package sqlite import ( "fmt" "go.uber.org/zap" "gorm.io/gorm" "gorm.io/gorm/clause" "hub.krkn.tech/KrakenTech/crowsnest/internal/files" ) type QueryOptions struct { gorm.Model MaxRecords int `json:"max_records"` MaxRequests int `json:"max_requests"` StartingPage int `json:"starting_page"` OutputFormat files.FileType `json:"output_format"` OutputFile string `json:"output_file"` RegexMatch bool `json:"regex_match"` WildcardMatch bool `json:"wildcard_match"` UsernameQuery string `json:"username_query"` EmailQuery string `json:"email_query"` IpQuery string `json:"ip_query"` PassQuery string `json:"pass_query"` HashQuery string `json:"hash_query"` NameQuery string `json:"name_query"` DomainQuery string `json:"domain_query"` VinQuery string `json:"vin_query"` LicensePlateQuery string `json:"license_plate_query"` AddressQuery string `json:"address_query"` PhoneQuery string `json:"phone_query"` SocialQuery string `json:"social_query"` CryptoAddressQuery string `json:"crypto_address_query"` PrintBalance bool `json:"print_balance"` CredsOnly bool `json:"creds_only"` Debug bool `json:"debug"` } func (QueryOptions) TableName() string { return "query_options" } func NewQueryOptions(maxRecords, maxRequests, startingPage int, outputFormat, outputFile, usernameQuery, emailQuery, ipQuery, passQuery, hashQuery, nameQuery, domainQuery, vinQuery, licensePlateQuery, addressQuery, phoneQuery, socialQuery, cryptoAddressQuery string, regexMatch, wildcardMatch, printBalance, credsOnly, debug bool) *QueryOptions { return &QueryOptions{ MaxRecords: maxRecords, MaxRequests: maxRequests, StartingPage: startingPage, OutputFormat: files.GetFileType(outputFormat), OutputFile: outputFile, PrintBalance: printBalance, CredsOnly: credsOnly, RegexMatch: regexMatch, WildcardMatch: wildcardMatch, UsernameQuery: usernameQuery, EmailQuery: emailQuery, IpQuery: ipQuery, PassQuery: passQuery, HashQuery: hashQuery, NameQuery: nameQuery, DomainQuery: domainQuery, VinQuery: vinQuery, LicensePlateQuery: licensePlateQuery, AddressQuery: addressQuery, PhoneQuery: phoneQuery, SocialQuery: socialQuery, CryptoAddressQuery: cryptoAddressQuery, Debug: debug, } } type DehashedSearchRequest struct { Page int `json:"page"` Query string `json:"query"` Size int `json:"size"` Wildcard bool `json:"wildcard"` Regex bool `json:"regex"` DeDupe bool `json:"de_dupe"` } type DehashedResponse struct { Balance int `json:"balance"` Entries []Result `json:"entries"` Success bool `json:"success"` Took string `json:"took"` TotalResults int `json:"total"` } type Result struct { gorm.Model DehashedId string `json:"id" xml:"id" yaml:"id" gorm:"uniqueIndex"` Email []string `json:"email,omitempty" xml:"email,omitempty" yaml:"email,omitempty" gorm:"serializer:json"` IpAddress []string `json:"ip_address,omitempty" xml:"ip_address,omitempty" yaml:"ip_address,omitempty" gorm:"serializer:json"` Username []string `json:"username,omitempty" xml:"username,omitempty" yaml:"username,omitempty" gorm:"serializer:json"` Password []string `json:"password,omitempty" xml:"password,omitempty" yaml:"password,omitempty" gorm:"serializer:json"` HashedPassword []string `json:"hashed_password,omitempty" xml:"hashed_password,omitempty" yaml:"hashed_password,omitempty" gorm:"serializer:json"` HashType string `json:"hash_type,omitempty" xml:"hash_type,omitempty" yaml:"hash_type,omitempty"` Name []string `json:"name,omitempty" xml:"name,omitempty" yaml:"name,omitempty" gorm:"serializer:json"` Vin []string `json:"vin,omitempty" xml:"vin,omitempty" yaml:"vin,omitempty" gorm:"serializer:json"` LicensePlate []string `json:"license_plate,omitempty" xml:"license_plate,omitempty" yaml:"license_plate,omitempty" gorm:"serializer:json"` Url []string `json:"url,omitempty" xml:"url,omitempty" yaml:"url,omitempty" gorm:"serializer:json"` Social []string `json:"social,omitempty" xml:"social,omitempty" yaml:"social,omitempty" gorm:"serializer:json"` CryptoCurrencyAddress []string `json:"cryptocurrency_address,omitempty" xml:"cryptocurrency_address,omitempty" yaml:"cryptocurrency_address,omitempty" gorm:"serializer:json"` Address []string `json:"address,omitempty" xml:"address,omitempty" yaml:"address,omitempty" gorm:"serializer:json"` Phone []string `json:"phone,omitempty" xml:"phone,omitempty" yaml:"phone,omitempty" gorm:"serializer:json"` Company []string `json:"company,omitempty" xml:"company,omitempty" yaml:"company,omitempty" gorm:"serializer:json"` DatabaseName string `json:"database_name,omitempty" xml:"database_name,omitempty" yaml:"database_name,omitempty"` } func (Result) TableName() string { return "dehashed" } type DehashedResults struct { Results []Result `json:"results"` } func (dr *DehashedResults) ExtractUsers() []User { var creds []User results := dr.Results for _, r := range results { if len(r.Password) > 0 { // Get first email if available email := "" if len(r.Email) > 0 { email = r.Email[0] } // Get first username if available username := "" if len(r.Username) > 0 { username = r.Username[0] } // Get first password password := r.Password[0] cred := User{Email: email, Password: password, Username: username} creds = append(creds, cred) } } go func() { err := StoreUsers(creds) if err != nil { zap.L().Error("store_creds", zap.String("message", "failed to store creds"), zap.Error(err), ) fmt.Printf("Error Storing Results: %v", err) } }() return creds } func (User) TableName() string { return "creds" } func (c User) ToString() string { return fmt.Sprintf("%s%s%s", c.Username, "%", c.Password) } func StoreDehashedResults(results DehashedResults) error { if len(results.Results) == 0 { return nil } zap.L().Info("Storing results", zap.Int("count", len(results.Results))) db := GetDB() // Use batch insert with conflict handling const batchSize = 100 var lastErr error // Extract the slice of results resultSlice := results.Results for i := 0; i < len(resultSlice); i += batchSize { end := i + batchSize if end > len(resultSlice) { end = len(resultSlice) } batch := resultSlice[i:end] // Use Clauses with OnConflict DoNothing to skip conflicts err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error if err != nil { zap.L().Warn("Error storing some results", zap.Error(err)) lastErr = err // Continue with next batch despite error } } return lastErr } func StoreDehashedQueryOptions(queryOptions *QueryOptions) error { db := GetDB() return db.Create(queryOptions).Error }