From 00d9a6b57e52dce31d970f233a703d0cd6ab11ed Mon Sep 17 00:00:00 2001 From: Evan Hosinski Date: Sat, 17 May 2025 10:25:16 -0400 Subject: [PATCH] fixed output of coffee and primary banner --- cmd/dehashed.go | 2 +- cmd/root.go | 12 +- cmd/whois.go | 4 +- dehasher.go => crowsnest.go | 0 internal/dehashed/dehashed.go | 6 +- internal/sqlite/db.go | 143 +++++++++++++++ internal/sqlite/dehashed.go | 235 +++++++++++++++++++++++++ internal/sqlite/gorm.go | 319 ---------------------------------- internal/sqlite/hunter.io.go | 67 ++++++- internal/sqlite/query.go | 20 --- internal/sqlite/result.go | 94 ---------- internal/sqlite/structs.go | 89 ---------- internal/sqlite/tables.go | 68 -------- internal/sqlite/whois.go | 116 +++++++++++++ internal/whois/whois.go | 6 +- 15 files changed, 574 insertions(+), 607 deletions(-) rename dehasher.go => crowsnest.go (100%) create mode 100644 internal/sqlite/db.go create mode 100644 internal/sqlite/dehashed.go delete mode 100644 internal/sqlite/gorm.go delete mode 100644 internal/sqlite/query.go delete mode 100644 internal/sqlite/result.go delete mode 100644 internal/sqlite/tables.go diff --git a/cmd/dehashed.go b/cmd/dehashed.go index 4d6894a..f79c9bc 100644 --- a/cmd/dehashed.go +++ b/cmd/dehashed.go @@ -118,7 +118,7 @@ var ( dehasher.Start() fmt.Println("\n[*] Completing Process") - err := sqlite.StoreQueryOptions(queryOptions) + err := sqlite.StoreDehashedQueryOptions(queryOptions) if err != nil { if debugGlobal { debug.PrintInfo("failed to store query options") diff --git a/cmd/root.go b/cmd/root.go index 5d61acd..9414493 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -21,12 +21,12 @@ var ( Long: fmt.Sprintf( "%s\n", ` - ╔═╗┬─┐┌─┐┬ ┬┌─┐╔╗╔┌─┐┌─┐┌┬┐ - ║ ├┬┘│ ││││└─┐║║║├┤ └─┐ │ - ╚═╝┴└─└─┘└┴┘└─┘╝╚╝└─┘└─┘ ┴ + ╔═╗┬─┐┌─┐┬ ┬┌─┐╔╗╔┌─┐┌─┐┌┬┐ + ║ ├┬┘│ ││││└─┐║║║├┤ └─┐ │ + ╚═╝┴└─└─┘└┴┘└─┘╝╚╝└─┘└─┘ ┴ - Crow’s Nest OSINT Recon Suite -⚓ A KrakenTech Intelligence Tool + Crow’s Nest OSINT Recon Suite + ⚓ A KrakenTech Intelligence Tool `, ), Version: "v1.2.1", @@ -124,7 +124,7 @@ var buyMeCoffeeCmd = &cobra.Command{ Use: "coffee", Short: "Support the project by buying a coffee", Run: func(cmd *cobra.Command, args []string) { - fmt.Println(color.HiRedString(" ;)( ;")) + fmt.Println(color.HiRedString(" ;)(; ")) fmt.Println(color.HiCyanString(" We Hope You Enjoy Our Product :----:")) fmt.Println(color.HiCyanString(" C|====|")) fmt.Println(color.HiCyanString(" | |")) diff --git a/cmd/whois.go b/cmd/whois.go index 6556906..cddf5a2 100644 --- a/cmd/whois.go +++ b/cmd/whois.go @@ -208,7 +208,7 @@ var ( fmt.Printf("[!] Error writing WHOIS history to file: %v\n", writeErr) } - err = sqlite.StoreHistoryRecord(historyRecords) + err = sqlite.StoreWhoisHistoryRecords(historyRecords) if err != nil { if debugGlobal { debug.PrintInfo("failed to store history record") @@ -254,7 +254,7 @@ var ( fmt.Printf("Error performing subdomain scan: %v\n", err) } else { fmt.Println("Subdomain Scan:") - err = sqlite.StoreSubdomainRecords(subdomains) + err = sqlite.StoreWhoisSubdomainRecords(subdomains) if err != nil { if debugGlobal { debug.PrintInfo("failed to store subdomain record") diff --git a/dehasher.go b/crowsnest.go similarity index 100% rename from dehasher.go rename to crowsnest.go diff --git a/internal/dehashed/dehashed.go b/internal/dehashed/dehashed.go index d3f2f49..6c8a7cf 100644 --- a/internal/dehashed/dehashed.go +++ b/internal/dehashed/dehashed.go @@ -130,7 +130,7 @@ func (dh *Dehasher) Start() { if len(dh.client.results) > 0 { fmt.Printf(" [!] Partial results retrieved. Storing Results...\n") - err := sqlite.StoreResults(dh.client.GetResults()) + err := sqlite.StoreDehashedResults(dh.client.GetResults()) if err != nil { zap.L().Error("store_results", zap.String("message", "failed to store results"), @@ -214,7 +214,7 @@ func (dh *Dehasher) parseResults() { results := dh.client.GetResults() creds := results.ExtractCredentials() fmt.Printf("\n\t[+] Discovered %d Credentials", len(creds)) - err := sqlite.StoreCreds(creds) + err := sqlite.StoreDehashedCreds(creds) if err != nil { zap.L().Error("store_creds", zap.String("message", "failed to store creds"), @@ -224,7 +224,7 @@ func (dh *Dehasher) parseResults() { zap.L().Info("creds_stored", zap.Int("count", len(creds))) zap.L().Info("storing_results") - err = sqlite.StoreResults(results) + err = sqlite.StoreDehashedResults(results) if err != nil { zap.L().Error("store_results", zap.String("message", "failed to store results"), diff --git a/internal/sqlite/db.go b/internal/sqlite/db.go new file mode 100644 index 0000000..1024a45 --- /dev/null +++ b/internal/sqlite/db.go @@ -0,0 +1,143 @@ +package sqlite + +import ( + "fmt" + "go.uber.org/zap" + "os" + "path/filepath" + "strings" + + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +var DB *gorm.DB + +// InitDB initializes the database connection +func InitDB(dbPath string) (*gorm.DB, error) { + zap.L().Info("Initializing database", zap.String("path", dbPath)) + + // Check if the path is a file or directory + fileInfo, err := os.Stat(dbPath) + var finalDbPath string + + // If path doesn't exist or is a directory + if os.IsNotExist(err) || (err == nil && fileInfo.IsDir()) { + // Treat as directory path + if err := os.MkdirAll(dbPath, 0755); err != nil { + zap.L().Error("Failed to create database directory", zap.Error(err)) + return nil, fmt.Errorf("failed to create database directory: %w", err) + } + finalDbPath = filepath.Join(dbPath, "dehashed.sqlite") + } else { + // Treat as file path + // Ensure the directory exists + dir := filepath.Dir(dbPath) + if err := os.MkdirAll(dir, 0755); err != nil { + zap.L().Error("Failed to create parent directory for database", zap.Error(err)) + return nil, fmt.Errorf("failed to create parent directory for database: %w", err) + } + finalDbPath = dbPath + } + + zap.L().Info("Opening database", zap.String("finalPath", finalDbPath)) + db, err := gorm.Open(sqlite.Open(finalDbPath), &gorm.Config{ + Logger: logger.Default.LogMode(logger.Silent), + }) + if err != nil { + zap.L().Error("Failed to connect to database", zap.Error(err)) + return nil, fmt.Errorf("failed to connect to database: %w", err) + } + + // Auto migrate your models + err = db.AutoMigrate(&Result{}, &Creds{}, &QueryOptions{}, &Creds{}, &WhoisRecord{}, &SubdomainRecord{}, + &HistoryRecord{}, &LookupResult{}, &HunterDomainData{}, &HunterEmail{}, &PersonData{}) + if err != nil { + zap.L().Error("Failed to migrate database", zap.Error(err)) + return nil, fmt.Errorf("failed to migrate database: %w", err) + } + + DB = db + return db, nil +} + +// GetDB returns the database connection +func GetDB() *gorm.DB { + if DB == nil { + zap.L().Error("database not initialized") + fmt.Println("sqlite database not initialized") + os.Exit(1) + } + return DB +} + +type Table int64 + +const ( + ResultsTable Table = iota + RunsTable + CredsTable + WhoIsTable + SubdomainsTable + HistoryTable + LookupTable + HunterDomainTable + HunterEmailTable + PersonTable + UnknownTable +) + +func GetTable(userInput string) Table { + switch strings.ToLower(userInput) { + case "results": + return ResultsTable + case "runs": + return RunsTable + case "creds": + return CredsTable + case "whois": + return WhoIsTable + case "subdomains": + return SubdomainsTable + case "history": + return HistoryTable + case "lookup": + return LookupTable + case "hunter_domain": + return HunterDomainTable + case "hunter_email": + return HunterEmailTable + case "person": + return PersonTable + default: + return UnknownTable + } +} + +func (t Table) Object() interface{} { + switch t { + case ResultsTable: + return Result{} + case RunsTable: + return QueryOptions{} + case CredsTable: + return Creds{} + case WhoIsTable: + return WhoisRecord{} + case SubdomainsTable: + return SubdomainRecord{} + case HistoryTable: + return HistoryRecord{} + case LookupTable: + return LookupResult{} + case HunterDomainTable: + return HunterDomainData{} + case HunterEmailTable: + return HunterEmail{} + case PersonTable: + return PersonData{} + default: + return nil + } +} diff --git a/internal/sqlite/dehashed.go b/internal/sqlite/dehashed.go new file mode 100644 index 0000000..ba78577 --- /dev/null +++ b/internal/sqlite/dehashed.go @@ -0,0 +1,235 @@ +package sqlite + +import ( + "crowsnest/internal/files" + "fmt" + "go.uber.org/zap" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +type QueryOptions struct { + gorm.Model + MaxRecords int `json:"max_records"` + MaxRequests int `json:"max_requests"` + StartingPage int `json:"starting_page"` + OutputFormat files.FileType `json:"output_format"` + OutputFile string `json:"output_file"` + RegexMatch bool `json:"regex_match"` + WildcardMatch bool `json:"wildcard_match"` + UsernameQuery string `json:"username_query"` + EmailQuery string `json:"email_query"` + IpQuery string `json:"ip_query"` + PassQuery string `json:"pass_query"` + HashQuery string `json:"hash_query"` + NameQuery string `json:"name_query"` + DomainQuery string `json:"domain_query"` + VinQuery string `json:"vin_query"` + LicensePlateQuery string `json:"license_plate_query"` + AddressQuery string `json:"address_query"` + PhoneQuery string `json:"phone_query"` + SocialQuery string `json:"social_query"` + CryptoAddressQuery string `json:"crypto_address_query"` + PrintBalance bool `json:"print_balance"` + CredsOnly bool `json:"creds_only"` + Debug bool `json:"debug"` +} + +func (QueryOptions) TableName() string { + return "query_options" +} + +func NewQueryOptions(maxRecords, maxRequests, startingPage int, outputFormat, outputFile, usernameQuery, emailQuery, ipQuery, passQuery, hashQuery, nameQuery, domainQuery, vinQuery, licensePlateQuery, addressQuery, phoneQuery, socialQuery, cryptoAddressQuery string, regexMatch, wildcardMatch, printBalance, credsOnly, debug bool) *QueryOptions { + return &QueryOptions{ + MaxRecords: maxRecords, + MaxRequests: maxRequests, + StartingPage: startingPage, + OutputFormat: files.GetFileType(outputFormat), + OutputFile: outputFile, + PrintBalance: printBalance, + CredsOnly: credsOnly, + RegexMatch: regexMatch, + WildcardMatch: wildcardMatch, + UsernameQuery: usernameQuery, + EmailQuery: emailQuery, + IpQuery: ipQuery, + PassQuery: passQuery, + HashQuery: hashQuery, + NameQuery: nameQuery, + DomainQuery: domainQuery, + VinQuery: vinQuery, + LicensePlateQuery: licensePlateQuery, + AddressQuery: addressQuery, + PhoneQuery: phoneQuery, + SocialQuery: socialQuery, + CryptoAddressQuery: cryptoAddressQuery, + Debug: debug, + } +} + +type DehashedSearchRequest struct { + Page int `json:"page"` + Query string `json:"query"` + Size int `json:"size"` + Wildcard bool `json:"wildcard"` + Regex bool `json:"regex"` + DeDupe bool `json:"de_dupe"` +} + +type DehashedResponse struct { + Balance int `json:"balance"` + Entries []Result `json:"entries"` + Success bool `json:"success"` + Took string `json:"took"` + TotalResults int `json:"total"` +} + +type Result struct { + gorm.Model + DehashedId string `json:"id" xml:"id" yaml:"id" gorm:"uniqueIndex"` + Email []string `json:"email,omitempty" xml:"email,omitempty" yaml:"email,omitempty" gorm:"serializer:json"` + IpAddress []string `json:"ip_address,omitempty" xml:"ip_address,omitempty" yaml:"ip_address,omitempty" gorm:"serializer:json"` + Username []string `json:"username,omitempty" xml:"username,omitempty" yaml:"username,omitempty" gorm:"serializer:json"` + Password []string `json:"password,omitempty" xml:"password,omitempty" yaml:"password,omitempty" gorm:"serializer:json"` + HashedPassword []string `json:"hashed_password,omitempty" xml:"hashed_password,omitempty" yaml:"hashed_password,omitempty" gorm:"serializer:json"` + HashType string `json:"hash_type,omitempty" xml:"hash_type,omitempty" yaml:"hash_type,omitempty"` + Name []string `json:"name,omitempty" xml:"name,omitempty" yaml:"name,omitempty" gorm:"serializer:json"` + Vin []string `json:"vin,omitempty" xml:"vin,omitempty" yaml:"vin,omitempty" gorm:"serializer:json"` + LicensePlate []string `json:"license_plate,omitempty" xml:"license_plate,omitempty" yaml:"license_plate,omitempty" gorm:"serializer:json"` + Url []string `json:"url,omitempty" xml:"url,omitempty" yaml:"url,omitempty" gorm:"serializer:json"` + Social []string `json:"social,omitempty" xml:"social,omitempty" yaml:"social,omitempty" gorm:"serializer:json"` + CryptoCurrencyAddress []string `json:"cryptocurrency_address,omitempty" xml:"cryptocurrency_address,omitempty" yaml:"cryptocurrency_address,omitempty" gorm:"serializer:json"` + Address []string `json:"address,omitempty" xml:"address,omitempty" yaml:"address,omitempty" gorm:"serializer:json"` + Phone []string `json:"phone,omitempty" xml:"phone,omitempty" yaml:"phone,omitempty" gorm:"serializer:json"` + Company []string `json:"company,omitempty" xml:"company,omitempty" yaml:"company,omitempty" gorm:"serializer:json"` + DatabaseName string `json:"database_name,omitempty" xml:"database_name,omitempty" yaml:"database_name,omitempty"` +} + +func (Result) TableName() string { + return "results" +} + +type DehashedResults struct { + Results []Result `json:"results"` +} + +func (dr *DehashedResults) ExtractCredentials() []Creds { + var creds []Creds + + results := dr.Results + + for _, r := range results { + if len(r.Password) > 0 { + // Get first email if available + email := "" + if len(r.Email) > 0 { + email = r.Email[0] + } + + // Get first password + password := r.Password[0] + + cred := Creds{Email: email, Password: password} + creds = append(creds, cred) + } + } + + go func() { + err := StoreDehashedCreds(creds) + if err != nil { + zap.L().Error("store_creds", + zap.String("message", "failed to store creds"), + zap.Error(err), + ) + fmt.Printf("Error Storing Results: %v", err) + } + }() + + return creds +} + +type Creds struct { + gorm.Model + Email string `json:"email" yaml:"email" xml:"email" gorm:"uniqueIndex:idx_email_username_password"` + Username string `json:"username" yaml:"username" xml:"username" gorm:"uniqueIndex:idx_email_username_password"` + Password string `json:"password" yaml:"password" xml:"password" gorm:"uniqueIndex:idx_email_username_password"` +} + +func (Creds) TableName() string { + return "creds" +} + +func (c Creds) ToString() string { + return fmt.Sprintf("%s%s%s", c.Username, "%", c.Password) +} + +func StoreDehashedResults(results DehashedResults) error { + if len(results.Results) == 0 { + return nil + } + + zap.L().Info("Storing results", zap.Int("count", len(results.Results))) + db := GetDB() + + // Use batch insert with conflict handling + const batchSize = 100 + var lastErr error + + // Extract the slice of results + resultSlice := results.Results + + for i := 0; i < len(resultSlice); i += batchSize { + end := i + batchSize + if end > len(resultSlice) { + end = len(resultSlice) + } + + batch := resultSlice[i:end] + // Use Clauses with OnConflict DoNothing to skip conflicts + err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error + if err != nil { + zap.L().Warn("Error storing some results", zap.Error(err)) + lastErr = err + // Continue with next batch despite error + } + } + + return lastErr +} + +func StoreDehashedCreds(creds []Creds) error { + if len(creds) == 0 { + return nil + } + + zap.L().Info("Storing credentials", zap.Int("count", len(creds))) + db := GetDB() + + // Use batch insert with conflict handling + // This will insert records in batches and continue even if some fail + const batchSize = 100 + var lastErr error + + for i := 0; i < len(creds); i += batchSize { + end := i + batchSize + if end > len(creds) { + end = len(creds) + } + + batch := creds[i:end] + // Use Clauses with OnConflict DoNothing to skip conflicts + err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error + if err != nil { + zap.L().Warn("Error storing some credentials", zap.Error(err)) + lastErr = err + // Continue with next batch despite error + } + } + + return lastErr +} + +func StoreDehashedQueryOptions(queryOptions *QueryOptions) error { + db := GetDB() + return db.Create(queryOptions).Error +} diff --git a/internal/sqlite/gorm.go b/internal/sqlite/gorm.go deleted file mode 100644 index bdc152d..0000000 --- a/internal/sqlite/gorm.go +++ /dev/null @@ -1,319 +0,0 @@ -package sqlite - -import ( - "fmt" - "go.uber.org/zap" - "gorm.io/gorm/clause" - "os" - "path/filepath" - - "gorm.io/driver/sqlite" - "gorm.io/gorm" - "gorm.io/gorm/logger" -) - -var DB *gorm.DB - -// InitDB initializes the database connection -func InitDB(dbPath string) (*gorm.DB, error) { - zap.L().Info("Initializing database", zap.String("path", dbPath)) - - // Check if the path is a file or directory - fileInfo, err := os.Stat(dbPath) - var finalDbPath string - - // If path doesn't exist or is a directory - if os.IsNotExist(err) || (err == nil && fileInfo.IsDir()) { - // Treat as directory path - if err := os.MkdirAll(dbPath, 0755); err != nil { - zap.L().Error("Failed to create database directory", zap.Error(err)) - return nil, fmt.Errorf("failed to create database directory: %w", err) - } - finalDbPath = filepath.Join(dbPath, "dehashed.sqlite") - } else { - // Treat as file path - // Ensure the directory exists - dir := filepath.Dir(dbPath) - if err := os.MkdirAll(dir, 0755); err != nil { - zap.L().Error("Failed to create parent directory for database", zap.Error(err)) - return nil, fmt.Errorf("failed to create parent directory for database: %w", err) - } - finalDbPath = dbPath - } - - zap.L().Info("Opening database", zap.String("finalPath", finalDbPath)) - db, err := gorm.Open(sqlite.Open(finalDbPath), &gorm.Config{ - Logger: logger.Default.LogMode(logger.Silent), - }) - if err != nil { - zap.L().Error("Failed to connect to database", zap.Error(err)) - return nil, fmt.Errorf("failed to connect to database: %w", err) - } - - // Auto migrate your models - err = db.AutoMigrate(&Result{}, &Creds{}, &QueryOptions{}, &Creds{}, &WhoisRecord{}, &SubdomainRecord{}, - &HistoryRecord{}, &LookupResult{}, &HunterDomainData{}, &HunterEmail{}, &PersonData{}) - if err != nil { - zap.L().Error("Failed to migrate database", zap.Error(err)) - return nil, fmt.Errorf("failed to migrate database: %w", err) - } - - DB = db - return db, nil -} - -// GetDB returns the database connection -func GetDB() *gorm.DB { - if DB == nil { - zap.L().Error("database not initialized") - fmt.Println("sqlite database not initialized") - os.Exit(1) - } - return DB -} - -func StoreResults(results DehashedResults) error { - if len(results.Results) == 0 { - return nil - } - - zap.L().Info("Storing results", zap.Int("count", len(results.Results))) - db := GetDB() - - // Use batch insert with conflict handling - const batchSize = 100 - var lastErr error - - // Extract the slice of results - resultSlice := results.Results - - for i := 0; i < len(resultSlice); i += batchSize { - end := i + batchSize - if end > len(resultSlice) { - end = len(resultSlice) - } - - batch := resultSlice[i:end] - // Use Clauses with OnConflict DoNothing to skip conflicts - err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error - if err != nil { - zap.L().Warn("Error storing some results", zap.Error(err)) - lastErr = err - // Continue with next batch despite error - } - } - - return lastErr -} - -func StoreCreds(creds []Creds) error { - if len(creds) == 0 { - return nil - } - - zap.L().Info("Storing credentials", zap.Int("count", len(creds))) - db := GetDB() - - // Use batch insert with conflict handling - // This will insert records in batches and continue even if some fail - const batchSize = 100 - var lastErr error - - for i := 0; i < len(creds); i += batchSize { - end := i + batchSize - if end > len(creds) { - end = len(creds) - } - - batch := creds[i:end] - // Use Clauses with OnConflict DoNothing to skip conflicts - err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error - if err != nil { - zap.L().Warn("Error storing some credentials", zap.Error(err)) - lastErr = err - // Continue with next batch despite error - } - } - - return lastErr -} - -func StoreQueryOptions(queryOptions *QueryOptions) error { - db := GetDB() - return db.Create(queryOptions).Error -} - -func StoreWhoisRecord(whoisRecord WhoisRecord) error { - // Create a pointer to the record to make it addressable - recordPtr := &whoisRecord - - zap.L().Info("Storing WHOIS record", - zap.String("domain", whoisRecord.DomainName)) - - db := GetDB() - - // Use OnConflict clause to handle duplicates - err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(recordPtr).Error - if err != nil { - zap.L().Error("store_whois_record", - zap.String("message", "failed to store whois record"), - zap.Error(err)) - return err - } - - return nil -} - -func StoreSubdomainRecords(subdomainRecords []SubdomainRecord) error { - if len(subdomainRecords) == 0 { - return nil - } - - zap.L().Info("Storing subdomain records", zap.Int("count", len(subdomainRecords))) - db := GetDB() - - // Use batch insert with conflict handling - const batchSize = 100 - var lastErr error - - for i := 0; i < len(subdomainRecords); i += batchSize { - end := i + batchSize - if end > len(subdomainRecords) { - end = len(subdomainRecords) - } - - batch := subdomainRecords[i:end] - // Use Clauses with OnConflict DoNothing to skip conflicts - err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error - if err != nil { - zap.L().Warn("Error storing some subdomain records", zap.Error(err)) - lastErr = err - // Continue with next batch despite error - } - } - - return lastErr -} - -func StoreHistoryRecord(historyRecords []HistoryRecord) error { - if len(historyRecords) == 0 { - return nil - } - - zap.L().Info("Storing history records", zap.Int("count", len(historyRecords))) - db := GetDB() - - // Use batch insert with conflict handling - const batchSize = 100 - var lastErr error - - for i := 0; i < len(historyRecords); i += batchSize { - end := i + batchSize - if end > len(historyRecords) { - end = len(historyRecords) - } - - batch := historyRecords[i:end] - // Use Clauses with OnConflict DoNothing to skip conflicts - err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error - if err != nil { - zap.L().Warn("Error storing some history records", zap.Error(err)) - lastErr = err - // Continue with next batch despite error - } - } - - return lastErr -} - -func StoreIPLookup(ipLookup []LookupResult) error { - if len(ipLookup) == 0 { - return nil - } - - zap.L().Info("Storing IP lookup records", zap.Int("count", len(ipLookup))) - db := GetDB() - - // Use batch insert with conflict handling - const batchSize = 100 - var lastErr error - - for i := 0; i < len(ipLookup); i += batchSize { - end := i + batchSize - if end > len(ipLookup) { - end = len(ipLookup) - } - - batch := ipLookup[i:end] - // Use Clauses with OnConflict DoNothing to skip conflicts - err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error - if err != nil { - zap.L().Warn("Error storing some IP lookup records", zap.Error(err)) - lastErr = err - // Continue with next batch despite error - } - } - - return lastErr -} - -func StoreHunterDomain(hunterDomain HunterDomainData) error { - db := GetDB() - - // Use OnConflict clause to handle duplicates - err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(&hunterDomain).Error - if err != nil { - zap.L().Error("store_hunter_domain", - zap.String("message", "failed to store hunter domain"), - zap.Error(err)) - return err - } - - return nil -} - -func StoreHunterEmails(hunterEmails []HunterEmail) error { - if len(hunterEmails) == 0 { - return nil - } - - zap.L().Info("Storing hunter emails", zap.Int("count", len(hunterEmails))) - db := GetDB() - - // Use batch insert with conflict handling - const batchSize = 100 - var lastErr error - - for i := 0; i < len(hunterEmails); i += batchSize { - end := i + batchSize - if end > len(hunterEmails) { - end = len(hunterEmails) - } - - batch := hunterEmails[i:end] - // Use Clauses with OnConflict DoNothing to skip conflicts - err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error - if err != nil { - zap.L().Warn("Error storing some hunter emails", zap.Error(err)) - lastErr = err - // Continue with next batch despite error - } - } - - return lastErr -} - -func StorePersonData(personData PersonData) error { - db := GetDB() - - // Use OnConflict clause to handle duplicates - err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(&personData).Error - if err != nil { - zap.L().Error("store_person_data", - zap.String("message", "failed to store person data"), - zap.Error(err)) - return err - } - - return nil -} diff --git a/internal/sqlite/hunter.io.go b/internal/sqlite/hunter.io.go index 9da19f1..5cb5a63 100644 --- a/internal/sqlite/hunter.io.go +++ b/internal/sqlite/hunter.io.go @@ -3,7 +3,9 @@ package sqlite import ( "fmt" "github.com/charmbracelet/lipgloss/tree" + "go.uber.org/zap" "gorm.io/gorm" + "gorm.io/gorm/clause" ) // HunterDomainSearchResult represents the response from Hunter.io domain search API @@ -14,8 +16,8 @@ type HunterDomainSearchResult struct { // HunterDomainData contains the main domain information type HunterDomainData struct { - IString gorm.Model + IString `gorm:"-"` Domain string `json:"domain" gorm:"unique"` Disposable bool `json:"disposable"` Webmail bool `json:"webmail"` @@ -512,8 +514,8 @@ type HunterPersonEnrichmentResponse struct { // PersonData contains the detailed person information type PersonData struct { - IString gorm.Model + IString `gorm:"-"` ID string `json:"id"` Name PersonName `json:"name" gorm:"embedded;embeddedPrefix:name_"` Email string `json:"email" gorm:"unique"` @@ -755,3 +757,64 @@ func (c *HunterCombinedEnrichmentResponse) String() string { c.Data.Person.String(), c.Data.Company.String()) } + +func StoreHunterDomain(hunterDomain HunterDomainData) error { + db := GetDB() + + // Use OnConflict clause to handle duplicates + err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(&hunterDomain).Error + if err != nil { + zap.L().Error("store_hunter_domain", + zap.String("message", "failed to store hunter domain"), + zap.Error(err)) + return err + } + + return nil +} + +func StoreHunterEmails(hunterEmails []HunterEmail) error { + if len(hunterEmails) == 0 { + return nil + } + + zap.L().Info("Storing hunter emails", zap.Int("count", len(hunterEmails))) + db := GetDB() + + // Use batch insert with conflict handling + const batchSize = 100 + var lastErr error + + for i := 0; i < len(hunterEmails); i += batchSize { + end := i + batchSize + if end > len(hunterEmails) { + end = len(hunterEmails) + } + + batch := hunterEmails[i:end] + // Use Clauses with OnConflict DoNothing to skip conflicts + err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error + if err != nil { + zap.L().Warn("Error storing some hunter emails", zap.Error(err)) + lastErr = err + // Continue with next batch despite error + } + } + + return lastErr +} + +func StoreHunterPersonData(personData PersonData) error { + db := GetDB() + + // Use OnConflict clause to handle duplicates + err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(&personData).Error + if err != nil { + zap.L().Error("store_person_data", + zap.String("message", "failed to store person data"), + zap.Error(err)) + return err + } + + return nil +} diff --git a/internal/sqlite/query.go b/internal/sqlite/query.go deleted file mode 100644 index 1b6382c..0000000 --- a/internal/sqlite/query.go +++ /dev/null @@ -1,20 +0,0 @@ -package sqlite - -type DehashedSearchRequest struct { - Page int `json:"page"` - Query string `json:"query"` - Size int `json:"size"` - Wildcard bool `json:"wildcard"` - Regex bool `json:"regex"` - DeDupe bool `json:"de_dupe"` -} - -func NewDehashedSearchRequest(size int, wildcard, regex bool) *DehashedSearchRequest { - return &DehashedSearchRequest{ - Page: 0, - Size: size, - Wildcard: false, - Regex: false, - DeDupe: true, - } -} diff --git a/internal/sqlite/result.go b/internal/sqlite/result.go deleted file mode 100644 index 1802c9c..0000000 --- a/internal/sqlite/result.go +++ /dev/null @@ -1,94 +0,0 @@ -package sqlite - -import ( - "encoding/json" - "fmt" - "go.uber.org/zap" - "gorm.io/gorm" - "io" - "os" -) - -type DehashedResponse struct { - Balance int `json:"balance"` - Entries []Result `json:"entries"` - Success bool `json:"success"` - Took string `json:"took"` - TotalResults int `json:"total"` -} - -type Result struct { - gorm.Model - DehashedId string `json:"id" xml:"id" yaml:"id" gorm:"uniqueIndex"` - Email []string `json:"email,omitempty" xml:"email,omitempty" yaml:"email,omitempty" gorm:"serializer:json"` - IpAddress []string `json:"ip_address,omitempty" xml:"ip_address,omitempty" yaml:"ip_address,omitempty" gorm:"serializer:json"` - Username []string `json:"username,omitempty" xml:"username,omitempty" yaml:"username,omitempty" gorm:"serializer:json"` - Password []string `json:"password,omitempty" xml:"password,omitempty" yaml:"password,omitempty" gorm:"serializer:json"` - HashedPassword []string `json:"hashed_password,omitempty" xml:"hashed_password,omitempty" yaml:"hashed_password,omitempty" gorm:"serializer:json"` - HashType string `json:"hash_type,omitempty" xml:"hash_type,omitempty" yaml:"hash_type,omitempty"` - Name []string `json:"name,omitempty" xml:"name,omitempty" yaml:"name,omitempty" gorm:"serializer:json"` - Vin []string `json:"vin,omitempty" xml:"vin,omitempty" yaml:"vin,omitempty" gorm:"serializer:json"` - LicensePlate []string `json:"license_plate,omitempty" xml:"license_plate,omitempty" yaml:"license_plate,omitempty" gorm:"serializer:json"` - Url []string `json:"url,omitempty" xml:"url,omitempty" yaml:"url,omitempty" gorm:"serializer:json"` - Social []string `json:"social,omitempty" xml:"social,omitempty" yaml:"social,omitempty" gorm:"serializer:json"` - CryptoCurrencyAddress []string `json:"cryptocurrency_address,omitempty" xml:"cryptocurrency_address,omitempty" yaml:"cryptocurrency_address,omitempty" gorm:"serializer:json"` - Address []string `json:"address,omitempty" xml:"address,omitempty" yaml:"address,omitempty" gorm:"serializer:json"` - Phone []string `json:"phone,omitempty" xml:"phone,omitempty" yaml:"phone,omitempty" gorm:"serializer:json"` - Company []string `json:"company,omitempty" xml:"company,omitempty" yaml:"company,omitempty" gorm:"serializer:json"` - DatabaseName string `json:"database_name,omitempty" xml:"database_name,omitempty" yaml:"database_name,omitempty"` -} - -func (Result) TableName() string { - return "results" -} - -type DehashedResults struct { - Results []Result `json:"results"` -} - -func (dr *DehashedResults) ExtractCredentials() []Creds { - var creds []Creds - - results := dr.Results - - for _, r := range results { - if len(r.Password) > 0 { - // Get first email if available - email := "" - if len(r.Email) > 0 { - email = r.Email[0] - } - - // Get first password - password := r.Password[0] - - cred := Creds{Email: email, Password: password} - creds = append(creds, cred) - } - } - - go func() { - err := StoreCreds(creds) - if err != nil { - zap.L().Error("store_creds", - zap.String("message", "failed to store creds"), - zap.Error(err), - ) - fmt.Printf("Error Storing Results: %v", err) - } - }() - - return creds -} - -func NewDehashedResults(body io.Reader) ([]Result, int, int) { - var response DehashedResponse - - err := json.NewDecoder(body).Decode(&response) - if err != nil { - fmt.Printf("Error Parsing Response Body: %v", err) - os.Exit(-1) - } - - return response.Entries, response.Balance, response.TotalResults -} diff --git a/internal/sqlite/structs.go b/internal/sqlite/structs.go index 078ec45..1adb693 100644 --- a/internal/sqlite/structs.go +++ b/internal/sqlite/structs.go @@ -1,11 +1,5 @@ package sqlite -import ( - "crowsnest/internal/files" - "fmt" - "gorm.io/gorm" -) - type IString interface { String() string } @@ -30,15 +24,6 @@ type DBOptions struct { DisplayFields []string // Fields to display in output } -func NewDBOptions() *DBOptions { - return &DBOptions{ - Limit: 100, // Default limit - ExactMatch: false, - NonEmptyFields: []string{}, - DisplayFields: []string{}, - } -} - func (o *DBOptions) Empty() bool { return o.Username == "" && o.Email == "" && o.IPAddress == "" && o.Password == "" && o.HashedPassword == "" && o.Name == "" && @@ -46,77 +31,3 @@ func (o *DBOptions) Empty() bool { o.Phone == "" && o.Social == "" && o.CryptoCurrencyAddress == "" && o.Domain == "" && len(o.NonEmptyFields) == 0 } - -type QueryOptions struct { - gorm.Model - MaxRecords int `json:"max_records"` - MaxRequests int `json:"max_requests"` - StartingPage int `json:"starting_page"` - OutputFormat files.FileType `json:"output_format"` - OutputFile string `json:"output_file"` - RegexMatch bool `json:"regex_match"` - WildcardMatch bool `json:"wildcard_match"` - UsernameQuery string `json:"username_query"` - EmailQuery string `json:"email_query"` - IpQuery string `json:"ip_query"` - PassQuery string `json:"pass_query"` - HashQuery string `json:"hash_query"` - NameQuery string `json:"name_query"` - DomainQuery string `json:"domain_query"` - VinQuery string `json:"vin_query"` - LicensePlateQuery string `json:"license_plate_query"` - AddressQuery string `json:"address_query"` - PhoneQuery string `json:"phone_query"` - SocialQuery string `json:"social_query"` - CryptoAddressQuery string `json:"crypto_address_query"` - PrintBalance bool `json:"print_balance"` - CredsOnly bool `json:"creds_only"` - Debug bool `json:"debug"` -} - -func (QueryOptions) TableName() string { - return "query_options" -} - -func NewQueryOptions(maxRecords, maxRequests, startingPage int, outputFormat, outputFile, usernameQuery, emailQuery, ipQuery, passQuery, hashQuery, nameQuery, domainQuery, vinQuery, licensePlateQuery, addressQuery, phoneQuery, socialQuery, cryptoAddressQuery string, regexMatch, wildcardMatch, printBalance, credsOnly, debug bool) *QueryOptions { - return &QueryOptions{ - MaxRecords: maxRecords, - MaxRequests: maxRequests, - StartingPage: startingPage, - OutputFormat: files.GetFileType(outputFormat), - OutputFile: outputFile, - PrintBalance: printBalance, - CredsOnly: credsOnly, - RegexMatch: regexMatch, - WildcardMatch: wildcardMatch, - UsernameQuery: usernameQuery, - EmailQuery: emailQuery, - IpQuery: ipQuery, - PassQuery: passQuery, - HashQuery: hashQuery, - NameQuery: nameQuery, - DomainQuery: domainQuery, - VinQuery: vinQuery, - LicensePlateQuery: licensePlateQuery, - AddressQuery: addressQuery, - PhoneQuery: phoneQuery, - SocialQuery: socialQuery, - CryptoAddressQuery: cryptoAddressQuery, - Debug: debug, - } -} - -type Creds struct { - gorm.Model - Email string `json:"email" yaml:"email" xml:"email" gorm:"uniqueIndex:idx_email_username_password"` - Username string `json:"username" yaml:"username" xml:"username" gorm:"uniqueIndex:idx_email_username_password"` - Password string `json:"password" yaml:"password" xml:"password" gorm:"uniqueIndex:idx_email_username_password"` -} - -func (Creds) TableName() string { - return "creds" -} - -func (c Creds) ToString() string { - return fmt.Sprintf("%s%s%s", c.Username, "%", c.Password) -} diff --git a/internal/sqlite/tables.go b/internal/sqlite/tables.go deleted file mode 100644 index 0e4493b..0000000 --- a/internal/sqlite/tables.go +++ /dev/null @@ -1,68 +0,0 @@ -package sqlite - -import "strings" - -type Table int64 - -const ( - ResultsTable Table = iota - RunsTable - CredsTable - WhoIsTable - SubdomainsTable - HistoryTable - LookupTable - HunterDomainTable - HunterEmailTable - UnknownTable -) - -func GetTable(userInput string) Table { - switch strings.ToLower(userInput) { - case "results": - return ResultsTable - case "runs": - return RunsTable - case "creds": - return CredsTable - case "whois": - return WhoIsTable - case "subdomains": - return SubdomainsTable - case "history": - return HistoryTable - case "lookup": - return LookupTable - case "hunter_domain": - return HunterDomainTable - case "hunter_email": - return HunterEmailTable - default: - return UnknownTable - } -} - -func (t Table) Object() interface{} { - switch t { - case ResultsTable: - return Result{} - case RunsTable: - return QueryOptions{} - case CredsTable: - return Creds{} - case WhoIsTable: - return WhoisRecord{} - case SubdomainsTable: - return SubdomainRecord{} - case HistoryTable: - return HistoryRecord{} - case LookupTable: - return LookupResult{} - case HunterDomainTable: - return HunterDomainData{} - case HunterEmailTable: - return HunterEmail{} - default: - return nil - } -} diff --git a/internal/sqlite/whois.go b/internal/sqlite/whois.go index d5b0f3f..201791f 100644 --- a/internal/sqlite/whois.go +++ b/internal/sqlite/whois.go @@ -2,7 +2,9 @@ package sqlite import ( "fmt" + "go.uber.org/zap" "gorm.io/gorm" + "gorm.io/gorm/clause" "strings" ) @@ -512,3 +514,117 @@ func formatWhoisContact(sb *strings.Builder, contact Contact, indent string) { sb.WriteString(indent + "Raw Text: " + rawTextPreview + "\n") } } + +func StoreWhoisRecord(whoisRecord WhoisRecord) error { + // Create a pointer to the record to make it addressable + recordPtr := &whoisRecord + + zap.L().Info("Storing WHOIS record", + zap.String("domain", whoisRecord.DomainName)) + + db := GetDB() + + // Use OnConflict clause to handle duplicates + err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(recordPtr).Error + if err != nil { + zap.L().Error("store_whois_record", + zap.String("message", "failed to store whois record"), + zap.Error(err)) + return err + } + + return nil +} + +func StoreWhoisSubdomainRecords(subdomainRecords []SubdomainRecord) error { + if len(subdomainRecords) == 0 { + return nil + } + + zap.L().Info("Storing subdomain records", zap.Int("count", len(subdomainRecords))) + db := GetDB() + + // Use batch insert with conflict handling + const batchSize = 100 + var lastErr error + + for i := 0; i < len(subdomainRecords); i += batchSize { + end := i + batchSize + if end > len(subdomainRecords) { + end = len(subdomainRecords) + } + + batch := subdomainRecords[i:end] + // Use Clauses with OnConflict DoNothing to skip conflicts + err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error + if err != nil { + zap.L().Warn("Error storing some subdomain records", zap.Error(err)) + lastErr = err + // Continue with next batch despite error + } + } + + return lastErr +} + +func StoreWhoisHistoryRecords(historyRecords []HistoryRecord) error { + if len(historyRecords) == 0 { + return nil + } + + zap.L().Info("Storing history records", zap.Int("count", len(historyRecords))) + db := GetDB() + + // Use batch insert with conflict handling + const batchSize = 100 + var lastErr error + + for i := 0; i < len(historyRecords); i += batchSize { + end := i + batchSize + if end > len(historyRecords) { + end = len(historyRecords) + } + + batch := historyRecords[i:end] + // Use Clauses with OnConflict DoNothing to skip conflicts + err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error + if err != nil { + zap.L().Warn("Error storing some history records", zap.Error(err)) + lastErr = err + // Continue with next batch despite error + } + } + + return lastErr +} + +func StoreWhoisLookup(lookup []LookupResult) error { + if len(lookup) == 0 { + return nil + } + + zap.L().Info("Storing IP lookup records", zap.Int("count", len(lookup))) + db := GetDB() + + // Use batch insert with conflict handling + const batchSize = 100 + var lastErr error + + for i := 0; i < len(lookup); i += batchSize { + end := i + batchSize + if end > len(lookup) { + end = len(lookup) + } + + batch := lookup[i:end] + // Use Clauses with OnConflict DoNothing to skip conflicts + err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error + if err != nil { + zap.L().Warn("Error storing some IP lookup records", zap.Error(err)) + lastErr = err + // Continue with next batch despite error + } + } + + return lastErr +} diff --git a/internal/whois/whois.go b/internal/whois/whois.go index 9e1c163..86fa787 100644 --- a/internal/whois/whois.go +++ b/internal/whois/whois.go @@ -552,7 +552,7 @@ func (w *DehashedWhoIs) WhoisIP(ipAddress string) ([]sqlite.LookupResult, error) }) } - sqlite.StoreIPLookup(lookups) + sqlite.StoreWhoisLookup(lookups) return lookups, nil } @@ -702,7 +702,7 @@ func (w *DehashedWhoIs) WhoisMX(mxHostname string) ([]sqlite.LookupResult, error }) } - sqlite.StoreIPLookup(mxLookups) + sqlite.StoreWhoisLookup(mxLookups) return mxLookups, nil } @@ -849,7 +849,7 @@ func (w *DehashedWhoIs) WhoisNS(nsHostname string) ([]sqlite.LookupResult, error }) } - sqlite.StoreIPLookup(nsLookups) + sqlite.StoreWhoisLookup(nsLookups) return nsLookups, nil }