fixed output of coffee and primary banner

cmd/dehashed.go

@@ -118,7 +118,7 @@ var (
 			dehasher.Start()
 			fmt.Println("\n[*] Completing Process")
 
-			err := sqlite.StoreQueryOptions(queryOptions)
+			err := sqlite.StoreDehashedQueryOptions(queryOptions)
 			if err != nil {
 				if debugGlobal {
 					debug.PrintInfo("failed to store query options")

cmd/root.go

@@ -21,12 +21,12 @@ var (
 		Long: fmt.Sprintf(
 			"%s\n",
 			`
-   ╔═╗┬─┐┌─┐┬ ┬┌─┐╔╗╔┌─┐┌─┐┌┬┐
+    ╔═╗┬─┐┌─┐┬ ┬┌─┐╔╗╔┌─┐┌─┐┌┬┐
-   ║  ├┬┘│ ││││└─┐║║║├┤ └─┐ │
+    ║  ├┬┘│ ││││└─┐║║║├┤ └─┐ │
-   ╚═╝┴└─└─┘└┴┘└─┘╝╚╝└─┘└─┘ ┴
+    ╚═╝┴└─└─┘└┴┘└─┘╝╚╝└─┘└─┘ ┴
 
-  Crow’s Nest OSINT Recon Suite
+   Crow’s Nest OSINT Recon Suite
-⚓ A KrakenTech Intelligence Tool
+ ⚓ A KrakenTech Intelligence Tool
 `,
 		),
 		Version: "v1.2.1",
@@ -124,7 +124,7 @@ var buyMeCoffeeCmd = &cobra.Command{
 	Use:   "coffee",
 	Short: "Support the project by buying a coffee",
 	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Println(color.HiRedString("                                                                                   ;)( ;"))
+		fmt.Println(color.HiRedString("                                                                                   ;)(; "))
 		fmt.Println(color.HiCyanString("                   We Hope You Enjoy Our Product                                  :----:"))
 		fmt.Println(color.HiCyanString("                                                                                 C|====|"))
 		fmt.Println(color.HiCyanString("                                                                                  |    |"))

cmd/whois.go

@@ -208,7 +208,7 @@ var (
 								fmt.Printf("[!] Error writing WHOIS history to file: %v\n", writeErr)
 							}
 
-							err = sqlite.StoreHistoryRecord(historyRecords)
+							err = sqlite.StoreWhoisHistoryRecords(historyRecords)
 							if err != nil {
 								if debugGlobal {
 									debug.PrintInfo("failed to store history record")
@@ -254,7 +254,7 @@ var (
 						fmt.Printf("Error performing subdomain scan: %v\n", err)
 					} else {
 						fmt.Println("Subdomain Scan:")
-						err = sqlite.StoreSubdomainRecords(subdomains)
+						err = sqlite.StoreWhoisSubdomainRecords(subdomains)
 						if err != nil {
 							if debugGlobal {
 								debug.PrintInfo("failed to store subdomain record")

internal/dehashed/dehashed.go

@@ -130,7 +130,7 @@ func (dh *Dehasher) Start() {
 
 			if len(dh.client.results) > 0 {
 				fmt.Printf("   [!] Partial results retrieved. Storing Results...\n")
-				err := sqlite.StoreResults(dh.client.GetResults())
+				err := sqlite.StoreDehashedResults(dh.client.GetResults())
 				if err != nil {
 					zap.L().Error("store_results",
 						zap.String("message", "failed to store results"),
@@ -214,7 +214,7 @@ func (dh *Dehasher) parseResults() {
 	results := dh.client.GetResults()
 	creds := results.ExtractCredentials()
 	fmt.Printf("\n\t[+] Discovered %d Credentials", len(creds))
-	err := sqlite.StoreCreds(creds)
+	err := sqlite.StoreDehashedCreds(creds)
 	if err != nil {
 		zap.L().Error("store_creds",
 			zap.String("message", "failed to store creds"),
@@ -224,7 +224,7 @@ func (dh *Dehasher) parseResults() {
 	zap.L().Info("creds_stored", zap.Int("count", len(creds)))
 
 	zap.L().Info("storing_results")
-	err = sqlite.StoreResults(results)
+	err = sqlite.StoreDehashedResults(results)
 	if err != nil {
 		zap.L().Error("store_results",
 			zap.String("message", "failed to store results"),

internal/sqlite/db.go

@@ -0,0 +1,143 @@
+package sqlite
+
+import (
+	"fmt"
+	"go.uber.org/zap"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
+)
+
+var DB *gorm.DB
+
+// InitDB initializes the database connection
+func InitDB(dbPath string) (*gorm.DB, error) {
+	zap.L().Info("Initializing database", zap.String("path", dbPath))
+
+	// Check if the path is a file or directory
+	fileInfo, err := os.Stat(dbPath)
+	var finalDbPath string
+
+	// If path doesn't exist or is a directory
+	if os.IsNotExist(err) || (err == nil && fileInfo.IsDir()) {
+		// Treat as directory path
+		if err := os.MkdirAll(dbPath, 0755); err != nil {
+			zap.L().Error("Failed to create database directory", zap.Error(err))
+			return nil, fmt.Errorf("failed to create database directory: %w", err)
+		}
+		finalDbPath = filepath.Join(dbPath, "dehashed.sqlite")
+	} else {
+		// Treat as file path
+		// Ensure the directory exists
+		dir := filepath.Dir(dbPath)
+		if err := os.MkdirAll(dir, 0755); err != nil {
+			zap.L().Error("Failed to create parent directory for database", zap.Error(err))
+			return nil, fmt.Errorf("failed to create parent directory for database: %w", err)
+		}
+		finalDbPath = dbPath
+	}
+
+	zap.L().Info("Opening database", zap.String("finalPath", finalDbPath))
+	db, err := gorm.Open(sqlite.Open(finalDbPath), &gorm.Config{
+		Logger: logger.Default.LogMode(logger.Silent),
+	})
+	if err != nil {
+		zap.L().Error("Failed to connect to database", zap.Error(err))
+		return nil, fmt.Errorf("failed to connect to database: %w", err)
+	}
+
+	// Auto migrate your models
+	err = db.AutoMigrate(&Result{}, &Creds{}, &QueryOptions{}, &Creds{}, &WhoisRecord{}, &SubdomainRecord{},
+		&HistoryRecord{}, &LookupResult{}, &HunterDomainData{}, &HunterEmail{}, &PersonData{})
+	if err != nil {
+		zap.L().Error("Failed to migrate database", zap.Error(err))
+		return nil, fmt.Errorf("failed to migrate database: %w", err)
+	}
+
+	DB = db
+	return db, nil
+}
+
+// GetDB returns the database connection
+func GetDB() *gorm.DB {
+	if DB == nil {
+		zap.L().Error("database not initialized")
+		fmt.Println("sqlite database not initialized")
+		os.Exit(1)
+	}
+	return DB
+}
+
+type Table int64
+
+const (
+	ResultsTable Table = iota
+	RunsTable
+	CredsTable
+	WhoIsTable
+	SubdomainsTable
+	HistoryTable
+	LookupTable
+	HunterDomainTable
+	HunterEmailTable
+	PersonTable
+	UnknownTable
+)
+
+func GetTable(userInput string) Table {
+	switch strings.ToLower(userInput) {
+	case "results":
+		return ResultsTable
+	case "runs":
+		return RunsTable
+	case "creds":
+		return CredsTable
+	case "whois":
+		return WhoIsTable
+	case "subdomains":
+		return SubdomainsTable
+	case "history":
+		return HistoryTable
+	case "lookup":
+		return LookupTable
+	case "hunter_domain":
+		return HunterDomainTable
+	case "hunter_email":
+		return HunterEmailTable
+	case "person":
+		return PersonTable
+	default:
+		return UnknownTable
+	}
+}
+
+func (t Table) Object() interface{} {
+	switch t {
+	case ResultsTable:
+		return Result{}
+	case RunsTable:
+		return QueryOptions{}
+	case CredsTable:
+		return Creds{}
+	case WhoIsTable:
+		return WhoisRecord{}
+	case SubdomainsTable:
+		return SubdomainRecord{}
+	case HistoryTable:
+		return HistoryRecord{}
+	case LookupTable:
+		return LookupResult{}
+	case HunterDomainTable:
+		return HunterDomainData{}
+	case HunterEmailTable:
+		return HunterEmail{}
+	case PersonTable:
+		return PersonData{}
+	default:
+		return nil
+	}
+}

internal/sqlite/dehashed.go

@@ -0,0 +1,235 @@
+package sqlite
+
+import (
+	"crowsnest/internal/files"
+	"fmt"
+	"go.uber.org/zap"
+	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
+)
+
+type QueryOptions struct {
+	gorm.Model
+	MaxRecords         int            `json:"max_records"`
+	MaxRequests        int            `json:"max_requests"`
+	StartingPage       int            `json:"starting_page"`
+	OutputFormat       files.FileType `json:"output_format"`
+	OutputFile         string         `json:"output_file"`
+	RegexMatch         bool           `json:"regex_match"`
+	WildcardMatch      bool           `json:"wildcard_match"`
+	UsernameQuery      string         `json:"username_query"`
+	EmailQuery         string         `json:"email_query"`
+	IpQuery            string         `json:"ip_query"`
+	PassQuery          string         `json:"pass_query"`
+	HashQuery          string         `json:"hash_query"`
+	NameQuery          string         `json:"name_query"`
+	DomainQuery        string         `json:"domain_query"`
+	VinQuery           string         `json:"vin_query"`
+	LicensePlateQuery  string         `json:"license_plate_query"`
+	AddressQuery       string         `json:"address_query"`
+	PhoneQuery         string         `json:"phone_query"`
+	SocialQuery        string         `json:"social_query"`
+	CryptoAddressQuery string         `json:"crypto_address_query"`
+	PrintBalance       bool           `json:"print_balance"`
+	CredsOnly          bool           `json:"creds_only"`
+	Debug              bool           `json:"debug"`
+}
+
+func (QueryOptions) TableName() string {
+	return "query_options"
+}
+
+func NewQueryOptions(maxRecords, maxRequests, startingPage int, outputFormat, outputFile, usernameQuery, emailQuery, ipQuery, passQuery, hashQuery, nameQuery, domainQuery, vinQuery, licensePlateQuery, addressQuery, phoneQuery, socialQuery, cryptoAddressQuery string, regexMatch, wildcardMatch, printBalance, credsOnly, debug bool) *QueryOptions {
+	return &QueryOptions{
+		MaxRecords:         maxRecords,
+		MaxRequests:        maxRequests,
+		StartingPage:       startingPage,
+		OutputFormat:       files.GetFileType(outputFormat),
+		OutputFile:         outputFile,
+		PrintBalance:       printBalance,
+		CredsOnly:          credsOnly,
+		RegexMatch:         regexMatch,
+		WildcardMatch:      wildcardMatch,
+		UsernameQuery:      usernameQuery,
+		EmailQuery:         emailQuery,
+		IpQuery:            ipQuery,
+		PassQuery:          passQuery,
+		HashQuery:          hashQuery,
+		NameQuery:          nameQuery,
+		DomainQuery:        domainQuery,
+		VinQuery:           vinQuery,
+		LicensePlateQuery:  licensePlateQuery,
+		AddressQuery:       addressQuery,
+		PhoneQuery:         phoneQuery,
+		SocialQuery:        socialQuery,
+		CryptoAddressQuery: cryptoAddressQuery,
+		Debug:              debug,
+	}
+}
+
+type DehashedSearchRequest struct {
+	Page     int    `json:"page"`
+	Query    string `json:"query"`
+	Size     int    `json:"size"`
+	Wildcard bool   `json:"wildcard"`
+	Regex    bool   `json:"regex"`
+	DeDupe   bool   `json:"de_dupe"`
+}
+
+type DehashedResponse struct {
+	Balance      int      `json:"balance"`
+	Entries      []Result `json:"entries"`
+	Success      bool     `json:"success"`
+	Took         string   `json:"took"`
+	TotalResults int      `json:"total"`
+}
+
+type Result struct {
+	gorm.Model
+	DehashedId            string   `json:"id" xml:"id" yaml:"id" gorm:"uniqueIndex"`
+	Email                 []string `json:"email,omitempty" xml:"email,omitempty" yaml:"email,omitempty" gorm:"serializer:json"`
+	IpAddress             []string `json:"ip_address,omitempty" xml:"ip_address,omitempty" yaml:"ip_address,omitempty" gorm:"serializer:json"`
+	Username              []string `json:"username,omitempty" xml:"username,omitempty" yaml:"username,omitempty" gorm:"serializer:json"`
+	Password              []string `json:"password,omitempty" xml:"password,omitempty" yaml:"password,omitempty" gorm:"serializer:json"`
+	HashedPassword        []string `json:"hashed_password,omitempty" xml:"hashed_password,omitempty" yaml:"hashed_password,omitempty" gorm:"serializer:json"`
+	HashType              string   `json:"hash_type,omitempty" xml:"hash_type,omitempty" yaml:"hash_type,omitempty"`
+	Name                  []string `json:"name,omitempty" xml:"name,omitempty" yaml:"name,omitempty" gorm:"serializer:json"`
+	Vin                   []string `json:"vin,omitempty" xml:"vin,omitempty" yaml:"vin,omitempty" gorm:"serializer:json"`
+	LicensePlate          []string `json:"license_plate,omitempty" xml:"license_plate,omitempty" yaml:"license_plate,omitempty" gorm:"serializer:json"`
+	Url                   []string `json:"url,omitempty" xml:"url,omitempty" yaml:"url,omitempty" gorm:"serializer:json"`
+	Social                []string `json:"social,omitempty" xml:"social,omitempty" yaml:"social,omitempty" gorm:"serializer:json"`
+	CryptoCurrencyAddress []string `json:"cryptocurrency_address,omitempty" xml:"cryptocurrency_address,omitempty" yaml:"cryptocurrency_address,omitempty" gorm:"serializer:json"`
+	Address               []string `json:"address,omitempty" xml:"address,omitempty" yaml:"address,omitempty" gorm:"serializer:json"`
+	Phone                 []string `json:"phone,omitempty" xml:"phone,omitempty" yaml:"phone,omitempty" gorm:"serializer:json"`
+	Company               []string `json:"company,omitempty" xml:"company,omitempty" yaml:"company,omitempty" gorm:"serializer:json"`
+	DatabaseName          string   `json:"database_name,omitempty" xml:"database_name,omitempty" yaml:"database_name,omitempty"`
+}
+
+func (Result) TableName() string {
+	return "results"
+}
+
+type DehashedResults struct {
+	Results []Result `json:"results"`
+}
+
+func (dr *DehashedResults) ExtractCredentials() []Creds {
+	var creds []Creds
+
+	results := dr.Results
+
+	for _, r := range results {
+		if len(r.Password) > 0 {
+			// Get first email if available
+			email := ""
+			if len(r.Email) > 0 {
+				email = r.Email[0]
+			}
+
+			// Get first password
+			password := r.Password[0]
+
+			cred := Creds{Email: email, Password: password}
+			creds = append(creds, cred)
+		}
+	}
+
+	go func() {
+		err := StoreDehashedCreds(creds)
+		if err != nil {
+			zap.L().Error("store_creds",
+				zap.String("message", "failed to store creds"),
+				zap.Error(err),
+			)
+			fmt.Printf("Error Storing Results: %v", err)
+		}
+	}()
+
+	return creds
+}
+
+type Creds struct {
+	gorm.Model
+	Email    string `json:"email" yaml:"email" xml:"email" gorm:"uniqueIndex:idx_email_username_password"`
+	Username string `json:"username" yaml:"username" xml:"username" gorm:"uniqueIndex:idx_email_username_password"`
+	Password string `json:"password" yaml:"password" xml:"password" gorm:"uniqueIndex:idx_email_username_password"`
+}
+
+func (Creds) TableName() string {
+	return "creds"
+}
+
+func (c Creds) ToString() string {
+	return fmt.Sprintf("%s%s%s", c.Username, "%", c.Password)
+}
+
+func StoreDehashedResults(results DehashedResults) error {
+	if len(results.Results) == 0 {
+		return nil
+	}
+
+	zap.L().Info("Storing results", zap.Int("count", len(results.Results)))
+	db := GetDB()
+
+	// Use batch insert with conflict handling
+	const batchSize = 100
+	var lastErr error
+
+	// Extract the slice of results
+	resultSlice := results.Results
+
+	for i := 0; i < len(resultSlice); i += batchSize {
+		end := i + batchSize
+		if end > len(resultSlice) {
+			end = len(resultSlice)
+		}
+
+		batch := resultSlice[i:end]
+		// Use Clauses with OnConflict DoNothing to skip conflicts
+		err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
+		if err != nil {
+			zap.L().Warn("Error storing some results", zap.Error(err))
+			lastErr = err
+			// Continue with next batch despite error
+		}
+	}
+
+	return lastErr
+}
+
+func StoreDehashedCreds(creds []Creds) error {
+	if len(creds) == 0 {
+		return nil
+	}
+
+	zap.L().Info("Storing credentials", zap.Int("count", len(creds)))
+	db := GetDB()
+
+	// Use batch insert with conflict handling
+	// This will insert records in batches and continue even if some fail
+	const batchSize = 100
+	var lastErr error
+
+	for i := 0; i < len(creds); i += batchSize {
+		end := i + batchSize
+		if end > len(creds) {
+			end = len(creds)
+		}
+
+		batch := creds[i:end]
+		// Use Clauses with OnConflict DoNothing to skip conflicts
+		err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
+		if err != nil {
+			zap.L().Warn("Error storing some credentials", zap.Error(err))
+			lastErr = err
+			// Continue with next batch despite error
+		}
+	}
+
+	return lastErr
+}
+
+func StoreDehashedQueryOptions(queryOptions *QueryOptions) error {
+	db := GetDB()
+	return db.Create(queryOptions).Error
+}

internal/sqlite/gorm.go

@@ -1,319 +0,0 @@
-package sqlite
-
-import (
-	"fmt"
-	"go.uber.org/zap"
-	"gorm.io/gorm/clause"
-	"os"
-	"path/filepath"
-
-	"gorm.io/driver/sqlite"
-	"gorm.io/gorm"
-	"gorm.io/gorm/logger"
-)
-
-var DB *gorm.DB
-
-// InitDB initializes the database connection
-func InitDB(dbPath string) (*gorm.DB, error) {
-	zap.L().Info("Initializing database", zap.String("path", dbPath))
-
-	// Check if the path is a file or directory
-	fileInfo, err := os.Stat(dbPath)
-	var finalDbPath string
-
-	// If path doesn't exist or is a directory
-	if os.IsNotExist(err) || (err == nil && fileInfo.IsDir()) {
-		// Treat as directory path
-		if err := os.MkdirAll(dbPath, 0755); err != nil {
-			zap.L().Error("Failed to create database directory", zap.Error(err))
-			return nil, fmt.Errorf("failed to create database directory: %w", err)
-		}
-		finalDbPath = filepath.Join(dbPath, "dehashed.sqlite")
-	} else {
-		// Treat as file path
-		// Ensure the directory exists
-		dir := filepath.Dir(dbPath)
-		if err := os.MkdirAll(dir, 0755); err != nil {
-			zap.L().Error("Failed to create parent directory for database", zap.Error(err))
-			return nil, fmt.Errorf("failed to create parent directory for database: %w", err)
-		}
-		finalDbPath = dbPath
-	}
-
-	zap.L().Info("Opening database", zap.String("finalPath", finalDbPath))
-	db, err := gorm.Open(sqlite.Open(finalDbPath), &gorm.Config{
-		Logger: logger.Default.LogMode(logger.Silent),
-	})
-	if err != nil {
-		zap.L().Error("Failed to connect to database", zap.Error(err))
-		return nil, fmt.Errorf("failed to connect to database: %w", err)
-	}
-
-	// Auto migrate your models
-	err = db.AutoMigrate(&Result{}, &Creds{}, &QueryOptions{}, &Creds{}, &WhoisRecord{}, &SubdomainRecord{},
-		&HistoryRecord{}, &LookupResult{}, &HunterDomainData{}, &HunterEmail{}, &PersonData{})
-	if err != nil {
-		zap.L().Error("Failed to migrate database", zap.Error(err))
-		return nil, fmt.Errorf("failed to migrate database: %w", err)
-	}
-
-	DB = db
-	return db, nil
-}
-
-// GetDB returns the database connection
-func GetDB() *gorm.DB {
-	if DB == nil {
-		zap.L().Error("database not initialized")
-		fmt.Println("sqlite database not initialized")
-		os.Exit(1)
-	}
-	return DB
-}
-
-func StoreResults(results DehashedResults) error {
-	if len(results.Results) == 0 {
-		return nil
-	}
-
-	zap.L().Info("Storing results", zap.Int("count", len(results.Results)))
-	db := GetDB()
-
-	// Use batch insert with conflict handling
-	const batchSize = 100
-	var lastErr error
-
-	// Extract the slice of results
-	resultSlice := results.Results
-
-	for i := 0; i < len(resultSlice); i += batchSize {
-		end := i + batchSize
-		if end > len(resultSlice) {
-			end = len(resultSlice)
-		}
-
-		batch := resultSlice[i:end]
-		// Use Clauses with OnConflict DoNothing to skip conflicts
-		err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
-		if err != nil {
-			zap.L().Warn("Error storing some results", zap.Error(err))
-			lastErr = err
-			// Continue with next batch despite error
-		}
-	}
-
-	return lastErr
-}
-
-func StoreCreds(creds []Creds) error {
-	if len(creds) == 0 {
-		return nil
-	}
-
-	zap.L().Info("Storing credentials", zap.Int("count", len(creds)))
-	db := GetDB()
-
-	// Use batch insert with conflict handling
-	// This will insert records in batches and continue even if some fail
-	const batchSize = 100
-	var lastErr error
-
-	for i := 0; i < len(creds); i += batchSize {
-		end := i + batchSize
-		if end > len(creds) {
-			end = len(creds)
-		}
-
-		batch := creds[i:end]
-		// Use Clauses with OnConflict DoNothing to skip conflicts
-		err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
-		if err != nil {
-			zap.L().Warn("Error storing some credentials", zap.Error(err))
-			lastErr = err
-			// Continue with next batch despite error
-		}
-	}
-
-	return lastErr
-}
-
-func StoreQueryOptions(queryOptions *QueryOptions) error {
-	db := GetDB()
-	return db.Create(queryOptions).Error
-}
-
-func StoreWhoisRecord(whoisRecord WhoisRecord) error {
-	// Create a pointer to the record to make it addressable
-	recordPtr := &whoisRecord
-
-	zap.L().Info("Storing WHOIS record",
-		zap.String("domain", whoisRecord.DomainName))
-
-	db := GetDB()
-
-	// Use OnConflict clause to handle duplicates
-	err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(recordPtr).Error
-	if err != nil {
-		zap.L().Error("store_whois_record",
-			zap.String("message", "failed to store whois record"),
-			zap.Error(err))
-		return err
-	}
-
-	return nil
-}
-
-func StoreSubdomainRecords(subdomainRecords []SubdomainRecord) error {
-	if len(subdomainRecords) == 0 {
-		return nil
-	}
-
-	zap.L().Info("Storing subdomain records", zap.Int("count", len(subdomainRecords)))
-	db := GetDB()
-
-	// Use batch insert with conflict handling
-	const batchSize = 100
-	var lastErr error
-
-	for i := 0; i < len(subdomainRecords); i += batchSize {
-		end := i + batchSize
-		if end > len(subdomainRecords) {
-			end = len(subdomainRecords)
-		}
-
-		batch := subdomainRecords[i:end]
-		// Use Clauses with OnConflict DoNothing to skip conflicts
-		err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
-		if err != nil {
-			zap.L().Warn("Error storing some subdomain records", zap.Error(err))
-			lastErr = err
-			// Continue with next batch despite error
-		}
-	}
-
-	return lastErr
-}
-
-func StoreHistoryRecord(historyRecords []HistoryRecord) error {
-	if len(historyRecords) == 0 {
-		return nil
-	}
-
-	zap.L().Info("Storing history records", zap.Int("count", len(historyRecords)))
-	db := GetDB()
-
-	// Use batch insert with conflict handling
-	const batchSize = 100
-	var lastErr error
-
-	for i := 0; i < len(historyRecords); i += batchSize {
-		end := i + batchSize
-		if end > len(historyRecords) {
-			end = len(historyRecords)
-		}
-
-		batch := historyRecords[i:end]
-		// Use Clauses with OnConflict DoNothing to skip conflicts
-		err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
-		if err != nil {
-			zap.L().Warn("Error storing some history records", zap.Error(err))
-			lastErr = err
-			// Continue with next batch despite error
-		}
-	}
-
-	return lastErr
-}
-
-func StoreIPLookup(ipLookup []LookupResult) error {
-	if len(ipLookup) == 0 {
-		return nil
-	}
-
-	zap.L().Info("Storing IP lookup records", zap.Int("count", len(ipLookup)))
-	db := GetDB()
-
-	// Use batch insert with conflict handling
-	const batchSize = 100
-	var lastErr error
-
-	for i := 0; i < len(ipLookup); i += batchSize {
-		end := i + batchSize
-		if end > len(ipLookup) {
-			end = len(ipLookup)
-		}
-
-		batch := ipLookup[i:end]
-		// Use Clauses with OnConflict DoNothing to skip conflicts
-		err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
-		if err != nil {
-			zap.L().Warn("Error storing some IP lookup records", zap.Error(err))
-			lastErr = err
-			// Continue with next batch despite error
-		}
-	}
-
-	return lastErr
-}
-
-func StoreHunterDomain(hunterDomain HunterDomainData) error {
-	db := GetDB()
-
-	// Use OnConflict clause to handle duplicates
-	err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(&hunterDomain).Error
-	if err != nil {
-		zap.L().Error("store_hunter_domain",
-			zap.String("message", "failed to store hunter domain"),
-			zap.Error(err))
-		return err
-	}
-
-	return nil
-}
-
-func StoreHunterEmails(hunterEmails []HunterEmail) error {
-	if len(hunterEmails) == 0 {
-		return nil
-	}
-
-	zap.L().Info("Storing hunter emails", zap.Int("count", len(hunterEmails)))
-	db := GetDB()
-
-	// Use batch insert with conflict handling
-	const batchSize = 100
-	var lastErr error
-
-	for i := 0; i < len(hunterEmails); i += batchSize {
-		end := i + batchSize
-		if end > len(hunterEmails) {
-			end = len(hunterEmails)
-		}
-
-		batch := hunterEmails[i:end]
-		// Use Clauses with OnConflict DoNothing to skip conflicts
-		err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
-		if err != nil {
-			zap.L().Warn("Error storing some hunter emails", zap.Error(err))
-			lastErr = err
-			// Continue with next batch despite error
-		}
-	}
-
-	return lastErr
-}
-
-func StorePersonData(personData PersonData) error {
-	db := GetDB()
-
-	// Use OnConflict clause to handle duplicates
-	err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(&personData).Error
-	if err != nil {
-		zap.L().Error("store_person_data",
-			zap.String("message", "failed to store person data"),
-			zap.Error(err))
-		return err
-	}
-
-	return nil
-}

internal/sqlite/hunter.io.go

@@ -3,7 +3,9 @@ package sqlite
 import (
 	"fmt"
 	"github.com/charmbracelet/lipgloss/tree"
+	"go.uber.org/zap"
 	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
 )
 
 // HunterDomainSearchResult represents the response from Hunter.io domain search API
@@ -14,8 +16,8 @@ type HunterDomainSearchResult struct {
 
 // HunterDomainData contains the main domain information
 type HunterDomainData struct {
-	IString
 	gorm.Model
+	IString       `gorm:"-"`
 	Domain        string        `json:"domain" gorm:"unique"`
 	Disposable    bool          `json:"disposable"`
 	Webmail       bool          `json:"webmail"`
@@ -512,8 +514,8 @@ type HunterPersonEnrichmentResponse struct {
 
 // PersonData contains the detailed person information
 type PersonData struct {
-	IString
 	gorm.Model
+	IString       `gorm:"-"`
 	ID            string     `json:"id"`
 	Name          PersonName `json:"name" gorm:"embedded;embeddedPrefix:name_"`
 	Email         string     `json:"email" gorm:"unique"`
@@ -755,3 +757,64 @@ func (c *HunterCombinedEnrichmentResponse) String() string {
 		c.Data.Person.String(),
 		c.Data.Company.String())
 }
+
+func StoreHunterDomain(hunterDomain HunterDomainData) error {
+	db := GetDB()
+
+	// Use OnConflict clause to handle duplicates
+	err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(&hunterDomain).Error
+	if err != nil {
+		zap.L().Error("store_hunter_domain",
+			zap.String("message", "failed to store hunter domain"),
+			zap.Error(err))
+		return err
+	}
+
+	return nil
+}
+
+func StoreHunterEmails(hunterEmails []HunterEmail) error {
+	if len(hunterEmails) == 0 {
+		return nil
+	}
+
+	zap.L().Info("Storing hunter emails", zap.Int("count", len(hunterEmails)))
+	db := GetDB()
+
+	// Use batch insert with conflict handling
+	const batchSize = 100
+	var lastErr error
+
+	for i := 0; i < len(hunterEmails); i += batchSize {
+		end := i + batchSize
+		if end > len(hunterEmails) {
+			end = len(hunterEmails)
+		}
+
+		batch := hunterEmails[i:end]
+		// Use Clauses with OnConflict DoNothing to skip conflicts
+		err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
+		if err != nil {
+			zap.L().Warn("Error storing some hunter emails", zap.Error(err))
+			lastErr = err
+			// Continue with next batch despite error
+		}
+	}
+
+	return lastErr
+}
+
+func StoreHunterPersonData(personData PersonData) error {
+	db := GetDB()
+
+	// Use OnConflict clause to handle duplicates
+	err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(&personData).Error
+	if err != nil {
+		zap.L().Error("store_person_data",
+			zap.String("message", "failed to store person data"),
+			zap.Error(err))
+		return err
+	}
+
+	return nil
+}

internal/sqlite/query.go

@@ -1,20 +0,0 @@
-package sqlite
-
-type DehashedSearchRequest struct {
-	Page     int    `json:"page"`
-	Query    string `json:"query"`
-	Size     int    `json:"size"`
-	Wildcard bool   `json:"wildcard"`
-	Regex    bool   `json:"regex"`
-	DeDupe   bool   `json:"de_dupe"`
-}
-
-func NewDehashedSearchRequest(size int, wildcard, regex bool) *DehashedSearchRequest {
-	return &DehashedSearchRequest{
-		Page:     0,
-		Size:     size,
-		Wildcard: false,
-		Regex:    false,
-		DeDupe:   true,
-	}
-}

internal/sqlite/result.go

@@ -1,94 +0,0 @@
-package sqlite
-
-import (
-	"encoding/json"
-	"fmt"
-	"go.uber.org/zap"
-	"gorm.io/gorm"
-	"io"
-	"os"
-)
-
-type DehashedResponse struct {
-	Balance      int      `json:"balance"`
-	Entries      []Result `json:"entries"`
-	Success      bool     `json:"success"`
-	Took         string   `json:"took"`
-	TotalResults int      `json:"total"`
-}
-
-type Result struct {
-	gorm.Model
-	DehashedId            string   `json:"id" xml:"id" yaml:"id" gorm:"uniqueIndex"`
-	Email                 []string `json:"email,omitempty" xml:"email,omitempty" yaml:"email,omitempty" gorm:"serializer:json"`
-	IpAddress             []string `json:"ip_address,omitempty" xml:"ip_address,omitempty" yaml:"ip_address,omitempty" gorm:"serializer:json"`
-	Username              []string `json:"username,omitempty" xml:"username,omitempty" yaml:"username,omitempty" gorm:"serializer:json"`
-	Password              []string `json:"password,omitempty" xml:"password,omitempty" yaml:"password,omitempty" gorm:"serializer:json"`
-	HashedPassword        []string `json:"hashed_password,omitempty" xml:"hashed_password,omitempty" yaml:"hashed_password,omitempty" gorm:"serializer:json"`
-	HashType              string   `json:"hash_type,omitempty" xml:"hash_type,omitempty" yaml:"hash_type,omitempty"`
-	Name                  []string `json:"name,omitempty" xml:"name,omitempty" yaml:"name,omitempty" gorm:"serializer:json"`
-	Vin                   []string `json:"vin,omitempty" xml:"vin,omitempty" yaml:"vin,omitempty" gorm:"serializer:json"`
-	LicensePlate          []string `json:"license_plate,omitempty" xml:"license_plate,omitempty" yaml:"license_plate,omitempty" gorm:"serializer:json"`
-	Url                   []string `json:"url,omitempty" xml:"url,omitempty" yaml:"url,omitempty" gorm:"serializer:json"`
-	Social                []string `json:"social,omitempty" xml:"social,omitempty" yaml:"social,omitempty" gorm:"serializer:json"`
-	CryptoCurrencyAddress []string `json:"cryptocurrency_address,omitempty" xml:"cryptocurrency_address,omitempty" yaml:"cryptocurrency_address,omitempty" gorm:"serializer:json"`
-	Address               []string `json:"address,omitempty" xml:"address,omitempty" yaml:"address,omitempty" gorm:"serializer:json"`
-	Phone                 []string `json:"phone,omitempty" xml:"phone,omitempty" yaml:"phone,omitempty" gorm:"serializer:json"`
-	Company               []string `json:"company,omitempty" xml:"company,omitempty" yaml:"company,omitempty" gorm:"serializer:json"`
-	DatabaseName          string   `json:"database_name,omitempty" xml:"database_name,omitempty" yaml:"database_name,omitempty"`
-}
-
-func (Result) TableName() string {
-	return "results"
-}
-
-type DehashedResults struct {
-	Results []Result `json:"results"`
-}
-
-func (dr *DehashedResults) ExtractCredentials() []Creds {
-	var creds []Creds
-
-	results := dr.Results
-
-	for _, r := range results {
-		if len(r.Password) > 0 {
-			// Get first email if available
-			email := ""
-			if len(r.Email) > 0 {
-				email = r.Email[0]
-			}
-
-			// Get first password
-			password := r.Password[0]
-
-			cred := Creds{Email: email, Password: password}
-			creds = append(creds, cred)
-		}
-	}
-
-	go func() {
-		err := StoreCreds(creds)
-		if err != nil {
-			zap.L().Error("store_creds",
-				zap.String("message", "failed to store creds"),
-				zap.Error(err),
-			)
-			fmt.Printf("Error Storing Results: %v", err)
-		}
-	}()
-
-	return creds
-}
-
-func NewDehashedResults(body io.Reader) ([]Result, int, int) {
-	var response DehashedResponse
-
-	err := json.NewDecoder(body).Decode(&response)
-	if err != nil {
-		fmt.Printf("Error Parsing Response Body: %v", err)
-		os.Exit(-1)
-	}
-
-	return response.Entries, response.Balance, response.TotalResults
-}

internal/sqlite/structs.go

@@ -1,11 +1,5 @@
 package sqlite
 
-import (
-	"crowsnest/internal/files"
-	"fmt"
-	"gorm.io/gorm"
-)
-
 type IString interface {
 	String() string
 }
@@ -30,15 +24,6 @@ type DBOptions struct {
 	DisplayFields         []string // Fields to display in output
 }
 
-func NewDBOptions() *DBOptions {
-	return &DBOptions{
-		Limit:          100, // Default limit
-		ExactMatch:     false,
-		NonEmptyFields: []string{},
-		DisplayFields:  []string{},
-	}
-}
-
 func (o *DBOptions) Empty() bool {
 	return o.Username == "" && o.Email == "" && o.IPAddress == "" &&
 		o.Password == "" && o.HashedPassword == "" && o.Name == "" &&
@@ -46,77 +31,3 @@ func (o *DBOptions) Empty() bool {
 		o.Phone == "" && o.Social == "" && o.CryptoCurrencyAddress == "" && o.Domain == "" &&
 		len(o.NonEmptyFields) == 0
 }
-
-type QueryOptions struct {
-	gorm.Model
-	MaxRecords         int            `json:"max_records"`
-	MaxRequests        int            `json:"max_requests"`
-	StartingPage       int            `json:"starting_page"`
-	OutputFormat       files.FileType `json:"output_format"`
-	OutputFile         string         `json:"output_file"`
-	RegexMatch         bool           `json:"regex_match"`
-	WildcardMatch      bool           `json:"wildcard_match"`
-	UsernameQuery      string         `json:"username_query"`
-	EmailQuery         string         `json:"email_query"`
-	IpQuery            string         `json:"ip_query"`
-	PassQuery          string         `json:"pass_query"`
-	HashQuery          string         `json:"hash_query"`
-	NameQuery          string         `json:"name_query"`
-	DomainQuery        string         `json:"domain_query"`
-	VinQuery           string         `json:"vin_query"`
-	LicensePlateQuery  string         `json:"license_plate_query"`
-	AddressQuery       string         `json:"address_query"`
-	PhoneQuery         string         `json:"phone_query"`
-	SocialQuery        string         `json:"social_query"`
-	CryptoAddressQuery string         `json:"crypto_address_query"`
-	PrintBalance       bool           `json:"print_balance"`
-	CredsOnly          bool           `json:"creds_only"`
-	Debug              bool           `json:"debug"`
-}
-
-func (QueryOptions) TableName() string {
-	return "query_options"
-}
-
-func NewQueryOptions(maxRecords, maxRequests, startingPage int, outputFormat, outputFile, usernameQuery, emailQuery, ipQuery, passQuery, hashQuery, nameQuery, domainQuery, vinQuery, licensePlateQuery, addressQuery, phoneQuery, socialQuery, cryptoAddressQuery string, regexMatch, wildcardMatch, printBalance, credsOnly, debug bool) *QueryOptions {
-	return &QueryOptions{
-		MaxRecords:         maxRecords,
-		MaxRequests:        maxRequests,
-		StartingPage:       startingPage,
-		OutputFormat:       files.GetFileType(outputFormat),
-		OutputFile:         outputFile,
-		PrintBalance:       printBalance,
-		CredsOnly:          credsOnly,
-		RegexMatch:         regexMatch,
-		WildcardMatch:      wildcardMatch,
-		UsernameQuery:      usernameQuery,
-		EmailQuery:         emailQuery,
-		IpQuery:            ipQuery,
-		PassQuery:          passQuery,
-		HashQuery:          hashQuery,
-		NameQuery:          nameQuery,
-		DomainQuery:        domainQuery,
-		VinQuery:           vinQuery,
-		LicensePlateQuery:  licensePlateQuery,
-		AddressQuery:       addressQuery,
-		PhoneQuery:         phoneQuery,
-		SocialQuery:        socialQuery,
-		CryptoAddressQuery: cryptoAddressQuery,
-		Debug:              debug,
-	}
-}
-
-type Creds struct {
-	gorm.Model
-	Email    string `json:"email" yaml:"email" xml:"email" gorm:"uniqueIndex:idx_email_username_password"`
-	Username string `json:"username" yaml:"username" xml:"username" gorm:"uniqueIndex:idx_email_username_password"`
-	Password string `json:"password" yaml:"password" xml:"password" gorm:"uniqueIndex:idx_email_username_password"`
-}
-
-func (Creds) TableName() string {
-	return "creds"
-}
-
-func (c Creds) ToString() string {
-	return fmt.Sprintf("%s%s%s", c.Username, "%", c.Password)
-}

internal/sqlite/tables.go

@@ -1,68 +0,0 @@
-package sqlite
-
-import "strings"
-
-type Table int64
-
-const (
-	ResultsTable Table = iota
-	RunsTable
-	CredsTable
-	WhoIsTable
-	SubdomainsTable
-	HistoryTable
-	LookupTable
-	HunterDomainTable
-	HunterEmailTable
-	UnknownTable
-)
-
-func GetTable(userInput string) Table {
-	switch strings.ToLower(userInput) {
-	case "results":
-		return ResultsTable
-	case "runs":
-		return RunsTable
-	case "creds":
-		return CredsTable
-	case "whois":
-		return WhoIsTable
-	case "subdomains":
-		return SubdomainsTable
-	case "history":
-		return HistoryTable
-	case "lookup":
-		return LookupTable
-	case "hunter_domain":
-		return HunterDomainTable
-	case "hunter_email":
-		return HunterEmailTable
-	default:
-		return UnknownTable
-	}
-}
-
-func (t Table) Object() interface{} {
-	switch t {
-	case ResultsTable:
-		return Result{}
-	case RunsTable:
-		return QueryOptions{}
-	case CredsTable:
-		return Creds{}
-	case WhoIsTable:
-		return WhoisRecord{}
-	case SubdomainsTable:
-		return SubdomainRecord{}
-	case HistoryTable:
-		return HistoryRecord{}
-	case LookupTable:
-		return LookupResult{}
-	case HunterDomainTable:
-		return HunterDomainData{}
-	case HunterEmailTable:
-		return HunterEmail{}
-	default:
-		return nil
-	}
-}

internal/sqlite/whois.go

@@ -2,7 +2,9 @@ package sqlite
 
 import (
 	"fmt"
+	"go.uber.org/zap"
 	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
 	"strings"
 )
 
@@ -512,3 +514,117 @@ func formatWhoisContact(sb *strings.Builder, contact Contact, indent string) {
 		sb.WriteString(indent + "Raw Text: " + rawTextPreview + "\n")
 	}
 }
+
+func StoreWhoisRecord(whoisRecord WhoisRecord) error {
+	// Create a pointer to the record to make it addressable
+	recordPtr := &whoisRecord
+
+	zap.L().Info("Storing WHOIS record",
+		zap.String("domain", whoisRecord.DomainName))
+
+	db := GetDB()
+
+	// Use OnConflict clause to handle duplicates
+	err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(recordPtr).Error
+	if err != nil {
+		zap.L().Error("store_whois_record",
+			zap.String("message", "failed to store whois record"),
+			zap.Error(err))
+		return err
+	}
+
+	return nil
+}
+
+func StoreWhoisSubdomainRecords(subdomainRecords []SubdomainRecord) error {
+	if len(subdomainRecords) == 0 {
+		return nil
+	}
+
+	zap.L().Info("Storing subdomain records", zap.Int("count", len(subdomainRecords)))
+	db := GetDB()
+
+	// Use batch insert with conflict handling
+	const batchSize = 100
+	var lastErr error
+
+	for i := 0; i < len(subdomainRecords); i += batchSize {
+		end := i + batchSize
+		if end > len(subdomainRecords) {
+			end = len(subdomainRecords)
+		}
+
+		batch := subdomainRecords[i:end]
+		// Use Clauses with OnConflict DoNothing to skip conflicts
+		err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
+		if err != nil {
+			zap.L().Warn("Error storing some subdomain records", zap.Error(err))
+			lastErr = err
+			// Continue with next batch despite error
+		}
+	}
+
+	return lastErr
+}
+
+func StoreWhoisHistoryRecords(historyRecords []HistoryRecord) error {
+	if len(historyRecords) == 0 {
+		return nil
+	}
+
+	zap.L().Info("Storing history records", zap.Int("count", len(historyRecords)))
+	db := GetDB()
+
+	// Use batch insert with conflict handling
+	const batchSize = 100
+	var lastErr error
+
+	for i := 0; i < len(historyRecords); i += batchSize {
+		end := i + batchSize
+		if end > len(historyRecords) {
+			end = len(historyRecords)
+		}
+
+		batch := historyRecords[i:end]
+		// Use Clauses with OnConflict DoNothing to skip conflicts
+		err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
+		if err != nil {
+			zap.L().Warn("Error storing some history records", zap.Error(err))
+			lastErr = err
+			// Continue with next batch despite error
+		}
+	}
+
+	return lastErr
+}
+
+func StoreWhoisLookup(lookup []LookupResult) error {
+	if len(lookup) == 0 {
+		return nil
+	}
+
+	zap.L().Info("Storing IP lookup records", zap.Int("count", len(lookup)))
+	db := GetDB()
+
+	// Use batch insert with conflict handling
+	const batchSize = 100
+	var lastErr error
+
+	for i := 0; i < len(lookup); i += batchSize {
+		end := i + batchSize
+		if end > len(lookup) {
+			end = len(lookup)
+		}
+
+		batch := lookup[i:end]
+		// Use Clauses with OnConflict DoNothing to skip conflicts
+		err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
+		if err != nil {
+			zap.L().Warn("Error storing some IP lookup records", zap.Error(err))
+			lastErr = err
+			// Continue with next batch despite error
+		}
+	}
+
+	return lastErr
+}

internal/whois/whois.go

@@ -552,7 +552,7 @@ func (w *DehashedWhoIs) WhoisIP(ipAddress string) ([]sqlite.LookupResult, error)
 		})
 	}
 
-	sqlite.StoreIPLookup(lookups)
+	sqlite.StoreWhoisLookup(lookups)
 
 	return lookups, nil
 }
@@ -702,7 +702,7 @@ func (w *DehashedWhoIs) WhoisMX(mxHostname string) ([]sqlite.LookupResult, error
 		})
 	}
 
-	sqlite.StoreIPLookup(mxLookups)
+	sqlite.StoreWhoisLookup(mxLookups)
 
 	return mxLookups, nil
 }
@@ -849,7 +849,7 @@ func (w *DehashedWhoIs) WhoisNS(nsHostname string) ([]sqlite.LookupResult, error
 		})
 	}
 
-	sqlite.StoreIPLookup(nsLookups)
+	sqlite.StoreWhoisLookup(nsLookups)
 
 	return nsLookups, nil
 }