Add mutually exclusive flags to targets command

Refactor user and credential handling: rename Creds to User, update d…

Makefile

@@ -1,4 +1,4 @@
-# Makefile for Dehasher
+# Makefile for CrowsNest
 
 # Go command
 GO=go
@@ -16,7 +16,7 @@ PLATFORMS=linux darwin windows
 ARCHS=amd64 arm64
 
 # Version info from git tag or default
-VERSION=$(shell git describe --tags 2>/dev/null || echo "v1.3.1")
+VERSION=$(shell git describe --tags 2>/dev/null || echo "v1.3.3")
 
 .PHONY: all clean build build-all
 

README.md

@@ -39,7 +39,7 @@ go build crowsnest.go
 
 CrowsNest supports two database storage options:
 
-1. **Default Path** (default): Stores the database at `~/.local/share/crowsnest/db/dehashed.sqlite`
+1. **Default Path** (default): Stores the database at `~/.local/share/crowsnest/db/crowsnest.sqlite`
 2. **Local Path**: Stores the database in the current directory as `./crowsnest.sqlite`
 
 The **Local Path** option allows for separate databases for different projects or engagements.

cmd/dehashed.go

@@ -77,7 +77,7 @@ var (
 
 			// Validate credentials
 			if key == "" {
-				fmt.Println("API key is required. Set the key with the \"set-key\" command. [dehasher set-key <api_key>]")
+				fmt.Println("API key is required. Set the key with the \"set-key\" command. [crowsnest set-key <api_key>]")
 				return
 			}
 
@@ -118,6 +118,7 @@ var (
 			dehasher.Start()
 			fmt.Println("\n[*] Completing Process")
 
+			// Store query options
 			err := sqlite.StoreDehashedQueryOptions(queryOptions)
 			if err != nil {
 				if debugGlobal {

cmd/export.go

@@ -1,311 +0,0 @@
-package cmd
-
-import (
-	"crowsnest/internal/export"
-	"crowsnest/internal/files"
-	"crowsnest/internal/sqlite"
-	"fmt"
-	"github.com/spf13/cobra"
-	"go.uber.org/zap"
-	"strings"
-)
-
-func init() {
-	// Add Subcommand to db command
-	rootCmd.AddCommand(exportCmd)
-
-	// Add flags specific to export command
-	exportCmd.Flags().IntVarP(&exportLimitRows, "limit", "l", 100, "Limit number of results")
-	exportCmd.Flags().BoolVarP(&exportListAll, "list-all", "a", false, "List all tables and their columns")
-	exportCmd.Flags().StringVarP(&exportTableName, "table", "t", "", "Table to export (results, creds, whois, subdomains, history, runs)")
-	exportCmd.Flags().StringVarP(&exportNotNull, "not-null", "n", "", "Filter for non-null values (comma-separated list, e.g., 'password,email')")
-	exportCmd.Flags().StringVarP(&exportColumns, "columns", "c", "", "Columns to display in output (comma-separated list, e.g., 'username,email,password')")
-	exportCmd.Flags().StringVarP(&exportUserQuery, "user-query", "q", "", "User query to execute")
-	exportCmd.Flags().StringVarP(&exportRawQuery, "raw-query", "r", "", "Raw SQL query to execute")
-	exportCmd.Flags().StringVarP(&exportFormat, "format", "f", "json", "Output format (json, yaml, xml, txt)")
-	exportCmd.Flags().StringVarP(&exportFile, "file", "o", "export", "File to output results to including extension")
-
-	// Add mutually exclusive flags to query and raw-query
-	// Cannot use query and raw-query at the same time
-	exportCmd.MarkFlagsMutuallyExclusive("user-query", "raw-query")
-	// Raw query does not require a table
-	exportCmd.MarkFlagsMutuallyExclusive("user-query", "table")
-	// List all columns does not require a query or raw-query
-	exportCmd.MarkFlagsMutuallyExclusive("raw-query", "list-all")
-}
-
-// DB export command
-var (
-	exportLimitRows int
-	exportListAll   bool
-	exportTableName string
-	exportNotNull   string
-	exportColumns   string
-	exportUserQuery string
-	exportRawQuery  string
-	exportFormat    string
-	exportFile      string
-
-	exportCmd = &cobra.Command{
-		Use:   "export",
-		Short: "Export database to file",
-		Run: func(cmd *cobra.Command, args []string) {
-			// If list-all flag is set, list all tables and columns
-			if exportListAll {
-				listAvailableTables()
-				return
-			}
-
-			fmt.Println("[*] Exporting database...")
-
-			// If Raw Query is set, execute it and export
-			if exportRawQuery != "" {
-				fmt.Println("[*] Executing Raw Query...")
-				exportRawDBQuery()
-				return
-			}
-
-			// Validate table name
-			if exportTableName == "" {
-				fmt.Println("[!] Error: Table name is required. Use -t or --table to specify a table.")
-				fmt.Println("[*] Available tables: results, creds, whois, subdomains, history, runs")
-				fmt.Println("[*] Use --list-all to see all tables and their columns.")
-				return
-			}
-
-			if !isValidTable(exportTableName) {
-				fmt.Printf("[!] Error: Unknown table '%s'.\n", exportTableName)
-				fmt.Println("[*] Available tables: results, creds, whois, subdomains, history, runs")
-				fmt.Println("[*] Use --list-all to see all tables and their columns.")
-				return
-			}
-
-			// Validate columns if specified
-			if exportColumns != "" {
-				columns := strings.Split(exportColumns, ",")
-				invalidColumns := validateColumns(exportTableName, columns)
-				if len(invalidColumns) > 0 {
-					fmt.Printf("[!] Error: Invalid column(s) for table '%s': %s\n",
-						exportTableName, strings.Join(invalidColumns, ", "))
-					fmt.Println("[*] Available columns for this table:")
-					for i := 0; i < len(availableTables[exportTableName]); i += 5 {
-						end := i + 5
-						if end > len(availableTables[exportTableName]) {
-							end = len(availableTables[exportTableName])
-						}
-						fmt.Printf("    %s\n", strings.Join(availableTables[exportTableName][i:end], ", "))
-					}
-					return
-				}
-			}
-
-			// Validate not-null fields if specified
-			if exportNotNull != "" {
-				notNullFields := strings.Split(exportNotNull, ",")
-				invalidFields := validateColumns(exportTableName, notNullFields)
-				if len(invalidFields) > 0 {
-					fmt.Printf("[!] Error: Invalid not-null field(s) for table '%s': %s\n",
-						exportTableName, strings.Join(invalidFields, ", "))
-					fmt.Println("[*] Available columns for this table:")
-					for i := 0; i < len(availableTables[exportTableName]); i += 5 {
-						end := i + 5
-						if end > len(availableTables[exportTableName]) {
-							end = len(availableTables[exportTableName])
-						}
-						fmt.Printf("    %s\n", strings.Join(availableTables[exportTableName][i:end], ", "))
-					}
-					return
-				}
-			}
-
-			// Determine which table to query based on the tableTypeDBQuery parameter
-			table := sqlite.GetTable(exportTableName)
-			if table == sqlite.UnknownTable {
-				fmt.Printf("[!] Error: Unknown table type '%s'.\n", exportTableName)
-				fmt.Println("[*] Available tables: results, creds, whois, subdomains, history, runs")
-				fmt.Println("[*] Use --list-all to see all tables and their columns.")
-				return
-			}
-
-			fmt.Println("[*] Querying Database...")
-			exportTableQuery(table)
-		},
-	}
-)
-
-// exportTableQuery queries a table and exports the results
-func exportTableQuery(table sqlite.Table) {
-	// Get the columns to query
-	columns := []string{"*"}
-	if exportColumns != "" {
-		columns = strings.Split(exportColumns, ",")
-	}
-
-	// Get the not null fields
-	notNullFields := []string{}
-	if exportNotNull != "" {
-		notNullFields = strings.Split(exportNotNull, ",")
-	}
-
-	// Get the user query
-	userQuery := ""
-	if exportUserQuery != "" {
-		userQuery = exportUserQuery
-	}
-
-	// Get the limit
-	limit := exportLimitRows
-
-	// Get the object for the table
-	object := table.Object()
-
-	// Check if object is nil (invalid table)
-	if object == nil {
-		fmt.Printf("[!] Error: Table '%s' is not valid or does not exist.\n", exportTableName)
-		return
-	}
-
-	// Query the database
-	db := sqlite.GetDB()
-	query := db.Model(object).Select(columns)
-	if len(notNullFields) > 0 {
-		for _, field := range notNullFields {
-			query = query.Where(fmt.Sprintf("%s IS NOT NULL", field))
-		}
-	}
-	if userQuery != "" {
-		query = query.Where(userQuery)
-	}
-	if limit > 0 {
-		query = query.Limit(limit)
-	}
-	rows, err := query.Rows()
-	if err != nil {
-		zap.L().Error("export_query",
-			zap.String("message", "failed to execute query"),
-			zap.Error(err),
-		)
-		fmt.Printf("[!] Error executing query: %v\n", err)
-		return
-	}
-	defer rows.Close()
-
-	// Get the columns
-	cols, err := rows.Columns()
-	if err != nil {
-		zap.L().Error("export_query",
-			zap.String("message", "failed to get columns from query"),
-			zap.Error(err),
-		)
-		fmt.Printf("[!] Error getting columns from query: %v\n", err)
-		return
-	}
-
-	// Prepare data for export
-	var results []map[string]interface{}
-
-	// Process the rows
-	for rows.Next() {
-		values := make([]interface{}, len(cols))
-		pointers := make([]interface{}, len(cols))
-		for i := range values {
-			pointers[i] = &values[i]
-		}
-		if err := rows.Scan(pointers...); err != nil {
-			zap.L().Error("export_query",
-				zap.String("message", "failed to scan row from query"),
-				zap.Error(err),
-			)
-			fmt.Printf("[!] Error scanning row from query: %v\n", err)
-			return
-		}
-
-		// Create a map for this row
-		rowMap := make(map[string]interface{})
-		for i, col := range cols {
-			val := values[i]
-			rowMap[col] = val
-		}
-
-		results = append(results, rowMap)
-	}
-
-	// Export the results
-	exportResults(results)
-}
-
-// exportRawDBQuery executes a raw query and exports the results
-func exportRawDBQuery() {
-	db := sqlite.GetDB()
-	rows, err := db.Raw(exportRawQuery).Rows()
-	if err != nil {
-		zap.L().Error("export_raw_query",
-			zap.String("message", "failed to execute raw query"),
-			zap.Error(err),
-		)
-		fmt.Printf("[!] Error executing raw query: %v\n", err)
-		return
-	}
-	defer rows.Close()
-
-	columns, err := rows.Columns()
-	if err != nil {
-		zap.L().Error("export_raw_query",
-			zap.String("message", "failed to get columns from raw query"),
-			zap.Error(err),
-		)
-		fmt.Printf("[!] Error getting columns from raw query: %v\n", err)
-		return
-	}
-
-	// Prepare data for export
-	var results []map[string]interface{}
-
-	// Process the rows
-	for rows.Next() {
-		values := make([]interface{}, len(columns))
-		pointers := make([]interface{}, len(columns))
-		for i := range values {
-			pointers[i] = &values[i]
-		}
-		if err := rows.Scan(pointers...); err != nil {
-			zap.L().Error("export_raw_query",
-				zap.String("message", "failed to scan row from raw query"),
-				zap.Error(err),
-			)
-			fmt.Printf("[!] Error scanning row from raw query: %v\n", err)
-			return
-		}
-
-		// Create a map for this row
-		rowMap := make(map[string]interface{})
-		for i, col := range columns {
-			val := values[i]
-			rowMap[col] = val
-		}
-
-		results = append(results, rowMap)
-	}
-
-	// Export the results
-	exportResults(results)
-}
-
-// exportResults exports the results to a file
-func exportResults(results []map[string]interface{}) {
-	// Get file type
-	fileType := files.GetFileType(exportFormat)
-
-	// Export results
-	err := export.WriteQueryResultsToFile(results, exportFile, fileType)
-	if err != nil {
-		zap.L().Error("export_results",
-			zap.String("message", "failed to write to file"),
-			zap.Error(err),
-		)
-		fmt.Printf("[!] Error writing to file: %v\n", err)
-		return
-	}
-
-	fmt.Printf("[+] Exported %d records to file: %s%s\n", len(results), exportFile, fileType.Extension())
-}

cmd/hunter.go

@@ -7,6 +7,7 @@ import (
 	"crowsnest/internal/files"
 	hunter "crowsnest/internal/hunter.io"
 	"crowsnest/internal/pretty"
+	"crowsnest/internal/sqlite"
 	"fmt"
 	"github.com/spf13/cobra"
 	"go.uber.org/zap"
@@ -111,6 +112,24 @@ var (
 					return
 				}
 
+				// Store the users discovered
+				var creds []sqlite.User
+				for _, email := range result.Emails {
+					creds = append(creds, sqlite.User{Email: email.Value})
+				}
+				err = sqlite.StoreUsers(creds)
+				if err != nil {
+					if debugGlobal {
+						debug.PrintInfo("failed to store hunter domain search")
+						debug.PrintError(err)
+					}
+					zap.L().Error("store_hunter_domain_search",
+						zap.String("message", "failed to store hunter domain search"),
+						zap.Error(err),
+					)
+					fmt.Printf("Error storing Hunter.io Domain Search Result: %v\n", err)
+				}
+
 				// Write Hunter.io Domain Search Result to file
 				fmt.Printf("[*] Writing Hunter.io Domain Search Result to file: %s%s\n", hunterOutputFile, fType.Extension())
 				err = export.WriteIStringToFile(result, hunterOutputFile, fType)
@@ -220,24 +239,17 @@ var (
 
 				// Pretty Print Hunter.io Email Verification Result
 				var (
-					headers = []string{"Email", "Status", "Result", "Score", "Regexp", "Gibberish", "Disposable", "Webmail", "MX Records", "SMTP Server", "SMTP Check", "Accept All", "Block", "Sources"}
+					headers = []string{"Email", "Result", "Score", "Disposable", "MX Records", "SMTP Server", "SMTP Check"}
 					rows    [][]string
 				)
 				rows = append(rows, []string{
 					result.Email,
-					result.Status,
 					result.Result,
 					fmt.Sprintf("%d", result.Score),
-					fmt.Sprintf("%t", result.Regexp),
-					fmt.Sprintf("%t", result.Gibberish),
 					fmt.Sprintf("%t", result.Disposable),
-					fmt.Sprintf("%t", result.Webmail),
 					fmt.Sprintf("%t", result.MXRecords),
 					fmt.Sprintf("%t", result.SMTPServer),
 					fmt.Sprintf("%t", result.SMTPCheck),
-					fmt.Sprintf("%t", result.AcceptAll),
-					fmt.Sprintf("%t", result.Block),
-					fmt.Sprintf("%v", result.Sources),
 				})
 
 				fmt.Println("Email Verification Result:")

cmd/query.go

@@ -1,6 +1,9 @@
 package cmd
 
 import (
+	"crowsnest/internal/debug"
+	"crowsnest/internal/export"
+	"crowsnest/internal/files"
 	"crowsnest/internal/pretty"
 	"crowsnest/internal/sqlite"
 	"encoding/json"
@@ -10,132 +13,6 @@ import (
 	"strings"
 )
 
-// Map of available tables and their columns
-var availableTables = map[string][]string{
-	"creds": {
-		"id", "created_at", "updated_at", "deleted_at", "email", "username", "password",
-	},
-	//"history": {
-	//	"id", "created_at", "updated_at", "deleted_at", "domain_name", "domain_type",
-	//	"registrar_name", "whois_server", "created_date_iso8601", "updated_date_iso8601", "expires_date_iso8601",
-	//},
-	"lookup": {
-		"id", "created_at", "updated_at", "deleted_at", "search_term", "type", "first_seen", "last_visit",
-		"name",
-	},
-	// Query Options
-	"runs": {
-		"id", "created_at", "updated_at", "deleted_at", "max_records", "max_requests", "starting_page",
-		"output_format", "output_file", "regex_match", "wildcard_match", "username_query", "email_query",
-		"ip_query", "pass_query", "hash_query", "name_query", "domain_query", "vin_query", "license_plate_query",
-		"address_query", "phone_query", "social_query", "crypto_address_query", "print_balance", "creds_only",
-	},
-	"results": {
-		"id", "created_at", "updated_at", "deleted_at", "dehashed_id", "email", "ip_address", "username",
-		"password", "hashed_password", "hash_type", "name", "vin", "license_plate", "url", "social",
-		"cryptocurrency_address", "address", "phone", "company", "database_name",
-	},
-	"subdomains": {
-		"id", "created_at", "updated_at", "deleted_at", "domain", "first_seen", "last_seen",
-	},
-	"whois": {
-		"id", "created_at", "updated_at", "deleted_at", "audit", "contact_email", "created_date", "created_date_normalized",
-		"domain_name", "domain_name_ext", "estimated_domain_age", "expires_date", "expires_date_normalized", "footer", "header",
-		"name_servers", "parse_code", "raw_text", "registrant", "registrar_iana_id", "registrar_name", "registry_data",
-		"status", "stripped_text", "updated_date", "updated_date_normalized",
-	},
-	"hunter_domain": {
-		"id", "created_at", "updated_at", "deleted_at", "domain", "disposable", "webmail", "accept_all", "pattern",
-		"organization", "description", "industry", "twitter", "facebook", "linkedin", "instagram", "youtube",
-		"technologies", "country", "state", "city", "postal_code", "street", "headcount", "company_type", "emails", "linked_domains",
-	},
-	"hunter_email": {
-		"id", "created_at", "updated_at", "deleted_at", "value", "type", "confidence", "sources", "first_name", "last_name",
-		"position", "position_raw", "seniority", "department", "linkedin", "twitter", "phone_number", "verification_date", "verification_status",
-	},
-}
-
-// Function to list available tables and their columns
-func listAvailableTables() {
-	fmt.Println("Available tables and columns:")
-
-	// Prepare data for pretty.Table
-	headers := []string{"Table", "Columns"}
-	var tableRows [][]string
-
-	// Sort tables alphabetically for consistent output
-	var tableNames []string
-	for tableName := range availableTables {
-		tableNames = append(tableNames, tableName)
-	}
-
-	// Simple bubble sort for table names
-	for i := 0; i < len(tableNames)-1; i++ {
-		for j := 0; j < len(tableNames)-i-1; j++ {
-			if tableNames[j] > tableNames[j+1] {
-				tableNames[j], tableNames[j+1] = tableNames[j+1], tableNames[j]
-			}
-		}
-	}
-
-	// Create rows for the table
-	for _, tableName := range tableNames {
-		columns := availableTables[tableName]
-
-		// Format columns with line breaks for better readability
-		var formattedColumns string
-		for i := 0; i < len(columns); i += 5 {
-			end := i + 5
-			if end > len(columns) {
-				end = len(columns)
-			}
-			if i > 0 {
-				formattedColumns += "\n"
-			}
-			formattedColumns += strings.Join(columns[i:end], ", ")
-		}
-
-		tableRows = append(tableRows, []string{tableName, formattedColumns})
-	}
-
-	// Display the table
-	pretty.Table(headers, tableRows)
-}
-
-// Function to validate table name
-func isValidTable(tableName string) bool {
-	_, exists := availableTables[tableName]
-	return exists
-}
-
-// Function to validate column names for a specific table
-func validateColumns(tableName string, columns []string) []string {
-	if tableName == "" || columns == nil || len(columns) == 0 || columns[0] == "*" {
-		return nil
-	}
-
-	tableColumns, exists := availableTables[tableName]
-	if !exists {
-		return []string{fmt.Sprintf("Table '%s' does not exist", tableName)}
-	}
-
-	var invalidColumns []string
-	for _, col := range columns {
-		valid := false
-		for _, tableCol := range tableColumns {
-			if col == tableCol {
-				valid = true
-				break
-			}
-		}
-		if !valid {
-			invalidColumns = append(invalidColumns, col)
-		}
-	}
-
-	return invalidColumns
-}
-
 func init() {
 	// Add whois command to root command
 	rootCmd.AddCommand(queryCmd)
@@ -148,6 +25,8 @@ func init() {
 	queryCmd.Flags().StringVarP(&dbQueryUserQuery, "user-query", "q", "", "User query to execute")
 	queryCmd.Flags().StringVarP(&dbQueryRawQuery, "raw-query", "r", "", "Raw SQL query to execute")
 	queryCmd.Flags().BoolVarP(&dbQueryListAll, "list-all", "a", false, "List all tables and their columns")
+	queryCmd.Flags().StringVarP(&dbQueryFormat, "format", "f", "json", "Output format (json, yaml, xml, txt)")
+	queryCmd.Flags().StringVarP(&dbQueryFile, "file", "o", "query", "File to output results to")
 
 	// Add mutually exclusive flags to query and raw-query
 	// Cannot use query and raw-query at the same time
@@ -166,11 +45,14 @@ var (
 	dbQueryUserQuery string
 	dbQueryRawQuery  string
 	dbQueryListAll   bool
+	dbQueryFormat    string
+	dbQueryFile      string
 
 	queryCmd = &cobra.Command{
 		Use:   "query",
 		Short: "Query the database",
-		Long:  `Query the database for various information.`,
+		Long: `Query the database for various information.
+If file is specified, results are written to file and not displayed in the terminal.`,
 		Run: func(cmd *cobra.Command, args []string) {
 			// If list-all flag is set, list all tables and columns
 			if dbQueryListAll {
@@ -321,6 +203,49 @@ func tableQuery(table sqlite.Table) {
 		return
 	}
 
+	// Export results if file name is specified
+	if len(strings.TrimSpace(dbQueryFile)) > 0 {
+		fmt.Println("[*] Exporting results to file...")
+
+		if debugGlobal {
+			debug.PrintInfo("exporting results to file: " + dbQueryFile)
+		}
+		// Prepare data for export
+		var results []map[string]interface{}
+
+		// Process the rows
+		for rows.Next() {
+			values := make([]interface{}, len(cols))
+			pointers := make([]interface{}, len(cols))
+			for i := range values {
+				pointers[i] = &values[i]
+			}
+			if err := rows.Scan(pointers...); err != nil {
+				zap.L().Error("export_query",
+					zap.String("message", "failed to scan row from query"),
+					zap.Error(err),
+				)
+				fmt.Printf("[!] Error scanning row from query: %v\n", err)
+				return
+			}
+
+			// Create a map for this row
+			rowMap := make(map[string]interface{})
+			for i, col := range cols {
+				val := values[i]
+				rowMap[col] = val
+			}
+
+			results = append(results, rowMap)
+		}
+
+		// Export the results
+		exportQueryResults(results)
+
+		return
+	}
+	fmt.Println("[*] Querying Database...")
+
 	// Prepare data for pretty.Table
 	headers := cols
 	var tableRows [][]string
@@ -411,6 +336,49 @@ func rawDBQuery() {
 		return
 	}
 
+	if len(strings.TrimSpace(dbQueryFile)) > 0 {
+		fmt.Println("[*] Exporting results to file...")
+
+		if debugGlobal {
+			debug.PrintInfo("exporting results to file: " + dbQueryFile)
+		}
+
+		// Prepare data for export
+		var results []map[string]interface{}
+
+		// Process the rows
+		for rows.Next() {
+			values := make([]interface{}, len(columns))
+			pointers := make([]interface{}, len(columns))
+			for i := range values {
+				pointers[i] = &values[i]
+			}
+			if err := rows.Scan(pointers...); err != nil {
+				zap.L().Error("export_raw_query",
+					zap.String("message", "failed to scan row from raw query"),
+					zap.Error(err),
+				)
+				fmt.Printf("[!] Error scanning row from raw query: %v\n", err)
+				return
+			}
+
+			// Create a map for this row
+			rowMap := make(map[string]interface{})
+			for i, col := range columns {
+				val := values[i]
+				rowMap[col] = val
+			}
+
+			results = append(results, rowMap)
+		}
+
+		// Export the results
+		exportQueryResults(results)
+
+		return
+	}
+	fmt.Println("[*] Querying Database...")
+
 	// Prepare data for pretty.Table
 	headers := columns
 	var tableRows [][]string
@@ -477,3 +445,22 @@ func rawDBQuery() {
 	// Display the table
 	pretty.Table(headers, tableRows)
 }
+
+// exportQueryResults exports the results to a file
+func exportQueryResults(results []map[string]interface{}) {
+	// Get file type
+	fileType := files.GetFileType(dbQueryFormat)
+
+	// Export results
+	err := export.WriteQueryResultsToFile(results, dbQueryFile, fileType)
+	if err != nil {
+		zap.L().Error("export_results",
+			zap.String("message", "failed to write to file"),
+			zap.Error(err),
+		)
+		fmt.Printf("[!] Error writing to file: %v\n", err)
+		return
+	}
+
+	fmt.Printf("[+] Exported %d records to file: %s%s\n", len(results), dbQueryFile, fileType.Extension())
+}

cmd/root.go

@@ -16,8 +16,8 @@ var (
 
 	// rootCmd is the base command for the CLI.
 	rootCmd = &cobra.Command{
-		Use:   "dehasher",
-		Short: `Dehasher is a cli tool for querying the dehashed api.`,
+		Use:   "crowsnest",
+		Short: `CrowsNest is a cli tool for querying the common OSINT api's.`,
 		Long: fmt.Sprintf(
 			"%s\n",
 			`
@@ -94,7 +94,7 @@ var setHunterKeyCmd = &cobra.Command{
 
 var setLocalDb = &cobra.Command{
 	Use:   "local-db [true|false]",
-	Short: "Set dehasher to use a local database path instead of the default path",
+	Short: "Set crowsnest to use a local database path instead of the default path",
 	Args:  cobra.ExactArgs(1),
 	Run: func(cmd *cobra.Command, args []string) {
 		var useLocalDatabase bool

cmd/tables.go

@@ -0,0 +1,133 @@
+package cmd
+
+import (
+	"crowsnest/internal/pretty"
+	"fmt"
+	"strings"
+)
+
+// Map of available tables and their columns
+var availableTables = map[string][]string{
+	"users": {
+		"id", "created_at", "updated_at", "deleted_at", "email", "username", "password",
+	},
+	//"history": {
+	//	"id", "created_at", "updated_at", "deleted_at", "domain_name", "domain_type",
+	//	"registrar_name", "whois_server", "created_date_iso8601", "updated_date_iso8601", "expires_date_iso8601",
+	//},
+	"lookup": {
+		"id", "created_at", "updated_at", "deleted_at", "search_term", "type", "first_seen", "last_visit",
+		"name",
+	},
+	// Query Options
+	"runs": {
+		"id", "created_at", "updated_at", "deleted_at", "max_records", "max_requests", "starting_page",
+		"output_format", "output_file", "regex_match", "wildcard_match", "username_query", "email_query",
+		"ip_query", "pass_query", "hash_query", "name_query", "domain_query", "vin_query", "license_plate_query",
+		"address_query", "phone_query", "social_query", "crypto_address_query", "print_balance", "creds_only",
+	},
+	"dehashed": {
+		"id", "created_at", "updated_at", "deleted_at", "dehashed_id", "email", "ip_address", "username",
+		"password", "hashed_password", "hash_type", "name", "vin", "license_plate", "url", "social",
+		"cryptocurrency_address", "address", "phone", "company", "database_name",
+	},
+	"subdomains": {
+		"id", "created_at", "updated_at", "deleted_at", "domain", "subdomain",
+	},
+	"whois": {
+		"id", "created_at", "updated_at", "deleted_at", "audit", "contact_email", "created_date", "created_date_normalized",
+		"domain_name", "domain_name_ext", "estimated_domain_age", "expires_date", "expires_date_normalized", "footer", "header",
+		"name_servers", "parse_code", "raw_text", "registrant", "registrar_iana_id", "registrar_name", "registry_data",
+		"status", "stripped_text", "updated_date", "updated_date_normalized",
+	},
+	"hunter_domain": {
+		"id", "created_at", "updated_at", "deleted_at", "domain", "disposable", "webmail", "accept_all", "pattern",
+		"organization", "description", "industry", "twitter", "facebook", "linkedin", "instagram", "youtube",
+		"technologies", "country", "state", "city", "postal_code", "street", "headcount", "company_type", "emails", "linked_domains",
+	},
+	"hunter_email": {
+		"id", "created_at", "updated_at", "deleted_at", "value", "type", "confidence", "sources", "first_name", "last_name",
+		"position", "position_raw", "seniority", "department", "linkedin", "twitter", "phone_number", "verification_date", "verification_status",
+	},
+}
+
+// Function to list available tables and their columns
+func listAvailableTables() {
+	fmt.Println("Available tables and columns:")
+
+	// Prepare data for pretty.Table
+	headers := []string{"Table", "Columns"}
+	var tableRows [][]string
+
+	// Sort tables alphabetically for consistent output
+	var tableNames []string
+	for tableName := range availableTables {
+		tableNames = append(tableNames, tableName)
+	}
+
+	// Simple bubble sort for table names
+	for i := 0; i < len(tableNames)-1; i++ {
+		for j := 0; j < len(tableNames)-i-1; j++ {
+			if tableNames[j] > tableNames[j+1] {
+				tableNames[j], tableNames[j+1] = tableNames[j+1], tableNames[j]
+			}
+		}
+	}
+
+	// Create rows for the table
+	for _, tableName := range tableNames {
+		columns := availableTables[tableName]
+
+		// Format columns with line breaks for better readability
+		var formattedColumns string
+		for i := 0; i < len(columns); i += 5 {
+			end := i + 5
+			if end > len(columns) {
+				end = len(columns)
+			}
+			if i > 0 {
+				formattedColumns += "\n"
+			}
+			formattedColumns += strings.Join(columns[i:end], ", ")
+		}
+
+		tableRows = append(tableRows, []string{tableName, formattedColumns})
+	}
+
+	// Display the table
+	pretty.Table(headers, tableRows)
+}
+
+// Function to validate table name
+func isValidTable(tableName string) bool {
+	_, exists := availableTables[tableName]
+	return exists
+}
+
+// Function to validate column names for a specific table
+func validateColumns(tableName string, columns []string) []string {
+	if tableName == "" || columns == nil || len(columns) == 0 || columns[0] == "*" {
+		return nil
+	}
+
+	tableColumns, exists := availableTables[tableName]
+	if !exists {
+		return []string{fmt.Sprintf("Table '%s' does not exist", tableName)}
+	}
+
+	var invalidColumns []string
+	for _, col := range columns {
+		valid := false
+		for _, tableCol := range tableColumns {
+			if col == tableCol {
+				valid = true
+				break
+			}
+		}
+		if !valid {
+			invalidColumns = append(invalidColumns, col)
+		}
+	}
+
+	return invalidColumns
+}

cmd/targets.go

@@ -0,0 +1,312 @@
+package cmd
+
+import (
+	"crowsnest/internal/sqlite"
+	"fmt"
+	"github.com/spf13/cobra"
+	"go.uber.org/zap"
+	"os"
+	"strings"
+)
+
+func init() {
+	// Add targets command to root command
+	rootCmd.AddCommand(targetsCmd)
+
+	// Add flags specific to targets command
+	targetsCmd.Flags().StringVarP(&targetsOutputFile, "output", "o", "targets", "Output file name (required)")
+	targetsCmd.Flags().BoolVarP(&targetsExternal, "external", "e", false, "Output external format (email:password)")
+	targetsCmd.Flags().BoolVarP(&targetsInternal, "internal", "i", false, "Output internal format (username:password)")
+	targetsCmd.Flags().BoolVarP(&targetsSubdomains, "subdomains", "s", false, "Output subdomains")
+	targetsCmd.Flags().BoolVarP(&targetsEmails, "emails", "E", false, "Output emails only (no passwords)")
+	targetsCmd.Flags().StringVarP(&targetsDomain, "domain", "d", "", "Filter by domain (for emails and subdomains)")
+
+	// Add mutually exclusive flags to targets command
+	targetsCmd.MarkFlagsMutuallyExclusive("external", "internal", "subdomains", "emails")
+}
+
+var (
+	// Targets command flags
+	targetsOutputFile string
+	targetsExternal   bool
+	targetsInternal   bool
+	targetsSubdomains bool
+	targetsEmails     bool
+	targetsDomain     string
+
+	// Targets command
+	targetsCmd = &cobra.Command{
+		Use:   "targets",
+		Short: "Export users and subdomains in formats suitable for external tools",
+		Long: `Export users and subdomains from the database in easily digestible formats for tools like sprays or other security testing tools.
+
+Formats:
+  --external (-e): Output in email:password format
+  --internal (-i): Output in username:password format
+  --emails (-E): Output emails only (no passwords)
+  --subdomains (-s): Output subdomains only
+
+Options:
+  --domain (-d): Filter results by domain (applies to emails and subdomains)
+  --output (-o): Specify output file name (required)
+
+Examples:
+  # Export all external credentials (email:password)
+  crowsnest targets -e -o external_creds
+
+  # Export internal credentials for a specific domain
+  crowsnest targets -i -d example.com -o internal_creds
+
+  # Export all emails
+  crowsnest targets -E -o all_emails
+
+  # Export emails for a specific domain
+  crowsnest targets -E -d example.com -o domain_emails
+
+  # Export subdomains for a specific domain
+  crowsnest targets -s -d example.com -o subdomains
+
+  # Export all subdomains
+  crowsnest targets -s -o all_subdomains`,
+		Run: func(cmd *cobra.Command, args []string) {
+			// Validate that at least one format is specified
+			if !targetsExternal && !targetsInternal && !targetsSubdomains && !targetsEmails {
+				fmt.Println("[!] Error: You must specify at least one output format:")
+				fmt.Println("    --external (-e) for email:password format")
+				fmt.Println("    --internal (-i) for username:password format")
+				fmt.Println("    --emails (-E) for emails only")
+				fmt.Println("    --subdomains (-s) for subdomains")
+				return
+			}
+
+			if debugGlobal {
+				zap.L().Info("targets_debug",
+					zap.String("message", "targets command started"),
+					zap.Bool("external", targetsExternal),
+					zap.Bool("internal", targetsInternal),
+					zap.Bool("subdomains", targetsSubdomains),
+					zap.Bool("emails", targetsEmails),
+					zap.String("domain", targetsDomain),
+					zap.String("output_file", targetsOutputFile),
+				)
+			}
+
+			// Execute the targets export
+			err := executeTargetsExport()
+			if err != nil {
+				fmt.Printf("[!] Error: %v\n", err)
+				return
+			}
+
+			fmt.Printf("[+] Successfully exported targets to: %s\n", targetsOutputFile)
+		},
+	}
+)
+
+// executeTargetsExport performs the main logic for exporting targets
+func executeTargetsExport() error {
+	var outputLines []string
+
+	// Export external credentials (email:password)
+	if targetsExternal {
+		if debugGlobal {
+			fmt.Println("[*] Exporting external credentials (email:password)...")
+		}
+
+		externalCreds, err := getExternalCredentials()
+		if err != nil {
+			return fmt.Errorf("failed to get external credentials: %v", err)
+		}
+
+		for _, cred := range externalCreds {
+			if cred.Email != "" && cred.Password != "" {
+				outputLines = append(outputLines, fmt.Sprintf("%s:%s", cred.Email, cred.Password))
+			}
+		}
+
+		if debugGlobal {
+			fmt.Printf("[*] Found %d external credentials\n", len(externalCreds))
+		}
+	}
+
+	// Export internal credentials (username:password)
+	if targetsInternal {
+		if debugGlobal {
+			fmt.Println("[*] Exporting internal credentials (username:password)...")
+		}
+
+		internalCreds, err := getInternalCredentials()
+		if err != nil {
+			return fmt.Errorf("failed to get internal credentials: %v", err)
+		}
+
+		for _, cred := range internalCreds {
+			if cred.Username != "" && cred.Password != "" {
+				outputLines = append(outputLines, fmt.Sprintf("%s:%s", cred.Username, cred.Password))
+			}
+		}
+
+		if debugGlobal {
+			fmt.Printf("[*] Found %d internal credentials\n", len(internalCreds))
+		}
+	}
+
+	// Export emails only
+	if targetsEmails {
+		if debugGlobal {
+			fmt.Println("[*] Exporting emails only...")
+		}
+
+		emails, err := getEmailsOnly()
+		if err != nil {
+			return fmt.Errorf("failed to get emails: %v", err)
+		}
+
+		for _, email := range emails {
+			if email.Email != "" {
+				outputLines = append(outputLines, email.Email)
+			}
+		}
+
+		if debugGlobal {
+			fmt.Printf("[*] Found %d emails\n", len(emails))
+		}
+	}
+
+	// Export subdomains
+	if targetsSubdomains {
+		if debugGlobal {
+			fmt.Println("[*] Exporting subdomains...")
+		}
+
+		subdomains, err := getSubdomains()
+		if err != nil {
+			return fmt.Errorf("failed to get subdomains: %v", err)
+		}
+
+		for _, subdomain := range subdomains {
+			if subdomain.Subdomain != "" {
+				outputLines = append(outputLines, subdomain.Subdomain)
+			}
+		}
+
+		if debugGlobal {
+			fmt.Printf("[*] Found %d subdomains\n", len(subdomains))
+		}
+	}
+
+	// Write to file
+	if len(outputLines) == 0 {
+		return fmt.Errorf("no data found to export")
+	}
+
+	// Join all lines with newlines and add a single newline at the end
+	content := strings.Join(outputLines, "\n") + "\n"
+
+	err := os.WriteFile(targetsOutputFile, []byte(content), 0644)
+	if err != nil {
+		return fmt.Errorf("failed to write to file: %v", err)
+	}
+
+	if debugGlobal {
+		fmt.Printf("[*] Wrote %d lines to %s\n", len(outputLines), targetsOutputFile)
+	}
+
+	return nil
+}
+
+// getExternalCredentials retrieves credentials for external format (email:password)
+func getExternalCredentials() ([]sqlite.User, error) {
+	db := sqlite.GetDB()
+	var users []sqlite.User
+
+	query := db.Where("email IS NOT NULL AND email != '' AND password IS NOT NULL AND password != ''")
+
+	// Apply domain filter if specified
+	if targetsDomain != "" {
+		query = query.Where("email LIKE ?", "%@"+targetsDomain)
+	}
+
+	err := query.Find(&users).Error
+	if err != nil {
+		zap.L().Error("get_external_credentials",
+			zap.String("message", "failed to query external credentials"),
+			zap.Error(err),
+		)
+		return nil, err
+	}
+
+	return users, nil
+}
+
+// getInternalCredentials retrieves credentials for internal format (username:password)
+func getInternalCredentials() ([]sqlite.User, error) {
+	db := sqlite.GetDB()
+	var users []sqlite.User
+
+	query := db.Where("username IS NOT NULL AND username != '' AND password IS NOT NULL AND password != ''")
+
+	// Apply domain filter if specified (filter usernames that might contain domain info)
+	if targetsDomain != "" {
+		query = query.Where("username LIKE ? OR email LIKE ?", "%"+targetsDomain+"%", "%@"+targetsDomain)
+	}
+
+	err := query.Find(&users).Error
+	if err != nil {
+		zap.L().Error("get_internal_credentials",
+			zap.String("message", "failed to query internal credentials"),
+			zap.Error(err),
+		)
+		return nil, err
+	}
+
+	return users, nil
+}
+
+// getEmailsOnly retrieves emails only (no passwords required)
+func getEmailsOnly() ([]sqlite.User, error) {
+	db := sqlite.GetDB()
+	var users []sqlite.User
+
+	query := db.Where("email IS NOT NULL AND email != ''")
+
+	// Apply domain filter if specified
+	if targetsDomain != "" {
+		query = query.Where("email LIKE ?", "%@"+targetsDomain)
+	}
+
+	err := query.Find(&users).Error
+	if err != nil {
+		zap.L().Error("get_emails_only",
+			zap.String("message", "failed to query emails"),
+			zap.Error(err),
+		)
+		return nil, err
+	}
+
+	return users, nil
+}
+
+// getSubdomains retrieves subdomains from the database
+func getSubdomains() ([]sqlite.Subdomain, error) {
+	db := sqlite.GetDB()
+	var subdomains []sqlite.Subdomain
+
+	query := db.Where("subdomain IS NOT NULL AND subdomain != ''")
+
+	// Apply domain filter if specified
+	if targetsDomain != "" {
+		query = query.Where("domain = ? OR subdomain LIKE ?", targetsDomain, "%."+targetsDomain)
+	}
+
+	err := query.Find(&subdomains).Error
+	if err != nil {
+		zap.L().Error("get_subdomains",
+			zap.String("message", "failed to query subdomains"),
+			zap.Error(err),
+		)
+		return nil, err
+	}
+
+	return subdomains, nil
+}

cmd/whois.go

@@ -59,7 +59,7 @@ var (
 
 			// Validate credentials
 			if key == "" {
-				fmt.Println("API key is required. Set the key with the \"set-key\" command. [dehasher set-key <api_key>]")
+				fmt.Println("API key is required. Set the key with the \"set-key\" command. [crowsnest set-key <api_key>]")
 				return
 			}
 
@@ -240,6 +240,7 @@ var (
 					fmt.Println("[*] Performing WHOIS subdomain scan...")
 					subdomains, err := w.WhoisSubdomainScan(whoisDomain)
 
+					// Get credits
 					if whoisShowCredits {
 						checkBalance(w)
 					}
@@ -255,7 +256,13 @@ var (
 						)
 						fmt.Printf("Error performing subdomain scan: %v\n", err)
 					} else {
-						err = sqlite.StoreWhoisSubdomainRecords(subdomains)
+						// Store subdomains in subdomains table
+						var subs []sqlite.Subdomain
+						for _, s := range subdomains {
+							subs = append(subs, sqlite.Subdomain{Domain: whoisDomain, Subdomain: s.Domain})
+						}
+
+						err = sqlite.StoreSubdomains(subs)
 						if err != nil {
 							if debugGlobal {
 								debug.PrintInfo("failed to store subdomain record")
@@ -265,7 +272,7 @@ var (
 								zap.String("message", "failed to store subdomain record"),
 								zap.Error(err),
 							)
-							fmt.Printf("Error storing WHOIS subdomain record: %v\n", err)
+							fmt.Printf("Error storing subdomain record: %v\n", err)
 						}
 
 						// Write the subdomains to file if any

crowsnest.go

@@ -21,7 +21,7 @@ var (
 )
 
 func init() {
-	basePath = filepath.Join(os.Getenv("HOME"), ".local", "share", "Dehasher")
+	basePath = filepath.Join(os.Getenv("HOME"), ".local", "share", "CrowsNest")
 	logPath = filepath.Join(basePath, "logs")
 	storePath = filepath.Join(basePath, "keystore")
 	// dbPath will be set in main() after badger is initialized
@@ -82,7 +82,7 @@ func main() {
 	useLocalDB := badger.GetUseLocalDB()
 	if useLocalDB {
 		// Use local database in current directory
-		dbPath = "./dehasher.sqlite"
+		dbPath = "./"
 		zap.L().Info("Using local database", zap.String("path", dbPath))
 	} else {
 		// Use default database path

internal/badger/badger.go

@@ -45,7 +45,7 @@ func GetHardwareEntropy() []byte {
 		username,
 		osInfo,
 		// You could add a static salt here for additional security
-		"Dehasher-static-salt-value",
+		"CrowsNest-static-salt-value",
 	}, ":")
 
 	// Hash the fingerprint to get a 32-byte key

internal/dehashed/dehashed.go

@@ -211,9 +211,9 @@ func (dh *Dehasher) buildRequest() {
 func (dh *Dehasher) parseResults() {
 	zap.L().Info("extracting_credentials")
 	results := dh.client.GetResults()
-	creds := results.ExtractCredentials()
+	creds := results.ExtractUsers()
 	fmt.Printf("   [+] Discovered %d Credentials\n", len(creds))
-	err := sqlite.StoreDehashedCreds(creds)
+	err := sqlite.StoreUsers(creds)
 	if err != nil {
 		zap.L().Error("store_creds",
 			zap.String("message", "failed to store creds"),
@@ -255,25 +255,25 @@ func (dh *Dehasher) parseResults() {
 				debug.PrintInfo("printing results table")
 			}
 
-			headers = []string{"Name", "Email", "Username", "Password", "Address", "Phone", "Social", "Crypto Address", "Company"}
+			headers = []string{"Email", "Username", "Password", "Phone", "Company"}
 			if len(results.Results) > 50 {
 				fmt.Println("   [-] Large number of results recovered, displaying first 50...")
 				for i := 0; i < 50; i++ {
 					r := results.Results[i]
 					rows = append(rows, []string{
-						strings.Join(r.Name, ", "), strings.Join(r.Email, ", "),
-						strings.Join(r.Username, ", "), strings.Join(r.Password, ", "),
-						strings.Join(r.Address, ", "), strings.Join(r.Phone, ", "),
-						strings.Join(r.Social, ", "), strings.Join(r.CryptoCurrencyAddress, ", "),
+						strings.Join(r.Email, ", "),
+						strings.Join(r.Username, ", "),
+						strings.Join(r.Password, ", "),
+						strings.Join(r.Phone, ", "),
 						strings.Join(r.Company, ", ")})
 				}
 			} else {
 				for _, r := range results.Results {
 					rows = append(rows, []string{
-						strings.Join(r.Name, ", "), strings.Join(r.Email, ", "),
-						strings.Join(r.Username, ", "), strings.Join(r.Password, ", "),
-						strings.Join(r.Address, ", "), strings.Join(r.Phone, ", "),
-						strings.Join(r.Social, ", "), strings.Join(r.CryptoCurrencyAddress, ", "),
+						strings.Join(r.Email, ", "),
+						strings.Join(r.Username, ", "),
+						strings.Join(r.Password, ", "),
+						strings.Join(r.Phone, ", "),
 						strings.Join(r.Company, ", ")})
 				}
 			}
@@ -284,7 +284,7 @@ func (dh *Dehasher) parseResults() {
 			if dh.debug {
 				debug.PrintInfo("extracting credentials")
 			}
-			creds := results.ExtractCredentials()
+			creds := results.ExtractUsers()
 			if dh.debug {
 				debug.PrintInfo("writing credentials to file")
 			}

internal/export/dehashed.go

@@ -14,7 +14,7 @@ import (
 	"time"
 )
 
-func WriteCredsToFile(creds []sqlite.Creds, outputFile string, fileType files.FileType) error {
+func WriteCredsToFile(creds []sqlite.User, outputFile string, fileType files.FileType) error {
 	var data []byte
 	var err error
 

internal/sqlite/db.go

@@ -29,7 +29,7 @@ func InitDB(dbPath string) (*gorm.DB, error) {
 			zap.L().Error("Failed to create database directory", zap.Error(err))
 			return nil, fmt.Errorf("failed to create database directory: %w", err)
 		}
-		finalDbPath = filepath.Join(dbPath, "dehashed.sqlite")
+		finalDbPath = filepath.Join(dbPath, "crowsnest.sqlite")
 	} else {
 		// Treat as file path
 		// Ensure the directory exists
@@ -51,8 +51,8 @@ func InitDB(dbPath string) (*gorm.DB, error) {
 	}
 
 	// Auto migrate your models
-	err = db.AutoMigrate(&Result{}, &Creds{}, &QueryOptions{}, &Creds{}, &WhoisRecord{}, &SubdomainRecord{},
-		&HistoryRecord{}, &LookupResult{}, &HunterDomainData{}, &HunterEmail{}, &PersonData{})
+	err = db.AutoMigrate(&Result{}, &User{}, &QueryOptions{}, &User{}, &WhoisRecord{}, &HistoryRecord{},
+		&LookupResult{}, &HunterDomainData{}, &HunterEmail{}, &PersonData{}, &Subdomain{})
 	if err != nil {
 		zap.L().Error("Failed to migrate database", zap.Error(err))
 		return nil, fmt.Errorf("failed to migrate database: %w", err)
@@ -122,7 +122,7 @@ func (t Table) Object() interface{} {
 	case RunsTable:
 		return QueryOptions{}
 	case CredsTable:
-		return Creds{}
+		return User{}
 	case WhoIsTable:
 		return WhoisRecord{}
 	case SubdomainsTable:

internal/sqlite/dehashed.go

@@ -106,15 +106,15 @@ type Result struct {
 }
 
 func (Result) TableName() string {
-	return "results"
+	return "dehashed"
 }
 
 type DehashedResults struct {
 	Results []Result `json:"results"`
 }
 
-func (dr *DehashedResults) ExtractCredentials() []Creds {
-	var creds []Creds
+func (dr *DehashedResults) ExtractUsers() []User {
+	var creds []User
 
 	results := dr.Results
 
@@ -126,16 +126,22 @@ func (dr *DehashedResults) ExtractCredentials() []Creds {
 				email = r.Email[0]
 			}
 
+			// Get first username if available
+			username := ""
+			if len(r.Username) > 0 {
+				username = r.Username[0]
+			}
+
 			// Get first password
 			password := r.Password[0]
 
-			cred := Creds{Email: email, Password: password}
+			cred := User{Email: email, Password: password, Username: username}
 			creds = append(creds, cred)
 		}
 	}
 
 	go func() {
-		err := StoreDehashedCreds(creds)
+		err := StoreUsers(creds)
 		if err != nil {
 			zap.L().Error("store_creds",
 				zap.String("message", "failed to store creds"),
@@ -148,18 +154,11 @@ func (dr *DehashedResults) ExtractCredentials() []Creds {
 	return creds
 }
 
-type Creds struct {
-	gorm.Model
-	Email    string `json:"email" yaml:"email" xml:"email" gorm:"uniqueIndex:idx_email_username_password"`
-	Username string `json:"username" yaml:"username" xml:"username" gorm:"uniqueIndex:idx_email_username_password"`
-	Password string `json:"password" yaml:"password" xml:"password" gorm:"uniqueIndex:idx_email_username_password"`
-}
-
-func (Creds) TableName() string {
+func (User) TableName() string {
 	return "creds"
 }
 
-func (c Creds) ToString() string {
+func (c User) ToString() string {
 	return fmt.Sprintf("%s%s%s", c.Username, "%", c.Password)
 }
 
@@ -197,38 +196,6 @@ func StoreDehashedResults(results DehashedResults) error {
 	return lastErr
 }
 
-func StoreDehashedCreds(creds []Creds) error {
-	if len(creds) == 0 {
-		return nil
-	}
-
-	zap.L().Info("Storing credentials", zap.Int("count", len(creds)))
-	db := GetDB()
-
-	// Use batch insert with conflict handling
-	// This will insert records in batches and continue even if some fail
-	const batchSize = 100
-	var lastErr error
-
-	for i := 0; i < len(creds); i += batchSize {
-		end := i + batchSize
-		if end > len(creds) {
-			end = len(creds)
-		}
-
-		batch := creds[i:end]
-		// Use Clauses with OnConflict DoNothing to skip conflicts
-		err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
-		if err != nil {
-			zap.L().Warn("Error storing some credentials", zap.Error(err))
-			lastErr = err
-			// Continue with next batch despite error
-		}
-	}
-
-	return lastErr
-}
-
 func StoreDehashedQueryOptions(queryOptions *QueryOptions) error {
 	db := GetDB()
 	return db.Create(queryOptions).Error

internal/sqlite/ports.go

@@ -0,0 +1 @@
+package sqlite

internal/sqlite/subdomains.go

@@ -0,0 +1,45 @@
+package sqlite
+
+import (
+	"go.uber.org/zap"
+	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
+)
+
+type Subdomain struct {
+	gorm.Model
+	Domain    string `json:"domain" yaml:"domain" xml:"domain"`
+	Subdomain string `json:"subdomain" yaml:"subdomain" xml:"subdomain" gorm:"uniqueIndex:idx_subdomain"`
+}
+
+func StoreSubdomains(subs []Subdomain) error {
+	if len(subs) == 0 {
+		return nil
+	}
+
+	zap.L().Info("Storing subdomains", zap.Int("count", len(subs)))
+	db := GetDB()
+
+	// Use batch insert with conflict handling
+	// This will insert records in batches and continue even if some fail
+	const batchSize = 100
+	var lastErr error
+
+	for i := 0; i < len(subs); i += batchSize {
+		end := i + batchSize
+		if end > len(subs) {
+			end = len(subs)
+		}
+
+		batch := subs[i:end]
+		// Use Clauses with OnConflict DoNothing to skip conflicts
+		err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
+		if err != nil {
+			zap.L().Warn("Error storing some credentials", zap.Error(err))
+			lastErr = err
+			// Continue with next batch despite error
+		}
+	}
+
+	return lastErr
+}

internal/sqlite/users.go

@@ -0,0 +1,58 @@
+package sqlite
+
+import (
+	"go.uber.org/zap"
+	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
+)
+
+type User struct {
+	gorm.Model
+	Company     string `json:"company" yaml:"company" xml:"company"`
+	Position    string `json:"position" yaml:"position" xml:"position"`
+	Department  string `json:"department" yaml:"department" xml:"department"`
+	PhoneNumber string `json:"phone_number" yaml:"phone_number" xml:"phone_number"`
+	FullName    string `json:"full_name" yaml:"full_name" xml:"full_name"`
+	Phone       string `json:"phone" yaml:"phone" xml:"phone"`
+	Linkedin    string `json:"linkedin" yaml:"linkedin" xml:"linkedin"`
+	Twitter     string `json:"twitter" yaml:"twitter" xml:"twitter"`
+	Facebook    string `json:"facebook" yaml:"facebook" xml:"facebook"`
+	Instagram   string `json:"instagram" yaml:"instagram" xml:"instagram"`
+	Youtube     string `json:"youtube" yaml:"youtube" xml:"youtube"`
+	Gravatar    string `json:"gravatar" yaml:"gravatar" xml:"gravatar"`
+	Email       string `json:"email" yaml:"email" xml:"email" gorm:"uniqueIndex:idx_email_username_password"`
+	Username    string `json:"username" yaml:"username" xml:"username" gorm:"uniqueIndex:idx_email_username_password"`
+	Password    string `json:"password" yaml:"password" xml:"password" gorm:"uniqueIndex:idx_email_username_password"`
+}
+
+func StoreUsers(users []User) error {
+	if len(users) == 0 {
+		return nil
+	}
+
+	zap.L().Info("Storing credentials", zap.Int("count", len(users)))
+	db := GetDB()
+
+	// Use batch insert with conflict handling
+	// This will insert records in batches and continue even if some fail
+	const batchSize = 100
+	var lastErr error
+
+	for i := 0; i < len(users); i += batchSize {
+		end := i + batchSize
+		if end > len(users) {
+			end = len(users)
+		}
+
+		batch := users[i:end]
+		// Use Clauses with OnConflict DoNothing to skip conflicts
+		err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
+		if err != nil {
+			zap.L().Warn("Error storing some credentials", zap.Error(err))
+			lastErr = err
+			// Continue with next batch despite error
+		}
+	}
+
+	return lastErr
+}

internal/sqlite/whois.go

@@ -536,37 +536,6 @@ func StoreWhoisRecord(whoisRecord WhoisRecord) error {
 	return nil
 }
 
-func StoreWhoisSubdomainRecords(subdomainRecords []SubdomainRecord) error {
-	if len(subdomainRecords) == 0 {
-		return nil
-	}
-
-	zap.L().Info("Storing subdomain records", zap.Int("count", len(subdomainRecords)))
-	db := GetDB()
-
-	// Use batch insert with conflict handling
-	const batchSize = 100
-	var lastErr error
-
-	for i := 0; i < len(subdomainRecords); i += batchSize {
-		end := i + batchSize
-		if end > len(subdomainRecords) {
-			end = len(subdomainRecords)
-		}
-
-		batch := subdomainRecords[i:end]
-		// Use Clauses with OnConflict DoNothing to skip conflicts
-		err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
-		if err != nil {
-			zap.L().Warn("Error storing some subdomain records", zap.Error(err))
-			lastErr = err
-			// Continue with next batch despite error
-		}
-	}
-
-	return lastErr
-}
-
 func StoreWhoisHistoryRecords(historyRecords []HistoryRecord) error {
 	if len(historyRecords) == 0 {
 		return nil