diff --git a/cmd/dehashed.go b/cmd/dehashed.go index ec57063..6cceabf 100644 --- a/cmd/dehashed.go +++ b/cmd/dehashed.go @@ -118,6 +118,7 @@ var ( dehasher.Start() fmt.Println("\n[*] Completing Process") + // Store query options err := sqlite.StoreDehashedQueryOptions(queryOptions) if err != nil { if debugGlobal { diff --git a/cmd/export.go b/cmd/export.go deleted file mode 100644 index 076a051..0000000 --- a/cmd/export.go +++ /dev/null @@ -1,311 +0,0 @@ -package cmd - -import ( - "crowsnest/internal/export" - "crowsnest/internal/files" - "crowsnest/internal/sqlite" - "fmt" - "github.com/spf13/cobra" - "go.uber.org/zap" - "strings" -) - -func init() { - // Add Subcommand to db command - rootCmd.AddCommand(exportCmd) - - // Add flags specific to export command - exportCmd.Flags().IntVarP(&exportLimitRows, "limit", "l", 100, "Limit number of results") - exportCmd.Flags().BoolVarP(&exportListAll, "list-all", "a", false, "List all tables and their columns") - exportCmd.Flags().StringVarP(&exportTableName, "table", "t", "", "Table to export (results, creds, whois, subdomains, history, runs)") - exportCmd.Flags().StringVarP(&exportNotNull, "not-null", "n", "", "Filter for non-null values (comma-separated list, e.g., 'password,email')") - exportCmd.Flags().StringVarP(&exportColumns, "columns", "c", "", "Columns to display in output (comma-separated list, e.g., 'username,email,password')") - exportCmd.Flags().StringVarP(&exportUserQuery, "user-query", "q", "", "User query to execute") - exportCmd.Flags().StringVarP(&exportRawQuery, "raw-query", "r", "", "Raw SQL query to execute") - exportCmd.Flags().StringVarP(&exportFormat, "format", "f", "json", "Output format (json, yaml, xml, txt)") - exportCmd.Flags().StringVarP(&exportFile, "file", "o", "export", "File to output results to including extension") - - // Add mutually exclusive flags to query and raw-query - // Cannot use query and raw-query at the same time - exportCmd.MarkFlagsMutuallyExclusive("user-query", "raw-query") - // Raw query does not require a table - exportCmd.MarkFlagsMutuallyExclusive("user-query", "table") - // List all columns does not require a query or raw-query - exportCmd.MarkFlagsMutuallyExclusive("raw-query", "list-all") -} - -// DB export command -var ( - exportLimitRows int - exportListAll bool - exportTableName string - exportNotNull string - exportColumns string - exportUserQuery string - exportRawQuery string - exportFormat string - exportFile string - - exportCmd = &cobra.Command{ - Use: "export", - Short: "Export database to file", - Run: func(cmd *cobra.Command, args []string) { - // If list-all flag is set, list all tables and columns - if exportListAll { - listAvailableTables() - return - } - - fmt.Println("[*] Exporting database...") - - // If Raw Query is set, execute it and export - if exportRawQuery != "" { - fmt.Println("[*] Executing Raw Query...") - exportRawDBQuery() - return - } - - // Validate table name - if exportTableName == "" { - fmt.Println("[!] Error: Table name is required. Use -t or --table to specify a table.") - fmt.Println("[*] Available tables: results, creds, whois, subdomains, history, runs") - fmt.Println("[*] Use --list-all to see all tables and their columns.") - return - } - - if !isValidTable(exportTableName) { - fmt.Printf("[!] Error: Unknown table '%s'.\n", exportTableName) - fmt.Println("[*] Available tables: results, creds, whois, subdomains, history, runs") - fmt.Println("[*] Use --list-all to see all tables and their columns.") - return - } - - // Validate columns if specified - if exportColumns != "" { - columns := strings.Split(exportColumns, ",") - invalidColumns := validateColumns(exportTableName, columns) - if len(invalidColumns) > 0 { - fmt.Printf("[!] Error: Invalid column(s) for table '%s': %s\n", - exportTableName, strings.Join(invalidColumns, ", ")) - fmt.Println("[*] Available columns for this table:") - for i := 0; i < len(availableTables[exportTableName]); i += 5 { - end := i + 5 - if end > len(availableTables[exportTableName]) { - end = len(availableTables[exportTableName]) - } - fmt.Printf(" %s\n", strings.Join(availableTables[exportTableName][i:end], ", ")) - } - return - } - } - - // Validate not-null fields if specified - if exportNotNull != "" { - notNullFields := strings.Split(exportNotNull, ",") - invalidFields := validateColumns(exportTableName, notNullFields) - if len(invalidFields) > 0 { - fmt.Printf("[!] Error: Invalid not-null field(s) for table '%s': %s\n", - exportTableName, strings.Join(invalidFields, ", ")) - fmt.Println("[*] Available columns for this table:") - for i := 0; i < len(availableTables[exportTableName]); i += 5 { - end := i + 5 - if end > len(availableTables[exportTableName]) { - end = len(availableTables[exportTableName]) - } - fmt.Printf(" %s\n", strings.Join(availableTables[exportTableName][i:end], ", ")) - } - return - } - } - - // Determine which table to query based on the tableTypeDBQuery parameter - table := sqlite.GetTable(exportTableName) - if table == sqlite.UnknownTable { - fmt.Printf("[!] Error: Unknown table type '%s'.\n", exportTableName) - fmt.Println("[*] Available tables: results, creds, whois, subdomains, history, runs") - fmt.Println("[*] Use --list-all to see all tables and their columns.") - return - } - - fmt.Println("[*] Querying Database...") - exportTableQuery(table) - }, - } -) - -// exportTableQuery queries a table and exports the results -func exportTableQuery(table sqlite.Table) { - // Get the columns to query - columns := []string{"*"} - if exportColumns != "" { - columns = strings.Split(exportColumns, ",") - } - - // Get the not null fields - notNullFields := []string{} - if exportNotNull != "" { - notNullFields = strings.Split(exportNotNull, ",") - } - - // Get the user query - userQuery := "" - if exportUserQuery != "" { - userQuery = exportUserQuery - } - - // Get the limit - limit := exportLimitRows - - // Get the object for the table - object := table.Object() - - // Check if object is nil (invalid table) - if object == nil { - fmt.Printf("[!] Error: Table '%s' is not valid or does not exist.\n", exportTableName) - return - } - - // Query the database - db := sqlite.GetDB() - query := db.Model(object).Select(columns) - if len(notNullFields) > 0 { - for _, field := range notNullFields { - query = query.Where(fmt.Sprintf("%s IS NOT NULL", field)) - } - } - if userQuery != "" { - query = query.Where(userQuery) - } - if limit > 0 { - query = query.Limit(limit) - } - rows, err := query.Rows() - if err != nil { - zap.L().Error("export_query", - zap.String("message", "failed to execute query"), - zap.Error(err), - ) - fmt.Printf("[!] Error executing query: %v\n", err) - return - } - defer rows.Close() - - // Get the columns - cols, err := rows.Columns() - if err != nil { - zap.L().Error("export_query", - zap.String("message", "failed to get columns from query"), - zap.Error(err), - ) - fmt.Printf("[!] Error getting columns from query: %v\n", err) - return - } - - // Prepare data for export - var results []map[string]interface{} - - // Process the rows - for rows.Next() { - values := make([]interface{}, len(cols)) - pointers := make([]interface{}, len(cols)) - for i := range values { - pointers[i] = &values[i] - } - if err := rows.Scan(pointers...); err != nil { - zap.L().Error("export_query", - zap.String("message", "failed to scan row from query"), - zap.Error(err), - ) - fmt.Printf("[!] Error scanning row from query: %v\n", err) - return - } - - // Create a map for this row - rowMap := make(map[string]interface{}) - for i, col := range cols { - val := values[i] - rowMap[col] = val - } - - results = append(results, rowMap) - } - - // Export the results - exportResults(results) -} - -// exportRawDBQuery executes a raw query and exports the results -func exportRawDBQuery() { - db := sqlite.GetDB() - rows, err := db.Raw(exportRawQuery).Rows() - if err != nil { - zap.L().Error("export_raw_query", - zap.String("message", "failed to execute raw query"), - zap.Error(err), - ) - fmt.Printf("[!] Error executing raw query: %v\n", err) - return - } - defer rows.Close() - - columns, err := rows.Columns() - if err != nil { - zap.L().Error("export_raw_query", - zap.String("message", "failed to get columns from raw query"), - zap.Error(err), - ) - fmt.Printf("[!] Error getting columns from raw query: %v\n", err) - return - } - - // Prepare data for export - var results []map[string]interface{} - - // Process the rows - for rows.Next() { - values := make([]interface{}, len(columns)) - pointers := make([]interface{}, len(columns)) - for i := range values { - pointers[i] = &values[i] - } - if err := rows.Scan(pointers...); err != nil { - zap.L().Error("export_raw_query", - zap.String("message", "failed to scan row from raw query"), - zap.Error(err), - ) - fmt.Printf("[!] Error scanning row from raw query: %v\n", err) - return - } - - // Create a map for this row - rowMap := make(map[string]interface{}) - for i, col := range columns { - val := values[i] - rowMap[col] = val - } - - results = append(results, rowMap) - } - - // Export the results - exportResults(results) -} - -// exportResults exports the results to a file -func exportResults(results []map[string]interface{}) { - // Get file type - fileType := files.GetFileType(exportFormat) - - // Export results - err := export.WriteQueryResultsToFile(results, exportFile, fileType) - if err != nil { - zap.L().Error("export_results", - zap.String("message", "failed to write to file"), - zap.Error(err), - ) - fmt.Printf("[!] Error writing to file: %v\n", err) - return - } - - fmt.Printf("[+] Exported %d records to file: %s%s\n", len(results), exportFile, fileType.Extension()) -} diff --git a/cmd/hunter.go b/cmd/hunter.go index 7d2fa9b..e268fa3 100644 --- a/cmd/hunter.go +++ b/cmd/hunter.go @@ -7,6 +7,7 @@ import ( "crowsnest/internal/files" hunter "crowsnest/internal/hunter.io" "crowsnest/internal/pretty" + "crowsnest/internal/sqlite" "fmt" "github.com/spf13/cobra" "go.uber.org/zap" @@ -111,6 +112,24 @@ var ( return } + // Store the users discovered + var creds []sqlite.User + for _, email := range result.Emails { + creds = append(creds, sqlite.User{Email: email.Value}) + } + err = sqlite.StoreUsers(creds) + if err != nil { + if debugGlobal { + debug.PrintInfo("failed to store hunter domain search") + debug.PrintError(err) + } + zap.L().Error("store_hunter_domain_search", + zap.String("message", "failed to store hunter domain search"), + zap.Error(err), + ) + fmt.Printf("Error storing Hunter.io Domain Search Result: %v\n", err) + } + // Write Hunter.io Domain Search Result to file fmt.Printf("[*] Writing Hunter.io Domain Search Result to file: %s%s\n", hunterOutputFile, fType.Extension()) err = export.WriteIStringToFile(result, hunterOutputFile, fType) diff --git a/cmd/query.go b/cmd/query.go index 1ffff46..b54abdd 100644 --- a/cmd/query.go +++ b/cmd/query.go @@ -1,6 +1,9 @@ package cmd import ( + "crowsnest/internal/debug" + "crowsnest/internal/export" + "crowsnest/internal/files" "crowsnest/internal/pretty" "crowsnest/internal/sqlite" "encoding/json" @@ -10,132 +13,6 @@ import ( "strings" ) -// Map of available tables and their columns -var availableTables = map[string][]string{ - "creds": { - "id", "created_at", "updated_at", "deleted_at", "email", "username", "password", - }, - //"history": { - // "id", "created_at", "updated_at", "deleted_at", "domain_name", "domain_type", - // "registrar_name", "whois_server", "created_date_iso8601", "updated_date_iso8601", "expires_date_iso8601", - //}, - "lookup": { - "id", "created_at", "updated_at", "deleted_at", "search_term", "type", "first_seen", "last_visit", - "name", - }, - // Query Options - "runs": { - "id", "created_at", "updated_at", "deleted_at", "max_records", "max_requests", "starting_page", - "output_format", "output_file", "regex_match", "wildcard_match", "username_query", "email_query", - "ip_query", "pass_query", "hash_query", "name_query", "domain_query", "vin_query", "license_plate_query", - "address_query", "phone_query", "social_query", "crypto_address_query", "print_balance", "creds_only", - }, - "results": { - "id", "created_at", "updated_at", "deleted_at", "dehashed_id", "email", "ip_address", "username", - "password", "hashed_password", "hash_type", "name", "vin", "license_plate", "url", "social", - "cryptocurrency_address", "address", "phone", "company", "database_name", - }, - "subdomains": { - "id", "created_at", "updated_at", "deleted_at", "domain", "first_seen", "last_seen", - }, - "whois": { - "id", "created_at", "updated_at", "deleted_at", "audit", "contact_email", "created_date", "created_date_normalized", - "domain_name", "domain_name_ext", "estimated_domain_age", "expires_date", "expires_date_normalized", "footer", "header", - "name_servers", "parse_code", "raw_text", "registrant", "registrar_iana_id", "registrar_name", "registry_data", - "status", "stripped_text", "updated_date", "updated_date_normalized", - }, - "hunter_domain": { - "id", "created_at", "updated_at", "deleted_at", "domain", "disposable", "webmail", "accept_all", "pattern", - "organization", "description", "industry", "twitter", "facebook", "linkedin", "instagram", "youtube", - "technologies", "country", "state", "city", "postal_code", "street", "headcount", "company_type", "emails", "linked_domains", - }, - "hunter_email": { - "id", "created_at", "updated_at", "deleted_at", "value", "type", "confidence", "sources", "first_name", "last_name", - "position", "position_raw", "seniority", "department", "linkedin", "twitter", "phone_number", "verification_date", "verification_status", - }, -} - -// Function to list available tables and their columns -func listAvailableTables() { - fmt.Println("Available tables and columns:") - - // Prepare data for pretty.Table - headers := []string{"Table", "Columns"} - var tableRows [][]string - - // Sort tables alphabetically for consistent output - var tableNames []string - for tableName := range availableTables { - tableNames = append(tableNames, tableName) - } - - // Simple bubble sort for table names - for i := 0; i < len(tableNames)-1; i++ { - for j := 0; j < len(tableNames)-i-1; j++ { - if tableNames[j] > tableNames[j+1] { - tableNames[j], tableNames[j+1] = tableNames[j+1], tableNames[j] - } - } - } - - // Create rows for the table - for _, tableName := range tableNames { - columns := availableTables[tableName] - - // Format columns with line breaks for better readability - var formattedColumns string - for i := 0; i < len(columns); i += 5 { - end := i + 5 - if end > len(columns) { - end = len(columns) - } - if i > 0 { - formattedColumns += "\n" - } - formattedColumns += strings.Join(columns[i:end], ", ") - } - - tableRows = append(tableRows, []string{tableName, formattedColumns}) - } - - // Display the table - pretty.Table(headers, tableRows) -} - -// Function to validate table name -func isValidTable(tableName string) bool { - _, exists := availableTables[tableName] - return exists -} - -// Function to validate column names for a specific table -func validateColumns(tableName string, columns []string) []string { - if tableName == "" || columns == nil || len(columns) == 0 || columns[0] == "*" { - return nil - } - - tableColumns, exists := availableTables[tableName] - if !exists { - return []string{fmt.Sprintf("Table '%s' does not exist", tableName)} - } - - var invalidColumns []string - for _, col := range columns { - valid := false - for _, tableCol := range tableColumns { - if col == tableCol { - valid = true - break - } - } - if !valid { - invalidColumns = append(invalidColumns, col) - } - } - - return invalidColumns -} - func init() { // Add whois command to root command rootCmd.AddCommand(queryCmd) @@ -148,6 +25,8 @@ func init() { queryCmd.Flags().StringVarP(&dbQueryUserQuery, "user-query", "q", "", "User query to execute") queryCmd.Flags().StringVarP(&dbQueryRawQuery, "raw-query", "r", "", "Raw SQL query to execute") queryCmd.Flags().BoolVarP(&dbQueryListAll, "list-all", "a", false, "List all tables and their columns") + queryCmd.Flags().StringVarP(&dbQueryFormat, "format", "f", "json", "Output format (json, yaml, xml, txt)") + queryCmd.Flags().StringVarP(&dbQueryFile, "file", "o", "query", "File to output results to including extension") // Add mutually exclusive flags to query and raw-query // Cannot use query and raw-query at the same time @@ -166,6 +45,8 @@ var ( dbQueryUserQuery string dbQueryRawQuery string dbQueryListAll bool + dbQueryFormat string + dbQueryFile string queryCmd = &cobra.Command{ Use: "query", @@ -321,6 +202,49 @@ func tableQuery(table sqlite.Table) { return } + // Export results if file name is specified + if len(strings.TrimSpace(dbQueryFile)) > 0 { + fmt.Println("[*] Exporting results to file...") + + if debugGlobal { + debug.PrintInfo("exporting results to file: " + dbQueryFile) + } + // Prepare data for export + var results []map[string]interface{} + + // Process the rows + for rows.Next() { + values := make([]interface{}, len(cols)) + pointers := make([]interface{}, len(cols)) + for i := range values { + pointers[i] = &values[i] + } + if err := rows.Scan(pointers...); err != nil { + zap.L().Error("export_query", + zap.String("message", "failed to scan row from query"), + zap.Error(err), + ) + fmt.Printf("[!] Error scanning row from query: %v\n", err) + return + } + + // Create a map for this row + rowMap := make(map[string]interface{}) + for i, col := range cols { + val := values[i] + rowMap[col] = val + } + + results = append(results, rowMap) + } + + // Export the results + exportQueryResults(results) + + return + } + fmt.Println("[*] Querying Database...") + // Prepare data for pretty.Table headers := cols var tableRows [][]string @@ -411,6 +335,49 @@ func rawDBQuery() { return } + if len(strings.TrimSpace(dbQueryFile)) > 0 { + fmt.Println("[*] Exporting results to file...") + + if debugGlobal { + debug.PrintInfo("exporting results to file: " + dbQueryFile) + } + + // Prepare data for export + var results []map[string]interface{} + + // Process the rows + for rows.Next() { + values := make([]interface{}, len(columns)) + pointers := make([]interface{}, len(columns)) + for i := range values { + pointers[i] = &values[i] + } + if err := rows.Scan(pointers...); err != nil { + zap.L().Error("export_raw_query", + zap.String("message", "failed to scan row from raw query"), + zap.Error(err), + ) + fmt.Printf("[!] Error scanning row from raw query: %v\n", err) + return + } + + // Create a map for this row + rowMap := make(map[string]interface{}) + for i, col := range columns { + val := values[i] + rowMap[col] = val + } + + results = append(results, rowMap) + } + + // Export the results + exportQueryResults(results) + + return + } + fmt.Println("[*] Querying Database...") + // Prepare data for pretty.Table headers := columns var tableRows [][]string @@ -477,3 +444,22 @@ func rawDBQuery() { // Display the table pretty.Table(headers, tableRows) } + +// exportQueryResults exports the results to a file +func exportQueryResults(results []map[string]interface{}) { + // Get file type + fileType := files.GetFileType(dbQueryFormat) + + // Export results + err := export.WriteQueryResultsToFile(results, dbQueryFile, fileType) + if err != nil { + zap.L().Error("export_results", + zap.String("message", "failed to write to file"), + zap.Error(err), + ) + fmt.Printf("[!] Error writing to file: %v\n", err) + return + } + + fmt.Printf("[+] Exported %d records to file: %s%s\n", len(results), dbQueryFile, fileType.Extension()) +} diff --git a/cmd/tables.go b/cmd/tables.go new file mode 100644 index 0000000..1851c5d --- /dev/null +++ b/cmd/tables.go @@ -0,0 +1,133 @@ +package cmd + +import ( + "crowsnest/internal/pretty" + "fmt" + "strings" +) + +// Map of available tables and their columns +var availableTables = map[string][]string{ + "users": { + "id", "created_at", "updated_at", "deleted_at", "email", "username", "password", + }, + //"history": { + // "id", "created_at", "updated_at", "deleted_at", "domain_name", "domain_type", + // "registrar_name", "whois_server", "created_date_iso8601", "updated_date_iso8601", "expires_date_iso8601", + //}, + "lookup": { + "id", "created_at", "updated_at", "deleted_at", "search_term", "type", "first_seen", "last_visit", + "name", + }, + // Query Options + "runs": { + "id", "created_at", "updated_at", "deleted_at", "max_records", "max_requests", "starting_page", + "output_format", "output_file", "regex_match", "wildcard_match", "username_query", "email_query", + "ip_query", "pass_query", "hash_query", "name_query", "domain_query", "vin_query", "license_plate_query", + "address_query", "phone_query", "social_query", "crypto_address_query", "print_balance", "creds_only", + }, + "dehashed": { + "id", "created_at", "updated_at", "deleted_at", "dehashed_id", "email", "ip_address", "username", + "password", "hashed_password", "hash_type", "name", "vin", "license_plate", "url", "social", + "cryptocurrency_address", "address", "phone", "company", "database_name", + }, + "subdomains": { + "id", "created_at", "updated_at", "deleted_at", "domain", "subdomain", + }, + "whois": { + "id", "created_at", "updated_at", "deleted_at", "audit", "contact_email", "created_date", "created_date_normalized", + "domain_name", "domain_name_ext", "estimated_domain_age", "expires_date", "expires_date_normalized", "footer", "header", + "name_servers", "parse_code", "raw_text", "registrant", "registrar_iana_id", "registrar_name", "registry_data", + "status", "stripped_text", "updated_date", "updated_date_normalized", + }, + "hunter_domain": { + "id", "created_at", "updated_at", "deleted_at", "domain", "disposable", "webmail", "accept_all", "pattern", + "organization", "description", "industry", "twitter", "facebook", "linkedin", "instagram", "youtube", + "technologies", "country", "state", "city", "postal_code", "street", "headcount", "company_type", "emails", "linked_domains", + }, + "hunter_email": { + "id", "created_at", "updated_at", "deleted_at", "value", "type", "confidence", "sources", "first_name", "last_name", + "position", "position_raw", "seniority", "department", "linkedin", "twitter", "phone_number", "verification_date", "verification_status", + }, +} + +// Function to list available tables and their columns +func listAvailableTables() { + fmt.Println("Available tables and columns:") + + // Prepare data for pretty.Table + headers := []string{"Table", "Columns"} + var tableRows [][]string + + // Sort tables alphabetically for consistent output + var tableNames []string + for tableName := range availableTables { + tableNames = append(tableNames, tableName) + } + + // Simple bubble sort for table names + for i := 0; i < len(tableNames)-1; i++ { + for j := 0; j < len(tableNames)-i-1; j++ { + if tableNames[j] > tableNames[j+1] { + tableNames[j], tableNames[j+1] = tableNames[j+1], tableNames[j] + } + } + } + + // Create rows for the table + for _, tableName := range tableNames { + columns := availableTables[tableName] + + // Format columns with line breaks for better readability + var formattedColumns string + for i := 0; i < len(columns); i += 5 { + end := i + 5 + if end > len(columns) { + end = len(columns) + } + if i > 0 { + formattedColumns += "\n" + } + formattedColumns += strings.Join(columns[i:end], ", ") + } + + tableRows = append(tableRows, []string{tableName, formattedColumns}) + } + + // Display the table + pretty.Table(headers, tableRows) +} + +// Function to validate table name +func isValidTable(tableName string) bool { + _, exists := availableTables[tableName] + return exists +} + +// Function to validate column names for a specific table +func validateColumns(tableName string, columns []string) []string { + if tableName == "" || columns == nil || len(columns) == 0 || columns[0] == "*" { + return nil + } + + tableColumns, exists := availableTables[tableName] + if !exists { + return []string{fmt.Sprintf("Table '%s' does not exist", tableName)} + } + + var invalidColumns []string + for _, col := range columns { + valid := false + for _, tableCol := range tableColumns { + if col == tableCol { + valid = true + break + } + } + if !valid { + invalidColumns = append(invalidColumns, col) + } + } + + return invalidColumns +} diff --git a/cmd/targets.go b/cmd/targets.go new file mode 100644 index 0000000..4445a84 --- /dev/null +++ b/cmd/targets.go @@ -0,0 +1,312 @@ +package cmd + +import ( + "crowsnest/internal/sqlite" + "fmt" + "github.com/spf13/cobra" + "go.uber.org/zap" + "os" + "strings" +) + +func init() { + // Add targets command to root command + rootCmd.AddCommand(targetsCmd) + + // Add flags specific to targets command + targetsCmd.Flags().StringVarP(&targetsOutputFile, "output", "o", "targets", "Output file name (required)") + targetsCmd.Flags().BoolVarP(&targetsExternal, "external", "e", false, "Output external format (email:password)") + targetsCmd.Flags().BoolVarP(&targetsInternal, "internal", "i", false, "Output internal format (username:password)") + targetsCmd.Flags().BoolVarP(&targetsSubdomains, "subdomains", "s", false, "Output subdomains") + targetsCmd.Flags().BoolVarP(&targetsEmails, "emails", "E", false, "Output emails only (no passwords)") + targetsCmd.Flags().StringVarP(&targetsDomain, "domain", "d", "", "Filter by domain (for emails and subdomains)") + + // Mark output flag as required + targetsCmd.MarkFlagRequired("output") +} + +var ( + // Targets command flags + targetsOutputFile string + targetsExternal bool + targetsInternal bool + targetsSubdomains bool + targetsEmails bool + targetsDomain string + + // Targets command + targetsCmd = &cobra.Command{ + Use: "targets", + Short: "Export users and subdomains in formats suitable for external tools", + Long: `Export users and subdomains from the database in easily digestible formats for tools like sprays or other security testing tools. + +Formats: + --external (-e): Output in email:password format + --internal (-i): Output in username:password format + --emails (-E): Output emails only (no passwords) + --subdomains (-s): Output subdomains only + +Options: + --domain (-d): Filter results by domain (applies to emails and subdomains) + --output (-o): Specify output file name (required) + +Examples: + # Export all external credentials (email:password) + crowsnest targets -e -o external_creds + + # Export internal credentials for a specific domain + crowsnest targets -i -d example.com -o internal_creds + + # Export all emails + crowsnest targets -E -o all_emails + + # Export emails for a specific domain + crowsnest targets -E -d example.com -o domain_emails + + # Export subdomains for a specific domain + crowsnest targets -s -d example.com -o subdomains + + # Export all subdomains + crowsnest targets -s -o all_subdomains`, + Run: func(cmd *cobra.Command, args []string) { + // Validate that at least one format is specified + if !targetsExternal && !targetsInternal && !targetsSubdomains && !targetsEmails { + fmt.Println("[!] Error: You must specify at least one output format:") + fmt.Println(" --external (-e) for email:password format") + fmt.Println(" --internal (-i) for username:password format") + fmt.Println(" --emails (-E) for emails only") + fmt.Println(" --subdomains (-s) for subdomains") + return + } + + if debugGlobal { + zap.L().Info("targets_debug", + zap.String("message", "targets command started"), + zap.Bool("external", targetsExternal), + zap.Bool("internal", targetsInternal), + zap.Bool("subdomains", targetsSubdomains), + zap.Bool("emails", targetsEmails), + zap.String("domain", targetsDomain), + zap.String("output_file", targetsOutputFile), + ) + } + + // Execute the targets export + err := executeTargetsExport() + if err != nil { + fmt.Printf("[!] Error: %v\n", err) + return + } + + fmt.Printf("[+] Successfully exported targets to: %s\n", targetsOutputFile) + }, + } +) + +// executeTargetsExport performs the main logic for exporting targets +func executeTargetsExport() error { + var outputLines []string + + // Export external credentials (email:password) + if targetsExternal { + if debugGlobal { + fmt.Println("[*] Exporting external credentials (email:password)...") + } + + externalCreds, err := getExternalCredentials() + if err != nil { + return fmt.Errorf("failed to get external credentials: %v", err) + } + + for _, cred := range externalCreds { + if cred.Email != "" && cred.Password != "" { + outputLines = append(outputLines, fmt.Sprintf("%s:%s", cred.Email, cred.Password)) + } + } + + if debugGlobal { + fmt.Printf("[*] Found %d external credentials\n", len(externalCreds)) + } + } + + // Export internal credentials (username:password) + if targetsInternal { + if debugGlobal { + fmt.Println("[*] Exporting internal credentials (username:password)...") + } + + internalCreds, err := getInternalCredentials() + if err != nil { + return fmt.Errorf("failed to get internal credentials: %v", err) + } + + for _, cred := range internalCreds { + if cred.Username != "" && cred.Password != "" { + outputLines = append(outputLines, fmt.Sprintf("%s:%s", cred.Username, cred.Password)) + } + } + + if debugGlobal { + fmt.Printf("[*] Found %d internal credentials\n", len(internalCreds)) + } + } + + // Export emails only + if targetsEmails { + if debugGlobal { + fmt.Println("[*] Exporting emails only...") + } + + emails, err := getEmailsOnly() + if err != nil { + return fmt.Errorf("failed to get emails: %v", err) + } + + for _, email := range emails { + if email.Email != "" { + outputLines = append(outputLines, email.Email) + } + } + + if debugGlobal { + fmt.Printf("[*] Found %d emails\n", len(emails)) + } + } + + // Export subdomains + if targetsSubdomains { + if debugGlobal { + fmt.Println("[*] Exporting subdomains...") + } + + subdomains, err := getSubdomains() + if err != nil { + return fmt.Errorf("failed to get subdomains: %v", err) + } + + for _, subdomain := range subdomains { + if subdomain.Subdomain != "" { + outputLines = append(outputLines, subdomain.Subdomain) + } + } + + if debugGlobal { + fmt.Printf("[*] Found %d subdomains\n", len(subdomains)) + } + } + + // Write to file + if len(outputLines) == 0 { + return fmt.Errorf("no data found to export") + } + + // Join all lines with newlines and add a single newline at the end + content := strings.Join(outputLines, "\n") + "\n" + + err := os.WriteFile(targetsOutputFile, []byte(content), 0644) + if err != nil { + return fmt.Errorf("failed to write to file: %v", err) + } + + if debugGlobal { + fmt.Printf("[*] Wrote %d lines to %s\n", len(outputLines), targetsOutputFile) + } + + return nil +} + +// getExternalCredentials retrieves credentials for external format (email:password) +func getExternalCredentials() ([]sqlite.User, error) { + db := sqlite.GetDB() + var users []sqlite.User + + query := db.Where("email IS NOT NULL AND email != '' AND password IS NOT NULL AND password != ''") + + // Apply domain filter if specified + if targetsDomain != "" { + query = query.Where("email LIKE ?", "%@"+targetsDomain) + } + + err := query.Find(&users).Error + if err != nil { + zap.L().Error("get_external_credentials", + zap.String("message", "failed to query external credentials"), + zap.Error(err), + ) + return nil, err + } + + return users, nil +} + +// getInternalCredentials retrieves credentials for internal format (username:password) +func getInternalCredentials() ([]sqlite.User, error) { + db := sqlite.GetDB() + var users []sqlite.User + + query := db.Where("username IS NOT NULL AND username != '' AND password IS NOT NULL AND password != ''") + + // Apply domain filter if specified (filter usernames that might contain domain info) + if targetsDomain != "" { + query = query.Where("username LIKE ? OR email LIKE ?", "%"+targetsDomain+"%", "%@"+targetsDomain) + } + + err := query.Find(&users).Error + if err != nil { + zap.L().Error("get_internal_credentials", + zap.String("message", "failed to query internal credentials"), + zap.Error(err), + ) + return nil, err + } + + return users, nil +} + +// getEmailsOnly retrieves emails only (no passwords required) +func getEmailsOnly() ([]sqlite.User, error) { + db := sqlite.GetDB() + var users []sqlite.User + + query := db.Where("email IS NOT NULL AND email != ''") + + // Apply domain filter if specified + if targetsDomain != "" { + query = query.Where("email LIKE ?", "%@"+targetsDomain) + } + + err := query.Find(&users).Error + if err != nil { + zap.L().Error("get_emails_only", + zap.String("message", "failed to query emails"), + zap.Error(err), + ) + return nil, err + } + + return users, nil +} + +// getSubdomains retrieves subdomains from the database +func getSubdomains() ([]sqlite.Subdomain, error) { + db := sqlite.GetDB() + var subdomains []sqlite.Subdomain + + query := db.Where("subdomain IS NOT NULL AND subdomain != ''") + + // Apply domain filter if specified + if targetsDomain != "" { + query = query.Where("domain = ? OR subdomain LIKE ?", targetsDomain, "%."+targetsDomain) + } + + err := query.Find(&subdomains).Error + if err != nil { + zap.L().Error("get_subdomains", + zap.String("message", "failed to query subdomains"), + zap.Error(err), + ) + return nil, err + } + + return subdomains, nil +} diff --git a/cmd/whois.go b/cmd/whois.go index 806e6fb..a30ecce 100644 --- a/cmd/whois.go +++ b/cmd/whois.go @@ -240,6 +240,7 @@ var ( fmt.Println("[*] Performing WHOIS subdomain scan...") subdomains, err := w.WhoisSubdomainScan(whoisDomain) + // Get credits if whoisShowCredits { checkBalance(w) } @@ -255,7 +256,13 @@ var ( ) fmt.Printf("Error performing subdomain scan: %v\n", err) } else { - err = sqlite.StoreWhoisSubdomainRecords(subdomains) + // Store subdomains in subdomains table + var subs []sqlite.Subdomain + for _, s := range subdomains { + subs = append(subs, sqlite.Subdomain{Domain: whoisDomain, Subdomain: s.Domain}) + } + + err = sqlite.StoreSubdomains(subs) if err != nil { if debugGlobal { debug.PrintInfo("failed to store subdomain record") @@ -265,7 +272,7 @@ var ( zap.String("message", "failed to store subdomain record"), zap.Error(err), ) - fmt.Printf("Error storing WHOIS subdomain record: %v\n", err) + fmt.Printf("Error storing subdomain record: %v\n", err) } // Write the subdomains to file if any diff --git a/internal/dehashed/dehashed.go b/internal/dehashed/dehashed.go index b67f1ba..f481c28 100644 --- a/internal/dehashed/dehashed.go +++ b/internal/dehashed/dehashed.go @@ -211,9 +211,9 @@ func (dh *Dehasher) buildRequest() { func (dh *Dehasher) parseResults() { zap.L().Info("extracting_credentials") results := dh.client.GetResults() - creds := results.ExtractCredentials() + creds := results.ExtractUsers() fmt.Printf(" [+] Discovered %d Credentials\n", len(creds)) - err := sqlite.StoreDehashedCreds(creds) + err := sqlite.StoreUsers(creds) if err != nil { zap.L().Error("store_creds", zap.String("message", "failed to store creds"), @@ -284,7 +284,7 @@ func (dh *Dehasher) parseResults() { if dh.debug { debug.PrintInfo("extracting credentials") } - creds := results.ExtractCredentials() + creds := results.ExtractUsers() if dh.debug { debug.PrintInfo("writing credentials to file") } diff --git a/internal/export/dehashed.go b/internal/export/dehashed.go index 9484eb6..f9df613 100644 --- a/internal/export/dehashed.go +++ b/internal/export/dehashed.go @@ -14,7 +14,7 @@ import ( "time" ) -func WriteCredsToFile(creds []sqlite.Creds, outputFile string, fileType files.FileType) error { +func WriteCredsToFile(creds []sqlite.User, outputFile string, fileType files.FileType) error { var data []byte var err error diff --git a/internal/sqlite/db.go b/internal/sqlite/db.go index 16a71f7..d48472b 100644 --- a/internal/sqlite/db.go +++ b/internal/sqlite/db.go @@ -51,8 +51,8 @@ func InitDB(dbPath string) (*gorm.DB, error) { } // Auto migrate your models - err = db.AutoMigrate(&Result{}, &Creds{}, &QueryOptions{}, &Creds{}, &WhoisRecord{}, &SubdomainRecord{}, - &HistoryRecord{}, &LookupResult{}, &HunterDomainData{}, &HunterEmail{}, &PersonData{}) + err = db.AutoMigrate(&Result{}, &User{}, &QueryOptions{}, &User{}, &WhoisRecord{}, &HistoryRecord{}, + &LookupResult{}, &HunterDomainData{}, &HunterEmail{}, &PersonData{}, &Subdomain{}) if err != nil { zap.L().Error("Failed to migrate database", zap.Error(err)) return nil, fmt.Errorf("failed to migrate database: %w", err) @@ -122,7 +122,7 @@ func (t Table) Object() interface{} { case RunsTable: return QueryOptions{} case CredsTable: - return Creds{} + return User{} case WhoIsTable: return WhoisRecord{} case SubdomainsTable: diff --git a/internal/sqlite/structs.go b/internal/sqlite/dbOptions.go similarity index 100% rename from internal/sqlite/structs.go rename to internal/sqlite/dbOptions.go diff --git a/internal/sqlite/dehashed.go b/internal/sqlite/dehashed.go index ba78577..76392ad 100644 --- a/internal/sqlite/dehashed.go +++ b/internal/sqlite/dehashed.go @@ -106,15 +106,15 @@ type Result struct { } func (Result) TableName() string { - return "results" + return "dehashed" } type DehashedResults struct { Results []Result `json:"results"` } -func (dr *DehashedResults) ExtractCredentials() []Creds { - var creds []Creds +func (dr *DehashedResults) ExtractUsers() []User { + var creds []User results := dr.Results @@ -126,16 +126,22 @@ func (dr *DehashedResults) ExtractCredentials() []Creds { email = r.Email[0] } + // Get first username if available + username := "" + if len(r.Username) > 0 { + username = r.Username[0] + } + // Get first password password := r.Password[0] - cred := Creds{Email: email, Password: password} + cred := User{Email: email, Password: password, Username: username} creds = append(creds, cred) } } go func() { - err := StoreDehashedCreds(creds) + err := StoreUsers(creds) if err != nil { zap.L().Error("store_creds", zap.String("message", "failed to store creds"), @@ -148,18 +154,11 @@ func (dr *DehashedResults) ExtractCredentials() []Creds { return creds } -type Creds struct { - gorm.Model - Email string `json:"email" yaml:"email" xml:"email" gorm:"uniqueIndex:idx_email_username_password"` - Username string `json:"username" yaml:"username" xml:"username" gorm:"uniqueIndex:idx_email_username_password"` - Password string `json:"password" yaml:"password" xml:"password" gorm:"uniqueIndex:idx_email_username_password"` -} - -func (Creds) TableName() string { +func (User) TableName() string { return "creds" } -func (c Creds) ToString() string { +func (c User) ToString() string { return fmt.Sprintf("%s%s%s", c.Username, "%", c.Password) } @@ -197,38 +196,6 @@ func StoreDehashedResults(results DehashedResults) error { return lastErr } -func StoreDehashedCreds(creds []Creds) error { - if len(creds) == 0 { - return nil - } - - zap.L().Info("Storing credentials", zap.Int("count", len(creds))) - db := GetDB() - - // Use batch insert with conflict handling - // This will insert records in batches and continue even if some fail - const batchSize = 100 - var lastErr error - - for i := 0; i < len(creds); i += batchSize { - end := i + batchSize - if end > len(creds) { - end = len(creds) - } - - batch := creds[i:end] - // Use Clauses with OnConflict DoNothing to skip conflicts - err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error - if err != nil { - zap.L().Warn("Error storing some credentials", zap.Error(err)) - lastErr = err - // Continue with next batch despite error - } - } - - return lastErr -} - func StoreDehashedQueryOptions(queryOptions *QueryOptions) error { db := GetDB() return db.Create(queryOptions).Error diff --git a/internal/sqlite/ports.go b/internal/sqlite/ports.go new file mode 100644 index 0000000..fef43c1 --- /dev/null +++ b/internal/sqlite/ports.go @@ -0,0 +1 @@ +package sqlite diff --git a/internal/sqlite/subdomains.go b/internal/sqlite/subdomains.go new file mode 100644 index 0000000..785c978 --- /dev/null +++ b/internal/sqlite/subdomains.go @@ -0,0 +1,45 @@ +package sqlite + +import ( + "go.uber.org/zap" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +type Subdomain struct { + gorm.Model + Domain string `json:"domain" yaml:"domain" xml:"domain"` + Subdomain string `json:"subdomain" yaml:"subdomain" xml:"subdomain" gorm:"uniqueIndex:idx_subdomain"` +} + +func StoreSubdomains(subs []Subdomain) error { + if len(subs) == 0 { + return nil + } + + zap.L().Info("Storing subdomains", zap.Int("count", len(subs))) + db := GetDB() + + // Use batch insert with conflict handling + // This will insert records in batches and continue even if some fail + const batchSize = 100 + var lastErr error + + for i := 0; i < len(subs); i += batchSize { + end := i + batchSize + if end > len(subs) { + end = len(subs) + } + + batch := subs[i:end] + // Use Clauses with OnConflict DoNothing to skip conflicts + err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error + if err != nil { + zap.L().Warn("Error storing some credentials", zap.Error(err)) + lastErr = err + // Continue with next batch despite error + } + } + + return lastErr +} diff --git a/internal/sqlite/users.go b/internal/sqlite/users.go new file mode 100644 index 0000000..e13865d --- /dev/null +++ b/internal/sqlite/users.go @@ -0,0 +1,58 @@ +package sqlite + +import ( + "go.uber.org/zap" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +type User struct { + gorm.Model + Company string `json:"company" yaml:"company" xml:"company"` + Position string `json:"position" yaml:"position" xml:"position"` + Department string `json:"department" yaml:"department" xml:"department"` + PhoneNumber string `json:"phone_number" yaml:"phone_number" xml:"phone_number"` + FullName string `json:"full_name" yaml:"full_name" xml:"full_name"` + Phone string `json:"phone" yaml:"phone" xml:"phone"` + Linkedin string `json:"linkedin" yaml:"linkedin" xml:"linkedin"` + Twitter string `json:"twitter" yaml:"twitter" xml:"twitter"` + Facebook string `json:"facebook" yaml:"facebook" xml:"facebook"` + Instagram string `json:"instagram" yaml:"instagram" xml:"instagram"` + Youtube string `json:"youtube" yaml:"youtube" xml:"youtube"` + Gravatar string `json:"gravatar" yaml:"gravatar" xml:"gravatar"` + Email string `json:"email" yaml:"email" xml:"email" gorm:"uniqueIndex:idx_email_username_password"` + Username string `json:"username" yaml:"username" xml:"username" gorm:"uniqueIndex:idx_email_username_password"` + Password string `json:"password" yaml:"password" xml:"password" gorm:"uniqueIndex:idx_email_username_password"` +} + +func StoreUsers(users []User) error { + if len(users) == 0 { + return nil + } + + zap.L().Info("Storing credentials", zap.Int("count", len(users))) + db := GetDB() + + // Use batch insert with conflict handling + // This will insert records in batches and continue even if some fail + const batchSize = 100 + var lastErr error + + for i := 0; i < len(users); i += batchSize { + end := i + batchSize + if end > len(users) { + end = len(users) + } + + batch := users[i:end] + // Use Clauses with OnConflict DoNothing to skip conflicts + err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error + if err != nil { + zap.L().Warn("Error storing some credentials", zap.Error(err)) + lastErr = err + // Continue with next batch despite error + } + } + + return lastErr +} diff --git a/internal/sqlite/whois.go b/internal/sqlite/whois.go index adc9d59..c360c96 100644 --- a/internal/sqlite/whois.go +++ b/internal/sqlite/whois.go @@ -536,37 +536,6 @@ func StoreWhoisRecord(whoisRecord WhoisRecord) error { return nil } -func StoreWhoisSubdomainRecords(subdomainRecords []SubdomainRecord) error { - if len(subdomainRecords) == 0 { - return nil - } - - zap.L().Info("Storing subdomain records", zap.Int("count", len(subdomainRecords))) - db := GetDB() - - // Use batch insert with conflict handling - const batchSize = 100 - var lastErr error - - for i := 0; i < len(subdomainRecords); i += batchSize { - end := i + batchSize - if end > len(subdomainRecords) { - end = len(subdomainRecords) - } - - batch := subdomainRecords[i:end] - // Use Clauses with OnConflict DoNothing to skip conflicts - err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error - if err != nil { - zap.L().Warn("Error storing some subdomain records", zap.Error(err)) - lastErr = err - // Continue with next batch despite error - } - } - - return lastErr -} - func StoreWhoisHistoryRecords(historyRecords []HistoryRecord) error { if len(historyRecords) == 0 { return nil