From f23bd0411496149192f06541f8341d86f5e6c6c7 Mon Sep 17 00:00:00 2001 From: Evan Hosinski Date: Tue, 7 Apr 2026 09:37:41 -0400 Subject: [PATCH] Fixed issue where query only exported to file --- cmd/query.go | 23 +++++------ internal/dehashed/datawells.go | 30 ++++++++------ internal/dehashed/dehashed_test.go | 22 +++++++++++ internal/export/dehashed.go | 63 +++++++++++++++++------------- internal/export/dehashed_test.go | 29 ++++++++++++++ internal/sqlite/db.go | 4 +- internal/sqlite/db_test.go | 18 +++++++++ 7 files changed, 133 insertions(+), 56 deletions(-) create mode 100644 internal/export/dehashed_test.go create mode 100644 internal/sqlite/db_test.go diff --git a/cmd/query.go b/cmd/query.go index 01c6c02..2d65f83 100644 --- a/cmd/query.go +++ b/cmd/query.go @@ -18,15 +18,16 @@ func init() { rootCmd.AddCommand(queryCmd) // Add flags specific to whois command - queryCmd.Flags().StringVarP(&dbQueryTableName, "table", "t", "", "Table to query (results, creds, whois, subdomains, history, runs)") + queryCmd.Flags().StringVarP(&dbQueryTableName, "table", "t", "", "Table to query (dehashed, users, whois, subdomains, lookup, runs)") queryCmd.Flags().IntVarP(&dbQueryLimitRows, "limit", "l", 100, "Limit number of results") queryCmd.Flags().StringVarP(&dbQueryNotNull, "not-null", "n", "", "Filter for non-null values (comma-separated list, e.g., 'password,email')") queryCmd.Flags().StringVarP(&dbQueryColumns, "columns", "c", "", "Columns to display in output (comma-separated list, e.g., 'username,email,password')") queryCmd.Flags().StringVarP(&dbQueryUserQuery, "user-query", "q", "", "User query to execute") queryCmd.Flags().StringVarP(&dbQueryRawQuery, "raw-query", "r", "", "Raw SQL query to execute") queryCmd.Flags().BoolVarP(&dbQueryListAll, "list-all", "a", false, "List all tables and their columns") + queryCmd.Flags().BoolVarP(&dbQueryExport, "export", "x", false, "Export results to file using --file and --format") queryCmd.Flags().StringVarP(&dbQueryFormat, "format", "f", "json", "Output format (json, yaml, xml, txt, grep)") - queryCmd.Flags().StringVarP(&dbQueryFile, "file", "o", "query", "File to output results to") + queryCmd.Flags().StringVarP(&dbQueryFile, "file", "o", "query", "File to output results to when --export is set") // Add mutually exclusive flags to query and raw-query // Cannot use query and raw-query at the same time @@ -45,6 +46,7 @@ var ( dbQueryUserQuery string dbQueryRawQuery string dbQueryListAll bool + dbQueryExport bool dbQueryFormat string dbQueryFile string @@ -52,7 +54,7 @@ var ( Use: "query", Short: "Query the database", Long: `Query the database for various information. -If file is specified, results are written to file and not displayed in the terminal.`, +Use --export with --file and --format to write results to a file instead of displaying them.`, Run: func(cmd *cobra.Command, args []string) { // If list-all flag is set, list all tables and columns if dbQueryListAll { @@ -70,14 +72,14 @@ If file is specified, results are written to file and not displayed in the termi // Validate table name if dbQueryTableName == "" { fmt.Println("[!] Error: Table name is required. Use -t or --table to specify a table.") - fmt.Println("[*] Available tables: results, creds, whois, subdomains, history, runs") + fmt.Println("[*] Available tables: dehashed, users, whois, subdomains, lookup, runs") fmt.Println("[*] Use --list-all to see all tables and their columns.") return } if !isValidTable(dbQueryTableName) { fmt.Printf("[!] Error: Unknown table '%s'.\n", dbQueryTableName) - fmt.Println("[*] Available tables: results, creds, whois, subdomains, history, runs") + fmt.Println("[*] Available tables: dehashed, users, whois, subdomains, lookup, runs") fmt.Println("[*] Use --list-all to see all tables and their columns.") return } @@ -124,7 +126,7 @@ If file is specified, results are written to file and not displayed in the termi table := sqlite.GetTable(dbQueryTableName) if table == sqlite.UnknownTable { fmt.Printf("[!] Error: Unknown table type '%s'.\n", dbQueryTableName) - fmt.Println("[*] Available tables: results, creds, whois, subdomains, history, runs") + fmt.Println("[*] Available tables: dehashed, users, whois, subdomains, lookup, runs") fmt.Println("[*] Use --list-all to see all tables and their columns.") return } @@ -203,8 +205,7 @@ func tableQuery(table sqlite.Table) { return } - // Export results if file name is specified - if len(strings.TrimSpace(dbQueryFile)) > 0 { + if dbQueryExport { fmt.Println("[*] Exporting results to file...") if debugGlobal { @@ -244,8 +245,6 @@ func tableQuery(table sqlite.Table) { return } - fmt.Println("[*] Querying Database...") - // Prepare data for pretty.Table headers := cols var tableRows [][]string @@ -336,7 +335,7 @@ func rawDBQuery() { return } - if len(strings.TrimSpace(dbQueryFile)) > 0 { + if dbQueryExport { fmt.Println("[*] Exporting results to file...") if debugGlobal { @@ -377,8 +376,6 @@ func rawDBQuery() { return } - fmt.Println("[*] Querying Database...") - // Prepare data for pretty.Table headers := columns var tableRows [][]string diff --git a/internal/dehashed/datawells.go b/internal/dehashed/datawells.go index e30ca3b..c8f175c 100644 --- a/internal/dehashed/datawells.go +++ b/internal/dehashed/datawells.go @@ -163,20 +163,24 @@ func (dwr DataWellsResponse) String() string { } func dataWellGreppable(well DataWell) string { - fields := []string{ - "name=" + cleanGreppableValue(well.Name), - "date=" + cleanGreppableValue(well.Date), - "records=" + strconv.Itoa(well.Records), - "is_sensitive=" + strconv.FormatBool(well.IsSensitive), - "data=" + cleanGreppableValue(well.Data), - "description=" + cleanGreppableValue(well.Description), - } - return strings.Join(fields, "\t") + var fields []string + fields = appendDataWellGreppableField(fields, "name", well.Name) + fields = appendDataWellGreppableField(fields, "date", well.Date) + fields = appendDataWellGreppableField(fields, "records", strconv.Itoa(well.Records)) + fields = appendDataWellGreppableField(fields, "is_sensitive", strconv.FormatBool(well.IsSensitive)) + fields = appendDataWellGreppableField(fields, "data", well.Data) + fields = appendDataWellGreppableField(fields, "description", well.Description) + return strings.Join(fields, " ") } func cleanGreppableValue(value string) string { - value = strings.ReplaceAll(value, "\r", " ") - value = strings.ReplaceAll(value, "\n", " ") - value = strings.ReplaceAll(value, "\t", " ") - return strings.TrimSpace(value) + return strings.Join(strings.Fields(value), "_") +} + +func appendDataWellGreppableField(fields []string, key, value string) []string { + value = cleanGreppableValue(value) + if value == "" { + return fields + } + return append(fields, fmt.Sprintf("%s=%s", key, value)) } diff --git a/internal/dehashed/dehashed_test.go b/internal/dehashed/dehashed_test.go index d89a378..4f0d1a8 100644 --- a/internal/dehashed/dehashed_test.go +++ b/internal/dehashed/dehashed_test.go @@ -70,3 +70,25 @@ func TestDataWellsURLDoesNotRequireAPIKey(t *testing.T) { } } } + +func TestDataWellGreppableUsesSpaceSeparatedNonEmptyTokens(t *testing.T) { + got := dataWellGreppable(DataWell{ + Name: "Example Breach", + Date: "2025-03-01", + Records: 500000, + IsSensitive: true, + Data: "name,email,address", + }) + + if strings.Contains(got, "\t") { + t.Fatalf("greppable output contains tab: %q", got) + } + if strings.Contains(got, "description=") { + t.Fatalf("greppable output contains empty field: %q", got) + } + for _, want := range []string{"name=Example_Breach", "date=2025-03-01", "records=500000", "is_sensitive=true", "data=name,email,address"} { + if !strings.Contains(got, want) { + t.Fatalf("greppable output = %q, want token %q", got, want) + } + } +} diff --git a/internal/export/dehashed.go b/internal/export/dehashed.go index 56067f9..b5d47c1 100644 --- a/internal/export/dehashed.go +++ b/internal/export/dehashed.go @@ -34,8 +34,11 @@ func WriteCredsToFile(creds []sqlite.User, outputFile string, fileType files.Fil case files.GREPPABLE: var outStrings []string for _, c := range creds { - outStrings = append(outStrings, fmt.Sprintf("email=%s\tusername=%s\tpassword=%s\n", - greppableValue(c.Email), greppableValue(c.Username), greppableValue(c.Password))) + var fields []string + fields = appendGreppableField(fields, "email", c.Email) + fields = appendGreppableField(fields, "username", c.Username) + fields = appendGreppableField(fields, "password", c.Password) + outStrings = append(outStrings, strings.Join(fields, " ")+"\n") } data = []byte(strings.Join(outStrings, "")) default: @@ -145,9 +148,9 @@ func WriteQueryResultsToFile(results []map[string]interface{}, outputFile string rowStrings := make([]string, 0, len(keys)) for _, k := range keys { - rowStrings = append(rowStrings, fmt.Sprintf("%s=%s", k, greppableAnyValue(r[k]))) + rowStrings = appendGreppableField(rowStrings, k, greppableAnyValue(r[k])) } - outStrings = append(outStrings, strings.Join(rowStrings, "\t")+"\n") + outStrings = append(outStrings, strings.Join(rowStrings, " ")+"\n") } data = []byte(strings.Join(outStrings, "")) default: @@ -163,26 +166,25 @@ func WriteQueryResultsToFile(results []map[string]interface{}, outputFile string } func dehashedResultGreppable(r sqlite.Result) string { - fields := []string{ - "id=" + greppableValue(r.DehashedId), - "email=" + greppableValue(strings.Join(r.Email, ",")), - "ip_address=" + greppableValue(strings.Join(r.IpAddress, ",")), - "username=" + greppableValue(strings.Join(r.Username, ",")), - "password=" + greppableValue(strings.Join(r.Password, ",")), - "hashed_password=" + greppableValue(strings.Join(r.HashedPassword, ",")), - "hash_type=" + greppableValue(r.HashType), - "name=" + greppableValue(strings.Join(r.Name, ",")), - "vin=" + greppableValue(strings.Join(r.Vin, ",")), - "license_plate=" + greppableValue(strings.Join(r.LicensePlate, ",")), - "url=" + greppableValue(strings.Join(r.Url, ",")), - "social=" + greppableValue(strings.Join(r.Social, ",")), - "cryptocurrency_address=" + greppableValue(strings.Join(r.CryptoCurrencyAddress, ",")), - "address=" + greppableValue(strings.Join(r.Address, ",")), - "phone=" + greppableValue(strings.Join(r.Phone, ",")), - "company=" + greppableValue(strings.Join(r.Company, ",")), - "database_name=" + greppableValue(r.DatabaseName), - } - return strings.Join(fields, "\t") + var fields []string + fields = appendGreppableField(fields, "id", r.DehashedId) + fields = appendGreppableField(fields, "email", strings.Join(r.Email, ",")) + fields = appendGreppableField(fields, "ip_address", strings.Join(r.IpAddress, ",")) + fields = appendGreppableField(fields, "username", strings.Join(r.Username, ",")) + fields = appendGreppableField(fields, "password", strings.Join(r.Password, ",")) + fields = appendGreppableField(fields, "hashed_password", strings.Join(r.HashedPassword, ",")) + fields = appendGreppableField(fields, "hash_type", r.HashType) + fields = appendGreppableField(fields, "name", strings.Join(r.Name, ",")) + fields = appendGreppableField(fields, "vin", strings.Join(r.Vin, ",")) + fields = appendGreppableField(fields, "license_plate", strings.Join(r.LicensePlate, ",")) + fields = appendGreppableField(fields, "url", strings.Join(r.Url, ",")) + fields = appendGreppableField(fields, "social", strings.Join(r.Social, ",")) + fields = appendGreppableField(fields, "cryptocurrency_address", strings.Join(r.CryptoCurrencyAddress, ",")) + fields = appendGreppableField(fields, "address", strings.Join(r.Address, ",")) + fields = appendGreppableField(fields, "phone", strings.Join(r.Phone, ",")) + fields = appendGreppableField(fields, "company", strings.Join(r.Company, ",")) + fields = appendGreppableField(fields, "database_name", r.DatabaseName) + return strings.Join(fields, " ") } func greppableAnyValue(value interface{}) string { @@ -205,10 +207,15 @@ func greppableAnyValue(value interface{}) string { } func greppableValue(value string) string { - value = strings.ReplaceAll(value, "\r", " ") - value = strings.ReplaceAll(value, "\n", " ") - value = strings.ReplaceAll(value, "\t", " ") - return strings.TrimSpace(value) + return strings.Join(strings.Fields(value), "_") +} + +func appendGreppableField(fields []string, key, value string) []string { + value = greppableValue(value) + if value == "" { + return fields + } + return append(fields, fmt.Sprintf("%s=%s", key, value)) } func WriteWhoIsHistoryToFile(results []sqlite.HistoryRecord, outputFile string, fileType files.FileType) error { diff --git a/internal/export/dehashed_test.go b/internal/export/dehashed_test.go new file mode 100644 index 0000000..010d864 --- /dev/null +++ b/internal/export/dehashed_test.go @@ -0,0 +1,29 @@ +package export + +import ( + "strings" + "testing" + + "crowsnest/internal/sqlite" +) + +func TestDehashedResultGreppableUsesSpaceSeparatedNonEmptyTokens(t *testing.T) { + got := dehashedResultGreppable(sqlite.Result{ + DehashedId: "123", + Name: []string{"Hargrave Mall"}, + Address: []string{"irving tx"}, + Url: []string{"gdt.com", "GDT.COM"}, + }) + + if strings.Contains(got, "\t") { + t.Fatalf("greppable output contains tab: %q", got) + } + if strings.Contains(got, "vin=") { + t.Fatalf("greppable output contains empty field: %q", got) + } + for _, want := range []string{"id=123", "name=Hargrave_Mall", "address=irving_tx", "url=gdt.com,GDT.COM"} { + if !strings.Contains(got, want) { + t.Fatalf("greppable output = %q, want token %q", got, want) + } + } +} diff --git a/internal/sqlite/db.go b/internal/sqlite/db.go index d48472b..91ad1ec 100644 --- a/internal/sqlite/db.go +++ b/internal/sqlite/db.go @@ -90,11 +90,11 @@ const ( func GetTable(userInput string) Table { switch strings.ToLower(userInput) { - case "results": + case "dehashed", "results": return ResultsTable case "runs": return RunsTable - case "creds": + case "users", "creds": return CredsTable case "whois": return WhoIsTable diff --git a/internal/sqlite/db_test.go b/internal/sqlite/db_test.go new file mode 100644 index 0000000..d06b76c --- /dev/null +++ b/internal/sqlite/db_test.go @@ -0,0 +1,18 @@ +package sqlite + +import "testing" + +func TestGetTableAcceptsDisplayedTableNames(t *testing.T) { + tests := map[string]Table{ + "dehashed": ResultsTable, + "results": ResultsTable, + "users": CredsTable, + "creds": CredsTable, + } + + for input, want := range tests { + if got := GetTable(input); got != want { + t.Fatalf("GetTable(%q) = %v, want %v", input, got, want) + } + } +}