diff --git a/README.md b/README.md index 1798089..0056905 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ To configure the database location: CrowsNest requires an API key from Dehashed. Set it up with: ![Alt text](.img/set-dehashed.png "Set Dehashed Key") ```bash -ar1ste1a@kali:~$ crowsnest set-dehashed +ar1ste1a@kali:~$ crowsnest set dehashed ``` ### Simple Query @@ -134,10 +134,23 @@ crowsnest dehashed -R -E 'joh?n(ath[oa]n)' -D hotmail.com' CrowsNest is capable of handling output formats. The default output format is JSON. To change the output format, use the `-f` flag. -CrowsNest currently supports JSON, YAML, XML, and TEXT output formats. +CrowsNest currently supports JSON, YAML, XML, TEXT, and GREP output formats. ``` go # Return matches for usernames exactly matching "admin" and write to text file 'admins_file.txt' crowsnest dehashed -U admin -o admins_file -f txt + +# Return one key=value record per line in a greppable file 'admins_file.grep' +crowsnest dehashed -U admin -o admins_file -f grep +``` + +### Data Wells +DeHashed data wells are free to query and do not require a paid API account. +``` go +# List the first page of data wells and write 'data_wells.json' +crowsnest dehashed data-wells + +# Sort by record count and write one key=value record per line +crowsnest dehashed data-wells --sort records-DESC --count 50 -f grep -o data_wells ``` --- @@ -216,11 +229,11 @@ crowsnest whois -n google.com ## 🌐 Hunter.io CrowsNest supports Hunter.io lookups. Hunter.io lookups require a separate API Key from the Dehashed API. -This can be set using the `set-hunter` command. +This can be set using the `set hunter` command. ![Alt text](.img/set-hunter.png "Set Dehashed Key") ```bash # Set the Hunter.io API key -crowsnest set-hunter +crowsnest set hunter ``` ### Domain Search diff --git a/cmd/dehashed.go b/cmd/dehashed.go index 6cceabf..82052e3 100644 --- a/cmd/dehashed.go +++ b/cmd/dehashed.go @@ -4,6 +4,8 @@ import ( "crowsnest/internal/badger" "crowsnest/internal/debug" "crowsnest/internal/dehashed" + "crowsnest/internal/files" + "crowsnest/internal/pretty" "crowsnest/internal/sqlite" "fmt" "github.com/spf13/cobra" @@ -13,19 +15,20 @@ import ( func init() { // Add api command to root command rootCmd.AddCommand(dehashedCmd) + dehashedCmd.AddCommand(dehashedDataWellsCmd) // Add flags specific to api command - dehashedCmd.Flags().IntVarP(&maxRecords, "max-records", "m", 30000, "Maximum amount of records to return") + dehashedCmd.Flags().IntVarP(&maxRecords, "max-records", "m", 50000, "Maximum total records to return (max 50000)") dehashedCmd.Flags().IntVarP(&maxRequests, "max-requests", "r", -1, "Maximum number of requests to make") dehashedCmd.Flags().IntVarP(&startingPage, "starting-page", "s", 1, "Starting page for requests") dehashedCmd.Flags().BoolVarP(&printBalance, "print-balance", "b", false, "Print remaining balance after requests") dehashedCmd.Flags().BoolVarP(®exMatch, "regex-match", "R", false, "Use regex matching on query fields") dehashedCmd.Flags().BoolVarP(&wildcardMatch, "wildcard-match", "W", false, "Use wildcard matching on query fields (Use ? to replace a single character, and * for multiple characters)") dehashedCmd.Flags().BoolVarP(&credsOnly, "creds-only", "C", false, "Return credentials only") - dehashedCmd.Flags().StringVarP(&outputFormat, "format", "f", "json", "Output format (json, yaml, xml, txt)") - dehashedCmd.Flags().StringVarP(&outputFile, "output", "o", "query", "File to output results to including extension") + dehashedCmd.Flags().StringVarP(&outputFormat, "format", "f", "json", "Output format (json, yaml, xml, txt, grep)") + dehashedCmd.Flags().StringVarP(&outputFile, "output", "o", "query", "File to output results to without extension") dehashedCmd.Flags().StringVarP(&usernameQuery, "username", "U", "", "Username query") - dehashedCmd.Flags().StringVarP(&emailQuery, "email-query", "E", "", "HunterEmail query") + dehashedCmd.Flags().StringVarP(&emailQuery, "email-query", "E", "", "Email query") dehashedCmd.Flags().StringVarP(&ipQuery, "ip", "I", "", "IP address query") dehashedCmd.Flags().StringVarP(&domainQuery, "domain", "D", "", "Domain query") dehashedCmd.Flags().StringVarP(&passwordQuery, "password", "P", "", "Password query") @@ -40,6 +43,12 @@ func init() { // Add mutually exclusive flags to wildcard match and regex match dehashedCmd.MarkFlagsMutuallyExclusive("regex-match", "wildcard-match") + + dehashedDataWellsCmd.Flags().IntVar(&dataWellsCount, "count", 20, "Number of data wells to return (20 or 50)") + dehashedDataWellsCmd.Flags().IntVarP(&dataWellsPage, "page", "p", 1, "Data wells page to request") + dehashedDataWellsCmd.Flags().StringVar(&dataWellsSort, "sort", "", "Sort data wells by added, name, date, or records; optionally suffix -ASC or -DESC") + dehashedDataWellsCmd.Flags().StringVarP(&dataWellsOutputFormat, "format", "f", "json", "Output format (json, yaml, xml, txt, grep)") + dehashedDataWellsCmd.Flags().StringVarP(&dataWellsOutputFile, "output", "o", "data_wells", "File to output data wells to without extension") } var ( @@ -66,6 +75,11 @@ var ( phoneQuery string socialQuery string cryptoCurrencyAddressQuery string + dataWellsCount int + dataWellsPage int + dataWellsSort string + dataWellsOutputFormat string + dataWellsOutputFile string // Query command dehashedCmd = &cobra.Command{ @@ -77,7 +91,7 @@ var ( // Validate credentials if key == "" { - fmt.Println("API key is required. Set the key with the \"set-key\" command. [crowsnest set-key ]") + fmt.Println("API key is required. Set the key with the \"set dehashed\" command. [crowsnest set dehashed ]") return } @@ -133,9 +147,54 @@ var ( } }, } + + dehashedDataWellsCmd = &cobra.Command{ + Use: "data-wells", + Short: "List DeHashed data wells", + Long: `List DeHashed data wells. This endpoint is free and does not require a DeHashed API key or subscription.`, + Run: func(cmd *cobra.Command, args []string) { + client := dehashed.NewDehashedClientV2("", debugGlobal) + response, err := client.DataWells(dehashed.DataWellsRequest{ + Count: dataWellsCount, + Page: dataWellsPage, + Sort: dataWellsSort, + }) + if err != nil { + fmt.Printf("[!] Error querying data wells: %v\n", err) + return + } + + fType := files.GetFileType(dataWellsOutputFormat) + if dataWellsOutputFile != "" { + fmt.Printf("[*] Writing data wells to file: %s%s\n", dataWellsOutputFile, fType.Extension()) + if err := dehashed.WriteDataWellsToFile(response, dataWellsOutputFile, fType); err != nil { + fmt.Printf("[!] Error writing data wells to file: %v\n", err) + return + } + } + + fmt.Printf("[+] Retrieved %d data wells (total: %d, next page: %t)\n", len(response.DataWells), response.Total, response.NextPage) + printDataWellsTable(response.DataWells) + }, + } ) // Helper functions to get stored API credentials func getDehashedApiKey() string { return badger.GetDehashedKey() } + +func printDataWellsTable(dataWells []dehashed.DataWell) { + headers := []string{"Name", "Date", "Records", "Sensitive", "Data"} + rows := make([][]string, 0, len(dataWells)) + for _, well := range dataWells { + rows = append(rows, []string{ + well.Name, + well.Date, + fmt.Sprintf("%d", well.Records), + fmt.Sprintf("%t", well.IsSensitive), + well.Data, + }) + } + pretty.Table(headers, rows) +} diff --git a/cmd/query.go b/cmd/query.go index 8115356..01c6c02 100644 --- a/cmd/query.go +++ b/cmd/query.go @@ -25,7 +25,7 @@ func init() { queryCmd.Flags().StringVarP(&dbQueryUserQuery, "user-query", "q", "", "User query to execute") queryCmd.Flags().StringVarP(&dbQueryRawQuery, "raw-query", "r", "", "Raw SQL query to execute") queryCmd.Flags().BoolVarP(&dbQueryListAll, "list-all", "a", false, "List all tables and their columns") - queryCmd.Flags().StringVarP(&dbQueryFormat, "format", "f", "json", "Output format (json, yaml, xml, txt)") + queryCmd.Flags().StringVarP(&dbQueryFormat, "format", "f", "json", "Output format (json, yaml, xml, txt, grep)") queryCmd.Flags().StringVarP(&dbQueryFile, "file", "o", "query", "File to output results to") // Add mutually exclusive flags to query and raw-query diff --git a/cmd/root.go b/cmd/root.go index d7b9bd1..ad910ef 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -53,15 +53,23 @@ func init() { rootCmd.PersistentFlags().BoolVar(&debugGlobal, "debug", false, "Show debug information") // Add subcommands - rootCmd.AddCommand(setDehashedKeyCmd) - rootCmd.AddCommand(setHunterKeyCmd) + rootCmd.AddCommand(setCmd) rootCmd.AddCommand(setLocalDb) rootCmd.AddCommand(buyMeCoffeeCmd) + + setCmd.AddCommand(setDehashedKeyCmd) + setCmd.AddCommand(setHunterKeyCmd) +} + +var setCmd = &cobra.Command{ + Use: "set", + Short: "Set CrowsNest configuration values", + Long: "Set CrowsNest configuration values such as API keys.", } // Command to set API key var setDehashedKeyCmd = &cobra.Command{ - Use: "set-dehashed [key]", + Use: "dehashed [key]", Short: "Set and store Dehashed.com API key", Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { @@ -77,7 +85,7 @@ var setDehashedKeyCmd = &cobra.Command{ } var setHunterKeyCmd = &cobra.Command{ - Use: "set-hunter [key]", + Use: "hunter [key]", Short: "Set and store Hunter.io API key", Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { diff --git a/cmd/whois.go b/cmd/whois.go index 9fda540..85487db 100644 --- a/cmd/whois.go +++ b/cmd/whois.go @@ -59,7 +59,7 @@ var ( // Validate credentials if key == "" { - fmt.Println("API key is required. Set the key with the \"set-key\" command. [crowsnest set-key ]") + fmt.Println("API key is required. Set the key with the \"set dehashed\" command. [crowsnest set dehashed ]") return } diff --git a/internal/dehashed/clientv2.go b/internal/dehashed/clientv2.go index 5828a1d..a381f4b 100644 --- a/internal/dehashed/clientv2.go +++ b/internal/dehashed/clientv2.go @@ -198,7 +198,7 @@ func (dcv2 *DehashedClientV2) Search(searchRequest DehashedSearchRequest) (int, req.Header.Set("Dehashed-Api-Key", dcv2.apiKey) if dcv2.debug { - headers := req.Header.Clone() + headers := redactedHeaders(req.Header) h := fmt.Sprintf("Headers: %v\n", headers) debug.PrintJson(h) zap.L().Info("v2_search_debug", @@ -286,7 +286,7 @@ func (dcv2 *DehashedClientV2) Search(searchRequest DehashedSearchRequest) (int, } dcv2.results = append(dcv2.results, responseResults.Entries...) - return responseResults.TotalResults, responseResults.Balance, nil + return len(responseResults.Entries), responseResults.Balance, nil } func (dcv2 *DehashedClientV2) GetResults() sqlite.DehashedResults { @@ -304,3 +304,11 @@ func enquoteSpaced(s string) string { } return s } + +func redactedHeaders(headers http.Header) http.Header { + redacted := headers.Clone() + if redacted.Get("Dehashed-Api-Key") != "" { + redacted.Set("Dehashed-Api-Key", "[REDACTED]") + } + return redacted +} diff --git a/internal/dehashed/datawells.go b/internal/dehashed/datawells.go new file mode 100644 index 0000000..e30ca3b --- /dev/null +++ b/internal/dehashed/datawells.go @@ -0,0 +1,182 @@ +package dehashed + +import ( + "encoding/json" + "encoding/xml" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strconv" + "strings" + + "crowsnest/internal/files" + "gopkg.in/yaml.v3" +) + +const dataWellsEndpoint = "https://api.dehashed.com/data-wells" + +type DataWellsRequest struct { + Count int + Page int + Sort string +} + +type DataWellsResponse struct { + NextPage bool `json:"next_page" xml:"next_page" yaml:"next_page"` + Total int `json:"total" xml:"total" yaml:"total"` + DataWells []DataWell `json:"data_wells" xml:"data_wells" yaml:"data_wells"` +} + +type DataWell struct { + Data string `json:"data" xml:"data" yaml:"data"` + Date string `json:"date" xml:"date" yaml:"date"` + Description string `json:"description" xml:"description" yaml:"description"` + Name string `json:"name" xml:"name" yaml:"name"` + Records int `json:"records" xml:"records" yaml:"records"` + IsSensitive bool `json:"is_sensitive" xml:"is_sensitive" yaml:"is_sensitive"` +} + +func (dcv2 *DehashedClientV2) DataWells(request DataWellsRequest) (DataWellsResponse, error) { + var dataWells DataWellsResponse + + endpoint, err := dataWellsURL(request) + if err != nil { + return dataWells, err + } + + req, err := http.NewRequest("GET", endpoint, nil) + if err != nil { + return dataWells, err + } + req.Header.Set("Accept", "application/json") + + res, err := http.DefaultClient.Do(req) + if res != nil { + defer res.Body.Close() + } + if err != nil { + return dataWells, err + } + if res == nil { + return dataWells, errors.New("response was nil") + } + + body, err := io.ReadAll(res.Body) + if err != nil { + return dataWells, err + } + + if res.StatusCode != http.StatusOK { + return dataWells, fmt.Errorf("data wells request failed: status=%d body=%s", res.StatusCode, string(body)) + } + + if err := json.Unmarshal(body, &dataWells); err != nil { + return dataWells, err + } + + return dataWells, nil +} + +func dataWellsURL(request DataWellsRequest) (string, error) { + if request.Page <= 0 { + return "", errors.New("page must be 1 or greater") + } + if request.Count != 20 && request.Count != 50 { + return "", errors.New("count must be 20 or 50") + } + if request.Sort != "" && !validDataWellsSort(request.Sort) { + return "", fmt.Errorf("invalid sort %q; use added, name, date, records, optionally suffixed with -ASC or -DESC", request.Sort) + } + + values := url.Values{} + values.Set("page", strconv.Itoa(request.Page)) + values.Set("count", strconv.Itoa(request.Count)) + if request.Sort != "" { + values.Set("sort", request.Sort) + } + + return dataWellsEndpoint + "?" + values.Encode(), nil +} + +func validDataWellsSort(sortValue string) bool { + sortValue = strings.ToLower(strings.TrimSpace(sortValue)) + field := sortValue + if before, _, ok := strings.Cut(sortValue, "-"); ok { + field = before + } + + switch field { + case "added", "name", "date", "records": + return strings.HasSuffix(sortValue, "-asc") || strings.HasSuffix(sortValue, "-desc") || !strings.Contains(sortValue, "-") + default: + return false + } +} + +func WriteDataWellsToFile(dataWells DataWellsResponse, outputFile string, fileType files.FileType) error { + var data []byte + var err error + + switch fileType { + case files.JSON: + data, err = json.MarshalIndent(dataWells, "", " ") + case files.XML: + data, err = xml.MarshalIndent(dataWells, "", " ") + case files.YAML: + data, err = yaml.Marshal(dataWells) + case files.TEXT: + data = []byte(dataWells.String()) + case files.GREPPABLE: + var outStrings []string + for _, well := range dataWells.DataWells { + outStrings = append(outStrings, dataWellGreppable(well)+"\n") + } + data = []byte(strings.Join(outStrings, "")) + default: + return errors.New("unsupported file type") + } + + if err != nil { + return err + } + + return os.WriteFile(outputFile+fileType.Extension(), data, 0644) +} + +func (dwr DataWellsResponse) String() string { + var b strings.Builder + fmt.Fprintf(&b, "Total: %d\nNext Page: %t\n\n", dwr.Total, dwr.NextPage) + for _, well := range dwr.DataWells { + fmt.Fprintf(&b, "Name: %s\nDate: %s\nRecords: %d\nSensitive: %t\nData: %s\nDescription: %s\n\n", + well.Name, + well.Date, + well.Records, + well.IsSensitive, + well.Data, + well.Description, + ) + } + return b.String() +} + +func dataWellGreppable(well DataWell) string { + fields := []string{ + "name=" + cleanGreppableValue(well.Name), + "date=" + cleanGreppableValue(well.Date), + "records=" + strconv.Itoa(well.Records), + "is_sensitive=" + strconv.FormatBool(well.IsSensitive), + "data=" + cleanGreppableValue(well.Data), + "description=" + cleanGreppableValue(well.Description), + } + return strings.Join(fields, "\t") +} + +func cleanGreppableValue(value string) string { + value = strings.ReplaceAll(value, "\r", " ") + value = strings.ReplaceAll(value, "\n", " ") + value = strings.ReplaceAll(value, "\t", " ") + return strings.TrimSpace(value) +} diff --git a/internal/dehashed/dehashed.go b/internal/dehashed/dehashed.go index 385a7cc..973542e 100644 --- a/internal/dehashed/dehashed.go +++ b/internal/dehashed/dehashed.go @@ -11,14 +11,20 @@ import ( "strings" ) +const ( + maxSearchResultsPerPage = 10000 + maxSearchResultsPerQuery = 50000 +) + // Dehasher is a struct for querying the Dehashed API type Dehasher struct { - options sqlite.QueryOptions - nextPage int - debug bool - balance int - request *DehashedSearchRequest - client *DehashedClientV2 + options sqlite.QueryOptions + nextPage int + debug bool + balance int + maxResults int + request *DehashedSearchRequest + client *DehashedClientV2 } // NewDehasher creates a new Dehasher @@ -51,55 +57,55 @@ func (dh *Dehasher) getNextPage() int { // setQueries sets the number of queries to make based on the number of records and requests func (dh *Dehasher) setQueries() { - var numQueries int - if dh.debug { debug.PrintInfo("setting queries") } - switch { - case dh.options.MaxRequests == 0: + if dh.options.MaxRequests == 0 { zap.L().Error("max requests cannot be zero") fmt.Println("[!] Max Requests cannot be zero") os.Exit(1) - case dh.options.MaxRecords <= 10000 || dh.options.MaxRequests == 1: - numQueries = 1 - if dh.options.MaxRecords > 10000 { - dh.options.MaxRecords = 10000 - } - zap.L().Info("max requests set to 1", zap.Int("max_records", dh.options.MaxRecords)) - case dh.options.MaxRequests < 0 && dh.options.MaxRecords > 20000: - numQueries = 3 - dh.options.MaxRecords = 10000 - zap.L().Info("max requests set to 3", zap.Int("max_records", dh.options.MaxRecords)) - case dh.options.MaxRequests < 0 && dh.options.MaxRecords > 10000: - numQueries = 2 - dh.options.MaxRecords = 10000 - zap.L().Info("max requests set to 2", zap.Int("max_records", dh.options.MaxRecords)) - case dh.options.MaxRecords < 0 && dh.options.MaxRecords < 10000: - numQueries = 1 - zap.L().Info("max requests set to 1", zap.Int("max_records", dh.options.MaxRecords)) - case dh.options.MaxRequests == 2 && dh.options.MaxRecords > 20000: - numQueries = 2 - dh.options.MaxRecords = 10000 - zap.L().Info("max requests set to 2", zap.Int("max_records", dh.options.MaxRecords)) - case dh.options.MaxRequests == 2 && dh.options.MaxRecords <= 10000: - numQueries = 1 - zap.L().Info("max requests set to 1", zap.Int("max_records", dh.options.MaxRecords)) - default: - numQueries = 3 - dh.options.MaxRecords = 10000 - zap.L().Info("max requests set to 3", zap.Int("max_records", dh.options.MaxRecords)) } + requestedMaxResults := dh.options.MaxRecords + if requestedMaxResults <= 0 { + requestedMaxResults = maxSearchResultsPerQuery + } + if requestedMaxResults > maxSearchResultsPerQuery { + requestedMaxResults = maxSearchResultsPerQuery + } + + pageSize := requestedMaxResults + if pageSize > maxSearchResultsPerPage { + pageSize = maxSearchResultsPerPage + } + + numQueries := (requestedMaxResults + pageSize - 1) / pageSize + if dh.options.MaxRequests > 0 && dh.options.MaxRequests < numQueries { + numQueries = dh.options.MaxRequests + } + + dh.maxResults = requestedMaxResults + if requestLimit := numQueries * pageSize; requestLimit < dh.maxResults { + dh.maxResults = requestLimit + } + + dh.options.MaxRecords = pageSize dh.options.MaxRequests = numQueries + zap.L().Info("dehashed_search_pagination", + zap.Int("max_results", dh.maxResults), + zap.Int("page_size", dh.options.MaxRecords), + zap.Int("max_requests", dh.options.MaxRequests), + ) + if dh.debug { debug.PrintInfo(fmt.Sprintf("setting max requests: %d", numQueries)) - debug.PrintInfo(fmt.Sprintf("setting max records: %d", dh.options.MaxRecords)) + debug.PrintInfo(fmt.Sprintf("setting page size: %d", dh.options.MaxRecords)) + debug.PrintInfo(fmt.Sprintf("setting max results: %d", dh.maxResults)) } - fmt.Printf("Making %d Requests for %d Records (%d Total)\n", dh.options.MaxRequests, dh.options.MaxRecords, dh.options.MaxRequests*dh.options.MaxRecords) + fmt.Printf("Making %d Requests for up to %d Records (%d per request)\n", dh.options.MaxRequests, dh.maxResults, dh.options.MaxRecords) } // Start starts the querying process @@ -151,7 +157,7 @@ func (dh *Dehasher) Start() { fmt.Printf(" [-] Not enough entries, ending queries\n") break } else { - fmt.Printf(" [+] Retrieved %d records\n", dh.options.MaxRecords) + fmt.Printf(" [+] Retrieved %d records\n", count) } if dh.options.PrintBalance { @@ -211,6 +217,9 @@ func (dh *Dehasher) buildRequest() { func (dh *Dehasher) parseResults() { zap.L().Info("extracting_credentials") results := dh.client.GetResults() + if dh.maxResults > 0 && len(results.Results) > dh.maxResults { + results.Results = results.Results[:dh.maxResults] + } creds := results.ExtractUsers() fmt.Printf(" [+] Discovered %d Credentials\n", len(creds)) err := sqlite.StoreUsers(creds) diff --git a/internal/dehashed/dehashed_test.go b/internal/dehashed/dehashed_test.go new file mode 100644 index 0000000..d89a378 --- /dev/null +++ b/internal/dehashed/dehashed_test.go @@ -0,0 +1,72 @@ +package dehashed + +import ( + "strings" + "testing" + + "crowsnest/internal/sqlite" +) + +func TestSetQueriesCapsSearchAtFiftyThousandResults(t *testing.T) { + options := &sqlite.QueryOptions{ + MaxRecords: 75000, + MaxRequests: -1, + StartingPage: 1, + } + + dehasher := NewDehasher(options) + + if dehasher.maxResults != maxSearchResultsPerQuery { + t.Fatalf("maxResults = %d, want %d", dehasher.maxResults, maxSearchResultsPerQuery) + } + if dehasher.options.MaxRecords != maxSearchResultsPerPage { + t.Fatalf("page size = %d, want %d", dehasher.options.MaxRecords, maxSearchResultsPerPage) + } + if dehasher.options.MaxRequests != 5 { + t.Fatalf("max requests = %d, want 5", dehasher.options.MaxRequests) + } +} + +func TestSetQueriesHonorsExplicitRequestLimit(t *testing.T) { + options := &sqlite.QueryOptions{ + MaxRecords: 50000, + MaxRequests: 1, + StartingPage: 1, + } + + dehasher := NewDehasher(options) + + if dehasher.maxResults != maxSearchResultsPerPage { + t.Fatalf("maxResults = %d, want %d", dehasher.maxResults, maxSearchResultsPerPage) + } + if dehasher.options.MaxRecords != maxSearchResultsPerPage { + t.Fatalf("page size = %d, want %d", dehasher.options.MaxRecords, maxSearchResultsPerPage) + } + if dehasher.options.MaxRequests != 1 { + t.Fatalf("max requests = %d, want 1", dehasher.options.MaxRequests) + } +} + +func TestDataWellsURLDoesNotRequireAPIKey(t *testing.T) { + got, err := dataWellsURL(DataWellsRequest{ + Count: 50, + Page: 2, + Sort: "records-DESC", + }) + if err != nil { + t.Fatalf("dataWellsURL returned error: %v", err) + } + + if !strings.HasPrefix(got, dataWellsEndpoint+"?") { + t.Fatalf("url = %q, want prefix %q", got, dataWellsEndpoint+"?") + } + gotLower := strings.ToLower(got) + if strings.Contains(gotLower, "api_key") || strings.Contains(gotLower, "dehashed-api-key") { + t.Fatalf("url contains API key material: %q", got) + } + for _, want := range []string{"count=50", "page=2", "sort=records-DESC"} { + if !strings.Contains(got, want) { + t.Fatalf("url = %q, want %q", got, want) + } + } +} diff --git a/internal/export/dehashed.go b/internal/export/dehashed.go index f9df613..56067f9 100644 --- a/internal/export/dehashed.go +++ b/internal/export/dehashed.go @@ -8,8 +8,8 @@ import ( "errors" "fmt" "gopkg.in/yaml.v3" - "io/ioutil" "os" + "sort" "strings" "time" ) @@ -31,6 +31,13 @@ func WriteCredsToFile(creds []sqlite.User, outputFile string, fileType files.Fil outStrings = append(outStrings, c.ToString()+"\n") } data = []byte(strings.Join(outStrings, "")) + case files.GREPPABLE: + var outStrings []string + for _, c := range creds { + outStrings = append(outStrings, fmt.Sprintf("email=%s\tusername=%s\tpassword=%s\n", + greppableValue(c.Email), greppableValue(c.Username), greppableValue(c.Password))) + } + data = []byte(strings.Join(outStrings, "")) default: return errors.New("unsupported file type") } @@ -65,6 +72,12 @@ func WriteToFile(results sqlite.DehashedResults, outputFile string, fileType fil outStrings = append(outStrings, out) } data = []byte(strings.Join(outStrings, "")) + case files.GREPPABLE: + var outStrings []string + for _, r := range result { + outStrings = append(outStrings, dehashedResultGreppable(r)+"\n") + } + data = []byte(strings.Join(outStrings, "")) default: return errors.New("unsupported file type") } @@ -73,8 +86,8 @@ func WriteToFile(results sqlite.DehashedResults, outputFile string, fileType fil return err } - filePath := fmt.Sprintf("%s.%s", outputFile, fileType) - return ioutil.WriteFile(filePath, data, 0644) + filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String()) + return os.WriteFile(filePath, data, 0644) } // WriteQueryResultsToFile writes query results to a file in the specified format @@ -121,6 +134,22 @@ func WriteQueryResultsToFile(results []map[string]interface{}, outputFile string outStrings = append(outStrings, strings.Join(rowStrings, "\n")+"\n\n") } data = []byte(strings.Join(outStrings, "")) + case files.GREPPABLE: + var outStrings []string + for _, r := range results { + keys := make([]string, 0, len(r)) + for k := range r { + keys = append(keys, k) + } + sort.Strings(keys) + + rowStrings := make([]string, 0, len(keys)) + for _, k := range keys { + rowStrings = append(rowStrings, fmt.Sprintf("%s=%s", k, greppableAnyValue(r[k]))) + } + outStrings = append(outStrings, strings.Join(rowStrings, "\t")+"\n") + } + data = []byte(strings.Join(outStrings, "")) default: return errors.New("unsupported file type") } @@ -133,6 +162,55 @@ func WriteQueryResultsToFile(results []map[string]interface{}, outputFile string return os.WriteFile(filePath, data, 0644) } +func dehashedResultGreppable(r sqlite.Result) string { + fields := []string{ + "id=" + greppableValue(r.DehashedId), + "email=" + greppableValue(strings.Join(r.Email, ",")), + "ip_address=" + greppableValue(strings.Join(r.IpAddress, ",")), + "username=" + greppableValue(strings.Join(r.Username, ",")), + "password=" + greppableValue(strings.Join(r.Password, ",")), + "hashed_password=" + greppableValue(strings.Join(r.HashedPassword, ",")), + "hash_type=" + greppableValue(r.HashType), + "name=" + greppableValue(strings.Join(r.Name, ",")), + "vin=" + greppableValue(strings.Join(r.Vin, ",")), + "license_plate=" + greppableValue(strings.Join(r.LicensePlate, ",")), + "url=" + greppableValue(strings.Join(r.Url, ",")), + "social=" + greppableValue(strings.Join(r.Social, ",")), + "cryptocurrency_address=" + greppableValue(strings.Join(r.CryptoCurrencyAddress, ",")), + "address=" + greppableValue(strings.Join(r.Address, ",")), + "phone=" + greppableValue(strings.Join(r.Phone, ",")), + "company=" + greppableValue(strings.Join(r.Company, ",")), + "database_name=" + greppableValue(r.DatabaseName), + } + return strings.Join(fields, "\t") +} + +func greppableAnyValue(value interface{}) string { + switch v := value.(type) { + case nil: + return "" + case []string: + return greppableValue(strings.Join(v, ",")) + case []interface{}: + values := make([]string, 0, len(v)) + for _, item := range v { + values = append(values, fmt.Sprintf("%v", item)) + } + return greppableValue(strings.Join(values, ",")) + case []byte: + return greppableValue(string(v)) + default: + return greppableValue(fmt.Sprintf("%v", v)) + } +} + +func greppableValue(value string) string { + value = strings.ReplaceAll(value, "\r", " ") + value = strings.ReplaceAll(value, "\n", " ") + value = strings.ReplaceAll(value, "\t", " ") + return strings.TrimSpace(value) +} + func WriteWhoIsHistoryToFile(results []sqlite.HistoryRecord, outputFile string, fileType files.FileType) error { var data []byte var err error diff --git a/internal/files/filetype.go b/internal/files/filetype.go index aa085cb..816a52f 100644 --- a/internal/files/filetype.go +++ b/internal/files/filetype.go @@ -1,5 +1,7 @@ package files +import "strings" + type FileType int32 const ( @@ -7,19 +9,22 @@ const ( XML YAML TEXT + GREPPABLE UNKNOWN ) func GetFileType(filetype string) FileType { - switch filetype { + switch strings.ToLower(strings.TrimSpace(filetype)) { case "json": return JSON case "xml": return XML case "yaml": return YAML - case "txt": + case "txt", "text": return TEXT + case "grep", "greppable": + return GREPPABLE default: return JSON } @@ -35,6 +40,8 @@ func (ft FileType) String() string { return "yaml" case TEXT: return "txt" + case GREPPABLE: + return "grep" default: return "json" }