- Added datawells subcommand

- Altered the request format to match the new api request structure
- Altered max results per page to reflect updated Dehashed API max (10000)

internal/dehashed/clientv2.go

@@ -198,7 +198,7 @@ func (dcv2 *DehashedClientV2) Search(searchRequest DehashedSearchRequest) (int,
 	req.Header.Set("Dehashed-Api-Key", dcv2.apiKey)
 
 	if dcv2.debug {
-		headers := req.Header.Clone()
+		headers := redactedHeaders(req.Header)
 		h := fmt.Sprintf("Headers: %v\n", headers)
 		debug.PrintJson(h)
 		zap.L().Info("v2_search_debug",
@@ -286,7 +286,7 @@ func (dcv2 *DehashedClientV2) Search(searchRequest DehashedSearchRequest) (int,
 	}
 
 	dcv2.results = append(dcv2.results, responseResults.Entries...)
-	return responseResults.TotalResults, responseResults.Balance, nil
+	return len(responseResults.Entries), responseResults.Balance, nil
 }
 
 func (dcv2 *DehashedClientV2) GetResults() sqlite.DehashedResults {
@@ -304,3 +304,11 @@ func enquoteSpaced(s string) string {
 	}
 	return s
 }
+
+func redactedHeaders(headers http.Header) http.Header {
+	redacted := headers.Clone()
+	if redacted.Get("Dehashed-Api-Key") != "" {
+		redacted.Set("Dehashed-Api-Key", "[REDACTED]")
+	}
+	return redacted
+}

internal/dehashed/datawells.go

@@ -0,0 +1,182 @@
+package dehashed
+
+import (
+	"encoding/json"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"strconv"
+	"strings"
+
+	"crowsnest/internal/files"
+	"gopkg.in/yaml.v3"
+)
+
+const dataWellsEndpoint = "https://api.dehashed.com/data-wells"
+
+type DataWellsRequest struct {
+	Count int
+	Page  int
+	Sort  string
+}
+
+type DataWellsResponse struct {
+	NextPage  bool       `json:"next_page" xml:"next_page" yaml:"next_page"`
+	Total     int        `json:"total" xml:"total" yaml:"total"`
+	DataWells []DataWell `json:"data_wells" xml:"data_wells" yaml:"data_wells"`
+}
+
+type DataWell struct {
+	Data        string `json:"data" xml:"data" yaml:"data"`
+	Date        string `json:"date" xml:"date" yaml:"date"`
+	Description string `json:"description" xml:"description" yaml:"description"`
+	Name        string `json:"name" xml:"name" yaml:"name"`
+	Records     int    `json:"records" xml:"records" yaml:"records"`
+	IsSensitive bool   `json:"is_sensitive" xml:"is_sensitive" yaml:"is_sensitive"`
+}
+
+func (dcv2 *DehashedClientV2) DataWells(request DataWellsRequest) (DataWellsResponse, error) {
+	var dataWells DataWellsResponse
+
+	endpoint, err := dataWellsURL(request)
+	if err != nil {
+		return dataWells, err
+	}
+
+	req, err := http.NewRequest("GET", endpoint, nil)
+	if err != nil {
+		return dataWells, err
+	}
+	req.Header.Set("Accept", "application/json")
+
+	res, err := http.DefaultClient.Do(req)
+	if res != nil {
+		defer res.Body.Close()
+	}
+	if err != nil {
+		return dataWells, err
+	}
+	if res == nil {
+		return dataWells, errors.New("response was nil")
+	}
+
+	body, err := io.ReadAll(res.Body)
+	if err != nil {
+		return dataWells, err
+	}
+
+	if res.StatusCode != http.StatusOK {
+		return dataWells, fmt.Errorf("data wells request failed: status=%d body=%s", res.StatusCode, string(body))
+	}
+
+	if err := json.Unmarshal(body, &dataWells); err != nil {
+		return dataWells, err
+	}
+
+	return dataWells, nil
+}
+
+func dataWellsURL(request DataWellsRequest) (string, error) {
+	if request.Page <= 0 {
+		return "", errors.New("page must be 1 or greater")
+	}
+	if request.Count != 20 && request.Count != 50 {
+		return "", errors.New("count must be 20 or 50")
+	}
+	if request.Sort != "" && !validDataWellsSort(request.Sort) {
+		return "", fmt.Errorf("invalid sort %q; use added, name, date, records, optionally suffixed with -ASC or -DESC", request.Sort)
+	}
+
+	values := url.Values{}
+	values.Set("page", strconv.Itoa(request.Page))
+	values.Set("count", strconv.Itoa(request.Count))
+	if request.Sort != "" {
+		values.Set("sort", request.Sort)
+	}
+
+	return dataWellsEndpoint + "?" + values.Encode(), nil
+}
+
+func validDataWellsSort(sortValue string) bool {
+	sortValue = strings.ToLower(strings.TrimSpace(sortValue))
+	field := sortValue
+	if before, _, ok := strings.Cut(sortValue, "-"); ok {
+		field = before
+	}
+
+	switch field {
+	case "added", "name", "date", "records":
+		return strings.HasSuffix(sortValue, "-asc") || strings.HasSuffix(sortValue, "-desc") || !strings.Contains(sortValue, "-")
+	default:
+		return false
+	}
+}
+
+func WriteDataWellsToFile(dataWells DataWellsResponse, outputFile string, fileType files.FileType) error {
+	var data []byte
+	var err error
+
+	switch fileType {
+	case files.JSON:
+		data, err = json.MarshalIndent(dataWells, "", "  ")
+	case files.XML:
+		data, err = xml.MarshalIndent(dataWells, "", "  ")
+	case files.YAML:
+		data, err = yaml.Marshal(dataWells)
+	case files.TEXT:
+		data = []byte(dataWells.String())
+	case files.GREPPABLE:
+		var outStrings []string
+		for _, well := range dataWells.DataWells {
+			outStrings = append(outStrings, dataWellGreppable(well)+"\n")
+		}
+		data = []byte(strings.Join(outStrings, ""))
+	default:
+		return errors.New("unsupported file type")
+	}
+
+	if err != nil {
+		return err
+	}
+
+	return os.WriteFile(outputFile+fileType.Extension(), data, 0644)
+}
+
+func (dwr DataWellsResponse) String() string {
+	var b strings.Builder
+	fmt.Fprintf(&b, "Total: %d\nNext Page: %t\n\n", dwr.Total, dwr.NextPage)
+	for _, well := range dwr.DataWells {
+		fmt.Fprintf(&b, "Name: %s\nDate: %s\nRecords: %d\nSensitive: %t\nData: %s\nDescription: %s\n\n",
+			well.Name,
+			well.Date,
+			well.Records,
+			well.IsSensitive,
+			well.Data,
+			well.Description,
+		)
+	}
+	return b.String()
+}
+
+func dataWellGreppable(well DataWell) string {
+	fields := []string{
+		"name=" + cleanGreppableValue(well.Name),
+		"date=" + cleanGreppableValue(well.Date),
+		"records=" + strconv.Itoa(well.Records),
+		"is_sensitive=" + strconv.FormatBool(well.IsSensitive),
+		"data=" + cleanGreppableValue(well.Data),
+		"description=" + cleanGreppableValue(well.Description),
+	}
+	return strings.Join(fields, "\t")
+}
+
+func cleanGreppableValue(value string) string {
+	value = strings.ReplaceAll(value, "\r", " ")
+	value = strings.ReplaceAll(value, "\n", " ")
+	value = strings.ReplaceAll(value, "\t", " ")
+	return strings.TrimSpace(value)
+}

internal/dehashed/dehashed.go

@@ -11,14 +11,20 @@ import (
 	"strings"
 )
 
+const (
+	maxSearchResultsPerPage  = 10000
+	maxSearchResultsPerQuery = 50000
+)
+
 // Dehasher is a struct for querying the Dehashed API
 type Dehasher struct {
-	options  sqlite.QueryOptions
-	nextPage int
-	debug    bool
-	balance  int
-	request  *DehashedSearchRequest
-	client   *DehashedClientV2
+	options    sqlite.QueryOptions
+	nextPage   int
+	debug      bool
+	balance    int
+	maxResults int
+	request    *DehashedSearchRequest
+	client     *DehashedClientV2
 }
 
 // NewDehasher creates a new Dehasher
@@ -51,55 +57,55 @@ func (dh *Dehasher) getNextPage() int {
 
 // setQueries sets the number of queries to make based on the number of records and requests
 func (dh *Dehasher) setQueries() {
-	var numQueries int
-
 	if dh.debug {
 		debug.PrintInfo("setting queries")
 	}
 
-	switch {
-	case dh.options.MaxRequests == 0:
+	if dh.options.MaxRequests == 0 {
 		zap.L().Error("max requests cannot be zero")
 		fmt.Println("[!] Max Requests cannot be zero")
 		os.Exit(1)
-	case dh.options.MaxRecords <= 10000 || dh.options.MaxRequests == 1:
-		numQueries = 1
-		if dh.options.MaxRecords > 10000 {
-			dh.options.MaxRecords = 10000
-		}
-		zap.L().Info("max requests set to 1", zap.Int("max_records", dh.options.MaxRecords))
-	case dh.options.MaxRequests < 0 && dh.options.MaxRecords > 20000:
-		numQueries = 3
-		dh.options.MaxRecords = 10000
-		zap.L().Info("max requests set to 3", zap.Int("max_records", dh.options.MaxRecords))
-	case dh.options.MaxRequests < 0 && dh.options.MaxRecords > 10000:
-		numQueries = 2
-		dh.options.MaxRecords = 10000
-		zap.L().Info("max requests set to 2", zap.Int("max_records", dh.options.MaxRecords))
-	case dh.options.MaxRecords < 0 && dh.options.MaxRecords < 10000:
-		numQueries = 1
-		zap.L().Info("max requests set to 1", zap.Int("max_records", dh.options.MaxRecords))
-	case dh.options.MaxRequests == 2 && dh.options.MaxRecords > 20000:
-		numQueries = 2
-		dh.options.MaxRecords = 10000
-		zap.L().Info("max requests set to 2", zap.Int("max_records", dh.options.MaxRecords))
-	case dh.options.MaxRequests == 2 && dh.options.MaxRecords <= 10000:
-		numQueries = 1
-		zap.L().Info("max requests set to 1", zap.Int("max_records", dh.options.MaxRecords))
-	default:
-		numQueries = 3
-		dh.options.MaxRecords = 10000
-		zap.L().Info("max requests set to 3", zap.Int("max_records", dh.options.MaxRecords))
 	}
 
+	requestedMaxResults := dh.options.MaxRecords
+	if requestedMaxResults <= 0 {
+		requestedMaxResults = maxSearchResultsPerQuery
+	}
+	if requestedMaxResults > maxSearchResultsPerQuery {
+		requestedMaxResults = maxSearchResultsPerQuery
+	}
+
+	pageSize := requestedMaxResults
+	if pageSize > maxSearchResultsPerPage {
+		pageSize = maxSearchResultsPerPage
+	}
+
+	numQueries := (requestedMaxResults + pageSize - 1) / pageSize
+	if dh.options.MaxRequests > 0 && dh.options.MaxRequests < numQueries {
+		numQueries = dh.options.MaxRequests
+	}
+
+	dh.maxResults = requestedMaxResults
+	if requestLimit := numQueries * pageSize; requestLimit < dh.maxResults {
+		dh.maxResults = requestLimit
+	}
+
+	dh.options.MaxRecords = pageSize
 	dh.options.MaxRequests = numQueries
 
+	zap.L().Info("dehashed_search_pagination",
+		zap.Int("max_results", dh.maxResults),
+		zap.Int("page_size", dh.options.MaxRecords),
+		zap.Int("max_requests", dh.options.MaxRequests),
+	)
+
 	if dh.debug {
 		debug.PrintInfo(fmt.Sprintf("setting max requests: %d", numQueries))
-		debug.PrintInfo(fmt.Sprintf("setting max records: %d", dh.options.MaxRecords))
+		debug.PrintInfo(fmt.Sprintf("setting page size: %d", dh.options.MaxRecords))
+		debug.PrintInfo(fmt.Sprintf("setting max results: %d", dh.maxResults))
 	}
 
-	fmt.Printf("Making %d Requests for %d Records (%d Total)\n", dh.options.MaxRequests, dh.options.MaxRecords, dh.options.MaxRequests*dh.options.MaxRecords)
+	fmt.Printf("Making %d Requests for up to %d Records (%d per request)\n", dh.options.MaxRequests, dh.maxResults, dh.options.MaxRecords)
 }
 
 // Start starts the querying process
@@ -151,7 +157,7 @@ func (dh *Dehasher) Start() {
 			fmt.Printf("      [-] Not enough entries, ending queries\n")
 			break
 		} else {
-			fmt.Printf("      [+] Retrieved %d records\n", dh.options.MaxRecords)
+			fmt.Printf("      [+] Retrieved %d records\n", count)
 		}
 
 		if dh.options.PrintBalance {
@@ -211,6 +217,9 @@ func (dh *Dehasher) buildRequest() {
 func (dh *Dehasher) parseResults() {
 	zap.L().Info("extracting_credentials")
 	results := dh.client.GetResults()
+	if dh.maxResults > 0 && len(results.Results) > dh.maxResults {
+		results.Results = results.Results[:dh.maxResults]
+	}
 	creds := results.ExtractUsers()
 	fmt.Printf("   [+] Discovered %d Credentials\n", len(creds))
 	err := sqlite.StoreUsers(creds)

internal/dehashed/dehashed_test.go

@@ -0,0 +1,72 @@
+package dehashed
+
+import (
+	"strings"
+	"testing"
+
+	"crowsnest/internal/sqlite"
+)
+
+func TestSetQueriesCapsSearchAtFiftyThousandResults(t *testing.T) {
+	options := &sqlite.QueryOptions{
+		MaxRecords:   75000,
+		MaxRequests:  -1,
+		StartingPage: 1,
+	}
+
+	dehasher := NewDehasher(options)
+
+	if dehasher.maxResults != maxSearchResultsPerQuery {
+		t.Fatalf("maxResults = %d, want %d", dehasher.maxResults, maxSearchResultsPerQuery)
+	}
+	if dehasher.options.MaxRecords != maxSearchResultsPerPage {
+		t.Fatalf("page size = %d, want %d", dehasher.options.MaxRecords, maxSearchResultsPerPage)
+	}
+	if dehasher.options.MaxRequests != 5 {
+		t.Fatalf("max requests = %d, want 5", dehasher.options.MaxRequests)
+	}
+}
+
+func TestSetQueriesHonorsExplicitRequestLimit(t *testing.T) {
+	options := &sqlite.QueryOptions{
+		MaxRecords:   50000,
+		MaxRequests:  1,
+		StartingPage: 1,
+	}
+
+	dehasher := NewDehasher(options)
+
+	if dehasher.maxResults != maxSearchResultsPerPage {
+		t.Fatalf("maxResults = %d, want %d", dehasher.maxResults, maxSearchResultsPerPage)
+	}
+	if dehasher.options.MaxRecords != maxSearchResultsPerPage {
+		t.Fatalf("page size = %d, want %d", dehasher.options.MaxRecords, maxSearchResultsPerPage)
+	}
+	if dehasher.options.MaxRequests != 1 {
+		t.Fatalf("max requests = %d, want 1", dehasher.options.MaxRequests)
+	}
+}
+
+func TestDataWellsURLDoesNotRequireAPIKey(t *testing.T) {
+	got, err := dataWellsURL(DataWellsRequest{
+		Count: 50,
+		Page:  2,
+		Sort:  "records-DESC",
+	})
+	if err != nil {
+		t.Fatalf("dataWellsURL returned error: %v", err)
+	}
+
+	if !strings.HasPrefix(got, dataWellsEndpoint+"?") {
+		t.Fatalf("url = %q, want prefix %q", got, dataWellsEndpoint+"?")
+	}
+	gotLower := strings.ToLower(got)
+	if strings.Contains(gotLower, "api_key") || strings.Contains(gotLower, "dehashed-api-key") {
+		t.Fatalf("url contains API key material: %q", got)
+	}
+	for _, want := range []string{"count=50", "page=2", "sort=records-DESC"} {
+		if !strings.Contains(got, want) {
+			t.Fatalf("url = %q, want %q", got, want)
+		}
+	}
+}

internal/export/dehashed.go

@@ -8,8 +8,8 @@ import (
 	"errors"
 	"fmt"
 	"gopkg.in/yaml.v3"
-	"io/ioutil"
 	"os"
+	"sort"
 	"strings"
 	"time"
 )
@@ -31,6 +31,13 @@ func WriteCredsToFile(creds []sqlite.User, outputFile string, fileType files.Fil
 			outStrings = append(outStrings, c.ToString()+"\n")
 		}
 		data = []byte(strings.Join(outStrings, ""))
+	case files.GREPPABLE:
+		var outStrings []string
+		for _, c := range creds {
+			outStrings = append(outStrings, fmt.Sprintf("email=%s\tusername=%s\tpassword=%s\n",
+				greppableValue(c.Email), greppableValue(c.Username), greppableValue(c.Password)))
+		}
+		data = []byte(strings.Join(outStrings, ""))
 	default:
 		return errors.New("unsupported file type")
 	}
@@ -65,6 +72,12 @@ func WriteToFile(results sqlite.DehashedResults, outputFile string, fileType fil
 			outStrings = append(outStrings, out)
 		}
 		data = []byte(strings.Join(outStrings, ""))
+	case files.GREPPABLE:
+		var outStrings []string
+		for _, r := range result {
+			outStrings = append(outStrings, dehashedResultGreppable(r)+"\n")
+		}
+		data = []byte(strings.Join(outStrings, ""))
 	default:
 		return errors.New("unsupported file type")
 	}
@@ -73,8 +86,8 @@ func WriteToFile(results sqlite.DehashedResults, outputFile string, fileType fil
 		return err
 	}
 
-	filePath := fmt.Sprintf("%s.%s", outputFile, fileType)
-	return ioutil.WriteFile(filePath, data, 0644)
+	filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String())
+	return os.WriteFile(filePath, data, 0644)
 }
 
 // WriteQueryResultsToFile writes query results to a file in the specified format
@@ -121,6 +134,22 @@ func WriteQueryResultsToFile(results []map[string]interface{}, outputFile string
 			outStrings = append(outStrings, strings.Join(rowStrings, "\n")+"\n\n")
 		}
 		data = []byte(strings.Join(outStrings, ""))
+	case files.GREPPABLE:
+		var outStrings []string
+		for _, r := range results {
+			keys := make([]string, 0, len(r))
+			for k := range r {
+				keys = append(keys, k)
+			}
+			sort.Strings(keys)
+
+			rowStrings := make([]string, 0, len(keys))
+			for _, k := range keys {
+				rowStrings = append(rowStrings, fmt.Sprintf("%s=%s", k, greppableAnyValue(r[k])))
+			}
+			outStrings = append(outStrings, strings.Join(rowStrings, "\t")+"\n")
+		}
+		data = []byte(strings.Join(outStrings, ""))
 	default:
 		return errors.New("unsupported file type")
 	}
@@ -133,6 +162,55 @@ func WriteQueryResultsToFile(results []map[string]interface{}, outputFile string
 	return os.WriteFile(filePath, data, 0644)
 }
 
+func dehashedResultGreppable(r sqlite.Result) string {
+	fields := []string{
+		"id=" + greppableValue(r.DehashedId),
+		"email=" + greppableValue(strings.Join(r.Email, ",")),
+		"ip_address=" + greppableValue(strings.Join(r.IpAddress, ",")),
+		"username=" + greppableValue(strings.Join(r.Username, ",")),
+		"password=" + greppableValue(strings.Join(r.Password, ",")),
+		"hashed_password=" + greppableValue(strings.Join(r.HashedPassword, ",")),
+		"hash_type=" + greppableValue(r.HashType),
+		"name=" + greppableValue(strings.Join(r.Name, ",")),
+		"vin=" + greppableValue(strings.Join(r.Vin, ",")),
+		"license_plate=" + greppableValue(strings.Join(r.LicensePlate, ",")),
+		"url=" + greppableValue(strings.Join(r.Url, ",")),
+		"social=" + greppableValue(strings.Join(r.Social, ",")),
+		"cryptocurrency_address=" + greppableValue(strings.Join(r.CryptoCurrencyAddress, ",")),
+		"address=" + greppableValue(strings.Join(r.Address, ",")),
+		"phone=" + greppableValue(strings.Join(r.Phone, ",")),
+		"company=" + greppableValue(strings.Join(r.Company, ",")),
+		"database_name=" + greppableValue(r.DatabaseName),
+	}
+	return strings.Join(fields, "\t")
+}
+
+func greppableAnyValue(value interface{}) string {
+	switch v := value.(type) {
+	case nil:
+		return ""
+	case []string:
+		return greppableValue(strings.Join(v, ","))
+	case []interface{}:
+		values := make([]string, 0, len(v))
+		for _, item := range v {
+			values = append(values, fmt.Sprintf("%v", item))
+		}
+		return greppableValue(strings.Join(values, ","))
+	case []byte:
+		return greppableValue(string(v))
+	default:
+		return greppableValue(fmt.Sprintf("%v", v))
+	}
+}
+
+func greppableValue(value string) string {
+	value = strings.ReplaceAll(value, "\r", " ")
+	value = strings.ReplaceAll(value, "\n", " ")
+	value = strings.ReplaceAll(value, "\t", " ")
+	return strings.TrimSpace(value)
+}
+
 func WriteWhoIsHistoryToFile(results []sqlite.HistoryRecord, outputFile string, fileType files.FileType) error {
 	var data []byte
 	var err error

internal/files/filetype.go

@@ -1,5 +1,7 @@
 package files
 
+import "strings"
+
 type FileType int32
 
 const (
@@ -7,19 +9,22 @@ const (
 	XML
 	YAML
 	TEXT
+	GREPPABLE
 	UNKNOWN
 )
 
 func GetFileType(filetype string) FileType {
-	switch filetype {
+	switch strings.ToLower(strings.TrimSpace(filetype)) {
 	case "json":
 		return JSON
 	case "xml":
 		return XML
 	case "yaml":
 		return YAML
-	case "txt":
+	case "txt", "text":
 		return TEXT
+	case "grep", "greppable":
+		return GREPPABLE
 	default:
 		return JSON
 	}
@@ -35,6 +40,8 @@ func (ft FileType) String() string {
 		return "yaml"
 	case TEXT:
 		return "txt"
+	case GREPPABLE:
+		return "grep"
 	default:
 		return "json"
 	}