Remove SQLite dependencies from go.mod and go.sum

- Altered the request format to match the new api request structure
- Altered max results per page to reflect updated Dehashed API max (10000)

Makefile

@@ -30,14 +30,14 @@ clean:
 
 # Build for current platform
 build:
-	CGO_ENABLED=0 $(GO) build -o $(BUILD_DIR)/$(BINARY_NAME) -ldflags "-X main.version=$(VERSION)" crowsnest.go
+	CGO_ENABLED=0 $(GO) build -o $(BUILD_DIR)/$(BINARY_NAME) -ldflags "-X main.version=$(VERSION) -s -w" crowsnest.go
 
 # Build for all platforms
 build-all: clean
 	@for platform in $(PLATFORMS); do \
 	    for arch in $(ARCHS); do \
 	        echo "Building for $$platform/$$arch..."; \
-	        GOOS=$$platform GOARCH=$$arch CGO_ENABLED=0 $(GO) build -o $(BUILD_DIR)/$(BINARY_NAME)-$$platform-$$arch -ldflags "-X main.version=$(VERSION)" crowsnest.go; \
+	        GOOS=$$platform GOARCH=$$arch CGO_ENABLED=0 $(GO) build -o $(BUILD_DIR)/$(BINARY_NAME)-$$platform-$$arch -ldflags "-X main.version=$(VERSION) -s -w" crowsnest.go; \
 	        if [ "$$platform" = "windows" ]; then \
 	            mv $(BUILD_DIR)/$(BINARY_NAME)-$$platform-$$arch $(BUILD_DIR)/$(BINARY_NAME)-$$platform-$$arch.exe; \
 	        fi; \

README.md

@@ -63,7 +63,7 @@ To configure the database location:
 CrowsNest requires an API key from Dehashed. Set it up with:
 ![Alt text](.img/set-dehashed.png "Set Dehashed Key")
 ```bash
-ar1ste1a@kali:~$ crowsnest set-dehashed <redacted>
+ar1ste1a@kali:~$ crowsnest set dehashed <redacted>
 ```
 
 ### Simple Query
@@ -134,10 +134,23 @@ crowsnest dehashed -R -E 'joh?n(ath[oa]n)' -D hotmail.com'
 CrowsNest is capable of handling output formats.  
 The default output format is JSON.  
 To change the output format, use the `-f` flag.  
-CrowsNest currently supports JSON, YAML, XML, and TEXT output formats.
+CrowsNest currently supports JSON, YAML, XML, TEXT, and GREP output formats.
 ``` go
 # Return matches for usernames exactly matching "admin" and write to text file 'admins_file.txt'
 crowsnest dehashed -U admin -o admins_file -f txt
+
+# Return one key=value record per line in a greppable file 'admins_file.grep'
+crowsnest dehashed -U admin -o admins_file -f grep
+```
+
+### Data Wells
+DeHashed data wells are free to query and do not require a paid API account.
+``` go
+# List the first page of data wells and write 'data_wells.json'
+crowsnest dehashed data-wells
+
+# Sort by record count and write one key=value record per line
+crowsnest dehashed data-wells --sort records-DESC --count 50 -f grep -o data_wells
 ```
 
 ---
@@ -216,11 +229,11 @@ crowsnest whois -n google.com
 ## 🌐 Hunter.io
 CrowsNest supports Hunter.io lookups.  
 Hunter.io lookups require a separate API Key from the Dehashed API.
-This can be set using the `set-hunter` command.
+This can be set using the `set hunter` command.
 ![Alt text](.img/set-hunter.png "Set Dehashed Key")
 ```bash
 # Set the Hunter.io API key
-crowsnest set-hunter <redacted>
+crowsnest set hunter <redacted>
 ```
 
 ### Domain Search

cmd/dehashed.go

@@ -1,31 +1,35 @@
 package cmd
 
 import (
-	"crowsnest/internal/badger"
-	"crowsnest/internal/debug"
-	"crowsnest/internal/dehashed"
-	"crowsnest/internal/sqlite"
 	"fmt"
+
 	"github.com/spf13/cobra"
 	"go.uber.org/zap"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/badger"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/debug"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/dehashed"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/files"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/pretty"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/sqlite"
 )
 
 func init() {
 	// Add api command to root command
 	rootCmd.AddCommand(dehashedCmd)
+	dehashedCmd.AddCommand(dehashedDataWellsCmd)
 
 	// Add flags specific to api command
-	dehashedCmd.Flags().IntVarP(&maxRecords, "max-records", "m", 30000, "Maximum amount of records to return")
+	dehashedCmd.Flags().IntVarP(&maxRecords, "max-records", "m", 50000, "Maximum total records to return (max 50000)")
 	dehashedCmd.Flags().IntVarP(&maxRequests, "max-requests", "r", -1, "Maximum number of requests to make")
 	dehashedCmd.Flags().IntVarP(&startingPage, "starting-page", "s", 1, "Starting page for requests")
 	dehashedCmd.Flags().BoolVarP(&printBalance, "print-balance", "b", false, "Print remaining balance after requests")
 	dehashedCmd.Flags().BoolVarP(&regexMatch, "regex-match", "R", false, "Use regex matching on query fields")
 	dehashedCmd.Flags().BoolVarP(&wildcardMatch, "wildcard-match", "W", false, "Use wildcard matching on query fields (Use ? to replace a single character, and * for multiple characters)")
 	dehashedCmd.Flags().BoolVarP(&credsOnly, "creds-only", "C", false, "Return credentials only")
-	dehashedCmd.Flags().StringVarP(&outputFormat, "format", "f", "json", "Output format (json, yaml, xml, txt)")
-	dehashedCmd.Flags().StringVarP(&outputFile, "output", "o", "query", "File to output results to including extension")
+	dehashedCmd.Flags().StringVarP(&outputFormat, "format", "f", "json", "Output format (json, yaml, xml, txt, grep)")
+	dehashedCmd.Flags().StringVarP(&outputFile, "output", "o", "query", "File to output results to without extension")
 	dehashedCmd.Flags().StringVarP(&usernameQuery, "username", "U", "", "Username query")
-	dehashedCmd.Flags().StringVarP(&emailQuery, "email-query", "E", "", "HunterEmail query")
+	dehashedCmd.Flags().StringVarP(&emailQuery, "email-query", "E", "", "Email query")
 	dehashedCmd.Flags().StringVarP(&ipQuery, "ip", "I", "", "IP address query")
 	dehashedCmd.Flags().StringVarP(&domainQuery, "domain", "D", "", "Domain query")
 	dehashedCmd.Flags().StringVarP(&passwordQuery, "password", "P", "", "Password query")
@@ -40,6 +44,12 @@ func init() {
 
 	// Add mutually exclusive flags to wildcard match and regex match
 	dehashedCmd.MarkFlagsMutuallyExclusive("regex-match", "wildcard-match")
+
+	dehashedDataWellsCmd.Flags().IntVar(&dataWellsCount, "count", 20, "Number of data wells to return (20 or 50)")
+	dehashedDataWellsCmd.Flags().IntVarP(&dataWellsPage, "page", "p", 1, "Data wells page to request")
+	dehashedDataWellsCmd.Flags().StringVar(&dataWellsSort, "sort", "", "Sort data wells by added, name, date, or records; optionally suffix -ASC or -DESC")
+	dehashedDataWellsCmd.Flags().StringVarP(&dataWellsOutputFormat, "format", "f", "json", "Output format (json, yaml, xml, txt, grep)")
+	dehashedDataWellsCmd.Flags().StringVarP(&dataWellsOutputFile, "output", "o", "data_wells", "File to output data wells to without extension")
 }
 
 var (
@@ -66,6 +76,11 @@ var (
 	phoneQuery                 string
 	socialQuery                string
 	cryptoCurrencyAddressQuery string
+	dataWellsCount             int
+	dataWellsPage              int
+	dataWellsSort              string
+	dataWellsOutputFormat      string
+	dataWellsOutputFile        string
 
 	// Query command
 	dehashedCmd = &cobra.Command{
@@ -77,7 +92,7 @@ var (
 
 			// Validate credentials
 			if key == "" {
-				fmt.Println("API key is required. Set the key with the \"set-key\" command. [crowsnest set-key <api_key>]")
+				fmt.Println("API key is required. Set the key with the \"set dehashed\" command. [crowsnest set dehashed <api_key>]")
 				return
 			}
 
@@ -133,9 +148,54 @@ var (
 			}
 		},
 	}
+
+	dehashedDataWellsCmd = &cobra.Command{
+		Use:   "data-wells",
+		Short: "List DeHashed data wells",
+		Long:  `List DeHashed data wells. This endpoint is free and does not require a DeHashed API key or subscription.`,
+		Run: func(cmd *cobra.Command, args []string) {
+			client := dehashed.NewDehashedClientV2("", debugGlobal)
+			response, err := client.DataWells(dehashed.DataWellsRequest{
+				Count: dataWellsCount,
+				Page:  dataWellsPage,
+				Sort:  dataWellsSort,
+			})
+			if err != nil {
+				fmt.Printf("[!] Error querying data wells: %v\n", err)
+				return
+			}
+
+			fType := files.GetFileType(dataWellsOutputFormat)
+			if dataWellsOutputFile != "" {
+				fmt.Printf("[*] Writing data wells to file: %s%s\n", dataWellsOutputFile, fType.Extension())
+				if err := dehashed.WriteDataWellsToFile(response, dataWellsOutputFile, fType); err != nil {
+					fmt.Printf("[!] Error writing data wells to file: %v\n", err)
+					return
+				}
+			}
+
+			fmt.Printf("[+] Retrieved %d data wells (total: %d, next page: %t)\n", len(response.DataWells), response.Total, response.NextPage)
+			printDataWellsTable(response.DataWells)
+		},
+	}
 )
 
 // Helper functions to get stored API credentials
 func getDehashedApiKey() string {
 	return badger.GetDehashedKey()
 }
+
+func printDataWellsTable(dataWells []dehashed.DataWell) {
+	headers := []string{"Name", "Date", "Records", "Sensitive", "Data"}
+	rows := make([][]string, 0, len(dataWells))
+	for _, well := range dataWells {
+		rows = append(rows, []string{
+			well.Name,
+			well.Date,
+			fmt.Sprintf("%d", well.Records),
+			fmt.Sprintf("%t", well.IsSensitive),
+			well.Data,
+		})
+	}
+	pretty.Table(headers, rows)
+}

cmd/hunter.go

@@ -1,17 +1,18 @@
 package cmd
 
 import (
-	"crowsnest/internal/badger"
-	"crowsnest/internal/debug"
-	"crowsnest/internal/export"
-	"crowsnest/internal/files"
-	hunter "crowsnest/internal/hunter.io"
-	"crowsnest/internal/pretty"
-	"crowsnest/internal/sqlite"
 	"fmt"
+	"time"
+
 	"github.com/spf13/cobra"
 	"go.uber.org/zap"
-	"time"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/badger"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/debug"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/export"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/files"
+	hunter "hub.krkn.tech/KrakenTech/crowsnest/internal/hunter.io"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/pretty"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/sqlite"
 )
 
 func init() {

cmd/logs.go

@@ -1,15 +1,16 @@
 package cmd
 
 import (
-	"crowsnest/internal/easyTime"
-	"crowsnest/internal/pretty"
 	"encoding/json"
 	"fmt"
-	"github.com/spf13/cobra"
 	"os"
 	"path/filepath"
 	"strings"
 	"time"
+
+	"github.com/spf13/cobra"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/easyTime"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/pretty"
 )
 
 func init() {

cmd/query.go

@@ -1,16 +1,17 @@
 package cmd
 
 import (
-	"crowsnest/internal/debug"
-	"crowsnest/internal/export"
-	"crowsnest/internal/files"
-	"crowsnest/internal/pretty"
-	"crowsnest/internal/sqlite"
 	"encoding/json"
 	"fmt"
+	"strings"
+
 	"github.com/spf13/cobra"
 	"go.uber.org/zap"
-	"strings"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/debug"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/export"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/files"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/pretty"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/sqlite"
 )
 
 func init() {
@@ -18,15 +19,16 @@ func init() {
 	rootCmd.AddCommand(queryCmd)
 
 	// Add flags specific to whois command
-	queryCmd.Flags().StringVarP(&dbQueryTableName, "table", "t", "", "Table to query (results, creds, whois, subdomains, history, runs)")
+	queryCmd.Flags().StringVarP(&dbQueryTableName, "table", "t", "", "Table to query (dehashed, users, whois, subdomains, lookup, runs)")
 	queryCmd.Flags().IntVarP(&dbQueryLimitRows, "limit", "l", 100, "Limit number of results")
 	queryCmd.Flags().StringVarP(&dbQueryNotNull, "not-null", "n", "", "Filter for non-null values (comma-separated list, e.g., 'password,email')")
 	queryCmd.Flags().StringVarP(&dbQueryColumns, "columns", "c", "", "Columns to display in output (comma-separated list, e.g., 'username,email,password')")
 	queryCmd.Flags().StringVarP(&dbQueryUserQuery, "user-query", "q", "", "User query to execute")
 	queryCmd.Flags().StringVarP(&dbQueryRawQuery, "raw-query", "r", "", "Raw SQL query to execute")
 	queryCmd.Flags().BoolVarP(&dbQueryListAll, "list-all", "a", false, "List all tables and their columns")
-	queryCmd.Flags().StringVarP(&dbQueryFormat, "format", "f", "json", "Output format (json, yaml, xml, txt)")
-	queryCmd.Flags().StringVarP(&dbQueryFile, "file", "o", "query", "File to output results to")
+	queryCmd.Flags().BoolVarP(&dbQueryExport, "export", "x", false, "Export results to file using --file and --format")
+	queryCmd.Flags().StringVarP(&dbQueryFormat, "format", "f", "json", "Output format (json, yaml, xml, txt, grep)")
+	queryCmd.Flags().StringVarP(&dbQueryFile, "file", "o", "query", "File to output results to when --export is set")
 
 	// Add mutually exclusive flags to query and raw-query
 	// Cannot use query and raw-query at the same time
@@ -45,6 +47,7 @@ var (
 	dbQueryUserQuery string
 	dbQueryRawQuery  string
 	dbQueryListAll   bool
+	dbQueryExport    bool
 	dbQueryFormat    string
 	dbQueryFile      string
 
@@ -52,7 +55,7 @@ var (
 		Use:   "query",
 		Short: "Query the database",
 		Long: `Query the database for various information.
-If file is specified, results are written to file and not displayed in the terminal.`,
+Use --export with --file and --format to write results to a file instead of displaying them.`,
 		Run: func(cmd *cobra.Command, args []string) {
 			// If list-all flag is set, list all tables and columns
 			if dbQueryListAll {
@@ -70,14 +73,14 @@ If file is specified, results are written to file and not displayed in the termi
 			// Validate table name
 			if dbQueryTableName == "" {
 				fmt.Println("[!] Error: Table name is required. Use -t or --table to specify a table.")
-				fmt.Println("[*] Available tables: results, creds, whois, subdomains, history, runs")
+				fmt.Println("[*] Available tables: dehashed, users, whois, subdomains, lookup, runs")
 				fmt.Println("[*] Use --list-all to see all tables and their columns.")
 				return
 			}
 
 			if !isValidTable(dbQueryTableName) {
 				fmt.Printf("[!] Error: Unknown table '%s'.\n", dbQueryTableName)
-				fmt.Println("[*] Available tables: results, creds, whois, subdomains, history, runs")
+				fmt.Println("[*] Available tables: dehashed, users, whois, subdomains, lookup, runs")
 				fmt.Println("[*] Use --list-all to see all tables and their columns.")
 				return
 			}
@@ -124,7 +127,7 @@ If file is specified, results are written to file and not displayed in the termi
 			table := sqlite.GetTable(dbQueryTableName)
 			if table == sqlite.UnknownTable {
 				fmt.Printf("[!] Error: Unknown table type '%s'.\n", dbQueryTableName)
-				fmt.Println("[*] Available tables: results, creds, whois, subdomains, history, runs")
+				fmt.Println("[*] Available tables: dehashed, users, whois, subdomains, lookup, runs")
 				fmt.Println("[*] Use --list-all to see all tables and their columns.")
 				return
 			}
@@ -203,8 +206,7 @@ func tableQuery(table sqlite.Table) {
 		return
 	}
 
-	// Export results if file name is specified
-	if len(strings.TrimSpace(dbQueryFile)) > 0 {
+	if dbQueryExport {
 		fmt.Println("[*] Exporting results to file...")
 
 		if debugGlobal {
@@ -244,8 +246,6 @@ func tableQuery(table sqlite.Table) {
 
 		return
 	}
-	fmt.Println("[*] Querying Database...")
-
 	// Prepare data for pretty.Table
 	headers := cols
 	var tableRows [][]string
@@ -336,7 +336,7 @@ func rawDBQuery() {
 		return
 	}
 
-	if len(strings.TrimSpace(dbQueryFile)) > 0 {
+	if dbQueryExport {
 		fmt.Println("[*] Exporting results to file...")
 
 		if debugGlobal {
@@ -377,8 +377,6 @@ func rawDBQuery() {
 
 		return
 	}
-	fmt.Println("[*] Querying Database...")
-
 	// Prepare data for pretty.Table
 	headers := columns
 	var tableRows [][]string

cmd/root.go

@@ -1,13 +1,14 @@
 package cmd
 
 import (
-	"crowsnest/internal/badger"
 	"fmt"
+	"os"
+	"strings"
+
 	"github.com/fatih/color"
 	"github.com/spf13/cobra"
 	"go.uber.org/zap"
-	"os"
-	"strings"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/badger"
 )
 
 var (
@@ -53,15 +54,23 @@ func init() {
 	rootCmd.PersistentFlags().BoolVar(&debugGlobal, "debug", false, "Show debug information")
 
 	// Add subcommands
-	rootCmd.AddCommand(setDehashedKeyCmd)
-	rootCmd.AddCommand(setHunterKeyCmd)
+	rootCmd.AddCommand(setCmd)
 	rootCmd.AddCommand(setLocalDb)
 	rootCmd.AddCommand(buyMeCoffeeCmd)
+
+	setCmd.AddCommand(setDehashedKeyCmd)
+	setCmd.AddCommand(setHunterKeyCmd)
+}
+
+var setCmd = &cobra.Command{
+	Use:   "set",
+	Short: "Set CrowsNest configuration values",
+	Long:  "Set CrowsNest configuration values such as API keys.",
 }
 
 // Command to set API key
 var setDehashedKeyCmd = &cobra.Command{
-	Use:   "set-dehashed [key]",
+	Use:   "dehashed [key]",
 	Short: "Set and store Dehashed.com API key",
 	Args:  cobra.ExactArgs(1),
 	Run: func(cmd *cobra.Command, args []string) {
@@ -77,7 +86,7 @@ var setDehashedKeyCmd = &cobra.Command{
 }
 
 var setHunterKeyCmd = &cobra.Command{
-	Use:   "set-hunter [key]",
+	Use:   "hunter [key]",
 	Short: "Set and store Hunter.io API key",
 	Args:  cobra.ExactArgs(1),
 	Run: func(cmd *cobra.Command, args []string) {

cmd/tables.go

@@ -1,9 +1,10 @@
 package cmd
 
 import (
-	"crowsnest/internal/pretty"
 	"fmt"
 	"strings"
+
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/pretty"
 )
 
 // Map of available tables and their columns

cmd/targets.go

@@ -1,12 +1,13 @@
 package cmd
 
 import (
-	"crowsnest/internal/sqlite"
 	"fmt"
-	"github.com/spf13/cobra"
-	"go.uber.org/zap"
 	"os"
 	"strings"
+
+	"github.com/spf13/cobra"
+	"go.uber.org/zap"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/sqlite"
 )
 
 func init() {
@@ -22,7 +23,7 @@ func init() {
 	targetsCmd.Flags().StringVarP(&targetsDomain, "domain", "d", "", "Filter by domain (for emails and subdomains)")
 
 	// Mark output flag as required
-	targetsCmd.MarkFlagRequired("output")
+	targetsCmd.MarkFlagsMutuallyExclusive("external", "internal", "subdomains", "emails")
 }
 
 var (

cmd/whois.go

@@ -1,18 +1,19 @@
 package cmd
 
 import (
-	"crowsnest/internal/debug"
-	"crowsnest/internal/export"
-	"crowsnest/internal/files"
-	"crowsnest/internal/pretty"
-	"crowsnest/internal/sqlite"
-	"crowsnest/internal/whois"
 	"fmt"
-	"github.com/spf13/cobra"
-	"go.uber.org/zap"
 	"os"
 	"strings"
 	"time"
+
+	"github.com/spf13/cobra"
+	"go.uber.org/zap"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/debug"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/export"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/files"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/pretty"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/sqlite"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/whois"
 )
 
 func init() {
@@ -59,7 +60,7 @@ var (
 
 			// Validate credentials
 			if key == "" {
-				fmt.Println("API key is required. Set the key with the \"set-key\" command. [crowsnest set-key <api_key>]")
+				fmt.Println("API key is required. Set the key with the \"set dehashed\" command. [crowsnest set dehashed <api_key>]")
 				return
 			}
 
@@ -195,8 +196,8 @@ var (
 
 						// Write history records to file if any
 						if len(historyRecords) > 0 {
-							fmt.Println("[*] Records Found: %d\n", len(historyRecords))
-							fmt.Println("[*] WHOIS History being written to file: %s%s\n", whoisOutputFile, fType.Extension())
+							fmt.Printf("[*] Records Found: %d\n", len(historyRecords))
+							fmt.Printf("[*] WHOIS History being written to file: %s%s\n", whoisOutputFile, fType.Extension())
 							writeErr := export.WriteWhoIsHistoryToFile(historyRecords, filename, fType)
 							if writeErr != nil {
 								if debugGlobal {

crowsnest.go

@@ -1,16 +1,17 @@
 package main
 
 import (
-	"crowsnest/cmd"
-	"crowsnest/internal/badger"
-	"crowsnest/internal/sqlite"
 	"fmt"
+	"os"
+	"path/filepath"
+
 	"github.com/winking324/rzap"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 	"gopkg.in/natefinch/lumberjack.v2"
-	"os"
-	"path/filepath"
+	"hub.krkn.tech/KrakenTech/crowsnest/cmd"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/badger"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/sqlite"
 )
 
 var (

go.mod

@@ -1,4 +1,4 @@
-module crowsnest
+module hub.krkn.tech/KrakenTech/crowsnest
 
 go 1.23.0
 
@@ -14,7 +14,6 @@ require (
 	go.uber.org/zap v1.20.0
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 	gopkg.in/yaml.v3 v3.0.1
-	gorm.io/driver/sqlite v1.5.7
 	gorm.io/gorm v1.26.1
 )
 
@@ -41,7 +40,6 @@ require (
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.16 // indirect
-	github.com/mattn/go-sqlite3 v1.14.22 // indirect
 	github.com/muesli/termenv v0.16.0 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect

go.sum

@@ -75,8 +75,6 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
 github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
-github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
 github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
@@ -171,8 +169,6 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gorm.io/driver/sqlite v1.5.7 h1:8NvsrhP0ifM7LX9G4zPB97NwovUakUxc+2V2uuf3Z1I=
-gorm.io/driver/sqlite v1.5.7/go.mod h1:U+J8craQU6Fzkcvu8oLeAQmi50TkwPEhHDEjQZXDah4=
 gorm.io/gorm v1.26.1 h1:ghB2gUI9FkS46luZtn6DLZ0f6ooBJ5IbVej2ENFDjRw=
 gorm.io/gorm v1.26.1/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE=
 modernc.org/libc v1.22.5 h1:91BNch/e5B0uPbJFgqbxXuOnxBQjlS//icfQEGmvyjE=

internal/badger/badger.go

@@ -3,15 +3,19 @@ package badger
 import (
 	"crypto/sha256"
 	"errors"
-	"github.com/dgraph-io/badger/v4"
-	"go.uber.org/zap"
+	"fmt"
 	"log"
 	"os"
+	"os/exec"
 	"os/user"
 	"path/filepath"
 	"runtime"
 	"strings"
 	"sync"
+	"time"
+
+	"github.com/dgraph-io/badger/v4"
+	"go.uber.org/zap"
 )
 
 var (
@@ -21,13 +25,35 @@ var (
 	once          sync.Once
 )
 
-func GetHardwareEntropy() []byte {
+const fingerprintSalt = "CrowsNest-static-salt-value"
+
+func GetHardwareEntropy() ([]byte, error) {
+	source, machineID, err := getMachineID()
+	if err != nil {
+		return nil, err
+	}
+
+	fingerprint := strings.Join([]string{
+		"v2",
+		runtime.GOOS,
+		source,
+		machineID,
+		fingerprintSalt,
+	}, ":")
+
+	return hashFingerprint(fingerprint), nil
+}
+
+func GetLegacyHardwareEntropy() []byte {
 	// Get hostname
 	hostname, err := os.Hostname()
 	if err != nil {
 		hostname = "unknown-host"
 		log.Printf("Error getting hostname: %v", err)
 	}
+	if legacyHostname := strings.TrimSpace(os.Getenv("CROWSNEST_LEGACY_HOSTNAME")); legacyHostname != "" {
+		hostname = legacyHostname
+	}
 
 	// Get username
 	currentUser, err := user.Current()
@@ -35,9 +61,15 @@ func GetHardwareEntropy() []byte {
 	if err == nil && currentUser != nil {
 		username = currentUser.Username
 	}
+	if legacyUsername := strings.TrimSpace(os.Getenv("CROWSNEST_LEGACY_USERNAME")); legacyUsername != "" {
+		username = legacyUsername
+	}
 
 	// Get OS and architecture info
 	osInfo := runtime.GOOS + "-" + runtime.GOARCH
+	if legacyOSInfo := strings.TrimSpace(os.Getenv("CROWSNEST_LEGACY_OSINFO")); legacyOSInfo != "" {
+		osInfo = legacyOSInfo
+	}
 
 	// Combine all information for a unique but consistent fingerprint
 	fingerprint := strings.Join([]string{
@@ -45,14 +77,93 @@ func GetHardwareEntropy() []byte {
 		username,
 		osInfo,
 		// You could add a static salt here for additional security
-		"CrowsNest-static-salt-value",
+		fingerprintSalt,
 	}, ":")
 
 	// Hash the fingerprint to get a 32-byte key
+	return hashFingerprint(fingerprint)
+}
+
+func hashFingerprint(fingerprint string) []byte {
 	sum := sha256.Sum256([]byte(fingerprint))
 	return sum[:]
 }
 
+func getMachineID() (string, string, error) {
+	switch runtime.GOOS {
+	case "darwin":
+		return getDarwinMachineID()
+	case "linux":
+		return getLinuxMachineID()
+	case "windows":
+		return getWindowsMachineID()
+	default:
+		return "", "", fmt.Errorf("stable machine id is not implemented for %s", runtime.GOOS)
+	}
+}
+
+func getDarwinMachineID() (string, string, error) {
+	out, err := exec.Command("ioreg", "-rd1", "-c", "IOPlatformExpertDevice").Output()
+	if err != nil {
+		return "", "", fmt.Errorf("run ioreg: %w", err)
+	}
+
+	for _, line := range strings.Split(string(out), "\n") {
+		if !strings.Contains(line, "IOPlatformUUID") {
+			continue
+		}
+		if id := normalizeMachineID(lastQuotedValue(line)); id != "" {
+			return "darwin-ioplatformuuid", id, nil
+		}
+	}
+
+	return "", "", errors.New("IOPlatformUUID not found")
+}
+
+func getLinuxMachineID() (string, string, error) {
+	for _, path := range []string{"/etc/machine-id", "/var/lib/dbus/machine-id"} {
+		out, err := os.ReadFile(path)
+		if err != nil {
+			continue
+		}
+		if id := normalizeMachineID(string(out)); id != "" {
+			return "linux-machine-id", id, nil
+		}
+	}
+
+	return "", "", errors.New("machine-id not found")
+}
+
+func getWindowsMachineID() (string, string, error) {
+	out, err := exec.Command("reg", "query", `HKLM\SOFTWARE\Microsoft\Cryptography`, "/v", "MachineGuid").Output()
+	if err != nil {
+		return "", "", fmt.Errorf("query MachineGuid: %w", err)
+	}
+
+	for _, line := range strings.Split(string(out), "\n") {
+		fields := strings.Fields(line)
+		if len(fields) >= 3 && strings.EqualFold(fields[0], "MachineGuid") {
+			if id := normalizeMachineID(fields[len(fields)-1]); id != "" {
+				return "windows-machineguid", id, nil
+			}
+		}
+	}
+
+	return "", "", errors.New("MachineGuid not found")
+}
+
+func lastQuotedValue(line string) string {
+	values := strings.Split(line, "\"")
+	if len(values) < 4 {
+		return ""
+	}
+	return values[len(values)-2]
+}
+
+func normalizeMachineID(id string) string {
+	return strings.ToLower(strings.TrimSpace(id))
+}
+
 func Start(dirPath string) *badger.DB {
 	var err error
 
@@ -65,7 +176,7 @@ func Start(dirPath string) *badger.DB {
 		}
 		rootDir = dirPath
 
-		encryptionKey = GetHardwareEntropy()
+		encryptionKey, err = GetHardwareEntropy()
 		if err != nil {
 			zap.L().Fatal("get_encryption_key",
 				zap.String("message", "failed to get encryption key"),
@@ -74,22 +185,134 @@ func Start(dirPath string) *badger.DB {
 		}
 
 		badgerDB := filepath.Join(rootDir, "badger.db")
-		opts := badger.DefaultOptions(badgerDB).
-			WithEncryptionKey(encryptionKey).
-			WithIndexCacheSize(10 << 20). // 10MB
-			WithLoggingLevel(badger.ERROR)
-		db, err = badger.Open(opts)
+		db, err = openBadger(badgerDB, encryptionKey)
 		if err != nil {
-			zap.L().Fatal("new_badger_db",
-				zap.String("message", "failed to open badger database"),
+			zap.L().Warn("open_badger_db",
+				zap.String("message", "failed to open badger database with stable machine key; trying legacy key"),
 				zap.Error(err),
 			)
+			db, err = openBadgerWithLegacyMigration(badgerDB, encryptionKey)
+			if err != nil {
+				zap.L().Fatal("new_badger_db",
+					zap.String("message", "failed to open badger database"),
+					zap.Error(err),
+				)
+			}
 		}
 	})
 
 	return db
 }
 
+func openBadger(dbPath string, key []byte) (*badger.DB, error) {
+	opts := badger.DefaultOptions(dbPath).
+		WithEncryptionKey(key).
+		WithIndexCacheSize(10 << 20). // 10MB
+		WithLoggingLevel(badger.ERROR)
+	return badger.Open(opts)
+}
+
+func openBadgerWithLegacyMigration(dbPath string, stableKey []byte) (*badger.DB, error) {
+	legacyKey := GetLegacyHardwareEntropy()
+	legacyDB, err := openBadger(dbPath, legacyKey)
+	if err != nil {
+		return nil, fmt.Errorf("stable key failed and legacy key failed: %w", err)
+	}
+
+	migratedDB, err := migrateBadgerEncryption(dbPath, legacyDB, stableKey)
+	if err != nil {
+		if closeErr := legacyDB.Close(); closeErr != nil {
+			zap.L().Error("close_legacy_badger_db", zap.Error(closeErr))
+		}
+		return nil, err
+	}
+
+	return migratedDB, nil
+}
+
+func migrateBadgerEncryption(dbPath string, legacyDB *badger.DB, stableKey []byte) (*badger.DB, error) {
+	parentDir := filepath.Dir(dbPath)
+	timestamp := time.Now().Format("20060102-150405")
+	migrationPath := filepath.Join(parentDir, fmt.Sprintf(".%s.migrating-%s", filepath.Base(dbPath), timestamp))
+	backupPath := filepath.Join(parentDir, fmt.Sprintf("%s.legacy-backup-%s", filepath.Base(dbPath), timestamp))
+
+	newDB, err := openBadger(migrationPath, stableKey)
+	if err != nil {
+		return nil, fmt.Errorf("open migration badger db: %w", err)
+	}
+
+	if err := copyBadgerData(legacyDB, newDB); err != nil {
+		_ = newDB.Close()
+		_ = os.RemoveAll(migrationPath)
+		return nil, fmt.Errorf("copy legacy badger data: %w", err)
+	}
+
+	if err := legacyDB.Close(); err != nil {
+		_ = newDB.Close()
+		_ = os.RemoveAll(migrationPath)
+		return nil, fmt.Errorf("close legacy badger db: %w", err)
+	}
+
+	if err := newDB.Close(); err != nil {
+		_ = os.RemoveAll(migrationPath)
+		return nil, fmt.Errorf("close migration badger db: %w", err)
+	}
+
+	if err := os.Rename(dbPath, backupPath); err != nil {
+		_ = os.RemoveAll(migrationPath)
+		return nil, fmt.Errorf("backup legacy badger db: %w", err)
+	}
+
+	if err := os.Rename(migrationPath, dbPath); err != nil {
+		if restoreErr := os.Rename(backupPath, dbPath); restoreErr != nil {
+			return nil, fmt.Errorf("promote migrated badger db: %w; restore legacy backup: %v", err, restoreErr)
+		}
+		return nil, fmt.Errorf("promote migrated badger db: %w", err)
+	}
+
+	db, err := openBadger(dbPath, stableKey)
+	if err != nil {
+		return nil, fmt.Errorf("open migrated badger db: %w", err)
+	}
+
+	zap.L().Info("migrated_badger_encryption",
+		zap.String("backup", backupPath),
+		zap.String("path", dbPath),
+	)
+	return db, nil
+}
+
+func copyBadgerData(src *badger.DB, dst *badger.DB) error {
+	return src.View(func(srcTxn *badger.Txn) error {
+		opts := badger.DefaultIteratorOptions
+		opts.PrefetchValues = true
+		iter := srcTxn.NewIterator(opts)
+		defer iter.Close()
+
+		return dst.Update(func(dstTxn *badger.Txn) error {
+			for iter.Rewind(); iter.Valid(); iter.Next() {
+				item := iter.Item()
+				if item.IsDeletedOrExpired() {
+					continue
+				}
+
+				key := item.KeyCopy(nil)
+				value, err := item.ValueCopy(nil)
+				if err != nil {
+					return err
+				}
+
+				entry := badger.NewEntry(key, value).WithMeta(item.UserMeta())
+				entry.ExpiresAt = item.ExpiresAt()
+				if err := dstTxn.SetEntry(entry); err != nil {
+					return err
+				}
+			}
+			return nil
+		})
+	})
+}
+
 func Close() {
 	err := db.Close()
 	if err != nil {

internal/badger/badger_test.go

@@ -0,0 +1,73 @@
+package badger
+
+import (
+	"crypto/sha256"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	badgerapi "github.com/dgraph-io/badger/v4"
+)
+
+func TestMigrateBadgerEncryptionCopiesDataToStableKey(t *testing.T) {
+	dir := t.TempDir()
+	dbPath := filepath.Join(dir, "badger.db")
+	legacyKey := testKey("legacy-key")
+	stableKey := testKey("stable-key")
+
+	legacyDB, err := openBadger(dbPath, legacyKey)
+	if err != nil {
+		t.Fatalf("open legacy db: %v", err)
+	}
+
+	if err := legacyDB.Update(func(txn *badgerapi.Txn) error {
+		return txn.Set([]byte("cfg:api_key"), []byte("secret"))
+	}); err != nil {
+		t.Fatalf("seed legacy db: %v", err)
+	}
+
+	migratedDB, err := migrateBadgerEncryption(dbPath, legacyDB, stableKey)
+	if err != nil {
+		t.Fatalf("migrate db: %v", err)
+	}
+	defer migratedDB.Close()
+
+	var got string
+	if err := migratedDB.View(func(txn *badgerapi.Txn) error {
+		item, err := txn.Get([]byte("cfg:api_key"))
+		if err != nil {
+			return err
+		}
+		return item.Value(func(value []byte) error {
+			got = string(value)
+			return nil
+		})
+	}); err != nil {
+		t.Fatalf("read migrated db: %v", err)
+	}
+
+	if got != "secret" {
+		t.Fatalf("migrated value = %q, want %q", got, "secret")
+	}
+
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		t.Fatalf("read temp dir: %v", err)
+	}
+	foundBackup := false
+	for _, entry := range entries {
+		if strings.HasPrefix(entry.Name(), "badger.db.legacy-backup-") {
+			foundBackup = true
+			break
+		}
+	}
+	if !foundBackup {
+		t.Fatal("legacy backup directory was not created")
+	}
+}
+
+func testKey(value string) []byte {
+	sum := sha256.Sum256([]byte(value))
+	return sum[:]
+}

internal/dehashed/clientv2.go

@@ -2,17 +2,18 @@ package dehashed
 
 import (
 	"bytes"
-	"crowsnest/internal/debug"
-	"crowsnest/internal/sqlite"
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
 	"errors"
 	"fmt"
-	"go.uber.org/zap"
 	"io"
 	"net/http"
 	"strings"
+
+	"go.uber.org/zap"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/debug"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/sqlite"
 )
 
 type DehashedParameter string
@@ -198,7 +199,7 @@ func (dcv2 *DehashedClientV2) Search(searchRequest DehashedSearchRequest) (int,
 	req.Header.Set("Dehashed-Api-Key", dcv2.apiKey)
 
 	if dcv2.debug {
-		headers := req.Header.Clone()
+		headers := redactedHeaders(req.Header)
 		h := fmt.Sprintf("Headers: %v\n", headers)
 		debug.PrintJson(h)
 		zap.L().Info("v2_search_debug",
@@ -286,7 +287,7 @@ func (dcv2 *DehashedClientV2) Search(searchRequest DehashedSearchRequest) (int,
 	}
 
 	dcv2.results = append(dcv2.results, responseResults.Entries...)
-	return responseResults.TotalResults, responseResults.Balance, nil
+	return len(responseResults.Entries), responseResults.Balance, nil
 }
 
 func (dcv2 *DehashedClientV2) GetResults() sqlite.DehashedResults {
@@ -304,3 +305,11 @@ func enquoteSpaced(s string) string {
 	}
 	return s
 }
+
+func redactedHeaders(headers http.Header) http.Header {
+	redacted := headers.Clone()
+	if redacted.Get("Dehashed-Api-Key") != "" {
+		redacted.Set("Dehashed-Api-Key", "[REDACTED]")
+	}
+	return redacted
+}

internal/dehashed/datawells.go

@@ -0,0 +1,186 @@
+package dehashed
+
+import (
+	"encoding/json"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"strconv"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/files"
+)
+
+const dataWellsEndpoint = "https://api.dehashed.com/data-wells"
+
+type DataWellsRequest struct {
+	Count int
+	Page  int
+	Sort  string
+}
+
+type DataWellsResponse struct {
+	NextPage  bool       `json:"next_page" xml:"next_page" yaml:"next_page"`
+	Total     int        `json:"total" xml:"total" yaml:"total"`
+	DataWells []DataWell `json:"data_wells" xml:"data_wells" yaml:"data_wells"`
+}
+
+type DataWell struct {
+	Data        string `json:"data" xml:"data" yaml:"data"`
+	Date        string `json:"date" xml:"date" yaml:"date"`
+	Description string `json:"description" xml:"description" yaml:"description"`
+	Name        string `json:"name" xml:"name" yaml:"name"`
+	Records     int    `json:"records" xml:"records" yaml:"records"`
+	IsSensitive bool   `json:"is_sensitive" xml:"is_sensitive" yaml:"is_sensitive"`
+}
+
+func (dcv2 *DehashedClientV2) DataWells(request DataWellsRequest) (DataWellsResponse, error) {
+	var dataWells DataWellsResponse
+
+	endpoint, err := dataWellsURL(request)
+	if err != nil {
+		return dataWells, err
+	}
+
+	req, err := http.NewRequest("GET", endpoint, nil)
+	if err != nil {
+		return dataWells, err
+	}
+	req.Header.Set("Accept", "application/json")
+
+	res, err := http.DefaultClient.Do(req)
+	if res != nil {
+		defer res.Body.Close()
+	}
+	if err != nil {
+		return dataWells, err
+	}
+	if res == nil {
+		return dataWells, errors.New("response was nil")
+	}
+
+	body, err := io.ReadAll(res.Body)
+	if err != nil {
+		return dataWells, err
+	}
+
+	if res.StatusCode != http.StatusOK {
+		return dataWells, fmt.Errorf("data wells request failed: status=%d body=%s", res.StatusCode, string(body))
+	}
+
+	if err := json.Unmarshal(body, &dataWells); err != nil {
+		return dataWells, err
+	}
+
+	return dataWells, nil
+}
+
+func dataWellsURL(request DataWellsRequest) (string, error) {
+	if request.Page <= 0 {
+		return "", errors.New("page must be 1 or greater")
+	}
+	if request.Count != 20 && request.Count != 50 {
+		return "", errors.New("count must be 20 or 50")
+	}
+	if request.Sort != "" && !validDataWellsSort(request.Sort) {
+		return "", fmt.Errorf("invalid sort %q; use added, name, date, records, optionally suffixed with -ASC or -DESC", request.Sort)
+	}
+
+	values := url.Values{}
+	values.Set("page", strconv.Itoa(request.Page))
+	values.Set("count", strconv.Itoa(request.Count))
+	if request.Sort != "" {
+		values.Set("sort", request.Sort)
+	}
+
+	return dataWellsEndpoint + "?" + values.Encode(), nil
+}
+
+func validDataWellsSort(sortValue string) bool {
+	sortValue = strings.ToLower(strings.TrimSpace(sortValue))
+	field := sortValue
+	if before, _, ok := strings.Cut(sortValue, "-"); ok {
+		field = before
+	}
+
+	switch field {
+	case "added", "name", "date", "records":
+		return strings.HasSuffix(sortValue, "-asc") || strings.HasSuffix(sortValue, "-desc") || !strings.Contains(sortValue, "-")
+	default:
+		return false
+	}
+}
+
+func WriteDataWellsToFile(dataWells DataWellsResponse, outputFile string, fileType files.FileType) error {
+	var data []byte
+	var err error
+
+	switch fileType {
+	case files.JSON:
+		data, err = json.MarshalIndent(dataWells, "", "  ")
+	case files.XML:
+		data, err = xml.MarshalIndent(dataWells, "", "  ")
+	case files.YAML:
+		data, err = yaml.Marshal(dataWells)
+	case files.TEXT:
+		data = []byte(dataWells.String())
+	case files.GREPPABLE:
+		var outStrings []string
+		for _, well := range dataWells.DataWells {
+			outStrings = append(outStrings, dataWellGreppable(well)+"\n")
+		}
+		data = []byte(strings.Join(outStrings, ""))
+	default:
+		return errors.New("unsupported file type")
+	}
+
+	if err != nil {
+		return err
+	}
+
+	return os.WriteFile(outputFile+fileType.Extension(), data, 0644)
+}
+
+func (dwr DataWellsResponse) String() string {
+	var b strings.Builder
+	fmt.Fprintf(&b, "Total: %d\nNext Page: %t\n\n", dwr.Total, dwr.NextPage)
+	for _, well := range dwr.DataWells {
+		fmt.Fprintf(&b, "Name: %s\nDate: %s\nRecords: %d\nSensitive: %t\nData: %s\nDescription: %s\n\n",
+			well.Name,
+			well.Date,
+			well.Records,
+			well.IsSensitive,
+			well.Data,
+			well.Description,
+		)
+	}
+	return b.String()
+}
+
+func dataWellGreppable(well DataWell) string {
+	var fields []string
+	fields = appendDataWellGreppableField(fields, "name", well.Name)
+	fields = appendDataWellGreppableField(fields, "date", well.Date)
+	fields = appendDataWellGreppableField(fields, "records", strconv.Itoa(well.Records))
+	fields = appendDataWellGreppableField(fields, "is_sensitive", strconv.FormatBool(well.IsSensitive))
+	fields = appendDataWellGreppableField(fields, "data", well.Data)
+	fields = appendDataWellGreppableField(fields, "description", well.Description)
+	return strings.Join(fields, " ")
+}
+
+func cleanGreppableValue(value string) string {
+	return strings.Join(strings.Fields(value), "_")
+}
+
+func appendDataWellGreppableField(fields []string, key, value string) []string {
+	value = cleanGreppableValue(value)
+	if value == "" {
+		return fields
+	}
+	return append(fields, fmt.Sprintf("%s=%s", key, value))
+}

internal/dehashed/dehashed.go

@@ -1,24 +1,31 @@
 package dehashed
 
 import (
-	"crowsnest/internal/debug"
-	"crowsnest/internal/export"
-	"crowsnest/internal/pretty"
-	"crowsnest/internal/sqlite"
 	"fmt"
-	"go.uber.org/zap"
 	"os"
 	"strings"
+
+	"go.uber.org/zap"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/debug"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/export"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/pretty"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/sqlite"
+)
+
+const (
+	maxSearchResultsPerPage  = 10000
+	maxSearchResultsPerQuery = 50000
 )
 
 // Dehasher is a struct for querying the Dehashed API
 type Dehasher struct {
-	options  sqlite.QueryOptions
-	nextPage int
-	debug    bool
-	balance  int
-	request  *DehashedSearchRequest
-	client   *DehashedClientV2
+	options    sqlite.QueryOptions
+	nextPage   int
+	debug      bool
+	balance    int
+	maxResults int
+	request    *DehashedSearchRequest
+	client     *DehashedClientV2
 }
 
 // NewDehasher creates a new Dehasher
@@ -51,55 +58,55 @@ func (dh *Dehasher) getNextPage() int {
 
 // setQueries sets the number of queries to make based on the number of records and requests
 func (dh *Dehasher) setQueries() {
-	var numQueries int
-
 	if dh.debug {
 		debug.PrintInfo("setting queries")
 	}
 
-	switch {
-	case dh.options.MaxRequests == 0:
+	if dh.options.MaxRequests == 0 {
 		zap.L().Error("max requests cannot be zero")
 		fmt.Println("[!] Max Requests cannot be zero")
 		os.Exit(1)
-	case dh.options.MaxRecords <= 10000 || dh.options.MaxRequests == 1:
-		numQueries = 1
-		if dh.options.MaxRecords > 10000 {
-			dh.options.MaxRecords = 10000
-		}
-		zap.L().Info("max requests set to 1", zap.Int("max_records", dh.options.MaxRecords))
-	case dh.options.MaxRequests < 0 && dh.options.MaxRecords > 20000:
-		numQueries = 3
-		dh.options.MaxRecords = 10000
-		zap.L().Info("max requests set to 3", zap.Int("max_records", dh.options.MaxRecords))
-	case dh.options.MaxRequests < 0 && dh.options.MaxRecords > 10000:
-		numQueries = 2
-		dh.options.MaxRecords = 10000
-		zap.L().Info("max requests set to 2", zap.Int("max_records", dh.options.MaxRecords))
-	case dh.options.MaxRecords < 0 && dh.options.MaxRecords < 10000:
-		numQueries = 1
-		zap.L().Info("max requests set to 1", zap.Int("max_records", dh.options.MaxRecords))
-	case dh.options.MaxRequests == 2 && dh.options.MaxRecords > 20000:
-		numQueries = 2
-		dh.options.MaxRecords = 10000
-		zap.L().Info("max requests set to 2", zap.Int("max_records", dh.options.MaxRecords))
-	case dh.options.MaxRequests == 2 && dh.options.MaxRecords <= 10000:
-		numQueries = 1
-		zap.L().Info("max requests set to 1", zap.Int("max_records", dh.options.MaxRecords))
-	default:
-		numQueries = 3
-		dh.options.MaxRecords = 10000
-		zap.L().Info("max requests set to 3", zap.Int("max_records", dh.options.MaxRecords))
 	}
 
+	requestedMaxResults := dh.options.MaxRecords
+	if requestedMaxResults <= 0 {
+		requestedMaxResults = maxSearchResultsPerQuery
+	}
+	if requestedMaxResults > maxSearchResultsPerQuery {
+		requestedMaxResults = maxSearchResultsPerQuery
+	}
+
+	pageSize := requestedMaxResults
+	if pageSize > maxSearchResultsPerPage {
+		pageSize = maxSearchResultsPerPage
+	}
+
+	numQueries := (requestedMaxResults + pageSize - 1) / pageSize
+	if dh.options.MaxRequests > 0 && dh.options.MaxRequests < numQueries {
+		numQueries = dh.options.MaxRequests
+	}
+
+	dh.maxResults = requestedMaxResults
+	if requestLimit := numQueries * pageSize; requestLimit < dh.maxResults {
+		dh.maxResults = requestLimit
+	}
+
+	dh.options.MaxRecords = pageSize
 	dh.options.MaxRequests = numQueries
 
+	zap.L().Info("dehashed_search_pagination",
+		zap.Int("max_results", dh.maxResults),
+		zap.Int("page_size", dh.options.MaxRecords),
+		zap.Int("max_requests", dh.options.MaxRequests),
+	)
+
 	if dh.debug {
 		debug.PrintInfo(fmt.Sprintf("setting max requests: %d", numQueries))
-		debug.PrintInfo(fmt.Sprintf("setting max records: %d", dh.options.MaxRecords))
+		debug.PrintInfo(fmt.Sprintf("setting page size: %d", dh.options.MaxRecords))
+		debug.PrintInfo(fmt.Sprintf("setting max results: %d", dh.maxResults))
 	}
 
-	fmt.Printf("Making %d Requests for %d Records (%d Total)\n", dh.options.MaxRequests, dh.options.MaxRecords, dh.options.MaxRequests*dh.options.MaxRecords)
+	fmt.Printf("Making %d Requests for up to %d Records (%d per request)\n", dh.options.MaxRequests, dh.maxResults, dh.options.MaxRecords)
 }
 
 // Start starts the querying process
@@ -151,7 +158,7 @@ func (dh *Dehasher) Start() {
 			fmt.Printf("      [-] Not enough entries, ending queries\n")
 			break
 		} else {
-			fmt.Printf("      [+] Retrieved %d records\n", dh.options.MaxRecords)
+			fmt.Printf("      [+] Retrieved %d records\n", count)
 		}
 
 		if dh.options.PrintBalance {
@@ -211,6 +218,9 @@ func (dh *Dehasher) buildRequest() {
 func (dh *Dehasher) parseResults() {
 	zap.L().Info("extracting_credentials")
 	results := dh.client.GetResults()
+	if dh.maxResults > 0 && len(results.Results) > dh.maxResults {
+		results.Results = results.Results[:dh.maxResults]
+	}
 	creds := results.ExtractUsers()
 	fmt.Printf("   [+] Discovered %d Credentials\n", len(creds))
 	err := sqlite.StoreUsers(creds)

internal/dehashed/dehashed_test.go

@@ -0,0 +1,94 @@
+package dehashed
+
+import (
+	"strings"
+	"testing"
+
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/sqlite"
+)
+
+func TestSetQueriesCapsSearchAtFiftyThousandResults(t *testing.T) {
+	options := &sqlite.QueryOptions{
+		MaxRecords:   75000,
+		MaxRequests:  -1,
+		StartingPage: 1,
+	}
+
+	dehasher := NewDehasher(options)
+
+	if dehasher.maxResults != maxSearchResultsPerQuery {
+		t.Fatalf("maxResults = %d, want %d", dehasher.maxResults, maxSearchResultsPerQuery)
+	}
+	if dehasher.options.MaxRecords != maxSearchResultsPerPage {
+		t.Fatalf("page size = %d, want %d", dehasher.options.MaxRecords, maxSearchResultsPerPage)
+	}
+	if dehasher.options.MaxRequests != 5 {
+		t.Fatalf("max requests = %d, want 5", dehasher.options.MaxRequests)
+	}
+}
+
+func TestSetQueriesHonorsExplicitRequestLimit(t *testing.T) {
+	options := &sqlite.QueryOptions{
+		MaxRecords:   50000,
+		MaxRequests:  1,
+		StartingPage: 1,
+	}
+
+	dehasher := NewDehasher(options)
+
+	if dehasher.maxResults != maxSearchResultsPerPage {
+		t.Fatalf("maxResults = %d, want %d", dehasher.maxResults, maxSearchResultsPerPage)
+	}
+	if dehasher.options.MaxRecords != maxSearchResultsPerPage {
+		t.Fatalf("page size = %d, want %d", dehasher.options.MaxRecords, maxSearchResultsPerPage)
+	}
+	if dehasher.options.MaxRequests != 1 {
+		t.Fatalf("max requests = %d, want 1", dehasher.options.MaxRequests)
+	}
+}
+
+func TestDataWellsURLDoesNotRequireAPIKey(t *testing.T) {
+	got, err := dataWellsURL(DataWellsRequest{
+		Count: 50,
+		Page:  2,
+		Sort:  "records-DESC",
+	})
+	if err != nil {
+		t.Fatalf("dataWellsURL returned error: %v", err)
+	}
+
+	if !strings.HasPrefix(got, dataWellsEndpoint+"?") {
+		t.Fatalf("url = %q, want prefix %q", got, dataWellsEndpoint+"?")
+	}
+	gotLower := strings.ToLower(got)
+	if strings.Contains(gotLower, "api_key") || strings.Contains(gotLower, "dehashed-api-key") {
+		t.Fatalf("url contains API key material: %q", got)
+	}
+	for _, want := range []string{"count=50", "page=2", "sort=records-DESC"} {
+		if !strings.Contains(got, want) {
+			t.Fatalf("url = %q, want %q", got, want)
+		}
+	}
+}
+
+func TestDataWellGreppableUsesSpaceSeparatedNonEmptyTokens(t *testing.T) {
+	got := dataWellGreppable(DataWell{
+		Name:        "Example Breach",
+		Date:        "2025-03-01",
+		Records:     500000,
+		IsSensitive: true,
+		Data:        "name,email,address",
+	})
+
+	if strings.Contains(got, "\t") {
+		t.Fatalf("greppable output contains tab: %q", got)
+	}
+	if strings.Contains(got, "description=") {
+		t.Fatalf("greppable output contains empty field: %q", got)
+	}
+	for _, want := range []string{"name=Example_Breach", "date=2025-03-01", "records=500000", "is_sensitive=true", "data=name,email,address"} {
+		if !strings.Contains(got, want) {
+			t.Fatalf("greppable output = %q, want token %q", got, want)
+		}
+	}
+}

internal/easyTime/parser.go

@@ -1,13 +1,14 @@
 package easyTime
 
 import (
-	"crowsnest/internal/debug"
 	"fmt"
-	"go.uber.org/zap"
 	"os"
 	"strconv"
 	"strings"
 	"time"
+
+	"go.uber.org/zap"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/debug"
 )
 
 type TimeChunk struct {

internal/export/dehashed.go

@@ -1,17 +1,18 @@
 package export
 
 import (
-	"crowsnest/internal/files"
-	"crowsnest/internal/sqlite"
 	"encoding/json"
 	"encoding/xml"
 	"errors"
 	"fmt"
-	"gopkg.in/yaml.v3"
-	"io/ioutil"
 	"os"
+	"sort"
 	"strings"
 	"time"
+
+	"gopkg.in/yaml.v3"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/files"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/sqlite"
 )
 
 func WriteCredsToFile(creds []sqlite.User, outputFile string, fileType files.FileType) error {
@@ -31,6 +32,16 @@ func WriteCredsToFile(creds []sqlite.User, outputFile string, fileType files.Fil
 			outStrings = append(outStrings, c.ToString()+"\n")
 		}
 		data = []byte(strings.Join(outStrings, ""))
+	case files.GREPPABLE:
+		var outStrings []string
+		for _, c := range creds {
+			var fields []string
+			fields = appendGreppableField(fields, "email", c.Email)
+			fields = appendGreppableField(fields, "username", c.Username)
+			fields = appendGreppableField(fields, "password", c.Password)
+			outStrings = append(outStrings, strings.Join(fields, " ")+"\n")
+		}
+		data = []byte(strings.Join(outStrings, ""))
 	default:
 		return errors.New("unsupported file type")
 	}
@@ -65,6 +76,12 @@ func WriteToFile(results sqlite.DehashedResults, outputFile string, fileType fil
 			outStrings = append(outStrings, out)
 		}
 		data = []byte(strings.Join(outStrings, ""))
+	case files.GREPPABLE:
+		var outStrings []string
+		for _, r := range result {
+			outStrings = append(outStrings, dehashedResultGreppable(r)+"\n")
+		}
+		data = []byte(strings.Join(outStrings, ""))
 	default:
 		return errors.New("unsupported file type")
 	}
@@ -73,8 +90,8 @@ func WriteToFile(results sqlite.DehashedResults, outputFile string, fileType fil
 		return err
 	}
 
-	filePath := fmt.Sprintf("%s.%s", outputFile, fileType)
-	return ioutil.WriteFile(filePath, data, 0644)
+	filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String())
+	return os.WriteFile(filePath, data, 0644)
 }
 
 // WriteQueryResultsToFile writes query results to a file in the specified format
@@ -121,6 +138,22 @@ func WriteQueryResultsToFile(results []map[string]interface{}, outputFile string
 			outStrings = append(outStrings, strings.Join(rowStrings, "\n")+"\n\n")
 		}
 		data = []byte(strings.Join(outStrings, ""))
+	case files.GREPPABLE:
+		var outStrings []string
+		for _, r := range results {
+			keys := make([]string, 0, len(r))
+			for k := range r {
+				keys = append(keys, k)
+			}
+			sort.Strings(keys)
+
+			rowStrings := make([]string, 0, len(keys))
+			for _, k := range keys {
+				rowStrings = appendGreppableField(rowStrings, k, greppableAnyValue(r[k]))
+			}
+			outStrings = append(outStrings, strings.Join(rowStrings, " ")+"\n")
+		}
+		data = []byte(strings.Join(outStrings, ""))
 	default:
 		return errors.New("unsupported file type")
 	}
@@ -133,6 +166,59 @@ func WriteQueryResultsToFile(results []map[string]interface{}, outputFile string
 	return os.WriteFile(filePath, data, 0644)
 }
 
+func dehashedResultGreppable(r sqlite.Result) string {
+	var fields []string
+	fields = appendGreppableField(fields, "id", r.DehashedId)
+	fields = appendGreppableField(fields, "email", strings.Join(r.Email, ","))
+	fields = appendGreppableField(fields, "ip_address", strings.Join(r.IpAddress, ","))
+	fields = appendGreppableField(fields, "username", strings.Join(r.Username, ","))
+	fields = appendGreppableField(fields, "password", strings.Join(r.Password, ","))
+	fields = appendGreppableField(fields, "hashed_password", strings.Join(r.HashedPassword, ","))
+	fields = appendGreppableField(fields, "hash_type", r.HashType)
+	fields = appendGreppableField(fields, "name", strings.Join(r.Name, ","))
+	fields = appendGreppableField(fields, "vin", strings.Join(r.Vin, ","))
+	fields = appendGreppableField(fields, "license_plate", strings.Join(r.LicensePlate, ","))
+	fields = appendGreppableField(fields, "url", strings.Join(r.Url, ","))
+	fields = appendGreppableField(fields, "social", strings.Join(r.Social, ","))
+	fields = appendGreppableField(fields, "cryptocurrency_address", strings.Join(r.CryptoCurrencyAddress, ","))
+	fields = appendGreppableField(fields, "address", strings.Join(r.Address, ","))
+	fields = appendGreppableField(fields, "phone", strings.Join(r.Phone, ","))
+	fields = appendGreppableField(fields, "company", strings.Join(r.Company, ","))
+	fields = appendGreppableField(fields, "database_name", r.DatabaseName)
+	return strings.Join(fields, " ")
+}
+
+func greppableAnyValue(value interface{}) string {
+	switch v := value.(type) {
+	case nil:
+		return ""
+	case []string:
+		return greppableValue(strings.Join(v, ","))
+	case []interface{}:
+		values := make([]string, 0, len(v))
+		for _, item := range v {
+			values = append(values, fmt.Sprintf("%v", item))
+		}
+		return greppableValue(strings.Join(values, ","))
+	case []byte:
+		return greppableValue(string(v))
+	default:
+		return greppableValue(fmt.Sprintf("%v", v))
+	}
+}
+
+func greppableValue(value string) string {
+	return strings.Join(strings.Fields(value), "_")
+}
+
+func appendGreppableField(fields []string, key, value string) []string {
+	value = greppableValue(value)
+	if value == "" {
+		return fields
+	}
+	return append(fields, fmt.Sprintf("%s=%s", key, value))
+}
+
 func WriteWhoIsHistoryToFile(results []sqlite.HistoryRecord, outputFile string, fileType files.FileType) error {
 	var data []byte
 	var err error

internal/export/dehashed_test.go

@@ -0,0 +1,29 @@
+package export
+
+import (
+	"strings"
+	"testing"
+
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/sqlite"
+)
+
+func TestDehashedResultGreppableUsesSpaceSeparatedNonEmptyTokens(t *testing.T) {
+	got := dehashedResultGreppable(sqlite.Result{
+		DehashedId: "123",
+		Name:       []string{"Hargrave Mall"},
+		Address:    []string{"irving tx"},
+		Url:        []string{"gdt.com", "GDT.COM"},
+	})
+
+	if strings.Contains(got, "\t") {
+		t.Fatalf("greppable output contains tab: %q", got)
+	}
+	if strings.Contains(got, "vin=") {
+		t.Fatalf("greppable output contains empty field: %q", got)
+	}
+	for _, want := range []string{"id=123", "name=Hargrave_Mall", "address=irving_tx", "url=gdt.com,GDT.COM"} {
+		if !strings.Contains(got, want) {
+			t.Fatalf("greppable output = %q, want token %q", got, want)
+		}
+	}
+}

internal/export/iString.go

@@ -1,13 +1,14 @@
 package export
 
 import (
-	"crowsnest/internal/files"
-	"crowsnest/internal/sqlite"
 	"encoding/json"
 	"encoding/xml"
 	"fmt"
-	"gopkg.in/yaml.v3"
 	"os"
+
+	"gopkg.in/yaml.v3"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/files"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/sqlite"
 )
 
 func WriteIStringToFile(iString sqlite.IString, outputFile string, fileType files.FileType) error {

internal/files/filetype.go

@@ -1,5 +1,7 @@
 package files
 
+import "strings"
+
 type FileType int32
 
 const (
@@ -7,19 +9,22 @@ const (
 	XML
 	YAML
 	TEXT
+	GREPPABLE
 	UNKNOWN
 )
 
 func GetFileType(filetype string) FileType {
-	switch filetype {
+	switch strings.ToLower(strings.TrimSpace(filetype)) {
 	case "json":
 		return JSON
 	case "xml":
 		return XML
 	case "yaml":
 		return YAML
-	case "txt":
+	case "txt", "text":
 		return TEXT
+	case "grep", "greppable":
+		return GREPPABLE
 	default:
 		return JSON
 	}
@@ -35,6 +40,8 @@ func (ft FileType) String() string {
 		return "yaml"
 	case TEXT:
 		return "txt"
+	case GREPPABLE:
+		return "grep"
 	default:
 		return "json"
 	}

internal/hunter.io/hunter.io.go

@@ -1,14 +1,15 @@
 package hunter_io
 
 import (
-	"crowsnest/internal/debug"
-	"crowsnest/internal/sqlite"
 	"encoding/json"
 	"fmt"
-	"go.uber.org/zap"
 	"io"
 	"net/http"
 	"strings"
+
+	"go.uber.org/zap"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/debug"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/sqlite"
 )
 
 const (

internal/pretty/trees.go

@@ -1,10 +1,11 @@
 package pretty
 
 import (
-	"crowsnest/internal/sqlite"
 	"fmt"
+
 	"github.com/charmbracelet/lipgloss"
 	"github.com/charmbracelet/lipgloss/tree"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/sqlite"
 )
 
 func WhoIsTree(root string, record sqlite.WhoisRecord) {

internal/sqlite/db.go

@@ -90,11 +90,11 @@ const (
 
 func GetTable(userInput string) Table {
 	switch strings.ToLower(userInput) {
-	case "results":
+	case "dehashed", "results":
 		return ResultsTable
 	case "runs":
 		return RunsTable
-	case "creds":
+	case "users", "creds":
 		return CredsTable
 	case "whois":
 		return WhoIsTable

internal/sqlite/db_test.go

@@ -0,0 +1,18 @@
+package sqlite
+
+import "testing"
+
+func TestGetTableAcceptsDisplayedTableNames(t *testing.T) {
+	tests := map[string]Table{
+		"dehashed": ResultsTable,
+		"results":  ResultsTable,
+		"users":    CredsTable,
+		"creds":    CredsTable,
+	}
+
+	for input, want := range tests {
+		if got := GetTable(input); got != want {
+			t.Fatalf("GetTable(%q) = %v, want %v", input, got, want)
+		}
+	}
+}

internal/sqlite/dehashed.go

@@ -1,11 +1,12 @@
 package sqlite
 
 import (
-	"crowsnest/internal/files"
 	"fmt"
+
 	"go.uber.org/zap"
 	"gorm.io/gorm"
 	"gorm.io/gorm/clause"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/files"
 )
 
 type QueryOptions struct {

internal/whois/whois.go

@@ -2,15 +2,16 @@ package whois
 
 import (
 	"bytes"
-	"crowsnest/internal/debug"
-	"crowsnest/internal/dehashed"
-	"crowsnest/internal/sqlite"
 	"encoding/json"
 	"errors"
 	"fmt"
-	"go.uber.org/zap"
 	"io"
 	"net/http"
+
+	"go.uber.org/zap"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/debug"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/dehashed"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/sqlite"
 )
 
 type DehashedWHOISSearchRequest struct {