Remove SQLite dependencies from go.mod and go.sum

- Altered the request format to match the new api request structure
- Altered max results per page to reflect updated Dehashed API max (10000)

.gitignore

@@ -1,2 +1,3 @@
 .idea/*
 build/*
+.DS_Store

Makefile

@@ -1,10 +1,10 @@
-# Makefile for Dehasher
+# Makefile for CrowsNest
 
 # Go command
 GO=go
 
 # Binary name
-BINARY_NAME=dehasher
+BINARY_NAME=crowsnest
 
 # Build directory
 BUILD_DIR=build/bin
@@ -16,7 +16,7 @@ PLATFORMS=linux darwin windows
 ARCHS=amd64 arm64
 
 # Version info from git tag or default
-VERSION=$(shell git describe --tags 2>/dev/null || echo "v1.0.1")
+VERSION=$(shell git describe --tags 2>/dev/null || echo "v1.3.3")
 
 .PHONY: all clean build build-all
 
@@ -30,14 +30,14 @@ clean:
 
 # Build for current platform
 build:
-	$(GO) build -o $(BUILD_DIR)/$(BINARY_NAME) -ldflags "-X main.version=$(VERSION)" dehasher.go
+	CGO_ENABLED=0 $(GO) build -o $(BUILD_DIR)/$(BINARY_NAME) -ldflags "-X main.version=$(VERSION) -s -w" crowsnest.go
 
 # Build for all platforms
 build-all: clean
 	@for platform in $(PLATFORMS); do \
 	    for arch in $(ARCHS); do \
 	        echo "Building for $$platform/$$arch..."; \
-	        GOOS=$$platform GOARCH=$$arch $(GO) build -o $(BUILD_DIR)/$(BINARY_NAME)-$$platform-$$arch -ldflags "-X main.version=$(VERSION)" dehasher.go; \
+	        GOOS=$$platform GOARCH=$$arch CGO_ENABLED=0 $(GO) build -o $(BUILD_DIR)/$(BINARY_NAME)-$$platform-$$arch -ldflags "-X main.version=$(VERSION) -s -w" crowsnest.go; \
 	        if [ "$$platform" = "windows" ]; then \
 	            mv $(BUILD_DIR)/$(BINARY_NAME)-$$platform-$$arch $(BUILD_DIR)/$(BINARY_NAME)-$$platform-$$arch.exe; \
 	        fi; \
@@ -50,4 +50,4 @@ install: build
 
 # Run tests
 test:
-	$(GO) test ./...
+	$(GO) test ./...

README.md

@@ -1,5 +1,8 @@
-# 🚀 Dehasher
-### A CLI tool for seamless interaction with the Dehashed API
+<div align="center">
+    <img src=.img/crowsnest.png  style="width: 500px; height: auto" alt="CrowsNest Logo" title="CrowsNest Logo">
+</div>
+
+### A CLI tool for seamless interaction with the Dehashed and Hunter.io APIs.
 
 ---
 
@@ -16,46 +19,28 @@
 - **API Key Management**: Securely store and manage API keys.
 - **Formatted Output**: Easy to read and understand.
 - **Intuitive Database Querying**: Query for specific information.
+- **Person and Company Enrichment**: Retrieve detailed information about people and companies.
+- **Email Verification**: Verify the existence and quality of email addresses.
 
 ---
 
-## 📦 Installation
-
-Clone the repository and build the tool:
-```bash
-git clone https://github.com/Ar1ste1a/Dehasher.git
-cd Dehasher
-go build dehasher.go
-```
-
-<hr></hr>
-
 ## 🔰 Getting Started
 
 To begin, clone the repository
 ``` bash-session
-git clone https://github.com/Ar1ste1a/Dehasher.git
-cd Dehasher
-go build dehasher.go
+git clone https://github.com/Kraken-OffSec/CrowsNest.git
+cd crowsnest
+go build crowsnest.go
 ```
 
-<hr></hr>
-
-## 🛠️ Initial Setup
-
-Dehasher requires an API key from Dehashed. Set it up with:
-```bash
-ar1ste1a@kali:~$ dehasher set-key <redacted>
-```
-
-<hr></hr>
+---
 
 ## 🗄️ Database Configuration
 
-Dehasher supports two database storage options:
+CrowsNest supports two database storage options:
 
-1. **Default Path** (default): Stores the database at `~/.local/share/Dehasher/db/dehashed.sqlite`
-2. **Local Path**: Stores the database in the current directory as `./dehasher.sqlite`
+1. **Default Path** (default): Stores the database at `~/.local/share/crowsnest/db/crowsnest.sqlite`
+2. **Local Path**: Stores the database in the current directory as `./crowsnest.sqlite`
 
 The **Local Path** option allows for separate databases for different projects or engagements.
 
@@ -63,213 +48,372 @@ To configure the database location:
 
 ```bash
 # Use local database in current directory
-./dehasher set-local-db true
+./crowsnest set-local-db true
 
 # Use default database path
-./dehasher set-local-db false
+./crowsnest set-local-db false
 ```
 
-<hr></hr>
+---
 
-## 🔍 Crafting Queries
+## 🌐 Dehashed
+
+### Initial Setup
+
+CrowsNest requires an API key from Dehashed. Set it up with:
+![Alt text](.img/set-dehashed.png "Set Dehashed Key")
+```bash
+ar1ste1a@kali:~$ crowsnest set dehashed <redacted>
+```
 
 ### Simple Query
-Dehasher can be used simply for example to query for credentials matching a given email domain.
+CrowsNest can be used simply for example to query for credentials matching a given email domain.
+![Alt text](.img/simple_query.png "Simple Query")
 ``` go
-# Provide credentials for emails matching @target.com
-dehasher -k ddq<redacted> -a ar1ste1a@domain.tld -E @target.com
+# Provide credentials for domains matching target.com
+crowsnest dehashed -D target.com
 ```
 
 ### Simple Credentials Query
-Dehasher can also be used to return only credentials for a given query.
+CrowsNest can also be used to return only credentials for a given query.
+![Alt text](.img/simple_creds_query.png "Creds Only Query")
 ``` go
 # Provide credentials for emails matching @target.com
-dehasher -E @target.com -C
-```
-
-### Multiple Match Query
-Dehasher is capable of handling multiple queries for the same field.  
-This is useful for when you want to search for multiple domains, or multiple usernames.
-``` go
-# Provide credentials for emails matching @target.com and @target2.com
-dehasher -E @target.com,@target2.com -C
+crowsnest dehashed -D @target.com -C
 ```
 
 ### Wildcard Query
-Dehasher is capable of handling wildcard queries.  
+CrowsNest is capable of handling wildcard queries.  
 A wildcard query cannot begin with a wildcard.  
 This is a limitation of the Dehashed API.
 An asterisk can be used to denote multiple characters, and a question mark can be used to denote a single character.
+<br>
+![Alt text](.img/wildcard_query.png "Wildcard Query")
 ``` go
 # Provide credentials for emails matching @target.com and @target2.com
-dehasher -E @target?.com -C -W
+crowsnest dehashed -E @target?.com -C -W
 ```
 
+### Email Query
+Dehashed has dictated that emails should be searched in the following format:
+`email:target.name&domain:target.com`.
+As such, to query an email, please use the following format (note, wildcard is not required but can be useful):
+<br>
+*see photo above in Wildcard Query*
+``` go
+# Provide credentials for emails matching target.*@target.com
+crowsnest dehashed -W -E 'target*' -D target.com
+```
+You may also query the domain and find emails as well
+``` go
+# Provide credentials for emails matching target.com
+crowsnest dehashed -D target.com -C
+```
+
+### Combining Queries
+CrowsNest is capable of combining queries.  
+This is useful for when you want to query for credentials matching a given email or domain, but only for a specific username.
+![Alt text](.img/combining_queries.png "Combined Query")
+``` go
+# Provide credentials for emails matching @target.com and username containing 'admin'
+crowsnest dehashed -D target.com -U admin
+```
 
 ### Regex Query
-Dehasher is capable of handling regex queries.  
+CrowsNest is capable of handling regex queries.  
 Simply denote regex queries with the `-R` flag.
 Place all regex queries in quotes with the corresponding query flag in single quotes.
+<br>
+!!!! *Currently, the Regex Operators appear to be broken. I am waiting on a response from Dehashed* !!!!
 ``` go
 # Return matches for emails matching this given regex query
-dehasher -R -e '[a-zA-Z0-9]+(?:\.[a-zA-Z0-9]+)?@target.com'
+crowsnest dehashed -R -E 'joh?n(ath[oa]n)' -D hotmail.com'
 ```
 
 ### Output Text (default JSON)
-Dehasher is capable of handling output formats.  
+CrowsNest is capable of handling output formats.  
 The default output format is JSON.  
 To change the output format, use the `-f` flag.  
-Dehasher currently supports JSON, YAML, XML, and TEXT output formats.
+CrowsNest currently supports JSON, YAML, XML, TEXT, and GREP output formats.
 ``` go
 # Return matches for usernames exactly matching "admin" and write to text file 'admins_file.txt'
-dehasher -U admin -o admins_file -f txt
+crowsnest dehashed -U admin -o admins_file -f txt
+
+# Return one key=value record per line in a greppable file 'admins_file.grep'
+crowsnest dehashed -U admin -o admins_file -f grep
 ```
 
-<hr></hr>
+### Data Wells
+DeHashed data wells are free to query and do not require a paid API account.
+``` go
+# List the first page of data wells and write 'data_wells.json'
+crowsnest dehashed data-wells
+
+# Sort by record count and write one key=value record per line
+crowsnest dehashed data-wells --sort records-DESC --count 50 -f grep -o data_wells
+```
+
+---
 
 ## 🌐 WhoIs Lookups
-Dehasher supports WHOIS lookups, history searches, reverse WHOIS searches, IP lookups, MX lookups, NS lookups, and subdomain scans.
+CrowsNest supports WHOIS lookups, history searches, reverse WHOIS searches, IP lookups, MX lookups, NS lookups, and subdomain scans.
 The WhoIs Lookups require a separate API Credit from the Dehashed API.
 
 ### Domain Lookup
+CrowsNest can perform a domain lookup for a given domain.
+This provides a tree view of the domain's WHOIS information.
+![Alt text](.img/whois_domain.png "WhoIs Tree View")
 ```bash
 # Perform a WHOIS lookup for example.com
-dehasher whois -d example.com
+crowsnest whois -d example.com
 ```
 
 ### History Lookup
 History Lookups require 25 credits. 
 This is a Dehashed API limitation.
+The history lookup is immediately written to file and not displayed in the terminal or stored in the database.
 ```bash
 # Perform a WHOIS history search for example.com
-dehasher whois -d example.com -H
+crowsnest whois -d example.com -H
+```
+
+### Subdomain Scan
+CrowsNest can perform a subdomain scan for a given domain.  
+This provides a list of all subdomains that match the given query.
+![Alt text](.img/whois_subdomain.png "WhoIs Tree View")
+```bash
+# Perform a WHOIS subdomain scan for google.com
+crowsnest whois -d google.com -s
 ```
 
 ### Reverse WHOIS Lookup
+CrowsNest can perform a reverse WHOIS lookup for given criteria.  
+This provides a list of all domains that match the given query.  
+The reverse WHOIS lookup is immediately written to file and not stored in the database.
+![Alt text](.img/whois_reverse.png "WhoIs Tree View")
 ```bash
 # Perform a reverse WHOIS lookup for example.com
-dehasher whois -I example.com
+crowsnest whois -I example.com
 ```
 
 ### IP Lookup
+CrowsNest can perform a reverse IP lookup for a given IP address.  
+This provides a list of all domains that match the given query.
+![Alt text](.img/whois_ip.png "WhoIs View")
 ```bash
 # Perform a reverse IP lookup for 8.8.8.8
-dehasher whois -i 8.8.8.8
+crowsnest whois -i 8.8.8.8
 ```
 
 ### MX Lookup
+CrowsNest can perform an MX lookup for a given MX hostname.  
+This provides a list of all domains that match the given query.
+![Alt text](.img/whois_mx.png "WhoIs Tree View")
 ```bash
 # Perform a reverse MX lookup for google.com
-dehasher whois -m google.com
+crowsnest whois -m stmp.google.com
 ```
 ### NS Lookup
+CrowsNest can perform an NS lookup for a given NS hostname.  
+This provides a list of all domains that match the given query.
+The picture below also includes the --debug global flag.
+![Alt text](.img/whois_ns.png "WhoIs Tree View")
 ```bash
 # Perform a reverse NS lookup for google.com
-dehasher whois -n google.com
+crowsnest whois -n google.com
 ```
-### Subdomain Scan
+
+
+---
+
+## 🌐 Hunter.io
+CrowsNest supports Hunter.io lookups.  
+Hunter.io lookups require a separate API Key from the Dehashed API.
+This can be set using the `set hunter` command.
+![Alt text](.img/set-hunter.png "Set Dehashed Key")
 ```bash
-# Perform a WHOIS subdomain scan for google.com
-dehasher whois -d google.com -s
+# Set the Hunter.io API key
+crowsnest set hunter <redacted>
 ```
 
-<hr></hr>
+### Domain Search
+CrowsNest can perform a domain search for a given domain.  
+This provides information about  company including a description, social media information and any technologies in use.<br>
+![Alt text](.img/hunter_domain.png "Hunter.io Domain Search")
+```bash
+# Perform a Hunter.io domain search for example.com
+crowsnest hunter -d example.com -D
+```
 
+### Email Verification
+CrowsNest can perform an email verification search for a given email.  
+This provides a verification and score of a given email address.
+![Alt text](.img/hunter_emailverification.png "Hunter.io Email Verification")
+```bash
+# Perform a Hunter.io email verification search for example@target.com
+crowsnest hunter -e example@target.com -V
+```
+
+### Company Enrichment
+CrowsNest can perform a company enrichment search for a given domain.  
+This provides information about a company given its domain.
+![Alt text](.img/hunter_company.png "Hunter.io Company Enrichment")
+```bash
+# Perform a Hunter.io company enrichment search for example.com
+crowsnest hunter -d example.com -C
+```
+
+### Email Finder
+CrowsNest can perform an email finder search for a given domain, first name, and last name.  
+This provides information about a user including a confidence score, and any social media accounts linked to a first name, last name and email.
+![Alt text](.img/hunter_emailfind.png "Hunter.io Email Finder")
+```bash
+# Perform a Hunter.io email finder search for example.com
+crowsnest hunter -d example.com -F John -L Doe -E
+```
+
+### Person Enrichment
+CrowsNest can perform a person enrichment search for a given email.  
+This provides information about a user given an email address..
+![Alt text](.img/hunter_person.png "Hunter.io Person Enrichment")
+```bash
+# Perform a Hunter.io person enrichment search for example@target.com
+crowsnest hunter -e example@target.com -P
+```
+
+### Combined Enrichment
+CrowsNest can perform a combined enrichment search for a given email.  
+This is a combination of the company and person enrichments given an email address.
+![Alt text](.img/combined_enrichment_1.png "Hunter.io Combined Enrichment")
+![Alt text](.img/combined_enrichment_2.png "Hunter.io Combined Enrichment")
+```bash
+# Perform a Hunter.io combined enrichment search for example@target.com
+crowsnest hunter -e example@target.com -B
+```
+## Debugging
+CrowsNest supports debugging.  This can be enabled using the `--debug` flag in the root command.
+![Alt text](.img/crowsnest_debugging_global.png "Debugging")
+```bash
+# Perform a Hunter.io combined enrichment search for example@target.com with debugging enabled
+crowsnest --debug hunter -e example@target.com -B
+```
+---
 ## 📊 Database Querying
-Dehasher stores query results in a local database.  
+CrowsNest stores query results in a local database.  
 This database can be queried for previous results.
+This is useful for when you want to query for specific information.
 This database also includes WhoIs Information and Subdomain Scan results, but does **not** include historical lookups.
 
 ## Simple Query
-![Alt text](.img/simple_query_db.png "Simple Query")
-
-Dehasher supports querying the database for previous results.  
-This is useful for when you want to query for specific information.
+#### It's possible to query the database using shorthand and without knowing any SQL at all.
+#### The following queries the results table where username is not null, only showing the username, email and password columns.
+![Alt text](.img/query_simple.png "Simple Query")
+#### You may also add in a simple query using the `-q` flag. The following displays a 'LIKE' clause on the email column.
+#### Note the '%\<clause\>%' is still required.
+![Alt text](.img/query_where.png "Simple Query")
+  
 ```bash
 # Query the database for all results containing the word 'admin' in the username
-dehasher query -t results -q "username LIKE '%admin%'"
+crowsnest query -t results -q "username LIKE '%admin%'"
 ```
 
 
 ## Raw SQL Queries
-![Alt text](.img/raw_query_db.png "Raw Query")
+![Alt text](.img/query_raw.png "Raw Query")
 
-Dehasher also supports raw SQL queries.  This is useful for when you want to query for specific information.
+CrowsNest also supports raw SQL queries.  This is useful for when you want to query for specific information.
 ```bash
 # Query the database for all results containing the word 'admin' in the username
-dehasher query -r "SELECT * FROM results WHERE username LIKE '%admin%'"
+crowsnest query -r "SELECT * FROM results WHERE username LIKE '%admin%'"
 ```
 
 ## Query Options
-Dehasher supports a number of query options.  These options can be used to filter the results of a query.
+CrowsNest supports a number of query options.  These options can be used to filter the results of a query.
 ```bash
 # Query the database for all results containing the word 'admin' in the username
-dehasher query -t results -q "username LIKE '%admin%'" -n username,email,password
+crowsnest query -t results -q "username LIKE '%admin%'" -n username,email,password
 ```
 
 ## Listing Tables and Columns
-Dehasher supports listing all available tables and columns.  
+CrowsNest supports listing all available tables and columns.  
 This is useful for when you want to query for specific information.
+![Alt text](.img/query_alltables.png "List All Tables")
 ```bash
 # List all available tables and columns
-dehasher query -a
+crowsnest query -a
 ```
 
 The current tables available for query are:
 - results
+  - Results from a dehashed query
 - creds
+  - Credentials parsed from dehashed results
 - whois
+  - Results from a whois record lookup
 - subdomains
-- history
+  - Subdomains discovered in a whois subdomain scan 
 - runs
+  - Previous query runs to the dehashed API
+- lookup 
+  - Results of any Whois NS, MX, or IP lookup
+- hunter_domain
+  - Results from a hunter.io domain search
+- hunter_email
+  - Results extracted from a domain saerch and email finder.
 
 ---
 
 # Exporting Results
-Dehasher supports exporting results to a file.  
+CrowsNest supports exporting results to a file.  
 This is useful for when you want to requery for specific information without touching the Dehashed API.
 The export subcommand supports all the same options as the query subcommand.
 The export subcommand also supports file naming and output format control.
+![Alt text](.img/export_raw.png "Export Results")
 ```bash
 # Export all results containing the word 'admin' in the username to a text file
-dehasher export -t results -q "username LIKE '%admin%'" -o admins_file -f txt
+crowsnest export -t results -q "username LIKE '%admin%'" -o admins_file -f txt
 ```
 
 ## 🐛 Debugging
 
-Dehasher uses the `zap` logging library for logging.  The logs are stored in `~/.local/share/Dehasher/logs`.
-The logs can be easily queried from the Dehasher CLI.
+CrowsNest uses the `zap` logging library for logging.  The logs are stored in `~/.local/share/crowsnest/logs`.
+The logs can be easily queried from the crowsnest CLI.
+
+### Filtering by Date
+#### CrowsNest utilizes 'easy time' to determine the appropriate time for a given query.
+![Alt text](.img/easy_time_parsing.png "Easy Time")
+#### You may also used dates mixed with easy time to perform queries.
+![Alt text](.img/mixed_time_query.png "Mixed Time")
+#### The following formats are supported:
+- `last 24 hours`
+- `last 2 days`
+- `30 minutes ago`
+- `45 seconds ago`
+- `1 week ago`
+- `05-01-2025`
+- `05/01/2025`
+- `05/01/25`
+- `05-01-25`
+- `May 01, 2025`
+
 ```bash
 # Show the last 10 logs
-dehasher logs -l 10
+crowsnest logs -l 10
 
 # Show logs from the last 24 hours
-dehasher logs -s "24 hours ago"
+crowsnest logs -s "last 24 hours"
 
 # Show logs from the last 24 hours with a severity of error or fatal
-dehasher logs -s "24 hours ago" -v error,fatal
-```
-## 🎉 Sample Run
-```bash
-ar1ste1a@kali:~$ dehasher api -D <redacted>.com -o <redacted> -f json
-Making 3 Requests for 10000 Records (30000 Total)
-[*] Querying Dehashed API...
-	[*] Performing Request...
-		[+] Retrieved 2740 Records
-        [-] Not Enough Entries, ending queries
-	[+] Discovered 10 Credentials
-	[*] Writing entries to file: <redacted>.json
-		[*] Success
-[*] Completing Process
+crowsnest logs -s "05-01-2025" -v error,fatal
 ```
 
 ## 🤝 Contributing
-Contributions are welcome! Submit a pull request to help improve Dehasher.
-
+Contributions are welcome! Submit a pull request to help improve CrowsNest.
 
+## [Buy Me A Coffee](https://buymeacoffee.com/ehosinskiz)
 
 <div align="center">
-    <img src="https://img.wanman.io/fUSu0/jUtovIFE52.png/raw" style="width: 350px; height: auto" alt="Ar1ste1a" title="Ar1ste1a Offensive Security">
+    <img src=.img/kraken_k.png style="width: 350px; height: auto" alt="Ar1ste1a" title="Ar1ste1a Offensive Security">
 </div>
 
 ## **Release The Kraken**

cmd/api.go

@@ -1,126 +0,0 @@
-package cmd
-
-import (
-	"dehasher/internal/badger"
-	"dehasher/internal/query"
-	"dehasher/internal/sqlite"
-	"fmt"
-	"github.com/spf13/cobra"
-)
-
-func init() {
-	// Add query command to root command
-	rootCmd.AddCommand(apiCmd)
-
-	// Add flags specific to query command
-	apiCmd.Flags().IntVarP(&maxRecords, "max-records", "m", 30000, "Maximum amount of records to return")
-	apiCmd.Flags().IntVarP(&maxRequests, "max-requests", "r", -1, "Maximum number of requests to make")
-	apiCmd.Flags().IntVarP(&startingPage, "starting-page", "s", 1, "Starting page for requests")
-	apiCmd.Flags().BoolVarP(&printBalance, "print-balance", "b", false, "Print remaining balance after requests")
-	apiCmd.Flags().BoolVarP(&regexMatch, "regex-match", "R", false, "Use regex matching on query fields")
-	apiCmd.Flags().BoolVarP(&wildcardMatch, "wildcard-match", "W", false, "Use wildcard matching on query fields (Use ? to replace a single character, and * for multiple characters)")
-	apiCmd.Flags().BoolVarP(&credsOnly, "creds-only", "C", false, "Return credentials only")
-	apiCmd.Flags().StringVarP(&outputFormat, "format", "f", "json", "Output format (json, yaml, xml, txt)")
-	apiCmd.Flags().StringVarP(&outputFile, "output", "o", "query", "File to output results to including extension")
-	apiCmd.Flags().StringVarP(&usernameQuery, "username", "U", "", "Username query")
-	apiCmd.Flags().StringVarP(&emailQuery, "email-query", "E", "", "Email query")
-	apiCmd.Flags().StringVarP(&ipQuery, "ip", "I", "", "IP address query")
-	apiCmd.Flags().StringVarP(&domainQuery, "domain", "D", "", "Domain query")
-	apiCmd.Flags().StringVarP(&passwordQuery, "password", "P", "", "Password query")
-	apiCmd.Flags().StringVarP(&vinQuery, "vin", "V", "", "VIN query")
-	apiCmd.Flags().StringVarP(&licensePlateQuery, "license", "L", "", "License plate query")
-	apiCmd.Flags().StringVarP(&addressQuery, "address", "A", "", "Address query")
-	apiCmd.Flags().StringVarP(&phoneQuery, "phone", "M", "", "Phone query")
-	apiCmd.Flags().StringVarP(&socialQuery, "social", "S", "", "Social query")
-	apiCmd.Flags().StringVarP(&cryptoCurrencyAddressQuery, "crypto", "B", "", "Crypto currency address query")
-	apiCmd.Flags().StringVarP(&hashQuery, "hash", "Q", "", "Hashed password query")
-	apiCmd.Flags().StringVarP(&nameQuery, "name", "N", "", "Name query")
-
-	// Add mutually exclusive flags to exact match and regex match
-	apiCmd.MarkFlagsMutuallyExclusive("regex-match", "wildcard-match")
-}
-
-var (
-	// Query command flags
-	maxRecords                 int
-	maxRequests                int
-	startingPage               int
-	credsOnly                  bool
-	printBalance               bool
-	regexMatch                 bool
-	wildcardMatch              bool
-	outputFormat               string
-	outputFile                 string
-	usernameQuery              string
-	emailQuery                 string
-	ipQuery                    string
-	passwordQuery              string
-	hashQuery                  string
-	nameQuery                  string
-	domainQuery                string
-	vinQuery                   string
-	licensePlateQuery          string
-	addressQuery               string
-	phoneQuery                 string
-	socialQuery                string
-	cryptoCurrencyAddressQuery string
-
-	// Query command
-	apiCmd = &cobra.Command{
-		Use:   "api",
-		Short: "Query the Dehashed API",
-		Long:  `Query the Dehashed API for emails, usernames, passwords, hashes, IP addresses, and names.`,
-		Run: func(cmd *cobra.Command, args []string) {
-			key := getStoredApiKey()
-
-			// Validate credentials
-			if key == "" {
-				fmt.Println("API key is required. Set the key with the \"set-key\" command. [dehasher set-key <api_key>]")
-				return
-			}
-
-			// Create new QueryOptions
-			queryOptions := sqlite.NewQueryOptions(
-				maxRecords,
-				maxRequests,
-				startingPage,
-				outputFormat,
-				outputFile,
-				usernameQuery,
-				emailQuery,
-				ipQuery,
-				passwordQuery,
-				hashQuery,
-				nameQuery,
-				domainQuery,
-				vinQuery,
-				licensePlateQuery,
-				addressQuery,
-				phoneQuery,
-				socialQuery,
-				cryptoCurrencyAddressQuery,
-				regexMatch,
-				wildcardMatch,
-				printBalance,
-				credsOnly,
-			)
-
-			// Create new Dehasher
-			dehasher := query.NewDehasher(queryOptions)
-			dehasher.SetClientCredentials(
-				key,
-			)
-
-			// Start querying
-			dehasher.Start()
-			fmt.Println("\n[*] Completing Process")
-
-			sqlite.StoreQueryOptions(queryOptions)
-		},
-	}
-)
-
-// Helper functions to get stored API credentials
-func getStoredApiKey() string {
-	return badger.GetKey()
-}

cmd/dehashed.go

@@ -0,0 +1,201 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+	"go.uber.org/zap"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/badger"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/debug"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/dehashed"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/files"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/pretty"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/sqlite"
+)
+
+func init() {
+	// Add api command to root command
+	rootCmd.AddCommand(dehashedCmd)
+	dehashedCmd.AddCommand(dehashedDataWellsCmd)
+
+	// Add flags specific to api command
+	dehashedCmd.Flags().IntVarP(&maxRecords, "max-records", "m", 50000, "Maximum total records to return (max 50000)")
+	dehashedCmd.Flags().IntVarP(&maxRequests, "max-requests", "r", -1, "Maximum number of requests to make")
+	dehashedCmd.Flags().IntVarP(&startingPage, "starting-page", "s", 1, "Starting page for requests")
+	dehashedCmd.Flags().BoolVarP(&printBalance, "print-balance", "b", false, "Print remaining balance after requests")
+	dehashedCmd.Flags().BoolVarP(&regexMatch, "regex-match", "R", false, "Use regex matching on query fields")
+	dehashedCmd.Flags().BoolVarP(&wildcardMatch, "wildcard-match", "W", false, "Use wildcard matching on query fields (Use ? to replace a single character, and * for multiple characters)")
+	dehashedCmd.Flags().BoolVarP(&credsOnly, "creds-only", "C", false, "Return credentials only")
+	dehashedCmd.Flags().StringVarP(&outputFormat, "format", "f", "json", "Output format (json, yaml, xml, txt, grep)")
+	dehashedCmd.Flags().StringVarP(&outputFile, "output", "o", "query", "File to output results to without extension")
+	dehashedCmd.Flags().StringVarP(&usernameQuery, "username", "U", "", "Username query")
+	dehashedCmd.Flags().StringVarP(&emailQuery, "email-query", "E", "", "Email query")
+	dehashedCmd.Flags().StringVarP(&ipQuery, "ip", "I", "", "IP address query")
+	dehashedCmd.Flags().StringVarP(&domainQuery, "domain", "D", "", "Domain query")
+	dehashedCmd.Flags().StringVarP(&passwordQuery, "password", "P", "", "Password query")
+	dehashedCmd.Flags().StringVarP(&vinQuery, "vin", "V", "", "VIN query")
+	dehashedCmd.Flags().StringVarP(&licensePlateQuery, "license", "L", "", "License plate query")
+	dehashedCmd.Flags().StringVarP(&addressQuery, "address", "A", "", "Address query")
+	dehashedCmd.Flags().StringVarP(&phoneQuery, "phone", "M", "", "Phone query")
+	dehashedCmd.Flags().StringVarP(&socialQuery, "social", "S", "", "Social query")
+	dehashedCmd.Flags().StringVarP(&cryptoCurrencyAddressQuery, "crypto", "B", "", "Crypto currency address query")
+	dehashedCmd.Flags().StringVarP(&hashQuery, "hash", "Q", "", "Hashed password query")
+	dehashedCmd.Flags().StringVarP(&nameQuery, "name", "N", "", "Name query")
+
+	// Add mutually exclusive flags to wildcard match and regex match
+	dehashedCmd.MarkFlagsMutuallyExclusive("regex-match", "wildcard-match")
+
+	dehashedDataWellsCmd.Flags().IntVar(&dataWellsCount, "count", 20, "Number of data wells to return (20 or 50)")
+	dehashedDataWellsCmd.Flags().IntVarP(&dataWellsPage, "page", "p", 1, "Data wells page to request")
+	dehashedDataWellsCmd.Flags().StringVar(&dataWellsSort, "sort", "", "Sort data wells by added, name, date, or records; optionally suffix -ASC or -DESC")
+	dehashedDataWellsCmd.Flags().StringVarP(&dataWellsOutputFormat, "format", "f", "json", "Output format (json, yaml, xml, txt, grep)")
+	dehashedDataWellsCmd.Flags().StringVarP(&dataWellsOutputFile, "output", "o", "data_wells", "File to output data wells to without extension")
+}
+
+var (
+	// Query command flags
+	maxRecords                 int
+	maxRequests                int
+	startingPage               int
+	credsOnly                  bool
+	printBalance               bool
+	regexMatch                 bool
+	wildcardMatch              bool
+	outputFormat               string
+	outputFile                 string
+	usernameQuery              string
+	emailQuery                 string
+	ipQuery                    string
+	passwordQuery              string
+	hashQuery                  string
+	nameQuery                  string
+	domainQuery                string
+	vinQuery                   string
+	licensePlateQuery          string
+	addressQuery               string
+	phoneQuery                 string
+	socialQuery                string
+	cryptoCurrencyAddressQuery string
+	dataWellsCount             int
+	dataWellsPage              int
+	dataWellsSort              string
+	dataWellsOutputFormat      string
+	dataWellsOutputFile        string
+
+	// Query command
+	dehashedCmd = &cobra.Command{
+		Use:   "dehashed",
+		Short: "Query the Dehashed API",
+		Long:  `Query the Dehashed API for emails, usernames, passwords, hashes, IP addresses, and names.`,
+		Run: func(cmd *cobra.Command, args []string) {
+			key := getDehashedApiKey()
+
+			// Validate credentials
+			if key == "" {
+				fmt.Println("API key is required. Set the key with the \"set dehashed\" command. [crowsnest set dehashed <api_key>]")
+				return
+			}
+
+			// Create new QueryOptions
+			queryOptions := sqlite.NewQueryOptions(
+				maxRecords,
+				maxRequests,
+				startingPage,
+				outputFormat,
+				outputFile,
+				usernameQuery,
+				emailQuery,
+				ipQuery,
+				passwordQuery,
+				hashQuery,
+				nameQuery,
+				domainQuery,
+				vinQuery,
+				licensePlateQuery,
+				addressQuery,
+				phoneQuery,
+				socialQuery,
+				cryptoCurrencyAddressQuery,
+				regexMatch,
+				wildcardMatch,
+				printBalance,
+				credsOnly,
+				debugGlobal,
+			)
+
+			// Create new Dehasher
+			dehasher := dehashed.NewDehasher(queryOptions)
+			dehasher.SetClientCredentials(
+				key,
+			)
+
+			// Start querying
+			dehasher.Start()
+			fmt.Println("\n[*] Completing Process")
+
+			// Store query options
+			err := sqlite.StoreDehashedQueryOptions(queryOptions)
+			if err != nil {
+				if debugGlobal {
+					debug.PrintInfo("failed to store query options")
+					debug.PrintError(err)
+				}
+				zap.L().Error("store_query_options",
+					zap.String("message", "failed to store query options"),
+					zap.Error(err),
+				)
+				fmt.Printf("Error storing query options: %v\n", err)
+			}
+		},
+	}
+
+	dehashedDataWellsCmd = &cobra.Command{
+		Use:   "data-wells",
+		Short: "List DeHashed data wells",
+		Long:  `List DeHashed data wells. This endpoint is free and does not require a DeHashed API key or subscription.`,
+		Run: func(cmd *cobra.Command, args []string) {
+			client := dehashed.NewDehashedClientV2("", debugGlobal)
+			response, err := client.DataWells(dehashed.DataWellsRequest{
+				Count: dataWellsCount,
+				Page:  dataWellsPage,
+				Sort:  dataWellsSort,
+			})
+			if err != nil {
+				fmt.Printf("[!] Error querying data wells: %v\n", err)
+				return
+			}
+
+			fType := files.GetFileType(dataWellsOutputFormat)
+			if dataWellsOutputFile != "" {
+				fmt.Printf("[*] Writing data wells to file: %s%s\n", dataWellsOutputFile, fType.Extension())
+				if err := dehashed.WriteDataWellsToFile(response, dataWellsOutputFile, fType); err != nil {
+					fmt.Printf("[!] Error writing data wells to file: %v\n", err)
+					return
+				}
+			}
+
+			fmt.Printf("[+] Retrieved %d data wells (total: %d, next page: %t)\n", len(response.DataWells), response.Total, response.NextPage)
+			printDataWellsTable(response.DataWells)
+		},
+	}
+)
+
+// Helper functions to get stored API credentials
+func getDehashedApiKey() string {
+	return badger.GetDehashedKey()
+}
+
+func printDataWellsTable(dataWells []dehashed.DataWell) {
+	headers := []string{"Name", "Date", "Records", "Sensitive", "Data"}
+	rows := make([][]string, 0, len(dataWells))
+	for _, well := range dataWells {
+		rows = append(rows, []string{
+			well.Name,
+			well.Date,
+			fmt.Sprintf("%d", well.Records),
+			fmt.Sprintf("%t", well.IsSensitive),
+			well.Data,
+		})
+	}
+	pretty.Table(headers, rows)
+}

cmd/export.go

@@ -1,311 +0,0 @@
-package cmd
-
-import (
-	"dehasher/internal/export"
-	"dehasher/internal/files"
-	"dehasher/internal/sqlite"
-	"fmt"
-	"github.com/spf13/cobra"
-	"go.uber.org/zap"
-	"strings"
-)
-
-func init() {
-	// Add Subcommand to db command
-	rootCmd.AddCommand(exportCmd)
-
-	// Add flags specific to export command
-	exportCmd.Flags().IntVarP(&exportLimitRows, "limit", "l", 100, "Limit number of results")
-	exportCmd.Flags().BoolVarP(&exportListAll, "list-all", "a", false, "List all tables and their columns")
-	exportCmd.Flags().StringVarP(&exportTableName, "table", "t", "", "Table to export (results, creds, whois, subdomains, history, runs)")
-	exportCmd.Flags().StringVarP(&exportNotNull, "not-null", "n", "", "Filter for non-null values (comma-separated list, e.g., 'password,email')")
-	exportCmd.Flags().StringVarP(&exportColumns, "columns", "c", "", "Columns to display in output (comma-separated list, e.g., 'username,email,password')")
-	exportCmd.Flags().StringVarP(&exportUserQuery, "user-query", "q", "", "User query to execute")
-	exportCmd.Flags().StringVarP(&exportRawQuery, "raw-query", "r", "", "Raw SQL query to execute")
-	exportCmd.Flags().StringVarP(&exportFormat, "format", "f", "json", "Output format (json, yaml, xml, txt)")
-	exportCmd.Flags().StringVarP(&exportFile, "file", "o", "export", "File to output results to including extension")
-
-	// Add mutually exclusive flags to query and raw-query
-	// Cannot use query and raw-query at the same time
-	exportCmd.MarkFlagsMutuallyExclusive("user-query", "raw-query")
-	// Raw query does not require a table
-	exportCmd.MarkFlagsMutuallyExclusive("user-query", "table")
-	// List all columns does not require a query or raw-query
-	exportCmd.MarkFlagsMutuallyExclusive("raw-query", "list-all")
-}
-
-// DB export command
-var (
-	exportLimitRows int
-	exportListAll   bool
-	exportTableName string
-	exportNotNull   string
-	exportColumns   string
-	exportUserQuery string
-	exportRawQuery  string
-	exportFormat    string
-	exportFile      string
-
-	exportCmd = &cobra.Command{
-		Use:   "export",
-		Short: "Export database to file",
-		Run: func(cmd *cobra.Command, args []string) {
-			// If list-all flag is set, list all tables and columns
-			if exportListAll {
-				listAvailableTables()
-				return
-			}
-
-			fmt.Println("[*] Exporting database...")
-
-			// If Raw Query is set, execute it and export
-			if exportRawQuery != "" {
-				fmt.Println("[*] Executing Raw Query...")
-				exportRawDBQuery()
-				return
-			}
-
-			// Validate table name
-			if exportTableName == "" {
-				fmt.Println("[!] Error: Table name is required. Use -t or --table to specify a table.")
-				fmt.Println("[*] Available tables: results, creds, whois, subdomains, history, runs")
-				fmt.Println("[*] Use --list-all to see all tables and their columns.")
-				return
-			}
-
-			if !isValidTable(exportTableName) {
-				fmt.Printf("[!] Error: Unknown table '%s'.\n", exportTableName)
-				fmt.Println("[*] Available tables: results, creds, whois, subdomains, history, runs")
-				fmt.Println("[*] Use --list-all to see all tables and their columns.")
-				return
-			}
-
-			// Validate columns if specified
-			if exportColumns != "" {
-				columns := strings.Split(exportColumns, ",")
-				invalidColumns := validateColumns(exportTableName, columns)
-				if len(invalidColumns) > 0 {
-					fmt.Printf("[!] Error: Invalid column(s) for table '%s': %s\n",
-						exportTableName, strings.Join(invalidColumns, ", "))
-					fmt.Println("[*] Available columns for this table:")
-					for i := 0; i < len(availableTables[exportTableName]); i += 5 {
-						end := i + 5
-						if end > len(availableTables[exportTableName]) {
-							end = len(availableTables[exportTableName])
-						}
-						fmt.Printf("    %s\n", strings.Join(availableTables[exportTableName][i:end], ", "))
-					}
-					return
-				}
-			}
-
-			// Validate not-null fields if specified
-			if exportNotNull != "" {
-				notNullFields := strings.Split(exportNotNull, ",")
-				invalidFields := validateColumns(exportTableName, notNullFields)
-				if len(invalidFields) > 0 {
-					fmt.Printf("[!] Error: Invalid not-null field(s) for table '%s': %s\n",
-						exportTableName, strings.Join(invalidFields, ", "))
-					fmt.Println("[*] Available columns for this table:")
-					for i := 0; i < len(availableTables[exportTableName]); i += 5 {
-						end := i + 5
-						if end > len(availableTables[exportTableName]) {
-							end = len(availableTables[exportTableName])
-						}
-						fmt.Printf("    %s\n", strings.Join(availableTables[exportTableName][i:end], ", "))
-					}
-					return
-				}
-			}
-
-			// Determine which table to query based on the tableTypeDBQuery parameter
-			table := sqlite.GetTable(exportTableName)
-			if table == sqlite.UnknownTable {
-				fmt.Printf("[!] Error: Unknown table type '%s'.\n", exportTableName)
-				fmt.Println("[*] Available tables: results, creds, whois, subdomains, history, runs")
-				fmt.Println("[*] Use --list-all to see all tables and their columns.")
-				return
-			}
-
-			fmt.Println("[*] Querying Database...")
-			exportTableQuery(table)
-		},
-	}
-)
-
-// exportTableQuery queries a table and exports the results
-func exportTableQuery(table sqlite.Table) {
-	// Get the columns to query
-	columns := []string{"*"}
-	if exportColumns != "" {
-		columns = strings.Split(exportColumns, ",")
-	}
-
-	// Get the not null fields
-	notNullFields := []string{}
-	if exportNotNull != "" {
-		notNullFields = strings.Split(exportNotNull, ",")
-	}
-
-	// Get the user query
-	userQuery := ""
-	if exportUserQuery != "" {
-		userQuery = exportUserQuery
-	}
-
-	// Get the limit
-	limit := exportLimitRows
-
-	// Get the object for the table
-	object := table.Object()
-
-	// Check if object is nil (invalid table)
-	if object == nil {
-		fmt.Printf("[!] Error: Table '%s' is not valid or does not exist.\n", exportTableName)
-		return
-	}
-
-	// Query the database
-	db := sqlite.GetDB()
-	query := db.Model(object).Select(columns)
-	if len(notNullFields) > 0 {
-		for _, field := range notNullFields {
-			query = query.Where(fmt.Sprintf("%s IS NOT NULL", field))
-		}
-	}
-	if userQuery != "" {
-		query = query.Where(userQuery)
-	}
-	if limit > 0 {
-		query = query.Limit(limit)
-	}
-	rows, err := query.Rows()
-	if err != nil {
-		zap.L().Error("export_query",
-			zap.String("message", "failed to execute query"),
-			zap.Error(err),
-		)
-		fmt.Printf("[!] Error executing query: %v\n", err)
-		return
-	}
-	defer rows.Close()
-
-	// Get the columns
-	cols, err := rows.Columns()
-	if err != nil {
-		zap.L().Error("export_query",
-			zap.String("message", "failed to get columns from query"),
-			zap.Error(err),
-		)
-		fmt.Printf("[!] Error getting columns from query: %v\n", err)
-		return
-	}
-
-	// Prepare data for export
-	var results []map[string]interface{}
-
-	// Process the rows
-	for rows.Next() {
-		values := make([]interface{}, len(cols))
-		pointers := make([]interface{}, len(cols))
-		for i := range values {
-			pointers[i] = &values[i]
-		}
-		if err := rows.Scan(pointers...); err != nil {
-			zap.L().Error("export_query",
-				zap.String("message", "failed to scan row from query"),
-				zap.Error(err),
-			)
-			fmt.Printf("[!] Error scanning row from query: %v\n", err)
-			return
-		}
-
-		// Create a map for this row
-		rowMap := make(map[string]interface{})
-		for i, col := range cols {
-			val := values[i]
-			rowMap[col] = val
-		}
-
-		results = append(results, rowMap)
-	}
-
-	// Export the results
-	exportResults(results)
-}
-
-// exportRawDBQuery executes a raw query and exports the results
-func exportRawDBQuery() {
-	db := sqlite.GetDB()
-	rows, err := db.Raw(exportRawQuery).Rows()
-	if err != nil {
-		zap.L().Error("export_raw_query",
-			zap.String("message", "failed to execute raw query"),
-			zap.Error(err),
-		)
-		fmt.Printf("[!] Error executing raw query: %v\n", err)
-		return
-	}
-	defer rows.Close()
-
-	columns, err := rows.Columns()
-	if err != nil {
-		zap.L().Error("export_raw_query",
-			zap.String("message", "failed to get columns from raw query"),
-			zap.Error(err),
-		)
-		fmt.Printf("[!] Error getting columns from raw query: %v\n", err)
-		return
-	}
-
-	// Prepare data for export
-	var results []map[string]interface{}
-
-	// Process the rows
-	for rows.Next() {
-		values := make([]interface{}, len(columns))
-		pointers := make([]interface{}, len(columns))
-		for i := range values {
-			pointers[i] = &values[i]
-		}
-		if err := rows.Scan(pointers...); err != nil {
-			zap.L().Error("export_raw_query",
-				zap.String("message", "failed to scan row from raw query"),
-				zap.Error(err),
-			)
-			fmt.Printf("[!] Error scanning row from raw query: %v\n", err)
-			return
-		}
-
-		// Create a map for this row
-		rowMap := make(map[string]interface{})
-		for i, col := range columns {
-			val := values[i]
-			rowMap[col] = val
-		}
-
-		results = append(results, rowMap)
-	}
-
-	// Export the results
-	exportResults(results)
-}
-
-// exportResults exports the results to a file
-func exportResults(results []map[string]interface{}) {
-	// Get file type
-	fileType := files.GetFileType(exportFormat)
-
-	// Export results
-	err := export.WriteQueryResultsToFile(results, exportFile, fileType)
-	if err != nil {
-		zap.L().Error("export_results",
-			zap.String("message", "failed to write to file"),
-			zap.Error(err),
-		)
-		fmt.Printf("[!] Error writing to file: %v\n", err)
-		return
-	}
-
-	fmt.Printf("[+] Exported %d records to file: %s%s\n", len(results), exportFile, fileType.Extension())
-}

cmd/hunter.go

@@ -0,0 +1,441 @@
+package cmd
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/spf13/cobra"
+	"go.uber.org/zap"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/badger"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/debug"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/export"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/files"
+	hunter "hub.krkn.tech/KrakenTech/crowsnest/internal/hunter.io"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/pretty"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/sqlite"
+)
+
+func init() {
+	// Add hunter command to root command
+	rootCmd.AddCommand(hunterCmd)
+
+	// Add flags specific to hunter command
+	hunterCmd.Flags().StringVarP(&hunterDomain, "domain", "d", "", "Domain to query")
+	hunterCmd.Flags().StringVarP(&hunterEmail, "email", "e", "", "Email to query")
+	hunterCmd.Flags().StringVarP(&hunterFirstName, "first-name", "F", "", "First name to query")
+	hunterCmd.Flags().StringVarP(&hunterLastName, "last-name", "L", "", "Last name to query")
+	hunterCmd.Flags().BoolVarP(&hunterDomainSearch, "domain-search", "D", false, "Search for domain")
+	hunterCmd.Flags().BoolVarP(&hunterEmailFind, "email-find", "E", false, "Find emails for user using domain, first name, and last name")
+	hunterCmd.Flags().BoolVarP(&hunterEmailVerify, "email-verify", "V", false, "Verify email")
+	hunterCmd.Flags().BoolVarP(&hunterCompanyEnrichmentDomain, "company-enrichment", "C", false, "Company enrichment for domain")
+	hunterCmd.Flags().BoolVarP(&hunterPersonEnrichmentEmail, "person-enrichment", "P", false, "Person enrichment for email")
+	hunterCmd.Flags().BoolVarP(&hunterCombinedEnrichmentEmail, "combined-enrichment", "B", false, "Combined Company and Person enrichment for email")
+	hunterCmd.Flags().StringVarP(&hunterOutputFormat, "format", "f", "json", "Output format (json, yaml, xml, txt)")
+	hunterCmd.Flags().StringVarP(&hunterOutputFile, "output", "o", "hunter", "File to output results to including extension")
+
+	// Add mutually exclusive flags to hunter command
+	hunterCmd.MarkFlagsMutuallyExclusive("email-find")
+
+}
+
+var (
+	// Hunter Commands Flags
+	hunterDomain                  string
+	hunterEmail                   string
+	hunterFirstName               string
+	hunterLastName                string
+	hunterDomainSearch            bool
+	hunterEmailFind               bool
+	hunterEmailVerify             bool
+	hunterCompanyEnrichmentDomain bool
+	hunterPersonEnrichmentEmail   bool
+	hunterCombinedEnrichmentEmail bool
+	hunterOutputFormat            string
+	hunterOutputFile              string
+
+	hunterCmd = &cobra.Command{
+		Use:   "hunter",
+		Short: "Hunter.io API interaction",
+		Long:  `Interact with the Hunter.io API for email and domain information.`,
+		Run: func(cmd *cobra.Command, args []string) {
+			if debugGlobal {
+				debug.PrintInfo("debug mode enabled")
+				zap.L().Info("hunter_debug",
+					zap.String("message", "debug mode enabled"),
+				)
+			}
+
+			// Flag Checks
+			if !hunterFlagCheck() {
+				return
+			}
+
+			if hunterOutputFile == "" {
+				if debugGlobal {
+					debug.PrintInfo("output file not specified, using default")
+				}
+				hunterOutputFile = "hunter_" + time.Now().Format("05_04_05")
+			}
+
+			if hunterOutputFormat == "" {
+				if debugGlobal {
+					debug.PrintInfo("output format not specified, using default")
+				}
+				hunterOutputFormat = "json"
+			}
+
+			fType := files.GetFileType(hunterOutputFormat)
+			if fType == files.UNKNOWN {
+				fmt.Println("[!] Error: Invalid output format. Must be 'json', 'xml', 'yaml', or 'txt'.")
+				return
+			}
+			if debugGlobal {
+				debug.PrintInfo("using output format: " + hunterOutputFormat)
+			}
+
+			fmt.Println("[*] Hunter.io API interaction [Beta]")
+
+			h := hunter.NewHunterIO(getHunterApiKey(), debugGlobal)
+
+			if hunterDomainSearch {
+				fmt.Println("[*] Performing domain search search...")
+				result, err := h.DomainSearch(hunterDomain)
+				if err != nil {
+					if debugGlobal {
+						debug.PrintInfo("failed to perform domain search")
+						debug.PrintError(err)
+					}
+					zap.L().Error("hunter_domain_search",
+						zap.String("message", "failed to perform domain search"),
+						zap.Error(err),
+					)
+					fmt.Printf("Error performing domain search: %v\n", err)
+					return
+				}
+
+				// Store the users discovered
+				var creds []sqlite.User
+				for _, email := range result.Emails {
+					creds = append(creds, sqlite.User{Email: email.Value})
+				}
+				err = sqlite.StoreUsers(creds)
+				if err != nil {
+					if debugGlobal {
+						debug.PrintInfo("failed to store hunter domain search")
+						debug.PrintError(err)
+					}
+					zap.L().Error("store_hunter_domain_search",
+						zap.String("message", "failed to store hunter domain search"),
+						zap.Error(err),
+					)
+					fmt.Printf("Error storing Hunter.io Domain Search Result: %v\n", err)
+				}
+
+				// Write Hunter.io Domain Search Result to file
+				fmt.Printf("[*] Writing Hunter.io Domain Search Result to file: %s%s\n", hunterOutputFile, fType.Extension())
+				err = export.WriteIStringToFile(result, hunterOutputFile, fType)
+				if err != nil {
+					if debugGlobal {
+						debug.PrintInfo("failed to write hunter domain search to file")
+						debug.PrintError(err)
+					}
+					zap.L().Error("write_hunter_domain_search",
+						zap.String("message", "failed to write hunter domain search to file"),
+						zap.Error(err),
+					)
+					fmt.Printf("Error writing Hunter.io Domain Search Result to file: %v\n", err)
+				}
+
+				// Pretty Print Hunter.io Domain Search Result
+				fmt.Println("Domain Search Result:")
+				pretty.HunterDomainTree(hunterDomain, result)
+				return
+			}
+
+			if hunterEmailFind {
+				fmt.Println("[*] Performing email find search...")
+				result, err := h.EmailFinder(hunterDomain, hunterFirstName, hunterLastName)
+				if err != nil {
+					if debugGlobal {
+						debug.PrintInfo("failed to perform email find")
+						debug.PrintError(err)
+					}
+					zap.L().Error("hunter_email_find",
+						zap.String("message", "failed to perform email find"),
+						zap.Error(err),
+					)
+					fmt.Printf("Error performing email find: %v\n", err)
+					return
+				}
+
+				// Write Hunter.io Email Finder Result to file
+				fmt.Printf("[*] Writing Hunter.io Email Finder Result to file: %s%s\n", hunterOutputFile, fType.Extension())
+				err = export.WriteIStringToFile(result, hunterOutputFile, fType)
+				if err != nil {
+					if debugGlobal {
+						debug.PrintInfo("failed to write hunter email find to file")
+						debug.PrintError(err)
+					}
+					zap.L().Error("write_hunter_email_find",
+						zap.String("message", "failed to write hunter email find to file"),
+						zap.Error(err),
+					)
+					fmt.Printf("Error writing Hunter.io Email Finder Result to file: %v\n", err)
+				}
+
+				fmt.Println("Email Find Result:")
+
+				var (
+					headers = []string{"Email", "Score", "Domain", "Accept All", "Position", "Twitter", "Linkedin", "Phone Number", "Company", "Verification"}
+					rows    [][]string
+				)
+
+				rows = append(rows, []string{
+					result.Email,
+					fmt.Sprintf("%d", result.Score),
+					result.Domain,
+					fmt.Sprintf("%t", result.AcceptAll),
+					result.Position,
+					result.Twitter,
+					result.LinkedinURL,
+					result.PhoneNumber,
+					result.Company,
+					fmt.Sprintf("%v", result.Verification),
+				})
+
+				pretty.Table(headers, rows)
+
+				return
+			}
+
+			if hunterEmailVerify {
+				fmt.Println("[*] Performing email verification search...")
+				result, err := h.EmailVerification(hunterEmail)
+				if err != nil {
+					if debugGlobal {
+						debug.PrintInfo("failed to perform email verification")
+						debug.PrintError(err)
+					}
+					zap.L().Error("hunter_email_verification",
+						zap.String("message", "failed to perform email verification"),
+						zap.Error(err),
+					)
+					fmt.Printf("Error performing email verification: %v\n", err)
+					return
+				}
+				// Write Hunter.io Email Verification Result to file
+				fmt.Printf("[*] Writing Hunter.io Email Verification Result to file: %s%s\n", hunterOutputFile, fType.Extension())
+				err = export.WriteIStringToFile(result, hunterOutputFile, fType)
+				if err != nil {
+					if debugGlobal {
+						debug.PrintInfo("failed to write hunter email verification to file")
+						debug.PrintError(err)
+					}
+					zap.L().Error("write_hunter_email_verification",
+						zap.String("message", "failed to write hunter email verification to file"),
+						zap.Error(err),
+					)
+					fmt.Printf("Error writing Hunter.io Email Verification Result to file: %v\n", err)
+				}
+
+				// Pretty Print Hunter.io Email Verification Result
+				var (
+					headers = []string{"Email", "Result", "Score", "Disposable", "MX Records", "SMTP Server", "SMTP Check"}
+					rows    [][]string
+				)
+				rows = append(rows, []string{
+					result.Email,
+					result.Result,
+					fmt.Sprintf("%d", result.Score),
+					fmt.Sprintf("%t", result.Disposable),
+					fmt.Sprintf("%t", result.MXRecords),
+					fmt.Sprintf("%t", result.SMTPServer),
+					fmt.Sprintf("%t", result.SMTPCheck),
+				})
+
+				fmt.Println("Email Verification Result:")
+				pretty.Table(headers, rows)
+				return
+			}
+
+			if hunterCompanyEnrichmentDomain {
+				fmt.Println("[*] Performing company enrichment search...")
+				result, err := h.CompanyEnrichment(hunterDomain)
+				if err != nil {
+					if debugGlobal {
+						debug.PrintInfo("failed to perform company enrichment")
+						debug.PrintError(err)
+					}
+					zap.L().Error("hunter_company_enrichment",
+						zap.String("message", "failed to perform company enrichment"),
+						zap.Error(err),
+					)
+					fmt.Printf("Error performing company enrichment: %v\n", err)
+					return
+				}
+
+				// Write to file
+				fmt.Printf("[*] Writing Hunter.io Company Enrichment Result to file: %s%s\n", hunterOutputFile, fType.Extension())
+				err = export.WriteIStringToFile(result, hunterOutputFile, fType)
+				if err != nil {
+					if debugGlobal {
+						debug.PrintInfo("failed to write hunter company enrichment to file")
+						debug.PrintError(err)
+					}
+					zap.L().Error("write_hunter_company_enrichment",
+						zap.String("message", "failed to write hunter company enrichment to file"),
+						zap.Error(err),
+					)
+					fmt.Printf("Error writing Hunter.io Company Enrichment Result to file: %v\n", err)
+				}
+
+				// Pretty Print Hunter.io Company Enrichment Result
+				fmt.Println("Company Enrichment Result:")
+				pretty.HunterCompanyEnrichmentTree(hunterDomain, result)
+
+				return
+			}
+
+			if hunterPersonEnrichmentEmail {
+				fmt.Println("[*] Performing person enrichment search...")
+				result, err := h.PersonEnrichment(hunterEmail)
+				if err != nil {
+					if debugGlobal {
+						debug.PrintInfo("failed to perform person enrichment")
+						debug.PrintError(err)
+					}
+					zap.L().Error("hunter_person_enrichment",
+						zap.String("message", "failed to perform person enrichment"),
+						zap.Error(err),
+					)
+					fmt.Printf("Error performing person enrichment: %v\n", err)
+					return
+				}
+
+				// Write to file
+				fmt.Printf("[*] Writing Hunter.io Person Enrichment Result to file: %s%s\n", hunterOutputFile, fType.Extension())
+				err = export.WriteIStringToFile(result, hunterOutputFile, fType)
+				if err != nil {
+					if debugGlobal {
+						debug.PrintInfo("failed to write hunter person enrichment to file")
+						debug.PrintError(err)
+					}
+					zap.L().Error("write_hunter_person_enrichment",
+						zap.String("message", "failed to write hunter person enrichment to file"),
+						zap.Error(err),
+					)
+					fmt.Printf("Error writing Hunter.io Person Enrichment Result to file: %v\n", err)
+				}
+
+				// Pretty Print Hunter.io Person Enrichment Result
+				fmt.Println("Person Enrichment Result:")
+				pretty.HunterPersonEnrichmentTree(hunterEmail, result)
+				return
+			}
+
+			if hunterCombinedEnrichmentEmail {
+				fmt.Println("[*] Performing combined enrichment search...")
+				result, err := h.CombinedEnrichment(hunterEmail)
+				if err != nil {
+					if debugGlobal {
+						debug.PrintInfo("failed to perform combined enrichment")
+						debug.PrintError(err)
+					}
+					zap.L().Error("hunter_combined_enrichment",
+						zap.String("message", "failed to perform combined enrichment"),
+						zap.Error(err),
+					)
+					fmt.Printf("Error performing combined enrichment: %v\n", err)
+					return
+				}
+
+				// Write to file
+				fmt.Printf("[*] Writing Hunter.io Combined Enrichment Result to file: %s%s\n", hunterOutputFile, fType.Extension())
+				err = export.WriteIStringToFile(result, hunterOutputFile, fType)
+				if err != nil {
+					if debugGlobal {
+						debug.PrintInfo("failed to write hunter combined enrichment to file")
+						debug.PrintError(err)
+					}
+					zap.L().Error("write_hunter_combined_enrichment",
+						zap.String("message", "failed to write hunter combined enrichment to file"),
+						zap.Error(err),
+					)
+					fmt.Printf("Error writing Hunter.io Combined Enrichment Result to file: %v\n", err)
+				}
+
+				fmt.Println("Combined Enrichment Result:")
+				pretty.HunterCombinedEnrichmentTree(hunterEmail, result)
+				return
+			}
+
+		},
+	}
+)
+
+func hunterFlagCheck() bool {
+	if debugGlobal {
+		debug.PrintInfo("checking flags")
+	}
+
+	var optionSet bool
+
+	if hunterDomainSearch {
+		if hunterDomain == "" {
+			fmt.Println("Domain is required for domain search")
+			return false
+		}
+		optionSet = true
+	}
+	if hunterEmailVerify {
+		if hunterEmail == "" {
+			fmt.Println("Email is required for email verification")
+			return false
+		}
+		optionSet = true
+	}
+	if hunterCompanyEnrichmentDomain {
+		if hunterDomain == "" {
+			fmt.Println("Domain is required for company enrichment")
+			return false
+		}
+		optionSet = true
+	}
+	if hunterPersonEnrichmentEmail {
+		if hunterEmail == "" {
+			fmt.Println("Email is required for person enrichment")
+			return false
+		}
+		optionSet = true
+	}
+	if hunterCombinedEnrichmentEmail {
+		if hunterEmail == "" {
+			fmt.Println("Email is required for combined enrichment")
+			return false
+		}
+		optionSet = true
+	}
+	if hunterEmailFind {
+		if hunterFirstName == "" || hunterLastName == "" {
+			fmt.Println("First name and last name are required for email find")
+			return false
+		}
+		if hunterDomain == "" {
+			fmt.Println("Domain is required for email find")
+			return false
+		}
+		optionSet = true
+	}
+
+	if !optionSet {
+		fmt.Println("[!] No options selected")
+		return false
+	}
+
+	return true
+}
+
+// Helper functions to get stored API credentials
+func getHunterApiKey() string {
+	return badger.GetHunterKey()
+}

cmd/logs.go

@@ -1,14 +1,16 @@
 package cmd
 
 import (
-	"dehasher/internal/pretty"
 	"encoding/json"
 	"fmt"
-	"github.com/spf13/cobra"
 	"os"
 	"path/filepath"
 	"strings"
 	"time"
+
+	"github.com/spf13/cobra"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/easyTime"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/pretty"
 )
 
 func init() {
@@ -73,6 +75,11 @@ var (
 				allLogs = append(allLogs, filepath.Join(logsPath, "info.log"), filepath.Join(logsPath, "error.log"))
 			}
 
+			var timeChunk easyTime.TimeChunk
+			if logStartDate != "" {
+				timeChunk = easyTime.NewTimeChunk(logStartDate, logEndDate, debugGlobal)
+			}
+
 			var parsedLogs []LogEntry
 			for _, logFile := range allLogs {
 				// Read the log file
@@ -97,7 +104,7 @@ var (
 						continue
 					}
 
-					// Also unmarshal to get additional fields
+					// Unmarshal to get additional fields
 					if err := json.Unmarshal([]byte(line), &rawEntry); err != nil {
 						fmt.Printf("Error parsing raw log entry: %v\n", err)
 						continue
@@ -106,10 +113,10 @@ var (
 					// Parse the timestamp
 					parsedTime, err := time.Parse("2006-01-02T15:04:05.999-0700", entry.Timestamp)
 					if err != nil {
-						// Try alternative formats
+						// Try RFC3339
 						parsedTime, err = time.Parse(time.RFC3339, entry.Timestamp)
 						if err != nil {
-							// Try another format
+							// Try RFC3339Nano
 							parsedTime, err = time.Parse(time.RFC3339Nano, entry.Timestamp)
 							if err != nil {
 								fmt.Printf("Error parsing timestamp '%s': %v\n", entry.Timestamp, err)
@@ -133,22 +140,8 @@ var (
 						(logFatal && strings.EqualFold(entry.Level, "FATAL")) {
 
 						// Filter by date range if specified
-						if logStartDate != "" {
-							startDate, err := time.Parse("2006-01-02", logStartDate)
-							if err != nil {
-								fmt.Printf("Error parsing start date: %v\n", err)
-							} else if entry.ParsedTime.Before(startDate) {
-								continue
-							}
-						}
-
-						if logEndDate != "" {
-							endDate, err := time.Parse("2006-01-02", logEndDate)
-							// Add one day to include the end date
-							endDate = endDate.Add(24 * time.Hour)
-							if err != nil {
-								fmt.Printf("Error parsing end date: %v\n", err)
-							} else if entry.ParsedTime.After(endDate) {
+						if timeChunk.IsSet() {
+							if entry.ParsedTime.Before(timeChunk.StartTime) || entry.ParsedTime.After(timeChunk.EndTime) {
 								continue
 							}
 						}
@@ -211,16 +204,3 @@ const (
 	FATAL
 	UNKNOWN Severity = -1
 )
-
-func getSeverity(logLevel string) Severity {
-	switch logLevel {
-	case "INFO":
-		return INFO
-	case "ERROR":
-		return ERROR
-	case "FATAL":
-		return FATAL
-	default:
-		return UNKNOWN
-	}
-}

cmd/query.go

@@ -1,137 +1,34 @@
 package cmd
 
 import (
-	"dehasher/internal/pretty"
-	"dehasher/internal/sqlite"
 	"encoding/json"
 	"fmt"
+	"strings"
+
 	"github.com/spf13/cobra"
 	"go.uber.org/zap"
-	"strings"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/debug"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/export"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/files"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/pretty"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/sqlite"
 )
 
-// Map of available tables and their columns
-var availableTables = map[string][]string{
-	"results": {
-		"id", "created_at", "updated_at", "deleted_at", "dehashed_id", "email", "ip_address", "username",
-		"password", "hashed_password", "hash_type", "name", "vin", "license_plate", "url", "social",
-		"cryptocurrency_address", "address", "phone", "company", "database_name",
-	},
-	"creds": {
-		"id", "created_at", "updated_at", "deleted_at", "email", "username", "password",
-	},
-	"whois": {
-		"id", "created_at", "updated_at", "deleted_at", "contact_email", "created_date",
-		"domain_name", "domain_name_ext", "expires_date", "status", "whois_server",
-	},
-	"subdomains": {
-		"id", "created_at", "updated_at", "deleted_at", "domain", "first_seen", "last_seen",
-	},
-	"history": {
-		"id", "created_at", "updated_at", "deleted_at", "domain_name", "domain_type",
-		"registrar_name", "whois_server", "created_date_iso8601", "updated_date_iso8601", "expires_date_iso8601",
-	},
-	"runs": {
-		"id", "created_at", "updated_at", "deleted_at", "max_records", "max_requests", "starting_page",
-		"output_format", "output_file", "regex_match", "wildcard_match", "username_query", "email_query",
-		"ip_query", "pass_query", "hash_query", "name_query", "domain_query", "vin_query", "license_plate_query",
-		"address_query", "phone_query", "social_query", "crypto_address_query", "print_balance", "creds_only",
-	},
-}
-
-// Function to list available tables and their columns
-func listAvailableTables() {
-	fmt.Println("Available tables and columns:")
-
-	// Prepare data for pretty.Table
-	headers := []string{"Table", "Columns"}
-	var tableRows [][]string
-
-	// Sort tables alphabetically for consistent output
-	var tableNames []string
-	for tableName := range availableTables {
-		tableNames = append(tableNames, tableName)
-	}
-
-	// Simple bubble sort for table names
-	for i := 0; i < len(tableNames)-1; i++ {
-		for j := 0; j < len(tableNames)-i-1; j++ {
-			if tableNames[j] > tableNames[j+1] {
-				tableNames[j], tableNames[j+1] = tableNames[j+1], tableNames[j]
-			}
-		}
-	}
-
-	// Create rows for the table
-	for _, tableName := range tableNames {
-		columns := availableTables[tableName]
-
-		// Format columns with line breaks for better readability
-		var formattedColumns string
-		for i := 0; i < len(columns); i += 5 {
-			end := i + 5
-			if end > len(columns) {
-				end = len(columns)
-			}
-			if i > 0 {
-				formattedColumns += "\n"
-			}
-			formattedColumns += strings.Join(columns[i:end], ", ")
-		}
-
-		tableRows = append(tableRows, []string{tableName, formattedColumns})
-	}
-
-	// Display the table
-	pretty.Table(headers, tableRows)
-}
-
-// Function to validate table name
-func isValidTable(tableName string) bool {
-	_, exists := availableTables[tableName]
-	return exists
-}
-
-// Function to validate column names for a specific table
-func validateColumns(tableName string, columns []string) []string {
-	if tableName == "" || columns == nil || len(columns) == 0 || columns[0] == "*" {
-		return nil
-	}
-
-	tableColumns, exists := availableTables[tableName]
-	if !exists {
-		return []string{fmt.Sprintf("Table '%s' does not exist", tableName)}
-	}
-
-	var invalidColumns []string
-	for _, col := range columns {
-		valid := false
-		for _, tableCol := range tableColumns {
-			if col == tableCol {
-				valid = true
-				break
-			}
-		}
-		if !valid {
-			invalidColumns = append(invalidColumns, col)
-		}
-	}
-
-	return invalidColumns
-}
-
 func init() {
 	// Add whois command to root command
 	rootCmd.AddCommand(queryCmd)
 
 	// Add flags specific to whois command
-	queryCmd.Flags().StringVarP(&dbQueryTableName, "table", "t", "", "Table to query (results, creds, whois, subdomains, history, runs)")
+	queryCmd.Flags().StringVarP(&dbQueryTableName, "table", "t", "", "Table to query (dehashed, users, whois, subdomains, lookup, runs)")
 	queryCmd.Flags().IntVarP(&dbQueryLimitRows, "limit", "l", 100, "Limit number of results")
 	queryCmd.Flags().StringVarP(&dbQueryNotNull, "not-null", "n", "", "Filter for non-null values (comma-separated list, e.g., 'password,email')")
 	queryCmd.Flags().StringVarP(&dbQueryColumns, "columns", "c", "", "Columns to display in output (comma-separated list, e.g., 'username,email,password')")
 	queryCmd.Flags().StringVarP(&dbQueryUserQuery, "user-query", "q", "", "User query to execute")
 	queryCmd.Flags().StringVarP(&dbQueryRawQuery, "raw-query", "r", "", "Raw SQL query to execute")
 	queryCmd.Flags().BoolVarP(&dbQueryListAll, "list-all", "a", false, "List all tables and their columns")
+	queryCmd.Flags().BoolVarP(&dbQueryExport, "export", "x", false, "Export results to file using --file and --format")
+	queryCmd.Flags().StringVarP(&dbQueryFormat, "format", "f", "json", "Output format (json, yaml, xml, txt, grep)")
+	queryCmd.Flags().StringVarP(&dbQueryFile, "file", "o", "query", "File to output results to when --export is set")
 
 	// Add mutually exclusive flags to query and raw-query
 	// Cannot use query and raw-query at the same time
@@ -150,11 +47,15 @@ var (
 	dbQueryUserQuery string
 	dbQueryRawQuery  string
 	dbQueryListAll   bool
+	dbQueryExport    bool
+	dbQueryFormat    string
+	dbQueryFile      string
 
 	queryCmd = &cobra.Command{
 		Use:   "query",
 		Short: "Query the database",
-		Long:  `Query the database for various information.`,
+		Long: `Query the database for various information.
+Use --export with --file and --format to write results to a file instead of displaying them.`,
 		Run: func(cmd *cobra.Command, args []string) {
 			// If list-all flag is set, list all tables and columns
 			if dbQueryListAll {
@@ -172,14 +73,14 @@ var (
 			// Validate table name
 			if dbQueryTableName == "" {
 				fmt.Println("[!] Error: Table name is required. Use -t or --table to specify a table.")
-				fmt.Println("[*] Available tables: results, creds, whois, subdomains, history, runs")
+				fmt.Println("[*] Available tables: dehashed, users, whois, subdomains, lookup, runs")
 				fmt.Println("[*] Use --list-all to see all tables and their columns.")
 				return
 			}
 
 			if !isValidTable(dbQueryTableName) {
 				fmt.Printf("[!] Error: Unknown table '%s'.\n", dbQueryTableName)
-				fmt.Println("[*] Available tables: results, creds, whois, subdomains, history, runs")
+				fmt.Println("[*] Available tables: dehashed, users, whois, subdomains, lookup, runs")
 				fmt.Println("[*] Use --list-all to see all tables and their columns.")
 				return
 			}
@@ -226,7 +127,7 @@ var (
 			table := sqlite.GetTable(dbQueryTableName)
 			if table == sqlite.UnknownTable {
 				fmt.Printf("[!] Error: Unknown table type '%s'.\n", dbQueryTableName)
-				fmt.Println("[*] Available tables: results, creds, whois, subdomains, history, runs")
+				fmt.Println("[*] Available tables: dehashed, users, whois, subdomains, lookup, runs")
 				fmt.Println("[*] Use --list-all to see all tables and their columns.")
 				return
 			}
@@ -305,6 +206,46 @@ func tableQuery(table sqlite.Table) {
 		return
 	}
 
+	if dbQueryExport {
+		fmt.Println("[*] Exporting results to file...")
+
+		if debugGlobal {
+			debug.PrintInfo("exporting results to file: " + dbQueryFile)
+		}
+		// Prepare data for export
+		var results []map[string]interface{}
+
+		// Process the rows
+		for rows.Next() {
+			values := make([]interface{}, len(cols))
+			pointers := make([]interface{}, len(cols))
+			for i := range values {
+				pointers[i] = &values[i]
+			}
+			if err := rows.Scan(pointers...); err != nil {
+				zap.L().Error("export_query",
+					zap.String("message", "failed to scan row from query"),
+					zap.Error(err),
+				)
+				fmt.Printf("[!] Error scanning row from query: %v\n", err)
+				return
+			}
+
+			// Create a map for this row
+			rowMap := make(map[string]interface{})
+			for i, col := range cols {
+				val := values[i]
+				rowMap[col] = val
+			}
+
+			results = append(results, rowMap)
+		}
+
+		// Export the results
+		exportQueryResults(results)
+
+		return
+	}
 	// Prepare data for pretty.Table
 	headers := cols
 	var tableRows [][]string
@@ -395,6 +336,47 @@ func rawDBQuery() {
 		return
 	}
 
+	if dbQueryExport {
+		fmt.Println("[*] Exporting results to file...")
+
+		if debugGlobal {
+			debug.PrintInfo("exporting results to file: " + dbQueryFile)
+		}
+
+		// Prepare data for export
+		var results []map[string]interface{}
+
+		// Process the rows
+		for rows.Next() {
+			values := make([]interface{}, len(columns))
+			pointers := make([]interface{}, len(columns))
+			for i := range values {
+				pointers[i] = &values[i]
+			}
+			if err := rows.Scan(pointers...); err != nil {
+				zap.L().Error("export_raw_query",
+					zap.String("message", "failed to scan row from raw query"),
+					zap.Error(err),
+				)
+				fmt.Printf("[!] Error scanning row from raw query: %v\n", err)
+				return
+			}
+
+			// Create a map for this row
+			rowMap := make(map[string]interface{})
+			for i, col := range columns {
+				val := values[i]
+				rowMap[col] = val
+			}
+
+			results = append(results, rowMap)
+		}
+
+		// Export the results
+		exportQueryResults(results)
+
+		return
+	}
 	// Prepare data for pretty.Table
 	headers := columns
 	var tableRows [][]string
@@ -461,3 +443,22 @@ func rawDBQuery() {
 	// Display the table
 	pretty.Table(headers, tableRows)
 }
+
+// exportQueryResults exports the results to a file
+func exportQueryResults(results []map[string]interface{}) {
+	// Get file type
+	fileType := files.GetFileType(dbQueryFormat)
+
+	// Export results
+	err := export.WriteQueryResultsToFile(results, dbQueryFile, fileType)
+	if err != nil {
+		zap.L().Error("export_results",
+			zap.String("message", "failed to write to file"),
+			zap.Error(err),
+		)
+		fmt.Printf("[!] Error writing to file: %v\n", err)
+		return
+	}
+
+	fmt.Printf("[+] Exported %d records to file: %s%s\n", len(results), dbQueryFile, fileType.Extension())
+}

cmd/root.go

@@ -1,47 +1,36 @@
 package cmd
 
 import (
-	"dehasher/internal/badger"
 	"fmt"
-	"github.com/spf13/cobra"
-	"go.uber.org/zap"
 	"os"
 	"strings"
+
+	"github.com/fatih/color"
+	"github.com/spf13/cobra"
+	"go.uber.org/zap"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/badger"
 )
 
 var (
 	// Global Flags
+	debugGlobal bool
 
 	// rootCmd is the base command for the CLI.
 	rootCmd = &cobra.Command{
-		Use:   "dehasher",
-		Short: `Dehasher is a cli tool for querying query.`,
+		Use:   "crowsnest",
+		Short: `CrowsNest is a cli tool for querying the common OSINT api's.`,
 		Long: fmt.Sprintf(
-			"%s\n%s",
+			"%s\n",
 			`
- ______   _______           _______  _______           _______  _______
-(  __  \ (  ____ \|\     /|(  ___  )(  ____ \|\     /|(  ____ \(  ____ )
-| (  \  )| (    \/| )   ( || (   ) || (    \/| )   ( || (    \/| (    )|
-| |   ) || (__    | (___) || (___) || (_____ | (___) || (__    | (____)|
-| |   | ||  __)   |  ___  ||  ___  |(_____  )|  ___  ||  __)   |     __)
-| |   ) || (      | (   ) || (   ) |      ) || (   ) || (      | (\ (
-| (__/  )| (____/\| )   ( || )   ( |/\____) || )   ( || (____/\| ) \ \__
-(______/ (_______/|/     \||/     \|\_______)|/     \|(_______/|/   \__/
-						     An Ar1ste1a Project
-`,
-			`––•–√\/––√\/––•––––•–√\/––√\/––•––––•–√\/––√\/––•––√\/––•––––•–√\/––√\/––•––
-  Dehasher can query the query API for:
-  - Emails		- Usernames 		- Password
-  - Hashes 		- IP Addresses		- Names
-  - VINs		- License Plates	- Addresses
-  - Phones		- Social Media		- Crypto Currency Addresses
-  Dehasher supports:
-  - Regex Matching
-  - Exact Matching
-––•–√\/––√\/––•––––•–√\/––√\/––•––––•–√\/––√\/––•––√\/––•––––•–√\/––√\/––•––
+    ╔═╗┬─┐┌─┐┬ ┬┌─┐╔╗╔┌─┐┌─┐┌┬┐
+    ║  ├┬┘│ ││││└─┐║║║├┤ └─┐ │
+    ╚═╝┴└─└─┘└┴┘└─┘╝╚╝└─┘└─┘ ┴
+
+   Crow’s Nest OSINT Recon Suite
+ ⚓ A KrakenTech Intelligence Tool
 `,
 		),
-		Version: "v1.0",
+		Version: "v1.2.1",
 	}
 )
 
@@ -58,31 +47,54 @@ func Execute() {
 }
 
 func init() {
-	//// Attempt to retrieve the useLocalDB
-	//useLocalDatabase := badger.GetUseLocalDB()
-
 	// Hide the default help command
 	rootCmd.CompletionOptions.HiddenDefaultCmd = true
 
-	//// Add global flags
-	//rootCmd.PersistentFlags().BoolVar(&useLocalDatabase, "local-db", useLocalDatabase, "Use local database in current directory instead of default path")
+	// Add global flags
+	rootCmd.PersistentFlags().BoolVar(&debugGlobal, "debug", false, "Show debug information")
 
 	// Add subcommands
-	rootCmd.AddCommand(setKeyCmd)
+	rootCmd.AddCommand(setCmd)
 	rootCmd.AddCommand(setLocalDb)
+	rootCmd.AddCommand(buyMeCoffeeCmd)
+
+	setCmd.AddCommand(setDehashedKeyCmd)
+	setCmd.AddCommand(setHunterKeyCmd)
+}
+
+var setCmd = &cobra.Command{
+	Use:   "set",
+	Short: "Set CrowsNest configuration values",
+	Long:  "Set CrowsNest configuration values such as API keys.",
 }
 
 // Command to set API key
-var setKeyCmd = &cobra.Command{
-	Use:   "set-key [key]",
-	Short: "Set and store API key",
+var setDehashedKeyCmd = &cobra.Command{
+	Use:   "dehashed [key]",
+	Short: "Set and store Dehashed.com API key",
 	Args:  cobra.ExactArgs(1),
 	Run: func(cmd *cobra.Command, args []string) {
 		key := args[0]
 		// Store key in badger DB
-		err := storeApiKey(key)
+		err := storeDehashedApiKey(key)
 		if err != nil {
-			fmt.Printf("Error storing API key: %v\n", err)
+			fmt.Printf("Error storing Dehashed API key: %v\n", err)
+			return
+		}
+		fmt.Println("API key stored successfully")
+	},
+}
+
+var setHunterKeyCmd = &cobra.Command{
+	Use:   "hunter [key]",
+	Short: "Set and store Hunter.io API key",
+	Args:  cobra.ExactArgs(1),
+	Run: func(cmd *cobra.Command, args []string) {
+		key := args[0]
+		// Store key in badger DB
+		err := storeHunterApiKey(key)
+		if err != nil {
+			fmt.Printf("Error storing Hunter API key: %v\n", err)
 			return
 		}
 		fmt.Println("API key stored successfully")
@@ -91,7 +103,7 @@ var setKeyCmd = &cobra.Command{
 
 var setLocalDb = &cobra.Command{
 	Use:   "local-db [true|false]",
-	Short: "Set dehasher to use a local database path instead of the default path",
+	Short: "Set crowsnest to use a local database path instead of the default path",
 	Args:  cobra.ExactArgs(1),
 	Run: func(cmd *cobra.Command, args []string) {
 		var useLocalDatabase bool
@@ -117,9 +129,32 @@ var setLocalDb = &cobra.Command{
 	},
 }
 
+var buyMeCoffeeCmd = &cobra.Command{
+	Use:   "coffee",
+	Short: "Support the project by buying a coffee",
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println(color.HiRedString("                                                                                   ;)(; "))
+		fmt.Println(color.HiCyanString("                   We Hope You Enjoy Our Product                                  :----:"))
+		fmt.Println(color.HiCyanString("                                                                                 C|====|"))
+		fmt.Println(color.HiCyanString("                                                                                  |    |"))
+		fmt.Print(color.HiGreenString(" Support the project by buying a coffee: "))
+		fmt.Print(color.BlueString("https://buymeacoffee.com/ehosinskiz      "))
+		fmt.Println(color.HiCyanString("`----'"))
+	},
+}
+
 // Helper functions to store API credentials
-func storeApiKey(key string) error {
-	err := badger.StoreKey(key)
+func storeDehashedApiKey(key string) error {
+	err := badger.StoreDehashedKey(key)
+	if err != nil {
+		fmt.Printf("Error storing API key: %v\n", err)
+		return err
+	}
+	return nil
+}
+
+func storeHunterApiKey(key string) error {
+	err := badger.StoreHunterKey(key)
 	if err != nil {
 		fmt.Printf("Error storing API key: %v\n", err)
 		return err

cmd/tables.go

@@ -0,0 +1,134 @@
+package cmd
+
+import (
+	"fmt"
+	"strings"
+
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/pretty"
+)
+
+// Map of available tables and their columns
+var availableTables = map[string][]string{
+	"users": {
+		"id", "created_at", "updated_at", "deleted_at", "email", "username", "password",
+	},
+	//"history": {
+	//	"id", "created_at", "updated_at", "deleted_at", "domain_name", "domain_type",
+	//	"registrar_name", "whois_server", "created_date_iso8601", "updated_date_iso8601", "expires_date_iso8601",
+	//},
+	"lookup": {
+		"id", "created_at", "updated_at", "deleted_at", "search_term", "type", "first_seen", "last_visit",
+		"name",
+	},
+	// Query Options
+	"runs": {
+		"id", "created_at", "updated_at", "deleted_at", "max_records", "max_requests", "starting_page",
+		"output_format", "output_file", "regex_match", "wildcard_match", "username_query", "email_query",
+		"ip_query", "pass_query", "hash_query", "name_query", "domain_query", "vin_query", "license_plate_query",
+		"address_query", "phone_query", "social_query", "crypto_address_query", "print_balance", "creds_only",
+	},
+	"dehashed": {
+		"id", "created_at", "updated_at", "deleted_at", "dehashed_id", "email", "ip_address", "username",
+		"password", "hashed_password", "hash_type", "name", "vin", "license_plate", "url", "social",
+		"cryptocurrency_address", "address", "phone", "company", "database_name",
+	},
+	"subdomains": {
+		"id", "created_at", "updated_at", "deleted_at", "domain", "subdomain",
+	},
+	"whois": {
+		"id", "created_at", "updated_at", "deleted_at", "audit", "contact_email", "created_date", "created_date_normalized",
+		"domain_name", "domain_name_ext", "estimated_domain_age", "expires_date", "expires_date_normalized", "footer", "header",
+		"name_servers", "parse_code", "raw_text", "registrant", "registrar_iana_id", "registrar_name", "registry_data",
+		"status", "stripped_text", "updated_date", "updated_date_normalized",
+	},
+	"hunter_domain": {
+		"id", "created_at", "updated_at", "deleted_at", "domain", "disposable", "webmail", "accept_all", "pattern",
+		"organization", "description", "industry", "twitter", "facebook", "linkedin", "instagram", "youtube",
+		"technologies", "country", "state", "city", "postal_code", "street", "headcount", "company_type", "emails", "linked_domains",
+	},
+	"hunter_email": {
+		"id", "created_at", "updated_at", "deleted_at", "value", "type", "confidence", "sources", "first_name", "last_name",
+		"position", "position_raw", "seniority", "department", "linkedin", "twitter", "phone_number", "verification_date", "verification_status",
+	},
+}
+
+// Function to list available tables and their columns
+func listAvailableTables() {
+	fmt.Println("Available tables and columns:")
+
+	// Prepare data for pretty.Table
+	headers := []string{"Table", "Columns"}
+	var tableRows [][]string
+
+	// Sort tables alphabetically for consistent output
+	var tableNames []string
+	for tableName := range availableTables {
+		tableNames = append(tableNames, tableName)
+	}
+
+	// Simple bubble sort for table names
+	for i := 0; i < len(tableNames)-1; i++ {
+		for j := 0; j < len(tableNames)-i-1; j++ {
+			if tableNames[j] > tableNames[j+1] {
+				tableNames[j], tableNames[j+1] = tableNames[j+1], tableNames[j]
+			}
+		}
+	}
+
+	// Create rows for the table
+	for _, tableName := range tableNames {
+		columns := availableTables[tableName]
+
+		// Format columns with line breaks for better readability
+		var formattedColumns string
+		for i := 0; i < len(columns); i += 5 {
+			end := i + 5
+			if end > len(columns) {
+				end = len(columns)
+			}
+			if i > 0 {
+				formattedColumns += "\n"
+			}
+			formattedColumns += strings.Join(columns[i:end], ", ")
+		}
+
+		tableRows = append(tableRows, []string{tableName, formattedColumns})
+	}
+
+	// Display the table
+	pretty.Table(headers, tableRows)
+}
+
+// Function to validate table name
+func isValidTable(tableName string) bool {
+	_, exists := availableTables[tableName]
+	return exists
+}
+
+// Function to validate column names for a specific table
+func validateColumns(tableName string, columns []string) []string {
+	if tableName == "" || columns == nil || len(columns) == 0 || columns[0] == "*" {
+		return nil
+	}
+
+	tableColumns, exists := availableTables[tableName]
+	if !exists {
+		return []string{fmt.Sprintf("Table '%s' does not exist", tableName)}
+	}
+
+	var invalidColumns []string
+	for _, col := range columns {
+		valid := false
+		for _, tableCol := range tableColumns {
+			if col == tableCol {
+				valid = true
+				break
+			}
+		}
+		if !valid {
+			invalidColumns = append(invalidColumns, col)
+		}
+	}
+
+	return invalidColumns
+}

cmd/targets.go

@@ -0,0 +1,313 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/spf13/cobra"
+	"go.uber.org/zap"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/sqlite"
+)
+
+func init() {
+	// Add targets command to root command
+	rootCmd.AddCommand(targetsCmd)
+
+	// Add flags specific to targets command
+	targetsCmd.Flags().StringVarP(&targetsOutputFile, "output", "o", "targets", "Output file name (required)")
+	targetsCmd.Flags().BoolVarP(&targetsExternal, "external", "e", false, "Output external format (email:password)")
+	targetsCmd.Flags().BoolVarP(&targetsInternal, "internal", "i", false, "Output internal format (username:password)")
+	targetsCmd.Flags().BoolVarP(&targetsSubdomains, "subdomains", "s", false, "Output subdomains")
+	targetsCmd.Flags().BoolVarP(&targetsEmails, "emails", "E", false, "Output emails only (no passwords)")
+	targetsCmd.Flags().StringVarP(&targetsDomain, "domain", "d", "", "Filter by domain (for emails and subdomains)")
+
+	// Mark output flag as required
+	targetsCmd.MarkFlagsMutuallyExclusive("external", "internal", "subdomains", "emails")
+}
+
+var (
+	// Targets command flags
+	targetsOutputFile string
+	targetsExternal   bool
+	targetsInternal   bool
+	targetsSubdomains bool
+	targetsEmails     bool
+	targetsDomain     string
+
+	// Targets command
+	targetsCmd = &cobra.Command{
+		Use:   "targets",
+		Short: "Export users and subdomains in formats suitable for external tools",
+		Long: `Export users and subdomains from the database in easily digestible formats for tools like sprays or other security testing tools.
+
+Formats:
+  --external (-e): Output in email:password format
+  --internal (-i): Output in username:password format
+  --emails (-E): Output emails only (no passwords)
+  --subdomains (-s): Output subdomains only
+
+Options:
+  --domain (-d): Filter results by domain (applies to emails and subdomains)
+  --output (-o): Specify output file name (required)
+
+Examples:
+  # Export all external credentials (email:password)
+  crowsnest targets -e -o external_creds
+
+  # Export internal credentials for a specific domain
+  crowsnest targets -i -d example.com -o internal_creds
+
+  # Export all emails
+  crowsnest targets -E -o all_emails
+
+  # Export emails for a specific domain
+  crowsnest targets -E -d example.com -o domain_emails
+
+  # Export subdomains for a specific domain
+  crowsnest targets -s -d example.com -o subdomains
+
+  # Export all subdomains
+  crowsnest targets -s -o all_subdomains`,
+		Run: func(cmd *cobra.Command, args []string) {
+			// Validate that at least one format is specified
+			if !targetsExternal && !targetsInternal && !targetsSubdomains && !targetsEmails {
+				fmt.Println("[!] Error: You must specify at least one output format:")
+				fmt.Println("    --external (-e) for email:password format")
+				fmt.Println("    --internal (-i) for username:password format")
+				fmt.Println("    --emails (-E) for emails only")
+				fmt.Println("    --subdomains (-s) for subdomains")
+				return
+			}
+
+			if debugGlobal {
+				zap.L().Info("targets_debug",
+					zap.String("message", "targets command started"),
+					zap.Bool("external", targetsExternal),
+					zap.Bool("internal", targetsInternal),
+					zap.Bool("subdomains", targetsSubdomains),
+					zap.Bool("emails", targetsEmails),
+					zap.String("domain", targetsDomain),
+					zap.String("output_file", targetsOutputFile),
+				)
+			}
+
+			// Execute the targets export
+			err := executeTargetsExport()
+			if err != nil {
+				fmt.Printf("[!] Error: %v\n", err)
+				return
+			}
+
+			fmt.Printf("[+] Successfully exported targets to: %s\n", targetsOutputFile)
+		},
+	}
+)
+
+// executeTargetsExport performs the main logic for exporting targets
+func executeTargetsExport() error {
+	var outputLines []string
+
+	// Export external credentials (email:password)
+	if targetsExternal {
+		if debugGlobal {
+			fmt.Println("[*] Exporting external credentials (email:password)...")
+		}
+
+		externalCreds, err := getExternalCredentials()
+		if err != nil {
+			return fmt.Errorf("failed to get external credentials: %v", err)
+		}
+
+		for _, cred := range externalCreds {
+			if cred.Email != "" && cred.Password != "" {
+				outputLines = append(outputLines, fmt.Sprintf("%s:%s", cred.Email, cred.Password))
+			}
+		}
+
+		if debugGlobal {
+			fmt.Printf("[*] Found %d external credentials\n", len(externalCreds))
+		}
+	}
+
+	// Export internal credentials (username:password)
+	if targetsInternal {
+		if debugGlobal {
+			fmt.Println("[*] Exporting internal credentials (username:password)...")
+		}
+
+		internalCreds, err := getInternalCredentials()
+		if err != nil {
+			return fmt.Errorf("failed to get internal credentials: %v", err)
+		}
+
+		for _, cred := range internalCreds {
+			if cred.Username != "" && cred.Password != "" {
+				outputLines = append(outputLines, fmt.Sprintf("%s:%s", cred.Username, cred.Password))
+			}
+		}
+
+		if debugGlobal {
+			fmt.Printf("[*] Found %d internal credentials\n", len(internalCreds))
+		}
+	}
+
+	// Export emails only
+	if targetsEmails {
+		if debugGlobal {
+			fmt.Println("[*] Exporting emails only...")
+		}
+
+		emails, err := getEmailsOnly()
+		if err != nil {
+			return fmt.Errorf("failed to get emails: %v", err)
+		}
+
+		for _, email := range emails {
+			if email.Email != "" {
+				outputLines = append(outputLines, email.Email)
+			}
+		}
+
+		if debugGlobal {
+			fmt.Printf("[*] Found %d emails\n", len(emails))
+		}
+	}
+
+	// Export subdomains
+	if targetsSubdomains {
+		if debugGlobal {
+			fmt.Println("[*] Exporting subdomains...")
+		}
+
+		subdomains, err := getSubdomains()
+		if err != nil {
+			return fmt.Errorf("failed to get subdomains: %v", err)
+		}
+
+		for _, subdomain := range subdomains {
+			if subdomain.Subdomain != "" {
+				outputLines = append(outputLines, subdomain.Subdomain)
+			}
+		}
+
+		if debugGlobal {
+			fmt.Printf("[*] Found %d subdomains\n", len(subdomains))
+		}
+	}
+
+	// Write to file
+	if len(outputLines) == 0 {
+		return fmt.Errorf("no data found to export")
+	}
+
+	// Join all lines with newlines and add a single newline at the end
+	content := strings.Join(outputLines, "\n") + "\n"
+
+	err := os.WriteFile(targetsOutputFile, []byte(content), 0644)
+	if err != nil {
+		return fmt.Errorf("failed to write to file: %v", err)
+	}
+
+	if debugGlobal {
+		fmt.Printf("[*] Wrote %d lines to %s\n", len(outputLines), targetsOutputFile)
+	}
+
+	return nil
+}
+
+// getExternalCredentials retrieves credentials for external format (email:password)
+func getExternalCredentials() ([]sqlite.User, error) {
+	db := sqlite.GetDB()
+	var users []sqlite.User
+
+	query := db.Where("email IS NOT NULL AND email != '' AND password IS NOT NULL AND password != ''")
+
+	// Apply domain filter if specified
+	if targetsDomain != "" {
+		query = query.Where("email LIKE ?", "%@"+targetsDomain)
+	}
+
+	err := query.Find(&users).Error
+	if err != nil {
+		zap.L().Error("get_external_credentials",
+			zap.String("message", "failed to query external credentials"),
+			zap.Error(err),
+		)
+		return nil, err
+	}
+
+	return users, nil
+}
+
+// getInternalCredentials retrieves credentials for internal format (username:password)
+func getInternalCredentials() ([]sqlite.User, error) {
+	db := sqlite.GetDB()
+	var users []sqlite.User
+
+	query := db.Where("username IS NOT NULL AND username != '' AND password IS NOT NULL AND password != ''")
+
+	// Apply domain filter if specified (filter usernames that might contain domain info)
+	if targetsDomain != "" {
+		query = query.Where("username LIKE ? OR email LIKE ?", "%"+targetsDomain+"%", "%@"+targetsDomain)
+	}
+
+	err := query.Find(&users).Error
+	if err != nil {
+		zap.L().Error("get_internal_credentials",
+			zap.String("message", "failed to query internal credentials"),
+			zap.Error(err),
+		)
+		return nil, err
+	}
+
+	return users, nil
+}
+
+// getEmailsOnly retrieves emails only (no passwords required)
+func getEmailsOnly() ([]sqlite.User, error) {
+	db := sqlite.GetDB()
+	var users []sqlite.User
+
+	query := db.Where("email IS NOT NULL AND email != ''")
+
+	// Apply domain filter if specified
+	if targetsDomain != "" {
+		query = query.Where("email LIKE ?", "%@"+targetsDomain)
+	}
+
+	err := query.Find(&users).Error
+	if err != nil {
+		zap.L().Error("get_emails_only",
+			zap.String("message", "failed to query emails"),
+			zap.Error(err),
+		)
+		return nil, err
+	}
+
+	return users, nil
+}
+
+// getSubdomains retrieves subdomains from the database
+func getSubdomains() ([]sqlite.Subdomain, error) {
+	db := sqlite.GetDB()
+	var subdomains []sqlite.Subdomain
+
+	query := db.Where("subdomain IS NOT NULL AND subdomain != ''")
+
+	// Apply domain filter if specified
+	if targetsDomain != "" {
+		query = query.Where("domain = ? OR subdomain LIKE ?", targetsDomain, "%."+targetsDomain)
+	}
+
+	err := query.Find(&subdomains).Error
+	if err != nil {
+		zap.L().Error("get_subdomains",
+			zap.String("message", "failed to query subdomains"),
+			zap.Error(err),
+		)
+		return nil, err
+	}
+
+	return subdomains, nil
+}

cmd/whois.go

@@ -1,14 +1,40 @@
 package cmd
 
 import (
-	"dehasher/internal/sqlite"
-	"dehasher/internal/whois"
 	"fmt"
+	"os"
+	"strings"
+	"time"
+
 	"github.com/spf13/cobra"
 	"go.uber.org/zap"
-	"strings"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/debug"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/export"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/files"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/pretty"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/sqlite"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/whois"
 )
 
+func init() {
+	// Add whois subcommand to root command
+	rootCmd.AddCommand(whoisCmd)
+
+	// Add flags specific to whois command
+	whoisCmd.Flags().StringVarP(&whoisDomain, "domain", "d", "", "Domain for WHOIS lookup, history search, and subdomain scan")
+	whoisCmd.Flags().StringVarP(&whoisIPAddress, "ip", "i", "", "IP address for reverse IP lookup")
+	whoisCmd.Flags().StringVarP(&whoisMXAddress, "mx", "m", "", "MX hostname for reverse MX lookup")
+	whoisCmd.Flags().StringVarP(&whoisNSAddress, "ns", "n", "", "NS hostname for reverse NS lookup")
+	whoisCmd.Flags().StringVarP(&whoisInclude, "include", "I", "", "Up to 4 Terms to include in reverse WHOIS search (comma-separated)")
+	whoisCmd.Flags().StringVarP(&whoisExclude, "exclude", "E", "", "Up to 4 Terms to exclude in reverse WHOIS search (comma-separated)")
+	whoisCmd.Flags().StringVarP(&whoisReverseType, "type", "t", "current", "Type of reverse WHOIS search ([default] current or historic)")
+	whoisCmd.Flags().StringVarP(&whoisOutputFormat, "format", "f", "text", "Output format (text, json)")
+	whoisCmd.Flags().StringVarP(&whoisOutputFile, "output", "o", "whois", "File to output results to including extension")
+	whoisCmd.Flags().BoolVarP(&whoisShowCredits, "credits", "c", false, "Show remaining WHOIS credits")
+	whoisCmd.Flags().BoolVarP(&whoisHistory, "history", "H", false, "Perform WHOIS history search [25 Credits]")
+	whoisCmd.Flags().BoolVarP(&whoisSubdomainScan, "subdomains", "s", false, "Perform WHOIS subdomain scan")
+}
+
 var (
 	// WHOIS command flags
 	whoisDomain        string
@@ -19,6 +45,7 @@ var (
 	whoisExclude       string
 	whoisReverseType   string
 	whoisOutputFormat  string
+	whoisOutputFile    string
 	whoisShowCredits   bool
 	whoisHistory       bool
 	whoisSubdomainScan bool
@@ -29,28 +56,52 @@ var (
 		Short: "Dehashed WHOIS lookups and reverse WHOIS searches",
 		Long:  `Perform WHOIS lookups, history searches, reverse WHOIS searches, IP lookups, MX lookups, NS lookups, and subdomain scans.`,
 		Run: func(cmd *cobra.Command, args []string) {
-			key := getStoredApiKey()
+			key := getDehashedApiKey()
 
 			// Validate credentials
 			if key == "" {
-				fmt.Println("API key is required. Set the key with the \"set-key\" command. [dehasher set-key <api_key>]")
+				fmt.Println("API key is required. Set the key with the \"set dehashed\" command. [crowsnest set dehashed <api_key>]")
 				return
 			}
 
+			if debugGlobal {
+				debug.PrintInfo("debug mode enabled")
+				zap.L().Info("whois_debug",
+					zap.String("message", "debug mode enabled"),
+				)
+			}
+
+			if whoisOutputFile == "" {
+				if debugGlobal {
+					debug.PrintInfo("output file not specified, using default")
+				}
+				whoisOutputFile = "whois_" + time.Now().Format("05_04_05")
+			}
+
+			if whoisOutputFormat == "" {
+				if debugGlobal {
+					debug.PrintInfo("output format not specified, using default")
+				}
+				whoisOutputFormat = "json"
+			}
+
+			fType := files.GetFileType(whoisOutputFormat)
+			if fType == files.UNKNOWN {
+				fmt.Println("[!] Error: Invalid output format. Must be 'json', 'xml', 'yaml', or 'txt'.")
+				return
+			}
+			if debugGlobal {
+				debug.PrintInfo("using output format: " + whoisOutputFormat)
+			}
+
+			w := whois.NewWhoIs(key, debugGlobal)
+
 			// Show credits if requested
 			if whoisShowCredits {
-				fmt.Println("[*] Getting WHOIS credits...")
-				credits, err := whois.GetWHOISCredits(key)
-				if err != nil {
-					zap.L().Error("get_whois_credits",
-						zap.String("message", "failed to get whois credits"),
-						zap.Error(err),
-					)
-					fmt.Printf("Error getting WHOIS credits: %v\n", err)
-					return
+				fmt.Println("[*] Getting WHOIS balance...")
+				if whoisShowCredits {
+					checkBalance(w)
 				}
-				fmt.Printf("WHOIS Credits: %d\n", credits.WhoisCredits)
-				return
 			}
 
 			// Check if domain is provided for history and subdomain scan
@@ -59,105 +110,220 @@ var (
 					fmt.Println("Domain is required for history and subdomain scan.")
 					return
 				}
+				if whoisShowCredits {
+					checkBalance(w)
+				}
 			}
 
 			// Determine which operation to perform based on flags
 			if whoisDomain != "" {
 				fmt.Println("[*] Performing WHOIS lookup...")
-				// Domain lookup
-				result, err := whois.WhoisSearch(whoisDomain, key)
-				if err != nil {
-					zap.L().Error("whois_search",
-						zap.String("message", "failed to perform whois search"),
-						zap.Error(err),
-					)
-					fmt.Printf("Error performing WHOIS lookup: %v\n", err)
-					return
-				}
 
-				// Fix the output format to use proper formatting
-				fmt.Printf("WHOIS Lookup Result:\n%+v\n", result.Data.WhoisRecord)
+				if !whoisHistory && !whoisSubdomainScan {
+					// Domain lookup
+					result, err := w.WhoisSearch(whoisDomain)
+					if err != nil {
+						if debugGlobal {
+							debug.PrintInfo("failed to perform whois search")
+							debug.PrintError(err)
+						}
+						zap.L().Error("whois_search",
+							zap.String("message", "failed to perform whois search"),
+							zap.Error(err),
+						)
+						fmt.Printf("Error performing WHOIS lookup: %v\n", err)
+						return
+					}
 
-				// Store the record
-				err = sqlite.StoreWhoisRecord(result.Data.WhoisRecord)
-				if err != nil {
-					zap.L().Error("store_whois_record",
-						zap.String("message", "failed to store whois record"),
-						zap.Error(err),
-					)
-					fmt.Printf("Error storing WHOIS record: %v\n", err)
-					// Continue execution even if storage fails
+					if whoisShowCredits {
+						checkBalance(w)
+					}
+
+					// Fix the output format to use proper formatting
+					fmt.Println("WHOIS Lookup Result:")
+
+					// Store the record
+					err = sqlite.StoreWhoisRecord(result)
+					if err != nil {
+						if debugGlobal {
+							debug.PrintInfo("failed to store whois record")
+							debug.PrintError(err)
+						}
+						zap.L().Error("store_whois_record",
+							zap.String("message", "failed to store whois record"),
+							zap.Error(err),
+						)
+						fmt.Printf("Error storing WHOIS record: %v\n", err)
+						// Continue execution even if storage fails
+					}
+
+					// Pretty Print WhoIs Record
+					pretty.WhoIsTree(whoisDomain, result)
+
+					// Write WhoIs Record to file
+					if len(result.DomainName) != 0 {
+						fmt.Printf("[*] Writing WHOIS record to file: %s%s\n", whoisOutputFile, fType.Extension())
+						err = export.WriteWhoIsRecordToFile(result, whoisOutputFile, fType)
+					} else {
+						if debugGlobal {
+							debug.PrintInfo("no whois record to write to file")
+						}
+						zap.L().Info("write_whois_record",
+							zap.String("message", "no whois record to write to file"),
+						)
+					}
 				}
 
 				if whoisHistory {
+					filename := whoisOutputFile + "_history"
 					fmt.Println("[*] Performing WHOIS history search...")
 					// Perform history search
-					history, err := whois.WhoisHistory(whoisDomain, key)
+					historyRecords, err := w.WhoisHistory(whoisDomain)
 					if err != nil {
+						if debugGlobal {
+							debug.PrintInfo("failed to perform whois history lookup")
+							debug.PrintError(err)
+						}
 						zap.L().Error("whois_history",
 							zap.String("message", "failed to perform whois history lookup"),
 							zap.Error(err),
 						)
-						fmt.Printf("Error performing WHOIS history lookup: %v\n", err)
+						fmt.Printf("[!] Error performing WHOIS history lookup: %v\n", err)
 					} else {
-						fmt.Println("\nWHOIS History:")
-						fmt.Println(history)
-					}
+						if whoisShowCredits {
+							checkBalance(w)
+						}
 
-					err = sqlite.StoreHistoryRecord(history.Data.Records)
-					if err != nil {
-						zap.L().Error("store_history_record",
-							zap.String("message", "failed to store history record"),
-							zap.Error(err),
-						)
-						fmt.Printf("Error storing WHOIS history record: %v\n", err)
+						// Write history records to file if any
+						if len(historyRecords) > 0 {
+							fmt.Printf("[*] Records Found: %d\n", len(historyRecords))
+							fmt.Printf("[*] WHOIS History being written to file: %s%s\n", whoisOutputFile, fType.Extension())
+							writeErr := export.WriteWhoIsHistoryToFile(historyRecords, filename, fType)
+							if writeErr != nil {
+								if debugGlobal {
+									debug.PrintInfo("failed to write whois history to file")
+									debug.PrintError(writeErr)
+								}
+								zap.L().Error("write_whois_history",
+									zap.String("message", "failed to write whois history to file"),
+									zap.Error(writeErr),
+								)
+								fmt.Printf("[!] Error writing WHOIS history to file: %v\n", writeErr)
+							}
+
+							err = sqlite.StoreWhoisHistoryRecords(historyRecords)
+							if err != nil {
+								if debugGlobal {
+									debug.PrintInfo("failed to store history record")
+									debug.PrintError(err)
+								}
+								zap.L().Error("store_history_record",
+									zap.String("message", "failed to store history record"),
+									zap.Error(err),
+								)
+								fmt.Printf("Error storing WHOIS history record: %v\n", err)
+							}
+
+						} else {
+							if debugGlobal {
+								debug.PrintInfo("no whois history records to write to file")
+							}
+							zap.L().Info("write_whois_history",
+								zap.String("message", "no whois history records to write to file"),
+							)
+						}
 					}
 				}
 
 				// Perform subdomain scan
 				if whoisSubdomainScan {
+					filename := whoisOutputFile + "_subdomains"
 					fmt.Println("[*] Performing WHOIS subdomain scan...")
-					subdomains, err := whois.WhoisSubdomainScan(whoisDomain, key)
+					subdomains, err := w.WhoisSubdomainScan(whoisDomain)
+
+					// Get credits
+					if whoisShowCredits {
+						checkBalance(w)
+					}
+
 					if err != nil {
+						if debugGlobal {
+							debug.PrintInfo("failed to perform subdomain scan")
+							debug.PrintError(err)
+						}
 						zap.L().Error("whois_subdomain_scan",
 							zap.String("message", "failed to perform subdomain scan"),
 							zap.Error(err),
 						)
 						fmt.Printf("Error performing subdomain scan: %v\n", err)
 					} else {
-						fmt.Println("\nSubdomain Scan:")
-						fmt.Println(subdomains)
-					}
+						// Store subdomains in subdomains table
+						var subs []sqlite.Subdomain
+						for _, s := range subdomains {
+							subs = append(subs, sqlite.Subdomain{Domain: whoisDomain, Subdomain: s.Domain})
+						}
 
-					err = sqlite.StoreSubdomainRecord(subdomains.Data.Result.Records)
-					if err != nil {
-						zap.L().Error("store_subdomain_record",
-							zap.String("message", "failed to store subdomain record"),
-							zap.Error(err),
-						)
-						fmt.Printf("Error storing WHOIS subdomain record: %v\n", err)
+						err = sqlite.StoreSubdomains(subs)
+						if err != nil {
+							if debugGlobal {
+								debug.PrintInfo("failed to store subdomain record")
+								debug.PrintError(err)
+							}
+							zap.L().Error("store_subdomain_record",
+								zap.String("message", "failed to store subdomain record"),
+								zap.Error(err),
+							)
+							fmt.Printf("Error storing subdomain record: %v\n", err)
+						}
+
+						// Write the subdomains to file if any
+						if len(subdomains) > 0 {
+							fmt.Printf("[*] Writing subdomains to file: %s%s\n", whoisOutputFile, fType.Extension())
+							err = export.WriteSubdomainsToFile(subdomains, filename, fType)
+							if err != nil {
+								zap.L().Error("write_whois_subdomain",
+									zap.String("message", "failed to write whois subdomain to file"),
+									zap.Error(err),
+								)
+								fmt.Printf("Error writing WHOIS subdomain to file: %v\n", err)
+							}
+
+							var (
+								headers = []string{"Domain", "First Seen", "Last Seen"}
+								rows    [][]string
+							)
+
+							// Create the rows
+							for _, r := range subdomains {
+								rows = append(rows, []string{r.Domain, time.Unix(r.FirstSeen, 0).String(), time.Unix(r.LastSeen, 0).String()})
+							}
+
+							// Store the subdomains
+							fmt.Println("Subdomain Scan:")
+							pretty.Table(headers, rows)
+
+						} else {
+							fmt.Println("[!] No subdomains found")
+							zap.L().Info("write_whois_subdomain",
+								zap.String("message", "no whois subdomain records to write to file"),
+								zap.String("domain", whoisDomain),
+							)
+						}
 					}
 				}
 
-				// Get credits
-				credits, err := whois.GetWHOISCredits(key)
-				if err != nil {
-					zap.L().Error("get_whois_credits",
-						zap.String("message", "failed to get whois credits"),
-						zap.Error(err),
-					)
-					fmt.Printf("Error getting WHOIS credits: %v\n", err)
-					return
-				}
-				fmt.Printf("\nWHOIS Credits Remaining: %d\n", credits.WhoisCredits)
 				return
 			}
 
 			if whoisIPAddress != "" {
 				fmt.Println("[*] Performing reverse IP lookup...")
 				// IP lookup
-				result, err := whois.WhoisIP(whoisIPAddress, key)
+				result, err := w.WhoisIP(whoisIPAddress)
 				if err != nil {
+					if debugGlobal {
+						debug.PrintInfo("failed to perform ip lookup")
+						debug.PrintError(err)
+					}
 					zap.L().Error("whois_ip",
 						zap.String("message", "failed to perform ip lookup"),
 						zap.Error(err),
@@ -165,28 +331,61 @@ var (
 					fmt.Printf("Error performing IP lookup: %v\n", err)
 					return
 				}
-				fmt.Println("IP Lookup Result:")
-				fmt.Println(string(result))
 
 				// Get credits
-				credits, err := whois.GetWHOISCredits(key)
-				if err != nil {
-					zap.L().Error("get_whois_credits",
-						zap.String("message", "failed to get whois credits"),
-						zap.Error(err),
+				if whoisShowCredits {
+					checkBalance(w)
+				}
+
+				if len(result) == 0 {
+					fmt.Println("[!] No results found")
+					zap.L().Info("whois_ip",
+						zap.String("message", "no results found"),
+						zap.String("ip", whoisIPAddress),
 					)
-					fmt.Printf("Error getting WHOIS credits: %v\n", err)
 					return
 				}
-				fmt.Printf("\nWHOIS Credits Remaining: %d\n", credits.WhoisCredits)
+
+				// Write the results to file
+				fmt.Printf("[*] Writing IP lookup results to file: %s%s\n", whoisOutputFile, fType.Extension())
+				err = export.WriteIPLookupToFile(result, whoisOutputFile, fType)
+				if err != nil {
+					if debugGlobal {
+						debug.PrintInfo("failed to write ip lookup to file")
+						debug.PrintError(err)
+					}
+					zap.L().Error("write_ip_lookup",
+						zap.String("message", "failed to write ip lookup to file"),
+						zap.Error(err),
+					)
+					fmt.Printf("Error writing IP lookup to file: %v\n", err)
+				}
+
+				// Pretty Print the JSON
+				var (
+					headers = []string{"Name", "Search Term", "First Seen", "Last Visit", "Type"}
+					rows    [][]string
+				)
+				fmt.Println("IP Lookup Result:")
+
+				for _, r := range result {
+					rows = append(rows, []string{r.Name, r.SearchTerm, time.Unix(r.FirstSeen, 0).String(), time.Unix(r.LastVisit, 0).String(), r.Type})
+				}
+
+				pretty.Table(headers, rows)
+
 				return
 			}
 
 			if whoisMXAddress != "" {
 				fmt.Println("[*] Performing reverse MX lookup...")
 				// MX lookup
-				result, err := whois.WhoisMX(whoisMXAddress, key)
+				result, err := w.WhoisMX(whoisMXAddress)
 				if err != nil {
+					if debugGlobal {
+						debug.PrintInfo("failed to perform mx lookup")
+						debug.PrintError(err)
+					}
 					zap.L().Error("whois_mx",
 						zap.String("message", "failed to perform mx lookup"),
 						zap.Error(err),
@@ -195,29 +394,60 @@ var (
 					return
 				}
 
-				// todo unmarshal mx lookup
-				fmt.Println("MX Lookup Result:")
-				fmt.Println(result)
-
 				// Get credits
-				credits, err := whois.GetWHOISCredits(key)
-				if err != nil {
-					zap.L().Error("get_whois_credits",
-						zap.String("message", "failed to get whois credits"),
-						zap.Error(err),
+				if whoisShowCredits {
+					checkBalance(w)
+				}
+
+				if len(result) == 0 {
+					fmt.Println("[!] No results found")
+					zap.L().Info("whois_mx",
+						zap.String("message", "no results found"),
+						zap.String("mx", whoisMXAddress),
 					)
-					fmt.Printf("Error getting WHOIS credits: %v\n", err)
 					return
 				}
-				fmt.Printf("\nWHOIS Credits Remaining: %d\n", credits.WhoisCredits)
+
+				// Write the results to file
+				fmt.Printf("[*] Writing MX lookup results to file: %s%s\n", whoisOutputFile, fType.Extension())
+				err = export.WriteIPLookupToFile(result, whoisOutputFile, fType)
+				if err != nil {
+					if debugGlobal {
+						debug.PrintInfo("failed to write mx lookup to file")
+						debug.PrintError(err)
+					}
+					zap.L().Error("write_mx_lookup",
+						zap.String("message", "failed to write mx lookup to file"),
+						zap.Error(err),
+					)
+					fmt.Printf("Error writing MX lookup to file: %v\n", err)
+				}
+
+				// Pretty Print the JSON
+				var (
+					headers = []string{"Name", "Search Term", "First Seen", "Last Visit", "Type"}
+					rows    [][]string
+				)
+				fmt.Println("MX Lookup Result:")
+
+				for _, r := range result {
+					rows = append(rows, []string{r.Name, r.SearchTerm, time.Unix(r.FirstSeen, 0).String(), time.Unix(r.LastVisit, 0).String(), r.Type})
+				}
+
+				pretty.Table(headers, rows)
+
 				return
 			}
 
 			if whoisNSAddress != "" {
 				fmt.Println("[*] Performing reverse NS lookup...")
 				// NS lookup
-				result, err := whois.WhoisNS(whoisNSAddress, key)
+				result, err := w.WhoisNS(whoisNSAddress)
 				if err != nil {
+					if debugGlobal {
+						debug.PrintInfo("failed to perform ns lookup")
+						debug.PrintError(err)
+					}
 					zap.L().Error("whois_ns",
 						zap.String("message", "failed to perform ns lookup"),
 						zap.Error(err),
@@ -225,47 +455,138 @@ var (
 					fmt.Printf("Error performing NS lookup: %v\n", err)
 					return
 				}
-				fmt.Println("NS Lookup Result:")
-				fmt.Println(result)
 
 				// Get credits
-				credits, err := whois.GetWHOISCredits(key)
-				if err != nil {
-					zap.L().Error("get_whois_credits",
-						zap.String("message", "failed to get whois credits"),
-						zap.Error(err),
+				if whoisShowCredits {
+					checkBalance(w)
+				}
+
+				if len(result) == 0 {
+					fmt.Println("[!] No results found")
+					zap.L().Info("whois_ns",
+						zap.String("message", "no results found"),
+						zap.String("ns", whoisNSAddress),
 					)
-					fmt.Printf("Error getting WHOIS credits: %v\n", err)
 					return
 				}
-				fmt.Printf("\nWHOIS Credits Remaining: %d\n", credits.WhoisCredits)
+
+				// Write the results to file
+				fmt.Printf("[*] Writing NS lookup results to file: %s%s\n", whoisOutputFile, fType.Extension())
+				err = export.WriteIPLookupToFile(result, whoisOutputFile, fType)
+				if err != nil {
+					if debugGlobal {
+						debug.PrintInfo("failed to write ns lookup to file")
+						debug.PrintError(err)
+					}
+					zap.L().Error("write_ns_lookup",
+						zap.String("message", "failed to write ns lookup to file"),
+						zap.Error(err),
+					)
+					fmt.Printf("Error writing NS lookup to file: %v\n", err)
+				}
+
+				// Pretty Print the JSON
+				var (
+					headers = []string{"Name", "Search Term", "First Seen", "Last Visit", "Type"}
+					rows    [][]string
+				)
+				fmt.Println("NS Lookup Result:")
+
+				for _, r := range result {
+					rows = append(rows, []string{r.Name, r.SearchTerm, time.Unix(r.FirstSeen, 0).String(), time.Unix(r.LastVisit, 0).String(), r.Type})
+				}
+
+				pretty.Table(headers, rows)
+
 				return
 			}
 
 			if whoisInclude != "" || whoisExclude != "" {
+				if debugGlobal {
+					debug.PrintInfo("performing reverse whois")
+					debug.PrintInfo("include: " + whoisInclude)
+					debug.PrintInfo("exclude: " + whoisExclude)
+					debug.PrintInfo("reverse type: " + whoisReverseType)
+				}
 				// Reverse WHOIS
 				includeTerms := []string{}
 				if whoisInclude != "" {
 					includeTerms = strings.Split(whoisInclude, ",")
+					if len(includeTerms) > 4 {
+						fmt.Println("[!] Error: Maximum of 4 include terms allowed.")
+						return
+					}
 				}
 
 				excludeTerms := []string{}
 				if whoisExclude != "" {
 					excludeTerms = strings.Split(whoisExclude, ",")
+					if len(excludeTerms) > 4 {
+						fmt.Println("[!] Error: Maximum of 4 exclude terms allowed.")
+						return
+					}
 				}
 
-				if whoisReverseType == "" {
-					whoisReverseType = "registrant"
+				toLower := strings.ToLower(whoisReverseType)
+				if toLower != "current" && toLower != "historic" {
+					fmt.Println("[!] Error: Invalid reverse type. Must be 'current' or 'historic'.")
+					return
 				}
 
 				fmt.Println("[*] Performing reverse WHOIS lookup...")
-				result, err := whois.ReverseWHOIS(includeTerms, excludeTerms, whoisReverseType, key)
+				result, err := w.ReverseWHOIS(includeTerms, excludeTerms, whoisReverseType)
 				if err != nil {
+					if debugGlobal {
+						debug.PrintInfo("failed to perform reverse whois")
+						debug.PrintError(err)
+					}
+					zap.L().Error("reverse_whois",
+						zap.String("message", "failed to perform reverse whois"),
+						zap.Error(err),
+					)
 					fmt.Printf("Error performing reverse WHOIS: %v\n", err)
 					return
 				}
-				fmt.Println("Reverse WHOIS Result:")
-				fmt.Println(result)
+
+				// Write to file
+				if len(result.DomainsList) > 0 {
+					fmt.Printf("[*] Writing reverse WHOIS results to file: %s%s\n", whoisOutputFile, fType.Extension())
+					err = export.WriteIStringToFile(result, whoisOutputFile, fType)
+					if err != nil {
+						if debugGlobal {
+							debug.PrintInfo("failed to write reverse whois to file")
+							debug.PrintError(err)
+						}
+						zap.L().Error("write_reverse_whois",
+							zap.String("message", "failed to write reverse whois to file"),
+							zap.Error(err),
+						)
+						fmt.Printf("Error writing reverse WHOIS to file: %v\n", err)
+					}
+
+					fmt.Println("Reverse WHOIS Result:")
+					fmt.Printf("Total Domains: %d\n", result.DomainsCount)
+
+					var (
+						headers = []string{"Domain"}
+						rows    [][]string
+					)
+
+					for _, r := range result.DomainsList {
+						rows = append(rows, []string{r})
+					}
+
+					pretty.Table(headers, rows)
+				} else {
+					fmt.Println("[!] No results found")
+					zap.L().Info("reverse_whois",
+						zap.String("message", "no results found"),
+					)
+				}
+
+				if whoisShowCredits {
+					checkBalance(w)
+				}
 				return
 			}
 
@@ -275,20 +596,22 @@ var (
 	}
 )
 
-func init() {
-	// Add whois command to root command
-	rootCmd.AddCommand(whoisCmd)
-
-	// Add flags specific to whois command
-	whoisCmd.Flags().StringVarP(&whoisDomain, "domain", "d", "", "Domain for WHOIS lookup, history search, and subdomain scan")
-	whoisCmd.Flags().StringVarP(&whoisIPAddress, "ip", "i", "", "IP address for reverse IP lookup")
-	whoisCmd.Flags().StringVarP(&whoisMXAddress, "mx", "m", "", "MX address for reverse MX lookup")
-	whoisCmd.Flags().StringVarP(&whoisNSAddress, "ns", "n", "", "NS address for reverse NS lookup")
-	whoisCmd.Flags().StringVarP(&whoisInclude, "include", "I", "", "Terms to include in reverse WHOIS search (comma-separated)")
-	whoisCmd.Flags().StringVarP(&whoisExclude, "exclude", "E", "", "Terms to exclude in reverse WHOIS search (comma-separated)")
-	whoisCmd.Flags().StringVarP(&whoisReverseType, "type", "t", "registrant", "Type of reverse WHOIS search (registrant, email, organization, address, phone)")
-	whoisCmd.Flags().StringVarP(&whoisOutputFormat, "format", "f", "text", "Output format (text, json)")
-	whoisCmd.Flags().BoolVarP(&whoisShowCredits, "credits", "c", false, "Show remaining WHOIS credits")
-	whoisCmd.Flags().BoolVarP(&whoisHistory, "history", "H", false, "Perform WHOIS history search [25 Credits]")
-	whoisCmd.Flags().BoolVarP(&whoisSubdomainScan, "subdomains", "s", false, "Perform WHOIS subdomain scan")
+func checkBalance(w *whois.DehashedWhoIs) {
+	balance, err := w.Balance()
+	if err != nil {
+		if debugGlobal {
+			debug.PrintInfo("failed to get whois balance")
+			debug.PrintError(err)
+		}
+		zap.L().Error("get_whois_credits",
+			zap.String("message", "failed to get whois balance"),
+			zap.Error(err),
+		)
+		fmt.Printf("Error getting WHOIS balance: %v\n", err)
+	}
+	fmt.Println("WHOIS Credits: ", balance)
+	if balance == 0 {
+		fmt.Println("[!] No WHOIS credits remaining.")
+		os.Exit(0)
+	}
 }

crowsnest.go

@@ -1,16 +1,17 @@
 package main
 
 import (
-	"dehasher/cmd"
-	"dehasher/internal/badger"
-	"dehasher/internal/sqlite"
 	"fmt"
+	"os"
+	"path/filepath"
+
 	"github.com/winking324/rzap"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 	"gopkg.in/natefinch/lumberjack.v2"
-	"os"
-	"path/filepath"
+	"hub.krkn.tech/KrakenTech/crowsnest/cmd"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/badger"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/sqlite"
 )
 
 var (
@@ -21,7 +22,7 @@ var (
 )
 
 func init() {
-	basePath = filepath.Join(os.Getenv("HOME"), ".local", "share", "Dehasher")
+	basePath = filepath.Join(os.Getenv("HOME"), ".local", "share", "CrowsNest")
 	logPath = filepath.Join(basePath, "logs")
 	storePath = filepath.Join(basePath, "keystore")
 	// dbPath will be set in main() after badger is initialized
@@ -82,7 +83,7 @@ func main() {
 	useLocalDB := badger.GetUseLocalDB()
 	if useLocalDB {
 		// Use local database in current directory
-		dbPath = "./dehasher.sqlite"
+		dbPath = "./"
 		zap.L().Info("Using local database", zap.String("path", dbPath))
 	} else {
 		// Use default database path

go.mod

@@ -1,4 +1,4 @@
-module dehasher
+module hub.krkn.tech/KrakenTech/crowsnest
 
 go 1.23.0
 
@@ -7,13 +7,13 @@ toolchain go1.24.3
 require (
 	github.com/charmbracelet/lipgloss v1.1.0
 	github.com/dgraph-io/badger/v4 v4.7.0
-	github.com/olekukonko/tablewriter v1.0.5
+	github.com/fatih/color v1.15.0
+	github.com/glebarez/sqlite v1.11.0
 	github.com/spf13/cobra v1.9.1
 	github.com/winking324/rzap v0.1.0
 	go.uber.org/zap v1.20.0
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 	gopkg.in/yaml.v3 v3.0.1
-	gorm.io/driver/sqlite v1.5.7
 	gorm.io/gorm v1.26.1
 )
 
@@ -27,10 +27,11 @@ require (
 	github.com/charmbracelet/x/term v0.2.1 // indirect
 	github.com/dgraph-io/ristretto/v2 v2.2.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
-	github.com/fatih/color v1.15.0 // indirect
+	github.com/glebarez/go-sqlite v1.21.2 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/google/flatbuffers v25.2.10+incompatible // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
@@ -39,10 +40,8 @@ require (
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.16 // indirect
-	github.com/mattn/go-sqlite3 v1.14.22 // indirect
 	github.com/muesli/termenv v0.16.0 // indirect
-	github.com/olekukonko/errors v0.0.0-20250405072817-4e6d85265da6 // indirect
-	github.com/olekukonko/ll v0.0.7 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/spf13/pflag v1.0.6 // indirect
 	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
@@ -56,4 +55,8 @@ require (
 	golang.org/x/sys v0.31.0 // indirect
 	golang.org/x/text v0.23.0 // indirect
 	google.golang.org/protobuf v1.36.6 // indirect
+	modernc.org/libc v1.22.5 // indirect
+	modernc.org/mathutil v1.5.0 // indirect
+	modernc.org/memory v1.5.0 // indirect
+	modernc.org/sqlite v1.23.1 // indirect
 )

go.sum

@@ -34,6 +34,10 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
 github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
+github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9gAXWo=
+github.com/glebarez/go-sqlite v1.21.2/go.mod h1:sfxdZyhQjTM2Wry3gVYWaW072Ri1WMdWJi0k6+3382k=
+github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw=
+github.com/glebarez/sqlite v1.11.0/go.mod h1:h8/o8j5wiAsqSPoWELDUdJXhjAhsVliSn7bWZjOhrgQ=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
 github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
@@ -43,6 +47,10 @@ github.com/google/flatbuffers v25.2.10+incompatible h1:F3vclr7C3HpB1k9mxCGRMXq6F
 github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ=
+github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
@@ -67,20 +75,15 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
 github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
-github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
 github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
-github.com/olekukonko/errors v0.0.0-20250405072817-4e6d85265da6 h1:r3FaAI0NZK3hSmtTDrBVREhKULp8oUeqLT5Eyl2mSPo=
-github.com/olekukonko/errors v0.0.0-20250405072817-4e6d85265da6/go.mod h1:ppzxA5jBKcO1vIpCXQ9ZqgDh8iwODz6OXIGKU8r5m4Y=
-github.com/olekukonko/ll v0.0.7 h1:K66xcUlG2qWRhPoLw/cidmbv4pDDJtZuvJGsR5QTzXo=
-github.com/olekukonko/ll v0.0.7/go.mod h1:En+sEW0JNETl26+K8eZ6/W4UQ7CYSrrgg/EdIYT2H8g=
-github.com/olekukonko/tablewriter v1.0.5 h1:8+uKJXxYcl29TcpfQdd0vL+l6Kul7Sk7sWolfgErDv0=
-github.com/olekukonko/tablewriter v1.0.5/go.mod h1:Z22i2ywMkT9sw64nuWAUaH62kb+umiwucGaQNbFh8Bg=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
@@ -166,7 +169,13 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gorm.io/driver/sqlite v1.5.7 h1:8NvsrhP0ifM7LX9G4zPB97NwovUakUxc+2V2uuf3Z1I=
-gorm.io/driver/sqlite v1.5.7/go.mod h1:U+J8craQU6Fzkcvu8oLeAQmi50TkwPEhHDEjQZXDah4=
 gorm.io/gorm v1.26.1 h1:ghB2gUI9FkS46luZtn6DLZ0f6ooBJ5IbVej2ENFDjRw=
 gorm.io/gorm v1.26.1/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE=
+modernc.org/libc v1.22.5 h1:91BNch/e5B0uPbJFgqbxXuOnxBQjlS//icfQEGmvyjE=
+modernc.org/libc v1.22.5/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY=
+modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ=
+modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
+modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds=
+modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU=
+modernc.org/sqlite v1.23.1 h1:nrSBg4aRQQwq59JpvGEQ15tNxoO5pX/kUjcRNwSAGQM=
+modernc.org/sqlite v1.23.1/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk=

internal/badger/badger.go

@@ -3,15 +3,19 @@ package badger
 import (
 	"crypto/sha256"
 	"errors"
-	"github.com/dgraph-io/badger/v4"
-	"go.uber.org/zap"
+	"fmt"
 	"log"
 	"os"
+	"os/exec"
 	"os/user"
 	"path/filepath"
 	"runtime"
 	"strings"
 	"sync"
+	"time"
+
+	"github.com/dgraph-io/badger/v4"
+	"go.uber.org/zap"
 )
 
 var (
@@ -21,13 +25,35 @@ var (
 	once          sync.Once
 )
 
-func GetHardwareEntropy() []byte {
+const fingerprintSalt = "CrowsNest-static-salt-value"
+
+func GetHardwareEntropy() ([]byte, error) {
+	source, machineID, err := getMachineID()
+	if err != nil {
+		return nil, err
+	}
+
+	fingerprint := strings.Join([]string{
+		"v2",
+		runtime.GOOS,
+		source,
+		machineID,
+		fingerprintSalt,
+	}, ":")
+
+	return hashFingerprint(fingerprint), nil
+}
+
+func GetLegacyHardwareEntropy() []byte {
 	// Get hostname
 	hostname, err := os.Hostname()
 	if err != nil {
 		hostname = "unknown-host"
 		log.Printf("Error getting hostname: %v", err)
 	}
+	if legacyHostname := strings.TrimSpace(os.Getenv("CROWSNEST_LEGACY_HOSTNAME")); legacyHostname != "" {
+		hostname = legacyHostname
+	}
 
 	// Get username
 	currentUser, err := user.Current()
@@ -35,9 +61,15 @@ func GetHardwareEntropy() []byte {
 	if err == nil && currentUser != nil {
 		username = currentUser.Username
 	}
+	if legacyUsername := strings.TrimSpace(os.Getenv("CROWSNEST_LEGACY_USERNAME")); legacyUsername != "" {
+		username = legacyUsername
+	}
 
 	// Get OS and architecture info
 	osInfo := runtime.GOOS + "-" + runtime.GOARCH
+	if legacyOSInfo := strings.TrimSpace(os.Getenv("CROWSNEST_LEGACY_OSINFO")); legacyOSInfo != "" {
+		osInfo = legacyOSInfo
+	}
 
 	// Combine all information for a unique but consistent fingerprint
 	fingerprint := strings.Join([]string{
@@ -45,14 +77,93 @@ func GetHardwareEntropy() []byte {
 		username,
 		osInfo,
 		// You could add a static salt here for additional security
-		"Dehasher-static-salt-value",
+		fingerprintSalt,
 	}, ":")
 
 	// Hash the fingerprint to get a 32-byte key
+	return hashFingerprint(fingerprint)
+}
+
+func hashFingerprint(fingerprint string) []byte {
 	sum := sha256.Sum256([]byte(fingerprint))
 	return sum[:]
 }
 
+func getMachineID() (string, string, error) {
+	switch runtime.GOOS {
+	case "darwin":
+		return getDarwinMachineID()
+	case "linux":
+		return getLinuxMachineID()
+	case "windows":
+		return getWindowsMachineID()
+	default:
+		return "", "", fmt.Errorf("stable machine id is not implemented for %s", runtime.GOOS)
+	}
+}
+
+func getDarwinMachineID() (string, string, error) {
+	out, err := exec.Command("ioreg", "-rd1", "-c", "IOPlatformExpertDevice").Output()
+	if err != nil {
+		return "", "", fmt.Errorf("run ioreg: %w", err)
+	}
+
+	for _, line := range strings.Split(string(out), "\n") {
+		if !strings.Contains(line, "IOPlatformUUID") {
+			continue
+		}
+		if id := normalizeMachineID(lastQuotedValue(line)); id != "" {
+			return "darwin-ioplatformuuid", id, nil
+		}
+	}
+
+	return "", "", errors.New("IOPlatformUUID not found")
+}
+
+func getLinuxMachineID() (string, string, error) {
+	for _, path := range []string{"/etc/machine-id", "/var/lib/dbus/machine-id"} {
+		out, err := os.ReadFile(path)
+		if err != nil {
+			continue
+		}
+		if id := normalizeMachineID(string(out)); id != "" {
+			return "linux-machine-id", id, nil
+		}
+	}
+
+	return "", "", errors.New("machine-id not found")
+}
+
+func getWindowsMachineID() (string, string, error) {
+	out, err := exec.Command("reg", "query", `HKLM\SOFTWARE\Microsoft\Cryptography`, "/v", "MachineGuid").Output()
+	if err != nil {
+		return "", "", fmt.Errorf("query MachineGuid: %w", err)
+	}
+
+	for _, line := range strings.Split(string(out), "\n") {
+		fields := strings.Fields(line)
+		if len(fields) >= 3 && strings.EqualFold(fields[0], "MachineGuid") {
+			if id := normalizeMachineID(fields[len(fields)-1]); id != "" {
+				return "windows-machineguid", id, nil
+			}
+		}
+	}
+
+	return "", "", errors.New("MachineGuid not found")
+}
+
+func lastQuotedValue(line string) string {
+	values := strings.Split(line, "\"")
+	if len(values) < 4 {
+		return ""
+	}
+	return values[len(values)-2]
+}
+
+func normalizeMachineID(id string) string {
+	return strings.ToLower(strings.TrimSpace(id))
+}
+
 func Start(dirPath string) *badger.DB {
 	var err error
 
@@ -65,7 +176,7 @@ func Start(dirPath string) *badger.DB {
 		}
 		rootDir = dirPath
 
-		encryptionKey = GetHardwareEntropy()
+		encryptionKey, err = GetHardwareEntropy()
 		if err != nil {
 			zap.L().Fatal("get_encryption_key",
 				zap.String("message", "failed to get encryption key"),
@@ -74,22 +185,134 @@ func Start(dirPath string) *badger.DB {
 		}
 
 		badgerDB := filepath.Join(rootDir, "badger.db")
-		opts := badger.DefaultOptions(badgerDB).
-			WithEncryptionKey(encryptionKey).
-			WithIndexCacheSize(10 << 20). // 10MB
-			WithLoggingLevel(badger.ERROR)
-		db, err = badger.Open(opts)
+		db, err = openBadger(badgerDB, encryptionKey)
 		if err != nil {
-			zap.L().Fatal("new_badger_db",
-				zap.String("message", "failed to open badger database"),
+			zap.L().Warn("open_badger_db",
+				zap.String("message", "failed to open badger database with stable machine key; trying legacy key"),
 				zap.Error(err),
 			)
+			db, err = openBadgerWithLegacyMigration(badgerDB, encryptionKey)
+			if err != nil {
+				zap.L().Fatal("new_badger_db",
+					zap.String("message", "failed to open badger database"),
+					zap.Error(err),
+				)
+			}
 		}
 	})
 
 	return db
 }
 
+func openBadger(dbPath string, key []byte) (*badger.DB, error) {
+	opts := badger.DefaultOptions(dbPath).
+		WithEncryptionKey(key).
+		WithIndexCacheSize(10 << 20). // 10MB
+		WithLoggingLevel(badger.ERROR)
+	return badger.Open(opts)
+}
+
+func openBadgerWithLegacyMigration(dbPath string, stableKey []byte) (*badger.DB, error) {
+	legacyKey := GetLegacyHardwareEntropy()
+	legacyDB, err := openBadger(dbPath, legacyKey)
+	if err != nil {
+		return nil, fmt.Errorf("stable key failed and legacy key failed: %w", err)
+	}
+
+	migratedDB, err := migrateBadgerEncryption(dbPath, legacyDB, stableKey)
+	if err != nil {
+		if closeErr := legacyDB.Close(); closeErr != nil {
+			zap.L().Error("close_legacy_badger_db", zap.Error(closeErr))
+		}
+		return nil, err
+	}
+
+	return migratedDB, nil
+}
+
+func migrateBadgerEncryption(dbPath string, legacyDB *badger.DB, stableKey []byte) (*badger.DB, error) {
+	parentDir := filepath.Dir(dbPath)
+	timestamp := time.Now().Format("20060102-150405")
+	migrationPath := filepath.Join(parentDir, fmt.Sprintf(".%s.migrating-%s", filepath.Base(dbPath), timestamp))
+	backupPath := filepath.Join(parentDir, fmt.Sprintf("%s.legacy-backup-%s", filepath.Base(dbPath), timestamp))
+
+	newDB, err := openBadger(migrationPath, stableKey)
+	if err != nil {
+		return nil, fmt.Errorf("open migration badger db: %w", err)
+	}
+
+	if err := copyBadgerData(legacyDB, newDB); err != nil {
+		_ = newDB.Close()
+		_ = os.RemoveAll(migrationPath)
+		return nil, fmt.Errorf("copy legacy badger data: %w", err)
+	}
+
+	if err := legacyDB.Close(); err != nil {
+		_ = newDB.Close()
+		_ = os.RemoveAll(migrationPath)
+		return nil, fmt.Errorf("close legacy badger db: %w", err)
+	}
+
+	if err := newDB.Close(); err != nil {
+		_ = os.RemoveAll(migrationPath)
+		return nil, fmt.Errorf("close migration badger db: %w", err)
+	}
+
+	if err := os.Rename(dbPath, backupPath); err != nil {
+		_ = os.RemoveAll(migrationPath)
+		return nil, fmt.Errorf("backup legacy badger db: %w", err)
+	}
+
+	if err := os.Rename(migrationPath, dbPath); err != nil {
+		if restoreErr := os.Rename(backupPath, dbPath); restoreErr != nil {
+			return nil, fmt.Errorf("promote migrated badger db: %w; restore legacy backup: %v", err, restoreErr)
+		}
+		return nil, fmt.Errorf("promote migrated badger db: %w", err)
+	}
+
+	db, err := openBadger(dbPath, stableKey)
+	if err != nil {
+		return nil, fmt.Errorf("open migrated badger db: %w", err)
+	}
+
+	zap.L().Info("migrated_badger_encryption",
+		zap.String("backup", backupPath),
+		zap.String("path", dbPath),
+	)
+	return db, nil
+}
+
+func copyBadgerData(src *badger.DB, dst *badger.DB) error {
+	return src.View(func(srcTxn *badger.Txn) error {
+		opts := badger.DefaultIteratorOptions
+		opts.PrefetchValues = true
+		iter := srcTxn.NewIterator(opts)
+		defer iter.Close()
+
+		return dst.Update(func(dstTxn *badger.Txn) error {
+			for iter.Rewind(); iter.Valid(); iter.Next() {
+				item := iter.Item()
+				if item.IsDeletedOrExpired() {
+					continue
+				}
+
+				key := item.KeyCopy(nil)
+				value, err := item.ValueCopy(nil)
+				if err != nil {
+					return err
+				}
+
+				entry := badger.NewEntry(key, value).WithMeta(item.UserMeta())
+				entry.ExpiresAt = item.ExpiresAt()
+				if err := dstTxn.SetEntry(entry); err != nil {
+					return err
+				}
+			}
+			return nil
+		})
+	})
+}
+
 func Close() {
 	err := db.Close()
 	if err != nil {
@@ -100,7 +323,7 @@ func Close() {
 	}
 }
 
-func GetKey() string {
+func GetDehashedKey() string {
 	var apiKey string
 
 	err := db.View(func(txn *badger.Txn) error {
@@ -124,6 +347,29 @@ func GetKey() string {
 	return apiKey
 }
 
+func GetHunterKey() string {
+	var apiKey string
+
+	err := db.View(func(txn *badger.Txn) error {
+		item, err := txn.Get([]byte("cfg:hunter_api_key"))
+		if err != nil {
+			return err // could be ErrKeyNotFound
+		}
+		return item.Value(func(val []byte) error {
+			apiKey = string(val)
+			return nil
+		})
+	})
+
+	if err != nil {
+		zap.L().Error("get_hunter_api_key",
+			zap.String("message", "failed to get hunter_api_key"),
+			zap.Error(err),
+		)
+	}
+	return apiKey
+}
+
 func GetUseLocalDB() bool {
 	var useLocal bool
 
@@ -162,13 +408,26 @@ func GetUseLocalDB() bool {
 	return useLocal
 }
 
-func StoreKey(apiKey string) error {
+func StoreDehashedKey(apiKey string) error {
 	err := db.Update(func(txn *badger.Txn) error {
 		return txn.Set([]byte("cfg:api_key"), []byte(apiKey))
 	})
 	if err != nil {
 		zap.L().Error("set_api_key",
-			zap.String("message", "failed to set api_key"),
+			zap.String("message", "failed to set dehashed api_key"),
+			zap.Error(err),
+		)
+	}
+	return err
+}
+
+func StoreHunterKey(apiKey string) error {
+	err := db.Update(func(txn *badger.Txn) error {
+		return txn.Set([]byte("cfg:hunter_api_key"), []byte(apiKey))
+	})
+	if err != nil {
+		zap.L().Error("set_api_key",
+			zap.String("message", "failed to set hunter api_key"),
 			zap.Error(err),
 		)
 	}

internal/badger/badger_test.go

@@ -0,0 +1,73 @@
+package badger
+
+import (
+	"crypto/sha256"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	badgerapi "github.com/dgraph-io/badger/v4"
+)
+
+func TestMigrateBadgerEncryptionCopiesDataToStableKey(t *testing.T) {
+	dir := t.TempDir()
+	dbPath := filepath.Join(dir, "badger.db")
+	legacyKey := testKey("legacy-key")
+	stableKey := testKey("stable-key")
+
+	legacyDB, err := openBadger(dbPath, legacyKey)
+	if err != nil {
+		t.Fatalf("open legacy db: %v", err)
+	}
+
+	if err := legacyDB.Update(func(txn *badgerapi.Txn) error {
+		return txn.Set([]byte("cfg:api_key"), []byte("secret"))
+	}); err != nil {
+		t.Fatalf("seed legacy db: %v", err)
+	}
+
+	migratedDB, err := migrateBadgerEncryption(dbPath, legacyDB, stableKey)
+	if err != nil {
+		t.Fatalf("migrate db: %v", err)
+	}
+	defer migratedDB.Close()
+
+	var got string
+	if err := migratedDB.View(func(txn *badgerapi.Txn) error {
+		item, err := txn.Get([]byte("cfg:api_key"))
+		if err != nil {
+			return err
+		}
+		return item.Value(func(value []byte) error {
+			got = string(value)
+			return nil
+		})
+	}); err != nil {
+		t.Fatalf("read migrated db: %v", err)
+	}
+
+	if got != "secret" {
+		t.Fatalf("migrated value = %q, want %q", got, "secret")
+	}
+
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		t.Fatalf("read temp dir: %v", err)
+	}
+	foundBackup := false
+	for _, entry := range entries {
+		if strings.HasPrefix(entry.Name(), "badger.db.legacy-backup-") {
+			foundBackup = true
+			break
+		}
+	}
+	if !foundBackup {
+		t.Fatal("legacy backup directory was not created")
+	}
+}
+
+func testKey(value string) []byte {
+	sum := sha256.Sum256([]byte(value))
+	return sum[:]
+}

internal/debug/print.go

@@ -0,0 +1,21 @@
+package debug
+
+import (
+	"fmt"
+	"github.com/fatih/color"
+)
+
+func PrintError(err error) {
+	errLine := fmt.Sprintf("[DEBUG-ERROR] %s", err)
+	fmt.Println(color.RedString(errLine))
+}
+
+func PrintJson(json string) {
+	jsonLine := fmt.Sprintf("[DEBUG-JSON] %s", json)
+	fmt.Print(color.GreenString(jsonLine))
+}
+
+func PrintInfo(info string) {
+	infoLine := fmt.Sprintf("[DEBUG-INFO] %s", info)
+	fmt.Println(color.BlueString(infoLine))
+}

internal/dehashed/clientv2.go

@@ -1,17 +1,19 @@
-package query
+package dehashed
 
 import (
 	"bytes"
 	"crypto/sha256"
-	"dehasher/internal/sqlite"
 	"encoding/hex"
 	"encoding/json"
 	"errors"
 	"fmt"
-	"go.uber.org/zap"
 	"io"
 	"net/http"
 	"strings"
+
+	"go.uber.org/zap"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/debug"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/sqlite"
 )
 
 type DehashedParameter string
@@ -38,6 +40,7 @@ func (dp DehashedParameter) GetArgumentString(arg string) string {
 
 type DehashedSearchRequest struct {
 	ForcePlaintext bool   `json:"-"`
+	Debug          bool   `json:"-"`
 	Page           int    `json:"page"`
 	Query          string `json:"query"`
 	Size           int    `json:"size"`
@@ -46,42 +49,60 @@ type DehashedSearchRequest struct {
 	DeDupe         bool   `json:"de_dupe"`
 }
 
-func NewDehashedSearchRequest(page, size int, wildcard, regex, forcePlaintext bool) *DehashedSearchRequest {
-	return &DehashedSearchRequest{Page: page, Query: "", Size: size, Wildcard: wildcard, Regex: regex, DeDupe: true, ForcePlaintext: forcePlaintext}
+func NewDehashedSearchRequest(page, size int, wildcard, regex, forcePlaintext, debug bool) *DehashedSearchRequest {
+	return &DehashedSearchRequest{Page: page, Query: "", Size: size, Wildcard: wildcard, Regex: regex, DeDupe: true, ForcePlaintext: forcePlaintext, Debug: debug}
 }
 
-func (dsr *DehashedSearchRequest) buildQuery(query string, param DehashedParameter) {
+func (dsr *DehashedSearchRequest) buildQuery(query string) {
+	if dsr.Debug {
+		debug.PrintInfo(fmt.Sprintf("building query: %s", query))
+	}
+	// Ensure query is properly formatted
+	query = strings.TrimSpace(query)
+
+	// For regex queries, we need to ensure the regex pattern is properly escaped
+	// and not enquoted, as that would break the regex pattern
+	if dsr.Regex && !strings.HasPrefix(query, "\"") && !strings.HasSuffix(query, "\"") {
+		// Don't add extra quotes for regex patterns
+	} else if strings.Contains(query, " ") && !strings.HasPrefix(query, "\"") {
+		query = fmt.Sprintf("\"%s\"", query)
+	}
+
 	if len(dsr.Query) > 0 {
-		dsr.Query = fmt.Sprintf("%s&%s", strings.TrimSpace(dsr.Query), strings.TrimSpace(query))
+		dsr.Query = fmt.Sprintf("%s&%s", strings.TrimSpace(dsr.Query), query)
 	} else {
 		dsr.Query = query
 	}
+
+	if dsr.Debug {
+		debug.PrintInfo(fmt.Sprintf("query built: %s", dsr.Query))
+	}
 }
 
 func (dsr *DehashedSearchRequest) AddUsernameQuery(query string) {
 	query = enquoteSpaced(query)
-	dsr.buildQuery(Username.GetArgumentString(query), Username)
+	dsr.buildQuery(Username.GetArgumentString(query))
 }
 
 func (dsr *DehashedSearchRequest) AddEmailQuery(query string) {
 	query = enquoteSpaced(query)
-	dsr.buildQuery(Email.GetArgumentString(query), Email)
+	dsr.buildQuery(Email.GetArgumentString(query))
 }
 
 func (dsr *DehashedSearchRequest) AddIpAddressQuery(query string) {
 	query = enquoteSpaced(query)
-	dsr.buildQuery(IpAddress.GetArgumentString(query), IpAddress)
+	dsr.buildQuery(IpAddress.GetArgumentString(query))
 }
 
 func (dsr *DehashedSearchRequest) AddDomainQuery(query string) {
 	query = enquoteSpaced(query)
-	dsr.buildQuery(Domain.GetArgumentString(query), Domain)
+	dsr.buildQuery(Domain.GetArgumentString(query))
 }
 
 func (dsr *DehashedSearchRequest) AddPasswordQuery(query string) {
 	if dsr.ForcePlaintext {
 		query = enquoteSpaced(query)
-		dsr.buildQuery(Password.GetArgumentString(query), Password)
+		dsr.buildQuery(Password.GetArgumentString(query))
 		return
 	}
 	hash := sha256.Sum256([]byte(query))
@@ -91,89 +112,130 @@ func (dsr *DehashedSearchRequest) AddPasswordQuery(query string) {
 
 func (dsr *DehashedSearchRequest) AddVinQuery(query string) {
 	query = enquoteSpaced(query)
-	dsr.buildQuery(Vin.GetArgumentString(query), Vin)
+	dsr.buildQuery(Vin.GetArgumentString(query))
 }
 
 func (dsr *DehashedSearchRequest) AddLicensePlateQuery(query string) {
 	query = enquoteSpaced(query)
-	dsr.buildQuery(LicensePlate.GetArgumentString(query), LicensePlate)
+	dsr.buildQuery(LicensePlate.GetArgumentString(query))
 }
 
 func (dsr *DehashedSearchRequest) AddAddressQuery(query string) {
 	query = enquoteSpaced(query)
-	dsr.buildQuery(Address.GetArgumentString(query), Address)
+	dsr.buildQuery(Address.GetArgumentString(query))
 }
 
 func (dsr *DehashedSearchRequest) AddPhoneQuery(query string) {
 	query = enquoteSpaced(query)
-	dsr.buildQuery(Phone.GetArgumentString(query), Phone)
+	dsr.buildQuery(Phone.GetArgumentString(query))
 }
 
 func (dsr *DehashedSearchRequest) AddSocialQuery(query string) {
 	query = enquoteSpaced(query)
-	dsr.buildQuery(Social.GetArgumentString(query), Social)
+	dsr.buildQuery(Social.GetArgumentString(query))
 }
 
 func (dsr *DehashedSearchRequest) AddCryptoAddressQuery(query string) {
 	query = enquoteSpaced(query)
-	dsr.buildQuery(CryptoAddress.GetArgumentString(query), CryptoAddress)
+	dsr.buildQuery(CryptoAddress.GetArgumentString(query))
 }
 
 func (dsr *DehashedSearchRequest) AddHashedPasswordQuery(query string) {
 	query = strings.TrimSpace(query)
-	dsr.buildQuery(HashedPassword.GetArgumentString(query), HashedPassword)
+	dsr.buildQuery(HashedPassword.GetArgumentString(query))
 }
 
 func (dsr *DehashedSearchRequest) AddNameQuery(query string) {
 	query = enquoteSpaced(query)
-	dsr.buildQuery(Name.GetArgumentString(query), Name)
+	dsr.buildQuery(Name.GetArgumentString(query))
 }
 
 type DehashedClientV2 struct {
 	apiKey  string
 	results []sqlite.Result
+	debug   bool
 }
 
-func NewDehashedClientV2(apiKey string) *DehashedClientV2 {
-	return &DehashedClientV2{apiKey: apiKey}
+func NewDehashedClientV2(apiKey string, debug bool) *DehashedClientV2 {
+	return &DehashedClientV2{apiKey: apiKey, debug: debug}
 }
 
-func (dcv2 *DehashedClientV2) Search(searchRequest DehashedSearchRequest) (int, error) {
-	reqBody, _ := json.Marshal(searchRequest)
+func (dcv2 *DehashedClientV2) Search(searchRequest DehashedSearchRequest) (int, int, error) {
+	if dcv2.debug {
+		debug.PrintInfo("preparing search request")
+		zap.L().Info("v2_search_debug",
+			zap.String("message", "preparing search request"),
+		)
+	}
+
+	// Create a copy of the search request to avoid modifying the original
+	requestCopy := searchRequest
+
+	reqBody, _ := json.Marshal(requestCopy)
+
+	if dcv2.debug {
+		j := string(reqBody)
+		jReq := fmt.Sprintf("Request Body: %s\n", j)
+		debug.PrintJson(jReq)
+		zap.L().Info("v2_search_debug",
+			zap.String("message", jReq),
+			zap.String("body", j),
+		)
+	}
+
 	req, err := http.NewRequest("POST", "https://api.dehashed.com/v2/search", bytes.NewReader(reqBody))
 	if err != nil {
-		return -1, err
+		return -1, -1, err
 	}
+
+	if dcv2.debug {
+		debug.PrintInfo("setting headers")
+		zap.L().Info("v2_search_debug",
+			zap.String("message", "setting headers"),
+		)
+	}
+
 	req.Header.Set("Content-Type", "application/json")
 	req.Header.Set("Dehashed-Api-Key", dcv2.apiKey)
+
+	if dcv2.debug {
+		headers := redactedHeaders(req.Header)
+		h := fmt.Sprintf("Headers: %v\n", headers)
+		debug.PrintJson(h)
+		zap.L().Info("v2_search_debug",
+			zap.String("message", h),
+			zap.String("headers", fmt.Sprintf("%v", headers)),
+		)
+
+		debug.PrintInfo("performing request")
+		zap.L().Info("v2_search_debug",
+			zap.String("message", "performing request"),
+		)
+	}
+
 	res, err := http.DefaultClient.Do(req)
 	if res != nil {
 		defer res.Body.Close()
 	}
 	if err != nil {
+		if dcv2.debug {
+			debug.PrintInfo("failed to perform request")
+			debug.PrintError(err)
+		}
 		zap.L().Error("v2_search",
 			zap.String("message", "failed to perform request"),
 			zap.Error(err),
 		)
-		return -1, err
+		return -1, -1, err
 	}
 	if res == nil {
+		if dcv2.debug {
+			debug.PrintInfo("response was nil")
+		}
 		zap.L().Error("v2_search",
 			zap.String("message", "response was nil"),
 		)
-		return -1, errors.New("response was nil")
-	}
-
-	// Check for HTTP status code errors
-	if res.StatusCode != 200 {
-		dhErr := GetDehashedError(res.StatusCode)
-		fmt.Printf("[%d] API Error message: %s\n", res.StatusCode, dhErr.Error())
-		zap.L().Error("v2_search",
-			zap.String("message", "received error status code"),
-			zap.Int("status_code", res.StatusCode),
-			zap.String("error", dhErr.Error()),
-		)
-		return -1, &dhErr
+		return -1, -1, errors.New("response was nil")
 	}
 
 	b, err := io.ReadAll(res.Body)
@@ -182,21 +244,50 @@ func (dcv2 *DehashedClientV2) Search(searchRequest DehashedSearchRequest) (int,
 			zap.String("message", "failed to read response body"),
 			zap.Error(err),
 		)
-		return -1, err
+		return -1, -1, err
+	}
+
+	// Check for HTTP status code errors
+	if res.StatusCode != 200 {
+		if dcv2.debug {
+			debug.PrintInfo("received error status code")
+			debug.PrintJson(fmt.Sprintf("Status Code: %d\n", res.StatusCode))
+			debug.PrintJson(fmt.Sprintf("Body: %s\n", string(b[:])))
+		}
+
+		dhErr := GetDehashedError(res.StatusCode)
+		zap.L().Error("v2_search",
+			zap.String("message", "received error status code"),
+			zap.Int("status_code", res.StatusCode),
+			zap.String("error", dhErr.Error()),
+			zap.String("body_error", string(b)),
+		)
+		return -1, -1, &dhErr
 	}
 
 	var responseResults sqlite.DehashedResponse
 	err = json.Unmarshal(b, &responseResults)
 	if err != nil {
+		if dcv2.debug {
+			debug.PrintInfo("failed to unmarshal response body")
+			debug.PrintError(err)
+		}
 		zap.L().Error("v2_search",
 			zap.String("message", "failed to unmarshal response body"),
 			zap.Error(err),
 		)
-		return -1, err
+		return -1, -1, err
+	}
+
+	if dcv2.debug {
+		debug.PrintInfo("appending results")
+		debug.PrintJson(fmt.Sprintf("Total Results: %d\n", responseResults.TotalResults))
+		debug.PrintJson(fmt.Sprintf("Balance: %d\n", responseResults.Balance))
+		debug.PrintJson(fmt.Sprintf("Entries: %d\n", len(responseResults.Entries)))
 	}
 
 	dcv2.results = append(dcv2.results, responseResults.Entries...)
-	return responseResults.TotalResults, nil
+	return len(responseResults.Entries), responseResults.Balance, nil
 }
 
 func (dcv2 *DehashedClientV2) GetResults() sqlite.DehashedResults {
@@ -214,3 +305,11 @@ func enquoteSpaced(s string) string {
 	}
 	return s
 }
+
+func redactedHeaders(headers http.Header) http.Header {
+	redacted := headers.Clone()
+	if redacted.Get("Dehashed-Api-Key") != "" {
+		redacted.Set("Dehashed-Api-Key", "[REDACTED]")
+	}
+	return redacted
+}

internal/dehashed/datawells.go

@@ -0,0 +1,186 @@
+package dehashed
+
+import (
+	"encoding/json"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"strconv"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/files"
+)
+
+const dataWellsEndpoint = "https://api.dehashed.com/data-wells"
+
+type DataWellsRequest struct {
+	Count int
+	Page  int
+	Sort  string
+}
+
+type DataWellsResponse struct {
+	NextPage  bool       `json:"next_page" xml:"next_page" yaml:"next_page"`
+	Total     int        `json:"total" xml:"total" yaml:"total"`
+	DataWells []DataWell `json:"data_wells" xml:"data_wells" yaml:"data_wells"`
+}
+
+type DataWell struct {
+	Data        string `json:"data" xml:"data" yaml:"data"`
+	Date        string `json:"date" xml:"date" yaml:"date"`
+	Description string `json:"description" xml:"description" yaml:"description"`
+	Name        string `json:"name" xml:"name" yaml:"name"`
+	Records     int    `json:"records" xml:"records" yaml:"records"`
+	IsSensitive bool   `json:"is_sensitive" xml:"is_sensitive" yaml:"is_sensitive"`
+}
+
+func (dcv2 *DehashedClientV2) DataWells(request DataWellsRequest) (DataWellsResponse, error) {
+	var dataWells DataWellsResponse
+
+	endpoint, err := dataWellsURL(request)
+	if err != nil {
+		return dataWells, err
+	}
+
+	req, err := http.NewRequest("GET", endpoint, nil)
+	if err != nil {
+		return dataWells, err
+	}
+	req.Header.Set("Accept", "application/json")
+
+	res, err := http.DefaultClient.Do(req)
+	if res != nil {
+		defer res.Body.Close()
+	}
+	if err != nil {
+		return dataWells, err
+	}
+	if res == nil {
+		return dataWells, errors.New("response was nil")
+	}
+
+	body, err := io.ReadAll(res.Body)
+	if err != nil {
+		return dataWells, err
+	}
+
+	if res.StatusCode != http.StatusOK {
+		return dataWells, fmt.Errorf("data wells request failed: status=%d body=%s", res.StatusCode, string(body))
+	}
+
+	if err := json.Unmarshal(body, &dataWells); err != nil {
+		return dataWells, err
+	}
+
+	return dataWells, nil
+}
+
+func dataWellsURL(request DataWellsRequest) (string, error) {
+	if request.Page <= 0 {
+		return "", errors.New("page must be 1 or greater")
+	}
+	if request.Count != 20 && request.Count != 50 {
+		return "", errors.New("count must be 20 or 50")
+	}
+	if request.Sort != "" && !validDataWellsSort(request.Sort) {
+		return "", fmt.Errorf("invalid sort %q; use added, name, date, records, optionally suffixed with -ASC or -DESC", request.Sort)
+	}
+
+	values := url.Values{}
+	values.Set("page", strconv.Itoa(request.Page))
+	values.Set("count", strconv.Itoa(request.Count))
+	if request.Sort != "" {
+		values.Set("sort", request.Sort)
+	}
+
+	return dataWellsEndpoint + "?" + values.Encode(), nil
+}
+
+func validDataWellsSort(sortValue string) bool {
+	sortValue = strings.ToLower(strings.TrimSpace(sortValue))
+	field := sortValue
+	if before, _, ok := strings.Cut(sortValue, "-"); ok {
+		field = before
+	}
+
+	switch field {
+	case "added", "name", "date", "records":
+		return strings.HasSuffix(sortValue, "-asc") || strings.HasSuffix(sortValue, "-desc") || !strings.Contains(sortValue, "-")
+	default:
+		return false
+	}
+}
+
+func WriteDataWellsToFile(dataWells DataWellsResponse, outputFile string, fileType files.FileType) error {
+	var data []byte
+	var err error
+
+	switch fileType {
+	case files.JSON:
+		data, err = json.MarshalIndent(dataWells, "", "  ")
+	case files.XML:
+		data, err = xml.MarshalIndent(dataWells, "", "  ")
+	case files.YAML:
+		data, err = yaml.Marshal(dataWells)
+	case files.TEXT:
+		data = []byte(dataWells.String())
+	case files.GREPPABLE:
+		var outStrings []string
+		for _, well := range dataWells.DataWells {
+			outStrings = append(outStrings, dataWellGreppable(well)+"\n")
+		}
+		data = []byte(strings.Join(outStrings, ""))
+	default:
+		return errors.New("unsupported file type")
+	}
+
+	if err != nil {
+		return err
+	}
+
+	return os.WriteFile(outputFile+fileType.Extension(), data, 0644)
+}
+
+func (dwr DataWellsResponse) String() string {
+	var b strings.Builder
+	fmt.Fprintf(&b, "Total: %d\nNext Page: %t\n\n", dwr.Total, dwr.NextPage)
+	for _, well := range dwr.DataWells {
+		fmt.Fprintf(&b, "Name: %s\nDate: %s\nRecords: %d\nSensitive: %t\nData: %s\nDescription: %s\n\n",
+			well.Name,
+			well.Date,
+			well.Records,
+			well.IsSensitive,
+			well.Data,
+			well.Description,
+		)
+	}
+	return b.String()
+}
+
+func dataWellGreppable(well DataWell) string {
+	var fields []string
+	fields = appendDataWellGreppableField(fields, "name", well.Name)
+	fields = appendDataWellGreppableField(fields, "date", well.Date)
+	fields = appendDataWellGreppableField(fields, "records", strconv.Itoa(well.Records))
+	fields = appendDataWellGreppableField(fields, "is_sensitive", strconv.FormatBool(well.IsSensitive))
+	fields = appendDataWellGreppableField(fields, "data", well.Data)
+	fields = appendDataWellGreppableField(fields, "description", well.Description)
+	return strings.Join(fields, " ")
+}
+
+func cleanGreppableValue(value string) string {
+	return strings.Join(strings.Fields(value), "_")
+}
+
+func appendDataWellGreppableField(fields []string, key, value string) []string {
+	value = cleanGreppableValue(value)
+	if value == "" {
+		return fields
+	}
+	return append(fields, fmt.Sprintf("%s=%s", key, value))
+}

internal/dehashed/dehashed.go

@@ -0,0 +1,335 @@
+package dehashed
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"go.uber.org/zap"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/debug"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/export"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/pretty"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/sqlite"
+)
+
+const (
+	maxSearchResultsPerPage  = 10000
+	maxSearchResultsPerQuery = 50000
+)
+
+// Dehasher is a struct for querying the Dehashed API
+type Dehasher struct {
+	options    sqlite.QueryOptions
+	nextPage   int
+	debug      bool
+	balance    int
+	maxResults int
+	request    *DehashedSearchRequest
+	client     *DehashedClientV2
+}
+
+// NewDehasher creates a new Dehasher
+func NewDehasher(options *sqlite.QueryOptions) *Dehasher {
+	dh := &Dehasher{
+		options:  *options,
+		nextPage: options.StartingPage + 1,
+		debug:    options.Debug,
+		balance:  0,
+	}
+	dh.setQueries()
+	dh.request = NewDehashedSearchRequest(dh.options.StartingPage, dh.options.MaxRecords, dh.options.WildcardMatch, dh.options.RegexMatch, false, options.Debug)
+	dh.buildRequest()
+	return dh
+}
+
+// SetClientCredentials sets the client credentials for the dehasher
+func (dh *Dehasher) SetClientCredentials(key string) {
+	dh.client = NewDehashedClientV2(key, dh.debug)
+}
+
+func (dh *Dehasher) getNextPage() int {
+	if dh.debug {
+		debug.PrintInfo(fmt.Sprintf("getting next page: %d", dh.nextPage))
+	}
+	nextPage := dh.nextPage
+	dh.nextPage += 1
+	return nextPage
+}
+
+// setQueries sets the number of queries to make based on the number of records and requests
+func (dh *Dehasher) setQueries() {
+	if dh.debug {
+		debug.PrintInfo("setting queries")
+	}
+
+	if dh.options.MaxRequests == 0 {
+		zap.L().Error("max requests cannot be zero")
+		fmt.Println("[!] Max Requests cannot be zero")
+		os.Exit(1)
+	}
+
+	requestedMaxResults := dh.options.MaxRecords
+	if requestedMaxResults <= 0 {
+		requestedMaxResults = maxSearchResultsPerQuery
+	}
+	if requestedMaxResults > maxSearchResultsPerQuery {
+		requestedMaxResults = maxSearchResultsPerQuery
+	}
+
+	pageSize := requestedMaxResults
+	if pageSize > maxSearchResultsPerPage {
+		pageSize = maxSearchResultsPerPage
+	}
+
+	numQueries := (requestedMaxResults + pageSize - 1) / pageSize
+	if dh.options.MaxRequests > 0 && dh.options.MaxRequests < numQueries {
+		numQueries = dh.options.MaxRequests
+	}
+
+	dh.maxResults = requestedMaxResults
+	if requestLimit := numQueries * pageSize; requestLimit < dh.maxResults {
+		dh.maxResults = requestLimit
+	}
+
+	dh.options.MaxRecords = pageSize
+	dh.options.MaxRequests = numQueries
+
+	zap.L().Info("dehashed_search_pagination",
+		zap.Int("max_results", dh.maxResults),
+		zap.Int("page_size", dh.options.MaxRecords),
+		zap.Int("max_requests", dh.options.MaxRequests),
+	)
+
+	if dh.debug {
+		debug.PrintInfo(fmt.Sprintf("setting max requests: %d", numQueries))
+		debug.PrintInfo(fmt.Sprintf("setting page size: %d", dh.options.MaxRecords))
+		debug.PrintInfo(fmt.Sprintf("setting max results: %d", dh.maxResults))
+	}
+
+	fmt.Printf("Making %d Requests for up to %d Records (%d per request)\n", dh.options.MaxRequests, dh.maxResults, dh.options.MaxRecords)
+}
+
+// Start starts the querying process
+func (dh *Dehasher) Start() {
+	fmt.Printf("[*] Querying Dehashed API...\n")
+	for i := 0; i < dh.options.MaxRequests; i++ {
+		fmt.Printf("   [*] Performing Request...\n")
+		count, balance, err := dh.client.Search(*dh.request)
+		if err != nil {
+			if dh.debug {
+				debug.PrintInfo("error performing request")
+				debug.PrintError(err)
+			}
+
+			// Check if it's a DehashError
+			if dhErr, ok := err.(*DehashError); ok {
+				fmt.Printf("      [!] Dehashed API Error: %s (Code: %d)\n", dhErr.Message, dhErr.Code)
+				zap.L().Error("dehashed_api_error",
+					zap.String("message", dhErr.Message),
+					zap.Int("code", dhErr.Code),
+				)
+			} else {
+				fmt.Printf("   [!] Error performing request: %v\n", err)
+				zap.L().Error("request_error",
+					zap.String("message", "failed to perform request"),
+					zap.Error(err),
+				)
+			}
+
+			if len(dh.client.results) > 0 {
+				fmt.Printf("   [!] Partial results retrieved. Storing Results...\n")
+				err := sqlite.StoreDehashedResults(dh.client.GetResults())
+				if err != nil {
+					zap.L().Error("store_results",
+						zap.String("message", "failed to store results"),
+						zap.Error(err),
+					)
+					fmt.Printf("   [!] Error storing results: %v\n", err)
+				}
+			}
+			dh.parseResults()
+			os.Exit(-1)
+		}
+
+		dh.balance = balance
+
+		if count < dh.options.MaxRecords {
+			fmt.Printf("      [+] Retrieved %d records\n", count)
+			fmt.Printf("      [-] Not enough entries, ending queries\n")
+			break
+		} else {
+			fmt.Printf("      [+] Retrieved %d records\n", count)
+		}
+
+		if dh.options.PrintBalance {
+			fmt.Printf("      [*] Balance: %d\n", balance)
+		}
+
+		dh.request.Page = dh.getNextPage()
+	}
+
+	dh.parseResults()
+}
+
+// buildRequest constructs the query map
+func (dh *Dehasher) buildRequest() {
+	if len(dh.options.UsernameQuery) > 0 {
+		dh.request.AddUsernameQuery(dh.options.UsernameQuery)
+	}
+	if len(dh.options.EmailQuery) > 0 {
+		dh.request.AddEmailQuery(dh.options.EmailQuery)
+	}
+	if len(dh.options.IpQuery) > 0 {
+		dh.request.AddIpAddressQuery(dh.options.IpQuery)
+	}
+	if len(dh.options.HashQuery) > 0 {
+		dh.request.AddHashedPasswordQuery(dh.options.HashQuery)
+	}
+	if len(dh.options.PassQuery) > 0 {
+		dh.request.AddPasswordQuery(dh.options.PassQuery)
+	}
+	if len(dh.options.NameQuery) > 0 {
+		dh.request.AddNameQuery(dh.options.NameQuery)
+	}
+	if len(dh.options.DomainQuery) > 0 {
+		dh.request.AddDomainQuery(dh.options.DomainQuery)
+	}
+	if len(dh.options.VinQuery) > 0 {
+		dh.request.AddVinQuery(dh.options.VinQuery)
+	}
+	if len(dh.options.LicensePlateQuery) > 0 {
+		dh.request.AddLicensePlateQuery(dh.options.LicensePlateQuery)
+	}
+	if len(dh.options.AddressQuery) > 0 {
+		dh.request.AddAddressQuery(dh.options.AddressQuery)
+	}
+	if len(dh.options.PhoneQuery) > 0 {
+		dh.request.AddPhoneQuery(dh.options.PhoneQuery)
+	}
+	if len(dh.options.SocialQuery) > 0 {
+		dh.request.AddSocialQuery(dh.options.SocialQuery)
+	}
+	if len(dh.options.CryptoAddressQuery) > 0 {
+		dh.request.AddCryptoAddressQuery(dh.options.CryptoAddressQuery)
+	}
+}
+
+// parseResults parses the results and writes them to a file
+func (dh *Dehasher) parseResults() {
+	zap.L().Info("extracting_credentials")
+	results := dh.client.GetResults()
+	if dh.maxResults > 0 && len(results.Results) > dh.maxResults {
+		results.Results = results.Results[:dh.maxResults]
+	}
+	creds := results.ExtractUsers()
+	fmt.Printf("   [+] Discovered %d Credentials\n", len(creds))
+	err := sqlite.StoreUsers(creds)
+	if err != nil {
+		zap.L().Error("store_creds",
+			zap.String("message", "failed to store creds"),
+			zap.Error(err),
+		)
+	}
+	zap.L().Info("creds_stored", zap.Int("count", len(creds)))
+
+	zap.L().Info("storing_results")
+	err = sqlite.StoreDehashedResults(results)
+	if err != nil {
+		zap.L().Error("store_results",
+			zap.String("message", "failed to store results"),
+			zap.Error(err),
+		)
+	}
+	zap.L().Info("results_stored", zap.Int("count", len(results.Results)))
+
+	if len(results.Results) > 0 {
+		var (
+			headers = []string{"Email", "Username", "Password"}
+			rows    [][]string
+		)
+
+		fmt.Printf("   [*] Writing entries to file: %s.%s\n", dh.options.OutputFile, dh.options.OutputFormat.String())
+		if !dh.options.CredsOnly {
+			err := export.WriteToFile(results, dh.options.OutputFile, dh.options.OutputFormat)
+			if err != nil {
+				fmt.Printf("[!] Error Writing to file: %v      Outputting to terminal.\n", err)
+				zap.L().Error("write_results",
+					zap.String("message", "failed to write results to file"),
+					zap.Error(err),
+				)
+			} else {
+				fmt.Println("      [*] Success")
+			}
+
+			if dh.debug {
+				debug.PrintInfo("printing results table")
+			}
+
+			headers = []string{"Email", "Username", "Password", "Phone", "Company"}
+			if len(results.Results) > 50 {
+				fmt.Println("   [-] Large number of results recovered, displaying first 50...")
+				for i := 0; i < 50; i++ {
+					r := results.Results[i]
+					rows = append(rows, []string{
+						strings.Join(r.Email, ", "),
+						strings.Join(r.Username, ", "),
+						strings.Join(r.Password, ", "),
+						strings.Join(r.Phone, ", "),
+						strings.Join(r.Company, ", ")})
+				}
+			} else {
+				for _, r := range results.Results {
+					rows = append(rows, []string{
+						strings.Join(r.Email, ", "),
+						strings.Join(r.Username, ", "),
+						strings.Join(r.Password, ", "),
+						strings.Join(r.Phone, ", "),
+						strings.Join(r.Company, ", ")})
+				}
+			}
+
+			// Print Table
+			pretty.Table(headers, rows)
+		} else {
+			if dh.debug {
+				debug.PrintInfo("extracting credentials")
+			}
+			creds := results.ExtractUsers()
+			if dh.debug {
+				debug.PrintInfo("writing credentials to file")
+			}
+			err := export.WriteCredsToFile(creds, dh.options.OutputFile, dh.options.OutputFormat)
+			if err != nil {
+				fmt.Printf("[!] Error Writing to file: %v\n   Outputting to terminal.", err)
+				zap.L().Error("write_creds",
+					zap.String("message", "failed to write creds to file"),
+					zap.Error(err),
+				)
+			} else {
+				fmt.Println("      [*] Success")
+			}
+
+			if dh.debug {
+				debug.PrintInfo("printing credentials table")
+			}
+
+			headers = []string{"Email", "Username", "Password"}
+			if len(creds) > 50 {
+				fmt.Println("   [-] Large number of results recovered, displaying first 50...")
+				for i := 0; i < 50; i++ {
+					c := creds[i]
+					rows = append(rows, []string{c.Email, c.Username, c.Password})
+				}
+			} else {
+				for _, c := range creds {
+					rows = append(rows, []string{c.Email, c.Username, c.Password})
+				}
+			}
+
+			// Print Table
+			pretty.Table(headers, rows)
+		}
+	} else {
+		fmt.Println("   [-] No results found")
+	}
+}

internal/dehashed/dehashed_test.go

@@ -0,0 +1,94 @@
+package dehashed
+
+import (
+	"strings"
+	"testing"
+
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/sqlite"
+)
+
+func TestSetQueriesCapsSearchAtFiftyThousandResults(t *testing.T) {
+	options := &sqlite.QueryOptions{
+		MaxRecords:   75000,
+		MaxRequests:  -1,
+		StartingPage: 1,
+	}
+
+	dehasher := NewDehasher(options)
+
+	if dehasher.maxResults != maxSearchResultsPerQuery {
+		t.Fatalf("maxResults = %d, want %d", dehasher.maxResults, maxSearchResultsPerQuery)
+	}
+	if dehasher.options.MaxRecords != maxSearchResultsPerPage {
+		t.Fatalf("page size = %d, want %d", dehasher.options.MaxRecords, maxSearchResultsPerPage)
+	}
+	if dehasher.options.MaxRequests != 5 {
+		t.Fatalf("max requests = %d, want 5", dehasher.options.MaxRequests)
+	}
+}
+
+func TestSetQueriesHonorsExplicitRequestLimit(t *testing.T) {
+	options := &sqlite.QueryOptions{
+		MaxRecords:   50000,
+		MaxRequests:  1,
+		StartingPage: 1,
+	}
+
+	dehasher := NewDehasher(options)
+
+	if dehasher.maxResults != maxSearchResultsPerPage {
+		t.Fatalf("maxResults = %d, want %d", dehasher.maxResults, maxSearchResultsPerPage)
+	}
+	if dehasher.options.MaxRecords != maxSearchResultsPerPage {
+		t.Fatalf("page size = %d, want %d", dehasher.options.MaxRecords, maxSearchResultsPerPage)
+	}
+	if dehasher.options.MaxRequests != 1 {
+		t.Fatalf("max requests = %d, want 1", dehasher.options.MaxRequests)
+	}
+}
+
+func TestDataWellsURLDoesNotRequireAPIKey(t *testing.T) {
+	got, err := dataWellsURL(DataWellsRequest{
+		Count: 50,
+		Page:  2,
+		Sort:  "records-DESC",
+	})
+	if err != nil {
+		t.Fatalf("dataWellsURL returned error: %v", err)
+	}
+
+	if !strings.HasPrefix(got, dataWellsEndpoint+"?") {
+		t.Fatalf("url = %q, want prefix %q", got, dataWellsEndpoint+"?")
+	}
+	gotLower := strings.ToLower(got)
+	if strings.Contains(gotLower, "api_key") || strings.Contains(gotLower, "dehashed-api-key") {
+		t.Fatalf("url contains API key material: %q", got)
+	}
+	for _, want := range []string{"count=50", "page=2", "sort=records-DESC"} {
+		if !strings.Contains(got, want) {
+			t.Fatalf("url = %q, want %q", got, want)
+		}
+	}
+}
+
+func TestDataWellGreppableUsesSpaceSeparatedNonEmptyTokens(t *testing.T) {
+	got := dataWellGreppable(DataWell{
+		Name:        "Example Breach",
+		Date:        "2025-03-01",
+		Records:     500000,
+		IsSensitive: true,
+		Data:        "name,email,address",
+	})
+
+	if strings.Contains(got, "\t") {
+		t.Fatalf("greppable output contains tab: %q", got)
+	}
+	if strings.Contains(got, "description=") {
+		t.Fatalf("greppable output contains empty field: %q", got)
+	}
+	for _, want := range []string{"name=Example_Breach", "date=2025-03-01", "records=500000", "is_sensitive=true", "data=name,email,address"} {
+		if !strings.Contains(got, want) {
+			t.Fatalf("greppable output = %q, want token %q", got, want)
+		}
+	}
+}

internal/dehashed/errors.go

@@ -1,4 +1,4 @@
-package query
+package dehashed
 
 type DehashError struct {
 	Message string

internal/easyTime/parser.go

@@ -0,0 +1,267 @@
+package easyTime
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"go.uber.org/zap"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/debug"
+)
+
+type TimeChunk struct {
+	StartTime time.Time
+	EndTime   time.Time
+	set       bool
+}
+
+func (tc *TimeChunk) isValid() bool {
+	if !tc.StartTime.IsZero() && !tc.EndTime.IsZero() && tc.StartTime.Before(tc.EndTime) {
+		tc.set = true
+		return true
+	}
+	tc.set = false
+	return false
+}
+
+func (tc *TimeChunk) IsSet() bool {
+	return tc.set
+}
+
+func NewTimeChunk(start, end string, debugOn bool) TimeChunk {
+	if debugOn {
+		debug.PrintInfo("parsing time chunk")
+		debug.PrintInfo(fmt.Sprintf("Start: %s, End: %s", start, end))
+		zap.L().Info("parsing time chunk",
+			zap.String("start", start),
+			zap.String("end", end),
+		)
+	}
+
+	if end == "" {
+		if debugOn {
+			debug.PrintInfo("no end time provided, using now")
+		}
+		end = "now"
+	}
+
+	tc := TimeChunk{
+		StartTime: parseUserTime(start),
+		EndTime:   parseUserTime(end),
+	}
+
+	if debugOn {
+		debug.PrintInfo("checking if time chunk is valid")
+		debug.PrintInfo(fmt.Sprintf("Start: %s, End: %s", tc.StartTime, tc.EndTime))
+	}
+	if !tc.isValid() {
+		fmt.Println("[!] Invalid time chunk")
+		zap.L().Fatal("invalid_time_chunk",
+			zap.String("message", "invalid time chunk"),
+		)
+		os.Exit(1)
+	}
+
+	return tc
+}
+
+func parseUserTime(args string) time.Time {
+	args = strings.TrimSpace(args)
+
+	if strings.EqualFold(args, "now") {
+		return time.Now()
+	}
+
+	// Check if time contains a space, if so, it's in 'last 24 hours' format
+	if strings.Contains(args, " ") && !containsMonth(strings.Split(args, " ")) {
+		splitArgs := strings.Split(args, " ")
+		if len(splitArgs) == 0 {
+			fmt.Println("[!] No time provided")
+			zap.L().Fatal("no_time_provided",
+				zap.String("message", "no time provided"),
+			)
+			os.Exit(1)
+		} else if len(splitArgs) < 3 {
+			fmt.Println("[!] Invalid time format")
+			zap.L().Fatal("invalid_time_format",
+				zap.String("message", "invalid time format"),
+			)
+			os.Exit(1)
+		}
+
+		// Handle 'last 24 hours' format
+		var (
+			tense    string
+			amount   int
+			duration time.Duration
+		)
+		for _, arg := range splitArgs {
+			if isPasteTense(arg) {
+				tense = arg
+			} else if isNumber(arg) {
+				amount, _ = strconv.Atoi(arg)
+			} else if isDuration(arg) {
+				duration = getDuration(arg)
+			}
+		}
+
+		if tense == "" {
+			fmt.Println("[!] Invalid time format: tense not found")
+			zap.L().Fatal("invalid_time_format",
+				zap.String("message", "invalid time format"),
+			)
+			os.Exit(1)
+		} else if amount == 0 {
+			fmt.Println("[!] Invalid time format: amount not found")
+			zap.L().Fatal("invalid_time_format",
+				zap.String("message", "invalid time format"),
+			)
+			os.Exit(1)
+		} else if duration == 0 {
+			fmt.Println("[!] Invalid time format: duration not found")
+			zap.L().Fatal("invalid_time_format",
+				zap.String("message", "invalid time format"),
+			)
+			os.Exit(1)
+		}
+
+		// Return the appropriate time
+		if tense == "last" {
+			return time.Now().Add(-time.Duration(amount) * duration)
+		} else if tense == "ago" {
+			return time.Now().Add(-time.Duration(amount) * duration)
+		}
+	}
+
+	// Handle possible formats 'May 01, 2025', '05-01-2025', '05/01/2025', '05/01/25', '05-01-25'
+	var (
+		t     time.Time
+		err   error
+		found bool
+	)
+	possible := []string{"01-02-2006", "01/02/2006", "01/02/06", "01-02-06", "Jan 02, 2006", "Jan 2, 2006"}
+	for _, format := range possible {
+		t, err = time.Parse(format, args)
+		if err == nil {
+			found = true
+			break
+		}
+	}
+
+	if !found {
+		fmt.Println("[!] Invalid time format")
+		zap.L().Fatal("invalid_time_format",
+			zap.String("message", "invalid time format"),
+		)
+		os.Exit(1)
+	}
+
+	// Convert UTC time to local time
+	local, err := time.LoadLocation("Local")
+	if err != nil {
+		fmt.Println("[!] Error loading local timezone")
+		zap.L().Error("load_timezone",
+			zap.String("message", "failed to load local timezone"),
+			zap.Error(err),
+		)
+		return t
+	}
+
+	// Convert the parsed time to local time
+	return time.Date(
+		t.Year(),
+		t.Month(),
+		t.Day(),
+		t.Hour(),
+		t.Minute(),
+		t.Second(),
+		t.Nanosecond(),
+		local,
+	)
+}
+
+func isPasteTense(value string) bool {
+	for _, v := range []string{"last", "ago"} {
+		if strings.EqualFold(value, v) {
+			return true
+		}
+	}
+	return false
+}
+
+func isDuration(value string) bool {
+	for _, v := range []string{"hour", "hours", "minute", "minutes", "second", "seconds", "day", "days", "week", "weeks", "month", "months", "year", "years"} {
+		if strings.EqualFold(value, v) {
+			return true
+		}
+	}
+	return false
+}
+
+func isNumber(value string) bool {
+	_, err := strconv.Atoi(value)
+	return err == nil
+}
+
+func getDuration(timeBlock string) time.Duration {
+	timeBlock = strings.TrimSpace(strings.ToLower(timeBlock))
+
+	switch timeBlock {
+	case "hour":
+		return time.Hour
+	case "hours":
+		return time.Hour
+	case "minute":
+		return time.Minute
+	case "minutes":
+		return time.Minute
+	case "second":
+		return time.Second
+	case "seconds":
+		return time.Second
+	case "day":
+		return 24 * time.Hour
+	case "days":
+		return 24 * time.Hour
+	case "week":
+		return 7 * 24 * time.Hour
+	case "weeks":
+		return 7 * 24 * time.Hour
+	case "month":
+		return 30 * 24 * time.Hour
+	case "months":
+		return 30 * 24 * time.Hour
+	case "year":
+		return 365 * 24 * time.Hour
+	case "years":
+		return 365 * 24 * time.Hour
+	default:
+		fmt.Printf("[!] Unknown duration: %s", timeBlock)
+		zap.L().Fatal("unknown_duration",
+			zap.String("message", "unknown duration"),
+			zap.String("duration", timeBlock),
+		)
+		os.Exit(1)
+	}
+	return 0
+}
+
+func containsMonth(arr []string) bool {
+	for _, v := range arr {
+		if isMonth(v) {
+			return true
+		}
+	}
+	return false
+}
+
+func isMonth(value string) bool {
+	for _, v := range []string{"jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"} {
+		if strings.EqualFold(value, v) {
+			return true
+		}
+	}
+	return false
+}

internal/export/dehashed.go

@@ -0,0 +1,338 @@
+package export
+
+import (
+	"encoding/json"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"os"
+	"sort"
+	"strings"
+	"time"
+
+	"gopkg.in/yaml.v3"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/files"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/sqlite"
+)
+
+func WriteCredsToFile(creds []sqlite.User, outputFile string, fileType files.FileType) error {
+	var data []byte
+	var err error
+
+	switch fileType {
+	case files.JSON:
+		data, err = json.MarshalIndent(creds, "", "  ")
+	case files.XML:
+		data, err = xml.MarshalIndent(creds, "", "  ")
+	case files.YAML:
+		data, err = yaml.Marshal(creds)
+	case files.TEXT:
+		var outStrings []string
+		for _, c := range creds {
+			outStrings = append(outStrings, c.ToString()+"\n")
+		}
+		data = []byte(strings.Join(outStrings, ""))
+	case files.GREPPABLE:
+		var outStrings []string
+		for _, c := range creds {
+			var fields []string
+			fields = appendGreppableField(fields, "email", c.Email)
+			fields = appendGreppableField(fields, "username", c.Username)
+			fields = appendGreppableField(fields, "password", c.Password)
+			outStrings = append(outStrings, strings.Join(fields, " ")+"\n")
+		}
+		data = []byte(strings.Join(outStrings, ""))
+	default:
+		return errors.New("unsupported file type")
+	}
+
+	if err != nil {
+		return err
+	}
+
+	filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String())
+	return os.WriteFile(filePath, data, 0644)
+}
+
+func WriteToFile(results sqlite.DehashedResults, outputFile string, fileType files.FileType) error {
+	var data []byte
+	var err error
+
+	result := results.Results
+
+	switch fileType {
+	case files.JSON:
+		data, err = json.MarshalIndent(result, "", "  ")
+	case files.XML:
+		data, err = xml.MarshalIndent(result, "", "  ")
+	case files.YAML:
+		data, err = yaml.Marshal(result)
+	case files.TEXT:
+		var outStrings []string
+		for _, r := range result {
+			out := fmt.Sprintf(
+				"Id: %s\nEmail: %s\nIpAddress: %s\nUsername: %s\nPassword: %s\nHashedPassword: %s\nHashType: %s\nName: %s\nVin: %s\nAddress: %s\nPhone: %s\nDatabaseName: %s\n\n",
+				r.DehashedId, r.Email, r.IpAddress, r.Username, r.Password, r.HashedPassword, r.HashType, r.Name, r.Vin, r.Address, r.Phone, r.DatabaseName)
+			outStrings = append(outStrings, out)
+		}
+		data = []byte(strings.Join(outStrings, ""))
+	case files.GREPPABLE:
+		var outStrings []string
+		for _, r := range result {
+			outStrings = append(outStrings, dehashedResultGreppable(r)+"\n")
+		}
+		data = []byte(strings.Join(outStrings, ""))
+	default:
+		return errors.New("unsupported file type")
+	}
+
+	if err != nil {
+		return err
+	}
+
+	filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String())
+	return os.WriteFile(filePath, data, 0644)
+}
+
+// WriteQueryResultsToFile writes query results to a file in the specified format
+func WriteQueryResultsToFile(results []map[string]interface{}, outputFile string, fileType files.FileType) error {
+	var data []byte
+	var err error
+
+	switch fileType {
+	case files.JSON:
+		data, err = json.MarshalIndent(results, "", "  ")
+	case files.XML:
+		data, err = xml.MarshalIndent(results, "", "  ")
+	case files.YAML:
+		data, err = yaml.Marshal(results)
+	case files.TEXT:
+		var outStrings []string
+		for _, r := range results {
+			var rowStrings []string
+			for k, v := range r {
+				// Format the value to avoid array notation
+				var valueStr string
+				switch val := v.(type) {
+				case []string:
+					valueStr = strings.Join(val, ", ")
+				case []interface{}:
+					strSlice := make([]string, len(val))
+					for i, item := range val {
+						if item == nil {
+							strSlice[i] = ""
+						} else {
+							strSlice[i] = fmt.Sprintf("%v", item)
+						}
+					}
+					valueStr = strings.Join(strSlice, ", ")
+				default:
+					if v == nil {
+						valueStr = ""
+					} else {
+						valueStr = fmt.Sprintf("%v", v)
+					}
+				}
+				rowStrings = append(rowStrings, fmt.Sprintf("%s: %s", k, valueStr))
+			}
+			outStrings = append(outStrings, strings.Join(rowStrings, "\n")+"\n\n")
+		}
+		data = []byte(strings.Join(outStrings, ""))
+	case files.GREPPABLE:
+		var outStrings []string
+		for _, r := range results {
+			keys := make([]string, 0, len(r))
+			for k := range r {
+				keys = append(keys, k)
+			}
+			sort.Strings(keys)
+
+			rowStrings := make([]string, 0, len(keys))
+			for _, k := range keys {
+				rowStrings = appendGreppableField(rowStrings, k, greppableAnyValue(r[k]))
+			}
+			outStrings = append(outStrings, strings.Join(rowStrings, " ")+"\n")
+		}
+		data = []byte(strings.Join(outStrings, ""))
+	default:
+		return errors.New("unsupported file type")
+	}
+
+	if err != nil {
+		return err
+	}
+
+	filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String())
+	return os.WriteFile(filePath, data, 0644)
+}
+
+func dehashedResultGreppable(r sqlite.Result) string {
+	var fields []string
+	fields = appendGreppableField(fields, "id", r.DehashedId)
+	fields = appendGreppableField(fields, "email", strings.Join(r.Email, ","))
+	fields = appendGreppableField(fields, "ip_address", strings.Join(r.IpAddress, ","))
+	fields = appendGreppableField(fields, "username", strings.Join(r.Username, ","))
+	fields = appendGreppableField(fields, "password", strings.Join(r.Password, ","))
+	fields = appendGreppableField(fields, "hashed_password", strings.Join(r.HashedPassword, ","))
+	fields = appendGreppableField(fields, "hash_type", r.HashType)
+	fields = appendGreppableField(fields, "name", strings.Join(r.Name, ","))
+	fields = appendGreppableField(fields, "vin", strings.Join(r.Vin, ","))
+	fields = appendGreppableField(fields, "license_plate", strings.Join(r.LicensePlate, ","))
+	fields = appendGreppableField(fields, "url", strings.Join(r.Url, ","))
+	fields = appendGreppableField(fields, "social", strings.Join(r.Social, ","))
+	fields = appendGreppableField(fields, "cryptocurrency_address", strings.Join(r.CryptoCurrencyAddress, ","))
+	fields = appendGreppableField(fields, "address", strings.Join(r.Address, ","))
+	fields = appendGreppableField(fields, "phone", strings.Join(r.Phone, ","))
+	fields = appendGreppableField(fields, "company", strings.Join(r.Company, ","))
+	fields = appendGreppableField(fields, "database_name", r.DatabaseName)
+	return strings.Join(fields, " ")
+}
+
+func greppableAnyValue(value interface{}) string {
+	switch v := value.(type) {
+	case nil:
+		return ""
+	case []string:
+		return greppableValue(strings.Join(v, ","))
+	case []interface{}:
+		values := make([]string, 0, len(v))
+		for _, item := range v {
+			values = append(values, fmt.Sprintf("%v", item))
+		}
+		return greppableValue(strings.Join(values, ","))
+	case []byte:
+		return greppableValue(string(v))
+	default:
+		return greppableValue(fmt.Sprintf("%v", v))
+	}
+}
+
+func greppableValue(value string) string {
+	return strings.Join(strings.Fields(value), "_")
+}
+
+func appendGreppableField(fields []string, key, value string) []string {
+	value = greppableValue(value)
+	if value == "" {
+		return fields
+	}
+	return append(fields, fmt.Sprintf("%s=%s", key, value))
+}
+
+func WriteWhoIsHistoryToFile(results []sqlite.HistoryRecord, outputFile string, fileType files.FileType) error {
+	var data []byte
+	var err error
+
+	switch fileType {
+	case files.JSON:
+		data, err = json.MarshalIndent(results, "", "  ")
+	case files.XML:
+		data, err = xml.MarshalIndent(results, "", "  ")
+	case files.YAML:
+		data, err = yaml.Marshal(results)
+	case files.TEXT:
+		var outStrings []string
+		for _, r := range results {
+			outStrings = append(outStrings, r.String()+"\n\n")
+		}
+		data = []byte(strings.Join(outStrings, ""))
+	default:
+		return errors.New("unsupported file type")
+	}
+
+	if err != nil {
+		return err
+	}
+
+	filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String())
+	return os.WriteFile(filePath, data, 0644)
+}
+
+func WriteWhoIsRecordToFile(record sqlite.WhoisRecord, outputFile string, fileType files.FileType) error {
+	var data []byte
+	var err error
+
+	switch fileType {
+	case files.JSON:
+		data, err = json.MarshalIndent(record, "", "  ")
+	case files.XML:
+		data, err = xml.MarshalIndent(record, "", "  ")
+	case files.YAML:
+		data, err = yaml.Marshal(record)
+	case files.TEXT:
+		data = []byte(record.String())
+	default:
+		return errors.New("unsupported file type")
+	}
+
+	if err != nil {
+		return err
+	}
+
+	filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String())
+	return os.WriteFile(filePath, data, 0644)
+}
+
+func WriteSubdomainsToFile(records []sqlite.SubdomainRecord, outputFile string, fileType files.FileType) error {
+	var data []byte
+	var err error
+
+	switch fileType {
+	case files.JSON:
+		data, err = json.MarshalIndent(records, "", "  ")
+	case files.XML:
+		data, err = xml.MarshalIndent(records, "", "  ")
+	case files.YAML:
+		data, err = yaml.Marshal(records)
+	case files.TEXT:
+		var outStrings []string
+		for _, r := range records {
+			out := fmt.Sprintf(
+				"Domain: %s\nFirst Seen: %s\nLast Seen: %s\n\n",
+				r.Domain, time.Unix(r.FirstSeen, 0).String(), time.Unix(r.LastSeen, 0).String())
+			outStrings = append(outStrings, out)
+		}
+		data = []byte(strings.Join(outStrings, ""))
+	default:
+		return errors.New("unsupported file type")
+	}
+
+	if err != nil {
+		return err
+	}
+
+	filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String())
+	return os.WriteFile(filePath, data, 0644)
+}
+
+func WriteIPLookupToFile(records []sqlite.LookupResult, outputFile string, fileType files.FileType) error {
+	var data []byte
+	var err error
+
+	switch fileType {
+	case files.JSON:
+		data, err = json.MarshalIndent(records, "", "  ")
+	case files.XML:
+		data, err = xml.MarshalIndent(records, "", "  ")
+	case files.YAML:
+		data, err = yaml.Marshal(records)
+	case files.TEXT:
+		var outStrings []string
+		for _, r := range records {
+			out := fmt.Sprintf(
+				"Name: %s\nSearch Term: %s\nFirst Seen: %s\nLast Visit: %s\nType: %s\n\n",
+				r.Name, r.SearchTerm, time.Unix(r.FirstSeen, 0).String(), time.Unix(r.LastVisit, 0).String(), r.Type)
+			outStrings = append(outStrings, out)
+		}
+		data = []byte(strings.Join(outStrings, ""))
+	default:
+		return errors.New("unsupported file type")
+	}
+
+	if err != nil {
+		return err
+	}
+
+	filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String())
+	return os.WriteFile(filePath, data, 0644)
+}

internal/export/dehashed_test.go

@@ -0,0 +1,29 @@
+package export
+
+import (
+	"strings"
+	"testing"
+
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/sqlite"
+)
+
+func TestDehashedResultGreppableUsesSpaceSeparatedNonEmptyTokens(t *testing.T) {
+	got := dehashedResultGreppable(sqlite.Result{
+		DehashedId: "123",
+		Name:       []string{"Hargrave Mall"},
+		Address:    []string{"irving tx"},
+		Url:        []string{"gdt.com", "GDT.COM"},
+	})
+
+	if strings.Contains(got, "\t") {
+		t.Fatalf("greppable output contains tab: %q", got)
+	}
+	if strings.Contains(got, "vin=") {
+		t.Fatalf("greppable output contains empty field: %q", got)
+	}
+	for _, want := range []string{"id=123", "name=Hargrave_Mall", "address=irving_tx", "url=gdt.com,GDT.COM"} {
+		if !strings.Contains(got, want) {
+			t.Fatalf("greppable output = %q, want token %q", got, want)
+		}
+	}
+}

internal/export/export.go

@@ -1,133 +0,0 @@
-package export
-
-import (
-	"dehasher/internal/files"
-	"dehasher/internal/sqlite"
-	"encoding/json"
-	"encoding/xml"
-	"errors"
-	"fmt"
-	"gopkg.in/yaml.v3"
-	"io/ioutil"
-	"os"
-	"strings"
-)
-
-func WriteCredsToFile(creds []sqlite.Creds, outputFile string, fileType files.FileType) error {
-	var data []byte
-	var err error
-
-	switch fileType {
-	case files.JSON:
-		data, err = json.MarshalIndent(creds, "", "  ")
-	case files.XML:
-		data, err = xml.MarshalIndent(creds, "", "  ")
-	case files.YAML:
-		data, err = yaml.Marshal(creds)
-	case files.TEXT:
-		var outStrings []string
-		for _, c := range creds {
-			outStrings = append(outStrings, c.ToString()+"\n")
-		}
-		data = []byte(strings.Join(outStrings, ""))
-	default:
-		return errors.New("unsupported file type")
-	}
-
-	if err != nil {
-		return err
-	}
-
-	filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String())
-	return os.WriteFile(filePath, data, 0644)
-}
-
-func WriteToFile(results sqlite.DehashedResults, outputFile string, fileType files.FileType) error {
-	var data []byte
-	var err error
-
-	result := results.Results
-
-	switch fileType {
-	case files.JSON:
-		data, err = json.MarshalIndent(result, "", "  ")
-	case files.XML:
-		data, err = xml.MarshalIndent(result, "", "  ")
-	case files.YAML:
-		data, err = yaml.Marshal(result)
-	case files.TEXT:
-		var outStrings []string
-		for _, r := range result {
-			out := fmt.Sprintf(
-				"Id: %s\nEmail: %s\nIpAddress: %s\nUsername: %s\nPassword: %s\nHashedPassword: %s\nHashType: %s\nName: %s\nVin: %s\nAddress: %s\nPhone: %s\nDatabaseName: %s\n\n",
-				r.DehashedId, r.Email, r.IpAddress, r.Username, r.Password, r.HashedPassword, r.HashType, r.Name, r.Vin, r.Address, r.Phone, r.DatabaseName)
-			outStrings = append(outStrings, out)
-		}
-		data = []byte(strings.Join(outStrings, ""))
-	default:
-		return errors.New("unsupported file type")
-	}
-
-	if err != nil {
-		return err
-	}
-
-	filePath := fmt.Sprintf("%s.%s", outputFile, fileType)
-	return ioutil.WriteFile(filePath, data, 0644)
-}
-
-// WriteQueryResultsToFile writes query results to a file in the specified format
-func WriteQueryResultsToFile(results []map[string]interface{}, outputFile string, fileType files.FileType) error {
-	var data []byte
-	var err error
-
-	switch fileType {
-	case files.JSON:
-		data, err = json.MarshalIndent(results, "", "  ")
-	case files.XML:
-		data, err = xml.MarshalIndent(results, "", "  ")
-	case files.YAML:
-		data, err = yaml.Marshal(results)
-	case files.TEXT:
-		var outStrings []string
-		for _, r := range results {
-			var rowStrings []string
-			for k, v := range r {
-				// Format the value to avoid array notation
-				var valueStr string
-				switch val := v.(type) {
-				case []string:
-					valueStr = strings.Join(val, ", ")
-				case []interface{}:
-					strSlice := make([]string, len(val))
-					for i, item := range val {
-						if item == nil {
-							strSlice[i] = ""
-						} else {
-							strSlice[i] = fmt.Sprintf("%v", item)
-						}
-					}
-					valueStr = strings.Join(strSlice, ", ")
-				default:
-					if v == nil {
-						valueStr = ""
-					} else {
-						valueStr = fmt.Sprintf("%v", v)
-					}
-				}
-				rowStrings = append(rowStrings, fmt.Sprintf("%s: %s", k, valueStr))
-			}
-			outStrings = append(outStrings, strings.Join(rowStrings, "\n")+"\n\n")
-		}
-		data = []byte(strings.Join(outStrings, ""))
-	default:
-		return errors.New("unsupported file type")
-	}
-
-	if err != nil {
-		return err
-	}
-
-	filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String())
-	return os.WriteFile(filePath, data, 0644)
-}

internal/export/iString.go

@@ -0,0 +1,37 @@
+package export
+
+import (
+	"encoding/json"
+	"encoding/xml"
+	"fmt"
+	"os"
+
+	"gopkg.in/yaml.v3"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/files"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/sqlite"
+)
+
+func WriteIStringToFile(iString sqlite.IString, outputFile string, fileType files.FileType) error {
+	var data []byte
+	var err error
+
+	switch fileType {
+	case files.JSON:
+		data, err = json.MarshalIndent(iString, "", "  ")
+	case files.XML:
+		data, err = xml.MarshalIndent(iString, "", "  ")
+	case files.YAML:
+		data, err = yaml.Marshal(iString)
+	case files.TEXT:
+		data = []byte(iString.String())
+	default:
+		return err
+	}
+
+	if err != nil {
+		return err
+	}
+
+	filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String())
+	return os.WriteFile(filePath, data, 0644)
+}

internal/files/filetype.go

@@ -1,5 +1,7 @@
 package files
 
+import "strings"
+
 type FileType int32
 
 const (
@@ -7,18 +9,22 @@ const (
 	XML
 	YAML
 	TEXT
+	GREPPABLE
+	UNKNOWN
 )
 
 func GetFileType(filetype string) FileType {
-	switch filetype {
+	switch strings.ToLower(strings.TrimSpace(filetype)) {
 	case "json":
 		return JSON
 	case "xml":
 		return XML
 	case "yaml":
 		return YAML
-	case "txt":
+	case "txt", "text":
 		return TEXT
+	case "grep", "greppable":
+		return GREPPABLE
 	default:
 		return JSON
 	}
@@ -34,6 +40,8 @@ func (ft FileType) String() string {
 		return "yaml"
 	case TEXT:
 		return "txt"
+	case GREPPABLE:
+		return "grep"
 	default:
 		return "json"
 	}

internal/hunter.io/hunter.io.go

@@ -0,0 +1,697 @@
+package hunter_io
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+
+	"go.uber.org/zap"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/debug"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/sqlite"
+)
+
+const (
+	DOMAIN_SEARCH       = "https://api.hunter.io/v2/domain-search?domain={{domain}}&api_key={{apikey}}"
+	EMAIL_FINDER        = "https://api.hunter.io/v2/email-finder?domain={{domain}}&first_name={{first_name}}&last_name={{last_name}}&api_key={{apikey}}"
+	EMAIL_VERIFICATION  = "https://api.hunter.io/v2/email-verifier?email={{email}}&api_key={{apikey}}"
+	COMPANY_ENRICHMENT  = "https://api.hunter.io/v2/companies/find?domain={{domain}}&api_key={{apikey}}"
+	PERSON_ENRICHMENT   = "https://api.hunter.io/v2/people/find?email={{email}}&api_key={{apikey}}"
+	COMBINED_ENRICHMENT = "https://api.hunter.io/v2/combined/find?email={{email}}&api_key={{apikey}}"
+)
+
+type HunterIO struct {
+	apiKey string
+	debug  bool
+}
+
+func NewHunterIO(apiKey string, debugEnabled bool) *HunterIO {
+	return &HunterIO{apiKey: apiKey, debug: debugEnabled}
+}
+
+func (h *HunterIO) DomainSearch(domain string) (sqlite.HunterDomainData, error) {
+	var hunterDomainData sqlite.HunterDomainData
+
+	if h.debug {
+		debug.PrintInfo("performing domain search")
+		zap.L().Info("hunter_domain_search_debug",
+			zap.String("message", "performing domain search"),
+		)
+	}
+
+	url := DOMAIN_SEARCH
+	url = strings.Replace(url, "{{domain}}", domain, -1)
+	url = strings.Replace(url, "{{apikey}}", h.apiKey, -1)
+
+	if h.debug {
+		debug.PrintInfo("performing request")
+		debug.PrintInfo(fmt.Sprintf("URL: %s\n", url))
+		zap.L().Info("hunter_domain_search_debug",
+			zap.String("message", "performing request"),
+			zap.String("url", url),
+		)
+	}
+
+	resp, err := http.Get(url)
+	if err != nil {
+		if h.debug {
+			debug.PrintInfo("failed to perform request")
+			debug.PrintError(err)
+		}
+		zap.L().Error("hunter_domain_search",
+			zap.String("message", "failed to perform request"),
+			zap.Error(err),
+		)
+		return hunterDomainData, err
+	}
+	defer resp.Body.Close()
+
+	b, err := io.ReadAll(resp.Body)
+	if err != nil {
+		if h.debug {
+			debug.PrintInfo("failed to read response body")
+			debug.PrintError(err)
+		}
+		zap.L().Error("hunter_domain_search",
+			zap.String("message", "failed to read response body"),
+			zap.Error(err),
+		)
+		return hunterDomainData, err
+	}
+
+	if resp.StatusCode != 200 {
+		if h.debug {
+			debug.PrintInfo("received error status code")
+			debug.PrintJson(fmt.Sprintf("Status Code: %d\n", resp.StatusCode))
+			debug.PrintJson(fmt.Sprintf("Body: %s\n", string(b)))
+			zap.L().Info("hunter_domain_search_debug",
+				zap.String("message", "received error status code"),
+				zap.Int("status_code", resp.StatusCode),
+				zap.String("body_error", string(b)),
+			)
+
+		}
+		zap.L().Error("hunter_domain_search",
+			zap.String("message", "received error status code"),
+			zap.Int("status_code", resp.StatusCode),
+			zap.String("body_error", string(b)),
+		)
+		return hunterDomainData, fmt.Errorf("received error status code: %d", resp.StatusCode)
+	}
+
+	if h.debug {
+		debug.PrintInfo("unmarshalled response body")
+		debug.PrintJson(fmt.Sprintf("Body: %s\n", string(b)))
+		zap.L().Info("hunter_domain_search_debug",
+			zap.String("message", "unmarshalled response body"),
+			zap.String("body", string(b)),
+		)
+	}
+
+	var hunterDomainSearchResult sqlite.HunterDomainSearchResult
+	err = json.Unmarshal(b, &hunterDomainSearchResult)
+	if err != nil {
+		if h.debug {
+			debug.PrintInfo("failed to unmarshal response body")
+			debug.PrintError(err)
+		}
+		zap.L().Error("hunter_domain_search",
+			zap.String("message", "failed to unmarshal response body"),
+			zap.Error(err),
+		)
+		return hunterDomainData, err
+	}
+
+	hunterDomainData = hunterDomainSearchResult.Data
+
+	// Create a list of email object associated with the domain
+	var emails []sqlite.HunterEmail
+	for _, email := range hunterDomainData.Emails {
+		emails = append(emails, sqlite.HunterEmail{
+			Domain:       domain,
+			Value:        email.Value,
+			Type:         email.Type,
+			Confidence:   email.Confidence,
+			Sources:      email.Sources,
+			FirstName:    email.FirstName,
+			LastName:     email.LastName,
+			Position:     email.Position,
+			PositionRaw:  email.PositionRaw,
+			Seniority:    email.Seniority,
+			Department:   email.Department,
+			Linkedin:     email.Linkedin,
+			Twitter:      email.Twitter,
+			PhoneNumber:  email.PhoneNumber,
+			Verification: email.Verification,
+		})
+	}
+
+	err = sqlite.StoreHunterEmails(emails)
+	if err != nil {
+		if h.debug {
+			debug.PrintInfo("failed to store hunter emails")
+			debug.PrintError(err)
+		}
+		zap.L().Error("store_hunter_emails",
+			zap.String("message", "failed to store hunter emails"),
+			zap.Error(err),
+		)
+		return hunterDomainData, err
+	}
+
+	err = sqlite.StoreHunterDomain(hunterDomainData)
+	if err != nil {
+		if h.debug {
+			debug.PrintInfo("failed to store hunter domain")
+			debug.PrintError(err)
+		}
+		zap.L().Error("store_hunter_domain",
+			zap.String("message", "failed to store hunter domain"),
+			zap.Error(err),
+		)
+		return hunterDomainData, err
+	}
+
+	return hunterDomainData, nil
+}
+
+func (h *HunterIO) EmailFinder(domain, firstName, lastName string) (sqlite.HunterEmailFinderData, error) {
+	var hunterEmailFinderData sqlite.HunterEmailFinderData
+
+	if h.debug {
+		debug.PrintInfo("performing email find")
+		zap.L().Info("hunter_email_find_debug",
+			zap.String("message", "performing email find"),
+		)
+	}
+
+	url := EMAIL_FINDER
+	url = strings.Replace(url, "{{domain}}", domain, -1)
+	url = strings.Replace(url, "{{first_name}}", firstName, -1)
+	url = strings.Replace(url, "{{last_name}}", lastName, -1)
+	url = strings.Replace(url, "{{apikey}}", h.apiKey, -1)
+
+	if h.debug {
+		debug.PrintInfo("performing request")
+		debug.PrintInfo(fmt.Sprintf("URL: %s\n", url))
+		zap.L().Info("hunter_email_find_debug",
+			zap.String("message", "performing request"),
+			zap.String("url", url),
+		)
+	}
+
+	resp, err := http.Get(url)
+	if err != nil {
+		if h.debug {
+			debug.PrintInfo("failed to perform request")
+			debug.PrintError(err)
+		}
+		zap.L().Error("hunter_email_find",
+			zap.String("message", "failed to perform request"),
+			zap.Error(err),
+		)
+		return hunterEmailFinderData, err
+	}
+	defer resp.Body.Close()
+
+	b, err := io.ReadAll(resp.Body)
+	if err != nil {
+		if h.debug {
+			debug.PrintInfo("failed to read response body")
+			debug.PrintError(err)
+		}
+		zap.L().Error("hunter_email_find",
+			zap.String("message", "failed to read response body"),
+			zap.Error(err),
+		)
+		return hunterEmailFinderData, err
+	}
+
+	if resp.StatusCode != 200 {
+		if h.debug {
+			debug.PrintInfo("received error status code")
+			debug.PrintJson(fmt.Sprintf("Status Code: %d\n", resp.StatusCode))
+			debug.PrintJson(fmt.Sprintf("Body: %s\n", string(b)))
+			zap.L().Info("hunter_email_find_debug",
+				zap.String("message", "received error status code"),
+				zap.Int("status_code", resp.StatusCode),
+				zap.String("body_error", string(b)),
+			)
+		}
+		zap.L().Error("hunter_email_find",
+			zap.String("message", "received error status code"),
+			zap.Int("status_code", resp.StatusCode),
+			zap.String("body_error", string(b)),
+		)
+		return hunterEmailFinderData, fmt.Errorf("received error status code: %d", resp.StatusCode)
+	}
+
+	if h.debug {
+		debug.PrintInfo("unmarshalled response body")
+		debug.PrintJson(fmt.Sprintf("Body: %s\n", string(b)))
+		zap.L().Info("hunter_email_find_debug",
+			zap.String("message", "unmarshalled response body"),
+			zap.String("body", string(b)),
+		)
+	}
+
+	var hunterEmailFinderResult sqlite.HunterEmailFinderResponse
+	err = json.Unmarshal(b, &hunterEmailFinderResult)
+	if err != nil {
+		if h.debug {
+			debug.PrintInfo("failed to unmarshal response body")
+			debug.PrintError(err)
+		}
+		zap.L().Error("hunter_email_find",
+			zap.String("message", "failed to unmarshal response body"),
+			zap.Error(err),
+		)
+		return hunterEmailFinderData, err
+	}
+
+	hunterEmailFinderData = hunterEmailFinderResult.Data
+
+	var hunterEmails []sqlite.HunterEmail
+
+	hunterEmails = append(hunterEmails, sqlite.HunterEmail{
+		Domain:       hunterEmailFinderData.Domain,
+		Value:        hunterEmailFinderData.Email,
+		Type:         "personal",
+		Confidence:   100,
+		Sources:      hunterEmailFinderData.Sources,
+		FirstName:    hunterEmailFinderData.FirstName,
+		LastName:     hunterEmailFinderData.LastName,
+		Position:     hunterEmailFinderData.Position,
+		PositionRaw:  "",
+		Seniority:    "",
+		Department:   "",
+		Linkedin:     hunterEmailFinderData.LinkedinURL,
+		Twitter:      hunterEmailFinderData.Twitter,
+		PhoneNumber:  hunterEmailFinderData.PhoneNumber,
+		Verification: hunterEmailFinderData.Verification,
+	})
+
+	err = sqlite.StoreHunterEmails(hunterEmails)
+	if err != nil {
+		if h.debug {
+			debug.PrintInfo("failed to store hunter email finder")
+			debug.PrintError(err)
+		}
+		zap.L().Error("store_hunter_email_finder",
+			zap.String("message", "failed to store hunter email finder"),
+			zap.Error(err),
+		)
+		return hunterEmailFinderData, err
+	}
+
+	return hunterEmailFinderData, nil
+}
+
+func (h *HunterIO) EmailVerification(email string) (sqlite.HunterEmailVerifyData, error) {
+	var hunterEmailVerifyData sqlite.HunterEmailVerifyData
+
+	if h.debug {
+		debug.PrintInfo("performing email verification")
+		zap.L().Info("hunter_email_verification_debug",
+			zap.String("message", "performing email verification"),
+		)
+	}
+
+	url := EMAIL_VERIFICATION
+	url = strings.Replace(url, "{{email}}", email, -1)
+	url = strings.Replace(url, "{{apikey}}", h.apiKey, -1)
+
+	if h.debug {
+		debug.PrintInfo("performing request")
+		debug.PrintInfo(fmt.Sprintf("URL: %s\n", url))
+		zap.L().Info("hunter_email_verification_debug",
+			zap.String("message", "performing request"),
+			zap.String("url", url),
+		)
+	}
+
+	resp, err := http.Get(url)
+	if err != nil {
+		if h.debug {
+			debug.PrintInfo("failed to perform request")
+			debug.PrintError(err)
+		}
+		zap.L().Error("hunter_email_verification",
+			zap.String("message", "failed to perform request"),
+			zap.Error(err),
+		)
+		return hunterEmailVerifyData, err
+	}
+	defer resp.Body.Close()
+
+	b, err := io.ReadAll(resp.Body)
+	if err != nil {
+		if h.debug {
+			debug.PrintInfo("failed to read response body")
+			debug.PrintError(err)
+		}
+		zap.L().Error("hunter_email_verification",
+			zap.String("message", "failed to read response body"),
+			zap.Error(err),
+		)
+		return hunterEmailVerifyData, err
+	}
+
+	if resp.StatusCode != 200 {
+		if h.debug {
+			debug.PrintInfo("received error status code")
+			debug.PrintJson(fmt.Sprintf("Status Code: %d\n", resp.StatusCode))
+			debug.PrintJson(fmt.Sprintf("Body: %s\n", string(b)))
+			zap.L().Info("hunter_email_verification_debug",
+				zap.String("message", "received error status code"),
+				zap.Int("status_code", resp.StatusCode),
+				zap.String("body_error", string(b)),
+			)
+		}
+		zap.L().Error("hunter_email_verification",
+			zap.String("message", "received error status code"),
+			zap.Int("status_code", resp.StatusCode),
+			zap.String("body_error", string(b)),
+		)
+		return hunterEmailVerifyData, fmt.Errorf("received error status code: %d", resp.StatusCode)
+	}
+
+	if h.debug {
+		debug.PrintInfo("unmarshalled response body")
+		debug.PrintJson(fmt.Sprintf("Body: %s\n", string(b)))
+		zap.L().Info("hunter_email_verification_debug",
+			zap.String("message", "unmarshalled response body"),
+			zap.String("body", string(b)),
+		)
+	}
+
+	var hunterEmailVerifyResult sqlite.HunterEmailVerifyResponse
+	err = json.Unmarshal(b, &hunterEmailVerifyResult)
+	if err != nil {
+		if h.debug {
+			debug.PrintInfo("failed to unmarshal response body")
+			debug.PrintError(err)
+		}
+		zap.L().Error("hunter_email_verification",
+			zap.String("message", "failed to unmarshal response body"),
+			zap.Error(err),
+		)
+		return hunterEmailVerifyData, err
+	}
+
+	hunterEmailVerifyData = hunterEmailVerifyResult.Data
+
+	return hunterEmailVerifyData, nil
+}
+
+func (h *HunterIO) CompanyEnrichment(domain string) (sqlite.CompanyData, error) {
+	var companyData sqlite.CompanyData
+
+	if h.debug {
+		debug.PrintInfo("performing company enrichment")
+		zap.L().Info("hunter_company_enrichment_debug",
+			zap.String("message", "performing company enrichment"),
+		)
+	}
+
+	url := COMPANY_ENRICHMENT
+	url = strings.Replace(url, "{{domain}}", domain, -1)
+	url = strings.Replace(url, "{{apikey}}", h.apiKey, -1)
+
+	if h.debug {
+		debug.PrintInfo("performing request")
+		debug.PrintInfo(fmt.Sprintf("URL: %s\n", url))
+		zap.L().Info("hunter_company_enrichment_debug",
+			zap.String("message", "performing request"),
+			zap.String("url", url),
+		)
+	}
+
+	resp, err := http.Get(url)
+	if err != nil {
+		if h.debug {
+			debug.PrintInfo("failed to perform request")
+			debug.PrintError(err)
+		}
+		zap.L().Error("hunter_company_enrichment",
+			zap.String("message", "failed to perform request"),
+			zap.Error(err),
+		)
+		return companyData, err
+	}
+	defer resp.Body.Close()
+
+	b, err := io.ReadAll(resp.Body)
+	if err != nil {
+		if h.debug {
+			debug.PrintInfo("failed to read response body")
+			debug.PrintError(err)
+		}
+		zap.L().Error("hunter_company_enrichment",
+			zap.String("message", "failed to read response body"),
+			zap.Error(err),
+		)
+		return companyData, err
+	}
+
+	if resp.StatusCode != 200 {
+		if h.debug {
+			debug.PrintInfo("received error status code")
+			debug.PrintJson(fmt.Sprintf("Status Code: %d\n", resp.StatusCode))
+			debug.PrintJson(fmt.Sprintf("Body: %s\n", string(b)))
+			zap.L().Info("hunter_company_enrichment_debug",
+				zap.String("message", "received error status code"),
+				zap.Int("status_code", resp.StatusCode),
+				zap.String("body_error", string(b)),
+			)
+		}
+		zap.L().Error("hunter_company_enrichment",
+			zap.String("message", "received error status code"),
+			zap.Int("status_code", resp.StatusCode),
+			zap.String("body_error", string(b)),
+		)
+		return companyData, fmt.Errorf("received error status code: %d", resp.StatusCode)
+	}
+
+	if h.debug {
+		debug.PrintInfo("unmarshalled response body")
+		debug.PrintJson(fmt.Sprintf("Body: %s\n", string(b)))
+		zap.L().Info("hunter_company_enrichment_debug",
+			zap.String("message", "unmarshalled response body"),
+			zap.String("body", string(b)),
+		)
+	}
+
+	var hunterCompanyEnrichmentResult sqlite.HunterCompanyEnrichmentResponse
+	err = json.Unmarshal(b, &hunterCompanyEnrichmentResult)
+	if err != nil {
+		if h.debug {
+			debug.PrintInfo("failed to unmarshal response body")
+			debug.PrintError(err)
+		}
+		zap.L().Error("hunter_company_enrichment",
+			zap.String("message", "failed to unmarshal response body"),
+			zap.Error(err),
+		)
+		return companyData, err
+	}
+
+	companyData = hunterCompanyEnrichmentResult.Data
+
+	return companyData, nil
+}
+
+func (h *HunterIO) PersonEnrichment(email string) (sqlite.PersonData, error) {
+	var personData sqlite.PersonData
+
+	if h.debug {
+		debug.PrintInfo("performing person enrichment")
+		zap.L().Info("hunter_person_enrichment_debug",
+			zap.String("message", "performing person enrichment"),
+		)
+	}
+
+	url := PERSON_ENRICHMENT
+	url = strings.Replace(url, "{{email}}", email, -1)
+	url = strings.Replace(url, "{{apikey}}", h.apiKey, -1)
+
+	if h.debug {
+		debug.PrintInfo("performing request")
+		debug.PrintInfo(fmt.Sprintf("URL: %s\n", url))
+		zap.L().Info("hunter_person_enrichment_debug",
+			zap.String("message", "performing request"),
+			zap.String("url", url),
+		)
+	}
+
+	resp, err := http.Get(url)
+	if err != nil {
+		if h.debug {
+			debug.PrintInfo("failed to perform request")
+			debug.PrintError(err)
+		}
+		zap.L().Error("hunter_person_enrichment",
+			zap.String("message", "failed to perform request"),
+			zap.Error(err),
+		)
+		return personData, err
+	}
+	defer resp.Body.Close()
+
+	b, err := io.ReadAll(resp.Body)
+	if err != nil {
+		if h.debug {
+			debug.PrintInfo("failed to read response body")
+			debug.PrintError(err)
+		}
+		zap.L().Error("hunter_person_enrichment",
+			zap.String("message", "failed to read response body"),
+			zap.Error(err),
+		)
+		return personData, err
+	}
+
+	if resp.StatusCode != 200 {
+		if h.debug {
+			debug.PrintInfo("received error status code")
+			debug.PrintJson(fmt.Sprintf("Status Code: %d\n", resp.StatusCode))
+			debug.PrintJson(fmt.Sprintf("Body: %s\n", string(b)))
+			zap.L().Info("hunter_person_enrichment_debug",
+				zap.String("message", "received error status code"),
+				zap.Int("status_code", resp.StatusCode),
+				zap.String("body_error", string(b)),
+			)
+		}
+		zap.L().Error("hunter_person_enrichment",
+			zap.String("message", "received error status code"),
+			zap.Int("status_code", resp.StatusCode),
+			zap.String("body_error", string(b)),
+		)
+		return personData, fmt.Errorf("received error status code: %d", resp.StatusCode)
+	}
+
+	if h.debug {
+		debug.PrintInfo("unmarshalled response body")
+		debug.PrintJson(fmt.Sprintf("Body: %s\n", string(b)))
+		zap.L().Info("hunter_person_enrichment_debug",
+			zap.String("message", "unmarshalled response body"),
+			zap.String("body", string(b)),
+		)
+	}
+
+	var hunterPersonEnrichmentResult sqlite.HunterPersonEnrichmentResponse
+	err = json.Unmarshal(b, &hunterPersonEnrichmentResult)
+	if err != nil {
+		if h.debug {
+			debug.PrintInfo("failed to unmarshal response body")
+			debug.PrintError(err)
+		}
+		zap.L().Error("hunter_person_enrichment",
+			zap.String("message", "failed to unmarshal response body"),
+			zap.Error(err),
+		)
+		return personData, err
+	}
+
+	personData = hunterPersonEnrichmentResult.Data
+
+	return personData, nil
+}
+
+func (h *HunterIO) CombinedEnrichment(email string) (sqlite.CombinedData, error) {
+	var combinedData sqlite.CombinedData
+
+	if h.debug {
+		debug.PrintInfo("performing combined enrichment")
+		zap.L().Info("hunter_combined_enrichment_debug",
+			zap.String("message", "performing combined enrichment"),
+		)
+	}
+
+	url := COMBINED_ENRICHMENT
+	url = strings.Replace(url, "{{email}}", email, -1)
+	url = strings.Replace(url, "{{apikey}}", h.apiKey, -1)
+
+	if h.debug {
+		debug.PrintInfo("performing request")
+		debug.PrintInfo(fmt.Sprintf("URL: %s\n", url))
+		zap.L().Info("hunter_combined_enrichment_debug",
+			zap.String("message", "performing request"),
+			zap.String("url", url),
+		)
+	}
+
+	resp, err := http.Get(url)
+	if err != nil {
+		if h.debug {
+			debug.PrintInfo("failed to perform request")
+			debug.PrintError(err)
+		}
+		zap.L().Error("hunter_combined_enrichment",
+			zap.String("message", "failed to perform request"),
+			zap.Error(err),
+		)
+		return combinedData, err
+	}
+	defer resp.Body.Close()
+
+	b, err := io.ReadAll(resp.Body)
+	if err != nil {
+		if h.debug {
+			debug.PrintInfo("failed to read response body")
+			debug.PrintError(err)
+		}
+		zap.L().Error("hunter_combined_enrichment",
+			zap.String("message", "failed to read response body"),
+			zap.Error(err),
+		)
+		return combinedData, err
+	}
+
+	if resp.StatusCode != 200 {
+		if h.debug {
+			debug.PrintInfo("received error status code")
+			debug.PrintJson(fmt.Sprintf("Status Code: %d\n", resp.StatusCode))
+			debug.PrintJson(fmt.Sprintf("Body: %s\n", string(b)))
+			zap.L().Info("hunter_combined_enrichment_debug",
+				zap.String("message", "received error status code"),
+				zap.Int("status_code", resp.StatusCode),
+				zap.String("body_error", string(b)),
+			)
+		}
+		zap.L().Error("hunter_combined_enrichment",
+			zap.String("message", "received error status code"),
+			zap.Int("status_code", resp.StatusCode),
+			zap.String("body_error", string(b)),
+		)
+		return combinedData, fmt.Errorf("received error status code: %d", resp.StatusCode)
+	}
+
+	if h.debug {
+		debug.PrintInfo("unmarshalled response body")
+		debug.PrintJson(fmt.Sprintf("Body: %s\n", string(b)))
+		zap.L().Info("hunter_combined_enrichment_debug",
+			zap.String("message", "unmarshalled response body"),
+			zap.String("body", string(b)),
+		)
+	}
+
+	var hunterCombinedEnrichmentResult sqlite.HunterCombinedEnrichmentResponse
+	err = json.Unmarshal(b, &hunterCombinedEnrichmentResult)
+	if err != nil {
+		if h.debug {
+			debug.PrintInfo("failed to unmarshal response body")
+			debug.PrintError(err)
+		}
+		zap.L().Error("hunter_combined_enrichment",
+			zap.String("message", "failed to unmarshal response body"),
+			zap.Error(err),
+		)
+		return combinedData, err
+	}
+
+	combinedData = hunterCombinedEnrichmentResult.Data
+
+	return combinedData, nil
+}

internal/pretty/trees.go

@@ -0,0 +1,346 @@
+package pretty
+
+import (
+	"fmt"
+
+	"github.com/charmbracelet/lipgloss"
+	"github.com/charmbracelet/lipgloss/tree"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/sqlite"
+)
+
+func WhoIsTree(root string, record sqlite.WhoisRecord) {
+	enumeratorStyle := lipgloss.NewStyle().Foreground(purple).MarginRight(1)
+	rootStyle := lipgloss.NewStyle().Foreground(lipgloss.Color("63"))
+	itemStyle := lipgloss.NewStyle().Foreground(gray)
+
+	rootTree := tree.Root(root)
+
+	// Child Trees
+	// Root Audit Tree
+	auditTree := tree.Root("Audit")
+	auditTree.Child(fmt.Sprintf("Created Date: %s", record.Audit.CreatedDate))
+	auditTree.Child(fmt.Sprintf("Updated Date: %s", record.Audit.UpdatedDate))
+	rootTree.Child(auditTree)
+
+	// Root Name Servers Tree
+	nameServersTree := tree.Root("Name Servers")
+	nameServersTree.Child("Host Names: " + fmt.Sprintf("%v", record.NameServers.HostNames))
+	nameServersTree.Child("IPs: " + fmt.Sprintf("%v", record.NameServers.IPs))
+	nameServersTree.Child("Raw Text: " + record.NameServers.RawText)
+
+	// Root Registry Data Tree
+	registryDataTree := tree.Root("Registry Data")
+	registryDataTree.Child("Audit: " + fmt.Sprintf("%v", record.RegistryData.Audit))
+	registryDataTree.Child("Created Date: " + record.RegistryData.CreatedDate)
+	registryDataTree.Child("Created Date Normalized: " + record.RegistryData.CreatedDateNormalized)
+	registryDataTree.Child("Domain Name: " + record.RegistryData.DomainName)
+	registryDataTree.Child("Expires Date: " + record.RegistryData.ExpiresDate)
+	registryDataTree.Child("Expires Date Normalized: " + record.RegistryData.ExpiresDateNormalized)
+	registryDataTree.Child("Footer: " + record.RegistryData.Footer)
+	registryDataTree.Child("Header: " + record.RegistryData.Header)
+
+	// Registry Data Name Servers Tree
+	registryNameServersTree := tree.Root("Name Servers")
+	registryNameServersTree.Child("Host Names: " + fmt.Sprintf("%v", record.RegistryData.NameServers.HostNames))
+	registryNameServersTree.Child("IPs: " + fmt.Sprintf("%v", record.RegistryData.NameServers.IPs))
+	registryNameServersTree.Child("Raw Text: " + record.RegistryData.NameServers.RawText)
+	registryDataTree.Child(registryNameServersTree)
+
+	// Root Registry Data Tree
+	registryDataTree.Child("Parse Code: " + fmt.Sprintf("%d", record.RegistryData.ParseCode))
+	registryDataTree.Child("Raw Text: " + record.RegistryData.RawText)
+	registryDataTree.Child("Registrar IANA ID: " + record.RegistryData.RegistrarIANAID)
+	registryDataTree.Child("Registrar Name: " + record.RegistryData.RegistrarName)
+	registryDataTree.Child("Status: " + record.RegistryData.Status)
+	registryDataTree.Child("Stripped Text: " + record.RegistryData.StrippedText)
+	registryDataTree.Child("Updated Date: " + record.RegistryData.UpdatedDate)
+	registryDataTree.Child("Updated Date Normalized: " + record.RegistryData.UpdatedDateNormalized)
+	registryDataTree.Child("Whois Server: " + record.RegistryData.WhoisServer)
+
+	// Root Contract Tree
+	technicalContactTree := tree.Root("Technical Contact")
+	technicalContactTree.Child("City: " + record.TechnicalContact.City)
+	technicalContactTree.Child("Country: " + record.TechnicalContact.Country)
+	technicalContactTree.Child("Country Code: " + record.TechnicalContact.CountryCode)
+	technicalContactTree.Child("Name: " + record.TechnicalContact.Name)
+	technicalContactTree.Child("Organization: " + record.TechnicalContact.Organization)
+	technicalContactTree.Child("Postal Code: " + record.TechnicalContact.PostalCode)
+	technicalContactTree.Child("Raw Text: " + record.TechnicalContact.RawText)
+	technicalContactTree.Child("State: " + record.TechnicalContact.State)
+	technicalContactTree.Child("Street 1: " + record.TechnicalContact.Street1)
+	technicalContactTree.Child("Telephone: " + record.TechnicalContact.Telephone)
+
+	// Root Tree Children
+	rootTree.Child("Contact HunterEmail: " + record.ContactEmail)
+	rootTree.Child("Created Date: " + record.CreatedDate)
+	rootTree.Child("Created Date Normalized: " + record.CreatedDateNormalized)
+	rootTree.Child("Domain Name: " + record.DomainName)
+	rootTree.Child("Domain Name Ext: " + record.DomainNameExt)
+	rootTree.Child("Estimated Domain Age: " + fmt.Sprintf("%d", record.EstimatedDomainAge))
+	rootTree.Child("Expires Date: " + record.ExpiresDate)
+	rootTree.Child("Expires Date Normalized: " + record.ExpiresDateNormalized)
+	rootTree.Child("Footer: " + record.Footer)
+	rootTree.Child("Header: " + record.Header)
+	rootTree.Child(nameServersTree)
+	rootTree.Child("Parse Code: " + fmt.Sprintf("%d", record.ParseCode))
+	rootTree.Child("Raw Text: " + record.RawText)
+	rootTree.Child("Registrant: " + fmt.Sprintf("%v", record.Registrant))
+	rootTree.Child("Registrar IANA ID: " + record.RegistrarIANAID)
+	rootTree.Child("Registrar Name: " + record.RegistrarName)
+	rootTree.Child(registryDataTree)
+	rootTree.Child("Status: " + record.Status)
+	rootTree.Child("Stripped Text: " + record.StrippedText)
+	rootTree.Child(technicalContactTree)
+	rootTree.Child("Updated Date: " + record.UpdatedDate)
+	rootTree.Child("Updated Date Normalized: " + record.UpdatedDateNormalized)
+
+	// Styles
+	rootTree.Enumerator(tree.RoundedEnumerator)
+	rootTree.EnumeratorStyle(enumeratorStyle)
+	rootTree.RootStyle(rootStyle)
+	rootTree.ItemStyle(itemStyle)
+
+	// Print Tree
+	fmt.Println(rootTree)
+}
+
+func HunterDomainTree(root string, record sqlite.HunterDomainData) {
+	enumeratorStyle := lipgloss.NewStyle().Foreground(purple).MarginRight(1)
+	rootStyle := lipgloss.NewStyle().Foreground(lipgloss.Color("63"))
+	itemStyle := lipgloss.NewStyle().Foreground(gray)
+
+	rootTree := tree.Root(root)
+
+	// Root Tree Children
+	rootTree.Child("Domain: " + record.Domain)
+	rootTree.Child("Disposable: " + fmt.Sprintf("%t", record.Disposable))
+	rootTree.Child("Webmail: " + fmt.Sprintf("%t", record.Webmail))
+	rootTree.Child("Accept All: " + fmt.Sprintf("%t", record.AcceptAll))
+	rootTree.Child("Pattern: " + record.Pattern)
+	rootTree.Child("Organization: " + record.Organization)
+	rootTree.Child("Description: " + record.Description)
+	rootTree.Child("Industry: " + record.Industry)
+	rootTree.Child("Twitter: " + record.Twitter)
+	rootTree.Child("Facebook: " + record.Facebook)
+	rootTree.Child("Linkedin: " + record.Linkedin)
+	rootTree.Child("Instagram: " + record.Instagram)
+	rootTree.Child("Youtube: " + record.Youtube)
+
+	techTree := tree.Root("Technologies")
+	for _, tech := range record.Technologies {
+		techTree.Child(tech)
+	}
+	rootTree.Child(techTree)
+
+	rootTree.Child("Country: " + record.Country)
+	rootTree.Child("State: " + record.State)
+	rootTree.Child("City: " + record.City)
+	rootTree.Child("Postal Code: " + record.PostalCode)
+	rootTree.Child("Street: " + record.Street)
+	rootTree.Child("Headcount: " + record.Headcount)
+	rootTree.Child("Company Type: " + record.CompanyType)
+
+	emailTree := tree.Root("Emails")
+	for _, email := range record.Emails {
+		emailTree.Child(email.ToTree())
+	}
+	rootTree.Child(emailTree)
+
+	linkedDomainTree := tree.Root("Linked Domains")
+	for _, domain := range record.LinkedDomains {
+		linkedDomainTree.Child(domain)
+	}
+	rootTree.Child(linkedDomainTree)
+
+	// Styles
+	rootTree.Enumerator(tree.RoundedEnumerator)
+	rootTree.EnumeratorStyle(enumeratorStyle)
+	rootTree.RootStyle(rootStyle)
+	rootTree.ItemStyle(itemStyle)
+
+	// Print Tree
+	fmt.Println(rootTree)
+}
+
+func HunterCompanyEnrichmentTree(root string, record sqlite.CompanyData) {
+	enumeratorStyle := lipgloss.NewStyle().Foreground(purple).MarginRight(1)
+	rootStyle := lipgloss.NewStyle().Foreground(lipgloss.Color("63"))
+	itemStyle := lipgloss.NewStyle().Foreground(gray)
+
+	rootTree := tree.Root(root)
+
+	// Root Tree Children
+	rootTree.Child("ID: " + record.ID)
+	rootTree.Child("Name: " + record.Name)
+	rootTree.Child("Legal Name: " + record.LegalName)
+	rootTree.Child("Domain: " + record.Domain)
+	rootTree.Child(record.DomainAliasesTree())
+	rootTree.Child(record.SiteTree())
+	rootTree.Child(record.CategoryTree())
+	rootTree.Child(record.TagsTree())
+	rootTree.Child("Description: " + record.Description)
+	rootTree.Child("Founded Year: " + fmt.Sprintf("%d", record.FoundedYear))
+	rootTree.Child("Location: " + record.Location)
+	rootTree.Child("Time Zone: " + record.TimeZone)
+	rootTree.Child("UTC Offset: " + fmt.Sprintf("%d", record.UTCOffset))
+	rootTree.Child(record.GeoTree())
+	rootTree.Child("Logo: " + record.Logo)
+	rootTree.Child(record.FacebookTree())
+	rootTree.Child(record.LinkedInTree())
+	rootTree.Child(record.TwitterTree())
+	rootTree.Child(record.CrunchbaseTree())
+	rootTree.Child(record.YouTubeTree())
+	rootTree.Child("Email Provider: " + record.EmailProvider)
+	rootTree.Child("Type: " + record.Type)
+	rootTree.Child("Ticker: " + record.Ticker)
+	rootTree.Child(record.IdentifiersTree())
+	rootTree.Child("Phone: " + record.Phone)
+	rootTree.Child(record.MetricsTree())
+	rootTree.Child("Indexed At: " + record.IndexedAt)
+	rootTree.Child(record.TechTree())
+	rootTree.Child(record.TechCategoriesTree())
+	rootTree.Child(record.ParentTree())
+	rootTree.Child(record.UltimateParentTree())
+
+	// Styles
+	rootTree.Enumerator(tree.RoundedEnumerator)
+	rootTree.EnumeratorStyle(enumeratorStyle)
+	rootTree.RootStyle(rootStyle)
+	rootTree.ItemStyle(itemStyle)
+
+	// Print Tree
+	fmt.Println(rootTree)
+}
+
+func HunterPersonEnrichmentTree(root string, record sqlite.PersonData) {
+	enumeratorStyle := lipgloss.NewStyle().Foreground(purple).MarginRight(1)
+	rootStyle := lipgloss.NewStyle().Foreground(lipgloss.Color("63"))
+	itemStyle := lipgloss.NewStyle().Foreground(gray)
+
+	rootTree := tree.Root(root)
+
+	// Root Tree Children
+	rootTree.Child("ID: " + record.ID)
+	rootTree.Child(record.NameTree())
+	rootTree.Child("Email: " + record.Email)
+	rootTree.Child("Location: " + record.Location)
+	rootTree.Child("Time Zone: " + record.TimeZone)
+	rootTree.Child("UTC Offset: " + fmt.Sprintf("%d", record.UTCOffset))
+	rootTree.Child(record.GeoTree())
+	rootTree.Child("Bio: " + record.Bio)
+	rootTree.Child("Site: " + record.Site)
+	rootTree.Child("Avatar: " + record.Avatar)
+	rootTree.Child(record.EmploymentTree())
+	rootTree.Child(record.FacebookTree())
+	rootTree.Child(record.GitHubTree())
+	rootTree.Child(record.TwitterTree())
+	rootTree.Child(record.LinkedInTree())
+	rootTree.Child(record.GooglePlusTree())
+	rootTree.Child(record.GravatarTree())
+	rootTree.Child("Fuzzy: " + fmt.Sprintf("%t", record.Fuzzy))
+	rootTree.Child("Email Provider: " + record.EmailProvider)
+	rootTree.Child("Indexed At: " + record.IndexedAt)
+	rootTree.Child("Phone: " + record.Phone)
+	rootTree.Child("Active At: " + record.ActiveAt)
+	rootTree.Child("Inactive At: " + record.InactiveAt)
+
+	// Styles
+	rootTree.Enumerator(tree.RoundedEnumerator)
+	rootTree.EnumeratorStyle(enumeratorStyle)
+	rootTree.RootStyle(rootStyle)
+	rootTree.ItemStyle(itemStyle)
+
+	// Print Tree
+	fmt.Println(rootTree)
+}
+
+func HunterCombinedEnrichmentTree(root string, record sqlite.CombinedData) {
+	enumeratorStyle := lipgloss.NewStyle().Foreground(purple).MarginRight(1)
+	rootStyle := lipgloss.NewStyle().Foreground(lipgloss.Color("63"))
+	itemStyle := lipgloss.NewStyle().Foreground(gray)
+
+	rootTree := tree.Root(root)
+
+	// Root Tree Children
+	rootTree.Child(personTree(record.Person))
+	rootTree.Child(companyTree(record.Company))
+
+	// Styles
+	rootTree.Enumerator(tree.RoundedEnumerator)
+	rootTree.EnumeratorStyle(enumeratorStyle)
+	rootTree.RootStyle(rootStyle)
+	rootTree.ItemStyle(itemStyle)
+
+	// Print Tree
+	fmt.Println(rootTree)
+}
+
+func companyTree(record sqlite.CompanyData) *tree.Tree {
+	companyTree := tree.Root("Company")
+
+	// Company Tree Children
+	companyTree.Child("ID: " + record.ID)
+	companyTree.Child("Name: " + record.Name)
+	companyTree.Child("Legal Name: " + record.LegalName)
+	companyTree.Child("Domain: " + record.Domain)
+	companyTree.Child(record.DomainAliasesTree())
+	companyTree.Child(record.SiteTree())
+	companyTree.Child(record.CategoryTree())
+	companyTree.Child(record.TagsTree())
+	companyTree.Child("Description: " + record.Description)
+	companyTree.Child("Founded Year: " + fmt.Sprintf("%d", record.FoundedYear))
+	companyTree.Child("Location: " + record.Location)
+	companyTree.Child("Time Zone: " + record.TimeZone)
+	companyTree.Child("UTC Offset: " + fmt.Sprintf("%d", record.UTCOffset))
+	companyTree.Child(record.GeoTree())
+	companyTree.Child("Logo: " + record.Logo)
+	companyTree.Child(record.FacebookTree())
+	companyTree.Child(record.LinkedInTree())
+	companyTree.Child(record.TwitterTree())
+	companyTree.Child(record.CrunchbaseTree())
+	companyTree.Child(record.YouTubeTree())
+	companyTree.Child("Email Provider: " + record.EmailProvider)
+	companyTree.Child("Type: " + record.Type)
+	companyTree.Child("Ticker: " + record.Ticker)
+	companyTree.Child(record.IdentifiersTree())
+	companyTree.Child("Phone: " + record.Phone)
+	companyTree.Child(record.MetricsTree())
+	companyTree.Child("Indexed At: " + record.IndexedAt)
+	companyTree.Child(record.TechTree())
+	companyTree.Child(record.TechCategoriesTree())
+	companyTree.Child(record.ParentTree())
+	companyTree.Child(record.UltimateParentTree())
+
+	return companyTree
+}
+
+func personTree(record sqlite.PersonData) *tree.Tree {
+	personTree := tree.Root("Person")
+
+	// Person Tree Children
+	personTree.Child("ID: " + record.ID)
+	personTree.Child(record.NameTree())
+	personTree.Child("Email: " + record.Email)
+	personTree.Child("Location: " + record.Location)
+	personTree.Child("Time Zone: " + record.TimeZone)
+	personTree.Child("UTC Offset: " + fmt.Sprintf("%d", record.UTCOffset))
+	personTree.Child(record.GeoTree())
+	personTree.Child("Bio: " + record.Bio)
+	personTree.Child("Site: " + record.Site)
+	personTree.Child("Avatar: " + record.Avatar)
+	personTree.Child(record.EmploymentTree())
+	personTree.Child(record.FacebookTree())
+	personTree.Child(record.GitHubTree())
+	personTree.Child(record.TwitterTree())
+	personTree.Child(record.LinkedInTree())
+	personTree.Child(record.GooglePlusTree())
+	personTree.Child(record.GravatarTree())
+	personTree.Child("Fuzzy: " + fmt.Sprintf("%t", record.Fuzzy))
+	personTree.Child("Email Provider: " + record.EmailProvider)
+	personTree.Child("Indexed At: " + record.IndexedAt)
+	personTree.Child("Phone: " + record.Phone)
+	personTree.Child("Active At: " + record.ActiveAt)
+	personTree.Child("Inactive At: " + record.InactiveAt)
+
+	return personTree
+}

internal/query/dehashed.go

@@ -1,219 +0,0 @@
-package query
-
-import (
-	"dehasher/internal/export"
-	"dehasher/internal/sqlite"
-	"encoding/json"
-	"fmt"
-	"go.uber.org/zap"
-	"os"
-)
-
-// Dehasher is a struct for querying the Dehashed API
-type Dehasher struct {
-	options  sqlite.QueryOptions
-	nextPage int
-	request  *DehashedSearchRequest
-	client   *DehashedClientV2
-}
-
-// NewDehasher creates a new Dehasher
-func NewDehasher(options *sqlite.QueryOptions) *Dehasher {
-	dh := &Dehasher{
-		options:  *options,
-		nextPage: options.StartingPage + 1,
-	}
-	dh.setQueries()
-	dh.request = NewDehashedSearchRequest(dh.options.StartingPage, dh.options.MaxRecords, dh.options.WildcardMatch, dh.options.RegexMatch, false)
-	dh.buildRequest()
-	return dh
-}
-
-// SetClientCredentials sets the client credentials for the dehasher
-func (dh *Dehasher) SetClientCredentials(key string) {
-	dh.client = NewDehashedClientV2(key)
-}
-
-func (dh *Dehasher) getNextPage() int {
-	nextPage := dh.nextPage
-	dh.nextPage += 1
-	return nextPage
-}
-
-// setQueries sets the number of queries to make based on the number of records and requests
-func (dh *Dehasher) setQueries() {
-	var numQueries int
-
-	switch {
-	case dh.options.MaxRequests == 0:
-		zap.L().Error("max requests cannot be zero")
-		fmt.Println("[!] Max Requests cannot be zero")
-		os.Exit(1)
-	case dh.options.MaxRecords <= 10000 || dh.options.MaxRequests == 1:
-		numQueries = 1
-		if dh.options.MaxRecords > 10000 {
-			dh.options.MaxRecords = 10000
-		}
-		zap.L().Info("max requests set to 1", zap.Int("max_records", dh.options.MaxRecords))
-	case dh.options.MaxRequests < 0 && dh.options.MaxRecords > 20000:
-		numQueries = 3
-		dh.options.MaxRecords = 10000
-		zap.L().Info("max requests set to 3", zap.Int("max_records", dh.options.MaxRecords))
-	case dh.options.MaxRequests < 0 && dh.options.MaxRecords > 10000:
-		numQueries = 2
-		dh.options.MaxRecords = 10000
-		zap.L().Info("max requests set to 2", zap.Int("max_records", dh.options.MaxRecords))
-	case dh.options.MaxRecords < 0 && dh.options.MaxRecords < 10000:
-		numQueries = 1
-		zap.L().Info("max requests set to 1", zap.Int("max_records", dh.options.MaxRecords))
-	case dh.options.MaxRequests == 2 && dh.options.MaxRecords > 20000:
-		numQueries = 2
-		dh.options.MaxRecords = 10000
-		zap.L().Info("max requests set to 2", zap.Int("max_records", dh.options.MaxRecords))
-	case dh.options.MaxRequests == 2 && dh.options.MaxRecords <= 10000:
-		numQueries = 1
-		zap.L().Info("max requests set to 1", zap.Int("max_records", dh.options.MaxRecords))
-	default:
-		numQueries = 3
-		dh.options.MaxRecords = 10000
-		zap.L().Info("max requests set to 3", zap.Int("max_records", dh.options.MaxRecords))
-	}
-
-	dh.options.MaxRequests = numQueries
-	fmt.Printf("Making %d Requests for %d Records (%d Total)\n", dh.options.MaxRequests, dh.options.MaxRecords, dh.options.MaxRequests*dh.options.MaxRecords)
-}
-
-// Start starts the querying process
-func (dh *Dehasher) Start() {
-	fmt.Printf("[*] Querying Dehashed API...\n")
-	for i := 0; i < dh.options.MaxRequests; i++ {
-		fmt.Printf("   [*] Performing Request...\n")
-		count, err := dh.client.Search(*dh.request)
-		if err != nil {
-			// Check if it's a DehashError
-			if dhErr, ok := err.(*DehashError); ok {
-				fmt.Printf("   [!] Dehashed API Error: %s (Code: %d)\n", dhErr.Message, dhErr.Code)
-				zap.L().Error("dehashed_api_error",
-					zap.String("message", dhErr.Message),
-					zap.Int("code", dhErr.Code),
-				)
-			} else {
-				fmt.Printf("   [!] Error performing request: %v\n", err)
-				zap.L().Error("request_error",
-					zap.String("message", "failed to perform request"),
-					zap.Error(err),
-				)
-			}
-			os.Exit(-1)
-		}
-
-		if count < dh.options.MaxRecords {
-			fmt.Printf("      [+] Retrieved %d records\n", count)
-			fmt.Printf("      [-] Not enough entries, ending queries\n")
-			break
-		} else {
-			fmt.Printf("      [+] Retrieved %d records\n", dh.options.MaxRecords)
-		}
-
-		dh.request.Page = dh.getNextPage()
-	}
-
-	dh.parseResults()
-}
-
-// buildRequest constructs the query map
-func (dh *Dehasher) buildRequest() {
-	if len(dh.options.UsernameQuery) > 0 {
-		dh.request.AddUsernameQuery(dh.options.UsernameQuery)
-	}
-	if len(dh.options.EmailQuery) > 0 {
-		dh.request.AddEmailQuery(dh.options.EmailQuery)
-	}
-	if len(dh.options.IpQuery) > 0 {
-		dh.request.AddIpAddressQuery(dh.options.IpQuery)
-	}
-	if len(dh.options.HashQuery) > 0 {
-		dh.request.AddHashedPasswordQuery(dh.options.HashQuery)
-	}
-	if len(dh.options.PassQuery) > 0 {
-		dh.request.AddPasswordQuery(dh.options.PassQuery)
-	}
-	if len(dh.options.NameQuery) > 0 {
-		dh.request.AddNameQuery(dh.options.NameQuery)
-	}
-	if len(dh.options.DomainQuery) > 0 {
-		dh.request.AddDomainQuery(dh.options.DomainQuery)
-	}
-	if len(dh.options.VinQuery) > 0 {
-		dh.request.AddVinQuery(dh.options.VinQuery)
-	}
-	if len(dh.options.LicensePlateQuery) > 0 {
-		dh.request.AddLicensePlateQuery(dh.options.LicensePlateQuery)
-	}
-	if len(dh.options.AddressQuery) > 0 {
-		dh.request.AddAddressQuery(dh.options.AddressQuery)
-	}
-	if len(dh.options.PhoneQuery) > 0 {
-		dh.request.AddPhoneQuery(dh.options.PhoneQuery)
-	}
-	if len(dh.options.SocialQuery) > 0 {
-		dh.request.AddSocialQuery(dh.options.SocialQuery)
-	}
-	if len(dh.options.CryptoAddressQuery) > 0 {
-		dh.request.AddCryptoAddressQuery(dh.options.CryptoAddressQuery)
-	}
-}
-
-// parseResults parses the results and writes them to a file
-func (dh *Dehasher) parseResults() {
-	var data []byte
-
-	zap.L().Info("extracting_credentials")
-	results := dh.client.GetResults()
-	creds := results.ExtractCredentials()
-	fmt.Printf("\n\t[+] Discovered %d Credentials", len(creds))
-	err := sqlite.StoreCreds(creds)
-	if err != nil {
-		zap.L().Error("store_creds",
-			zap.String("message", "failed to store creds"),
-			zap.Error(err),
-		)
-	}
-	zap.L().Info("creds_stored", zap.Int("count", len(creds)))
-
-	zap.L().Info("storing_results")
-	err = sqlite.StoreResults(results)
-	if err != nil {
-		zap.L().Error("store_results",
-			zap.String("message", "failed to store results"),
-			zap.Error(err),
-		)
-	}
-	zap.L().Info("results_stored", zap.Int("count", len(results.Results)))
-
-	if len(results.Results) > 0 {
-		fmt.Printf("\n\t[*] Writing entries to file: %s.%s", dh.options.OutputFile, dh.options.OutputFormat.String())
-		if !dh.options.CredsOnly {
-			err := export.WriteToFile(results, dh.options.OutputFile, dh.options.OutputFormat)
-			if err != nil {
-				fmt.Printf("\n[!] Error Writing to file: %v\n\tOutputting to terminal.", err)
-				data, err = json.MarshalIndent(results, "", "  ")
-				fmt.Println(string(data))
-				os.Exit(0)
-			} else {
-				fmt.Println("\n\t\t[*] Success\n")
-			}
-		} else {
-			creds := results.ExtractCredentials()
-			err := export.WriteCredsToFile(creds, dh.options.OutputFile, dh.options.OutputFormat)
-			if err != nil {
-				fmt.Printf("\n[!] Error Writing to file: %v\n\tOutputting to terminal.", err)
-				data, err = json.MarshalIndent(creds, "", "  ")
-				fmt.Println(string(data))
-				os.Exit(0)
-			} else {
-				fmt.Println("\n\t\t[*] Success\n")
-			}
-		}
-	}
-}

internal/sqlite/db.go

@@ -3,417 +3,141 @@ package sqlite
 import (
 	"fmt"
 	"go.uber.org/zap"
+	"os"
+	"path/filepath"
+	"strings"
+
+	sql "github.com/glebarez/sqlite"
 	"gorm.io/gorm"
-	"time"
+	"gorm.io/gorm/logger"
 )
 
-// QueryResults queries the database for results based on the provided options
-func QueryResults(options *DBOptions) ([]Result, error) {
-	db := GetDB()
-	var results []Result
-	query := db.Model(&Result{})
+var DB *gorm.DB
 
-	// Apply filters based on the provided options
-	query = applyFilters(query, options)
+// InitDB initializes the database connection
+func InitDB(dbPath string) (*gorm.DB, error) {
+	zap.L().Info("Initializing database", zap.String("path", dbPath))
 
-	// Apply limit
-	if options.Limit > 0 {
-		query = query.Limit(options.Limit)
-	}
+	// Check if the path is a file or directory
+	fileInfo, err := os.Stat(dbPath)
+	var finalDbPath string
 
-	// Execute the query
-	if err := query.Find(&results).Error; err != nil {
-		zap.L().Error("query_results",
-			zap.String("message", "failed to query results"),
-			zap.Error(err),
-		)
-		return nil, fmt.Errorf("failed to query results: %w", err)
-	}
-
-	return results, nil
-}
-
-// applyFilters applies filters to the query based on the provided options
-func applyFilters(query *gorm.DB, options *DBOptions) *gorm.DB {
-	// Helper function to apply filter based on exact match setting
-	applyFilter := func(field, value string) *gorm.DB {
-		if value == "" {
-			return query
+	// If path doesn't exist or is a directory
+	if os.IsNotExist(err) || (err == nil && fileInfo.IsDir()) {
+		// Treat as directory path
+		if err := os.MkdirAll(dbPath, 0755); err != nil {
+			zap.L().Error("Failed to create database directory", zap.Error(err))
+			return nil, fmt.Errorf("failed to create database directory: %w", err)
 		}
-
-		if options.ExactMatch {
-			return query.Where(field+" = ?", value)
-		} else {
-			return query.Where(field+" LIKE ?", "%"+value+"%")
+		finalDbPath = filepath.Join(dbPath, "crowsnest.sqlite")
+	} else {
+		// Treat as file path
+		// Ensure the directory exists
+		dir := filepath.Dir(dbPath)
+		if err := os.MkdirAll(dir, 0755); err != nil {
+			zap.L().Error("Failed to create parent directory for database", zap.Error(err))
+			return nil, fmt.Errorf("failed to create parent directory for database: %w", err)
 		}
+		finalDbPath = dbPath
 	}
 
-	// Apply filters for each field if provided
-	if options.Email != "" {
-		query = applyFilter("email", options.Email)
-	}
-
-	if options.Username != "" {
-		query = applyFilter("username", options.Username)
-	}
-
-	if options.IPAddress != "" {
-		query = applyFilter("ip_address", options.IPAddress)
-	}
-
-	if options.Password != "" {
-		query = applyFilter("password", options.Password)
-	}
-
-	if options.HashedPassword != "" {
-		query = applyFilter("hashed_password", options.HashedPassword)
-	}
-
-	if options.Name != "" {
-		query = applyFilter("name", options.Name)
-	}
-
-	if options.Vin != "" {
-		query = applyFilter("vin", options.Vin)
-	}
-
-	if options.LicensePlate != "" {
-		query = applyFilter("license_plate", options.LicensePlate)
-	}
-
-	if options.Address != "" {
-		query = applyFilter("address", options.Address)
-	}
-
-	if options.Phone != "" {
-		query = applyFilter("phone", options.Phone)
-	}
-
-	if options.Social != "" {
-		query = applyFilter("social", options.Social)
-	}
-
-	if options.CryptoCurrencyAddress != "" {
-		query = applyFilter("cryptocurrency_address", options.CryptoCurrencyAddress)
-	}
-
-	if options.Domain != "" {
-		query = applyFilter("url", options.Domain)
-	}
-
-	// Apply non-empty field filters
-	for _, field := range options.NonEmptyFields {
-		switch field {
-		case "username":
-			query = query.Where("JSON_ARRAY_LENGTH(username) > 0")
-		case "email":
-			query = query.Where("JSON_ARRAY_LENGTH(email) > 0")
-		case "ip_address", "ipaddress", "ip":
-			query = query.Where("JSON_ARRAY_LENGTH(ip_address) > 0")
-		case "password":
-			query = query.Where("JSON_ARRAY_LENGTH(password) > 0")
-		case "hashed_password", "hash":
-			query = query.Where("JSON_ARRAY_LENGTH(hashed_password) > 0")
-		case "name":
-			query = query.Where("JSON_ARRAY_LENGTH(name) > 0")
-		case "vin":
-			query = query.Where("JSON_ARRAY_LENGTH(vin) > 0")
-		case "license_plate", "license":
-			query = query.Where("JSON_ARRAY_LENGTH(license_plate) > 0")
-		case "address":
-			query = query.Where("JSON_ARRAY_LENGTH(address) > 0")
-		case "phone":
-			query = query.Where("JSON_ARRAY_LENGTH(phone) > 0")
-		case "social":
-			query = query.Where("JSON_ARRAY_LENGTH(social) > 0")
-		case "cryptocurrency_address", "crypto":
-			query = query.Where("JSON_ARRAY_LENGTH(cryptocurrency_address) > 0")
-		case "url", "domain":
-			query = query.Where("JSON_ARRAY_LENGTH(url) > 0")
-		}
-	}
-
-	return query
-}
-
-// GetResultsCount returns the count of results matching the provided options
-func GetResultsCount(options *DBOptions) (int64, error) {
-	db := GetDB()
-	var count int64
-	query := db.Model(&Result{})
-
-	// Apply filters based on the provided options
-	query = applyFilters(query, options)
-
-	// Count the results
-	if err := query.Count(&count).Error; err != nil {
-		zap.L().Error("get_results_count",
-			zap.String("message", "failed to count results"),
-			zap.Error(err),
-		)
-		return 0, fmt.Errorf("failed to count results: %w", err)
-	}
-
-	return count, nil
-}
-
-// QueryRuns queries the database for previous query runs (QueryOptions) based on the provided filters
-func QueryRuns(limit, lastXRuns int, startDate, endDate time.Time, containsQuery string) ([]QueryOptions, error) {
-	db := GetDB()
-	var runs []QueryOptions
-	query := db.Model(&QueryOptions{})
-
-	// Apply date range filter if provided
-	if lastXRuns > 0 {
-		query = query.Order("created_at DESC").Limit(lastXRuns)
-	} else if !startDate.IsZero() && !endDate.IsZero() {
-		query = query.Where("created_at BETWEEN ? AND ?", startDate, endDate)
-	} else if !startDate.IsZero() {
-		query = query.Where("created_at >= ?", startDate)
-	} else if !endDate.IsZero() {
-		query = query.Where("created_at <= ?", endDate)
-	}
-
-	// Apply query filter if provided
-	if containsQuery != "" {
-		// Search in all query fields
-		query = query.Where(
-			"username_query LIKE ? OR "+
-				"email_query LIKE ? OR "+
-				"ip_query LIKE ? OR "+
-				"pass_query LIKE ? OR "+
-				"hash_query LIKE ? OR "+
-				"name_query LIKE ? OR "+
-				"domain_query LIKE ? OR "+
-				"vin_query LIKE ? OR "+
-				"license_plate_query LIKE ? OR "+
-				"address_query LIKE ? OR "+
-				"phone_query LIKE ? OR "+
-				"social_query LIKE ? OR "+
-				"crypto_address_query LIKE ?",
-			"%"+containsQuery+"%", "%"+containsQuery+"%", "%"+containsQuery+"%",
-			"%"+containsQuery+"%", "%"+containsQuery+"%", "%"+containsQuery+"%",
-			"%"+containsQuery+"%", "%"+containsQuery+"%", "%"+containsQuery+"%",
-			"%"+containsQuery+"%", "%"+containsQuery+"%", "%"+containsQuery+"%",
-			"%"+containsQuery+"%",
-		)
-	}
-
-	// Apply limit
-	if limit > 0 {
-		query = query.Limit(limit)
-	}
-
-	// Order by most recent first
-	query = query.Order("created_at DESC")
-
-	// Execute the query
-	if err := query.Find(&runs).Error; err != nil {
-		zap.L().Error("query_runs",
-			zap.String("message", "failed to query runs"),
-			zap.Error(err),
-		)
-		return nil, fmt.Errorf("failed to query runs: %w", err)
-	}
-
-	return runs, nil
-}
-
-// GetRunsCount returns the count of runs matching the provided filters
-func GetRunsCount(lastXRuns int, startDate, endDate time.Time, containsQuery string) (int64, error) {
-	db := GetDB()
-	var count int64
-	query := db.Model(&QueryOptions{})
-
-	// Apply date range filter if provided
-	if lastXRuns > 0 {
-		query = query.Order("created_at DESC").Limit(lastXRuns)
-	} else if !startDate.IsZero() && !endDate.IsZero() {
-		query = query.Where("created_at BETWEEN ? AND ?", startDate, endDate)
-	} else if !startDate.IsZero() {
-		query = query.Where("created_at >= ?", startDate)
-	} else if !endDate.IsZero() {
-		query = query.Where("created_at <= ?", endDate)
-	}
-
-	// Apply query filter if provided
-	if containsQuery != "" {
-		// Search in all query fields
-		query = query.Where(
-			"username_query LIKE ? OR "+
-				"email_query LIKE ? OR "+
-				"ip_query LIKE ? OR "+
-				"pass_query LIKE ? OR "+
-				"hash_query LIKE ? OR "+
-				"name_query LIKE ? OR "+
-				"domain_query LIKE ? OR "+
-				"vin_query LIKE ? OR "+
-				"license_plate_query LIKE ? OR "+
-				"address_query LIKE ? OR "+
-				"phone_query LIKE ? OR "+
-				"social_query LIKE ? OR "+
-				"crypto_address_query LIKE ?",
-			"%"+containsQuery+"%", "%"+containsQuery+"%", "%"+containsQuery+"%",
-			"%"+containsQuery+"%", "%"+containsQuery+"%", "%"+containsQuery+"%",
-			"%"+containsQuery+"%", "%"+containsQuery+"%", "%"+containsQuery+"%",
-			"%"+containsQuery+"%", "%"+containsQuery+"%", "%"+containsQuery+"%",
-			"%"+containsQuery+"%",
-		)
-	}
-
-	// Count the results
-	if err := query.Count(&count).Error; err != nil {
-		zap.L().Error("get_runs_count",
-			zap.String("message", "failed to count runs"),
-			zap.Error(err),
-		)
-		return 0, fmt.Errorf("failed to count runs: %w", err)
-	}
-
-	return count, nil
-}
-
-// QueryCreds queries the database for credentials based on the provided filters
-func QueryCreds(options *DBOptions) ([]Creds, error) {
-	db := GetDB()
-	var creds []Creds
-	query := db.Model(&Creds{})
-
-	// Apply filters based on the provided options
-	if options.Username != "" {
-		if options.ExactMatch {
-			query = query.Where("username = ?", options.Username)
-		} else {
-			query = query.Where("username LIKE ?", "%"+options.Username+"%")
-		}
-	}
-
-	if options.Email != "" {
-		if options.ExactMatch {
-			query = query.Where("email = ?", options.Email)
-		} else {
-			query = query.Where("email LIKE ?", "%"+options.Email+"%")
-		}
-	}
-
-	if options.Password != "" {
-		if options.ExactMatch {
-			query = query.Where("password = ?", options.Password)
-		} else {
-			query = query.Where("password LIKE ?", "%"+options.Password+"%")
-		}
-	}
-
-	// Apply limit
-	if options.Limit > 0 {
-		query = query.Limit(options.Limit)
-	}
-
-	// Execute the query
-	if err := query.Find(&creds).Error; err != nil {
-		zap.L().Error("query_creds",
-			zap.String("message", "failed to query credentials"),
-			zap.Error(err),
-		)
-		return nil, fmt.Errorf("failed to query credentials: %w", err)
-	}
-
-	return creds, nil
-}
-
-// GetCredsCount returns the count of credentials matching the provided filters
-func GetCredsCount(options *DBOptions) (int64, error) {
-	db := GetDB()
-	var count int64
-	query := db.Model(&Creds{})
-
-	// Apply filters based on the provided options
-	if options.Username != "" {
-		if options.ExactMatch {
-			query = query.Where("username = ?", options.Username)
-		} else {
-			query = query.Where("username LIKE ?", "%"+options.Username+"%")
-		}
-	}
-
-	if options.Email != "" {
-		if options.ExactMatch {
-			query = query.Where("email = ?", options.Email)
-		} else {
-			query = query.Where("email LIKE ?", "%"+options.Email+"%")
-		}
-	}
-
-	if options.Password != "" {
-		if options.ExactMatch {
-			query = query.Where("password = ?", options.Password)
-		} else {
-			query = query.Where("password LIKE ?", "%"+options.Password+"%")
-		}
-	}
-
-	// Count the results
-	if err := query.Count(&count).Error; err != nil {
-		zap.L().Error("get_creds_count",
-			zap.String("message", "failed to count credentials"),
-			zap.Error(err),
-		)
-		return 0, fmt.Errorf("failed to count credentials: %w", err)
-	}
-
-	return count, nil
-}
-
-// ExecuteRawQuery executes a raw SQL query and returns the results as a slice of maps
-func ExecuteRawQuery(query string) ([]map[string]interface{}, error) {
-	db := GetDB()
-	rows, err := db.Raw(query).Rows()
+	zap.L().Info("Opening database", zap.String("finalPath", finalDbPath))
+	db, err := gorm.Open(sql.Open(finalDbPath), &gorm.Config{
+		Logger: logger.Default.LogMode(logger.Silent),
+	})
 	if err != nil {
-		zap.L().Error("raw_query",
-			zap.String("message", "failed to execute raw query"),
-			zap.Error(err),
-		)
-		return nil, fmt.Errorf("failed to execute raw query: %w", err)
+		zap.L().Error("Failed to connect to database", zap.Error(err))
+		return nil, fmt.Errorf("failed to connect to database: %w", err)
 	}
-	defer rows.Close()
 
-	columns, err := rows.Columns()
+	// Auto migrate your models
+	err = db.AutoMigrate(&Result{}, &User{}, &QueryOptions{}, &User{}, &WhoisRecord{}, &HistoryRecord{},
+		&LookupResult{}, &HunterDomainData{}, &HunterEmail{}, &PersonData{}, &Subdomain{})
 	if err != nil {
-		zap.L().Error("raw_query",
-			zap.String("message", "failed to get columns from raw query"),
-			zap.Error(err),
-		)
-		return nil, fmt.Errorf("failed to get columns from raw query: %w", err)
+		zap.L().Error("Failed to migrate database", zap.Error(err))
+		return nil, fmt.Errorf("failed to migrate database: %w", err)
 	}
 
-	var results []map[string]interface{}
-
-	for rows.Next() {
-		// Create a slice of interface{} to hold the values
-		values := make([]interface{}, len(columns))
-		pointers := make([]interface{}, len(columns))
-		for i := range values {
-			pointers[i] = &values[i]
-		}
-
-		// Scan the result into the pointers
-		if err := rows.Scan(pointers...); err != nil {
-			zap.L().Error("raw_query",
-				zap.String("message", "failed to scan row from raw query"),
-				zap.Error(err),
-			)
-			return nil, fmt.Errorf("failed to scan row from raw query: %w", err)
-		}
-
-		// Create a map for this row
-		rowMap := make(map[string]interface{})
-		for i, col := range columns {
-			val := values[i]
-			rowMap[col] = val
-		}
-
-		results = append(results, rowMap)
-	}
-
-	return results, nil
+	DB = db
+	return db, nil
+}
+
+// GetDB returns the database connection
+func GetDB() *gorm.DB {
+	if DB == nil {
+		zap.L().Error("database not initialized")
+		fmt.Println("sqlite database not initialized")
+		os.Exit(1)
+	}
+	return DB
+}
+
+type Table int64
+
+const (
+	ResultsTable Table = iota
+	RunsTable
+	CredsTable
+	WhoIsTable
+	SubdomainsTable
+	HistoryTable
+	LookupTable
+	HunterDomainTable
+	HunterEmailTable
+	PersonTable
+	UnknownTable
+)
+
+func GetTable(userInput string) Table {
+	switch strings.ToLower(userInput) {
+	case "dehashed", "results":
+		return ResultsTable
+	case "runs":
+		return RunsTable
+	case "users", "creds":
+		return CredsTable
+	case "whois":
+		return WhoIsTable
+	case "subdomains":
+		return SubdomainsTable
+	case "history":
+		return HistoryTable
+	case "lookup":
+		return LookupTable
+	case "hunter_domain":
+		return HunterDomainTable
+	case "hunter_email":
+		return HunterEmailTable
+	case "person":
+		return PersonTable
+	default:
+		return UnknownTable
+	}
+}
+
+func (t Table) Object() interface{} {
+	switch t {
+	case ResultsTable:
+		return Result{}
+	case RunsTable:
+		return QueryOptions{}
+	case CredsTable:
+		return User{}
+	case WhoIsTable:
+		return WhoisRecord{}
+	case SubdomainsTable:
+		return SubdomainRecord{}
+	case HistoryTable:
+		return HistoryRecord{}
+	case LookupTable:
+		return LookupResult{}
+	case HunterDomainTable:
+		return HunterDomainData{}
+	case HunterEmailTable:
+		return HunterEmail{}
+	case PersonTable:
+		return PersonData{}
+	default:
+		return nil
+	}
 }

internal/sqlite/dbOptions.go

@@ -0,0 +1,33 @@
+package sqlite
+
+type IString interface {
+	String() string
+}
+
+type DBOptions struct {
+	Username              string
+	Email                 string
+	IPAddress             string
+	Password              string
+	HashedPassword        string
+	Name                  string
+	Vin                   string
+	LicensePlate          string
+	Address               string
+	Phone                 string
+	Social                string
+	CryptoCurrencyAddress string
+	Domain                string
+	Limit                 int
+	ExactMatch            bool
+	NonEmptyFields        []string // Fields that should not be empty
+	DisplayFields         []string // Fields to display in output
+}
+
+func (o *DBOptions) Empty() bool {
+	return o.Username == "" && o.Email == "" && o.IPAddress == "" &&
+		o.Password == "" && o.HashedPassword == "" && o.Name == "" &&
+		o.Vin == "" && o.LicensePlate == "" && o.Address == "" &&
+		o.Phone == "" && o.Social == "" && o.CryptoCurrencyAddress == "" && o.Domain == "" &&
+		len(o.NonEmptyFields) == 0
+}

internal/sqlite/db_test.go

@@ -0,0 +1,18 @@
+package sqlite
+
+import "testing"
+
+func TestGetTableAcceptsDisplayedTableNames(t *testing.T) {
+	tests := map[string]Table{
+		"dehashed": ResultsTable,
+		"results":  ResultsTable,
+		"users":    CredsTable,
+		"creds":    CredsTable,
+	}
+
+	for input, want := range tests {
+		if got := GetTable(input); got != want {
+			t.Fatalf("GetTable(%q) = %v, want %v", input, got, want)
+		}
+	}
+}

internal/sqlite/dehashed.go

@@ -0,0 +1,203 @@
+package sqlite
+
+import (
+	"fmt"
+
+	"go.uber.org/zap"
+	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
+	"hub.krkn.tech/KrakenTech/crowsnest/internal/files"
+)
+
+type QueryOptions struct {
+	gorm.Model
+	MaxRecords         int            `json:"max_records"`
+	MaxRequests        int            `json:"max_requests"`
+	StartingPage       int            `json:"starting_page"`
+	OutputFormat       files.FileType `json:"output_format"`
+	OutputFile         string         `json:"output_file"`
+	RegexMatch         bool           `json:"regex_match"`
+	WildcardMatch      bool           `json:"wildcard_match"`
+	UsernameQuery      string         `json:"username_query"`
+	EmailQuery         string         `json:"email_query"`
+	IpQuery            string         `json:"ip_query"`
+	PassQuery          string         `json:"pass_query"`
+	HashQuery          string         `json:"hash_query"`
+	NameQuery          string         `json:"name_query"`
+	DomainQuery        string         `json:"domain_query"`
+	VinQuery           string         `json:"vin_query"`
+	LicensePlateQuery  string         `json:"license_plate_query"`
+	AddressQuery       string         `json:"address_query"`
+	PhoneQuery         string         `json:"phone_query"`
+	SocialQuery        string         `json:"social_query"`
+	CryptoAddressQuery string         `json:"crypto_address_query"`
+	PrintBalance       bool           `json:"print_balance"`
+	CredsOnly          bool           `json:"creds_only"`
+	Debug              bool           `json:"debug"`
+}
+
+func (QueryOptions) TableName() string {
+	return "query_options"
+}
+
+func NewQueryOptions(maxRecords, maxRequests, startingPage int, outputFormat, outputFile, usernameQuery, emailQuery, ipQuery, passQuery, hashQuery, nameQuery, domainQuery, vinQuery, licensePlateQuery, addressQuery, phoneQuery, socialQuery, cryptoAddressQuery string, regexMatch, wildcardMatch, printBalance, credsOnly, debug bool) *QueryOptions {
+	return &QueryOptions{
+		MaxRecords:         maxRecords,
+		MaxRequests:        maxRequests,
+		StartingPage:       startingPage,
+		OutputFormat:       files.GetFileType(outputFormat),
+		OutputFile:         outputFile,
+		PrintBalance:       printBalance,
+		CredsOnly:          credsOnly,
+		RegexMatch:         regexMatch,
+		WildcardMatch:      wildcardMatch,
+		UsernameQuery:      usernameQuery,
+		EmailQuery:         emailQuery,
+		IpQuery:            ipQuery,
+		PassQuery:          passQuery,
+		HashQuery:          hashQuery,
+		NameQuery:          nameQuery,
+		DomainQuery:        domainQuery,
+		VinQuery:           vinQuery,
+		LicensePlateQuery:  licensePlateQuery,
+		AddressQuery:       addressQuery,
+		PhoneQuery:         phoneQuery,
+		SocialQuery:        socialQuery,
+		CryptoAddressQuery: cryptoAddressQuery,
+		Debug:              debug,
+	}
+}
+
+type DehashedSearchRequest struct {
+	Page     int    `json:"page"`
+	Query    string `json:"query"`
+	Size     int    `json:"size"`
+	Wildcard bool   `json:"wildcard"`
+	Regex    bool   `json:"regex"`
+	DeDupe   bool   `json:"de_dupe"`
+}
+
+type DehashedResponse struct {
+	Balance      int      `json:"balance"`
+	Entries      []Result `json:"entries"`
+	Success      bool     `json:"success"`
+	Took         string   `json:"took"`
+	TotalResults int      `json:"total"`
+}
+
+type Result struct {
+	gorm.Model
+	DehashedId            string   `json:"id" xml:"id" yaml:"id" gorm:"uniqueIndex"`
+	Email                 []string `json:"email,omitempty" xml:"email,omitempty" yaml:"email,omitempty" gorm:"serializer:json"`
+	IpAddress             []string `json:"ip_address,omitempty" xml:"ip_address,omitempty" yaml:"ip_address,omitempty" gorm:"serializer:json"`
+	Username              []string `json:"username,omitempty" xml:"username,omitempty" yaml:"username,omitempty" gorm:"serializer:json"`
+	Password              []string `json:"password,omitempty" xml:"password,omitempty" yaml:"password,omitempty" gorm:"serializer:json"`
+	HashedPassword        []string `json:"hashed_password,omitempty" xml:"hashed_password,omitempty" yaml:"hashed_password,omitempty" gorm:"serializer:json"`
+	HashType              string   `json:"hash_type,omitempty" xml:"hash_type,omitempty" yaml:"hash_type,omitempty"`
+	Name                  []string `json:"name,omitempty" xml:"name,omitempty" yaml:"name,omitempty" gorm:"serializer:json"`
+	Vin                   []string `json:"vin,omitempty" xml:"vin,omitempty" yaml:"vin,omitempty" gorm:"serializer:json"`
+	LicensePlate          []string `json:"license_plate,omitempty" xml:"license_plate,omitempty" yaml:"license_plate,omitempty" gorm:"serializer:json"`
+	Url                   []string `json:"url,omitempty" xml:"url,omitempty" yaml:"url,omitempty" gorm:"serializer:json"`
+	Social                []string `json:"social,omitempty" xml:"social,omitempty" yaml:"social,omitempty" gorm:"serializer:json"`
+	CryptoCurrencyAddress []string `json:"cryptocurrency_address,omitempty" xml:"cryptocurrency_address,omitempty" yaml:"cryptocurrency_address,omitempty" gorm:"serializer:json"`
+	Address               []string `json:"address,omitempty" xml:"address,omitempty" yaml:"address,omitempty" gorm:"serializer:json"`
+	Phone                 []string `json:"phone,omitempty" xml:"phone,omitempty" yaml:"phone,omitempty" gorm:"serializer:json"`
+	Company               []string `json:"company,omitempty" xml:"company,omitempty" yaml:"company,omitempty" gorm:"serializer:json"`
+	DatabaseName          string   `json:"database_name,omitempty" xml:"database_name,omitempty" yaml:"database_name,omitempty"`
+}
+
+func (Result) TableName() string {
+	return "dehashed"
+}
+
+type DehashedResults struct {
+	Results []Result `json:"results"`
+}
+
+func (dr *DehashedResults) ExtractUsers() []User {
+	var creds []User
+
+	results := dr.Results
+
+	for _, r := range results {
+		if len(r.Password) > 0 {
+			// Get first email if available
+			email := ""
+			if len(r.Email) > 0 {
+				email = r.Email[0]
+			}
+
+			// Get first username if available
+			username := ""
+			if len(r.Username) > 0 {
+				username = r.Username[0]
+			}
+
+			// Get first password
+			password := r.Password[0]
+
+			cred := User{Email: email, Password: password, Username: username}
+			creds = append(creds, cred)
+		}
+	}
+
+	go func() {
+		err := StoreUsers(creds)
+		if err != nil {
+			zap.L().Error("store_creds",
+				zap.String("message", "failed to store creds"),
+				zap.Error(err),
+			)
+			fmt.Printf("Error Storing Results: %v", err)
+		}
+	}()
+
+	return creds
+}
+
+func (User) TableName() string {
+	return "creds"
+}
+
+func (c User) ToString() string {
+	return fmt.Sprintf("%s%s%s", c.Username, "%", c.Password)
+}
+
+func StoreDehashedResults(results DehashedResults) error {
+	if len(results.Results) == 0 {
+		return nil
+	}
+
+	zap.L().Info("Storing results", zap.Int("count", len(results.Results)))
+	db := GetDB()
+
+	// Use batch insert with conflict handling
+	const batchSize = 100
+	var lastErr error
+
+	// Extract the slice of results
+	resultSlice := results.Results
+
+	for i := 0; i < len(resultSlice); i += batchSize {
+		end := i + batchSize
+		if end > len(resultSlice) {
+			end = len(resultSlice)
+		}
+
+		batch := resultSlice[i:end]
+		// Use Clauses with OnConflict DoNothing to skip conflicts
+		err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
+		if err != nil {
+			zap.L().Warn("Error storing some results", zap.Error(err))
+			lastErr = err
+			// Continue with next batch despite error
+		}
+	}
+
+	return lastErr
+}
+
+func StoreDehashedQueryOptions(queryOptions *QueryOptions) error {
+	db := GetDB()
+	return db.Create(queryOptions).Error
+}

internal/sqlite/gorm.go

@@ -1,226 +0,0 @@
-package sqlite
-
-import (
-	"fmt"
-	"go.uber.org/zap"
-	"gorm.io/gorm/clause"
-	"os"
-	"path/filepath"
-
-	"gorm.io/driver/sqlite"
-	"gorm.io/gorm"
-	"gorm.io/gorm/logger"
-)
-
-var DB *gorm.DB
-
-// InitDB initializes the database connection
-func InitDB(dbPath string) (*gorm.DB, error) {
-	zap.L().Info("Initializing database", zap.String("path", dbPath))
-
-	// Check if the path is a file or directory
-	fileInfo, err := os.Stat(dbPath)
-	var finalDbPath string
-
-	// If path doesn't exist or is a directory
-	if os.IsNotExist(err) || (err == nil && fileInfo.IsDir()) {
-		// Treat as directory path
-		if err := os.MkdirAll(dbPath, 0755); err != nil {
-			zap.L().Error("Failed to create database directory", zap.Error(err))
-			return nil, fmt.Errorf("failed to create database directory: %w", err)
-		}
-		finalDbPath = filepath.Join(dbPath, "dehashed.sqlite")
-	} else {
-		// Treat as file path
-		// Ensure the directory exists
-		dir := filepath.Dir(dbPath)
-		if err := os.MkdirAll(dir, 0755); err != nil {
-			zap.L().Error("Failed to create parent directory for database", zap.Error(err))
-			return nil, fmt.Errorf("failed to create parent directory for database: %w", err)
-		}
-		finalDbPath = dbPath
-	}
-
-	zap.L().Info("Opening database", zap.String("finalPath", finalDbPath))
-	db, err := gorm.Open(sqlite.Open(finalDbPath), &gorm.Config{
-		Logger: logger.Default.LogMode(logger.Silent),
-	})
-	if err != nil {
-		zap.L().Error("Failed to connect to database", zap.Error(err))
-		return nil, fmt.Errorf("failed to connect to database: %w", err)
-	}
-
-	// Auto migrate your models
-	err = db.AutoMigrate(&Result{}, &Creds{}, QueryOptions{}, Creds{}, WhoisRecord{}, SubdomainRecord{}, HistoryRecord{})
-	if err != nil {
-		zap.L().Error("Failed to migrate database", zap.Error(err))
-		return nil, fmt.Errorf("failed to migrate database: %w", err)
-	}
-
-	DB = db
-	return db, nil
-}
-
-// GetDB returns the database connection
-func GetDB() *gorm.DB {
-	if DB == nil {
-		zap.L().Error("database not initialized")
-		fmt.Println("sqlite database not initialized")
-		os.Exit(1)
-	}
-	return DB
-}
-
-func StoreResults(results DehashedResults) error {
-	if len(results.Results) == 0 {
-		return nil
-	}
-
-	zap.L().Info("Storing results", zap.Int("count", len(results.Results)))
-	db := GetDB()
-
-	// Use batch insert with conflict handling
-	const batchSize = 100
-	var lastErr error
-
-	// Extract the slice of results
-	resultSlice := results.Results
-
-	for i := 0; i < len(resultSlice); i += batchSize {
-		end := i + batchSize
-		if end > len(resultSlice) {
-			end = len(resultSlice)
-		}
-
-		batch := resultSlice[i:end]
-		// Use Clauses with OnConflict DoNothing to skip conflicts
-		err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
-		if err != nil {
-			zap.L().Warn("Error storing some results", zap.Error(err))
-			lastErr = err
-			// Continue with next batch despite error
-		}
-	}
-
-	return lastErr
-}
-
-func StoreCreds(creds []Creds) error {
-	if len(creds) == 0 {
-		return nil
-	}
-
-	zap.L().Info("Storing credentials", zap.Int("count", len(creds)))
-	db := GetDB()
-
-	// Use batch insert with conflict handling
-	// This will insert records in batches and continue even if some fail
-	const batchSize = 100
-	var lastErr error
-
-	for i := 0; i < len(creds); i += batchSize {
-		end := i + batchSize
-		if end > len(creds) {
-			end = len(creds)
-		}
-
-		batch := creds[i:end]
-		// Use Clauses with OnConflict DoNothing to skip conflicts
-		err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
-		if err != nil {
-			zap.L().Warn("Error storing some credentials", zap.Error(err))
-			lastErr = err
-			// Continue with next batch despite error
-		}
-	}
-
-	return lastErr
-}
-
-func StoreQueryOptions(queryOptions *QueryOptions) error {
-	db := GetDB()
-	return db.Create(queryOptions).Error
-}
-
-func StoreWhoisRecord(whoisRecord WhoisRecord) error {
-	// Create a pointer to the record to make it addressable
-	recordPtr := &whoisRecord
-
-	zap.L().Info("Storing WHOIS record",
-		zap.String("domain", whoisRecord.DomainName))
-
-	db := GetDB()
-
-	// Use OnConflict clause to handle duplicates
-	err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(recordPtr).Error
-	if err != nil {
-		zap.L().Error("store_whois_record",
-			zap.String("message", "failed to store whois record"),
-			zap.Error(err))
-		return err
-	}
-
-	return nil
-}
-
-func StoreSubdomainRecord(subdomainRecords []SubdomainRecord) error {
-	if len(subdomainRecords) == 0 {
-		return nil
-	}
-
-	zap.L().Info("Storing subdomain records", zap.Int("count", len(subdomainRecords)))
-	db := GetDB()
-
-	// Use batch insert with conflict handling
-	const batchSize = 100
-	var lastErr error
-
-	for i := 0; i < len(subdomainRecords); i += batchSize {
-		end := i + batchSize
-		if end > len(subdomainRecords) {
-			end = len(subdomainRecords)
-		}
-
-		batch := subdomainRecords[i:end]
-		// Use Clauses with OnConflict DoNothing to skip conflicts
-		err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
-		if err != nil {
-			zap.L().Warn("Error storing some subdomain records", zap.Error(err))
-			lastErr = err
-			// Continue with next batch despite error
-		}
-	}
-
-	return lastErr
-}
-
-func StoreHistoryRecord(historyRecords []HistoryRecord) error {
-	if len(historyRecords) == 0 {
-		return nil
-	}
-
-	zap.L().Info("Storing history records", zap.Int("count", len(historyRecords)))
-	db := GetDB()
-
-	// Use batch insert with conflict handling
-	const batchSize = 100
-	var lastErr error
-
-	for i := 0; i < len(historyRecords); i += batchSize {
-		end := i + batchSize
-		if end > len(historyRecords) {
-			end = len(historyRecords)
-		}
-
-		batch := historyRecords[i:end]
-		// Use Clauses with OnConflict DoNothing to skip conflicts
-		err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
-		if err != nil {
-			zap.L().Warn("Error storing some history records", zap.Error(err))
-			lastErr = err
-			// Continue with next batch despite error
-		}
-	}
-
-	return lastErr
-}

internal/sqlite/hunter.io.go

@@ -0,0 +1,820 @@
+package sqlite
+
+import (
+	"fmt"
+	"github.com/charmbracelet/lipgloss/tree"
+	"go.uber.org/zap"
+	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
+)
+
+// HunterDomainSearchResult represents the response from Hunter.io domain search API
+type HunterDomainSearchResult struct {
+	Data HunterDomainData `json:"data" gorm:"embedded;embeddedPrefix:data_"`
+	Meta HunterMeta       `json:"meta" gorm:"embedded;embeddedPrefix:meta_"`
+}
+
+// HunterDomainData contains the main domain information
+type HunterDomainData struct {
+	gorm.Model
+	IString       `gorm:"-"`
+	Domain        string        `json:"domain" gorm:"unique"`
+	Disposable    bool          `json:"disposable"`
+	Webmail       bool          `json:"webmail"`
+	AcceptAll     bool          `json:"accept_all"`
+	Pattern       string        `json:"pattern"`
+	Organization  string        `json:"organization"`
+	Description   string        `json:"description"`
+	Industry      string        `json:"industry"`
+	Twitter       string        `json:"twitter"`
+	Facebook      string        `json:"facebook"`
+	Linkedin      string        `json:"linkedin"`
+	Instagram     string        `json:"instagram"`
+	Youtube       string        `json:"youtube"`
+	Technologies  []string      `json:"technologies" gorm:"serializer:json"`
+	Country       string        `json:"country"`
+	State         string        `json:"state"`
+	City          string        `json:"city"`
+	PostalCode    string        `json:"postal_code"`
+	Street        string        `json:"street"`
+	Headcount     string        `json:"headcount"`
+	CompanyType   string        `json:"company_type"`
+	Emails        []HunterEmail `json:"emails" gorm:"serializer:json"`
+	LinkedDomains []string      `json:"linked_domains" gorm:"serializer:json"`
+}
+
+func (h HunterDomainData) String() string {
+	return fmt.Sprintf("Domain: %s\nDisposable: %t\nWebmail: %t\nAcceptAll: %t\nPattern: %s\nOrganization: %s\nDescription: %s\nIndustry: %s\nTwitter: %s\nFacebook: %s\nLinkedin: %s\nInstagram: %s\nYoutube: %s\nTechnologies: %v\nCountry: %s\nState: %s\nCity: %s\nPostalCode: %s\nStreet: %s\nHeadcount: %s\nCompanyType: %s\nEmails: %v\nLinkedDomains: %v\n",
+		h.Domain, h.Disposable, h.Webmail, h.AcceptAll, h.Pattern, h.Organization, h.Description, h.Industry, h.Twitter, h.Facebook, h.Linkedin, h.Instagram, h.Youtube, h.Technologies, h.Country, h.State, h.City, h.PostalCode, h.Street, h.Headcount, h.CompanyType, h.Emails, h.LinkedDomains)
+}
+
+func (HunterDomainData) TableName() string {
+	return "hunter_domain"
+}
+
+// HunterEmail represents an email found for the domain
+type HunterEmail struct {
+	gorm.Model
+	Domain       string             `json:"domain,omitempty"`
+	Value        string             `json:"value" gorm:"unique"`
+	Type         string             `json:"type"`
+	Confidence   int                `json:"confidence"`
+	Sources      []HunterSource     `json:"sources" gorm:"serializer:json"`
+	FirstName    string             `json:"first_name"`
+	LastName     string             `json:"last_name"`
+	Position     string             `json:"position"`
+	PositionRaw  string             `json:"position_raw"`
+	Seniority    string             `json:"seniority"`
+	Department   string             `json:"department"`
+	Linkedin     string             `json:"linkedin"`
+	Twitter      string             `json:"twitter"`
+	PhoneNumber  string             `json:"phone_number"`
+	Verification HunterVerification `json:"verification" gorm:"embedded;embeddedPrefix:verification_"`
+}
+
+func (he *HunterEmail) ToTree() *tree.Tree {
+	emailTree := tree.Root(he.Value)
+	emailTree.Child("Type: " + he.Type)
+	emailTree.Child("Confidence: " + fmt.Sprintf("%d", he.Confidence))
+	emailTree.Child("FirstName: " + he.FirstName)
+	emailTree.Child("LastName: " + he.LastName)
+	emailTree.Child("Position: " + he.Position)
+	emailTree.Child("PositionRaw: " + he.PositionRaw)
+	emailTree.Child("Seniority: " + he.Seniority)
+	emailTree.Child("Department: " + he.Department)
+	emailTree.Child("Linkedin: " + he.Linkedin)
+	emailTree.Child("Twitter: " + he.Twitter)
+	emailTree.Child("PhoneNumber: " + he.PhoneNumber)
+	emailTree.Child(he.Verification.ToTree())
+	return emailTree
+}
+
+func (he *HunterEmail) String() string {
+	return fmt.Sprintf("Value: %s\nType: %s\nConfidence: %d\nSources: %v\nFirstName: %s\nLastName: %s\nPosition: %s\nPositionRaw: %s\nSeniority: %s\nDepartment: %s\nLinkedin: %s\nTwitter: %s\nPhoneNumber: %s\nVerification: %v\n",
+		he.Value, he.Type, he.Confidence, he.Sources, he.FirstName, he.LastName, he.Position, he.PositionRaw, he.Seniority, he.Department, he.Linkedin, he.Twitter, he.PhoneNumber, he.Verification)
+}
+
+func (HunterEmail) TableName() string {
+	return "hunter_email"
+}
+
+// HunterSource represents where an email was found
+type HunterSource struct {
+	Domain      string `json:"domain"`
+	URI         string `json:"uri"`
+	ExtractedOn string `json:"extracted_on"`
+	LastSeenOn  string `json:"last_seen_on"`
+	StillOnPage bool   `json:"still_on_page"`
+}
+
+// HunterVerification represents the verification status of an email
+type HunterVerification struct {
+	Date   string `json:"date"`
+	Status string `json:"status"`
+}
+
+func (hv *HunterVerification) ToTree() *tree.Tree {
+	verificationTree := tree.Root("Verification")
+	verificationTree.Child("Date: " + hv.Date)
+	verificationTree.Child("Status: " + hv.Status)
+	return verificationTree
+}
+
+// HunterMeta contains metadata about the API response
+type HunterMeta struct {
+	Results int                `json:"results"`
+	Limit   int                `json:"limit"`
+	Offset  int                `json:"offset"`
+	Params  HunterSearchParams `json:"params" gorm:"embedded;embeddedPrefix:params_"`
+}
+
+// HunterSearchParams contains the parameters used in the search
+type HunterSearchParams struct {
+	Domain     string `json:"domain"`
+	Company    string `json:"company"`
+	Type       string `json:"type"`
+	Seniority  string `json:"seniority"`
+	Department string `json:"department"`
+}
+
+// HunterEmailFinderResponse represents the response from Hunter.io email finder API
+type HunterEmailFinderResponse struct {
+	Data HunterEmailFinderData `json:"data" gorm:"embedded;embeddedPrefix:data_"`
+	Meta EmailFinderMeta       `json:"meta" gorm:"embedded;embeddedPrefix:meta_"`
+}
+
+// HunterEmailFinderData contains the main email information
+type HunterEmailFinderData struct {
+	IString
+	FirstName    string             `json:"first_name"`
+	LastName     string             `json:"last_name"`
+	Email        string             `json:"email"`
+	Score        int                `json:"score"`
+	Domain       string             `json:"domain"`
+	AcceptAll    bool               `json:"accept_all"`
+	Position     string             `json:"position"`
+	Twitter      string             `json:"twitter"`
+	LinkedinURL  string             `json:"linkedin_url"`
+	PhoneNumber  string             `json:"phone_number"`
+	Company      string             `json:"company"`
+	Sources      []HunterSource     `json:"sources" gorm:"serializer:json"`
+	Verification HunterVerification `json:"verification" gorm:"embedded;embeddedPrefix:verification_"`
+}
+
+func (he HunterEmailFinderData) String() string {
+	return fmt.Sprintf("FirstName: %s\nLastName: %s\nEmail: %s\nScore: %d\nDomain: %s\nAcceptAll: %t\nPosition: %s\nTwitter: %s\nLinkedinURL: %s\nPhoneNumber: %s\nCompany: %s\nSources: %v\nVerification: %v\n",
+		he.FirstName, he.LastName, he.Email, he.Score, he.Domain, he.AcceptAll, he.Position, he.Twitter, he.LinkedinURL, he.PhoneNumber, he.Company, he.Sources, he.Verification)
+}
+
+// EmailFinderMeta contains metadata about the API response
+type EmailFinderMeta struct {
+	Params EmailFinderParams `json:"params" gorm:"embedded;embeddedPrefix:params_"`
+}
+
+// EmailFinderParams contains the parameters used in the search
+type EmailFinderParams struct {
+	FirstName   string `json:"first_name"`
+	LastName    string `json:"last_name"`
+	FullName    string `json:"full_name"`
+	Domain      string `json:"domain"`
+	Company     string `json:"company"`
+	MaxDuration string `json:"max_duration"`
+}
+
+func (HunterEmailFinderResponse) TableName() string {
+	return "hunter_email_finder"
+}
+
+// HunterEmailVerifyResponse represents the response from Hunter.io email verification API
+type HunterEmailVerifyResponse struct {
+	Data HunterEmailVerifyData `json:"data" gorm:"embedded;embeddedPrefix:data_"`
+	Meta EmailVerifyMeta       `json:"meta" gorm:"embedded;embeddedPrefix:meta_"`
+}
+
+// HunterEmailVerifyData contains the email verification information
+type HunterEmailVerifyData struct {
+	IString
+	Status            string         `json:"status"`
+	Result            string         `json:"result"`
+	DeprecationNotice string         `json:"_deprecation_notice"`
+	Score             int            `json:"score"`
+	Email             string         `json:"email"`
+	Regexp            bool           `json:"regexp"`
+	Gibberish         bool           `json:"gibberish"`
+	Disposable        bool           `json:"disposable"`
+	Webmail           bool           `json:"webmail"`
+	MXRecords         bool           `json:"mx_records"`
+	SMTPServer        bool           `json:"smtp_server"`
+	SMTPCheck         bool           `json:"smtp_check"`
+	AcceptAll         bool           `json:"accept_all"`
+	Block             bool           `json:"block"`
+	Sources           []HunterSource `json:"sources" gorm:"serializer:json"`
+}
+
+func (ev HunterEmailVerifyData) String() string {
+	return fmt.Sprintf("Status: %s\nResult: %s\nDeprecationNotice: %s\nScore: %d\nEmail: %s\nRegexp: %t\nGibberish: %t\nDisposable: %t\nWebmail: %t\nMXRecords: %t\nSMTPServer: %t\nSMTPCheck: %t\nAcceptAll: %t\nBlock: %t\nSources: %v\n",
+		ev.Status, ev.Result, ev.DeprecationNotice, ev.Score, ev.Email, ev.Regexp, ev.Gibberish, ev.Disposable, ev.Webmail, ev.MXRecords, ev.SMTPServer, ev.SMTPCheck, ev.AcceptAll, ev.Block, ev.Sources)
+}
+
+// EmailVerifyMeta contains metadata about the API response
+type EmailVerifyMeta struct {
+	Params EmailVerifyParams `json:"params" gorm:"embedded;embeddedPrefix:params_"`
+}
+
+// EmailVerifyParams contains the parameters used in the verification
+type EmailVerifyParams struct {
+	Email string `json:"email"`
+}
+
+// HunterCompanyEnrichmentResponse represents the response from Hunter.io company enrichment API
+type HunterCompanyEnrichmentResponse struct {
+	Data CompanyData `json:"data" gorm:"embedded;embeddedPrefix:data_"`
+	Meta CompanyMeta `json:"meta" gorm:"embedded;embeddedPrefix:meta_"`
+}
+
+// CompanyData contains the detailed company information
+type CompanyData struct {
+	IString
+	ID             string        `json:"id"`
+	Name           string        `json:"name"`
+	LegalName      string        `json:"legalName"`
+	Domain         string        `json:"domain"`
+	DomainAliases  []string      `json:"domainAliases" gorm:"serializer:json"`
+	Site           CompanySite   `json:"site" gorm:"embedded;embeddedPrefix:site_"`
+	Category       Category      `json:"category" gorm:"embedded;embeddedPrefix:category_"`
+	Tags           []string      `json:"tags" gorm:"serializer:json"`
+	Description    string        `json:"description"`
+	FoundedYear    int           `json:"foundedYear"`
+	Location       string        `json:"location"`
+	TimeZone       string        `json:"timeZone"`
+	UTCOffset      int           `json:"utcOffset"`
+	Geo            Geography     `json:"geo" gorm:"embedded;embeddedPrefix:geo_"`
+	Logo           string        `json:"logo"`
+	Facebook       Facebook      `json:"facebook" gorm:"embedded;embeddedPrefix:facebook_"`
+	LinkedIn       LinkedIn      `json:"linkedin" gorm:"embedded;embeddedPrefix:linkedin_"`
+	Twitter        Twitter       `json:"twitter" gorm:"embedded;embeddedPrefix:twitter_"`
+	Crunchbase     Crunchbase    `json:"crunchbase" gorm:"embedded;embeddedPrefix:crunchbase_"`
+	YouTube        YouTube       `json:"youtube" gorm:"embedded;embeddedPrefix:youtube_"`
+	EmailProvider  string        `json:"emailProvider"`
+	Type           string        `json:"type"`
+	Ticker         string        `json:"ticker"`
+	Identifiers    Identifiers   `json:"identifiers" gorm:"embedded;embeddedPrefix:identifiers_"`
+	Phone          string        `json:"phone"`
+	Metrics        Metrics       `json:"metrics" gorm:"embedded;embeddedPrefix:metrics_"`
+	IndexedAt      string        `json:"indexedAt"`
+	Tech           []string      `json:"tech" gorm:"serializer:json"`
+	TechCategories []string      `json:"techCategories" gorm:"serializer:json"`
+	Parent         ParentCompany `json:"parent" gorm:"embedded;embeddedPrefix:parent_"`
+	UltimateParent ParentCompany `json:"ultimateParent" gorm:"embedded;embeddedPrefix:ultimate_parent_"`
+}
+
+func (cd CompanyData) String() string {
+	return fmt.Sprintf("ID: %s\nName: %s\nLegalName: %s\nDomain: %s\nDomainAliases: %v\nSite: %v\nCategory: %v\nTags: %v\nDescription: %s\nFoundedYear: %d\nLocation: %s\nTimeZone: %s\nUTCOffset: %d\nGeo: %v\nLogo: %s\nFacebook: %v\nLinkedIn: %v\nTwitter: %v\nCrunchbase: %v\nYouTube: %v\nEmailProvider: %s\nType: %s\nTicker: %s\nIdentifiers: %v\nPhone: %s\nMetrics: %v\nIndexedAt: %s\nTech: %v\nTechCategories: %v\nParent: %v\nUltimateParent: %v\n",
+		cd.ID, cd.Name, cd.LegalName, cd.Domain, cd.DomainAliases, cd.Site, cd.Category, cd.Tags, cd.Description, cd.FoundedYear, cd.Location, cd.TimeZone, cd.UTCOffset, cd.Geo, cd.Logo, cd.Facebook, cd.LinkedIn, cd.Twitter, cd.Crunchbase, cd.YouTube, cd.EmailProvider, cd.Type, cd.Ticker, cd.Identifiers, cd.Phone, cd.Metrics, cd.IndexedAt, cd.Tech, cd.TechCategories, cd.Parent, cd.UltimateParent)
+}
+
+func (cd *CompanyData) DomainAliasesTree() *tree.Tree {
+	domainAliasesTree := tree.Root("Domain Aliases")
+	for _, domainAlias := range cd.DomainAliases {
+		domainAliasesTree.Child(domainAlias)
+	}
+	return domainAliasesTree
+}
+
+func (cd *CompanyData) SiteTree() *tree.Tree {
+	siteTree := tree.Root("Site")
+	phoneTree := tree.Root("Phone Numbers")
+	for _, phoneNumber := range cd.Site.PhoneNumbers {
+		phoneTree.Child(phoneNumber)
+	}
+	emailTree := tree.Root("Email Addresses")
+	for _, emailAddress := range cd.Site.EmailAddresses {
+		emailTree.Child(emailAddress)
+	}
+	siteTree.Child(phoneTree)
+	siteTree.Child(emailTree)
+	return siteTree
+}
+
+func (cd *CompanyData) CategoryTree() *tree.Tree {
+	categoryTree := tree.Root("Category")
+	categoryTree.Child("Sector: " + cd.Category.Sector)
+	categoryTree.Child("Industry Group: " + cd.Category.IndustryGroup)
+	categoryTree.Child("Industry: " + cd.Category.Industry)
+	categoryTree.Child("Sub Industry: " + cd.Category.SubIndustry)
+	categoryTree.Child("GICS Code: " + cd.Category.GICSCode)
+	categoryTree.Child("SIC Code: " + cd.Category.SICCode)
+
+	sic4CodesTree := tree.Root("SIC 4 Codes")
+	for _, sic4Code := range cd.Category.SIC4Codes {
+		sic4CodesTree.Child(sic4Code)
+	}
+	categoryTree.Child(sic4CodesTree)
+
+	categoryTree.Child("NAICS Code: " + cd.Category.NAICSCode)
+
+	naics6CodesTree := tree.Root("NAICS 6 Codes")
+	for _, naics6Code := range cd.Category.NAICS6Codes {
+		naics6CodesTree.Child(naics6Code)
+	}
+	categoryTree.Child(naics6CodesTree)
+
+	naics6Codes2022Tree := tree.Root("NAICS 6 Codes 2022")
+	for _, naics6Code2022 := range cd.Category.NAICS6Codes2022 {
+		naics6Codes2022Tree.Child(naics6Code2022)
+	}
+	categoryTree.Child(naics6Codes2022Tree)
+	return categoryTree
+}
+
+func (cd *CompanyData) GeoTree() *tree.Tree {
+	geoTree := tree.Root("Geo")
+	geoTree.Child("Street Number: " + cd.Geo.StreetNumber)
+	geoTree.Child("Street Name: " + cd.Geo.StreetName)
+	geoTree.Child("Sub Premise: " + cd.Geo.SubPremise)
+	geoTree.Child("Street Address: " + cd.Geo.StreetAddress)
+	geoTree.Child("City: " + cd.Geo.City)
+	geoTree.Child("Postal Code: " + cd.Geo.PostalCode)
+	geoTree.Child("State: " + cd.Geo.State)
+	geoTree.Child("State Code: " + cd.Geo.StateCode)
+	geoTree.Child("Country: " + cd.Geo.Country)
+	geoTree.Child("Country Code: " + cd.Geo.CountryCode)
+	geoTree.Child("Latitude: " + fmt.Sprintf("%f", cd.Geo.Lat))
+	geoTree.Child("Longitude: " + fmt.Sprintf("%f", cd.Geo.Lng))
+	return geoTree
+}
+
+func (cd *CompanyData) FacebookTree() *tree.Tree {
+	facebookTree := tree.Root("Facebook")
+	facebookTree.Child("Handle: " + cd.Facebook.Handle)
+	facebookTree.Child("Likes: " + fmt.Sprintf("%d", cd.Facebook.Likes))
+	return facebookTree
+}
+
+func (cd *CompanyData) LinkedInTree() *tree.Tree {
+	linkedinTree := tree.Root("LinkedIn")
+	linkedinTree.Child("Handle: " + cd.LinkedIn.Handle)
+	return linkedinTree
+}
+
+func (cd *CompanyData) TwitterTree() *tree.Tree {
+	twitterTree := tree.Root("Twitter")
+	twitterTree.Child("Handle: " + cd.Twitter.Handle)
+	twitterTree.Child("ID: " + cd.Twitter.ID)
+	twitterTree.Child("Bio: " + cd.Twitter.Bio)
+	twitterTree.Child("Followers: " + fmt.Sprintf("%d", cd.Twitter.Followers))
+	twitterTree.Child("Following: " + fmt.Sprintf("%d", cd.Twitter.Following))
+	twitterTree.Child("Location: " + cd.Twitter.Location)
+	twitterTree.Child("Site: " + cd.Twitter.Site)
+	twitterTree.Child("Avatar" + cd.Twitter.Avatar)
+	return twitterTree
+}
+
+func (cd *CompanyData) CrunchbaseTree() *tree.Tree {
+	crunchbaseTree := tree.Root("Crunchbase")
+	crunchbaseTree.Child("Handle: " + cd.Crunchbase.Handle)
+	return crunchbaseTree
+}
+
+func (cd *CompanyData) YouTubeTree() *tree.Tree {
+	youtubeTree := tree.Root("YouTube")
+	youtubeTree.Child("Handle: " + cd.YouTube.Handle)
+	return youtubeTree
+}
+
+func (cd *CompanyData) IdentifiersTree() *tree.Tree {
+	identifiersTree := tree.Root("Identifiers")
+	identifiersTree.Child("UsEIN: " + cd.Identifiers.UsEIN)
+	return identifiersTree
+}
+
+func (cd *CompanyData) MetricsTree() *tree.Tree {
+	metricsTree := tree.Root("Metrics")
+	metricsTree.Child("Alexa Us Rank: " + fmt.Sprintf("%d", cd.Metrics.AlexaUsRank))
+	metricsTree.Child("Alexa Global Rank: " + fmt.Sprintf("%d", cd.Metrics.AlexaGlobalRank))
+	metricsTree.Child("Traffic Rank: " + cd.Metrics.TrafficRank)
+	metricsTree.Child("Employees: " + cd.Metrics.Employees)
+	metricsTree.Child("Market Cap: " + cd.Metrics.MarketCap)
+	metricsTree.Child("Raised: " + cd.Metrics.Raised)
+	metricsTree.Child("Annual Revenue: " + cd.Metrics.AnnualRevenue)
+	metricsTree.Child("Estimated Annual Revenue: " + cd.Metrics.EstimatedAnnualRevenue)
+	metricsTree.Child("Fiscal Year End: " + cd.Metrics.FiscalYearEnd)
+	return metricsTree
+}
+
+func (cd *CompanyData) TagsTree() *tree.Tree {
+	tagsTree := tree.Root("Tags")
+	for _, tag := range cd.Tags {
+		tagsTree.Child(tag)
+	}
+	return tagsTree
+}
+
+func (cd *CompanyData) TechTree() *tree.Tree {
+	techTree := tree.Root("Tech")
+	for _, tech := range cd.Tech {
+		techTree.Child(tech)
+	}
+	return techTree
+}
+
+func (cd *CompanyData) TechCategoriesTree() *tree.Tree {
+	techCategoriesTree := tree.Root("Tech Categories")
+	for _, techCategory := range cd.TechCategories {
+		techCategoriesTree.Child(techCategory)
+	}
+	return techCategoriesTree
+}
+
+func (cd *CompanyData) ParentTree() *tree.Tree {
+	parentTree := tree.Root("Parent")
+	parentTree.Child("Domain: " + cd.Parent.Domain)
+	return parentTree
+}
+
+func (cd *CompanyData) UltimateParentTree() *tree.Tree {
+	ultimateParentTree := tree.Root("Ultimate Parent")
+	ultimateParentTree.Child("Domain: " + cd.UltimateParent.Domain)
+	return ultimateParentTree
+}
+
+// CompanySite contains contact information from the company website
+type CompanySite struct {
+	PhoneNumbers   []string `json:"phoneNumbers" gorm:"serializer:json"`
+	EmailAddresses []string `json:"emailAddresses" gorm:"serializer:json"`
+}
+
+// Category contains industry classification information
+type Category struct {
+	Sector          string   `json:"sector"`
+	IndustryGroup   string   `json:"industryGroup"`
+	Industry        string   `json:"industry"`
+	SubIndustry     string   `json:"subIndustry"`
+	GICSCode        string   `json:"gicsCode"`
+	SICCode         string   `json:"sicCode"`
+	SIC4Codes       []string `json:"sic4Codes" gorm:"serializer:json"`
+	NAICSCode       string   `json:"naicsCode"`
+	NAICS6Codes     []string `json:"naics6Codes" gorm:"serializer:json"`
+	NAICS6Codes2022 []string `json:"naics6Codes2022" gorm:"serializer:json"`
+}
+
+// Geography contains location information
+type Geography struct {
+	StreetNumber  string  `json:"streetNumber"`
+	StreetName    string  `json:"streetName"`
+	SubPremise    string  `json:"subPremise"`
+	StreetAddress string  `json:"streetAddress"`
+	City          string  `json:"city"`
+	PostalCode    string  `json:"postalCode"`
+	State         string  `json:"state"`
+	StateCode     string  `json:"stateCode"`
+	Country       string  `json:"country"`
+	CountryCode   string  `json:"countryCode"`
+	Lat           float64 `json:"lat"`
+	Lng           float64 `json:"lng"`
+}
+
+// Identifiers contains company identification numbers
+type Identifiers struct {
+	UsEIN string `json:"usEIN"`
+}
+
+// Metrics contains company performance metrics
+type Metrics struct {
+	AlexaUsRank            int    `json:"alexaUsRank"`
+	AlexaGlobalRank        int    `json:"alexaGlobalRank"`
+	TrafficRank            string `json:"trafficRank"`
+	Employees              string `json:"employees"`
+	MarketCap              string `json:"marketCap"`
+	Raised                 string `json:"raised"`
+	AnnualRevenue          string `json:"annualRevenue"`
+	EstimatedAnnualRevenue string `json:"estimatedAnnualRevenue"`
+	FiscalYearEnd          string `json:"fiscalYearEnd"`
+}
+
+// ParentCompany contains information about parent companies
+type ParentCompany struct {
+	Domain string `json:"domain"`
+}
+
+// CompanyMeta contains metadata about the API response
+type CompanyMeta struct {
+	Domain string `json:"domain"`
+}
+
+func (HunterCompanyEnrichmentResponse) TableName() string {
+	return "hunter_company_enrichment"
+}
+
+// HunterPersonEnrichmentResponse represents the response from Hunter.io person enrichment API
+type HunterPersonEnrichmentResponse struct {
+	Data PersonData `json:"data" gorm:"embedded;embeddedPrefix:data_"`
+	Meta PersonMeta `json:"meta" gorm:"embedded;embeddedPrefix:meta_"`
+}
+
+// PersonData contains the detailed person information
+type PersonData struct {
+	gorm.Model
+	IString       `gorm:"-"`
+	ID            string     `json:"id"`
+	Name          PersonName `json:"name" gorm:"embedded;embeddedPrefix:name_"`
+	Email         string     `json:"email" gorm:"unique"`
+	Location      string     `json:"location"`
+	TimeZone      string     `json:"timeZone"`
+	UTCOffset     int        `json:"utcOffset"`
+	Geo           PersonGeo  `json:"geo" gorm:"embedded;embeddedPrefix:geo_"`
+	Bio           string     `json:"bio"`
+	Site          string     `json:"site"`
+	Avatar        string     `json:"avatar"`
+	Employment    Employment `json:"employment" gorm:"embedded;embeddedPrefix:employment_"`
+	Facebook      Facebook   `json:"facebook" gorm:"embedded;embeddedPrefix:facebook_"`
+	GitHub        GitHub     `json:"github" gorm:"embedded;embeddedPrefix:github_"`
+	Twitter       Twitter    `json:"twitter" gorm:"embedded;embeddedPrefix:twitter_"`
+	LinkedIn      LinkedIn   `json:"linkedin" gorm:"embedded;embeddedPrefix:linkedin_"`
+	GooglePlus    GooglePlus `json:"googleplus" gorm:"embedded;embeddedPrefix:googleplus_"`
+	Gravatar      Gravatar   `json:"gravatar" gorm:"embedded;embeddedPrefix:gravatar_"`
+	Fuzzy         bool       `json:"fuzzy"`
+	EmailProvider string     `json:"emailProvider"`
+	IndexedAt     string     `json:"indexedAt"`
+	Phone         string     `json:"phone"`
+	ActiveAt      string     `json:"activeAt"`
+	InactiveAt    string     `json:"inactiveAt"`
+}
+
+func (pd PersonData) String() string {
+	return fmt.Sprintf("ID: %s\nName: %v\nEmail: %s\nLocation: %s\nTimeZone: %s\nUTCOffset: %d\nGeo: %v\nBio: %s\nSite: %s\nAvatar: %s\nEmployment: %v\nFacebook: %v\nGitHub: %v\nTwitter: %v\nLinkedIn: %v\nGooglePlus: %v\nGravatar: %v\nFuzzy: %t\nEmailProvider: %s\nIndexedAt: %s\nPhone: %s\nActiveAt: %s\nInactiveAt: %s\n",
+		pd.ID, pd.Name, pd.Email, pd.Location, pd.TimeZone, pd.UTCOffset, pd.Geo, pd.Bio, pd.Site, pd.Avatar, pd.Employment, pd.Facebook, pd.GitHub, pd.Twitter, pd.LinkedIn, pd.GooglePlus, pd.Gravatar, pd.Fuzzy, pd.EmailProvider, pd.IndexedAt, pd.Phone, pd.ActiveAt, pd.InactiveAt)
+}
+
+func (pd *PersonData) NameTree() *tree.Tree {
+	nameTree := tree.Root("Name")
+	nameTree.Child("Full Name: " + pd.Name.FullName)
+	nameTree.Child("Given Name: " + pd.Name.GivenName)
+	nameTree.Child("Family Name: " + pd.Name.FamilyName)
+	return nameTree
+}
+
+func (pd *PersonData) GeoTree() *tree.Tree {
+	geoTree := tree.Root("Geo")
+	geoTree.Child("City: " + pd.Geo.City)
+	geoTree.Child("State: " + pd.Geo.State)
+	geoTree.Child("State Code: " + pd.Geo.StateCode)
+	geoTree.Child("Country: " + pd.Geo.Country)
+	geoTree.Child("Country Code: " + pd.Geo.CountryCode)
+	geoTree.Child("Latitude: " + fmt.Sprintf("%f", pd.Geo.Lat))
+	geoTree.Child("Longitude: " + fmt.Sprintf("%f", pd.Geo.Lng))
+	return geoTree
+}
+
+func (pd *PersonData) EmploymentTree() *tree.Tree {
+	employmentTree := tree.Root("Employment")
+	employmentTree.Child("Domain: " + pd.Employment.Domain)
+	employmentTree.Child("Name: " + pd.Employment.Name)
+	employmentTree.Child("Title: " + pd.Employment.Title)
+	employmentTree.Child("Role: " + pd.Employment.Role)
+	employmentTree.Child("Sub Role: " + pd.Employment.SubRole)
+	employmentTree.Child("Seniority: " + pd.Employment.Seniority)
+	return employmentTree
+}
+
+func (pd *PersonData) FacebookTree() *tree.Tree {
+	facebookTree := tree.Root("Facebook")
+	facebookTree.Child("Handle: " + pd.Facebook.Handle)
+	facebookTree.Child("Likes: " + fmt.Sprintf("%d", pd.Facebook.Likes))
+	return facebookTree
+}
+
+func (pd *PersonData) GitHubTree() *tree.Tree {
+	githubTree := tree.Root("GitHub")
+	githubTree.Child("Handle: " + pd.GitHub.Handle)
+	githubTree.Child("ID: " + pd.GitHub.ID)
+	githubTree.Child("Avatar: " + pd.GitHub.Avatar)
+	githubTree.Child("Company: " + pd.GitHub.Company)
+	githubTree.Child("Blog: " + pd.GitHub.Blog)
+	githubTree.Child("Followers: " + fmt.Sprintf("%d", pd.GitHub.Followers))
+	githubTree.Child("Following: " + fmt.Sprintf("%d", pd.GitHub.Following))
+	return githubTree
+}
+
+func (pd *PersonData) TwitterTree() *tree.Tree {
+	twitterTree := tree.Root("Twitter")
+	twitterTree.Child("Handle: " + pd.Twitter.Handle)
+	twitterTree.Child("ID: " + pd.Twitter.ID)
+	twitterTree.Child("Bio: " + pd.Twitter.Bio)
+	twitterTree.Child("Followers: " + fmt.Sprintf("%d", pd.Twitter.Followers))
+	twitterTree.Child("Following: " + fmt.Sprintf("%d", pd.Twitter.Following))
+	twitterTree.Child("Location: " + pd.Twitter.Location)
+	twitterTree.Child("Site: " + pd.Twitter.Site)
+	twitterTree.Child("Avatar: " + pd.Twitter.Avatar)
+	return twitterTree
+}
+
+func (pd *PersonData) LinkedInTree() *tree.Tree {
+	linkedinTree := tree.Root("LinkedIn")
+	linkedinTree.Child("Handle: " + pd.LinkedIn.Handle)
+	return linkedinTree
+}
+
+func (pd *PersonData) GooglePlusTree() *tree.Tree {
+	googlePlusTree := tree.Root("GooglePlus")
+	googlePlusTree.Child("Handle: " + pd.GooglePlus.Handle)
+	return googlePlusTree
+}
+
+func (pd *PersonData) GravatarTree() *tree.Tree {
+	gravatarTree := tree.Root("Gravatar")
+	gravatarTree.Child("Handle: " + pd.Gravatar.Handle)
+	gravatarTree.Child("Avatar: " + pd.Gravatar.Avatar)
+	return gravatarTree
+}
+
+func (PersonData) TableName() string {
+	return "person"
+}
+
+// PersonName contains the person's name components
+type PersonName struct {
+	FullName   string `json:"fullName"`
+	GivenName  string `json:"givenName"`
+	FamilyName string `json:"familyName"`
+}
+
+// PersonGeo contains location information for a person
+type PersonGeo struct {
+	City        string  `json:"city"`
+	State       string  `json:"state"`
+	StateCode   string  `json:"stateCode"`
+	Country     string  `json:"country"`
+	CountryCode string  `json:"countryCode"`
+	Lat         float64 `json:"lat"`
+	Lng         float64 `json:"lng"`
+}
+
+// Employment contains employment information
+type Employment struct {
+	Domain    string `json:"domain"`
+	Name      string `json:"name"`
+	Title     string `json:"title"`
+	Role      string `json:"role"`
+	SubRole   string `json:"subRole"`
+	Seniority string `json:"seniority"`
+}
+
+// GitHub contains GitHub profile information
+type GitHub struct {
+	gorm.Model
+	Handle    string `json:"handle"`
+	ID        string `json:"id"`
+	Avatar    string `json:"avatar"`
+	Company   string `json:"company"`
+	Blog      string `json:"blog"`
+	Followers int    `json:"followers"`
+	Following int    `json:"following"`
+	PersonID  int    `json:"person_id,omitempty"`
+}
+
+// GooglePlus contains Google+ profile information
+type GooglePlus struct {
+	Handle   string `json:"handle"`
+	PersonID int    `json:"person_id,omitempty"`
+}
+
+// Gravatar contains Gravatar profile information
+type Gravatar struct {
+	gorm.Model
+	Handle   string   `json:"handle"`
+	URLs     []string `json:"urls" gorm:"serializer:json"`
+	Avatar   string   `json:"avatar"`
+	Avatars  []string `json:"avatars" gorm:"serializer:json"`
+	PersonID int      `json:"person_id,omitempty"`
+}
+
+// Facebook contains Facebook profile information
+type Facebook struct {
+	Handle string `json:"handle"`
+	Likes  int    `json:"likes"`
+}
+
+// LinkedIn contains LinkedIn profile information
+type LinkedIn struct {
+	Handle string `json:"handle"`
+}
+
+// Twitter contains Twitter profile information
+type Twitter struct {
+	Handle    string `json:"handle"`
+	ID        string `json:"id"`
+	Bio       string `json:"bio"`
+	Followers int    `json:"followers"`
+	Following int    `json:"following"`
+	Location  string `json:"location"`
+	Site      string `json:"site"`
+	Avatar    string `json:"avatar"`
+}
+
+// Crunchbase contains Crunchbase profile information
+type Crunchbase struct {
+	Handle string `json:"handle"`
+}
+
+// YouTube contains YouTube profile information
+type YouTube struct {
+	Handle string `json:"handle"`
+}
+
+// PersonMeta contains metadata about the API response
+type PersonMeta struct {
+	Email string `json:"email"`
+}
+
+// HunterCombinedEnrichmentResponse represents the response from Hunter.io combined enrichment API
+type HunterCombinedEnrichmentResponse struct {
+	Data CombinedData `json:"data" gorm:"embedded;embeddedPrefix:data_"`
+	Meta CombinedMeta `json:"meta" gorm:"embedded;embeddedPrefix:meta_"`
+}
+
+// CombinedData contains both person and company information
+type CombinedData struct {
+	IString
+	Person  PersonData  `json:"person" gorm:"embedded;embeddedPrefix:person_"`
+	Company CompanyData `json:"company" gorm:"embedded;embeddedPrefix:company_"`
+}
+
+func (cbd CombinedData) String() string {
+	return fmt.Sprintf("Person: %s\nCompany: %s",
+		cbd.Person.String(),
+		cbd.Company.String())
+}
+
+// CombinedMeta contains metadata about the API response
+type CombinedMeta struct {
+	Email string `json:"email"`
+}
+
+// String returns a string representation of the combined enrichment response
+func (c *HunterCombinedEnrichmentResponse) String() string {
+	return fmt.Sprintf("Person:\n%s\n\nCompany:\n%s",
+		c.Data.Person.String(),
+		c.Data.Company.String())
+}
+
+func StoreHunterDomain(hunterDomain HunterDomainData) error {
+	db := GetDB()
+
+	// Use OnConflict clause to handle duplicates
+	err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(&hunterDomain).Error
+	if err != nil {
+		zap.L().Error("store_hunter_domain",
+			zap.String("message", "failed to store hunter domain"),
+			zap.Error(err))
+		return err
+	}
+
+	return nil
+}
+
+func StoreHunterEmails(hunterEmails []HunterEmail) error {
+	if len(hunterEmails) == 0 {
+		return nil
+	}
+
+	zap.L().Info("Storing hunter emails", zap.Int("count", len(hunterEmails)))
+	db := GetDB()
+
+	// Use batch insert with conflict handling
+	const batchSize = 100
+	var lastErr error
+
+	for i := 0; i < len(hunterEmails); i += batchSize {
+		end := i + batchSize
+		if end > len(hunterEmails) {
+			end = len(hunterEmails)
+		}
+
+		batch := hunterEmails[i:end]
+		// Use Clauses with OnConflict DoNothing to skip conflicts
+		err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
+		if err != nil {
+			zap.L().Warn("Error storing some hunter emails", zap.Error(err))
+			lastErr = err
+			// Continue with next batch despite error
+		}
+	}
+
+	return lastErr
+}
+
+func StoreHunterPersonData(personData PersonData) error {
+	db := GetDB()
+
+	// Use OnConflict clause to handle duplicates
+	err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(&personData).Error
+	if err != nil {
+		zap.L().Error("store_person_data",
+			zap.String("message", "failed to store person data"),
+			zap.Error(err))
+		return err
+	}
+
+	return nil
+}

internal/sqlite/ports.go

@@ -0,0 +1 @@
+package sqlite

internal/sqlite/query.go

@@ -1,20 +0,0 @@
-package sqlite
-
-type DehashedSearchRequest struct {
-	Page     int    `json:"page"`
-	Query    string `json:"query"`
-	Size     int    `json:"size"`
-	Wildcard bool   `json:"wildcard"`
-	Regex    bool   `json:"regex"`
-	DeDupe   bool   `json:"de_dupe"`
-}
-
-func NewDehashedSearchRequest(size int, wildcard, regex bool) *DehashedSearchRequest {
-	return &DehashedSearchRequest{
-		Page:     0,
-		Size:     size,
-		Wildcard: false,
-		Regex:    false,
-		DeDupe:   true,
-	}
-}

internal/sqlite/result.go

@@ -1,94 +0,0 @@
-package sqlite
-
-import (
-	"encoding/json"
-	"fmt"
-	"go.uber.org/zap"
-	"gorm.io/gorm"
-	"io"
-	"os"
-)
-
-type DehashedResponse struct {
-	Balance      int      `json:"balance"`
-	Entries      []Result `json:"entries"`
-	Success      bool     `json:"success"`
-	Took         string   `json:"took"`
-	TotalResults int      `json:"total"`
-}
-
-type Result struct {
-	gorm.Model
-	DehashedId            string   `json:"id" xml:"id" yaml:"id" gorm:"uniqueIndex"`
-	Email                 []string `json:"email,omitempty" xml:"email,omitempty" yaml:"email,omitempty" gorm:"serializer:json"`
-	IpAddress             []string `json:"ip_address,omitempty" xml:"ip_address,omitempty" yaml:"ip_address,omitempty" gorm:"serializer:json"`
-	Username              []string `json:"username,omitempty" xml:"username,omitempty" yaml:"username,omitempty" gorm:"serializer:json"`
-	Password              []string `json:"password,omitempty" xml:"password,omitempty" yaml:"password,omitempty" gorm:"serializer:json"`
-	HashedPassword        []string `json:"hashed_password,omitempty" xml:"hashed_password,omitempty" yaml:"hashed_password,omitempty" gorm:"serializer:json"`
-	HashType              string   `json:"hash_type,omitempty" xml:"hash_type,omitempty" yaml:"hash_type,omitempty"`
-	Name                  []string `json:"name,omitempty" xml:"name,omitempty" yaml:"name,omitempty" gorm:"serializer:json"`
-	Vin                   []string `json:"vin,omitempty" xml:"vin,omitempty" yaml:"vin,omitempty" gorm:"serializer:json"`
-	LicensePlate          []string `json:"license_plate,omitempty" xml:"license_plate,omitempty" yaml:"license_plate,omitempty" gorm:"serializer:json"`
-	Url                   []string `json:"url,omitempty" xml:"url,omitempty" yaml:"url,omitempty" gorm:"serializer:json"`
-	Social                []string `json:"social,omitempty" xml:"social,omitempty" yaml:"social,omitempty" gorm:"serializer:json"`
-	CryptoCurrencyAddress []string `json:"cryptocurrency_address,omitempty" xml:"cryptocurrency_address,omitempty" yaml:"cryptocurrency_address,omitempty" gorm:"serializer:json"`
-	Address               []string `json:"address,omitempty" xml:"address,omitempty" yaml:"address,omitempty" gorm:"serializer:json"`
-	Phone                 []string `json:"phone,omitempty" xml:"phone,omitempty" yaml:"phone,omitempty" gorm:"serializer:json"`
-	Company               []string `json:"company,omitempty" xml:"company,omitempty" yaml:"company,omitempty" gorm:"serializer:json"`
-	DatabaseName          string   `json:"database_name,omitempty" xml:"database_name,omitempty" yaml:"database_name,omitempty"`
-}
-
-func (Result) TableName() string {
-	return "results"
-}
-
-type DehashedResults struct {
-	Results []Result `json:"results"`
-}
-
-func (dr *DehashedResults) ExtractCredentials() []Creds {
-	var creds []Creds
-
-	results := dr.Results
-
-	for _, r := range results {
-		if len(r.Password) > 0 {
-			// Get first email if available
-			email := ""
-			if len(r.Email) > 0 {
-				email = r.Email[0]
-			}
-
-			// Get first password
-			password := r.Password[0]
-
-			cred := Creds{Email: email, Password: password}
-			creds = append(creds, cred)
-		}
-	}
-
-	go func() {
-		err := StoreCreds(creds)
-		if err != nil {
-			zap.L().Error("store_creds",
-				zap.String("message", "failed to store creds"),
-				zap.Error(err),
-			)
-			fmt.Printf("Error Storing Results: %v", err)
-		}
-	}()
-
-	return creds
-}
-
-func NewDehashedResults(body io.Reader) ([]Result, int, int) {
-	var response DehashedResponse
-
-	err := json.NewDecoder(body).Decode(&response)
-	if err != nil {
-		fmt.Printf("Error Parsing Response Body: %v", err)
-		os.Exit(-1)
-	}
-
-	return response.Entries, response.Balance, response.TotalResults
-}

internal/sqlite/structs.go

@@ -1,116 +0,0 @@
-package sqlite
-
-import (
-	"dehasher/internal/files"
-	"fmt"
-	"gorm.io/gorm"
-)
-
-type DBOptions struct {
-	Username              string
-	Email                 string
-	IPAddress             string
-	Password              string
-	HashedPassword        string
-	Name                  string
-	Vin                   string
-	LicensePlate          string
-	Address               string
-	Phone                 string
-	Social                string
-	CryptoCurrencyAddress string
-	Domain                string
-	Limit                 int
-	ExactMatch            bool
-	NonEmptyFields        []string // Fields that should not be empty
-	DisplayFields         []string // Fields to display in output
-}
-
-func NewDBOptions() *DBOptions {
-	return &DBOptions{
-		Limit:          100, // Default limit
-		ExactMatch:     false,
-		NonEmptyFields: []string{},
-		DisplayFields:  []string{},
-	}
-}
-
-func (o *DBOptions) Empty() bool {
-	return o.Username == "" && o.Email == "" && o.IPAddress == "" &&
-		o.Password == "" && o.HashedPassword == "" && o.Name == "" &&
-		o.Vin == "" && o.LicensePlate == "" && o.Address == "" &&
-		o.Phone == "" && o.Social == "" && o.CryptoCurrencyAddress == "" && o.Domain == "" &&
-		len(o.NonEmptyFields) == 0
-}
-
-type QueryOptions struct {
-	gorm.Model
-	MaxRecords         int            `json:"max_records"`
-	MaxRequests        int            `json:"max_requests"`
-	StartingPage       int            `json:"starting_page"`
-	OutputFormat       files.FileType `json:"output_format"`
-	OutputFile         string         `json:"output_file"`
-	RegexMatch         bool           `json:"regex_match"`
-	WildcardMatch      bool           `json:"wildcard_match"`
-	UsernameQuery      string         `json:"username_query"`
-	EmailQuery         string         `json:"email_query"`
-	IpQuery            string         `json:"ip_query"`
-	PassQuery          string         `json:"pass_query"`
-	HashQuery          string         `json:"hash_query"`
-	NameQuery          string         `json:"name_query"`
-	DomainQuery        string         `json:"domain_query"`
-	VinQuery           string         `json:"vin_query"`
-	LicensePlateQuery  string         `json:"license_plate_query"`
-	AddressQuery       string         `json:"address_query"`
-	PhoneQuery         string         `json:"phone_query"`
-	SocialQuery        string         `json:"social_query"`
-	CryptoAddressQuery string         `json:"crypto_address_query"`
-	PrintBalance       bool           `json:"print_balance"`
-	CredsOnly          bool           `json:"creds_only"`
-}
-
-func (QueryOptions) TableName() string {
-	return "query_options"
-}
-
-func NewQueryOptions(maxRecords, maxRequests, startingPage int, outputFormat, outputFile, usernameQuery, emailQuery, ipQuery, passQuery, hashQuery, nameQuery, domainQuery, vinQuery, licensePlateQuery, addressQuery, phoneQuery, socialQuery, cryptoAddressQuery string, regexMatch, wildcardMatch, printBalance, credsOnly bool) *QueryOptions {
-	return &QueryOptions{
-		MaxRecords:         maxRecords,
-		MaxRequests:        maxRequests,
-		StartingPage:       startingPage,
-		OutputFormat:       files.GetFileType(outputFormat),
-		OutputFile:         outputFile,
-		PrintBalance:       printBalance,
-		CredsOnly:          credsOnly,
-		RegexMatch:         regexMatch,
-		WildcardMatch:      wildcardMatch,
-		UsernameQuery:      usernameQuery,
-		EmailQuery:         emailQuery,
-		IpQuery:            ipQuery,
-		PassQuery:          passQuery,
-		HashQuery:          hashQuery,
-		NameQuery:          nameQuery,
-		DomainQuery:        domainQuery,
-		VinQuery:           vinQuery,
-		LicensePlateQuery:  licensePlateQuery,
-		AddressQuery:       addressQuery,
-		PhoneQuery:         phoneQuery,
-		SocialQuery:        socialQuery,
-		CryptoAddressQuery: cryptoAddressQuery,
-	}
-}
-
-type Creds struct {
-	gorm.Model
-	Email    string `json:"email" yaml:"email" xml:"email"`
-	Username string `json:"username" yaml:"username" xml:"username"`
-	Password string `json:"password" yaml:"password" xml:"password"`
-}
-
-func (Creds) TableName() string {
-	return "creds"
-}
-
-func (c Creds) ToString() string {
-	return fmt.Sprintf("%s%s%s", c.Username, "%", c.Password)
-}

internal/sqlite/subdomains.go

@@ -0,0 +1,45 @@
+package sqlite
+
+import (
+	"go.uber.org/zap"
+	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
+)
+
+type Subdomain struct {
+	gorm.Model
+	Domain    string `json:"domain" yaml:"domain" xml:"domain"`
+	Subdomain string `json:"subdomain" yaml:"subdomain" xml:"subdomain" gorm:"uniqueIndex:idx_subdomain"`
+}
+
+func StoreSubdomains(subs []Subdomain) error {
+	if len(subs) == 0 {
+		return nil
+	}
+
+	zap.L().Info("Storing subdomains", zap.Int("count", len(subs)))
+	db := GetDB()
+
+	// Use batch insert with conflict handling
+	// This will insert records in batches and continue even if some fail
+	const batchSize = 100
+	var lastErr error
+
+	for i := 0; i < len(subs); i += batchSize {
+		end := i + batchSize
+		if end > len(subs) {
+			end = len(subs)
+		}
+
+		batch := subs[i:end]
+		// Use Clauses with OnConflict DoNothing to skip conflicts
+		err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
+		if err != nil {
+			zap.L().Warn("Error storing some credentials", zap.Error(err))
+			lastErr = err
+			// Continue with next batch despite error
+		}
+	}
+
+	return lastErr
+}

internal/sqlite/tables.go

@@ -1,53 +0,0 @@
-package sqlite
-
-import "strings"
-
-type Table int64
-
-const (
-	ResultsTable Table = iota
-	RunsTable
-	CredsTable
-	WhoIsTable
-	SubdomainsTable
-	HistoryTable
-	UnknownTable
-)
-
-func GetTable(userInput string) Table {
-	switch strings.ToLower(userInput) {
-	case "results":
-		return ResultsTable
-	case "runs":
-		return RunsTable
-	case "creds":
-		return CredsTable
-	case "whois":
-		return WhoIsTable
-	case "subdomains":
-		return SubdomainsTable
-	case "history":
-		return HistoryTable
-	default:
-		return UnknownTable
-	}
-}
-
-func (t Table) Object() interface{} {
-	switch t {
-	case ResultsTable:
-		return Result{}
-	case RunsTable:
-		return QueryOptions{}
-	case CredsTable:
-		return Creds{}
-	case WhoIsTable:
-		return WhoisRecord{}
-	case SubdomainsTable:
-		return SubdomainRecord{}
-	case HistoryTable:
-		return HistoryRecord{}
-	default:
-		return nil
-	}
-}

internal/sqlite/users.go

@@ -0,0 +1,58 @@
+package sqlite
+
+import (
+	"go.uber.org/zap"
+	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
+)
+
+type User struct {
+	gorm.Model
+	Company     string `json:"company" yaml:"company" xml:"company"`
+	Position    string `json:"position" yaml:"position" xml:"position"`
+	Department  string `json:"department" yaml:"department" xml:"department"`
+	PhoneNumber string `json:"phone_number" yaml:"phone_number" xml:"phone_number"`
+	FullName    string `json:"full_name" yaml:"full_name" xml:"full_name"`
+	Phone       string `json:"phone" yaml:"phone" xml:"phone"`
+	Linkedin    string `json:"linkedin" yaml:"linkedin" xml:"linkedin"`
+	Twitter     string `json:"twitter" yaml:"twitter" xml:"twitter"`
+	Facebook    string `json:"facebook" yaml:"facebook" xml:"facebook"`
+	Instagram   string `json:"instagram" yaml:"instagram" xml:"instagram"`
+	Youtube     string `json:"youtube" yaml:"youtube" xml:"youtube"`
+	Gravatar    string `json:"gravatar" yaml:"gravatar" xml:"gravatar"`
+	Email       string `json:"email" yaml:"email" xml:"email" gorm:"uniqueIndex:idx_email_username_password"`
+	Username    string `json:"username" yaml:"username" xml:"username" gorm:"uniqueIndex:idx_email_username_password"`
+	Password    string `json:"password" yaml:"password" xml:"password" gorm:"uniqueIndex:idx_email_username_password"`
+}
+
+func StoreUsers(users []User) error {
+	if len(users) == 0 {
+		return nil
+	}
+
+	zap.L().Info("Storing credentials", zap.Int("count", len(users)))
+	db := GetDB()
+
+	// Use batch insert with conflict handling
+	// This will insert records in batches and continue even if some fail
+	const batchSize = 100
+	var lastErr error
+
+	for i := 0; i < len(users); i += batchSize {
+		end := i + batchSize
+		if end > len(users) {
+			end = len(users)
+		}
+
+		batch := users[i:end]
+		// Use Clauses with OnConflict DoNothing to skip conflicts
+		err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
+		if err != nil {
+			zap.L().Warn("Error storing some credentials", zap.Error(err))
+			lastErr = err
+			// Continue with next batch despite error
+		}
+	}
+
+	return lastErr
+}

internal/sqlite/whois.go

@@ -1,6 +1,12 @@
 package sqlite
 
-import "gorm.io/gorm"
+import (
+	"fmt"
+	"go.uber.org/zap"
+	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
+	"strings"
+)
 
 type WhoIsLookupResult struct {
 	RemainingCredits int  `json:"remaining_credits"`
@@ -17,7 +23,7 @@ type WhoisRecord struct {
 	ContactEmail          string       `json:"contactEmail"`
 	CreatedDate           string       `json:"createdDate"`
 	CreatedDateNormalized string       `json:"createdDateNormalized"`
-	DomainName            string       `json:"domainName"`
+	DomainName            string       `json:"domainName" gorm:"unique"`
 	DomainNameExt         string       `json:"domainNameExt"`
 	EstimatedDomainAge    int          `json:"estimatedDomainAge"`
 	ExpiresDate           string       `json:"expiresDate"`
@@ -38,6 +44,142 @@ type WhoisRecord struct {
 	UpdatedDateNormalized string       `json:"updatedDateNormalized"`
 }
 
+func (w WhoisRecord) String() string {
+	var sb strings.Builder
+
+	// Main domain information
+	sb.WriteString(fmt.Sprintf("Domain Name: %s\n", w.DomainName))
+	sb.WriteString(fmt.Sprintf("Domain Name Ext: %s\n", w.DomainNameExt))
+	sb.WriteString(fmt.Sprintf("Registrar Name: %s\n", w.RegistrarName))
+	sb.WriteString(fmt.Sprintf("Registrar IANA ID: %s\n", w.RegistrarIANAID))
+	sb.WriteString(fmt.Sprintf("Contact HunterEmail: %s\n", w.ContactEmail))
+	sb.WriteString(fmt.Sprintf("Estimated Domain Age: %d days\n", w.EstimatedDomainAge))
+
+	// Dates
+	sb.WriteString(fmt.Sprintf("Created Date: %s (Normalized: %s)\n", w.CreatedDate, w.CreatedDateNormalized))
+	sb.WriteString(fmt.Sprintf("Updated Date: %s (Normalized: %s)\n", w.UpdatedDate, w.UpdatedDateNormalized))
+	sb.WriteString(fmt.Sprintf("Expires Date: %s (Normalized: %s)\n", w.ExpiresDate, w.ExpiresDateNormalized))
+
+	// Status
+	sb.WriteString(fmt.Sprintf("Status: %s\n", w.Status))
+
+	// Parse code
+	sb.WriteString(fmt.Sprintf("Parse Code: %d\n", w.ParseCode))
+
+	// Audit information
+	sb.WriteString("\nAudit Information:\n")
+	sb.WriteString(fmt.Sprintf("  Created Date: %s\n", w.Audit.CreatedDate))
+	sb.WriteString(fmt.Sprintf("  Updated Date: %s\n", w.Audit.UpdatedDate))
+
+	// Name servers
+	sb.WriteString("\nName Servers:\n")
+	if len(w.NameServers.HostNames) > 0 {
+		for i, ns := range w.NameServers.HostNames {
+			ip := ""
+			if i < len(w.NameServers.IPs) {
+				ip = fmt.Sprintf(" (%s)", w.NameServers.IPs[i])
+			}
+			sb.WriteString(fmt.Sprintf("  %d. %s%s\n", i+1, ns, ip))
+		}
+	} else {
+		sb.WriteString("  None listed\n")
+	}
+
+	if w.NameServers.RawText != "" {
+		sb.WriteString(fmt.Sprintf("  Raw Text: %s\n", w.NameServers.RawText))
+	}
+
+	// Contact information
+	sb.WriteString("\nRegistrant Contact:\n")
+	formatWhoisContact(&sb, w.Registrant, "  ")
+
+	sb.WriteString("\nTechnical Contact:\n")
+	formatWhoisContact(&sb, w.TechnicalContact, "  ")
+
+	// Registry Data
+	sb.WriteString("\nRegistry Data:\n")
+	if w.RegistryData.DomainName != "" {
+		sb.WriteString(fmt.Sprintf("  Domain Name: %s\n", w.RegistryData.DomainName))
+		sb.WriteString(fmt.Sprintf("  Registrar Name: %s\n", w.RegistryData.RegistrarName))
+		sb.WriteString(fmt.Sprintf("  Registrar IANA ID: %s\n", w.RegistryData.RegistrarIANAID))
+		sb.WriteString(fmt.Sprintf("  Whois Server: %s\n", w.RegistryData.WhoisServer))
+		sb.WriteString(fmt.Sprintf("  Status: %s\n", w.RegistryData.Status))
+
+		// Registry dates
+		sb.WriteString(fmt.Sprintf("  Created Date: %s (Normalized: %s)\n",
+			w.RegistryData.CreatedDate, w.RegistryData.CreatedDateNormalized))
+		sb.WriteString(fmt.Sprintf("  Updated Date: %s (Normalized: %s)\n",
+			w.RegistryData.UpdatedDate, w.RegistryData.UpdatedDateNormalized))
+		sb.WriteString(fmt.Sprintf("  Expires Date: %s (Normalized: %s)\n",
+			w.RegistryData.ExpiresDate, w.RegistryData.ExpiresDateNormalized))
+
+		// Registry nameservers
+		sb.WriteString("  Name Servers:\n")
+		if len(w.RegistryData.NameServers.HostNames) > 0 {
+			for i, ns := range w.RegistryData.NameServers.HostNames {
+				ip := ""
+				if i < len(w.RegistryData.NameServers.IPs) {
+					ip = fmt.Sprintf(" (%s)", w.RegistryData.NameServers.IPs[i])
+				}
+				sb.WriteString(fmt.Sprintf("    %d. %s%s\n", i+1, ns, ip))
+			}
+		} else {
+			sb.WriteString("    None listed\n")
+		}
+
+		// Registry audit
+		sb.WriteString("  Audit Information:\n")
+		sb.WriteString(fmt.Sprintf("    Created Date: %s\n", w.RegistryData.Audit.CreatedDate))
+		sb.WriteString(fmt.Sprintf("    Updated Date: %s\n", w.RegistryData.Audit.UpdatedDate))
+	} else {
+		sb.WriteString("  No registry data available\n")
+	}
+
+	// Header and footer
+	if w.Header != "" {
+		headerPreview := w.Header
+		if len(headerPreview) > 100 {
+			headerPreview = headerPreview[:100] + "... [truncated]"
+		}
+		sb.WriteString("\nHeader:\n")
+		sb.WriteString(headerPreview)
+		sb.WriteString("\n")
+	}
+
+	if w.Footer != "" {
+		footerPreview := w.Footer
+		if len(footerPreview) > 100 {
+			footerPreview = footerPreview[:100] + "... [truncated]"
+		}
+		sb.WriteString("\nFooter:\n")
+		sb.WriteString(footerPreview)
+		sb.WriteString("\n")
+	}
+
+	// Raw text (truncated if too long)
+	if w.RawText != "" {
+		rawTextPreview := w.RawText
+		if len(rawTextPreview) > 500 {
+			rawTextPreview = rawTextPreview[:500] + "... [truncated]"
+		}
+		sb.WriteString("\nRaw Text:\n")
+		sb.WriteString(rawTextPreview)
+		sb.WriteString("\n")
+	}
+
+	if w.StrippedText != "" {
+		strippedTextPreview := w.StrippedText
+		if len(strippedTextPreview) > 500 {
+			strippedTextPreview = strippedTextPreview[:500] + "... [truncated]"
+		}
+		sb.WriteString("\nStripped Text:\n")
+		sb.WriteString(strippedTextPreview)
+		sb.WriteString("\n")
+	}
+
+	return sb.String()
+}
+
 func (WhoisRecord) TableName() string {
 	return "whois"
 }
@@ -104,7 +246,7 @@ type ScanResult struct {
 
 type SubdomainRecord struct {
 	gorm.Model
-	Domain    string `json:"domain"`
+	Domain    string `json:"domain" gorm:"unique"`
 	FirstSeen int64  `json:"firstSeen"`
 	LastSeen  int64  `json:"lastSeen"`
 }
@@ -131,7 +273,7 @@ type HistoryRecord struct {
 	CleanText             string      `json:"cleanText"`
 	CreatedDateISO8601    string      `json:"createdDateISO8601"`
 	CreatedDateRaw        string      `json:"createdDateRaw"`
-	DomainName            string      `json:"domainName"`
+	DomainName            string      `json:"domainName" gorm:"unique"`
 	DomainType            string      `json:"domainType"`
 	ExpiresDateISO8601    string      `json:"expiresDateISO8601"`
 	ExpiresDateRaw        string      `json:"expiresDateRaw"`
@@ -147,6 +289,131 @@ type HistoryRecord struct {
 	ZoneContact           ContactInfo `json:"zoneContact" gorm:"serializer:json"`
 }
 
+func (h HistoryRecord) String() string {
+	var sb strings.Builder
+
+	// Main domain information
+	sb.WriteString(fmt.Sprintf("Domain Name: %s\n", h.DomainName))
+	sb.WriteString(fmt.Sprintf("Domain Type: %s\n", h.DomainType))
+	sb.WriteString(fmt.Sprintf("Registrar Name: %s\n", h.RegistrarName))
+	sb.WriteString(fmt.Sprintf("Whois Server: %s\n", h.WhoisServer))
+
+	// Dates
+	sb.WriteString(fmt.Sprintf("Created Date: %s (Raw: %s)\n", h.CreatedDateISO8601, h.CreatedDateRaw))
+	sb.WriteString(fmt.Sprintf("Updated Date: %s (Raw: %s)\n", h.UpdatedDateISO8601, h.UpdatedDateRaw))
+	sb.WriteString(fmt.Sprintf("Expires Date: %s (Raw: %s)\n", h.ExpiresDateISO8601, h.ExpiresDateRaw))
+
+	// Status
+	sb.WriteString("Status: ")
+	if len(h.Status) > 0 {
+		sb.WriteString(strings.Join(h.Status, ", "))
+	} else {
+		sb.WriteString("N/A")
+	}
+	sb.WriteString("\n")
+
+	// Audit information
+	sb.WriteString("\nAudit Information:\n")
+	sb.WriteString(fmt.Sprintf("  Created Date: %s\n", h.Audit.CreatedDate))
+	sb.WriteString(fmt.Sprintf("  Updated Date: %s\n", h.Audit.UpdatedDate))
+
+	// Name servers
+	sb.WriteString("\nName Servers:\n")
+	if len(h.NameServers) > 0 {
+		for i, ns := range h.NameServers {
+			sb.WriteString(fmt.Sprintf("  %d. %s\n", i+1, ns))
+		}
+	} else {
+		sb.WriteString("  None listed\n")
+	}
+
+	// Contact information
+	sb.WriteString("\nRegistrant Contact:\n")
+	formatContact(&sb, h.RegistrantContact, "  ")
+
+	sb.WriteString("\nAdministrative Contact:\n")
+	formatContact(&sb, h.AdministrativeContact, "  ")
+
+	sb.WriteString("\nTechnical Contact:\n")
+	formatContact(&sb, h.TechnicalContact, "  ")
+
+	sb.WriteString("\nBilling Contact:\n")
+	formatContact(&sb, h.BillingContact, "  ")
+
+	sb.WriteString("\nZone Contact:\n")
+	formatContact(&sb, h.ZoneContact, "  ")
+
+	// Raw text (truncated if too long)
+	if len(h.RawText) > 0 {
+		rawTextPreview := h.RawText
+		if len(rawTextPreview) > 500 {
+			rawTextPreview = rawTextPreview[:500] + "... [truncated]"
+		}
+		sb.WriteString("\nRaw Text:\n")
+		sb.WriteString(rawTextPreview)
+		sb.WriteString("\n")
+	}
+
+	if len(h.CleanText) > 0 {
+		cleanTextPreview := h.CleanText
+		if len(cleanTextPreview) > 500 {
+			cleanTextPreview = cleanTextPreview[:500] + "... [truncated]"
+		}
+		sb.WriteString("\nClean Text:\n")
+		sb.WriteString(cleanTextPreview)
+		sb.WriteString("\n")
+	}
+
+	return sb.String()
+}
+
+// Helper function to format contact information
+func formatContact(sb *strings.Builder, contact ContactInfo, indent string) {
+	if contact.Name == "" && contact.Organization == "" && contact.Email == "" {
+		sb.WriteString(indent + "No contact information available\n")
+		return
+	}
+
+	if contact.Name != "" {
+		sb.WriteString(indent + "Name: " + contact.Name + "\n")
+	}
+	if contact.Organization != "" {
+		sb.WriteString(indent + "Organization: " + contact.Organization + "\n")
+	}
+	if contact.Email != "" {
+		sb.WriteString(indent + "HunterEmail: " + contact.Email + "\n")
+	}
+	if contact.Street != "" {
+		sb.WriteString(indent + "Street: " + contact.Street + "\n")
+	}
+	if contact.City != "" {
+		sb.WriteString(indent + "City: " + contact.City + "\n")
+	}
+	if contact.State != "" {
+		sb.WriteString(indent + "State: " + contact.State + "\n")
+	}
+	if contact.PostalCode != "" {
+		sb.WriteString(indent + "Postal Code: " + contact.PostalCode + "\n")
+	}
+	if contact.Country != "" {
+		sb.WriteString(indent + "Country: " + contact.Country + "\n")
+	}
+	if contact.Telephone != "" {
+		phone := contact.Telephone
+		if contact.TelephoneExt != "" {
+			phone += " ext. " + contact.TelephoneExt
+		}
+		sb.WriteString(indent + "Telephone: " + phone + "\n")
+	}
+	if contact.Fax != "" {
+		fax := contact.Fax
+		if contact.FaxExt != "" {
+			fax += " ext. " + contact.FaxExt
+		}
+		sb.WriteString(indent + "Fax: " + fax + "\n")
+	}
+}
+
 func (HistoryRecord) TableName() string {
 	return "history"
 }
@@ -170,3 +437,180 @@ type ContactInfo struct {
 type WhoIsCredits struct {
 	WhoisCredits int `json:"whois_credits"`
 }
+
+type WhoIsIPLookup struct {
+	RemainingCredits int    `json:"remaining_credits"`
+	Data             IPData `json:"data"`
+}
+
+type WhoIsMXLookup struct {
+	RemainingCredits int    `json:"remaining_credits"`
+	Data             IPData `json:"data"`
+}
+
+type WhoIsNSLookup struct {
+	RemainingCredits int    `json:"remaining_credits"`
+	Data             IPData `json:"data"`
+}
+
+type IPData struct {
+	CurrentPage string         `json:"current_page"`
+	Result      []LookupResult `json:"result"`
+	Size        int            `json:"size"`
+}
+
+type LookupResult struct {
+	gorm.Model
+	FirstSeen  int64  `json:"first_seen"`
+	LastVisit  int64  `json:"last_visit"`
+	Name       string `json:"name" gorm:"unique"`
+	SearchTerm string `json:"search_term,omitempty"` // For storing the IP address this domain is associated with
+	Type       string `json:"type,omitempty"`        // For storing the MX address this domain is associated with
+}
+
+func (LookupResult) TableName() string {
+	return "lookup"
+}
+
+// Helper function to format contact information for WhoisRecord
+func formatWhoisContact(sb *strings.Builder, contact Contact, indent string) {
+	if contact.Name == "" && contact.Organization == "" {
+		sb.WriteString(indent + "No contact information available\n")
+		return
+	}
+
+	if contact.Name != "" {
+		sb.WriteString(indent + "Name: " + contact.Name + "\n")
+	}
+	if contact.Organization != "" {
+		sb.WriteString(indent + "Organization: " + contact.Organization + "\n")
+	}
+	if contact.Street1 != "" {
+		sb.WriteString(indent + "Street: " + contact.Street1 + "\n")
+	}
+	if contact.City != "" {
+		sb.WriteString(indent + "City: " + contact.City + "\n")
+	}
+	if contact.State != "" {
+		sb.WriteString(indent + "State: " + contact.State + "\n")
+	}
+	if contact.PostalCode != "" {
+		sb.WriteString(indent + "Postal Code: " + contact.PostalCode + "\n")
+	}
+	if contact.Country != "" {
+		sb.WriteString(indent + "Country: " + contact.Country + "\n")
+	}
+	if contact.CountryCode != "" {
+		sb.WriteString(indent + "Country Code: " + contact.CountryCode + "\n")
+	}
+	if contact.Telephone != "" {
+		sb.WriteString(indent + "Telephone: " + contact.Telephone + "\n")
+	}
+	if contact.RawText != "" {
+		rawTextPreview := contact.RawText
+		if len(rawTextPreview) > 100 {
+			rawTextPreview = rawTextPreview[:100] + "... [truncated]"
+		}
+		sb.WriteString(indent + "Raw Text: " + rawTextPreview + "\n")
+	}
+}
+
+func StoreWhoisRecord(whoisRecord WhoisRecord) error {
+	// Create a pointer to the record to make it addressable
+	recordPtr := &whoisRecord
+
+	zap.L().Info("Storing WHOIS record",
+		zap.String("domain", whoisRecord.DomainName))
+
+	db := GetDB()
+
+	// Use OnConflict clause to handle duplicates
+	err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(recordPtr).Error
+	if err != nil {
+		zap.L().Error("store_whois_record",
+			zap.String("message", "failed to store whois record"),
+			zap.Error(err))
+		return err
+	}
+
+	return nil
+}
+
+func StoreWhoisHistoryRecords(historyRecords []HistoryRecord) error {
+	if len(historyRecords) == 0 {
+		return nil
+	}
+
+	zap.L().Info("Storing history records", zap.Int("count", len(historyRecords)))
+	db := GetDB()
+
+	// Use batch insert with conflict handling
+	const batchSize = 100
+	var lastErr error
+
+	for i := 0; i < len(historyRecords); i += batchSize {
+		end := i + batchSize
+		if end > len(historyRecords) {
+			end = len(historyRecords)
+		}
+
+		batch := historyRecords[i:end]
+		// Use Clauses with OnConflict DoNothing to skip conflicts
+		err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
+		if err != nil {
+			zap.L().Warn("Error storing some history records", zap.Error(err))
+			lastErr = err
+			// Continue with next batch despite error
+		}
+	}
+
+	return lastErr
+}
+
+func StoreWhoisLookup(lookup []LookupResult) error {
+	if len(lookup) == 0 {
+		return nil
+	}
+
+	zap.L().Info("Storing IP lookup records", zap.Int("count", len(lookup)))
+	db := GetDB()
+
+	// Use batch insert with conflict handling
+	const batchSize = 100
+	var lastErr error
+
+	for i := 0; i < len(lookup); i += batchSize {
+		end := i + batchSize
+		if end > len(lookup) {
+			end = len(lookup)
+		}
+
+		batch := lookup[i:end]
+		// Use Clauses with OnConflict DoNothing to skip conflicts
+		err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
+		if err != nil {
+			zap.L().Warn("Error storing some IP lookup records", zap.Error(err))
+			lastErr = err
+			// Continue with next batch despite error
+		}
+	}
+
+	return lastErr
+}
+
+// ReverseWhoisResponse represents the response from a reverse WHOIS lookup
+type ReverseWhoisResponse struct {
+	RemainingCredits int              `json:"remaining_credits"`
+	Data             ReverseWhoisData `json:"data"`
+}
+
+// ReverseWhoisData contains the domain count and list from a reverse WHOIS lookup
+type ReverseWhoisData struct {
+	DomainsCount        int      `json:"domainsCount"`
+	DomainsList         []string `json:"domainsList"`
+	NextPageSearchAfter *string  `json:"nextPageSearchAfter"`
+}
+
+func (rwd ReverseWhoisData) String() string {
+	return fmt.Sprintf("Domains Count: %d\nDomains List: %v\nNext Page Search After: %v\n", rwd.DomainsCount, rwd.DomainsList, rwd.NextPageSearchAfter)
+}