Merge pull request #4 from Kraken-OffSec/crows-nest-rebrand

Crows nest rebrand

Makefile

@@ -4,7 +4,7 @@
 GO=go
 
 # Binary name
-BINARY_NAME=dehasher
+BINARY_NAME=crowsnest
 
 # Build directory
 BUILD_DIR=build/bin
@@ -16,7 +16,7 @@ PLATFORMS=linux darwin windows
 ARCHS=amd64 arm64
 
 # Version info from git tag or default
-VERSION=$(shell git describe --tags 2>/dev/null || echo "v1.2.1")
+VERSION=$(shell git describe --tags 2>/dev/null || echo "v1.3.1")
 
 .PHONY: all clean build build-all
 
@@ -30,14 +30,14 @@ clean:
 
 # Build for current platform
 build:
-	$(GO) build -o $(BUILD_DIR)/$(BINARY_NAME) -ldflags "-X main.version=$(VERSION)" dehasher.go
+	$(GO) build -o $(BUILD_DIR)/$(BINARY_NAME) -ldflags "-X main.version=$(VERSION)" crowsnest.go
 
 # Build for all platforms
 build-all: clean
 	@for platform in $(PLATFORMS); do \
 	    for arch in $(ARCHS); do \
 	        echo "Building for $$platform/$$arch..."; \
-	        GOOS=$$platform GOARCH=$$arch $(GO) build -o $(BUILD_DIR)/$(BINARY_NAME)-$$platform-$$arch -ldflags "-X main.version=$(VERSION)" dehasher.go; \
+	        GOOS=$$platform GOARCH=$$arch $(GO) build -o $(BUILD_DIR)/$(BINARY_NAME)-$$platform-$$arch -ldflags "-X main.version=$(VERSION)" crowsnest.go; \
 	        if [ "$$platform" = "windows" ]; then \
 	            mv $(BUILD_DIR)/$(BINARY_NAME)-$$platform-$$arch $(BUILD_DIR)/$(BINARY_NAME)-$$platform-$$arch.exe; \
 	        fi; \

README.md

@@ -221,7 +221,7 @@ dehasher set-hunter <redacted>
 
 ### Domain Search
 Dehasher can perform a domain search for a given domain.  
-This provides a list of all emails that match the given query.
+This provides information about  company including a description, social media information and any technologies in use.
 ![Alt text](.img/hunter_domain_search.png "Hunter.io Domain Search")
 ```bash
 # Perform a Hunter.io domain search for example.com
@@ -230,7 +230,7 @@ dehasher hunter -d example.com -D
 
 ### Email Finder
 Dehasher can perform an email finder search for a given domain, first name, and last name.  
-This provides a list of all emails that match the given query.
+This provides information about a user including a confidence score, and any social media accounts linked to a first name, last name and email.
 ![Alt text](.img/hunter_email_finder.png "Hunter.io Email Finder")
 ```bash
 # Perform a Hunter.io email finder search for example.com
@@ -239,7 +239,7 @@ dehasher hunter -d example.com -F John -L Doe -E
 
 ### Email Verification
 Dehasher can perform an email verification search for a given email.  
-This provides a list of all emails that match the given query.
+This provides a verification and score of a given email address.
 ![Alt text](.img/email_verification.png "Hunter.io Email Verification")
 ```bash
 # Perform a Hunter.io email verification search for example@target.com
@@ -248,7 +248,7 @@ dehasher hunter -e example@target.com -V
 
 ### Company Enrichment
 Dehasher can perform a company enrichment search for a given domain.  
-This provides a list of all emails that match the given query.
+This provides information about a company given its domain.
 ![Alt text](.img/company_enrichment.png "Hunter.io Company Enrichment")
 ```bash
 # Perform a Hunter.io company enrichment search for example.com
@@ -257,7 +257,7 @@ dehasher hunter -d example.com -C
 
 ### Person Enrichment
 Dehasher can perform a person enrichment search for a given email.  
-This provides a list of all emails that match the given query.
+This provides information about a user given an email address..
 ![Alt text](.img/person_enrichment.png "Hunter.io Person Enrichment")
 ```bash
 # Perform a Hunter.io person enrichment search for example@target.com
@@ -266,7 +266,7 @@ dehasher hunter -e example@target.com -P
 
 ### Combined Enrichment
 Dehasher can perform a combined enrichment search for a given email.  
-This provides a list of all emails that match the given query.
+This is a combination of the company and person enrichments given an email address.
 ![Alt text](.img/combined_enrichment_1.png "Hunter.io Combined Enrichment")
 ![Alt text](.img/combined_enrichment_2.png "Hunter.io Combined Enrichment")
 ```bash

cmd/dehashed.go

@@ -1,10 +1,10 @@
 package cmd
 
 import (
-	"dehasher/internal/badger"
+	"crowsnest/internal/badger"
-	"dehasher/internal/debug"
+	"crowsnest/internal/debug"
-	"dehasher/internal/dehashed"
+	"crowsnest/internal/dehashed"
-	"dehasher/internal/sqlite"
+	"crowsnest/internal/sqlite"
 	"fmt"
 	"github.com/spf13/cobra"
 	"go.uber.org/zap"
@@ -12,34 +12,34 @@ import (
 
 func init() {
 	// Add api command to root command
-	rootCmd.AddCommand(apiCmd)
+	rootCmd.AddCommand(dehashedCmd)
 
 	// Add flags specific to api command
-	apiCmd.Flags().IntVarP(&maxRecords, "max-records", "m", 30000, "Maximum amount of records to return")
+	dehashedCmd.Flags().IntVarP(&maxRecords, "max-records", "m", 30000, "Maximum amount of records to return")
-	apiCmd.Flags().IntVarP(&maxRequests, "max-requests", "r", -1, "Maximum number of requests to make")
+	dehashedCmd.Flags().IntVarP(&maxRequests, "max-requests", "r", -1, "Maximum number of requests to make")
-	apiCmd.Flags().IntVarP(&startingPage, "starting-page", "s", 1, "Starting page for requests")
+	dehashedCmd.Flags().IntVarP(&startingPage, "starting-page", "s", 1, "Starting page for requests")
-	apiCmd.Flags().BoolVarP(&printBalance, "print-balance", "b", false, "Print remaining balance after requests")
+	dehashedCmd.Flags().BoolVarP(&printBalance, "print-balance", "b", false, "Print remaining balance after requests")
-	apiCmd.Flags().BoolVarP(&regexMatch, "regex-match", "R", false, "Use regex matching on query fields")
+	dehashedCmd.Flags().BoolVarP(&regexMatch, "regex-match", "R", false, "Use regex matching on query fields")
-	apiCmd.Flags().BoolVarP(&wildcardMatch, "wildcard-match", "W", false, "Use wildcard matching on query fields (Use ? to replace a single character, and * for multiple characters)")
+	dehashedCmd.Flags().BoolVarP(&wildcardMatch, "wildcard-match", "W", false, "Use wildcard matching on query fields (Use ? to replace a single character, and * for multiple characters)")
-	apiCmd.Flags().BoolVarP(&credsOnly, "creds-only", "C", false, "Return credentials only")
+	dehashedCmd.Flags().BoolVarP(&credsOnly, "creds-only", "C", false, "Return credentials only")
-	apiCmd.Flags().StringVarP(&outputFormat, "format", "f", "json", "Output format (json, yaml, xml, txt)")
+	dehashedCmd.Flags().StringVarP(&outputFormat, "format", "f", "json", "Output format (json, yaml, xml, txt)")
-	apiCmd.Flags().StringVarP(&outputFile, "output", "o", "query", "File to output results to including extension")
+	dehashedCmd.Flags().StringVarP(&outputFile, "output", "o", "query", "File to output results to including extension")
-	apiCmd.Flags().StringVarP(&usernameQuery, "username", "U", "", "Username query")
+	dehashedCmd.Flags().StringVarP(&usernameQuery, "username", "U", "", "Username query")
-	apiCmd.Flags().StringVarP(&emailQuery, "email-query", "E", "", "HunterEmail query")
+	dehashedCmd.Flags().StringVarP(&emailQuery, "email-query", "E", "", "HunterEmail query")
-	apiCmd.Flags().StringVarP(&ipQuery, "ip", "I", "", "IP address query")
+	dehashedCmd.Flags().StringVarP(&ipQuery, "ip", "I", "", "IP address query")
-	apiCmd.Flags().StringVarP(&domainQuery, "domain", "D", "", "Domain query")
+	dehashedCmd.Flags().StringVarP(&domainQuery, "domain", "D", "", "Domain query")
-	apiCmd.Flags().StringVarP(&passwordQuery, "password", "P", "", "Password query")
+	dehashedCmd.Flags().StringVarP(&passwordQuery, "password", "P", "", "Password query")
-	apiCmd.Flags().StringVarP(&vinQuery, "vin", "V", "", "VIN query")
+	dehashedCmd.Flags().StringVarP(&vinQuery, "vin", "V", "", "VIN query")
-	apiCmd.Flags().StringVarP(&licensePlateQuery, "license", "L", "", "License plate query")
+	dehashedCmd.Flags().StringVarP(&licensePlateQuery, "license", "L", "", "License plate query")
-	apiCmd.Flags().StringVarP(&addressQuery, "address", "A", "", "Address query")
+	dehashedCmd.Flags().StringVarP(&addressQuery, "address", "A", "", "Address query")
-	apiCmd.Flags().StringVarP(&phoneQuery, "phone", "M", "", "Phone query")
+	dehashedCmd.Flags().StringVarP(&phoneQuery, "phone", "M", "", "Phone query")
-	apiCmd.Flags().StringVarP(&socialQuery, "social", "S", "", "Social query")
+	dehashedCmd.Flags().StringVarP(&socialQuery, "social", "S", "", "Social query")
-	apiCmd.Flags().StringVarP(&cryptoCurrencyAddressQuery, "crypto", "B", "", "Crypto currency address query")
+	dehashedCmd.Flags().StringVarP(&cryptoCurrencyAddressQuery, "crypto", "B", "", "Crypto currency address query")
-	apiCmd.Flags().StringVarP(&hashQuery, "hash", "Q", "", "Hashed password query")
+	dehashedCmd.Flags().StringVarP(&hashQuery, "hash", "Q", "", "Hashed password query")
-	apiCmd.Flags().StringVarP(&nameQuery, "name", "N", "", "Name query")
+	dehashedCmd.Flags().StringVarP(&nameQuery, "name", "N", "", "Name query")
 
 	// Add mutually exclusive flags to wildcard match and regex match
-	apiCmd.MarkFlagsMutuallyExclusive("regex-match", "wildcard-match")
+	dehashedCmd.MarkFlagsMutuallyExclusive("regex-match", "wildcard-match")
 }
 
 var (
@@ -68,8 +68,8 @@ var (
 	cryptoCurrencyAddressQuery string
 
 	// Query command
-	apiCmd = &cobra.Command{
+	dehashedCmd = &cobra.Command{
-		Use:   "api",
+		Use:   "dehashed",
 		Short: "Query the Dehashed API",
 		Long:  `Query the Dehashed API for emails, usernames, passwords, hashes, IP addresses, and names.`,
 		Run: func(cmd *cobra.Command, args []string) {
@@ -118,7 +118,7 @@ var (
 			dehasher.Start()
 			fmt.Println("\n[*] Completing Process")
 
-			err := sqlite.StoreQueryOptions(queryOptions)
+			err := sqlite.StoreDehashedQueryOptions(queryOptions)
 			if err != nil {
 				if debugGlobal {
 					debug.PrintInfo("failed to store query options")

cmd/export.go

@@ -1,9 +1,9 @@
 package cmd
 
 import (
-	"dehasher/internal/export"
+	"crowsnest/internal/export"
-	"dehasher/internal/files"
+	"crowsnest/internal/files"
-	"dehasher/internal/sqlite"
+	"crowsnest/internal/sqlite"
 	"fmt"
 	"github.com/spf13/cobra"
 	"go.uber.org/zap"

cmd/hunter.go

@@ -1,12 +1,12 @@
 package cmd
 
 import (
-	"dehasher/internal/badger"
+	"crowsnest/internal/badger"
-	"dehasher/internal/debug"
+	"crowsnest/internal/debug"
-	"dehasher/internal/export"
+	"crowsnest/internal/export"
-	"dehasher/internal/files"
+	"crowsnest/internal/files"
-	hunter "dehasher/internal/hunter.io"
+	hunter "crowsnest/internal/hunter.io"
-	"dehasher/internal/pretty"
+	"crowsnest/internal/pretty"
 	"fmt"
 	"github.com/spf13/cobra"
 	"go.uber.org/zap"
@@ -113,7 +113,7 @@ var (
 
 				// Write Hunter.io Domain Search Result to file
 				fmt.Printf("[*] Writing Hunter.io Domain Search Result to file: %s%s\n", hunterOutputFile, fType.Extension())
-				err = export.WriteHunterDomainToFile(result, hunterOutputFile, fType)
+				err = export.WriteIStringToFile(result, hunterOutputFile, fType)
 				if err != nil {
 					if debugGlobal {
 						debug.PrintInfo("failed to write hunter domain search to file")
@@ -150,7 +150,7 @@ var (
 
 				// Write Hunter.io Email Finder Result to file
 				fmt.Printf("[*] Writing Hunter.io Email Finder Result to file: %s%s\n", hunterOutputFile, fType.Extension())
-				err = export.WriteHunterEmailToFile(result, hunterOutputFile, fType)
+				err = export.WriteIStringToFile(result, hunterOutputFile, fType)
 				if err != nil {
 					if debugGlobal {
 						debug.PrintInfo("failed to write hunter email find to file")
@@ -206,7 +206,7 @@ var (
 				}
 				// Write Hunter.io Email Verification Result to file
 				fmt.Printf("[*] Writing Hunter.io Email Verification Result to file: %s%s\n", hunterOutputFile, fType.Extension())
-				err = export.WriteHunterEmailVerifyToFile(result, hunterOutputFile, fType)
+				err = export.WriteIStringToFile(result, hunterOutputFile, fType)
 				if err != nil {
 					if debugGlobal {
 						debug.PrintInfo("failed to write hunter email verification to file")
@@ -264,7 +264,7 @@ var (
 
 				// Write to file
 				fmt.Printf("[*] Writing Hunter.io Company Enrichment Result to file: %s%s\n", hunterOutputFile, fType.Extension())
-				err = export.WriteHunterCompanyEnrichmentToFile(result, hunterOutputFile, fType)
+				err = export.WriteIStringToFile(result, hunterOutputFile, fType)
 				if err != nil {
 					if debugGlobal {
 						debug.PrintInfo("failed to write hunter company enrichment to file")
@@ -302,7 +302,7 @@ var (
 
 				// Write to file
 				fmt.Printf("[*] Writing Hunter.io Person Enrichment Result to file: %s%s\n", hunterOutputFile, fType.Extension())
-				err = export.WriteHunterPersonEnrichmentToFile(result, hunterOutputFile, fType)
+				err = export.WriteIStringToFile(result, hunterOutputFile, fType)
 				if err != nil {
 					if debugGlobal {
 						debug.PrintInfo("failed to write hunter person enrichment to file")
@@ -339,7 +339,7 @@ var (
 
 				// Write to file
 				fmt.Printf("[*] Writing Hunter.io Combined Enrichment Result to file: %s%s\n", hunterOutputFile, fType.Extension())
-				err = export.WriteHunterCombinedEnrichmentToFile(result, hunterOutputFile, fType)
+				err = export.WriteIStringToFile(result, hunterOutputFile, fType)
 				if err != nil {
 					if debugGlobal {
 						debug.PrintInfo("failed to write hunter combined enrichment to file")

cmd/logs.go

@@ -1,8 +1,8 @@
 package cmd
 
 import (
-	"dehasher/internal/easyTime"
+	"crowsnest/internal/easyTime"
-	"dehasher/internal/pretty"
+	"crowsnest/internal/pretty"
 	"encoding/json"
 	"fmt"
 	"github.com/spf13/cobra"

cmd/query.go

@@ -1,8 +1,8 @@
 package cmd
 
 import (
-	"dehasher/internal/pretty"
+	"crowsnest/internal/pretty"
-	"dehasher/internal/sqlite"
+	"crowsnest/internal/sqlite"
 	"encoding/json"
 	"fmt"
 	"github.com/spf13/cobra"

cmd/root.go

@@ -1,8 +1,9 @@
 package cmd
 
 import (
-	"dehasher/internal/badger"
+	"crowsnest/internal/badger"
 	"fmt"
+	"github.com/fatih/color"
 	"github.com/spf13/cobra"
 	"go.uber.org/zap"
 	"os"
@@ -18,28 +19,14 @@ var (
 		Use:   "dehasher",
 		Short: `Dehasher is a cli tool for querying the dehashed api.`,
 		Long: fmt.Sprintf(
-			"%s\n%s",
+			"%s\n",
 			`
- ______   _______           _______  _______           _______  _______
+    ╔═╗┬─┐┌─┐┬ ┬┌─┐╔╗╔┌─┐┌─┐┌┬┐
-(  __  \ (  ____ \|\     /|(  ___  )(  ____ \|\     /|(  ____ \(  ____ )
+    ║  ├┬┘│ ││││└─┐║║║├┤ └─┐ │
-| (  \  )| (    \/| )   ( || (   ) || (    \/| )   ( || (    \/| (    )|
+    ╚═╝┴└─└─┘└┴┘└─┘╝╚╝└─┘└─┘ ┴
-| |   ) || (__    | (___) || (___) || (_____ | (___) || (__    | (____)|
+
-| |   | ||  __)   |  ___  ||  ___  |(_____  )|  ___  ||  __)   |     __)
+   Crow’s Nest OSINT Recon Suite
-| |   ) || (      | (   ) || (   ) |      ) || (   ) || (      | (\ (
+ ⚓ A KrakenTech Intelligence Tool
-| (__/  )| (____/\| )   ( || )   ( |/\____) || )   ( || (____/\| ) \ \__
-(______/ (_______/|/     \||/     \|\_______)|/     \|(_______/|/   \__/
-						     An Ar1ste1a Project
-`,
-			`––•–√\/––√\/––•––––•–√\/––√\/––•––––•–√\/––√\/––•––√\/––•––––•–√\/––√\/––•––
-  Dehasher can query the query API for:
-  - Emails		- Usernames 		- Password
-  - Hashes 		- IP Addresses		- Names
-  - VINs		- License Plates	- Addresses
-  - Phones		- Social Media		- Crypto Currency Addresses
-  Dehasher supports:
-  - Regex Matching
-  - Exact Matching
-––•–√\/––√\/––•––––•–√\/––√\/––•––––•–√\/––√\/––•––√\/––•––––•–√\/––√\/––•––
 `,
 		),
 		Version: "v1.2.1",
@@ -69,6 +56,7 @@ func init() {
 	rootCmd.AddCommand(setDehashedKeyCmd)
 	rootCmd.AddCommand(setHunterKeyCmd)
 	rootCmd.AddCommand(setLocalDb)
+	rootCmd.AddCommand(buyMeCoffeeCmd)
 }
 
 // Command to set API key
@@ -132,6 +120,20 @@ var setLocalDb = &cobra.Command{
 	},
 }
 
+var buyMeCoffeeCmd = &cobra.Command{
+	Use:   "coffee",
+	Short: "Support the project by buying a coffee",
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println(color.HiRedString("                                                                                   ;)(; "))
+		fmt.Println(color.HiCyanString("                   We Hope You Enjoy Our Product                                  :----:"))
+		fmt.Println(color.HiCyanString("                                                                                 C|====|"))
+		fmt.Println(color.HiCyanString("                                                                                  |    |"))
+		fmt.Print(color.HiGreenString(" Support the project by buying a coffee: "))
+		fmt.Print(color.BlueString("https://buymeacoffee.com/ehosinskiz      "))
+		fmt.Println(color.HiCyanString("`----'"))
+	},
+}
+
 // Helper functions to store API credentials
 func storeDehashedApiKey(key string) error {
 	err := badger.StoreDehashedKey(key)

cmd/whois.go

@@ -1,12 +1,12 @@
 package cmd
 
 import (
-	"dehasher/internal/debug"
+	"crowsnest/internal/debug"
-	"dehasher/internal/export"
+	"crowsnest/internal/export"
-	"dehasher/internal/files"
+	"crowsnest/internal/files"
-	"dehasher/internal/pretty"
+	"crowsnest/internal/pretty"
-	"dehasher/internal/sqlite"
+	"crowsnest/internal/sqlite"
-	"dehasher/internal/whois"
+	"crowsnest/internal/whois"
 	"fmt"
 	"github.com/spf13/cobra"
 	"go.uber.org/zap"
@@ -208,7 +208,7 @@ var (
 								fmt.Printf("[!] Error writing WHOIS history to file: %v\n", writeErr)
 							}
 
-							err = sqlite.StoreHistoryRecord(historyRecords)
+							err = sqlite.StoreWhoisHistoryRecords(historyRecords)
 							if err != nil {
 								if debugGlobal {
 									debug.PrintInfo("failed to store history record")
@@ -254,7 +254,7 @@ var (
 						fmt.Printf("Error performing subdomain scan: %v\n", err)
 					} else {
 						fmt.Println("Subdomain Scan:")
-						err = sqlite.StoreSubdomainRecords(subdomains)
+						err = sqlite.StoreWhoisSubdomainRecords(subdomains)
 						if err != nil {
 							if debugGlobal {
 								debug.PrintInfo("failed to store subdomain record")

crowsnest.go

@@ -1,9 +1,9 @@
 package main
 
 import (
-	"dehasher/cmd"
+	"crowsnest/cmd"
-	"dehasher/internal/badger"
+	"crowsnest/internal/badger"
-	"dehasher/internal/sqlite"
+	"crowsnest/internal/sqlite"
 	"fmt"
 	"github.com/winking324/rzap"
 	"go.uber.org/zap"

go.mod

@@ -1,4 +1,4 @@
-module dehasher
+module crowsnest
 
 go 1.23.0
 
@@ -7,7 +7,7 @@ toolchain go1.24.3
 require (
 	github.com/charmbracelet/lipgloss v1.1.0
 	github.com/dgraph-io/badger/v4 v4.7.0
-	github.com/olekukonko/tablewriter v1.0.5
+	github.com/fatih/color v1.15.0
 	github.com/spf13/cobra v1.9.1
 	github.com/winking324/rzap v0.1.0
 	go.uber.org/zap v1.20.0
@@ -27,7 +27,6 @@ require (
 	github.com/charmbracelet/x/term v0.2.1 // indirect
 	github.com/dgraph-io/ristretto/v2 v2.2.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
-	github.com/fatih/color v1.15.0 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/google/flatbuffers v25.2.10+incompatible // indirect
@@ -41,8 +40,6 @@ require (
 	github.com/mattn/go-runewidth v0.0.16 // indirect
 	github.com/mattn/go-sqlite3 v1.14.22 // indirect
 	github.com/muesli/termenv v0.16.0 // indirect
-	github.com/olekukonko/errors v0.0.0-20250405072817-4e6d85265da6 // indirect
-	github.com/olekukonko/ll v0.0.7 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/spf13/pflag v1.0.6 // indirect
 	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect

go.sum

@@ -71,12 +71,6 @@ github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o
 github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
 github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
-github.com/olekukonko/errors v0.0.0-20250405072817-4e6d85265da6 h1:r3FaAI0NZK3hSmtTDrBVREhKULp8oUeqLT5Eyl2mSPo=
-github.com/olekukonko/errors v0.0.0-20250405072817-4e6d85265da6/go.mod h1:ppzxA5jBKcO1vIpCXQ9ZqgDh8iwODz6OXIGKU8r5m4Y=
-github.com/olekukonko/ll v0.0.7 h1:K66xcUlG2qWRhPoLw/cidmbv4pDDJtZuvJGsR5QTzXo=
-github.com/olekukonko/ll v0.0.7/go.mod h1:En+sEW0JNETl26+K8eZ6/W4UQ7CYSrrgg/EdIYT2H8g=
-github.com/olekukonko/tablewriter v1.0.5 h1:8+uKJXxYcl29TcpfQdd0vL+l6Kul7Sk7sWolfgErDv0=
-github.com/olekukonko/tablewriter v1.0.5/go.mod h1:Z22i2ywMkT9sw64nuWAUaH62kb+umiwucGaQNbFh8Bg=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=

internal/dehashed/clientv2.go

@@ -2,9 +2,9 @@ package dehashed
 
 import (
 	"bytes"
+	"crowsnest/internal/debug"
+	"crowsnest/internal/sqlite"
 	"crypto/sha256"
-	"dehasher/internal/debug"
-	"dehasher/internal/sqlite"
 	"encoding/hex"
 	"encoding/json"
 	"errors"

internal/dehashed/dehashed.go

@@ -1,9 +1,9 @@
 package dehashed
 
 import (
-	"dehasher/internal/debug"
+	"crowsnest/internal/debug"
-	"dehasher/internal/export"
+	"crowsnest/internal/export"
-	"dehasher/internal/sqlite"
+	"crowsnest/internal/sqlite"
 	"encoding/json"
 	"fmt"
 	"go.uber.org/zap"
@@ -130,7 +130,7 @@ func (dh *Dehasher) Start() {
 
 			if len(dh.client.results) > 0 {
 				fmt.Printf("   [!] Partial results retrieved. Storing Results...\n")
-				err := sqlite.StoreResults(dh.client.GetResults())
+				err := sqlite.StoreDehashedResults(dh.client.GetResults())
 				if err != nil {
 					zap.L().Error("store_results",
 						zap.String("message", "failed to store results"),
@@ -214,7 +214,7 @@ func (dh *Dehasher) parseResults() {
 	results := dh.client.GetResults()
 	creds := results.ExtractCredentials()
 	fmt.Printf("\n\t[+] Discovered %d Credentials", len(creds))
-	err := sqlite.StoreCreds(creds)
+	err := sqlite.StoreDehashedCreds(creds)
 	if err != nil {
 		zap.L().Error("store_creds",
 			zap.String("message", "failed to store creds"),
@@ -224,7 +224,7 @@ func (dh *Dehasher) parseResults() {
 	zap.L().Info("creds_stored", zap.Int("count", len(creds)))
 
 	zap.L().Info("storing_results")
-	err = sqlite.StoreResults(results)
+	err = sqlite.StoreDehashedResults(results)
 	if err != nil {
 		zap.L().Error("store_results",
 			zap.String("message", "failed to store results"),

internal/easyTime/parser.go

@@ -1,7 +1,7 @@
 package easyTime
 
 import (
-	"dehasher/internal/debug"
+	"crowsnest/internal/debug"
 	"fmt"
 	"go.uber.org/zap"
 	"os"

internal/export/dehashed.go

@@ -1,8 +1,8 @@
 package export
 
 import (
-	"dehasher/internal/files"
+	"crowsnest/internal/files"
-	"dehasher/internal/sqlite"
+	"crowsnest/internal/sqlite"
 	"encoding/json"
 	"encoding/xml"
 	"errors"

internal/export/hunter.go

@@ -1,161 +0,0 @@
-package export
-
-import (
-	"dehasher/internal/files"
-	"dehasher/internal/sqlite"
-	"encoding/json"
-	"encoding/xml"
-	"fmt"
-	"gopkg.in/yaml.v3"
-	"os"
-)
-
-func WriteHunterDomainToFile(result sqlite.HunterDomainData, outputFile string, fileType files.FileType) error {
-	var data []byte
-	var err error
-
-	switch fileType {
-	case files.JSON:
-		data, err = json.MarshalIndent(result, "", "  ")
-	case files.XML:
-		data, err = xml.MarshalIndent(result, "", "  ")
-	case files.YAML:
-		data, err = yaml.Marshal(result)
-	case files.TEXT:
-		data = []byte(result.String())
-	default:
-		return err
-	}
-
-	if err != nil {
-		return err
-	}
-
-	filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String())
-	return os.WriteFile(filePath, data, 0644)
-}
-
-func WriteHunterEmailToFile(result sqlite.HunterEmailFinderData, outputFile string, fileType files.FileType) error {
-	var data []byte
-	var err error
-
-	switch fileType {
-	case files.JSON:
-		data, err = json.MarshalIndent(result, "", "  ")
-	case files.XML:
-		data, err = xml.MarshalIndent(result, "", "  ")
-	case files.YAML:
-		data, err = yaml.Marshal(result)
-	case files.TEXT:
-		data = []byte(result.String())
-	default:
-		return err
-	}
-
-	if err != nil {
-		return err
-	}
-
-	filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String())
-	return os.WriteFile(filePath, data, 0644)
-}
-
-func WriteHunterEmailVerifyToFile(result sqlite.HunterEmailVerifyData, outputFile string, fileType files.FileType) error {
-	var data []byte
-	var err error
-
-	switch fileType {
-	case files.JSON:
-		data, err = json.MarshalIndent(result, "", "  ")
-	case files.XML:
-		data, err = xml.MarshalIndent(result, "", "  ")
-	case files.YAML:
-		data, err = yaml.Marshal(result)
-	case files.TEXT:
-		data = []byte(result.String())
-	default:
-		return err
-	}
-
-	if err != nil {
-		return err
-	}
-
-	filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String())
-	return os.WriteFile(filePath, data, 0644)
-}
-
-func WriteHunterCompanyEnrichmentToFile(result sqlite.CompanyData, outputFile string, fileType files.FileType) error {
-	var data []byte
-	var err error
-
-	switch fileType {
-	case files.JSON:
-		data, err = json.MarshalIndent(result, "", "  ")
-	case files.XML:
-		data, err = xml.MarshalIndent(result, "", "  ")
-	case files.YAML:
-		data, err = yaml.Marshal(result)
-	case files.TEXT:
-		data = []byte(result.String())
-	default:
-		return err
-	}
-
-	if err != nil {
-		return err
-	}
-
-	filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String())
-	return os.WriteFile(filePath, data, 0644)
-}
-
-func WriteHunterPersonEnrichmentToFile(result sqlite.PersonData, outputFile string, fileType files.FileType) error {
-	var data []byte
-	var err error
-
-	switch fileType {
-	case files.JSON:
-		data, err = json.MarshalIndent(result, "", "  ")
-	case files.XML:
-		data, err = xml.MarshalIndent(result, "", "  ")
-	case files.YAML:
-		data, err = yaml.Marshal(result)
-	case files.TEXT:
-		data = []byte(result.String())
-	default:
-		return err
-	}
-
-	if err != nil {
-		return err
-	}
-
-	filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String())
-	return os.WriteFile(filePath, data, 0644)
-}
-
-func WriteHunterCombinedEnrichmentToFile(result sqlite.CombinedData, outputFile string, fileType files.FileType) error {
-	var data []byte
-	var err error
-
-	switch fileType {
-	case files.JSON:
-		data, err = json.MarshalIndent(result, "", "  ")
-	case files.XML:
-		data, err = xml.MarshalIndent(result, "", "  ")
-	case files.YAML:
-		data, err = yaml.Marshal(result)
-	case files.TEXT:
-		data = []byte(result.String())
-	default:
-		return err
-	}
-
-	if err != nil {
-		return err
-	}
-
-	filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String())
-	return os.WriteFile(filePath, data, 0644)
-}

internal/export/iString.go

@@ -0,0 +1,36 @@
+package export
+
+import (
+	"crowsnest/internal/files"
+	"crowsnest/internal/sqlite"
+	"encoding/json"
+	"encoding/xml"
+	"fmt"
+	"gopkg.in/yaml.v3"
+	"os"
+)
+
+func WriteIStringToFile(iString sqlite.IString, outputFile string, fileType files.FileType) error {
+	var data []byte
+	var err error
+
+	switch fileType {
+	case files.JSON:
+		data, err = json.MarshalIndent(iString, "", "  ")
+	case files.XML:
+		data, err = xml.MarshalIndent(iString, "", "  ")
+	case files.YAML:
+		data, err = yaml.Marshal(iString)
+	case files.TEXT:
+		data = []byte(iString.String())
+	default:
+		return err
+	}
+
+	if err != nil {
+		return err
+	}
+
+	filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String())
+	return os.WriteFile(filePath, data, 0644)
+}

internal/hunter.io/hunter.io.go

@@ -1,8 +1,8 @@
 package hunter_io
 
 import (
-	"dehasher/internal/debug"
+	"crowsnest/internal/debug"
-	"dehasher/internal/sqlite"
+	"crowsnest/internal/sqlite"
 	"encoding/json"
 	"fmt"
 	"go.uber.org/zap"

internal/pretty/trees.go

@@ -1,7 +1,7 @@
 package pretty
 
 import (
-	"dehasher/internal/sqlite"
+	"crowsnest/internal/sqlite"
 	"fmt"
 	"github.com/charmbracelet/lipgloss"
 	"github.com/charmbracelet/lipgloss/tree"

internal/sqlite/db.go

@@ -0,0 +1,143 @@
+package sqlite
+
+import (
+	"fmt"
+	"go.uber.org/zap"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
+)
+
+var DB *gorm.DB
+
+// InitDB initializes the database connection
+func InitDB(dbPath string) (*gorm.DB, error) {
+	zap.L().Info("Initializing database", zap.String("path", dbPath))
+
+	// Check if the path is a file or directory
+	fileInfo, err := os.Stat(dbPath)
+	var finalDbPath string
+
+	// If path doesn't exist or is a directory
+	if os.IsNotExist(err) || (err == nil && fileInfo.IsDir()) {
+		// Treat as directory path
+		if err := os.MkdirAll(dbPath, 0755); err != nil {
+			zap.L().Error("Failed to create database directory", zap.Error(err))
+			return nil, fmt.Errorf("failed to create database directory: %w", err)
+		}
+		finalDbPath = filepath.Join(dbPath, "dehashed.sqlite")
+	} else {
+		// Treat as file path
+		// Ensure the directory exists
+		dir := filepath.Dir(dbPath)
+		if err := os.MkdirAll(dir, 0755); err != nil {
+			zap.L().Error("Failed to create parent directory for database", zap.Error(err))
+			return nil, fmt.Errorf("failed to create parent directory for database: %w", err)
+		}
+		finalDbPath = dbPath
+	}
+
+	zap.L().Info("Opening database", zap.String("finalPath", finalDbPath))
+	db, err := gorm.Open(sqlite.Open(finalDbPath), &gorm.Config{
+		Logger: logger.Default.LogMode(logger.Silent),
+	})
+	if err != nil {
+		zap.L().Error("Failed to connect to database", zap.Error(err))
+		return nil, fmt.Errorf("failed to connect to database: %w", err)
+	}
+
+	// Auto migrate your models
+	err = db.AutoMigrate(&Result{}, &Creds{}, &QueryOptions{}, &Creds{}, &WhoisRecord{}, &SubdomainRecord{},
+		&HistoryRecord{}, &LookupResult{}, &HunterDomainData{}, &HunterEmail{}, &PersonData{})
+	if err != nil {
+		zap.L().Error("Failed to migrate database", zap.Error(err))
+		return nil, fmt.Errorf("failed to migrate database: %w", err)
+	}
+
+	DB = db
+	return db, nil
+}
+
+// GetDB returns the database connection
+func GetDB() *gorm.DB {
+	if DB == nil {
+		zap.L().Error("database not initialized")
+		fmt.Println("sqlite database not initialized")
+		os.Exit(1)
+	}
+	return DB
+}
+
+type Table int64
+
+const (
+	ResultsTable Table = iota
+	RunsTable
+	CredsTable
+	WhoIsTable
+	SubdomainsTable
+	HistoryTable
+	LookupTable
+	HunterDomainTable
+	HunterEmailTable
+	PersonTable
+	UnknownTable
+)
+
+func GetTable(userInput string) Table {
+	switch strings.ToLower(userInput) {
+	case "results":
+		return ResultsTable
+	case "runs":
+		return RunsTable
+	case "creds":
+		return CredsTable
+	case "whois":
+		return WhoIsTable
+	case "subdomains":
+		return SubdomainsTable
+	case "history":
+		return HistoryTable
+	case "lookup":
+		return LookupTable
+	case "hunter_domain":
+		return HunterDomainTable
+	case "hunter_email":
+		return HunterEmailTable
+	case "person":
+		return PersonTable
+	default:
+		return UnknownTable
+	}
+}
+
+func (t Table) Object() interface{} {
+	switch t {
+	case ResultsTable:
+		return Result{}
+	case RunsTable:
+		return QueryOptions{}
+	case CredsTable:
+		return Creds{}
+	case WhoIsTable:
+		return WhoisRecord{}
+	case SubdomainsTable:
+		return SubdomainRecord{}
+	case HistoryTable:
+		return HistoryRecord{}
+	case LookupTable:
+		return LookupResult{}
+	case HunterDomainTable:
+		return HunterDomainData{}
+	case HunterEmailTable:
+		return HunterEmail{}
+	case PersonTable:
+		return PersonData{}
+	default:
+		return nil
+	}
+}

internal/sqlite/dehashed.go

@@ -0,0 +1,235 @@
+package sqlite
+
+import (
+	"crowsnest/internal/files"
+	"fmt"
+	"go.uber.org/zap"
+	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
+)
+
+type QueryOptions struct {
+	gorm.Model
+	MaxRecords         int            `json:"max_records"`
+	MaxRequests        int            `json:"max_requests"`
+	StartingPage       int            `json:"starting_page"`
+	OutputFormat       files.FileType `json:"output_format"`
+	OutputFile         string         `json:"output_file"`
+	RegexMatch         bool           `json:"regex_match"`
+	WildcardMatch      bool           `json:"wildcard_match"`
+	UsernameQuery      string         `json:"username_query"`
+	EmailQuery         string         `json:"email_query"`
+	IpQuery            string         `json:"ip_query"`
+	PassQuery          string         `json:"pass_query"`
+	HashQuery          string         `json:"hash_query"`
+	NameQuery          string         `json:"name_query"`
+	DomainQuery        string         `json:"domain_query"`
+	VinQuery           string         `json:"vin_query"`
+	LicensePlateQuery  string         `json:"license_plate_query"`
+	AddressQuery       string         `json:"address_query"`
+	PhoneQuery         string         `json:"phone_query"`
+	SocialQuery        string         `json:"social_query"`
+	CryptoAddressQuery string         `json:"crypto_address_query"`
+	PrintBalance       bool           `json:"print_balance"`
+	CredsOnly          bool           `json:"creds_only"`
+	Debug              bool           `json:"debug"`
+}
+
+func (QueryOptions) TableName() string {
+	return "query_options"
+}
+
+func NewQueryOptions(maxRecords, maxRequests, startingPage int, outputFormat, outputFile, usernameQuery, emailQuery, ipQuery, passQuery, hashQuery, nameQuery, domainQuery, vinQuery, licensePlateQuery, addressQuery, phoneQuery, socialQuery, cryptoAddressQuery string, regexMatch, wildcardMatch, printBalance, credsOnly, debug bool) *QueryOptions {
+	return &QueryOptions{
+		MaxRecords:         maxRecords,
+		MaxRequests:        maxRequests,
+		StartingPage:       startingPage,
+		OutputFormat:       files.GetFileType(outputFormat),
+		OutputFile:         outputFile,
+		PrintBalance:       printBalance,
+		CredsOnly:          credsOnly,
+		RegexMatch:         regexMatch,
+		WildcardMatch:      wildcardMatch,
+		UsernameQuery:      usernameQuery,
+		EmailQuery:         emailQuery,
+		IpQuery:            ipQuery,
+		PassQuery:          passQuery,
+		HashQuery:          hashQuery,
+		NameQuery:          nameQuery,
+		DomainQuery:        domainQuery,
+		VinQuery:           vinQuery,
+		LicensePlateQuery:  licensePlateQuery,
+		AddressQuery:       addressQuery,
+		PhoneQuery:         phoneQuery,
+		SocialQuery:        socialQuery,
+		CryptoAddressQuery: cryptoAddressQuery,
+		Debug:              debug,
+	}
+}
+
+type DehashedSearchRequest struct {
+	Page     int    `json:"page"`
+	Query    string `json:"query"`
+	Size     int    `json:"size"`
+	Wildcard bool   `json:"wildcard"`
+	Regex    bool   `json:"regex"`
+	DeDupe   bool   `json:"de_dupe"`
+}
+
+type DehashedResponse struct {
+	Balance      int      `json:"balance"`
+	Entries      []Result `json:"entries"`
+	Success      bool     `json:"success"`
+	Took         string   `json:"took"`
+	TotalResults int      `json:"total"`
+}
+
+type Result struct {
+	gorm.Model
+	DehashedId            string   `json:"id" xml:"id" yaml:"id" gorm:"uniqueIndex"`
+	Email                 []string `json:"email,omitempty" xml:"email,omitempty" yaml:"email,omitempty" gorm:"serializer:json"`
+	IpAddress             []string `json:"ip_address,omitempty" xml:"ip_address,omitempty" yaml:"ip_address,omitempty" gorm:"serializer:json"`
+	Username              []string `json:"username,omitempty" xml:"username,omitempty" yaml:"username,omitempty" gorm:"serializer:json"`
+	Password              []string `json:"password,omitempty" xml:"password,omitempty" yaml:"password,omitempty" gorm:"serializer:json"`
+	HashedPassword        []string `json:"hashed_password,omitempty" xml:"hashed_password,omitempty" yaml:"hashed_password,omitempty" gorm:"serializer:json"`
+	HashType              string   `json:"hash_type,omitempty" xml:"hash_type,omitempty" yaml:"hash_type,omitempty"`
+	Name                  []string `json:"name,omitempty" xml:"name,omitempty" yaml:"name,omitempty" gorm:"serializer:json"`
+	Vin                   []string `json:"vin,omitempty" xml:"vin,omitempty" yaml:"vin,omitempty" gorm:"serializer:json"`
+	LicensePlate          []string `json:"license_plate,omitempty" xml:"license_plate,omitempty" yaml:"license_plate,omitempty" gorm:"serializer:json"`
+	Url                   []string `json:"url,omitempty" xml:"url,omitempty" yaml:"url,omitempty" gorm:"serializer:json"`
+	Social                []string `json:"social,omitempty" xml:"social,omitempty" yaml:"social,omitempty" gorm:"serializer:json"`
+	CryptoCurrencyAddress []string `json:"cryptocurrency_address,omitempty" xml:"cryptocurrency_address,omitempty" yaml:"cryptocurrency_address,omitempty" gorm:"serializer:json"`
+	Address               []string `json:"address,omitempty" xml:"address,omitempty" yaml:"address,omitempty" gorm:"serializer:json"`
+	Phone                 []string `json:"phone,omitempty" xml:"phone,omitempty" yaml:"phone,omitempty" gorm:"serializer:json"`
+	Company               []string `json:"company,omitempty" xml:"company,omitempty" yaml:"company,omitempty" gorm:"serializer:json"`
+	DatabaseName          string   `json:"database_name,omitempty" xml:"database_name,omitempty" yaml:"database_name,omitempty"`
+}
+
+func (Result) TableName() string {
+	return "results"
+}
+
+type DehashedResults struct {
+	Results []Result `json:"results"`
+}
+
+func (dr *DehashedResults) ExtractCredentials() []Creds {
+	var creds []Creds
+
+	results := dr.Results
+
+	for _, r := range results {
+		if len(r.Password) > 0 {
+			// Get first email if available
+			email := ""
+			if len(r.Email) > 0 {
+				email = r.Email[0]
+			}
+
+			// Get first password
+			password := r.Password[0]
+
+			cred := Creds{Email: email, Password: password}
+			creds = append(creds, cred)
+		}
+	}
+
+	go func() {
+		err := StoreDehashedCreds(creds)
+		if err != nil {
+			zap.L().Error("store_creds",
+				zap.String("message", "failed to store creds"),
+				zap.Error(err),
+			)
+			fmt.Printf("Error Storing Results: %v", err)
+		}
+	}()
+
+	return creds
+}
+
+type Creds struct {
+	gorm.Model
+	Email    string `json:"email" yaml:"email" xml:"email" gorm:"uniqueIndex:idx_email_username_password"`
+	Username string `json:"username" yaml:"username" xml:"username" gorm:"uniqueIndex:idx_email_username_password"`
+	Password string `json:"password" yaml:"password" xml:"password" gorm:"uniqueIndex:idx_email_username_password"`
+}
+
+func (Creds) TableName() string {
+	return "creds"
+}
+
+func (c Creds) ToString() string {
+	return fmt.Sprintf("%s%s%s", c.Username, "%", c.Password)
+}
+
+func StoreDehashedResults(results DehashedResults) error {
+	if len(results.Results) == 0 {
+		return nil
+	}
+
+	zap.L().Info("Storing results", zap.Int("count", len(results.Results)))
+	db := GetDB()
+
+	// Use batch insert with conflict handling
+	const batchSize = 100
+	var lastErr error
+
+	// Extract the slice of results
+	resultSlice := results.Results
+
+	for i := 0; i < len(resultSlice); i += batchSize {
+		end := i + batchSize
+		if end > len(resultSlice) {
+			end = len(resultSlice)
+		}
+
+		batch := resultSlice[i:end]
+		// Use Clauses with OnConflict DoNothing to skip conflicts
+		err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
+		if err != nil {
+			zap.L().Warn("Error storing some results", zap.Error(err))
+			lastErr = err
+			// Continue with next batch despite error
+		}
+	}
+
+	return lastErr
+}
+
+func StoreDehashedCreds(creds []Creds) error {
+	if len(creds) == 0 {
+		return nil
+	}
+
+	zap.L().Info("Storing credentials", zap.Int("count", len(creds)))
+	db := GetDB()
+
+	// Use batch insert with conflict handling
+	// This will insert records in batches and continue even if some fail
+	const batchSize = 100
+	var lastErr error
+
+	for i := 0; i < len(creds); i += batchSize {
+		end := i + batchSize
+		if end > len(creds) {
+			end = len(creds)
+		}
+
+		batch := creds[i:end]
+		// Use Clauses with OnConflict DoNothing to skip conflicts
+		err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
+		if err != nil {
+			zap.L().Warn("Error storing some credentials", zap.Error(err))
+			lastErr = err
+			// Continue with next batch despite error
+		}
+	}
+
+	return lastErr
+}
+
+func StoreDehashedQueryOptions(queryOptions *QueryOptions) error {
+	db := GetDB()
+	return db.Create(queryOptions).Error
+}

internal/sqlite/gorm.go

@@ -1,319 +0,0 @@
-package sqlite
-
-import (
-	"fmt"
-	"go.uber.org/zap"
-	"gorm.io/gorm/clause"
-	"os"
-	"path/filepath"
-
-	"gorm.io/driver/sqlite"
-	"gorm.io/gorm"
-	"gorm.io/gorm/logger"
-)
-
-var DB *gorm.DB
-
-// InitDB initializes the database connection
-func InitDB(dbPath string) (*gorm.DB, error) {
-	zap.L().Info("Initializing database", zap.String("path", dbPath))
-
-	// Check if the path is a file or directory
-	fileInfo, err := os.Stat(dbPath)
-	var finalDbPath string
-
-	// If path doesn't exist or is a directory
-	if os.IsNotExist(err) || (err == nil && fileInfo.IsDir()) {
-		// Treat as directory path
-		if err := os.MkdirAll(dbPath, 0755); err != nil {
-			zap.L().Error("Failed to create database directory", zap.Error(err))
-			return nil, fmt.Errorf("failed to create database directory: %w", err)
-		}
-		finalDbPath = filepath.Join(dbPath, "dehashed.sqlite")
-	} else {
-		// Treat as file path
-		// Ensure the directory exists
-		dir := filepath.Dir(dbPath)
-		if err := os.MkdirAll(dir, 0755); err != nil {
-			zap.L().Error("Failed to create parent directory for database", zap.Error(err))
-			return nil, fmt.Errorf("failed to create parent directory for database: %w", err)
-		}
-		finalDbPath = dbPath
-	}
-
-	zap.L().Info("Opening database", zap.String("finalPath", finalDbPath))
-	db, err := gorm.Open(sqlite.Open(finalDbPath), &gorm.Config{
-		Logger: logger.Default.LogMode(logger.Silent),
-	})
-	if err != nil {
-		zap.L().Error("Failed to connect to database", zap.Error(err))
-		return nil, fmt.Errorf("failed to connect to database: %w", err)
-	}
-
-	// Auto migrate your models
-	err = db.AutoMigrate(&Result{}, &Creds{}, &QueryOptions{}, &Creds{}, &WhoisRecord{}, &SubdomainRecord{},
-		&HistoryRecord{}, &LookupResult{}, &HunterDomainData{}, &HunterEmail{}, &PersonData{})
-	if err != nil {
-		zap.L().Error("Failed to migrate database", zap.Error(err))
-		return nil, fmt.Errorf("failed to migrate database: %w", err)
-	}
-
-	DB = db
-	return db, nil
-}
-
-// GetDB returns the database connection
-func GetDB() *gorm.DB {
-	if DB == nil {
-		zap.L().Error("database not initialized")
-		fmt.Println("sqlite database not initialized")
-		os.Exit(1)
-	}
-	return DB
-}
-
-func StoreResults(results DehashedResults) error {
-	if len(results.Results) == 0 {
-		return nil
-	}
-
-	zap.L().Info("Storing results", zap.Int("count", len(results.Results)))
-	db := GetDB()
-
-	// Use batch insert with conflict handling
-	const batchSize = 100
-	var lastErr error
-
-	// Extract the slice of results
-	resultSlice := results.Results
-
-	for i := 0; i < len(resultSlice); i += batchSize {
-		end := i + batchSize
-		if end > len(resultSlice) {
-			end = len(resultSlice)
-		}
-
-		batch := resultSlice[i:end]
-		// Use Clauses with OnConflict DoNothing to skip conflicts
-		err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
-		if err != nil {
-			zap.L().Warn("Error storing some results", zap.Error(err))
-			lastErr = err
-			// Continue with next batch despite error
-		}
-	}
-
-	return lastErr
-}
-
-func StoreCreds(creds []Creds) error {
-	if len(creds) == 0 {
-		return nil
-	}
-
-	zap.L().Info("Storing credentials", zap.Int("count", len(creds)))
-	db := GetDB()
-
-	// Use batch insert with conflict handling
-	// This will insert records in batches and continue even if some fail
-	const batchSize = 100
-	var lastErr error
-
-	for i := 0; i < len(creds); i += batchSize {
-		end := i + batchSize
-		if end > len(creds) {
-			end = len(creds)
-		}
-
-		batch := creds[i:end]
-		// Use Clauses with OnConflict DoNothing to skip conflicts
-		err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
-		if err != nil {
-			zap.L().Warn("Error storing some credentials", zap.Error(err))
-			lastErr = err
-			// Continue with next batch despite error
-		}
-	}
-
-	return lastErr
-}
-
-func StoreQueryOptions(queryOptions *QueryOptions) error {
-	db := GetDB()
-	return db.Create(queryOptions).Error
-}
-
-func StoreWhoisRecord(whoisRecord WhoisRecord) error {
-	// Create a pointer to the record to make it addressable
-	recordPtr := &whoisRecord
-
-	zap.L().Info("Storing WHOIS record",
-		zap.String("domain", whoisRecord.DomainName))
-
-	db := GetDB()
-
-	// Use OnConflict clause to handle duplicates
-	err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(recordPtr).Error
-	if err != nil {
-		zap.L().Error("store_whois_record",
-			zap.String("message", "failed to store whois record"),
-			zap.Error(err))
-		return err
-	}
-
-	return nil
-}
-
-func StoreSubdomainRecords(subdomainRecords []SubdomainRecord) error {
-	if len(subdomainRecords) == 0 {
-		return nil
-	}
-
-	zap.L().Info("Storing subdomain records", zap.Int("count", len(subdomainRecords)))
-	db := GetDB()
-
-	// Use batch insert with conflict handling
-	const batchSize = 100
-	var lastErr error
-
-	for i := 0; i < len(subdomainRecords); i += batchSize {
-		end := i + batchSize
-		if end > len(subdomainRecords) {
-			end = len(subdomainRecords)
-		}
-
-		batch := subdomainRecords[i:end]
-		// Use Clauses with OnConflict DoNothing to skip conflicts
-		err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
-		if err != nil {
-			zap.L().Warn("Error storing some subdomain records", zap.Error(err))
-			lastErr = err
-			// Continue with next batch despite error
-		}
-	}
-
-	return lastErr
-}
-
-func StoreHistoryRecord(historyRecords []HistoryRecord) error {
-	if len(historyRecords) == 0 {
-		return nil
-	}
-
-	zap.L().Info("Storing history records", zap.Int("count", len(historyRecords)))
-	db := GetDB()
-
-	// Use batch insert with conflict handling
-	const batchSize = 100
-	var lastErr error
-
-	for i := 0; i < len(historyRecords); i += batchSize {
-		end := i + batchSize
-		if end > len(historyRecords) {
-			end = len(historyRecords)
-		}
-
-		batch := historyRecords[i:end]
-		// Use Clauses with OnConflict DoNothing to skip conflicts
-		err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
-		if err != nil {
-			zap.L().Warn("Error storing some history records", zap.Error(err))
-			lastErr = err
-			// Continue with next batch despite error
-		}
-	}
-
-	return lastErr
-}
-
-func StoreIPLookup(ipLookup []LookupResult) error {
-	if len(ipLookup) == 0 {
-		return nil
-	}
-
-	zap.L().Info("Storing IP lookup records", zap.Int("count", len(ipLookup)))
-	db := GetDB()
-
-	// Use batch insert with conflict handling
-	const batchSize = 100
-	var lastErr error
-
-	for i := 0; i < len(ipLookup); i += batchSize {
-		end := i + batchSize
-		if end > len(ipLookup) {
-			end = len(ipLookup)
-		}
-
-		batch := ipLookup[i:end]
-		// Use Clauses with OnConflict DoNothing to skip conflicts
-		err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
-		if err != nil {
-			zap.L().Warn("Error storing some IP lookup records", zap.Error(err))
-			lastErr = err
-			// Continue with next batch despite error
-		}
-	}
-
-	return lastErr
-}
-
-func StoreHunterDomain(hunterDomain HunterDomainData) error {
-	db := GetDB()
-
-	// Use OnConflict clause to handle duplicates
-	err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(&hunterDomain).Error
-	if err != nil {
-		zap.L().Error("store_hunter_domain",
-			zap.String("message", "failed to store hunter domain"),
-			zap.Error(err))
-		return err
-	}
-
-	return nil
-}
-
-func StoreHunterEmails(hunterEmails []HunterEmail) error {
-	if len(hunterEmails) == 0 {
-		return nil
-	}
-
-	zap.L().Info("Storing hunter emails", zap.Int("count", len(hunterEmails)))
-	db := GetDB()
-
-	// Use batch insert with conflict handling
-	const batchSize = 100
-	var lastErr error
-
-	for i := 0; i < len(hunterEmails); i += batchSize {
-		end := i + batchSize
-		if end > len(hunterEmails) {
-			end = len(hunterEmails)
-		}
-
-		batch := hunterEmails[i:end]
-		// Use Clauses with OnConflict DoNothing to skip conflicts
-		err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
-		if err != nil {
-			zap.L().Warn("Error storing some hunter emails", zap.Error(err))
-			lastErr = err
-			// Continue with next batch despite error
-		}
-	}
-
-	return lastErr
-}
-
-func StorePersonData(personData PersonData) error {
-	db := GetDB()
-
-	// Use OnConflict clause to handle duplicates
-	err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(&personData).Error
-	if err != nil {
-		zap.L().Error("store_person_data",
-			zap.String("message", "failed to store person data"),
-			zap.Error(err))
-		return err
-	}
-
-	return nil
-}

internal/sqlite/hunter.io.go

@@ -3,7 +3,9 @@ package sqlite
 import (
 	"fmt"
 	"github.com/charmbracelet/lipgloss/tree"
+	"go.uber.org/zap"
 	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
 )
 
 // HunterDomainSearchResult represents the response from Hunter.io domain search API
@@ -15,6 +17,7 @@ type HunterDomainSearchResult struct {
 // HunterDomainData contains the main domain information
 type HunterDomainData struct {
 	gorm.Model
+	IString       `gorm:"-"`
 	Domain        string        `json:"domain" gorm:"unique"`
 	Disposable    bool          `json:"disposable"`
 	Webmail       bool          `json:"webmail"`
@@ -40,7 +43,7 @@ type HunterDomainData struct {
 	LinkedDomains []string      `json:"linked_domains" gorm:"serializer:json"`
 }
 
-func (h *HunterDomainData) String() string {
+func (h HunterDomainData) String() string {
 	return fmt.Sprintf("Domain: %s\nDisposable: %t\nWebmail: %t\nAcceptAll: %t\nPattern: %s\nOrganization: %s\nDescription: %s\nIndustry: %s\nTwitter: %s\nFacebook: %s\nLinkedin: %s\nInstagram: %s\nYoutube: %s\nTechnologies: %v\nCountry: %s\nState: %s\nCity: %s\nPostalCode: %s\nStreet: %s\nHeadcount: %s\nCompanyType: %s\nEmails: %v\nLinkedDomains: %v\n",
 		h.Domain, h.Disposable, h.Webmail, h.AcceptAll, h.Pattern, h.Organization, h.Description, h.Industry, h.Twitter, h.Facebook, h.Linkedin, h.Instagram, h.Youtube, h.Technologies, h.Country, h.State, h.City, h.PostalCode, h.Street, h.Headcount, h.CompanyType, h.Emails, h.LinkedDomains)
 }
@@ -142,6 +145,7 @@ type HunterEmailFinderResponse struct {
 
 // HunterEmailFinderData contains the main email information
 type HunterEmailFinderData struct {
+	IString
 	FirstName    string             `json:"first_name"`
 	LastName     string             `json:"last_name"`
 	Email        string             `json:"email"`
@@ -157,7 +161,7 @@ type HunterEmailFinderData struct {
 	Verification HunterVerification `json:"verification" gorm:"embedded;embeddedPrefix:verification_"`
 }
 
-func (he *HunterEmailFinderData) String() string {
+func (he HunterEmailFinderData) String() string {
 	return fmt.Sprintf("FirstName: %s\nLastName: %s\nEmail: %s\nScore: %d\nDomain: %s\nAcceptAll: %t\nPosition: %s\nTwitter: %s\nLinkedinURL: %s\nPhoneNumber: %s\nCompany: %s\nSources: %v\nVerification: %v\n",
 		he.FirstName, he.LastName, he.Email, he.Score, he.Domain, he.AcceptAll, he.Position, he.Twitter, he.LinkedinURL, he.PhoneNumber, he.Company, he.Sources, he.Verification)
 }
@@ -189,6 +193,7 @@ type HunterEmailVerifyResponse struct {
 
 // HunterEmailVerifyData contains the email verification information
 type HunterEmailVerifyData struct {
+	IString
 	Status            string         `json:"status"`
 	Result            string         `json:"result"`
 	DeprecationNotice string         `json:"_deprecation_notice"`
@@ -206,7 +211,7 @@ type HunterEmailVerifyData struct {
 	Sources           []HunterSource `json:"sources" gorm:"serializer:json"`
 }
 
-func (ev *HunterEmailVerifyData) String() string {
+func (ev HunterEmailVerifyData) String() string {
 	return fmt.Sprintf("Status: %s\nResult: %s\nDeprecationNotice: %s\nScore: %d\nEmail: %s\nRegexp: %t\nGibberish: %t\nDisposable: %t\nWebmail: %t\nMXRecords: %t\nSMTPServer: %t\nSMTPCheck: %t\nAcceptAll: %t\nBlock: %t\nSources: %v\n",
 		ev.Status, ev.Result, ev.DeprecationNotice, ev.Score, ev.Email, ev.Regexp, ev.Gibberish, ev.Disposable, ev.Webmail, ev.MXRecords, ev.SMTPServer, ev.SMTPCheck, ev.AcceptAll, ev.Block, ev.Sources)
 }
@@ -229,6 +234,7 @@ type HunterCompanyEnrichmentResponse struct {
 
 // CompanyData contains the detailed company information
 type CompanyData struct {
+	IString
 	ID             string        `json:"id"`
 	Name           string        `json:"name"`
 	LegalName      string        `json:"legalName"`
@@ -262,7 +268,7 @@ type CompanyData struct {
 	UltimateParent ParentCompany `json:"ultimateParent" gorm:"embedded;embeddedPrefix:ultimate_parent_"`
 }
 
-func (cd *CompanyData) String() string {
+func (cd CompanyData) String() string {
 	return fmt.Sprintf("ID: %s\nName: %s\nLegalName: %s\nDomain: %s\nDomainAliases: %v\nSite: %v\nCategory: %v\nTags: %v\nDescription: %s\nFoundedYear: %d\nLocation: %s\nTimeZone: %s\nUTCOffset: %d\nGeo: %v\nLogo: %s\nFacebook: %v\nLinkedIn: %v\nTwitter: %v\nCrunchbase: %v\nYouTube: %v\nEmailProvider: %s\nType: %s\nTicker: %s\nIdentifiers: %v\nPhone: %s\nMetrics: %v\nIndexedAt: %s\nTech: %v\nTechCategories: %v\nParent: %v\nUltimateParent: %v\n",
 		cd.ID, cd.Name, cd.LegalName, cd.Domain, cd.DomainAliases, cd.Site, cd.Category, cd.Tags, cd.Description, cd.FoundedYear, cd.Location, cd.TimeZone, cd.UTCOffset, cd.Geo, cd.Logo, cd.Facebook, cd.LinkedIn, cd.Twitter, cd.Crunchbase, cd.YouTube, cd.EmailProvider, cd.Type, cd.Ticker, cd.Identifiers, cd.Phone, cd.Metrics, cd.IndexedAt, cd.Tech, cd.TechCategories, cd.Parent, cd.UltimateParent)
 }
@@ -509,6 +515,7 @@ type HunterPersonEnrichmentResponse struct {
 // PersonData contains the detailed person information
 type PersonData struct {
 	gorm.Model
+	IString       `gorm:"-"`
 	ID            string     `json:"id"`
 	Name          PersonName `json:"name" gorm:"embedded;embeddedPrefix:name_"`
 	Email         string     `json:"email" gorm:"unique"`
@@ -534,7 +541,7 @@ type PersonData struct {
 	InactiveAt    string     `json:"inactiveAt"`
 }
 
-func (pd *PersonData) String() string {
+func (pd PersonData) String() string {
 	return fmt.Sprintf("ID: %s\nName: %v\nEmail: %s\nLocation: %s\nTimeZone: %s\nUTCOffset: %d\nGeo: %v\nBio: %s\nSite: %s\nAvatar: %s\nEmployment: %v\nFacebook: %v\nGitHub: %v\nTwitter: %v\nLinkedIn: %v\nGooglePlus: %v\nGravatar: %v\nFuzzy: %t\nEmailProvider: %s\nIndexedAt: %s\nPhone: %s\nActiveAt: %s\nInactiveAt: %s\n",
 		pd.ID, pd.Name, pd.Email, pd.Location, pd.TimeZone, pd.UTCOffset, pd.Geo, pd.Bio, pd.Site, pd.Avatar, pd.Employment, pd.Facebook, pd.GitHub, pd.Twitter, pd.LinkedIn, pd.GooglePlus, pd.Gravatar, pd.Fuzzy, pd.EmailProvider, pd.IndexedAt, pd.Phone, pd.ActiveAt, pd.InactiveAt)
 }
@@ -728,11 +735,12 @@ type HunterCombinedEnrichmentResponse struct {
 
 // CombinedData contains both person and company information
 type CombinedData struct {
+	IString
 	Person  PersonData  `json:"person" gorm:"embedded;embeddedPrefix:person_"`
 	Company CompanyData `json:"company" gorm:"embedded;embeddedPrefix:company_"`
 }
 
-func (cbd *CombinedData) String() string {
+func (cbd CombinedData) String() string {
 	return fmt.Sprintf("Person: %s\nCompany: %s",
 		cbd.Person.String(),
 		cbd.Company.String())
@@ -749,3 +757,64 @@ func (c *HunterCombinedEnrichmentResponse) String() string {
 		c.Data.Person.String(),
 		c.Data.Company.String())
 }
+
+func StoreHunterDomain(hunterDomain HunterDomainData) error {
+	db := GetDB()
+
+	// Use OnConflict clause to handle duplicates
+	err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(&hunterDomain).Error
+	if err != nil {
+		zap.L().Error("store_hunter_domain",
+			zap.String("message", "failed to store hunter domain"),
+			zap.Error(err))
+		return err
+	}
+
+	return nil
+}
+
+func StoreHunterEmails(hunterEmails []HunterEmail) error {
+	if len(hunterEmails) == 0 {
+		return nil
+	}
+
+	zap.L().Info("Storing hunter emails", zap.Int("count", len(hunterEmails)))
+	db := GetDB()
+
+	// Use batch insert with conflict handling
+	const batchSize = 100
+	var lastErr error
+
+	for i := 0; i < len(hunterEmails); i += batchSize {
+		end := i + batchSize
+		if end > len(hunterEmails) {
+			end = len(hunterEmails)
+		}
+
+		batch := hunterEmails[i:end]
+		// Use Clauses with OnConflict DoNothing to skip conflicts
+		err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
+		if err != nil {
+			zap.L().Warn("Error storing some hunter emails", zap.Error(err))
+			lastErr = err
+			// Continue with next batch despite error
+		}
+	}
+
+	return lastErr
+}
+
+func StoreHunterPersonData(personData PersonData) error {
+	db := GetDB()
+
+	// Use OnConflict clause to handle duplicates
+	err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(&personData).Error
+	if err != nil {
+		zap.L().Error("store_person_data",
+			zap.String("message", "failed to store person data"),
+			zap.Error(err))
+		return err
+	}
+
+	return nil
+}

internal/sqlite/query.go

@@ -1,20 +0,0 @@
-package sqlite
-
-type DehashedSearchRequest struct {
-	Page     int    `json:"page"`
-	Query    string `json:"query"`
-	Size     int    `json:"size"`
-	Wildcard bool   `json:"wildcard"`
-	Regex    bool   `json:"regex"`
-	DeDupe   bool   `json:"de_dupe"`
-}
-
-func NewDehashedSearchRequest(size int, wildcard, regex bool) *DehashedSearchRequest {
-	return &DehashedSearchRequest{
-		Page:     0,
-		Size:     size,
-		Wildcard: false,
-		Regex:    false,
-		DeDupe:   true,
-	}
-}

internal/sqlite/result.go

@@ -1,94 +0,0 @@
-package sqlite
-
-import (
-	"encoding/json"
-	"fmt"
-	"go.uber.org/zap"
-	"gorm.io/gorm"
-	"io"
-	"os"
-)
-
-type DehashedResponse struct {
-	Balance      int      `json:"balance"`
-	Entries      []Result `json:"entries"`
-	Success      bool     `json:"success"`
-	Took         string   `json:"took"`
-	TotalResults int      `json:"total"`
-}
-
-type Result struct {
-	gorm.Model
-	DehashedId            string   `json:"id" xml:"id" yaml:"id" gorm:"uniqueIndex"`
-	Email                 []string `json:"email,omitempty" xml:"email,omitempty" yaml:"email,omitempty" gorm:"serializer:json"`
-	IpAddress             []string `json:"ip_address,omitempty" xml:"ip_address,omitempty" yaml:"ip_address,omitempty" gorm:"serializer:json"`
-	Username              []string `json:"username,omitempty" xml:"username,omitempty" yaml:"username,omitempty" gorm:"serializer:json"`
-	Password              []string `json:"password,omitempty" xml:"password,omitempty" yaml:"password,omitempty" gorm:"serializer:json"`
-	HashedPassword        []string `json:"hashed_password,omitempty" xml:"hashed_password,omitempty" yaml:"hashed_password,omitempty" gorm:"serializer:json"`
-	HashType              string   `json:"hash_type,omitempty" xml:"hash_type,omitempty" yaml:"hash_type,omitempty"`
-	Name                  []string `json:"name,omitempty" xml:"name,omitempty" yaml:"name,omitempty" gorm:"serializer:json"`
-	Vin                   []string `json:"vin,omitempty" xml:"vin,omitempty" yaml:"vin,omitempty" gorm:"serializer:json"`
-	LicensePlate          []string `json:"license_plate,omitempty" xml:"license_plate,omitempty" yaml:"license_plate,omitempty" gorm:"serializer:json"`
-	Url                   []string `json:"url,omitempty" xml:"url,omitempty" yaml:"url,omitempty" gorm:"serializer:json"`
-	Social                []string `json:"social,omitempty" xml:"social,omitempty" yaml:"social,omitempty" gorm:"serializer:json"`
-	CryptoCurrencyAddress []string `json:"cryptocurrency_address,omitempty" xml:"cryptocurrency_address,omitempty" yaml:"cryptocurrency_address,omitempty" gorm:"serializer:json"`
-	Address               []string `json:"address,omitempty" xml:"address,omitempty" yaml:"address,omitempty" gorm:"serializer:json"`
-	Phone                 []string `json:"phone,omitempty" xml:"phone,omitempty" yaml:"phone,omitempty" gorm:"serializer:json"`
-	Company               []string `json:"company,omitempty" xml:"company,omitempty" yaml:"company,omitempty" gorm:"serializer:json"`
-	DatabaseName          string   `json:"database_name,omitempty" xml:"database_name,omitempty" yaml:"database_name,omitempty"`
-}
-
-func (Result) TableName() string {
-	return "results"
-}
-
-type DehashedResults struct {
-	Results []Result `json:"results"`
-}
-
-func (dr *DehashedResults) ExtractCredentials() []Creds {
-	var creds []Creds
-
-	results := dr.Results
-
-	for _, r := range results {
-		if len(r.Password) > 0 {
-			// Get first email if available
-			email := ""
-			if len(r.Email) > 0 {
-				email = r.Email[0]
-			}
-
-			// Get first password
-			password := r.Password[0]
-
-			cred := Creds{Email: email, Password: password}
-			creds = append(creds, cred)
-		}
-	}
-
-	go func() {
-		err := StoreCreds(creds)
-		if err != nil {
-			zap.L().Error("store_creds",
-				zap.String("message", "failed to store creds"),
-				zap.Error(err),
-			)
-			fmt.Printf("Error Storing Results: %v", err)
-		}
-	}()
-
-	return creds
-}
-
-func NewDehashedResults(body io.Reader) ([]Result, int, int) {
-	var response DehashedResponse
-
-	err := json.NewDecoder(body).Decode(&response)
-	if err != nil {
-		fmt.Printf("Error Parsing Response Body: %v", err)
-		os.Exit(-1)
-	}
-
-	return response.Entries, response.Balance, response.TotalResults
-}

internal/sqlite/structs.go

@@ -1,10 +1,8 @@
 package sqlite
 
-import (
+type IString interface {
-	"dehasher/internal/files"
+	String() string
-	"fmt"
+}
-	"gorm.io/gorm"
-)
 
 type DBOptions struct {
 	Username              string
@@ -26,15 +24,6 @@ type DBOptions struct {
 	DisplayFields         []string // Fields to display in output
 }
 
-func NewDBOptions() *DBOptions {
-	return &DBOptions{
-		Limit:          100, // Default limit
-		ExactMatch:     false,
-		NonEmptyFields: []string{},
-		DisplayFields:  []string{},
-	}
-}
-
 func (o *DBOptions) Empty() bool {
 	return o.Username == "" && o.Email == "" && o.IPAddress == "" &&
 		o.Password == "" && o.HashedPassword == "" && o.Name == "" &&
@@ -42,77 +31,3 @@ func (o *DBOptions) Empty() bool {
 		o.Phone == "" && o.Social == "" && o.CryptoCurrencyAddress == "" && o.Domain == "" &&
 		len(o.NonEmptyFields) == 0
 }
-
-type QueryOptions struct {
-	gorm.Model
-	MaxRecords         int            `json:"max_records"`
-	MaxRequests        int            `json:"max_requests"`
-	StartingPage       int            `json:"starting_page"`
-	OutputFormat       files.FileType `json:"output_format"`
-	OutputFile         string         `json:"output_file"`
-	RegexMatch         bool           `json:"regex_match"`
-	WildcardMatch      bool           `json:"wildcard_match"`
-	UsernameQuery      string         `json:"username_query"`
-	EmailQuery         string         `json:"email_query"`
-	IpQuery            string         `json:"ip_query"`
-	PassQuery          string         `json:"pass_query"`
-	HashQuery          string         `json:"hash_query"`
-	NameQuery          string         `json:"name_query"`
-	DomainQuery        string         `json:"domain_query"`
-	VinQuery           string         `json:"vin_query"`
-	LicensePlateQuery  string         `json:"license_plate_query"`
-	AddressQuery       string         `json:"address_query"`
-	PhoneQuery         string         `json:"phone_query"`
-	SocialQuery        string         `json:"social_query"`
-	CryptoAddressQuery string         `json:"crypto_address_query"`
-	PrintBalance       bool           `json:"print_balance"`
-	CredsOnly          bool           `json:"creds_only"`
-	Debug              bool           `json:"debug"`
-}
-
-func (QueryOptions) TableName() string {
-	return "query_options"
-}
-
-func NewQueryOptions(maxRecords, maxRequests, startingPage int, outputFormat, outputFile, usernameQuery, emailQuery, ipQuery, passQuery, hashQuery, nameQuery, domainQuery, vinQuery, licensePlateQuery, addressQuery, phoneQuery, socialQuery, cryptoAddressQuery string, regexMatch, wildcardMatch, printBalance, credsOnly, debug bool) *QueryOptions {
-	return &QueryOptions{
-		MaxRecords:         maxRecords,
-		MaxRequests:        maxRequests,
-		StartingPage:       startingPage,
-		OutputFormat:       files.GetFileType(outputFormat),
-		OutputFile:         outputFile,
-		PrintBalance:       printBalance,
-		CredsOnly:          credsOnly,
-		RegexMatch:         regexMatch,
-		WildcardMatch:      wildcardMatch,
-		UsernameQuery:      usernameQuery,
-		EmailQuery:         emailQuery,
-		IpQuery:            ipQuery,
-		PassQuery:          passQuery,
-		HashQuery:          hashQuery,
-		NameQuery:          nameQuery,
-		DomainQuery:        domainQuery,
-		VinQuery:           vinQuery,
-		LicensePlateQuery:  licensePlateQuery,
-		AddressQuery:       addressQuery,
-		PhoneQuery:         phoneQuery,
-		SocialQuery:        socialQuery,
-		CryptoAddressQuery: cryptoAddressQuery,
-		Debug:              debug,
-	}
-}
-
-type Creds struct {
-	gorm.Model
-	Email    string `json:"email" yaml:"email" xml:"email" gorm:"uniqueIndex:idx_email_username_password"`
-	Username string `json:"username" yaml:"username" xml:"username" gorm:"uniqueIndex:idx_email_username_password"`
-	Password string `json:"password" yaml:"password" xml:"password" gorm:"uniqueIndex:idx_email_username_password"`
-}
-
-func (Creds) TableName() string {
-	return "creds"
-}
-
-func (c Creds) ToString() string {
-	return fmt.Sprintf("%s%s%s", c.Username, "%", c.Password)
-}

internal/sqlite/tables.go

@@ -1,68 +0,0 @@
-package sqlite
-
-import "strings"
-
-type Table int64
-
-const (
-	ResultsTable Table = iota
-	RunsTable
-	CredsTable
-	WhoIsTable
-	SubdomainsTable
-	HistoryTable
-	LookupTable
-	HunterDomainTable
-	HunterEmailTable
-	UnknownTable
-)
-
-func GetTable(userInput string) Table {
-	switch strings.ToLower(userInput) {
-	case "results":
-		return ResultsTable
-	case "runs":
-		return RunsTable
-	case "creds":
-		return CredsTable
-	case "whois":
-		return WhoIsTable
-	case "subdomains":
-		return SubdomainsTable
-	case "history":
-		return HistoryTable
-	case "lookup":
-		return LookupTable
-	case "hunter_domain":
-		return HunterDomainTable
-	case "hunter_email":
-		return HunterEmailTable
-	default:
-		return UnknownTable
-	}
-}
-
-func (t Table) Object() interface{} {
-	switch t {
-	case ResultsTable:
-		return Result{}
-	case RunsTable:
-		return QueryOptions{}
-	case CredsTable:
-		return Creds{}
-	case WhoIsTable:
-		return WhoisRecord{}
-	case SubdomainsTable:
-		return SubdomainRecord{}
-	case HistoryTable:
-		return HistoryRecord{}
-	case LookupTable:
-		return LookupResult{}
-	case HunterDomainTable:
-		return HunterDomainData{}
-	case HunterEmailTable:
-		return HunterEmail{}
-	default:
-		return nil
-	}
-}

internal/sqlite/whois.go

@@ -2,7 +2,9 @@ package sqlite
 
 import (
 	"fmt"
+	"go.uber.org/zap"
 	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
 	"strings"
 )
 
@@ -512,3 +514,117 @@ func formatWhoisContact(sb *strings.Builder, contact Contact, indent string) {
 		sb.WriteString(indent + "Raw Text: " + rawTextPreview + "\n")
 	}
 }
+
+func StoreWhoisRecord(whoisRecord WhoisRecord) error {
+	// Create a pointer to the record to make it addressable
+	recordPtr := &whoisRecord
+
+	zap.L().Info("Storing WHOIS record",
+		zap.String("domain", whoisRecord.DomainName))
+
+	db := GetDB()
+
+	// Use OnConflict clause to handle duplicates
+	err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(recordPtr).Error
+	if err != nil {
+		zap.L().Error("store_whois_record",
+			zap.String("message", "failed to store whois record"),
+			zap.Error(err))
+		return err
+	}
+
+	return nil
+}
+
+func StoreWhoisSubdomainRecords(subdomainRecords []SubdomainRecord) error {
+	if len(subdomainRecords) == 0 {
+		return nil
+	}
+
+	zap.L().Info("Storing subdomain records", zap.Int("count", len(subdomainRecords)))
+	db := GetDB()
+
+	// Use batch insert with conflict handling
+	const batchSize = 100
+	var lastErr error
+
+	for i := 0; i < len(subdomainRecords); i += batchSize {
+		end := i + batchSize
+		if end > len(subdomainRecords) {
+			end = len(subdomainRecords)
+		}
+
+		batch := subdomainRecords[i:end]
+		// Use Clauses with OnConflict DoNothing to skip conflicts
+		err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
+		if err != nil {
+			zap.L().Warn("Error storing some subdomain records", zap.Error(err))
+			lastErr = err
+			// Continue with next batch despite error
+		}
+	}
+
+	return lastErr
+}
+
+func StoreWhoisHistoryRecords(historyRecords []HistoryRecord) error {
+	if len(historyRecords) == 0 {
+		return nil
+	}
+
+	zap.L().Info("Storing history records", zap.Int("count", len(historyRecords)))
+	db := GetDB()
+
+	// Use batch insert with conflict handling
+	const batchSize = 100
+	var lastErr error
+
+	for i := 0; i < len(historyRecords); i += batchSize {
+		end := i + batchSize
+		if end > len(historyRecords) {
+			end = len(historyRecords)
+		}
+
+		batch := historyRecords[i:end]
+		// Use Clauses with OnConflict DoNothing to skip conflicts
+		err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
+		if err != nil {
+			zap.L().Warn("Error storing some history records", zap.Error(err))
+			lastErr = err
+			// Continue with next batch despite error
+		}
+	}
+
+	return lastErr
+}
+
+func StoreWhoisLookup(lookup []LookupResult) error {
+	if len(lookup) == 0 {
+		return nil
+	}
+
+	zap.L().Info("Storing IP lookup records", zap.Int("count", len(lookup)))
+	db := GetDB()
+
+	// Use batch insert with conflict handling
+	const batchSize = 100
+	var lastErr error
+
+	for i := 0; i < len(lookup); i += batchSize {
+		end := i + batchSize
+		if end > len(lookup) {
+			end = len(lookup)
+		}
+
+		batch := lookup[i:end]
+		// Use Clauses with OnConflict DoNothing to skip conflicts
+		err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
+		if err != nil {
+			zap.L().Warn("Error storing some IP lookup records", zap.Error(err))
+			lastErr = err
+			// Continue with next batch despite error
+		}
+	}
+
+	return lastErr
+}

internal/whois/whois.go

@@ -2,9 +2,9 @@ package whois
 
 import (
 	"bytes"
-	"dehasher/internal/debug"
+	"crowsnest/internal/debug"
-	"dehasher/internal/dehashed"
+	"crowsnest/internal/dehashed"
-	"dehasher/internal/sqlite"
+	"crowsnest/internal/sqlite"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -552,7 +552,7 @@ func (w *DehashedWhoIs) WhoisIP(ipAddress string) ([]sqlite.LookupResult, error)
 		})
 	}
 
-	sqlite.StoreIPLookup(lookups)
+	sqlite.StoreWhoisLookup(lookups)
 
 	return lookups, nil
 }
@@ -702,7 +702,7 @@ func (w *DehashedWhoIs) WhoisMX(mxHostname string) ([]sqlite.LookupResult, error
 		})
 	}
 
-	sqlite.StoreIPLookup(mxLookups)
+	sqlite.StoreWhoisLookup(mxLookups)
 
 	return mxLookups, nil
 }
@@ -849,7 +849,7 @@ func (w *DehashedWhoIs) WhoisNS(nsHostname string) ([]sqlite.LookupResult, error
 		})
 	}
 
-	sqlite.StoreIPLookup(nsLookups)
+	sqlite.StoreWhoisLookup(nsLookups)
 
 	return nsLookups, nil
 }