From 1152a1910cc6609274286331aca3cb296fff09c2 Mon Sep 17 00:00:00 2001 From: Evan Hosinski Date: Sat, 17 May 2025 09:03:04 -0400 Subject: [PATCH 1/5] updated api to dehashed to match naming convention of hunter. --- cmd/{api.go => dehashed.go} | 52 ++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 26 deletions(-) rename cmd/{api.go => dehashed.go} (54%) diff --git a/cmd/api.go b/cmd/dehashed.go similarity index 54% rename from cmd/api.go rename to cmd/dehashed.go index 0e84c98..18e501e 100644 --- a/cmd/api.go +++ b/cmd/dehashed.go @@ -12,34 +12,34 @@ import ( func init() { // Add api command to root command - rootCmd.AddCommand(apiCmd) + rootCmd.AddCommand(dehashedCmd) // Add flags specific to api command - apiCmd.Flags().IntVarP(&maxRecords, "max-records", "m", 30000, "Maximum amount of records to return") - apiCmd.Flags().IntVarP(&maxRequests, "max-requests", "r", -1, "Maximum number of requests to make") - apiCmd.Flags().IntVarP(&startingPage, "starting-page", "s", 1, "Starting page for requests") - apiCmd.Flags().BoolVarP(&printBalance, "print-balance", "b", false, "Print remaining balance after requests") - apiCmd.Flags().BoolVarP(®exMatch, "regex-match", "R", false, "Use regex matching on query fields") - apiCmd.Flags().BoolVarP(&wildcardMatch, "wildcard-match", "W", false, "Use wildcard matching on query fields (Use ? to replace a single character, and * for multiple characters)") - apiCmd.Flags().BoolVarP(&credsOnly, "creds-only", "C", false, "Return credentials only") - apiCmd.Flags().StringVarP(&outputFormat, "format", "f", "json", "Output format (json, yaml, xml, txt)") - apiCmd.Flags().StringVarP(&outputFile, "output", "o", "query", "File to output results to including extension") - apiCmd.Flags().StringVarP(&usernameQuery, "username", "U", "", "Username query") - apiCmd.Flags().StringVarP(&emailQuery, "email-query", "E", "", "HunterEmail query") - apiCmd.Flags().StringVarP(&ipQuery, "ip", "I", "", "IP address query") - apiCmd.Flags().StringVarP(&domainQuery, "domain", "D", "", "Domain query") - apiCmd.Flags().StringVarP(&passwordQuery, "password", "P", "", "Password query") - apiCmd.Flags().StringVarP(&vinQuery, "vin", "V", "", "VIN query") - apiCmd.Flags().StringVarP(&licensePlateQuery, "license", "L", "", "License plate query") - apiCmd.Flags().StringVarP(&addressQuery, "address", "A", "", "Address query") - apiCmd.Flags().StringVarP(&phoneQuery, "phone", "M", "", "Phone query") - apiCmd.Flags().StringVarP(&socialQuery, "social", "S", "", "Social query") - apiCmd.Flags().StringVarP(&cryptoCurrencyAddressQuery, "crypto", "B", "", "Crypto currency address query") - apiCmd.Flags().StringVarP(&hashQuery, "hash", "Q", "", "Hashed password query") - apiCmd.Flags().StringVarP(&nameQuery, "name", "N", "", "Name query") + dehashedCmd.Flags().IntVarP(&maxRecords, "max-records", "m", 30000, "Maximum amount of records to return") + dehashedCmd.Flags().IntVarP(&maxRequests, "max-requests", "r", -1, "Maximum number of requests to make") + dehashedCmd.Flags().IntVarP(&startingPage, "starting-page", "s", 1, "Starting page for requests") + dehashedCmd.Flags().BoolVarP(&printBalance, "print-balance", "b", false, "Print remaining balance after requests") + dehashedCmd.Flags().BoolVarP(®exMatch, "regex-match", "R", false, "Use regex matching on query fields") + dehashedCmd.Flags().BoolVarP(&wildcardMatch, "wildcard-match", "W", false, "Use wildcard matching on query fields (Use ? to replace a single character, and * for multiple characters)") + dehashedCmd.Flags().BoolVarP(&credsOnly, "creds-only", "C", false, "Return credentials only") + dehashedCmd.Flags().StringVarP(&outputFormat, "format", "f", "json", "Output format (json, yaml, xml, txt)") + dehashedCmd.Flags().StringVarP(&outputFile, "output", "o", "query", "File to output results to including extension") + dehashedCmd.Flags().StringVarP(&usernameQuery, "username", "U", "", "Username query") + dehashedCmd.Flags().StringVarP(&emailQuery, "email-query", "E", "", "HunterEmail query") + dehashedCmd.Flags().StringVarP(&ipQuery, "ip", "I", "", "IP address query") + dehashedCmd.Flags().StringVarP(&domainQuery, "domain", "D", "", "Domain query") + dehashedCmd.Flags().StringVarP(&passwordQuery, "password", "P", "", "Password query") + dehashedCmd.Flags().StringVarP(&vinQuery, "vin", "V", "", "VIN query") + dehashedCmd.Flags().StringVarP(&licensePlateQuery, "license", "L", "", "License plate query") + dehashedCmd.Flags().StringVarP(&addressQuery, "address", "A", "", "Address query") + dehashedCmd.Flags().StringVarP(&phoneQuery, "phone", "M", "", "Phone query") + dehashedCmd.Flags().StringVarP(&socialQuery, "social", "S", "", "Social query") + dehashedCmd.Flags().StringVarP(&cryptoCurrencyAddressQuery, "crypto", "B", "", "Crypto currency address query") + dehashedCmd.Flags().StringVarP(&hashQuery, "hash", "Q", "", "Hashed password query") + dehashedCmd.Flags().StringVarP(&nameQuery, "name", "N", "", "Name query") // Add mutually exclusive flags to wildcard match and regex match - apiCmd.MarkFlagsMutuallyExclusive("regex-match", "wildcard-match") + dehashedCmd.MarkFlagsMutuallyExclusive("regex-match", "wildcard-match") } var ( @@ -68,8 +68,8 @@ var ( cryptoCurrencyAddressQuery string // Query command - apiCmd = &cobra.Command{ - Use: "api", + dehashedCmd = &cobra.Command{ + Use: "dehashed", Short: "Query the Dehashed API", Long: `Query the Dehashed API for emails, usernames, passwords, hashes, IP addresses, and names.`, Run: func(cmd *cobra.Command, args []string) { From dc15315403199e2cc3a83af07267c6bda7fc7143 Mon Sep 17 00:00:00 2001 From: Evan Hosinski Date: Sat, 17 May 2025 10:00:59 -0400 Subject: [PATCH 2/5] Rebrand to CrowsNest Added coffee rootCmd --- Makefile | 4 +-- README.md | 12 ++++----- cmd/dehashed.go | 8 +++--- cmd/export.go | 6 ++--- cmd/hunter.go | 12 ++++----- cmd/logs.go | 4 +-- cmd/query.go | 4 +-- cmd/root.go | 46 +++++++++++++++++---------------- cmd/whois.go | 12 ++++----- dehasher.go | 6 ++--- go.mod | 2 +- internal/dehashed/clientv2.go | 4 +-- internal/dehashed/dehashed.go | 6 ++--- internal/easyTime/parser.go | 2 +- internal/export/export.go | 4 +-- internal/export/hunter.go | 4 +-- internal/hunter.io/hunter.io.go | 4 +-- internal/pretty/trees.go | 2 +- internal/sqlite/structs.go | 6 ++++- internal/whois/whois.go | 6 ++--- 20 files changed, 80 insertions(+), 74 deletions(-) diff --git a/Makefile b/Makefile index 4150194..9a730c6 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ GO=go # Binary name -BINARY_NAME=dehasher +BINARY_NAME=crowsnest # Build directory BUILD_DIR=build/bin @@ -16,7 +16,7 @@ PLATFORMS=linux darwin windows ARCHS=amd64 arm64 # Version info from git tag or default -VERSION=$(shell git describe --tags 2>/dev/null || echo "v1.2.1") +VERSION=$(shell git describe --tags 2>/dev/null || echo "v1.3.1") .PHONY: all clean build build-all diff --git a/README.md b/README.md index ee46b8f..50dd60d 100644 --- a/README.md +++ b/README.md @@ -221,7 +221,7 @@ dehasher set-hunter ### Domain Search Dehasher can perform a domain search for a given domain. -This provides a list of all emails that match the given query. +This provides information about company including a description, social media information and any technologies in use. ![Alt text](.img/hunter_domain_search.png "Hunter.io Domain Search") ```bash # Perform a Hunter.io domain search for example.com @@ -230,7 +230,7 @@ dehasher hunter -d example.com -D ### Email Finder Dehasher can perform an email finder search for a given domain, first name, and last name. -This provides a list of all emails that match the given query. +This provides information about a user including a confidence score, and any social media accounts linked to a first name, last name and email. ![Alt text](.img/hunter_email_finder.png "Hunter.io Email Finder") ```bash # Perform a Hunter.io email finder search for example.com @@ -239,7 +239,7 @@ dehasher hunter -d example.com -F John -L Doe -E ### Email Verification Dehasher can perform an email verification search for a given email. -This provides a list of all emails that match the given query. +This provides a verification and score of a given email address. ![Alt text](.img/email_verification.png "Hunter.io Email Verification") ```bash # Perform a Hunter.io email verification search for example@target.com @@ -248,7 +248,7 @@ dehasher hunter -e example@target.com -V ### Company Enrichment Dehasher can perform a company enrichment search for a given domain. -This provides a list of all emails that match the given query. +This provides information about a company given its domain. ![Alt text](.img/company_enrichment.png "Hunter.io Company Enrichment") ```bash # Perform a Hunter.io company enrichment search for example.com @@ -257,7 +257,7 @@ dehasher hunter -d example.com -C ### Person Enrichment Dehasher can perform a person enrichment search for a given email. -This provides a list of all emails that match the given query. +This provides information about a user given an email address.. ![Alt text](.img/person_enrichment.png "Hunter.io Person Enrichment") ```bash # Perform a Hunter.io person enrichment search for example@target.com @@ -266,7 +266,7 @@ dehasher hunter -e example@target.com -P ### Combined Enrichment Dehasher can perform a combined enrichment search for a given email. -This provides a list of all emails that match the given query. +This is a combination of the company and person enrichments given an email address. ![Alt text](.img/combined_enrichment_1.png "Hunter.io Combined Enrichment") ![Alt text](.img/combined_enrichment_2.png "Hunter.io Combined Enrichment") ```bash diff --git a/cmd/dehashed.go b/cmd/dehashed.go index 18e501e..4d6894a 100644 --- a/cmd/dehashed.go +++ b/cmd/dehashed.go @@ -1,10 +1,10 @@ package cmd import ( - "dehasher/internal/badger" - "dehasher/internal/debug" - "dehasher/internal/dehashed" - "dehasher/internal/sqlite" + "crowsnest/internal/badger" + "crowsnest/internal/debug" + "crowsnest/internal/dehashed" + "crowsnest/internal/sqlite" "fmt" "github.com/spf13/cobra" "go.uber.org/zap" diff --git a/cmd/export.go b/cmd/export.go index 533d946..076a051 100644 --- a/cmd/export.go +++ b/cmd/export.go @@ -1,9 +1,9 @@ package cmd import ( - "dehasher/internal/export" - "dehasher/internal/files" - "dehasher/internal/sqlite" + "crowsnest/internal/export" + "crowsnest/internal/files" + "crowsnest/internal/sqlite" "fmt" "github.com/spf13/cobra" "go.uber.org/zap" diff --git a/cmd/hunter.go b/cmd/hunter.go index ae0c715..a024547 100644 --- a/cmd/hunter.go +++ b/cmd/hunter.go @@ -1,12 +1,12 @@ package cmd import ( - "dehasher/internal/badger" - "dehasher/internal/debug" - "dehasher/internal/export" - "dehasher/internal/files" - hunter "dehasher/internal/hunter.io" - "dehasher/internal/pretty" + "crowsnest/internal/badger" + "crowsnest/internal/debug" + "crowsnest/internal/export" + "crowsnest/internal/files" + hunter "crowsnest/internal/hunter.io" + "crowsnest/internal/pretty" "fmt" "github.com/spf13/cobra" "go.uber.org/zap" diff --git a/cmd/logs.go b/cmd/logs.go index 6f5b509..43aac04 100644 --- a/cmd/logs.go +++ b/cmd/logs.go @@ -1,8 +1,8 @@ package cmd import ( - "dehasher/internal/easyTime" - "dehasher/internal/pretty" + "crowsnest/internal/easyTime" + "crowsnest/internal/pretty" "encoding/json" "fmt" "github.com/spf13/cobra" diff --git a/cmd/query.go b/cmd/query.go index 16ad8c7..1ffff46 100644 --- a/cmd/query.go +++ b/cmd/query.go @@ -1,8 +1,8 @@ package cmd import ( - "dehasher/internal/pretty" - "dehasher/internal/sqlite" + "crowsnest/internal/pretty" + "crowsnest/internal/sqlite" "encoding/json" "fmt" "github.com/spf13/cobra" diff --git a/cmd/root.go b/cmd/root.go index 5d80bd8..5d61acd 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,8 +1,9 @@ package cmd import ( - "dehasher/internal/badger" + "crowsnest/internal/badger" "fmt" + "github.com/fatih/color" "github.com/spf13/cobra" "go.uber.org/zap" "os" @@ -18,28 +19,14 @@ var ( Use: "dehasher", Short: `Dehasher is a cli tool for querying the dehashed api.`, Long: fmt.Sprintf( - "%s\n%s", + "%s\n", ` - ______ _______ _______ _______ _______ _______ -( __ \ ( ____ \|\ /|( ___ )( ____ \|\ /|( ____ \( ____ ) -| ( \ )| ( \/| ) ( || ( ) || ( \/| ) ( || ( \/| ( )| -| | ) || (__ | (___) || (___) || (_____ | (___) || (__ | (____)| -| | | || __) | ___ || ___ |(_____ )| ___ || __) | __) -| | ) || ( | ( ) || ( ) | ) || ( ) || ( | (\ ( -| (__/ )| (____/\| ) ( || ) ( |/\____) || ) ( || (____/\| ) \ \__ -(______/ (_______/|/ \||/ \|\_______)|/ \|(_______/|/ \__/ - An Ar1ste1a Project -`, - `––•–√\/––√\/––•––––•–√\/––√\/––•––––•–√\/––√\/––•––√\/––•––––•–√\/––√\/––•–– - Dehasher can query the query API for: - - Emails - Usernames - Password - - Hashes - IP Addresses - Names - - VINs - License Plates - Addresses - - Phones - Social Media - Crypto Currency Addresses - Dehasher supports: - - Regex Matching - - Exact Matching -––•–√\/––√\/––•––––•–√\/––√\/––•––––•–√\/––√\/––•––√\/––•––––•–√\/––√\/––•–– + ╔═╗┬─┐┌─┐┬ ┬┌─┐╔╗╔┌─┐┌─┐┌┬┐ + ║ ├┬┘│ ││││└─┐║║║├┤ └─┐ │ + ╚═╝┴└─└─┘└┴┘└─┘╝╚╝└─┘└─┘ ┴ + + Crow’s Nest OSINT Recon Suite +⚓ A KrakenTech Intelligence Tool `, ), Version: "v1.2.1", @@ -69,6 +56,7 @@ func init() { rootCmd.AddCommand(setDehashedKeyCmd) rootCmd.AddCommand(setHunterKeyCmd) rootCmd.AddCommand(setLocalDb) + rootCmd.AddCommand(buyMeCoffeeCmd) } // Command to set API key @@ -132,6 +120,20 @@ var setLocalDb = &cobra.Command{ }, } +var buyMeCoffeeCmd = &cobra.Command{ + Use: "coffee", + Short: "Support the project by buying a coffee", + Run: func(cmd *cobra.Command, args []string) { + fmt.Println(color.HiRedString(" ;)( ;")) + fmt.Println(color.HiCyanString(" We Hope You Enjoy Our Product :----:")) + fmt.Println(color.HiCyanString(" C|====|")) + fmt.Println(color.HiCyanString(" | |")) + fmt.Print(color.HiGreenString(" Support the project by buying a coffee: ")) + fmt.Print(color.BlueString("https://buymeacoffee.com/ehosinskiz ")) + fmt.Println(color.HiCyanString("`----'")) + }, +} + // Helper functions to store API credentials func storeDehashedApiKey(key string) error { err := badger.StoreDehashedKey(key) diff --git a/cmd/whois.go b/cmd/whois.go index eda1b04..6556906 100644 --- a/cmd/whois.go +++ b/cmd/whois.go @@ -1,12 +1,12 @@ package cmd import ( - "dehasher/internal/debug" - "dehasher/internal/export" - "dehasher/internal/files" - "dehasher/internal/pretty" - "dehasher/internal/sqlite" - "dehasher/internal/whois" + "crowsnest/internal/debug" + "crowsnest/internal/export" + "crowsnest/internal/files" + "crowsnest/internal/pretty" + "crowsnest/internal/sqlite" + "crowsnest/internal/whois" "fmt" "github.com/spf13/cobra" "go.uber.org/zap" diff --git a/dehasher.go b/dehasher.go index 44e8c6b..51f57ae 100644 --- a/dehasher.go +++ b/dehasher.go @@ -1,9 +1,9 @@ package main import ( - "dehasher/cmd" - "dehasher/internal/badger" - "dehasher/internal/sqlite" + "crowsnest/cmd" + "crowsnest/internal/badger" + "crowsnest/internal/sqlite" "fmt" "github.com/winking324/rzap" "go.uber.org/zap" diff --git a/go.mod b/go.mod index 76e44e5..77744a1 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module dehasher +module crowsnest go 1.23.0 diff --git a/internal/dehashed/clientv2.go b/internal/dehashed/clientv2.go index 0a50b51..f2087ae 100644 --- a/internal/dehashed/clientv2.go +++ b/internal/dehashed/clientv2.go @@ -2,9 +2,9 @@ package dehashed import ( "bytes" + "crowsnest/internal/debug" + "crowsnest/internal/sqlite" "crypto/sha256" - "dehasher/internal/debug" - "dehasher/internal/sqlite" "encoding/hex" "encoding/json" "errors" diff --git a/internal/dehashed/dehashed.go b/internal/dehashed/dehashed.go index 6c0e249..d3f2f49 100644 --- a/internal/dehashed/dehashed.go +++ b/internal/dehashed/dehashed.go @@ -1,9 +1,9 @@ package dehashed import ( - "dehasher/internal/debug" - "dehasher/internal/export" - "dehasher/internal/sqlite" + "crowsnest/internal/debug" + "crowsnest/internal/export" + "crowsnest/internal/sqlite" "encoding/json" "fmt" "go.uber.org/zap" diff --git a/internal/easyTime/parser.go b/internal/easyTime/parser.go index 879dd3b..075a010 100644 --- a/internal/easyTime/parser.go +++ b/internal/easyTime/parser.go @@ -1,7 +1,7 @@ package easyTime import ( - "dehasher/internal/debug" + "crowsnest/internal/debug" "fmt" "go.uber.org/zap" "os" diff --git a/internal/export/export.go b/internal/export/export.go index d97a78e..9484eb6 100644 --- a/internal/export/export.go +++ b/internal/export/export.go @@ -1,8 +1,8 @@ package export import ( - "dehasher/internal/files" - "dehasher/internal/sqlite" + "crowsnest/internal/files" + "crowsnest/internal/sqlite" "encoding/json" "encoding/xml" "errors" diff --git a/internal/export/hunter.go b/internal/export/hunter.go index 99d093e..8c191c0 100644 --- a/internal/export/hunter.go +++ b/internal/export/hunter.go @@ -1,8 +1,8 @@ package export import ( - "dehasher/internal/files" - "dehasher/internal/sqlite" + "crowsnest/internal/files" + "crowsnest/internal/sqlite" "encoding/json" "encoding/xml" "fmt" diff --git a/internal/hunter.io/hunter.io.go b/internal/hunter.io/hunter.io.go index b933779..d1415d7 100644 --- a/internal/hunter.io/hunter.io.go +++ b/internal/hunter.io/hunter.io.go @@ -1,8 +1,8 @@ package hunter_io import ( - "dehasher/internal/debug" - "dehasher/internal/sqlite" + "crowsnest/internal/debug" + "crowsnest/internal/sqlite" "encoding/json" "fmt" "go.uber.org/zap" diff --git a/internal/pretty/trees.go b/internal/pretty/trees.go index cd79b7c..a1ddce9 100644 --- a/internal/pretty/trees.go +++ b/internal/pretty/trees.go @@ -1,7 +1,7 @@ package pretty import ( - "dehasher/internal/sqlite" + "crowsnest/internal/sqlite" "fmt" "github.com/charmbracelet/lipgloss" "github.com/charmbracelet/lipgloss/tree" diff --git a/internal/sqlite/structs.go b/internal/sqlite/structs.go index 9417990..078ec45 100644 --- a/internal/sqlite/structs.go +++ b/internal/sqlite/structs.go @@ -1,11 +1,15 @@ package sqlite import ( - "dehasher/internal/files" + "crowsnest/internal/files" "fmt" "gorm.io/gorm" ) +type IString interface { + String() string +} + type DBOptions struct { Username string Email string diff --git a/internal/whois/whois.go b/internal/whois/whois.go index a18b035..9e1c163 100644 --- a/internal/whois/whois.go +++ b/internal/whois/whois.go @@ -2,9 +2,9 @@ package whois import ( "bytes" - "dehasher/internal/debug" - "dehasher/internal/dehashed" - "dehasher/internal/sqlite" + "crowsnest/internal/debug" + "crowsnest/internal/dehashed" + "crowsnest/internal/sqlite" "encoding/json" "errors" "fmt" From 61052b3308f4e2a26698f9e01c61d1f9748b043c Mon Sep 17 00:00:00 2001 From: Evan Hosinski Date: Sat, 17 May 2025 10:07:49 -0400 Subject: [PATCH 3/5] Implemented IString for file writing for Hunter Structs. --- cmd/hunter.go | 12 +- internal/export/{export.go => dehashed.go} | 0 internal/export/hunter.go | 161 --------------------- internal/export/iString.go | 36 +++++ internal/sqlite/hunter.io.go | 18 ++- 5 files changed, 54 insertions(+), 173 deletions(-) rename internal/export/{export.go => dehashed.go} (100%) delete mode 100644 internal/export/hunter.go create mode 100644 internal/export/iString.go diff --git a/cmd/hunter.go b/cmd/hunter.go index a024547..af3fb7d 100644 --- a/cmd/hunter.go +++ b/cmd/hunter.go @@ -113,7 +113,7 @@ var ( // Write Hunter.io Domain Search Result to file fmt.Printf("[*] Writing Hunter.io Domain Search Result to file: %s%s\n", hunterOutputFile, fType.Extension()) - err = export.WriteHunterDomainToFile(result, hunterOutputFile, fType) + err = export.WriteIStringToFile(result, hunterOutputFile, fType) if err != nil { if debugGlobal { debug.PrintInfo("failed to write hunter domain search to file") @@ -150,7 +150,7 @@ var ( // Write Hunter.io Email Finder Result to file fmt.Printf("[*] Writing Hunter.io Email Finder Result to file: %s%s\n", hunterOutputFile, fType.Extension()) - err = export.WriteHunterEmailToFile(result, hunterOutputFile, fType) + err = export.WriteIStringToFile(result, hunterOutputFile, fType) if err != nil { if debugGlobal { debug.PrintInfo("failed to write hunter email find to file") @@ -206,7 +206,7 @@ var ( } // Write Hunter.io Email Verification Result to file fmt.Printf("[*] Writing Hunter.io Email Verification Result to file: %s%s\n", hunterOutputFile, fType.Extension()) - err = export.WriteHunterEmailVerifyToFile(result, hunterOutputFile, fType) + err = export.WriteIStringToFile(result, hunterOutputFile, fType) if err != nil { if debugGlobal { debug.PrintInfo("failed to write hunter email verification to file") @@ -264,7 +264,7 @@ var ( // Write to file fmt.Printf("[*] Writing Hunter.io Company Enrichment Result to file: %s%s\n", hunterOutputFile, fType.Extension()) - err = export.WriteHunterCompanyEnrichmentToFile(result, hunterOutputFile, fType) + err = export.WriteIStringToFile(result, hunterOutputFile, fType) if err != nil { if debugGlobal { debug.PrintInfo("failed to write hunter company enrichment to file") @@ -302,7 +302,7 @@ var ( // Write to file fmt.Printf("[*] Writing Hunter.io Person Enrichment Result to file: %s%s\n", hunterOutputFile, fType.Extension()) - err = export.WriteHunterPersonEnrichmentToFile(result, hunterOutputFile, fType) + err = export.WriteIStringToFile(result, hunterOutputFile, fType) if err != nil { if debugGlobal { debug.PrintInfo("failed to write hunter person enrichment to file") @@ -339,7 +339,7 @@ var ( // Write to file fmt.Printf("[*] Writing Hunter.io Combined Enrichment Result to file: %s%s\n", hunterOutputFile, fType.Extension()) - err = export.WriteHunterCombinedEnrichmentToFile(result, hunterOutputFile, fType) + err = export.WriteIStringToFile(result, hunterOutputFile, fType) if err != nil { if debugGlobal { debug.PrintInfo("failed to write hunter combined enrichment to file") diff --git a/internal/export/export.go b/internal/export/dehashed.go similarity index 100% rename from internal/export/export.go rename to internal/export/dehashed.go diff --git a/internal/export/hunter.go b/internal/export/hunter.go deleted file mode 100644 index 8c191c0..0000000 --- a/internal/export/hunter.go +++ /dev/null @@ -1,161 +0,0 @@ -package export - -import ( - "crowsnest/internal/files" - "crowsnest/internal/sqlite" - "encoding/json" - "encoding/xml" - "fmt" - "gopkg.in/yaml.v3" - "os" -) - -func WriteHunterDomainToFile(result sqlite.HunterDomainData, outputFile string, fileType files.FileType) error { - var data []byte - var err error - - switch fileType { - case files.JSON: - data, err = json.MarshalIndent(result, "", " ") - case files.XML: - data, err = xml.MarshalIndent(result, "", " ") - case files.YAML: - data, err = yaml.Marshal(result) - case files.TEXT: - data = []byte(result.String()) - default: - return err - } - - if err != nil { - return err - } - - filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String()) - return os.WriteFile(filePath, data, 0644) -} - -func WriteHunterEmailToFile(result sqlite.HunterEmailFinderData, outputFile string, fileType files.FileType) error { - var data []byte - var err error - - switch fileType { - case files.JSON: - data, err = json.MarshalIndent(result, "", " ") - case files.XML: - data, err = xml.MarshalIndent(result, "", " ") - case files.YAML: - data, err = yaml.Marshal(result) - case files.TEXT: - data = []byte(result.String()) - default: - return err - } - - if err != nil { - return err - } - - filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String()) - return os.WriteFile(filePath, data, 0644) -} - -func WriteHunterEmailVerifyToFile(result sqlite.HunterEmailVerifyData, outputFile string, fileType files.FileType) error { - var data []byte - var err error - - switch fileType { - case files.JSON: - data, err = json.MarshalIndent(result, "", " ") - case files.XML: - data, err = xml.MarshalIndent(result, "", " ") - case files.YAML: - data, err = yaml.Marshal(result) - case files.TEXT: - data = []byte(result.String()) - default: - return err - } - - if err != nil { - return err - } - - filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String()) - return os.WriteFile(filePath, data, 0644) -} - -func WriteHunterCompanyEnrichmentToFile(result sqlite.CompanyData, outputFile string, fileType files.FileType) error { - var data []byte - var err error - - switch fileType { - case files.JSON: - data, err = json.MarshalIndent(result, "", " ") - case files.XML: - data, err = xml.MarshalIndent(result, "", " ") - case files.YAML: - data, err = yaml.Marshal(result) - case files.TEXT: - data = []byte(result.String()) - default: - return err - } - - if err != nil { - return err - } - - filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String()) - return os.WriteFile(filePath, data, 0644) -} - -func WriteHunterPersonEnrichmentToFile(result sqlite.PersonData, outputFile string, fileType files.FileType) error { - var data []byte - var err error - - switch fileType { - case files.JSON: - data, err = json.MarshalIndent(result, "", " ") - case files.XML: - data, err = xml.MarshalIndent(result, "", " ") - case files.YAML: - data, err = yaml.Marshal(result) - case files.TEXT: - data = []byte(result.String()) - default: - return err - } - - if err != nil { - return err - } - - filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String()) - return os.WriteFile(filePath, data, 0644) -} - -func WriteHunterCombinedEnrichmentToFile(result sqlite.CombinedData, outputFile string, fileType files.FileType) error { - var data []byte - var err error - - switch fileType { - case files.JSON: - data, err = json.MarshalIndent(result, "", " ") - case files.XML: - data, err = xml.MarshalIndent(result, "", " ") - case files.YAML: - data, err = yaml.Marshal(result) - case files.TEXT: - data = []byte(result.String()) - default: - return err - } - - if err != nil { - return err - } - - filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String()) - return os.WriteFile(filePath, data, 0644) -} diff --git a/internal/export/iString.go b/internal/export/iString.go new file mode 100644 index 0000000..809b26a --- /dev/null +++ b/internal/export/iString.go @@ -0,0 +1,36 @@ +package export + +import ( + "crowsnest/internal/files" + "crowsnest/internal/sqlite" + "encoding/json" + "encoding/xml" + "fmt" + "gopkg.in/yaml.v3" + "os" +) + +func WriteIStringToFile(iString sqlite.IString, outputFile string, fileType files.FileType) error { + var data []byte + var err error + + switch fileType { + case files.JSON: + data, err = json.MarshalIndent(iString, "", " ") + case files.XML: + data, err = xml.MarshalIndent(iString, "", " ") + case files.YAML: + data, err = yaml.Marshal(iString) + case files.TEXT: + data = []byte(iString.String()) + default: + return err + } + + if err != nil { + return err + } + + filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String()) + return os.WriteFile(filePath, data, 0644) +} diff --git a/internal/sqlite/hunter.io.go b/internal/sqlite/hunter.io.go index f78a0ed..9da19f1 100644 --- a/internal/sqlite/hunter.io.go +++ b/internal/sqlite/hunter.io.go @@ -14,6 +14,7 @@ type HunterDomainSearchResult struct { // HunterDomainData contains the main domain information type HunterDomainData struct { + IString gorm.Model Domain string `json:"domain" gorm:"unique"` Disposable bool `json:"disposable"` @@ -40,7 +41,7 @@ type HunterDomainData struct { LinkedDomains []string `json:"linked_domains" gorm:"serializer:json"` } -func (h *HunterDomainData) String() string { +func (h HunterDomainData) String() string { return fmt.Sprintf("Domain: %s\nDisposable: %t\nWebmail: %t\nAcceptAll: %t\nPattern: %s\nOrganization: %s\nDescription: %s\nIndustry: %s\nTwitter: %s\nFacebook: %s\nLinkedin: %s\nInstagram: %s\nYoutube: %s\nTechnologies: %v\nCountry: %s\nState: %s\nCity: %s\nPostalCode: %s\nStreet: %s\nHeadcount: %s\nCompanyType: %s\nEmails: %v\nLinkedDomains: %v\n", h.Domain, h.Disposable, h.Webmail, h.AcceptAll, h.Pattern, h.Organization, h.Description, h.Industry, h.Twitter, h.Facebook, h.Linkedin, h.Instagram, h.Youtube, h.Technologies, h.Country, h.State, h.City, h.PostalCode, h.Street, h.Headcount, h.CompanyType, h.Emails, h.LinkedDomains) } @@ -142,6 +143,7 @@ type HunterEmailFinderResponse struct { // HunterEmailFinderData contains the main email information type HunterEmailFinderData struct { + IString FirstName string `json:"first_name"` LastName string `json:"last_name"` Email string `json:"email"` @@ -157,7 +159,7 @@ type HunterEmailFinderData struct { Verification HunterVerification `json:"verification" gorm:"embedded;embeddedPrefix:verification_"` } -func (he *HunterEmailFinderData) String() string { +func (he HunterEmailFinderData) String() string { return fmt.Sprintf("FirstName: %s\nLastName: %s\nEmail: %s\nScore: %d\nDomain: %s\nAcceptAll: %t\nPosition: %s\nTwitter: %s\nLinkedinURL: %s\nPhoneNumber: %s\nCompany: %s\nSources: %v\nVerification: %v\n", he.FirstName, he.LastName, he.Email, he.Score, he.Domain, he.AcceptAll, he.Position, he.Twitter, he.LinkedinURL, he.PhoneNumber, he.Company, he.Sources, he.Verification) } @@ -189,6 +191,7 @@ type HunterEmailVerifyResponse struct { // HunterEmailVerifyData contains the email verification information type HunterEmailVerifyData struct { + IString Status string `json:"status"` Result string `json:"result"` DeprecationNotice string `json:"_deprecation_notice"` @@ -206,7 +209,7 @@ type HunterEmailVerifyData struct { Sources []HunterSource `json:"sources" gorm:"serializer:json"` } -func (ev *HunterEmailVerifyData) String() string { +func (ev HunterEmailVerifyData) String() string { return fmt.Sprintf("Status: %s\nResult: %s\nDeprecationNotice: %s\nScore: %d\nEmail: %s\nRegexp: %t\nGibberish: %t\nDisposable: %t\nWebmail: %t\nMXRecords: %t\nSMTPServer: %t\nSMTPCheck: %t\nAcceptAll: %t\nBlock: %t\nSources: %v\n", ev.Status, ev.Result, ev.DeprecationNotice, ev.Score, ev.Email, ev.Regexp, ev.Gibberish, ev.Disposable, ev.Webmail, ev.MXRecords, ev.SMTPServer, ev.SMTPCheck, ev.AcceptAll, ev.Block, ev.Sources) } @@ -229,6 +232,7 @@ type HunterCompanyEnrichmentResponse struct { // CompanyData contains the detailed company information type CompanyData struct { + IString ID string `json:"id"` Name string `json:"name"` LegalName string `json:"legalName"` @@ -262,7 +266,7 @@ type CompanyData struct { UltimateParent ParentCompany `json:"ultimateParent" gorm:"embedded;embeddedPrefix:ultimate_parent_"` } -func (cd *CompanyData) String() string { +func (cd CompanyData) String() string { return fmt.Sprintf("ID: %s\nName: %s\nLegalName: %s\nDomain: %s\nDomainAliases: %v\nSite: %v\nCategory: %v\nTags: %v\nDescription: %s\nFoundedYear: %d\nLocation: %s\nTimeZone: %s\nUTCOffset: %d\nGeo: %v\nLogo: %s\nFacebook: %v\nLinkedIn: %v\nTwitter: %v\nCrunchbase: %v\nYouTube: %v\nEmailProvider: %s\nType: %s\nTicker: %s\nIdentifiers: %v\nPhone: %s\nMetrics: %v\nIndexedAt: %s\nTech: %v\nTechCategories: %v\nParent: %v\nUltimateParent: %v\n", cd.ID, cd.Name, cd.LegalName, cd.Domain, cd.DomainAliases, cd.Site, cd.Category, cd.Tags, cd.Description, cd.FoundedYear, cd.Location, cd.TimeZone, cd.UTCOffset, cd.Geo, cd.Logo, cd.Facebook, cd.LinkedIn, cd.Twitter, cd.Crunchbase, cd.YouTube, cd.EmailProvider, cd.Type, cd.Ticker, cd.Identifiers, cd.Phone, cd.Metrics, cd.IndexedAt, cd.Tech, cd.TechCategories, cd.Parent, cd.UltimateParent) } @@ -508,6 +512,7 @@ type HunterPersonEnrichmentResponse struct { // PersonData contains the detailed person information type PersonData struct { + IString gorm.Model ID string `json:"id"` Name PersonName `json:"name" gorm:"embedded;embeddedPrefix:name_"` @@ -534,7 +539,7 @@ type PersonData struct { InactiveAt string `json:"inactiveAt"` } -func (pd *PersonData) String() string { +func (pd PersonData) String() string { return fmt.Sprintf("ID: %s\nName: %v\nEmail: %s\nLocation: %s\nTimeZone: %s\nUTCOffset: %d\nGeo: %v\nBio: %s\nSite: %s\nAvatar: %s\nEmployment: %v\nFacebook: %v\nGitHub: %v\nTwitter: %v\nLinkedIn: %v\nGooglePlus: %v\nGravatar: %v\nFuzzy: %t\nEmailProvider: %s\nIndexedAt: %s\nPhone: %s\nActiveAt: %s\nInactiveAt: %s\n", pd.ID, pd.Name, pd.Email, pd.Location, pd.TimeZone, pd.UTCOffset, pd.Geo, pd.Bio, pd.Site, pd.Avatar, pd.Employment, pd.Facebook, pd.GitHub, pd.Twitter, pd.LinkedIn, pd.GooglePlus, pd.Gravatar, pd.Fuzzy, pd.EmailProvider, pd.IndexedAt, pd.Phone, pd.ActiveAt, pd.InactiveAt) } @@ -728,11 +733,12 @@ type HunterCombinedEnrichmentResponse struct { // CombinedData contains both person and company information type CombinedData struct { + IString Person PersonData `json:"person" gorm:"embedded;embeddedPrefix:person_"` Company CompanyData `json:"company" gorm:"embedded;embeddedPrefix:company_"` } -func (cbd *CombinedData) String() string { +func (cbd CombinedData) String() string { return fmt.Sprintf("Person: %s\nCompany: %s", cbd.Person.String(), cbd.Company.String()) From 00d9a6b57e52dce31d970f233a703d0cd6ab11ed Mon Sep 17 00:00:00 2001 From: Evan Hosinski Date: Sat, 17 May 2025 10:25:16 -0400 Subject: [PATCH 4/5] fixed output of coffee and primary banner --- cmd/dehashed.go | 2 +- cmd/root.go | 12 +- cmd/whois.go | 4 +- dehasher.go => crowsnest.go | 0 internal/dehashed/dehashed.go | 6 +- internal/sqlite/db.go | 143 +++++++++++++++ internal/sqlite/dehashed.go | 235 +++++++++++++++++++++++++ internal/sqlite/gorm.go | 319 ---------------------------------- internal/sqlite/hunter.io.go | 67 ++++++- internal/sqlite/query.go | 20 --- internal/sqlite/result.go | 94 ---------- internal/sqlite/structs.go | 89 ---------- internal/sqlite/tables.go | 68 -------- internal/sqlite/whois.go | 116 +++++++++++++ internal/whois/whois.go | 6 +- 15 files changed, 574 insertions(+), 607 deletions(-) rename dehasher.go => crowsnest.go (100%) create mode 100644 internal/sqlite/db.go create mode 100644 internal/sqlite/dehashed.go delete mode 100644 internal/sqlite/gorm.go delete mode 100644 internal/sqlite/query.go delete mode 100644 internal/sqlite/result.go delete mode 100644 internal/sqlite/tables.go diff --git a/cmd/dehashed.go b/cmd/dehashed.go index 4d6894a..f79c9bc 100644 --- a/cmd/dehashed.go +++ b/cmd/dehashed.go @@ -118,7 +118,7 @@ var ( dehasher.Start() fmt.Println("\n[*] Completing Process") - err := sqlite.StoreQueryOptions(queryOptions) + err := sqlite.StoreDehashedQueryOptions(queryOptions) if err != nil { if debugGlobal { debug.PrintInfo("failed to store query options") diff --git a/cmd/root.go b/cmd/root.go index 5d61acd..9414493 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -21,12 +21,12 @@ var ( Long: fmt.Sprintf( "%s\n", ` - ╔═╗┬─┐┌─┐┬ ┬┌─┐╔╗╔┌─┐┌─┐┌┬┐ - ║ ├┬┘│ ││││└─┐║║║├┤ └─┐ │ - ╚═╝┴└─└─┘└┴┘└─┘╝╚╝└─┘└─┘ ┴ + ╔═╗┬─┐┌─┐┬ ┬┌─┐╔╗╔┌─┐┌─┐┌┬┐ + ║ ├┬┘│ ││││└─┐║║║├┤ └─┐ │ + ╚═╝┴└─└─┘└┴┘└─┘╝╚╝└─┘└─┘ ┴ - Crow’s Nest OSINT Recon Suite -⚓ A KrakenTech Intelligence Tool + Crow’s Nest OSINT Recon Suite + ⚓ A KrakenTech Intelligence Tool `, ), Version: "v1.2.1", @@ -124,7 +124,7 @@ var buyMeCoffeeCmd = &cobra.Command{ Use: "coffee", Short: "Support the project by buying a coffee", Run: func(cmd *cobra.Command, args []string) { - fmt.Println(color.HiRedString(" ;)( ;")) + fmt.Println(color.HiRedString(" ;)(; ")) fmt.Println(color.HiCyanString(" We Hope You Enjoy Our Product :----:")) fmt.Println(color.HiCyanString(" C|====|")) fmt.Println(color.HiCyanString(" | |")) diff --git a/cmd/whois.go b/cmd/whois.go index 6556906..cddf5a2 100644 --- a/cmd/whois.go +++ b/cmd/whois.go @@ -208,7 +208,7 @@ var ( fmt.Printf("[!] Error writing WHOIS history to file: %v\n", writeErr) } - err = sqlite.StoreHistoryRecord(historyRecords) + err = sqlite.StoreWhoisHistoryRecords(historyRecords) if err != nil { if debugGlobal { debug.PrintInfo("failed to store history record") @@ -254,7 +254,7 @@ var ( fmt.Printf("Error performing subdomain scan: %v\n", err) } else { fmt.Println("Subdomain Scan:") - err = sqlite.StoreSubdomainRecords(subdomains) + err = sqlite.StoreWhoisSubdomainRecords(subdomains) if err != nil { if debugGlobal { debug.PrintInfo("failed to store subdomain record") diff --git a/dehasher.go b/crowsnest.go similarity index 100% rename from dehasher.go rename to crowsnest.go diff --git a/internal/dehashed/dehashed.go b/internal/dehashed/dehashed.go index d3f2f49..6c8a7cf 100644 --- a/internal/dehashed/dehashed.go +++ b/internal/dehashed/dehashed.go @@ -130,7 +130,7 @@ func (dh *Dehasher) Start() { if len(dh.client.results) > 0 { fmt.Printf(" [!] Partial results retrieved. Storing Results...\n") - err := sqlite.StoreResults(dh.client.GetResults()) + err := sqlite.StoreDehashedResults(dh.client.GetResults()) if err != nil { zap.L().Error("store_results", zap.String("message", "failed to store results"), @@ -214,7 +214,7 @@ func (dh *Dehasher) parseResults() { results := dh.client.GetResults() creds := results.ExtractCredentials() fmt.Printf("\n\t[+] Discovered %d Credentials", len(creds)) - err := sqlite.StoreCreds(creds) + err := sqlite.StoreDehashedCreds(creds) if err != nil { zap.L().Error("store_creds", zap.String("message", "failed to store creds"), @@ -224,7 +224,7 @@ func (dh *Dehasher) parseResults() { zap.L().Info("creds_stored", zap.Int("count", len(creds))) zap.L().Info("storing_results") - err = sqlite.StoreResults(results) + err = sqlite.StoreDehashedResults(results) if err != nil { zap.L().Error("store_results", zap.String("message", "failed to store results"), diff --git a/internal/sqlite/db.go b/internal/sqlite/db.go new file mode 100644 index 0000000..1024a45 --- /dev/null +++ b/internal/sqlite/db.go @@ -0,0 +1,143 @@ +package sqlite + +import ( + "fmt" + "go.uber.org/zap" + "os" + "path/filepath" + "strings" + + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +var DB *gorm.DB + +// InitDB initializes the database connection +func InitDB(dbPath string) (*gorm.DB, error) { + zap.L().Info("Initializing database", zap.String("path", dbPath)) + + // Check if the path is a file or directory + fileInfo, err := os.Stat(dbPath) + var finalDbPath string + + // If path doesn't exist or is a directory + if os.IsNotExist(err) || (err == nil && fileInfo.IsDir()) { + // Treat as directory path + if err := os.MkdirAll(dbPath, 0755); err != nil { + zap.L().Error("Failed to create database directory", zap.Error(err)) + return nil, fmt.Errorf("failed to create database directory: %w", err) + } + finalDbPath = filepath.Join(dbPath, "dehashed.sqlite") + } else { + // Treat as file path + // Ensure the directory exists + dir := filepath.Dir(dbPath) + if err := os.MkdirAll(dir, 0755); err != nil { + zap.L().Error("Failed to create parent directory for database", zap.Error(err)) + return nil, fmt.Errorf("failed to create parent directory for database: %w", err) + } + finalDbPath = dbPath + } + + zap.L().Info("Opening database", zap.String("finalPath", finalDbPath)) + db, err := gorm.Open(sqlite.Open(finalDbPath), &gorm.Config{ + Logger: logger.Default.LogMode(logger.Silent), + }) + if err != nil { + zap.L().Error("Failed to connect to database", zap.Error(err)) + return nil, fmt.Errorf("failed to connect to database: %w", err) + } + + // Auto migrate your models + err = db.AutoMigrate(&Result{}, &Creds{}, &QueryOptions{}, &Creds{}, &WhoisRecord{}, &SubdomainRecord{}, + &HistoryRecord{}, &LookupResult{}, &HunterDomainData{}, &HunterEmail{}, &PersonData{}) + if err != nil { + zap.L().Error("Failed to migrate database", zap.Error(err)) + return nil, fmt.Errorf("failed to migrate database: %w", err) + } + + DB = db + return db, nil +} + +// GetDB returns the database connection +func GetDB() *gorm.DB { + if DB == nil { + zap.L().Error("database not initialized") + fmt.Println("sqlite database not initialized") + os.Exit(1) + } + return DB +} + +type Table int64 + +const ( + ResultsTable Table = iota + RunsTable + CredsTable + WhoIsTable + SubdomainsTable + HistoryTable + LookupTable + HunterDomainTable + HunterEmailTable + PersonTable + UnknownTable +) + +func GetTable(userInput string) Table { + switch strings.ToLower(userInput) { + case "results": + return ResultsTable + case "runs": + return RunsTable + case "creds": + return CredsTable + case "whois": + return WhoIsTable + case "subdomains": + return SubdomainsTable + case "history": + return HistoryTable + case "lookup": + return LookupTable + case "hunter_domain": + return HunterDomainTable + case "hunter_email": + return HunterEmailTable + case "person": + return PersonTable + default: + return UnknownTable + } +} + +func (t Table) Object() interface{} { + switch t { + case ResultsTable: + return Result{} + case RunsTable: + return QueryOptions{} + case CredsTable: + return Creds{} + case WhoIsTable: + return WhoisRecord{} + case SubdomainsTable: + return SubdomainRecord{} + case HistoryTable: + return HistoryRecord{} + case LookupTable: + return LookupResult{} + case HunterDomainTable: + return HunterDomainData{} + case HunterEmailTable: + return HunterEmail{} + case PersonTable: + return PersonData{} + default: + return nil + } +} diff --git a/internal/sqlite/dehashed.go b/internal/sqlite/dehashed.go new file mode 100644 index 0000000..ba78577 --- /dev/null +++ b/internal/sqlite/dehashed.go @@ -0,0 +1,235 @@ +package sqlite + +import ( + "crowsnest/internal/files" + "fmt" + "go.uber.org/zap" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +type QueryOptions struct { + gorm.Model + MaxRecords int `json:"max_records"` + MaxRequests int `json:"max_requests"` + StartingPage int `json:"starting_page"` + OutputFormat files.FileType `json:"output_format"` + OutputFile string `json:"output_file"` + RegexMatch bool `json:"regex_match"` + WildcardMatch bool `json:"wildcard_match"` + UsernameQuery string `json:"username_query"` + EmailQuery string `json:"email_query"` + IpQuery string `json:"ip_query"` + PassQuery string `json:"pass_query"` + HashQuery string `json:"hash_query"` + NameQuery string `json:"name_query"` + DomainQuery string `json:"domain_query"` + VinQuery string `json:"vin_query"` + LicensePlateQuery string `json:"license_plate_query"` + AddressQuery string `json:"address_query"` + PhoneQuery string `json:"phone_query"` + SocialQuery string `json:"social_query"` + CryptoAddressQuery string `json:"crypto_address_query"` + PrintBalance bool `json:"print_balance"` + CredsOnly bool `json:"creds_only"` + Debug bool `json:"debug"` +} + +func (QueryOptions) TableName() string { + return "query_options" +} + +func NewQueryOptions(maxRecords, maxRequests, startingPage int, outputFormat, outputFile, usernameQuery, emailQuery, ipQuery, passQuery, hashQuery, nameQuery, domainQuery, vinQuery, licensePlateQuery, addressQuery, phoneQuery, socialQuery, cryptoAddressQuery string, regexMatch, wildcardMatch, printBalance, credsOnly, debug bool) *QueryOptions { + return &QueryOptions{ + MaxRecords: maxRecords, + MaxRequests: maxRequests, + StartingPage: startingPage, + OutputFormat: files.GetFileType(outputFormat), + OutputFile: outputFile, + PrintBalance: printBalance, + CredsOnly: credsOnly, + RegexMatch: regexMatch, + WildcardMatch: wildcardMatch, + UsernameQuery: usernameQuery, + EmailQuery: emailQuery, + IpQuery: ipQuery, + PassQuery: passQuery, + HashQuery: hashQuery, + NameQuery: nameQuery, + DomainQuery: domainQuery, + VinQuery: vinQuery, + LicensePlateQuery: licensePlateQuery, + AddressQuery: addressQuery, + PhoneQuery: phoneQuery, + SocialQuery: socialQuery, + CryptoAddressQuery: cryptoAddressQuery, + Debug: debug, + } +} + +type DehashedSearchRequest struct { + Page int `json:"page"` + Query string `json:"query"` + Size int `json:"size"` + Wildcard bool `json:"wildcard"` + Regex bool `json:"regex"` + DeDupe bool `json:"de_dupe"` +} + +type DehashedResponse struct { + Balance int `json:"balance"` + Entries []Result `json:"entries"` + Success bool `json:"success"` + Took string `json:"took"` + TotalResults int `json:"total"` +} + +type Result struct { + gorm.Model + DehashedId string `json:"id" xml:"id" yaml:"id" gorm:"uniqueIndex"` + Email []string `json:"email,omitempty" xml:"email,omitempty" yaml:"email,omitempty" gorm:"serializer:json"` + IpAddress []string `json:"ip_address,omitempty" xml:"ip_address,omitempty" yaml:"ip_address,omitempty" gorm:"serializer:json"` + Username []string `json:"username,omitempty" xml:"username,omitempty" yaml:"username,omitempty" gorm:"serializer:json"` + Password []string `json:"password,omitempty" xml:"password,omitempty" yaml:"password,omitempty" gorm:"serializer:json"` + HashedPassword []string `json:"hashed_password,omitempty" xml:"hashed_password,omitempty" yaml:"hashed_password,omitempty" gorm:"serializer:json"` + HashType string `json:"hash_type,omitempty" xml:"hash_type,omitempty" yaml:"hash_type,omitempty"` + Name []string `json:"name,omitempty" xml:"name,omitempty" yaml:"name,omitempty" gorm:"serializer:json"` + Vin []string `json:"vin,omitempty" xml:"vin,omitempty" yaml:"vin,omitempty" gorm:"serializer:json"` + LicensePlate []string `json:"license_plate,omitempty" xml:"license_plate,omitempty" yaml:"license_plate,omitempty" gorm:"serializer:json"` + Url []string `json:"url,omitempty" xml:"url,omitempty" yaml:"url,omitempty" gorm:"serializer:json"` + Social []string `json:"social,omitempty" xml:"social,omitempty" yaml:"social,omitempty" gorm:"serializer:json"` + CryptoCurrencyAddress []string `json:"cryptocurrency_address,omitempty" xml:"cryptocurrency_address,omitempty" yaml:"cryptocurrency_address,omitempty" gorm:"serializer:json"` + Address []string `json:"address,omitempty" xml:"address,omitempty" yaml:"address,omitempty" gorm:"serializer:json"` + Phone []string `json:"phone,omitempty" xml:"phone,omitempty" yaml:"phone,omitempty" gorm:"serializer:json"` + Company []string `json:"company,omitempty" xml:"company,omitempty" yaml:"company,omitempty" gorm:"serializer:json"` + DatabaseName string `json:"database_name,omitempty" xml:"database_name,omitempty" yaml:"database_name,omitempty"` +} + +func (Result) TableName() string { + return "results" +} + +type DehashedResults struct { + Results []Result `json:"results"` +} + +func (dr *DehashedResults) ExtractCredentials() []Creds { + var creds []Creds + + results := dr.Results + + for _, r := range results { + if len(r.Password) > 0 { + // Get first email if available + email := "" + if len(r.Email) > 0 { + email = r.Email[0] + } + + // Get first password + password := r.Password[0] + + cred := Creds{Email: email, Password: password} + creds = append(creds, cred) + } + } + + go func() { + err := StoreDehashedCreds(creds) + if err != nil { + zap.L().Error("store_creds", + zap.String("message", "failed to store creds"), + zap.Error(err), + ) + fmt.Printf("Error Storing Results: %v", err) + } + }() + + return creds +} + +type Creds struct { + gorm.Model + Email string `json:"email" yaml:"email" xml:"email" gorm:"uniqueIndex:idx_email_username_password"` + Username string `json:"username" yaml:"username" xml:"username" gorm:"uniqueIndex:idx_email_username_password"` + Password string `json:"password" yaml:"password" xml:"password" gorm:"uniqueIndex:idx_email_username_password"` +} + +func (Creds) TableName() string { + return "creds" +} + +func (c Creds) ToString() string { + return fmt.Sprintf("%s%s%s", c.Username, "%", c.Password) +} + +func StoreDehashedResults(results DehashedResults) error { + if len(results.Results) == 0 { + return nil + } + + zap.L().Info("Storing results", zap.Int("count", len(results.Results))) + db := GetDB() + + // Use batch insert with conflict handling + const batchSize = 100 + var lastErr error + + // Extract the slice of results + resultSlice := results.Results + + for i := 0; i < len(resultSlice); i += batchSize { + end := i + batchSize + if end > len(resultSlice) { + end = len(resultSlice) + } + + batch := resultSlice[i:end] + // Use Clauses with OnConflict DoNothing to skip conflicts + err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error + if err != nil { + zap.L().Warn("Error storing some results", zap.Error(err)) + lastErr = err + // Continue with next batch despite error + } + } + + return lastErr +} + +func StoreDehashedCreds(creds []Creds) error { + if len(creds) == 0 { + return nil + } + + zap.L().Info("Storing credentials", zap.Int("count", len(creds))) + db := GetDB() + + // Use batch insert with conflict handling + // This will insert records in batches and continue even if some fail + const batchSize = 100 + var lastErr error + + for i := 0; i < len(creds); i += batchSize { + end := i + batchSize + if end > len(creds) { + end = len(creds) + } + + batch := creds[i:end] + // Use Clauses with OnConflict DoNothing to skip conflicts + err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error + if err != nil { + zap.L().Warn("Error storing some credentials", zap.Error(err)) + lastErr = err + // Continue with next batch despite error + } + } + + return lastErr +} + +func StoreDehashedQueryOptions(queryOptions *QueryOptions) error { + db := GetDB() + return db.Create(queryOptions).Error +} diff --git a/internal/sqlite/gorm.go b/internal/sqlite/gorm.go deleted file mode 100644 index bdc152d..0000000 --- a/internal/sqlite/gorm.go +++ /dev/null @@ -1,319 +0,0 @@ -package sqlite - -import ( - "fmt" - "go.uber.org/zap" - "gorm.io/gorm/clause" - "os" - "path/filepath" - - "gorm.io/driver/sqlite" - "gorm.io/gorm" - "gorm.io/gorm/logger" -) - -var DB *gorm.DB - -// InitDB initializes the database connection -func InitDB(dbPath string) (*gorm.DB, error) { - zap.L().Info("Initializing database", zap.String("path", dbPath)) - - // Check if the path is a file or directory - fileInfo, err := os.Stat(dbPath) - var finalDbPath string - - // If path doesn't exist or is a directory - if os.IsNotExist(err) || (err == nil && fileInfo.IsDir()) { - // Treat as directory path - if err := os.MkdirAll(dbPath, 0755); err != nil { - zap.L().Error("Failed to create database directory", zap.Error(err)) - return nil, fmt.Errorf("failed to create database directory: %w", err) - } - finalDbPath = filepath.Join(dbPath, "dehashed.sqlite") - } else { - // Treat as file path - // Ensure the directory exists - dir := filepath.Dir(dbPath) - if err := os.MkdirAll(dir, 0755); err != nil { - zap.L().Error("Failed to create parent directory for database", zap.Error(err)) - return nil, fmt.Errorf("failed to create parent directory for database: %w", err) - } - finalDbPath = dbPath - } - - zap.L().Info("Opening database", zap.String("finalPath", finalDbPath)) - db, err := gorm.Open(sqlite.Open(finalDbPath), &gorm.Config{ - Logger: logger.Default.LogMode(logger.Silent), - }) - if err != nil { - zap.L().Error("Failed to connect to database", zap.Error(err)) - return nil, fmt.Errorf("failed to connect to database: %w", err) - } - - // Auto migrate your models - err = db.AutoMigrate(&Result{}, &Creds{}, &QueryOptions{}, &Creds{}, &WhoisRecord{}, &SubdomainRecord{}, - &HistoryRecord{}, &LookupResult{}, &HunterDomainData{}, &HunterEmail{}, &PersonData{}) - if err != nil { - zap.L().Error("Failed to migrate database", zap.Error(err)) - return nil, fmt.Errorf("failed to migrate database: %w", err) - } - - DB = db - return db, nil -} - -// GetDB returns the database connection -func GetDB() *gorm.DB { - if DB == nil { - zap.L().Error("database not initialized") - fmt.Println("sqlite database not initialized") - os.Exit(1) - } - return DB -} - -func StoreResults(results DehashedResults) error { - if len(results.Results) == 0 { - return nil - } - - zap.L().Info("Storing results", zap.Int("count", len(results.Results))) - db := GetDB() - - // Use batch insert with conflict handling - const batchSize = 100 - var lastErr error - - // Extract the slice of results - resultSlice := results.Results - - for i := 0; i < len(resultSlice); i += batchSize { - end := i + batchSize - if end > len(resultSlice) { - end = len(resultSlice) - } - - batch := resultSlice[i:end] - // Use Clauses with OnConflict DoNothing to skip conflicts - err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error - if err != nil { - zap.L().Warn("Error storing some results", zap.Error(err)) - lastErr = err - // Continue with next batch despite error - } - } - - return lastErr -} - -func StoreCreds(creds []Creds) error { - if len(creds) == 0 { - return nil - } - - zap.L().Info("Storing credentials", zap.Int("count", len(creds))) - db := GetDB() - - // Use batch insert with conflict handling - // This will insert records in batches and continue even if some fail - const batchSize = 100 - var lastErr error - - for i := 0; i < len(creds); i += batchSize { - end := i + batchSize - if end > len(creds) { - end = len(creds) - } - - batch := creds[i:end] - // Use Clauses with OnConflict DoNothing to skip conflicts - err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error - if err != nil { - zap.L().Warn("Error storing some credentials", zap.Error(err)) - lastErr = err - // Continue with next batch despite error - } - } - - return lastErr -} - -func StoreQueryOptions(queryOptions *QueryOptions) error { - db := GetDB() - return db.Create(queryOptions).Error -} - -func StoreWhoisRecord(whoisRecord WhoisRecord) error { - // Create a pointer to the record to make it addressable - recordPtr := &whoisRecord - - zap.L().Info("Storing WHOIS record", - zap.String("domain", whoisRecord.DomainName)) - - db := GetDB() - - // Use OnConflict clause to handle duplicates - err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(recordPtr).Error - if err != nil { - zap.L().Error("store_whois_record", - zap.String("message", "failed to store whois record"), - zap.Error(err)) - return err - } - - return nil -} - -func StoreSubdomainRecords(subdomainRecords []SubdomainRecord) error { - if len(subdomainRecords) == 0 { - return nil - } - - zap.L().Info("Storing subdomain records", zap.Int("count", len(subdomainRecords))) - db := GetDB() - - // Use batch insert with conflict handling - const batchSize = 100 - var lastErr error - - for i := 0; i < len(subdomainRecords); i += batchSize { - end := i + batchSize - if end > len(subdomainRecords) { - end = len(subdomainRecords) - } - - batch := subdomainRecords[i:end] - // Use Clauses with OnConflict DoNothing to skip conflicts - err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error - if err != nil { - zap.L().Warn("Error storing some subdomain records", zap.Error(err)) - lastErr = err - // Continue with next batch despite error - } - } - - return lastErr -} - -func StoreHistoryRecord(historyRecords []HistoryRecord) error { - if len(historyRecords) == 0 { - return nil - } - - zap.L().Info("Storing history records", zap.Int("count", len(historyRecords))) - db := GetDB() - - // Use batch insert with conflict handling - const batchSize = 100 - var lastErr error - - for i := 0; i < len(historyRecords); i += batchSize { - end := i + batchSize - if end > len(historyRecords) { - end = len(historyRecords) - } - - batch := historyRecords[i:end] - // Use Clauses with OnConflict DoNothing to skip conflicts - err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error - if err != nil { - zap.L().Warn("Error storing some history records", zap.Error(err)) - lastErr = err - // Continue with next batch despite error - } - } - - return lastErr -} - -func StoreIPLookup(ipLookup []LookupResult) error { - if len(ipLookup) == 0 { - return nil - } - - zap.L().Info("Storing IP lookup records", zap.Int("count", len(ipLookup))) - db := GetDB() - - // Use batch insert with conflict handling - const batchSize = 100 - var lastErr error - - for i := 0; i < len(ipLookup); i += batchSize { - end := i + batchSize - if end > len(ipLookup) { - end = len(ipLookup) - } - - batch := ipLookup[i:end] - // Use Clauses with OnConflict DoNothing to skip conflicts - err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error - if err != nil { - zap.L().Warn("Error storing some IP lookup records", zap.Error(err)) - lastErr = err - // Continue with next batch despite error - } - } - - return lastErr -} - -func StoreHunterDomain(hunterDomain HunterDomainData) error { - db := GetDB() - - // Use OnConflict clause to handle duplicates - err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(&hunterDomain).Error - if err != nil { - zap.L().Error("store_hunter_domain", - zap.String("message", "failed to store hunter domain"), - zap.Error(err)) - return err - } - - return nil -} - -func StoreHunterEmails(hunterEmails []HunterEmail) error { - if len(hunterEmails) == 0 { - return nil - } - - zap.L().Info("Storing hunter emails", zap.Int("count", len(hunterEmails))) - db := GetDB() - - // Use batch insert with conflict handling - const batchSize = 100 - var lastErr error - - for i := 0; i < len(hunterEmails); i += batchSize { - end := i + batchSize - if end > len(hunterEmails) { - end = len(hunterEmails) - } - - batch := hunterEmails[i:end] - // Use Clauses with OnConflict DoNothing to skip conflicts - err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error - if err != nil { - zap.L().Warn("Error storing some hunter emails", zap.Error(err)) - lastErr = err - // Continue with next batch despite error - } - } - - return lastErr -} - -func StorePersonData(personData PersonData) error { - db := GetDB() - - // Use OnConflict clause to handle duplicates - err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(&personData).Error - if err != nil { - zap.L().Error("store_person_data", - zap.String("message", "failed to store person data"), - zap.Error(err)) - return err - } - - return nil -} diff --git a/internal/sqlite/hunter.io.go b/internal/sqlite/hunter.io.go index 9da19f1..5cb5a63 100644 --- a/internal/sqlite/hunter.io.go +++ b/internal/sqlite/hunter.io.go @@ -3,7 +3,9 @@ package sqlite import ( "fmt" "github.com/charmbracelet/lipgloss/tree" + "go.uber.org/zap" "gorm.io/gorm" + "gorm.io/gorm/clause" ) // HunterDomainSearchResult represents the response from Hunter.io domain search API @@ -14,8 +16,8 @@ type HunterDomainSearchResult struct { // HunterDomainData contains the main domain information type HunterDomainData struct { - IString gorm.Model + IString `gorm:"-"` Domain string `json:"domain" gorm:"unique"` Disposable bool `json:"disposable"` Webmail bool `json:"webmail"` @@ -512,8 +514,8 @@ type HunterPersonEnrichmentResponse struct { // PersonData contains the detailed person information type PersonData struct { - IString gorm.Model + IString `gorm:"-"` ID string `json:"id"` Name PersonName `json:"name" gorm:"embedded;embeddedPrefix:name_"` Email string `json:"email" gorm:"unique"` @@ -755,3 +757,64 @@ func (c *HunterCombinedEnrichmentResponse) String() string { c.Data.Person.String(), c.Data.Company.String()) } + +func StoreHunterDomain(hunterDomain HunterDomainData) error { + db := GetDB() + + // Use OnConflict clause to handle duplicates + err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(&hunterDomain).Error + if err != nil { + zap.L().Error("store_hunter_domain", + zap.String("message", "failed to store hunter domain"), + zap.Error(err)) + return err + } + + return nil +} + +func StoreHunterEmails(hunterEmails []HunterEmail) error { + if len(hunterEmails) == 0 { + return nil + } + + zap.L().Info("Storing hunter emails", zap.Int("count", len(hunterEmails))) + db := GetDB() + + // Use batch insert with conflict handling + const batchSize = 100 + var lastErr error + + for i := 0; i < len(hunterEmails); i += batchSize { + end := i + batchSize + if end > len(hunterEmails) { + end = len(hunterEmails) + } + + batch := hunterEmails[i:end] + // Use Clauses with OnConflict DoNothing to skip conflicts + err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error + if err != nil { + zap.L().Warn("Error storing some hunter emails", zap.Error(err)) + lastErr = err + // Continue with next batch despite error + } + } + + return lastErr +} + +func StoreHunterPersonData(personData PersonData) error { + db := GetDB() + + // Use OnConflict clause to handle duplicates + err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(&personData).Error + if err != nil { + zap.L().Error("store_person_data", + zap.String("message", "failed to store person data"), + zap.Error(err)) + return err + } + + return nil +} diff --git a/internal/sqlite/query.go b/internal/sqlite/query.go deleted file mode 100644 index 1b6382c..0000000 --- a/internal/sqlite/query.go +++ /dev/null @@ -1,20 +0,0 @@ -package sqlite - -type DehashedSearchRequest struct { - Page int `json:"page"` - Query string `json:"query"` - Size int `json:"size"` - Wildcard bool `json:"wildcard"` - Regex bool `json:"regex"` - DeDupe bool `json:"de_dupe"` -} - -func NewDehashedSearchRequest(size int, wildcard, regex bool) *DehashedSearchRequest { - return &DehashedSearchRequest{ - Page: 0, - Size: size, - Wildcard: false, - Regex: false, - DeDupe: true, - } -} diff --git a/internal/sqlite/result.go b/internal/sqlite/result.go deleted file mode 100644 index 1802c9c..0000000 --- a/internal/sqlite/result.go +++ /dev/null @@ -1,94 +0,0 @@ -package sqlite - -import ( - "encoding/json" - "fmt" - "go.uber.org/zap" - "gorm.io/gorm" - "io" - "os" -) - -type DehashedResponse struct { - Balance int `json:"balance"` - Entries []Result `json:"entries"` - Success bool `json:"success"` - Took string `json:"took"` - TotalResults int `json:"total"` -} - -type Result struct { - gorm.Model - DehashedId string `json:"id" xml:"id" yaml:"id" gorm:"uniqueIndex"` - Email []string `json:"email,omitempty" xml:"email,omitempty" yaml:"email,omitempty" gorm:"serializer:json"` - IpAddress []string `json:"ip_address,omitempty" xml:"ip_address,omitempty" yaml:"ip_address,omitempty" gorm:"serializer:json"` - Username []string `json:"username,omitempty" xml:"username,omitempty" yaml:"username,omitempty" gorm:"serializer:json"` - Password []string `json:"password,omitempty" xml:"password,omitempty" yaml:"password,omitempty" gorm:"serializer:json"` - HashedPassword []string `json:"hashed_password,omitempty" xml:"hashed_password,omitempty" yaml:"hashed_password,omitempty" gorm:"serializer:json"` - HashType string `json:"hash_type,omitempty" xml:"hash_type,omitempty" yaml:"hash_type,omitempty"` - Name []string `json:"name,omitempty" xml:"name,omitempty" yaml:"name,omitempty" gorm:"serializer:json"` - Vin []string `json:"vin,omitempty" xml:"vin,omitempty" yaml:"vin,omitempty" gorm:"serializer:json"` - LicensePlate []string `json:"license_plate,omitempty" xml:"license_plate,omitempty" yaml:"license_plate,omitempty" gorm:"serializer:json"` - Url []string `json:"url,omitempty" xml:"url,omitempty" yaml:"url,omitempty" gorm:"serializer:json"` - Social []string `json:"social,omitempty" xml:"social,omitempty" yaml:"social,omitempty" gorm:"serializer:json"` - CryptoCurrencyAddress []string `json:"cryptocurrency_address,omitempty" xml:"cryptocurrency_address,omitempty" yaml:"cryptocurrency_address,omitempty" gorm:"serializer:json"` - Address []string `json:"address,omitempty" xml:"address,omitempty" yaml:"address,omitempty" gorm:"serializer:json"` - Phone []string `json:"phone,omitempty" xml:"phone,omitempty" yaml:"phone,omitempty" gorm:"serializer:json"` - Company []string `json:"company,omitempty" xml:"company,omitempty" yaml:"company,omitempty" gorm:"serializer:json"` - DatabaseName string `json:"database_name,omitempty" xml:"database_name,omitempty" yaml:"database_name,omitempty"` -} - -func (Result) TableName() string { - return "results" -} - -type DehashedResults struct { - Results []Result `json:"results"` -} - -func (dr *DehashedResults) ExtractCredentials() []Creds { - var creds []Creds - - results := dr.Results - - for _, r := range results { - if len(r.Password) > 0 { - // Get first email if available - email := "" - if len(r.Email) > 0 { - email = r.Email[0] - } - - // Get first password - password := r.Password[0] - - cred := Creds{Email: email, Password: password} - creds = append(creds, cred) - } - } - - go func() { - err := StoreCreds(creds) - if err != nil { - zap.L().Error("store_creds", - zap.String("message", "failed to store creds"), - zap.Error(err), - ) - fmt.Printf("Error Storing Results: %v", err) - } - }() - - return creds -} - -func NewDehashedResults(body io.Reader) ([]Result, int, int) { - var response DehashedResponse - - err := json.NewDecoder(body).Decode(&response) - if err != nil { - fmt.Printf("Error Parsing Response Body: %v", err) - os.Exit(-1) - } - - return response.Entries, response.Balance, response.TotalResults -} diff --git a/internal/sqlite/structs.go b/internal/sqlite/structs.go index 078ec45..1adb693 100644 --- a/internal/sqlite/structs.go +++ b/internal/sqlite/structs.go @@ -1,11 +1,5 @@ package sqlite -import ( - "crowsnest/internal/files" - "fmt" - "gorm.io/gorm" -) - type IString interface { String() string } @@ -30,15 +24,6 @@ type DBOptions struct { DisplayFields []string // Fields to display in output } -func NewDBOptions() *DBOptions { - return &DBOptions{ - Limit: 100, // Default limit - ExactMatch: false, - NonEmptyFields: []string{}, - DisplayFields: []string{}, - } -} - func (o *DBOptions) Empty() bool { return o.Username == "" && o.Email == "" && o.IPAddress == "" && o.Password == "" && o.HashedPassword == "" && o.Name == "" && @@ -46,77 +31,3 @@ func (o *DBOptions) Empty() bool { o.Phone == "" && o.Social == "" && o.CryptoCurrencyAddress == "" && o.Domain == "" && len(o.NonEmptyFields) == 0 } - -type QueryOptions struct { - gorm.Model - MaxRecords int `json:"max_records"` - MaxRequests int `json:"max_requests"` - StartingPage int `json:"starting_page"` - OutputFormat files.FileType `json:"output_format"` - OutputFile string `json:"output_file"` - RegexMatch bool `json:"regex_match"` - WildcardMatch bool `json:"wildcard_match"` - UsernameQuery string `json:"username_query"` - EmailQuery string `json:"email_query"` - IpQuery string `json:"ip_query"` - PassQuery string `json:"pass_query"` - HashQuery string `json:"hash_query"` - NameQuery string `json:"name_query"` - DomainQuery string `json:"domain_query"` - VinQuery string `json:"vin_query"` - LicensePlateQuery string `json:"license_plate_query"` - AddressQuery string `json:"address_query"` - PhoneQuery string `json:"phone_query"` - SocialQuery string `json:"social_query"` - CryptoAddressQuery string `json:"crypto_address_query"` - PrintBalance bool `json:"print_balance"` - CredsOnly bool `json:"creds_only"` - Debug bool `json:"debug"` -} - -func (QueryOptions) TableName() string { - return "query_options" -} - -func NewQueryOptions(maxRecords, maxRequests, startingPage int, outputFormat, outputFile, usernameQuery, emailQuery, ipQuery, passQuery, hashQuery, nameQuery, domainQuery, vinQuery, licensePlateQuery, addressQuery, phoneQuery, socialQuery, cryptoAddressQuery string, regexMatch, wildcardMatch, printBalance, credsOnly, debug bool) *QueryOptions { - return &QueryOptions{ - MaxRecords: maxRecords, - MaxRequests: maxRequests, - StartingPage: startingPage, - OutputFormat: files.GetFileType(outputFormat), - OutputFile: outputFile, - PrintBalance: printBalance, - CredsOnly: credsOnly, - RegexMatch: regexMatch, - WildcardMatch: wildcardMatch, - UsernameQuery: usernameQuery, - EmailQuery: emailQuery, - IpQuery: ipQuery, - PassQuery: passQuery, - HashQuery: hashQuery, - NameQuery: nameQuery, - DomainQuery: domainQuery, - VinQuery: vinQuery, - LicensePlateQuery: licensePlateQuery, - AddressQuery: addressQuery, - PhoneQuery: phoneQuery, - SocialQuery: socialQuery, - CryptoAddressQuery: cryptoAddressQuery, - Debug: debug, - } -} - -type Creds struct { - gorm.Model - Email string `json:"email" yaml:"email" xml:"email" gorm:"uniqueIndex:idx_email_username_password"` - Username string `json:"username" yaml:"username" xml:"username" gorm:"uniqueIndex:idx_email_username_password"` - Password string `json:"password" yaml:"password" xml:"password" gorm:"uniqueIndex:idx_email_username_password"` -} - -func (Creds) TableName() string { - return "creds" -} - -func (c Creds) ToString() string { - return fmt.Sprintf("%s%s%s", c.Username, "%", c.Password) -} diff --git a/internal/sqlite/tables.go b/internal/sqlite/tables.go deleted file mode 100644 index 0e4493b..0000000 --- a/internal/sqlite/tables.go +++ /dev/null @@ -1,68 +0,0 @@ -package sqlite - -import "strings" - -type Table int64 - -const ( - ResultsTable Table = iota - RunsTable - CredsTable - WhoIsTable - SubdomainsTable - HistoryTable - LookupTable - HunterDomainTable - HunterEmailTable - UnknownTable -) - -func GetTable(userInput string) Table { - switch strings.ToLower(userInput) { - case "results": - return ResultsTable - case "runs": - return RunsTable - case "creds": - return CredsTable - case "whois": - return WhoIsTable - case "subdomains": - return SubdomainsTable - case "history": - return HistoryTable - case "lookup": - return LookupTable - case "hunter_domain": - return HunterDomainTable - case "hunter_email": - return HunterEmailTable - default: - return UnknownTable - } -} - -func (t Table) Object() interface{} { - switch t { - case ResultsTable: - return Result{} - case RunsTable: - return QueryOptions{} - case CredsTable: - return Creds{} - case WhoIsTable: - return WhoisRecord{} - case SubdomainsTable: - return SubdomainRecord{} - case HistoryTable: - return HistoryRecord{} - case LookupTable: - return LookupResult{} - case HunterDomainTable: - return HunterDomainData{} - case HunterEmailTable: - return HunterEmail{} - default: - return nil - } -} diff --git a/internal/sqlite/whois.go b/internal/sqlite/whois.go index d5b0f3f..201791f 100644 --- a/internal/sqlite/whois.go +++ b/internal/sqlite/whois.go @@ -2,7 +2,9 @@ package sqlite import ( "fmt" + "go.uber.org/zap" "gorm.io/gorm" + "gorm.io/gorm/clause" "strings" ) @@ -512,3 +514,117 @@ func formatWhoisContact(sb *strings.Builder, contact Contact, indent string) { sb.WriteString(indent + "Raw Text: " + rawTextPreview + "\n") } } + +func StoreWhoisRecord(whoisRecord WhoisRecord) error { + // Create a pointer to the record to make it addressable + recordPtr := &whoisRecord + + zap.L().Info("Storing WHOIS record", + zap.String("domain", whoisRecord.DomainName)) + + db := GetDB() + + // Use OnConflict clause to handle duplicates + err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(recordPtr).Error + if err != nil { + zap.L().Error("store_whois_record", + zap.String("message", "failed to store whois record"), + zap.Error(err)) + return err + } + + return nil +} + +func StoreWhoisSubdomainRecords(subdomainRecords []SubdomainRecord) error { + if len(subdomainRecords) == 0 { + return nil + } + + zap.L().Info("Storing subdomain records", zap.Int("count", len(subdomainRecords))) + db := GetDB() + + // Use batch insert with conflict handling + const batchSize = 100 + var lastErr error + + for i := 0; i < len(subdomainRecords); i += batchSize { + end := i + batchSize + if end > len(subdomainRecords) { + end = len(subdomainRecords) + } + + batch := subdomainRecords[i:end] + // Use Clauses with OnConflict DoNothing to skip conflicts + err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error + if err != nil { + zap.L().Warn("Error storing some subdomain records", zap.Error(err)) + lastErr = err + // Continue with next batch despite error + } + } + + return lastErr +} + +func StoreWhoisHistoryRecords(historyRecords []HistoryRecord) error { + if len(historyRecords) == 0 { + return nil + } + + zap.L().Info("Storing history records", zap.Int("count", len(historyRecords))) + db := GetDB() + + // Use batch insert with conflict handling + const batchSize = 100 + var lastErr error + + for i := 0; i < len(historyRecords); i += batchSize { + end := i + batchSize + if end > len(historyRecords) { + end = len(historyRecords) + } + + batch := historyRecords[i:end] + // Use Clauses with OnConflict DoNothing to skip conflicts + err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error + if err != nil { + zap.L().Warn("Error storing some history records", zap.Error(err)) + lastErr = err + // Continue with next batch despite error + } + } + + return lastErr +} + +func StoreWhoisLookup(lookup []LookupResult) error { + if len(lookup) == 0 { + return nil + } + + zap.L().Info("Storing IP lookup records", zap.Int("count", len(lookup))) + db := GetDB() + + // Use batch insert with conflict handling + const batchSize = 100 + var lastErr error + + for i := 0; i < len(lookup); i += batchSize { + end := i + batchSize + if end > len(lookup) { + end = len(lookup) + } + + batch := lookup[i:end] + // Use Clauses with OnConflict DoNothing to skip conflicts + err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error + if err != nil { + zap.L().Warn("Error storing some IP lookup records", zap.Error(err)) + lastErr = err + // Continue with next batch despite error + } + } + + return lastErr +} diff --git a/internal/whois/whois.go b/internal/whois/whois.go index 9e1c163..86fa787 100644 --- a/internal/whois/whois.go +++ b/internal/whois/whois.go @@ -552,7 +552,7 @@ func (w *DehashedWhoIs) WhoisIP(ipAddress string) ([]sqlite.LookupResult, error) }) } - sqlite.StoreIPLookup(lookups) + sqlite.StoreWhoisLookup(lookups) return lookups, nil } @@ -702,7 +702,7 @@ func (w *DehashedWhoIs) WhoisMX(mxHostname string) ([]sqlite.LookupResult, error }) } - sqlite.StoreIPLookup(mxLookups) + sqlite.StoreWhoisLookup(mxLookups) return mxLookups, nil } @@ -849,7 +849,7 @@ func (w *DehashedWhoIs) WhoisNS(nsHostname string) ([]sqlite.LookupResult, error }) } - sqlite.StoreIPLookup(nsLookups) + sqlite.StoreWhoisLookup(nsLookups) return nsLookups, nil } From edbf9d57dc2b8655c72d93bd143f79bbf66ed86a Mon Sep 17 00:00:00 2001 From: Evan Hosinski Date: Sat, 17 May 2025 10:27:33 -0400 Subject: [PATCH 5/5] updated dehasher.go in Makefile --- Makefile | 4 ++-- go.mod | 5 +---- go.sum | 6 ------ 3 files changed, 3 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 9a730c6..0de3316 100644 --- a/Makefile +++ b/Makefile @@ -30,14 +30,14 @@ clean: # Build for current platform build: - $(GO) build -o $(BUILD_DIR)/$(BINARY_NAME) -ldflags "-X main.version=$(VERSION)" dehasher.go + $(GO) build -o $(BUILD_DIR)/$(BINARY_NAME) -ldflags "-X main.version=$(VERSION)" crowsnest.go # Build for all platforms build-all: clean @for platform in $(PLATFORMS); do \ for arch in $(ARCHS); do \ echo "Building for $$platform/$$arch..."; \ - GOOS=$$platform GOARCH=$$arch $(GO) build -o $(BUILD_DIR)/$(BINARY_NAME)-$$platform-$$arch -ldflags "-X main.version=$(VERSION)" dehasher.go; \ + GOOS=$$platform GOARCH=$$arch $(GO) build -o $(BUILD_DIR)/$(BINARY_NAME)-$$platform-$$arch -ldflags "-X main.version=$(VERSION)" crowsnest.go; \ if [ "$$platform" = "windows" ]; then \ mv $(BUILD_DIR)/$(BINARY_NAME)-$$platform-$$arch $(BUILD_DIR)/$(BINARY_NAME)-$$platform-$$arch.exe; \ fi; \ diff --git a/go.mod b/go.mod index 77744a1..a69c521 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ toolchain go1.24.3 require ( github.com/charmbracelet/lipgloss v1.1.0 github.com/dgraph-io/badger/v4 v4.7.0 - github.com/olekukonko/tablewriter v1.0.5 + github.com/fatih/color v1.15.0 github.com/spf13/cobra v1.9.1 github.com/winking324/rzap v0.1.0 go.uber.org/zap v1.20.0 @@ -27,7 +27,6 @@ require ( github.com/charmbracelet/x/term v0.2.1 // indirect github.com/dgraph-io/ristretto/v2 v2.2.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/fatih/color v1.15.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/google/flatbuffers v25.2.10+incompatible // indirect @@ -41,8 +40,6 @@ require ( github.com/mattn/go-runewidth v0.0.16 // indirect github.com/mattn/go-sqlite3 v1.14.22 // indirect github.com/muesli/termenv v0.16.0 // indirect - github.com/olekukonko/errors v0.0.0-20250405072817-4e6d85265da6 // indirect - github.com/olekukonko/ll v0.0.7 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/spf13/pflag v1.0.6 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect diff --git a/go.sum b/go.sum index 4ae6881..fb227f2 100644 --- a/go.sum +++ b/go.sum @@ -71,12 +71,6 @@ github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= -github.com/olekukonko/errors v0.0.0-20250405072817-4e6d85265da6 h1:r3FaAI0NZK3hSmtTDrBVREhKULp8oUeqLT5Eyl2mSPo= -github.com/olekukonko/errors v0.0.0-20250405072817-4e6d85265da6/go.mod h1:ppzxA5jBKcO1vIpCXQ9ZqgDh8iwODz6OXIGKU8r5m4Y= -github.com/olekukonko/ll v0.0.7 h1:K66xcUlG2qWRhPoLw/cidmbv4pDDJtZuvJGsR5QTzXo= -github.com/olekukonko/ll v0.0.7/go.mod h1:En+sEW0JNETl26+K8eZ6/W4UQ7CYSrrgg/EdIYT2H8g= -github.com/olekukonko/tablewriter v1.0.5 h1:8+uKJXxYcl29TcpfQdd0vL+l6Kul7Sk7sWolfgErDv0= -github.com/olekukonko/tablewriter v1.0.5/go.mod h1:Z22i2ywMkT9sw64nuWAUaH62kb+umiwucGaQNbFh8Bg= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=