From 65c4ea6a1573983ba2bd12651c271ea454d215c7 Mon Sep 17 00:00:00 2001 From: Evan Hosinski Date: Fri, 16 May 2025 15:33:29 -0400 Subject: [PATCH] Updates to allow for enhanced debugging. Added structs for whois calls. Added ability to write WhoIs to file. Added structured output for Whois Records. Added String Method for WhoIsRecord and WhoIsHistory Records. --- cmd/api.go | 9 +- cmd/root.go | 8 +- cmd/whois.go | 547 +++++++++++--- internal/{query => dehashed}/clientv2.go | 164 ++++- internal/{query => dehashed}/dehashed.go | 52 +- internal/{query => dehashed}/errors.go | 2 +- internal/export/export.go | 119 ++++ internal/files/filetype.go | 1 + internal/sqlite/db.go | 4 +- internal/sqlite/gorm.go | 35 +- internal/sqlite/structs.go | 10 +- internal/sqlite/whois.go | 350 ++++++++- internal/whois/whois.go | 869 +++++++++++++++++++---- 13 files changed, 1869 insertions(+), 301 deletions(-) rename internal/{query => dehashed}/clientv2.go (57%) rename internal/{query => dehashed}/dehashed.go (84%) rename internal/{query => dehashed}/errors.go (98%) diff --git a/cmd/api.go b/cmd/api.go index 6e3ece0..2a7bbaa 100644 --- a/cmd/api.go +++ b/cmd/api.go @@ -2,7 +2,7 @@ package cmd import ( "dehasher/internal/badger" - "dehasher/internal/query" + "dehasher/internal/dehashed" "dehasher/internal/sqlite" "fmt" "github.com/spf13/cobra" @@ -12,7 +12,7 @@ func init() { // Add query command to root command rootCmd.AddCommand(apiCmd) - // Add flags specific to query command + // Add flags specific to api command apiCmd.Flags().IntVarP(&maxRecords, "max-records", "m", 30000, "Maximum amount of records to return") apiCmd.Flags().IntVarP(&maxRequests, "max-requests", "r", -1, "Maximum number of requests to make") apiCmd.Flags().IntVarP(&startingPage, "starting-page", "s", 1, "Starting page for requests") @@ -36,7 +36,7 @@ func init() { apiCmd.Flags().StringVarP(&hashQuery, "hash", "Q", "", "Hashed password query") apiCmd.Flags().StringVarP(&nameQuery, "name", "N", "", "Name query") - // Add mutually exclusive flags to exact match and regex match + // Add mutually exclusive flags to wildcard match and regex match apiCmd.MarkFlagsMutuallyExclusive("regex-match", "wildcard-match") } @@ -103,10 +103,11 @@ var ( wildcardMatch, printBalance, credsOnly, + debugGlobal, ) // Create new Dehasher - dehasher := query.NewDehasher(queryOptions) + dehasher := dehashed.NewDehasher(queryOptions) dehasher.SetClientCredentials( key, ) diff --git a/cmd/root.go b/cmd/root.go index 0eab842..e6f699a 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -11,6 +11,7 @@ import ( var ( // Global Flags + debugGlobal bool // rootCmd is the base command for the CLI. rootCmd = &cobra.Command{ @@ -58,14 +59,11 @@ func Execute() { } func init() { - //// Attempt to retrieve the useLocalDB - //useLocalDatabase := badger.GetUseLocalDB() - // Hide the default help command rootCmd.CompletionOptions.HiddenDefaultCmd = true - //// Add global flags - //rootCmd.PersistentFlags().BoolVar(&useLocalDatabase, "local-db", useLocalDatabase, "Use local database in current directory instead of default path") + // Add global flags + rootCmd.PersistentFlags().BoolVar(&debugGlobal, "debug", false, "Show debugGlobal information") // Add subcommands rootCmd.AddCommand(setKeyCmd) diff --git a/cmd/whois.go b/cmd/whois.go index fa0107c..6d1d6b3 100644 --- a/cmd/whois.go +++ b/cmd/whois.go @@ -1,14 +1,38 @@ package cmd import ( + "dehasher/internal/debug" + "dehasher/internal/export" + "dehasher/internal/files" + "dehasher/internal/pretty" "dehasher/internal/sqlite" "dehasher/internal/whois" "fmt" "github.com/spf13/cobra" "go.uber.org/zap" "strings" + "time" ) +func init() { + // Add whois subcommand to root command + rootCmd.AddCommand(whoisCmd) + + // Add flags specific to whois command + whoisCmd.Flags().StringVarP(&whoisDomain, "domain", "d", "", "Domain for WHOIS lookup, history search, and subdomain scan") + whoisCmd.Flags().StringVarP(&whoisIPAddress, "ip", "i", "", "IP address for reverse IP lookup") + whoisCmd.Flags().StringVarP(&whoisMXAddress, "mx", "m", "", "MX hostname for reverse MX lookup") + whoisCmd.Flags().StringVarP(&whoisNSAddress, "ns", "n", "", "NS hostname for reverse NS lookup") + whoisCmd.Flags().StringVarP(&whoisInclude, "include", "I", "", "Up to 4 Terms to include in reverse WHOIS search (comma-separated)") + whoisCmd.Flags().StringVarP(&whoisExclude, "exclude", "E", "", "Up to 4 Terms to exclude in reverse WHOIS search (comma-separated)") + whoisCmd.Flags().StringVarP(&whoisReverseType, "type", "t", "registrant", "Type of reverse WHOIS search ([default] current or historic)") + whoisCmd.Flags().StringVarP(&whoisOutputFormat, "format", "f", "text", "Output format (text, json)") + whoisCmd.Flags().StringVarP(&whoisOutputFile, "output", "o", "whois", "File to output results to including extension") + whoisCmd.Flags().BoolVarP(&whoisShowCredits, "credits", "c", false, "Show remaining WHOIS credits") + whoisCmd.Flags().BoolVarP(&whoisHistory, "history", "H", false, "Perform WHOIS history search [25 Credits]") + whoisCmd.Flags().BoolVarP(&whoisSubdomainScan, "subdomains", "s", false, "Perform WHOIS subdomain scan") +} + var ( // WHOIS command flags whoisDomain string @@ -19,6 +43,7 @@ var ( whoisExclude string whoisReverseType string whoisOutputFormat string + whoisOutputFile string whoisShowCredits bool whoisHistory bool whoisSubdomainScan bool @@ -37,20 +62,50 @@ var ( return } + if debugGlobal { + debug.PrintInfo("debug mode enabled") + zap.L().Info("whois_debug", + zap.String("message", "debug mode enabled"), + ) + } + + if whoisOutputFile == "" { + if debugGlobal { + debug.PrintInfo("output file not specified, using default") + } + whoisOutputFile = "whois_" + time.Now().Format("05_04_05") + } + + if whoisOutputFormat == "" { + if debugGlobal { + debug.PrintInfo("output format not specified, using default") + } + whoisOutputFormat = "json" + } + + fType := files.GetFileType(whoisOutputFormat) + if fType == files.UNKNOWN { + fmt.Println("[!] Error: Invalid output format. Must be 'json', 'xml', 'yaml', or 'txt'.") + return + } + if debugGlobal { + debug.PrintInfo("using output format: " + whoisOutputFormat) + } + + w := whois.NewWhoIs(key, debugGlobal) + // Show credits if requested if whoisShowCredits { - fmt.Println("[*] Getting WHOIS credits...") - credits, err := whois.GetWHOISCredits(key) + fmt.Println("[*] Getting WHOIS balance...") + balance, err := w.Balance() if err != nil { zap.L().Error("get_whois_credits", - zap.String("message", "failed to get whois credits"), + zap.String("message", "failed to get whois balance"), zap.Error(err), ) - fmt.Printf("Error getting WHOIS credits: %v\n", err) - return + fmt.Printf("Error getting WHOIS balance: %v\n", err) } - fmt.Printf("WHOIS Credits: %d\n", credits.WhoisCredits) - return + fmt.Printf("WHOIS Credits: %d\n", balance) } // Check if domain is provided for history and subdomain scan @@ -59,14 +114,30 @@ var ( fmt.Println("Domain is required for history and subdomain scan.") return } + if whoisShowCredits { + balance, err := w.Balance() + if err != nil { + zap.L().Error("get_whois_credits", + zap.String("message", "failed to get whois balance"), + zap.Error(err), + ) + fmt.Printf("Error getting WHOIS balance: %v\n", err) + } + fmt.Println("WHOIS Credits: ", balance) + } } // Determine which operation to perform based on flags if whoisDomain != "" { fmt.Println("[*] Performing WHOIS lookup...") + // Domain lookup - result, err := whois.WhoisSearch(whoisDomain, key) + result, err := w.WhoisSearch(whoisDomain) if err != nil { + if debugGlobal { + debug.PrintInfo("failed to perform whois search") + debug.PrintError(err) + } zap.L().Error("whois_search", zap.String("message", "failed to perform whois search"), zap.Error(err), @@ -75,12 +146,32 @@ var ( return } + if whoisShowCredits { + balance, err := w.Balance() + if err != nil { + if debugGlobal { + debug.PrintInfo("failed to get whois balance") + debug.PrintError(err) + } + zap.L().Error("get_whois_credits", + zap.String("message", "failed to get whois balance"), + zap.Error(err), + ) + fmt.Printf("Error getting WHOIS balance: %v\n", err) + } + fmt.Println("WHOIS Credits: ", balance) + } + // Fix the output format to use proper formatting - fmt.Printf("WHOIS Lookup Result:\n%+v\n", result.Data.WhoisRecord) + fmt.Println("WHOIS Lookup Result:") // Store the record - err = sqlite.StoreWhoisRecord(result.Data.WhoisRecord) + err = sqlite.StoreWhoisRecord(result) if err != nil { + if debugGlobal { + debug.PrintInfo("failed to store whois record") + debug.PrintError(err) + } zap.L().Error("store_whois_record", zap.String("message", "failed to store whois record"), zap.Error(err), @@ -89,75 +180,187 @@ var ( // Continue execution even if storage fails } + // Pretty Print WhoIs Record + pretty.WhoIsTree(whoisDomain, result) + + // Write WhoIs Record to file + if len(result.DomainName) != 0 { + fmt.Printf("[*] Writing WHOIS record to file: %s%s\n", whoisOutputFile, fType.Extension()) + err = export.WriteWhoIsRecordToFile(result, whoisOutputFile, fType) + } else { + if debugGlobal { + debug.PrintInfo("no whois record to write to file") + } + zap.L().Info("write_whois_record", + zap.String("message", "no whois record to write to file"), + ) + } + if whoisHistory { fmt.Println("[*] Performing WHOIS history search...") // Perform history search - history, err := whois.WhoisHistory(whoisDomain, key) + historyRecords, err := w.WhoisHistory(whoisDomain) if err != nil { + if debugGlobal { + debug.PrintInfo("failed to perform whois history lookup") + debug.PrintError(err) + } zap.L().Error("whois_history", zap.String("message", "failed to perform whois history lookup"), zap.Error(err), ) - fmt.Printf("Error performing WHOIS history lookup: %v\n", err) + fmt.Printf("[!] Error performing WHOIS history lookup: %v\n", err) } else { - fmt.Println("\nWHOIS History:") - fmt.Println(history) - } + if whoisShowCredits { + balance, err := w.Balance() + if err != nil { + if debugGlobal { + debug.PrintInfo("failed to get whois balance") + debug.PrintError(err) + } + zap.L().Error("get_whois_credits", + zap.String("message", "failed to get whois balance"), + zap.Error(err), + ) + fmt.Printf("Error getting WHOIS balance: %v\n", err) + } + fmt.Println("WHOIS Credits: ", balance) + } - err = sqlite.StoreHistoryRecord(history.Data.Records) - if err != nil { - zap.L().Error("store_history_record", - zap.String("message", "failed to store history record"), - zap.Error(err), - ) - fmt.Printf("Error storing WHOIS history record: %v\n", err) + // Write history records to file if any + if len(historyRecords) > 0 { + fmt.Println("[*] Records Found: %d\n", len(historyRecords)) + fmt.Println("[*] WHOIS History being written to file: %s%s\n", whoisOutputFile, fType.Extension()) + writeErr := export.WriteWhoIsHistoryToFile(historyRecords, whoisOutputFile, fType) + if writeErr != nil { + if debugGlobal { + debug.PrintInfo("failed to write whois history to file") + debug.PrintError(writeErr) + } + zap.L().Error("write_whois_history", + zap.String("message", "failed to write whois history to file"), + zap.Error(writeErr), + ) + fmt.Printf("[!] Error writing WHOIS history to file: %v\n", writeErr) + } + + err = sqlite.StoreHistoryRecord(historyRecords) + if err != nil { + if debugGlobal { + debug.PrintInfo("failed to store history record") + debug.PrintError(err) + } + zap.L().Error("store_history_record", + zap.String("message", "failed to store history record"), + zap.Error(err), + ) + fmt.Printf("Error storing WHOIS history record: %v\n", err) + } + + } else { + if debugGlobal { + debug.PrintInfo("no whois history records to write to file") + } + zap.L().Info("write_whois_history", + zap.String("message", "no whois history records to write to file"), + ) + } } } // Perform subdomain scan if whoisSubdomainScan { fmt.Println("[*] Performing WHOIS subdomain scan...") - subdomains, err := whois.WhoisSubdomainScan(whoisDomain, key) + subdomains, err := w.WhoisSubdomainScan(whoisDomain) + + if whoisShowCredits { + balance, err := w.Balance() + if err != nil { + if debugGlobal { + debug.PrintInfo("failed to get whois balance") + debug.PrintError(err) + } + zap.L().Error("get_whois_credits", + zap.String("message", "failed to get whois balance"), + zap.Error(err), + ) + fmt.Printf("Error getting WHOIS balance: %v\n", err) + } + fmt.Println("WHOIS Credits: ", balance) + } + if err != nil { + if debugGlobal { + debug.PrintInfo("failed to perform subdomain scan") + debug.PrintError(err) + } zap.L().Error("whois_subdomain_scan", zap.String("message", "failed to perform subdomain scan"), zap.Error(err), ) fmt.Printf("Error performing subdomain scan: %v\n", err) } else { - fmt.Println("\nSubdomain Scan:") - fmt.Println(subdomains) - } + fmt.Println("Subdomain Scan:") + err = sqlite.StoreSubdomainRecords(subdomains) + if err != nil { + if debugGlobal { + debug.PrintInfo("failed to store subdomain record") + debug.PrintError(err) + } + zap.L().Error("store_subdomain_record", + zap.String("message", "failed to store subdomain record"), + zap.Error(err), + ) + fmt.Printf("Error storing WHOIS subdomain record: %v\n", err) + } - err = sqlite.StoreSubdomainRecord(subdomains.Data.Result.Records) - if err != nil { - zap.L().Error("store_subdomain_record", - zap.String("message", "failed to store subdomain record"), - zap.Error(err), - ) - fmt.Printf("Error storing WHOIS subdomain record: %v\n", err) + // Write the subdomains to file if any + if len(subdomains) > 0 { + fmt.Printf("[*] Writing subdomains to file: %s%s\n", whoisOutputFile, fType.Extension()) + err = export.WriteSubdomainsToFile(subdomains, whoisOutputFile, fType) + if err != nil { + zap.L().Error("write_whois_subdomain", + zap.String("message", "failed to write whois subdomain to file"), + zap.Error(err), + ) + fmt.Printf("Error writing WHOIS subdomain to file: %v\n", err) + } + + var ( + headers = []string{"Domain", "First Seen", "Last Seen"} + rows [][]string + ) + + // Create the rows + for _, r := range subdomains { + rows = append(rows, []string{r.Domain, time.Unix(r.FirstSeen, 0).String(), time.Unix(r.LastSeen, 0).String()}) + } + + // Store the subdomains + pretty.Table(headers, rows) + + } else { + fmt.Println("[!] No subdomains found") + zap.L().Info("write_whois_subdomain", + zap.String("message", "no whois subdomain records to write to file"), + zap.String("domain", whoisDomain), + ) + } } } - // Get credits - credits, err := whois.GetWHOISCredits(key) - if err != nil { - zap.L().Error("get_whois_credits", - zap.String("message", "failed to get whois credits"), - zap.Error(err), - ) - fmt.Printf("Error getting WHOIS credits: %v\n", err) - return - } - fmt.Printf("\nWHOIS Credits Remaining: %d\n", credits.WhoisCredits) return } if whoisIPAddress != "" { fmt.Println("[*] Performing reverse IP lookup...") // IP lookup - result, err := whois.WhoisIP(whoisIPAddress, key) + result, err := w.WhoisIP(whoisIPAddress) if err != nil { + if debugGlobal { + debug.PrintInfo("failed to perform ip lookup") + debug.PrintError(err) + } zap.L().Error("whois_ip", zap.String("message", "failed to perform ip lookup"), zap.Error(err), @@ -165,28 +368,73 @@ var ( fmt.Printf("Error performing IP lookup: %v\n", err) return } - fmt.Println("IP Lookup Result:") - fmt.Println(string(result)) // Get credits - credits, err := whois.GetWHOISCredits(key) - if err != nil { - zap.L().Error("get_whois_credits", - zap.String("message", "failed to get whois credits"), - zap.Error(err), + if whoisShowCredits { + balance, err := w.Balance() + if err != nil { + if debugGlobal { + debug.PrintInfo("failed to get whois balance") + debug.PrintError(err) + } + zap.L().Error("get_whois_credits", + zap.String("message", "failed to get whois balance"), + zap.Error(err), + ) + fmt.Printf("Error getting WHOIS balance: %v\n", err) + } + fmt.Println("WHOIS Credits: ", balance) + } + + if len(result) == 0 { + fmt.Println("[!] No results found") + zap.L().Info("whois_ip", + zap.String("message", "no results found"), + zap.String("ip", whoisIPAddress), ) - fmt.Printf("Error getting WHOIS credits: %v\n", err) return } - fmt.Printf("\nWHOIS Credits Remaining: %d\n", credits.WhoisCredits) + + // Write the results to file + fmt.Printf("[*] Writing IP lookup results to file: %s%s\n", whoisOutputFile, fType.Extension()) + err = export.WriteIPLookupToFile(result, whoisOutputFile, fType) + if err != nil { + if debugGlobal { + debug.PrintInfo("failed to write ip lookup to file") + debug.PrintError(err) + } + zap.L().Error("write_ip_lookup", + zap.String("message", "failed to write ip lookup to file"), + zap.Error(err), + ) + fmt.Printf("Error writing IP lookup to file: %v\n", err) + } + + // Pretty Print the JSON + var ( + headers = []string{"Name", "Search Term", "First Seen", "Last Visit", "Type"} + rows [][]string + ) + fmt.Println("IP Lookup Result:") + + for _, r := range result { + rows = append(rows, []string{r.Name, r.SearchTerm, time.Unix(r.FirstSeen, 0).String(), time.Unix(r.LastVisit, 0).String(), r.Type}) + } + + pretty.Table(headers, rows) + return } if whoisMXAddress != "" { fmt.Println("[*] Performing reverse MX lookup...") // MX lookup - result, err := whois.WhoisMX(whoisMXAddress, key) + result, err := w.WhoisMX(whoisMXAddress) if err != nil { + if debugGlobal { + debug.PrintInfo("failed to perform mx lookup") + debug.PrintError(err) + } zap.L().Error("whois_mx", zap.String("message", "failed to perform mx lookup"), zap.Error(err), @@ -195,29 +443,72 @@ var ( return } - // todo unmarshal mx lookup - fmt.Println("MX Lookup Result:") - fmt.Println(result) - // Get credits - credits, err := whois.GetWHOISCredits(key) - if err != nil { - zap.L().Error("get_whois_credits", - zap.String("message", "failed to get whois credits"), - zap.Error(err), + if whoisShowCredits { + balance, err := w.Balance() + if err != nil { + if debugGlobal { + debug.PrintInfo("failed to get whois balance") + debug.PrintError(err) + } + zap.L().Error("get_whois_credits", + zap.String("message", "failed to get whois balance"), + zap.Error(err), + ) + fmt.Printf("Error getting WHOIS balance: %v\n", err) + } + fmt.Println("WHOIS Credits: ", balance) + } + + if len(result) == 0 { + fmt.Println("[!] No results found") + zap.L().Info("whois_mx", + zap.String("message", "no results found"), + zap.String("mx", whoisMXAddress), ) - fmt.Printf("Error getting WHOIS credits: %v\n", err) return } - fmt.Printf("\nWHOIS Credits Remaining: %d\n", credits.WhoisCredits) + + // Write the results to file + fmt.Printf("[*] Writing MX lookup results to file: %s%s\n", whoisOutputFile, fType.Extension()) + err = export.WriteIPLookupToFile(result, whoisOutputFile, fType) + if err != nil { + if debugGlobal { + debug.PrintInfo("failed to write mx lookup to file") + debug.PrintError(err) + } + zap.L().Error("write_mx_lookup", + zap.String("message", "failed to write mx lookup to file"), + zap.Error(err), + ) + fmt.Printf("Error writing MX lookup to file: %v\n", err) + } + + // Pretty Print the JSON + var ( + headers = []string{"Name", "Search Term", "First Seen", "Last Visit", "Type"} + rows [][]string + ) + fmt.Println("MX Lookup Result:") + + for _, r := range result { + rows = append(rows, []string{r.Name, r.SearchTerm, time.Unix(r.FirstSeen, 0).String(), time.Unix(r.LastVisit, 0).String(), r.Type}) + } + + pretty.Table(headers, rows) + return } if whoisNSAddress != "" { fmt.Println("[*] Performing reverse NS lookup...") // NS lookup - result, err := whois.WhoisNS(whoisNSAddress, key) + result, err := w.WhoisNS(whoisNSAddress) if err != nil { + if debugGlobal { + debug.PrintInfo("failed to perform ns lookup") + debug.PrintError(err) + } zap.L().Error("whois_ns", zap.String("message", "failed to perform ns lookup"), zap.Error(err), @@ -225,20 +516,61 @@ var ( fmt.Printf("Error performing NS lookup: %v\n", err) return } - fmt.Println("NS Lookup Result:") - fmt.Println(result) // Get credits - credits, err := whois.GetWHOISCredits(key) - if err != nil { - zap.L().Error("get_whois_credits", - zap.String("message", "failed to get whois credits"), - zap.Error(err), + if whoisShowCredits { + balance, err := w.Balance() + if err != nil { + if debugGlobal { + debug.PrintInfo("failed to get whois balance") + debug.PrintError(err) + } + zap.L().Error("get_whois_credits", + zap.String("message", "failed to get whois balance"), + zap.Error(err), + ) + fmt.Printf("Error getting WHOIS balance: %v\n", err) + } + fmt.Println("WHOIS Credits: ", balance) + } + + if len(result) == 0 { + fmt.Println("[!] No results found") + zap.L().Info("whois_ns", + zap.String("message", "no results found"), + zap.String("ns", whoisNSAddress), ) - fmt.Printf("Error getting WHOIS credits: %v\n", err) return } - fmt.Printf("\nWHOIS Credits Remaining: %d\n", credits.WhoisCredits) + + // Write the results to file + fmt.Printf("[*] Writing NS lookup results to file: %s%s\n", whoisOutputFile, fType.Extension()) + err = export.WriteIPLookupToFile(result, whoisOutputFile, fType) + if err != nil { + if debugGlobal { + debug.PrintInfo("failed to write ns lookup to file") + debug.PrintError(err) + } + zap.L().Error("write_ns_lookup", + zap.String("message", "failed to write ns lookup to file"), + zap.Error(err), + ) + fmt.Printf("Error writing NS lookup to file: %v\n", err) + } + + // Pretty Print the JSON + var ( + headers = []string{"Name", "Search Term", "First Seen", "Last Visit", "Type"} + rows [][]string + ) + fmt.Println("NS Lookup Result:") + + for _, r := range result { + rows = append(rows, []string{r.Name, r.SearchTerm, time.Unix(r.FirstSeen, 0).String(), time.Unix(r.LastVisit, 0).String(), r.Type}) + } + + pretty.Table(headers, rows) + return } @@ -247,25 +579,66 @@ var ( includeTerms := []string{} if whoisInclude != "" { includeTerms = strings.Split(whoisInclude, ",") + if len(includeTerms) > 4 { + fmt.Println("[!] Error: Maximum of 4 include terms allowed.") + return + } } excludeTerms := []string{} if whoisExclude != "" { excludeTerms = strings.Split(whoisExclude, ",") + if len(excludeTerms) > 4 { + fmt.Println("[!] Error: Maximum of 4 exclude terms allowed.") + return + } } if whoisReverseType == "" { - whoisReverseType = "registrant" + if debugGlobal { + debug.PrintInfo("reverse type not specified, using default") + } + whoisReverseType = "current" + } else { + toLower := strings.ToLower(whoisReverseType) + if toLower != "current" && toLower != "historic" { + fmt.Println("[!] Error: Invalid reverse type. Must be 'current' or 'historic'.") + return + } } fmt.Println("[*] Performing reverse WHOIS lookup...") - result, err := whois.ReverseWHOIS(includeTerms, excludeTerms, whoisReverseType, key) + result, err := w.ReverseWHOIS(includeTerms, excludeTerms, whoisReverseType) if err != nil { + if debugGlobal { + debug.PrintInfo("failed to perform reverse whois") + debug.PrintError(err) + } + zap.L().Error("reverse_whois", + zap.String("message", "failed to perform reverse whois"), + zap.Error(err), + ) fmt.Printf("Error performing reverse WHOIS: %v\n", err) return } fmt.Println("Reverse WHOIS Result:") fmt.Println(result) + + if whoisShowCredits { + balance, err := w.Balance() + if err != nil { + if debugGlobal { + debug.PrintInfo("failed to get whois balance") + debug.PrintError(err) + } + zap.L().Error("get_whois_credits", + zap.String("message", "failed to get whois balance"), + zap.Error(err), + ) + fmt.Printf("Error getting WHOIS balance: %v\n", err) + } + fmt.Println("WHOIS Credits: ", balance) + } return } @@ -274,21 +647,3 @@ var ( }, } ) - -func init() { - // Add whois command to root command - rootCmd.AddCommand(whoisCmd) - - // Add flags specific to whois command - whoisCmd.Flags().StringVarP(&whoisDomain, "domain", "d", "", "Domain for WHOIS lookup, history search, and subdomain scan") - whoisCmd.Flags().StringVarP(&whoisIPAddress, "ip", "i", "", "IP address for reverse IP lookup") - whoisCmd.Flags().StringVarP(&whoisMXAddress, "mx", "m", "", "MX address for reverse MX lookup") - whoisCmd.Flags().StringVarP(&whoisNSAddress, "ns", "n", "", "NS address for reverse NS lookup") - whoisCmd.Flags().StringVarP(&whoisInclude, "include", "I", "", "Terms to include in reverse WHOIS search (comma-separated)") - whoisCmd.Flags().StringVarP(&whoisExclude, "exclude", "E", "", "Terms to exclude in reverse WHOIS search (comma-separated)") - whoisCmd.Flags().StringVarP(&whoisReverseType, "type", "t", "registrant", "Type of reverse WHOIS search (registrant, email, organization, address, phone)") - whoisCmd.Flags().StringVarP(&whoisOutputFormat, "format", "f", "text", "Output format (text, json)") - whoisCmd.Flags().BoolVarP(&whoisShowCredits, "credits", "c", false, "Show remaining WHOIS credits") - whoisCmd.Flags().BoolVarP(&whoisHistory, "history", "H", false, "Perform WHOIS history search [25 Credits]") - whoisCmd.Flags().BoolVarP(&whoisSubdomainScan, "subdomains", "s", false, "Perform WHOIS subdomain scan") -} diff --git a/internal/query/clientv2.go b/internal/dehashed/clientv2.go similarity index 57% rename from internal/query/clientv2.go rename to internal/dehashed/clientv2.go index 894f68b..0a50b51 100644 --- a/internal/query/clientv2.go +++ b/internal/dehashed/clientv2.go @@ -1,8 +1,9 @@ -package query +package dehashed import ( "bytes" "crypto/sha256" + "dehasher/internal/debug" "dehasher/internal/sqlite" "encoding/hex" "encoding/json" @@ -38,6 +39,7 @@ func (dp DehashedParameter) GetArgumentString(arg string) string { type DehashedSearchRequest struct { ForcePlaintext bool `json:"-"` + Debug bool `json:"-"` Page int `json:"page"` Query string `json:"query"` Size int `json:"size"` @@ -46,42 +48,60 @@ type DehashedSearchRequest struct { DeDupe bool `json:"de_dupe"` } -func NewDehashedSearchRequest(page, size int, wildcard, regex, forcePlaintext bool) *DehashedSearchRequest { - return &DehashedSearchRequest{Page: page, Query: "", Size: size, Wildcard: wildcard, Regex: regex, DeDupe: true, ForcePlaintext: forcePlaintext} +func NewDehashedSearchRequest(page, size int, wildcard, regex, forcePlaintext, debug bool) *DehashedSearchRequest { + return &DehashedSearchRequest{Page: page, Query: "", Size: size, Wildcard: wildcard, Regex: regex, DeDupe: true, ForcePlaintext: forcePlaintext, Debug: debug} } -func (dsr *DehashedSearchRequest) buildQuery(query string, param DehashedParameter) { +func (dsr *DehashedSearchRequest) buildQuery(query string) { + if dsr.Debug { + debug.PrintInfo(fmt.Sprintf("building query: %s", query)) + } + // Ensure query is properly formatted + query = strings.TrimSpace(query) + + // For regex queries, we need to ensure the regex pattern is properly escaped + // and not enquoted, as that would break the regex pattern + if dsr.Regex && !strings.HasPrefix(query, "\"") && !strings.HasSuffix(query, "\"") { + // Don't add extra quotes for regex patterns + } else if strings.Contains(query, " ") && !strings.HasPrefix(query, "\"") { + query = fmt.Sprintf("\"%s\"", query) + } + if len(dsr.Query) > 0 { - dsr.Query = fmt.Sprintf("%s&%s", strings.TrimSpace(dsr.Query), strings.TrimSpace(query)) + dsr.Query = fmt.Sprintf("%s&%s", strings.TrimSpace(dsr.Query), query) } else { dsr.Query = query } + + if dsr.Debug { + debug.PrintInfo(fmt.Sprintf("query built: %s", dsr.Query)) + } } func (dsr *DehashedSearchRequest) AddUsernameQuery(query string) { query = enquoteSpaced(query) - dsr.buildQuery(Username.GetArgumentString(query), Username) + dsr.buildQuery(Username.GetArgumentString(query)) } func (dsr *DehashedSearchRequest) AddEmailQuery(query string) { query = enquoteSpaced(query) - dsr.buildQuery(Email.GetArgumentString(query), Email) + dsr.buildQuery(Email.GetArgumentString(query)) } func (dsr *DehashedSearchRequest) AddIpAddressQuery(query string) { query = enquoteSpaced(query) - dsr.buildQuery(IpAddress.GetArgumentString(query), IpAddress) + dsr.buildQuery(IpAddress.GetArgumentString(query)) } func (dsr *DehashedSearchRequest) AddDomainQuery(query string) { query = enquoteSpaced(query) - dsr.buildQuery(Domain.GetArgumentString(query), Domain) + dsr.buildQuery(Domain.GetArgumentString(query)) } func (dsr *DehashedSearchRequest) AddPasswordQuery(query string) { if dsr.ForcePlaintext { query = enquoteSpaced(query) - dsr.buildQuery(Password.GetArgumentString(query), Password) + dsr.buildQuery(Password.GetArgumentString(query)) return } hash := sha256.Sum256([]byte(query)) @@ -91,89 +111,126 @@ func (dsr *DehashedSearchRequest) AddPasswordQuery(query string) { func (dsr *DehashedSearchRequest) AddVinQuery(query string) { query = enquoteSpaced(query) - dsr.buildQuery(Vin.GetArgumentString(query), Vin) + dsr.buildQuery(Vin.GetArgumentString(query)) } func (dsr *DehashedSearchRequest) AddLicensePlateQuery(query string) { query = enquoteSpaced(query) - dsr.buildQuery(LicensePlate.GetArgumentString(query), LicensePlate) + dsr.buildQuery(LicensePlate.GetArgumentString(query)) } func (dsr *DehashedSearchRequest) AddAddressQuery(query string) { query = enquoteSpaced(query) - dsr.buildQuery(Address.GetArgumentString(query), Address) + dsr.buildQuery(Address.GetArgumentString(query)) } func (dsr *DehashedSearchRequest) AddPhoneQuery(query string) { query = enquoteSpaced(query) - dsr.buildQuery(Phone.GetArgumentString(query), Phone) + dsr.buildQuery(Phone.GetArgumentString(query)) } func (dsr *DehashedSearchRequest) AddSocialQuery(query string) { query = enquoteSpaced(query) - dsr.buildQuery(Social.GetArgumentString(query), Social) + dsr.buildQuery(Social.GetArgumentString(query)) } func (dsr *DehashedSearchRequest) AddCryptoAddressQuery(query string) { query = enquoteSpaced(query) - dsr.buildQuery(CryptoAddress.GetArgumentString(query), CryptoAddress) + dsr.buildQuery(CryptoAddress.GetArgumentString(query)) } func (dsr *DehashedSearchRequest) AddHashedPasswordQuery(query string) { query = strings.TrimSpace(query) - dsr.buildQuery(HashedPassword.GetArgumentString(query), HashedPassword) + dsr.buildQuery(HashedPassword.GetArgumentString(query)) } func (dsr *DehashedSearchRequest) AddNameQuery(query string) { query = enquoteSpaced(query) - dsr.buildQuery(Name.GetArgumentString(query), Name) + dsr.buildQuery(Name.GetArgumentString(query)) } type DehashedClientV2 struct { apiKey string results []sqlite.Result + debug bool } -func NewDehashedClientV2(apiKey string) *DehashedClientV2 { - return &DehashedClientV2{apiKey: apiKey} +func NewDehashedClientV2(apiKey string, debug bool) *DehashedClientV2 { + return &DehashedClientV2{apiKey: apiKey, debug: debug} } -func (dcv2 *DehashedClientV2) Search(searchRequest DehashedSearchRequest) (int, error) { +func (dcv2 *DehashedClientV2) Search(searchRequest DehashedSearchRequest) (int, int, error) { + if dcv2.debug { + debug.PrintInfo("preparing search request") + zap.L().Info("v2_search_debug", + zap.String("message", "preparing search request"), + ) + } reqBody, _ := json.Marshal(searchRequest) + + if dcv2.debug { + j := string(reqBody) + jReq := fmt.Sprintf("Request Body: %s\n", j) + debug.PrintJson(jReq) + zap.L().Info("v2_search_debug", + zap.String("message", jReq), + zap.String("body", j), + ) + } + req, err := http.NewRequest("POST", "https://api.dehashed.com/v2/search", bytes.NewReader(reqBody)) if err != nil { - return -1, err + return -1, -1, err } + + if dcv2.debug { + debug.PrintInfo("setting headers") + zap.L().Info("v2_search_debug", + zap.String("message", "setting headers"), + ) + } + req.Header.Set("Content-Type", "application/json") req.Header.Set("Dehashed-Api-Key", dcv2.apiKey) + + if dcv2.debug { + headers := req.Header.Clone() + h := fmt.Sprintf("Headers: %v\n", headers) + debug.PrintJson(h) + zap.L().Info("v2_search_debug", + zap.String("message", h), + zap.String("headers", fmt.Sprintf("%v", headers)), + ) + + debug.PrintInfo("performing request") + zap.L().Info("v2_search_debug", + zap.String("message", "performing request"), + ) + } + res, err := http.DefaultClient.Do(req) if res != nil { defer res.Body.Close() } if err != nil { + if dcv2.debug { + debug.PrintInfo("failed to perform request") + debug.PrintError(err) + } zap.L().Error("v2_search", zap.String("message", "failed to perform request"), zap.Error(err), ) - return -1, err + return -1, -1, err } if res == nil { + if dcv2.debug { + debug.PrintInfo("response was nil") + } zap.L().Error("v2_search", zap.String("message", "response was nil"), ) - return -1, errors.New("response was nil") - } - - // Check for HTTP status code errors - if res.StatusCode != 200 { - dhErr := GetDehashedError(res.StatusCode) - fmt.Printf("[%d] API Error message: %s\n", res.StatusCode, dhErr.Error()) - zap.L().Error("v2_search", - zap.String("message", "received error status code"), - zap.Int("status_code", res.StatusCode), - zap.String("error", dhErr.Error()), - ) - return -1, &dhErr + return -1, -1, errors.New("response was nil") } b, err := io.ReadAll(res.Body) @@ -182,21 +239,50 @@ func (dcv2 *DehashedClientV2) Search(searchRequest DehashedSearchRequest) (int, zap.String("message", "failed to read response body"), zap.Error(err), ) - return -1, err + return -1, -1, err + } + + // Check for HTTP status code errors + if res.StatusCode != 200 { + if dcv2.debug { + debug.PrintInfo("received error status code") + debug.PrintJson(fmt.Sprintf("Status Code: %d\n", res.StatusCode)) + debug.PrintJson(fmt.Sprintf("Body: %s\n", string(b[:]))) + } + + dhErr := GetDehashedError(res.StatusCode) + zap.L().Error("v2_search", + zap.String("message", "received error status code"), + zap.Int("status_code", res.StatusCode), + zap.String("error", dhErr.Error()), + zap.String("body_error", string(b)), + ) + return -1, -1, &dhErr } var responseResults sqlite.DehashedResponse err = json.Unmarshal(b, &responseResults) if err != nil { + if dcv2.debug { + debug.PrintInfo("failed to unmarshal response body") + debug.PrintError(err) + } zap.L().Error("v2_search", zap.String("message", "failed to unmarshal response body"), zap.Error(err), ) - return -1, err + return -1, -1, err + } + + if dcv2.debug { + debug.PrintInfo("appending results") + debug.PrintJson(fmt.Sprintf("Total Results: %d\n", responseResults.TotalResults)) + debug.PrintJson(fmt.Sprintf("Balance: %d\n", responseResults.Balance)) + debug.PrintJson(fmt.Sprintf("Entries: %d\n", len(responseResults.Entries))) } dcv2.results = append(dcv2.results, responseResults.Entries...) - return responseResults.TotalResults, nil + return responseResults.TotalResults, responseResults.Balance, nil } func (dcv2 *DehashedClientV2) GetResults() sqlite.DehashedResults { diff --git a/internal/query/dehashed.go b/internal/dehashed/dehashed.go similarity index 84% rename from internal/query/dehashed.go rename to internal/dehashed/dehashed.go index 9511978..6c0e249 100644 --- a/internal/query/dehashed.go +++ b/internal/dehashed/dehashed.go @@ -1,6 +1,7 @@ -package query +package dehashed import ( + "dehasher/internal/debug" "dehasher/internal/export" "dehasher/internal/sqlite" "encoding/json" @@ -13,6 +14,8 @@ import ( type Dehasher struct { options sqlite.QueryOptions nextPage int + debug bool + balance int request *DehashedSearchRequest client *DehashedClientV2 } @@ -22,19 +25,24 @@ func NewDehasher(options *sqlite.QueryOptions) *Dehasher { dh := &Dehasher{ options: *options, nextPage: options.StartingPage + 1, + debug: options.Debug, + balance: 0, } dh.setQueries() - dh.request = NewDehashedSearchRequest(dh.options.StartingPage, dh.options.MaxRecords, dh.options.WildcardMatch, dh.options.RegexMatch, false) + dh.request = NewDehashedSearchRequest(dh.options.StartingPage, dh.options.MaxRecords, dh.options.WildcardMatch, dh.options.RegexMatch, false, options.Debug) dh.buildRequest() return dh } // SetClientCredentials sets the client credentials for the dehasher func (dh *Dehasher) SetClientCredentials(key string) { - dh.client = NewDehashedClientV2(key) + dh.client = NewDehashedClientV2(key, dh.debug) } func (dh *Dehasher) getNextPage() int { + if dh.debug { + debug.PrintInfo(fmt.Sprintf("getting next page: %d", dh.nextPage)) + } nextPage := dh.nextPage dh.nextPage += 1 return nextPage @@ -44,6 +52,10 @@ func (dh *Dehasher) getNextPage() int { func (dh *Dehasher) setQueries() { var numQueries int + if dh.debug { + debug.PrintInfo("setting queries") + } + switch { case dh.options.MaxRequests == 0: zap.L().Error("max requests cannot be zero") @@ -80,6 +92,12 @@ func (dh *Dehasher) setQueries() { } dh.options.MaxRequests = numQueries + + if dh.debug { + debug.PrintInfo(fmt.Sprintf("setting max requests: %d", numQueries)) + debug.PrintInfo(fmt.Sprintf("setting max records: %d", dh.options.MaxRecords)) + } + fmt.Printf("Making %d Requests for %d Records (%d Total)\n", dh.options.MaxRequests, dh.options.MaxRecords, dh.options.MaxRequests*dh.options.MaxRecords) } @@ -88,11 +106,16 @@ func (dh *Dehasher) Start() { fmt.Printf("[*] Querying Dehashed API...\n") for i := 0; i < dh.options.MaxRequests; i++ { fmt.Printf(" [*] Performing Request...\n") - count, err := dh.client.Search(*dh.request) + count, balance, err := dh.client.Search(*dh.request) if err != nil { + if dh.debug { + debug.PrintInfo("error performing request") + debug.PrintError(err) + } + // Check if it's a DehashError if dhErr, ok := err.(*DehashError); ok { - fmt.Printf(" [!] Dehashed API Error: %s (Code: %d)\n", dhErr.Message, dhErr.Code) + fmt.Printf(" [!] Dehashed API Error: %s (Code: %d)\n", dhErr.Message, dhErr.Code) zap.L().Error("dehashed_api_error", zap.String("message", dhErr.Message), zap.Int("code", dhErr.Code), @@ -104,9 +127,24 @@ func (dh *Dehasher) Start() { zap.Error(err), ) } + + if len(dh.client.results) > 0 { + fmt.Printf(" [!] Partial results retrieved. Storing Results...\n") + err := sqlite.StoreResults(dh.client.GetResults()) + if err != nil { + zap.L().Error("store_results", + zap.String("message", "failed to store results"), + zap.Error(err), + ) + fmt.Printf(" [!] Error storing results: %v\n", err) + } + } + dh.parseResults() os.Exit(-1) } + dh.balance = balance + if count < dh.options.MaxRecords { fmt.Printf(" [+] Retrieved %d records\n", count) fmt.Printf(" [-] Not enough entries, ending queries\n") @@ -115,6 +153,10 @@ func (dh *Dehasher) Start() { fmt.Printf(" [+] Retrieved %d records\n", dh.options.MaxRecords) } + if dh.options.PrintBalance { + fmt.Printf(" [*] Balance: %d\n", balance) + } + dh.request.Page = dh.getNextPage() } diff --git a/internal/query/errors.go b/internal/dehashed/errors.go similarity index 98% rename from internal/query/errors.go rename to internal/dehashed/errors.go index 12027e1..7c8b391 100644 --- a/internal/query/errors.go +++ b/internal/dehashed/errors.go @@ -1,4 +1,4 @@ -package query +package dehashed type DehashError struct { Message string diff --git a/internal/export/export.go b/internal/export/export.go index 18781ad..d97a78e 100644 --- a/internal/export/export.go +++ b/internal/export/export.go @@ -11,6 +11,7 @@ import ( "io/ioutil" "os" "strings" + "time" ) func WriteCredsToFile(creds []sqlite.Creds, outputFile string, fileType files.FileType) error { @@ -131,3 +132,121 @@ func WriteQueryResultsToFile(results []map[string]interface{}, outputFile string filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String()) return os.WriteFile(filePath, data, 0644) } + +func WriteWhoIsHistoryToFile(results []sqlite.HistoryRecord, outputFile string, fileType files.FileType) error { + var data []byte + var err error + + switch fileType { + case files.JSON: + data, err = json.MarshalIndent(results, "", " ") + case files.XML: + data, err = xml.MarshalIndent(results, "", " ") + case files.YAML: + data, err = yaml.Marshal(results) + case files.TEXT: + var outStrings []string + for _, r := range results { + outStrings = append(outStrings, r.String()+"\n\n") + } + data = []byte(strings.Join(outStrings, "")) + default: + return errors.New("unsupported file type") + } + + if err != nil { + return err + } + + filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String()) + return os.WriteFile(filePath, data, 0644) +} + +func WriteWhoIsRecordToFile(record sqlite.WhoisRecord, outputFile string, fileType files.FileType) error { + var data []byte + var err error + + switch fileType { + case files.JSON: + data, err = json.MarshalIndent(record, "", " ") + case files.XML: + data, err = xml.MarshalIndent(record, "", " ") + case files.YAML: + data, err = yaml.Marshal(record) + case files.TEXT: + data = []byte(record.String()) + default: + return errors.New("unsupported file type") + } + + if err != nil { + return err + } + + filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String()) + return os.WriteFile(filePath, data, 0644) +} + +func WriteSubdomainsToFile(records []sqlite.SubdomainRecord, outputFile string, fileType files.FileType) error { + var data []byte + var err error + + switch fileType { + case files.JSON: + data, err = json.MarshalIndent(records, "", " ") + case files.XML: + data, err = xml.MarshalIndent(records, "", " ") + case files.YAML: + data, err = yaml.Marshal(records) + case files.TEXT: + var outStrings []string + for _, r := range records { + out := fmt.Sprintf( + "Domain: %s\nFirst Seen: %s\nLast Seen: %s\n\n", + r.Domain, time.Unix(r.FirstSeen, 0).String(), time.Unix(r.LastSeen, 0).String()) + outStrings = append(outStrings, out) + } + data = []byte(strings.Join(outStrings, "")) + default: + return errors.New("unsupported file type") + } + + if err != nil { + return err + } + + filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String()) + return os.WriteFile(filePath, data, 0644) +} + +func WriteIPLookupToFile(records []sqlite.LookupResult, outputFile string, fileType files.FileType) error { + var data []byte + var err error + + switch fileType { + case files.JSON: + data, err = json.MarshalIndent(records, "", " ") + case files.XML: + data, err = xml.MarshalIndent(records, "", " ") + case files.YAML: + data, err = yaml.Marshal(records) + case files.TEXT: + var outStrings []string + for _, r := range records { + out := fmt.Sprintf( + "Name: %s\nSearch Term: %s\nFirst Seen: %s\nLast Visit: %s\nType: %s\n\n", + r.Name, r.SearchTerm, time.Unix(r.FirstSeen, 0).String(), time.Unix(r.LastVisit, 0).String(), r.Type) + outStrings = append(outStrings, out) + } + data = []byte(strings.Join(outStrings, "")) + default: + return errors.New("unsupported file type") + } + + if err != nil { + return err + } + + filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String()) + return os.WriteFile(filePath, data, 0644) +} diff --git a/internal/files/filetype.go b/internal/files/filetype.go index 11cecae..aa085cb 100644 --- a/internal/files/filetype.go +++ b/internal/files/filetype.go @@ -7,6 +7,7 @@ const ( XML YAML TEXT + UNKNOWN ) func GetFileType(filetype string) FileType { diff --git a/internal/sqlite/db.go b/internal/sqlite/db.go index b5e3db9..31dba17 100644 --- a/internal/sqlite/db.go +++ b/internal/sqlite/db.go @@ -176,7 +176,7 @@ func QueryRuns(limit, lastXRuns int, startDate, endDate time.Time, containsQuery // Apply query filter if provided if containsQuery != "" { - // Search in all query fields + // SearchTerm in all query fields query = query.Where( "username_query LIKE ? OR "+ "email_query LIKE ? OR "+ @@ -238,7 +238,7 @@ func GetRunsCount(lastXRuns int, startDate, endDate time.Time, containsQuery str // Apply query filter if provided if containsQuery != "" { - // Search in all query fields + // SearchTerm in all query fields query = query.Where( "username_query LIKE ? OR "+ "email_query LIKE ? OR "+ diff --git a/internal/sqlite/gorm.go b/internal/sqlite/gorm.go index d4cc7ca..affb4da 100644 --- a/internal/sqlite/gorm.go +++ b/internal/sqlite/gorm.go @@ -51,7 +51,7 @@ func InitDB(dbPath string) (*gorm.DB, error) { } // Auto migrate your models - err = db.AutoMigrate(&Result{}, &Creds{}, QueryOptions{}, Creds{}, WhoisRecord{}, SubdomainRecord{}, HistoryRecord{}) + err = db.AutoMigrate(&Result{}, &Creds{}, &QueryOptions{}, &Creds{}, &WhoisRecord{}, &SubdomainRecord{}, &HistoryRecord{}, &LookupResult{}) if err != nil { zap.L().Error("Failed to migrate database", zap.Error(err)) return nil, fmt.Errorf("failed to migrate database: %w", err) @@ -163,7 +163,7 @@ func StoreWhoisRecord(whoisRecord WhoisRecord) error { return nil } -func StoreSubdomainRecord(subdomainRecords []SubdomainRecord) error { +func StoreSubdomainRecords(subdomainRecords []SubdomainRecord) error { if len(subdomainRecords) == 0 { return nil } @@ -224,3 +224,34 @@ func StoreHistoryRecord(historyRecords []HistoryRecord) error { return lastErr } + +func StoreIPLookup(ipLookup []LookupResult) error { + if len(ipLookup) == 0 { + return nil + } + + zap.L().Info("Storing IP lookup records", zap.Int("count", len(ipLookup))) + db := GetDB() + + // Use batch insert with conflict handling + const batchSize = 100 + var lastErr error + + for i := 0; i < len(ipLookup); i += batchSize { + end := i + batchSize + if end > len(ipLookup) { + end = len(ipLookup) + } + + batch := ipLookup[i:end] + // Use Clauses with OnConflict DoNothing to skip conflicts + err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error + if err != nil { + zap.L().Warn("Error storing some IP lookup records", zap.Error(err)) + lastErr = err + // Continue with next batch despite error + } + } + + return lastErr +} diff --git a/internal/sqlite/structs.go b/internal/sqlite/structs.go index 43e4088..9417990 100644 --- a/internal/sqlite/structs.go +++ b/internal/sqlite/structs.go @@ -67,13 +67,14 @@ type QueryOptions struct { CryptoAddressQuery string `json:"crypto_address_query"` PrintBalance bool `json:"print_balance"` CredsOnly bool `json:"creds_only"` + Debug bool `json:"debug"` } func (QueryOptions) TableName() string { return "query_options" } -func NewQueryOptions(maxRecords, maxRequests, startingPage int, outputFormat, outputFile, usernameQuery, emailQuery, ipQuery, passQuery, hashQuery, nameQuery, domainQuery, vinQuery, licensePlateQuery, addressQuery, phoneQuery, socialQuery, cryptoAddressQuery string, regexMatch, wildcardMatch, printBalance, credsOnly bool) *QueryOptions { +func NewQueryOptions(maxRecords, maxRequests, startingPage int, outputFormat, outputFile, usernameQuery, emailQuery, ipQuery, passQuery, hashQuery, nameQuery, domainQuery, vinQuery, licensePlateQuery, addressQuery, phoneQuery, socialQuery, cryptoAddressQuery string, regexMatch, wildcardMatch, printBalance, credsOnly, debug bool) *QueryOptions { return &QueryOptions{ MaxRecords: maxRecords, MaxRequests: maxRequests, @@ -97,14 +98,15 @@ func NewQueryOptions(maxRecords, maxRequests, startingPage int, outputFormat, ou PhoneQuery: phoneQuery, SocialQuery: socialQuery, CryptoAddressQuery: cryptoAddressQuery, + Debug: debug, } } type Creds struct { gorm.Model - Email string `json:"email" yaml:"email" xml:"email"` - Username string `json:"username" yaml:"username" xml:"username"` - Password string `json:"password" yaml:"password" xml:"password"` + Email string `json:"email" yaml:"email" xml:"email" gorm:"uniqueIndex:idx_email_username_password"` + Username string `json:"username" yaml:"username" xml:"username" gorm:"uniqueIndex:idx_email_username_password"` + Password string `json:"password" yaml:"password" xml:"password" gorm:"uniqueIndex:idx_email_username_password"` } func (Creds) TableName() string { diff --git a/internal/sqlite/whois.go b/internal/sqlite/whois.go index eacb9f6..886ab62 100644 --- a/internal/sqlite/whois.go +++ b/internal/sqlite/whois.go @@ -1,6 +1,10 @@ package sqlite -import "gorm.io/gorm" +import ( + "fmt" + "gorm.io/gorm" + "strings" +) type WhoIsLookupResult struct { RemainingCredits int `json:"remaining_credits"` @@ -17,7 +21,7 @@ type WhoisRecord struct { ContactEmail string `json:"contactEmail"` CreatedDate string `json:"createdDate"` CreatedDateNormalized string `json:"createdDateNormalized"` - DomainName string `json:"domainName"` + DomainName string `json:"domainName" gorm:"unique"` DomainNameExt string `json:"domainNameExt"` EstimatedDomainAge int `json:"estimatedDomainAge"` ExpiresDate string `json:"expiresDate"` @@ -38,6 +42,142 @@ type WhoisRecord struct { UpdatedDateNormalized string `json:"updatedDateNormalized"` } +func (w WhoisRecord) String() string { + var sb strings.Builder + + // Main domain information + sb.WriteString(fmt.Sprintf("Domain Name: %s\n", w.DomainName)) + sb.WriteString(fmt.Sprintf("Domain Name Ext: %s\n", w.DomainNameExt)) + sb.WriteString(fmt.Sprintf("Registrar Name: %s\n", w.RegistrarName)) + sb.WriteString(fmt.Sprintf("Registrar IANA ID: %s\n", w.RegistrarIANAID)) + sb.WriteString(fmt.Sprintf("Contact Email: %s\n", w.ContactEmail)) + sb.WriteString(fmt.Sprintf("Estimated Domain Age: %d days\n", w.EstimatedDomainAge)) + + // Dates + sb.WriteString(fmt.Sprintf("Created Date: %s (Normalized: %s)\n", w.CreatedDate, w.CreatedDateNormalized)) + sb.WriteString(fmt.Sprintf("Updated Date: %s (Normalized: %s)\n", w.UpdatedDate, w.UpdatedDateNormalized)) + sb.WriteString(fmt.Sprintf("Expires Date: %s (Normalized: %s)\n", w.ExpiresDate, w.ExpiresDateNormalized)) + + // Status + sb.WriteString(fmt.Sprintf("Status: %s\n", w.Status)) + + // Parse code + sb.WriteString(fmt.Sprintf("Parse Code: %d\n", w.ParseCode)) + + // Audit information + sb.WriteString("\nAudit Information:\n") + sb.WriteString(fmt.Sprintf(" Created Date: %s\n", w.Audit.CreatedDate)) + sb.WriteString(fmt.Sprintf(" Updated Date: %s\n", w.Audit.UpdatedDate)) + + // Name servers + sb.WriteString("\nName Servers:\n") + if len(w.NameServers.HostNames) > 0 { + for i, ns := range w.NameServers.HostNames { + ip := "" + if i < len(w.NameServers.IPs) { + ip = fmt.Sprintf(" (%s)", w.NameServers.IPs[i]) + } + sb.WriteString(fmt.Sprintf(" %d. %s%s\n", i+1, ns, ip)) + } + } else { + sb.WriteString(" None listed\n") + } + + if w.NameServers.RawText != "" { + sb.WriteString(fmt.Sprintf(" Raw Text: %s\n", w.NameServers.RawText)) + } + + // Contact information + sb.WriteString("\nRegistrant Contact:\n") + formatWhoisContact(&sb, w.Registrant, " ") + + sb.WriteString("\nTechnical Contact:\n") + formatWhoisContact(&sb, w.TechnicalContact, " ") + + // Registry Data + sb.WriteString("\nRegistry Data:\n") + if w.RegistryData.DomainName != "" { + sb.WriteString(fmt.Sprintf(" Domain Name: %s\n", w.RegistryData.DomainName)) + sb.WriteString(fmt.Sprintf(" Registrar Name: %s\n", w.RegistryData.RegistrarName)) + sb.WriteString(fmt.Sprintf(" Registrar IANA ID: %s\n", w.RegistryData.RegistrarIANAID)) + sb.WriteString(fmt.Sprintf(" Whois Server: %s\n", w.RegistryData.WhoisServer)) + sb.WriteString(fmt.Sprintf(" Status: %s\n", w.RegistryData.Status)) + + // Registry dates + sb.WriteString(fmt.Sprintf(" Created Date: %s (Normalized: %s)\n", + w.RegistryData.CreatedDate, w.RegistryData.CreatedDateNormalized)) + sb.WriteString(fmt.Sprintf(" Updated Date: %s (Normalized: %s)\n", + w.RegistryData.UpdatedDate, w.RegistryData.UpdatedDateNormalized)) + sb.WriteString(fmt.Sprintf(" Expires Date: %s (Normalized: %s)\n", + w.RegistryData.ExpiresDate, w.RegistryData.ExpiresDateNormalized)) + + // Registry nameservers + sb.WriteString(" Name Servers:\n") + if len(w.RegistryData.NameServers.HostNames) > 0 { + for i, ns := range w.RegistryData.NameServers.HostNames { + ip := "" + if i < len(w.RegistryData.NameServers.IPs) { + ip = fmt.Sprintf(" (%s)", w.RegistryData.NameServers.IPs[i]) + } + sb.WriteString(fmt.Sprintf(" %d. %s%s\n", i+1, ns, ip)) + } + } else { + sb.WriteString(" None listed\n") + } + + // Registry audit + sb.WriteString(" Audit Information:\n") + sb.WriteString(fmt.Sprintf(" Created Date: %s\n", w.RegistryData.Audit.CreatedDate)) + sb.WriteString(fmt.Sprintf(" Updated Date: %s\n", w.RegistryData.Audit.UpdatedDate)) + } else { + sb.WriteString(" No registry data available\n") + } + + // Header and footer + if w.Header != "" { + headerPreview := w.Header + if len(headerPreview) > 100 { + headerPreview = headerPreview[:100] + "... [truncated]" + } + sb.WriteString("\nHeader:\n") + sb.WriteString(headerPreview) + sb.WriteString("\n") + } + + if w.Footer != "" { + footerPreview := w.Footer + if len(footerPreview) > 100 { + footerPreview = footerPreview[:100] + "... [truncated]" + } + sb.WriteString("\nFooter:\n") + sb.WriteString(footerPreview) + sb.WriteString("\n") + } + + // Raw text (truncated if too long) + if w.RawText != "" { + rawTextPreview := w.RawText + if len(rawTextPreview) > 500 { + rawTextPreview = rawTextPreview[:500] + "... [truncated]" + } + sb.WriteString("\nRaw Text:\n") + sb.WriteString(rawTextPreview) + sb.WriteString("\n") + } + + if w.StrippedText != "" { + strippedTextPreview := w.StrippedText + if len(strippedTextPreview) > 500 { + strippedTextPreview = strippedTextPreview[:500] + "... [truncated]" + } + sb.WriteString("\nStripped Text:\n") + sb.WriteString(strippedTextPreview) + sb.WriteString("\n") + } + + return sb.String() +} + func (WhoisRecord) TableName() string { return "whois" } @@ -104,7 +244,7 @@ type ScanResult struct { type SubdomainRecord struct { gorm.Model - Domain string `json:"domain"` + Domain string `json:"domain" gorm:"unique"` FirstSeen int64 `json:"firstSeen"` LastSeen int64 `json:"lastSeen"` } @@ -131,7 +271,7 @@ type HistoryRecord struct { CleanText string `json:"cleanText"` CreatedDateISO8601 string `json:"createdDateISO8601"` CreatedDateRaw string `json:"createdDateRaw"` - DomainName string `json:"domainName"` + DomainName string `json:"domainName" gorm:"unique"` DomainType string `json:"domainType"` ExpiresDateISO8601 string `json:"expiresDateISO8601"` ExpiresDateRaw string `json:"expiresDateRaw"` @@ -147,6 +287,131 @@ type HistoryRecord struct { ZoneContact ContactInfo `json:"zoneContact" gorm:"serializer:json"` } +func (h HistoryRecord) String() string { + var sb strings.Builder + + // Main domain information + sb.WriteString(fmt.Sprintf("Domain Name: %s\n", h.DomainName)) + sb.WriteString(fmt.Sprintf("Domain Type: %s\n", h.DomainType)) + sb.WriteString(fmt.Sprintf("Registrar Name: %s\n", h.RegistrarName)) + sb.WriteString(fmt.Sprintf("Whois Server: %s\n", h.WhoisServer)) + + // Dates + sb.WriteString(fmt.Sprintf("Created Date: %s (Raw: %s)\n", h.CreatedDateISO8601, h.CreatedDateRaw)) + sb.WriteString(fmt.Sprintf("Updated Date: %s (Raw: %s)\n", h.UpdatedDateISO8601, h.UpdatedDateRaw)) + sb.WriteString(fmt.Sprintf("Expires Date: %s (Raw: %s)\n", h.ExpiresDateISO8601, h.ExpiresDateRaw)) + + // Status + sb.WriteString("Status: ") + if len(h.Status) > 0 { + sb.WriteString(strings.Join(h.Status, ", ")) + } else { + sb.WriteString("N/A") + } + sb.WriteString("\n") + + // Audit information + sb.WriteString("\nAudit Information:\n") + sb.WriteString(fmt.Sprintf(" Created Date: %s\n", h.Audit.CreatedDate)) + sb.WriteString(fmt.Sprintf(" Updated Date: %s\n", h.Audit.UpdatedDate)) + + // Name servers + sb.WriteString("\nName Servers:\n") + if len(h.NameServers) > 0 { + for i, ns := range h.NameServers { + sb.WriteString(fmt.Sprintf(" %d. %s\n", i+1, ns)) + } + } else { + sb.WriteString(" None listed\n") + } + + // Contact information + sb.WriteString("\nRegistrant Contact:\n") + formatContact(&sb, h.RegistrantContact, " ") + + sb.WriteString("\nAdministrative Contact:\n") + formatContact(&sb, h.AdministrativeContact, " ") + + sb.WriteString("\nTechnical Contact:\n") + formatContact(&sb, h.TechnicalContact, " ") + + sb.WriteString("\nBilling Contact:\n") + formatContact(&sb, h.BillingContact, " ") + + sb.WriteString("\nZone Contact:\n") + formatContact(&sb, h.ZoneContact, " ") + + // Raw text (truncated if too long) + if len(h.RawText) > 0 { + rawTextPreview := h.RawText + if len(rawTextPreview) > 500 { + rawTextPreview = rawTextPreview[:500] + "... [truncated]" + } + sb.WriteString("\nRaw Text:\n") + sb.WriteString(rawTextPreview) + sb.WriteString("\n") + } + + if len(h.CleanText) > 0 { + cleanTextPreview := h.CleanText + if len(cleanTextPreview) > 500 { + cleanTextPreview = cleanTextPreview[:500] + "... [truncated]" + } + sb.WriteString("\nClean Text:\n") + sb.WriteString(cleanTextPreview) + sb.WriteString("\n") + } + + return sb.String() +} + +// Helper function to format contact information +func formatContact(sb *strings.Builder, contact ContactInfo, indent string) { + if contact.Name == "" && contact.Organization == "" && contact.Email == "" { + sb.WriteString(indent + "No contact information available\n") + return + } + + if contact.Name != "" { + sb.WriteString(indent + "Name: " + contact.Name + "\n") + } + if contact.Organization != "" { + sb.WriteString(indent + "Organization: " + contact.Organization + "\n") + } + if contact.Email != "" { + sb.WriteString(indent + "Email: " + contact.Email + "\n") + } + if contact.Street != "" { + sb.WriteString(indent + "Street: " + contact.Street + "\n") + } + if contact.City != "" { + sb.WriteString(indent + "City: " + contact.City + "\n") + } + if contact.State != "" { + sb.WriteString(indent + "State: " + contact.State + "\n") + } + if contact.PostalCode != "" { + sb.WriteString(indent + "Postal Code: " + contact.PostalCode + "\n") + } + if contact.Country != "" { + sb.WriteString(indent + "Country: " + contact.Country + "\n") + } + if contact.Telephone != "" { + phone := contact.Telephone + if contact.TelephoneExt != "" { + phone += " ext. " + contact.TelephoneExt + } + sb.WriteString(indent + "Telephone: " + phone + "\n") + } + if contact.Fax != "" { + fax := contact.Fax + if contact.FaxExt != "" { + fax += " ext. " + contact.FaxExt + } + sb.WriteString(indent + "Fax: " + fax + "\n") + } +} + func (HistoryRecord) TableName() string { return "history" } @@ -170,3 +435,80 @@ type ContactInfo struct { type WhoIsCredits struct { WhoisCredits int `json:"whois_credits"` } + +type WhoIsIPLookup struct { + RemainingCredits int `json:"remaining_credits"` + Data IPData `json:"data"` +} + +type WhoIsMXLookup struct { + RemainingCredits int `json:"remaining_credits"` + Data IPData `json:"data"` +} + +type WhoIsNSLookup struct { + RemainingCredits int `json:"remaining_credits"` + Data IPData `json:"data"` +} + +type IPData struct { + CurrentPage string `json:"current_page"` + Result []LookupResult `json:"result"` + Size int `json:"size"` +} + +type LookupResult struct { + gorm.Model + FirstSeen int64 `json:"first_seen"` + LastVisit int64 `json:"last_visit"` + Name string `json:"name" gorm:"unique"` + SearchTerm string `json:"search_term,omitempty"` // For storing the IP address this domain is associated with + Type string `json:"type,omitempty"` // For storing the MX address this domain is associated with +} + +func (LookupResult) TableName() string { + return "lookup" +} + +// Helper function to format contact information for WhoisRecord +func formatWhoisContact(sb *strings.Builder, contact Contact, indent string) { + if contact.Name == "" && contact.Organization == "" { + sb.WriteString(indent + "No contact information available\n") + return + } + + if contact.Name != "" { + sb.WriteString(indent + "Name: " + contact.Name + "\n") + } + if contact.Organization != "" { + sb.WriteString(indent + "Organization: " + contact.Organization + "\n") + } + if contact.Street1 != "" { + sb.WriteString(indent + "Street: " + contact.Street1 + "\n") + } + if contact.City != "" { + sb.WriteString(indent + "City: " + contact.City + "\n") + } + if contact.State != "" { + sb.WriteString(indent + "State: " + contact.State + "\n") + } + if contact.PostalCode != "" { + sb.WriteString(indent + "Postal Code: " + contact.PostalCode + "\n") + } + if contact.Country != "" { + sb.WriteString(indent + "Country: " + contact.Country + "\n") + } + if contact.CountryCode != "" { + sb.WriteString(indent + "Country Code: " + contact.CountryCode + "\n") + } + if contact.Telephone != "" { + sb.WriteString(indent + "Telephone: " + contact.Telephone + "\n") + } + if contact.RawText != "" { + rawTextPreview := contact.RawText + if len(rawTextPreview) > 100 { + rawTextPreview = rawTextPreview[:100] + "... [truncated]" + } + sb.WriteString(indent + "Raw Text: " + rawTextPreview + "\n") + } +} diff --git a/internal/whois/whois.go b/internal/whois/whois.go index b40530e..a18b035 100644 --- a/internal/whois/whois.go +++ b/internal/whois/whois.go @@ -2,7 +2,8 @@ package whois import ( "bytes" - "dehasher/internal/query" + "dehasher/internal/debug" + "dehasher/internal/dehashed" "dehasher/internal/sqlite" "encoding/json" "errors" @@ -23,47 +24,89 @@ type DehashedWHOISSearchRequest struct { SearchType string `json:"search_type,omitempty"` } -func WhoisSearch(domain, apiKey string) (sqlite.WhoIsLookupResult, error) { +type DehashedWhoIs struct { + balance int + debug bool + apiKey string +} + +func NewWhoIs(apiKey string, debug bool) *DehashedWhoIs { + return &DehashedWhoIs{apiKey: apiKey, debug: debug, balance: -1} +} + +func (w *DehashedWhoIs) WhoisSearch(domain string) (sqlite.WhoisRecord, error) { var whois sqlite.WhoIsLookupResult + var whoisRecord sqlite.WhoisRecord + + if w.debug { + debug.PrintInfo("performing whois search") + zap.L().Info("whois_search_debug", + zap.String("message", "performing whois search"), + ) + } + whoisSearchRequest := DehashedWHOISSearchRequest{ Domain: domain, SearchType: "whois", } + + if w.debug { + debug.PrintInfo("building request body") + debug.PrintJson(fmt.Sprintf("Request Body: %v\n", whoisSearchRequest)) + zap.L().Info("whois_search_debug", + zap.String("message", "building request body"), + zap.String("body", fmt.Sprintf("%v", whoisSearchRequest)), + ) + } + reqBody, _ := json.Marshal(whoisSearchRequest) req, err := http.NewRequest("POST", "https://api.dehashed.com/v2/whois/search", bytes.NewReader(reqBody)) if err != nil { - return whois, err + return whoisRecord, err } + + if w.debug { + debug.PrintInfo("setting headers") + zap.L().Info("whois_search_debug", + zap.String("message", "setting headers"), + ) + } + req.Header.Set("Content-Type", "application/json") - req.Header.Set("Dehashed-Api-Key", apiKey) + req.Header.Set("Dehashed-Api-Key", w.apiKey) + + if w.debug { + debug.PrintInfo("headers set") + debug.PrintJson(fmt.Sprintf("Headers: %v\n", req.Header.Clone())) + debug.PrintInfo("performing request") + zap.L().Info("whois_search_debug", + zap.String("message", "performing request"), + ) + } + res, err := http.DefaultClient.Do(req) if res != nil { defer res.Body.Close() } if err != nil { + if w.debug { + debug.PrintInfo("failed to perform request") + debug.PrintError(err) + } zap.L().Error("whois_search", zap.String("message", "failed to perform request"), zap.Error(err), ) - return whois, err + return whoisRecord, err } if res == nil { + if w.debug { + debug.PrintInfo("response was nil") + } zap.L().Error("whois_search", zap.String("message", "response was nil"), ) - return whois, errors.New("response was nil") - } - - // Check for HTTP status code errors - if res.StatusCode != 200 { - dhErr := query.GetDehashedError(res.StatusCode) - fmt.Printf("[%d] API Error message: %s\n", res.StatusCode, dhErr.Error()) - zap.L().Error("whois_search", - zap.String("message", "received error status code"), - zap.Int("status_code", res.StatusCode), - zap.String("error", dhErr.Error()), - ) - return whois, &dhErr + return whoisRecord, errors.New("response was nil") } b, err := io.ReadAll(res.Body) @@ -72,93 +115,197 @@ func WhoisSearch(domain, apiKey string) (sqlite.WhoIsLookupResult, error) { zap.String("message", "failed to read response body"), zap.Error(err), ) - return whois, err + return whoisRecord, err + } + + // Check for HTTP status code errors + if res.StatusCode != 200 { + if w.debug { + debug.PrintInfo("received error status code") + debug.PrintJson(fmt.Sprintf("Status Code: %d\n", res.StatusCode)) + debug.PrintJson(fmt.Sprintf("Body: %s\n", string(b[:]))) + } + + dhErr := dehashed.GetDehashedError(res.StatusCode) + fmt.Printf("[%d] API Error message: %s\n", res.StatusCode, dhErr.Error()) + zap.L().Error("whois_search", + zap.String("message", "received error status code"), + zap.Int("status_code", res.StatusCode), + zap.String("error", dhErr.Error()), + ) + return whoisRecord, &dhErr } err = json.Unmarshal(b, &whois) if err != nil { + if w.debug { + debug.PrintInfo("failed to unmarshal response body") + debug.PrintError(err) + } zap.L().Error("whois_search", zap.String("message", "failed to unmarshal response body"), zap.Error(err), ) fmt.Println("Error unmarshalling response body:", err) fmt.Println("Response body:", string(b)) - return whois, err + return whoisRecord, err } - return whois, nil + if w.debug { + debug.PrintInfo("unmarshalled response body") + debug.PrintJson(fmt.Sprintf("Remaining Credits: %d\n", whois.RemainingCredits)) + debug.PrintJson(fmt.Sprintf("Data: %v\n", whois.Data)) + } + w.balance = whois.RemainingCredits + + return whois.Data.WhoisRecord, nil } -func WhoisHistory(domain, apiKey string) (sqlite.WhoIsHistory, error) { +func (w *DehashedWhoIs) WhoisHistory(domain string) ([]sqlite.HistoryRecord, error) { var whois sqlite.WhoIsHistory + var historyRecords []sqlite.HistoryRecord + + if w.debug { + debug.PrintInfo("performing whois history search") + zap.L().Info("whois_history_debug", + zap.String("message", "performing whois history search"), + ) + } + whoisSearchRequest := DehashedWHOISSearchRequest{ Domain: domain, SearchType: "whois-history", } reqBody, _ := json.Marshal(whoisSearchRequest) + + if w.debug { + debug.PrintInfo("building request body") + debug.PrintJson(fmt.Sprintf("Request Body: %v\n", whoisSearchRequest)) + zap.L().Info("whois_history_debug", + zap.String("message", "building request body"), + zap.String("body", fmt.Sprintf("%v", whoisSearchRequest)), + ) + } + req, err := http.NewRequest("POST", "https://api.dehashed.com/v2/whois/search", bytes.NewReader(reqBody)) if err != nil { - return whois, err + if w.debug { + debug.PrintInfo("failed to create request") + debug.PrintError(err) + } + zap.L().Error("whois_history", + zap.String("message", "failed to create request"), + zap.Error(err), + ) + return historyRecords, err } + req.Header.Set("Content-Type", "application/json") - req.Header.Set("Dehashed-Api-Key", apiKey) + req.Header.Set("Dehashed-Api-Key", w.apiKey) + + if w.debug { + debug.PrintInfo("performing request") + debug.PrintJson(fmt.Sprintf("Headers: %v\n", req.Header.Clone())) + zap.L().Info("whois_history_debug", + zap.String("message", "performing request"), + ) + } + res, err := http.DefaultClient.Do(req) if res != nil { + if w.debug { + debug.PrintInfo("response was not nil") + } zap.L().Info("whois_history", zap.String("message", "response was not nil"), ) defer res.Body.Close() } if err != nil { + if w.debug { + debug.PrintInfo("failed to perform request") + debug.PrintError(err) + } zap.L().Error("whois_history", zap.String("message", "failed to perform request"), zap.Error(err), ) - return whois, err + return historyRecords, err } if res == nil { + if w.debug { + debug.PrintInfo("response was nil") + } zap.L().Error("whois_history", zap.String("message", "response was nil"), ) - return whois, errors.New("response was nil") + return historyRecords, errors.New("response was nil") + } + + b, err := io.ReadAll(res.Body) + if err != nil { + if w.debug { + debug.PrintInfo("failed to read response body") + debug.PrintError(err) + } + zap.L().Error("whois_history", + zap.String("message", "failed to read response body"), + zap.Error(err), + ) + return historyRecords, err } // Check for HTTP status code errors if res.StatusCode != 200 { - dhErr := query.GetDehashedError(res.StatusCode) + if w.debug { + debug.PrintInfo("received error status code") + debug.PrintJson(fmt.Sprintf("Status Code: %d\n", res.StatusCode)) + debug.PrintJson(fmt.Sprintf("Body: %s\n", string(b[:]))) + } + dhErr := dehashed.GetDehashedError(res.StatusCode) fmt.Printf("[%d] API Error message: %s\n", res.StatusCode, dhErr.Error()) zap.L().Error("whois_history", zap.String("message", "received error status code"), zap.Int("status_code", res.StatusCode), zap.String("error", dhErr.Error()), + zap.String("body_error", string(b)), ) - return whois, &dhErr - } - - b, err := io.ReadAll(res.Body) - if err != nil { - zap.L().Error("whois_history", - zap.String("message", "failed to read response body"), - zap.Error(err), - ) - return whois, err + return historyRecords, &dhErr } err = json.Unmarshal(b, &whois) if err != nil { + if w.debug { + debug.PrintInfo("failed to unmarshal response body") + debug.PrintError(err) + } zap.L().Error("whois_history", zap.String("message", "failed to unmarshal response body"), zap.Error(err), ) fmt.Println("Error unmarshalling response body:", err) fmt.Println("Response body:", string(b)) - return whois, err + return historyRecords, err } - return whois, nil + if w.debug { + debug.PrintInfo("unmarshalled response body") + debug.PrintJson(fmt.Sprintf("Remaining Credits: %d\n", whois.RemainingCredits)) + debug.PrintJson(fmt.Sprintf("Data: %v\n", whois.Data)) + } + w.balance = whois.RemainingCredits + + return whois.Data.Records, nil } -func ReverseWHOIS(include []string, exclude []string, reverseType, apiKey string) (string, error) { +func (w *DehashedWhoIs) ReverseWHOIS(include []string, exclude []string, reverseType string) (string, error) { + if w.debug { + debug.PrintInfo("performing reverse whois search") + zap.L().Info("reverse_whois_debug", + zap.String("message", "performing reverse whois search"), + ) + } + whoisSearchRequest := DehashedWHOISSearchRequest{ Include: include, Exclude: exclude, @@ -166,17 +313,45 @@ func ReverseWHOIS(include []string, exclude []string, reverseType, apiKey string SearchType: "reverse-whois", } reqBody, _ := json.Marshal(whoisSearchRequest) + + if w.debug { + debug.PrintInfo("building request body") + debug.PrintJson(fmt.Sprintf("Request Body: %v\n", whoisSearchRequest)) + zap.L().Info("reverse_whois_debug", + zap.String("message", "building request body"), + zap.String("body", fmt.Sprintf("%v", whoisSearchRequest)), + ) + } + req, err := http.NewRequest("POST", "https://api.dehashed.com/v2/whois/search", bytes.NewReader(reqBody)) if err != nil { + zap.L().Error("reverse_whois", + zap.String("message", "failed to create request"), + zap.Error(err), + ) return "", err } + req.Header.Set("Content-Type", "application/json") - req.Header.Set("Dehashed-Api-Key", apiKey) + req.Header.Set("Dehashed-Api-Key", w.apiKey) + + if w.debug { + debug.PrintInfo("performing reverse whois search") + debug.PrintJson(fmt.Sprintf("Headers: %v\n", req.Header.Clone())) + zap.L().Info("reverse_whois_debug", + zap.String("message", "performing reverse whois search"), + ) + } + res, err := http.DefaultClient.Do(req) if res != nil { defer res.Body.Close() } if err != nil { + if w.debug { + debug.PrintInfo("failed to perform request") + debug.PrintError(err) + } zap.L().Error("reverse_whois", zap.String("message", "failed to perform request"), zap.Error(err), @@ -184,52 +359,111 @@ func ReverseWHOIS(include []string, exclude []string, reverseType, apiKey string return "", err } if res == nil { + if w.debug { + debug.PrintInfo("response was nil") + } zap.L().Error("reverse_whois", zap.String("message", "response was nil"), ) return "", errors.New("response was nil") } - // Check for HTTP status code errors - if res.StatusCode != 200 { - dhErr := query.GetDehashedError(res.StatusCode) - fmt.Printf("[%d] API Error message: %s\n", res.StatusCode, dhErr.Error()) - zap.L().Error("reverse_whois", - zap.String("message", "received error status code"), - zap.Int("status_code", res.StatusCode), - zap.String("error", dhErr.Error()), - ) - return "", &dhErr - } - b, err := io.ReadAll(res.Body) if err != nil { + if w.debug { + debug.PrintInfo("failed to read response body") + debug.PrintError(err) + } zap.L().Error("reverse_whois", zap.String("message", "failed to read response body"), zap.Error(err), ) return "", err } + + // Check for HTTP status code errors + if res.StatusCode != 200 { + if w.debug { + debug.PrintInfo("received error status code") + debug.PrintJson(fmt.Sprintf("Status Code: %d\n", res.StatusCode)) + debug.PrintJson(fmt.Sprintf("Body: %s\n", string(b[:]))) + } + dhErr := dehashed.GetDehashedError(res.StatusCode) + fmt.Printf("[%d] API Error message: %s\n", res.StatusCode, dhErr.Error()) + zap.L().Error("reverse_whois", + zap.String("message", "received error status code"), + zap.Int("status_code", res.StatusCode), + zap.String("error", dhErr.Error()), + zap.String("body_error", string(b)), + ) + return "", &dhErr + } + + if w.debug { + debug.PrintInfo("unmarshalled response body") + debug.PrintJson(fmt.Sprintf("Body: %s\n", string(b[:]))) + } + return string(b), nil } -func WhoisIP(ipAddress, apiKey string) ([]byte, error) { - whoisSearchRequest := DehashedWHOISSearchRequest{ +func (w *DehashedWhoIs) WhoisIP(ipAddress string) ([]sqlite.LookupResult, error) { + if w.debug { + debug.PrintInfo("performing whois ip search") + zap.L().Info("whois_ip_debug", + zap.String("message", "performing whois ip search"), + ) + } + + type IPSearchRequest struct { + IPAddress string `json:"domain"` + SearchType string `json:"search_type"` + } + + whoisSearchRequest := IPSearchRequest{ IPAddress: ipAddress, SearchType: "reverse-ip", } reqBody, _ := json.Marshal(whoisSearchRequest) + + if w.debug { + debug.PrintInfo("building request body") + debug.PrintJson(fmt.Sprintf("Request Body: %v\n", whoisSearchRequest)) + zap.L().Info("whois_ip_debug", + zap.String("message", "building request body"), + zap.String("body", fmt.Sprintf("%v", whoisSearchRequest)), + ) + } + req, err := http.NewRequest("POST", "https://api.dehashed.com/v2/whois/search", bytes.NewReader(reqBody)) if err != nil { + zap.L().Error("whois_ip", + zap.String("message", "failed to create request"), + zap.Error(err), + ) return nil, err } + req.Header.Set("Content-Type", "application/json") - req.Header.Set("Dehashed-Api-Key", apiKey) + req.Header.Set("Dehashed-Api-Key", w.apiKey) + + if w.debug { + debug.PrintInfo("performing whois ip search") + debug.PrintJson(fmt.Sprintf("Headers: %v\n", req.Header.Clone())) + zap.L().Info("whois_ip_debug", + zap.String("message", "performing whois ip search"), + ) + } + res, err := http.DefaultClient.Do(req) if res != nil { defer res.Body.Close() } if err != nil { + if w.debug { + debug.PrintInfo("failed to perform request") + debug.PrintError(err) + } zap.L().Error("whois_ip", zap.String("message", "failed to perform request"), zap.Error(err), @@ -237,128 +471,314 @@ func WhoisIP(ipAddress, apiKey string) ([]byte, error) { return nil, err } if res == nil { + if w.debug { + debug.PrintInfo("response was nil") + } zap.L().Error("whois_ip", zap.String("message", "response was nil"), ) return nil, errors.New("response was nil") } - // Check for HTTP status code errors - if res.StatusCode != 200 { - dhErr := query.GetDehashedError(res.StatusCode) - fmt.Printf("[%d] API Error message: %s\n", res.StatusCode, dhErr.Error()) - zap.L().Error("whois_ip", - zap.String("message", "received error status code"), - zap.Int("status_code", res.StatusCode), - zap.String("error", dhErr.Error()), - ) - return nil, &dhErr - } - b, err := io.ReadAll(res.Body) if err != nil { + if w.debug { + debug.PrintInfo("failed to read response body") + debug.PrintError(err) + } zap.L().Error("whois_ip", zap.String("message", "failed to read response body"), zap.Error(err), ) return nil, err } - return b, nil + + // Check for HTTP status code errors + if res.StatusCode != 200 { + if w.debug { + debug.PrintInfo("received error status code") + debug.PrintJson(fmt.Sprintf("Status Code: %d\n", res.StatusCode)) + debug.PrintJson(fmt.Sprintf("Body: %s\n", string(b[:]))) + } + dhErr := dehashed.GetDehashedError(res.StatusCode) + fmt.Printf("[%d] API Error message: %s\n", res.StatusCode, dhErr.Error()) + zap.L().Error("whois_ip", + zap.String("message", "received error status code"), + zap.Int("status_code", res.StatusCode), + zap.String("error", dhErr.Error()), + zap.String("body_error", string(b)), + ) + return nil, &dhErr + } + + if w.debug { + debug.PrintInfo("read response body") + debug.PrintJson(fmt.Sprintf("Response Body: %s\n", string(b))) + zap.L().Info("whois_ip_debug", + zap.String("message", "read response body"), + zap.String("body", string(b)), + ) + } + + var whois sqlite.WhoIsIPLookup + err = json.Unmarshal(b, &whois) + if err != nil { + if w.debug { + debug.PrintInfo("failed to unmarshal response body") + debug.PrintError(err) + } + zap.L().Error("whois_ip", + zap.String("message", "failed to unmarshal response body"), + zap.Error(err), + ) + return nil, err + } + + if w.debug { + debug.PrintInfo("unmarshalled response body") + debug.PrintJson(fmt.Sprintf("Remaining Credits: %d\n", whois.RemainingCredits)) + debug.PrintJson(fmt.Sprintf("Data: %v\n", whois.Data)) + } + w.balance = whois.RemainingCredits + var lookups []sqlite.LookupResult + + for _, v := range whois.Data.Result { + lookups = append(lookups, sqlite.LookupResult{ + FirstSeen: v.FirstSeen, + LastVisit: v.LastVisit, + Name: v.Name, + SearchTerm: ipAddress, + Type: "Reverse IP", + }) + } + + sqlite.StoreIPLookup(lookups) + + return lookups, nil } -func WhoisMX(mxAddress, apiKey string) (string, error) { - whoisSearchRequest := DehashedWHOISSearchRequest{ - MXAddress: mxAddress, +func (w *DehashedWhoIs) WhoisMX(mxHostname string) ([]sqlite.LookupResult, error) { + if w.debug { + debug.PrintInfo("performing whois mx search") + zap.L().Info("whois_mx_debug", + zap.String("message", "performing whois mx search"), + ) + } + + type ReverseMX struct { + Domain string `json:"domain"` + SearchType string `json:"search_type"` + } + + whoisSearchRequest := ReverseMX{ + Domain: mxHostname, SearchType: "reverse-mx", } reqBody, _ := json.Marshal(whoisSearchRequest) + + if w.debug { + debug.PrintInfo("building request body") + debug.PrintJson(fmt.Sprintf("Request Body: %v\n", whoisSearchRequest)) + zap.L().Info("whois_mx_debug", + zap.String("message", "building request body"), + zap.String("body", fmt.Sprintf("%v", whoisSearchRequest)), + ) + } + req, err := http.NewRequest("POST", "https://api.dehashed.com/v2/whois/search", bytes.NewReader(reqBody)) if err != nil { - return "", err + if w.debug { + debug.PrintInfo("failed to create request") + debug.PrintError(err) + } + zap.L().Error("whois_mx", + zap.String("message", "failed to create request"), + zap.Error(err), + ) + return nil, err } + req.Header.Set("Content-Type", "application/json") - req.Header.Set("Dehashed-Api-Key", apiKey) + req.Header.Set("Dehashed-Api-Key", w.apiKey) + + if w.debug { + debug.PrintInfo("performing whois mx search") + debug.PrintJson(fmt.Sprintf("Headers: %v\n", req.Header.Clone())) + zap.L().Info("whois_mx_debug", + zap.String("message", "performing whois mx search"), + ) + } + res, err := http.DefaultClient.Do(req) if res != nil { defer res.Body.Close() } if err != nil { + if w.debug { + debug.PrintInfo("failed to perform request") + debug.PrintError(err) + } zap.L().Error("whois_mx", zap.String("message", "failed to perform request"), zap.Error(err), ) - return "", err + return nil, err } if res == nil { + if w.debug { + debug.PrintInfo("response was nil") + } zap.L().Error("whois_mx", zap.String("message", "response was nil"), ) - return "", errors.New("response was nil") + return nil, errors.New("response was nil") + } + + b, err := io.ReadAll(res.Body) + if err != nil { + if w.debug { + debug.PrintInfo("failed to read response body") + debug.PrintError(err) + } + zap.L().Error("whois_mx", + zap.String("message", "failed to read response body"), + zap.Error(err), + ) + return nil, err } // Check for HTTP status code errors if res.StatusCode != 200 { - dhErr := query.GetDehashedError(res.StatusCode) + if w.debug { + debug.PrintInfo("received error status code") + debug.PrintJson(fmt.Sprintf("Status Code: %d\n", res.StatusCode)) + debug.PrintJson(fmt.Sprintf("Body: %s\n", string(b[:]))) + } + dhErr := dehashed.GetDehashedError(res.StatusCode) fmt.Printf("[%d] API Error message: %s\n", res.StatusCode, dhErr.Error()) zap.L().Error("whois_mx", zap.String("message", "received error status code"), zap.Int("status_code", res.StatusCode), zap.String("error", dhErr.Error()), + zap.String("body_error", string(b)), ) - return "", &dhErr + return nil, &dhErr } - b, err := io.ReadAll(res.Body) + if w.debug { + debug.PrintInfo("read response body") + debug.PrintJson(fmt.Sprintf("Body: %s\n", string(b[:]))) + } + + var whois sqlite.WhoIsMXLookup + err = json.Unmarshal(b, &whois) if err != nil { + if w.debug { + debug.PrintInfo("failed to unmarshal response body") + debug.PrintError(err) + } zap.L().Error("whois_mx", - zap.String("message", "failed to read response body"), + zap.String("message", "failed to unmarshal response body"), zap.Error(err), ) - return "", err + return nil, err } - return string(b), nil + + if w.debug { + debug.PrintInfo("unmarshalled response body") + debug.PrintJson(fmt.Sprintf("Remaining Credits: %d\n", whois.RemainingCredits)) + debug.PrintJson(fmt.Sprintf("Data: %v\n", whois.Data)) + } + + var mxLookups []sqlite.LookupResult + + for _, v := range whois.Data.Result { + mxLookups = append(mxLookups, sqlite.LookupResult{ + FirstSeen: v.FirstSeen, + LastVisit: v.LastVisit, + Name: v.Name, + SearchTerm: mxHostname, + Type: "MX", + }) + } + + sqlite.StoreIPLookup(mxLookups) + + return mxLookups, nil } -func WhoisNS(nsAddress, apiKey string) (string, error) { - whoisSearchRequest := DehashedWHOISSearchRequest{ - NSAddress: nsAddress, +func (w *DehashedWhoIs) WhoisNS(nsHostname string) ([]sqlite.LookupResult, error) { + if w.debug { + debug.PrintInfo("performing whois ns search") + zap.L().Info("whois_ns_debug", + zap.String("message", "performing whois ns search"), + ) + } + + type NSLookup struct { + Domain string `json:"domain"` + SearchType string `json:"search_type"` + } + + whoisSearchRequest := NSLookup{ + Domain: nsHostname, SearchType: "reverse-ns", } + reqBody, _ := json.Marshal(whoisSearchRequest) + + if w.debug { + debug.PrintInfo("building request body") + debug.PrintJson(fmt.Sprintf("Request Body: %v\n", whoisSearchRequest)) + zap.L().Info("whois_ns_debug", + zap.String("message", "building request body"), + zap.String("body", fmt.Sprintf("%v", whoisSearchRequest)), + ) + } + req, err := http.NewRequest("POST", "https://api.dehashed.com/v2/whois/search", bytes.NewReader(reqBody)) if err != nil { - return "", err + zap.L().Error("whois_ns", + zap.String("message", "failed to create request"), + zap.Error(err), + ) + return nil, err } + req.Header.Set("Content-Type", "application/json") - req.Header.Set("Dehashed-Api-Key", apiKey) + req.Header.Set("Dehashed-Api-Key", w.apiKey) + + if w.debug { + debug.PrintInfo("performing request") + debug.PrintJson(fmt.Sprintf("Headers: %v\n", req.Header.Clone())) + zap.L().Info("whois_ns_debug", + zap.String("message", "performing request"), + ) + } + res, err := http.DefaultClient.Do(req) if res != nil { defer res.Body.Close() } if err != nil { + if w.debug { + debug.PrintInfo("failed to perform request") + debug.PrintError(err) + } zap.L().Error("whois_ns", zap.String("message", "failed to perform request"), zap.Error(err), ) - return "", err + return nil, err } if res == nil { + if w.debug { + debug.PrintInfo("response was nil") + } zap.L().Error("whois_ns", zap.String("message", "response was nil"), ) - return "", errors.New("response was nil") - } - - // Check for HTTP status code errors - if res.StatusCode != 200 { - dhErr := query.GetDehashedError(res.StatusCode) - fmt.Printf("[%d] API Error message: %s\n", res.StatusCode, dhErr.Error()) - zap.L().Error("whois_ns", - zap.String("message", "received error status code"), - zap.Int("status_code", res.StatusCode), - zap.String("error", dhErr.Error()), - ) - return "", &dhErr + return nil, errors.New("response was nil") } b, err := io.ReadAll(res.Body) @@ -367,61 +787,178 @@ func WhoisNS(nsAddress, apiKey string) (string, error) { zap.String("message", "failed to read response body"), zap.Error(err), ) - return "", err + return nil, err } - return string(b), nil + + // Check for HTTP status code errors + if res.StatusCode != 200 { + if w.debug { + debug.PrintInfo("received error status code") + debug.PrintJson(fmt.Sprintf("Status Code: %d\n", res.StatusCode)) + debug.PrintJson(fmt.Sprintf("Body: %s\n", string(b[:]))) + } + dhErr := dehashed.GetDehashedError(res.StatusCode) + fmt.Printf("[%d] API Error message: %s\n", res.StatusCode, dhErr.Error()) + zap.L().Error("whois_ns", + zap.String("message", "received error status code"), + zap.Int("status_code", res.StatusCode), + zap.String("error", dhErr.Error()), + zap.String("body_error", string(b)), + ) + return nil, &dhErr + } + + if w.debug { + debug.PrintInfo("read response body") + debug.PrintJson(fmt.Sprintf("Body: %s\n", string(b[:]))) + zap.L().Info("whois_ns_debug", + zap.String("message", "read response body"), + zap.String("body", string(b)), + ) + } + + var whois sqlite.WhoIsNSLookup + err = json.Unmarshal(b, &whois) + if err != nil { + if w.debug { + debug.PrintInfo("failed to unmarshal response body") + debug.PrintError(err) + } + zap.L().Error("whois_ns", + zap.String("message", "failed to unmarshal response body"), + zap.Error(err), + ) + return nil, err + } + + if w.debug { + debug.PrintInfo("unmarshalled response body") + debug.PrintJson(fmt.Sprintf("Remaining Credits: %d\n", whois.RemainingCredits)) + debug.PrintJson(fmt.Sprintf("Data: %v\n", whois.Data)) + } + w.balance = whois.RemainingCredits + var nsLookups []sqlite.LookupResult + + for _, v := range whois.Data.Result { + nsLookups = append(nsLookups, sqlite.LookupResult{ + FirstSeen: v.FirstSeen, + LastVisit: v.LastVisit, + Name: v.Name, + SearchTerm: nsHostname, + Type: "NS", + }) + } + + sqlite.StoreIPLookup(nsLookups) + + return nsLookups, nil } -func WhoisSubdomainScan(domain, apiKey string) (sqlite.WhoIsSubdomainScan, error) { +func (w *DehashedWhoIs) WhoisSubdomainScan(domain string) ([]sqlite.SubdomainRecord, error) { var whois sqlite.WhoIsSubdomainScan + var subdomains []sqlite.SubdomainRecord + + if w.debug { + debug.PrintInfo("performing whois subdomain scan") + zap.L().Info("whois_subdomain_scan_debug", + zap.String("message", "performing whois subdomain scan"), + ) + } + whoisSearchRequest := DehashedWHOISSearchRequest{ Domain: domain, SearchType: "subdomain-scan", } + reqBody, _ := json.Marshal(whoisSearchRequest) + + if w.debug { + debug.PrintInfo("building request body") + debug.PrintJson(fmt.Sprintf("Request Body: %v\n", whoisSearchRequest)) + zap.L().Info("whois_subdomain_scan_debug", + zap.String("message", "building request body"), + zap.String("body", fmt.Sprintf("%v", whoisSearchRequest)), + ) + } + req, err := http.NewRequest("POST", "https://api.dehashed.com/v2/whois/search", bytes.NewReader(reqBody)) if err != nil { - return whois, err + if w.debug { + debug.PrintInfo("failed to create request") + debug.PrintError(err) + } + zap.L().Error("whois_subdomain_scan", + zap.String("message", "failed to create request"), + zap.Error(err), + ) + return subdomains, err } + req.Header.Set("Content-Type", "application/json") - req.Header.Set("Dehashed-Api-Key", apiKey) + req.Header.Set("Dehashed-Api-Key", w.apiKey) + + if w.debug { + debug.PrintInfo("performing request") + debug.PrintJson(fmt.Sprintf("Headers: %v\n", req.Header.Clone())) + zap.L().Info("whois_subdomain_scan_debug", + zap.String("message", "performing request"), + ) + } + res, err := http.DefaultClient.Do(req) if res != nil { defer res.Body.Close() } if err != nil { + if w.debug { + debug.PrintInfo("failed to perform request") + debug.PrintError(err) + } zap.L().Error("whois_subdomain_scan", zap.String("message", "failed to perform request"), zap.Error(err), ) - return whois, err + return subdomains, err } if res == nil { + if w.debug { + debug.PrintInfo("response was nil") + } zap.L().Error("whois_subdomain_scan", zap.String("message", "response was nil"), ) - return whois, errors.New("response was nil") + return subdomains, errors.New("response was nil") + } + + b, err := io.ReadAll(res.Body) + if err != nil { + if w.debug { + debug.PrintInfo("failed to read response body") + debug.PrintError(err) + } + zap.L().Error("whois_subdomain_scan", + zap.String("message", "failed to read response body"), + zap.Error(err), + ) + return subdomains, err } // Check for HTTP status code errors if res.StatusCode != 200 { - dhErr := query.GetDehashedError(res.StatusCode) + if w.debug { + debug.PrintInfo("received error status code") + debug.PrintJson(fmt.Sprintf("Status Code: %d\n", res.StatusCode)) + debug.PrintJson(fmt.Sprintf("Body: %s\n", string(b[:]))) + } + dhErr := dehashed.GetDehashedError(res.StatusCode) fmt.Printf("[%d] API Error message: %s\n", res.StatusCode, dhErr.Error()) zap.L().Error("whois_subdomain_scan", zap.String("message", "received error status code"), zap.Int("status_code", res.StatusCode), zap.String("error", dhErr.Error()), + zap.String("body_error", string(b)), ) - return whois, &dhErr - } - - b, err := io.ReadAll(res.Body) - if err != nil { - zap.L().Error("whois_subdomain_scan", - zap.String("message", "failed to read response body"), - zap.Error(err), - ) - return whois, err + return subdomains, &dhErr } err = json.Unmarshal(b, &whois) @@ -432,58 +969,107 @@ func WhoisSubdomainScan(domain, apiKey string) (sqlite.WhoIsSubdomainScan, error ) fmt.Println("Error unmarshalling response body:", err) fmt.Println("Response body:", string(b)) - return whois, err + return subdomains, err } - return whois, nil + if w.debug { + debug.PrintInfo("unmarshalled response body") + debug.PrintJson(fmt.Sprintf("Remaining Credits: %d\n", whois.RemainingCredits)) + debug.PrintJson(fmt.Sprintf("Data: %v\n", whois.Data)) + } + w.balance = whois.RemainingCredits + + return whois.Data.Result.Records, nil } -func GetWHOISCredits(apiKey string) (sqlite.WhoIsCredits, error) { +func (w *DehashedWhoIs) Balance() (int, error) { + if w.debug { + debug.PrintInfo("getting whois credits") + zap.L().Info("whois_debug", + zap.String("message", "getting whois credits"), + ) + } + return w.getBalance() +} + +func (w *DehashedWhoIs) getBalance() (int, error) { var whoisCredits sqlite.WhoIsCredits req, err := http.NewRequest("GET", "https://api.dehashed.com/v2/whois/credits", nil) if err != nil { - return whoisCredits, err + if w.debug { + debug.PrintInfo("failed to create request") + debug.PrintError(err) + } + return whoisCredits.WhoisCredits, err } + req.Header.Set("Content-Type", "application/json") - req.Header.Set("Dehashed-Api-Key", apiKey) + req.Header.Set("Dehashed-Api-Key", w.apiKey) + + if w.debug { + debug.PrintInfo("performing request") + h := req.Header.Clone() + debug.PrintJson(fmt.Sprintf("Headers: %v\n", h)) + zap.L().Info("whois_debug", + zap.String("message", "performing request"), + ) + } + res, err := http.DefaultClient.Do(req) if res != nil { defer res.Body.Close() } if err != nil { + if w.debug { + debug.PrintInfo("failed to perform request") + debug.PrintError(err) + } zap.L().Error("get_whois_credits", zap.String("message", "failed to perform request"), zap.Error(err), ) - return whoisCredits, err + return whoisCredits.WhoisCredits, err } if res == nil { + if w.debug { + debug.PrintInfo("response was nil") + } zap.L().Error("get_whois_credits", zap.String("message", "response was nil"), ) - return whoisCredits, errors.New("response was nil") + return whoisCredits.WhoisCredits, errors.New("response was nil") + } + + b, err := io.ReadAll(res.Body) + if err != nil { + if w.debug { + debug.PrintInfo("failed to read response body") + debug.PrintError(err) + } + zap.L().Error("get_whois_credits", + zap.String("message", "failed to read response body"), + zap.Error(err), + ) + return whoisCredits.WhoisCredits, err } // Check for HTTP status code errors if res.StatusCode != 200 { - dhErr := query.GetDehashedError(res.StatusCode) + if w.debug { + debug.PrintInfo("received error status code") + debug.PrintJson(fmt.Sprintf("Status Code: %d\n", res.StatusCode)) + debug.PrintJson(fmt.Sprintf("Body: %s\n", string(b[:]))) + } + dhErr := dehashed.GetDehashedError(res.StatusCode) fmt.Printf("[%d] API Error message: %s\n", res.StatusCode, dhErr.Error()) zap.L().Error("get_whois_credits", zap.String("message", "received error status code"), zap.Int("status_code", res.StatusCode), zap.String("error", dhErr.Error()), + zap.String("body_error", string(b)), ) - return whoisCredits, &dhErr - } - - b, err := io.ReadAll(res.Body) - if err != nil { - zap.L().Error("get_whois_credits", - zap.String("message", "failed to read response body"), - zap.Error(err), - ) - return whoisCredits, err + return whoisCredits.WhoisCredits, &dhErr } err = json.Unmarshal(b, &whoisCredits) @@ -494,8 +1080,13 @@ func GetWHOISCredits(apiKey string) (sqlite.WhoIsCredits, error) { ) fmt.Println("Error unmarshalling response body:", err) fmt.Println("Response body:", string(b)) - return whoisCredits, err + return whoisCredits.WhoisCredits, err } - return whoisCredits, nil + if w.debug { + debug.PrintInfo("unmarshalled response body") + debug.PrintJson(fmt.Sprintf("Remaining Credits: %d\n", whoisCredits.WhoisCredits)) + } + + return whoisCredits.WhoisCredits, nil }