From c29f18a056a77f1863cb5545dd7d331e4845435f Mon Sep 17 00:00:00 2001 From: Evan Hosinski Date: Thu, 15 May 2025 16:23:59 -0400 Subject: [PATCH] Added enhanced logging for specific errors related to dehash tied into the apiu calls. Added logs subcommand. --- cmd/logs.go | 226 +++++++++++++++++++++++++++++++++++++ dehasher.go | 2 +- internal/query/client.go | 111 ------------------ internal/query/clientv2.go | 13 +++ internal/query/dehashed.go | 15 ++- internal/whois/whois.go | 182 +++++++++++++++++++++++++++++ 6 files changed, 436 insertions(+), 113 deletions(-) create mode 100644 cmd/logs.go delete mode 100644 internal/query/client.go diff --git a/cmd/logs.go b/cmd/logs.go new file mode 100644 index 0000000..118a3f3 --- /dev/null +++ b/cmd/logs.go @@ -0,0 +1,226 @@ +package cmd + +import ( + "dehasher/internal/pretty" + "encoding/json" + "fmt" + "github.com/spf13/cobra" + "os" + "path/filepath" + "strings" + "time" +) + +func init() { + // Add Subcommand to db command + rootCmd.AddCommand(logsCmd) + + logsCmd.Flags().IntVarP(&logLastXLogs, "last", "l", 0, "Number of logs to show") + logsCmd.Flags().StringVarP(&logStartDate, "start", "s", "", "Start date for logs") + logsCmd.Flags().StringVarP(&logEndDate, "end", "e", "", "End date for logs") + logsCmd.Flags().StringVarP(&logSeverity, "severity", "v", "", "Comma delimited Log severity to show (info, error, fatal)") +} + +// LogEntry represents a parsed log entry +type LogEntry struct { + Level string `json:"level"` + Timestamp string `json:"ts"` // Store as string initially + Message string `json:"msg"` + Details map[string]interface{} `json:"-"` + ParsedTime time.Time // Parsed time for sorting and filtering +} + +var ( + logLastXLogs int + logStartDate string + logEndDate string + logSeverity string + + logsCmd = &cobra.Command{ + Use: "logs", + Short: "View logs", + Long: `View logs for the Dehasher CLI.`, + Run: func(cmd *cobra.Command, args []string) { + logsPath := filepath.Join(os.Getenv("HOME"), ".local", "share", "Dehasher", "logs") + + if logSeverity == "" { + logSeverity = "info,error,fatal" + } else { + logSeverity = strings.ToLower(logSeverity) + } + + var ( + logInfo bool + logError bool + logFatal bool + ) + if strings.Contains(logSeverity, "info") { + logInfo = true + } + if strings.Contains(logSeverity, "error") { + logError = true + } + if strings.Contains(logSeverity, "fatal") { + logFatal = true + } + + var allLogs []string + if logSeverity == "info" { + allLogs = append(allLogs, filepath.Join(logsPath, "info.log")) + } else if logSeverity == "error" || logSeverity == "fatal" { + allLogs = append(allLogs, filepath.Join(logsPath, "error.log")) + } else { + allLogs = append(allLogs, filepath.Join(logsPath, "info.log"), filepath.Join(logsPath, "error.log")) + } + + var parsedLogs []LogEntry + for _, logFile := range allLogs { + // Read the log file + logData, err := os.ReadFile(logFile) + if err != nil { + fmt.Printf("Error reading log file %s: %v\n", logFile, err) + continue + } + + // Split the file into lines + logLines := strings.Split(strings.TrimSpace(string(logData)), "\n") + for _, line := range logLines { + if line == "" { + continue + } + + // Parse the JSON log entry + var entry LogEntry + var rawEntry map[string]interface{} + if err := json.Unmarshal([]byte(line), &entry); err != nil { + fmt.Printf("Error parsing log entry: %v\n", err) + continue + } + + // Also unmarshal to get additional fields + if err := json.Unmarshal([]byte(line), &rawEntry); err != nil { + fmt.Printf("Error parsing raw log entry: %v\n", err) + continue + } + + // Parse the timestamp + parsedTime, err := time.Parse("2006-01-02T15:04:05.999-0700", entry.Timestamp) + if err != nil { + // Try alternative formats + parsedTime, err = time.Parse(time.RFC3339, entry.Timestamp) + if err != nil { + // Try another format + parsedTime, err = time.Parse(time.RFC3339Nano, entry.Timestamp) + if err != nil { + fmt.Printf("Error parsing timestamp '%s': %v\n", entry.Timestamp, err) + continue + } + } + } + entry.ParsedTime = parsedTime + + // Store additional fields in Details + entry.Details = make(map[string]interface{}) + for k, v := range rawEntry { + if k != "level" && k != "ts" && k != "msg" { + entry.Details[k] = v + } + } + + // Filter by severity + if (logInfo && strings.EqualFold(entry.Level, "INFO")) || + (logError && strings.EqualFold(entry.Level, "ERROR")) || + (logFatal && strings.EqualFold(entry.Level, "FATAL")) { + + // Filter by date range if specified + if logStartDate != "" { + startDate, err := time.Parse("2006-01-02", logStartDate) + if err != nil { + fmt.Printf("Error parsing start date: %v\n", err) + } else if entry.ParsedTime.Before(startDate) { + continue + } + } + + if logEndDate != "" { + endDate, err := time.Parse("2006-01-02", logEndDate) + // Add one day to include the end date + endDate = endDate.Add(24 * time.Hour) + if err != nil { + fmt.Printf("Error parsing end date: %v\n", err) + } else if entry.ParsedTime.After(endDate) { + continue + } + } + + parsedLogs = append(parsedLogs, entry) + } + } + } + + // Limit the number of logs if specified + if logLastXLogs > 0 && len(parsedLogs) > logLastXLogs { + // Sort logs by timestamp (newest first) + // This is a simple bubble sort for demonstration + for i := 0; i < len(parsedLogs)-1; i++ { + for j := 0; j < len(parsedLogs)-i-1; j++ { + if parsedLogs[j].ParsedTime.Before(parsedLogs[j+1].ParsedTime) { + parsedLogs[j], parsedLogs[j+1] = parsedLogs[j+1], parsedLogs[j] + } + } + } + parsedLogs = parsedLogs[:logLastXLogs] + } + + // Display logs in a table + if len(parsedLogs) > 0 { + // Prepare table headers and rows + headers := []string{"Date", "Severity", "Message", "Details"} + rows := make([][]string, len(parsedLogs)) + + for i, log := range parsedLogs { + // Format timestamp + timeStr := log.ParsedTime.Format("2006-01-02 15:04:05") + + // Format details + detailsStr := "" + for k, v := range log.Details { + detailsStr += fmt.Sprintf("%s: %v, ", k, v) + } + if len(detailsStr) > 2 { + detailsStr = detailsStr[:len(detailsStr)-2] // Remove trailing comma and space + } + + rows[i] = []string{timeStr, log.Level, log.Message, detailsStr} + } + + // Display the table + pretty.Table(headers, rows) + } else { + fmt.Println("No logs found matching the specified criteria.") + } + }, + } +) + +type Severity int + +const ( + INFO Severity = iota + ERROR + FATAL + UNKNOWN Severity = -1 +) + +func getSeverity(logLevel string) Severity { + switch logLevel { + case "INFO": + return INFO + case "ERROR": + return ERROR + case "FATAL": + return FATAL + default: + return UNKNOWN + } +} diff --git a/dehasher.go b/dehasher.go index 3478e92..44e8c6b 100644 --- a/dehasher.go +++ b/dehasher.go @@ -65,7 +65,7 @@ func initializeLogger() { })), }) - zap.L().Info("some message", zap.Int("status", 0)) + zap.L().Info("logger initialized", zap.Int("status", 0)) } func main() { diff --git a/internal/query/client.go b/internal/query/client.go deleted file mode 100644 index d31e940..0000000 --- a/internal/query/client.go +++ /dev/null @@ -1,111 +0,0 @@ -package query - -import ( - "dehasher/internal/sqlite" - "fmt" - "log" - "net/http" - "net/url" - "os" - "strings" -) - -type DehashedClient struct { - key string - email string - results []sqlite.Result - client *http.Client - query string - params string - printBal bool - total int - balance int -} - -var baseUrl = "https://api.dehashed.com/v2/search" - -func NewDehashedClient(key, email string, printBal bool) *DehashedClient { - return &DehashedClient{key: key, email: email, results: make([]sqlite.Result, 0), client: &http.Client{}, printBal: printBal} -} - -func (dc *DehashedClient) getKey() string { - return dc.key -} - -func (dc *DehashedClient) getEmail() string { - return dc.email -} - -func (dc *DehashedClient) GetResults() sqlite.DehashedResults { - return sqlite.DehashedResults{Results: dc.results} -} - -func (dc *DehashedClient) buildQuery(params map[string]string) { - urlParams := url.Values{} - urlString := baseUrl - - if len(params) > 0 { - urlString += "?query=" - - for k, v := range params { - if len(v) > 0 { - urlParams.Add(k, v) - } - } - } - - tmp, _ := url.QueryUnescape(urlParams.Encode()) - tmp2 := strings.Replace(tmp, "=", ":", -1) - dc.params = tmp2 - urlString += dc.params - dc.query = urlString -} - -func (dc *DehashedClient) setResults(results int) { - dc.query = fmt.Sprintf("%s?query=%s&size=%d", baseUrl, dc.params, results) -} - -func (dc *DehashedClient) setPage(page int) { - dc.query = fmt.Sprintf("%s&nextPage=%d", dc.query, page) -} - -func (dc *DehashedClient) Do() int { - fmt.Printf("\n\t[*] Performing Request...") - req, err := http.NewRequest("GET", dc.query, nil) - if err != nil { - fmt.Printf("[!] Error constructing request: %v", err) - os.Exit(-1) - } - - dc.setAuth(req) - req.Header.Add("Dehashed-Api-Key", dc.getKey()) - req.Header.Add("Accept", "application/json") - resp, err := dc.client.Do(req) - if err != nil { - fmt.Printf("[!] Error performing request: %s\n%v", dc.query, err) - os.Exit(-1) - } - - if resp.StatusCode != 200 { - dhErr := GetDehashedError(resp.StatusCode) - fmt.Println() - log.Fatal(dhErr.Error()) - } - - entries, balance, total := sqlite.NewDehashedResults(resp.Body) - dc.results = append(dc.results, entries...) - dc.balance = balance - dc.total += total - if dc.printBal { - fmt.Printf("\n\t\t[*] Balance Remaining: %d", balance) - } - return total -} - -func (dc *DehashedClient) setAuth(r *http.Request) { - r.SetBasicAuth(dc.email, dc.key) -} - -func (dc *DehashedClient) GetDomains() int { - return dc.balance -} diff --git a/internal/query/clientv2.go b/internal/query/clientv2.go index b54a06b..894f68b 100644 --- a/internal/query/clientv2.go +++ b/internal/query/clientv2.go @@ -163,6 +163,19 @@ func (dcv2 *DehashedClientV2) Search(searchRequest DehashedSearchRequest) (int, ) return -1, errors.New("response was nil") } + + // Check for HTTP status code errors + if res.StatusCode != 200 { + dhErr := GetDehashedError(res.StatusCode) + fmt.Printf("[%d] API Error message: %s\n", res.StatusCode, dhErr.Error()) + zap.L().Error("v2_search", + zap.String("message", "received error status code"), + zap.Int("status_code", res.StatusCode), + zap.String("error", dhErr.Error()), + ) + return -1, &dhErr + } + b, err := io.ReadAll(res.Body) if err != nil { zap.L().Error("v2_search", diff --git a/internal/query/dehashed.go b/internal/query/dehashed.go index a14949f..a237f60 100644 --- a/internal/query/dehashed.go +++ b/internal/query/dehashed.go @@ -90,7 +90,20 @@ func (dh *Dehasher) Start() { fmt.Printf("\n\t[*] Performing Request...") count, err := dh.client.Search(*dh.request) if err != nil { - fmt.Printf("[!] Error performing request: %v", err) + // Check if it's a DehashError + if dhErr, ok := err.(*DehashError); ok { + fmt.Printf("\n\t[!] Dehashed API Error: %s (Code: %d)", dhErr.Message, dhErr.Code) + zap.L().Error("dehashed_api_error", + zap.String("message", dhErr.Message), + zap.Int("code", dhErr.Code), + ) + } else { + fmt.Printf("\n\t[!] Error performing request: %v", err) + zap.L().Error("request_error", + zap.String("message", "failed to perform request"), + zap.Error(err), + ) + } os.Exit(-1) } diff --git a/internal/whois/whois.go b/internal/whois/whois.go index 93dfef5..b40530e 100644 --- a/internal/whois/whois.go +++ b/internal/whois/whois.go @@ -2,6 +2,7 @@ package whois import ( "bytes" + "dehasher/internal/query" "dehasher/internal/sqlite" "encoding/json" "errors" @@ -40,13 +41,37 @@ func WhoisSearch(domain, apiKey string) (sqlite.WhoIsLookupResult, error) { defer res.Body.Close() } if err != nil { + zap.L().Error("whois_search", + zap.String("message", "failed to perform request"), + zap.Error(err), + ) return whois, err } if res == nil { + zap.L().Error("whois_search", + zap.String("message", "response was nil"), + ) return whois, errors.New("response was nil") } + + // Check for HTTP status code errors + if res.StatusCode != 200 { + dhErr := query.GetDehashedError(res.StatusCode) + fmt.Printf("[%d] API Error message: %s\n", res.StatusCode, dhErr.Error()) + zap.L().Error("whois_search", + zap.String("message", "received error status code"), + zap.Int("status_code", res.StatusCode), + zap.String("error", dhErr.Error()), + ) + return whois, &dhErr + } + b, err := io.ReadAll(res.Body) if err != nil { + zap.L().Error("whois_search", + zap.String("message", "failed to read response body"), + zap.Error(err), + ) return whois, err } @@ -97,6 +122,19 @@ func WhoisHistory(domain, apiKey string) (sqlite.WhoIsHistory, error) { ) return whois, errors.New("response was nil") } + + // Check for HTTP status code errors + if res.StatusCode != 200 { + dhErr := query.GetDehashedError(res.StatusCode) + fmt.Printf("[%d] API Error message: %s\n", res.StatusCode, dhErr.Error()) + zap.L().Error("whois_history", + zap.String("message", "received error status code"), + zap.Int("status_code", res.StatusCode), + zap.String("error", dhErr.Error()), + ) + return whois, &dhErr + } + b, err := io.ReadAll(res.Body) if err != nil { zap.L().Error("whois_history", @@ -139,13 +177,37 @@ func ReverseWHOIS(include []string, exclude []string, reverseType, apiKey string defer res.Body.Close() } if err != nil { + zap.L().Error("reverse_whois", + zap.String("message", "failed to perform request"), + zap.Error(err), + ) return "", err } if res == nil { + zap.L().Error("reverse_whois", + zap.String("message", "response was nil"), + ) return "", errors.New("response was nil") } + + // Check for HTTP status code errors + if res.StatusCode != 200 { + dhErr := query.GetDehashedError(res.StatusCode) + fmt.Printf("[%d] API Error message: %s\n", res.StatusCode, dhErr.Error()) + zap.L().Error("reverse_whois", + zap.String("message", "received error status code"), + zap.Int("status_code", res.StatusCode), + zap.String("error", dhErr.Error()), + ) + return "", &dhErr + } + b, err := io.ReadAll(res.Body) if err != nil { + zap.L().Error("reverse_whois", + zap.String("message", "failed to read response body"), + zap.Error(err), + ) return "", err } return string(b), nil @@ -168,13 +230,37 @@ func WhoisIP(ipAddress, apiKey string) ([]byte, error) { defer res.Body.Close() } if err != nil { + zap.L().Error("whois_ip", + zap.String("message", "failed to perform request"), + zap.Error(err), + ) return nil, err } if res == nil { + zap.L().Error("whois_ip", + zap.String("message", "response was nil"), + ) return nil, errors.New("response was nil") } + + // Check for HTTP status code errors + if res.StatusCode != 200 { + dhErr := query.GetDehashedError(res.StatusCode) + fmt.Printf("[%d] API Error message: %s\n", res.StatusCode, dhErr.Error()) + zap.L().Error("whois_ip", + zap.String("message", "received error status code"), + zap.Int("status_code", res.StatusCode), + zap.String("error", dhErr.Error()), + ) + return nil, &dhErr + } + b, err := io.ReadAll(res.Body) if err != nil { + zap.L().Error("whois_ip", + zap.String("message", "failed to read response body"), + zap.Error(err), + ) return nil, err } return b, nil @@ -197,13 +283,37 @@ func WhoisMX(mxAddress, apiKey string) (string, error) { defer res.Body.Close() } if err != nil { + zap.L().Error("whois_mx", + zap.String("message", "failed to perform request"), + zap.Error(err), + ) return "", err } if res == nil { + zap.L().Error("whois_mx", + zap.String("message", "response was nil"), + ) return "", errors.New("response was nil") } + + // Check for HTTP status code errors + if res.StatusCode != 200 { + dhErr := query.GetDehashedError(res.StatusCode) + fmt.Printf("[%d] API Error message: %s\n", res.StatusCode, dhErr.Error()) + zap.L().Error("whois_mx", + zap.String("message", "received error status code"), + zap.Int("status_code", res.StatusCode), + zap.String("error", dhErr.Error()), + ) + return "", &dhErr + } + b, err := io.ReadAll(res.Body) if err != nil { + zap.L().Error("whois_mx", + zap.String("message", "failed to read response body"), + zap.Error(err), + ) return "", err } return string(b), nil @@ -226,13 +336,37 @@ func WhoisNS(nsAddress, apiKey string) (string, error) { defer res.Body.Close() } if err != nil { + zap.L().Error("whois_ns", + zap.String("message", "failed to perform request"), + zap.Error(err), + ) return "", err } if res == nil { + zap.L().Error("whois_ns", + zap.String("message", "response was nil"), + ) return "", errors.New("response was nil") } + + // Check for HTTP status code errors + if res.StatusCode != 200 { + dhErr := query.GetDehashedError(res.StatusCode) + fmt.Printf("[%d] API Error message: %s\n", res.StatusCode, dhErr.Error()) + zap.L().Error("whois_ns", + zap.String("message", "received error status code"), + zap.Int("status_code", res.StatusCode), + zap.String("error", dhErr.Error()), + ) + return "", &dhErr + } + b, err := io.ReadAll(res.Body) if err != nil { + zap.L().Error("whois_ns", + zap.String("message", "failed to read response body"), + zap.Error(err), + ) return "", err } return string(b), nil @@ -256,13 +390,37 @@ func WhoisSubdomainScan(domain, apiKey string) (sqlite.WhoIsSubdomainScan, error defer res.Body.Close() } if err != nil { + zap.L().Error("whois_subdomain_scan", + zap.String("message", "failed to perform request"), + zap.Error(err), + ) return whois, err } if res == nil { + zap.L().Error("whois_subdomain_scan", + zap.String("message", "response was nil"), + ) return whois, errors.New("response was nil") } + + // Check for HTTP status code errors + if res.StatusCode != 200 { + dhErr := query.GetDehashedError(res.StatusCode) + fmt.Printf("[%d] API Error message: %s\n", res.StatusCode, dhErr.Error()) + zap.L().Error("whois_subdomain_scan", + zap.String("message", "received error status code"), + zap.Int("status_code", res.StatusCode), + zap.String("error", dhErr.Error()), + ) + return whois, &dhErr + } + b, err := io.ReadAll(res.Body) if err != nil { + zap.L().Error("whois_subdomain_scan", + zap.String("message", "failed to read response body"), + zap.Error(err), + ) return whois, err } @@ -294,13 +452,37 @@ func GetWHOISCredits(apiKey string) (sqlite.WhoIsCredits, error) { defer res.Body.Close() } if err != nil { + zap.L().Error("get_whois_credits", + zap.String("message", "failed to perform request"), + zap.Error(err), + ) return whoisCredits, err } if res == nil { + zap.L().Error("get_whois_credits", + zap.String("message", "response was nil"), + ) return whoisCredits, errors.New("response was nil") } + + // Check for HTTP status code errors + if res.StatusCode != 200 { + dhErr := query.GetDehashedError(res.StatusCode) + fmt.Printf("[%d] API Error message: %s\n", res.StatusCode, dhErr.Error()) + zap.L().Error("get_whois_credits", + zap.String("message", "received error status code"), + zap.Int("status_code", res.StatusCode), + zap.String("error", dhErr.Error()), + ) + return whoisCredits, &dhErr + } + b, err := io.ReadAll(res.Body) if err != nil { + zap.L().Error("get_whois_credits", + zap.String("message", "failed to read response body"), + zap.Error(err), + ) return whoisCredits, err }