Added enhanced logging for specific errors related to dehash tied into the apiu calls. Added logs subcommand.

cmd/logs.go

@@ -0,0 +1,226 @@
+package cmd
+
+import (
+	"dehasher/internal/pretty"
+	"encoding/json"
+	"fmt"
+	"github.com/spf13/cobra"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+func init() {
+	// Add Subcommand to db command
+	rootCmd.AddCommand(logsCmd)
+
+	logsCmd.Flags().IntVarP(&logLastXLogs, "last", "l", 0, "Number of logs to show")
+	logsCmd.Flags().StringVarP(&logStartDate, "start", "s", "", "Start date for logs")
+	logsCmd.Flags().StringVarP(&logEndDate, "end", "e", "", "End date for logs")
+	logsCmd.Flags().StringVarP(&logSeverity, "severity", "v", "", "Comma delimited Log severity to show (info, error, fatal)")
+}
+
+// LogEntry represents a parsed log entry
+type LogEntry struct {
+	Level      string                 `json:"level"`
+	Timestamp  string                 `json:"ts"` // Store as string initially
+	Message    string                 `json:"msg"`
+	Details    map[string]interface{} `json:"-"`
+	ParsedTime time.Time              // Parsed time for sorting and filtering
+}
+
+var (
+	logLastXLogs int
+	logStartDate string
+	logEndDate   string
+	logSeverity  string
+
+	logsCmd = &cobra.Command{
+		Use:   "logs",
+		Short: "View logs",
+		Long:  `View logs for the Dehasher CLI.`,
+		Run: func(cmd *cobra.Command, args []string) {
+			logsPath := filepath.Join(os.Getenv("HOME"), ".local", "share", "Dehasher", "logs")
+
+			if logSeverity == "" {
+				logSeverity = "info,error,fatal"
+			} else {
+				logSeverity = strings.ToLower(logSeverity)
+			}
+
+			var (
+				logInfo  bool
+				logError bool
+				logFatal bool
+			)
+			if strings.Contains(logSeverity, "info") {
+				logInfo = true
+			}
+			if strings.Contains(logSeverity, "error") {
+				logError = true
+			}
+			if strings.Contains(logSeverity, "fatal") {
+				logFatal = true
+			}
+
+			var allLogs []string
+			if logSeverity == "info" {
+				allLogs = append(allLogs, filepath.Join(logsPath, "info.log"))
+			} else if logSeverity == "error" || logSeverity == "fatal" {
+				allLogs = append(allLogs, filepath.Join(logsPath, "error.log"))
+			} else {
+				allLogs = append(allLogs, filepath.Join(logsPath, "info.log"), filepath.Join(logsPath, "error.log"))
+			}
+
+			var parsedLogs []LogEntry
+			for _, logFile := range allLogs {
+				// Read the log file
+				logData, err := os.ReadFile(logFile)
+				if err != nil {
+					fmt.Printf("Error reading log file %s: %v\n", logFile, err)
+					continue
+				}
+
+				// Split the file into lines
+				logLines := strings.Split(strings.TrimSpace(string(logData)), "\n")
+				for _, line := range logLines {
+					if line == "" {
+						continue
+					}
+
+					// Parse the JSON log entry
+					var entry LogEntry
+					var rawEntry map[string]interface{}
+					if err := json.Unmarshal([]byte(line), &entry); err != nil {
+						fmt.Printf("Error parsing log entry: %v\n", err)
+						continue
+					}
+
+					// Also unmarshal to get additional fields
+					if err := json.Unmarshal([]byte(line), &rawEntry); err != nil {
+						fmt.Printf("Error parsing raw log entry: %v\n", err)
+						continue
+					}
+
+					// Parse the timestamp
+					parsedTime, err := time.Parse("2006-01-02T15:04:05.999-0700", entry.Timestamp)
+					if err != nil {
+						// Try alternative formats
+						parsedTime, err = time.Parse(time.RFC3339, entry.Timestamp)
+						if err != nil {
+							// Try another format
+							parsedTime, err = time.Parse(time.RFC3339Nano, entry.Timestamp)
+							if err != nil {
+								fmt.Printf("Error parsing timestamp '%s': %v\n", entry.Timestamp, err)
+								continue
+							}
+						}
+					}
+					entry.ParsedTime = parsedTime
+
+					// Store additional fields in Details
+					entry.Details = make(map[string]interface{})
+					for k, v := range rawEntry {
+						if k != "level" && k != "ts" && k != "msg" {
+							entry.Details[k] = v
+						}
+					}
+
+					// Filter by severity
+					if (logInfo && strings.EqualFold(entry.Level, "INFO")) ||
+						(logError && strings.EqualFold(entry.Level, "ERROR")) ||
+						(logFatal && strings.EqualFold(entry.Level, "FATAL")) {
+
+						// Filter by date range if specified
+						if logStartDate != "" {
+							startDate, err := time.Parse("2006-01-02", logStartDate)
+							if err != nil {
+								fmt.Printf("Error parsing start date: %v\n", err)
+							} else if entry.ParsedTime.Before(startDate) {
+								continue
+							}
+						}
+
+						if logEndDate != "" {
+							endDate, err := time.Parse("2006-01-02", logEndDate)
+							// Add one day to include the end date
+							endDate = endDate.Add(24 * time.Hour)
+							if err != nil {
+								fmt.Printf("Error parsing end date: %v\n", err)
+							} else if entry.ParsedTime.After(endDate) {
+								continue
+							}
+						}
+
+						parsedLogs = append(parsedLogs, entry)
+					}
+				}
+			}
+
+			// Limit the number of logs if specified
+			if logLastXLogs > 0 && len(parsedLogs) > logLastXLogs {
+				// Sort logs by timestamp (newest first)
+				// This is a simple bubble sort for demonstration
+				for i := 0; i < len(parsedLogs)-1; i++ {
+					for j := 0; j < len(parsedLogs)-i-1; j++ {
+						if parsedLogs[j].ParsedTime.Before(parsedLogs[j+1].ParsedTime) {
+							parsedLogs[j], parsedLogs[j+1] = parsedLogs[j+1], parsedLogs[j]
+						}
+					}
+				}
+				parsedLogs = parsedLogs[:logLastXLogs]
+			}
+
+			// Display logs in a table
+			if len(parsedLogs) > 0 {
+				// Prepare table headers and rows
+				headers := []string{"Date", "Severity", "Message", "Details"}
+				rows := make([][]string, len(parsedLogs))
+
+				for i, log := range parsedLogs {
+					// Format timestamp
+					timeStr := log.ParsedTime.Format("2006-01-02 15:04:05")
+
+					// Format details
+					detailsStr := ""
+					for k, v := range log.Details {
+						detailsStr += fmt.Sprintf("%s: %v, ", k, v)
+					}
+					if len(detailsStr) > 2 {
+						detailsStr = detailsStr[:len(detailsStr)-2] // Remove trailing comma and space
+					}
+
+					rows[i] = []string{timeStr, log.Level, log.Message, detailsStr}
+				}
+
+				// Display the table
+				pretty.Table(headers, rows)
+			} else {
+				fmt.Println("No logs found matching the specified criteria.")
+			}
+		},
+	}
+)
+
+type Severity int
+
+const (
+	INFO Severity = iota
+	ERROR
+	FATAL
+	UNKNOWN Severity = -1
+)
+
+func getSeverity(logLevel string) Severity {
+	switch logLevel {
+	case "INFO":
+		return INFO
+	case "ERROR":
+		return ERROR
+	case "FATAL":
+		return FATAL
+	default:
+		return UNKNOWN
+	}
+}

dehasher.go

@@ -65,7 +65,7 @@ func initializeLogger() {
 		})),
 	})
 
-	zap.L().Info("some message", zap.Int("status", 0))
+	zap.L().Info("logger initialized", zap.Int("status", 0))
 }
 
 func main() {

internal/query/client.go

@@ -1,111 +0,0 @@
-package query
-
-import (
-	"dehasher/internal/sqlite"
-	"fmt"
-	"log"
-	"net/http"
-	"net/url"
-	"os"
-	"strings"
-)
-
-type DehashedClient struct {
-	key      string
-	email    string
-	results  []sqlite.Result
-	client   *http.Client
-	query    string
-	params   string
-	printBal bool
-	total    int
-	balance  int
-}
-
-var baseUrl = "https://api.dehashed.com/v2/search"
-
-func NewDehashedClient(key, email string, printBal bool) *DehashedClient {
-	return &DehashedClient{key: key, email: email, results: make([]sqlite.Result, 0), client: &http.Client{}, printBal: printBal}
-}
-
-func (dc *DehashedClient) getKey() string {
-	return dc.key
-}
-
-func (dc *DehashedClient) getEmail() string {
-	return dc.email
-}
-
-func (dc *DehashedClient) GetResults() sqlite.DehashedResults {
-	return sqlite.DehashedResults{Results: dc.results}
-}
-
-func (dc *DehashedClient) buildQuery(params map[string]string) {
-	urlParams := url.Values{}
-	urlString := baseUrl
-
-	if len(params) > 0 {
-		urlString += "?query="
-
-		for k, v := range params {
-			if len(v) > 0 {
-				urlParams.Add(k, v)
-			}
-		}
-	}
-
-	tmp, _ := url.QueryUnescape(urlParams.Encode())
-	tmp2 := strings.Replace(tmp, "=", ":", -1)
-	dc.params = tmp2
-	urlString += dc.params
-	dc.query = urlString
-}
-
-func (dc *DehashedClient) setResults(results int) {
-	dc.query = fmt.Sprintf("%s?query=%s&size=%d", baseUrl, dc.params, results)
-}
-
-func (dc *DehashedClient) setPage(page int) {
-	dc.query = fmt.Sprintf("%s&nextPage=%d", dc.query, page)
-}
-
-func (dc *DehashedClient) Do() int {
-	fmt.Printf("\n\t[*] Performing Request...")
-	req, err := http.NewRequest("GET", dc.query, nil)
-	if err != nil {
-		fmt.Printf("[!] Error constructing request: %v", err)
-		os.Exit(-1)
-	}
-
-	dc.setAuth(req)
-	req.Header.Add("Dehashed-Api-Key", dc.getKey())
-	req.Header.Add("Accept", "application/json")
-	resp, err := dc.client.Do(req)
-	if err != nil {
-		fmt.Printf("[!] Error performing request: %s\n%v", dc.query, err)
-		os.Exit(-1)
-	}
-
-	if resp.StatusCode != 200 {
-		dhErr := GetDehashedError(resp.StatusCode)
-		fmt.Println()
-		log.Fatal(dhErr.Error())
-	}
-
-	entries, balance, total := sqlite.NewDehashedResults(resp.Body)
-	dc.results = append(dc.results, entries...)
-	dc.balance = balance
-	dc.total += total
-	if dc.printBal {
-		fmt.Printf("\n\t\t[*] Balance Remaining: %d", balance)
-	}
-	return total
-}
-
-func (dc *DehashedClient) setAuth(r *http.Request) {
-	r.SetBasicAuth(dc.email, dc.key)
-}
-
-func (dc *DehashedClient) GetDomains() int {
-	return dc.balance
-}

internal/query/clientv2.go

@@ -163,6 +163,19 @@ func (dcv2 *DehashedClientV2) Search(searchRequest DehashedSearchRequest) (int,
 		)
 		return -1, errors.New("response was nil")
 	}
+
+	// Check for HTTP status code errors
+	if res.StatusCode != 200 {
+		dhErr := GetDehashedError(res.StatusCode)
+		fmt.Printf("[%d] API Error message: %s\n", res.StatusCode, dhErr.Error())
+		zap.L().Error("v2_search",
+			zap.String("message", "received error status code"),
+			zap.Int("status_code", res.StatusCode),
+			zap.String("error", dhErr.Error()),
+		)
+		return -1, &dhErr
+	}
+
 	b, err := io.ReadAll(res.Body)
 	if err != nil {
 		zap.L().Error("v2_search",

internal/query/dehashed.go

@@ -90,7 +90,20 @@ func (dh *Dehasher) Start() {
 		fmt.Printf("\n\t[*] Performing Request...")
 		count, err := dh.client.Search(*dh.request)
 		if err != nil {
-			fmt.Printf("[!] Error performing request: %v", err)
+			// Check if it's a DehashError
+			if dhErr, ok := err.(*DehashError); ok {
+				fmt.Printf("\n\t[!] Dehashed API Error: %s (Code: %d)", dhErr.Message, dhErr.Code)
+				zap.L().Error("dehashed_api_error",
+					zap.String("message", dhErr.Message),
+					zap.Int("code", dhErr.Code),
+				)
+			} else {
+				fmt.Printf("\n\t[!] Error performing request: %v", err)
+				zap.L().Error("request_error",
+					zap.String("message", "failed to perform request"),
+					zap.Error(err),
+				)
+			}
 			os.Exit(-1)
 		}
 

internal/whois/whois.go

@@ -2,6 +2,7 @@ package whois
 
 import (
 	"bytes"
+	"dehasher/internal/query"
 	"dehasher/internal/sqlite"
 	"encoding/json"
 	"errors"
@@ -40,13 +41,37 @@ func WhoisSearch(domain, apiKey string) (sqlite.WhoIsLookupResult, error) {
 		defer res.Body.Close()
 	}
 	if err != nil {
+		zap.L().Error("whois_search",
+			zap.String("message", "failed to perform request"),
+			zap.Error(err),
+		)
 		return whois, err
 	}
 	if res == nil {
+		zap.L().Error("whois_search",
+			zap.String("message", "response was nil"),
+		)
 		return whois, errors.New("response was nil")
 	}
+
+	// Check for HTTP status code errors
+	if res.StatusCode != 200 {
+		dhErr := query.GetDehashedError(res.StatusCode)
+		fmt.Printf("[%d] API Error message: %s\n", res.StatusCode, dhErr.Error())
+		zap.L().Error("whois_search",
+			zap.String("message", "received error status code"),
+			zap.Int("status_code", res.StatusCode),
+			zap.String("error", dhErr.Error()),
+		)
+		return whois, &dhErr
+	}
+
 	b, err := io.ReadAll(res.Body)
 	if err != nil {
+		zap.L().Error("whois_search",
+			zap.String("message", "failed to read response body"),
+			zap.Error(err),
+		)
 		return whois, err
 	}
 
@@ -97,6 +122,19 @@ func WhoisHistory(domain, apiKey string) (sqlite.WhoIsHistory, error) {
 		)
 		return whois, errors.New("response was nil")
 	}
+
+	// Check for HTTP status code errors
+	if res.StatusCode != 200 {
+		dhErr := query.GetDehashedError(res.StatusCode)
+		fmt.Printf("[%d] API Error message: %s\n", res.StatusCode, dhErr.Error())
+		zap.L().Error("whois_history",
+			zap.String("message", "received error status code"),
+			zap.Int("status_code", res.StatusCode),
+			zap.String("error", dhErr.Error()),
+		)
+		return whois, &dhErr
+	}
+
 	b, err := io.ReadAll(res.Body)
 	if err != nil {
 		zap.L().Error("whois_history",
@@ -139,13 +177,37 @@ func ReverseWHOIS(include []string, exclude []string, reverseType, apiKey string
 		defer res.Body.Close()
 	}
 	if err != nil {
+		zap.L().Error("reverse_whois",
+			zap.String("message", "failed to perform request"),
+			zap.Error(err),
+		)
 		return "", err
 	}
 	if res == nil {
+		zap.L().Error("reverse_whois",
+			zap.String("message", "response was nil"),
+		)
 		return "", errors.New("response was nil")
 	}
+
+	// Check for HTTP status code errors
+	if res.StatusCode != 200 {
+		dhErr := query.GetDehashedError(res.StatusCode)
+		fmt.Printf("[%d] API Error message: %s\n", res.StatusCode, dhErr.Error())
+		zap.L().Error("reverse_whois",
+			zap.String("message", "received error status code"),
+			zap.Int("status_code", res.StatusCode),
+			zap.String("error", dhErr.Error()),
+		)
+		return "", &dhErr
+	}
+
 	b, err := io.ReadAll(res.Body)
 	if err != nil {
+		zap.L().Error("reverse_whois",
+			zap.String("message", "failed to read response body"),
+			zap.Error(err),
+		)
 		return "", err
 	}
 	return string(b), nil
@@ -168,13 +230,37 @@ func WhoisIP(ipAddress, apiKey string) ([]byte, error) {
 		defer res.Body.Close()
 	}
 	if err != nil {
+		zap.L().Error("whois_ip",
+			zap.String("message", "failed to perform request"),
+			zap.Error(err),
+		)
 		return nil, err
 	}
 	if res == nil {
+		zap.L().Error("whois_ip",
+			zap.String("message", "response was nil"),
+		)
 		return nil, errors.New("response was nil")
 	}
+
+	// Check for HTTP status code errors
+	if res.StatusCode != 200 {
+		dhErr := query.GetDehashedError(res.StatusCode)
+		fmt.Printf("[%d] API Error message: %s\n", res.StatusCode, dhErr.Error())
+		zap.L().Error("whois_ip",
+			zap.String("message", "received error status code"),
+			zap.Int("status_code", res.StatusCode),
+			zap.String("error", dhErr.Error()),
+		)
+		return nil, &dhErr
+	}
+
 	b, err := io.ReadAll(res.Body)
 	if err != nil {
+		zap.L().Error("whois_ip",
+			zap.String("message", "failed to read response body"),
+			zap.Error(err),
+		)
 		return nil, err
 	}
 	return b, nil
@@ -197,13 +283,37 @@ func WhoisMX(mxAddress, apiKey string) (string, error) {
 		defer res.Body.Close()
 	}
 	if err != nil {
+		zap.L().Error("whois_mx",
+			zap.String("message", "failed to perform request"),
+			zap.Error(err),
+		)
 		return "", err
 	}
 	if res == nil {
+		zap.L().Error("whois_mx",
+			zap.String("message", "response was nil"),
+		)
 		return "", errors.New("response was nil")
 	}
+
+	// Check for HTTP status code errors
+	if res.StatusCode != 200 {
+		dhErr := query.GetDehashedError(res.StatusCode)
+		fmt.Printf("[%d] API Error message: %s\n", res.StatusCode, dhErr.Error())
+		zap.L().Error("whois_mx",
+			zap.String("message", "received error status code"),
+			zap.Int("status_code", res.StatusCode),
+			zap.String("error", dhErr.Error()),
+		)
+		return "", &dhErr
+	}
+
 	b, err := io.ReadAll(res.Body)
 	if err != nil {
+		zap.L().Error("whois_mx",
+			zap.String("message", "failed to read response body"),
+			zap.Error(err),
+		)
 		return "", err
 	}
 	return string(b), nil
@@ -226,13 +336,37 @@ func WhoisNS(nsAddress, apiKey string) (string, error) {
 		defer res.Body.Close()
 	}
 	if err != nil {
+		zap.L().Error("whois_ns",
+			zap.String("message", "failed to perform request"),
+			zap.Error(err),
+		)
 		return "", err
 	}
 	if res == nil {
+		zap.L().Error("whois_ns",
+			zap.String("message", "response was nil"),
+		)
 		return "", errors.New("response was nil")
 	}
+
+	// Check for HTTP status code errors
+	if res.StatusCode != 200 {
+		dhErr := query.GetDehashedError(res.StatusCode)
+		fmt.Printf("[%d] API Error message: %s\n", res.StatusCode, dhErr.Error())
+		zap.L().Error("whois_ns",
+			zap.String("message", "received error status code"),
+			zap.Int("status_code", res.StatusCode),
+			zap.String("error", dhErr.Error()),
+		)
+		return "", &dhErr
+	}
+
 	b, err := io.ReadAll(res.Body)
 	if err != nil {
+		zap.L().Error("whois_ns",
+			zap.String("message", "failed to read response body"),
+			zap.Error(err),
+		)
 		return "", err
 	}
 	return string(b), nil
@@ -256,13 +390,37 @@ func WhoisSubdomainScan(domain, apiKey string) (sqlite.WhoIsSubdomainScan, error
 		defer res.Body.Close()
 	}
 	if err != nil {
+		zap.L().Error("whois_subdomain_scan",
+			zap.String("message", "failed to perform request"),
+			zap.Error(err),
+		)
 		return whois, err
 	}
 	if res == nil {
+		zap.L().Error("whois_subdomain_scan",
+			zap.String("message", "response was nil"),
+		)
 		return whois, errors.New("response was nil")
 	}
+
+	// Check for HTTP status code errors
+	if res.StatusCode != 200 {
+		dhErr := query.GetDehashedError(res.StatusCode)
+		fmt.Printf("[%d] API Error message: %s\n", res.StatusCode, dhErr.Error())
+		zap.L().Error("whois_subdomain_scan",
+			zap.String("message", "received error status code"),
+			zap.Int("status_code", res.StatusCode),
+			zap.String("error", dhErr.Error()),
+		)
+		return whois, &dhErr
+	}
+
 	b, err := io.ReadAll(res.Body)
 	if err != nil {
+		zap.L().Error("whois_subdomain_scan",
+			zap.String("message", "failed to read response body"),
+			zap.Error(err),
+		)
 		return whois, err
 	}
 
@@ -294,13 +452,37 @@ func GetWHOISCredits(apiKey string) (sqlite.WhoIsCredits, error) {
 		defer res.Body.Close()
 	}
 	if err != nil {
+		zap.L().Error("get_whois_credits",
+			zap.String("message", "failed to perform request"),
+			zap.Error(err),
+		)
 		return whoisCredits, err
 	}
 	if res == nil {
+		zap.L().Error("get_whois_credits",
+			zap.String("message", "response was nil"),
+		)
 		return whoisCredits, errors.New("response was nil")
 	}
+
+	// Check for HTTP status code errors
+	if res.StatusCode != 200 {
+		dhErr := query.GetDehashedError(res.StatusCode)
+		fmt.Printf("[%d] API Error message: %s\n", res.StatusCode, dhErr.Error())
+		zap.L().Error("get_whois_credits",
+			zap.String("message", "received error status code"),
+			zap.Int("status_code", res.StatusCode),
+			zap.String("error", dhErr.Error()),
+		)
+		return whoisCredits, &dhErr
+	}
+
 	b, err := io.ReadAll(res.Body)
 	if err != nil {
+		zap.L().Error("get_whois_credits",
+			zap.String("message", "failed to read response body"),
+			zap.Error(err),
+		)
 		return whoisCredits, err
 	}