KrakenTech/CrowsNest
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added enhanced logging for specific errors related to dehash tied into the apiu calls. Added logs subcommand.

Browse Source
This commit is contained in:
Evan Hosinski
2025-05-15 16:23:59 -04:00
parent 749f2b5eb8
commit c29f18a056
6 changed files with 436 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/logs.go
+226
View File
@@ -0,0 +1,226 @@
package cmd
import (
"dehasher/internal/pretty"
"encoding/json"
"fmt"
"github.com/spf13/cobra"
"os"
"path/filepath"
"strings"
"time"
)
func init() {
// Add Subcommand to db command
rootCmd.AddCommand(logsCmd)
logsCmd.Flags().IntVarP(&logLastXLogs, "last", "l", 0, "Number of logs to show")
logsCmd.Flags().StringVarP(&logStartDate, "start", "s", "", "Start date for logs")
logsCmd.Flags().StringVarP(&logEndDate, "end", "e", "", "End date for logs")
logsCmd.Flags().StringVarP(&logSeverity, "severity", "v", "", "Comma delimited Log severity to show (info, error, fatal)")
}
// LogEntry represents a parsed log entry
type LogEntry struct {
Level string `json:"level"`
Timestamp string `json:"ts"` // Store as string initially
Message string `json:"msg"`
Details map[string]interface{} `json:"-"`
ParsedTime time.Time // Parsed time for sorting and filtering
}
var (
logLastXLogs int
logStartDate string
logEndDate string
logSeverity string
logsCmd = &cobra.Command{
Use: "logs",
Short: "View logs",
Long: `View logs for the Dehasher CLI.`,
Run: func(cmd *cobra.Command, args []string) {
logsPath := filepath.Join(os.Getenv("HOME"), ".local", "share", "Dehasher", "logs")
if logSeverity == "" {
logSeverity = "info,error,fatal"
} else {
logSeverity = strings.ToLower(logSeverity)
}
var (
logInfo bool
logError bool
logFatal bool
)
if strings.Contains(logSeverity, "info") {
logInfo = true
}
if strings.Contains(logSeverity, "error") {
logError = true
}
if strings.Contains(logSeverity, "fatal") {
logFatal = true
}
var allLogs []string
if logSeverity == "info" {
allLogs = append(allLogs, filepath.Join(logsPath, "info.log"))
} else if logSeverity == "error" || logSeverity == "fatal" {
allLogs = append(allLogs, filepath.Join(logsPath, "error.log"))
} else {
allLogs = append(allLogs, filepath.Join(logsPath, "info.log"), filepath.Join(logsPath, "error.log"))
}
var parsedLogs []LogEntry
for _, logFile := range allLogs {
// Read the log file
logData, err := os.ReadFile(logFile)
if err != nil {
fmt.Printf("Error reading log file %s: %v\n", logFile, err)
continue
}
// Split the file into lines
logLines := strings.Split(strings.TrimSpace(string(logData)), "\n")
for _, line := range logLines {
if line == "" {
continue
}
// Parse the JSON log entry
var entry LogEntry
var rawEntry map[string]interface{}
if err := json.Unmarshal([]byte(line), &entry); err != nil {
fmt.Printf("Error parsing log entry: %v\n", err)
continue
}
// Also unmarshal to get additional fields
if err := json.Unmarshal([]byte(line), &rawEntry); err != nil {
fmt.Printf("Error parsing raw log entry: %v\n", err)
continue
}
// Parse the timestamp
parsedTime, err := time.Parse("2006-01-02T15:04:05.999-0700", entry.Timestamp)
if err != nil {
// Try alternative formats
parsedTime, err = time.Parse(time.RFC3339, entry.Timestamp)
if err != nil {
// Try another format
parsedTime, err = time.Parse(time.RFC3339Nano, entry.Timestamp)
if err != nil {
fmt.Printf("Error parsing timestamp '%s': %v\n", entry.Timestamp, err)
continue
}
}
}
entry.ParsedTime = parsedTime
// Store additional fields in Details
entry.Details = make(map[string]interface{})
for k, v := range rawEntry {
if k != "level" && k != "ts" && k != "msg" {
entry.Details[k] = v
}
}
// Filter by severity
if (logInfo && strings.EqualFold(entry.Level, "INFO")) ||
(logError && strings.EqualFold(entry.Level, "ERROR")) ||
(logFatal && strings.EqualFold(entry.Level, "FATAL")) {
// Filter by date range if specified
if logStartDate != "" {
startDate, err := time.Parse("2006-01-02", logStartDate)
if err != nil {
fmt.Printf("Error parsing start date: %v\n", err)
} else if entry.ParsedTime.Before(startDate) {
continue
}
}
if logEndDate != "" {
endDate, err := time.Parse("2006-01-02", logEndDate)
// Add one day to include the end date
endDate = endDate.Add(24 * time.Hour)
if err != nil {
fmt.Printf("Error parsing end date: %v\n", err)
} else if entry.ParsedTime.After(endDate) {
continue
}
}
parsedLogs = append(parsedLogs, entry)
}
}
}
// Limit the number of logs if specified
if logLastXLogs > 0 && len(parsedLogs) > logLastXLogs {
// Sort logs by timestamp (newest first)
// This is a simple bubble sort for demonstration
for i := 0; i < len(parsedLogs)-1; i++ {
for j := 0; j < len(parsedLogs)-i-1; j++ {
if parsedLogs[j].ParsedTime.Before(parsedLogs[j+1].ParsedTime) {
parsedLogs[j], parsedLogs[j+1] = parsedLogs[j+1], parsedLogs[j]
}
}
}
parsedLogs = parsedLogs[:logLastXLogs]
}
// Display logs in a table
if len(parsedLogs) > 0 {
// Prepare table headers and rows
headers := []string{"Date", "Severity", "Message", "Details"}
rows := make([][]string, len(parsedLogs))
for i, log := range parsedLogs {
// Format timestamp
timeStr := log.ParsedTime.Format("2006-01-02 15:04:05")
// Format details
detailsStr := ""
for k, v := range log.Details {
detailsStr += fmt.Sprintf("%s: %v, ", k, v)
}
if len(detailsStr) > 2 {
detailsStr = detailsStr[:len(detailsStr)-2] // Remove trailing comma and space
}
rows[i] = []string{timeStr, log.Level, log.Message, detailsStr}
}
// Display the table
pretty.Table(headers, rows)
} else {
fmt.Println("No logs found matching the specified criteria.")
}
},
}
)
type Severity int
const (
INFO Severity = iota
ERROR
FATAL
UNKNOWN Severity = -1
)
func getSeverity(logLevel string) Severity {
switch logLevel {
case "INFO":
return INFO
case "ERROR":
return ERROR
case "FATAL":
return FATAL
default:
return UNKNOWN
}
}
dehasher.go
+1 -1
View File
@@ -65,7 +65,7 @@ func initializeLogger() {
})),
})
zap.L().Info("some message", zap.Int("status", 0))
zap.L().Info("logger initialized", zap.Int("status", 0))
}
func main() {
internal/query/client.go
-111
View File
@@ -1,111 +0,0 @@
package query
import (
"dehasher/internal/sqlite"
"fmt"
"log"
"net/http"
"net/url"
"os"
"strings"
)
type DehashedClient struct {
key string
email string
results []sqlite.Result
client *http.Client
query string
params string
printBal bool
total int
balance int
}
var baseUrl = "https://api.dehashed.com/v2/search"
func NewDehashedClient(key, email string, printBal bool) *DehashedClient {
return &DehashedClient{key: key, email: email, results: make([]sqlite.Result, 0), client: &http.Client{}, printBal: printBal}
}
func (dc *DehashedClient) getKey() string {
return dc.key
}
func (dc *DehashedClient) getEmail() string {
return dc.email
}
func (dc *DehashedClient) GetResults() sqlite.DehashedResults {
return sqlite.DehashedResults{Results: dc.results}
}
func (dc *DehashedClient) buildQuery(params map[string]string) {
urlParams := url.Values{}
urlString := baseUrl
if len(params) > 0 {
urlString += "?query="
for k, v := range params {
if len(v) > 0 {
urlParams.Add(k, v)
}
}
}
tmp, _ := url.QueryUnescape(urlParams.Encode())
tmp2 := strings.Replace(tmp, "=", ":", -1)
dc.params = tmp2
urlString += dc.params
dc.query = urlString
}
func (dc *DehashedClient) setResults(results int) {
dc.query = fmt.Sprintf("%s?query=%s&size=%d", baseUrl, dc.params, results)
}
func (dc *DehashedClient) setPage(page int) {
dc.query = fmt.Sprintf("%s&nextPage=%d", dc.query, page)
}
func (dc *DehashedClient) Do() int {
fmt.Printf("\n\t[*] Performing Request...")
req, err := http.NewRequest("GET", dc.query, nil)
if err != nil {
fmt.Printf("[!] Error constructing request: %v", err)
os.Exit(-1)
}
dc.setAuth(req)
req.Header.Add("Dehashed-Api-Key", dc.getKey())
req.Header.Add("Accept", "application/json")
resp, err := dc.client.Do(req)
if err != nil {
fmt.Printf("[!] Error performing request: %s\n%v", dc.query, err)
os.Exit(-1)
}
if resp.StatusCode != 200 {
dhErr := GetDehashedError(resp.StatusCode)
fmt.Println()
log.Fatal(dhErr.Error())
}
entries, balance, total := sqlite.NewDehashedResults(resp.Body)
dc.results = append(dc.results, entries...)
dc.balance = balance
dc.total += total
if dc.printBal {
fmt.Printf("\n\t\t[*] Balance Remaining: %d", balance)
}
return total
}
func (dc *DehashedClient) setAuth(r *http.Request) {
r.SetBasicAuth(dc.email, dc.key)
}
func (dc *DehashedClient) GetDomains() int {
return dc.balance
}
internal/query/clientv2.go
+13
View File
@@ -163,6 +163,19 @@ func (dcv2 *DehashedClientV2) Search(searchRequest DehashedSearchRequest) (int,
)
return -1, errors.New("response was nil")
}
// Check for HTTP status code errors
if res.StatusCode != 200 {
dhErr := GetDehashedError(res.StatusCode)
fmt.Printf("[%d] API Error message: %s\n", res.StatusCode, dhErr.Error())
zap.L().Error("v2_search",
zap.String("message", "received error status code"),
zap.Int("status_code", res.StatusCode),
zap.String("error", dhErr.Error()),
)
return -1, &dhErr
}
b, err := io.ReadAll(res.Body)
if err != nil {
zap.L().Error("v2_search",
internal/query/dehashed.go
+14 -1
View File
@@ -90,7 +90,20 @@ func (dh *Dehasher) Start() {
fmt.Printf("\n\t[*] Performing Request...")
count, err := dh.client.Search(*dh.request)
if err != nil {
fmt.Printf("[!] Error performing request: %v", err)
// Check if it's a DehashError
if dhErr, ok := err.(*DehashError); ok {
fmt.Printf("\n\t[!] Dehashed API Error: %s (Code: %d)", dhErr.Message, dhErr.Code)
zap.L().Error("dehashed_api_error",
zap.String("message", dhErr.Message),
zap.Int("code", dhErr.Code),
)
} else {
fmt.Printf("\n\t[!] Error performing request: %v", err)
zap.L().Error("request_error",
zap.String("message", "failed to perform request"),
zap.Error(err),
)
}
os.Exit(-1)
}
internal/whois/whois.go
+182
View File
@@ -2,6 +2,7 @@ package whois
import (
"bytes"
"dehasher/internal/query"
"dehasher/internal/sqlite"
"encoding/json"
"errors"
@@ -40,13 +41,37 @@ func WhoisSearch(domain, apiKey string) (sqlite.WhoIsLookupResult, error) {
defer res.Body.Close()
}
if err != nil {
zap.L().Error("whois_search",
zap.String("message", "failed to perform request"),
zap.Error(err),
)
return whois, err
}
if res == nil {
zap.L().Error("whois_search",
zap.String("message", "response was nil"),
)
return whois, errors.New("response was nil")
}
// Check for HTTP status code errors
if res.StatusCode != 200 {
dhErr := query.GetDehashedError(res.StatusCode)
fmt.Printf("[%d] API Error message: %s\n", res.StatusCode, dhErr.Error())
zap.L().Error("whois_search",
zap.String("message", "received error status code"),
zap.Int("status_code", res.StatusCode),
zap.String("error", dhErr.Error()),
)
return whois, &dhErr
}
b, err := io.ReadAll(res.Body)
if err != nil {
zap.L().Error("whois_search",
zap.String("message", "failed to read response body"),
zap.Error(err),
)
return whois, err
}
@@ -97,6 +122,19 @@ func WhoisHistory(domain, apiKey string) (sqlite.WhoIsHistory, error) {
)
return whois, errors.New("response was nil")
}
// Check for HTTP status code errors
if res.StatusCode != 200 {
dhErr := query.GetDehashedError(res.StatusCode)
fmt.Printf("[%d] API Error message: %s\n", res.StatusCode, dhErr.Error())
zap.L().Error("whois_history",
zap.String("message", "received error status code"),
zap.Int("status_code", res.StatusCode),
zap.String("error", dhErr.Error()),
)
return whois, &dhErr
}
b, err := io.ReadAll(res.Body)
if err != nil {
zap.L().Error("whois_history",
@@ -139,13 +177,37 @@ func ReverseWHOIS(include []string, exclude []string, reverseType, apiKey string
defer res.Body.Close()
}
if err != nil {
zap.L().Error("reverse_whois",
zap.String("message", "failed to perform request"),
zap.Error(err),
)
return "", err
}
if res == nil {
zap.L().Error("reverse_whois",
zap.String("message", "response was nil"),
)
return "", errors.New("response was nil")
}
// Check for HTTP status code errors
if res.StatusCode != 200 {
dhErr := query.GetDehashedError(res.StatusCode)
fmt.Printf("[%d] API Error message: %s\n", res.StatusCode, dhErr.Error())
zap.L().Error("reverse_whois",
zap.String("message", "received error status code"),
zap.Int("status_code", res.StatusCode),
zap.String("error", dhErr.Error()),
)
return "", &dhErr
}
b, err := io.ReadAll(res.Body)
if err != nil {
zap.L().Error("reverse_whois",
zap.String("message", "failed to read response body"),
zap.Error(err),
)
return "", err
}
return string(b), nil
@@ -168,13 +230,37 @@ func WhoisIP(ipAddress, apiKey string) ([]byte, error) {
defer res.Body.Close()
}
if err != nil {
zap.L().Error("whois_ip",
zap.String("message", "failed to perform request"),
zap.Error(err),
)
return nil, err
}
if res == nil {
zap.L().Error("whois_ip",
zap.String("message", "response was nil"),
)
return nil, errors.New("response was nil")
}
// Check for HTTP status code errors
if res.StatusCode != 200 {
dhErr := query.GetDehashedError(res.StatusCode)
fmt.Printf("[%d] API Error message: %s\n", res.StatusCode, dhErr.Error())
zap.L().Error("whois_ip",
zap.String("message", "received error status code"),
zap.Int("status_code", res.StatusCode),
zap.String("error", dhErr.Error()),
)
return nil, &dhErr
}
b, err := io.ReadAll(res.Body)
if err != nil {
zap.L().Error("whois_ip",
zap.String("message", "failed to read response body"),
zap.Error(err),
)
return nil, err
}
return b, nil
@@ -197,13 +283,37 @@ func WhoisMX(mxAddress, apiKey string) (string, error) {
defer res.Body.Close()
}
if err != nil {
zap.L().Error("whois_mx",
zap.String("message", "failed to perform request"),
zap.Error(err),
)
return "", err
}
if res == nil {
zap.L().Error("whois_mx",
zap.String("message", "response was nil"),
)
return "", errors.New("response was nil")
}
// Check for HTTP status code errors
if res.StatusCode != 200 {
dhErr := query.GetDehashedError(res.StatusCode)
fmt.Printf("[%d] API Error message: %s\n", res.StatusCode, dhErr.Error())
zap.L().Error("whois_mx",
zap.String("message", "received error status code"),
zap.Int("status_code", res.StatusCode),
zap.String("error", dhErr.Error()),
)
return "", &dhErr
}
b, err := io.ReadAll(res.Body)
if err != nil {
zap.L().Error("whois_mx",
zap.String("message", "failed to read response body"),
zap.Error(err),
)
return "", err
}
return string(b), nil
@@ -226,13 +336,37 @@ func WhoisNS(nsAddress, apiKey string) (string, error) {
defer res.Body.Close()
}
if err != nil {
zap.L().Error("whois_ns",
zap.String("message", "failed to perform request"),
zap.Error(err),
)
return "", err
}
if res == nil {
zap.L().Error("whois_ns",
zap.String("message", "response was nil"),
)
return "", errors.New("response was nil")
}
// Check for HTTP status code errors
if res.StatusCode != 200 {
dhErr := query.GetDehashedError(res.StatusCode)
fmt.Printf("[%d] API Error message: %s\n", res.StatusCode, dhErr.Error())
zap.L().Error("whois_ns",
zap.String("message", "received error status code"),
zap.Int("status_code", res.StatusCode),
zap.String("error", dhErr.Error()),
)
return "", &dhErr
}
b, err := io.ReadAll(res.Body)
if err != nil {
zap.L().Error("whois_ns",
zap.String("message", "failed to read response body"),
zap.Error(err),
)
return "", err
}
return string(b), nil
@@ -256,13 +390,37 @@ func WhoisSubdomainScan(domain, apiKey string) (sqlite.WhoIsSubdomainScan, error
defer res.Body.Close()
}
if err != nil {
zap.L().Error("whois_subdomain_scan",
zap.String("message", "failed to perform request"),
zap.Error(err),
)
return whois, err
}
if res == nil {
zap.L().Error("whois_subdomain_scan",
zap.String("message", "response was nil"),
)
return whois, errors.New("response was nil")
}
// Check for HTTP status code errors
if res.StatusCode != 200 {
dhErr := query.GetDehashedError(res.StatusCode)
fmt.Printf("[%d] API Error message: %s\n", res.StatusCode, dhErr.Error())
zap.L().Error("whois_subdomain_scan",
zap.String("message", "received error status code"),
zap.Int("status_code", res.StatusCode),
zap.String("error", dhErr.Error()),
)
return whois, &dhErr
}
b, err := io.ReadAll(res.Body)
if err != nil {
zap.L().Error("whois_subdomain_scan",
zap.String("message", "failed to read response body"),
zap.Error(err),
)
return whois, err
}
@@ -294,13 +452,37 @@ func GetWHOISCredits(apiKey string) (sqlite.WhoIsCredits, error) {
defer res.Body.Close()
}
if err != nil {
zap.L().Error("get_whois_credits",
zap.String("message", "failed to perform request"),
zap.Error(err),
)
return whoisCredits, err
}
if res == nil {
zap.L().Error("get_whois_credits",
zap.String("message", "response was nil"),
)
return whoisCredits, errors.New("response was nil")
}
// Check for HTTP status code errors
if res.StatusCode != 200 {
dhErr := query.GetDehashedError(res.StatusCode)
fmt.Printf("[%d] API Error message: %s\n", res.StatusCode, dhErr.Error())
zap.L().Error("get_whois_credits",
zap.String("message", "received error status code"),
zap.Int("status_code", res.StatusCode),
zap.String("error", dhErr.Error()),
)
return whoisCredits, &dhErr
}
b, err := io.ReadAll(res.Body)
if err != nil {
zap.L().Error("get_whois_credits",
zap.String("message", "failed to read response body"),
zap.Error(err),
)
return whoisCredits, err
}