Updates to allow for enhanced debugging.

Added structs for whois calls.

Added ability to write WhoIs to file.

Added structured output for Whois Records.

Added String Method for WhoIsRecord and WhoIsHistory Records.

internal/dehashed/clientv2.go

@@ -1,8 +1,9 @@
-package query
+package dehashed
 
 import (
 	"bytes"
 	"crypto/sha256"
+	"dehasher/internal/debug"
 	"dehasher/internal/sqlite"
 	"encoding/hex"
 	"encoding/json"
@@ -38,6 +39,7 @@ func (dp DehashedParameter) GetArgumentString(arg string) string {
 
 type DehashedSearchRequest struct {
 	ForcePlaintext bool   `json:"-"`
+	Debug          bool   `json:"-"`
 	Page           int    `json:"page"`
 	Query          string `json:"query"`
 	Size           int    `json:"size"`
@@ -46,42 +48,60 @@ type DehashedSearchRequest struct {
 	DeDupe         bool   `json:"de_dupe"`
 }
 
-func NewDehashedSearchRequest(page, size int, wildcard, regex, forcePlaintext bool) *DehashedSearchRequest {
-	return &DehashedSearchRequest{Page: page, Query: "", Size: size, Wildcard: wildcard, Regex: regex, DeDupe: true, ForcePlaintext: forcePlaintext}
+func NewDehashedSearchRequest(page, size int, wildcard, regex, forcePlaintext, debug bool) *DehashedSearchRequest {
+	return &DehashedSearchRequest{Page: page, Query: "", Size: size, Wildcard: wildcard, Regex: regex, DeDupe: true, ForcePlaintext: forcePlaintext, Debug: debug}
 }
 
-func (dsr *DehashedSearchRequest) buildQuery(query string, param DehashedParameter) {
+func (dsr *DehashedSearchRequest) buildQuery(query string) {
+	if dsr.Debug {
+		debug.PrintInfo(fmt.Sprintf("building query: %s", query))
+	}
+	// Ensure query is properly formatted
+	query = strings.TrimSpace(query)
+
+	// For regex queries, we need to ensure the regex pattern is properly escaped
+	// and not enquoted, as that would break the regex pattern
+	if dsr.Regex && !strings.HasPrefix(query, "\"") && !strings.HasSuffix(query, "\"") {
+		// Don't add extra quotes for regex patterns
+	} else if strings.Contains(query, " ") && !strings.HasPrefix(query, "\"") {
+		query = fmt.Sprintf("\"%s\"", query)
+	}
+
 	if len(dsr.Query) > 0 {
-		dsr.Query = fmt.Sprintf("%s&%s", strings.TrimSpace(dsr.Query), strings.TrimSpace(query))
+		dsr.Query = fmt.Sprintf("%s&%s", strings.TrimSpace(dsr.Query), query)
 	} else {
 		dsr.Query = query
 	}
+
+	if dsr.Debug {
+		debug.PrintInfo(fmt.Sprintf("query built: %s", dsr.Query))
+	}
 }
 
 func (dsr *DehashedSearchRequest) AddUsernameQuery(query string) {
 	query = enquoteSpaced(query)
-	dsr.buildQuery(Username.GetArgumentString(query), Username)
+	dsr.buildQuery(Username.GetArgumentString(query))
 }
 
 func (dsr *DehashedSearchRequest) AddEmailQuery(query string) {
 	query = enquoteSpaced(query)
-	dsr.buildQuery(Email.GetArgumentString(query), Email)
+	dsr.buildQuery(Email.GetArgumentString(query))
 }
 
 func (dsr *DehashedSearchRequest) AddIpAddressQuery(query string) {
 	query = enquoteSpaced(query)
-	dsr.buildQuery(IpAddress.GetArgumentString(query), IpAddress)
+	dsr.buildQuery(IpAddress.GetArgumentString(query))
 }
 
 func (dsr *DehashedSearchRequest) AddDomainQuery(query string) {
 	query = enquoteSpaced(query)
-	dsr.buildQuery(Domain.GetArgumentString(query), Domain)
+	dsr.buildQuery(Domain.GetArgumentString(query))
 }
 
 func (dsr *DehashedSearchRequest) AddPasswordQuery(query string) {
 	if dsr.ForcePlaintext {
 		query = enquoteSpaced(query)
-		dsr.buildQuery(Password.GetArgumentString(query), Password)
+		dsr.buildQuery(Password.GetArgumentString(query))
 		return
 	}
 	hash := sha256.Sum256([]byte(query))
@@ -91,89 +111,126 @@ func (dsr *DehashedSearchRequest) AddPasswordQuery(query string) {
 
 func (dsr *DehashedSearchRequest) AddVinQuery(query string) {
 	query = enquoteSpaced(query)
-	dsr.buildQuery(Vin.GetArgumentString(query), Vin)
+	dsr.buildQuery(Vin.GetArgumentString(query))
 }
 
 func (dsr *DehashedSearchRequest) AddLicensePlateQuery(query string) {
 	query = enquoteSpaced(query)
-	dsr.buildQuery(LicensePlate.GetArgumentString(query), LicensePlate)
+	dsr.buildQuery(LicensePlate.GetArgumentString(query))
 }
 
 func (dsr *DehashedSearchRequest) AddAddressQuery(query string) {
 	query = enquoteSpaced(query)
-	dsr.buildQuery(Address.GetArgumentString(query), Address)
+	dsr.buildQuery(Address.GetArgumentString(query))
 }
 
 func (dsr *DehashedSearchRequest) AddPhoneQuery(query string) {
 	query = enquoteSpaced(query)
-	dsr.buildQuery(Phone.GetArgumentString(query), Phone)
+	dsr.buildQuery(Phone.GetArgumentString(query))
 }
 
 func (dsr *DehashedSearchRequest) AddSocialQuery(query string) {
 	query = enquoteSpaced(query)
-	dsr.buildQuery(Social.GetArgumentString(query), Social)
+	dsr.buildQuery(Social.GetArgumentString(query))
 }
 
 func (dsr *DehashedSearchRequest) AddCryptoAddressQuery(query string) {
 	query = enquoteSpaced(query)
-	dsr.buildQuery(CryptoAddress.GetArgumentString(query), CryptoAddress)
+	dsr.buildQuery(CryptoAddress.GetArgumentString(query))
 }
 
 func (dsr *DehashedSearchRequest) AddHashedPasswordQuery(query string) {
 	query = strings.TrimSpace(query)
-	dsr.buildQuery(HashedPassword.GetArgumentString(query), HashedPassword)
+	dsr.buildQuery(HashedPassword.GetArgumentString(query))
 }
 
 func (dsr *DehashedSearchRequest) AddNameQuery(query string) {
 	query = enquoteSpaced(query)
-	dsr.buildQuery(Name.GetArgumentString(query), Name)
+	dsr.buildQuery(Name.GetArgumentString(query))
 }
 
 type DehashedClientV2 struct {
 	apiKey  string
 	results []sqlite.Result
+	debug   bool
 }
 
-func NewDehashedClientV2(apiKey string) *DehashedClientV2 {
-	return &DehashedClientV2{apiKey: apiKey}
+func NewDehashedClientV2(apiKey string, debug bool) *DehashedClientV2 {
+	return &DehashedClientV2{apiKey: apiKey, debug: debug}
 }
 
-func (dcv2 *DehashedClientV2) Search(searchRequest DehashedSearchRequest) (int, error) {
+func (dcv2 *DehashedClientV2) Search(searchRequest DehashedSearchRequest) (int, int, error) {
+	if dcv2.debug {
+		debug.PrintInfo("preparing search request")
+		zap.L().Info("v2_search_debug",
+			zap.String("message", "preparing search request"),
+		)
+	}
 	reqBody, _ := json.Marshal(searchRequest)
+
+	if dcv2.debug {
+		j := string(reqBody)
+		jReq := fmt.Sprintf("Request Body: %s\n", j)
+		debug.PrintJson(jReq)
+		zap.L().Info("v2_search_debug",
+			zap.String("message", jReq),
+			zap.String("body", j),
+		)
+	}
+
 	req, err := http.NewRequest("POST", "https://api.dehashed.com/v2/search", bytes.NewReader(reqBody))
 	if err != nil {
-		return -1, err
+		return -1, -1, err
 	}
+
+	if dcv2.debug {
+		debug.PrintInfo("setting headers")
+		zap.L().Info("v2_search_debug",
+			zap.String("message", "setting headers"),
+		)
+	}
+
 	req.Header.Set("Content-Type", "application/json")
 	req.Header.Set("Dehashed-Api-Key", dcv2.apiKey)
+
+	if dcv2.debug {
+		headers := req.Header.Clone()
+		h := fmt.Sprintf("Headers: %v\n", headers)
+		debug.PrintJson(h)
+		zap.L().Info("v2_search_debug",
+			zap.String("message", h),
+			zap.String("headers", fmt.Sprintf("%v", headers)),
+		)
+
+		debug.PrintInfo("performing request")
+		zap.L().Info("v2_search_debug",
+			zap.String("message", "performing request"),
+		)
+	}
+
 	res, err := http.DefaultClient.Do(req)
 	if res != nil {
 		defer res.Body.Close()
 	}
 	if err != nil {
+		if dcv2.debug {
+			debug.PrintInfo("failed to perform request")
+			debug.PrintError(err)
+		}
 		zap.L().Error("v2_search",
 			zap.String("message", "failed to perform request"),
 			zap.Error(err),
 		)
-		return -1, err
+		return -1, -1, err
 	}
 	if res == nil {
+		if dcv2.debug {
+			debug.PrintInfo("response was nil")
+		}
 		zap.L().Error("v2_search",
 			zap.String("message", "response was nil"),
 		)
-		return -1, errors.New("response was nil")
-	}
-
-	// Check for HTTP status code errors
-	if res.StatusCode != 200 {
-		dhErr := GetDehashedError(res.StatusCode)
-		fmt.Printf("[%d] API Error message: %s\n", res.StatusCode, dhErr.Error())
-		zap.L().Error("v2_search",
-			zap.String("message", "received error status code"),
-			zap.Int("status_code", res.StatusCode),
-			zap.String("error", dhErr.Error()),
-		)
-		return -1, &dhErr
+		return -1, -1, errors.New("response was nil")
 	}
 
 	b, err := io.ReadAll(res.Body)
@@ -182,21 +239,50 @@ func (dcv2 *DehashedClientV2) Search(searchRequest DehashedSearchRequest) (int,
 			zap.String("message", "failed to read response body"),
 			zap.Error(err),
 		)
-		return -1, err
+		return -1, -1, err
+	}
+
+	// Check for HTTP status code errors
+	if res.StatusCode != 200 {
+		if dcv2.debug {
+			debug.PrintInfo("received error status code")
+			debug.PrintJson(fmt.Sprintf("Status Code: %d\n", res.StatusCode))
+			debug.PrintJson(fmt.Sprintf("Body: %s\n", string(b[:])))
+		}
+
+		dhErr := GetDehashedError(res.StatusCode)
+		zap.L().Error("v2_search",
+			zap.String("message", "received error status code"),
+			zap.Int("status_code", res.StatusCode),
+			zap.String("error", dhErr.Error()),
+			zap.String("body_error", string(b)),
+		)
+		return -1, -1, &dhErr
 	}
 
 	var responseResults sqlite.DehashedResponse
 	err = json.Unmarshal(b, &responseResults)
 	if err != nil {
+		if dcv2.debug {
+			debug.PrintInfo("failed to unmarshal response body")
+			debug.PrintError(err)
+		}
 		zap.L().Error("v2_search",
 			zap.String("message", "failed to unmarshal response body"),
 			zap.Error(err),
 		)
-		return -1, err
+		return -1, -1, err
+	}
+
+	if dcv2.debug {
+		debug.PrintInfo("appending results")
+		debug.PrintJson(fmt.Sprintf("Total Results: %d\n", responseResults.TotalResults))
+		debug.PrintJson(fmt.Sprintf("Balance: %d\n", responseResults.Balance))
+		debug.PrintJson(fmt.Sprintf("Entries: %d\n", len(responseResults.Entries)))
 	}
 
 	dcv2.results = append(dcv2.results, responseResults.Entries...)
-	return responseResults.TotalResults, nil
+	return responseResults.TotalResults, responseResults.Balance, nil
 }
 
 func (dcv2 *DehashedClientV2) GetResults() sqlite.DehashedResults {

internal/dehashed/dehashed.go

@@ -1,6 +1,7 @@
-package query
+package dehashed
 
 import (
+	"dehasher/internal/debug"
 	"dehasher/internal/export"
 	"dehasher/internal/sqlite"
 	"encoding/json"
@@ -13,6 +14,8 @@ import (
 type Dehasher struct {
 	options  sqlite.QueryOptions
 	nextPage int
+	debug    bool
+	balance  int
 	request  *DehashedSearchRequest
 	client   *DehashedClientV2
 }
@@ -22,19 +25,24 @@ func NewDehasher(options *sqlite.QueryOptions) *Dehasher {
 	dh := &Dehasher{
 		options:  *options,
 		nextPage: options.StartingPage + 1,
+		debug:    options.Debug,
+		balance:  0,
 	}
 	dh.setQueries()
-	dh.request = NewDehashedSearchRequest(dh.options.StartingPage, dh.options.MaxRecords, dh.options.WildcardMatch, dh.options.RegexMatch, false)
+	dh.request = NewDehashedSearchRequest(dh.options.StartingPage, dh.options.MaxRecords, dh.options.WildcardMatch, dh.options.RegexMatch, false, options.Debug)
 	dh.buildRequest()
 	return dh
 }
 
 // SetClientCredentials sets the client credentials for the dehasher
 func (dh *Dehasher) SetClientCredentials(key string) {
-	dh.client = NewDehashedClientV2(key)
+	dh.client = NewDehashedClientV2(key, dh.debug)
 }
 
 func (dh *Dehasher) getNextPage() int {
+	if dh.debug {
+		debug.PrintInfo(fmt.Sprintf("getting next page: %d", dh.nextPage))
+	}
 	nextPage := dh.nextPage
 	dh.nextPage += 1
 	return nextPage
@@ -44,6 +52,10 @@ func (dh *Dehasher) getNextPage() int {
 func (dh *Dehasher) setQueries() {
 	var numQueries int
 
+	if dh.debug {
+		debug.PrintInfo("setting queries")
+	}
+
 	switch {
 	case dh.options.MaxRequests == 0:
 		zap.L().Error("max requests cannot be zero")
@@ -80,6 +92,12 @@ func (dh *Dehasher) setQueries() {
 	}
 
 	dh.options.MaxRequests = numQueries
+
+	if dh.debug {
+		debug.PrintInfo(fmt.Sprintf("setting max requests: %d", numQueries))
+		debug.PrintInfo(fmt.Sprintf("setting max records: %d", dh.options.MaxRecords))
+	}
+
 	fmt.Printf("Making %d Requests for %d Records (%d Total)\n", dh.options.MaxRequests, dh.options.MaxRecords, dh.options.MaxRequests*dh.options.MaxRecords)
 }
 
@@ -88,11 +106,16 @@ func (dh *Dehasher) Start() {
 	fmt.Printf("[*] Querying Dehashed API...\n")
 	for i := 0; i < dh.options.MaxRequests; i++ {
 		fmt.Printf("   [*] Performing Request...\n")
-		count, err := dh.client.Search(*dh.request)
+		count, balance, err := dh.client.Search(*dh.request)
 		if err != nil {
+			if dh.debug {
+				debug.PrintInfo("error performing request")
+				debug.PrintError(err)
+			}
+
 			// Check if it's a DehashError
 			if dhErr, ok := err.(*DehashError); ok {
-				fmt.Printf("   [!] Dehashed API Error: %s (Code: %d)\n", dhErr.Message, dhErr.Code)
+				fmt.Printf("      [!] Dehashed API Error: %s (Code: %d)\n", dhErr.Message, dhErr.Code)
 				zap.L().Error("dehashed_api_error",
 					zap.String("message", dhErr.Message),
 					zap.Int("code", dhErr.Code),
@@ -104,9 +127,24 @@ func (dh *Dehasher) Start() {
 					zap.Error(err),
 				)
 			}
+
+			if len(dh.client.results) > 0 {
+				fmt.Printf("   [!] Partial results retrieved. Storing Results...\n")
+				err := sqlite.StoreResults(dh.client.GetResults())
+				if err != nil {
+					zap.L().Error("store_results",
+						zap.String("message", "failed to store results"),
+						zap.Error(err),
+					)
+					fmt.Printf("   [!] Error storing results: %v\n", err)
+				}
+			}
+			dh.parseResults()
 			os.Exit(-1)
 		}
 
+		dh.balance = balance
+
 		if count < dh.options.MaxRecords {
 			fmt.Printf("      [+] Retrieved %d records\n", count)
 			fmt.Printf("      [-] Not enough entries, ending queries\n")
@@ -115,6 +153,10 @@ func (dh *Dehasher) Start() {
 			fmt.Printf("      [+] Retrieved %d records\n", dh.options.MaxRecords)
 		}
 
+		if dh.options.PrintBalance {
+			fmt.Printf("      [*] Balance: %d\n", balance)
+		}
+
 		dh.request.Page = dh.getNextPage()
 	}
 

internal/dehashed/errors.go

@@ -1,4 +1,4 @@
-package query
+package dehashed
 
 type DehashError struct {
 	Message string

internal/export/export.go

@@ -11,6 +11,7 @@ import (
 	"io/ioutil"
 	"os"
 	"strings"
+	"time"
 )
 
 func WriteCredsToFile(creds []sqlite.Creds, outputFile string, fileType files.FileType) error {
@@ -131,3 +132,121 @@ func WriteQueryResultsToFile(results []map[string]interface{}, outputFile string
 	filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String())
 	return os.WriteFile(filePath, data, 0644)
 }
+
+func WriteWhoIsHistoryToFile(results []sqlite.HistoryRecord, outputFile string, fileType files.FileType) error {
+	var data []byte
+	var err error
+
+	switch fileType {
+	case files.JSON:
+		data, err = json.MarshalIndent(results, "", "  ")
+	case files.XML:
+		data, err = xml.MarshalIndent(results, "", "  ")
+	case files.YAML:
+		data, err = yaml.Marshal(results)
+	case files.TEXT:
+		var outStrings []string
+		for _, r := range results {
+			outStrings = append(outStrings, r.String()+"\n\n")
+		}
+		data = []byte(strings.Join(outStrings, ""))
+	default:
+		return errors.New("unsupported file type")
+	}
+
+	if err != nil {
+		return err
+	}
+
+	filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String())
+	return os.WriteFile(filePath, data, 0644)
+}
+
+func WriteWhoIsRecordToFile(record sqlite.WhoisRecord, outputFile string, fileType files.FileType) error {
+	var data []byte
+	var err error
+
+	switch fileType {
+	case files.JSON:
+		data, err = json.MarshalIndent(record, "", "  ")
+	case files.XML:
+		data, err = xml.MarshalIndent(record, "", "  ")
+	case files.YAML:
+		data, err = yaml.Marshal(record)
+	case files.TEXT:
+		data = []byte(record.String())
+	default:
+		return errors.New("unsupported file type")
+	}
+
+	if err != nil {
+		return err
+	}
+
+	filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String())
+	return os.WriteFile(filePath, data, 0644)
+}
+
+func WriteSubdomainsToFile(records []sqlite.SubdomainRecord, outputFile string, fileType files.FileType) error {
+	var data []byte
+	var err error
+
+	switch fileType {
+	case files.JSON:
+		data, err = json.MarshalIndent(records, "", "  ")
+	case files.XML:
+		data, err = xml.MarshalIndent(records, "", "  ")
+	case files.YAML:
+		data, err = yaml.Marshal(records)
+	case files.TEXT:
+		var outStrings []string
+		for _, r := range records {
+			out := fmt.Sprintf(
+				"Domain: %s\nFirst Seen: %s\nLast Seen: %s\n\n",
+				r.Domain, time.Unix(r.FirstSeen, 0).String(), time.Unix(r.LastSeen, 0).String())
+			outStrings = append(outStrings, out)
+		}
+		data = []byte(strings.Join(outStrings, ""))
+	default:
+		return errors.New("unsupported file type")
+	}
+
+	if err != nil {
+		return err
+	}
+
+	filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String())
+	return os.WriteFile(filePath, data, 0644)
+}
+
+func WriteIPLookupToFile(records []sqlite.LookupResult, outputFile string, fileType files.FileType) error {
+	var data []byte
+	var err error
+
+	switch fileType {
+	case files.JSON:
+		data, err = json.MarshalIndent(records, "", "  ")
+	case files.XML:
+		data, err = xml.MarshalIndent(records, "", "  ")
+	case files.YAML:
+		data, err = yaml.Marshal(records)
+	case files.TEXT:
+		var outStrings []string
+		for _, r := range records {
+			out := fmt.Sprintf(
+				"Name: %s\nSearch Term: %s\nFirst Seen: %s\nLast Visit: %s\nType: %s\n\n",
+				r.Name, r.SearchTerm, time.Unix(r.FirstSeen, 0).String(), time.Unix(r.LastVisit, 0).String(), r.Type)
+			outStrings = append(outStrings, out)
+		}
+		data = []byte(strings.Join(outStrings, ""))
+	default:
+		return errors.New("unsupported file type")
+	}
+
+	if err != nil {
+		return err
+	}
+
+	filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String())
+	return os.WriteFile(filePath, data, 0644)
+}

internal/files/filetype.go

@@ -7,6 +7,7 @@ const (
 	XML
 	YAML
 	TEXT
+	UNKNOWN
 )
 
 func GetFileType(filetype string) FileType {

internal/sqlite/db.go

@@ -176,7 +176,7 @@ func QueryRuns(limit, lastXRuns int, startDate, endDate time.Time, containsQuery
 
 	// Apply query filter if provided
 	if containsQuery != "" {
-		// Search in all query fields
+		// SearchTerm in all query fields
 		query = query.Where(
 			"username_query LIKE ? OR "+
 				"email_query LIKE ? OR "+
@@ -238,7 +238,7 @@ func GetRunsCount(lastXRuns int, startDate, endDate time.Time, containsQuery str
 
 	// Apply query filter if provided
 	if containsQuery != "" {
-		// Search in all query fields
+		// SearchTerm in all query fields
 		query = query.Where(
 			"username_query LIKE ? OR "+
 				"email_query LIKE ? OR "+

internal/sqlite/gorm.go

@@ -51,7 +51,7 @@ func InitDB(dbPath string) (*gorm.DB, error) {
 	}
 
 	// Auto migrate your models
-	err = db.AutoMigrate(&Result{}, &Creds{}, QueryOptions{}, Creds{}, WhoisRecord{}, SubdomainRecord{}, HistoryRecord{})
+	err = db.AutoMigrate(&Result{}, &Creds{}, &QueryOptions{}, &Creds{}, &WhoisRecord{}, &SubdomainRecord{}, &HistoryRecord{}, &LookupResult{})
 	if err != nil {
 		zap.L().Error("Failed to migrate database", zap.Error(err))
 		return nil, fmt.Errorf("failed to migrate database: %w", err)
@@ -163,7 +163,7 @@ func StoreWhoisRecord(whoisRecord WhoisRecord) error {
 	return nil
 }
 
-func StoreSubdomainRecord(subdomainRecords []SubdomainRecord) error {
+func StoreSubdomainRecords(subdomainRecords []SubdomainRecord) error {
 	if len(subdomainRecords) == 0 {
 		return nil
 	}
@@ -224,3 +224,34 @@ func StoreHistoryRecord(historyRecords []HistoryRecord) error {
 
 	return lastErr
 }
+
+func StoreIPLookup(ipLookup []LookupResult) error {
+	if len(ipLookup) == 0 {
+		return nil
+	}
+
+	zap.L().Info("Storing IP lookup records", zap.Int("count", len(ipLookup)))
+	db := GetDB()
+
+	// Use batch insert with conflict handling
+	const batchSize = 100
+	var lastErr error
+
+	for i := 0; i < len(ipLookup); i += batchSize {
+		end := i + batchSize
+		if end > len(ipLookup) {
+			end = len(ipLookup)
+		}
+
+		batch := ipLookup[i:end]
+		// Use Clauses with OnConflict DoNothing to skip conflicts
+		err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
+		if err != nil {
+			zap.L().Warn("Error storing some IP lookup records", zap.Error(err))
+			lastErr = err
+			// Continue with next batch despite error
+		}
+	}
+
+	return lastErr
+}

internal/sqlite/structs.go

@@ -67,13 +67,14 @@ type QueryOptions struct {
 	CryptoAddressQuery string         `json:"crypto_address_query"`
 	PrintBalance       bool           `json:"print_balance"`
 	CredsOnly          bool           `json:"creds_only"`
+	Debug              bool           `json:"debug"`
 }
 
 func (QueryOptions) TableName() string {
 	return "query_options"
 }
 
-func NewQueryOptions(maxRecords, maxRequests, startingPage int, outputFormat, outputFile, usernameQuery, emailQuery, ipQuery, passQuery, hashQuery, nameQuery, domainQuery, vinQuery, licensePlateQuery, addressQuery, phoneQuery, socialQuery, cryptoAddressQuery string, regexMatch, wildcardMatch, printBalance, credsOnly bool) *QueryOptions {
+func NewQueryOptions(maxRecords, maxRequests, startingPage int, outputFormat, outputFile, usernameQuery, emailQuery, ipQuery, passQuery, hashQuery, nameQuery, domainQuery, vinQuery, licensePlateQuery, addressQuery, phoneQuery, socialQuery, cryptoAddressQuery string, regexMatch, wildcardMatch, printBalance, credsOnly, debug bool) *QueryOptions {
 	return &QueryOptions{
 		MaxRecords:         maxRecords,
 		MaxRequests:        maxRequests,
@@ -97,14 +98,15 @@ func NewQueryOptions(maxRecords, maxRequests, startingPage int, outputFormat, ou
 		PhoneQuery:         phoneQuery,
 		SocialQuery:        socialQuery,
 		CryptoAddressQuery: cryptoAddressQuery,
+		Debug:              debug,
 	}
 }
 
 type Creds struct {
 	gorm.Model
-	Email    string `json:"email" yaml:"email" xml:"email"`
-	Username string `json:"username" yaml:"username" xml:"username"`
-	Password string `json:"password" yaml:"password" xml:"password"`
+	Email    string `json:"email" yaml:"email" xml:"email" gorm:"uniqueIndex:idx_email_username_password"`
+	Username string `json:"username" yaml:"username" xml:"username" gorm:"uniqueIndex:idx_email_username_password"`
+	Password string `json:"password" yaml:"password" xml:"password" gorm:"uniqueIndex:idx_email_username_password"`
 }
 
 func (Creds) TableName() string {

internal/sqlite/whois.go

@@ -1,6 +1,10 @@
 package sqlite
 
-import "gorm.io/gorm"
+import (
+	"fmt"
+	"gorm.io/gorm"
+	"strings"
+)
 
 type WhoIsLookupResult struct {
 	RemainingCredits int  `json:"remaining_credits"`
@@ -17,7 +21,7 @@ type WhoisRecord struct {
 	ContactEmail          string       `json:"contactEmail"`
 	CreatedDate           string       `json:"createdDate"`
 	CreatedDateNormalized string       `json:"createdDateNormalized"`
-	DomainName            string       `json:"domainName"`
+	DomainName            string       `json:"domainName" gorm:"unique"`
 	DomainNameExt         string       `json:"domainNameExt"`
 	EstimatedDomainAge    int          `json:"estimatedDomainAge"`
 	ExpiresDate           string       `json:"expiresDate"`
@@ -38,6 +42,142 @@ type WhoisRecord struct {
 	UpdatedDateNormalized string       `json:"updatedDateNormalized"`
 }
 
+func (w WhoisRecord) String() string {
+	var sb strings.Builder
+
+	// Main domain information
+	sb.WriteString(fmt.Sprintf("Domain Name: %s\n", w.DomainName))
+	sb.WriteString(fmt.Sprintf("Domain Name Ext: %s\n", w.DomainNameExt))
+	sb.WriteString(fmt.Sprintf("Registrar Name: %s\n", w.RegistrarName))
+	sb.WriteString(fmt.Sprintf("Registrar IANA ID: %s\n", w.RegistrarIANAID))
+	sb.WriteString(fmt.Sprintf("Contact Email: %s\n", w.ContactEmail))
+	sb.WriteString(fmt.Sprintf("Estimated Domain Age: %d days\n", w.EstimatedDomainAge))
+
+	// Dates
+	sb.WriteString(fmt.Sprintf("Created Date: %s (Normalized: %s)\n", w.CreatedDate, w.CreatedDateNormalized))
+	sb.WriteString(fmt.Sprintf("Updated Date: %s (Normalized: %s)\n", w.UpdatedDate, w.UpdatedDateNormalized))
+	sb.WriteString(fmt.Sprintf("Expires Date: %s (Normalized: %s)\n", w.ExpiresDate, w.ExpiresDateNormalized))
+
+	// Status
+	sb.WriteString(fmt.Sprintf("Status: %s\n", w.Status))
+
+	// Parse code
+	sb.WriteString(fmt.Sprintf("Parse Code: %d\n", w.ParseCode))
+
+	// Audit information
+	sb.WriteString("\nAudit Information:\n")
+	sb.WriteString(fmt.Sprintf("  Created Date: %s\n", w.Audit.CreatedDate))
+	sb.WriteString(fmt.Sprintf("  Updated Date: %s\n", w.Audit.UpdatedDate))
+
+	// Name servers
+	sb.WriteString("\nName Servers:\n")
+	if len(w.NameServers.HostNames) > 0 {
+		for i, ns := range w.NameServers.HostNames {
+			ip := ""
+			if i < len(w.NameServers.IPs) {
+				ip = fmt.Sprintf(" (%s)", w.NameServers.IPs[i])
+			}
+			sb.WriteString(fmt.Sprintf("  %d. %s%s\n", i+1, ns, ip))
+		}
+	} else {
+		sb.WriteString("  None listed\n")
+	}
+
+	if w.NameServers.RawText != "" {
+		sb.WriteString(fmt.Sprintf("  Raw Text: %s\n", w.NameServers.RawText))
+	}
+
+	// Contact information
+	sb.WriteString("\nRegistrant Contact:\n")
+	formatWhoisContact(&sb, w.Registrant, "  ")
+
+	sb.WriteString("\nTechnical Contact:\n")
+	formatWhoisContact(&sb, w.TechnicalContact, "  ")
+
+	// Registry Data
+	sb.WriteString("\nRegistry Data:\n")
+	if w.RegistryData.DomainName != "" {
+		sb.WriteString(fmt.Sprintf("  Domain Name: %s\n", w.RegistryData.DomainName))
+		sb.WriteString(fmt.Sprintf("  Registrar Name: %s\n", w.RegistryData.RegistrarName))
+		sb.WriteString(fmt.Sprintf("  Registrar IANA ID: %s\n", w.RegistryData.RegistrarIANAID))
+		sb.WriteString(fmt.Sprintf("  Whois Server: %s\n", w.RegistryData.WhoisServer))
+		sb.WriteString(fmt.Sprintf("  Status: %s\n", w.RegistryData.Status))
+
+		// Registry dates
+		sb.WriteString(fmt.Sprintf("  Created Date: %s (Normalized: %s)\n",
+			w.RegistryData.CreatedDate, w.RegistryData.CreatedDateNormalized))
+		sb.WriteString(fmt.Sprintf("  Updated Date: %s (Normalized: %s)\n",
+			w.RegistryData.UpdatedDate, w.RegistryData.UpdatedDateNormalized))
+		sb.WriteString(fmt.Sprintf("  Expires Date: %s (Normalized: %s)\n",
+			w.RegistryData.ExpiresDate, w.RegistryData.ExpiresDateNormalized))
+
+		// Registry nameservers
+		sb.WriteString("  Name Servers:\n")
+		if len(w.RegistryData.NameServers.HostNames) > 0 {
+			for i, ns := range w.RegistryData.NameServers.HostNames {
+				ip := ""
+				if i < len(w.RegistryData.NameServers.IPs) {
+					ip = fmt.Sprintf(" (%s)", w.RegistryData.NameServers.IPs[i])
+				}
+				sb.WriteString(fmt.Sprintf("    %d. %s%s\n", i+1, ns, ip))
+			}
+		} else {
+			sb.WriteString("    None listed\n")
+		}
+
+		// Registry audit
+		sb.WriteString("  Audit Information:\n")
+		sb.WriteString(fmt.Sprintf("    Created Date: %s\n", w.RegistryData.Audit.CreatedDate))
+		sb.WriteString(fmt.Sprintf("    Updated Date: %s\n", w.RegistryData.Audit.UpdatedDate))
+	} else {
+		sb.WriteString("  No registry data available\n")
+	}
+
+	// Header and footer
+	if w.Header != "" {
+		headerPreview := w.Header
+		if len(headerPreview) > 100 {
+			headerPreview = headerPreview[:100] + "... [truncated]"
+		}
+		sb.WriteString("\nHeader:\n")
+		sb.WriteString(headerPreview)
+		sb.WriteString("\n")
+	}
+
+	if w.Footer != "" {
+		footerPreview := w.Footer
+		if len(footerPreview) > 100 {
+			footerPreview = footerPreview[:100] + "... [truncated]"
+		}
+		sb.WriteString("\nFooter:\n")
+		sb.WriteString(footerPreview)
+		sb.WriteString("\n")
+	}
+
+	// Raw text (truncated if too long)
+	if w.RawText != "" {
+		rawTextPreview := w.RawText
+		if len(rawTextPreview) > 500 {
+			rawTextPreview = rawTextPreview[:500] + "... [truncated]"
+		}
+		sb.WriteString("\nRaw Text:\n")
+		sb.WriteString(rawTextPreview)
+		sb.WriteString("\n")
+	}
+
+	if w.StrippedText != "" {
+		strippedTextPreview := w.StrippedText
+		if len(strippedTextPreview) > 500 {
+			strippedTextPreview = strippedTextPreview[:500] + "... [truncated]"
+		}
+		sb.WriteString("\nStripped Text:\n")
+		sb.WriteString(strippedTextPreview)
+		sb.WriteString("\n")
+	}
+
+	return sb.String()
+}
+
 func (WhoisRecord) TableName() string {
 	return "whois"
 }
@@ -104,7 +244,7 @@ type ScanResult struct {
 
 type SubdomainRecord struct {
 	gorm.Model
-	Domain    string `json:"domain"`
+	Domain    string `json:"domain" gorm:"unique"`
 	FirstSeen int64  `json:"firstSeen"`
 	LastSeen  int64  `json:"lastSeen"`
 }
@@ -131,7 +271,7 @@ type HistoryRecord struct {
 	CleanText             string      `json:"cleanText"`
 	CreatedDateISO8601    string      `json:"createdDateISO8601"`
 	CreatedDateRaw        string      `json:"createdDateRaw"`
-	DomainName            string      `json:"domainName"`
+	DomainName            string      `json:"domainName" gorm:"unique"`
 	DomainType            string      `json:"domainType"`
 	ExpiresDateISO8601    string      `json:"expiresDateISO8601"`
 	ExpiresDateRaw        string      `json:"expiresDateRaw"`
@@ -147,6 +287,131 @@ type HistoryRecord struct {
 	ZoneContact           ContactInfo `json:"zoneContact" gorm:"serializer:json"`
 }
 
+func (h HistoryRecord) String() string {
+	var sb strings.Builder
+
+	// Main domain information
+	sb.WriteString(fmt.Sprintf("Domain Name: %s\n", h.DomainName))
+	sb.WriteString(fmt.Sprintf("Domain Type: %s\n", h.DomainType))
+	sb.WriteString(fmt.Sprintf("Registrar Name: %s\n", h.RegistrarName))
+	sb.WriteString(fmt.Sprintf("Whois Server: %s\n", h.WhoisServer))
+
+	// Dates
+	sb.WriteString(fmt.Sprintf("Created Date: %s (Raw: %s)\n", h.CreatedDateISO8601, h.CreatedDateRaw))
+	sb.WriteString(fmt.Sprintf("Updated Date: %s (Raw: %s)\n", h.UpdatedDateISO8601, h.UpdatedDateRaw))
+	sb.WriteString(fmt.Sprintf("Expires Date: %s (Raw: %s)\n", h.ExpiresDateISO8601, h.ExpiresDateRaw))
+
+	// Status
+	sb.WriteString("Status: ")
+	if len(h.Status) > 0 {
+		sb.WriteString(strings.Join(h.Status, ", "))
+	} else {
+		sb.WriteString("N/A")
+	}
+	sb.WriteString("\n")
+
+	// Audit information
+	sb.WriteString("\nAudit Information:\n")
+	sb.WriteString(fmt.Sprintf("  Created Date: %s\n", h.Audit.CreatedDate))
+	sb.WriteString(fmt.Sprintf("  Updated Date: %s\n", h.Audit.UpdatedDate))
+
+	// Name servers
+	sb.WriteString("\nName Servers:\n")
+	if len(h.NameServers) > 0 {
+		for i, ns := range h.NameServers {
+			sb.WriteString(fmt.Sprintf("  %d. %s\n", i+1, ns))
+		}
+	} else {
+		sb.WriteString("  None listed\n")
+	}
+
+	// Contact information
+	sb.WriteString("\nRegistrant Contact:\n")
+	formatContact(&sb, h.RegistrantContact, "  ")
+
+	sb.WriteString("\nAdministrative Contact:\n")
+	formatContact(&sb, h.AdministrativeContact, "  ")
+
+	sb.WriteString("\nTechnical Contact:\n")
+	formatContact(&sb, h.TechnicalContact, "  ")
+
+	sb.WriteString("\nBilling Contact:\n")
+	formatContact(&sb, h.BillingContact, "  ")
+
+	sb.WriteString("\nZone Contact:\n")
+	formatContact(&sb, h.ZoneContact, "  ")
+
+	// Raw text (truncated if too long)
+	if len(h.RawText) > 0 {
+		rawTextPreview := h.RawText
+		if len(rawTextPreview) > 500 {
+			rawTextPreview = rawTextPreview[:500] + "... [truncated]"
+		}
+		sb.WriteString("\nRaw Text:\n")
+		sb.WriteString(rawTextPreview)
+		sb.WriteString("\n")
+	}
+
+	if len(h.CleanText) > 0 {
+		cleanTextPreview := h.CleanText
+		if len(cleanTextPreview) > 500 {
+			cleanTextPreview = cleanTextPreview[:500] + "... [truncated]"
+		}
+		sb.WriteString("\nClean Text:\n")
+		sb.WriteString(cleanTextPreview)
+		sb.WriteString("\n")
+	}
+
+	return sb.String()
+}
+
+// Helper function to format contact information
+func formatContact(sb *strings.Builder, contact ContactInfo, indent string) {
+	if contact.Name == "" && contact.Organization == "" && contact.Email == "" {
+		sb.WriteString(indent + "No contact information available\n")
+		return
+	}
+
+	if contact.Name != "" {
+		sb.WriteString(indent + "Name: " + contact.Name + "\n")
+	}
+	if contact.Organization != "" {
+		sb.WriteString(indent + "Organization: " + contact.Organization + "\n")
+	}
+	if contact.Email != "" {
+		sb.WriteString(indent + "Email: " + contact.Email + "\n")
+	}
+	if contact.Street != "" {
+		sb.WriteString(indent + "Street: " + contact.Street + "\n")
+	}
+	if contact.City != "" {
+		sb.WriteString(indent + "City: " + contact.City + "\n")
+	}
+	if contact.State != "" {
+		sb.WriteString(indent + "State: " + contact.State + "\n")
+	}
+	if contact.PostalCode != "" {
+		sb.WriteString(indent + "Postal Code: " + contact.PostalCode + "\n")
+	}
+	if contact.Country != "" {
+		sb.WriteString(indent + "Country: " + contact.Country + "\n")
+	}
+	if contact.Telephone != "" {
+		phone := contact.Telephone
+		if contact.TelephoneExt != "" {
+			phone += " ext. " + contact.TelephoneExt
+		}
+		sb.WriteString(indent + "Telephone: " + phone + "\n")
+	}
+	if contact.Fax != "" {
+		fax := contact.Fax
+		if contact.FaxExt != "" {
+			fax += " ext. " + contact.FaxExt
+		}
+		sb.WriteString(indent + "Fax: " + fax + "\n")
+	}
+}
+
 func (HistoryRecord) TableName() string {
 	return "history"
 }
@@ -170,3 +435,80 @@ type ContactInfo struct {
 type WhoIsCredits struct {
 	WhoisCredits int `json:"whois_credits"`
 }
+
+type WhoIsIPLookup struct {
+	RemainingCredits int    `json:"remaining_credits"`
+	Data             IPData `json:"data"`
+}
+
+type WhoIsMXLookup struct {
+	RemainingCredits int    `json:"remaining_credits"`
+	Data             IPData `json:"data"`
+}
+
+type WhoIsNSLookup struct {
+	RemainingCredits int    `json:"remaining_credits"`
+	Data             IPData `json:"data"`
+}
+
+type IPData struct {
+	CurrentPage string         `json:"current_page"`
+	Result      []LookupResult `json:"result"`
+	Size        int            `json:"size"`
+}
+
+type LookupResult struct {
+	gorm.Model
+	FirstSeen  int64  `json:"first_seen"`
+	LastVisit  int64  `json:"last_visit"`
+	Name       string `json:"name" gorm:"unique"`
+	SearchTerm string `json:"search_term,omitempty"` // For storing the IP address this domain is associated with
+	Type       string `json:"type,omitempty"`        // For storing the MX address this domain is associated with
+}
+
+func (LookupResult) TableName() string {
+	return "lookup"
+}
+
+// Helper function to format contact information for WhoisRecord
+func formatWhoisContact(sb *strings.Builder, contact Contact, indent string) {
+	if contact.Name == "" && contact.Organization == "" {
+		sb.WriteString(indent + "No contact information available\n")
+		return
+	}
+
+	if contact.Name != "" {
+		sb.WriteString(indent + "Name: " + contact.Name + "\n")
+	}
+	if contact.Organization != "" {
+		sb.WriteString(indent + "Organization: " + contact.Organization + "\n")
+	}
+	if contact.Street1 != "" {
+		sb.WriteString(indent + "Street: " + contact.Street1 + "\n")
+	}
+	if contact.City != "" {
+		sb.WriteString(indent + "City: " + contact.City + "\n")
+	}
+	if contact.State != "" {
+		sb.WriteString(indent + "State: " + contact.State + "\n")
+	}
+	if contact.PostalCode != "" {
+		sb.WriteString(indent + "Postal Code: " + contact.PostalCode + "\n")
+	}
+	if contact.Country != "" {
+		sb.WriteString(indent + "Country: " + contact.Country + "\n")
+	}
+	if contact.CountryCode != "" {
+		sb.WriteString(indent + "Country Code: " + contact.CountryCode + "\n")
+	}
+	if contact.Telephone != "" {
+		sb.WriteString(indent + "Telephone: " + contact.Telephone + "\n")
+	}
+	if contact.RawText != "" {
+		rawTextPreview := contact.RawText
+		if len(rawTextPreview) > 100 {
+			rawTextPreview = rawTextPreview[:100] + "... [truncated]"
+		}
+		sb.WriteString(indent + "Raw Text: " + rawTextPreview + "\n")
+	}
+}