KrakenTech/CrowsNest
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #4 from Kraken-OffSec/crows-nest-rebrand

Browse Source 
Crows nest rebrand
This commit is contained in:
KrakenTech
2025-05-17 10:28:53 -04:00
committed by GitHub
parent 174071e472 edbf9d57dc
commit 89d8872407
30 changed files with 725 additions and 880 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+4 -4
View File
@@ -4,7 +4,7 @@
GO=go
# Binary name
BINARY_NAME=dehasher
BINARY_NAME=crowsnest
# Build directory
BUILD_DIR=build/bin
@@ -16,7 +16,7 @@ PLATFORMS=linux darwin windows
ARCHS=amd64 arm64
# Version info from git tag or default
VERSION=$(shell git describe --tags 2>/dev/null || echo "v1.2.1")
VERSION=$(shell git describe --tags 2>/dev/null || echo "v1.3.1")
.PHONY: all clean build build-all
@@ -30,14 +30,14 @@ clean:
# Build for current platform
build:
$(GO) build -o $(BUILD_DIR)/$(BINARY_NAME) -ldflags "-X main.version=$(VERSION)" dehasher.go
$(GO) build -o $(BUILD_DIR)/$(BINARY_NAME) -ldflags "-X main.version=$(VERSION)" crowsnest.go
# Build for all platforms
build-all: clean
@for platform in $(PLATFORMS); do \
for arch in $(ARCHS); do \
echo "Building for $$platform/$$arch..."; \
GOOS=$$platform GOARCH=$$arch $(GO) build -o $(BUILD_DIR)/$(BINARY_NAME)-$$platform-$$arch -ldflags "-X main.version=$(VERSION)" dehasher.go; \
GOOS=$$platform GOARCH=$$arch $(GO) build -o $(BUILD_DIR)/$(BINARY_NAME)-$$platform-$$arch -ldflags "-X main.version=$(VERSION)" crowsnest.go; \
if [ "$$platform" = "windows" ]; then \
mv $(BUILD_DIR)/$(BINARY_NAME)-$$platform-$$arch $(BUILD_DIR)/$(BINARY_NAME)-$$platform-$$arch.exe; \
fi; \
README.md
+6 -6
View File
@@ -221,7 +221,7 @@ dehasher set-hunter <redacted>
### Domain Search
Dehasher can perform a domain search for a given domain.
This provides a list of all emails that match the given query.
This provides information about company including a description, social media information and any technologies in use.
![Alt text](.img/hunter_domain_search.png "Hunter.io Domain Search")
```bash
# Perform a Hunter.io domain search for example.com
@@ -230,7 +230,7 @@ dehasher hunter -d example.com -D
### Email Finder
Dehasher can perform an email finder search for a given domain, first name, and last name.
This provides a list of all emails that match the given query.
This provides information about a user including a confidence score, and any social media accounts linked to a first name, last name and email.
![Alt text](.img/hunter_email_finder.png "Hunter.io Email Finder")
```bash
# Perform a Hunter.io email finder search for example.com
@@ -239,7 +239,7 @@ dehasher hunter -d example.com -F John -L Doe -E
### Email Verification
Dehasher can perform an email verification search for a given email.
This provides a list of all emails that match the given query.
This provides a verification and score of a given email address.
![Alt text](.img/email_verification.png "Hunter.io Email Verification")
```bash
# Perform a Hunter.io email verification search for example@target.com
@@ -248,7 +248,7 @@ dehasher hunter -e example@target.com -V
### Company Enrichment
Dehasher can perform a company enrichment search for a given domain.
This provides a list of all emails that match the given query.
This provides information about a company given its domain.
![Alt text](.img/company_enrichment.png "Hunter.io Company Enrichment")
```bash
# Perform a Hunter.io company enrichment search for example.com
@@ -257,7 +257,7 @@ dehasher hunter -d example.com -C
### Person Enrichment
Dehasher can perform a person enrichment search for a given email.
This provides a list of all emails that match the given query.
This provides information about a user given an email address..
![Alt text](.img/person_enrichment.png "Hunter.io Person Enrichment")
```bash
# Perform a Hunter.io person enrichment search for example@target.com
@@ -266,7 +266,7 @@ dehasher hunter -e example@target.com -P
### Combined Enrichment
Dehasher can perform a combined enrichment search for a given email.
This provides a list of all emails that match the given query.
This is a combination of the company and person enrichments given an email address.
![Alt text](.img/combined_enrichment_1.png "Hunter.io Combined Enrichment")
![Alt text](.img/combined_enrichment_2.png "Hunter.io Combined Enrichment")
```bash
cmd/api.go → cmd/dehashed.go
+31 -31
View File
@@ -1,10 +1,10 @@
package cmd
import (
"dehasher/internal/badger"
"dehasher/internal/debug"
"dehasher/internal/dehashed"
"dehasher/internal/sqlite"
"crowsnest/internal/badger"
"crowsnest/internal/debug"
"crowsnest/internal/dehashed"
"crowsnest/internal/sqlite"
"fmt"
"github.com/spf13/cobra"
"go.uber.org/zap"
@@ -12,34 +12,34 @@ import (
func init() {
// Add api command to root command
rootCmd.AddCommand(apiCmd)
rootCmd.AddCommand(dehashedCmd)
// Add flags specific to api command
apiCmd.Flags().IntVarP(&maxRecords, "max-records", "m", 30000, "Maximum amount of records to return")
apiCmd.Flags().IntVarP(&maxRequests, "max-requests", "r", -1, "Maximum number of requests to make")
apiCmd.Flags().IntVarP(&startingPage, "starting-page", "s", 1, "Starting page for requests")
apiCmd.Flags().BoolVarP(&printBalance, "print-balance", "b", false, "Print remaining balance after requests")
apiCmd.Flags().BoolVarP(&regexMatch, "regex-match", "R", false, "Use regex matching on query fields")
apiCmd.Flags().BoolVarP(&wildcardMatch, "wildcard-match", "W", false, "Use wildcard matching on query fields (Use ? to replace a single character, and * for multiple characters)")
apiCmd.Flags().BoolVarP(&credsOnly, "creds-only", "C", false, "Return credentials only")
apiCmd.Flags().StringVarP(&outputFormat, "format", "f", "json", "Output format (json, yaml, xml, txt)")
apiCmd.Flags().StringVarP(&outputFile, "output", "o", "query", "File to output results to including extension")
apiCmd.Flags().StringVarP(&usernameQuery, "username", "U", "", "Username query")
apiCmd.Flags().StringVarP(&emailQuery, "email-query", "E", "", "HunterEmail query")
apiCmd.Flags().StringVarP(&ipQuery, "ip", "I", "", "IP address query")
apiCmd.Flags().StringVarP(&domainQuery, "domain", "D", "", "Domain query")
apiCmd.Flags().StringVarP(&passwordQuery, "password", "P", "", "Password query")
apiCmd.Flags().StringVarP(&vinQuery, "vin", "V", "", "VIN query")
apiCmd.Flags().StringVarP(&licensePlateQuery, "license", "L", "", "License plate query")
apiCmd.Flags().StringVarP(&addressQuery, "address", "A", "", "Address query")
apiCmd.Flags().StringVarP(&phoneQuery, "phone", "M", "", "Phone query")
apiCmd.Flags().StringVarP(&socialQuery, "social", "S", "", "Social query")
apiCmd.Flags().StringVarP(&cryptoCurrencyAddressQuery, "crypto", "B", "", "Crypto currency address query")
apiCmd.Flags().StringVarP(&hashQuery, "hash", "Q", "", "Hashed password query")
apiCmd.Flags().StringVarP(&nameQuery, "name", "N", "", "Name query")
dehashedCmd.Flags().IntVarP(&maxRecords, "max-records", "m", 30000, "Maximum amount of records to return")
dehashedCmd.Flags().IntVarP(&maxRequests, "max-requests", "r", -1, "Maximum number of requests to make")
dehashedCmd.Flags().IntVarP(&startingPage, "starting-page", "s", 1, "Starting page for requests")
dehashedCmd.Flags().BoolVarP(&printBalance, "print-balance", "b", false, "Print remaining balance after requests")
dehashedCmd.Flags().BoolVarP(&regexMatch, "regex-match", "R", false, "Use regex matching on query fields")
dehashedCmd.Flags().BoolVarP(&wildcardMatch, "wildcard-match", "W", false, "Use wildcard matching on query fields (Use ? to replace a single character, and * for multiple characters)")
dehashedCmd.Flags().BoolVarP(&credsOnly, "creds-only", "C", false, "Return credentials only")
dehashedCmd.Flags().StringVarP(&outputFormat, "format", "f", "json", "Output format (json, yaml, xml, txt)")
dehashedCmd.Flags().StringVarP(&outputFile, "output", "o", "query", "File to output results to including extension")
dehashedCmd.Flags().StringVarP(&usernameQuery, "username", "U", "", "Username query")
dehashedCmd.Flags().StringVarP(&emailQuery, "email-query", "E", "", "HunterEmail query")
dehashedCmd.Flags().StringVarP(&ipQuery, "ip", "I", "", "IP address query")
dehashedCmd.Flags().StringVarP(&domainQuery, "domain", "D", "", "Domain query")
dehashedCmd.Flags().StringVarP(&passwordQuery, "password", "P", "", "Password query")
dehashedCmd.Flags().StringVarP(&vinQuery, "vin", "V", "", "VIN query")
dehashedCmd.Flags().StringVarP(&licensePlateQuery, "license", "L", "", "License plate query")
dehashedCmd.Flags().StringVarP(&addressQuery, "address", "A", "", "Address query")
dehashedCmd.Flags().StringVarP(&phoneQuery, "phone", "M", "", "Phone query")
dehashedCmd.Flags().StringVarP(&socialQuery, "social", "S", "", "Social query")
dehashedCmd.Flags().StringVarP(&cryptoCurrencyAddressQuery, "crypto", "B", "", "Crypto currency address query")
dehashedCmd.Flags().StringVarP(&hashQuery, "hash", "Q", "", "Hashed password query")
dehashedCmd.Flags().StringVarP(&nameQuery, "name", "N", "", "Name query")
// Add mutually exclusive flags to wildcard match and regex match
apiCmd.MarkFlagsMutuallyExclusive("regex-match", "wildcard-match")
dehashedCmd.MarkFlagsMutuallyExclusive("regex-match", "wildcard-match")
}
var (
@@ -68,8 +68,8 @@ var (
cryptoCurrencyAddressQuery string
// Query command
apiCmd = &cobra.Command{
Use: "api",
dehashedCmd = &cobra.Command{
Use: "dehashed",
Short: "Query the Dehashed API",
Long: `Query the Dehashed API for emails, usernames, passwords, hashes, IP addresses, and names.`,
Run: func(cmd *cobra.Command, args []string) {
@@ -118,7 +118,7 @@ var (
dehasher.Start()
fmt.Println("\n[*] Completing Process")
err := sqlite.StoreQueryOptions(queryOptions)
err := sqlite.StoreDehashedQueryOptions(queryOptions)
if err != nil {
if debugGlobal {
debug.PrintInfo("failed to store query options")
cmd/export.go
+3 -3
View File
@@ -1,9 +1,9 @@
package cmd
import (
"dehasher/internal/export"
"dehasher/internal/files"
"dehasher/internal/sqlite"
"crowsnest/internal/export"
"crowsnest/internal/files"
"crowsnest/internal/sqlite"
"fmt"
"github.com/spf13/cobra"
"go.uber.org/zap"
cmd/hunter.go
+12 -12
View File
@@ -1,12 +1,12 @@
package cmd
import (
"dehasher/internal/badger"
"dehasher/internal/debug"
"dehasher/internal/export"
"dehasher/internal/files"
hunter "dehasher/internal/hunter.io"
"dehasher/internal/pretty"
"crowsnest/internal/badger"
"crowsnest/internal/debug"
"crowsnest/internal/export"
"crowsnest/internal/files"
hunter "crowsnest/internal/hunter.io"
"crowsnest/internal/pretty"
"fmt"
"github.com/spf13/cobra"
"go.uber.org/zap"
@@ -113,7 +113,7 @@ var (
// Write Hunter.io Domain Search Result to file
fmt.Printf("[*] Writing Hunter.io Domain Search Result to file: %s%s\n", hunterOutputFile, fType.Extension())
err = export.WriteHunterDomainToFile(result, hunterOutputFile, fType)
err = export.WriteIStringToFile(result, hunterOutputFile, fType)
if err != nil {
if debugGlobal {
debug.PrintInfo("failed to write hunter domain search to file")
@@ -150,7 +150,7 @@ var (
// Write Hunter.io Email Finder Result to file
fmt.Printf("[*] Writing Hunter.io Email Finder Result to file: %s%s\n", hunterOutputFile, fType.Extension())
err = export.WriteHunterEmailToFile(result, hunterOutputFile, fType)
err = export.WriteIStringToFile(result, hunterOutputFile, fType)
if err != nil {
if debugGlobal {
debug.PrintInfo("failed to write hunter email find to file")
@@ -206,7 +206,7 @@ var (
}
// Write Hunter.io Email Verification Result to file
fmt.Printf("[*] Writing Hunter.io Email Verification Result to file: %s%s\n", hunterOutputFile, fType.Extension())
err = export.WriteHunterEmailVerifyToFile(result, hunterOutputFile, fType)
err = export.WriteIStringToFile(result, hunterOutputFile, fType)
if err != nil {
if debugGlobal {
debug.PrintInfo("failed to write hunter email verification to file")
@@ -264,7 +264,7 @@ var (
// Write to file
fmt.Printf("[*] Writing Hunter.io Company Enrichment Result to file: %s%s\n", hunterOutputFile, fType.Extension())
err = export.WriteHunterCompanyEnrichmentToFile(result, hunterOutputFile, fType)
err = export.WriteIStringToFile(result, hunterOutputFile, fType)
if err != nil {
if debugGlobal {
debug.PrintInfo("failed to write hunter company enrichment to file")
@@ -302,7 +302,7 @@ var (
// Write to file
fmt.Printf("[*] Writing Hunter.io Person Enrichment Result to file: %s%s\n", hunterOutputFile, fType.Extension())
err = export.WriteHunterPersonEnrichmentToFile(result, hunterOutputFile, fType)
err = export.WriteIStringToFile(result, hunterOutputFile, fType)
if err != nil {
if debugGlobal {
debug.PrintInfo("failed to write hunter person enrichment to file")
@@ -339,7 +339,7 @@ var (
// Write to file
fmt.Printf("[*] Writing Hunter.io Combined Enrichment Result to file: %s%s\n", hunterOutputFile, fType.Extension())
err = export.WriteHunterCombinedEnrichmentToFile(result, hunterOutputFile, fType)
err = export.WriteIStringToFile(result, hunterOutputFile, fType)
if err != nil {
if debugGlobal {
debug.PrintInfo("failed to write hunter combined enrichment to file")
cmd/logs.go
+2 -2
View File
@@ -1,8 +1,8 @@
package cmd
import (
"dehasher/internal/easyTime"
"dehasher/internal/pretty"
"crowsnest/internal/easyTime"
"crowsnest/internal/pretty"
"encoding/json"
"fmt"
"github.com/spf13/cobra"
cmd/query.go
+2 -2
View File
@@ -1,8 +1,8 @@
package cmd
import (
"dehasher/internal/pretty"
"dehasher/internal/sqlite"
"crowsnest/internal/pretty"
"crowsnest/internal/sqlite"
"encoding/json"
"fmt"
"github.com/spf13/cobra"
cmd/root.go
+24 -22
View File
@@ -1,8 +1,9 @@
package cmd
import (
"dehasher/internal/badger"
"crowsnest/internal/badger"
"fmt"
"github.com/fatih/color"
"github.com/spf13/cobra"
"go.uber.org/zap"
"os"
@@ -18,28 +19,14 @@ var (
Use: "dehasher",
Short: `Dehasher is a cli tool for querying the dehashed api.`,
Long: fmt.Sprintf(
"%s\n%s",
"%s\n",
`
______ _______ _______ _______ _______ _______
( __ \ ( ____ \|\ /|( ___ )( ____ \|\ /|( ____ \( ____ )
| ( \ )| ( \/| ) ( || ( ) || ( \/| ) ( || ( \/| ( )|
| | ) || (__ | (___) || (___) || (_____ | (___) || (__ | (____)|
| | | || __) | ___ || ___ |(_____ )| ___ || __) | __)
| | ) || ( | ( ) || ( ) | ) || ( ) || ( | (\ (
| (__/ )| (____/\| ) ( || ) ( |/\____) || ) ( || (____/\| ) \ \__
(______/ (_______/|/ \||/ \|\_______)|/ \|(_______/|/ \__/
An Ar1ste1a Project
`,
`––•–√\/––√\/––•––––•–√\/––√\/––•––––•–√\/––√\/––•––√\/––•––––•–√\/––√\/––•––
Dehasher can query the query API for:
- Emails - Usernames - Password
- Hashes - IP Addresses - Names
- VINs - License Plates - Addresses
- Phones - Social Media - Crypto Currency Addresses
Dehasher supports:
- Regex Matching
- Exact Matching
––•–√\/––√\/––•––––•–√\/––√\/––•––––•–√\/––√\/––•––√\/––•––––•–√\/––√\/––•––
╔═╗┬─┐┌─┐┬ ┬┌─┐╔╗╔┌─┐┌─┐┌┬┐
║ ├┬┘│ ││││└─┐║║║├┤ └─┐ │
╚═╝┴└─└─┘└┴┘└─┘╝╚╝└─┘└─┘ ┴
Crow’s Nest OSINT Recon Suite
⚓ A KrakenTech Intelligence Tool
`,
),
Version: "v1.2.1",
@@ -69,6 +56,7 @@ func init() {
rootCmd.AddCommand(setDehashedKeyCmd)
rootCmd.AddCommand(setHunterKeyCmd)
rootCmd.AddCommand(setLocalDb)
rootCmd.AddCommand(buyMeCoffeeCmd)
}
// Command to set API key
@@ -132,6 +120,20 @@ var setLocalDb = &cobra.Command{
},
}
var buyMeCoffeeCmd = &cobra.Command{
Use: "coffee",
Short: "Support the project by buying a coffee",
Run: func(cmd *cobra.Command, args []string) {
fmt.Println(color.HiRedString(" ;)(; "))
fmt.Println(color.HiCyanString(" We Hope You Enjoy Our Product :----:"))
fmt.Println(color.HiCyanString(" C|====|"))
fmt.Println(color.HiCyanString(" | |"))
fmt.Print(color.HiGreenString(" Support the project by buying a coffee: "))
fmt.Print(color.BlueString("https://buymeacoffee.com/ehosinskiz "))
fmt.Println(color.HiCyanString("`----'"))
},
}
// Helper functions to store API credentials
func storeDehashedApiKey(key string) error {
err := badger.StoreDehashedKey(key)
cmd/whois.go
+8 -8
View File
@@ -1,12 +1,12 @@
package cmd
import (
"dehasher/internal/debug"
"dehasher/internal/export"
"dehasher/internal/files"
"dehasher/internal/pretty"
"dehasher/internal/sqlite"
"dehasher/internal/whois"
"crowsnest/internal/debug"
"crowsnest/internal/export"
"crowsnest/internal/files"
"crowsnest/internal/pretty"
"crowsnest/internal/sqlite"
"crowsnest/internal/whois"
"fmt"
"github.com/spf13/cobra"
"go.uber.org/zap"
@@ -208,7 +208,7 @@ var (
fmt.Printf("[!] Error writing WHOIS history to file: %v\n", writeErr)
}
err = sqlite.StoreHistoryRecord(historyRecords)
err = sqlite.StoreWhoisHistoryRecords(historyRecords)
if err != nil {
if debugGlobal {
debug.PrintInfo("failed to store history record")
@@ -254,7 +254,7 @@ var (
fmt.Printf("Error performing subdomain scan: %v\n", err)
} else {
fmt.Println("Subdomain Scan:")
err = sqlite.StoreSubdomainRecords(subdomains)
err = sqlite.StoreWhoisSubdomainRecords(subdomains)
if err != nil {
if debugGlobal {
debug.PrintInfo("failed to store subdomain record")
dehasher.go → crowsnest.go
+3 -3
View File
@@ -1,9 +1,9 @@
package main
import (
"dehasher/cmd"
"dehasher/internal/badger"
"dehasher/internal/sqlite"
"crowsnest/cmd"
"crowsnest/internal/badger"
"crowsnest/internal/sqlite"
"fmt"
"github.com/winking324/rzap"
"go.uber.org/zap"
go.mod
+2 -5
View File
@@ -1,4 +1,4 @@
module dehasher
module crowsnest
go 1.23.0
@@ -7,7 +7,7 @@ toolchain go1.24.3
require (
github.com/charmbracelet/lipgloss v1.1.0
github.com/dgraph-io/badger/v4 v4.7.0
github.com/olekukonko/tablewriter v1.0.5
github.com/fatih/color v1.15.0
github.com/spf13/cobra v1.9.1
github.com/winking324/rzap v0.1.0
go.uber.org/zap v1.20.0
@@ -27,7 +27,6 @@ require (
github.com/charmbracelet/x/term v0.2.1 // indirect
github.com/dgraph-io/ristretto/v2 v2.2.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/fatih/color v1.15.0 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/google/flatbuffers v25.2.10+incompatible // indirect
@@ -41,8 +40,6 @@ require (
github.com/mattn/go-runewidth v0.0.16 // indirect
github.com/mattn/go-sqlite3 v1.14.22 // indirect
github.com/muesli/termenv v0.16.0 // indirect
github.com/olekukonko/errors v0.0.0-20250405072817-4e6d85265da6 // indirect
github.com/olekukonko/ll v0.0.7 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/spf13/pflag v1.0.6 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
go.sum
-6
View File
@@ -71,12 +71,6 @@ github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
github.com/olekukonko/errors v0.0.0-20250405072817-4e6d85265da6 h1:r3FaAI0NZK3hSmtTDrBVREhKULp8oUeqLT5Eyl2mSPo=
github.com/olekukonko/errors v0.0.0-20250405072817-4e6d85265da6/go.mod h1:ppzxA5jBKcO1vIpCXQ9ZqgDh8iwODz6OXIGKU8r5m4Y=
github.com/olekukonko/ll v0.0.7 h1:K66xcUlG2qWRhPoLw/cidmbv4pDDJtZuvJGsR5QTzXo=
github.com/olekukonko/ll v0.0.7/go.mod h1:En+sEW0JNETl26+K8eZ6/W4UQ7CYSrrgg/EdIYT2H8g=
github.com/olekukonko/tablewriter v1.0.5 h1:8+uKJXxYcl29TcpfQdd0vL+l6Kul7Sk7sWolfgErDv0=
github.com/olekukonko/tablewriter v1.0.5/go.mod h1:Z22i2ywMkT9sw64nuWAUaH62kb+umiwucGaQNbFh8Bg=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
internal/dehashed/clientv2.go
+2 -2
View File
@@ -2,9 +2,9 @@ package dehashed
import (
"bytes"
"crowsnest/internal/debug"
"crowsnest/internal/sqlite"
"crypto/sha256"
"dehasher/internal/debug"
"dehasher/internal/sqlite"
"encoding/hex"
"encoding/json"
"errors"
internal/dehashed/dehashed.go
+6 -6
View File
@@ -1,9 +1,9 @@
package dehashed
import (
"dehasher/internal/debug"
"dehasher/internal/export"
"dehasher/internal/sqlite"
"crowsnest/internal/debug"
"crowsnest/internal/export"
"crowsnest/internal/sqlite"
"encoding/json"
"fmt"
"go.uber.org/zap"
@@ -130,7 +130,7 @@ func (dh *Dehasher) Start() {
if len(dh.client.results) > 0 {
fmt.Printf(" [!] Partial results retrieved. Storing Results...\n")
err := sqlite.StoreResults(dh.client.GetResults())
err := sqlite.StoreDehashedResults(dh.client.GetResults())
if err != nil {
zap.L().Error("store_results",
zap.String("message", "failed to store results"),
@@ -214,7 +214,7 @@ func (dh *Dehasher) parseResults() {
results := dh.client.GetResults()
creds := results.ExtractCredentials()
fmt.Printf("\n\t[+] Discovered %d Credentials", len(creds))
err := sqlite.StoreCreds(creds)
err := sqlite.StoreDehashedCreds(creds)
if err != nil {
zap.L().Error("store_creds",
zap.String("message", "failed to store creds"),
@@ -224,7 +224,7 @@ func (dh *Dehasher) parseResults() {
zap.L().Info("creds_stored", zap.Int("count", len(creds)))
zap.L().Info("storing_results")
err = sqlite.StoreResults(results)
err = sqlite.StoreDehashedResults(results)
if err != nil {
zap.L().Error("store_results",
zap.String("message", "failed to store results"),
internal/easyTime/parser.go
+1 -1
View File
@@ -1,7 +1,7 @@
package easyTime
import (
"dehasher/internal/debug"
"crowsnest/internal/debug"
"fmt"
"go.uber.org/zap"
"os"
internal/export/export.go → internal/export/dehashed.go
+2 -2
View File
@@ -1,8 +1,8 @@
package export
import (
"dehasher/internal/files"
"dehasher/internal/sqlite"
"crowsnest/internal/files"
"crowsnest/internal/sqlite"
"encoding/json"
"encoding/xml"
"errors"
internal/export/hunter.go
-161
View File
@@ -1,161 +0,0 @@
package export
import (
"dehasher/internal/files"
"dehasher/internal/sqlite"
"encoding/json"
"encoding/xml"
"fmt"
"gopkg.in/yaml.v3"
"os"
)
func WriteHunterDomainToFile(result sqlite.HunterDomainData, outputFile string, fileType files.FileType) error {
var data []byte
var err error
switch fileType {
case files.JSON:
data, err = json.MarshalIndent(result, "", " ")
case files.XML:
data, err = xml.MarshalIndent(result, "", " ")
case files.YAML:
data, err = yaml.Marshal(result)
case files.TEXT:
data = []byte(result.String())
default:
return err
}
if err != nil {
return err
}
filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String())
return os.WriteFile(filePath, data, 0644)
}
func WriteHunterEmailToFile(result sqlite.HunterEmailFinderData, outputFile string, fileType files.FileType) error {
var data []byte
var err error
switch fileType {
case files.JSON:
data, err = json.MarshalIndent(result, "", " ")
case files.XML:
data, err = xml.MarshalIndent(result, "", " ")
case files.YAML:
data, err = yaml.Marshal(result)
case files.TEXT:
data = []byte(result.String())
default:
return err
}
if err != nil {
return err
}
filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String())
return os.WriteFile(filePath, data, 0644)
}
func WriteHunterEmailVerifyToFile(result sqlite.HunterEmailVerifyData, outputFile string, fileType files.FileType) error {
var data []byte
var err error
switch fileType {
case files.JSON:
data, err = json.MarshalIndent(result, "", " ")
case files.XML:
data, err = xml.MarshalIndent(result, "", " ")
case files.YAML:
data, err = yaml.Marshal(result)
case files.TEXT:
data = []byte(result.String())
default:
return err
}
if err != nil {
return err
}
filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String())
return os.WriteFile(filePath, data, 0644)
}
func WriteHunterCompanyEnrichmentToFile(result sqlite.CompanyData, outputFile string, fileType files.FileType) error {
var data []byte
var err error
switch fileType {
case files.JSON:
data, err = json.MarshalIndent(result, "", " ")
case files.XML:
data, err = xml.MarshalIndent(result, "", " ")
case files.YAML:
data, err = yaml.Marshal(result)
case files.TEXT:
data = []byte(result.String())
default:
return err
}
if err != nil {
return err
}
filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String())
return os.WriteFile(filePath, data, 0644)
}
func WriteHunterPersonEnrichmentToFile(result sqlite.PersonData, outputFile string, fileType files.FileType) error {
var data []byte
var err error
switch fileType {
case files.JSON:
data, err = json.MarshalIndent(result, "", " ")
case files.XML:
data, err = xml.MarshalIndent(result, "", " ")
case files.YAML:
data, err = yaml.Marshal(result)
case files.TEXT:
data = []byte(result.String())
default:
return err
}
if err != nil {
return err
}
filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String())
return os.WriteFile(filePath, data, 0644)
}
func WriteHunterCombinedEnrichmentToFile(result sqlite.CombinedData, outputFile string, fileType files.FileType) error {
var data []byte
var err error
switch fileType {
case files.JSON:
data, err = json.MarshalIndent(result, "", " ")
case files.XML:
data, err = xml.MarshalIndent(result, "", " ")
case files.YAML:
data, err = yaml.Marshal(result)
case files.TEXT:
data = []byte(result.String())
default:
return err
}
if err != nil {
return err
}
filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String())
return os.WriteFile(filePath, data, 0644)
}
internal/export/iString.go
+36
View File
@@ -0,0 +1,36 @@
package export
import (
"crowsnest/internal/files"
"crowsnest/internal/sqlite"
"encoding/json"
"encoding/xml"
"fmt"
"gopkg.in/yaml.v3"
"os"
)
func WriteIStringToFile(iString sqlite.IString, outputFile string, fileType files.FileType) error {
var data []byte
var err error
switch fileType {
case files.JSON:
data, err = json.MarshalIndent(iString, "", " ")
case files.XML:
data, err = xml.MarshalIndent(iString, "", " ")
case files.YAML:
data, err = yaml.Marshal(iString)
case files.TEXT:
data = []byte(iString.String())
default:
return err
}
if err != nil {
return err
}
filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String())
return os.WriteFile(filePath, data, 0644)
}
internal/hunter.io/hunter.io.go
+2 -2
View File
@@ -1,8 +1,8 @@
package hunter_io
import (
"dehasher/internal/debug"
"dehasher/internal/sqlite"
"crowsnest/internal/debug"
"crowsnest/internal/sqlite"
"encoding/json"
"fmt"
"go.uber.org/zap"
internal/pretty/trees.go
+1 -1
View File
@@ -1,7 +1,7 @@
package pretty
import (
"dehasher/internal/sqlite"
"crowsnest/internal/sqlite"
"fmt"
"github.com/charmbracelet/lipgloss"
"github.com/charmbracelet/lipgloss/tree"
internal/sqlite/db.go
+143
View File
@@ -0,0 +1,143 @@
package sqlite
import (
"fmt"
"go.uber.org/zap"
"os"
"path/filepath"
"strings"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
"gorm.io/gorm/logger"
)
var DB *gorm.DB
// InitDB initializes the database connection
func InitDB(dbPath string) (*gorm.DB, error) {
zap.L().Info("Initializing database", zap.String("path", dbPath))
// Check if the path is a file or directory
fileInfo, err := os.Stat(dbPath)
var finalDbPath string
// If path doesn't exist or is a directory
if os.IsNotExist(err) || (err == nil && fileInfo.IsDir()) {
// Treat as directory path
if err := os.MkdirAll(dbPath, 0755); err != nil {
zap.L().Error("Failed to create database directory", zap.Error(err))
return nil, fmt.Errorf("failed to create database directory: %w", err)
}
finalDbPath = filepath.Join(dbPath, "dehashed.sqlite")
} else {
// Treat as file path
// Ensure the directory exists
dir := filepath.Dir(dbPath)
if err := os.MkdirAll(dir, 0755); err != nil {
zap.L().Error("Failed to create parent directory for database", zap.Error(err))
return nil, fmt.Errorf("failed to create parent directory for database: %w", err)
}
finalDbPath = dbPath
}
zap.L().Info("Opening database", zap.String("finalPath", finalDbPath))
db, err := gorm.Open(sqlite.Open(finalDbPath), &gorm.Config{
Logger: logger.Default.LogMode(logger.Silent),
})
if err != nil {
zap.L().Error("Failed to connect to database", zap.Error(err))
return nil, fmt.Errorf("failed to connect to database: %w", err)
}
// Auto migrate your models
err = db.AutoMigrate(&Result{}, &Creds{}, &QueryOptions{}, &Creds{}, &WhoisRecord{}, &SubdomainRecord{},
&HistoryRecord{}, &LookupResult{}, &HunterDomainData{}, &HunterEmail{}, &PersonData{})
if err != nil {
zap.L().Error("Failed to migrate database", zap.Error(err))
return nil, fmt.Errorf("failed to migrate database: %w", err)
}
DB = db
return db, nil
}
// GetDB returns the database connection
func GetDB() *gorm.DB {
if DB == nil {
zap.L().Error("database not initialized")
fmt.Println("sqlite database not initialized")
os.Exit(1)
}
return DB
}
type Table int64
const (
ResultsTable Table = iota
RunsTable
CredsTable
WhoIsTable
SubdomainsTable
HistoryTable
LookupTable
HunterDomainTable
HunterEmailTable
PersonTable
UnknownTable
)
func GetTable(userInput string) Table {
switch strings.ToLower(userInput) {
case "results":
return ResultsTable
case "runs":
return RunsTable
case "creds":
return CredsTable
case "whois":
return WhoIsTable
case "subdomains":
return SubdomainsTable
case "history":
return HistoryTable
case "lookup":
return LookupTable
case "hunter_domain":
return HunterDomainTable
case "hunter_email":
return HunterEmailTable
case "person":
return PersonTable
default:
return UnknownTable
}
}
func (t Table) Object() interface{} {
switch t {
case ResultsTable:
return Result{}
case RunsTable:
return QueryOptions{}
case CredsTable:
return Creds{}
case WhoIsTable:
return WhoisRecord{}
case SubdomainsTable:
return SubdomainRecord{}
case HistoryTable:
return HistoryRecord{}
case LookupTable:
return LookupResult{}
case HunterDomainTable:
return HunterDomainData{}
case HunterEmailTable:
return HunterEmail{}
case PersonTable:
return PersonData{}
default:
return nil
}
}
internal/sqlite/dehashed.go
+235
View File
@@ -0,0 +1,235 @@
package sqlite
import (
"crowsnest/internal/files"
"fmt"
"go.uber.org/zap"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
type QueryOptions struct {
gorm.Model
MaxRecords int `json:"max_records"`
MaxRequests int `json:"max_requests"`
StartingPage int `json:"starting_page"`
OutputFormat files.FileType `json:"output_format"`
OutputFile string `json:"output_file"`
RegexMatch bool `json:"regex_match"`
WildcardMatch bool `json:"wildcard_match"`
UsernameQuery string `json:"username_query"`
EmailQuery string `json:"email_query"`
IpQuery string `json:"ip_query"`
PassQuery string `json:"pass_query"`
HashQuery string `json:"hash_query"`
NameQuery string `json:"name_query"`
DomainQuery string `json:"domain_query"`
VinQuery string `json:"vin_query"`
LicensePlateQuery string `json:"license_plate_query"`
AddressQuery string `json:"address_query"`
PhoneQuery string `json:"phone_query"`
SocialQuery string `json:"social_query"`
CryptoAddressQuery string `json:"crypto_address_query"`
PrintBalance bool `json:"print_balance"`
CredsOnly bool `json:"creds_only"`
Debug bool `json:"debug"`
}
func (QueryOptions) TableName() string {
return "query_options"
}
func NewQueryOptions(maxRecords, maxRequests, startingPage int, outputFormat, outputFile, usernameQuery, emailQuery, ipQuery, passQuery, hashQuery, nameQuery, domainQuery, vinQuery, licensePlateQuery, addressQuery, phoneQuery, socialQuery, cryptoAddressQuery string, regexMatch, wildcardMatch, printBalance, credsOnly, debug bool) *QueryOptions {
return &QueryOptions{
MaxRecords: maxRecords,
MaxRequests: maxRequests,
StartingPage: startingPage,
OutputFormat: files.GetFileType(outputFormat),
OutputFile: outputFile,
PrintBalance: printBalance,
CredsOnly: credsOnly,
RegexMatch: regexMatch,
WildcardMatch: wildcardMatch,
UsernameQuery: usernameQuery,
EmailQuery: emailQuery,
IpQuery: ipQuery,
PassQuery: passQuery,
HashQuery: hashQuery,
NameQuery: nameQuery,
DomainQuery: domainQuery,
VinQuery: vinQuery,
LicensePlateQuery: licensePlateQuery,
AddressQuery: addressQuery,
PhoneQuery: phoneQuery,
SocialQuery: socialQuery,
CryptoAddressQuery: cryptoAddressQuery,
Debug: debug,
}
}
type DehashedSearchRequest struct {
Page int `json:"page"`
Query string `json:"query"`
Size int `json:"size"`
Wildcard bool `json:"wildcard"`
Regex bool `json:"regex"`
DeDupe bool `json:"de_dupe"`
}
type DehashedResponse struct {
Balance int `json:"balance"`
Entries []Result `json:"entries"`
Success bool `json:"success"`
Took string `json:"took"`
TotalResults int `json:"total"`
}
type Result struct {
gorm.Model
DehashedId string `json:"id" xml:"id" yaml:"id" gorm:"uniqueIndex"`
Email []string `json:"email,omitempty" xml:"email,omitempty" yaml:"email,omitempty" gorm:"serializer:json"`
IpAddress []string `json:"ip_address,omitempty" xml:"ip_address,omitempty" yaml:"ip_address,omitempty" gorm:"serializer:json"`
Username []string `json:"username,omitempty" xml:"username,omitempty" yaml:"username,omitempty" gorm:"serializer:json"`
Password []string `json:"password,omitempty" xml:"password,omitempty" yaml:"password,omitempty" gorm:"serializer:json"`
HashedPassword []string `json:"hashed_password,omitempty" xml:"hashed_password,omitempty" yaml:"hashed_password,omitempty" gorm:"serializer:json"`
HashType string `json:"hash_type,omitempty" xml:"hash_type,omitempty" yaml:"hash_type,omitempty"`
Name []string `json:"name,omitempty" xml:"name,omitempty" yaml:"name,omitempty" gorm:"serializer:json"`
Vin []string `json:"vin,omitempty" xml:"vin,omitempty" yaml:"vin,omitempty" gorm:"serializer:json"`
LicensePlate []string `json:"license_plate,omitempty" xml:"license_plate,omitempty" yaml:"license_plate,omitempty" gorm:"serializer:json"`
Url []string `json:"url,omitempty" xml:"url,omitempty" yaml:"url,omitempty" gorm:"serializer:json"`
Social []string `json:"social,omitempty" xml:"social,omitempty" yaml:"social,omitempty" gorm:"serializer:json"`
CryptoCurrencyAddress []string `json:"cryptocurrency_address,omitempty" xml:"cryptocurrency_address,omitempty" yaml:"cryptocurrency_address,omitempty" gorm:"serializer:json"`
Address []string `json:"address,omitempty" xml:"address,omitempty" yaml:"address,omitempty" gorm:"serializer:json"`
Phone []string `json:"phone,omitempty" xml:"phone,omitempty" yaml:"phone,omitempty" gorm:"serializer:json"`
Company []string `json:"company,omitempty" xml:"company,omitempty" yaml:"company,omitempty" gorm:"serializer:json"`
DatabaseName string `json:"database_name,omitempty" xml:"database_name,omitempty" yaml:"database_name,omitempty"`
}
func (Result) TableName() string {
return "results"
}
type DehashedResults struct {
Results []Result `json:"results"`
}
func (dr *DehashedResults) ExtractCredentials() []Creds {
var creds []Creds
results := dr.Results
for _, r := range results {
if len(r.Password) > 0 {
// Get first email if available
email := ""
if len(r.Email) > 0 {
email = r.Email[0]
}
// Get first password
password := r.Password[0]
cred := Creds{Email: email, Password: password}
creds = append(creds, cred)
}
}
go func() {
err := StoreDehashedCreds(creds)
if err != nil {
zap.L().Error("store_creds",
zap.String("message", "failed to store creds"),
zap.Error(err),
)
fmt.Printf("Error Storing Results: %v", err)
}
}()
return creds
}
type Creds struct {
gorm.Model
Email string `json:"email" yaml:"email" xml:"email" gorm:"uniqueIndex:idx_email_username_password"`
Username string `json:"username" yaml:"username" xml:"username" gorm:"uniqueIndex:idx_email_username_password"`
Password string `json:"password" yaml:"password" xml:"password" gorm:"uniqueIndex:idx_email_username_password"`
}
func (Creds) TableName() string {
return "creds"
}
func (c Creds) ToString() string {
return fmt.Sprintf("%s%s%s", c.Username, "%", c.Password)
}
func StoreDehashedResults(results DehashedResults) error {
if len(results.Results) == 0 {
return nil
}
zap.L().Info("Storing results", zap.Int("count", len(results.Results)))
db := GetDB()
// Use batch insert with conflict handling
const batchSize = 100
var lastErr error
// Extract the slice of results
resultSlice := results.Results
for i := 0; i < len(resultSlice); i += batchSize {
end := i + batchSize
if end > len(resultSlice) {
end = len(resultSlice)
}
batch := resultSlice[i:end]
// Use Clauses with OnConflict DoNothing to skip conflicts
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
if err != nil {
zap.L().Warn("Error storing some results", zap.Error(err))
lastErr = err
// Continue with next batch despite error
}
}
return lastErr
}
func StoreDehashedCreds(creds []Creds) error {
if len(creds) == 0 {
return nil
}
zap.L().Info("Storing credentials", zap.Int("count", len(creds)))
db := GetDB()
// Use batch insert with conflict handling
// This will insert records in batches and continue even if some fail
const batchSize = 100
var lastErr error
for i := 0; i < len(creds); i += batchSize {
end := i + batchSize
if end > len(creds) {
end = len(creds)
}
batch := creds[i:end]
// Use Clauses with OnConflict DoNothing to skip conflicts
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
if err != nil {
zap.L().Warn("Error storing some credentials", zap.Error(err))
lastErr = err
// Continue with next batch despite error
}
}
return lastErr
}
func StoreDehashedQueryOptions(queryOptions *QueryOptions) error {
db := GetDB()
return db.Create(queryOptions).Error
}
internal/sqlite/gorm.go
-319
View File
@@ -1,319 +0,0 @@
package sqlite
import (
"fmt"
"go.uber.org/zap"
"gorm.io/gorm/clause"
"os"
"path/filepath"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
"gorm.io/gorm/logger"
)
var DB *gorm.DB
// InitDB initializes the database connection
func InitDB(dbPath string) (*gorm.DB, error) {
zap.L().Info("Initializing database", zap.String("path", dbPath))
// Check if the path is a file or directory
fileInfo, err := os.Stat(dbPath)
var finalDbPath string
// If path doesn't exist or is a directory
if os.IsNotExist(err) || (err == nil && fileInfo.IsDir()) {
// Treat as directory path
if err := os.MkdirAll(dbPath, 0755); err != nil {
zap.L().Error("Failed to create database directory", zap.Error(err))
return nil, fmt.Errorf("failed to create database directory: %w", err)
}
finalDbPath = filepath.Join(dbPath, "dehashed.sqlite")
} else {
// Treat as file path
// Ensure the directory exists
dir := filepath.Dir(dbPath)
if err := os.MkdirAll(dir, 0755); err != nil {
zap.L().Error("Failed to create parent directory for database", zap.Error(err))
return nil, fmt.Errorf("failed to create parent directory for database: %w", err)
}
finalDbPath = dbPath
}
zap.L().Info("Opening database", zap.String("finalPath", finalDbPath))
db, err := gorm.Open(sqlite.Open(finalDbPath), &gorm.Config{
Logger: logger.Default.LogMode(logger.Silent),
})
if err != nil {
zap.L().Error("Failed to connect to database", zap.Error(err))
return nil, fmt.Errorf("failed to connect to database: %w", err)
}
// Auto migrate your models
err = db.AutoMigrate(&Result{}, &Creds{}, &QueryOptions{}, &Creds{}, &WhoisRecord{}, &SubdomainRecord{},
&HistoryRecord{}, &LookupResult{}, &HunterDomainData{}, &HunterEmail{}, &PersonData{})
if err != nil {
zap.L().Error("Failed to migrate database", zap.Error(err))
return nil, fmt.Errorf("failed to migrate database: %w", err)
}
DB = db
return db, nil
}
// GetDB returns the database connection
func GetDB() *gorm.DB {
if DB == nil {
zap.L().Error("database not initialized")
fmt.Println("sqlite database not initialized")
os.Exit(1)
}
return DB
}
func StoreResults(results DehashedResults) error {
if len(results.Results) == 0 {
return nil
}
zap.L().Info("Storing results", zap.Int("count", len(results.Results)))
db := GetDB()
// Use batch insert with conflict handling
const batchSize = 100
var lastErr error
// Extract the slice of results
resultSlice := results.Results
for i := 0; i < len(resultSlice); i += batchSize {
end := i + batchSize
if end > len(resultSlice) {
end = len(resultSlice)
}
batch := resultSlice[i:end]
// Use Clauses with OnConflict DoNothing to skip conflicts
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
if err != nil {
zap.L().Warn("Error storing some results", zap.Error(err))
lastErr = err
// Continue with next batch despite error
}
}
return lastErr
}
func StoreCreds(creds []Creds) error {
if len(creds) == 0 {
return nil
}
zap.L().Info("Storing credentials", zap.Int("count", len(creds)))
db := GetDB()
// Use batch insert with conflict handling
// This will insert records in batches and continue even if some fail
const batchSize = 100
var lastErr error
for i := 0; i < len(creds); i += batchSize {
end := i + batchSize
if end > len(creds) {
end = len(creds)
}
batch := creds[i:end]
// Use Clauses with OnConflict DoNothing to skip conflicts
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
if err != nil {
zap.L().Warn("Error storing some credentials", zap.Error(err))
lastErr = err
// Continue with next batch despite error
}
}
return lastErr
}
func StoreQueryOptions(queryOptions *QueryOptions) error {
db := GetDB()
return db.Create(queryOptions).Error
}
func StoreWhoisRecord(whoisRecord WhoisRecord) error {
// Create a pointer to the record to make it addressable
recordPtr := &whoisRecord
zap.L().Info("Storing WHOIS record",
zap.String("domain", whoisRecord.DomainName))
db := GetDB()
// Use OnConflict clause to handle duplicates
err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(recordPtr).Error
if err != nil {
zap.L().Error("store_whois_record",
zap.String("message", "failed to store whois record"),
zap.Error(err))
return err
}
return nil
}
func StoreSubdomainRecords(subdomainRecords []SubdomainRecord) error {
if len(subdomainRecords) == 0 {
return nil
}
zap.L().Info("Storing subdomain records", zap.Int("count", len(subdomainRecords)))
db := GetDB()
// Use batch insert with conflict handling
const batchSize = 100
var lastErr error
for i := 0; i < len(subdomainRecords); i += batchSize {
end := i + batchSize
if end > len(subdomainRecords) {
end = len(subdomainRecords)
}
batch := subdomainRecords[i:end]
// Use Clauses with OnConflict DoNothing to skip conflicts
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
if err != nil {
zap.L().Warn("Error storing some subdomain records", zap.Error(err))
lastErr = err
// Continue with next batch despite error
}
}
return lastErr
}
func StoreHistoryRecord(historyRecords []HistoryRecord) error {
if len(historyRecords) == 0 {
return nil
}
zap.L().Info("Storing history records", zap.Int("count", len(historyRecords)))
db := GetDB()
// Use batch insert with conflict handling
const batchSize = 100
var lastErr error
for i := 0; i < len(historyRecords); i += batchSize {
end := i + batchSize
if end > len(historyRecords) {
end = len(historyRecords)
}
batch := historyRecords[i:end]
// Use Clauses with OnConflict DoNothing to skip conflicts
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
if err != nil {
zap.L().Warn("Error storing some history records", zap.Error(err))
lastErr = err
// Continue with next batch despite error
}
}
return lastErr
}
func StoreIPLookup(ipLookup []LookupResult) error {
if len(ipLookup) == 0 {
return nil
}
zap.L().Info("Storing IP lookup records", zap.Int("count", len(ipLookup)))
db := GetDB()
// Use batch insert with conflict handling
const batchSize = 100
var lastErr error
for i := 0; i < len(ipLookup); i += batchSize {
end := i + batchSize
if end > len(ipLookup) {
end = len(ipLookup)
}
batch := ipLookup[i:end]
// Use Clauses with OnConflict DoNothing to skip conflicts
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
if err != nil {
zap.L().Warn("Error storing some IP lookup records", zap.Error(err))
lastErr = err
// Continue with next batch despite error
}
}
return lastErr
}
func StoreHunterDomain(hunterDomain HunterDomainData) error {
db := GetDB()
// Use OnConflict clause to handle duplicates
err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(&hunterDomain).Error
if err != nil {
zap.L().Error("store_hunter_domain",
zap.String("message", "failed to store hunter domain"),
zap.Error(err))
return err
}
return nil
}
func StoreHunterEmails(hunterEmails []HunterEmail) error {
if len(hunterEmails) == 0 {
return nil
}
zap.L().Info("Storing hunter emails", zap.Int("count", len(hunterEmails)))
db := GetDB()
// Use batch insert with conflict handling
const batchSize = 100
var lastErr error
for i := 0; i < len(hunterEmails); i += batchSize {
end := i + batchSize
if end > len(hunterEmails) {
end = len(hunterEmails)
}
batch := hunterEmails[i:end]
// Use Clauses with OnConflict DoNothing to skip conflicts
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
if err != nil {
zap.L().Warn("Error storing some hunter emails", zap.Error(err))
lastErr = err
// Continue with next batch despite error
}
}
return lastErr
}
func StorePersonData(personData PersonData) error {
db := GetDB()
// Use OnConflict clause to handle duplicates
err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(&personData).Error
if err != nil {
zap.L().Error("store_person_data",
zap.String("message", "failed to store person data"),
zap.Error(err))
return err
}
return nil
}
internal/sqlite/hunter.io.go
+75 -6
View File
@@ -3,7 +3,9 @@ package sqlite
import (
"fmt"
"github.com/charmbracelet/lipgloss/tree"
"go.uber.org/zap"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
// HunterDomainSearchResult represents the response from Hunter.io domain search API
@@ -15,6 +17,7 @@ type HunterDomainSearchResult struct {
// HunterDomainData contains the main domain information
type HunterDomainData struct {
gorm.Model
IString `gorm:"-"`
Domain string `json:"domain" gorm:"unique"`
Disposable bool `json:"disposable"`
Webmail bool `json:"webmail"`
@@ -40,7 +43,7 @@ type HunterDomainData struct {
LinkedDomains []string `json:"linked_domains" gorm:"serializer:json"`
}
func (h *HunterDomainData) String() string {
func (h HunterDomainData) String() string {
return fmt.Sprintf("Domain: %s\nDisposable: %t\nWebmail: %t\nAcceptAll: %t\nPattern: %s\nOrganization: %s\nDescription: %s\nIndustry: %s\nTwitter: %s\nFacebook: %s\nLinkedin: %s\nInstagram: %s\nYoutube: %s\nTechnologies: %v\nCountry: %s\nState: %s\nCity: %s\nPostalCode: %s\nStreet: %s\nHeadcount: %s\nCompanyType: %s\nEmails: %v\nLinkedDomains: %v\n",
h.Domain, h.Disposable, h.Webmail, h.AcceptAll, h.Pattern, h.Organization, h.Description, h.Industry, h.Twitter, h.Facebook, h.Linkedin, h.Instagram, h.Youtube, h.Technologies, h.Country, h.State, h.City, h.PostalCode, h.Street, h.Headcount, h.CompanyType, h.Emails, h.LinkedDomains)
}
@@ -142,6 +145,7 @@ type HunterEmailFinderResponse struct {
// HunterEmailFinderData contains the main email information
type HunterEmailFinderData struct {
IString
FirstName string `json:"first_name"`
LastName string `json:"last_name"`
Email string `json:"email"`
@@ -157,7 +161,7 @@ type HunterEmailFinderData struct {
Verification HunterVerification `json:"verification" gorm:"embedded;embeddedPrefix:verification_"`
}
func (he *HunterEmailFinderData) String() string {
func (he HunterEmailFinderData) String() string {
return fmt.Sprintf("FirstName: %s\nLastName: %s\nEmail: %s\nScore: %d\nDomain: %s\nAcceptAll: %t\nPosition: %s\nTwitter: %s\nLinkedinURL: %s\nPhoneNumber: %s\nCompany: %s\nSources: %v\nVerification: %v\n",
he.FirstName, he.LastName, he.Email, he.Score, he.Domain, he.AcceptAll, he.Position, he.Twitter, he.LinkedinURL, he.PhoneNumber, he.Company, he.Sources, he.Verification)
}
@@ -189,6 +193,7 @@ type HunterEmailVerifyResponse struct {
// HunterEmailVerifyData contains the email verification information
type HunterEmailVerifyData struct {
IString
Status string `json:"status"`
Result string `json:"result"`
DeprecationNotice string `json:"_deprecation_notice"`
@@ -206,7 +211,7 @@ type HunterEmailVerifyData struct {
Sources []HunterSource `json:"sources" gorm:"serializer:json"`
}
func (ev *HunterEmailVerifyData) String() string {
func (ev HunterEmailVerifyData) String() string {
return fmt.Sprintf("Status: %s\nResult: %s\nDeprecationNotice: %s\nScore: %d\nEmail: %s\nRegexp: %t\nGibberish: %t\nDisposable: %t\nWebmail: %t\nMXRecords: %t\nSMTPServer: %t\nSMTPCheck: %t\nAcceptAll: %t\nBlock: %t\nSources: %v\n",
ev.Status, ev.Result, ev.DeprecationNotice, ev.Score, ev.Email, ev.Regexp, ev.Gibberish, ev.Disposable, ev.Webmail, ev.MXRecords, ev.SMTPServer, ev.SMTPCheck, ev.AcceptAll, ev.Block, ev.Sources)
}
@@ -229,6 +234,7 @@ type HunterCompanyEnrichmentResponse struct {
// CompanyData contains the detailed company information
type CompanyData struct {
IString
ID string `json:"id"`
Name string `json:"name"`
LegalName string `json:"legalName"`
@@ -262,7 +268,7 @@ type CompanyData struct {
UltimateParent ParentCompany `json:"ultimateParent" gorm:"embedded;embeddedPrefix:ultimate_parent_"`
}
func (cd *CompanyData) String() string {
func (cd CompanyData) String() string {
return fmt.Sprintf("ID: %s\nName: %s\nLegalName: %s\nDomain: %s\nDomainAliases: %v\nSite: %v\nCategory: %v\nTags: %v\nDescription: %s\nFoundedYear: %d\nLocation: %s\nTimeZone: %s\nUTCOffset: %d\nGeo: %v\nLogo: %s\nFacebook: %v\nLinkedIn: %v\nTwitter: %v\nCrunchbase: %v\nYouTube: %v\nEmailProvider: %s\nType: %s\nTicker: %s\nIdentifiers: %v\nPhone: %s\nMetrics: %v\nIndexedAt: %s\nTech: %v\nTechCategories: %v\nParent: %v\nUltimateParent: %v\n",
cd.ID, cd.Name, cd.LegalName, cd.Domain, cd.DomainAliases, cd.Site, cd.Category, cd.Tags, cd.Description, cd.FoundedYear, cd.Location, cd.TimeZone, cd.UTCOffset, cd.Geo, cd.Logo, cd.Facebook, cd.LinkedIn, cd.Twitter, cd.Crunchbase, cd.YouTube, cd.EmailProvider, cd.Type, cd.Ticker, cd.Identifiers, cd.Phone, cd.Metrics, cd.IndexedAt, cd.Tech, cd.TechCategories, cd.Parent, cd.UltimateParent)
}
@@ -509,6 +515,7 @@ type HunterPersonEnrichmentResponse struct {
// PersonData contains the detailed person information
type PersonData struct {
gorm.Model
IString `gorm:"-"`
ID string `json:"id"`
Name PersonName `json:"name" gorm:"embedded;embeddedPrefix:name_"`
Email string `json:"email" gorm:"unique"`
@@ -534,7 +541,7 @@ type PersonData struct {
InactiveAt string `json:"inactiveAt"`
}
func (pd *PersonData) String() string {
func (pd PersonData) String() string {
return fmt.Sprintf("ID: %s\nName: %v\nEmail: %s\nLocation: %s\nTimeZone: %s\nUTCOffset: %d\nGeo: %v\nBio: %s\nSite: %s\nAvatar: %s\nEmployment: %v\nFacebook: %v\nGitHub: %v\nTwitter: %v\nLinkedIn: %v\nGooglePlus: %v\nGravatar: %v\nFuzzy: %t\nEmailProvider: %s\nIndexedAt: %s\nPhone: %s\nActiveAt: %s\nInactiveAt: %s\n",
pd.ID, pd.Name, pd.Email, pd.Location, pd.TimeZone, pd.UTCOffset, pd.Geo, pd.Bio, pd.Site, pd.Avatar, pd.Employment, pd.Facebook, pd.GitHub, pd.Twitter, pd.LinkedIn, pd.GooglePlus, pd.Gravatar, pd.Fuzzy, pd.EmailProvider, pd.IndexedAt, pd.Phone, pd.ActiveAt, pd.InactiveAt)
}
@@ -728,11 +735,12 @@ type HunterCombinedEnrichmentResponse struct {
// CombinedData contains both person and company information
type CombinedData struct {
IString
Person PersonData `json:"person" gorm:"embedded;embeddedPrefix:person_"`
Company CompanyData `json:"company" gorm:"embedded;embeddedPrefix:company_"`
}
func (cbd *CombinedData) String() string {
func (cbd CombinedData) String() string {
return fmt.Sprintf("Person: %s\nCompany: %s",
cbd.Person.String(),
cbd.Company.String())
@@ -749,3 +757,64 @@ func (c *HunterCombinedEnrichmentResponse) String() string {
c.Data.Person.String(),
c.Data.Company.String())
}
func StoreHunterDomain(hunterDomain HunterDomainData) error {
db := GetDB()
// Use OnConflict clause to handle duplicates
err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(&hunterDomain).Error
if err != nil {
zap.L().Error("store_hunter_domain",
zap.String("message", "failed to store hunter domain"),
zap.Error(err))
return err
}
return nil
}
func StoreHunterEmails(hunterEmails []HunterEmail) error {
if len(hunterEmails) == 0 {
return nil
}
zap.L().Info("Storing hunter emails", zap.Int("count", len(hunterEmails)))
db := GetDB()
// Use batch insert with conflict handling
const batchSize = 100
var lastErr error
for i := 0; i < len(hunterEmails); i += batchSize {
end := i + batchSize
if end > len(hunterEmails) {
end = len(hunterEmails)
}
batch := hunterEmails[i:end]
// Use Clauses with OnConflict DoNothing to skip conflicts
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
if err != nil {
zap.L().Warn("Error storing some hunter emails", zap.Error(err))
lastErr = err
// Continue with next batch despite error
}
}
return lastErr
}
func StoreHunterPersonData(personData PersonData) error {
db := GetDB()
// Use OnConflict clause to handle duplicates
err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(&personData).Error
if err != nil {
zap.L().Error("store_person_data",
zap.String("message", "failed to store person data"),
zap.Error(err))
return err
}
return nil
}
internal/sqlite/query.go
-20
View File
@@ -1,20 +0,0 @@
package sqlite
type DehashedSearchRequest struct {
Page int `json:"page"`
Query string `json:"query"`
Size int `json:"size"`
Wildcard bool `json:"wildcard"`
Regex bool `json:"regex"`
DeDupe bool `json:"de_dupe"`
}
func NewDehashedSearchRequest(size int, wildcard, regex bool) *DehashedSearchRequest {
return &DehashedSearchRequest{
Page: 0,
Size: size,
Wildcard: false,
Regex: false,
DeDupe: true,
}
}
internal/sqlite/result.go
-94
View File
@@ -1,94 +0,0 @@
package sqlite
import (
"encoding/json"
"fmt"
"go.uber.org/zap"
"gorm.io/gorm"
"io"
"os"
)
type DehashedResponse struct {
Balance int `json:"balance"`
Entries []Result `json:"entries"`
Success bool `json:"success"`
Took string `json:"took"`
TotalResults int `json:"total"`
}
type Result struct {
gorm.Model
DehashedId string `json:"id" xml:"id" yaml:"id" gorm:"uniqueIndex"`
Email []string `json:"email,omitempty" xml:"email,omitempty" yaml:"email,omitempty" gorm:"serializer:json"`
IpAddress []string `json:"ip_address,omitempty" xml:"ip_address,omitempty" yaml:"ip_address,omitempty" gorm:"serializer:json"`
Username []string `json:"username,omitempty" xml:"username,omitempty" yaml:"username,omitempty" gorm:"serializer:json"`
Password []string `json:"password,omitempty" xml:"password,omitempty" yaml:"password,omitempty" gorm:"serializer:json"`
HashedPassword []string `json:"hashed_password,omitempty" xml:"hashed_password,omitempty" yaml:"hashed_password,omitempty" gorm:"serializer:json"`
HashType string `json:"hash_type,omitempty" xml:"hash_type,omitempty" yaml:"hash_type,omitempty"`
Name []string `json:"name,omitempty" xml:"name,omitempty" yaml:"name,omitempty" gorm:"serializer:json"`
Vin []string `json:"vin,omitempty" xml:"vin,omitempty" yaml:"vin,omitempty" gorm:"serializer:json"`
LicensePlate []string `json:"license_plate,omitempty" xml:"license_plate,omitempty" yaml:"license_plate,omitempty" gorm:"serializer:json"`
Url []string `json:"url,omitempty" xml:"url,omitempty" yaml:"url,omitempty" gorm:"serializer:json"`
Social []string `json:"social,omitempty" xml:"social,omitempty" yaml:"social,omitempty" gorm:"serializer:json"`
CryptoCurrencyAddress []string `json:"cryptocurrency_address,omitempty" xml:"cryptocurrency_address,omitempty" yaml:"cryptocurrency_address,omitempty" gorm:"serializer:json"`
Address []string `json:"address,omitempty" xml:"address,omitempty" yaml:"address,omitempty" gorm:"serializer:json"`
Phone []string `json:"phone,omitempty" xml:"phone,omitempty" yaml:"phone,omitempty" gorm:"serializer:json"`
Company []string `json:"company,omitempty" xml:"company,omitempty" yaml:"company,omitempty" gorm:"serializer:json"`
DatabaseName string `json:"database_name,omitempty" xml:"database_name,omitempty" yaml:"database_name,omitempty"`
}
func (Result) TableName() string {
return "results"
}
type DehashedResults struct {
Results []Result `json:"results"`
}
func (dr *DehashedResults) ExtractCredentials() []Creds {
var creds []Creds
results := dr.Results
for _, r := range results {
if len(r.Password) > 0 {
// Get first email if available
email := ""
if len(r.Email) > 0 {
email = r.Email[0]
}
// Get first password
password := r.Password[0]
cred := Creds{Email: email, Password: password}
creds = append(creds, cred)
}
}
go func() {
err := StoreCreds(creds)
if err != nil {
zap.L().Error("store_creds",
zap.String("message", "failed to store creds"),
zap.Error(err),
)
fmt.Printf("Error Storing Results: %v", err)
}
}()
return creds
}
func NewDehashedResults(body io.Reader) ([]Result, int, int) {
var response DehashedResponse
err := json.NewDecoder(body).Decode(&response)
if err != nil {
fmt.Printf("Error Parsing Response Body: %v", err)
os.Exit(-1)
}
return response.Entries, response.Balance, response.TotalResults
}
internal/sqlite/structs.go
+3 -88
View File
@@ -1,10 +1,8 @@
package sqlite
import (
"dehasher/internal/files"
"fmt"
"gorm.io/gorm"
)
type IString interface {
String() string
}
type DBOptions struct {
Username string
@@ -26,15 +24,6 @@ type DBOptions struct {
DisplayFields []string // Fields to display in output
}
func NewDBOptions() *DBOptions {
return &DBOptions{
Limit: 100, // Default limit
ExactMatch: false,
NonEmptyFields: []string{},
DisplayFields: []string{},
}
}
func (o *DBOptions) Empty() bool {
return o.Username == "" && o.Email == "" && o.IPAddress == "" &&
o.Password == "" && o.HashedPassword == "" && o.Name == "" &&
@@ -42,77 +31,3 @@ func (o *DBOptions) Empty() bool {
o.Phone == "" && o.Social == "" && o.CryptoCurrencyAddress == "" && o.Domain == "" &&
len(o.NonEmptyFields) == 0
}
type QueryOptions struct {
gorm.Model
MaxRecords int `json:"max_records"`
MaxRequests int `json:"max_requests"`
StartingPage int `json:"starting_page"`
OutputFormat files.FileType `json:"output_format"`
OutputFile string `json:"output_file"`
RegexMatch bool `json:"regex_match"`
WildcardMatch bool `json:"wildcard_match"`
UsernameQuery string `json:"username_query"`
EmailQuery string `json:"email_query"`
IpQuery string `json:"ip_query"`
PassQuery string `json:"pass_query"`
HashQuery string `json:"hash_query"`
NameQuery string `json:"name_query"`
DomainQuery string `json:"domain_query"`
VinQuery string `json:"vin_query"`
LicensePlateQuery string `json:"license_plate_query"`
AddressQuery string `json:"address_query"`
PhoneQuery string `json:"phone_query"`
SocialQuery string `json:"social_query"`
CryptoAddressQuery string `json:"crypto_address_query"`
PrintBalance bool `json:"print_balance"`
CredsOnly bool `json:"creds_only"`
Debug bool `json:"debug"`
}
func (QueryOptions) TableName() string {
return "query_options"
}
func NewQueryOptions(maxRecords, maxRequests, startingPage int, outputFormat, outputFile, usernameQuery, emailQuery, ipQuery, passQuery, hashQuery, nameQuery, domainQuery, vinQuery, licensePlateQuery, addressQuery, phoneQuery, socialQuery, cryptoAddressQuery string, regexMatch, wildcardMatch, printBalance, credsOnly, debug bool) *QueryOptions {
return &QueryOptions{
MaxRecords: maxRecords,
MaxRequests: maxRequests,
StartingPage: startingPage,
OutputFormat: files.GetFileType(outputFormat),
OutputFile: outputFile,
PrintBalance: printBalance,
CredsOnly: credsOnly,
RegexMatch: regexMatch,
WildcardMatch: wildcardMatch,
UsernameQuery: usernameQuery,
EmailQuery: emailQuery,
IpQuery: ipQuery,
PassQuery: passQuery,
HashQuery: hashQuery,
NameQuery: nameQuery,
DomainQuery: domainQuery,
VinQuery: vinQuery,
LicensePlateQuery: licensePlateQuery,
AddressQuery: addressQuery,
PhoneQuery: phoneQuery,
SocialQuery: socialQuery,
CryptoAddressQuery: cryptoAddressQuery,
Debug: debug,
}
}
type Creds struct {
gorm.Model
Email string `json:"email" yaml:"email" xml:"email" gorm:"uniqueIndex:idx_email_username_password"`
Username string `json:"username" yaml:"username" xml:"username" gorm:"uniqueIndex:idx_email_username_password"`
Password string `json:"password" yaml:"password" xml:"password" gorm:"uniqueIndex:idx_email_username_password"`
}
func (Creds) TableName() string {
return "creds"
}
func (c Creds) ToString() string {
return fmt.Sprintf("%s%s%s", c.Username, "%", c.Password)
}
internal/sqlite/tables.go
-68
View File
@@ -1,68 +0,0 @@
package sqlite
import "strings"
type Table int64
const (
ResultsTable Table = iota
RunsTable
CredsTable
WhoIsTable
SubdomainsTable
HistoryTable
LookupTable
HunterDomainTable
HunterEmailTable
UnknownTable
)
func GetTable(userInput string) Table {
switch strings.ToLower(userInput) {
case "results":
return ResultsTable
case "runs":
return RunsTable
case "creds":
return CredsTable
case "whois":
return WhoIsTable
case "subdomains":
return SubdomainsTable
case "history":
return HistoryTable
case "lookup":
return LookupTable
case "hunter_domain":
return HunterDomainTable
case "hunter_email":
return HunterEmailTable
default:
return UnknownTable
}
}
func (t Table) Object() interface{} {
switch t {
case ResultsTable:
return Result{}
case RunsTable:
return QueryOptions{}
case CredsTable:
return Creds{}
case WhoIsTable:
return WhoisRecord{}
case SubdomainsTable:
return SubdomainRecord{}
case HistoryTable:
return HistoryRecord{}
case LookupTable:
return LookupResult{}
case HunterDomainTable:
return HunterDomainData{}
case HunterEmailTable:
return HunterEmail{}
default:
return nil
}
}
internal/sqlite/whois.go
+116
View File
@@ -2,7 +2,9 @@ package sqlite
import (
"fmt"
"go.uber.org/zap"
"gorm.io/gorm"
"gorm.io/gorm/clause"
"strings"
)
@@ -512,3 +514,117 @@ func formatWhoisContact(sb *strings.Builder, contact Contact, indent string) {
sb.WriteString(indent + "Raw Text: " + rawTextPreview + "\n")
}
}
func StoreWhoisRecord(whoisRecord WhoisRecord) error {
// Create a pointer to the record to make it addressable
recordPtr := &whoisRecord
zap.L().Info("Storing WHOIS record",
zap.String("domain", whoisRecord.DomainName))
db := GetDB()
// Use OnConflict clause to handle duplicates
err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(recordPtr).Error
if err != nil {
zap.L().Error("store_whois_record",
zap.String("message", "failed to store whois record"),
zap.Error(err))
return err
}
return nil
}
func StoreWhoisSubdomainRecords(subdomainRecords []SubdomainRecord) error {
if len(subdomainRecords) == 0 {
return nil
}
zap.L().Info("Storing subdomain records", zap.Int("count", len(subdomainRecords)))
db := GetDB()
// Use batch insert with conflict handling
const batchSize = 100
var lastErr error
for i := 0; i < len(subdomainRecords); i += batchSize {
end := i + batchSize
if end > len(subdomainRecords) {
end = len(subdomainRecords)
}
batch := subdomainRecords[i:end]
// Use Clauses with OnConflict DoNothing to skip conflicts
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
if err != nil {
zap.L().Warn("Error storing some subdomain records", zap.Error(err))
lastErr = err
// Continue with next batch despite error
}
}
return lastErr
}
func StoreWhoisHistoryRecords(historyRecords []HistoryRecord) error {
if len(historyRecords) == 0 {
return nil
}
zap.L().Info("Storing history records", zap.Int("count", len(historyRecords)))
db := GetDB()
// Use batch insert with conflict handling
const batchSize = 100
var lastErr error
for i := 0; i < len(historyRecords); i += batchSize {
end := i + batchSize
if end > len(historyRecords) {
end = len(historyRecords)
}
batch := historyRecords[i:end]
// Use Clauses with OnConflict DoNothing to skip conflicts
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
if err != nil {
zap.L().Warn("Error storing some history records", zap.Error(err))
lastErr = err
// Continue with next batch despite error
}
}
return lastErr
}
func StoreWhoisLookup(lookup []LookupResult) error {
if len(lookup) == 0 {
return nil
}
zap.L().Info("Storing IP lookup records", zap.Int("count", len(lookup)))
db := GetDB()
// Use batch insert with conflict handling
const batchSize = 100
var lastErr error
for i := 0; i < len(lookup); i += batchSize {
end := i + batchSize
if end > len(lookup) {
end = len(lookup)
}
batch := lookup[i:end]
// Use Clauses with OnConflict DoNothing to skip conflicts
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
if err != nil {
zap.L().Warn("Error storing some IP lookup records", zap.Error(err))
lastErr = err
// Continue with next batch despite error
}
}
return lastErr
}
internal/whois/whois.go
+6 -6
View File
@@ -2,9 +2,9 @@ package whois
import (
"bytes"
"dehasher/internal/debug"
"dehasher/internal/dehashed"
"dehasher/internal/sqlite"
"crowsnest/internal/debug"
"crowsnest/internal/dehashed"
"crowsnest/internal/sqlite"
"encoding/json"
"errors"
"fmt"
@@ -552,7 +552,7 @@ func (w *DehashedWhoIs) WhoisIP(ipAddress string) ([]sqlite.LookupResult, error)
})
}
sqlite.StoreIPLookup(lookups)
sqlite.StoreWhoisLookup(lookups)
return lookups, nil
}
@@ -702,7 +702,7 @@ func (w *DehashedWhoIs) WhoisMX(mxHostname string) ([]sqlite.LookupResult, error
})
}
sqlite.StoreIPLookup(mxLookups)
sqlite.StoreWhoisLookup(mxLookups)
return mxLookups, nil
}
@@ -849,7 +849,7 @@ func (w *DehashedWhoIs) WhoisNS(nsHostname string) ([]sqlite.LookupResult, error
})
}
sqlite.StoreIPLookup(nsLookups)
sqlite.StoreWhoisLookup(nsLookups)
return nsLookups, nil
}