KrakenTech/CrowsNest
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fixed output of coffee and primary banner

Browse Source
This commit is contained in:
Evan Hosinski
2025-05-17 10:25:16 -04:00
parent 61052b3308
commit 00d9a6b57e
15 changed files with 574 additions and 607 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/dehashed.go
+1 -1
View File
@@ -118,7 +118,7 @@ var (
dehasher.Start()
fmt.Println("\n[*] Completing Process")
err := sqlite.StoreQueryOptions(queryOptions)
err := sqlite.StoreDehashedQueryOptions(queryOptions)
if err != nil {
if debugGlobal {
debug.PrintInfo("failed to store query options")
cmd/whois.go
+2 -2
View File
@@ -208,7 +208,7 @@ var (
fmt.Printf("[!] Error writing WHOIS history to file: %v\n", writeErr)
}
err = sqlite.StoreHistoryRecord(historyRecords)
err = sqlite.StoreWhoisHistoryRecords(historyRecords)
if err != nil {
if debugGlobal {
debug.PrintInfo("failed to store history record")
@@ -254,7 +254,7 @@ var (
fmt.Printf("Error performing subdomain scan: %v\n", err)
} else {
fmt.Println("Subdomain Scan:")
err = sqlite.StoreSubdomainRecords(subdomains)
err = sqlite.StoreWhoisSubdomainRecords(subdomains)
if err != nil {
if debugGlobal {
debug.PrintInfo("failed to store subdomain record")
dehasher.go → crowsnest.go
View File
internal/dehashed/dehashed.go
+3 -3
View File
@@ -130,7 +130,7 @@ func (dh *Dehasher) Start() {
if len(dh.client.results) > 0 {
fmt.Printf(" [!] Partial results retrieved. Storing Results...\n")
err := sqlite.StoreResults(dh.client.GetResults())
err := sqlite.StoreDehashedResults(dh.client.GetResults())
if err != nil {
zap.L().Error("store_results",
zap.String("message", "failed to store results"),
@@ -214,7 +214,7 @@ func (dh *Dehasher) parseResults() {
results := dh.client.GetResults()
creds := results.ExtractCredentials()
fmt.Printf("\n\t[+] Discovered %d Credentials", len(creds))
err := sqlite.StoreCreds(creds)
err := sqlite.StoreDehashedCreds(creds)
if err != nil {
zap.L().Error("store_creds",
zap.String("message", "failed to store creds"),
@@ -224,7 +224,7 @@ func (dh *Dehasher) parseResults() {
zap.L().Info("creds_stored", zap.Int("count", len(creds)))
zap.L().Info("storing_results")
err = sqlite.StoreResults(results)
err = sqlite.StoreDehashedResults(results)
if err != nil {
zap.L().Error("store_results",
zap.String("message", "failed to store results"),
internal/sqlite/db.go
+143
View File
@@ -0,0 +1,143 @@
package sqlite
import (
"fmt"
"go.uber.org/zap"
"os"
"path/filepath"
"strings"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
"gorm.io/gorm/logger"
)
var DB *gorm.DB
// InitDB initializes the database connection
func InitDB(dbPath string) (*gorm.DB, error) {
zap.L().Info("Initializing database", zap.String("path", dbPath))
// Check if the path is a file or directory
fileInfo, err := os.Stat(dbPath)
var finalDbPath string
// If path doesn't exist or is a directory
if os.IsNotExist(err) || (err == nil && fileInfo.IsDir()) {
// Treat as directory path
if err := os.MkdirAll(dbPath, 0755); err != nil {
zap.L().Error("Failed to create database directory", zap.Error(err))
return nil, fmt.Errorf("failed to create database directory: %w", err)
}
finalDbPath = filepath.Join(dbPath, "dehashed.sqlite")
} else {
// Treat as file path
// Ensure the directory exists
dir := filepath.Dir(dbPath)
if err := os.MkdirAll(dir, 0755); err != nil {
zap.L().Error("Failed to create parent directory for database", zap.Error(err))
return nil, fmt.Errorf("failed to create parent directory for database: %w", err)
}
finalDbPath = dbPath
}
zap.L().Info("Opening database", zap.String("finalPath", finalDbPath))
db, err := gorm.Open(sqlite.Open(finalDbPath), &gorm.Config{
Logger: logger.Default.LogMode(logger.Silent),
})
if err != nil {
zap.L().Error("Failed to connect to database", zap.Error(err))
return nil, fmt.Errorf("failed to connect to database: %w", err)
}
// Auto migrate your models
err = db.AutoMigrate(&Result{}, &Creds{}, &QueryOptions{}, &Creds{}, &WhoisRecord{}, &SubdomainRecord{},
&HistoryRecord{}, &LookupResult{}, &HunterDomainData{}, &HunterEmail{}, &PersonData{})
if err != nil {
zap.L().Error("Failed to migrate database", zap.Error(err))
return nil, fmt.Errorf("failed to migrate database: %w", err)
}
DB = db
return db, nil
}
// GetDB returns the database connection
func GetDB() *gorm.DB {
if DB == nil {
zap.L().Error("database not initialized")
fmt.Println("sqlite database not initialized")
os.Exit(1)
}
return DB
}
type Table int64
const (
ResultsTable Table = iota
RunsTable
CredsTable
WhoIsTable
SubdomainsTable
HistoryTable
LookupTable
HunterDomainTable
HunterEmailTable
PersonTable
UnknownTable
)
func GetTable(userInput string) Table {
switch strings.ToLower(userInput) {
case "results":
return ResultsTable
case "runs":
return RunsTable
case "creds":
return CredsTable
case "whois":
return WhoIsTable
case "subdomains":
return SubdomainsTable
case "history":
return HistoryTable
case "lookup":
return LookupTable
case "hunter_domain":
return HunterDomainTable
case "hunter_email":
return HunterEmailTable
case "person":
return PersonTable
default:
return UnknownTable
}
}
func (t Table) Object() interface{} {
switch t {
case ResultsTable:
return Result{}
case RunsTable:
return QueryOptions{}
case CredsTable:
return Creds{}
case WhoIsTable:
return WhoisRecord{}
case SubdomainsTable:
return SubdomainRecord{}
case HistoryTable:
return HistoryRecord{}
case LookupTable:
return LookupResult{}
case HunterDomainTable:
return HunterDomainData{}
case HunterEmailTable:
return HunterEmail{}
case PersonTable:
return PersonData{}
default:
return nil
}
}
internal/sqlite/dehashed.go
+235
View File
@@ -0,0 +1,235 @@
package sqlite
import (
"crowsnest/internal/files"
"fmt"
"go.uber.org/zap"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
type QueryOptions struct {
gorm.Model
MaxRecords int `json:"max_records"`
MaxRequests int `json:"max_requests"`
StartingPage int `json:"starting_page"`
OutputFormat files.FileType `json:"output_format"`
OutputFile string `json:"output_file"`
RegexMatch bool `json:"regex_match"`
WildcardMatch bool `json:"wildcard_match"`
UsernameQuery string `json:"username_query"`
EmailQuery string `json:"email_query"`
IpQuery string `json:"ip_query"`
PassQuery string `json:"pass_query"`
HashQuery string `json:"hash_query"`
NameQuery string `json:"name_query"`
DomainQuery string `json:"domain_query"`
VinQuery string `json:"vin_query"`
LicensePlateQuery string `json:"license_plate_query"`
AddressQuery string `json:"address_query"`
PhoneQuery string `json:"phone_query"`
SocialQuery string `json:"social_query"`
CryptoAddressQuery string `json:"crypto_address_query"`
PrintBalance bool `json:"print_balance"`
CredsOnly bool `json:"creds_only"`
Debug bool `json:"debug"`
}
func (QueryOptions) TableName() string {
return "query_options"
}
func NewQueryOptions(maxRecords, maxRequests, startingPage int, outputFormat, outputFile, usernameQuery, emailQuery, ipQuery, passQuery, hashQuery, nameQuery, domainQuery, vinQuery, licensePlateQuery, addressQuery, phoneQuery, socialQuery, cryptoAddressQuery string, regexMatch, wildcardMatch, printBalance, credsOnly, debug bool) *QueryOptions {
return &QueryOptions{
MaxRecords: maxRecords,
MaxRequests: maxRequests,
StartingPage: startingPage,
OutputFormat: files.GetFileType(outputFormat),
OutputFile: outputFile,
PrintBalance: printBalance,
CredsOnly: credsOnly,
RegexMatch: regexMatch,
WildcardMatch: wildcardMatch,
UsernameQuery: usernameQuery,
EmailQuery: emailQuery,
IpQuery: ipQuery,
PassQuery: passQuery,
HashQuery: hashQuery,
NameQuery: nameQuery,
DomainQuery: domainQuery,
VinQuery: vinQuery,
LicensePlateQuery: licensePlateQuery,
AddressQuery: addressQuery,
PhoneQuery: phoneQuery,
SocialQuery: socialQuery,
CryptoAddressQuery: cryptoAddressQuery,
Debug: debug,
}
}
type DehashedSearchRequest struct {
Page int `json:"page"`
Query string `json:"query"`
Size int `json:"size"`
Wildcard bool `json:"wildcard"`
Regex bool `json:"regex"`
DeDupe bool `json:"de_dupe"`
}
type DehashedResponse struct {
Balance int `json:"balance"`
Entries []Result `json:"entries"`
Success bool `json:"success"`
Took string `json:"took"`
TotalResults int `json:"total"`
}
type Result struct {
gorm.Model
DehashedId string `json:"id" xml:"id" yaml:"id" gorm:"uniqueIndex"`
Email []string `json:"email,omitempty" xml:"email,omitempty" yaml:"email,omitempty" gorm:"serializer:json"`
IpAddress []string `json:"ip_address,omitempty" xml:"ip_address,omitempty" yaml:"ip_address,omitempty" gorm:"serializer:json"`
Username []string `json:"username,omitempty" xml:"username,omitempty" yaml:"username,omitempty" gorm:"serializer:json"`
Password []string `json:"password,omitempty" xml:"password,omitempty" yaml:"password,omitempty" gorm:"serializer:json"`
HashedPassword []string `json:"hashed_password,omitempty" xml:"hashed_password,omitempty" yaml:"hashed_password,omitempty" gorm:"serializer:json"`
HashType string `json:"hash_type,omitempty" xml:"hash_type,omitempty" yaml:"hash_type,omitempty"`
Name []string `json:"name,omitempty" xml:"name,omitempty" yaml:"name,omitempty" gorm:"serializer:json"`
Vin []string `json:"vin,omitempty" xml:"vin,omitempty" yaml:"vin,omitempty" gorm:"serializer:json"`
LicensePlate []string `json:"license_plate,omitempty" xml:"license_plate,omitempty" yaml:"license_plate,omitempty" gorm:"serializer:json"`
Url []string `json:"url,omitempty" xml:"url,omitempty" yaml:"url,omitempty" gorm:"serializer:json"`
Social []string `json:"social,omitempty" xml:"social,omitempty" yaml:"social,omitempty" gorm:"serializer:json"`
CryptoCurrencyAddress []string `json:"cryptocurrency_address,omitempty" xml:"cryptocurrency_address,omitempty" yaml:"cryptocurrency_address,omitempty" gorm:"serializer:json"`
Address []string `json:"address,omitempty" xml:"address,omitempty" yaml:"address,omitempty" gorm:"serializer:json"`
Phone []string `json:"phone,omitempty" xml:"phone,omitempty" yaml:"phone,omitempty" gorm:"serializer:json"`
Company []string `json:"company,omitempty" xml:"company,omitempty" yaml:"company,omitempty" gorm:"serializer:json"`
DatabaseName string `json:"database_name,omitempty" xml:"database_name,omitempty" yaml:"database_name,omitempty"`
}
func (Result) TableName() string {
return "results"
}
type DehashedResults struct {
Results []Result `json:"results"`
}
func (dr *DehashedResults) ExtractCredentials() []Creds {
var creds []Creds
results := dr.Results
for _, r := range results {
if len(r.Password) > 0 {
// Get first email if available
email := ""
if len(r.Email) > 0 {
email = r.Email[0]
}
// Get first password
password := r.Password[0]
cred := Creds{Email: email, Password: password}
creds = append(creds, cred)
}
}
go func() {
err := StoreDehashedCreds(creds)
if err != nil {
zap.L().Error("store_creds",
zap.String("message", "failed to store creds"),
zap.Error(err),
)
fmt.Printf("Error Storing Results: %v", err)
}
}()
return creds
}
type Creds struct {
gorm.Model
Email string `json:"email" yaml:"email" xml:"email" gorm:"uniqueIndex:idx_email_username_password"`
Username string `json:"username" yaml:"username" xml:"username" gorm:"uniqueIndex:idx_email_username_password"`
Password string `json:"password" yaml:"password" xml:"password" gorm:"uniqueIndex:idx_email_username_password"`
}
func (Creds) TableName() string {
return "creds"
}
func (c Creds) ToString() string {
return fmt.Sprintf("%s%s%s", c.Username, "%", c.Password)
}
func StoreDehashedResults(results DehashedResults) error {
if len(results.Results) == 0 {
return nil
}
zap.L().Info("Storing results", zap.Int("count", len(results.Results)))
db := GetDB()
// Use batch insert with conflict handling
const batchSize = 100
var lastErr error
// Extract the slice of results
resultSlice := results.Results
for i := 0; i < len(resultSlice); i += batchSize {
end := i + batchSize
if end > len(resultSlice) {
end = len(resultSlice)
}
batch := resultSlice[i:end]
// Use Clauses with OnConflict DoNothing to skip conflicts
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
if err != nil {
zap.L().Warn("Error storing some results", zap.Error(err))
lastErr = err
// Continue with next batch despite error
}
}
return lastErr
}
func StoreDehashedCreds(creds []Creds) error {
if len(creds) == 0 {
return nil
}
zap.L().Info("Storing credentials", zap.Int("count", len(creds)))
db := GetDB()
// Use batch insert with conflict handling
// This will insert records in batches and continue even if some fail
const batchSize = 100
var lastErr error
for i := 0; i < len(creds); i += batchSize {
end := i + batchSize
if end > len(creds) {
end = len(creds)
}
batch := creds[i:end]
// Use Clauses with OnConflict DoNothing to skip conflicts
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
if err != nil {
zap.L().Warn("Error storing some credentials", zap.Error(err))
lastErr = err
// Continue with next batch despite error
}
}
return lastErr
}
func StoreDehashedQueryOptions(queryOptions *QueryOptions) error {
db := GetDB()
return db.Create(queryOptions).Error
}
internal/sqlite/gorm.go
-319
View File
@@ -1,319 +0,0 @@
package sqlite
import (
"fmt"
"go.uber.org/zap"
"gorm.io/gorm/clause"
"os"
"path/filepath"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
"gorm.io/gorm/logger"
)
var DB *gorm.DB
// InitDB initializes the database connection
func InitDB(dbPath string) (*gorm.DB, error) {
zap.L().Info("Initializing database", zap.String("path", dbPath))
// Check if the path is a file or directory
fileInfo, err := os.Stat(dbPath)
var finalDbPath string
// If path doesn't exist or is a directory
if os.IsNotExist(err) || (err == nil && fileInfo.IsDir()) {
// Treat as directory path
if err := os.MkdirAll(dbPath, 0755); err != nil {
zap.L().Error("Failed to create database directory", zap.Error(err))
return nil, fmt.Errorf("failed to create database directory: %w", err)
}
finalDbPath = filepath.Join(dbPath, "dehashed.sqlite")
} else {
// Treat as file path
// Ensure the directory exists
dir := filepath.Dir(dbPath)
if err := os.MkdirAll(dir, 0755); err != nil {
zap.L().Error("Failed to create parent directory for database", zap.Error(err))
return nil, fmt.Errorf("failed to create parent directory for database: %w", err)
}
finalDbPath = dbPath
}
zap.L().Info("Opening database", zap.String("finalPath", finalDbPath))
db, err := gorm.Open(sqlite.Open(finalDbPath), &gorm.Config{
Logger: logger.Default.LogMode(logger.Silent),
})
if err != nil {
zap.L().Error("Failed to connect to database", zap.Error(err))
return nil, fmt.Errorf("failed to connect to database: %w", err)
}
// Auto migrate your models
err = db.AutoMigrate(&Result{}, &Creds{}, &QueryOptions{}, &Creds{}, &WhoisRecord{}, &SubdomainRecord{},
&HistoryRecord{}, &LookupResult{}, &HunterDomainData{}, &HunterEmail{}, &PersonData{})
if err != nil {
zap.L().Error("Failed to migrate database", zap.Error(err))
return nil, fmt.Errorf("failed to migrate database: %w", err)
}
DB = db
return db, nil
}
// GetDB returns the database connection
func GetDB() *gorm.DB {
if DB == nil {
zap.L().Error("database not initialized")
fmt.Println("sqlite database not initialized")
os.Exit(1)
}
return DB
}
func StoreResults(results DehashedResults) error {
if len(results.Results) == 0 {
return nil
}
zap.L().Info("Storing results", zap.Int("count", len(results.Results)))
db := GetDB()
// Use batch insert with conflict handling
const batchSize = 100
var lastErr error
// Extract the slice of results
resultSlice := results.Results
for i := 0; i < len(resultSlice); i += batchSize {
end := i + batchSize
if end > len(resultSlice) {
end = len(resultSlice)
}
batch := resultSlice[i:end]
// Use Clauses with OnConflict DoNothing to skip conflicts
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
if err != nil {
zap.L().Warn("Error storing some results", zap.Error(err))
lastErr = err
// Continue with next batch despite error
}
}
return lastErr
}
func StoreCreds(creds []Creds) error {
if len(creds) == 0 {
return nil
}
zap.L().Info("Storing credentials", zap.Int("count", len(creds)))
db := GetDB()
// Use batch insert with conflict handling
// This will insert records in batches and continue even if some fail
const batchSize = 100
var lastErr error
for i := 0; i < len(creds); i += batchSize {
end := i + batchSize
if end > len(creds) {
end = len(creds)
}
batch := creds[i:end]
// Use Clauses with OnConflict DoNothing to skip conflicts
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
if err != nil {
zap.L().Warn("Error storing some credentials", zap.Error(err))
lastErr = err
// Continue with next batch despite error
}
}
return lastErr
}
func StoreQueryOptions(queryOptions *QueryOptions) error {
db := GetDB()
return db.Create(queryOptions).Error
}
func StoreWhoisRecord(whoisRecord WhoisRecord) error {
// Create a pointer to the record to make it addressable
recordPtr := &whoisRecord
zap.L().Info("Storing WHOIS record",
zap.String("domain", whoisRecord.DomainName))
db := GetDB()
// Use OnConflict clause to handle duplicates
err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(recordPtr).Error
if err != nil {
zap.L().Error("store_whois_record",
zap.String("message", "failed to store whois record"),
zap.Error(err))
return err
}
return nil
}
func StoreSubdomainRecords(subdomainRecords []SubdomainRecord) error {
if len(subdomainRecords) == 0 {
return nil
}
zap.L().Info("Storing subdomain records", zap.Int("count", len(subdomainRecords)))
db := GetDB()
// Use batch insert with conflict handling
const batchSize = 100
var lastErr error
for i := 0; i < len(subdomainRecords); i += batchSize {
end := i + batchSize
if end > len(subdomainRecords) {
end = len(subdomainRecords)
}
batch := subdomainRecords[i:end]
// Use Clauses with OnConflict DoNothing to skip conflicts
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
if err != nil {
zap.L().Warn("Error storing some subdomain records", zap.Error(err))
lastErr = err
// Continue with next batch despite error
}
}
return lastErr
}
func StoreHistoryRecord(historyRecords []HistoryRecord) error {
if len(historyRecords) == 0 {
return nil
}
zap.L().Info("Storing history records", zap.Int("count", len(historyRecords)))
db := GetDB()
// Use batch insert with conflict handling
const batchSize = 100
var lastErr error
for i := 0; i < len(historyRecords); i += batchSize {
end := i + batchSize
if end > len(historyRecords) {
end = len(historyRecords)
}
batch := historyRecords[i:end]
// Use Clauses with OnConflict DoNothing to skip conflicts
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
if err != nil {
zap.L().Warn("Error storing some history records", zap.Error(err))
lastErr = err
// Continue with next batch despite error
}
}
return lastErr
}
func StoreIPLookup(ipLookup []LookupResult) error {
if len(ipLookup) == 0 {
return nil
}
zap.L().Info("Storing IP lookup records", zap.Int("count", len(ipLookup)))
db := GetDB()
// Use batch insert with conflict handling
const batchSize = 100
var lastErr error
for i := 0; i < len(ipLookup); i += batchSize {
end := i + batchSize
if end > len(ipLookup) {
end = len(ipLookup)
}
batch := ipLookup[i:end]
// Use Clauses with OnConflict DoNothing to skip conflicts
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
if err != nil {
zap.L().Warn("Error storing some IP lookup records", zap.Error(err))
lastErr = err
// Continue with next batch despite error
}
}
return lastErr
}
func StoreHunterDomain(hunterDomain HunterDomainData) error {
db := GetDB()
// Use OnConflict clause to handle duplicates
err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(&hunterDomain).Error
if err != nil {
zap.L().Error("store_hunter_domain",
zap.String("message", "failed to store hunter domain"),
zap.Error(err))
return err
}
return nil
}
func StoreHunterEmails(hunterEmails []HunterEmail) error {
if len(hunterEmails) == 0 {
return nil
}
zap.L().Info("Storing hunter emails", zap.Int("count", len(hunterEmails)))
db := GetDB()
// Use batch insert with conflict handling
const batchSize = 100
var lastErr error
for i := 0; i < len(hunterEmails); i += batchSize {
end := i + batchSize
if end > len(hunterEmails) {
end = len(hunterEmails)
}
batch := hunterEmails[i:end]
// Use Clauses with OnConflict DoNothing to skip conflicts
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
if err != nil {
zap.L().Warn("Error storing some hunter emails", zap.Error(err))
lastErr = err
// Continue with next batch despite error
}
}
return lastErr
}
func StorePersonData(personData PersonData) error {
db := GetDB()
// Use OnConflict clause to handle duplicates
err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(&personData).Error
if err != nil {
zap.L().Error("store_person_data",
zap.String("message", "failed to store person data"),
zap.Error(err))
return err
}
return nil
}
internal/sqlite/hunter.io.go
+65 -2
View File
@@ -3,7 +3,9 @@ package sqlite
import (
"fmt"
"github.com/charmbracelet/lipgloss/tree"
"go.uber.org/zap"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
// HunterDomainSearchResult represents the response from Hunter.io domain search API
@@ -14,8 +16,8 @@ type HunterDomainSearchResult struct {
// HunterDomainData contains the main domain information
type HunterDomainData struct {
IString
gorm.Model
IString `gorm:"-"`
Domain string `json:"domain" gorm:"unique"`
Disposable bool `json:"disposable"`
Webmail bool `json:"webmail"`
@@ -512,8 +514,8 @@ type HunterPersonEnrichmentResponse struct {
// PersonData contains the detailed person information
type PersonData struct {
IString
gorm.Model
IString `gorm:"-"`
ID string `json:"id"`
Name PersonName `json:"name" gorm:"embedded;embeddedPrefix:name_"`
Email string `json:"email" gorm:"unique"`
@@ -755,3 +757,64 @@ func (c *HunterCombinedEnrichmentResponse) String() string {
c.Data.Person.String(),
c.Data.Company.String())
}
func StoreHunterDomain(hunterDomain HunterDomainData) error {
db := GetDB()
// Use OnConflict clause to handle duplicates
err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(&hunterDomain).Error
if err != nil {
zap.L().Error("store_hunter_domain",
zap.String("message", "failed to store hunter domain"),
zap.Error(err))
return err
}
return nil
}
func StoreHunterEmails(hunterEmails []HunterEmail) error {
if len(hunterEmails) == 0 {
return nil
}
zap.L().Info("Storing hunter emails", zap.Int("count", len(hunterEmails)))
db := GetDB()
// Use batch insert with conflict handling
const batchSize = 100
var lastErr error
for i := 0; i < len(hunterEmails); i += batchSize {
end := i + batchSize
if end > len(hunterEmails) {
end = len(hunterEmails)
}
batch := hunterEmails[i:end]
// Use Clauses with OnConflict DoNothing to skip conflicts
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
if err != nil {
zap.L().Warn("Error storing some hunter emails", zap.Error(err))
lastErr = err
// Continue with next batch despite error
}
}
return lastErr
}
func StoreHunterPersonData(personData PersonData) error {
db := GetDB()
// Use OnConflict clause to handle duplicates
err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(&personData).Error
if err != nil {
zap.L().Error("store_person_data",
zap.String("message", "failed to store person data"),
zap.Error(err))
return err
}
return nil
}
internal/sqlite/query.go
-20
View File
@@ -1,20 +0,0 @@
package sqlite
type DehashedSearchRequest struct {
Page int `json:"page"`
Query string `json:"query"`
Size int `json:"size"`
Wildcard bool `json:"wildcard"`
Regex bool `json:"regex"`
DeDupe bool `json:"de_dupe"`
}
func NewDehashedSearchRequest(size int, wildcard, regex bool) *DehashedSearchRequest {
return &DehashedSearchRequest{
Page: 0,
Size: size,
Wildcard: false,
Regex: false,
DeDupe: true,
}
}
internal/sqlite/result.go
-94
View File
@@ -1,94 +0,0 @@
package sqlite
import (
"encoding/json"
"fmt"
"go.uber.org/zap"
"gorm.io/gorm"
"io"
"os"
)
type DehashedResponse struct {
Balance int `json:"balance"`
Entries []Result `json:"entries"`
Success bool `json:"success"`
Took string `json:"took"`
TotalResults int `json:"total"`
}
type Result struct {
gorm.Model
DehashedId string `json:"id" xml:"id" yaml:"id" gorm:"uniqueIndex"`
Email []string `json:"email,omitempty" xml:"email,omitempty" yaml:"email,omitempty" gorm:"serializer:json"`
IpAddress []string `json:"ip_address,omitempty" xml:"ip_address,omitempty" yaml:"ip_address,omitempty" gorm:"serializer:json"`
Username []string `json:"username,omitempty" xml:"username,omitempty" yaml:"username,omitempty" gorm:"serializer:json"`
Password []string `json:"password,omitempty" xml:"password,omitempty" yaml:"password,omitempty" gorm:"serializer:json"`
HashedPassword []string `json:"hashed_password,omitempty" xml:"hashed_password,omitempty" yaml:"hashed_password,omitempty" gorm:"serializer:json"`
HashType string `json:"hash_type,omitempty" xml:"hash_type,omitempty" yaml:"hash_type,omitempty"`
Name []string `json:"name,omitempty" xml:"name,omitempty" yaml:"name,omitempty" gorm:"serializer:json"`
Vin []string `json:"vin,omitempty" xml:"vin,omitempty" yaml:"vin,omitempty" gorm:"serializer:json"`
LicensePlate []string `json:"license_plate,omitempty" xml:"license_plate,omitempty" yaml:"license_plate,omitempty" gorm:"serializer:json"`
Url []string `json:"url,omitempty" xml:"url,omitempty" yaml:"url,omitempty" gorm:"serializer:json"`
Social []string `json:"social,omitempty" xml:"social,omitempty" yaml:"social,omitempty" gorm:"serializer:json"`
CryptoCurrencyAddress []string `json:"cryptocurrency_address,omitempty" xml:"cryptocurrency_address,omitempty" yaml:"cryptocurrency_address,omitempty" gorm:"serializer:json"`
Address []string `json:"address,omitempty" xml:"address,omitempty" yaml:"address,omitempty" gorm:"serializer:json"`
Phone []string `json:"phone,omitempty" xml:"phone,omitempty" yaml:"phone,omitempty" gorm:"serializer:json"`
Company []string `json:"company,omitempty" xml:"company,omitempty" yaml:"company,omitempty" gorm:"serializer:json"`
DatabaseName string `json:"database_name,omitempty" xml:"database_name,omitempty" yaml:"database_name,omitempty"`
}
func (Result) TableName() string {
return "results"
}
type DehashedResults struct {
Results []Result `json:"results"`
}
func (dr *DehashedResults) ExtractCredentials() []Creds {
var creds []Creds
results := dr.Results
for _, r := range results {
if len(r.Password) > 0 {
// Get first email if available
email := ""
if len(r.Email) > 0 {
email = r.Email[0]
}
// Get first password
password := r.Password[0]
cred := Creds{Email: email, Password: password}
creds = append(creds, cred)
}
}
go func() {
err := StoreCreds(creds)
if err != nil {
zap.L().Error("store_creds",
zap.String("message", "failed to store creds"),
zap.Error(err),
)
fmt.Printf("Error Storing Results: %v", err)
}
}()
return creds
}
func NewDehashedResults(body io.Reader) ([]Result, int, int) {
var response DehashedResponse
err := json.NewDecoder(body).Decode(&response)
if err != nil {
fmt.Printf("Error Parsing Response Body: %v", err)
os.Exit(-1)
}
return response.Entries, response.Balance, response.TotalResults
}
internal/sqlite/structs.go
-89
View File
@@ -1,11 +1,5 @@
package sqlite
import (
"crowsnest/internal/files"
"fmt"
"gorm.io/gorm"
)
type IString interface {
String() string
}
@@ -30,15 +24,6 @@ type DBOptions struct {
DisplayFields []string // Fields to display in output
}
func NewDBOptions() *DBOptions {
return &DBOptions{
Limit: 100, // Default limit
ExactMatch: false,
NonEmptyFields: []string{},
DisplayFields: []string{},
}
}
func (o *DBOptions) Empty() bool {
return o.Username == "" && o.Email == "" && o.IPAddress == "" &&
o.Password == "" && o.HashedPassword == "" && o.Name == "" &&
@@ -46,77 +31,3 @@ func (o *DBOptions) Empty() bool {
o.Phone == "" && o.Social == "" && o.CryptoCurrencyAddress == "" && o.Domain == "" &&
len(o.NonEmptyFields) == 0
}
type QueryOptions struct {
gorm.Model
MaxRecords int `json:"max_records"`
MaxRequests int `json:"max_requests"`
StartingPage int `json:"starting_page"`
OutputFormat files.FileType `json:"output_format"`
OutputFile string `json:"output_file"`
RegexMatch bool `json:"regex_match"`
WildcardMatch bool `json:"wildcard_match"`
UsernameQuery string `json:"username_query"`
EmailQuery string `json:"email_query"`
IpQuery string `json:"ip_query"`
PassQuery string `json:"pass_query"`
HashQuery string `json:"hash_query"`
NameQuery string `json:"name_query"`
DomainQuery string `json:"domain_query"`
VinQuery string `json:"vin_query"`
LicensePlateQuery string `json:"license_plate_query"`
AddressQuery string `json:"address_query"`
PhoneQuery string `json:"phone_query"`
SocialQuery string `json:"social_query"`
CryptoAddressQuery string `json:"crypto_address_query"`
PrintBalance bool `json:"print_balance"`
CredsOnly bool `json:"creds_only"`
Debug bool `json:"debug"`
}
func (QueryOptions) TableName() string {
return "query_options"
}
func NewQueryOptions(maxRecords, maxRequests, startingPage int, outputFormat, outputFile, usernameQuery, emailQuery, ipQuery, passQuery, hashQuery, nameQuery, domainQuery, vinQuery, licensePlateQuery, addressQuery, phoneQuery, socialQuery, cryptoAddressQuery string, regexMatch, wildcardMatch, printBalance, credsOnly, debug bool) *QueryOptions {
return &QueryOptions{
MaxRecords: maxRecords,
MaxRequests: maxRequests,
StartingPage: startingPage,
OutputFormat: files.GetFileType(outputFormat),
OutputFile: outputFile,
PrintBalance: printBalance,
CredsOnly: credsOnly,
RegexMatch: regexMatch,
WildcardMatch: wildcardMatch,
UsernameQuery: usernameQuery,
EmailQuery: emailQuery,
IpQuery: ipQuery,
PassQuery: passQuery,
HashQuery: hashQuery,
NameQuery: nameQuery,
DomainQuery: domainQuery,
VinQuery: vinQuery,
LicensePlateQuery: licensePlateQuery,
AddressQuery: addressQuery,
PhoneQuery: phoneQuery,
SocialQuery: socialQuery,
CryptoAddressQuery: cryptoAddressQuery,
Debug: debug,
}
}
type Creds struct {
gorm.Model
Email string `json:"email" yaml:"email" xml:"email" gorm:"uniqueIndex:idx_email_username_password"`
Username string `json:"username" yaml:"username" xml:"username" gorm:"uniqueIndex:idx_email_username_password"`
Password string `json:"password" yaml:"password" xml:"password" gorm:"uniqueIndex:idx_email_username_password"`
}
func (Creds) TableName() string {
return "creds"
}
func (c Creds) ToString() string {
return fmt.Sprintf("%s%s%s", c.Username, "%", c.Password)
}
internal/sqlite/tables.go
-68
View File
@@ -1,68 +0,0 @@
package sqlite
import "strings"
type Table int64
const (
ResultsTable Table = iota
RunsTable
CredsTable
WhoIsTable
SubdomainsTable
HistoryTable
LookupTable
HunterDomainTable
HunterEmailTable
UnknownTable
)
func GetTable(userInput string) Table {
switch strings.ToLower(userInput) {
case "results":
return ResultsTable
case "runs":
return RunsTable
case "creds":
return CredsTable
case "whois":
return WhoIsTable
case "subdomains":
return SubdomainsTable
case "history":
return HistoryTable
case "lookup":
return LookupTable
case "hunter_domain":
return HunterDomainTable
case "hunter_email":
return HunterEmailTable
default:
return UnknownTable
}
}
func (t Table) Object() interface{} {
switch t {
case ResultsTable:
return Result{}
case RunsTable:
return QueryOptions{}
case CredsTable:
return Creds{}
case WhoIsTable:
return WhoisRecord{}
case SubdomainsTable:
return SubdomainRecord{}
case HistoryTable:
return HistoryRecord{}
case LookupTable:
return LookupResult{}
case HunterDomainTable:
return HunterDomainData{}
case HunterEmailTable:
return HunterEmail{}
default:
return nil
}
}
internal/sqlite/whois.go
+116
View File
@@ -2,7 +2,9 @@ package sqlite
import (
"fmt"
"go.uber.org/zap"
"gorm.io/gorm"
"gorm.io/gorm/clause"
"strings"
)
@@ -512,3 +514,117 @@ func formatWhoisContact(sb *strings.Builder, contact Contact, indent string) {
sb.WriteString(indent + "Raw Text: " + rawTextPreview + "\n")
}
}
func StoreWhoisRecord(whoisRecord WhoisRecord) error {
// Create a pointer to the record to make it addressable
recordPtr := &whoisRecord
zap.L().Info("Storing WHOIS record",
zap.String("domain", whoisRecord.DomainName))
db := GetDB()
// Use OnConflict clause to handle duplicates
err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(recordPtr).Error
if err != nil {
zap.L().Error("store_whois_record",
zap.String("message", "failed to store whois record"),
zap.Error(err))
return err
}
return nil
}
func StoreWhoisSubdomainRecords(subdomainRecords []SubdomainRecord) error {
if len(subdomainRecords) == 0 {
return nil
}
zap.L().Info("Storing subdomain records", zap.Int("count", len(subdomainRecords)))
db := GetDB()
// Use batch insert with conflict handling
const batchSize = 100
var lastErr error
for i := 0; i < len(subdomainRecords); i += batchSize {
end := i + batchSize
if end > len(subdomainRecords) {
end = len(subdomainRecords)
}
batch := subdomainRecords[i:end]
// Use Clauses with OnConflict DoNothing to skip conflicts
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
if err != nil {
zap.L().Warn("Error storing some subdomain records", zap.Error(err))
lastErr = err
// Continue with next batch despite error
}
}
return lastErr
}
func StoreWhoisHistoryRecords(historyRecords []HistoryRecord) error {
if len(historyRecords) == 0 {
return nil
}
zap.L().Info("Storing history records", zap.Int("count", len(historyRecords)))
db := GetDB()
// Use batch insert with conflict handling
const batchSize = 100
var lastErr error
for i := 0; i < len(historyRecords); i += batchSize {
end := i + batchSize
if end > len(historyRecords) {
end = len(historyRecords)
}
batch := historyRecords[i:end]
// Use Clauses with OnConflict DoNothing to skip conflicts
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
if err != nil {
zap.L().Warn("Error storing some history records", zap.Error(err))
lastErr = err
// Continue with next batch despite error
}
}
return lastErr
}
func StoreWhoisLookup(lookup []LookupResult) error {
if len(lookup) == 0 {
return nil
}
zap.L().Info("Storing IP lookup records", zap.Int("count", len(lookup)))
db := GetDB()
// Use batch insert with conflict handling
const batchSize = 100
var lastErr error
for i := 0; i < len(lookup); i += batchSize {
end := i + batchSize
if end > len(lookup) {
end = len(lookup)
}
batch := lookup[i:end]
// Use Clauses with OnConflict DoNothing to skip conflicts
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
if err != nil {
zap.L().Warn("Error storing some IP lookup records", zap.Error(err))
lastErr = err
// Continue with next batch despite error
}
}
return lastErr
}
internal/whois/whois.go
+3 -3
View File
@@ -552,7 +552,7 @@ func (w *DehashedWhoIs) WhoisIP(ipAddress string) ([]sqlite.LookupResult, error)
})
}
sqlite.StoreIPLookup(lookups)
sqlite.StoreWhoisLookup(lookups)
return lookups, nil
}
@@ -702,7 +702,7 @@ func (w *DehashedWhoIs) WhoisMX(mxHostname string) ([]sqlite.LookupResult, error
})
}
sqlite.StoreIPLookup(mxLookups)
sqlite.StoreWhoisLookup(mxLookups)
return mxLookups, nil
}
@@ -849,7 +849,7 @@ func (w *DehashedWhoIs) WhoisNS(nsHostname string) ([]sqlite.LookupResult, error
})
}
sqlite.StoreIPLookup(nsLookups)
sqlite.StoreWhoisLookup(nsLookups)
return nsLookups, nil
}