KrakenTech/CrowsNest
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor user and credential handling: rename Creds to User, update database migrations, and add targets subcommand for exporting users and subdomains

Browse Source
This commit is contained in:
Evan Hosinski
2025-06-03 19:29:10 -04:00
parent 9a22445e55
commit 0bd9347074
16 changed files with 710 additions and 523 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/dehashed/dehashed.go
+3 -3
View File
@@ -211,9 +211,9 @@ func (dh *Dehasher) buildRequest() {
func (dh *Dehasher) parseResults() {
zap.L().Info("extracting_credentials")
results := dh.client.GetResults()
creds := results.ExtractCredentials()
creds := results.ExtractUsers()
fmt.Printf(" [+] Discovered %d Credentials\n", len(creds))
err := sqlite.StoreDehashedCreds(creds)
err := sqlite.StoreUsers(creds)
if err != nil {
zap.L().Error("store_creds",
zap.String("message", "failed to store creds"),
@@ -284,7 +284,7 @@ func (dh *Dehasher) parseResults() {
if dh.debug {
debug.PrintInfo("extracting credentials")
}
creds := results.ExtractCredentials()
creds := results.ExtractUsers()
if dh.debug {
debug.PrintInfo("writing credentials to file")
}
internal/export/dehashed.go
+1 -1
View File
@@ -14,7 +14,7 @@ import (
"time"
)
func WriteCredsToFile(creds []sqlite.Creds, outputFile string, fileType files.FileType) error {
func WriteCredsToFile(creds []sqlite.User, outputFile string, fileType files.FileType) error {
var data []byte
var err error
internal/sqlite/db.go
+3 -3
View File
@@ -51,8 +51,8 @@ func InitDB(dbPath string) (*gorm.DB, error) {
}
// Auto migrate your models
err = db.AutoMigrate(&Result{}, &Creds{}, &QueryOptions{}, &Creds{}, &WhoisRecord{}, &SubdomainRecord{},
&HistoryRecord{}, &LookupResult{}, &HunterDomainData{}, &HunterEmail{}, &PersonData{})
err = db.AutoMigrate(&Result{}, &User{}, &QueryOptions{}, &User{}, &WhoisRecord{}, &HistoryRecord{},
&LookupResult{}, &HunterDomainData{}, &HunterEmail{}, &PersonData{}, &Subdomain{})
if err != nil {
zap.L().Error("Failed to migrate database", zap.Error(err))
return nil, fmt.Errorf("failed to migrate database: %w", err)
@@ -122,7 +122,7 @@ func (t Table) Object() interface{} {
case RunsTable:
return QueryOptions{}
case CredsTable:
return Creds{}
return User{}
case WhoIsTable:
return WhoisRecord{}
case SubdomainsTable:
internal/sqlite/structs.go → internal/sqlite/dbOptions.go
View File
internal/sqlite/dehashed.go
+13 -46
View File
@@ -106,15 +106,15 @@ type Result struct {
}
func (Result) TableName() string {
return "results"
return "dehashed"
}
type DehashedResults struct {
Results []Result `json:"results"`
}
func (dr *DehashedResults) ExtractCredentials() []Creds {
var creds []Creds
func (dr *DehashedResults) ExtractUsers() []User {
var creds []User
results := dr.Results
@@ -126,16 +126,22 @@ func (dr *DehashedResults) ExtractCredentials() []Creds {
email = r.Email[0]
}
// Get first username if available
username := ""
if len(r.Username) > 0 {
username = r.Username[0]
}
// Get first password
password := r.Password[0]
cred := Creds{Email: email, Password: password}
cred := User{Email: email, Password: password, Username: username}
creds = append(creds, cred)
}
}
go func() {
err := StoreDehashedCreds(creds)
err := StoreUsers(creds)
if err != nil {
zap.L().Error("store_creds",
zap.String("message", "failed to store creds"),
@@ -148,18 +154,11 @@ func (dr *DehashedResults) ExtractCredentials() []Creds {
return creds
}
type Creds struct {
gorm.Model
Email string `json:"email" yaml:"email" xml:"email" gorm:"uniqueIndex:idx_email_username_password"`
Username string `json:"username" yaml:"username" xml:"username" gorm:"uniqueIndex:idx_email_username_password"`
Password string `json:"password" yaml:"password" xml:"password" gorm:"uniqueIndex:idx_email_username_password"`
}
func (Creds) TableName() string {
func (User) TableName() string {
return "creds"
}
func (c Creds) ToString() string {
func (c User) ToString() string {
return fmt.Sprintf("%s%s%s", c.Username, "%", c.Password)
}
@@ -197,38 +196,6 @@ func StoreDehashedResults(results DehashedResults) error {
return lastErr
}
func StoreDehashedCreds(creds []Creds) error {
if len(creds) == 0 {
return nil
}
zap.L().Info("Storing credentials", zap.Int("count", len(creds)))
db := GetDB()
// Use batch insert with conflict handling
// This will insert records in batches and continue even if some fail
const batchSize = 100
var lastErr error
for i := 0; i < len(creds); i += batchSize {
end := i + batchSize
if end > len(creds) {
end = len(creds)
}
batch := creds[i:end]
// Use Clauses with OnConflict DoNothing to skip conflicts
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
if err != nil {
zap.L().Warn("Error storing some credentials", zap.Error(err))
lastErr = err
// Continue with next batch despite error
}
}
return lastErr
}
func StoreDehashedQueryOptions(queryOptions *QueryOptions) error {
db := GetDB()
return db.Create(queryOptions).Error
internal/sqlite/ports.go
+1
View File
@@ -0,0 +1 @@
package sqlite
internal/sqlite/subdomains.go
+45
View File
@@ -0,0 +1,45 @@
package sqlite
import (
"go.uber.org/zap"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
type Subdomain struct {
gorm.Model
Domain string `json:"domain" yaml:"domain" xml:"domain"`
Subdomain string `json:"subdomain" yaml:"subdomain" xml:"subdomain" gorm:"uniqueIndex:idx_subdomain"`
}
func StoreSubdomains(subs []Subdomain) error {
if len(subs) == 0 {
return nil
}
zap.L().Info("Storing subdomains", zap.Int("count", len(subs)))
db := GetDB()
// Use batch insert with conflict handling
// This will insert records in batches and continue even if some fail
const batchSize = 100
var lastErr error
for i := 0; i < len(subs); i += batchSize {
end := i + batchSize
if end > len(subs) {
end = len(subs)
}
batch := subs[i:end]
// Use Clauses with OnConflict DoNothing to skip conflicts
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
if err != nil {
zap.L().Warn("Error storing some credentials", zap.Error(err))
lastErr = err
// Continue with next batch despite error
}
}
return lastErr
}
internal/sqlite/users.go
+58
View File
@@ -0,0 +1,58 @@
package sqlite
import (
"go.uber.org/zap"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
type User struct {
gorm.Model
Company string `json:"company" yaml:"company" xml:"company"`
Position string `json:"position" yaml:"position" xml:"position"`
Department string `json:"department" yaml:"department" xml:"department"`
PhoneNumber string `json:"phone_number" yaml:"phone_number" xml:"phone_number"`
FullName string `json:"full_name" yaml:"full_name" xml:"full_name"`
Phone string `json:"phone" yaml:"phone" xml:"phone"`
Linkedin string `json:"linkedin" yaml:"linkedin" xml:"linkedin"`
Twitter string `json:"twitter" yaml:"twitter" xml:"twitter"`
Facebook string `json:"facebook" yaml:"facebook" xml:"facebook"`
Instagram string `json:"instagram" yaml:"instagram" xml:"instagram"`
Youtube string `json:"youtube" yaml:"youtube" xml:"youtube"`
Gravatar string `json:"gravatar" yaml:"gravatar" xml:"gravatar"`
Email string `json:"email" yaml:"email" xml:"email" gorm:"uniqueIndex:idx_email_username_password"`
Username string `json:"username" yaml:"username" xml:"username" gorm:"uniqueIndex:idx_email_username_password"`
Password string `json:"password" yaml:"password" xml:"password" gorm:"uniqueIndex:idx_email_username_password"`
}
func StoreUsers(users []User) error {
if len(users) == 0 {
return nil
}
zap.L().Info("Storing credentials", zap.Int("count", len(users)))
db := GetDB()
// Use batch insert with conflict handling
// This will insert records in batches and continue even if some fail
const batchSize = 100
var lastErr error
for i := 0; i < len(users); i += batchSize {
end := i + batchSize
if end > len(users) {
end = len(users)
}
batch := users[i:end]
// Use Clauses with OnConflict DoNothing to skip conflicts
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
if err != nil {
zap.L().Warn("Error storing some credentials", zap.Error(err))
lastErr = err
// Continue with next batch despite error
}
}
return lastErr
}
internal/sqlite/whois.go
-31
View File
@@ -536,37 +536,6 @@ func StoreWhoisRecord(whoisRecord WhoisRecord) error {
return nil
}
func StoreWhoisSubdomainRecords(subdomainRecords []SubdomainRecord) error {
if len(subdomainRecords) == 0 {
return nil
}
zap.L().Info("Storing subdomain records", zap.Int("count", len(subdomainRecords)))
db := GetDB()
// Use batch insert with conflict handling
const batchSize = 100
var lastErr error
for i := 0; i < len(subdomainRecords); i += batchSize {
end := i + batchSize
if end > len(subdomainRecords) {
end = len(subdomainRecords)
}
batch := subdomainRecords[i:end]
// Use Clauses with OnConflict DoNothing to skip conflicts
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
if err != nil {
zap.L().Warn("Error storing some subdomain records", zap.Error(err))
lastErr = err
// Continue with next batch despite error
}
}
return lastErr
}
func StoreWhoisHistoryRecords(historyRecords []HistoryRecord) error {
if len(historyRecords) == 0 {
return nil