first commit
This commit is contained in:
Ar1ste1a
@@ -0,0 +1,365 @@
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"go.uber.org/zap"
|
||||
"gorm.io/gorm"
|
||||
"time"
|
||||
)
|
||||
|
||||
// QueryResults queries the database for results based on the provided options
|
||||
func QueryResults(options *DBOptions) ([]Result, error) {
|
||||
db := GetDB()
|
||||
var results []Result
|
||||
query := db.Model(&Result{})
|
||||
|
||||
// Apply filters based on the provided options
|
||||
query = applyFilters(query, options)
|
||||
|
||||
// Apply limit
|
||||
if options.Limit > 0 {
|
||||
query = query.Limit(options.Limit)
|
||||
}
|
||||
|
||||
// Execute the query
|
||||
if err := query.Find(&results).Error; err != nil {
|
||||
zap.L().Error("query_results",
|
||||
zap.String("message", "failed to query results"),
|
||||
zap.Error(err),
|
||||
)
|
||||
return nil, fmt.Errorf("failed to query results: %w", err)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// applyFilters applies filters to the query based on the provided options
|
||||
func applyFilters(query *gorm.DB, options *DBOptions) *gorm.DB {
|
||||
// Helper function to apply filter based on exact match setting
|
||||
applyFilter := func(field, value string) *gorm.DB {
|
||||
if value == "" {
|
||||
return query
|
||||
}
|
||||
|
||||
if options.ExactMatch {
|
||||
return query.Where(field+" = ?", value)
|
||||
} else {
|
||||
return query.Where(field+" LIKE ?", "%"+value+"%")
|
||||
}
|
||||
}
|
||||
|
||||
// Apply filters for each field if provided
|
||||
if options.Email != "" {
|
||||
query = applyFilter("email", options.Email)
|
||||
}
|
||||
|
||||
if options.Username != "" {
|
||||
query = applyFilter("username", options.Username)
|
||||
}
|
||||
|
||||
if options.IPAddress != "" {
|
||||
query = applyFilter("ip_address", options.IPAddress)
|
||||
}
|
||||
|
||||
if options.Password != "" {
|
||||
query = applyFilter("password", options.Password)
|
||||
}
|
||||
|
||||
if options.HashedPassword != "" {
|
||||
query = applyFilter("hashed_password", options.HashedPassword)
|
||||
}
|
||||
|
||||
if options.Name != "" {
|
||||
query = applyFilter("name", options.Name)
|
||||
}
|
||||
|
||||
if options.Vin != "" {
|
||||
query = applyFilter("vin", options.Vin)
|
||||
}
|
||||
|
||||
if options.LicensePlate != "" {
|
||||
query = applyFilter("license_plate", options.LicensePlate)
|
||||
}
|
||||
|
||||
if options.Address != "" {
|
||||
query = applyFilter("address", options.Address)
|
||||
}
|
||||
|
||||
if options.Phone != "" {
|
||||
query = applyFilter("phone", options.Phone)
|
||||
}
|
||||
|
||||
if options.Social != "" {
|
||||
query = applyFilter("social", options.Social)
|
||||
}
|
||||
|
||||
if options.CryptoCurrencyAddress != "" {
|
||||
query = applyFilter("cryptocurrency_address", options.CryptoCurrencyAddress)
|
||||
}
|
||||
|
||||
if options.Domain != "" {
|
||||
query = applyFilter("url", options.Domain)
|
||||
}
|
||||
|
||||
// Apply non-empty field filters
|
||||
for _, field := range options.NonEmptyFields {
|
||||
switch field {
|
||||
case "username":
|
||||
query = query.Where("JSON_ARRAY_LENGTH(username) > 0")
|
||||
case "email":
|
||||
query = query.Where("JSON_ARRAY_LENGTH(email) > 0")
|
||||
case "ip_address", "ipaddress", "ip":
|
||||
query = query.Where("JSON_ARRAY_LENGTH(ip_address) > 0")
|
||||
case "password":
|
||||
query = query.Where("JSON_ARRAY_LENGTH(password) > 0")
|
||||
case "hashed_password", "hash":
|
||||
query = query.Where("JSON_ARRAY_LENGTH(hashed_password) > 0")
|
||||
case "name":
|
||||
query = query.Where("JSON_ARRAY_LENGTH(name) > 0")
|
||||
case "vin":
|
||||
query = query.Where("JSON_ARRAY_LENGTH(vin) > 0")
|
||||
case "license_plate", "license":
|
||||
query = query.Where("JSON_ARRAY_LENGTH(license_plate) > 0")
|
||||
case "address":
|
||||
query = query.Where("JSON_ARRAY_LENGTH(address) > 0")
|
||||
case "phone":
|
||||
query = query.Where("JSON_ARRAY_LENGTH(phone) > 0")
|
||||
case "social":
|
||||
query = query.Where("JSON_ARRAY_LENGTH(social) > 0")
|
||||
case "cryptocurrency_address", "crypto":
|
||||
query = query.Where("JSON_ARRAY_LENGTH(cryptocurrency_address) > 0")
|
||||
case "url", "domain":
|
||||
query = query.Where("JSON_ARRAY_LENGTH(url) > 0")
|
||||
}
|
||||
}
|
||||
|
||||
return query
|
||||
}
|
||||
|
||||
// GetResultsCount returns the count of results matching the provided options
|
||||
func GetResultsCount(options *DBOptions) (int64, error) {
|
||||
db := GetDB()
|
||||
var count int64
|
||||
query := db.Model(&Result{})
|
||||
|
||||
// Apply filters based on the provided options
|
||||
query = applyFilters(query, options)
|
||||
|
||||
// Count the results
|
||||
if err := query.Count(&count).Error; err != nil {
|
||||
zap.L().Error("get_results_count",
|
||||
zap.String("message", "failed to count results"),
|
||||
zap.Error(err),
|
||||
)
|
||||
return 0, fmt.Errorf("failed to count results: %w", err)
|
||||
}
|
||||
|
||||
return count, nil
|
||||
}
|
||||
|
||||
// QueryRuns queries the database for previous query runs (QueryOptions) based on the provided filters
|
||||
func QueryRuns(limit, lastXRuns int, startDate, endDate time.Time, containsQuery string) ([]QueryOptions, error) {
|
||||
db := GetDB()
|
||||
var runs []QueryOptions
|
||||
query := db.Model(&QueryOptions{})
|
||||
|
||||
// Apply date range filter if provided
|
||||
if lastXRuns > 0 {
|
||||
query = query.Order("created_at DESC").Limit(lastXRuns)
|
||||
} else if !startDate.IsZero() && !endDate.IsZero() {
|
||||
query = query.Where("created_at BETWEEN ? AND ?", startDate, endDate)
|
||||
} else if !startDate.IsZero() {
|
||||
query = query.Where("created_at >= ?", startDate)
|
||||
} else if !endDate.IsZero() {
|
||||
query = query.Where("created_at <= ?", endDate)
|
||||
}
|
||||
|
||||
// Apply query filter if provided
|
||||
if containsQuery != "" {
|
||||
// Search in all query fields
|
||||
query = query.Where(
|
||||
"username_query LIKE ? OR "+
|
||||
"email_query LIKE ? OR "+
|
||||
"ip_query LIKE ? OR "+
|
||||
"pass_query LIKE ? OR "+
|
||||
"hash_query LIKE ? OR "+
|
||||
"name_query LIKE ? OR "+
|
||||
"domain_query LIKE ? OR "+
|
||||
"vin_query LIKE ? OR "+
|
||||
"license_plate_query LIKE ? OR "+
|
||||
"address_query LIKE ? OR "+
|
||||
"phone_query LIKE ? OR "+
|
||||
"social_query LIKE ? OR "+
|
||||
"crypto_address_query LIKE ?",
|
||||
"%"+containsQuery+"%", "%"+containsQuery+"%", "%"+containsQuery+"%",
|
||||
"%"+containsQuery+"%", "%"+containsQuery+"%", "%"+containsQuery+"%",
|
||||
"%"+containsQuery+"%", "%"+containsQuery+"%", "%"+containsQuery+"%",
|
||||
"%"+containsQuery+"%", "%"+containsQuery+"%", "%"+containsQuery+"%",
|
||||
"%"+containsQuery+"%",
|
||||
)
|
||||
}
|
||||
|
||||
// Apply limit
|
||||
if limit > 0 {
|
||||
query = query.Limit(limit)
|
||||
}
|
||||
|
||||
// Order by most recent first
|
||||
query = query.Order("created_at DESC")
|
||||
|
||||
// Execute the query
|
||||
if err := query.Find(&runs).Error; err != nil {
|
||||
zap.L().Error("query_runs",
|
||||
zap.String("message", "failed to query runs"),
|
||||
zap.Error(err),
|
||||
)
|
||||
return nil, fmt.Errorf("failed to query runs: %w", err)
|
||||
}
|
||||
|
||||
return runs, nil
|
||||
}
|
||||
|
||||
// GetRunsCount returns the count of runs matching the provided filters
|
||||
func GetRunsCount(lastXRuns int, startDate, endDate time.Time, containsQuery string) (int64, error) {
|
||||
db := GetDB()
|
||||
var count int64
|
||||
query := db.Model(&QueryOptions{})
|
||||
|
||||
// Apply date range filter if provided
|
||||
if lastXRuns > 0 {
|
||||
query = query.Order("created_at DESC").Limit(lastXRuns)
|
||||
} else if !startDate.IsZero() && !endDate.IsZero() {
|
||||
query = query.Where("created_at BETWEEN ? AND ?", startDate, endDate)
|
||||
} else if !startDate.IsZero() {
|
||||
query = query.Where("created_at >= ?", startDate)
|
||||
} else if !endDate.IsZero() {
|
||||
query = query.Where("created_at <= ?", endDate)
|
||||
}
|
||||
|
||||
// Apply query filter if provided
|
||||
if containsQuery != "" {
|
||||
// Search in all query fields
|
||||
query = query.Where(
|
||||
"username_query LIKE ? OR "+
|
||||
"email_query LIKE ? OR "+
|
||||
"ip_query LIKE ? OR "+
|
||||
"pass_query LIKE ? OR "+
|
||||
"hash_query LIKE ? OR "+
|
||||
"name_query LIKE ? OR "+
|
||||
"domain_query LIKE ? OR "+
|
||||
"vin_query LIKE ? OR "+
|
||||
"license_plate_query LIKE ? OR "+
|
||||
"address_query LIKE ? OR "+
|
||||
"phone_query LIKE ? OR "+
|
||||
"social_query LIKE ? OR "+
|
||||
"crypto_address_query LIKE ?",
|
||||
"%"+containsQuery+"%", "%"+containsQuery+"%", "%"+containsQuery+"%",
|
||||
"%"+containsQuery+"%", "%"+containsQuery+"%", "%"+containsQuery+"%",
|
||||
"%"+containsQuery+"%", "%"+containsQuery+"%", "%"+containsQuery+"%",
|
||||
"%"+containsQuery+"%", "%"+containsQuery+"%", "%"+containsQuery+"%",
|
||||
"%"+containsQuery+"%",
|
||||
)
|
||||
}
|
||||
|
||||
// Count the results
|
||||
if err := query.Count(&count).Error; err != nil {
|
||||
zap.L().Error("get_runs_count",
|
||||
zap.String("message", "failed to count runs"),
|
||||
zap.Error(err),
|
||||
)
|
||||
return 0, fmt.Errorf("failed to count runs: %w", err)
|
||||
}
|
||||
|
||||
return count, nil
|
||||
}
|
||||
|
||||
// QueryCreds queries the database for credentials based on the provided filters
|
||||
func QueryCreds(options *DBOptions) ([]Creds, error) {
|
||||
db := GetDB()
|
||||
var creds []Creds
|
||||
query := db.Model(&Creds{})
|
||||
|
||||
// Apply filters based on the provided options
|
||||
if options.Username != "" {
|
||||
if options.ExactMatch {
|
||||
query = query.Where("username = ?", options.Username)
|
||||
} else {
|
||||
query = query.Where("username LIKE ?", "%"+options.Username+"%")
|
||||
}
|
||||
}
|
||||
|
||||
if options.Email != "" {
|
||||
if options.ExactMatch {
|
||||
query = query.Where("email = ?", options.Email)
|
||||
} else {
|
||||
query = query.Where("email LIKE ?", "%"+options.Email+"%")
|
||||
}
|
||||
}
|
||||
|
||||
if options.Password != "" {
|
||||
if options.ExactMatch {
|
||||
query = query.Where("password = ?", options.Password)
|
||||
} else {
|
||||
query = query.Where("password LIKE ?", "%"+options.Password+"%")
|
||||
}
|
||||
}
|
||||
|
||||
// Apply limit
|
||||
if options.Limit > 0 {
|
||||
query = query.Limit(options.Limit)
|
||||
}
|
||||
|
||||
// Execute the query
|
||||
if err := query.Find(&creds).Error; err != nil {
|
||||
zap.L().Error("query_creds",
|
||||
zap.String("message", "failed to query credentials"),
|
||||
zap.Error(err),
|
||||
)
|
||||
return nil, fmt.Errorf("failed to query credentials: %w", err)
|
||||
}
|
||||
|
||||
return creds, nil
|
||||
}
|
||||
|
||||
// GetCredsCount returns the count of credentials matching the provided filters
|
||||
func GetCredsCount(options *DBOptions) (int64, error) {
|
||||
db := GetDB()
|
||||
var count int64
|
||||
query := db.Model(&Creds{})
|
||||
|
||||
// Apply filters based on the provided options
|
||||
if options.Username != "" {
|
||||
if options.ExactMatch {
|
||||
query = query.Where("username = ?", options.Username)
|
||||
} else {
|
||||
query = query.Where("username LIKE ?", "%"+options.Username+"%")
|
||||
}
|
||||
}
|
||||
|
||||
if options.Email != "" {
|
||||
if options.ExactMatch {
|
||||
query = query.Where("email = ?", options.Email)
|
||||
} else {
|
||||
query = query.Where("email LIKE ?", "%"+options.Email+"%")
|
||||
}
|
||||
}
|
||||
|
||||
if options.Password != "" {
|
||||
if options.ExactMatch {
|
||||
query = query.Where("password = ?", options.Password)
|
||||
} else {
|
||||
query = query.Where("password LIKE ?", "%"+options.Password+"%")
|
||||
}
|
||||
}
|
||||
|
||||
// Count the results
|
||||
if err := query.Count(&count).Error; err != nil {
|
||||
zap.L().Error("get_creds_count",
|
||||
zap.String("message", "failed to count credentials"),
|
||||
zap.Error(err),
|
||||
)
|
||||
return 0, fmt.Errorf("failed to count credentials: %w", err)
|
||||
}
|
||||
|
||||
return count, nil
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"go.uber.org/zap"
|
||||
"gorm.io/gorm/clause"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/logger"
|
||||
)
|
||||
|
||||
var DB *gorm.DB
|
||||
|
||||
// InitDB initializes the database connection
|
||||
func InitDB(dbDir string) (*gorm.DB, error) {
|
||||
zap.L().Info("Initializing database")
|
||||
|
||||
// Create directory if it doesn't exist
|
||||
if err := os.MkdirAll(dbDir, 0755); err != nil {
|
||||
zap.L().Error("Failed to create database directory", zap.Error(err))
|
||||
return nil, fmt.Errorf("failed to create database directory: %w", err)
|
||||
}
|
||||
|
||||
dbPath := filepath.Join(dbDir, "dehashed.sqlite")
|
||||
db, err := gorm.Open(sqlite.Open(dbPath), &gorm.Config{
|
||||
Logger: logger.Default.LogMode(logger.Silent),
|
||||
})
|
||||
if err != nil {
|
||||
zap.L().Error("Failed to connect to database", zap.Error(err))
|
||||
return nil, fmt.Errorf("failed to connect to database: %w", err)
|
||||
}
|
||||
|
||||
// Auto migrate your models
|
||||
err = db.AutoMigrate(&Result{}, &Creds{}, QueryOptions{}, Creds{}, WhoisRecord{}, SubdomainRecord{}, HistoryRecord{})
|
||||
if err != nil {
|
||||
zap.L().Error("Failed to migrate database", zap.Error(err))
|
||||
return nil, fmt.Errorf("failed to migrate database: %w", err)
|
||||
}
|
||||
|
||||
DB = db
|
||||
return db, nil
|
||||
}
|
||||
|
||||
// GetDB returns the database connection
|
||||
func GetDB() *gorm.DB {
|
||||
if DB == nil {
|
||||
zap.L().Error("database not initialized")
|
||||
fmt.Println("sqlite database not initialized")
|
||||
os.Exit(1)
|
||||
}
|
||||
return DB
|
||||
}
|
||||
|
||||
func StoreResults(results DehashedResults) error {
|
||||
if len(results.Results) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
zap.L().Info("Storing results", zap.Int("count", len(results.Results)))
|
||||
db := GetDB()
|
||||
|
||||
// Use batch insert with conflict handling
|
||||
const batchSize = 100
|
||||
var lastErr error
|
||||
|
||||
// Extract the slice of results
|
||||
resultSlice := results.Results
|
||||
|
||||
for i := 0; i < len(resultSlice); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(resultSlice) {
|
||||
end = len(resultSlice)
|
||||
}
|
||||
|
||||
batch := resultSlice[i:end]
|
||||
// Use Clauses with OnConflict DoNothing to skip conflicts
|
||||
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
|
||||
if err != nil {
|
||||
zap.L().Warn("Error storing some results", zap.Error(err))
|
||||
lastErr = err
|
||||
// Continue with next batch despite error
|
||||
}
|
||||
}
|
||||
|
||||
return lastErr
|
||||
}
|
||||
|
||||
func StoreCreds(creds []Creds) error {
|
||||
if len(creds) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
zap.L().Info("Storing credentials", zap.Int("count", len(creds)))
|
||||
db := GetDB()
|
||||
|
||||
// Use batch insert with conflict handling
|
||||
// This will insert records in batches and continue even if some fail
|
||||
const batchSize = 100
|
||||
var lastErr error
|
||||
|
||||
for i := 0; i < len(creds); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(creds) {
|
||||
end = len(creds)
|
||||
}
|
||||
|
||||
batch := creds[i:end]
|
||||
// Use Clauses with OnConflict DoNothing to skip conflicts
|
||||
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
|
||||
if err != nil {
|
||||
zap.L().Warn("Error storing some credentials", zap.Error(err))
|
||||
lastErr = err
|
||||
// Continue with next batch despite error
|
||||
}
|
||||
}
|
||||
|
||||
return lastErr
|
||||
}
|
||||
|
||||
func StoreQueryOptions(queryOptions *QueryOptions) error {
|
||||
db := GetDB()
|
||||
return db.Create(queryOptions).Error
|
||||
}
|
||||
|
||||
func StoreWhoisRecord(whoisRecord WhoisRecord) error {
|
||||
// Create a pointer to the record to make it addressable
|
||||
recordPtr := &whoisRecord
|
||||
|
||||
zap.L().Info("Storing WHOIS record",
|
||||
zap.String("domain", whoisRecord.DomainName))
|
||||
|
||||
db := GetDB()
|
||||
|
||||
// Use OnConflict clause to handle duplicates
|
||||
err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(recordPtr).Error
|
||||
if err != nil {
|
||||
zap.L().Error("store_whois_record",
|
||||
zap.String("message", "failed to store whois record"),
|
||||
zap.Error(err))
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func StoreSubdomainRecord(subdomainRecords []SubdomainRecord) error {
|
||||
if len(subdomainRecords) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
zap.L().Info("Storing subdomain records", zap.Int("count", len(subdomainRecords)))
|
||||
db := GetDB()
|
||||
|
||||
// Use batch insert with conflict handling
|
||||
const batchSize = 100
|
||||
var lastErr error
|
||||
|
||||
for i := 0; i < len(subdomainRecords); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(subdomainRecords) {
|
||||
end = len(subdomainRecords)
|
||||
}
|
||||
|
||||
batch := subdomainRecords[i:end]
|
||||
// Use Clauses with OnConflict DoNothing to skip conflicts
|
||||
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
|
||||
if err != nil {
|
||||
zap.L().Warn("Error storing some subdomain records", zap.Error(err))
|
||||
lastErr = err
|
||||
// Continue with next batch despite error
|
||||
}
|
||||
}
|
||||
|
||||
return lastErr
|
||||
}
|
||||
|
||||
func StoreHistoryRecord(historyRecords []HistoryRecord) error {
|
||||
if len(historyRecords) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
zap.L().Info("Storing history records", zap.Int("count", len(historyRecords)))
|
||||
db := GetDB()
|
||||
|
||||
// Use batch insert with conflict handling
|
||||
const batchSize = 100
|
||||
var lastErr error
|
||||
|
||||
for i := 0; i < len(historyRecords); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(historyRecords) {
|
||||
end = len(historyRecords)
|
||||
}
|
||||
|
||||
batch := historyRecords[i:end]
|
||||
// Use Clauses with OnConflict DoNothing to skip conflicts
|
||||
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
|
||||
if err != nil {
|
||||
zap.L().Warn("Error storing some history records", zap.Error(err))
|
||||
lastErr = err
|
||||
// Continue with next batch despite error
|
||||
}
|
||||
}
|
||||
|
||||
return lastErr
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
package sqlite
|
||||
|
||||
type DehashedSearchRequest struct {
|
||||
Page int `json:"page"`
|
||||
Query string `json:"query"`
|
||||
Size int `json:"size"`
|
||||
Wildcard bool `json:"wildcard"`
|
||||
Regex bool `json:"regex"`
|
||||
DeDupe bool `json:"de_dupe"`
|
||||
}
|
||||
|
||||
func NewDehashedSearchRequest(size int, wildcard, regex bool) *DehashedSearchRequest {
|
||||
return &DehashedSearchRequest{
|
||||
Page: 0,
|
||||
Size: size,
|
||||
Wildcard: false,
|
||||
Regex: false,
|
||||
DeDupe: true,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"go.uber.org/zap"
|
||||
"gorm.io/gorm"
|
||||
"io"
|
||||
"os"
|
||||
)
|
||||
|
||||
type DehashedResponse struct {
|
||||
Balance int `json:"balance"`
|
||||
Entries []Result `json:"entries"`
|
||||
Success bool `json:"success"`
|
||||
Took string `json:"took"`
|
||||
TotalResults int `json:"total"`
|
||||
}
|
||||
|
||||
type Result struct {
|
||||
gorm.Model
|
||||
DehashedId string `json:"id" xml:"id" yaml:"id" gorm:"uniqueIndex"`
|
||||
Email []string `json:"email,omitempty" xml:"email,omitempty" yaml:"email,omitempty" gorm:"serializer:json"`
|
||||
IpAddress []string `json:"ip_address,omitempty" xml:"ip_address,omitempty" yaml:"ip_address,omitempty" gorm:"serializer:json"`
|
||||
Username []string `json:"username,omitempty" xml:"username,omitempty" yaml:"username,omitempty" gorm:"serializer:json"`
|
||||
Password []string `json:"password,omitempty" xml:"password,omitempty" yaml:"password,omitempty" gorm:"serializer:json"`
|
||||
HashedPassword []string `json:"hashed_password,omitempty" xml:"hashed_password,omitempty" yaml:"hashed_password,omitempty" gorm:"serializer:json"`
|
||||
HashType string `json:"hash_type,omitempty" xml:"hash_type,omitempty" yaml:"hash_type,omitempty"`
|
||||
Name []string `json:"name,omitempty" xml:"name,omitempty" yaml:"name,omitempty" gorm:"serializer:json"`
|
||||
Vin []string `json:"vin,omitempty" xml:"vin,omitempty" yaml:"vin,omitempty" gorm:"serializer:json"`
|
||||
LicensePlate []string `json:"license_plate,omitempty" xml:"license_plate,omitempty" yaml:"license_plate,omitempty" gorm:"serializer:json"`
|
||||
Url []string `json:"url,omitempty" xml:"url,omitempty" yaml:"url,omitempty" gorm:"serializer:json"`
|
||||
Social []string `json:"social,omitempty" xml:"social,omitempty" yaml:"social,omitempty" gorm:"serializer:json"`
|
||||
CryptoCurrencyAddress []string `json:"cryptocurrency_address,omitempty" xml:"cryptocurrency_address,omitempty" yaml:"cryptocurrency_address,omitempty" gorm:"serializer:json"`
|
||||
Address []string `json:"address,omitempty" xml:"address,omitempty" yaml:"address,omitempty" gorm:"serializer:json"`
|
||||
Phone []string `json:"phone,omitempty" xml:"phone,omitempty" yaml:"phone,omitempty" gorm:"serializer:json"`
|
||||
Company []string `json:"company,omitempty" xml:"company,omitempty" yaml:"company,omitempty" gorm:"serializer:json"`
|
||||
DatabaseName string `json:"database_name,omitempty" xml:"database_name,omitempty" yaml:"database_name,omitempty"`
|
||||
}
|
||||
|
||||
type DehashedResults struct {
|
||||
Results []Result `json:"results"`
|
||||
}
|
||||
|
||||
func (dr *DehashedResults) ExtractCredentials() []Creds {
|
||||
var creds []Creds
|
||||
|
||||
results := dr.Results
|
||||
|
||||
for _, r := range results {
|
||||
if len(r.Password) > 0 {
|
||||
// Get first email if available
|
||||
email := ""
|
||||
if len(r.Email) > 0 {
|
||||
email = r.Email[0]
|
||||
}
|
||||
|
||||
// Get first password
|
||||
password := r.Password[0]
|
||||
|
||||
cred := Creds{Email: email, Password: password}
|
||||
creds = append(creds, cred)
|
||||
}
|
||||
}
|
||||
|
||||
go func() {
|
||||
err := StoreCreds(creds)
|
||||
if err != nil {
|
||||
zap.L().Error("store_creds",
|
||||
zap.String("message", "failed to store creds"),
|
||||
zap.Error(err),
|
||||
)
|
||||
fmt.Printf("Error Storing Results: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
return creds
|
||||
}
|
||||
|
||||
func NewDehashedResults(body io.Reader) ([]Result, int, int) {
|
||||
var response DehashedResponse
|
||||
|
||||
err := json.NewDecoder(body).Decode(&response)
|
||||
if err != nil {
|
||||
fmt.Printf("Error Parsing Response Body: %v", err)
|
||||
os.Exit(-1)
|
||||
}
|
||||
|
||||
return response.Entries, response.Balance, response.TotalResults
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"dehasher/internal/files"
|
||||
"fmt"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
type DBOptions struct {
|
||||
Username string
|
||||
Email string
|
||||
IPAddress string
|
||||
Password string
|
||||
HashedPassword string
|
||||
Name string
|
||||
Vin string
|
||||
LicensePlate string
|
||||
Address string
|
||||
Phone string
|
||||
Social string
|
||||
CryptoCurrencyAddress string
|
||||
Domain string
|
||||
Limit int
|
||||
ExactMatch bool
|
||||
NonEmptyFields []string // Fields that should not be empty
|
||||
DisplayFields []string // Fields to display in output
|
||||
}
|
||||
|
||||
func NewDBOptions() *DBOptions {
|
||||
return &DBOptions{
|
||||
Limit: 100, // Default limit
|
||||
ExactMatch: false,
|
||||
NonEmptyFields: []string{},
|
||||
DisplayFields: []string{},
|
||||
}
|
||||
}
|
||||
|
||||
func (o *DBOptions) Empty() bool {
|
||||
return o.Username == "" && o.Email == "" && o.IPAddress == "" &&
|
||||
o.Password == "" && o.HashedPassword == "" && o.Name == "" &&
|
||||
o.Vin == "" && o.LicensePlate == "" && o.Address == "" &&
|
||||
o.Phone == "" && o.Social == "" && o.CryptoCurrencyAddress == "" && o.Domain == "" &&
|
||||
len(o.NonEmptyFields) == 0
|
||||
}
|
||||
|
||||
type QueryOptions struct {
|
||||
gorm.Model
|
||||
MaxRecords int `json:"max_records"`
|
||||
MaxRequests int `json:"max_requests"`
|
||||
StartingPage int `json:"starting_page"`
|
||||
OutputFormat files.FileType `json:"output_format"`
|
||||
OutputFile string `json:"output_file"`
|
||||
RegexMatch bool `json:"regex_match"`
|
||||
WildcardMatch bool `json:"wildcard_match"`
|
||||
UsernameQuery string `json:"username_query"`
|
||||
EmailQuery string `json:"email_query"`
|
||||
IpQuery string `json:"ip_query"`
|
||||
PassQuery string `json:"pass_query"`
|
||||
HashQuery string `json:"hash_query"`
|
||||
NameQuery string `json:"name_query"`
|
||||
DomainQuery string `json:"domain_query"`
|
||||
VinQuery string `json:"vin_query"`
|
||||
LicensePlateQuery string `json:"license_plate_query"`
|
||||
AddressQuery string `json:"address_query"`
|
||||
PhoneQuery string `json:"phone_query"`
|
||||
SocialQuery string `json:"social_query"`
|
||||
CryptoAddressQuery string `json:"crypto_address_query"`
|
||||
PrintBalance bool `json:"print_balance"`
|
||||
CredsOnly bool `json:"creds_only"`
|
||||
}
|
||||
|
||||
func NewQueryOptions(maxRecords, maxRequests, startingPage int, outputFormat, outputFile, usernameQuery, emailQuery, ipQuery, passQuery, hashQuery, nameQuery, domainQuery, vinQuery, licensePlateQuery, addressQuery, phoneQuery, socialQuery, cryptoAddressQuery string, regexMatch, wildcardMatch, printBalance, credsOnly bool) *QueryOptions {
|
||||
return &QueryOptions{
|
||||
MaxRecords: maxRecords,
|
||||
MaxRequests: maxRequests,
|
||||
StartingPage: startingPage,
|
||||
OutputFormat: files.GetFileType(outputFormat),
|
||||
OutputFile: outputFile,
|
||||
PrintBalance: printBalance,
|
||||
CredsOnly: credsOnly,
|
||||
RegexMatch: regexMatch,
|
||||
WildcardMatch: wildcardMatch,
|
||||
UsernameQuery: usernameQuery,
|
||||
EmailQuery: emailQuery,
|
||||
IpQuery: ipQuery,
|
||||
PassQuery: passQuery,
|
||||
HashQuery: hashQuery,
|
||||
NameQuery: nameQuery,
|
||||
DomainQuery: domainQuery,
|
||||
VinQuery: vinQuery,
|
||||
LicensePlateQuery: licensePlateQuery,
|
||||
AddressQuery: addressQuery,
|
||||
PhoneQuery: phoneQuery,
|
||||
SocialQuery: socialQuery,
|
||||
CryptoAddressQuery: cryptoAddressQuery,
|
||||
}
|
||||
}
|
||||
|
||||
type Creds struct {
|
||||
gorm.Model
|
||||
Email string `json:"email" yaml:"email" xml:"email"`
|
||||
Username string `json:"username" yaml:"username" xml:"username"`
|
||||
Password string `json:"password" yaml:"password" xml:"password"`
|
||||
}
|
||||
|
||||
func (c Creds) ToString() string {
|
||||
return fmt.Sprintf("%s%s%s", c.Username, "%", c.Password)
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
package sqlite
|
||||
|
||||
import "gorm.io/gorm"
|
||||
|
||||
type WhoIsLookupResult struct {
|
||||
RemainingCredits int `json:"remaining_credits"`
|
||||
Data Data `json:"data"`
|
||||
}
|
||||
|
||||
type Data struct {
|
||||
WhoisRecord WhoisRecord `json:"WhoisRecord"`
|
||||
}
|
||||
|
||||
type WhoisRecord struct {
|
||||
gorm.Model
|
||||
Audit Audit `json:"audit" gorm:"serializer:json"`
|
||||
ContactEmail string `json:"contactEmail"`
|
||||
CreatedDate string `json:"createdDate"`
|
||||
CreatedDateNormalized string `json:"createdDateNormalized"`
|
||||
DomainName string `json:"domainName"`
|
||||
DomainNameExt string `json:"domainNameExt"`
|
||||
EstimatedDomainAge int `json:"estimatedDomainAge"`
|
||||
ExpiresDate string `json:"expiresDate"`
|
||||
ExpiresDateNormalized string `json:"expiresDateNormalized"`
|
||||
Footer string `json:"footer"`
|
||||
Header string `json:"header"`
|
||||
NameServers NameServers `json:"nameServers" gorm:"serializer:json"`
|
||||
ParseCode int `json:"parseCode"`
|
||||
RawText string `json:"rawText"`
|
||||
Registrant Contact `json:"registrant" gorm:"serializer:json"`
|
||||
RegistrarIANAID string `json:"registrarIANAID"`
|
||||
RegistrarName string `json:"registrarName"`
|
||||
RegistryData RegistryData `json:"registryData" gorm:"serializer:json"`
|
||||
Status string `json:"status"`
|
||||
StrippedText string `json:"strippedText"`
|
||||
TechnicalContact Contact `json:"technicalContact" gorm:"serializer:json"`
|
||||
UpdatedDate string `json:"updatedDate"`
|
||||
UpdatedDateNormalized string `json:"updatedDateNormalized"`
|
||||
}
|
||||
|
||||
type Audit struct {
|
||||
CreatedDate string `json:"createdDate"`
|
||||
UpdatedDate string `json:"updatedDate"`
|
||||
}
|
||||
|
||||
type NameServers struct {
|
||||
HostNames []string `json:"hostNames"`
|
||||
IPs []string `json:"ips"`
|
||||
RawText string `json:"rawText"`
|
||||
}
|
||||
|
||||
type Contact struct {
|
||||
City string `json:"city"`
|
||||
Country string `json:"country"`
|
||||
CountryCode string `json:"countryCode"`
|
||||
Name string `json:"name"`
|
||||
Organization string `json:"organization"`
|
||||
PostalCode string `json:"postalCode"`
|
||||
RawText string `json:"rawText"`
|
||||
State string `json:"state"`
|
||||
Street1 string `json:"street1"`
|
||||
Telephone string `json:"telephone"`
|
||||
}
|
||||
|
||||
type RegistryData struct {
|
||||
Audit Audit `json:"audit"`
|
||||
CreatedDate string `json:"createdDate"`
|
||||
CreatedDateNormalized string `json:"createdDateNormalized"`
|
||||
DomainName string `json:"domainName"`
|
||||
ExpiresDate string `json:"expiresDate"`
|
||||
ExpiresDateNormalized string `json:"expiresDateNormalized"`
|
||||
Footer string `json:"footer"`
|
||||
Header string `json:"header"`
|
||||
NameServers NameServers `json:"nameServers"`
|
||||
ParseCode int `json:"parseCode"`
|
||||
RawText string `json:"rawText"`
|
||||
RegistrarIANAID string `json:"registrarIANAID"`
|
||||
RegistrarName string `json:"registrarName"`
|
||||
Status string `json:"status"`
|
||||
StrippedText string `json:"strippedText"`
|
||||
UpdatedDate string `json:"updatedDate"`
|
||||
UpdatedDateNormalized string `json:"updatedDateNormalized"`
|
||||
WhoisServer string `json:"whoisServer"`
|
||||
}
|
||||
|
||||
type WhoIsSubdomainScan struct {
|
||||
RemainingCredits int `json:"remaining_credits"`
|
||||
Data ScanData `json:"data"`
|
||||
}
|
||||
|
||||
type ScanData struct {
|
||||
Result ScanResult `json:"result"`
|
||||
Search string `json:"search"`
|
||||
}
|
||||
|
||||
type ScanResult struct {
|
||||
Count int `json:"count"`
|
||||
Records []SubdomainRecord `json:"records"`
|
||||
}
|
||||
|
||||
type SubdomainRecord struct {
|
||||
gorm.Model
|
||||
Domain string `json:"domain"`
|
||||
FirstSeen int64 `json:"firstSeen"`
|
||||
LastSeen int64 `json:"lastSeen"`
|
||||
}
|
||||
|
||||
type WhoIsHistory struct {
|
||||
RemainingCredits int `json:"remaining_credits"`
|
||||
Data HistoryData `json:"data"`
|
||||
}
|
||||
|
||||
type HistoryData struct {
|
||||
Records []HistoryRecord `json:"records"`
|
||||
RecordsCount int `json:"recordsCount"`
|
||||
}
|
||||
|
||||
type HistoryRecord struct {
|
||||
gorm.Model
|
||||
AdministrativeContact ContactInfo `json:"administrativeContact" gorm:"serializer:json"`
|
||||
Audit Audit `json:"audit" gorm:"serializer:json"`
|
||||
BillingContact ContactInfo `json:"billingContact" gorm:"serializer:json"`
|
||||
CleanText string `json:"cleanText"`
|
||||
CreatedDateISO8601 string `json:"createdDateISO8601"`
|
||||
CreatedDateRaw string `json:"createdDateRaw"`
|
||||
DomainName string `json:"domainName"`
|
||||
DomainType string `json:"domainType"`
|
||||
ExpiresDateISO8601 string `json:"expiresDateISO8601"`
|
||||
ExpiresDateRaw string `json:"expiresDateRaw"`
|
||||
NameServers []string `json:"nameServers" gorm:"serializer:json"`
|
||||
RawText string `json:"rawText"`
|
||||
RegistrantContact ContactInfo `json:"registrantContact" gorm:"serializer:json"`
|
||||
RegistrarName string `json:"registrarName"`
|
||||
Status []string `json:"status" gorm:"serializer:json"`
|
||||
TechnicalContact ContactInfo `json:"technicalContact" gorm:"serializer:json"`
|
||||
UpdatedDateISO8601 string `json:"updatedDateISO8601"`
|
||||
UpdatedDateRaw string `json:"updatedDateRaw"`
|
||||
WhoisServer string `json:"whoisServer"`
|
||||
ZoneContact ContactInfo `json:"zoneContact" gorm:"serializer:json"`
|
||||
}
|
||||
|
||||
type ContactInfo struct {
|
||||
City string `json:"city"`
|
||||
Country string `json:"country"`
|
||||
Email string `json:"email"`
|
||||
Fax string `json:"fax"`
|
||||
FaxExt string `json:"faxExt"`
|
||||
Name string `json:"name"`
|
||||
Organization string `json:"organization"`
|
||||
PostalCode string `json:"postalCode"`
|
||||
RawText string `json:"rawText"`
|
||||
State string `json:"state"`
|
||||
Street string `json:"street"`
|
||||
Telephone string `json:"telephone"`
|
||||
TelephoneExt string `json:"telephoneExt"`
|
||||
}
|
||||
|
||||
type WhoIsCredits struct {
|
||||
WhoisCredits int `json:"whois_credits"`
|
||||
}
|
||||
Reference in New Issue
Block a user