Refactor user and credential handling: rename Creds to User, update database migrations, and add targets subcommand for exporting users and subdomains
This commit is contained in:
Evan Hosinski
+112
-126
@@ -1,6 +1,9 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"crowsnest/internal/debug"
|
||||
"crowsnest/internal/export"
|
||||
"crowsnest/internal/files"
|
||||
"crowsnest/internal/pretty"
|
||||
"crowsnest/internal/sqlite"
|
||||
"encoding/json"
|
||||
@@ -10,132 +13,6 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Map of available tables and their columns
|
||||
var availableTables = map[string][]string{
|
||||
"creds": {
|
||||
"id", "created_at", "updated_at", "deleted_at", "email", "username", "password",
|
||||
},
|
||||
//"history": {
|
||||
// "id", "created_at", "updated_at", "deleted_at", "domain_name", "domain_type",
|
||||
// "registrar_name", "whois_server", "created_date_iso8601", "updated_date_iso8601", "expires_date_iso8601",
|
||||
//},
|
||||
"lookup": {
|
||||
"id", "created_at", "updated_at", "deleted_at", "search_term", "type", "first_seen", "last_visit",
|
||||
"name",
|
||||
},
|
||||
// Query Options
|
||||
"runs": {
|
||||
"id", "created_at", "updated_at", "deleted_at", "max_records", "max_requests", "starting_page",
|
||||
"output_format", "output_file", "regex_match", "wildcard_match", "username_query", "email_query",
|
||||
"ip_query", "pass_query", "hash_query", "name_query", "domain_query", "vin_query", "license_plate_query",
|
||||
"address_query", "phone_query", "social_query", "crypto_address_query", "print_balance", "creds_only",
|
||||
},
|
||||
"results": {
|
||||
"id", "created_at", "updated_at", "deleted_at", "dehashed_id", "email", "ip_address", "username",
|
||||
"password", "hashed_password", "hash_type", "name", "vin", "license_plate", "url", "social",
|
||||
"cryptocurrency_address", "address", "phone", "company", "database_name",
|
||||
},
|
||||
"subdomains": {
|
||||
"id", "created_at", "updated_at", "deleted_at", "domain", "first_seen", "last_seen",
|
||||
},
|
||||
"whois": {
|
||||
"id", "created_at", "updated_at", "deleted_at", "audit", "contact_email", "created_date", "created_date_normalized",
|
||||
"domain_name", "domain_name_ext", "estimated_domain_age", "expires_date", "expires_date_normalized", "footer", "header",
|
||||
"name_servers", "parse_code", "raw_text", "registrant", "registrar_iana_id", "registrar_name", "registry_data",
|
||||
"status", "stripped_text", "updated_date", "updated_date_normalized",
|
||||
},
|
||||
"hunter_domain": {
|
||||
"id", "created_at", "updated_at", "deleted_at", "domain", "disposable", "webmail", "accept_all", "pattern",
|
||||
"organization", "description", "industry", "twitter", "facebook", "linkedin", "instagram", "youtube",
|
||||
"technologies", "country", "state", "city", "postal_code", "street", "headcount", "company_type", "emails", "linked_domains",
|
||||
},
|
||||
"hunter_email": {
|
||||
"id", "created_at", "updated_at", "deleted_at", "value", "type", "confidence", "sources", "first_name", "last_name",
|
||||
"position", "position_raw", "seniority", "department", "linkedin", "twitter", "phone_number", "verification_date", "verification_status",
|
||||
},
|
||||
}
|
||||
|
||||
// Function to list available tables and their columns
|
||||
func listAvailableTables() {
|
||||
fmt.Println("Available tables and columns:")
|
||||
|
||||
// Prepare data for pretty.Table
|
||||
headers := []string{"Table", "Columns"}
|
||||
var tableRows [][]string
|
||||
|
||||
// Sort tables alphabetically for consistent output
|
||||
var tableNames []string
|
||||
for tableName := range availableTables {
|
||||
tableNames = append(tableNames, tableName)
|
||||
}
|
||||
|
||||
// Simple bubble sort for table names
|
||||
for i := 0; i < len(tableNames)-1; i++ {
|
||||
for j := 0; j < len(tableNames)-i-1; j++ {
|
||||
if tableNames[j] > tableNames[j+1] {
|
||||
tableNames[j], tableNames[j+1] = tableNames[j+1], tableNames[j]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Create rows for the table
|
||||
for _, tableName := range tableNames {
|
||||
columns := availableTables[tableName]
|
||||
|
||||
// Format columns with line breaks for better readability
|
||||
var formattedColumns string
|
||||
for i := 0; i < len(columns); i += 5 {
|
||||
end := i + 5
|
||||
if end > len(columns) {
|
||||
end = len(columns)
|
||||
}
|
||||
if i > 0 {
|
||||
formattedColumns += "\n"
|
||||
}
|
||||
formattedColumns += strings.Join(columns[i:end], ", ")
|
||||
}
|
||||
|
||||
tableRows = append(tableRows, []string{tableName, formattedColumns})
|
||||
}
|
||||
|
||||
// Display the table
|
||||
pretty.Table(headers, tableRows)
|
||||
}
|
||||
|
||||
// Function to validate table name
|
||||
func isValidTable(tableName string) bool {
|
||||
_, exists := availableTables[tableName]
|
||||
return exists
|
||||
}
|
||||
|
||||
// Function to validate column names for a specific table
|
||||
func validateColumns(tableName string, columns []string) []string {
|
||||
if tableName == "" || columns == nil || len(columns) == 0 || columns[0] == "*" {
|
||||
return nil
|
||||
}
|
||||
|
||||
tableColumns, exists := availableTables[tableName]
|
||||
if !exists {
|
||||
return []string{fmt.Sprintf("Table '%s' does not exist", tableName)}
|
||||
}
|
||||
|
||||
var invalidColumns []string
|
||||
for _, col := range columns {
|
||||
valid := false
|
||||
for _, tableCol := range tableColumns {
|
||||
if col == tableCol {
|
||||
valid = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !valid {
|
||||
invalidColumns = append(invalidColumns, col)
|
||||
}
|
||||
}
|
||||
|
||||
return invalidColumns
|
||||
}
|
||||
|
||||
func init() {
|
||||
// Add whois command to root command
|
||||
rootCmd.AddCommand(queryCmd)
|
||||
@@ -148,6 +25,8 @@ func init() {
|
||||
queryCmd.Flags().StringVarP(&dbQueryUserQuery, "user-query", "q", "", "User query to execute")
|
||||
queryCmd.Flags().StringVarP(&dbQueryRawQuery, "raw-query", "r", "", "Raw SQL query to execute")
|
||||
queryCmd.Flags().BoolVarP(&dbQueryListAll, "list-all", "a", false, "List all tables and their columns")
|
||||
queryCmd.Flags().StringVarP(&dbQueryFormat, "format", "f", "json", "Output format (json, yaml, xml, txt)")
|
||||
queryCmd.Flags().StringVarP(&dbQueryFile, "file", "o", "query", "File to output results to including extension")
|
||||
|
||||
// Add mutually exclusive flags to query and raw-query
|
||||
// Cannot use query and raw-query at the same time
|
||||
@@ -166,6 +45,8 @@ var (
|
||||
dbQueryUserQuery string
|
||||
dbQueryRawQuery string
|
||||
dbQueryListAll bool
|
||||
dbQueryFormat string
|
||||
dbQueryFile string
|
||||
|
||||
queryCmd = &cobra.Command{
|
||||
Use: "query",
|
||||
@@ -321,6 +202,49 @@ func tableQuery(table sqlite.Table) {
|
||||
return
|
||||
}
|
||||
|
||||
// Export results if file name is specified
|
||||
if len(strings.TrimSpace(dbQueryFile)) > 0 {
|
||||
fmt.Println("[*] Exporting results to file...")
|
||||
|
||||
if debugGlobal {
|
||||
debug.PrintInfo("exporting results to file: " + dbQueryFile)
|
||||
}
|
||||
// Prepare data for export
|
||||
var results []map[string]interface{}
|
||||
|
||||
// Process the rows
|
||||
for rows.Next() {
|
||||
values := make([]interface{}, len(cols))
|
||||
pointers := make([]interface{}, len(cols))
|
||||
for i := range values {
|
||||
pointers[i] = &values[i]
|
||||
}
|
||||
if err := rows.Scan(pointers...); err != nil {
|
||||
zap.L().Error("export_query",
|
||||
zap.String("message", "failed to scan row from query"),
|
||||
zap.Error(err),
|
||||
)
|
||||
fmt.Printf("[!] Error scanning row from query: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Create a map for this row
|
||||
rowMap := make(map[string]interface{})
|
||||
for i, col := range cols {
|
||||
val := values[i]
|
||||
rowMap[col] = val
|
||||
}
|
||||
|
||||
results = append(results, rowMap)
|
||||
}
|
||||
|
||||
// Export the results
|
||||
exportQueryResults(results)
|
||||
|
||||
return
|
||||
}
|
||||
fmt.Println("[*] Querying Database...")
|
||||
|
||||
// Prepare data for pretty.Table
|
||||
headers := cols
|
||||
var tableRows [][]string
|
||||
@@ -411,6 +335,49 @@ func rawDBQuery() {
|
||||
return
|
||||
}
|
||||
|
||||
if len(strings.TrimSpace(dbQueryFile)) > 0 {
|
||||
fmt.Println("[*] Exporting results to file...")
|
||||
|
||||
if debugGlobal {
|
||||
debug.PrintInfo("exporting results to file: " + dbQueryFile)
|
||||
}
|
||||
|
||||
// Prepare data for export
|
||||
var results []map[string]interface{}
|
||||
|
||||
// Process the rows
|
||||
for rows.Next() {
|
||||
values := make([]interface{}, len(columns))
|
||||
pointers := make([]interface{}, len(columns))
|
||||
for i := range values {
|
||||
pointers[i] = &values[i]
|
||||
}
|
||||
if err := rows.Scan(pointers...); err != nil {
|
||||
zap.L().Error("export_raw_query",
|
||||
zap.String("message", "failed to scan row from raw query"),
|
||||
zap.Error(err),
|
||||
)
|
||||
fmt.Printf("[!] Error scanning row from raw query: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Create a map for this row
|
||||
rowMap := make(map[string]interface{})
|
||||
for i, col := range columns {
|
||||
val := values[i]
|
||||
rowMap[col] = val
|
||||
}
|
||||
|
||||
results = append(results, rowMap)
|
||||
}
|
||||
|
||||
// Export the results
|
||||
exportQueryResults(results)
|
||||
|
||||
return
|
||||
}
|
||||
fmt.Println("[*] Querying Database...")
|
||||
|
||||
// Prepare data for pretty.Table
|
||||
headers := columns
|
||||
var tableRows [][]string
|
||||
@@ -477,3 +444,22 @@ func rawDBQuery() {
|
||||
// Display the table
|
||||
pretty.Table(headers, tableRows)
|
||||
}
|
||||
|
||||
// exportQueryResults exports the results to a file
|
||||
func exportQueryResults(results []map[string]interface{}) {
|
||||
// Get file type
|
||||
fileType := files.GetFileType(dbQueryFormat)
|
||||
|
||||
// Export results
|
||||
err := export.WriteQueryResultsToFile(results, dbQueryFile, fileType)
|
||||
if err != nil {
|
||||
zap.L().Error("export_results",
|
||||
zap.String("message", "failed to write to file"),
|
||||
zap.Error(err),
|
||||
)
|
||||
fmt.Printf("[!] Error writing to file: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Printf("[+] Exported %d records to file: %s%s\n", len(results), dbQueryFile, fileType.Extension())
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user