updates to db queries and exports
Updates to how queries to the db are made.
This commit is contained in:
Evan Hosinski
+258
-103
@@ -1,126 +1,281 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"dehasher/internal/badger"
|
||||
"dehasher/internal/query"
|
||||
"dehasher/internal/pretty"
|
||||
"dehasher/internal/sqlite"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/zap"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var (
|
||||
// Query command flags
|
||||
maxRecords int
|
||||
maxRequests int
|
||||
startingPage int
|
||||
credsOnly bool
|
||||
printBalance bool
|
||||
regexMatch bool
|
||||
wildcardMatch bool
|
||||
outputFormat string
|
||||
outputFile string
|
||||
usernameQuery string
|
||||
emailQuery string
|
||||
ipQuery string
|
||||
passwordQuery string
|
||||
hashQuery string
|
||||
nameQuery string
|
||||
domainQuery string
|
||||
vinQuery string
|
||||
licensePlateQuery string
|
||||
addressQuery string
|
||||
phoneQuery string
|
||||
socialQuery string
|
||||
cryptoCurrencyAddressQuery string
|
||||
func init() {
|
||||
// Add whois command to root command
|
||||
rootCmd.AddCommand(queryCmd)
|
||||
|
||||
// Add flags specific to whois command
|
||||
queryCmd.Flags().StringVarP(&dbQueryTableName, "table", "t", "", "Table to query (results, creds, whois, subdomains, history, query_options)")
|
||||
queryCmd.Flags().IntVarP(&dbQueryLimitRows, "limit", "l", 100, "Limit number of results")
|
||||
queryCmd.Flags().StringVarP(&dbQueryNotNull, "not-null", "n", "", "Filter for non-null values (comma-separated list, e.g., 'password,email')")
|
||||
queryCmd.Flags().StringVarP(&dbQueryColumns, "columns", "c", "", "Columns to display in output (comma-separated list, e.g., 'username,email,password')")
|
||||
queryCmd.Flags().StringVarP(&dbQueryUserQuery, "user-query", "q", "", "User query to execute")
|
||||
queryCmd.Flags().StringVarP(&dbQueryRawQuery, "raw-query", "r", "", "Raw SQL query to execute")
|
||||
queryCmd.Flags().BoolVarP(&dbQueryListAll, "list-all", "a", false, "List all columns")
|
||||
|
||||
// Add mutually exclusive flags to query and raw-query
|
||||
// Cannot use query and raw-query at the same time
|
||||
queryCmd.MarkFlagsMutuallyExclusive("user-query", "raw-query")
|
||||
// Raw query does not require a table
|
||||
queryCmd.MarkFlagsMutuallyExclusive("user-query", "table")
|
||||
// List all columns does not require a query or raw-query
|
||||
queryCmd.MarkFlagsMutuallyExclusive("raw-query", "list-all")
|
||||
}
|
||||
|
||||
var (
|
||||
dbQueryTableName string
|
||||
dbQueryLimitRows int
|
||||
dbQueryNotNull string
|
||||
dbQueryColumns string
|
||||
dbQueryUserQuery string
|
||||
dbQueryRawQuery string
|
||||
dbQueryListAll bool
|
||||
|
||||
// Query command
|
||||
queryCmd = &cobra.Command{
|
||||
Use: "query",
|
||||
Short: "Query the Dehashed API",
|
||||
Long: `Query the Dehashed API for emails, usernames, passwords, hashes, IP addresses, and names.`,
|
||||
Short: "Query the database",
|
||||
Long: `Query the database for various information.`,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
key := getStoredApiKey()
|
||||
|
||||
// Validate credentials
|
||||
if key == "" {
|
||||
fmt.Println("API key is required. Set the key with the \"set-key\" command. [dehasher set-key <api_key>]")
|
||||
return
|
||||
// If Raw Query is set, execute it and return
|
||||
if dbQueryRawQuery != "" {
|
||||
fmt.Println("[*] Executing Raw Query...")
|
||||
rawDBQuery()
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Create new QueryOptions
|
||||
queryOptions := sqlite.NewQueryOptions(
|
||||
maxRecords,
|
||||
maxRequests,
|
||||
startingPage,
|
||||
outputFormat,
|
||||
outputFile,
|
||||
usernameQuery,
|
||||
emailQuery,
|
||||
ipQuery,
|
||||
passwordQuery,
|
||||
hashQuery,
|
||||
nameQuery,
|
||||
domainQuery,
|
||||
vinQuery,
|
||||
licensePlateQuery,
|
||||
addressQuery,
|
||||
phoneQuery,
|
||||
socialQuery,
|
||||
cryptoCurrencyAddressQuery,
|
||||
regexMatch,
|
||||
wildcardMatch,
|
||||
printBalance,
|
||||
credsOnly,
|
||||
)
|
||||
|
||||
// Create new Dehasher
|
||||
dehasher := query.NewDehasher(queryOptions)
|
||||
dehasher.SetClientCredentials(
|
||||
key,
|
||||
)
|
||||
|
||||
// Start querying
|
||||
dehasher.Start()
|
||||
fmt.Println("\n[*] Completing Process")
|
||||
|
||||
sqlite.StoreQueryOptions(queryOptions)
|
||||
// Determine which table to query based on the tableTypeDBQuery parameter
|
||||
table := sqlite.GetTable(dbQueryTableName)
|
||||
if table == sqlite.UnknownTable {
|
||||
fmt.Printf("Error: Unknown table type '%s'.\n", dbQueryTableName)
|
||||
cmd.Help()
|
||||
return
|
||||
}
|
||||
fmt.Println("[*] Querying Database...")
|
||||
tableQuery(table)
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
func init() {
|
||||
// Add query command to root command
|
||||
rootCmd.AddCommand(queryCmd)
|
||||
func tableQuery(table sqlite.Table) {
|
||||
|
||||
// Add flags specific to query command
|
||||
queryCmd.Flags().IntVarP(&maxRecords, "max-records", "m", 30000, "Maximum amount of records to return")
|
||||
queryCmd.Flags().IntVarP(&maxRequests, "max-requests", "r", -1, "Maximum number of requests to make")
|
||||
queryCmd.Flags().IntVarP(&startingPage, "starting-page", "s", 1, "Starting page for requests")
|
||||
queryCmd.Flags().BoolVarP(&printBalance, "print-balance", "b", false, "Print remaining balance after requests")
|
||||
queryCmd.Flags().BoolVarP(®exMatch, "regex-match", "R", false, "Use regex matching on query fields")
|
||||
queryCmd.Flags().BoolVarP(&wildcardMatch, "wildcard-match", "W", false, "Use wildcard matching on query fields (Use ? to replace a single character, and * for multiple characters)")
|
||||
queryCmd.Flags().BoolVarP(&credsOnly, "creds-only", "C", false, "Return credentials only")
|
||||
queryCmd.Flags().StringVarP(&outputFormat, "format", "f", "json", "Output format (json, yaml, xml, txt)")
|
||||
queryCmd.Flags().StringVarP(&outputFile, "output", "o", "query", "File to output results to including extension")
|
||||
queryCmd.Flags().StringVarP(&usernameQuery, "username", "U", "", "Username query")
|
||||
queryCmd.Flags().StringVarP(&emailQuery, "email-query", "E", "", "Email query")
|
||||
queryCmd.Flags().StringVarP(&ipQuery, "ip", "I", "", "IP address query")
|
||||
queryCmd.Flags().StringVarP(&domainQuery, "domain", "D", "", "Domain query")
|
||||
queryCmd.Flags().StringVarP(&passwordQuery, "password", "P", "", "Password query")
|
||||
queryCmd.Flags().StringVarP(&vinQuery, "vin", "V", "", "VIN query")
|
||||
queryCmd.Flags().StringVarP(&licensePlateQuery, "license", "L", "", "License plate query")
|
||||
queryCmd.Flags().StringVarP(&addressQuery, "address", "A", "", "Address query")
|
||||
queryCmd.Flags().StringVarP(&phoneQuery, "phone", "M", "", "Phone query")
|
||||
queryCmd.Flags().StringVarP(&socialQuery, "social", "S", "", "Social query")
|
||||
queryCmd.Flags().StringVarP(&cryptoCurrencyAddressQuery, "crypto", "B", "", "Crypto currency address query")
|
||||
queryCmd.Flags().StringVarP(&hashQuery, "hash", "Q", "", "Hashed password query")
|
||||
queryCmd.Flags().StringVarP(&nameQuery, "name", "N", "", "Name query")
|
||||
// Get the columns to query
|
||||
columns := []string{"*"}
|
||||
if dbQueryColumns != "" {
|
||||
columns = strings.Split(dbQueryColumns, ",")
|
||||
}
|
||||
|
||||
// Add mutually exclusive flags to exact match and regex match
|
||||
queryCmd.MarkFlagsMutuallyExclusive("regex-match", "wildcard-match")
|
||||
// Get the not null fields
|
||||
notNullFields := []string{}
|
||||
if dbQueryNotNull != "" {
|
||||
notNullFields = strings.Split(dbQueryNotNull, ",")
|
||||
}
|
||||
|
||||
// Get the user query
|
||||
userQuery := ""
|
||||
if dbQueryUserQuery != "" {
|
||||
userQuery = dbQueryUserQuery
|
||||
}
|
||||
|
||||
// Get the limit
|
||||
limit := dbQueryLimitRows
|
||||
|
||||
// Get the object for the table
|
||||
object := table.Object()
|
||||
|
||||
// Query the database
|
||||
db := sqlite.GetDB()
|
||||
query := db.Model(object).Select(columns)
|
||||
if len(notNullFields) > 0 {
|
||||
for _, field := range notNullFields {
|
||||
query = query.Where(fmt.Sprintf("%s IS NOT NULL", field))
|
||||
}
|
||||
}
|
||||
if userQuery != "" {
|
||||
query = query.Where(userQuery)
|
||||
}
|
||||
if limit > 0 {
|
||||
query = query.Limit(limit)
|
||||
}
|
||||
rows, err := query.Rows()
|
||||
if err != nil {
|
||||
zap.L().Error("db_query",
|
||||
zap.String("message", "failed to execute query"),
|
||||
zap.Error(err),
|
||||
)
|
||||
fmt.Printf("[!] Error executing query: %v\n", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
// Get the columns
|
||||
cols, err := rows.Columns()
|
||||
if err != nil {
|
||||
zap.L().Error("db_query",
|
||||
zap.String("message", "failed to get columns from query"),
|
||||
zap.Error(err),
|
||||
)
|
||||
fmt.Printf("[!] Error getting columns from query: %v\n", err)
|
||||
}
|
||||
|
||||
// Prepare data for pretty.Table
|
||||
headers := cols
|
||||
var tableRows [][]string
|
||||
|
||||
// Process the rows
|
||||
for rows.Next() {
|
||||
values := make([]interface{}, len(cols))
|
||||
pointers := make([]interface{}, len(cols))
|
||||
for i := range values {
|
||||
pointers[i] = &values[i]
|
||||
}
|
||||
if err := rows.Scan(pointers...); err != nil {
|
||||
zap.L().Error("db_query",
|
||||
zap.String("message", "failed to scan row from query"),
|
||||
zap.Error(err),
|
||||
)
|
||||
fmt.Printf("[!] Error scanning row from query: %v\n", err)
|
||||
}
|
||||
|
||||
// Convert row values to strings
|
||||
rowStrings := make([]string, len(values))
|
||||
for i, value := range values {
|
||||
if value == nil {
|
||||
rowStrings[i] = " "
|
||||
} else {
|
||||
// Check if the value is a slice or array
|
||||
switch v := value.(type) {
|
||||
case []string:
|
||||
// Join string slices with commas, no brackets
|
||||
rowStrings[i] = strings.Join(v, ", ")
|
||||
case []interface{}:
|
||||
// Convert interface slice to strings and join
|
||||
strSlice := make([]string, len(v))
|
||||
for j, item := range v {
|
||||
if item == nil {
|
||||
strSlice[j] = ""
|
||||
} else {
|
||||
strSlice[j] = fmt.Sprintf("%v", item)
|
||||
}
|
||||
}
|
||||
rowStrings[i] = strings.Join(strSlice, ", ")
|
||||
case string:
|
||||
// Handle JSON strings that might be arrays
|
||||
if strings.HasPrefix(v, "[") && strings.HasSuffix(v, "]") {
|
||||
// Try to unmarshal JSON array
|
||||
var strArray []string
|
||||
if err := json.Unmarshal([]byte(v), &strArray); err == nil {
|
||||
rowStrings[i] = strings.Join(strArray, ", ")
|
||||
} else {
|
||||
rowStrings[i] = v
|
||||
}
|
||||
} else {
|
||||
rowStrings[i] = v
|
||||
}
|
||||
default:
|
||||
rowStrings[i] = fmt.Sprintf("%v", v)
|
||||
}
|
||||
}
|
||||
}
|
||||
tableRows = append(tableRows, rowStrings)
|
||||
}
|
||||
|
||||
// Display the table
|
||||
pretty.Table(headers, tableRows)
|
||||
}
|
||||
|
||||
// Helper functions to get stored API credentials
|
||||
func getStoredApiKey() string {
|
||||
return badger.GetKey()
|
||||
func rawDBQuery() {
|
||||
db := sqlite.GetDB()
|
||||
rows, err := db.Raw(dbQueryRawQuery).Rows()
|
||||
if err != nil {
|
||||
zap.L().Error("raw_query",
|
||||
zap.String("message", "failed to execute raw query"),
|
||||
zap.Error(err),
|
||||
)
|
||||
fmt.Printf("[!] Error executing raw query: %v\n", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
columns, err := rows.Columns()
|
||||
if err != nil {
|
||||
zap.L().Error("raw_query",
|
||||
zap.String("message", "failed to get columns from raw query"),
|
||||
zap.Error(err),
|
||||
)
|
||||
fmt.Printf("[!] Error getting columns from raw query: %v\n", err)
|
||||
}
|
||||
|
||||
// Prepare data for pretty.Table
|
||||
headers := columns
|
||||
var tableRows [][]string
|
||||
|
||||
// Process the rows
|
||||
for rows.Next() {
|
||||
values := make([]interface{}, len(columns))
|
||||
pointers := make([]interface{}, len(columns))
|
||||
for i := range values {
|
||||
pointers[i] = &values[i]
|
||||
}
|
||||
if err := rows.Scan(pointers...); err != nil {
|
||||
zap.L().Error("raw_query",
|
||||
zap.String("message", "failed to scan row from raw query"),
|
||||
zap.Error(err),
|
||||
)
|
||||
fmt.Printf("[!] Error scanning row from raw query: %v\n", err)
|
||||
}
|
||||
|
||||
// Convert row values to strings
|
||||
rowStrings := make([]string, len(values))
|
||||
for i, value := range values {
|
||||
if value == nil {
|
||||
rowStrings[i] = " "
|
||||
} else {
|
||||
// Check if the value is a slice or array
|
||||
switch v := value.(type) {
|
||||
case []string:
|
||||
// Join string slices with commas, no brackets
|
||||
rowStrings[i] = strings.Join(v, ", ")
|
||||
case []interface{}:
|
||||
// Convert interface slice to strings and join
|
||||
strSlice := make([]string, len(v))
|
||||
for j, item := range v {
|
||||
if item == nil {
|
||||
strSlice[j] = ""
|
||||
} else {
|
||||
strSlice[j] = fmt.Sprintf("%v", item)
|
||||
}
|
||||
}
|
||||
rowStrings[i] = strings.Join(strSlice, ", ")
|
||||
case string:
|
||||
// Handle JSON strings that might be arrays
|
||||
if strings.HasPrefix(v, "[") && strings.HasSuffix(v, "]") {
|
||||
// Try to unmarshal JSON array
|
||||
var strArray []string
|
||||
if err := json.Unmarshal([]byte(v), &strArray); err == nil {
|
||||
rowStrings[i] = strings.Join(strArray, ", ")
|
||||
} else {
|
||||
rowStrings[i] = v
|
||||
}
|
||||
} else {
|
||||
rowStrings[i] = v
|
||||
}
|
||||
default:
|
||||
rowStrings[i] = fmt.Sprintf("%v", v)
|
||||
}
|
||||
}
|
||||
}
|
||||
tableRows = append(tableRows, rowStrings)
|
||||
}
|
||||
|
||||
// Display the table
|
||||
pretty.Table(headers, tableRows)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user