Compare commits
25 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 24b1f99413 | |||
| 22beaf2310 | |||
| 7cff4e70b8 | |||
| 0bd9347074 | |||
| 9a22445e55 | |||
| e167a10fcc | |||
| 84f3becdf2 | |||
| a2358d0714 | |||
| ded3e4ae71 | |||
| fbe1eda8e9 | |||
| 63f302604f | |||
| a0f216508d | |||
| 7d1b7a2225 | |||
| acf6336516 | |||
| ce8079f72e | |||
| 54ee9a192f | |||
| 95791312d0 | |||
| d4e626d574 | |||
| dd2050ce64 | |||
| 075f826816 | |||
| de34de60d9 | |||
| 67c4e0394e | |||
| 49424c1603 | |||
| d4db32c8b9 | |||
| d2cc0d1022 |
|
After Width: | Height: | Size: 23 KiB |
|
Before Width: | Height: | Size: 2.5 MiB |
|
After Width: | Height: | Size: 56 KiB |
|
Before Width: | Height: | Size: 188 KiB |
|
Before Width: | Height: | Size: 495 KiB |
|
Before Width: | Height: | Size: 996 KiB |
|
After Width: | Height: | Size: 89 KiB |
|
After Width: | Height: | Size: 71 KiB |
|
After Width: | Height: | Size: 52 KiB |
|
After Width: | Height: | Size: 26 KiB |
|
After Width: | Height: | Size: 28 KiB |
|
After Width: | Height: | Size: 63 KiB |
|
After Width: | Height: | Size: 172 KiB |
|
Before Width: | Height: | Size: 110 KiB |
|
Before Width: | Height: | Size: 2.1 MiB |
|
After Width: | Height: | Size: 88 KiB |
|
After Width: | Height: | Size: 30 KiB |
|
After Width: | Height: | Size: 16 KiB |
|
After Width: | Height: | Size: 15 KiB |
|
Before Width: | Height: | Size: 966 KiB |
|
Before Width: | Height: | Size: 112 KiB |
|
After Width: | Height: | Size: 7.4 KiB |
|
After Width: | Height: | Size: 7.5 KiB |
|
After Width: | Height: | Size: 47 KiB |
|
After Width: | Height: | Size: 44 KiB |
|
Before Width: | Height: | Size: 1.5 MiB |
|
Before Width: | Height: | Size: 604 KiB |
|
Before Width: | Height: | Size: 100 KiB |
|
Before Width: | Height: | Size: 116 KiB |
|
After Width: | Height: | Size: 69 KiB |
|
After Width: | Height: | Size: 87 KiB |
|
After Width: | Height: | Size: 76 KiB |
|
After Width: | Height: | Size: 29 KiB |
|
After Width: | Height: | Size: 59 KiB |
|
After Width: | Height: | Size: 78 KiB |
|
After Width: | Height: | Size: 101 KiB |
|
Before Width: | Height: | Size: 33 KiB |
@@ -1,4 +1,4 @@
|
||||
# Makefile for Dehasher
|
||||
# Makefile for CrowsNest
|
||||
|
||||
# Go command
|
||||
GO=go
|
||||
@@ -16,7 +16,7 @@ PLATFORMS=linux darwin windows
|
||||
ARCHS=amd64 arm64
|
||||
|
||||
# Version info from git tag or default
|
||||
VERSION=$(shell git describe --tags 2>/dev/null || echo "v1.3.1")
|
||||
VERSION=$(shell git describe --tags 2>/dev/null || echo "v1.3.3")
|
||||
|
||||
.PHONY: all clean build build-all
|
||||
|
||||
@@ -30,14 +30,14 @@ clean:
|
||||
|
||||
# Build for current platform
|
||||
build:
|
||||
$(GO) build -o $(BUILD_DIR)/$(BINARY_NAME) -ldflags "-X main.version=$(VERSION)" crowsnest.go
|
||||
CGO_ENABLED=0 $(GO) build -o $(BUILD_DIR)/$(BINARY_NAME) -ldflags "-X main.version=$(VERSION)" crowsnest.go
|
||||
|
||||
# Build for all platforms
|
||||
build-all: clean
|
||||
@for platform in $(PLATFORMS); do \
|
||||
for arch in $(ARCHS); do \
|
||||
echo "Building for $$platform/$$arch..."; \
|
||||
GOOS=$$platform GOARCH=$$arch $(GO) build -o $(BUILD_DIR)/$(BINARY_NAME)-$$platform-$$arch -ldflags "-X main.version=$(VERSION)" crowsnest.go; \
|
||||
GOOS=$$platform GOARCH=$$arch CGO_ENABLED=0 $(GO) build -o $(BUILD_DIR)/$(BINARY_NAME)-$$platform-$$arch -ldflags "-X main.version=$(VERSION)" crowsnest.go; \
|
||||
if [ "$$platform" = "windows" ]; then \
|
||||
mv $(BUILD_DIR)/$(BINARY_NAME)-$$platform-$$arch $(BUILD_DIR)/$(BINARY_NAME)-$$platform-$$arch.exe; \
|
||||
fi; \
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
<div align="center">
|
||||
<img src=.img/crowsnest.png style="width: 500px; height: auto" alt="Ar1ste1a" title="CrowsNest Logo">
|
||||
<img src=.img/crowsnest.png style="width: 500px; height: auto" alt="CrowsNest Logo" title="CrowsNest Logo">
|
||||
</div>
|
||||
|
||||
### A CLI tool for seamless interaction with the Dehashed and Hunter.io APIs.
|
||||
@@ -28,7 +28,7 @@
|
||||
|
||||
To begin, clone the repository
|
||||
``` bash-session
|
||||
git clone https://github.com/Ar1ste1a/CrowsNest.git
|
||||
git clone https://github.com/Kraken-OffSec/CrowsNest.git
|
||||
cd crowsnest
|
||||
go build crowsnest.go
|
||||
```
|
||||
@@ -39,7 +39,7 @@ go build crowsnest.go
|
||||
|
||||
CrowsNest supports two database storage options:
|
||||
|
||||
1. **Default Path** (default): Stores the database at `~/.local/share/crowsnest/db/dehashed.sqlite`
|
||||
1. **Default Path** (default): Stores the database at `~/.local/share/crowsnest/db/crowsnest.sqlite`
|
||||
2. **Local Path**: Stores the database in the current directory as `./crowsnest.sqlite`
|
||||
|
||||
The **Local Path** option allows for separate databases for different projects or engagements.
|
||||
@@ -58,33 +58,28 @@ To configure the database location:
|
||||
|
||||
## 🌐 Dehashed
|
||||
|
||||
###️ Initial Setup
|
||||
### Initial Setup
|
||||
|
||||
CrowsNest requires an API key from Dehashed. Set it up with:
|
||||

|
||||
```bash
|
||||
ar1ste1a@kali:~$ crowsnest set-dehashed <redacted>
|
||||
```
|
||||
|
||||
### Simple Query
|
||||
CrowsNest can be used simply for example to query for credentials matching a given email domain.
|
||||

|
||||
``` go
|
||||
# Provide credentials for domains matching target.com
|
||||
crowsnest api -D target.com -C
|
||||
crowsnest dehashed -D target.com
|
||||
```
|
||||
|
||||
### Simple Credentials Query
|
||||
CrowsNest can also be used to return only credentials for a given query.
|
||||

|
||||
``` go
|
||||
# Provide credentials for emails matching @target.com
|
||||
crowsnest api -E @target.com -C
|
||||
```
|
||||
|
||||
### Multiple Match Query
|
||||
CrowsNest is capable of handling multiple queries for the same field.
|
||||
This is useful for when you want to search for multiple domains, or multiple usernames.
|
||||
``` go
|
||||
# Provide credentials for domains matching target.com and target2.com, retrieving only credentials
|
||||
crowsnest api -D target.com,target2.com -C
|
||||
crowsnest dehashed -D @target.com -C
|
||||
```
|
||||
|
||||
### Wildcard Query
|
||||
@@ -92,34 +87,47 @@ CrowsNest is capable of handling wildcard queries.
|
||||
A wildcard query cannot begin with a wildcard.
|
||||
This is a limitation of the Dehashed API.
|
||||
An asterisk can be used to denote multiple characters, and a question mark can be used to denote a single character.
|
||||

|
||||
<br>
|
||||

|
||||
``` go
|
||||
# Provide credentials for emails matching @target.com and @target2.com
|
||||
crowsnest api -E @target?.com -C -W
|
||||
crowsnest dehashed -E @target?.com -C -W
|
||||
```
|
||||
|
||||
### Email Query
|
||||
Dehashed has dictated that emails should be searched in the following format:
|
||||
`email:target.name&domain:target.com`.
|
||||
As such, to query an email, please use the following format (note, wildcard is not required but can be useful):
|
||||
<br>
|
||||
*see photo above in Wildcard Query*
|
||||
``` go
|
||||
# Provide credentials for emails matching target.*@target.com
|
||||
crowsnest api -W -E 'target*' -D target.com
|
||||
crowsnest dehashed -W -E 'target*' -D target.com
|
||||
```
|
||||
You may also query the domain and find emails as well
|
||||
``` go
|
||||
# Provide credentials for emails matching target.com
|
||||
crowsnest api -D target.com -C
|
||||
crowsnest dehashed -D target.com -C
|
||||
```
|
||||
|
||||
### Combining Queries
|
||||
CrowsNest is capable of combining queries.
|
||||
This is useful for when you want to query for credentials matching a given email or domain, but only for a specific username.
|
||||

|
||||
``` go
|
||||
# Provide credentials for emails matching @target.com and username containing 'admin'
|
||||
crowsnest dehashed -D target.com -U admin
|
||||
```
|
||||
|
||||
### Regex Query
|
||||
CrowsNest is capable of handling regex queries.
|
||||
Simply denote regex queries with the `-R` flag.
|
||||
Place all regex queries in quotes with the corresponding query flag in single quotes.
|
||||
<br>
|
||||
!!!! *Currently, the Regex Operators appear to be broken. I am waiting on a response from Dehashed* !!!!
|
||||
``` go
|
||||
# Return matches for emails matching this given regex query
|
||||
crowsnest api -R -E '[a-zA-Z0-9]+(?:\.[a-zA-Z0-9]+)?@target.com'
|
||||
crowsnest dehashed -R -E 'joh?n(ath[oa]n)' -D hotmail.com'
|
||||
```
|
||||
|
||||
### Output Text (default JSON)
|
||||
@@ -129,7 +137,7 @@ To change the output format, use the `-f` flag.
|
||||
CrowsNest currently supports JSON, YAML, XML, and TEXT output formats.
|
||||
``` go
|
||||
# Return matches for usernames exactly matching "admin" and write to text file 'admins_file.txt'
|
||||
crowsnest api -U admin -o admins_file -f txt
|
||||
crowsnest dehashed -U admin -o admins_file -f txt
|
||||
```
|
||||
|
||||
---
|
||||
@@ -141,7 +149,7 @@ The WhoIs Lookups require a separate API Credit from the Dehashed API.
|
||||
### Domain Lookup
|
||||
CrowsNest can perform a domain lookup for a given domain.
|
||||
This provides a tree view of the domain's WHOIS information.
|
||||

|
||||

|
||||
```bash
|
||||
# Perform a WHOIS lookup for example.com
|
||||
crowsnest whois -d example.com
|
||||
@@ -156,10 +164,20 @@ The history lookup is immediately written to file and not displayed in the termi
|
||||
crowsnest whois -d example.com -H
|
||||
```
|
||||
|
||||
### Subdomain Scan
|
||||
CrowsNest can perform a subdomain scan for a given domain.
|
||||
This provides a list of all subdomains that match the given query.
|
||||

|
||||
```bash
|
||||
# Perform a WHOIS subdomain scan for google.com
|
||||
crowsnest whois -d google.com -s
|
||||
```
|
||||
|
||||
### Reverse WHOIS Lookup
|
||||
CrowsNest can perform a reverse WHOIS lookup for given criteria.
|
||||
This provides a list of all domains that match the given query.
|
||||
The reverse WHOIS lookup is immediately written to file and not displayed in the terminal or stored in the database.
|
||||
The reverse WHOIS lookup is immediately written to file and not stored in the database.
|
||||

|
||||
```bash
|
||||
# Perform a reverse WHOIS lookup for example.com
|
||||
crowsnest whois -I example.com
|
||||
@@ -168,7 +186,7 @@ crowsnest whois -I example.com
|
||||
### IP Lookup
|
||||
CrowsNest can perform a reverse IP lookup for a given IP address.
|
||||
This provides a list of all domains that match the given query.
|
||||

|
||||

|
||||
```bash
|
||||
# Perform a reverse IP lookup for 8.8.8.8
|
||||
crowsnest whois -i 8.8.8.8
|
||||
@@ -177,28 +195,21 @@ crowsnest whois -i 8.8.8.8
|
||||
### MX Lookup
|
||||
CrowsNest can perform an MX lookup for a given MX hostname.
|
||||
This provides a list of all domains that match the given query.
|
||||

|
||||

|
||||
```bash
|
||||
# Perform a reverse MX lookup for google.com
|
||||
crowsnest whois -m google.com
|
||||
crowsnest whois -m stmp.google.com
|
||||
```
|
||||
### NS Lookup
|
||||
CrowsNest can perform an NS lookup for a given NS hostname.
|
||||
This provides a list of all domains that match the given query.
|
||||
The picture below also includes the --debug global flag.
|
||||

|
||||

|
||||
```bash
|
||||
# Perform a reverse NS lookup for google.com
|
||||
crowsnest whois -n google.com
|
||||
```
|
||||
### Subdomain Scan
|
||||
CrowsNest can perform a subdomain scan for a given domain.
|
||||
This provides a list of all subdomains that match the given query.
|
||||

|
||||
```bash
|
||||
# Perform a WHOIS subdomain scan for google.com
|
||||
crowsnest whois -d google.com -s
|
||||
```
|
||||
|
||||
|
||||
---
|
||||
|
||||
@@ -206,6 +217,7 @@ crowsnest whois -d google.com -s
|
||||
CrowsNest supports Hunter.io lookups.
|
||||
Hunter.io lookups require a separate API Key from the Dehashed API.
|
||||
This can be set using the `set-hunter` command.
|
||||

|
||||
```bash
|
||||
# Set the Hunter.io API key
|
||||
crowsnest set-hunter <redacted>
|
||||
@@ -213,26 +225,17 @@ crowsnest set-hunter <redacted>
|
||||
|
||||
### Domain Search
|
||||
CrowsNest can perform a domain search for a given domain.
|
||||
This provides information about company including a description, social media information and any technologies in use.
|
||||

|
||||
This provides information about company including a description, social media information and any technologies in use.<br>
|
||||

|
||||
```bash
|
||||
# Perform a Hunter.io domain search for example.com
|
||||
crowsnest hunter -d example.com -D
|
||||
```
|
||||
|
||||
### Email Finder
|
||||
CrowsNest can perform an email finder search for a given domain, first name, and last name.
|
||||
This provides information about a user including a confidence score, and any social media accounts linked to a first name, last name and email.
|
||||

|
||||
```bash
|
||||
# Perform a Hunter.io email finder search for example.com
|
||||
crowsnest hunter -d example.com -F John -L Doe -E
|
||||
```
|
||||
|
||||
### Email Verification
|
||||
CrowsNest can perform an email verification search for a given email.
|
||||
This provides a verification and score of a given email address.
|
||||

|
||||

|
||||
```bash
|
||||
# Perform a Hunter.io email verification search for example@target.com
|
||||
crowsnest hunter -e example@target.com -V
|
||||
@@ -241,16 +244,25 @@ crowsnest hunter -e example@target.com -V
|
||||
### Company Enrichment
|
||||
CrowsNest can perform a company enrichment search for a given domain.
|
||||
This provides information about a company given its domain.
|
||||

|
||||

|
||||
```bash
|
||||
# Perform a Hunter.io company enrichment search for example.com
|
||||
crowsnest hunter -d example.com -C
|
||||
```
|
||||
|
||||
### Email Finder
|
||||
CrowsNest can perform an email finder search for a given domain, first name, and last name.
|
||||
This provides information about a user including a confidence score, and any social media accounts linked to a first name, last name and email.
|
||||

|
||||
```bash
|
||||
# Perform a Hunter.io email finder search for example.com
|
||||
crowsnest hunter -d example.com -F John -L Doe -E
|
||||
```
|
||||
|
||||
### Person Enrichment
|
||||
CrowsNest can perform a person enrichment search for a given email.
|
||||
This provides information about a user given an email address..
|
||||

|
||||

|
||||
```bash
|
||||
# Perform a Hunter.io person enrichment search for example@target.com
|
||||
crowsnest hunter -e example@target.com -P
|
||||
@@ -265,7 +277,13 @@ This is a combination of the company and person enrichments given an email addre
|
||||
# Perform a Hunter.io combined enrichment search for example@target.com
|
||||
crowsnest hunter -e example@target.com -B
|
||||
```
|
||||
|
||||
## Debugging
|
||||
CrowsNest supports debugging. This can be enabled using the `--debug` flag in the root command.
|
||||

|
||||
```bash
|
||||
# Perform a Hunter.io combined enrichment search for example@target.com with debugging enabled
|
||||
crowsnest --debug hunter -e example@target.com -B
|
||||
```
|
||||
---
|
||||
## 📊 Database Querying
|
||||
CrowsNest stores query results in a local database.
|
||||
@@ -276,10 +294,10 @@ This database also includes WhoIs Information and Subdomain Scan results, but do
|
||||
## Simple Query
|
||||
#### It's possible to query the database using shorthand and without knowing any SQL at all.
|
||||
#### The following queries the results table where username is not null, only showing the username, email and password columns.
|
||||

|
||||

|
||||
#### You may also add in a simple query using the `-q` flag. The following displays a 'LIKE' clause on the email column.
|
||||
#### Note the '%\<clause\>%' is still required.
|
||||

|
||||

|
||||
|
||||
```bash
|
||||
# Query the database for all results containing the word 'admin' in the username
|
||||
@@ -288,7 +306,7 @@ crowsnest query -t results -q "username LIKE '%admin%'"
|
||||
|
||||
|
||||
## Raw SQL Queries
|
||||

|
||||

|
||||
|
||||
CrowsNest also supports raw SQL queries. This is useful for when you want to query for specific information.
|
||||
```bash
|
||||
@@ -306,6 +324,7 @@ crowsnest query -t results -q "username LIKE '%admin%'" -n username,email,passwo
|
||||
## Listing Tables and Columns
|
||||
CrowsNest supports listing all available tables and columns.
|
||||
This is useful for when you want to query for specific information.
|
||||

|
||||
```bash
|
||||
# List all available tables and columns
|
||||
crowsnest query -a
|
||||
@@ -336,6 +355,7 @@ CrowsNest supports exporting results to a file.
|
||||
This is useful for when you want to requery for specific information without touching the Dehashed API.
|
||||
The export subcommand supports all the same options as the query subcommand.
|
||||
The export subcommand also supports file naming and output format control.
|
||||

|
||||
```bash
|
||||
# Export all results containing the word 'admin' in the username to a text file
|
||||
crowsnest export -t results -q "username LIKE '%admin%'" -o admins_file -f txt
|
||||
@@ -346,8 +366,8 @@ crowsnest export -t results -q "username LIKE '%admin%'" -o admins_file -f txt
|
||||
CrowsNest uses the `zap` logging library for logging. The logs are stored in `~/.local/share/crowsnest/logs`.
|
||||
The logs can be easily queried from the crowsnest CLI.
|
||||
|
||||
### Logs Dates
|
||||
#### crowsnest utilized 'easy time' to determine the appropriate time for a given query.
|
||||
### Filtering by Date
|
||||
#### CrowsNest utilizes 'easy time' to determine the appropriate time for a given query.
|
||||

|
||||
#### You may also used dates mixed with easy time to perform queries.
|
||||

|
||||
@@ -374,27 +394,13 @@ crowsnest logs -s "last 24 hours"
|
||||
crowsnest logs -s "05-01-2025" -v error,fatal
|
||||
```
|
||||
|
||||
## 🎉 Sample Run
|
||||
```bash
|
||||
ar1ste1a@kali:~$ crowsnest api -D <redacted>.com -o <redacted> -f json
|
||||
Making 3 Requests for 10000 Records (30000 Total)
|
||||
[*] Querying Dehashed API...
|
||||
[*] Performing Request...
|
||||
[+] Retrieved 2740 Records
|
||||
[-] Not Enough Entries, ending queries
|
||||
[+] Discovered 10 Credentials
|
||||
[*] Writing entries to file: <redacted>.json
|
||||
[*] Success
|
||||
[*] Completing Process
|
||||
```
|
||||
|
||||
## 🤝 Contributing
|
||||
Contributions are welcome! Submit a pull request to help improve CrowsNest.
|
||||
|
||||
|
||||
## [Buy Me A Coffee](https://buymeacoffee.com/ehosinskiz)
|
||||
|
||||
<div align="center">
|
||||
<img src="https://img.wanman.io/fUSu0/jUtovIFE52.png/raw" style="width: 350px; height: auto" alt="Ar1ste1a" title="Ar1ste1a Offensive Security">
|
||||
<img src=.img/kraken_k.png style="width: 350px; height: auto" alt="Ar1ste1a" title="Ar1ste1a Offensive Security">
|
||||
</div>
|
||||
|
||||
## **Release The Kraken**
|
||||
|
||||
@@ -77,7 +77,7 @@ var (
|
||||
|
||||
// Validate credentials
|
||||
if key == "" {
|
||||
fmt.Println("API key is required. Set the key with the \"set-key\" command. [dehasher set-key <api_key>]")
|
||||
fmt.Println("API key is required. Set the key with the \"set-key\" command. [crowsnest set-key <api_key>]")
|
||||
return
|
||||
}
|
||||
|
||||
@@ -118,6 +118,7 @@ var (
|
||||
dehasher.Start()
|
||||
fmt.Println("\n[*] Completing Process")
|
||||
|
||||
// Store query options
|
||||
err := sqlite.StoreDehashedQueryOptions(queryOptions)
|
||||
if err != nil {
|
||||
if debugGlobal {
|
||||
|
||||
@@ -1,311 +0,0 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"crowsnest/internal/export"
|
||||
"crowsnest/internal/files"
|
||||
"crowsnest/internal/sqlite"
|
||||
"fmt"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/zap"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func init() {
|
||||
// Add Subcommand to db command
|
||||
rootCmd.AddCommand(exportCmd)
|
||||
|
||||
// Add flags specific to export command
|
||||
exportCmd.Flags().IntVarP(&exportLimitRows, "limit", "l", 100, "Limit number of results")
|
||||
exportCmd.Flags().BoolVarP(&exportListAll, "list-all", "a", false, "List all tables and their columns")
|
||||
exportCmd.Flags().StringVarP(&exportTableName, "table", "t", "", "Table to export (results, creds, whois, subdomains, history, runs)")
|
||||
exportCmd.Flags().StringVarP(&exportNotNull, "not-null", "n", "", "Filter for non-null values (comma-separated list, e.g., 'password,email')")
|
||||
exportCmd.Flags().StringVarP(&exportColumns, "columns", "c", "", "Columns to display in output (comma-separated list, e.g., 'username,email,password')")
|
||||
exportCmd.Flags().StringVarP(&exportUserQuery, "user-query", "q", "", "User query to execute")
|
||||
exportCmd.Flags().StringVarP(&exportRawQuery, "raw-query", "r", "", "Raw SQL query to execute")
|
||||
exportCmd.Flags().StringVarP(&exportFormat, "format", "f", "json", "Output format (json, yaml, xml, txt)")
|
||||
exportCmd.Flags().StringVarP(&exportFile, "file", "o", "export", "File to output results to including extension")
|
||||
|
||||
// Add mutually exclusive flags to query and raw-query
|
||||
// Cannot use query and raw-query at the same time
|
||||
exportCmd.MarkFlagsMutuallyExclusive("user-query", "raw-query")
|
||||
// Raw query does not require a table
|
||||
exportCmd.MarkFlagsMutuallyExclusive("user-query", "table")
|
||||
// List all columns does not require a query or raw-query
|
||||
exportCmd.MarkFlagsMutuallyExclusive("raw-query", "list-all")
|
||||
}
|
||||
|
||||
// DB export command
|
||||
var (
|
||||
exportLimitRows int
|
||||
exportListAll bool
|
||||
exportTableName string
|
||||
exportNotNull string
|
||||
exportColumns string
|
||||
exportUserQuery string
|
||||
exportRawQuery string
|
||||
exportFormat string
|
||||
exportFile string
|
||||
|
||||
exportCmd = &cobra.Command{
|
||||
Use: "export",
|
||||
Short: "Export database to file",
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
// If list-all flag is set, list all tables and columns
|
||||
if exportListAll {
|
||||
listAvailableTables()
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Println("[*] Exporting database...")
|
||||
|
||||
// If Raw Query is set, execute it and export
|
||||
if exportRawQuery != "" {
|
||||
fmt.Println("[*] Executing Raw Query...")
|
||||
exportRawDBQuery()
|
||||
return
|
||||
}
|
||||
|
||||
// Validate table name
|
||||
if exportTableName == "" {
|
||||
fmt.Println("[!] Error: Table name is required. Use -t or --table to specify a table.")
|
||||
fmt.Println("[*] Available tables: results, creds, whois, subdomains, history, runs")
|
||||
fmt.Println("[*] Use --list-all to see all tables and their columns.")
|
||||
return
|
||||
}
|
||||
|
||||
if !isValidTable(exportTableName) {
|
||||
fmt.Printf("[!] Error: Unknown table '%s'.\n", exportTableName)
|
||||
fmt.Println("[*] Available tables: results, creds, whois, subdomains, history, runs")
|
||||
fmt.Println("[*] Use --list-all to see all tables and their columns.")
|
||||
return
|
||||
}
|
||||
|
||||
// Validate columns if specified
|
||||
if exportColumns != "" {
|
||||
columns := strings.Split(exportColumns, ",")
|
||||
invalidColumns := validateColumns(exportTableName, columns)
|
||||
if len(invalidColumns) > 0 {
|
||||
fmt.Printf("[!] Error: Invalid column(s) for table '%s': %s\n",
|
||||
exportTableName, strings.Join(invalidColumns, ", "))
|
||||
fmt.Println("[*] Available columns for this table:")
|
||||
for i := 0; i < len(availableTables[exportTableName]); i += 5 {
|
||||
end := i + 5
|
||||
if end > len(availableTables[exportTableName]) {
|
||||
end = len(availableTables[exportTableName])
|
||||
}
|
||||
fmt.Printf(" %s\n", strings.Join(availableTables[exportTableName][i:end], ", "))
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Validate not-null fields if specified
|
||||
if exportNotNull != "" {
|
||||
notNullFields := strings.Split(exportNotNull, ",")
|
||||
invalidFields := validateColumns(exportTableName, notNullFields)
|
||||
if len(invalidFields) > 0 {
|
||||
fmt.Printf("[!] Error: Invalid not-null field(s) for table '%s': %s\n",
|
||||
exportTableName, strings.Join(invalidFields, ", "))
|
||||
fmt.Println("[*] Available columns for this table:")
|
||||
for i := 0; i < len(availableTables[exportTableName]); i += 5 {
|
||||
end := i + 5
|
||||
if end > len(availableTables[exportTableName]) {
|
||||
end = len(availableTables[exportTableName])
|
||||
}
|
||||
fmt.Printf(" %s\n", strings.Join(availableTables[exportTableName][i:end], ", "))
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Determine which table to query based on the tableTypeDBQuery parameter
|
||||
table := sqlite.GetTable(exportTableName)
|
||||
if table == sqlite.UnknownTable {
|
||||
fmt.Printf("[!] Error: Unknown table type '%s'.\n", exportTableName)
|
||||
fmt.Println("[*] Available tables: results, creds, whois, subdomains, history, runs")
|
||||
fmt.Println("[*] Use --list-all to see all tables and their columns.")
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Println("[*] Querying Database...")
|
||||
exportTableQuery(table)
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
// exportTableQuery queries a table and exports the results
|
||||
func exportTableQuery(table sqlite.Table) {
|
||||
// Get the columns to query
|
||||
columns := []string{"*"}
|
||||
if exportColumns != "" {
|
||||
columns = strings.Split(exportColumns, ",")
|
||||
}
|
||||
|
||||
// Get the not null fields
|
||||
notNullFields := []string{}
|
||||
if exportNotNull != "" {
|
||||
notNullFields = strings.Split(exportNotNull, ",")
|
||||
}
|
||||
|
||||
// Get the user query
|
||||
userQuery := ""
|
||||
if exportUserQuery != "" {
|
||||
userQuery = exportUserQuery
|
||||
}
|
||||
|
||||
// Get the limit
|
||||
limit := exportLimitRows
|
||||
|
||||
// Get the object for the table
|
||||
object := table.Object()
|
||||
|
||||
// Check if object is nil (invalid table)
|
||||
if object == nil {
|
||||
fmt.Printf("[!] Error: Table '%s' is not valid or does not exist.\n", exportTableName)
|
||||
return
|
||||
}
|
||||
|
||||
// Query the database
|
||||
db := sqlite.GetDB()
|
||||
query := db.Model(object).Select(columns)
|
||||
if len(notNullFields) > 0 {
|
||||
for _, field := range notNullFields {
|
||||
query = query.Where(fmt.Sprintf("%s IS NOT NULL", field))
|
||||
}
|
||||
}
|
||||
if userQuery != "" {
|
||||
query = query.Where(userQuery)
|
||||
}
|
||||
if limit > 0 {
|
||||
query = query.Limit(limit)
|
||||
}
|
||||
rows, err := query.Rows()
|
||||
if err != nil {
|
||||
zap.L().Error("export_query",
|
||||
zap.String("message", "failed to execute query"),
|
||||
zap.Error(err),
|
||||
)
|
||||
fmt.Printf("[!] Error executing query: %v\n", err)
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
// Get the columns
|
||||
cols, err := rows.Columns()
|
||||
if err != nil {
|
||||
zap.L().Error("export_query",
|
||||
zap.String("message", "failed to get columns from query"),
|
||||
zap.Error(err),
|
||||
)
|
||||
fmt.Printf("[!] Error getting columns from query: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Prepare data for export
|
||||
var results []map[string]interface{}
|
||||
|
||||
// Process the rows
|
||||
for rows.Next() {
|
||||
values := make([]interface{}, len(cols))
|
||||
pointers := make([]interface{}, len(cols))
|
||||
for i := range values {
|
||||
pointers[i] = &values[i]
|
||||
}
|
||||
if err := rows.Scan(pointers...); err != nil {
|
||||
zap.L().Error("export_query",
|
||||
zap.String("message", "failed to scan row from query"),
|
||||
zap.Error(err),
|
||||
)
|
||||
fmt.Printf("[!] Error scanning row from query: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Create a map for this row
|
||||
rowMap := make(map[string]interface{})
|
||||
for i, col := range cols {
|
||||
val := values[i]
|
||||
rowMap[col] = val
|
||||
}
|
||||
|
||||
results = append(results, rowMap)
|
||||
}
|
||||
|
||||
// Export the results
|
||||
exportResults(results)
|
||||
}
|
||||
|
||||
// exportRawDBQuery executes a raw query and exports the results
|
||||
func exportRawDBQuery() {
|
||||
db := sqlite.GetDB()
|
||||
rows, err := db.Raw(exportRawQuery).Rows()
|
||||
if err != nil {
|
||||
zap.L().Error("export_raw_query",
|
||||
zap.String("message", "failed to execute raw query"),
|
||||
zap.Error(err),
|
||||
)
|
||||
fmt.Printf("[!] Error executing raw query: %v\n", err)
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
columns, err := rows.Columns()
|
||||
if err != nil {
|
||||
zap.L().Error("export_raw_query",
|
||||
zap.String("message", "failed to get columns from raw query"),
|
||||
zap.Error(err),
|
||||
)
|
||||
fmt.Printf("[!] Error getting columns from raw query: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Prepare data for export
|
||||
var results []map[string]interface{}
|
||||
|
||||
// Process the rows
|
||||
for rows.Next() {
|
||||
values := make([]interface{}, len(columns))
|
||||
pointers := make([]interface{}, len(columns))
|
||||
for i := range values {
|
||||
pointers[i] = &values[i]
|
||||
}
|
||||
if err := rows.Scan(pointers...); err != nil {
|
||||
zap.L().Error("export_raw_query",
|
||||
zap.String("message", "failed to scan row from raw query"),
|
||||
zap.Error(err),
|
||||
)
|
||||
fmt.Printf("[!] Error scanning row from raw query: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Create a map for this row
|
||||
rowMap := make(map[string]interface{})
|
||||
for i, col := range columns {
|
||||
val := values[i]
|
||||
rowMap[col] = val
|
||||
}
|
||||
|
||||
results = append(results, rowMap)
|
||||
}
|
||||
|
||||
// Export the results
|
||||
exportResults(results)
|
||||
}
|
||||
|
||||
// exportResults exports the results to a file
|
||||
func exportResults(results []map[string]interface{}) {
|
||||
// Get file type
|
||||
fileType := files.GetFileType(exportFormat)
|
||||
|
||||
// Export results
|
||||
err := export.WriteQueryResultsToFile(results, exportFile, fileType)
|
||||
if err != nil {
|
||||
zap.L().Error("export_results",
|
||||
zap.String("message", "failed to write to file"),
|
||||
zap.Error(err),
|
||||
)
|
||||
fmt.Printf("[!] Error writing to file: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Printf("[+] Exported %d records to file: %s%s\n", len(results), exportFile, fileType.Extension())
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"crowsnest/internal/files"
|
||||
hunter "crowsnest/internal/hunter.io"
|
||||
"crowsnest/internal/pretty"
|
||||
"crowsnest/internal/sqlite"
|
||||
"fmt"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/zap"
|
||||
@@ -111,6 +112,24 @@ var (
|
||||
return
|
||||
}
|
||||
|
||||
// Store the users discovered
|
||||
var creds []sqlite.User
|
||||
for _, email := range result.Emails {
|
||||
creds = append(creds, sqlite.User{Email: email.Value})
|
||||
}
|
||||
err = sqlite.StoreUsers(creds)
|
||||
if err != nil {
|
||||
if debugGlobal {
|
||||
debug.PrintInfo("failed to store hunter domain search")
|
||||
debug.PrintError(err)
|
||||
}
|
||||
zap.L().Error("store_hunter_domain_search",
|
||||
zap.String("message", "failed to store hunter domain search"),
|
||||
zap.Error(err),
|
||||
)
|
||||
fmt.Printf("Error storing Hunter.io Domain Search Result: %v\n", err)
|
||||
}
|
||||
|
||||
// Write Hunter.io Domain Search Result to file
|
||||
fmt.Printf("[*] Writing Hunter.io Domain Search Result to file: %s%s\n", hunterOutputFile, fType.Extension())
|
||||
err = export.WriteIStringToFile(result, hunterOutputFile, fType)
|
||||
@@ -166,7 +185,7 @@ var (
|
||||
fmt.Println("Email Find Result:")
|
||||
|
||||
var (
|
||||
headers = []string{"Email", "Score", "Domain", "Accept All", "Position", "Twitter", "Linkedin", "Phone Number", "Company", "Sources", "Verification"}
|
||||
headers = []string{"Email", "Score", "Domain", "Accept All", "Position", "Twitter", "Linkedin", "Phone Number", "Company", "Verification"}
|
||||
rows [][]string
|
||||
)
|
||||
|
||||
@@ -180,7 +199,6 @@ var (
|
||||
result.LinkedinURL,
|
||||
result.PhoneNumber,
|
||||
result.Company,
|
||||
fmt.Sprintf("%v", result.Sources),
|
||||
fmt.Sprintf("%v", result.Verification),
|
||||
})
|
||||
|
||||
@@ -221,24 +239,17 @@ var (
|
||||
|
||||
// Pretty Print Hunter.io Email Verification Result
|
||||
var (
|
||||
headers = []string{"Email", "Status", "Result", "Score", "Regexp", "Gibberish", "Disposable", "Webmail", "MX Records", "SMTP Server", "SMTP Check", "Accept All", "Block", "Sources"}
|
||||
headers = []string{"Email", "Result", "Score", "Disposable", "MX Records", "SMTP Server", "SMTP Check"}
|
||||
rows [][]string
|
||||
)
|
||||
rows = append(rows, []string{
|
||||
result.Email,
|
||||
result.Status,
|
||||
result.Result,
|
||||
fmt.Sprintf("%d", result.Score),
|
||||
fmt.Sprintf("%t", result.Regexp),
|
||||
fmt.Sprintf("%t", result.Gibberish),
|
||||
fmt.Sprintf("%t", result.Disposable),
|
||||
fmt.Sprintf("%t", result.Webmail),
|
||||
fmt.Sprintf("%t", result.MXRecords),
|
||||
fmt.Sprintf("%t", result.SMTPServer),
|
||||
fmt.Sprintf("%t", result.SMTPCheck),
|
||||
fmt.Sprintf("%t", result.AcceptAll),
|
||||
fmt.Sprintf("%t", result.Block),
|
||||
fmt.Sprintf("%v", result.Sources),
|
||||
})
|
||||
|
||||
fmt.Println("Email Verification Result:")
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"crowsnest/internal/debug"
|
||||
"crowsnest/internal/export"
|
||||
"crowsnest/internal/files"
|
||||
"crowsnest/internal/pretty"
|
||||
"crowsnest/internal/sqlite"
|
||||
"encoding/json"
|
||||
@@ -10,132 +13,6 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Map of available tables and their columns
|
||||
var availableTables = map[string][]string{
|
||||
"creds": {
|
||||
"id", "created_at", "updated_at", "deleted_at", "email", "username", "password",
|
||||
},
|
||||
//"history": {
|
||||
// "id", "created_at", "updated_at", "deleted_at", "domain_name", "domain_type",
|
||||
// "registrar_name", "whois_server", "created_date_iso8601", "updated_date_iso8601", "expires_date_iso8601",
|
||||
//},
|
||||
"lookup": {
|
||||
"id", "created_at", "updated_at", "deleted_at", "search_term", "type", "first_seen", "last_visit",
|
||||
"name",
|
||||
},
|
||||
// Query Options
|
||||
"runs": {
|
||||
"id", "created_at", "updated_at", "deleted_at", "max_records", "max_requests", "starting_page",
|
||||
"output_format", "output_file", "regex_match", "wildcard_match", "username_query", "email_query",
|
||||
"ip_query", "pass_query", "hash_query", "name_query", "domain_query", "vin_query", "license_plate_query",
|
||||
"address_query", "phone_query", "social_query", "crypto_address_query", "print_balance", "creds_only",
|
||||
},
|
||||
"results": {
|
||||
"id", "created_at", "updated_at", "deleted_at", "dehashed_id", "email", "ip_address", "username",
|
||||
"password", "hashed_password", "hash_type", "name", "vin", "license_plate", "url", "social",
|
||||
"cryptocurrency_address", "address", "phone", "company", "database_name",
|
||||
},
|
||||
"subdomains": {
|
||||
"id", "created_at", "updated_at", "deleted_at", "domain", "first_seen", "last_seen",
|
||||
},
|
||||
"whois": {
|
||||
"id", "created_at", "updated_at", "deleted_at", "audit", "contact_email", "created_date", "created_date_normalized",
|
||||
"domain_name", "domain_name_ext", "estimated_domain_age", "expires_date", "expires_date_normalized", "footer", "header",
|
||||
"name_servers", "parse_code", "raw_text", "registrant", "registrar_iana_id", "registrar_name", "registry_data",
|
||||
"status", "stripped_text", "updated_date", "updated_date_normalized",
|
||||
},
|
||||
"hunter_domain": {
|
||||
"id", "created_at", "updated_at", "deleted_at", "domain", "disposable", "webmail", "accept_all", "pattern",
|
||||
"organization", "description", "industry", "twitter", "facebook", "linkedin", "instagram", "youtube",
|
||||
"technologies", "country", "state", "city", "postal_code", "street", "headcount", "company_type", "emails", "linked_domains",
|
||||
},
|
||||
"hunter_email": {
|
||||
"id", "created_at", "updated_at", "deleted_at", "value", "type", "confidence", "sources", "first_name", "last_name",
|
||||
"position", "position_raw", "seniority", "department", "linkedin", "twitter", "phone_number", "verification_date", "verification_status",
|
||||
},
|
||||
}
|
||||
|
||||
// Function to list available tables and their columns
|
||||
func listAvailableTables() {
|
||||
fmt.Println("Available tables and columns:")
|
||||
|
||||
// Prepare data for pretty.Table
|
||||
headers := []string{"Table", "Columns"}
|
||||
var tableRows [][]string
|
||||
|
||||
// Sort tables alphabetically for consistent output
|
||||
var tableNames []string
|
||||
for tableName := range availableTables {
|
||||
tableNames = append(tableNames, tableName)
|
||||
}
|
||||
|
||||
// Simple bubble sort for table names
|
||||
for i := 0; i < len(tableNames)-1; i++ {
|
||||
for j := 0; j < len(tableNames)-i-1; j++ {
|
||||
if tableNames[j] > tableNames[j+1] {
|
||||
tableNames[j], tableNames[j+1] = tableNames[j+1], tableNames[j]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Create rows for the table
|
||||
for _, tableName := range tableNames {
|
||||
columns := availableTables[tableName]
|
||||
|
||||
// Format columns with line breaks for better readability
|
||||
var formattedColumns string
|
||||
for i := 0; i < len(columns); i += 5 {
|
||||
end := i + 5
|
||||
if end > len(columns) {
|
||||
end = len(columns)
|
||||
}
|
||||
if i > 0 {
|
||||
formattedColumns += "\n"
|
||||
}
|
||||
formattedColumns += strings.Join(columns[i:end], ", ")
|
||||
}
|
||||
|
||||
tableRows = append(tableRows, []string{tableName, formattedColumns})
|
||||
}
|
||||
|
||||
// Display the table
|
||||
pretty.Table(headers, tableRows)
|
||||
}
|
||||
|
||||
// Function to validate table name
|
||||
func isValidTable(tableName string) bool {
|
||||
_, exists := availableTables[tableName]
|
||||
return exists
|
||||
}
|
||||
|
||||
// Function to validate column names for a specific table
|
||||
func validateColumns(tableName string, columns []string) []string {
|
||||
if tableName == "" || columns == nil || len(columns) == 0 || columns[0] == "*" {
|
||||
return nil
|
||||
}
|
||||
|
||||
tableColumns, exists := availableTables[tableName]
|
||||
if !exists {
|
||||
return []string{fmt.Sprintf("Table '%s' does not exist", tableName)}
|
||||
}
|
||||
|
||||
var invalidColumns []string
|
||||
for _, col := range columns {
|
||||
valid := false
|
||||
for _, tableCol := range tableColumns {
|
||||
if col == tableCol {
|
||||
valid = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !valid {
|
||||
invalidColumns = append(invalidColumns, col)
|
||||
}
|
||||
}
|
||||
|
||||
return invalidColumns
|
||||
}
|
||||
|
||||
func init() {
|
||||
// Add whois command to root command
|
||||
rootCmd.AddCommand(queryCmd)
|
||||
@@ -148,6 +25,8 @@ func init() {
|
||||
queryCmd.Flags().StringVarP(&dbQueryUserQuery, "user-query", "q", "", "User query to execute")
|
||||
queryCmd.Flags().StringVarP(&dbQueryRawQuery, "raw-query", "r", "", "Raw SQL query to execute")
|
||||
queryCmd.Flags().BoolVarP(&dbQueryListAll, "list-all", "a", false, "List all tables and their columns")
|
||||
queryCmd.Flags().StringVarP(&dbQueryFormat, "format", "f", "json", "Output format (json, yaml, xml, txt)")
|
||||
queryCmd.Flags().StringVarP(&dbQueryFile, "file", "o", "query", "File to output results to")
|
||||
|
||||
// Add mutually exclusive flags to query and raw-query
|
||||
// Cannot use query and raw-query at the same time
|
||||
@@ -166,11 +45,14 @@ var (
|
||||
dbQueryUserQuery string
|
||||
dbQueryRawQuery string
|
||||
dbQueryListAll bool
|
||||
dbQueryFormat string
|
||||
dbQueryFile string
|
||||
|
||||
queryCmd = &cobra.Command{
|
||||
Use: "query",
|
||||
Short: "Query the database",
|
||||
Long: `Query the database for various information.`,
|
||||
Long: `Query the database for various information.
|
||||
If file is specified, results are written to file and not displayed in the terminal.`,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
// If list-all flag is set, list all tables and columns
|
||||
if dbQueryListAll {
|
||||
@@ -321,6 +203,49 @@ func tableQuery(table sqlite.Table) {
|
||||
return
|
||||
}
|
||||
|
||||
// Export results if file name is specified
|
||||
if len(strings.TrimSpace(dbQueryFile)) > 0 {
|
||||
fmt.Println("[*] Exporting results to file...")
|
||||
|
||||
if debugGlobal {
|
||||
debug.PrintInfo("exporting results to file: " + dbQueryFile)
|
||||
}
|
||||
// Prepare data for export
|
||||
var results []map[string]interface{}
|
||||
|
||||
// Process the rows
|
||||
for rows.Next() {
|
||||
values := make([]interface{}, len(cols))
|
||||
pointers := make([]interface{}, len(cols))
|
||||
for i := range values {
|
||||
pointers[i] = &values[i]
|
||||
}
|
||||
if err := rows.Scan(pointers...); err != nil {
|
||||
zap.L().Error("export_query",
|
||||
zap.String("message", "failed to scan row from query"),
|
||||
zap.Error(err),
|
||||
)
|
||||
fmt.Printf("[!] Error scanning row from query: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Create a map for this row
|
||||
rowMap := make(map[string]interface{})
|
||||
for i, col := range cols {
|
||||
val := values[i]
|
||||
rowMap[col] = val
|
||||
}
|
||||
|
||||
results = append(results, rowMap)
|
||||
}
|
||||
|
||||
// Export the results
|
||||
exportQueryResults(results)
|
||||
|
||||
return
|
||||
}
|
||||
fmt.Println("[*] Querying Database...")
|
||||
|
||||
// Prepare data for pretty.Table
|
||||
headers := cols
|
||||
var tableRows [][]string
|
||||
@@ -411,6 +336,49 @@ func rawDBQuery() {
|
||||
return
|
||||
}
|
||||
|
||||
if len(strings.TrimSpace(dbQueryFile)) > 0 {
|
||||
fmt.Println("[*] Exporting results to file...")
|
||||
|
||||
if debugGlobal {
|
||||
debug.PrintInfo("exporting results to file: " + dbQueryFile)
|
||||
}
|
||||
|
||||
// Prepare data for export
|
||||
var results []map[string]interface{}
|
||||
|
||||
// Process the rows
|
||||
for rows.Next() {
|
||||
values := make([]interface{}, len(columns))
|
||||
pointers := make([]interface{}, len(columns))
|
||||
for i := range values {
|
||||
pointers[i] = &values[i]
|
||||
}
|
||||
if err := rows.Scan(pointers...); err != nil {
|
||||
zap.L().Error("export_raw_query",
|
||||
zap.String("message", "failed to scan row from raw query"),
|
||||
zap.Error(err),
|
||||
)
|
||||
fmt.Printf("[!] Error scanning row from raw query: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Create a map for this row
|
||||
rowMap := make(map[string]interface{})
|
||||
for i, col := range columns {
|
||||
val := values[i]
|
||||
rowMap[col] = val
|
||||
}
|
||||
|
||||
results = append(results, rowMap)
|
||||
}
|
||||
|
||||
// Export the results
|
||||
exportQueryResults(results)
|
||||
|
||||
return
|
||||
}
|
||||
fmt.Println("[*] Querying Database...")
|
||||
|
||||
// Prepare data for pretty.Table
|
||||
headers := columns
|
||||
var tableRows [][]string
|
||||
@@ -477,3 +445,22 @@ func rawDBQuery() {
|
||||
// Display the table
|
||||
pretty.Table(headers, tableRows)
|
||||
}
|
||||
|
||||
// exportQueryResults exports the results to a file
|
||||
func exportQueryResults(results []map[string]interface{}) {
|
||||
// Get file type
|
||||
fileType := files.GetFileType(dbQueryFormat)
|
||||
|
||||
// Export results
|
||||
err := export.WriteQueryResultsToFile(results, dbQueryFile, fileType)
|
||||
if err != nil {
|
||||
zap.L().Error("export_results",
|
||||
zap.String("message", "failed to write to file"),
|
||||
zap.Error(err),
|
||||
)
|
||||
fmt.Printf("[!] Error writing to file: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Printf("[+] Exported %d records to file: %s%s\n", len(results), dbQueryFile, fileType.Extension())
|
||||
}
|
||||
|
||||
@@ -16,8 +16,8 @@ var (
|
||||
|
||||
// rootCmd is the base command for the CLI.
|
||||
rootCmd = &cobra.Command{
|
||||
Use: "dehasher",
|
||||
Short: `Dehasher is a cli tool for querying the dehashed api.`,
|
||||
Use: "crowsnest",
|
||||
Short: `CrowsNest is a cli tool for querying the common OSINT api's.`,
|
||||
Long: fmt.Sprintf(
|
||||
"%s\n",
|
||||
`
|
||||
@@ -94,7 +94,7 @@ var setHunterKeyCmd = &cobra.Command{
|
||||
|
||||
var setLocalDb = &cobra.Command{
|
||||
Use: "local-db [true|false]",
|
||||
Short: "Set dehasher to use a local database path instead of the default path",
|
||||
Short: "Set crowsnest to use a local database path instead of the default path",
|
||||
Args: cobra.ExactArgs(1),
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
var useLocalDatabase bool
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"crowsnest/internal/pretty"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Map of available tables and their columns
|
||||
var availableTables = map[string][]string{
|
||||
"users": {
|
||||
"id", "created_at", "updated_at", "deleted_at", "email", "username", "password",
|
||||
},
|
||||
//"history": {
|
||||
// "id", "created_at", "updated_at", "deleted_at", "domain_name", "domain_type",
|
||||
// "registrar_name", "whois_server", "created_date_iso8601", "updated_date_iso8601", "expires_date_iso8601",
|
||||
//},
|
||||
"lookup": {
|
||||
"id", "created_at", "updated_at", "deleted_at", "search_term", "type", "first_seen", "last_visit",
|
||||
"name",
|
||||
},
|
||||
// Query Options
|
||||
"runs": {
|
||||
"id", "created_at", "updated_at", "deleted_at", "max_records", "max_requests", "starting_page",
|
||||
"output_format", "output_file", "regex_match", "wildcard_match", "username_query", "email_query",
|
||||
"ip_query", "pass_query", "hash_query", "name_query", "domain_query", "vin_query", "license_plate_query",
|
||||
"address_query", "phone_query", "social_query", "crypto_address_query", "print_balance", "creds_only",
|
||||
},
|
||||
"dehashed": {
|
||||
"id", "created_at", "updated_at", "deleted_at", "dehashed_id", "email", "ip_address", "username",
|
||||
"password", "hashed_password", "hash_type", "name", "vin", "license_plate", "url", "social",
|
||||
"cryptocurrency_address", "address", "phone", "company", "database_name",
|
||||
},
|
||||
"subdomains": {
|
||||
"id", "created_at", "updated_at", "deleted_at", "domain", "subdomain",
|
||||
},
|
||||
"whois": {
|
||||
"id", "created_at", "updated_at", "deleted_at", "audit", "contact_email", "created_date", "created_date_normalized",
|
||||
"domain_name", "domain_name_ext", "estimated_domain_age", "expires_date", "expires_date_normalized", "footer", "header",
|
||||
"name_servers", "parse_code", "raw_text", "registrant", "registrar_iana_id", "registrar_name", "registry_data",
|
||||
"status", "stripped_text", "updated_date", "updated_date_normalized",
|
||||
},
|
||||
"hunter_domain": {
|
||||
"id", "created_at", "updated_at", "deleted_at", "domain", "disposable", "webmail", "accept_all", "pattern",
|
||||
"organization", "description", "industry", "twitter", "facebook", "linkedin", "instagram", "youtube",
|
||||
"technologies", "country", "state", "city", "postal_code", "street", "headcount", "company_type", "emails", "linked_domains",
|
||||
},
|
||||
"hunter_email": {
|
||||
"id", "created_at", "updated_at", "deleted_at", "value", "type", "confidence", "sources", "first_name", "last_name",
|
||||
"position", "position_raw", "seniority", "department", "linkedin", "twitter", "phone_number", "verification_date", "verification_status",
|
||||
},
|
||||
}
|
||||
|
||||
// Function to list available tables and their columns
|
||||
func listAvailableTables() {
|
||||
fmt.Println("Available tables and columns:")
|
||||
|
||||
// Prepare data for pretty.Table
|
||||
headers := []string{"Table", "Columns"}
|
||||
var tableRows [][]string
|
||||
|
||||
// Sort tables alphabetically for consistent output
|
||||
var tableNames []string
|
||||
for tableName := range availableTables {
|
||||
tableNames = append(tableNames, tableName)
|
||||
}
|
||||
|
||||
// Simple bubble sort for table names
|
||||
for i := 0; i < len(tableNames)-1; i++ {
|
||||
for j := 0; j < len(tableNames)-i-1; j++ {
|
||||
if tableNames[j] > tableNames[j+1] {
|
||||
tableNames[j], tableNames[j+1] = tableNames[j+1], tableNames[j]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Create rows for the table
|
||||
for _, tableName := range tableNames {
|
||||
columns := availableTables[tableName]
|
||||
|
||||
// Format columns with line breaks for better readability
|
||||
var formattedColumns string
|
||||
for i := 0; i < len(columns); i += 5 {
|
||||
end := i + 5
|
||||
if end > len(columns) {
|
||||
end = len(columns)
|
||||
}
|
||||
if i > 0 {
|
||||
formattedColumns += "\n"
|
||||
}
|
||||
formattedColumns += strings.Join(columns[i:end], ", ")
|
||||
}
|
||||
|
||||
tableRows = append(tableRows, []string{tableName, formattedColumns})
|
||||
}
|
||||
|
||||
// Display the table
|
||||
pretty.Table(headers, tableRows)
|
||||
}
|
||||
|
||||
// Function to validate table name
|
||||
func isValidTable(tableName string) bool {
|
||||
_, exists := availableTables[tableName]
|
||||
return exists
|
||||
}
|
||||
|
||||
// Function to validate column names for a specific table
|
||||
func validateColumns(tableName string, columns []string) []string {
|
||||
if tableName == "" || columns == nil || len(columns) == 0 || columns[0] == "*" {
|
||||
return nil
|
||||
}
|
||||
|
||||
tableColumns, exists := availableTables[tableName]
|
||||
if !exists {
|
||||
return []string{fmt.Sprintf("Table '%s' does not exist", tableName)}
|
||||
}
|
||||
|
||||
var invalidColumns []string
|
||||
for _, col := range columns {
|
||||
valid := false
|
||||
for _, tableCol := range tableColumns {
|
||||
if col == tableCol {
|
||||
valid = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !valid {
|
||||
invalidColumns = append(invalidColumns, col)
|
||||
}
|
||||
}
|
||||
|
||||
return invalidColumns
|
||||
}
|
||||
@@ -0,0 +1,312 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"crowsnest/internal/sqlite"
|
||||
"fmt"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/zap"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func init() {
|
||||
// Add targets command to root command
|
||||
rootCmd.AddCommand(targetsCmd)
|
||||
|
||||
// Add flags specific to targets command
|
||||
targetsCmd.Flags().StringVarP(&targetsOutputFile, "output", "o", "targets", "Output file name (required)")
|
||||
targetsCmd.Flags().BoolVarP(&targetsExternal, "external", "e", false, "Output external format (email:password)")
|
||||
targetsCmd.Flags().BoolVarP(&targetsInternal, "internal", "i", false, "Output internal format (username:password)")
|
||||
targetsCmd.Flags().BoolVarP(&targetsSubdomains, "subdomains", "s", false, "Output subdomains")
|
||||
targetsCmd.Flags().BoolVarP(&targetsEmails, "emails", "E", false, "Output emails only (no passwords)")
|
||||
targetsCmd.Flags().StringVarP(&targetsDomain, "domain", "d", "", "Filter by domain (for emails and subdomains)")
|
||||
|
||||
// Add mutually exclusive flags to targets command
|
||||
targetsCmd.MarkFlagsMutuallyExclusive("external", "internal", "subdomains", "emails")
|
||||
}
|
||||
|
||||
var (
|
||||
// Targets command flags
|
||||
targetsOutputFile string
|
||||
targetsExternal bool
|
||||
targetsInternal bool
|
||||
targetsSubdomains bool
|
||||
targetsEmails bool
|
||||
targetsDomain string
|
||||
|
||||
// Targets command
|
||||
targetsCmd = &cobra.Command{
|
||||
Use: "targets",
|
||||
Short: "Export users and subdomains in formats suitable for external tools",
|
||||
Long: `Export users and subdomains from the database in easily digestible formats for tools like sprays or other security testing tools.
|
||||
|
||||
Formats:
|
||||
--external (-e): Output in email:password format
|
||||
--internal (-i): Output in username:password format
|
||||
--emails (-E): Output emails only (no passwords)
|
||||
--subdomains (-s): Output subdomains only
|
||||
|
||||
Options:
|
||||
--domain (-d): Filter results by domain (applies to emails and subdomains)
|
||||
--output (-o): Specify output file name (required)
|
||||
|
||||
Examples:
|
||||
# Export all external credentials (email:password)
|
||||
crowsnest targets -e -o external_creds
|
||||
|
||||
# Export internal credentials for a specific domain
|
||||
crowsnest targets -i -d example.com -o internal_creds
|
||||
|
||||
# Export all emails
|
||||
crowsnest targets -E -o all_emails
|
||||
|
||||
# Export emails for a specific domain
|
||||
crowsnest targets -E -d example.com -o domain_emails
|
||||
|
||||
# Export subdomains for a specific domain
|
||||
crowsnest targets -s -d example.com -o subdomains
|
||||
|
||||
# Export all subdomains
|
||||
crowsnest targets -s -o all_subdomains`,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
// Validate that at least one format is specified
|
||||
if !targetsExternal && !targetsInternal && !targetsSubdomains && !targetsEmails {
|
||||
fmt.Println("[!] Error: You must specify at least one output format:")
|
||||
fmt.Println(" --external (-e) for email:password format")
|
||||
fmt.Println(" --internal (-i) for username:password format")
|
||||
fmt.Println(" --emails (-E) for emails only")
|
||||
fmt.Println(" --subdomains (-s) for subdomains")
|
||||
return
|
||||
}
|
||||
|
||||
if debugGlobal {
|
||||
zap.L().Info("targets_debug",
|
||||
zap.String("message", "targets command started"),
|
||||
zap.Bool("external", targetsExternal),
|
||||
zap.Bool("internal", targetsInternal),
|
||||
zap.Bool("subdomains", targetsSubdomains),
|
||||
zap.Bool("emails", targetsEmails),
|
||||
zap.String("domain", targetsDomain),
|
||||
zap.String("output_file", targetsOutputFile),
|
||||
)
|
||||
}
|
||||
|
||||
// Execute the targets export
|
||||
err := executeTargetsExport()
|
||||
if err != nil {
|
||||
fmt.Printf("[!] Error: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Printf("[+] Successfully exported targets to: %s\n", targetsOutputFile)
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
// executeTargetsExport performs the main logic for exporting targets
|
||||
func executeTargetsExport() error {
|
||||
var outputLines []string
|
||||
|
||||
// Export external credentials (email:password)
|
||||
if targetsExternal {
|
||||
if debugGlobal {
|
||||
fmt.Println("[*] Exporting external credentials (email:password)...")
|
||||
}
|
||||
|
||||
externalCreds, err := getExternalCredentials()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get external credentials: %v", err)
|
||||
}
|
||||
|
||||
for _, cred := range externalCreds {
|
||||
if cred.Email != "" && cred.Password != "" {
|
||||
outputLines = append(outputLines, fmt.Sprintf("%s:%s", cred.Email, cred.Password))
|
||||
}
|
||||
}
|
||||
|
||||
if debugGlobal {
|
||||
fmt.Printf("[*] Found %d external credentials\n", len(externalCreds))
|
||||
}
|
||||
}
|
||||
|
||||
// Export internal credentials (username:password)
|
||||
if targetsInternal {
|
||||
if debugGlobal {
|
||||
fmt.Println("[*] Exporting internal credentials (username:password)...")
|
||||
}
|
||||
|
||||
internalCreds, err := getInternalCredentials()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get internal credentials: %v", err)
|
||||
}
|
||||
|
||||
for _, cred := range internalCreds {
|
||||
if cred.Username != "" && cred.Password != "" {
|
||||
outputLines = append(outputLines, fmt.Sprintf("%s:%s", cred.Username, cred.Password))
|
||||
}
|
||||
}
|
||||
|
||||
if debugGlobal {
|
||||
fmt.Printf("[*] Found %d internal credentials\n", len(internalCreds))
|
||||
}
|
||||
}
|
||||
|
||||
// Export emails only
|
||||
if targetsEmails {
|
||||
if debugGlobal {
|
||||
fmt.Println("[*] Exporting emails only...")
|
||||
}
|
||||
|
||||
emails, err := getEmailsOnly()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get emails: %v", err)
|
||||
}
|
||||
|
||||
for _, email := range emails {
|
||||
if email.Email != "" {
|
||||
outputLines = append(outputLines, email.Email)
|
||||
}
|
||||
}
|
||||
|
||||
if debugGlobal {
|
||||
fmt.Printf("[*] Found %d emails\n", len(emails))
|
||||
}
|
||||
}
|
||||
|
||||
// Export subdomains
|
||||
if targetsSubdomains {
|
||||
if debugGlobal {
|
||||
fmt.Println("[*] Exporting subdomains...")
|
||||
}
|
||||
|
||||
subdomains, err := getSubdomains()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get subdomains: %v", err)
|
||||
}
|
||||
|
||||
for _, subdomain := range subdomains {
|
||||
if subdomain.Subdomain != "" {
|
||||
outputLines = append(outputLines, subdomain.Subdomain)
|
||||
}
|
||||
}
|
||||
|
||||
if debugGlobal {
|
||||
fmt.Printf("[*] Found %d subdomains\n", len(subdomains))
|
||||
}
|
||||
}
|
||||
|
||||
// Write to file
|
||||
if len(outputLines) == 0 {
|
||||
return fmt.Errorf("no data found to export")
|
||||
}
|
||||
|
||||
// Join all lines with newlines and add a single newline at the end
|
||||
content := strings.Join(outputLines, "\n") + "\n"
|
||||
|
||||
err := os.WriteFile(targetsOutputFile, []byte(content), 0644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to write to file: %v", err)
|
||||
}
|
||||
|
||||
if debugGlobal {
|
||||
fmt.Printf("[*] Wrote %d lines to %s\n", len(outputLines), targetsOutputFile)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// getExternalCredentials retrieves credentials for external format (email:password)
|
||||
func getExternalCredentials() ([]sqlite.User, error) {
|
||||
db := sqlite.GetDB()
|
||||
var users []sqlite.User
|
||||
|
||||
query := db.Where("email IS NOT NULL AND email != '' AND password IS NOT NULL AND password != ''")
|
||||
|
||||
// Apply domain filter if specified
|
||||
if targetsDomain != "" {
|
||||
query = query.Where("email LIKE ?", "%@"+targetsDomain)
|
||||
}
|
||||
|
||||
err := query.Find(&users).Error
|
||||
if err != nil {
|
||||
zap.L().Error("get_external_credentials",
|
||||
zap.String("message", "failed to query external credentials"),
|
||||
zap.Error(err),
|
||||
)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return users, nil
|
||||
}
|
||||
|
||||
// getInternalCredentials retrieves credentials for internal format (username:password)
|
||||
func getInternalCredentials() ([]sqlite.User, error) {
|
||||
db := sqlite.GetDB()
|
||||
var users []sqlite.User
|
||||
|
||||
query := db.Where("username IS NOT NULL AND username != '' AND password IS NOT NULL AND password != ''")
|
||||
|
||||
// Apply domain filter if specified (filter usernames that might contain domain info)
|
||||
if targetsDomain != "" {
|
||||
query = query.Where("username LIKE ? OR email LIKE ?", "%"+targetsDomain+"%", "%@"+targetsDomain)
|
||||
}
|
||||
|
||||
err := query.Find(&users).Error
|
||||
if err != nil {
|
||||
zap.L().Error("get_internal_credentials",
|
||||
zap.String("message", "failed to query internal credentials"),
|
||||
zap.Error(err),
|
||||
)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return users, nil
|
||||
}
|
||||
|
||||
// getEmailsOnly retrieves emails only (no passwords required)
|
||||
func getEmailsOnly() ([]sqlite.User, error) {
|
||||
db := sqlite.GetDB()
|
||||
var users []sqlite.User
|
||||
|
||||
query := db.Where("email IS NOT NULL AND email != ''")
|
||||
|
||||
// Apply domain filter if specified
|
||||
if targetsDomain != "" {
|
||||
query = query.Where("email LIKE ?", "%@"+targetsDomain)
|
||||
}
|
||||
|
||||
err := query.Find(&users).Error
|
||||
if err != nil {
|
||||
zap.L().Error("get_emails_only",
|
||||
zap.String("message", "failed to query emails"),
|
||||
zap.Error(err),
|
||||
)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return users, nil
|
||||
}
|
||||
|
||||
// getSubdomains retrieves subdomains from the database
|
||||
func getSubdomains() ([]sqlite.Subdomain, error) {
|
||||
db := sqlite.GetDB()
|
||||
var subdomains []sqlite.Subdomain
|
||||
|
||||
query := db.Where("subdomain IS NOT NULL AND subdomain != ''")
|
||||
|
||||
// Apply domain filter if specified
|
||||
if targetsDomain != "" {
|
||||
query = query.Where("domain = ? OR subdomain LIKE ?", targetsDomain, "%."+targetsDomain)
|
||||
}
|
||||
|
||||
err := query.Find(&subdomains).Error
|
||||
if err != nil {
|
||||
zap.L().Error("get_subdomains",
|
||||
zap.String("message", "failed to query subdomains"),
|
||||
zap.Error(err),
|
||||
)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return subdomains, nil
|
||||
}
|
||||
@@ -26,7 +26,7 @@ func init() {
|
||||
whoisCmd.Flags().StringVarP(&whoisNSAddress, "ns", "n", "", "NS hostname for reverse NS lookup")
|
||||
whoisCmd.Flags().StringVarP(&whoisInclude, "include", "I", "", "Up to 4 Terms to include in reverse WHOIS search (comma-separated)")
|
||||
whoisCmd.Flags().StringVarP(&whoisExclude, "exclude", "E", "", "Up to 4 Terms to exclude in reverse WHOIS search (comma-separated)")
|
||||
whoisCmd.Flags().StringVarP(&whoisReverseType, "type", "t", "registrant", "Type of reverse WHOIS search ([default] current or historic)")
|
||||
whoisCmd.Flags().StringVarP(&whoisReverseType, "type", "t", "current", "Type of reverse WHOIS search ([default] current or historic)")
|
||||
whoisCmd.Flags().StringVarP(&whoisOutputFormat, "format", "f", "text", "Output format (text, json)")
|
||||
whoisCmd.Flags().StringVarP(&whoisOutputFile, "output", "o", "whois", "File to output results to including extension")
|
||||
whoisCmd.Flags().BoolVarP(&whoisShowCredits, "credits", "c", false, "Show remaining WHOIS credits")
|
||||
@@ -59,7 +59,7 @@ var (
|
||||
|
||||
// Validate credentials
|
||||
if key == "" {
|
||||
fmt.Println("API key is required. Set the key with the \"set-key\" command. [dehasher set-key <api_key>]")
|
||||
fmt.Println("API key is required. Set the key with the \"set-key\" command. [crowsnest set-key <api_key>]")
|
||||
return
|
||||
}
|
||||
|
||||
@@ -118,57 +118,59 @@ var (
|
||||
if whoisDomain != "" {
|
||||
fmt.Println("[*] Performing WHOIS lookup...")
|
||||
|
||||
// Domain lookup
|
||||
result, err := w.WhoisSearch(whoisDomain)
|
||||
if err != nil {
|
||||
if debugGlobal {
|
||||
debug.PrintInfo("failed to perform whois search")
|
||||
debug.PrintError(err)
|
||||
if !whoisHistory && !whoisSubdomainScan {
|
||||
// Domain lookup
|
||||
result, err := w.WhoisSearch(whoisDomain)
|
||||
if err != nil {
|
||||
if debugGlobal {
|
||||
debug.PrintInfo("failed to perform whois search")
|
||||
debug.PrintError(err)
|
||||
}
|
||||
zap.L().Error("whois_search",
|
||||
zap.String("message", "failed to perform whois search"),
|
||||
zap.Error(err),
|
||||
)
|
||||
fmt.Printf("Error performing WHOIS lookup: %v\n", err)
|
||||
return
|
||||
}
|
||||
zap.L().Error("whois_search",
|
||||
zap.String("message", "failed to perform whois search"),
|
||||
zap.Error(err),
|
||||
)
|
||||
fmt.Printf("Error performing WHOIS lookup: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
if whoisShowCredits {
|
||||
checkBalance(w)
|
||||
}
|
||||
|
||||
// Fix the output format to use proper formatting
|
||||
fmt.Println("WHOIS Lookup Result:")
|
||||
|
||||
// Store the record
|
||||
err = sqlite.StoreWhoisRecord(result)
|
||||
if err != nil {
|
||||
if debugGlobal {
|
||||
debug.PrintInfo("failed to store whois record")
|
||||
debug.PrintError(err)
|
||||
if whoisShowCredits {
|
||||
checkBalance(w)
|
||||
}
|
||||
zap.L().Error("store_whois_record",
|
||||
zap.String("message", "failed to store whois record"),
|
||||
zap.Error(err),
|
||||
)
|
||||
fmt.Printf("Error storing WHOIS record: %v\n", err)
|
||||
// Continue execution even if storage fails
|
||||
}
|
||||
|
||||
// Pretty Print WhoIs Record
|
||||
pretty.WhoIsTree(whoisDomain, result)
|
||||
// Fix the output format to use proper formatting
|
||||
fmt.Println("WHOIS Lookup Result:")
|
||||
|
||||
// Write WhoIs Record to file
|
||||
if len(result.DomainName) != 0 {
|
||||
fmt.Printf("[*] Writing WHOIS record to file: %s%s\n", whoisOutputFile, fType.Extension())
|
||||
err = export.WriteWhoIsRecordToFile(result, whoisOutputFile, fType)
|
||||
} else {
|
||||
if debugGlobal {
|
||||
debug.PrintInfo("no whois record to write to file")
|
||||
// Store the record
|
||||
err = sqlite.StoreWhoisRecord(result)
|
||||
if err != nil {
|
||||
if debugGlobal {
|
||||
debug.PrintInfo("failed to store whois record")
|
||||
debug.PrintError(err)
|
||||
}
|
||||
zap.L().Error("store_whois_record",
|
||||
zap.String("message", "failed to store whois record"),
|
||||
zap.Error(err),
|
||||
)
|
||||
fmt.Printf("Error storing WHOIS record: %v\n", err)
|
||||
// Continue execution even if storage fails
|
||||
}
|
||||
|
||||
// Pretty Print WhoIs Record
|
||||
pretty.WhoIsTree(whoisDomain, result)
|
||||
|
||||
// Write WhoIs Record to file
|
||||
if len(result.DomainName) != 0 {
|
||||
fmt.Printf("[*] Writing WHOIS record to file: %s%s\n", whoisOutputFile, fType.Extension())
|
||||
err = export.WriteWhoIsRecordToFile(result, whoisOutputFile, fType)
|
||||
} else {
|
||||
if debugGlobal {
|
||||
debug.PrintInfo("no whois record to write to file")
|
||||
}
|
||||
zap.L().Info("write_whois_record",
|
||||
zap.String("message", "no whois record to write to file"),
|
||||
)
|
||||
}
|
||||
zap.L().Info("write_whois_record",
|
||||
zap.String("message", "no whois record to write to file"),
|
||||
)
|
||||
}
|
||||
|
||||
if whoisHistory {
|
||||
@@ -238,6 +240,7 @@ var (
|
||||
fmt.Println("[*] Performing WHOIS subdomain scan...")
|
||||
subdomains, err := w.WhoisSubdomainScan(whoisDomain)
|
||||
|
||||
// Get credits
|
||||
if whoisShowCredits {
|
||||
checkBalance(w)
|
||||
}
|
||||
@@ -253,8 +256,13 @@ var (
|
||||
)
|
||||
fmt.Printf("Error performing subdomain scan: %v\n", err)
|
||||
} else {
|
||||
fmt.Println("Subdomain Scan:")
|
||||
err = sqlite.StoreWhoisSubdomainRecords(subdomains)
|
||||
// Store subdomains in subdomains table
|
||||
var subs []sqlite.Subdomain
|
||||
for _, s := range subdomains {
|
||||
subs = append(subs, sqlite.Subdomain{Domain: whoisDomain, Subdomain: s.Domain})
|
||||
}
|
||||
|
||||
err = sqlite.StoreSubdomains(subs)
|
||||
if err != nil {
|
||||
if debugGlobal {
|
||||
debug.PrintInfo("failed to store subdomain record")
|
||||
@@ -264,7 +272,7 @@ var (
|
||||
zap.String("message", "failed to store subdomain record"),
|
||||
zap.Error(err),
|
||||
)
|
||||
fmt.Printf("Error storing WHOIS subdomain record: %v\n", err)
|
||||
fmt.Printf("Error storing subdomain record: %v\n", err)
|
||||
}
|
||||
|
||||
// Write the subdomains to file if any
|
||||
@@ -290,6 +298,7 @@ var (
|
||||
}
|
||||
|
||||
// Store the subdomains
|
||||
fmt.Println("Subdomain Scan:")
|
||||
pretty.Table(headers, rows)
|
||||
|
||||
} else {
|
||||
@@ -492,6 +501,12 @@ var (
|
||||
}
|
||||
|
||||
if whoisInclude != "" || whoisExclude != "" {
|
||||
if debugGlobal {
|
||||
debug.PrintInfo("performing reverse whois")
|
||||
debug.PrintInfo("include: " + whoisInclude)
|
||||
debug.PrintInfo("exclude: " + whoisExclude)
|
||||
debug.PrintInfo("reverse type: " + whoisReverseType)
|
||||
}
|
||||
// Reverse WHOIS
|
||||
includeTerms := []string{}
|
||||
if whoisInclude != "" {
|
||||
@@ -511,17 +526,10 @@ var (
|
||||
}
|
||||
}
|
||||
|
||||
if whoisReverseType == "" {
|
||||
if debugGlobal {
|
||||
debug.PrintInfo("reverse type not specified, using default")
|
||||
}
|
||||
whoisReverseType = "current"
|
||||
} else {
|
||||
toLower := strings.ToLower(whoisReverseType)
|
||||
if toLower != "current" && toLower != "historic" {
|
||||
fmt.Println("[!] Error: Invalid reverse type. Must be 'current' or 'historic'.")
|
||||
return
|
||||
}
|
||||
toLower := strings.ToLower(whoisReverseType)
|
||||
if toLower != "current" && toLower != "historic" {
|
||||
fmt.Println("[!] Error: Invalid reverse type. Must be 'current' or 'historic'.")
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Println("[*] Performing reverse WHOIS lookup...")
|
||||
@@ -538,8 +546,42 @@ var (
|
||||
fmt.Printf("Error performing reverse WHOIS: %v\n", err)
|
||||
return
|
||||
}
|
||||
fmt.Println("Reverse WHOIS Result:")
|
||||
fmt.Println(result)
|
||||
|
||||
// Write to file
|
||||
if len(result.DomainsList) > 0 {
|
||||
fmt.Printf("[*] Writing reverse WHOIS results to file: %s%s\n", whoisOutputFile, fType.Extension())
|
||||
err = export.WriteIStringToFile(result, whoisOutputFile, fType)
|
||||
if err != nil {
|
||||
if debugGlobal {
|
||||
debug.PrintInfo("failed to write reverse whois to file")
|
||||
debug.PrintError(err)
|
||||
}
|
||||
zap.L().Error("write_reverse_whois",
|
||||
zap.String("message", "failed to write reverse whois to file"),
|
||||
zap.Error(err),
|
||||
)
|
||||
fmt.Printf("Error writing reverse WHOIS to file: %v\n", err)
|
||||
}
|
||||
|
||||
fmt.Println("Reverse WHOIS Result:")
|
||||
fmt.Printf("Total Domains: %d\n", result.DomainsCount)
|
||||
|
||||
var (
|
||||
headers = []string{"Domain"}
|
||||
rows [][]string
|
||||
)
|
||||
|
||||
for _, r := range result.DomainsList {
|
||||
rows = append(rows, []string{r})
|
||||
}
|
||||
|
||||
pretty.Table(headers, rows)
|
||||
} else {
|
||||
fmt.Println("[!] No results found")
|
||||
zap.L().Info("reverse_whois",
|
||||
zap.String("message", "no results found"),
|
||||
)
|
||||
}
|
||||
|
||||
if whoisShowCredits {
|
||||
checkBalance(w)
|
||||
|
||||
@@ -21,7 +21,7 @@ var (
|
||||
)
|
||||
|
||||
func init() {
|
||||
basePath = filepath.Join(os.Getenv("HOME"), ".local", "share", "Dehasher")
|
||||
basePath = filepath.Join(os.Getenv("HOME"), ".local", "share", "CrowsNest")
|
||||
logPath = filepath.Join(basePath, "logs")
|
||||
storePath = filepath.Join(basePath, "keystore")
|
||||
// dbPath will be set in main() after badger is initialized
|
||||
@@ -82,7 +82,7 @@ func main() {
|
||||
useLocalDB := badger.GetUseLocalDB()
|
||||
if useLocalDB {
|
||||
// Use local database in current directory
|
||||
dbPath = "./dehasher.sqlite"
|
||||
dbPath = "./"
|
||||
zap.L().Info("Using local database", zap.String("path", dbPath))
|
||||
} else {
|
||||
// Use default database path
|
||||
|
||||
@@ -8,6 +8,7 @@ require (
|
||||
github.com/charmbracelet/lipgloss v1.1.0
|
||||
github.com/dgraph-io/badger/v4 v4.7.0
|
||||
github.com/fatih/color v1.15.0
|
||||
github.com/glebarez/sqlite v1.11.0
|
||||
github.com/spf13/cobra v1.9.1
|
||||
github.com/winking324/rzap v0.1.0
|
||||
go.uber.org/zap v1.20.0
|
||||
@@ -27,9 +28,11 @@ require (
|
||||
github.com/charmbracelet/x/term v0.2.1 // indirect
|
||||
github.com/dgraph-io/ristretto/v2 v2.2.0 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/glebarez/go-sqlite v1.21.2 // indirect
|
||||
github.com/go-logr/logr v1.4.2 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/google/flatbuffers v25.2.10+incompatible // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/jinzhu/inflection v1.0.0 // indirect
|
||||
github.com/jinzhu/now v1.1.5 // indirect
|
||||
@@ -40,6 +43,7 @@ require (
|
||||
github.com/mattn/go-runewidth v0.0.16 // indirect
|
||||
github.com/mattn/go-sqlite3 v1.14.22 // indirect
|
||||
github.com/muesli/termenv v0.16.0 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/spf13/pflag v1.0.6 // indirect
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
||||
@@ -53,4 +57,8 @@ require (
|
||||
golang.org/x/sys v0.31.0 // indirect
|
||||
golang.org/x/text v0.23.0 // indirect
|
||||
google.golang.org/protobuf v1.36.6 // indirect
|
||||
modernc.org/libc v1.22.5 // indirect
|
||||
modernc.org/mathutil v1.5.0 // indirect
|
||||
modernc.org/memory v1.5.0 // indirect
|
||||
modernc.org/sqlite v1.23.1 // indirect
|
||||
)
|
||||
|
||||
@@ -34,6 +34,10 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
|
||||
github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
|
||||
github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9gAXWo=
|
||||
github.com/glebarez/go-sqlite v1.21.2/go.mod h1:sfxdZyhQjTM2Wry3gVYWaW072Ri1WMdWJi0k6+3382k=
|
||||
github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw=
|
||||
github.com/glebarez/sqlite v1.11.0/go.mod h1:h8/o8j5wiAsqSPoWELDUdJXhjAhsVliSn7bWZjOhrgQ=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
|
||||
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
@@ -43,6 +47,10 @@ github.com/google/flatbuffers v25.2.10+incompatible h1:F3vclr7C3HpB1k9mxCGRMXq6F
|
||||
github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ=
|
||||
github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
|
||||
@@ -75,6 +83,9 @@ github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
@@ -164,3 +175,11 @@ gorm.io/driver/sqlite v1.5.7 h1:8NvsrhP0ifM7LX9G4zPB97NwovUakUxc+2V2uuf3Z1I=
|
||||
gorm.io/driver/sqlite v1.5.7/go.mod h1:U+J8craQU6Fzkcvu8oLeAQmi50TkwPEhHDEjQZXDah4=
|
||||
gorm.io/gorm v1.26.1 h1:ghB2gUI9FkS46luZtn6DLZ0f6ooBJ5IbVej2ENFDjRw=
|
||||
gorm.io/gorm v1.26.1/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE=
|
||||
modernc.org/libc v1.22.5 h1:91BNch/e5B0uPbJFgqbxXuOnxBQjlS//icfQEGmvyjE=
|
||||
modernc.org/libc v1.22.5/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY=
|
||||
modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ=
|
||||
modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
|
||||
modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds=
|
||||
modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU=
|
||||
modernc.org/sqlite v1.23.1 h1:nrSBg4aRQQwq59JpvGEQ15tNxoO5pX/kUjcRNwSAGQM=
|
||||
modernc.org/sqlite v1.23.1/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk=
|
||||
|
||||
@@ -45,7 +45,7 @@ func GetHardwareEntropy() []byte {
|
||||
username,
|
||||
osInfo,
|
||||
// You could add a static salt here for additional security
|
||||
"Dehasher-static-salt-value",
|
||||
"CrowsNest-static-salt-value",
|
||||
}, ":")
|
||||
|
||||
// Hash the fingerprint to get a 32-byte key
|
||||
|
||||
@@ -166,7 +166,11 @@ func (dcv2 *DehashedClientV2) Search(searchRequest DehashedSearchRequest) (int,
|
||||
zap.String("message", "preparing search request"),
|
||||
)
|
||||
}
|
||||
reqBody, _ := json.Marshal(searchRequest)
|
||||
|
||||
// Create a copy of the search request to avoid modifying the original
|
||||
requestCopy := searchRequest
|
||||
|
||||
reqBody, _ := json.Marshal(requestCopy)
|
||||
|
||||
if dcv2.debug {
|
||||
j := string(reqBody)
|
||||
|
||||
@@ -3,11 +3,12 @@ package dehashed
|
||||
import (
|
||||
"crowsnest/internal/debug"
|
||||
"crowsnest/internal/export"
|
||||
"crowsnest/internal/pretty"
|
||||
"crowsnest/internal/sqlite"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"go.uber.org/zap"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Dehasher is a struct for querying the Dehashed API
|
||||
@@ -208,13 +209,11 @@ func (dh *Dehasher) buildRequest() {
|
||||
|
||||
// parseResults parses the results and writes them to a file
|
||||
func (dh *Dehasher) parseResults() {
|
||||
var data []byte
|
||||
|
||||
zap.L().Info("extracting_credentials")
|
||||
results := dh.client.GetResults()
|
||||
creds := results.ExtractCredentials()
|
||||
fmt.Printf("\n\t[+] Discovered %d Credentials", len(creds))
|
||||
err := sqlite.StoreDehashedCreds(creds)
|
||||
creds := results.ExtractUsers()
|
||||
fmt.Printf(" [+] Discovered %d Credentials\n", len(creds))
|
||||
err := sqlite.StoreUsers(creds)
|
||||
if err != nil {
|
||||
zap.L().Error("store_creds",
|
||||
zap.String("message", "failed to store creds"),
|
||||
@@ -234,28 +233,93 @@ func (dh *Dehasher) parseResults() {
|
||||
zap.L().Info("results_stored", zap.Int("count", len(results.Results)))
|
||||
|
||||
if len(results.Results) > 0 {
|
||||
fmt.Printf("\n\t[*] Writing entries to file: %s.%s", dh.options.OutputFile, dh.options.OutputFormat.String())
|
||||
var (
|
||||
headers = []string{"Email", "Username", "Password"}
|
||||
rows [][]string
|
||||
)
|
||||
|
||||
fmt.Printf(" [*] Writing entries to file: %s.%s\n", dh.options.OutputFile, dh.options.OutputFormat.String())
|
||||
if !dh.options.CredsOnly {
|
||||
err := export.WriteToFile(results, dh.options.OutputFile, dh.options.OutputFormat)
|
||||
if err != nil {
|
||||
fmt.Printf("\n[!] Error Writing to file: %v\n\tOutputting to terminal.", err)
|
||||
data, err = json.MarshalIndent(results, "", " ")
|
||||
fmt.Println(string(data))
|
||||
os.Exit(0)
|
||||
fmt.Printf("[!] Error Writing to file: %v Outputting to terminal.\n", err)
|
||||
zap.L().Error("write_results",
|
||||
zap.String("message", "failed to write results to file"),
|
||||
zap.Error(err),
|
||||
)
|
||||
} else {
|
||||
fmt.Println("\n\t\t[*] Success\n")
|
||||
fmt.Println(" [*] Success")
|
||||
}
|
||||
|
||||
if dh.debug {
|
||||
debug.PrintInfo("printing results table")
|
||||
}
|
||||
|
||||
headers = []string{"Email", "Username", "Password", "Phone", "Company"}
|
||||
if len(results.Results) > 50 {
|
||||
fmt.Println(" [-] Large number of results recovered, displaying first 50...")
|
||||
for i := 0; i < 50; i++ {
|
||||
r := results.Results[i]
|
||||
rows = append(rows, []string{
|
||||
strings.Join(r.Email, ", "),
|
||||
strings.Join(r.Username, ", "),
|
||||
strings.Join(r.Password, ", "),
|
||||
strings.Join(r.Phone, ", "),
|
||||
strings.Join(r.Company, ", ")})
|
||||
}
|
||||
} else {
|
||||
for _, r := range results.Results {
|
||||
rows = append(rows, []string{
|
||||
strings.Join(r.Email, ", "),
|
||||
strings.Join(r.Username, ", "),
|
||||
strings.Join(r.Password, ", "),
|
||||
strings.Join(r.Phone, ", "),
|
||||
strings.Join(r.Company, ", ")})
|
||||
}
|
||||
}
|
||||
|
||||
// Print Table
|
||||
pretty.Table(headers, rows)
|
||||
} else {
|
||||
creds := results.ExtractCredentials()
|
||||
if dh.debug {
|
||||
debug.PrintInfo("extracting credentials")
|
||||
}
|
||||
creds := results.ExtractUsers()
|
||||
if dh.debug {
|
||||
debug.PrintInfo("writing credentials to file")
|
||||
}
|
||||
err := export.WriteCredsToFile(creds, dh.options.OutputFile, dh.options.OutputFormat)
|
||||
if err != nil {
|
||||
fmt.Printf("\n[!] Error Writing to file: %v\n\tOutputting to terminal.", err)
|
||||
data, err = json.MarshalIndent(creds, "", " ")
|
||||
fmt.Println(string(data))
|
||||
os.Exit(0)
|
||||
fmt.Printf("[!] Error Writing to file: %v\n Outputting to terminal.", err)
|
||||
zap.L().Error("write_creds",
|
||||
zap.String("message", "failed to write creds to file"),
|
||||
zap.Error(err),
|
||||
)
|
||||
} else {
|
||||
fmt.Println("\n\t\t[*] Success\n")
|
||||
fmt.Println(" [*] Success")
|
||||
}
|
||||
|
||||
if dh.debug {
|
||||
debug.PrintInfo("printing credentials table")
|
||||
}
|
||||
|
||||
headers = []string{"Email", "Username", "Password"}
|
||||
if len(creds) > 50 {
|
||||
fmt.Println(" [-] Large number of results recovered, displaying first 50...")
|
||||
for i := 0; i < 50; i++ {
|
||||
c := creds[i]
|
||||
rows = append(rows, []string{c.Email, c.Username, c.Password})
|
||||
}
|
||||
} else {
|
||||
for _, c := range creds {
|
||||
rows = append(rows, []string{c.Email, c.Username, c.Password})
|
||||
}
|
||||
}
|
||||
|
||||
// Print Table
|
||||
pretty.Table(headers, rows)
|
||||
}
|
||||
} else {
|
||||
fmt.Println(" [-] No results found")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
func WriteCredsToFile(creds []sqlite.Creds, outputFile string, fileType files.FileType) error {
|
||||
func WriteCredsToFile(creds []sqlite.User, outputFile string, fileType files.FileType) error {
|
||||
var data []byte
|
||||
var err error
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gorm.io/driver/sqlite"
|
||||
sql "github.com/glebarez/sqlite"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/logger"
|
||||
)
|
||||
@@ -29,7 +29,7 @@ func InitDB(dbPath string) (*gorm.DB, error) {
|
||||
zap.L().Error("Failed to create database directory", zap.Error(err))
|
||||
return nil, fmt.Errorf("failed to create database directory: %w", err)
|
||||
}
|
||||
finalDbPath = filepath.Join(dbPath, "dehashed.sqlite")
|
||||
finalDbPath = filepath.Join(dbPath, "crowsnest.sqlite")
|
||||
} else {
|
||||
// Treat as file path
|
||||
// Ensure the directory exists
|
||||
@@ -42,7 +42,7 @@ func InitDB(dbPath string) (*gorm.DB, error) {
|
||||
}
|
||||
|
||||
zap.L().Info("Opening database", zap.String("finalPath", finalDbPath))
|
||||
db, err := gorm.Open(sqlite.Open(finalDbPath), &gorm.Config{
|
||||
db, err := gorm.Open(sql.Open(finalDbPath), &gorm.Config{
|
||||
Logger: logger.Default.LogMode(logger.Silent),
|
||||
})
|
||||
if err != nil {
|
||||
@@ -51,8 +51,8 @@ func InitDB(dbPath string) (*gorm.DB, error) {
|
||||
}
|
||||
|
||||
// Auto migrate your models
|
||||
err = db.AutoMigrate(&Result{}, &Creds{}, &QueryOptions{}, &Creds{}, &WhoisRecord{}, &SubdomainRecord{},
|
||||
&HistoryRecord{}, &LookupResult{}, &HunterDomainData{}, &HunterEmail{}, &PersonData{})
|
||||
err = db.AutoMigrate(&Result{}, &User{}, &QueryOptions{}, &User{}, &WhoisRecord{}, &HistoryRecord{},
|
||||
&LookupResult{}, &HunterDomainData{}, &HunterEmail{}, &PersonData{}, &Subdomain{})
|
||||
if err != nil {
|
||||
zap.L().Error("Failed to migrate database", zap.Error(err))
|
||||
return nil, fmt.Errorf("failed to migrate database: %w", err)
|
||||
@@ -122,7 +122,7 @@ func (t Table) Object() interface{} {
|
||||
case RunsTable:
|
||||
return QueryOptions{}
|
||||
case CredsTable:
|
||||
return Creds{}
|
||||
return User{}
|
||||
case WhoIsTable:
|
||||
return WhoisRecord{}
|
||||
case SubdomainsTable:
|
||||
|
||||
@@ -106,15 +106,15 @@ type Result struct {
|
||||
}
|
||||
|
||||
func (Result) TableName() string {
|
||||
return "results"
|
||||
return "dehashed"
|
||||
}
|
||||
|
||||
type DehashedResults struct {
|
||||
Results []Result `json:"results"`
|
||||
}
|
||||
|
||||
func (dr *DehashedResults) ExtractCredentials() []Creds {
|
||||
var creds []Creds
|
||||
func (dr *DehashedResults) ExtractUsers() []User {
|
||||
var creds []User
|
||||
|
||||
results := dr.Results
|
||||
|
||||
@@ -126,16 +126,22 @@ func (dr *DehashedResults) ExtractCredentials() []Creds {
|
||||
email = r.Email[0]
|
||||
}
|
||||
|
||||
// Get first username if available
|
||||
username := ""
|
||||
if len(r.Username) > 0 {
|
||||
username = r.Username[0]
|
||||
}
|
||||
|
||||
// Get first password
|
||||
password := r.Password[0]
|
||||
|
||||
cred := Creds{Email: email, Password: password}
|
||||
cred := User{Email: email, Password: password, Username: username}
|
||||
creds = append(creds, cred)
|
||||
}
|
||||
}
|
||||
|
||||
go func() {
|
||||
err := StoreDehashedCreds(creds)
|
||||
err := StoreUsers(creds)
|
||||
if err != nil {
|
||||
zap.L().Error("store_creds",
|
||||
zap.String("message", "failed to store creds"),
|
||||
@@ -148,18 +154,11 @@ func (dr *DehashedResults) ExtractCredentials() []Creds {
|
||||
return creds
|
||||
}
|
||||
|
||||
type Creds struct {
|
||||
gorm.Model
|
||||
Email string `json:"email" yaml:"email" xml:"email" gorm:"uniqueIndex:idx_email_username_password"`
|
||||
Username string `json:"username" yaml:"username" xml:"username" gorm:"uniqueIndex:idx_email_username_password"`
|
||||
Password string `json:"password" yaml:"password" xml:"password" gorm:"uniqueIndex:idx_email_username_password"`
|
||||
}
|
||||
|
||||
func (Creds) TableName() string {
|
||||
func (User) TableName() string {
|
||||
return "creds"
|
||||
}
|
||||
|
||||
func (c Creds) ToString() string {
|
||||
func (c User) ToString() string {
|
||||
return fmt.Sprintf("%s%s%s", c.Username, "%", c.Password)
|
||||
}
|
||||
|
||||
@@ -197,38 +196,6 @@ func StoreDehashedResults(results DehashedResults) error {
|
||||
return lastErr
|
||||
}
|
||||
|
||||
func StoreDehashedCreds(creds []Creds) error {
|
||||
if len(creds) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
zap.L().Info("Storing credentials", zap.Int("count", len(creds)))
|
||||
db := GetDB()
|
||||
|
||||
// Use batch insert with conflict handling
|
||||
// This will insert records in batches and continue even if some fail
|
||||
const batchSize = 100
|
||||
var lastErr error
|
||||
|
||||
for i := 0; i < len(creds); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(creds) {
|
||||
end = len(creds)
|
||||
}
|
||||
|
||||
batch := creds[i:end]
|
||||
// Use Clauses with OnConflict DoNothing to skip conflicts
|
||||
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
|
||||
if err != nil {
|
||||
zap.L().Warn("Error storing some credentials", zap.Error(err))
|
||||
lastErr = err
|
||||
// Continue with next batch despite error
|
||||
}
|
||||
}
|
||||
|
||||
return lastErr
|
||||
}
|
||||
|
||||
func StoreDehashedQueryOptions(queryOptions *QueryOptions) error {
|
||||
db := GetDB()
|
||||
return db.Create(queryOptions).Error
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
package sqlite
|
||||
@@ -0,0 +1,45 @@
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"go.uber.org/zap"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
type Subdomain struct {
|
||||
gorm.Model
|
||||
Domain string `json:"domain" yaml:"domain" xml:"domain"`
|
||||
Subdomain string `json:"subdomain" yaml:"subdomain" xml:"subdomain" gorm:"uniqueIndex:idx_subdomain"`
|
||||
}
|
||||
|
||||
func StoreSubdomains(subs []Subdomain) error {
|
||||
if len(subs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
zap.L().Info("Storing subdomains", zap.Int("count", len(subs)))
|
||||
db := GetDB()
|
||||
|
||||
// Use batch insert with conflict handling
|
||||
// This will insert records in batches and continue even if some fail
|
||||
const batchSize = 100
|
||||
var lastErr error
|
||||
|
||||
for i := 0; i < len(subs); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(subs) {
|
||||
end = len(subs)
|
||||
}
|
||||
|
||||
batch := subs[i:end]
|
||||
// Use Clauses with OnConflict DoNothing to skip conflicts
|
||||
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
|
||||
if err != nil {
|
||||
zap.L().Warn("Error storing some credentials", zap.Error(err))
|
||||
lastErr = err
|
||||
// Continue with next batch despite error
|
||||
}
|
||||
}
|
||||
|
||||
return lastErr
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"go.uber.org/zap"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
type User struct {
|
||||
gorm.Model
|
||||
Company string `json:"company" yaml:"company" xml:"company"`
|
||||
Position string `json:"position" yaml:"position" xml:"position"`
|
||||
Department string `json:"department" yaml:"department" xml:"department"`
|
||||
PhoneNumber string `json:"phone_number" yaml:"phone_number" xml:"phone_number"`
|
||||
FullName string `json:"full_name" yaml:"full_name" xml:"full_name"`
|
||||
Phone string `json:"phone" yaml:"phone" xml:"phone"`
|
||||
Linkedin string `json:"linkedin" yaml:"linkedin" xml:"linkedin"`
|
||||
Twitter string `json:"twitter" yaml:"twitter" xml:"twitter"`
|
||||
Facebook string `json:"facebook" yaml:"facebook" xml:"facebook"`
|
||||
Instagram string `json:"instagram" yaml:"instagram" xml:"instagram"`
|
||||
Youtube string `json:"youtube" yaml:"youtube" xml:"youtube"`
|
||||
Gravatar string `json:"gravatar" yaml:"gravatar" xml:"gravatar"`
|
||||
Email string `json:"email" yaml:"email" xml:"email" gorm:"uniqueIndex:idx_email_username_password"`
|
||||
Username string `json:"username" yaml:"username" xml:"username" gorm:"uniqueIndex:idx_email_username_password"`
|
||||
Password string `json:"password" yaml:"password" xml:"password" gorm:"uniqueIndex:idx_email_username_password"`
|
||||
}
|
||||
|
||||
func StoreUsers(users []User) error {
|
||||
if len(users) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
zap.L().Info("Storing credentials", zap.Int("count", len(users)))
|
||||
db := GetDB()
|
||||
|
||||
// Use batch insert with conflict handling
|
||||
// This will insert records in batches and continue even if some fail
|
||||
const batchSize = 100
|
||||
var lastErr error
|
||||
|
||||
for i := 0; i < len(users); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(users) {
|
||||
end = len(users)
|
||||
}
|
||||
|
||||
batch := users[i:end]
|
||||
// Use Clauses with OnConflict DoNothing to skip conflicts
|
||||
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
|
||||
if err != nil {
|
||||
zap.L().Warn("Error storing some credentials", zap.Error(err))
|
||||
lastErr = err
|
||||
// Continue with next batch despite error
|
||||
}
|
||||
}
|
||||
|
||||
return lastErr
|
||||
}
|
||||
@@ -536,37 +536,6 @@ func StoreWhoisRecord(whoisRecord WhoisRecord) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func StoreWhoisSubdomainRecords(subdomainRecords []SubdomainRecord) error {
|
||||
if len(subdomainRecords) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
zap.L().Info("Storing subdomain records", zap.Int("count", len(subdomainRecords)))
|
||||
db := GetDB()
|
||||
|
||||
// Use batch insert with conflict handling
|
||||
const batchSize = 100
|
||||
var lastErr error
|
||||
|
||||
for i := 0; i < len(subdomainRecords); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(subdomainRecords) {
|
||||
end = len(subdomainRecords)
|
||||
}
|
||||
|
||||
batch := subdomainRecords[i:end]
|
||||
// Use Clauses with OnConflict DoNothing to skip conflicts
|
||||
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
|
||||
if err != nil {
|
||||
zap.L().Warn("Error storing some subdomain records", zap.Error(err))
|
||||
lastErr = err
|
||||
// Continue with next batch despite error
|
||||
}
|
||||
}
|
||||
|
||||
return lastErr
|
||||
}
|
||||
|
||||
func StoreWhoisHistoryRecords(historyRecords []HistoryRecord) error {
|
||||
if len(historyRecords) == 0 {
|
||||
return nil
|
||||
@@ -628,3 +597,20 @@ func StoreWhoisLookup(lookup []LookupResult) error {
|
||||
|
||||
return lastErr
|
||||
}
|
||||
|
||||
// ReverseWhoisResponse represents the response from a reverse WHOIS lookup
|
||||
type ReverseWhoisResponse struct {
|
||||
RemainingCredits int `json:"remaining_credits"`
|
||||
Data ReverseWhoisData `json:"data"`
|
||||
}
|
||||
|
||||
// ReverseWhoisData contains the domain count and list from a reverse WHOIS lookup
|
||||
type ReverseWhoisData struct {
|
||||
DomainsCount int `json:"domainsCount"`
|
||||
DomainsList []string `json:"domainsList"`
|
||||
NextPageSearchAfter *string `json:"nextPageSearchAfter"`
|
||||
}
|
||||
|
||||
func (rwd ReverseWhoisData) String() string {
|
||||
return fmt.Sprintf("Domains Count: %d\nDomains List: %v\nNext Page Search After: %v\n", rwd.DomainsCount, rwd.DomainsList, rwd.NextPageSearchAfter)
|
||||
}
|
||||
|
||||
@@ -298,7 +298,9 @@ func (w *DehashedWhoIs) WhoisHistory(domain string) ([]sqlite.HistoryRecord, err
|
||||
return whois.Data.Records, nil
|
||||
}
|
||||
|
||||
func (w *DehashedWhoIs) ReverseWHOIS(include []string, exclude []string, reverseType string) (string, error) {
|
||||
func (w *DehashedWhoIs) ReverseWHOIS(include []string, exclude []string, reverseType string) (sqlite.ReverseWhoisData, error) {
|
||||
var whois sqlite.ReverseWhoisData
|
||||
|
||||
if w.debug {
|
||||
debug.PrintInfo("performing reverse whois search")
|
||||
zap.L().Info("reverse_whois_debug",
|
||||
@@ -329,7 +331,7 @@ func (w *DehashedWhoIs) ReverseWHOIS(include []string, exclude []string, reverse
|
||||
zap.String("message", "failed to create request"),
|
||||
zap.Error(err),
|
||||
)
|
||||
return "", err
|
||||
return whois, err
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
@@ -356,7 +358,7 @@ func (w *DehashedWhoIs) ReverseWHOIS(include []string, exclude []string, reverse
|
||||
zap.String("message", "failed to perform request"),
|
||||
zap.Error(err),
|
||||
)
|
||||
return "", err
|
||||
return whois, err
|
||||
}
|
||||
if res == nil {
|
||||
if w.debug {
|
||||
@@ -365,7 +367,7 @@ func (w *DehashedWhoIs) ReverseWHOIS(include []string, exclude []string, reverse
|
||||
zap.L().Error("reverse_whois",
|
||||
zap.String("message", "response was nil"),
|
||||
)
|
||||
return "", errors.New("response was nil")
|
||||
return whois, errors.New("response was nil")
|
||||
}
|
||||
|
||||
b, err := io.ReadAll(res.Body)
|
||||
@@ -378,7 +380,7 @@ func (w *DehashedWhoIs) ReverseWHOIS(include []string, exclude []string, reverse
|
||||
zap.String("message", "failed to read response body"),
|
||||
zap.Error(err),
|
||||
)
|
||||
return "", err
|
||||
return whois, err
|
||||
}
|
||||
|
||||
// Check for HTTP status code errors
|
||||
@@ -396,7 +398,7 @@ func (w *DehashedWhoIs) ReverseWHOIS(include []string, exclude []string, reverse
|
||||
zap.String("error", dhErr.Error()),
|
||||
zap.String("body_error", string(b)),
|
||||
)
|
||||
return "", &dhErr
|
||||
return whois, &dhErr
|
||||
}
|
||||
|
||||
if w.debug {
|
||||
@@ -404,7 +406,30 @@ func (w *DehashedWhoIs) ReverseWHOIS(include []string, exclude []string, reverse
|
||||
debug.PrintJson(fmt.Sprintf("Body: %s\n", string(b[:])))
|
||||
}
|
||||
|
||||
return string(b), nil
|
||||
var whoisResponse sqlite.ReverseWhoisResponse
|
||||
err = json.Unmarshal(b, &whoisResponse)
|
||||
if err != nil {
|
||||
if w.debug {
|
||||
debug.PrintInfo("failed to unmarshal response body")
|
||||
debug.PrintError(err)
|
||||
}
|
||||
zap.L().Error("reverse_whois",
|
||||
zap.String("message", "failed to unmarshal response body"),
|
||||
zap.Error(err),
|
||||
)
|
||||
return whois, err
|
||||
}
|
||||
|
||||
if w.debug {
|
||||
debug.PrintInfo("unmarshalled response body")
|
||||
debug.PrintJson(fmt.Sprintf("Remaining Credits: %d\n", whoisResponse.RemainingCredits))
|
||||
debug.PrintJson(fmt.Sprintf("Data: %v\n", whoisResponse.Data))
|
||||
}
|
||||
w.balance = whoisResponse.RemainingCredits
|
||||
|
||||
whois = whoisResponse.Data
|
||||
|
||||
return whois, nil
|
||||
}
|
||||
|
||||
func (w *DehashedWhoIs) WhoisIP(ipAddress string) ([]sqlite.LookupResult, error) {
|
||||
|
||||