Compare commits
25 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 24b1f99413 | |||
| 22beaf2310 | |||
| 7cff4e70b8 | |||
| 0bd9347074 | |||
| 9a22445e55 | |||
| e167a10fcc | |||
| 84f3becdf2 | |||
| a2358d0714 | |||
| ded3e4ae71 | |||
| fbe1eda8e9 | |||
| 63f302604f | |||
| a0f216508d | |||
| 7d1b7a2225 | |||
| acf6336516 | |||
| ce8079f72e | |||
| 54ee9a192f | |||
| 95791312d0 | |||
| d4e626d574 | |||
| dd2050ce64 | |||
| 075f826816 | |||
| de34de60d9 | |||
| 67c4e0394e | |||
| 49424c1603 | |||
| d4db32c8b9 | |||
| d2cc0d1022 |
|
After Width: | Height: | Size: 23 KiB |
|
Before Width: | Height: | Size: 2.5 MiB |
|
After Width: | Height: | Size: 56 KiB |
|
Before Width: | Height: | Size: 188 KiB |
|
Before Width: | Height: | Size: 495 KiB |
|
Before Width: | Height: | Size: 996 KiB |
|
After Width: | Height: | Size: 89 KiB |
|
After Width: | Height: | Size: 71 KiB |
|
After Width: | Height: | Size: 52 KiB |
|
After Width: | Height: | Size: 26 KiB |
|
After Width: | Height: | Size: 28 KiB |
|
After Width: | Height: | Size: 63 KiB |
|
After Width: | Height: | Size: 172 KiB |
|
Before Width: | Height: | Size: 110 KiB |
|
Before Width: | Height: | Size: 2.1 MiB |
|
After Width: | Height: | Size: 88 KiB |
|
After Width: | Height: | Size: 30 KiB |
|
After Width: | Height: | Size: 16 KiB |
|
After Width: | Height: | Size: 15 KiB |
|
Before Width: | Height: | Size: 966 KiB |
|
Before Width: | Height: | Size: 112 KiB |
|
After Width: | Height: | Size: 7.4 KiB |
|
After Width: | Height: | Size: 7.5 KiB |
|
After Width: | Height: | Size: 47 KiB |
|
After Width: | Height: | Size: 44 KiB |
|
Before Width: | Height: | Size: 1.5 MiB |
|
Before Width: | Height: | Size: 604 KiB |
|
Before Width: | Height: | Size: 100 KiB |
|
Before Width: | Height: | Size: 116 KiB |
|
After Width: | Height: | Size: 69 KiB |
|
After Width: | Height: | Size: 87 KiB |
|
After Width: | Height: | Size: 76 KiB |
|
After Width: | Height: | Size: 29 KiB |
|
After Width: | Height: | Size: 59 KiB |
|
After Width: | Height: | Size: 78 KiB |
|
After Width: | Height: | Size: 101 KiB |
|
Before Width: | Height: | Size: 33 KiB |
@@ -1,4 +1,4 @@
|
|||||||
# Makefile for Dehasher
|
# Makefile for CrowsNest
|
||||||
|
|
||||||
# Go command
|
# Go command
|
||||||
GO=go
|
GO=go
|
||||||
@@ -16,7 +16,7 @@ PLATFORMS=linux darwin windows
|
|||||||
ARCHS=amd64 arm64
|
ARCHS=amd64 arm64
|
||||||
|
|
||||||
# Version info from git tag or default
|
# Version info from git tag or default
|
||||||
VERSION=$(shell git describe --tags 2>/dev/null || echo "v1.3.1")
|
VERSION=$(shell git describe --tags 2>/dev/null || echo "v1.3.3")
|
||||||
|
|
||||||
.PHONY: all clean build build-all
|
.PHONY: all clean build build-all
|
||||||
|
|
||||||
@@ -30,14 +30,14 @@ clean:
|
|||||||
|
|
||||||
# Build for current platform
|
# Build for current platform
|
||||||
build:
|
build:
|
||||||
$(GO) build -o $(BUILD_DIR)/$(BINARY_NAME) -ldflags "-X main.version=$(VERSION)" crowsnest.go
|
CGO_ENABLED=0 $(GO) build -o $(BUILD_DIR)/$(BINARY_NAME) -ldflags "-X main.version=$(VERSION)" crowsnest.go
|
||||||
|
|
||||||
# Build for all platforms
|
# Build for all platforms
|
||||||
build-all: clean
|
build-all: clean
|
||||||
@for platform in $(PLATFORMS); do \
|
@for platform in $(PLATFORMS); do \
|
||||||
for arch in $(ARCHS); do \
|
for arch in $(ARCHS); do \
|
||||||
echo "Building for $$platform/$$arch..."; \
|
echo "Building for $$platform/$$arch..."; \
|
||||||
GOOS=$$platform GOARCH=$$arch $(GO) build -o $(BUILD_DIR)/$(BINARY_NAME)-$$platform-$$arch -ldflags "-X main.version=$(VERSION)" crowsnest.go; \
|
GOOS=$$platform GOARCH=$$arch CGO_ENABLED=0 $(GO) build -o $(BUILD_DIR)/$(BINARY_NAME)-$$platform-$$arch -ldflags "-X main.version=$(VERSION)" crowsnest.go; \
|
||||||
if [ "$$platform" = "windows" ]; then \
|
if [ "$$platform" = "windows" ]; then \
|
||||||
mv $(BUILD_DIR)/$(BINARY_NAME)-$$platform-$$arch $(BUILD_DIR)/$(BINARY_NAME)-$$platform-$$arch.exe; \
|
mv $(BUILD_DIR)/$(BINARY_NAME)-$$platform-$$arch $(BUILD_DIR)/$(BINARY_NAME)-$$platform-$$arch.exe; \
|
||||||
fi; \
|
fi; \
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
<div align="center">
|
<div align="center">
|
||||||
<img src=.img/crowsnest.png style="width: 500px; height: auto" alt="Ar1ste1a" title="CrowsNest Logo">
|
<img src=.img/crowsnest.png style="width: 500px; height: auto" alt="CrowsNest Logo" title="CrowsNest Logo">
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
### A CLI tool for seamless interaction with the Dehashed and Hunter.io APIs.
|
### A CLI tool for seamless interaction with the Dehashed and Hunter.io APIs.
|
||||||
@@ -28,7 +28,7 @@
|
|||||||
|
|
||||||
To begin, clone the repository
|
To begin, clone the repository
|
||||||
``` bash-session
|
``` bash-session
|
||||||
git clone https://github.com/Ar1ste1a/CrowsNest.git
|
git clone https://github.com/Kraken-OffSec/CrowsNest.git
|
||||||
cd crowsnest
|
cd crowsnest
|
||||||
go build crowsnest.go
|
go build crowsnest.go
|
||||||
```
|
```
|
||||||
@@ -39,7 +39,7 @@ go build crowsnest.go
|
|||||||
|
|
||||||
CrowsNest supports two database storage options:
|
CrowsNest supports two database storage options:
|
||||||
|
|
||||||
1. **Default Path** (default): Stores the database at `~/.local/share/crowsnest/db/dehashed.sqlite`
|
1. **Default Path** (default): Stores the database at `~/.local/share/crowsnest/db/crowsnest.sqlite`
|
||||||
2. **Local Path**: Stores the database in the current directory as `./crowsnest.sqlite`
|
2. **Local Path**: Stores the database in the current directory as `./crowsnest.sqlite`
|
||||||
|
|
||||||
The **Local Path** option allows for separate databases for different projects or engagements.
|
The **Local Path** option allows for separate databases for different projects or engagements.
|
||||||
@@ -58,33 +58,28 @@ To configure the database location:
|
|||||||
|
|
||||||
## 🌐 Dehashed
|
## 🌐 Dehashed
|
||||||
|
|
||||||
###️ Initial Setup
|
### Initial Setup
|
||||||
|
|
||||||
CrowsNest requires an API key from Dehashed. Set it up with:
|
CrowsNest requires an API key from Dehashed. Set it up with:
|
||||||
|

|
||||||
```bash
|
```bash
|
||||||
ar1ste1a@kali:~$ crowsnest set-dehashed <redacted>
|
ar1ste1a@kali:~$ crowsnest set-dehashed <redacted>
|
||||||
```
|
```
|
||||||
|
|
||||||
### Simple Query
|
### Simple Query
|
||||||
CrowsNest can be used simply for example to query for credentials matching a given email domain.
|
CrowsNest can be used simply for example to query for credentials matching a given email domain.
|
||||||
|

|
||||||
``` go
|
``` go
|
||||||
# Provide credentials for domains matching target.com
|
# Provide credentials for domains matching target.com
|
||||||
crowsnest api -D target.com -C
|
crowsnest dehashed -D target.com
|
||||||
```
|
```
|
||||||
|
|
||||||
### Simple Credentials Query
|
### Simple Credentials Query
|
||||||
CrowsNest can also be used to return only credentials for a given query.
|
CrowsNest can also be used to return only credentials for a given query.
|
||||||
|

|
||||||
``` go
|
``` go
|
||||||
# Provide credentials for emails matching @target.com
|
# Provide credentials for emails matching @target.com
|
||||||
crowsnest api -E @target.com -C
|
crowsnest dehashed -D @target.com -C
|
||||||
```
|
|
||||||
|
|
||||||
### Multiple Match Query
|
|
||||||
CrowsNest is capable of handling multiple queries for the same field.
|
|
||||||
This is useful for when you want to search for multiple domains, or multiple usernames.
|
|
||||||
``` go
|
|
||||||
# Provide credentials for domains matching target.com and target2.com, retrieving only credentials
|
|
||||||
crowsnest api -D target.com,target2.com -C
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Wildcard Query
|
### Wildcard Query
|
||||||
@@ -92,34 +87,47 @@ CrowsNest is capable of handling wildcard queries.
|
|||||||
A wildcard query cannot begin with a wildcard.
|
A wildcard query cannot begin with a wildcard.
|
||||||
This is a limitation of the Dehashed API.
|
This is a limitation of the Dehashed API.
|
||||||
An asterisk can be used to denote multiple characters, and a question mark can be used to denote a single character.
|
An asterisk can be used to denote multiple characters, and a question mark can be used to denote a single character.
|
||||||

|
<br>
|
||||||
|

|
||||||
``` go
|
``` go
|
||||||
# Provide credentials for emails matching @target.com and @target2.com
|
# Provide credentials for emails matching @target.com and @target2.com
|
||||||
crowsnest api -E @target?.com -C -W
|
crowsnest dehashed -E @target?.com -C -W
|
||||||
```
|
```
|
||||||
|
|
||||||
### Email Query
|
### Email Query
|
||||||
Dehashed has dictated that emails should be searched in the following format:
|
Dehashed has dictated that emails should be searched in the following format:
|
||||||
`email:target.name&domain:target.com`.
|
`email:target.name&domain:target.com`.
|
||||||
As such, to query an email, please use the following format (note, wildcard is not required but can be useful):
|
As such, to query an email, please use the following format (note, wildcard is not required but can be useful):
|
||||||
|
<br>
|
||||||
|
*see photo above in Wildcard Query*
|
||||||
``` go
|
``` go
|
||||||
# Provide credentials for emails matching target.*@target.com
|
# Provide credentials for emails matching target.*@target.com
|
||||||
crowsnest api -W -E 'target*' -D target.com
|
crowsnest dehashed -W -E 'target*' -D target.com
|
||||||
```
|
```
|
||||||
You may also query the domain and find emails as well
|
You may also query the domain and find emails as well
|
||||||
``` go
|
``` go
|
||||||
# Provide credentials for emails matching target.com
|
# Provide credentials for emails matching target.com
|
||||||
crowsnest api -D target.com -C
|
crowsnest dehashed -D target.com -C
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Combining Queries
|
||||||
|
CrowsNest is capable of combining queries.
|
||||||
|
This is useful for when you want to query for credentials matching a given email or domain, but only for a specific username.
|
||||||
|

|
||||||
|
``` go
|
||||||
|
# Provide credentials for emails matching @target.com and username containing 'admin'
|
||||||
|
crowsnest dehashed -D target.com -U admin
|
||||||
|
```
|
||||||
|
|
||||||
### Regex Query
|
### Regex Query
|
||||||
CrowsNest is capable of handling regex queries.
|
CrowsNest is capable of handling regex queries.
|
||||||
Simply denote regex queries with the `-R` flag.
|
Simply denote regex queries with the `-R` flag.
|
||||||
Place all regex queries in quotes with the corresponding query flag in single quotes.
|
Place all regex queries in quotes with the corresponding query flag in single quotes.
|
||||||
|
<br>
|
||||||
|
!!!! *Currently, the Regex Operators appear to be broken. I am waiting on a response from Dehashed* !!!!
|
||||||
``` go
|
``` go
|
||||||
# Return matches for emails matching this given regex query
|
# Return matches for emails matching this given regex query
|
||||||
crowsnest api -R -E '[a-zA-Z0-9]+(?:\.[a-zA-Z0-9]+)?@target.com'
|
crowsnest dehashed -R -E 'joh?n(ath[oa]n)' -D hotmail.com'
|
||||||
```
|
```
|
||||||
|
|
||||||
### Output Text (default JSON)
|
### Output Text (default JSON)
|
||||||
@@ -129,7 +137,7 @@ To change the output format, use the `-f` flag.
|
|||||||
CrowsNest currently supports JSON, YAML, XML, and TEXT output formats.
|
CrowsNest currently supports JSON, YAML, XML, and TEXT output formats.
|
||||||
``` go
|
``` go
|
||||||
# Return matches for usernames exactly matching "admin" and write to text file 'admins_file.txt'
|
# Return matches for usernames exactly matching "admin" and write to text file 'admins_file.txt'
|
||||||
crowsnest api -U admin -o admins_file -f txt
|
crowsnest dehashed -U admin -o admins_file -f txt
|
||||||
```
|
```
|
||||||
|
|
||||||
---
|
---
|
||||||
@@ -141,7 +149,7 @@ The WhoIs Lookups require a separate API Credit from the Dehashed API.
|
|||||||
### Domain Lookup
|
### Domain Lookup
|
||||||
CrowsNest can perform a domain lookup for a given domain.
|
CrowsNest can perform a domain lookup for a given domain.
|
||||||
This provides a tree view of the domain's WHOIS information.
|
This provides a tree view of the domain's WHOIS information.
|
||||||

|

|
||||||
```bash
|
```bash
|
||||||
# Perform a WHOIS lookup for example.com
|
# Perform a WHOIS lookup for example.com
|
||||||
crowsnest whois -d example.com
|
crowsnest whois -d example.com
|
||||||
@@ -156,10 +164,20 @@ The history lookup is immediately written to file and not displayed in the termi
|
|||||||
crowsnest whois -d example.com -H
|
crowsnest whois -d example.com -H
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Subdomain Scan
|
||||||
|
CrowsNest can perform a subdomain scan for a given domain.
|
||||||
|
This provides a list of all subdomains that match the given query.
|
||||||
|

|
||||||
|
```bash
|
||||||
|
# Perform a WHOIS subdomain scan for google.com
|
||||||
|
crowsnest whois -d google.com -s
|
||||||
|
```
|
||||||
|
|
||||||
### Reverse WHOIS Lookup
|
### Reverse WHOIS Lookup
|
||||||
CrowsNest can perform a reverse WHOIS lookup for given criteria.
|
CrowsNest can perform a reverse WHOIS lookup for given criteria.
|
||||||
This provides a list of all domains that match the given query.
|
This provides a list of all domains that match the given query.
|
||||||
The reverse WHOIS lookup is immediately written to file and not displayed in the terminal or stored in the database.
|
The reverse WHOIS lookup is immediately written to file and not stored in the database.
|
||||||
|

|
||||||
```bash
|
```bash
|
||||||
# Perform a reverse WHOIS lookup for example.com
|
# Perform a reverse WHOIS lookup for example.com
|
||||||
crowsnest whois -I example.com
|
crowsnest whois -I example.com
|
||||||
@@ -168,7 +186,7 @@ crowsnest whois -I example.com
|
|||||||
### IP Lookup
|
### IP Lookup
|
||||||
CrowsNest can perform a reverse IP lookup for a given IP address.
|
CrowsNest can perform a reverse IP lookup for a given IP address.
|
||||||
This provides a list of all domains that match the given query.
|
This provides a list of all domains that match the given query.
|
||||||

|

|
||||||
```bash
|
```bash
|
||||||
# Perform a reverse IP lookup for 8.8.8.8
|
# Perform a reverse IP lookup for 8.8.8.8
|
||||||
crowsnest whois -i 8.8.8.8
|
crowsnest whois -i 8.8.8.8
|
||||||
@@ -177,28 +195,21 @@ crowsnest whois -i 8.8.8.8
|
|||||||
### MX Lookup
|
### MX Lookup
|
||||||
CrowsNest can perform an MX lookup for a given MX hostname.
|
CrowsNest can perform an MX lookup for a given MX hostname.
|
||||||
This provides a list of all domains that match the given query.
|
This provides a list of all domains that match the given query.
|
||||||

|

|
||||||
```bash
|
```bash
|
||||||
# Perform a reverse MX lookup for google.com
|
# Perform a reverse MX lookup for google.com
|
||||||
crowsnest whois -m google.com
|
crowsnest whois -m stmp.google.com
|
||||||
```
|
```
|
||||||
### NS Lookup
|
### NS Lookup
|
||||||
CrowsNest can perform an NS lookup for a given NS hostname.
|
CrowsNest can perform an NS lookup for a given NS hostname.
|
||||||
This provides a list of all domains that match the given query.
|
This provides a list of all domains that match the given query.
|
||||||
The picture below also includes the --debug global flag.
|
The picture below also includes the --debug global flag.
|
||||||

|

|
||||||
```bash
|
```bash
|
||||||
# Perform a reverse NS lookup for google.com
|
# Perform a reverse NS lookup for google.com
|
||||||
crowsnest whois -n google.com
|
crowsnest whois -n google.com
|
||||||
```
|
```
|
||||||
### Subdomain Scan
|
|
||||||
CrowsNest can perform a subdomain scan for a given domain.
|
|
||||||
This provides a list of all subdomains that match the given query.
|
|
||||||

|
|
||||||
```bash
|
|
||||||
# Perform a WHOIS subdomain scan for google.com
|
|
||||||
crowsnest whois -d google.com -s
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -206,6 +217,7 @@ crowsnest whois -d google.com -s
|
|||||||
CrowsNest supports Hunter.io lookups.
|
CrowsNest supports Hunter.io lookups.
|
||||||
Hunter.io lookups require a separate API Key from the Dehashed API.
|
Hunter.io lookups require a separate API Key from the Dehashed API.
|
||||||
This can be set using the `set-hunter` command.
|
This can be set using the `set-hunter` command.
|
||||||
|

|
||||||
```bash
|
```bash
|
||||||
# Set the Hunter.io API key
|
# Set the Hunter.io API key
|
||||||
crowsnest set-hunter <redacted>
|
crowsnest set-hunter <redacted>
|
||||||
@@ -213,26 +225,17 @@ crowsnest set-hunter <redacted>
|
|||||||
|
|
||||||
### Domain Search
|
### Domain Search
|
||||||
CrowsNest can perform a domain search for a given domain.
|
CrowsNest can perform a domain search for a given domain.
|
||||||
This provides information about company including a description, social media information and any technologies in use.
|
This provides information about company including a description, social media information and any technologies in use.<br>
|
||||||

|

|
||||||
```bash
|
```bash
|
||||||
# Perform a Hunter.io domain search for example.com
|
# Perform a Hunter.io domain search for example.com
|
||||||
crowsnest hunter -d example.com -D
|
crowsnest hunter -d example.com -D
|
||||||
```
|
```
|
||||||
|
|
||||||
### Email Finder
|
|
||||||
CrowsNest can perform an email finder search for a given domain, first name, and last name.
|
|
||||||
This provides information about a user including a confidence score, and any social media accounts linked to a first name, last name and email.
|
|
||||||

|
|
||||||
```bash
|
|
||||||
# Perform a Hunter.io email finder search for example.com
|
|
||||||
crowsnest hunter -d example.com -F John -L Doe -E
|
|
||||||
```
|
|
||||||
|
|
||||||
### Email Verification
|
### Email Verification
|
||||||
CrowsNest can perform an email verification search for a given email.
|
CrowsNest can perform an email verification search for a given email.
|
||||||
This provides a verification and score of a given email address.
|
This provides a verification and score of a given email address.
|
||||||

|

|
||||||
```bash
|
```bash
|
||||||
# Perform a Hunter.io email verification search for example@target.com
|
# Perform a Hunter.io email verification search for example@target.com
|
||||||
crowsnest hunter -e example@target.com -V
|
crowsnest hunter -e example@target.com -V
|
||||||
@@ -241,16 +244,25 @@ crowsnest hunter -e example@target.com -V
|
|||||||
### Company Enrichment
|
### Company Enrichment
|
||||||
CrowsNest can perform a company enrichment search for a given domain.
|
CrowsNest can perform a company enrichment search for a given domain.
|
||||||
This provides information about a company given its domain.
|
This provides information about a company given its domain.
|
||||||

|

|
||||||
```bash
|
```bash
|
||||||
# Perform a Hunter.io company enrichment search for example.com
|
# Perform a Hunter.io company enrichment search for example.com
|
||||||
crowsnest hunter -d example.com -C
|
crowsnest hunter -d example.com -C
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Email Finder
|
||||||
|
CrowsNest can perform an email finder search for a given domain, first name, and last name.
|
||||||
|
This provides information about a user including a confidence score, and any social media accounts linked to a first name, last name and email.
|
||||||
|

|
||||||
|
```bash
|
||||||
|
# Perform a Hunter.io email finder search for example.com
|
||||||
|
crowsnest hunter -d example.com -F John -L Doe -E
|
||||||
|
```
|
||||||
|
|
||||||
### Person Enrichment
|
### Person Enrichment
|
||||||
CrowsNest can perform a person enrichment search for a given email.
|
CrowsNest can perform a person enrichment search for a given email.
|
||||||
This provides information about a user given an email address..
|
This provides information about a user given an email address..
|
||||||

|

|
||||||
```bash
|
```bash
|
||||||
# Perform a Hunter.io person enrichment search for example@target.com
|
# Perform a Hunter.io person enrichment search for example@target.com
|
||||||
crowsnest hunter -e example@target.com -P
|
crowsnest hunter -e example@target.com -P
|
||||||
@@ -265,7 +277,13 @@ This is a combination of the company and person enrichments given an email addre
|
|||||||
# Perform a Hunter.io combined enrichment search for example@target.com
|
# Perform a Hunter.io combined enrichment search for example@target.com
|
||||||
crowsnest hunter -e example@target.com -B
|
crowsnest hunter -e example@target.com -B
|
||||||
```
|
```
|
||||||
|
## Debugging
|
||||||
|
CrowsNest supports debugging. This can be enabled using the `--debug` flag in the root command.
|
||||||
|

|
||||||
|
```bash
|
||||||
|
# Perform a Hunter.io combined enrichment search for example@target.com with debugging enabled
|
||||||
|
crowsnest --debug hunter -e example@target.com -B
|
||||||
|
```
|
||||||
---
|
---
|
||||||
## 📊 Database Querying
|
## 📊 Database Querying
|
||||||
CrowsNest stores query results in a local database.
|
CrowsNest stores query results in a local database.
|
||||||
@@ -276,10 +294,10 @@ This database also includes WhoIs Information and Subdomain Scan results, but do
|
|||||||
## Simple Query
|
## Simple Query
|
||||||
#### It's possible to query the database using shorthand and without knowing any SQL at all.
|
#### It's possible to query the database using shorthand and without knowing any SQL at all.
|
||||||
#### The following queries the results table where username is not null, only showing the username, email and password columns.
|
#### The following queries the results table where username is not null, only showing the username, email and password columns.
|
||||||

|

|
||||||
#### You may also add in a simple query using the `-q` flag. The following displays a 'LIKE' clause on the email column.
|
#### You may also add in a simple query using the `-q` flag. The following displays a 'LIKE' clause on the email column.
|
||||||
#### Note the '%\<clause\>%' is still required.
|
#### Note the '%\<clause\>%' is still required.
|
||||||

|

|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Query the database for all results containing the word 'admin' in the username
|
# Query the database for all results containing the word 'admin' in the username
|
||||||
@@ -288,7 +306,7 @@ crowsnest query -t results -q "username LIKE '%admin%'"
|
|||||||
|
|
||||||
|
|
||||||
## Raw SQL Queries
|
## Raw SQL Queries
|
||||||

|

|
||||||
|
|
||||||
CrowsNest also supports raw SQL queries. This is useful for when you want to query for specific information.
|
CrowsNest also supports raw SQL queries. This is useful for when you want to query for specific information.
|
||||||
```bash
|
```bash
|
||||||
@@ -306,6 +324,7 @@ crowsnest query -t results -q "username LIKE '%admin%'" -n username,email,passwo
|
|||||||
## Listing Tables and Columns
|
## Listing Tables and Columns
|
||||||
CrowsNest supports listing all available tables and columns.
|
CrowsNest supports listing all available tables and columns.
|
||||||
This is useful for when you want to query for specific information.
|
This is useful for when you want to query for specific information.
|
||||||
|

|
||||||
```bash
|
```bash
|
||||||
# List all available tables and columns
|
# List all available tables and columns
|
||||||
crowsnest query -a
|
crowsnest query -a
|
||||||
@@ -336,6 +355,7 @@ CrowsNest supports exporting results to a file.
|
|||||||
This is useful for when you want to requery for specific information without touching the Dehashed API.
|
This is useful for when you want to requery for specific information without touching the Dehashed API.
|
||||||
The export subcommand supports all the same options as the query subcommand.
|
The export subcommand supports all the same options as the query subcommand.
|
||||||
The export subcommand also supports file naming and output format control.
|
The export subcommand also supports file naming and output format control.
|
||||||
|

|
||||||
```bash
|
```bash
|
||||||
# Export all results containing the word 'admin' in the username to a text file
|
# Export all results containing the word 'admin' in the username to a text file
|
||||||
crowsnest export -t results -q "username LIKE '%admin%'" -o admins_file -f txt
|
crowsnest export -t results -q "username LIKE '%admin%'" -o admins_file -f txt
|
||||||
@@ -346,8 +366,8 @@ crowsnest export -t results -q "username LIKE '%admin%'" -o admins_file -f txt
|
|||||||
CrowsNest uses the `zap` logging library for logging. The logs are stored in `~/.local/share/crowsnest/logs`.
|
CrowsNest uses the `zap` logging library for logging. The logs are stored in `~/.local/share/crowsnest/logs`.
|
||||||
The logs can be easily queried from the crowsnest CLI.
|
The logs can be easily queried from the crowsnest CLI.
|
||||||
|
|
||||||
### Logs Dates
|
### Filtering by Date
|
||||||
#### crowsnest utilized 'easy time' to determine the appropriate time for a given query.
|
#### CrowsNest utilizes 'easy time' to determine the appropriate time for a given query.
|
||||||

|

|
||||||
#### You may also used dates mixed with easy time to perform queries.
|
#### You may also used dates mixed with easy time to perform queries.
|
||||||

|

|
||||||
@@ -374,27 +394,13 @@ crowsnest logs -s "last 24 hours"
|
|||||||
crowsnest logs -s "05-01-2025" -v error,fatal
|
crowsnest logs -s "05-01-2025" -v error,fatal
|
||||||
```
|
```
|
||||||
|
|
||||||
## 🎉 Sample Run
|
|
||||||
```bash
|
|
||||||
ar1ste1a@kali:~$ crowsnest api -D <redacted>.com -o <redacted> -f json
|
|
||||||
Making 3 Requests for 10000 Records (30000 Total)
|
|
||||||
[*] Querying Dehashed API...
|
|
||||||
[*] Performing Request...
|
|
||||||
[+] Retrieved 2740 Records
|
|
||||||
[-] Not Enough Entries, ending queries
|
|
||||||
[+] Discovered 10 Credentials
|
|
||||||
[*] Writing entries to file: <redacted>.json
|
|
||||||
[*] Success
|
|
||||||
[*] Completing Process
|
|
||||||
```
|
|
||||||
|
|
||||||
## 🤝 Contributing
|
## 🤝 Contributing
|
||||||
Contributions are welcome! Submit a pull request to help improve CrowsNest.
|
Contributions are welcome! Submit a pull request to help improve CrowsNest.
|
||||||
|
|
||||||
|
## [Buy Me A Coffee](https://buymeacoffee.com/ehosinskiz)
|
||||||
|
|
||||||
<div align="center">
|
<div align="center">
|
||||||
<img src="https://img.wanman.io/fUSu0/jUtovIFE52.png/raw" style="width: 350px; height: auto" alt="Ar1ste1a" title="Ar1ste1a Offensive Security">
|
<img src=.img/kraken_k.png style="width: 350px; height: auto" alt="Ar1ste1a" title="Ar1ste1a Offensive Security">
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
## **Release The Kraken**
|
## **Release The Kraken**
|
||||||
|
|||||||
@@ -77,7 +77,7 @@ var (
|
|||||||
|
|
||||||
// Validate credentials
|
// Validate credentials
|
||||||
if key == "" {
|
if key == "" {
|
||||||
fmt.Println("API key is required. Set the key with the \"set-key\" command. [dehasher set-key <api_key>]")
|
fmt.Println("API key is required. Set the key with the \"set-key\" command. [crowsnest set-key <api_key>]")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -118,6 +118,7 @@ var (
|
|||||||
dehasher.Start()
|
dehasher.Start()
|
||||||
fmt.Println("\n[*] Completing Process")
|
fmt.Println("\n[*] Completing Process")
|
||||||
|
|
||||||
|
// Store query options
|
||||||
err := sqlite.StoreDehashedQueryOptions(queryOptions)
|
err := sqlite.StoreDehashedQueryOptions(queryOptions)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if debugGlobal {
|
if debugGlobal {
|
||||||
|
|||||||
@@ -1,311 +0,0 @@
|
|||||||
package cmd
|
|
||||||
|
|
||||||
import (
|
|
||||||
"crowsnest/internal/export"
|
|
||||||
"crowsnest/internal/files"
|
|
||||||
"crowsnest/internal/sqlite"
|
|
||||||
"fmt"
|
|
||||||
"github.com/spf13/cobra"
|
|
||||||
"go.uber.org/zap"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
func init() {
|
|
||||||
// Add Subcommand to db command
|
|
||||||
rootCmd.AddCommand(exportCmd)
|
|
||||||
|
|
||||||
// Add flags specific to export command
|
|
||||||
exportCmd.Flags().IntVarP(&exportLimitRows, "limit", "l", 100, "Limit number of results")
|
|
||||||
exportCmd.Flags().BoolVarP(&exportListAll, "list-all", "a", false, "List all tables and their columns")
|
|
||||||
exportCmd.Flags().StringVarP(&exportTableName, "table", "t", "", "Table to export (results, creds, whois, subdomains, history, runs)")
|
|
||||||
exportCmd.Flags().StringVarP(&exportNotNull, "not-null", "n", "", "Filter for non-null values (comma-separated list, e.g., 'password,email')")
|
|
||||||
exportCmd.Flags().StringVarP(&exportColumns, "columns", "c", "", "Columns to display in output (comma-separated list, e.g., 'username,email,password')")
|
|
||||||
exportCmd.Flags().StringVarP(&exportUserQuery, "user-query", "q", "", "User query to execute")
|
|
||||||
exportCmd.Flags().StringVarP(&exportRawQuery, "raw-query", "r", "", "Raw SQL query to execute")
|
|
||||||
exportCmd.Flags().StringVarP(&exportFormat, "format", "f", "json", "Output format (json, yaml, xml, txt)")
|
|
||||||
exportCmd.Flags().StringVarP(&exportFile, "file", "o", "export", "File to output results to including extension")
|
|
||||||
|
|
||||||
// Add mutually exclusive flags to query and raw-query
|
|
||||||
// Cannot use query and raw-query at the same time
|
|
||||||
exportCmd.MarkFlagsMutuallyExclusive("user-query", "raw-query")
|
|
||||||
// Raw query does not require a table
|
|
||||||
exportCmd.MarkFlagsMutuallyExclusive("user-query", "table")
|
|
||||||
// List all columns does not require a query or raw-query
|
|
||||||
exportCmd.MarkFlagsMutuallyExclusive("raw-query", "list-all")
|
|
||||||
}
|
|
||||||
|
|
||||||
// DB export command
|
|
||||||
var (
|
|
||||||
exportLimitRows int
|
|
||||||
exportListAll bool
|
|
||||||
exportTableName string
|
|
||||||
exportNotNull string
|
|
||||||
exportColumns string
|
|
||||||
exportUserQuery string
|
|
||||||
exportRawQuery string
|
|
||||||
exportFormat string
|
|
||||||
exportFile string
|
|
||||||
|
|
||||||
exportCmd = &cobra.Command{
|
|
||||||
Use: "export",
|
|
||||||
Short: "Export database to file",
|
|
||||||
Run: func(cmd *cobra.Command, args []string) {
|
|
||||||
// If list-all flag is set, list all tables and columns
|
|
||||||
if exportListAll {
|
|
||||||
listAvailableTables()
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
fmt.Println("[*] Exporting database...")
|
|
||||||
|
|
||||||
// If Raw Query is set, execute it and export
|
|
||||||
if exportRawQuery != "" {
|
|
||||||
fmt.Println("[*] Executing Raw Query...")
|
|
||||||
exportRawDBQuery()
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Validate table name
|
|
||||||
if exportTableName == "" {
|
|
||||||
fmt.Println("[!] Error: Table name is required. Use -t or --table to specify a table.")
|
|
||||||
fmt.Println("[*] Available tables: results, creds, whois, subdomains, history, runs")
|
|
||||||
fmt.Println("[*] Use --list-all to see all tables and their columns.")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if !isValidTable(exportTableName) {
|
|
||||||
fmt.Printf("[!] Error: Unknown table '%s'.\n", exportTableName)
|
|
||||||
fmt.Println("[*] Available tables: results, creds, whois, subdomains, history, runs")
|
|
||||||
fmt.Println("[*] Use --list-all to see all tables and their columns.")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Validate columns if specified
|
|
||||||
if exportColumns != "" {
|
|
||||||
columns := strings.Split(exportColumns, ",")
|
|
||||||
invalidColumns := validateColumns(exportTableName, columns)
|
|
||||||
if len(invalidColumns) > 0 {
|
|
||||||
fmt.Printf("[!] Error: Invalid column(s) for table '%s': %s\n",
|
|
||||||
exportTableName, strings.Join(invalidColumns, ", "))
|
|
||||||
fmt.Println("[*] Available columns for this table:")
|
|
||||||
for i := 0; i < len(availableTables[exportTableName]); i += 5 {
|
|
||||||
end := i + 5
|
|
||||||
if end > len(availableTables[exportTableName]) {
|
|
||||||
end = len(availableTables[exportTableName])
|
|
||||||
}
|
|
||||||
fmt.Printf(" %s\n", strings.Join(availableTables[exportTableName][i:end], ", "))
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Validate not-null fields if specified
|
|
||||||
if exportNotNull != "" {
|
|
||||||
notNullFields := strings.Split(exportNotNull, ",")
|
|
||||||
invalidFields := validateColumns(exportTableName, notNullFields)
|
|
||||||
if len(invalidFields) > 0 {
|
|
||||||
fmt.Printf("[!] Error: Invalid not-null field(s) for table '%s': %s\n",
|
|
||||||
exportTableName, strings.Join(invalidFields, ", "))
|
|
||||||
fmt.Println("[*] Available columns for this table:")
|
|
||||||
for i := 0; i < len(availableTables[exportTableName]); i += 5 {
|
|
||||||
end := i + 5
|
|
||||||
if end > len(availableTables[exportTableName]) {
|
|
||||||
end = len(availableTables[exportTableName])
|
|
||||||
}
|
|
||||||
fmt.Printf(" %s\n", strings.Join(availableTables[exportTableName][i:end], ", "))
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Determine which table to query based on the tableTypeDBQuery parameter
|
|
||||||
table := sqlite.GetTable(exportTableName)
|
|
||||||
if table == sqlite.UnknownTable {
|
|
||||||
fmt.Printf("[!] Error: Unknown table type '%s'.\n", exportTableName)
|
|
||||||
fmt.Println("[*] Available tables: results, creds, whois, subdomains, history, runs")
|
|
||||||
fmt.Println("[*] Use --list-all to see all tables and their columns.")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
fmt.Println("[*] Querying Database...")
|
|
||||||
exportTableQuery(table)
|
|
||||||
},
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
// exportTableQuery queries a table and exports the results
|
|
||||||
func exportTableQuery(table sqlite.Table) {
|
|
||||||
// Get the columns to query
|
|
||||||
columns := []string{"*"}
|
|
||||||
if exportColumns != "" {
|
|
||||||
columns = strings.Split(exportColumns, ",")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get the not null fields
|
|
||||||
notNullFields := []string{}
|
|
||||||
if exportNotNull != "" {
|
|
||||||
notNullFields = strings.Split(exportNotNull, ",")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get the user query
|
|
||||||
userQuery := ""
|
|
||||||
if exportUserQuery != "" {
|
|
||||||
userQuery = exportUserQuery
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get the limit
|
|
||||||
limit := exportLimitRows
|
|
||||||
|
|
||||||
// Get the object for the table
|
|
||||||
object := table.Object()
|
|
||||||
|
|
||||||
// Check if object is nil (invalid table)
|
|
||||||
if object == nil {
|
|
||||||
fmt.Printf("[!] Error: Table '%s' is not valid or does not exist.\n", exportTableName)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Query the database
|
|
||||||
db := sqlite.GetDB()
|
|
||||||
query := db.Model(object).Select(columns)
|
|
||||||
if len(notNullFields) > 0 {
|
|
||||||
for _, field := range notNullFields {
|
|
||||||
query = query.Where(fmt.Sprintf("%s IS NOT NULL", field))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if userQuery != "" {
|
|
||||||
query = query.Where(userQuery)
|
|
||||||
}
|
|
||||||
if limit > 0 {
|
|
||||||
query = query.Limit(limit)
|
|
||||||
}
|
|
||||||
rows, err := query.Rows()
|
|
||||||
if err != nil {
|
|
||||||
zap.L().Error("export_query",
|
|
||||||
zap.String("message", "failed to execute query"),
|
|
||||||
zap.Error(err),
|
|
||||||
)
|
|
||||||
fmt.Printf("[!] Error executing query: %v\n", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer rows.Close()
|
|
||||||
|
|
||||||
// Get the columns
|
|
||||||
cols, err := rows.Columns()
|
|
||||||
if err != nil {
|
|
||||||
zap.L().Error("export_query",
|
|
||||||
zap.String("message", "failed to get columns from query"),
|
|
||||||
zap.Error(err),
|
|
||||||
)
|
|
||||||
fmt.Printf("[!] Error getting columns from query: %v\n", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Prepare data for export
|
|
||||||
var results []map[string]interface{}
|
|
||||||
|
|
||||||
// Process the rows
|
|
||||||
for rows.Next() {
|
|
||||||
values := make([]interface{}, len(cols))
|
|
||||||
pointers := make([]interface{}, len(cols))
|
|
||||||
for i := range values {
|
|
||||||
pointers[i] = &values[i]
|
|
||||||
}
|
|
||||||
if err := rows.Scan(pointers...); err != nil {
|
|
||||||
zap.L().Error("export_query",
|
|
||||||
zap.String("message", "failed to scan row from query"),
|
|
||||||
zap.Error(err),
|
|
||||||
)
|
|
||||||
fmt.Printf("[!] Error scanning row from query: %v\n", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create a map for this row
|
|
||||||
rowMap := make(map[string]interface{})
|
|
||||||
for i, col := range cols {
|
|
||||||
val := values[i]
|
|
||||||
rowMap[col] = val
|
|
||||||
}
|
|
||||||
|
|
||||||
results = append(results, rowMap)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Export the results
|
|
||||||
exportResults(results)
|
|
||||||
}
|
|
||||||
|
|
||||||
// exportRawDBQuery executes a raw query and exports the results
|
|
||||||
func exportRawDBQuery() {
|
|
||||||
db := sqlite.GetDB()
|
|
||||||
rows, err := db.Raw(exportRawQuery).Rows()
|
|
||||||
if err != nil {
|
|
||||||
zap.L().Error("export_raw_query",
|
|
||||||
zap.String("message", "failed to execute raw query"),
|
|
||||||
zap.Error(err),
|
|
||||||
)
|
|
||||||
fmt.Printf("[!] Error executing raw query: %v\n", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer rows.Close()
|
|
||||||
|
|
||||||
columns, err := rows.Columns()
|
|
||||||
if err != nil {
|
|
||||||
zap.L().Error("export_raw_query",
|
|
||||||
zap.String("message", "failed to get columns from raw query"),
|
|
||||||
zap.Error(err),
|
|
||||||
)
|
|
||||||
fmt.Printf("[!] Error getting columns from raw query: %v\n", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Prepare data for export
|
|
||||||
var results []map[string]interface{}
|
|
||||||
|
|
||||||
// Process the rows
|
|
||||||
for rows.Next() {
|
|
||||||
values := make([]interface{}, len(columns))
|
|
||||||
pointers := make([]interface{}, len(columns))
|
|
||||||
for i := range values {
|
|
||||||
pointers[i] = &values[i]
|
|
||||||
}
|
|
||||||
if err := rows.Scan(pointers...); err != nil {
|
|
||||||
zap.L().Error("export_raw_query",
|
|
||||||
zap.String("message", "failed to scan row from raw query"),
|
|
||||||
zap.Error(err),
|
|
||||||
)
|
|
||||||
fmt.Printf("[!] Error scanning row from raw query: %v\n", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create a map for this row
|
|
||||||
rowMap := make(map[string]interface{})
|
|
||||||
for i, col := range columns {
|
|
||||||
val := values[i]
|
|
||||||
rowMap[col] = val
|
|
||||||
}
|
|
||||||
|
|
||||||
results = append(results, rowMap)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Export the results
|
|
||||||
exportResults(results)
|
|
||||||
}
|
|
||||||
|
|
||||||
// exportResults exports the results to a file
|
|
||||||
func exportResults(results []map[string]interface{}) {
|
|
||||||
// Get file type
|
|
||||||
fileType := files.GetFileType(exportFormat)
|
|
||||||
|
|
||||||
// Export results
|
|
||||||
err := export.WriteQueryResultsToFile(results, exportFile, fileType)
|
|
||||||
if err != nil {
|
|
||||||
zap.L().Error("export_results",
|
|
||||||
zap.String("message", "failed to write to file"),
|
|
||||||
zap.Error(err),
|
|
||||||
)
|
|
||||||
fmt.Printf("[!] Error writing to file: %v\n", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
fmt.Printf("[+] Exported %d records to file: %s%s\n", len(results), exportFile, fileType.Extension())
|
|
||||||
}
|
|
||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"crowsnest/internal/files"
|
"crowsnest/internal/files"
|
||||||
hunter "crowsnest/internal/hunter.io"
|
hunter "crowsnest/internal/hunter.io"
|
||||||
"crowsnest/internal/pretty"
|
"crowsnest/internal/pretty"
|
||||||
|
"crowsnest/internal/sqlite"
|
||||||
"fmt"
|
"fmt"
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
@@ -111,6 +112,24 @@ var (
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Store the users discovered
|
||||||
|
var creds []sqlite.User
|
||||||
|
for _, email := range result.Emails {
|
||||||
|
creds = append(creds, sqlite.User{Email: email.Value})
|
||||||
|
}
|
||||||
|
err = sqlite.StoreUsers(creds)
|
||||||
|
if err != nil {
|
||||||
|
if debugGlobal {
|
||||||
|
debug.PrintInfo("failed to store hunter domain search")
|
||||||
|
debug.PrintError(err)
|
||||||
|
}
|
||||||
|
zap.L().Error("store_hunter_domain_search",
|
||||||
|
zap.String("message", "failed to store hunter domain search"),
|
||||||
|
zap.Error(err),
|
||||||
|
)
|
||||||
|
fmt.Printf("Error storing Hunter.io Domain Search Result: %v\n", err)
|
||||||
|
}
|
||||||
|
|
||||||
// Write Hunter.io Domain Search Result to file
|
// Write Hunter.io Domain Search Result to file
|
||||||
fmt.Printf("[*] Writing Hunter.io Domain Search Result to file: %s%s\n", hunterOutputFile, fType.Extension())
|
fmt.Printf("[*] Writing Hunter.io Domain Search Result to file: %s%s\n", hunterOutputFile, fType.Extension())
|
||||||
err = export.WriteIStringToFile(result, hunterOutputFile, fType)
|
err = export.WriteIStringToFile(result, hunterOutputFile, fType)
|
||||||
@@ -166,7 +185,7 @@ var (
|
|||||||
fmt.Println("Email Find Result:")
|
fmt.Println("Email Find Result:")
|
||||||
|
|
||||||
var (
|
var (
|
||||||
headers = []string{"Email", "Score", "Domain", "Accept All", "Position", "Twitter", "Linkedin", "Phone Number", "Company", "Sources", "Verification"}
|
headers = []string{"Email", "Score", "Domain", "Accept All", "Position", "Twitter", "Linkedin", "Phone Number", "Company", "Verification"}
|
||||||
rows [][]string
|
rows [][]string
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -180,7 +199,6 @@ var (
|
|||||||
result.LinkedinURL,
|
result.LinkedinURL,
|
||||||
result.PhoneNumber,
|
result.PhoneNumber,
|
||||||
result.Company,
|
result.Company,
|
||||||
fmt.Sprintf("%v", result.Sources),
|
|
||||||
fmt.Sprintf("%v", result.Verification),
|
fmt.Sprintf("%v", result.Verification),
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -221,24 +239,17 @@ var (
|
|||||||
|
|
||||||
// Pretty Print Hunter.io Email Verification Result
|
// Pretty Print Hunter.io Email Verification Result
|
||||||
var (
|
var (
|
||||||
headers = []string{"Email", "Status", "Result", "Score", "Regexp", "Gibberish", "Disposable", "Webmail", "MX Records", "SMTP Server", "SMTP Check", "Accept All", "Block", "Sources"}
|
headers = []string{"Email", "Result", "Score", "Disposable", "MX Records", "SMTP Server", "SMTP Check"}
|
||||||
rows [][]string
|
rows [][]string
|
||||||
)
|
)
|
||||||
rows = append(rows, []string{
|
rows = append(rows, []string{
|
||||||
result.Email,
|
result.Email,
|
||||||
result.Status,
|
|
||||||
result.Result,
|
result.Result,
|
||||||
fmt.Sprintf("%d", result.Score),
|
fmt.Sprintf("%d", result.Score),
|
||||||
fmt.Sprintf("%t", result.Regexp),
|
|
||||||
fmt.Sprintf("%t", result.Gibberish),
|
|
||||||
fmt.Sprintf("%t", result.Disposable),
|
fmt.Sprintf("%t", result.Disposable),
|
||||||
fmt.Sprintf("%t", result.Webmail),
|
|
||||||
fmt.Sprintf("%t", result.MXRecords),
|
fmt.Sprintf("%t", result.MXRecords),
|
||||||
fmt.Sprintf("%t", result.SMTPServer),
|
fmt.Sprintf("%t", result.SMTPServer),
|
||||||
fmt.Sprintf("%t", result.SMTPCheck),
|
fmt.Sprintf("%t", result.SMTPCheck),
|
||||||
fmt.Sprintf("%t", result.AcceptAll),
|
|
||||||
fmt.Sprintf("%t", result.Block),
|
|
||||||
fmt.Sprintf("%v", result.Sources),
|
|
||||||
})
|
})
|
||||||
|
|
||||||
fmt.Println("Email Verification Result:")
|
fmt.Println("Email Verification Result:")
|
||||||
|
|||||||
@@ -1,6 +1,9 @@
|
|||||||
package cmd
|
package cmd
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"crowsnest/internal/debug"
|
||||||
|
"crowsnest/internal/export"
|
||||||
|
"crowsnest/internal/files"
|
||||||
"crowsnest/internal/pretty"
|
"crowsnest/internal/pretty"
|
||||||
"crowsnest/internal/sqlite"
|
"crowsnest/internal/sqlite"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
@@ -10,132 +13,6 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Map of available tables and their columns
|
|
||||||
var availableTables = map[string][]string{
|
|
||||||
"creds": {
|
|
||||||
"id", "created_at", "updated_at", "deleted_at", "email", "username", "password",
|
|
||||||
},
|
|
||||||
//"history": {
|
|
||||||
// "id", "created_at", "updated_at", "deleted_at", "domain_name", "domain_type",
|
|
||||||
// "registrar_name", "whois_server", "created_date_iso8601", "updated_date_iso8601", "expires_date_iso8601",
|
|
||||||
//},
|
|
||||||
"lookup": {
|
|
||||||
"id", "created_at", "updated_at", "deleted_at", "search_term", "type", "first_seen", "last_visit",
|
|
||||||
"name",
|
|
||||||
},
|
|
||||||
// Query Options
|
|
||||||
"runs": {
|
|
||||||
"id", "created_at", "updated_at", "deleted_at", "max_records", "max_requests", "starting_page",
|
|
||||||
"output_format", "output_file", "regex_match", "wildcard_match", "username_query", "email_query",
|
|
||||||
"ip_query", "pass_query", "hash_query", "name_query", "domain_query", "vin_query", "license_plate_query",
|
|
||||||
"address_query", "phone_query", "social_query", "crypto_address_query", "print_balance", "creds_only",
|
|
||||||
},
|
|
||||||
"results": {
|
|
||||||
"id", "created_at", "updated_at", "deleted_at", "dehashed_id", "email", "ip_address", "username",
|
|
||||||
"password", "hashed_password", "hash_type", "name", "vin", "license_plate", "url", "social",
|
|
||||||
"cryptocurrency_address", "address", "phone", "company", "database_name",
|
|
||||||
},
|
|
||||||
"subdomains": {
|
|
||||||
"id", "created_at", "updated_at", "deleted_at", "domain", "first_seen", "last_seen",
|
|
||||||
},
|
|
||||||
"whois": {
|
|
||||||
"id", "created_at", "updated_at", "deleted_at", "audit", "contact_email", "created_date", "created_date_normalized",
|
|
||||||
"domain_name", "domain_name_ext", "estimated_domain_age", "expires_date", "expires_date_normalized", "footer", "header",
|
|
||||||
"name_servers", "parse_code", "raw_text", "registrant", "registrar_iana_id", "registrar_name", "registry_data",
|
|
||||||
"status", "stripped_text", "updated_date", "updated_date_normalized",
|
|
||||||
},
|
|
||||||
"hunter_domain": {
|
|
||||||
"id", "created_at", "updated_at", "deleted_at", "domain", "disposable", "webmail", "accept_all", "pattern",
|
|
||||||
"organization", "description", "industry", "twitter", "facebook", "linkedin", "instagram", "youtube",
|
|
||||||
"technologies", "country", "state", "city", "postal_code", "street", "headcount", "company_type", "emails", "linked_domains",
|
|
||||||
},
|
|
||||||
"hunter_email": {
|
|
||||||
"id", "created_at", "updated_at", "deleted_at", "value", "type", "confidence", "sources", "first_name", "last_name",
|
|
||||||
"position", "position_raw", "seniority", "department", "linkedin", "twitter", "phone_number", "verification_date", "verification_status",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
// Function to list available tables and their columns
|
|
||||||
func listAvailableTables() {
|
|
||||||
fmt.Println("Available tables and columns:")
|
|
||||||
|
|
||||||
// Prepare data for pretty.Table
|
|
||||||
headers := []string{"Table", "Columns"}
|
|
||||||
var tableRows [][]string
|
|
||||||
|
|
||||||
// Sort tables alphabetically for consistent output
|
|
||||||
var tableNames []string
|
|
||||||
for tableName := range availableTables {
|
|
||||||
tableNames = append(tableNames, tableName)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Simple bubble sort for table names
|
|
||||||
for i := 0; i < len(tableNames)-1; i++ {
|
|
||||||
for j := 0; j < len(tableNames)-i-1; j++ {
|
|
||||||
if tableNames[j] > tableNames[j+1] {
|
|
||||||
tableNames[j], tableNames[j+1] = tableNames[j+1], tableNames[j]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create rows for the table
|
|
||||||
for _, tableName := range tableNames {
|
|
||||||
columns := availableTables[tableName]
|
|
||||||
|
|
||||||
// Format columns with line breaks for better readability
|
|
||||||
var formattedColumns string
|
|
||||||
for i := 0; i < len(columns); i += 5 {
|
|
||||||
end := i + 5
|
|
||||||
if end > len(columns) {
|
|
||||||
end = len(columns)
|
|
||||||
}
|
|
||||||
if i > 0 {
|
|
||||||
formattedColumns += "\n"
|
|
||||||
}
|
|
||||||
formattedColumns += strings.Join(columns[i:end], ", ")
|
|
||||||
}
|
|
||||||
|
|
||||||
tableRows = append(tableRows, []string{tableName, formattedColumns})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Display the table
|
|
||||||
pretty.Table(headers, tableRows)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Function to validate table name
|
|
||||||
func isValidTable(tableName string) bool {
|
|
||||||
_, exists := availableTables[tableName]
|
|
||||||
return exists
|
|
||||||
}
|
|
||||||
|
|
||||||
// Function to validate column names for a specific table
|
|
||||||
func validateColumns(tableName string, columns []string) []string {
|
|
||||||
if tableName == "" || columns == nil || len(columns) == 0 || columns[0] == "*" {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
tableColumns, exists := availableTables[tableName]
|
|
||||||
if !exists {
|
|
||||||
return []string{fmt.Sprintf("Table '%s' does not exist", tableName)}
|
|
||||||
}
|
|
||||||
|
|
||||||
var invalidColumns []string
|
|
||||||
for _, col := range columns {
|
|
||||||
valid := false
|
|
||||||
for _, tableCol := range tableColumns {
|
|
||||||
if col == tableCol {
|
|
||||||
valid = true
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !valid {
|
|
||||||
invalidColumns = append(invalidColumns, col)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return invalidColumns
|
|
||||||
}
|
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
// Add whois command to root command
|
// Add whois command to root command
|
||||||
rootCmd.AddCommand(queryCmd)
|
rootCmd.AddCommand(queryCmd)
|
||||||
@@ -148,6 +25,8 @@ func init() {
|
|||||||
queryCmd.Flags().StringVarP(&dbQueryUserQuery, "user-query", "q", "", "User query to execute")
|
queryCmd.Flags().StringVarP(&dbQueryUserQuery, "user-query", "q", "", "User query to execute")
|
||||||
queryCmd.Flags().StringVarP(&dbQueryRawQuery, "raw-query", "r", "", "Raw SQL query to execute")
|
queryCmd.Flags().StringVarP(&dbQueryRawQuery, "raw-query", "r", "", "Raw SQL query to execute")
|
||||||
queryCmd.Flags().BoolVarP(&dbQueryListAll, "list-all", "a", false, "List all tables and their columns")
|
queryCmd.Flags().BoolVarP(&dbQueryListAll, "list-all", "a", false, "List all tables and their columns")
|
||||||
|
queryCmd.Flags().StringVarP(&dbQueryFormat, "format", "f", "json", "Output format (json, yaml, xml, txt)")
|
||||||
|
queryCmd.Flags().StringVarP(&dbQueryFile, "file", "o", "query", "File to output results to")
|
||||||
|
|
||||||
// Add mutually exclusive flags to query and raw-query
|
// Add mutually exclusive flags to query and raw-query
|
||||||
// Cannot use query and raw-query at the same time
|
// Cannot use query and raw-query at the same time
|
||||||
@@ -166,11 +45,14 @@ var (
|
|||||||
dbQueryUserQuery string
|
dbQueryUserQuery string
|
||||||
dbQueryRawQuery string
|
dbQueryRawQuery string
|
||||||
dbQueryListAll bool
|
dbQueryListAll bool
|
||||||
|
dbQueryFormat string
|
||||||
|
dbQueryFile string
|
||||||
|
|
||||||
queryCmd = &cobra.Command{
|
queryCmd = &cobra.Command{
|
||||||
Use: "query",
|
Use: "query",
|
||||||
Short: "Query the database",
|
Short: "Query the database",
|
||||||
Long: `Query the database for various information.`,
|
Long: `Query the database for various information.
|
||||||
|
If file is specified, results are written to file and not displayed in the terminal.`,
|
||||||
Run: func(cmd *cobra.Command, args []string) {
|
Run: func(cmd *cobra.Command, args []string) {
|
||||||
// If list-all flag is set, list all tables and columns
|
// If list-all flag is set, list all tables and columns
|
||||||
if dbQueryListAll {
|
if dbQueryListAll {
|
||||||
@@ -321,6 +203,49 @@ func tableQuery(table sqlite.Table) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Export results if file name is specified
|
||||||
|
if len(strings.TrimSpace(dbQueryFile)) > 0 {
|
||||||
|
fmt.Println("[*] Exporting results to file...")
|
||||||
|
|
||||||
|
if debugGlobal {
|
||||||
|
debug.PrintInfo("exporting results to file: " + dbQueryFile)
|
||||||
|
}
|
||||||
|
// Prepare data for export
|
||||||
|
var results []map[string]interface{}
|
||||||
|
|
||||||
|
// Process the rows
|
||||||
|
for rows.Next() {
|
||||||
|
values := make([]interface{}, len(cols))
|
||||||
|
pointers := make([]interface{}, len(cols))
|
||||||
|
for i := range values {
|
||||||
|
pointers[i] = &values[i]
|
||||||
|
}
|
||||||
|
if err := rows.Scan(pointers...); err != nil {
|
||||||
|
zap.L().Error("export_query",
|
||||||
|
zap.String("message", "failed to scan row from query"),
|
||||||
|
zap.Error(err),
|
||||||
|
)
|
||||||
|
fmt.Printf("[!] Error scanning row from query: %v\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create a map for this row
|
||||||
|
rowMap := make(map[string]interface{})
|
||||||
|
for i, col := range cols {
|
||||||
|
val := values[i]
|
||||||
|
rowMap[col] = val
|
||||||
|
}
|
||||||
|
|
||||||
|
results = append(results, rowMap)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Export the results
|
||||||
|
exportQueryResults(results)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fmt.Println("[*] Querying Database...")
|
||||||
|
|
||||||
// Prepare data for pretty.Table
|
// Prepare data for pretty.Table
|
||||||
headers := cols
|
headers := cols
|
||||||
var tableRows [][]string
|
var tableRows [][]string
|
||||||
@@ -411,6 +336,49 @@ func rawDBQuery() {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if len(strings.TrimSpace(dbQueryFile)) > 0 {
|
||||||
|
fmt.Println("[*] Exporting results to file...")
|
||||||
|
|
||||||
|
if debugGlobal {
|
||||||
|
debug.PrintInfo("exporting results to file: " + dbQueryFile)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Prepare data for export
|
||||||
|
var results []map[string]interface{}
|
||||||
|
|
||||||
|
// Process the rows
|
||||||
|
for rows.Next() {
|
||||||
|
values := make([]interface{}, len(columns))
|
||||||
|
pointers := make([]interface{}, len(columns))
|
||||||
|
for i := range values {
|
||||||
|
pointers[i] = &values[i]
|
||||||
|
}
|
||||||
|
if err := rows.Scan(pointers...); err != nil {
|
||||||
|
zap.L().Error("export_raw_query",
|
||||||
|
zap.String("message", "failed to scan row from raw query"),
|
||||||
|
zap.Error(err),
|
||||||
|
)
|
||||||
|
fmt.Printf("[!] Error scanning row from raw query: %v\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create a map for this row
|
||||||
|
rowMap := make(map[string]interface{})
|
||||||
|
for i, col := range columns {
|
||||||
|
val := values[i]
|
||||||
|
rowMap[col] = val
|
||||||
|
}
|
||||||
|
|
||||||
|
results = append(results, rowMap)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Export the results
|
||||||
|
exportQueryResults(results)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fmt.Println("[*] Querying Database...")
|
||||||
|
|
||||||
// Prepare data for pretty.Table
|
// Prepare data for pretty.Table
|
||||||
headers := columns
|
headers := columns
|
||||||
var tableRows [][]string
|
var tableRows [][]string
|
||||||
@@ -477,3 +445,22 @@ func rawDBQuery() {
|
|||||||
// Display the table
|
// Display the table
|
||||||
pretty.Table(headers, tableRows)
|
pretty.Table(headers, tableRows)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// exportQueryResults exports the results to a file
|
||||||
|
func exportQueryResults(results []map[string]interface{}) {
|
||||||
|
// Get file type
|
||||||
|
fileType := files.GetFileType(dbQueryFormat)
|
||||||
|
|
||||||
|
// Export results
|
||||||
|
err := export.WriteQueryResultsToFile(results, dbQueryFile, fileType)
|
||||||
|
if err != nil {
|
||||||
|
zap.L().Error("export_results",
|
||||||
|
zap.String("message", "failed to write to file"),
|
||||||
|
zap.Error(err),
|
||||||
|
)
|
||||||
|
fmt.Printf("[!] Error writing to file: %v\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf("[+] Exported %d records to file: %s%s\n", len(results), dbQueryFile, fileType.Extension())
|
||||||
|
}
|
||||||
|
|||||||
@@ -16,8 +16,8 @@ var (
|
|||||||
|
|
||||||
// rootCmd is the base command for the CLI.
|
// rootCmd is the base command for the CLI.
|
||||||
rootCmd = &cobra.Command{
|
rootCmd = &cobra.Command{
|
||||||
Use: "dehasher",
|
Use: "crowsnest",
|
||||||
Short: `Dehasher is a cli tool for querying the dehashed api.`,
|
Short: `CrowsNest is a cli tool for querying the common OSINT api's.`,
|
||||||
Long: fmt.Sprintf(
|
Long: fmt.Sprintf(
|
||||||
"%s\n",
|
"%s\n",
|
||||||
`
|
`
|
||||||
@@ -94,7 +94,7 @@ var setHunterKeyCmd = &cobra.Command{
|
|||||||
|
|
||||||
var setLocalDb = &cobra.Command{
|
var setLocalDb = &cobra.Command{
|
||||||
Use: "local-db [true|false]",
|
Use: "local-db [true|false]",
|
||||||
Short: "Set dehasher to use a local database path instead of the default path",
|
Short: "Set crowsnest to use a local database path instead of the default path",
|
||||||
Args: cobra.ExactArgs(1),
|
Args: cobra.ExactArgs(1),
|
||||||
Run: func(cmd *cobra.Command, args []string) {
|
Run: func(cmd *cobra.Command, args []string) {
|
||||||
var useLocalDatabase bool
|
var useLocalDatabase bool
|
||||||
|
|||||||
@@ -0,0 +1,133 @@
|
|||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crowsnest/internal/pretty"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Map of available tables and their columns
|
||||||
|
var availableTables = map[string][]string{
|
||||||
|
"users": {
|
||||||
|
"id", "created_at", "updated_at", "deleted_at", "email", "username", "password",
|
||||||
|
},
|
||||||
|
//"history": {
|
||||||
|
// "id", "created_at", "updated_at", "deleted_at", "domain_name", "domain_type",
|
||||||
|
// "registrar_name", "whois_server", "created_date_iso8601", "updated_date_iso8601", "expires_date_iso8601",
|
||||||
|
//},
|
||||||
|
"lookup": {
|
||||||
|
"id", "created_at", "updated_at", "deleted_at", "search_term", "type", "first_seen", "last_visit",
|
||||||
|
"name",
|
||||||
|
},
|
||||||
|
// Query Options
|
||||||
|
"runs": {
|
||||||
|
"id", "created_at", "updated_at", "deleted_at", "max_records", "max_requests", "starting_page",
|
||||||
|
"output_format", "output_file", "regex_match", "wildcard_match", "username_query", "email_query",
|
||||||
|
"ip_query", "pass_query", "hash_query", "name_query", "domain_query", "vin_query", "license_plate_query",
|
||||||
|
"address_query", "phone_query", "social_query", "crypto_address_query", "print_balance", "creds_only",
|
||||||
|
},
|
||||||
|
"dehashed": {
|
||||||
|
"id", "created_at", "updated_at", "deleted_at", "dehashed_id", "email", "ip_address", "username",
|
||||||
|
"password", "hashed_password", "hash_type", "name", "vin", "license_plate", "url", "social",
|
||||||
|
"cryptocurrency_address", "address", "phone", "company", "database_name",
|
||||||
|
},
|
||||||
|
"subdomains": {
|
||||||
|
"id", "created_at", "updated_at", "deleted_at", "domain", "subdomain",
|
||||||
|
},
|
||||||
|
"whois": {
|
||||||
|
"id", "created_at", "updated_at", "deleted_at", "audit", "contact_email", "created_date", "created_date_normalized",
|
||||||
|
"domain_name", "domain_name_ext", "estimated_domain_age", "expires_date", "expires_date_normalized", "footer", "header",
|
||||||
|
"name_servers", "parse_code", "raw_text", "registrant", "registrar_iana_id", "registrar_name", "registry_data",
|
||||||
|
"status", "stripped_text", "updated_date", "updated_date_normalized",
|
||||||
|
},
|
||||||
|
"hunter_domain": {
|
||||||
|
"id", "created_at", "updated_at", "deleted_at", "domain", "disposable", "webmail", "accept_all", "pattern",
|
||||||
|
"organization", "description", "industry", "twitter", "facebook", "linkedin", "instagram", "youtube",
|
||||||
|
"technologies", "country", "state", "city", "postal_code", "street", "headcount", "company_type", "emails", "linked_domains",
|
||||||
|
},
|
||||||
|
"hunter_email": {
|
||||||
|
"id", "created_at", "updated_at", "deleted_at", "value", "type", "confidence", "sources", "first_name", "last_name",
|
||||||
|
"position", "position_raw", "seniority", "department", "linkedin", "twitter", "phone_number", "verification_date", "verification_status",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
// Function to list available tables and their columns
|
||||||
|
func listAvailableTables() {
|
||||||
|
fmt.Println("Available tables and columns:")
|
||||||
|
|
||||||
|
// Prepare data for pretty.Table
|
||||||
|
headers := []string{"Table", "Columns"}
|
||||||
|
var tableRows [][]string
|
||||||
|
|
||||||
|
// Sort tables alphabetically for consistent output
|
||||||
|
var tableNames []string
|
||||||
|
for tableName := range availableTables {
|
||||||
|
tableNames = append(tableNames, tableName)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Simple bubble sort for table names
|
||||||
|
for i := 0; i < len(tableNames)-1; i++ {
|
||||||
|
for j := 0; j < len(tableNames)-i-1; j++ {
|
||||||
|
if tableNames[j] > tableNames[j+1] {
|
||||||
|
tableNames[j], tableNames[j+1] = tableNames[j+1], tableNames[j]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create rows for the table
|
||||||
|
for _, tableName := range tableNames {
|
||||||
|
columns := availableTables[tableName]
|
||||||
|
|
||||||
|
// Format columns with line breaks for better readability
|
||||||
|
var formattedColumns string
|
||||||
|
for i := 0; i < len(columns); i += 5 {
|
||||||
|
end := i + 5
|
||||||
|
if end > len(columns) {
|
||||||
|
end = len(columns)
|
||||||
|
}
|
||||||
|
if i > 0 {
|
||||||
|
formattedColumns += "\n"
|
||||||
|
}
|
||||||
|
formattedColumns += strings.Join(columns[i:end], ", ")
|
||||||
|
}
|
||||||
|
|
||||||
|
tableRows = append(tableRows, []string{tableName, formattedColumns})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Display the table
|
||||||
|
pretty.Table(headers, tableRows)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Function to validate table name
|
||||||
|
func isValidTable(tableName string) bool {
|
||||||
|
_, exists := availableTables[tableName]
|
||||||
|
return exists
|
||||||
|
}
|
||||||
|
|
||||||
|
// Function to validate column names for a specific table
|
||||||
|
func validateColumns(tableName string, columns []string) []string {
|
||||||
|
if tableName == "" || columns == nil || len(columns) == 0 || columns[0] == "*" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
tableColumns, exists := availableTables[tableName]
|
||||||
|
if !exists {
|
||||||
|
return []string{fmt.Sprintf("Table '%s' does not exist", tableName)}
|
||||||
|
}
|
||||||
|
|
||||||
|
var invalidColumns []string
|
||||||
|
for _, col := range columns {
|
||||||
|
valid := false
|
||||||
|
for _, tableCol := range tableColumns {
|
||||||
|
if col == tableCol {
|
||||||
|
valid = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !valid {
|
||||||
|
invalidColumns = append(invalidColumns, col)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return invalidColumns
|
||||||
|
}
|
||||||
@@ -0,0 +1,312 @@
|
|||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crowsnest/internal/sqlite"
|
||||||
|
"fmt"
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
"go.uber.org/zap"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
// Add targets command to root command
|
||||||
|
rootCmd.AddCommand(targetsCmd)
|
||||||
|
|
||||||
|
// Add flags specific to targets command
|
||||||
|
targetsCmd.Flags().StringVarP(&targetsOutputFile, "output", "o", "targets", "Output file name (required)")
|
||||||
|
targetsCmd.Flags().BoolVarP(&targetsExternal, "external", "e", false, "Output external format (email:password)")
|
||||||
|
targetsCmd.Flags().BoolVarP(&targetsInternal, "internal", "i", false, "Output internal format (username:password)")
|
||||||
|
targetsCmd.Flags().BoolVarP(&targetsSubdomains, "subdomains", "s", false, "Output subdomains")
|
||||||
|
targetsCmd.Flags().BoolVarP(&targetsEmails, "emails", "E", false, "Output emails only (no passwords)")
|
||||||
|
targetsCmd.Flags().StringVarP(&targetsDomain, "domain", "d", "", "Filter by domain (for emails and subdomains)")
|
||||||
|
|
||||||
|
// Add mutually exclusive flags to targets command
|
||||||
|
targetsCmd.MarkFlagsMutuallyExclusive("external", "internal", "subdomains", "emails")
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
// Targets command flags
|
||||||
|
targetsOutputFile string
|
||||||
|
targetsExternal bool
|
||||||
|
targetsInternal bool
|
||||||
|
targetsSubdomains bool
|
||||||
|
targetsEmails bool
|
||||||
|
targetsDomain string
|
||||||
|
|
||||||
|
// Targets command
|
||||||
|
targetsCmd = &cobra.Command{
|
||||||
|
Use: "targets",
|
||||||
|
Short: "Export users and subdomains in formats suitable for external tools",
|
||||||
|
Long: `Export users and subdomains from the database in easily digestible formats for tools like sprays or other security testing tools.
|
||||||
|
|
||||||
|
Formats:
|
||||||
|
--external (-e): Output in email:password format
|
||||||
|
--internal (-i): Output in username:password format
|
||||||
|
--emails (-E): Output emails only (no passwords)
|
||||||
|
--subdomains (-s): Output subdomains only
|
||||||
|
|
||||||
|
Options:
|
||||||
|
--domain (-d): Filter results by domain (applies to emails and subdomains)
|
||||||
|
--output (-o): Specify output file name (required)
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
# Export all external credentials (email:password)
|
||||||
|
crowsnest targets -e -o external_creds
|
||||||
|
|
||||||
|
# Export internal credentials for a specific domain
|
||||||
|
crowsnest targets -i -d example.com -o internal_creds
|
||||||
|
|
||||||
|
# Export all emails
|
||||||
|
crowsnest targets -E -o all_emails
|
||||||
|
|
||||||
|
# Export emails for a specific domain
|
||||||
|
crowsnest targets -E -d example.com -o domain_emails
|
||||||
|
|
||||||
|
# Export subdomains for a specific domain
|
||||||
|
crowsnest targets -s -d example.com -o subdomains
|
||||||
|
|
||||||
|
# Export all subdomains
|
||||||
|
crowsnest targets -s -o all_subdomains`,
|
||||||
|
Run: func(cmd *cobra.Command, args []string) {
|
||||||
|
// Validate that at least one format is specified
|
||||||
|
if !targetsExternal && !targetsInternal && !targetsSubdomains && !targetsEmails {
|
||||||
|
fmt.Println("[!] Error: You must specify at least one output format:")
|
||||||
|
fmt.Println(" --external (-e) for email:password format")
|
||||||
|
fmt.Println(" --internal (-i) for username:password format")
|
||||||
|
fmt.Println(" --emails (-E) for emails only")
|
||||||
|
fmt.Println(" --subdomains (-s) for subdomains")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if debugGlobal {
|
||||||
|
zap.L().Info("targets_debug",
|
||||||
|
zap.String("message", "targets command started"),
|
||||||
|
zap.Bool("external", targetsExternal),
|
||||||
|
zap.Bool("internal", targetsInternal),
|
||||||
|
zap.Bool("subdomains", targetsSubdomains),
|
||||||
|
zap.Bool("emails", targetsEmails),
|
||||||
|
zap.String("domain", targetsDomain),
|
||||||
|
zap.String("output_file", targetsOutputFile),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Execute the targets export
|
||||||
|
err := executeTargetsExport()
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("[!] Error: %v\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf("[+] Successfully exported targets to: %s\n", targetsOutputFile)
|
||||||
|
},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
// executeTargetsExport performs the main logic for exporting targets
|
||||||
|
func executeTargetsExport() error {
|
||||||
|
var outputLines []string
|
||||||
|
|
||||||
|
// Export external credentials (email:password)
|
||||||
|
if targetsExternal {
|
||||||
|
if debugGlobal {
|
||||||
|
fmt.Println("[*] Exporting external credentials (email:password)...")
|
||||||
|
}
|
||||||
|
|
||||||
|
externalCreds, err := getExternalCredentials()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to get external credentials: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, cred := range externalCreds {
|
||||||
|
if cred.Email != "" && cred.Password != "" {
|
||||||
|
outputLines = append(outputLines, fmt.Sprintf("%s:%s", cred.Email, cred.Password))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if debugGlobal {
|
||||||
|
fmt.Printf("[*] Found %d external credentials\n", len(externalCreds))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Export internal credentials (username:password)
|
||||||
|
if targetsInternal {
|
||||||
|
if debugGlobal {
|
||||||
|
fmt.Println("[*] Exporting internal credentials (username:password)...")
|
||||||
|
}
|
||||||
|
|
||||||
|
internalCreds, err := getInternalCredentials()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to get internal credentials: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, cred := range internalCreds {
|
||||||
|
if cred.Username != "" && cred.Password != "" {
|
||||||
|
outputLines = append(outputLines, fmt.Sprintf("%s:%s", cred.Username, cred.Password))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if debugGlobal {
|
||||||
|
fmt.Printf("[*] Found %d internal credentials\n", len(internalCreds))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Export emails only
|
||||||
|
if targetsEmails {
|
||||||
|
if debugGlobal {
|
||||||
|
fmt.Println("[*] Exporting emails only...")
|
||||||
|
}
|
||||||
|
|
||||||
|
emails, err := getEmailsOnly()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to get emails: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, email := range emails {
|
||||||
|
if email.Email != "" {
|
||||||
|
outputLines = append(outputLines, email.Email)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if debugGlobal {
|
||||||
|
fmt.Printf("[*] Found %d emails\n", len(emails))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Export subdomains
|
||||||
|
if targetsSubdomains {
|
||||||
|
if debugGlobal {
|
||||||
|
fmt.Println("[*] Exporting subdomains...")
|
||||||
|
}
|
||||||
|
|
||||||
|
subdomains, err := getSubdomains()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to get subdomains: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, subdomain := range subdomains {
|
||||||
|
if subdomain.Subdomain != "" {
|
||||||
|
outputLines = append(outputLines, subdomain.Subdomain)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if debugGlobal {
|
||||||
|
fmt.Printf("[*] Found %d subdomains\n", len(subdomains))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write to file
|
||||||
|
if len(outputLines) == 0 {
|
||||||
|
return fmt.Errorf("no data found to export")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Join all lines with newlines and add a single newline at the end
|
||||||
|
content := strings.Join(outputLines, "\n") + "\n"
|
||||||
|
|
||||||
|
err := os.WriteFile(targetsOutputFile, []byte(content), 0644)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to write to file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if debugGlobal {
|
||||||
|
fmt.Printf("[*] Wrote %d lines to %s\n", len(outputLines), targetsOutputFile)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// getExternalCredentials retrieves credentials for external format (email:password)
|
||||||
|
func getExternalCredentials() ([]sqlite.User, error) {
|
||||||
|
db := sqlite.GetDB()
|
||||||
|
var users []sqlite.User
|
||||||
|
|
||||||
|
query := db.Where("email IS NOT NULL AND email != '' AND password IS NOT NULL AND password != ''")
|
||||||
|
|
||||||
|
// Apply domain filter if specified
|
||||||
|
if targetsDomain != "" {
|
||||||
|
query = query.Where("email LIKE ?", "%@"+targetsDomain)
|
||||||
|
}
|
||||||
|
|
||||||
|
err := query.Find(&users).Error
|
||||||
|
if err != nil {
|
||||||
|
zap.L().Error("get_external_credentials",
|
||||||
|
zap.String("message", "failed to query external credentials"),
|
||||||
|
zap.Error(err),
|
||||||
|
)
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return users, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// getInternalCredentials retrieves credentials for internal format (username:password)
|
||||||
|
func getInternalCredentials() ([]sqlite.User, error) {
|
||||||
|
db := sqlite.GetDB()
|
||||||
|
var users []sqlite.User
|
||||||
|
|
||||||
|
query := db.Where("username IS NOT NULL AND username != '' AND password IS NOT NULL AND password != ''")
|
||||||
|
|
||||||
|
// Apply domain filter if specified (filter usernames that might contain domain info)
|
||||||
|
if targetsDomain != "" {
|
||||||
|
query = query.Where("username LIKE ? OR email LIKE ?", "%"+targetsDomain+"%", "%@"+targetsDomain)
|
||||||
|
}
|
||||||
|
|
||||||
|
err := query.Find(&users).Error
|
||||||
|
if err != nil {
|
||||||
|
zap.L().Error("get_internal_credentials",
|
||||||
|
zap.String("message", "failed to query internal credentials"),
|
||||||
|
zap.Error(err),
|
||||||
|
)
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return users, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// getEmailsOnly retrieves emails only (no passwords required)
|
||||||
|
func getEmailsOnly() ([]sqlite.User, error) {
|
||||||
|
db := sqlite.GetDB()
|
||||||
|
var users []sqlite.User
|
||||||
|
|
||||||
|
query := db.Where("email IS NOT NULL AND email != ''")
|
||||||
|
|
||||||
|
// Apply domain filter if specified
|
||||||
|
if targetsDomain != "" {
|
||||||
|
query = query.Where("email LIKE ?", "%@"+targetsDomain)
|
||||||
|
}
|
||||||
|
|
||||||
|
err := query.Find(&users).Error
|
||||||
|
if err != nil {
|
||||||
|
zap.L().Error("get_emails_only",
|
||||||
|
zap.String("message", "failed to query emails"),
|
||||||
|
zap.Error(err),
|
||||||
|
)
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return users, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// getSubdomains retrieves subdomains from the database
|
||||||
|
func getSubdomains() ([]sqlite.Subdomain, error) {
|
||||||
|
db := sqlite.GetDB()
|
||||||
|
var subdomains []sqlite.Subdomain
|
||||||
|
|
||||||
|
query := db.Where("subdomain IS NOT NULL AND subdomain != ''")
|
||||||
|
|
||||||
|
// Apply domain filter if specified
|
||||||
|
if targetsDomain != "" {
|
||||||
|
query = query.Where("domain = ? OR subdomain LIKE ?", targetsDomain, "%."+targetsDomain)
|
||||||
|
}
|
||||||
|
|
||||||
|
err := query.Find(&subdomains).Error
|
||||||
|
if err != nil {
|
||||||
|
zap.L().Error("get_subdomains",
|
||||||
|
zap.String("message", "failed to query subdomains"),
|
||||||
|
zap.Error(err),
|
||||||
|
)
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return subdomains, nil
|
||||||
|
}
|
||||||
@@ -26,7 +26,7 @@ func init() {
|
|||||||
whoisCmd.Flags().StringVarP(&whoisNSAddress, "ns", "n", "", "NS hostname for reverse NS lookup")
|
whoisCmd.Flags().StringVarP(&whoisNSAddress, "ns", "n", "", "NS hostname for reverse NS lookup")
|
||||||
whoisCmd.Flags().StringVarP(&whoisInclude, "include", "I", "", "Up to 4 Terms to include in reverse WHOIS search (comma-separated)")
|
whoisCmd.Flags().StringVarP(&whoisInclude, "include", "I", "", "Up to 4 Terms to include in reverse WHOIS search (comma-separated)")
|
||||||
whoisCmd.Flags().StringVarP(&whoisExclude, "exclude", "E", "", "Up to 4 Terms to exclude in reverse WHOIS search (comma-separated)")
|
whoisCmd.Flags().StringVarP(&whoisExclude, "exclude", "E", "", "Up to 4 Terms to exclude in reverse WHOIS search (comma-separated)")
|
||||||
whoisCmd.Flags().StringVarP(&whoisReverseType, "type", "t", "registrant", "Type of reverse WHOIS search ([default] current or historic)")
|
whoisCmd.Flags().StringVarP(&whoisReverseType, "type", "t", "current", "Type of reverse WHOIS search ([default] current or historic)")
|
||||||
whoisCmd.Flags().StringVarP(&whoisOutputFormat, "format", "f", "text", "Output format (text, json)")
|
whoisCmd.Flags().StringVarP(&whoisOutputFormat, "format", "f", "text", "Output format (text, json)")
|
||||||
whoisCmd.Flags().StringVarP(&whoisOutputFile, "output", "o", "whois", "File to output results to including extension")
|
whoisCmd.Flags().StringVarP(&whoisOutputFile, "output", "o", "whois", "File to output results to including extension")
|
||||||
whoisCmd.Flags().BoolVarP(&whoisShowCredits, "credits", "c", false, "Show remaining WHOIS credits")
|
whoisCmd.Flags().BoolVarP(&whoisShowCredits, "credits", "c", false, "Show remaining WHOIS credits")
|
||||||
@@ -59,7 +59,7 @@ var (
|
|||||||
|
|
||||||
// Validate credentials
|
// Validate credentials
|
||||||
if key == "" {
|
if key == "" {
|
||||||
fmt.Println("API key is required. Set the key with the \"set-key\" command. [dehasher set-key <api_key>]")
|
fmt.Println("API key is required. Set the key with the \"set-key\" command. [crowsnest set-key <api_key>]")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -118,57 +118,59 @@ var (
|
|||||||
if whoisDomain != "" {
|
if whoisDomain != "" {
|
||||||
fmt.Println("[*] Performing WHOIS lookup...")
|
fmt.Println("[*] Performing WHOIS lookup...")
|
||||||
|
|
||||||
// Domain lookup
|
if !whoisHistory && !whoisSubdomainScan {
|
||||||
result, err := w.WhoisSearch(whoisDomain)
|
// Domain lookup
|
||||||
if err != nil {
|
result, err := w.WhoisSearch(whoisDomain)
|
||||||
if debugGlobal {
|
if err != nil {
|
||||||
debug.PrintInfo("failed to perform whois search")
|
if debugGlobal {
|
||||||
debug.PrintError(err)
|
debug.PrintInfo("failed to perform whois search")
|
||||||
|
debug.PrintError(err)
|
||||||
|
}
|
||||||
|
zap.L().Error("whois_search",
|
||||||
|
zap.String("message", "failed to perform whois search"),
|
||||||
|
zap.Error(err),
|
||||||
|
)
|
||||||
|
fmt.Printf("Error performing WHOIS lookup: %v\n", err)
|
||||||
|
return
|
||||||
}
|
}
|
||||||
zap.L().Error("whois_search",
|
|
||||||
zap.String("message", "failed to perform whois search"),
|
|
||||||
zap.Error(err),
|
|
||||||
)
|
|
||||||
fmt.Printf("Error performing WHOIS lookup: %v\n", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if whoisShowCredits {
|
if whoisShowCredits {
|
||||||
checkBalance(w)
|
checkBalance(w)
|
||||||
}
|
|
||||||
|
|
||||||
// Fix the output format to use proper formatting
|
|
||||||
fmt.Println("WHOIS Lookup Result:")
|
|
||||||
|
|
||||||
// Store the record
|
|
||||||
err = sqlite.StoreWhoisRecord(result)
|
|
||||||
if err != nil {
|
|
||||||
if debugGlobal {
|
|
||||||
debug.PrintInfo("failed to store whois record")
|
|
||||||
debug.PrintError(err)
|
|
||||||
}
|
}
|
||||||
zap.L().Error("store_whois_record",
|
|
||||||
zap.String("message", "failed to store whois record"),
|
|
||||||
zap.Error(err),
|
|
||||||
)
|
|
||||||
fmt.Printf("Error storing WHOIS record: %v\n", err)
|
|
||||||
// Continue execution even if storage fails
|
|
||||||
}
|
|
||||||
|
|
||||||
// Pretty Print WhoIs Record
|
// Fix the output format to use proper formatting
|
||||||
pretty.WhoIsTree(whoisDomain, result)
|
fmt.Println("WHOIS Lookup Result:")
|
||||||
|
|
||||||
// Write WhoIs Record to file
|
// Store the record
|
||||||
if len(result.DomainName) != 0 {
|
err = sqlite.StoreWhoisRecord(result)
|
||||||
fmt.Printf("[*] Writing WHOIS record to file: %s%s\n", whoisOutputFile, fType.Extension())
|
if err != nil {
|
||||||
err = export.WriteWhoIsRecordToFile(result, whoisOutputFile, fType)
|
if debugGlobal {
|
||||||
} else {
|
debug.PrintInfo("failed to store whois record")
|
||||||
if debugGlobal {
|
debug.PrintError(err)
|
||||||
debug.PrintInfo("no whois record to write to file")
|
}
|
||||||
|
zap.L().Error("store_whois_record",
|
||||||
|
zap.String("message", "failed to store whois record"),
|
||||||
|
zap.Error(err),
|
||||||
|
)
|
||||||
|
fmt.Printf("Error storing WHOIS record: %v\n", err)
|
||||||
|
// Continue execution even if storage fails
|
||||||
|
}
|
||||||
|
|
||||||
|
// Pretty Print WhoIs Record
|
||||||
|
pretty.WhoIsTree(whoisDomain, result)
|
||||||
|
|
||||||
|
// Write WhoIs Record to file
|
||||||
|
if len(result.DomainName) != 0 {
|
||||||
|
fmt.Printf("[*] Writing WHOIS record to file: %s%s\n", whoisOutputFile, fType.Extension())
|
||||||
|
err = export.WriteWhoIsRecordToFile(result, whoisOutputFile, fType)
|
||||||
|
} else {
|
||||||
|
if debugGlobal {
|
||||||
|
debug.PrintInfo("no whois record to write to file")
|
||||||
|
}
|
||||||
|
zap.L().Info("write_whois_record",
|
||||||
|
zap.String("message", "no whois record to write to file"),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
zap.L().Info("write_whois_record",
|
|
||||||
zap.String("message", "no whois record to write to file"),
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if whoisHistory {
|
if whoisHistory {
|
||||||
@@ -238,6 +240,7 @@ var (
|
|||||||
fmt.Println("[*] Performing WHOIS subdomain scan...")
|
fmt.Println("[*] Performing WHOIS subdomain scan...")
|
||||||
subdomains, err := w.WhoisSubdomainScan(whoisDomain)
|
subdomains, err := w.WhoisSubdomainScan(whoisDomain)
|
||||||
|
|
||||||
|
// Get credits
|
||||||
if whoisShowCredits {
|
if whoisShowCredits {
|
||||||
checkBalance(w)
|
checkBalance(w)
|
||||||
}
|
}
|
||||||
@@ -253,8 +256,13 @@ var (
|
|||||||
)
|
)
|
||||||
fmt.Printf("Error performing subdomain scan: %v\n", err)
|
fmt.Printf("Error performing subdomain scan: %v\n", err)
|
||||||
} else {
|
} else {
|
||||||
fmt.Println("Subdomain Scan:")
|
// Store subdomains in subdomains table
|
||||||
err = sqlite.StoreWhoisSubdomainRecords(subdomains)
|
var subs []sqlite.Subdomain
|
||||||
|
for _, s := range subdomains {
|
||||||
|
subs = append(subs, sqlite.Subdomain{Domain: whoisDomain, Subdomain: s.Domain})
|
||||||
|
}
|
||||||
|
|
||||||
|
err = sqlite.StoreSubdomains(subs)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if debugGlobal {
|
if debugGlobal {
|
||||||
debug.PrintInfo("failed to store subdomain record")
|
debug.PrintInfo("failed to store subdomain record")
|
||||||
@@ -264,7 +272,7 @@ var (
|
|||||||
zap.String("message", "failed to store subdomain record"),
|
zap.String("message", "failed to store subdomain record"),
|
||||||
zap.Error(err),
|
zap.Error(err),
|
||||||
)
|
)
|
||||||
fmt.Printf("Error storing WHOIS subdomain record: %v\n", err)
|
fmt.Printf("Error storing subdomain record: %v\n", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Write the subdomains to file if any
|
// Write the subdomains to file if any
|
||||||
@@ -290,6 +298,7 @@ var (
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Store the subdomains
|
// Store the subdomains
|
||||||
|
fmt.Println("Subdomain Scan:")
|
||||||
pretty.Table(headers, rows)
|
pretty.Table(headers, rows)
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
@@ -492,6 +501,12 @@ var (
|
|||||||
}
|
}
|
||||||
|
|
||||||
if whoisInclude != "" || whoisExclude != "" {
|
if whoisInclude != "" || whoisExclude != "" {
|
||||||
|
if debugGlobal {
|
||||||
|
debug.PrintInfo("performing reverse whois")
|
||||||
|
debug.PrintInfo("include: " + whoisInclude)
|
||||||
|
debug.PrintInfo("exclude: " + whoisExclude)
|
||||||
|
debug.PrintInfo("reverse type: " + whoisReverseType)
|
||||||
|
}
|
||||||
// Reverse WHOIS
|
// Reverse WHOIS
|
||||||
includeTerms := []string{}
|
includeTerms := []string{}
|
||||||
if whoisInclude != "" {
|
if whoisInclude != "" {
|
||||||
@@ -511,17 +526,10 @@ var (
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if whoisReverseType == "" {
|
toLower := strings.ToLower(whoisReverseType)
|
||||||
if debugGlobal {
|
if toLower != "current" && toLower != "historic" {
|
||||||
debug.PrintInfo("reverse type not specified, using default")
|
fmt.Println("[!] Error: Invalid reverse type. Must be 'current' or 'historic'.")
|
||||||
}
|
return
|
||||||
whoisReverseType = "current"
|
|
||||||
} else {
|
|
||||||
toLower := strings.ToLower(whoisReverseType)
|
|
||||||
if toLower != "current" && toLower != "historic" {
|
|
||||||
fmt.Println("[!] Error: Invalid reverse type. Must be 'current' or 'historic'.")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Println("[*] Performing reverse WHOIS lookup...")
|
fmt.Println("[*] Performing reverse WHOIS lookup...")
|
||||||
@@ -538,8 +546,42 @@ var (
|
|||||||
fmt.Printf("Error performing reverse WHOIS: %v\n", err)
|
fmt.Printf("Error performing reverse WHOIS: %v\n", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
fmt.Println("Reverse WHOIS Result:")
|
|
||||||
fmt.Println(result)
|
// Write to file
|
||||||
|
if len(result.DomainsList) > 0 {
|
||||||
|
fmt.Printf("[*] Writing reverse WHOIS results to file: %s%s\n", whoisOutputFile, fType.Extension())
|
||||||
|
err = export.WriteIStringToFile(result, whoisOutputFile, fType)
|
||||||
|
if err != nil {
|
||||||
|
if debugGlobal {
|
||||||
|
debug.PrintInfo("failed to write reverse whois to file")
|
||||||
|
debug.PrintError(err)
|
||||||
|
}
|
||||||
|
zap.L().Error("write_reverse_whois",
|
||||||
|
zap.String("message", "failed to write reverse whois to file"),
|
||||||
|
zap.Error(err),
|
||||||
|
)
|
||||||
|
fmt.Printf("Error writing reverse WHOIS to file: %v\n", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Println("Reverse WHOIS Result:")
|
||||||
|
fmt.Printf("Total Domains: %d\n", result.DomainsCount)
|
||||||
|
|
||||||
|
var (
|
||||||
|
headers = []string{"Domain"}
|
||||||
|
rows [][]string
|
||||||
|
)
|
||||||
|
|
||||||
|
for _, r := range result.DomainsList {
|
||||||
|
rows = append(rows, []string{r})
|
||||||
|
}
|
||||||
|
|
||||||
|
pretty.Table(headers, rows)
|
||||||
|
} else {
|
||||||
|
fmt.Println("[!] No results found")
|
||||||
|
zap.L().Info("reverse_whois",
|
||||||
|
zap.String("message", "no results found"),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
if whoisShowCredits {
|
if whoisShowCredits {
|
||||||
checkBalance(w)
|
checkBalance(w)
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ var (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
basePath = filepath.Join(os.Getenv("HOME"), ".local", "share", "Dehasher")
|
basePath = filepath.Join(os.Getenv("HOME"), ".local", "share", "CrowsNest")
|
||||||
logPath = filepath.Join(basePath, "logs")
|
logPath = filepath.Join(basePath, "logs")
|
||||||
storePath = filepath.Join(basePath, "keystore")
|
storePath = filepath.Join(basePath, "keystore")
|
||||||
// dbPath will be set in main() after badger is initialized
|
// dbPath will be set in main() after badger is initialized
|
||||||
@@ -82,7 +82,7 @@ func main() {
|
|||||||
useLocalDB := badger.GetUseLocalDB()
|
useLocalDB := badger.GetUseLocalDB()
|
||||||
if useLocalDB {
|
if useLocalDB {
|
||||||
// Use local database in current directory
|
// Use local database in current directory
|
||||||
dbPath = "./dehasher.sqlite"
|
dbPath = "./"
|
||||||
zap.L().Info("Using local database", zap.String("path", dbPath))
|
zap.L().Info("Using local database", zap.String("path", dbPath))
|
||||||
} else {
|
} else {
|
||||||
// Use default database path
|
// Use default database path
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ require (
|
|||||||
github.com/charmbracelet/lipgloss v1.1.0
|
github.com/charmbracelet/lipgloss v1.1.0
|
||||||
github.com/dgraph-io/badger/v4 v4.7.0
|
github.com/dgraph-io/badger/v4 v4.7.0
|
||||||
github.com/fatih/color v1.15.0
|
github.com/fatih/color v1.15.0
|
||||||
|
github.com/glebarez/sqlite v1.11.0
|
||||||
github.com/spf13/cobra v1.9.1
|
github.com/spf13/cobra v1.9.1
|
||||||
github.com/winking324/rzap v0.1.0
|
github.com/winking324/rzap v0.1.0
|
||||||
go.uber.org/zap v1.20.0
|
go.uber.org/zap v1.20.0
|
||||||
@@ -27,9 +28,11 @@ require (
|
|||||||
github.com/charmbracelet/x/term v0.2.1 // indirect
|
github.com/charmbracelet/x/term v0.2.1 // indirect
|
||||||
github.com/dgraph-io/ristretto/v2 v2.2.0 // indirect
|
github.com/dgraph-io/ristretto/v2 v2.2.0 // indirect
|
||||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||||
|
github.com/glebarez/go-sqlite v1.21.2 // indirect
|
||||||
github.com/go-logr/logr v1.4.2 // indirect
|
github.com/go-logr/logr v1.4.2 // indirect
|
||||||
github.com/go-logr/stdr v1.2.2 // indirect
|
github.com/go-logr/stdr v1.2.2 // indirect
|
||||||
github.com/google/flatbuffers v25.2.10+incompatible // indirect
|
github.com/google/flatbuffers v25.2.10+incompatible // indirect
|
||||||
|
github.com/google/uuid v1.6.0 // indirect
|
||||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||||
github.com/jinzhu/inflection v1.0.0 // indirect
|
github.com/jinzhu/inflection v1.0.0 // indirect
|
||||||
github.com/jinzhu/now v1.1.5 // indirect
|
github.com/jinzhu/now v1.1.5 // indirect
|
||||||
@@ -40,6 +43,7 @@ require (
|
|||||||
github.com/mattn/go-runewidth v0.0.16 // indirect
|
github.com/mattn/go-runewidth v0.0.16 // indirect
|
||||||
github.com/mattn/go-sqlite3 v1.14.22 // indirect
|
github.com/mattn/go-sqlite3 v1.14.22 // indirect
|
||||||
github.com/muesli/termenv v0.16.0 // indirect
|
github.com/muesli/termenv v0.16.0 // indirect
|
||||||
|
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||||
github.com/rivo/uniseg v0.4.7 // indirect
|
github.com/rivo/uniseg v0.4.7 // indirect
|
||||||
github.com/spf13/pflag v1.0.6 // indirect
|
github.com/spf13/pflag v1.0.6 // indirect
|
||||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
||||||
@@ -53,4 +57,8 @@ require (
|
|||||||
golang.org/x/sys v0.31.0 // indirect
|
golang.org/x/sys v0.31.0 // indirect
|
||||||
golang.org/x/text v0.23.0 // indirect
|
golang.org/x/text v0.23.0 // indirect
|
||||||
google.golang.org/protobuf v1.36.6 // indirect
|
google.golang.org/protobuf v1.36.6 // indirect
|
||||||
|
modernc.org/libc v1.22.5 // indirect
|
||||||
|
modernc.org/mathutil v1.5.0 // indirect
|
||||||
|
modernc.org/memory v1.5.0 // indirect
|
||||||
|
modernc.org/sqlite v1.23.1 // indirect
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -34,6 +34,10 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
|
|||||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||||
github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
|
github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
|
||||||
github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
|
github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
|
||||||
|
github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9gAXWo=
|
||||||
|
github.com/glebarez/go-sqlite v1.21.2/go.mod h1:sfxdZyhQjTM2Wry3gVYWaW072Ri1WMdWJi0k6+3382k=
|
||||||
|
github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw=
|
||||||
|
github.com/glebarez/sqlite v1.11.0/go.mod h1:h8/o8j5wiAsqSPoWELDUdJXhjAhsVliSn7bWZjOhrgQ=
|
||||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||||
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
|
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
|
||||||
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||||
@@ -43,6 +47,10 @@ github.com/google/flatbuffers v25.2.10+incompatible h1:F3vclr7C3HpB1k9mxCGRMXq6F
|
|||||||
github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
|
github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
|
||||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||||
|
github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ=
|
||||||
|
github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo=
|
||||||
|
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||||
|
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||||
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
|
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
|
||||||
@@ -75,6 +83,9 @@ github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
|
|||||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
|
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||||
|
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||||
|
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||||
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||||
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||||
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||||
@@ -164,3 +175,11 @@ gorm.io/driver/sqlite v1.5.7 h1:8NvsrhP0ifM7LX9G4zPB97NwovUakUxc+2V2uuf3Z1I=
|
|||||||
gorm.io/driver/sqlite v1.5.7/go.mod h1:U+J8craQU6Fzkcvu8oLeAQmi50TkwPEhHDEjQZXDah4=
|
gorm.io/driver/sqlite v1.5.7/go.mod h1:U+J8craQU6Fzkcvu8oLeAQmi50TkwPEhHDEjQZXDah4=
|
||||||
gorm.io/gorm v1.26.1 h1:ghB2gUI9FkS46luZtn6DLZ0f6ooBJ5IbVej2ENFDjRw=
|
gorm.io/gorm v1.26.1 h1:ghB2gUI9FkS46luZtn6DLZ0f6ooBJ5IbVej2ENFDjRw=
|
||||||
gorm.io/gorm v1.26.1/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE=
|
gorm.io/gorm v1.26.1/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE=
|
||||||
|
modernc.org/libc v1.22.5 h1:91BNch/e5B0uPbJFgqbxXuOnxBQjlS//icfQEGmvyjE=
|
||||||
|
modernc.org/libc v1.22.5/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY=
|
||||||
|
modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ=
|
||||||
|
modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
|
||||||
|
modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds=
|
||||||
|
modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU=
|
||||||
|
modernc.org/sqlite v1.23.1 h1:nrSBg4aRQQwq59JpvGEQ15tNxoO5pX/kUjcRNwSAGQM=
|
||||||
|
modernc.org/sqlite v1.23.1/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk=
|
||||||
|
|||||||
@@ -45,7 +45,7 @@ func GetHardwareEntropy() []byte {
|
|||||||
username,
|
username,
|
||||||
osInfo,
|
osInfo,
|
||||||
// You could add a static salt here for additional security
|
// You could add a static salt here for additional security
|
||||||
"Dehasher-static-salt-value",
|
"CrowsNest-static-salt-value",
|
||||||
}, ":")
|
}, ":")
|
||||||
|
|
||||||
// Hash the fingerprint to get a 32-byte key
|
// Hash the fingerprint to get a 32-byte key
|
||||||
|
|||||||
@@ -166,7 +166,11 @@ func (dcv2 *DehashedClientV2) Search(searchRequest DehashedSearchRequest) (int,
|
|||||||
zap.String("message", "preparing search request"),
|
zap.String("message", "preparing search request"),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
reqBody, _ := json.Marshal(searchRequest)
|
|
||||||
|
// Create a copy of the search request to avoid modifying the original
|
||||||
|
requestCopy := searchRequest
|
||||||
|
|
||||||
|
reqBody, _ := json.Marshal(requestCopy)
|
||||||
|
|
||||||
if dcv2.debug {
|
if dcv2.debug {
|
||||||
j := string(reqBody)
|
j := string(reqBody)
|
||||||
|
|||||||
@@ -3,11 +3,12 @@ package dehashed
|
|||||||
import (
|
import (
|
||||||
"crowsnest/internal/debug"
|
"crowsnest/internal/debug"
|
||||||
"crowsnest/internal/export"
|
"crowsnest/internal/export"
|
||||||
|
"crowsnest/internal/pretty"
|
||||||
"crowsnest/internal/sqlite"
|
"crowsnest/internal/sqlite"
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
"os"
|
"os"
|
||||||
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Dehasher is a struct for querying the Dehashed API
|
// Dehasher is a struct for querying the Dehashed API
|
||||||
@@ -208,13 +209,11 @@ func (dh *Dehasher) buildRequest() {
|
|||||||
|
|
||||||
// parseResults parses the results and writes them to a file
|
// parseResults parses the results and writes them to a file
|
||||||
func (dh *Dehasher) parseResults() {
|
func (dh *Dehasher) parseResults() {
|
||||||
var data []byte
|
|
||||||
|
|
||||||
zap.L().Info("extracting_credentials")
|
zap.L().Info("extracting_credentials")
|
||||||
results := dh.client.GetResults()
|
results := dh.client.GetResults()
|
||||||
creds := results.ExtractCredentials()
|
creds := results.ExtractUsers()
|
||||||
fmt.Printf("\n\t[+] Discovered %d Credentials", len(creds))
|
fmt.Printf(" [+] Discovered %d Credentials\n", len(creds))
|
||||||
err := sqlite.StoreDehashedCreds(creds)
|
err := sqlite.StoreUsers(creds)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
zap.L().Error("store_creds",
|
zap.L().Error("store_creds",
|
||||||
zap.String("message", "failed to store creds"),
|
zap.String("message", "failed to store creds"),
|
||||||
@@ -234,28 +233,93 @@ func (dh *Dehasher) parseResults() {
|
|||||||
zap.L().Info("results_stored", zap.Int("count", len(results.Results)))
|
zap.L().Info("results_stored", zap.Int("count", len(results.Results)))
|
||||||
|
|
||||||
if len(results.Results) > 0 {
|
if len(results.Results) > 0 {
|
||||||
fmt.Printf("\n\t[*] Writing entries to file: %s.%s", dh.options.OutputFile, dh.options.OutputFormat.String())
|
var (
|
||||||
|
headers = []string{"Email", "Username", "Password"}
|
||||||
|
rows [][]string
|
||||||
|
)
|
||||||
|
|
||||||
|
fmt.Printf(" [*] Writing entries to file: %s.%s\n", dh.options.OutputFile, dh.options.OutputFormat.String())
|
||||||
if !dh.options.CredsOnly {
|
if !dh.options.CredsOnly {
|
||||||
err := export.WriteToFile(results, dh.options.OutputFile, dh.options.OutputFormat)
|
err := export.WriteToFile(results, dh.options.OutputFile, dh.options.OutputFormat)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Printf("\n[!] Error Writing to file: %v\n\tOutputting to terminal.", err)
|
fmt.Printf("[!] Error Writing to file: %v Outputting to terminal.\n", err)
|
||||||
data, err = json.MarshalIndent(results, "", " ")
|
zap.L().Error("write_results",
|
||||||
fmt.Println(string(data))
|
zap.String("message", "failed to write results to file"),
|
||||||
os.Exit(0)
|
zap.Error(err),
|
||||||
|
)
|
||||||
} else {
|
} else {
|
||||||
fmt.Println("\n\t\t[*] Success\n")
|
fmt.Println(" [*] Success")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if dh.debug {
|
||||||
|
debug.PrintInfo("printing results table")
|
||||||
|
}
|
||||||
|
|
||||||
|
headers = []string{"Email", "Username", "Password", "Phone", "Company"}
|
||||||
|
if len(results.Results) > 50 {
|
||||||
|
fmt.Println(" [-] Large number of results recovered, displaying first 50...")
|
||||||
|
for i := 0; i < 50; i++ {
|
||||||
|
r := results.Results[i]
|
||||||
|
rows = append(rows, []string{
|
||||||
|
strings.Join(r.Email, ", "),
|
||||||
|
strings.Join(r.Username, ", "),
|
||||||
|
strings.Join(r.Password, ", "),
|
||||||
|
strings.Join(r.Phone, ", "),
|
||||||
|
strings.Join(r.Company, ", ")})
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
for _, r := range results.Results {
|
||||||
|
rows = append(rows, []string{
|
||||||
|
strings.Join(r.Email, ", "),
|
||||||
|
strings.Join(r.Username, ", "),
|
||||||
|
strings.Join(r.Password, ", "),
|
||||||
|
strings.Join(r.Phone, ", "),
|
||||||
|
strings.Join(r.Company, ", ")})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Print Table
|
||||||
|
pretty.Table(headers, rows)
|
||||||
} else {
|
} else {
|
||||||
creds := results.ExtractCredentials()
|
if dh.debug {
|
||||||
|
debug.PrintInfo("extracting credentials")
|
||||||
|
}
|
||||||
|
creds := results.ExtractUsers()
|
||||||
|
if dh.debug {
|
||||||
|
debug.PrintInfo("writing credentials to file")
|
||||||
|
}
|
||||||
err := export.WriteCredsToFile(creds, dh.options.OutputFile, dh.options.OutputFormat)
|
err := export.WriteCredsToFile(creds, dh.options.OutputFile, dh.options.OutputFormat)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Printf("\n[!] Error Writing to file: %v\n\tOutputting to terminal.", err)
|
fmt.Printf("[!] Error Writing to file: %v\n Outputting to terminal.", err)
|
||||||
data, err = json.MarshalIndent(creds, "", " ")
|
zap.L().Error("write_creds",
|
||||||
fmt.Println(string(data))
|
zap.String("message", "failed to write creds to file"),
|
||||||
os.Exit(0)
|
zap.Error(err),
|
||||||
|
)
|
||||||
} else {
|
} else {
|
||||||
fmt.Println("\n\t\t[*] Success\n")
|
fmt.Println(" [*] Success")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if dh.debug {
|
||||||
|
debug.PrintInfo("printing credentials table")
|
||||||
|
}
|
||||||
|
|
||||||
|
headers = []string{"Email", "Username", "Password"}
|
||||||
|
if len(creds) > 50 {
|
||||||
|
fmt.Println(" [-] Large number of results recovered, displaying first 50...")
|
||||||
|
for i := 0; i < 50; i++ {
|
||||||
|
c := creds[i]
|
||||||
|
rows = append(rows, []string{c.Email, c.Username, c.Password})
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
for _, c := range creds {
|
||||||
|
rows = append(rows, []string{c.Email, c.Username, c.Password})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Print Table
|
||||||
|
pretty.Table(headers, rows)
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
fmt.Println(" [-] No results found")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
func WriteCredsToFile(creds []sqlite.Creds, outputFile string, fileType files.FileType) error {
|
func WriteCredsToFile(creds []sqlite.User, outputFile string, fileType files.FileType) error {
|
||||||
var data []byte
|
var data []byte
|
||||||
var err error
|
var err error
|
||||||
|
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gorm.io/driver/sqlite"
|
sql "github.com/glebarez/sqlite"
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
"gorm.io/gorm/logger"
|
"gorm.io/gorm/logger"
|
||||||
)
|
)
|
||||||
@@ -29,7 +29,7 @@ func InitDB(dbPath string) (*gorm.DB, error) {
|
|||||||
zap.L().Error("Failed to create database directory", zap.Error(err))
|
zap.L().Error("Failed to create database directory", zap.Error(err))
|
||||||
return nil, fmt.Errorf("failed to create database directory: %w", err)
|
return nil, fmt.Errorf("failed to create database directory: %w", err)
|
||||||
}
|
}
|
||||||
finalDbPath = filepath.Join(dbPath, "dehashed.sqlite")
|
finalDbPath = filepath.Join(dbPath, "crowsnest.sqlite")
|
||||||
} else {
|
} else {
|
||||||
// Treat as file path
|
// Treat as file path
|
||||||
// Ensure the directory exists
|
// Ensure the directory exists
|
||||||
@@ -42,7 +42,7 @@ func InitDB(dbPath string) (*gorm.DB, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
zap.L().Info("Opening database", zap.String("finalPath", finalDbPath))
|
zap.L().Info("Opening database", zap.String("finalPath", finalDbPath))
|
||||||
db, err := gorm.Open(sqlite.Open(finalDbPath), &gorm.Config{
|
db, err := gorm.Open(sql.Open(finalDbPath), &gorm.Config{
|
||||||
Logger: logger.Default.LogMode(logger.Silent),
|
Logger: logger.Default.LogMode(logger.Silent),
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -51,8 +51,8 @@ func InitDB(dbPath string) (*gorm.DB, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Auto migrate your models
|
// Auto migrate your models
|
||||||
err = db.AutoMigrate(&Result{}, &Creds{}, &QueryOptions{}, &Creds{}, &WhoisRecord{}, &SubdomainRecord{},
|
err = db.AutoMigrate(&Result{}, &User{}, &QueryOptions{}, &User{}, &WhoisRecord{}, &HistoryRecord{},
|
||||||
&HistoryRecord{}, &LookupResult{}, &HunterDomainData{}, &HunterEmail{}, &PersonData{})
|
&LookupResult{}, &HunterDomainData{}, &HunterEmail{}, &PersonData{}, &Subdomain{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
zap.L().Error("Failed to migrate database", zap.Error(err))
|
zap.L().Error("Failed to migrate database", zap.Error(err))
|
||||||
return nil, fmt.Errorf("failed to migrate database: %w", err)
|
return nil, fmt.Errorf("failed to migrate database: %w", err)
|
||||||
@@ -122,7 +122,7 @@ func (t Table) Object() interface{} {
|
|||||||
case RunsTable:
|
case RunsTable:
|
||||||
return QueryOptions{}
|
return QueryOptions{}
|
||||||
case CredsTable:
|
case CredsTable:
|
||||||
return Creds{}
|
return User{}
|
||||||
case WhoIsTable:
|
case WhoIsTable:
|
||||||
return WhoisRecord{}
|
return WhoisRecord{}
|
||||||
case SubdomainsTable:
|
case SubdomainsTable:
|
||||||
|
|||||||
@@ -106,15 +106,15 @@ type Result struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (Result) TableName() string {
|
func (Result) TableName() string {
|
||||||
return "results"
|
return "dehashed"
|
||||||
}
|
}
|
||||||
|
|
||||||
type DehashedResults struct {
|
type DehashedResults struct {
|
||||||
Results []Result `json:"results"`
|
Results []Result `json:"results"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (dr *DehashedResults) ExtractCredentials() []Creds {
|
func (dr *DehashedResults) ExtractUsers() []User {
|
||||||
var creds []Creds
|
var creds []User
|
||||||
|
|
||||||
results := dr.Results
|
results := dr.Results
|
||||||
|
|
||||||
@@ -126,16 +126,22 @@ func (dr *DehashedResults) ExtractCredentials() []Creds {
|
|||||||
email = r.Email[0]
|
email = r.Email[0]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Get first username if available
|
||||||
|
username := ""
|
||||||
|
if len(r.Username) > 0 {
|
||||||
|
username = r.Username[0]
|
||||||
|
}
|
||||||
|
|
||||||
// Get first password
|
// Get first password
|
||||||
password := r.Password[0]
|
password := r.Password[0]
|
||||||
|
|
||||||
cred := Creds{Email: email, Password: password}
|
cred := User{Email: email, Password: password, Username: username}
|
||||||
creds = append(creds, cred)
|
creds = append(creds, cred)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
err := StoreDehashedCreds(creds)
|
err := StoreUsers(creds)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
zap.L().Error("store_creds",
|
zap.L().Error("store_creds",
|
||||||
zap.String("message", "failed to store creds"),
|
zap.String("message", "failed to store creds"),
|
||||||
@@ -148,18 +154,11 @@ func (dr *DehashedResults) ExtractCredentials() []Creds {
|
|||||||
return creds
|
return creds
|
||||||
}
|
}
|
||||||
|
|
||||||
type Creds struct {
|
func (User) TableName() string {
|
||||||
gorm.Model
|
|
||||||
Email string `json:"email" yaml:"email" xml:"email" gorm:"uniqueIndex:idx_email_username_password"`
|
|
||||||
Username string `json:"username" yaml:"username" xml:"username" gorm:"uniqueIndex:idx_email_username_password"`
|
|
||||||
Password string `json:"password" yaml:"password" xml:"password" gorm:"uniqueIndex:idx_email_username_password"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func (Creds) TableName() string {
|
|
||||||
return "creds"
|
return "creds"
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c Creds) ToString() string {
|
func (c User) ToString() string {
|
||||||
return fmt.Sprintf("%s%s%s", c.Username, "%", c.Password)
|
return fmt.Sprintf("%s%s%s", c.Username, "%", c.Password)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -197,38 +196,6 @@ func StoreDehashedResults(results DehashedResults) error {
|
|||||||
return lastErr
|
return lastErr
|
||||||
}
|
}
|
||||||
|
|
||||||
func StoreDehashedCreds(creds []Creds) error {
|
|
||||||
if len(creds) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
zap.L().Info("Storing credentials", zap.Int("count", len(creds)))
|
|
||||||
db := GetDB()
|
|
||||||
|
|
||||||
// Use batch insert with conflict handling
|
|
||||||
// This will insert records in batches and continue even if some fail
|
|
||||||
const batchSize = 100
|
|
||||||
var lastErr error
|
|
||||||
|
|
||||||
for i := 0; i < len(creds); i += batchSize {
|
|
||||||
end := i + batchSize
|
|
||||||
if end > len(creds) {
|
|
||||||
end = len(creds)
|
|
||||||
}
|
|
||||||
|
|
||||||
batch := creds[i:end]
|
|
||||||
// Use Clauses with OnConflict DoNothing to skip conflicts
|
|
||||||
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
|
|
||||||
if err != nil {
|
|
||||||
zap.L().Warn("Error storing some credentials", zap.Error(err))
|
|
||||||
lastErr = err
|
|
||||||
// Continue with next batch despite error
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return lastErr
|
|
||||||
}
|
|
||||||
|
|
||||||
func StoreDehashedQueryOptions(queryOptions *QueryOptions) error {
|
func StoreDehashedQueryOptions(queryOptions *QueryOptions) error {
|
||||||
db := GetDB()
|
db := GetDB()
|
||||||
return db.Create(queryOptions).Error
|
return db.Create(queryOptions).Error
|
||||||
|
|||||||
@@ -0,0 +1 @@
|
|||||||
|
package sqlite
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
package sqlite
|
||||||
|
|
||||||
|
import (
|
||||||
|
"go.uber.org/zap"
|
||||||
|
"gorm.io/gorm"
|
||||||
|
"gorm.io/gorm/clause"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Subdomain struct {
|
||||||
|
gorm.Model
|
||||||
|
Domain string `json:"domain" yaml:"domain" xml:"domain"`
|
||||||
|
Subdomain string `json:"subdomain" yaml:"subdomain" xml:"subdomain" gorm:"uniqueIndex:idx_subdomain"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func StoreSubdomains(subs []Subdomain) error {
|
||||||
|
if len(subs) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
zap.L().Info("Storing subdomains", zap.Int("count", len(subs)))
|
||||||
|
db := GetDB()
|
||||||
|
|
||||||
|
// Use batch insert with conflict handling
|
||||||
|
// This will insert records in batches and continue even if some fail
|
||||||
|
const batchSize = 100
|
||||||
|
var lastErr error
|
||||||
|
|
||||||
|
for i := 0; i < len(subs); i += batchSize {
|
||||||
|
end := i + batchSize
|
||||||
|
if end > len(subs) {
|
||||||
|
end = len(subs)
|
||||||
|
}
|
||||||
|
|
||||||
|
batch := subs[i:end]
|
||||||
|
// Use Clauses with OnConflict DoNothing to skip conflicts
|
||||||
|
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
|
||||||
|
if err != nil {
|
||||||
|
zap.L().Warn("Error storing some credentials", zap.Error(err))
|
||||||
|
lastErr = err
|
||||||
|
// Continue with next batch despite error
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return lastErr
|
||||||
|
}
|
||||||
@@ -0,0 +1,58 @@
|
|||||||
|
package sqlite
|
||||||
|
|
||||||
|
import (
|
||||||
|
"go.uber.org/zap"
|
||||||
|
"gorm.io/gorm"
|
||||||
|
"gorm.io/gorm/clause"
|
||||||
|
)
|
||||||
|
|
||||||
|
type User struct {
|
||||||
|
gorm.Model
|
||||||
|
Company string `json:"company" yaml:"company" xml:"company"`
|
||||||
|
Position string `json:"position" yaml:"position" xml:"position"`
|
||||||
|
Department string `json:"department" yaml:"department" xml:"department"`
|
||||||
|
PhoneNumber string `json:"phone_number" yaml:"phone_number" xml:"phone_number"`
|
||||||
|
FullName string `json:"full_name" yaml:"full_name" xml:"full_name"`
|
||||||
|
Phone string `json:"phone" yaml:"phone" xml:"phone"`
|
||||||
|
Linkedin string `json:"linkedin" yaml:"linkedin" xml:"linkedin"`
|
||||||
|
Twitter string `json:"twitter" yaml:"twitter" xml:"twitter"`
|
||||||
|
Facebook string `json:"facebook" yaml:"facebook" xml:"facebook"`
|
||||||
|
Instagram string `json:"instagram" yaml:"instagram" xml:"instagram"`
|
||||||
|
Youtube string `json:"youtube" yaml:"youtube" xml:"youtube"`
|
||||||
|
Gravatar string `json:"gravatar" yaml:"gravatar" xml:"gravatar"`
|
||||||
|
Email string `json:"email" yaml:"email" xml:"email" gorm:"uniqueIndex:idx_email_username_password"`
|
||||||
|
Username string `json:"username" yaml:"username" xml:"username" gorm:"uniqueIndex:idx_email_username_password"`
|
||||||
|
Password string `json:"password" yaml:"password" xml:"password" gorm:"uniqueIndex:idx_email_username_password"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func StoreUsers(users []User) error {
|
||||||
|
if len(users) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
zap.L().Info("Storing credentials", zap.Int("count", len(users)))
|
||||||
|
db := GetDB()
|
||||||
|
|
||||||
|
// Use batch insert with conflict handling
|
||||||
|
// This will insert records in batches and continue even if some fail
|
||||||
|
const batchSize = 100
|
||||||
|
var lastErr error
|
||||||
|
|
||||||
|
for i := 0; i < len(users); i += batchSize {
|
||||||
|
end := i + batchSize
|
||||||
|
if end > len(users) {
|
||||||
|
end = len(users)
|
||||||
|
}
|
||||||
|
|
||||||
|
batch := users[i:end]
|
||||||
|
// Use Clauses with OnConflict DoNothing to skip conflicts
|
||||||
|
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
|
||||||
|
if err != nil {
|
||||||
|
zap.L().Warn("Error storing some credentials", zap.Error(err))
|
||||||
|
lastErr = err
|
||||||
|
// Continue with next batch despite error
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return lastErr
|
||||||
|
}
|
||||||
@@ -536,37 +536,6 @@ func StoreWhoisRecord(whoisRecord WhoisRecord) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func StoreWhoisSubdomainRecords(subdomainRecords []SubdomainRecord) error {
|
|
||||||
if len(subdomainRecords) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
zap.L().Info("Storing subdomain records", zap.Int("count", len(subdomainRecords)))
|
|
||||||
db := GetDB()
|
|
||||||
|
|
||||||
// Use batch insert with conflict handling
|
|
||||||
const batchSize = 100
|
|
||||||
var lastErr error
|
|
||||||
|
|
||||||
for i := 0; i < len(subdomainRecords); i += batchSize {
|
|
||||||
end := i + batchSize
|
|
||||||
if end > len(subdomainRecords) {
|
|
||||||
end = len(subdomainRecords)
|
|
||||||
}
|
|
||||||
|
|
||||||
batch := subdomainRecords[i:end]
|
|
||||||
// Use Clauses with OnConflict DoNothing to skip conflicts
|
|
||||||
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
|
|
||||||
if err != nil {
|
|
||||||
zap.L().Warn("Error storing some subdomain records", zap.Error(err))
|
|
||||||
lastErr = err
|
|
||||||
// Continue with next batch despite error
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return lastErr
|
|
||||||
}
|
|
||||||
|
|
||||||
func StoreWhoisHistoryRecords(historyRecords []HistoryRecord) error {
|
func StoreWhoisHistoryRecords(historyRecords []HistoryRecord) error {
|
||||||
if len(historyRecords) == 0 {
|
if len(historyRecords) == 0 {
|
||||||
return nil
|
return nil
|
||||||
@@ -628,3 +597,20 @@ func StoreWhoisLookup(lookup []LookupResult) error {
|
|||||||
|
|
||||||
return lastErr
|
return lastErr
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ReverseWhoisResponse represents the response from a reverse WHOIS lookup
|
||||||
|
type ReverseWhoisResponse struct {
|
||||||
|
RemainingCredits int `json:"remaining_credits"`
|
||||||
|
Data ReverseWhoisData `json:"data"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReverseWhoisData contains the domain count and list from a reverse WHOIS lookup
|
||||||
|
type ReverseWhoisData struct {
|
||||||
|
DomainsCount int `json:"domainsCount"`
|
||||||
|
DomainsList []string `json:"domainsList"`
|
||||||
|
NextPageSearchAfter *string `json:"nextPageSearchAfter"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (rwd ReverseWhoisData) String() string {
|
||||||
|
return fmt.Sprintf("Domains Count: %d\nDomains List: %v\nNext Page Search After: %v\n", rwd.DomainsCount, rwd.DomainsList, rwd.NextPageSearchAfter)
|
||||||
|
}
|
||||||
|
|||||||
@@ -298,7 +298,9 @@ func (w *DehashedWhoIs) WhoisHistory(domain string) ([]sqlite.HistoryRecord, err
|
|||||||
return whois.Data.Records, nil
|
return whois.Data.Records, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (w *DehashedWhoIs) ReverseWHOIS(include []string, exclude []string, reverseType string) (string, error) {
|
func (w *DehashedWhoIs) ReverseWHOIS(include []string, exclude []string, reverseType string) (sqlite.ReverseWhoisData, error) {
|
||||||
|
var whois sqlite.ReverseWhoisData
|
||||||
|
|
||||||
if w.debug {
|
if w.debug {
|
||||||
debug.PrintInfo("performing reverse whois search")
|
debug.PrintInfo("performing reverse whois search")
|
||||||
zap.L().Info("reverse_whois_debug",
|
zap.L().Info("reverse_whois_debug",
|
||||||
@@ -329,7 +331,7 @@ func (w *DehashedWhoIs) ReverseWHOIS(include []string, exclude []string, reverse
|
|||||||
zap.String("message", "failed to create request"),
|
zap.String("message", "failed to create request"),
|
||||||
zap.Error(err),
|
zap.Error(err),
|
||||||
)
|
)
|
||||||
return "", err
|
return whois, err
|
||||||
}
|
}
|
||||||
|
|
||||||
req.Header.Set("Content-Type", "application/json")
|
req.Header.Set("Content-Type", "application/json")
|
||||||
@@ -356,7 +358,7 @@ func (w *DehashedWhoIs) ReverseWHOIS(include []string, exclude []string, reverse
|
|||||||
zap.String("message", "failed to perform request"),
|
zap.String("message", "failed to perform request"),
|
||||||
zap.Error(err),
|
zap.Error(err),
|
||||||
)
|
)
|
||||||
return "", err
|
return whois, err
|
||||||
}
|
}
|
||||||
if res == nil {
|
if res == nil {
|
||||||
if w.debug {
|
if w.debug {
|
||||||
@@ -365,7 +367,7 @@ func (w *DehashedWhoIs) ReverseWHOIS(include []string, exclude []string, reverse
|
|||||||
zap.L().Error("reverse_whois",
|
zap.L().Error("reverse_whois",
|
||||||
zap.String("message", "response was nil"),
|
zap.String("message", "response was nil"),
|
||||||
)
|
)
|
||||||
return "", errors.New("response was nil")
|
return whois, errors.New("response was nil")
|
||||||
}
|
}
|
||||||
|
|
||||||
b, err := io.ReadAll(res.Body)
|
b, err := io.ReadAll(res.Body)
|
||||||
@@ -378,7 +380,7 @@ func (w *DehashedWhoIs) ReverseWHOIS(include []string, exclude []string, reverse
|
|||||||
zap.String("message", "failed to read response body"),
|
zap.String("message", "failed to read response body"),
|
||||||
zap.Error(err),
|
zap.Error(err),
|
||||||
)
|
)
|
||||||
return "", err
|
return whois, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check for HTTP status code errors
|
// Check for HTTP status code errors
|
||||||
@@ -396,7 +398,7 @@ func (w *DehashedWhoIs) ReverseWHOIS(include []string, exclude []string, reverse
|
|||||||
zap.String("error", dhErr.Error()),
|
zap.String("error", dhErr.Error()),
|
||||||
zap.String("body_error", string(b)),
|
zap.String("body_error", string(b)),
|
||||||
)
|
)
|
||||||
return "", &dhErr
|
return whois, &dhErr
|
||||||
}
|
}
|
||||||
|
|
||||||
if w.debug {
|
if w.debug {
|
||||||
@@ -404,7 +406,30 @@ func (w *DehashedWhoIs) ReverseWHOIS(include []string, exclude []string, reverse
|
|||||||
debug.PrintJson(fmt.Sprintf("Body: %s\n", string(b[:])))
|
debug.PrintJson(fmt.Sprintf("Body: %s\n", string(b[:])))
|
||||||
}
|
}
|
||||||
|
|
||||||
return string(b), nil
|
var whoisResponse sqlite.ReverseWhoisResponse
|
||||||
|
err = json.Unmarshal(b, &whoisResponse)
|
||||||
|
if err != nil {
|
||||||
|
if w.debug {
|
||||||
|
debug.PrintInfo("failed to unmarshal response body")
|
||||||
|
debug.PrintError(err)
|
||||||
|
}
|
||||||
|
zap.L().Error("reverse_whois",
|
||||||
|
zap.String("message", "failed to unmarshal response body"),
|
||||||
|
zap.Error(err),
|
||||||
|
)
|
||||||
|
return whois, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if w.debug {
|
||||||
|
debug.PrintInfo("unmarshalled response body")
|
||||||
|
debug.PrintJson(fmt.Sprintf("Remaining Credits: %d\n", whoisResponse.RemainingCredits))
|
||||||
|
debug.PrintJson(fmt.Sprintf("Data: %v\n", whoisResponse.Data))
|
||||||
|
}
|
||||||
|
w.balance = whoisResponse.RemainingCredits
|
||||||
|
|
||||||
|
whois = whoisResponse.Data
|
||||||
|
|
||||||
|
return whois, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (w *DehashedWhoIs) WhoisIP(ipAddress string) ([]sqlite.LookupResult, error) {
|
func (w *DehashedWhoIs) WhoisIP(ipAddress string) ([]sqlite.LookupResult, error) {
|
||||||
|
|||||||