45 Commits

Author SHA1 Message Date
Evan Hosinski 24b1f99413 Add mutually exclusive flags to targets command 2025-06-03 20:08:40 -04:00
Evan Hosinski 22beaf2310 Refactor user and credential handling: rename Creds to User, update database migrations, and add targets subcommand for exporting users and subdomains 2025-06-03 19:42:40 -04:00
KrakenTech 7cff4e70b8 Merge pull request #9 from Kraken-OffSec/add-targets-subcommand-option
Refactor user and credential handling: rename Creds to User, update d…
2025-06-03 19:29:44 -04:00
Evan Hosinski 0bd9347074 Refactor user and credential handling: rename Creds to User, update database migrations, and add targets subcommand for exporting users and subdomains 2025-06-03 19:29:10 -04:00
KrakenTech 9a22445e55 Merge pull request #8 from Kraken-OffSec/fixed-local-db-output-file-location
Fixed error where the local db instance would be written to crowsnest…
2025-06-03 18:10:18 -04:00
Evan Hosinski e167a10fcc Fixed error where the local db instance would be written to crowsnest.sql/crowsnest.sql 2025-06-03 18:08:57 -04:00
Evan Hosinski 84f3becdf2 Fixed root.go branding 2025-05-22 10:47:10 -04:00
KrakenTech a2358d0714 Update Makefile 2025-05-21 09:26:35 -04:00
Evan Hosinski ded3e4ae71 Altered Dehasher to CrowsNest
! WARNING !

This will break older versions of CrowsNest. The database file will no longer be compatible
2025-05-21 09:14:56 -04:00
Evan Hosinski fbe1eda8e9 Altered Dehasher to CrowsNest
! WARNING !

This will break older versions of CrowsNest. The database file will no longer be compatible
2025-05-21 09:12:20 -04:00
Evan Hosinski 63f302604f Changed output directory to be CrowsNest 2025-05-21 09:08:07 -04:00
KrakenTech a0f216508d Merge pull request #7 from Kraken-OffSec/sqlite-pure-go-driver
Sqlite pure go driver
2025-05-20 10:42:54 -04:00
Evan Hosinski 7d1b7a2225 Updated the makefile to reflect CGO_ENABLED=0 2025-05-20 10:41:09 -04:00
Evan Hosinski acf6336516 Added the kraken k to the .img folder 2025-05-20 10:39:08 -04:00
Evan Hosinski ce8079f72e Added the kraken k to the .img folder 2025-05-19 12:01:48 -04:00
KrakenTech 54ee9a192f Update README.md 2025-05-19 11:55:03 -04:00
KrakenTech 95791312d0 Update README.md 2025-05-17 14:45:01 -04:00
KrakenTech d4e626d574 Update README.md 2025-05-17 13:32:51 -04:00
KrakenTech dd2050ce64 Update README.md 2025-05-17 13:31:46 -04:00
KrakenTech 075f826816 Add files via upload 2025-05-17 13:19:15 -04:00
Evan Hosinski de34de60d9 Updated Readme to reflect new branding 2025-05-17 13:14:19 -04:00
Evan Hosinski 67c4e0394e Updated Readme to reflect new branding 2025-05-17 13:13:29 -04:00
Evan Hosinski 49424c1603 Merge remote-tracking branch 'origin/main' 2025-05-17 13:13:19 -04:00
Evan Hosinski d4db32c8b9 Updated Readme to reflect new branding 2025-05-17 12:58:37 -04:00
KrakenTech d2cc0d1022 Update Makefile 2025-05-17 11:11:33 -04:00
KrakenTech fe4d904c5f Update README.md 2025-05-17 10:48:14 -04:00
KrakenTech c246c88724 Merge pull request #6 from Kraken-OffSec/crows-nest-rebrand
Added logo png file
2025-05-17 10:41:19 -04:00
Evan Hosinski 056fe0d4a7 Added logo png file 2025-05-17 10:40:51 -04:00
KrakenTech 78186c71b2 Merge pull request #5 from Kraken-OffSec/crows-nest-rebrand
Added logo png file
2025-05-17 10:37:53 -04:00
Evan Hosinski ac835f22ca Added logo png file 2025-05-17 10:37:04 -04:00
KrakenTech 89d8872407 Merge pull request #4 from Kraken-OffSec/crows-nest-rebrand
Crows nest rebrand
2025-05-17 10:28:53 -04:00
Evan Hosinski edbf9d57dc updated dehasher.go in Makefile 2025-05-17 10:27:33 -04:00
Evan Hosinski 00d9a6b57e fixed output of coffee and primary banner 2025-05-17 10:25:16 -04:00
Evan Hosinski 61052b3308 Implemented IString for file writing for Hunter Structs. 2025-05-17 10:07:49 -04:00
Evan Hosinski dc15315403 Rebrand to CrowsNest
Added coffee rootCmd
2025-05-17 10:00:59 -04:00
Evan Hosinski 1152a1910c updated api to dehashed to match naming convention of hunter. 2025-05-17 09:03:04 -04:00
KrakenTech 174071e472 Merge pull request #3 from Kraken-OffSec/hunter.io-subcommand
Hunter.io subcommand
2025-05-17 00:02:40 -04:00
Ar1ste1a 1f20cff41b Added hunter.io functions and file writes. 2025-05-17 00:01:48 -04:00
Ar1ste1a 2caccbee9d Added hunter.io functions and file writes. 2025-05-16 23:46:55 -04:00
Ar1ste1a 59ca1d4e92 Updated README.md
updated typos

added 'easy time' upgrade to log filtering
2025-05-16 20:54:16 -04:00
Ar1ste1a ad4c9197a2 Updated README.md
updated typos

added 'easy time' upgrade to log filtering
2025-05-16 20:53:53 -04:00
Ar1ste1a fccb213cf3 Updated README.md
updated typos

added 'easy time' upgrade to log filtering
2025-05-16 20:51:29 -04:00
Ar1ste1a 61e777e379 Updated README.md
updated typos

added 'easy time' upgrade to log filtering
2025-05-16 20:50:43 -04:00
Ar1ste1a 0b5a4bfea0 Updated README.md
updated typos

added 'easy time' upgrade to log filtering
2025-05-16 20:48:53 -04:00
Ar1ste1a 40e583b787 Updated README.md
updated typos

added 'easy time' upgrade to log filtering
2025-05-16 20:26:14 -04:00
76 changed files with 4128 additions and 1422 deletions
Binary file not shown.

After

Width:  |  Height:  |  Size: 2.1 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.0 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 23 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.6 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 56 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 188 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 89 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 71 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 52 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.5 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 849 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.2 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 26 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 28 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 63 KiB

BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 172 KiB

Before

Width:  |  Height:  |  Size: 148 KiB

After

Width:  |  Height:  |  Size: 148 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 110 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 88 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 30 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 16 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 15 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 966 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 112 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 7.4 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 7.5 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 47 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 44 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.5 MiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 100 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 116 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 69 KiB

BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 87 KiB

BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 76 KiB

BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 29 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 59 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 78 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 101 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 33 KiB

+5 -5
View File
@@ -1,10 +1,10 @@
# Makefile for Dehasher
# Makefile for CrowsNest
# Go command
GO=go
# Binary name
BINARY_NAME=dehasher
BINARY_NAME=crowsnest
# Build directory
BUILD_DIR=build/bin
@@ -16,7 +16,7 @@ PLATFORMS=linux darwin windows
ARCHS=amd64 arm64
# Version info from git tag or default
VERSION=$(shell git describe --tags 2>/dev/null || echo "v1.2.0")
VERSION=$(shell git describe --tags 2>/dev/null || echo "v1.3.3")
.PHONY: all clean build build-all
@@ -30,14 +30,14 @@ clean:
# Build for current platform
build:
$(GO) build -o $(BUILD_DIR)/$(BINARY_NAME) -ldflags "-X main.version=$(VERSION)" dehasher.go
CGO_ENABLED=0 $(GO) build -o $(BUILD_DIR)/$(BINARY_NAME) -ldflags "-X main.version=$(VERSION)" crowsnest.go
# Build for all platforms
build-all: clean
@for platform in $(PLATFORMS); do \
for arch in $(ARCHS); do \
echo "Building for $$platform/$$arch..."; \
GOOS=$$platform GOARCH=$$arch $(GO) build -o $(BUILD_DIR)/$(BINARY_NAME)-$$platform-$$arch -ldflags "-X main.version=$(VERSION)" dehasher.go; \
GOOS=$$platform GOARCH=$$arch CGO_ENABLED=0 $(GO) build -o $(BUILD_DIR)/$(BINARY_NAME)-$$platform-$$arch -ldflags "-X main.version=$(VERSION)" crowsnest.go; \
if [ "$$platform" = "windows" ]; then \
mv $(BUILD_DIR)/$(BINARY_NAME)-$$platform-$$arch $(BUILD_DIR)/$(BINARY_NAME)-$$platform-$$arch.exe; \
fi; \
+210 -124
View File
@@ -1,5 +1,8 @@
# 🚀 Dehasher
### A CLI tool for seamless interaction with the Dehashed API
<div align="center">
<img src=.img/crowsnest.png style="width: 500px; height: auto" alt="CrowsNest Logo" title="CrowsNest Logo">
</div>
### A CLI tool for seamless interaction with the Dehashed and Hunter.io APIs.
---
@@ -16,46 +19,28 @@
- **API Key Management**: Securely store and manage API keys.
- **Formatted Output**: Easy to read and understand.
- **Intuitive Database Querying**: Query for specific information.
- **Person and Company Enrichment**: Retrieve detailed information about people and companies.
- **Email Verification**: Verify the existence and quality of email addresses.
---
## 📦 Installation
Clone the repository and build the tool:
```bash
git clone https://github.com/Ar1ste1a/Dehasher.git
cd Dehasher
go build dehasher.go
```
<hr></hr>
## 🔰 Getting Started
To begin, clone the repository
``` bash-session
git clone https://github.com/Ar1ste1a/Dehasher.git
cd Dehasher
go build dehasher.go
git clone https://github.com/Kraken-OffSec/CrowsNest.git
cd crowsnest
go build crowsnest.go
```
<hr></hr>
## 🛠️ Initial Setup
Dehasher requires an API key from Dehashed. Set it up with:
```bash
ar1ste1a@kali:~$ dehasher set-key <redacted>
```
<hr></hr>
---
## 🗄️ Database Configuration
Dehasher supports two database storage options:
CrowsNest supports two database storage options:
1. **Default Path** (default): Stores the database at `~/.local/share/Dehasher/db/dehashed.sqlite`
2. **Local Path**: Stores the database in the current directory as `./dehasher.sqlite`
1. **Default Path** (default): Stores the database at `~/.local/share/crowsnest/db/crowsnest.sqlite`
2. **Local Path**: Stores the database in the current directory as `./crowsnest.sqlite`
The **Local Path** option allows for separate databases for different projects or engagements.
@@ -63,82 +48,111 @@ To configure the database location:
```bash
# Use local database in current directory
./dehasher set-local-db true
./crowsnest set-local-db true
# Use default database path
./dehasher set-local-db false
./crowsnest set-local-db false
```
<hr></hr>
---
## 🔍 Crafting Queries
## 🌐 Dehashed
### Initial Setup
CrowsNest requires an API key from Dehashed. Set it up with:
![Alt text](.img/set-dehashed.png "Set Dehashed Key")
```bash
ar1ste1a@kali:~$ crowsnest set-dehashed <redacted>
```
### Simple Query
Dehasher can be used simply for example to query for credentials matching a given email domain.
CrowsNest can be used simply for example to query for credentials matching a given email domain.
![Alt text](.img/simple_query.png "Simple Query")
``` go
# Provide credentials for emails matching @target.com
dehasher api -D @target.com -C
# Provide credentials for domains matching target.com
crowsnest dehashed -D target.com
```
### Simple Credentials Query
Dehasher can also be used to return only credentials for a given query.
CrowsNest can also be used to return only credentials for a given query.
![Alt text](.img/simple_creds_query.png "Creds Only Query")
``` go
# Provide credentials for emails matching @target.com
dehasher api -E @target.com -C
```
### Multiple Match Query
Dehasher is capable of handling multiple queries for the same field.
This is useful for when you want to search for multiple domains, or multiple usernames.
``` go
# Provide credentials for emails matching @target.com and @target2.com
dehasher api -E @target.com,@target2.com -C
crowsnest dehashed -D @target.com -C
```
### Wildcard Query
Dehasher is capable of handling wildcard queries.
CrowsNest is capable of handling wildcard queries.
A wildcard query cannot begin with a wildcard.
This is a limitation of the Dehashed API.
An asterisk can be used to denote multiple characters, and a question mark can be used to denote a single character.
![Alt text](.img/wildcard_sample.png "Wildcard Query")
<br>
![Alt text](.img/wildcard_query.png "Wildcard Query")
``` go
# Provide credentials for emails matching @target.com and @target2.com
dehasher api -E @target?.com -C -W
crowsnest dehashed -E @target?.com -C -W
```
### Email Query
Dehashed has dictated that emails should be searched in the following format:
`email:target.name&domain:target.com`.
As such, to query an email, please use the following format (note, wildcard is not required but can be useful):
<br>
*see photo above in Wildcard Query*
``` go
# Provide credentials for emails matching target.*@target.com
crowsnest dehashed -W -E 'target*' -D target.com
```
You may also query the domain and find emails as well
``` go
# Provide credentials for emails matching target.com
crowsnest dehashed -D target.com -C
```
### Combining Queries
CrowsNest is capable of combining queries.
This is useful for when you want to query for credentials matching a given email or domain, but only for a specific username.
![Alt text](.img/combining_queries.png "Combined Query")
``` go
# Provide credentials for emails matching @target.com and username containing 'admin'
crowsnest dehashed -D target.com -U admin
```
### Regex Query
Dehasher is capable of handling regex queries.
CrowsNest is capable of handling regex queries.
Simply denote regex queries with the `-R` flag.
Place all regex queries in quotes with the corresponding query flag in single quotes.
<br>
!!!! *Currently, the Regex Operators appear to be broken. I am waiting on a response from Dehashed* !!!!
``` go
# Return matches for emails matching this given regex query
dehasher api -R -e '[a-zA-Z0-9]+(?:\.[a-zA-Z0-9]+)?@target.com'
crowsnest dehashed -R -E 'joh?n(ath[oa]n)' -D hotmail.com'
```
### Output Text (default JSON)
Dehasher is capable of handling output formats.
CrowsNest is capable of handling output formats.
The default output format is JSON.
To change the output format, use the `-f` flag.
Dehasher currently supports JSON, YAML, XML, and TEXT output formats.
CrowsNest currently supports JSON, YAML, XML, and TEXT output formats.
``` go
# Return matches for usernames exactly matching "admin" and write to text file 'admins_file.txt'
dehasher api -U admin -o admins_file -f txt
crowsnest dehashed -U admin -o admins_file -f txt
```
<hr></hr>
---
## 🌐 WhoIs Lookups
Dehasher supports WHOIS lookups, history searches, reverse WHOIS searches, IP lookups, MX lookups, NS lookups, and subdomain scans.
CrowsNest supports WHOIS lookups, history searches, reverse WHOIS searches, IP lookups, MX lookups, NS lookups, and subdomain scans.
The WhoIs Lookups require a separate API Credit from the Dehashed API.
### Domain Lookup
Dehasher can perform a domain lookup for a given domain.
CrowsNest can perform a domain lookup for a given domain.
This provides a tree view of the domain's WHOIS information.
![Alt text](.img/tree_whois_lookup.png "WhoIs Tree View")
![Alt text](.img/whois_domain.png "WhoIs Tree View")
```bash
# Perform a WHOIS lookup for example.com
dehasher whois -d example.com
crowsnest whois -d example.com
```
### History Lookup
@@ -147,93 +161,173 @@ This is a Dehashed API limitation.
The history lookup is immediately written to file and not displayed in the terminal or stored in the database.
```bash
# Perform a WHOIS history search for example.com
dehasher whois -d example.com -H
crowsnest whois -d example.com -H
```
### Subdomain Scan
CrowsNest can perform a subdomain scan for a given domain.
This provides a list of all subdomains that match the given query.
![Alt text](.img/whois_subdomain.png "WhoIs Tree View")
```bash
# Perform a WHOIS subdomain scan for google.com
crowsnest whois -d google.com -s
```
### Reverse WHOIS Lookup
Dehasher can perform a reverse WHOIS lookup for given criteria.
CrowsNest can perform a reverse WHOIS lookup for given criteria.
This provides a list of all domains that match the given query.
The reverse WHOIS lookup is immediately written to file and not displayed in the terminal or stored in the database.
The reverse WHOIS lookup is immediately written to file and not stored in the database.
![Alt text](.img/whois_reverse.png "WhoIs Tree View")
```bash
# Perform a reverse WHOIS lookup for example.com
dehasher whois -I example.com
crowsnest whois -I example.com
```
### IP Lookup
Dehasher can perform a reverse IP lookup for a given IP address.
CrowsNest can perform a reverse IP lookup for a given IP address.
This provides a list of all domains that match the given query.
![Alt text](.img/reverse_ip_lookup.png "WhoIs Tree View")
![Alt text](.img/whois_ip.png "WhoIs View")
```bash
# Perform a reverse IP lookup for 8.8.8.8
dehasher whois -i 8.8.8.8
crowsnest whois -i 8.8.8.8
```
### MX Lookup
Dehasher can perform an MX lookup for a given MX hostname.
CrowsNest can perform an MX lookup for a given MX hostname.
This provides a list of all domains that match the given query.
![Alt text](.img/mx_lookup.png "WhoIs Tree View")
![Alt text](.img/whois_mx.png "WhoIs Tree View")
```bash
# Perform a reverse MX lookup for google.com
dehasher whois -m google.com
crowsnest whois -m stmp.google.com
```
### NS Lookup
Dehasher can perform an NS lookup for a given NS hostname.
CrowsNest can perform an NS lookup for a given NS hostname.
This provides a list of all domains that match the given query.
The picture below also includes the --debug global flag.
![Alt text](.img/debug_ns_search.png "WhoIs Tree View")
![Alt text](.img/whois_ns.png "WhoIs Tree View")
```bash
# Perform a reverse NS lookup for google.com
dehasher whois -n google.com
```
### Subdomain Scan
Dehasher can perform a subdomain scan for a given domain.
This provides a list of all subdomains that match the given query.
![Alt text](.img/subdomains_lookup.png "WhoIs Tree View")
```bash
# Perform a WHOIS subdomain scan for google.com
dehasher whois -d google.com -s
crowsnest whois -n google.com
```
---
## 🌐 Hunter.io
CrowsNest supports Hunter.io lookups.
Hunter.io lookups require a separate API Key from the Dehashed API.
This can be set using the `set-hunter` command.
![Alt text](.img/set-hunter.png "Set Dehashed Key")
```bash
# Set the Hunter.io API key
crowsnest set-hunter <redacted>
```
### Domain Search
CrowsNest can perform a domain search for a given domain.
This provides information about company including a description, social media information and any technologies in use.<br>
![Alt text](.img/hunter_domain.png "Hunter.io Domain Search")
```bash
# Perform a Hunter.io domain search for example.com
crowsnest hunter -d example.com -D
```
### Email Verification
CrowsNest can perform an email verification search for a given email.
This provides a verification and score of a given email address.
![Alt text](.img/hunter_emailverification.png "Hunter.io Email Verification")
```bash
# Perform a Hunter.io email verification search for example@target.com
crowsnest hunter -e example@target.com -V
```
### Company Enrichment
CrowsNest can perform a company enrichment search for a given domain.
This provides information about a company given its domain.
![Alt text](.img/hunter_company.png "Hunter.io Company Enrichment")
```bash
# Perform a Hunter.io company enrichment search for example.com
crowsnest hunter -d example.com -C
```
### Email Finder
CrowsNest can perform an email finder search for a given domain, first name, and last name.
This provides information about a user including a confidence score, and any social media accounts linked to a first name, last name and email.
![Alt text](.img/hunter_emailfind.png "Hunter.io Email Finder")
```bash
# Perform a Hunter.io email finder search for example.com
crowsnest hunter -d example.com -F John -L Doe -E
```
### Person Enrichment
CrowsNest can perform a person enrichment search for a given email.
This provides information about a user given an email address..
![Alt text](.img/hunter_person.png "Hunter.io Person Enrichment")
```bash
# Perform a Hunter.io person enrichment search for example@target.com
crowsnest hunter -e example@target.com -P
```
### Combined Enrichment
CrowsNest can perform a combined enrichment search for a given email.
This is a combination of the company and person enrichments given an email address.
![Alt text](.img/combined_enrichment_1.png "Hunter.io Combined Enrichment")
![Alt text](.img/combined_enrichment_2.png "Hunter.io Combined Enrichment")
```bash
# Perform a Hunter.io combined enrichment search for example@target.com
crowsnest hunter -e example@target.com -B
```
## Debugging
CrowsNest supports debugging. This can be enabled using the `--debug` flag in the root command.
![Alt text](.img/crowsnest_debugging_global.png "Debugging")
```bash
# Perform a Hunter.io combined enrichment search for example@target.com with debugging enabled
crowsnest --debug hunter -e example@target.com -B
```
---
## 📊 Database Querying
Dehasher stores query results in a local database.
CrowsNest stores query results in a local database.
This database can be queried for previous results.
This is useful for when you want to query for specific information.
This database also includes WhoIs Information and Subdomain Scan results, but does **not** include historical lookups.
## Simple Query
![Alt text](.img/simple_query_db.png "Simple Query")
Dehasher supports querying the database for previous results.
This is useful for when you want to query for specific information.
#### It's possible to query the database using shorthand and without knowing any SQL at all.
#### The following queries the results table where username is not null, only showing the username, email and password columns.
![Alt text](.img/query_simple.png "Simple Query")
#### You may also add in a simple query using the `-q` flag. The following displays a 'LIKE' clause on the email column.
#### Note the '%\<clause\>%' is still required.
![Alt text](.img/query_where.png "Simple Query")
```bash
# Query the database for all results containing the word 'admin' in the username
dehasher query -t results -q "username LIKE '%admin%'"
crowsnest query -t results -q "username LIKE '%admin%'"
```
## Raw SQL Queries
![Alt text](.img/raw_query_db.png "Raw Query")
![Alt text](.img/query_raw.png "Raw Query")
Dehasher also supports raw SQL queries. This is useful for when you want to query for specific information.
CrowsNest also supports raw SQL queries. This is useful for when you want to query for specific information.
```bash
# Query the database for all results containing the word 'admin' in the username
dehasher query -r "SELECT * FROM results WHERE username LIKE '%admin%'"
crowsnest query -r "SELECT * FROM results WHERE username LIKE '%admin%'"
```
## Query Options
Dehasher supports a number of query options. These options can be used to filter the results of a query.
CrowsNest supports a number of query options. These options can be used to filter the results of a query.
```bash
# Query the database for all results containing the word 'admin' in the username
dehasher query -t results -q "username LIKE '%admin%'" -n username,email,password
crowsnest query -t results -q "username LIKE '%admin%'" -n username,email,password
```
## Listing Tables and Columns
Dehasher supports listing all available tables and columns.
CrowsNest supports listing all available tables and columns.
This is useful for when you want to query for specific information.
![Alt text](.img/query_alltables.png "List All Tables")
```bash
# List all available tables and columns
dehasher query -a
crowsnest query -a
```
The current tables available for query are:
@@ -249,39 +343,34 @@ The current tables available for query are:
- Previous query runs to the dehashed API
- lookup
- Results of any Whois NS, MX, or IP lookup
- hunter_domain
- Results from a hunter.io domain search
- hunter_email
- Results extracted from a domain saerch and email finder.
---
# Exporting Results
Dehasher supports exporting results to a file.
CrowsNest supports exporting results to a file.
This is useful for when you want to requery for specific information without touching the Dehashed API.
The export subcommand supports all the same options as the query subcommand.
The export subcommand also supports file naming and output format control.
![Alt text](.img/export_raw.png "Export Results")
```bash
# Export all results containing the word 'admin' in the username to a text file
dehasher export -t results -q "username LIKE '%admin%'" -o admins_file -f txt
crowsnest export -t results -q "username LIKE '%admin%'" -o admins_file -f txt
```
## 🐛 Debugging
Dehasher uses the `zap` logging library for logging. The logs are stored in `~/.local/share/Dehasher/logs`.
The logs can be easily queried from the Dehasher CLI.
```bash
# Show the last 10 logs
dehasher logs -l 10
CrowsNest uses the `zap` logging library for logging. The logs are stored in `~/.local/share/crowsnest/logs`.
The logs can be easily queried from the crowsnest CLI.
# Show logs from the last 24 hours
dehasher logs -s "last 24 hours"
# Show logs from the last 24 hours with a severity of error or fatal
dehasher logs -s "05-01-2025" -v error,fatal
```
### Logs Dates
#### Dehasher utilized 'easy time' to determine the appropriate time for a given query.
### Filtering by Date
#### CrowsNest utilizes 'easy time' to determine the appropriate time for a given query.
![Alt text](.img/easy_time_parsing.png "Easy Time")
#### You may also used dates mixed with easy time to perform queries.
![Alt text](.img/easy_time_query_2.png "Mixed Time")
![Alt text](.img/mixed_time_query.png "Mixed Time")
#### The following formats are supported:
- `last 24 hours`
- `last 2 days`
@@ -294,27 +383,24 @@ dehasher logs -s "05-01-2025" -v error,fatal
- `05-01-25`
- `May 01, 2025`
## 🎉 Sample Run
```bash
ar1ste1a@kali:~$ dehasher api -D <redacted>.com -o <redacted> -f json
Making 3 Requests for 10000 Records (30000 Total)
[*] Querying Dehashed API...
[*] Performing Request...
[+] Retrieved 2740 Records
[-] Not Enough Entries, ending queries
[+] Discovered 10 Credentials
[*] Writing entries to file: <redacted>.json
[*] Success
[*] Completing Process
# Show the last 10 logs
crowsnest logs -l 10
# Show logs from the last 24 hours
crowsnest logs -s "last 24 hours"
# Show logs from the last 24 hours with a severity of error or fatal
crowsnest logs -s "05-01-2025" -v error,fatal
```
## 🤝 Contributing
Contributions are welcome! Submit a pull request to help improve Dehasher.
Contributions are welcome! Submit a pull request to help improve CrowsNest.
## [Buy Me A Coffee](https://buymeacoffee.com/ehosinskiz)
<div align="center">
<img src="https://img.wanman.io/fUSu0/jUtovIFE52.png/raw" style="width: 350px; height: auto" alt="Ar1ste1a" title="Ar1ste1a Offensive Security">
<img src=.img/kraken_k.png style="width: 350px; height: auto" alt="Ar1ste1a" title="Ar1ste1a Offensive Security">
</div>
## **Release The Kraken**
-140
View File
@@ -1,140 +0,0 @@
package cmd
import (
"dehasher/internal/badger"
"dehasher/internal/debug"
"dehasher/internal/dehashed"
"dehasher/internal/sqlite"
"fmt"
"github.com/spf13/cobra"
"go.uber.org/zap"
)
func init() {
// Add query command to root command
rootCmd.AddCommand(apiCmd)
// Add flags specific to api command
apiCmd.Flags().IntVarP(&maxRecords, "max-records", "m", 30000, "Maximum amount of records to return")
apiCmd.Flags().IntVarP(&maxRequests, "max-requests", "r", -1, "Maximum number of requests to make")
apiCmd.Flags().IntVarP(&startingPage, "starting-page", "s", 1, "Starting page for requests")
apiCmd.Flags().BoolVarP(&printBalance, "print-balance", "b", false, "Print remaining balance after requests")
apiCmd.Flags().BoolVarP(&regexMatch, "regex-match", "R", false, "Use regex matching on query fields")
apiCmd.Flags().BoolVarP(&wildcardMatch, "wildcard-match", "W", false, "Use wildcard matching on query fields (Use ? to replace a single character, and * for multiple characters)")
apiCmd.Flags().BoolVarP(&credsOnly, "creds-only", "C", false, "Return credentials only")
apiCmd.Flags().StringVarP(&outputFormat, "format", "f", "json", "Output format (json, yaml, xml, txt)")
apiCmd.Flags().StringVarP(&outputFile, "output", "o", "query", "File to output results to including extension")
apiCmd.Flags().StringVarP(&usernameQuery, "username", "U", "", "Username query")
apiCmd.Flags().StringVarP(&emailQuery, "email-query", "E", "", "Email query")
apiCmd.Flags().StringVarP(&ipQuery, "ip", "I", "", "IP address query")
apiCmd.Flags().StringVarP(&domainQuery, "domain", "D", "", "Domain query")
apiCmd.Flags().StringVarP(&passwordQuery, "password", "P", "", "Password query")
apiCmd.Flags().StringVarP(&vinQuery, "vin", "V", "", "VIN query")
apiCmd.Flags().StringVarP(&licensePlateQuery, "license", "L", "", "License plate query")
apiCmd.Flags().StringVarP(&addressQuery, "address", "A", "", "Address query")
apiCmd.Flags().StringVarP(&phoneQuery, "phone", "M", "", "Phone query")
apiCmd.Flags().StringVarP(&socialQuery, "social", "S", "", "Social query")
apiCmd.Flags().StringVarP(&cryptoCurrencyAddressQuery, "crypto", "B", "", "Crypto currency address query")
apiCmd.Flags().StringVarP(&hashQuery, "hash", "Q", "", "Hashed password query")
apiCmd.Flags().StringVarP(&nameQuery, "name", "N", "", "Name query")
// Add mutually exclusive flags to wildcard match and regex match
apiCmd.MarkFlagsMutuallyExclusive("regex-match", "wildcard-match")
}
var (
// Query command flags
maxRecords int
maxRequests int
startingPage int
credsOnly bool
printBalance bool
regexMatch bool
wildcardMatch bool
outputFormat string
outputFile string
usernameQuery string
emailQuery string
ipQuery string
passwordQuery string
hashQuery string
nameQuery string
domainQuery string
vinQuery string
licensePlateQuery string
addressQuery string
phoneQuery string
socialQuery string
cryptoCurrencyAddressQuery string
// Query command
apiCmd = &cobra.Command{
Use: "api",
Short: "Query the Dehashed API",
Long: `Query the Dehashed API for emails, usernames, passwords, hashes, IP addresses, and names.`,
Run: func(cmd *cobra.Command, args []string) {
key := getStoredApiKey()
// Validate credentials
if key == "" {
fmt.Println("API key is required. Set the key with the \"set-key\" command. [dehasher set-key <api_key>]")
return
}
// Create new QueryOptions
queryOptions := sqlite.NewQueryOptions(
maxRecords,
maxRequests,
startingPage,
outputFormat,
outputFile,
usernameQuery,
emailQuery,
ipQuery,
passwordQuery,
hashQuery,
nameQuery,
domainQuery,
vinQuery,
licensePlateQuery,
addressQuery,
phoneQuery,
socialQuery,
cryptoCurrencyAddressQuery,
regexMatch,
wildcardMatch,
printBalance,
credsOnly,
debugGlobal,
)
// Create new Dehasher
dehasher := dehashed.NewDehasher(queryOptions)
dehasher.SetClientCredentials(
key,
)
// Start querying
dehasher.Start()
fmt.Println("\n[*] Completing Process")
err := sqlite.StoreQueryOptions(queryOptions)
if err != nil {
if debugGlobal {
debug.PrintInfo("failed to store query options")
debug.PrintError(err)
}
zap.L().Error("store_query_options",
zap.String("message", "failed to store query options"),
zap.Error(err),
)
fmt.Printf("Error storing query options: %v\n", err)
}
},
}
)
// Helper functions to get stored API credentials
func getStoredApiKey() string {
return badger.GetKey()
}
+141
View File
@@ -0,0 +1,141 @@
package cmd
import (
"crowsnest/internal/badger"
"crowsnest/internal/debug"
"crowsnest/internal/dehashed"
"crowsnest/internal/sqlite"
"fmt"
"github.com/spf13/cobra"
"go.uber.org/zap"
)
func init() {
// Add api command to root command
rootCmd.AddCommand(dehashedCmd)
// Add flags specific to api command
dehashedCmd.Flags().IntVarP(&maxRecords, "max-records", "m", 30000, "Maximum amount of records to return")
dehashedCmd.Flags().IntVarP(&maxRequests, "max-requests", "r", -1, "Maximum number of requests to make")
dehashedCmd.Flags().IntVarP(&startingPage, "starting-page", "s", 1, "Starting page for requests")
dehashedCmd.Flags().BoolVarP(&printBalance, "print-balance", "b", false, "Print remaining balance after requests")
dehashedCmd.Flags().BoolVarP(&regexMatch, "regex-match", "R", false, "Use regex matching on query fields")
dehashedCmd.Flags().BoolVarP(&wildcardMatch, "wildcard-match", "W", false, "Use wildcard matching on query fields (Use ? to replace a single character, and * for multiple characters)")
dehashedCmd.Flags().BoolVarP(&credsOnly, "creds-only", "C", false, "Return credentials only")
dehashedCmd.Flags().StringVarP(&outputFormat, "format", "f", "json", "Output format (json, yaml, xml, txt)")
dehashedCmd.Flags().StringVarP(&outputFile, "output", "o", "query", "File to output results to including extension")
dehashedCmd.Flags().StringVarP(&usernameQuery, "username", "U", "", "Username query")
dehashedCmd.Flags().StringVarP(&emailQuery, "email-query", "E", "", "HunterEmail query")
dehashedCmd.Flags().StringVarP(&ipQuery, "ip", "I", "", "IP address query")
dehashedCmd.Flags().StringVarP(&domainQuery, "domain", "D", "", "Domain query")
dehashedCmd.Flags().StringVarP(&passwordQuery, "password", "P", "", "Password query")
dehashedCmd.Flags().StringVarP(&vinQuery, "vin", "V", "", "VIN query")
dehashedCmd.Flags().StringVarP(&licensePlateQuery, "license", "L", "", "License plate query")
dehashedCmd.Flags().StringVarP(&addressQuery, "address", "A", "", "Address query")
dehashedCmd.Flags().StringVarP(&phoneQuery, "phone", "M", "", "Phone query")
dehashedCmd.Flags().StringVarP(&socialQuery, "social", "S", "", "Social query")
dehashedCmd.Flags().StringVarP(&cryptoCurrencyAddressQuery, "crypto", "B", "", "Crypto currency address query")
dehashedCmd.Flags().StringVarP(&hashQuery, "hash", "Q", "", "Hashed password query")
dehashedCmd.Flags().StringVarP(&nameQuery, "name", "N", "", "Name query")
// Add mutually exclusive flags to wildcard match and regex match
dehashedCmd.MarkFlagsMutuallyExclusive("regex-match", "wildcard-match")
}
var (
// Query command flags
maxRecords int
maxRequests int
startingPage int
credsOnly bool
printBalance bool
regexMatch bool
wildcardMatch bool
outputFormat string
outputFile string
usernameQuery string
emailQuery string
ipQuery string
passwordQuery string
hashQuery string
nameQuery string
domainQuery string
vinQuery string
licensePlateQuery string
addressQuery string
phoneQuery string
socialQuery string
cryptoCurrencyAddressQuery string
// Query command
dehashedCmd = &cobra.Command{
Use: "dehashed",
Short: "Query the Dehashed API",
Long: `Query the Dehashed API for emails, usernames, passwords, hashes, IP addresses, and names.`,
Run: func(cmd *cobra.Command, args []string) {
key := getDehashedApiKey()
// Validate credentials
if key == "" {
fmt.Println("API key is required. Set the key with the \"set-key\" command. [crowsnest set-key <api_key>]")
return
}
// Create new QueryOptions
queryOptions := sqlite.NewQueryOptions(
maxRecords,
maxRequests,
startingPage,
outputFormat,
outputFile,
usernameQuery,
emailQuery,
ipQuery,
passwordQuery,
hashQuery,
nameQuery,
domainQuery,
vinQuery,
licensePlateQuery,
addressQuery,
phoneQuery,
socialQuery,
cryptoCurrencyAddressQuery,
regexMatch,
wildcardMatch,
printBalance,
credsOnly,
debugGlobal,
)
// Create new Dehasher
dehasher := dehashed.NewDehasher(queryOptions)
dehasher.SetClientCredentials(
key,
)
// Start querying
dehasher.Start()
fmt.Println("\n[*] Completing Process")
// Store query options
err := sqlite.StoreDehashedQueryOptions(queryOptions)
if err != nil {
if debugGlobal {
debug.PrintInfo("failed to store query options")
debug.PrintError(err)
}
zap.L().Error("store_query_options",
zap.String("message", "failed to store query options"),
zap.Error(err),
)
fmt.Printf("Error storing query options: %v\n", err)
}
},
}
)
// Helper functions to get stored API credentials
func getDehashedApiKey() string {
return badger.GetDehashedKey()
}
-311
View File
@@ -1,311 +0,0 @@
package cmd
import (
"dehasher/internal/export"
"dehasher/internal/files"
"dehasher/internal/sqlite"
"fmt"
"github.com/spf13/cobra"
"go.uber.org/zap"
"strings"
)
func init() {
// Add Subcommand to db command
rootCmd.AddCommand(exportCmd)
// Add flags specific to export command
exportCmd.Flags().IntVarP(&exportLimitRows, "limit", "l", 100, "Limit number of results")
exportCmd.Flags().BoolVarP(&exportListAll, "list-all", "a", false, "List all tables and their columns")
exportCmd.Flags().StringVarP(&exportTableName, "table", "t", "", "Table to export (results, creds, whois, subdomains, history, runs)")
exportCmd.Flags().StringVarP(&exportNotNull, "not-null", "n", "", "Filter for non-null values (comma-separated list, e.g., 'password,email')")
exportCmd.Flags().StringVarP(&exportColumns, "columns", "c", "", "Columns to display in output (comma-separated list, e.g., 'username,email,password')")
exportCmd.Flags().StringVarP(&exportUserQuery, "user-query", "q", "", "User query to execute")
exportCmd.Flags().StringVarP(&exportRawQuery, "raw-query", "r", "", "Raw SQL query to execute")
exportCmd.Flags().StringVarP(&exportFormat, "format", "f", "json", "Output format (json, yaml, xml, txt)")
exportCmd.Flags().StringVarP(&exportFile, "file", "o", "export", "File to output results to including extension")
// Add mutually exclusive flags to query and raw-query
// Cannot use query and raw-query at the same time
exportCmd.MarkFlagsMutuallyExclusive("user-query", "raw-query")
// Raw query does not require a table
exportCmd.MarkFlagsMutuallyExclusive("user-query", "table")
// List all columns does not require a query or raw-query
exportCmd.MarkFlagsMutuallyExclusive("raw-query", "list-all")
}
// DB export command
var (
exportLimitRows int
exportListAll bool
exportTableName string
exportNotNull string
exportColumns string
exportUserQuery string
exportRawQuery string
exportFormat string
exportFile string
exportCmd = &cobra.Command{
Use: "export",
Short: "Export database to file",
Run: func(cmd *cobra.Command, args []string) {
// If list-all flag is set, list all tables and columns
if exportListAll {
listAvailableTables()
return
}
fmt.Println("[*] Exporting database...")
// If Raw Query is set, execute it and export
if exportRawQuery != "" {
fmt.Println("[*] Executing Raw Query...")
exportRawDBQuery()
return
}
// Validate table name
if exportTableName == "" {
fmt.Println("[!] Error: Table name is required. Use -t or --table to specify a table.")
fmt.Println("[*] Available tables: results, creds, whois, subdomains, history, runs")
fmt.Println("[*] Use --list-all to see all tables and their columns.")
return
}
if !isValidTable(exportTableName) {
fmt.Printf("[!] Error: Unknown table '%s'.\n", exportTableName)
fmt.Println("[*] Available tables: results, creds, whois, subdomains, history, runs")
fmt.Println("[*] Use --list-all to see all tables and their columns.")
return
}
// Validate columns if specified
if exportColumns != "" {
columns := strings.Split(exportColumns, ",")
invalidColumns := validateColumns(exportTableName, columns)
if len(invalidColumns) > 0 {
fmt.Printf("[!] Error: Invalid column(s) for table '%s': %s\n",
exportTableName, strings.Join(invalidColumns, ", "))
fmt.Println("[*] Available columns for this table:")
for i := 0; i < len(availableTables[exportTableName]); i += 5 {
end := i + 5
if end > len(availableTables[exportTableName]) {
end = len(availableTables[exportTableName])
}
fmt.Printf(" %s\n", strings.Join(availableTables[exportTableName][i:end], ", "))
}
return
}
}
// Validate not-null fields if specified
if exportNotNull != "" {
notNullFields := strings.Split(exportNotNull, ",")
invalidFields := validateColumns(exportTableName, notNullFields)
if len(invalidFields) > 0 {
fmt.Printf("[!] Error: Invalid not-null field(s) for table '%s': %s\n",
exportTableName, strings.Join(invalidFields, ", "))
fmt.Println("[*] Available columns for this table:")
for i := 0; i < len(availableTables[exportTableName]); i += 5 {
end := i + 5
if end > len(availableTables[exportTableName]) {
end = len(availableTables[exportTableName])
}
fmt.Printf(" %s\n", strings.Join(availableTables[exportTableName][i:end], ", "))
}
return
}
}
// Determine which table to query based on the tableTypeDBQuery parameter
table := sqlite.GetTable(exportTableName)
if table == sqlite.UnknownTable {
fmt.Printf("[!] Error: Unknown table type '%s'.\n", exportTableName)
fmt.Println("[*] Available tables: results, creds, whois, subdomains, history, runs")
fmt.Println("[*] Use --list-all to see all tables and their columns.")
return
}
fmt.Println("[*] Querying Database...")
exportTableQuery(table)
},
}
)
// exportTableQuery queries a table and exports the results
func exportTableQuery(table sqlite.Table) {
// Get the columns to query
columns := []string{"*"}
if exportColumns != "" {
columns = strings.Split(exportColumns, ",")
}
// Get the not null fields
notNullFields := []string{}
if exportNotNull != "" {
notNullFields = strings.Split(exportNotNull, ",")
}
// Get the user query
userQuery := ""
if exportUserQuery != "" {
userQuery = exportUserQuery
}
// Get the limit
limit := exportLimitRows
// Get the object for the table
object := table.Object()
// Check if object is nil (invalid table)
if object == nil {
fmt.Printf("[!] Error: Table '%s' is not valid or does not exist.\n", exportTableName)
return
}
// Query the database
db := sqlite.GetDB()
query := db.Model(object).Select(columns)
if len(notNullFields) > 0 {
for _, field := range notNullFields {
query = query.Where(fmt.Sprintf("%s IS NOT NULL", field))
}
}
if userQuery != "" {
query = query.Where(userQuery)
}
if limit > 0 {
query = query.Limit(limit)
}
rows, err := query.Rows()
if err != nil {
zap.L().Error("export_query",
zap.String("message", "failed to execute query"),
zap.Error(err),
)
fmt.Printf("[!] Error executing query: %v\n", err)
return
}
defer rows.Close()
// Get the columns
cols, err := rows.Columns()
if err != nil {
zap.L().Error("export_query",
zap.String("message", "failed to get columns from query"),
zap.Error(err),
)
fmt.Printf("[!] Error getting columns from query: %v\n", err)
return
}
// Prepare data for export
var results []map[string]interface{}
// Process the rows
for rows.Next() {
values := make([]interface{}, len(cols))
pointers := make([]interface{}, len(cols))
for i := range values {
pointers[i] = &values[i]
}
if err := rows.Scan(pointers...); err != nil {
zap.L().Error("export_query",
zap.String("message", "failed to scan row from query"),
zap.Error(err),
)
fmt.Printf("[!] Error scanning row from query: %v\n", err)
return
}
// Create a map for this row
rowMap := make(map[string]interface{})
for i, col := range cols {
val := values[i]
rowMap[col] = val
}
results = append(results, rowMap)
}
// Export the results
exportResults(results)
}
// exportRawDBQuery executes a raw query and exports the results
func exportRawDBQuery() {
db := sqlite.GetDB()
rows, err := db.Raw(exportRawQuery).Rows()
if err != nil {
zap.L().Error("export_raw_query",
zap.String("message", "failed to execute raw query"),
zap.Error(err),
)
fmt.Printf("[!] Error executing raw query: %v\n", err)
return
}
defer rows.Close()
columns, err := rows.Columns()
if err != nil {
zap.L().Error("export_raw_query",
zap.String("message", "failed to get columns from raw query"),
zap.Error(err),
)
fmt.Printf("[!] Error getting columns from raw query: %v\n", err)
return
}
// Prepare data for export
var results []map[string]interface{}
// Process the rows
for rows.Next() {
values := make([]interface{}, len(columns))
pointers := make([]interface{}, len(columns))
for i := range values {
pointers[i] = &values[i]
}
if err := rows.Scan(pointers...); err != nil {
zap.L().Error("export_raw_query",
zap.String("message", "failed to scan row from raw query"),
zap.Error(err),
)
fmt.Printf("[!] Error scanning row from raw query: %v\n", err)
return
}
// Create a map for this row
rowMap := make(map[string]interface{})
for i, col := range columns {
val := values[i]
rowMap[col] = val
}
results = append(results, rowMap)
}
// Export the results
exportResults(results)
}
// exportResults exports the results to a file
func exportResults(results []map[string]interface{}) {
// Get file type
fileType := files.GetFileType(exportFormat)
// Export results
err := export.WriteQueryResultsToFile(results, exportFile, fileType)
if err != nil {
zap.L().Error("export_results",
zap.String("message", "failed to write to file"),
zap.Error(err),
)
fmt.Printf("[!] Error writing to file: %v\n", err)
return
}
fmt.Printf("[+] Exported %d records to file: %s%s\n", len(results), exportFile, fileType.Extension())
}
+440
View File
@@ -0,0 +1,440 @@
package cmd
import (
"crowsnest/internal/badger"
"crowsnest/internal/debug"
"crowsnest/internal/export"
"crowsnest/internal/files"
hunter "crowsnest/internal/hunter.io"
"crowsnest/internal/pretty"
"crowsnest/internal/sqlite"
"fmt"
"github.com/spf13/cobra"
"go.uber.org/zap"
"time"
)
func init() {
// Add hunter command to root command
rootCmd.AddCommand(hunterCmd)
// Add flags specific to hunter command
hunterCmd.Flags().StringVarP(&hunterDomain, "domain", "d", "", "Domain to query")
hunterCmd.Flags().StringVarP(&hunterEmail, "email", "e", "", "Email to query")
hunterCmd.Flags().StringVarP(&hunterFirstName, "first-name", "F", "", "First name to query")
hunterCmd.Flags().StringVarP(&hunterLastName, "last-name", "L", "", "Last name to query")
hunterCmd.Flags().BoolVarP(&hunterDomainSearch, "domain-search", "D", false, "Search for domain")
hunterCmd.Flags().BoolVarP(&hunterEmailFind, "email-find", "E", false, "Find emails for user using domain, first name, and last name")
hunterCmd.Flags().BoolVarP(&hunterEmailVerify, "email-verify", "V", false, "Verify email")
hunterCmd.Flags().BoolVarP(&hunterCompanyEnrichmentDomain, "company-enrichment", "C", false, "Company enrichment for domain")
hunterCmd.Flags().BoolVarP(&hunterPersonEnrichmentEmail, "person-enrichment", "P", false, "Person enrichment for email")
hunterCmd.Flags().BoolVarP(&hunterCombinedEnrichmentEmail, "combined-enrichment", "B", false, "Combined Company and Person enrichment for email")
hunterCmd.Flags().StringVarP(&hunterOutputFormat, "format", "f", "json", "Output format (json, yaml, xml, txt)")
hunterCmd.Flags().StringVarP(&hunterOutputFile, "output", "o", "hunter", "File to output results to including extension")
// Add mutually exclusive flags to hunter command
hunterCmd.MarkFlagsMutuallyExclusive("email-find")
}
var (
// Hunter Commands Flags
hunterDomain string
hunterEmail string
hunterFirstName string
hunterLastName string
hunterDomainSearch bool
hunterEmailFind bool
hunterEmailVerify bool
hunterCompanyEnrichmentDomain bool
hunterPersonEnrichmentEmail bool
hunterCombinedEnrichmentEmail bool
hunterOutputFormat string
hunterOutputFile string
hunterCmd = &cobra.Command{
Use: "hunter",
Short: "Hunter.io API interaction",
Long: `Interact with the Hunter.io API for email and domain information.`,
Run: func(cmd *cobra.Command, args []string) {
if debugGlobal {
debug.PrintInfo("debug mode enabled")
zap.L().Info("hunter_debug",
zap.String("message", "debug mode enabled"),
)
}
// Flag Checks
if !hunterFlagCheck() {
return
}
if hunterOutputFile == "" {
if debugGlobal {
debug.PrintInfo("output file not specified, using default")
}
hunterOutputFile = "hunter_" + time.Now().Format("05_04_05")
}
if hunterOutputFormat == "" {
if debugGlobal {
debug.PrintInfo("output format not specified, using default")
}
hunterOutputFormat = "json"
}
fType := files.GetFileType(hunterOutputFormat)
if fType == files.UNKNOWN {
fmt.Println("[!] Error: Invalid output format. Must be 'json', 'xml', 'yaml', or 'txt'.")
return
}
if debugGlobal {
debug.PrintInfo("using output format: " + hunterOutputFormat)
}
fmt.Println("[*] Hunter.io API interaction [Beta]")
h := hunter.NewHunterIO(getHunterApiKey(), debugGlobal)
if hunterDomainSearch {
fmt.Println("[*] Performing domain search search...")
result, err := h.DomainSearch(hunterDomain)
if err != nil {
if debugGlobal {
debug.PrintInfo("failed to perform domain search")
debug.PrintError(err)
}
zap.L().Error("hunter_domain_search",
zap.String("message", "failed to perform domain search"),
zap.Error(err),
)
fmt.Printf("Error performing domain search: %v\n", err)
return
}
// Store the users discovered
var creds []sqlite.User
for _, email := range result.Emails {
creds = append(creds, sqlite.User{Email: email.Value})
}
err = sqlite.StoreUsers(creds)
if err != nil {
if debugGlobal {
debug.PrintInfo("failed to store hunter domain search")
debug.PrintError(err)
}
zap.L().Error("store_hunter_domain_search",
zap.String("message", "failed to store hunter domain search"),
zap.Error(err),
)
fmt.Printf("Error storing Hunter.io Domain Search Result: %v\n", err)
}
// Write Hunter.io Domain Search Result to file
fmt.Printf("[*] Writing Hunter.io Domain Search Result to file: %s%s\n", hunterOutputFile, fType.Extension())
err = export.WriteIStringToFile(result, hunterOutputFile, fType)
if err != nil {
if debugGlobal {
debug.PrintInfo("failed to write hunter domain search to file")
debug.PrintError(err)
}
zap.L().Error("write_hunter_domain_search",
zap.String("message", "failed to write hunter domain search to file"),
zap.Error(err),
)
fmt.Printf("Error writing Hunter.io Domain Search Result to file: %v\n", err)
}
// Pretty Print Hunter.io Domain Search Result
fmt.Println("Domain Search Result:")
pretty.HunterDomainTree(hunterDomain, result)
return
}
if hunterEmailFind {
fmt.Println("[*] Performing email find search...")
result, err := h.EmailFinder(hunterDomain, hunterFirstName, hunterLastName)
if err != nil {
if debugGlobal {
debug.PrintInfo("failed to perform email find")
debug.PrintError(err)
}
zap.L().Error("hunter_email_find",
zap.String("message", "failed to perform email find"),
zap.Error(err),
)
fmt.Printf("Error performing email find: %v\n", err)
return
}
// Write Hunter.io Email Finder Result to file
fmt.Printf("[*] Writing Hunter.io Email Finder Result to file: %s%s\n", hunterOutputFile, fType.Extension())
err = export.WriteIStringToFile(result, hunterOutputFile, fType)
if err != nil {
if debugGlobal {
debug.PrintInfo("failed to write hunter email find to file")
debug.PrintError(err)
}
zap.L().Error("write_hunter_email_find",
zap.String("message", "failed to write hunter email find to file"),
zap.Error(err),
)
fmt.Printf("Error writing Hunter.io Email Finder Result to file: %v\n", err)
}
fmt.Println("Email Find Result:")
var (
headers = []string{"Email", "Score", "Domain", "Accept All", "Position", "Twitter", "Linkedin", "Phone Number", "Company", "Verification"}
rows [][]string
)
rows = append(rows, []string{
result.Email,
fmt.Sprintf("%d", result.Score),
result.Domain,
fmt.Sprintf("%t", result.AcceptAll),
result.Position,
result.Twitter,
result.LinkedinURL,
result.PhoneNumber,
result.Company,
fmt.Sprintf("%v", result.Verification),
})
pretty.Table(headers, rows)
return
}
if hunterEmailVerify {
fmt.Println("[*] Performing email verification search...")
result, err := h.EmailVerification(hunterEmail)
if err != nil {
if debugGlobal {
debug.PrintInfo("failed to perform email verification")
debug.PrintError(err)
}
zap.L().Error("hunter_email_verification",
zap.String("message", "failed to perform email verification"),
zap.Error(err),
)
fmt.Printf("Error performing email verification: %v\n", err)
return
}
// Write Hunter.io Email Verification Result to file
fmt.Printf("[*] Writing Hunter.io Email Verification Result to file: %s%s\n", hunterOutputFile, fType.Extension())
err = export.WriteIStringToFile(result, hunterOutputFile, fType)
if err != nil {
if debugGlobal {
debug.PrintInfo("failed to write hunter email verification to file")
debug.PrintError(err)
}
zap.L().Error("write_hunter_email_verification",
zap.String("message", "failed to write hunter email verification to file"),
zap.Error(err),
)
fmt.Printf("Error writing Hunter.io Email Verification Result to file: %v\n", err)
}
// Pretty Print Hunter.io Email Verification Result
var (
headers = []string{"Email", "Result", "Score", "Disposable", "MX Records", "SMTP Server", "SMTP Check"}
rows [][]string
)
rows = append(rows, []string{
result.Email,
result.Result,
fmt.Sprintf("%d", result.Score),
fmt.Sprintf("%t", result.Disposable),
fmt.Sprintf("%t", result.MXRecords),
fmt.Sprintf("%t", result.SMTPServer),
fmt.Sprintf("%t", result.SMTPCheck),
})
fmt.Println("Email Verification Result:")
pretty.Table(headers, rows)
return
}
if hunterCompanyEnrichmentDomain {
fmt.Println("[*] Performing company enrichment search...")
result, err := h.CompanyEnrichment(hunterDomain)
if err != nil {
if debugGlobal {
debug.PrintInfo("failed to perform company enrichment")
debug.PrintError(err)
}
zap.L().Error("hunter_company_enrichment",
zap.String("message", "failed to perform company enrichment"),
zap.Error(err),
)
fmt.Printf("Error performing company enrichment: %v\n", err)
return
}
// Write to file
fmt.Printf("[*] Writing Hunter.io Company Enrichment Result to file: %s%s\n", hunterOutputFile, fType.Extension())
err = export.WriteIStringToFile(result, hunterOutputFile, fType)
if err != nil {
if debugGlobal {
debug.PrintInfo("failed to write hunter company enrichment to file")
debug.PrintError(err)
}
zap.L().Error("write_hunter_company_enrichment",
zap.String("message", "failed to write hunter company enrichment to file"),
zap.Error(err),
)
fmt.Printf("Error writing Hunter.io Company Enrichment Result to file: %v\n", err)
}
// Pretty Print Hunter.io Company Enrichment Result
fmt.Println("Company Enrichment Result:")
pretty.HunterCompanyEnrichmentTree(hunterDomain, result)
return
}
if hunterPersonEnrichmentEmail {
fmt.Println("[*] Performing person enrichment search...")
result, err := h.PersonEnrichment(hunterEmail)
if err != nil {
if debugGlobal {
debug.PrintInfo("failed to perform person enrichment")
debug.PrintError(err)
}
zap.L().Error("hunter_person_enrichment",
zap.String("message", "failed to perform person enrichment"),
zap.Error(err),
)
fmt.Printf("Error performing person enrichment: %v\n", err)
return
}
// Write to file
fmt.Printf("[*] Writing Hunter.io Person Enrichment Result to file: %s%s\n", hunterOutputFile, fType.Extension())
err = export.WriteIStringToFile(result, hunterOutputFile, fType)
if err != nil {
if debugGlobal {
debug.PrintInfo("failed to write hunter person enrichment to file")
debug.PrintError(err)
}
zap.L().Error("write_hunter_person_enrichment",
zap.String("message", "failed to write hunter person enrichment to file"),
zap.Error(err),
)
fmt.Printf("Error writing Hunter.io Person Enrichment Result to file: %v\n", err)
}
// Pretty Print Hunter.io Person Enrichment Result
fmt.Println("Person Enrichment Result:")
pretty.HunterPersonEnrichmentTree(hunterEmail, result)
return
}
if hunterCombinedEnrichmentEmail {
fmt.Println("[*] Performing combined enrichment search...")
result, err := h.CombinedEnrichment(hunterEmail)
if err != nil {
if debugGlobal {
debug.PrintInfo("failed to perform combined enrichment")
debug.PrintError(err)
}
zap.L().Error("hunter_combined_enrichment",
zap.String("message", "failed to perform combined enrichment"),
zap.Error(err),
)
fmt.Printf("Error performing combined enrichment: %v\n", err)
return
}
// Write to file
fmt.Printf("[*] Writing Hunter.io Combined Enrichment Result to file: %s%s\n", hunterOutputFile, fType.Extension())
err = export.WriteIStringToFile(result, hunterOutputFile, fType)
if err != nil {
if debugGlobal {
debug.PrintInfo("failed to write hunter combined enrichment to file")
debug.PrintError(err)
}
zap.L().Error("write_hunter_combined_enrichment",
zap.String("message", "failed to write hunter combined enrichment to file"),
zap.Error(err),
)
fmt.Printf("Error writing Hunter.io Combined Enrichment Result to file: %v\n", err)
}
fmt.Println("Combined Enrichment Result:")
pretty.HunterCombinedEnrichmentTree(hunterEmail, result)
return
}
},
}
)
func hunterFlagCheck() bool {
if debugGlobal {
debug.PrintInfo("checking flags")
}
var optionSet bool
if hunterDomainSearch {
if hunterDomain == "" {
fmt.Println("Domain is required for domain search")
return false
}
optionSet = true
}
if hunterEmailVerify {
if hunterEmail == "" {
fmt.Println("Email is required for email verification")
return false
}
optionSet = true
}
if hunterCompanyEnrichmentDomain {
if hunterDomain == "" {
fmt.Println("Domain is required for company enrichment")
return false
}
optionSet = true
}
if hunterPersonEnrichmentEmail {
if hunterEmail == "" {
fmt.Println("Email is required for person enrichment")
return false
}
optionSet = true
}
if hunterCombinedEnrichmentEmail {
if hunterEmail == "" {
fmt.Println("Email is required for combined enrichment")
return false
}
optionSet = true
}
if hunterEmailFind {
if hunterFirstName == "" || hunterLastName == "" {
fmt.Println("First name and last name are required for email find")
return false
}
if hunterDomain == "" {
fmt.Println("Domain is required for email find")
return false
}
optionSet = true
}
if !optionSet {
fmt.Println("[!] No options selected")
return false
}
return true
}
// Helper functions to get stored API credentials
func getHunterApiKey() string {
return badger.GetHunterKey()
}
+2 -2
View File
@@ -1,8 +1,8 @@
package cmd
import (
"dehasher/internal/easyTime"
"dehasher/internal/pretty"
"crowsnest/internal/easyTime"
"crowsnest/internal/pretty"
"encoding/json"
"fmt"
"github.com/spf13/cobra"
+116 -120
View File
@@ -1,8 +1,11 @@
package cmd
import (
"dehasher/internal/pretty"
"dehasher/internal/sqlite"
"crowsnest/internal/debug"
"crowsnest/internal/export"
"crowsnest/internal/files"
"crowsnest/internal/pretty"
"crowsnest/internal/sqlite"
"encoding/json"
"fmt"
"github.com/spf13/cobra"
@@ -10,123 +13,6 @@ import (
"strings"
)
// Map of available tables and their columns
var availableTables = map[string][]string{
"creds": {
"id", "created_at", "updated_at", "deleted_at", "email", "username", "password",
},
//"history": {
// "id", "created_at", "updated_at", "deleted_at", "domain_name", "domain_type",
// "registrar_name", "whois_server", "created_date_iso8601", "updated_date_iso8601", "expires_date_iso8601",
//},
"lookup": {
"id", "created_at", "updated_at", "deleted_at", "search_term", "type", "first_seen", "last_visit",
"name",
},
// Query Options
"runs": {
"id", "created_at", "updated_at", "deleted_at", "max_records", "max_requests", "starting_page",
"output_format", "output_file", "regex_match", "wildcard_match", "username_query", "email_query",
"ip_query", "pass_query", "hash_query", "name_query", "domain_query", "vin_query", "license_plate_query",
"address_query", "phone_query", "social_query", "crypto_address_query", "print_balance", "creds_only",
},
"results": {
"id", "created_at", "updated_at", "deleted_at", "dehashed_id", "email", "ip_address", "username",
"password", "hashed_password", "hash_type", "name", "vin", "license_plate", "url", "social",
"cryptocurrency_address", "address", "phone", "company", "database_name",
},
"subdomains": {
"id", "created_at", "updated_at", "deleted_at", "domain", "first_seen", "last_seen",
},
"whois": {
"id", "created_at", "updated_at", "deleted_at", "audit", "contact_email", "created_date", "created_date_normalized",
"domain_name", "domain_name_ext", "estimated_domain_age", "expires_date", "expires_date_normalized", "footer", "header",
"name_servers", "parse_code", "raw_text", "registrant", "registrar_iana_id", "registrar_name", "registry_data",
"status", "stripped_text", "updated_date", "updated_date_normalized",
},
}
// Function to list available tables and their columns
func listAvailableTables() {
fmt.Println("Available tables and columns:")
// Prepare data for pretty.Table
headers := []string{"Table", "Columns"}
var tableRows [][]string
// Sort tables alphabetically for consistent output
var tableNames []string
for tableName := range availableTables {
tableNames = append(tableNames, tableName)
}
// Simple bubble sort for table names
for i := 0; i < len(tableNames)-1; i++ {
for j := 0; j < len(tableNames)-i-1; j++ {
if tableNames[j] > tableNames[j+1] {
tableNames[j], tableNames[j+1] = tableNames[j+1], tableNames[j]
}
}
}
// Create rows for the table
for _, tableName := range tableNames {
columns := availableTables[tableName]
// Format columns with line breaks for better readability
var formattedColumns string
for i := 0; i < len(columns); i += 5 {
end := i + 5
if end > len(columns) {
end = len(columns)
}
if i > 0 {
formattedColumns += "\n"
}
formattedColumns += strings.Join(columns[i:end], ", ")
}
tableRows = append(tableRows, []string{tableName, formattedColumns})
}
// Display the table
pretty.Table(headers, tableRows)
}
// Function to validate table name
func isValidTable(tableName string) bool {
_, exists := availableTables[tableName]
return exists
}
// Function to validate column names for a specific table
func validateColumns(tableName string, columns []string) []string {
if tableName == "" || columns == nil || len(columns) == 0 || columns[0] == "*" {
return nil
}
tableColumns, exists := availableTables[tableName]
if !exists {
return []string{fmt.Sprintf("Table '%s' does not exist", tableName)}
}
var invalidColumns []string
for _, col := range columns {
valid := false
for _, tableCol := range tableColumns {
if col == tableCol {
valid = true
break
}
}
if !valid {
invalidColumns = append(invalidColumns, col)
}
}
return invalidColumns
}
func init() {
// Add whois command to root command
rootCmd.AddCommand(queryCmd)
@@ -139,6 +25,8 @@ func init() {
queryCmd.Flags().StringVarP(&dbQueryUserQuery, "user-query", "q", "", "User query to execute")
queryCmd.Flags().StringVarP(&dbQueryRawQuery, "raw-query", "r", "", "Raw SQL query to execute")
queryCmd.Flags().BoolVarP(&dbQueryListAll, "list-all", "a", false, "List all tables and their columns")
queryCmd.Flags().StringVarP(&dbQueryFormat, "format", "f", "json", "Output format (json, yaml, xml, txt)")
queryCmd.Flags().StringVarP(&dbQueryFile, "file", "o", "query", "File to output results to")
// Add mutually exclusive flags to query and raw-query
// Cannot use query and raw-query at the same time
@@ -157,11 +45,14 @@ var (
dbQueryUserQuery string
dbQueryRawQuery string
dbQueryListAll bool
dbQueryFormat string
dbQueryFile string
queryCmd = &cobra.Command{
Use: "query",
Short: "Query the database",
Long: `Query the database for various information.`,
Long: `Query the database for various information.
If file is specified, results are written to file and not displayed in the terminal.`,
Run: func(cmd *cobra.Command, args []string) {
// If list-all flag is set, list all tables and columns
if dbQueryListAll {
@@ -312,6 +203,49 @@ func tableQuery(table sqlite.Table) {
return
}
// Export results if file name is specified
if len(strings.TrimSpace(dbQueryFile)) > 0 {
fmt.Println("[*] Exporting results to file...")
if debugGlobal {
debug.PrintInfo("exporting results to file: " + dbQueryFile)
}
// Prepare data for export
var results []map[string]interface{}
// Process the rows
for rows.Next() {
values := make([]interface{}, len(cols))
pointers := make([]interface{}, len(cols))
for i := range values {
pointers[i] = &values[i]
}
if err := rows.Scan(pointers...); err != nil {
zap.L().Error("export_query",
zap.String("message", "failed to scan row from query"),
zap.Error(err),
)
fmt.Printf("[!] Error scanning row from query: %v\n", err)
return
}
// Create a map for this row
rowMap := make(map[string]interface{})
for i, col := range cols {
val := values[i]
rowMap[col] = val
}
results = append(results, rowMap)
}
// Export the results
exportQueryResults(results)
return
}
fmt.Println("[*] Querying Database...")
// Prepare data for pretty.Table
headers := cols
var tableRows [][]string
@@ -402,6 +336,49 @@ func rawDBQuery() {
return
}
if len(strings.TrimSpace(dbQueryFile)) > 0 {
fmt.Println("[*] Exporting results to file...")
if debugGlobal {
debug.PrintInfo("exporting results to file: " + dbQueryFile)
}
// Prepare data for export
var results []map[string]interface{}
// Process the rows
for rows.Next() {
values := make([]interface{}, len(columns))
pointers := make([]interface{}, len(columns))
for i := range values {
pointers[i] = &values[i]
}
if err := rows.Scan(pointers...); err != nil {
zap.L().Error("export_raw_query",
zap.String("message", "failed to scan row from raw query"),
zap.Error(err),
)
fmt.Printf("[!] Error scanning row from raw query: %v\n", err)
return
}
// Create a map for this row
rowMap := make(map[string]interface{})
for i, col := range columns {
val := values[i]
rowMap[col] = val
}
results = append(results, rowMap)
}
// Export the results
exportQueryResults(results)
return
}
fmt.Println("[*] Querying Database...")
// Prepare data for pretty.Table
headers := columns
var tableRows [][]string
@@ -468,3 +445,22 @@ func rawDBQuery() {
// Display the table
pretty.Table(headers, tableRows)
}
// exportQueryResults exports the results to a file
func exportQueryResults(results []map[string]interface{}) {
// Get file type
fileType := files.GetFileType(dbQueryFormat)
// Export results
err := export.WriteQueryResultsToFile(results, dbQueryFile, fileType)
if err != nil {
zap.L().Error("export_results",
zap.String("message", "failed to write to file"),
zap.Error(err),
)
fmt.Printf("[!] Error writing to file: %v\n", err)
return
}
fmt.Printf("[+] Exported %d records to file: %s%s\n", len(results), dbQueryFile, fileType.Extension())
}
+61 -33
View File
@@ -1,8 +1,9 @@
package cmd
import (
"dehasher/internal/badger"
"crowsnest/internal/badger"
"fmt"
"github.com/fatih/color"
"github.com/spf13/cobra"
"go.uber.org/zap"
"os"
@@ -15,31 +16,17 @@ var (
// rootCmd is the base command for the CLI.
rootCmd = &cobra.Command{
Use: "dehasher",
Short: `Dehasher is a cli tool for querying the dehashed api.`,
Use: "crowsnest",
Short: `CrowsNest is a cli tool for querying the common OSINT api's.`,
Long: fmt.Sprintf(
"%s\n%s",
"%s\n",
`
______ _______ _______ _______ _______ _______
( __ \ ( ____ \|\ /|( ___ )( ____ \|\ /|( ____ \( ____ )
| ( \ )| ( \/| ) ( || ( ) || ( \/| ) ( || ( \/| ( )|
| | ) || (__ | (___) || (___) || (_____ | (___) || (__ | (____)|
| | | || __) | ___ || ___ |(_____ )| ___ || __) | __)
| | ) || ( | ( ) || ( ) | ) || ( ) || ( | (\ (
| (__/ )| (____/\| ) ( || ) ( |/\____) || ) ( || (____/\| ) \ \__
(______/ (_______/|/ \||/ \|\_______)|/ \|(_______/|/ \__/
An Ar1ste1a Project
`,
`––•–√\/––√\/––•––––•–√\/––√\/––•––––•–√\/––√\/––•––√\/––•––––•–√\/––√\/––•––
Dehasher can query the query API for:
- Emails - Usernames - Password
- Hashes - IP Addresses - Names
- VINs - License Plates - Addresses
- Phones - Social Media - Crypto Currency Addresses
Dehasher supports:
- Regex Matching
- Exact Matching
––•–√\/––√\/––•––––•–√\/––√\/––•––––•–√\/––√\/––•––√\/––•––––•–√\/––√\/––•––
╔═╗┬─┐┌─┐┬ ┬┌─┐╔╗╔┌─┐┌─┐┌┬┐
║ ├┬┘│ ││││└─┐║║║├┤ └─┐ │
╚═╝┴└─└─┘└┴┘└─┘╝╚╝└─┘└─┘ ┴
Crows Nest OSINT Recon Suite
⚓ A KrakenTech Intelligence Tool
`,
),
Version: "v1.2.1",
@@ -66,21 +53,39 @@ func init() {
rootCmd.PersistentFlags().BoolVar(&debugGlobal, "debug", false, "Show debug information")
// Add subcommands
rootCmd.AddCommand(setKeyCmd)
rootCmd.AddCommand(setDehashedKeyCmd)
rootCmd.AddCommand(setHunterKeyCmd)
rootCmd.AddCommand(setLocalDb)
rootCmd.AddCommand(buyMeCoffeeCmd)
}
// Command to set API key
var setKeyCmd = &cobra.Command{
Use: "set-key [key]",
Short: "Set and store API key",
var setDehashedKeyCmd = &cobra.Command{
Use: "set-dehashed [key]",
Short: "Set and store Dehashed.com API key",
Args: cobra.ExactArgs(1),
Run: func(cmd *cobra.Command, args []string) {
key := args[0]
// Store key in badger DB
err := storeApiKey(key)
err := storeDehashedApiKey(key)
if err != nil {
fmt.Printf("Error storing API key: %v\n", err)
fmt.Printf("Error storing Dehashed API key: %v\n", err)
return
}
fmt.Println("API key stored successfully")
},
}
var setHunterKeyCmd = &cobra.Command{
Use: "set-hunter [key]",
Short: "Set and store Hunter.io API key",
Args: cobra.ExactArgs(1),
Run: func(cmd *cobra.Command, args []string) {
key := args[0]
// Store key in badger DB
err := storeHunterApiKey(key)
if err != nil {
fmt.Printf("Error storing Hunter API key: %v\n", err)
return
}
fmt.Println("API key stored successfully")
@@ -89,7 +94,7 @@ var setKeyCmd = &cobra.Command{
var setLocalDb = &cobra.Command{
Use: "local-db [true|false]",
Short: "Set dehasher to use a local database path instead of the default path",
Short: "Set crowsnest to use a local database path instead of the default path",
Args: cobra.ExactArgs(1),
Run: func(cmd *cobra.Command, args []string) {
var useLocalDatabase bool
@@ -115,9 +120,32 @@ var setLocalDb = &cobra.Command{
},
}
var buyMeCoffeeCmd = &cobra.Command{
Use: "coffee",
Short: "Support the project by buying a coffee",
Run: func(cmd *cobra.Command, args []string) {
fmt.Println(color.HiRedString(" ;)(; "))
fmt.Println(color.HiCyanString(" We Hope You Enjoy Our Product :----:"))
fmt.Println(color.HiCyanString(" C|====|"))
fmt.Println(color.HiCyanString(" | |"))
fmt.Print(color.HiGreenString(" Support the project by buying a coffee: "))
fmt.Print(color.BlueString("https://buymeacoffee.com/ehosinskiz "))
fmt.Println(color.HiCyanString("`----'"))
},
}
// Helper functions to store API credentials
func storeApiKey(key string) error {
err := badger.StoreKey(key)
func storeDehashedApiKey(key string) error {
err := badger.StoreDehashedKey(key)
if err != nil {
fmt.Printf("Error storing API key: %v\n", err)
return err
}
return nil
}
func storeHunterApiKey(key string) error {
err := badger.StoreHunterKey(key)
if err != nil {
fmt.Printf("Error storing API key: %v\n", err)
return err
+133
View File
@@ -0,0 +1,133 @@
package cmd
import (
"crowsnest/internal/pretty"
"fmt"
"strings"
)
// Map of available tables and their columns
var availableTables = map[string][]string{
"users": {
"id", "created_at", "updated_at", "deleted_at", "email", "username", "password",
},
//"history": {
// "id", "created_at", "updated_at", "deleted_at", "domain_name", "domain_type",
// "registrar_name", "whois_server", "created_date_iso8601", "updated_date_iso8601", "expires_date_iso8601",
//},
"lookup": {
"id", "created_at", "updated_at", "deleted_at", "search_term", "type", "first_seen", "last_visit",
"name",
},
// Query Options
"runs": {
"id", "created_at", "updated_at", "deleted_at", "max_records", "max_requests", "starting_page",
"output_format", "output_file", "regex_match", "wildcard_match", "username_query", "email_query",
"ip_query", "pass_query", "hash_query", "name_query", "domain_query", "vin_query", "license_plate_query",
"address_query", "phone_query", "social_query", "crypto_address_query", "print_balance", "creds_only",
},
"dehashed": {
"id", "created_at", "updated_at", "deleted_at", "dehashed_id", "email", "ip_address", "username",
"password", "hashed_password", "hash_type", "name", "vin", "license_plate", "url", "social",
"cryptocurrency_address", "address", "phone", "company", "database_name",
},
"subdomains": {
"id", "created_at", "updated_at", "deleted_at", "domain", "subdomain",
},
"whois": {
"id", "created_at", "updated_at", "deleted_at", "audit", "contact_email", "created_date", "created_date_normalized",
"domain_name", "domain_name_ext", "estimated_domain_age", "expires_date", "expires_date_normalized", "footer", "header",
"name_servers", "parse_code", "raw_text", "registrant", "registrar_iana_id", "registrar_name", "registry_data",
"status", "stripped_text", "updated_date", "updated_date_normalized",
},
"hunter_domain": {
"id", "created_at", "updated_at", "deleted_at", "domain", "disposable", "webmail", "accept_all", "pattern",
"organization", "description", "industry", "twitter", "facebook", "linkedin", "instagram", "youtube",
"technologies", "country", "state", "city", "postal_code", "street", "headcount", "company_type", "emails", "linked_domains",
},
"hunter_email": {
"id", "created_at", "updated_at", "deleted_at", "value", "type", "confidence", "sources", "first_name", "last_name",
"position", "position_raw", "seniority", "department", "linkedin", "twitter", "phone_number", "verification_date", "verification_status",
},
}
// Function to list available tables and their columns
func listAvailableTables() {
fmt.Println("Available tables and columns:")
// Prepare data for pretty.Table
headers := []string{"Table", "Columns"}
var tableRows [][]string
// Sort tables alphabetically for consistent output
var tableNames []string
for tableName := range availableTables {
tableNames = append(tableNames, tableName)
}
// Simple bubble sort for table names
for i := 0; i < len(tableNames)-1; i++ {
for j := 0; j < len(tableNames)-i-1; j++ {
if tableNames[j] > tableNames[j+1] {
tableNames[j], tableNames[j+1] = tableNames[j+1], tableNames[j]
}
}
}
// Create rows for the table
for _, tableName := range tableNames {
columns := availableTables[tableName]
// Format columns with line breaks for better readability
var formattedColumns string
for i := 0; i < len(columns); i += 5 {
end := i + 5
if end > len(columns) {
end = len(columns)
}
if i > 0 {
formattedColumns += "\n"
}
formattedColumns += strings.Join(columns[i:end], ", ")
}
tableRows = append(tableRows, []string{tableName, formattedColumns})
}
// Display the table
pretty.Table(headers, tableRows)
}
// Function to validate table name
func isValidTable(tableName string) bool {
_, exists := availableTables[tableName]
return exists
}
// Function to validate column names for a specific table
func validateColumns(tableName string, columns []string) []string {
if tableName == "" || columns == nil || len(columns) == 0 || columns[0] == "*" {
return nil
}
tableColumns, exists := availableTables[tableName]
if !exists {
return []string{fmt.Sprintf("Table '%s' does not exist", tableName)}
}
var invalidColumns []string
for _, col := range columns {
valid := false
for _, tableCol := range tableColumns {
if col == tableCol {
valid = true
break
}
}
if !valid {
invalidColumns = append(invalidColumns, col)
}
}
return invalidColumns
}
+312
View File
@@ -0,0 +1,312 @@
package cmd
import (
"crowsnest/internal/sqlite"
"fmt"
"github.com/spf13/cobra"
"go.uber.org/zap"
"os"
"strings"
)
func init() {
// Add targets command to root command
rootCmd.AddCommand(targetsCmd)
// Add flags specific to targets command
targetsCmd.Flags().StringVarP(&targetsOutputFile, "output", "o", "targets", "Output file name (required)")
targetsCmd.Flags().BoolVarP(&targetsExternal, "external", "e", false, "Output external format (email:password)")
targetsCmd.Flags().BoolVarP(&targetsInternal, "internal", "i", false, "Output internal format (username:password)")
targetsCmd.Flags().BoolVarP(&targetsSubdomains, "subdomains", "s", false, "Output subdomains")
targetsCmd.Flags().BoolVarP(&targetsEmails, "emails", "E", false, "Output emails only (no passwords)")
targetsCmd.Flags().StringVarP(&targetsDomain, "domain", "d", "", "Filter by domain (for emails and subdomains)")
// Add mutually exclusive flags to targets command
targetsCmd.MarkFlagsMutuallyExclusive("external", "internal", "subdomains", "emails")
}
var (
// Targets command flags
targetsOutputFile string
targetsExternal bool
targetsInternal bool
targetsSubdomains bool
targetsEmails bool
targetsDomain string
// Targets command
targetsCmd = &cobra.Command{
Use: "targets",
Short: "Export users and subdomains in formats suitable for external tools",
Long: `Export users and subdomains from the database in easily digestible formats for tools like sprays or other security testing tools.
Formats:
--external (-e): Output in email:password format
--internal (-i): Output in username:password format
--emails (-E): Output emails only (no passwords)
--subdomains (-s): Output subdomains only
Options:
--domain (-d): Filter results by domain (applies to emails and subdomains)
--output (-o): Specify output file name (required)
Examples:
# Export all external credentials (email:password)
crowsnest targets -e -o external_creds
# Export internal credentials for a specific domain
crowsnest targets -i -d example.com -o internal_creds
# Export all emails
crowsnest targets -E -o all_emails
# Export emails for a specific domain
crowsnest targets -E -d example.com -o domain_emails
# Export subdomains for a specific domain
crowsnest targets -s -d example.com -o subdomains
# Export all subdomains
crowsnest targets -s -o all_subdomains`,
Run: func(cmd *cobra.Command, args []string) {
// Validate that at least one format is specified
if !targetsExternal && !targetsInternal && !targetsSubdomains && !targetsEmails {
fmt.Println("[!] Error: You must specify at least one output format:")
fmt.Println(" --external (-e) for email:password format")
fmt.Println(" --internal (-i) for username:password format")
fmt.Println(" --emails (-E) for emails only")
fmt.Println(" --subdomains (-s) for subdomains")
return
}
if debugGlobal {
zap.L().Info("targets_debug",
zap.String("message", "targets command started"),
zap.Bool("external", targetsExternal),
zap.Bool("internal", targetsInternal),
zap.Bool("subdomains", targetsSubdomains),
zap.Bool("emails", targetsEmails),
zap.String("domain", targetsDomain),
zap.String("output_file", targetsOutputFile),
)
}
// Execute the targets export
err := executeTargetsExport()
if err != nil {
fmt.Printf("[!] Error: %v\n", err)
return
}
fmt.Printf("[+] Successfully exported targets to: %s\n", targetsOutputFile)
},
}
)
// executeTargetsExport performs the main logic for exporting targets
func executeTargetsExport() error {
var outputLines []string
// Export external credentials (email:password)
if targetsExternal {
if debugGlobal {
fmt.Println("[*] Exporting external credentials (email:password)...")
}
externalCreds, err := getExternalCredentials()
if err != nil {
return fmt.Errorf("failed to get external credentials: %v", err)
}
for _, cred := range externalCreds {
if cred.Email != "" && cred.Password != "" {
outputLines = append(outputLines, fmt.Sprintf("%s:%s", cred.Email, cred.Password))
}
}
if debugGlobal {
fmt.Printf("[*] Found %d external credentials\n", len(externalCreds))
}
}
// Export internal credentials (username:password)
if targetsInternal {
if debugGlobal {
fmt.Println("[*] Exporting internal credentials (username:password)...")
}
internalCreds, err := getInternalCredentials()
if err != nil {
return fmt.Errorf("failed to get internal credentials: %v", err)
}
for _, cred := range internalCreds {
if cred.Username != "" && cred.Password != "" {
outputLines = append(outputLines, fmt.Sprintf("%s:%s", cred.Username, cred.Password))
}
}
if debugGlobal {
fmt.Printf("[*] Found %d internal credentials\n", len(internalCreds))
}
}
// Export emails only
if targetsEmails {
if debugGlobal {
fmt.Println("[*] Exporting emails only...")
}
emails, err := getEmailsOnly()
if err != nil {
return fmt.Errorf("failed to get emails: %v", err)
}
for _, email := range emails {
if email.Email != "" {
outputLines = append(outputLines, email.Email)
}
}
if debugGlobal {
fmt.Printf("[*] Found %d emails\n", len(emails))
}
}
// Export subdomains
if targetsSubdomains {
if debugGlobal {
fmt.Println("[*] Exporting subdomains...")
}
subdomains, err := getSubdomains()
if err != nil {
return fmt.Errorf("failed to get subdomains: %v", err)
}
for _, subdomain := range subdomains {
if subdomain.Subdomain != "" {
outputLines = append(outputLines, subdomain.Subdomain)
}
}
if debugGlobal {
fmt.Printf("[*] Found %d subdomains\n", len(subdomains))
}
}
// Write to file
if len(outputLines) == 0 {
return fmt.Errorf("no data found to export")
}
// Join all lines with newlines and add a single newline at the end
content := strings.Join(outputLines, "\n") + "\n"
err := os.WriteFile(targetsOutputFile, []byte(content), 0644)
if err != nil {
return fmt.Errorf("failed to write to file: %v", err)
}
if debugGlobal {
fmt.Printf("[*] Wrote %d lines to %s\n", len(outputLines), targetsOutputFile)
}
return nil
}
// getExternalCredentials retrieves credentials for external format (email:password)
func getExternalCredentials() ([]sqlite.User, error) {
db := sqlite.GetDB()
var users []sqlite.User
query := db.Where("email IS NOT NULL AND email != '' AND password IS NOT NULL AND password != ''")
// Apply domain filter if specified
if targetsDomain != "" {
query = query.Where("email LIKE ?", "%@"+targetsDomain)
}
err := query.Find(&users).Error
if err != nil {
zap.L().Error("get_external_credentials",
zap.String("message", "failed to query external credentials"),
zap.Error(err),
)
return nil, err
}
return users, nil
}
// getInternalCredentials retrieves credentials for internal format (username:password)
func getInternalCredentials() ([]sqlite.User, error) {
db := sqlite.GetDB()
var users []sqlite.User
query := db.Where("username IS NOT NULL AND username != '' AND password IS NOT NULL AND password != ''")
// Apply domain filter if specified (filter usernames that might contain domain info)
if targetsDomain != "" {
query = query.Where("username LIKE ? OR email LIKE ?", "%"+targetsDomain+"%", "%@"+targetsDomain)
}
err := query.Find(&users).Error
if err != nil {
zap.L().Error("get_internal_credentials",
zap.String("message", "failed to query internal credentials"),
zap.Error(err),
)
return nil, err
}
return users, nil
}
// getEmailsOnly retrieves emails only (no passwords required)
func getEmailsOnly() ([]sqlite.User, error) {
db := sqlite.GetDB()
var users []sqlite.User
query := db.Where("email IS NOT NULL AND email != ''")
// Apply domain filter if specified
if targetsDomain != "" {
query = query.Where("email LIKE ?", "%@"+targetsDomain)
}
err := query.Find(&users).Error
if err != nil {
zap.L().Error("get_emails_only",
zap.String("message", "failed to query emails"),
zap.Error(err),
)
return nil, err
}
return users, nil
}
// getSubdomains retrieves subdomains from the database
func getSubdomains() ([]sqlite.Subdomain, error) {
db := sqlite.GetDB()
var subdomains []sqlite.Subdomain
query := db.Where("subdomain IS NOT NULL AND subdomain != ''")
// Apply domain filter if specified
if targetsDomain != "" {
query = query.Where("domain = ? OR subdomain LIKE ?", targetsDomain, "%."+targetsDomain)
}
err := query.Find(&subdomains).Error
if err != nil {
zap.L().Error("get_subdomains",
zap.String("message", "failed to query subdomains"),
zap.Error(err),
)
return nil, err
}
return subdomains, nil
}
+117 -73
View File
@@ -1,12 +1,12 @@
package cmd
import (
"dehasher/internal/debug"
"dehasher/internal/export"
"dehasher/internal/files"
"dehasher/internal/pretty"
"dehasher/internal/sqlite"
"dehasher/internal/whois"
"crowsnest/internal/debug"
"crowsnest/internal/export"
"crowsnest/internal/files"
"crowsnest/internal/pretty"
"crowsnest/internal/sqlite"
"crowsnest/internal/whois"
"fmt"
"github.com/spf13/cobra"
"go.uber.org/zap"
@@ -26,7 +26,7 @@ func init() {
whoisCmd.Flags().StringVarP(&whoisNSAddress, "ns", "n", "", "NS hostname for reverse NS lookup")
whoisCmd.Flags().StringVarP(&whoisInclude, "include", "I", "", "Up to 4 Terms to include in reverse WHOIS search (comma-separated)")
whoisCmd.Flags().StringVarP(&whoisExclude, "exclude", "E", "", "Up to 4 Terms to exclude in reverse WHOIS search (comma-separated)")
whoisCmd.Flags().StringVarP(&whoisReverseType, "type", "t", "registrant", "Type of reverse WHOIS search ([default] current or historic)")
whoisCmd.Flags().StringVarP(&whoisReverseType, "type", "t", "current", "Type of reverse WHOIS search ([default] current or historic)")
whoisCmd.Flags().StringVarP(&whoisOutputFormat, "format", "f", "text", "Output format (text, json)")
whoisCmd.Flags().StringVarP(&whoisOutputFile, "output", "o", "whois", "File to output results to including extension")
whoisCmd.Flags().BoolVarP(&whoisShowCredits, "credits", "c", false, "Show remaining WHOIS credits")
@@ -55,11 +55,11 @@ var (
Short: "Dehashed WHOIS lookups and reverse WHOIS searches",
Long: `Perform WHOIS lookups, history searches, reverse WHOIS searches, IP lookups, MX lookups, NS lookups, and subdomain scans.`,
Run: func(cmd *cobra.Command, args []string) {
key := getStoredApiKey()
key := getDehashedApiKey()
// Validate credentials
if key == "" {
fmt.Println("API key is required. Set the key with the \"set-key\" command. [dehasher set-key <api_key>]")
fmt.Println("API key is required. Set the key with the \"set-key\" command. [crowsnest set-key <api_key>]")
return
}
@@ -118,60 +118,63 @@ var (
if whoisDomain != "" {
fmt.Println("[*] Performing WHOIS lookup...")
// Domain lookup
result, err := w.WhoisSearch(whoisDomain)
if err != nil {
if debugGlobal {
debug.PrintInfo("failed to perform whois search")
debug.PrintError(err)
if !whoisHistory && !whoisSubdomainScan {
// Domain lookup
result, err := w.WhoisSearch(whoisDomain)
if err != nil {
if debugGlobal {
debug.PrintInfo("failed to perform whois search")
debug.PrintError(err)
}
zap.L().Error("whois_search",
zap.String("message", "failed to perform whois search"),
zap.Error(err),
)
fmt.Printf("Error performing WHOIS lookup: %v\n", err)
return
}
zap.L().Error("whois_search",
zap.String("message", "failed to perform whois search"),
zap.Error(err),
)
fmt.Printf("Error performing WHOIS lookup: %v\n", err)
return
}
if whoisShowCredits {
checkBalance(w)
}
// Fix the output format to use proper formatting
fmt.Println("WHOIS Lookup Result:")
// Store the record
err = sqlite.StoreWhoisRecord(result)
if err != nil {
if debugGlobal {
debug.PrintInfo("failed to store whois record")
debug.PrintError(err)
if whoisShowCredits {
checkBalance(w)
}
zap.L().Error("store_whois_record",
zap.String("message", "failed to store whois record"),
zap.Error(err),
)
fmt.Printf("Error storing WHOIS record: %v\n", err)
// Continue execution even if storage fails
}
// Pretty Print WhoIs Record
pretty.WhoIsTree(whoisDomain, result)
// Fix the output format to use proper formatting
fmt.Println("WHOIS Lookup Result:")
// Write WhoIs Record to file
if len(result.DomainName) != 0 {
fmt.Printf("[*] Writing WHOIS record to file: %s%s\n", whoisOutputFile, fType.Extension())
err = export.WriteWhoIsRecordToFile(result, whoisOutputFile, fType)
} else {
if debugGlobal {
debug.PrintInfo("no whois record to write to file")
// Store the record
err = sqlite.StoreWhoisRecord(result)
if err != nil {
if debugGlobal {
debug.PrintInfo("failed to store whois record")
debug.PrintError(err)
}
zap.L().Error("store_whois_record",
zap.String("message", "failed to store whois record"),
zap.Error(err),
)
fmt.Printf("Error storing WHOIS record: %v\n", err)
// Continue execution even if storage fails
}
// Pretty Print WhoIs Record
pretty.WhoIsTree(whoisDomain, result)
// Write WhoIs Record to file
if len(result.DomainName) != 0 {
fmt.Printf("[*] Writing WHOIS record to file: %s%s\n", whoisOutputFile, fType.Extension())
err = export.WriteWhoIsRecordToFile(result, whoisOutputFile, fType)
} else {
if debugGlobal {
debug.PrintInfo("no whois record to write to file")
}
zap.L().Info("write_whois_record",
zap.String("message", "no whois record to write to file"),
)
}
zap.L().Info("write_whois_record",
zap.String("message", "no whois record to write to file"),
)
}
if whoisHistory {
filename := whoisOutputFile + "_history"
fmt.Println("[*] Performing WHOIS history search...")
// Perform history search
historyRecords, err := w.WhoisHistory(whoisDomain)
@@ -194,7 +197,7 @@ var (
if len(historyRecords) > 0 {
fmt.Println("[*] Records Found: %d\n", len(historyRecords))
fmt.Println("[*] WHOIS History being written to file: %s%s\n", whoisOutputFile, fType.Extension())
writeErr := export.WriteWhoIsHistoryToFile(historyRecords, whoisOutputFile, fType)
writeErr := export.WriteWhoIsHistoryToFile(historyRecords, filename, fType)
if writeErr != nil {
if debugGlobal {
debug.PrintInfo("failed to write whois history to file")
@@ -207,7 +210,7 @@ var (
fmt.Printf("[!] Error writing WHOIS history to file: %v\n", writeErr)
}
err = sqlite.StoreHistoryRecord(historyRecords)
err = sqlite.StoreWhoisHistoryRecords(historyRecords)
if err != nil {
if debugGlobal {
debug.PrintInfo("failed to store history record")
@@ -233,9 +236,11 @@ var (
// Perform subdomain scan
if whoisSubdomainScan {
filename := whoisOutputFile + "_subdomains"
fmt.Println("[*] Performing WHOIS subdomain scan...")
subdomains, err := w.WhoisSubdomainScan(whoisDomain)
// Get credits
if whoisShowCredits {
checkBalance(w)
}
@@ -251,8 +256,13 @@ var (
)
fmt.Printf("Error performing subdomain scan: %v\n", err)
} else {
fmt.Println("Subdomain Scan:")
err = sqlite.StoreSubdomainRecords(subdomains)
// Store subdomains in subdomains table
var subs []sqlite.Subdomain
for _, s := range subdomains {
subs = append(subs, sqlite.Subdomain{Domain: whoisDomain, Subdomain: s.Domain})
}
err = sqlite.StoreSubdomains(subs)
if err != nil {
if debugGlobal {
debug.PrintInfo("failed to store subdomain record")
@@ -262,13 +272,13 @@ var (
zap.String("message", "failed to store subdomain record"),
zap.Error(err),
)
fmt.Printf("Error storing WHOIS subdomain record: %v\n", err)
fmt.Printf("Error storing subdomain record: %v\n", err)
}
// Write the subdomains to file if any
if len(subdomains) > 0 {
fmt.Printf("[*] Writing subdomains to file: %s%s\n", whoisOutputFile, fType.Extension())
err = export.WriteSubdomainsToFile(subdomains, whoisOutputFile, fType)
err = export.WriteSubdomainsToFile(subdomains, filename, fType)
if err != nil {
zap.L().Error("write_whois_subdomain",
zap.String("message", "failed to write whois subdomain to file"),
@@ -288,6 +298,7 @@ var (
}
// Store the subdomains
fmt.Println("Subdomain Scan:")
pretty.Table(headers, rows)
} else {
@@ -490,6 +501,12 @@ var (
}
if whoisInclude != "" || whoisExclude != "" {
if debugGlobal {
debug.PrintInfo("performing reverse whois")
debug.PrintInfo("include: " + whoisInclude)
debug.PrintInfo("exclude: " + whoisExclude)
debug.PrintInfo("reverse type: " + whoisReverseType)
}
// Reverse WHOIS
includeTerms := []string{}
if whoisInclude != "" {
@@ -509,17 +526,10 @@ var (
}
}
if whoisReverseType == "" {
if debugGlobal {
debug.PrintInfo("reverse type not specified, using default")
}
whoisReverseType = "current"
} else {
toLower := strings.ToLower(whoisReverseType)
if toLower != "current" && toLower != "historic" {
fmt.Println("[!] Error: Invalid reverse type. Must be 'current' or 'historic'.")
return
}
toLower := strings.ToLower(whoisReverseType)
if toLower != "current" && toLower != "historic" {
fmt.Println("[!] Error: Invalid reverse type. Must be 'current' or 'historic'.")
return
}
fmt.Println("[*] Performing reverse WHOIS lookup...")
@@ -536,8 +546,42 @@ var (
fmt.Printf("Error performing reverse WHOIS: %v\n", err)
return
}
fmt.Println("Reverse WHOIS Result:")
fmt.Println(result)
// Write to file
if len(result.DomainsList) > 0 {
fmt.Printf("[*] Writing reverse WHOIS results to file: %s%s\n", whoisOutputFile, fType.Extension())
err = export.WriteIStringToFile(result, whoisOutputFile, fType)
if err != nil {
if debugGlobal {
debug.PrintInfo("failed to write reverse whois to file")
debug.PrintError(err)
}
zap.L().Error("write_reverse_whois",
zap.String("message", "failed to write reverse whois to file"),
zap.Error(err),
)
fmt.Printf("Error writing reverse WHOIS to file: %v\n", err)
}
fmt.Println("Reverse WHOIS Result:")
fmt.Printf("Total Domains: %d\n", result.DomainsCount)
var (
headers = []string{"Domain"}
rows [][]string
)
for _, r := range result.DomainsList {
rows = append(rows, []string{r})
}
pretty.Table(headers, rows)
} else {
fmt.Println("[!] No results found")
zap.L().Info("reverse_whois",
zap.String("message", "no results found"),
)
}
if whoisShowCredits {
checkBalance(w)
+5 -5
View File
@@ -1,9 +1,9 @@
package main
import (
"dehasher/cmd"
"dehasher/internal/badger"
"dehasher/internal/sqlite"
"crowsnest/cmd"
"crowsnest/internal/badger"
"crowsnest/internal/sqlite"
"fmt"
"github.com/winking324/rzap"
"go.uber.org/zap"
@@ -21,7 +21,7 @@ var (
)
func init() {
basePath = filepath.Join(os.Getenv("HOME"), ".local", "share", "Dehasher")
basePath = filepath.Join(os.Getenv("HOME"), ".local", "share", "CrowsNest")
logPath = filepath.Join(basePath, "logs")
storePath = filepath.Join(basePath, "keystore")
// dbPath will be set in main() after badger is initialized
@@ -82,7 +82,7 @@ func main() {
useLocalDB := badger.GetUseLocalDB()
if useLocalDB {
// Use local database in current directory
dbPath = "./dehasher.sqlite"
dbPath = "./"
zap.L().Info("Using local database", zap.String("path", dbPath))
} else {
// Use default database path
+10 -5
View File
@@ -1,4 +1,4 @@
module dehasher
module crowsnest
go 1.23.0
@@ -7,7 +7,8 @@ toolchain go1.24.3
require (
github.com/charmbracelet/lipgloss v1.1.0
github.com/dgraph-io/badger/v4 v4.7.0
github.com/olekukonko/tablewriter v1.0.5
github.com/fatih/color v1.15.0
github.com/glebarez/sqlite v1.11.0
github.com/spf13/cobra v1.9.1
github.com/winking324/rzap v0.1.0
go.uber.org/zap v1.20.0
@@ -27,10 +28,11 @@ require (
github.com/charmbracelet/x/term v0.2.1 // indirect
github.com/dgraph-io/ristretto/v2 v2.2.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/fatih/color v1.15.0 // indirect
github.com/glebarez/go-sqlite v1.21.2 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/google/flatbuffers v25.2.10+incompatible // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
@@ -41,8 +43,7 @@ require (
github.com/mattn/go-runewidth v0.0.16 // indirect
github.com/mattn/go-sqlite3 v1.14.22 // indirect
github.com/muesli/termenv v0.16.0 // indirect
github.com/olekukonko/errors v0.0.0-20250405072817-4e6d85265da6 // indirect
github.com/olekukonko/ll v0.0.7 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/spf13/pflag v1.0.6 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
@@ -56,4 +57,8 @@ require (
golang.org/x/sys v0.31.0 // indirect
golang.org/x/text v0.23.0 // indirect
google.golang.org/protobuf v1.36.6 // indirect
modernc.org/libc v1.22.5 // indirect
modernc.org/mathutil v1.5.0 // indirect
modernc.org/memory v1.5.0 // indirect
modernc.org/sqlite v1.23.1 // indirect
)
+19 -6
View File
@@ -34,6 +34,10 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9gAXWo=
github.com/glebarez/go-sqlite v1.21.2/go.mod h1:sfxdZyhQjTM2Wry3gVYWaW072Ri1WMdWJi0k6+3382k=
github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw=
github.com/glebarez/sqlite v1.11.0/go.mod h1:h8/o8j5wiAsqSPoWELDUdJXhjAhsVliSn7bWZjOhrgQ=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
@@ -43,6 +47,10 @@ github.com/google/flatbuffers v25.2.10+incompatible h1:F3vclr7C3HpB1k9mxCGRMXq6F
github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ=
github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
@@ -71,16 +79,13 @@ github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
github.com/olekukonko/errors v0.0.0-20250405072817-4e6d85265da6 h1:r3FaAI0NZK3hSmtTDrBVREhKULp8oUeqLT5Eyl2mSPo=
github.com/olekukonko/errors v0.0.0-20250405072817-4e6d85265da6/go.mod h1:ppzxA5jBKcO1vIpCXQ9ZqgDh8iwODz6OXIGKU8r5m4Y=
github.com/olekukonko/ll v0.0.7 h1:K66xcUlG2qWRhPoLw/cidmbv4pDDJtZuvJGsR5QTzXo=
github.com/olekukonko/ll v0.0.7/go.mod h1:En+sEW0JNETl26+K8eZ6/W4UQ7CYSrrgg/EdIYT2H8g=
github.com/olekukonko/tablewriter v1.0.5 h1:8+uKJXxYcl29TcpfQdd0vL+l6Kul7Sk7sWolfgErDv0=
github.com/olekukonko/tablewriter v1.0.5/go.mod h1:Z22i2ywMkT9sw64nuWAUaH62kb+umiwucGaQNbFh8Bg=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
@@ -170,3 +175,11 @@ gorm.io/driver/sqlite v1.5.7 h1:8NvsrhP0ifM7LX9G4zPB97NwovUakUxc+2V2uuf3Z1I=
gorm.io/driver/sqlite v1.5.7/go.mod h1:U+J8craQU6Fzkcvu8oLeAQmi50TkwPEhHDEjQZXDah4=
gorm.io/gorm v1.26.1 h1:ghB2gUI9FkS46luZtn6DLZ0f6ooBJ5IbVej2ENFDjRw=
gorm.io/gorm v1.26.1/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE=
modernc.org/libc v1.22.5 h1:91BNch/e5B0uPbJFgqbxXuOnxBQjlS//icfQEGmvyjE=
modernc.org/libc v1.22.5/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY=
modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ=
modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds=
modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU=
modernc.org/sqlite v1.23.1 h1:nrSBg4aRQQwq59JpvGEQ15tNxoO5pX/kUjcRNwSAGQM=
modernc.org/sqlite v1.23.1/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk=
+40 -4
View File
@@ -45,7 +45,7 @@ func GetHardwareEntropy() []byte {
username,
osInfo,
// You could add a static salt here for additional security
"Dehasher-static-salt-value",
"CrowsNest-static-salt-value",
}, ":")
// Hash the fingerprint to get a 32-byte key
@@ -100,7 +100,7 @@ func Close() {
}
}
func GetKey() string {
func GetDehashedKey() string {
var apiKey string
err := db.View(func(txn *badger.Txn) error {
@@ -124,6 +124,29 @@ func GetKey() string {
return apiKey
}
func GetHunterKey() string {
var apiKey string
err := db.View(func(txn *badger.Txn) error {
item, err := txn.Get([]byte("cfg:hunter_api_key"))
if err != nil {
return err // could be ErrKeyNotFound
}
return item.Value(func(val []byte) error {
apiKey = string(val)
return nil
})
})
if err != nil {
zap.L().Error("get_hunter_api_key",
zap.String("message", "failed to get hunter_api_key"),
zap.Error(err),
)
}
return apiKey
}
func GetUseLocalDB() bool {
var useLocal bool
@@ -162,13 +185,26 @@ func GetUseLocalDB() bool {
return useLocal
}
func StoreKey(apiKey string) error {
func StoreDehashedKey(apiKey string) error {
err := db.Update(func(txn *badger.Txn) error {
return txn.Set([]byte("cfg:api_key"), []byte(apiKey))
})
if err != nil {
zap.L().Error("set_api_key",
zap.String("message", "failed to set api_key"),
zap.String("message", "failed to set dehashed api_key"),
zap.Error(err),
)
}
return err
}
func StoreHunterKey(apiKey string) error {
err := db.Update(func(txn *badger.Txn) error {
return txn.Set([]byte("cfg:hunter_api_key"), []byte(apiKey))
})
if err != nil {
zap.L().Error("set_api_key",
zap.String("message", "failed to set hunter api_key"),
zap.Error(err),
)
}
+7 -3
View File
@@ -2,9 +2,9 @@ package dehashed
import (
"bytes"
"crowsnest/internal/debug"
"crowsnest/internal/sqlite"
"crypto/sha256"
"dehasher/internal/debug"
"dehasher/internal/sqlite"
"encoding/hex"
"encoding/json"
"errors"
@@ -166,7 +166,11 @@ func (dcv2 *DehashedClientV2) Search(searchRequest DehashedSearchRequest) (int,
zap.String("message", "preparing search request"),
)
}
reqBody, _ := json.Marshal(searchRequest)
// Create a copy of the search request to avoid modifying the original
requestCopy := searchRequest
reqBody, _ := json.Marshal(requestCopy)
if dcv2.debug {
j := string(reqBody)
+87 -23
View File
@@ -1,13 +1,14 @@
package dehashed
import (
"dehasher/internal/debug"
"dehasher/internal/export"
"dehasher/internal/sqlite"
"encoding/json"
"crowsnest/internal/debug"
"crowsnest/internal/export"
"crowsnest/internal/pretty"
"crowsnest/internal/sqlite"
"fmt"
"go.uber.org/zap"
"os"
"strings"
)
// Dehasher is a struct for querying the Dehashed API
@@ -130,7 +131,7 @@ func (dh *Dehasher) Start() {
if len(dh.client.results) > 0 {
fmt.Printf(" [!] Partial results retrieved. Storing Results...\n")
err := sqlite.StoreResults(dh.client.GetResults())
err := sqlite.StoreDehashedResults(dh.client.GetResults())
if err != nil {
zap.L().Error("store_results",
zap.String("message", "failed to store results"),
@@ -208,13 +209,11 @@ func (dh *Dehasher) buildRequest() {
// parseResults parses the results and writes them to a file
func (dh *Dehasher) parseResults() {
var data []byte
zap.L().Info("extracting_credentials")
results := dh.client.GetResults()
creds := results.ExtractCredentials()
fmt.Printf("\n\t[+] Discovered %d Credentials", len(creds))
err := sqlite.StoreCreds(creds)
creds := results.ExtractUsers()
fmt.Printf(" [+] Discovered %d Credentials\n", len(creds))
err := sqlite.StoreUsers(creds)
if err != nil {
zap.L().Error("store_creds",
zap.String("message", "failed to store creds"),
@@ -224,7 +223,7 @@ func (dh *Dehasher) parseResults() {
zap.L().Info("creds_stored", zap.Int("count", len(creds)))
zap.L().Info("storing_results")
err = sqlite.StoreResults(results)
err = sqlite.StoreDehashedResults(results)
if err != nil {
zap.L().Error("store_results",
zap.String("message", "failed to store results"),
@@ -234,28 +233,93 @@ func (dh *Dehasher) parseResults() {
zap.L().Info("results_stored", zap.Int("count", len(results.Results)))
if len(results.Results) > 0 {
fmt.Printf("\n\t[*] Writing entries to file: %s.%s", dh.options.OutputFile, dh.options.OutputFormat.String())
var (
headers = []string{"Email", "Username", "Password"}
rows [][]string
)
fmt.Printf(" [*] Writing entries to file: %s.%s\n", dh.options.OutputFile, dh.options.OutputFormat.String())
if !dh.options.CredsOnly {
err := export.WriteToFile(results, dh.options.OutputFile, dh.options.OutputFormat)
if err != nil {
fmt.Printf("\n[!] Error Writing to file: %v\n\tOutputting to terminal.", err)
data, err = json.MarshalIndent(results, "", " ")
fmt.Println(string(data))
os.Exit(0)
fmt.Printf("[!] Error Writing to file: %v Outputting to terminal.\n", err)
zap.L().Error("write_results",
zap.String("message", "failed to write results to file"),
zap.Error(err),
)
} else {
fmt.Println("\n\t\t[*] Success\n")
fmt.Println(" [*] Success")
}
if dh.debug {
debug.PrintInfo("printing results table")
}
headers = []string{"Email", "Username", "Password", "Phone", "Company"}
if len(results.Results) > 50 {
fmt.Println(" [-] Large number of results recovered, displaying first 50...")
for i := 0; i < 50; i++ {
r := results.Results[i]
rows = append(rows, []string{
strings.Join(r.Email, ", "),
strings.Join(r.Username, ", "),
strings.Join(r.Password, ", "),
strings.Join(r.Phone, ", "),
strings.Join(r.Company, ", ")})
}
} else {
for _, r := range results.Results {
rows = append(rows, []string{
strings.Join(r.Email, ", "),
strings.Join(r.Username, ", "),
strings.Join(r.Password, ", "),
strings.Join(r.Phone, ", "),
strings.Join(r.Company, ", ")})
}
}
// Print Table
pretty.Table(headers, rows)
} else {
creds := results.ExtractCredentials()
if dh.debug {
debug.PrintInfo("extracting credentials")
}
creds := results.ExtractUsers()
if dh.debug {
debug.PrintInfo("writing credentials to file")
}
err := export.WriteCredsToFile(creds, dh.options.OutputFile, dh.options.OutputFormat)
if err != nil {
fmt.Printf("\n[!] Error Writing to file: %v\n\tOutputting to terminal.", err)
data, err = json.MarshalIndent(creds, "", " ")
fmt.Println(string(data))
os.Exit(0)
fmt.Printf("[!] Error Writing to file: %v\n Outputting to terminal.", err)
zap.L().Error("write_creds",
zap.String("message", "failed to write creds to file"),
zap.Error(err),
)
} else {
fmt.Println("\n\t\t[*] Success\n")
fmt.Println(" [*] Success")
}
if dh.debug {
debug.PrintInfo("printing credentials table")
}
headers = []string{"Email", "Username", "Password"}
if len(creds) > 50 {
fmt.Println(" [-] Large number of results recovered, displaying first 50...")
for i := 0; i < 50; i++ {
c := creds[i]
rows = append(rows, []string{c.Email, c.Username, c.Password})
}
} else {
for _, c := range creds {
rows = append(rows, []string{c.Email, c.Username, c.Password})
}
}
// Print Table
pretty.Table(headers, rows)
}
} else {
fmt.Println(" [-] No results found")
}
}
+1 -1
View File
@@ -1,7 +1,7 @@
package easyTime
import (
"dehasher/internal/debug"
"crowsnest/internal/debug"
"fmt"
"go.uber.org/zap"
"os"
@@ -1,8 +1,8 @@
package export
import (
"dehasher/internal/files"
"dehasher/internal/sqlite"
"crowsnest/internal/files"
"crowsnest/internal/sqlite"
"encoding/json"
"encoding/xml"
"errors"
@@ -14,7 +14,7 @@ import (
"time"
)
func WriteCredsToFile(creds []sqlite.Creds, outputFile string, fileType files.FileType) error {
func WriteCredsToFile(creds []sqlite.User, outputFile string, fileType files.FileType) error {
var data []byte
var err error
+36
View File
@@ -0,0 +1,36 @@
package export
import (
"crowsnest/internal/files"
"crowsnest/internal/sqlite"
"encoding/json"
"encoding/xml"
"fmt"
"gopkg.in/yaml.v3"
"os"
)
func WriteIStringToFile(iString sqlite.IString, outputFile string, fileType files.FileType) error {
var data []byte
var err error
switch fileType {
case files.JSON:
data, err = json.MarshalIndent(iString, "", " ")
case files.XML:
data, err = xml.MarshalIndent(iString, "", " ")
case files.YAML:
data, err = yaml.Marshal(iString)
case files.TEXT:
data = []byte(iString.String())
default:
return err
}
if err != nil {
return err
}
filePath := fmt.Sprintf("%s.%s", outputFile, fileType.String())
return os.WriteFile(filePath, data, 0644)
}
+696
View File
@@ -0,0 +1,696 @@
package hunter_io
import (
"crowsnest/internal/debug"
"crowsnest/internal/sqlite"
"encoding/json"
"fmt"
"go.uber.org/zap"
"io"
"net/http"
"strings"
)
const (
DOMAIN_SEARCH = "https://api.hunter.io/v2/domain-search?domain={{domain}}&api_key={{apikey}}"
EMAIL_FINDER = "https://api.hunter.io/v2/email-finder?domain={{domain}}&first_name={{first_name}}&last_name={{last_name}}&api_key={{apikey}}"
EMAIL_VERIFICATION = "https://api.hunter.io/v2/email-verifier?email={{email}}&api_key={{apikey}}"
COMPANY_ENRICHMENT = "https://api.hunter.io/v2/companies/find?domain={{domain}}&api_key={{apikey}}"
PERSON_ENRICHMENT = "https://api.hunter.io/v2/people/find?email={{email}}&api_key={{apikey}}"
COMBINED_ENRICHMENT = "https://api.hunter.io/v2/combined/find?email={{email}}&api_key={{apikey}}"
)
type HunterIO struct {
apiKey string
debug bool
}
func NewHunterIO(apiKey string, debugEnabled bool) *HunterIO {
return &HunterIO{apiKey: apiKey, debug: debugEnabled}
}
func (h *HunterIO) DomainSearch(domain string) (sqlite.HunterDomainData, error) {
var hunterDomainData sqlite.HunterDomainData
if h.debug {
debug.PrintInfo("performing domain search")
zap.L().Info("hunter_domain_search_debug",
zap.String("message", "performing domain search"),
)
}
url := DOMAIN_SEARCH
url = strings.Replace(url, "{{domain}}", domain, -1)
url = strings.Replace(url, "{{apikey}}", h.apiKey, -1)
if h.debug {
debug.PrintInfo("performing request")
debug.PrintInfo(fmt.Sprintf("URL: %s\n", url))
zap.L().Info("hunter_domain_search_debug",
zap.String("message", "performing request"),
zap.String("url", url),
)
}
resp, err := http.Get(url)
if err != nil {
if h.debug {
debug.PrintInfo("failed to perform request")
debug.PrintError(err)
}
zap.L().Error("hunter_domain_search",
zap.String("message", "failed to perform request"),
zap.Error(err),
)
return hunterDomainData, err
}
defer resp.Body.Close()
b, err := io.ReadAll(resp.Body)
if err != nil {
if h.debug {
debug.PrintInfo("failed to read response body")
debug.PrintError(err)
}
zap.L().Error("hunter_domain_search",
zap.String("message", "failed to read response body"),
zap.Error(err),
)
return hunterDomainData, err
}
if resp.StatusCode != 200 {
if h.debug {
debug.PrintInfo("received error status code")
debug.PrintJson(fmt.Sprintf("Status Code: %d\n", resp.StatusCode))
debug.PrintJson(fmt.Sprintf("Body: %s\n", string(b)))
zap.L().Info("hunter_domain_search_debug",
zap.String("message", "received error status code"),
zap.Int("status_code", resp.StatusCode),
zap.String("body_error", string(b)),
)
}
zap.L().Error("hunter_domain_search",
zap.String("message", "received error status code"),
zap.Int("status_code", resp.StatusCode),
zap.String("body_error", string(b)),
)
return hunterDomainData, fmt.Errorf("received error status code: %d", resp.StatusCode)
}
if h.debug {
debug.PrintInfo("unmarshalled response body")
debug.PrintJson(fmt.Sprintf("Body: %s\n", string(b)))
zap.L().Info("hunter_domain_search_debug",
zap.String("message", "unmarshalled response body"),
zap.String("body", string(b)),
)
}
var hunterDomainSearchResult sqlite.HunterDomainSearchResult
err = json.Unmarshal(b, &hunterDomainSearchResult)
if err != nil {
if h.debug {
debug.PrintInfo("failed to unmarshal response body")
debug.PrintError(err)
}
zap.L().Error("hunter_domain_search",
zap.String("message", "failed to unmarshal response body"),
zap.Error(err),
)
return hunterDomainData, err
}
hunterDomainData = hunterDomainSearchResult.Data
// Create a list of email object associated with the domain
var emails []sqlite.HunterEmail
for _, email := range hunterDomainData.Emails {
emails = append(emails, sqlite.HunterEmail{
Domain: domain,
Value: email.Value,
Type: email.Type,
Confidence: email.Confidence,
Sources: email.Sources,
FirstName: email.FirstName,
LastName: email.LastName,
Position: email.Position,
PositionRaw: email.PositionRaw,
Seniority: email.Seniority,
Department: email.Department,
Linkedin: email.Linkedin,
Twitter: email.Twitter,
PhoneNumber: email.PhoneNumber,
Verification: email.Verification,
})
}
err = sqlite.StoreHunterEmails(emails)
if err != nil {
if h.debug {
debug.PrintInfo("failed to store hunter emails")
debug.PrintError(err)
}
zap.L().Error("store_hunter_emails",
zap.String("message", "failed to store hunter emails"),
zap.Error(err),
)
return hunterDomainData, err
}
err = sqlite.StoreHunterDomain(hunterDomainData)
if err != nil {
if h.debug {
debug.PrintInfo("failed to store hunter domain")
debug.PrintError(err)
}
zap.L().Error("store_hunter_domain",
zap.String("message", "failed to store hunter domain"),
zap.Error(err),
)
return hunterDomainData, err
}
return hunterDomainData, nil
}
func (h *HunterIO) EmailFinder(domain, firstName, lastName string) (sqlite.HunterEmailFinderData, error) {
var hunterEmailFinderData sqlite.HunterEmailFinderData
if h.debug {
debug.PrintInfo("performing email find")
zap.L().Info("hunter_email_find_debug",
zap.String("message", "performing email find"),
)
}
url := EMAIL_FINDER
url = strings.Replace(url, "{{domain}}", domain, -1)
url = strings.Replace(url, "{{first_name}}", firstName, -1)
url = strings.Replace(url, "{{last_name}}", lastName, -1)
url = strings.Replace(url, "{{apikey}}", h.apiKey, -1)
if h.debug {
debug.PrintInfo("performing request")
debug.PrintInfo(fmt.Sprintf("URL: %s\n", url))
zap.L().Info("hunter_email_find_debug",
zap.String("message", "performing request"),
zap.String("url", url),
)
}
resp, err := http.Get(url)
if err != nil {
if h.debug {
debug.PrintInfo("failed to perform request")
debug.PrintError(err)
}
zap.L().Error("hunter_email_find",
zap.String("message", "failed to perform request"),
zap.Error(err),
)
return hunterEmailFinderData, err
}
defer resp.Body.Close()
b, err := io.ReadAll(resp.Body)
if err != nil {
if h.debug {
debug.PrintInfo("failed to read response body")
debug.PrintError(err)
}
zap.L().Error("hunter_email_find",
zap.String("message", "failed to read response body"),
zap.Error(err),
)
return hunterEmailFinderData, err
}
if resp.StatusCode != 200 {
if h.debug {
debug.PrintInfo("received error status code")
debug.PrintJson(fmt.Sprintf("Status Code: %d\n", resp.StatusCode))
debug.PrintJson(fmt.Sprintf("Body: %s\n", string(b)))
zap.L().Info("hunter_email_find_debug",
zap.String("message", "received error status code"),
zap.Int("status_code", resp.StatusCode),
zap.String("body_error", string(b)),
)
}
zap.L().Error("hunter_email_find",
zap.String("message", "received error status code"),
zap.Int("status_code", resp.StatusCode),
zap.String("body_error", string(b)),
)
return hunterEmailFinderData, fmt.Errorf("received error status code: %d", resp.StatusCode)
}
if h.debug {
debug.PrintInfo("unmarshalled response body")
debug.PrintJson(fmt.Sprintf("Body: %s\n", string(b)))
zap.L().Info("hunter_email_find_debug",
zap.String("message", "unmarshalled response body"),
zap.String("body", string(b)),
)
}
var hunterEmailFinderResult sqlite.HunterEmailFinderResponse
err = json.Unmarshal(b, &hunterEmailFinderResult)
if err != nil {
if h.debug {
debug.PrintInfo("failed to unmarshal response body")
debug.PrintError(err)
}
zap.L().Error("hunter_email_find",
zap.String("message", "failed to unmarshal response body"),
zap.Error(err),
)
return hunterEmailFinderData, err
}
hunterEmailFinderData = hunterEmailFinderResult.Data
var hunterEmails []sqlite.HunterEmail
hunterEmails = append(hunterEmails, sqlite.HunterEmail{
Domain: hunterEmailFinderData.Domain,
Value: hunterEmailFinderData.Email,
Type: "personal",
Confidence: 100,
Sources: hunterEmailFinderData.Sources,
FirstName: hunterEmailFinderData.FirstName,
LastName: hunterEmailFinderData.LastName,
Position: hunterEmailFinderData.Position,
PositionRaw: "",
Seniority: "",
Department: "",
Linkedin: hunterEmailFinderData.LinkedinURL,
Twitter: hunterEmailFinderData.Twitter,
PhoneNumber: hunterEmailFinderData.PhoneNumber,
Verification: hunterEmailFinderData.Verification,
})
err = sqlite.StoreHunterEmails(hunterEmails)
if err != nil {
if h.debug {
debug.PrintInfo("failed to store hunter email finder")
debug.PrintError(err)
}
zap.L().Error("store_hunter_email_finder",
zap.String("message", "failed to store hunter email finder"),
zap.Error(err),
)
return hunterEmailFinderData, err
}
return hunterEmailFinderData, nil
}
func (h *HunterIO) EmailVerification(email string) (sqlite.HunterEmailVerifyData, error) {
var hunterEmailVerifyData sqlite.HunterEmailVerifyData
if h.debug {
debug.PrintInfo("performing email verification")
zap.L().Info("hunter_email_verification_debug",
zap.String("message", "performing email verification"),
)
}
url := EMAIL_VERIFICATION
url = strings.Replace(url, "{{email}}", email, -1)
url = strings.Replace(url, "{{apikey}}", h.apiKey, -1)
if h.debug {
debug.PrintInfo("performing request")
debug.PrintInfo(fmt.Sprintf("URL: %s\n", url))
zap.L().Info("hunter_email_verification_debug",
zap.String("message", "performing request"),
zap.String("url", url),
)
}
resp, err := http.Get(url)
if err != nil {
if h.debug {
debug.PrintInfo("failed to perform request")
debug.PrintError(err)
}
zap.L().Error("hunter_email_verification",
zap.String("message", "failed to perform request"),
zap.Error(err),
)
return hunterEmailVerifyData, err
}
defer resp.Body.Close()
b, err := io.ReadAll(resp.Body)
if err != nil {
if h.debug {
debug.PrintInfo("failed to read response body")
debug.PrintError(err)
}
zap.L().Error("hunter_email_verification",
zap.String("message", "failed to read response body"),
zap.Error(err),
)
return hunterEmailVerifyData, err
}
if resp.StatusCode != 200 {
if h.debug {
debug.PrintInfo("received error status code")
debug.PrintJson(fmt.Sprintf("Status Code: %d\n", resp.StatusCode))
debug.PrintJson(fmt.Sprintf("Body: %s\n", string(b)))
zap.L().Info("hunter_email_verification_debug",
zap.String("message", "received error status code"),
zap.Int("status_code", resp.StatusCode),
zap.String("body_error", string(b)),
)
}
zap.L().Error("hunter_email_verification",
zap.String("message", "received error status code"),
zap.Int("status_code", resp.StatusCode),
zap.String("body_error", string(b)),
)
return hunterEmailVerifyData, fmt.Errorf("received error status code: %d", resp.StatusCode)
}
if h.debug {
debug.PrintInfo("unmarshalled response body")
debug.PrintJson(fmt.Sprintf("Body: %s\n", string(b)))
zap.L().Info("hunter_email_verification_debug",
zap.String("message", "unmarshalled response body"),
zap.String("body", string(b)),
)
}
var hunterEmailVerifyResult sqlite.HunterEmailVerifyResponse
err = json.Unmarshal(b, &hunterEmailVerifyResult)
if err != nil {
if h.debug {
debug.PrintInfo("failed to unmarshal response body")
debug.PrintError(err)
}
zap.L().Error("hunter_email_verification",
zap.String("message", "failed to unmarshal response body"),
zap.Error(err),
)
return hunterEmailVerifyData, err
}
hunterEmailVerifyData = hunterEmailVerifyResult.Data
return hunterEmailVerifyData, nil
}
func (h *HunterIO) CompanyEnrichment(domain string) (sqlite.CompanyData, error) {
var companyData sqlite.CompanyData
if h.debug {
debug.PrintInfo("performing company enrichment")
zap.L().Info("hunter_company_enrichment_debug",
zap.String("message", "performing company enrichment"),
)
}
url := COMPANY_ENRICHMENT
url = strings.Replace(url, "{{domain}}", domain, -1)
url = strings.Replace(url, "{{apikey}}", h.apiKey, -1)
if h.debug {
debug.PrintInfo("performing request")
debug.PrintInfo(fmt.Sprintf("URL: %s\n", url))
zap.L().Info("hunter_company_enrichment_debug",
zap.String("message", "performing request"),
zap.String("url", url),
)
}
resp, err := http.Get(url)
if err != nil {
if h.debug {
debug.PrintInfo("failed to perform request")
debug.PrintError(err)
}
zap.L().Error("hunter_company_enrichment",
zap.String("message", "failed to perform request"),
zap.Error(err),
)
return companyData, err
}
defer resp.Body.Close()
b, err := io.ReadAll(resp.Body)
if err != nil {
if h.debug {
debug.PrintInfo("failed to read response body")
debug.PrintError(err)
}
zap.L().Error("hunter_company_enrichment",
zap.String("message", "failed to read response body"),
zap.Error(err),
)
return companyData, err
}
if resp.StatusCode != 200 {
if h.debug {
debug.PrintInfo("received error status code")
debug.PrintJson(fmt.Sprintf("Status Code: %d\n", resp.StatusCode))
debug.PrintJson(fmt.Sprintf("Body: %s\n", string(b)))
zap.L().Info("hunter_company_enrichment_debug",
zap.String("message", "received error status code"),
zap.Int("status_code", resp.StatusCode),
zap.String("body_error", string(b)),
)
}
zap.L().Error("hunter_company_enrichment",
zap.String("message", "received error status code"),
zap.Int("status_code", resp.StatusCode),
zap.String("body_error", string(b)),
)
return companyData, fmt.Errorf("received error status code: %d", resp.StatusCode)
}
if h.debug {
debug.PrintInfo("unmarshalled response body")
debug.PrintJson(fmt.Sprintf("Body: %s\n", string(b)))
zap.L().Info("hunter_company_enrichment_debug",
zap.String("message", "unmarshalled response body"),
zap.String("body", string(b)),
)
}
var hunterCompanyEnrichmentResult sqlite.HunterCompanyEnrichmentResponse
err = json.Unmarshal(b, &hunterCompanyEnrichmentResult)
if err != nil {
if h.debug {
debug.PrintInfo("failed to unmarshal response body")
debug.PrintError(err)
}
zap.L().Error("hunter_company_enrichment",
zap.String("message", "failed to unmarshal response body"),
zap.Error(err),
)
return companyData, err
}
companyData = hunterCompanyEnrichmentResult.Data
return companyData, nil
}
func (h *HunterIO) PersonEnrichment(email string) (sqlite.PersonData, error) {
var personData sqlite.PersonData
if h.debug {
debug.PrintInfo("performing person enrichment")
zap.L().Info("hunter_person_enrichment_debug",
zap.String("message", "performing person enrichment"),
)
}
url := PERSON_ENRICHMENT
url = strings.Replace(url, "{{email}}", email, -1)
url = strings.Replace(url, "{{apikey}}", h.apiKey, -1)
if h.debug {
debug.PrintInfo("performing request")
debug.PrintInfo(fmt.Sprintf("URL: %s\n", url))
zap.L().Info("hunter_person_enrichment_debug",
zap.String("message", "performing request"),
zap.String("url", url),
)
}
resp, err := http.Get(url)
if err != nil {
if h.debug {
debug.PrintInfo("failed to perform request")
debug.PrintError(err)
}
zap.L().Error("hunter_person_enrichment",
zap.String("message", "failed to perform request"),
zap.Error(err),
)
return personData, err
}
defer resp.Body.Close()
b, err := io.ReadAll(resp.Body)
if err != nil {
if h.debug {
debug.PrintInfo("failed to read response body")
debug.PrintError(err)
}
zap.L().Error("hunter_person_enrichment",
zap.String("message", "failed to read response body"),
zap.Error(err),
)
return personData, err
}
if resp.StatusCode != 200 {
if h.debug {
debug.PrintInfo("received error status code")
debug.PrintJson(fmt.Sprintf("Status Code: %d\n", resp.StatusCode))
debug.PrintJson(fmt.Sprintf("Body: %s\n", string(b)))
zap.L().Info("hunter_person_enrichment_debug",
zap.String("message", "received error status code"),
zap.Int("status_code", resp.StatusCode),
zap.String("body_error", string(b)),
)
}
zap.L().Error("hunter_person_enrichment",
zap.String("message", "received error status code"),
zap.Int("status_code", resp.StatusCode),
zap.String("body_error", string(b)),
)
return personData, fmt.Errorf("received error status code: %d", resp.StatusCode)
}
if h.debug {
debug.PrintInfo("unmarshalled response body")
debug.PrintJson(fmt.Sprintf("Body: %s\n", string(b)))
zap.L().Info("hunter_person_enrichment_debug",
zap.String("message", "unmarshalled response body"),
zap.String("body", string(b)),
)
}
var hunterPersonEnrichmentResult sqlite.HunterPersonEnrichmentResponse
err = json.Unmarshal(b, &hunterPersonEnrichmentResult)
if err != nil {
if h.debug {
debug.PrintInfo("failed to unmarshal response body")
debug.PrintError(err)
}
zap.L().Error("hunter_person_enrichment",
zap.String("message", "failed to unmarshal response body"),
zap.Error(err),
)
return personData, err
}
personData = hunterPersonEnrichmentResult.Data
return personData, nil
}
func (h *HunterIO) CombinedEnrichment(email string) (sqlite.CombinedData, error) {
var combinedData sqlite.CombinedData
if h.debug {
debug.PrintInfo("performing combined enrichment")
zap.L().Info("hunter_combined_enrichment_debug",
zap.String("message", "performing combined enrichment"),
)
}
url := COMBINED_ENRICHMENT
url = strings.Replace(url, "{{email}}", email, -1)
url = strings.Replace(url, "{{apikey}}", h.apiKey, -1)
if h.debug {
debug.PrintInfo("performing request")
debug.PrintInfo(fmt.Sprintf("URL: %s\n", url))
zap.L().Info("hunter_combined_enrichment_debug",
zap.String("message", "performing request"),
zap.String("url", url),
)
}
resp, err := http.Get(url)
if err != nil {
if h.debug {
debug.PrintInfo("failed to perform request")
debug.PrintError(err)
}
zap.L().Error("hunter_combined_enrichment",
zap.String("message", "failed to perform request"),
zap.Error(err),
)
return combinedData, err
}
defer resp.Body.Close()
b, err := io.ReadAll(resp.Body)
if err != nil {
if h.debug {
debug.PrintInfo("failed to read response body")
debug.PrintError(err)
}
zap.L().Error("hunter_combined_enrichment",
zap.String("message", "failed to read response body"),
zap.Error(err),
)
return combinedData, err
}
if resp.StatusCode != 200 {
if h.debug {
debug.PrintInfo("received error status code")
debug.PrintJson(fmt.Sprintf("Status Code: %d\n", resp.StatusCode))
debug.PrintJson(fmt.Sprintf("Body: %s\n", string(b)))
zap.L().Info("hunter_combined_enrichment_debug",
zap.String("message", "received error status code"),
zap.Int("status_code", resp.StatusCode),
zap.String("body_error", string(b)),
)
}
zap.L().Error("hunter_combined_enrichment",
zap.String("message", "received error status code"),
zap.Int("status_code", resp.StatusCode),
zap.String("body_error", string(b)),
)
return combinedData, fmt.Errorf("received error status code: %d", resp.StatusCode)
}
if h.debug {
debug.PrintInfo("unmarshalled response body")
debug.PrintJson(fmt.Sprintf("Body: %s\n", string(b)))
zap.L().Info("hunter_combined_enrichment_debug",
zap.String("message", "unmarshalled response body"),
zap.String("body", string(b)),
)
}
var hunterCombinedEnrichmentResult sqlite.HunterCombinedEnrichmentResponse
err = json.Unmarshal(b, &hunterCombinedEnrichmentResult)
if err != nil {
if h.debug {
debug.PrintInfo("failed to unmarshal response body")
debug.PrintError(err)
}
zap.L().Error("hunter_combined_enrichment",
zap.String("message", "failed to unmarshal response body"),
zap.Error(err),
)
return combinedData, err
}
combinedData = hunterCombinedEnrichmentResult.Data
return combinedData, nil
}
+243 -2
View File
@@ -1,7 +1,7 @@
package pretty
import (
"dehasher/internal/sqlite"
"crowsnest/internal/sqlite"
"fmt"
"github.com/charmbracelet/lipgloss"
"github.com/charmbracelet/lipgloss/tree"
@@ -70,7 +70,7 @@ func WhoIsTree(root string, record sqlite.WhoisRecord) {
technicalContactTree.Child("Telephone: " + record.TechnicalContact.Telephone)
// Root Tree Children
rootTree.Child("Contact Email: " + record.ContactEmail)
rootTree.Child("Contact HunterEmail: " + record.ContactEmail)
rootTree.Child("Created Date: " + record.CreatedDate)
rootTree.Child("Created Date Normalized: " + record.CreatedDateNormalized)
rootTree.Child("Domain Name: " + record.DomainName)
@@ -102,3 +102,244 @@ func WhoIsTree(root string, record sqlite.WhoisRecord) {
// Print Tree
fmt.Println(rootTree)
}
func HunterDomainTree(root string, record sqlite.HunterDomainData) {
enumeratorStyle := lipgloss.NewStyle().Foreground(purple).MarginRight(1)
rootStyle := lipgloss.NewStyle().Foreground(lipgloss.Color("63"))
itemStyle := lipgloss.NewStyle().Foreground(gray)
rootTree := tree.Root(root)
// Root Tree Children
rootTree.Child("Domain: " + record.Domain)
rootTree.Child("Disposable: " + fmt.Sprintf("%t", record.Disposable))
rootTree.Child("Webmail: " + fmt.Sprintf("%t", record.Webmail))
rootTree.Child("Accept All: " + fmt.Sprintf("%t", record.AcceptAll))
rootTree.Child("Pattern: " + record.Pattern)
rootTree.Child("Organization: " + record.Organization)
rootTree.Child("Description: " + record.Description)
rootTree.Child("Industry: " + record.Industry)
rootTree.Child("Twitter: " + record.Twitter)
rootTree.Child("Facebook: " + record.Facebook)
rootTree.Child("Linkedin: " + record.Linkedin)
rootTree.Child("Instagram: " + record.Instagram)
rootTree.Child("Youtube: " + record.Youtube)
techTree := tree.Root("Technologies")
for _, tech := range record.Technologies {
techTree.Child(tech)
}
rootTree.Child(techTree)
rootTree.Child("Country: " + record.Country)
rootTree.Child("State: " + record.State)
rootTree.Child("City: " + record.City)
rootTree.Child("Postal Code: " + record.PostalCode)
rootTree.Child("Street: " + record.Street)
rootTree.Child("Headcount: " + record.Headcount)
rootTree.Child("Company Type: " + record.CompanyType)
emailTree := tree.Root("Emails")
for _, email := range record.Emails {
emailTree.Child(email.ToTree())
}
rootTree.Child(emailTree)
linkedDomainTree := tree.Root("Linked Domains")
for _, domain := range record.LinkedDomains {
linkedDomainTree.Child(domain)
}
rootTree.Child(linkedDomainTree)
// Styles
rootTree.Enumerator(tree.RoundedEnumerator)
rootTree.EnumeratorStyle(enumeratorStyle)
rootTree.RootStyle(rootStyle)
rootTree.ItemStyle(itemStyle)
// Print Tree
fmt.Println(rootTree)
}
func HunterCompanyEnrichmentTree(root string, record sqlite.CompanyData) {
enumeratorStyle := lipgloss.NewStyle().Foreground(purple).MarginRight(1)
rootStyle := lipgloss.NewStyle().Foreground(lipgloss.Color("63"))
itemStyle := lipgloss.NewStyle().Foreground(gray)
rootTree := tree.Root(root)
// Root Tree Children
rootTree.Child("ID: " + record.ID)
rootTree.Child("Name: " + record.Name)
rootTree.Child("Legal Name: " + record.LegalName)
rootTree.Child("Domain: " + record.Domain)
rootTree.Child(record.DomainAliasesTree())
rootTree.Child(record.SiteTree())
rootTree.Child(record.CategoryTree())
rootTree.Child(record.TagsTree())
rootTree.Child("Description: " + record.Description)
rootTree.Child("Founded Year: " + fmt.Sprintf("%d", record.FoundedYear))
rootTree.Child("Location: " + record.Location)
rootTree.Child("Time Zone: " + record.TimeZone)
rootTree.Child("UTC Offset: " + fmt.Sprintf("%d", record.UTCOffset))
rootTree.Child(record.GeoTree())
rootTree.Child("Logo: " + record.Logo)
rootTree.Child(record.FacebookTree())
rootTree.Child(record.LinkedInTree())
rootTree.Child(record.TwitterTree())
rootTree.Child(record.CrunchbaseTree())
rootTree.Child(record.YouTubeTree())
rootTree.Child("Email Provider: " + record.EmailProvider)
rootTree.Child("Type: " + record.Type)
rootTree.Child("Ticker: " + record.Ticker)
rootTree.Child(record.IdentifiersTree())
rootTree.Child("Phone: " + record.Phone)
rootTree.Child(record.MetricsTree())
rootTree.Child("Indexed At: " + record.IndexedAt)
rootTree.Child(record.TechTree())
rootTree.Child(record.TechCategoriesTree())
rootTree.Child(record.ParentTree())
rootTree.Child(record.UltimateParentTree())
// Styles
rootTree.Enumerator(tree.RoundedEnumerator)
rootTree.EnumeratorStyle(enumeratorStyle)
rootTree.RootStyle(rootStyle)
rootTree.ItemStyle(itemStyle)
// Print Tree
fmt.Println(rootTree)
}
func HunterPersonEnrichmentTree(root string, record sqlite.PersonData) {
enumeratorStyle := lipgloss.NewStyle().Foreground(purple).MarginRight(1)
rootStyle := lipgloss.NewStyle().Foreground(lipgloss.Color("63"))
itemStyle := lipgloss.NewStyle().Foreground(gray)
rootTree := tree.Root(root)
// Root Tree Children
rootTree.Child("ID: " + record.ID)
rootTree.Child(record.NameTree())
rootTree.Child("Email: " + record.Email)
rootTree.Child("Location: " + record.Location)
rootTree.Child("Time Zone: " + record.TimeZone)
rootTree.Child("UTC Offset: " + fmt.Sprintf("%d", record.UTCOffset))
rootTree.Child(record.GeoTree())
rootTree.Child("Bio: " + record.Bio)
rootTree.Child("Site: " + record.Site)
rootTree.Child("Avatar: " + record.Avatar)
rootTree.Child(record.EmploymentTree())
rootTree.Child(record.FacebookTree())
rootTree.Child(record.GitHubTree())
rootTree.Child(record.TwitterTree())
rootTree.Child(record.LinkedInTree())
rootTree.Child(record.GooglePlusTree())
rootTree.Child(record.GravatarTree())
rootTree.Child("Fuzzy: " + fmt.Sprintf("%t", record.Fuzzy))
rootTree.Child("Email Provider: " + record.EmailProvider)
rootTree.Child("Indexed At: " + record.IndexedAt)
rootTree.Child("Phone: " + record.Phone)
rootTree.Child("Active At: " + record.ActiveAt)
rootTree.Child("Inactive At: " + record.InactiveAt)
// Styles
rootTree.Enumerator(tree.RoundedEnumerator)
rootTree.EnumeratorStyle(enumeratorStyle)
rootTree.RootStyle(rootStyle)
rootTree.ItemStyle(itemStyle)
// Print Tree
fmt.Println(rootTree)
}
func HunterCombinedEnrichmentTree(root string, record sqlite.CombinedData) {
enumeratorStyle := lipgloss.NewStyle().Foreground(purple).MarginRight(1)
rootStyle := lipgloss.NewStyle().Foreground(lipgloss.Color("63"))
itemStyle := lipgloss.NewStyle().Foreground(gray)
rootTree := tree.Root(root)
// Root Tree Children
rootTree.Child(personTree(record.Person))
rootTree.Child(companyTree(record.Company))
// Styles
rootTree.Enumerator(tree.RoundedEnumerator)
rootTree.EnumeratorStyle(enumeratorStyle)
rootTree.RootStyle(rootStyle)
rootTree.ItemStyle(itemStyle)
// Print Tree
fmt.Println(rootTree)
}
func companyTree(record sqlite.CompanyData) *tree.Tree {
companyTree := tree.Root("Company")
// Company Tree Children
companyTree.Child("ID: " + record.ID)
companyTree.Child("Name: " + record.Name)
companyTree.Child("Legal Name: " + record.LegalName)
companyTree.Child("Domain: " + record.Domain)
companyTree.Child(record.DomainAliasesTree())
companyTree.Child(record.SiteTree())
companyTree.Child(record.CategoryTree())
companyTree.Child(record.TagsTree())
companyTree.Child("Description: " + record.Description)
companyTree.Child("Founded Year: " + fmt.Sprintf("%d", record.FoundedYear))
companyTree.Child("Location: " + record.Location)
companyTree.Child("Time Zone: " + record.TimeZone)
companyTree.Child("UTC Offset: " + fmt.Sprintf("%d", record.UTCOffset))
companyTree.Child(record.GeoTree())
companyTree.Child("Logo: " + record.Logo)
companyTree.Child(record.FacebookTree())
companyTree.Child(record.LinkedInTree())
companyTree.Child(record.TwitterTree())
companyTree.Child(record.CrunchbaseTree())
companyTree.Child(record.YouTubeTree())
companyTree.Child("Email Provider: " + record.EmailProvider)
companyTree.Child("Type: " + record.Type)
companyTree.Child("Ticker: " + record.Ticker)
companyTree.Child(record.IdentifiersTree())
companyTree.Child("Phone: " + record.Phone)
companyTree.Child(record.MetricsTree())
companyTree.Child("Indexed At: " + record.IndexedAt)
companyTree.Child(record.TechTree())
companyTree.Child(record.TechCategoriesTree())
companyTree.Child(record.ParentTree())
companyTree.Child(record.UltimateParentTree())
return companyTree
}
func personTree(record sqlite.PersonData) *tree.Tree {
personTree := tree.Root("Person")
// Person Tree Children
personTree.Child("ID: " + record.ID)
personTree.Child(record.NameTree())
personTree.Child("Email: " + record.Email)
personTree.Child("Location: " + record.Location)
personTree.Child("Time Zone: " + record.TimeZone)
personTree.Child("UTC Offset: " + fmt.Sprintf("%d", record.UTCOffset))
personTree.Child(record.GeoTree())
personTree.Child("Bio: " + record.Bio)
personTree.Child("Site: " + record.Site)
personTree.Child("Avatar: " + record.Avatar)
personTree.Child(record.EmploymentTree())
personTree.Child(record.FacebookTree())
personTree.Child(record.GitHubTree())
personTree.Child(record.TwitterTree())
personTree.Child(record.LinkedInTree())
personTree.Child(record.GooglePlusTree())
personTree.Child(record.GravatarTree())
personTree.Child("Fuzzy: " + fmt.Sprintf("%t", record.Fuzzy))
personTree.Child("Email Provider: " + record.EmailProvider)
personTree.Child("Indexed At: " + record.IndexedAt)
personTree.Child("Phone: " + record.Phone)
personTree.Child("Active At: " + record.ActiveAt)
personTree.Child("Inactive At: " + record.InactiveAt)
return personTree
}
+143
View File
@@ -0,0 +1,143 @@
package sqlite
import (
"fmt"
"go.uber.org/zap"
"os"
"path/filepath"
"strings"
sql "github.com/glebarez/sqlite"
"gorm.io/gorm"
"gorm.io/gorm/logger"
)
var DB *gorm.DB
// InitDB initializes the database connection
func InitDB(dbPath string) (*gorm.DB, error) {
zap.L().Info("Initializing database", zap.String("path", dbPath))
// Check if the path is a file or directory
fileInfo, err := os.Stat(dbPath)
var finalDbPath string
// If path doesn't exist or is a directory
if os.IsNotExist(err) || (err == nil && fileInfo.IsDir()) {
// Treat as directory path
if err := os.MkdirAll(dbPath, 0755); err != nil {
zap.L().Error("Failed to create database directory", zap.Error(err))
return nil, fmt.Errorf("failed to create database directory: %w", err)
}
finalDbPath = filepath.Join(dbPath, "crowsnest.sqlite")
} else {
// Treat as file path
// Ensure the directory exists
dir := filepath.Dir(dbPath)
if err := os.MkdirAll(dir, 0755); err != nil {
zap.L().Error("Failed to create parent directory for database", zap.Error(err))
return nil, fmt.Errorf("failed to create parent directory for database: %w", err)
}
finalDbPath = dbPath
}
zap.L().Info("Opening database", zap.String("finalPath", finalDbPath))
db, err := gorm.Open(sql.Open(finalDbPath), &gorm.Config{
Logger: logger.Default.LogMode(logger.Silent),
})
if err != nil {
zap.L().Error("Failed to connect to database", zap.Error(err))
return nil, fmt.Errorf("failed to connect to database: %w", err)
}
// Auto migrate your models
err = db.AutoMigrate(&Result{}, &User{}, &QueryOptions{}, &User{}, &WhoisRecord{}, &HistoryRecord{},
&LookupResult{}, &HunterDomainData{}, &HunterEmail{}, &PersonData{}, &Subdomain{})
if err != nil {
zap.L().Error("Failed to migrate database", zap.Error(err))
return nil, fmt.Errorf("failed to migrate database: %w", err)
}
DB = db
return db, nil
}
// GetDB returns the database connection
func GetDB() *gorm.DB {
if DB == nil {
zap.L().Error("database not initialized")
fmt.Println("sqlite database not initialized")
os.Exit(1)
}
return DB
}
type Table int64
const (
ResultsTable Table = iota
RunsTable
CredsTable
WhoIsTable
SubdomainsTable
HistoryTable
LookupTable
HunterDomainTable
HunterEmailTable
PersonTable
UnknownTable
)
func GetTable(userInput string) Table {
switch strings.ToLower(userInput) {
case "results":
return ResultsTable
case "runs":
return RunsTable
case "creds":
return CredsTable
case "whois":
return WhoIsTable
case "subdomains":
return SubdomainsTable
case "history":
return HistoryTable
case "lookup":
return LookupTable
case "hunter_domain":
return HunterDomainTable
case "hunter_email":
return HunterEmailTable
case "person":
return PersonTable
default:
return UnknownTable
}
}
func (t Table) Object() interface{} {
switch t {
case ResultsTable:
return Result{}
case RunsTable:
return QueryOptions{}
case CredsTable:
return User{}
case WhoIsTable:
return WhoisRecord{}
case SubdomainsTable:
return SubdomainRecord{}
case HistoryTable:
return HistoryRecord{}
case LookupTable:
return LookupResult{}
case HunterDomainTable:
return HunterDomainData{}
case HunterEmailTable:
return HunterEmail{}
case PersonTable:
return PersonData{}
default:
return nil
}
}
+33
View File
@@ -0,0 +1,33 @@
package sqlite
type IString interface {
String() string
}
type DBOptions struct {
Username string
Email string
IPAddress string
Password string
HashedPassword string
Name string
Vin string
LicensePlate string
Address string
Phone string
Social string
CryptoCurrencyAddress string
Domain string
Limit int
ExactMatch bool
NonEmptyFields []string // Fields that should not be empty
DisplayFields []string // Fields to display in output
}
func (o *DBOptions) Empty() bool {
return o.Username == "" && o.Email == "" && o.IPAddress == "" &&
o.Password == "" && o.HashedPassword == "" && o.Name == "" &&
o.Vin == "" && o.LicensePlate == "" && o.Address == "" &&
o.Phone == "" && o.Social == "" && o.CryptoCurrencyAddress == "" && o.Domain == "" &&
len(o.NonEmptyFields) == 0
}
+202
View File
@@ -0,0 +1,202 @@
package sqlite
import (
"crowsnest/internal/files"
"fmt"
"go.uber.org/zap"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
type QueryOptions struct {
gorm.Model
MaxRecords int `json:"max_records"`
MaxRequests int `json:"max_requests"`
StartingPage int `json:"starting_page"`
OutputFormat files.FileType `json:"output_format"`
OutputFile string `json:"output_file"`
RegexMatch bool `json:"regex_match"`
WildcardMatch bool `json:"wildcard_match"`
UsernameQuery string `json:"username_query"`
EmailQuery string `json:"email_query"`
IpQuery string `json:"ip_query"`
PassQuery string `json:"pass_query"`
HashQuery string `json:"hash_query"`
NameQuery string `json:"name_query"`
DomainQuery string `json:"domain_query"`
VinQuery string `json:"vin_query"`
LicensePlateQuery string `json:"license_plate_query"`
AddressQuery string `json:"address_query"`
PhoneQuery string `json:"phone_query"`
SocialQuery string `json:"social_query"`
CryptoAddressQuery string `json:"crypto_address_query"`
PrintBalance bool `json:"print_balance"`
CredsOnly bool `json:"creds_only"`
Debug bool `json:"debug"`
}
func (QueryOptions) TableName() string {
return "query_options"
}
func NewQueryOptions(maxRecords, maxRequests, startingPage int, outputFormat, outputFile, usernameQuery, emailQuery, ipQuery, passQuery, hashQuery, nameQuery, domainQuery, vinQuery, licensePlateQuery, addressQuery, phoneQuery, socialQuery, cryptoAddressQuery string, regexMatch, wildcardMatch, printBalance, credsOnly, debug bool) *QueryOptions {
return &QueryOptions{
MaxRecords: maxRecords,
MaxRequests: maxRequests,
StartingPage: startingPage,
OutputFormat: files.GetFileType(outputFormat),
OutputFile: outputFile,
PrintBalance: printBalance,
CredsOnly: credsOnly,
RegexMatch: regexMatch,
WildcardMatch: wildcardMatch,
UsernameQuery: usernameQuery,
EmailQuery: emailQuery,
IpQuery: ipQuery,
PassQuery: passQuery,
HashQuery: hashQuery,
NameQuery: nameQuery,
DomainQuery: domainQuery,
VinQuery: vinQuery,
LicensePlateQuery: licensePlateQuery,
AddressQuery: addressQuery,
PhoneQuery: phoneQuery,
SocialQuery: socialQuery,
CryptoAddressQuery: cryptoAddressQuery,
Debug: debug,
}
}
type DehashedSearchRequest struct {
Page int `json:"page"`
Query string `json:"query"`
Size int `json:"size"`
Wildcard bool `json:"wildcard"`
Regex bool `json:"regex"`
DeDupe bool `json:"de_dupe"`
}
type DehashedResponse struct {
Balance int `json:"balance"`
Entries []Result `json:"entries"`
Success bool `json:"success"`
Took string `json:"took"`
TotalResults int `json:"total"`
}
type Result struct {
gorm.Model
DehashedId string `json:"id" xml:"id" yaml:"id" gorm:"uniqueIndex"`
Email []string `json:"email,omitempty" xml:"email,omitempty" yaml:"email,omitempty" gorm:"serializer:json"`
IpAddress []string `json:"ip_address,omitempty" xml:"ip_address,omitempty" yaml:"ip_address,omitempty" gorm:"serializer:json"`
Username []string `json:"username,omitempty" xml:"username,omitempty" yaml:"username,omitempty" gorm:"serializer:json"`
Password []string `json:"password,omitempty" xml:"password,omitempty" yaml:"password,omitempty" gorm:"serializer:json"`
HashedPassword []string `json:"hashed_password,omitempty" xml:"hashed_password,omitempty" yaml:"hashed_password,omitempty" gorm:"serializer:json"`
HashType string `json:"hash_type,omitempty" xml:"hash_type,omitempty" yaml:"hash_type,omitempty"`
Name []string `json:"name,omitempty" xml:"name,omitempty" yaml:"name,omitempty" gorm:"serializer:json"`
Vin []string `json:"vin,omitempty" xml:"vin,omitempty" yaml:"vin,omitempty" gorm:"serializer:json"`
LicensePlate []string `json:"license_plate,omitempty" xml:"license_plate,omitempty" yaml:"license_plate,omitempty" gorm:"serializer:json"`
Url []string `json:"url,omitempty" xml:"url,omitempty" yaml:"url,omitempty" gorm:"serializer:json"`
Social []string `json:"social,omitempty" xml:"social,omitempty" yaml:"social,omitempty" gorm:"serializer:json"`
CryptoCurrencyAddress []string `json:"cryptocurrency_address,omitempty" xml:"cryptocurrency_address,omitempty" yaml:"cryptocurrency_address,omitempty" gorm:"serializer:json"`
Address []string `json:"address,omitempty" xml:"address,omitempty" yaml:"address,omitempty" gorm:"serializer:json"`
Phone []string `json:"phone,omitempty" xml:"phone,omitempty" yaml:"phone,omitempty" gorm:"serializer:json"`
Company []string `json:"company,omitempty" xml:"company,omitempty" yaml:"company,omitempty" gorm:"serializer:json"`
DatabaseName string `json:"database_name,omitempty" xml:"database_name,omitempty" yaml:"database_name,omitempty"`
}
func (Result) TableName() string {
return "dehashed"
}
type DehashedResults struct {
Results []Result `json:"results"`
}
func (dr *DehashedResults) ExtractUsers() []User {
var creds []User
results := dr.Results
for _, r := range results {
if len(r.Password) > 0 {
// Get first email if available
email := ""
if len(r.Email) > 0 {
email = r.Email[0]
}
// Get first username if available
username := ""
if len(r.Username) > 0 {
username = r.Username[0]
}
// Get first password
password := r.Password[0]
cred := User{Email: email, Password: password, Username: username}
creds = append(creds, cred)
}
}
go func() {
err := StoreUsers(creds)
if err != nil {
zap.L().Error("store_creds",
zap.String("message", "failed to store creds"),
zap.Error(err),
)
fmt.Printf("Error Storing Results: %v", err)
}
}()
return creds
}
func (User) TableName() string {
return "creds"
}
func (c User) ToString() string {
return fmt.Sprintf("%s%s%s", c.Username, "%", c.Password)
}
func StoreDehashedResults(results DehashedResults) error {
if len(results.Results) == 0 {
return nil
}
zap.L().Info("Storing results", zap.Int("count", len(results.Results)))
db := GetDB()
// Use batch insert with conflict handling
const batchSize = 100
var lastErr error
// Extract the slice of results
resultSlice := results.Results
for i := 0; i < len(resultSlice); i += batchSize {
end := i + batchSize
if end > len(resultSlice) {
end = len(resultSlice)
}
batch := resultSlice[i:end]
// Use Clauses with OnConflict DoNothing to skip conflicts
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
if err != nil {
zap.L().Warn("Error storing some results", zap.Error(err))
lastErr = err
// Continue with next batch despite error
}
}
return lastErr
}
func StoreDehashedQueryOptions(queryOptions *QueryOptions) error {
db := GetDB()
return db.Create(queryOptions).Error
}
-257
View File
@@ -1,257 +0,0 @@
package sqlite
import (
"fmt"
"go.uber.org/zap"
"gorm.io/gorm/clause"
"os"
"path/filepath"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
"gorm.io/gorm/logger"
)
var DB *gorm.DB
// InitDB initializes the database connection
func InitDB(dbPath string) (*gorm.DB, error) {
zap.L().Info("Initializing database", zap.String("path", dbPath))
// Check if the path is a file or directory
fileInfo, err := os.Stat(dbPath)
var finalDbPath string
// If path doesn't exist or is a directory
if os.IsNotExist(err) || (err == nil && fileInfo.IsDir()) {
// Treat as directory path
if err := os.MkdirAll(dbPath, 0755); err != nil {
zap.L().Error("Failed to create database directory", zap.Error(err))
return nil, fmt.Errorf("failed to create database directory: %w", err)
}
finalDbPath = filepath.Join(dbPath, "dehashed.sqlite")
} else {
// Treat as file path
// Ensure the directory exists
dir := filepath.Dir(dbPath)
if err := os.MkdirAll(dir, 0755); err != nil {
zap.L().Error("Failed to create parent directory for database", zap.Error(err))
return nil, fmt.Errorf("failed to create parent directory for database: %w", err)
}
finalDbPath = dbPath
}
zap.L().Info("Opening database", zap.String("finalPath", finalDbPath))
db, err := gorm.Open(sqlite.Open(finalDbPath), &gorm.Config{
Logger: logger.Default.LogMode(logger.Silent),
})
if err != nil {
zap.L().Error("Failed to connect to database", zap.Error(err))
return nil, fmt.Errorf("failed to connect to database: %w", err)
}
// Auto migrate your models
err = db.AutoMigrate(&Result{}, &Creds{}, &QueryOptions{}, &Creds{}, &WhoisRecord{}, &SubdomainRecord{}, &HistoryRecord{}, &LookupResult{})
if err != nil {
zap.L().Error("Failed to migrate database", zap.Error(err))
return nil, fmt.Errorf("failed to migrate database: %w", err)
}
DB = db
return db, nil
}
// GetDB returns the database connection
func GetDB() *gorm.DB {
if DB == nil {
zap.L().Error("database not initialized")
fmt.Println("sqlite database not initialized")
os.Exit(1)
}
return DB
}
func StoreResults(results DehashedResults) error {
if len(results.Results) == 0 {
return nil
}
zap.L().Info("Storing results", zap.Int("count", len(results.Results)))
db := GetDB()
// Use batch insert with conflict handling
const batchSize = 100
var lastErr error
// Extract the slice of results
resultSlice := results.Results
for i := 0; i < len(resultSlice); i += batchSize {
end := i + batchSize
if end > len(resultSlice) {
end = len(resultSlice)
}
batch := resultSlice[i:end]
// Use Clauses with OnConflict DoNothing to skip conflicts
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
if err != nil {
zap.L().Warn("Error storing some results", zap.Error(err))
lastErr = err
// Continue with next batch despite error
}
}
return lastErr
}
func StoreCreds(creds []Creds) error {
if len(creds) == 0 {
return nil
}
zap.L().Info("Storing credentials", zap.Int("count", len(creds)))
db := GetDB()
// Use batch insert with conflict handling
// This will insert records in batches and continue even if some fail
const batchSize = 100
var lastErr error
for i := 0; i < len(creds); i += batchSize {
end := i + batchSize
if end > len(creds) {
end = len(creds)
}
batch := creds[i:end]
// Use Clauses with OnConflict DoNothing to skip conflicts
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
if err != nil {
zap.L().Warn("Error storing some credentials", zap.Error(err))
lastErr = err
// Continue with next batch despite error
}
}
return lastErr
}
func StoreQueryOptions(queryOptions *QueryOptions) error {
db := GetDB()
return db.Create(queryOptions).Error
}
func StoreWhoisRecord(whoisRecord WhoisRecord) error {
// Create a pointer to the record to make it addressable
recordPtr := &whoisRecord
zap.L().Info("Storing WHOIS record",
zap.String("domain", whoisRecord.DomainName))
db := GetDB()
// Use OnConflict clause to handle duplicates
err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(recordPtr).Error
if err != nil {
zap.L().Error("store_whois_record",
zap.String("message", "failed to store whois record"),
zap.Error(err))
return err
}
return nil
}
func StoreSubdomainRecords(subdomainRecords []SubdomainRecord) error {
if len(subdomainRecords) == 0 {
return nil
}
zap.L().Info("Storing subdomain records", zap.Int("count", len(subdomainRecords)))
db := GetDB()
// Use batch insert with conflict handling
const batchSize = 100
var lastErr error
for i := 0; i < len(subdomainRecords); i += batchSize {
end := i + batchSize
if end > len(subdomainRecords) {
end = len(subdomainRecords)
}
batch := subdomainRecords[i:end]
// Use Clauses with OnConflict DoNothing to skip conflicts
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
if err != nil {
zap.L().Warn("Error storing some subdomain records", zap.Error(err))
lastErr = err
// Continue with next batch despite error
}
}
return lastErr
}
func StoreHistoryRecord(historyRecords []HistoryRecord) error {
if len(historyRecords) == 0 {
return nil
}
zap.L().Info("Storing history records", zap.Int("count", len(historyRecords)))
db := GetDB()
// Use batch insert with conflict handling
const batchSize = 100
var lastErr error
for i := 0; i < len(historyRecords); i += batchSize {
end := i + batchSize
if end > len(historyRecords) {
end = len(historyRecords)
}
batch := historyRecords[i:end]
// Use Clauses with OnConflict DoNothing to skip conflicts
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
if err != nil {
zap.L().Warn("Error storing some history records", zap.Error(err))
lastErr = err
// Continue with next batch despite error
}
}
return lastErr
}
func StoreIPLookup(ipLookup []LookupResult) error {
if len(ipLookup) == 0 {
return nil
}
zap.L().Info("Storing IP lookup records", zap.Int("count", len(ipLookup)))
db := GetDB()
// Use batch insert with conflict handling
const batchSize = 100
var lastErr error
for i := 0; i < len(ipLookup); i += batchSize {
end := i + batchSize
if end > len(ipLookup) {
end = len(ipLookup)
}
batch := ipLookup[i:end]
// Use Clauses with OnConflict DoNothing to skip conflicts
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
if err != nil {
zap.L().Warn("Error storing some IP lookup records", zap.Error(err))
lastErr = err
// Continue with next batch despite error
}
}
return lastErr
}
+820
View File
@@ -0,0 +1,820 @@
package sqlite
import (
"fmt"
"github.com/charmbracelet/lipgloss/tree"
"go.uber.org/zap"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
// HunterDomainSearchResult represents the response from Hunter.io domain search API
type HunterDomainSearchResult struct {
Data HunterDomainData `json:"data" gorm:"embedded;embeddedPrefix:data_"`
Meta HunterMeta `json:"meta" gorm:"embedded;embeddedPrefix:meta_"`
}
// HunterDomainData contains the main domain information
type HunterDomainData struct {
gorm.Model
IString `gorm:"-"`
Domain string `json:"domain" gorm:"unique"`
Disposable bool `json:"disposable"`
Webmail bool `json:"webmail"`
AcceptAll bool `json:"accept_all"`
Pattern string `json:"pattern"`
Organization string `json:"organization"`
Description string `json:"description"`
Industry string `json:"industry"`
Twitter string `json:"twitter"`
Facebook string `json:"facebook"`
Linkedin string `json:"linkedin"`
Instagram string `json:"instagram"`
Youtube string `json:"youtube"`
Technologies []string `json:"technologies" gorm:"serializer:json"`
Country string `json:"country"`
State string `json:"state"`
City string `json:"city"`
PostalCode string `json:"postal_code"`
Street string `json:"street"`
Headcount string `json:"headcount"`
CompanyType string `json:"company_type"`
Emails []HunterEmail `json:"emails" gorm:"serializer:json"`
LinkedDomains []string `json:"linked_domains" gorm:"serializer:json"`
}
func (h HunterDomainData) String() string {
return fmt.Sprintf("Domain: %s\nDisposable: %t\nWebmail: %t\nAcceptAll: %t\nPattern: %s\nOrganization: %s\nDescription: %s\nIndustry: %s\nTwitter: %s\nFacebook: %s\nLinkedin: %s\nInstagram: %s\nYoutube: %s\nTechnologies: %v\nCountry: %s\nState: %s\nCity: %s\nPostalCode: %s\nStreet: %s\nHeadcount: %s\nCompanyType: %s\nEmails: %v\nLinkedDomains: %v\n",
h.Domain, h.Disposable, h.Webmail, h.AcceptAll, h.Pattern, h.Organization, h.Description, h.Industry, h.Twitter, h.Facebook, h.Linkedin, h.Instagram, h.Youtube, h.Technologies, h.Country, h.State, h.City, h.PostalCode, h.Street, h.Headcount, h.CompanyType, h.Emails, h.LinkedDomains)
}
func (HunterDomainData) TableName() string {
return "hunter_domain"
}
// HunterEmail represents an email found for the domain
type HunterEmail struct {
gorm.Model
Domain string `json:"domain,omitempty"`
Value string `json:"value" gorm:"unique"`
Type string `json:"type"`
Confidence int `json:"confidence"`
Sources []HunterSource `json:"sources" gorm:"serializer:json"`
FirstName string `json:"first_name"`
LastName string `json:"last_name"`
Position string `json:"position"`
PositionRaw string `json:"position_raw"`
Seniority string `json:"seniority"`
Department string `json:"department"`
Linkedin string `json:"linkedin"`
Twitter string `json:"twitter"`
PhoneNumber string `json:"phone_number"`
Verification HunterVerification `json:"verification" gorm:"embedded;embeddedPrefix:verification_"`
}
func (he *HunterEmail) ToTree() *tree.Tree {
emailTree := tree.Root(he.Value)
emailTree.Child("Type: " + he.Type)
emailTree.Child("Confidence: " + fmt.Sprintf("%d", he.Confidence))
emailTree.Child("FirstName: " + he.FirstName)
emailTree.Child("LastName: " + he.LastName)
emailTree.Child("Position: " + he.Position)
emailTree.Child("PositionRaw: " + he.PositionRaw)
emailTree.Child("Seniority: " + he.Seniority)
emailTree.Child("Department: " + he.Department)
emailTree.Child("Linkedin: " + he.Linkedin)
emailTree.Child("Twitter: " + he.Twitter)
emailTree.Child("PhoneNumber: " + he.PhoneNumber)
emailTree.Child(he.Verification.ToTree())
return emailTree
}
func (he *HunterEmail) String() string {
return fmt.Sprintf("Value: %s\nType: %s\nConfidence: %d\nSources: %v\nFirstName: %s\nLastName: %s\nPosition: %s\nPositionRaw: %s\nSeniority: %s\nDepartment: %s\nLinkedin: %s\nTwitter: %s\nPhoneNumber: %s\nVerification: %v\n",
he.Value, he.Type, he.Confidence, he.Sources, he.FirstName, he.LastName, he.Position, he.PositionRaw, he.Seniority, he.Department, he.Linkedin, he.Twitter, he.PhoneNumber, he.Verification)
}
func (HunterEmail) TableName() string {
return "hunter_email"
}
// HunterSource represents where an email was found
type HunterSource struct {
Domain string `json:"domain"`
URI string `json:"uri"`
ExtractedOn string `json:"extracted_on"`
LastSeenOn string `json:"last_seen_on"`
StillOnPage bool `json:"still_on_page"`
}
// HunterVerification represents the verification status of an email
type HunterVerification struct {
Date string `json:"date"`
Status string `json:"status"`
}
func (hv *HunterVerification) ToTree() *tree.Tree {
verificationTree := tree.Root("Verification")
verificationTree.Child("Date: " + hv.Date)
verificationTree.Child("Status: " + hv.Status)
return verificationTree
}
// HunterMeta contains metadata about the API response
type HunterMeta struct {
Results int `json:"results"`
Limit int `json:"limit"`
Offset int `json:"offset"`
Params HunterSearchParams `json:"params" gorm:"embedded;embeddedPrefix:params_"`
}
// HunterSearchParams contains the parameters used in the search
type HunterSearchParams struct {
Domain string `json:"domain"`
Company string `json:"company"`
Type string `json:"type"`
Seniority string `json:"seniority"`
Department string `json:"department"`
}
// HunterEmailFinderResponse represents the response from Hunter.io email finder API
type HunterEmailFinderResponse struct {
Data HunterEmailFinderData `json:"data" gorm:"embedded;embeddedPrefix:data_"`
Meta EmailFinderMeta `json:"meta" gorm:"embedded;embeddedPrefix:meta_"`
}
// HunterEmailFinderData contains the main email information
type HunterEmailFinderData struct {
IString
FirstName string `json:"first_name"`
LastName string `json:"last_name"`
Email string `json:"email"`
Score int `json:"score"`
Domain string `json:"domain"`
AcceptAll bool `json:"accept_all"`
Position string `json:"position"`
Twitter string `json:"twitter"`
LinkedinURL string `json:"linkedin_url"`
PhoneNumber string `json:"phone_number"`
Company string `json:"company"`
Sources []HunterSource `json:"sources" gorm:"serializer:json"`
Verification HunterVerification `json:"verification" gorm:"embedded;embeddedPrefix:verification_"`
}
func (he HunterEmailFinderData) String() string {
return fmt.Sprintf("FirstName: %s\nLastName: %s\nEmail: %s\nScore: %d\nDomain: %s\nAcceptAll: %t\nPosition: %s\nTwitter: %s\nLinkedinURL: %s\nPhoneNumber: %s\nCompany: %s\nSources: %v\nVerification: %v\n",
he.FirstName, he.LastName, he.Email, he.Score, he.Domain, he.AcceptAll, he.Position, he.Twitter, he.LinkedinURL, he.PhoneNumber, he.Company, he.Sources, he.Verification)
}
// EmailFinderMeta contains metadata about the API response
type EmailFinderMeta struct {
Params EmailFinderParams `json:"params" gorm:"embedded;embeddedPrefix:params_"`
}
// EmailFinderParams contains the parameters used in the search
type EmailFinderParams struct {
FirstName string `json:"first_name"`
LastName string `json:"last_name"`
FullName string `json:"full_name"`
Domain string `json:"domain"`
Company string `json:"company"`
MaxDuration string `json:"max_duration"`
}
func (HunterEmailFinderResponse) TableName() string {
return "hunter_email_finder"
}
// HunterEmailVerifyResponse represents the response from Hunter.io email verification API
type HunterEmailVerifyResponse struct {
Data HunterEmailVerifyData `json:"data" gorm:"embedded;embeddedPrefix:data_"`
Meta EmailVerifyMeta `json:"meta" gorm:"embedded;embeddedPrefix:meta_"`
}
// HunterEmailVerifyData contains the email verification information
type HunterEmailVerifyData struct {
IString
Status string `json:"status"`
Result string `json:"result"`
DeprecationNotice string `json:"_deprecation_notice"`
Score int `json:"score"`
Email string `json:"email"`
Regexp bool `json:"regexp"`
Gibberish bool `json:"gibberish"`
Disposable bool `json:"disposable"`
Webmail bool `json:"webmail"`
MXRecords bool `json:"mx_records"`
SMTPServer bool `json:"smtp_server"`
SMTPCheck bool `json:"smtp_check"`
AcceptAll bool `json:"accept_all"`
Block bool `json:"block"`
Sources []HunterSource `json:"sources" gorm:"serializer:json"`
}
func (ev HunterEmailVerifyData) String() string {
return fmt.Sprintf("Status: %s\nResult: %s\nDeprecationNotice: %s\nScore: %d\nEmail: %s\nRegexp: %t\nGibberish: %t\nDisposable: %t\nWebmail: %t\nMXRecords: %t\nSMTPServer: %t\nSMTPCheck: %t\nAcceptAll: %t\nBlock: %t\nSources: %v\n",
ev.Status, ev.Result, ev.DeprecationNotice, ev.Score, ev.Email, ev.Regexp, ev.Gibberish, ev.Disposable, ev.Webmail, ev.MXRecords, ev.SMTPServer, ev.SMTPCheck, ev.AcceptAll, ev.Block, ev.Sources)
}
// EmailVerifyMeta contains metadata about the API response
type EmailVerifyMeta struct {
Params EmailVerifyParams `json:"params" gorm:"embedded;embeddedPrefix:params_"`
}
// EmailVerifyParams contains the parameters used in the verification
type EmailVerifyParams struct {
Email string `json:"email"`
}
// HunterCompanyEnrichmentResponse represents the response from Hunter.io company enrichment API
type HunterCompanyEnrichmentResponse struct {
Data CompanyData `json:"data" gorm:"embedded;embeddedPrefix:data_"`
Meta CompanyMeta `json:"meta" gorm:"embedded;embeddedPrefix:meta_"`
}
// CompanyData contains the detailed company information
type CompanyData struct {
IString
ID string `json:"id"`
Name string `json:"name"`
LegalName string `json:"legalName"`
Domain string `json:"domain"`
DomainAliases []string `json:"domainAliases" gorm:"serializer:json"`
Site CompanySite `json:"site" gorm:"embedded;embeddedPrefix:site_"`
Category Category `json:"category" gorm:"embedded;embeddedPrefix:category_"`
Tags []string `json:"tags" gorm:"serializer:json"`
Description string `json:"description"`
FoundedYear int `json:"foundedYear"`
Location string `json:"location"`
TimeZone string `json:"timeZone"`
UTCOffset int `json:"utcOffset"`
Geo Geography `json:"geo" gorm:"embedded;embeddedPrefix:geo_"`
Logo string `json:"logo"`
Facebook Facebook `json:"facebook" gorm:"embedded;embeddedPrefix:facebook_"`
LinkedIn LinkedIn `json:"linkedin" gorm:"embedded;embeddedPrefix:linkedin_"`
Twitter Twitter `json:"twitter" gorm:"embedded;embeddedPrefix:twitter_"`
Crunchbase Crunchbase `json:"crunchbase" gorm:"embedded;embeddedPrefix:crunchbase_"`
YouTube YouTube `json:"youtube" gorm:"embedded;embeddedPrefix:youtube_"`
EmailProvider string `json:"emailProvider"`
Type string `json:"type"`
Ticker string `json:"ticker"`
Identifiers Identifiers `json:"identifiers" gorm:"embedded;embeddedPrefix:identifiers_"`
Phone string `json:"phone"`
Metrics Metrics `json:"metrics" gorm:"embedded;embeddedPrefix:metrics_"`
IndexedAt string `json:"indexedAt"`
Tech []string `json:"tech" gorm:"serializer:json"`
TechCategories []string `json:"techCategories" gorm:"serializer:json"`
Parent ParentCompany `json:"parent" gorm:"embedded;embeddedPrefix:parent_"`
UltimateParent ParentCompany `json:"ultimateParent" gorm:"embedded;embeddedPrefix:ultimate_parent_"`
}
func (cd CompanyData) String() string {
return fmt.Sprintf("ID: %s\nName: %s\nLegalName: %s\nDomain: %s\nDomainAliases: %v\nSite: %v\nCategory: %v\nTags: %v\nDescription: %s\nFoundedYear: %d\nLocation: %s\nTimeZone: %s\nUTCOffset: %d\nGeo: %v\nLogo: %s\nFacebook: %v\nLinkedIn: %v\nTwitter: %v\nCrunchbase: %v\nYouTube: %v\nEmailProvider: %s\nType: %s\nTicker: %s\nIdentifiers: %v\nPhone: %s\nMetrics: %v\nIndexedAt: %s\nTech: %v\nTechCategories: %v\nParent: %v\nUltimateParent: %v\n",
cd.ID, cd.Name, cd.LegalName, cd.Domain, cd.DomainAliases, cd.Site, cd.Category, cd.Tags, cd.Description, cd.FoundedYear, cd.Location, cd.TimeZone, cd.UTCOffset, cd.Geo, cd.Logo, cd.Facebook, cd.LinkedIn, cd.Twitter, cd.Crunchbase, cd.YouTube, cd.EmailProvider, cd.Type, cd.Ticker, cd.Identifiers, cd.Phone, cd.Metrics, cd.IndexedAt, cd.Tech, cd.TechCategories, cd.Parent, cd.UltimateParent)
}
func (cd *CompanyData) DomainAliasesTree() *tree.Tree {
domainAliasesTree := tree.Root("Domain Aliases")
for _, domainAlias := range cd.DomainAliases {
domainAliasesTree.Child(domainAlias)
}
return domainAliasesTree
}
func (cd *CompanyData) SiteTree() *tree.Tree {
siteTree := tree.Root("Site")
phoneTree := tree.Root("Phone Numbers")
for _, phoneNumber := range cd.Site.PhoneNumbers {
phoneTree.Child(phoneNumber)
}
emailTree := tree.Root("Email Addresses")
for _, emailAddress := range cd.Site.EmailAddresses {
emailTree.Child(emailAddress)
}
siteTree.Child(phoneTree)
siteTree.Child(emailTree)
return siteTree
}
func (cd *CompanyData) CategoryTree() *tree.Tree {
categoryTree := tree.Root("Category")
categoryTree.Child("Sector: " + cd.Category.Sector)
categoryTree.Child("Industry Group: " + cd.Category.IndustryGroup)
categoryTree.Child("Industry: " + cd.Category.Industry)
categoryTree.Child("Sub Industry: " + cd.Category.SubIndustry)
categoryTree.Child("GICS Code: " + cd.Category.GICSCode)
categoryTree.Child("SIC Code: " + cd.Category.SICCode)
sic4CodesTree := tree.Root("SIC 4 Codes")
for _, sic4Code := range cd.Category.SIC4Codes {
sic4CodesTree.Child(sic4Code)
}
categoryTree.Child(sic4CodesTree)
categoryTree.Child("NAICS Code: " + cd.Category.NAICSCode)
naics6CodesTree := tree.Root("NAICS 6 Codes")
for _, naics6Code := range cd.Category.NAICS6Codes {
naics6CodesTree.Child(naics6Code)
}
categoryTree.Child(naics6CodesTree)
naics6Codes2022Tree := tree.Root("NAICS 6 Codes 2022")
for _, naics6Code2022 := range cd.Category.NAICS6Codes2022 {
naics6Codes2022Tree.Child(naics6Code2022)
}
categoryTree.Child(naics6Codes2022Tree)
return categoryTree
}
func (cd *CompanyData) GeoTree() *tree.Tree {
geoTree := tree.Root("Geo")
geoTree.Child("Street Number: " + cd.Geo.StreetNumber)
geoTree.Child("Street Name: " + cd.Geo.StreetName)
geoTree.Child("Sub Premise: " + cd.Geo.SubPremise)
geoTree.Child("Street Address: " + cd.Geo.StreetAddress)
geoTree.Child("City: " + cd.Geo.City)
geoTree.Child("Postal Code: " + cd.Geo.PostalCode)
geoTree.Child("State: " + cd.Geo.State)
geoTree.Child("State Code: " + cd.Geo.StateCode)
geoTree.Child("Country: " + cd.Geo.Country)
geoTree.Child("Country Code: " + cd.Geo.CountryCode)
geoTree.Child("Latitude: " + fmt.Sprintf("%f", cd.Geo.Lat))
geoTree.Child("Longitude: " + fmt.Sprintf("%f", cd.Geo.Lng))
return geoTree
}
func (cd *CompanyData) FacebookTree() *tree.Tree {
facebookTree := tree.Root("Facebook")
facebookTree.Child("Handle: " + cd.Facebook.Handle)
facebookTree.Child("Likes: " + fmt.Sprintf("%d", cd.Facebook.Likes))
return facebookTree
}
func (cd *CompanyData) LinkedInTree() *tree.Tree {
linkedinTree := tree.Root("LinkedIn")
linkedinTree.Child("Handle: " + cd.LinkedIn.Handle)
return linkedinTree
}
func (cd *CompanyData) TwitterTree() *tree.Tree {
twitterTree := tree.Root("Twitter")
twitterTree.Child("Handle: " + cd.Twitter.Handle)
twitterTree.Child("ID: " + cd.Twitter.ID)
twitterTree.Child("Bio: " + cd.Twitter.Bio)
twitterTree.Child("Followers: " + fmt.Sprintf("%d", cd.Twitter.Followers))
twitterTree.Child("Following: " + fmt.Sprintf("%d", cd.Twitter.Following))
twitterTree.Child("Location: " + cd.Twitter.Location)
twitterTree.Child("Site: " + cd.Twitter.Site)
twitterTree.Child("Avatar" + cd.Twitter.Avatar)
return twitterTree
}
func (cd *CompanyData) CrunchbaseTree() *tree.Tree {
crunchbaseTree := tree.Root("Crunchbase")
crunchbaseTree.Child("Handle: " + cd.Crunchbase.Handle)
return crunchbaseTree
}
func (cd *CompanyData) YouTubeTree() *tree.Tree {
youtubeTree := tree.Root("YouTube")
youtubeTree.Child("Handle: " + cd.YouTube.Handle)
return youtubeTree
}
func (cd *CompanyData) IdentifiersTree() *tree.Tree {
identifiersTree := tree.Root("Identifiers")
identifiersTree.Child("UsEIN: " + cd.Identifiers.UsEIN)
return identifiersTree
}
func (cd *CompanyData) MetricsTree() *tree.Tree {
metricsTree := tree.Root("Metrics")
metricsTree.Child("Alexa Us Rank: " + fmt.Sprintf("%d", cd.Metrics.AlexaUsRank))
metricsTree.Child("Alexa Global Rank: " + fmt.Sprintf("%d", cd.Metrics.AlexaGlobalRank))
metricsTree.Child("Traffic Rank: " + cd.Metrics.TrafficRank)
metricsTree.Child("Employees: " + cd.Metrics.Employees)
metricsTree.Child("Market Cap: " + cd.Metrics.MarketCap)
metricsTree.Child("Raised: " + cd.Metrics.Raised)
metricsTree.Child("Annual Revenue: " + cd.Metrics.AnnualRevenue)
metricsTree.Child("Estimated Annual Revenue: " + cd.Metrics.EstimatedAnnualRevenue)
metricsTree.Child("Fiscal Year End: " + cd.Metrics.FiscalYearEnd)
return metricsTree
}
func (cd *CompanyData) TagsTree() *tree.Tree {
tagsTree := tree.Root("Tags")
for _, tag := range cd.Tags {
tagsTree.Child(tag)
}
return tagsTree
}
func (cd *CompanyData) TechTree() *tree.Tree {
techTree := tree.Root("Tech")
for _, tech := range cd.Tech {
techTree.Child(tech)
}
return techTree
}
func (cd *CompanyData) TechCategoriesTree() *tree.Tree {
techCategoriesTree := tree.Root("Tech Categories")
for _, techCategory := range cd.TechCategories {
techCategoriesTree.Child(techCategory)
}
return techCategoriesTree
}
func (cd *CompanyData) ParentTree() *tree.Tree {
parentTree := tree.Root("Parent")
parentTree.Child("Domain: " + cd.Parent.Domain)
return parentTree
}
func (cd *CompanyData) UltimateParentTree() *tree.Tree {
ultimateParentTree := tree.Root("Ultimate Parent")
ultimateParentTree.Child("Domain: " + cd.UltimateParent.Domain)
return ultimateParentTree
}
// CompanySite contains contact information from the company website
type CompanySite struct {
PhoneNumbers []string `json:"phoneNumbers" gorm:"serializer:json"`
EmailAddresses []string `json:"emailAddresses" gorm:"serializer:json"`
}
// Category contains industry classification information
type Category struct {
Sector string `json:"sector"`
IndustryGroup string `json:"industryGroup"`
Industry string `json:"industry"`
SubIndustry string `json:"subIndustry"`
GICSCode string `json:"gicsCode"`
SICCode string `json:"sicCode"`
SIC4Codes []string `json:"sic4Codes" gorm:"serializer:json"`
NAICSCode string `json:"naicsCode"`
NAICS6Codes []string `json:"naics6Codes" gorm:"serializer:json"`
NAICS6Codes2022 []string `json:"naics6Codes2022" gorm:"serializer:json"`
}
// Geography contains location information
type Geography struct {
StreetNumber string `json:"streetNumber"`
StreetName string `json:"streetName"`
SubPremise string `json:"subPremise"`
StreetAddress string `json:"streetAddress"`
City string `json:"city"`
PostalCode string `json:"postalCode"`
State string `json:"state"`
StateCode string `json:"stateCode"`
Country string `json:"country"`
CountryCode string `json:"countryCode"`
Lat float64 `json:"lat"`
Lng float64 `json:"lng"`
}
// Identifiers contains company identification numbers
type Identifiers struct {
UsEIN string `json:"usEIN"`
}
// Metrics contains company performance metrics
type Metrics struct {
AlexaUsRank int `json:"alexaUsRank"`
AlexaGlobalRank int `json:"alexaGlobalRank"`
TrafficRank string `json:"trafficRank"`
Employees string `json:"employees"`
MarketCap string `json:"marketCap"`
Raised string `json:"raised"`
AnnualRevenue string `json:"annualRevenue"`
EstimatedAnnualRevenue string `json:"estimatedAnnualRevenue"`
FiscalYearEnd string `json:"fiscalYearEnd"`
}
// ParentCompany contains information about parent companies
type ParentCompany struct {
Domain string `json:"domain"`
}
// CompanyMeta contains metadata about the API response
type CompanyMeta struct {
Domain string `json:"domain"`
}
func (HunterCompanyEnrichmentResponse) TableName() string {
return "hunter_company_enrichment"
}
// HunterPersonEnrichmentResponse represents the response from Hunter.io person enrichment API
type HunterPersonEnrichmentResponse struct {
Data PersonData `json:"data" gorm:"embedded;embeddedPrefix:data_"`
Meta PersonMeta `json:"meta" gorm:"embedded;embeddedPrefix:meta_"`
}
// PersonData contains the detailed person information
type PersonData struct {
gorm.Model
IString `gorm:"-"`
ID string `json:"id"`
Name PersonName `json:"name" gorm:"embedded;embeddedPrefix:name_"`
Email string `json:"email" gorm:"unique"`
Location string `json:"location"`
TimeZone string `json:"timeZone"`
UTCOffset int `json:"utcOffset"`
Geo PersonGeo `json:"geo" gorm:"embedded;embeddedPrefix:geo_"`
Bio string `json:"bio"`
Site string `json:"site"`
Avatar string `json:"avatar"`
Employment Employment `json:"employment" gorm:"embedded;embeddedPrefix:employment_"`
Facebook Facebook `json:"facebook" gorm:"embedded;embeddedPrefix:facebook_"`
GitHub GitHub `json:"github" gorm:"embedded;embeddedPrefix:github_"`
Twitter Twitter `json:"twitter" gorm:"embedded;embeddedPrefix:twitter_"`
LinkedIn LinkedIn `json:"linkedin" gorm:"embedded;embeddedPrefix:linkedin_"`
GooglePlus GooglePlus `json:"googleplus" gorm:"embedded;embeddedPrefix:googleplus_"`
Gravatar Gravatar `json:"gravatar" gorm:"embedded;embeddedPrefix:gravatar_"`
Fuzzy bool `json:"fuzzy"`
EmailProvider string `json:"emailProvider"`
IndexedAt string `json:"indexedAt"`
Phone string `json:"phone"`
ActiveAt string `json:"activeAt"`
InactiveAt string `json:"inactiveAt"`
}
func (pd PersonData) String() string {
return fmt.Sprintf("ID: %s\nName: %v\nEmail: %s\nLocation: %s\nTimeZone: %s\nUTCOffset: %d\nGeo: %v\nBio: %s\nSite: %s\nAvatar: %s\nEmployment: %v\nFacebook: %v\nGitHub: %v\nTwitter: %v\nLinkedIn: %v\nGooglePlus: %v\nGravatar: %v\nFuzzy: %t\nEmailProvider: %s\nIndexedAt: %s\nPhone: %s\nActiveAt: %s\nInactiveAt: %s\n",
pd.ID, pd.Name, pd.Email, pd.Location, pd.TimeZone, pd.UTCOffset, pd.Geo, pd.Bio, pd.Site, pd.Avatar, pd.Employment, pd.Facebook, pd.GitHub, pd.Twitter, pd.LinkedIn, pd.GooglePlus, pd.Gravatar, pd.Fuzzy, pd.EmailProvider, pd.IndexedAt, pd.Phone, pd.ActiveAt, pd.InactiveAt)
}
func (pd *PersonData) NameTree() *tree.Tree {
nameTree := tree.Root("Name")
nameTree.Child("Full Name: " + pd.Name.FullName)
nameTree.Child("Given Name: " + pd.Name.GivenName)
nameTree.Child("Family Name: " + pd.Name.FamilyName)
return nameTree
}
func (pd *PersonData) GeoTree() *tree.Tree {
geoTree := tree.Root("Geo")
geoTree.Child("City: " + pd.Geo.City)
geoTree.Child("State: " + pd.Geo.State)
geoTree.Child("State Code: " + pd.Geo.StateCode)
geoTree.Child("Country: " + pd.Geo.Country)
geoTree.Child("Country Code: " + pd.Geo.CountryCode)
geoTree.Child("Latitude: " + fmt.Sprintf("%f", pd.Geo.Lat))
geoTree.Child("Longitude: " + fmt.Sprintf("%f", pd.Geo.Lng))
return geoTree
}
func (pd *PersonData) EmploymentTree() *tree.Tree {
employmentTree := tree.Root("Employment")
employmentTree.Child("Domain: " + pd.Employment.Domain)
employmentTree.Child("Name: " + pd.Employment.Name)
employmentTree.Child("Title: " + pd.Employment.Title)
employmentTree.Child("Role: " + pd.Employment.Role)
employmentTree.Child("Sub Role: " + pd.Employment.SubRole)
employmentTree.Child("Seniority: " + pd.Employment.Seniority)
return employmentTree
}
func (pd *PersonData) FacebookTree() *tree.Tree {
facebookTree := tree.Root("Facebook")
facebookTree.Child("Handle: " + pd.Facebook.Handle)
facebookTree.Child("Likes: " + fmt.Sprintf("%d", pd.Facebook.Likes))
return facebookTree
}
func (pd *PersonData) GitHubTree() *tree.Tree {
githubTree := tree.Root("GitHub")
githubTree.Child("Handle: " + pd.GitHub.Handle)
githubTree.Child("ID: " + pd.GitHub.ID)
githubTree.Child("Avatar: " + pd.GitHub.Avatar)
githubTree.Child("Company: " + pd.GitHub.Company)
githubTree.Child("Blog: " + pd.GitHub.Blog)
githubTree.Child("Followers: " + fmt.Sprintf("%d", pd.GitHub.Followers))
githubTree.Child("Following: " + fmt.Sprintf("%d", pd.GitHub.Following))
return githubTree
}
func (pd *PersonData) TwitterTree() *tree.Tree {
twitterTree := tree.Root("Twitter")
twitterTree.Child("Handle: " + pd.Twitter.Handle)
twitterTree.Child("ID: " + pd.Twitter.ID)
twitterTree.Child("Bio: " + pd.Twitter.Bio)
twitterTree.Child("Followers: " + fmt.Sprintf("%d", pd.Twitter.Followers))
twitterTree.Child("Following: " + fmt.Sprintf("%d", pd.Twitter.Following))
twitterTree.Child("Location: " + pd.Twitter.Location)
twitterTree.Child("Site: " + pd.Twitter.Site)
twitterTree.Child("Avatar: " + pd.Twitter.Avatar)
return twitterTree
}
func (pd *PersonData) LinkedInTree() *tree.Tree {
linkedinTree := tree.Root("LinkedIn")
linkedinTree.Child("Handle: " + pd.LinkedIn.Handle)
return linkedinTree
}
func (pd *PersonData) GooglePlusTree() *tree.Tree {
googlePlusTree := tree.Root("GooglePlus")
googlePlusTree.Child("Handle: " + pd.GooglePlus.Handle)
return googlePlusTree
}
func (pd *PersonData) GravatarTree() *tree.Tree {
gravatarTree := tree.Root("Gravatar")
gravatarTree.Child("Handle: " + pd.Gravatar.Handle)
gravatarTree.Child("Avatar: " + pd.Gravatar.Avatar)
return gravatarTree
}
func (PersonData) TableName() string {
return "person"
}
// PersonName contains the person's name components
type PersonName struct {
FullName string `json:"fullName"`
GivenName string `json:"givenName"`
FamilyName string `json:"familyName"`
}
// PersonGeo contains location information for a person
type PersonGeo struct {
City string `json:"city"`
State string `json:"state"`
StateCode string `json:"stateCode"`
Country string `json:"country"`
CountryCode string `json:"countryCode"`
Lat float64 `json:"lat"`
Lng float64 `json:"lng"`
}
// Employment contains employment information
type Employment struct {
Domain string `json:"domain"`
Name string `json:"name"`
Title string `json:"title"`
Role string `json:"role"`
SubRole string `json:"subRole"`
Seniority string `json:"seniority"`
}
// GitHub contains GitHub profile information
type GitHub struct {
gorm.Model
Handle string `json:"handle"`
ID string `json:"id"`
Avatar string `json:"avatar"`
Company string `json:"company"`
Blog string `json:"blog"`
Followers int `json:"followers"`
Following int `json:"following"`
PersonID int `json:"person_id,omitempty"`
}
// GooglePlus contains Google+ profile information
type GooglePlus struct {
Handle string `json:"handle"`
PersonID int `json:"person_id,omitempty"`
}
// Gravatar contains Gravatar profile information
type Gravatar struct {
gorm.Model
Handle string `json:"handle"`
URLs []string `json:"urls" gorm:"serializer:json"`
Avatar string `json:"avatar"`
Avatars []string `json:"avatars" gorm:"serializer:json"`
PersonID int `json:"person_id,omitempty"`
}
// Facebook contains Facebook profile information
type Facebook struct {
Handle string `json:"handle"`
Likes int `json:"likes"`
}
// LinkedIn contains LinkedIn profile information
type LinkedIn struct {
Handle string `json:"handle"`
}
// Twitter contains Twitter profile information
type Twitter struct {
Handle string `json:"handle"`
ID string `json:"id"`
Bio string `json:"bio"`
Followers int `json:"followers"`
Following int `json:"following"`
Location string `json:"location"`
Site string `json:"site"`
Avatar string `json:"avatar"`
}
// Crunchbase contains Crunchbase profile information
type Crunchbase struct {
Handle string `json:"handle"`
}
// YouTube contains YouTube profile information
type YouTube struct {
Handle string `json:"handle"`
}
// PersonMeta contains metadata about the API response
type PersonMeta struct {
Email string `json:"email"`
}
// HunterCombinedEnrichmentResponse represents the response from Hunter.io combined enrichment API
type HunterCombinedEnrichmentResponse struct {
Data CombinedData `json:"data" gorm:"embedded;embeddedPrefix:data_"`
Meta CombinedMeta `json:"meta" gorm:"embedded;embeddedPrefix:meta_"`
}
// CombinedData contains both person and company information
type CombinedData struct {
IString
Person PersonData `json:"person" gorm:"embedded;embeddedPrefix:person_"`
Company CompanyData `json:"company" gorm:"embedded;embeddedPrefix:company_"`
}
func (cbd CombinedData) String() string {
return fmt.Sprintf("Person: %s\nCompany: %s",
cbd.Person.String(),
cbd.Company.String())
}
// CombinedMeta contains metadata about the API response
type CombinedMeta struct {
Email string `json:"email"`
}
// String returns a string representation of the combined enrichment response
func (c *HunterCombinedEnrichmentResponse) String() string {
return fmt.Sprintf("Person:\n%s\n\nCompany:\n%s",
c.Data.Person.String(),
c.Data.Company.String())
}
func StoreHunterDomain(hunterDomain HunterDomainData) error {
db := GetDB()
// Use OnConflict clause to handle duplicates
err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(&hunterDomain).Error
if err != nil {
zap.L().Error("store_hunter_domain",
zap.String("message", "failed to store hunter domain"),
zap.Error(err))
return err
}
return nil
}
func StoreHunterEmails(hunterEmails []HunterEmail) error {
if len(hunterEmails) == 0 {
return nil
}
zap.L().Info("Storing hunter emails", zap.Int("count", len(hunterEmails)))
db := GetDB()
// Use batch insert with conflict handling
const batchSize = 100
var lastErr error
for i := 0; i < len(hunterEmails); i += batchSize {
end := i + batchSize
if end > len(hunterEmails) {
end = len(hunterEmails)
}
batch := hunterEmails[i:end]
// Use Clauses with OnConflict DoNothing to skip conflicts
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
if err != nil {
zap.L().Warn("Error storing some hunter emails", zap.Error(err))
lastErr = err
// Continue with next batch despite error
}
}
return lastErr
}
func StoreHunterPersonData(personData PersonData) error {
db := GetDB()
// Use OnConflict clause to handle duplicates
err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(&personData).Error
if err != nil {
zap.L().Error("store_person_data",
zap.String("message", "failed to store person data"),
zap.Error(err))
return err
}
return nil
}
+1
View File
@@ -0,0 +1 @@
package sqlite
-20
View File
@@ -1,20 +0,0 @@
package sqlite
type DehashedSearchRequest struct {
Page int `json:"page"`
Query string `json:"query"`
Size int `json:"size"`
Wildcard bool `json:"wildcard"`
Regex bool `json:"regex"`
DeDupe bool `json:"de_dupe"`
}
func NewDehashedSearchRequest(size int, wildcard, regex bool) *DehashedSearchRequest {
return &DehashedSearchRequest{
Page: 0,
Size: size,
Wildcard: false,
Regex: false,
DeDupe: true,
}
}
-94
View File
@@ -1,94 +0,0 @@
package sqlite
import (
"encoding/json"
"fmt"
"go.uber.org/zap"
"gorm.io/gorm"
"io"
"os"
)
type DehashedResponse struct {
Balance int `json:"balance"`
Entries []Result `json:"entries"`
Success bool `json:"success"`
Took string `json:"took"`
TotalResults int `json:"total"`
}
type Result struct {
gorm.Model
DehashedId string `json:"id" xml:"id" yaml:"id" gorm:"uniqueIndex"`
Email []string `json:"email,omitempty" xml:"email,omitempty" yaml:"email,omitempty" gorm:"serializer:json"`
IpAddress []string `json:"ip_address,omitempty" xml:"ip_address,omitempty" yaml:"ip_address,omitempty" gorm:"serializer:json"`
Username []string `json:"username,omitempty" xml:"username,omitempty" yaml:"username,omitempty" gorm:"serializer:json"`
Password []string `json:"password,omitempty" xml:"password,omitempty" yaml:"password,omitempty" gorm:"serializer:json"`
HashedPassword []string `json:"hashed_password,omitempty" xml:"hashed_password,omitempty" yaml:"hashed_password,omitempty" gorm:"serializer:json"`
HashType string `json:"hash_type,omitempty" xml:"hash_type,omitempty" yaml:"hash_type,omitempty"`
Name []string `json:"name,omitempty" xml:"name,omitempty" yaml:"name,omitempty" gorm:"serializer:json"`
Vin []string `json:"vin,omitempty" xml:"vin,omitempty" yaml:"vin,omitempty" gorm:"serializer:json"`
LicensePlate []string `json:"license_plate,omitempty" xml:"license_plate,omitempty" yaml:"license_plate,omitempty" gorm:"serializer:json"`
Url []string `json:"url,omitempty" xml:"url,omitempty" yaml:"url,omitempty" gorm:"serializer:json"`
Social []string `json:"social,omitempty" xml:"social,omitempty" yaml:"social,omitempty" gorm:"serializer:json"`
CryptoCurrencyAddress []string `json:"cryptocurrency_address,omitempty" xml:"cryptocurrency_address,omitempty" yaml:"cryptocurrency_address,omitempty" gorm:"serializer:json"`
Address []string `json:"address,omitempty" xml:"address,omitempty" yaml:"address,omitempty" gorm:"serializer:json"`
Phone []string `json:"phone,omitempty" xml:"phone,omitempty" yaml:"phone,omitempty" gorm:"serializer:json"`
Company []string `json:"company,omitempty" xml:"company,omitempty" yaml:"company,omitempty" gorm:"serializer:json"`
DatabaseName string `json:"database_name,omitempty" xml:"database_name,omitempty" yaml:"database_name,omitempty"`
}
func (Result) TableName() string {
return "results"
}
type DehashedResults struct {
Results []Result `json:"results"`
}
func (dr *DehashedResults) ExtractCredentials() []Creds {
var creds []Creds
results := dr.Results
for _, r := range results {
if len(r.Password) > 0 {
// Get first email if available
email := ""
if len(r.Email) > 0 {
email = r.Email[0]
}
// Get first password
password := r.Password[0]
cred := Creds{Email: email, Password: password}
creds = append(creds, cred)
}
}
go func() {
err := StoreCreds(creds)
if err != nil {
zap.L().Error("store_creds",
zap.String("message", "failed to store creds"),
zap.Error(err),
)
fmt.Printf("Error Storing Results: %v", err)
}
}()
return creds
}
func NewDehashedResults(body io.Reader) ([]Result, int, int) {
var response DehashedResponse
err := json.NewDecoder(body).Decode(&response)
if err != nil {
fmt.Printf("Error Parsing Response Body: %v", err)
os.Exit(-1)
}
return response.Entries, response.Balance, response.TotalResults
}
-118
View File
@@ -1,118 +0,0 @@
package sqlite
import (
"dehasher/internal/files"
"fmt"
"gorm.io/gorm"
)
type DBOptions struct {
Username string
Email string
IPAddress string
Password string
HashedPassword string
Name string
Vin string
LicensePlate string
Address string
Phone string
Social string
CryptoCurrencyAddress string
Domain string
Limit int
ExactMatch bool
NonEmptyFields []string // Fields that should not be empty
DisplayFields []string // Fields to display in output
}
func NewDBOptions() *DBOptions {
return &DBOptions{
Limit: 100, // Default limit
ExactMatch: false,
NonEmptyFields: []string{},
DisplayFields: []string{},
}
}
func (o *DBOptions) Empty() bool {
return o.Username == "" && o.Email == "" && o.IPAddress == "" &&
o.Password == "" && o.HashedPassword == "" && o.Name == "" &&
o.Vin == "" && o.LicensePlate == "" && o.Address == "" &&
o.Phone == "" && o.Social == "" && o.CryptoCurrencyAddress == "" && o.Domain == "" &&
len(o.NonEmptyFields) == 0
}
type QueryOptions struct {
gorm.Model
MaxRecords int `json:"max_records"`
MaxRequests int `json:"max_requests"`
StartingPage int `json:"starting_page"`
OutputFormat files.FileType `json:"output_format"`
OutputFile string `json:"output_file"`
RegexMatch bool `json:"regex_match"`
WildcardMatch bool `json:"wildcard_match"`
UsernameQuery string `json:"username_query"`
EmailQuery string `json:"email_query"`
IpQuery string `json:"ip_query"`
PassQuery string `json:"pass_query"`
HashQuery string `json:"hash_query"`
NameQuery string `json:"name_query"`
DomainQuery string `json:"domain_query"`
VinQuery string `json:"vin_query"`
LicensePlateQuery string `json:"license_plate_query"`
AddressQuery string `json:"address_query"`
PhoneQuery string `json:"phone_query"`
SocialQuery string `json:"social_query"`
CryptoAddressQuery string `json:"crypto_address_query"`
PrintBalance bool `json:"print_balance"`
CredsOnly bool `json:"creds_only"`
Debug bool `json:"debug"`
}
func (QueryOptions) TableName() string {
return "query_options"
}
func NewQueryOptions(maxRecords, maxRequests, startingPage int, outputFormat, outputFile, usernameQuery, emailQuery, ipQuery, passQuery, hashQuery, nameQuery, domainQuery, vinQuery, licensePlateQuery, addressQuery, phoneQuery, socialQuery, cryptoAddressQuery string, regexMatch, wildcardMatch, printBalance, credsOnly, debug bool) *QueryOptions {
return &QueryOptions{
MaxRecords: maxRecords,
MaxRequests: maxRequests,
StartingPage: startingPage,
OutputFormat: files.GetFileType(outputFormat),
OutputFile: outputFile,
PrintBalance: printBalance,
CredsOnly: credsOnly,
RegexMatch: regexMatch,
WildcardMatch: wildcardMatch,
UsernameQuery: usernameQuery,
EmailQuery: emailQuery,
IpQuery: ipQuery,
PassQuery: passQuery,
HashQuery: hashQuery,
NameQuery: nameQuery,
DomainQuery: domainQuery,
VinQuery: vinQuery,
LicensePlateQuery: licensePlateQuery,
AddressQuery: addressQuery,
PhoneQuery: phoneQuery,
SocialQuery: socialQuery,
CryptoAddressQuery: cryptoAddressQuery,
Debug: debug,
}
}
type Creds struct {
gorm.Model
Email string `json:"email" yaml:"email" xml:"email" gorm:"uniqueIndex:idx_email_username_password"`
Username string `json:"username" yaml:"username" xml:"username" gorm:"uniqueIndex:idx_email_username_password"`
Password string `json:"password" yaml:"password" xml:"password" gorm:"uniqueIndex:idx_email_username_password"`
}
func (Creds) TableName() string {
return "creds"
}
func (c Creds) ToString() string {
return fmt.Sprintf("%s%s%s", c.Username, "%", c.Password)
}
+45
View File
@@ -0,0 +1,45 @@
package sqlite
import (
"go.uber.org/zap"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
type Subdomain struct {
gorm.Model
Domain string `json:"domain" yaml:"domain" xml:"domain"`
Subdomain string `json:"subdomain" yaml:"subdomain" xml:"subdomain" gorm:"uniqueIndex:idx_subdomain"`
}
func StoreSubdomains(subs []Subdomain) error {
if len(subs) == 0 {
return nil
}
zap.L().Info("Storing subdomains", zap.Int("count", len(subs)))
db := GetDB()
// Use batch insert with conflict handling
// This will insert records in batches and continue even if some fail
const batchSize = 100
var lastErr error
for i := 0; i < len(subs); i += batchSize {
end := i + batchSize
if end > len(subs) {
end = len(subs)
}
batch := subs[i:end]
// Use Clauses with OnConflict DoNothing to skip conflicts
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
if err != nil {
zap.L().Warn("Error storing some credentials", zap.Error(err))
lastErr = err
// Continue with next batch despite error
}
}
return lastErr
}
-58
View File
@@ -1,58 +0,0 @@
package sqlite
import "strings"
type Table int64
const (
ResultsTable Table = iota
RunsTable
CredsTable
WhoIsTable
SubdomainsTable
HistoryTable
LookupTable
UnknownTable
)
func GetTable(userInput string) Table {
switch strings.ToLower(userInput) {
case "results":
return ResultsTable
case "runs":
return RunsTable
case "creds":
return CredsTable
case "whois":
return WhoIsTable
case "subdomains":
return SubdomainsTable
case "history":
return HistoryTable
case "lookup":
return LookupTable
default:
return UnknownTable
}
}
func (t Table) Object() interface{} {
switch t {
case ResultsTable:
return Result{}
case RunsTable:
return QueryOptions{}
case CredsTable:
return Creds{}
case WhoIsTable:
return WhoisRecord{}
case SubdomainsTable:
return SubdomainRecord{}
case HistoryTable:
return HistoryRecord{}
case LookupTable:
return LookupResult{}
default:
return nil
}
}
+58
View File
@@ -0,0 +1,58 @@
package sqlite
import (
"go.uber.org/zap"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
type User struct {
gorm.Model
Company string `json:"company" yaml:"company" xml:"company"`
Position string `json:"position" yaml:"position" xml:"position"`
Department string `json:"department" yaml:"department" xml:"department"`
PhoneNumber string `json:"phone_number" yaml:"phone_number" xml:"phone_number"`
FullName string `json:"full_name" yaml:"full_name" xml:"full_name"`
Phone string `json:"phone" yaml:"phone" xml:"phone"`
Linkedin string `json:"linkedin" yaml:"linkedin" xml:"linkedin"`
Twitter string `json:"twitter" yaml:"twitter" xml:"twitter"`
Facebook string `json:"facebook" yaml:"facebook" xml:"facebook"`
Instagram string `json:"instagram" yaml:"instagram" xml:"instagram"`
Youtube string `json:"youtube" yaml:"youtube" xml:"youtube"`
Gravatar string `json:"gravatar" yaml:"gravatar" xml:"gravatar"`
Email string `json:"email" yaml:"email" xml:"email" gorm:"uniqueIndex:idx_email_username_password"`
Username string `json:"username" yaml:"username" xml:"username" gorm:"uniqueIndex:idx_email_username_password"`
Password string `json:"password" yaml:"password" xml:"password" gorm:"uniqueIndex:idx_email_username_password"`
}
func StoreUsers(users []User) error {
if len(users) == 0 {
return nil
}
zap.L().Info("Storing credentials", zap.Int("count", len(users)))
db := GetDB()
// Use batch insert with conflict handling
// This will insert records in batches and continue even if some fail
const batchSize = 100
var lastErr error
for i := 0; i < len(users); i += batchSize {
end := i + batchSize
if end > len(users) {
end = len(users)
}
batch := users[i:end]
// Use Clauses with OnConflict DoNothing to skip conflicts
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
if err != nil {
zap.L().Warn("Error storing some credentials", zap.Error(err))
lastErr = err
// Continue with next batch despite error
}
}
return lastErr
}
+104 -2
View File
@@ -2,7 +2,9 @@ package sqlite
import (
"fmt"
"go.uber.org/zap"
"gorm.io/gorm"
"gorm.io/gorm/clause"
"strings"
)
@@ -50,7 +52,7 @@ func (w WhoisRecord) String() string {
sb.WriteString(fmt.Sprintf("Domain Name Ext: %s\n", w.DomainNameExt))
sb.WriteString(fmt.Sprintf("Registrar Name: %s\n", w.RegistrarName))
sb.WriteString(fmt.Sprintf("Registrar IANA ID: %s\n", w.RegistrarIANAID))
sb.WriteString(fmt.Sprintf("Contact Email: %s\n", w.ContactEmail))
sb.WriteString(fmt.Sprintf("Contact HunterEmail: %s\n", w.ContactEmail))
sb.WriteString(fmt.Sprintf("Estimated Domain Age: %d days\n", w.EstimatedDomainAge))
// Dates
@@ -379,7 +381,7 @@ func formatContact(sb *strings.Builder, contact ContactInfo, indent string) {
sb.WriteString(indent + "Organization: " + contact.Organization + "\n")
}
if contact.Email != "" {
sb.WriteString(indent + "Email: " + contact.Email + "\n")
sb.WriteString(indent + "HunterEmail: " + contact.Email + "\n")
}
if contact.Street != "" {
sb.WriteString(indent + "Street: " + contact.Street + "\n")
@@ -512,3 +514,103 @@ func formatWhoisContact(sb *strings.Builder, contact Contact, indent string) {
sb.WriteString(indent + "Raw Text: " + rawTextPreview + "\n")
}
}
func StoreWhoisRecord(whoisRecord WhoisRecord) error {
// Create a pointer to the record to make it addressable
recordPtr := &whoisRecord
zap.L().Info("Storing WHOIS record",
zap.String("domain", whoisRecord.DomainName))
db := GetDB()
// Use OnConflict clause to handle duplicates
err := db.Clauses(clause.OnConflict{DoNothing: true}).Create(recordPtr).Error
if err != nil {
zap.L().Error("store_whois_record",
zap.String("message", "failed to store whois record"),
zap.Error(err))
return err
}
return nil
}
func StoreWhoisHistoryRecords(historyRecords []HistoryRecord) error {
if len(historyRecords) == 0 {
return nil
}
zap.L().Info("Storing history records", zap.Int("count", len(historyRecords)))
db := GetDB()
// Use batch insert with conflict handling
const batchSize = 100
var lastErr error
for i := 0; i < len(historyRecords); i += batchSize {
end := i + batchSize
if end > len(historyRecords) {
end = len(historyRecords)
}
batch := historyRecords[i:end]
// Use Clauses with OnConflict DoNothing to skip conflicts
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
if err != nil {
zap.L().Warn("Error storing some history records", zap.Error(err))
lastErr = err
// Continue with next batch despite error
}
}
return lastErr
}
func StoreWhoisLookup(lookup []LookupResult) error {
if len(lookup) == 0 {
return nil
}
zap.L().Info("Storing IP lookup records", zap.Int("count", len(lookup)))
db := GetDB()
// Use batch insert with conflict handling
const batchSize = 100
var lastErr error
for i := 0; i < len(lookup); i += batchSize {
end := i + batchSize
if end > len(lookup) {
end = len(lookup)
}
batch := lookup[i:end]
// Use Clauses with OnConflict DoNothing to skip conflicts
err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&batch, batchSize).Error
if err != nil {
zap.L().Warn("Error storing some IP lookup records", zap.Error(err))
lastErr = err
// Continue with next batch despite error
}
}
return lastErr
}
// ReverseWhoisResponse represents the response from a reverse WHOIS lookup
type ReverseWhoisResponse struct {
RemainingCredits int `json:"remaining_credits"`
Data ReverseWhoisData `json:"data"`
}
// ReverseWhoisData contains the domain count and list from a reverse WHOIS lookup
type ReverseWhoisData struct {
DomainsCount int `json:"domainsCount"`
DomainsList []string `json:"domainsList"`
NextPageSearchAfter *string `json:"nextPageSearchAfter"`
}
func (rwd ReverseWhoisData) String() string {
return fmt.Sprintf("Domains Count: %d\nDomains List: %v\nNext Page Search After: %v\n", rwd.DomainsCount, rwd.DomainsList, rwd.NextPageSearchAfter)
}
+38 -13
View File
@@ -2,9 +2,9 @@ package whois
import (
"bytes"
"dehasher/internal/debug"
"dehasher/internal/dehashed"
"dehasher/internal/sqlite"
"crowsnest/internal/debug"
"crowsnest/internal/dehashed"
"crowsnest/internal/sqlite"
"encoding/json"
"errors"
"fmt"
@@ -298,7 +298,9 @@ func (w *DehashedWhoIs) WhoisHistory(domain string) ([]sqlite.HistoryRecord, err
return whois.Data.Records, nil
}
func (w *DehashedWhoIs) ReverseWHOIS(include []string, exclude []string, reverseType string) (string, error) {
func (w *DehashedWhoIs) ReverseWHOIS(include []string, exclude []string, reverseType string) (sqlite.ReverseWhoisData, error) {
var whois sqlite.ReverseWhoisData
if w.debug {
debug.PrintInfo("performing reverse whois search")
zap.L().Info("reverse_whois_debug",
@@ -329,7 +331,7 @@ func (w *DehashedWhoIs) ReverseWHOIS(include []string, exclude []string, reverse
zap.String("message", "failed to create request"),
zap.Error(err),
)
return "", err
return whois, err
}
req.Header.Set("Content-Type", "application/json")
@@ -356,7 +358,7 @@ func (w *DehashedWhoIs) ReverseWHOIS(include []string, exclude []string, reverse
zap.String("message", "failed to perform request"),
zap.Error(err),
)
return "", err
return whois, err
}
if res == nil {
if w.debug {
@@ -365,7 +367,7 @@ func (w *DehashedWhoIs) ReverseWHOIS(include []string, exclude []string, reverse
zap.L().Error("reverse_whois",
zap.String("message", "response was nil"),
)
return "", errors.New("response was nil")
return whois, errors.New("response was nil")
}
b, err := io.ReadAll(res.Body)
@@ -378,7 +380,7 @@ func (w *DehashedWhoIs) ReverseWHOIS(include []string, exclude []string, reverse
zap.String("message", "failed to read response body"),
zap.Error(err),
)
return "", err
return whois, err
}
// Check for HTTP status code errors
@@ -396,7 +398,7 @@ func (w *DehashedWhoIs) ReverseWHOIS(include []string, exclude []string, reverse
zap.String("error", dhErr.Error()),
zap.String("body_error", string(b)),
)
return "", &dhErr
return whois, &dhErr
}
if w.debug {
@@ -404,7 +406,30 @@ func (w *DehashedWhoIs) ReverseWHOIS(include []string, exclude []string, reverse
debug.PrintJson(fmt.Sprintf("Body: %s\n", string(b[:])))
}
return string(b), nil
var whoisResponse sqlite.ReverseWhoisResponse
err = json.Unmarshal(b, &whoisResponse)
if err != nil {
if w.debug {
debug.PrintInfo("failed to unmarshal response body")
debug.PrintError(err)
}
zap.L().Error("reverse_whois",
zap.String("message", "failed to unmarshal response body"),
zap.Error(err),
)
return whois, err
}
if w.debug {
debug.PrintInfo("unmarshalled response body")
debug.PrintJson(fmt.Sprintf("Remaining Credits: %d\n", whoisResponse.RemainingCredits))
debug.PrintJson(fmt.Sprintf("Data: %v\n", whoisResponse.Data))
}
w.balance = whoisResponse.RemainingCredits
whois = whoisResponse.Data
return whois, nil
}
func (w *DehashedWhoIs) WhoisIP(ipAddress string) ([]sqlite.LookupResult, error) {
@@ -552,7 +577,7 @@ func (w *DehashedWhoIs) WhoisIP(ipAddress string) ([]sqlite.LookupResult, error)
})
}
sqlite.StoreIPLookup(lookups)
sqlite.StoreWhoisLookup(lookups)
return lookups, nil
}
@@ -702,7 +727,7 @@ func (w *DehashedWhoIs) WhoisMX(mxHostname string) ([]sqlite.LookupResult, error
})
}
sqlite.StoreIPLookup(mxLookups)
sqlite.StoreWhoisLookup(mxLookups)
return mxLookups, nil
}
@@ -849,7 +874,7 @@ func (w *DehashedWhoIs) WhoisNS(nsHostname string) ([]sqlite.LookupResult, error
})
}
sqlite.StoreIPLookup(nsLookups)
sqlite.StoreWhoisLookup(nsLookups)
return nsLookups, nil
}