KrakenTech/CrowsNest
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

first commit

Browse Source
This commit is contained in:
Ar1ste1a
2025-05-14 22:00:38 -04:00
commit a4dffe61bf
27 changed files with 4742 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/query/client.go
+111
View File
@@ -0,0 +1,111 @@
package query
import (
"dehasher/internal/sqlite"
"fmt"
"log"
"net/http"
"net/url"
"os"
"strings"
)
type DehashedClient struct {
key string
email string
results []sqlite.Result
client *http.Client
query string
params string
printBal bool
total int
balance int
}
var baseUrl = "https://api.dehashed.com/v2/search"
func NewDehashedClient(key, email string, printBal bool) *DehashedClient {
return &DehashedClient{key: key, email: email, results: make([]sqlite.Result, 0), client: &http.Client{}, printBal: printBal}
}
func (dc *DehashedClient) getKey() string {
return dc.key
}
func (dc *DehashedClient) getEmail() string {
return dc.email
}
func (dc *DehashedClient) GetResults() sqlite.DehashedResults {
return sqlite.DehashedResults{Results: dc.results}
}
func (dc *DehashedClient) buildQuery(params map[string]string) {
urlParams := url.Values{}
urlString := baseUrl
if len(params) > 0 {
urlString += "?query="
for k, v := range params {
if len(v) > 0 {
urlParams.Add(k, v)
}
}
}
tmp, _ := url.QueryUnescape(urlParams.Encode())
tmp2 := strings.Replace(tmp, "=", ":", -1)
dc.params = tmp2
urlString += dc.params
dc.query = urlString
}
func (dc *DehashedClient) setResults(results int) {
dc.query = fmt.Sprintf("%s?query=%s&size=%d", baseUrl, dc.params, results)
}
func (dc *DehashedClient) setPage(page int) {
dc.query = fmt.Sprintf("%s&nextPage=%d", dc.query, page)
}
func (dc *DehashedClient) Do() int {
fmt.Printf("\n\t[*] Performing Request...")
req, err := http.NewRequest("GET", dc.query, nil)
if err != nil {
fmt.Printf("[!] Error constructing request: %v", err)
os.Exit(-1)
}
dc.setAuth(req)
req.Header.Add("Dehashed-Api-Key", dc.getKey())
req.Header.Add("Accept", "application/json")
resp, err := dc.client.Do(req)
if err != nil {
fmt.Printf("[!] Error performing request: %s\n%v", dc.query, err)
os.Exit(-1)
}
if resp.StatusCode != 200 {
dhErr := GetDehashedError(resp.StatusCode)
fmt.Println()
log.Fatal(dhErr.Error())
}
entries, balance, total := sqlite.NewDehashedResults(resp.Body)
dc.results = append(dc.results, entries...)
dc.balance = balance
dc.total += total
if dc.printBal {
fmt.Printf("\n\t\t[*] Balance Remaining: %d", balance)
}
return total
}
func (dc *DehashedClient) setAuth(r *http.Request) {
r.SetBasicAuth(dc.email, dc.key)
}
func (dc *DehashedClient) GetDomains() int {
return dc.balance
}
internal/query/clientv2.go
+193
View File
@@ -0,0 +1,193 @@
package query
import (
"bytes"
"crypto/sha256"
"dehasher/internal/sqlite"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"go.uber.org/zap"
"io"
"net/http"
"strings"
)
type DehashedParameter string
const (
Username DehashedParameter = "username"
Email DehashedParameter = "email"
Password DehashedParameter = "password"
HashedPassword DehashedParameter = "hashed_password"
Name DehashedParameter = "name"
IpAddress DehashedParameter = "ip_address"
Domain DehashedParameter = "domain"
Vin DehashedParameter = "vin"
LicensePlate DehashedParameter = "license_plate"
Address DehashedParameter = "address"
Phone DehashedParameter = "phone"
Social DehashedParameter = "social"
CryptoAddress DehashedParameter = "cryptocurrency_address"
)
func (dp DehashedParameter) GetArgumentString(arg string) string {
return fmt.Sprintf("%s:%s", string(dp), arg)
}
type DehashedSearchRequest struct {
ForcePlaintext bool `json:"-"`
Page int `json:"page"`
Query string `json:"query"`
Size int `json:"size"`
Wildcard bool `json:"wildcard"`
Regex bool `json:"regex"`
DeDupe bool `json:"de_dupe"`
}
func NewDehashedSearchRequest(page, size int, wildcard, regex, forcePlaintext bool) *DehashedSearchRequest {
return &DehashedSearchRequest{Page: page, Query: "", Size: size, Wildcard: wildcard, Regex: regex, DeDupe: true, ForcePlaintext: forcePlaintext}
}
func (dsr *DehashedSearchRequest) buildQuery(query string, param DehashedParameter) {
if len(dsr.Query) > 0 {
dsr.Query = fmt.Sprintf("%s&%s", strings.TrimSpace(dsr.Query), strings.TrimSpace(query))
} else {
dsr.Query = query
}
}
func (dsr *DehashedSearchRequest) AddUsernameQuery(query string) {
query = strings.TrimSpace(query)
dsr.buildQuery(Username.GetArgumentString(query), Username)
}
func (dsr *DehashedSearchRequest) AddEmailQuery(query string) {
query = strings.TrimSpace(query)
dsr.buildQuery(Email.GetArgumentString(query), Email)
}
func (dsr *DehashedSearchRequest) AddIpAddressQuery(query string) {
query = strings.TrimSpace(query)
dsr.buildQuery(IpAddress.GetArgumentString(query), IpAddress)
}
func (dsr *DehashedSearchRequest) AddDomainQuery(query string) {
query = strings.TrimSpace(query)
dsr.buildQuery(Domain.GetArgumentString(query), Domain)
}
func (dsr *DehashedSearchRequest) AddPasswordQuery(query string) {
if dsr.ForcePlaintext {
dsr.buildQuery(Password.GetArgumentString(query), Password)
return
}
hash := sha256.Sum256([]byte(query))
query = hex.EncodeToString(hash[:])
dsr.AddHashedPasswordQuery(query)
}
func (dsr *DehashedSearchRequest) AddVinQuery(query string) {
query = strings.TrimSpace(query)
dsr.buildQuery(Vin.GetArgumentString(query), Vin)
}
func (dsr *DehashedSearchRequest) AddLicensePlateQuery(query string) {
query = strings.TrimSpace(query)
dsr.buildQuery(LicensePlate.GetArgumentString(query), LicensePlate)
}
func (dsr *DehashedSearchRequest) AddAddressQuery(query string) {
query = strings.TrimSpace(query)
dsr.buildQuery(Address.GetArgumentString(query), Address)
}
func (dsr *DehashedSearchRequest) AddPhoneQuery(query string) {
query = strings.TrimSpace(query)
dsr.buildQuery(Phone.GetArgumentString(query), Phone)
}
func (dsr *DehashedSearchRequest) AddSocialQuery(query string) {
query = strings.TrimSpace(query)
dsr.buildQuery(Social.GetArgumentString(query), Social)
}
func (dsr *DehashedSearchRequest) AddCryptoAddressQuery(query string) {
query = strings.TrimSpace(query)
dsr.buildQuery(CryptoAddress.GetArgumentString(query), CryptoAddress)
}
func (dsr *DehashedSearchRequest) AddHashedPasswordQuery(query string) {
dsr.buildQuery(HashedPassword.GetArgumentString(query), HashedPassword)
}
func (dsr *DehashedSearchRequest) AddNameQuery(query string) {
query = strings.TrimSpace(query)
dsr.buildQuery(Name.GetArgumentString(query), Name)
}
type DehashedClientV2 struct {
apiKey string
results []sqlite.Result
}
func NewDehashedClientV2(apiKey string) *DehashedClientV2 {
return &DehashedClientV2{apiKey: apiKey}
}
func (dcv2 *DehashedClientV2) Search(searchRequest DehashedSearchRequest) (int, error) {
reqBody, _ := json.Marshal(searchRequest)
req, err := http.NewRequest("POST", "https://api.dehashed.com/v2/search", bytes.NewReader(reqBody))
if err != nil {
return -1, err
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Dehashed-Api-Key", dcv2.apiKey)
res, err := http.DefaultClient.Do(req)
if res != nil {
defer res.Body.Close()
}
if err != nil {
zap.L().Error("v2_search",
zap.String("message", "failed to perform request"),
zap.Error(err),
)
return -1, err
}
if res == nil {
zap.L().Error("v2_search",
zap.String("message", "response was nil"),
)
return -1, errors.New("response was nil")
}
b, err := io.ReadAll(res.Body)
if err != nil {
zap.L().Error("v2_search",
zap.String("message", "failed to read response body"),
zap.Error(err),
)
return -1, err
}
var responseResults sqlite.DehashedResponse
err = json.Unmarshal(b, &responseResults)
if err != nil {
zap.L().Error("v2_search",
zap.String("message", "failed to unmarshal response body"),
zap.Error(err),
)
return -1, err
}
dcv2.results = append(dcv2.results, responseResults.Entries...)
return responseResults.TotalResults, nil
}
func (dcv2 *DehashedClientV2) GetResults() sqlite.DehashedResults {
return sqlite.DehashedResults{Results: dcv2.results}
}
func (dcv2 *DehashedClientV2) GetTotalResults() int {
return len(dcv2.results)
}
internal/query/dehashed.go
+206
View File
@@ -0,0 +1,206 @@
package query
import (
"dehasher/internal/export"
"dehasher/internal/sqlite"
"encoding/json"
"fmt"
"go.uber.org/zap"
"os"
)
// Dehasher is a struct for querying the Dehashed API
type Dehasher struct {
options sqlite.QueryOptions
nextPage int
request *DehashedSearchRequest
client *DehashedClientV2
}
// NewDehasher creates a new Dehasher
func NewDehasher(options *sqlite.QueryOptions) *Dehasher {
dh := &Dehasher{
options: *options,
nextPage: options.StartingPage + 1,
}
dh.setQueries()
dh.request = NewDehashedSearchRequest(dh.options.StartingPage, dh.options.MaxRecords, dh.options.WildcardMatch, dh.options.RegexMatch, false)
dh.buildRequest()
return dh
}
// SetClientCredentials sets the client credentials for the dehasher
func (dh *Dehasher) SetClientCredentials(key string) {
dh.client = NewDehashedClientV2(key)
}
func (dh *Dehasher) getNextPage() int {
nextPage := dh.nextPage
dh.nextPage += 1
return nextPage
}
// setQueries sets the number of queries to make based on the number of records and requests
func (dh *Dehasher) setQueries() {
var numQueries int
switch {
case dh.options.MaxRequests == 0:
zap.L().Error("max requests cannot be zero")
fmt.Println("[!] Max Requests cannot be zero")
os.Exit(1)
case dh.options.MaxRecords <= 10000 || dh.options.MaxRequests == 1:
numQueries = 1
if dh.options.MaxRecords > 10000 {
dh.options.MaxRecords = 10000
}
zap.L().Info("max requests set to 1", zap.Int("max_records", dh.options.MaxRecords))
case dh.options.MaxRequests < 0 && dh.options.MaxRecords > 20000:
numQueries = 3
dh.options.MaxRecords = 10000
zap.L().Info("max requests set to 3", zap.Int("max_records", dh.options.MaxRecords))
case dh.options.MaxRequests < 0 && dh.options.MaxRecords > 10000:
numQueries = 2
dh.options.MaxRecords = 10000
zap.L().Info("max requests set to 2", zap.Int("max_records", dh.options.MaxRecords))
case dh.options.MaxRecords < 0 && dh.options.MaxRecords < 10000:
numQueries = 1
zap.L().Info("max requests set to 1", zap.Int("max_records", dh.options.MaxRecords))
case dh.options.MaxRequests == 2 && dh.options.MaxRecords > 20000:
numQueries = 2
dh.options.MaxRecords = 10000
zap.L().Info("max requests set to 2", zap.Int("max_records", dh.options.MaxRecords))
case dh.options.MaxRequests == 2 && dh.options.MaxRecords <= 10000:
numQueries = 1
zap.L().Info("max requests set to 1", zap.Int("max_records", dh.options.MaxRecords))
default:
numQueries = 3
dh.options.MaxRecords = 10000
zap.L().Info("max requests set to 3", zap.Int("max_records", dh.options.MaxRecords))
}
dh.options.MaxRequests = numQueries
fmt.Printf("Making %d Requests for %d Records (%d Total)\n", dh.options.MaxRequests, dh.options.MaxRecords, dh.options.MaxRequests*dh.options.MaxRecords)
}
// Start starts the querying process
func (dh *Dehasher) Start() {
fmt.Println("[*] Querying Dehashed API...")
for i := 0; i < dh.options.MaxRequests; i++ {
fmt.Printf("\n\t[*] Performing Request...")
count, err := dh.client.Search(*dh.request)
if err != nil {
fmt.Printf("[!] Error performing request: %v", err)
os.Exit(-1)
}
if count < dh.options.MaxRecords {
fmt.Printf("\n\t\t[+] Retrieved %d Records", count)
fmt.Printf("\n[-] Not Enough Entries, ending queries")
break
} else {
fmt.Printf("\n\t\t[+] Retrieved %d Records", dh.options.MaxRecords)
}
dh.request.Page = dh.getNextPage()
}
dh.parseResults()
}
// buildRequest constructs the query map
func (dh *Dehasher) buildRequest() {
if len(dh.options.UsernameQuery) > 0 {
dh.request.AddUsernameQuery(dh.options.UsernameQuery)
}
if len(dh.options.EmailQuery) > 0 {
dh.request.AddEmailQuery(dh.options.EmailQuery)
}
if len(dh.options.IpQuery) > 0 {
dh.request.AddIpAddressQuery(dh.options.IpQuery)
}
if len(dh.options.HashQuery) > 0 {
dh.request.AddHashedPasswordQuery(dh.options.HashQuery)
}
if len(dh.options.PassQuery) > 0 {
dh.request.AddPasswordQuery(dh.options.PassQuery)
}
if len(dh.options.NameQuery) > 0 {
dh.request.AddNameQuery(dh.options.NameQuery)
}
if len(dh.options.DomainQuery) > 0 {
dh.request.AddDomainQuery(dh.options.DomainQuery)
}
if len(dh.options.VinQuery) > 0 {
dh.request.AddVinQuery(dh.options.VinQuery)
}
if len(dh.options.LicensePlateQuery) > 0 {
dh.request.AddLicensePlateQuery(dh.options.LicensePlateQuery)
}
if len(dh.options.AddressQuery) > 0 {
dh.request.AddAddressQuery(dh.options.AddressQuery)
}
if len(dh.options.PhoneQuery) > 0 {
dh.request.AddPhoneQuery(dh.options.PhoneQuery)
}
if len(dh.options.SocialQuery) > 0 {
dh.request.AddSocialQuery(dh.options.SocialQuery)
}
if len(dh.options.CryptoAddressQuery) > 0 {
dh.request.AddCryptoAddressQuery(dh.options.CryptoAddressQuery)
}
}
// parseResults parses the results and writes them to a file
func (dh *Dehasher) parseResults() {
var data []byte
zap.L().Info("extracting_credentials")
results := dh.client.GetResults()
creds := results.ExtractCredentials()
fmt.Printf("\n\t[*] Discovered %d Credentials", len(creds))
err := sqlite.StoreCreds(creds)
if err != nil {
zap.L().Error("store_creds",
zap.String("message", "failed to store creds"),
zap.Error(err),
)
}
zap.L().Info("creds_stored", zap.Int("count", len(creds)))
zap.L().Info("storing_results")
err = sqlite.StoreResults(results)
if err != nil {
zap.L().Error("store_results",
zap.String("message", "failed to store results"),
zap.Error(err),
)
}
zap.L().Info("results_stored", zap.Int("count", len(results.Results)))
if len(results.Results) > 0 {
fmt.Printf("\n\t[*] Writing entries to file: %s.%s", dh.options.OutputFile, dh.options.OutputFormat.String())
if !dh.options.CredsOnly {
err := export.WriteToFile(results, dh.options.OutputFile, dh.options.OutputFormat)
if err != nil {
fmt.Printf("\n[!] Error Writing to file: %v\n\tOutputting to terminal.", err)
data, err = json.MarshalIndent(results, "", " ")
fmt.Println(string(data))
os.Exit(0)
} else {
fmt.Println("\n\t\t[*] Success\n")
}
} else {
creds := results.ExtractCredentials()
err := export.WriteCredsToFile(creds, dh.options.OutputFile, dh.options.OutputFormat)
if err != nil {
fmt.Printf("\n[!] Error Writing to file: %v\n\tOutputting to terminal.", err)
data, err = json.MarshalIndent(creds, "", " ")
fmt.Println(string(data))
os.Exit(0)
} else {
fmt.Println("\n\t\t[*] Success\n")
}
}
}
}
internal/query/errors.go
+33
View File
@@ -0,0 +1,33 @@
package query
type DehashError struct {
Message string
Code int
}
type DehashResponseError struct {
HttpResponse int `json:"HTTP Response Code"`
}
func (de *DehashError) Error() string {
return de.Message
}
func GetDehashedError(c int) DehashError {
switch c {
case 400:
return DehashError{Code: 400, Message: "There is an issue with authentication. Please check your API key and email. If you haven't, refresh your API Key "}
case 401:
return DehashError{Code: 401, Message: "You need a search subscription and API credits to use the API, please purchase a search subscription and add credits to your account."}
case 403:
return DehashError{Code: 403, Message: "Insufficient Credits"}
case 404:
return DehashError{Code: 404, Message: "Method not permitted"}
case 429:
return DehashError{Code: 420, Message: "Rate Limited"}
case 302:
return DehashError{Code: 302, Message: "Invalid/Missing Query"}
default:
return DehashError{Code: -1, Message: "An unknown error has occurred"}
}
}