Attempting to stay under the threshold but still hitting it somehow?

I cannot tell if its me or the dehashed devs at this point.

internal/dehashed/dehashed.go

@@ -12,24 +12,34 @@ import (
 
 // Dehasher is a struct for querying the Dehashed API
 type Dehasher struct {
-	options  sqlite.QueryOptions
+	options   sqlite.QueryOptions
-	nextPage int
+	nextPage  int
-	debug    bool
+	debug     bool
-	balance  int
+	balance   int
-	request  *DehashedSearchRequest
+	request   *DehashedSearchRequest
-	client   *DehashedClientV2
+	client    *DehashedClientV2
+	queryPlan []struct{ Page, Size int }
 }
 
 // NewDehasher creates a new Dehasher
 func NewDehasher(options *sqlite.QueryOptions) *Dehasher {
 	dh := &Dehasher{
-		options:  *options,
+		options:   *options,
-		nextPage: options.StartingPage + 1,
+		nextPage:  options.StartingPage + 1,
-		debug:    options.Debug,
+		debug:     options.Debug,
-		balance:  0,
+		balance:   0,
+		queryPlan: make([]struct{ Page, Size int }, 0),
 	}
 	dh.setQueries()
-	dh.request = NewDehashedSearchRequest(dh.options.StartingPage, dh.options.MaxRecords, dh.options.WildcardMatch, dh.options.RegexMatch, false, options.Debug)
+	dh.request = NewDehashedSearchRequest(
+		dh.queryPlan[0].Page,
+		dh.queryPlan[0].Size,
+		dh.options.WildcardMatch,
+		dh.options.RegexMatch,
+		false,
+		options.Debug,
+	)
+
 	dh.buildRequest()
 	return dh
 }
@@ -48,121 +58,172 @@ func (dh *Dehasher) getNextPage() int {
 	return nextPage
 }
 
+// generatePagination creates a list of (page, size) tuples such that page * size <= 10000
+func generatePagination(maxRecords int) []struct{ Page, Size int } {
+	const maxPageProduct = 9500
+	var queries []struct{ Page, Size int }
+
+	remaining := maxRecords
+	page := 1
+
+	for remaining > 0 {
+		size := (maxPageProduct - 1) / page // guarantees page * size < 10000
+		if size > remaining {
+			size = remaining
+		}
+		queries = append(queries, struct{ Page, Size int }{page, size})
+		remaining -= size
+		page++
+	}
+
+	return queries
+}
+
 // setQueries sets the number of queries to make based on the number of records and requests
 func (dh *Dehasher) setQueries() {
-	var numQueries int
+	if dh.options.MaxRecords <= 0 {
-
+		dh.options.MaxRecords = 10000
-	if dh.debug {
-		debug.PrintInfo("setting queries")
 	}
 
-	switch {
+	dh.queryPlan = generatePagination(dh.options.MaxRecords)
-	case dh.options.MaxRequests == 0:
+
-		zap.L().Error("max requests cannot be zero")
+	fmt.Printf("Making %d requests to retrieve %d records\n", len(dh.queryPlan), dh.options.MaxRecords)
-		fmt.Println("[!] Max Requests cannot be zero")
+
-		os.Exit(1)
+	if dh.debug {
-	case dh.options.MaxRecords <= 10000 || dh.options.MaxRequests == 1:
+		for i, q := range dh.queryPlan {
-		numQueries = 1
+			debug.PrintInfo(fmt.Sprintf("query %d: page=%d, size=%d", i+1, q.Page, q.Size))
-		if dh.options.MaxRecords > 10000 {
-			dh.options.MaxRecords = 10000
 		}
-		zap.L().Info("max requests set to 1", zap.Int("max_records", dh.options.MaxRecords))
-	case dh.options.MaxRequests < 0 && dh.options.MaxRecords > 20000:
-		numQueries = 3
-		dh.options.MaxRecords = 10000
-		zap.L().Info("max requests set to 3", zap.Int("max_records", dh.options.MaxRecords))
-	case dh.options.MaxRequests < 0 && dh.options.MaxRecords > 10000:
-		numQueries = 2
-		dh.options.MaxRecords = 10000
-		zap.L().Info("max requests set to 2", zap.Int("max_records", dh.options.MaxRecords))
-	case dh.options.MaxRecords < 0 && dh.options.MaxRecords < 10000:
-		numQueries = 1
-		zap.L().Info("max requests set to 1", zap.Int("max_records", dh.options.MaxRecords))
-	case dh.options.MaxRequests == 2 && dh.options.MaxRecords > 20000:
-		numQueries = 2
-		dh.options.MaxRecords = 10000
-		zap.L().Info("max requests set to 2", zap.Int("max_records", dh.options.MaxRecords))
-	case dh.options.MaxRequests == 2 && dh.options.MaxRecords <= 10000:
-		numQueries = 1
-		zap.L().Info("max requests set to 1", zap.Int("max_records", dh.options.MaxRecords))
-	default:
-		numQueries = 3
-		dh.options.MaxRecords = 10000
-		zap.L().Info("max requests set to 3", zap.Int("max_records", dh.options.MaxRecords))
 	}
-
-	dh.options.MaxRequests = numQueries
-
-	if dh.debug {
-		debug.PrintInfo(fmt.Sprintf("setting max requests: %d", numQueries))
-		debug.PrintInfo(fmt.Sprintf("setting max records: %d", dh.options.MaxRecords))
-	}
-
-	fmt.Printf("Making %d Requests for %d Records (%d Total)\n", dh.options.MaxRequests, dh.options.MaxRecords, dh.options.MaxRequests*dh.options.MaxRecords)
 }
 
 // Start starts the querying process
 func (dh *Dehasher) Start() {
 	fmt.Printf("[*] Querying Dehashed API...\n")
-	for i := 0; i < dh.options.MaxRequests; i++ {
-		fmt.Printf("   [*] Performing Request...\n")
-		count, balance, err := dh.client.Search(*dh.request)
-		if err != nil {
-			if dh.debug {
-				debug.PrintInfo("error performing request")
-				debug.PrintError(err)
-			}
 
-			// Check if it's a DehashError
+	// Make initial request to get total count
-			if dhErr, ok := err.(*DehashError); ok {
+	fmt.Printf("   [*] Performing initial request to determine total records...\n")
-				fmt.Printf("      [!] Dehashed API Error: %s (Code: %d)\n", dhErr.Message, dhErr.Code)
+	totalRecords, balance, err := dh.client.Search(*dh.request)
-				zap.L().Error("dehashed_api_error",
+	if err != nil {
-					zap.String("message", dhErr.Message),
+		handleSearchError(dh, err)
-					zap.Int("code", dhErr.Code),
+		return
-				)
+	}
-			} else {
-				fmt.Printf("   [!] Error performing request: %v\n", err)
-				zap.L().Error("request_error",
-					zap.String("message", "failed to perform request"),
-					zap.Error(err),
-				)
-			}
 
-			if len(dh.client.results) > 0 {
+	dh.balance = balance
-				fmt.Printf("   [!] Partial results retrieved. Storing Results...\n")
+	recordsRetrieved := len(dh.client.results)
-				err := sqlite.StoreResults(dh.client.GetResults())
+
-				if err != nil {
+	fmt.Printf("      [+] Retrieved %d records\n", recordsRetrieved)
-					zap.L().Error("store_results",
+	fmt.Printf("      [*] Total available records: %d\n", totalRecords)
-						zap.String("message", "failed to store results"),
+
-						zap.Error(err),
+	if dh.options.PrintBalance {
-					)
+		fmt.Printf("      [*] Balance: %d\n", balance)
-					fmt.Printf("   [!] Error storing results: %v\n", err)
+	}
-				}
+
-			}
+	// If we've already got all records or reached our limit, we're done
+	if recordsRetrieved >= totalRecords || recordsRetrieved >= dh.options.MaxRecords {
+		fmt.Printf("      [*] All requested records retrieved\n")
+		dh.parseResults()
+		return
+	}
+
+	// Calculate remaining records to fetch
+	remainingRecords := totalRecords - recordsRetrieved
+	if dh.options.MaxRecords > 0 && dh.options.MaxRecords < totalRecords {
+		remainingRecords = dh.options.MaxRecords - recordsRetrieved
+	}
+
+	// Check if we need user confirmation for large datasets
+	if remainingRecords > 30000 {
+		tokensRequired := (remainingRecords + 9999) / 10000 // Ceiling division
+		fmt.Printf("\n[!] Large dataset detected: %d additional records\n", remainingRecords)
+		fmt.Printf("[!] This will require approximately %d API tokens\n", tokensRequired)
+		fmt.Printf("[!] Your current balance: %d\n", balance)
+
+		if balance < tokensRequired {
+			fmt.Printf("[!] WARNING: Your balance (%d) is less than required tokens (%d)\n", balance, tokensRequired)
+		}
+
+		fmt.Printf("[?] Do you want to continue? (y/n): ")
+		var response string
+		fmt.Scanln(&response)
+
+		if response != "y" && response != "Y" {
+			fmt.Println("[*] Operation cancelled by user")
 			dh.parseResults()
-			os.Exit(-1)
+			return
+		}
+	}
+
+	// Make additional requests
+	for i, q := range dh.queryPlan {
+		if i == 0 {
+			// We already made the first request before this loop
+			continue
+		}
+
+		dh.request.Page = q.Page
+		dh.request.Size = q.Size
+
+		fmt.Printf("   [*] Performing Request %d of %d (page=%d, size=%d)...\n", i+1, len(dh.queryPlan), q.Page, q.Size)
+
+		_, balance, err := dh.client.Search(*dh.request)
+		if err != nil {
+			handleSearchError(dh, err)
+			break
 		}
 
 		dh.balance = balance
+		recordsRetrieved += len(dh.client.results)
 
-		if count < dh.options.MaxRecords {
+		fmt.Printf("      [+] Retrieved %d total records so far\n", recordsRetrieved)
-			fmt.Printf("      [+] Retrieved %d records\n", count)
-			fmt.Printf("      [-] Not enough entries, ending queries\n")
-			break
-		} else {
-			fmt.Printf("      [+] Retrieved %d records\n", dh.options.MaxRecords)
-		}
 
 		if dh.options.PrintBalance {
 			fmt.Printf("      [*] Balance: %d\n", balance)
 		}
 
-		dh.request.Page = dh.getNextPage()
+		if recordsRetrieved >= totalRecords || recordsRetrieved >= dh.options.MaxRecords {
+			fmt.Printf("      [*] All requested records retrieved\n")
+			break
+		}
 	}
 
 	dh.parseResults()
 }
 
+// Helper function to handle search errors
+func handleSearchError(dh *Dehasher, err error) {
+	if dh.debug {
+		debug.PrintInfo("error performing request")
+		debug.PrintError(err)
+	}
+
+	// Check if it's a DehashError
+	if dhErr, ok := err.(*DehashError); ok {
+		fmt.Printf("      [!] Dehashed API Error: %s (Code: %d)\n", dhErr.Message, dhErr.Code)
+		zap.L().Error("dehashed_api_error",
+			zap.String("message", dhErr.Message),
+			zap.Int("code", dhErr.Code),
+		)
+	} else {
+		fmt.Printf("   [!] Error performing request: %v\n", err)
+		zap.L().Error("request_error",
+			zap.String("message", "failed to perform request"),
+			zap.Error(err),
+		)
+	}
+
+	if len(dh.client.results) > 0 {
+		fmt.Printf("   [!] Partial results retrieved. Storing Results...\n")
+		err := sqlite.StoreResults(dh.client.GetResults())
+		if err != nil {
+			zap.L().Error("store_results",
+				zap.String("message", "failed to store results"),
+				zap.Error(err),
+			)
+			fmt.Printf("   [!] Error storing results: %v\n", err)
+		}
+	}
+}
+
 // buildRequest constructs the query map
 func (dh *Dehasher) buildRequest() {
 	if len(dh.options.UsernameQuery) > 0 {