CrowsNest/internal/dehashed/datawells.go

package dehashed

import (
	"encoding/json"
	"encoding/xml"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"

	"crowsnest/internal/files"
	"gopkg.in/yaml.v3"
)

const dataWellsEndpoint = "https://api.dehashed.com/data-wells"

type DataWellsRequest struct {
	Count int
	Page  int
	Sort  string
}

type DataWellsResponse struct {
	NextPage  bool       `json:"next_page" xml:"next_page" yaml:"next_page"`
	Total     int        `json:"total" xml:"total" yaml:"total"`
	DataWells []DataWell `json:"data_wells" xml:"data_wells" yaml:"data_wells"`
}

type DataWell struct {
	Data        string `json:"data" xml:"data" yaml:"data"`
	Date        string `json:"date" xml:"date" yaml:"date"`
	Description string `json:"description" xml:"description" yaml:"description"`
	Name        string `json:"name" xml:"name" yaml:"name"`
	Records     int    `json:"records" xml:"records" yaml:"records"`
	IsSensitive bool   `json:"is_sensitive" xml:"is_sensitive" yaml:"is_sensitive"`
}

func (dcv2 *DehashedClientV2) DataWells(request DataWellsRequest) (DataWellsResponse, error) {
	var dataWells DataWellsResponse

	endpoint, err := dataWellsURL(request)
	if err != nil {
		return dataWells, err
	}

	req, err := http.NewRequest("GET", endpoint, nil)
	if err != nil {
		return dataWells, err
	}
	req.Header.Set("Accept", "application/json")

	res, err := http.DefaultClient.Do(req)
	if res != nil {
		defer res.Body.Close()
	}
	if err != nil {
		return dataWells, err
	}
	if res == nil {
		return dataWells, errors.New("response was nil")
	}

	body, err := io.ReadAll(res.Body)
	if err != nil {
		return dataWells, err
	}

	if res.StatusCode != http.StatusOK {
		return dataWells, fmt.Errorf("data wells request failed: status=%d body=%s", res.StatusCode, string(body))
	}

	if err := json.Unmarshal(body, &dataWells); err != nil {
		return dataWells, err
	}

	return dataWells, nil
}

func dataWellsURL(request DataWellsRequest) (string, error) {
	if request.Page <= 0 {
		return "", errors.New("page must be 1 or greater")
	}
	if request.Count != 20 && request.Count != 50 {
		return "", errors.New("count must be 20 or 50")
	}
	if request.Sort != "" && !validDataWellsSort(request.Sort) {
		return "", fmt.Errorf("invalid sort %q; use added, name, date, records, optionally suffixed with -ASC or -DESC", request.Sort)
	}

	values := url.Values{}
	values.Set("page", strconv.Itoa(request.Page))
	values.Set("count", strconv.Itoa(request.Count))
	if request.Sort != "" {
		values.Set("sort", request.Sort)
	}

	return dataWellsEndpoint + "?" + values.Encode(), nil
}

func validDataWellsSort(sortValue string) bool {
	sortValue = strings.ToLower(strings.TrimSpace(sortValue))
	field := sortValue
	if before, _, ok := strings.Cut(sortValue, "-"); ok {
		field = before
	}

	switch field {
	case "added", "name", "date", "records":
		return strings.HasSuffix(sortValue, "-asc") || strings.HasSuffix(sortValue, "-desc") || !strings.Contains(sortValue, "-")
	default:
		return false
	}
}

func WriteDataWellsToFile(dataWells DataWellsResponse, outputFile string, fileType files.FileType) error {
	var data []byte
	var err error

	switch fileType {
	case files.JSON:
		data, err = json.MarshalIndent(dataWells, "", "  ")
	case files.XML:
		data, err = xml.MarshalIndent(dataWells, "", "  ")
	case files.YAML:
		data, err = yaml.Marshal(dataWells)
	case files.TEXT:
		data = []byte(dataWells.String())
	case files.GREPPABLE:
		var outStrings []string
		for _, well := range dataWells.DataWells {
			outStrings = append(outStrings, dataWellGreppable(well)+"\n")
		}
		data = []byte(strings.Join(outStrings, ""))
	default:
		return errors.New("unsupported file type")
	}

	if err != nil {
		return err
	}

	return os.WriteFile(outputFile+fileType.Extension(), data, 0644)
}

func (dwr DataWellsResponse) String() string {
	var b strings.Builder
	fmt.Fprintf(&b, "Total: %d\nNext Page: %t\n\n", dwr.Total, dwr.NextPage)
	for _, well := range dwr.DataWells {
		fmt.Fprintf(&b, "Name: %s\nDate: %s\nRecords: %d\nSensitive: %t\nData: %s\nDescription: %s\n\n",
			well.Name,
			well.Date,
			well.Records,
			well.IsSensitive,
			well.Data,
			well.Description,
		)
	}
	return b.String()
}

func dataWellGreppable(well DataWell) string {
	var fields []string
	fields = appendDataWellGreppableField(fields, "name", well.Name)
	fields = appendDataWellGreppableField(fields, "date", well.Date)
	fields = appendDataWellGreppableField(fields, "records", strconv.Itoa(well.Records))
	fields = appendDataWellGreppableField(fields, "is_sensitive", strconv.FormatBool(well.IsSensitive))
	fields = appendDataWellGreppableField(fields, "data", well.Data)
	fields = appendDataWellGreppableField(fields, "description", well.Description)
	return strings.Join(fields, " ")
}

func cleanGreppableValue(value string) string {
	return strings.Join(strings.Fields(value), "_")
}

func appendDataWellGreppableField(fields []string, key, value string) []string {
	value = cleanGreppableValue(value)
	if value == "" {
		return fields
	}
	return append(fields, fmt.Sprintf("%s=%s", key, value))
}