add bing bot + create generic struct for bots

internal/helpers/bot.go

@@ -1,119 +0,0 @@
-package helpers
-
-import (
-	"encoding/json"
-	"fmt"
-	"io"
-	"math/rand"
-	"net/http"
-	"time"
-
-	"github.com/3th1nk/cidr"
-)
-
-type Bot interface {
-	UpdatePool() error
-	GetRandomIdentity() string
-}
-
-type GoogleBot struct {
-	UserAgent   string
-	Fingerprint string
-	IPPool      googleBotPool
-}
-
-type googleBotPool struct {
-	Timestamp string            `json:"creationTime"`
-	Prefixes  []googleBotPrefix `json:"prefixes"`
-}
-
-type googleBotPrefix struct {
-	IPv6 string `json:"ipv6Prefix,omitempty"`
-	IPv4 string `json:"ipv4Prefix,omitempty"`
-}
-
-// const googleBotTimestampFormat string = "2006-01-02T15:04:05.999999"
-
-// TODO: move this thing's pointer aound, not use it as a global variable
-var GlobalGoogleBot = GoogleBot{
-	UserAgent: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; http://www.google.com/bot.html) Chrome/79.0.3945.120 Safari/537.36",
-
-	// https://github.com/trisulnsm/trisul-scripts/blob/master/lua/frontend_scripts/reassembly/ja3/prints/ja3fingerprint.json
-	Fingerprint: "769,49195-49199-49196-49200-52393-52392-52244-52243-49161-49171-49162-49172-156-157-47-53-10,65281-0-23-35-13-5-18-16-11-10-21,29-23-24,0",
-
-	IPPool: googleBotPool{
-		Timestamp: "2023-11-28T23:00:56.000000",
-		Prefixes: []googleBotPrefix{
-			{
-				IPv4: "34.100.182.96/28",
-			},
-		},
-	},
-}
-
-func (bot *GoogleBot) UpdatePool() error {
-	client := &http.Client{Timeout: 10 * time.Second}
-
-	resp, err := client.Get("https://developers.google.com/static/search/apis/ipranges/googlebot.json")
-	if err != nil {
-		return err
-	}
-
-	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("failed to update googlebot IP pool: status code %s", resp.Status)
-	}
-
-	defer resp.Body.Close()
-
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return err
-	}
-
-	err = json.Unmarshal(body, &bot.IPPool)
-
-	return err
-}
-
-func (bot *GoogleBot) GetRandomIP() string {
-	count := len(bot.IPPool.Prefixes)
-
-	var prefix googleBotPrefix
-
-	if count == 1 {
-		prefix = bot.IPPool.Prefixes[0]
-	} else {
-		idx := rand.Intn(count)
-		prefix = bot.IPPool.Prefixes[idx]
-	}
-
-	if prefix.IPv4 != "" {
-		ip, err := randomIPFromSubnet(prefix.IPv4)
-		if err == nil {
-			return ip
-		}
-	}
-
-	if prefix.IPv6 != "" {
-		ip, err := randomIPFromSubnet(prefix.IPv6)
-		if err == nil {
-			return ip
-		}
-	}
-
-	// fallback to default IP which is known to work
-	ip, _ := randomIPFromSubnet(bot.IPPool.Prefixes[0].IPv4)
-
-	return ip
-}
-
-func randomIPFromSubnet(c string) (string, error) {
-	block, err := cidr.Parse(c)
-	if err != nil {
-		return "", err
-	}
-
-	// TODO: the beginning of the network is technically a viable IP to use
-	// but maybe a different solution would be better here
-	return block.Network().String(), nil
-}