add bing bot + create generic struct for bots

proxychain/requestmodifers/bot/bot.go

@@ -0,0 +1,129 @@
+package bot
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"math/rand"
+	"net/http"
+	"time"
+
+	"github.com/3th1nk/cidr"
+)
+
+type Bot interface {
+	UpdatePool() error
+	GetRandomIdentity() string
+}
+
+type bot struct {
+	UserAgent   string
+	Fingerprint string
+	IPPool      botPool
+}
+
+type botPool struct {
+	Timestamp string      `json:"creationTime"`
+	Prefixes  []botPrefix `json:"prefixes"`
+}
+
+type botPrefix struct {
+	IPv6 string `json:"ipv6Prefix,omitempty"`
+	IPv4 string `json:"ipv4Prefix,omitempty"`
+}
+
+// TODO: move pointers around, not global variables
+var GoogleBot = bot{
+	UserAgent: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; http://www.google.com/bot.html) Chrome/79.0.3945.120 Safari/537.36",
+
+	// https://github.com/trisulnsm/trisul-scripts/blob/master/lua/frontend_scripts/reassembly/ja3/prints/ja3fingerprint.json
+	Fingerprint: "769,49195-49199-49196-49200-52393-52392-52244-52243-49161-49171-49162-49172-156-157-47-53-10,65281-0-23-35-13-5-18-16-11-10-21,29-23-24,0",
+
+	IPPool: botPool{
+		Timestamp: "2023-11-28T23:00:56.000000",
+		Prefixes: []botPrefix{
+			{
+				IPv4: "34.100.182.96/28",
+			},
+		},
+	},
+}
+
+var BingBot = bot{
+	UserAgent: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/79.0.3945.120 Safari/537.36",
+	IPPool: botPool{
+		Timestamp: "2023-03-08T10:00:00.121331",
+		Prefixes: []botPrefix{
+			{
+				IPv4: "207.46.13.0/24",
+			},
+		},
+	},
+}
+
+func (b *bot) UpdatePool(url string) error {
+	client := &http.Client{Timeout: 10 * time.Second}
+
+	resp, err := client.Get(url)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("failed to update googlebot IP pool: status code %s", resp.Status)
+	}
+
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	err = json.Unmarshal(body, &b.IPPool)
+
+	return err
+}
+
+func (b *bot) GetRandomIP() string {
+	count := len(b.IPPool.Prefixes)
+
+	var prefix botPrefix
+
+	if count == 1 {
+		prefix = b.IPPool.Prefixes[0]
+	} else {
+		idx := rand.Intn(count)
+		prefix = b.IPPool.Prefixes[idx]
+	}
+
+	if prefix.IPv4 != "" {
+		ip, err := randomIPFromSubnet(prefix.IPv4)
+		if err == nil {
+			return ip
+		}
+	}
+
+	if prefix.IPv6 != "" {
+		ip, err := randomIPFromSubnet(prefix.IPv6)
+		if err == nil {
+			return ip
+		}
+	}
+
+	// fallback to default IP which is known to work
+	ip, _ := randomIPFromSubnet(b.IPPool.Prefixes[0].IPv4)
+
+	return ip
+}
+
+func randomIPFromSubnet(c string) (string, error) {
+	block, err := cidr.Parse(c)
+	if err != nil {
+		return "", err
+	}
+
+	// TODO: the beginning of the network is technically a viable IP to use
+	// but maybe a different solution would be better here
+	return block.Network().String(), nil
+}

proxychain/requestmodifers/masquerade_as_trusted_bot.go

@@ -1,24 +1,24 @@
 package requestmodifers
 
 import (
-	"ladder/internal/helpers"
 	"ladder/proxychain"
+	"ladder/proxychain/requestmodifers/bot"
 )
 
 // MasqueradeAsGoogleBot modifies user agent and x-forwarded for
 // to appear to be a Google Bot
 func MasqueradeAsGoogleBot() proxychain.RequestModification {
-	ip := helpers.GlobalGoogleBot.GetRandomIP()
+	ip := bot.GoogleBot.GetRandomIP()
 
-	return masqueradeAsTrustedBot(helpers.GlobalGoogleBot.UserAgent, ip, helpers.GlobalGoogleBot.Fingerprint)
+	return masqueradeAsTrustedBot(bot.GoogleBot.UserAgent, ip, bot.GoogleBot.Fingerprint)
 }
 
 // MasqueradeAsBingBot modifies user agent and x-forwarded for
 // to appear to be a Bing Bot
 func MasqueradeAsBingBot() proxychain.RequestModification {
-	const botUA string = "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/79.0.3945.120 Safari/537.36"
-	const botIP string = "13.66.144.9" // https://www.bing.com/toolbox/bingbot.json
-	return masqueradeAsTrustedBot(botUA, botIP, "")
+	ip := bot.BingBot.GetRandomIP()
+
+	return masqueradeAsTrustedBot(bot.BingBot.Fingerprint, ip, "")
 }
 
 // MasqueradeAsWaybackMachineBot modifies user agent and x-forwarded for