add bing bot + create generic struct for bots

cmd/main.go

@@ -9,7 +9,7 @@ import (
 
 	"ladder/handlers"
 	"ladder/internal/cli"
-	"ladder/internal/helpers"
+	"ladder/proxychain/requestmodifers/bot"
 
 	"github.com/akamensky/argparse"
 	"github.com/gofiber/fiber/v2"
@@ -50,9 +50,14 @@ func main() {
 		Help:     "Adds verbose logging",
 	})
 
-	randomGooglebot := parser.Flag("", "random-googlebot", &argparse.Options{
+	randomGoogleBot := parser.Flag("", "random-googlebot", &argparse.Options{
 		Required: false,
-		Help:     "Uses a random trusted Googlebot IP for each masqueraded request",
+		Help:     "Update the list of trusted Googlebot IPs, and use a random one for each masqueraded request",
+	})
+
+	randomBingBot := parser.Flag("", "random-bingbot", &argparse.Options{
+		Required: false,
+		Help:     "Update the list of trusted Bingbot IPs, and use a random one for each masqueraded request",
 	})
 
 	// TODO: add version flag that reads from handers/VERSION
@@ -82,14 +87,22 @@ func main() {
 		fmt.Print(parser.Usage(err))
 	}
 
-	if *randomGooglebot {
+	if *randomGoogleBot {
-		err := helpers.GlobalGoogleBot.UpdatePool()
+		err := bot.GoogleBot.UpdatePool("https://developers.google.com/static/search/apis/ipranges/googlebot.json")
 		if err != nil {
 			fmt.Println("error while retrieving list of Googlebot IPs: " + err.Error())
 			fmt.Println("defaulting to known trusted Googlebot identity")
 		}
 	}
 
+	if *randomBingBot {
+		err := bot.GoogleBot.UpdatePool("https://www.bing.com/toolbox/bingbot.json")
+		if err != nil {
+			fmt.Println("error while retrieving list of Bingbot IPs: " + err.Error())
+			fmt.Println("defaulting to known trusted Bingbot identity")
+		}
+	}
+
 	// utility cli flag to compile ruleset directory into single ruleset.yaml
 	if *mergeRulesets || *mergeRulesetsGzip {
 		output := os.Stdout

proxychain/requestmodifers/bot/bot.go

@@ -1,4 +1,4 @@
-package helpers
+package bot
 
 import (
 	"encoding/json"
@@ -16,34 +16,32 @@ type Bot interface {
 	GetRandomIdentity() string
 }
 
-type GoogleBot struct {
+type bot struct {
 	UserAgent   string
 	Fingerprint string
-	IPPool      googleBotPool
+	IPPool      botPool
 }
 
-type googleBotPool struct {
+type botPool struct {
-	Timestamp string            `json:"creationTime"`
+	Timestamp string      `json:"creationTime"`
-	Prefixes  []googleBotPrefix `json:"prefixes"`
+	Prefixes  []botPrefix `json:"prefixes"`
 }
 
-type googleBotPrefix struct {
+type botPrefix struct {
 	IPv6 string `json:"ipv6Prefix,omitempty"`
 	IPv4 string `json:"ipv4Prefix,omitempty"`
 }
 
-// const googleBotTimestampFormat string = "2006-01-02T15:04:05.999999"
+// TODO: move pointers around, not global variables
-
+var GoogleBot = bot{
-// TODO: move this thing's pointer aound, not use it as a global variable
-var GlobalGoogleBot = GoogleBot{
 	UserAgent: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; http://www.google.com/bot.html) Chrome/79.0.3945.120 Safari/537.36",
 
 	// https://github.com/trisulnsm/trisul-scripts/blob/master/lua/frontend_scripts/reassembly/ja3/prints/ja3fingerprint.json
 	Fingerprint: "769,49195-49199-49196-49200-52393-52392-52244-52243-49161-49171-49162-49172-156-157-47-53-10,65281-0-23-35-13-5-18-16-11-10-21,29-23-24,0",
 
-	IPPool: googleBotPool{
+	IPPool: botPool{
 		Timestamp: "2023-11-28T23:00:56.000000",
-		Prefixes: []googleBotPrefix{
+		Prefixes: []botPrefix{
 			{
 				IPv4: "34.100.182.96/28",
 			},
@@ -51,10 +49,22 @@ var GlobalGoogleBot = GoogleBot{
 	},
 }
 
-func (bot *GoogleBot) UpdatePool() error {
+var BingBot = bot{
+	UserAgent: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/79.0.3945.120 Safari/537.36",
+	IPPool: botPool{
+		Timestamp: "2023-03-08T10:00:00.121331",
+		Prefixes: []botPrefix{
+			{
+				IPv4: "207.46.13.0/24",
+			},
+		},
+	},
+}
+
+func (b *bot) UpdatePool(url string) error {
 	client := &http.Client{Timeout: 10 * time.Second}
 
-	resp, err := client.Get("https://developers.google.com/static/search/apis/ipranges/googlebot.json")
+	resp, err := client.Get(url)
 	if err != nil {
 		return err
 	}
@@ -70,21 +80,21 @@ func (bot *GoogleBot) UpdatePool() error {
 		return err
 	}
 
-	err = json.Unmarshal(body, &bot.IPPool)
+	err = json.Unmarshal(body, &b.IPPool)
 
 	return err
 }
 
-func (bot *GoogleBot) GetRandomIP() string {
+func (b *bot) GetRandomIP() string {
-	count := len(bot.IPPool.Prefixes)
+	count := len(b.IPPool.Prefixes)
 
-	var prefix googleBotPrefix
+	var prefix botPrefix
 
 	if count == 1 {
-		prefix = bot.IPPool.Prefixes[0]
+		prefix = b.IPPool.Prefixes[0]
 	} else {
 		idx := rand.Intn(count)
-		prefix = bot.IPPool.Prefixes[idx]
+		prefix = b.IPPool.Prefixes[idx]
 	}
 
 	if prefix.IPv4 != "" {
@@ -102,7 +112,7 @@ func (bot *GoogleBot) GetRandomIP() string {
 	}
 
 	// fallback to default IP which is known to work
-	ip, _ := randomIPFromSubnet(bot.IPPool.Prefixes[0].IPv4)
+	ip, _ := randomIPFromSubnet(b.IPPool.Prefixes[0].IPv4)
 
 	return ip
 }

proxychain/requestmodifers/masquerade_as_trusted_bot.go

@@ -1,24 +1,24 @@
 package requestmodifers
 
 import (
-	"ladder/internal/helpers"
 	"ladder/proxychain"
+	"ladder/proxychain/requestmodifers/bot"
 )
 
 // MasqueradeAsGoogleBot modifies user agent and x-forwarded for
 // to appear to be a Google Bot
 func MasqueradeAsGoogleBot() proxychain.RequestModification {
-	ip := helpers.GlobalGoogleBot.GetRandomIP()
+	ip := bot.GoogleBot.GetRandomIP()
 
-	return masqueradeAsTrustedBot(helpers.GlobalGoogleBot.UserAgent, ip, helpers.GlobalGoogleBot.Fingerprint)
+	return masqueradeAsTrustedBot(bot.GoogleBot.UserAgent, ip, bot.GoogleBot.Fingerprint)
 }
 
 // MasqueradeAsBingBot modifies user agent and x-forwarded for
 // to appear to be a Bing Bot
 func MasqueradeAsBingBot() proxychain.RequestModification {
-	const botUA string = "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/79.0.3945.120 Safari/537.36"
+	ip := bot.BingBot.GetRandomIP()
-	const botIP string = "13.66.144.9" // https://www.bing.com/toolbox/bingbot.json
+
-	return masqueradeAsTrustedBot(botUA, botIP, "")
+	return masqueradeAsTrustedBot(bot.BingBot.Fingerprint, ip, "")
 }
 
 // MasqueradeAsWaybackMachineBot modifies user agent and x-forwarded for