add bing bot + create generic struct for bots

cmd/main.go

@@ -9,7 +9,7 @@ import (
 
 	"ladder/handlers"
 	"ladder/internal/cli"
-	"ladder/internal/helpers"
+	"ladder/proxychain/requestmodifers/bot"
 
 	"github.com/akamensky/argparse"
 	"github.com/gofiber/fiber/v2"
@@ -50,9 +50,14 @@ func main() {
 		Help:     "Adds verbose logging",
 	})
 
-	randomGooglebot := parser.Flag("", "random-googlebot", &argparse.Options{
+	randomGoogleBot := parser.Flag("", "random-googlebot", &argparse.Options{
 		Required: false,
-		Help:     "Uses a random trusted Googlebot IP for each masqueraded request",
+		Help:     "Update the list of trusted Googlebot IPs, and use a random one for each masqueraded request",
+	})
+
+	randomBingBot := parser.Flag("", "random-bingbot", &argparse.Options{
+		Required: false,
+		Help:     "Update the list of trusted Bingbot IPs, and use a random one for each masqueraded request",
 	})
 
 	// TODO: add version flag that reads from handers/VERSION
@@ -82,14 +87,22 @@ func main() {
 		fmt.Print(parser.Usage(err))
 	}
 
-	if *randomGooglebot {
-		err := helpers.GlobalGoogleBot.UpdatePool()
+	if *randomGoogleBot {
+		err := bot.GoogleBot.UpdatePool("https://developers.google.com/static/search/apis/ipranges/googlebot.json")
 		if err != nil {
 			fmt.Println("error while retrieving list of Googlebot IPs: " + err.Error())
 			fmt.Println("defaulting to known trusted Googlebot identity")
 		}
 	}
 
+	if *randomBingBot {
+		err := bot.GoogleBot.UpdatePool("https://www.bing.com/toolbox/bingbot.json")
+		if err != nil {
+			fmt.Println("error while retrieving list of Bingbot IPs: " + err.Error())
+			fmt.Println("defaulting to known trusted Bingbot identity")
+		}
+	}
+
 	// utility cli flag to compile ruleset directory into single ruleset.yaml
 	if *mergeRulesets || *mergeRulesetsGzip {
 		output := os.Stdout