add option to use random ip from googlebot pool

cmd/main.go

@@ -9,6 +9,7 @@ import (
 
 	"ladder/handlers"
 	"ladder/internal/cli"
+	"ladder/internal/helpers"
 
 	"github.com/akamensky/argparse"
 	"github.com/gofiber/fiber/v2"
@@ -22,7 +23,6 @@ var faviconData string
 //go:embed styles.css
 var cssData embed.FS
 
-//go:embed VERSION
 var version string
 
 func main() {
@@ -49,6 +49,11 @@ func main() {
 		Help:     "Adds verbose logging",
 	})
 
+	randomGooglebot := parser.Flag("", "random-googlebot", &argparse.Options{
+		Required: false,
+		Help:     "Uses a random trusted Googlebot IP for each masqueraded request",
+	})
+
 	// TODO: add version flag that reads from handers/VERSION
 
 	ruleset := parser.String("r", "ruleset", &argparse.Options{
@@ -76,6 +81,15 @@ func main() {
 		fmt.Print(parser.Usage(err))
 	}
 
+	if *randomGooglebot {
+		err := helpers.UpdateGooglebotIPs()
+
+		if err != nil {
+			fmt.Println("error while retrieving list of Googlebot IPs: " + err.Error())
+			fmt.Println("defaulting to known trusted Googlebot identity")
+		}
+	}
+
 	// utility cli flag to compile ruleset directory into single ruleset.yaml
 	if *mergeRulesets || *mergeRulesetsGzip {
 		output := os.Stdout

handlers/api.go

@@ -7,7 +7,7 @@ import (
 )
 
 //nolint:all
-//go:embed VERSION
+
 var version string
 
 func Api(c *fiber.Ctx) error {

internal/helpers/googlebot.go

@@ -0,0 +1,84 @@
+package helpers
+
+import (
+	"encoding/json"
+	"errors"
+	"io"
+	"math/rand"
+	"net/http"
+	"strings"
+	"time"
+)
+
+type googlebotResp struct {
+	Timestamp time.Time
+	IPs       []string
+}
+
+var GooglebotIPs = googlebotResp{
+	IPs: []string{"34.165.18.176"},
+}
+
+const timeFormat string = "2006-01-02T15:04:05.999999"
+
+func UpdateGooglebotIPs() error {
+	resp, err := http.Get("https://developers.google.com/static/search/apis/ipranges/googlebot.json")
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return errors.New("non-200 status code recieved")
+	}
+
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	j := map[string]any{}
+	json.Unmarshal(body, &j)
+
+	timestamp, err := time.Parse(timeFormat, j["creationTime"].(string))
+	if err != nil {
+		return err
+	}
+
+	prefixes := j["prefixes"].([]any)
+
+	ips := make([]string, 0, 127)
+
+	for _, prefix := range prefixes {
+		p := prefix.(map[string]any)
+
+		if val, exists := p["ipv4Prefix"]; exists {
+			v := val.(string)
+
+			v = strings.ReplaceAll(v, "/27", "")
+			v = strings.ReplaceAll(v, "/28", "")
+
+			ips = append(ips, v)
+		}
+
+	}
+
+	GooglebotIPs = googlebotResp{
+		Timestamp: timestamp,
+		IPs:       ips,
+	}
+
+	return nil
+}
+
+func RandomGooglebotIP() string {
+	count := len(GooglebotIPs.IPs)
+	idx := 0
+
+	if count != 1 {
+		idx = rand.Intn(count)
+	}
+
+	return GooglebotIPs.IPs[idx]
+}

proxychain/requestmodifers/masquerade_as_trusted_bot.go

@@ -1,6 +1,7 @@
 package requestmodifers
 
 import (
+	"ladder/internal/helpers"
 	"ladder/proxychain"
 )
 
@@ -8,7 +9,8 @@ import (
 // to appear to be a Google Bot
 func MasqueradeAsGoogleBot() proxychain.RequestModification {
 	const botUA string = "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; http://www.google.com/bot.html) Chrome/79.0.3945.120 Safari/537.36"
-	const botIP string = "66.249.78.8" // TODO: create a random ip pool from https://developers.google.com/static/search/apis/ipranges/googlebot.json
+	var botIP string = helpers.RandomGooglebotIP()
+
 	// https://github.com/trisulnsm/trisul-scripts/blob/master/lua/frontend_scripts/reassembly/ja3/prints/ja3fingerprint.json
 	const ja3 string = "769,49195-49199-49196-49200-52393-52392-52244-52243-49161-49171-49162-49172-156-157-47-53-10,65281-0-23-35-13-5-18-16-11-10-21,29-23-24,0"
 	//	"741,49195-49199-49200-49161-49171-49162-49172-156-157-47-10-53-51-57,65281-0-23-35-13-13172-11-10,29-23-24,0"