rough draft of more modular design

Makefile

@@ -8,3 +8,6 @@ lint:
 install-linters:
 	go install mvdan.cc/gofumpt@latest
 	go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.55.2
+
+run:
+	go run ./cmd/.

cmd/main.go

@@ -83,8 +83,7 @@ func main() {
 	}
 
 	if *randomGooglebot {
-		err := helpers.UpdateGooglebotIPs()
+		err := helpers.GlobalGoogleBot.UpdatePool()
-
 		if err != nil {
 			fmt.Println("error while retrieving list of Googlebot IPs: " + err.Error())
 			fmt.Println("defaulting to known trusted Googlebot identity")

go.mod

@@ -3,6 +3,7 @@ module ladder
 go 1.21.1
 
 require (
+	github.com/3th1nk/cidr v0.2.0
 	github.com/akamensky/argparse v1.4.0
 	github.com/bogdanfinn/fhttp v0.5.24
 	github.com/bogdanfinn/tls-client v1.6.1

go.sum

@@ -1,3 +1,5 @@
+github.com/3th1nk/cidr v0.2.0 h1:81jjEknszD8SHPLVTPPk+BZjNVqq1ND2YXLSChl6Lrs=
+github.com/3th1nk/cidr v0.2.0/go.mod h1:XsSQnS4rEYyB2veDfnIGgViulFpIITPKtp3f0VxpiLw=
 github.com/abadojack/whatlanggo v1.0.1 h1:19N6YogDnf71CTHm3Mp2qhYfkRdyvbgwWdd2EPxJRG4=
 github.com/abadojack/whatlanggo v1.0.1/go.mod h1:66WiQbSbJBIlOZMsvbKe5m6pzQovxCH9B/K8tQB2uoc=
 github.com/akamensky/argparse v1.4.0 h1:YGzvsTqCvbEZhL8zZu2AiA5nq805NZh75JNj4ajn1xc=
@@ -76,6 +78,7 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5 h1:YqAladjX7xpA6BM04leXMWAEjS0mTZ5kUU9KRBriQJc=

internal/helpers/bot.go

@@ -0,0 +1,119 @@
+package helpers
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"math/rand"
+	"net/http"
+	"time"
+
+	"github.com/3th1nk/cidr"
+)
+
+type Bot interface {
+	UpdatePool() error
+	GetRandomIdentity() string
+}
+
+type GoogleBot struct {
+	UserAgent   string
+	Fingerprint string
+	IPPool      googleBotPool
+}
+
+type googleBotPool struct {
+	Timestamp string            `json:"creationTime"`
+	Prefixes  []googleBotPrefix `json:"prefixes"`
+}
+
+type googleBotPrefix struct {
+	IPv6 string `json:"ipv6Prefix,omitempty"`
+	IPv4 string `json:"ipv4Prefix,omitempty"`
+}
+
+// const googleBotTimestampFormat string = "2006-01-02T15:04:05.999999"
+
+// TODO: move this thing's pointer aound, not use it as a global variable
+var GlobalGoogleBot = GoogleBot{
+	UserAgent: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; http://www.google.com/bot.html) Chrome/79.0.3945.120 Safari/537.36",
+
+	// https://github.com/trisulnsm/trisul-scripts/blob/master/lua/frontend_scripts/reassembly/ja3/prints/ja3fingerprint.json
+	Fingerprint: "769,49195-49199-49196-49200-52393-52392-52244-52243-49161-49171-49162-49172-156-157-47-53-10,65281-0-23-35-13-5-18-16-11-10-21,29-23-24,0",
+
+	IPPool: googleBotPool{
+		Timestamp: "2023-11-28T23:00:56.000000",
+		Prefixes: []googleBotPrefix{
+			{
+				IPv4: "34.100.182.96/28",
+			},
+		},
+	},
+}
+
+func (bot *GoogleBot) UpdatePool() error {
+	client := &http.Client{Timeout: 10 * time.Second}
+
+	resp, err := client.Get("https://developers.google.com/static/search/apis/ipranges/googlebot.json")
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("failed to update googlebot IP pool: status code %s", resp.Status)
+	}
+
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	err = json.Unmarshal(body, &bot.IPPool)
+
+	return err
+}
+
+func (bot *GoogleBot) GetRandomIP() string {
+	count := len(bot.IPPool.Prefixes)
+
+	var prefix googleBotPrefix
+
+	if count == 1 {
+		prefix = bot.IPPool.Prefixes[0]
+	} else {
+		idx := rand.Intn(count)
+		prefix = bot.IPPool.Prefixes[idx]
+	}
+
+	if prefix.IPv4 != "" {
+		ip, err := randomIPFromSubnet(prefix.IPv4)
+		if err == nil {
+			return ip
+		}
+	}
+
+	if prefix.IPv6 != "" {
+		ip, err := randomIPFromSubnet(prefix.IPv6)
+		if err == nil {
+			return ip
+		}
+	}
+
+	// fallback to default IP which is known to work
+	ip, _ := randomIPFromSubnet(bot.IPPool.Prefixes[0].IPv4)
+
+	return ip
+}
+
+func randomIPFromSubnet(c string) (string, error) {
+	block, err := cidr.Parse(c)
+	if err != nil {
+		return "", err
+	}
+
+	// TODO: the beginning of the network is technically a viable IP to use
+	// but maybe a different solution would be better here
+	return block.Network().String(), nil
+}

internal/helpers/googlebot.go

@@ -1,84 +0,0 @@
-package helpers
-
-import (
-	"encoding/json"
-	"errors"
-	"io"
-	"math/rand"
-	"net/http"
-	"strings"
-	"time"
-)
-
-type googlebotResp struct {
-	Timestamp time.Time
-	IPs       []string
-}
-
-var GooglebotIPs = googlebotResp{
-	IPs: []string{"34.165.18.176"},
-}
-
-const timeFormat string = "2006-01-02T15:04:05.999999"
-
-func UpdateGooglebotIPs() error {
-	resp, err := http.Get("https://developers.google.com/static/search/apis/ipranges/googlebot.json")
-	if err != nil {
-		return err
-	}
-
-	if resp.StatusCode != http.StatusOK {
-		return errors.New("non-200 status code recieved")
-	}
-
-	defer resp.Body.Close()
-
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return err
-	}
-
-	j := map[string]any{}
-	json.Unmarshal(body, &j)
-
-	timestamp, err := time.Parse(timeFormat, j["creationTime"].(string))
-	if err != nil {
-		return err
-	}
-
-	prefixes := j["prefixes"].([]any)
-
-	ips := make([]string, 0, 127)
-
-	for _, prefix := range prefixes {
-		p := prefix.(map[string]any)
-
-		if val, exists := p["ipv4Prefix"]; exists {
-			v := val.(string)
-
-			v = strings.ReplaceAll(v, "/27", "")
-			v = strings.ReplaceAll(v, "/28", "")
-
-			ips = append(ips, v)
-		}
-
-	}
-
-	GooglebotIPs = googlebotResp{
-		Timestamp: timestamp,
-		IPs:       ips,
-	}
-
-	return nil
-}
-
-func RandomGooglebotIP() string {
-	count := len(GooglebotIPs.IPs)
-	idx := 0
-
-	if count != 1 {
-		idx = rand.Intn(count)
-	}
-
-	return GooglebotIPs.IPs[idx]
-}

proxychain/requestmodifers/masquerade_as_trusted_bot.go

@@ -8,14 +8,9 @@ import (
 // MasqueradeAsGoogleBot modifies user agent and x-forwarded for
 // to appear to be a Google Bot
 func MasqueradeAsGoogleBot() proxychain.RequestModification {
-	const botUA string = "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; http://www.google.com/bot.html) Chrome/79.0.3945.120 Safari/537.36"
+	ip := helpers.GlobalGoogleBot.GetRandomIP()
-	var botIP string = helpers.RandomGooglebotIP()
 
-	// https://github.com/trisulnsm/trisul-scripts/blob/master/lua/frontend_scripts/reassembly/ja3/prints/ja3fingerprint.json
+	return masqueradeAsTrustedBot(helpers.GlobalGoogleBot.UserAgent, ip, helpers.GlobalGoogleBot.Fingerprint)
-	const ja3 string = "769,49195-49199-49196-49200-52393-52392-52244-52243-49161-49171-49162-49172-156-157-47-53-10,65281-0-23-35-13-5-18-16-11-10-21,29-23-24,0"
-	//	"741,49195-49199-49200-49161-49171-49162-49172-156-157-47-10-53-51-57,65281-0-23-35-13-13172-11-10,29-23-24,0"
-
-	return masqueradeAsTrustedBot(botUA, botIP, ja3)
 }
 
 // MasqueradeAsBingBot modifies user agent and x-forwarded for