diff --git a/cmd/main.go b/cmd/main.go index a9fff89..513b83f 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -9,7 +9,7 @@ import ( "ladder/handlers" "ladder/internal/cli" - "ladder/internal/helpers" + "ladder/proxychain/requestmodifers/bot" "github.com/akamensky/argparse" "github.com/gofiber/fiber/v2" @@ -50,9 +50,14 @@ func main() { Help: "Adds verbose logging", }) - randomGooglebot := parser.Flag("", "random-googlebot", &argparse.Options{ + randomGoogleBot := parser.Flag("", "random-googlebot", &argparse.Options{ Required: false, - Help: "Uses a random trusted Googlebot IP for each masqueraded request", + Help: "Update the list of trusted Googlebot IPs, and use a random one for each masqueraded request", + }) + + randomBingBot := parser.Flag("", "random-bingbot", &argparse.Options{ + Required: false, + Help: "Update the list of trusted Bingbot IPs, and use a random one for each masqueraded request", }) // TODO: add version flag that reads from handers/VERSION @@ -82,14 +87,22 @@ func main() { fmt.Print(parser.Usage(err)) } - if *randomGooglebot { - err := helpers.GlobalGoogleBot.UpdatePool() + if *randomGoogleBot { + err := bot.GoogleBot.UpdatePool("https://developers.google.com/static/search/apis/ipranges/googlebot.json") if err != nil { fmt.Println("error while retrieving list of Googlebot IPs: " + err.Error()) fmt.Println("defaulting to known trusted Googlebot identity") } } + if *randomBingBot { + err := bot.GoogleBot.UpdatePool("https://www.bing.com/toolbox/bingbot.json") + if err != nil { + fmt.Println("error while retrieving list of Bingbot IPs: " + err.Error()) + fmt.Println("defaulting to known trusted Bingbot identity") + } + } + // utility cli flag to compile ruleset directory into single ruleset.yaml if *mergeRulesets || *mergeRulesetsGzip { output := os.Stdout diff --git a/internal/helpers/bot.go b/proxychain/requestmodifers/bot/bot.go similarity index 65% rename from internal/helpers/bot.go rename to proxychain/requestmodifers/bot/bot.go index 4e1632c..96f0ad7 100644 --- a/internal/helpers/bot.go +++ b/proxychain/requestmodifers/bot/bot.go @@ -1,4 +1,4 @@ -package helpers +package bot import ( "encoding/json" @@ -16,34 +16,32 @@ type Bot interface { GetRandomIdentity() string } -type GoogleBot struct { +type bot struct { UserAgent string Fingerprint string - IPPool googleBotPool + IPPool botPool } -type googleBotPool struct { - Timestamp string `json:"creationTime"` - Prefixes []googleBotPrefix `json:"prefixes"` +type botPool struct { + Timestamp string `json:"creationTime"` + Prefixes []botPrefix `json:"prefixes"` } -type googleBotPrefix struct { +type botPrefix struct { IPv6 string `json:"ipv6Prefix,omitempty"` IPv4 string `json:"ipv4Prefix,omitempty"` } -// const googleBotTimestampFormat string = "2006-01-02T15:04:05.999999" - -// TODO: move this thing's pointer aound, not use it as a global variable -var GlobalGoogleBot = GoogleBot{ +// TODO: move pointers around, not global variables +var GoogleBot = bot{ UserAgent: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; http://www.google.com/bot.html) Chrome/79.0.3945.120 Safari/537.36", // https://github.com/trisulnsm/trisul-scripts/blob/master/lua/frontend_scripts/reassembly/ja3/prints/ja3fingerprint.json Fingerprint: "769,49195-49199-49196-49200-52393-52392-52244-52243-49161-49171-49162-49172-156-157-47-53-10,65281-0-23-35-13-5-18-16-11-10-21,29-23-24,0", - IPPool: googleBotPool{ + IPPool: botPool{ Timestamp: "2023-11-28T23:00:56.000000", - Prefixes: []googleBotPrefix{ + Prefixes: []botPrefix{ { IPv4: "34.100.182.96/28", }, @@ -51,10 +49,22 @@ var GlobalGoogleBot = GoogleBot{ }, } -func (bot *GoogleBot) UpdatePool() error { +var BingBot = bot{ + UserAgent: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/79.0.3945.120 Safari/537.36", + IPPool: botPool{ + Timestamp: "2023-03-08T10:00:00.121331", + Prefixes: []botPrefix{ + { + IPv4: "207.46.13.0/24", + }, + }, + }, +} + +func (b *bot) UpdatePool(url string) error { client := &http.Client{Timeout: 10 * time.Second} - resp, err := client.Get("https://developers.google.com/static/search/apis/ipranges/googlebot.json") + resp, err := client.Get(url) if err != nil { return err } @@ -70,21 +80,21 @@ func (bot *GoogleBot) UpdatePool() error { return err } - err = json.Unmarshal(body, &bot.IPPool) + err = json.Unmarshal(body, &b.IPPool) return err } -func (bot *GoogleBot) GetRandomIP() string { - count := len(bot.IPPool.Prefixes) +func (b *bot) GetRandomIP() string { + count := len(b.IPPool.Prefixes) - var prefix googleBotPrefix + var prefix botPrefix if count == 1 { - prefix = bot.IPPool.Prefixes[0] + prefix = b.IPPool.Prefixes[0] } else { idx := rand.Intn(count) - prefix = bot.IPPool.Prefixes[idx] + prefix = b.IPPool.Prefixes[idx] } if prefix.IPv4 != "" { @@ -102,7 +112,7 @@ func (bot *GoogleBot) GetRandomIP() string { } // fallback to default IP which is known to work - ip, _ := randomIPFromSubnet(bot.IPPool.Prefixes[0].IPv4) + ip, _ := randomIPFromSubnet(b.IPPool.Prefixes[0].IPv4) return ip } diff --git a/proxychain/requestmodifers/masquerade_as_trusted_bot.go b/proxychain/requestmodifers/masquerade_as_trusted_bot.go index 49cadad..f9bab95 100644 --- a/proxychain/requestmodifers/masquerade_as_trusted_bot.go +++ b/proxychain/requestmodifers/masquerade_as_trusted_bot.go @@ -1,24 +1,24 @@ package requestmodifers import ( - "ladder/internal/helpers" "ladder/proxychain" + "ladder/proxychain/requestmodifers/bot" ) // MasqueradeAsGoogleBot modifies user agent and x-forwarded for // to appear to be a Google Bot func MasqueradeAsGoogleBot() proxychain.RequestModification { - ip := helpers.GlobalGoogleBot.GetRandomIP() + ip := bot.GoogleBot.GetRandomIP() - return masqueradeAsTrustedBot(helpers.GlobalGoogleBot.UserAgent, ip, helpers.GlobalGoogleBot.Fingerprint) + return masqueradeAsTrustedBot(bot.GoogleBot.UserAgent, ip, bot.GoogleBot.Fingerprint) } // MasqueradeAsBingBot modifies user agent and x-forwarded for // to appear to be a Bing Bot func MasqueradeAsBingBot() proxychain.RequestModification { - const botUA string = "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/79.0.3945.120 Safari/537.36" - const botIP string = "13.66.144.9" // https://www.bing.com/toolbox/bingbot.json - return masqueradeAsTrustedBot(botUA, botIP, "") + ip := bot.BingBot.GetRandomIP() + + return masqueradeAsTrustedBot(bot.BingBot.Fingerprint, ip, "") } // MasqueradeAsWaybackMachineBot modifies user agent and x-forwarded for