DWS/hadrian
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add bing bot + create generic struct for bots

Browse Source
This commit is contained in:
Damian Bednarczyk
2023-11-30 19:55:34 -06:00
parent ba252f5750
commit 130fdb6b5b
3 changed files with 56 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
cmd/main.go
View File

@@ -9,7 +9,7 @@ import (
"ladder/handlers"
"ladder/internal/cli"
"ladder/internal/helpers"
"ladder/proxychain/requestmodifers/bot"
"github.com/akamensky/argparse"
"github.com/gofiber/fiber/v2"
@@ -50,9 +50,14 @@ func main() {
Help: "Adds verbose logging",
})
randomGooglebot := parser.Flag("", "random-googlebot", &argparse.Options{
randomGoogleBot := parser.Flag("", "random-googlebot", &argparse.Options{
Required: false,
Help: "Uses a random trusted Googlebot IP for each masqueraded request",
Help: "Update the list of trusted Googlebot IPs, and use a random one for each masqueraded request",
})
randomBingBot := parser.Flag("", "random-bingbot", &argparse.Options{
Required: false,
Help: "Update the list of trusted Bingbot IPs, and use a random one for each masqueraded request",
})
// TODO: add version flag that reads from handers/VERSION
@@ -82,14 +87,22 @@ func main() {
fmt.Print(parser.Usage(err))
}
if *randomGooglebot {
err := helpers.GlobalGoogleBot.UpdatePool()
if *randomGoogleBot {
err := bot.GoogleBot.UpdatePool("https://developers.google.com/static/search/apis/ipranges/googlebot.json")
if err != nil {
fmt.Println("error while retrieving list of Googlebot IPs: " + err.Error())
fmt.Println("defaulting to known trusted Googlebot identity")
}
}
if *randomBingBot {
err := bot.GoogleBot.UpdatePool("https://www.bing.com/toolbox/bingbot.json")
if err != nil {
fmt.Println("error while retrieving list of Bingbot IPs: " + err.Error())
fmt.Println("defaulting to known trusted Bingbot identity")
}
}
// utility cli flag to compile ruleset directory into single ruleset.yaml
if *mergeRulesets || *mergeRulesetsGzip {
output := os.Stdout

54
internal/helpers/bot.go → proxychain/requestmodifers/bot/bot.go
View File

@@ -1,4 +1,4 @@
package helpers
package bot
import (
"encoding/json"
@@ -16,34 +16,32 @@ type Bot interface {
GetRandomIdentity() string
}
type GoogleBot struct {
type bot struct {
UserAgent string
Fingerprint string
IPPool googleBotPool
IPPool botPool
}
type googleBotPool struct {
Timestamp string `json:"creationTime"`
Prefixes []googleBotPrefix `json:"prefixes"`
type botPool struct {
Timestamp string `json:"creationTime"`
Prefixes []botPrefix `json:"prefixes"`
}
type googleBotPrefix struct {
type botPrefix struct {
IPv6 string `json:"ipv6Prefix,omitempty"`
IPv4 string `json:"ipv4Prefix,omitempty"`
}
// const googleBotTimestampFormat string = "2006-01-02T15:04:05.999999"
// TODO: move this thing's pointer aound, not use it as a global variable
var GlobalGoogleBot = GoogleBot{
// TODO: move pointers around, not global variables
var GoogleBot = bot{
UserAgent: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; http://www.google.com/bot.html) Chrome/79.0.3945.120 Safari/537.36",
// https://github.com/trisulnsm/trisul-scripts/blob/master/lua/frontend_scripts/reassembly/ja3/prints/ja3fingerprint.json
Fingerprint: "769,49195-49199-49196-49200-52393-52392-52244-52243-49161-49171-49162-49172-156-157-47-53-10,65281-0-23-35-13-5-18-16-11-10-21,29-23-24,0",
IPPool: googleBotPool{
IPPool: botPool{
Timestamp: "2023-11-28T23:00:56.000000",
Prefixes: []googleBotPrefix{
Prefixes: []botPrefix{
{
IPv4: "34.100.182.96/28",
},
@@ -51,10 +49,22 @@ var GlobalGoogleBot = GoogleBot{
},
}
func (bot *GoogleBot) UpdatePool() error {
var BingBot = bot{
UserAgent: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/79.0.3945.120 Safari/537.36",
IPPool: botPool{
Timestamp: "2023-03-08T10:00:00.121331",
Prefixes: []botPrefix{
{
IPv4: "207.46.13.0/24",
},
},
},
}
func (b *bot) UpdatePool(url string) error {
client := &http.Client{Timeout: 10 * time.Second}
resp, err := client.Get("https://developers.google.com/static/search/apis/ipranges/googlebot.json")
resp, err := client.Get(url)
if err != nil {
return err
}
@@ -70,21 +80,21 @@ func (bot *GoogleBot) UpdatePool() error {
return err
}
err = json.Unmarshal(body, &bot.IPPool)
err = json.Unmarshal(body, &b.IPPool)
return err
}
func (bot *GoogleBot) GetRandomIP() string {
count := len(bot.IPPool.Prefixes)
func (b *bot) GetRandomIP() string {
count := len(b.IPPool.Prefixes)
var prefix googleBotPrefix
var prefix botPrefix
if count == 1 {
prefix = bot.IPPool.Prefixes[0]
prefix = b.IPPool.Prefixes[0]
} else {
idx := rand.Intn(count)
prefix = bot.IPPool.Prefixes[idx]
prefix = b.IPPool.Prefixes[idx]
}
if prefix.IPv4 != "" {
@@ -102,7 +112,7 @@ func (bot *GoogleBot) GetRandomIP() string {
}
// fallback to default IP which is known to work
ip, _ := randomIPFromSubnet(bot.IPPool.Prefixes[0].IPv4)
ip, _ := randomIPFromSubnet(b.IPPool.Prefixes[0].IPv4)
return ip
}

12
proxychain/requestmodifers/masquerade_as_trusted_bot.go
View File

@@ -1,24 +1,24 @@
package requestmodifers
import (
"ladder/internal/helpers"
"ladder/proxychain"
"ladder/proxychain/requestmodifers/bot"
)
// MasqueradeAsGoogleBot modifies user agent and x-forwarded for
// to appear to be a Google Bot
func MasqueradeAsGoogleBot() proxychain.RequestModification {
ip := helpers.GlobalGoogleBot.GetRandomIP()
ip := bot.GoogleBot.GetRandomIP()
return masqueradeAsTrustedBot(helpers.GlobalGoogleBot.UserAgent, ip, helpers.GlobalGoogleBot.Fingerprint)
return masqueradeAsTrustedBot(bot.GoogleBot.UserAgent, ip, bot.GoogleBot.Fingerprint)
}
// MasqueradeAsBingBot modifies user agent and x-forwarded for
// to appear to be a Bing Bot
func MasqueradeAsBingBot() proxychain.RequestModification {
const botUA string = "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/79.0.3945.120 Safari/537.36"
const botIP string = "13.66.144.9" // https://www.bing.com/toolbox/bingbot.json
return masqueradeAsTrustedBot(botUA, botIP, "")
ip := bot.BingBot.GetRandomIP()
return masqueradeAsTrustedBot(bot.BingBot.Fingerprint, ip, "")
}
// MasqueradeAsWaybackMachineBot modifies user agent and x-forwarded for