diff --git a/cmd/main.go b/cmd/main.go index ab9d8bf..4089160 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -9,6 +9,7 @@ import ( "ladder/handlers" "ladder/internal/cli" + "ladder/internal/helpers" "github.com/akamensky/argparse" "github.com/gofiber/fiber/v2" @@ -22,7 +23,6 @@ var faviconData string //go:embed styles.css var cssData embed.FS -//go:embed VERSION var version string func main() { @@ -49,6 +49,11 @@ func main() { Help: "Adds verbose logging", }) + randomGooglebot := parser.Flag("", "random-googlebot", &argparse.Options{ + Required: false, + Help: "Uses a random trusted Googlebot IP for each masqueraded request", + }) + // TODO: add version flag that reads from handers/VERSION ruleset := parser.String("r", "ruleset", &argparse.Options{ @@ -76,6 +81,15 @@ func main() { fmt.Print(parser.Usage(err)) } + if *randomGooglebot { + err := helpers.UpdateGooglebotIPs() + + if err != nil { + fmt.Println("error while retrieving list of Googlebot IPs: " + err.Error()) + fmt.Println("defaulting to known trusted Googlebot identity") + } + } + // utility cli flag to compile ruleset directory into single ruleset.yaml if *mergeRulesets || *mergeRulesetsGzip { output := os.Stdout diff --git a/handlers/api.go b/handlers/api.go index 96ff82a..1a34864 100644 --- a/handlers/api.go +++ b/handlers/api.go @@ -7,7 +7,7 @@ import ( ) //nolint:all -//go:embed VERSION + var version string func Api(c *fiber.Ctx) error { diff --git a/internal/helpers/googlebot.go b/internal/helpers/googlebot.go new file mode 100644 index 0000000..58b09a2 --- /dev/null +++ b/internal/helpers/googlebot.go @@ -0,0 +1,84 @@ +package helpers + +import ( + "encoding/json" + "errors" + "io" + "math/rand" + "net/http" + "strings" + "time" +) + +type googlebotResp struct { + Timestamp time.Time + IPs []string +} + +var GooglebotIPs = googlebotResp{ + IPs: []string{"34.165.18.176"}, +} + +const timeFormat string = "2006-01-02T15:04:05.999999" + +func UpdateGooglebotIPs() error { + resp, err := http.Get("https://developers.google.com/static/search/apis/ipranges/googlebot.json") + if err != nil { + return err + } + + if resp.StatusCode != http.StatusOK { + return errors.New("non-200 status code recieved") + } + + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + + j := map[string]any{} + json.Unmarshal(body, &j) + + timestamp, err := time.Parse(timeFormat, j["creationTime"].(string)) + if err != nil { + return err + } + + prefixes := j["prefixes"].([]any) + + ips := make([]string, 0, 127) + + for _, prefix := range prefixes { + p := prefix.(map[string]any) + + if val, exists := p["ipv4Prefix"]; exists { + v := val.(string) + + v = strings.ReplaceAll(v, "/27", "") + v = strings.ReplaceAll(v, "/28", "") + + ips = append(ips, v) + } + + } + + GooglebotIPs = googlebotResp{ + Timestamp: timestamp, + IPs: ips, + } + + return nil +} + +func RandomGooglebotIP() string { + count := len(GooglebotIPs.IPs) + idx := 0 + + if count != 1 { + idx = rand.Intn(count) + } + + return GooglebotIPs.IPs[idx] +} diff --git a/proxychain/requestmodifers/masquerade_as_trusted_bot.go b/proxychain/requestmodifers/masquerade_as_trusted_bot.go index 4f1c75a..4d4d81a 100644 --- a/proxychain/requestmodifers/masquerade_as_trusted_bot.go +++ b/proxychain/requestmodifers/masquerade_as_trusted_bot.go @@ -1,6 +1,7 @@ package requestmodifers import ( + "ladder/internal/helpers" "ladder/proxychain" ) @@ -8,7 +9,8 @@ import ( // to appear to be a Google Bot func MasqueradeAsGoogleBot() proxychain.RequestModification { const botUA string = "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; http://www.google.com/bot.html) Chrome/79.0.3945.120 Safari/537.36" - const botIP string = "66.249.78.8" // TODO: create a random ip pool from https://developers.google.com/static/search/apis/ipranges/googlebot.json + var botIP string = helpers.RandomGooglebotIP() + // https://github.com/trisulnsm/trisul-scripts/blob/master/lua/frontend_scripts/reassembly/ja3/prints/ja3fingerprint.json const ja3 string = "769,49195-49199-49196-49200-52393-52392-52244-52243-49161-49171-49162-49172-156-157-47-53-10,65281-0-23-35-13-5-18-16-11-10-21,29-23-24,0" // "741,49195-49199-49200-49161-49171-49162-49172-156-157-47-10-53-51-57,65281-0-23-35-13-13172-11-10,29-23-24,0"