120 lines
2.6 KiB
Go
120 lines
2.6 KiB
Go
package helpers
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"math/rand"
|
|
"net/http"
|
|
"time"
|
|
|
|
"github.com/3th1nk/cidr"
|
|
)
|
|
|
|
type Bot interface {
|
|
UpdatePool() error
|
|
GetRandomIdentity() string
|
|
}
|
|
|
|
type GoogleBot struct {
|
|
UserAgent string
|
|
Fingerprint string
|
|
IPPool googleBotPool
|
|
}
|
|
|
|
type googleBotPool struct {
|
|
Timestamp string `json:"creationTime"`
|
|
Prefixes []googleBotPrefix `json:"prefixes"`
|
|
}
|
|
|
|
type googleBotPrefix struct {
|
|
IPv6 string `json:"ipv6Prefix,omitempty"`
|
|
IPv4 string `json:"ipv4Prefix,omitempty"`
|
|
}
|
|
|
|
// const googleBotTimestampFormat string = "2006-01-02T15:04:05.999999"
|
|
|
|
// TODO: move this thing's pointer aound, not use it as a global variable
|
|
var GlobalGoogleBot = GoogleBot{
|
|
UserAgent: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; http://www.google.com/bot.html) Chrome/79.0.3945.120 Safari/537.36",
|
|
|
|
// https://github.com/trisulnsm/trisul-scripts/blob/master/lua/frontend_scripts/reassembly/ja3/prints/ja3fingerprint.json
|
|
Fingerprint: "769,49195-49199-49196-49200-52393-52392-52244-52243-49161-49171-49162-49172-156-157-47-53-10,65281-0-23-35-13-5-18-16-11-10-21,29-23-24,0",
|
|
|
|
IPPool: googleBotPool{
|
|
Timestamp: "2023-11-28T23:00:56.000000",
|
|
Prefixes: []googleBotPrefix{
|
|
{
|
|
IPv4: "34.100.182.96/28",
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
func (bot *GoogleBot) UpdatePool() error {
|
|
client := &http.Client{Timeout: 10 * time.Second}
|
|
|
|
resp, err := client.Get("https://developers.google.com/static/search/apis/ipranges/googlebot.json")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
return fmt.Errorf("failed to update googlebot IP pool: status code %s", resp.Status)
|
|
}
|
|
|
|
defer resp.Body.Close()
|
|
|
|
body, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = json.Unmarshal(body, &bot.IPPool)
|
|
|
|
return err
|
|
}
|
|
|
|
func (bot *GoogleBot) GetRandomIP() string {
|
|
count := len(bot.IPPool.Prefixes)
|
|
|
|
var prefix googleBotPrefix
|
|
|
|
if count == 1 {
|
|
prefix = bot.IPPool.Prefixes[0]
|
|
} else {
|
|
idx := rand.Intn(count)
|
|
prefix = bot.IPPool.Prefixes[idx]
|
|
}
|
|
|
|
if prefix.IPv4 != "" {
|
|
ip, err := randomIPFromSubnet(prefix.IPv4)
|
|
if err == nil {
|
|
return ip
|
|
}
|
|
}
|
|
|
|
if prefix.IPv6 != "" {
|
|
ip, err := randomIPFromSubnet(prefix.IPv6)
|
|
if err == nil {
|
|
return ip
|
|
}
|
|
}
|
|
|
|
// fallback to default IP which is known to work
|
|
ip, _ := randomIPFromSubnet(bot.IPPool.Prefixes[0].IPv4)
|
|
|
|
return ip
|
|
}
|
|
|
|
func randomIPFromSubnet(c string) (string, error) {
|
|
block, err := cidr.Parse(c)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
// TODO: the beginning of the network is technically a viable IP to use
|
|
// but maybe a different solution would be better here
|
|
return block.Network().String(), nil
|
|
}
|