143 lines
2.9 KiB
Go
143 lines
2.9 KiB
Go
package bot
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"math/big"
|
|
"math/bits"
|
|
"math/rand"
|
|
"net"
|
|
"net/http"
|
|
"time"
|
|
)
|
|
|
|
type Bot interface {
|
|
UpdatePool() error
|
|
GetRandomIdentity() string
|
|
}
|
|
|
|
type bot struct {
|
|
UserAgent string
|
|
Fingerprint string
|
|
IPPool botPool
|
|
}
|
|
|
|
type botPool struct {
|
|
Timestamp string `json:"creationTime"`
|
|
Prefixes []botPrefix `json:"prefixes"`
|
|
}
|
|
|
|
type botPrefix struct {
|
|
IPv6 string `json:"ipv6Prefix,omitempty"`
|
|
IPv4 string `json:"ipv4Prefix,omitempty"`
|
|
}
|
|
|
|
// TODO: move pointers around, not global variables
|
|
var GoogleBot = bot{
|
|
UserAgent: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; http://www.google.com/bot.html) Chrome/79.0.3945.120 Safari/537.36",
|
|
|
|
// https://github.com/trisulnsm/trisul-scripts/blob/master/lua/frontend_scripts/reassembly/ja3/prints/ja3fingerprint.json
|
|
Fingerprint: "769,49195-49199-49196-49200-52393-52392-52244-52243-49161-49171-49162-49172-156-157-47-53-10,65281-0-23-35-13-5-18-16-11-10-21,29-23-24,0",
|
|
|
|
IPPool: botPool{
|
|
Timestamp: "2023-11-28T23:00:56.000000",
|
|
Prefixes: []botPrefix{
|
|
{
|
|
IPv4: "34.100.182.96/28",
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
var BingBot = bot{
|
|
UserAgent: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/79.0.3945.120 Safari/537.36",
|
|
IPPool: botPool{
|
|
Timestamp: "2023-03-08T10:00:00.121331",
|
|
Prefixes: []botPrefix{
|
|
{
|
|
IPv4: "207.46.13.0/24",
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
func (b *bot) UpdatePool(url string) error {
|
|
client := &http.Client{Timeout: 10 * time.Second}
|
|
|
|
resp, err := client.Get(url)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
return fmt.Errorf("failed to update googlebot IP pool: status code %s", resp.Status)
|
|
}
|
|
|
|
defer resp.Body.Close()
|
|
|
|
body, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = json.Unmarshal(body, &b.IPPool)
|
|
|
|
return err
|
|
}
|
|
|
|
func (b *bot) GetRandomIP() string {
|
|
count := len(b.IPPool.Prefixes)
|
|
|
|
var prefix botPrefix
|
|
|
|
if count == 1 {
|
|
prefix = b.IPPool.Prefixes[0]
|
|
} else {
|
|
idx := rand.Intn(count)
|
|
prefix = b.IPPool.Prefixes[idx]
|
|
}
|
|
|
|
if prefix.IPv4 != "" {
|
|
ip, err := randomIPFromSubnet(prefix.IPv4)
|
|
if err == nil {
|
|
return ip.String()
|
|
}
|
|
}
|
|
|
|
if prefix.IPv6 != "" {
|
|
ip, err := randomIPFromSubnet(prefix.IPv6)
|
|
if err == nil {
|
|
return ip.String()
|
|
}
|
|
}
|
|
|
|
// fallback to default IP which is known to work
|
|
ip, _ := randomIPFromSubnet(b.IPPool.Prefixes[0].IPv4)
|
|
|
|
return ip.String()
|
|
}
|
|
|
|
func randomIPFromSubnet(c string) (net.IP, error) {
|
|
ip, ipnet, err := net.ParseCIDR(c)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// int representation of byte mask
|
|
mask := big.NewInt(0).SetBytes(ipnet.Mask).Uint64()
|
|
|
|
// how many unset bits there are at the end of the mask
|
|
offset := bits.TrailingZeros8(byte(0) ^ byte(mask))
|
|
|
|
// total number of ips available in the block
|
|
offset *= offset
|
|
|
|
toAdd := rand.Intn(offset)
|
|
|
|
last := len(ip) - 1
|
|
ip[last] = ip[last] + byte(toAdd)
|
|
|
|
return ip, nil
|
|
}
|