fix "modifer" -> "modifier" typo everywhere

This commit is contained in:
Kevin Pham
2023-12-03 17:04:30 -06:00
parent 117ded5653
commit 6c0721dcb8
67 changed files with 73 additions and 77 deletions

View File

@@ -0,0 +1,142 @@
package bot
import (
"encoding/json"
"fmt"
"io"
"math/big"
"math/bits"
"math/rand"
"net"
"net/http"
"time"
)
type Bot interface {
UpdatePool() error
GetRandomIdentity() string
}
type bot struct {
UserAgent string
Fingerprint string
IPPool botPool
}
type botPool struct {
Timestamp string `json:"creationTime"`
Prefixes []botPrefix `json:"prefixes"`
}
type botPrefix struct {
IPv6 string `json:"ipv6Prefix,omitempty"`
IPv4 string `json:"ipv4Prefix,omitempty"`
}
// TODO: move pointers around, not global variables
var GoogleBot = bot{
UserAgent: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; http://www.google.com/bot.html) Chrome/79.0.3945.120 Safari/537.36",
// https://github.com/trisulnsm/trisul-scripts/blob/master/lua/frontend_scripts/reassembly/ja3/prints/ja3fingerprint.json
Fingerprint: "769,49195-49199-49196-49200-52393-52392-52244-52243-49161-49171-49162-49172-156-157-47-53-10,65281-0-23-35-13-5-18-16-11-10-21,29-23-24,0",
IPPool: botPool{
Timestamp: "2023-11-28T23:00:56.000000",
Prefixes: []botPrefix{
{
IPv4: "34.100.182.96/28",
},
},
},
}
var BingBot = bot{
UserAgent: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/79.0.3945.120 Safari/537.36",
IPPool: botPool{
Timestamp: "2023-03-08T10:00:00.121331",
Prefixes: []botPrefix{
{
IPv4: "207.46.13.0/24",
},
},
},
}
func (b *bot) UpdatePool(url string) error {
client := &http.Client{Timeout: 10 * time.Second}
resp, err := client.Get(url)
if err != nil {
return err
}
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("failed to update googlebot IP pool: status code %s", resp.Status)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return err
}
err = json.Unmarshal(body, &b.IPPool)
return err
}
func (b *bot) GetRandomIP() string {
count := len(b.IPPool.Prefixes)
var prefix botPrefix
if count == 1 {
prefix = b.IPPool.Prefixes[0]
} else {
idx := rand.Intn(count)
prefix = b.IPPool.Prefixes[idx]
}
if prefix.IPv4 != "" {
ip, err := randomIPFromSubnet(prefix.IPv4)
if err == nil {
return ip.String()
}
}
if prefix.IPv6 != "" {
ip, err := randomIPFromSubnet(prefix.IPv6)
if err == nil {
return ip.String()
}
}
// fallback to default IP which is known to work
ip, _ := randomIPFromSubnet(b.IPPool.Prefixes[0].IPv4)
return ip.String()
}
func randomIPFromSubnet(c string) (net.IP, error) {
ip, ipnet, err := net.ParseCIDR(c)
if err != nil {
return nil, err
}
// int representation of byte mask
mask := big.NewInt(0).SetBytes(ipnet.Mask).Uint64()
// how many unset bits there are at the end of the mask
offset := bits.TrailingZeros8(byte(0) ^ byte(mask))
// total number of ips available in the block
offset *= offset
toAdd := rand.Intn(offset)
last := len(ip) - 1
ip[last] = ip[last] + byte(toAdd)
return ip, nil
}

View File

@@ -0,0 +1,36 @@
package bot
import (
"net"
"testing"
)
func TestRandomIPFromSubnet(t *testing.T) {
err := GoogleBot.UpdatePool("https://developers.google.com/static/search/apis/ipranges/googlebot.json")
if err != nil {
t.Error(err)
}
for _, prefix := range GoogleBot.IPPool.Prefixes {
subnet := prefix.IPv4
if prefix.IPv6 != "" {
subnet = prefix.IPv6
}
t.Run(subnet, func(t *testing.T) {
_, ipnet, err := net.ParseCIDR(subnet)
if err != nil {
t.Error(err)
}
ip, err := randomIPFromSubnet(subnet)
if err != nil {
t.Error(err)
}
if !ipnet.Contains(ip) {
t.Fail()
}
})
}
}

View File

@@ -0,0 +1,45 @@
package requestmodifiers
import (
"strings"
//"fmt"
"ladder/proxychain"
)
var forwardBlacklist map[string]bool
func init() {
forwardBlacklist = map[string]bool{
"host": true,
"connection": true,
"keep-alive": true,
"content-length": true,
"content-encoding": true,
"transfer-encoding": true,
"referer": true,
"x-forwarded-for": true,
"x-real-ip": true,
"forwarded": true,
"accept-encoding": true,
}
}
// ForwardRequestHeaders forwards the requests headers sent from the client to the upstream server
func ForwardRequestHeaders() proxychain.RequestModification {
return func(chain *proxychain.ProxyChain) error {
forwardHeaders := func(key, value []byte) {
k := strings.ToLower(string(key))
v := string(value)
if forwardBlacklist[k] {
return
}
// fmt.Println(k, v)
chain.Request.Header.Set(k, v)
}
chain.Context.Request().
Header.VisitAll(forwardHeaders)
return nil
}
}

View File

@@ -0,0 +1,126 @@
package requestmodifiers
import (
"ladder/proxychain"
"ladder/proxychain/requestmodifiers/bot"
)
// MasqueradeAsGoogleBot modifies user agent and x-forwarded for
// to appear to be a Google Bot
func MasqueradeAsGoogleBot() proxychain.RequestModification {
ip := bot.GoogleBot.GetRandomIP()
return masqueradeAsTrustedBot(bot.GoogleBot.UserAgent, ip, bot.GoogleBot.Fingerprint)
}
// MasqueradeAsBingBot modifies user agent and x-forwarded for
// to appear to be a Bing Bot
func MasqueradeAsBingBot() proxychain.RequestModification {
ip := bot.BingBot.GetRandomIP()
return masqueradeAsTrustedBot(bot.BingBot.Fingerprint, ip, "")
}
// MasqueradeAsWaybackMachineBot modifies user agent and x-forwarded for
// to appear to be a archive.org (wayback machine) Bot
func MasqueradeAsWaybackMachineBot() proxychain.RequestModification {
const botUA string = "Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)"
const botIP string = "207.241.235.164"
return masqueradeAsTrustedBot(botUA, botIP, "")
}
// MasqueradeAsFacebookBot modifies user agent and x-forwarded for
// to appear to be a Facebook Bot (link previews?)
func MasqueradeAsFacebookBot() proxychain.RequestModification {
const botUA string = "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)"
// 31.13.97.0/24, 31.13.99.0/24, 31.13.100.0/24, 66.220.144.0/20, 69.63.189.0/24, 69.63.190.0/24, 69.171.224.0/20, 69.171.240.0/21, 69.171.248.0/24, 173.252.73.0/24, 173.252.74.0/24, 173.252.77.0/24, 173.252.100.0/22, 173.252.104.0/21, 173.252.112.0/24, 2a03:2880:10::/48, 2a03:2880:10ff::/48, 2a03:2880:11::/48, 2a03:2880:11ff::/48, 2a03:2880:20::/48, 2a03:2880:20ff::/48, 2a03:2880:21ff::/48, 2a03:2880:30ff::/48, 2a03:2880:31ff::/48, 2a03:2880:1010::/48, 2a03:2880:1020::/48, 2a03:2880:2020::/48, 2a03:2880:2050::/48, 2a03:2880:2040::/48, 2a03:2880:2110::/48, 2a03:2880:2130::/48, 2a03:2880:3010::/48, 2a03:2880:3020::/48
const botIP string = "31.13.99.8"
const ja3 string = "771,49199-49195-49171-49161-49200-49196-49172-49162-51-57-50-49169-49159-47-53-10-5-4-255,0-11-10-13-13172-16,23-25-28-27-24-26-22-14-13-11-12-9-10,0-1-2"
return masqueradeAsTrustedBot(botUA, botIP, ja3)
}
// MasqueradeAsYandexBot modifies user agent and x-forwarded for
// to appear to be a Yandex Spider Bot
func MasqueradeAsYandexBot() proxychain.RequestModification {
const botUA string = "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)"
// 100.43.90.0/24, 37.9.115.0/24, 37.140.165.0/24, 77.88.22.0/25, 77.88.29.0/24, 77.88.31.0/24, 77.88.59.0/24, 84.201.146.0/24, 84.201.148.0/24, 84.201.149.0/24, 87.250.243.0/24, 87.250.253.0/24, 93.158.147.0/24, 93.158.148.0/24, 93.158.151.0/24, 93.158.153.0/32, 95.108.128.0/24, 95.108.138.0/24, 95.108.150.0/23, 95.108.158.0/24, 95.108.156.0/24, 95.108.188.128/25, 95.108.234.0/24, 95.108.248.0/24, 100.43.80.0/24, 130.193.62.0/24, 141.8.153.0/24, 178.154.165.0/24, 178.154.166.128/25, 178.154.173.29, 178.154.200.158, 178.154.202.0/24, 178.154.205.0/24, 178.154.239.0/24, 178.154.243.0/24, 37.9.84.253, 199.21.99.99, 178.154.162.29, 178.154.203.251, 178.154.211.250, 178.154.171.0/24, 178.154.200.0/24, 178.154.244.0/24, 178.154.246.0/24, 95.108.181.0/24, 95.108.246.252, 5.45.254.0/24, 5.255.253.0/24, 37.140.141.0/24, 37.140.188.0/24, 100.43.81.0/24, 100.43.85.0/24, 100.43.91.0/24, 199.21.99.0/24, 2a02:6b8:b000::/32, 2a02:6b8:b010::/32, 2a02:6b8:b011::/32, 2a02:6b8:c0e::/32
const botIP string = "37.9.115.9"
const ja3 string = "769,49200-49196-49192-49188-49172-49162-165-163-161-159-107-106-105-104-57-56-55-54-136-135-134-133-49202-49198-49194-49190-49167-49157-157-61-53-132-49199-49195-49191-49187-49171-49161-164-162-160-158-103-64-63-62-51-50-49-48-154-153-152-151-69-68-67-66-49201-49197-49193-49189-49166-49156-156-60-47-150-65-7-49169-49159-49164-49154-5-4-49170-49160-22-19-16-13-49165-49155-10-255,0-11-10-35-13-15,23-25-28-27-24-26-22-14-13-11-12-9-10,0-1-2"
return masqueradeAsTrustedBot(botUA, botIP, ja3)
}
// MasqueradeAsBaiduBot modifies user agent and x-forwarded for
// to appear to be a Baidu Spider Bot
func MasqueradeAsBaiduBot() proxychain.RequestModification {
const botUA string = "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)"
// 180.76.15.0/24, 119.63.196.0/24, 115.239.212./24, 119.63.199.0/24, 122.81.208.0/22, 123.125.71.0/24, 180.76.4.0/24, 180.76.5.0/24, 180.76.6.0/24, 185.10.104.0/24, 220.181.108.0/24, 220.181.51.0/24, 111.13.102.0/24, 123.125.67.144/29, 123.125.67.152/31, 61.135.169.0/24, 123.125.68.68/30, 123.125.68.72/29, 123.125.68.80/28, 123.125.68.96/30, 202.46.48.0/20, 220.181.38.0/24, 123.125.68.80/30, 123.125.68.84/31, 123.125.68.0/24
const botIP string = "180.76.15.7"
return masqueradeAsTrustedBot(botUA, botIP, "")
}
// MasqueradeAsDuckDuckBot modifies user agent and x-forwarded for
// to appear to be a DuckDuckGo Bot
func MasqueradeAsDuckDuckBot() proxychain.RequestModification {
const botUA string = "DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)"
// 46.51.197.88, 46.51.197.89, 50.18.192.250, 50.18.192.251, 107.21.1.61, 176.34.131.233, 176.34.135.167, 184.72.106.52, 184.72.115.86
const botIP string = "46.51.197.88"
return masqueradeAsTrustedBot(botUA, botIP, "")
}
// MasqueradeAsYahooBot modifies user agent and x-forwarded for
// to appear to be a Yahoo Bot
func MasqueradeAsYahooBot() proxychain.RequestModification {
const botUA string = "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
// 5.255.250.0/24, 37.9.87.0/24, 67.195.37.0/24, 67.195.50.0/24, 67.195.110.0/24, 67.195.111.0/24, 67.195.112.0/23, 67.195.114.0/24, 67.195.115.0/24, 68.180.224.0/21, 72.30.132.0/24, 72.30.142.0/24, 72.30.161.0/24, 72.30.196.0/24, 72.30.198.0/24, 74.6.254.0/24, 74.6.8.0/24, 74.6.13.0/24, 74.6.17.0/24, 74.6.18.0/24, 74.6.22.0/24, 74.6.27.0/24, 74.6.168.0/24, 77.88.5.0/24, 77.88.47.0/24, 93.158.161.0/24, 98.137.72.0/24, 98.137.206.0/24, 98.137.207.0/24, 98.139.168.0/24, 114.111.95.0/24, 124.83.159.0/24, 124.83.179.0/24, 124.83.223.0/24, 141.8.144.0/24, 183.79.63.0/24, 183.79.92.0/24, 203.216.255.0/24, 211.14.11.0/24
const ja3 = "769,49200-49196-49192-49188-49172-49162-163-159-107-106-57-56-136-135-49202-49198-49194-49190-49167-49157-157-61-53-132-49199-49195-49191-49187-49171-49161-162-158-103-64-51-50-49170-49160-154-153-69-68-22-19-49201-49197-49193-49189-49166-49156-49165-49155-156-60-47-150-65-10-7-49169-49159-49164-49154-5-4-255,0-11-10-13-15,25-24-23,0-1-2"
const botIP string = "37.9.87.5"
return masqueradeAsTrustedBot(botUA, botIP, ja3)
}
func masqueradeAsTrustedBot(botUA string, botIP string, ja3 string) proxychain.RequestModification {
return func(chain *proxychain.ProxyChain) error {
chain.AddOnceRequestModifications(
SpoofUserAgent(botUA),
// general / nginx
SetRequestHeader("X-Forwarded-For", botIP),
SetRequestHeader("X-Real-IP", botIP),
SetRequestHeader("True-Client-IP", botIP),
SetRequestHeader("WL-Proxy-Client-IP", botIP),
SetRequestHeader("X-Cluster-Client-IP", botIP),
/*
// akamai
SetRequestHeader("True-Client-IP", botIP),
// cloudflare
// TODO: this seems to cause issues with CF... figure out workaround or remove
Error 1000
Ray ID: xxxxxxxxxxxxxxxx •
2023-12-01 20:09:22 UTC
DNS points to prohibited IP
What happened?
You've requested a page on a website (xxxxxxxxxxxxxxxxxxx) that is on the Cloudflare network. Unfortunately, it is resolving to an IP address that is creating a conflict within Cloudflare's system
SetRequestHeader("CF-Connecting-IP", botIP),
// weblogic
SetRequestHeader("WL-Proxy-Client-IP", botIP),
// azure
SetRequestHeader("X-Cluster-Client-IP", botIP),
*/
DeleteRequestHeader("referrer"),
DeleteRequestHeader("origin"),
)
/*
if ja3 != "" {
chain.AddOnceRequestModifications(
SpoofJA3fingerprint(ja3, botUA),
)
}
*/
return nil
}
}

View File

@@ -0,0 +1,14 @@
package requestmodifiers
import (
"regexp"
"ladder/proxychain"
)
func ModifyDomainWithRegex(match regexp.Regexp, replacement string) proxychain.RequestModification {
return func(px *proxychain.ProxyChain) error {
px.Request.URL.Host = match.ReplaceAllString(px.Request.URL.Host, replacement)
return nil
}
}

View File

@@ -0,0 +1,100 @@
package requestmodifiers
import (
//"net/http"
//http "github.com/Danny-Dasilva/fhttp"
http "github.com/bogdanfinn/fhttp"
"ladder/proxychain"
)
// SetOutgoingCookie modifes a specific cookie name
// by modifying the request cookie headers going to the upstream server.
// If the cookie name does not already exist, it is created.
func SetOutgoingCookie(name string, val string) proxychain.RequestModification {
return func(chain *proxychain.ProxyChain) error {
cookies := chain.Request.Cookies()
hasCookie := false
for _, cookie := range cookies {
if cookie.Name != name {
continue
}
hasCookie = true
cookie.Value = val
}
if hasCookie {
return nil
}
chain.Request.AddCookie(&http.Cookie{
Domain: chain.Request.URL.Host,
Name: name,
Value: val,
})
return nil
}
}
// SetOutgoingCookies modifies a client request's cookie header
// to a raw Cookie string, overwriting existing cookies
func SetOutgoingCookies(cookies string) proxychain.RequestModification {
return func(chain *proxychain.ProxyChain) error {
chain.Request.Header.Set("Cookies", cookies)
return nil
}
}
// DeleteOutgoingCookie modifies the http request's cookies header to
// delete a specific request cookie going to the upstream server.
// If the cookie does not exist, it does not do anything.
func DeleteOutgoingCookie(name string) proxychain.RequestModification {
return func(chain *proxychain.ProxyChain) error {
cookies := chain.Request.Cookies()
chain.Request.Header.Del("Cookies")
for _, cookie := range cookies {
if cookie.Name == name {
chain.Request.AddCookie(cookie)
}
}
return nil
}
}
// DeleteOutgoingCookies removes the cookie header entirely,
// preventing any cookies from reaching the upstream server.
func DeleteOutgoingCookies() proxychain.RequestModification {
return func(px *proxychain.ProxyChain) error {
px.Request.Header.Del("Cookie")
return nil
}
}
// DeleteOutGoingCookiesExcept prevents non-whitelisted cookies from being sent from the client
// to the upstream proxy server. Cookies whose names are in the whitelist are not removed.
func DeleteOutgoingCookiesExcept(whitelist ...string) proxychain.RequestModification {
return func(px *proxychain.ProxyChain) error {
// Convert whitelist slice to a map for efficient lookups
whitelistMap := make(map[string]struct{})
for _, cookieName := range whitelist {
whitelistMap[cookieName] = struct{}{}
}
// Get all cookies from the request header
cookies := px.Request.Cookies()
// Clear the original Cookie header
px.Request.Header.Del("Cookie")
// Re-add cookies that are in the whitelist
for _, cookie := range cookies {
if _, found := whitelistMap[cookie.Name]; found {
px.Request.AddCookie(cookie)
}
}
return nil
}
}

View File

@@ -0,0 +1,14 @@
package requestmodifiers
import (
"regexp"
"ladder/proxychain"
)
func ModifyPathWithRegex(match regexp.Regexp, replacement string) proxychain.RequestModification {
return func(px *proxychain.ProxyChain) error {
px.Request.URL.Path = match.ReplaceAllString(px.Request.URL.Path, replacement)
return nil
}
}

View File

@@ -0,0 +1,26 @@
package requestmodifiers
import (
"net/url"
"ladder/proxychain"
)
// ModifyQueryParams replaces query parameter values in URL's query params in a ProxyChain's URL.
// If the query param key doesn't exist, it is created.
func ModifyQueryParams(key string, value string) proxychain.RequestModification {
return func(chain *proxychain.ProxyChain) error {
q := chain.Request.URL.Query()
chain.Request.URL.RawQuery = modifyQueryParams(key, value, q)
return nil
}
}
func modifyQueryParams(key string, value string, q url.Values) string {
if value == "" {
q.Del(key)
return q.Encode()
}
q.Set(key, value)
return q.Encode()
}

View File

@@ -0,0 +1,23 @@
package requestmodifiers
import (
"ladder/proxychain"
)
// SetRequestHeader modifies a specific outgoing header
// This is the header that the upstream server will see.
func SetRequestHeader(name string, val string) proxychain.RequestModification {
return func(px *proxychain.ProxyChain) error {
px.Request.Header.Set(name, val)
return nil
}
}
// DeleteRequestHeader modifies a specific outgoing header
// This is the header that the upstream server will see.
func DeleteRequestHeader(name string) proxychain.RequestModification {
return func(px *proxychain.ProxyChain) error {
px.Request.Header.Del(name)
return nil
}
}

View File

@@ -0,0 +1,47 @@
package requestmodifiers
import (
"fmt"
"net/url"
"regexp"
"ladder/proxychain"
tx "ladder/proxychain/responsemodifiers"
)
const archivistUrl string = "https://archive.is/latest"
// RequestArchiveIs modifies a ProxyChain's URL to request an archived version from archive.is
func RequestArchiveIs() proxychain.RequestModification {
return func(chain *proxychain.ProxyChain) error {
rURL := preventRecursiveArchivistURLs(chain.Request.URL.String())
chain.Request.URL.RawQuery = ""
newURL, err := url.Parse(fmt.Sprintf("%s/%s", archivistUrl, rURL))
if err != nil {
return err
}
// archivist seems to sabotage requests from cloudflare's DNS
// bypass this just in case
chain.AddOnceRequestModifications(ResolveWithGoogleDoH())
chain.Request.URL = newURL
// cleanup archivst headers
script := `[...document.querySelector("body > center").childNodes].filter(e => e.id != "SOLID").forEach(e => e.remove())`
chain.AddOnceResponseModifications(
tx.InjectScriptAfterDOMContentLoaded(script),
)
return nil
}
}
// https://archive.is/20200421201055/https://rt.live/ -> http://rt.live/
func preventRecursiveArchivistURLs(url string) string {
re := regexp.MustCompile(`https?:\/\/archive\.is\/\d+\/(https?:\/\/.*)`)
match := re.FindStringSubmatch(url)
if match != nil {
return match[1]
}
return url
}

View File

@@ -0,0 +1,22 @@
package requestmodifiers
import (
"net/url"
"ladder/proxychain"
)
const googleCacheUrl string = "https://webcache.googleusercontent.com/search?q=cache:"
// RequestGoogleCache modifies a ProxyChain's URL to request its Google Cache version.
func RequestGoogleCache() proxychain.RequestModification {
return func(px *proxychain.ProxyChain) error {
encodedURL := url.QueryEscape(px.Request.URL.String())
newURL, err := url.Parse(googleCacheUrl + encodedURL)
if err != nil {
return err
}
px.Request.URL = newURL
return nil
}
}

View File

@@ -0,0 +1,43 @@
package requestmodifiers
import (
"net/url"
"regexp"
"ladder/proxychain"
tx "ladder/proxychain/responsemodifiers"
)
const waybackUrl string = "https://web.archive.org/web/"
// RequestWaybackMachine modifies a ProxyChain's URL to request the wayback machine (archive.org) version.
func RequestWaybackMachine() proxychain.RequestModification {
return func(chain *proxychain.ProxyChain) error {
chain.Request.URL.RawQuery = ""
rURL := preventRecursiveWaybackURLs(chain.Request.URL.String())
newURLString := waybackUrl + rURL
newURL, err := url.Parse(newURLString)
if err != nil {
return err
}
chain.Request.URL = newURL
// cleanup wayback headers
script := `["wm-ipp-print", "wm-ipp-base"].forEach(id => { try { document.getElementById(id).remove() } catch{ } })`
chain.AddOnceResponseModifications(
tx.InjectScriptAfterDOMContentLoaded(script),
)
return nil
}
}
func preventRecursiveWaybackURLs(url string) string {
re := regexp.MustCompile(`https:\/\/web\.archive\.org\/web\/\d+\/\*(https?:\/\/.*)`)
match := re.FindStringSubmatch(url)
if match != nil {
return match[1]
}
return url
}

View File

@@ -0,0 +1,94 @@
package requestmodifiers
import (
"context"
"encoding/json"
"fmt"
"net"
"time"
http "github.com/bogdanfinn/fhttp"
/*
tls_client "github.com/bogdanfinn/tls-client"
//"net/http"
*/
"ladder/proxychain"
)
// resolveWithGoogleDoH resolves DNS using Google's DNS-over-HTTPS
func resolveWithGoogleDoH(host string) (string, error) {
url := "https://dns.google/resolve?name=" + host + "&type=A"
resp, err := http.Get(url)
if err != nil {
return "", err
}
defer resp.Body.Close()
var result struct {
Answer []struct {
Data string `json:"data"`
} `json:"Answer"`
}
err = json.NewDecoder(resp.Body).Decode(&result)
if err != nil {
return "", err
}
// Get the first A record
if len(result.Answer) > 0 {
return result.Answer[0].Data, nil
}
return "", fmt.Errorf("no DoH DNS record found for %s", host)
}
type CustomDialer struct {
*net.Dialer
}
func NewCustomDialer(timeout, keepAlive time.Duration) *CustomDialer {
return &CustomDialer{
Dialer: &net.Dialer{
Timeout: timeout,
KeepAlive: keepAlive,
},
}
}
func (cd *CustomDialer) DialContext(ctx context.Context, network, addr string) (net.Conn, error) {
host, port, err := net.SplitHostPort(addr)
if err != nil {
port = "443"
}
resolvedHost, err := resolveWithGoogleDoH(host)
if err != nil {
return nil, err
}
return cd.Dialer.DialContext(ctx, network, net.JoinHostPort(resolvedHost, port))
}
// ResolveWithGoogleDoH modifies a ProxyChain's client to make the request by resolving the URL
// using Google's DNS over HTTPs service
func ResolveWithGoogleDoH() proxychain.RequestModification {
///customDialer := NewCustomDialer(10*time.Second, 10*time.Second)
return func(chain *proxychain.ProxyChain) error {
/*
options := []tls_client.HttpClientOption{
tls_client.WithTimeoutSeconds(30),
tls_client.WithRandomTLSExtensionOrder(),
tls_client.WithDialer(*customDialer.Dialer),
//tls_client.WithClientProfile(profiles.Chrome_105),
}
client, err := tls_client.NewHttpClient(tls_client.NewNoopLogger(), options...)
if err != nil {
return err
}
chain.SetOnceHTTPClient(client)
*/
return nil
}
}

View File

@@ -0,0 +1,52 @@
package requestmodifiers
// removed due to using a different TLS spoofing technique
/*
import (
//"github.com/Danny-Dasilva/CycleTLS/cycletls"
//http "github.com/Danny-Dasilva/fhttp"
//http "github.com/bogdanfinn/fhttp"
"golang.org/x/net/proxy"
"ladder/proxychain"
)
// SpoofJA3fingerprint modifies the TLS client and user agent to spoof a particular JA3 fingerprint
// Some anti-bot WAFs such as cloudflare can fingerprint the fields of the TLS hello packet, and the order in which they appear
// https://web.archive.org/web/20231126224326/https://engineering.salesforce.com/tls-fingerprinting-with-ja3-and-ja3s-247362855967/
// https://web.archive.org/web/20231119065253/https://developers.cloudflare.com/bots/concepts/ja3-fingerprint/
func SpoofJA3fingerprint(ja3 string, userAgent string) proxychain.RequestModification {
//fmt.Println(ja3)
return func(chain *proxychain.ProxyChain) error {
// deep copy existing client while modifying http transport
ja3SpoofClient := &http.Client{
Transport: cycletls.NewTransport(ja3, userAgent),
Timeout: chain.Client.Timeout,
CheckRedirect: chain.Client.CheckRedirect,
}
chain.SetOnceHTTPClient(ja3SpoofClient)
return nil
}
}
// SpoofJA3fingerprintWithProxy modifies the TLS client and user agent to spoof a particular JA3 fingerprint and use a proxy.ContextDialer from the "golang.org/x/net/proxy"
// Some anti-bot WAFs such as cloudflare can fingerprint the fields of the TLS hello packet, and the order in which they appear
// https://web.archive.org/web/20231126224326/https://engineering.salesforce.com/tls-fingerprinting-with-ja3-and-ja3s-247362855967/
// https://web.archive.org/web/20231119065253/https://developers.cloudflare.com/bots/concepts/ja3-fingerprint/
func SpoofJA3fingerprintWithProxy(ja3 string, userAgent string, proxy proxy.ContextDialer) proxychain.RequestModification {
return func(chain *proxychain.ProxyChain) error {
// deep copy existing client while modifying http transport
ja3SpoofClient := &http.Client{
Transport: cycletls.NewTransportWithProxy(ja3, userAgent, proxy),
Timeout: chain.Client.Timeout,
CheckRedirect: chain.Client.CheckRedirect,
}
chain.SetOnceHTTPClient(ja3SpoofClient)
return nil
}
}
*/

View File

@@ -0,0 +1,24 @@
package requestmodifiers
import (
"ladder/proxychain"
)
// SpoofOrigin modifies the origin header
// if the upstream server returns a Vary header
// it means you might get a different response if you change this
func SpoofOrigin(url string) proxychain.RequestModification {
return func(px *proxychain.ProxyChain) error {
px.Request.Header.Set("origin", url)
return nil
}
}
// HideOrigin modifies the origin header
// so that it is the original origin, not the proxy
func HideOrigin() proxychain.RequestModification {
return func(px *proxychain.ProxyChain) error {
px.Request.Header.Set("origin", px.Request.URL.String())
return nil
}
}

View File

@@ -0,0 +1,38 @@
package requestmodifiers
import (
"fmt"
"ladder/proxychain"
tx "ladder/proxychain/responsemodifiers"
)
// SpoofReferrer modifies the referrer header.
// It is useful if the page can be accessed from a search engine
// or social media site, but not by browsing the website itself.
// if url is "", then the referrer header is removed.
func SpoofReferrer(url string) proxychain.RequestModification {
return func(chain *proxychain.ProxyChain) error {
// change refer on client side js
script := fmt.Sprintf(`document.referrer = "%s"`, url)
chain.AddOnceResponseModifications(
tx.InjectScriptBeforeDOMContentLoaded(script),
)
if url == "" {
chain.Request.Header.Del("referrer")
return nil
}
chain.Request.Header.Set("referrer", url)
return nil
}
}
// HideReferrer modifies the referrer header
// so that it is the original referrer, not the proxy
func HideReferrer() proxychain.RequestModification {
return func(px *proxychain.ProxyChain) error {
px.Request.Header.Set("referrer", px.Request.URL.String())
return nil
}
}

View File

@@ -0,0 +1,43 @@
package requestmodifiers
import (
"fmt"
"math/rand"
"strings"
"time"
"ladder/proxychain"
)
// SpoofReferrerFromBaiduSearch modifies the referrer header
// pretending to be from a BaiduSearch
func SpoofReferrerFromBaiduSearch() proxychain.RequestModification {
return func(chain *proxychain.ProxyChain) error {
// https://www.baidu.com/link?url=5biIeDvUIihawf3Zbbysach2Xn4H3w3FzO6LZKgSs-B5Yt4M4RUFikokOk5zetf2&wd=&eqid=9da80d8208009b8480000706655d5ed6
referrer := fmt.Sprintf("https://baidu.com/link?url=%s", generateRandomBaiduURL())
chain.Request.Header.Set("referrer", referrer)
chain.Request.Header.Set("sec-fetch-site", "cross-site")
chain.Request.Header.Set("sec-fetch-dest", "document")
chain.Request.Header.Set("sec-fetch-mode", "navigate")
return nil
}
}
// utility functions ==================
func generateRandomString(charset string, length int) string {
var seededRand *rand.Rand = rand.New(rand.NewSource(time.Now().UnixNano()))
var stringBuilder strings.Builder
for i := 0; i < length; i++ {
stringBuilder.WriteByte(charset[seededRand.Intn(len(charset))])
}
return stringBuilder.String()
}
func generateRandomBaiduURL() string {
const alphanumericCharset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
const hexCharset = "0123456789abcdef"
randomAlphanumeric := generateRandomString(alphanumericCharset, 30) // Length before "-"
randomHex := generateRandomString(hexCharset, 16) // Length of eqid
return randomAlphanumeric + "-" + "&wd=&eqid=" + randomHex
}

View File

@@ -0,0 +1,20 @@
package requestmodifiers
import (
"ladder/proxychain"
)
// SpoofReferrerFromBingSearch modifies the referrer header
// pretending to be from a bing search site
func SpoofReferrerFromBingSearch() proxychain.RequestModification {
return func(chain *proxychain.ProxyChain) error {
chain.AddOnceRequestModifications(
SpoofReferrer("https://www.bing.com/"),
SetRequestHeader("sec-fetch-site", "cross-site"),
SetRequestHeader("sec-fetch-dest", "document"),
SetRequestHeader("sec-fetch-mode", "navigate"),
ModifyQueryParams("utm_source", "bing"),
)
return nil
}
}

View File

@@ -0,0 +1,20 @@
package requestmodifiers
import (
"ladder/proxychain"
)
// SpoofReferrerFromGoogleSearch modifies the referrer header
// pretending to be from a google search site
func SpoofReferrerFromGoogleSearch() proxychain.RequestModification {
return func(chain *proxychain.ProxyChain) error {
chain.AddOnceRequestModifications(
SpoofReferrer("https://www.google.com"),
SetRequestHeader("sec-fetch-site", "cross-site"),
SetRequestHeader("sec-fetch-dest", "document"),
SetRequestHeader("sec-fetch-mode", "navigate"),
ModifyQueryParams("utm_source", "google"),
)
return nil
}
}

View File

@@ -0,0 +1,21 @@
package requestmodifiers
import (
"ladder/proxychain"
)
// SpoofReferrerFromLinkedInPost modifies the referrer header
// pretending to be from a linkedin post
func SpoofReferrerFromLinkedInPost() proxychain.RequestModification {
return func(chain *proxychain.ProxyChain) error {
chain.AddOnceRequestModifications(
SpoofReferrer("https://www.linkedin.com/"),
SetRequestHeader("sec-fetch-site", "cross-site"),
SetRequestHeader("sec-fetch-dest", "document"),
SetRequestHeader("sec-fetch-mode", "navigate"),
ModifyQueryParams("utm_campaign", "post"),
ModifyQueryParams("utm_medium", "web"),
)
return nil
}
}

View File

@@ -0,0 +1,23 @@
package requestmodifiers
import (
"fmt"
"ladder/proxychain"
)
// SpoofReferrerFromNaverSearch modifies the referrer header
// pretending to be from a Naver search (popular in South Korea)
func SpoofReferrerFromNaverSearch() proxychain.RequestModification {
return func(chain *proxychain.ProxyChain) error {
referrer := fmt.Sprintf(
"https://search.naver.com/search.naver?where=nexearch&sm=top_hty&fbm=0&ie=utf8&query=%s",
chain.Request.URL.Host,
)
chain.Request.Header.Set("referrer", referrer)
chain.Request.Header.Set("sec-fetch-site", "cross-site")
chain.Request.Header.Set("sec-fetch-dest", "document")
chain.Request.Header.Set("sec-fetch-mode", "navigate")
return nil
}
}

View File

@@ -0,0 +1,17 @@
package requestmodifiers
import (
"ladder/proxychain"
)
// SpoofReferrerFromPinterestPost modifies the referrer header
// pretending to be from a pinterest post
func SpoofReferrerFromPinterestPost() proxychain.RequestModification {
return func(chain *proxychain.ProxyChain) error {
chain.Request.Header.Set("referrer", "https://www.pinterest.com/")
chain.Request.Header.Set("sec-fetch-site", "cross-site")
chain.Request.Header.Set("sec-fetch-dest", "document")
chain.Request.Header.Set("sec-fetch-mode", "navigate")
return nil
}
}

View File

@@ -0,0 +1,16 @@
package requestmodifiers
import (
"ladder/proxychain"
)
// SpoofReferrerFromQQPost modifies the referrer header
// pretending to be from a QQ post (popular social media in China)
func SpoofReferrerFromQQPost() proxychain.RequestModification {
return func(chain *proxychain.ProxyChain) error {
chain.Request.Header.Set("referrer", "https://new.qq.com/")
chain.Request.Header.Set("sec-fetch-site", "cross-site")
chain.Request.Header.Set("sec-fetch-dest", "document")
return nil
}
}

View File

@@ -0,0 +1,17 @@
package requestmodifiers
import (
"ladder/proxychain"
)
// SpoofReferrerFromRedditPost modifies the referrer header
// pretending to be from a reddit post
func SpoofReferrerFromRedditPost() proxychain.RequestModification {
return func(chain *proxychain.ProxyChain) error {
chain.Request.Header.Set("referrer", "https://www.reddit.com/")
chain.Request.Header.Set("sec-fetch-site", "cross-site")
chain.Request.Header.Set("sec-fetch-dest", "document")
chain.Request.Header.Set("sec-fetch-mode", "navigate")
return nil
}
}

View File

@@ -0,0 +1,19 @@
package requestmodifiers
import (
"ladder/proxychain"
)
// SpoofReferrerFromTumblrPost modifies the referrer header
// pretending to be from a tumblr post
func SpoofReferrerFromTumblrPost() proxychain.RequestModification {
return func(chain *proxychain.ProxyChain) error {
chain.AddOnceRequestModifications(
SpoofReferrer("https://www.tumblr.com/"),
SetRequestHeader("sec-fetch-site", "cross-site"),
SetRequestHeader("sec-fetch-dest", "document"),
SetRequestHeader("sec-fetch-mode", "navigate"),
)
return nil
}
}

View File

@@ -0,0 +1,19 @@
package requestmodifiers
import (
"ladder/proxychain"
)
// SpoofReferrerFromTwitterPost modifies the referrer header
// pretending to be from a twitter post
func SpoofReferrerFromTwitterPost() proxychain.RequestModification {
return func(chain *proxychain.ProxyChain) error {
chain.AddOnceRequestModifications(
SpoofReferrer("https://t.co/"),
SetRequestHeader("sec-fetch-site", "cross-site"),
SetRequestHeader("sec-fetch-dest", "document"),
SetRequestHeader("sec-fetch-mode", "navigate"),
)
return nil
}
}

View File

@@ -0,0 +1,19 @@
package requestmodifiers
import (
"ladder/proxychain"
)
// SpoofReferrerFromVkontaktePost modifies the referrer header
// pretending to be from a vkontakte post (popular in Russia)
func SpoofReferrerFromVkontaktePost() proxychain.RequestModification {
return func(chain *proxychain.ProxyChain) error {
chain.AddOnceRequestModifications(
SpoofReferrer("https://away.vk.com/"),
SetRequestHeader("sec-fetch-site", "cross-site"),
SetRequestHeader("sec-fetch-dest", "document"),
SetRequestHeader("sec-fetch-mode", "navigate"),
)
return nil
}
}

View File

@@ -0,0 +1,21 @@
package requestmodifiers
import (
"fmt"
"math/rand"
"ladder/proxychain"
)
// SpoofReferrerFromWeiboPost modifies the referrer header
// pretending to be from a Weibo post (popular in China)
func SpoofReferrerFromWeiboPost() proxychain.RequestModification {
return func(chain *proxychain.ProxyChain) error {
referrer := fmt.Sprintf("http://weibo.com/u/%d", rand.Intn(90001))
chain.Request.Header.Set("referrer", referrer)
chain.Request.Header.Set("sec-fetch-site", "cross-site")
chain.Request.Header.Set("sec-fetch-dest", "document")
chain.Request.Header.Set("sec-fetch-mode", "navigate")
return nil
}
}

View File

@@ -0,0 +1,40 @@
package requestmodifiers
import (
_ "embed"
"strings"
"ladder/proxychain"
tx "ladder/proxychain/responsemodifiers"
)
// https://github.com/faisalman/ua-parser-js/tree/master
// update using:
// git submodule update --remote --merge
//
//go:embed vendor/ua-parser-js/dist/ua-parser.min.js
var UAParserJS string
// note: spoof_user_agent.js has a dependency on ua-parser.min.js
// ua-parser.min.js should be loaded first.
//
//go:embed spoof_user_agent.js
var spoofUserAgentJS string
// SpoofUserAgent modifies the user agent
func SpoofUserAgent(ua string) proxychain.RequestModification {
return func(chain *proxychain.ProxyChain) error {
// modify ua headers
chain.AddOnceRequestModifications(
SetRequestHeader("user-agent", ua),
)
script := strings.ReplaceAll(spoofUserAgentJS, "{{USER_AGENT}}", ua)
chain.AddOnceResponseModifications(
tx.InjectScriptBeforeDOMContentLoaded(script),
tx.InjectScriptBeforeDOMContentLoaded(UAParserJS),
)
return nil
}
}

View File

@@ -0,0 +1,100 @@
(() => {
const UA = "{{USER_AGENT}}";
// monkey-patch navigator.userAgent
{
const { get } = Object.getOwnPropertyDescriptor(
Navigator.prototype,
"userAgent",
);
Object.defineProperty(Navigator.prototype, "userAgent", {
get: new Proxy(get, {
apply() {
return UA;
},
}),
});
}
// monkey-patch navigator.appVersion
{
const { get } = Object.getOwnPropertyDescriptor(
Navigator.prototype,
"appVersion",
);
Object.defineProperty(Navigator.prototype, "appVersion", {
get: new Proxy(get, {
apply() {
return UA.replace("Mozilla/", "");
},
}),
});
}
// monkey-patch navigator.UserAgentData
// Assuming UAParser is already loaded and available
function spoofUserAgentData(uaString) {
// Parse the user-agent string
const parser = new UAParser(uaString);
const parsedData = parser.getResult();
// Extracted data
const platform = parsedData.os.name;
const browserName = parsedData.browser.name;
const browserMajorVersion = parsedData.browser.major;
const isMobile =
/Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test(
uaString,
);
// Overwrite navigator.userAgentData
self.NavigatorUAData = self.NavigatorUAData || new class NavigatorUAData {
brands = [{
brand: browserName,
version: browserMajorVersion,
}];
mobile = isMobile;
platform = platform;
toJSON() {
return {
brands: this.brands,
mobile: this.mobile,
platform: this.platform,
};
}
getHighEntropyValues(hints) {
const result = this.toJSON();
// Add additional high entropy values based on hints
// Modify these as per your requirements
if (hints.includes("architecture")) {
result.architecture = "x86";
}
if (hints.includes("bitness")) {
result.bitness = "64";
}
if (hints.includes("model")) {
result.model = "";
}
if (hints.includes("platformVersion")) {
result.platformVersion = "10.0.0"; // Example value
}
if (hints.includes("uaFullVersion")) {
result.uaFullVersion = browserMajorVersion;
}
if (hints.includes("fullVersionList")) {
result.fullVersionList = this.brands;
}
return Promise.resolve(result);
}
}();
// Apply the monkey patch
Object.defineProperty(navigator, "userAgentData", {
value: new self.NavigatorUAData(),
writable: false,
});
}
spoofUserAgentData(UA);
// TODO: use hideMonkeyPatch to hide overrides
})();

View File

@@ -0,0 +1,14 @@
package requestmodifiers
import (
"ladder/proxychain"
)
// SpoofXForwardedFor modifies the X-Forwarded-For header
// in some cases, a forward proxy may interpret this as the source IP
func SpoofXForwardedFor(ip string) proxychain.RequestModification {
return func(px *proxychain.ProxyChain) error {
px.Request.Header.Set("X-FORWARDED-FOR", ip)
return nil
}
}