From 59cf0317ecfb6f1c7dc3be14202eaf9f9378db31 Mon Sep 17 00:00:00 2001 From: Kevin Pham Date: Mon, 27 Nov 2023 12:00:12 -0600 Subject: [PATCH] improve ua modifier with client side spoofing --- .gitignore | 3 +- .gitmodules | 3 + handlers/proxy.go | 3 +- proxychain/proxychain.go | 109 +++--- .../masquerade_as_trusted_bot.go | 63 +++- .../modify_domain_with_regex.go | 3 +- .../modify_outgoing_cookies.go | 3 +- .../requestmodifers/modify_path_with_regex.go | 3 +- .../requestmodifers/modify_query_params.go | 9 +- .../requestmodifers/request_archive_is.go | 17 +- .../requestmodifers/request_google_cache.go | 3 +- .../request_wayback_machine.go | 3 +- .../resolve_with_google_doh.go | 3 +- proxychain/requestmodifers/spoof_referrer.go | 23 +- .../spoof_referrer_from_baidu_post.go | 3 +- .../spoof_referrer_from_bing_search.go | 2 +- .../spoof_referrer_from_google_search.go | 12 +- .../spoof_referrer_from_linkedin_post.go | 2 +- .../spoof_referrer_from_naver_post.go | 1 + .../spoof_referrer_from_tumblr_post.go | 2 +- .../spoof_referrer_from_twitter_post.go | 2 +- .../spoof_referrer_from_vkontake_post.go | 2 +- .../spoof_referrer_from_weibo_post.go | 3 +- .../requestmodifers/spoof_user_agent.go | 30 +- .../requestmodifers/spoof_user_agent.js | 100 ++++++ .../requestmodifers/spoof_x_forwarded_for.go | 2 +- .../requestmodifers/vendor/ua-parser-js | 1 + proxychain/responsemodifers/inject_script.go | 9 +- .../modify_incoming_cookies.go | 3 +- .../patch_dynamic_resource_urls.go | 3 +- .../patch_dynamic_resource_urls.js | 315 +++++++++--------- .../rewrite_http_resource_urls.go | 3 +- .../rewriters/html_rewriter.go | 1 - .../rewriters/html_token_url_rewriter.go | 20 +- 34 files changed, 490 insertions(+), 274 deletions(-) create mode 100644 .gitmodules create mode 100644 proxychain/requestmodifers/spoof_user_agent.js create mode 160000 proxychain/requestmodifers/vendor/ua-parser-js diff --git a/.gitignore b/.gitignore index 212e502..7f05744 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ ladder VERSION -output.css \ No newline at end of file +output.css +.aider* diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..66e57b4 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "proxychain/requestmodifers/vendor/ua-parser-js"] + path = proxychain/requestmodifers/vendor/ua-parser-js + url = https://github.com/faisalman/ua-parser-js.git diff --git a/handlers/proxy.go b/handlers/proxy.go index 196353a..838cf93 100644 --- a/handlers/proxy.go +++ b/handlers/proxy.go @@ -31,8 +31,9 @@ func NewProxySiteHandler(opts *ProxyOptions) fiber.Handler { SetFiberCtx(c). SetDebugLogging(opts.Verbose). SetRequestModifications( + rx.MasqueradeAsGoogleBot(), rx.DeleteOutgoingCookies(), - //rx.RequestArchiveIs(), + // rx.RequestArchiveIs(), rx.MasqueradeAsGoogleBot(), ). AddResponseModifications( diff --git a/proxychain/proxychain.go b/proxychain/proxychain.go index f9acd05..738aa00 100644 --- a/proxychain/proxychain.go +++ b/proxychain/proxychain.go @@ -83,18 +83,18 @@ proxychain.NewProxyChain(). └─────────┘ └────────────────────────┘ └─────────┘ */ type ProxyChain struct { - Context *fiber.Ctx - Client *http.Client - Request *http.Request - Response *http.Response - requestModifications []RequestModification - onceRequestModifications []RequestModification - onceResultModifications []ResponseModification - resultModifications []ResponseModification - htmlTokenRewriters []rr.IHTMLTokenRewriter - Ruleset *ruleset.RuleSet - debugMode bool - abortErr error + Context *fiber.Ctx + Client *http.Client + Request *http.Request + Response *http.Response + requestModifications []RequestModification + onceRequestModifications []RequestModification + onceResponseModifications []ResponseModification + resultModifications []ResponseModification + htmlTokenRewriters []rr.IHTMLTokenRewriter + Ruleset *ruleset.RuleSet + debugMode bool + abortErr error } // a ProxyStrategy is a pre-built proxychain with purpose-built defaults @@ -122,17 +122,17 @@ func (chain *ProxyChain) AddRequestModifications(mods ...RequestModification) *P return chain } -// AddOnceRequestModification adds a request modifier to the ProxyChain that should only fire once +// AddOnceRequestModifications adds a request modifier to the ProxyChain that should only fire once // the modifier will not fire until ProxyChain.Execute() is run and will be removed after it has been applied. -func (chain *ProxyChain) AddOnceRequestModification(mod ...RequestModification) *ProxyChain { - chain.onceRequestModifications = append(chain.onceRequestModifications, mod...) +func (chain *ProxyChain) AddOnceRequestModifications(mods ...RequestModification) *ProxyChain { + chain.onceRequestModifications = append(chain.onceRequestModifications, mods...) return chain } -// AddOnceResponseModification adds a response modifier to the ProxyChain that should only fire once +// AddOnceResponseModifications adds a response modifier to the ProxyChain that should only fire once // the modifier will not fire until ProxyChain.Execute() is run and will be removed after it has been applied. -func (chain *ProxyChain) AddOnceResponseModification(mod ...ResponseModification) *ProxyChain { - chain.onceResultModifications = append(chain.onceResultModifications, mod...) +func (chain *ProxyChain) AddOnceResponseModifications(mods ...ResponseModification) *ProxyChain { + chain.onceResponseModifications = append(chain.onceResponseModifications, mods...) return chain } @@ -188,26 +188,25 @@ func (chain *ProxyChain) _initialize_request() (*http.Request, error) { return req, nil } -// reconstructUrlFromReferer reconstructs the URL using the referer's scheme, host, and the relative path / queries -func reconstructUrlFromReferer(referer *url.URL, relativeUrl *url.URL) (*url.URL, error) { - +// reconstructURLFromReferer reconstructs the URL using the referer's scheme, host, and the relative path / queries +func reconstructURLFromReferer(referer *url.URL, relativeURL *url.URL) (*url.URL, error) { // Extract the real url from referer path - realUrl, err := url.Parse(strings.TrimPrefix(referer.Path, "/")) + realURL, err := url.Parse(strings.TrimPrefix(referer.Path, "/")) if err != nil { return nil, fmt.Errorf("error parsing real URL from referer '%s': %v", referer.Path, err) } - if realUrl.Scheme == "" || realUrl.Host == "" { - return nil, fmt.Errorf("invalid referer URL: '%s' on request '%s", referer.String(), relativeUrl.String()) + if realURL.Scheme == "" || realURL.Host == "" { + return nil, fmt.Errorf("invalid referer URL: '%s' on request '%s", referer.String(), relativeURL.String()) } - log.Printf("rewrite relative URL using referer: '%s' -> '%s'\n", relativeUrl.String(), realUrl.String()) + log.Printf("rewrite relative URL using referer: '%s' -> '%s'\n", relativeURL.String(), realURL.String()) return &url.URL{ Scheme: referer.Scheme, Host: referer.Host, - Path: realUrl.Path, - RawQuery: realUrl.RawQuery, + Path: realURL.Path, + RawQuery: realURL.RawQuery, }, nil } @@ -227,27 +226,27 @@ func preventRecursiveProxyRequest(urlQuery *url.URL, baseProxyURL string) *url.U return preventRecursiveProxyRequest(fixedURL, baseProxyURL) } -// extractUrl extracts a URL from the request ctx. If the URL in the request +// extractURL extracts a URL from the request ctx. If the URL in the request // is a relative path, it reconstructs the full URL using the referer header. -func (chain *ProxyChain) extractUrl() (*url.URL, error) { - reqUrl := chain.Context.Params("*") +func (chain *ProxyChain) extractURL() (*url.URL, error) { + reqURL := chain.Context.Params("*") // sometimes client requests doubleroot '//' // there is a bug somewhere else, but this is a workaround until we find it - if strings.HasPrefix(reqUrl, "/") || strings.HasPrefix(reqUrl, `%2F`) { - reqUrl = strings.TrimPrefix(reqUrl, "/") - reqUrl = strings.TrimPrefix(reqUrl, `%2F`) + if strings.HasPrefix(reqURL, "/") || strings.HasPrefix(reqURL, `%2F`) { + reqURL = strings.TrimPrefix(reqURL, "/") + reqURL = strings.TrimPrefix(reqURL, `%2F`) } // unescape url query - uReqUrl, err := url.QueryUnescape(reqUrl) + uReqURL, err := url.QueryUnescape(reqURL) if err == nil { - reqUrl = uReqUrl + reqURL = uReqURL } - urlQuery, err := url.Parse(reqUrl) + urlQuery, err := url.Parse(reqURL) if err != nil { - return nil, fmt.Errorf("error parsing request URL '%s': %v", reqUrl, err) + return nil, fmt.Errorf("error parsing request URL '%s': %v", reqURL, err) } // prevent recursive proxy requests @@ -269,7 +268,7 @@ func (chain *ProxyChain) extractUrl() (*url.URL, error) { if err != nil { return nil, fmt.Errorf("error parsing referer URL from req: '%s': %v", relativePath, err) } - return reconstructUrlFromReferer(referer, relativePath) + return reconstructURLFromReferer(referer, relativePath) } // SetFiberCtx takes the request ctx from the client @@ -288,7 +287,7 @@ func (chain *ProxyChain) SetFiberCtx(ctx *fiber.Ctx) *ProxyChain { chain.Request = req // extract the URL for the request and add it to the new request - url, err := chain.extractUrl() + url, err := chain.extractURL() if err != nil { chain.abortErr = chain.abort(err) } @@ -307,9 +306,9 @@ func (chain *ProxyChain) validateCtxIsSet() error { return chain.abortErr } -// SetHttpClient sets a new upstream http client transport +// SetHTTPClient sets a new upstream http client transport // useful for modifying TLS -func (chain *ProxyChain) SetHttpClient(httpClient *http.Client) *ProxyChain { +func (chain *ProxyChain) SetHTTPClient(httpClient *http.Client) *ProxyChain { chain.Client = httpClient return chain } @@ -325,7 +324,7 @@ func (chain *ProxyChain) SetDebugLogging(isDebugMode bool) *ProxyChain { // this will prevent Execute from firing and reset the state // returns the initial error enriched with context func (chain *ProxyChain) abort(err error) error { - //defer chain._reset() + // defer chain._reset() chain.abortErr = err chain.Context.Response().SetStatusCode(500) e := fmt.Errorf("ProxyChain error for '%s': %s", chain.Request.URL.String(), err.Error()) @@ -338,7 +337,7 @@ func (chain *ProxyChain) abort(err error) error { func (chain *ProxyChain) _reset() { chain.abortErr = nil chain.Request = nil - //chain.Response = nil + // chain.Response = nil chain.Context = nil } @@ -373,22 +372,14 @@ func (chain *ProxyChain) _execute() (io.Reader, error) { } } - // Apply onceRequestModifications and onceResultModifications to proxychain and clear them + // Apply onceRequestModifications to proxychain and clear them for _, applyOnceRequestModificationsTo := range chain.onceRequestModifications { err := applyOnceRequestModificationsTo(chain) if err != nil { return nil, chain.abort(err) } } - chain.onceRequestModifications = nil - - for _, applyOnceResultModificationsTo := range chain.onceResultModifications { - err := applyOnceResultModificationsTo(chain) - if err != nil { - return nil, chain.abort(err) - } - } - chain.onceResultModifications = nil + chain.onceRequestModifications = []RequestModification{} // Send Request Upstream resp, err := chain.Client.Do(chain.Request) @@ -411,8 +402,16 @@ func (chain *ProxyChain) _execute() (io.Reader, error) { } } - return chain.Response.Body, nil + // Apply onceResponseModifications to proxychain and clear them + for _, applyOnceResponseModificationsTo := range chain.onceResponseModifications { + err := applyOnceResponseModificationsTo(chain) + if err != nil { + return nil, chain.abort(err) + } + } + chain.onceResponseModifications = []ResponseModification{} + return chain.Response.Body, nil } // Execute sends the request for the ProxyChain and returns the request to the sender @@ -434,5 +433,5 @@ func (chain *ProxyChain) Execute() error { chain.Context.Set("content-type", chain.Response.Header.Get("content-type")) return chain.Context.SendStream(body) - //return chain.Context.SendStream(body) + // return chain.Context.SendStream(body) } diff --git a/proxychain/requestmodifers/masquerade_as_trusted_bot.go b/proxychain/requestmodifers/masquerade_as_trusted_bot.go index bb0e0f8..3ae51e1 100644 --- a/proxychain/requestmodifers/masquerade_as_trusted_bot.go +++ b/proxychain/requestmodifers/masquerade_as_trusted_bot.go @@ -20,12 +20,67 @@ func MasqueradeAsBingBot() proxychain.RequestModification { return masqueradeAsTrustedBot(botUA, botIP) } +// MasqueradeAsWaybackMachineBot modifies user agent and x-forwarded for +// to appear to be a archive.org (wayback machine) Bot +func MasqueradeAsWaybackMachineBot() proxychain.RequestModification { + const botUA string = "Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)" + const botIP string = "207.241.235.164" + return masqueradeAsTrustedBot(botUA, botIP) +} + +// MasqueradeAsFacebookBot modifies user agent and x-forwarded for +// to appear to be a Facebook Bot (link previews?) +func MasqueradeAsFacebookBot() proxychain.RequestModification { + const botUA string = "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)" + // 31.13.97.0/24, 31.13.99.0/24, 31.13.100.0/24, 66.220.144.0/20, 69.63.189.0/24, 69.63.190.0/24, 69.171.224.0/20, 69.171.240.0/21, 69.171.248.0/24, 173.252.73.0/24, 173.252.74.0/24, 173.252.77.0/24, 173.252.100.0/22, 173.252.104.0/21, 173.252.112.0/24, 2a03:2880:10::/48, 2a03:2880:10ff::/48, 2a03:2880:11::/48, 2a03:2880:11ff::/48, 2a03:2880:20::/48, 2a03:2880:20ff::/48, 2a03:2880:21ff::/48, 2a03:2880:30ff::/48, 2a03:2880:31ff::/48, 2a03:2880:1010::/48, 2a03:2880:1020::/48, 2a03:2880:2020::/48, 2a03:2880:2050::/48, 2a03:2880:2040::/48, 2a03:2880:2110::/48, 2a03:2880:2130::/48, 2a03:2880:3010::/48, 2a03:2880:3020::/48 + const botIP string = "31.13.99.8" + return masqueradeAsTrustedBot(botUA, botIP) +} + +// MasqueradeAsYandexBot modifies user agent and x-forwarded for +// to appear to be a Yandex Spider Bot +func MasqueradeAsYandexBot() proxychain.RequestModification { + const botUA string = "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)" + // 100.43.90.0/24, 37.9.115.0/24, 37.140.165.0/24, 77.88.22.0/25, 77.88.29.0/24, 77.88.31.0/24, 77.88.59.0/24, 84.201.146.0/24, 84.201.148.0/24, 84.201.149.0/24, 87.250.243.0/24, 87.250.253.0/24, 93.158.147.0/24, 93.158.148.0/24, 93.158.151.0/24, 93.158.153.0/32, 95.108.128.0/24, 95.108.138.0/24, 95.108.150.0/23, 95.108.158.0/24, 95.108.156.0/24, 95.108.188.128/25, 95.108.234.0/24, 95.108.248.0/24, 100.43.80.0/24, 130.193.62.0/24, 141.8.153.0/24, 178.154.165.0/24, 178.154.166.128/25, 178.154.173.29, 178.154.200.158, 178.154.202.0/24, 178.154.205.0/24, 178.154.239.0/24, 178.154.243.0/24, 37.9.84.253, 199.21.99.99, 178.154.162.29, 178.154.203.251, 178.154.211.250, 178.154.171.0/24, 178.154.200.0/24, 178.154.244.0/24, 178.154.246.0/24, 95.108.181.0/24, 95.108.246.252, 5.45.254.0/24, 5.255.253.0/24, 37.140.141.0/24, 37.140.188.0/24, 100.43.81.0/24, 100.43.85.0/24, 100.43.91.0/24, 199.21.99.0/24, 2a02:6b8:b000::/32, 2a02:6b8:b010::/32, 2a02:6b8:b011::/32, 2a02:6b8:c0e::/32 + const botIP string = "37.9.115.9" + return masqueradeAsTrustedBot(botUA, botIP) +} + +// MasqueradeAsBaiduBot modifies user agent and x-forwarded for +// to appear to be a Baidu Spider Bot +func MasqueradeAsBaiduBot() proxychain.RequestModification { + const botUA string = "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)" + // 180.76.15.0/24, 119.63.196.0/24, 115.239.212./24, 119.63.199.0/24, 122.81.208.0/22, 123.125.71.0/24, 180.76.4.0/24, 180.76.5.0/24, 180.76.6.0/24, 185.10.104.0/24, 220.181.108.0/24, 220.181.51.0/24, 111.13.102.0/24, 123.125.67.144/29, 123.125.67.152/31, 61.135.169.0/24, 123.125.68.68/30, 123.125.68.72/29, 123.125.68.80/28, 123.125.68.96/30, 202.46.48.0/20, 220.181.38.0/24, 123.125.68.80/30, 123.125.68.84/31, 123.125.68.0/24 + const botIP string = "180.76.15.7" + return masqueradeAsTrustedBot(botUA, botIP) +} + +// MasqueradeAsDuckDuckBot modifies user agent and x-forwarded for +// to appear to be a DuckDuckGo Bot +func MasqueradeAsDuckDuckBot() proxychain.RequestModification { + const botUA string = "DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)" + // 46.51.197.88, 46.51.197.89, 50.18.192.250, 50.18.192.251, 107.21.1.61, 176.34.131.233, 176.34.135.167, 184.72.106.52, 184.72.115.86 + const botIP string = "46.51.197.88" + return masqueradeAsTrustedBot(botUA, botIP) +} + +// MasqueradeAsYahooBot modifies user agent and x-forwarded for +// to appear to be a Yahoo Bot +func MasqueradeAsYahooBot() proxychain.RequestModification { + const botUA string = "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" + // 5.255.250.0/24, 37.9.87.0/24, 67.195.37.0/24, 67.195.50.0/24, 67.195.110.0/24, 67.195.111.0/24, 67.195.112.0/23, 67.195.114.0/24, 67.195.115.0/24, 68.180.224.0/21, 72.30.132.0/24, 72.30.142.0/24, 72.30.161.0/24, 72.30.196.0/24, 72.30.198.0/24, 74.6.254.0/24, 74.6.8.0/24, 74.6.13.0/24, 74.6.17.0/24, 74.6.18.0/24, 74.6.22.0/24, 74.6.27.0/24, 74.6.168.0/24, 77.88.5.0/24, 77.88.47.0/24, 93.158.161.0/24, 98.137.72.0/24, 98.137.206.0/24, 98.137.207.0/24, 98.139.168.0/24, 114.111.95.0/24, 124.83.159.0/24, 124.83.179.0/24, 124.83.223.0/24, 141.8.144.0/24, 183.79.63.0/24, 183.79.92.0/24, 203.216.255.0/24, 211.14.11.0/24 + const botIP string = "37.9.87.5" + return masqueradeAsTrustedBot(botUA, botIP) +} + func masqueradeAsTrustedBot(botUA string, botIP string) proxychain.RequestModification { return func(chain *proxychain.ProxyChain) error { - chain.Request.Header.Set("user-agent", botUA) - chain.Request.Header.Set("x-forwarded-for", botIP) - chain.Request.Header.Del("referrer") - chain.Request.Header.Del("origin") + chain.AddOnceRequestModifications( + SpoofUserAgent(botUA), + SetRequestHeader("x-forwarded-for", botIP), + DeleteRequestHeader("referrer"), + DeleteRequestHeader("origin"), + ) return nil } } diff --git a/proxychain/requestmodifers/modify_domain_with_regex.go b/proxychain/requestmodifers/modify_domain_with_regex.go index 3352d54..d6d97e8 100644 --- a/proxychain/requestmodifers/modify_domain_with_regex.go +++ b/proxychain/requestmodifers/modify_domain_with_regex.go @@ -1,8 +1,9 @@ package requestmodifers import ( - "ladder/proxychain" "regexp" + + "ladder/proxychain" ) func ModifyDomainWithRegex(match regexp.Regexp, replacement string) proxychain.RequestModification { diff --git a/proxychain/requestmodifers/modify_outgoing_cookies.go b/proxychain/requestmodifers/modify_outgoing_cookies.go index 86f8e35..9c8be53 100644 --- a/proxychain/requestmodifers/modify_outgoing_cookies.go +++ b/proxychain/requestmodifers/modify_outgoing_cookies.go @@ -1,8 +1,9 @@ package requestmodifers import ( - "ladder/proxychain" "net/http" + + "ladder/proxychain" ) // SetOutgoingCookie modifes a specific cookie name diff --git a/proxychain/requestmodifers/modify_path_with_regex.go b/proxychain/requestmodifers/modify_path_with_regex.go index 97163ee..109570a 100644 --- a/proxychain/requestmodifers/modify_path_with_regex.go +++ b/proxychain/requestmodifers/modify_path_with_regex.go @@ -1,8 +1,9 @@ package requestmodifers import ( - "ladder/proxychain" "regexp" + + "ladder/proxychain" ) func ModifyPathWithRegex(match regexp.Regexp, replacement string) proxychain.RequestModification { diff --git a/proxychain/requestmodifers/modify_query_params.go b/proxychain/requestmodifers/modify_query_params.go index 7f42f65..0b8448e 100644 --- a/proxychain/requestmodifers/modify_query_params.go +++ b/proxychain/requestmodifers/modify_query_params.go @@ -1,16 +1,17 @@ package requestmodifers import ( - "ladder/proxychain" "net/url" + + "ladder/proxychain" ) // ModifyQueryParams replaces query parameter values in URL's query params in a ProxyChain's URL. // If the query param key doesn't exist, it is created. func ModifyQueryParams(key string, value string) proxychain.RequestModification { - return func(px *proxychain.ProxyChain) error { - q := px.Request.URL.Query() - px.Request.URL.RawQuery = modifyQueryParams(key, value, q) + return func(chain *proxychain.ProxyChain) error { + q := chain.Request.URL.Query() + chain.Request.URL.RawQuery = modifyQueryParams(key, value, q) return nil } } diff --git a/proxychain/requestmodifers/request_archive_is.go b/proxychain/requestmodifers/request_archive_is.go index 674a48a..c8253f5 100644 --- a/proxychain/requestmodifers/request_archive_is.go +++ b/proxychain/requestmodifers/request_archive_is.go @@ -1,27 +1,28 @@ package requestmodifers import ( - "ladder/proxychain" + "fmt" "net/url" + + "ladder/proxychain" ) -const archivistUrl string = "https://archive.is/latest/" +const archivistUrl string = "https://archive.is/latest" // RequestArchiveIs modifies a ProxyChain's URL to request an archived version from archive.is func RequestArchiveIs() proxychain.RequestModification { - return func(px *proxychain.ProxyChain) error { - px.Request.URL.RawQuery = "" - newURLString := archivistUrl + px.Request.URL.String() - newURL, err := url.Parse(newURLString) + return func(chain *proxychain.ProxyChain) error { + chain.Request.URL.RawQuery = "" + newURL, err := url.Parse(fmt.Sprintf("%s/%s", archivistUrl, chain.Request.URL.String())) if err != nil { return err } // archivist seems to sabotage requests from cloudflare's DNS // bypass this just in case - px.AddRequestModifications(ResolveWithGoogleDoH()) + chain.AddOnceRequestModifications(ResolveWithGoogleDoH()) - px.Request.URL = newURL + chain.Request.URL = newURL return nil } } diff --git a/proxychain/requestmodifers/request_google_cache.go b/proxychain/requestmodifers/request_google_cache.go index 1e967a0..a3abfaf 100644 --- a/proxychain/requestmodifers/request_google_cache.go +++ b/proxychain/requestmodifers/request_google_cache.go @@ -1,8 +1,9 @@ package requestmodifers import ( - "ladder/proxychain" "net/url" + + "ladder/proxychain" ) const googleCacheUrl string = "https://webcache.googleusercontent.com/search?q=cache:" diff --git a/proxychain/requestmodifers/request_wayback_machine.go b/proxychain/requestmodifers/request_wayback_machine.go index 36363dc..536cc09 100644 --- a/proxychain/requestmodifers/request_wayback_machine.go +++ b/proxychain/requestmodifers/request_wayback_machine.go @@ -1,8 +1,9 @@ package requestmodifers import ( - "ladder/proxychain" "net/url" + + "ladder/proxychain" ) const waybackUrl string = "https://web.archive.org/web/" diff --git a/proxychain/requestmodifers/resolve_with_google_doh.go b/proxychain/requestmodifers/resolve_with_google_doh.go index 9f626e5..eeab810 100644 --- a/proxychain/requestmodifers/resolve_with_google_doh.go +++ b/proxychain/requestmodifers/resolve_with_google_doh.go @@ -4,10 +4,11 @@ import ( "context" "encoding/json" "fmt" - "ladder/proxychain" "net" "net/http" "time" + + "ladder/proxychain" ) // resolveWithGoogleDoH resolves DNS using Google's DNS-over-HTTPS diff --git a/proxychain/requestmodifers/spoof_referrer.go b/proxychain/requestmodifers/spoof_referrer.go index b2e5301..1ab8cf0 100644 --- a/proxychain/requestmodifers/spoof_referrer.go +++ b/proxychain/requestmodifers/spoof_referrer.go @@ -1,20 +1,29 @@ package requestmodifers import ( + "fmt" "ladder/proxychain" + tx "ladder/proxychain/responsemodifers" ) -// SpoofReferrer modifies the referrer header -// useful if the page can be accessed from a search engine -// or social media site, but not by browsing the website itself -// if url is "", then the referrer header is removed +// SpoofReferrer modifies the referrer header. +// It is useful if the page can be accessed from a search engine +// or social media site, but not by browsing the website itself. +// if url is "", then the referrer header is removed. func SpoofReferrer(url string) proxychain.RequestModification { - return func(px *proxychain.ProxyChain) error { + return func(chain *proxychain.ProxyChain) error { + + // change refer on client side js + script := fmt.Sprintf(`document.referrer = "%s"`, url) + chain.AddOnceResponseModifications( + tx.InjectScriptBeforeDOMContentLoaded(script), + ) + if url == "" { - px.Request.Header.Del("referrer") + chain.Request.Header.Del("referrer") return nil } - px.Request.Header.Set("referrer", url) + chain.Request.Header.Set("referrer", url) return nil } } diff --git a/proxychain/requestmodifers/spoof_referrer_from_baidu_post.go b/proxychain/requestmodifers/spoof_referrer_from_baidu_post.go index 40960eb..2e057c5 100644 --- a/proxychain/requestmodifers/spoof_referrer_from_baidu_post.go +++ b/proxychain/requestmodifers/spoof_referrer_from_baidu_post.go @@ -2,10 +2,11 @@ package requestmodifers import ( "fmt" - "ladder/proxychain" "math/rand" "strings" "time" + + "ladder/proxychain" ) // SpoofReferrerFromBaiduSearch modifies the referrer header diff --git a/proxychain/requestmodifers/spoof_referrer_from_bing_search.go b/proxychain/requestmodifers/spoof_referrer_from_bing_search.go index 8b7fb9f..c85b80a 100644 --- a/proxychain/requestmodifers/spoof_referrer_from_bing_search.go +++ b/proxychain/requestmodifers/spoof_referrer_from_bing_search.go @@ -8,7 +8,7 @@ import ( // pretending to be from a bing search site func SpoofReferrerFromBingSearch() proxychain.RequestModification { return func(chain *proxychain.ProxyChain) error { - chain.AddRequestModifications( + chain.AddOnceRequestModifications( SpoofReferrer("https://www.bing.com/"), SetRequestHeader("sec-fetch-site", "cross-site"), SetRequestHeader("sec-fetch-dest", "document"), diff --git a/proxychain/requestmodifers/spoof_referrer_from_google_search.go b/proxychain/requestmodifers/spoof_referrer_from_google_search.go index f5ff0b2..9193639 100644 --- a/proxychain/requestmodifers/spoof_referrer_from_google_search.go +++ b/proxychain/requestmodifers/spoof_referrer_from_google_search.go @@ -8,11 +8,13 @@ import ( // pretending to be from a google search site func SpoofReferrerFromGoogleSearch() proxychain.RequestModification { return func(chain *proxychain.ProxyChain) error { - chain.Request.Header.Set("referrer", "https://www.google.com/") - chain.Request.Header.Set("sec-fetch-site", "cross-site") - chain.Request.Header.Set("sec-fetch-dest", "document") - chain.Request.Header.Set("sec-fetch-mode", "navigate") - ModifyQueryParams("utm_source", "google") + chain.AddOnceRequestModifications( + SpoofReferrer("https://www.google.com"), + SetRequestHeader("sec-fetch-site", "cross-site"), + SetRequestHeader("sec-fetch-dest", "document"), + SetRequestHeader("sec-fetch-mode", "navigate"), + ModifyQueryParams("utm_source", "google"), + ) return nil } } diff --git a/proxychain/requestmodifers/spoof_referrer_from_linkedin_post.go b/proxychain/requestmodifers/spoof_referrer_from_linkedin_post.go index 147beb4..d288931 100644 --- a/proxychain/requestmodifers/spoof_referrer_from_linkedin_post.go +++ b/proxychain/requestmodifers/spoof_referrer_from_linkedin_post.go @@ -8,7 +8,7 @@ import ( // pretending to be from a linkedin post func SpoofReferrerFromLinkedInPost() proxychain.RequestModification { return func(chain *proxychain.ProxyChain) error { - chain.AddRequestModifications( + chain.AddOnceRequestModifications( SpoofReferrer("https://www.linkedin.com/"), SetRequestHeader("sec-fetch-site", "cross-site"), SetRequestHeader("sec-fetch-dest", "document"), diff --git a/proxychain/requestmodifers/spoof_referrer_from_naver_post.go b/proxychain/requestmodifers/spoof_referrer_from_naver_post.go index ae73588..7a7852d 100644 --- a/proxychain/requestmodifers/spoof_referrer_from_naver_post.go +++ b/proxychain/requestmodifers/spoof_referrer_from_naver_post.go @@ -2,6 +2,7 @@ package requestmodifers import ( "fmt" + "ladder/proxychain" ) diff --git a/proxychain/requestmodifers/spoof_referrer_from_tumblr_post.go b/proxychain/requestmodifers/spoof_referrer_from_tumblr_post.go index 5c47e04..3679936 100644 --- a/proxychain/requestmodifers/spoof_referrer_from_tumblr_post.go +++ b/proxychain/requestmodifers/spoof_referrer_from_tumblr_post.go @@ -8,7 +8,7 @@ import ( // pretending to be from a tumblr post func SpoofReferrerFromTumblrPost() proxychain.RequestModification { return func(chain *proxychain.ProxyChain) error { - chain.AddRequestModifications( + chain.AddOnceRequestModifications( SpoofReferrer("https://www.tumblr.com/"), SetRequestHeader("sec-fetch-site", "cross-site"), SetRequestHeader("sec-fetch-dest", "document"), diff --git a/proxychain/requestmodifers/spoof_referrer_from_twitter_post.go b/proxychain/requestmodifers/spoof_referrer_from_twitter_post.go index c3ae34c..348e4cf 100644 --- a/proxychain/requestmodifers/spoof_referrer_from_twitter_post.go +++ b/proxychain/requestmodifers/spoof_referrer_from_twitter_post.go @@ -8,7 +8,7 @@ import ( // pretending to be from a twitter post func SpoofReferrerFromTwitterPost() proxychain.RequestModification { return func(chain *proxychain.ProxyChain) error { - chain.AddRequestModifications( + chain.AddOnceRequestModifications( SpoofReferrer("https://t.co/"), SetRequestHeader("sec-fetch-site", "cross-site"), SetRequestHeader("sec-fetch-dest", "document"), diff --git a/proxychain/requestmodifers/spoof_referrer_from_vkontake_post.go b/proxychain/requestmodifers/spoof_referrer_from_vkontake_post.go index e05be73..1bae306 100644 --- a/proxychain/requestmodifers/spoof_referrer_from_vkontake_post.go +++ b/proxychain/requestmodifers/spoof_referrer_from_vkontake_post.go @@ -8,7 +8,7 @@ import ( // pretending to be from a vkontakte post (popular in Russia) func SpoofReferrerFromVkontaktePost() proxychain.RequestModification { return func(chain *proxychain.ProxyChain) error { - chain.AddRequestModifications( + chain.AddOnceRequestModifications( SpoofReferrer("https://away.vk.com/"), SetRequestHeader("sec-fetch-site", "cross-site"), SetRequestHeader("sec-fetch-dest", "document"), diff --git a/proxychain/requestmodifers/spoof_referrer_from_weibo_post.go b/proxychain/requestmodifers/spoof_referrer_from_weibo_post.go index 11ca11a..69e32fc 100644 --- a/proxychain/requestmodifers/spoof_referrer_from_weibo_post.go +++ b/proxychain/requestmodifers/spoof_referrer_from_weibo_post.go @@ -2,8 +2,9 @@ package requestmodifers import ( "fmt" - "ladder/proxychain" "math/rand" + + "ladder/proxychain" ) // SpoofReferrerFromWeiboPost modifies the referrer header diff --git a/proxychain/requestmodifers/spoof_user_agent.go b/proxychain/requestmodifers/spoof_user_agent.go index 4c2083c..cc706bc 100644 --- a/proxychain/requestmodifers/spoof_user_agent.go +++ b/proxychain/requestmodifers/spoof_user_agent.go @@ -1,13 +1,39 @@ package requestmodifers import ( + _ "embed" "ladder/proxychain" + tx "ladder/proxychain/responsemodifers" + "strings" ) +// https://github.com/faisalman/ua-parser-js/tree/master +// update using: +// git submodule update --remote --merge +// +//go:embed vendor/ua-parser-js/dist/ua-parser.min.js +var UAParserJS string + +// note: spoof_user_agent.js has a dependency on ua-parser.min.js +// ua-parser.min.js should be loaded first. +// +//go:embed spoof_user_agent.js +var spoofUserAgentJS string + // SpoofUserAgent modifies the user agent func SpoofUserAgent(ua string) proxychain.RequestModification { - return func(px *proxychain.ProxyChain) error { - px.Request.Header.Set("user-agent", ua) + return func(chain *proxychain.ProxyChain) error { + // modify ua headers + chain.AddOnceRequestModifications( + SetRequestHeader("user-agent", ua), + ) + + script := strings.ReplaceAll(spoofUserAgentJS, "{{USER_AGENT}}", ua) + chain.AddOnceResponseModifications( + tx.InjectScriptBeforeDOMContentLoaded(script), + tx.InjectScriptBeforeDOMContentLoaded(UAParserJS), + ) + return nil } } diff --git a/proxychain/requestmodifers/spoof_user_agent.js b/proxychain/requestmodifers/spoof_user_agent.js new file mode 100644 index 0000000..d37b93d --- /dev/null +++ b/proxychain/requestmodifers/spoof_user_agent.js @@ -0,0 +1,100 @@ +(() => { + const UA = "{{USER_AGENT}}"; + + // monkey-patch navigator.userAgent + { + const { get } = Object.getOwnPropertyDescriptor( + Navigator.prototype, + "userAgent", + ); + Object.defineProperty(Navigator.prototype, "userAgent", { + get: new Proxy(get, { + apply() { + return UA; + }, + }), + }); + } + + // monkey-patch navigator.appVersion + { + const { get } = Object.getOwnPropertyDescriptor( + Navigator.prototype, + "appVersion", + ); + Object.defineProperty(Navigator.prototype, "appVersion", { + get: new Proxy(get, { + apply() { + return UA.replace("Mozilla/", ""); + }, + }), + }); + } + + // monkey-patch navigator.UserAgentData + // Assuming UAParser is already loaded and available + function spoofUserAgentData(uaString) { + // Parse the user-agent string + const parser = new UAParser(uaString); + const parsedData = parser.getResult(); + + // Extracted data + const platform = parsedData.os.name; + const browserName = parsedData.browser.name; + const browserMajorVersion = parsedData.browser.major; + const isMobile = + /Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test( + uaString, + ); + + // Overwrite navigator.userAgentData + self.NavigatorUAData = self.NavigatorUAData || new class NavigatorUAData { + brands = [{ + brand: browserName, + version: browserMajorVersion, + }]; + mobile = isMobile; + platform = platform; + toJSON() { + return { + brands: this.brands, + mobile: this.mobile, + platform: this.platform, + }; + } + getHighEntropyValues(hints) { + const result = this.toJSON(); + // Add additional high entropy values based on hints + // Modify these as per your requirements + if (hints.includes("architecture")) { + result.architecture = "x86"; + } + if (hints.includes("bitness")) { + result.bitness = "64"; + } + if (hints.includes("model")) { + result.model = ""; + } + if (hints.includes("platformVersion")) { + result.platformVersion = "10.0.0"; // Example value + } + if (hints.includes("uaFullVersion")) { + result.uaFullVersion = browserMajorVersion; + } + if (hints.includes("fullVersionList")) { + result.fullVersionList = this.brands; + } + return Promise.resolve(result); + } + }(); + + // Apply the monkey patch + Object.defineProperty(navigator, "userAgentData", { + value: new self.NavigatorUAData(), + writable: false, + }); + } + + spoofUserAgentData(UA); + // TODO: use hideMonkeyPatch to hide overrides +})(); diff --git a/proxychain/requestmodifers/spoof_x_forwarded_for.go b/proxychain/requestmodifers/spoof_x_forwarded_for.go index 33d77e7..45a9dff 100644 --- a/proxychain/requestmodifers/spoof_x_forwarded_for.go +++ b/proxychain/requestmodifers/spoof_x_forwarded_for.go @@ -11,4 +11,4 @@ func SpoofXForwardedFor(ip string) proxychain.RequestModification { px.Request.Header.Set("X-FORWARDED-FOR", ip) return nil } -} \ No newline at end of file +} diff --git a/proxychain/requestmodifers/vendor/ua-parser-js b/proxychain/requestmodifers/vendor/ua-parser-js new file mode 160000 index 0000000..5173a54 --- /dev/null +++ b/proxychain/requestmodifers/vendor/ua-parser-js @@ -0,0 +1 @@ +Subproject commit 5173a5442f4af04b4f9c51519587e7969de9fc15 diff --git a/proxychain/responsemodifers/inject_script.go b/proxychain/responsemodifers/inject_script.go index 12b2e5d..5e5ef87 100644 --- a/proxychain/responsemodifers/inject_script.go +++ b/proxychain/responsemodifers/inject_script.go @@ -2,9 +2,10 @@ package responsemodifers import ( _ "embed" + "strings" + "ladder/proxychain" "ladder/proxychain/responsemodifers/rewriters" - "strings" ) // injectScript modifies HTTP responses @@ -25,16 +26,16 @@ func injectScript(js string, execTime rewriters.ScriptExecTime) proxychain.Respo } // InjectScriptBeforeDOMContentLoaded modifies HTTP responses to inject a JS before DOM Content is loaded (script tag in head) -func InjectScriptBeforeDOMContentLoaded(js string, execTime rewriters.ScriptExecTime) proxychain.ResponseModification { +func InjectScriptBeforeDOMContentLoaded(js string) proxychain.ResponseModification { return injectScript(js, rewriters.BeforeDOMContentLoaded) } // InjectScriptAfterDOMContentLoaded modifies HTTP responses to inject a JS after DOM Content is loaded (script tag in head) -func InjectScriptAfterDOMContentLoaded(js string, execTime rewriters.ScriptExecTime) proxychain.ResponseModification { +func InjectScriptAfterDOMContentLoaded(js string) proxychain.ResponseModification { return injectScript(js, rewriters.AfterDOMContentLoaded) } // InjectScriptAfterDOMIdle modifies HTTP responses to inject a JS after the DOM is idle (ie: js framework loaded) -func InjectScriptAfterDOMIdle(js string, execTime rewriters.ScriptExecTime) proxychain.ResponseModification { +func InjectScriptAfterDOMIdle(js string) proxychain.ResponseModification { return injectScript(js, rewriters.AfterDOMIdle) } diff --git a/proxychain/responsemodifers/modify_incoming_cookies.go b/proxychain/responsemodifers/modify_incoming_cookies.go index df6aa53..494f9a7 100644 --- a/proxychain/responsemodifers/modify_incoming_cookies.go +++ b/proxychain/responsemodifers/modify_incoming_cookies.go @@ -2,8 +2,9 @@ package responsemodifers import ( "fmt" - "ladder/proxychain" "net/http" + + "ladder/proxychain" ) // DeleteIncomingCookies prevents ALL cookies from being sent from the proxy server diff --git a/proxychain/responsemodifers/patch_dynamic_resource_urls.go b/proxychain/responsemodifers/patch_dynamic_resource_urls.go index 6ab2e85..21c3483 100644 --- a/proxychain/responsemodifers/patch_dynamic_resource_urls.go +++ b/proxychain/responsemodifers/patch_dynamic_resource_urls.go @@ -3,9 +3,10 @@ package responsemodifers import ( _ "embed" "fmt" + "strings" + "ladder/proxychain" "ladder/proxychain/responsemodifers/rewriters" - "strings" ) //go:embed patch_dynamic_resource_urls.js diff --git a/proxychain/responsemodifers/patch_dynamic_resource_urls.js b/proxychain/responsemodifers/patch_dynamic_resource_urls.js index cc4b4d1..337c9ca 100644 --- a/proxychain/responsemodifers/patch_dynamic_resource_urls.js +++ b/proxychain/responsemodifers/patch_dynamic_resource_urls.js @@ -2,43 +2,46 @@ // Also overrides the attribute setter prototype to modify the request URLs // fetch("/relative_script.js") -> fetch("http://localhost:8080/relative_script.js") (() => { - // ============== PARAMS =========================== - // if the original request was: http://localhost:8080/http://proxiedsite.com/foo/bar - // proxyOrigin is http://localhost:8080 + // if the original request was: http://localhost:8080/http://proxiedsite.com/foo/bar + // proxyOrigin is http://localhost:8080 const proxyOrigin = "{{PROXY_ORIGIN}}"; //const proxyOrigin = globalThis.window.location.origin; - // if the original request was: http://localhost:8080/http://proxiedsite.com/foo/bar - // origin is http://proxiedsite.com + // if the original request was: http://localhost:8080/http://proxiedsite.com/foo/bar + // origin is http://proxiedsite.com const origin = "{{ORIGIN}}"; //const origin = (new URL(decodeURIComponent(globalThis.window.location.pathname.substring(1)))).origin // ============== END PARAMS ====================== - const blacklistedSchemes = [ - "ftp:", - "mailto:", - "tel:", - "file:", - "blob:", - "javascript:", - "about:", - "magnet:", - "ws:", - "wss:", - ]; + const blacklistedSchemes = [ + "ftp:", + "mailto:", + "tel:", + "file:", + "blob:", + "javascript:", + "about:", + "magnet:", + "ws:", + "wss:", + ]; - function rewriteURL(url) { - const oldUrl = url - if (!url) return url - let isStr = (typeof url.startsWith === 'function') - if (!isStr) return url + function rewriteURL(url) { + const oldUrl = url; + if (!url) return url; + let isStr = typeof url.startsWith === "function"; + if (!isStr) return url; // don't rewrite special URIs if (blacklistedSchemes.includes(url)) return url; // don't rewrite invalid URIs - try { new URL(url, origin) } catch { return url } + try { + new URL(url, origin); + } catch { + return url; + } // don't double rewrite if (url.startsWith(proxyOrigin)) return url; @@ -57,132 +60,163 @@ } else if (url.startsWith("/")) { url = `/${origin}/${encodeURIComponent(url.substring(1))}`; } else if (url.startsWith(origin)) { - url = `/${encodeURIComponent(url)}` + url = `/${encodeURIComponent(url)}`; } else if (url.startsWith("http://") || url.startsWith("https://")) { url = `/${proxyOrigin}/${encodeURIComponent(url)}`; } - console.log(`proxychain: rewrite JS URL: ${oldUrl} -> ${url}`) + console.log(`proxychain: rewrite JS URL: ${oldUrl} -> ${url}`); return url; - }; + } // sometimes anti-bot protections like cloudflare or akamai bot manager check if JS is hooked function hideMonkeyPatch(objectOrName, method, originalToString) { let obj; let isGlobalFunction = false; - - if (typeof objectOrName === 'string') { + + if (typeof objectOrName === "string") { obj = globalThis[objectOrName]; - isGlobalFunction = (typeof obj === 'function') && (method === objectOrName); + isGlobalFunction = (typeof obj === "function") && + (method === objectOrName); } else { obj = objectOrName; } - + if (isGlobalFunction) { const originalFunction = obj; globalThis[objectOrName] = function(...args) { return originalFunction.apply(this, args); }; globalThis[objectOrName].toString = () => originalToString; - } else if (obj && typeof obj[method] === 'function') { + } else if (obj && typeof obj[method] === "function") { const originalMethod = obj[method]; obj[method] = function(...args) { return originalMethod.apply(this, args); }; obj[method].toString = () => originalToString; } else { - console.warn(`proxychain: cannot hide monkey patch: ${method} is not a function on the provided object.`); + console.warn( + `proxychain: cannot hide monkey patch: ${method} is not a function on the provided object.`, + ); } } - // monkey patch fetch - const oldFetch = fetch; - fetch = async (url, init) => { - return oldFetch(rewriteURL(url), init) - } - hideMonkeyPatch('fetch', 'fetch', 'function fetch() { [native code] }') + // monkey patch fetch + const oldFetch = fetch; + fetch = async (url, init) => { + return oldFetch(rewriteURL(url), init); + }; + hideMonkeyPatch("fetch", "fetch", "function fetch() { [native code] }"); - // monkey patch xmlhttprequest - const oldOpen = XMLHttpRequest.prototype.open; - XMLHttpRequest.prototype.open = function(method, url, async = true, user = null, password = null) { - return oldOpen.call(this, method, rewriteURL(url), async, user, password); - }; - hideMonkeyPatch(XMLHttpRequest.prototype, 'open', 'function(){if("function"==typeof eo)return eo.apply(this,arguments)}'); + // monkey patch xmlhttprequest + const oldOpen = XMLHttpRequest.prototype.open; + XMLHttpRequest.prototype.open = function( + method, + url, + async = true, + user = null, + password = null, + ) { + return oldOpen.call(this, method, rewriteURL(url), async, user, password); + }; + hideMonkeyPatch( + XMLHttpRequest.prototype, + "open", + 'function(){if("function"==typeof eo)return eo.apply(this,arguments)}', + ); - const oldSend = XMLHttpRequest.prototype.send; - XMLHttpRequest.prototype.send = function(method, url) { - return oldSend.call(this, method, rewriteURL(url)); - }; - hideMonkeyPatch(XMLHttpRequest.prototype, 'send', 'function(){if("function"==typeof eo)return eo.apply(this,arguments)}'); + const oldSend = XMLHttpRequest.prototype.send; + XMLHttpRequest.prototype.send = function(method, url) { + return oldSend.call(this, method, rewriteURL(url)); + }; + hideMonkeyPatch( + XMLHttpRequest.prototype, + "send", + 'function(){if("function"==typeof eo)return eo.apply(this,arguments)}', + ); + // monkey patch service worker registration + const oldRegister = ServiceWorkerContainer.prototype.register; + ServiceWorkerContainer.prototype.register = function(scriptURL, options) { + return oldRegister.call(this, rewriteURL(scriptURL), options); + }; + hideMonkeyPatch( + ServiceWorkerContainer.prototype, + "register", + "function register() { [native code] }", + ); - // monkey patch service worker registration - const oldRegister = ServiceWorkerContainer.prototype.register; - ServiceWorkerContainer.prototype.register = function(scriptURL, options) { - return oldRegister.call(this, rewriteURL(scriptURL), options) - } - hideMonkeyPatch(ServiceWorkerContainer.prototype, 'register', 'function register() { [native code] }') + // monkey patch URL.toString() method + const oldToString = URL.prototype.toString; + URL.prototype.toString = function() { + let originalURL = oldToString.call(this); + return rewriteURL(originalURL); + }; + hideMonkeyPatch( + URL.prototype, + "toString", + "function toString() { [native code] }", + ); - // monkey patch URL.toString() method - const oldToString = URL.prototype.toString - URL.prototype.toString = function() { - let originalURL = oldToString.call(this) - return rewriteURL(originalURL) - } - hideMonkeyPatch(URL.prototype, 'toString', 'function toString() { [native code] }') + // monkey patch URL.toJSON() method + const oldToJson = URL.prototype.toString; + URL.prototype.toString = function() { + let originalURL = oldToJson.call(this); + return rewriteURL(originalURL); + }; + hideMonkeyPatch( + URL.prototype, + "toString", + "function toJSON() { [native code] }", + ); - // monkey patch URL.toJSON() method - const oldToJson = URL.prototype.toString - URL.prototype.toString = function() { - let originalURL = oldToJson.call(this) - return rewriteURL(originalURL) - } - hideMonkeyPatch(URL.prototype, 'toString', 'function toJSON() { [native code] }') - - // Monkey patch URL.href getter and setter - const originalHrefDescriptor = Object.getOwnPropertyDescriptor(URL.prototype, 'href'); - Object.defineProperty(URL.prototype, 'href', { + // Monkey patch URL.href getter and setter + const originalHrefDescriptor = Object.getOwnPropertyDescriptor( + URL.prototype, + "href", + ); + Object.defineProperty(URL.prototype, "href", { get: function() { let originalHref = originalHrefDescriptor.get.call(this); - return rewriteURL(originalHref) + return rewriteURL(originalHref); }, set: function(newValue) { originalHrefDescriptor.set.call(this, rewriteURL(newValue)); - } + }, }); // TODO: do one more pass of this by manually traversing the DOM - // AFTER all the JS and page has loaded just in case + // AFTER all the JS and page has loaded just in case - // Monkey patch setter + // Monkey patch setter const elements = [ - { tag: 'a', attribute: 'href' }, - { tag: 'img', attribute: 'src' }, + { tag: "a", attribute: "href" }, + { tag: "img", attribute: "src" }, // { tag: 'img', attribute: 'srcset' }, // TODO: handle srcset - { tag: 'script', attribute: 'src' }, - { tag: 'link', attribute: 'href' }, - { tag: 'link', attribute: 'icon' }, - { tag: 'iframe', attribute: 'src' }, - { tag: 'audio', attribute: 'src' }, - { tag: 'video', attribute: 'src' }, - { tag: 'source', attribute: 'src' }, + { tag: "script", attribute: "src" }, + { tag: "link", attribute: "href" }, + { tag: "link", attribute: "icon" }, + { tag: "iframe", attribute: "src" }, + { tag: "audio", attribute: "src" }, + { tag: "video", attribute: "src" }, + { tag: "source", attribute: "src" }, // { tag: 'source', attribute: 'srcset' }, // TODO: handle srcset - { tag: 'embed', attribute: 'src' }, - { tag: 'embed', attribute: 'pluginspage' }, - { tag: 'html', attribute: 'manifest' }, - { tag: 'object', attribute: 'src' }, - { tag: 'input', attribute: 'src' }, - { tag: 'track', attribute: 'src' }, - { tag: 'form', attribute: 'action' }, - { tag: 'area', attribute: 'href' }, - { tag: 'base', attribute: 'href' }, - { tag: 'blockquote', attribute: 'cite' }, - { tag: 'del', attribute: 'cite' }, - { tag: 'ins', attribute: 'cite' }, - { tag: 'q', attribute: 'cite' }, - { tag: 'button', attribute: 'formaction' }, - { tag: 'input', attribute: 'formaction' }, - { tag: 'meta', attribute: 'content' }, - { tag: 'object', attribute: 'data' }, + { tag: "embed", attribute: "src" }, + { tag: "embed", attribute: "pluginspage" }, + { tag: "html", attribute: "manifest" }, + { tag: "object", attribute: "src" }, + { tag: "input", attribute: "src" }, + { tag: "track", attribute: "src" }, + { tag: "form", attribute: "action" }, + { tag: "area", attribute: "href" }, + { tag: "base", attribute: "href" }, + { tag: "blockquote", attribute: "cite" }, + { tag: "del", attribute: "cite" }, + { tag: "ins", attribute: "cite" }, + { tag: "q", attribute: "cite" }, + { tag: "button", attribute: "formaction" }, + { tag: "input", attribute: "formaction" }, + { tag: "meta", attribute: "content" }, + { tag: "object", attribute: "data" }, ]; elements.forEach(({ tag, attribute }) => { @@ -195,7 +229,7 @@ // calling rewriteURL will end up calling a setter for href, // leading to a recusive loop and a Maximum call stack size exceeded // error, so we guard against this with a local semaphore flag - const isRewritingSetKey = Symbol.for('isRewritingSet'); + const isRewritingSetKey = Symbol.for("isRewritingSet"); if (!this[isRewritingSetKey]) { this[isRewritingSetKey] = true; descriptor.set.call(this, rewriteURL(value)); @@ -207,44 +241,43 @@ } }, get() { - const isRewritingGetKey = Symbol.for('isRewritingGet'); + const isRewritingGetKey = Symbol.for("isRewritingGet"); if (!this[isRewritingGetKey]) { this[isRewritingGetKey] = true; let oldURL = descriptor.get.call(this); let newURL = rewriteURL(oldURL); this[isRewritingGetKey] = false; - return newURL + return newURL; } else { return descriptor.get.call(this); } - } + }, }); } }); - // sometimes, libraries will set the Element.innerHTML or Element.outerHTML directly with a string instead of setters. // in this case, we intercept it, create a fake DOM, parse it and then rewrite all attributes that could // contain a URL. Then we return the replacement innerHTML/outerHTML with redirected links. function rewriteInnerHTML(html, elements) { - const isRewritingHTMLKey = Symbol.for('isRewritingHTML'); - + const isRewritingHTMLKey = Symbol.for("isRewritingHTML"); + // Check if already processing if (document[isRewritingHTMLKey]) { return html; } - - const tempContainer = document.createElement('div'); + + const tempContainer = document.createElement("div"); document[isRewritingHTMLKey] = true; - + try { tempContainer.innerHTML = html; - + // Create a map for quick lookup - const elementsMap = new Map(elements.map(e => [e.tag, e.attribute])); - + const elementsMap = new Map(elements.map((e) => [e.tag, e.attribute])); + // Loop-based DOM traversal - const nodes = [...tempContainer.querySelectorAll('*')]; + const nodes = [...tempContainer.querySelectorAll("*")]; for (const node of nodes) { const attribute = elementsMap.get(node.tagName.toLowerCase()); if (attribute && node.hasAttribute(attribute)) { @@ -253,7 +286,7 @@ node.setAttribute(attribute, rewrittenUrl); } } - + return tempContainer.innerHTML; } finally { // Clear the flag @@ -261,19 +294,21 @@ } } - // Store original setters -const originalSetters = {}; + const originalSetters = {}; - ['innerHTML', 'outerHTML'].forEach(property => { - const descriptor = Object.getOwnPropertyDescriptor(Element.prototype, property); + ["innerHTML", "outerHTML"].forEach((property) => { + const descriptor = Object.getOwnPropertyDescriptor( + Element.prototype, + property, + ); if (descriptor && descriptor.set) { originalSetters[property] = descriptor.set; Object.defineProperty(Element.prototype, property, { ...descriptor, set(value) { - const isRewritingHTMLKey = Symbol.for('isRewritingHTML'); + const isRewritingHTMLKey = Symbol.for("isRewritingHTML"); if (!this[isRewritingHTMLKey]) { this[isRewritingHTMLKey] = true; try { @@ -286,40 +321,8 @@ const originalSetters = {}; // Use original setter in recursive call originalSetters[property].call(this, value); } - } + }, }); } }); - - -})(); - - - -(() => { - document.addEventListener('DOMContentLoaded', (event) => { - initIdleMutationObserver(); - }); - - function initIdleMutationObserver() { - let debounceTimer; - const debounceDelay = 500; // adjust the delay as needed - - const observer = new MutationObserver((mutations) => { - // Clear the previous timer and set a new one - clearTimeout(debounceTimer); - debounceTimer = setTimeout(() => { - execute(); - observer.disconnect(); // Disconnect after first execution - }, debounceDelay); - }); - - const config = { attributes: false, childList: true, subtree: true }; - observer.observe(document.body, config); - } - - function execute() { - console.log('DOM is now idle. Executing...'); - } - })(); diff --git a/proxychain/responsemodifers/rewrite_http_resource_urls.go b/proxychain/responsemodifers/rewrite_http_resource_urls.go index bd3b92a..5320d13 100644 --- a/proxychain/responsemodifers/rewrite_http_resource_urls.go +++ b/proxychain/responsemodifers/rewrite_http_resource_urls.go @@ -3,9 +3,10 @@ package responsemodifers import ( _ "embed" "fmt" + "strings" + "ladder/proxychain" "ladder/proxychain/responsemodifers/rewriters" - "strings" ) // RewriteHTMLResourceURLs modifies HTTP responses diff --git a/proxychain/responsemodifers/rewriters/html_rewriter.go b/proxychain/responsemodifers/rewriters/html_rewriter.go index e18a045..d14626f 100644 --- a/proxychain/responsemodifers/rewriters/html_rewriter.go +++ b/proxychain/responsemodifers/rewriters/html_rewriter.go @@ -71,7 +71,6 @@ func (r *HTMLRewriter) Close() error { // Read processes the HTML content, rewriting URLs and managing the state of tokens. func (r *HTMLRewriter) Read(p []byte) (int, error) { - if r.currentToken == nil || r.currentToken.Data == "" || r.currentTokenProcessed { tokenType := r.tokenizer.Next() diff --git a/proxychain/responsemodifers/rewriters/html_token_url_rewriter.go b/proxychain/responsemodifers/rewriters/html_token_url_rewriter.go index d7fca47..bc112bd 100644 --- a/proxychain/responsemodifers/rewriters/html_token_url_rewriter.go +++ b/proxychain/responsemodifers/rewriters/html_token_url_rewriter.go @@ -3,18 +3,21 @@ package rewriters import ( _ "embed" "fmt" - "golang.org/x/net/html/atom" "log" "net/url" "regexp" "strings" + "golang.org/x/net/html/atom" + "golang.org/x/net/html" ) -var rewriteAttrs map[string]map[string]bool -var specialRewriteAttrs map[string]map[string]bool -var schemeBlacklist map[string]bool +var ( + rewriteAttrs map[string]map[string]bool + specialRewriteAttrs map[string]map[string]bool + schemeBlacklist map[string]bool +) func init() { // define all tag/attributes which might contain URLs @@ -66,7 +69,6 @@ func init() { "wss": true, "ftp": true, } - } // HTMLTokenURLRewriter implements HTMLTokenRewriter @@ -159,10 +161,10 @@ func handleRootRelativePath(attr *html.Attribute, baseURL *url.URL) { return } - //log.Printf("BASEURL patch: %s\n", baseURL) + // log.Printf("BASEURL patch: %s\n", baseURL) attr.Val = fmt.Sprintf( - "/%s://%s/%s", + "%s://%s/%s", baseURL.Scheme, baseURL.Host, strings.TrimPrefix(attr.Val, "/"), @@ -184,7 +186,7 @@ func handleDocumentRelativePath(attr *html.Attribute, baseURL *url.URL) { strings.Trim(attr.Val, "/"), ) attr.Val = escape(attr.Val) - attr.Val = fmt.Sprintf("/%s", attr.Val) + attr.Val = fmt.Sprintf("%s://%s/%s", baseURL.Scheme, baseURL.Host, attr.Val) log.Printf("doc rel url rewritten-> '%s'='%s'", attr.Key, attr.Val) } @@ -199,7 +201,7 @@ func handleAbsolutePath(attr *html.Attribute, baseURL *url.URL) { if !(u.Scheme == "http" || u.Scheme == "https") { return } - attr.Val = fmt.Sprintf("/%s", escape(strings.TrimPrefix(attr.Val, "/"))) + attr.Val = fmt.Sprintf("%s://%s/%s", baseURL.Scheme, baseURL.Host, escape(strings.TrimPrefix(attr.Val, "/"))) log.Printf("abs url rewritten-> '%s'='%s'", attr.Key, attr.Val) }