From f6341f2c3e9bb62c285f3c6a35d95476f6fb1ef8 Mon Sep 17 00:00:00 2001 From: Kevin Pham Date: Sat, 18 Nov 2023 08:31:59 -0600 Subject: [PATCH] begin refactor of proxy engine --- cmd/main.go | 22 +- handlers/proxy.go | 95 +---- handlers/proxy.test.go | 2 +- .../cli.go => internal/cli/ruleset_merge.go | 0 proxychain/proxychain.go | 384 ++++++++++++++++++ proxychain/proxychain_pool.go | 11 + proxychain/rqm/block_outgoing_cookies.go | 41 ++ proxychain/rqm/masquerade_as_trusted_bot.go | 32 ++ proxychain/rqm/modify_domain_with_regex.go | 13 + proxychain/rqm/modify_path_with_regex.go | 13 + proxychain/rqm/modify_query_params.go | 20 + proxychain/rqm/request_archive_is.go | 27 ++ proxychain/rqm/request_google_cache.go | 21 + proxychain/rqm/request_wayback_machine.go | 22 + proxychain/rqm/resolve_with_google_doh.go | 80 ++++ proxychain/rqm/spoof_origin.go | 24 ++ proxychain/rqm/spoof_referrer.go | 29 ++ proxychain/rqm/spoof_user_agent.go | 13 + proxychain/rqm/spoof_x_forwarded_for.go | 14 + proxychain/rsm/block_incoming_cookies.go | 58 +++ proxychain/rsm/bypass_cors.go | 20 + proxychain/rsm/bypass_csp.go | 19 + proxychain/rsm/modify_response_header.go | 26 ++ proxychain/strategy/cloudflare.go | 8 + 24 files changed, 917 insertions(+), 77 deletions(-) rename handlers/cli/cli.go => internal/cli/ruleset_merge.go (100%) create mode 100644 proxychain/proxychain.go create mode 100644 proxychain/proxychain_pool.go create mode 100644 proxychain/rqm/block_outgoing_cookies.go create mode 100644 proxychain/rqm/masquerade_as_trusted_bot.go create mode 100644 proxychain/rqm/modify_domain_with_regex.go create mode 100644 proxychain/rqm/modify_path_with_regex.go create mode 100644 proxychain/rqm/modify_query_params.go create mode 100644 proxychain/rqm/request_archive_is.go create mode 100644 proxychain/rqm/request_google_cache.go create mode 100644 proxychain/rqm/request_wayback_machine.go create mode 100644 proxychain/rqm/resolve_with_google_doh.go create mode 100644 proxychain/rqm/spoof_origin.go create mode 100644 proxychain/rqm/spoof_referrer.go create mode 100644 proxychain/rqm/spoof_user_agent.go create mode 100644 proxychain/rqm/spoof_x_forwarded_for.go create mode 100644 proxychain/rsm/block_incoming_cookies.go create mode 100644 proxychain/rsm/bypass_cors.go create mode 100644 proxychain/rsm/bypass_csp.go create mode 100644 proxychain/rsm/modify_response_header.go create mode 100644 proxychain/strategy/cloudflare.go diff --git a/cmd/main.go b/cmd/main.go index e9ead89..9671012 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -8,7 +8,7 @@ import ( "strings" "ladder/handlers" - "ladder/handlers/cli" + "ladder/internal/cli" "github.com/akamensky/argparse" "github.com/gofiber/fiber/v2" @@ -40,6 +40,13 @@ func main() { Help: "This will spawn multiple processes listening", }) + verbose := parser.Flag("v", "verbose", &argparse.Options{ + Required: false, + Help: "Adds verbose logging", + }) + + // TODO: add version flag that reads from handers/VERSION + ruleset := parser.String("r", "ruleset", &argparse.Options{ Required: false, Help: "File, Directory or URL to a ruleset.yaml. Overrides RULESET environment variable.", @@ -84,6 +91,7 @@ func main() { }, ) + // TODO: move to cmd/auth.go userpass := os.Getenv("USERPASS") if userpass != "" { userpass := strings.Split(userpass, ":") @@ -94,6 +102,7 @@ func main() { })) } + // TODO: move to handlers/favicon.go app.Use(favicon.New(favicon.Config{ Data: []byte(faviconData), URL: "/favicon.ico", @@ -107,6 +116,8 @@ func main() { } app.Get("/", handlers.Form) + + // TODO: move this logic to handers/styles.go app.Get("/styles.css", func(c *fiber.Ctx) error { cssData, err := cssData.ReadFile("styles.css") if err != nil { @@ -115,10 +126,17 @@ func main() { c.Set("Content-Type", "text/css") return c.Send(cssData) }) + app.Get("ruleset", handlers.Ruleset) app.Get("raw/*", handlers.Raw) app.Get("api/*", handlers.Api) - app.Get("/*", handlers.ProxySite(*ruleset)) + + proxyOpts := &handlers.ProxyOptions{ + Verbose: *verbose, + RulesetPath: *ruleset, + } + + app.Get("/*", handlers.NewProxySiteHandler(proxyOpts)) log.Fatal(app.Listen(":" + *port)) } diff --git a/handlers/proxy.go b/handlers/proxy.go index 128314e..2a0f0c6 100644 --- a/handlers/proxy.go +++ b/handlers/proxy.go @@ -11,6 +11,9 @@ import ( "strings" "ladder/pkg/ruleset" + "ladder/proxychain" + "ladder/proxychain/rqm" + "ladder/proxychain/rsm" "github.com/PuerkitoBio/goquery" "github.com/gofiber/fiber/v2" @@ -30,88 +33,32 @@ func init() { } } -// extracts a URL from the request ctx. If the URL in the request -// is a relative path, it reconstructs the full URL using the referer header. -func extractUrl(c *fiber.Ctx) (string, error) { - // try to extract url-encoded - reqUrl, err := url.QueryUnescape(c.Params("*")) - if err != nil { - // fallback - reqUrl = c.Params("*") - } - - // Extract the actual path from req ctx - urlQuery, err := url.Parse(reqUrl) - if err != nil { - return "", fmt.Errorf("error parsing request URL '%s': %v", reqUrl, err) - } - - isRelativePath := urlQuery.Scheme == "" - - // eg: https://localhost:8080/images/foobar.jpg -> https://realsite.com/images/foobar.jpg - if isRelativePath { - // Parse the referer URL from the request header. - refererUrl, err := url.Parse(c.Get("referer")) - if err != nil { - return "", fmt.Errorf("error parsing referer URL from req: '%s': %v", reqUrl, err) - } - - // Extract the real url from referer path - realUrl, err := url.Parse(strings.TrimPrefix(refererUrl.Path, "/")) - if err != nil { - return "", fmt.Errorf("error parsing real URL from referer '%s': %v", refererUrl.Path, err) - } - - // reconstruct the full URL using the referer's scheme, host, and the relative path / queries - fullUrl := &url.URL{ - Scheme: realUrl.Scheme, - Host: realUrl.Host, - Path: urlQuery.Path, - RawQuery: urlQuery.RawQuery, - } - - if os.Getenv("LOG_URLS") == "true" { - log.Printf("modified relative URL: '%s' -> '%s'", reqUrl, fullUrl.String()) - } - return fullUrl.String(), nil - - } - - // default behavior: - // eg: https://localhost:8080/https://realsite.com/images/foobar.jpg -> https://realsite.com/images/foobar.jpg - return urlQuery.String(), nil - +type ProxyOptions struct { + RulesetPath string + Verbose bool } -func ProxySite(rulesetPath string) fiber.Handler { - if rulesetPath != "" { - rs, err := ruleset.NewRuleset(rulesetPath) +func NewProxySiteHandler(opts *ProxyOptions) fiber.Handler { + var rs ruleset.RuleSet + if opts.RulesetPath != "" { + r, err := ruleset.NewRuleset(opts.RulesetPath) if err != nil { panic(err) } - rulesSet = rs + rs = r } return func(c *fiber.Ctx) error { - // Get the url from the URL - url, err := extractUrl(c) - if err != nil { - log.Println("ERROR In URL extraction:", err) - } - - queries := c.Queries() - body, _, resp, err := fetchSite(url, queries) - if err != nil { - log.Println("ERROR:", err) - c.SendStatus(fiber.StatusInternalServerError) - return c.SendString(err.Error()) - } - - c.Cookie(&fiber.Cookie{}) - c.Set("Content-Type", resp.Header.Get("Content-Type")) - c.Set("Content-Security-Policy", resp.Header.Get("Content-Security-Policy")) - - return c.SendString(body) + return proxychain.NewProxyChain(). + SetCtx(c). + AddRuleset(&rs). + SetRequestModifications( + rqm.BlockOutgoingCookies(), + ). + SetResultModifications( + rsm.BlockIncomingCookies(), + ). + Execute() } } diff --git a/handlers/proxy.test.go b/handlers/proxy.test.go index 07f72bd..ef4349c 100644 --- a/handlers/proxy.test.go +++ b/handlers/proxy.test.go @@ -14,7 +14,7 @@ import ( func TestProxySite(t *testing.T) { app := fiber.New() - app.Get("/:url", ProxySite("")) + app.Get("/:url", NewProxySiteHandler(nil)) req := httptest.NewRequest("GET", "/https://example.com", nil) resp, err := app.Test(req) diff --git a/handlers/cli/cli.go b/internal/cli/ruleset_merge.go similarity index 100% rename from handlers/cli/cli.go rename to internal/cli/ruleset_merge.go diff --git a/proxychain/proxychain.go b/proxychain/proxychain.go new file mode 100644 index 0000000..56d27fc --- /dev/null +++ b/proxychain/proxychain.go @@ -0,0 +1,384 @@ +package proxychain + +import ( + "errors" + "fmt" + "io" + "log" + "net/http" + "net/url" + "strings" + + "ladder/pkg/ruleset" + + "github.com/gofiber/fiber/v2" +) + +var defaultClient *http.Client + +func DefaultClient() { + defaultClient = &http.Client{ + Timeout: 15, + } +} + +/* +ProxyChain manages the process of forwarding an HTTP request to an upstream server, +applying request and response modifications along the way. + + - It accepts incoming HTTP requests (as a Fiber *ctx), and applies + request modifiers (ReqMods) and response modifiers (ResMods) before passing the + upstream response back to the client. + + - ProxyChains can be reused to avoid memory allocations. + +--- + +# EXAMPLE + +``` + +import ( + + "ladder/internal/proxychain/rqm" + "ladder/internal/proxychain/rsm" + "ladder/internal/proxychain" + +) + +proxychain.NewProxyChain(). + + SetCtx(c). + AddRuleset(&rs). + SetRequestModifications( + rqm.BlockOutgoingCookies(), + ). + SetResultModifications( + rsm.BlockIncomingCookies(), + ). + Execute() + +``` + + client ladder service upstream + +┌─────────┐ ┌────────────────────────┐ ┌─────────┐ +│ │GET │ │ │ │ +│ req────┼───► ProxyChain │ │ │ +│ │ │ │ │ │ │ +│ │ │ ▼ │ │ │ +│ │ │ apply │ │ │ +│ │ │ RequestModifications │ │ │ +│ │ │ │ │ │ │ +│ │ │ ▼ │ │ │ +│ │ │ send GET │ │ │ +│ │ │ Request req────────┼─► │ │ +│ │ │ │ │ │ +│ │ │ 200 OK │ │ │ +│ │ │ ┌────────────────┼─response │ +│ │ │ ▼ │ │ │ +│ │ │ apply │ │ │ +│ │ │ ResultModifications │ │ │ +│ │ │ │ │ │ │ +│ │◄───┼───────┘ │ │ │ +│ │ │ 200 OK │ │ │ +│ │ │ │ │ │ +└─────────┘ └────────────────────────┘ └─────────┘ +*/ +type ProxyChain struct { + Context *fiber.Ctx + Client *http.Client + Request *http.Request + Response *http.Response + Body []byte + requestModifications []RequestModification + resultModifications []ResponseModification + ruleset *ruleset.RuleSet + verbose bool + _abort_err error +} + +// a ProxyStrategy is a pre-built proxychain with purpose-built defaults +type ProxyStrategy ProxyChain + +// A RequestModification is a function that should operate on the +// ProxyChain Req or Client field, using the fiber ctx as needed. +type RequestModification func(*ProxyChain) error + +// A ResponseModification is a function that should operate on the +// ProxyChain Res (http result) & Body (buffered http response body) field +type ResponseModification func(*ProxyChain) error + +// SetRequestModifications sets the ProxyChain's request modifers +// the modifier will not fire until ProxyChain.Execute() is run. +func (chain *ProxyChain) SetRequestModifications(mods ...RequestModification) *ProxyChain { + chain.requestModifications = mods + return chain +} + +// AddRequestModifications sets the ProxyChain's request modifers +// the modifier will not fire until ProxyChain.Execute() is run. +func (chain *ProxyChain) AddRequestModifications(mods ...RequestModification) *ProxyChain { + chain.requestModifications = append(chain.requestModifications, mods...) + return chain +} + +// SetResultModifications sets the ProxyChain's response modifers +// the modifier will not fire until ProxyChain.Execute() is run. +func (chain *ProxyChain) SetResultModifications(mods ...ResponseModification) *ProxyChain { + chain.resultModifications = mods + return chain +} + +// AddResultModifications adds to the ProxyChain's response modifers +// the modifier will not fire until ProxyChain.Execute() is run. +func (chain *ProxyChain) AddResultModifications(mods ...ResponseModification) *ProxyChain { + chain.resultModifications = append(chain.resultModifications, mods...) + return chain +} + +// Adds a ruleset to ProxyChain +func (chain *ProxyChain) AddRuleset(rs *ruleset.RuleSet) *ProxyChain { + chain.ruleset = rs + // TODO: add _applyRuleset method + return chain +} + +func (chain *ProxyChain) _initialize_request() (*http.Request, error) { + // initialize a request (without url) + req, err := http.NewRequest(chain.Context.Method(), "", nil) + if err != nil { + return nil, err + } + chain.Request = req + switch chain.Context.Method() { + case "GET": + case "DELETE": + case "HEAD": + case "OPTIONS": + break + case "POST": + case "PUT": + case "PATCH": + // stream content of body from client request to upstream request + chain.Request.Body = io.NopCloser(chain.Context.Request().BodyStream()) + default: + return nil, fmt.Errorf("unsupported request method from client: '%s'", chain.Context.Method()) + } + + // copy client request headers to upstream request headers + forwardHeaders := func(key []byte, val []byte) { + req.Header.Set(string(key), string(val)) + } + clientHeaders := &chain.Context.Request().Header + clientHeaders.VisitAll(forwardHeaders) + + return req, nil +} + +// _execute sends the request for the ProxyChain and returns the raw body only +// the caller is responsible for returning a response back to the requestor +// the caller is also responsible for calling pxc._reset() when they are done with the body +func (chain *ProxyChain) _execute() (*[]byte, error) { + chain._validate_ctx_is_set() + if chain._abort_err != nil { + return nil, chain._abort_err + } + if chain.Context == nil { + return nil, errors.New("request ctx not set. Use ProxyChain.SetCtx()") + } + if chain.Request.URL.Scheme == "" { + return nil, errors.New("request url not set or invalid. Check ProxyChain ReqMods for issues") + } + + // Apply requestModifications to proxychain (pxc) + for _, applyRequestModificationsTo := range chain.requestModifications { + err := applyRequestModificationsTo(chain) + if err != nil { + return nil, chain.abort(err) + } + } + + // Send Request Upstream + resp, err := chain.Client.Do(chain.Request) + if err != nil { + return nil, chain.abort(err) + } + chain.Response = resp + + // Buffer response into memory + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, chain.abort(err) + } + chain.Body = body + defer resp.Body.Close() + + // Apply ResponseModifiers to proxychain (pxc) + for _, applyResultModificationsTo := range chain.resultModifications { + err := applyResultModificationsTo(chain) + if err != nil { + return nil, chain.abort(err) + } + } + + return &chain.Body, nil +} + +// Execute sends the request for the ProxyChain and returns the request to the sender +// and resets the fields so that the ProxyChain can be reused. +// if any step in the ProxyChain fails, the request will abort and a 500 error will +// be returned to the client +func (chain *ProxyChain) Execute() error { + defer chain._reset() + body, err := chain._execute() + if err != nil { + return err + } + // Return request back to client + return chain.Context.Send(*body) +} + +// ExecuteAPIContent sends the request for the ProxyChain and returns the response body as +// a structured API response to the client +// if any step in the ProxyChain fails, the request will abort and a 500 error will +// be returned to the client +func (chain *ProxyChain) ExecuteAPIContent() error { + defer chain._reset() + body, err := chain._execute() + if err != nil { + return err + } + // TODO: implement reader API + // Return request back to client + return chain.Context.Send(*body) +} + +// reconstructUrlFromReferer reconstructs the URL using the referer's scheme, host, and the relative path / queries +func reconstructUrlFromReferer(referer *url.URL, relativeUrl *url.URL) (*url.URL, error) { + + // Extract the real url from referer path + realUrl, err := url.Parse(strings.TrimPrefix(referer.Path, "/")) + if err != nil { + return nil, fmt.Errorf("error parsing real URL from referer '%s': %v", referer.Path, err) + } + + if realUrl.Scheme == "" || realUrl.Host == "" { + return nil, fmt.Errorf("invalid referer URL: %s", referer) + } + + return &url.URL{ + Scheme: referer.Scheme, + Host: referer.Host, + Path: realUrl.Path, + RawQuery: realUrl.RawQuery, + }, nil +} + +// extractUrl extracts a URL from the request ctx. If the URL in the request +// is a relative path, it reconstructs the full URL using the referer header. +func (chain *ProxyChain) extractUrl() (*url.URL, error) { + // try to extract url-encoded + reqUrl, err := url.QueryUnescape(chain.Context.Params("*")) + if err != nil { + reqUrl = chain.Context.Params("*") // fallback + } + + urlQuery, err := url.Parse(reqUrl) + if err != nil { + return nil, fmt.Errorf("error parsing request URL '%s': %v", reqUrl, err) + } + + // Handle standard paths + // eg: https://localhost:8080/https://realsite.com/images/foobar.jpg -> https://realsite.com/images/foobar.jpg + isRelativePath := urlQuery.Scheme == "" + if !isRelativePath { + return urlQuery, nil + } + + // Handle relative URLs + // eg: https://localhost:8080/images/foobar.jpg -> https://realsite.com/images/foobar.jpg + referer, err := url.Parse(chain.Context.Get("referer")) + relativePath := urlQuery + if err != nil { + return nil, fmt.Errorf("error parsing referer URL from req: '%s': %v", relativePath, err) + } + return reconstructUrlFromReferer(referer, relativePath) +} + +// SetCtx takes the request ctx from the client +// for the modifiers and execute function to use. +// it must be set everytime a new request comes through +// if the upstream request url cannot be extracted from the ctx, +// a 500 error will be sent back to the client +func (chain *ProxyChain) SetCtx(ctx *fiber.Ctx) *ProxyChain { + chain.Context = ctx + + // initialize the request and prepare it for modification + req, err := chain._initialize_request() + if err != nil { + chain._abort_err = chain.abort(err) + } + chain.Request = req + + // extract the URL for the request and add it to the new request + url, err := chain.extractUrl() + if err != nil { + chain._abort_err = chain.abort(err) + } + chain.Request.URL = url + + return chain +} + +func (pxc *ProxyChain) _validate_ctx_is_set() { + if pxc.Context != nil { + return + } + err := errors.New("proxyChain was called without setting a fiber Ctx. Use ProxyChain.SetCtx()") + pxc._abort_err = pxc.abort(err) +} + +// SetClient sets a new upstream http client transport +// useful for modifying TLS +func (pxc *ProxyChain) SetClient(httpClient *http.Client) *ProxyChain { + pxc.Client = httpClient + return pxc +} + +// SetVerbose changes the logging behavior to print +// the modification steps and applied rulesets for debugging +func (pxc *ProxyChain) SetVerbose() *ProxyChain { + pxc.verbose = true + return pxc +} + +// abort proxychain and return 500 error to client +// this will prevent Execute from firing and reset the state +// returns the initial error enriched with context +func (pxc *ProxyChain) abort(err error) error { + defer pxc._reset() + pxc._abort_err = err + pxc.Context.Response().SetStatusCode(500) + e := fmt.Errorf("ProxyChain error for '%s': %s", pxc.Request.URL.String(), err.Error()) + pxc.Context.SendString(e.Error()) + log.Println(e.Error()) + return e +} + +// internal function to reset state of ProxyChain for reuse +func (pxc *ProxyChain) _reset() { + pxc._abort_err = nil + pxc.Body = nil + pxc.Request = nil + pxc.Response = nil + pxc.Context = nil + pxc.Request.URL = nil +} + +// NewProxyChain initializes a new ProxyChain +func NewProxyChain() *ProxyChain { + px := new(ProxyChain) + px.Client = defaultClient + return px +} diff --git a/proxychain/proxychain_pool.go b/proxychain/proxychain_pool.go new file mode 100644 index 0000000..005bd28 --- /dev/null +++ b/proxychain/proxychain_pool.go @@ -0,0 +1,11 @@ +package proxychain + +import ( + "net/url" +) + +type ProxyChainPool map[url.URL]ProxyChain + +func NewProxyChainPool() ProxyChainPool { + return map[url.URL]ProxyChain{} +} diff --git a/proxychain/rqm/block_outgoing_cookies.go b/proxychain/rqm/block_outgoing_cookies.go new file mode 100644 index 0000000..b59a9ff --- /dev/null +++ b/proxychain/rqm/block_outgoing_cookies.go @@ -0,0 +1,41 @@ +package rqm // ReQuestModifier + +import ( + "ladder/proxychain" +) + +// BlockOutgoingCookies prevents ALL cookies from being sent from the client +// to the upstream proxy server. +func BlockOutgoingCookies() proxychain.RequestModification { + return func(px *proxychain.ProxyChain) error { + px.Request.Header.Del("Cookie") + return nil + } +} + +// BlockOutgoingCookiesExcept prevents non-whitelisted cookies from being sent from the client +// to the upstream proxy server. Cookies whose names are in the whitelist are not removed. +func BlockOutgoingCookiesExcept(whitelist ...string) proxychain.RequestModification { + return func(px *proxychain.ProxyChain) error { + // Convert whitelist slice to a map for efficient lookups + whitelistMap := make(map[string]struct{}) + for _, cookieName := range whitelist { + whitelistMap[cookieName] = struct{}{} + } + + // Get all cookies from the request header + cookies := px.Request.Cookies() + + // Clear the original Cookie header + px.Request.Header.Del("Cookie") + + // Re-add cookies that are in the whitelist + for _, cookie := range cookies { + if _, found := whitelistMap[cookie.Name]; found { + px.Request.AddCookie(cookie) + } + } + + return nil + } +} diff --git a/proxychain/rqm/masquerade_as_trusted_bot.go b/proxychain/rqm/masquerade_as_trusted_bot.go new file mode 100644 index 0000000..550329a --- /dev/null +++ b/proxychain/rqm/masquerade_as_trusted_bot.go @@ -0,0 +1,32 @@ +package rqm // ReQuestModifier + +import ( + "ladder/proxychain" +) + +// MasqueradeAsGoogleBot modifies user agent and x-forwarded for +// to appear to be a Google Bot +func MasqueradeAsGoogleBot() proxychain.RequestModification { + const botUA string = "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; http://www.google.com/bot.html) Chrome/79.0.3945.120 Safari/537.36" + const botIP string = "66.249.78.8" // TODO: create a random ip pool from https://developers.google.com/static/search/apis/ipranges/googlebot.json + return masqueradeAsTrustedBot(botUA, botIP) +} + +// MasqueradeAsBingBot modifies user agent and x-forwarded for +// to appear to be a Bing Bot +func MasqueradeAsBingBot() proxychain.RequestModification { + const botUA string = "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/79.0.3945.120 Safari/537.36" + const botIP string = "13.66.144.9" // https://www.bing.com/toolbox/bingbot.json + return masqueradeAsTrustedBot(botUA, botIP) +} + +func masqueradeAsTrustedBot(botUA string, botIP string) proxychain.RequestModification { + return func(px *proxychain.ProxyChain) error { + px.AddRequestModifications( + SpoofUserAgent(botUA), + SpoofXForwardedFor(botIP), + SpoofReferrer(""), + ) + return nil + } +} diff --git a/proxychain/rqm/modify_domain_with_regex.go b/proxychain/rqm/modify_domain_with_regex.go new file mode 100644 index 0000000..d3b93dc --- /dev/null +++ b/proxychain/rqm/modify_domain_with_regex.go @@ -0,0 +1,13 @@ +package rqm // ReQuestModifier + +import ( + "ladder/proxychain" + "regexp" +) + +func ModifyDomainWithRegex(match regexp.Regexp, replacement string) proxychain.RequestModification { + return func(px *proxychain.ProxyChain) error { + px.Request.URL.Host = match.ReplaceAllString(px.Request.URL.Host, replacement) + return nil + } +} diff --git a/proxychain/rqm/modify_path_with_regex.go b/proxychain/rqm/modify_path_with_regex.go new file mode 100644 index 0000000..8b96c6f --- /dev/null +++ b/proxychain/rqm/modify_path_with_regex.go @@ -0,0 +1,13 @@ +package rqm // ReQuestModifier + +import ( + "ladder/proxychain" + "regexp" +) + +func ModifyPathWithRegex(match regexp.Regexp, replacement string) proxychain.RequestModification { + return func(px *proxychain.ProxyChain) error { + px.Request.URL.Path = match.ReplaceAllString(px.Request.URL.Path, replacement) + return nil + } +} diff --git a/proxychain/rqm/modify_query_params.go b/proxychain/rqm/modify_query_params.go new file mode 100644 index 0000000..1018a00 --- /dev/null +++ b/proxychain/rqm/modify_query_params.go @@ -0,0 +1,20 @@ +package rqm // ReQuestModifier + +import ( + "ladder/proxychain" +) + +// ModifyQueryParams replaces query parameter values in URL's query params in a ProxyChain's URL. +// If the query param key doesn't exist, it is created. +func ModifyQueryParams(key string, value string) proxychain.RequestModification { + return func(px *proxychain.ProxyChain) error { + q := px.Request.URL.Query() + if value == "" { + q.Del(key) + return nil + } + q.Set(key, value) + px.Request.URL.RawQuery = q.Encode() + return nil + } +} diff --git a/proxychain/rqm/request_archive_is.go b/proxychain/rqm/request_archive_is.go new file mode 100644 index 0000000..525b092 --- /dev/null +++ b/proxychain/rqm/request_archive_is.go @@ -0,0 +1,27 @@ +package rqm + +import ( + "ladder/proxychain" + "net/url" +) + +const archivistUrl string = "https://archive.is/latest/" + +// RequestArchiveIs modifies a ProxyChain's URL to request an archived version from archive.is +func RequestArchiveIs() proxychain.RequestModification { + return func(px *proxychain.ProxyChain) error { + px.Request.URL.RawQuery = "" + newURLString := archivistUrl + px.Request.URL.String() + newURL, err := url.Parse(newURLString) + if err != nil { + return err + } + + // archivist seems to sabotage requests from cloudflare's DNS + // bypass this just in case + px.AddRequestModifications(ResolveWithGoogleDoH()) + + px.Request.URL = newURL + return nil + } +} diff --git a/proxychain/rqm/request_google_cache.go b/proxychain/rqm/request_google_cache.go new file mode 100644 index 0000000..c9d8d9b --- /dev/null +++ b/proxychain/rqm/request_google_cache.go @@ -0,0 +1,21 @@ +package rqm // ReQuestModifier + +import ( + "ladder/proxychain" + "net/url" +) + +const googleCacheUrl string = "https://webcache.googleusercontent.com/search?q=cache:" + +// RequestGoogleCache modifies a ProxyChain's URL to request its Google Cache version. +func RequestGoogleCache() proxychain.RequestModification { + return func(px *proxychain.ProxyChain) error { + encodedURL := url.QueryEscape(px.Request.URL.String()) + newURL, err := url.Parse(googleCacheUrl + encodedURL) + if err != nil { + return err + } + px.Request.URL = newURL + return nil + } +} diff --git a/proxychain/rqm/request_wayback_machine.go b/proxychain/rqm/request_wayback_machine.go new file mode 100644 index 0000000..a01926b --- /dev/null +++ b/proxychain/rqm/request_wayback_machine.go @@ -0,0 +1,22 @@ +package rqm // ReQuestModifier + +import ( + "ladder/proxychain" + "net/url" +) + +const waybackUrl string = "https://web.archive.org/web/" + +// RequestWaybackMachine modifies a ProxyChain's URL to request the wayback machine (archive.org) version. +func RequestWaybackMachine() proxychain.RequestModification { + return func(px *proxychain.ProxyChain) error { + px.Request.URL.RawQuery = "" + newURLString := waybackUrl + px.Request.URL.String() + newURL, err := url.Parse(newURLString) + if err != nil { + return err + } + px.Request.URL = newURL + return nil + } +} diff --git a/proxychain/rqm/resolve_with_google_doh.go b/proxychain/rqm/resolve_with_google_doh.go new file mode 100644 index 0000000..2e2fb05 --- /dev/null +++ b/proxychain/rqm/resolve_with_google_doh.go @@ -0,0 +1,80 @@ +package rqm + +import ( + "context" + "encoding/json" + "fmt" + "ladder/proxychain" + "net" + "net/http" + "time" +) + +// resolveWithGoogleDoH resolves DNS using Google's DNS-over-HTTPS +func resolveWithGoogleDoH(host string) (string, error) { + url := "https://dns.google/resolve?name=" + host + "&type=A" + resp, err := http.Get(url) + if err != nil { + return "", err + } + defer resp.Body.Close() + + var result struct { + Answer []struct { + Data string `json:"data"` + } `json:"Answer"` + } + err = json.NewDecoder(resp.Body).Decode(&result) + if err != nil { + return "", err + } + + // Get the first A record + if len(result.Answer) > 0 { + return result.Answer[0].Data, nil + } + return "", fmt.Errorf("no DoH DNS record found for %s", host) +} + +// ResolveWithGoogleDoH modifies a ProxyChain's client to make the request but resolve the URL +// using Google's DNS over HTTPs service +func ResolveWithGoogleDoH() proxychain.RequestModification { + return func(px *proxychain.ProxyChain) error { + client := &http.Client{ + Timeout: px.Client.Timeout, + } + + dialer := &net.Dialer{ + Timeout: 5 * time.Second, + KeepAlive: 5 * time.Second, + } + + customDialContext := func(ctx context.Context, network, addr string) (net.Conn, error) { + host, port, err := net.SplitHostPort(addr) + if err != nil { + // If the addr doesn't include a port, determine it based on the URL scheme + if px.Request.URL.Scheme == "https" { + port = "443" + } else { + port = "80" + } + host = addr // assume the entire addr is the host + } + + resolvedHost, err := resolveWithGoogleDoH(host) + if err != nil { + return nil, err + } + + return dialer.DialContext(ctx, network, net.JoinHostPort(resolvedHost, port)) + } + + patchedTransportWithDoH := &http.Transport{ + DialContext: customDialContext, + } + + client.Transport = patchedTransportWithDoH + px.Client = client // Assign the modified client to the ProxyChain + return nil + } +} diff --git a/proxychain/rqm/spoof_origin.go b/proxychain/rqm/spoof_origin.go new file mode 100644 index 0000000..07bbd8a --- /dev/null +++ b/proxychain/rqm/spoof_origin.go @@ -0,0 +1,24 @@ +package rqm // ReQuestModifier + +import ( + "ladder/proxychain" +) + +// SpoofOrigin modifies the origin header +// if the upstream server returns a Vary header +// it means you might get a different response if you change this +func SpoofOrigin(url string) proxychain.RequestModification { + return func(px *proxychain.ProxyChain) error { + px.Request.Header.Set("origin", url) + return nil + } +} + +// HideOrigin modifies the origin header +// so that it is the original origin, not the proxy +func HideOrigin() proxychain.RequestModification { + return func(px *proxychain.ProxyChain) error { + px.Request.Header.Set("origin", px.Request.URL.String()) + return nil + } +} diff --git a/proxychain/rqm/spoof_referrer.go b/proxychain/rqm/spoof_referrer.go new file mode 100644 index 0000000..1f0ae46 --- /dev/null +++ b/proxychain/rqm/spoof_referrer.go @@ -0,0 +1,29 @@ +package rqm // ReQuestModifier + +import ( + "ladder/proxychain" +) + +// SpoofReferrer modifies the referrer header +// useful if the page can be accessed from a search engine +// or social media site, but not by browsing the website itself +// if url is "", then the referrer header is removed +func SpoofReferrer(url string) proxychain.RequestModification { + return func(px *proxychain.ProxyChain) error { + if url == "" { + px.Request.Header.Del("referrer") + return nil + } + px.Request.Header.Set("referrer", url) + return nil + } +} + +// HideReferrer modifies the referrer header +// so that it is the original referrer, not the proxy +func HideReferrer() proxychain.RequestModification { + return func(px *proxychain.ProxyChain) error { + px.Request.Header.Set("referrer", px.Request.URL.String()) + return nil + } +} diff --git a/proxychain/rqm/spoof_user_agent.go b/proxychain/rqm/spoof_user_agent.go new file mode 100644 index 0000000..4950282 --- /dev/null +++ b/proxychain/rqm/spoof_user_agent.go @@ -0,0 +1,13 @@ +package rqm // ReQuestModifier + +import ( + "ladder/proxychain" +) + +// SpoofUserAgent modifies the user agent +func SpoofUserAgent(ua string) proxychain.RequestModification { + return func(px *proxychain.ProxyChain) error { + px.Request.Header.Set("user-agent", ua) + return nil + } +} diff --git a/proxychain/rqm/spoof_x_forwarded_for.go b/proxychain/rqm/spoof_x_forwarded_for.go new file mode 100644 index 0000000..93a4e91 --- /dev/null +++ b/proxychain/rqm/spoof_x_forwarded_for.go @@ -0,0 +1,14 @@ +package rqm // ReQuestModifier + +import ( + "ladder/proxychain" +) + +// SpoofXForwardedFor modifies the X-Forwarded-For header +// in some cases, a forward proxy may interpret this as the source IP +func SpoofXForwardedFor(ip string) proxychain.RequestModification { + return func(px *proxychain.ProxyChain) error { + px.Request.Header.Set("X-FORWARDED-FOR", ip) + return nil + } +} diff --git a/proxychain/rsm/block_incoming_cookies.go b/proxychain/rsm/block_incoming_cookies.go new file mode 100644 index 0000000..0884a6f --- /dev/null +++ b/proxychain/rsm/block_incoming_cookies.go @@ -0,0 +1,58 @@ +package rsm // ReSponseModifers + +import ( + "ladder/proxychain" + "net/http" +) + +// BlockIncomingCookies prevents ALL cookies from being sent from the proxy server +// to the client. +func BlockIncomingCookies(whitelist ...string) proxychain.ResponseModification { + return func(px *proxychain.ProxyChain) error { + px.Response.Header.Del("Set-Cookie") + return nil + } +} + +// BlockIncomingCookiesExcept prevents non-whitelisted cookies from being sent from the proxy server +// to the client. Cookies whose names are in the whitelist are not removed. +func BlockIncomingCookiesExcept(whitelist ...string) proxychain.ResponseModification { + return func(px *proxychain.ProxyChain) error { + // Convert whitelist slice to a map for efficient lookups + whitelistMap := make(map[string]struct{}) + for _, cookieName := range whitelist { + whitelistMap[cookieName] = struct{}{} + } + + // If the response has no cookies, return early + if px.Response.Header == nil { + return nil + } + + // Filter the cookies in the response + filteredCookies := []string{} + for _, cookieStr := range px.Response.Header["Set-Cookie"] { + cookie := parseCookie(cookieStr) + if _, found := whitelistMap[cookie.Name]; found { + filteredCookies = append(filteredCookies, cookieStr) + } + } + + // Update the Set-Cookie header with the filtered cookies + if len(filteredCookies) > 0 { + px.Response.Header["Set-Cookie"] = filteredCookies + } else { + px.Response.Header.Del("Set-Cookie") + } + + return nil + } +} + +// parseCookie parses a cookie string and returns an http.Cookie object. +func parseCookie(cookieStr string) *http.Cookie { + header := http.Header{} + header.Add("Set-Cookie", cookieStr) + request := http.Request{Header: header} + return request.Cookies()[0] +} diff --git a/proxychain/rsm/bypass_cors.go b/proxychain/rsm/bypass_cors.go new file mode 100644 index 0000000..dbf7013 --- /dev/null +++ b/proxychain/rsm/bypass_cors.go @@ -0,0 +1,20 @@ +package rsm // ReSponseModifers + +import ( + "ladder/proxychain" +) + +// BypassCORs modifies response headers to prevent the browser +// from enforcing any CORS restrictions +func BypassCORS() proxychain.ResponseModification { + return func(px *proxychain.ProxyChain) error { + px.AddResultModifications( + ModifyResponseHeader("Access-Control-Allow-Origin", "*"), + ModifyResponseHeader("Access-Control-Expose-Headers", "*"), + ModifyResponseHeader("Access-Control-Allow-Credentials", "true"), + ModifyResponseHeader("Access-Control-Allow-Methods", "GET, PUT, POST, DELETE, HEAD, OPTIONS, PATCH"), + DeleteResponseHeader("X-Frame-Options"), + ) + return nil + } +} diff --git a/proxychain/rsm/bypass_csp.go b/proxychain/rsm/bypass_csp.go new file mode 100644 index 0000000..4262590 --- /dev/null +++ b/proxychain/rsm/bypass_csp.go @@ -0,0 +1,19 @@ +package rsm // ReSponseModifers + +import ( + "ladder/proxychain" +) + +// BypassCSP modifies response headers to prevent the browser +// from enforcing any CORS restrictions +func BypassCSP() proxychain.ResponseModification { + return func(px *proxychain.ProxyChain) error { + px.AddResultModifications( + ModifyResponseHeader("Access-Control-Allow-Origin", "*"), + ModifyResponseHeader("Access-Control-Expose-Headers", "*"), + ModifyResponseHeader("Access-Control-Allow-Credentials", "true"), + ModifyResponseHeader("Access-Control-Allow-Methods", ""), + ) + return nil + } +} diff --git a/proxychain/rsm/modify_response_header.go b/proxychain/rsm/modify_response_header.go new file mode 100644 index 0000000..d7a473e --- /dev/null +++ b/proxychain/rsm/modify_response_header.go @@ -0,0 +1,26 @@ +package rsm // ReSponseModifers + +import ( + "ladder/proxychain" +) + +// ModifyResponseHeader modifies response headers from the upstream server +// if value is "", then the response header is deleted. +func ModifyResponseHeader(key string, value string) proxychain.ResponseModification { + return func(px *proxychain.ProxyChain) error { + if value == "" { + px.Context.Response().Header.Del(key) + return nil + } + px.Context.Response().Header.Set(key, value) + return nil + } +} + +// DeleteResponseHeader removes response headers from the upstream server +func DeleteResponseHeader(key string) proxychain.ResponseModification { + return func(px *proxychain.ProxyChain) error { + px.Context.Response().Header.Del(key) + return nil + } +} diff --git a/proxychain/strategy/cloudflare.go b/proxychain/strategy/cloudflare.go new file mode 100644 index 0000000..6387736 --- /dev/null +++ b/proxychain/strategy/cloudflare.go @@ -0,0 +1,8 @@ +package strategy + +/* +var Cloudflare = proxy.Strategy{ + tactic.NoCookie(), + // ... other tactics ... +} +*/