begin refactor of proxy engine

cmd/main.go

@@ -8,7 +8,7 @@ import (
 	"strings"
 
 	"ladder/handlers"
-	"ladder/handlers/cli"
+	"ladder/internal/cli"
 
 	"github.com/akamensky/argparse"
 	"github.com/gofiber/fiber/v2"
@@ -40,6 +40,13 @@ func main() {
 		Help:     "This will spawn multiple processes listening",
 	})
 
+	verbose := parser.Flag("v", "verbose", &argparse.Options{
+		Required: false,
+		Help:     "Adds verbose logging",
+	})
+
+	// TODO: add version flag that reads from handers/VERSION
+
 	ruleset := parser.String("r", "ruleset", &argparse.Options{
 		Required: false,
 		Help:     "File, Directory or URL to a ruleset.yaml. Overrides RULESET environment variable.",
@@ -84,6 +91,7 @@ func main() {
 		},
 	)
 
+	// TODO: move to cmd/auth.go
 	userpass := os.Getenv("USERPASS")
 	if userpass != "" {
 		userpass := strings.Split(userpass, ":")
@@ -94,6 +102,7 @@ func main() {
 		}))
 	}
 
+	// TODO: move to handlers/favicon.go
 	app.Use(favicon.New(favicon.Config{
 		Data: []byte(faviconData),
 		URL:  "/favicon.ico",
@@ -107,6 +116,8 @@ func main() {
 	}
 
 	app.Get("/", handlers.Form)
+
+	// TODO: move this logic to handers/styles.go
 	app.Get("/styles.css", func(c *fiber.Ctx) error {
 		cssData, err := cssData.ReadFile("styles.css")
 		if err != nil {
@@ -115,10 +126,17 @@ func main() {
 		c.Set("Content-Type", "text/css")
 		return c.Send(cssData)
 	})
+
 	app.Get("ruleset", handlers.Ruleset)
 
 	app.Get("raw/*", handlers.Raw)
 	app.Get("api/*", handlers.Api)
-	app.Get("/*", handlers.ProxySite(*ruleset))
+
+	proxyOpts := &handlers.ProxyOptions{
+		Verbose:     *verbose,
+		RulesetPath: *ruleset,
+	}
+
+	app.Get("/*", handlers.NewProxySiteHandler(proxyOpts))
 	log.Fatal(app.Listen(":" + *port))
 }

handlers/proxy.go

@@ -11,6 +11,9 @@ import (
 	"strings"
 
 	"ladder/pkg/ruleset"
+	"ladder/proxychain"
+	"ladder/proxychain/rqm"
+	"ladder/proxychain/rsm"
 
 	"github.com/PuerkitoBio/goquery"
 	"github.com/gofiber/fiber/v2"
@@ -30,88 +33,32 @@ func init() {
 	}
 }
 
-// extracts a URL from the request ctx. If the URL in the request
+type ProxyOptions struct {
-// is a relative path, it reconstructs the full URL using the referer header.
+	RulesetPath string
-func extractUrl(c *fiber.Ctx) (string, error) {
+	Verbose     bool
-	// try to extract url-encoded
-	reqUrl, err := url.QueryUnescape(c.Params("*"))
-	if err != nil {
-		// fallback
-		reqUrl = c.Params("*")
 }
 
-	// Extract the actual path from req ctx
+func NewProxySiteHandler(opts *ProxyOptions) fiber.Handler {
-	urlQuery, err := url.Parse(reqUrl)
+	var rs ruleset.RuleSet
-	if err != nil {
+	if opts.RulesetPath != "" {
-		return "", fmt.Errorf("error parsing request URL '%s': %v", reqUrl, err)
+		r, err := ruleset.NewRuleset(opts.RulesetPath)
-	}
-
-	isRelativePath := urlQuery.Scheme == ""
-
-	// eg: https://localhost:8080/images/foobar.jpg -> https://realsite.com/images/foobar.jpg
-	if isRelativePath {
-		// Parse the referer URL from the request header.
-		refererUrl, err := url.Parse(c.Get("referer"))
-		if err != nil {
-			return "", fmt.Errorf("error parsing referer URL from req: '%s': %v", reqUrl, err)
-		}
-
-		// Extract the real url from referer path
-		realUrl, err := url.Parse(strings.TrimPrefix(refererUrl.Path, "/"))
-		if err != nil {
-			return "", fmt.Errorf("error parsing real URL from referer '%s': %v", refererUrl.Path, err)
-		}
-
-		// reconstruct the full URL using the referer's scheme, host, and the relative path / queries
-		fullUrl := &url.URL{
-			Scheme:   realUrl.Scheme,
-			Host:     realUrl.Host,
-			Path:     urlQuery.Path,
-			RawQuery: urlQuery.RawQuery,
-		}
-
-		if os.Getenv("LOG_URLS") == "true" {
-			log.Printf("modified relative URL: '%s' -> '%s'", reqUrl, fullUrl.String())
-		}
-		return fullUrl.String(), nil
-
-	}
-
-	// default behavior:
-	// eg: https://localhost:8080/https://realsite.com/images/foobar.jpg -> https://realsite.com/images/foobar.jpg
-	return urlQuery.String(), nil
-
-}
-
-func ProxySite(rulesetPath string) fiber.Handler {
-	if rulesetPath != "" {
-		rs, err := ruleset.NewRuleset(rulesetPath)
 		if err != nil {
 			panic(err)
 		}
-		rulesSet = rs
+		rs = r
 	}
 
 	return func(c *fiber.Ctx) error {
-		// Get the url from the URL
+		return proxychain.NewProxyChain().
-		url, err := extractUrl(c)
+			SetCtx(c).
-		if err != nil {
+			AddRuleset(&rs).
-			log.Println("ERROR In URL extraction:", err)
+			SetRequestModifications(
-		}
+				rqm.BlockOutgoingCookies(),
-
+			).
-		queries := c.Queries()
+			SetResultModifications(
-		body, _, resp, err := fetchSite(url, queries)
+				rsm.BlockIncomingCookies(),
-		if err != nil {
+			).
-			log.Println("ERROR:", err)
+			Execute()
-			c.SendStatus(fiber.StatusInternalServerError)
-			return c.SendString(err.Error())
-		}
-
-		c.Cookie(&fiber.Cookie{})
-		c.Set("Content-Type", resp.Header.Get("Content-Type"))
-		c.Set("Content-Security-Policy", resp.Header.Get("Content-Security-Policy"))
-
-		return c.SendString(body)
 	}
 }
 

handlers/proxy.test.go

@@ -14,7 +14,7 @@ import (
 
 func TestProxySite(t *testing.T) {
 	app := fiber.New()
-	app.Get("/:url", ProxySite(""))
+	app.Get("/:url", NewProxySiteHandler(nil))
 
 	req := httptest.NewRequest("GET", "/https://example.com", nil)
 	resp, err := app.Test(req)

proxychain/proxychain.go

@@ -0,0 +1,384 @@
+package proxychain
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"ladder/pkg/ruleset"
+
+	"github.com/gofiber/fiber/v2"
+)
+
+var defaultClient *http.Client
+
+func DefaultClient() {
+	defaultClient = &http.Client{
+		Timeout: 15,
+	}
+}
+
+/*
+ProxyChain manages the process of forwarding an HTTP request to an upstream server,
+applying request and response modifications along the way.
+
+  - It accepts incoming HTTP requests (as a Fiber *ctx), and applies
+    request modifiers (ReqMods) and response modifiers (ResMods) before passing the
+    upstream response back to the client.
+
+  - ProxyChains can be reused to avoid memory allocations.
+
+---
+
+# EXAMPLE
+
+```
+
+import (
+
+	"ladder/internal/proxychain/rqm"
+	"ladder/internal/proxychain/rsm"
+	"ladder/internal/proxychain"
+
+)
+
+proxychain.NewProxyChain().
+
+	SetCtx(c).
+	AddRuleset(&rs).
+	SetRequestModifications(
+		rqm.BlockOutgoingCookies(),
+	).
+	SetResultModifications(
+		rsm.BlockIncomingCookies(),
+	).
+	Execute()
+
+```
+
+	client            ladder service            upstream
+
+┌─────────┐    ┌────────────────────────┐    ┌─────────┐
+│         │GET │                        │    │         │
+│       req────┼───► ProxyChain         │    │         │
+│         │    │       │                │    │         │
+│         │    │       ▼                │    │         │
+│         │    │     apply              │    │         │
+│         │    │ RequestModifications   │    │         │
+│         │    │       │                │    │         │
+│         │    │       ▼                │    │         │
+│         │    │     send        GET    │    │         │
+│         │    │     Request req────────┼─►  │         │
+│         │    │                        │    │         │
+│         │    │                 200 OK │    │         │
+│         │    │       ┌────────────────┼─response     │
+│         │    │       ▼                │    │         │
+│         │    │     apply              │    │         │
+│         │    │ ResultModifications    │    │         │
+│         │    │       │                │    │         │
+│         │◄───┼───────┘                │    │         │
+│         │    │ 200 OK                 │    │         │
+│         │    │                        │    │         │
+└─────────┘    └────────────────────────┘    └─────────┘
+*/
+type ProxyChain struct {
+	Context              *fiber.Ctx
+	Client               *http.Client
+	Request              *http.Request
+	Response             *http.Response
+	Body                 []byte
+	requestModifications []RequestModification
+	resultModifications  []ResponseModification
+	ruleset              *ruleset.RuleSet
+	verbose              bool
+	_abort_err           error
+}
+
+// a ProxyStrategy is a pre-built proxychain with purpose-built defaults
+type ProxyStrategy ProxyChain
+
+// A RequestModification is a function that should operate on the
+// ProxyChain Req or Client field, using the fiber ctx as needed.
+type RequestModification func(*ProxyChain) error
+
+// A ResponseModification is a function that should operate on the
+// ProxyChain Res (http result) & Body (buffered http response body) field
+type ResponseModification func(*ProxyChain) error
+
+// SetRequestModifications sets the ProxyChain's request modifers
+// the modifier will not fire until ProxyChain.Execute() is run.
+func (chain *ProxyChain) SetRequestModifications(mods ...RequestModification) *ProxyChain {
+	chain.requestModifications = mods
+	return chain
+}
+
+// AddRequestModifications sets the ProxyChain's request modifers
+// the modifier will not fire until ProxyChain.Execute() is run.
+func (chain *ProxyChain) AddRequestModifications(mods ...RequestModification) *ProxyChain {
+	chain.requestModifications = append(chain.requestModifications, mods...)
+	return chain
+}
+
+// SetResultModifications sets the ProxyChain's response modifers
+// the modifier will not fire until ProxyChain.Execute() is run.
+func (chain *ProxyChain) SetResultModifications(mods ...ResponseModification) *ProxyChain {
+	chain.resultModifications = mods
+	return chain
+}
+
+// AddResultModifications adds to the ProxyChain's response modifers
+// the modifier will not fire until ProxyChain.Execute() is run.
+func (chain *ProxyChain) AddResultModifications(mods ...ResponseModification) *ProxyChain {
+	chain.resultModifications = append(chain.resultModifications, mods...)
+	return chain
+}
+
+// Adds a ruleset to ProxyChain
+func (chain *ProxyChain) AddRuleset(rs *ruleset.RuleSet) *ProxyChain {
+	chain.ruleset = rs
+	// TODO: add _applyRuleset method
+	return chain
+}
+
+func (chain *ProxyChain) _initialize_request() (*http.Request, error) {
+	// initialize a request (without url)
+	req, err := http.NewRequest(chain.Context.Method(), "", nil)
+	if err != nil {
+		return nil, err
+	}
+	chain.Request = req
+	switch chain.Context.Method() {
+	case "GET":
+	case "DELETE":
+	case "HEAD":
+	case "OPTIONS":
+		break
+	case "POST":
+	case "PUT":
+	case "PATCH":
+		// stream content of body from client request to upstream request
+		chain.Request.Body = io.NopCloser(chain.Context.Request().BodyStream())
+	default:
+		return nil, fmt.Errorf("unsupported request method from client: '%s'", chain.Context.Method())
+	}
+
+	// copy client request headers to upstream request headers
+	forwardHeaders := func(key []byte, val []byte) {
+		req.Header.Set(string(key), string(val))
+	}
+	clientHeaders := &chain.Context.Request().Header
+	clientHeaders.VisitAll(forwardHeaders)
+
+	return req, nil
+}
+
+// _execute sends the request for the ProxyChain and returns the raw body only
+// the caller is responsible for returning a response back to the requestor
+// the caller is also responsible for calling pxc._reset() when they are done with the body
+func (chain *ProxyChain) _execute() (*[]byte, error) {
+	chain._validate_ctx_is_set()
+	if chain._abort_err != nil {
+		return nil, chain._abort_err
+	}
+	if chain.Context == nil {
+		return nil, errors.New("request ctx not set. Use ProxyChain.SetCtx()")
+	}
+	if chain.Request.URL.Scheme == "" {
+		return nil, errors.New("request url not set or invalid. Check ProxyChain ReqMods for issues")
+	}
+
+	// Apply requestModifications to proxychain (pxc)
+	for _, applyRequestModificationsTo := range chain.requestModifications {
+		err := applyRequestModificationsTo(chain)
+		if err != nil {
+			return nil, chain.abort(err)
+		}
+	}
+
+	// Send Request Upstream
+	resp, err := chain.Client.Do(chain.Request)
+	if err != nil {
+		return nil, chain.abort(err)
+	}
+	chain.Response = resp
+
+	// Buffer response into memory
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, chain.abort(err)
+	}
+	chain.Body = body
+	defer resp.Body.Close()
+
+	// Apply ResponseModifiers to proxychain (pxc)
+	for _, applyResultModificationsTo := range chain.resultModifications {
+		err := applyResultModificationsTo(chain)
+		if err != nil {
+			return nil, chain.abort(err)
+		}
+	}
+
+	return &chain.Body, nil
+}
+
+// Execute sends the request for the ProxyChain and returns the request to the sender
+// and resets the fields so that the ProxyChain can be reused.
+// if any step in the ProxyChain fails, the request will abort and a 500 error will
+// be returned to the client
+func (chain *ProxyChain) Execute() error {
+	defer chain._reset()
+	body, err := chain._execute()
+	if err != nil {
+		return err
+	}
+	// Return request back to client
+	return chain.Context.Send(*body)
+}
+
+// ExecuteAPIContent sends the request for the ProxyChain and returns the response body as
+// a structured API response to the client
+// if any step in the ProxyChain fails, the request will abort and a 500 error will
+// be returned to the client
+func (chain *ProxyChain) ExecuteAPIContent() error {
+	defer chain._reset()
+	body, err := chain._execute()
+	if err != nil {
+		return err
+	}
+	// TODO: implement reader API
+	// Return request back to client
+	return chain.Context.Send(*body)
+}
+
+// reconstructUrlFromReferer reconstructs the URL using the referer's scheme, host, and the relative path / queries
+func reconstructUrlFromReferer(referer *url.URL, relativeUrl *url.URL) (*url.URL, error) {
+
+	// Extract the real url from referer path
+	realUrl, err := url.Parse(strings.TrimPrefix(referer.Path, "/"))
+	if err != nil {
+		return nil, fmt.Errorf("error parsing real URL from referer '%s': %v", referer.Path, err)
+	}
+
+	if realUrl.Scheme == "" || realUrl.Host == "" {
+		return nil, fmt.Errorf("invalid referer URL: %s", referer)
+	}
+
+	return &url.URL{
+		Scheme:   referer.Scheme,
+		Host:     referer.Host,
+		Path:     realUrl.Path,
+		RawQuery: realUrl.RawQuery,
+	}, nil
+}
+
+// extractUrl extracts a URL from the request ctx. If the URL in the request
+// is a relative path, it reconstructs the full URL using the referer header.
+func (chain *ProxyChain) extractUrl() (*url.URL, error) {
+	// try to extract url-encoded
+	reqUrl, err := url.QueryUnescape(chain.Context.Params("*"))
+	if err != nil {
+		reqUrl = chain.Context.Params("*") // fallback
+	}
+
+	urlQuery, err := url.Parse(reqUrl)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing request URL '%s': %v", reqUrl, err)
+	}
+
+	// Handle standard paths
+	// eg: https://localhost:8080/https://realsite.com/images/foobar.jpg -> https://realsite.com/images/foobar.jpg
+	isRelativePath := urlQuery.Scheme == ""
+	if !isRelativePath {
+		return urlQuery, nil
+	}
+
+	// Handle relative URLs
+	// eg: https://localhost:8080/images/foobar.jpg -> https://realsite.com/images/foobar.jpg
+	referer, err := url.Parse(chain.Context.Get("referer"))
+	relativePath := urlQuery
+	if err != nil {
+		return nil, fmt.Errorf("error parsing referer URL from req: '%s': %v", relativePath, err)
+	}
+	return reconstructUrlFromReferer(referer, relativePath)
+}
+
+// SetCtx takes the request ctx from the client
+// for the modifiers and execute function to use.
+// it must be set everytime a new request comes through
+// if the upstream request url cannot be extracted from the ctx,
+// a 500 error will be sent back to the client
+func (chain *ProxyChain) SetCtx(ctx *fiber.Ctx) *ProxyChain {
+	chain.Context = ctx
+
+	// initialize the request and prepare it for modification
+	req, err := chain._initialize_request()
+	if err != nil {
+		chain._abort_err = chain.abort(err)
+	}
+	chain.Request = req
+
+	// extract the URL for the request and add it to the new request
+	url, err := chain.extractUrl()
+	if err != nil {
+		chain._abort_err = chain.abort(err)
+	}
+	chain.Request.URL = url
+
+	return chain
+}
+
+func (pxc *ProxyChain) _validate_ctx_is_set() {
+	if pxc.Context != nil {
+		return
+	}
+	err := errors.New("proxyChain was called without setting a fiber Ctx. Use ProxyChain.SetCtx()")
+	pxc._abort_err = pxc.abort(err)
+}
+
+// SetClient sets a new upstream http client transport
+// useful for modifying TLS
+func (pxc *ProxyChain) SetClient(httpClient *http.Client) *ProxyChain {
+	pxc.Client = httpClient
+	return pxc
+}
+
+// SetVerbose changes the logging behavior to print
+// the modification steps and applied rulesets for debugging
+func (pxc *ProxyChain) SetVerbose() *ProxyChain {
+	pxc.verbose = true
+	return pxc
+}
+
+// abort proxychain and return 500 error to client
+// this will prevent Execute from firing and reset the state
+// returns the initial error enriched with context
+func (pxc *ProxyChain) abort(err error) error {
+	defer pxc._reset()
+	pxc._abort_err = err
+	pxc.Context.Response().SetStatusCode(500)
+	e := fmt.Errorf("ProxyChain error for '%s': %s", pxc.Request.URL.String(), err.Error())
+	pxc.Context.SendString(e.Error())
+	log.Println(e.Error())
+	return e
+}
+
+// internal function to reset state of ProxyChain for reuse
+func (pxc *ProxyChain) _reset() {
+	pxc._abort_err = nil
+	pxc.Body = nil
+	pxc.Request = nil
+	pxc.Response = nil
+	pxc.Context = nil
+	pxc.Request.URL = nil
+}
+
+// NewProxyChain initializes a new ProxyChain
+func NewProxyChain() *ProxyChain {
+	px := new(ProxyChain)
+	px.Client = defaultClient
+	return px
+}

proxychain/proxychain_pool.go

@@ -0,0 +1,11 @@
+package proxychain
+
+import (
+	"net/url"
+)
+
+type ProxyChainPool map[url.URL]ProxyChain
+
+func NewProxyChainPool() ProxyChainPool {
+	return map[url.URL]ProxyChain{}
+}

proxychain/rqm/block_outgoing_cookies.go

@@ -0,0 +1,41 @@
+package rqm // ReQuestModifier
+
+import (
+	"ladder/proxychain"
+)
+
+// BlockOutgoingCookies prevents ALL cookies from being sent from the client
+// to the upstream proxy server.
+func BlockOutgoingCookies() proxychain.RequestModification {
+	return func(px *proxychain.ProxyChain) error {
+		px.Request.Header.Del("Cookie")
+		return nil
+	}
+}
+
+// BlockOutgoingCookiesExcept prevents non-whitelisted cookies from being sent from the client
+// to the upstream proxy server. Cookies whose names are in the whitelist are not removed.
+func BlockOutgoingCookiesExcept(whitelist ...string) proxychain.RequestModification {
+	return func(px *proxychain.ProxyChain) error {
+		// Convert whitelist slice to a map for efficient lookups
+		whitelistMap := make(map[string]struct{})
+		for _, cookieName := range whitelist {
+			whitelistMap[cookieName] = struct{}{}
+		}
+
+		// Get all cookies from the request header
+		cookies := px.Request.Cookies()
+
+		// Clear the original Cookie header
+		px.Request.Header.Del("Cookie")
+
+		// Re-add cookies that are in the whitelist
+		for _, cookie := range cookies {
+			if _, found := whitelistMap[cookie.Name]; found {
+				px.Request.AddCookie(cookie)
+			}
+		}
+
+		return nil
+	}
+}

proxychain/rqm/masquerade_as_trusted_bot.go

@@ -0,0 +1,32 @@
+package rqm // ReQuestModifier
+
+import (
+	"ladder/proxychain"
+)
+
+// MasqueradeAsGoogleBot modifies user agent and x-forwarded for
+// to appear to be a Google Bot
+func MasqueradeAsGoogleBot() proxychain.RequestModification {
+	const botUA string = "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; http://www.google.com/bot.html) Chrome/79.0.3945.120 Safari/537.36"
+	const botIP string = "66.249.78.8" // TODO: create a random ip pool from https://developers.google.com/static/search/apis/ipranges/googlebot.json
+	return masqueradeAsTrustedBot(botUA, botIP)
+}
+
+// MasqueradeAsBingBot modifies user agent and x-forwarded for
+// to appear to be a Bing Bot
+func MasqueradeAsBingBot() proxychain.RequestModification {
+	const botUA string = "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/79.0.3945.120 Safari/537.36"
+	const botIP string = "13.66.144.9" // https://www.bing.com/toolbox/bingbot.json
+	return masqueradeAsTrustedBot(botUA, botIP)
+}
+
+func masqueradeAsTrustedBot(botUA string, botIP string) proxychain.RequestModification {
+	return func(px *proxychain.ProxyChain) error {
+		px.AddRequestModifications(
+			SpoofUserAgent(botUA),
+			SpoofXForwardedFor(botIP),
+			SpoofReferrer(""),
+		)
+		return nil
+	}
+}

proxychain/rqm/modify_domain_with_regex.go

@@ -0,0 +1,13 @@
+package rqm // ReQuestModifier
+
+import (
+	"ladder/proxychain"
+	"regexp"
+)
+
+func ModifyDomainWithRegex(match regexp.Regexp, replacement string) proxychain.RequestModification {
+	return func(px *proxychain.ProxyChain) error {
+		px.Request.URL.Host = match.ReplaceAllString(px.Request.URL.Host, replacement)
+		return nil
+	}
+}

proxychain/rqm/modify_path_with_regex.go

@@ -0,0 +1,13 @@
+package rqm // ReQuestModifier
+
+import (
+	"ladder/proxychain"
+	"regexp"
+)
+
+func ModifyPathWithRegex(match regexp.Regexp, replacement string) proxychain.RequestModification {
+	return func(px *proxychain.ProxyChain) error {
+		px.Request.URL.Path = match.ReplaceAllString(px.Request.URL.Path, replacement)
+		return nil
+	}
+}

proxychain/rqm/modify_query_params.go

@@ -0,0 +1,20 @@
+package rqm // ReQuestModifier
+
+import (
+	"ladder/proxychain"
+)
+
+// ModifyQueryParams replaces query parameter values in URL's query params in a ProxyChain's URL.
+// If the query param key doesn't exist, it is created.
+func ModifyQueryParams(key string, value string) proxychain.RequestModification {
+	return func(px *proxychain.ProxyChain) error {
+		q := px.Request.URL.Query()
+		if value == "" {
+			q.Del(key)
+			return nil
+		}
+		q.Set(key, value)
+		px.Request.URL.RawQuery = q.Encode()
+		return nil
+	}
+}

proxychain/rqm/request_archive_is.go

@@ -0,0 +1,27 @@
+package rqm
+
+import (
+	"ladder/proxychain"
+	"net/url"
+)
+
+const archivistUrl string = "https://archive.is/latest/"
+
+// RequestArchiveIs modifies a ProxyChain's URL to request an archived version from archive.is
+func RequestArchiveIs() proxychain.RequestModification {
+	return func(px *proxychain.ProxyChain) error {
+		px.Request.URL.RawQuery = ""
+		newURLString := archivistUrl + px.Request.URL.String()
+		newURL, err := url.Parse(newURLString)
+		if err != nil {
+			return err
+		}
+
+		// archivist seems to sabotage requests from cloudflare's DNS
+		// bypass this just in case
+		px.AddRequestModifications(ResolveWithGoogleDoH())
+
+		px.Request.URL = newURL
+		return nil
+	}
+}

proxychain/rqm/request_google_cache.go

@@ -0,0 +1,21 @@
+package rqm // ReQuestModifier
+
+import (
+	"ladder/proxychain"
+	"net/url"
+)
+
+const googleCacheUrl string = "https://webcache.googleusercontent.com/search?q=cache:"
+
+// RequestGoogleCache modifies a ProxyChain's URL to request its Google Cache version.
+func RequestGoogleCache() proxychain.RequestModification {
+	return func(px *proxychain.ProxyChain) error {
+		encodedURL := url.QueryEscape(px.Request.URL.String())
+		newURL, err := url.Parse(googleCacheUrl + encodedURL)
+		if err != nil {
+			return err
+		}
+		px.Request.URL = newURL
+		return nil
+	}
+}

proxychain/rqm/request_wayback_machine.go

@@ -0,0 +1,22 @@
+package rqm // ReQuestModifier
+
+import (
+	"ladder/proxychain"
+	"net/url"
+)
+
+const waybackUrl string = "https://web.archive.org/web/"
+
+// RequestWaybackMachine modifies a ProxyChain's URL to request the wayback machine (archive.org) version.
+func RequestWaybackMachine() proxychain.RequestModification {
+	return func(px *proxychain.ProxyChain) error {
+		px.Request.URL.RawQuery = ""
+		newURLString := waybackUrl + px.Request.URL.String()
+		newURL, err := url.Parse(newURLString)
+		if err != nil {
+			return err
+		}
+		px.Request.URL = newURL
+		return nil
+	}
+}

proxychain/rqm/resolve_with_google_doh.go

@@ -0,0 +1,80 @@
+package rqm
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"ladder/proxychain"
+	"net"
+	"net/http"
+	"time"
+)
+
+// resolveWithGoogleDoH resolves DNS using Google's DNS-over-HTTPS
+func resolveWithGoogleDoH(host string) (string, error) {
+	url := "https://dns.google/resolve?name=" + host + "&type=A"
+	resp, err := http.Get(url)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	var result struct {
+		Answer []struct {
+			Data string `json:"data"`
+		} `json:"Answer"`
+	}
+	err = json.NewDecoder(resp.Body).Decode(&result)
+	if err != nil {
+		return "", err
+	}
+
+	// Get the first A record
+	if len(result.Answer) > 0 {
+		return result.Answer[0].Data, nil
+	}
+	return "", fmt.Errorf("no DoH DNS record found for %s", host)
+}
+
+// ResolveWithGoogleDoH modifies a ProxyChain's client to make the request but resolve the URL
+// using Google's DNS over HTTPs service
+func ResolveWithGoogleDoH() proxychain.RequestModification {
+	return func(px *proxychain.ProxyChain) error {
+		client := &http.Client{
+			Timeout: px.Client.Timeout,
+		}
+
+		dialer := &net.Dialer{
+			Timeout:   5 * time.Second,
+			KeepAlive: 5 * time.Second,
+		}
+
+		customDialContext := func(ctx context.Context, network, addr string) (net.Conn, error) {
+			host, port, err := net.SplitHostPort(addr)
+			if err != nil {
+				// If the addr doesn't include a port, determine it based on the URL scheme
+				if px.Request.URL.Scheme == "https" {
+					port = "443"
+				} else {
+					port = "80"
+				}
+				host = addr // assume the entire addr is the host
+			}
+
+			resolvedHost, err := resolveWithGoogleDoH(host)
+			if err != nil {
+				return nil, err
+			}
+
+			return dialer.DialContext(ctx, network, net.JoinHostPort(resolvedHost, port))
+		}
+
+		patchedTransportWithDoH := &http.Transport{
+			DialContext: customDialContext,
+		}
+
+		client.Transport = patchedTransportWithDoH
+		px.Client = client // Assign the modified client to the ProxyChain
+		return nil
+	}
+}

proxychain/rqm/spoof_origin.go

@@ -0,0 +1,24 @@
+package rqm // ReQuestModifier
+
+import (
+	"ladder/proxychain"
+)
+
+// SpoofOrigin modifies the origin header
+// if the upstream server returns a Vary header
+// it means you might get a different response if you change this
+func SpoofOrigin(url string) proxychain.RequestModification {
+	return func(px *proxychain.ProxyChain) error {
+		px.Request.Header.Set("origin", url)
+		return nil
+	}
+}
+
+// HideOrigin modifies the origin header
+// so that it is the original origin, not the proxy
+func HideOrigin() proxychain.RequestModification {
+	return func(px *proxychain.ProxyChain) error {
+		px.Request.Header.Set("origin", px.Request.URL.String())
+		return nil
+	}
+}

proxychain/rqm/spoof_referrer.go

@@ -0,0 +1,29 @@
+package rqm // ReQuestModifier
+
+import (
+	"ladder/proxychain"
+)
+
+// SpoofReferrer modifies the referrer header
+// useful if the page can be accessed from a search engine
+// or social media site, but not by browsing the website itself
+// if url is "", then the referrer header is removed
+func SpoofReferrer(url string) proxychain.RequestModification {
+	return func(px *proxychain.ProxyChain) error {
+		if url == "" {
+			px.Request.Header.Del("referrer")
+			return nil
+		}
+		px.Request.Header.Set("referrer", url)
+		return nil
+	}
+}
+
+// HideReferrer modifies the referrer header
+// so that it is the original referrer, not the proxy
+func HideReferrer() proxychain.RequestModification {
+	return func(px *proxychain.ProxyChain) error {
+		px.Request.Header.Set("referrer", px.Request.URL.String())
+		return nil
+	}
+}

proxychain/rqm/spoof_user_agent.go

@@ -0,0 +1,13 @@
+package rqm // ReQuestModifier
+
+import (
+	"ladder/proxychain"
+)
+
+// SpoofUserAgent modifies the user agent
+func SpoofUserAgent(ua string) proxychain.RequestModification {
+	return func(px *proxychain.ProxyChain) error {
+		px.Request.Header.Set("user-agent", ua)
+		return nil
+	}
+}

proxychain/rqm/spoof_x_forwarded_for.go

@@ -0,0 +1,14 @@
+package rqm // ReQuestModifier
+
+import (
+	"ladder/proxychain"
+)
+
+// SpoofXForwardedFor modifies the X-Forwarded-For header
+// in some cases, a forward proxy may interpret this as the source IP
+func SpoofXForwardedFor(ip string) proxychain.RequestModification {
+	return func(px *proxychain.ProxyChain) error {
+		px.Request.Header.Set("X-FORWARDED-FOR", ip)
+		return nil
+	}
+}

proxychain/rsm/block_incoming_cookies.go

@@ -0,0 +1,58 @@
+package rsm // ReSponseModifers
+
+import (
+	"ladder/proxychain"
+	"net/http"
+)
+
+// BlockIncomingCookies prevents ALL cookies from being sent from the proxy server
+// to the client.
+func BlockIncomingCookies(whitelist ...string) proxychain.ResponseModification {
+	return func(px *proxychain.ProxyChain) error {
+		px.Response.Header.Del("Set-Cookie")
+		return nil
+	}
+}
+
+// BlockIncomingCookiesExcept prevents non-whitelisted cookies from being sent from the proxy server
+// to the client. Cookies whose names are in the whitelist are not removed.
+func BlockIncomingCookiesExcept(whitelist ...string) proxychain.ResponseModification {
+	return func(px *proxychain.ProxyChain) error {
+		// Convert whitelist slice to a map for efficient lookups
+		whitelistMap := make(map[string]struct{})
+		for _, cookieName := range whitelist {
+			whitelistMap[cookieName] = struct{}{}
+		}
+
+		// If the response has no cookies, return early
+		if px.Response.Header == nil {
+			return nil
+		}
+
+		// Filter the cookies in the response
+		filteredCookies := []string{}
+		for _, cookieStr := range px.Response.Header["Set-Cookie"] {
+			cookie := parseCookie(cookieStr)
+			if _, found := whitelistMap[cookie.Name]; found {
+				filteredCookies = append(filteredCookies, cookieStr)
+			}
+		}
+
+		// Update the Set-Cookie header with the filtered cookies
+		if len(filteredCookies) > 0 {
+			px.Response.Header["Set-Cookie"] = filteredCookies
+		} else {
+			px.Response.Header.Del("Set-Cookie")
+		}
+
+		return nil
+	}
+}
+
+// parseCookie parses a cookie string and returns an http.Cookie object.
+func parseCookie(cookieStr string) *http.Cookie {
+	header := http.Header{}
+	header.Add("Set-Cookie", cookieStr)
+	request := http.Request{Header: header}
+	return request.Cookies()[0]
+}

proxychain/rsm/bypass_cors.go

@@ -0,0 +1,20 @@
+package rsm // ReSponseModifers
+
+import (
+	"ladder/proxychain"
+)
+
+// BypassCORs modifies response headers to prevent the browser
+// from enforcing any CORS restrictions
+func BypassCORS() proxychain.ResponseModification {
+	return func(px *proxychain.ProxyChain) error {
+		px.AddResultModifications(
+			ModifyResponseHeader("Access-Control-Allow-Origin", "*"),
+			ModifyResponseHeader("Access-Control-Expose-Headers", "*"),
+			ModifyResponseHeader("Access-Control-Allow-Credentials", "true"),
+			ModifyResponseHeader("Access-Control-Allow-Methods", "GET, PUT, POST, DELETE, HEAD, OPTIONS, PATCH"),
+			DeleteResponseHeader("X-Frame-Options"),
+		)
+		return nil
+	}
+}

proxychain/rsm/bypass_csp.go

@@ -0,0 +1,19 @@
+package rsm // ReSponseModifers
+
+import (
+	"ladder/proxychain"
+)
+
+// BypassCSP modifies response headers to prevent the browser
+// from enforcing any CORS restrictions
+func BypassCSP() proxychain.ResponseModification {
+	return func(px *proxychain.ProxyChain) error {
+		px.AddResultModifications(
+			ModifyResponseHeader("Access-Control-Allow-Origin", "*"),
+			ModifyResponseHeader("Access-Control-Expose-Headers", "*"),
+			ModifyResponseHeader("Access-Control-Allow-Credentials", "true"),
+			ModifyResponseHeader("Access-Control-Allow-Methods", ""),
+		)
+		return nil
+	}
+}

proxychain/rsm/modify_response_header.go

@@ -0,0 +1,26 @@
+package rsm // ReSponseModifers
+
+import (
+	"ladder/proxychain"
+)
+
+// ModifyResponseHeader modifies response headers from the upstream server
+// if value is "", then the response header is deleted.
+func ModifyResponseHeader(key string, value string) proxychain.ResponseModification {
+	return func(px *proxychain.ProxyChain) error {
+		if value == "" {
+			px.Context.Response().Header.Del(key)
+			return nil
+		}
+		px.Context.Response().Header.Set(key, value)
+		return nil
+	}
+}
+
+// DeleteResponseHeader removes response headers from the upstream server
+func DeleteResponseHeader(key string) proxychain.ResponseModification {
+	return func(px *proxychain.ProxyChain) error {
+		px.Context.Response().Header.Del(key)
+		return nil
+	}
+}

proxychain/strategy/cloudflare.go

@@ -0,0 +1,8 @@
+package strategy
+
+/*
+var Cloudflare = proxy.Strategy{
+    tactic.NoCookie(),
+    // ... other tactics ...
+}
+*/