Simpler error return for non text/html

feat: Playground

.air.toml

@@ -8,11 +8,11 @@ tmp_dir = "tmp"
   cmd = "go build -o ./tmp/main ./cmd"
   delay = 1000
   exclude_dir = ["assets", "tmp", "vendor", "testdata",]
-  exclude_file = ["proxychain/ruleset/rule_resmod_types.gen.go", "proxychain/ruleset/rule_reqmod_types.gen.go"]
+  exclude_file = ["proxychain/ruleset/rule_resmod_types.gen.go", "proxychain/ruleset/rule_reqmod_types.gen.go", "handlers/api_modifiers_structdef.gen.go"]
   exclude_regex = ["_test.go"]
   exclude_unchanged = false
   follow_symlink = false
-  full_bin = "./tmp/main --ruleset ./ruleset.yaml"
+  full_bin = "./tmp/main --ruleset ./rulesets_v2"
   include_dir = []
   include_ext = ["go", "tpl", "tmpl", "yaml", "html", "js"]
   include_file = []
@@ -21,7 +21,7 @@ tmp_dir = "tmp"
   poll = false
   poll_interval = 0
   post_cmd = []
-  pre_cmd = ["git submodule update --init --recursive; git rev-parse --short HEAD > handlers/VERSION; git rev-parse --short HEAD > cmd/VERSION; cd proxychain/codegen && go run codegen.go"]
+  pre_cmd = ["git submodule update --init --recursive; git rev-parse --short HEAD > handlers/VERSION; git rev-parse --short HEAD > cmd/VERSION; cd proxychain/codegen && go run codegen.go && cd ../../handlers/api_modifiers_codegen && go run api_modifiers_codegen.go"]
   rerun = false
   rerun_delay = 500
   send_interrupt = false

.github/workflows/build-css.yaml

@@ -3,8 +3,10 @@ name: Build Tailwind CSS
 on:
   push:
     paths:
+      - "handlers/error_page.html"
       - "handlers/form.html"
-      - "proxychain/responsemodifiers/generate_readable_outline.html"
+      - "handlers/playground.html"
+      - "proxychain/responsemodifiers/vendor/generate_readable_outline.html"
   workflow_dispatch:
 
 jobs:
@@ -28,16 +30,17 @@ jobs:
         name: Build Tailwind CSS
         run: pnpm build
       -
-        name: Commit generated stylesheet
+        name: Commit generated stylesheet for handlers/styles.css
         run: |
-          if git diff --quiet cmd/styles.css; then
+          if git diff --quiet handlers/styles.css; then
             echo "No changes to commit."
             exit 0
           else
             echo "Changes detected, committing..."
             git config --global user.name "Github action"
             git config --global user.email "username@users.noreply.github.com"
-            git add cmd
+            git add handlers
+            git add proxychain/responsemodifiers/vendor/
             git commit -m "Generated stylesheet"
             git push
           fi

.gitignore

@@ -1,5 +1,7 @@
 # dev binary
 ladder
+tmp/main
+tmp
 
 VERSION
 output.css

Dockerfile

@@ -7,7 +7,7 @@ COPY . .
 
 RUN go mod download
 
-RUN CGO_ENABLED=0 GOOS=linux go build -o ladder cmd/main.go
+RUN make build
 
 FROM debian:12-slim as release
 
@@ -18,8 +18,4 @@ RUN chmod +x /app/ladder
 
 RUN apt update && apt install -y ca-certificates && rm -rf /var/lib/apt/lists/*
 
-#EXPOSE 8080
-
-#ENTRYPOINT ["/usr/bin/dumb-init", "--"]
-
 CMD ["sh", "-c", "/app/ladder"]

Makefile

@@ -1,5 +1,6 @@
 build:
 	cd proxychain/codegen && go run codegen.go
+	cd handlers/api_modifiers_codegen && go run api_modifiers_codegen.go
 	git submodule update --init --recursive
 	git rev-parse --short HEAD > handlers/VERSION
 	git rev-parse --short HEAD > cmd/VERSION

README.md

@@ -91,18 +91,28 @@ Or create a bookmark with the following URL:
 ```javascript
 javascript:window.location.href="http://localhost:8080/"+location.href
 ```
+### Outline
+```bash
+curl -X GET "http://localhost:8080/outline/https://www.example.com"
+```
 
 ### API
 ```bash
-curl -X GET "http://localhost:8080/api/https://www.example.com"
+curl -X GET "http://localhost:8080/api/content/https://www.example.com"
 ```
 
 ### RAW
-http://localhost:8080/raw/https://www.example.com
+http://localhost:8080/api/raw/https://www.example.com
 
 
 ### Running Ruleset
-http://localhost:8080/ruleset
+http://localhost:8080/api/ruleset
+
+### Running Rule
+http://localhost:8080/api/ruleset/https://example.com
+
+### List available modifiers
+http://localhost:8080/api/modifiers
 
 ## Configuration
 

cmd/main.go

@@ -3,17 +3,17 @@ package main
 import (
 	_ "embed"
 	"fmt"
-	"html/template"
 	"log"
 	"os"
 
-	"ladder/handlers"
+	"github.com/everywall/ladder/handlers"
-	"ladder/internal/cli"
+	"github.com/everywall/ladder/internal/cli"
-	"ladder/proxychain/requestmodifiers/bot"
+
+	"github.com/everywall/ladder/proxychain/requestmodifiers/bot"
+	ruleset_v2 "github.com/everywall/ladder/proxychain/ruleset"
 
 	"github.com/akamensky/argparse"
 	"github.com/gofiber/fiber/v2"
-	"github.com/gofiber/template/html/v2"
 )
 
 //go:embed VERSION
@@ -65,14 +65,9 @@ func main() {
 		Help:     "Compiles a directory of yaml files into a single ruleset.yaml. Requires --ruleset arg.",
 	})
 
-	mergeRulesetsGzip := parser.Flag("", "merge-rulesets-gzip", &argparse.Options{
-		Required: false,
-		Help:     "Compiles a directory of yaml files into a single ruleset.gz Requires --ruleset arg.",
-	})
-
 	mergeRulesetsOutput := parser.String("", "merge-rulesets-output", &argparse.Options{
 		Required: false,
-		Help:     "Specify output file for --merge-rulesets and --merge-rulesets-gzip. Requires --ruleset and --merge-rulesets args.",
+		Help:     "Specify output file for --merge-rulesets. Requires --ruleset and --merge-rulesets args.",
 	})
 
 	err := parser.Parse(os.Args)
@@ -97,7 +92,7 @@ func main() {
 	}
 
 	// utility cli flag to compile ruleset directory into single ruleset.yaml
-	if *mergeRulesets || *mergeRulesetsGzip {
+	if *mergeRulesets {
 		output := os.Stdout
 
 		if *mergeRulesetsOutput != "" {
@@ -109,7 +104,7 @@ func main() {
 			}
 		}
 
-		err = cli.HandleRulesetMerge(*ruleset, *mergeRulesets, *mergeRulesetsGzip, output)
+		err = cli.HandleRulesetMerge(*ruleset, *mergeRulesets, output)
 		if err != nil {
 			fmt.Println(err)
 			os.Exit(1)
@@ -121,13 +116,17 @@ func main() {
 		*prefork = true
 	}
 
-	engine := html.New("./handlers", ".html")
+	var rs ruleset_v2.IRuleset
-	engine.AddFunc(
+
-		// add unescape function
+	switch {
-		"unescape", func(s string) template.HTML {
+	case *ruleset != "":
-			return template.HTML(s)
+		rs, err = ruleset_v2.NewRuleset(*ruleset)
-		},
+		if err != nil {
-	)
+			fmt.Printf("ERROR: failed to load ruleset from %s\n", *ruleset)
+		}
+	case os.Getenv("RULESET") != "":
+		rs = ruleset_v2.NewRulesetFromEnv()
+	}
 
 	app := fiber.New(
 		fiber.Config{
@@ -135,12 +134,12 @@ func main() {
 			GETOnly:               false,
 			ReadBufferSize:        4096 * 4, // increase max header size
 			DisableStartupMessage: true,
-			Views:                 engine,
 		},
 	)
 
 	app.Use(handlers.Auth())
 	app.Use(handlers.Favicon())
+	app.Use(handlers.RenderErrorPage())
 
 	if os.Getenv("NOLOGS") != "true" {
 		app.Use(func(c *fiber.Ctx) error {
@@ -150,21 +149,25 @@ func main() {
 		})
 	}
 
+	proxyOpts := &handlers.ProxyOptions{
+		Verbose: *verbose,
+		Ruleset: rs,
+	}
+
 	app.Get("/", handlers.Form)
 
 	app.Get("styles.css", handlers.Styles)
 	app.Get("script.js", handlers.Script)
-	app.Get("ruleset", handlers.Ruleset)
+	app.Get("playground-script.js", handlers.Script)
-	app.Get("raw/*", handlers.Raw)
 
-	proxyOpts := &handlers.ProxyOptions{
+	app.All("api/raw/*", handlers.NewRawProxySiteHandler(proxyOpts))
-		Verbose:     *verbose,
-		RulesetPath: *ruleset,
-	}
 
+	app.Get("api/modifiers", handlers.NewAPIModifersListHandler(proxyOpts))
+	app.Get("api/ruleset/*", handlers.NewRulesetSiteHandler(proxyOpts))
 	app.Get("api/content/*", handlers.NewAPIContentHandler("api/outline/*", proxyOpts))
 
 	app.Get("outline/*", handlers.NewOutlineHandler("outline/*", proxyOpts))
+	app.All("playground/*", handlers.PlaygroundHandler("playground/*", proxyOpts))
 
 	app.All("/*", handlers.NewProxySiteHandler(proxyOpts))
 

docker-compose.yaml

@@ -3,7 +3,7 @@ services:
   ladder:
     image: ghcr.io/everywall/ladder:latest
     container_name: ladder
-    #build: .
+    build: .
     #restart: always
     #command: sh -c ./ladder
     environment:

go.mod

@@ -1,4 +1,4 @@
-module ladder
+module github.com/everywall/ladder
 
 go 1.21.1
 
@@ -7,56 +7,53 @@ require (
 	github.com/bogdanfinn/fhttp v0.5.24
 	github.com/bogdanfinn/tls-client v1.6.1
 	github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c
-	github.com/gofiber/fiber/v2 v2.50.0
+	github.com/gofiber/fiber/v2 v2.51.0
 	github.com/markusmobius/go-trafilatura v1.5.1
 	github.com/stretchr/testify v1.8.4
+	golang.org/x/net v0.19.0
+	golang.org/x/term v0.15.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
 	github.com/abadojack/whatlanggo v1.0.1 // indirect
+	github.com/andybalholm/brotli v1.0.6 // indirect
 	github.com/andybalholm/cascadia v1.3.2 // indirect
 	github.com/bogdanfinn/utls v1.5.16 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/elliotchance/pie/v2 v2.8.0 // indirect
 	github.com/forPelevin/gomoji v1.1.8 // indirect
 	github.com/go-shiori/go-readability v0.0.0-20231029095239-6b97d5aba789 // indirect
 	github.com/gofiber/template v1.8.2 // indirect
+	github.com/gofiber/template/html/v2 v2.0.5
 	github.com/gofiber/utils v1.1.0 // indirect
 	github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect
+	github.com/google/uuid v1.4.0 // indirect
 	github.com/hablullah/go-hijri v1.0.2 // indirect
 	github.com/hablullah/go-juliandays v1.0.0 // indirect
 	github.com/jalaali/go-jalaali v0.0.0-20210801064154-80525e88d958 // indirect
+	github.com/klauspost/compress v1.17.4 // indirect
 	github.com/magefile/mage v1.15.0 // indirect
 	github.com/markusmobius/go-dateparser v1.2.1 // indirect
 	github.com/markusmobius/go-domdistiller v0.0.0-20230515154422-71af71939ff3 // indirect
 	github.com/markusmobius/go-htmldate v1.2.2 // indirect
-	github.com/rs/zerolog v1.31.0 // indirect
-	github.com/sirupsen/logrus v1.9.3 // indirect
-	github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5 // indirect
-	github.com/tetratelabs/wazero v1.5.0 // indirect
-	github.com/wasilibs/go-re2 v1.4.1 // indirect
-	github.com/yosssi/gohtml v0.0.0-20201013000340-ee4748c638f4 // indirect
-	golang.org/x/crypto v0.16.0 // indirect
-	golang.org/x/exp v0.0.0-20231127185646-65229373498e // indirect
-	golang.org/x/text v0.14.0 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
-)
-
-require (
-	github.com/andybalholm/brotli v1.0.6 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/gofiber/template/html/v2 v2.0.5
-	github.com/google/uuid v1.4.0 // indirect
-	github.com/klauspost/compress v1.17.3 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rivo/uniseg v0.4.4 // indirect
+	github.com/rs/zerolog v1.31.0 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
+	github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5 // indirect
+	github.com/tetratelabs/wazero v1.5.0 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
-	github.com/valyala/fasthttp v1.50.0 // indirect
+	github.com/valyala/fasthttp v1.51.0 // indirect
 	github.com/valyala/tcplisten v1.0.0 // indirect
-	golang.org/x/net v0.19.0
+	github.com/wasilibs/go-re2 v1.4.1 // indirect
+	github.com/yosssi/gohtml v0.0.0-20201013000340-ee4748c638f4 // indirect
+	golang.org/x/crypto v0.16.0 // indirect
+	golang.org/x/exp v0.0.0-20231127185646-65229373498e // indirect
 	golang.org/x/sys v0.15.0 // indirect
-	golang.org/x/term v0.15.0
+	golang.org/x/text v0.14.0 // indirect
+
 )

go.sum

@@ -25,8 +25,8 @@ github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c/go.mod h1:oVDCh3qjJM
 github.com/go-shiori/go-readability v0.0.0-20231029095239-6b97d5aba789 h1:G6wSuUyCoLB9jrUokipsmFuRi8aJozt3phw/g9Sl4Xs=
 github.com/go-shiori/go-readability v0.0.0-20231029095239-6b97d5aba789/go.mod h1:2DpZlTJO/ycxp/vsc/C11oUyveStOgIXB88SYV1lncI=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/gofiber/fiber/v2 v2.50.0 h1:ia0JaB+uw3GpNSCR5nvC5dsaxXjRU5OEu36aytx+zGw=
+github.com/gofiber/fiber/v2 v2.51.0 h1:JNACcZy5e2tGApWB2QrRpenTWn0fq0hkFm6k0C86gKQ=
-github.com/gofiber/fiber/v2 v2.50.0/go.mod h1:21eytvay9Is7S6z+OgPi7c7n4++tnClWmhpimVHMimw=
+github.com/gofiber/fiber/v2 v2.51.0/go.mod h1:xaQRZQJGqnKOQnbQw+ltvku3/h8QxvNi8o6JiJ7Ll0U=
 github.com/gofiber/template v1.8.2 h1:PIv9s/7Uq6m+Fm2MDNd20pAFFKt5wWs7ZBd8iV9pWwk=
 github.com/gofiber/template v1.8.2/go.mod h1:bs/2n0pSNPOkRa5VJ8zTIvedcI/lEYxzV3+YPXdBvq8=
 github.com/gofiber/template/html/v2 v2.0.5 h1:BKLJ6Qr940NjntbGmpO3zVa4nFNGDCi/IfUiDB9OC20=
@@ -43,8 +43,8 @@ github.com/hablullah/go-juliandays v1.0.0 h1:A8YM7wIj16SzlKT0SRJc9CD29iiaUzpBLzh
 github.com/hablullah/go-juliandays v1.0.0/go.mod h1:0JOYq4oFOuDja+oospuc61YoX+uNEn7Z6uHYTbBzdGc=
 github.com/jalaali/go-jalaali v0.0.0-20210801064154-80525e88d958 h1:qxLoi6CAcXVzjfvu+KXIXJOAsQB62LXjsfbOaErsVzE=
 github.com/jalaali/go-jalaali v0.0.0-20210801064154-80525e88d958/go.mod h1:Wqfu7mjUHj9WDzSSPI5KfBclTTEnLveRUFr/ujWnTgE=
-github.com/klauspost/compress v1.17.3 h1:qkRjuerhUU1EmXLYGkSH6EZL+vPSxIrYjLNAK4slzwA=
+github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4=
-github.com/klauspost/compress v1.17.3/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
+github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg=
@@ -90,8 +90,8 @@ github.com/tetratelabs/wazero v1.5.0 h1:Yz3fZHivfDiZFUXnWMPUoiW7s8tC1sjdBtlJn08q
 github.com/tetratelabs/wazero v1.5.0/go.mod h1:0U0G41+ochRKoPKCJlh0jMg1CHkyfK8kDqiirMmKY8A=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasthttp v1.50.0 h1:H7fweIlBm0rXLs2q0XbalvJ6r0CUPFWK3/bB4N13e9M=
+github.com/valyala/fasthttp v1.51.0 h1:8b30A5JlZ6C7AS81RsWjYMQmrZG6feChmgAolCl1SqA=
-github.com/valyala/fasthttp v1.50.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA=
+github.com/valyala/fasthttp v1.51.0/go.mod h1:oI2XroL+lI7vdXyYoQk03bXBThfFl2cVdIA3Xl7cH8g=
 github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
 github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
 github.com/wasilibs/go-re2 v1.4.1 h1:E5+9O1M8UoGeqLB2A9omeoaWImqpuYDs9cKwvTJq/Oo=
@@ -153,8 +153,6 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U=
 gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
-gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

handlers/api.test.go

@@ -1,44 +0,0 @@
-// BEGIN: 7d5e1f7c7d5e
-package handlers
-
-import (
-	"net/http"
-	"net/http/httptest"
-	"testing"
-
-	"github.com/gofiber/fiber/v2"
-	"github.com/stretchr/testify/assert"
-)
-
-func TestApi(t *testing.T) {
-	app := fiber.New()
-	app.Get("/api/*", Api)
-
-	tests := []struct {
-		name           string
-		url            string
-		expectedStatus int
-	}{
-		{
-			name:           "valid url",
-			url:            "https://www.google.com",
-			expectedStatus: http.StatusOK,
-		},
-		{
-			name:           "invalid url",
-			url:            "invalid-url",
-			expectedStatus: http.StatusBadRequest,
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			req := httptest.NewRequest(http.MethodGet, "/api/"+tt.url, nil)
-			resp, err := app.Test(req)
-			assert.NoError(t, err)
-			assert.Equal(t, tt.expectedStatus, resp.StatusCode)
-		})
-	}
-}
-
-// END: 7d5e1f7c7d5e

handlers/api_content.go

@@ -1,9 +1,10 @@
 package handlers
 
 import (
-	"ladder/proxychain"
+	rx "github.com/everywall/ladder/proxychain/requestmodifiers"
-	rx "ladder/proxychain/requestmodifiers"
+	tx "github.com/everywall/ladder/proxychain/responsemodifiers"
-	tx "ladder/proxychain/responsemodifiers"
+
+	"github.com/everywall/ladder/proxychain"
 
 	"github.com/gofiber/fiber/v2"
 )

handlers/api_modifiers.go

@@ -0,0 +1,29 @@
+package handlers
+
+import (
+	"encoding/json"
+
+	"github.com/everywall/ladder/proxychain/responsemodifiers/api"
+	"github.com/gofiber/fiber/v2"
+)
+
+func NewAPIModifersListHandler(opts *ProxyOptions) fiber.Handler {
+	payload := ModifiersAPIResponse{
+		Success: true,
+		Result:  AllMods,
+	}
+	body, err := json.MarshalIndent(payload, "", "  ")
+	if err != nil {
+		panic(err)
+	}
+
+	return func(c *fiber.Ctx) error {
+		c.Set("content-type", "application/json")
+		if err != nil {
+			c.SendStatus(500)
+			return c.SendStream(api.CreateAPIErrReader(err))
+		}
+
+		return c.Send(body)
+	}
+}

handlers/api_modifiers_codegen/api_modifiers_codegen.go

@@ -0,0 +1,196 @@
+package main
+
+import (
+	"fmt"
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"io"
+	"io/fs"
+	"os/exec"
+
+	//"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+	//"strings"
+)
+
+func genModStruct(fn *ast.FuncDecl, githubEditLink string, filename string) string {
+	params := []string{}
+	for _, fd := range fn.Type.Params.List {
+		p := fmt.Sprintf(`    {Name: "%s", Type: "%+v"},`, fd.Names[0], fd.Type)
+		params = append(params, p)
+	}
+
+	block := fmt.Sprintf(`{
+  Name:        "%s",
+  Description: "%s",
+  CodeEditLink: "%s%s",
+  Params: []Param{
+%s
+  },
+},`,
+		fn.Name.String(),
+		strings.ReplaceAll(strings.ReplaceAll(strings.TrimSpace(fn.Doc.Text()), "\n", " "), `"`, `\"`),
+		githubEditLink, filename,
+		strings.Join(params, "\n"),
+	)
+
+	return block
+}
+
+func modCodeGen(dir string, githubEditLink string) (code string, err error) {
+	fset := token.NewFileSet()
+
+	files, err := os.ReadDir(dir)
+	if err != nil {
+		panic(err)
+	}
+
+	modStructs := []string{}
+	for _, file := range files {
+		if !shouldGenCodeFor(file) {
+			continue
+		}
+
+		// Parse each Go file
+		node, err := parser.ParseFile(fset, filepath.Join(dir, file.Name()), nil, parser.ParseComments)
+		if err != nil {
+			return "", err
+		}
+
+		ast.Inspect(node, func(n ast.Node) bool {
+			fn, ok := n.(*ast.FuncDecl)
+			if ok && fn.Recv == nil && fn.Name.IsExported() {
+				modStructs = append(modStructs, genModStruct(fn, githubEditLink, file.Name()))
+			}
+			return true
+		})
+
+	}
+
+	code = strings.Join(modStructs, "\n")
+	return code, nil
+}
+
+func shouldGenCodeFor(file fs.DirEntry) bool {
+	if file.IsDir() {
+		return false
+	}
+	if filepath.Ext(file.Name()) != ".go" {
+		return false
+	}
+	if strings.HasSuffix(file.Name(), "_test.go") {
+		return false
+	}
+	return true
+}
+
+func getGitRemoteURL(remoteName string) (string, error) {
+	cmd := exec.Command("git", "remote", "get-url", remoteName)
+	output, err := cmd.Output()
+	if err != nil {
+		return "", err
+	}
+	url := strings.TrimSpace(string(output))
+
+	// Convert SSH format to HTTPS format
+	if strings.HasPrefix(url, "git@") {
+		url = strings.Replace(url, ":", "/", 1)
+		url = strings.Replace(url, "git@", "https://", 1)
+		url = strings.TrimSuffix(url, ".git")
+	}
+
+	return url, nil
+}
+
+func getCurrentGitBranch() (string, error) {
+	cmd := exec.Command("git", "rev-parse", "--abbrev-ref", "HEAD")
+	output, err := cmd.Output()
+	if err != nil {
+		return "", err
+	}
+	return strings.TrimSpace(string(output)), nil
+}
+
+func main() {
+	gitURL, err := getGitRemoteURL("origin")
+	if err != nil {
+		fmt.Println("Error getting Git remote URL:", err)
+		return
+	}
+
+	branchName, err := getCurrentGitBranch()
+	if err != nil {
+		fmt.Println("Error getting current Git branch:", err)
+		return
+	}
+
+	githubEditLink := fmt.Sprintf("%s/edit/%s/proxychain/requestmodifiers/", gitURL, branchName)
+	rqmCode, err := modCodeGen("../../proxychain/requestmodifiers/", githubEditLink)
+	if err != nil {
+		panic(err)
+	}
+
+	githubEditLink = fmt.Sprintf("%s/edit/%s/proxychain/responsemodifiers/", gitURL, branchName)
+	rsmCode, err := modCodeGen("../../proxychain/responsemodifiers/", githubEditLink)
+	if err != nil {
+		panic(err)
+	}
+
+	code := fmt.Sprintf(`
+package handlers
+// DO NOT EDIT THIS FILE. It is automatically generated by ladder/handlers/api_modifiers_codegen/api_modifiers_codegen.go
+// The purpose of this is to produce an API reponse listing all the available modifier, their parameters and usage instructions.
+// for use in proxychains.
+
+import (
+	"github.com/everywall/ladder/proxychain/responsemodifiers/api"
+)
+
+type ModifiersAPIResponse struct {
+	Success bool      ||json:"success"||
+	Error   api.ErrorDetails ||json:"error"||
+	Result  Modifiers ||json:"result"||
+}
+
+type Modifiers struct {
+	RequestModifiers  []Modifier ||json:"requestmodifiers"||
+	ResponseModifiers []Modifier ||json:"responsemodifiers"||
+}
+
+type Modifier struct {
+	Name        string  ||json:"name"||
+	Description string  ||json:"description"||
+	CodeEditLink string ||json:"code_edit_link"||
+	Params      []Param ||json:"params"||
+}
+
+type Param struct {
+	Name string ||json:"name"||
+	Type string ||json:"type"||
+}
+
+var AllMods Modifiers = Modifiers{
+	RequestModifiers: []Modifier{
+%s
+	},
+	ResponseModifiers: []Modifier{
+%s
+	},
+}
+`, rqmCode, rsmCode)
+	code = strings.ReplaceAll(code, "||", "`")
+
+	//fmt.Println(code)
+
+	fq, err := os.Create("../api_modifiers_structdef.gen.go")
+	if err != nil {
+		panic(err)
+	}
+	_, err = io.WriteString(fq, code)
+	if err != nil {
+		panic(err)
+	}
+}

handlers/api_modifiers_structdef.gen.go

@@ -0,0 +1,567 @@
+
+package handlers
+// DO NOT EDIT THIS FILE. It is automatically generated by ladder/handlers/api_modifiers_codegen/api_modifiers_codegen.go
+// The purpose of this is to produce an API reponse listing all the available modifier, their parameters and usage instructions.
+// for use in proxychains.
+
+import (
+	"github.com/everywall/ladder/proxychain/responsemodifiers/api"
+)
+
+type ModifiersAPIResponse struct {
+	Success bool      `json:"success"`
+	Error   api.ErrorDetails `json:"error"`
+	Result  Modifiers `json:"result"`
+}
+
+type Modifiers struct {
+	RequestModifiers  []Modifier `json:"requestmodifiers"`
+	ResponseModifiers []Modifier `json:"responsemodifiers"`
+}
+
+type Modifier struct {
+	Name        string  `json:"name"`
+	Description string  `json:"description"`
+	CodeEditLink string `json:"code_edit_link"`
+	Params      []Param `json:"params"`
+}
+
+type Param struct {
+	Name string `json:"name"`
+	Type string `json:"type"`
+}
+
+var AllMods Modifiers = Modifiers{
+	RequestModifiers: []Modifier{
+{
+  Name:        "AddCacheBusterQuery",
+  Description: "AddCacheBusterQuery modifies query params to add a random parameter key In order to get the upstream network stack to serve a fresh copy of the page.",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/requestmodifiers/add_cache_buster_query.go",
+  Params: []Param{
+
+  },
+},
+{
+  Name:        "ForwardRequestHeaders",
+  Description: "ForwardRequestHeaders forwards the requests headers sent from the client to the upstream server",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/requestmodifiers/forward_request_headers.go",
+  Params: []Param{
+
+  },
+},
+{
+  Name:        "MasqueradeAsGoogleBot",
+  Description: "MasqueradeAsGoogleBot modifies user agent and x-forwarded for to appear to be a Google Bot",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/requestmodifiers/masquerade_as_trusted_bot.go",
+  Params: []Param{
+
+  },
+},
+{
+  Name:        "MasqueradeAsBingBot",
+  Description: "MasqueradeAsBingBot modifies user agent and x-forwarded for to appear to be a Bing Bot",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/requestmodifiers/masquerade_as_trusted_bot.go",
+  Params: []Param{
+
+  },
+},
+{
+  Name:        "MasqueradeAsWaybackMachineBot",
+  Description: "MasqueradeAsWaybackMachineBot modifies user agent and x-forwarded for to appear to be a archive.org (wayback machine) Bot",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/requestmodifiers/masquerade_as_trusted_bot.go",
+  Params: []Param{
+
+  },
+},
+{
+  Name:        "MasqueradeAsFacebookBot",
+  Description: "MasqueradeAsFacebookBot modifies user agent and x-forwarded for to appear to be a Facebook Bot (link previews?)",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/requestmodifiers/masquerade_as_trusted_bot.go",
+  Params: []Param{
+
+  },
+},
+{
+  Name:        "MasqueradeAsYandexBot",
+  Description: "MasqueradeAsYandexBot modifies user agent and x-forwarded for to appear to be a Yandex Spider Bot",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/requestmodifiers/masquerade_as_trusted_bot.go",
+  Params: []Param{
+
+  },
+},
+{
+  Name:        "MasqueradeAsBaiduBot",
+  Description: "MasqueradeAsBaiduBot modifies user agent and x-forwarded for to appear to be a Baidu Spider Bot",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/requestmodifiers/masquerade_as_trusted_bot.go",
+  Params: []Param{
+
+  },
+},
+{
+  Name:        "MasqueradeAsDuckDuckBot",
+  Description: "MasqueradeAsDuckDuckBot modifies user agent and x-forwarded for to appear to be a DuckDuckGo Bot",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/requestmodifiers/masquerade_as_trusted_bot.go",
+  Params: []Param{
+
+  },
+},
+{
+  Name:        "MasqueradeAsYahooBot",
+  Description: "MasqueradeAsYahooBot modifies user agent and x-forwarded for to appear to be a Yahoo Bot",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/requestmodifiers/masquerade_as_trusted_bot.go",
+  Params: []Param{
+
+  },
+},
+{
+  Name:        "ModifyDomainWithRegex",
+  Description: "",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/requestmodifiers/modify_domain_with_regex.go",
+  Params: []Param{
+    {Name: "matchRegex", Type: "string"},
+    {Name: "replacement", Type: "string"},
+  },
+},
+{
+  Name:        "SetOutgoingCookie",
+  Description: "SetOutgoingCookie modifes a specific cookie name by modifying the request cookie headers going to the upstream server. If the cookie name does not already exist, it is created.",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/requestmodifiers/modify_outgoing_cookies.go",
+  Params: []Param{
+    {Name: "name", Type: "string"},
+    {Name: "val", Type: "string"},
+  },
+},
+{
+  Name:        "SetOutgoingCookies",
+  Description: "SetOutgoingCookies modifies a client request's cookie header to a raw Cookie string, overwriting existing cookies",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/requestmodifiers/modify_outgoing_cookies.go",
+  Params: []Param{
+    {Name: "cookies", Type: "string"},
+  },
+},
+{
+  Name:        "DeleteOutgoingCookie",
+  Description: "DeleteOutgoingCookie modifies the http request's cookies header to delete a specific request cookie going to the upstream server. If the cookie does not exist, it does not do anything.",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/requestmodifiers/modify_outgoing_cookies.go",
+  Params: []Param{
+    {Name: "name", Type: "string"},
+  },
+},
+{
+  Name:        "DeleteOutgoingCookies",
+  Description: "DeleteOutgoingCookies removes the cookie header entirely, preventing any cookies from reaching the upstream server.",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/requestmodifiers/modify_outgoing_cookies.go",
+  Params: []Param{
+
+  },
+},
+{
+  Name:        "DeleteOutgoingCookiesExcept",
+  Description: "DeleteOutGoingCookiesExcept prevents non-whitelisted cookies from being sent from the client to the upstream proxy server. Cookies whose names are in the whitelist are not removed.",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/requestmodifiers/modify_outgoing_cookies.go",
+  Params: []Param{
+    {Name: "whitelist", Type: "&{Ellipsis:12476 Elt:string}"},
+  },
+},
+{
+  Name:        "ModifyPathWithRegex",
+  Description: "",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/requestmodifiers/modify_path_with_regex.go",
+  Params: []Param{
+    {Name: "matchRegex", Type: "string"},
+    {Name: "replacement", Type: "string"},
+  },
+},
+{
+  Name:        "ModifyQueryParams",
+  Description: "ModifyQueryParams replaces query parameter values in URL's query params in a ProxyChain's URL. If the query param key doesn't exist, it is created.",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/requestmodifiers/modify_query_params.go",
+  Params: []Param{
+    {Name: "key", Type: "string"},
+    {Name: "value", Type: "string"},
+  },
+},
+{
+  Name:        "SetRequestHeader",
+  Description: "SetRequestHeader modifies a specific outgoing header This is the header that the upstream server will see.",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/requestmodifiers/modify_request_headers.go",
+  Params: []Param{
+    {Name: "name", Type: "string"},
+    {Name: "val", Type: "string"},
+  },
+},
+{
+  Name:        "DeleteRequestHeader",
+  Description: "DeleteRequestHeader modifies a specific outgoing header This is the header that the upstream server will see.",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/requestmodifiers/modify_request_headers.go",
+  Params: []Param{
+    {Name: "name", Type: "string"},
+  },
+},
+{
+  Name:        "RequestArchiveIs",
+  Description: "RequestArchiveIs modifies a ProxyChain's URL to request an archived version from archive.is",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/requestmodifiers/request_archive_is.go",
+  Params: []Param{
+
+  },
+},
+{
+  Name:        "RequestGoogleCache",
+  Description: "RequestGoogleCache modifies a ProxyChain's URL to request its Google Cache version.",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/requestmodifiers/request_google_cache.go",
+  Params: []Param{
+
+  },
+},
+{
+  Name:        "RequestWaybackMachine",
+  Description: "RequestWaybackMachine modifies a ProxyChain's URL to request the wayback machine (archive.org) version.",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/requestmodifiers/request_wayback_machine.go",
+  Params: []Param{
+
+  },
+},
+{
+  Name:        "ResolveWithGoogleDoH",
+  Description: "ResolveWithGoogleDoH modifies a ProxyChain's client to make the request by resolving the URL using Google's DNS over HTTPs service",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/requestmodifiers/resolve_with_google_doh.go",
+  Params: []Param{
+
+  },
+},
+{
+  Name:        "SpoofOrigin",
+  Description: "SpoofOrigin modifies the origin header if the upstream server returns a Vary header it means you might get a different response if you change this",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/requestmodifiers/spoof_origin.go",
+  Params: []Param{
+    {Name: "url", Type: "string"},
+  },
+},
+{
+  Name:        "HideOrigin",
+  Description: "HideOrigin modifies the origin header so that it is the original origin, not the proxy",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/requestmodifiers/spoof_origin.go",
+  Params: []Param{
+
+  },
+},
+{
+  Name:        "SpoofReferrer",
+  Description: "SpoofReferrer modifies the referrer header. It is useful if the page can be accessed from a search engine or social media site, but not by browsing the website itself. if url is \"\", then the referrer header is removed.",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/requestmodifiers/spoof_referrer.go",
+  Params: []Param{
+    {Name: "url", Type: "string"},
+  },
+},
+{
+  Name:        "HideReferrer",
+  Description: "HideReferrer modifies the referrer header so that it is the original referrer, not the proxy",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/requestmodifiers/spoof_referrer.go",
+  Params: []Param{
+
+  },
+},
+{
+  Name:        "SpoofReferrerFromBaiduSearch",
+  Description: "SpoofReferrerFromBaiduSearch modifies the referrer header pretending to be from a BaiduSearch",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/requestmodifiers/spoof_referrer_from_baidu_post.go",
+  Params: []Param{
+
+  },
+},
+{
+  Name:        "SpoofReferrerFromBingSearch",
+  Description: "SpoofReferrerFromBingSearch modifies the referrer header pretending to be from a bing search site",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/requestmodifiers/spoof_referrer_from_bing_search.go",
+  Params: []Param{
+
+  },
+},
+{
+  Name:        "SpoofReferrerFromGoogleSearch",
+  Description: "SpoofReferrerFromGoogleSearch modifies the referrer header pretending to be from a google search site",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/requestmodifiers/spoof_referrer_from_google_search.go",
+  Params: []Param{
+
+  },
+},
+{
+  Name:        "SpoofReferrerFromLinkedInPost",
+  Description: "SpoofReferrerFromLinkedInPost modifies the referrer header pretending to be from a linkedin post",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/requestmodifiers/spoof_referrer_from_linkedin_post.go",
+  Params: []Param{
+
+  },
+},
+{
+  Name:        "SpoofReferrerFromNaverSearch",
+  Description: "SpoofReferrerFromNaverSearch modifies the referrer header pretending to be from a Naver search (popular in South Korea)",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/requestmodifiers/spoof_referrer_from_naver_post.go",
+  Params: []Param{
+
+  },
+},
+{
+  Name:        "SpoofReferrerFromPinterestPost",
+  Description: "SpoofReferrerFromPinterestPost modifies the referrer header pretending to be from a pinterest post",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/requestmodifiers/spoof_referrer_from_pinterest_post.go",
+  Params: []Param{
+
+  },
+},
+{
+  Name:        "SpoofReferrerFromQQPost",
+  Description: "SpoofReferrerFromQQPost modifies the referrer header pretending to be from a QQ post (popular social media in China)",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/requestmodifiers/spoof_referrer_from_qq_post.go",
+  Params: []Param{
+
+  },
+},
+{
+  Name:        "SpoofReferrerFromRedditPost",
+  Description: "SpoofReferrerFromRedditPost modifies the referrer header pretending to be from a reddit post",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/requestmodifiers/spoof_referrer_from_reddit_post.go",
+  Params: []Param{
+
+  },
+},
+{
+  Name:        "SpoofReferrerFromTumblrPost",
+  Description: "SpoofReferrerFromTumblrPost modifies the referrer header pretending to be from a tumblr post",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/requestmodifiers/spoof_referrer_from_tumblr_post.go",
+  Params: []Param{
+
+  },
+},
+{
+  Name:        "SpoofReferrerFromTwitterPost",
+  Description: "SpoofReferrerFromTwitterPost modifies the referrer header pretending to be from a twitter post",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/requestmodifiers/spoof_referrer_from_twitter_post.go",
+  Params: []Param{
+
+  },
+},
+{
+  Name:        "SpoofReferrerFromVkontaktePost",
+  Description: "SpoofReferrerFromVkontaktePost modifies the referrer header pretending to be from a vkontakte post (popular in Russia)",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/requestmodifiers/spoof_referrer_from_vkontake_post.go",
+  Params: []Param{
+
+  },
+},
+{
+  Name:        "SpoofReferrerFromWeiboPost",
+  Description: "SpoofReferrerFromWeiboPost modifies the referrer header pretending to be from a Weibo post (popular in China)",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/requestmodifiers/spoof_referrer_from_weibo_post.go",
+  Params: []Param{
+
+  },
+},
+{
+  Name:        "SpoofUserAgent",
+  Description: "SpoofUserAgent modifies the user agent",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/requestmodifiers/spoof_user_agent.go",
+  Params: []Param{
+    {Name: "ua", Type: "string"},
+  },
+},
+{
+  Name:        "SpoofXForwardedFor",
+  Description: "SpoofXForwardedFor modifies the X-Forwarded-For header in some cases, a forward proxy may interpret this as the source IP",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/requestmodifiers/spoof_x_forwarded_for.go",
+  Params: []Param{
+    {Name: "ip", Type: "string"},
+  },
+},
+	},
+	ResponseModifiers: []Modifier{
+{
+  Name:        "APIContent",
+  Description: "APIContent creates an JSON representation of the article and returns it as an API response.",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/responsemodifiers/api_content.go",
+  Params: []Param{
+
+  },
+},
+{
+  Name:        "BlockElementRemoval",
+  Description: "BlockElementRemoval prevents paywall javascript from removing a particular element by detecting the removal, then immediately reinserting it. This is useful when a page will return a \"fake\" 404, after flashing the content briefly. If the /outline/ API works, but the regular API doesn't, try this modifier.",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/responsemodifiers/block_element_removal.go",
+  Params: []Param{
+    {Name: "cssSelector", Type: "string"},
+  },
+},
+{
+  Name:        "BlockThirdPartyScripts",
+  Description: "BlockThirdPartyScripts rewrites HTML and injects JS to block all third party JS from loading.",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/responsemodifiers/block_third_party_scripts.go",
+  Params: []Param{
+
+  },
+},
+{
+  Name:        "BypassCORS",
+  Description: "BypassCORS modifies response headers to prevent the browser from enforcing any CORS restrictions. This should run at the end of the chain.",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/responsemodifiers/bypass_cors.go",
+  Params: []Param{
+
+  },
+},
+{
+  Name:        "BypassContentSecurityPolicy",
+  Description: "BypassContentSecurityPolicy modifies response headers to prevent the browser from enforcing any CSP restrictions. This should run at the end of the chain.",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/responsemodifiers/bypass_csp.go",
+  Params: []Param{
+
+  },
+},
+{
+  Name:        "SetContentSecurityPolicy",
+  Description: "SetContentSecurityPolicy modifies response headers to a specific CSP",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/responsemodifiers/bypass_csp.go",
+  Params: []Param{
+    {Name: "csp", Type: "string"},
+  },
+},
+{
+  Name:        "DeleteLocalStorageData",
+  Description: "DeleteLocalStorageData deletes localstorage cookies. If the page works once in a fresh incognito window, but fails for subsequent loads, try this response modifier alongside DeleteSessionStorageData and DeleteIncomingCookies",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/responsemodifiers/delete_localstorage_data.go",
+  Params: []Param{
+
+  },
+},
+{
+  Name:        "DeleteSessionStorageData",
+  Description: "DeleteSessionStorageData deletes localstorage cookies. If the page works once in a fresh incognito window, but fails for subsequent loads, try this response modifier alongside DeleteLocalStorageData and DeleteIncomingCookies",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/responsemodifiers/delete_sessionstorage_data.go",
+  Params: []Param{
+
+  },
+},
+{
+  Name:        "ForwardResponseHeaders",
+  Description: "ForwardResponseHeaders forwards the response headers from the upstream server to the client",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/responsemodifiers/forward_response_headers.go",
+  Params: []Param{
+
+  },
+},
+{
+  Name:        "GenerateReadableOutline",
+  Description: "GenerateReadableOutline creates an reader-friendly distilled representation of the article. This is a reliable way of bypassing soft-paywalled articles, where the content is hidden, but still present in the DOM.",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/responsemodifiers/generate_readable_outline.go",
+  Params: []Param{
+
+  },
+},
+{
+  Name:        "InjectScriptBeforeDOMContentLoaded",
+  Description: "InjectScriptBeforeDOMContentLoaded modifies HTTP responses to inject a JS before DOM Content is loaded (script tag in head)",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/responsemodifiers/inject_script.go",
+  Params: []Param{
+    {Name: "js", Type: "string"},
+  },
+},
+{
+  Name:        "InjectScriptAfterDOMContentLoaded",
+  Description: "InjectScriptAfterDOMContentLoaded modifies HTTP responses to inject a JS after DOM Content is loaded (script tag in head)",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/responsemodifiers/inject_script.go",
+  Params: []Param{
+    {Name: "js", Type: "string"},
+  },
+},
+{
+  Name:        "InjectScriptAfterDOMIdle",
+  Description: "InjectScriptAfterDOMIdle modifies HTTP responses to inject a JS after the DOM is idle (ie: js framework loaded)",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/responsemodifiers/inject_script.go",
+  Params: []Param{
+    {Name: "js", Type: "string"},
+  },
+},
+{
+  Name:        "DeleteIncomingCookies",
+  Description: "DeleteIncomingCookies prevents ALL cookies from being sent from the proxy server back down to the client.",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/responsemodifiers/modify_incoming_cookies.go",
+  Params: []Param{
+    {Name: "_", Type: "&{Ellipsis:18780 Elt:string}"},
+  },
+},
+{
+  Name:        "DeleteIncomingCookiesExcept",
+  Description: "DeleteIncomingCookiesExcept prevents non-whitelisted cookies from being sent from the proxy server to the client. Cookies whose names are in the whitelist are not removed.",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/responsemodifiers/modify_incoming_cookies.go",
+  Params: []Param{
+    {Name: "whitelist", Type: "&{Ellipsis:19325 Elt:string}"},
+  },
+},
+{
+  Name:        "SetIncomingCookies",
+  Description: "SetIncomingCookies adds a raw cookie string being sent from the proxy server down to the client",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/responsemodifiers/modify_incoming_cookies.go",
+  Params: []Param{
+    {Name: "cookies", Type: "string"},
+  },
+},
+{
+  Name:        "SetIncomingCookie",
+  Description: "SetIncomingCookie modifies a specific cookie in the response from the proxy server to the client.",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/responsemodifiers/modify_incoming_cookies.go",
+  Params: []Param{
+    {Name: "name", Type: "string"},
+    {Name: "val", Type: "string"},
+  },
+},
+{
+  Name:        "ModifyIncomingScriptsWithRegex",
+  Description: "ModifyIncomingScriptsWithRegex modifies all incoming javascript (application/javascript and inline <script> in text/html) using a regex match and replacement.",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/responsemodifiers/modify_incoming_scripts_with_regex.go",
+  Params: []Param{
+    {Name: "matchRegex", Type: "string"},
+    {Name: "replacement", Type: "string"},
+  },
+},
+{
+  Name:        "SetResponseHeader",
+  Description: "SetResponseHeader modifies response headers from the upstream server",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/responsemodifiers/modify_response_header.go",
+  Params: []Param{
+    {Name: "key", Type: "string"},
+    {Name: "value", Type: "string"},
+  },
+},
+{
+  Name:        "DeleteResponseHeader",
+  Description: "DeleteResponseHeader removes response headers from the upstream server",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/responsemodifiers/modify_response_header.go",
+  Params: []Param{
+    {Name: "key", Type: "string"},
+  },
+},
+{
+  Name:        "PatchDynamicResourceURLs",
+  Description: "PatchDynamicResourceURLs patches the javascript runtime to rewrite URLs client-side.   - This function is designed to allow the proxified page     to still be browsible by routing all resource URLs through the proxy.   - Native APIs capable of network requests will be hooked     and the URLs arguments modified to point to the proxy instead.   - fetch('/relative_path') -> fetch('/https://proxiedsite.com/relative_path')   - Element.setAttribute('src', \"/assets/img.jpg\") -> Element.setAttribute('src', \"/https://proxiedsite.com/assets/img.jpg\") -> fetch('/https://proxiedsite.com/relative_path')",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/responsemodifiers/patch_dynamic_resource_urls.go",
+  Params: []Param{
+
+  },
+},
+{
+  Name:        "PatchTrackerScripts",
+  Description: "PatchTrackerScripts replaces any request to tracker scripts such as google analytics with a no-op stub that mocks the API structure of the original scripts they replace. Some pages depend on the existence of these structures for proper loading, so this may fix some broken elements. Surrogate script code borrowed from: DuckDuckGo Privacy Essentials browser extension for Firefox, Chrome. (Apache 2.0 license)",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/responsemodifiers/patch_tracker_scripts.go",
+  Params: []Param{
+
+  },
+},
+{
+  Name:        "RewriteHTMLResourceURLs",
+  Description: "RewriteHTMLResourceURLs modifies HTTP responses to rewrite URLs attributes in HTML content (such as src, href)   - `<img src='/relative_path'>` -> `<img src='/https://proxiedsite.com/relative_path'>`   - This function is designed to allow the proxified page     to still be browsible by routing all resource URLs through the proxy.",
+  CodeEditLink: "https://github.com/everywall/ladder.git/edit/heads/origin/proxy_v2/proxychain/responsemodifiers/rewrite_http_resource_urls.go",
+  Params: []Param{
+
+  },
+},
+	},
+}

handlers/api_raw.go

@@ -0,0 +1,51 @@
+package handlers
+
+import (
+	rx "github.com/everywall/ladder/proxychain/requestmodifiers"
+	tx "github.com/everywall/ladder/proxychain/responsemodifiers"
+
+	"github.com/everywall/ladder/proxychain"
+
+	"github.com/gofiber/fiber/v2"
+)
+
+func NewRawProxySiteHandler(opts *ProxyOptions) fiber.Handler {
+
+	return func(c *fiber.Ctx) error {
+		proxychain := proxychain.
+			NewProxyChain().
+			SetFiberCtx(c).
+			SetRequestModifications(
+				rx.AddCacheBusterQuery(),
+				rx.MasqueradeAsGoogleBot(),
+				rx.ForwardRequestHeaders(),
+				rx.HideOrigin(),
+				rx.DeleteOutgoingCookies(),
+				rx.SpoofReferrerFromRedditPost(),
+			)
+
+		// no options passed in, return early
+		if opts == nil {
+			// return as plaintext, overriding any rules
+			proxychain.AddOnceResponseModifications(
+				tx.SetResponseHeader("content-type", "text/plain; charset=UTF-8"),
+			)
+
+			return proxychain.Execute()
+		}
+
+		// load ruleset
+		rule, exists := opts.Ruleset.GetRule(proxychain.Request.URL)
+		if exists {
+			proxychain.AddOnceRequestModifications(rule.RequestModifications...)
+			proxychain.AddOnceResponseModifications(rule.ResponseModifications...)
+		}
+
+		// return as plaintext, overriding any rules
+		proxychain.AddOnceResponseModifications(
+			tx.SetResponseHeader("content-type", "text/plain; charset=UTF-8"),
+		)
+
+		return proxychain.Execute()
+	}
+}

handlers/api_ruleset.go

@@ -0,0 +1,96 @@
+package handlers
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/gofiber/fiber/v2"
+	"gopkg.in/yaml.v3"
+	"net/url"
+	"strings"
+)
+
+func NewRulesetSiteHandler(opts *ProxyOptions) fiber.Handler {
+
+	return func(c *fiber.Ctx) error {
+		if opts == nil {
+			c.SendStatus(404)
+			c.SendString("No ruleset specified. Set the RULESET environment variable or use the --ruleset flag.")
+		}
+
+		// no specific rule requested, return the entire ruleset
+		if c.Params("*") == "" {
+			switch c.Get("accept") {
+			case "application/json":
+				jsn, err := opts.Ruleset.JSON()
+				if err != nil {
+					return err
+				}
+				c.Set("content-type", "application/json")
+				return c.Send([]byte(jsn))
+
+			default:
+				yml, err := opts.Ruleset.YAML()
+				if err != nil {
+					return err
+				}
+				c.Set("content-type", "text/yaml")
+				return c.Send([]byte(yml))
+			}
+		}
+
+		// a specific rule was requested by path /ruleset/https://example.com
+		// return only that particular rule
+		reqURL, err := extractURLFromContext(c, "api/ruleset/")
+		if err != nil {
+			c.SendStatus(404)
+			return c.SendString(fmt.Sprintf("A rule that matches '%s' was not found in the ruleset. Possible URL formatting issue.", c.Params("*")))
+		}
+		rule, exists := opts.Ruleset.GetRule(reqURL)
+		if !exists {
+			c.SendStatus(404)
+			return c.SendString(fmt.Sprintf("A rule that matches '%s' was not found in the ruleset.", reqURL))
+		}
+
+		switch c.Get("accept") {
+		case "application/json":
+			jsn, err := json.MarshalIndent(rule, "", "  ")
+			if err != nil {
+				return err
+			}
+			c.Set("content-type", "application/json")
+			return c.Send(jsn)
+		default:
+			yml, err := yaml.Marshal(rule)
+			if err != nil {
+				return err
+			}
+			c.Set("content-type", "text/yaml")
+			return c.Send(yml)
+		}
+	}
+}
+
+// extractURLFromContext extracts a URL from the request ctx.
+func extractURLFromContext(ctx *fiber.Ctx, apiPrefix string) (*url.URL, error) {
+	reqURL := ctx.Params("*")
+
+	reqURL = strings.TrimPrefix(reqURL, apiPrefix)
+	if !strings.HasPrefix(reqURL, "http") {
+		reqURL = "https://" + reqURL
+	}
+
+	// sometimes client requests doubleroot '//'
+	// there is a bug somewhere else, but this is a workaround until we find it
+	if strings.HasPrefix(reqURL, "/") || strings.HasPrefix(reqURL, `%2F`) {
+		reqURL = strings.TrimPrefix(reqURL, "/")
+		reqURL = strings.TrimPrefix(reqURL, `%2F`)
+	}
+
+	// unescape url query
+	uReqURL, err := url.QueryUnescape(reqURL)
+	if err == nil {
+		reqURL = uReqURL
+	}
+
+	return url.Parse(reqURL)
+}

handlers/error_page.go

@@ -0,0 +1,32 @@
+package handlers
+
+import (
+	"embed"
+	"fmt"
+	"html/template"
+	"strings"
+
+	"github.com/gofiber/fiber/v2"
+)
+
+//go:embed error_page.html
+var errorHTML embed.FS
+
+func RenderErrorPage() fiber.Handler {
+	f := "error_page.html"
+	tmpl, err := template.ParseFS(errorHTML, f)
+	if err != nil {
+		panic(fmt.Errorf("RenderErrorPage Error: %s not found", f))
+	}
+	return func(c *fiber.Ctx) error {
+		if err := c.Next(); err != nil {
+			if strings.Contains(c.Get("Accept"), "text/html") {
+				c.Set("Content-Type", "text/html")
+				tmpl.Execute(c.Response().BodyWriter(), err.Error())
+				return nil
+			}
+			return c.SendString(err.Error())
+		}
+		return err
+	}
+}

handlers/error_page.html

@@ -0,0 +1,238 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <link rel="stylesheet" href="/styles.css" />
+    <script src="/script.js" defer></script>
+    <script>
+      const handleThemeChange = () => {
+        let theme = localStorage.getItem("theme");
+        if (theme === null) {
+          localStorage.setItem("theme", "system");
+          theme = "system";
+        }
+        if (
+          theme === "dark" ||
+          (theme === "system" &&
+            window.matchMedia("(prefers-color-scheme: dark)").matches)
+        ) {
+          document.documentElement.classList.add("dark");
+        } else {
+          document.documentElement.classList.remove("dark");
+        }
+      };
+      handleThemeChange();
+    </script>
+    <title>ladder | error</title>
+  </head>
+
+  <body
+    class="antialiased bg-white dark:bg-slate-900 text-slate-900 dark:text-slate-200"
+  >
+    <div class="flex flex-col gap-4 max-w-3xl mx-auto pt-10">
+      <div class="flex justify-between place-items-center">
+        <div
+          class="hover:drop-shadow-[0_0px_4px_rgba(122,167,209,.3)] ring-offset-white dark:ring-offset-slate-900 transition-colors duration-300 focus:outline-none focus:ring ring-offset-2"
+        >
+          <div class="flex">
+            <a
+              href="/"
+              aria-label="ladder"
+              class="flex -ml-2 h-8 font-extrabold tracking-tight no-underline focus:outline-none ring-offset-white dark:ring-offset-slate-900 focus:ring ring-offset-2"
+            >
+              <svg
+                xmlns="http://www.w3.org/2000/svg"
+                xmlns:xlink="http://www.w3.org/1999/xlink"
+                viewBox="0 0 512 512"
+                class="h-8 focus:outline-none focus:ring ring-offset-white dark:ring-offset-slate-900 ring-offset-2"
+              >
+                <path
+                  fill="#7AA7D1"
+                  d="M262.074 485.246C254.809 485.265 247.407 485.534 240.165 484.99L226.178 483.306C119.737 468.826 34.1354 383.43 25.3176 274.714C24.3655 262.975 23.5876 253.161 24.3295 241.148C31.4284 126.212 123.985 31.919 238.633 24.1259L250.022 23.8366C258.02 23.8001 266.212 23.491 274.183 24.1306C320.519 27.8489 366.348 45.9743 402.232 75.4548L416.996 88.2751C444.342 114.373 464.257 146.819 475.911 182.72L480.415 197.211C486.174 219.054 488.67 242.773 487.436 265.259L486.416 275.75C478.783 352.041 436.405 418.1 369.36 455.394L355.463 462.875C326.247 477.031 294.517 484.631 262.074 485.246ZM253.547 72.4475C161.905 73.0454 83.5901 144.289 73.0095 234.5C69.9101 260.926 74.7763 292.594 83.9003 317.156C104.53 372.691 153.9 416.616 211.281 430.903C226.663 434.733 242.223 436.307 258.044 436.227C353.394 435.507 430.296 361.835 438.445 267.978C439.794 252.442 438.591 236.759 435.59 221.5C419.554 139.955 353.067 79.4187 269.856 72.7052C264.479 72.2714 258.981 72.423 253.586 72.4127L253.547 72.4475Z"
+                />
+                <path
+                  fill="#7AA7D1"
+                  d="M153.196 310.121L133.153 285.021C140.83 283.798 148.978 285.092 156.741 284.353L156.637 277.725L124.406 278.002C123.298 277.325 122.856 276.187 122.058 275.193L116.089 267.862C110.469 260.975 103.827 254.843 98.6026 247.669C103.918 246.839 105.248 246.537 111.14 246.523L129.093 246.327C130.152 238.785 128.62 240.843 122.138 240.758C111.929 240.623 110.659 242.014 105.004 234.661L97.9953 225.654C94.8172 221.729 91.2219 218.104 88.2631 214.005C84.1351 208.286 90.1658 209.504 94.601 209.489L236.752 209.545C257.761 209.569 268.184 211.009 285.766 221.678L285.835 206.051C285.837 197.542 286.201 189.141 284.549 180.748C280.22 158.757 260.541 143.877 240.897 135.739C238.055 134.561 232.259 133.654 235.575 129.851C244.784 119.288 263.680 111.990 277.085 111.105C288.697 109.828 301.096 113.537 311.75 117.703C360.649 136.827 393.225 183.042 398.561 234.866C402.204 270.253 391.733 308.356 367.999 335.1C332.832 374.727 269.877 384.883 223.294 360.397C206.156 351.388 183.673 333.299 175.08 316.6C173.511 313.551 174.005 313.555 170.443 313.52L160.641 313.449C158.957 313.435 156.263 314.031 155.122 312.487L153.196 310.121Z"
+                />
+              </svg>
+              <span class="text-3xl ml-1 text-[#7AA7D1] leading-8 align-middle"
+                >ladder</span
+              >
+            </a>
+          </div>
+        </div>
+
+        <div class="flex justify-center z-10">
+          <div class="relative" id="dropdown">
+            <button
+              aria-expanded="false"
+              id="dropdownButton"
+              aria-label="Toggle dropdown menu"
+              onclick="toggleDropdown()"
+              type="button"
+              class="inline-flex items-center justify-center whitespace-nowrap rounded-full h-12 px-4 py-2 text-sm font-medium text-slate-600 dark:text-slate-400 ring-offset-white dark:ring-offset transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50 bg-white dark:bg-slate-900 hover:bg-slate-200 dark:hover:bg-slate-700 hover:text-slate-500 dark:hover:text-slate-200"
+            >
+              <svg
+                xmlns="http://www.w3.org/2000/svg"
+                viewBox="0 0 24 24"
+                fill="none"
+                stroke="currentColor"
+                stroke-width="2"
+                stroke-linecap="round"
+                stroke-linejoin="round"
+                class="h-5 w-5"
+              >
+                <path
+                  d="M12.22 2h-.44a2 2 0 0 0-2 2v.18a2 2 0 0 1-1 1.73l-.43.25a2 2 0 0 1-2 0l-.15-.08a2 2 0 0 0-2.73.73l-.22.38a2 2 0 0 0 .73 2.73l.15.1a2 2 0 0 1 1 1.72v.51a2 2 0 0 1-1 1.74l-.15.09a2 2 0 0 0-.73 2.73l.22.38a2 2 0 0 0 2.73.73l.15-.08a2 2 0 0 1 2 0l.43.25a2 2 0 0 1 1 1.73V20a2 2 0 0 0 2 2h.44a2 2 0 0 0 2-2v-.18a2 2 0 0 1 1-1.73l.43-.25a2 2 0 0 1 2 0l.15.08a2 2 0 0 0 2.73-.73l.22-.39a2 2 0 0 0-.73-2.73l-.15-.08a2 2 0 0 1-1-1.74v-.5a2 2 0 0 1 1-1.74l.15-.09a2 2 0 0 0 .73-2.73l-.22-.38a2 2 0 0 0-2.73-.73l-.15.08a2 2 0 0 1-2 0l-.43-.25a2 2 0 0 1-1-1.73V4a2 2 0 0 0-2-2z"
+                />
+                <circle cx="12" cy="12" r="3" />
+              </svg>
+            </button>
+
+            <div
+              id="dropdown_panel"
+              class="hidden absolute right-0 mt-2 w-52 rounded-md bg-white dark:bg-slate-900 shadow-md border border-slate-400 dark:border-slate-700"
+            >
+              <div
+                class="flex flex-col gap-2 w-full first-of-type:rounded-t-md last-of-type:rounded-b-md px-4 py-2.5 text-left text-sm"
+              >
+                Appearance
+                <div class="grid grid-cols-4 gap-2">
+                  <div>
+                    <input
+                      type="radio"
+                      name="theme"
+                      id="light"
+                      value="light"
+                      class="peer hidden"
+                    />
+                    <label
+                      for="light"
+                      tabindex="0"
+                      title="Light"
+                      class="flex items-end justify-center h-10 w-10 cursor-pointer select-none rounded-md p-2 text-sm text-slate-600 dark:text-slate-200 text-center hover:bg-slate-200 dark:hover:bg-slate-700 peer-checked:bg-slate-200 dark:peer-checked:bg-slate-700"
+                    >
+                      <svg
+                        xmlns="http://www.w3.org/2000/svg"
+                        viewBox="0 0 24 24"
+                        fill="none"
+                        stroke="currentColor"
+                        stroke-width="2"
+                        stroke-linecap="round"
+                        stroke-linejoin="round"
+                        class="h-5 w-5"
+                      >
+                        <circle cx="12" cy="12" r="4" />
+                        <path d="M12 2v2" />
+                        <path d="M12 20v2" />
+                        <path d="m4.93 4.93 1.41 1.41" />
+                        <path d="m17.66 17.66 1.41 1.41" />
+                        <path d="M2 12h2" />
+                        <path d="M20 12h2" />
+                        <path d="m6.34 17.66-1.41 1.41" />
+                        <path d="m19.07 4.93-1.41 1.41" />
+                      </svg>
+                    </label>
+                  </div>
+                  <div>
+                    <input
+                      type="radio"
+                      name="theme"
+                      id="dark"
+                      value="dark"
+                      class="peer hidden"
+                    />
+                    <label
+                      for="dark"
+                      tabindex="0"
+                      title="Dark"
+                      class="flex items-end justify-center h-10 w-10 cursor-pointer select-none rounded-md p-2 text-base text-slate-600 dark:text-slate-200 text-center hover:bg-slate-200 dark:hover:bg-slate-700 peer-checked:bg-slate-200 dark:peer-checked:bg-slate-700"
+                    >
+                      <svg
+                        xmlns="http://www.w3.org/2000/svg"
+                        width="24"
+                        height="24"
+                        viewBox="0 0 24 24"
+                        fill="none"
+                        stroke="currentColor"
+                        stroke-width="2"
+                        stroke-linecap="round"
+                        stroke-linejoin="round"
+                        class="h-5 w-5"
+                      >
+                        <path d="M12 3a6 6 0 0 0 9 9 9 9 0 1 1-9-9Z" />
+                      </svg>
+                    </label>
+                  </div>
+                  <div>
+                    <input
+                      type="radio"
+                      name="theme"
+                      id="system"
+                      value="system"
+                      class="peer hidden"
+                      checked
+                    />
+                    <label
+                      for="system"
+                      tabindex="0"
+                      title="System preference"
+                      class="flex items-end justify-center h-10 w-10 cursor-pointer select-none rounded-md p-2 text-lg text-slate-600 dark:text-slate-200 text-center hover:bg-slate-200 dark:hover:bg-slate-700 peer-checked:bg-slate-200 dark:peer-checked:bg-slate-700"
+                    >
+                      <svg
+                        xmlns="http://www.w3.org/2000/svg"
+                        width="24"
+                        height="24"
+                        viewBox="0 0 24 24"
+                        fill="none"
+                        stroke="currentColor"
+                        stroke-width="2"
+                        stroke-linecap="round"
+                        stroke-linejoin="round"
+                        class="h-5 w-5"
+                      >
+                        <path d="M12 8a2.83 2.83 0 0 0 4 4 4 4 0 1 1-4-4" />
+                        <path d="M12 2v2" />
+                        <path d="M12 20v2" />
+                        <path d="m4.9 4.9 1.4 1.4" />
+                        <path d="m17.7 17.7 1.4 1.4" />
+                        <path d="M2 12h2" />
+                        <path d="M20 12h2" />
+                        <path d="m6.3 17.7-1.4 1.4" />
+                        <path d="m19.1 4.9-1.4 1.4" />
+                      </svg>
+                    </label>
+                  </div>
+                </div>
+              </div>
+            </div>
+          </div>
+        </div>
+      </div>
+
+      <div class="flex flex-col space-y-3">
+        <h1>Error</h1>
+        <div class="my-4"></div>
+        <code class="p-4 mx-auto text-red-500 dark:text-red-400"> {{.}} </code>
+      </div>
+
+      <footer class="mx-4 my-2 text-center text-slate-600 dark:text-slate-400">
+        <small>
+          <a
+            href="https://github.com/everywall"
+            class="hover:text-blue-500 dark:hover:text-blue-500 hover:underline underline-offset-2 transition-colors duration-300"
+            >Everywall</a
+          >
+          |
+          <a
+            href="https://github.com/everywall/ladder"
+            class="hover:text-blue-500 dark:hover:text-blue-500 hover:underline underline-offset-2 transition-colors duration-300"
+            >Source</a
+          >
+          | Code Licensed Under GPL v3.0
+        </small>
+      </footer>
+    </div>
+  </body>
+</html>

handlers/form.html

@@ -32,7 +32,9 @@
       <div class="place-self-end z-10">
         <div class="relative" id="dropdown">
           <button
-            aria-expanded="closed"
+            aria-expanded="false"
+            id="dropdownButton"
+            aria-label="Toggle dropdown menu"
             onclick="toggleDropdown()"
             type="button"
             class="inline-flex items-center justify-center whitespace-nowrap rounded-full h-12 px-4 py-2 text-sm font-medium text-slate-600 dark:text-slate-400 ring-offset-white dark:ring-offset-slate-900 transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50 bg-white dark:bg-slate-900 hover:bg-slate-200 dark:hover:bg-slate-700 hover:text-slate-500 dark:hover:text-slate-200"
@@ -210,6 +212,7 @@
             class="w-full text-sm leading-6 text-slate-400 rounded-md ring-1 ring-slate-900/10 shadow-sm py-1.5 pl-2 pr-3 hover:ring-slate-300 dark:bg-slate-800 dark:highlight-white/5 dark:hover:bg-slate-700"
             autocomplete="off"
             autofocus
+            required
           />
           <button
             id="clearButton"
@@ -233,10 +236,6 @@
               <path d="m6 6 12 12" />
             </svg>
           </button>
-          <p
-            id="errorContainer"
-            class="absolute ml-2 left-0 -bottom-6 text-red-700 dark:text-red-400 text-sm"
-          />
         </div>
 
         <div
@@ -251,6 +250,7 @@
           </button>
           <button
             type="submit"
+            id="submitButton"
             class="inline-flex items-center justify-center h-11 px-8 whitespace-nowrap rounded-md text-sm font-medium text-slate-200 dark:text-slate-900 ring-offset-white dark:ring-offset-slate-900 transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50 bg-slate-800 dark:bg-slate-200 hover:bg-slate-800/90 dark:hover:bg-slate-200/90"
           >
             Proxy Search
@@ -258,55 +258,57 @@
         </div>
       </form>
 
-      <footer class="mx-4 text-center text-slate-600 dark:text-slate-400">
+      <div class="flex flex-col justify-center my-4 mx-auto">
-        <p>
-          Code Licensed Under GPL v3.0 |
         <a
-            href="https://github.com/everywall/ladder"
+          href="/playground"
-            class="hover:text-blue-500 dark:hover:text-blue-500 hover:underline underline-offset-2 transition-colors duration-300"
+          class="inline-flex items-center justify-center h-8 px-8 whitespace-nowrap no-underline rounded-md text-sm font-medium text-slate-900 dark:text-slate-200 hover:text-slate-900 hover:dark:text-slate-200 ring-offset-white dark:ring-offset-slate-900 transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none hover:bg-slate-200/90 dark:hover:bg-slate-800/90 border border-slate-600 dark:border-slate-400"
-            >View Source</a
         >
-          |
+          Experiment with modifiers in the playground
+          <svg
+            xmlns="http://www.w3.org/2000/svg"
+            width="24"
+            height="24"
+            viewBox="0 0 24 24"
+            fill="none"
+            stroke="currentColor"
+            stroke-width="2"
+            stroke-linecap="round"
+            stroke-linejoin="round"
+            class="h-4 w-4 ml-1"
+          >
+            <path d="M13 5H19V11" />
+            <path d="M19 5L5 19" />
+          </svg>
+        </a>
+      </div>
+
+      <footer class="mx-4 my-2 text-center text-slate-600 dark:text-slate-400">
+        <small>
           <a
             href="https://github.com/everywall"
             class="hover:text-blue-500 dark:hover:text-blue-500 hover:underline underline-offset-2 transition-colors duration-300"
             >Everywall</a
           >
-        </p>
+          |
+          <a
+            href="https://github.com/everywall/ladder"
+            class="hover:text-blue-500 dark:hover:text-blue-500 hover:underline underline-offset-2 transition-colors duration-300"
+            >Source</a
+          >
+          | Code Licensed Under GPL v3.0
+        </small>
       </footer>
     </div>
 
     <script>
       function validateAndRedirect(destination) {
         let url = inputField.value;
-        let error = "";
-        if (!url || typeof url !== "string") {
-          error = "Please enter a valid URL.";
-        }
-        if (typeof url === "string" && url.indexOf("http") === -1) {
-          url = "https://" + url;
-        }
-        const urlPattern = /^(https?:\/\/)?([\w.-]+)\.([a-z]{2,})(\/\S*)?$/i;
-        if (!urlPattern.test(url)) {
-          error = "Please enter a valid URL.";
-        }
-        if (error) {
-          errorContainer.textContent = error;
-          return false;
-        }
         const redirectUrl =
           destination === "outline" ? "/outline/" + url : "/" + url;
         window.location.href = redirectUrl;
         return true;
       }
 
-      function clearInput() {
-        inputField.value = "";
-        clearButton.style.display = "none";
-        errorContainer.textContent = "";
-        inputField.focus();
-      }
-
       document
         .getElementById("inputForm")
         .addEventListener("submit", function (e) {
@@ -319,33 +321,6 @@
         .addEventListener("click", function () {
           validateAndRedirect("outline");
         });
-
-      const inputField = document.getElementById("inputField");
-      const clearButton = document.getElementById("clearButton");
-      const errorContainer = document.getElementById("errorContainer");
-
-      if (inputField !== null && clearButton !== null) {
-        inputField.addEventListener("input", () => {
-          const clearButton = document.getElementById("clearButton");
-          if (clearButton !== null) {
-            if (inputField.value.trim().length > 0) {
-              clearButton.style.display = "block";
-            } else {
-              clearButton.style.display = "none";
-            }
-          }
-        });
-
-        inputField.addEventListener("keydown", (event) => {
-          if (event.code === "Escape") {
-            clearInput();
-          }
-        });
-
-        clearButton.addEventListener("click", () => {
-          clearInput();
-        });
-      }
     </script>
   </body>
 </html>

handlers/outline.go

@@ -1,9 +1,10 @@
 package handlers
 
 import (
-	"ladder/proxychain"
+	rx "github.com/everywall/ladder/proxychain/requestmodifiers"
-	rx "ladder/proxychain/requestmodifiers"
+	tx "github.com/everywall/ladder/proxychain/responsemodifiers"
-	tx "ladder/proxychain/responsemodifiers"
+
+	"github.com/everywall/ladder/proxychain"
 
 	"github.com/gofiber/fiber/v2"
 )
@@ -20,6 +21,7 @@ func NewOutlineHandler(path string, opts *ProxyOptions) fiber.Handler {
 				rx.SpoofReferrerFromGoogleSearch(),
 			).
 			AddResponseModifications(
+				tx.SetResponseHeader("content-type", "text/html"),
 				tx.DeleteIncomingCookies(),
 				tx.RewriteHTMLResourceURLs(),
 				tx.GenerateReadableOutline(), // <-- this response modification does the outline rendering

handlers/playground-script.js

@@ -0,0 +1,491 @@
+const modifierContainer = document.getElementById("modifierContainer");
+const modalContainer = document.getElementById("modalContainer");
+const modalBody = document.getElementById("modal-body");
+const modalContent = document.getElementById("modal-content");
+const modalSubmitButton = document.getElementById("modal-submit");
+const modalClose = document.getElementById("modal-close");
+
+let hasFetched = false;
+let payload = {
+  requestmodifications: [],
+  responsemodifications: [],
+};
+let ninjaData = [];
+
+initialize();
+
+// Rerun handleThemeChange() so style is applied to Ninja Keys
+handleThemeChange();
+
+// Add event listener to the iframe so it closes dropdown when clicked
+closeDropdownOnClickWithinIframe();
+
+async function initialize() {
+  if (!hasFetched) {
+    try {
+      await fetchPayload();
+      hasFetched = true;
+    } catch (error) {
+      console.error("Fetch error:", error);
+    }
+  }
+}
+
+function closeDropdownOnClickWithinIframe() {
+  const iframe = document.getElementById("resultIframe");
+  iframe.contentWindow.document.addEventListener(
+    "click",
+    () => {
+      if (
+        !document.getElementById("dropdown_panel").classList.contains("hidden")
+      ) {
+        toggleDropdown();
+      }
+    },
+    true
+  );
+}
+
+async function fetchPayload() {
+  try {
+    const response = await fetch("/api/modifiers");
+    const data = await response.json();
+
+    Object.entries(data.result.requestmodifiers ?? []).forEach(([_, value]) => {
+      addModifierToNinjaData(
+        value.name,
+        value.description,
+        value.params,
+        "requestmodifications"
+      );
+    });
+
+    Object.entries(data.result.responsemodifiers ?? []).forEach(
+      ([_, value]) => {
+        addModifierToNinjaData(
+          value.name,
+          value.description,
+          value.params,
+          "responsemodifications"
+        );
+      }
+    );
+
+    return data;
+  } catch (error) {
+    console.error("Fetch error:", error);
+    throw error;
+  }
+}
+
+async function submitForm() {
+  if (!document.getElementById("inputForm").checkValidity()) {
+    return;
+  }
+
+  try {
+    const response = await fetch("/playground/" + inputField.value, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify(payload),
+    });
+
+    if (!response.ok) {
+      throw new Error("Request failed");
+    }
+
+    const result = await response.text();
+    updateResultIframe(result);
+  } catch (error) {
+    console.error(error);
+  }
+}
+
+function updateResultIframe(result) {
+  const resultIframe = parent.document.getElementById("resultIframe");
+  resultIframe.contentDocument.open();
+  resultIframe.contentDocument.write(result);
+  closeDropdownOnClickWithinIframe();
+  resultIframe.contentDocument.close();
+}
+
+document.getElementById("inputForm").addEventListener("submit", function (e) {
+  e.preventDefault();
+  submitForm();
+});
+
+if (navigator.userAgent.includes("Mac")) {
+  document.getElementById("ninjaKey").textContent = "⌘";
+} else {
+  document.getElementById("ninjaKey").textContent = "Ctrl";
+}
+
+function downloadYaml() {
+  function jsonToYaml(payload) {
+    const jsonObject = {
+      rules: [
+        {
+          domains: [hostname],
+          responsemodifications: [],
+          requestmodifications: [],
+          ...payload,
+        },
+      ],
+    };
+    return jsyaml.dump(jsonObject);
+  }
+
+  if (!document.getElementById("inputForm").checkValidity()) {
+    alert("Please enter a valid URL.");
+    return;
+  }
+  const hostname = new URL(inputField.value).hostname;
+  const ruleHostname = hostname.replace(/^www\./, "").replace(/\./g, "-");
+  const yamlString = jsonToYaml(payload);
+  const blob = new Blob([yamlString], { type: "text/yaml;charset=utf-8" });
+  const href = URL.createObjectURL(blob);
+  const link = document.createElement("a");
+  link.href = href;
+  link.download = `${ruleHostname}.yaml`;
+  link.click();
+  URL.revokeObjectURL(href);
+}
+
+function getValues(type, id, description, params) {
+  const focusTrap = trap(modalBody);
+  let values = [];
+  let existingValues = [];
+  const inputs = [];
+  const inputEventListeners = [];
+
+  function closeModal() {
+    focusTrap.destroy();
+    modalBody.removeEventListener("keydown", handleKeyboardEvents);
+    modalContainer.removeEventListener("click", handleClickOutside);
+    modalSubmitButton.removeEventListener("click", closeModal);
+    modalClose.removeEventListener("click", closeModal);
+    inputEventListeners.forEach((listener, index) => {
+      if (listener !== undefined && inputs[index] !== undefined)
+        inputs[index].removeEventListener("input", listener);
+    });
+    modalContent.classList.remove("relative", "h-[220px]");
+    inputEventListeners.length = 0;
+    inputs.length = 0;
+    modalContainer.classList.add("hidden");
+    modalContent.innerHTML = "";
+  }
+
+  function handleClickOutside(e) {
+    if (modalBody !== null && !modalBody.contains(e.target)) {
+      closeModal();
+    }
+  }
+
+  function handleKeyboardEvents(e) {
+    if (e.key === "Escape") {
+      closeModal();
+    }
+    if (e.key === "Enter") {
+      if (e.target.tagName.toLowerCase() === "textarea") {
+        return;
+      } else {
+        modalSubmitButton.click();
+      }
+    }
+  }
+
+  document.getElementById("modal-title").textContent = id;
+  document.getElementById("modal-description").textContent = description;
+
+  existingValues =
+    payload[type].find(
+      (modifier) => modifier.name === id && modifier.params !== undefined
+    )?.params ?? [];
+
+  params.map((param, i) => {
+    function textareaEventListener(e) {
+      const codeElement = document.querySelector("code");
+      let text = e.target.value;
+
+      if (text[text.length - 1] == "\n") {
+        text += " ";
+      }
+
+      codeElement.innerHTML = text
+        .replace(new RegExp("&", "g"), "&")
+        .replace(new RegExp("<", "g"), "&lt;");
+
+      Prism.highlightElement(codeElement);
+      values[i] = text;
+      syncScroll(e.target);
+    }
+
+    function textareaKeyEventListener(e) {
+      if (e.key === "Tab" && !e.shiftKey) {
+        e.preventDefault();
+        e.stopPropagation();
+        let text = e.target.value;
+        const start = e.target.selectionStart;
+        const end = e.target.selectionEnd;
+        e.target.value = text.substring(0, start) + "\t" + text.substring(end);
+        e.target.setSelectionRange(start + 1, start + 1);
+        e.target.dispatchEvent(new Event("input"));
+      }
+      syncScroll(e.target);
+    }
+
+    function syncScroll(el) {
+      const codeElement = document.querySelector("code");
+      codeElement.scrollTop = el.scrollTop;
+      codeElement.scrollLeft = el.scrollLeft;
+    }
+
+    function inputEventListener(e) {
+      if (e.key !== "Enter") {
+        values[i] = e.target.value;
+      }
+    }
+
+    const label = document.createElement("label");
+    label.textContent = param.name;
+    label.setAttribute("for", `input-${i}`);
+    let input;
+    if (param.name === "js") {
+      input = document.createElement("textarea");
+      input.type = "textarea";
+      input.setAttribute("spellcheck", "false");
+      input.placeholder = "Enter your JavaScript injection code ...";
+      input.classList.add(
+        "h-[200px]",
+        "w-full",
+        "font-mono",
+        "whitespace-nowrap",
+        "font-semibold",
+        "absolute",
+        "text-base",
+        "leading-6",
+        "rounded-md",
+        "ring-1",
+        "ring-slate-900/10",
+        "shadow-sm",
+        "z-10",
+        "p-4",
+        "m-0",
+        "my-2",
+        "bg-transparent",
+        "dark:bg-transparent",
+        "text-transparent",
+        "overflow-auto",
+        "resize-none",
+        "caret-white",
+        "hover:ring-slate-300",
+        "hyphens-none"
+      );
+      input.style.tabSize = "4";
+    } else {
+      input = document.createElement("input");
+      input.type = "text";
+      input.classList.add(
+        "w-full",
+        "text-sm",
+        "leading-6",
+        "text-slate-400",
+        "rounded-md",
+        "ring-1",
+        "ring-slate-900/10",
+        "shadow-sm",
+        "py-1.5",
+        "pl-2",
+        "pr-3",
+        "mt-0",
+        "hover:ring-slate-300",
+        "dark:bg-slate-800",
+        "dark:highlight-white/5",
+        "overflow-auto"
+      );
+    }
+    input.id = `input-${i}`;
+    input.value = existingValues[i] ?? "";
+    modalContent.appendChild(label);
+    modalContent.appendChild(input);
+    if (input.type === "textarea") {
+      label.classList.add("sr-only", "hidden");
+      preElement = document.createElement("pre");
+      codeElement = document.createElement("code");
+      preElement.setAttribute("aria-hidden", "true");
+      preElement.classList.add(
+        "bg-[#2d2d2d]",
+        "dark:bg-[#2d2d2d]",
+        "h-[200px]",
+        "w-full",
+        "rounded-md",
+        "ring-1",
+        "ring-slate-900/10",
+        "shadow-sm",
+        "p-0",
+        "m-0",
+        "my-2",
+        "font-mono",
+        "text-base",
+        "leading-6",
+        "overflow-auto",
+        "whitespace-nowrap",
+        "font-semibold",
+        "absolute",
+        "z-0",
+        "hyphens-none"
+      );
+      modalContent.classList.add("relative", "h-[220px]");
+      preElement.setAttribute("tabindex", "-1");
+      codeElement.classList.add(
+        "language-javascript",
+        "absolute",
+        "w-full",
+        "font-mono",
+        "text-base",
+        "leading-6",
+        "z-0",
+        "p-4",
+        "-mx-4",
+        "-my-4",
+        "h-full",
+        "whitespace-nowrap",
+        "overflow-auto",
+        "hyphens-none"
+      );
+      preElement.appendChild(codeElement);
+      modalContent.appendChild(preElement);
+      codeElement.innerHTML = input.value
+        .replace(new RegExp("&", "g"), "&amp;")
+        .replace(new RegExp("<", "g"), "&lt;");
+      Prism.highlightElement(codeElement);
+      input.addEventListener("input", textareaEventListener);
+      input.addEventListener("keydown", textareaKeyEventListener);
+      input.addEventListener("scroll", () => syncScroll(input));
+      inputEventListeners.push(
+        textareaEventListener,
+        textareaKeyEventListener,
+        syncScroll
+      );
+    } else {
+      input.addEventListener("input", inputEventListener);
+      inputEventListeners.push(inputEventListener);
+    }
+    inputs.push(input);
+  });
+
+  modalContainer.classList.remove("hidden");
+  document.getElementById("input-0").focus();
+
+  return new Promise((resolve) => {
+    modalBody.addEventListener("keydown", handleKeyboardEvents);
+    modalContainer.addEventListener("click", handleClickOutside);
+    modalClose.addEventListener("click", () => {
+      closeModal();
+    });
+    modalSubmitButton.addEventListener("click", (e) => {
+      inputs.forEach((input, i) => {
+        values[i] = input.value;
+      });
+      resolve(values);
+      closeModal();
+    });
+  });
+}
+
+function toggleModifier(type, id, params = []) {
+  function pillClickHandler(pill) {
+    toggleModifier(pill.getAttribute("type"), pill.id);
+    pill.removeEventListener("click", () => pillClickHandler(pill));
+    pill.remove();
+  }
+
+  function createPill(type, id) {
+    const pill = document.createElement("span");
+    pill.classList.add(
+      "inline-flex",
+      "items-center",
+      "rounded-md",
+      "bg-slate-100",
+      "dark:bg-slate-800",
+      "px-2",
+      "py-1",
+      "h-4",
+      "text-xs",
+      "font-medium",
+      "border",
+      "border-slate-400",
+      "dark:border-slate-700",
+      "cursor-pointer"
+    );
+    pill.id = id;
+    pill.setAttribute("type", type);
+    pill.textContent = id;
+    modifierContainer.appendChild(pill);
+    pill.addEventListener("click", () => pillClickHandler(pill));
+  }
+
+  if (
+    params === undefined &&
+    payload[type].some((modifier) => modifier.name === id)
+  ) {
+    payload[type] = payload[type].filter((modifier) => modifier.name !== id);
+    const existingPill = document.getElementById(id);
+    if (existingPill !== null) {
+      existingPill.removeEventListener("click", () => pillClickHandler(pill));
+      existingPill.remove();
+    }
+  } else {
+    const existingModifier = payload[type].find(
+      (modifier) => modifier.name === id
+    );
+    if (existingModifier) {
+      existingModifier.params = params;
+    } else {
+      payload[type].push({ name: id, params: params });
+    }
+    const existingPill = document.getElementById(id);
+    if (existingPill === null) {
+      createPill(type, id);
+    }
+  }
+
+  submitForm();
+}
+
+function addModifierToNinjaData(id, description, params, type) {
+  const section =
+    type === "requestmodifications"
+      ? "Request Modifiers"
+      : "Response Modifiers";
+  const modifier = {
+    id: id,
+    title: id,
+    section: section,
+
+    handler: () => {
+      if (Object.keys(params).length === 0) {
+        toggleModifier(type, id);
+      } else {
+        if (params[0].name === "_") {
+          toggleModifier(type, id, (params = [""]));
+        } else {
+          getValues(type, id, description, params).then((values) => {
+            if (Object.keys(values).length === 0) return;
+            toggleModifier(type, id, values);
+          });
+        }
+      }
+    },
+  };
+
+  ninjaData.push(modifier);
+}
+
+const ninja = document.querySelector("ninja-keys");
+ninja.data = ninjaData;
+document.getElementById("btnNinja").addEventListener("click", () => {
+  ninja.open();
+});

handlers/playground.go

@@ -0,0 +1,42 @@
+package handlers
+
+import (
+	_ "embed"
+
+	"github.com/everywall/ladder/proxychain"
+	ruleset_v2 "github.com/everywall/ladder/proxychain/ruleset"
+
+	"net/http"
+
+	"github.com/gofiber/fiber/v2"
+)
+
+//go:embed playground.html
+var playgroundHtml string
+
+func PlaygroundHandler(path string, opts *ProxyOptions) fiber.Handler {
+	return func(c *fiber.Ctx) error {
+		if c.Method() == fiber.MethodGet {
+			c.Set("Content-Type", "text/html")
+
+			return c.SendString(playgroundHtml)
+		} else if c.Method() == fiber.MethodPost {
+			var modificationData ruleset_v2.Rule
+			if err := c.BodyParser(&modificationData); err != nil {
+				return err
+			}
+
+			c.Method(fiber.MethodGet)
+
+			return proxychain.
+				NewProxyChain().
+				SetFiberCtx(c).
+				WithAPIPath(path).
+				AddOnceRequestModifications(modificationData.RequestModifications...).
+				AddOnceResponseModifications(modificationData.ResponseModifications...).
+				Execute()
+		}
+
+		return c.Status(http.StatusMethodNotAllowed).SendString("Method not allowed")
+	}
+}

handlers/playground.html

@@ -0,0 +1,451 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>ladder | playground</title>
+    <link rel="stylesheet" href="/styles.css" />
+    <style>
+      #modifierContainer::-webkit-scrollbar {
+        width: 8px;
+      }
+      #modifierContainer::-webkit-scrollbar-track {
+        background: rgb(226 232 240);
+        border-radius: 5px;
+      }
+      #modifierContainer::-webkit-scrollbar-thumb {
+        background: rgb(100 116 139);
+        border-radius: 14px;
+      }
+      ninja-keys {
+        --ninja-accent-color: #7aa7d1;
+      }
+    </style>
+    <script src="/script.js" defer></script>
+    <script src="/playground-script.js" defer></script>
+    <script type="importmap">
+      {
+        "imports": {
+          "https://unpkg.com/lit-html@latest/directives/ref.js?module": "https://unpkg.com/lit-html@2.2.6/directives/ref.js?module"
+        }
+      }
+    </script>
+    <script type="module" src="https://unpkg.com/ninja-keys?module"></script>
+    <link
+      rel="stylesheet"
+      href="https://cdnjs.cloudflare.com/ajax/libs/prism/9000.0.1/themes/prism-tomorrow.min.css"
+      integrity="sha512-kSwGoyIkfz4+hMo5jkJngSByil9jxJPKbweYec/UgS+S1EgE45qm4Gea7Ks2oxQ7qiYyyZRn66A9df2lMtjIsw=="
+      crossorigin="anonymous"
+      referrerpolicy="no-referrer"
+    />
+    <script
+      src="https://cdnjs.cloudflare.com/ajax/libs/prism/9000.0.1/prism.min.js"
+      integrity="sha512-UOoJElONeUNzQbbKQbjldDf9MwOHqxNz49NNJJ1d90yp+X9edsHyJoAs6O4K19CZGaIdjI5ohK+O2y5lBTW6uQ=="
+      crossorigin="anonymous"
+      referrerpolicy="no-referrer"
+    ></script>
+    <script
+      src="https://cdnjs.cloudflare.com/ajax/libs/js-yaml/4.1.0/js-yaml.min.js"
+      integrity="sha512-CSBhVREyzHAjAFfBlIBakjoRUKp5h7VSweP0InR/pAJyptH7peuhCsqAI/snV+TwZmXZqoUklpXp6R6wMnYf5Q=="
+      crossorigin="anonymous"
+      referrerpolicy="no-referrer"
+    ></script>
+    <script>
+      const handleThemeChange = () => {
+        let theme = localStorage.getItem("theme");
+        if (theme === null) {
+          localStorage.setItem("theme", "system");
+          theme = "system";
+        }
+        if (
+          theme === "dark" ||
+          (theme === "system" &&
+            window.matchMedia("(prefers-color-scheme: dark)").matches)
+        ) {
+          document.documentElement.classList.add("dark");
+          const ninjaKeys = document.querySelector("ninja-keys");
+          if (ninjaKeys !== null) ninjaKeys.classList.add("dark");
+        } else {
+          document.documentElement.classList.remove("dark");
+          const ninjaKeys = document.querySelector("ninja-keys");
+          if (ninjaKeys !== null) ninjaKeys.classList.remove("dark");
+        }
+      };
+      handleThemeChange();
+    </script>
+  </head>
+
+  <body
+    class="antialiased bg-white dark:bg-slate-900 text-slate-900 dark:text-slate-200"
+  >
+    <div class="flex flex-col h-screen">
+      <div
+        class="fixed top-0 inset-x-0 h-48 flex-col gap-4 max-w-3xl mx-4 lg:mx-auto pt-4"
+      >
+        <header class="flex flex-col gap-2">
+          <div class="flex justify-between place-items-center">
+            <div
+              class="hover:drop-shadow-[0_0px_4px_rgba(122,167,209,.3)] ring-offset-white dark:ring-offset-slate-900 transition-colors duration-300 focus:outline-none focus:ring ring-offset-2"
+            >
+              <div class="flex">
+                <a
+                  href="/"
+                  aria-label="ladder"
+                  class="flex -ml-2 h-8 font-extrabold tracking-tight no-underline focus:outline-none ring-offset-white dark:ring-offset-slate-900 focus:ring ring-offset-2"
+                >
+                  <svg
+                    xmlns="http://www.w3.org/2000/svg"
+                    xmlns:xlink="http://www.w3.org/1999/xlink"
+                    viewBox="0 0 512 512"
+                    class="h-8 focus:outline-none focus:ring ring-offset-white dark:ring-offset-slate-900 ring-offset-2"
+                  >
+                    <path
+                      fill="#7AA7D1"
+                      d="M262.074 485.246C254.809 485.265 247.407 485.534 240.165 484.99L226.178 483.306C119.737 468.826 34.1354 383.43 25.3176 274.714C24.3655 262.975 23.5876 253.161 24.3295 241.148C31.4284 126.212 123.985 31.919 238.633 24.1259L250.022 23.8366C258.02 23.8001 266.212 23.491 274.183 24.1306C320.519 27.8489 366.348 45.9743 402.232 75.4548L416.996 88.2751C444.342 114.373 464.257 146.819 475.911 182.72L480.415 197.211C486.174 219.054 488.67 242.773 487.436 265.259L486.416 275.75C478.783 352.041 436.405 418.1 369.36 455.394L355.463 462.875C326.247 477.031 294.517 484.631 262.074 485.246ZM253.547 72.4475C161.905 73.0454 83.5901 144.289 73.0095 234.5C69.9101 260.926 74.7763 292.594 83.9003 317.156C104.53 372.691 153.9 416.616 211.281 430.903C226.663 434.733 242.223 436.307 258.044 436.227C353.394 435.507 430.296 361.835 438.445 267.978C439.794 252.442 438.591 236.759 435.59 221.5C419.554 139.955 353.067 79.4187 269.856 72.7052C264.479 72.2714 258.981 72.423 253.586 72.4127L253.547 72.4475Z"
+                    />
+                    <path
+                      fill="#7AA7D1"
+                      d="M153.196 310.121L133.153 285.021C140.83 283.798 148.978 285.092 156.741 284.353L156.637 277.725L124.406 278.002C123.298 277.325 122.856 276.187 122.058 275.193L116.089 267.862C110.469 260.975 103.827 254.843 98.6026 247.669C103.918 246.839 105.248 246.537 111.14 246.523L129.093 246.327C130.152 238.785 128.62 240.843 122.138 240.758C111.929 240.623 110.659 242.014 105.004 234.661L97.9953 225.654C94.8172 221.729 91.2219 218.104 88.2631 214.005C84.1351 208.286 90.1658 209.504 94.601 209.489L236.752 209.545C257.761 209.569 268.184 211.009 285.766 221.678L285.835 206.051C285.837 197.542 286.201 189.141 284.549 180.748C280.22 158.757 260.541 143.877 240.897 135.739C238.055 134.561 232.259 133.654 235.575 129.851C244.784 119.288 263.680 111.990 277.085 111.105C288.697 109.828 301.096 113.537 311.75 117.703C360.649 136.827 393.225 183.042 398.561 234.866C402.204 270.253 391.733 308.356 367.999 335.1C332.832 374.727 269.877 384.883 223.294 360.397C206.156 351.388 183.673 333.299 175.08 316.6C173.511 313.551 174.005 313.555 170.443 313.52L160.641 313.449C158.957 313.435 156.263 314.031 155.122 312.487L153.196 310.121Z"
+                    />
+                  </svg>
+                  <span
+                    class="text-3xl ml-1 text-[#7AA7D1] leading-8 align-middle"
+                    >ladder</span
+                  >
+                </a>
+                <span
+                  class="text-3xl mx-1 text-[#7AA7D1] leading-8 align-middle"
+                  >|</span
+                >
+                <a
+                  href="/playground"
+                  class="flex h-8 font-extrabold tracking-tight no-underline focus:outline-none focus:ring ring-offset-2 ring-offset-white dark:ring-offset"
+                >
+                  <span class="text-3xl text-[#7AA7D1] leading-8 align-middle"
+                    >playground</span
+                  >
+                </a>
+              </div>
+            </div>
+
+            <div class="flex justify-center z-10">
+              <div class="relative" id="dropdown">
+                <button
+                  aria-expanded="false"
+                  id="dropdownButton"
+                  aria-label="Toggle dropdown menu"
+                  onclick="toggleDropdown()"
+                  type="button"
+                  class="inline-flex items-center justify-center whitespace-nowrap rounded-full h-12 px-4 py-2 text-sm font-medium text-slate-600 dark:text-slate-400 ring-offset-white dark:ring-offset transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50 bg-white dark:bg-slate-900 hover:bg-slate-200 dark:hover:bg-slate-700 hover:text-slate-500 dark:hover:text-slate-200"
+                >
+                  <svg
+                    xmlns="http://www.w3.org/2000/svg"
+                    viewBox="0 0 24 24"
+                    fill="none"
+                    stroke="currentColor"
+                    stroke-width="2"
+                    stroke-linecap="round"
+                    stroke-linejoin="round"
+                    class="h-5 w-5"
+                  >
+                    <path
+                      d="M12.22 2h-.44a2 2 0 0 0-2 2v.18a2 2 0 0 1-1 1.73l-.43.25a2 2 0 0 1-2 0l-.15-.08a2 2 0 0 0-2.73.73l-.22.38a2 2 0 0 0 .73 2.73l.15.1a2 2 0 0 1 1 1.72v.51a2 2 0 0 1-1 1.74l-.15.09a2 2 0 0 0-.73 2.73l.22.38a2 2 0 0 0 2.73.73l.15-.08a2 2 0 0 1 2 0l.43.25a2 2 0 0 1 1 1.73V20a2 2 0 0 0 2 2h.44a2 2 0 0 0 2-2v-.18a2 2 0 0 1 1-1.73l.43-.25a2 2 0 0 1 2 0l.15.08a2 2 0 0 0 2.73-.73l.22-.39a2 2 0 0 0-.73-2.73l-.15-.08a2 2 0 0 1-1-1.74v-.5a2 2 0 0 1 1-1.74l.15-.09a2 2 0 0 0 .73-2.73l-.22-.38a2 2 0 0 0-2.73-.73l-.15.08a2 2 0 0 1-2 0l-.43-.25a2 2 0 0 1-1-1.73V4a2 2 0 0 0-2-2z"
+                    />
+                    <circle cx="12" cy="12" r="3" />
+                  </svg>
+                </button>
+
+                <div
+                  id="dropdown_panel"
+                  class="hidden absolute right-0 mt-2 w-52 rounded-md bg-white dark:bg-slate-900 shadow-md border border-slate-400 dark:border-slate-700"
+                >
+                  <div
+                    class="flex flex-col gap-2 w-full first-of-type:rounded-t-md last-of-type:rounded-b-md px-4 py-2.5 text-left text-sm"
+                  >
+                    Appearance
+                    <div class="grid grid-cols-4 gap-2">
+                      <div>
+                        <input
+                          type="radio"
+                          name="theme"
+                          id="light"
+                          value="light"
+                          class="peer hidden"
+                        />
+                        <label
+                          for="light"
+                          tabindex="0"
+                          title="Light"
+                          class="flex items-end justify-center h-10 w-10 cursor-pointer select-none rounded-md p-2 text-sm text-slate-600 dark:text-slate-200 text-center hover:bg-slate-200 dark:hover:bg-slate-700 peer-checked:bg-slate-200 dark:peer-checked:bg-slate-700"
+                        >
+                          <svg
+                            xmlns="http://www.w3.org/2000/svg"
+                            viewBox="0 0 24 24"
+                            fill="none"
+                            stroke="currentColor"
+                            stroke-width="2"
+                            stroke-linecap="round"
+                            stroke-linejoin="round"
+                            class="h-5 w-5"
+                          >
+                            <circle cx="12" cy="12" r="4" />
+                            <path d="M12 2v2" />
+                            <path d="M12 20v2" />
+                            <path d="m4.93 4.93 1.41 1.41" />
+                            <path d="m17.66 17.66 1.41 1.41" />
+                            <path d="M2 12h2" />
+                            <path d="M20 12h2" />
+                            <path d="m6.34 17.66-1.41 1.41" />
+                            <path d="m19.07 4.93-1.41 1.41" />
+                          </svg>
+                        </label>
+                      </div>
+                      <div>
+                        <input
+                          type="radio"
+                          name="theme"
+                          id="dark"
+                          value="dark"
+                          class="peer hidden"
+                        />
+                        <label
+                          for="dark"
+                          tabindex="0"
+                          title="Dark"
+                          class="flex items-end justify-center h-10 w-10 cursor-pointer select-none rounded-md p-2 text-base text-slate-600 dark:text-slate-200 text-center hover:bg-slate-200 dark:hover:bg-slate-700 peer-checked:bg-slate-200 dark:peer-checked:bg-slate-700"
+                        >
+                          <svg
+                            xmlns="http://www.w3.org/2000/svg"
+                            width="24"
+                            height="24"
+                            viewBox="0 0 24 24"
+                            fill="none"
+                            stroke="currentColor"
+                            stroke-width="2"
+                            stroke-linecap="round"
+                            stroke-linejoin="round"
+                            class="h-5 w-5"
+                          >
+                            <path d="M12 3a6 6 0 0 0 9 9 9 9 0 1 1-9-9Z" />
+                          </svg>
+                        </label>
+                      </div>
+                      <div>
+                        <input
+                          type="radio"
+                          name="theme"
+                          id="system"
+                          value="system"
+                          class="peer hidden"
+                          checked
+                        />
+                        <label
+                          for="system"
+                          tabindex="0"
+                          title="System preference"
+                          class="flex items-end justify-center h-10 w-10 cursor-pointer select-none rounded-md p-2 text-lg text-slate-600 dark:text-slate-200 text-center hover:bg-slate-200 dark:hover:bg-slate-700 peer-checked:bg-slate-200 dark:peer-checked:bg-slate-700"
+                        >
+                          <svg
+                            xmlns="http://www.w3.org/2000/svg"
+                            width="24"
+                            height="24"
+                            viewBox="0 0 24 24"
+                            fill="none"
+                            stroke="currentColor"
+                            stroke-width="2"
+                            stroke-linecap="round"
+                            stroke-linejoin="round"
+                            class="h-5 w-5"
+                          >
+                            <path d="M12 8a2.83 2.83 0 0 0 4 4 4 4 0 1 1-4-4" />
+                            <path d="M12 2v2" />
+                            <path d="M12 20v2" />
+                            <path d="m4.9 4.9 1.4 1.4" />
+                            <path d="m17.7 17.7 1.4 1.4" />
+                            <path d="M2 12h2" />
+                            <path d="M20 12h2" />
+                            <path d="m6.3 17.7-1.4 1.4" />
+                            <path d="m19.1 4.9-1.4 1.4" />
+                          </svg>
+                        </label>
+                      </div>
+                    </div>
+                  </div>
+                </div>
+              </div>
+            </div>
+          </div>
+          <form id="inputForm" method="get" class="flex flex-col gap-2 mx-4">
+            <div class="flex gap-6">
+              <div class="flex-1 relative">
+                <input
+                  type="url"
+                  id="inputField"
+                  placeholder="Enter URL"
+                  name="inputField"
+                  class="w-full text-sm leading-6 text-slate-400 rounded-md ring-1 ring-slate-900/10 shadow-sm py-1.5 pl-2 pr-3 hover:ring-slate-300 dark:bg-slate-800 dark:highlight-white/5 dark:hover:bg-slate-700"
+                  autocomplete="off"
+                  autofocus
+                  required
+                />
+                <button
+                  id="clearButton"
+                  type="reset"
+                  aria-label="Clear Search"
+                  title="Clear Search"
+                  class="hidden absolute inset-y-0 right-0 items-center pr-2 text-slate-600 dark:text-slate-400 hover:text-slate-400 hover:dark:text-slate-300"
+                  tabindex="-1"
+                >
+                  <svg
+                    xmlns="http://www.w3.org/2000/svg"
+                    viewBox="0 0 24 24"
+                    fill="none"
+                    stroke="currentColor"
+                    stroke-width="2"
+                    stroke-linecap="round"
+                    stroke-linejoin="round"
+                    class="h-4 w-4"
+                  >
+                    <path d="M18 6 6 18" />
+                    <path d="m6 6 12 12" />
+                  </svg>
+                </button>
+              </div>
+              <button
+                id="btnNinja"
+                type="button"
+                class="inline-flex items-center justify-center h-8 px-8 my-auto whitespace-nowrap rounded-xl text-sm font-medium text-slate-900 dark:text-slate-200 ring-offset-white dark:ring-offset-slate-900 transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50 bg-white dark:bg-slate-900 hover:bg-slate-200/90 dark:hover:bg-slate-800/90 border hover:bg-slate-200 dark:hover:bg-slate-500"
+              >
+                Press &nbsp;<span id="ninjaKey">⌘</span>&nbsp; + K to Apply
+                Modifiers
+              </button>
+            </div>
+          </form>
+          <div class="mt-2 mx-4 flex justify-between gap-2">
+            <div
+              id="modifierContainer"
+              class="flex flex-wrap grow overflow-y-auto max-h-14 w-full gap-1"
+            ></div>
+            <button
+              type="button"
+              class="inline-flex items-center justify-center h-11 px-4 py-2 whitespace-nowrap rounded-md text-sm font-medium text-slate-200 dark:text-slate-900 ring-offset-white dark:ring-offset-slate-900 transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50 bg-slate-800 dark:bg-slate-200 hover:bg-slate-800/90 dark:hover:bg-slate-200/90"
+              onclick="downloadYaml()"
+            >
+              <svg
+                xmlns="http://www.w3.org/2000/svg"
+                viewBox="0 0 24 24"
+                fill="none"
+                stroke="currentColor"
+                stroke-width="2"
+                stroke-linecap="round"
+                stroke-linejoin="round"
+                class="pl-0 mr-2 w-5 h-5"
+              >
+                <path
+                  d="M14.5 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V7.5L14.5 2z"
+                />
+                <polyline points="14 2 14 8 20 8" />
+                <path d="M12 18v-6" />
+                <path d="m9 15 3 3 3-3" />
+              </svg>
+              Export Rule
+            </button>
+          </div>
+        </header>
+      </div>
+
+      <iframe
+        name="resultIframe"
+        id="resultIframe"
+        title="resultIframe"
+        class="mt-48 h-[calc(100vh-14.5rem)] w-full overflow-x-hidden overflow-y-auto border-t-2 border-b-2 border-slate-400 dark:border-slate-700"
+      ></iframe>
+
+      <footer class="mx-4 my-2 text-center text-slate-600 dark:text-slate-400">
+        <small>
+          <a
+            href="https://github.com/everywall"
+            class="hover:text-blue-500 dark:hover:text-blue-500 hover:underline underline-offset-2 transition-colors duration-300"
+            >Everywall</a
+          >
+          |
+          <a
+            href="https://github.com/everywall/ladder"
+            class="hover:text-blue-500 dark:hover:text-blue-500 hover:underline underline-offset-2 transition-colors duration-300"
+            >Source</a
+          >
+          | Code Licensed Under GPL v3.0
+        </small>
+      </footer>
+      <div
+        id="modalContainer"
+        tabindex="-1"
+        class="hidden relative z-10"
+        aria-labelledby="modal"
+        role="dialog"
+        aria-modal="true"
+        aria-hidden="true"
+      >
+        <div class="fixed inset-0 bg-slate-500/50 transition-opacity"></div>
+        <div class="fixed inset-0 z-10 w-screen overflow-y-auto">
+          <div
+            class="flex min-h-full items-center justify-center p-4 text-center sm:p-0"
+          >
+            <div
+              id="modal-body"
+              class="relative transform overflow-hidden rounded-lg bg-white dark:bg-slate-900 text-left shadow-xl transition-all sm:my-8 sm:w-full sm:max-w-lg"
+            >
+              <form>
+                <div class="px-4 pb-4 pt-5 sm:p-6 sm:pb-4">
+                  <div class="mt-3 text-left sm:ml-4 sm:mt-0">
+                    <div class="flex w-full justify-between items-center">
+                      <h3 id="modal-title">MODAL HEADER</h3>
+                      <button
+                        type="button"
+                        id="modal-close"
+                        class="ml-2 text-slate-600 dark:text-slate-400 hover:text-slate-400 hover:dark:text-slate-300 bg-transparent text-sm w-6 h-6 inline-flex justify-center items-center"
+                      >
+                        <svg
+                          xmlns="http://www.w3.org/2000/svg"
+                          viewBox="0 0 24 24"
+                          fill="none"
+                          stroke="currentColor"
+                          stroke-width="2"
+                          stroke-linecap="round"
+                          stroke-linejoin="round"
+                        >
+                          <path d="M18 6 6 18" />
+                          <path d="m6 6 12 12" />
+                        </svg>
+                        <span class="sr-only">Close modal</span>
+                      </button>
+                    </div>
+                    <div class="mt-3">
+                      <p id="modal-description">DESCRIPTION</p>
+                    </div>
+                    <div
+                      class="my-2 flex flex-col gap-2 w-full"
+                      id="modal-content"
+                    ></div>
+                  </div>
+                </div>
+                <div
+                  class="bg-slate-200 dark:bg-slate-800 px-4 py-3 sm:flex sm:flex-row-reverse sm:px-6"
+                >
+                  <button
+                    type="button"
+                    id="modal-submit"
+                    class="inline-flex items-center justify-center h-11 px-4 py-2 whitespace-nowrap rounded-md text-sm font-medium text-slate-200 dark:text-slate-900 ring-offset-white dark:ring-offset-slate-900 transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50 bg-slate-800 dark:bg-slate-200 hover:bg-slate-800/90 dark:hover:bg-slate-200/90"
+                  >
+                    Apply modification
+                  </button>
+                </div>
+              </form>
+            </div>
+          </div>
+        </div>
+      </div>
+      <ninja-keys> </ninja-keys>
+    </div>
+  </body>
+</html>

handlers/proxy.go

@@ -1,15 +1,17 @@
 package handlers
 
 import (
-	"ladder/proxychain"
+	rx "github.com/everywall/ladder/proxychain/requestmodifiers"
-	rx "ladder/proxychain/requestmodifiers"
+	tx "github.com/everywall/ladder/proxychain/responsemodifiers"
-	tx "ladder/proxychain/responsemodifiers"
+
+	"github.com/everywall/ladder/proxychain"
+	ruleset_v2 "github.com/everywall/ladder/proxychain/ruleset"
 
 	"github.com/gofiber/fiber/v2"
 )
 
 type ProxyOptions struct {
-	RulesetPath string
+	Ruleset ruleset_v2.IRuleset
 	Verbose bool
 }
 
@@ -32,28 +34,43 @@ func NewProxySiteHandler(opts *ProxyOptions) fiber.Handler {
 			SetDebugLogging(opts.Verbose).
 			SetRequestModifications(
 				//rx.SpoofJA3fingerprint(ja3, "Googlebot"),
-				// rx.MasqueradeAsFacebookBot(),
+				rx.AddCacheBusterQuery(),
-				// rx.MasqueradeAsGoogleBot(),
+				rx.MasqueradeAsGoogleBot(),
-				rx.DeleteOutgoingCookies(),
 				rx.ForwardRequestHeaders(),
-				// rx.SpoofReferrerFromGoogleSearch(),
+				rx.DeleteOutgoingCookies(),
-				rx.SpoofReferrerFromLinkedInPost(),
+				rx.SpoofReferrerFromRedditPost(),
+				//rx.SpoofReferrerFromLinkedInPost(),
 				//rx.RequestWaybackMachine(),
 				//rx.RequestArchiveIs(),
 			).
 			AddResponseModifications(
 				tx.ForwardResponseHeaders(),
+				tx.BlockThirdPartyScripts(),
+				tx.DeleteIncomingCookies(),
+				tx.DeleteLocalStorageData(),
+				tx.DeleteSessionStorageData(),
 				tx.BypassCORS(),
 				tx.BypassContentSecurityPolicy(),
-				// tx.DeleteIncomingCookies(),
 				tx.RewriteHTMLResourceURLs(),
-				tx.PatchTrackerScripts(),
 				tx.PatchDynamicResourceURLs(),
-				tx.BlockElementRemoval(".article-content"),
+				tx.PatchTrackerScripts(),
+				//tx.BlockElementRemoval(".article-content"), // techcrunch
+				//tx.BlockElementRemoval(".available-content"), // substack
 			// tx.SetContentSecurityPolicy("default-src * 'unsafe-inline' 'unsafe-eval' data: blob:;"),
-			).
+			)
-			Execute()
 
-		return proxychain
+		// no options passed in, return early
+		if opts == nil {
+			return proxychain.Execute()
+		}
+
+		// load ruleset
+		rule, exists := opts.Ruleset.GetRule(proxychain.Request.URL)
+		if exists {
+			proxychain.AddOnceRequestModifications(rule.RequestModifications...)
+			proxychain.AddOnceResponseModifications(rule.ResponseModifications...)
+		}
+
+		return proxychain.Execute()
 	}
 }

handlers/raw.go

@@ -1,9 +0,0 @@
-package handlers
-
-import (
-	"github.com/gofiber/fiber/v2"
-)
-
-func Raw(c *fiber.Ctx) error {
-	return nil
-}

handlers/raw.test.go

@@ -1,60 +0,0 @@
-// BEGIN: 7f8d9e6d4b5c
-package handlers
-
-import (
-	"io"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"testing"
-
-	"github.com/gofiber/fiber/v2"
-)
-
-func TestRaw(t *testing.T) {
-	app := fiber.New()
-	app.Get("/raw/*", Raw)
-
-	testCases := []struct {
-		name     string
-		url      string
-		expected string
-	}{
-		{
-			name:     "valid url",
-			url:      "https://www.google.com",
-			expected: "<!doctype html>",
-		},
-		{
-			name:     "invalid url",
-			url:      "invalid-url",
-			expected: "parse invalid-url: invalid URI for request",
-		},
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			req := httptest.NewRequest(http.MethodGet, "/raw/"+tc.url, nil)
-			resp, err := app.Test(req)
-			if err != nil {
-				t.Fatalf("unexpected error: %v", err)
-			}
-			defer resp.Body.Close()
-
-			if resp.StatusCode != http.StatusOK {
-				t.Errorf("expected status OK; got %v", resp.Status)
-			}
-
-			body, err := io.ReadAll(resp.Body)
-			if err != nil {
-				t.Fatalf("unexpected error: %v", err)
-			}
-
-			if !strings.Contains(string(body), tc.expected) {
-				t.Errorf("expected body to contain %q; got %q", tc.expected, string(body))
-			}
-		})
-	}
-}
-
-// END: 7f8d9e6d4b5c

handlers/ruleset.go

@@ -1,9 +0,0 @@
-package handlers
-
-import (
-	"github.com/gofiber/fiber/v2"
-)
-
-func Ruleset(c *fiber.Ctx) error {
-	return nil
-}

handlers/script.go

@@ -9,15 +9,25 @@ import (
 //go:embed script.js
 var scriptData embed.FS
 
-func Script(c *fiber.Ctx) error {
+//go:embed playground-script.js
+var playgroundScriptData embed.FS
 
+func Script(c *fiber.Ctx) error {
+	if c.Path() == "/script.js" {
 		scriptData, err := scriptData.ReadFile("script.js")
 		if err != nil {
 			return c.Status(fiber.StatusInternalServerError).SendString("Internal Server Error")
 		}
-
 		c.Set("Content-Type", "text/javascript")
-
 		return c.Send(scriptData)
-
+	}
+	if c.Path() == "/playground-script.js" {
+		playgroundScriptData, err := playgroundScriptData.ReadFile("playground-script.js")
+		if err != nil {
+			return c.Status(fiber.StatusInternalServerError).SendString("Internal Server Error")
+		}
+		c.Set("Content-Type", "text/javascript")
+		return c.Send(playgroundScriptData)
+	}
+	return nil
 }

handlers/script.js

@@ -1,8 +1,12 @@
 const labels = document.querySelectorAll("label");
 const inputs = document.querySelectorAll('input[type="radio"]');
 const mainElement = document.querySelector("main");
+const inputField = document.getElementById("inputField");
+const clearButton = document.getElementById("clearButton");
 
-const handleDOMContentLoaded = () => {
+window.addEventListener("DOMContentLoaded", handleDOMContentLoaded);
+
+function handleDOMContentLoaded() {
   handleFontChange();
   handleFontSizeChange();
   inputs.forEach((input) => {
@@ -12,7 +16,36 @@ const handleDOMContentLoaded = () => {
     }
   });
   window.removeEventListener("DOMContentLoaded", handleDOMContentLoaded);
-};
+}
+
+function clearInput() {
+  inputField.value = "";
+  clearButton.style.display = "none";
+  inputField.focus();
+}
+
+if (inputField !== null && clearButton !== null) {
+  inputField.addEventListener("input", () => {
+    const clearButton = document.getElementById("clearButton");
+    if (clearButton !== null) {
+      if (inputField.value.trim().length > 0) {
+        clearButton.style.display = "block";
+      } else {
+        clearButton.style.display = "none";
+      }
+    }
+  });
+
+  inputField.addEventListener("keydown", (event) => {
+    if (event.code === "Escape") {
+      clearInput();
+    }
+  });
+
+  clearButton.addEventListener("click", () => {
+    clearInput();
+  });
+}
 
 function focusable_children(node) {
   const nodes = Array.from(
@@ -78,11 +111,13 @@ function trap(node) {
 
 const toggleDropdown = () => {
   const dropdown = document.getElementById("dropdown");
+  const dropdown_button = dropdown.querySelector("button");
   const dropdown_panel = document.getElementById("dropdown_panel");
   const focusTrap = trap(dropdown);
 
   const closeDropdown = () => {
     dropdown_panel.classList.add("hidden");
+    dropdown_button.setAttribute("aria-expanded", "false");
     focusTrap.destroy();
     dropdown.removeEventListener("keydown", handleEscapeKey);
     document.removeEventListener("click", handleClickOutside);
@@ -95,7 +130,7 @@ const toggleDropdown = () => {
   };
 
   const handleClickOutside = (e) => {
-    if (!dropdown.contains(e.target)) {
+    if (dropdown !== null && !dropdown.contains(e.target)) {
       closeDropdown();
     }
   };
@@ -142,6 +177,7 @@ const toggleDropdown = () => {
 
   if (dropdown_panel.classList.contains("hidden")) {
     dropdown_panel.classList.remove("hidden");
+    dropdown_button.setAttribute("aria-expanded", "true");
     dropdown.addEventListener("keydown", handleEscapeKey);
     inputs.forEach((input) => {
       input.addEventListener("change", handleInputChange);
@@ -292,5 +328,3 @@ const handleFontSizeChange = () => {
     changeFontSize(node, classes);
   });
 };
-
-window.addEventListener("DOMContentLoaded", handleDOMContentLoaded);

internal/cli/ruleset_merge.go

@@ -5,9 +5,7 @@ import (
 	"io"
 	"os"
 
-	"ladder/pkg/ruleset"
+	ruleset_v2 "github.com/everywall/ladder/proxychain/ruleset"
-
-	"golang.org/x/term"
 )
 
 // HandleRulesetMerge merges a set of ruleset files, specified by the rulesetPath or RULESET env variable, into either YAML or Gzip format.
@@ -21,7 +19,7 @@ import (
 //
 // Returns:
 // - An error if the ruleset loading or merging process fails, otherwise nil.
-func HandleRulesetMerge(rulesetPath string, mergeRulesets bool, useGzip bool, output *os.File) error {
+func HandleRulesetMerge(rulesetPath string, mergeRulesets bool, output *os.File) error {
 	if !mergeRulesets {
 		return nil
 	}
@@ -35,55 +33,15 @@ func HandleRulesetMerge(rulesetPath string, mergeRulesets bool, useGzip bool, ou
 		os.Exit(1)
 	}
 
-	rs, err := ruleset.NewRuleset(rulesetPath)
+	rs, err := ruleset_v2.NewRuleset(rulesetPath)
 	if err != nil {
 		fmt.Println(err)
 		os.Exit(1)
 	}
 
-	if useGzip {
-		return gzipMerge(rs, output)
-	}
-
 	return yamlMerge(rs, output)
 }
 
-// gzipMerge takes a RuleSet and an optional output file path pointer. It compresses the RuleSet into Gzip format.
-// If the output file path is provided, the compressed data is written to this file. Otherwise, it prints a warning
-// and outputs the binary data to stdout
-//
-// Parameters:
-// - rs: The ruleset.RuleSet to be compressed.
-// - output: The output for the gzip data. If nil, stdout will be used.
-//
-// Returns:
-// - An error if compression or file writing fails, otherwise nil.
-func gzipMerge(rs ruleset.RuleSet, output io.Writer) error {
-	gzip, err := rs.GzipYaml()
-	if err != nil {
-		return err
-	}
-
-	if output != nil {
-		_, err = io.Copy(output, gzip)
-		if err != nil {
-			return err
-		}
-	}
-
-	if term.IsTerminal(int(os.Stdout.Fd())) {
-		println("warning: binary output can mess up your terminal. Use '--merge-rulesets-output <ruleset.gz>' or pipe it to a file.")
-		os.Exit(1)
-	}
-
-	_, err = io.Copy(os.Stdout, gzip)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
 // yamlMerge takes a RuleSet and an optional output file path pointer. It converts the RuleSet into YAML format.
 // If the output file path is provided, the YAML data is written to this file. If not, the YAML data is printed to stdout.
 //
@@ -93,8 +51,8 @@ func gzipMerge(rs ruleset.RuleSet, output io.Writer) error {
 //
 // Returns:
 // - An error if YAML conversion or file writing fails, otherwise nil.
-func yamlMerge(rs ruleset.RuleSet, output io.Writer) error {
+func yamlMerge(rs ruleset_v2.Ruleset, output io.Writer) error {
-	yaml, err := rs.Yaml()
+	yaml, err := rs.YAML()
 	if err != nil {
 		return err
 	}

pkg/ruleset/ruleset.go

@@ -1,347 +0,0 @@
-package ruleset
-
-import (
-	"compress/gzip"
-	"errors"
-	"fmt"
-	"io"
-	"log"
-	"net/http"
-	"os"
-	"path/filepath"
-	"regexp"
-	"strings"
-
-	"gopkg.in/yaml.v3"
-)
-
-type Regex struct {
-	Match   string `yaml:"match"`
-	Replace string `yaml:"replace"`
-}
-type KV struct {
-	Key   string `yaml:"key"`
-	Value string `yaml:"value"`
-}
-
-type RuleSet []Rule
-
-type Rule struct {
-	Domain  string   `yaml:"domain,omitempty"`
-	Domains []string `yaml:"domains,omitempty"`
-	Paths   []string `yaml:"paths,omitempty"`
-	Headers struct {
-		UserAgent     string `yaml:"user-agent,omitempty"`
-		XForwardedFor string `yaml:"x-forwarded-for,omitempty"`
-		Referer       string `yaml:"referer,omitempty"`
-		Cookie        string `yaml:"cookie,omitempty"`
-		CSP           string `yaml:"content-security-policy,omitempty"`
-	} `yaml:"headers,omitempty"`
-	GoogleCache bool    `yaml:"googleCache,omitempty"`
-	RegexRules  []Regex `yaml:"regexRules,omitempty"`
-
-	URLMods struct {
-		Domain []Regex `yaml:"domain,omitempty"`
-		Path   []Regex `yaml:"path,omitempty"`
-		Query  []KV    `yaml:"query,omitempty"`
-	} `yaml:"urlMods,omitempty"`
-
-	Injections []struct {
-		Position string `yaml:"position,omitempty"`
-		Append   string `yaml:"append,omitempty"`
-		Prepend  string `yaml:"prepend,omitempty"`
-		Replace  string `yaml:"replace,omitempty"`
-	} `yaml:"injections,omitempty"`
-}
-
-var remoteRegex = regexp.MustCompile(`^https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()!@:%_\+.~#?&\/\/=]*)`)
-
-// NewRulesetFromEnv creates a new RuleSet based on the RULESET environment variable.
-// It logs a warning and returns an empty RuleSet if the RULESET environment variable is not set.
-// If the RULESET is set but the rules cannot be loaded, it panics.
-func NewRulesetFromEnv() RuleSet {
-	rulesPath, ok := os.LookupEnv("RULESET")
-	if !ok {
-		log.Printf("WARN: No ruleset specified. Set the `RULESET` environment variable to load one for a better success rate.")
-		return RuleSet{}
-	}
-
-	ruleSet, err := NewRuleset(rulesPath)
-	if err != nil {
-		log.Println(err)
-	}
-
-	return ruleSet
-}
-
-// NewRuleset loads a RuleSet from a given string of rule paths, separated by semicolons.
-// It supports loading rules from both local file paths and remote URLs.
-// Returns a RuleSet and an error if any issues occur during loading.
-func NewRuleset(rulePaths string) (RuleSet, error) {
-	var ruleSet RuleSet
-
-	var errs []error
-
-	rp := strings.Split(rulePaths, ";")
-	for _, rule := range rp {
-		var err error
-
-		rulePath := strings.Trim(rule, " ")
-		isRemote := remoteRegex.MatchString(rulePath)
-
-		if isRemote {
-			err = ruleSet.loadRulesFromRemoteFile(rulePath)
-		} else {
-			err = ruleSet.loadRulesFromLocalDir(rulePath)
-		}
-
-		if err != nil {
-			e := fmt.Errorf("WARN: failed to load ruleset from '%s'", rulePath)
-			errs = append(errs, errors.Join(e, err))
-
-			continue
-		}
-	}
-
-	if len(errs) != 0 {
-		e := fmt.Errorf("WARN: failed to load %d rulesets", len(rp))
-		errs = append(errs, e)
-
-		// panic if the user specified a local ruleset, but it wasn't found on disk
-		// don't fail silently
-		for _, err := range errs {
-			if errors.Is(os.ErrNotExist, err) {
-				e := fmt.Errorf("PANIC: ruleset '%s' not found", err)
-				panic(errors.Join(e, err))
-			}
-		}
-
-		// else, bubble up any errors, such as syntax or remote host issues
-		return ruleSet, errors.Join(errs...)
-	}
-
-	ruleSet.PrintStats()
-
-	return ruleSet, nil
-}
-
-// ================== RULESET loading logic ===================================
-
-// loadRulesFromLocalDir loads rules from a local directory specified by the path.
-// It walks through the directory, loading rules from YAML files.
-// Returns an error if the directory cannot be accessed
-// If there is an issue loading any file, it will be skipped
-func (rs *RuleSet) loadRulesFromLocalDir(path string) error {
-	_, err := os.Stat(path)
-	if err != nil {
-		return err
-	}
-
-	yamlRegex := regexp.MustCompile(`.*\.ya?ml`)
-
-	err = filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-		if info.IsDir() {
-			return nil
-		}
-		if isYaml := yamlRegex.MatchString(path); !isYaml {
-			return nil
-		}
-
-		err = rs.loadRulesFromLocalFile(path)
-		if err != nil {
-			log.Printf("WARN: failed to load directory ruleset '%s': %s, skipping", path, err)
-			return nil
-		}
-
-		log.Printf("INFO: loaded ruleset %s\n", path)
-
-		return nil
-	})
-
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// loadRulesFromLocalFile loads rules from a local YAML file specified by the path.
-// Returns an error if the file cannot be read or if there's a syntax error in the YAML.
-func (rs *RuleSet) loadRulesFromLocalFile(path string) error {
-	yamlFile, err := os.ReadFile(path)
-	if err != nil {
-		e := fmt.Errorf("failed to read rules from local file: '%s'", path)
-		return errors.Join(e, err)
-	}
-
-	var r RuleSet
-	err = yaml.Unmarshal(yamlFile, &r)
-
-	if err != nil {
-		e := fmt.Errorf("failed to load rules from local file, possible syntax error in '%s'", path)
-		ee := errors.Join(e, err)
-
-		if _, ok := os.LookupEnv("DEBUG"); ok {
-			debugPrintRule(string(yamlFile), ee)
-		}
-
-		return ee
-	}
-
-	*rs = append(*rs, r...)
-
-	return nil
-}
-
-// loadRulesFromRemoteFile loads rules from a remote URL.
-// It supports plain and gzip compressed content.
-// Returns an error if there's an issue accessing the URL or if there's a syntax error in the YAML.
-func (rs *RuleSet) loadRulesFromRemoteFile(rulesURL string) error {
-	var r RuleSet
-
-	resp, err := http.Get(rulesURL)
-	if err != nil {
-		e := fmt.Errorf("failed to load rules from remote url '%s'", rulesURL)
-		return errors.Join(e, err)
-	}
-
-	defer resp.Body.Close()
-
-	if resp.StatusCode >= 400 {
-		e := fmt.Errorf("failed to load rules from remote url (%s) on '%s'", resp.Status, rulesURL)
-		return errors.Join(e, err)
-	}
-
-	var reader io.Reader
-
-	isGzip := strings.HasSuffix(rulesURL, ".gz") || strings.HasSuffix(rulesURL, ".gzip") || resp.Header.Get("content-encoding") == "gzip"
-
-	if isGzip {
-		reader, err = gzip.NewReader(resp.Body)
-
-		if err != nil {
-			return fmt.Errorf("failed to create gzip reader for URL '%s' with status code '%s': %w", rulesURL, resp.Status, err)
-		}
-	} else {
-		reader = resp.Body
-	}
-
-	err = yaml.NewDecoder(reader).Decode(&r)
-
-	if err != nil {
-		e := fmt.Errorf("failed to load rules from remote url '%s' with status code '%s' and possible syntax error", rulesURL, resp.Status)
-		ee := errors.Join(e, err)
-
-		return ee
-	}
-
-	*rs = append(*rs, r...)
-
-	return nil
-}
-
-// ================= utility methods ==========================
-
-// Yaml returns the ruleset as a Yaml string
-func (rs *RuleSet) Yaml() (string, error) {
-	y, err := yaml.Marshal(rs)
-	if err != nil {
-		return "", err
-	}
-
-	return string(y), nil
-}
-
-// GzipYaml returns an io.Reader that streams the Gzip-compressed YAML representation of the RuleSet.
-func (rs *RuleSet) GzipYaml() (io.Reader, error) {
-	pr, pw := io.Pipe()
-
-	go func() {
-		defer pw.Close()
-
-		gw := gzip.NewWriter(pw)
-		defer gw.Close()
-
-		if err := yaml.NewEncoder(gw).Encode(rs); err != nil {
-			gw.Close() // Ensure to close the gzip writer
-			pw.CloseWithError(err)
-			return
-		}
-	}()
-
-	return pr, nil
-}
-
-// Domains extracts and returns a slice of all domains present in the RuleSet.
-func (rs *RuleSet) Domains() []string {
-	var domains []string
-	for _, rule := range *rs {
-		domains = append(domains, rule.Domain)
-		domains = append(domains, rule.Domains...)
-	}
-	return domains
-}
-
-// DomainCount returns the count of unique domains present in the RuleSet.
-func (rs *RuleSet) DomainCount() int {
-	return len(rs.Domains())
-}
-
-// Count returns the total number of rules in the RuleSet.
-func (rs *RuleSet) Count() int {
-	return len(*rs)
-}
-
-// PrintStats logs the number of rules and domains loaded in the RuleSet.
-func (rs *RuleSet) PrintStats() {
-	log.Printf("INFO: Loaded %d rules for %d domains\n", rs.Count(), rs.DomainCount())
-}
-
-// debugPrintRule is a utility function for printing a rule and associated error for debugging purposes.
-func debugPrintRule(rule string, err error) {
-	fmt.Println("------------------------------ BEGIN DEBUG RULESET -----------------------------")
-	fmt.Printf("%s\n", err.Error())
-	fmt.Println("--------------------------------------------------------------------------------")
-	fmt.Println(rule)
-	fmt.Println("------------------------------ END DEBUG RULESET -------------------------------")
-}
-
-// ======================= RuleSetMap implementation =================================================
-
-// RuleSetMap: A map with domain names as keys and pointers to the corresponding Rules as values.
-// This type is used to efficiently access rules based on domain names.
-type RuleSetMap map[string]*Rule
-
-// ToMap converts a RuleSet into a RuleSetMap. It transforms each Rule in the RuleSet
-// into a map entry where the key is the Rule's domain (lowercase)
-// and the value is a pointer to the Rule. This method is used to
-// efficiently access rules based on domain names.
-// The RuleSetMap may be accessed with or without a "www." prefix in the domain.
-func (rs *RuleSet) ToMap() RuleSetMap {
-	rsm := make(RuleSetMap)
-
-	addMapEntry := func(d string, rule *Rule) {
-		d = strings.ToLower(d)
-		rsm[d] = rule
-		if strings.HasPrefix(d, "www.") {
-			d = strings.TrimPrefix(d, "www.")
-			rsm[d] = rule
-		} else {
-			d = fmt.Sprintf("www.%s", d)
-			rsm[d] = rule
-		}
-	}
-
-	for i, rule := range *rs {
-		rulePtr := &(*rs)[i]
-		addMapEntry(rule.Domain, rulePtr)
-		for _, domain := range rule.Domains {
-			addMapEntry(domain, rulePtr)
-		}
-	}
-
-	return rsm
-}

pkg/ruleset/ruleset_test.go

@@ -1,225 +0,0 @@
-package ruleset
-
-import (
-	"os"
-	"path/filepath"
-	"testing"
-	"time"
-
-	"github.com/gofiber/fiber/v2"
-	"github.com/stretchr/testify/assert"
-)
-
-var (
-	validYAML = `
-- domain: example.com
-  regexRules:
-    - match: "^http:"
-      replace: "https:"`
-
-	invalidYAML = `
-- domain: [thisIsATestYamlThatIsMeantToFail.example]
-  regexRules:
-    - match: "^http:"
-      replace: "https:"
-    - match: "[incomplete"`
-)
-
-func TestLoadRulesFromRemoteFile(t *testing.T) {
-	app := fiber.New()
-	defer app.Shutdown()
-
-	app.Get("/valid-config.yml", func(c *fiber.Ctx) error {
-		c.SendString(validYAML)
-		return nil
-	})
-
-	app.Get("/invalid-config.yml", func(c *fiber.Ctx) error {
-		c.SendString(invalidYAML)
-		return nil
-	})
-
-	app.Get("/valid-config.gz", func(c *fiber.Ctx) error {
-		c.Set("Content-Type", "application/octet-stream")
-
-		rs, err := loadRuleFromString(validYAML)
-		if err != nil {
-			t.Errorf("failed to load valid yaml from string: %s", err.Error())
-		}
-
-		s, err := rs.GzipYaml()
-		if err != nil {
-			t.Errorf("failed to load gzip serialize yaml: %s", err.Error())
-		}
-
-		err = c.SendStream(s)
-		if err != nil {
-			t.Errorf("failed to stream gzip serialized yaml: %s", err.Error())
-		}
-		return nil
-	})
-
-	// Start the server in a goroutine
-	go func() {
-		if err := app.Listen("127.0.0.1:9999"); err != nil {
-			t.Errorf("Server failed to start: %s", err.Error())
-		}
-	}()
-
-	// Wait for the server to start
-	time.Sleep(time.Second * 1)
-
-	rs, err := NewRuleset("http://127.0.0.1:9999/valid-config.yml")
-	if err != nil {
-		t.Errorf("failed to load plaintext ruleset from http server: %s", err.Error())
-	}
-
-	assert.Equal(t, rs[0].Domain, "example.com")
-
-	rs, err = NewRuleset("http://127.0.0.1:9999/valid-config.gz")
-	if err != nil {
-		t.Errorf("failed to load gzipped ruleset from http server: %s", err.Error())
-	}
-
-	assert.Equal(t, rs[0].Domain, "example.com")
-
-	os.Setenv("RULESET", "http://127.0.0.1:9999/valid-config.gz")
-
-	rs = NewRulesetFromEnv()
-	if !assert.Equal(t, rs[0].Domain, "example.com") {
-		t.Error("expected no errors loading ruleset from gzip url using environment variable, but got one")
-	}
-}
-
-func loadRuleFromString(yaml string) (RuleSet, error) {
-	// Create a temporary file and load it
-	tmpFile, _ := os.CreateTemp("", "ruleset*.yaml")
-
-	defer os.Remove(tmpFile.Name())
-
-	tmpFile.WriteString(yaml)
-
-	rs := RuleSet{}
-	err := rs.loadRulesFromLocalFile(tmpFile.Name())
-
-	return rs, err
-}
-
-// TestLoadRulesFromLocalFile tests the loading of rules from a local YAML file.
-func TestLoadRulesFromLocalFile(t *testing.T) {
-	rs, err := loadRuleFromString(validYAML)
-	if err != nil {
-		t.Errorf("Failed to load rules from valid YAML: %s", err)
-	}
-
-	assert.Equal(t, rs[0].Domain, "example.com")
-	assert.Equal(t, rs[0].RegexRules[0].Match, "^http:")
-	assert.Equal(t, rs[0].RegexRules[0].Replace, "https:")
-
-	_, err = loadRuleFromString(invalidYAML)
-	if err == nil {
-		t.Errorf("Expected an error when loading invalid YAML, but got none")
-	}
-}
-
-// TestLoadRulesFromLocalDir tests the loading of rules from a local nested directory full of yaml rulesets
-func TestLoadRulesFromLocalDir(t *testing.T) {
-	// Create a temporary directory
-	baseDir, err := os.MkdirTemp("", "ruleset_test")
-	if err != nil {
-		t.Fatalf("Failed to create temporary directory: %s", err)
-	}
-
-	defer os.RemoveAll(baseDir)
-
-	// Create a nested subdirectory
-	nestedDir := filepath.Join(baseDir, "nested")
-	err = os.Mkdir(nestedDir, 0o755)
-
-	if err != nil {
-		t.Fatalf("Failed to create nested directory: %s", err)
-	}
-
-	// Create a nested subdirectory
-	nestedTwiceDir := filepath.Join(nestedDir, "nestedTwice")
-	err = os.Mkdir(nestedTwiceDir, 0o755)
-	if err != nil {
-		t.Fatalf("Failed to create twice-nested directory: %s", err)
-	}
-
-	testCases := []string{"test.yaml", "test2.yaml", "test-3.yaml", "test 4.yaml", "1987.test.yaml.yml", "foobar.example.com.yaml", "foobar.com.yml"}
-	for _, fileName := range testCases {
-		filePath := filepath.Join(nestedDir, "2x-"+fileName)
-		os.WriteFile(filePath, []byte(validYAML), 0o644)
-
-		filePath = filepath.Join(nestedDir, fileName)
-		os.WriteFile(filePath, []byte(validYAML), 0o644)
-
-		filePath = filepath.Join(baseDir, "base-"+fileName)
-		os.WriteFile(filePath, []byte(validYAML), 0o644)
-	}
-
-	rs := RuleSet{}
-	err = rs.loadRulesFromLocalDir(baseDir)
-
-	assert.NoError(t, err)
-	assert.Equal(t, rs.Count(), len(testCases)*3)
-
-	for _, rule := range rs {
-		assert.Equal(t, rule.Domain, "example.com")
-		assert.Equal(t, rule.RegexRules[0].Match, "^http:")
-		assert.Equal(t, rule.RegexRules[0].Replace, "https:")
-	}
-}
-
-func TestToMap(t *testing.T) {
-	// Prepare a ruleset with multiple rules, including "www." prefixed domains
-	rules := RuleSet{
-		{
-			Domain:     "Example.com",
-			RegexRules: []Regex{{Match: "match1", Replace: "replace1"}},
-		},
-		{
-			Domain:     "www.AnotherExample.com",
-			RegexRules: []Regex{{Match: "match2", Replace: "replace2"}},
-		},
-		{
-			Domain:     "www.foo.bAr.baz.bOol.quX.com",
-			RegexRules: []Regex{{Match: "match3", Replace: "replace3"}},
-		},
-	}
-
-	// Convert to RuleSetMap
-	rsm := rules.ToMap()
-
-	// Test for correct number of entries
-	if len(rsm) != 6 {
-		t.Errorf("Expected 6 entries in RuleSetMap, got %d", len(rsm))
-	}
-
-	// Test for correct mapping
-	testDomains := []struct {
-		domain        string
-		expectedMatch string
-	}{
-		{"example.com", "match1"},
-		{"www.example.com", "match1"},
-		{"anotherexample.com", "match2"},
-		{"www.anotherexample.com", "match2"},
-		{"foo.bar.baz.bool.qux.com", "match3"},
-		{"no.ruleset.domain.com", ""},
-	}
-
-	for _, test := range testDomains {
-		if test.domain == "no.ruleset.domain.com" {
-			assert.Empty(t, test.expectedMatch)
-			continue
-		}
-		rule, exists := rsm[test.domain]
-		if !exists {
-			t.Errorf("Expected domain %s to exist in RuleSetMap", test.domain)
-		} else if rule.RegexRules[0].Match != test.expectedMatch {
-			t.Errorf("Expected match for %s to be %s, got %s", test.domain, test.expectedMatch, rule.RegexRules[0].Match)
-		}
-	}
-}

proxychain/codegen/codegen.go

@@ -70,8 +70,8 @@ package ruleset_v2
 // for use in proxychains.
 
 import (
-	"ladder/proxychain"
+	"github.com/everywall/ladder/proxychain"
-	tx "ladder/proxychain/responsemodifiers"
+	tx "github.com/everywall/ladder/proxychain/responsemodifiers"
 )
 
 type ResponseModifierFactory func(params ...string) proxychain.ResponseModification
@@ -142,8 +142,8 @@ package ruleset_v2
 // for use in proxychains.
 
 import (
-	"ladder/proxychain"
+	"github.com/everywall/ladder/proxychain"
-	rx "ladder/proxychain/requestmodifiers"
+	rx "github.com/everywall/ladder/proxychain/requestmodifiers"
 )
 
 type RequestModifierFactory func(params ...string) proxychain.RequestModification

proxychain/proxychain.go

@@ -8,17 +8,9 @@ import (
 	"net/url"
 	"strings"
 
-	//"time"
-
-	//"net/http"
-	//"github.com/Danny-Dasilva/CycleTLS/cycletls"
-	//http "github.com/Danny-Dasilva/fhttp"
 	http "github.com/bogdanfinn/fhttp"
 	tls_client "github.com/bogdanfinn/tls-client"
-
+	profiles "github.com/bogdanfinn/tls-client/profiles"
-	//"github.com/bogdanfinn/tls-client/profiles"
-
-	"ladder/pkg/ruleset"
 
 	"github.com/gofiber/fiber/v2"
 )
@@ -100,7 +92,6 @@ type ProxyChain struct {
 	onceRequestModifications  []RequestModification
 	onceResponseModifications []ResponseModification
 	responseModifications     []ResponseModification
-	Ruleset                   *ruleset.RuleSet
 	debugMode                 bool
 	abortErr                  error
 	APIPrefix                 string
@@ -177,13 +168,6 @@ func (chain *ProxyChain) WithAPIPath(path string) *ProxyChain {
 	return chain
 }
 
-// Adds a ruleset to ProxyChain
-func (chain *ProxyChain) AddRuleset(rs *ruleset.RuleSet) *ProxyChain {
-	chain.Ruleset = rs
-	// TODO: add _applyRuleset method
-	return chain
-}
-
 func (chain *ProxyChain) _initializeRequest() (*http.Request, error) {
 	if chain.Context == nil {
 		chain.abortErr = chain.abort(errors.New("no context set"))
@@ -210,15 +194,6 @@ func (chain *ProxyChain) _initializeRequest() (*http.Request, error) {
 		return nil, fmt.Errorf("unsupported request method from client: '%s'", chain.Context.Method())
 	}
 
-	/*
-		// copy client request headers to upstream request headers
-		forwardHeaders := func(key []byte, val []byte) {
-			req.Header.Set(string(key), string(val))
-		}
-		clientHeaders := &chain.Context.Request().Header
-		clientHeaders.VisitAll(forwardHeaders)
-	*/
-
 	return req, nil
 }
 
@@ -260,14 +235,54 @@ func preventRecursiveProxyRequest(urlQuery *url.URL, baseProxyURL string) *url.U
 	return preventRecursiveProxyRequest(fixedURL, baseProxyURL)
 }
 
-// extractURL extracts a URL from the request ctx. If the URL in the request
+// extractURL extracts a URL from the request ctx
-// is a relative path, it reconstructs the full URL using the referer header.
 func (chain *ProxyChain) extractURL() (*url.URL, error) {
-	reqURL := chain.Context.Params("*")
+	isLocal := strings.HasPrefix(chain.Context.BaseURL(), "http://localhost") || strings.HasPrefix(chain.Context.BaseURL(), "http://127.0.0.1")
+	isReqPath := strings.HasPrefix(chain.Context.Path(), "/http")
+	isAPI := strings.HasPrefix(chain.Context.Path(), "/api")
+	isOutline := strings.HasPrefix(chain.Context.Path(), "/outline")
 
-	fmt.Println("XXXXXXXXXXXXXXXX")
+	if isLocal || isReqPath || isAPI || isOutline {
-	fmt.Println(reqURL)
+		return chain.extractURLFromPath()
-	fmt.Println(chain.APIPrefix)
+	}
+
+	u, err := url.Parse(chain.Context.BaseURL())
+	if err != nil {
+		return &url.URL{}, err
+	}
+	parts := strings.Split(u.Hostname(), ".")
+	if len(parts) < 2 {
+		fmt.Println("path")
+		return chain.extractURLFromPath()
+	}
+
+	return chain.extractURLFromSubdomain()
+}
+
+// extractURLFromPath extracts a URL from the request ctx if subdomains are used.
+func (chain *ProxyChain) extractURLFromSubdomain() (*url.URL, error) {
+	u, err := url.Parse(chain.Context.BaseURL())
+	if err != nil {
+		return &url.URL{}, err
+	}
+	parts := strings.Split(u.Hostname(), ".")
+	if len(parts) < 2 {
+		// no subdomain set, fallback to path extraction
+		//panic("asdf")
+		return chain.extractURLFromPath()
+	}
+	subdomain := strings.Join(parts[:len(parts)-2], ".")
+	subURL := subdomain
+	subURL = strings.ReplaceAll(subURL, "--", "|")
+	subURL = strings.ReplaceAll(subURL, "-", ".")
+	subURL = strings.ReplaceAll(subURL, "|", "-")
+	return url.Parse(fmt.Sprintf("https://%s/%s", subURL, u.Path))
+}
+
+// extractURLFromPath extracts a URL from the request ctx. If the URL in the request
+// is a relative path, it reconstructs the full URL using the referer header.
+func (chain *ProxyChain) extractURLFromPath() (*url.URL, error) {
+	reqURL := chain.Context.Params("*")
 
 	reqURL = strings.TrimPrefix(reqURL, chain.APIPrefix)
 
@@ -330,9 +345,10 @@ func (chain *ProxyChain) SetFiberCtx(ctx *fiber.Ctx) *ProxyChain {
 	url, err := chain.extractURL()
 	if err != nil {
 		chain.abortErr = chain.abort(err)
-	}
+	} else {
 		chain.Request.URL = url
 		fmt.Printf("extracted URL: %s\n", chain.Request.URL)
+	}
 
 	return chain
 }
@@ -341,7 +357,7 @@ func (chain *ProxyChain) validateCtxIsSet() error {
 	if chain.Context != nil {
 		return nil
 	}
-	err := errors.New("proxyChain was called without setting a fiber Ctx. Use ProxyChain.SetCtx()")
+	err := errors.New("proxyChain was called without setting a fiber Ctx. Use ProxyChain.SetFiberCtx()")
 	chain.abortErr = chain.abort(err)
 	return chain.abortErr
 }
@@ -377,8 +393,13 @@ func (chain *ProxyChain) abort(err error) error {
 	// defer chain._reset()
 	chain.abortErr = err
 	chain.Context.Response().SetStatusCode(500)
-	e := fmt.Errorf("ProxyChain error for '%s': %s", chain.Request.URL.String(), err.Error())
+	var e error
-	chain.Context.SendString(e.Error())
+	if chain.Request.URL != nil {
+		e = fmt.Errorf("ProxyChain error for '%s': %s", chain.Request.URL.String(), err.Error())
+	} else {
+		e = fmt.Errorf("ProxyChain error: '%s'", err.Error())
+	}
+	// chain.Context.SendString(e.Error()) // <- RenderErrorPage middleware to render error
 	log.Println(e.Error())
 	return e
 }
@@ -400,8 +421,8 @@ func NewProxyChain() *ProxyChain {
 
 	options := []tls_client.HttpClientOption{
 		tls_client.WithTimeoutSeconds(20),
-		tls_client.WithRandomTLSExtensionOrder(),
+		//tls_client.WithRandomTLSExtensionOrder(),
-		// tls_client.WithClientProfile(profiles.Chrome_117),
+		tls_client.WithClientProfile(profiles.Chrome_117),
 		// tls_client.WithNotFollowRedirects(),
 		// tls_client.WithCookieJar(jar), // create cookieJar instance and pass it as argument
 	}
@@ -505,10 +526,14 @@ func (chain *ProxyChain) Execute() error {
 		return errors.New("no context set")
 	}
 
+	// TODO: this seems broken
 	// in case api user did not set or forward content-type, we do it for them
-	if chain.Context.Get("content-type") == "" {
+	/*
+		ct := string(chain.Context.Response().Header.Peek("content-type"))
+		if ct == "" {
 			chain.Context.Set("content-type", chain.Response.Header.Get("content-type"))
 		}
+	*/
 
 	// Return request back to client
 	return chain.Context.SendStream(body)

proxychain/requestmodifiers/add_cache_buster_query.go

@@ -0,0 +1,29 @@
+package requestmodifiers
+
+import (
+	"math/rand"
+
+	"github.com/everywall/ladder/proxychain"
+)
+
+// AddCacheBusterQuery modifies query params to add a random parameter key
+// In order to get the upstream network stack to serve a fresh copy of the page.
+func AddCacheBusterQuery() proxychain.RequestModification {
+	return func(chain *proxychain.ProxyChain) error {
+		chain.AddOnceRequestModifications(
+			ModifyQueryParams("ord", randomString(15)),
+		)
+
+		return nil
+	}
+}
+
+func randomString(length int) string {
+	const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789."
+
+	b := make([]byte, length)
+	for i := range b {
+		b[i] = charset[rand.Intn(len(charset))]
+	}
+	return string(b)
+}

proxychain/requestmodifiers/forward_request_headers.go

@@ -3,7 +3,7 @@ package requestmodifiers
 import (
 	"strings"
 	//"fmt"
-	"ladder/proxychain"
+	"github.com/everywall/ladder/proxychain"
 )
 
 var forwardBlacklist map[string]bool

proxychain/requestmodifiers/masquerade_as_trusted_bot.go

@@ -1,8 +1,9 @@
 package requestmodifiers
 
 import (
-	"ladder/proxychain"
+	"github.com/everywall/ladder/proxychain/requestmodifiers/bot"
-	"ladder/proxychain/requestmodifiers/bot"
+
+	"github.com/everywall/ladder/proxychain"
 )
 
 // MasqueradeAsGoogleBot modifies user agent and x-forwarded for

proxychain/requestmodifiers/modify_domain_with_regex.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"regexp"
 
-	"ladder/proxychain"
+	"github.com/everywall/ladder/proxychain"
 )
 
 func ModifyDomainWithRegex(matchRegex string, replacement string) proxychain.RequestModification {

proxychain/requestmodifiers/modify_outgoing_cookies.go

@@ -5,7 +5,7 @@ import (
 	//http "github.com/Danny-Dasilva/fhttp"
 	http "github.com/bogdanfinn/fhttp"
 
-	"ladder/proxychain"
+	"github.com/everywall/ladder/proxychain"
 )
 
 // SetOutgoingCookie modifes a specific cookie name

proxychain/requestmodifiers/modify_path_with_regex.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"regexp"
 
-	"ladder/proxychain"
+	"github.com/everywall/ladder/proxychain"
 )
 
 func ModifyPathWithRegex(matchRegex string, replacement string) proxychain.RequestModification {

proxychain/requestmodifiers/modify_query_params.go

@@ -1,9 +1,10 @@
 package requestmodifiers
 
 import (
+	//"fmt"
 	"net/url"
 
-	"ladder/proxychain"
+	"github.com/everywall/ladder/proxychain"
 )
 
 // ModifyQueryParams replaces query parameter values in URL's query params in a ProxyChain's URL.
@@ -12,6 +13,7 @@ func ModifyQueryParams(key string, value string) proxychain.RequestModification
 	return func(chain *proxychain.ProxyChain) error {
 		q := chain.Request.URL.Query()
 		chain.Request.URL.RawQuery = modifyQueryParams(key, value, q)
+		//fmt.Println(chain.Request.URL.String())
 		return nil
 	}
 }

proxychain/requestmodifiers/modify_request_headers.go

@@ -1,7 +1,7 @@
 package requestmodifiers
 
 import (
-	"ladder/proxychain"
+	"github.com/everywall/ladder/proxychain"
 )
 
 // SetRequestHeader modifies a specific outgoing header

proxychain/requestmodifiers/request_archive_is.go

@@ -5,8 +5,9 @@ import (
 	"net/url"
 	"regexp"
 
-	"ladder/proxychain"
+	tx "github.com/everywall/ladder/proxychain/responsemodifiers"
-	tx "ladder/proxychain/responsemodifiers"
+
+	"github.com/everywall/ladder/proxychain"
 )
 
 const archivistUrl string = "https://archive.is/latest"

proxychain/requestmodifiers/request_google_cache.go

@@ -3,7 +3,7 @@ package requestmodifiers
 import (
 	"net/url"
 
-	"ladder/proxychain"
+	"github.com/everywall/ladder/proxychain"
 )
 
 const googleCacheUrl string = "https://webcache.googleusercontent.com/search?q=cache:"

proxychain/requestmodifiers/request_wayback_machine.go

@@ -4,8 +4,9 @@ import (
 	"net/url"
 	"regexp"
 
-	"ladder/proxychain"
+	tx "github.com/everywall/ladder/proxychain/responsemodifiers"
-	tx "ladder/proxychain/responsemodifiers"
+
+	"github.com/everywall/ladder/proxychain"
 )
 
 const waybackUrl string = "https://web.archive.org/web/"

proxychain/requestmodifiers/resolve_with_google_doh.go

@@ -14,7 +14,7 @@ import (
 		//"net/http"
 	*/
 
-	"ladder/proxychain"
+	"github.com/everywall/ladder/proxychain"
 )
 
 // resolveWithGoogleDoH resolves DNS using Google's DNS-over-HTTPS

proxychain/requestmodifiers/spoof_ja3_fingerprint.go

@@ -1,52 +0,0 @@
-package requestmodifiers
-
-// removed due to using a different TLS spoofing technique
-
-/*
-import (
-	//"github.com/Danny-Dasilva/CycleTLS/cycletls"
-	//http "github.com/Danny-Dasilva/fhttp"
-	//http "github.com/bogdanfinn/fhttp"
-
-	"golang.org/x/net/proxy"
-	"ladder/proxychain"
-)
-
-// SpoofJA3fingerprint modifies the TLS client and user agent to spoof a particular JA3 fingerprint
-// Some anti-bot WAFs such as cloudflare can fingerprint the fields of the TLS hello packet, and the order in which they appear
-// https://web.archive.org/web/20231126224326/https://engineering.salesforce.com/tls-fingerprinting-with-ja3-and-ja3s-247362855967/
-// https://web.archive.org/web/20231119065253/https://developers.cloudflare.com/bots/concepts/ja3-fingerprint/
-func SpoofJA3fingerprint(ja3 string, userAgent string) proxychain.RequestModification {
-	//fmt.Println(ja3)
-	return func(chain *proxychain.ProxyChain) error {
-			// deep copy existing client while modifying http transport
-			ja3SpoofClient := &http.Client{
-				Transport:     cycletls.NewTransport(ja3, userAgent),
-				Timeout:       chain.Client.Timeout,
-				CheckRedirect: chain.Client.CheckRedirect,
-			}
-
-			chain.SetOnceHTTPClient(ja3SpoofClient)
-		return nil
-	}
-}
-
-// SpoofJA3fingerprintWithProxy modifies the TLS client and user agent to spoof a particular JA3 fingerprint and use a proxy.ContextDialer from the "golang.org/x/net/proxy"
-// Some anti-bot WAFs such as cloudflare can fingerprint the fields of the TLS hello packet, and the order in which they appear
-// https://web.archive.org/web/20231126224326/https://engineering.salesforce.com/tls-fingerprinting-with-ja3-and-ja3s-247362855967/
-// https://web.archive.org/web/20231119065253/https://developers.cloudflare.com/bots/concepts/ja3-fingerprint/
-func SpoofJA3fingerprintWithProxy(ja3 string, userAgent string, proxy proxy.ContextDialer) proxychain.RequestModification {
-	return func(chain *proxychain.ProxyChain) error {
-
-			// deep copy existing client while modifying http transport
-			ja3SpoofClient := &http.Client{
-				Transport:     cycletls.NewTransportWithProxy(ja3, userAgent, proxy),
-				Timeout:       chain.Client.Timeout,
-				CheckRedirect: chain.Client.CheckRedirect,
-			}
-
-			chain.SetOnceHTTPClient(ja3SpoofClient)
-		return nil
-	}
-}
-*/

proxychain/requestmodifiers/spoof_origin.go

@@ -1,7 +1,7 @@
 package requestmodifiers
 
 import (
-	"ladder/proxychain"
+	"github.com/everywall/ladder/proxychain"
 )
 
 // SpoofOrigin modifies the origin header

proxychain/requestmodifiers/spoof_referrer.go

@@ -3,8 +3,9 @@ package requestmodifiers
 import (
 	"fmt"
 
-	"ladder/proxychain"
+	tx "github.com/everywall/ladder/proxychain/responsemodifiers"
-	tx "ladder/proxychain/responsemodifiers"
+
+	"github.com/everywall/ladder/proxychain"
 )
 
 // SpoofReferrer modifies the referrer header.

proxychain/requestmodifiers/spoof_referrer_from_baidu_post.go

@@ -6,7 +6,7 @@ import (
 	"strings"
 	"time"
 
-	"ladder/proxychain"
+	"github.com/everywall/ladder/proxychain"
 )
 
 // SpoofReferrerFromBaiduSearch modifies the referrer header

proxychain/requestmodifiers/spoof_referrer_from_bing_search.go

@@ -1,7 +1,7 @@
 package requestmodifiers
 
 import (
-	"ladder/proxychain"
+	"github.com/everywall/ladder/proxychain"
 )
 
 // SpoofReferrerFromBingSearch modifies the referrer header

proxychain/requestmodifiers/spoof_referrer_from_google_search.go

@@ -1,7 +1,7 @@
 package requestmodifiers
 
 import (
-	"ladder/proxychain"
+	"github.com/everywall/ladder/proxychain"
 )
 
 // SpoofReferrerFromGoogleSearch modifies the referrer header

proxychain/requestmodifiers/spoof_referrer_from_linkedin_post.go

@@ -1,7 +1,7 @@
 package requestmodifiers
 
 import (
-	"ladder/proxychain"
+	"github.com/everywall/ladder/proxychain"
 )
 
 // SpoofReferrerFromLinkedInPost modifies the referrer header

proxychain/requestmodifiers/spoof_referrer_from_naver_post.go

@@ -3,7 +3,7 @@ package requestmodifiers
 import (
 	"fmt"
 
-	"ladder/proxychain"
+	"github.com/everywall/ladder/proxychain"
 )
 
 // SpoofReferrerFromNaverSearch modifies the referrer header

proxychain/requestmodifiers/spoof_referrer_from_pinterest_post.go

@@ -1,7 +1,7 @@
 package requestmodifiers
 
 import (
-	"ladder/proxychain"
+	"github.com/everywall/ladder/proxychain"
 )
 
 // SpoofReferrerFromPinterestPost modifies the referrer header

proxychain/requestmodifiers/spoof_referrer_from_qq_post.go

@@ -1,7 +1,7 @@
 package requestmodifiers
 
 import (
-	"ladder/proxychain"
+	"github.com/everywall/ladder/proxychain"
 )
 
 // SpoofReferrerFromQQPost modifies the referrer header

proxychain/requestmodifiers/spoof_referrer_from_reddit_post.go

@@ -1,7 +1,7 @@
 package requestmodifiers
 
 import (
-	"ladder/proxychain"
+	"github.com/everywall/ladder/proxychain"
 )
 
 // SpoofReferrerFromRedditPost modifies the referrer header

proxychain/requestmodifiers/spoof_referrer_from_tumblr_post.go

@@ -1,7 +1,7 @@
 package requestmodifiers
 
 import (
-	"ladder/proxychain"
+	"github.com/everywall/ladder/proxychain"
 )
 
 // SpoofReferrerFromTumblrPost modifies the referrer header

proxychain/requestmodifiers/spoof_referrer_from_twitter_post.go

@@ -1,7 +1,7 @@
 package requestmodifiers
 
 import (
-	"ladder/proxychain"
+	"github.com/everywall/ladder/proxychain"
 )
 
 // SpoofReferrerFromTwitterPost modifies the referrer header

proxychain/requestmodifiers/spoof_referrer_from_vkontake_post.go

@@ -1,7 +1,7 @@
 package requestmodifiers
 
 import (
-	"ladder/proxychain"
+	"github.com/everywall/ladder/proxychain"
 )
 
 // SpoofReferrerFromVkontaktePost modifies the referrer header

proxychain/requestmodifiers/spoof_referrer_from_weibo_post.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"math/rand"
 
-	"ladder/proxychain"
+	"github.com/everywall/ladder/proxychain"
 )
 
 // SpoofReferrerFromWeiboPost modifies the referrer header

proxychain/requestmodifiers/spoof_user_agent.go

@@ -4,8 +4,9 @@ import (
 	_ "embed"
 	"strings"
 
-	"ladder/proxychain"
+	tx "github.com/everywall/ladder/proxychain/responsemodifiers"
-	tx "ladder/proxychain/responsemodifiers"
+
+	"github.com/everywall/ladder/proxychain"
 )
 
 // https://github.com/faisalman/ua-parser-js/tree/master

proxychain/requestmodifiers/spoof_x_forwarded_for.go

@@ -1,7 +1,7 @@
 package requestmodifiers
 
 import (
-	"ladder/proxychain"
+	"github.com/everywall/ladder/proxychain"
 )
 
 // SpoofXForwardedFor modifies the X-Forwarded-For header

proxychain/responsemodifiers/api_content.go

@@ -7,8 +7,9 @@ import (
 
 	"github.com/markusmobius/go-trafilatura"
 
-	"ladder/proxychain"
+	"github.com/everywall/ladder/proxychain/responsemodifiers/api"
-	"ladder/proxychain/responsemodifiers/api"
+
+	"github.com/everywall/ladder/proxychain"
 )
 
 // APIContent creates an JSON representation of the article and returns it as an API response.

proxychain/responsemodifiers/api_content_test.go

@@ -7,7 +7,7 @@ import (
 	"net/url"
 	"testing"
 
-	"ladder/proxychain/responsemodifiers/api"
+	"github.com/everywall/ladder/proxychain/responsemodifiers/api"
 )
 
 func TestCreateAPIErrReader(t *testing.T) {

proxychain/responsemodifiers/block_element_removal.go

@@ -4,11 +4,12 @@ import (
 	_ "embed"
 	"strings"
 
-	"ladder/proxychain"
+	"github.com/everywall/ladder/proxychain/responsemodifiers/rewriters"
-	"ladder/proxychain/responsemodifiers/rewriters"
+
+	"github.com/everywall/ladder/proxychain"
 )
 
-//go:embed block_element_removal.js
+//go:embed vendor/block_element_removal.js
 var blockElementRemoval string
 
 // BlockElementRemoval prevents paywall javascript from removing a

proxychain/responsemodifiers/block_third_party_scripts.go

@@ -0,0 +1,34 @@
+package responsemodifiers
+
+import (
+	_ "embed"
+	"fmt"
+	"strings"
+
+	"github.com/everywall/ladder/proxychain/responsemodifiers/rewriters"
+
+	"github.com/everywall/ladder/proxychain"
+)
+
+// BlockThirdPartyScripts rewrites HTML and injects JS to block all third party JS from loading.
+func BlockThirdPartyScripts() proxychain.ResponseModification {
+	// TODO: monkey patch fetch and XMLHttpRequest to firewall 3P JS as well.
+	return func(chain *proxychain.ProxyChain) error {
+		// don't add rewriter if it's not even html
+		ct := chain.Response.Header.Get("content-type")
+		if !strings.HasPrefix(ct, "text/html") {
+			return nil
+		}
+
+		// proxyURL is the URL of the ladder: http://localhost:8080 (ladder)
+		originalURI := chain.Context.Request().URI()
+		proxyURL := fmt.Sprintf("%s://%s", originalURI.Scheme(), originalURI.Host())
+
+		// replace http.Response.Body with a readcloser that wraps the original, modifying the html attributes
+		rr := rewriters.NewBlockThirdPartyScriptsRewriter(chain.Request.URL, proxyURL)
+		blockJSRewriter := rewriters.NewHTMLRewriter(chain.Response.Body, rr)
+		chain.Response.Body = blockJSRewriter
+
+		return nil
+	}
+}

proxychain/responsemodifiers/bypass_cors.go

@@ -1,7 +1,7 @@
 package responsemodifiers
 
 import (
-	"ladder/proxychain"
+	"github.com/everywall/ladder/proxychain"
 )
 
 // BypassCORS modifies response headers to prevent the browser

proxychain/responsemodifiers/bypass_csp.go

@@ -1,7 +1,7 @@
 package responsemodifiers
 
 import (
-	"ladder/proxychain"
+	"github.com/everywall/ladder/proxychain"
 )
 
 // TODO: handle edge case where CSP is specified in meta tag:

proxychain/responsemodifiers/delete_localstorage_data.go

@@ -0,0 +1,28 @@
+package responsemodifiers
+
+import (
+	_ "embed"
+	"strings"
+
+	"github.com/everywall/ladder/proxychain"
+)
+
+// DeleteLocalStorageData deletes localstorage cookies.
+// If the page works once in a fresh incognito window, but fails
+// for subsequent loads, try this response modifier alongside
+// DeleteSessionStorageData and DeleteIncomingCookies
+func DeleteLocalStorageData() proxychain.ResponseModification {
+	return func(chain *proxychain.ProxyChain) error {
+		// don't add rewriter if it's not even html
+		ct := chain.Response.Header.Get("content-type")
+		if !strings.HasPrefix(ct, "text/html") {
+			return nil
+		}
+
+		chain.AddOnceResponseModifications(
+			InjectScriptBeforeDOMContentLoaded(`window.sessionStorage.clear()`),
+			InjectScriptAfterDOMContentLoaded(`window.sessionStorage.clear()`),
+		)
+		return nil
+	}
+}

proxychain/responsemodifiers/delete_sessionstorage_data.go

@@ -0,0 +1,28 @@
+package responsemodifiers
+
+import (
+	_ "embed"
+	"strings"
+
+	"github.com/everywall/ladder/proxychain"
+)
+
+// DeleteSessionStorageData deletes localstorage cookies.
+// If the page works once in a fresh incognito window, but fails
+// for subsequent loads, try this response modifier alongside
+// DeleteLocalStorageData and DeleteIncomingCookies
+func DeleteSessionStorageData() proxychain.ResponseModification {
+	return func(chain *proxychain.ProxyChain) error {
+		// don't add rewriter if it's not even html
+		ct := chain.Response.Header.Get("content-type")
+		if !strings.HasPrefix(ct, "text/html") {
+			return nil
+		}
+
+		chain.AddOnceResponseModifications(
+			InjectScriptBeforeDOMContentLoaded(`window.sessionStorage.clear()`),
+			InjectScriptAfterDOMContentLoaded(`window.sessionStorage.clear()`),
+		)
+		return nil
+	}
+}

proxychain/responsemodifiers/forward_response_headers.go

@@ -5,7 +5,7 @@ import (
 	"net/url"
 	"strings"
 
-	"ladder/proxychain"
+	"github.com/everywall/ladder/proxychain"
 )
 
 var forwardBlacklist map[string]bool

proxychain/responsemodifiers/generate_readable_outline.go

@@ -7,26 +7,25 @@ import (
 	"html/template"
 	"io"
 	"log"
+	"math"
 	"net/url"
 	"strings"
+	"time"
 
-	"ladder/proxychain"
+	"github.com/everywall/ladder/proxychain"
-
+	"github.com/markusmobius/go-trafilatura"
 	"golang.org/x/net/html"
 	"golang.org/x/net/html/atom"
-
-	//"github.com/go-shiori/dom"
-	"github.com/markusmobius/go-trafilatura"
 )
 
-//go:embed generate_readable_outline.html
+//go:embed vendor/generate_readable_outline.html
 var templateFS embed.FS
 
 // GenerateReadableOutline creates an reader-friendly distilled representation of the article.
 // This is a reliable way of bypassing soft-paywalled articles, where the content is hidden, but still present in the DOM.
 func GenerateReadableOutline() proxychain.ResponseModification {
 	// get template only once, and resuse for subsequent calls
-	f := "generate_readable_outline.html"
+	f := "vendor/generate_readable_outline.html"
 	tmpl, err := template.ParseFS(templateFS, f)
 	if err != nil {
 		panic(fmt.Errorf("tx.GenerateReadableOutline Error: %s not found", f))
@@ -63,19 +62,24 @@ func GenerateReadableOutline() proxychain.ResponseModification {
 		html.Render(&b, extract.ContentNode)
 		distilledHTML := b.String()
 
+		siteName := strings.Split(extract.Metadata.Sitename, ";")[0]
+		title := strings.Split(extract.Metadata.Title, "|")[0]
+		fmtDate := createWikipediaDateLink(extract.Metadata.Date)
+		readingTime := formatDuration(estimateReadingTime(extract.ContentText))
+
 		// populate template parameters
 		data := map[string]interface{}{
 			"Success":     true,
 			"Image":       extract.Metadata.Image,
 			"Description": extract.Metadata.Description,
-			"Sitename":    extract.Metadata.Sitename,
+			"Sitename":    siteName,
 			"Hostname":    extract.Metadata.Hostname,
 			"Url":         "/" + chain.Request.URL.String(),
-			"Title":       extract.Metadata.Title, // todo: modify CreateReadableDocument so we don't have <h1> titles duplicated?
+			"Title":       title,
-			"Date":        extract.Metadata.Date.String(),
+			"Date":        fmtDate,
-			"Author":      createWikipediaSearchLinks(extract.Metadata.Author),
+			"Author":      createDDGFeelingLuckyLinks(extract.Metadata.Author, extract.Metadata.Hostname),
-			//"Author": extract.Metadata.Author,
 			"Body":        distilledHTML,
+			"ReadingTime": readingTime,
 		}
 
 		// ============================================================================
@@ -157,9 +161,20 @@ func rewriteHrefLinks(n *html.Node, baseURL string, apiPath string) {
 	recurse(n)
 }
 
-// createWikipediaSearchLinks takes in comma or semicolon separated terms,
+// createWikipediaDateLink takes in a date
-// then turns them into <a> links searching for the term.
+// and returns an <a> link pointing to the current events page for that day
-func createWikipediaSearchLinks(searchTerms string) string {
+func createWikipediaDateLink(t time.Time) string {
+	url := fmt.Sprintf("https://en.wikipedia.org/wiki/Portal:Current_events#%s", t.Format("2006_January_02"))
+	date := t.Format("January 02, 2006")
+	return fmt.Sprintf("<a rel=\"noreferrer\" href=\"%s\">%s</a>", url, date)
+}
+
+// createDDGFeelingLuckyLinks takes in comma or semicolon separated terms,
+// then turns them into <a> links searching for the term using DuckDuckGo's I'm
+// feeling lucky feature. It will redirect the user immediately to the first search result.
+func createDDGFeelingLuckyLinks(searchTerms string, siteHostname string) string {
+
+	siteHostname = strings.TrimSpace(siteHostname)
 	semiColonSplit := strings.Split(searchTerms, ";")
 
 	var links []string
@@ -171,11 +186,13 @@ func createWikipediaSearchLinks(searchTerms string) string {
 				continue
 			}
 
-			encodedTerm := url.QueryEscape(trimmedTerm)
+			ddgQuery := fmt.Sprintf(` site:%s intitle:"%s"`, strings.TrimPrefix(siteHostname, "www."), trimmedTerm)
 
-			wikiURL := fmt.Sprintf("https://en.wikipedia.org/w/index.php?search=%s", encodedTerm)
+			encodedTerm := `\%s:` + url.QueryEscape(ddgQuery)
+			//ddgURL := `https://html.duckduckgo.com/html/?q=` + encodedTerm
+			ddgURL := `https://www.duckduckgo.com/?q=` + encodedTerm
 
-			link := fmt.Sprintf("<a href=\"%s\">%s</a>", wikiURL, trimmedTerm)
+			link := fmt.Sprintf("<a rel=\"noreferrer\" href=\"%s\">%s</a>", ddgURL, trimmedTerm)
 			links = append(links, link)
 		}
 
@@ -187,3 +204,66 @@ func createWikipediaSearchLinks(searchTerms string) string {
 
 	return strings.Join(links, " ")
 }
+
+// estimateReadingTime estimates how long the given text will take to read using the given configuration.
+func estimateReadingTime(text string) time.Duration {
+	if len(text) == 0 {
+		return 0
+	}
+
+	// Init options with default values.
+	WordsPerMinute := 200
+	WordBound := func(b byte) bool {
+		return b == ' ' || b == '\n' || b == '\r' || b == '\t'
+	}
+
+	words := 0
+	start := 0
+	end := len(text) - 1
+
+	// Fetch bounds.
+	for WordBound(text[start]) {
+		start++
+	}
+	for WordBound(text[end]) {
+		end--
+	}
+
+	// Calculate the number of words.
+	for i := start; i <= end; {
+		for i <= end && !WordBound(text[i]) {
+			i++
+		}
+
+		words++
+
+		for i <= end && WordBound(text[i]) {
+			i++
+		}
+	}
+
+	// Reading time stats.
+	minutes := math.Ceil(float64(words) / float64(WordsPerMinute))
+	duration := time.Duration(math.Ceil(minutes) * float64(time.Minute))
+
+	return duration
+
+}
+
+func formatDuration(d time.Duration) string {
+	// Check if the duration is less than one minute
+	if d < time.Minute {
+		seconds := int(d.Seconds())
+		return fmt.Sprintf("%d seconds", seconds)
+	}
+
+	// Convert the duration to minutes
+	minutes := int(d.Minutes())
+
+	// Format the string for one or more minutes
+	if minutes == 1 {
+		return "1 minute"
+	} else {
+		return fmt.Sprintf("%d minutes", minutes)
+	}
+}

proxychain/responsemodifiers/inject_script.go

@@ -4,8 +4,9 @@ import (
 	_ "embed"
 	"strings"
 
-	"ladder/proxychain"
+	"github.com/everywall/ladder/proxychain/responsemodifiers/rewriters"
-	"ladder/proxychain/responsemodifiers/rewriters"
+
+	"github.com/everywall/ladder/proxychain"
 )
 
 // injectScript modifies HTTP responses

proxychain/responsemodifiers/modify_incoming_cookies.go

@@ -7,14 +7,18 @@ import (
 	//"net/http"
 	//http "github.com/Danny-Dasilva/fhttp"
 
-	"ladder/proxychain"
+	"github.com/everywall/ladder/proxychain"
 )
 
 // DeleteIncomingCookies prevents ALL cookies from being sent from the proxy server
 // back down to the client.
 func DeleteIncomingCookies(_ ...string) proxychain.ResponseModification {
-	return func(px *proxychain.ProxyChain) error {
+	return func(chain *proxychain.ProxyChain) error {
-		px.Response.Header.Del("Set-Cookie")
+		chain.Response.Header.Del("Set-Cookie")
+		chain.AddOnceResponseModifications(
+			InjectScriptBeforeDOMContentLoaded(`document.cookie = ""`),
+			InjectScriptAfterDOMContentLoaded(`document.cookie = ""`),
+		)
 		return nil
 	}
 }

proxychain/responsemodifiers/modify_incoming_scripts_with_regex.go

@@ -0,0 +1,53 @@
+package responsemodifiers
+
+import (
+	"bytes"
+	"io"
+	"regexp"
+	"strings"
+
+	"github.com/everywall/ladder/proxychain"
+)
+
+// ModifyIncomingScriptsWithRegex modifies all incoming javascript (application/javascript and inline <script> in text/html) using a regex match and replacement.
+func ModifyIncomingScriptsWithRegex(matchRegex string, replacement string) proxychain.ResponseModification {
+	return func(chain *proxychain.ProxyChain) error {
+		path := chain.Request.URL.Path
+		ct := chain.Response.Header.Get("content-type")
+		isJavascript := strings.HasSuffix(path, ".js") || ct == "text/javascript" || ct == "application/javascript"
+		isHTML := strings.HasSuffix(chain.Request.URL.Path, ".html") || ct == "text/html"
+
+		switch {
+		case isJavascript:
+			rBody, err := modifyResponse(chain.Response.Body, matchRegex, replacement)
+			if err != nil {
+				return err
+			}
+			chain.Response.Body = rBody
+		case isHTML:
+		default:
+			return nil
+		}
+		return nil
+	}
+}
+
+func modifyResponse(body io.ReadCloser, matchRegex, replacement string) (io.ReadCloser, error) {
+	content, err := io.ReadAll(body)
+	if err != nil {
+		return nil, err
+	}
+
+	re, err := regexp.Compile(matchRegex)
+	if err != nil {
+		return nil, err
+	}
+	err = body.Close()
+	if err != nil {
+		return body, err
+	}
+
+	modifiedContent := re.ReplaceAll(content, []byte(replacement))
+
+	return io.NopCloser(bytes.NewReader(modifiedContent)), nil
+}

proxychain/responsemodifiers/modify_response_header.go

@@ -1,21 +1,21 @@
 package responsemodifiers
 
 import (
-	"ladder/proxychain"
+	"github.com/everywall/ladder/proxychain"
 )
 
 // SetResponseHeader modifies response headers from the upstream server
 func SetResponseHeader(key string, value string) proxychain.ResponseModification {
-	return func(px *proxychain.ProxyChain) error {
+	return func(chain *proxychain.ProxyChain) error {
-		px.Context.Response().Header.Set(key, value)
+		chain.Context.Set(key, value)
 		return nil
 	}
 }
 
 // DeleteResponseHeader removes response headers from the upstream server
 func DeleteResponseHeader(key string) proxychain.ResponseModification {
-	return func(px *proxychain.ProxyChain) error {
+	return func(chain *proxychain.ProxyChain) error {
-		px.Context.Response().Header.Del(key)
+		chain.Context.Response().Header.Del(key)
 		return nil
 	}
 }

proxychain/responsemodifiers/patch_dynamic_resource_urls.go

@@ -5,11 +5,12 @@ import (
 	"fmt"
 	"strings"
 
-	"ladder/proxychain"
+	"github.com/everywall/ladder/proxychain/responsemodifiers/rewriters"
-	"ladder/proxychain/responsemodifiers/rewriters"
+
+	"github.com/everywall/ladder/proxychain"
 )
 
-//go:embed patch_dynamic_resource_urls.js
+//go:embed vendor/patch_dynamic_resource_urls.js
 var patchDynamicResourceURLsScript string
 
 // PatchDynamicResourceURLs patches the javascript runtime to rewrite URLs client-side.
@@ -51,6 +52,19 @@ func PatchDynamicResourceURLs() proxychain.ResponseModification {
 		htmlRewriter := rewriters.NewHTMLRewriter(chain.Response.Body, rr)
 		chain.Response.Body = htmlRewriter
 
+		// window.location
+		/*
+				spoofedLocationAPI := fmt.Sprintf(`{href:"%s", origin:"%s", pathname:"%s", protocol:"%s:", port:"%s"}`,
+					reqURL.String(), reqURL.Host,
+					reqURL.Path, reqURL.Scheme, reqURL.Port())
+			spoofedLocationAPI := fmt.Sprintf(`{origin: "%s"}`, reqURL.Host)
+			fmt.Println(spoofedLocationAPI)
+
+			chain.AddOnceResponseModifications(
+				ModifyIncomingScriptsWithRegex(`window\.location`, spoofedLocationAPI),
+			)
+		*/
+
 		return nil
 	}
 }

proxychain/responsemodifiers/patch_google_analytics.go

@@ -1,34 +0,0 @@
-package responsemodifiers
-
-import (
-	_ "embed"
-	"io"
-	"strings"
-
-	"ladder/proxychain"
-)
-
-//go:embed patch_google_analytics.js
-var gaPatch string
-
-// PatchGoogleAnalytics replaces any request to google analytics with a no-op stub function.
-// Some sites will not display content until GA is loaded, so we fake one instead.
-// Credit to Raymond Hill @ github.com/gorhill/uBlock
-func PatchGoogleAnalytics() proxychain.ResponseModification {
-	return func(chain *proxychain.ProxyChain) error {
-
-		// preflight check
-		isGADomain := chain.Request.URL.Host == "www.google-analytics.com" || chain.Request.URL.Host == "google-analytics.com"
-		isGAPath := strings.HasSuffix(chain.Request.URL.Path, "analytics.js")
-		if !(isGADomain || isGAPath) {
-			return nil
-		}
-
-		// send modified js payload to client containing
-		// stub functions from patch_google_analytics.js
-		gaPatchReader := io.NopCloser(strings.NewReader(gaPatch))
-		chain.Response.Body = gaPatchReader
-		chain.Context.Set("content-type", "text/javascript")
-		return nil
-	}
-}

proxychain/responsemodifiers/patch_tracker_scripts.go

@@ -7,7 +7,7 @@ import (
 	"log"
 	"regexp"
 
-	"ladder/proxychain"
+	"github.com/everywall/ladder/proxychain"
 )
 
 //go:embed vendor/ddg-tracker-surrogates/mapping.json

proxychain/responsemodifiers/rewrite_http_resource_urls.go

@@ -5,8 +5,9 @@ import (
 	"fmt"
 	"strings"
 
-	"ladder/proxychain"
+	"github.com/everywall/ladder/proxychain/responsemodifiers/rewriters"
-	"ladder/proxychain/responsemodifiers/rewriters"
+
+	"github.com/everywall/ladder/proxychain"
 )
 
 // RewriteHTMLResourceURLs modifies HTTP responses

proxychain/responsemodifiers/rewriters/block_third_party_scripts.go

@@ -0,0 +1,69 @@
+package rewriters
+
+import (
+	_ "embed"
+	"fmt"
+	"log"
+	"net/url"
+	"strings"
+
+	"golang.org/x/net/html"
+	"golang.org/x/net/html/atom"
+)
+
+// BlockThirdPartyScriptsRewriter implements HTMLTokenRewriter
+// and blocks 3rd party JS in script tags by replacing the src attribute value "blocked"
+type BlockThirdPartyScriptsRewriter struct {
+	baseURL  *url.URL
+	proxyURL string // ladder URL, not proxied site URL
+}
+
+// NewBlockThirdPartyScriptsRewriter creates a new instance of BlockThirdPartyScriptsRewriter.
+// This rewriter will strip out 3rd party JS URLs from script tags.
+func NewBlockThirdPartyScriptsRewriter(baseURL *url.URL, proxyURL string) *BlockThirdPartyScriptsRewriter {
+	return &BlockThirdPartyScriptsRewriter{
+		baseURL:  baseURL,
+		proxyURL: proxyURL,
+	}
+}
+
+func (r *BlockThirdPartyScriptsRewriter) ShouldModify(token *html.Token) bool {
+	if token.DataAtom != atom.Script {
+		return false
+	}
+
+	// check for 3p .js urls in html elements
+	for i := range token.Attr {
+		attr := token.Attr[i]
+		switch {
+		case attr.Key != "src":
+			continue
+		case strings.HasPrefix(attr.Val, "/"):
+			return false
+		case !strings.HasPrefix(attr.Val, "http"):
+			return false
+		case strings.HasPrefix(attr.Val, r.proxyURL):
+			return false
+		case strings.HasPrefix(attr.Val, fmt.Sprintf("%s://%s", r.baseURL.Scheme, r.baseURL.Hostname())):
+			return false
+		}
+	}
+
+	return true
+}
+
+func (r *BlockThirdPartyScriptsRewriter) ModifyToken(token *html.Token) (string, string) {
+	for i := range token.Attr {
+		attr := &token.Attr[i]
+		if attr.Key != "src" {
+			continue
+		}
+
+		if !strings.HasPrefix(attr.Val, "http") {
+			continue
+		}
+		log.Printf("INFO: blocked 3P js: '%s' on '%s'\n", attr.Val, r.baseURL.String())
+		attr.Key = "blocked"
+	}
+	return "", ""
+}

proxychain/responsemodifiers/rewriters/script_injector_rewriter.go

@@ -6,6 +6,8 @@ import (
 	"sort"
 	"strings"
 
+	"crypto/md5"
+	"encoding/hex"
 	"golang.org/x/net/html"
 	"golang.org/x/net/html/atom"
 )
@@ -16,6 +18,7 @@ import (
 type ScriptInjectorRewriter struct {
 	execTime  ScriptExecTime
 	script    string
+	scriptMD5 string
 }
 
 type ScriptExecTime int
@@ -37,20 +40,27 @@ var afterDomIdleScriptInjector string
 func (r *ScriptInjectorRewriter) ModifyToken(_ *html.Token) (string, string) {
 	switch {
 	case r.execTime == BeforeDOMContentLoaded:
-		return "", fmt.Sprintf("\n<script>\n%s\n</script>\n", r.script)
+		return "", fmt.Sprintf("\n<script id='%s'>\n%s\n</script>\n", r.scriptMD5, r.script)
 
 	case r.execTime == AfterDOMContentLoaded:
-		return "", fmt.Sprintf("\n<script>\ndocument.addEventListener('DOMContentLoaded', () => { %s });\n</script>", r.script)
+		return "", fmt.Sprintf("\n<script id='%s'>\ndocument.addEventListener('DOMContentLoaded', () => { %s });\n</script>", r.scriptMD5, r.script)
 
 	case r.execTime == AfterDOMIdle:
 		s := strings.Replace(afterDomIdleScriptInjector, `'{{AFTER_DOM_IDLE_SCRIPT}}'`, r.script, 1)
-		return "", fmt.Sprintf("\n<script>\n%s\n</script>\n", s)
+		return "", fmt.Sprintf("\n<script id='%s'>\n%s\n</script>\n", r.scriptMD5, s)
 
 	default:
 		return "", ""
 	}
 }
 
+// GenerateMD5Hash takes a string and returns its MD5 hash as a hexadecimal string
+func generateMD5Hash(input string) string {
+	hasher := md5.New()
+	hasher.Write([]byte(input))
+	return hex.EncodeToString(hasher.Sum(nil))
+}
+
 // applies parameters by string replacement of the template script
 func (r *ScriptInjectorRewriter) applyParams(params map[string]string) {
 	// Sort the keys by length in descending order
@@ -71,9 +81,13 @@ func (r *ScriptInjectorRewriter) applyParams(params map[string]string) {
 // NewScriptInjectorRewriter implements a HtmlTokenRewriter
 // and injects JS into the page for execution at a particular time
 func NewScriptInjectorRewriter(script string, execTime ScriptExecTime) *ScriptInjectorRewriter {
+	scriptMD5 := generateMD5Hash(script)
+	executeOnceScript := fmt.Sprintf(`if (!document.getElementById("x-%s")) { %s; document.getElementById("%s").id = "x-%s" };`, scriptMD5, script, scriptMD5, scriptMD5)
+
 	return &ScriptInjectorRewriter{
 		execTime:  execTime,
-		script:   script,
+		script:    executeOnceScript,
+		scriptMD5: scriptMD5,
 	}
 }
 
@@ -83,10 +97,7 @@ func NewScriptInjectorRewriter(script string, execTime ScriptExecTime) *ScriptIn
 // the params map represents the key-value pair of the params.
 // the key will be string replaced with the value
 func NewScriptInjectorRewriterWithParams(script string, execTime ScriptExecTime, params map[string]string) *ScriptInjectorRewriter {
-	rr := &ScriptInjectorRewriter{
+	rr := NewScriptInjectorRewriter(script, execTime)
-		execTime: execTime,
-		script:   script,
-	}
 	rr.applyParams(params)
 	return rr
 }

proxychain/responsemodifiers/vendor/generate_readable_outline.html

@@ -23,6 +23,26 @@
         }
       };
       handleThemeChange();
+      function prepareForPrint() {
+        document.getElementById("readingtime").innerText =
+          "Date Accessed: " +
+          new Date().toLocaleDateString("en-US", {
+            year: "numeric",
+            month: "long",
+            day: "numeric",
+          });
+        [...document.querySelectorAll(".noprint")].forEach((e) =>
+          e.classList.toggle("hidden")
+        );
+        window.addEventListener("afterprint", handleAfterPrint);
+        window.print();
+      }
+      function handleAfterPrint() {
+        [...document.querySelectorAll(".noprint")].forEach((e) =>
+          e.classList.toggle("hidden")
+        );
+        window.removeEventListener("afterprint", handleAfterPrint);
+      }
     </script>
     <title>ladder | {{.Title}}</title>
   </head>
@@ -35,17 +55,17 @@
         <div
           class="hover:drop-shadow-[0_0px_4px_rgba(122,167,209,.3)] transition-colors duration-300 focus:outline-none focus:ring focus:border-[#7AA7D1] ring-offset-2"
         >
-
           <div class="flex">
             <a
               href="/"
-            class="flex -ml-2 h-8 font-extrabold tracking-tight hover:no-underline focus:outline-none focus:ring focus:border-[#7AA7D1] ring-offset-2"
+              aria-label="ladder"
+              class="flex -ml-2 h-8 font-extrabold tracking-tight no-underline focus:outline-none focus:ring focus:border-[#7AA7D1] ring-offset-2"
             >
               <svg
                 xmlns="http://www.w3.org/2000/svg"
                 xmlns:xlink="http://www.w3.org/1999/xlink"
                 viewBox="0 0 512 512"
-              class="h-8 focus:outline-none focus:ring focus:border-[#7AA7D1] ring-offset-2"
+                class="noprint h-8 focus:outline-none focus:ring focus:border-[#7AA7D1] ring-offset-2"
               >
                 <path
                   fill="#7AA7D1"
@@ -59,21 +79,24 @@
             </a>
             <a
               href="/https://{{.Hostname}}"
-            class="flex ml-1 h-8 font-extrabold tracking-tight hover:no-underline focus:outline-none focus:ring focus:border-[#7AA7D1] ring-offset-2"
+              class="flex ml-1 h-8 font-extrabold tracking-tight no-underline focus:outline-none focus:ring focus:border-[#7AA7D1] ring-offset-2"
+            >
+              <span class="text-3xl mr-1 text-[#7AA7D1] leading-8 align-middle"
+                >{{.Sitename}}</span
               >
-            <span class="text-3xl mr-1 text-[#7AA7D1] leading-8 align-middle">{{.Sitename}}</span>
             </a>
           </div>
-
         </div>
 
-        <div class="flex justify-center z-10">
+        <div class="noprint flex justify-center z-10">
           <div class="relative" id="dropdown">
             <button
-              aria-expanded="closed"
+              aria-expanded="false"
+              id="dropdownButton"
+              aria-label="Toggle preferences menu"
               onclick="toggleDropdown()"
               type="button"
-              class="inline-flex items-center justify-center whitespace-nowrap rounded-full h-12 px-4 py-2 text-sm font-medium text-slate-600 dark:text-slate-400 ring-offset-background transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50 bg-white dark:bg-slate-900 hover:bg-slate-200 dark:hover:bg-slate-700 hover:text-slate-500 dark:hover:text-slate-200"
+              class="inline-flex items-center justify-center align-middle whitespace-nowrap rounded-full h-12 px-4 py-2 text-sm font-medium text-slate-600 dark:text-slate-400 ring-offset-background transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50 bg-white dark:bg-slate-900 hover:bg-slate-200 dark:hover:bg-slate-700 hover:text-slate-500 dark:hover:text-slate-200"
             >
               <svg
                 xmlns="http://www.w3.org/2000/svg"
@@ -92,6 +115,25 @@
               </svg>
             </button>
 
+            <button
+              aria-expanded="false"
+              title="Print"
+              onclick="prepareForPrint()"
+              type="button"
+              class="inline-flex items-center justify-center align-middle whitespace-nowrap rounded-full h-12 px-4 py-2 text-sm font-medium text-slate-600 dark:text-slate-400 ring-offset-background transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50 bg-white dark:bg-slate-900 hover:bg-slate-200 dark:hover:bg-slate-700 hover:text-slate-500 dark:hover:text-slate-200"
+            >
+              <svg
+                xmlns="http://www.w3.org/2000/svg"
+                fill="currentColor"
+                viewBox="0 0 512 512"
+                class="h-4 w-4"
+              >
+                <path
+                  d="M128 0C92.7 0 64 28.7 64 64v96h64V64H354.7L384 93.3V160h64V93.3c0-17-6.7-33.3-18.7-45.3L400 18.7C388 6.7 371.7 0 354.7 0H128zM384 352v32 64H128V384 368 352H384zm64 32h32c17.7 0 32-14.3 32-32V256c0-35.3-28.7-64-64-64H64c-35.3 0-64 28.7-64 64v96c0 17.7 14.3 32 32 32H64v64c0 35.3 28.7 64 64 64H384c35.3 0 64-28.7 64-64V384zM432 248a24 24 0 1 1 0 48 24 24 0 1 1 0-48z"
+                />
+              </svg>
+            </button>
+
             <div
               id="dropdown_panel"
               class="hidden absolute right-0 mt-2 w-52 rounded-md bg-white dark:bg-slate-900 shadow-md border border-slate-400 dark:border-slate-700"
@@ -315,28 +357,29 @@
 
       <main class="flex flex-col space-y-3">
         {{if not .Success}}
-        <h1>
+        <h1>Error</h1>
-          Error
-        </h1>
         <p>
           There was a problem querying
           <a href="{{.Params}}">{{.Params}}</a>
         </p>
-        <code class="text-red-500 dark:text-red-400">
+        <code class="text-red-500 dark:text-red-400"> {{.Error}} </code>
-          {{.Error}}
-        </code>
         {{else}}
+
         <div class="flex flex-col gap-1 mt-3">
           <h1>
-            <a href="{{.Url}}" class="text-slate-900 dark:text-slate-200"> {{.Title}} </a>
+            <a
+              href="{{.Url}}"
+              class="text-slate-900 dark:text-slate-200 no-underline hover:underline"
+            >
+              {{.Title}}
+            </a>
           </h1>
           {{if ne .Date ""}}
           <small
             class="text-sm font-medium leading-none text-slate-600 dark:text-slate-400"
             >{{.Date}}</small
           >
-          {{end}} 
+          {{end}} {{if ne .Author ""}}
-          {{if ne .Author ""}}
           <small
             class="text-sm font-medium leading-none text-slate-600 dark:text-slate-400"
             >{{.Author}}</small
@@ -344,11 +387,50 @@
           {{end}}
         </div>
 
-        <div class="flex flex-col space-y-3">
+        <h2>
+          <a
+            href="{{.Url}}"
+            class="text-slate-900 dark:text-slate-200 no-underline hover:underline"
+          >
+            {{.Title}}
+          </a>
+        </h2>
+
+        <div class="flex justify-between items-center gap-1 mt-3">
           <div>
+            {{if ne .Author ""}}
+            <small
+              class="text-sm font-medium leading-none text-slate-600 dark:text-slate-400"
+              >{{.Author}} |
+            </small>
+            {{end}} {{if ne .Date ""}}
+            <small
+              class="text-sm font-medium leading-none text-slate-600 dark:text-slate-400"
+              >{{.Date}}</small
+            >
+            {{end}}
+          </div>
+
+          <div>
+            <small
+              id="readingtime"
+              class="text-sm font-medium leading-none text-slate-600 dark:text-slate-400"
+              >Reading Time: {{.ReadingTime}}</small
+            >
+          </div>
+        </div>
+        <div class="flex flex-col space-y-3">
           <div class="grid grid-cols-1 justify-items-center">
-            <div><img src="{{.Image}}" alt="{{.Description}}" class="h-auto w-auto object-cover max-w-full mx-auto rounded-md shadow-md dark:shadow-slate-700"/></div>
+            <div>
-            <div class="mt-2 text-sm text-slate-600 dark:text-slate-400">{{.Description}}</div>
+              <img
+                src="{{.Image}}"
+                alt="{{.Description}}"
+                class="h-auto w-auto object-cover max-w-full mx-auto rounded-md shadow-md dark:shadow-slate-700"
+              />
+            </div>
+            <div class="mt-2 text-sm text-slate-600 dark:text-slate-400">
+              {{.Description}}
+            </div>
           </div>
         </div>
 
@@ -356,25 +438,24 @@
         {{end}}
       </main>
 
-      <div class="my-2"></div>
+      <footer
-      <footer class="mx-4 text-center text-slate-600 dark:text-slate-400">
+        class="noprint mx-4 my-2 pt-2 border-t border-gray-300 dark:border-gray-700 text-center text-slate-600 dark:text-slate-400"
-        <p>
-          Code Licensed Under GPL v3.0 |
-          <a
-            href="https://github.com/everywall/ladder"
-            class="hover:text-blue-500 dark:hover:text-blue-500 hover:underline underline-offset-2 transition-colors duration-300"
-            >View Source</a
       >
-          |
+        <small>
           <a
             href="https://github.com/everywall"
             class="hover:text-blue-500 dark:hover:text-blue-500 hover:underline underline-offset-2 transition-colors duration-300"
             >Everywall</a
           >
-        </p>
+          |
+          <a
+            href="https://github.com/everywall/ladder"
+            class="hover:text-blue-500 dark:hover:text-blue-500 hover:underline underline-offset-2 transition-colors duration-300"
+            >Source</a
+          >
+          | Code Licensed Under GPL v3.0
+        </small>
       </footer>
-      <div class="my-2"></div>
     </div>
-
   </body>
 </html>

proxychain/responsemodifiers/vendor/patch_dynamic_resource_urls.js

@@ -160,6 +160,7 @@
   );
 
   // monkey patch service worker registration
+  /*
   const oldRegister = ServiceWorkerContainer.prototype.register;
   ServiceWorkerContainer.prototype.register = function (scriptURL, options) {
     return oldRegister.call(this, rewriteURL(scriptURL), options);
@@ -169,6 +170,7 @@
     "register",
     "function register() { [native code] }",
   );
+  */
 
   // monkey patch URL.toString() method
   const oldToString = URL.prototype.toString;

proxychain/ruleset/rule.go

@@ -1,11 +1,12 @@
 package ruleset_v2
 
 import (
+	//"bytes"
 	"encoding/json"
 	"fmt"
 
-	"gopkg.in/yaml.v3"
+	//"gopkg.in/yaml.v3"
-	"ladder/proxychain"
+	"github.com/everywall/ladder/proxychain"
 )
 
 type Rule struct {
@@ -128,11 +129,9 @@ func (rule *Rule) MarshalYAML() (interface{}, error) {
 		ResponseModifications []_rsm   `yaml:"responsemodifications"`
 	}
 
-	aux := &Aux{
+	return &Aux{
 		Domains:               rule.Domains,
 		RequestModifications:  rule._rqms,
 		ResponseModifications: rule._rsms,
-	}
+	}, nil
-
-	return yaml.Marshal(aux)
 }

proxychain/ruleset/rule_reqmod_types.gen.go

@@ -1,12 +1,12 @@
-package ruleset_v2
 
+package ruleset_v2
 // DO NOT EDIT THIS FILE. It is automatically generated by ladder/proxychain/codegen/codegen.go
 // The purpose of this is serialization of rulesets from JSON or YAML into functional options suitable
 // for use in proxychains.
 
 import (
-	"ladder/proxychain"
+	"github.com/everywall/ladder/proxychain"
-	rx "ladder/proxychain/requestmodifiers"
+	rx "github.com/everywall/ladder/proxychain/requestmodifiers"
 )
 
 type RequestModifierFactory func(params ...string) proxychain.RequestModification
@@ -16,6 +16,10 @@ var rqmModMap map[string]RequestModifierFactory
 func init() {
 	rqmModMap = make(map[string]RequestModifierFactory)
 
+	  rqmModMap["AddCacheBusterQuery"] = func(_ ...string) proxychain.RequestModification {
+    return rx.AddCacheBusterQuery()
+  }
+
   rqmModMap["ForwardRequestHeaders"] = func(_ ...string) proxychain.RequestModification {
     return rx.ForwardRequestHeaders()
   }
@@ -179,4 +183,5 @@ func init() {
   rqmModMap["SpoofXForwardedFor"] = func(params ...string) proxychain.RequestModification {
     return rx.SpoofXForwardedFor(params[0])
   }
+
 }

proxychain/ruleset/rule_resmod_types.gen.go

@@ -1,12 +1,12 @@
-package ruleset_v2
 
+package ruleset_v2
 // DO NOT EDIT THIS FILE. It is automatically generated by ladder/proxychain/codegen/codegen.go
 // The purpose of this is serialization of rulesets from JSON or YAML into functional options suitable
 // for use in proxychains.
 
 import (
-	"ladder/proxychain"
+	"github.com/everywall/ladder/proxychain"
-	tx "ladder/proxychain/responsemodifiers"
+	tx "github.com/everywall/ladder/proxychain/responsemodifiers"
 )
 
 type ResponseModifierFactory func(params ...string) proxychain.ResponseModification
@@ -24,6 +24,10 @@ func init() {
     return tx.BlockElementRemoval(params[0])
   }
 
+  rsmModMap["BlockThirdPartyScripts"] = func(_ ...string) proxychain.ResponseModification {
+    return tx.BlockThirdPartyScripts()
+  }
+
   rsmModMap["BypassCORS"] = func(_ ...string) proxychain.ResponseModification {
     return tx.BypassCORS()
   }
@@ -36,6 +40,14 @@ func init() {
     return tx.SetContentSecurityPolicy(params[0])
   }
 
+  rsmModMap["DeleteLocalStorageData"] = func(_ ...string) proxychain.ResponseModification {
+    return tx.DeleteLocalStorageData()
+  }
+
+  rsmModMap["DeleteSessionStorageData"] = func(_ ...string) proxychain.ResponseModification {
+    return tx.DeleteSessionStorageData()
+  }
+
   rsmModMap["ForwardResponseHeaders"] = func(_ ...string) proxychain.ResponseModification {
     return tx.ForwardResponseHeaders()
   }
@@ -72,6 +84,10 @@ func init() {
     return tx.SetIncomingCookie(params[0], params[1])
   }
 
+  rsmModMap["ModifyIncomingScriptsWithRegex"] = func(params ...string) proxychain.ResponseModification {
+    return tx.ModifyIncomingScriptsWithRegex(params[0], params[1])
+  }
+
   rsmModMap["SetResponseHeader"] = func(params ...string) proxychain.ResponseModification {
     return tx.SetResponseHeader(params[0], params[1])
   }
@@ -84,10 +100,6 @@ func init() {
     return tx.PatchDynamicResourceURLs()
   }
 
-	rsmModMap["PatchGoogleAnalytics"] = func(_ ...string) proxychain.ResponseModification {
-		return tx.PatchGoogleAnalytics()
-	}
-
   rsmModMap["PatchTrackerScripts"] = func(_ ...string) proxychain.ResponseModification {
     return tx.PatchTrackerScripts()
   }
@@ -95,4 +107,5 @@ func init() {
   rsmModMap["RewriteHTMLResourceURLs"] = func(_ ...string) proxychain.ResponseModification {
     return tx.RewriteHTMLResourceURLs()
   }
+
 }

proxychain/ruleset/rule_test.go

@@ -103,7 +103,6 @@ requestmodifications:
 	if len(rule.RequestModifications) != 1 {
 		t.Errorf("expected number of RequestModifications to be 1, got %d", len(rule.RequestModifications))
 	}
-	fmt.Println(rule.RequestModifications[0].Name)
 }
 
 func TestRuleMarshalYAML(t *testing.T) {

proxychain/ruleset/ruleset.go

@@ -1,32 +1,382 @@
 package ruleset_v2
 
 import (
+	//"bytes"
+	"compress/gzip"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
 	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"encoding/json"
+	"gopkg.in/yaml.v3"
 )
 
 type IRuleset interface {
-	HasRule(url url.URL) bool
+	YAML() (string, error)
-	GetRule(url url.URL) (rule Rule, exists bool)
+	JSON() (string, error)
+	HasRule(url *url.URL) bool
+	GetRule(url *url.URL) (rule *Rule, exists bool)
 }
 
 type Ruleset struct {
-	rulesetPath string
+	Rules    []Rule           `json:"rules" yaml:"rules"`
-	rules       map[string]Rule
+	_rulemap map[string]*Rule // internal map for fast lookups; points at a rule in the Rules slice
 }
 
-func (rs Ruleset) GetRule(url url.URL) (rule Rule, exists bool) {
+func (rs *Ruleset) UnmarshalYAML(unmarshal func(interface{}) error) error {
-	rule, exists = rs.rules[url.Hostname()]
+	type AuxRuleset struct {
+		Rules []Rule `yaml:"rules"`
+	}
+	yamlRuleset := &AuxRuleset{}
+
+	// First, try to unmarshal as AuxRuleset
+	err := unmarshal(yamlRuleset)
+	if err != nil {
+		// If that fails, try to unmarshal directly into a slice of Rules
+		var directRules []Rule
+		if err := unmarshal(&directRules); err != nil {
+			return err
+		}
+		yamlRuleset.Rules = directRules
+	}
+
+	rs._rulemap = make(map[string]*Rule)
+	rs.Rules = yamlRuleset.Rules
+
+	// create a map of pointers to rules loaded above based on domain string keys
+	// this way we don't have two copies of the rule in ruleset
+	for i, rule := range rs.Rules {
+		rulePtr := &rs.Rules[i]
+		for _, domain := range rule.Domains {
+			rs._rulemap[domain] = rulePtr
+			if !strings.HasPrefix(domain, "www.") {
+				rs._rulemap["www."+domain] = rulePtr
+			}
+		}
+	}
+
+	return nil
+}
+
+// MarshalYAML implements the yaml.Marshaler interface.
+// It customizes the marshaling of a Ruleset object into YAML
+func (rs *Ruleset) MarshalYAML() (interface{}, error) {
+
+	type AuxRule struct {
+		Domains               []string `yaml:"domains"`
+		RequestModifications  []_rqm   `yaml:"requestmodifications"`
+		ResponseModifications []_rsm   `yaml:"responsemodifications"`
+	}
+
+	type Aux struct {
+		Rules []AuxRule `yaml:"rules"`
+	}
+
+	aux := Aux{}
+
+	for _, rule := range rs.Rules {
+		auxRule := AuxRule{
+			Domains:               rule.Domains,
+			RequestModifications:  rule._rqms,
+			ResponseModifications: rule._rsms,
+		}
+		aux.Rules = append(aux.Rules, auxRule)
+	}
+	return aux, nil
+
+	/*
+		var b bytes.Buffer
+		y := yaml.NewEncoder(&b)
+		y.SetIndent(2)
+		err := y.Encode(&aux)
+
+		return b.String(), err
+	*/
+}
+
+// ==========================================================
+
+func (rs *Ruleset) UnmarshalJSON(data []byte) error {
+	type AuxRuleset struct {
+		Rules []Rule `json:"rules"`
+	}
+	ar := &AuxRuleset{}
+
+	if err := json.Unmarshal(data, ar); err != nil {
+		return err
+	}
+
+	rs._rulemap = make(map[string]*Rule)
+	rs.Rules = ar.Rules
+
+	for i, rule := range rs.Rules {
+		rulePtr := &rs.Rules[i]
+		for _, domain := range rule.Domains {
+			rs._rulemap[domain] = rulePtr
+			if !strings.HasPrefix(domain, "www.") {
+				rs._rulemap["www."+domain] = rulePtr
+			}
+		}
+	}
+
+	return nil
+}
+
+func (rs *Ruleset) MarshalJSON() ([]byte, error) {
+	type AuxRule struct {
+		Domains               []string `json:"domains"`
+		RequestModifications  []_rqm   `json:"requestmodifications"`
+		ResponseModifications []_rsm   `json:"responsemodifications"`
+	}
+
+	type Aux struct {
+		Rules []AuxRule `json:"rules"`
+	}
+
+	aux := Aux{}
+	for _, rule := range rs.Rules {
+		auxRule := AuxRule{
+			Domains:               rule.Domains,
+			RequestModifications:  rule._rqms,
+			ResponseModifications: rule._rsms,
+		}
+		aux.Rules = append(aux.Rules, auxRule)
+	}
+
+	return json.Marshal(aux)
+}
+
+// ===========================================================
+
+func (rs Ruleset) GetRule(url *url.URL) (rule *Rule, exists bool) {
+	rule, exists = rs._rulemap[url.Hostname()]
 	return rule, exists
 }
 
-func (rs Ruleset) HasRule(url url.URL) bool {
+func (rs Ruleset) HasRule(url *url.URL) bool {
 	_, exists := rs.GetRule(url)
 	return exists
 }
 
+// NewRuleset loads a new RuleSet from a path
 func NewRuleset(path string) (Ruleset, error) {
 	rs := Ruleset{
-		rulesetPath: path,
+		_rulemap: map[string]*Rule{},
+		Rules:    []Rule{},
 	}
-	return rs, nil
+
+	switch {
+	case strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://"):
+		err := rs.loadRulesFromRemoteFile(path)
+		return rs, err
+	default:
+		err := rs.loadRulesFromLocalDir(path)
+		return rs, err
+	}
+}
+
+// NewRulesetFromEnv creates a new RuleSet based on the RULESET environment variable.
+// It logs a warning and returns an empty RuleSet if the RULESET environment variable is not set.
+// If the RULESET is set but the rules cannot be loaded, it panics.
+func NewRulesetFromEnv() Ruleset {
+	rulesPath, ok := os.LookupEnv("RULESET")
+	if !ok {
+		log.Printf("WARN: No ruleset specified. Set the `RULESET` environment variable to load one for a better success rate.")
+		return Ruleset{}
+	}
+
+	ruleSet, err := NewRuleset(rulesPath)
+	if err != nil {
+		log.Println(err)
+	}
+
+	return ruleSet
+}
+
+// loadRulesFromLocalDir loads rules from a local directory specified by the path.
+// It walks through the directory, loading rules from YAML files.
+// Returns an error if the directory cannot be accessed
+// If there is an issue loading any file, it will be skipped
+func (rs *Ruleset) loadRulesFromLocalDir(path string) error {
+	_, err := os.Stat(path)
+	if err != nil {
+		return fmt.Errorf("loadRulesFromLocalDir: invalid path - %s", err)
+	}
+
+	err = filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+
+		isYAML := filepath.Ext(path) == ".yaml" || filepath.Ext(path) == ".yml"
+		if !isYAML {
+			return nil
+		}
+		fmt.Printf("loadRulesFromLocalDir :: loading rule: %s\n", path)
+
+		tmpRs := Ruleset{_rulemap: make(map[string]*Rule)}
+		err = tmpRs.loadRulesFromLocalFile(path)
+		if err != nil {
+			log.Printf("WARN: failed to load directory ruleset '%s': %s, skipping", path, err)
+			return nil
+		}
+		rs.Rules = append(rs.Rules, tmpRs.Rules...)
+
+		//log.Printf("INFO: loaded ruleset %s\n", path)
+
+		return nil
+	})
+
+	// create a map of pointers to rules loaded above based on domain string keys
+	// this way we don't have two copies of the rule in ruleset
+	if rs._rulemap == nil {
+		rs._rulemap = make(map[string]*Rule)
+	}
+	for i, rule := range rs.Rules {
+		rulePtr := &rs.Rules[i]
+		for _, domain := range rule.Domains {
+			rs._rulemap[domain] = rulePtr
+			if !strings.HasPrefix(domain, "www.") {
+				rs._rulemap["www."+domain] = rulePtr
+			}
+		}
+	}
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// loadRulesFromLocalFile loads rules from a local YAML file specified by the path.
+// Returns an error if the file cannot be read or if there's a syntax error in the YAML.
+func (rs *Ruleset) loadRulesFromLocalFile(path string) error {
+	file, err := os.ReadFile(path)
+	if err != nil {
+		e := fmt.Errorf("failed to read rules from local file: '%s'", path)
+		return errors.Join(e, err)
+	}
+	fmt.Printf("loadRulesFromLocalFile :: %s\n", path)
+
+	isJSON := strings.HasSuffix(path, ".json")
+	if isJSON {
+		err = json.Unmarshal(file, rs)
+	} else {
+		err = yaml.Unmarshal(file, rs)
+	}
+
+	if err != nil {
+		e := fmt.Errorf("failed to load rules from local file, possible syntax error in '%s' - %s", path, err)
+		debugPrintRule(string(file), e)
+		return e
+	}
+
+	return nil
+}
+
+// loadRulesFromRemoteFile loads rules from a remote URL.
+// It supports plain and gzip compressed content.
+// Returns an error if there's an issue accessing the URL or if there's a syntax error in the YAML.
+func (rs *Ruleset) loadRulesFromRemoteFile(rulesURL string) error {
+
+	resp, err := http.Get(rulesURL)
+	if err != nil {
+		return fmt.Errorf("failed to load rules from remote url '%s' - %s", rulesURL, err)
+	}
+
+	defer resp.Body.Close()
+
+	if resp.StatusCode >= 400 {
+		return fmt.Errorf("failed to load rules from remote url (%s) on '%s' - %s", resp.Status, rulesURL, err)
+	}
+
+	var reader io.Reader
+
+	// in case remote server did not set content-encoding gzip header
+	isGzip := strings.HasSuffix(rulesURL, ".gz") || strings.HasSuffix(rulesURL, ".gzip") || resp.Header.Get("content-encoding") == "gzip"
+	if isGzip {
+		reader, err = gzip.NewReader(resp.Body)
+
+		if err != nil {
+			return fmt.Errorf("failed to create gzip reader for URL '%s' with status code '%s': %w", rulesURL, resp.Status, err)
+		}
+	} else {
+		reader = resp.Body
+	}
+
+	isJSON := strings.HasSuffix(rulesURL, ".json") || resp.Header.Get("content-type") == "application/json"
+	if isJSON {
+		err = json.NewDecoder(reader).Decode(&rs)
+	} else {
+		err = yaml.NewDecoder(reader).Decode(&rs)
+	}
+
+	if err != nil {
+		return fmt.Errorf("failed to load rules from remote url '%s' with status code '%s' and possible syntax error - %s", rulesURL, resp.Status, err)
+	}
+
+	return nil
+}
+
+// ================= utility methods ==========================
+
+// YAML returns the ruleset as a Yaml string
+func (rs Ruleset) YAML() (string, error) {
+	yml, err := yaml.Marshal(&rs)
+	if err != nil {
+		return "", err
+	}
+	return string(yml), nil
+}
+
+// JSON returns the ruleset as a JSON string
+func (rs Ruleset) JSON() (string, error) {
+	jsn, err := json.Marshal(&rs)
+	if err != nil {
+		return "", err
+	}
+	return string(jsn), nil
+}
+
+// Domains extracts and returns a slice of all domains present in the RuleSet.
+func (rs *Ruleset) Domains() []string {
+	var domains []string
+	for _, rule := range rs.Rules {
+		domains = append(domains, rule.Domains...)
+	}
+	return domains
+}
+
+// DomainCount returns the count of unique domains present in the RuleSet.
+func (rs *Ruleset) DomainCount() int {
+	return len(rs.Domains())
+}
+
+// Count returns the total number of rules in the RuleSet.
+func (rs *Ruleset) Count() int {
+	return len(rs.Rules)
+}
+
+// PrintStats logs the number of rules and domains loaded in the RuleSet.
+func (rs *Ruleset) PrintStats() {
+	log.Printf("INFO: Loaded %d rules for %d domains\n", rs.Count(), rs.DomainCount())
+}
+
+// debugPrintRule is a utility function for printing a rule and associated error for debugging purposes.
+func debugPrintRule(rule string, err error) {
+	fmt.Println("------------------------------ BEGIN DEBUG RULESET -----------------------------")
+	fmt.Printf("%s\n", err.Error())
+	fmt.Println("--------------------------------------------------------------------------------")
+	fmt.Println(rule)
+	fmt.Println("------------------------------ END DEBUG RULESET -------------------------------")
 }

proxychain/ruleset/ruleset_test.go

@@ -0,0 +1,214 @@
+package ruleset_v2
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+
+	//"strings"
+	"testing"
+	"time"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/stretchr/testify/assert"
+	//"gopkg.in/yaml.v3"
+)
+
+var (
+	validYAML = `rules:
+  - domains:
+    - example.com
+    - www.example.com
+    responsemodifications:
+    - name: APIContent
+      params: []
+    - name: SetContentSecurityPolicy
+      params:
+      - foobar
+    - name: SetIncomingCookie
+      params:
+      - authorization-bearer
+      - hunter2
+    requestmodifications:
+    - name: ForwardRequestHeaders
+      params: []
+`
+
+	invalidYAML = `
+rules:
+  domains:
+  - example.com
+  - www.example.com
+  responsemodifications:
+  - name: APIContent
+  - name: SetContentSecurityPolicy
+  - name: INVALIDSetIncomingCookie
+    params:
+    - authorization-bearer
+    - hunter2
+  requestmodifications:
+  - name: ForwardRequestHeaders
+    params: []
+`
+)
+
+func TestYAMLUnmarshal(t *testing.T) {
+	rs, err := loadRuleFromString(validYAML)
+	fmt.Println(validYAML)
+	assert.NoError(t, err, "expected no error loading valid yml")
+	yml, err := rs.YAML()
+	assert.NoError(t, err, "expected no error marshalling ruleset")
+
+	rs2, err := loadRuleFromString(yml)
+	assert.NoError(t, err, "expected no error loading yaml marshalled -> unmarshalled -> marshalled ruleset")
+	assert.Equal(t, 1, rs2.Count(), "expected one rule to be returned after marshalled -> unmarshalled -> marshalled ruleset")
+}
+
+func TestJSONUnmarshal(t *testing.T) {
+	rs, err := loadRuleFromString(validYAML)
+	assert.NoError(t, err, "expected no error loading valid yml")
+	j, err := json.Marshal(&rs)
+	assert.NoError(t, err, "expected no error marshalling ruleset to json")
+
+	fmt.Println(string(j))
+
+	rs2, err := loadRuleFromString(string(j))
+	assert.NoError(t, err, "expected no error loading JSON marshalled -> unmarshalled -> marshalled ruleset")
+
+	assert.Equal(t, 1, rs2.Count(), "expected one rule to be returned after JSON marshalled -> unmarshalled -> marshalled ruleset")
+}
+
+func TestLoadRulesFromRemoteFile(t *testing.T) {
+	app := fiber.New()
+	defer app.Shutdown()
+
+	app.Get("/valid-config.yml", func(c *fiber.Ctx) error {
+		c.SendString(validYAML)
+		return nil
+	})
+
+	app.Get("/invalid-config.yml", func(c *fiber.Ctx) error {
+		c.SendString(invalidYAML)
+		return nil
+	})
+
+	// Start the server in a goroutine
+	go func() {
+		if err := app.Listen("127.0.0.1:9999"); err != nil {
+			t.Errorf("Server failed to start: %s", err.Error())
+		}
+	}()
+
+	// Wait for the server to start
+	time.Sleep(time.Second * 1)
+
+	rs, err := NewRuleset("http://127.0.0.1:9999/valid-config.yml")
+	if err != nil {
+		t.Errorf("failed to load plaintext ruleset from http server: %s", err.Error())
+	}
+
+	u, _ := url.Parse("http://example.com")
+	r, exists := rs.GetRule(u)
+	assert.True(t, exists, "expected example.com rule to be present")
+	assert.Equal(t, r.Domains[0], "example.com")
+
+	u, _ = url.Parse("http://www.www.foobar.com")
+	_, exists = rs.GetRule(u)
+	assert.False(t, exists, "expected www.www.foobar.com rule to NOT be present")
+
+	u, _ = url.Parse("http://example.com")
+	r, exists = rs.GetRule(u)
+	assert.Equal(t, r.Domains[0], "example.com")
+
+	os.Setenv("RULESET", "http://127.0.0.1:9999/valid-config.yml")
+
+	rs = NewRulesetFromEnv()
+	r, exists = rs.GetRule(u)
+	assert.True(t, exists, "expected example.com rule to be present from env")
+	if !assert.Equal(t, r.Domains[0], "example.com") {
+		t.Error("expected no errors loading ruleset from url using environment variable, but got one")
+	}
+}
+
+func loadRuleFromString(yamlOrJSON string) (Ruleset, error) {
+	// Create a temporary file and load it
+	var tmpFile *os.File
+	if strings.HasPrefix(yamlOrJSON, "{") {
+		tmpFile, _ = os.CreateTemp("", "ruleset*.json")
+	} else {
+		tmpFile, _ = os.CreateTemp("", "ruleset*.yaml")
+	}
+
+	defer os.Remove(tmpFile.Name())
+
+	tmpFile.WriteString(yamlOrJSON)
+
+	rs := Ruleset{
+		_rulemap: map[string]*Rule{},
+		Rules:    []Rule{},
+	}
+	err := rs.loadRulesFromLocalFile(tmpFile.Name())
+
+	return rs, err
+}
+
+// TestLoadRulesFromLocalFile tests the loading of rules from a local YAML file.
+func TestLoadRulesFromLocalFile(t *testing.T) {
+	_, err := loadRuleFromString(validYAML)
+	if err != nil {
+		t.Errorf("Failed to load rules from valid YAML: %s", err)
+	}
+
+	_, err = loadRuleFromString(invalidYAML)
+	if err == nil {
+		t.Errorf("Expected an error when loading invalid YAML, but got none")
+	}
+}
+
+// TestLoadRulesFromLocalDir tests the loading of rules from a local nested directory full of yaml rulesets
+func TestLoadRulesFromLocalDir(t *testing.T) {
+	// Create a temporary directory
+	baseDir, err := os.MkdirTemp("", "ruleset_test")
+	if err != nil {
+		t.Fatalf("Failed to create temporary directory: %s", err)
+	}
+
+	defer os.RemoveAll(baseDir)
+
+	// Create a nested subdirectory
+	nestedDir := filepath.Join(baseDir, "nested")
+	err = os.Mkdir(nestedDir, 0o755)
+
+	if err != nil {
+		t.Fatalf("Failed to create nested directory: %s", err)
+	}
+
+	// Create a nested subdirectory
+	nestedTwiceDir := filepath.Join(nestedDir, "nestedTwice")
+	err = os.Mkdir(nestedTwiceDir, 0o755)
+	if err != nil {
+		t.Fatalf("Failed to create twice-nested directory: %s", err)
+	}
+
+	testCases := []string{"test.yaml", "test2.yaml", "test-3.yaml", "test 4.yaml", "1987.test.yaml.yml", "foobar.example.com.yaml", "foobar.com.yml"}
+	for _, fileName := range testCases {
+		filePath := filepath.Join(nestedDir, "2x-"+fileName)
+		os.WriteFile(filePath, []byte(validYAML), 0o644)
+
+		filePath = filepath.Join(nestedDir, fileName)
+		os.WriteFile(filePath, []byte(validYAML), 0o644)
+
+		filePath = filepath.Join(baseDir, "base-"+fileName)
+		os.WriteFile(filePath, []byte(validYAML), 0o644)
+	}
+
+	rs := Ruleset{}
+	fmt.Println(baseDir)
+	err = rs.loadRulesFromLocalDir(baseDir)
+
+	assert.NoError(t, err)
+	assert.Equal(t, len(testCases)*3, rs.Count())
+}

proxychain/ruleset/todo.md

@@ -0,0 +1 @@
+ruleset loading rule tests are failing; maybe concurrency issue with assigning to nil map?

ruleset_v2.yaml

@@ -0,0 +1,36 @@
+rules:
+  - domains:
+    - example.com
+    - www.example.com
+    responsemodifications:
+    - name: APIContent
+      params: []
+    - name: SetContentSecurityPolicy
+      params:
+      - foobar
+    - name: SetIncomingCookie
+      params:
+      - authorization-bearer
+      - hunter2
+    requestmodifications:
+    - name: ForwardRequestHeaders
+      params: []
+
+  - domains:
+    - quantamagzine.org
+    responsemodifications:
+    - name: BlockElementRemoval
+      params:
+      - "#postContent"
+
+  - domains:
+    - techcrunch.com
+    responsemodifications:
+    - name: ModifyIncomingScriptsWithRegex
+      params:
+      - "window\\.location"
+      - |
+        {origin: "techcrunch.com"} 
+    - name: BlockElementRemoval
+      params:
+      - ".article-content"

rulesets_v2/BE_Belgium/demorgen-be.yaml

@@ -0,0 +1,23 @@
+- domains:
+  - demorgen.be
+
+  requestmodifiers:
+    - name: MasqueradeAsGoogleBot
+
+    - name: SpoofReferrer
+      params: ["https://news.google.com"]
+
+    - name: SetOutgoingCookie
+      params: ["isBot", "true"]
+
+    - name: SetOutgoingCookie
+      params: ["authId", "1"]
+
+  responsemodifiers:
+    - name: BypassContentSecurityPolicy
+    - name: InjectScriptAfterDOMContentLoaded
+      params:
+        - | 
+            let paywall = document.querySelectorAll('script[src*="advertising-cdn.dpgmedia.cloud"], div[data-temptation-position="ARTICLE_BOTTOM"]');
+            paywall.forEach(el => { el.remove(); });
+            document.querySelector('div[data-advert-placeholder-collapses]').remove();

rulesets_v2/BE_Belgium/dpg-media.yaml

@@ -0,0 +1,44 @@
+- domains:
+  - myprivacy.dpgmedia.be
+  - myprivacy.dpgmedia.nl
+
+  requestmodifiers:
+    - name: SpoofReferrer
+      params: ["https://news.google.com"]
+
+    - name: SetOutgoingCookie
+      params: ["isBot", "true"]
+
+    - name: SetOutgoingCookie
+      params: ["authId", "1"]
+
+    - name: SpoofXForwardedFor
+      params: ["none"]
+
+
+- domains:
+  - demorgen.be
+
+  requestmodifiers:
+    - name: MasqueradeAsGoogleBot
+
+    - name: SpoofReferrer
+      params: ["https://news.google.com"]
+
+    - name: SetOutgoingCookie
+      params: ["isBot", "true"]
+
+    - name: SetOutgoingCookie
+      params: ["authId", "1"]
+
+    - name: SpoofXForwardedFor
+      params: ["none"]
+
+  responsemodifiers:
+    - name: BypassContentSecurityPolicy
+    - name: InjectScriptAfterDOMContentLoaded
+      params:
+        - | 
+            let paywall = document.querySelectorAll('script[src*="advertising-cdn.dpgmedia.cloud"], div[data-temptation-position="ARTICLE_BOTTOM"]');
+            paywall.forEach(el => { el.remove(); });
+            document.querySelector('div[data-advert-placeholder-collapses]').remove();

rulesets_v2/CA_Canada/_multi-metroland-media-group.yaml

@@ -0,0 +1,34 @@
+rules:
+  - domains: 
+    - thestar.com
+    - niagarafallsreview.ca
+    - stcatharinesstandard.ca
+    - thepeterboroughexaminer.com
+    - therecord.com
+    - thespec.com
+    - wellandtribune.ca
+    responsemodifications:
+      - name: DeleteLocalStorageData
+      - name: DeleteSessionStorageData
+      - name: InjectScriptAfterDOMContentLoaded
+        params:
+          - |
+              const paywall = document.querySelectorAll('div.subscriber-offers');
+              paywall.forEach(el => { el.remove(); });
+              const subscriber_only = document.querySelectorAll('div.subscriber-only');
+              for (const elem of subscriber_only) {
+                if (elem.classList.contains('encrypted-content') && dompurify_loaded) {
+                  const parser = new DOMParser();
+                  const doc = parser.parseFromString('<div>' + DOMPurify.sanitize(unscramble(elem.innerText)) + '</div>', 'text/html');
+                  const content_new = doc.querySelector('div');
+                  elem.parentNode.replaceChild(content_new, elem);
+                }
+                elem.removeAttribute('style');
+                elem.removeAttribute('class');
+              }
+              const banners = document.querySelectorAll('div.subscription-required, div.redacted-overlay, div.subscriber-hide, div.tnt-ads-container');
+              banners.forEach(el => { el.remove(); });
+              const ads = document.querySelectorAll('div.tnt-ads-container, div[class*="adLabelWrapper"]');
+              ads.forEach(el => { el.remove(); });
+              const recommendations = document.querySelectorAll('div[id^="tncms-region-article"]');
+              recommendations.forEach(el => { el.remove(); });