Merge pull request #64 from lutfuahmet/main

HTTP Timeout

.air.toml

@@ -0,0 +1,46 @@
+root = "./"
+testdata_dir = "testdata"
+tmp_dir = "tmp"
+
+[build]
+  args_bin = []
+  bin = "./tmp/main"
+  cmd = "go build -o ./tmp/main ./cmd"
+  delay = 1000
+  exclude_dir = ["assets", "tmp", "vendor", "testdata"]
+  exclude_file = []
+  exclude_regex = ["_test.go"]
+  exclude_unchanged = false
+  follow_symlink = false
+  full_bin = "RULESET=./ruleset.yaml ./tmp/main"
+  include_dir = []
+  include_ext = ["go", "tpl", "tmpl", "yaml", "html"]
+  include_file = []
+  kill_delay = "0s"
+  log = "build-errors.log"
+  poll = false
+  poll_interval = 0
+  post_cmd = []
+  pre_cmd = ["echo 'dev' > handlers/VERSION"]
+  rerun = false
+  rerun_delay = 500
+  send_interrupt = false
+  stop_on_error = false
+
+[color]
+  app = ""
+  build = "yellow"
+  main = "magenta"
+  runner = "green"
+  watcher = "cyan"
+
+[log]
+  main_only = false
+  time = false
+
+[misc]
+  clean_on_exit = true
+
+[screen]
+  clear_on_rebuild = true
+  keep_scroll = true

.github/workflows/build-css.yaml

@@ -0,0 +1,42 @@
+name: Build Tailwind CSS
+
+on:
+  push:
+    paths:
+      - "handlers/form.html"
+  workflow_dispatch:
+
+jobs:
+  tailwindbuilder:
+    permissions:
+      # Give the default GITHUB_TOKEN write permission to commit and push the
+      # added or changed files to the repository.
+      contents: write
+      
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Install pnpm
+        uses: pnpm/action-setup@v2
+        with:
+          version: 8
+      -
+        name: Build Tailwind CSS
+        run: pnpm build
+      -
+        name: Commit generated stylesheet
+        run: |
+          if git diff --quiet cmd/styles.css; then
+            echo "No changes to commit."
+            exit 0
+          else
+            echo "Changes detected, committing..."
+            git config --global user.name "Github action"
+            git config --global user.email "username@users.noreply.github.com"
+            git add cmd
+            git commit -m "Generated stylesheet"
+            git push
+          fi

.gitignore

@@ -1,4 +1,5 @@
 # dev binary
 ladder
 
-VERSION
+VERSION
+output.css

.golangci-lint.yaml

@@ -0,0 +1,24 @@
+linters:
+  enable:
+    - errcheck
+    - gosimple
+    - govet
+    - ineffassign
+    - staticcheck
+    - unused
+    - cyclop
+    - dupword
+    - wsl
+    - varnamelen
+    - usestdlibvars
+    - unparam
+    - revive
+    - prealloc
+    - misspell
+    - gocyclo
+    - funlen
+    - bodyclose
+
+linters-settings:
+  cyclop:
+    max-complexity: 15

Makefile

@@ -0,0 +1,10 @@
+lint:
+	gofumpt -l -w .
+	golangci-lint run -c .golangci-lint.yaml --fix
+
+	go mod tidy
+	go clean
+
+install-linters:
+	go install mvdan.cc/gofumpt@latest
+	go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.55.2

README.md

@@ -14,10 +14,22 @@ Freedom of information is an essential pillar of democracy and informed decision
 
 > **Disclaimer:** This project is intended for educational purposes only. The author does not endorse or encourage any unethical or illegal activity. Use this tool at your own risk.
 
+### How it works
+
+```mermaid
+sequenceDiagram
+    client->>+ladder: GET
+    ladder-->>ladder: apply RequestModifications
+    ladder->>+website: GET
+    website->>-ladder: 200 OK
+    ladder-->>ladder: apply ResultModifications
+    ladder->>-client: 200 OK
+```
+
 ### Features
 - [x] Bypass Paywalls
 - [x] Remove CORS headers from responses, assets, and images ...
-- [x] Apply domain based ruleset/code to modify response
+- [x] Apply domain based ruleset/code to modify response / requested URL
 - [x] Keep site browsable
 - [x] API
 - [x] Fetch RAW HTML
@@ -33,27 +45,27 @@ Freedom of information is an essential pillar of democracy and informed decision
 - [x] No Tracking
 - [x] Limit the proxy to a list of domains
 - [x] Expose Ruleset to other ladders
+- [x] Fetch from Google Cache
 - [ ] Optional TOR proxy
 - [ ] A key to share only one URL
-- [ ] Fetch from Google Cache if not available
 
 ### Limitations
-Certain sites may display missing images or encounter formatting issues. This can be attributed to the site's reliance on JavaScript or CSS for image and resource loading, which presents a limitation when accessed through this proxy. If you prefer a full experience, please consider buying a subscription for the site.
-
 Some sites do not expose their content to search engines, which means that the proxy cannot access the content. A future version will try to fetch the content from Google Cache.
 
+Certain sites may display missing images or encounter formatting issues. This can be attributed to the site's reliance on JavaScript or CSS for image and resource loading, which presents a limitation when accessed through this proxy. If you prefer a full experience, please consider buying a subscription for the site.
+
 ## Installation
 
 > **Warning:** If your instance will be publicly accessible, make sure to enable Basic Auth. This will prevent unauthorized users from using your proxy. If you do not enable Basic Auth, anyone can use your proxy to browse nasty/illegal stuff. And you will be responsible for it.
 
 ### Binary
 1) Download binary [here](https://github.com/everywall/ladder/releases/latest)
-2) Unpack and run the binary `./ladder`
+2) Unpack and run the binary `./ladder -r https://t.ly/14PSf`
 3) Open Browser (Default: http://localhost:8080)
 
 ### Docker
 ```bash
-docker run -p 8080:8080 -d --name ladder ghcr.io/everywall/ladder:latest
+docker run -p 8080:8080 -d --env RULESET=https://t.ly/14PSf --name ladder ghcr.io/everywall/ladder:latest
 ```
 
 ### Docker Compose
@@ -106,7 +118,7 @@ http://localhost:8080/ruleset
 | `LOG_URLS` | Log fetched URL's | `true` |
 | `DISABLE_FORM` | Disables URL Form Frontpage | `false` |
 | `FORM_PATH` | Path to custom Form HTML | `` |
-| `RULESET` | URL to a ruleset file | `https://raw.githubusercontent.com/everywall/ladder/main/ruleset.yaml` or `/path/to/my/rules.yaml` |
+| `RULESET` | Path or URL to a ruleset file, accepts local directories | `https://raw.githubusercontent.com/everywall/ladder-rules/main/ruleset.yaml` or `/path/to/my/rules.yaml` or `/path/to/my/rules/` |
 | `EXPOSE_RULESET` | Make your Ruleset available to other ladders | `true` |
 | `ALLOWED_DOMAINS` | Comma separated list of allowed domains. Empty = no limitations | `` |
 | `ALLOWED_DOMAINS_RULESET` | Allow Domains from Ruleset. false = no limitations | `false` |
@@ -115,12 +127,13 @@ http://localhost:8080/ruleset
 
 ### Ruleset
 
-It is possible to apply custom rules to modify the response. This can be used to remove unwanted or modify elements from the page. The ruleset is a YAML file that contains a list of rules for each domain and is loaded on startup
+It is possible to apply custom rules to modify the response or the requested URL. This can be used to remove unwanted or modify elements from the page. The ruleset is a YAML file, a directory with YAML Files, or an URL to a YAML file that contains a list of rules for each domain. These rules are loaded on startup.
+
+There is a basic ruleset available in a separate repository [ruleset.yaml](https://raw.githubusercontent.com/everywall/ladder-rules/main/ruleset.yaml). Feel free to add your own rules and create a pull request.
 
-See in [ruleset.yaml](ruleset.yaml) for an example.
 
 ```yaml
-- domain: example.com          # Inbcludes all subdomains
+- domain: example.com          # Includes all subdomains
   domains:                     # Additional domains to apply the rule
     - www.example.de
     - www.beispiel.de
@@ -154,5 +167,40 @@ See in [ruleset.yaml](ruleset.yaml) for an example.
         <h1>My Custom Title</h1>
     - position: .left-content article # Position where to inject the code into DOM
       prepend: | 
-        <h2>Suptitle</h2>
+        <h2>Subtitle</h2>
+- domain: demo.com
+  headers:
+    content-security-policy: script-src 'self';
+    user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36
+  urlMods:              # Modify the URL
+    query:              
+      - key: amp        # (this will append ?amp=1 to the URL)
+        value: 1 
+    domain:             
+      - match: www      # regex to match part of domain
+        replace: amp    # (this would modify the domain from www.demo.de to amp.demo.de)
+    path:               
+      - match: ^        # regex to match part of path
+        replace: /amp/  # (modify the url from https://www.demo.com/article/ to https://www.demo.de/amp/article/)
 ```
+
+## Development
+
+To run a development server at http://localhost:8080:
+
+```bash
+echo "dev" > handlers/VERSION
+RULESET="./ruleset.yaml" go run cmd/main.go
+```
+
+### Optional: Live reloading development server with [cosmtrek/air](https://github.com/cosmtrek/air)
+
+Install air according to the [installation instructions](https://github.com/cosmtrek/air#installation). 
+
+Run a development server at http://localhost:8080:
+
+```bash
+air # or the path to air if you haven't added a path alias to your .bashrc or .zshrc
+```
+
+This project uses [pnpm](https://pnpm.io/) to build a stylesheet with the [Tailwind CSS](https://tailwindcss.com/) classes. For local development, if you modify styles in `form.html`, run `pnpm build` to generate a new stylesheet.

cmd/main.go

@@ -1,14 +1,15 @@
 package main
 
 import (
-	_ "embed"
+	"embed"
 	"fmt"
-
-	"ladder/handlers"
 	"log"
 	"os"
 	"strings"
 
+	"ladder/handlers"
+	"ladder/handlers/cli"
+
 	"github.com/akamensky/argparse"
 	"github.com/gofiber/fiber/v2"
 	"github.com/gofiber/fiber/v2/middleware/basicauth"
@@ -18,28 +19,74 @@ import (
 //go:embed favicon.ico
 var faviconData string
 
-func main() {
+//go:embed styles.css
+var cssData embed.FS
 
+func main() {
 	parser := argparse.NewParser("ladder", "Every Wall needs a Ladder")
 
 	portEnv := os.Getenv("PORT")
 	if os.Getenv("PORT") == "" {
 		portEnv = "8080"
 	}
+
 	port := parser.String("p", "port", &argparse.Options{
 		Required: false,
 		Default:  portEnv,
-		Help:     "Port the webserver will listen on"})
+		Help:     "Port the webserver will listen on",
+	})
 
 	prefork := parser.Flag("P", "prefork", &argparse.Options{
 		Required: false,
-		Help:     "This will spawn multiple processes listening"})
+		Help:     "This will spawn multiple processes listening",
+	})
+
+	ruleset := parser.String("r", "ruleset", &argparse.Options{
+		Required: false,
+		Help:     "File, Directory or URL to a ruleset.yaml. Overrides RULESET environment variable.",
+	})
+
+	mergeRulesets := parser.Flag("", "merge-rulesets", &argparse.Options{
+		Required: false,
+		Help:     "Compiles a directory of yaml files into a single ruleset.yaml. Requires --ruleset arg.",
+	})
+
+	mergeRulesetsGzip := parser.Flag("", "merge-rulesets-gzip", &argparse.Options{
+		Required: false,
+		Help:     "Compiles a directory of yaml files into a single ruleset.gz Requires --ruleset arg.",
+	})
+
+	mergeRulesetsOutput := parser.String("", "merge-rulesets-output", &argparse.Options{
+		Required: false,
+		Help:     "Specify output file for --merge-rulesets and --merge-rulesets-gzip. Requires --ruleset and --merge-rulesets args.",
+	})
 
 	err := parser.Parse(os.Args)
 	if err != nil {
 		fmt.Print(parser.Usage(err))
 	}
 
+	// utility cli flag to compile ruleset directory into single ruleset.yaml
+	if *mergeRulesets || *mergeRulesetsGzip {
+		output := os.Stdout
+
+		if *mergeRulesetsOutput != "" {
+			output, err = os.Create(*mergeRulesetsOutput)
+			
+			if err != nil {
+				fmt.Println(err)
+				os.Exit(1)
+			}
+		}
+
+		err = cli.HandleRulesetMerge(*ruleset, *mergeRulesets, *mergeRulesetsGzip, output)
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+		os.Exit(0)
+	}
+
 	if os.Getenv("PREFORK") == "true" {
 		*prefork = true
 	}
@@ -54,6 +101,7 @@ func main() {
 	userpass := os.Getenv("USERPASS")
 	if userpass != "" {
 		userpass := strings.Split(userpass, ":")
+
 		app.Use(basicauth.New(basicauth.Config{
 			Users: map[string]string{
 				userpass[0]: userpass[1],
@@ -69,18 +117,28 @@ func main() {
 	if os.Getenv("NOLOGS") != "true" {
 		app.Use(func(c *fiber.Ctx) error {
 			log.Println(c.Method(), c.Path())
+
 			return c.Next()
 		})
 	}
 
 	app.Get("/", handlers.Form)
-	app.Get("ruleset", handlers.Ruleset)
 
+	app.Get("/styles.css", func(c *fiber.Ctx) error {
+		cssData, err := cssData.ReadFile("styles.css")
+		if err != nil {
+			return c.Status(fiber.StatusInternalServerError).SendString("Internal Server Error")
+		}
+
+		c.Set("Content-Type", "text/css")
+
+		return c.Send(cssData)
+	})
+
+	app.Get("ruleset", handlers.Ruleset)
 	app.Get("raw/*", handlers.Raw)
 	app.Get("api/*", handlers.Api)
-	app.Get("ruleset", handlers.Raw)
+	app.Get("/*", handlers.ProxySite(*ruleset))
-	app.Get("/*", handlers.ProxySite)
 
 	log.Fatal(app.Listen(":" + *port))
-
 }

docker-compose.yaml

@@ -9,10 +9,11 @@ services:
     environment:
       - PORT=8080
       - RULESET=/app/ruleset.yaml
+      #- ALLOWED_DOMAINS=example.com,example.org
       #- ALLOWED_DOMAINS_RULESET=false
       #- EXPOSE_RULESET=true
       #- PREFORK=false
-      #- DISABLE_FORM=fase
+      #- DISABLE_FORM=false
       #- FORM_PATH=/app/form.html
       #- X_FORWARDED_FOR=66.249.66.1
       #- USER_AGENT=Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)

go.mod

@@ -2,17 +2,17 @@ module ladder
 
 go 1.21.1
 
-require github.com/gofiber/fiber/v2 v2.50.0
-
-require (
-	github.com/andybalholm/cascadia v1.3.1 // indirect
-	golang.org/x/net v0.8.0 // indirect
-)
-
 require (
 	github.com/PuerkitoBio/goquery v1.8.1
 	github.com/akamensky/argparse v1.4.0
+	github.com/gofiber/fiber/v2 v2.50.0
+	github.com/stretchr/testify v1.8.4
+	gopkg.in/yaml.v3 v3.0.1
+)
+
+require (
 	github.com/andybalholm/brotli v1.0.6 // indirect
+	github.com/andybalholm/cascadia v1.3.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/google/uuid v1.4.0 // indirect
 	github.com/klauspost/compress v1.17.2 // indirect
@@ -21,11 +21,10 @@ require (
 	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rivo/uniseg v0.4.4 // indirect
-	github.com/stretchr/objx v0.5.0 // indirect
-	github.com/stretchr/testify v1.8.4 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasthttp v1.50.0 // indirect
 	github.com/valyala/tcplisten v1.0.0 // indirect
-	golang.org/x/sys v0.13.0 // indirect
+	golang.org/x/net v0.18.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
+	golang.org/x/sys v0.14.0 // indirect
+	golang.org/x/term v0.14.0
 )

go.sum

@@ -4,9 +4,9 @@ github.com/akamensky/argparse v1.4.0 h1:YGzvsTqCvbEZhL8zZu2AiA5nq805NZh75JNj4ajn
 github.com/akamensky/argparse v1.4.0/go.mod h1:S5kwC7IuDcEr5VeXtGPRVZ5o/FdhcMlQz4IZQuw64xA=
 github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI=
 github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
-github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
 github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
+github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/gofiber/fiber/v2 v2.50.0 h1:ia0JaB+uw3GpNSCR5nvC5dsaxXjRU5OEu36aytx+zGw=
@@ -27,12 +27,6 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
 github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
-github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
-github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
@@ -45,15 +39,19 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
+golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
-golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
+golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg=
+golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -63,21 +61,27 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
+golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
+golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
+golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8=
+golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

handlers/api.go

@@ -2,12 +2,12 @@ package handlers
 
 import (
 	_ "embed"
-
 	"log"
 
 	"github.com/gofiber/fiber/v2"
 )
 
+//nolint:all
 //go:embed VERSION
 var version string
 

handlers/cli/cli.go

@@ -0,0 +1,113 @@
+package cli
+
+import (
+	"fmt"
+	"io"
+	"os"
+
+	"ladder/pkg/ruleset"
+
+	"golang.org/x/term"
+)
+
+// HandleRulesetMerge merges a set of ruleset files, specified by the rulesetPath or RULESET env variable, into either YAML or Gzip format.
+// Exits the program with an error message if the ruleset path is not provided or if loading the ruleset fails.
+//
+// Parameters:
+// - rulesetPath: Specifies the path to the ruleset file.
+// - mergeRulesets: Indicates if a merge operation should be performed.
+// - useGzip: Indicates if the merged rulesets should be gzip-ped.
+// - output: Specifies the output file. If nil, stdout will be used.
+//
+// Returns:
+// - An error if the ruleset loading or merging process fails, otherwise nil.
+func HandleRulesetMerge(rulesetPath string, mergeRulesets bool, useGzip bool, output *os.File) error {
+	if !mergeRulesets {
+		return nil
+	}
+
+	if rulesetPath == "" {
+		rulesetPath = os.Getenv("RULESET")
+	}
+
+	if rulesetPath == "" {
+		fmt.Println("error: no ruleset provided. Try again with --ruleset <ruleset.yaml>")
+		os.Exit(1)
+	}
+
+	rs, err := ruleset.NewRuleset(rulesetPath)
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	if useGzip {
+		return gzipMerge(rs, output)
+	}
+
+	return yamlMerge(rs, output)
+}
+
+// gzipMerge takes a RuleSet and an optional output file path pointer. It compresses the RuleSet into Gzip format.
+// If the output file path is provided, the compressed data is written to this file. Otherwise, it prints a warning
+// and outputs the binary data to stdout
+//
+// Parameters:
+// - rs: The ruleset.RuleSet to be compressed.
+// - output: The output for the gzip data. If nil, stdout will be used.
+//
+// Returns:
+// - An error if compression or file writing fails, otherwise nil.
+func gzipMerge(rs ruleset.RuleSet, output io.Writer) error {
+	gzip, err := rs.GzipYaml()
+	if err != nil {
+		return err
+	}
+
+	if output != nil {
+		_, err = io.Copy(output, gzip)
+		if err != nil {
+			return err
+		}
+	}
+
+	if term.IsTerminal(int(os.Stdout.Fd())) {
+		println("warning: binary output can mess up your terminal. Use '--merge-rulesets-output <ruleset.gz>' or pipe it to a file.")
+		os.Exit(1)
+	}
+
+	_, err = io.Copy(os.Stdout, gzip)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// yamlMerge takes a RuleSet and an optional output file path pointer. It converts the RuleSet into YAML format.
+// If the output file path is provided, the YAML data is written to this file. If not, the YAML data is printed to stdout.
+//
+// Parameters:
+// - rs: The ruleset.RuleSet to be converted to YAML.
+// - output: The output for the merged data. If nil, stdout will be used.
+//
+// Returns:
+// - An error if YAML conversion or file writing fails, otherwise nil.
+func yamlMerge(rs ruleset.RuleSet, output io.Writer) error {
+	yaml, err := rs.Yaml()
+	if err != nil {
+		return err
+	}
+
+	if output == nil {
+		fmt.Println(yaml)
+		os.Exit(0)
+	}
+
+	_, err = io.WriteString(output, yaml)
+	if err != nil {
+		return fmt.Errorf("failed to write merged YAML ruleset: %v", err)
+	}
+
+	return nil
+}

handlers/form.html

@@ -1,162 +1,40 @@
 <!DOCTYPE html>
-<html>
+<html lang="en">
+
 <head>
     <meta charset="UTF-8">
-    <title>Ladder</title>
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/materialize/1.0.0/css/materialize.min.css">
+    <title>ladder</title>
+    <link rel="stylesheet" href="/styles.css">
 </head>
-    <style>
-        body {
-            background-color: #ffffff;
-        }
 
-        header h1 {
+<body class="antialiased text-slate-500 dark:text-slate-400 bg-white dark:bg-slate-900">
-            text-transform: uppercase;
+    <div class="grid grid-cols-1 gap-4 max-w-3xl mx-auto pt-10">
-            font-size: 70px;
+        <svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="100%" height="250" viewBox="0 0 512 512">
-            font-weight: 600;
+            <path fill="#7AA7D1" d="M262.074 485.246C254.809 485.265 247.407 485.534 240.165 484.99L226.178 483.306C119.737 468.826 34.1354 383.43 25.3176 274.714C24.3655 262.975 23.5876 253.161 24.3295 241.148C31.4284 126.212 123.985 31.919 238.633 24.1259L250.022 23.8366C258.02 23.8001 266.212 23.491 274.183 24.1306C320.519 27.8489 366.348 45.9743 402.232 75.4548L416.996 88.2751C444.342 114.373 464.257 146.819 475.911 182.72L480.415 197.211C486.174 219.054 488.67 242.773 487.436 265.259L486.416 275.75C478.783 352.041 436.405 418.1 369.36 455.394L355.463 462.875C326.247 477.031 294.517 484.631 262.074 485.246ZM253.547 72.4475C161.905 73.0454 83.5901 144.289 73.0095 234.5C69.9101 260.926 74.7763 292.594 83.9003 317.156C104.53 372.691 153.9 416.616 211.281 430.903C226.663 434.733 242.223 436.307 258.044 436.227C353.394 435.507 430.296 361.835 438.445 267.978C439.794 252.442 438.591 236.759 435.59 221.5C419.554 139.955 353.067 79.4187 269.856 72.7052C264.479 72.2714 258.981 72.423 253.586 72.4127L253.547 72.4475Z"/>
-            color: #fdfdfe;
+            <path fill="#7AA7D1" d="M153.196 310.121L133.153 285.021C140.83 283.798 148.978 285.092 156.741 284.353L156.637 277.725L124.406 278.002C123.298 277.325 122.856 276.187 122.058 275.193L116.089 267.862C110.469 260.975 103.827 254.843 98.6026 247.669C103.918 246.839 105.248 246.537 111.14 246.523L129.093 246.327C130.152 238.785 128.62 240.843 122.138 240.758C111.929 240.623 110.659 242.014 105.004 234.661L97.9953 225.654C94.8172 221.729 91.2219 218.104 88.2631 214.005C84.1351 208.286 90.1658 209.504 94.601 209.489L236.752 209.545C257.761 209.569 268.184 211.009 285.766 221.678L285.835 206.051C285.837 197.542 286.201 189.141 284.549 180.748C280.22 158.757 260.541 143.877 240.897 135.739C238.055 134.561 232.259 133.654 235.575 129.851C244.784 119.288 263.680 111.990 277.085 111.105C288.697 109.828 301.096 113.537 311.75 117.703C360.649 136.827 393.225 183.042 398.561 234.866C402.204 270.253 391.733 308.356 367.999 335.1C332.832 374.727 269.877 384.883 223.294 360.397C206.156 351.388 183.673 333.299 175.08 316.6C173.511 313.551 174.005 313.555 170.443 313.52L160.641 313.449C158.957 313.435 156.263 314.031 155.122 312.487L153.196 310.121Z"/>
-            text-shadow: 0px 0px 5px #7AA7D1, 0px 0px 10px #7AA7D1, 0px 0px 10px #7AA7D1,
+        </svg>
-                0px 0px 20px #7AA7D1;
-        }
-        .logo-title {
-            font-family: 'Arial', sans-serif;
-            font-size: 2rem;
-            color: #fff;
-            margin-bottom: 20px;
-        }
-        .logo {
-            text-align: center;
-        }
-        .github-corner {
-            animation: octocat-wave 560ms ease-in-out;
-            position:absolute;
-            top:0;
-            right:0;
-        }
-    </style>
-</head>
-<body>
-    <a href="https://github.com/everywall/ladder">
-        <div class="github-corner" aria-label="View source on GitHub">
-            <svg
-                xmlns:svg="http://www.w3.org/2000/svg"
-                xmlns="http://www.w3.org/2000/svg"
-                xmlns:xlink="http://www.w3.org/1999/xlink"
-                version="1.1"
-                width="146"
-                height="146"
-                id="svg2">
-                <defs
-                    id="defs8">
-                    <filter
-                    height="1.096"
-                    y="-0.048"
-                    width="1.096"
-                    x="-0.048"
-                    style="color-interpolation-filters:sRGB"
-                    id="filter6">
-                    <feGaussianBlur
-                        stdDeviation="3"
-                        id="feGaussianBlur4" />
-                    </filter>
-                </defs>
-                <path
-                    d="M 152,140 6,-6 H 48 L 152,98 Z"
-                    style="opacity:0.8;filter:url(#filter6)"
-                    id="path10" />
-                <path
-                    d="M 146,134 12,0 h 42 l 92,92 z"
-                    style="fill:#007200"
-                    id="path12" />
-                <g
-                    aria-label="Fork me on GitHub"
-                    transform="rotate(45)"
-                    style="font-family:Collegiate;fill:#ffffff"
-                    id="g42">
-                    <path
-                    d="m 53.643,-19.486 c 0,0.688 -0.016,1.2 -0.064,1.504 h 2.08 c -0.048,-0.32 -0.064,-0.8 -0.064,-1.424 v -3.344 h 1.76 c 0.416,0 0.736,0.016 0.944,0.048 v -1.76 c -0.24,0.032 -0.592,0.048 -1.088,0.048 h -1.616 v -2.496 h 1.936 c 0.56,0 0.944,0.016 1.184,0.048 v -1.792 h -5.136 c 0.048,0.272 0.064,0.784 0.064,1.504 z"
-                    id="path14" />
-                    <path
-                    d="m 62.424,-17.87 c 1.008,0 1.776,-0.368 2.272,-1.088 0.432,-0.624 0.656,-1.472 0.656,-2.544 0,-2.416 -0.976,-3.616 -2.928,-3.616 -1.968,0 -2.96,1.2 -2.96,3.616 0,1.072 0.224,1.936 0.656,2.56 0.512,0.72 1.28,1.072 2.304,1.072 z m -0.016,-5.68 c 0.496,0 0.816,0.24 0.976,0.704 0.096,0.272 0.144,0.72 0.144,1.344 0,0.64 -0.048,1.088 -0.144,1.36 -0.16,0.464 -0.48,0.688 -0.976,0.688 -0.496,0 -0.816,-0.24 -0.976,-0.704 -0.096,-0.272 -0.144,-0.72 -0.144,-1.344 0,-0.624 0.048,-1.072 0.144,-1.344 0.16,-0.464 0.48,-0.704 0.976,-0.704 z"
-                    id="path16" />
-                    <path
-                    d="m 68.293,-17.982 c -0.032,-0.24 -0.048,-0.64 -0.048,-1.184 v -3.888 c 0.352,-0.304 0.672,-0.464 0.976,-0.464 0.224,0 0.48,0.096 0.752,0.288 v -1.808 c -0.224,-0.08 -0.432,-0.128 -0.624,-0.128 -0.448,0 -0.832,0.192 -1.152,0.56 v -0.48 h -1.744 c 0.032,0.192 0.048,0.544 0.048,1.04 v 4.976 c 0,0.512 -0.016,0.88 -0.048,1.088 z"
-                    id="path18" />
-                    <path
-                    d="m 72.857,-17.982 c -0.032,-0.24 -0.048,-0.64 -0.048,-1.184 v -2.448 l 1.472,2.816 c 0.208,0.384 0.32,0.656 0.368,0.816 h 1.872 l -2.352,-4.416 2.144,-2.752 h -2.064 c -0.08,0.176 -0.192,0.352 -0.352,0.56 l -1.088,1.44 v -4.496 c 0,-0.464 0.016,-0.8 0.048,-1.008 h -1.824 c 0.032,0.192 0.048,0.544 0.048,1.04 v 8.544 c 0,0.512 -0.016,0.88 -0.048,1.088 z"
-                    id="path20" />
-                    <path
-                    d="m 85.08,-24.478 c -0.384,-0.432 -0.896,-0.656 -1.52,-0.656 -0.416,0 -0.864,0.192 -1.328,0.56 v -0.512 l -1.76,-0.016 c 0.032,0.176 0.048,0.544 0.048,1.12 v 4.992 c 0,0.496 -0.016,0.832 -0.048,1.008 h 1.856 c 0,-0.064 -0.048,-0.64 -0.048,-1.008 v -3.984 c 0.304,-0.288 0.608,-0.432 0.928,-0.432 0.656,0 0.864,0.416 0.864,1.76 l -0.016,2.16 c 0,0.656 -0.032,1.168 -0.08,1.504 h 1.92 c -0.048,-0.256 -0.064,-0.752 -0.064,-1.472 v -2.192 c 0,-0.56 -0.048,-1.056 -0.144,-1.504 0.208,-0.176 0.544,-0.256 0.976,-0.256 0.64,0 0.96,0.592 0.96,1.76 v 2.16 c 0,0.656 -0.032,1.168 -0.08,1.504 h 1.904 c -0.048,-0.256 -0.064,-0.752 -0.064,-1.472 v -2.192 c 0,-0.96 -0.176,-1.744 -0.512,-2.368 -0.432,-0.752 -1.056,-1.12 -1.888,-1.12 -0.736,0 -1.376,0.224 -1.904,0.656 z"
-                    id="path22" />
-                    <path
-                    d="m 95.905,-20.99 c 0.032,-0.304 0.048,-0.624 0.048,-0.992 0,-0.944 -0.224,-1.696 -0.656,-2.256 -0.464,-0.592 -1.136,-0.896 -2.016,-0.896 -0.896,0 -1.6,0.368 -2.112,1.088 -0.464,0.656 -0.688,1.456 -0.688,2.432 0,1.136 0.272,2.048 0.832,2.704 0.576,0.72 1.392,1.072 2.448,1.072 0.496,0 1.056,-0.128 1.712,-0.368 v -1.712 c -0.464,0.304 -1.008,0.464 -1.6,0.464 -0.944,0 -1.44,-0.512 -1.52,-1.536 z m -2.576,-2.672 c 0.64,0 0.96,0.4 0.976,1.216 h -1.968 c 0.048,-0.816 0.368,-1.216 0.992,-1.216 z"
-                    id="path24" />
-                    <use
-                    xlink:href="#path16"
-                    transform="translate(40.438)"
-                    id="use26" />
-                    <path
-                    d="m 110.187,-25.15 c -0.496,0 -0.992,0.208 -1.472,0.64 v -0.576 h -1.76 c 0.032,0.176 0.048,0.56 0.048,1.184 v 4.912 c 0,0.496 -0.016,0.832 -0.048,1.008 h 1.856 c 0,-0.064 -0.048,-0.64 -0.048,-1.008 v -3.936 c 0.368,-0.352 0.736,-0.528 1.088,-0.528 0.784,0 1.168,0.608 1.152,1.808 l -0.016,2.16 c -0.016,0.752 -0.032,1.264 -0.064,1.504 h 1.92 c -0.048,-0.256 -0.064,-0.752 -0.064,-1.472 v -2.192 c 0,-0.944 -0.192,-1.744 -0.592,-2.384 -0.464,-0.752 -1.136,-1.136 -2,-1.12 z"
-                    id="path28" />
-                    <path
-                    d="m 123.877,-17.982 c 0.144,0.016 0.256,0.016 0.336,0 0,-0.192 -0.064,-0.768 -0.064,-1.36 v -1.856 c 0,-0.56 0.016,-1.056 0.064,-1.52 h -1.952 c 0.032,0.704 0.048,1.12 0.032,1.248 0,1.28 -0.512,1.92 -1.552,1.92 -1.264,0 -1.904,-1.264 -1.904,-3.776 0,-2.48 0.784,-3.728 2.352,-3.728 0.752,0 1.472,0.288 2.16,0.88 v -1.824 c -0.608,-0.528 -1.328,-0.8 -2.16,-0.8 -2.896,0 -4.416,1.904 -4.416,5.424 0,3.696 1.328,5.552 3.968,5.552 0.592,0 1.12,-0.128 1.584,-0.368 0.368,-0.208 0.624,-0.432 0.768,-0.688 z"
-                    id="path30" />
-                    <path
-                    d="m 126.49,-26.334 c 0.592,0 1.104,-0.544 1.104,-1.184 0,-0.656 -0.512,-1.2 -1.104,-1.2 -0.624,0 -1.12,0.544 -1.12,1.2 0,0.64 0.496,1.184 1.12,1.184 z m 0.896,8.352 c -0.016,-0.24 -0.032,-0.64 -0.032,-1.184 v -4.912 c 0,-0.464 0.016,-0.8 0.032,-1.008 h -1.808 c 0.016,0.192 0.032,0.544 0.032,1.04 v 4.976 c 0,0.512 -0.016,0.88 -0.032,1.088 z"
-                    id="path32" />
-                    <path
-                    d="m 130.783,-25.742 c 0,-0.256 0.016,-0.48 0.048,-0.688 h -1.856 c 0.032,0.176 0.048,0.416 0.048,0.72 v 0.624 h -0.784 v 1.552 c 0.224,-0.032 0.4,-0.048 0.544,-0.048 l 0.24,0.016 v 0.032 h -0.016 v 2.864 c 0,0.896 0.112,1.552 0.336,1.968 0.304,0.56 0.832,0.832 1.616,0.832 0.56,0 1.024,-0.112 1.424,-0.32 v -1.6 c -0.272,0.176 -0.56,0.272 -0.896,0.272 -0.464,0 -0.704,-0.352 -0.704,-1.072 v -2.976 h 0.688 c 0.256,0 0.592,0.032 0.704,0.032 v -1.552 h -1.392 z"
-                    id="path34" />
-                    <path
-                    d="m 140.259,-27.678 c 0,-0.416 0.016,-0.736 0.064,-0.976 h -2.096 c 0.048,0.24 0.064,0.688 0.064,1.344 v 2.8 h -2.912 v -3.024 c 0,-0.48 0.016,-0.848 0.064,-1.12 h -2.08 c 0.048,0.256 0.064,0.624 0.064,1.12 v 8.432 c 0,0.496 -0.016,0.864 -0.064,1.12 h 2.08 c -0.048,-0.24 -0.064,-0.656 -0.064,-1.232 v -3.552 h 2.912 v 3.568 c 0,0.528 -0.016,0.944 -0.064,1.216 h 2.096 c -0.048,-0.24 -0.064,-0.624 -0.064,-1.12 v -3.664 h 0.528 v -1.744 h -0.528 z"
-                    id="path36" />
-                    <path
-                    d="m 144.402,-17.918 c 0.56,0 1.072,-0.208 1.568,-0.64 v 0.576 h 1.744 c -0.016,-0.176 -0.032,-0.576 -0.032,-1.2 v -4.896 c 0,-0.496 0.016,-0.832 0.032,-1.008 h -1.856 c 0,0.048 0.064,0.64 0.064,1.008 v 3.936 c -0.368,0.352 -0.704,0.528 -1.008,0.528 -0.432,0 -0.72,-0.16 -0.88,-0.496 -0.144,-0.272 -0.208,-0.704 -0.208,-1.312 l 0.016,-2.16 c 0.016,-0.768 0.032,-1.264 0.064,-1.504 h -1.92 c 0.048,0.256 0.064,0.752 0.064,1.472 v 2.192 c 0,0.976 0.16,1.776 0.48,2.384 0.4,0.752 1.024,1.12 1.872,1.12 z"
-                    id="path38" />
-                    <path
-                    d="m 152.31,-17.934 c 0.848,0 1.536,-0.416 2.048,-1.232 0.432,-0.704 0.64,-1.536 0.64,-2.48 0,-0.928 -0.208,-1.712 -0.608,-2.368 -0.48,-0.752 -1.136,-1.12 -1.984,-1.12 -0.464,0 -0.944,0.16 -1.44,0.464 v -2.784 c 0,-0.608 0.016,-1.008 0.032,-1.2 h -1.824 c 0.032,0.176 0.048,0.576 0.048,1.2 v 8.464 c 0,0.496 -0.016,0.832 -0.048,1.008 h 1.696 v -0.576 c 0.384,0.416 0.864,0.624 1.44,0.624 z m -0.24,-5.52 c 0.736,0 1.104,0.608 1.104,1.808 0,0.496 -0.08,0.928 -0.256,1.296 -0.208,0.464 -0.528,0.688 -0.944,0.688 -0.336,0 -0.672,-0.16 -1.008,-0.496 v -2.768 c 0.384,-0.352 0.752,-0.528 1.104,-0.528 z"
-                    id="path40" />
-                </g>
-                <path
-                    d="m 52,0 94,94 M 14,0 146,132"
-                    style="fill:none;stroke:#ffffff;stroke-dasharray:2, 1;stroke-opacity:0.95"
-                    id="path44" />
-                </svg>
-        </div>
-    </a>
-    <div class="container">
-		<div class="logo">
-			<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="250" height="250" viewBox="0 0 512 512">
-				<path fill="#7AA7D1" d="M262.074 485.246C254.809 485.265 247.407 485.534 240.165 484.99L226.178 483.306C119.737 468.826 34.1354 383.43 25.3176 274.714C24.3655 262.975 23.5876 253.161 24.3295 241.148C31.4284 126.212 123.985 31.919 238.633 24.1259L250.022 23.8366C258.02 23.8001 266.212 23.491 274.183 24.1306C320.519 27.8489 366.348 45.9743 402.232 75.4548L416.996 88.2751C444.342 114.373 464.257 146.819 475.911 182.72L480.415 197.211C486.174 219.054 488.67 242.773 487.436 265.259L486.416 275.75C478.783 352.041 436.405 418.1 369.36 455.394L355.463 462.875C326.247 477.031 294.517 484.631 262.074 485.246ZM253.547 72.4475C161.905 73.0454 83.5901 144.289 73.0095 234.5C69.9101 260.926 74.7763 292.594 83.9003 317.156C104.53 372.691 153.9 416.616 211.281 430.903C226.663 434.733 242.223 436.307 258.044 436.227C353.394 435.507 430.296 361.835 438.445 267.978C439.794 252.442 438.591 236.759 435.59 221.5C419.554 139.955 353.067 79.4187 269.856 72.7052C264.479 72.2714 258.981 72.423 253.586 72.4127L253.547 72.4475Z"/>
-				<path fill="#7AA7D1" d="M153.196 310.121L133.153 285.021C140.83 283.798 148.978 285.092 156.741 284.353L156.637 277.725L124.406 278.002C123.298 277.325 122.856 276.187 122.058 275.193L116.089 267.862C110.469 260.975 103.827 254.843 98.6026 247.669C103.918 246.839 105.248 246.537 111.14 246.523L129.093 246.327C130.152 238.785 128.62 240.843 122.138 240.758C111.929 240.623 110.659 242.014 105.004 234.661L97.9953 225.654C94.8172 221.729 91.2219 218.104 88.2631 214.005C84.1351 208.286 90.1658 209.504 94.601 209.489L236.752 209.545C257.761 209.569 268.184 211.009 285.766 221.678L285.835 206.051C285.837 197.542 286.201 189.141 284.549 180.748C280.22 158.757 260.541 143.877 240.897 135.739C238.055 134.561 232.259 133.654 235.575 129.851C244.784 119.288 263.68 111.99 277.085 111.105C288.697 109.828 301.096 113.537 311.75 117.703C360.649 136.827 393.225 183.042 398.561 234.866C402.204 270.253 391.733 308.356 367.999 335.1C332.832 374.727 269.877 384.883 223.294 360.397C206.156 351.388 183.673 333.299 175.08 316.6C173.511 313.551 174.005 313.555 170.443 313.52L160.641 313.449C158.957 313.435 156.263 314.031 155.122 312.487L153.196 310.121Z"/>
-			</svg>
-		</div>
         <header>
-            <h1 class="center-align logo-title">ladddddddder</h1>
+            <h1 class="text-center text-3xl sm:text-4xl font-extrabold text-slate-900 tracking-tight dark:text-slate-200">ladddddddder</h1>
         </header>
-        <form id="inputForm" class="col s12" method="get">
+        <form id="inputForm" method="get" class="mx-4 relative">
-            <div class="row">
+            <div>
-                <div class="input-field col s12">
+                <input type="url" id="inputField" placeholder="Proxy Search" name="inputField" class="w-full text-sm leading-6 text-slate-400 rounded-md ring-1 ring-slate-900/10 shadow-sm py-1.5 pl-2 pr-3 hover:ring-slate-300 dark:bg-slate-800 dark:highlight-white/5 dark:hover:bg-slate-700" required autofocus>
-                    <input type="text" id="inputField" name="inputField" class="validate" required>
+                <button id="clearButton" type="button" aria-label="Clear Search" title="Clear Search" class="hidden absolute inset-y-0 right-0 items-center pr-2 hover:text-slate-400 hover:dark:text-slate-300" tabindex="-1">
-                    <label for="inputField">URL</label>
+                    <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round""><path d="M18 6 6 18"/><path d="m6 6 12 12"/></svg>
-                </div>
+                </button>
-				<!--
-                <div class="input-field col s2">
-                    <button class="btn waves-effect waves-light" type="submit" name="action">Submit
-                        <i class="material-icons right">go</i>
-                    </button>
-                </div>
-				-->
             </div>
         </form>
+        <footer class="mt-10 mx-4 text-center text-slate-600 dark:text-slate-400">
+            <p>
+                Code Licensed Under GPL v3.0 |
+                <a href="https://github.com/everywall/ladder" class="hover:text-blue-500 hover:underline underline-offset-2 transition-colors duration-300">View Source</a> |
+                <a href="https://github.com/everywall" class="hover:text-blue-500 hover:underline underline-offset-2 transition-colors duration-300">Everywall</a>
+            </p>
+        </footer>
     </div>
 
-    <script src="https://code.jquery.com/jquery-3.6.0.min.js"></script>
-    <script src="https://cdnjs.cloudflare.com/ajax/libs/materialize/1.0.0/js/materialize.min.js"></script>
     <script>
-        document.addEventListener('DOMContentLoaded', function() {
-            M.AutoInit();
-        });
         document.getElementById('inputForm').addEventListener('submit', function (e) {
             e.preventDefault();
             let url = document.getElementById('inputField').value;
@@ -166,6 +44,36 @@
             window.location.href = '/' + url;
             return false;
         });
+        document.getElementById('inputField').addEventListener('input', function() {
+            const clearButton = document.getElementById('clearButton');
+            if (this.value.trim().length > 0) {
+                clearButton.style.display = 'block';
+            } else {
+                clearButton.style.display = 'none';
+            }
+        });
+        document.getElementById('clearButton').addEventListener('click', function() {
+            document.getElementById('inputField').value = '';
+            this.style.display = 'none';
+            document.getElementById('inputField').focus();
+        });
     </script>
+
+    <style>
+        @media (prefers-color-scheme: light) {
+            body {
+                background-color: #ffffff;
+                color: #333333;
+            }
+        }
+
+        @media (prefers-color-scheme: dark) {
+            body {
+                background-color: #1a202c;
+                color: #ffffff;
+            }
+        }
+    </style>
 </body>
-</html>
+
+</html>

handlers/proxy.go

@@ -8,38 +8,155 @@ import (
 	"net/url"
 	"os"
 	"regexp"
+	"strconv"
 	"strings"
+	"time"
+
+	"ladder/pkg/ruleset"
 
 	"github.com/PuerkitoBio/goquery"
 	"github.com/gofiber/fiber/v2"
-	"gopkg.in/yaml.v3"
 )
 
-var UserAgent = getenv("USER_AGENT", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)")
+var (
-var ForwardedFor = getenv("X_FORWARDED_FOR", "66.249.66.1")
+	UserAgent      = getenv("USER_AGENT", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)")
-var rulesSet = loadRules()
+	ForwardedFor   = getenv("X_FORWARDED_FOR", "66.249.66.1")
-var allowedDomains = strings.Split(os.Getenv("ALLOWED_DOMAINS"), ",")
+	rulesSet       = ruleset.NewRulesetFromEnv()
+	allowedDomains = []string{}
+	defaultTimeout = 15 // in seconds
+)
 
-func ProxySite(c *fiber.Ctx) error {
+func init() {
-	// Get the url from the URL
+	allowedDomains = strings.Split(os.Getenv("ALLOWED_DOMAINS"), ",")
-	url := c.Params("*")
+	if os.Getenv("ALLOWED_DOMAINS_RULESET") == "true" {
+		allowedDomains = append(allowedDomains, rulesSet.Domains()...)
+	}
+	if timeoutStr := os.Getenv("HTTP_TIMEOUT"); timeoutStr != "" {
+		defaultTimeout, _ = strconv.Atoi(timeoutStr)
+	}
+}
 
-	queries := c.Queries()
+// extracts a URL from the request ctx. If the URL in the request
-	body, _, resp, err := fetchSite(url, queries)
+// is a relative path, it reconstructs the full URL using the referer header.
+func extractUrl(c *fiber.Ctx) (string, error) {
+	// try to extract url-encoded
+	reqUrl, err := url.QueryUnescape(c.Params("*"))
 	if err != nil {
-		log.Println("ERROR:", err)
+		// fallback
-		c.SendStatus(fiber.StatusInternalServerError)
+		reqUrl = c.Params("*")
-		return c.SendString(err.Error())
 	}
 
-	c.Set("Content-Type", resp.Header.Get("Content-Type"))
+	// Extract the actual path from req ctx
-	c.Set("Content-Security-Policy", resp.Header.Get("Content-Security-Policy"))
+	urlQuery, err := url.Parse(reqUrl)
+	if err != nil {
+		return "", fmt.Errorf("error parsing request URL '%s': %v", reqUrl, err)
+	}
 
-	return c.SendString(body)
+	isRelativePath := urlQuery.Scheme == ""
+
+	// eg: https://localhost:8080/images/foobar.jpg -> https://realsite.com/images/foobar.jpg
+	if isRelativePath {
+		// Parse the referer URL from the request header.
+		refererUrl, err := url.Parse(c.Get("referer"))
+		if err != nil {
+			return "", fmt.Errorf("error parsing referer URL from req: '%s': %v", reqUrl, err)
+		}
+
+		// Extract the real url from referer path
+		realUrl, err := url.Parse(strings.TrimPrefix(refererUrl.Path, "/"))
+		if err != nil {
+			return "", fmt.Errorf("error parsing real URL from referer '%s': %v", refererUrl.Path, err)
+		}
+
+		// reconstruct the full URL using the referer's scheme, host, and the relative path / queries
+		fullUrl := &url.URL{
+			Scheme:   realUrl.Scheme,
+			Host:     realUrl.Host,
+			Path:     urlQuery.Path,
+			RawQuery: urlQuery.RawQuery,
+		}
+
+		if os.Getenv("LOG_URLS") == "true" {
+			log.Printf("modified relative URL: '%s' -> '%s'", reqUrl, fullUrl.String())
+		}
+		return fullUrl.String(), nil
+
+	}
+
+	// default behavior:
+	// eg: https://localhost:8080/https://realsite.com/images/foobar.jpg -> https://realsite.com/images/foobar.jpg
+	return urlQuery.String(), nil
+}
+
+func ProxySite(rulesetPath string) fiber.Handler {
+	if rulesetPath != "" {
+		rs, err := ruleset.NewRuleset(rulesetPath)
+		if err != nil {
+			panic(err)
+		}
+		rulesSet = rs
+	}
+
+	return func(c *fiber.Ctx) error {
+		// Get the url from the URL
+		url, err := extractUrl(c)
+		if err != nil {
+			log.Println("ERROR In URL extraction:", err)
+		}
+
+		queries := c.Queries()
+		body, _, resp, err := fetchSite(url, queries)
+		if err != nil {
+			log.Println("ERROR:", err)
+			c.SendStatus(fiber.StatusInternalServerError)
+			return c.SendString(err.Error())
+		}
+
+		c.Cookie(&fiber.Cookie{})
+		c.Set("Content-Type", resp.Header.Get("Content-Type"))
+		c.Set("Content-Security-Policy", resp.Header.Get("Content-Security-Policy"))
+
+		return c.SendString(body)
+	}
+}
+
+func modifyURL(uri string, rule ruleset.Rule) (string, error) {
+	newUrl, err := url.Parse(uri)
+	if err != nil {
+		return "", err
+	}
+
+	for _, urlMod := range rule.URLMods.Domain {
+		re := regexp.MustCompile(urlMod.Match)
+		newUrl.Host = re.ReplaceAllString(newUrl.Host, urlMod.Replace)
+	}
+
+	for _, urlMod := range rule.URLMods.Path {
+		re := regexp.MustCompile(urlMod.Match)
+		newUrl.Path = re.ReplaceAllString(newUrl.Path, urlMod.Replace)
+	}
+
+	v := newUrl.Query()
+	for _, query := range rule.URLMods.Query {
+		if query.Value == "" {
+			v.Del(query.Key)
+			continue
+		}
+		v.Set(query.Key, query.Value)
+	}
+	newUrl.RawQuery = v.Encode()
+
+	if rule.GoogleCache {
+		newUrl, err = url.Parse("https://webcache.googleusercontent.com/search?q=cache:" + newUrl.String())
+		if err != nil {
+			return "", err
+		}
+	}
+
+	return newUrl.String(), nil
 }
 
 func fetchSite(urlpath string, queries map[string]string) (string, *http.Request, *http.Response, error) {
-
 	urlQuery := "?"
 	if len(queries) > 0 {
 		for k, v := range queries {
@@ -62,18 +179,18 @@ func fetchSite(urlpath string, queries map[string]string) (string, *http.Request
 		log.Println(u.String() + urlQuery)
 	}
 
+	// Modify the URI according to ruleset
 	rule := fetchRule(u.Host, u.Path)
-
+	url, err := modifyURL(u.String()+urlQuery, rule)
-	if rule.GoogleCache {
+	if err != nil {
-		u, err = url.Parse("https://webcache.googleusercontent.com/search?q=cache:" + u.String())
+		return "", nil, nil, err
-		if err != nil {
-			return "", nil, nil, err
-		}
 	}
 
 	// Fetch the site
-	client := &http.Client{}
+	client := &http.Client{
-	req, _ := http.NewRequest("GET", u.String()+urlQuery, nil)
+		Timeout: time.Second * time.Duration(defaultTimeout),
+	}
+	req, _ := http.NewRequest("GET", url, nil)
 
 	if rule.Headers.UserAgent != "" {
 		req.Header.Set("User-Agent", rule.Headers.UserAgent)
@@ -102,7 +219,6 @@ func fetchSite(urlpath string, queries map[string]string) (string, *http.Request
 	}
 
 	resp, err := client.Do(req)
-
 	if err != nil {
 		return "", nil, nil, err
 	}
@@ -114,15 +230,16 @@ func fetchSite(urlpath string, queries map[string]string) (string, *http.Request
 	}
 
 	if rule.Headers.CSP != "" {
+		// log.Println(rule.Headers.CSP)
 		resp.Header.Set("Content-Security-Policy", rule.Headers.CSP)
 	}
 
-	log.Print("rule", rule)
+	// log.Print("rule", rule) TODO: Add a debug mode to print the rule
 	body := rewriteHtml(bodyB, u, rule)
 	return body, req, resp, nil
 }
 
-func rewriteHtml(bodyB []byte, u *url.URL, rule Rule) string {
+func rewriteHtml(bodyB []byte, u *url.URL, rule ruleset.Rule) string {
 	// Rewrite the HTML
 	body := string(bodyB)
 
@@ -136,7 +253,7 @@ func rewriteHtml(bodyB []byte, u *url.URL, rule Rule) string {
 	reScript := regexp.MustCompile(scriptPattern)
 	body = reScript.ReplaceAllString(body, fmt.Sprintf(`<script $1 script="%s$3"`, "/https://"+u.Host+"/"))
 
-	//body = strings.ReplaceAll(body, "srcset=\"/", "srcset=\"/https://"+u.Host+"/") // TODO: Needs a regex to rewrite the URL's
+	// body = strings.ReplaceAll(body, "srcset=\"/", "srcset=\"/https://"+u.Host+"/") // TODO: Needs a regex to rewrite the URL's
 	body = strings.ReplaceAll(body, "href=\"/", "href=\"/https://"+u.Host+"/")
 	body = strings.ReplaceAll(body, "url('/", "url('/https://"+u.Host+"/")
 	body = strings.ReplaceAll(body, "url(/", "url(/https://"+u.Host+"/")
@@ -156,63 +273,11 @@ func getenv(key, fallback string) string {
 	return value
 }
 
-func loadRules() RuleSet {
+func fetchRule(domain string, path string) ruleset.Rule {
-	rulesUrl := os.Getenv("RULESET")
-	if rulesUrl == "" {
-		RulesList := RuleSet{}
-		return RulesList
-	}
-	log.Println("Loading rules")
-
-	var ruleSet RuleSet
-	if strings.HasPrefix(rulesUrl, "http") {
-
-		resp, err := http.Get(rulesUrl)
-		if err != nil {
-			log.Println("ERROR:", err)
-		}
-		defer resp.Body.Close()
-
-		if resp.StatusCode >= 400 {
-			log.Println("ERROR:", resp.StatusCode, rulesUrl)
-		}
-
-		body, err := io.ReadAll(resp.Body)
-		if err != nil {
-			log.Println("ERROR:", err)
-		}
-		yaml.Unmarshal(body, &ruleSet)
-
-		if err != nil {
-			log.Println("ERROR:", err)
-		}
-	} else {
-		yamlFile, err := os.ReadFile(rulesUrl)
-		if err != nil {
-			log.Println("ERROR:", err)
-		}
-		yaml.Unmarshal(yamlFile, &ruleSet)
-	}
-
-	domains := []string{}
-	for _, rule := range ruleSet {
-
-		domains = append(domains, rule.Domain)
-		domains = append(domains, rule.Domains...)
-		if os.Getenv("ALLOWED_DOMAINS_RULESET") == "true" {
-			allowedDomains = append(allowedDomains, domains...)
-		}
-	}
-
-	log.Println("Loaded ", len(ruleSet), " rules for", len(domains), "Domains")
-	return ruleSet
-}
-
-func fetchRule(domain string, path string) Rule {
 	if len(rulesSet) == 0 {
-		return Rule{}
+		return ruleset.Rule{}
 	}
-	rule := Rule{}
+	rule := ruleset.Rule{}
 	for _, rule := range rulesSet {
 		domains := rule.Domains
 		if rule.Domain != "" {
@@ -231,7 +296,7 @@ func fetchRule(domain string, path string) Rule {
 	return rule
 }
 
-func applyRules(body string, rule Rule) string {
+func applyRules(body string, rule ruleset.Rule) string {
 	if len(rulesSet) == 0 {
 		return body
 	}

handlers/proxy.test.go

@@ -7,13 +7,15 @@ import (
 	"net/url"
 	"testing"
 
+	"ladder/pkg/ruleset"
+
 	"github.com/gofiber/fiber/v2"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestProxySite(t *testing.T) {
 	app := fiber.New()
-	app.Get("/:url", ProxySite)
+	app.Get("/:url", ProxySite(""))
 
 	req := httptest.NewRequest("GET", "/https://example.com", nil)
 	resp, err := app.Test(req)
@@ -51,7 +53,7 @@ func TestRewriteHtml(t *testing.T) {
 		</html>
 	`
 
-	actual := rewriteHtml(bodyB, u, Rule{})
+	actual := rewriteHtml(bodyB, u, ruleset.Rule{})
 	assert.Equal(t, expected, actual)
 }
 

handlers/ruleset.go

@@ -8,7 +8,6 @@ import (
 )
 
 func Ruleset(c *fiber.Ctx) error {
-
 	if os.Getenv("EXPOSE_RULESET") == "false" {
 		c.SendStatus(fiber.StatusForbidden)
 		return c.SendString("Rules Disabled")

handlers/types.go

@@ -1,29 +0,0 @@
-package handlers
-
-type Regex struct {
-	Match   string `yaml:"match"`
-	Replace string `yaml:"replace"`
-}
-
-type RuleSet []Rule
-
-type Rule struct {
-	Domain  string   `yaml:"domain,omitempty"`
-	Domains []string `yaml:"domains,omitempty"`
-	Paths   []string `yaml:"paths,omitempty"`
-	Headers struct {
-		UserAgent     string `yaml:"user-agent,omitempty"`
-		XForwardedFor string `yaml:"x-forwarded-for,omitempty"`
-		Referer       string `yaml:"referer,omitempty"`
-		Cookie        string `yaml:"cookie,omitempty"`
-		CSP           string `yaml:"content-security-policy,omitempty"`
-	} `yaml:"headers,omitempty"`
-	GoogleCache bool    `yaml:"googleCache,omitempty"`
-	RegexRules  []Regex `yaml:"regexRules"`
-	Injections  []struct {
-		Position string `yaml:"position"`
-		Append   string `yaml:"append"`
-		Prepend  string `yaml:"prepend"`
-		Replace  string `yaml:"replace"`
-	} `yaml:"injections"`
-}

package.json

@@ -0,0 +1,9 @@
+{
+	"scripts": {
+		"build": "pnpx tailwindcss -i ./styles/input.css -o ./styles/output.css --build && pnpx minify ./styles/output.css > ./cmd/styles.css"
+	},
+	"devDependencies": {
+		"minify": "^10.5.2",
+		"tailwindcss": "^3.3.5"
+	}
+}

pkg/ruleset/ruleset.go

@@ -0,0 +1,310 @@
+package ruleset
+
+import (
+	"compress/gzip"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+)
+
+type Regex struct {
+	Match   string `yaml:"match"`
+	Replace string `yaml:"replace"`
+}
+type KV struct {
+	Key   string `yaml:"key"`
+	Value string `yaml:"value"`
+}
+
+type RuleSet []Rule
+
+type Rule struct {
+	Domain  string   `yaml:"domain,omitempty"`
+	Domains []string `yaml:"domains,omitempty"`
+	Paths   []string `yaml:"paths,omitempty"`
+	Headers struct {
+		UserAgent     string `yaml:"user-agent,omitempty"`
+		XForwardedFor string `yaml:"x-forwarded-for,omitempty"`
+		Referer       string `yaml:"referer,omitempty"`
+		Cookie        string `yaml:"cookie,omitempty"`
+		CSP           string `yaml:"content-security-policy,omitempty"`
+	} `yaml:"headers,omitempty"`
+	GoogleCache bool    `yaml:"googleCache,omitempty"`
+	RegexRules  []Regex `yaml:"regexRules,omitempty"`
+
+	URLMods struct {
+		Domain []Regex `yaml:"domain,omitempty"`
+		Path   []Regex `yaml:"path,omitempty"`
+		Query  []KV    `yaml:"query,omitempty"`
+	} `yaml:"urlMods,omitempty"`
+
+	Injections []struct {
+		Position string `yaml:"position,omitempty"`
+		Append   string `yaml:"append,omitempty"`
+		Prepend  string `yaml:"prepend,omitempty"`
+		Replace  string `yaml:"replace,omitempty"`
+	} `yaml:"injections,omitempty"`
+}
+
+var remoteRegex = regexp.MustCompile(`^https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()!@:%_\+.~#?&\/\/=]*)`)
+
+// NewRulesetFromEnv creates a new RuleSet based on the RULESET environment variable.
+// It logs a warning and returns an empty RuleSet if the RULESET environment variable is not set.
+// If the RULESET is set but the rules cannot be loaded, it panics.
+func NewRulesetFromEnv() RuleSet {
+	rulesPath, ok := os.LookupEnv("RULESET")
+	if !ok {
+		log.Printf("WARN: No ruleset specified. Set the `RULESET` environment variable to load one for a better success rate.")
+		return RuleSet{}
+	}
+
+	ruleSet, err := NewRuleset(rulesPath)
+	if err != nil {
+		log.Println(err)
+	}
+
+	return ruleSet
+}
+
+// NewRuleset loads a RuleSet from a given string of rule paths, separated by semicolons.
+// It supports loading rules from both local file paths and remote URLs.
+// Returns a RuleSet and an error if any issues occur during loading.
+func NewRuleset(rulePaths string) (RuleSet, error) {
+	var ruleSet RuleSet
+
+	var errs []error
+
+	rp := strings.Split(rulePaths, ";")
+	for _, rule := range rp {
+		var err error
+
+		rulePath := strings.Trim(rule, " ")
+		isRemote := remoteRegex.MatchString(rulePath)
+
+		if isRemote {
+			err = ruleSet.loadRulesFromRemoteFile(rulePath)
+		} else {
+			err = ruleSet.loadRulesFromLocalDir(rulePath)
+		}
+
+		if err != nil {
+			e := fmt.Errorf("WARN: failed to load ruleset from '%s'", rulePath)
+			errs = append(errs, errors.Join(e, err))
+
+			continue
+		}
+	}
+
+	if len(errs) != 0 {
+		e := fmt.Errorf("WARN: failed to load %d rulesets", len(rp))
+		errs = append(errs, e)
+
+		// panic if the user specified a local ruleset, but it wasn't found on disk
+		// don't fail silently
+		for _, err := range errs {
+			if errors.Is(os.ErrNotExist, err) {
+				e := fmt.Errorf("PANIC: ruleset '%s' not found", err)
+				panic(errors.Join(e, err))
+			}
+		}
+
+		// else, bubble up any errors, such as syntax or remote host issues
+		return ruleSet, errors.Join(errs...)
+	}
+
+	ruleSet.PrintStats()
+
+	return ruleSet, nil
+}
+
+// ================== RULESET loading logic ===================================
+
+// loadRulesFromLocalDir loads rules from a local directory specified by the path.
+// It walks through the directory, loading rules from YAML files.
+// Returns an error if the directory cannot be accessed
+// If there is an issue loading any file, it will be skipped
+func (rs *RuleSet) loadRulesFromLocalDir(path string) error {
+	_, err := os.Stat(path)
+	if err != nil {
+		return err
+	}
+
+	yamlRegex := regexp.MustCompile(`.*\.ya?ml`)
+
+	err = filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+		if isYaml := yamlRegex.MatchString(path); !isYaml {
+			return nil
+		}
+
+		err = rs.loadRulesFromLocalFile(path)
+		if err != nil {
+			log.Printf("WARN: failed to load directory ruleset '%s': %s, skipping", path, err)
+			return nil
+		}
+
+		log.Printf("INFO: loaded ruleset %s\n", path)
+
+		return nil
+	})
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// loadRulesFromLocalFile loads rules from a local YAML file specified by the path.
+// Returns an error if the file cannot be read or if there's a syntax error in the YAML.
+func (rs *RuleSet) loadRulesFromLocalFile(path string) error {
+	yamlFile, err := os.ReadFile(path)
+	if err != nil {
+		e := fmt.Errorf("failed to read rules from local file: '%s'", path)
+		return errors.Join(e, err)
+	}
+
+	var r RuleSet
+	err = yaml.Unmarshal(yamlFile, &r)
+
+	if err != nil {
+		e := fmt.Errorf("failed to load rules from local file, possible syntax error in '%s'", path)
+		ee := errors.Join(e, err)
+
+		if _, ok := os.LookupEnv("DEBUG"); ok {
+			debugPrintRule(string(yamlFile), ee)
+		}
+
+		return ee
+	}
+
+	*rs = append(*rs, r...)
+
+	return nil
+}
+
+// loadRulesFromRemoteFile loads rules from a remote URL.
+// It supports plain and gzip compressed content.
+// Returns an error if there's an issue accessing the URL or if there's a syntax error in the YAML.
+func (rs *RuleSet) loadRulesFromRemoteFile(rulesURL string) error {
+	var r RuleSet
+
+	resp, err := http.Get(rulesURL)
+	if err != nil {
+		e := fmt.Errorf("failed to load rules from remote url '%s'", rulesURL)
+		return errors.Join(e, err)
+	}
+
+	defer resp.Body.Close()
+
+	if resp.StatusCode >= 400 {
+		e := fmt.Errorf("failed to load rules from remote url (%s) on '%s'", resp.Status, rulesURL)
+		return errors.Join(e, err)
+	}
+
+	var reader io.Reader
+
+	isGzip := strings.HasSuffix(rulesURL, ".gz") || strings.HasSuffix(rulesURL, ".gzip") || resp.Header.Get("content-encoding") == "gzip"
+
+	if isGzip {
+		reader, err = gzip.NewReader(resp.Body)
+
+		if err != nil {
+			return fmt.Errorf("failed to create gzip reader for URL '%s' with status code '%s': %w", rulesURL, resp.Status, err)
+		}
+	} else {
+		reader = resp.Body
+	}
+
+	err = yaml.NewDecoder(reader).Decode(&r)
+
+	if err != nil {
+		e := fmt.Errorf("failed to load rules from remote url '%s' with status code '%s' and possible syntax error", rulesURL, resp.Status)
+		ee := errors.Join(e, err)
+
+		return ee
+	}
+
+	*rs = append(*rs, r...)
+
+	return nil
+}
+
+// ================= utility methods ==========================
+
+// Yaml returns the ruleset as a Yaml string
+func (rs *RuleSet) Yaml() (string, error) {
+	y, err := yaml.Marshal(rs)
+	if err != nil {
+		return "", err
+	}
+
+	return string(y), nil
+}
+
+// GzipYaml returns an io.Reader that streams the Gzip-compressed YAML representation of the RuleSet.
+func (rs *RuleSet) GzipYaml() (io.Reader, error) {
+	pr, pw := io.Pipe()
+
+	go func() {
+		defer pw.Close()
+
+		gw := gzip.NewWriter(pw)
+		defer gw.Close()
+
+		if err := yaml.NewEncoder(gw).Encode(rs); err != nil {
+			gw.Close() // Ensure to close the gzip writer
+			pw.CloseWithError(err)
+			return
+		}
+	}()
+
+	return pr, nil
+}
+
+// Domains extracts and returns a slice of all domains present in the RuleSet.
+func (rs *RuleSet) Domains() []string {
+	var domains []string
+	for _, rule := range *rs {
+		domains = append(domains, rule.Domain)
+		domains = append(domains, rule.Domains...)
+	}
+	return domains
+}
+
+// DomainCount returns the count of unique domains present in the RuleSet.
+func (rs *RuleSet) DomainCount() int {
+	return len(rs.Domains())
+}
+
+// Count returns the total number of rules in the RuleSet.
+func (rs *RuleSet) Count() int {
+	return len(*rs)
+}
+
+// PrintStats logs the number of rules and domains loaded in the RuleSet.
+func (rs *RuleSet) PrintStats() {
+	log.Printf("INFO: Loaded %d rules for %d domains\n", rs.Count(), rs.DomainCount())
+}
+
+// debugPrintRule is a utility function for printing a rule and associated error for debugging purposes.
+func debugPrintRule(rule string, err error) {
+	fmt.Println("------------------------------ BEGIN DEBUG RULESET -----------------------------")
+	fmt.Printf("%s\n", err.Error())
+	fmt.Println("--------------------------------------------------------------------------------")
+	fmt.Println(rule)
+	fmt.Println("------------------------------ END DEBUG RULESET -------------------------------")
+}

pkg/ruleset/ruleset_test.go

@@ -0,0 +1,173 @@
+package ruleset
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/stretchr/testify/assert"
+)
+
+var (
+	validYAML = `
+- domain: example.com
+  regexRules:
+    - match: "^http:"
+      replace: "https:"`
+
+	invalidYAML = `
+- domain: [thisIsATestYamlThatIsMeantToFail.example]
+  regexRules:
+    - match: "^http:"
+      replace: "https:"
+    - match: "[incomplete"`
+)
+
+func TestLoadRulesFromRemoteFile(t *testing.T) {
+	app := fiber.New()
+	defer app.Shutdown()
+
+	app.Get("/valid-config.yml", func(c *fiber.Ctx) error {
+		c.SendString(validYAML)
+		return nil
+	})
+
+	app.Get("/invalid-config.yml", func(c *fiber.Ctx) error {
+		c.SendString(invalidYAML)
+		return nil
+	})
+
+	app.Get("/valid-config.gz", func(c *fiber.Ctx) error {
+		c.Set("Content-Type", "application/octet-stream")
+
+		rs, err := loadRuleFromString(validYAML)
+		if err != nil {
+			t.Errorf("failed to load valid yaml from string: %s", err.Error())
+		}
+
+		s, err := rs.GzipYaml()
+		if err != nil {
+			t.Errorf("failed to load gzip serialize yaml: %s", err.Error())
+		}
+
+		err = c.SendStream(s)
+		if err != nil {
+			t.Errorf("failed to stream gzip serialized yaml: %s", err.Error())
+		}
+		return nil
+	})
+
+	// Start the server in a goroutine
+	go func() {
+		if err := app.Listen("127.0.0.1:9999"); err != nil {
+			t.Errorf("Server failed to start: %s", err.Error())
+		}
+	}()
+
+	// Wait for the server to start
+	time.Sleep(time.Second * 1)
+
+	rs, err := NewRuleset("http://127.0.0.1:9999/valid-config.yml")
+	if err != nil {
+		t.Errorf("failed to load plaintext ruleset from http server: %s", err.Error())
+	}
+
+	assert.Equal(t, rs[0].Domain, "example.com")
+
+	rs, err = NewRuleset("http://127.0.0.1:9999/valid-config.gz")
+	if err != nil {
+		t.Errorf("failed to load gzipped ruleset from http server: %s", err.Error())
+	}
+
+	assert.Equal(t, rs[0].Domain, "example.com")
+
+	os.Setenv("RULESET", "http://127.0.0.1:9999/valid-config.gz")
+
+	rs = NewRulesetFromEnv()
+	if !assert.Equal(t, rs[0].Domain, "example.com") {
+		t.Error("expected no errors loading ruleset from gzip url using environment variable, but got one")
+	}
+}
+
+func loadRuleFromString(yaml string) (RuleSet, error) {
+	// Create a temporary file and load it
+	tmpFile, _ := os.CreateTemp("", "ruleset*.yaml")
+
+	defer os.Remove(tmpFile.Name())
+
+	tmpFile.WriteString(yaml)
+
+	rs := RuleSet{}
+	err := rs.loadRulesFromLocalFile(tmpFile.Name())
+
+	return rs, err
+}
+
+// TestLoadRulesFromLocalFile tests the loading of rules from a local YAML file.
+func TestLoadRulesFromLocalFile(t *testing.T) {
+	rs, err := loadRuleFromString(validYAML)
+	if err != nil {
+		t.Errorf("Failed to load rules from valid YAML: %s", err)
+	}
+
+	assert.Equal(t, rs[0].Domain, "example.com")
+	assert.Equal(t, rs[0].RegexRules[0].Match, "^http:")
+	assert.Equal(t, rs[0].RegexRules[0].Replace, "https:")
+
+	_, err = loadRuleFromString(invalidYAML)
+	if err == nil {
+		t.Errorf("Expected an error when loading invalid YAML, but got none")
+	}
+}
+
+// TestLoadRulesFromLocalDir tests the loading of rules from a local nested directory full of yaml rulesets
+func TestLoadRulesFromLocalDir(t *testing.T) {
+	// Create a temporary directory
+	baseDir, err := os.MkdirTemp("", "ruleset_test")
+	if err != nil {
+		t.Fatalf("Failed to create temporary directory: %s", err)
+	}
+
+	defer os.RemoveAll(baseDir)
+
+	// Create a nested subdirectory
+	nestedDir := filepath.Join(baseDir, "nested")
+	err = os.Mkdir(nestedDir, 0o755)
+
+	if err != nil {
+		t.Fatalf("Failed to create nested directory: %s", err)
+	}
+
+	// Create a nested subdirectory
+	nestedTwiceDir := filepath.Join(nestedDir, "nestedTwice")
+	err = os.Mkdir(nestedTwiceDir, 0o755)
+	if err != nil {
+		t.Fatalf("Failed to create twice-nested directory: %s", err)
+	}
+
+	testCases := []string{"test.yaml", "test2.yaml", "test-3.yaml", "test 4.yaml", "1987.test.yaml.yml", "foobar.example.com.yaml", "foobar.com.yml"}
+	for _, fileName := range testCases {
+		filePath := filepath.Join(nestedDir, "2x-"+fileName)
+		os.WriteFile(filePath, []byte(validYAML), 0o644)
+
+		filePath = filepath.Join(nestedDir, fileName)
+		os.WriteFile(filePath, []byte(validYAML), 0o644)
+
+		filePath = filepath.Join(baseDir, "base-"+fileName)
+		os.WriteFile(filePath, []byte(validYAML), 0o644)
+	}
+
+	rs := RuleSet{}
+	err = rs.loadRulesFromLocalDir(baseDir)
+
+	assert.NoError(t, err)
+	assert.Equal(t, rs.Count(), len(testCases)*3)
+
+	for _, rule := range rs {
+		assert.Equal(t, rule.Domain, "example.com")
+		assert.Equal(t, rule.RegexRules[0].Match, "^http:")
+		assert.Equal(t, rule.RegexRules[0].Replace, "https:")
+	}
+}

ruleset.yaml

@@ -20,146 +20,4 @@
         </script>
     - position: h1
       replace: |
-        <h1>An example with a ladder ;-)</h1>
+        <h1>An example with a ladder ;-)</h1>
-- domain: www.americanbanker.com
-  paths:
-    - /news
-  injections:
-    - position: head
-      append: |
-        <script>
-          document.addEventListener("DOMContentLoaded", () => {
-            const inlineGate = document.querySelector('.inline-gate');
-            if (inlineGate) {
-              inlineGate.classList.remove('inline-gate');
-              const inlineGated = document.querySelectorAll('.inline-gated');
-              for (const elem of inlineGated) { elem.classList.remove('inline-gated'); }
-            }
-          });
-        </script>
-- domain: www.nzz.ch
-  paths:
-    - /international
-    - /sport
-    - /wirtschaft
-    - /technologie
-    - /feuilleton
-    - /zuerich
-    - /wissenschaft
-    - /gesellschaft
-    - /panorama
-    - /mobilitaet
-    - /reisen
-    - /meinung
-    - /finanze
-  injections:
-    - position: head
-      append: |
-        <script>
-          document.addEventListener("DOMContentLoaded", () => {
-            const paywall = document.querySelector('.dynamic-regwall');
-            removeDOMElement(paywall)
-          });
-        </script>
-- domains: 
-  - www.architecturaldigest.com
-  - www.bonappetit.com
-  - www.cntraveler.com
-  - www.epicurious.com
-  - www.gq.com
-  - www.newyorker.com
-  - www.vanityfair.com
-  - www.vogue.com
-  - www.wired.com
-  injections:
-    - position: head
-      append: |
-        <script>
-          document.addEventListener("DOMContentLoaded", () => {
-            const banners = document.querySelectorAll('.paywall-bar, div[class^="MessageBannerWrapper-"');
-            banners.forEach(el => { el.remove(); });
-          });
-        </script>
-- domains: 
-  - www.nytimes.com
-  - www.time.com
-  headers:
-    ueser-agent: Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
-    cookie: nyt-a=; nyt-gdpr=0; nyt-geo=DE; nyt-privacy=1
-    referer: https://www.google.com/ 
-  injections:
-    - position: head
-      append: |
-        <script>
-          window.localStorage.clear();
-          document.addEventListener("DOMContentLoaded", () => {
-            const banners = document.querySelectorAll('div[data-testid="inline-message"], div[id^="ad-"], div[id^="leaderboard-"], div.expanded-dock, div.pz-ad-box, div[id="top-wrapper"], div[id="bottom-wrapper"]');
-            banners.forEach(el => { el.remove(); });
-          });
-        </script>
-- domains: 
-  - www.thestar.com
-  - www.niagarafallsreview.ca
-  - www.stcatharinesstandard.ca
-  - www.thepeterboroughexaminer.com
-  - www.therecord.com
-  - www.thespec.com
-  - www.wellandtribune.ca
-  injections:
-    - position: head
-      append: |
-        <script>
-          window.localStorage.clear();
-          document.addEventListener("DOMContentLoaded", () => {
-            const paywall = document.querySelectorAll('div.subscriber-offers');
-            paywall.forEach(el => { el.remove(); });
-            const subscriber_only = document.querySelectorAll('div.subscriber-only');
-            for (const elem of subscriber_only) {
-              if (elem.classList.contains('encrypted-content') && dompurify_loaded) {
-                const parser = new DOMParser();
-                const doc = parser.parseFromString('<div>' + DOMPurify.sanitize(unscramble(elem.innerText)) + '</div>', 'text/html');
-                const content_new = doc.querySelector('div');
-                elem.parentNode.replaceChild(content_new, elem);
-              }
-              elem.removeAttribute('style');
-              elem.removeAttribute('class');
-            }
-            const banners = document.querySelectorAll('div.subscription-required, div.redacted-overlay, div.subscriber-hide, div.tnt-ads-container');
-            banners.forEach(el => { el.remove(); });
-            const ads = document.querySelectorAll('div.tnt-ads-container, div[class*="adLabelWrapper"]');
-            ads.forEach(el => { el.remove(); });
-            const recommendations = document.querySelectorAll('div[id^="tncms-region-article"]');
-            recommendations.forEach(el => { el.remove(); });
-          });
-        </script>
-- domain: www.usatoday.com
-  injections:
-    - position: head
-      append: |
-        <script>
-          document.addEventListener("DOMContentLoaded", () => {
-            const banners = document.querySelectorAll('div.roadblock-container, .gnt_nb, [aria-label="advertisement"], div[id="main-frame-error"]');
-            banners.forEach(el => { el.remove(); });
-          });
-        </script>
-- domain: www.washingtonpost.com
-  injections:
-    - position: head
-      append: |
-        <script>
-          document.addEventListener("DOMContentLoaded", () => {
-            let paywall = document.querySelectorAll('div[data-qa$="-ad"], div[id="leaderboard-wrapper"], div[data-qa="subscribe-promo"]');
-            paywall.forEach(el => { el.remove(); });
-            const images = document.querySelectorAll('img');
-            images.forEach(image => { image.parentElement.style.filter = ''; });
-            const headimage = document.querySelectorAll('div .aspect-custom');
-            headimage.forEach(image => { image.style.filter = ''; });
-          });
-        </script>
-- domain: medium.com
-  headers:
-    referer: https://t.co/x?amp=1
-    x-forwarded-for: none
-    user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36
-    content-security-policy: script-src 'self';
-    cookie: 

rulesets/ca/_multi-metroland-media-group.yaml

@@ -0,0 +1,35 @@
+- domains: 
+  - www.thestar.com
+  - www.niagarafallsreview.ca
+  - www.stcatharinesstandard.ca
+  - www.thepeterboroughexaminer.com
+  - www.therecord.com
+  - www.thespec.com
+  - www.wellandtribune.ca
+  injections:
+    - position: head
+      append: |
+        <script>
+          window.localStorage.clear();
+          document.addEventListener("DOMContentLoaded", () => {
+            const paywall = document.querySelectorAll('div.subscriber-offers');
+            paywall.forEach(el => { el.remove(); });
+            const subscriber_only = document.querySelectorAll('div.subscriber-only');
+            for (const elem of subscriber_only) {
+              if (elem.classList.contains('encrypted-content') && dompurify_loaded) {
+                const parser = new DOMParser();
+                const doc = parser.parseFromString('<div>' + DOMPurify.sanitize(unscramble(elem.innerText)) + '</div>', 'text/html');
+                const content_new = doc.querySelector('div');
+                elem.parentNode.replaceChild(content_new, elem);
+              }
+              elem.removeAttribute('style');
+              elem.removeAttribute('class');
+            }
+            const banners = document.querySelectorAll('div.subscription-required, div.redacted-overlay, div.subscriber-hide, div.tnt-ads-container');
+            banners.forEach(el => { el.remove(); });
+            const ads = document.querySelectorAll('div.tnt-ads-container, div[class*="adLabelWrapper"]');
+            ads.forEach(el => { el.remove(); });
+            const recommendations = document.querySelectorAll('div[id^="tncms-region-article"]');
+            recommendations.forEach(el => { el.remove(); });
+          });
+        </script>

rulesets/ch/nzz-ch.yaml

@@ -0,0 +1,24 @@
+- domain: www.nzz.ch
+  paths:
+    - /international
+    - /sport
+    - /wirtschaft
+    - /technologie
+    - /feuilleton
+    - /zuerich
+    - /wissenschaft
+    - /gesellschaft
+    - /panorama
+    - /mobilitaet
+    - /reisen
+    - /meinung
+    - /finanze
+  injections:
+    - position: head
+      append: |
+        <script>
+          document.addEventListener("DOMContentLoaded", () => {
+            const paywall = document.querySelector('.dynamic-regwall');
+            removeDOMElement(paywall)
+          });
+        </script>

rulesets/de/tagesspiegel-de.yaml

@@ -0,0 +1,9 @@
+# loads amp version of page
+- domain: tagesspiegel.de
+  headers:
+    content-security-policy: script-src 'self';
+    user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36
+  urlMods:
+    query:
+      - key: amp
+        value: 1

rulesets/gb/ft-com.yaml

@@ -0,0 +1,20 @@
+- domain: www.ft.com
+  headers:
+    referer: https://t.co/x?amp=1
+  injections:
+    - position: head
+      append: |
+        <script>
+          document.addEventListener("DOMContentLoaded", () => {
+            const styleTags = document.querySelectorAll('link[rel="stylesheet"]');
+            styleTags.forEach(el => { 
+              const href = el.getAttribute('href').substring(1);
+              const updatedHref = href.replace(/(https?:\/\/.+?)\/{2,}/, '$1/');
+              el.setAttribute('href', updatedHref);
+            });
+            setTimeout(() => {
+              const cookie = document.querySelectorAll('.o-cookie-message, .js-article-ribbon, .o-ads, .o-banner, .o-message, .article__content-sign-up');
+              cookie.forEach(el => { el.remove(); });
+            }, 1000);
+          })
+        </script>

rulesets/us/_multi-conde-nast.yaml

@@ -0,0 +1,19 @@
+- domains: 
+  - www.architecturaldigest.com
+  - www.bonappetit.com
+  - www.cntraveler.com
+  - www.epicurious.com
+  - www.gq.com
+  - www.newyorker.com
+  - www.vanityfair.com
+  - www.vogue.com
+  - www.wired.com
+  injections:
+    - position: head
+      append: |
+        <script>
+          document.addEventListener("DOMContentLoaded", () => {
+            const banners = document.querySelectorAll('.paywall-bar, div[class^="MessageBannerWrapper-"');
+            banners.forEach(el => { el.remove(); });
+          });
+        </script>

rulesets/us/americanbanker-com.yaml

@@ -0,0 +1,16 @@
+- domain: americanbanker.com
+  paths:
+    - /news
+  injections:
+    - position: head
+      append: |
+        <script>
+          document.addEventListener("DOMContentLoaded", () => {
+            const inlineGate = document.querySelector('.inline-gate');
+            if (inlineGate) {
+              inlineGate.classList.remove('inline-gate');
+              const inlineGated = document.querySelectorAll('.inline-gated');
+              for (const elem of inlineGated) { elem.classList.remove('inline-gated'); }
+            }
+          });
+        </script>

rulesets/us/medium-com.yaml

@@ -0,0 +1,7 @@
+- domain: medium.com
+  headers:
+    referer: https://t.co/x?amp=1
+    x-forwarded-for: none
+    user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36
+    content-security-policy: script-src 'self';
+    cookie: 

rulesets/us/nytimes-com.yaml

@@ -0,0 +1,17 @@
+- domains: 
+  - www.nytimes.com
+  - www.time.com
+  headers:
+    ueser-agent: Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
+    cookie: nyt-a=; nyt-gdpr=0; nyt-geo=DE; nyt-privacy=1
+    referer: https://www.google.com/ 
+  injections:
+    - position: head
+      append: |
+        <script>
+          window.localStorage.clear();
+          document.addEventListener("DOMContentLoaded", () => {
+            const banners = document.querySelectorAll('div[data-testid="inline-message"], div[id^="ad-"], div[id^="leaderboard-"], div.expanded-dock, div.pz-ad-box, div[id="top-wrapper"], div[id="bottom-wrapper"]');
+            banners.forEach(el => { el.remove(); });
+          });
+        </script>

rulesets/us/usatoday-com.yaml

@@ -0,0 +1,10 @@
+- domain: www.usatoday.com
+  injections:
+    - position: head
+      append: |
+        <script>
+          document.addEventListener("DOMContentLoaded", () => {
+            const banners = document.querySelectorAll('div.roadblock-container, .gnt_nb, [aria-label="advertisement"], div[id="main-frame-error"]');
+            banners.forEach(el => { el.remove(); });
+          });
+        </script>

rulesets/us/washingtonpost-com.yaml

@@ -0,0 +1,14 @@
+- domain: www.washingtonpost.com
+  injections:
+    - position: head
+      append: |
+        <script>
+          document.addEventListener("DOMContentLoaded", () => {
+            let paywall = document.querySelectorAll('div[data-qa$="-ad"], div[id="leaderboard-wrapper"], div[data-qa="subscribe-promo"]');
+            paywall.forEach(el => { el.remove(); });
+            const images = document.querySelectorAll('img');
+            images.forEach(image => { image.parentElement.style.filter = ''; });
+            const headimage = document.querySelectorAll('div .aspect-custom');
+            headimage.forEach(image => { image.style.filter = ''; });
+          });
+        </script>

styles/input.css

@@ -0,0 +1,3 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;

tailwind.config.js

@@ -0,0 +1,9 @@
+/** @type {import('tailwindcss').Config} */
+module.exports = {
+    content: ["./handlers/**/*.html"],
+    theme: {
+      extend: {},
+    },
+    plugins: [],
+  }
+