Merge pull request #47 from dxbednarczyk/main

Idiomatize(?) ruleset package and run lint

.air.toml

@@ -0,0 +1,46 @@
+root = "./"
+testdata_dir = "testdata"
+tmp_dir = "tmp"
+
+[build]
+  args_bin = []
+  bin = "./tmp/main"
+  cmd = "go build -o ./tmp/main ./cmd"
+  delay = 1000
+  exclude_dir = ["assets", "tmp", "vendor", "testdata"]
+  exclude_file = []
+  exclude_regex = ["_test.go"]
+  exclude_unchanged = false
+  follow_symlink = false
+  full_bin = "RULESET=./ruleset.yaml ./tmp/main"
+  include_dir = []
+  include_ext = ["go", "tpl", "tmpl", "yaml", "html"]
+  include_file = []
+  kill_delay = "0s"
+  log = "build-errors.log"
+  poll = false
+  poll_interval = 0
+  post_cmd = []
+  pre_cmd = ["echo 'dev' > handlers/VERSION"]
+  rerun = false
+  rerun_delay = 500
+  send_interrupt = false
+  stop_on_error = false
+
+[color]
+  app = ""
+  build = "yellow"
+  main = "magenta"
+  runner = "green"
+  watcher = "cyan"
+
+[log]
+  main_only = false
+  time = false
+
+[misc]
+  clean_on_exit = true
+
+[screen]
+  clear_on_rebuild = true
+  keep_scroll = true

Makefile

@@ -1,6 +1,6 @@
 lint:
 	gofumpt -l -w .
-	golangci-lint run -c .golangci-lint.yaml
+	golangci-lint run -c .golangci-lint.yaml --fix
 
 	go mod tidy
 	go clean

README.md

@@ -14,6 +14,18 @@ Freedom of information is an essential pillar of democracy and informed decision
 
 > **Disclaimer:** This project is intended for educational purposes only. The author does not endorse or encourage any unethical or illegal activity. Use this tool at your own risk.
 
+### How it works
+
+```mermaid
+sequenceDiagram
+    client->>+ladder: GET
+    ladder-->>ladder: apply RequestModifications
+    ladder->>+website: GET
+    website->>-ladder: 200 OK
+    ladder-->>ladder: apply ResultModifications
+    ladder->>-client: 200 OK
+```
+
 ### Features
 - [x] Bypass Paywalls
 - [x] Remove CORS headers from responses, assets, and images ...
@@ -48,12 +60,12 @@ Certain sites may display missing images or encounter formatting issues. This ca
 
 ### Binary
 1) Download binary [here](https://github.com/everywall/ladder/releases/latest)
-2) Unpack and run the binary `./ladder`
+2) Unpack and run the binary `./ladder -r https://t.ly/14PSf`
 3) Open Browser (Default: http://localhost:8080)
 
 ### Docker
 ```bash
-docker run -p 8080:8080 -d --name ladder ghcr.io/everywall/ladder:latest
+docker run -p 8080:8080 -d --env RULESET=https://t.ly/14PSf --name ladder ghcr.io/everywall/ladder:latest
 ```
 
 ### Docker Compose
@@ -106,7 +118,7 @@ http://localhost:8080/ruleset
 | `LOG_URLS` | Log fetched URL's | `true` |
 | `DISABLE_FORM` | Disables URL Form Frontpage | `false` |
 | `FORM_PATH` | Path to custom Form HTML | `` |
-| `RULESET` | Path or URL to a ruleset file, accepts local directories | `https://raw.githubusercontent.com/everywall/ladder/main/ruleset.yaml` or `/path/to/my/rules.yaml` or `/path/to/my/rules/` |
+| `RULESET` | Path or URL to a ruleset file, accepts local directories | `https://raw.githubusercontent.com/everywall/ladder-rules/main/ruleset.yaml` or `/path/to/my/rules.yaml` or `/path/to/my/rules/` |
 | `EXPOSE_RULESET` | Make your Ruleset available to other ladders | `true` |
 | `ALLOWED_DOMAINS` | Comma separated list of allowed domains. Empty = no limitations | `` |
 | `ALLOWED_DOMAINS_RULESET` | Allow Domains from Ruleset. false = no limitations | `false` |
@@ -115,9 +127,10 @@ http://localhost:8080/ruleset
 
 ### Ruleset
 
-It is possible to apply custom rules to modify the response or the requested URL. This can be used to remove unwanted or modify elements from the page. The ruleset is a YAML file that contains a list of rules for each domain and is loaded on startup
+It is possible to apply custom rules to modify the response or the requested URL. This can be used to remove unwanted or modify elements from the page. The ruleset is a YAML file, a directory with YAML Files, or an URL to a YAML file that contains a list of rules for each domain. These rules are loaded on startup.
+
+There is a basic ruleset available in a separate repository [ruleset.yaml](https://raw.githubusercontent.com/everywall/ladder-rules/main/ruleset.yaml). Feel free to add your own rules and create a pull request.
 
-See in [ruleset.yaml](ruleset.yaml) for an example.
 
 ```yaml
 - domain: example.com          # Includes all subdomains
@@ -180,4 +193,14 @@ echo "dev" > handlers/VERSION
 RULESET="./ruleset.yaml" go run cmd/main.go
 ```
 
+### Optional: Live reloading development server with [cosmtrek/air](https://github.com/cosmtrek/air)
+
+Install air according to the [installation instructions](https://github.com/cosmtrek/air#installation). 
+
+Run a development server at http://localhost:8080:
+
+```bash
+air # or the path to air if you haven't added a path alias to your .bashrc or .zshrc
+```
+
 This project uses [pnpm](https://pnpm.io/) to build a stylesheet with the [Tailwind CSS](https://tailwindcss.com/) classes. For local development, if you modify styles in `form.html`, run `pnpm build` to generate a new stylesheet.

cmd/main.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 
 	"ladder/handlers"
+	"ladder/handlers/cli"
 
 	"github.com/akamensky/argparse"
 	"github.com/gofiber/fiber/v2"
@@ -17,6 +18,7 @@ import (
 
 //go:embed favicon.ico
 var faviconData string
+
 //go:embed styles.css
 var cssData embed.FS
 
@@ -27,6 +29,7 @@ func main() {
 	if os.Getenv("PORT") == "" {
 		portEnv = "8080"
 	}
+
 	port := parser.String("p", "port", &argparse.Options{
 		Required: false,
 		Default:  portEnv,
@@ -40,7 +43,22 @@ func main() {
 
 	ruleset := parser.String("r", "ruleset", &argparse.Options{
 		Required: false,
-		Help:     "File, Directory or URL to a ruleset.yml. Overrides RULESET environment variable.",
+		Help:     "File, Directory or URL to a ruleset.yaml. Overrides RULESET environment variable.",
+	})
+
+	mergeRulesets := parser.Flag("", "merge-rulesets", &argparse.Options{
+		Required: false,
+		Help:     "Compiles a directory of yaml files into a single ruleset.yaml. Requires --ruleset arg.",
+	})
+
+	mergeRulesetsGzip := parser.Flag("", "merge-rulesets-gzip", &argparse.Options{
+		Required: false,
+		Help:     "Compiles a directory of yaml files into a single ruleset.gz Requires --ruleset arg.",
+	})
+
+	mergeRulesetsOutput := parser.String("", "merge-rulesets-output", &argparse.Options{
+		Required: false,
+		Help:     "Specify output file for --merge-rulesets and --merge-rulesets-gzip. Requires --ruleset and --merge-rulesets args.",
 	})
 
 	err := parser.Parse(os.Args)
@@ -48,6 +66,27 @@ func main() {
 		fmt.Print(parser.Usage(err))
 	}
 
+	// utility cli flag to compile ruleset directory into single ruleset.yaml
+	if *mergeRulesets || *mergeRulesetsGzip {
+		output := os.Stdout
+
+		if *mergeRulesetsOutput != "" {
+			output, err = os.Create(*mergeRulesetsOutput)
+			
+			if err != nil {
+				fmt.Println(err)
+				os.Exit(1)
+			}
+		}
+
+		err = cli.HandleRulesetMerge(*ruleset, *mergeRulesets, *mergeRulesetsGzip, output)
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+		os.Exit(0)
+	}
+
 	if os.Getenv("PREFORK") == "true" {
 		*prefork = true
 	}
@@ -62,6 +101,7 @@ func main() {
 	userpass := os.Getenv("USERPASS")
 	if userpass != "" {
 		userpass := strings.Split(userpass, ":")
+
 		app.Use(basicauth.New(basicauth.Config{
 			Users: map[string]string{
 				userpass[0]: userpass[1],
@@ -77,23 +117,28 @@ func main() {
 	if os.Getenv("NOLOGS") != "true" {
 		app.Use(func(c *fiber.Ctx) error {
 			log.Println(c.Method(), c.Path())
+
 			return c.Next()
 		})
 	}
 
 	app.Get("/", handlers.Form)
+
 	app.Get("/styles.css", func(c *fiber.Ctx) error {
 		cssData, err := cssData.ReadFile("styles.css")
 		if err != nil {
 			return c.Status(fiber.StatusInternalServerError).SendString("Internal Server Error")
 		}
+
 		c.Set("Content-Type", "text/css")
+
 		return c.Send(cssData)
 	})
-	app.Get("ruleset", handlers.Ruleset)
 
+	app.Get("ruleset", handlers.Ruleset)
 	app.Get("raw/*", handlers.Raw)
 	app.Get("api/*", handlers.Api)
 	app.Get("/*", handlers.ProxySite(*ruleset))
+
 	log.Fatal(app.Listen(":" + *port))
 }

go.mod

@@ -26,4 +26,5 @@ require (
 	github.com/valyala/tcplisten v1.0.0 // indirect
 	golang.org/x/net v0.18.0 // indirect
 	golang.org/x/sys v0.14.0 // indirect
+	golang.org/x/term v0.14.0
 )

go.sum

@@ -68,6 +68,8 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
+golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8=
+golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=

handlers/cli/cli.go

@@ -0,0 +1,113 @@
+package cli
+
+import (
+	"fmt"
+	"io"
+	"os"
+
+	"ladder/pkg/ruleset"
+
+	"golang.org/x/term"
+)
+
+// HandleRulesetMerge merges a set of ruleset files, specified by the rulesetPath or RULESET env variable, into either YAML or Gzip format.
+// Exits the program with an error message if the ruleset path is not provided or if loading the ruleset fails.
+//
+// Parameters:
+// - rulesetPath: Specifies the path to the ruleset file.
+// - mergeRulesets: Indicates if a merge operation should be performed.
+// - useGzip: Indicates if the merged rulesets should be gzip-ped.
+// - output: Specifies the output file. If nil, stdout will be used.
+//
+// Returns:
+// - An error if the ruleset loading or merging process fails, otherwise nil.
+func HandleRulesetMerge(rulesetPath string, mergeRulesets bool, useGzip bool, output *os.File) error {
+	if !mergeRulesets {
+		return nil
+	}
+
+	if rulesetPath == "" {
+		rulesetPath = os.Getenv("RULESET")
+	}
+
+	if rulesetPath == "" {
+		fmt.Println("error: no ruleset provided. Try again with --ruleset <ruleset.yaml>")
+		os.Exit(1)
+	}
+
+	rs, err := ruleset.NewRuleset(rulesetPath)
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	if useGzip {
+		return gzipMerge(rs, output)
+	}
+
+	return yamlMerge(rs, output)
+}
+
+// gzipMerge takes a RuleSet and an optional output file path pointer. It compresses the RuleSet into Gzip format.
+// If the output file path is provided, the compressed data is written to this file. Otherwise, it prints a warning
+// and outputs the binary data to stdout
+//
+// Parameters:
+// - rs: The ruleset.RuleSet to be compressed.
+// - output: The output for the gzip data. If nil, stdout will be used.
+//
+// Returns:
+// - An error if compression or file writing fails, otherwise nil.
+func gzipMerge(rs ruleset.RuleSet, output io.Writer) error {
+	gzip, err := rs.GzipYaml()
+	if err != nil {
+		return err
+	}
+
+	if output != nil {
+		_, err = io.Copy(output, gzip)
+		if err != nil {
+			return err
+		}
+	}
+
+	if term.IsTerminal(int(os.Stdout.Fd())) {
+		println("warning: binary output can mess up your terminal. Use '--merge-rulesets-output <ruleset.gz>' or pipe it to a file.")
+		os.Exit(1)
+	}
+
+	_, err = io.Copy(os.Stdout, gzip)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// yamlMerge takes a RuleSet and an optional output file path pointer. It converts the RuleSet into YAML format.
+// If the output file path is provided, the YAML data is written to this file. If not, the YAML data is printed to stdout.
+//
+// Parameters:
+// - rs: The ruleset.RuleSet to be converted to YAML.
+// - output: The output for the merged data. If nil, stdout will be used.
+//
+// Returns:
+// - An error if YAML conversion or file writing fails, otherwise nil.
+func yamlMerge(rs ruleset.RuleSet, output io.Writer) error {
+	yaml, err := rs.Yaml()
+	if err != nil {
+		return err
+	}
+
+	if output == nil {
+		fmt.Println(yaml)
+		os.Exit(0)
+	}
+
+	_, err = io.WriteString(output, yaml)
+	if err != nil {
+		return fmt.Errorf("failed to write merged YAML ruleset: %v", err)
+	}
+
+	return nil
+}

handlers/proxy.go

@@ -80,7 +80,6 @@ func extractUrl(c *fiber.Ctx) (string, error) {
 	// default behavior:
 	// eg: https://localhost:8080/https://realsite.com/images/foobar.jpg -> https://realsite.com/images/foobar.jpg
 	return urlQuery.String(), nil
-
 }
 
 func ProxySite(rulesetPath string) fiber.Handler {
@@ -121,18 +120,18 @@ func modifyURL(uri string, rule ruleset.Rule) (string, error) {
 		return "", err
 	}
 
-	for _, urlMod := range rule.UrlMods.Domain {
+	for _, urlMod := range rule.URLMods.Domain {
 		re := regexp.MustCompile(urlMod.Match)
 		newUrl.Host = re.ReplaceAllString(newUrl.Host, urlMod.Replace)
 	}
 
-	for _, urlMod := range rule.UrlMods.Path {
+	for _, urlMod := range rule.URLMods.Path {
 		re := regexp.MustCompile(urlMod.Match)
 		newUrl.Path = re.ReplaceAllString(newUrl.Path, urlMod.Replace)
 	}
 
 	v := newUrl.Query()
-	for _, query := range rule.UrlMods.Query {
+	for _, query := range rule.URLMods.Query {
 		if query.Value == "" {
 			v.Del(query.Key)
 			continue
@@ -223,11 +222,11 @@ func fetchSite(urlpath string, queries map[string]string) (string, *http.Request
 	}
 
 	if rule.Headers.CSP != "" {
-		//log.Println(rule.Headers.CSP)
+		// log.Println(rule.Headers.CSP)
 		resp.Header.Set("Content-Security-Policy", rule.Headers.CSP)
 	}
 
-	//log.Print("rule", rule) TODO: Add a debug mode to print the rule
+	// log.Print("rule", rule) TODO: Add a debug mode to print the rule
 	body := rewriteHtml(bodyB, u, rule)
 	return body, req, resp, nil
 }

handlers/proxy.test.go

@@ -2,12 +2,13 @@
 package handlers
 
 import (
-	"ladder/pkg/ruleset"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
 	"testing"
 
+	"ladder/pkg/ruleset"
+
 	"github.com/gofiber/fiber/v2"
 	"github.com/stretchr/testify/assert"
 )

pkg/ruleset/ruleset.go

@@ -1,6 +1,7 @@
 package ruleset
 
 import (
+	"compress/gzip"
 	"errors"
 	"fmt"
 	"io"
@@ -11,8 +12,6 @@ import (
 	"regexp"
 	"strings"
 
-	"compress/gzip"
-
 	"gopkg.in/yaml.v3"
 )
 
@@ -39,22 +38,24 @@ type Rule struct {
 		CSP           string `yaml:"content-security-policy,omitempty"`
 	} `yaml:"headers,omitempty"`
 	GoogleCache bool    `yaml:"googleCache,omitempty"`
-	RegexRules  []Regex `yaml:"regexRules"`
+	RegexRules  []Regex `yaml:"regexRules,omitempty"`
 
-	UrlMods struct {
+	URLMods struct {
-		Domain []Regex `yaml:"domain"`
+		Domain []Regex `yaml:"domain,omitempty"`
-		Path   []Regex `yaml:"path"`
+		Path   []Regex `yaml:"path,omitempty"`
-		Query  []KV    `yaml:"query"`
+		Query  []KV    `yaml:"query,omitempty"`
-	} `yaml:"urlMods"`
+	} `yaml:"urlMods,omitempty"`
 
 	Injections []struct {
-		Position string `yaml:"position"`
+		Position string `yaml:"position,omitempty"`
-		Append   string `yaml:"append"`
+		Append   string `yaml:"append,omitempty"`
-		Prepend  string `yaml:"prepend"`
+		Prepend  string `yaml:"prepend,omitempty"`
-		Replace  string `yaml:"replace"`
+		Replace  string `yaml:"replace,omitempty"`
-	} `yaml:"injections"`
+	} `yaml:"injections,omitempty"`
 }
 
+var remoteRegex = regexp.MustCompile(`^https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()!@:%_\+.~#?&\/\/=]*)`)
+
 // NewRulesetFromEnv creates a new RuleSet based on the RULESET environment variable.
 // It logs a warning and returns an empty RuleSet if the RULESET environment variable is not set.
 // If the RULESET is set but the rules cannot be loaded, it panics.
@@ -64,10 +65,12 @@ func NewRulesetFromEnv() RuleSet {
 		log.Printf("WARN: No ruleset specified. Set the `RULESET` environment variable to load one for a better success rate.")
 		return RuleSet{}
 	}
+
 	ruleSet, err := NewRuleset(rulesPath)
 	if err != nil {
 		log.Println(err)
 	}
+
 	return ruleSet
 }
 
@@ -75,16 +78,17 @@ func NewRulesetFromEnv() RuleSet {
 // It supports loading rules from both local file paths and remote URLs.
 // Returns a RuleSet and an error if any issues occur during loading.
 func NewRuleset(rulePaths string) (RuleSet, error) {
-	ruleSet := RuleSet{}
+	var ruleSet RuleSet
-	errs := []error{}
+
+	var errs []error
 
 	rp := strings.Split(rulePaths, ";")
-	var remoteRegex = regexp.MustCompile(`^https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()!@:%_\+.~#?&\/\/=]*)`)
 	for _, rule := range rp {
-		rulePath := strings.Trim(rule, " ")
 		var err error
 
+		rulePath := strings.Trim(rule, " ")
 		isRemote := remoteRegex.MatchString(rulePath)
+
 		if isRemote {
 			err = ruleSet.loadRulesFromRemoteFile(rulePath)
 		} else {
@@ -94,6 +98,7 @@ func NewRuleset(rulePaths string) (RuleSet, error) {
 		if err != nil {
 			e := fmt.Errorf("WARN: failed to load ruleset from '%s'", rulePath)
 			errs = append(errs, errors.Join(e, err))
+
 			continue
 		}
 	}
@@ -101,6 +106,7 @@ func NewRuleset(rulePaths string) (RuleSet, error) {
 	if len(errs) != 0 {
 		e := fmt.Errorf("WARN: failed to load %d rulesets", len(rp))
 		errs = append(errs, e)
+
 		// panic if the user specified a local ruleset, but it wasn't found on disk
 		// don't fail silently
 		for _, err := range errs {
@@ -109,10 +115,13 @@ func NewRuleset(rulePaths string) (RuleSet, error) {
 				panic(errors.Join(e, err))
 			}
 		}
+
 		// else, bubble up any errors, such as syntax or remote host issues
 		return ruleSet, errors.Join(errs...)
 	}
+
 	ruleSet.PrintStats()
+
 	return ruleSet, nil
 }
 
@@ -146,13 +155,16 @@ func (rs *RuleSet) loadRulesFromLocalDir(path string) error {
 			log.Printf("WARN: failed to load directory ruleset '%s': %s, skipping", path, err)
 			return nil
 		}
+
 		log.Printf("INFO: loaded ruleset %s\n", path)
+
 		return nil
 	})
 
 	if err != nil {
 		return err
 	}
+
 	return nil
 }
 
@@ -167,42 +179,51 @@ func (rs *RuleSet) loadRulesFromLocalFile(path string) error {
 
 	var r RuleSet
 	err = yaml.Unmarshal(yamlFile, &r)
+
 	if err != nil {
 		e := fmt.Errorf("failed to load rules from local file, possible syntax error in '%s'", path)
 		ee := errors.Join(e, err)
+
 		if _, ok := os.LookupEnv("DEBUG"); ok {
 			debugPrintRule(string(yamlFile), ee)
 		}
+
 		return ee
 	}
+
 	*rs = append(*rs, r...)
+
 	return nil
 }
 
 // loadRulesFromRemoteFile loads rules from a remote URL.
 // It supports plain and gzip compressed content.
 // Returns an error if there's an issue accessing the URL or if there's a syntax error in the YAML.
-func (rs *RuleSet) loadRulesFromRemoteFile(rulesUrl string) error {
+func (rs *RuleSet) loadRulesFromRemoteFile(rulesURL string) error {
 	var r RuleSet
-	resp, err := http.Get(rulesUrl)
+
+	resp, err := http.Get(rulesURL)
 	if err != nil {
-		e := fmt.Errorf("failed to load rules from remote url '%s'", rulesUrl)
+		e := fmt.Errorf("failed to load rules from remote url '%s'", rulesURL)
 		return errors.Join(e, err)
 	}
+
 	defer resp.Body.Close()
 
 	if resp.StatusCode >= 400 {
-		e := fmt.Errorf("failed to load rules from remote url (%s) on '%s'", resp.Status, rulesUrl)
+		e := fmt.Errorf("failed to load rules from remote url (%s) on '%s'", resp.Status, rulesURL)
 		return errors.Join(e, err)
 	}
 
 	var reader io.Reader
-	isGzip := strings.HasSuffix(rulesUrl, ".gz") || strings.HasSuffix(rulesUrl, ".gzip") || resp.Header.Get("content-encoding") == "gzip"
+
+	isGzip := strings.HasSuffix(rulesURL, ".gz") || strings.HasSuffix(rulesURL, ".gzip") || resp.Header.Get("content-encoding") == "gzip"
 
 	if isGzip {
 		reader, err = gzip.NewReader(resp.Body)
+
 		if err != nil {
-			return fmt.Errorf("failed to create gzip reader for URL '%s' with status code '%s': %w", rulesUrl, resp.Status, err)
+			return fmt.Errorf("failed to create gzip reader for URL '%s' with status code '%s': %w", rulesURL, resp.Status, err)
 		}
 	} else {
 		reader = resp.Body
@@ -211,12 +232,14 @@ func (rs *RuleSet) loadRulesFromRemoteFile(rulesUrl string) error {
 	err = yaml.NewDecoder(reader).Decode(&r)
 
 	if err != nil {
-		e := fmt.Errorf("failed to load rules from remote url '%s' with status code '%s' and possible syntax error", rulesUrl, resp.Status)
+		e := fmt.Errorf("failed to load rules from remote url '%s' with status code '%s' and possible syntax error", rulesURL, resp.Status)
 		ee := errors.Join(e, err)
+
 		return ee
 	}
 
 	*rs = append(*rs, r...)
+
 	return nil
 }
 
@@ -228,6 +251,7 @@ func (rs *RuleSet) Yaml() (string, error) {
 	if err != nil {
 		return "", err
 	}
+
 	return string(y), nil
 }
 

pkg/ruleset/ruleset_test.go

@@ -33,6 +33,7 @@ func TestLoadRulesFromRemoteFile(t *testing.T) {
 		c.SendString(validYAML)
 		return nil
 	})
+
 	app.Get("/invalid-config.yml", func(c *fiber.Ctx) error {
 		c.SendString(invalidYAML)
 		return nil
@@ -40,10 +41,12 @@ func TestLoadRulesFromRemoteFile(t *testing.T) {
 
 	app.Get("/valid-config.gz", func(c *fiber.Ctx) error {
 		c.Set("Content-Type", "application/octet-stream")
+
 		rs, err := loadRuleFromString(validYAML)
 		if err != nil {
 			t.Errorf("failed to load valid yaml from string: %s", err.Error())
 		}
+
 		s, err := rs.GzipYaml()
 		if err != nil {
 			t.Errorf("failed to load gzip serialize yaml: %s", err.Error())
@@ -70,15 +73,18 @@ func TestLoadRulesFromRemoteFile(t *testing.T) {
 	if err != nil {
 		t.Errorf("failed to load plaintext ruleset from http server: %s", err.Error())
 	}
+
 	assert.Equal(t, rs[0].Domain, "example.com")
 
 	rs, err = NewRuleset("http://127.0.0.1:9999/valid-config.gz")
 	if err != nil {
 		t.Errorf("failed to load gzipped ruleset from http server: %s", err.Error())
 	}
+
 	assert.Equal(t, rs[0].Domain, "example.com")
 
 	os.Setenv("RULESET", "http://127.0.0.1:9999/valid-config.gz")
+
 	rs = NewRulesetFromEnv()
 	if !assert.Equal(t, rs[0].Domain, "example.com") {
 		t.Error("expected no errors loading ruleset from gzip url using environment variable, but got one")
@@ -88,10 +94,14 @@ func TestLoadRulesFromRemoteFile(t *testing.T) {
 func loadRuleFromString(yaml string) (RuleSet, error) {
 	// Create a temporary file and load it
 	tmpFile, _ := os.CreateTemp("", "ruleset*.yaml")
+
 	defer os.Remove(tmpFile.Name())
+
 	tmpFile.WriteString(yaml)
+
 	rs := RuleSet{}
 	err := rs.loadRulesFromLocalFile(tmpFile.Name())
+
 	return rs, err
 }
 
@@ -101,6 +111,7 @@ func TestLoadRulesFromLocalFile(t *testing.T) {
 	if err != nil {
 		t.Errorf("Failed to load rules from valid YAML: %s", err)
 	}
+
 	assert.Equal(t, rs[0].Domain, "example.com")
 	assert.Equal(t, rs[0].RegexRules[0].Match, "^http:")
 	assert.Equal(t, rs[0].RegexRules[0].Replace, "https:")
@@ -118,30 +129,39 @@ func TestLoadRulesFromLocalDir(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Failed to create temporary directory: %s", err)
 	}
+
 	defer os.RemoveAll(baseDir)
 
 	// Create a nested subdirectory
 	nestedDir := filepath.Join(baseDir, "nested")
-	err = os.Mkdir(nestedDir, 0755)
+	err = os.Mkdir(nestedDir, 0o755)
+
 	if err != nil {
 		t.Fatalf("Failed to create nested directory: %s", err)
 	}
 
 	// Create a nested subdirectory
 	nestedTwiceDir := filepath.Join(nestedDir, "nestedTwice")
-	err = os.Mkdir(nestedTwiceDir, 0755)
+	err = os.Mkdir(nestedTwiceDir, 0o755)
+	if err != nil {
+		t.Fatalf("Failed to create twice-nested directory: %s", err)
+	}
 
 	testCases := []string{"test.yaml", "test2.yaml", "test-3.yaml", "test 4.yaml", "1987.test.yaml.yml", "foobar.example.com.yaml", "foobar.com.yml"}
 	for _, fileName := range testCases {
 		filePath := filepath.Join(nestedDir, "2x-"+fileName)
-		os.WriteFile(filePath, []byte(validYAML), 0644)
+		os.WriteFile(filePath, []byte(validYAML), 0o644)
+
 		filePath = filepath.Join(nestedDir, fileName)
-		os.WriteFile(filePath, []byte(validYAML), 0644)
+		os.WriteFile(filePath, []byte(validYAML), 0o644)
+
 		filePath = filepath.Join(baseDir, "base-"+fileName)
-		os.WriteFile(filePath, []byte(validYAML), 0644)
+		os.WriteFile(filePath, []byte(validYAML), 0o644)
 	}
+
 	rs := RuleSet{}
 	err = rs.loadRulesFromLocalDir(baseDir)
+
 	assert.NoError(t, err)
 	assert.Equal(t, rs.Count(), len(testCases)*3)
 

ruleset.yaml

@@ -21,174 +21,3 @@
     - position: h1
       replace: |
         <h1>An example with a ladder ;-)</h1>
-- domain: www.americanbanker.com
-  paths:
-    - /news
-  injections:
-    - position: head
-      append: |
-        <script>
-          document.addEventListener("DOMContentLoaded", () => {
-            const inlineGate = document.querySelector('.inline-gate');
-            if (inlineGate) {
-              inlineGate.classList.remove('inline-gate');
-              const inlineGated = document.querySelectorAll('.inline-gated');
-              for (const elem of inlineGated) { elem.classList.remove('inline-gated'); }
-            }
-          });
-        </script>
-- domain: www.nzz.ch
-  paths:
-    - /international
-    - /sport
-    - /wirtschaft
-    - /technologie
-    - /feuilleton
-    - /zuerich
-    - /wissenschaft
-    - /gesellschaft
-    - /panorama
-    - /mobilitaet
-    - /reisen
-    - /meinung
-    - /finanze
-  injections:
-    - position: head
-      append: |
-        <script>
-          document.addEventListener("DOMContentLoaded", () => {
-            const paywall = document.querySelector('.dynamic-regwall');
-            removeDOMElement(paywall)
-          });
-        </script>
-- domains: 
-  - www.architecturaldigest.com
-  - www.bonappetit.com
-  - www.cntraveler.com
-  - www.epicurious.com
-  - www.gq.com
-  - www.newyorker.com
-  - www.vanityfair.com
-  - www.vogue.com
-  - www.wired.com
-  injections:
-    - position: head
-      append: |
-        <script>
-          document.addEventListener("DOMContentLoaded", () => {
-            const banners = document.querySelectorAll('.paywall-bar, div[class^="MessageBannerWrapper-"');
-            banners.forEach(el => { el.remove(); });
-          });
-        </script>
-- domains: 
-  - www.nytimes.com
-  - www.time.com
-  headers:
-    ueser-agent: Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
-    cookie: nyt-a=; nyt-gdpr=0; nyt-geo=DE; nyt-privacy=1
-    referer: https://www.google.com/ 
-  injections:
-    - position: head
-      append: |
-        <script>
-          window.localStorage.clear();
-          document.addEventListener("DOMContentLoaded", () => {
-            const banners = document.querySelectorAll('div[data-testid="inline-message"], div[id^="ad-"], div[id^="leaderboard-"], div.expanded-dock, div.pz-ad-box, div[id="top-wrapper"], div[id="bottom-wrapper"]');
-            banners.forEach(el => { el.remove(); });
-          });
-        </script>
-- domains: 
-  - www.thestar.com
-  - www.niagarafallsreview.ca
-  - www.stcatharinesstandard.ca
-  - www.thepeterboroughexaminer.com
-  - www.therecord.com
-  - www.thespec.com
-  - www.wellandtribune.ca
-  injections:
-    - position: head
-      append: |
-        <script>
-          window.localStorage.clear();
-          document.addEventListener("DOMContentLoaded", () => {
-            const paywall = document.querySelectorAll('div.subscriber-offers');
-            paywall.forEach(el => { el.remove(); });
-            const subscriber_only = document.querySelectorAll('div.subscriber-only');
-            for (const elem of subscriber_only) {
-              if (elem.classList.contains('encrypted-content') && dompurify_loaded) {
-                const parser = new DOMParser();
-                const doc = parser.parseFromString('<div>' + DOMPurify.sanitize(unscramble(elem.innerText)) + '</div>', 'text/html');
-                const content_new = doc.querySelector('div');
-                elem.parentNode.replaceChild(content_new, elem);
-              }
-              elem.removeAttribute('style');
-              elem.removeAttribute('class');
-            }
-            const banners = document.querySelectorAll('div.subscription-required, div.redacted-overlay, div.subscriber-hide, div.tnt-ads-container');
-            banners.forEach(el => { el.remove(); });
-            const ads = document.querySelectorAll('div.tnt-ads-container, div[class*="adLabelWrapper"]');
-            ads.forEach(el => { el.remove(); });
-            const recommendations = document.querySelectorAll('div[id^="tncms-region-article"]');
-            recommendations.forEach(el => { el.remove(); });
-          });
-        </script>
-- domain: www.usatoday.com
-  injections:
-    - position: head
-      append: |
-        <script>
-          document.addEventListener("DOMContentLoaded", () => {
-            const banners = document.querySelectorAll('div.roadblock-container, .gnt_nb, [aria-label="advertisement"], div[id="main-frame-error"]');
-            banners.forEach(el => { el.remove(); });
-          });
-        </script>
-- domain: www.washingtonpost.com
-  injections:
-    - position: head
-      append: |
-        <script>
-          document.addEventListener("DOMContentLoaded", () => {
-            let paywall = document.querySelectorAll('div[data-qa$="-ad"], div[id="leaderboard-wrapper"], div[data-qa="subscribe-promo"]');
-            paywall.forEach(el => { el.remove(); });
-            const images = document.querySelectorAll('img');
-            images.forEach(image => { image.parentElement.style.filter = ''; });
-            const headimage = document.querySelectorAll('div .aspect-custom');
-            headimage.forEach(image => { image.style.filter = ''; });
-          });
-        </script>
-- domain: medium.com
-  headers:
-    referer: https://t.co/x?amp=1
-    x-forwarded-for: none
-    user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36
-    content-security-policy: script-src 'self';
-    cookie: 
-- domain: tagesspiegel.de
-  headers:
-    content-security-policy: script-src 'self';
-    user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36
-  urlMods:
-    query:
-      - key: amp
-        value: 1
-- domain: www.ft.com
-  headers:
-    referer: https://t.co/x?amp=1
-  injections:
-    - position: head
-      append: |
-        <script>
-          document.addEventListener("DOMContentLoaded", () => {
-            const styleTags = document.querySelectorAll('link[rel="stylesheet"]');
-            styleTags.forEach(el => { 
-              const href = el.getAttribute('href').substring(1);
-              const updatedHref = href.replace(/(https?:\/\/.+?)\/{2,}/, '$1/');
-              el.setAttribute('href', updatedHref);
-            });
-            setTimeout(() => {
-              const cookie = document.querySelectorAll('.o-cookie-message, .js-article-ribbon, .o-ads, .o-banner, .o-message, .article__content-sign-up');
-              cookie.forEach(el => { el.remove(); });
-            }, 1000);
-          })
-        </script>
-

rulesets/ca/_multi-metroland-media-group.yaml

@@ -0,0 +1,35 @@
+- domains: 
+  - www.thestar.com
+  - www.niagarafallsreview.ca
+  - www.stcatharinesstandard.ca
+  - www.thepeterboroughexaminer.com
+  - www.therecord.com
+  - www.thespec.com
+  - www.wellandtribune.ca
+  injections:
+    - position: head
+      append: |
+        <script>
+          window.localStorage.clear();
+          document.addEventListener("DOMContentLoaded", () => {
+            const paywall = document.querySelectorAll('div.subscriber-offers');
+            paywall.forEach(el => { el.remove(); });
+            const subscriber_only = document.querySelectorAll('div.subscriber-only');
+            for (const elem of subscriber_only) {
+              if (elem.classList.contains('encrypted-content') && dompurify_loaded) {
+                const parser = new DOMParser();
+                const doc = parser.parseFromString('<div>' + DOMPurify.sanitize(unscramble(elem.innerText)) + '</div>', 'text/html');
+                const content_new = doc.querySelector('div');
+                elem.parentNode.replaceChild(content_new, elem);
+              }
+              elem.removeAttribute('style');
+              elem.removeAttribute('class');
+            }
+            const banners = document.querySelectorAll('div.subscription-required, div.redacted-overlay, div.subscriber-hide, div.tnt-ads-container');
+            banners.forEach(el => { el.remove(); });
+            const ads = document.querySelectorAll('div.tnt-ads-container, div[class*="adLabelWrapper"]');
+            ads.forEach(el => { el.remove(); });
+            const recommendations = document.querySelectorAll('div[id^="tncms-region-article"]');
+            recommendations.forEach(el => { el.remove(); });
+          });
+        </script>

rulesets/ch/nzz-ch.yaml

@@ -0,0 +1,24 @@
+- domain: www.nzz.ch
+  paths:
+    - /international
+    - /sport
+    - /wirtschaft
+    - /technologie
+    - /feuilleton
+    - /zuerich
+    - /wissenschaft
+    - /gesellschaft
+    - /panorama
+    - /mobilitaet
+    - /reisen
+    - /meinung
+    - /finanze
+  injections:
+    - position: head
+      append: |
+        <script>
+          document.addEventListener("DOMContentLoaded", () => {
+            const paywall = document.querySelector('.dynamic-regwall');
+            removeDOMElement(paywall)
+          });
+        </script>

rulesets/de/tagesspiegel-de.yaml

@@ -0,0 +1,9 @@
+# loads amp version of page
+- domain: tagesspiegel.de
+  headers:
+    content-security-policy: script-src 'self';
+    user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36
+  urlMods:
+    query:
+      - key: amp
+        value: 1

rulesets/gb/ft-com.yaml

@@ -0,0 +1,20 @@
+- domain: www.ft.com
+  headers:
+    referer: https://t.co/x?amp=1
+  injections:
+    - position: head
+      append: |
+        <script>
+          document.addEventListener("DOMContentLoaded", () => {
+            const styleTags = document.querySelectorAll('link[rel="stylesheet"]');
+            styleTags.forEach(el => { 
+              const href = el.getAttribute('href').substring(1);
+              const updatedHref = href.replace(/(https?:\/\/.+?)\/{2,}/, '$1/');
+              el.setAttribute('href', updatedHref);
+            });
+            setTimeout(() => {
+              const cookie = document.querySelectorAll('.o-cookie-message, .js-article-ribbon, .o-ads, .o-banner, .o-message, .article__content-sign-up');
+              cookie.forEach(el => { el.remove(); });
+            }, 1000);
+          })
+        </script>

rulesets/us/_multi-conde-nast.yaml

@@ -0,0 +1,19 @@
+- domains: 
+  - www.architecturaldigest.com
+  - www.bonappetit.com
+  - www.cntraveler.com
+  - www.epicurious.com
+  - www.gq.com
+  - www.newyorker.com
+  - www.vanityfair.com
+  - www.vogue.com
+  - www.wired.com
+  injections:
+    - position: head
+      append: |
+        <script>
+          document.addEventListener("DOMContentLoaded", () => {
+            const banners = document.querySelectorAll('.paywall-bar, div[class^="MessageBannerWrapper-"');
+            banners.forEach(el => { el.remove(); });
+          });
+        </script>

rulesets/us/americanbanker-com.yaml

@@ -0,0 +1,16 @@
+- domain: americanbanker.com
+  paths:
+    - /news
+  injections:
+    - position: head
+      append: |
+        <script>
+          document.addEventListener("DOMContentLoaded", () => {
+            const inlineGate = document.querySelector('.inline-gate');
+            if (inlineGate) {
+              inlineGate.classList.remove('inline-gate');
+              const inlineGated = document.querySelectorAll('.inline-gated');
+              for (const elem of inlineGated) { elem.classList.remove('inline-gated'); }
+            }
+          });
+        </script>

rulesets/us/medium-com.yaml

@@ -0,0 +1,7 @@
+- domain: medium.com
+  headers:
+    referer: https://t.co/x?amp=1
+    x-forwarded-for: none
+    user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36
+    content-security-policy: script-src 'self';
+    cookie: 

rulesets/us/nytimes-com.yaml

@@ -0,0 +1,17 @@
+- domains: 
+  - www.nytimes.com
+  - www.time.com
+  headers:
+    ueser-agent: Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
+    cookie: nyt-a=; nyt-gdpr=0; nyt-geo=DE; nyt-privacy=1
+    referer: https://www.google.com/ 
+  injections:
+    - position: head
+      append: |
+        <script>
+          window.localStorage.clear();
+          document.addEventListener("DOMContentLoaded", () => {
+            const banners = document.querySelectorAll('div[data-testid="inline-message"], div[id^="ad-"], div[id^="leaderboard-"], div.expanded-dock, div.pz-ad-box, div[id="top-wrapper"], div[id="bottom-wrapper"]');
+            banners.forEach(el => { el.remove(); });
+          });
+        </script>

rulesets/us/usatoday-com.yaml

@@ -0,0 +1,10 @@
+- domain: www.usatoday.com
+  injections:
+    - position: head
+      append: |
+        <script>
+          document.addEventListener("DOMContentLoaded", () => {
+            const banners = document.querySelectorAll('div.roadblock-container, .gnt_nb, [aria-label="advertisement"], div[id="main-frame-error"]');
+            banners.forEach(el => { el.remove(); });
+          });
+        </script>

rulesets/us/washingtonpost-com.yaml

@@ -0,0 +1,14 @@
+- domain: www.washingtonpost.com
+  injections:
+    - position: head
+      append: |
+        <script>
+          document.addEventListener("DOMContentLoaded", () => {
+            let paywall = document.querySelectorAll('div[data-qa$="-ad"], div[id="leaderboard-wrapper"], div[data-qa="subscribe-promo"]');
+            paywall.forEach(el => { el.remove(); });
+            const images = document.querySelectorAll('img');
+            images.forEach(image => { image.parentElement.style.filter = ''; });
+            const headimage = document.querySelectorAll('div .aspect-custom');
+            headimage.forEach(image => { image.style.filter = ''; });
+          });
+        </script>