diff --git a/cmd/main.go b/cmd/main.go index a7370dc..e9ead89 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -8,6 +8,7 @@ import ( "strings" "ladder/handlers" + "ladder/handlers/cli" "github.com/akamensky/argparse" "github.com/gofiber/fiber/v2" @@ -17,6 +18,7 @@ import ( //go:embed favicon.ico var faviconData string + //go:embed styles.css var cssData embed.FS @@ -40,7 +42,20 @@ func main() { ruleset := parser.String("r", "ruleset", &argparse.Options{ Required: false, - Help: "File, Directory or URL to a ruleset.yml. Overrides RULESET environment variable.", + Help: "File, Directory or URL to a ruleset.yaml. Overrides RULESET environment variable.", + }) + + mergeRulesets := parser.Flag("", "merge-rulesets", &argparse.Options{ + Required: false, + Help: "Compiles a directory of yaml files into a single ruleset.yaml. Requires --ruleset arg.", + }) + mergeRulesetsGzip := parser.Flag("", "merge-rulesets-gzip", &argparse.Options{ + Required: false, + Help: "Compiles a directory of yaml files into a single ruleset.gz Requires --ruleset arg.", + }) + mergeRulesetsOutput := parser.String("", "merge-rulesets-output", &argparse.Options{ + Required: false, + Help: "Specify output file for --merge-rulesets and --merge-rulesets-gzip. Requires --ruleset and --merge-rulesets args.", }) err := parser.Parse(os.Args) @@ -48,6 +63,16 @@ func main() { fmt.Print(parser.Usage(err)) } + // utility cli flag to compile ruleset directory into single ruleset.yaml + if *mergeRulesets || *mergeRulesetsGzip { + err = cli.HandleRulesetMerge(ruleset, mergeRulesets, mergeRulesetsGzip, mergeRulesetsOutput) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + os.Exit(0) + } + if os.Getenv("PREFORK") == "true" { *prefork = true } diff --git a/go.mod b/go.mod index e35c2b6..5d6ca5d 100644 --- a/go.mod +++ b/go.mod @@ -26,4 +26,5 @@ require ( github.com/valyala/tcplisten v1.0.0 // indirect golang.org/x/net v0.18.0 // indirect golang.org/x/sys v0.14.0 // indirect + golang.org/x/term v0.14.0 ) diff --git a/go.sum b/go.sum index 16347de..f171d69 100644 --- a/go.sum +++ b/go.sum @@ -68,6 +68,8 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8= +golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/handlers/cli/cli.go b/handlers/cli/cli.go new file mode 100644 index 0000000..ab355d7 --- /dev/null +++ b/handlers/cli/cli.go @@ -0,0 +1,105 @@ +package cli + +import ( + "fmt" + "io" + "io/fs" + "ladder/pkg/ruleset" + "os" + + "golang.org/x/term" +) + +// HandleRulesetMerge merges a set of ruleset files, specified by the rulesetPath or RULESET env variable, into either YAML or Gzip format. +// Exits the program with an error message if the ruleset path is not provided or if loading the ruleset fails. +// +// Parameters: +// - rulesetPath: A pointer to a string specifying the path to the ruleset file. +// - mergeRulesets: A pointer to a boolean indicating if a merge operation should be performed. +// - mergeRulesetsGzip: A pointer to a boolean indicating if the merge should be in Gzip format. +// - mergeRulesetsOutput: A pointer to a string specifying the output file path. If empty, the output is printed to stdout. +// +// Returns: +// - An error if the ruleset loading or merging process fails, otherwise nil. +func HandleRulesetMerge(rulesetPath *string, mergeRulesets *bool, mergeRulesetsGzip *bool, mergeRulesetsOutput *string) error { + if *rulesetPath == "" { + *rulesetPath = os.Getenv("RULESET") + } + if *rulesetPath == "" { + fmt.Println("ERROR: no ruleset provided. Try again with --ruleset ") + os.Exit(1) + } + + rs, err := ruleset.NewRuleset(*rulesetPath) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + if *mergeRulesetsGzip { + return gzipMerge(rs, mergeRulesetsOutput) + } + return yamlMerge(rs, mergeRulesetsOutput) +} + +// gzipMerge takes a RuleSet and an optional output file path pointer. It compresses the RuleSet into Gzip format. +// If the output file path is provided, the compressed data is written to this file. Otherwise, it prints a warning +// and outputs the binary data to stdout +// +// Parameters: +// - rs: The ruleset.RuleSet to be compressed. +// - mergeRulesetsOutput: A pointer to a string specifying the output file path. If empty, the output is directed to stdout. +// +// Returns: +// - An error if compression or file writing fails, otherwise nil. +func gzipMerge(rs ruleset.RuleSet, mergeRulesetsOutput *string) error { + gzip, err := rs.GzipYaml() + if err != nil { + return err + } + + if *mergeRulesetsOutput != "" { + out, err := os.Create(*mergeRulesetsOutput) + defer out.Close() + _, err = io.Copy(out, gzip) + if err != nil { + return err + } + } + + if term.IsTerminal(int(os.Stdout.Fd())) { + println("WARNING: binary output can mess up your terminal. Use '--merge-rulesets-output ' or pipe it to a file.") + os.Exit(1) + } + _, err = io.Copy(os.Stdout, gzip) + if err != nil { + return err + } + return nil +} + +// yamlMerge takes a RuleSet and an optional output file path pointer. It converts the RuleSet into YAML format. +// If the output file path is provided, the YAML data is written to this file. If not, the YAML data is printed to stdout. +// +// Parameters: +// - rs: The ruleset.RuleSet to be converted to YAML. +// - mergeRulesetsOutput: A pointer to a string specifying the output file path. If empty, the output is printed to stdout. +// +// Returns: +// - An error if YAML conversion or file writing fails, otherwise nil. +func yamlMerge(rs ruleset.RuleSet, mergeRulesetsOutput *string) error { + yaml, err := rs.Yaml() + if err != nil { + return err + } + if *mergeRulesetsOutput == "" { + fmt.Printf(yaml) + os.Exit(0) + } + + err = os.WriteFile(*mergeRulesetsOutput, []byte(yaml), fs.FileMode(os.O_RDWR)) + if err != nil { + return fmt.Errorf("ERROR: failed to write merged YAML ruleset to '%s'\n", *mergeRulesetsOutput) + } + return nil +} diff --git a/pkg/ruleset/ruleset.go b/pkg/ruleset/ruleset.go index 9029a65..06c1389 100644 --- a/pkg/ruleset/ruleset.go +++ b/pkg/ruleset/ruleset.go @@ -39,20 +39,20 @@ type Rule struct { CSP string `yaml:"content-security-policy,omitempty"` } `yaml:"headers,omitempty"` GoogleCache bool `yaml:"googleCache,omitempty"` - RegexRules []Regex `yaml:"regexRules"` + RegexRules []Regex `yaml:"regexRules,omitempty"` UrlMods struct { - Domain []Regex `yaml:"domain"` - Path []Regex `yaml:"path"` - Query []KV `yaml:"query"` - } `yaml:"urlMods"` + Domain []Regex `yaml:"domain,omitempty"` + Path []Regex `yaml:"path,omitempty"` + Query []KV `yaml:"queryomitempty"` + } `yaml:"urlMods,omitempty"` Injections []struct { - Position string `yaml:"position"` - Append string `yaml:"append"` - Prepend string `yaml:"prepend"` - Replace string `yaml:"replace"` - } `yaml:"injections"` + Position string `yaml:"position,omitempty"` + Append string `yaml:"append,omitempty"` + Prepend string `yaml:"prepend,omitempty"` + Replace string `yaml:"replace,omitempty"` + } `yaml:"injections,omitempty"` } // NewRulesetFromEnv creates a new RuleSet based on the RULESET environment variable. diff --git a/ruleset.yaml b/ruleset.yaml deleted file mode 100644 index 572b980..0000000 --- a/ruleset.yaml +++ /dev/null @@ -1,194 +0,0 @@ -- domain: example.com - domains: - - www.beispiel.de - googleCache: true - headers: - x-forwarded-for: none - referer: none - user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 - cookie: privacy=1 - regexRules: - - match: ]*\s+)?src="(/)([^"]*)" - replace: - - position: h1 - replace: | -

An example with a ladder ;-)

-- domain: www.americanbanker.com - paths: - - /news - injections: - - position: head - append: | - -- domain: www.nzz.ch - paths: - - /international - - /sport - - /wirtschaft - - /technologie - - /feuilleton - - /zuerich - - /wissenschaft - - /gesellschaft - - /panorama - - /mobilitaet - - /reisen - - /meinung - - /finanze - injections: - - position: head - append: | - -- domains: - - www.architecturaldigest.com - - www.bonappetit.com - - www.cntraveler.com - - www.epicurious.com - - www.gq.com - - www.newyorker.com - - www.vanityfair.com - - www.vogue.com - - www.wired.com - injections: - - position: head - append: | - -- domains: - - www.nytimes.com - - www.time.com - headers: - ueser-agent: Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html) - cookie: nyt-a=; nyt-gdpr=0; nyt-geo=DE; nyt-privacy=1 - referer: https://www.google.com/ - injections: - - position: head - append: | - -- domains: - - www.thestar.com - - www.niagarafallsreview.ca - - www.stcatharinesstandard.ca - - www.thepeterboroughexaminer.com - - www.therecord.com - - www.thespec.com - - www.wellandtribune.ca - injections: - - position: head - append: | - -- domain: www.usatoday.com - injections: - - position: head - append: | - -- domain: www.washingtonpost.com - injections: - - position: head - append: | - -- domain: medium.com - headers: - referer: https://t.co/x?amp=1 - x-forwarded-for: none - user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 - content-security-policy: script-src 'self'; - cookie: -- domain: tagesspiegel.de - headers: - content-security-policy: script-src 'self'; - user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 - urlMods: - query: - - key: amp - value: 1 -- domain: www.ft.com - headers: - referer: https://t.co/x?amp=1 - injections: - - position: head - append: | - - diff --git a/rulesets/ca/_multi-metroland-media-group.yaml b/rulesets/ca/_multi-metroland-media-group.yaml new file mode 100644 index 0000000..9ec8f94 --- /dev/null +++ b/rulesets/ca/_multi-metroland-media-group.yaml @@ -0,0 +1,35 @@ +- domains: + - www.thestar.com + - www.niagarafallsreview.ca + - www.stcatharinesstandard.ca + - www.thepeterboroughexaminer.com + - www.therecord.com + - www.thespec.com + - www.wellandtribune.ca + injections: + - position: head + append: | + diff --git a/rulesets/ch/nzz-ch.yaml b/rulesets/ch/nzz-ch.yaml new file mode 100644 index 0000000..c949925 --- /dev/null +++ b/rulesets/ch/nzz-ch.yaml @@ -0,0 +1,24 @@ +- domain: www.nzz.ch + paths: + - /international + - /sport + - /wirtschaft + - /technologie + - /feuilleton + - /zuerich + - /wissenschaft + - /gesellschaft + - /panorama + - /mobilitaet + - /reisen + - /meinung + - /finanze + injections: + - position: head + append: | + diff --git a/rulesets/de/tagesspiegel-de.yaml b/rulesets/de/tagesspiegel-de.yaml new file mode 100644 index 0000000..cadbe5b --- /dev/null +++ b/rulesets/de/tagesspiegel-de.yaml @@ -0,0 +1,9 @@ +# loads amp version of page +- domain: tagesspiegel.de + headers: + content-security-policy: script-src 'self'; + user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 + urlMods: + query: + - key: amp + value: 1 diff --git a/rulesets/gb/ft-com.yaml b/rulesets/gb/ft-com.yaml new file mode 100644 index 0000000..0011c1d --- /dev/null +++ b/rulesets/gb/ft-com.yaml @@ -0,0 +1,20 @@ +- domain: www.ft.com + headers: + referer: https://t.co/x?amp=1 + injections: + - position: head + append: | + diff --git a/rulesets/us/_multi-conde-nast.yaml b/rulesets/us/_multi-conde-nast.yaml new file mode 100644 index 0000000..ac08d09 --- /dev/null +++ b/rulesets/us/_multi-conde-nast.yaml @@ -0,0 +1,19 @@ +- domains: + - www.architecturaldigest.com + - www.bonappetit.com + - www.cntraveler.com + - www.epicurious.com + - www.gq.com + - www.newyorker.com + - www.vanityfair.com + - www.vogue.com + - www.wired.com + injections: + - position: head + append: | + diff --git a/rulesets/us/americanbanker-com.yaml b/rulesets/us/americanbanker-com.yaml new file mode 100644 index 0000000..d9d199e --- /dev/null +++ b/rulesets/us/americanbanker-com.yaml @@ -0,0 +1,16 @@ +- domain: americanbanker.com + paths: + - /news + injections: + - position: head + append: | + diff --git a/rulesets/us/medium-com.yaml b/rulesets/us/medium-com.yaml new file mode 100644 index 0000000..b9b72fc --- /dev/null +++ b/rulesets/us/medium-com.yaml @@ -0,0 +1,7 @@ +- domain: medium.com + headers: + referer: https://t.co/x?amp=1 + x-forwarded-for: none + user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 + content-security-policy: script-src 'self'; + cookie: diff --git a/rulesets/us/nytimes-com.yaml b/rulesets/us/nytimes-com.yaml new file mode 100644 index 0000000..1408549 --- /dev/null +++ b/rulesets/us/nytimes-com.yaml @@ -0,0 +1,17 @@ +- domains: + - www.nytimes.com + - www.time.com + headers: + ueser-agent: Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html) + cookie: nyt-a=; nyt-gdpr=0; nyt-geo=DE; nyt-privacy=1 + referer: https://www.google.com/ + injections: + - position: head + append: | + diff --git a/rulesets/us/usatoday-com.yaml b/rulesets/us/usatoday-com.yaml new file mode 100644 index 0000000..22eccdc --- /dev/null +++ b/rulesets/us/usatoday-com.yaml @@ -0,0 +1,10 @@ +- domain: www.usatoday.com + injections: + - position: head + append: | + diff --git a/rulesets/us/washingtonpost-com.yaml b/rulesets/us/washingtonpost-com.yaml new file mode 100644 index 0000000..0d3d5ed --- /dev/null +++ b/rulesets/us/washingtonpost-com.yaml @@ -0,0 +1,14 @@ +- domain: www.washingtonpost.com + injections: + - position: head + append: | +