ruleset_v2 integration test working

cmd/main.go

@@ -9,7 +9,6 @@ import (
 
 	"ladder/handlers"
 	"ladder/internal/cli"
-	"ladder/proxychain/requestmodifiers/bot"
 
 	"github.com/akamensky/argparse"
 	"github.com/gofiber/fiber/v2"
@@ -43,6 +42,7 @@ func main() {
 		Help:     "Adds verbose logging",
 	})
 
+	/*
 		randomGoogleBot := parser.Flag("", "random-googlebot", &argparse.Options{
 			Required: false,
 			Help:     "Update the list of trusted Googlebot IPs, and use a random one for each masqueraded request",
@@ -52,6 +52,7 @@ func main() {
 			Required: false,
 			Help:     "Update the list of trusted Bingbot IPs, and use a random one for each masqueraded request",
 		})
+	*/
 
 	// TODO: add version flag that reads from handers/VERSION
 
@@ -65,14 +66,9 @@ func main() {
 		Help:     "Compiles a directory of yaml files into a single ruleset.yaml. Requires --ruleset arg.",
 	})
 
-	mergeRulesetsGzip := parser.Flag("", "merge-rulesets-gzip", &argparse.Options{
-		Required: false,
-		Help:     "Compiles a directory of yaml files into a single ruleset.gz Requires --ruleset arg.",
-	})
-
 	mergeRulesetsOutput := parser.String("", "merge-rulesets-output", &argparse.Options{
 		Required: false,
-		Help:     "Specify output file for --merge-rulesets and --merge-rulesets-gzip. Requires --ruleset and --merge-rulesets args.",
+		Help:     "Specify output file for --merge-rulesets. Requires --ruleset and --merge-rulesets args.",
 	})
 
 	err := parser.Parse(os.Args)
@@ -80,6 +76,7 @@ func main() {
 		fmt.Print(parser.Usage(err))
 	}
 
+	/*
 		if *randomGoogleBot {
 			err := bot.GoogleBot.UpdatePool("https://developers.google.com/static/search/apis/ipranges/googlebot.json")
 			if err != nil {
@@ -95,9 +92,10 @@ func main() {
 				fmt.Println("defaulting to known trusted Bingbot identity")
 			}
 		}
+	*/
 
 	// utility cli flag to compile ruleset directory into single ruleset.yaml
-	if *mergeRulesets || *mergeRulesetsGzip {
+	if *mergeRulesets {
 		output := os.Stdout
 
 		if *mergeRulesetsOutput != "" {
@@ -109,7 +107,7 @@ func main() {
 			}
 		}
 
-		err = cli.HandleRulesetMerge(*ruleset, *mergeRulesets, *mergeRulesetsGzip, output)
+		err = cli.HandleRulesetMerge(*ruleset, *mergeRulesets, output)
 		if err != nil {
 			fmt.Println(err)
 			os.Exit(1)

handlers/proxy.go

@@ -1,9 +1,11 @@
 package handlers
 
 import (
+	"fmt"
 	"ladder/proxychain"
 	rx "ladder/proxychain/requestmodifiers"
 	tx "ladder/proxychain/responsemodifiers"
+	"ladder/proxychain/ruleset"
 
 	"github.com/gofiber/fiber/v2"
 )
@@ -24,6 +26,10 @@ func NewProxySiteHandler(opts *ProxyOptions) fiber.Handler {
 			rs = r
 		}
 	*/
+	rs, err := ruleset_v2.NewRuleset("ruleset_v2.yaml")
+	if err != nil {
+		panic(err)
+	}
 
 	return func(c *fiber.Ctx) error {
 		proxychain := proxychain.
@@ -51,9 +57,20 @@ func NewProxySiteHandler(opts *ProxyOptions) fiber.Handler {
 				tx.PatchDynamicResourceURLs(),
 				tx.BlockElementRemoval(".article-content"),
 			// tx.SetContentSecurityPolicy("default-src * 'unsafe-inline' 'unsafe-eval' data: blob:;"),
-			).
+			)
-			Execute()
 
-		return proxychain
+		// load ruleset
+		rule, exists := rs.GetRule(proxychain.Request.URL)
+		fmt.Println("============")
+		fmt.Println(proxychain.Request.URL)
+		fmt.Println(rs)
+		fmt.Println("============")
+		if exists {
+			fmt.Println("===========EXISTS=")
+			proxychain.AddOnceRequestModifications(rule.RequestModifications...)
+			proxychain.AddOnceResponseModifications(rule.ResponseModifications...)
+		}
+
+		return proxychain.Execute()
 	}
 }

internal/cli/ruleset_merge.go

@@ -5,9 +5,7 @@ import (
 	"io"
 	"os"
 
-	"ladder/pkg/ruleset"
+	"ladder/proxychain/ruleset"
-
-	"golang.org/x/term"
 )
 
 // HandleRulesetMerge merges a set of ruleset files, specified by the rulesetPath or RULESET env variable, into either YAML or Gzip format.
@@ -21,7 +19,7 @@ import (
 //
 // Returns:
 // - An error if the ruleset loading or merging process fails, otherwise nil.
-func HandleRulesetMerge(rulesetPath string, mergeRulesets bool, useGzip bool, output *os.File) error {
+func HandleRulesetMerge(rulesetPath string, mergeRulesets bool, output *os.File) error {
 	if !mergeRulesets {
 		return nil
 	}
@@ -35,55 +33,15 @@ func HandleRulesetMerge(rulesetPath string, mergeRulesets bool, useGzip bool, ou
 		os.Exit(1)
 	}
 
-	rs, err := ruleset.NewRuleset(rulesetPath)
+	rs, err := ruleset_v2.NewRuleset(rulesetPath)
 	if err != nil {
 		fmt.Println(err)
 		os.Exit(1)
 	}
 
-	if useGzip {
-		return gzipMerge(rs, output)
-	}
-
 	return yamlMerge(rs, output)
 }
 
-// gzipMerge takes a RuleSet and an optional output file path pointer. It compresses the RuleSet into Gzip format.
-// If the output file path is provided, the compressed data is written to this file. Otherwise, it prints a warning
-// and outputs the binary data to stdout
-//
-// Parameters:
-// - rs: The ruleset.RuleSet to be compressed.
-// - output: The output for the gzip data. If nil, stdout will be used.
-//
-// Returns:
-// - An error if compression or file writing fails, otherwise nil.
-func gzipMerge(rs ruleset.RuleSet, output io.Writer) error {
-	gzip, err := rs.GzipYaml()
-	if err != nil {
-		return err
-	}
-
-	if output != nil {
-		_, err = io.Copy(output, gzip)
-		if err != nil {
-			return err
-		}
-	}
-
-	if term.IsTerminal(int(os.Stdout.Fd())) {
-		println("warning: binary output can mess up your terminal. Use '--merge-rulesets-output <ruleset.gz>' or pipe it to a file.")
-		os.Exit(1)
-	}
-
-	_, err = io.Copy(os.Stdout, gzip)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
 // yamlMerge takes a RuleSet and an optional output file path pointer. It converts the RuleSet into YAML format.
 // If the output file path is provided, the YAML data is written to this file. If not, the YAML data is printed to stdout.
 //
@@ -93,7 +51,7 @@ func gzipMerge(rs ruleset.RuleSet, output io.Writer) error {
 //
 // Returns:
 // - An error if YAML conversion or file writing fails, otherwise nil.
-func yamlMerge(rs ruleset.RuleSet, output io.Writer) error {
+func yamlMerge(rs ruleset_v2.Ruleset, output io.Writer) error {
 	yaml, err := rs.Yaml()
 	if err != nil {
 		return err

proxychain/proxychain.go

@@ -3,23 +3,13 @@ package proxychain
 import (
 	"errors"
 	"fmt"
+	http "github.com/bogdanfinn/fhttp"
+	tls_client "github.com/bogdanfinn/tls-client"
 	"io"
 	"log"
 	"net/url"
 	"strings"
 
-	//"time"
-
-	//"net/http"
-	//"github.com/Danny-Dasilva/CycleTLS/cycletls"
-	//http "github.com/Danny-Dasilva/fhttp"
-	http "github.com/bogdanfinn/fhttp"
-	tls_client "github.com/bogdanfinn/tls-client"
-
-	//"github.com/bogdanfinn/tls-client/profiles"
-
-	"ladder/pkg/ruleset"
-
 	"github.com/gofiber/fiber/v2"
 )
 
@@ -100,7 +90,6 @@ type ProxyChain struct {
 	onceRequestModifications  []RequestModification
 	onceResponseModifications []ResponseModification
 	responseModifications     []ResponseModification
-	Ruleset                   *ruleset.RuleSet
 	debugMode                 bool
 	abortErr                  error
 	APIPrefix                 string
@@ -177,13 +166,6 @@ func (chain *ProxyChain) WithAPIPath(path string) *ProxyChain {
 	return chain
 }
 
-// Adds a ruleset to ProxyChain
-func (chain *ProxyChain) AddRuleset(rs *ruleset.RuleSet) *ProxyChain {
-	chain.Ruleset = rs
-	// TODO: add _applyRuleset method
-	return chain
-}
-
 func (chain *ProxyChain) _initializeRequest() (*http.Request, error) {
 	if chain.Context == nil {
 		chain.abortErr = chain.abort(errors.New("no context set"))

proxychain/ruleset/rule_reqmod_types.gen.go

@@ -1,5 +1,5 @@
-package ruleset_v2
 
+package ruleset_v2
 // DO NOT EDIT THIS FILE. It is automatically generated by ladder/proxychain/codegen/codegen.go
 // The purpose of this is serialization of rulesets from JSON or YAML into functional options suitable
 // for use in proxychains.
@@ -179,4 +179,5 @@ func init() {
   rqmModMap["SpoofXForwardedFor"] = func(params ...string) proxychain.RequestModification {
     return rx.SpoofXForwardedFor(params[0])
   }
+
 }

proxychain/ruleset/rule_resmod_types.gen.go

@@ -1,5 +1,5 @@
-package ruleset_v2
 
+package ruleset_v2
 // DO NOT EDIT THIS FILE. It is automatically generated by ladder/proxychain/codegen/codegen.go
 // The purpose of this is serialization of rulesets from JSON or YAML into functional options suitable
 // for use in proxychains.
@@ -95,4 +95,5 @@ func init() {
   rsmModMap["RewriteHTMLResourceURLs"] = func(_ ...string) proxychain.ResponseModification {
     return tx.RewriteHTMLResourceURLs()
   }
+
 }

proxychain/ruleset/ruleset.go

@@ -163,6 +163,18 @@ func (rs *Ruleset) loadRulesFromLocalDir(path string) error {
 		return nil
 	})
 
+	// create a map of pointers to rules loaded above based on domain string keys
+	// this way we don't have two copies of the rule in ruleset
+	for i, rule := range rs.Rules {
+		rulePtr := &rs.Rules[i]
+		for _, domain := range rule.Domains {
+			rs._rulemap[domain] = rulePtr
+			if !strings.HasPrefix(domain, "www.") {
+				rs._rulemap["www."+domain] = rulePtr
+			}
+		}
+	}
+
 	if err != nil {
 		return err
 	}

ruleset_v2.yaml

@@ -0,0 +1,17 @@
+rules:
+  - domains:
+    - example.com
+    - www.example.com
+    responsemodifications:
+    - name: APIContent
+      params: []
+    - name: SetContentSecurityPolicy
+      params:
+      - foobar
+    - name: SetIncomingCookie
+      params:
+      - authorization-bearer
+      - hunter2
+    requestmodifications:
+    - name: ForwardRequestHeaders
+      params: []