From 52d12dd1ac7721ce00bc30edab07b488b0b64365 Mon Sep 17 00:00:00 2001 From: Kevin Pham Date: Tue, 5 Dec 2023 15:02:18 -0600 Subject: [PATCH] ruleset_v2 integration test working --- cmd/main.go | 56 +++-- handlers/proxy.go | 23 +- internal/cli/ruleset_merge.go | 50 +--- proxychain/proxychain.go | 22 +- proxychain/ruleset/rule_reqmod_types.gen.go | 251 ++++++++++---------- proxychain/ruleset/rule_resmod_types.gen.go | 125 +++++----- proxychain/ruleset/ruleset.go | 12 + ruleset_v2.yaml | 17 ++ 8 files changed, 271 insertions(+), 285 deletions(-) create mode 100644 ruleset_v2.yaml diff --git a/cmd/main.go b/cmd/main.go index 4af5534..32792e5 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -9,7 +9,6 @@ import ( "ladder/handlers" "ladder/internal/cli" - "ladder/proxychain/requestmodifiers/bot" "github.com/akamensky/argparse" "github.com/gofiber/fiber/v2" @@ -43,15 +42,17 @@ func main() { Help: "Adds verbose logging", }) - randomGoogleBot := parser.Flag("", "random-googlebot", &argparse.Options{ - Required: false, - Help: "Update the list of trusted Googlebot IPs, and use a random one for each masqueraded request", - }) + /* + randomGoogleBot := parser.Flag("", "random-googlebot", &argparse.Options{ + Required: false, + Help: "Update the list of trusted Googlebot IPs, and use a random one for each masqueraded request", + }) - randomBingBot := parser.Flag("", "random-bingbot", &argparse.Options{ - Required: false, - Help: "Update the list of trusted Bingbot IPs, and use a random one for each masqueraded request", - }) + randomBingBot := parser.Flag("", "random-bingbot", &argparse.Options{ + Required: false, + Help: "Update the list of trusted Bingbot IPs, and use a random one for each masqueraded request", + }) + */ // TODO: add version flag that reads from handers/VERSION @@ -65,14 +66,9 @@ func main() { Help: "Compiles a directory of yaml files into a single ruleset.yaml. Requires --ruleset arg.", }) - mergeRulesetsGzip := parser.Flag("", "merge-rulesets-gzip", &argparse.Options{ - Required: false, - Help: "Compiles a directory of yaml files into a single ruleset.gz Requires --ruleset arg.", - }) - mergeRulesetsOutput := parser.String("", "merge-rulesets-output", &argparse.Options{ Required: false, - Help: "Specify output file for --merge-rulesets and --merge-rulesets-gzip. Requires --ruleset and --merge-rulesets args.", + Help: "Specify output file for --merge-rulesets. Requires --ruleset and --merge-rulesets args.", }) err := parser.Parse(os.Args) @@ -80,24 +76,26 @@ func main() { fmt.Print(parser.Usage(err)) } - if *randomGoogleBot { - err := bot.GoogleBot.UpdatePool("https://developers.google.com/static/search/apis/ipranges/googlebot.json") - if err != nil { - fmt.Println("error while retrieving list of Googlebot IPs: " + err.Error()) - fmt.Println("defaulting to known trusted Googlebot identity") + /* + if *randomGoogleBot { + err := bot.GoogleBot.UpdatePool("https://developers.google.com/static/search/apis/ipranges/googlebot.json") + if err != nil { + fmt.Println("error while retrieving list of Googlebot IPs: " + err.Error()) + fmt.Println("defaulting to known trusted Googlebot identity") + } } - } - if *randomBingBot { - err := bot.BingBot.UpdatePool("https://www.bing.com/toolbox/bingbot.json") - if err != nil { - fmt.Println("error while retrieving list of Bingbot IPs: " + err.Error()) - fmt.Println("defaulting to known trusted Bingbot identity") + if *randomBingBot { + err := bot.BingBot.UpdatePool("https://www.bing.com/toolbox/bingbot.json") + if err != nil { + fmt.Println("error while retrieving list of Bingbot IPs: " + err.Error()) + fmt.Println("defaulting to known trusted Bingbot identity") + } } - } + */ // utility cli flag to compile ruleset directory into single ruleset.yaml - if *mergeRulesets || *mergeRulesetsGzip { + if *mergeRulesets { output := os.Stdout if *mergeRulesetsOutput != "" { @@ -109,7 +107,7 @@ func main() { } } - err = cli.HandleRulesetMerge(*ruleset, *mergeRulesets, *mergeRulesetsGzip, output) + err = cli.HandleRulesetMerge(*ruleset, *mergeRulesets, output) if err != nil { fmt.Println(err) os.Exit(1) diff --git a/handlers/proxy.go b/handlers/proxy.go index d22678f..5377a89 100644 --- a/handlers/proxy.go +++ b/handlers/proxy.go @@ -1,9 +1,11 @@ package handlers import ( + "fmt" "ladder/proxychain" rx "ladder/proxychain/requestmodifiers" tx "ladder/proxychain/responsemodifiers" + "ladder/proxychain/ruleset" "github.com/gofiber/fiber/v2" ) @@ -24,6 +26,10 @@ func NewProxySiteHandler(opts *ProxyOptions) fiber.Handler { rs = r } */ + rs, err := ruleset_v2.NewRuleset("ruleset_v2.yaml") + if err != nil { + panic(err) + } return func(c *fiber.Ctx) error { proxychain := proxychain. @@ -51,9 +57,20 @@ func NewProxySiteHandler(opts *ProxyOptions) fiber.Handler { tx.PatchDynamicResourceURLs(), tx.BlockElementRemoval(".article-content"), // tx.SetContentSecurityPolicy("default-src * 'unsafe-inline' 'unsafe-eval' data: blob:;"), - ). - Execute() + ) - return proxychain + // load ruleset + rule, exists := rs.GetRule(proxychain.Request.URL) + fmt.Println("============") + fmt.Println(proxychain.Request.URL) + fmt.Println(rs) + fmt.Println("============") + if exists { + fmt.Println("===========EXISTS=") + proxychain.AddOnceRequestModifications(rule.RequestModifications...) + proxychain.AddOnceResponseModifications(rule.ResponseModifications...) + } + + return proxychain.Execute() } } diff --git a/internal/cli/ruleset_merge.go b/internal/cli/ruleset_merge.go index ca7b7e0..98524b4 100644 --- a/internal/cli/ruleset_merge.go +++ b/internal/cli/ruleset_merge.go @@ -5,9 +5,7 @@ import ( "io" "os" - "ladder/pkg/ruleset" - - "golang.org/x/term" + "ladder/proxychain/ruleset" ) // HandleRulesetMerge merges a set of ruleset files, specified by the rulesetPath or RULESET env variable, into either YAML or Gzip format. @@ -21,7 +19,7 @@ import ( // // Returns: // - An error if the ruleset loading or merging process fails, otherwise nil. -func HandleRulesetMerge(rulesetPath string, mergeRulesets bool, useGzip bool, output *os.File) error { +func HandleRulesetMerge(rulesetPath string, mergeRulesets bool, output *os.File) error { if !mergeRulesets { return nil } @@ -35,55 +33,15 @@ func HandleRulesetMerge(rulesetPath string, mergeRulesets bool, useGzip bool, ou os.Exit(1) } - rs, err := ruleset.NewRuleset(rulesetPath) + rs, err := ruleset_v2.NewRuleset(rulesetPath) if err != nil { fmt.Println(err) os.Exit(1) } - if useGzip { - return gzipMerge(rs, output) - } - return yamlMerge(rs, output) } -// gzipMerge takes a RuleSet and an optional output file path pointer. It compresses the RuleSet into Gzip format. -// If the output file path is provided, the compressed data is written to this file. Otherwise, it prints a warning -// and outputs the binary data to stdout -// -// Parameters: -// - rs: The ruleset.RuleSet to be compressed. -// - output: The output for the gzip data. If nil, stdout will be used. -// -// Returns: -// - An error if compression or file writing fails, otherwise nil. -func gzipMerge(rs ruleset.RuleSet, output io.Writer) error { - gzip, err := rs.GzipYaml() - if err != nil { - return err - } - - if output != nil { - _, err = io.Copy(output, gzip) - if err != nil { - return err - } - } - - if term.IsTerminal(int(os.Stdout.Fd())) { - println("warning: binary output can mess up your terminal. Use '--merge-rulesets-output ' or pipe it to a file.") - os.Exit(1) - } - - _, err = io.Copy(os.Stdout, gzip) - if err != nil { - return err - } - - return nil -} - // yamlMerge takes a RuleSet and an optional output file path pointer. It converts the RuleSet into YAML format. // If the output file path is provided, the YAML data is written to this file. If not, the YAML data is printed to stdout. // @@ -93,7 +51,7 @@ func gzipMerge(rs ruleset.RuleSet, output io.Writer) error { // // Returns: // - An error if YAML conversion or file writing fails, otherwise nil. -func yamlMerge(rs ruleset.RuleSet, output io.Writer) error { +func yamlMerge(rs ruleset_v2.Ruleset, output io.Writer) error { yaml, err := rs.Yaml() if err != nil { return err diff --git a/proxychain/proxychain.go b/proxychain/proxychain.go index bc5a847..7bd4b90 100644 --- a/proxychain/proxychain.go +++ b/proxychain/proxychain.go @@ -3,23 +3,13 @@ package proxychain import ( "errors" "fmt" + http "github.com/bogdanfinn/fhttp" + tls_client "github.com/bogdanfinn/tls-client" "io" "log" "net/url" "strings" - //"time" - - //"net/http" - //"github.com/Danny-Dasilva/CycleTLS/cycletls" - //http "github.com/Danny-Dasilva/fhttp" - http "github.com/bogdanfinn/fhttp" - tls_client "github.com/bogdanfinn/tls-client" - - //"github.com/bogdanfinn/tls-client/profiles" - - "ladder/pkg/ruleset" - "github.com/gofiber/fiber/v2" ) @@ -100,7 +90,6 @@ type ProxyChain struct { onceRequestModifications []RequestModification onceResponseModifications []ResponseModification responseModifications []ResponseModification - Ruleset *ruleset.RuleSet debugMode bool abortErr error APIPrefix string @@ -177,13 +166,6 @@ func (chain *ProxyChain) WithAPIPath(path string) *ProxyChain { return chain } -// Adds a ruleset to ProxyChain -func (chain *ProxyChain) AddRuleset(rs *ruleset.RuleSet) *ProxyChain { - chain.Ruleset = rs - // TODO: add _applyRuleset method - return chain -} - func (chain *ProxyChain) _initializeRequest() (*http.Request, error) { if chain.Context == nil { chain.abortErr = chain.abort(errors.New("no context set")) diff --git a/proxychain/ruleset/rule_reqmod_types.gen.go b/proxychain/ruleset/rule_reqmod_types.gen.go index f055d28..7f22c3e 100644 --- a/proxychain/ruleset/rule_reqmod_types.gen.go +++ b/proxychain/ruleset/rule_reqmod_types.gen.go @@ -1,5 +1,5 @@ -package ruleset_v2 +package ruleset_v2 // DO NOT EDIT THIS FILE. It is automatically generated by ladder/proxychain/codegen/codegen.go // The purpose of this is serialization of rulesets from JSON or YAML into functional options suitable // for use in proxychains. @@ -16,167 +16,168 @@ var rqmModMap map[string]RequestModifierFactory func init() { rqmModMap = make(map[string]RequestModifierFactory) - rqmModMap["ForwardRequestHeaders"] = func(_ ...string) proxychain.RequestModification { - return rx.ForwardRequestHeaders() - } + rqmModMap["ForwardRequestHeaders"] = func(_ ...string) proxychain.RequestModification { + return rx.ForwardRequestHeaders() + } - rqmModMap["MasqueradeAsGoogleBot"] = func(_ ...string) proxychain.RequestModification { - return rx.MasqueradeAsGoogleBot() - } + rqmModMap["MasqueradeAsGoogleBot"] = func(_ ...string) proxychain.RequestModification { + return rx.MasqueradeAsGoogleBot() + } - rqmModMap["MasqueradeAsBingBot"] = func(_ ...string) proxychain.RequestModification { - return rx.MasqueradeAsBingBot() - } + rqmModMap["MasqueradeAsBingBot"] = func(_ ...string) proxychain.RequestModification { + return rx.MasqueradeAsBingBot() + } - rqmModMap["MasqueradeAsWaybackMachineBot"] = func(_ ...string) proxychain.RequestModification { - return rx.MasqueradeAsWaybackMachineBot() - } + rqmModMap["MasqueradeAsWaybackMachineBot"] = func(_ ...string) proxychain.RequestModification { + return rx.MasqueradeAsWaybackMachineBot() + } - rqmModMap["MasqueradeAsFacebookBot"] = func(_ ...string) proxychain.RequestModification { - return rx.MasqueradeAsFacebookBot() - } + rqmModMap["MasqueradeAsFacebookBot"] = func(_ ...string) proxychain.RequestModification { + return rx.MasqueradeAsFacebookBot() + } - rqmModMap["MasqueradeAsYandexBot"] = func(_ ...string) proxychain.RequestModification { - return rx.MasqueradeAsYandexBot() - } + rqmModMap["MasqueradeAsYandexBot"] = func(_ ...string) proxychain.RequestModification { + return rx.MasqueradeAsYandexBot() + } - rqmModMap["MasqueradeAsBaiduBot"] = func(_ ...string) proxychain.RequestModification { - return rx.MasqueradeAsBaiduBot() - } + rqmModMap["MasqueradeAsBaiduBot"] = func(_ ...string) proxychain.RequestModification { + return rx.MasqueradeAsBaiduBot() + } - rqmModMap["MasqueradeAsDuckDuckBot"] = func(_ ...string) proxychain.RequestModification { - return rx.MasqueradeAsDuckDuckBot() - } + rqmModMap["MasqueradeAsDuckDuckBot"] = func(_ ...string) proxychain.RequestModification { + return rx.MasqueradeAsDuckDuckBot() + } - rqmModMap["MasqueradeAsYahooBot"] = func(_ ...string) proxychain.RequestModification { - return rx.MasqueradeAsYahooBot() - } + rqmModMap["MasqueradeAsYahooBot"] = func(_ ...string) proxychain.RequestModification { + return rx.MasqueradeAsYahooBot() + } - rqmModMap["ModifyDomainWithRegex"] = func(params ...string) proxychain.RequestModification { - return rx.ModifyDomainWithRegex(params[0], params[1]) - } + rqmModMap["ModifyDomainWithRegex"] = func(params ...string) proxychain.RequestModification { + return rx.ModifyDomainWithRegex(params[0], params[1]) + } - rqmModMap["SetOutgoingCookie"] = func(params ...string) proxychain.RequestModification { - return rx.SetOutgoingCookie(params[0], params[1]) - } + rqmModMap["SetOutgoingCookie"] = func(params ...string) proxychain.RequestModification { + return rx.SetOutgoingCookie(params[0], params[1]) + } - rqmModMap["SetOutgoingCookies"] = func(params ...string) proxychain.RequestModification { - return rx.SetOutgoingCookies(params[0]) - } + rqmModMap["SetOutgoingCookies"] = func(params ...string) proxychain.RequestModification { + return rx.SetOutgoingCookies(params[0]) + } - rqmModMap["DeleteOutgoingCookie"] = func(params ...string) proxychain.RequestModification { - return rx.DeleteOutgoingCookie(params[0]) - } + rqmModMap["DeleteOutgoingCookie"] = func(params ...string) proxychain.RequestModification { + return rx.DeleteOutgoingCookie(params[0]) + } - rqmModMap["DeleteOutgoingCookies"] = func(_ ...string) proxychain.RequestModification { - return rx.DeleteOutgoingCookies() - } + rqmModMap["DeleteOutgoingCookies"] = func(_ ...string) proxychain.RequestModification { + return rx.DeleteOutgoingCookies() + } - rqmModMap["DeleteOutgoingCookiesExcept"] = func(params ...string) proxychain.RequestModification { - return rx.DeleteOutgoingCookiesExcept(params[0]) - } + rqmModMap["DeleteOutgoingCookiesExcept"] = func(params ...string) proxychain.RequestModification { + return rx.DeleteOutgoingCookiesExcept(params[0]) + } - rqmModMap["ModifyPathWithRegex"] = func(params ...string) proxychain.RequestModification { - return rx.ModifyPathWithRegex(params[0], params[1]) - } + rqmModMap["ModifyPathWithRegex"] = func(params ...string) proxychain.RequestModification { + return rx.ModifyPathWithRegex(params[0], params[1]) + } - rqmModMap["ModifyQueryParams"] = func(params ...string) proxychain.RequestModification { - return rx.ModifyQueryParams(params[0], params[1]) - } + rqmModMap["ModifyQueryParams"] = func(params ...string) proxychain.RequestModification { + return rx.ModifyQueryParams(params[0], params[1]) + } - rqmModMap["SetRequestHeader"] = func(params ...string) proxychain.RequestModification { - return rx.SetRequestHeader(params[0], params[1]) - } + rqmModMap["SetRequestHeader"] = func(params ...string) proxychain.RequestModification { + return rx.SetRequestHeader(params[0], params[1]) + } - rqmModMap["DeleteRequestHeader"] = func(params ...string) proxychain.RequestModification { - return rx.DeleteRequestHeader(params[0]) - } + rqmModMap["DeleteRequestHeader"] = func(params ...string) proxychain.RequestModification { + return rx.DeleteRequestHeader(params[0]) + } - rqmModMap["RequestArchiveIs"] = func(_ ...string) proxychain.RequestModification { - return rx.RequestArchiveIs() - } + rqmModMap["RequestArchiveIs"] = func(_ ...string) proxychain.RequestModification { + return rx.RequestArchiveIs() + } - rqmModMap["RequestGoogleCache"] = func(_ ...string) proxychain.RequestModification { - return rx.RequestGoogleCache() - } + rqmModMap["RequestGoogleCache"] = func(_ ...string) proxychain.RequestModification { + return rx.RequestGoogleCache() + } - rqmModMap["RequestWaybackMachine"] = func(_ ...string) proxychain.RequestModification { - return rx.RequestWaybackMachine() - } + rqmModMap["RequestWaybackMachine"] = func(_ ...string) proxychain.RequestModification { + return rx.RequestWaybackMachine() + } - rqmModMap["ResolveWithGoogleDoH"] = func(_ ...string) proxychain.RequestModification { - return rx.ResolveWithGoogleDoH() - } + rqmModMap["ResolveWithGoogleDoH"] = func(_ ...string) proxychain.RequestModification { + return rx.ResolveWithGoogleDoH() + } - rqmModMap["SpoofOrigin"] = func(params ...string) proxychain.RequestModification { - return rx.SpoofOrigin(params[0]) - } + rqmModMap["SpoofOrigin"] = func(params ...string) proxychain.RequestModification { + return rx.SpoofOrigin(params[0]) + } - rqmModMap["HideOrigin"] = func(_ ...string) proxychain.RequestModification { - return rx.HideOrigin() - } + rqmModMap["HideOrigin"] = func(_ ...string) proxychain.RequestModification { + return rx.HideOrigin() + } - rqmModMap["SpoofReferrer"] = func(params ...string) proxychain.RequestModification { - return rx.SpoofReferrer(params[0]) - } + rqmModMap["SpoofReferrer"] = func(params ...string) proxychain.RequestModification { + return rx.SpoofReferrer(params[0]) + } - rqmModMap["HideReferrer"] = func(_ ...string) proxychain.RequestModification { - return rx.HideReferrer() - } + rqmModMap["HideReferrer"] = func(_ ...string) proxychain.RequestModification { + return rx.HideReferrer() + } - rqmModMap["SpoofReferrerFromBaiduSearch"] = func(_ ...string) proxychain.RequestModification { - return rx.SpoofReferrerFromBaiduSearch() - } + rqmModMap["SpoofReferrerFromBaiduSearch"] = func(_ ...string) proxychain.RequestModification { + return rx.SpoofReferrerFromBaiduSearch() + } - rqmModMap["SpoofReferrerFromBingSearch"] = func(_ ...string) proxychain.RequestModification { - return rx.SpoofReferrerFromBingSearch() - } + rqmModMap["SpoofReferrerFromBingSearch"] = func(_ ...string) proxychain.RequestModification { + return rx.SpoofReferrerFromBingSearch() + } - rqmModMap["SpoofReferrerFromGoogleSearch"] = func(_ ...string) proxychain.RequestModification { - return rx.SpoofReferrerFromGoogleSearch() - } + rqmModMap["SpoofReferrerFromGoogleSearch"] = func(_ ...string) proxychain.RequestModification { + return rx.SpoofReferrerFromGoogleSearch() + } - rqmModMap["SpoofReferrerFromLinkedInPost"] = func(_ ...string) proxychain.RequestModification { - return rx.SpoofReferrerFromLinkedInPost() - } + rqmModMap["SpoofReferrerFromLinkedInPost"] = func(_ ...string) proxychain.RequestModification { + return rx.SpoofReferrerFromLinkedInPost() + } - rqmModMap["SpoofReferrerFromNaverSearch"] = func(_ ...string) proxychain.RequestModification { - return rx.SpoofReferrerFromNaverSearch() - } + rqmModMap["SpoofReferrerFromNaverSearch"] = func(_ ...string) proxychain.RequestModification { + return rx.SpoofReferrerFromNaverSearch() + } - rqmModMap["SpoofReferrerFromPinterestPost"] = func(_ ...string) proxychain.RequestModification { - return rx.SpoofReferrerFromPinterestPost() - } + rqmModMap["SpoofReferrerFromPinterestPost"] = func(_ ...string) proxychain.RequestModification { + return rx.SpoofReferrerFromPinterestPost() + } - rqmModMap["SpoofReferrerFromQQPost"] = func(_ ...string) proxychain.RequestModification { - return rx.SpoofReferrerFromQQPost() - } + rqmModMap["SpoofReferrerFromQQPost"] = func(_ ...string) proxychain.RequestModification { + return rx.SpoofReferrerFromQQPost() + } - rqmModMap["SpoofReferrerFromRedditPost"] = func(_ ...string) proxychain.RequestModification { - return rx.SpoofReferrerFromRedditPost() - } + rqmModMap["SpoofReferrerFromRedditPost"] = func(_ ...string) proxychain.RequestModification { + return rx.SpoofReferrerFromRedditPost() + } - rqmModMap["SpoofReferrerFromTumblrPost"] = func(_ ...string) proxychain.RequestModification { - return rx.SpoofReferrerFromTumblrPost() - } + rqmModMap["SpoofReferrerFromTumblrPost"] = func(_ ...string) proxychain.RequestModification { + return rx.SpoofReferrerFromTumblrPost() + } - rqmModMap["SpoofReferrerFromTwitterPost"] = func(_ ...string) proxychain.RequestModification { - return rx.SpoofReferrerFromTwitterPost() - } + rqmModMap["SpoofReferrerFromTwitterPost"] = func(_ ...string) proxychain.RequestModification { + return rx.SpoofReferrerFromTwitterPost() + } - rqmModMap["SpoofReferrerFromVkontaktePost"] = func(_ ...string) proxychain.RequestModification { - return rx.SpoofReferrerFromVkontaktePost() - } + rqmModMap["SpoofReferrerFromVkontaktePost"] = func(_ ...string) proxychain.RequestModification { + return rx.SpoofReferrerFromVkontaktePost() + } - rqmModMap["SpoofReferrerFromWeiboPost"] = func(_ ...string) proxychain.RequestModification { - return rx.SpoofReferrerFromWeiboPost() - } + rqmModMap["SpoofReferrerFromWeiboPost"] = func(_ ...string) proxychain.RequestModification { + return rx.SpoofReferrerFromWeiboPost() + } - rqmModMap["SpoofUserAgent"] = func(params ...string) proxychain.RequestModification { - return rx.SpoofUserAgent(params[0]) - } + rqmModMap["SpoofUserAgent"] = func(params ...string) proxychain.RequestModification { + return rx.SpoofUserAgent(params[0]) + } - rqmModMap["SpoofXForwardedFor"] = func(params ...string) proxychain.RequestModification { - return rx.SpoofXForwardedFor(params[0]) - } -} + rqmModMap["SpoofXForwardedFor"] = func(params ...string) proxychain.RequestModification { + return rx.SpoofXForwardedFor(params[0]) + } + +} \ No newline at end of file diff --git a/proxychain/ruleset/rule_resmod_types.gen.go b/proxychain/ruleset/rule_resmod_types.gen.go index ea869a3..f6ec8c0 100644 --- a/proxychain/ruleset/rule_resmod_types.gen.go +++ b/proxychain/ruleset/rule_resmod_types.gen.go @@ -1,5 +1,5 @@ -package ruleset_v2 +package ruleset_v2 // DO NOT EDIT THIS FILE. It is automatically generated by ladder/proxychain/codegen/codegen.go // The purpose of this is serialization of rulesets from JSON or YAML into functional options suitable // for use in proxychains. @@ -16,83 +16,84 @@ var rsmModMap map[string]ResponseModifierFactory func init() { rsmModMap = make(map[string]ResponseModifierFactory) - rsmModMap["APIContent"] = func(_ ...string) proxychain.ResponseModification { - return tx.APIContent() - } + rsmModMap["APIContent"] = func(_ ...string) proxychain.ResponseModification { + return tx.APIContent() + } - rsmModMap["BlockElementRemoval"] = func(params ...string) proxychain.ResponseModification { - return tx.BlockElementRemoval(params[0]) - } + rsmModMap["BlockElementRemoval"] = func(params ...string) proxychain.ResponseModification { + return tx.BlockElementRemoval(params[0]) + } - rsmModMap["BypassCORS"] = func(_ ...string) proxychain.ResponseModification { - return tx.BypassCORS() - } + rsmModMap["BypassCORS"] = func(_ ...string) proxychain.ResponseModification { + return tx.BypassCORS() + } - rsmModMap["BypassContentSecurityPolicy"] = func(_ ...string) proxychain.ResponseModification { - return tx.BypassContentSecurityPolicy() - } + rsmModMap["BypassContentSecurityPolicy"] = func(_ ...string) proxychain.ResponseModification { + return tx.BypassContentSecurityPolicy() + } - rsmModMap["SetContentSecurityPolicy"] = func(params ...string) proxychain.ResponseModification { - return tx.SetContentSecurityPolicy(params[0]) - } + rsmModMap["SetContentSecurityPolicy"] = func(params ...string) proxychain.ResponseModification { + return tx.SetContentSecurityPolicy(params[0]) + } - rsmModMap["ForwardResponseHeaders"] = func(_ ...string) proxychain.ResponseModification { - return tx.ForwardResponseHeaders() - } + rsmModMap["ForwardResponseHeaders"] = func(_ ...string) proxychain.ResponseModification { + return tx.ForwardResponseHeaders() + } - rsmModMap["GenerateReadableOutline"] = func(_ ...string) proxychain.ResponseModification { - return tx.GenerateReadableOutline() - } + rsmModMap["GenerateReadableOutline"] = func(_ ...string) proxychain.ResponseModification { + return tx.GenerateReadableOutline() + } - rsmModMap["InjectScriptBeforeDOMContentLoaded"] = func(params ...string) proxychain.ResponseModification { - return tx.InjectScriptBeforeDOMContentLoaded(params[0]) - } + rsmModMap["InjectScriptBeforeDOMContentLoaded"] = func(params ...string) proxychain.ResponseModification { + return tx.InjectScriptBeforeDOMContentLoaded(params[0]) + } - rsmModMap["InjectScriptAfterDOMContentLoaded"] = func(params ...string) proxychain.ResponseModification { - return tx.InjectScriptAfterDOMContentLoaded(params[0]) - } + rsmModMap["InjectScriptAfterDOMContentLoaded"] = func(params ...string) proxychain.ResponseModification { + return tx.InjectScriptAfterDOMContentLoaded(params[0]) + } - rsmModMap["InjectScriptAfterDOMIdle"] = func(params ...string) proxychain.ResponseModification { - return tx.InjectScriptAfterDOMIdle(params[0]) - } + rsmModMap["InjectScriptAfterDOMIdle"] = func(params ...string) proxychain.ResponseModification { + return tx.InjectScriptAfterDOMIdle(params[0]) + } - rsmModMap["DeleteIncomingCookies"] = func(params ...string) proxychain.ResponseModification { - return tx.DeleteIncomingCookies(params[0]) - } + rsmModMap["DeleteIncomingCookies"] = func(params ...string) proxychain.ResponseModification { + return tx.DeleteIncomingCookies(params[0]) + } - rsmModMap["DeleteIncomingCookiesExcept"] = func(params ...string) proxychain.ResponseModification { - return tx.DeleteIncomingCookiesExcept(params[0]) - } + rsmModMap["DeleteIncomingCookiesExcept"] = func(params ...string) proxychain.ResponseModification { + return tx.DeleteIncomingCookiesExcept(params[0]) + } - rsmModMap["SetIncomingCookies"] = func(params ...string) proxychain.ResponseModification { - return tx.SetIncomingCookies(params[0]) - } + rsmModMap["SetIncomingCookies"] = func(params ...string) proxychain.ResponseModification { + return tx.SetIncomingCookies(params[0]) + } - rsmModMap["SetIncomingCookie"] = func(params ...string) proxychain.ResponseModification { - return tx.SetIncomingCookie(params[0], params[1]) - } + rsmModMap["SetIncomingCookie"] = func(params ...string) proxychain.ResponseModification { + return tx.SetIncomingCookie(params[0], params[1]) + } - rsmModMap["SetResponseHeader"] = func(params ...string) proxychain.ResponseModification { - return tx.SetResponseHeader(params[0], params[1]) - } + rsmModMap["SetResponseHeader"] = func(params ...string) proxychain.ResponseModification { + return tx.SetResponseHeader(params[0], params[1]) + } - rsmModMap["DeleteResponseHeader"] = func(params ...string) proxychain.ResponseModification { - return tx.DeleteResponseHeader(params[0]) - } + rsmModMap["DeleteResponseHeader"] = func(params ...string) proxychain.ResponseModification { + return tx.DeleteResponseHeader(params[0]) + } - rsmModMap["PatchDynamicResourceURLs"] = func(_ ...string) proxychain.ResponseModification { - return tx.PatchDynamicResourceURLs() - } + rsmModMap["PatchDynamicResourceURLs"] = func(_ ...string) proxychain.ResponseModification { + return tx.PatchDynamicResourceURLs() + } - rsmModMap["PatchGoogleAnalytics"] = func(_ ...string) proxychain.ResponseModification { - return tx.PatchGoogleAnalytics() - } + rsmModMap["PatchGoogleAnalytics"] = func(_ ...string) proxychain.ResponseModification { + return tx.PatchGoogleAnalytics() + } - rsmModMap["PatchTrackerScripts"] = func(_ ...string) proxychain.ResponseModification { - return tx.PatchTrackerScripts() - } + rsmModMap["PatchTrackerScripts"] = func(_ ...string) proxychain.ResponseModification { + return tx.PatchTrackerScripts() + } - rsmModMap["RewriteHTMLResourceURLs"] = func(_ ...string) proxychain.ResponseModification { - return tx.RewriteHTMLResourceURLs() - } -} + rsmModMap["RewriteHTMLResourceURLs"] = func(_ ...string) proxychain.ResponseModification { + return tx.RewriteHTMLResourceURLs() + } + +} \ No newline at end of file diff --git a/proxychain/ruleset/ruleset.go b/proxychain/ruleset/ruleset.go index 9731df6..68056d2 100644 --- a/proxychain/ruleset/ruleset.go +++ b/proxychain/ruleset/ruleset.go @@ -163,6 +163,18 @@ func (rs *Ruleset) loadRulesFromLocalDir(path string) error { return nil }) + // create a map of pointers to rules loaded above based on domain string keys + // this way we don't have two copies of the rule in ruleset + for i, rule := range rs.Rules { + rulePtr := &rs.Rules[i] + for _, domain := range rule.Domains { + rs._rulemap[domain] = rulePtr + if !strings.HasPrefix(domain, "www.") { + rs._rulemap["www."+domain] = rulePtr + } + } + } + if err != nil { return err } diff --git a/ruleset_v2.yaml b/ruleset_v2.yaml new file mode 100644 index 0000000..0eea0ec --- /dev/null +++ b/ruleset_v2.yaml @@ -0,0 +1,17 @@ +rules: + - domains: + - example.com + - www.example.com + responsemodifications: + - name: APIContent + params: [] + - name: SetContentSecurityPolicy + params: + - foobar + - name: SetIncomingCookie + params: + - authorization-bearer + - hunter2 + requestmodifications: + - name: ForwardRequestHeaders + params: []