work on rule reqmod resmod codegen for serialization

This commit is contained in:
Kevin Pham
2023-12-03 23:34:46 -06:00
parent a83a95c5a5
commit 7883b32335
10 changed files with 454 additions and 25 deletions

View File

@@ -0,0 +1,86 @@
package main
import (
"fmt"
"go/ast"
"go/parser"
"go/token"
//"io/fs"
"os"
"path/filepath"
"strings"
//"strings"
)
func modToFactoryMap(fn *ast.FuncDecl) (modMap string) {
paramCount := len(fn.Type.Params.List)
name := fn.Name.Name
var x string
switch paramCount {
case 0:
x = fmt.Sprintf(" resModMap[\"%s\"] = func(_ ...string) proxychain.ResponseModification {\n return rx.%s()\n }\n", name, name)
default:
p := []string{}
for i := 0; i < paramCount; i++ {
p = append(p, fmt.Sprintf("params[%d]", i))
}
params := strings.Join(p, ", ")
x = fmt.Sprintf(" resModMap[\"%s\"] = func(params, ...string) proxychain.ResponseModification {\n return rx.%s(%s)\n }\n", name, name, params)
}
return x
}
func main() {
fset := token.NewFileSet()
// Directory containing your Go files
dir := "../requestmodifiers/"
files, err := os.ReadDir(dir)
if err != nil {
panic(err)
}
factoryMaps := []string{}
for _, file := range files {
if filepath.Ext(file.Name()) != ".go" {
continue
}
// Parse each Go file
node, err := parser.ParseFile(fset, filepath.Join(dir, file.Name()), nil, parser.ParseComments)
if err != nil {
panic(err)
}
ast.Inspect(node, func(n ast.Node) bool {
fn, ok := n.(*ast.FuncDecl)
if ok && fn.Recv == nil && fn.Name.IsExported() {
factoryMaps = append(factoryMaps, modToFactoryMap(fn))
}
return true
})
}
code := fmt.Sprintf(`
package ruleset_v2
import (
"ladder/proxychain"
rx "ladder/proxychain/responsemodifiers"
)
type ResponseModifierFactory func(params ...string) proxychain.ResponseModification
var resModMap map[string]ResponseModifierFactory
// TODO: create codegen using AST parsing of exported methods in ladder/proxychain/responsemodifiers/*.go
func init() {
resModMap = make(map[string]ResponseModifierFactory)
%s
}`, strings.Join(factoryMaps, "\n"))
fmt.Println(code)
}

View File

@@ -0,0 +1,49 @@
package ruleset_v2
import (
"encoding/json"
"fmt"
"ladder/proxychain"
)
type Rule struct {
Domains []string
RequestModifications []proxychain.RequestModification
ResponseModifications []proxychain.ResponseModification
}
// implement type encoding/json/Marshaler
func (rule *Rule) UnmarshalJSON(data []byte) error {
type Aux struct {
Domains []string `json:"domains"`
RequestModifications []string `json:"request_modifications"`
ResponseModifications []string `json:"response_modifications"`
}
aux := &Aux{}
if err := json.Unmarshal(data, aux); err != nil {
return err
}
//fmt.Println(aux.Domains)
rule.Domains = aux.Domains
// convert requestModification function call string into actual functional option
for _, resModStr := range aux.RequestModifications {
name, params, err := parseFuncCall(resModStr)
if err != nil {
return fmt.Errorf("Rule::UnmarshalJSON invalid function call syntax => '%s'", err)
}
f, exists := resModMap[name]
if !exists {
return fmt.Errorf("Rule::UnmarshalJSON => requestModifier '%s' does not exist, please check spelling", err)
}
rule.ResponseModifications = append(rule.ResponseModifications, f(params...))
}
return nil
}
func (r *Rule) MarshalJSON() ([]byte, error) {
return []byte{}, nil
}

View File

@@ -0,0 +1,184 @@
package ruleset
import (
"ladder/proxychain"
rx "ladder/proxychain/responsemodifiers"
)
type ResponseModifierFactory func(params ...string) proxychain.ResponseModification
var resModMap map[string]ResponseModifierFactory
// TODO: create codegen using AST parsing of exported methods in ladder/proxychain/responsemodifiers/*.go
func init() {
resModMap = make(map[string]ResponseModifierFactory)
resModMap["ForwardRequestHeaders"] = func(_ ...string) proxychain.ResponseModification {
return rx.ForwardRequestHeaders()
}
resModMap["MasqueradeAsGoogleBot"] = func(_ ...string) proxychain.ResponseModification {
return rx.MasqueradeAsGoogleBot()
}
resModMap["MasqueradeAsBingBot"] = func(_ ...string) proxychain.ResponseModification {
return rx.MasqueradeAsBingBot()
}
resModMap["MasqueradeAsWaybackMachineBot"] = func(_ ...string) proxychain.ResponseModification {
return rx.MasqueradeAsWaybackMachineBot()
}
resModMap["MasqueradeAsFacebookBot"] = func(_ ...string) proxychain.ResponseModification {
return rx.MasqueradeAsFacebookBot()
}
resModMap["MasqueradeAsYandexBot"] = func(_ ...string) proxychain.ResponseModification {
return rx.MasqueradeAsYandexBot()
}
resModMap["MasqueradeAsBaiduBot"] = func(_ ...string) proxychain.ResponseModification {
return rx.MasqueradeAsBaiduBot()
}
resModMap["MasqueradeAsDuckDuckBot"] = func(_ ...string) proxychain.ResponseModification {
return rx.MasqueradeAsDuckDuckBot()
}
resModMap["MasqueradeAsYahooBot"] = func(_ ...string) proxychain.ResponseModification {
return rx.MasqueradeAsYahooBot()
}
resModMap["ModifyDomainWithRegex"] = func(params, ...string) proxychain.ResponseModification {
return rx.ModifyDomainWithRegex(params[0], params[1])
}
resModMap["SetOutgoingCookie"] = func(params, ...string) proxychain.ResponseModification {
return rx.SetOutgoingCookie(params[0], params[1])
}
resModMap["SetOutgoingCookies"] = func(params, ...string) proxychain.ResponseModification {
return rx.SetOutgoingCookies(params[0])
}
resModMap["DeleteOutgoingCookie"] = func(params, ...string) proxychain.ResponseModification {
return rx.DeleteOutgoingCookie(params[0])
}
resModMap["DeleteOutgoingCookies"] = func(_ ...string) proxychain.ResponseModification {
return rx.DeleteOutgoingCookies()
}
resModMap["DeleteOutgoingCookiesExcept"] = func(params, ...string) proxychain.ResponseModification {
return rx.DeleteOutgoingCookiesExcept(params[0])
}
resModMap["ModifyPathWithRegex"] = func(params, ...string) proxychain.ResponseModification {
return rx.ModifyPathWithRegex(params[0], params[1])
}
resModMap["ModifyQueryParams"] = func(params, ...string) proxychain.ResponseModification {
return rx.ModifyQueryParams(params[0], params[1])
}
resModMap["SetRequestHeader"] = func(params, ...string) proxychain.ResponseModification {
return rx.SetRequestHeader(params[0], params[1])
}
resModMap["DeleteRequestHeader"] = func(params, ...string) proxychain.ResponseModification {
return rx.DeleteRequestHeader(params[0])
}
resModMap["RequestArchiveIs"] = func(_ ...string) proxychain.ResponseModification {
return rx.RequestArchiveIs()
}
resModMap["RequestGoogleCache"] = func(_ ...string) proxychain.ResponseModification {
return rx.RequestGoogleCache()
}
resModMap["RequestWaybackMachine"] = func(_ ...string) proxychain.ResponseModification {
return rx.RequestWaybackMachine()
}
resModMap["NewCustomDialer"] = func(params, ...string) proxychain.ResponseModification {
return rx.NewCustomDialer(params[0])
}
resModMap["ResolveWithGoogleDoH"] = func(_ ...string) proxychain.ResponseModification {
return rx.ResolveWithGoogleDoH()
}
resModMap["SpoofOrigin"] = func(params, ...string) proxychain.ResponseModification {
return rx.SpoofOrigin(params[0])
}
resModMap["HideOrigin"] = func(_ ...string) proxychain.ResponseModification {
return rx.HideOrigin()
}
resModMap["SpoofReferrer"] = func(params, ...string) proxychain.ResponseModification {
return rx.SpoofReferrer(params[0])
}
resModMap["HideReferrer"] = func(_ ...string) proxychain.ResponseModification {
return rx.HideReferrer()
}
resModMap["SpoofReferrerFromBaiduSearch"] = func(_ ...string) proxychain.ResponseModification {
return rx.SpoofReferrerFromBaiduSearch()
}
resModMap["SpoofReferrerFromBingSearch"] = func(_ ...string) proxychain.ResponseModification {
return rx.SpoofReferrerFromBingSearch()
}
resModMap["SpoofReferrerFromGoogleSearch"] = func(_ ...string) proxychain.ResponseModification {
return rx.SpoofReferrerFromGoogleSearch()
}
resModMap["SpoofReferrerFromLinkedInPost"] = func(_ ...string) proxychain.ResponseModification {
return rx.SpoofReferrerFromLinkedInPost()
}
resModMap["SpoofReferrerFromNaverSearch"] = func(_ ...string) proxychain.ResponseModification {
return rx.SpoofReferrerFromNaverSearch()
}
resModMap["SpoofReferrerFromPinterestPost"] = func(_ ...string) proxychain.ResponseModification {
return rx.SpoofReferrerFromPinterestPost()
}
resModMap["SpoofReferrerFromQQPost"] = func(_ ...string) proxychain.ResponseModification {
return rx.SpoofReferrerFromQQPost()
}
resModMap["SpoofReferrerFromRedditPost"] = func(_ ...string) proxychain.ResponseModification {
return rx.SpoofReferrerFromRedditPost()
}
resModMap["SpoofReferrerFromTumblrPost"] = func(_ ...string) proxychain.ResponseModification {
return rx.SpoofReferrerFromTumblrPost()
}
resModMap["SpoofReferrerFromTwitterPost"] = func(_ ...string) proxychain.ResponseModification {
return rx.SpoofReferrerFromTwitterPost()
}
resModMap["SpoofReferrerFromVkontaktePost"] = func(_ ...string) proxychain.ResponseModification {
return rx.SpoofReferrerFromVkontaktePost()
}
resModMap["SpoofReferrerFromWeiboPost"] = func(_ ...string) proxychain.ResponseModification {
return rx.SpoofReferrerFromWeiboPost()
}
resModMap["SpoofUserAgent"] = func(params, ...string) proxychain.ResponseModification {
return rx.SpoofUserAgent(params[0])
}
resModMap["SpoofXForwardedFor"] = func(params, ...string) proxychain.ResponseModification {
return rx.SpoofXForwardedFor(params[0])
}
}

View File

@@ -0,0 +1,184 @@
package ruleset_v2
import (
"ladder/proxychain"
rx "ladder/proxychain/responsemodifiers"
)
type ResponseModifierFactory func(params ...string) proxychain.ResponseModification
var resModMap map[string]ResponseModifierFactory
// TODO: create codegen using AST parsing of exported methods in ladder/proxychain/responsemodifiers/*.go
func init() {
resModMap = make(map[string]ResponseModifierFactory)
resModMap["ForwardRequestHeaders"] = func(_ ...string) proxychain.ResponseModification {
return rx.ForwardRequestHeaders()
}
resModMap["MasqueradeAsGoogleBot"] = func(_ ...string) proxychain.ResponseModification {
return rx.MasqueradeAsGoogleBot()
}
resModMap["MasqueradeAsBingBot"] = func(_ ...string) proxychain.ResponseModification {
return rx.MasqueradeAsBingBot()
}
resModMap["MasqueradeAsWaybackMachineBot"] = func(_ ...string) proxychain.ResponseModification {
return rx.MasqueradeAsWaybackMachineBot()
}
resModMap["MasqueradeAsFacebookBot"] = func(_ ...string) proxychain.ResponseModification {
return rx.MasqueradeAsFacebookBot()
}
resModMap["MasqueradeAsYandexBot"] = func(_ ...string) proxychain.ResponseModification {
return rx.MasqueradeAsYandexBot()
}
resModMap["MasqueradeAsBaiduBot"] = func(_ ...string) proxychain.ResponseModification {
return rx.MasqueradeAsBaiduBot()
}
resModMap["MasqueradeAsDuckDuckBot"] = func(_ ...string) proxychain.ResponseModification {
return rx.MasqueradeAsDuckDuckBot()
}
resModMap["MasqueradeAsYahooBot"] = func(_ ...string) proxychain.ResponseModification {
return rx.MasqueradeAsYahooBot()
}
resModMap["ModifyDomainWithRegex"] = func(params, ...string) proxychain.ResponseModification {
return rx.ModifyDomainWithRegex(params[0], params[1])
}
resModMap["SetOutgoingCookie"] = func(params, ...string) proxychain.ResponseModification {
return rx.SetOutgoingCookie(params[0], params[1])
}
resModMap["SetOutgoingCookies"] = func(params, ...string) proxychain.ResponseModification {
return rx.SetOutgoingCookies(params[0])
}
resModMap["DeleteOutgoingCookie"] = func(params, ...string) proxychain.ResponseModification {
return rx.DeleteOutgoingCookie(params[0])
}
resModMap["DeleteOutgoingCookies"] = func(_ ...string) proxychain.ResponseModification {
return rx.DeleteOutgoingCookies()
}
resModMap["DeleteOutgoingCookiesExcept"] = func(params, ...string) proxychain.ResponseModification {
return rx.DeleteOutgoingCookiesExcept(params[0])
}
resModMap["ModifyPathWithRegex"] = func(params, ...string) proxychain.ResponseModification {
return rx.ModifyPathWithRegex(params[0], params[1])
}
resModMap["ModifyQueryParams"] = func(params, ...string) proxychain.ResponseModification {
return rx.ModifyQueryParams(params[0], params[1])
}
resModMap["SetRequestHeader"] = func(params, ...string) proxychain.ResponseModification {
return rx.SetRequestHeader(params[0], params[1])
}
resModMap["DeleteRequestHeader"] = func(params, ...string) proxychain.ResponseModification {
return rx.DeleteRequestHeader(params[0])
}
resModMap["RequestArchiveIs"] = func(_ ...string) proxychain.ResponseModification {
return rx.RequestArchiveIs()
}
resModMap["RequestGoogleCache"] = func(_ ...string) proxychain.ResponseModification {
return rx.RequestGoogleCache()
}
resModMap["RequestWaybackMachine"] = func(_ ...string) proxychain.ResponseModification {
return rx.RequestWaybackMachine()
}
resModMap["NewCustomDialer"] = func(params, ...string) proxychain.ResponseModification {
return rx.NewCustomDialer(params[0])
}
resModMap["ResolveWithGoogleDoH"] = func(_ ...string) proxychain.ResponseModification {
return rx.ResolveWithGoogleDoH()
}
resModMap["SpoofOrigin"] = func(params, ...string) proxychain.ResponseModification {
return rx.SpoofOrigin(params[0])
}
resModMap["HideOrigin"] = func(_ ...string) proxychain.ResponseModification {
return rx.HideOrigin()
}
resModMap["SpoofReferrer"] = func(params, ...string) proxychain.ResponseModification {
return rx.SpoofReferrer(params[0])
}
resModMap["HideReferrer"] = func(_ ...string) proxychain.ResponseModification {
return rx.HideReferrer()
}
resModMap["SpoofReferrerFromBaiduSearch"] = func(_ ...string) proxychain.ResponseModification {
return rx.SpoofReferrerFromBaiduSearch()
}
resModMap["SpoofReferrerFromBingSearch"] = func(_ ...string) proxychain.ResponseModification {
return rx.SpoofReferrerFromBingSearch()
}
resModMap["SpoofReferrerFromGoogleSearch"] = func(_ ...string) proxychain.ResponseModification {
return rx.SpoofReferrerFromGoogleSearch()
}
resModMap["SpoofReferrerFromLinkedInPost"] = func(_ ...string) proxychain.ResponseModification {
return rx.SpoofReferrerFromLinkedInPost()
}
resModMap["SpoofReferrerFromNaverSearch"] = func(_ ...string) proxychain.ResponseModification {
return rx.SpoofReferrerFromNaverSearch()
}
resModMap["SpoofReferrerFromPinterestPost"] = func(_ ...string) proxychain.ResponseModification {
return rx.SpoofReferrerFromPinterestPost()
}
resModMap["SpoofReferrerFromQQPost"] = func(_ ...string) proxychain.ResponseModification {
return rx.SpoofReferrerFromQQPost()
}
resModMap["SpoofReferrerFromRedditPost"] = func(_ ...string) proxychain.ResponseModification {
return rx.SpoofReferrerFromRedditPost()
}
resModMap["SpoofReferrerFromTumblrPost"] = func(_ ...string) proxychain.ResponseModification {
return rx.SpoofReferrerFromTumblrPost()
}
resModMap["SpoofReferrerFromTwitterPost"] = func(_ ...string) proxychain.ResponseModification {
return rx.SpoofReferrerFromTwitterPost()
}
resModMap["SpoofReferrerFromVkontaktePost"] = func(_ ...string) proxychain.ResponseModification {
return rx.SpoofReferrerFromVkontaktePost()
}
resModMap["SpoofReferrerFromWeiboPost"] = func(_ ...string) proxychain.ResponseModification {
return rx.SpoofReferrerFromWeiboPost()
}
resModMap["SpoofUserAgent"] = func(params, ...string) proxychain.ResponseModification {
return rx.SpoofUserAgent(params[0])
}
resModMap["SpoofXForwardedFor"] = func(params, ...string) proxychain.ResponseModification {
return rx.SpoofXForwardedFor(params[0])
}
}

View File

@@ -0,0 +1,45 @@
package ruleset_v2
import (
"encoding/json"
"fmt"
//"io"
"testing"
)
func TestRuleUnmarshalJSON(t *testing.T) {
ruleJSON := `{
"domains": [
"example.com",
"www.example.com"
],
"response_modifiers": [
"APIContent()",
"SetContentSecurityPolicy(\"foobar\")",
"SetIncomingCookie(\"authorization-bearer\", \"hunter2\")"
],
"response_modifiers": []
}`
//fmt.Println(ruleJSON)
rule := &Rule{}
err := json.Unmarshal([]byte(ruleJSON), rule)
if err != nil {
t.Errorf("expected no error in Unmarshal, got '%s'", err)
return
}
if len(rule.Domains) != 2 {
t.Errorf("expected number of domains to be 2")
return
}
if !(rule.Domains[0] == "example.com" || rule.Domains[1] == "example.com") {
t.Errorf("expected domain to be example.com")
return
}
if len(rule.ResponseModifications) == 3 {
t.Errorf("expected number of ResponseModifications to be 3")
}
fmt.Println(rule.ResponseModifications)
}

View File

@@ -0,0 +1,59 @@
package ruleset_v2
import (
"errors"
"strings"
)
// parseFuncCall takes a string that look like foo("bar", "baz") and breaks it down
// into funcName = "foo" and params = []string{"bar", "baz"}]
func parseFuncCall(funcCall string) (funcName string, params []string, err error) {
// Splitting the input string into two parts: functionName and parameters
parts := strings.SplitN(funcCall, "(", 2)
if len(parts) != 2 {
return "", nil, errors.New("invalid function call format")
}
// get function name
funcName = strings.TrimSpace(parts[0])
// Removing the closing parenthesis from the parameters part
paramsPart := strings.TrimSuffix(parts[1], ")")
if len(paramsPart) == 0 {
// No parameters
return funcName, []string{}, nil
}
inQuote := false
inEscape := false
param := ""
for _, r := range paramsPart {
switch {
case inQuote && !inEscape && r == '\\':
inEscape = true
continue
case inEscape && inQuote && r == '"':
param += string(r)
inEscape = false
continue
case inEscape:
param += string(r)
inEscape = false
continue
case r == '"':
inQuote = !inQuote
if !inQuote {
params = append(params, param)
param = ""
}
continue
case !inQuote && r == ',':
continue
case inQuote:
param += string(r)
continue
}
}
return funcName, params, nil
}

View File

@@ -0,0 +1,138 @@
package ruleset_v2
import (
"errors"
"reflect"
"testing"
)
func TestParseFuncCall(t *testing.T) {
testCases := []struct {
name string
input string
expected struct {
funcName string
params []string
err error
}
}{
{
name: "Normal case, one param",
input: `one("baz")`,
expected: struct {
funcName string
params []string
err error
}{funcName: "one", params: []string{"baz"}, err: nil},
},
{
name: "Normal case, one param, extra space in function call",
input: `two("baz" )`,
expected: struct {
funcName string
params []string
err error
}{funcName: "two", params: []string{"baz"}, err: nil},
},
{
name: "Normal case, one param, extra space in param",
input: `three("baz ")`,
expected: struct {
funcName string
params []string
err error
}{funcName: "three", params: []string{"baz "}, err: nil},
},
{
name: "Space in front of function",
input: ` three("baz")`,
expected: struct {
funcName string
params []string
err error
}{funcName: "three", params: []string{"baz"}, err: nil},
},
{
name: "Normal case, two params",
input: `foobar("baz", "qux")`,
expected: struct {
funcName string
params []string
err error
}{funcName: "foobar", params: []string{"baz", "qux"}, err: nil},
},
{
name: "Normal case, two params, no spaces between param comma",
input: `foobar("baz","qux")`,
expected: struct {
funcName string
params []string
err error
}{funcName: "foobar", params: []string{"baz", "qux"}, err: nil},
},
{
name: "Escaped parenthesis",
input: `testFunc("hello\(world", "anotherParam")`,
expected: struct {
funcName string
params []string
err error
}{funcName: "testFunc", params: []string{`hello(world`, "anotherParam"}, err: nil},
},
{
name: "Escaped quote",
input: `testFunc("hello\"world", "anotherParam")`,
expected: struct {
funcName string
params []string
err error
}{funcName: "testFunc", params: []string{`hello"world`, "anotherParam"}, err: nil},
},
{
name: "Two Escaped quote",
input: `testFunc("hello: \"world\"", "anotherParam")`,
expected: struct {
funcName string
params []string
err error
}{funcName: "testFunc", params: []string{`hello: "world"`, "anotherParam"}, err: nil},
},
{
name: "No parameters",
input: `emptyFunc()`,
expected: struct {
funcName string
params []string
err error
}{funcName: "emptyFunc", params: []string{}, err: nil},
},
{
name: "Invalid format",
input: `invalidFunc`,
expected: struct {
funcName string
params []string
err error
}{funcName: "", params: nil, err: errors.New("invalid function call format")},
},
{
name: "Invalid format 2",
input: `invalidFunc "foo", "bar"`,
expected: struct {
funcName string
params []string
err error
}{funcName: "", params: nil, err: errors.New("invalid function call format")},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
funcName, params, err := parseFuncCall(tc.input)
if funcName != tc.expected.funcName || !reflect.DeepEqual(params, tc.expected.params) || (err != nil && tc.expected.err != nil && err.Error() != tc.expected.err.Error()) {
//if funcName != tc.expected.funcName || (err != nil && tc.expected.err != nil && err.Error() != tc.expected.err.Error()) {
t.Errorf("Test %s failed: got (%s, %v, %v), want (%s, %v, %v)", tc.name, funcName, params, err, tc.expected.funcName, tc.expected.params, tc.expected.err)
}
})
}
}

View File

@@ -0,0 +1,32 @@
package ruleset_v2
import (
"net/url"
)
type IRuleset interface {
HasRule(url url.URL) bool
GetRule(url url.URL) (rule Rule, exists bool)
}
type Ruleset struct {
rulesetPath string
rules map[string]Rule
}
func (rs Ruleset) GetRule(url url.URL) (rule Rule, exists bool) {
rule, exists = rs.rules[url.Hostname()]
return rule, exists
}
func (rs Ruleset) HasRule(url url.URL) bool {
_, exists := rs.GetRule(url)
return exists
}
func NewRuleset(path string) (Ruleset, error) {
rs := Ruleset{
rulesetPath: path,
}
return rs, nil
}