begin refactor of proxy engine
This commit is contained in:
Kevin Pham
@@ -1,105 +0,0 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"ladder/pkg/ruleset"
|
||||
"os"
|
||||
|
||||
"golang.org/x/term"
|
||||
)
|
||||
|
||||
// HandleRulesetMerge merges a set of ruleset files, specified by the rulesetPath or RULESET env variable, into either YAML or Gzip format.
|
||||
// Exits the program with an error message if the ruleset path is not provided or if loading the ruleset fails.
|
||||
//
|
||||
// Parameters:
|
||||
// - rulesetPath: A pointer to a string specifying the path to the ruleset file.
|
||||
// - mergeRulesets: A pointer to a boolean indicating if a merge operation should be performed.
|
||||
// - mergeRulesetsGzip: A pointer to a boolean indicating if the merge should be in Gzip format.
|
||||
// - mergeRulesetsOutput: A pointer to a string specifying the output file path. If empty, the output is printed to stdout.
|
||||
//
|
||||
// Returns:
|
||||
// - An error if the ruleset loading or merging process fails, otherwise nil.
|
||||
func HandleRulesetMerge(rulesetPath *string, mergeRulesets *bool, mergeRulesetsGzip *bool, mergeRulesetsOutput *string) error {
|
||||
if *rulesetPath == "" {
|
||||
*rulesetPath = os.Getenv("RULESET")
|
||||
}
|
||||
if *rulesetPath == "" {
|
||||
fmt.Println("ERROR: no ruleset provided. Try again with --ruleset <ruleset.yaml>")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
rs, err := ruleset.NewRuleset(*rulesetPath)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if *mergeRulesetsGzip {
|
||||
return gzipMerge(rs, mergeRulesetsOutput)
|
||||
}
|
||||
return yamlMerge(rs, mergeRulesetsOutput)
|
||||
}
|
||||
|
||||
// gzipMerge takes a RuleSet and an optional output file path pointer. It compresses the RuleSet into Gzip format.
|
||||
// If the output file path is provided, the compressed data is written to this file. Otherwise, it prints a warning
|
||||
// and outputs the binary data to stdout
|
||||
//
|
||||
// Parameters:
|
||||
// - rs: The ruleset.RuleSet to be compressed.
|
||||
// - mergeRulesetsOutput: A pointer to a string specifying the output file path. If empty, the output is directed to stdout.
|
||||
//
|
||||
// Returns:
|
||||
// - An error if compression or file writing fails, otherwise nil.
|
||||
func gzipMerge(rs ruleset.RuleSet, mergeRulesetsOutput *string) error {
|
||||
gzip, err := rs.GzipYaml()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if *mergeRulesetsOutput != "" {
|
||||
out, err := os.Create(*mergeRulesetsOutput)
|
||||
defer out.Close()
|
||||
_, err = io.Copy(out, gzip)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if term.IsTerminal(int(os.Stdout.Fd())) {
|
||||
println("WARNING: binary output can mess up your terminal. Use '--merge-rulesets-output <ruleset.gz>' or pipe it to a file.")
|
||||
os.Exit(1)
|
||||
}
|
||||
_, err = io.Copy(os.Stdout, gzip)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// yamlMerge takes a RuleSet and an optional output file path pointer. It converts the RuleSet into YAML format.
|
||||
// If the output file path is provided, the YAML data is written to this file. If not, the YAML data is printed to stdout.
|
||||
//
|
||||
// Parameters:
|
||||
// - rs: The ruleset.RuleSet to be converted to YAML.
|
||||
// - mergeRulesetsOutput: A pointer to a string specifying the output file path. If empty, the output is printed to stdout.
|
||||
//
|
||||
// Returns:
|
||||
// - An error if YAML conversion or file writing fails, otherwise nil.
|
||||
func yamlMerge(rs ruleset.RuleSet, mergeRulesetsOutput *string) error {
|
||||
yaml, err := rs.Yaml()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *mergeRulesetsOutput == "" {
|
||||
fmt.Printf(yaml)
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
err = os.WriteFile(*mergeRulesetsOutput, []byte(yaml), fs.FileMode(os.O_RDWR))
|
||||
if err != nil {
|
||||
return fmt.Errorf("ERROR: failed to write merged YAML ruleset to '%s'\n", *mergeRulesetsOutput)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -11,6 +11,9 @@ import (
|
||||
"strings"
|
||||
|
||||
"ladder/pkg/ruleset"
|
||||
"ladder/proxychain"
|
||||
"ladder/proxychain/rqm"
|
||||
"ladder/proxychain/rsm"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/gofiber/fiber/v2"
|
||||
@@ -30,88 +33,32 @@ func init() {
|
||||
}
|
||||
}
|
||||
|
||||
// extracts a URL from the request ctx. If the URL in the request
|
||||
// is a relative path, it reconstructs the full URL using the referer header.
|
||||
func extractUrl(c *fiber.Ctx) (string, error) {
|
||||
// try to extract url-encoded
|
||||
reqUrl, err := url.QueryUnescape(c.Params("*"))
|
||||
if err != nil {
|
||||
// fallback
|
||||
reqUrl = c.Params("*")
|
||||
}
|
||||
|
||||
// Extract the actual path from req ctx
|
||||
urlQuery, err := url.Parse(reqUrl)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("error parsing request URL '%s': %v", reqUrl, err)
|
||||
}
|
||||
|
||||
isRelativePath := urlQuery.Scheme == ""
|
||||
|
||||
// eg: https://localhost:8080/images/foobar.jpg -> https://realsite.com/images/foobar.jpg
|
||||
if isRelativePath {
|
||||
// Parse the referer URL from the request header.
|
||||
refererUrl, err := url.Parse(c.Get("referer"))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("error parsing referer URL from req: '%s': %v", reqUrl, err)
|
||||
}
|
||||
|
||||
// Extract the real url from referer path
|
||||
realUrl, err := url.Parse(strings.TrimPrefix(refererUrl.Path, "/"))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("error parsing real URL from referer '%s': %v", refererUrl.Path, err)
|
||||
}
|
||||
|
||||
// reconstruct the full URL using the referer's scheme, host, and the relative path / queries
|
||||
fullUrl := &url.URL{
|
||||
Scheme: realUrl.Scheme,
|
||||
Host: realUrl.Host,
|
||||
Path: urlQuery.Path,
|
||||
RawQuery: urlQuery.RawQuery,
|
||||
}
|
||||
|
||||
if os.Getenv("LOG_URLS") == "true" {
|
||||
log.Printf("modified relative URL: '%s' -> '%s'", reqUrl, fullUrl.String())
|
||||
}
|
||||
return fullUrl.String(), nil
|
||||
|
||||
}
|
||||
|
||||
// default behavior:
|
||||
// eg: https://localhost:8080/https://realsite.com/images/foobar.jpg -> https://realsite.com/images/foobar.jpg
|
||||
return urlQuery.String(), nil
|
||||
|
||||
type ProxyOptions struct {
|
||||
RulesetPath string
|
||||
Verbose bool
|
||||
}
|
||||
|
||||
func ProxySite(rulesetPath string) fiber.Handler {
|
||||
if rulesetPath != "" {
|
||||
rs, err := ruleset.NewRuleset(rulesetPath)
|
||||
func NewProxySiteHandler(opts *ProxyOptions) fiber.Handler {
|
||||
var rs ruleset.RuleSet
|
||||
if opts.RulesetPath != "" {
|
||||
r, err := ruleset.NewRuleset(opts.RulesetPath)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
rulesSet = rs
|
||||
rs = r
|
||||
}
|
||||
|
||||
return func(c *fiber.Ctx) error {
|
||||
// Get the url from the URL
|
||||
url, err := extractUrl(c)
|
||||
if err != nil {
|
||||
log.Println("ERROR In URL extraction:", err)
|
||||
}
|
||||
|
||||
queries := c.Queries()
|
||||
body, _, resp, err := fetchSite(url, queries)
|
||||
if err != nil {
|
||||
log.Println("ERROR:", err)
|
||||
c.SendStatus(fiber.StatusInternalServerError)
|
||||
return c.SendString(err.Error())
|
||||
}
|
||||
|
||||
c.Cookie(&fiber.Cookie{})
|
||||
c.Set("Content-Type", resp.Header.Get("Content-Type"))
|
||||
c.Set("Content-Security-Policy", resp.Header.Get("Content-Security-Policy"))
|
||||
|
||||
return c.SendString(body)
|
||||
return proxychain.NewProxyChain().
|
||||
SetCtx(c).
|
||||
AddRuleset(&rs).
|
||||
SetRequestModifications(
|
||||
rqm.BlockOutgoingCookies(),
|
||||
).
|
||||
SetResultModifications(
|
||||
rsm.BlockIncomingCookies(),
|
||||
).
|
||||
Execute()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ import (
|
||||
|
||||
func TestProxySite(t *testing.T) {
|
||||
app := fiber.New()
|
||||
app.Get("/:url", ProxySite(""))
|
||||
app.Get("/:url", NewProxySiteHandler(nil))
|
||||
|
||||
req := httptest.NewRequest("GET", "/https://example.com", nil)
|
||||
resp, err := app.Test(req)
|
||||
|
||||
Reference in New Issue
Block a user