Compare commits
1 Commits
proxy_v2_f
...
docs/add-h
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
24ad760119 |
12
README.md
12
README.md
@@ -14,6 +14,18 @@ Freedom of information is an essential pillar of democracy and informed decision
|
||||
|
||||
> **Disclaimer:** This project is intended for educational purposes only. The author does not endorse or encourage any unethical or illegal activity. Use this tool at your own risk.
|
||||
|
||||
### How it works
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
client->>+ladder: GET
|
||||
ladder-->>ladder: apply RequestModifications
|
||||
ladder->>+website: GET
|
||||
website->>-ladder: 200 OK
|
||||
ladder-->>ladder: apply ResultModifications
|
||||
ladder->>-client: 200 OK
|
||||
```
|
||||
|
||||
### Features
|
||||
- [x] Bypass Paywalls
|
||||
- [x] Remove CORS headers from responses, assets, and images ...
|
||||
|
||||
22
cmd/main.go
22
cmd/main.go
@@ -8,7 +8,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"ladder/handlers"
|
||||
"ladder/internal/cli"
|
||||
"ladder/handlers/cli"
|
||||
|
||||
"github.com/akamensky/argparse"
|
||||
"github.com/gofiber/fiber/v2"
|
||||
@@ -40,13 +40,6 @@ func main() {
|
||||
Help: "This will spawn multiple processes listening",
|
||||
})
|
||||
|
||||
verbose := parser.Flag("v", "verbose", &argparse.Options{
|
||||
Required: false,
|
||||
Help: "Adds verbose logging",
|
||||
})
|
||||
|
||||
// TODO: add version flag that reads from handers/VERSION
|
||||
|
||||
ruleset := parser.String("r", "ruleset", &argparse.Options{
|
||||
Required: false,
|
||||
Help: "File, Directory or URL to a ruleset.yaml. Overrides RULESET environment variable.",
|
||||
@@ -91,7 +84,6 @@ func main() {
|
||||
},
|
||||
)
|
||||
|
||||
// TODO: move to cmd/auth.go
|
||||
userpass := os.Getenv("USERPASS")
|
||||
if userpass != "" {
|
||||
userpass := strings.Split(userpass, ":")
|
||||
@@ -102,7 +94,6 @@ func main() {
|
||||
}))
|
||||
}
|
||||
|
||||
// TODO: move to handlers/favicon.go
|
||||
app.Use(favicon.New(favicon.Config{
|
||||
Data: []byte(faviconData),
|
||||
URL: "/favicon.ico",
|
||||
@@ -116,8 +107,6 @@ func main() {
|
||||
}
|
||||
|
||||
app.Get("/", handlers.Form)
|
||||
|
||||
// TODO: move this logic to handers/styles.go
|
||||
app.Get("/styles.css", func(c *fiber.Ctx) error {
|
||||
cssData, err := cssData.ReadFile("styles.css")
|
||||
if err != nil {
|
||||
@@ -126,17 +115,10 @@ func main() {
|
||||
c.Set("Content-Type", "text/css")
|
||||
return c.Send(cssData)
|
||||
})
|
||||
|
||||
app.Get("ruleset", handlers.Ruleset)
|
||||
|
||||
app.Get("raw/*", handlers.Raw)
|
||||
app.Get("api/*", handlers.Api)
|
||||
|
||||
proxyOpts := &handlers.ProxyOptions{
|
||||
Verbose: *verbose,
|
||||
RulesetPath: *ruleset,
|
||||
}
|
||||
|
||||
app.Get("/*", handlers.NewProxySiteHandler(proxyOpts))
|
||||
app.Get("/*", handlers.ProxySite(*ruleset))
|
||||
log.Fatal(app.Listen(":" + *port))
|
||||
}
|
||||
|
||||
2
go.mod
2
go.mod
@@ -24,7 +24,7 @@ require (
|
||||
github.com/valyala/bytebufferpool v1.0.0 // indirect
|
||||
github.com/valyala/fasthttp v1.50.0 // indirect
|
||||
github.com/valyala/tcplisten v1.0.0 // indirect
|
||||
golang.org/x/net v0.18.0
|
||||
golang.org/x/net v0.18.0 // indirect
|
||||
golang.org/x/sys v0.14.0 // indirect
|
||||
golang.org/x/term v0.14.0
|
||||
)
|
||||
|
||||
@@ -11,9 +11,6 @@ import (
|
||||
"strings"
|
||||
|
||||
"ladder/pkg/ruleset"
|
||||
"ladder/proxychain"
|
||||
rx "ladder/proxychain/requestmodifers"
|
||||
tx "ladder/proxychain/responsemodifers"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/gofiber/fiber/v2"
|
||||
@@ -33,38 +30,89 @@ func init() {
|
||||
}
|
||||
}
|
||||
|
||||
type ProxyOptions struct {
|
||||
RulesetPath string
|
||||
Verbose bool
|
||||
// extracts a URL from the request ctx. If the URL in the request
|
||||
// is a relative path, it reconstructs the full URL using the referer header.
|
||||
func extractUrl(c *fiber.Ctx) (string, error) {
|
||||
// try to extract url-encoded
|
||||
reqUrl, err := url.QueryUnescape(c.Params("*"))
|
||||
if err != nil {
|
||||
// fallback
|
||||
reqUrl = c.Params("*")
|
||||
}
|
||||
|
||||
// Extract the actual path from req ctx
|
||||
urlQuery, err := url.Parse(reqUrl)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("error parsing request URL '%s': %v", reqUrl, err)
|
||||
}
|
||||
|
||||
isRelativePath := urlQuery.Scheme == ""
|
||||
|
||||
// eg: https://localhost:8080/images/foobar.jpg -> https://realsite.com/images/foobar.jpg
|
||||
if isRelativePath {
|
||||
// Parse the referer URL from the request header.
|
||||
refererUrl, err := url.Parse(c.Get("referer"))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("error parsing referer URL from req: '%s': %v", reqUrl, err)
|
||||
}
|
||||
|
||||
// Extract the real url from referer path
|
||||
realUrl, err := url.Parse(strings.TrimPrefix(refererUrl.Path, "/"))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("error parsing real URL from referer '%s': %v", refererUrl.Path, err)
|
||||
}
|
||||
|
||||
// reconstruct the full URL using the referer's scheme, host, and the relative path / queries
|
||||
fullUrl := &url.URL{
|
||||
Scheme: realUrl.Scheme,
|
||||
Host: realUrl.Host,
|
||||
Path: urlQuery.Path,
|
||||
RawQuery: urlQuery.RawQuery,
|
||||
}
|
||||
|
||||
if os.Getenv("LOG_URLS") == "true" {
|
||||
log.Printf("modified relative URL: '%s' -> '%s'", reqUrl, fullUrl.String())
|
||||
}
|
||||
return fullUrl.String(), nil
|
||||
|
||||
}
|
||||
|
||||
// default behavior:
|
||||
// eg: https://localhost:8080/https://realsite.com/images/foobar.jpg -> https://realsite.com/images/foobar.jpg
|
||||
return urlQuery.String(), nil
|
||||
|
||||
}
|
||||
|
||||
func NewProxySiteHandler(opts *ProxyOptions) fiber.Handler {
|
||||
/*
|
||||
var rs ruleset.RuleSet
|
||||
if opts.RulesetPath != "" {
|
||||
r, err := ruleset.NewRuleset(opts.RulesetPath)
|
||||
func ProxySite(rulesetPath string) fiber.Handler {
|
||||
if rulesetPath != "" {
|
||||
rs, err := ruleset.NewRuleset(rulesetPath)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
rs = r
|
||||
rulesSet = rs
|
||||
}
|
||||
*/
|
||||
|
||||
return func(c *fiber.Ctx) error {
|
||||
proxychain := proxychain.
|
||||
NewProxyChain().
|
||||
SetDebugLogging(opts.Verbose).
|
||||
SetRequestModifications(
|
||||
rx.DeleteOutgoingCookies(),
|
||||
//rx.RequestArchiveIs(),
|
||||
).
|
||||
AddResponseModifications(
|
||||
tx.DeleteIncomingCookies(),
|
||||
tx.RewriteHTMLResourceURLs(),
|
||||
)
|
||||
return proxychain.SetFiberCtx(c).Execute()
|
||||
// Get the url from the URL
|
||||
url, err := extractUrl(c)
|
||||
if err != nil {
|
||||
log.Println("ERROR In URL extraction:", err)
|
||||
}
|
||||
|
||||
queries := c.Queries()
|
||||
body, _, resp, err := fetchSite(url, queries)
|
||||
if err != nil {
|
||||
log.Println("ERROR:", err)
|
||||
c.SendStatus(fiber.StatusInternalServerError)
|
||||
return c.SendString(err.Error())
|
||||
}
|
||||
|
||||
c.Cookie(&fiber.Cookie{})
|
||||
c.Set("Content-Type", resp.Header.Get("Content-Type"))
|
||||
c.Set("Content-Security-Policy", resp.Header.Get("Content-Security-Policy"))
|
||||
|
||||
return c.SendString(body)
|
||||
}
|
||||
}
|
||||
|
||||
func modifyURL(uri string, rule ruleset.Rule) (string, error) {
|
||||
|
||||
@@ -14,7 +14,7 @@ import (
|
||||
|
||||
func TestProxySite(t *testing.T) {
|
||||
app := fiber.New()
|
||||
app.Get("/:url", NewProxySiteHandler(nil))
|
||||
app.Get("/:url", ProxySite(""))
|
||||
|
||||
req := httptest.NewRequest("GET", "/https://example.com", nil)
|
||||
resp, err := app.Test(req)
|
||||
|
||||
@@ -1,370 +0,0 @@
|
||||
package proxychain
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"ladder/pkg/ruleset"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
)
|
||||
|
||||
/*
|
||||
ProxyChain manages the process of forwarding an HTTP request to an upstream server,
|
||||
applying request and response modifications along the way.
|
||||
|
||||
- It accepts incoming HTTP requests (as a Fiber *ctx), and applies
|
||||
request modifiers (ReqMods) and response modifiers (ResMods) before passing the
|
||||
upstream response back to the client.
|
||||
|
||||
- ProxyChains can be reused to avoid memory allocations. However, they are not concurrent-safe
|
||||
so a ProxyChainPool should be used with mutexes to avoid memory errors.
|
||||
|
||||
---
|
||||
|
||||
# EXAMPLE
|
||||
|
||||
```
|
||||
|
||||
import (
|
||||
|
||||
rx "ladder/pkg/proxychain/requestmodifers"
|
||||
tx "ladder/pkg/proxychain/responsemodifers"
|
||||
"ladder/internal/proxychain"
|
||||
|
||||
)
|
||||
|
||||
proxychain.NewProxyChain().
|
||||
|
||||
SetFiberCtx(c).
|
||||
SetRequestModifications(
|
||||
rx.BlockOutgoingCookies(),
|
||||
rx.SpoofOrigin(),
|
||||
rx.SpoofReferrer(),
|
||||
).
|
||||
SetResultModifications(
|
||||
tx.BlockIncomingCookies(),
|
||||
tx.RewriteHTMLResourceURLs()
|
||||
).
|
||||
Execute()
|
||||
|
||||
```
|
||||
|
||||
client ladder service upstream
|
||||
|
||||
┌─────────┐ ┌────────────────────────┐ ┌─────────┐
|
||||
│ │GET │ │ │ │
|
||||
│ req────┼───► ProxyChain │ │ │
|
||||
│ │ │ │ │ │ │
|
||||
│ │ │ ▼ │ │ │
|
||||
│ │ │ apply │ │ │
|
||||
│ │ │ RequestModifications │ │ │
|
||||
│ │ │ │ │ │ │
|
||||
│ │ │ ▼ │ │ │
|
||||
│ │ │ send GET │ │ │
|
||||
│ │ │ Request req────────┼─► │ │
|
||||
│ │ │ │ │ │
|
||||
│ │ │ 200 OK │ │ │
|
||||
│ │ │ ┌────────────────┼─response │
|
||||
│ │ │ ▼ │ │ │
|
||||
│ │ │ apply │ │ │
|
||||
│ │ │ ResultModifications │ │ │
|
||||
│ │ │ │ │ │ │
|
||||
│ │◄───┼───────┘ │ │ │
|
||||
│ │ │ 200 OK │ │ │
|
||||
│ │ │ │ │ │
|
||||
└─────────┘ └────────────────────────┘ └─────────┘
|
||||
*/
|
||||
type ProxyChain struct {
|
||||
Context *fiber.Ctx
|
||||
Client *http.Client
|
||||
Request *http.Request
|
||||
Response *http.Response
|
||||
requestModifications []RequestModification
|
||||
resultModifications []ResponseModification
|
||||
Ruleset *ruleset.RuleSet
|
||||
debugMode bool
|
||||
abortErr error
|
||||
}
|
||||
|
||||
// a ProxyStrategy is a pre-built proxychain with purpose-built defaults
|
||||
type ProxyStrategy ProxyChain
|
||||
|
||||
// A RequestModification is a function that should operate on the
|
||||
// ProxyChain Req or Client field, using the fiber ctx as needed.
|
||||
type RequestModification func(*ProxyChain) error
|
||||
|
||||
// A ResponseModification is a function that should operate on the
|
||||
// ProxyChain Res (http result) & Body (buffered http response body) field
|
||||
type ResponseModification func(*ProxyChain) error
|
||||
|
||||
// SetRequestModifications sets the ProxyChain's request modifers
|
||||
// the modifier will not fire until ProxyChain.Execute() is run.
|
||||
func (chain *ProxyChain) SetRequestModifications(mods ...RequestModification) *ProxyChain {
|
||||
chain.requestModifications = mods
|
||||
return chain
|
||||
}
|
||||
|
||||
// AddRequestModifications sets the ProxyChain's request modifers
|
||||
// the modifier will not fire until ProxyChain.Execute() is run.
|
||||
func (chain *ProxyChain) AddRequestModifications(mods ...RequestModification) *ProxyChain {
|
||||
chain.requestModifications = append(chain.requestModifications, mods...)
|
||||
return chain
|
||||
}
|
||||
|
||||
// AddResponseModifications sets the ProxyChain's response modifers
|
||||
// the modifier will not fire until ProxyChain.Execute() is run.
|
||||
func (chain *ProxyChain) AddResponseModifications(mods ...ResponseModification) *ProxyChain {
|
||||
chain.resultModifications = mods
|
||||
return chain
|
||||
}
|
||||
|
||||
// Adds a ruleset to ProxyChain
|
||||
func (chain *ProxyChain) AddRuleset(rs *ruleset.RuleSet) *ProxyChain {
|
||||
chain.Ruleset = rs
|
||||
// TODO: add _applyRuleset method
|
||||
return chain
|
||||
}
|
||||
|
||||
func (chain *ProxyChain) _initialize_request() (*http.Request, error) {
|
||||
if chain.Context == nil {
|
||||
chain.abortErr = chain.abort(errors.New("no context set"))
|
||||
return nil, chain.abortErr
|
||||
}
|
||||
// initialize a request (without url)
|
||||
req, err := http.NewRequest(chain.Context.Method(), "", nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
chain.Request = req
|
||||
switch chain.Context.Method() {
|
||||
case "GET":
|
||||
case "DELETE":
|
||||
case "HEAD":
|
||||
case "OPTIONS":
|
||||
break
|
||||
case "POST":
|
||||
case "PUT":
|
||||
case "PATCH":
|
||||
// stream content of body from client request to upstream request
|
||||
chain.Request.Body = io.NopCloser(chain.Context.Request().BodyStream())
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported request method from client: '%s'", chain.Context.Method())
|
||||
}
|
||||
|
||||
/*
|
||||
// copy client request headers to upstream request headers
|
||||
forwardHeaders := func(key []byte, val []byte) {
|
||||
req.Header.Set(string(key), string(val))
|
||||
}
|
||||
clientHeaders := &chain.Context.Request().Header
|
||||
clientHeaders.VisitAll(forwardHeaders)
|
||||
*/
|
||||
|
||||
return req, nil
|
||||
}
|
||||
|
||||
// _execute sends the request for the ProxyChain and returns the raw body only
|
||||
// the caller is responsible for returning a response back to the requestor
|
||||
// the caller is also responsible for calling chain._reset() when they are done with the body
|
||||
func (chain *ProxyChain) _execute() (io.Reader, error) {
|
||||
if chain.validateCtxIsSet() != nil || chain.abortErr != nil {
|
||||
return nil, chain.abortErr
|
||||
}
|
||||
if chain.Request == nil {
|
||||
return nil, errors.New("proxychain request not yet initialized")
|
||||
}
|
||||
if chain.Request.URL.Scheme == "" {
|
||||
return nil, errors.New("request url not set or invalid. Check ProxyChain ReqMods for issues")
|
||||
}
|
||||
|
||||
// Apply requestModifications to proxychain
|
||||
for _, applyRequestModificationsTo := range chain.requestModifications {
|
||||
err := applyRequestModificationsTo(chain)
|
||||
if err != nil {
|
||||
return nil, chain.abort(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Send Request Upstream
|
||||
resp, err := chain.Client.Do(chain.Request)
|
||||
if err != nil {
|
||||
return nil, chain.abort(err)
|
||||
}
|
||||
chain.Response = resp
|
||||
|
||||
//defer resp.Body.Close()
|
||||
|
||||
/* todo: move to rsm
|
||||
for k, v := range resp.Header {
|
||||
chain.Context.Set(k, resp.Header.Get(k))
|
||||
}
|
||||
*/
|
||||
|
||||
// Apply ResponseModifiers to proxychain
|
||||
for _, applyResultModificationsTo := range chain.resultModifications {
|
||||
err := applyResultModificationsTo(chain)
|
||||
if err != nil {
|
||||
return nil, chain.abort(err)
|
||||
}
|
||||
}
|
||||
|
||||
return chain.Response.Body, nil
|
||||
}
|
||||
|
||||
// Execute sends the request for the ProxyChain and returns the request to the sender
|
||||
// and resets the fields so that the ProxyChain can be reused.
|
||||
// if any step in the ProxyChain fails, the request will abort and a 500 error will
|
||||
// be returned to the client
|
||||
func (chain *ProxyChain) Execute() error {
|
||||
defer chain._reset()
|
||||
body, err := chain._execute()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
return err
|
||||
}
|
||||
if chain.Context == nil {
|
||||
return errors.New("no context set")
|
||||
}
|
||||
// Return request back to client
|
||||
chain.Context.Set("content-type", chain.Response.Header.Get("content-type"))
|
||||
return chain.Context.SendStream(body)
|
||||
|
||||
//return chain.Context.SendStream(body)
|
||||
}
|
||||
|
||||
// reconstructUrlFromReferer reconstructs the URL using the referer's scheme, host, and the relative path / queries
|
||||
func reconstructUrlFromReferer(referer *url.URL, relativeUrl *url.URL) (*url.URL, error) {
|
||||
|
||||
// Extract the real url from referer path
|
||||
realUrl, err := url.Parse(strings.TrimPrefix(referer.Path, "/"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error parsing real URL from referer '%s': %v", referer.Path, err)
|
||||
}
|
||||
|
||||
if realUrl.Scheme == "" || realUrl.Host == "" {
|
||||
return nil, fmt.Errorf("invalid referer URL: '%s' on request '%s", referer, relativeUrl)
|
||||
}
|
||||
|
||||
log.Printf("'%s' -> '%s'\n", relativeUrl.String(), realUrl.String())
|
||||
|
||||
return &url.URL{
|
||||
Scheme: referer.Scheme,
|
||||
Host: referer.Host,
|
||||
Path: realUrl.Path,
|
||||
RawQuery: realUrl.RawQuery,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// extractUrl extracts a URL from the request ctx. If the URL in the request
|
||||
// is a relative path, it reconstructs the full URL using the referer header.
|
||||
func (chain *ProxyChain) extractUrl() (*url.URL, error) {
|
||||
// try to extract url-encoded
|
||||
reqUrl, err := url.QueryUnescape(chain.Context.Params("*"))
|
||||
if err != nil {
|
||||
reqUrl = chain.Context.Params("*") // fallback
|
||||
}
|
||||
|
||||
urlQuery, err := url.Parse(reqUrl)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error parsing request URL '%s': %v", reqUrl, err)
|
||||
}
|
||||
|
||||
// Handle standard paths
|
||||
// eg: https://localhost:8080/https://realsite.com/images/foobar.jpg -> https://realsite.com/images/foobar.jpg
|
||||
isRelativePath := urlQuery.Scheme == ""
|
||||
if !isRelativePath {
|
||||
return urlQuery, nil
|
||||
}
|
||||
|
||||
// Handle relative URLs
|
||||
// eg: https://localhost:8080/images/foobar.jpg -> https://realsite.com/images/foobar.jpg
|
||||
referer, err := url.Parse(chain.Context.Get("referer"))
|
||||
relativePath := urlQuery
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error parsing referer URL from req: '%s': %v", relativePath, err)
|
||||
}
|
||||
return reconstructUrlFromReferer(referer, relativePath)
|
||||
}
|
||||
|
||||
// SetFiberCtx takes the request ctx from the client
|
||||
// for the modifiers and execute function to use.
|
||||
// it must be set everytime a new request comes through
|
||||
// if the upstream request url cannot be extracted from the ctx,
|
||||
// a 500 error will be sent back to the client
|
||||
func (chain *ProxyChain) SetFiberCtx(ctx *fiber.Ctx) *ProxyChain {
|
||||
chain.Context = ctx
|
||||
|
||||
// initialize the request and prepare it for modification
|
||||
req, err := chain._initialize_request()
|
||||
if err != nil {
|
||||
chain.abortErr = chain.abort(err)
|
||||
}
|
||||
chain.Request = req
|
||||
|
||||
// extract the URL for the request and add it to the new request
|
||||
url, err := chain.extractUrl()
|
||||
if err != nil {
|
||||
chain.abortErr = chain.abort(err)
|
||||
}
|
||||
chain.Request.URL = url
|
||||
fmt.Printf("extracted URL: %s\n", chain.Request.URL)
|
||||
|
||||
return chain
|
||||
}
|
||||
|
||||
func (chain *ProxyChain) validateCtxIsSet() error {
|
||||
if chain.Context != nil {
|
||||
return nil
|
||||
}
|
||||
err := errors.New("proxyChain was called without setting a fiber Ctx. Use ProxyChain.SetCtx()")
|
||||
chain.abortErr = chain.abort(err)
|
||||
return chain.abortErr
|
||||
}
|
||||
|
||||
// SetHttpClient sets a new upstream http client transport
|
||||
// useful for modifying TLS
|
||||
func (chain *ProxyChain) SetHttpClient(httpClient *http.Client) *ProxyChain {
|
||||
chain.Client = httpClient
|
||||
return chain
|
||||
}
|
||||
|
||||
// SetVerbose changes the logging behavior to print
|
||||
// the modification steps and applied rulesets for debugging
|
||||
func (chain *ProxyChain) SetDebugLogging(isDebugMode bool) *ProxyChain {
|
||||
chain.debugMode = isDebugMode
|
||||
return chain
|
||||
}
|
||||
|
||||
// abort proxychain and return 500 error to client
|
||||
// this will prevent Execute from firing and reset the state
|
||||
// returns the initial error enriched with context
|
||||
func (chain *ProxyChain) abort(err error) error {
|
||||
//defer chain._reset()
|
||||
chain.abortErr = err
|
||||
chain.Context.Response().SetStatusCode(500)
|
||||
e := fmt.Errorf("ProxyChain error for '%s': %s", chain.Request.URL.String(), err.Error())
|
||||
chain.Context.SendString(e.Error())
|
||||
log.Println(e.Error())
|
||||
return e
|
||||
}
|
||||
|
||||
// internal function to reset state of ProxyChain for reuse
|
||||
func (chain *ProxyChain) _reset() {
|
||||
chain.abortErr = nil
|
||||
chain.Request = nil
|
||||
//chain.Response = nil
|
||||
chain.Context = nil
|
||||
}
|
||||
|
||||
// NewProxyChain initializes a new ProxyChain
|
||||
func NewProxyChain() *ProxyChain {
|
||||
chain := new(ProxyChain)
|
||||
chain.Client = http.DefaultClient
|
||||
return chain
|
||||
}
|
||||
@@ -1,11 +0,0 @@
|
||||
package proxychain
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
)
|
||||
|
||||
type ProxyChainPool map[url.URL]ProxyChain
|
||||
|
||||
func NewProxyChainPool() ProxyChainPool {
|
||||
return map[url.URL]ProxyChain{}
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
package requestmodifers
|
||||
|
||||
import (
|
||||
"ladder/proxychain"
|
||||
)
|
||||
|
||||
// MasqueradeAsGoogleBot modifies user agent and x-forwarded for
|
||||
// to appear to be a Google Bot
|
||||
func MasqueradeAsGoogleBot() proxychain.RequestModification {
|
||||
const botUA string = "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; http://www.google.com/bot.html) Chrome/79.0.3945.120 Safari/537.36"
|
||||
const botIP string = "66.249.78.8" // TODO: create a random ip pool from https://developers.google.com/static/search/apis/ipranges/googlebot.json
|
||||
return masqueradeAsTrustedBot(botUA, botIP)
|
||||
}
|
||||
|
||||
// MasqueradeAsBingBot modifies user agent and x-forwarded for
|
||||
// to appear to be a Bing Bot
|
||||
func MasqueradeAsBingBot() proxychain.RequestModification {
|
||||
const botUA string = "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/79.0.3945.120 Safari/537.36"
|
||||
const botIP string = "13.66.144.9" // https://www.bing.com/toolbox/bingbot.json
|
||||
return masqueradeAsTrustedBot(botUA, botIP)
|
||||
}
|
||||
|
||||
func masqueradeAsTrustedBot(botUA string, botIP string) proxychain.RequestModification {
|
||||
return func(px *proxychain.ProxyChain) error {
|
||||
px.AddRequestModifications(
|
||||
SpoofUserAgent(botUA),
|
||||
SpoofXForwardedFor(botIP),
|
||||
SpoofReferrer(""),
|
||||
SpoofOrigin(""),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
package requestmodifers
|
||||
|
||||
import (
|
||||
"ladder/proxychain"
|
||||
"regexp"
|
||||
)
|
||||
|
||||
func ModifyDomainWithRegex(match regexp.Regexp, replacement string) proxychain.RequestModification {
|
||||
return func(px *proxychain.ProxyChain) error {
|
||||
px.Request.URL.Host = match.ReplaceAllString(px.Request.URL.Host, replacement)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -1,97 +0,0 @@
|
||||
package requestmodifers
|
||||
|
||||
import (
|
||||
"ladder/proxychain"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// SetOutgoingCookie modifes a specific cookie name
|
||||
// by modifying the request cookie headers going to the upstream server.
|
||||
// If the cookie name does not already exist, it is created.
|
||||
func SetOutgoingCookie(name string, val string) proxychain.RequestModification {
|
||||
return func(chain *proxychain.ProxyChain) error {
|
||||
cookies := chain.Request.Cookies()
|
||||
hasCookie := false
|
||||
for _, cookie := range cookies {
|
||||
if cookie.Name != name {
|
||||
continue
|
||||
}
|
||||
hasCookie = true
|
||||
cookie.Value = val
|
||||
}
|
||||
|
||||
if hasCookie {
|
||||
return nil
|
||||
}
|
||||
|
||||
chain.Request.AddCookie(&http.Cookie{
|
||||
Domain: chain.Request.URL.Host,
|
||||
Name: name,
|
||||
Value: val,
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// SetOutgoingCookies modifies a client request's cookie header
|
||||
// to a raw Cookie string, overwriting existing cookies
|
||||
func SetOutgoingCookies(cookies string) proxychain.RequestModification {
|
||||
return func(chain *proxychain.ProxyChain) error {
|
||||
chain.Request.Header.Set("Cookies", cookies)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// DeleteOutgoingCookie modifies the http request's cookies header to
|
||||
// delete a specific request cookie going to the upstream server.
|
||||
// If the cookie does not exist, it does not do anything.
|
||||
func DeleteOutgoingCookie(name string) proxychain.RequestModification {
|
||||
return func(chain *proxychain.ProxyChain) error {
|
||||
cookies := chain.Request.Cookies()
|
||||
chain.Request.Header.Del("Cookies")
|
||||
|
||||
for _, cookie := range cookies {
|
||||
if cookie.Name == name {
|
||||
chain.Request.AddCookie(cookie)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// DeleteOutgoingCookies removes the cookie header entirely,
|
||||
// preventing any cookies from reaching the upstream server.
|
||||
func DeleteOutgoingCookies() proxychain.RequestModification {
|
||||
return func(px *proxychain.ProxyChain) error {
|
||||
px.Request.Header.Del("Cookie")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// DeleteOutGoingCookiesExcept prevents non-whitelisted cookies from being sent from the client
|
||||
// to the upstream proxy server. Cookies whose names are in the whitelist are not removed.
|
||||
func DeleteOutgoingCookiesExcept(whitelist ...string) proxychain.RequestModification {
|
||||
return func(px *proxychain.ProxyChain) error {
|
||||
// Convert whitelist slice to a map for efficient lookups
|
||||
whitelistMap := make(map[string]struct{})
|
||||
for _, cookieName := range whitelist {
|
||||
whitelistMap[cookieName] = struct{}{}
|
||||
}
|
||||
|
||||
// Get all cookies from the request header
|
||||
cookies := px.Request.Cookies()
|
||||
|
||||
// Clear the original Cookie header
|
||||
px.Request.Header.Del("Cookie")
|
||||
|
||||
// Re-add cookies that are in the whitelist
|
||||
for _, cookie := range cookies {
|
||||
if _, found := whitelistMap[cookie.Name]; found {
|
||||
px.Request.AddCookie(cookie)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
package requestmodifers
|
||||
|
||||
import (
|
||||
"ladder/proxychain"
|
||||
"regexp"
|
||||
)
|
||||
|
||||
func ModifyPathWithRegex(match regexp.Regexp, replacement string) proxychain.RequestModification {
|
||||
return func(px *proxychain.ProxyChain) error {
|
||||
px.Request.URL.Path = match.ReplaceAllString(px.Request.URL.Path, replacement)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
package requestmodifers
|
||||
|
||||
import (
|
||||
"ladder/proxychain"
|
||||
)
|
||||
|
||||
// ModifyQueryParams replaces query parameter values in URL's query params in a ProxyChain's URL.
|
||||
// If the query param key doesn't exist, it is created.
|
||||
func ModifyQueryParams(key string, value string) proxychain.RequestModification {
|
||||
return func(px *proxychain.ProxyChain) error {
|
||||
q := px.Request.URL.Query()
|
||||
if value == "" {
|
||||
q.Del(key)
|
||||
return nil
|
||||
}
|
||||
q.Set(key, value)
|
||||
px.Request.URL.RawQuery = q.Encode()
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -1,23 +0,0 @@
|
||||
package requestmodifers
|
||||
|
||||
import (
|
||||
"ladder/proxychain"
|
||||
)
|
||||
|
||||
// SetRequestHeader modifies a specific outgoing header
|
||||
// This is the header that the upstream server will see.
|
||||
func SetRequestHeader(name string, val string) proxychain.RequestModification {
|
||||
return func(px *proxychain.ProxyChain) error {
|
||||
px.Request.Header.Set(name, val)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// DeleteRequestHeader modifies a specific outgoing header
|
||||
// This is the header that the upstream server will see.
|
||||
func DeleteRequestHeader(name string) proxychain.RequestModification {
|
||||
return func(px *proxychain.ProxyChain) error {
|
||||
px.Request.Header.Del(name)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -1,27 +0,0 @@
|
||||
package requestmodifers
|
||||
|
||||
import (
|
||||
"ladder/proxychain"
|
||||
"net/url"
|
||||
)
|
||||
|
||||
const archivistUrl string = "https://archive.is/latest/"
|
||||
|
||||
// RequestArchiveIs modifies a ProxyChain's URL to request an archived version from archive.is
|
||||
func RequestArchiveIs() proxychain.RequestModification {
|
||||
return func(px *proxychain.ProxyChain) error {
|
||||
px.Request.URL.RawQuery = ""
|
||||
newURLString := archivistUrl + px.Request.URL.String()
|
||||
newURL, err := url.Parse(newURLString)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// archivist seems to sabotage requests from cloudflare's DNS
|
||||
// bypass this just in case
|
||||
px.AddRequestModifications(ResolveWithGoogleDoH())
|
||||
|
||||
px.Request.URL = newURL
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
package requestmodifers
|
||||
|
||||
import (
|
||||
"ladder/proxychain"
|
||||
"net/url"
|
||||
)
|
||||
|
||||
const googleCacheUrl string = "https://webcache.googleusercontent.com/search?q=cache:"
|
||||
|
||||
// RequestGoogleCache modifies a ProxyChain's URL to request its Google Cache version.
|
||||
func RequestGoogleCache() proxychain.RequestModification {
|
||||
return func(px *proxychain.ProxyChain) error {
|
||||
encodedURL := url.QueryEscape(px.Request.URL.String())
|
||||
newURL, err := url.Parse(googleCacheUrl + encodedURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
px.Request.URL = newURL
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
package requestmodifers
|
||||
|
||||
import (
|
||||
"ladder/proxychain"
|
||||
"net/url"
|
||||
)
|
||||
|
||||
const waybackUrl string = "https://web.archive.org/web/"
|
||||
|
||||
// RequestWaybackMachine modifies a ProxyChain's URL to request the wayback machine (archive.org) version.
|
||||
func RequestWaybackMachine() proxychain.RequestModification {
|
||||
return func(px *proxychain.ProxyChain) error {
|
||||
px.Request.URL.RawQuery = ""
|
||||
newURLString := waybackUrl + px.Request.URL.String()
|
||||
newURL, err := url.Parse(newURLString)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
px.Request.URL = newURL
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -1,80 +0,0 @@
|
||||
package requestmodifers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"ladder/proxychain"
|
||||
"net"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
// resolveWithGoogleDoH resolves DNS using Google's DNS-over-HTTPS
|
||||
func resolveWithGoogleDoH(host string) (string, error) {
|
||||
url := "https://dns.google/resolve?name=" + host + "&type=A"
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
var result struct {
|
||||
Answer []struct {
|
||||
Data string `json:"data"`
|
||||
} `json:"Answer"`
|
||||
}
|
||||
err = json.NewDecoder(resp.Body).Decode(&result)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Get the first A record
|
||||
if len(result.Answer) > 0 {
|
||||
return result.Answer[0].Data, nil
|
||||
}
|
||||
return "", fmt.Errorf("no DoH DNS record found for %s", host)
|
||||
}
|
||||
|
||||
// ResolveWithGoogleDoH modifies a ProxyChain's client to make the request but resolve the URL
|
||||
// using Google's DNS over HTTPs service
|
||||
func ResolveWithGoogleDoH() proxychain.RequestModification {
|
||||
return func(px *proxychain.ProxyChain) error {
|
||||
client := &http.Client{
|
||||
Timeout: px.Client.Timeout,
|
||||
}
|
||||
|
||||
dialer := &net.Dialer{
|
||||
Timeout: 5 * time.Second,
|
||||
KeepAlive: 5 * time.Second,
|
||||
}
|
||||
|
||||
customDialContext := func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
host, port, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
// If the addr doesn't include a port, determine it based on the URL scheme
|
||||
if px.Request.URL.Scheme == "https" {
|
||||
port = "443"
|
||||
} else {
|
||||
port = "80"
|
||||
}
|
||||
host = addr // assume the entire addr is the host
|
||||
}
|
||||
|
||||
resolvedHost, err := resolveWithGoogleDoH(host)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return dialer.DialContext(ctx, network, net.JoinHostPort(resolvedHost, port))
|
||||
}
|
||||
|
||||
patchedTransportWithDoH := &http.Transport{
|
||||
DialContext: customDialContext,
|
||||
}
|
||||
|
||||
client.Transport = patchedTransportWithDoH
|
||||
px.Client = client // Assign the modified client to the ProxyChain
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
package requestmodifers
|
||||
|
||||
import (
|
||||
"ladder/proxychain"
|
||||
)
|
||||
|
||||
// SpoofOrigin modifies the origin header
|
||||
// if the upstream server returns a Vary header
|
||||
// it means you might get a different response if you change this
|
||||
func SpoofOrigin(url string) proxychain.RequestModification {
|
||||
return func(px *proxychain.ProxyChain) error {
|
||||
px.Request.Header.Set("origin", url)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// HideOrigin modifies the origin header
|
||||
// so that it is the original origin, not the proxy
|
||||
func HideOrigin() proxychain.RequestModification {
|
||||
return func(px *proxychain.ProxyChain) error {
|
||||
px.Request.Header.Set("origin", px.Request.URL.String())
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
package requestmodifers
|
||||
|
||||
import (
|
||||
"ladder/proxychain"
|
||||
)
|
||||
|
||||
// SpoofReferrer modifies the referrer header
|
||||
// useful if the page can be accessed from a search engine
|
||||
// or social media site, but not by browsing the website itself
|
||||
// if url is "", then the referrer header is removed
|
||||
func SpoofReferrer(url string) proxychain.RequestModification {
|
||||
return func(px *proxychain.ProxyChain) error {
|
||||
if url == "" {
|
||||
px.Request.Header.Del("referrer")
|
||||
return nil
|
||||
}
|
||||
px.Request.Header.Set("referrer", url)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// HideReferrer modifies the referrer header
|
||||
// so that it is the original referrer, not the proxy
|
||||
func HideReferrer() proxychain.RequestModification {
|
||||
return func(px *proxychain.ProxyChain) error {
|
||||
px.Request.Header.Set("referrer", px.Request.URL.String())
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
package requestmodifers
|
||||
|
||||
import (
|
||||
"ladder/proxychain"
|
||||
)
|
||||
|
||||
// SpoofUserAgent modifies the user agent
|
||||
func SpoofUserAgent(ua string) proxychain.RequestModification {
|
||||
return func(px *proxychain.ProxyChain) error {
|
||||
px.Request.Header.Set("user-agent", ua)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
package requestmodifers
|
||||
|
||||
import (
|
||||
"ladder/proxychain"
|
||||
)
|
||||
|
||||
// SpoofXForwardedFor modifies the X-Forwarded-For header
|
||||
// in some cases, a forward proxy may interpret this as the source IP
|
||||
func SpoofXForwardedFor(ip string) proxychain.RequestModification {
|
||||
return func(px *proxychain.ProxyChain) error {
|
||||
px.Request.Header.Set("X-FORWARDED-FOR", ip)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
package responsemodifers
|
||||
|
||||
import (
|
||||
"ladder/proxychain"
|
||||
)
|
||||
|
||||
// BypassCORS modifies response headers to prevent the browser
|
||||
// from enforcing any CORS restrictions. This should run at the end of the chain.
|
||||
func BypassCORS() proxychain.ResponseModification {
|
||||
return func(chain *proxychain.ProxyChain) error {
|
||||
chain.AddResponseModifications(
|
||||
SetResponseHeader("Access-Control-Allow-Origin", "*"),
|
||||
SetResponseHeader("Access-Control-Expose-Headers", "*"),
|
||||
SetResponseHeader("Access-Control-Allow-Credentials", "true"),
|
||||
SetResponseHeader("Access-Control-Allow-Methods", "GET, PUT, POST, DELETE, HEAD, OPTIONS, PATCH"),
|
||||
SetResponseHeader("Access-Control-Allow-Headers", "*"),
|
||||
DeleteResponseHeader("X-Frame-Options"),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -1,27 +0,0 @@
|
||||
package responsemodifers
|
||||
|
||||
import (
|
||||
"ladder/proxychain"
|
||||
)
|
||||
|
||||
// BypassContentSecurityPolicy modifies response headers to prevent the browser
|
||||
// from enforcing any CSP restrictions. This should run at the end of the chain.
|
||||
func BypassContentSecurityPolicy() proxychain.ResponseModification {
|
||||
return func(chain *proxychain.ProxyChain) error {
|
||||
chain.AddResponseModifications(
|
||||
DeleteResponseHeader("Content-Security-Policy"),
|
||||
DeleteResponseHeader("Content-Security-Policy-Report-Only"),
|
||||
DeleteResponseHeader("X-Content-Security-Policy"),
|
||||
DeleteResponseHeader("X-WebKit-CSP"),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// SetContentSecurityPolicy modifies response headers to a specific CSP
|
||||
func SetContentSecurityPolicy(csp string) proxychain.ResponseModification {
|
||||
return func(chain *proxychain.ProxyChain) error {
|
||||
chain.Response.Header.Set("Content-Security-Policy", csp)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -1,102 +0,0 @@
|
||||
package responsemodifers
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"ladder/proxychain"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// DeleteIncomingCookies prevents ALL cookies from being sent from the proxy server
|
||||
// back down to the client.
|
||||
func DeleteIncomingCookies(whitelist ...string) proxychain.ResponseModification {
|
||||
return func(px *proxychain.ProxyChain) error {
|
||||
px.Response.Header.Del("Set-Cookie")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// DeleteIncomingCookiesExcept prevents non-whitelisted cookies from being sent from the proxy server
|
||||
// to the client. Cookies whose names are in the whitelist are not removed.
|
||||
func DeleteIncomingCookiesExcept(whitelist ...string) proxychain.ResponseModification {
|
||||
return func(px *proxychain.ProxyChain) error {
|
||||
// Convert whitelist slice to a map for efficient lookups
|
||||
whitelistMap := make(map[string]struct{})
|
||||
for _, cookieName := range whitelist {
|
||||
whitelistMap[cookieName] = struct{}{}
|
||||
}
|
||||
|
||||
// If the response has no cookies, return early
|
||||
if px.Response.Header == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Filter the cookies in the response
|
||||
filteredCookies := []string{}
|
||||
for _, cookieStr := range px.Response.Header["Set-Cookie"] {
|
||||
cookie := parseCookie(cookieStr)
|
||||
if _, found := whitelistMap[cookie.Name]; found {
|
||||
filteredCookies = append(filteredCookies, cookieStr)
|
||||
}
|
||||
}
|
||||
|
||||
// Update the Set-Cookie header with the filtered cookies
|
||||
if len(filteredCookies) > 0 {
|
||||
px.Response.Header["Set-Cookie"] = filteredCookies
|
||||
} else {
|
||||
px.Response.Header.Del("Set-Cookie")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// parseCookie parses a cookie string and returns an http.Cookie object.
|
||||
func parseCookie(cookieStr string) *http.Cookie {
|
||||
header := http.Header{}
|
||||
header.Add("Set-Cookie", cookieStr)
|
||||
request := http.Request{Header: header}
|
||||
return request.Cookies()[0]
|
||||
}
|
||||
|
||||
// SetIncomingCookies adds a raw cookie string being sent from the proxy server down to the client
|
||||
func SetIncomingCookies(cookies string) proxychain.ResponseModification {
|
||||
return func(px *proxychain.ProxyChain) error {
|
||||
px.Response.Header.Set("Set-Cookie", cookies)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// SetIncomingCookie modifies a specific cookie in the response from the proxy server to the client.
|
||||
func SetIncomingCookie(name string, val string) proxychain.ResponseModification {
|
||||
return func(px *proxychain.ProxyChain) error {
|
||||
if px.Response.Header == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
updatedCookies := []string{}
|
||||
found := false
|
||||
|
||||
// Iterate over existing cookies and modify the one that matches the cookieName
|
||||
for _, cookieStr := range px.Response.Header["Set-Cookie"] {
|
||||
cookie := parseCookie(cookieStr)
|
||||
if cookie.Name == name {
|
||||
// Replace the cookie with the new value
|
||||
updatedCookies = append(updatedCookies, fmt.Sprintf("%s=%s", name, val))
|
||||
found = true
|
||||
} else {
|
||||
// Keep the cookie as is
|
||||
updatedCookies = append(updatedCookies, cookieStr)
|
||||
}
|
||||
}
|
||||
|
||||
// If the specified cookie wasn't found, add it
|
||||
if !found {
|
||||
updatedCookies = append(updatedCookies, fmt.Sprintf("%s=%s", name, val))
|
||||
}
|
||||
|
||||
// Update the Set-Cookie header
|
||||
px.Response.Header["Set-Cookie"] = updatedCookies
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
package responsemodifers
|
||||
|
||||
import (
|
||||
"ladder/proxychain"
|
||||
)
|
||||
|
||||
// SetResponseHeader modifies response headers from the upstream server
|
||||
func SetResponseHeader(key string, value string) proxychain.ResponseModification {
|
||||
return func(px *proxychain.ProxyChain) error {
|
||||
px.Context.Response().Header.Set(key, value)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// DeleteResponseHeader removes response headers from the upstream server
|
||||
func DeleteResponseHeader(key string) proxychain.ResponseModification {
|
||||
return func(px *proxychain.ProxyChain) error {
|
||||
px.Context.Response().Header.Del(key)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -1,62 +0,0 @@
|
||||
// Overrides the global fetch and XMLHttpRequest open methods to modify the request URLs.
|
||||
// Also overrides the attribute setter prototype to modify the request URLs
|
||||
// fetch("/relative_script.js") -> fetch("http://localhost:8080/relative_script.js")
|
||||
(() => {
|
||||
function rewriteURL(url) {
|
||||
if (!url) return url
|
||||
if (url.startsWith(window.location.origin)) return url
|
||||
|
||||
if (url.startsWith("//")) {
|
||||
url = `${window.location.origin}/${encodeURIComponent(url.substring(2))}`;
|
||||
} else if (url.startsWith("/")) {
|
||||
url = `${window.location.origin}/${encodeURIComponent(url.substring(1))}`;
|
||||
} else if (url.startsWith("http://") || url.startsWith("https://")) {
|
||||
url = `${window.location.origin}/${encodeURIComponent(url)}`;
|
||||
}
|
||||
return url;
|
||||
};
|
||||
|
||||
// monkey patch fetch
|
||||
const oldFetch = globalThis.fetch ;
|
||||
globalThis.fetch = async (url, init) => {
|
||||
return oldFetch(rewriteURL(url), init)
|
||||
}
|
||||
|
||||
// monkey patch xmlhttprequest
|
||||
const oldOpen = XMLHttpRequest.prototype.open;
|
||||
XMLHttpRequest.prototype.open = function(method, url, async = true, user = null, password = null) {
|
||||
return oldOpen.call(this, method, rewriteURL(url), async, user, password);
|
||||
};
|
||||
|
||||
|
||||
// Monkey patch setter methods
|
||||
const elements = [
|
||||
{ tag: 'a', attribute: 'href' },
|
||||
{ tag: 'img', attribute: 'src' },
|
||||
{ tag: 'script', attribute: 'src' },
|
||||
{ tag: 'link', attribute: 'href' },
|
||||
{ tag: 'iframe', attribute: 'src' },
|
||||
{ tag: 'audio', attribute: 'src' },
|
||||
{ tag: 'video', attribute: 'src' },
|
||||
{ tag: 'source', attribute: 'src' },
|
||||
{ tag: 'embed', attribute: 'src' },
|
||||
{ tag: 'object', attribute: 'src' },
|
||||
{ tag: 'input', attribute: 'src' },
|
||||
{ tag: 'track', attribute: 'src' },
|
||||
{ tag: 'form', attribute: 'action' },
|
||||
];
|
||||
|
||||
elements.forEach(({ tag, attribute }) => {
|
||||
const proto = document.createElement(tag).constructor.prototype;
|
||||
const descriptor = Object.getOwnPropertyDescriptor(proto, attribute);
|
||||
if (descriptor && descriptor.set) {
|
||||
Object.defineProperty(proto, attribute, {
|
||||
...descriptor,
|
||||
set(value) {
|
||||
return descriptor.set.call(this, rewriteURL(value));
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
})();
|
||||
@@ -1,262 +0,0 @@
|
||||
package responsemodifers
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"ladder/proxychain"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
// Define list of HTML attributes to try to rewrite
|
||||
var AttributesToRewrite map[string]bool
|
||||
|
||||
func init() {
|
||||
AttributesToRewrite = map[string]bool{
|
||||
"src": true,
|
||||
"href": true,
|
||||
"action": true,
|
||||
"srcset": true, // TODO: fix
|
||||
"poster": true,
|
||||
"data": true,
|
||||
"cite": true,
|
||||
"formaction": true,
|
||||
"background": true,
|
||||
"usemap": true,
|
||||
"longdesc": true,
|
||||
"manifest": true,
|
||||
"archive": true,
|
||||
"codebase": true,
|
||||
"icon": true,
|
||||
"pluginspage": true,
|
||||
}
|
||||
}
|
||||
|
||||
// HTMLResourceURLRewriter is a struct that rewrites URLs within HTML resources to use a specified proxy URL.
|
||||
// It uses an HTML tokenizer to process HTML content and rewrites URLs in src/href attributes.
|
||||
// <img src='/relative_path'> -> <img src='/https://proxiedsite.com/relative_path'>
|
||||
type HTMLResourceURLRewriter struct {
|
||||
baseURL *url.URL
|
||||
tokenizer *html.Tokenizer
|
||||
currentToken html.Token
|
||||
tokenBuffer *bytes.Buffer
|
||||
currentTokenIndex int
|
||||
currentTokenProcessed bool
|
||||
}
|
||||
|
||||
// NewHTMLResourceURLRewriter creates a new instance of HTMLResourceURLRewriter.
|
||||
// It initializes the tokenizer with the provided source and sets the proxy URL.
|
||||
func NewHTMLResourceURLRewriter(src io.ReadCloser, baseURL *url.URL) *HTMLResourceURLRewriter {
|
||||
return &HTMLResourceURLRewriter{
|
||||
tokenizer: html.NewTokenizer(src),
|
||||
currentToken: html.Token{},
|
||||
currentTokenIndex: 0,
|
||||
tokenBuffer: new(bytes.Buffer),
|
||||
baseURL: baseURL,
|
||||
}
|
||||
}
|
||||
|
||||
// Close resets the internal state of HTMLResourceURLRewriter, clearing buffers and token data.
|
||||
func (r *HTMLResourceURLRewriter) Close() error {
|
||||
r.tokenBuffer.Reset()
|
||||
r.currentToken = html.Token{}
|
||||
r.currentTokenIndex = 0
|
||||
r.currentTokenProcessed = false
|
||||
return nil
|
||||
}
|
||||
|
||||
// Read processes the HTML content, rewriting URLs and managing the state of tokens.
|
||||
// It reads HTML content, token by token, rewriting URLs to route through the specified proxy.
|
||||
func (r *HTMLResourceURLRewriter) Read(p []byte) (int, error) {
|
||||
|
||||
if r.currentToken.Data == "" || r.currentTokenProcessed {
|
||||
tokenType := r.tokenizer.Next()
|
||||
|
||||
// done reading html, close out reader
|
||||
if tokenType == html.ErrorToken {
|
||||
if r.tokenizer.Err() == io.EOF {
|
||||
return 0, io.EOF
|
||||
}
|
||||
return 0, r.tokenizer.Err()
|
||||
}
|
||||
|
||||
// flush the current token into an internal buffer
|
||||
// to handle fragmented tokens
|
||||
r.currentToken = r.tokenizer.Token()
|
||||
|
||||
// patch tokens with URLs
|
||||
isTokenWithAttribute := r.currentToken.Type == html.StartTagToken || r.currentToken.Type == html.SelfClosingTagToken
|
||||
if isTokenWithAttribute {
|
||||
patchResourceURL(&r.currentToken, r.baseURL)
|
||||
}
|
||||
|
||||
r.tokenBuffer.Reset()
|
||||
r.tokenBuffer.WriteString(html.UnescapeString(r.currentToken.String()))
|
||||
r.currentTokenProcessed = false
|
||||
r.currentTokenIndex = 0
|
||||
}
|
||||
|
||||
n, err := r.tokenBuffer.Read(p)
|
||||
if err == io.EOF || r.tokenBuffer.Len() == 0 {
|
||||
r.currentTokenProcessed = true
|
||||
err = nil // EOF in this context is expected and not an actual error
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Root-relative URLs: These are relative to the root path and start with a "/".
|
||||
func handleRootRelativePath(attr *html.Attribute, baseURL *url.URL) {
|
||||
// doublecheck this is a valid relative URL
|
||||
_, err := url.Parse(fmt.Sprintf("http://localhost.com%s", attr.Val))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
//log.Printf("BASEURL patch: %s\n", baseURL)
|
||||
|
||||
attr.Val = fmt.Sprintf(
|
||||
"/%s://%s/%s",
|
||||
baseURL.Scheme,
|
||||
baseURL.Host,
|
||||
strings.TrimPrefix(attr.Val, "/"),
|
||||
)
|
||||
attr.Val = url.QueryEscape(attr.Val)
|
||||
attr.Val = fmt.Sprintf("/%s", attr.Val)
|
||||
|
||||
//log.Printf("root rel url rewritten-> '%s'='%s'", attr.Key, attr.Val)
|
||||
}
|
||||
|
||||
// Document-relative URLs: These are relative to the current document's path and don't start with a "/".
|
||||
func handleDocumentRelativePath(attr *html.Attribute, baseURL *url.URL) {
|
||||
attr.Val = fmt.Sprintf(
|
||||
"%s://%s/%s%s",
|
||||
baseURL.Scheme,
|
||||
strings.Trim(baseURL.Host, "/"),
|
||||
strings.Trim(baseURL.RawPath, "/"),
|
||||
strings.Trim(attr.Val, "/"),
|
||||
)
|
||||
attr.Val = url.QueryEscape(attr.Val)
|
||||
attr.Val = fmt.Sprintf("/%s", attr.Val)
|
||||
//log.Printf("doc rel url rewritten-> '%s'='%s'", attr.Key, attr.Val)
|
||||
}
|
||||
|
||||
// Protocol-relative URLs: These start with "//" and will use the same protocol (http or https) as the current page.
|
||||
func handleProtocolRelativePath(attr *html.Attribute, baseURL *url.URL) {
|
||||
attr.Val = strings.TrimPrefix(attr.Val, "/")
|
||||
handleRootRelativePath(attr, baseURL)
|
||||
//log.Printf("proto rel url rewritten-> '%s'='%s'", attr.Key, attr.Val)
|
||||
}
|
||||
|
||||
func handleAbsolutePath(attr *html.Attribute, baseURL *url.URL) {
|
||||
// check if valid URL
|
||||
u, err := url.Parse(attr.Val)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if !(u.Scheme == "http" || u.Scheme == "https") {
|
||||
return
|
||||
}
|
||||
attr.Val = fmt.Sprintf(
|
||||
"/%s",
|
||||
url.QueryEscape(
|
||||
strings.TrimPrefix(attr.Val, "/"),
|
||||
),
|
||||
)
|
||||
//log.Printf("abs url rewritten-> '%s'='%s'", attr.Key, attr.Val)
|
||||
}
|
||||
|
||||
func handleSrcSet(attr *html.Attribute, baseURL *url.URL) {
|
||||
for i, src := range strings.Split(attr.Val, ",") {
|
||||
src = strings.Trim(src, " ")
|
||||
for j, s := range strings.Split(src, " ") {
|
||||
s = strings.Trim(s, " ")
|
||||
if j == 0 {
|
||||
f := &html.Attribute{Val: s, Key: attr.Key}
|
||||
switch {
|
||||
case strings.HasPrefix(s, "//"):
|
||||
handleProtocolRelativePath(f, baseURL)
|
||||
case strings.HasPrefix(s, "/"):
|
||||
handleRootRelativePath(f, baseURL)
|
||||
case strings.HasPrefix(s, "https://") || strings.HasPrefix(s, "http://"):
|
||||
handleAbsolutePath(f, baseURL)
|
||||
default:
|
||||
handleDocumentRelativePath(f, baseURL)
|
||||
}
|
||||
s = f.Val
|
||||
}
|
||||
if i == 0 && j == 0 {
|
||||
attr.Val = s
|
||||
continue
|
||||
}
|
||||
attr.Val = fmt.Sprintf("%s %s", attr.Val, s)
|
||||
}
|
||||
attr.Val = fmt.Sprintf("%s,", attr.Val)
|
||||
}
|
||||
attr.Val = strings.TrimSuffix(attr.Val, ",")
|
||||
}
|
||||
|
||||
// TODO: figure out how to handle these
|
||||
// srcset
|
||||
func patchResourceURL(token *html.Token, baseURL *url.URL) {
|
||||
for i := range token.Attr {
|
||||
attr := &token.Attr[i]
|
||||
|
||||
switch {
|
||||
// dont touch attributes except for the ones we defined
|
||||
case !AttributesToRewrite[attr.Key]:
|
||||
continue
|
||||
case attr.Key == "srcset":
|
||||
handleSrcSet(attr, baseURL)
|
||||
continue
|
||||
case strings.HasPrefix(attr.Val, "//"):
|
||||
handleProtocolRelativePath(attr, baseURL)
|
||||
continue
|
||||
case strings.HasPrefix(attr.Val, "/"):
|
||||
handleRootRelativePath(attr, baseURL)
|
||||
continue
|
||||
case strings.HasPrefix(attr.Val, "https://") || strings.HasPrefix(attr.Val, "http://"):
|
||||
handleAbsolutePath(attr, baseURL)
|
||||
continue
|
||||
default:
|
||||
handleDocumentRelativePath(attr, baseURL)
|
||||
continue
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
// RewriteHTMLResourceURLs modifies HTTP responses
|
||||
// to rewrite URLs attributes in HTML content (such as src, href)
|
||||
// - `<img src='/relative_path'>` -> `<img src='/https://proxiedsite.com/relative_path'>`
|
||||
// - This function is designed to allow the proxified page
|
||||
// to still be browsible by routing all resource URLs through the proxy.
|
||||
//
|
||||
// ---
|
||||
//
|
||||
// - It works by replacing the io.ReadCloser of the http.Response.Body
|
||||
// with another io.ReaderCloser (HTMLResourceRewriter) that wraps the first one.
|
||||
//
|
||||
// - This process can be done multiple times, so that the response will
|
||||
// be streamed and modified through each pass without buffering the entire response in memory.
|
||||
//
|
||||
// - HTMLResourceRewriter reads the http.Response.Body stream,
|
||||
// parsing each HTML token one at a time and replacing attribute tags.
|
||||
//
|
||||
// - When ProxyChain.Execute() is called, the response body will be read from the server
|
||||
// and pulled through each ResponseModification which wraps the ProxyChain.Response.Body
|
||||
// without ever buffering the entire HTTP response in memory.
|
||||
func RewriteHTMLResourceURLs() proxychain.ResponseModification {
|
||||
return func(chain *proxychain.ProxyChain) error {
|
||||
// return early if it's not HTML
|
||||
ct := chain.Response.Header.Get("content-type")
|
||||
if !strings.HasPrefix(ct, "text/html") {
|
||||
return nil
|
||||
}
|
||||
|
||||
chain.Response.Body = NewHTMLResourceURLRewriter(chain.Response.Body, chain.Request.URL)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user