DWS/hadrian
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #52 from dxbednarczyk/origin/proxy_v2

Browse Source 
tldr
This commit is contained in:
Kevin Pham
2023-11-29 15:08:24 -06:00
committed by GitHub
parent 53ce3ff297 439ec61de9
commit 23fb0253b3
16 changed files with 51 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
proxychain/proxychain.go
View File

@@ -5,6 +5,9 @@ import (
"fmt"
"io"
"log"
"net/url"
"strings"
//"time"
//"net/http"
@@ -12,10 +15,8 @@ import (
//http "github.com/Danny-Dasilva/fhttp"
http "github.com/bogdanfinn/fhttp"
tls_client "github.com/bogdanfinn/tls-client"
//"github.com/bogdanfinn/tls-client/profiles"
"net/url"
"strings"
//"github.com/bogdanfinn/tls-client/profiles"
"ladder/pkg/ruleset"
@@ -122,7 +123,7 @@ type HTTPClient interface {
SetCookies(u *url.URL, cookies []*http.Cookie)
SetCookieJar(jar http.CookieJar)
GetCookieJar() http.CookieJar
SetProxy(proxyUrl string) error
SetProxy(proxyURL string) error
GetProxy() string
SetFollowRedirect(followRedirect bool)
GetFollowRedirect() bool
@@ -184,7 +185,7 @@ func (chain *ProxyChain) AddRuleset(rs *ruleset.RuleSet) *ProxyChain {
return chain
}
func (chain *ProxyChain) _initialize_request() (*http.Request, error) {
func (chain *ProxyChain) _initializeRequest() (*http.Request, error) {
if chain.Context == nil {
chain.abortErr = chain.abort(errors.New("no context set"))
return nil, chain.abortErr
@@ -264,9 +265,11 @@ func preventRecursiveProxyRequest(urlQuery *url.URL, baseProxyURL string) *url.U
// is a relative path, it reconstructs the full URL using the referer header.
func (chain *ProxyChain) extractURL() (*url.URL, error) {
reqURL := chain.Context.Params("*")
fmt.Println("XXXXXXXXXXXXXXXX")
fmt.Println(reqURL)
fmt.Println(chain._apiPrefix)
reqURL = strings.TrimPrefix(reqURL, chain._apiPrefix)
// sometimes client requests doubleroot '//'
@@ -318,7 +321,7 @@ func (chain *ProxyChain) SetFiberCtx(ctx *fiber.Ctx) *ProxyChain {
chain.Context = ctx
// initialize the request and prepare it for modification
req, err := chain._initialize_request()
req, err := chain._initializeRequest()
if err != nil {
chain.abortErr = chain.abort(err)
}

4
proxychain/proxychain_pool.go
View File

@@ -4,8 +4,8 @@ import (
"net/url"
)
type ProxyChainPool map[url.URL]ProxyChain
type Pool map[url.URL]ProxyChain
func NewProxyChainPool() ProxyChainPool {
func NewPool() Pool {
return map[url.URL]ProxyChain{}
}

3
proxychain/requestmodifers/forward_request_headers.go
View File

@@ -1,9 +1,9 @@
package requestmodifers
import (
"strings"
//"fmt"
"ladder/proxychain"
"strings"
)
var forwardBlacklist map[string]bool
@@ -27,7 +27,6 @@ func init() {
// ForwardRequestHeaders forwards the requests headers sent from the client to the upstream server
func ForwardRequestHeaders() proxychain.RequestModification {
return func(chain *proxychain.ProxyChain) error {
forwardHeaders := func(key, value []byte) {
k := strings.ToLower(string(key))
v := string(value)

2
proxychain/requestmodifers/request_archive_is.go
View File

@@ -3,10 +3,10 @@ package requestmodifers
import (
"fmt"
"net/url"
"regexp"
"ladder/proxychain"
tx "ladder/proxychain/responsemodifers"
"regexp"
)
const archivistUrl string = "https://archive.is/latest"

2
proxychain/requestmodifers/request_wayback_machine.go
View File

@@ -2,10 +2,10 @@ package requestmodifers
import (
"net/url"
"regexp"
"ladder/proxychain"
tx "ladder/proxychain/responsemodifers"
"regexp"
)
const waybackUrl string = "https://web.archive.org/web/"

8
proxychain/requestmodifers/resolve_with_google_doh.go
View File

@@ -4,15 +4,16 @@ import (
"context"
"encoding/json"
"fmt"
http "github.com/bogdanfinn/fhttp"
"net"
"time"
http "github.com/bogdanfinn/fhttp"
/*
tls_client "github.com/bogdanfinn/tls-client"
//"net/http"
*/
"time"
"ladder/proxychain"
)
@@ -71,7 +72,6 @@ func (cd *CustomDialer) DialContext(ctx context.Context, network, addr string) (
// ResolveWithGoogleDoH modifies a ProxyChain's client to make the request by resolving the URL
// using Google's DNS over HTTPs service
func ResolveWithGoogleDoH() proxychain.RequestModification {
///customDialer := NewCustomDialer(10*time.Second, 10*time.Second)
return func(chain *proxychain.ProxyChain) error {
/*

2
proxychain/requestmodifers/spoof_referrer.go
View File

@@ -2,6 +2,7 @@ package requestmodifers
import (
"fmt"
"ladder/proxychain"
tx "ladder/proxychain/responsemodifers"
)
@@ -12,7 +13,6 @@ import (
// if url is "", then the referrer header is removed.
func SpoofReferrer(url string) proxychain.RequestModification {
return func(chain *proxychain.ProxyChain) error {
// change refer on client side js
script := fmt.Sprintf(`document.referrer = "%s"`, url)
chain.AddOnceResponseModifications(

3
proxychain/requestmodifers/spoof_user_agent.go
View File

@@ -2,9 +2,10 @@ package requestmodifers
import (
_ "embed"
"strings"
"ladder/proxychain"
tx "ladder/proxychain/responsemodifers"
"strings"
)
// https://github.com/faisalman/ua-parser-js/tree/master

5
proxychain/responsemodifers/api/error_api.go
View File

@@ -8,7 +8,7 @@ import (
"reflect"
)
type APIError struct {
type Error struct {
Success bool `json:"success"`
Error ErrorDetails `json:"error"`
}
@@ -25,7 +25,7 @@ func CreateAPIErrReader(err error) io.ReadCloser {
}
baseErr := getBaseError(err)
apiErr := APIError{
apiErr := Error{
Success: false,
Error: ErrorDetails{
Message: err.Error(),
@@ -50,6 +50,7 @@ func getBaseError(err error) error {
if unwrapped == nil {
return err
}
err = unwrapped
}
}

4
proxychain/responsemodifers/forward_response_headers.go
View File

@@ -2,9 +2,10 @@ package responsemodifers
import (
"fmt"
"ladder/proxychain"
"net/url"
"strings"
"ladder/proxychain"
)
var forwardBlacklist map[string]bool
@@ -23,7 +24,6 @@ func init() {
// ForwardResponseHeaders forwards the response headers from the upstream server to the client
func ForwardResponseHeaders() proxychain.ResponseModification {
return func(chain *proxychain.ProxyChain) error {
//fmt.Println(chain.Response.Header)
for uname, headers := range chain.Response.Header {
name := strings.ToLower(uname)

4
proxychain/responsemodifers/modify_incoming_cookies.go
View File

@@ -2,6 +2,7 @@ package responsemodifers
import (
"fmt"
http "github.com/bogdanfinn/fhttp"
//"net/http"
//http "github.com/Danny-Dasilva/fhttp"
@@ -11,7 +12,7 @@ import (
// DeleteIncomingCookies prevents ALL cookies from being sent from the proxy server
// back down to the client.
func DeleteIncomingCookies(whitelist ...string) proxychain.ResponseModification {
func DeleteIncomingCookies(_ ...string) proxychain.ResponseModification {
return func(px *proxychain.ProxyChain) error {
px.Response.Header.Del("Set-Cookie")
return nil
@@ -37,6 +38,7 @@ func DeleteIncomingCookiesExcept(whitelist ...string) proxychain.ResponseModific
filteredCookies := []string{}
for _, cookieStr := range px.Response.Header["Set-Cookie"] {
cookie := parseCookie(cookieStr)
if _, found := whitelistMap[cookie.Name]; found {
filteredCookies = append(filteredCookies, cookieStr)
}

5
proxychain/responsemodifers/outline.go
View File

@@ -3,10 +3,12 @@ package responsemodifers
import (
"bytes"
"encoding/json"
"io"
//"github.com/go-shiori/dom"
"github.com/markusmobius/go-trafilatura"
//"golang.org/x/net/html"
"io"
"ladder/proxychain"
"ladder/proxychain/responsemodifers/api"
)
@@ -46,5 +48,4 @@ func APIOutline() proxychain.ResponseModification {
chain.Response.Body = io.NopCloser(buf)
return nil
}
}

10
proxychain/responsemodifers/outline_test.go
View File

@@ -6,13 +6,15 @@ import (
"io"
"net/url"
"testing"
"ladder/proxychain/responsemodifers/api"
)
func TestCreateAPIErrReader(t *testing.T) {
_, baseErr := url.Parse("://this is an invalid url")
wrappedErr := fmt.Errorf("wrapped error: %w", baseErr)
readCloser := CreateAPIErrReader(wrappedErr)
readCloser := api.CreateAPIErrReader(wrappedErr)
defer readCloser.Close()
// Read and unmarshal the JSON output
@@ -22,7 +24,7 @@ func TestCreateAPIErrReader(t *testing.T) {
}
fmt.Println(string(data))
var apiErr APIError
var apiErr api.Error
err = json.Unmarshal(data, &apiErr)
if err != nil {
t.Fatalf("Failed to unmarshal JSON: %v", err)
@@ -41,7 +43,7 @@ func TestCreateAPIErrReader(t *testing.T) {
func TestCreateAPIErrReader2(t *testing.T) {
_, baseErr := url.Parse("://this is an invalid url")
readCloser := CreateAPIErrReader(baseErr)
readCloser := api.CreateAPIErrReader(baseErr)
defer readCloser.Close()
// Read and unmarshal the JSON output
@@ -51,7 +53,7 @@ func TestCreateAPIErrReader2(t *testing.T) {
}
fmt.Println(string(data))
var apiErr APIError
var apiErr api.Error
err = json.Unmarshal(data, &apiErr)
if err != nil {
t.Fatalf("Failed to unmarshal JSON: %v", err)

4
proxychain/responsemodifers/rewriters/html_rewriter.go
View File

@@ -66,6 +66,7 @@ func (r *HTMLRewriter) Close() error {
r.tokenBuffer.Reset()
r.currentToken = nil
r.currentTokenProcessed = false
return nil
}
@@ -79,6 +80,7 @@ func (r *HTMLRewriter) Read(p []byte) (int, error) {
if r.tokenizer.Err() == io.EOF {
return 0, io.EOF
}
return 0, r.tokenizer.Err()
}
@@ -96,6 +98,7 @@ func (r *HTMLRewriter) Read(p []byte) (int, error) {
if !rewriter.ShouldModify(r.currentToken) {
continue
}
prepend, a := rewriter.ModifyToken(r.currentToken)
appends = append(appends, a)
// add <prepends> to buffer
@@ -125,5 +128,6 @@ func (r *HTMLRewriter) Read(p []byte) (int, error) {
r.currentTokenProcessed = true
err = nil // EOF in this context is expected and not an actual error
}
return n, err
}

8
proxychain/responsemodifers/rewriters/html_token_url_rewriter.go
View File

@@ -111,6 +111,7 @@ func (r *HTMLTokenURLRewriter) ShouldModify(token *html.Token) bool {
func (r *HTMLTokenURLRewriter) ModifyToken(token *html.Token) (string, string) {
for i := range token.Attr {
attr := &token.Attr[i]
switch {
// don't touch tag/attributes that don't contain URIs
case !rewriteAttrs[token.Data][attr.Key]:
@@ -192,9 +193,11 @@ func handleRootRelativePath(attr *html.Attribute, baseURL *url.URL) {
// Document-relative URLs: These are relative to the current document's path and don't start with a "/".
func handleDocumentRelativePath(attr *html.Attribute, baseURL *url.URL) {
log.Printf("PROCESSING: key: %s val: %s\n", attr.Key, attr.Val)
if strings.HasPrefix(attr.Val, "#") {
return
}
relativePath := path.Join(strings.Trim(baseURL.RawPath, "/"), strings.Trim(attr.Val, "/"))
attr.Val = fmt.Sprintf(
"%s://%s/%s",
@@ -204,13 +207,15 @@ func handleDocumentRelativePath(attr *html.Attribute, baseURL *url.URL) {
)
attr.Val = escape(attr.Val)
attr.Val = fmt.Sprintf("/%s", attr.Val)
log.Printf("doc rel url rewritten-> '%s'='%s'", attr.Key, attr.Val)
}
// full URIs beginning with https?://proxiedsite.com
func handleAbsolutePath(attr *html.Attribute, baseURL *url.URL) {
func handleAbsolutePath(attr *html.Attribute, _ *url.URL) {
// check if valid URL
log.Printf("PROCESSING: key: %s val: %s\n", attr.Key, attr.Val)
u, err := url.Parse(attr.Val)
if err != nil {
return
@@ -218,6 +223,7 @@ func handleAbsolutePath(attr *html.Attribute, baseURL *url.URL) {
if !(u.Scheme == "http" || u.Scheme == "https") {
return
}
attr.Val = fmt.Sprintf("/%s", escape(strings.TrimPrefix(attr.Val, "/")))
//attr.Val = fmt.Sprintf("/%s", escape(attr.Val))

3
proxychain/responsemodifers/rewriters/script_injector_rewriter.go
View File

@@ -34,7 +34,7 @@ func (r *ScriptInjectorRewriter) ShouldModify(token *html.Token) bool {
//go:embed after_dom_idle_script_injector.js
var afterDomIdleScriptInjector string
func (r *ScriptInjectorRewriter) ModifyToken(token *html.Token) (string, string) {
func (r *ScriptInjectorRewriter) ModifyToken(_ *html.Token) (string, string) {
switch {
case r.execTime == BeforeDOMContentLoaded:
return "", fmt.Sprintf("\n<script>\n%s\n</script>\n", r.script)
@@ -58,6 +58,7 @@ func (r *ScriptInjectorRewriter) applyParams(params map[string]string) {
for key := range params {
keys = append(keys, key)
}
sort.Slice(keys, func(i, j int) bool {
return len(keys[i]) > len(keys[j])
})