replace cycletls with bogdanfinn/tls-client for better tls fingerprint spoofing reliability

2023-11-28 12:33:53 -06:00
parent 9d31f7eb59
commit 31200cf9e2
10 changed files with 147 additions and 451 deletions
--- a/proxychain/proxychain.go
+++ b/proxychain/proxychain.go
@@ -5,14 +5,21 @@ import (
 	"fmt"
 	"io"
 	"log"
+	//"time"
+
 	//"net/http"
-	http "github.com/Danny-Dasilva/fhttp"
+	//"github.com/Danny-Dasilva/CycleTLS/cycletls"
+	//http "github.com/Danny-Dasilva/fhttp"
+	http "github.com/bogdanfinn/fhttp"
+	tls_client "github.com/bogdanfinn/tls-client"
+	//"github.com/bogdanfinn/tls-client/profiles"

 	"net/url"
 	"strings"

-	"github.com/gofiber/fiber/v2"
 	"ladder/pkg/ruleset"
+
+	"github.com/gofiber/fiber/v2"
 )

 /*
@@ -84,8 +91,8 @@ proxychain.NewProxyChain().
 */
 type ProxyChain struct {
 	Context                   *fiber.Ctx
-	Client                    *http.Client
-	onceClient                *http.Client
+	Client                    HTTPClient
+	onceClient                HTTPClient
 	Request                   *http.Request
 	Response                  *http.Response
 	requestModifications      []RequestModification
@@ -108,6 +115,23 @@ type RequestModification func(*ProxyChain) error
 // ProxyChain Res (http result) & Body (buffered http response body) field
 type ResponseModification func(*ProxyChain) error

+// abstraction over HTTPClient
+type HTTPClient interface {
+	GetCookies(u *url.URL) []*http.Cookie
+	SetCookies(u *url.URL, cookies []*http.Cookie)
+	SetCookieJar(jar http.CookieJar)
+	GetCookieJar() http.CookieJar
+	SetProxy(proxyUrl string) error
+	GetProxy() string
+	SetFollowRedirect(followRedirect bool)
+	GetFollowRedirect() bool
+	CloseIdleConnections()
+	Do(req *http.Request) (*http.Response, error)
+	Get(url string) (resp *http.Response, err error)
+	Head(url string) (resp *http.Response, err error)
+	Post(url, contentType string, body io.Reader) (resp *http.Response, err error)
+}
+
 // SetRequestModifications sets the ProxyChain's request modifers
 // the modifier will not fire until ProxyChain.Execute() is run.
 func (chain *ProxyChain) SetRequestModifications(mods ...RequestModification) *ProxyChain {
@@ -308,14 +332,14 @@ func (chain *ProxyChain) validateCtxIsSet() error {

 // SetHTTPClient sets a new upstream http client transport
 // useful for modifying TLS
-func (chain *ProxyChain) SetHTTPClient(httpClient *http.Client) *ProxyChain {
+func (chain *ProxyChain) SetHTTPClient(httpClient HTTPClient) *ProxyChain {
 	chain.Client = httpClient
 	return chain
 }

 // SetOnceHTTPClient sets a new upstream http client transport temporarily
 // and clears it once it is used.
-func (chain *ProxyChain) SetOnceHTTPClient(httpClient *http.Client) *ProxyChain {
+func (chain *ProxyChain) SetOnceHTTPClient(httpClient HTTPClient) *ProxyChain {
 	chain.onceClient = httpClient
 	return chain
 }
@@ -354,8 +378,20 @@ func (chain *ProxyChain) _reset() {
 // NewProxyChain initializes a new ProxyChain
 func NewProxyChain() *ProxyChain {
 	chain := new(ProxyChain)
-	//chain.Client = http.DefaultClient
-	chain.Client = &http.Client{}
+
+	options := []tls_client.HttpClientOption{
+		tls_client.WithTimeoutSeconds(20),
+		tls_client.WithRandomTLSExtensionOrder(),
+		//tls_client.WithClientProfile(profiles.Chrome_117),
+		//tls_client.WithNotFollowRedirects(),
+		//tls_client.WithCookieJar(jar), // create cookieJar instance and pass it as argument
+	}
+	client, err := tls_client.NewHttpClient(tls_client.NewNoopLogger(), options...)
+	if err != nil {
+		panic(err)
+	}
+	chain.Client = client
+
 	return chain
 }

--- a/proxychain/requestmodifers/masquerade_as_trusted_bot.go
+++ b/proxychain/requestmodifers/masquerade_as_trusted_bot.go
@@ -10,7 +10,8 @@ func MasqueradeAsGoogleBot() proxychain.RequestModification {
 	const botUA string = "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; http://www.google.com/bot.html) Chrome/79.0.3945.120 Safari/537.36"
 	const botIP string = "66.249.78.8" // TODO: create a random ip pool from https://developers.google.com/static/search/apis/ipranges/googlebot.json
 	// https://github.com/trisulnsm/trisul-scripts/blob/master/lua/frontend_scripts/reassembly/ja3/prints/ja3fingerprint.json
-	const ja3 string = "769,49195-49199-49200-49161-49171-49162-49172-156-157-47-10-53-51-57,65281-0-23-35-13-13172-11-10,29-23-24,0"
+	const ja3 string = "769,49195-49199-49196-49200-52393-52392-52244-52243-49161-49171-49162-49172-156-157-47-53-10,65281-0-23-35-13-5-18-16-11-10-21,29-23-24,0"
+	//	"741,49195-49199-49200-49161-49171-49162-49172-156-157-47-10-53-51-57,65281-0-23-35-13-13172-11-10,29-23-24,0"

 	return masqueradeAsTrustedBot(botUA, botIP, ja3)
 }
--- a/proxychain/requestmodifers/modify_outgoing_cookies.go
+++ b/proxychain/requestmodifers/modify_outgoing_cookies.go
@@ -2,7 +2,8 @@ package requestmodifers

 import (
 	//"net/http"
-	http "github.com/Danny-Dasilva/fhttp"
+	//http "github.com/Danny-Dasilva/fhttp"
+	http "github.com/bogdanfinn/fhttp"

 	"ladder/proxychain"
 )
--- a/proxychain/requestmodifers/resolve_with_google_doh.go
+++ b/proxychain/requestmodifers/resolve_with_google_doh.go
@@ -4,9 +4,13 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	http "github.com/bogdanfinn/fhttp"
 	"net"
-	//"net/http"
-	http "github.com/Danny-Dasilva/fhttp"
+	/*
+		tls_client "github.com/bogdanfinn/tls-client"
+		//"net/http"
+	*/
+
 	"time"

 	"ladder/proxychain"
@@ -38,45 +42,53 @@ func resolveWithGoogleDoH(host string) (string, error) {
 	return "", fmt.Errorf("no DoH DNS record found for %s", host)
 }

+type CustomDialer struct {
+	*net.Dialer
+}
+
+func NewCustomDialer(timeout, keepAlive time.Duration) *CustomDialer {
+	return &CustomDialer{
+		Dialer: &net.Dialer{
+			Timeout:   timeout,
+			KeepAlive: keepAlive,
+		},
+	}
+}
+
+func (cd *CustomDialer) DialContext(ctx context.Context, network, addr string) (net.Conn, error) {
+	host, port, err := net.SplitHostPort(addr)
+	if err != nil {
+		port = "443"
+	}
+
+	resolvedHost, err := resolveWithGoogleDoH(host)
+	if err != nil {
+		return nil, err
+	}
+	return cd.Dialer.DialContext(ctx, network, net.JoinHostPort(resolvedHost, port))
+}
+
 // ResolveWithGoogleDoH modifies a ProxyChain's client to make the request by resolving the URL
 // using Google's DNS over HTTPs service
 func ResolveWithGoogleDoH() proxychain.RequestModification {
-	return func(px *proxychain.ProxyChain) error {
-		client := &http.Client{
-			Timeout: px.Client.Timeout,
-		}

-		dialer := &net.Dialer{
-			Timeout:   5 * time.Second,
-			KeepAlive: 5 * time.Second,
-		}
-
-		customDialContext := func(ctx context.Context, network, addr string) (net.Conn, error) {
-			host, port, err := net.SplitHostPort(addr)
-			if err != nil {
-				// If the addr doesn't include a port, determine it based on the URL scheme
-				if px.Request.URL.Scheme == "https" {
-					port = "443"
-				} else {
-					port = "80"
-				}
-				host = addr // assume the entire addr is the host
+	///customDialer := NewCustomDialer(10*time.Second, 10*time.Second)
+	return func(chain *proxychain.ProxyChain) error {
+		/*
+			options := []tls_client.HttpClientOption{
+				tls_client.WithTimeoutSeconds(30),
+				tls_client.WithRandomTLSExtensionOrder(),
+				tls_client.WithDialer(*customDialer.Dialer),
+				//tls_client.WithClientProfile(profiles.Chrome_105),
 			}

-			resolvedHost, err := resolveWithGoogleDoH(host)
+			client, err := tls_client.NewHttpClient(tls_client.NewNoopLogger(), options...)
 			if err != nil {
-				return nil, err
+				return err
 			}

-			return dialer.DialContext(ctx, network, net.JoinHostPort(resolvedHost, port))
-		}
-
-		patchedTransportWithDoH := &http.Transport{
-			DialContext: customDialContext,
-		}
-
-		client.Transport = patchedTransportWithDoH
-		px.Client = client // Assign the modified client to the ProxyChain
+			chain.SetOnceHTTPClient(client)
+		*/
 		return nil
 	}
 }
--- a/proxychain/requestmodifers/spoof_ja3_fingerprint.go
+++ b/proxychain/requestmodifers/spoof_ja3_fingerprint.go
@@ -1,8 +1,13 @@
 package requestmodifers

+// removed due to using a different TLS spoofing technique
+
+/*
 import (
-	"github.com/Danny-Dasilva/CycleTLS/cycletls"
-	http "github.com/Danny-Dasilva/fhttp"
+	//"github.com/Danny-Dasilva/CycleTLS/cycletls"
+	//http "github.com/Danny-Dasilva/fhttp"
+	//http "github.com/bogdanfinn/fhttp"
+
 	"golang.org/x/net/proxy"
 	"ladder/proxychain"
 )
@@ -14,14 +19,14 @@ import (
 func SpoofJA3fingerprint(ja3 string, userAgent string) proxychain.RequestModification {
 	//fmt.Println(ja3)
 	return func(chain *proxychain.ProxyChain) error {
-		// deep copy existing client while modifying http transport
-		ja3SpoofClient := &http.Client{
-			Transport:     cycletls.NewTransport(ja3, userAgent),
-			Timeout:       chain.Client.Timeout,
-			CheckRedirect: chain.Client.CheckRedirect,
-		}
+			// deep copy existing client while modifying http transport
+			ja3SpoofClient := &http.Client{
+				Transport:     cycletls.NewTransport(ja3, userAgent),
+				Timeout:       chain.Client.Timeout,
+				CheckRedirect: chain.Client.CheckRedirect,
+			}

-		chain.SetOnceHTTPClient(ja3SpoofClient)
+			chain.SetOnceHTTPClient(ja3SpoofClient)
 		return nil
 	}
 }
@@ -33,14 +38,15 @@ func SpoofJA3fingerprint(ja3 string, userAgent string) proxychain.RequestModific
 func SpoofJA3fingerprintWithProxy(ja3 string, userAgent string, proxy proxy.ContextDialer) proxychain.RequestModification {
 	return func(chain *proxychain.ProxyChain) error {

-		// deep copy existing client while modifying http transport
-		ja3SpoofClient := &http.Client{
-			Transport:     cycletls.NewTransportWithProxy(ja3, userAgent, proxy),
-			Timeout:       chain.Client.Timeout,
-			CheckRedirect: chain.Client.CheckRedirect,
-		}
+			// deep copy existing client while modifying http transport
+			ja3SpoofClient := &http.Client{
+				Transport:     cycletls.NewTransportWithProxy(ja3, userAgent, proxy),
+				Timeout:       chain.Client.Timeout,
+				CheckRedirect: chain.Client.CheckRedirect,
+			}

-		chain.SetOnceHTTPClient(ja3SpoofClient)
+			chain.SetOnceHTTPClient(ja3SpoofClient)
 		return nil
 	}
 }
+*/
--- a/proxychain/responsemodifers/modify_incoming_cookies.go
+++ b/proxychain/responsemodifers/modify_incoming_cookies.go
@@ -2,8 +2,9 @@ package responsemodifers

 import (
 	"fmt"
+	http "github.com/bogdanfinn/fhttp"
 	//"net/http"
-	http "github.com/Danny-Dasilva/fhttp"
+	//http "github.com/Danny-Dasilva/fhttp"

 	"ladder/proxychain"
 )
--- a/proxychain/responsemodifers/rewriters/html_token_url_rewriter.go
+++ b/proxychain/responsemodifers/rewriters/html_token_url_rewriter.go
@@ -82,6 +82,8 @@ type HTMLTokenURLRewriter struct {

 // NewHTMLTokenURLRewriter creates a new instance of HTMLResourceURLRewriter.
 // It initializes the tokenizer with the provided source and sets the proxy URL.
+// baseURL might be https://medium.com/foobar
+// proxyURL is http://localhost:8080
 func NewHTMLTokenURLRewriter(baseURL *url.URL, proxyURL string) *HTMLTokenURLRewriter {
 	return &HTMLTokenURLRewriter{
 		baseURL:  baseURL,
@@ -160,6 +162,11 @@ func handleProtocolRelativePath(attr *html.Attribute, baseURL *url.URL) {

 // Root-relative URLs: These are relative to the root path and start with a "/".
 func handleRootRelativePath(attr *html.Attribute, baseURL *url.URL) {
+	// Skip processing if it's already in the correct format
+	if strings.HasPrefix(attr.Val, "/http://") || strings.HasPrefix(attr.Val, "/https://") {
+		return
+	}
+
 	// doublecheck this is a valid relative URL
 	log.Printf("PROCESSING: key: %s val: %s\n", attr.Key, attr.Val)
 	_, err := url.Parse(fmt.Sprintf("http://localhost.com%s", attr.Val))
@@ -212,6 +219,8 @@ func handleAbsolutePath(attr *html.Attribute, baseURL *url.URL) {
 		return
 	}
 	attr.Val = fmt.Sprintf("/%s", escape(strings.TrimPrefix(attr.Val, "/")))
+	//attr.Val = fmt.Sprintf("/%s", escape(attr.Val))
+
 	log.Printf("abs url rewritten-> '%s'='%s'", attr.Key, attr.Val)
 }