fix srcset handler in html rewriter

handlers/proxy.go

@@ -31,16 +31,27 @@ func NewProxySiteHandler(opts *ProxyOptions) fiber.Handler {
 			SetFiberCtx(c).
 			SetDebugLogging(opts.Verbose).
 			SetRequestModifications(
-				rx.MasqueradeAsFacebookBot(),
+				//rx.MasqueradeAsFacebookBot(),
-				rx.DeleteOutgoingCookies(),
+				rx.MasqueradeAsGoogleBot(),
-			// rx.RequestArchiveIs(),
+				//rx.DeleteOutgoingCookies(),
+				rx.ForwardRequestHeaders(),
+				rx.SetOutgoingCookie("nyt-a", " "),
+				rx.SetOutgoingCookie("nyt-gdpr", "0"),
+				rx.SetOutgoingCookie("nyt-gdpr", "0"),
+				rx.SetOutgoingCookie("nyt-geo", "DE"),
+				rx.SetOutgoingCookie("nyt-privacy", "1"),
+				rx.SpoofReferrerFromGoogleSearch(),
+				//rx.RequestWaybackMachine(),
+				//rx.RequestArchiveIs(),
 			).
 			AddResponseModifications(
 				tx.BypassCORS(),
-				//tx.BypassContentSecurityPolicy(),
+				tx.BypassContentSecurityPolicy(),
 				//tx.DeleteIncomingCookies(),
-				//tx.RewriteHTMLResourceURLs(),
+				tx.ForwardResponseHeaders(),
-				//tx.PatchDynamicResourceURLs(),
+				tx.RewriteHTMLResourceURLs(),
+				tx.PatchDynamicResourceURLs(),
+				//tx.SetContentSecurityPolicy("default-src * 'unsafe-inline' 'unsafe-eval' data: blob:;"),
 			).
 			Execute()
 

proxychain/proxychain.go

@@ -5,8 +5,8 @@ import (
 	"fmt"
 	"io"
 	"log"
-	//"net/http"
+	"net/http"
-	http "github.com/Danny-Dasilva/fhttp"
+	//http "github.com/Danny-Dasilva/fhttp"
 
 	"net/url"
 	"strings"

proxychain/requestmodifers/forward_request_headers.go

@@ -19,11 +19,12 @@ func init() {
 		"x-forwarded-for":   true,
 		"x-real-ip":         true,
 		"forwarded":         true,
+		"accept-encoding":   true,
 	}
 }
 
 // ForwardRequestHeaders forwards the requests headers sent from the client to the upstream server
-func ForwardRequestHeaders(ua string) proxychain.RequestModification {
+func ForwardRequestHeaders() proxychain.RequestModification {
 	return func(chain *proxychain.ProxyChain) error {
 
 		forwardHeaders := func(key, value []byte) {
@@ -32,6 +33,7 @@ func ForwardRequestHeaders(ua string) proxychain.RequestModification {
 			if forwardBlacklist[k] {
 				return
 			}
+			//fmt.Println(k, v)
 			chain.Request.Header.Set(k, v)
 		}
 

proxychain/requestmodifers/modify_outgoing_cookies.go

@@ -1,8 +1,8 @@
 package requestmodifers
 
 import (
-	//"net/http"
+	"net/http"
-	http "github.com/Danny-Dasilva/fhttp"
+	//http "github.com/Danny-Dasilva/fhttp"
 
 	"ladder/proxychain"
 )

proxychain/requestmodifers/resolve_with_google_doh.go

@@ -5,8 +5,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"net"
-	//"net/http"
+	"net/http"
-	http "github.com/Danny-Dasilva/fhttp"
+	//http "github.com/Danny-Dasilva/fhttp"
 	"time"
 
 	"ladder/proxychain"

proxychain/requestmodifers/spoof_ja3_fingerprint.go

@@ -1,5 +1,15 @@
 package requestmodifers
 
+import "ladder/proxychain"
+
+// TODO: fix
+func SpoofJA3fingerprint(ja3 string, userAgent string) proxychain.RequestModification {
+	return func(chain *proxychain.ProxyChain) error {
+		return nil
+	}
+}
+
+/*
 import (
 	"github.com/Danny-Dasilva/CycleTLS/cycletls"
 	http "github.com/Danny-Dasilva/fhttp"
@@ -44,3 +54,4 @@ func SpoofJA3fingerprintWithProxy(ja3 string, userAgent string, proxy proxy.Cont
 		return nil
 	}
 }
+*/

proxychain/responsemodifers/bypass_csp.go

@@ -11,7 +11,7 @@ import (
 // from enforcing any CSP restrictions. This should run at the end of the chain.
 func BypassContentSecurityPolicy() proxychain.ResponseModification {
 	return func(chain *proxychain.ProxyChain) error {
-		chain.AddResponseModifications(
+		chain.AddOnceResponseModifications(
 			DeleteResponseHeader("Content-Security-Policy"),
 			DeleteResponseHeader("Content-Security-Policy-Report-Only"),
 			DeleteResponseHeader("X-Content-Security-Policy"),

proxychain/responsemodifers/rewriters/html_token_url_rewriter.go

@@ -89,14 +89,20 @@ func NewHTMLTokenURLRewriter(baseURL *url.URL, proxyURL string) *HTMLTokenURLRew
 }
 
 func (r *HTMLTokenURLRewriter) ShouldModify(token *html.Token) bool {
+	//fmt.Printf("touch token: %s\n", token.String())
 	attrLen := len(token.Attr)
 	if attrLen == 0 {
 		return false
 	}
-	if !(token.Type == html.StartTagToken || token.Type == html.SelfClosingTagToken) {
+
-		return false
+	if token.Type == html.StartTagToken {
+		return true
 	}
-	return true
+
+	if token.Type == html.SelfClosingTagToken {
+		return true
+	}
+	return false
 }
 
 func (r *HTMLTokenURLRewriter) ModifyToken(token *html.Token) (string, string) {
@@ -198,10 +204,10 @@ func handleAbsolutePath(attr *html.Attribute, baseURL *url.URL) {
 	if err != nil {
 		return
 	}
-	if !(u.Scheme == "http" || u.Scheme == "https") {
+	if u.Scheme != "http" || u.Scheme != "https" {
 		return
 	}
-	attr.Val = fmt.Sprintf("%s://%s/%s", baseURL.Scheme, baseURL.Host, escape(strings.TrimPrefix(attr.Val, "/")))
+	attr.Val = fmt.Sprintf("/%s", escape(strings.TrimPrefix(attr.Val, "/")))
 	log.Printf("abs url rewritten-> '%s'='%s'", attr.Key, attr.Val)
 }
 
@@ -234,31 +240,27 @@ func handleSrcSet(attr *html.Attribute, baseURL *url.URL) {
 	srcSetItems := strings.Split(attr.Val, ",")
 
 	for i, srcItem := range srcSetItems {
-		srcParts := strings.Fields(srcItem) // Fields splits around whitespace, trimming them
+		srcParts := strings.Fields(srcItem)
 
 		if len(srcParts) == 0 {
-			continue // skip empty items
+			continue
 		}
 
-		// rewrite each URL part by passing in fake attribute
+		fakeAttr := &html.Attribute{Val: srcParts[0], Key: "src"}
-		f := &html.Attribute{Val: srcParts[0], Key: "src"}
+		handleURLPart(fakeAttr, baseURL)
-		handleURLPart(f, baseURL)
-		urlPart := f.Key
 
-		// First srcset item without a descriptor
+		if i > 0 {
-		if i == 0 && (len(srcParts) == 1 || !strings.HasSuffix(srcParts[1], "x")) {
+			srcSetBuilder.WriteString(", ")
-			srcSetBuilder.WriteString(urlPart)
-		} else {
-			srcSetBuilder.WriteString(fmt.Sprintf("%s %s", urlPart, srcParts[1]))
 		}
 
-		if i < len(srcSetItems)-1 {
+		srcSetBuilder.WriteString(fakeAttr.Val)
-			srcSetBuilder.WriteString(",") // Add comma for all but last item
+		if len(srcParts) > 1 {
+			srcSetBuilder.WriteString(" ")
+			srcSetBuilder.WriteString(strings.Join(srcParts[1:], " "))
 		}
 	}
 
 	attr.Val = srcSetBuilder.String()
-	log.Printf("srcset url rewritten-> '%s'='%s'", attr.Key, attr.Val)
 }
 
 func escape(str string) string {