add /api/outline path

cmd/main.go

@@ -147,13 +147,14 @@ func main() {
 
 	app.Get("ruleset", handlers.Ruleset)
 	app.Get("raw/*", handlers.Raw)
-	app.Get("api/*", handlers.Api)
 
 	proxyOpts := &handlers.ProxyOptions{
 		Verbose:     *verbose,
 		RulesetPath: *ruleset,
 	}
 
+	app.Get("api/outline/*", handlers.NewAPIOutlineHandler("api/outline/*", proxyOpts))
+
 	app.Get("/*", handlers.NewProxySiteHandler(proxyOpts))
 	app.Post("/*", handlers.NewProxySiteHandler(proxyOpts))
 

handlers/api_outline.go

@@ -0,0 +1,44 @@
+package handlers
+
+import (
+	"ladder/proxychain"
+	rx "ladder/proxychain/requestmodifers"
+	tx "ladder/proxychain/responsemodifers"
+
+	"github.com/gofiber/fiber/v2"
+)
+
+func NewAPIOutlineHandler(path string, opts *ProxyOptions) fiber.Handler {
+	// TODO: implement ruleset logic
+	/*
+		var rs ruleset.RuleSet
+		if opts.RulesetPath != "" {
+			r, err := ruleset.NewRuleset(opts.RulesetPath)
+			if err != nil {
+				panic(err)
+			}
+			rs = r
+		}
+	*/
+
+	return func(c *fiber.Ctx) error {
+		proxychain := proxychain.
+			NewProxyChain().
+			WithAPIPath(path).
+			SetDebugLogging(opts.Verbose).
+			SetRequestModifications(
+				rx.MasqueradeAsGoogleBot(),
+				rx.ForwardRequestHeaders(),
+				rx.SpoofReferrerFromGoogleSearch(),
+			).
+			AddResponseModifications(
+				tx.DeleteIncomingCookies(),
+				tx.RewriteHTMLResourceURLs(),
+				tx.APIOutline(),
+			).
+			SetFiberCtx(c).
+			Execute()
+
+		return proxychain
+	}
+}

handlers/proxy.go

@@ -24,10 +24,6 @@ func NewProxySiteHandler(opts *ProxyOptions) fiber.Handler {
 			rs = r
 		}
 	*/
-	const botUA string = "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
-	// 5.255.250.0/24, 37.9.87.0/24, 67.195.37.0/24, 67.195.50.0/24, 67.195.110.0/24, 67.195.111.0/24, 67.195.112.0/23, 67.195.114.0/24, 67.195.115.0/24, 68.180.224.0/21, 72.30.132.0/24, 72.30.142.0/24, 72.30.161.0/24, 72.30.196.0/24, 72.30.198.0/24, 74.6.254.0/24, 74.6.8.0/24, 74.6.13.0/24, 74.6.17.0/24, 74.6.18.0/24, 74.6.22.0/24, 74.6.27.0/24, 74.6.168.0/24, 77.88.5.0/24, 77.88.47.0/24, 93.158.161.0/24, 98.137.72.0/24, 98.137.206.0/24, 98.137.207.0/24, 98.139.168.0/24, 114.111.95.0/24, 124.83.159.0/24, 124.83.179.0/24, 124.83.223.0/24, 141.8.144.0/24, 183.79.63.0/24, 183.79.92.0/24, 203.216.255.0/24, 211.14.11.0/24
-	//const ja3 string = "769,49195-49199-49200-49161-49171-49162-49172-156-157-47-10-53-51-57,65281-0-23-35-13-13172-11-10,29-23-24,0"
-	const ja3 string = "771,49199-49195-49171-49161-49200-49196-49172-49162-51-57-50-49169-49159-47-53-10-5-4-255,0-11-10-13-13172-16,23-25-28-27-24-26-22-14-13-11-12-9-10,0-1-2"
 
 	return func(c *fiber.Ctx) error {
 		proxychain := proxychain.
@@ -38,25 +34,19 @@ func NewProxySiteHandler(opts *ProxyOptions) fiber.Handler {
 				//rx.SpoofJA3fingerprint(ja3, "Googlebot"),
 				//rx.MasqueradeAsFacebookBot(),
 				rx.MasqueradeAsGoogleBot(),
-				//rx.DeleteOutgoingCookies(),
+				rx.DeleteOutgoingCookies(),
 				rx.ForwardRequestHeaders(),
-				rx.SetOutgoingCookie("nyt-a", " "),
-				rx.SetOutgoingCookie("nyt-gdpr", "0"),
-				rx.SetOutgoingCookie("nyt-gdpr", "0"),
-				rx.SetOutgoingCookie("nyt-geo", "DE"),
-				rx.SetOutgoingCookie("nyt-privacy", "1"),
 				rx.SpoofReferrerFromGoogleSearch(),
 				//rx.RequestWaybackMachine(),
 				//rx.RequestArchiveIs(),
 			).
 			AddResponseModifications(
-				//tx.ForwardResponseHeaders(),
+				tx.ForwardResponseHeaders(),
 				tx.BypassCORS(),
 				tx.BypassContentSecurityPolicy(),
 				//tx.DeleteIncomingCookies(),
 				tx.RewriteHTMLResourceURLs(),
-				//tx.PatchDynamicResourceURLs(),
+				tx.PatchDynamicResourceURLs(),
-				tx.APIOutline(),
 			//tx.SetContentSecurityPolicy("default-src * 'unsafe-inline' 'unsafe-eval' data: blob:;"),
 			).
 			Execute()

proxychain/proxychain.go

@@ -102,6 +102,7 @@ type ProxyChain struct {
 	Ruleset                   *ruleset.RuleSet
 	debugMode                 bool
 	abortErr                  error
+	_apiPrefix                string
 }
 
 // a ProxyStrategy is a pre-built proxychain with purpose-built defaults
@@ -167,6 +168,15 @@ func (chain *ProxyChain) AddResponseModifications(mods ...ResponseModification)
 	return chain
 }
 
+// WithAPIPath trims the path during URL extraction.
+// example: using path = "api/outline/", a path like "http://localhost:8080/api/outline/https://example.com" becomes "https://example.com"
+func (chain *ProxyChain) WithAPIPath(path string) *ProxyChain {
+	fmt.Println("===================")
+	fmt.Printf("set path %s\n", path)
+	chain._apiPrefix = path
+	return chain
+}
+
 // Adds a ruleset to ProxyChain
 func (chain *ProxyChain) AddRuleset(rs *ruleset.RuleSet) *ProxyChain {
 	chain.Ruleset = rs
@@ -254,6 +264,10 @@ func preventRecursiveProxyRequest(urlQuery *url.URL, baseProxyURL string) *url.U
 // is a relative path, it reconstructs the full URL using the referer header.
 func (chain *ProxyChain) extractURL() (*url.URL, error) {
 	reqURL := chain.Context.Params("*")
+	fmt.Println("XXXXXXXXXXXXXXXX")
+	fmt.Println(reqURL)
+	fmt.Println(chain._apiPrefix)
+	reqURL = strings.TrimPrefix(reqURL, chain._apiPrefix)
 
 	// sometimes client requests doubleroot '//'
 	// there is a bug somewhere else, but this is a workaround until we find it
@@ -347,6 +361,9 @@ func (chain *ProxyChain) SetOnceHTTPClient(httpClient HTTPClient) *ProxyChain {
 // SetVerbose changes the logging behavior to print
 // the modification steps and applied rulesets for debugging
 func (chain *ProxyChain) SetDebugLogging(isDebugMode bool) *ProxyChain {
+	if isDebugMode {
+		log.Println("DEBUG MODE ENABLED")
+	}
 	chain.debugMode = isDebugMode
 	return chain
 }