diff --git a/cmd/main.go b/cmd/main.go index 2106713..554353a 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -147,13 +147,14 @@ func main() { app.Get("ruleset", handlers.Ruleset) app.Get("raw/*", handlers.Raw) - app.Get("api/*", handlers.Api) proxyOpts := &handlers.ProxyOptions{ Verbose: *verbose, RulesetPath: *ruleset, } + app.Get("api/outline/*", handlers.NewAPIOutlineHandler("api/outline/*", proxyOpts)) + app.Get("/*", handlers.NewProxySiteHandler(proxyOpts)) app.Post("/*", handlers.NewProxySiteHandler(proxyOpts)) diff --git a/handlers/api_outline.go b/handlers/api_outline.go new file mode 100644 index 0000000..f14fe5a --- /dev/null +++ b/handlers/api_outline.go @@ -0,0 +1,44 @@ +package handlers + +import ( + "ladder/proxychain" + rx "ladder/proxychain/requestmodifers" + tx "ladder/proxychain/responsemodifers" + + "github.com/gofiber/fiber/v2" +) + +func NewAPIOutlineHandler(path string, opts *ProxyOptions) fiber.Handler { + // TODO: implement ruleset logic + /* + var rs ruleset.RuleSet + if opts.RulesetPath != "" { + r, err := ruleset.NewRuleset(opts.RulesetPath) + if err != nil { + panic(err) + } + rs = r + } + */ + + return func(c *fiber.Ctx) error { + proxychain := proxychain. + NewProxyChain(). + WithAPIPath(path). + SetDebugLogging(opts.Verbose). + SetRequestModifications( + rx.MasqueradeAsGoogleBot(), + rx.ForwardRequestHeaders(), + rx.SpoofReferrerFromGoogleSearch(), + ). + AddResponseModifications( + tx.DeleteIncomingCookies(), + tx.RewriteHTMLResourceURLs(), + tx.APIOutline(), + ). + SetFiberCtx(c). + Execute() + + return proxychain + } +} diff --git a/handlers/proxy.go b/handlers/proxy.go index 42c47b4..8682419 100644 --- a/handlers/proxy.go +++ b/handlers/proxy.go @@ -24,10 +24,6 @@ func NewProxySiteHandler(opts *ProxyOptions) fiber.Handler { rs = r } */ - const botUA string = "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" - // 5.255.250.0/24, 37.9.87.0/24, 67.195.37.0/24, 67.195.50.0/24, 67.195.110.0/24, 67.195.111.0/24, 67.195.112.0/23, 67.195.114.0/24, 67.195.115.0/24, 68.180.224.0/21, 72.30.132.0/24, 72.30.142.0/24, 72.30.161.0/24, 72.30.196.0/24, 72.30.198.0/24, 74.6.254.0/24, 74.6.8.0/24, 74.6.13.0/24, 74.6.17.0/24, 74.6.18.0/24, 74.6.22.0/24, 74.6.27.0/24, 74.6.168.0/24, 77.88.5.0/24, 77.88.47.0/24, 93.158.161.0/24, 98.137.72.0/24, 98.137.206.0/24, 98.137.207.0/24, 98.139.168.0/24, 114.111.95.0/24, 124.83.159.0/24, 124.83.179.0/24, 124.83.223.0/24, 141.8.144.0/24, 183.79.63.0/24, 183.79.92.0/24, 203.216.255.0/24, 211.14.11.0/24 - //const ja3 string = "769,49195-49199-49200-49161-49171-49162-49172-156-157-47-10-53-51-57,65281-0-23-35-13-13172-11-10,29-23-24,0" - const ja3 string = "771,49199-49195-49171-49161-49200-49196-49172-49162-51-57-50-49169-49159-47-53-10-5-4-255,0-11-10-13-13172-16,23-25-28-27-24-26-22-14-13-11-12-9-10,0-1-2" return func(c *fiber.Ctx) error { proxychain := proxychain. @@ -38,25 +34,19 @@ func NewProxySiteHandler(opts *ProxyOptions) fiber.Handler { //rx.SpoofJA3fingerprint(ja3, "Googlebot"), //rx.MasqueradeAsFacebookBot(), rx.MasqueradeAsGoogleBot(), - //rx.DeleteOutgoingCookies(), + rx.DeleteOutgoingCookies(), rx.ForwardRequestHeaders(), - rx.SetOutgoingCookie("nyt-a", " "), - rx.SetOutgoingCookie("nyt-gdpr", "0"), - rx.SetOutgoingCookie("nyt-gdpr", "0"), - rx.SetOutgoingCookie("nyt-geo", "DE"), - rx.SetOutgoingCookie("nyt-privacy", "1"), rx.SpoofReferrerFromGoogleSearch(), //rx.RequestWaybackMachine(), //rx.RequestArchiveIs(), ). AddResponseModifications( - //tx.ForwardResponseHeaders(), + tx.ForwardResponseHeaders(), tx.BypassCORS(), tx.BypassContentSecurityPolicy(), //tx.DeleteIncomingCookies(), tx.RewriteHTMLResourceURLs(), - //tx.PatchDynamicResourceURLs(), - tx.APIOutline(), + tx.PatchDynamicResourceURLs(), //tx.SetContentSecurityPolicy("default-src * 'unsafe-inline' 'unsafe-eval' data: blob:;"), ). Execute() diff --git a/proxychain/proxychain.go b/proxychain/proxychain.go index 151334e..ad5da59 100644 --- a/proxychain/proxychain.go +++ b/proxychain/proxychain.go @@ -102,6 +102,7 @@ type ProxyChain struct { Ruleset *ruleset.RuleSet debugMode bool abortErr error + _apiPrefix string } // a ProxyStrategy is a pre-built proxychain with purpose-built defaults @@ -167,6 +168,15 @@ func (chain *ProxyChain) AddResponseModifications(mods ...ResponseModification) return chain } +// WithAPIPath trims the path during URL extraction. +// example: using path = "api/outline/", a path like "http://localhost:8080/api/outline/https://example.com" becomes "https://example.com" +func (chain *ProxyChain) WithAPIPath(path string) *ProxyChain { + fmt.Println("===================") + fmt.Printf("set path %s\n", path) + chain._apiPrefix = path + return chain +} + // Adds a ruleset to ProxyChain func (chain *ProxyChain) AddRuleset(rs *ruleset.RuleSet) *ProxyChain { chain.Ruleset = rs @@ -254,6 +264,10 @@ func preventRecursiveProxyRequest(urlQuery *url.URL, baseProxyURL string) *url.U // is a relative path, it reconstructs the full URL using the referer header. func (chain *ProxyChain) extractURL() (*url.URL, error) { reqURL := chain.Context.Params("*") + fmt.Println("XXXXXXXXXXXXXXXX") + fmt.Println(reqURL) + fmt.Println(chain._apiPrefix) + reqURL = strings.TrimPrefix(reqURL, chain._apiPrefix) // sometimes client requests doubleroot '//' // there is a bug somewhere else, but this is a workaround until we find it @@ -347,6 +361,9 @@ func (chain *ProxyChain) SetOnceHTTPClient(httpClient HTTPClient) *ProxyChain { // SetVerbose changes the logging behavior to print // the modification steps and applied rulesets for debugging func (chain *ProxyChain) SetDebugLogging(isDebugMode bool) *ProxyChain { + if isDebugMode { + log.Println("DEBUG MODE ENABLED") + } chain.debugMode = isDebugMode return chain }