package handlers import ( "fmt" "io" "log" "net/http" "net/url" "os" "regexp" "strings" "ladder/pkg/ruleset" "ladder/proxychain" rx "ladder/proxychain/requestmodifers" tx "ladder/proxychain/responsemodifers" "github.com/PuerkitoBio/goquery" "github.com/gofiber/fiber/v2" ) var ( UserAgent = getenv("USER_AGENT", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)") ForwardedFor = getenv("X_FORWARDED_FOR", "66.249.66.1") rulesSet = ruleset.NewRulesetFromEnv() allowedDomains = []string{} ) func init() { allowedDomains = strings.Split(os.Getenv("ALLOWED_DOMAINS"), ",") if os.Getenv("ALLOWED_DOMAINS_RULESET") == "true" { allowedDomains = append(allowedDomains, rulesSet.Domains()...) } } type ProxyOptions struct { RulesetPath string Verbose bool } func NewProxySiteHandler(opts *ProxyOptions) fiber.Handler { /* var rs ruleset.RuleSet if opts.RulesetPath != "" { r, err := ruleset.NewRuleset(opts.RulesetPath) if err != nil { panic(err) } rs = r } */ return func(c *fiber.Ctx) error { proxychain := proxychain. NewProxyChain(). SetFiberCtx(c). SetDebugLogging(opts.Verbose). SetRequestModifications( rx.DeleteOutgoingCookies(), ). AddResponseModifications( tx.DeleteIncomingCookies(), tx.RewriteHTMLResourceURLs(), ). Execute() return proxychain } } func modifyURL(uri string, rule ruleset.Rule) (string, error) { newUrl, err := url.Parse(uri) if err != nil { return "", err } for _, urlMod := range rule.UrlMods.Domain { re := regexp.MustCompile(urlMod.Match) newUrl.Host = re.ReplaceAllString(newUrl.Host, urlMod.Replace) } for _, urlMod := range rule.UrlMods.Path { re := regexp.MustCompile(urlMod.Match) newUrl.Path = re.ReplaceAllString(newUrl.Path, urlMod.Replace) } v := newUrl.Query() for _, query := range rule.UrlMods.Query { if query.Value == "" { v.Del(query.Key) continue } v.Set(query.Key, query.Value) } newUrl.RawQuery = v.Encode() if rule.GoogleCache { newUrl, err = url.Parse("https://webcache.googleusercontent.com/search?q=cache:" + newUrl.String()) if err != nil { return "", err } } return newUrl.String(), nil } func fetchSite(urlpath string, queries map[string]string) (string, *http.Request, *http.Response, error) { urlQuery := "?" if len(queries) > 0 { for k, v := range queries { urlQuery += k + "=" + v + "&" } } urlQuery = strings.TrimSuffix(urlQuery, "&") urlQuery = strings.TrimSuffix(urlQuery, "?") u, err := url.Parse(urlpath) if err != nil { return "", nil, nil, err } if len(allowedDomains) > 0 && !StringInSlice(u.Host, allowedDomains) { return "", nil, nil, fmt.Errorf("domain not allowed. %s not in %s", u.Host, allowedDomains) } if os.Getenv("LOG_URLS") == "true" { log.Println(u.String() + urlQuery) } // Modify the URI according to ruleset rule := fetchRule(u.Host, u.Path) url, err := modifyURL(u.String()+urlQuery, rule) if err != nil { return "", nil, nil, err } // Fetch the site client := &http.Client{} req, _ := http.NewRequest("GET", url, nil) if rule.Headers.UserAgent != "" { req.Header.Set("User-Agent", rule.Headers.UserAgent) } else { req.Header.Set("User-Agent", UserAgent) } if rule.Headers.XForwardedFor != "" { if rule.Headers.XForwardedFor != "none" { req.Header.Set("X-Forwarded-For", rule.Headers.XForwardedFor) } } else { req.Header.Set("X-Forwarded-For", ForwardedFor) } if rule.Headers.Referer != "" { if rule.Headers.Referer != "none" { req.Header.Set("Referer", rule.Headers.Referer) } } else { req.Header.Set("Referer", u.String()) } if rule.Headers.Cookie != "" { req.Header.Set("Cookie", rule.Headers.Cookie) } resp, err := client.Do(req) if err != nil { return "", nil, nil, err } defer resp.Body.Close() bodyB, err := io.ReadAll(resp.Body) if err != nil { return "", nil, nil, err } if rule.Headers.CSP != "" { //log.Println(rule.Headers.CSP) resp.Header.Set("Content-Security-Policy", rule.Headers.CSP) } //log.Print("rule", rule) TODO: Add a debug mode to print the rule body := rewriteHtml(bodyB, u, rule) return body, req, resp, nil } func rewriteHtml(bodyB []byte, u *url.URL, rule ruleset.Rule) string { // Rewrite the HTML body := string(bodyB) // images imagePattern := `]*\s+)?src="(/)([^"]*)"` re := regexp.MustCompile(imagePattern) body = re.ReplaceAllString(body, fmt.Sprintf(`]*\s+)?src="(/)([^"]*)"` reScript := regexp.MustCompile(scriptPattern) body = reScript.ReplaceAllString(body, fmt.Sprintf(`