package handlers
import (
"fmt"
"io"
"log"
"net/http"
"net/url"
"os"
"regexp"
"strings"
"ladder/pkg/ruleset"
"ladder/proxychain"
rx "ladder/proxychain/requestmodifers"
tx "ladder/proxychain/responsemodifers"
"github.com/PuerkitoBio/goquery"
"github.com/gofiber/fiber/v2"
)
var (
UserAgent = getenv("USER_AGENT", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)")
ForwardedFor = getenv("X_FORWARDED_FOR", "66.249.66.1")
rulesSet = ruleset.NewRulesetFromEnv()
allowedDomains = []string{}
)
func init() {
allowedDomains = strings.Split(os.Getenv("ALLOWED_DOMAINS"), ",")
if os.Getenv("ALLOWED_DOMAINS_RULESET") == "true" {
allowedDomains = append(allowedDomains, rulesSet.Domains()...)
}
}
type ProxyOptions struct {
RulesetPath string
Verbose bool
}
func NewProxySiteHandler(opts *ProxyOptions) fiber.Handler {
/*
var rs ruleset.RuleSet
if opts.RulesetPath != "" {
r, err := ruleset.NewRuleset(opts.RulesetPath)
if err != nil {
panic(err)
}
rs = r
}
*/
return func(c *fiber.Ctx) error {
proxychain := proxychain.
NewProxyChain().
SetFiberCtx(c).
SetDebugLogging(opts.Verbose).
SetRequestModifications(
rx.DeleteOutgoingCookies(),
//rx.RequestArchiveIs(),
rx.MasqueradeAsGoogleBot(),
).
AddResponseModifications(
tx.BypassCORS(),
tx.BypassContentSecurityPolicy(),
tx.DeleteIncomingCookies(),
tx.RewriteHTMLResourceURLs(),
tx.PatchDynamicResourceURLs(),
).
Execute()
return proxychain
}
}
func modifyURL(uri string, rule ruleset.Rule) (string, error) {
newUrl, err := url.Parse(uri)
if err != nil {
return "", err
}
for _, urlMod := range rule.UrlMods.Domain {
re := regexp.MustCompile(urlMod.Match)
newUrl.Host = re.ReplaceAllString(newUrl.Host, urlMod.Replace)
}
for _, urlMod := range rule.UrlMods.Path {
re := regexp.MustCompile(urlMod.Match)
newUrl.Path = re.ReplaceAllString(newUrl.Path, urlMod.Replace)
}
v := newUrl.Query()
for _, query := range rule.UrlMods.Query {
if query.Value == "" {
v.Del(query.Key)
continue
}
v.Set(query.Key, query.Value)
}
newUrl.RawQuery = v.Encode()
if rule.GoogleCache {
newUrl, err = url.Parse("https://webcache.googleusercontent.com/search?q=cache:" + newUrl.String())
if err != nil {
return "", err
}
}
return newUrl.String(), nil
}
func fetchSite(urlpath string, queries map[string]string) (string, *http.Request, *http.Response, error) {
urlQuery := "?"
if len(queries) > 0 {
for k, v := range queries {
urlQuery += k + "=" + v + "&"
}
}
urlQuery = strings.TrimSuffix(urlQuery, "&")
urlQuery = strings.TrimSuffix(urlQuery, "?")
u, err := url.Parse(urlpath)
if err != nil {
return "", nil, nil, err
}
if len(allowedDomains) > 0 && !StringInSlice(u.Host, allowedDomains) {
return "", nil, nil, fmt.Errorf("domain not allowed. %s not in %s", u.Host, allowedDomains)
}
if os.Getenv("LOG_URLS") == "true" {
log.Println(u.String() + urlQuery)
}
// Modify the URI according to ruleset
rule := fetchRule(u.Host, u.Path)
url, err := modifyURL(u.String()+urlQuery, rule)
if err != nil {
return "", nil, nil, err
}
// Fetch the site
client := &http.Client{}
req, _ := http.NewRequest("GET", url, nil)
if rule.Headers.UserAgent != "" {
req.Header.Set("User-Agent", rule.Headers.UserAgent)
} else {
req.Header.Set("User-Agent", UserAgent)
}
if rule.Headers.XForwardedFor != "" {
if rule.Headers.XForwardedFor != "none" {
req.Header.Set("X-Forwarded-For", rule.Headers.XForwardedFor)
}
} else {
req.Header.Set("X-Forwarded-For", ForwardedFor)
}
if rule.Headers.Referer != "" {
if rule.Headers.Referer != "none" {
req.Header.Set("Referer", rule.Headers.Referer)
}
} else {
req.Header.Set("Referer", u.String())
}
if rule.Headers.Cookie != "" {
req.Header.Set("Cookie", rule.Headers.Cookie)
}
resp, err := client.Do(req)
if err != nil {
return "", nil, nil, err
}
defer resp.Body.Close()
bodyB, err := io.ReadAll(resp.Body)
if err != nil {
return "", nil, nil, err
}
if rule.Headers.CSP != "" {
//log.Println(rule.Headers.CSP)
resp.Header.Set("Content-Security-Policy", rule.Headers.CSP)
}
//log.Print("rule", rule) TODO: Add a debug mode to print the rule
body := rewriteHtml(bodyB, u, rule)
return body, req, resp, nil
}
func rewriteHtml(bodyB []byte, u *url.URL, rule ruleset.Rule) string {
// Rewrite the HTML
body := string(bodyB)
// images
imagePattern := `
]*\s+)?src="(/)([^"]*)"`
re := regexp.MustCompile(imagePattern)
body = re.ReplaceAllString(body, fmt.Sprintf(`
]*\s+)?src="(/)([^"]*)"`
reScript := regexp.MustCompile(scriptPattern)
body = reScript.ReplaceAllString(body, fmt.Sprintf(`