merge main
This commit is contained in:
Kevin Pham
@@ -1,38 +1,13 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"ladder/pkg/ruleset"
|
||||
"ladder/proxychain"
|
||||
rx "ladder/proxychain/requestmodifers"
|
||||
tx "ladder/proxychain/responsemodifers"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/gofiber/fiber/v2"
|
||||
)
|
||||
|
||||
var (
|
||||
UserAgent = getenv("USER_AGENT", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)")
|
||||
ForwardedFor = getenv("X_FORWARDED_FOR", "66.249.66.1")
|
||||
rulesSet = ruleset.NewRulesetFromEnv()
|
||||
allowedDomains = []string{}
|
||||
)
|
||||
|
||||
func init() {
|
||||
allowedDomains = strings.Split(os.Getenv("ALLOWED_DOMAINS"), ",")
|
||||
if os.Getenv("ALLOWED_DOMAINS_RULESET") == "true" {
|
||||
allowedDomains = append(allowedDomains, rulesSet.Domains()...)
|
||||
}
|
||||
}
|
||||
|
||||
type ProxyOptions struct {
|
||||
RulesetPath string
|
||||
Verbose bool
|
||||
@@ -71,220 +46,4 @@ func NewProxySiteHandler(opts *ProxyOptions) fiber.Handler {
|
||||
|
||||
return proxychain
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func modifyURL(uri string, rule ruleset.Rule) (string, error) {
|
||||
newUrl, err := url.Parse(uri)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
for _, urlMod := range rule.UrlMods.Domain {
|
||||
re := regexp.MustCompile(urlMod.Match)
|
||||
newUrl.Host = re.ReplaceAllString(newUrl.Host, urlMod.Replace)
|
||||
}
|
||||
|
||||
for _, urlMod := range rule.UrlMods.Path {
|
||||
re := regexp.MustCompile(urlMod.Match)
|
||||
newUrl.Path = re.ReplaceAllString(newUrl.Path, urlMod.Replace)
|
||||
}
|
||||
|
||||
v := newUrl.Query()
|
||||
for _, query := range rule.UrlMods.Query {
|
||||
if query.Value == "" {
|
||||
v.Del(query.Key)
|
||||
continue
|
||||
}
|
||||
v.Set(query.Key, query.Value)
|
||||
}
|
||||
newUrl.RawQuery = v.Encode()
|
||||
|
||||
if rule.GoogleCache {
|
||||
newUrl, err = url.Parse("https://webcache.googleusercontent.com/search?q=cache:" + newUrl.String())
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
|
||||
return newUrl.String(), nil
|
||||
}
|
||||
|
||||
func fetchSite(urlpath string, queries map[string]string) (string, *http.Request, *http.Response, error) {
|
||||
urlQuery := "?"
|
||||
if len(queries) > 0 {
|
||||
for k, v := range queries {
|
||||
urlQuery += k + "=" + v + "&"
|
||||
}
|
||||
}
|
||||
urlQuery = strings.TrimSuffix(urlQuery, "&")
|
||||
urlQuery = strings.TrimSuffix(urlQuery, "?")
|
||||
|
||||
u, err := url.Parse(urlpath)
|
||||
if err != nil {
|
||||
return "", nil, nil, err
|
||||
}
|
||||
|
||||
if len(allowedDomains) > 0 && !StringInSlice(u.Host, allowedDomains) {
|
||||
return "", nil, nil, fmt.Errorf("domain not allowed. %s not in %s", u.Host, allowedDomains)
|
||||
}
|
||||
|
||||
if os.Getenv("LOG_URLS") == "true" {
|
||||
log.Println(u.String() + urlQuery)
|
||||
}
|
||||
|
||||
// Modify the URI according to ruleset
|
||||
rule := fetchRule(u.Host, u.Path)
|
||||
url, err := modifyURL(u.String()+urlQuery, rule)
|
||||
if err != nil {
|
||||
return "", nil, nil, err
|
||||
}
|
||||
|
||||
// Fetch the site
|
||||
client := &http.Client{}
|
||||
req, _ := http.NewRequest("GET", url, nil)
|
||||
|
||||
if rule.Headers.UserAgent != "" {
|
||||
req.Header.Set("User-Agent", rule.Headers.UserAgent)
|
||||
} else {
|
||||
req.Header.Set("User-Agent", UserAgent)
|
||||
}
|
||||
|
||||
if rule.Headers.XForwardedFor != "" {
|
||||
if rule.Headers.XForwardedFor != "none" {
|
||||
req.Header.Set("X-Forwarded-For", rule.Headers.XForwardedFor)
|
||||
}
|
||||
} else {
|
||||
req.Header.Set("X-Forwarded-For", ForwardedFor)
|
||||
}
|
||||
|
||||
if rule.Headers.Referer != "" {
|
||||
if rule.Headers.Referer != "none" {
|
||||
req.Header.Set("Referer", rule.Headers.Referer)
|
||||
}
|
||||
} else {
|
||||
req.Header.Set("Referer", u.String())
|
||||
}
|
||||
|
||||
if rule.Headers.Cookie != "" {
|
||||
req.Header.Set("Cookie", rule.Headers.Cookie)
|
||||
}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "", nil, nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
bodyB, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", nil, nil, err
|
||||
}
|
||||
|
||||
if rule.Headers.CSP != "" {
|
||||
//log.Println(rule.Headers.CSP)
|
||||
resp.Header.Set("Content-Security-Policy", rule.Headers.CSP)
|
||||
}
|
||||
|
||||
//log.Print("rule", rule) TODO: Add a debug mode to print the rule
|
||||
body := rewriteHtml(bodyB, u, rule)
|
||||
return body, req, resp, nil
|
||||
}
|
||||
|
||||
func rewriteHtml(bodyB []byte, u *url.URL, rule ruleset.Rule) string {
|
||||
// Rewrite the HTML
|
||||
body := string(bodyB)
|
||||
|
||||
// images
|
||||
imagePattern := `<img\s+([^>]*\s+)?src="(/)([^"]*)"`
|
||||
re := regexp.MustCompile(imagePattern)
|
||||
body = re.ReplaceAllString(body, fmt.Sprintf(`<img $1 src="%s$3"`, "/https://"+u.Host+"/"))
|
||||
|
||||
// scripts
|
||||
scriptPattern := `<script\s+([^>]*\s+)?src="(/)([^"]*)"`
|
||||
reScript := regexp.MustCompile(scriptPattern)
|
||||
body = reScript.ReplaceAllString(body, fmt.Sprintf(`<script $1 script="%s$3"`, "/https://"+u.Host+"/"))
|
||||
|
||||
// body = strings.ReplaceAll(body, "srcset=\"/", "srcset=\"/https://"+u.Host+"/") // TODO: Needs a regex to rewrite the URL's
|
||||
body = strings.ReplaceAll(body, "href=\"/", "href=\"/https://"+u.Host+"/")
|
||||
body = strings.ReplaceAll(body, "url('/", "url('/https://"+u.Host+"/")
|
||||
body = strings.ReplaceAll(body, "url(/", "url(/https://"+u.Host+"/")
|
||||
body = strings.ReplaceAll(body, "href=\"https://"+u.Host, "href=\"/https://"+u.Host+"/")
|
||||
|
||||
if os.Getenv("RULESET") != "" {
|
||||
body = applyRules(body, rule)
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
func getenv(key, fallback string) string {
|
||||
value := os.Getenv(key)
|
||||
if len(value) == 0 {
|
||||
return fallback
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func fetchRule(domain string, path string) ruleset.Rule {
|
||||
if len(rulesSet) == 0 {
|
||||
return ruleset.Rule{}
|
||||
}
|
||||
rule := ruleset.Rule{}
|
||||
for _, rule := range rulesSet {
|
||||
domains := rule.Domains
|
||||
if rule.Domain != "" {
|
||||
domains = append(domains, rule.Domain)
|
||||
}
|
||||
for _, ruleDomain := range domains {
|
||||
if ruleDomain == domain || strings.HasSuffix(domain, ruleDomain) {
|
||||
if len(rule.Paths) > 0 && !StringInSlice(path, rule.Paths) {
|
||||
continue
|
||||
}
|
||||
// return first match
|
||||
return rule
|
||||
}
|
||||
}
|
||||
}
|
||||
return rule
|
||||
}
|
||||
|
||||
func applyRules(body string, rule ruleset.Rule) string {
|
||||
if len(rulesSet) == 0 {
|
||||
return body
|
||||
}
|
||||
|
||||
for _, regexRule := range rule.RegexRules {
|
||||
re := regexp.MustCompile(regexRule.Match)
|
||||
body = re.ReplaceAllString(body, regexRule.Replace)
|
||||
}
|
||||
for _, injection := range rule.Injections {
|
||||
doc, err := goquery.NewDocumentFromReader(strings.NewReader(body))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if injection.Replace != "" {
|
||||
doc.Find(injection.Position).ReplaceWithHtml(injection.Replace)
|
||||
}
|
||||
if injection.Append != "" {
|
||||
doc.Find(injection.Position).AppendHtml(injection.Append)
|
||||
}
|
||||
if injection.Prepend != "" {
|
||||
doc.Find(injection.Position).PrependHtml(injection.Prepend)
|
||||
}
|
||||
body, err = doc.Html()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
return body
|
||||
}
|
||||
|
||||
func StringInSlice(s string, list []string) bool {
|
||||
for _, x := range list {
|
||||
if strings.HasPrefix(s, x) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -1,59 +0,0 @@
|
||||
// BEGIN: 6f8b3f5d5d5d
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"ladder/pkg/ruleset"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"testing"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestProxySite(t *testing.T) {
|
||||
app := fiber.New()
|
||||
app.Get("/:url", NewProxySiteHandler(nil))
|
||||
|
||||
req := httptest.NewRequest("GET", "/https://example.com", nil)
|
||||
resp, err := app.Test(req)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
}
|
||||
|
||||
func TestRewriteHtml(t *testing.T) {
|
||||
bodyB := []byte(`
|
||||
<html>
|
||||
<head>
|
||||
<title>Test Page</title>
|
||||
</head>
|
||||
<body>
|
||||
<img src="/image.jpg">
|
||||
<script src="/script.js"></script>
|
||||
<a href="/about">About Us</a>
|
||||
<div style="background-image: url('/background.jpg')"></div>
|
||||
</body>
|
||||
</html>
|
||||
`)
|
||||
u := &url.URL{Host: "example.com"}
|
||||
|
||||
expected := `
|
||||
<html>
|
||||
<head>
|
||||
<title>Test Page</title>
|
||||
</head>
|
||||
<body>
|
||||
<img src="/https://example.com/image.jpg">
|
||||
<script script="/https://example.com/script.js"></script>
|
||||
<a href="/https://example.com/about">About Us</a>
|
||||
<div style="background-image: url('/https://example.com/background.jpg')"></div>
|
||||
</body>
|
||||
</html>
|
||||
`
|
||||
|
||||
actual := rewriteHtml(bodyB, u, ruleset.Rule{})
|
||||
assert.Equal(t, expected, actual)
|
||||
}
|
||||
|
||||
// END: 6f8b3f5d5d5d
|
||||
Reference in New Issue
Block a user