Merge pull request #21 from joncrangle/csp-override

Allow the user to specify the Content Security Policy for a domain

README.md

@@ -120,13 +120,22 @@ It is possible to apply custom rules to modify the response. This can be used to
 See in [ruleset.yaml](ruleset.yaml) for an example.
 
 ```yaml
-- domain: www.example.com
+- domain: example.com          # Inbcludes all subdomains
+  domains:                     # Additional domains to apply the rule
+    - www.example.de
+    - www.beispiel.de
+  headers:
+    x-forwarded-for: none      # override X-Forwarded-For header or delete with none
+    referer: none              # override Referer header or delete with none
+    user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36
+    content-security-policy: script-src 'self'; # override response header
+    cookie: privacy=1
   regexRules:
     - match: <script\s+([^>]*\s+)?src="(/)([^"]*)"
       replace: <script $1 script="/https://www.example.com/$3"
   injections:
     - position: head # Position where to inject the code
-      append: | 
+      append: |      # possible keys: append, prepend, replace
         <script>
           window.localStorage.clear();
           console.log("test");
@@ -135,7 +144,7 @@ See in [ruleset.yaml](ruleset.yaml) for an example.
 - domain: www.anotherdomain.com # Domain where the rule applies
   paths:                        # Paths where the rule applies
     - /article
-  googleCache: false            # Search also in Google Cache
+  googleCache: false            # Use Google Cache to fetch the content
   regexRules:                   # Regex rules to apply
     - match: <script\s+([^>]*\s+)?src="(/)([^"]*)"
       replace: <script $1 script="/https://www.example.com/$3"

cmd/main.go

@@ -22,9 +22,13 @@ func main() {
 
 	parser := argparse.NewParser("ladder", "Every Wall needs a Ladder")
 
+	portEnv := os.Getenv("PORT")
+	if os.Getenv("PORT") == "" {
+		portEnv = "8080"
+	}
 	port := parser.String("p", "port", &argparse.Options{
 		Required: false,
-		Default:  os.Getenv("PORT"),
+		Default:  portEnv,
 		Help:     "Port the webserver will listen on"})
 
 	prefork := parser.Flag("P", "prefork", &argparse.Options{
@@ -43,6 +47,7 @@ func main() {
 	app := fiber.New(
 		fiber.Config{
 			Prefork: *prefork,
+			GETOnly: true,
 		},
 	)
 

docker-compose.yaml

@@ -3,13 +3,17 @@ services:
   ladder:
     image: ghcr.io/kubero-dev/ladder:latest
     container_name: ladder
-    build: .
+    #build: .
     #restart: always
     #command: sh -c ./ladder
     environment:
       - PORT=8080
       - RULESET=/app/ruleset.yaml
-      #- PREFORK=true
+      #- ALLOWED_DOMAINS_RULESET=false
+      #- EXPOSE_RULESET=true
+      #- PREFORK=false
+      #- DISABLE_FORM=fase
+      #- FORM_PATH=/app/form.html
       #- X_FORWARDED_FOR=66.249.66.1
       #- USER_AGENT=Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
       #- USERPASS=foo:bar
@@ -19,11 +23,4 @@ services:
       - "8080:8080"
     volumes:
       - ./ruleset.yaml:/app/ruleset.yaml
-    deploy:
-      resources:
-        limits:
-          cpus: "0.50"
-          memory: 512M
-        reservations:
-          cpus: "0.25"
-          memory: 128M
+      - ./handlers/form.html:/app/form.html

handlers/proxy.go

@@ -33,6 +33,8 @@ func ProxySite(c *fiber.Ctx) error {
 	}
 
 	c.Set("Content-Type", resp.Header.Get("Content-Type"))
+	c.Set("Content-Security-Policy", resp.Header.Get("Content-Security-Policy"))
+
 	return c.SendString(body)
 }
 
@@ -56,17 +58,49 @@ func fetchSite(urlpath string, queries map[string]string) (string, *http.Request
 		return "", nil, nil, fmt.Errorf("domain not allowed. %s not in %s", u.Host, allowedDomains)
 	}
 
-	if os.Getenv("LOG_URLS	") == "true" {
+	if os.Getenv("LOG_URLS") == "true" {
 		log.Println(u.String() + urlQuery)
 	}
 
+	rule := fetchRule(u.Host, u.Path)
+
+	if rule.GoogleCache {
+		u, err = url.Parse("https://webcache.googleusercontent.com/search?q=cache:" + u.String())
+		if err != nil {
+			return "", nil, nil, err
+		}
+	}
+
 	// Fetch the site
 	client := &http.Client{}
 	req, _ := http.NewRequest("GET", u.String()+urlQuery, nil)
-	req.Header.Set("User-Agent", UserAgent)
-	req.Header.Set("X-Forwarded-For", ForwardedFor)
-	req.Header.Set("Referer", u.String())
-	req.Header.Set("Host", u.Host)
+
+	if rule.Headers.UserAgent != "" {
+		req.Header.Set("User-Agent", rule.Headers.UserAgent)
+	} else {
+		req.Header.Set("User-Agent", UserAgent)
+	}
+
+	if rule.Headers.XForwardedFor != "" {
+		if rule.Headers.XForwardedFor != "none" {
+			req.Header.Set("X-Forwarded-For", rule.Headers.XForwardedFor)
+		}
+	} else {
+		req.Header.Set("X-Forwarded-For", ForwardedFor)
+	}
+
+	if rule.Headers.Referer != "" {
+		if rule.Headers.Referer != "none" {
+			req.Header.Set("Referer", rule.Headers.Referer)
+		}
+	} else {
+		req.Header.Set("Referer", u.String())
+	}
+
+	if rule.Headers.Cookie != "" {
+		req.Header.Set("Cookie", rule.Headers.Cookie)
+	}
+
 	resp, err := client.Do(req)
 
 	if err != nil {
@@ -79,11 +113,16 @@ func fetchSite(urlpath string, queries map[string]string) (string, *http.Request
 		return "", nil, nil, err
 	}
 
-	body := rewriteHtml(bodyB, u)
+	if rule.Headers.CSP != "" {
+		resp.Header.Set("Content-Security-Policy", rule.Headers.CSP)
+	}
+
+	log.Print("rule", rule)
+	body := rewriteHtml(bodyB, u, rule)
 	return body, req, resp, nil
 }
 
-func rewriteHtml(bodyB []byte, u *url.URL) string {
+func rewriteHtml(bodyB []byte, u *url.URL, rule Rule) string {
 	// Rewrite the HTML
 	body := string(bodyB)
 
@@ -104,7 +143,7 @@ func rewriteHtml(bodyB []byte, u *url.URL) string {
 	body = strings.ReplaceAll(body, "href=\"https://"+u.Host, "href=\"/https://"+u.Host+"/")
 
 	if os.Getenv("RULESET") != "" {
-		body = applyRules(u.Host, u.Path, body)
+		body = applyRules(body, rule)
 	}
 	return body
 }
@@ -155,75 +194,75 @@ func loadRules() RuleSet {
 		yaml.Unmarshal(yamlFile, &ruleSet)
 	}
 
+	domains := []string{}
 	for _, rule := range ruleSet {
-		//log.Println("Loaded rules for", rule.Domain)
+
+		domains = append(domains, rule.Domain)
+		domains = append(domains, rule.Domains...)
 		if os.Getenv("ALLOWED_DOMAINS_RULESET") == "true" {
-			allowedDomains = append(allowedDomains, rule.Domain)
+			allowedDomains = append(allowedDomains, domains...)
 		}
 	}
 
-	log.Println("Loaded rules for", len(ruleSet), "Domains")
+	log.Println("Loaded ", len(ruleSet), " rules for", len(domains), "Domains")
 	return ruleSet
 }
 
-func applyRules(domain string, path string, body string) string {
+func fetchRule(domain string, path string) Rule {
+	if len(rulesSet) == 0 {
+		return Rule{}
+	}
+	rule := Rule{}
+	for _, rule := range rulesSet {
+		domains := rule.Domains
+		if rule.Domain != "" {
+			domains = append(domains, rule.Domain)
+		}
+		for _, ruleDomain := range domains {
+			if ruleDomain == domain || strings.HasSuffix(domain, ruleDomain) {
+				if len(rule.Paths) > 0 && !StringInSlice(path, rule.Paths) {
+					continue
+				}
+				// return first match
+				return rule
+			}
+		}
+	}
+	return rule
+}
+
+func applyRules(body string, rule Rule) string {
 	if len(rulesSet) == 0 {
 		return body
 	}
 
-	for _, rule := range rulesSet {
-		if rule.Domain != domain {
-			continue
+	for _, regexRule := range rule.RegexRules {
+		re := regexp.MustCompile(regexRule.Match)
+		body = re.ReplaceAllString(body, regexRule.Replace)
+	}
+	for _, injection := range rule.Injections {
+		doc, err := goquery.NewDocumentFromReader(strings.NewReader(body))
+		if err != nil {
+			log.Fatal(err)
 		}
-		if len(rule.Paths) > 0 && !StringInSlice(path, rule.Paths) {
-			continue
+		if injection.Replace != "" {
+			doc.Find(injection.Position).ReplaceWithHtml(injection.Replace)
 		}
-		for _, regexRule := range rule.RegexRules {
-			re := regexp.MustCompile(regexRule.Match)
-			body = re.ReplaceAllString(body, regexRule.Replace)
+		if injection.Append != "" {
+			doc.Find(injection.Position).AppendHtml(injection.Append)
 		}
-		for _, injection := range rule.Injections {
-			doc, err := goquery.NewDocumentFromReader(strings.NewReader(body))
-			if err != nil {
-				log.Fatal(err)
-			}
-			if injection.Replace != "" {
-				doc.Find(injection.Position).ReplaceWithHtml(injection.Replace)
-			}
-			if injection.Append != "" {
-				doc.Find(injection.Position).AppendHtml(injection.Append)
-			}
-			if injection.Prepend != "" {
-				doc.Find(injection.Position).PrependHtml(injection.Prepend)
-			}
-			body, err = doc.Html()
-			if err != nil {
-				log.Fatal(err)
-			}
+		if injection.Prepend != "" {
+			doc.Find(injection.Position).PrependHtml(injection.Prepend)
+		}
+		body, err = doc.Html()
+		if err != nil {
+			log.Fatal(err)
 		}
 	}
 
 	return body
 }
 
-type Rule struct {
-	Match   string `yaml:"match"`
-	Replace string `yaml:"replace"`
-}
-
-type RuleSet []struct {
-	Domain      string   `yaml:"domain"`
-	Paths       []string `yaml:"paths,omitempty"`
-	GoogleCache bool     `yaml:"googleCache,omitempty"`
-	RegexRules  []Rule   `yaml:"regexRules"`
-	Injections  []struct {
-		Position string `yaml:"position"`
-		Append   string `yaml:"append"`
-		Prepend  string `yaml:"prepend"`
-		Replace  string `yaml:"replace"`
-	} `yaml:"injections"`
-}
-
 func StringInSlice(s string, list []string) bool {
 	for _, x := range list {
 		if strings.HasPrefix(s, x) {

handlers/proxy.test.go

@@ -51,7 +51,7 @@ func TestRewriteHtml(t *testing.T) {
 		</html>
 	`
 
-	actual := rewriteHtml(bodyB, u)
+	actual := rewriteHtml(bodyB, u, Rule{})
 	assert.Equal(t, expected, actual)
 }
 

handlers/types.go

@@ -0,0 +1,29 @@
+package handlers
+
+type Regex struct {
+	Match   string `yaml:"match"`
+	Replace string `yaml:"replace"`
+}
+
+type RuleSet []Rule
+
+type Rule struct {
+	Domain  string   `yaml:"domain,omitempty"`
+	Domains []string `yaml:"domains,omitempty"`
+	Paths   []string `yaml:"paths,omitempty"`
+	Headers struct {
+		UserAgent     string `yaml:"user-agent,omitempty"`
+		XForwardedFor string `yaml:"x-forwarded-for,omitempty"`
+		Referer       string `yaml:"referer,omitempty"`
+		Cookie        string `yaml:"cookie,omitempty"`
+		CSP           string `yaml:"content-security-policy,omitempty"`
+	} `yaml:"headers,omitempty"`
+	GoogleCache bool    `yaml:"googleCache,omitempty"`
+	RegexRules  []Regex `yaml:"regexRules"`
+	Injections  []struct {
+		Position string `yaml:"position"`
+		Append   string `yaml:"append"`
+		Prepend  string `yaml:"prepend"`
+		Replace  string `yaml:"replace"`
+	} `yaml:"injections"`
+}

ruleset.yaml

@@ -1,24 +1,32 @@
-- domain: www.example.com
+- domain: example.com
+  domains: 
+  - www.beispiel.de
+  googleCache: true
+  headers:
+    x-forwarded-for: none
+    referer: none
+    user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36
+    cookie: privacy=1
   regexRules:
     - match: <script\s+([^>]*\s+)?src="(/)([^"]*)"
       replace: <script $1 script="/https://www.example.com/$3"
   injections:
     - position: head # Position where to inject the code
-      append: | 
+      append: |
         <script>
           window.localStorage.clear();
           console.log("test");
           alert("Hello!");
         </script>
     - position: h1
-      replace: | 
+      replace: |
         <h1>An example with a ladder ;-)</h1>
 - domain: www.americanbanker.com
-  paths: 
+  paths:
     - /news
   injections:
     - position: head
-      append: | 
+      append: |
         <script>
           document.addEventListener("DOMContentLoaded", () => {
             const inlineGate = document.querySelector('.inline-gate');
@@ -30,7 +38,7 @@
           });
         </script>
 - domain: www.nzz.ch
-  paths: 
+  paths:
     - /international
     - /sport
     - /wirtschaft
@@ -46,10 +54,112 @@
     - /finanze
   injections:
     - position: head
-      append: | 
+      append: |
         <script>
           document.addEventListener("DOMContentLoaded", () => {
             const paywall = document.querySelector('.dynamic-regwall');
             removeDOMElement(paywall)
           });
-        </script>
+        </script>
+- domains: 
+  - www.architecturaldigest.com
+  - www.bonappetit.com
+  - www.cntraveler.com
+  - www.epicurious.com
+  - www.gq.com
+  - www.newyorker.com
+  - www.vanityfair.com
+  - www.vogue.com
+  - www.wired.com
+  injections:
+    - position: head
+      append: |
+        <script>
+          document.addEventListener("DOMContentLoaded", () => {
+            const banners = document.querySelectorAll('.paywall-bar, div[class^="MessageBannerWrapper-"');
+            banners.forEach(el => { el.remove(); });
+          });
+        </script>
+- domains: 
+  - www.nytimes.com
+  - www.time.com
+  headers:
+    ueser-agent: Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
+    cookie: nyt-a=; nyt-gdpr=0; nyt-geo=DE; nyt-privacy=1
+    referer: https://www.google.com/ 
+  injections:
+    - position: head
+      append: |
+        <script>
+          window.localStorage.clear();
+          document.addEventListener("DOMContentLoaded", () => {
+            const banners = document.querySelectorAll('div[data-testid="inline-message"], div[id^="ad-"], div[id^="leaderboard-"], div.expanded-dock, div.pz-ad-box, div[id="top-wrapper"], div[id="bottom-wrapper"]');
+            banners.forEach(el => { el.remove(); });
+          });
+        </script>
+- domains: 
+  - www.thestar.com
+  - www.niagarafallsreview.ca
+  - www.stcatharinesstandard.ca
+  - www.thepeterboroughexaminer.com
+  - www.therecord.com
+  - www.thespec.com
+  - www.wellandtribune.ca
+  injections:
+    - position: head
+      append: |
+        <script>
+          window.localStorage.clear();
+          document.addEventListener("DOMContentLoaded", () => {
+            const paywall = document.querySelectorAll('div.subscriber-offers');
+            paywall.forEach(el => { el.remove(); });
+            const subscriber_only = document.querySelectorAll('div.subscriber-only');
+            for (const elem of subscriber_only) {
+              if (elem.classList.contains('encrypted-content') && dompurify_loaded) {
+                const parser = new DOMParser();
+                const doc = parser.parseFromString('<div>' + DOMPurify.sanitize(unscramble(elem.innerText)) + '</div>', 'text/html');
+                const content_new = doc.querySelector('div');
+                elem.parentNode.replaceChild(content_new, elem);
+              }
+              elem.removeAttribute('style');
+              elem.removeAttribute('class');
+            }
+            const banners = document.querySelectorAll('div.subscription-required, div.redacted-overlay, div.subscriber-hide, div.tnt-ads-container');
+            banners.forEach(el => { el.remove(); });
+            const ads = document.querySelectorAll('div.tnt-ads-container, div[class*="adLabelWrapper"]');
+            ads.forEach(el => { el.remove(); });
+            const recommendations = document.querySelectorAll('div[id^="tncms-region-article"]');
+            recommendations.forEach(el => { el.remove(); });
+          });
+        </script>
+- domain: www.usatoday.com
+  injections:
+    - position: head
+      append: |
+        <script>
+          document.addEventListener("DOMContentLoaded", () => {
+            const banners = document.querySelectorAll('div.roadblock-container, .gnt_nb, [aria-label="advertisement"], div[id="main-frame-error"]');
+            banners.forEach(el => { el.remove(); });
+          });
+        </script>
+- domain: www.washingtonpost.com
+  injections:
+    - position: head
+      append: |
+        <script>
+          document.addEventListener("DOMContentLoaded", () => {
+            let paywall = document.querySelectorAll('div[data-qa$="-ad"], div[id="leaderboard-wrapper"], div[data-qa="subscribe-promo"]');
+            paywall.forEach(el => { el.remove(); });
+            const images = document.querySelectorAll('img');
+            images.forEach(image => { image.parentElement.style.filter = ''; });
+            const headimage = document.querySelectorAll('div .aspect-custom');
+            headimage.forEach(image => { image.style.filter = ''; });
+          });
+        </script>
+- domain: medium.com
+  headers:
+    referer: https://t.co/x?amp=1
+    x-forwarded-for: none
+    user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36
+    content-security-policy: script-src 'self';
+    cookie: