From 6c1f58e2e7643f727c6fc7592172657ac926231b Mon Sep 17 00:00:00 2001
From: Gianni Carafa <gianni.carafa@srf.ch>
Date: Thu, 9 Nov 2023 23:32:43 +0100
Subject: [PATCH] Add headers field in ruleset. Enable Google Cache.

---
 README.md              |   7 ++-
 cmd/main.go            |   1 +
 handlers/proxy.go      | 137 ++++++++++++++++++++++++-----------------
 handlers/proxy.test.go |   2 +-
 handlers/types.go      |  28 +++++++++
 ruleset.yaml           |  20 +++++-
 6 files changed, 135 insertions(+), 60 deletions(-)
 create mode 100644 handlers/types.go

diff --git a/README.md b/README.md
index 25cea80..3fede33 100644
--- a/README.md
+++ b/README.md
@@ -124,6 +124,11 @@ See in [ruleset.yaml](ruleset.yaml) for an example.
   domains:                     # Additional domains to apply the rule
     - www.example.com
     - www.beispiel.de
+  headers:
+    x-forwarded-for: none      # override X-Forwarded-For header or delete with none
+    referer: none              # override Referer header or delete with none   
+    user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36
+    cookie: privacy=1
   regexRules:
     - match: <script\s+([^>]*\s+)?src="(/)([^"]*)"
       replace: <script $1 script="/https://www.example.com/$3"
@@ -138,7 +143,7 @@ See in [ruleset.yaml](ruleset.yaml) for an example.
 - domain: www.anotherdomain.com # Domain where the rule applies
   paths:                        # Paths where the rule applies
     - /article
-  googleCache: false            # Search also in Google Cache
+  googleCache: false            # Use Google Cache to fetch the content
   regexRules:                   # Regex rules to apply
     - match: <script\s+([^>]*\s+)?src="(/)([^"]*)"
       replace: <script $1 script="/https://www.example.com/$3"
diff --git a/cmd/main.go b/cmd/main.go
index 239b1c4..c6c22e5 100644
--- a/cmd/main.go
+++ b/cmd/main.go
@@ -47,6 +47,7 @@ func main() {
 	app := fiber.New(
 		fiber.Config{
 			Prefork: *prefork,
+			GETOnly: true,
 		},
 	)
 
diff --git a/handlers/proxy.go b/handlers/proxy.go
index 22d82d8..67edbee 100644
--- a/handlers/proxy.go
+++ b/handlers/proxy.go
@@ -60,13 +60,45 @@ func fetchSite(urlpath string, queries map[string]string) (string, *http.Request
 		log.Println(u.String() + urlQuery)
 	}
 
+	rule := fetchRule(u.Host, u.Path)
+
+	if rule.GoogleCache {
+		u, err = url.Parse("https://webcache.googleusercontent.com/search?q=cache:" + u.String())
+		if err != nil {
+			return "", nil, nil, err
+		}
+	}
+
 	// Fetch the site
 	client := &http.Client{}
 	req, _ := http.NewRequest("GET", u.String()+urlQuery, nil)
-	req.Header.Set("User-Agent", UserAgent)
-	req.Header.Set("X-Forwarded-For", ForwardedFor)
-	req.Header.Set("Referer", u.String())
-	req.Header.Set("Host", u.Host)
+
+	if rule.Headers.UserAgent != "" {
+		req.Header.Set("User-Agent", rule.Headers.UserAgent)
+	} else {
+		req.Header.Set("User-Agent", UserAgent)
+	}
+
+	if rule.Headers.XForwardedFor != "" {
+		if rule.Headers.XForwardedFor != "none" {
+			req.Header.Set("X-Forwarded-For", rule.Headers.XForwardedFor)
+		}
+	} else {
+		req.Header.Set("X-Forwarded-For", ForwardedFor)
+	}
+
+	if rule.Headers.Referer != "" {
+		if rule.Headers.Referer != "none" {
+			req.Header.Set("Referer", rule.Headers.Referer)
+		}
+	} else {
+		req.Header.Set("Referer", u.String())
+	}
+
+	if rule.Headers.Cookie != "" {
+		req.Header.Set("Cookie", rule.Headers.Cookie)
+	}
+
 	resp, err := client.Do(req)
 
 	if err != nil {
@@ -79,11 +111,12 @@ func fetchSite(urlpath string, queries map[string]string) (string, *http.Request
 		return "", nil, nil, err
 	}
 
-	body := rewriteHtml(bodyB, u)
+	log.Print("rule", rule)
+	body := rewriteHtml(bodyB, u, rule)
 	return body, req, resp, nil
 }
 
-func rewriteHtml(bodyB []byte, u *url.URL) string {
+func rewriteHtml(bodyB []byte, u *url.URL, rule Rule) string {
 	// Rewrite the HTML
 	body := string(bodyB)
 
@@ -104,7 +137,7 @@ func rewriteHtml(bodyB []byte, u *url.URL) string {
 	body = strings.ReplaceAll(body, "href=\"https://"+u.Host, "href=\"/https://"+u.Host+"/")
 
 	if os.Getenv("RULESET") != "" {
-		body = applyRules(u.Host, u.Path, body)
+		body = applyRules(body, rule)
 	}
 	return body
 }
@@ -169,69 +202,59 @@ func loadRules() RuleSet {
 	return ruleSet
 }
 
-func applyRules(domain string, path string, body string) string {
+func fetchRule(domain string, path string) Rule {
 	if len(rulesSet) == 0 {
-		return body
+		return Rule{}
 	}
-
+	rule := Rule{}
 	for _, rule := range rulesSet {
 		domains := rule.Domains
 		domains = append(domains, rule.Domain)
 		for _, ruleDomain := range domains {
-			if ruleDomain != domain {
-				continue
-			}
-			if len(rule.Paths) > 0 && !StringInSlice(path, rule.Paths) {
-				continue
-			}
-			for _, regexRule := range rule.RegexRules {
-				re := regexp.MustCompile(regexRule.Match)
-				body = re.ReplaceAllString(body, regexRule.Replace)
-			}
-			for _, injection := range rule.Injections {
-				doc, err := goquery.NewDocumentFromReader(strings.NewReader(body))
-				if err != nil {
-					log.Fatal(err)
-				}
-				if injection.Replace != "" {
-					doc.Find(injection.Position).ReplaceWithHtml(injection.Replace)
-				}
-				if injection.Append != "" {
-					doc.Find(injection.Position).AppendHtml(injection.Append)
-				}
-				if injection.Prepend != "" {
-					doc.Find(injection.Position).PrependHtml(injection.Prepend)
-				}
-				body, err = doc.Html()
-				if err != nil {
-					log.Fatal(err)
-				}
+			if ruleDomain == domain {
+				if len(rule.Paths) > 0 && !StringInSlice(path, rule.Paths) {
+					continue
+				}
+				// return first match
+				return rule
 			}
 		}
 	}
+	return rule
+}
+
+func applyRules(body string, rule Rule) string {
+	if len(rulesSet) == 0 {
+		return body
+	}
+
+	for _, regexRule := range rule.RegexRules {
+		re := regexp.MustCompile(regexRule.Match)
+		body = re.ReplaceAllString(body, regexRule.Replace)
+	}
+	for _, injection := range rule.Injections {
+		doc, err := goquery.NewDocumentFromReader(strings.NewReader(body))
+		if err != nil {
+			log.Fatal(err)
+		}
+		if injection.Replace != "" {
+			doc.Find(injection.Position).ReplaceWithHtml(injection.Replace)
+		}
+		if injection.Append != "" {
+			doc.Find(injection.Position).AppendHtml(injection.Append)
+		}
+		if injection.Prepend != "" {
+			doc.Find(injection.Position).PrependHtml(injection.Prepend)
+		}
+		body, err = doc.Html()
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
 
 	return body
 }
 
-type Rule struct {
-	Match   string `yaml:"match"`
-	Replace string `yaml:"replace"`
-}
-
-type RuleSet []struct {
-	Domain      string   `yaml:"domain"`
-	Domains     []string `yaml:"domains,omitempty"`
-	Paths       []string `yaml:"paths,omitempty"`
-	GoogleCache bool     `yaml:"googleCache,omitempty"`
-	RegexRules  []Rule   `yaml:"regexRules"`
-	Injections  []struct {
-		Position string `yaml:"position"`
-		Append   string `yaml:"append"`
-		Prepend  string `yaml:"prepend"`
-		Replace  string `yaml:"replace"`
-	} `yaml:"injections"`
-}
-
 func StringInSlice(s string, list []string) bool {
 	for _, x := range list {
 		if strings.HasPrefix(s, x) {
diff --git a/handlers/proxy.test.go b/handlers/proxy.test.go
index a4f6dc7..af66e54 100644
--- a/handlers/proxy.test.go
+++ b/handlers/proxy.test.go
@@ -51,7 +51,7 @@ func TestRewriteHtml(t *testing.T) {
 		</html>
 	`
 
-	actual := rewriteHtml(bodyB, u)
+	actual := rewriteHtml(bodyB, u, Rule{})
 	assert.Equal(t, expected, actual)
 }
 
diff --git a/handlers/types.go b/handlers/types.go
new file mode 100644
index 0000000..eacc722
--- /dev/null
+++ b/handlers/types.go
@@ -0,0 +1,28 @@
+package handlers
+
+type Regex struct {
+	Match   string `yaml:"match"`
+	Replace string `yaml:"replace"`
+}
+
+type RuleSet []Rule
+
+type Rule struct {
+	Domain  string   `yaml:"domain"`
+	Domains []string `yaml:"domains,omitempty"`
+	Paths   []string `yaml:"paths,omitempty"`
+	Headers struct {
+		UserAgent     string `yaml:"user-agent,omitempty"`
+		XForwardedFor string `yaml:"x-forwarded-for,omitempty"`
+		Referer       string `yaml:"referer,omitempty"`
+		Cookie        string `yaml:"cookie,omitempty"`
+	} `yaml:"headers,omitempty"`
+	GoogleCache bool    `yaml:"googleCache,omitempty"`
+	RegexRules  []Regex `yaml:"regexRules"`
+	Injections  []struct {
+		Position string `yaml:"position"`
+		Append   string `yaml:"append"`
+		Prepend  string `yaml:"prepend"`
+		Replace  string `yaml:"replace"`
+	} `yaml:"injections"`
+}
diff --git a/ruleset.yaml b/ruleset.yaml
index f1cc8f7..bf8b346 100644
--- a/ruleset.yaml
+++ b/ruleset.yaml
@@ -1,6 +1,12 @@
 - domain: www.example.com
   domains: 
-  - www.beispiel.com
+  - www.beispiel.de
+  googleCache: true
+  headers:
+    x-forwarded-for: none
+    referer: none
+    user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36
+    cookie: privacy=1
   regexRules:
     - match: <script\s+([^>]*\s+)?src="(/)([^"]*)"
       replace: <script $1 script="/https://www.example.com/$3"
@@ -77,6 +83,10 @@
 - domains: 
   - www.nytimes.com
   - www.time.com
+  headers:
+    ueser-agent: Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
+    cookie: nyt-a=; nyt-gdpr=0; nyt-geo=DE; nyt-privacy=1
+    referer: https://www.google.com/ 
   injections:
     - position: head
       append: |
@@ -146,3 +156,11 @@
             headimage.forEach(image => { image.style.filter = ''; });
           });
         </script>
+- domain: www.medium.com
+  domains:
+  - medium.com
+  headers:
+    referer: https://t.co/x?amp=1
+    x-forwarded-for: none
+    user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36
+    cookie: 
\ No newline at end of file