WIP: write version on build to form HTML

Fix typo

.github/workflows/build-css.yaml

@@ -0,0 +1,42 @@
+name: Build Tailwind CSS
+
+on:
+  push:
+    paths:
+      - "handlers/form.html"
+  workflow_dispatch:
+
+jobs:
+  tailwindbuilder:
+    permissions:
+      # Give the default GITHUB_TOKEN write permission to commit and push the
+      # added or changed files to the repository.
+      contents: write
+      
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Install pnpm
+        uses: pnpm/action-setup@v2
+        with:
+          version: 8
+      -
+        name: Build Tailwind CSS
+        run: pnpm build
+      -
+        name: Commit generated stylesheet
+        run: |
+          if git diff --quiet cmd/styles.css; then
+            echo "No changes to commit."
+            exit 0
+          else
+            echo "Changes detected, committing..."
+            git config --global user.name "Github action"
+            git config --global user.email "username@users.noreply.github.com"
+            git add cmd
+            git commit -m "Generated stylesheet"
+            git push
+          fi

.github/workflows/release-binaries.yaml

@@ -22,7 +22,11 @@ jobs:
       -
         name: Set version
         run: |
-          echo -n $(git describe --tags --abbrev=0) > handlers/VERSION
+          VERSION=$(git describe --tags --abbrev=0)
+          echo -n $VERSION > handlers/VERSION
+          sed -i 's\VERSION\${VERSION}\g' handlers/form.html
+          echo handlers/form.html >> .gitignore
+          echo .gitignore >> .gitignore
       -
         name: Set up Go
         uses: actions/setup-go@v3

.github/workflows/release-docker.yaml

@@ -42,7 +42,11 @@ jobs:
       - name: Set version
         id: version
         run: |
-          echo ${GITHUB_REF#refs/tags/v} > handlers/VERSION
+          VERSION=$(git describe --tags --abbrev=0)
+          echo -n $VERSION > handlers/VERSION
+          sed -i 's\VERSION\${VERSION}\g' handlers/form.html
+          echo handlers/form.html >> .gitignore
+          echo .gitignore >> .gitignore
 
       # Install the cosign tool except on PR
       # https://github.com/sigstore/cosign-installer

.gitignore

@@ -2,3 +2,4 @@
 ladder
 
 VERSION
+output.css

README.md

@@ -17,7 +17,7 @@ Freedom of information is an essential pillar of democracy and informed decision
 ### Features
 - [x] Bypass Paywalls
 - [x] Remove CORS headers from responses, assets, and images ...
-- [x] Apply domain based ruleset/code to modify response
+- [x] Apply domain based ruleset/code to modify response / requested URL
 - [x] Keep site browsable
 - [x] API
 - [x] Fetch RAW HTML
@@ -38,10 +38,10 @@ Freedom of information is an essential pillar of democracy and informed decision
 - [ ] A key to share only one URL
 
 ### Limitations
-Certain sites may display missing images or encounter formatting issues. This can be attributed to the site's reliance on JavaScript or CSS for image and resource loading, which presents a limitation when accessed through this proxy. If you prefer a full experience, please consider buying a subscription for the site.
-
 Some sites do not expose their content to search engines, which means that the proxy cannot access the content. A future version will try to fetch the content from Google Cache.
 
+Certain sites may display missing images or encounter formatting issues. This can be attributed to the site's reliance on JavaScript or CSS for image and resource loading, which presents a limitation when accessed through this proxy. If you prefer a full experience, please consider buying a subscription for the site.
+
 ## Installation
 
 > **Warning:** If your instance will be publicly accessible, make sure to enable Basic Auth. This will prevent unauthorized users from using your proxy. If you do not enable Basic Auth, anyone can use your proxy to browse nasty/illegal stuff. And you will be responsible for it.
@@ -115,12 +115,12 @@ http://localhost:8080/ruleset
 
 ### Ruleset
 
-It is possible to apply custom rules to modify the response. This can be used to remove unwanted or modify elements from the page. The ruleset is a YAML file that contains a list of rules for each domain and is loaded on startup
+It is possible to apply custom rules to modify the response or the requested URL. This can be used to remove unwanted or modify elements from the page. The ruleset is a YAML file that contains a list of rules for each domain and is loaded on startup
 
 See in [ruleset.yaml](ruleset.yaml) for an example.
 
 ```yaml
-- domain: example.com          # Inbcludes all subdomains
+- domain: example.com          # Includes all subdomains
   domains:                     # Additional domains to apply the rule
     - www.example.de
     - www.beispiel.de
@@ -154,5 +154,30 @@ See in [ruleset.yaml](ruleset.yaml) for an example.
         <h1>My Custom Title</h1>
     - position: .left-content article # Position where to inject the code into DOM
       prepend: | 
-        <h2>Suptitle</h2>
+        <h2>Subtitle</h2>
+- domain: demo.com
+  headers:
+    content-security-policy: script-src 'self';
+    user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36
+  urlMods:              # Modify the URL
+    query:              
+      - key: amp        # (this will append ?amp=1 to the URL)
+        value: 1 
+    domain:             
+      - match: www      # regex to match part of domain
+        replace: amp    # (this would modify the domain from www.demo.de to amp.demo.de)
+    path:               
+      - match: ^        # regex to match part of path
+        replace: /amp/  # (modify the url from https://www.demo.com/article/ to https://www.demo.de/amp/article/)
 ```
+
+## Development
+
+To run a development server at http://localhost:8080:
+
+```bash
+echo "DEV" > handler/VERSION
+RULESET="./ruleset.yaml" go run cmd/main.go
+```
+
+This project uses [pnpm](https://pnpm.io/) to build a stylesheet with the [Tailwind CSS](https://tailwindcss.com/) classes. For local development, if you modify styles in `form.html`, run `pnpm build` to generate a new stylesheet.

cmd/main.go

@@ -1,7 +1,7 @@
 package main
 
 import (
-	_ "embed"
+	"embed"
 	"fmt"
 	"log"
 	"os"
@@ -17,6 +17,8 @@ import (
 
 //go:embed favicon.ico
 var faviconData string
+//go:embed styles.css
+var cssData embed.FS
 
 func main() {
 	parser := argparse.NewParser("ladder", "Every Wall needs a Ladder")
@@ -75,11 +77,18 @@ func main() {
 	}
 
 	app.Get("/", handlers.Form)
+	app.Get("/styles.css", func(c *fiber.Ctx) error {
+		cssData, err := cssData.ReadFile("styles.css")
+		if err != nil {
+			return c.Status(fiber.StatusInternalServerError).SendString("Internal Server Error")
+		}
+		c.Set("Content-Type", "text/css")
+		return c.Send(cssData)
+	})
 	app.Get("ruleset", handlers.Ruleset)
 
 	app.Get("raw/*", handlers.Raw)
 	app.Get("api/*", handlers.Api)
-	app.Get("ruleset", handlers.Raw)
 	app.Get("/*", handlers.ProxySite)
 
 	log.Fatal(app.Listen(":" + *port))

handlers/form.html

@@ -3,8 +3,9 @@
 
 <head>
     <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <title>ladder</title>
-    <script src="https://cdn.tailwindcss.com"></script>
+    <link rel="stylesheet" href="/styles.css">
 </head>
 
 <body class="antialiased text-slate-500 dark:text-slate-400 bg-white dark:bg-slate-900">
@@ -16,24 +17,24 @@
         <header>
             <h1 class="text-center text-3xl sm:text-4xl font-extrabold text-slate-900 tracking-tight dark:text-slate-200">ladddddddder</h1>
         </header>
-        <form id="inputForm" method="get" class="mx-4">
+        <form id="inputForm" method="get" class="mx-4 relative">
             <div>
                 <input type="text" id="inputField" placeholder="Proxy Search" name="inputField" class="w-full text-sm leading-6 text-slate-400 rounded-md ring-1 ring-slate-900/10 shadow-sm py-1.5 pl-2 pr-3 hover:ring-slate-300 dark:bg-slate-800 dark:highlight-white/5 dark:hover:bg-slate-700" required autofocus>
+                <button id="clearButton" type="button" aria-label="Clear Search" title="Clear Search" class="hidden absolute inset-y-0 right-0 items-center pr-2 hover:text-slate-400 hover:dark:text-slate-300" tabindex="-1">
+                    <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round""><path d="M18 6 6 18"/><path d="m6 6 12 12"/></svg>
+                </button>
             </div>
         </form>
-        <footer class="mt-10 text-center text-slate-600 dark:text-slate-400">
+        <footer class="mt-10 mx-4 text-center text-slate-600 dark:text-slate-400">
             <p>
                 Code Licensed Under GPL v3.0 |
-                <a href="https://github.com/everywall/ladder" class="hover:text-blue-500 hover:underline underline-offset-2 transition-colors duration-300">View Source</a> |
+                <a href="https://github.com/everywall/ladder" class="hover:text-blue-500 hover:underline underline-offset-2 transition-colors duration-300">Source</a> |
-                <a href="https://github.com/everywall" class="hover:text-blue-500 hover:underline underline-offset-2 transition-colors duration-300">Everywall</a>
+                <a href="https://github.com/everywall/ladder/releases" class="hover:text-blue-500 hover:underline underline-offset-2 transition-colors duration-300">VERSION</a>
             </p>
         </footer>
     </div>
 
     <script>
-        document.addEventListener('DOMContentLoaded', function () {
-            M.AutoInit();
-        });
         document.getElementById('inputForm').addEventListener('submit', function (e) {
             e.preventDefault();
             let url = document.getElementById('inputField').value;
@@ -43,6 +44,19 @@
             window.location.href = '/' + url;
             return false;
         });
+        document.getElementById('inputField').addEventListener('input', function() {
+            const clearButton = document.getElementById('clearButton');
+            if (this.value.trim().length > 0) {
+                clearButton.style.display = 'block';
+            } else {
+                clearButton.style.display = 'none';
+            }
+        });
+        document.getElementById('clearButton').addEventListener('click', function() {
+            document.getElementById('inputField').value = '';
+            this.style.display = 'none';
+            document.getElementById('inputField').focus();
+        });
     </script>
 
     <style>

handlers/proxy.go

@@ -22,9 +22,65 @@ var (
 	allowedDomains = strings.Split(os.Getenv("ALLOWED_DOMAINS"), ",")
 )
 
+// extracts a URL from the request ctx. If the URL in the request
+// is a relative path, it reconstructs the full URL using the referer header.
+func extractUrl(c *fiber.Ctx) (string, error) {
+	// try to extract url-encoded
+	reqUrl, err := url.QueryUnescape(c.Params("*"))
+	if err != nil {
+		// fallback
+		reqUrl = c.Params("*")
+	}
+
+	// Extract the actual path from req ctx
+	urlQuery, err := url.Parse(reqUrl)
+	if err != nil {
+		return "", fmt.Errorf("error parsing request URL '%s': %v", reqUrl, err)
+	}
+
+	isRelativePath := urlQuery.Scheme == ""
+
+	// eg: https://localhost:8080/images/foobar.jpg -> https://realsite.com/images/foobar.jpg
+	if isRelativePath {
+		// Parse the referer URL from the request header.
+		refererUrl, err := url.Parse(c.Get("referer"))
+		if err != nil {
+			return "", fmt.Errorf("error parsing referer URL from req: '%s': %v", reqUrl, err)
+		}
+
+		// Extract the real url from referer path
+		realUrl, err := url.Parse(strings.TrimPrefix(refererUrl.Path, "/"))
+		if err != nil {
+			return "", fmt.Errorf("error parsing real URL from referer '%s': %v", refererUrl.Path, err)
+		}
+
+		// reconstruct the full URL using the referer's scheme, host, and the relative path / queries
+		fullUrl := &url.URL{
+			Scheme:   realUrl.Scheme,
+			Host:     realUrl.Host,
+			Path:     urlQuery.Path,
+			RawQuery: urlQuery.RawQuery,
+		}
+
+		if os.Getenv("LOG_URLS") == "true" {
+			log.Printf("modified relative URL: '%s' -> '%s'", reqUrl, fullUrl.String())
+		}
+		return fullUrl.String(), nil
+
+	}
+
+	// default behavior:
+	// eg: https://localhost:8080/https://realsite.com/images/foobar.jpg -> https://realsite.com/images/foobar.jpg
+	return urlQuery.String(), nil
+
+}
+
 func ProxySite(c *fiber.Ctx) error {
 	// Get the url from the URL
-	url := c.Params("*")
+	url, err := extractUrl(c)
+	if err != nil {
+		log.Println("ERROR In URL extraction:", err)
+	}
 
 	queries := c.Queries()
 	body, _, resp, err := fetchSite(url, queries)
@@ -40,6 +96,42 @@ func ProxySite(c *fiber.Ctx) error {
 	return c.SendString(body)
 }
 
+func modifyURL(uri string, rule Rule) (string, error) {
+	newUrl, err := url.Parse(uri)
+	if err != nil {
+		return "", err
+	}
+
+	for _, urlMod := range rule.UrlMods.Domain {
+		re := regexp.MustCompile(urlMod.Match)
+		newUrl.Host = re.ReplaceAllString(newUrl.Host, urlMod.Replace)
+	}
+
+	for _, urlMod := range rule.UrlMods.Path {
+		re := regexp.MustCompile(urlMod.Match)
+		newUrl.Path = re.ReplaceAllString(newUrl.Path, urlMod.Replace)
+	}
+
+	v := newUrl.Query()
+	for _, query := range rule.UrlMods.Query {
+		if query.Value == "" {
+			v.Del(query.Key)
+			continue
+		}
+		v.Set(query.Key, query.Value)
+	}
+	newUrl.RawQuery = v.Encode()
+
+	if rule.GoogleCache {
+		newUrl, err = url.Parse("https://webcache.googleusercontent.com/search?q=cache:" + newUrl.String())
+		if err != nil {
+			return "", err
+		}
+	}
+
+	return newUrl.String(), nil
+}
+
 func fetchSite(urlpath string, queries map[string]string) (string, *http.Request, *http.Response, error) {
 	urlQuery := "?"
 	if len(queries) > 0 {
@@ -63,18 +155,16 @@ func fetchSite(urlpath string, queries map[string]string) (string, *http.Request
 		log.Println(u.String() + urlQuery)
 	}
 
+	// Modify the URI according to ruleset
 	rule := fetchRule(u.Host, u.Path)
-
+	url, err := modifyURL(u.String()+urlQuery, rule)
-	if rule.GoogleCache {
-		u, err = url.Parse("https://webcache.googleusercontent.com/search?q=cache:" + u.String())
 	if err != nil {
 		return "", nil, nil, err
 	}
-	}
 
 	// Fetch the site
 	client := &http.Client{}
-	req, _ := http.NewRequest("GET", u.String()+urlQuery, nil)
+	req, _ := http.NewRequest("GET", url, nil)
 
 	if rule.Headers.UserAgent != "" {
 		req.Header.Set("User-Agent", rule.Headers.UserAgent)
@@ -114,6 +204,7 @@ func fetchSite(urlpath string, queries map[string]string) (string, *http.Request
 	}
 
 	if rule.Headers.CSP != "" {
+		log.Println(rule.Headers.CSP)
 		resp.Header.Set("Content-Security-Policy", rule.Headers.CSP)
 	}
 

handlers/types.go

@@ -4,6 +4,10 @@ type Regex struct {
 	Match   string `yaml:"match"`
 	Replace string `yaml:"replace"`
 }
+type KV struct {
+	Key   string `yaml:"key"`
+	Value string `yaml:"value"`
+}
 
 type RuleSet []Rule
 
@@ -20,6 +24,13 @@ type Rule struct {
 	} `yaml:"headers,omitempty"`
 	GoogleCache bool    `yaml:"googleCache,omitempty"`
 	RegexRules  []Regex `yaml:"regexRules"`
+
+	UrlMods struct {
+		Domain []Regex `yaml:"domain"`
+		Path   []Regex `yaml:"path"`
+		Query  []KV    `yaml:"query"`
+	} `yaml:"urlMods"`
+
 	Injections []struct {
 		Position string `yaml:"position"`
 		Append   string `yaml:"append"`

package.json

@@ -0,0 +1,9 @@
+{
+	"scripts": {
+		"build": "pnpx tailwindcss -i ./styles/input.css -o ./styles/output.css --build && pnpx minify ./styles/output.css > ./cmd/styles.css"
+	},
+	"devDependencies": {
+		"minify": "^10.5.2",
+		"tailwindcss": "^3.3.5"
+	}
+}

ruleset.yaml

@@ -163,3 +163,32 @@
     user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36
     content-security-policy: script-src 'self';
     cookie: 
+- domain: tagesspiegel.de
+  headers:
+    content-security-policy: script-src 'self';
+    user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36
+  urlMods:
+    query:
+      - key: amp
+        value: 1
+- domain: www.ft.com
+  headers:
+    referer: https://t.co/x?amp=1
+  injections:
+    - position: head
+      append: |
+        <script>
+          document.addEventListener("DOMContentLoaded", () => {
+            const styleTags = document.querySelectorAll('link[rel="stylesheet"]');
+            styleTags.forEach(el => { 
+              const href = el.getAttribute('href').substring(1);
+              const updatedHref = href.replace(/(https?:\/\/.+?)\/{2,}/, '$1/');
+              el.setAttribute('href', updatedHref);
+            });
+            setTimeout(() => {
+              const cookie = document.querySelectorAll('.o-cookie-message, .js-article-ribbon, .o-ads, .o-banner, .o-message, .article__content-sign-up');
+              cookie.forEach(el => { el.remove(); });
+            }, 1000);
+          })
+        </script>
+

styles/input.css

@@ -0,0 +1,3 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;

tailwind.config.js

@@ -0,0 +1,9 @@
+/** @type {import('tailwindcss').Config} */
+module.exports = {
+    content: ["./handlers/**/*.html"],
+    theme: {
+      extend: {},
+    },
+    plugins: [],
+  }
+