fix Buildjob

.github/workflows/release-binaries.yaml

@@ -22,7 +22,7 @@ jobs:
       -
         name: Set version
         run: |
-          echo -n $(git describe --tags --abbrev=0) > cmd/VERSION
+          echo -n $(git describe --tags --abbrev=0) > handlers/VERSION
       -
         name: Set up Go
         uses: actions/setup-go@v3

.github/workflows/release-docker.yaml

@@ -42,7 +42,7 @@ jobs:
       - name: Set version
         id: version
         run: |
-          echo ${GITHUB_REF#refs/tags/v} > cmd/VERSION
+          echo ${GITHUB_REF#refs/tags/v} > handlers/VERSION
 
       # Install the cosign tool except on PR
       # https://github.com/sigstore/cosign-installer

.goreleaser.yaml

@@ -7,7 +7,7 @@ before:
 builds:
   -
     main: cmd/main.go
-    binary: kubero
+    binary: ladder
     env:
       - CGO_ENABLED=0
     goos:

README.md

@@ -14,16 +14,19 @@ Certain sites may display missing images or encounter formatting issues. This ca
 
 ### Features
 - [x] Bypass Paywalls
-- [x] Remove CORS headers from responses, Assets, and images ...
+- [x] Remove CORS headers from responses, assets, and images ...
 - [x] Keep site browsable
-- [x] Add a debug path
-- [x] Add a API
+- [x] API
+- [x] Show RAW HTML
 - [x] Docker container
 - [x] Linux binary
 - [x] Mac OS binary
-- [x] Windows binary (Untested)
+- [x] Windows binary (untested)
 - [x] Remove most of the ads (unexpected side effect)
-- [ ] Basic Auth
+- [x] Basic Auth
+- [x] Disable logs
+- [x] Custom User Agent
+- [x] Custom X-Forwarded-For IP
 
 ## Installation
 
@@ -39,7 +42,7 @@ docker run -p 8080:8080 -d --name ladder ghcr.io/kubero-dev/ladder:latest
 
 ### Docker Compose
 ```bash
-wget https://raw.githubusercontent.com/kubero-dev/ladder/main/docker-compose.yaml
+curl https://raw.githubusercontent.com/kubero-dev/ladder/main/docker-compose.yaml --output docker-compose.yaml
 docker-compose up -d
 ```
 
@@ -53,15 +56,13 @@ docker-compose up -d
 Or direct by appending the URL to the end of the proxy URL:
 http://localhost:8080/https://www.google.com
 
-
-
 ### API
 ```bash
 curl -X GET "http://localhost:8080/api/https://www.google.com"
 ```
 
-### Debug
-http://localhost:8080/debug/https://www.google.com
+### RAW
+http://localhost:8080/raw/https://www.google.com
 
 ## Configuration
 
@@ -71,3 +72,8 @@ http://localhost:8080/debug/https://www.google.com
 | --- | --- | --- |
 | `PORT` | Port to listen on | `8080` |
 | `PREFORK` | Spawn multiple server instances | `false` |
+| `USER_AGENT` | User agent to emulate | `Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)` |
+| `X_FORWARDED_FOR` | IP forwarder address | `66.249.66.1` |
+| `USERPASS` | Enables Basic Auth, format `admin:123456` | `` |
+| `LOG_URLS` | Log fetched URL's | `true` |
+| `DISABLE_FORM` | Disables URL Form Frontpage | `false` |

assets/index.html

@@ -159,8 +159,11 @@
         });
         document.getElementById('inputForm').addEventListener('submit', function (e) {
             e.preventDefault();
-            const inputValue = document.getElementById('inputField').value;
-            window.location.href = '/' + inputValue;
+            let url = document.getElementById('inputField').value;
+            if (url.indexOf('http') === -1) {
+                url = 'https://' + url;
+            }
+            window.location.href = '/' + url;
             return false;
         });
     </script>

cmd/main.go

@@ -1,14 +1,22 @@
 package main
 
 import (
+	_ "embed"
+
 	"ladder/handlers"
 	"log"
 	"os"
 	"strconv"
+	"strings"
 
 	"github.com/gofiber/fiber/v2"
+	"github.com/gofiber/fiber/v2/middleware/basicauth"
+	"github.com/gofiber/fiber/v2/middleware/favicon"
 )
 
+//go:embed favicon.ico
+var faviconData string
+
 func main() {
 
 	prefork, _ := strconv.ParseBool(os.Getenv("PREFORK"))
@@ -18,8 +26,31 @@ func main() {
 		},
 	)
 
+	userpass := os.Getenv("USERPASS")
+	if userpass != "" {
+		userpass := strings.Split(userpass, ":")
+		app.Use(basicauth.New(basicauth.Config{
+			Users: map[string]string{
+				userpass[0]: userpass[1],
+			},
+		}))
+	}
+
+	app.Use(favicon.New(favicon.Config{
+		Data: []byte(faviconData),
+		URL:  "/favicon.ico",
+	}))
+
+	if os.Getenv("NOLOGS") != "true" {
+		app.Use(func(c *fiber.Ctx) error {
+			log.Println(c.Method(), c.Path())
+			return c.Next()
+		})
+	}
+
 	app.Get("/", handlers.Form)
-	app.Get("debug/*", handlers.Debug)
+
+	app.Get("raw/*", handlers.Raw)
 	app.Get("api/*", handlers.Api)
 	app.Get("/*", handlers.ProxySite)
 

docker-compose.yaml

@@ -5,12 +5,15 @@ services:
     container_name: ladder
     build: .
     #restart: always
-    #command: tail -f /dev/null
     #command: sh -c ./ladder
     environment:
       - PORT=8080
-      - PREFORK=true
-      #- GODEBUG=netdns=go+4
+      #- PREFORK=true
+      #- X_FORWARDED_FOR=66.249.66.1
+      #- USER_AGENT=Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
+      #- USERPASS=foo:bar
+      #- LOG_URLS=true
+      #- GODEBUG=netdns=go
     ports:
       - "8080:8080"
     deploy:

handlers/api.go

@@ -1,46 +1,30 @@
 package handlers
 
 import (
-	"io"
+	_ "embed"
+
 	"log"
-	"net/http"
-	"net/url"
 
 	"github.com/gofiber/fiber/v2"
 )
 
+//go:embed VERSION
+var version string
+
 func Api(c *fiber.Ctx) error {
 	// Get the url from the URL
 	urlQuery := c.Params("*")
 
-	u, err := url.Parse(urlQuery)
+	queries := c.Queries()
+	body, req, resp, err := fetchSite(urlQuery, queries)
 	if err != nil {
+		log.Println("ERROR:", err)
+		c.SendStatus(500)
 		return c.SendString(err.Error())
 	}
 
-	log.Println(u.String())
-
-	// Fetch the site
-	client := &http.Client{}
-	req, _ := http.NewRequest("GET", u.String(), nil)
-	req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)")
-	req.Header.Set("X-Forwarded-For", "66.249.66.1")
-	req.Header.Set("Referer", u.String())
-	req.Header.Set("Host", u.Host)
-	resp, err := client.Do(req)
-
-	if err != nil {
-		return c.SendString(err.Error())
-	}
-	defer resp.Body.Close()
-
-	bodyB, err := io.ReadAll(resp.Body)
-	if err != nil {
-		log.Println("ERROR", err)
-		return c.SendString(err.Error())
-	}
-	body := rewriteHtml(bodyB, u)
 	response := Response{
+		Version: version,
 		Body:    body,
 	}
 	response.Request.Headers = make([]interface{}, 0)
@@ -63,6 +47,7 @@ func Api(c *fiber.Ctx) error {
 }
 
 type Response struct {
+	Version string `json:"version"`
 	Body    string `json:"body"`
 	Request struct {
 		Headers []interface{} `json:"headers"`

handlers/debug.go

@@ -1,44 +0,0 @@
-package handlers
-
-import (
-	"io"
-	"log"
-	"net/http"
-	"net/url"
-
-	"github.com/gofiber/fiber/v2"
-)
-
-func Debug(c *fiber.Ctx) error {
-	// Get the url from the URL
-	urlQuery := c.Params("*")
-
-	u, err := url.Parse(urlQuery)
-	if err != nil {
-		return c.SendString(err.Error())
-	}
-
-	log.Println(u.String())
-
-	// Fetch the site
-	client := &http.Client{}
-	req, _ := http.NewRequest("GET", u.String(), nil)
-	req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)")
-	req.Header.Set("X-Forwarded-For", "66.249.66.1")
-	req.Header.Set("Referer", u.String())
-	req.Header.Set("Host", u.Host)
-	resp, err := client.Do(req)
-
-	if err != nil {
-		return c.SendString(err.Error())
-	}
-	defer resp.Body.Close()
-
-	bodyB, err := io.ReadAll(resp.Body)
-	if err != nil {
-		log.Println("ERROR", err)
-		return c.SendString(err.Error())
-	}
-	body := rewriteHtml(bodyB, u)
-	return c.SendString(body)
-}

handlers/form.go

@@ -1,11 +1,21 @@
 package handlers
 
-import "github.com/gofiber/fiber/v2"
+import (
+	"os"
+
+	"github.com/gofiber/fiber/v2"
+)
 
 func Form(c *fiber.Ctx) error {
+	if os.Getenv("DISABLE_FORM") == "true" {
+		c.Set("Content-Type", "text/html")
+		c.SendStatus(fiber.StatusNotFound)
+		return c.SendString("Form Disabled")
+	} else {
 		c.Set("Content-Type", "text/html")
 		return c.SendString(html)
 	}
+}
 
 const html = `
 <!DOCTYPE html>
@@ -169,8 +179,11 @@ const html = `
         });
         document.getElementById('inputForm').addEventListener('submit', function (e) {
             e.preventDefault();
-            const inputValue = document.getElementById('inputField').value;
-            window.location.href = '/' + inputValue;
+            let url = document.getElementById('inputField').value;
+            if (url.indexOf('http') === -1) {
+                url = 'https://' + url;
+            }
+            window.location.href = '/' + url;
             return false;
         });
     </script>

handlers/proxy.go

@@ -6,49 +6,73 @@ import (
 	"log"
 	"net/http"
 	"net/url"
+	"os"
 	"regexp"
 	"strings"
 
 	"github.com/gofiber/fiber/v2"
 )
 
+var UserAgent = getenv("USER_AGENT", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)")
+var ForwardedFor = getenv("X_FORWARDED_FOR", "66.249.66.1")
+
 func ProxySite(c *fiber.Ctx) error {
 	// Get the url from the URL
-	urlQuery := c.Params("*")
+	url := c.Params("*")
 
-	u, err := url.Parse(urlQuery)
+	queries := c.Queries()
+	body, _, resp, err := fetchSite(url, queries)
 	if err != nil {
-		log.Println("ERROR", err)
-		c.SendStatus(500)
+		log.Println("ERROR:", err)
+		c.SendStatus(fiber.StatusInternalServerError)
 		return c.SendString(err.Error())
 	}
 
-	log.Println(u.String())
+	c.Set("Content-Type", resp.Header.Get("Content-Type"))
+	return c.SendString(body)
+}
+
+func fetchSite(urlpath string, queries map[string]string) (string, *http.Request, *http.Response, error) {
+
+	urlQuery := "?"
+	if len(queries) > 0 {
+		for k, v := range queries {
+			urlQuery += k + "=" + v + "&"
+		}
+	}
+	urlQuery = strings.TrimSuffix(urlQuery, "&")
+	urlQuery = strings.TrimSuffix(urlQuery, "?")
+
+	u, err := url.Parse(urlpath)
+	if err != nil {
+		return "", nil, nil, err
+	}
+
+	if os.Getenv("DEBUG	") == "true" {
+		log.Println(u.String() + urlQuery)
+	}
 
 	// Fetch the site
 	client := &http.Client{}
-	req, _ := http.NewRequest("GET", u.String(), nil)
-	req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)")
-	req.Header.Set("X-Forwarded-For", "66.249.66.1")
+	req, _ := http.NewRequest("GET", u.String()+urlQuery, nil)
+	req.Header.Set("User-Agent", UserAgent)
+	req.Header.Set("X-Forwarded-For", ForwardedFor)
 	req.Header.Set("Referer", u.String())
 	req.Header.Set("Host", u.Host)
 	resp, err := client.Do(req)
 
 	if err != nil {
-		return c.SendString(err.Error())
+		return "", nil, nil, err
 	}
 	defer resp.Body.Close()
 
 	bodyB, err := io.ReadAll(resp.Body)
 	if err != nil {
-		log.Println("ERROR", err)
-		c.SendStatus(500)
-		return c.SendString(err.Error())
+		return "", nil, nil, err
 	}
 
 	body := rewriteHtml(bodyB, u)
-	c.Set("Content-Type", resp.Header.Get("Content-Type"))
-	return c.SendString(body)
+	return body, req, resp, nil
 }
 
 func rewriteHtml(bodyB []byte, u *url.URL) string {
@@ -73,3 +97,11 @@ func rewriteHtml(bodyB []byte, u *url.URL) string {
 
 	return body
 }
+
+func getenv(key, fallback string) string {
+	value := os.Getenv(key)
+	if len(value) == 0 {
+		return fallback
+	}
+	return value
+}

handlers/raw.go

@@ -0,0 +1,21 @@
+package handlers
+
+import (
+	"log"
+
+	"github.com/gofiber/fiber/v2"
+)
+
+func Raw(c *fiber.Ctx) error {
+	// Get the url from the URL
+	urlQuery := c.Params("*")
+
+	queries := c.Queries()
+	body, _, _, err := fetchSite(urlQuery, queries)
+	if err != nil {
+		log.Println("ERROR:", err)
+		c.SendStatus(500)
+		return c.SendString(err.Error())
+	}
+	return c.SendString(body)
+}

handlers/raw.test.go

@@ -11,9 +11,9 @@ import (
 	"github.com/gofiber/fiber/v2"
 )
 
-func TestDebug(t *testing.T) {
+func TestRaw(t *testing.T) {
 	app := fiber.New()
-	app.Get("/debug/*", Debug)
+	app.Get("/raw/*", Raw)
 
 	testCases := []struct {
 		name     string
@@ -34,7 +34,7 @@ func TestDebug(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			req := httptest.NewRequest(http.MethodGet, "/debug/"+tc.url, nil)
+			req := httptest.NewRequest(http.MethodGet, "/raw/"+tc.url, nil)
 			resp, err := app.Test(req)
 			if err != nil {
 				t.Fatalf("unexpected error: %v", err)