add some tests

Dockerfile

@@ -20,4 +20,6 @@ RUN apt update && apt install -y ca-certificates && rm -rf /var/lib/apt/lists/*
 
 #EXPOSE 8080
 
-#ENTRYPOINT ["/usr/bin/dumb-init", "--"]
+#ENTRYPOINT ["/usr/bin/dumb-init", "--"]
+
+CMD ["sh", "-c", "/app/ladder"]

README.md

@@ -1,5 +1,5 @@
 <p align="center">
-    <img src="public/pigeon.svg" width="100px">
+    <img src="assets/pigeon.svg" width="100px">
 </p>
 
 <h1 align="center">Ladder</h1>
@@ -10,22 +10,26 @@
 
 Freedom of information is an essential pillar of democracy and informed decision-making. While media organizations have legitimate financial interests, it is crucial to strike a balance between profitability and the public's right to access information. The proliferation of paywalls raises concerns about the erosion of this fundamental freedom, and it is imperative for society to find innovative ways to preserve access to vital information without compromising the sustainability of journalism. In a world where knowledge should be shared and not commodified, paywalls should be critically examined to ensure that they do not undermine the principles of an open and informed society.
 
-Some site might have missing images or other formating issues. This is due to the fact that the site using javascript or CSS to load the images/JS/CSS. This is a limitation of this proxy. If you prefer a full experience, please concider buying a subscription for the site.
+Certain sites may display missing images or encounter formatting issues. This can be attributed to the site's reliance on JavaScript or CSS for image and resource loading, which presents a limitation when accessed through this proxy. If you prefer a full experience, please concider buying a subscription for the site.
 
 ### Features
 - [x] Bypass Paywalls
-- [x] Remove CORS
-- [x] Keep Site browsable
-- [x] Docker Container
-- [x] Linux Binary
-- [x] Mac OS Binary
-- [x] Windows Binary (Untested)
+- [x] Remove CORS headers from responses, Assets, and images ...
+- [x] Keep site browsable
+- [x] Add a debug path
+- [x] Add a API
+- [x] Docker container
+- [x] Linux binary
+- [x] Mac OS binary
+- [x] Windows binary (Untested)
+- [x] Remove most of the ads (unexpected side effect)
+- [ ] Basic Auth
 
-## How to use
+## Installation
 
 ### Binary
-1) Download Binary
-2) Run Binary
+1) Download binary [here](https://github.com/kubero-dev/ladder/releases/latest)
+2) Unpack and run the binary `./ladder`
 3) Open Browser (Default: http://localhost:8080)
 
 ### Docker
@@ -35,10 +39,30 @@ docker run -p 8080:8080 -d --name ladder ghcr.io/kubero-dev/ladder:latest
 
 ### Docker Compose
 ```bash
-wget https://raw.githubusercontent.com/kubero-dev/ladder/main/docker-compose.yml
+wget https://raw.githubusercontent.com/kubero-dev/ladder/main/docker-compose.yaml
 docker-compose up -d
 ```
 
+## Usage
+
+### Browser
+1) Open Browser (Default: http://localhost:8080)
+2) Enter URL
+3) Press Enter
+
+Or direct by appending the URL to the end of the proxy URL:
+http://localhost:8080/https://www.google.com
+
+
+
+### API
+```bash
+curl -X GET "http://localhost:8080/api/https://www.google.com"
+```
+
+### Debug
+http://localhost:8080/debug/https://www.google.com
+
 ## Configuration
 
 ### Environment Variables

cmd/main.go

@@ -20,6 +20,7 @@ func main() {
 
 	app.Get("/", handlers.Form)
 	app.Get("debug/*", handlers.Debug)
+	app.Get("api/*", handlers.Api)
 	app.Get("/*", handlers.ProxySite)
 
 	port := os.Getenv("PORT")

docker-compose.yaml

@@ -6,7 +6,7 @@ services:
     build: .
     #restart: always
     #command: tail -f /dev/null
-    command: sh -c ./ladder
+    #command: sh -c ./ladder
     environment:
       - PORT=8080
       - PREFORK=true

go.mod

@@ -2,17 +2,23 @@ module ladder
 
 go 1.21.1
 
+require github.com/gofiber/fiber/v2 v2.50.0
+
 require (
 	github.com/andybalholm/brotli v1.0.6 // indirect
-	github.com/gofiber/fiber/v2 v2.50.0
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/google/uuid v1.4.0 // indirect
 	github.com/klauspost/compress v1.17.2 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.15 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rivo/uniseg v0.4.4 // indirect
+	github.com/stretchr/objx v0.5.0 // indirect
+	github.com/stretchr/testify v1.8.4 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasthttp v1.50.0 // indirect
 	github.com/valyala/tcplisten v1.0.0 // indirect
 	golang.org/x/sys v0.13.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -1,5 +1,8 @@
 github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI=
 github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/gofiber/fiber/v2 v2.50.0 h1:ia0JaB+uw3GpNSCR5nvC5dsaxXjRU5OEu36aytx+zGw=
 github.com/gofiber/fiber/v2 v2.50.0/go.mod h1:21eytvay9Is7S6z+OgPi7c7n4++tnClWmhpimVHMimw=
 github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
@@ -13,9 +16,19 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
 github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
 github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasthttp v1.50.0 h1:H7fweIlBm0rXLs2q0XbalvJ6r0CUPFWK3/bB4N13e9M=
@@ -26,3 +39,7 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

handlers/api.go

@@ -0,0 +1,73 @@
+package handlers
+
+import (
+	"io"
+	"log"
+	"net/http"
+	"net/url"
+
+	"github.com/gofiber/fiber/v2"
+)
+
+func Api(c *fiber.Ctx) error {
+	// Get the url from the URL
+	urlQuery := c.Params("*")
+
+	u, err := url.Parse(urlQuery)
+	if err != nil {
+		return c.SendString(err.Error())
+	}
+
+	log.Println(u.String())
+
+	// Fetch the site
+	client := &http.Client{}
+	req, _ := http.NewRequest("GET", u.String(), nil)
+	req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)")
+	req.Header.Set("X-Forwarded-For", "66.249.66.1")
+	req.Header.Set("Referer", u.String())
+	req.Header.Set("Host", u.Host)
+	resp, err := client.Do(req)
+
+	if err != nil {
+		return c.SendString(err.Error())
+	}
+	defer resp.Body.Close()
+
+	bodyB, err := io.ReadAll(resp.Body)
+	if err != nil {
+		log.Println("ERROR", err)
+		return c.SendString(err.Error())
+	}
+	body := rewriteHtml(bodyB, u)
+	response := Response{
+		Body: body,
+	}
+	response.Request.Headers = make([]interface{}, 0)
+	for k, v := range req.Header {
+		response.Request.Headers = append(response.Request.Headers, map[string]string{
+			"key":   k,
+			"value": v[0],
+		})
+	}
+
+	response.Response.Headers = make([]interface{}, 0)
+	for k, v := range resp.Header {
+		response.Response.Headers = append(response.Response.Headers, map[string]string{
+			"key":   k,
+			"value": v[0],
+		})
+	}
+
+	return c.JSON(response)
+}
+
+type Response struct {
+	Body    string `json:"body"`
+	Request struct {
+		Headers []interface{} `json:"headers"`
+	} `json:"request"`
+	Response struct {
+		Headers []interface{} `json:"headers"`
+	} `json:"response"`
+}

handlers/api.test.go

@@ -0,0 +1,44 @@
+// BEGIN: 7d5e1f7c7d5e
+package handlers
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestApi(t *testing.T) {
+	app := fiber.New()
+	app.Get("/api/*", Api)
+
+	tests := []struct {
+		name           string
+		url            string
+		expectedStatus int
+	}{
+		{
+			name:           "valid url",
+			url:            "https://www.google.com",
+			expectedStatus: http.StatusOK,
+		},
+		{
+			name:           "invalid url",
+			url:            "invalid-url",
+			expectedStatus: http.StatusBadRequest,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			req := httptest.NewRequest(http.MethodGet, "/api/"+tt.url, nil)
+			resp, err := app.Test(req)
+			assert.NoError(t, err)
+			assert.Equal(t, tt.expectedStatus, resp.StatusCode)
+		})
+	}
+}
+
+// END: 7d5e1f7c7d5e

handlers/debug.test.go

@@ -0,0 +1,60 @@
+// BEGIN: 7f8d9e6d4b5c
+package handlers
+
+import (
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/gofiber/fiber/v2"
+)
+
+func TestDebug(t *testing.T) {
+	app := fiber.New()
+	app.Get("/debug/*", Debug)
+
+	testCases := []struct {
+		name     string
+		url      string
+		expected string
+	}{
+		{
+			name:     "valid url",
+			url:      "https://www.google.com",
+			expected: "<!doctype html>",
+		},
+		{
+			name:     "invalid url",
+			url:      "invalid-url",
+			expected: "parse invalid-url: invalid URI for request",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			req := httptest.NewRequest(http.MethodGet, "/debug/"+tc.url, nil)
+			resp, err := app.Test(req)
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+			defer resp.Body.Close()
+
+			if resp.StatusCode != http.StatusOK {
+				t.Errorf("expected status OK; got %v", resp.Status)
+			}
+
+			body, err := io.ReadAll(resp.Body)
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+
+			if !strings.Contains(string(body), tc.expected) {
+				t.Errorf("expected body to contain %q; got %q", tc.expected, string(body))
+			}
+		})
+	}
+}
+
+// END: 7f8d9e6d4b5c

handlers/proxy.go

@@ -63,7 +63,7 @@ func rewriteHtml(bodyB []byte, u *url.URL) string {
 	// scripts
 	scriptPattern := `<script\s+([^>]*\s+)?src="(/)([^"]*)"`
 	reScript := regexp.MustCompile(scriptPattern)
-	body = reScript.ReplaceAllString(body, fmt.Sprintf(`<script $1 script="%s$3"`, "/https://"+u.Host))
+	body = reScript.ReplaceAllString(body, fmt.Sprintf(`<script $1 script="%s$3"`, "/https://"+u.Host+"/"))
 
 	//body = strings.ReplaceAll(body, "srcset=\"/", "srcset=\"/https://"+u.Host+"/") // TODO: Needs a regex to rewrite the URL's
 	body = strings.ReplaceAll(body, "href=\"/", "href=\"/https://"+u.Host+"/")

handlers/proxy.test.go

@@ -0,0 +1,58 @@
+// BEGIN: 6f8b3f5d5d5d
+package handlers
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestProxySite(t *testing.T) {
+	app := fiber.New()
+	app.Get("/:url", ProxySite)
+
+	req := httptest.NewRequest("GET", "/https://example.com", nil)
+	resp, err := app.Test(req)
+	assert.NoError(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+}
+
+func TestRewriteHtml(t *testing.T) {
+	bodyB := []byte(`
+		<html>
+			<head>
+				<title>Test Page</title>
+			</head>
+			<body>
+				<img src="/image.jpg">
+				<script src="/script.js"></script>
+				<a href="/about">About Us</a>
+				<div style="background-image: url('/background.jpg')"></div>
+			</body>
+		</html>
+	`)
+	u := &url.URL{Host: "example.com"}
+
+	expected := `
+		<html>
+			<head>
+				<title>Test Page</title>
+			</head>
+			<body>
+				<img src="/https://example.com/image.jpg">
+				<script script="/https://example.com/script.js"></script>
+				<a href="/https://example.com/about">About Us</a>
+				<div style="background-image: url('/https://example.com/background.jpg')"></div>
+			</body>
+		</html>
+	`
+
+	actual := rewriteHtml(bodyB, u)
+	assert.Equal(t, expected, actual)
+}
+
+// END: 6f8b3f5d5d5d