accept urls without scheme

README.md

@@ -14,16 +14,19 @@ Certain sites may display missing images or encounter formatting issues. This ca
 
 ### Features
 - [x] Bypass Paywalls
-- [x] Remove CORS headers from responses, Assets, and images ...
+- [x] Remove CORS headers from responses, assets, and images ...
 - [x] Keep site browsable
-- [x] Add a debug path
-- [x] Add a API
+- [x] API
+- [x] Show RAW HTML
 - [x] Docker container
 - [x] Linux binary
 - [x] Mac OS binary
-- [x] Windows binary (Untested)
+- [x] Windows binary (untested)
 - [x] Remove most of the ads (unexpected side effect)
-- [ ] Basic Auth
+- [x] Basic Auth
+- [x] Disable logs
+- [x] Custom User Agent
+- [x] Custom X-Forwarded-For IP
 
 ## Installation
 
@@ -39,7 +42,7 @@ docker run -p 8080:8080 -d --name ladder ghcr.io/kubero-dev/ladder:latest
 
 ### Docker Compose
 ```bash
-wget https://raw.githubusercontent.com/kubero-dev/ladder/main/docker-compose.yaml
+curl https://raw.githubusercontent.com/kubero-dev/ladder/main/docker-compose.yaml --output docker-compose.yaml
 docker-compose up -d
 ```
 
@@ -53,15 +56,13 @@ docker-compose up -d
 Or direct by appending the URL to the end of the proxy URL:
 http://localhost:8080/https://www.google.com
 
-
-
 ### API
 ```bash
 curl -X GET "http://localhost:8080/api/https://www.google.com"
 ```
 
-### Debug
-http://localhost:8080/debug/https://www.google.com
+### RAW
+http://localhost:8080/raw/https://www.google.com
 
 ## Configuration
 
@@ -72,4 +73,7 @@ http://localhost:8080/debug/https://www.google.com
 | `PORT` | Port to listen on | `8080` |
 | `PREFORK` | Spawn multiple server instances | `false` |
 | `USER_AGENT` | User agent to emulate | `Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)` |
-| `X_FORWARDED_FOR` | IP Forwarder address | `66.249.66.1` |
+| `X_FORWARDED_FOR` | IP forwarder address | `66.249.66.1` |
+| `USERPASS` | Enables Basic Auth, format `admin:123456` | `` |
+| `LOG_URLS` | Log fetched URL's | `true` |
+| `DISABLE_FORM` | Disables URL Form Frontpage | `false` |

assets/index.html

@@ -159,8 +159,11 @@
         });
         document.getElementById('inputForm').addEventListener('submit', function (e) {
             e.preventDefault();
-            const inputValue = document.getElementById('inputField').value;
-            window.location.href = '/' + inputValue;
+            let url = document.getElementById('inputField').value;
+            if (url.indexOf('http') === -1) {
+                url = 'https://' + url;
+            }
+            window.location.href = '/' + url;
             return false;
         });
     </script>

cmd/main.go

@@ -1,14 +1,22 @@
 package main
 
 import (
+	_ "embed"
+
 	"ladder/handlers"
 	"log"
 	"os"
 	"strconv"
+	"strings"
 
 	"github.com/gofiber/fiber/v2"
+	"github.com/gofiber/fiber/v2/middleware/basicauth"
+	"github.com/gofiber/fiber/v2/middleware/favicon"
 )
 
+//go:embed favicon.ico
+var faviconData string
+
 func main() {
 
 	prefork, _ := strconv.ParseBool(os.Getenv("PREFORK"))
@@ -18,7 +26,30 @@ func main() {
 		},
 	)
 
+	userpass := os.Getenv("USERPASS")
+	if userpass != "" {
+		userpass := strings.Split(userpass, ":")
+		app.Use(basicauth.New(basicauth.Config{
+			Users: map[string]string{
+				userpass[0]: userpass[1],
+			},
+		}))
+	}
+
+	app.Use(favicon.New(favicon.Config{
+		Data: []byte(faviconData),
+		URL:  "/favicon.ico",
+	}))
+
+	if os.Getenv("NOLOGS") != "true" {
+		app.Use(func(c *fiber.Ctx) error {
+			log.Println(c.Method(), c.Path())
+			return c.Next()
+		})
+	}
+
 	app.Get("/", handlers.Form)
+
 	app.Get("raw/*", handlers.Raw)
 	app.Get("api/*", handlers.Api)
 	app.Get("/*", handlers.ProxySite)

docker-compose.yaml

@@ -5,14 +5,15 @@ services:
     container_name: ladder
     build: .
     #restart: always
-    #command: tail -f /dev/null
     #command: sh -c ./ladder
     environment:
       - PORT=8080
-      - PREFORK=true
-      - X_FORWARDED_FOR=66.249.66.1
-      - USER_AGENT=Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
-      #- GODEBUG=netdns=go+4
+      #- PREFORK=true
+      #- X_FORWARDED_FOR=66.249.66.1
+      #- USER_AGENT=Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
+      #- USERPASS=foo:bar
+      #- LOG_URLS=true
+      #- GODEBUG=netdns=go
     ports:
       - "8080:8080"
     deploy:

handlers/api.go

@@ -1,46 +1,30 @@
 package handlers
 
 import (
-	"io"
+	_ "embed"
+
 	"log"
-	"net/http"
-	"net/url"
 
 	"github.com/gofiber/fiber/v2"
 )
 
+//go:embed VERSION
+var version string
+
 func Api(c *fiber.Ctx) error {
 	// Get the url from the URL
 	urlQuery := c.Params("*")
 
-	u, err := url.Parse(urlQuery)
+	queries := c.Queries()
+	body, req, resp, err := fetchSite(urlQuery, queries)
 	if err != nil {
+		log.Println("ERROR:", err)
+		c.SendStatus(500)
 		return c.SendString(err.Error())
 	}
 
-	log.Println(u.String())
-
-	// Fetch the site
-	client := &http.Client{}
-	req, _ := http.NewRequest("GET", u.String(), nil)
-	req.Header.Set("User-Agent", UserAgent)
-	req.Header.Set("X-Forwarded-For", ForwardedFor)
-	req.Header.Set("Referer", u.String())
-	req.Header.Set("Host", u.Host)
-	resp, err := client.Do(req)
-
-	if err != nil {
-		return c.SendString(err.Error())
-	}
-	defer resp.Body.Close()
-
-	bodyB, err := io.ReadAll(resp.Body)
-	if err != nil {
-		log.Println("ERROR", err)
-		return c.SendString(err.Error())
-	}
-	body := rewriteHtml(bodyB, u)
 	response := Response{
+		Version: version,
 		Body:    body,
 	}
 	response.Request.Headers = make([]interface{}, 0)
@@ -63,6 +47,7 @@ func Api(c *fiber.Ctx) error {
 }
 
 type Response struct {
+	Version string `json:"version"`
 	Body    string `json:"body"`
 	Request struct {
 		Headers []interface{} `json:"headers"`

handlers/form.go

@@ -1,10 +1,20 @@
 package handlers
 
-import "github.com/gofiber/fiber/v2"
+import (
+	"os"
+
+	"github.com/gofiber/fiber/v2"
+)
 
 func Form(c *fiber.Ctx) error {
+	if os.Getenv("DISABLE_FORM") == "true" {
+		c.Set("Content-Type", "text/html")
+		c.SendStatus(fiber.StatusNotFound)
+		return c.SendString("Form Disabled")
+	} else {
 		c.Set("Content-Type", "text/html")
 		return c.SendString(html)
+	}
 }
 
 const html = `
@@ -169,8 +179,11 @@ const html = `
         });
         document.getElementById('inputForm').addEventListener('submit', function (e) {
             e.preventDefault();
-            const inputValue = document.getElementById('inputField').value;
-            window.location.href = '/' + inputValue;
+            let url = document.getElementById('inputField').value;
+            if (url.indexOf('http') === -1) {
+                url = 'https://' + url;
+            }
+            window.location.href = '/' + url;
             return false;
         });
     </script>

handlers/proxy.go

@@ -18,20 +18,43 @@ var ForwardedFor = getenv("X_FORWARDED_FOR", "66.249.66.1")
 
 func ProxySite(c *fiber.Ctx) error {
 	// Get the url from the URL
-	urlQuery := c.Params("*")
+	url := c.Params("*")
 
-	u, err := url.Parse(urlQuery)
+	queries := c.Queries()
+	body, _, resp, err := fetchSite(url, queries)
 	if err != nil {
-		log.Println("ERROR", err)
-		c.SendStatus(500)
+		log.Println("ERROR:", err)
+		c.SendStatus(fiber.StatusInternalServerError)
 		return c.SendString(err.Error())
 	}
 
-	log.Println(u.String())
+	c.Set("Content-Type", resp.Header.Get("Content-Type"))
+	return c.SendString(body)
+}
+
+func fetchSite(urlpath string, queries map[string]string) (string, *http.Request, *http.Response, error) {
+
+	urlQuery := "?"
+	if len(queries) > 0 {
+		for k, v := range queries {
+			urlQuery += k + "=" + v + "&"
+		}
+	}
+	urlQuery = strings.TrimSuffix(urlQuery, "&")
+	urlQuery = strings.TrimSuffix(urlQuery, "?")
+
+	u, err := url.Parse(urlpath)
+	if err != nil {
+		return "", nil, nil, err
+	}
+
+	if os.Getenv("DEBUG	") == "true" {
+		log.Println(u.String() + urlQuery)
+	}
 
 	// Fetch the site
 	client := &http.Client{}
-	req, _ := http.NewRequest("GET", u.String(), nil)
+	req, _ := http.NewRequest("GET", u.String()+urlQuery, nil)
 	req.Header.Set("User-Agent", UserAgent)
 	req.Header.Set("X-Forwarded-For", ForwardedFor)
 	req.Header.Set("Referer", u.String())
@@ -39,20 +62,17 @@ func ProxySite(c *fiber.Ctx) error {
 	resp, err := client.Do(req)
 
 	if err != nil {
-		return c.SendString(err.Error())
+		return "", nil, nil, err
 	}
 	defer resp.Body.Close()
 
 	bodyB, err := io.ReadAll(resp.Body)
 	if err != nil {
-		log.Println("ERROR", err)
-		c.SendStatus(500)
-		return c.SendString(err.Error())
+		return "", nil, nil, err
 	}
 
 	body := rewriteHtml(bodyB, u)
-	c.Set("Content-Type", resp.Header.Get("Content-Type"))
-	return c.SendString(body)
+	return body, req, resp, nil
 }
 
 func rewriteHtml(bodyB []byte, u *url.URL) string {

handlers/raw.go

@@ -1,10 +1,7 @@
 package handlers
 
 import (
-	"io"
 	"log"
-	"net/http"
-	"net/url"
 
 	"github.com/gofiber/fiber/v2"
 )
@@ -13,32 +10,12 @@ func Raw(c *fiber.Ctx) error {
 	// Get the url from the URL
 	urlQuery := c.Params("*")
 
-	u, err := url.Parse(urlQuery)
+	queries := c.Queries()
+	body, _, _, err := fetchSite(urlQuery, queries)
 	if err != nil {
+		log.Println("ERROR:", err)
+		c.SendStatus(500)
 		return c.SendString(err.Error())
 	}
-
-	log.Println(u.String())
-
-	// Fetch the site
-	client := &http.Client{}
-	req, _ := http.NewRequest("GET", u.String(), nil)
-	req.Header.Set("User-Agent", UserAgent)
-	req.Header.Set("X-Forwarded-For", ForwardedFor)
-	req.Header.Set("Referer", u.String())
-	req.Header.Set("Host", u.Host)
-	resp, err := client.Do(req)
-
-	if err != nil {
-		return c.SendString(err.Error())
-	}
-	defer resp.Body.Close()
-
-	bodyB, err := io.ReadAll(resp.Body)
-	if err != nil {
-		log.Println("ERROR", err)
-		return c.SendString(err.Error())
-	}
-	body := rewriteHtml(bodyB, u)
 	return c.SendString(body)
 }