Merge pull request #68 from everywall/fix/proxy_v2/add-package-urls

#65 use full URL in package line
This commit is contained in:
Kevin Pham
2023-12-08 07:19:30 -06:00
committed by GitHub
58 changed files with 104 additions and 86 deletions

View File

@@ -7,10 +7,11 @@ import (
"log" "log"
"os" "os"
"ladder/handlers" "github.com/everywall/ladder/handlers"
"ladder/internal/cli" "github.com/everywall/ladder/internal/cli"
"ladder/proxychain/requestmodifiers/bot"
"ladder/proxychain/ruleset" "github.com/everywall/ladder/proxychain/requestmodifiers/bot"
ruleset_v2 "github.com/everywall/ladder/proxychain/ruleset"
"github.com/akamensky/argparse" "github.com/akamensky/argparse"
"github.com/gofiber/fiber/v2" "github.com/gofiber/fiber/v2"

2
go.mod
View File

@@ -1,4 +1,4 @@
module ladder module github.com/everywall/ladder
go 1.21.1 go 1.21.1

View File

@@ -1,9 +1,10 @@
package handlers package handlers
import ( import (
"ladder/proxychain" rx "github.com/everywall/ladder/proxychain/requestmodifiers"
rx "ladder/proxychain/requestmodifiers" tx "github.com/everywall/ladder/proxychain/responsemodifiers"
tx "ladder/proxychain/responsemodifiers"
"github.com/everywall/ladder/proxychain"
"github.com/gofiber/fiber/v2" "github.com/gofiber/fiber/v2"
) )

View File

@@ -2,8 +2,9 @@ package handlers
import ( import (
"encoding/json" "encoding/json"
"github.com/everywall/ladder/proxychain/responsemodifiers/api"
"github.com/gofiber/fiber/v2" "github.com/gofiber/fiber/v2"
"ladder/proxychain/responsemodifiers/api"
) )
func NewAPIModifersListHandler(opts *ProxyOptions) fiber.Handler { func NewAPIModifersListHandler(opts *ProxyOptions) fiber.Handler {

View File

@@ -146,7 +146,7 @@ package handlers
// for use in proxychains. // for use in proxychains.
import ( import (
"ladder/proxychain/responsemodifiers/api" "github.com/everywall/ladder/proxychain/responsemodifiers/api"
) )
type ModifiersAPIResponse struct { type ModifiersAPIResponse struct {

View File

@@ -5,7 +5,7 @@ package handlers
// for use in proxychains. // for use in proxychains.
import ( import (
"ladder/proxychain/responsemodifiers/api" "github.com/everywall/ladder/proxychain/responsemodifiers/api"
) )
type ModifiersAPIResponse struct { type ModifiersAPIResponse struct {

View File

@@ -1,9 +1,10 @@
package handlers package handlers
import ( import (
"ladder/proxychain" rx "github.com/everywall/ladder/proxychain/requestmodifiers"
rx "ladder/proxychain/requestmodifiers" tx "github.com/everywall/ladder/proxychain/responsemodifiers"
tx "ladder/proxychain/responsemodifiers"
"github.com/everywall/ladder/proxychain"
"github.com/gofiber/fiber/v2" "github.com/gofiber/fiber/v2"
) )

View File

@@ -1,9 +1,10 @@
package handlers package handlers
import ( import (
"ladder/proxychain" rx "github.com/everywall/ladder/proxychain/requestmodifiers"
rx "ladder/proxychain/requestmodifiers" tx "github.com/everywall/ladder/proxychain/responsemodifiers"
tx "ladder/proxychain/responsemodifiers"
"github.com/everywall/ladder/proxychain"
"github.com/gofiber/fiber/v2" "github.com/gofiber/fiber/v2"
) )

View File

@@ -1,10 +1,11 @@
package handlers package handlers
import ( import (
"ladder/proxychain" rx "github.com/everywall/ladder/proxychain/requestmodifiers"
rx "ladder/proxychain/requestmodifiers" tx "github.com/everywall/ladder/proxychain/responsemodifiers"
tx "ladder/proxychain/responsemodifiers"
"ladder/proxychain/ruleset" "github.com/everywall/ladder/proxychain"
ruleset_v2 "github.com/everywall/ladder/proxychain/ruleset"
"github.com/gofiber/fiber/v2" "github.com/gofiber/fiber/v2"
) )

View File

@@ -5,7 +5,7 @@ import (
"io" "io"
"os" "os"
"ladder/proxychain/ruleset" ruleset_v2 "github.com/everywall/ladder/proxychain/ruleset"
) )
// HandleRulesetMerge merges a set of ruleset files, specified by the rulesetPath or RULESET env variable, into either YAML or Gzip format. // HandleRulesetMerge merges a set of ruleset files, specified by the rulesetPath or RULESET env variable, into either YAML or Gzip format.

View File

@@ -70,8 +70,8 @@ package ruleset_v2
// for use in proxychains. // for use in proxychains.
import ( import (
"ladder/proxychain" "github.com/everywall/ladder/proxychain"
tx "ladder/proxychain/responsemodifiers" tx "github.com/everywall/ladder/proxychain/responsemodifiers"
) )
type ResponseModifierFactory func(params ...string) proxychain.ResponseModification type ResponseModifierFactory func(params ...string) proxychain.ResponseModification
@@ -142,8 +142,8 @@ package ruleset_v2
// for use in proxychains. // for use in proxychains.
import ( import (
"ladder/proxychain" "github.com/everywall/ladder/proxychain"
rx "ladder/proxychain/requestmodifiers" rx "github.com/everywall/ladder/proxychain/requestmodifiers"
) )
type RequestModifierFactory func(params ...string) proxychain.RequestModification type RequestModifierFactory func(params ...string) proxychain.RequestModification

View File

@@ -1,8 +1,9 @@
package requestmodifiers package requestmodifiers
import ( import (
"ladder/proxychain"
"math/rand" "math/rand"
"github.com/everywall/ladder/proxychain"
) )
// AddCacheBusterQuery modifies query params to add a random parameter key // AddCacheBusterQuery modifies query params to add a random parameter key

View File

@@ -3,7 +3,7 @@ package requestmodifiers
import ( import (
"strings" "strings"
//"fmt" //"fmt"
"ladder/proxychain" "github.com/everywall/ladder/proxychain"
) )
var forwardBlacklist map[string]bool var forwardBlacklist map[string]bool

View File

@@ -1,8 +1,9 @@
package requestmodifiers package requestmodifiers
import ( import (
"ladder/proxychain" "github.com/everywall/ladder/proxychain/requestmodifiers/bot"
"ladder/proxychain/requestmodifiers/bot"
"github.com/everywall/ladder/proxychain"
) )
// MasqueradeAsGoogleBot modifies user agent and x-forwarded for // MasqueradeAsGoogleBot modifies user agent and x-forwarded for

View File

@@ -4,7 +4,7 @@ import (
"fmt" "fmt"
"regexp" "regexp"
"ladder/proxychain" "github.com/everywall/ladder/proxychain"
) )
func ModifyDomainWithRegex(matchRegex string, replacement string) proxychain.RequestModification { func ModifyDomainWithRegex(matchRegex string, replacement string) proxychain.RequestModification {

View File

@@ -5,7 +5,7 @@ import (
//http "github.com/Danny-Dasilva/fhttp" //http "github.com/Danny-Dasilva/fhttp"
http "github.com/bogdanfinn/fhttp" http "github.com/bogdanfinn/fhttp"
"ladder/proxychain" "github.com/everywall/ladder/proxychain"
) )
// SetOutgoingCookie modifes a specific cookie name // SetOutgoingCookie modifes a specific cookie name

View File

@@ -4,7 +4,7 @@ import (
"fmt" "fmt"
"regexp" "regexp"
"ladder/proxychain" "github.com/everywall/ladder/proxychain"
) )
func ModifyPathWithRegex(matchRegex string, replacement string) proxychain.RequestModification { func ModifyPathWithRegex(matchRegex string, replacement string) proxychain.RequestModification {

View File

@@ -4,7 +4,7 @@ import (
//"fmt" //"fmt"
"net/url" "net/url"
"ladder/proxychain" "github.com/everywall/ladder/proxychain"
) )
// ModifyQueryParams replaces query parameter values in URL's query params in a ProxyChain's URL. // ModifyQueryParams replaces query parameter values in URL's query params in a ProxyChain's URL.

View File

@@ -1,7 +1,7 @@
package requestmodifiers package requestmodifiers
import ( import (
"ladder/proxychain" "github.com/everywall/ladder/proxychain"
) )
// SetRequestHeader modifies a specific outgoing header // SetRequestHeader modifies a specific outgoing header

View File

@@ -5,8 +5,9 @@ import (
"net/url" "net/url"
"regexp" "regexp"
"ladder/proxychain" tx "github.com/everywall/ladder/proxychain/responsemodifiers"
tx "ladder/proxychain/responsemodifiers"
"github.com/everywall/ladder/proxychain"
) )
const archivistUrl string = "https://archive.is/latest" const archivistUrl string = "https://archive.is/latest"

View File

@@ -3,7 +3,7 @@ package requestmodifiers
import ( import (
"net/url" "net/url"
"ladder/proxychain" "github.com/everywall/ladder/proxychain"
) )
const googleCacheUrl string = "https://webcache.googleusercontent.com/search?q=cache:" const googleCacheUrl string = "https://webcache.googleusercontent.com/search?q=cache:"

View File

@@ -4,8 +4,9 @@ import (
"net/url" "net/url"
"regexp" "regexp"
"ladder/proxychain" tx "github.com/everywall/ladder/proxychain/responsemodifiers"
tx "ladder/proxychain/responsemodifiers"
"github.com/everywall/ladder/proxychain"
) )
const waybackUrl string = "https://web.archive.org/web/" const waybackUrl string = "https://web.archive.org/web/"

View File

@@ -14,7 +14,7 @@ import (
//"net/http" //"net/http"
*/ */
"ladder/proxychain" "github.com/everywall/ladder/proxychain"
) )
// resolveWithGoogleDoH resolves DNS using Google's DNS-over-HTTPS // resolveWithGoogleDoH resolves DNS using Google's DNS-over-HTTPS

View File

@@ -1,7 +1,7 @@
package requestmodifiers package requestmodifiers
import ( import (
"ladder/proxychain" "github.com/everywall/ladder/proxychain"
) )
// SpoofOrigin modifies the origin header // SpoofOrigin modifies the origin header

View File

@@ -3,8 +3,9 @@ package requestmodifiers
import ( import (
"fmt" "fmt"
"ladder/proxychain" tx "github.com/everywall/ladder/proxychain/responsemodifiers"
tx "ladder/proxychain/responsemodifiers"
"github.com/everywall/ladder/proxychain"
) )
// SpoofReferrer modifies the referrer header. // SpoofReferrer modifies the referrer header.

View File

@@ -6,7 +6,7 @@ import (
"strings" "strings"
"time" "time"
"ladder/proxychain" "github.com/everywall/ladder/proxychain"
) )
// SpoofReferrerFromBaiduSearch modifies the referrer header // SpoofReferrerFromBaiduSearch modifies the referrer header

View File

@@ -1,7 +1,7 @@
package requestmodifiers package requestmodifiers
import ( import (
"ladder/proxychain" "github.com/everywall/ladder/proxychain"
) )
// SpoofReferrerFromBingSearch modifies the referrer header // SpoofReferrerFromBingSearch modifies the referrer header

View File

@@ -1,7 +1,7 @@
package requestmodifiers package requestmodifiers
import ( import (
"ladder/proxychain" "github.com/everywall/ladder/proxychain"
) )
// SpoofReferrerFromGoogleSearch modifies the referrer header // SpoofReferrerFromGoogleSearch modifies the referrer header

View File

@@ -1,7 +1,7 @@
package requestmodifiers package requestmodifiers
import ( import (
"ladder/proxychain" "github.com/everywall/ladder/proxychain"
) )
// SpoofReferrerFromLinkedInPost modifies the referrer header // SpoofReferrerFromLinkedInPost modifies the referrer header

View File

@@ -3,7 +3,7 @@ package requestmodifiers
import ( import (
"fmt" "fmt"
"ladder/proxychain" "github.com/everywall/ladder/proxychain"
) )
// SpoofReferrerFromNaverSearch modifies the referrer header // SpoofReferrerFromNaverSearch modifies the referrer header

View File

@@ -1,7 +1,7 @@
package requestmodifiers package requestmodifiers
import ( import (
"ladder/proxychain" "github.com/everywall/ladder/proxychain"
) )
// SpoofReferrerFromPinterestPost modifies the referrer header // SpoofReferrerFromPinterestPost modifies the referrer header

View File

@@ -1,7 +1,7 @@
package requestmodifiers package requestmodifiers
import ( import (
"ladder/proxychain" "github.com/everywall/ladder/proxychain"
) )
// SpoofReferrerFromQQPost modifies the referrer header // SpoofReferrerFromQQPost modifies the referrer header

View File

@@ -1,7 +1,7 @@
package requestmodifiers package requestmodifiers
import ( import (
"ladder/proxychain" "github.com/everywall/ladder/proxychain"
) )
// SpoofReferrerFromRedditPost modifies the referrer header // SpoofReferrerFromRedditPost modifies the referrer header

View File

@@ -1,7 +1,7 @@
package requestmodifiers package requestmodifiers
import ( import (
"ladder/proxychain" "github.com/everywall/ladder/proxychain"
) )
// SpoofReferrerFromTumblrPost modifies the referrer header // SpoofReferrerFromTumblrPost modifies the referrer header

View File

@@ -1,7 +1,7 @@
package requestmodifiers package requestmodifiers
import ( import (
"ladder/proxychain" "github.com/everywall/ladder/proxychain"
) )
// SpoofReferrerFromTwitterPost modifies the referrer header // SpoofReferrerFromTwitterPost modifies the referrer header

View File

@@ -1,7 +1,7 @@
package requestmodifiers package requestmodifiers
import ( import (
"ladder/proxychain" "github.com/everywall/ladder/proxychain"
) )
// SpoofReferrerFromVkontaktePost modifies the referrer header // SpoofReferrerFromVkontaktePost modifies the referrer header

View File

@@ -4,7 +4,7 @@ import (
"fmt" "fmt"
"math/rand" "math/rand"
"ladder/proxychain" "github.com/everywall/ladder/proxychain"
) )
// SpoofReferrerFromWeiboPost modifies the referrer header // SpoofReferrerFromWeiboPost modifies the referrer header

View File

@@ -4,8 +4,9 @@ import (
_ "embed" _ "embed"
"strings" "strings"
"ladder/proxychain" tx "github.com/everywall/ladder/proxychain/responsemodifiers"
tx "ladder/proxychain/responsemodifiers"
"github.com/everywall/ladder/proxychain"
) )
// https://github.com/faisalman/ua-parser-js/tree/master // https://github.com/faisalman/ua-parser-js/tree/master

View File

@@ -1,7 +1,7 @@
package requestmodifiers package requestmodifiers
import ( import (
"ladder/proxychain" "github.com/everywall/ladder/proxychain"
) )
// SpoofXForwardedFor modifies the X-Forwarded-For header // SpoofXForwardedFor modifies the X-Forwarded-For header

View File

@@ -7,8 +7,9 @@ import (
"github.com/markusmobius/go-trafilatura" "github.com/markusmobius/go-trafilatura"
"ladder/proxychain" "github.com/everywall/ladder/proxychain/responsemodifiers/api"
"ladder/proxychain/responsemodifiers/api"
"github.com/everywall/ladder/proxychain"
) )
// APIContent creates an JSON representation of the article and returns it as an API response. // APIContent creates an JSON representation of the article and returns it as an API response.

View File

@@ -7,7 +7,7 @@ import (
"net/url" "net/url"
"testing" "testing"
"ladder/proxychain/responsemodifiers/api" "github.com/everywall/ladder/proxychain/responsemodifiers/api"
) )
func TestCreateAPIErrReader(t *testing.T) { func TestCreateAPIErrReader(t *testing.T) {

View File

@@ -4,8 +4,9 @@ import (
_ "embed" _ "embed"
"strings" "strings"
"ladder/proxychain" "github.com/everywall/ladder/proxychain/responsemodifiers/rewriters"
"ladder/proxychain/responsemodifiers/rewriters"
"github.com/everywall/ladder/proxychain"
) )
//go:embed vendor/block_element_removal.js //go:embed vendor/block_element_removal.js

View File

@@ -5,8 +5,9 @@ import (
"fmt" "fmt"
"strings" "strings"
"ladder/proxychain" "github.com/everywall/ladder/proxychain/responsemodifiers/rewriters"
"ladder/proxychain/responsemodifiers/rewriters"
"github.com/everywall/ladder/proxychain"
) )
// BlockThirdPartyScripts rewrites HTML and injects JS to block all third party JS from loading. // BlockThirdPartyScripts rewrites HTML and injects JS to block all third party JS from loading.

View File

@@ -1,7 +1,7 @@
package responsemodifiers package responsemodifiers
import ( import (
"ladder/proxychain" "github.com/everywall/ladder/proxychain"
) )
// BypassCORS modifies response headers to prevent the browser // BypassCORS modifies response headers to prevent the browser

View File

@@ -1,7 +1,7 @@
package responsemodifiers package responsemodifiers
import ( import (
"ladder/proxychain" "github.com/everywall/ladder/proxychain"
) )
// TODO: handle edge case where CSP is specified in meta tag: // TODO: handle edge case where CSP is specified in meta tag:

View File

@@ -4,7 +4,7 @@ import (
_ "embed" _ "embed"
"strings" "strings"
"ladder/proxychain" "github.com/everywall/ladder/proxychain"
) )
// DeleteLocalStorageData deletes localstorage cookies. // DeleteLocalStorageData deletes localstorage cookies.

View File

@@ -4,7 +4,7 @@ import (
_ "embed" _ "embed"
"strings" "strings"
"ladder/proxychain" "github.com/everywall/ladder/proxychain"
) )
// DeleteSessionStorageData deletes localstorage cookies. // DeleteSessionStorageData deletes localstorage cookies.

View File

@@ -5,7 +5,7 @@ import (
"net/url" "net/url"
"strings" "strings"
"ladder/proxychain" "github.com/everywall/ladder/proxychain"
) )
var forwardBlacklist map[string]bool var forwardBlacklist map[string]bool

View File

@@ -10,7 +10,7 @@ import (
"net/url" "net/url"
"strings" "strings"
"ladder/proxychain" "github.com/everywall/ladder/proxychain"
"golang.org/x/net/html" "golang.org/x/net/html"
"golang.org/x/net/html/atom" "golang.org/x/net/html/atom"

View File

@@ -4,8 +4,9 @@ import (
_ "embed" _ "embed"
"strings" "strings"
"ladder/proxychain" "github.com/everywall/ladder/proxychain/responsemodifiers/rewriters"
"ladder/proxychain/responsemodifiers/rewriters"
"github.com/everywall/ladder/proxychain"
) )
// injectScript modifies HTTP responses // injectScript modifies HTTP responses

View File

@@ -7,7 +7,7 @@ import (
//"net/http" //"net/http"
//http "github.com/Danny-Dasilva/fhttp" //http "github.com/Danny-Dasilva/fhttp"
"ladder/proxychain" "github.com/everywall/ladder/proxychain"
) )
// DeleteIncomingCookies prevents ALL cookies from being sent from the proxy server // DeleteIncomingCookies prevents ALL cookies from being sent from the proxy server

View File

@@ -1,7 +1,7 @@
package responsemodifiers package responsemodifiers
import ( import (
"ladder/proxychain" "github.com/everywall/ladder/proxychain"
) )
// SetResponseHeader modifies response headers from the upstream server // SetResponseHeader modifies response headers from the upstream server

View File

@@ -5,8 +5,9 @@ import (
"fmt" "fmt"
"strings" "strings"
"ladder/proxychain" "github.com/everywall/ladder/proxychain/responsemodifiers/rewriters"
"ladder/proxychain/responsemodifiers/rewriters"
"github.com/everywall/ladder/proxychain"
) )
//go:embed vendor/patch_dynamic_resource_urls.js //go:embed vendor/patch_dynamic_resource_urls.js

View File

@@ -7,7 +7,7 @@ import (
"log" "log"
"regexp" "regexp"
"ladder/proxychain" "github.com/everywall/ladder/proxychain"
) )
//go:embed vendor/ddg-tracker-surrogates/mapping.json //go:embed vendor/ddg-tracker-surrogates/mapping.json

View File

@@ -5,8 +5,9 @@ import (
"fmt" "fmt"
"strings" "strings"
"ladder/proxychain" "github.com/everywall/ladder/proxychain/responsemodifiers/rewriters"
"ladder/proxychain/responsemodifiers/rewriters"
"github.com/everywall/ladder/proxychain"
) )
// RewriteHTMLResourceURLs modifies HTTP responses // RewriteHTMLResourceURLs modifies HTTP responses

View File

@@ -6,7 +6,7 @@ import (
"fmt" "fmt"
//"gopkg.in/yaml.v3" //"gopkg.in/yaml.v3"
"ladder/proxychain" "github.com/everywall/ladder/proxychain"
) )
type Rule struct { type Rule struct {

View File

@@ -5,8 +5,8 @@ package ruleset_v2
// for use in proxychains. // for use in proxychains.
import ( import (
"ladder/proxychain" "github.com/everywall/ladder/proxychain"
rx "ladder/proxychain/requestmodifiers" rx "github.com/everywall/ladder/proxychain/requestmodifiers"
) )
type RequestModifierFactory func(params ...string) proxychain.RequestModification type RequestModifierFactory func(params ...string) proxychain.RequestModification

View File

@@ -5,8 +5,8 @@ package ruleset_v2
// for use in proxychains. // for use in proxychains.
import ( import (
"ladder/proxychain" "github.com/everywall/ladder/proxychain"
tx "ladder/proxychain/responsemodifiers" tx "github.com/everywall/ladder/proxychain/responsemodifiers"
) )
type ResponseModifierFactory func(params ...string) proxychain.ResponseModification type ResponseModifierFactory func(params ...string) proxychain.ResponseModification