DWS/hadrian
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rough draft of more modular design

Browse Source
This commit is contained in:
Damian Bednarczyk
2023-11-30 16:05:14 -06:00
parent dbaf1029c5
commit 8862b7de8b
13 changed files with 147 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
proxychain/proxychain.go
View File

@@ -392,7 +392,7 @@ func (chain *ProxyChain) _reset() {
chain.Context = nil
chain.onceResponseModifications = []ResponseModification{}
chain.onceRequestModifications = []RequestModification{}
//chain.onceClient = nil
// chain.onceClient = nil
}
// NewProxyChain initializes a new ProxyChain
@@ -402,9 +402,9 @@ func NewProxyChain() *ProxyChain {
options := []tls_client.HttpClientOption{
tls_client.WithTimeoutSeconds(20),
tls_client.WithRandomTLSExtensionOrder(),
//tls_client.WithClientProfile(profiles.Chrome_117),
//tls_client.WithNotFollowRedirects(),
//tls_client.WithCookieJar(jar), // create cookieJar instance and pass it as argument
// tls_client.WithClientProfile(profiles.Chrome_117),
// tls_client.WithNotFollowRedirects(),
// tls_client.WithCookieJar(jar), // create cookieJar instance and pass it as argument
}
client, err := tls_client.NewHttpClient(tls_client.NewNoopLogger(), options...)
if err != nil {
@@ -460,7 +460,7 @@ func (chain *ProxyChain) _execute() (io.Reader, error) {
return nil, chain.abort(err)
}
chain.Response = resp
//chain.onceClient = nil
// chain.onceClient = nil
} else {
resp, err := chain.Client.Do(chain.Request)
if err != nil {

2
proxychain/requestmodifers/forward_request_headers.go
View File

@@ -33,7 +33,7 @@ func ForwardRequestHeaders() proxychain.RequestModification {
if forwardBlacklist[k] {
return
}
//fmt.Println(k, v)
// fmt.Println(k, v)
chain.Request.Header.Set(k, v)
}

9
proxychain/requestmodifers/masquerade_as_trusted_bot.go
View File

@@ -8,14 +8,9 @@ import (
// MasqueradeAsGoogleBot modifies user agent and x-forwarded for
// to appear to be a Google Bot
func MasqueradeAsGoogleBot() proxychain.RequestModification {
const botUA string = "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; http://www.google.com/bot.html) Chrome/79.0.3945.120 Safari/537.36"
var botIP string = helpers.RandomGooglebotIP()
ip := helpers.GlobalGoogleBot.GetRandomIP()
// https://github.com/trisulnsm/trisul-scripts/blob/master/lua/frontend_scripts/reassembly/ja3/prints/ja3fingerprint.json
const ja3 string = "769,49195-49199-49196-49200-52393-52392-52244-52243-49161-49171-49162-49172-156-157-47-53-10,65281-0-23-35-13-5-18-16-11-10-21,29-23-24,0"
// "741,49195-49199-49200-49161-49171-49162-49172-156-157-47-10-53-51-57,65281-0-23-35-13-13172-11-10,29-23-24,0"
return masqueradeAsTrustedBot(botUA, botIP, ja3)
return masqueradeAsTrustedBot(helpers.GlobalGoogleBot.UserAgent, ip, helpers.GlobalGoogleBot.Fingerprint)
}
// MasqueradeAsBingBot modifies user agent and x-forwarded for

2
proxychain/responsemodifers/forward_response_headers.go
View File

@@ -24,7 +24,7 @@ func init() {
// ForwardResponseHeaders forwards the response headers from the upstream server to the client
func ForwardResponseHeaders() proxychain.ResponseModification {
return func(chain *proxychain.ProxyChain) error {
//fmt.Println(chain.Response.Header)
// fmt.Println(chain.Response.Header)
for uname, headers := range chain.Response.Header {
name := strings.ToLower(uname)
if forwardBlacklist[name] {

2
proxychain/responsemodifers/outline.go
View File

@@ -25,7 +25,7 @@ func APIOutline() proxychain.ResponseModification {
opts := trafilatura.Options{
IncludeImages: true,
IncludeLinks: true,
//FavorPrecision: true,
// FavorPrecision: true,
FallbackCandidates: nil, // TODO: https://github.com/markusmobius/go-trafilatura/blob/main/examples/chained/main.go
// implement fallbacks from "github.com/markusmobius/go-domdistiller" and "github.com/go-shiori/go-readability"
OriginalURL: chain.Request.URL,

6
proxychain/responsemodifers/rewriters/html_token_url_rewriter.go
View File

@@ -92,7 +92,7 @@ func NewHTMLTokenURLRewriter(baseURL *url.URL, proxyURL string) *HTMLTokenURLRew
}
func (r *HTMLTokenURLRewriter) ShouldModify(token *html.Token) bool {
//fmt.Printf("touch token: %s\n", token.String())
// fmt.Printf("touch token: %s\n", token.String())
attrLen := len(token.Attr)
if attrLen == 0 {
return false
@@ -225,7 +225,7 @@ func handleAbsolutePath(attr *html.Attribute, _ *url.URL) {
}
attr.Val = fmt.Sprintf("/%s", escape(strings.TrimPrefix(attr.Val, "/")))
//attr.Val = fmt.Sprintf("/%s", escape(attr.Val))
// attr.Val = fmt.Sprintf("/%s", escape(attr.Val))
log.Printf("abs url rewritten-> '%s'='%s'", attr.Key, attr.Val)
}
@@ -283,6 +283,6 @@ func handleSrcSet(attr *html.Attribute, baseURL *url.URL) {
}
func escape(str string) string {
//return str
// return str
return strings.ReplaceAll(url.PathEscape(str), "%2F", "/")
}