diff --git a/go.mod b/go.mod index 9bab4b3..0a20934 100644 --- a/go.mod +++ b/go.mod @@ -3,16 +3,22 @@ module ladder go 1.21.1 require ( - github.com/PuerkitoBio/goquery v1.8.1 + github.com/Danny-Dasilva/fhttp v0.0.0-20220524230104-f801520157d6 + github.com/Danny-Dasilva/utls v0.0.0-20220604023528-30cb107b834e github.com/akamensky/argparse v1.4.0 github.com/gofiber/fiber/v2 v2.50.0 github.com/stretchr/testify v1.8.4 gopkg.in/yaml.v3 v3.0.1 ) +require ( + github.com/dsnet/compress v0.0.1 // indirect + golang.org/x/crypto v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect +) + require ( github.com/andybalholm/brotli v1.0.6 // indirect - github.com/andybalholm/cascadia v1.3.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/google/uuid v1.4.0 // indirect github.com/klauspost/compress v1.17.2 // indirect diff --git a/go.sum b/go.sum index f171d69..dd9defb 100644 --- a/go.sum +++ b/go.sum @@ -1,20 +1,25 @@ -github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= -github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= +github.com/Danny-Dasilva/fhttp v0.0.0-20220524230104-f801520157d6 h1:Wzbitazy0HugGNRACX7ZB1En21LT/TiVF6YbxoTTqN8= +github.com/Danny-Dasilva/fhttp v0.0.0-20220524230104-f801520157d6/go.mod h1:2IT2IFG+d+zzFuj3+ksGtVytcCBsF402zMNWHsWhD2U= +github.com/Danny-Dasilva/utls v0.0.0-20220418175931-f38e470e04f2/go.mod h1:A2g8gPTJWDD3Y4iCTNon2vG3VcjdTBcgWBlZtopfNxU= +github.com/Danny-Dasilva/utls v0.0.0-20220604023528-30cb107b834e h1:tqiguW0yAcIwQBQtD+d2rjBnboqB7CwG1OZ12F8avX8= +github.com/Danny-Dasilva/utls v0.0.0-20220604023528-30cb107b834e/go.mod h1:ssfbVNUfWJVRfW41RTpedOUlGXSq3J6aLmirUVkDgJk= github.com/akamensky/argparse v1.4.0 h1:YGzvsTqCvbEZhL8zZu2AiA5nq805NZh75JNj4ajn1xc= github.com/akamensky/argparse v1.4.0/go.mod h1:S5kwC7IuDcEr5VeXtGPRVZ5o/FdhcMlQz4IZQuw64xA= github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI= github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= -github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= -github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= +github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= +github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/gofiber/fiber/v2 v2.50.0 h1:ia0JaB+uw3GpNSCR5nvC5dsaxXjRU5OEu36aytx+zGw= github.com/gofiber/fiber/v2 v2.50.0/go.mod h1:21eytvay9Is7S6z+OgPi7c7n4++tnClWmhpimVHMimw= github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -29,58 +34,42 @@ github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.50.0 h1:H7fweIlBm0rXLs2q0XbalvJ6r0CUPFWK3/bB4N13e9M= github.com/valyala/fasthttp v1.50.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA= github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +gitlab.com/yawning/bsaes.git v0.0.0-20190805113838-0a714cd429ec/go.mod h1:BZ1RAoRPbCxum9Grlv5aeksu2H8BiKehBYooU2LFiOQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= +golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= +golang.org/x/net v0.0.0-20190328230028-74de082e2cca/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8= golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/handlers/proxy.go b/handlers/proxy.go index 838cf93..c9e7a31 100644 --- a/handlers/proxy.go +++ b/handlers/proxy.go @@ -31,17 +31,16 @@ func NewProxySiteHandler(opts *ProxyOptions) fiber.Handler { SetFiberCtx(c). SetDebugLogging(opts.Verbose). SetRequestModifications( - rx.MasqueradeAsGoogleBot(), + rx.MasqueradeAsFacebookBot(), rx.DeleteOutgoingCookies(), - // rx.RequestArchiveIs(), - rx.MasqueradeAsGoogleBot(), + // rx.RequestArchiveIs(), ). AddResponseModifications( tx.BypassCORS(), - tx.BypassContentSecurityPolicy(), - tx.DeleteIncomingCookies(), - tx.RewriteHTMLResourceURLs(), - tx.PatchDynamicResourceURLs(), + //tx.BypassContentSecurityPolicy(), + //tx.DeleteIncomingCookies(), + //tx.RewriteHTMLResourceURLs(), + //tx.PatchDynamicResourceURLs(), ). Execute() diff --git a/proxychain/proxychain.go b/proxychain/proxychain.go index 98be1d0..748b84f 100644 --- a/proxychain/proxychain.go +++ b/proxychain/proxychain.go @@ -5,14 +5,14 @@ import ( "fmt" "io" "log" - "net/http" + //"net/http" + http "github.com/Danny-Dasilva/fhttp" + "net/url" "strings" - "ladder/pkg/ruleset" - rr "ladder/proxychain/responsemodifers/rewriters" - "github.com/gofiber/fiber/v2" + "ladder/pkg/ruleset" ) /* @@ -85,6 +85,7 @@ proxychain.NewProxyChain(). type ProxyChain struct { Context *fiber.Ctx Client *http.Client + onceClient *http.Client Request *http.Request Response *http.Response requestModifications []RequestModification @@ -312,6 +313,13 @@ func (chain *ProxyChain) SetHTTPClient(httpClient *http.Client) *ProxyChain { return chain } +// SetOnceHTTPClient sets a new upstream http client transport temporarily +// and clears it once it is used. +func (chain *ProxyChain) SetOnceHTTPClient(httpClient *http.Client) *ProxyChain { + chain.onceClient = httpClient + return chain +} + // SetVerbose changes the logging behavior to print // the modification steps and applied rulesets for debugging func (chain *ProxyChain) SetDebugLogging(isDebugMode bool) *ProxyChain { @@ -340,6 +348,7 @@ func (chain *ProxyChain) _reset() { chain.Context = nil chain.onceResponseModifications = []ResponseModification{} chain.onceRequestModifications = []RequestModification{} + //chain.onceClient = nil } // NewProxyChain initializes a new ProxyChain @@ -355,6 +364,7 @@ func NewProxyChain() *ProxyChain { // the caller is responsible for returning a response back to the requestor // the caller is also responsible for calling chain._reset() when they are done with the body func (chain *ProxyChain) _execute() (io.Reader, error) { + // ================== PREFLIGHT CHECKS ============================= if chain.validateCtxIsSet() != nil || chain.abortErr != nil { return nil, chain.abortErr } @@ -365,6 +375,7 @@ func (chain *ProxyChain) _execute() (io.Reader, error) { return nil, errors.New("request url not set or invalid. Check ProxyChain ReqMods for issues") } + // ======== REQUEST MODIFICATIONS :: [client -> ladder] -> upstream -> ladder -> client ============================= // Apply requestModifications to proxychain for _, applyRequestModificationsTo := range chain.requestModifications { err := applyRequestModificationsTo(chain) @@ -382,19 +393,26 @@ func (chain *ProxyChain) _execute() (io.Reader, error) { } chain.onceRequestModifications = []RequestModification{} + // ======== SEND REQUEST UPSTREAM :: client -> [ladder -> upstream] -> ladder -> client ============================= // Send Request Upstream - resp, err := chain.Client.Do(chain.Request) - if err != nil { - return nil, chain.abort(err) + if chain.onceClient != nil { + // if chain.SetOnceClient() is used, use that client instead of the + // default http client temporarily. + resp, err := chain.onceClient.Do(chain.Request) + if err != nil { + return nil, chain.abort(err) + } + chain.Response = resp + //chain.onceClient = nil + } else { + resp, err := chain.Client.Do(chain.Request) + if err != nil { + return nil, chain.abort(err) + } + chain.Response = resp } - chain.Response = resp - - /* todo: move to rsm - for k, v := range resp.Header { - chain.Context.Set(k, resp.Header.Get(k)) - } - */ + // ======== APPLY RESPONSE MODIFIERS :: client -> ladder -> [upstream -> ladder] -> client ============================= // Apply ResponseModifiers to proxychain for _, applyResultModificationsTo := range chain.responseModifications { err := applyResultModificationsTo(chain) @@ -412,6 +430,7 @@ func (chain *ProxyChain) _execute() (io.Reader, error) { } chain.onceResponseModifications = []ResponseModification{} + // ======== RETURN BODY TO CLIENT :: client -> ladder -> upstream -> [ladder -> client] ============================= return chain.Response.Body, nil } @@ -430,8 +449,12 @@ func (chain *ProxyChain) Execute() error { return errors.New("no context set") } + // in case api user did not set or forward content-type, we do it for them + if chain.Context.Get("content-type") == "" { + chain.Context.Set("content-type", chain.Response.Header.Get("content-type")) + } + // Return request back to client - chain.Context.Set("content-type", chain.Response.Header.Get("content-type")) return chain.Context.SendStream(body) // return chain.Context.SendStream(body) diff --git a/proxychain/requestmodifers/masquerade_as_trusted_bot.go b/proxychain/requestmodifers/masquerade_as_trusted_bot.go index 3ae51e1..7a4f687 100644 --- a/proxychain/requestmodifers/masquerade_as_trusted_bot.go +++ b/proxychain/requestmodifers/masquerade_as_trusted_bot.go @@ -9,7 +9,10 @@ import ( func MasqueradeAsGoogleBot() proxychain.RequestModification { const botUA string = "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; http://www.google.com/bot.html) Chrome/79.0.3945.120 Safari/537.36" const botIP string = "66.249.78.8" // TODO: create a random ip pool from https://developers.google.com/static/search/apis/ipranges/googlebot.json - return masqueradeAsTrustedBot(botUA, botIP) + // https://github.com/trisulnsm/trisul-scripts/blob/master/lua/frontend_scripts/reassembly/ja3/prints/ja3fingerprint.json + const ja3 string = "769,49195-49199-49200-49161-49171-49162-49172-156-157-47-10-53-51-57,65281-0-23-35-13-13172-11-10,29-23-24,0" + + return masqueradeAsTrustedBot(botUA, botIP, ja3) } // MasqueradeAsBingBot modifies user agent and x-forwarded for @@ -17,7 +20,7 @@ func MasqueradeAsGoogleBot() proxychain.RequestModification { func MasqueradeAsBingBot() proxychain.RequestModification { const botUA string = "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/79.0.3945.120 Safari/537.36" const botIP string = "13.66.144.9" // https://www.bing.com/toolbox/bingbot.json - return masqueradeAsTrustedBot(botUA, botIP) + return masqueradeAsTrustedBot(botUA, botIP, "") } // MasqueradeAsWaybackMachineBot modifies user agent and x-forwarded for @@ -25,7 +28,7 @@ func MasqueradeAsBingBot() proxychain.RequestModification { func MasqueradeAsWaybackMachineBot() proxychain.RequestModification { const botUA string = "Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)" const botIP string = "207.241.235.164" - return masqueradeAsTrustedBot(botUA, botIP) + return masqueradeAsTrustedBot(botUA, botIP, "") } // MasqueradeAsFacebookBot modifies user agent and x-forwarded for @@ -34,7 +37,8 @@ func MasqueradeAsFacebookBot() proxychain.RequestModification { const botUA string = "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)" // 31.13.97.0/24, 31.13.99.0/24, 31.13.100.0/24, 66.220.144.0/20, 69.63.189.0/24, 69.63.190.0/24, 69.171.224.0/20, 69.171.240.0/21, 69.171.248.0/24, 173.252.73.0/24, 173.252.74.0/24, 173.252.77.0/24, 173.252.100.0/22, 173.252.104.0/21, 173.252.112.0/24, 2a03:2880:10::/48, 2a03:2880:10ff::/48, 2a03:2880:11::/48, 2a03:2880:11ff::/48, 2a03:2880:20::/48, 2a03:2880:20ff::/48, 2a03:2880:21ff::/48, 2a03:2880:30ff::/48, 2a03:2880:31ff::/48, 2a03:2880:1010::/48, 2a03:2880:1020::/48, 2a03:2880:2020::/48, 2a03:2880:2050::/48, 2a03:2880:2040::/48, 2a03:2880:2110::/48, 2a03:2880:2130::/48, 2a03:2880:3010::/48, 2a03:2880:3020::/48 const botIP string = "31.13.99.8" - return masqueradeAsTrustedBot(botUA, botIP) + const ja3 string = "771,49199-49195-49171-49161-49200-49196-49172-49162-51-57-50-49169-49159-47-53-10-5-4-255,0-11-10-13-13172-16,23-25-28-27-24-26-22-14-13-11-12-9-10,0-1-2" + return masqueradeAsTrustedBot(botUA, botIP, ja3) } // MasqueradeAsYandexBot modifies user agent and x-forwarded for @@ -43,7 +47,8 @@ func MasqueradeAsYandexBot() proxychain.RequestModification { const botUA string = "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)" // 100.43.90.0/24, 37.9.115.0/24, 37.140.165.0/24, 77.88.22.0/25, 77.88.29.0/24, 77.88.31.0/24, 77.88.59.0/24, 84.201.146.0/24, 84.201.148.0/24, 84.201.149.0/24, 87.250.243.0/24, 87.250.253.0/24, 93.158.147.0/24, 93.158.148.0/24, 93.158.151.0/24, 93.158.153.0/32, 95.108.128.0/24, 95.108.138.0/24, 95.108.150.0/23, 95.108.158.0/24, 95.108.156.0/24, 95.108.188.128/25, 95.108.234.0/24, 95.108.248.0/24, 100.43.80.0/24, 130.193.62.0/24, 141.8.153.0/24, 178.154.165.0/24, 178.154.166.128/25, 178.154.173.29, 178.154.200.158, 178.154.202.0/24, 178.154.205.0/24, 178.154.239.0/24, 178.154.243.0/24, 37.9.84.253, 199.21.99.99, 178.154.162.29, 178.154.203.251, 178.154.211.250, 178.154.171.0/24, 178.154.200.0/24, 178.154.244.0/24, 178.154.246.0/24, 95.108.181.0/24, 95.108.246.252, 5.45.254.0/24, 5.255.253.0/24, 37.140.141.0/24, 37.140.188.0/24, 100.43.81.0/24, 100.43.85.0/24, 100.43.91.0/24, 199.21.99.0/24, 2a02:6b8:b000::/32, 2a02:6b8:b010::/32, 2a02:6b8:b011::/32, 2a02:6b8:c0e::/32 const botIP string = "37.9.115.9" - return masqueradeAsTrustedBot(botUA, botIP) + const ja3 string = "769,49200-49196-49192-49188-49172-49162-165-163-161-159-107-106-105-104-57-56-55-54-136-135-134-133-49202-49198-49194-49190-49167-49157-157-61-53-132-49199-49195-49191-49187-49171-49161-164-162-160-158-103-64-63-62-51-50-49-48-154-153-152-151-69-68-67-66-49201-49197-49193-49189-49166-49156-156-60-47-150-65-7-49169-49159-49164-49154-5-4-49170-49160-22-19-16-13-49165-49155-10-255,0-11-10-35-13-15,23-25-28-27-24-26-22-14-13-11-12-9-10,0-1-2" + return masqueradeAsTrustedBot(botUA, botIP, ja3) } // MasqueradeAsBaiduBot modifies user agent and x-forwarded for @@ -52,7 +57,7 @@ func MasqueradeAsBaiduBot() proxychain.RequestModification { const botUA string = "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)" // 180.76.15.0/24, 119.63.196.0/24, 115.239.212./24, 119.63.199.0/24, 122.81.208.0/22, 123.125.71.0/24, 180.76.4.0/24, 180.76.5.0/24, 180.76.6.0/24, 185.10.104.0/24, 220.181.108.0/24, 220.181.51.0/24, 111.13.102.0/24, 123.125.67.144/29, 123.125.67.152/31, 61.135.169.0/24, 123.125.68.68/30, 123.125.68.72/29, 123.125.68.80/28, 123.125.68.96/30, 202.46.48.0/20, 220.181.38.0/24, 123.125.68.80/30, 123.125.68.84/31, 123.125.68.0/24 const botIP string = "180.76.15.7" - return masqueradeAsTrustedBot(botUA, botIP) + return masqueradeAsTrustedBot(botUA, botIP, "") } // MasqueradeAsDuckDuckBot modifies user agent and x-forwarded for @@ -61,7 +66,7 @@ func MasqueradeAsDuckDuckBot() proxychain.RequestModification { const botUA string = "DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)" // 46.51.197.88, 46.51.197.89, 50.18.192.250, 50.18.192.251, 107.21.1.61, 176.34.131.233, 176.34.135.167, 184.72.106.52, 184.72.115.86 const botIP string = "46.51.197.88" - return masqueradeAsTrustedBot(botUA, botIP) + return masqueradeAsTrustedBot(botUA, botIP, "") } // MasqueradeAsYahooBot modifies user agent and x-forwarded for @@ -69,11 +74,12 @@ func MasqueradeAsDuckDuckBot() proxychain.RequestModification { func MasqueradeAsYahooBot() proxychain.RequestModification { const botUA string = "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" // 5.255.250.0/24, 37.9.87.0/24, 67.195.37.0/24, 67.195.50.0/24, 67.195.110.0/24, 67.195.111.0/24, 67.195.112.0/23, 67.195.114.0/24, 67.195.115.0/24, 68.180.224.0/21, 72.30.132.0/24, 72.30.142.0/24, 72.30.161.0/24, 72.30.196.0/24, 72.30.198.0/24, 74.6.254.0/24, 74.6.8.0/24, 74.6.13.0/24, 74.6.17.0/24, 74.6.18.0/24, 74.6.22.0/24, 74.6.27.0/24, 74.6.168.0/24, 77.88.5.0/24, 77.88.47.0/24, 93.158.161.0/24, 98.137.72.0/24, 98.137.206.0/24, 98.137.207.0/24, 98.139.168.0/24, 114.111.95.0/24, 124.83.159.0/24, 124.83.179.0/24, 124.83.223.0/24, 141.8.144.0/24, 183.79.63.0/24, 183.79.92.0/24, 203.216.255.0/24, 211.14.11.0/24 + const ja3 = "769,49200-49196-49192-49188-49172-49162-163-159-107-106-57-56-136-135-49202-49198-49194-49190-49167-49157-157-61-53-132-49199-49195-49191-49187-49171-49161-162-158-103-64-51-50-49170-49160-154-153-69-68-22-19-49201-49197-49193-49189-49166-49156-49165-49155-156-60-47-150-65-10-7-49169-49159-49164-49154-5-4-255,0-11-10-13-15,25-24-23,0-1-2" const botIP string = "37.9.87.5" - return masqueradeAsTrustedBot(botUA, botIP) + return masqueradeAsTrustedBot(botUA, botIP, ja3) } -func masqueradeAsTrustedBot(botUA string, botIP string) proxychain.RequestModification { +func masqueradeAsTrustedBot(botUA string, botIP string, ja3 string) proxychain.RequestModification { return func(chain *proxychain.ProxyChain) error { chain.AddOnceRequestModifications( SpoofUserAgent(botUA), @@ -81,6 +87,14 @@ func masqueradeAsTrustedBot(botUA string, botIP string) proxychain.RequestModifi DeleteRequestHeader("referrer"), DeleteRequestHeader("origin"), ) + + if ja3 != "" { + chain.AddOnceRequestModifications( + //SpoofJA3fingerprint(ja3, botUA), + SpoofJA3fingerprint(ja3, ""), + ) + } + return nil } } diff --git a/proxychain/requestmodifers/modify_outgoing_cookies.go b/proxychain/requestmodifers/modify_outgoing_cookies.go index 9c8be53..b34d1ce 100644 --- a/proxychain/requestmodifers/modify_outgoing_cookies.go +++ b/proxychain/requestmodifers/modify_outgoing_cookies.go @@ -1,7 +1,8 @@ package requestmodifers import ( - "net/http" + //"net/http" + http "github.com/Danny-Dasilva/fhttp" "ladder/proxychain" ) diff --git a/proxychain/requestmodifers/resolve_with_google_doh.go b/proxychain/requestmodifers/resolve_with_google_doh.go index eeab810..6c4194c 100644 --- a/proxychain/requestmodifers/resolve_with_google_doh.go +++ b/proxychain/requestmodifers/resolve_with_google_doh.go @@ -5,7 +5,8 @@ import ( "encoding/json" "fmt" "net" - "net/http" + //"net/http" + http "github.com/Danny-Dasilva/fhttp" "time" "ladder/proxychain" diff --git a/proxychain/requestmodifers/spoof_ja3_fingerprint.go b/proxychain/requestmodifers/spoof_ja3_fingerprint.go new file mode 100644 index 0000000..c38fd69 --- /dev/null +++ b/proxychain/requestmodifers/spoof_ja3_fingerprint.go @@ -0,0 +1,451 @@ +package requestmodifers + +import ( + "ladder/proxychain" + + "strconv" + + "crypto/sha256" + + // "crypto/tls" + "context" + "errors" + "fmt" + "net" + + "strings" + "sync" + + http "github.com/Danny-Dasilva/fhttp" + "github.com/Danny-Dasilva/fhttp/http2" + utls "github.com/Danny-Dasilva/utls" + "golang.org/x/net/proxy" +) + +// SpoofJA3fingerprint modifies the TLS client and user agent to spoof a particular JA3 fingerprint +// Some anti-bot WAFs such as cloudflare can fingerprint the fields of the TLS hello packet, and the order in which they appear +// https://web.archive.org/web/20231126224326/https://engineering.salesforce.com/tls-fingerprinting-with-ja3-and-ja3s-247362855967/ +// https://web.archive.org/web/20231119065253/https://developers.cloudflare.com/bots/concepts/ja3-fingerprint/ +func SpoofJA3fingerprint(ja3 string, userAgent string) proxychain.RequestModification { + //fmt.Println(ja3) + return func(chain *proxychain.ProxyChain) error { + spoofConfig := browser{ + UserAgent: userAgent, + JA3: ja3, + InsecureSkipVerify: false, + } + + // deep copy existing client while modifying http transport + ja3SpoofClient := &http.Client{ + //Transport: newJA3SpooferTransport(spoofConfig, proxy), + Transport: newJA3SpooferTransport(spoofConfig), + Timeout: chain.Client.Timeout, + CheckRedirect: chain.Client.CheckRedirect, + } + + chain.SetOnceHTTPClient(ja3SpoofClient) + return nil + } +} + +// SpoofJA3fingerprintWithProxy modifies the TLS client and user agent to spoof a particular JA3 fingerprint and use a proxy.ContextDialer from the "golang.org/x/net/proxy" +// Some anti-bot WAFs such as cloudflare can fingerprint the fields of the TLS hello packet, and the order in which they appear +// https://web.archive.org/web/20231126224326/https://engineering.salesforce.com/tls-fingerprinting-with-ja3-and-ja3s-247362855967/ +// https://web.archive.org/web/20231119065253/https://developers.cloudflare.com/bots/concepts/ja3-fingerprint/ +func SpoofJA3fingerprintWithProxy(ja3 string, userAgent string, proxy proxy.ContextDialer) proxychain.RequestModification { + return func(chain *proxychain.ProxyChain) error { + spoofConfig := browser{ + UserAgent: userAgent, + JA3: ja3, + InsecureSkipVerify: false, + } + + // deep copy existing client while modifying http transport + ja3SpoofClient := &http.Client{ + //Transport: newJA3SpooferTransport(spoofConfig, proxy), + Transport: newJA3SpooferTransport(spoofConfig, proxy), + Timeout: chain.Client.Timeout, + CheckRedirect: chain.Client.CheckRedirect, + } + + chain.SetOnceHTTPClient(ja3SpoofClient) + return nil + } +} + +// ============================================================================================== // +// The following code is adapated from https://github.com/Danny-Dasilva/CycleTLS with GPL3 license +// ============================================================================================== // +var errProtocolNegotiated = errors.New("protocol negotiated") + +type roundTripper struct { + sync.Mutex + // fix typing + JA3 string + UserAgent string + + InsecureSkipVerify bool + cachedConnections map[string]net.Conn + cachedTransports map[string]http.RoundTripper + + dialer proxy.ContextDialer +} + +func (rt *roundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + req.Header.Set("User-Agent", rt.UserAgent) + addr := rt.getDialTLSAddr(req) + if _, ok := rt.cachedTransports[addr]; !ok { + if err := rt.getTransport(req, addr); err != nil { + return nil, err + } + } + return rt.cachedTransports[addr].RoundTrip(req) +} + +func (rt *roundTripper) getTransport(req *http.Request, addr string) error { + switch strings.ToLower(req.URL.Scheme) { + case "http": + rt.cachedTransports[addr] = &http.Transport{DialContext: rt.dialer.DialContext, DisableKeepAlives: true} + return nil + case "https": + default: + return fmt.Errorf("invalid URL scheme: [%v]", req.URL.Scheme) + } + + _, err := rt.dialTLS(req.Context(), "tcp", addr) + switch err { + case errProtocolNegotiated: + case nil: + // Should never happen. + panic("dialTLS returned no error when determining cachedTransports") + default: + return err + } + + return nil +} + +func (rt *roundTripper) dialTLS(ctx context.Context, network, addr string) (net.Conn, error) { + rt.Lock() + defer rt.Unlock() + + // If we have the connection from when we determined the HTTPS + // cachedTransports to use, return that. + if conn := rt.cachedConnections[addr]; conn != nil { + return conn, nil + } + rawConn, err := rt.dialer.DialContext(ctx, network, addr) + if err != nil { + return nil, err + } + + var host string + if host, _, err = net.SplitHostPort(addr); err != nil { + host = addr + } + ////////////////// + + spec, err := StringToSpec(rt.JA3, rt.UserAgent) + if err != nil { + return nil, err + } + + conn := utls.UClient(rawConn, &utls.Config{ServerName: host, InsecureSkipVerify: rt.InsecureSkipVerify}, // MinVersion: tls.VersionTLS10, + // MaxVersion: tls.VersionTLS13, + + utls.HelloCustom) + + if err := conn.ApplyPreset(spec); err != nil { + return nil, err + } + + if err = conn.Handshake(); err != nil { + _ = conn.Close() + + if err.Error() == "tls: CurvePreferences includes unsupported curve" { + //fix this + return nil, fmt.Errorf("conn.Handshake() error for tls 1.3 (please retry request): %+v", err) + } + return nil, fmt.Errorf("uTlsConn.Handshake() error: %+v", err) + } + + ////////// + if rt.cachedTransports[addr] != nil { + return conn, nil + } + + // No http.Transport constructed yet, create one based on the results + // of ALPN. + switch conn.ConnectionState().NegotiatedProtocol { + case http2.NextProtoTLS: + parsedUserAgent := parseUserAgent(rt.UserAgent).UserAgent + t2 := http2.Transport{DialTLS: rt.dialTLSHTTP2, + PushHandler: &http2.DefaultPushHandler{}, + Navigator: parsedUserAgent, + } + rt.cachedTransports[addr] = &t2 + default: + // Assume the remote peer is speaking HTTP 1.x + TLS. + rt.cachedTransports[addr] = &http.Transport{DialTLSContext: rt.dialTLS} + + } + + // Stash the connection just established for use servicing the + // actual request (should be near-immediate). + rt.cachedConnections[addr] = conn + + return nil, errProtocolNegotiated +} + +func (rt *roundTripper) dialTLSHTTP2(network, addr string, _ *utls.Config) (net.Conn, error) { + return rt.dialTLS(context.Background(), network, addr) +} + +func (rt *roundTripper) getDialTLSAddr(req *http.Request) string { + host, port, err := net.SplitHostPort(req.URL.Host) + if err == nil { + return net.JoinHostPort(host, port) + } + return net.JoinHostPort(req.URL.Host, "443") // we can assume port is 443 at this point +} + +func (rt *roundTripper) CloseIdleConnections() { + for addr, conn := range rt.cachedConnections { + _ = conn.Close() + delete(rt.cachedConnections, addr) + } +} + +func newJA3SpooferTransport(browser browser, dialer ...proxy.ContextDialer) http.RoundTripper { + if dialer != nil && len(dialer) > 0 { + return &roundTripper{ + dialer: dialer[0], + JA3: browser.JA3, + UserAgent: browser.UserAgent, + cachedTransports: make(map[string]http.RoundTripper), + cachedConnections: make(map[string]net.Conn), + InsecureSkipVerify: browser.InsecureSkipVerify, + } + } + + return &roundTripper{ + dialer: proxy.Direct, + + JA3: browser.JA3, + UserAgent: browser.UserAgent, + cachedTransports: make(map[string]http.RoundTripper), + cachedConnections: make(map[string]net.Conn), + InsecureSkipVerify: browser.InsecureSkipVerify, + } +} + +type browser struct { + // Return a greeting that embeds the name in a message. + JA3 string + UserAgent string + InsecureSkipVerify bool +} + +// adapted from +// https://github.com/Danny-Dasilva/CycleTLS/blob/963daacd92414dc84a0ec6930dd252381cb1247a/cycletls/utils.go#L27 +const ( + chrome = "chrome" //chrome User agent enum + firefox = "firefox" //firefox User agent enum +) + +type UserAgent struct { + UserAgent string + HeaderOrder []string +} + +// ParseUserAgent returns the pseudo header order and user agent string for chrome/firefox +func parseUserAgent(userAgent string) UserAgent { + switch { + case strings.Contains(strings.ToLower(userAgent), "chrome"): + return UserAgent{chrome, []string{":method", ":authority", ":scheme", ":path"}} + case strings.Contains(strings.ToLower(userAgent), "firefox"): + return UserAgent{firefox, []string{":method", ":path", ":authority", ":scheme"}} + default: + return UserAgent{chrome, []string{":method", ":authority", ":scheme", ":path"}} + } + +} + +// ============== + +// StringToSpec creates a ClientHelloSpec based on a JA3 string +func StringToSpec(ja3 string, userAgent string) (*utls.ClientHelloSpec, error) { + parsedUserAgent := parseUserAgent(userAgent).UserAgent + extMap := genMap() + tokens := strings.Split(ja3, ",") + + version := tokens[0] + ciphers := strings.Split(tokens[1], "-") + extensions := strings.Split(tokens[2], "-") + curves := strings.Split(tokens[3], "-") + if len(curves) == 1 && curves[0] == "" { + curves = []string{} + } + pointFormats := strings.Split(tokens[4], "-") + if len(pointFormats) == 1 && pointFormats[0] == "" { + pointFormats = []string{} + } + // parse curves + var targetCurves []utls.CurveID + targetCurves = append(targetCurves, utls.CurveID(utls.GREASE_PLACEHOLDER)) //append grease for Chrome browsers + for _, c := range curves { + cid, err := strconv.ParseUint(c, 10, 16) + if err != nil { + return nil, err + } + targetCurves = append(targetCurves, utls.CurveID(cid)) + // if cid != uint64(utls.CurveP521) { + // CurveP521 sometimes causes handshake errors + // } + } + extMap["10"] = &utls.SupportedCurvesExtension{Curves: targetCurves} + + // parse point formats + var targetPointFormats []byte + for _, p := range pointFormats { + pid, err := strconv.ParseUint(p, 10, 8) + if err != nil { + return nil, err + } + targetPointFormats = append(targetPointFormats, byte(pid)) + } + extMap["11"] = &utls.SupportedPointsExtension{SupportedPoints: targetPointFormats} + + // set extension 43 + vid64, err := strconv.ParseUint(version, 10, 16) + if err != nil { + return nil, err + } + vid := uint16(vid64) + // extMap["43"] = &utls.SupportedVersionsExtension{ + // Versions: []uint16{ + // utls.VersionTLS12, + // }, + // } + + // build extenions list + var exts []utls.TLSExtension + //Optionally Add Chrome Grease Extension + if parsedUserAgent == chrome { + exts = append(exts, &utls.UtlsGREASEExtension{}) + } + for _, e := range extensions { + te, ok := extMap[e] + if !ok { + return nil, errors.New("string2spec extension error") + } + // //Optionally add Chrome Grease Extension + if e == "21" && parsedUserAgent == chrome { + exts = append(exts, &utls.UtlsGREASEExtension{}) + } + exts = append(exts, te) + } + //Add this back in if user agent is chrome and no padding extension is given + // if parsedUserAgent == chrome { + // exts = append(exts, &utls.UtlsGREASEExtension{}) + // exts = append(exts, &utls.UtlsPaddingExtension{GetPaddingLen: utls.BoringPaddingStyle}) + // } + // build SSLVersion + // vid64, err := strconv.ParseUint(version, 10, 16) + // if err != nil { + // return nil, err + // } + + // build CipherSuites + var suites []uint16 + //Optionally Add Chrome Grease Extension + if parsedUserAgent == chrome { + suites = append(suites, utls.GREASE_PLACEHOLDER) + } + for _, c := range ciphers { + cid, err := strconv.ParseUint(c, 10, 16) + if err != nil { + return nil, err + } + suites = append(suites, uint16(cid)) + } + _ = vid + return &utls.ClientHelloSpec{ + // TLSVersMin: vid, + // TLSVersMax: vid, + CipherSuites: suites, + CompressionMethods: []byte{0}, + Extensions: exts, + GetSessionID: sha256.Sum256, + }, nil +} + +func genMap() (extMap map[string]utls.TLSExtension) { + extMap = map[string]utls.TLSExtension{ + "0": &utls.SNIExtension{}, + "5": &utls.StatusRequestExtension{}, + // These are applied later + // "10": &tls.SupportedCurvesExtension{...} + // "11": &tls.SupportedPointsExtension{...} + "13": &utls.SignatureAlgorithmsExtension{ + SupportedSignatureAlgorithms: []utls.SignatureScheme{ + utls.ECDSAWithP256AndSHA256, + utls.ECDSAWithP384AndSHA384, + utls.ECDSAWithP521AndSHA512, + utls.PSSWithSHA256, + utls.PSSWithSHA384, + utls.PSSWithSHA512, + utls.PKCS1WithSHA256, + utls.PKCS1WithSHA384, + utls.PKCS1WithSHA512, + utls.ECDSAWithSHA1, + utls.PKCS1WithSHA1, + }, + }, + "16": &utls.ALPNExtension{ + AlpnProtocols: []string{"h2", "http/1.1"}, + }, + "17": &utls.GenericExtension{Id: 17}, // status_request_v2 + "18": &utls.SCTExtension{}, + "21": &utls.UtlsPaddingExtension{GetPaddingLen: utls.BoringPaddingStyle}, + "22": &utls.GenericExtension{Id: 22}, // encrypt_then_mac + "23": &utls.UtlsExtendedMasterSecretExtension{}, + "27": &utls.CompressCertificateExtension{ + Algorithms: []utls.CertCompressionAlgo{utls.CertCompressionBrotli}, + }, + "28": &utls.FakeRecordSizeLimitExtension{}, //Limit: 0x4001 + "35": &utls.SessionTicketExtension{}, + "34": &utls.GenericExtension{Id: 34}, + "41": &utls.GenericExtension{Id: 41}, //FIXME pre_shared_key + "43": &utls.SupportedVersionsExtension{Versions: []uint16{ + utls.GREASE_PLACEHOLDER, + utls.VersionTLS13, + utls.VersionTLS12, + utls.VersionTLS11, + utls.VersionTLS10}}, + "44": &utls.CookieExtension{}, + "45": &utls.PSKKeyExchangeModesExtension{Modes: []uint8{ + utls.PskModeDHE, + }}, + "49": &utls.GenericExtension{Id: 49}, // post_handshake_auth + "50": &utls.GenericExtension{Id: 50}, // signature_algorithms_cert + "51": &utls.KeyShareExtension{KeyShares: []utls.KeyShare{ + {Group: utls.CurveID(utls.GREASE_PLACEHOLDER), Data: []byte{0}}, + {Group: utls.X25519}, + + // {Group: utls.CurveP384}, known bug missing correct extensions for handshake + }}, + "30032": &utls.GenericExtension{Id: 0x7550, Data: []byte{0}}, //FIXME + "13172": &utls.NPNExtension{}, + "17513": &utls.ApplicationSettingsExtension{ + SupportedALPNList: []string{ + "h2", + }, + }, + "65281": &utls.RenegotiationInfoExtension{ + Renegotiation: utls.RenegotiateOnceAsClient, + }, + "65037": &utls.GenericExtension{Id: 65037}, + } + return +}