Commit Graph

64 Commits

Author SHA1 Message Date
Kevin Pham
47def5c610 add patch tracker scripts response modifier 2023-12-01 16:26:59 -06:00
Kevin Pham
cdac75be06 add troubleshooting comment for CF-Connecting-IP header spoofing issue with cloudflare 2023-12-01 14:13:29 -06:00
Kevin Pham
f7ddd67b4e add patch google analytics response modifier 2023-12-01 14:12:52 -06:00
Kevin Pham
8aba4a18c2 lint 2023-12-01 13:10:37 -06:00
Damian
d7b827a154 Merge branch 'origin/proxy_v2' into origin/proxy_v2 2023-12-01 11:26:44 -06:00
Kevin Pham
140a38fd39 improve /api/content metadata and tag handling 2023-11-30 22:47:15 -06:00
joncrangle
0aa34e2cfd Update caption styling 2023-11-30 23:14:36 -05:00
joncrangle
85be697741 Apply consistent template format 2023-11-30 22:56:44 -05:00
joncrangle
8bd4ca8a57 Ladder logo link to / 2023-11-30 22:42:50 -05:00
joncrangle
0fc84cbe96 Styling updates 2023-11-30 22:29:14 -05:00
Kevin Pham
518a94139b author metadata in outline links to wikipedia; sitename metadata is now header 2023-11-30 20:20:43 -06:00
Damian Bednarczyk
5877a49002 Merge branch 'origin/proxy_v2' of https://github.com/everywall/ladder into origin/proxy_v2 2023-11-30 20:10:39 -06:00
Damian Bednarczyk
130fdb6b5b add bing bot + create generic struct for bots 2023-11-30 19:55:34 -06:00
Kevin Pham
0043b67d20 rewrite a href URLs for outline pages 2023-11-30 19:45:51 -06:00
Damian Bednarczyk
ba252f5750 add other headers 2023-11-30 19:24:50 -06:00
Kevin Pham
cd891d88a8 tweak outline template rendering 2023-11-30 19:06:33 -06:00
Damian Bednarczyk
8862b7de8b rough draft of more modular design 2023-11-30 16:05:14 -06:00
Kevin Pham
1ec0d2c09e refactor outline api to become a response modifier 2023-11-30 15:50:02 -06:00
Kevin Pham
73cae94e53 handle request and url param types in fetch monkey patch 2023-11-30 00:37:55 -06:00
joncrangle
2beeeca89f Rename ExecuteForAPI to ExecuteForOutline 2023-11-30 01:13:27 -05:00
joncrangle
6749f76b0b Begin work on outline page and route 2023-11-30 00:57:32 -05:00
Damian Bednarczyk
0122b2f1cf add option to use random ip from googlebot pool 2023-11-29 23:24:32 -06:00
Damian Bednarczyk
439ec61de9 rename api.ApiError to api.Error 2023-11-29 15:03:44 -06:00
Damian Bednarczyk
382d5f3744 ignore unused params 2023-11-29 15:03:30 -06:00
Damian Bednarczyk
dc558b38fa fix import orders 2023-11-29 15:02:05 -06:00
Damian Bednarczyk
3fdbf3524d rename proxychain.ProxyChainPool to proxychain.Pool 2023-11-29 15:00:35 -06:00
Kevin Pham
53ce3ff297 add /api/outline path 2023-11-29 11:38:28 -06:00
Kevin Pham
909d29230f improve outline api with error handling and proper content-type response 2023-11-29 09:50:02 -06:00
Kevin Pham
e5bd5df2c0 structured article json first draft 2023-11-29 00:33:46 -06:00
Kevin Pham
b7a012d75b add outline test 2023-11-28 20:32:36 -06:00
Kevin Pham
31200cf9e2 replace cycletls with bogdanfinn/tls-client for better tls fingerprint spoofing reliability 2023-11-28 12:33:53 -06:00
Kevin Pham
9d31f7eb59 improve token rewriting 2023-11-28 07:37:28 -06:00
Kevin Pham
214790f905 fix fwd response headers 2023-11-28 00:06:41 -06:00
Kevin Pham
9a8d4ac261 fix cycletls imports 2023-11-27 22:40:34 -06:00
Kevin Pham
0fe02c397d fix srcset handler in html rewriter 2023-11-27 21:36:50 -06:00
Kevin Pham
78e15d8342 fix archivist response modifier 2023-11-27 20:29:03 -06:00
Kevin Pham
81cda1101c improve wayback/archist display / refactor cycletls transport with PR in ext lib 2023-11-27 20:09:03 -06:00
Kevin Pham
7b3aecca7d add TLS fingerprint (ja3) spoofer request modifier 2023-11-27 16:30:13 -06:00
Kevin Pham
2dccc7ca35 add req/resp header forwarding modifiers 2023-11-27 12:45:23 -06:00
Kevin Pham
59cf0317ec improve ua modifier with client side spoofing 2023-11-27 12:00:12 -06:00
Kevin Pham
fb63dba8a0 add response / request modifier API to only fire once 2023-11-26 23:15:00 -06:00
Kevin Pham
a295f2a167 aider: Add public API to add a ResponseModification to fire only once.
# Aider chat conversation:

USER: Modify proxychain.go such that there is a public API to add a ResponseModification to fire only once.

ASSISTANT: To make this change we need to modify `proxychain/proxychain.go` to:

1. Add a new field `onceResultModifications` in the `ProxyChain` struct to store the response modifications that should only fire once.
2. Add a new method `AddOnceResponseModification` to add a response modification that should only fire once.
3. Update the `_execute` method to apply the `onceResultModifications` and clear them after they have been applied.

Here are the *SEARCH/REPLACE* blocks:

proxychain/proxychain.go
<source>
<<<<<<< SEARCH
onceRequestModifications []RequestModification
resultModifications     []ResponseModification
=======
onceRequestModifications []RequestModification
onceResultModifications []ResponseModification
resultModifications     []ResponseModification
>>>>>>> REPLACE
</source>

proxychain/proxychain.go
<source>
<<<<<<< SEARCH
// AddOnceRequestModification adds a request modifier to the ProxyChain that should only fire once
// the modifier will not fire until ProxyChain.Execute() is run and will be removed after it has been applied.
func (chain *ProxyChain) AddOnceRequestModification(mod ...RequestModification) *ProxyChain {
	chain.onceRequestModifications = append(chain.onceRequestModifications, mod...)
	return chain
}
=======
// AddOnceRequestModification adds a request modifier to the ProxyChain that should only fire once
// the modifier will not fire until ProxyChain.Execute() is run and will be removed after it has been applied.
func (chain *ProxyChain) AddOnceRequestModification(mod ...RequestModification) *ProxyChain {
	chain.onceRequestModifications = append(chain.onceRequestModifications, mod...)
	return chain
}

// AddOnceResponseModification adds a response modifier to the ProxyChain that should only fire once
// the modifier will not fire until ProxyChain.Execute() is run and will be removed after it has been applied.
func (chain *ProxyChain) AddOnceResponseModification(mod ...ResponseModification) *ProxyChain {
	chain.onceResultModifications = append(chain.onceResultModifications, mod...)
	return chain
}
>>>>>>> REPLACE
</source>

proxychain/proxychain.go
<source>
<<<<<<< SEARCH
// Apply onceRequestModifications to proxychain and clear them
for _, applyOnceRequestModificationsTo := range chain.onceRequestModifications {
	err := applyOnceRequestModificationsTo(chain)
	if err != nil {
		return nil, chain.abort(err)
	}
}
chain.onceRequestModifications = nil
=======
// Apply onceRequestModifications and onceResultModifications to proxychain and clear them
for _, applyOnceRequestModificationsTo := range chain.onceRequestModifications {
	err := applyOnceRequestModificationsTo(chain)
	if err != nil {
		return nil, chain.abort(err)
	}
}
chain.onceRequestModifications = nil

for _, applyOnceResultModificationsTo := range chain.onceResultModifications {
	err := applyOnceResultModificationsTo(chain)
	if err != nil {
		return nil, chain.abort(err)
	}
}
chain.onceResultModifications = nil
>>>>>>> REPLACE
</source>
2023-11-26 23:14:03 -06:00
Kevin Pham
642736126d aider: Add public API to add a RequestModification to fire only once.
# Aider chat conversation:

USER: Modify proxychain.go such that there is a public API to add a RequestModification to fire only once.

ASSISTANT: To make this change we need to modify `proxychain/proxychain.go` to:

1. Add a new field `onceRequestModifications` in the `ProxyChain` struct to store the request modifications that should only fire once.
2. Add a new method `AddOnceRequestModification` to add a request modification that should only fire once.
3. Update the `_execute` method to apply the `onceRequestModifications` and clear them after they have been applied.

Here are the *SEARCH/REPLACE* blocks:

proxychain/proxychain.go
<source>
<<<<<<< SEARCH
type ProxyChain struct {
	Context              *fiber.Ctx
	Client               *http.Client
	Request              *http.Request
	Response             *http.Response
	requestModifications []RequestModification
	resultModifications  []ResponseModification
	htmlTokenRewriters   []rr.IHTMLTokenRewriter
	Ruleset              *ruleset.RuleSet
	debugMode            bool
	abortErr             error
}
=======
type ProxyChain struct {
	Context                 *fiber.Ctx
	Client                  *http.Client
	Request                 *http.Request
	Response                *http.Response
	requestModifications    []RequestModification
	onceRequestModifications []RequestModification
	resultModifications     []ResponseModification
	htmlTokenRewriters      []rr.IHTMLTokenRewriter
	Ruleset                 *ruleset.RuleSet
	debugMode               bool
	abortErr                error
}
>>>>>>> REPLACE
</source>

proxychain/proxychain.go
<source>
<<<<<<< SEARCH
// AddRequestModifications sets the ProxyChain's request modifers
// the modifier will not fire until ProxyChain.Execute() is run.
func (chain *ProxyChain) AddRequestModifications(mods ...RequestModification) *ProxyChain {
	chain.requestModifications = append(chain.requestModifications, mods...)
	return chain
}
=======
// AddRequestModifications adds more request modifers to the ProxyChain
// the modifier will not fire until ProxyChain.Execute() is run.
func (chain *ProxyChain) AddRequestModifications(mods ...RequestModification) *ProxyChain {
	chain.requestModifications = append(chain.requestModifications, mods...)
	return chain
}

// AddOnceRequestModification adds a request modifier to the ProxyChain that should only fire once
// the modifier will not fire until ProxyChain.Execute() is run and will be removed after it has been applied.
func (chain *ProxyChain) AddOnceRequestModification(mod ...RequestModification) *ProxyChain {
	chain.onceRequestModifications = append(chain.onceRequestModifications, mod...)
	return chain
}
>>>>>>> REPLACE
</source>

proxychain/proxychain.go
<source>
<<<<<<< SEARCH
// _execute sends the request for the ProxyChain and returns the raw body only
// the caller is responsible for returning a response back to the requestor
// the caller is also responsible for calling chain._reset() when they are done with the body
func (chain *ProxyChain) _execute() (io.Reader, error) {
	if chain.validateCtxIsSet() != nil || chain.abortErr != nil {
		return nil, chain.abortErr
	}
	if chain.Request == nil {
		return nil, errors.New("proxychain request not yet initialized")
	}
	if chain.Request.URL.Scheme == "" {
		return nil, errors.New("request url not set or invalid. Check ProxyChain ReqMods for issues")
	}

	// Apply requestModifications to proxychain
	for _, applyRequestModificationsTo := range chain.requestModifications {
		err := applyRequestModificationsTo(chain)
		if err != nil {
			return nil, chain.abort(err)
		}
	}
=======
// _execute sends the request for the ProxyChain and returns the raw body only
// the caller is responsible for returning a response back to the requestor
// the caller is also responsible for calling chain._reset() when they are done with the body
func (chain *ProxyChain) _execute() (io.Reader, error) {
	if chain.validateCtxIsSet() != nil || chain.abortErr != nil {
		return nil, chain.abortErr
	}
	if chain.Request == nil {
		return nil, errors.New("proxychain request not yet initialized")
	}
	if chain.Request.URL.Scheme == "" {
		return nil, errors.New("request url not set or invalid. Check ProxyChain ReqMods for issues")
	}

	// Apply requestModifications to proxychain
	for _, applyRequestModificationsTo := range chain.requestModifications {
		err := applyRequestModificationsTo(chain)
		if err != nil {
			return nil, chain.abort(err)
		}
	}

	// Apply onceRequestModifications to proxychain and clear them
	for _, applyOnceRequestModificationsTo := range chain.onceRequestModifications {
		err := applyOnceRequestModificationsTo(chain)
		if err != nil {
			return nil, chain.abort(err)
		}
	}
	chain.onceRequestModifications = nil
>>>>>>> REPLACE
</source>
2023-11-26 23:12:27 -06:00
Kevin Pham
98d6b65057 simplify rewriters api usage 2023-11-26 22:51:00 -06:00
Kevin Pham
ae48429da7 merge main 2023-11-26 21:55:17 -06:00
Kevin Pham
6c54d31086 add dynamic resource url patcher as standalone responsemodifier 2023-11-23 08:14:52 -06:00
Kevin Pham
5d55a2f3f0 refactor rewriters to modify html in single pass with multiple rewriters; improve html rewriter edge case handling 2023-11-22 23:51:52 -06:00
Kevin Pham
7668713b1a fix recursive proxy calls 2023-11-22 07:21:48 -06:00
Kevin Pham
bfd647e526 url rewrite improvements 2023-11-21 23:25:34 -06:00
Kevin Pham
efa43a6f36 minor fix with parameterization 2023-11-21 20:58:16 -06:00