DWS/hadrian
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

tweak dynamic url rewriting logic

Browse Source
This commit is contained in:
Kevin Pham
2023-12-01 18:45:07 -06:00
parent 47def5c610
commit 59a57f2dbd
3 changed files with 316 additions and 298 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
handlers/proxy.go
View File

@@ -33,7 +33,7 @@ func NewProxySiteHandler(opts *ProxyOptions) fiber.Handler {
SetRequestModifications( SetRequestModifications(
// rx.SpoofJA3fingerprint(ja3, "Googlebot"), // rx.SpoofJA3fingerprint(ja3, "Googlebot"),
// rx.MasqueradeAsFacebookBot(), // rx.MasqueradeAsFacebookBot(),
//rx.MasqueradeAsGoogleBot(), rx.MasqueradeAsGoogleBot(),
rx.DeleteOutgoingCookies(), rx.DeleteOutgoingCookies(),
rx.ForwardRequestHeaders(), rx.ForwardRequestHeaders(),
//rx.SpoofReferrerFromGoogleSearch(), //rx.SpoofReferrerFromGoogleSearch(),
@@ -42,6 +42,7 @@ func NewProxySiteHandler(opts *ProxyOptions) fiber.Handler {
// rx.RequestArchiveIs(), // rx.RequestArchiveIs(),
). ).
AddResponseModifications( AddResponseModifications(
tx.InjectScriptBeforeDOMContentLoaded(`(() => {let d = document.createElement("div"); d.id = "adb-check"; document.body.append(d) })()`),
tx.ForwardResponseHeaders(), tx.ForwardResponseHeaders(),
tx.BypassCORS(), tx.BypassCORS(),
tx.BypassContentSecurityPolicy(), tx.BypassContentSecurityPolicy(),

606
proxychain/responsemodifers/patch_dynamic_resource_urls.js
View File

@@ -2,90 +2,93 @@
// Also overrides the attribute setter prototype to modify the request URLs // Also overrides the attribute setter prototype to modify the request URLs
// fetch("/relative_script.js") -> fetch("http://localhost:8080/relative_script.js") // fetch("/relative_script.js") -> fetch("http://localhost:8080/relative_script.js")
(() => { (() => {
// ============== PARAMS =========================== // ============== PARAMS ===========================
// if the original request was: http://localhost:8080/http://proxiedsite.com/foo/bar // if the original request was: http://localhost:8080/http://proxiedsite.com/foo/bar
// proxyOrigin is http://localhost:8080 // proxyOrigin is http://localhost:8080
const proxyOrigin = "{{PROXY_ORIGIN}}"; const proxyOrigin = "{{PROXY_ORIGIN}}";
//const proxyOrigin = globalThis.window.location.origin; //const proxyOrigin = globalThis.window.location.origin;
// if the original request was: http://localhost:8080/http://proxiedsite.com/foo/bar // if the original request was: http://localhost:8080/http://proxiedsite.com/foo/bar
// origin is http://proxiedsite.com // origin is http://proxiedsite.com
const origin = "{{ORIGIN}}"; const origin = "{{ORIGIN}}";
//const origin = (new URL(decodeURIComponent(globalThis.window.location.pathname.substring(1)))).origin //const origin = (new URL(decodeURIComponent(globalThis.window.location.pathname.substring(1)))).origin
// ============== END PARAMS ====================== // ============== END PARAMS ======================
const blacklistedSchemes = [ const blacklistedSchemes = [
"ftp:", "ftp:",
"mailto:", "mailto:",
"tel:", "tel:",
"file:", "file:",
"blob:", "blob:",
"javascript:", "javascript:",
"about:", "about:",
"magnet:", "magnet:",
"ws:", "ws:",
"wss:", "wss:",
]; ];
function rewriteURL(url) { function rewriteURL(url) {
if (!url) return url; if (!url) return url;
// fetch url might be string, url, or request object // fetch url might be string, url, or request object
// handle all three by downcasting to string // handle all three by downcasting to string
const isStr = typeof url === "string"; const isStr = typeof url === "string";
if (!isStr) { if (!isStr) {
x = String(url); x = String(url);
if (x == "[object Request]") { if (x == "[object Request]") {
url = url.url; url = url.url;
} else { } else {
url = String(url); url = String(url);
} }
}
const oldUrl = url;
// don't rewrite special URIs
if (blacklistedSchemes.includes(url)) return url;
// don't rewrite invalid URIs
try {
new URL(url, origin);
} catch {
return url;
}
// don't double rewrite
if (url.startsWith(`${proxyOrigin}/http://`)) return url;
if (url.startsWith(`${proxyOrigin}/https://`)) return url;
if (url.startsWith(`/${proxyOrigin}`)) return url;
if (url.startsWith(`/${origin}`)) return url;
if (url.startsWith(`/http://`)) return url;
if (url.startsWith(`/https://`)) return url;
if (url.startsWith(`/http%3A%2F%2F`)) return url;
if (url.startsWith(`/https%3A%2F%2F`)) return url;
if (url.startsWith(`/%2Fhttp`)) return url;
//console.log(`proxychain: origin: ${origin} // proxyOrigin: ${proxyOrigin} // original: ${oldUrl}`)
if (url.startsWith("//")) {
url = `/${origin}/${encodeURIComponent(url.substring(2))}`;
} else if (url.startsWith("/")) {
url = `/${origin}/${encodeURIComponent(url.substring(1))}`;
} else if (
url.startsWith(proxyOrigin) && !url.startsWith(`${proxyOrigin}/http`)
) {
// edge case where client js uses current url host to write an absolute path
url = "".replace(proxyOrigin, `${proxyOrigin}/${origin}`);
} else if (url.startsWith(origin)) {
url = `/${encodeURIComponent(url)}`;
} else if (url.startsWith("http://") || url.startsWith("https://")) {
url = `/${proxyOrigin}/${encodeURIComponent(url)}`;
}
console.log(`proxychain: rewrite JS URL: ${oldUrl} -> ${url}`);
return url;
} }
/* const oldUrl = url;
// don't rewrite special URIs
if (blacklistedSchemes.includes(url)) return url;
// don't rewrite invalid URIs
try {
new URL(url, origin);
} catch {
return url;
}
// don't double rewrite
if (url.startsWith(`${proxyOrigin}/http://`)) return url;
if (url.startsWith(`${proxyOrigin}/https://`)) return url;
if (url.startsWith(`/${proxyOrigin}`)) return url;
if (url.startsWith(`/${origin}`)) return url;
if (url.startsWith(`/http://`)) return url;
if (url.startsWith(`/https://`)) return url;
if (url.startsWith(`/http%3A%2F%2F`)) return url;
if (url.startsWith(`/https%3A%2F%2F`)) return url;
if (url.startsWith(`/%2Fhttp`)) return url;
//console.log(`proxychain: origin: ${origin} // proxyOrigin: ${proxyOrigin} // original: ${oldUrl}`)
//originDomain = origin.replace("https://", "");
let scheme = origin.split(":")[0];
if (url.startsWith("//")) {
url = `/${scheme}://${encodeURIComponent(url.substring(2))}`;
} else if (url.startsWith("/")) {
url = `/${origin}/${encodeURIComponent(url.substring(1))}`;
} else if (
url.startsWith(proxyOrigin) && !url.startsWith(`${proxyOrigin}/http`)
) {
// edge case where client js uses current url host to write an absolute path
url = "".replace(proxyOrigin, `${proxyOrigin}/${origin}`);
} else if (url.startsWith(origin)) {
url = `/${encodeURIComponent(url)}`;
} else if (url.startsWith("http://") || url.startsWith("https://")) {
url = `/${proxyOrigin}/${encodeURIComponent(url)}`;
}
console.log(`proxychain: rewrite JS URL: ${oldUrl} -> ${url}`);
return url;
}
/*
// sometimes anti-bot protections like cloudflare or akamai bot manager check if JS is hooked // sometimes anti-bot protections like cloudflare or akamai bot manager check if JS is hooked
function hideMonkeyPatch(objectOrName, method, originalToString) { function hideMonkeyPatch(objectOrName, method, originalToString) {
let obj; let obj;
@@ -118,233 +121,246 @@
} }
} }
*/ */
function hideMonkeyPatch(objectOrName, method, originalToString) { function hideMonkeyPatch(objectOrName, method, originalToString) {
return; return;
}
// monkey patch fetch
const oldFetch = fetch;
fetch = async (url, init) => {
return oldFetch(rewriteURL(url), init);
};
hideMonkeyPatch("fetch", "fetch", "function fetch() { [native code] }");
// monkey patch xmlhttprequest
const oldOpen = XMLHttpRequest.prototype.open;
XMLHttpRequest.prototype.open = function (
method,
url,
async = true,
user = null,
password = null,
) {
return oldOpen.call(this, method, rewriteURL(url), async, user, password);
};
hideMonkeyPatch(
XMLHttpRequest.prototype,
"open",
'function(){if("function"==typeof eo)return eo.apply(this,arguments)}',
);
const oldSend = XMLHttpRequest.prototype.send;
XMLHttpRequest.prototype.send = function (method, url) {
return oldSend.call(this, method, rewriteURL(url));
};
hideMonkeyPatch(
XMLHttpRequest.prototype,
"send",
'function(){if("function"==typeof eo)return eo.apply(this,arguments)}',
);
// monkey patch service worker registration
const oldRegister = ServiceWorkerContainer.prototype.register;
ServiceWorkerContainer.prototype.register = function (scriptURL, options) {
return oldRegister.call(this, rewriteURL(scriptURL), options);
};
hideMonkeyPatch(
ServiceWorkerContainer.prototype,
"register",
"function register() { [native code] }",
);
// monkey patch URL.toString() method
const oldToString = URL.prototype.toString;
URL.prototype.toString = function () {
let originalURL = oldToString.call(this);
return rewriteURL(originalURL);
};
hideMonkeyPatch(
URL.prototype,
"toString",
"function toString() { [native code] }",
);
// monkey patch URL.toJSON() method
const oldToJson = URL.prototype.toString;
URL.prototype.toString = function () {
let originalURL = oldToJson.call(this);
return rewriteURL(originalURL);
};
hideMonkeyPatch(
URL.prototype,
"toString",
"function toJSON() { [native code] }",
);
// Monkey patch URL.href getter and setter
const originalHrefDescriptor = Object.getOwnPropertyDescriptor(
URL.prototype,
"href",
);
Object.defineProperty(URL.prototype, "href", {
get: function () {
let originalHref = originalHrefDescriptor.get.call(this);
return rewriteURL(originalHref);
},
set: function (newValue) {
originalHrefDescriptor.set.call(this, rewriteURL(newValue));
},
});
// TODO: do one more pass of this by manually traversing the DOM
// AFTER all the JS and page has loaded just in case
// Monkey patch setter
const elements = [
{ tag: "a", attribute: "href" },
{ tag: "img", attribute: "src" },
// { tag: 'img', attribute: 'srcset' }, // TODO: handle srcset
{ tag: "script", attribute: "src" },
{ tag: "link", attribute: "href" },
{ tag: "link", attribute: "icon" },
{ tag: "iframe", attribute: "src" },
{ tag: "audio", attribute: "src" },
{ tag: "video", attribute: "src" },
{ tag: "source", attribute: "src" },
// { tag: 'source', attribute: 'srcset' }, // TODO: handle srcset
{ tag: "embed", attribute: "src" },
{ tag: "embed", attribute: "pluginspage" },
{ tag: "html", attribute: "manifest" },
{ tag: "object", attribute: "src" },
{ tag: "input", attribute: "src" },
{ tag: "track", attribute: "src" },
{ tag: "form", attribute: "action" },
{ tag: "area", attribute: "href" },
{ tag: "base", attribute: "href" },
{ tag: "blockquote", attribute: "cite" },
{ tag: "del", attribute: "cite" },
{ tag: "ins", attribute: "cite" },
{ tag: "q", attribute: "cite" },
{ tag: "button", attribute: "formaction" },
{ tag: "input", attribute: "formaction" },
{ tag: "meta", attribute: "content" },
{ tag: "object", attribute: "data" },
];
elements.forEach(({ tag, attribute }) => {
const proto = document.createElement(tag).constructor.prototype;
const descriptor = Object.getOwnPropertyDescriptor(proto, attribute);
if (descriptor && descriptor.set) {
Object.defineProperty(proto, attribute, {
...descriptor,
set(value) {
// calling rewriteURL will end up calling a setter for href,
// leading to a recusive loop and a Maximum call stack size exceeded
// error, so we guard against this with a local semaphore flag
const isRewritingSetKey = Symbol.for("isRewritingSet");
if (!this[isRewritingSetKey]) {
this[isRewritingSetKey] = true;
descriptor.set.call(this, rewriteURL(value));
//descriptor.set.call(this, value);
this[isRewritingSetKey] = false;
} else {
// Directly set the value without rewriting
descriptor.set.call(this, value);
}
},
get() {
const isRewritingGetKey = Symbol.for("isRewritingGet");
if (!this[isRewritingGetKey]) {
this[isRewritingGetKey] = true;
let oldURL = descriptor.get.call(this);
let newURL = rewriteURL(oldURL);
this[isRewritingGetKey] = false;
return newURL;
} else {
return descriptor.get.call(this);
}
},
});
}
});
// monkey-patching Element.setAttribute
const originalSetAttribute = Element.prototype.setAttribute;
Element.prototype.setAttribute = function (name, value) {
const isMatchingElement = elements.some((element) => {
return this.tagName.toLowerCase() === element.tag &&
name.toLowerCase() === element.attribute;
});
if (isMatchingElement) {
value = rewriteURL(value);
}
originalSetAttribute.call(this, name, value);
};
// sometimes, libraries will set the Element.innerHTML or Element.outerHTML directly with a string instead of setters.
// in this case, we intercept it, create a fake DOM, parse it and then rewrite all attributes that could
// contain a URL. Then we return the replacement innerHTML/outerHTML with redirected links.
function rewriteInnerHTML(html, elements) {
const isRewritingHTMLKey = Symbol.for("isRewritingHTML");
// Check if already processing
if (document[isRewritingHTMLKey]) {
return html;
} }
// monkey patch fetch const tempContainer = document.createElement("div");
const oldFetch = fetch; document[isRewritingHTMLKey] = true;
fetch = async (url, init) => {
return oldFetch(rewriteURL(url), init);
};
hideMonkeyPatch("fetch", "fetch", "function fetch() { [native code] }");
// monkey patch xmlhttprequest try {
const oldOpen = XMLHttpRequest.prototype.open; tempContainer.innerHTML = html;
XMLHttpRequest.prototype.open = function(
method,
url,
async = true,
user = null,
password = null,
) {
return oldOpen.call(this, method, rewriteURL(url), async, user, password);
};
hideMonkeyPatch(
XMLHttpRequest.prototype,
"open",
'function(){if("function"==typeof eo)return eo.apply(this,arguments)}',
);
const oldSend = XMLHttpRequest.prototype.send; // Create a map for quick lookup
XMLHttpRequest.prototype.send = function(method, url) { const elementsMap = new Map(elements.map((e) => [e.tag, e.attribute]));
return oldSend.call(this, method, rewriteURL(url));
};
hideMonkeyPatch(
XMLHttpRequest.prototype,
"send",
'function(){if("function"==typeof eo)return eo.apply(this,arguments)}',
);
// monkey patch service worker registration // Loop-based DOM traversal
const oldRegister = ServiceWorkerContainer.prototype.register; const nodes = [...tempContainer.querySelectorAll("*")];
ServiceWorkerContainer.prototype.register = function(scriptURL, options) { for (const node of nodes) {
return oldRegister.call(this, rewriteURL(scriptURL), options); const attribute = elementsMap.get(node.tagName.toLowerCase());
}; if (attribute && node.hasAttribute(attribute)) {
hideMonkeyPatch( const originalUrl = node.getAttribute(attribute);
ServiceWorkerContainer.prototype, const rewrittenUrl = rewriteURL(originalUrl);
"register", node.setAttribute(attribute, rewrittenUrl);
"function register() { [native code] }",
);
// monkey patch URL.toString() method
const oldToString = URL.prototype.toString;
URL.prototype.toString = function() {
let originalURL = oldToString.call(this);
return rewriteURL(originalURL);
};
hideMonkeyPatch(
URL.prototype,
"toString",
"function toString() { [native code] }",
);
// monkey patch URL.toJSON() method
const oldToJson = URL.prototype.toString;
URL.prototype.toString = function() {
let originalURL = oldToJson.call(this);
return rewriteURL(originalURL);
};
hideMonkeyPatch(
URL.prototype,
"toString",
"function toJSON() { [native code] }",
);
// Monkey patch URL.href getter and setter
const originalHrefDescriptor = Object.getOwnPropertyDescriptor(
URL.prototype,
"href",
);
Object.defineProperty(URL.prototype, "href", {
get: function() {
let originalHref = originalHrefDescriptor.get.call(this);
return rewriteURL(originalHref);
},
set: function(newValue) {
originalHrefDescriptor.set.call(this, rewriteURL(newValue));
},
});
// TODO: do one more pass of this by manually traversing the DOM
// AFTER all the JS and page has loaded just in case
// Monkey patch setter
const elements = [
{ tag: "a", attribute: "href" },
{ tag: "img", attribute: "src" },
// { tag: 'img', attribute: 'srcset' }, // TODO: handle srcset
{ tag: "script", attribute: "src" },
{ tag: "link", attribute: "href" },
{ tag: "link", attribute: "icon" },
{ tag: "iframe", attribute: "src" },
{ tag: "audio", attribute: "src" },
{ tag: "video", attribute: "src" },
{ tag: "source", attribute: "src" },
// { tag: 'source', attribute: 'srcset' }, // TODO: handle srcset
{ tag: "embed", attribute: "src" },
{ tag: "embed", attribute: "pluginspage" },
{ tag: "html", attribute: "manifest" },
{ tag: "object", attribute: "src" },
{ tag: "input", attribute: "src" },
{ tag: "track", attribute: "src" },
{ tag: "form", attribute: "action" },
{ tag: "area", attribute: "href" },
{ tag: "base", attribute: "href" },
{ tag: "blockquote", attribute: "cite" },
{ tag: "del", attribute: "cite" },
{ tag: "ins", attribute: "cite" },
{ tag: "q", attribute: "cite" },
{ tag: "button", attribute: "formaction" },
{ tag: "input", attribute: "formaction" },
{ tag: "meta", attribute: "content" },
{ tag: "object", attribute: "data" },
];
elements.forEach(({ tag, attribute }) => {
const proto = document.createElement(tag).constructor.prototype;
const descriptor = Object.getOwnPropertyDescriptor(proto, attribute);
if (descriptor && descriptor.set) {
Object.defineProperty(proto, attribute, {
...descriptor,
set(value) {
// calling rewriteURL will end up calling a setter for href,
// leading to a recusive loop and a Maximum call stack size exceeded
// error, so we guard against this with a local semaphore flag
const isRewritingSetKey = Symbol.for("isRewritingSet");
if (!this[isRewritingSetKey]) {
this[isRewritingSetKey] = true;
descriptor.set.call(this, rewriteURL(value));
//descriptor.set.call(this, value);
this[isRewritingSetKey] = false;
} else {
// Directly set the value without rewriting
descriptor.set.call(this, value);
}
},
get() {
const isRewritingGetKey = Symbol.for("isRewritingGet");
if (!this[isRewritingGetKey]) {
this[isRewritingGetKey] = true;
let oldURL = descriptor.get.call(this);
let newURL = rewriteURL(oldURL);
this[isRewritingGetKey] = false;
return newURL;
} else {
return descriptor.get.call(this);
}
},
});
} }
}); }
// sometimes, libraries will set the Element.innerHTML or Element.outerHTML directly with a string instead of setters. return tempContainer.innerHTML;
// in this case, we intercept it, create a fake DOM, parse it and then rewrite all attributes that could } finally {
// contain a URL. Then we return the replacement innerHTML/outerHTML with redirected links. // Clear the flag
function rewriteInnerHTML(html, elements) { document[isRewritingHTMLKey] = false;
const isRewritingHTMLKey = Symbol.for("isRewritingHTML"); }
}
// Check if already processing // Store original setters
if (document[isRewritingHTMLKey]) { const originalSetters = {};
return html;
}
const tempContainer = document.createElement("div"); ["innerHTML", "outerHTML"].forEach((property) => {
document[isRewritingHTMLKey] = true; const descriptor = Object.getOwnPropertyDescriptor(
Element.prototype,
property,
);
if (descriptor && descriptor.set) {
originalSetters[property] = descriptor.set;
try { Object.defineProperty(Element.prototype, property, {
tempContainer.innerHTML = html; ...descriptor,
set(value) {
// Create a map for quick lookup const isRewritingHTMLKey = Symbol.for("isRewritingHTML");
const elementsMap = new Map(elements.map((e) => [e.tag, e.attribute])); if (!this[isRewritingHTMLKey]) {
this[isRewritingHTMLKey] = true;
// Loop-based DOM traversal try {
const nodes = [...tempContainer.querySelectorAll("*")]; // Use custom logic
for (const node of nodes) { descriptor.set.call(this, rewriteInnerHTML(value, elements));
const attribute = elementsMap.get(node.tagName.toLowerCase()); } finally {
if (attribute && node.hasAttribute(attribute)) { this[isRewritingHTMLKey] = false;
const originalUrl = node.getAttribute(attribute);
const rewrittenUrl = rewriteURL(originalUrl);
node.setAttribute(attribute, rewrittenUrl);
}
} }
} else {
return tempContainer.innerHTML; // Use original setter in recursive call
} finally { originalSetters[property].call(this, value);
// Clear the flag }
document[isRewritingHTMLKey] = false; },
} });
} }
});
// Store original setters
const originalSetters = {};
["innerHTML", "outerHTML"].forEach((property) => {
const descriptor = Object.getOwnPropertyDescriptor(
Element.prototype,
property,
);
if (descriptor && descriptor.set) {
originalSetters[property] = descriptor.set;
Object.defineProperty(Element.prototype, property, {
...descriptor,
set(value) {
const isRewritingHTMLKey = Symbol.for("isRewritingHTML");
if (!this[isRewritingHTMLKey]) {
this[isRewritingHTMLKey] = true;
try {
// Use custom logic
descriptor.set.call(this, rewriteInnerHTML(value, elements));
} finally {
this[isRewritingHTMLKey] = false;
}
} else {
// Use original setter in recursive call
originalSetters[property].call(this, value);
}
},
});
}
});
})(); })();

1
proxychain/responsemodifers/patch_tracker_scripts.go
View File

@@ -66,6 +66,7 @@ func PatchTrackerScripts() proxychain.ResponseModification {
// preflight checks // preflight checks
reqURL := chain.Request.URL.String() reqURL := chain.Request.URL.String()
isTracker := false isTracker := false
//
var surrogateScript io.ReadCloser var surrogateScript io.ReadCloser
for domain, domainRules := range rules { for domain, domainRules := range rules {