refactor outline api to become a response modifier

This commit is contained in:
Kevin Pham
2023-11-30 15:50:02 -06:00
parent 1c810ad8e2
commit 1ec0d2c09e
7 changed files with 110 additions and 48 deletions

View File

@@ -1,21 +1,17 @@
package responsemodifers
import (
"io"
"strings"
//"github.com/go-shiori/dom"
"github.com/go-shiori/dom"
"bytes"
"encoding/json"
"github.com/markusmobius/go-trafilatura"
//"golang.org/x/net/html"
"io"
"ladder/proxychain"
"ladder/proxychain/responsemodifers/api"
)
// APIOutline creates an JSON representation of the article and returns it as an API response.
func APIOutline() proxychain.ResponseModification {
// APIContent creates an JSON representation of the article and returns it as an API response.
func APIContent() proxychain.ResponseModification {
return func(chain *proxychain.ProxyChain) error {
// we set content-type twice here, in case another response modifier
// tries to forward over the original headers
@@ -38,9 +34,14 @@ func APIOutline() proxychain.ResponseModification {
return nil
}
doc := trafilatura.CreateReadableDocument(result)
reader := io.NopCloser(strings.NewReader(dom.OuterHTML(doc)))
chain.Response.Body = reader
res := api.ExtractResultToAPIResponse(result)
jsonData, err := json.MarshalIndent(res, "", " ")
if err != nil {
return err
}
chain.Response.Body = io.NopCloser(bytes.NewReader(jsonData))
return nil
}
}

View File

@@ -0,0 +1,84 @@
package responsemodifers
import (
"embed"
"fmt"
"html/template"
"io"
"ladder/proxychain"
"log"
"github.com/go-shiori/dom"
"github.com/markusmobius/go-trafilatura"
)
//go:embed generate_readable_outline.html
var templateFS embed.FS
// GenerateReadableOutline creates an reader-friendly distilled representation of the article.
// This is a reliable way of bypassing soft-paywalled articles, where the content is hidden, but still present in the DOM.
func GenerateReadableOutline() proxychain.ResponseModification {
// get template only once, and resuse for subsequent calls
f := "generate_readable_outline.html"
tmpl, err := template.ParseFS(templateFS, f)
if err != nil {
panic(fmt.Errorf("tx.GenerateReadableOutline Error: %s not found", f))
}
return func(chain *proxychain.ProxyChain) error {
// ===========================================================
// 1. extract dom contents using reading mode algo
// ===========================================================
opts := trafilatura.Options{
IncludeImages: true,
IncludeLinks: true,
//FavorPrecision: true,
FallbackCandidates: nil, // TODO: https://github.com/markusmobius/go-trafilatura/blob/main/examples/chained/main.go
// implement fallbacks from "github.com/markusmobius/go-domdistiller" and "github.com/go-shiori/go-readability"
OriginalURL: chain.Request.URL,
}
result, err := trafilatura.Extract(chain.Response.Body, opts)
if err != nil {
return err
}
doc := trafilatura.CreateReadableDocument(result)
distilledHTML := dom.OuterHTML(doc)
// ============================================================================
// 2. render generate_readable_outline.html template using metadata from step 1
// ============================================================================
data := map[string]interface{}{
"Success": true,
"Params": chain.Request.URL,
//"Title": result.Metadata.Title, // todo: modify CreateReadableDocument so we don't have <h1> titles duplicated?
"Date": result.Metadata.Date.String(),
"Author": result.Metadata.Author,
"Body": distilledHTML,
}
// ============================================================================
// 3. queue sending the response back to the client by replacing the response body
// (the response body will be read as a stream in proxychain.Execute() later on.)
// ============================================================================
pr, pw := io.Pipe() // pipe io.writer contents into io.reader
// Use a goroutine for writing to the pipe so we don't deadlock the request
go func() {
defer pw.Close()
err := tmpl.Execute(pw, data) // <- render template
if err != nil {
log.Printf("WARN: GenerateReadableOutline template rendering error: %s\n", err)
}
}()
chain.Context.Set("content-type", "text/html")
chain.Response.Body = pr // <- replace reponse body reader with our new reader from pipe
return nil
}
}

View File

@@ -0,0 +1,387 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<link rel="stylesheet" href="/styles.css" />
<script>
const handleThemeChange = () => {
const theme = localStorage.getItem("theme");
if (
theme === "dark" ||
(theme === "system" &&
window.matchMedia("(prefers-color-scheme: dark)").matches)
) {
document.documentElement.classList.add("dark");
} else {
document.documentElement.classList.remove("dark");
}
};
handleThemeChange();
</script>
<title>ladder | {{ .Title }}</title>
</head>
<body
class="antialiased bg-white dark:bg-slate-900 text-slate-900 dark:text-slate-200"
>
<div class="flex flex-col gap-4 max-w-3xl mx-4 lg:mx-auto pt-10">
<div class="flex justify-between place-items-center">
<div
class="hover:drop-shadow-[0_0px_4px_rgba(122,167,209,.3)] transition-colors duration-300 focus:outline-none focus:ring focus:border-[#7AA7D1] ring-offset-2"
>
<a
href="/"
class="flex -ml-2 gap-1 h-8 font-extrabold tracking-tight hover:no-underline focus:outline-none focus:ring focus:border-[#7AA7D1] ring-offset-2"
>
<svg
xmlns="http://www.w3.org/2000/svg"
xmlns:xlink="http://www.w3.org/1999/xlink"
viewBox="0 0 512 512"
class="h-8 focus:outline-none focus:ring focus:border-[#7AA7D1] ring-offset-2"
>
<path
fill="#7AA7D1"
d="M262.074 485.246C254.809 485.265 247.407 485.534 240.165 484.99L226.178 483.306C119.737 468.826 34.1354 383.43 25.3176 274.714C24.3655 262.975 23.5876 253.161 24.3295 241.148C31.4284 126.212 123.985 31.919 238.633 24.1259L250.022 23.8366C258.02 23.8001 266.212 23.491 274.183 24.1306C320.519 27.8489 366.348 45.9743 402.232 75.4548L416.996 88.2751C444.342 114.373 464.257 146.819 475.911 182.72L480.415 197.211C486.174 219.054 488.67 242.773 487.436 265.259L486.416 275.75C478.783 352.041 436.405 418.1 369.36 455.394L355.463 462.875C326.247 477.031 294.517 484.631 262.074 485.246ZM253.547 72.4475C161.905 73.0454 83.5901 144.289 73.0095 234.5C69.9101 260.926 74.7763 292.594 83.9003 317.156C104.53 372.691 153.9 416.616 211.281 430.903C226.663 434.733 242.223 436.307 258.044 436.227C353.394 435.507 430.296 361.835 438.445 267.978C439.794 252.442 438.591 236.759 435.59 221.5C419.554 139.955 353.067 79.4187 269.856 72.7052C264.479 72.2714 258.981 72.423 253.586 72.4127L253.547 72.4475Z"
/>
<path
fill="#7AA7D1"
d="M153.196 310.121L133.153 285.021C140.83 283.798 148.978 285.092 156.741 284.353L156.637 277.725L124.406 278.002C123.298 277.325 122.856 276.187 122.058 275.193L116.089 267.862C110.469 260.975 103.827 254.843 98.6026 247.669C103.918 246.839 105.248 246.537 111.14 246.523L129.093 246.327C130.152 238.785 128.62 240.843 122.138 240.758C111.929 240.623 110.659 242.014 105.004 234.661L97.9953 225.654C94.8172 221.729 91.2219 218.104 88.2631 214.005C84.1351 208.286 90.1658 209.504 94.601 209.489L236.752 209.545C257.761 209.569 268.184 211.009 285.766 221.678L285.835 206.051C285.837 197.542 286.201 189.141 284.549 180.748C280.22 158.757 260.541 143.877 240.897 135.739C238.055 134.561 232.259 133.654 235.575 129.851C244.784 119.288 263.680 111.990 277.085 111.105C288.697 109.828 301.096 113.537 311.75 117.703C360.649 136.827 393.225 183.042 398.561 234.866C402.204 270.253 391.733 308.356 367.999 335.1C332.832 374.727 269.877 384.883 223.294 360.397C206.156 351.388 183.673 333.299 175.08 316.6C173.511 313.551 174.005 313.555 170.443 313.52L160.641 313.449C158.957 313.435 156.263 314.031 155.122 312.487L153.196 310.121Z"
/>
</svg>
<span class="text-3xl mr-1 text-[#7AA7D1] leading-8 align-middle"
>ladder</span
>
</a>
</div>
<div class="flex justify-center z-10">
<div class="relative" id="dropdown">
<button
aria-expanded="closed"
onclick="toggleDropdown()"
type="button"
class="inline-flex items-center justify-center whitespace-nowrap rounded-full h-12 px-4 py-2 text-sm font-medium text-slate-600 dark:text-slate-400 ring-offset-background transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50 bg-white dark:bg-slate-900 hover:bg-slate-200 dark:hover:bg-slate-700 hover:text-slate-500 dark:hover:text-slate-200"
>
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="none"
stroke="currentColor"
stroke-width="2"
stroke-linecap="round"
stroke-linejoin="round"
class="h-5 w-5"
>
<path
d="M12.22 2h-.44a2 2 0 0 0-2 2v.18a2 2 0 0 1-1 1.73l-.43.25a2 2 0 0 1-2 0l-.15-.08a2 2 0 0 0-2.73.73l-.22.38a2 2 0 0 0 .73 2.73l.15.1a2 2 0 0 1 1 1.72v.51a2 2 0 0 1-1 1.74l-.15.09a2 2 0 0 0-.73 2.73l.22.38a2 2 0 0 0 2.73.73l.15-.08a2 2 0 0 1 2 0l.43.25a2 2 0 0 1 1 1.73V20a2 2 0 0 0 2 2h.44a2 2 0 0 0 2-2v-.18a2 2 0 0 1 1-1.73l.43-.25a2 2 0 0 1 2 0l.15.08a2 2 0 0 0 2.73-.73l.22-.39a2 2 0 0 0-.73-2.73l-.15-.08a2 2 0 0 1-1-1.74v-.5a2 2 0 0 1 1-1.74l.15-.09a2 2 0 0 0 .73-2.73l-.22-.38a2 2 0 0 0-2.73-.73l-.15.08a2 2 0 0 1-2 0l-.43-.25a2 2 0 0 1-1-1.73V4a2 2 0 0 0-2-2z"
/>
<circle cx="12" cy="12" r="3" />
</svg>
</button>
<div
id="dropdown_panel"
class="hidden absolute right-0 mt-2 w-52 rounded-md bg-white dark:bg-slate-900 shadow-md border border-slate-400 dark:border-slate-700"
>
<div
class="flex flex-col gap-2 w-full first-of-type:rounded-t-md last-of-type:rounded-b-md px-4 py-2.5 text-left text-sm"
>
Font family
<div class="grid grid-cols-2 gap-2">
<div>
<input
type="radio"
name="font"
id="sans-serif"
value="sans-serif"
class="peer hidden"
checked
/>
<label
for="sans-serif"
tabindex="0"
class="flex items-center justify-center h-10 cursor-pointer select-none rounded-md p-2 text-sm font-sans text-slate-600 dark:text-slate-200 text-center hover:bg-slate-200 dark:hover:bg-slate-700 peer-checked:bg-slate-200 dark:peer-checked:bg-slate-700"
>Sans-serif</label
>
</div>
<div>
<input
type="radio"
name="font"
id="serif"
value="serif"
class="peer hidden"
/>
<label
for="serif"
tabindex="0"
class="flex items-center justify-center h-10 cursor-pointer select-none rounded-md p-2 text-sm font-serif text-slate-600 dark:text-slate-200 text-center hover:bg-slate-200 dark:hover:bg-slate-700 peer-checked:bg-slate-200 dark:peer-checked:bg-slate-700"
>Serif</label
>
</div>
</div>
</div>
<div
class="shrink-0 bg-slate-400 dark:bg-slate-700 h-[1px] w-full"
></div>
<div
class="flex flex-col gap-2 w-full first-of-type:rounded-t-md last-of-type:rounded-b-md px-4 py-2.5 text-left text-sm"
>
Font size
<div class="grid grid-cols-4 gap-2">
<div>
<input
type="radio"
name="fontsize"
id="sm"
value="text-sm"
class="peer hidden"
/>
<label
for="sm"
tabindex="0"
title="Small"
class="flex items-end justify-center h-10 w-10 cursor-pointer select-none rounded-md p-2 text-sm text-slate-600 dark:text-slate-200 text-center hover:bg-slate-200 dark:hover:bg-slate-700 peer-checked:bg-slate-200 dark:peer-checked:bg-slate-700"
>sm</label
>
</div>
<div>
<input
type="radio"
name="fontsize"
id="base"
value="text-base"
class="peer hidden"
checked
/>
<label
for="base"
tabindex="0"
title="Medium"
class="flex items-end justify-center h-10 w-10 cursor-pointer select-none rounded-md p-2 text-base text-slate-600 dark:text-slate-200 text-center hover:bg-slate-200 dark:hover:bg-slate-700 peer-checked:bg-slate-200 dark:peer-checked:bg-slate-700"
>md</label
>
</div>
<div>
<input
type="radio"
name="fontsize"
id="lg"
value="text-lg"
class="peer hidden"
/>
<label
for="lg"
tabindex="0"
title="Large"
class="flex items-end justify-center h-10 w-10 cursor-pointer select-none rounded-md p-2 text-lg text-slate-600 dark:text-slate-200 text-center hover:bg-slate-200 dark:hover:bg-slate-700 peer-checked:bg-slate-200 dark:peer-checked:bg-slate-700"
>lg</label
>
</div>
</div>
</div>
<div
class="shrink-0 bg-slate-200 dark:bg-slate-700 h-[1px] w-full"
></div>
<div
class="flex flex-col gap-2 w-full first-of-type:rounded-t-md last-of-type:rounded-b-md px-4 py-2.5 text-left text-sm"
>
Appearance
<div class="grid grid-cols-4 gap-2">
<div>
<input
type="radio"
name="theme"
id="light"
value="light"
class="peer hidden"
/>
<label
for="light"
tabindex="0"
title="Light"
class="flex items-end justify-center h-10 w-10 cursor-pointer select-none rounded-md p-2 text-sm text-slate-600 dark:text-slate-200 text-center hover:bg-slate-200 dark:hover:bg-slate-700 peer-checked:bg-slate-200 dark:peer-checked:bg-slate-700"
>
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="none"
stroke="currentColor"
stroke-width="2"
stroke-linecap="round"
stroke-linejoin="round"
class="h-5 w-5"
>
<circle cx="12" cy="12" r="4" />
<path d="M12 2v2" />
<path d="M12 20v2" />
<path d="m4.93 4.93 1.41 1.41" />
<path d="m17.66 17.66 1.41 1.41" />
<path d="M2 12h2" />
<path d="M20 12h2" />
<path d="m6.34 17.66-1.41 1.41" />
<path d="m19.07 4.93-1.41 1.41" />
</svg>
</label>
</div>
<div>
<input
type="radio"
name="theme"
id="dark"
value="dark"
class="peer hidden"
/>
<label
for="dark"
tabindex="0"
title="Dark"
class="flex items-end justify-center h-10 w-10 cursor-pointer select-none rounded-md p-2 text-base text-slate-600 dark:text-slate-200 text-center hover:bg-slate-200 dark:hover:bg-slate-700 peer-checked:bg-slate-200 dark:peer-checked:bg-slate-700"
>
<svg
xmlns="http://www.w3.org/2000/svg"
width="24"
height="24"
viewBox="0 0 24 24"
fill="none"
stroke="currentColor"
stroke-width="2"
stroke-linecap="round"
stroke-linejoin="round"
class="h-5 w-5"
>
<path d="M12 3a6 6 0 0 0 9 9 9 9 0 1 1-9-9Z" />
</svg>
</label>
</div>
<div>
<input
type="radio"
name="theme"
id="system"
value="system"
class="peer hidden"
checked
/>
<label
for="system"
tabindex="0"
title="System preference"
class="flex items-end justify-center h-10 w-10 cursor-pointer select-none rounded-md p-2 text-lg text-slate-600 dark:text-slate-200 text-center hover:bg-slate-200 dark:hover:bg-slate-700 peer-checked:bg-slate-200 dark:peer-checked:bg-slate-700"
>
<svg
xmlns="http://www.w3.org/2000/svg"
width="24"
height="24"
viewBox="0 0 24 24"
fill="none"
stroke="currentColor"
stroke-width="2"
stroke-linecap="round"
stroke-linejoin="round"
class="h-5 w-5"
>
<path d="M12 8a2.83 2.83 0 0 0 4 4 4 4 0 1 1-4-4" />
<path d="M12 2v2" />
<path d="M12 20v2" />
<path d="m4.9 4.9 1.4 1.4" />
<path d="m17.7 17.7 1.4 1.4" />
<path d="M2 12h2" />
<path d="M20 12h2" />
<path d="m6.3 17.7-1.4 1.4" />
<path d="m19.1 4.9-1.4 1.4" />
</svg>
</label>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
<main class="flex flex-col space-y-3">
{{ if not .Success}}
<h1
class="text-3xl sm:text-4xl font-extrabold text-slate-900 tracking-tight dark:text-slate-200"
>
Error
</h1>
<p
class="leading-7 [&:not(:first-child)]:mt-6 text-slate-900 dark:text-slate-200"
>
There was a problem querying
<a
href="{{ .Params }}"
class="hover:text-blue-500 hover:underline underline-offset-2 transition-colors duration-300"
>{{ .Params }}</a
>
</p>
<code
class="m-auto text-red-500 dark:text-red-400 relative rounded bg-slate-200 dark:bg-slate-800 px-[0.3rem] py-[0.2rem] font-mono text-sm font-semibold"
>
{{ .Type }}: {{ .Message }}
<div class="my-2">Cause:</div>
{{ .Cause }}
</code>
{{else}}
<div class="flex flex-col gap-1 mt-3">
<div>
<small class="text-sm font-medium leading-none">
<a
href="{{.Url}}"
class="text-slate-600 dark:text-slate-400 hover:text-blue-500 hover:underline underline-offset-2 transition-colors duration-300"
>{{.Url}}</a
>
</small>
</div>
<h1
class="text-3xl sm:text-4xl font-extrabold text-slate-900 tracking-tight dark:text-slate-200"
>
{{.Title}}
</h1>
{{ if ne .Date "" }}
<small
class="text-sm font-medium leading-none text-slate-600 dark:text-slate-400"
>{{.date}}</small
>
{{ end }} {{ if ne .Author "" }}
<small
class="text-sm font-medium leading-none text-slate-600 dark:text-slate-400"
>{{.Author}}</small
>
{{ end }}
</div>
<div class="flex flex-col space-y-3">
<div>{{ .Params }}</div>
<div>{{ .Body }}</div>
{{ end }}
<!-- Trick Tailwind into compiling these styles into styles.css -->
<!-- <div class="hidden text-xs text-sm text-base text-xl text-2xl text-3xl text-4xl sm:text-3xl sm:text-4xl sm:text-5xl"></div> -->
</main>
<div class="my-2"></div>
<footer class="mx-4 text-center text-slate-600 dark:text-slate-400">
<p>
Code Licensed Under GPL v3.0 |
<a
href="https://github.com/everywall/ladder"
class="hover:text-blue-500 dark:hover:text-blue-500 hover:underline underline-offset-2 transition-colors duration-300"
>View Source</a
>
|
<a
href="https://github.com/everywall"
class="hover:text-blue-500 dark:hover:text-blue-500 hover:underline underline-offset-2 transition-colors duration-300"
>Everywall</a
>
</p>
</footer>
<div class="my-2"></div>
</div>
<script src="/script.js"></script>
</body>
</html>