36 lines
976 B
Go
36 lines
976 B
Go
package responsemodifers
|
|
|
|
import (
|
|
"github.com/go-shiori/dom"
|
|
"github.com/markusmobius/go-trafilatura"
|
|
"io"
|
|
"ladder/proxychain"
|
|
"strings"
|
|
)
|
|
|
|
// Outline creates an JSON representation of the article
|
|
func Outline() proxychain.ResponseModification {
|
|
return func(chain *proxychain.ProxyChain) error {
|
|
// Use readability
|
|
opts := trafilatura.Options{
|
|
IncludeImages: true,
|
|
IncludeLinks: true,
|
|
//FavorPrecision: true,
|
|
FallbackCandidates: nil, // TODO: https://github.com/markusmobius/go-trafilatura/blob/main/examples/chained/main.go
|
|
// implement fallbacks from "github.com/markusmobius/go-domdistiller" and "github.com/go-shiori/go-readability"
|
|
OriginalURL: chain.Request.URL,
|
|
}
|
|
|
|
result, err := trafilatura.Extract(chain.Response.Body, opts)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
doc := trafilatura.CreateReadableDocument(result)
|
|
reader := io.NopCloser(strings.NewReader(dom.OuterHTML(doc)))
|
|
chain.Response.Body = reader
|
|
return nil
|
|
}
|
|
|
|
}
|