improve outline api with error handling and proper content-type response
This commit is contained in:
Kevin Pham
29
go.mod
29
go.mod
@@ -4,21 +4,38 @@ go 1.21.1
|
||||
|
||||
require (
|
||||
github.com/akamensky/argparse v1.4.0
|
||||
github.com/bogdanfinn/fhttp v0.5.24
|
||||
github.com/bogdanfinn/tls-client v1.6.1
|
||||
github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c
|
||||
github.com/gofiber/fiber/v2 v2.50.0
|
||||
github.com/markusmobius/go-trafilatura v1.5.1
|
||||
github.com/stretchr/testify v1.8.4
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/bogdanfinn/fhttp v0.5.24 // indirect
|
||||
github.com/bogdanfinn/tls-client v1.6.1 // indirect
|
||||
github.com/abadojack/whatlanggo v1.0.1 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.2 // indirect
|
||||
github.com/bogdanfinn/utls v1.5.16 // indirect
|
||||
github.com/cloudflare/circl v1.3.6 // indirect
|
||||
github.com/gaukas/godicttls v0.0.4 // indirect
|
||||
github.com/kr/pretty v0.1.0 // indirect
|
||||
github.com/quic-go/quic-go v0.40.0 // indirect
|
||||
github.com/elliotchance/pie/v2 v2.8.0 // indirect
|
||||
github.com/forPelevin/gomoji v1.1.8 // indirect
|
||||
github.com/go-shiori/go-readability v0.0.0-20231029095239-6b97d5aba789 // indirect
|
||||
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect
|
||||
github.com/hablullah/go-hijri v1.0.2 // indirect
|
||||
github.com/hablullah/go-juliandays v1.0.0 // indirect
|
||||
github.com/jalaali/go-jalaali v0.0.0-20210801064154-80525e88d958 // indirect
|
||||
github.com/magefile/mage v1.15.0 // indirect
|
||||
github.com/markusmobius/go-dateparser v1.2.1 // indirect
|
||||
github.com/markusmobius/go-domdistiller v0.0.0-20230515154422-71af71939ff3 // indirect
|
||||
github.com/markusmobius/go-htmldate v1.2.2 // indirect
|
||||
github.com/rs/zerolog v1.31.0 // indirect
|
||||
github.com/sirupsen/logrus v1.9.3 // indirect
|
||||
github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5 // indirect
|
||||
github.com/tetratelabs/wazero v1.5.0 // indirect
|
||||
github.com/wasilibs/go-re2 v1.4.1 // indirect
|
||||
github.com/yosssi/gohtml v0.0.0-20201013000340-ee4748c638f4 // indirect
|
||||
golang.org/x/crypto v0.16.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20231127185646-65229373498e // indirect
|
||||
golang.org/x/text v0.14.0 // indirect
|
||||
)
|
||||
|
||||
|
||||
132
go.sum
132
go.sum
@@ -1,90 +1,152 @@
|
||||
github.com/abadojack/whatlanggo v1.0.1 h1:19N6YogDnf71CTHm3Mp2qhYfkRdyvbgwWdd2EPxJRG4=
|
||||
github.com/abadojack/whatlanggo v1.0.1/go.mod h1:66WiQbSbJBIlOZMsvbKe5m6pzQovxCH9B/K8tQB2uoc=
|
||||
github.com/akamensky/argparse v1.4.0 h1:YGzvsTqCvbEZhL8zZu2AiA5nq805NZh75JNj4ajn1xc=
|
||||
github.com/akamensky/argparse v1.4.0/go.mod h1:S5kwC7IuDcEr5VeXtGPRVZ5o/FdhcMlQz4IZQuw64xA=
|
||||
github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI=
|
||||
github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
|
||||
github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
|
||||
github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
|
||||
github.com/bogdanfinn/fhttp v0.5.24 h1:OlyBKjvJp6a3TotN3wuj4mQHHRbfK7QUMrzCPOZGhRc=
|
||||
github.com/bogdanfinn/fhttp v0.5.24/go.mod h1:brqi5woc5eSCVHdKYBV8aZLbO7HGqpwyDLeXW+fT18I=
|
||||
github.com/bogdanfinn/fhttp v0.5.25-0.20231126232510-b5ab2425c56f h1:mmyiFfcYbiAyJhMQ0R9XjLfa7Xmd5dmpPswe6Rn/MJQ=
|
||||
github.com/bogdanfinn/fhttp v0.5.25-0.20231126232510-b5ab2425c56f/go.mod h1:brqi5woc5eSCVHdKYBV8aZLbO7HGqpwyDLeXW+fT18I=
|
||||
github.com/bogdanfinn/fhttp v0.6.0-beta h1:EmRFT7L39EWtdhd/4n/84hduTjR3F8g6ME3Gsgf/OYQ=
|
||||
github.com/bogdanfinn/fhttp v0.6.0-beta/go.mod h1:5Y+1PZFdQ1fZST6AYm4f1lEDc91Qa4rqPXH9+lKQxp0=
|
||||
github.com/bogdanfinn/tls-client v1.6.1 h1:GTIqQssFoIvLaDf4btoYRzDhUzudLqYD4axvfUCXl3I=
|
||||
github.com/bogdanfinn/tls-client v1.6.1/go.mod h1:FtwQ3DndVZ0xAOO704v4iNAgbHOcEc5kPk9tjICTNQ0=
|
||||
github.com/bogdanfinn/tls-client v1.6.2-0.20231126231757-ac864669c643 h1:ZqHtnRfVg8r95y1PuIC3AzHzQc+ezgRMOveX1um14/s=
|
||||
github.com/bogdanfinn/tls-client v1.6.2-0.20231126231757-ac864669c643/go.mod h1:FtwQ3DndVZ0xAOO704v4iNAgbHOcEc5kPk9tjICTNQ0=
|
||||
github.com/bogdanfinn/utls v1.5.16 h1:NhhWkegEcYETBMj9nvgO4lwvc6NcLH+znrXzO3gnw4M=
|
||||
github.com/bogdanfinn/utls v1.5.16/go.mod h1:mHeRCi69cUiEyVBkKONB1cAbLjRcZnlJbGzttmiuK4o=
|
||||
github.com/bogdanfinn/utls v1.6.0-beta-2 h1:f/u3WICv2/GD44JofwajjMGhqgyCPMpc9Ou1HYVMjnA=
|
||||
github.com/bogdanfinn/utls v1.6.0-beta-2/go.mod h1:VQBQ4r9Ks905rxLidZpdRWO1bFk4SyILUVPEyC7zBIY=
|
||||
github.com/cloudflare/circl v1.3.6 h1:/xbKIqSHbZXHwkhbrhrt2YOHIwYJlXH94E3tI/gDlUg=
|
||||
github.com/cloudflare/circl v1.3.6/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
|
||||
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/gaukas/godicttls v0.0.4 h1:NlRaXb3J6hAnTmWdsEKb9bcSBD6BvcIjdGdeb0zfXbk=
|
||||
github.com/gaukas/godicttls v0.0.4/go.mod h1:l6EenT4TLWgTdwslVb4sEMOCf7Bv0JAK67deKr9/NCI=
|
||||
github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
|
||||
github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
|
||||
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
|
||||
github.com/elliotchance/pie/v2 v2.8.0 h1://QS43W8sEha8XV/fjngO5iMudN3XARJV5cpBayAcVY=
|
||||
github.com/elliotchance/pie/v2 v2.8.0/go.mod h1:18t0dgGFH006g4eVdDtWfgFZPQEgl10IoEO8YWEq3Og=
|
||||
github.com/forPelevin/gomoji v1.1.8 h1:JElzDdt0TyiUlecy6PfITDL6eGvIaxqYH1V52zrd0qQ=
|
||||
github.com/forPelevin/gomoji v1.1.8/go.mod h1:8+Z3KNGkdslmeGZBC3tCrwMrcPy5GRzAD+gL9NAwMXg=
|
||||
github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c h1:wpkoddUomPfHiOziHZixGO5ZBS73cKqVzZipfrLmO1w=
|
||||
github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c/go.mod h1:oVDCh3qjJMLVUSILBRwrm+Bc6RNXGZYtoh9xdvf1ffM=
|
||||
github.com/go-shiori/go-readability v0.0.0-20231029095239-6b97d5aba789 h1:G6wSuUyCoLB9jrUokipsmFuRi8aJozt3phw/g9Sl4Xs=
|
||||
github.com/go-shiori/go-readability v0.0.0-20231029095239-6b97d5aba789/go.mod h1:2DpZlTJO/ycxp/vsc/C11oUyveStOgIXB88SYV1lncI=
|
||||
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
|
||||
github.com/gofiber/fiber/v2 v2.50.0 h1:ia0JaB+uw3GpNSCR5nvC5dsaxXjRU5OEu36aytx+zGw=
|
||||
github.com/gofiber/fiber/v2 v2.50.0/go.mod h1:21eytvay9Is7S6z+OgPi7c7n4++tnClWmhpimVHMimw=
|
||||
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
|
||||
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE=
|
||||
github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs=
|
||||
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14=
|
||||
github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
|
||||
github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/hablullah/go-hijri v1.0.2 h1:drT/MZpSZJQXo7jftf5fthArShcaMtsal0Zf/dnmp6k=
|
||||
github.com/hablullah/go-hijri v1.0.2/go.mod h1:OS5qyYLDjORXzK4O1adFw9Q5WfhOcMdAKglDkcTxgWQ=
|
||||
github.com/hablullah/go-juliandays v1.0.0 h1:A8YM7wIj16SzlKT0SRJc9CD29iiaUzpBLzh5hr0/5p0=
|
||||
github.com/hablullah/go-juliandays v1.0.0/go.mod h1:0JOYq4oFOuDja+oospuc61YoX+uNEn7Z6uHYTbBzdGc=
|
||||
github.com/jalaali/go-jalaali v0.0.0-20210801064154-80525e88d958 h1:qxLoi6CAcXVzjfvu+KXIXJOAsQB62LXjsfbOaErsVzE=
|
||||
github.com/jalaali/go-jalaali v0.0.0-20210801064154-80525e88d958/go.mod h1:Wqfu7mjUHj9WDzSSPI5KfBclTTEnLveRUFr/ujWnTgE=
|
||||
github.com/klauspost/compress v1.17.3 h1:qkRjuerhUU1EmXLYGkSH6EZL+vPSxIrYjLNAK4slzwA=
|
||||
github.com/klauspost/compress v1.17.3/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
|
||||
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg=
|
||||
github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
|
||||
github.com/markusmobius/go-dateparser v1.2.1 h1:mYRRdu3TzpAeE6fSl2Gn3arfxEtoTRvFOKlumlVsUtg=
|
||||
github.com/markusmobius/go-dateparser v1.2.1/go.mod h1:5xYsZ1h7iB3sE1BSu8bkjYpbFST7EU1/AFxcyO3mgYg=
|
||||
github.com/markusmobius/go-domdistiller v0.0.0-20230515154422-71af71939ff3 h1:D83RvMz1lQ0ilKlJt6DWc65+Q77CXGRFmfihR0bfQvc=
|
||||
github.com/markusmobius/go-domdistiller v0.0.0-20230515154422-71af71939ff3/go.mod h1:n1AYw0wiJDT3YXnIsElJPiDR63YGXT2yv3uq0CboGmU=
|
||||
github.com/markusmobius/go-htmldate v1.2.2 h1:tp1IxhefCYpEoL9CM1LiU6l+2YayTpuTjkkdnik6hXE=
|
||||
github.com/markusmobius/go-htmldate v1.2.2/go.mod h1:26VRz16sCosuiv42MNRW9iPBGnGLo+q/Z6TWitt8uzs=
|
||||
github.com/markusmobius/go-trafilatura v1.5.1 h1:EXhZY2AVRyepUlLZHeuZUme3v7Ms9G8lDOLl4u+Jp5M=
|
||||
github.com/markusmobius/go-trafilatura v1.5.1/go.mod h1:FhuBBPZ9ph4ufpGBKAkuq5oQwEhg0KKnIOUlv5h7EHg=
|
||||
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
|
||||
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
|
||||
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q=
|
||||
github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k=
|
||||
github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE=
|
||||
github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg=
|
||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
|
||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/quic-go/quic-go v0.40.0 h1:GYd1iznlKm7dpHD7pOVpUvItgMPo/jrMgDWZhMCecqw=
|
||||
github.com/quic-go/quic-go v0.40.0/go.mod h1:PeN7kuVJ4xZbxSv/4OX6S1USOX8MJvydwpTx31vx60c=
|
||||
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
|
||||
github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
|
||||
github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A=
|
||||
github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
|
||||
github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
|
||||
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
|
||||
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
|
||||
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5 h1:YqAladjX7xpA6BM04leXMWAEjS0mTZ5kUU9KRBriQJc=
|
||||
github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5/go.mod h1:2JjD2zLQYH5HO74y5+aE3remJQvl6q4Sn6aWA2wD1Ng=
|
||||
github.com/tetratelabs/wazero v1.5.0 h1:Yz3fZHivfDiZFUXnWMPUoiW7s8tC1sjdBtlJn08qYa0=
|
||||
github.com/tetratelabs/wazero v1.5.0/go.mod h1:0U0G41+ochRKoPKCJlh0jMg1CHkyfK8kDqiirMmKY8A=
|
||||
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
|
||||
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||
github.com/valyala/fasthttp v1.50.0 h1:H7fweIlBm0rXLs2q0XbalvJ6r0CUPFWK3/bB4N13e9M=
|
||||
github.com/valyala/fasthttp v1.50.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA=
|
||||
github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
|
||||
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
|
||||
github.com/wasilibs/go-re2 v1.4.1 h1:E5+9O1M8UoGeqLB2A9omeoaWImqpuYDs9cKwvTJq/Oo=
|
||||
github.com/wasilibs/go-re2 v1.4.1/go.mod h1:ynB8eCwd9JsqUnsk8WlPDk6cEeme8BguZmnqOSURE4Y=
|
||||
github.com/wasilibs/nottinygc v0.4.0 h1:h1TJMihMC4neN6Zq+WKpLxgd9xCFMw7O9ETLwY2exJQ=
|
||||
github.com/wasilibs/nottinygc v0.4.0/go.mod h1:oDcIotskuYNMpqMF23l7Z8uzD4TC0WXHK8jetlB3HIo=
|
||||
github.com/yosssi/gohtml v0.0.0-20201013000340-ee4748c638f4 h1:0sw0nJM544SpsihWx1bkXdYLQDlzRflMgFJQ4Yih9ts=
|
||||
github.com/yosssi/gohtml v0.0.0-20201013000340-ee4748c638f4/go.mod h1:+ccdNT0xMY1dtc5XBxumbYfOUhmduiGudqaDgD2rVRE=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY=
|
||||
golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
|
||||
golang.org/x/exp v0.0.0-20231127185646-65229373498e h1:Gvh4YaCaXNs6dKTlfgismwWZKyjVZXwOPfIyUaqU3No=
|
||||
golang.org/x/exp v0.0.0-20231127185646-65229373498e/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
|
||||
golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
|
||||
golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
|
||||
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
|
||||
golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
|
||||
golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo=
|
||||
golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U=
|
||||
gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
@@ -35,9 +35,9 @@ func NewProxySiteHandler(opts *ProxyOptions) fiber.Handler {
|
||||
SetFiberCtx(c).
|
||||
SetDebugLogging(opts.Verbose).
|
||||
SetRequestModifications(
|
||||
rx.SpoofJA3fingerprint(ja3, "Googlebot"),
|
||||
//rx.SpoofJA3fingerprint(ja3, "Googlebot"),
|
||||
//rx.MasqueradeAsFacebookBot(),
|
||||
//rx.MasqueradeAsGoogleBot(),
|
||||
rx.MasqueradeAsGoogleBot(),
|
||||
//rx.DeleteOutgoingCookies(),
|
||||
rx.ForwardRequestHeaders(),
|
||||
rx.SetOutgoingCookie("nyt-a", " "),
|
||||
@@ -55,7 +55,8 @@ func NewProxySiteHandler(opts *ProxyOptions) fiber.Handler {
|
||||
tx.BypassContentSecurityPolicy(),
|
||||
//tx.DeleteIncomingCookies(),
|
||||
tx.RewriteHTMLResourceURLs(),
|
||||
tx.PatchDynamicResourceURLs(),
|
||||
//tx.PatchDynamicResourceURLs(),
|
||||
tx.APIOutline(),
|
||||
//tx.SetContentSecurityPolicy("default-src * 'unsafe-inline' 'unsafe-eval' data: blob:;"),
|
||||
).
|
||||
Execute()
|
||||
|
||||
@@ -487,9 +487,11 @@ func (chain *ProxyChain) Execute() error {
|
||||
}
|
||||
|
||||
// in case api user did not set or forward content-type, we do it for them
|
||||
/*
|
||||
if chain.Context.Get("content-type") == "" {
|
||||
chain.Context.Set("content-type", chain.Response.Header.Get("content-type"))
|
||||
}
|
||||
*/
|
||||
|
||||
// Return request back to client
|
||||
return chain.Context.SendStream(body)
|
||||
|
||||
55
proxychain/responsemodifers/api/error_api.go
Normal file
55
proxychain/responsemodifers/api/error_api.go
Normal file
@@ -0,0 +1,55 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"reflect"
|
||||
)
|
||||
|
||||
type APIError struct {
|
||||
Success bool `json:"success"`
|
||||
Error ErrorDetails `json:"error"`
|
||||
}
|
||||
|
||||
type ErrorDetails struct {
|
||||
Message string `json:"message"`
|
||||
Type string `json:"type"`
|
||||
Cause string `json:"cause"`
|
||||
}
|
||||
|
||||
func CreateAPIErrReader(err error) io.ReadCloser {
|
||||
if err == nil {
|
||||
return io.NopCloser(bytes.NewBufferString(`{"success":false, "error": "No error provided"}`))
|
||||
}
|
||||
|
||||
baseErr := getBaseError(err)
|
||||
apiErr := APIError{
|
||||
Success: false,
|
||||
Error: ErrorDetails{
|
||||
Message: err.Error(),
|
||||
Type: reflect.TypeOf(err).String(),
|
||||
Cause: baseErr.Error(),
|
||||
},
|
||||
}
|
||||
|
||||
// Serialize the APIError into JSON
|
||||
jsonData, jsonErr := json.Marshal(apiErr)
|
||||
if jsonErr != nil {
|
||||
return io.NopCloser(bytes.NewBufferString(`{"success":false, "error": "Failed to serialize error"}`))
|
||||
}
|
||||
|
||||
// Return the JSON data as an io.ReadCloser
|
||||
return io.NopCloser(bytes.NewBuffer(jsonData))
|
||||
}
|
||||
|
||||
func getBaseError(err error) error {
|
||||
for {
|
||||
unwrapped := errors.Unwrap(err)
|
||||
if unwrapped == nil {
|
||||
return err
|
||||
}
|
||||
err = unwrapped
|
||||
}
|
||||
}
|
||||
135
proxychain/responsemodifers/api/outline_api.go
Normal file
135
proxychain/responsemodifers/api/outline_api.go
Normal file
@@ -0,0 +1,135 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"github.com/go-shiori/dom"
|
||||
"github.com/markusmobius/go-trafilatura"
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
// =======================================================================================
|
||||
// credit @joncrangle https://github.com/everywall/ladder/issues/38#issuecomment-1831252934
|
||||
|
||||
type ImageContent struct {
|
||||
Type string `json:"type"`
|
||||
URL string `json:"url"`
|
||||
Alt string `json:"alt"`
|
||||
Caption string `json:"caption"`
|
||||
}
|
||||
|
||||
type LinkContent struct {
|
||||
Type string `json:"type"`
|
||||
Href string `json:"href"`
|
||||
Data string `json:"data"`
|
||||
}
|
||||
|
||||
type TextContent struct {
|
||||
Type string `json:"type"`
|
||||
Data string `json:"data"`
|
||||
}
|
||||
|
||||
type JSONDocument struct {
|
||||
Success bool `json:"success"`
|
||||
Error ErrorDetails `json:"error"`
|
||||
Metadata struct {
|
||||
Title string `json:"title"`
|
||||
Author string `json:"author"`
|
||||
URL string `json:"url"`
|
||||
Hostname string `json:"hostname"`
|
||||
Description string `json:"description"`
|
||||
Sitename string `json:"sitename"`
|
||||
Date string `json:"date"`
|
||||
Categories []string `json:"categories"`
|
||||
Tags []string `json:"tags"`
|
||||
License string `json:"license"`
|
||||
} `json:"metadata"`
|
||||
Content []interface{} `json:"content"`
|
||||
Comments string `json:"comments"`
|
||||
}
|
||||
|
||||
func ExtractResultToAPIResponse(extract *trafilatura.ExtractResult) *JSONDocument {
|
||||
jsonDoc := &JSONDocument{}
|
||||
|
||||
// Populate success
|
||||
jsonDoc.Success = true
|
||||
|
||||
// Populate metadata
|
||||
jsonDoc.Metadata.Title = extract.Metadata.Title
|
||||
jsonDoc.Metadata.Author = extract.Metadata.Author
|
||||
jsonDoc.Metadata.URL = extract.Metadata.URL
|
||||
jsonDoc.Metadata.Hostname = extract.Metadata.Hostname
|
||||
jsonDoc.Metadata.Description = extract.Metadata.Description
|
||||
jsonDoc.Metadata.Sitename = extract.Metadata.Sitename
|
||||
jsonDoc.Metadata.Date = extract.Metadata.Date.Format("2006-01-02")
|
||||
jsonDoc.Metadata.Categories = extract.Metadata.Categories
|
||||
jsonDoc.Metadata.Tags = extract.Metadata.Tags
|
||||
jsonDoc.Metadata.License = extract.Metadata.License
|
||||
|
||||
// Populate content
|
||||
if extract.ContentNode != nil {
|
||||
jsonDoc.Content = parseContent(extract.ContentNode)
|
||||
}
|
||||
|
||||
// Populate comments
|
||||
if extract.CommentsNode != nil {
|
||||
jsonDoc.Comments = dom.OuterHTML(extract.CommentsNode)
|
||||
}
|
||||
|
||||
return jsonDoc
|
||||
}
|
||||
|
||||
func parseContent(node *html.Node) []interface{} {
|
||||
var content []interface{}
|
||||
|
||||
for child := node.FirstChild; child != nil; child = child.NextSibling {
|
||||
switch child.Data {
|
||||
case "img":
|
||||
image := ImageContent{
|
||||
Type: "img",
|
||||
URL: dom.GetAttribute(child, "src"),
|
||||
Alt: dom.GetAttribute(child, "alt"),
|
||||
Caption: dom.GetAttribute(child, "caption"),
|
||||
}
|
||||
content = append(content, image)
|
||||
|
||||
case "a":
|
||||
link := LinkContent{
|
||||
Type: "a",
|
||||
Href: dom.GetAttribute(child, "href"),
|
||||
Data: dom.InnerText(child),
|
||||
}
|
||||
content = append(content, link)
|
||||
|
||||
case "h1":
|
||||
text := TextContent{
|
||||
Type: "h1",
|
||||
Data: dom.InnerText(child),
|
||||
}
|
||||
content = append(content, text)
|
||||
|
||||
case "h2":
|
||||
text := TextContent{
|
||||
Type: "h2",
|
||||
Data: dom.InnerText(child),
|
||||
}
|
||||
content = append(content, text)
|
||||
|
||||
case "h3":
|
||||
text := TextContent{
|
||||
Type: "h3",
|
||||
Data: dom.InnerText(child),
|
||||
}
|
||||
content = append(content, text)
|
||||
|
||||
// continue with other tags
|
||||
|
||||
default:
|
||||
text := TextContent{
|
||||
Type: "p",
|
||||
Data: dom.InnerText(child),
|
||||
}
|
||||
content = append(content, text)
|
||||
}
|
||||
}
|
||||
|
||||
return content
|
||||
}
|
||||
@@ -3,18 +3,23 @@ package responsemodifers
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"github.com/go-shiori/dom"
|
||||
//"github.com/go-shiori/dom"
|
||||
"github.com/markusmobius/go-trafilatura"
|
||||
"golang.org/x/net/html"
|
||||
//"golang.org/x/net/html"
|
||||
"io"
|
||||
"ladder/proxychain"
|
||||
//"strings"
|
||||
"ladder/proxychain/responsemodifers/api"
|
||||
)
|
||||
|
||||
// Outline creates an JSON representation of the article
|
||||
func Outline() proxychain.ResponseModification {
|
||||
// APIOutline creates an JSON representation of the article and returns it as an API response.
|
||||
func APIOutline() proxychain.ResponseModification {
|
||||
return func(chain *proxychain.ProxyChain) error {
|
||||
// Use readability
|
||||
// we set content-type twice here, in case another response modifier
|
||||
// tries to forward over the original headers
|
||||
chain.Context.Set("content-type", "application/json")
|
||||
chain.Response.Header.Set("content-type", "application/json")
|
||||
|
||||
// extract dom contents
|
||||
opts := trafilatura.Options{
|
||||
IncludeImages: true,
|
||||
IncludeLinks: true,
|
||||
@@ -26,151 +31,20 @@ func Outline() proxychain.ResponseModification {
|
||||
|
||||
result, err := trafilatura.Extract(chain.Response.Body, opts)
|
||||
if err != nil {
|
||||
return err
|
||||
chain.Response.Body = api.CreateAPIErrReader(err)
|
||||
return nil
|
||||
}
|
||||
|
||||
doc := createJSONDocument(result)
|
||||
jsonData, err := json.MarshalIndent(doc, "", " ")
|
||||
doc := api.ExtractResultToAPIResponse(result)
|
||||
jsonData, err := json.MarshalIndent(doc, "", "\t")
|
||||
if err != nil {
|
||||
return err
|
||||
chain.Response.Body = api.CreateAPIErrReader(err)
|
||||
return nil
|
||||
}
|
||||
|
||||
buf := bytes.NewBuffer(jsonData)
|
||||
//doc := trafilatura.CreateReadableDocument(result)
|
||||
//reader := io.NopCloser(strings.NewReader(dom.OuterHTML(doc)))
|
||||
chain.Response.Body = io.NopCloser(buf)
|
||||
return nil
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// =======================================================================================
|
||||
// credit @joncrangle https://github.com/everywall/ladder/issues/38#issuecomment-1831252934
|
||||
|
||||
type ImageContent struct {
|
||||
Type string `json:"type"`
|
||||
URL string `json:"url"`
|
||||
Alt string `json:"alt"`
|
||||
Caption string `json:"caption"`
|
||||
}
|
||||
|
||||
type LinkContent struct {
|
||||
Type string `json:"type"`
|
||||
Href string `json:"href"`
|
||||
Data string `json:"data"`
|
||||
}
|
||||
|
||||
type TextContent struct {
|
||||
Type string `json:"type"`
|
||||
Data string `json:"data"`
|
||||
}
|
||||
|
||||
type JSONDocument struct {
|
||||
Success bool `json:"success"`
|
||||
Error struct {
|
||||
Message string `json:"message"`
|
||||
Type string `json:"type"`
|
||||
Cause string `json:"cause"`
|
||||
} `json:"error"`
|
||||
Metadata struct {
|
||||
Title string `json:"title"`
|
||||
Author string `json:"author"`
|
||||
URL string `json:"url"`
|
||||
Hostname string `json:"hostname"`
|
||||
Description string `json:"description"`
|
||||
Sitename string `json:"sitename"`
|
||||
Date string `json:"date"`
|
||||
Categories []string `json:"categories"`
|
||||
Tags []string `json:"tags"`
|
||||
License string `json:"license"`
|
||||
} `json:"metadata"`
|
||||
Content []interface{} `json:"content"`
|
||||
Comments string `json:"comments"`
|
||||
}
|
||||
|
||||
func createJSONDocument(extract *trafilatura.ExtractResult) *JSONDocument {
|
||||
jsonDoc := &JSONDocument{}
|
||||
|
||||
// Populate success
|
||||
jsonDoc.Success = true
|
||||
|
||||
// Populate metadata
|
||||
jsonDoc.Metadata.Title = extract.Metadata.Title
|
||||
jsonDoc.Metadata.Author = extract.Metadata.Author
|
||||
jsonDoc.Metadata.URL = extract.Metadata.URL
|
||||
jsonDoc.Metadata.Hostname = extract.Metadata.Hostname
|
||||
jsonDoc.Metadata.Description = extract.Metadata.Description
|
||||
jsonDoc.Metadata.Sitename = extract.Metadata.Sitename
|
||||
jsonDoc.Metadata.Date = extract.Metadata.Date.Format("2006-01-02")
|
||||
jsonDoc.Metadata.Categories = extract.Metadata.Categories
|
||||
jsonDoc.Metadata.Tags = extract.Metadata.Tags
|
||||
jsonDoc.Metadata.License = extract.Metadata.License
|
||||
|
||||
// Populate content
|
||||
if extract.ContentNode != nil {
|
||||
jsonDoc.Content = parseContent(extract.ContentNode)
|
||||
}
|
||||
|
||||
// Populate comments
|
||||
if extract.CommentsNode != nil {
|
||||
jsonDoc.Comments = dom.OuterHTML(extract.CommentsNode)
|
||||
}
|
||||
|
||||
return jsonDoc
|
||||
}
|
||||
|
||||
func parseContent(node *html.Node) []interface{} {
|
||||
var content []interface{}
|
||||
|
||||
for child := node.FirstChild; child != nil; child = child.NextSibling {
|
||||
switch child.Data {
|
||||
case "img":
|
||||
image := ImageContent{
|
||||
Type: "img",
|
||||
URL: dom.GetAttribute(child, "src"),
|
||||
Alt: dom.GetAttribute(child, "alt"),
|
||||
Caption: dom.GetAttribute(child, "caption"),
|
||||
}
|
||||
content = append(content, image)
|
||||
|
||||
case "a":
|
||||
link := LinkContent{
|
||||
Type: "a",
|
||||
Href: dom.GetAttribute(child, "href"),
|
||||
Data: dom.InnerText(child),
|
||||
}
|
||||
content = append(content, link)
|
||||
|
||||
case "h1":
|
||||
text := TextContent{
|
||||
Type: "h1",
|
||||
Data: dom.InnerText(child),
|
||||
}
|
||||
content = append(content, text)
|
||||
|
||||
case "h2":
|
||||
text := TextContent{
|
||||
Type: "h2",
|
||||
Data: dom.InnerText(child),
|
||||
}
|
||||
content = append(content, text)
|
||||
|
||||
case "h3":
|
||||
text := TextContent{
|
||||
Type: "h3",
|
||||
Data: dom.InnerText(child),
|
||||
}
|
||||
content = append(content, text)
|
||||
|
||||
// continue with other tags
|
||||
|
||||
default:
|
||||
text := TextContent{
|
||||
Type: "p",
|
||||
Data: dom.InnerText(child),
|
||||
}
|
||||
content = append(content, text)
|
||||
}
|
||||
}
|
||||
|
||||
return content
|
||||
}
|
||||
|
||||
68
proxychain/responsemodifers/outline_test.go
Normal file
68
proxychain/responsemodifers/outline_test.go
Normal file
@@ -0,0 +1,68 @@
|
||||
package responsemodifers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCreateAPIErrReader(t *testing.T) {
|
||||
_, baseErr := url.Parse("://this is an invalid url")
|
||||
wrappedErr := fmt.Errorf("wrapped error: %w", baseErr)
|
||||
|
||||
readCloser := CreateAPIErrReader(wrappedErr)
|
||||
defer readCloser.Close()
|
||||
|
||||
// Read and unmarshal the JSON output
|
||||
data, err := io.ReadAll(readCloser)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to read from ReadCloser: %v", err)
|
||||
}
|
||||
fmt.Println(string(data))
|
||||
|
||||
var apiErr APIError
|
||||
err = json.Unmarshal(data, &apiErr)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to unmarshal JSON: %v", err)
|
||||
}
|
||||
|
||||
// Verify the structure of the APIError
|
||||
if apiErr.Success {
|
||||
t.Errorf("Expected Success to be false, got true")
|
||||
}
|
||||
|
||||
if apiErr.Error.Message != wrappedErr.Error() {
|
||||
t.Errorf("Expected error message to be '%v', got '%v'", wrappedErr.Error(), apiErr.Error.Message)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateAPIErrReader2(t *testing.T) {
|
||||
_, baseErr := url.Parse("://this is an invalid url")
|
||||
|
||||
readCloser := CreateAPIErrReader(baseErr)
|
||||
defer readCloser.Close()
|
||||
|
||||
// Read and unmarshal the JSON output
|
||||
data, err := io.ReadAll(readCloser)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to read from ReadCloser: %v", err)
|
||||
}
|
||||
fmt.Println(string(data))
|
||||
|
||||
var apiErr APIError
|
||||
err = json.Unmarshal(data, &apiErr)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to unmarshal JSON: %v", err)
|
||||
}
|
||||
|
||||
// Verify the structure of the APIError
|
||||
if apiErr.Success {
|
||||
t.Errorf("Expected Success to be false, got true")
|
||||
}
|
||||
|
||||
if apiErr.Error.Message != baseErr.Error() {
|
||||
t.Errorf("Expected error message to be '%v', got '%v'", baseErr.Error(), apiErr.Error.Message)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user