package handlers
import (
"fmt"
"io"
"log"
"net/http"
"net/http/httputil"
"net/url"
"regexp"
"strings"
"time"
"github.com/gofiber/fiber/v2"
"github.com/gojek/heimdall/v7/httpclient"
)
type loggingTransport struct{}
func (s *loggingTransport) RoundTrip(r *http.Request) (*http.Response, error) {
bytes, _ := httputil.DumpRequestOut(r, true)
resp, err := http.DefaultTransport.RoundTrip(r)
// err is returned after dumping the response
respBytes, _ := httputil.DumpResponse(resp, false)
bytes = append(bytes, respBytes...)
fmt.Printf("%s\n", bytes)
return resp, err
}
func FetchSite(c *fiber.Ctx) error {
// Get the url from the URL
urlQuery := c.Params("*")
u, err := url.Parse(urlQuery)
if err != nil {
log.Fatal(err)
}
log.Println(u.String())
if u.Scheme == "" {
u.Scheme = "https"
}
// Fetch the site
//resp, err := http.Get(url)
req, _ := http.NewRequest("GET", u.String(), nil)
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)")
req.Header.Set("X-Forwarded-For", "66.249.66.1")
//fmt.Println(c.GetReqHeaders()["Cookie"])
//req.Header.Set("Cookie", fmt.Sprint(c.GetReqHeaders()["Cookie"]))
//resp, err := http.DefaultClient.Do(req)
//client := &http.Client{}
//client.Timeout = time.Duration(5 * time.Second)
client := http.Client{}
client.Transport = &loggingTransport{}
fmt.Println("DEBUG1")
resp, err := client.Do(req)
fmt.Println("DEBUG2")
/*
timeout := 1000 * time.Millisecond
client := httpclient.NewClient(httpclient.WithHTTPTimeout(timeout))
headers := http.Header{}
headers.Set("User-Agent", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)")
headers.Set("X-Forwarded-For", "66.249.66.1")
fmt.Println("DEBUG1")
resp, err := client.Get(u.String(), headers)
if err != nil {
panic(err)
}
fmt.Println("DEBUG2")
*/
if err != nil {
return c.SendString(err.Error())
}
defer resp.Body.Close()
bodyB, err := io.ReadAll(resp.Body)
if err != nil {
//log.Fatalln(err)
return c.SendString(err.Error())
}
body := string(bodyB)
imagePattern := `
]*\s+)?src="(/)([^"]*)"`
re := regexp.MustCompile(imagePattern)
body = re.ReplaceAllString(body, fmt.Sprintf(`
]*\s+)?src="(/)([^"]*)"`
reScript := regexp.MustCompile(scriptPattern)
body = reScript.ReplaceAllString(body, fmt.Sprintf(`