What I am trying to do :
Dump the dom after it is rendered using chrome headless and go.
I saw over the code of aqautone to get an idea as to how to use chrome headless for my needs.
Problem
The problem is if I add many urls it just time outs printing Dumping dom timed out and currently when there's only two urls it prints :
<html><head></head><body></body></html>
<html><head></head><body></body></html>
<html><head></head><body></body></html>
<html><head></head><body></body></html>
Checking bingBot
1.73 URL : https://www.facebook.com
<html><head></head><body></body></html>
Checking yahooBot
1.75 URL : https://www.facebook.com
Checking bingBot
1.74 URL : https://www.google.com
Checking googleBot
1.75 URL : https://www.google.com
Checking googleBot
1.76 URL : https://www.facebook.com
<html><head></head><body></body></html>
Checking yahooBot
1.76 URL : https://www.google.com
Total time taken 1.78s elapsed
Here's my go code : Playground ( However since there's no chrome exec there so it is no point)
In case you don't want to go to the Playground :
package main
import (
"context"
"fmt"
"io"
"log"
"os"
"os/exec"
"sync"
"time"
//"github.com/remeh/sizedwaitgroup"
)
// Taken from https://deviceatlas.com/blog/list-of-user-agent-strings
var (
useragentstrings = map[string]string{
"googleBot": "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
"bingBot": "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)",
"yahooBot": "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)",
}
chromePath = "/root/tools/chromium-latest-linux/latest/chrome"
)
func fetchUsingChrome(url string, nameOfAgent string, useragent string, ch chan<- string, wg *sync.WaitGroup) {
// func fetchUsingChrome(url string, nameOfAgent string, useragent string, ch chan<- string, wg *sizedwaitgroup.SizedWaitGroup) {
start := time.Now()
defer wg.Done()
var chromeArguments = []string{
"--headless", "--disable-gpu", "--hide-scrollbars", "--mute-audio", "--disable-notifications",
"--disable-crash-reporter",
"--ignore-certificate-errors",
"--dump-dom",
// "--screenshot=" + url + ".png",
"--user-agent=" + useragent,
}
if os.Geteuid() == 0 {
chromeArguments = append(chromeArguments, "--no-sandbox")
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
cmd := exec.CommandContext(ctx, chromePath, chromeArguments...)
stdout, _ := cmd.StdoutPipe()
if err := cmd.Start(); err != nil {
fmt.Printf("[%s]Dumping dom failed Error: %v
", err)
return
}
// buf := new(bytes.Buffer)
// if err := buf.ReadFrom(stdout); err != nil {
// fmt.Println(err)
// return
// }
// fmt.Println(buf.String())
if _, err := io.Copy(os.Stdout, stdout); err != nil {
log.Fatal(err)
}
if err := cmd.Wait(); err != nil {
if ctx.Err() == context.DeadlineExceeded {
fmt.Printf("Dumping dom timed out : %s
", url)
return
}
fmt.Printf("[%s]Dumping dom failed Error: %v
", err)
return
}
secs := time.Since(start).Seconds()
fmt.Println("Checking ", nameOfAgent)
ch <- fmt.Sprintf("%.2f URL : %s", secs, url)
}
func main() {
var wg sync.WaitGroup
output := []string{
"https://www.facebook.com",
"https://www.google.com",
}
start := time.Now()
ch := make(chan string)
//swg := sizedwaitgroup.New(10)
for _, url := range output {
for k, v := range useragentstrings {
wg.Add(1)
//swg.Add()
go fetchUsingChrome(url, k, v, ch, &wg)
// go fetchUsingChrome(url, k, v, ch, &swg)
}
}
go func() {
wg.Wait()
// swg.Wait()
close(ch)
}()
for val := range ch {
fmt.Println(val)
}
fmt.Printf("Total time taken %.2fs elapsed
", time.Since(start).Seconds())
}
P.S. : I've deliberately left some commented code, to give an insight as to what all I've tried and failed.
Thanks
Temporarya
( A golang noobie )
