如何在Golang请求中执行JavaScript异步代码

I need to read the content of a dynamically generated page with from a website with this code in , it works fine for non ajax pages but i can't find a package or example to do this. Thanks.

package main

import(

    "fmt"
    "time"
    "net/http"
    "github.com/PuerkitoBio/goquery"
    "strings"
    "strconv"

)

func main() {

    var masterURI [1]string
    masterURI[0] = "http://uri1"
    /*masterURI[1] = "http://uri2"
    masterURI[2] = "http://uri3"*/

    for _, uri := range masterURI {

        doc, err := extractHTML(uri)

        check(err)

        search := doc.Find(".pagination li a span").Eq(-1)

        numPages, err := strconv.Atoi(search.Text())

        check(err)

        var i int

        for i = 1; i <= numPages; i++ {

            page := uri + "#/page-" + strconv.Itoa(i)

            fmt.Println("
========> " + page)

            doc, err := extractHTML(page)

            check(err)

            search := doc.Find(".product-name")

            for i := range search.Nodes {
                product := strings.TrimSpace(search.Eq(i).Text())
                fmt.Println(product)
                // use `single` as a selection of 1 node
            }

        }

        fmt.Println(" --- ")

    }

}

func extractHTML(url string) (doc *goquery.Document, err error) {

    userAgent := "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.93 Safari/537.36 OPR/32.0.1948.69";

    trans := &http.Transport{
       DisableKeepAlives: true,
    }

    timeout := time.Duration(600 * time.Second)

    myClient := &http.Client{Transport: trans, Timeout: timeout}

    req, err := http.NewRequest("GET", url, nil)

    check(err)

    req.Header.Add("User-Agent", userAgent)

    resp, err := myClient.Do(req)

    req.Close = true

    check(err)

    if resp.StatusCode == 200 {

        doc, err = goquery.NewDocumentFromResponse(resp) 

    }

    check(err)

    resp.Body.Close()

    return doc, err

}

func check(err error) {

    if err != nil {

        panic(err)

    }

}
查看全部
dongmi4720
dongmi4720
2015/10/23 16:44
  • Golang
  • request
  • asynchronous
  • ajax
  • javascript
  • 点赞
  • 收藏
  • 回答
    私信
满意答案
查看全部

1个回复