donglan7594 2017-07-19 19:51
浏览 25
已采纳

这是并行编程的更好方法吗?

I made this script for getting the follower count of "influencers" from instagram

the "runtime" number I am getting from it is between 550-750ms. It is not that bad, but I am wondering whether it could be better or not (as I am a golang noob - learning it 3 weeks only)

package main

import (
    "encoding/json"
    "fmt"
    "io/ioutil"
    "log"
    "net/http"
    "sync"
    "time"
)

type user struct {
    User userData `json:"user"`
}

type userData struct {
    Followers count `json:"followed_by"`
}

type count struct {
    Count int `json:"count"`
}

func getFollowerCount(in <-chan string) <-chan int {
    out := make(chan int)
    go func() {
        for un := range in {
            URL := "https://www.instagram.com/" + un + "/?__a=1"
            resp, err := http.Get(URL)
            if err != nil {
                // handle error
                fmt.Println(err)
            }
            defer resp.Body.Close()
            body, err := ioutil.ReadAll(resp.Body)
            var u user
            err = json.Unmarshal(body, &u)
            if err != nil {
                fmt.Println(err)
            }
            // return u.User.Followers.Count
            out <- u.User.Followers.Count
        }
        close(out)
    }()
    return out
}

func merge(cs ...<-chan int) <-chan int {
    var wg sync.WaitGroup
    out := make(chan int)
    output := func(c <-chan int) {
        for n := range c {
            out <- n
        }
        wg.Done()
    }

    wg.Add(len(cs))
    for _, c := range cs {
        go output(c)
    }
    go func() {
        wg.Wait()
        close(out)
    }()
    return out
}

func gen(users ...string) <-chan string {
    out := make(chan string)
    go func() {
        for _, u := range users {
            out <- u
        }
        close(out)
    }()
    return out
}

func main() {
    start := time.Now()
    fmt.Println("STARTING UP")
    usrs := []string{"kanywest", "kimkardashian", "groovyq", "kendricklamar", "barackobama", "asaprocky", "champagnepapi", "eminem", "drdre", "g_eazy", "skrillex"}
    in := gen(usrs...)
    d1 := getFollowerCount(in)
    d2 := getFollowerCount(in)
    d3 := getFollowerCount(in)
    d4 := getFollowerCount(in)
    d5 := getFollowerCount(in)
    d6 := getFollowerCount(in)
    d7 := getFollowerCount(in)
    d8 := getFollowerCount(in)
    d9 := getFollowerCount(in)
    d10 := getFollowerCount(in)

    for d := range merge(d1, d2, d3, d4, d5, d6, d7, d8, d9, d10) {
        fmt.Println(d)
    }

    elapsed := time.Since(start)
    log.Println("runtime", elapsed)
}

展开全部

  • 写回答

2条回答 默认 最新

  • drbd65446 2017-07-19 20:48
    关注

    Welcome to Go, happy learning.

    You're doing good, you can improve your program many ways (such as json decoder, less no of chan, etc). Following is one of the approach. Execution time is between 352-446ms (take it with grain of salt, since network call is involved in your code. Might vary based on server response time).

    Your updated code:

    package main
    
    import (
        "encoding/json"
        "fmt"
        "log"
        "net/http"
        "sync"
        "time"
    )
    
    type user struct {
        User userData `json:"user"`
    }
    
    type userData struct {
        Followers count `json:"followed_by"`
    }
    
    type count struct {
        Count int `json:"count"`
    }
    
    func getFollowerCount(username string, result chan<- int, wg *sync.WaitGroup) {
        defer wg.Done()
        reqURL := "https://www.instagram.com/" + username + "/?__a=1"
        resp, err := http.Get(reqURL)
        if err != nil {
            log.Println(err)
            return
        }
        defer resp.Body.Close()
    
        var u user
        if err := json.NewDecoder(resp.Body).Decode(&u); err != nil {
            log.Println(err)
            return
        }
        result <- u.User.Followers.Count
    }
    
    func execute(users []string, result chan<- int) {
        wg := &sync.WaitGroup{}
        for _, username := range users {
            wg.Add(1)
            go getFollowerCount(username, result, wg)
        }
        wg.Wait()
        result <- -1
    }
    
    func main() {
        start := time.Now()
        fmt.Println("STARTING UP")
        usrs := []string{"kanywest", "kimkardashian", "groovyq", "kendricklamar", "barackobama", "asaprocky", "champagnepapi", "eminem", "drdre", "g_eazy", "skrillex"}
    
        result := make(chan int)
        go execute(usrs, result)
    
        for v := range result {
            if v == -1 {
                break
            }
            fmt.Println(v)
        }
    
        elapsed := time.Since(start)
        fmt.Println("runtime:", elapsed)
    }
    

    展开全部

    本回答被题主选为最佳回答 , 对您是否有帮助呢?
    评论
查看更多回答(1条)
编辑
预览

报告相同问题?

悬赏问题

  • ¥15 PADS Logic 原理图
  • ¥15 PADS Logic 图标
  • ¥15 电脑和power bi环境都是英文如何将日期层次结构转换成英文
  • ¥20 气象站点数据求取中~
  • ¥15 如何获取APP内弹出的网址链接
  • ¥15 wifi 图标不见了 不知道怎么办 上不了网 变成小地球了
手机看
程序员都在用的中文IT技术交流社区

程序员都在用的中文IT技术交流社区

专业的中文 IT 技术社区,与千万技术人共成长

专业的中文 IT 技术社区,与千万技术人共成长

关注【CSDN】视频号,行业资讯、技术分享精彩不断,直播好礼送不停!

关注【CSDN】视频号,行业资讯、技术分享精彩不断,直播好礼送不停!

客服 返回
顶部