这是并行编程的更好方法吗？

Question

I made this script for getting the follower count of "influencers" from instagram

the "runtime" number I am getting from it is between 550-750ms. It is not that bad, but I am wondering whether it could be better or not (as I am a golang noob - learning it 3 weeks only)

package main

import (
    "encoding/json"
    "fmt"
    "io/ioutil"
    "log"
    "net/http"
    "sync"
    "time"
)

type user struct {
    User userData `json:"user"`
}

type userData struct {
    Followers count `json:"followed_by"`
}

type count struct {
    Count int `json:"count"`
}

func getFollowerCount(in <-chan string) <-chan int {
    out := make(chan int)
    go func() {
        for un := range in {
            URL := "https://www.instagram.com/" + un + "/?__a=1"
            resp, err := http.Get(URL)
            if err != nil {
                // handle error
                fmt.Println(err)
            }
            defer resp.Body.Close()
            body, err := ioutil.ReadAll(resp.Body)
            var u user
            err = json.Unmarshal(body, &u)
            if err != nil {
                fmt.Println(err)
            }
            // return u.User.Followers.Count
            out <- u.User.Followers.Count
        }
        close(out)
    }()
    return out
}

func merge(cs ...<-chan int) <-chan int {
    var wg sync.WaitGroup
    out := make(chan int)
    output := func(c <-chan int) {
        for n := range c {
            out <- n
        }
        wg.Done()
    }

    wg.Add(len(cs))
    for _, c := range cs {
        go output(c)
    }
    go func() {
        wg.Wait()
        close(out)
    }()
    return out
}

func gen(users ...string) <-chan string {
    out := make(chan string)
    go func() {
        for _, u := range users {
            out <- u
        }
        close(out)
    }()
    return out
}

func main() {
    start := time.Now()
    fmt.Println("STARTING UP")
    usrs := []string{"kanywest", "kimkardashian", "groovyq", "kendricklamar", "barackobama", "asaprocky", "champagnepapi", "eminem", "drdre", "g_eazy", "skrillex"}
    in := gen(usrs...)
    d1 := getFollowerCount(in)
    d2 := getFollowerCount(in)
    d3 := getFollowerCount(in)
    d4 := getFollowerCount(in)
    d5 := getFollowerCount(in)
    d6 := getFollowerCount(in)
    d7 := getFollowerCount(in)
    d8 := getFollowerCount(in)
    d9 := getFollowerCount(in)
    d10 := getFollowerCount(in)

    for d := range merge(d1, d2, d3, d4, d5, d6, d7, d8, d9, d10) {
        fmt.Println(d)
    }

    elapsed := time.Since(start)
    log.Println("runtime", elapsed)
}

Answer 1

Welcome to Go, happy learning.

You're doing good, you can improve your program many ways (such as json decoder, less no of chan, etc). Following is one of the approach. Execution time is between 352-446ms (take it with grain of salt, since network call is involved in your code. Might vary based on server response time).

Your updated code:

package main

import (
    "encoding/json"
    "fmt"
    "log"
    "net/http"
    "sync"
    "time"
)

type user struct {
    User userData `json:"user"`
}

type userData struct {
    Followers count `json:"followed_by"`
}

type count struct {
    Count int `json:"count"`
}

func getFollowerCount(username string, result chan<- int, wg *sync.WaitGroup) {
    defer wg.Done()
    reqURL := "https://www.instagram.com/" + username + "/?__a=1"
    resp, err := http.Get(reqURL)
    if err != nil {
        log.Println(err)
        return
    }
    defer resp.Body.Close()

    var u user
    if err := json.NewDecoder(resp.Body).Decode(&u); err != nil {
        log.Println(err)
        return
    }
    result <- u.User.Followers.Count
}

func execute(users []string, result chan<- int) {
    wg := &sync.WaitGroup{}
    for _, username := range users {
        wg.Add(1)
        go getFollowerCount(username, result, wg)
    }
    wg.Wait()
    result <- -1
}

func main() {
    start := time.Now()
    fmt.Println("STARTING UP")
    usrs := []string{"kanywest", "kimkardashian", "groovyq", "kendricklamar", "barackobama", "asaprocky", "champagnepapi", "eminem", "drdre", "g_eazy", "skrillex"}

    result := make(chan int)
    go execute(usrs, result)

    for v := range result {
        if v == -1 {
            break
        }
        fmt.Println(v)
    }

    elapsed := time.Since(start)
    fmt.Println("runtime:", elapsed)
}