切片并行

Question

I want to make it run parallel based on number of thread. But the result was not as i expected. I dont know how to make it efficient and fast.

I ended up with this code.

package main

import (
    "fmt"
    "io/ioutil"
    "net/http"
    "os"
    "runtime"
    "strconv"
    "strings"
    "sync"
    "time"
)

func main() {
    start := time.Now()
    target := os.Args[1]
    thread, _ := strconv.Atoi(os.Args[3])
    file, err := ioutil.ReadFile(os.Args[2])
    if err != nil {
        fmt.Println("Error: Please double check if the file " + os.Args[2] + " is exist!")
        os.Exit(0)
    }
    wordlist := strings.Split(string(file), "
")

    var wg sync.WaitGroup
    runtime.GOMAXPROCS(runtime.NumCPU())
    jobs := make(chan string)
    for i := 0; i < thread; i++ {
        wg.Add(1)
        defer wg.Done()
        for _, word := range wordlist {
            go func(word string) {
                jobs <- word
            }(word)
        }
    }

    go func() {
        for job := range jobs {
            code := visit(target + job)
            fmt.Println(target + job + " - " + strconv.Itoa(code))
        }
    }()
    wg.Wait()

    elapsed := time.Since(start)
    fmt.Printf("Timer: %s
", elapsed)
}

func visit(url string) int {
    data, err := http.Get(url)
    if err != nil {
        panic(err)
    }
    return data.StatusCode
}

Any help would be appreciated. Thank you.

Update This is my current results :

$ go run test.go http://localhost/ word.txt 2
http://localhost/1 - 404
http://localhost/1 - 404
http://localhost/7 - 404
http://localhost/8 - 404
http://localhost/9 - 404
http://localhost/0 - 404
http://localhost/ - 200
http://localhost/3 - 404
http://localhost/2 - 404
http://localhost/4 - 404
http://localhost/6 - 404
http://localhost/2 - 404
http://localhost/3 - 404
http://localhost/4 - 404
http://localhost/5 - 404
http://localhost/9 - 404
http://localhost/7 - 404
http://localhost/8 - 404
http://localhost/0 - 404
http://localhost/5 - 404
http://localhost/ - 200
http://localhost/6 - 404

Answer 1

You are not using the waitgroup correctly. The defer's in main's for loop are never called since main never returns and as a result, wg.Wait() call never unblocks.

Put defer calls in the goroutine sending the message:

// ...
for i := 0; i < thread; i++ {
    wg.Add(1)

    for _, word := range wordlist {
        go func(word string) {
            defer wg.Done()
            jobs <- word
        }(word)
    }
}
// ...

Also close the jobs channel:

// ...
wg.Wait()
close(jobs)
// ...