My golang program (url monitor) has a memory leak, it finially killed by kernel (oom). the env:
$ go version
go version go1.0.3
$ go env
GOARCH="amd64"
GOBIN=""
GOCHAR="6"
GOEXE=""
GOGCCFLAGS="-g -O2 -fPIC -m64 -pthread"
GOHOSTARCH="amd64"
GOHOSTOS="linux"
GOOS="linux"
GOPATH="/data/apps/go"
GOROOT="/usr/local/go"
GOTOOLDIR="/usr/local/go/pkg/tool/linux_amd64"
CGO_ENABLED="1"
code:
package main
import (
"bytes"
"database/sql"
"flag"
"fmt"
_ "github.com/Go-SQL-Driver/MySQL"
"ijinshan.com/cfg"
"log"
"net"
"net/http"
"net/smtp"
"os"
"strconv"
"strings"
"sync"
"time"
)
var (
Log *log.Logger
Conf cfg.KVConfig
Debug bool
CpuCore int
HttpTransport = &http.Transport{
Dial: func(netw, addr string) (net.Conn, error) {
deadline := time.Now().Add(30 * time.Second)
c, err := net.DialTimeout(netw, addr, 20*time.Second)
if err != nil {
return nil, err
}
c.SetDeadline(deadline)
return c, nil
},
DisableKeepAlives: true,
}
HttpClient = &http.Client{
Transport: HttpTransport,
}
WG sync.WaitGroup
)
const (
LogFileFlag = os.O_WRONLY | os.O_CREATE | os.O_APPEND
LogFileMode = 0644
LogFlag = log.LstdFlags | log.Lshortfile
GET_VIDEO_SQL = `SELECT B.Name, A.TSID, A.Chapter, A.ChapterNum,
IFNULL(A.Website, ''), IFNULL(A.Descr, ''),
IFNULL(A.VideoId, ''), IFNULL(AndroidWebURL, ''), IFNULL(IOSWebURL, ''),
IFNULL(AndroidURL, ''), IFNULL(AndroidURL2, ''), IFNULL(IOSURL, '')
FROM Video A INNER JOIN TVS B ON A.TSID = B.ID LIMIT 200`
HtmlHead = `<table border=1 width=100% height=100%><tr><td>节目名
</td><td>tsid</td><td>章节</td><td>章节号</td><td>描述
</td><td>videoid</td><td>网站</td><td>地址</td></tr>`
HtmlTail = "</table>"
)
type videoInfo struct {
name string
tsid uint
chapter string
chapterNum uint
descr string
videoId string
website string
androidWebUrl string
iosWebUrl string
androidUrl string
androidUrl2 string
iosUrl string
}
func init() {
var (
confFile string
err error
)
// parse command argument:w
flag.StringVar(&confFile, "c", "./vsmonitor.conf", " set config file path")
flag.Parse()
// read config
if Conf, err = cfg.Read(confFile); err != nil {
panic(fmt.Sprintf("Read config file \"%s\" failed (%s)",
confFile, err.Error()))
}
// open log file
file, err := os.OpenFile(Conf["log.file"], LogFileFlag, LogFileMode)
if err != nil {
panic(fmt.Sprintf("OpenFile \"%s\" failed (%s)", Conf["log.file"],
err.Error()))
}
// init LOG
Log = log.New(file, "", LogFlag)
Debug = false
i, err := strconv.ParseInt(Conf["cpucore.num"], 10, 32)
if err != nil {
panic(fmt.Sprintf("ParseInt \"%s\" failed (%s)", Conf["cpucore.num"],
err.Error()))
}
CpuCore = int(i)
}
func getHttpStatusCode(url string) int {
if url == "" {
return 200
}
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return 0
}
req.Header.Add("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.57 Safari/537.17")
req.Header.Add("Connection", "close")
resp, err := HttpClient.Do(req)
if err != nil {
return 0
}
defer resp.Body.Close()
return resp.StatusCode
}
func sendMail(host, user, pwd, from, to, subject, body, mailType string) error {
auth := smtp.PlainAuth("", user, pwd, strings.Split(host, ":")[0])
cntType := fmt.Sprintf("Content-Type: text/%s;charset=UTF-8", mailType)
msg := fmt.Sprintf("To: %s
From: %s<%s>
Subject: %s
%s
%s",
to, from, user, subject, cntType, body)
return smtp.SendMail(host, auth, user, strings.Split(to, ","), []byte(msg))
}
func getVideos(videoChan chan *videoInfo, htmlBuf *bytes.Buffer) error {
defer HttpTransport.CloseIdleConnections()
db, err := sql.Open("mysql", Conf["weikan.mysql"])
if err != nil {
return err
}
rows, err := db.Query(GET_VIDEO_SQL)
if err != nil {
db.Close()
return err
}
for rows.Next() {
video := &videoInfo{}
err = rows.Scan(&video.name, &video.tsid, &video.chapter,
&video.chapterNum,
&video.website, &video.descr, &video.videoId, &video.androidWebUrl,
&video.iosWebUrl, &video.androidUrl, &video.androidUrl2,
&video.iosUrl)
if err != nil {
db.Close()
return err
}
videoChan <- video
WG.Add(1)
}
db.Close()
// wait check url finish
WG.Wait()
// send mail
for {
if htmlBuf.Len() == 0 {
Log.Print("no error found!!!!!!!!")
break
}
Log.Print("found error !!!!!!!!")
/*
err := sendMail("smtp.gmail.com:587", "xxxx",
"xxx", "xxx <xxx>",
Conf["mail.to"], "xxxxx",
HtmlHead+htmlBuf.String()+HtmlTail, "html")
if err != nil {
Log.Printf("sendMail failed (%s)", err.Error())
time.Sleep(10 * time.Second)
continue
}
*/
Log.Print("send mail")
break
}
Log.Print("reset buf")
htmlBuf.Reset()
return nil
}
func checkUrl(videoChan chan *videoInfo, errChan chan string) {
defer func() {
if err := recover(); err != nil {
Log.Print("rouintes failed : ", err)
}
}()
for {
video := <-videoChan
ok := true
errUrl := ""
if code := getHttpStatusCode(video.androidWebUrl); code >= 400 {
errUrl += fmt.Sprintf("%s (%d)<br />",
video.androidWebUrl, code)
ok = false
}
if code := getHttpStatusCode(video.iosWebUrl); code >= 400 {
errUrl += fmt.Sprintf("%s (%d)<br />",
video.iosWebUrl, code)
ok = false
}
if code := getHttpStatusCode(video.androidUrl); code >= 400 {
errUrl += fmt.Sprintf("%s (%d)<br />",
video.androidUrl, code)
ok = false
}
if code := getHttpStatusCode(video.androidUrl2); code >= 400 {
errUrl += fmt.Sprintf("%s (%d)<br />",
video.androidUrl2, code)
ok = false
}
if code := getHttpStatusCode(video.iosUrl); code >= 400 {
errUrl += fmt.Sprintf("%s (%d)<br />",
video.iosUrl, code)
ok = false
}
if !ok {
errChan <- fmt.Sprintf(`<tr><td>%s</td><td>%d</td><td>%s</td>
<td>%d</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>`,
video.name, video.tsid, video.chapter, video.chapterNum,
video.descr, video.videoId,
video.website, errUrl)
Log.Printf("\"%s\" (%s) —— \"%s\" checkurl err", video.name,
video.chapter, video.descr)
} else {
Log.Printf("\"%s\" (%s) —— \"%s\" checkurl ok", video.name,
video.chapter, video.descr)
WG.Done()
}
}
}
func mergeErr(errChan chan string, htmlBuf *bytes.Buffer) {
defer func() {
if err := recover(); err != nil {
Log.Print("rouintes failed : ", err)
}
}()
for {
html := <-errChan
_, err := htmlBuf.WriteString(html)
if err != nil {
Log.Printf("htmlBuf WriteString \"%s\" failed (%s)", html,
err.Error())
panic(err)
}
WG.Done()
}
}
func main() {
videoChan := make(chan *videoInfo, 100000)
errChan := make(chan string, 100000)
htmlBuf := &bytes.Buffer{}
defer func() {
if err := recover(); err != nil {
Log.Print("rouintes failed : ", err)
}
}()
// check url
for i := 0; i < CpuCore; i++ {
go checkUrl(videoChan, errChan)
}
// merge error string then send mail
go mergeErr(errChan, htmlBuf)
for {
// get Video and LiveSrc video source
if err := getVideos(videoChan, htmlBuf); err != nil {
Log.Printf("getVideos failed (%s)", err.Error())
time.Sleep(10 * time.Second)
continue
}
// time.Sleep(1 * time.Hour)
}
Log.Print("exit...")
}
the code has four funcs:
getHttpStatusCode
free resource use resp.Body.Close()
sendMail
I don't need to free the resource manually
mergeErr
concat the err string by using a htmlBuf(*bytes.Buffer)
getVideos
First it gets the Video urls and then sends them to videoChan then it waits all the routines finish their check jobs. then sendmail and reset htmlBuf.
I don't find any resource that needs free, but.
$ top
shows:
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
6451 root 20 0 3946m 115m 2808 S 0.7 0.2 6:11.20 vsmonitor
the VIRT and RES will grow ...
memory profiling:
(pprof) top
Total: 10.8 MB
2.3 21.2% 21.2% 2.3 21.2% main.main
2.0 18.5% 39.8% 2.0 18.5% bufio.NewWriterSize
1.5 13.9% 53.7% 1.5 13.9% bufio.NewReaderSize
1.5 13.9% 67.6% 1.5 13.9% compress/flate.NewReader
0.5 4.6% 72.2% 0.5 4.6% net.newFD
0.5 4.6% 76.8% 0.5 4.6% net.sockaddrToTCP
0.5 4.6% 81.5% 4.5 41.7% net/http.(*Transport).getConn
0.5 4.6% 86.1% 2.5 23.2% net/http.(*persistConn).readLoop
0.5 4.6% 90.7% 0.5 4.6% net/textproto.(*Reader).ReadMIMEHeader
0.5 4.6% 95.4% 0.5 4.6% net/url.(*URL).ResolveReference
