douyi3676
douyi3676
2016-03-03 09:36

Go Lang HTTP处理程序中的内存泄漏

已采纳

I have the following code of http handler which on subsequent request download the original image from Amazon S3 and convert it in the desired aspect ratio and save it back on s3. This code leaks memory and after sometime it crashes.I have handled everything from my side and also did the profiling on the code. But, still couldn't figure out the issue. Would appreciate if anyone can figure out here. FYI, I am using the go version go1.5.3 linux/amd64 version .

Output from profiling :

3259.27kB of 3302.42kB total (98.69%)
Dropped 258 nodes (cum <= 16.51kB)
Showing top 30 nodes out of 91 (cum >= 27.76kB)
      flat  flat%   sum%        cum   cum%
 1552.59kB 47.01% 47.01%  1552.59kB 47.01%  bytes.makeSlice
     584kB 17.68% 64.70%      584kB 17.68%  imagick._Cfunc_GoBytes
  257.38kB  7.79% 72.49%   257.38kB  7.79%  encoding/pem.Decode
  168.11kB  5.09% 77.58%   168.11kB  5.09%  crypto/tls.(*block).reserve
  165.09kB  5.00% 82.58%   389.49kB 11.79%  crypto/x509.parseCertificate
  105.32kB  3.19% 85.77%   105.32kB  3.19%  reflect.unsafe_NewArray
   83.64kB  2.53% 88.30%    83.64kB  2.53%  math/big.nat.make
   75.55kB  2.29% 90.59%    75.55kB  2.29%  net/http.(*Transport).dialConn
   64.02kB  1.94% 92.53%    64.02kB  1.94%  regexp.(*bitState).reset
   43.77kB  1.33% 93.85%    43.77kB  1.33%  crypto/x509.(*CertPool).AddCert
   40.44kB  1.22% 95.08%    40.44kB  1.22%  crypto/x509/pkix.(*Name).FillFromRDNSequence
   40.16kB  1.22% 96.29%    40.16kB  1.22%  encoding/asn1.parsePrintableString
   24.07kB  0.73% 97.02%    24.07kB  0.73%  net/http.newBufioWriterSize
   18.98kB  0.57% 97.60%    18.98kB  0.57%  net/http.newBufioReader
   16.14kB  0.49% 98.09%    64.77kB  1.96%  crypto/tls.(*Conn).readHandshake
   12.01kB  0.36% 98.45%   237.09kB  7.18%  encoding/asn1.parseField
    8.01kB  0.24% 98.69%    91.65kB  2.78%  crypto/x509.parsePublicKey
         0     0% 98.69%   112.33kB  3.40%  bufio.(*Reader).Read
         0     0% 98.69%    80.32kB  2.43%  bufio.(*Reader).fill
         0     0% 98.69%    27.76kB  0.84%  bufio.(*Writer).ReadFrom
         0     0% 98.69%    27.76kB  0.84%  bufio.(*Writer).flush
         0     0% 98.69%  1648.33kB 49.91%  bytes.(*Buffer).ReadFrom
         0     0% 98.69%    16.59kB   0.5%  bytes.(*Buffer).Write
         0     0% 98.69%    16.59kB   0.5%  bytes.(*Buffer).grow
         0     0% 98.69%   843.06kB 25.53%  crypto/tls.(*Conn).Handshake
         0     0% 98.69%   112.33kB  3.40%  crypto/tls.(*Conn).Read
         0     0% 98.69%    27.76kB  0.84%  crypto/tls.(*Conn).Write
         0     0% 98.69%   843.06kB 25.53%  crypto/tls.(*Conn).clientHandshake
         0     0% 98.69%   160.96kB  4.87%  crypto/tls.(*Conn).readRecord
         0     0% 98.69%    27.76kB  0.84%  crypto/tls.(*Conn).writeRecord

Code :

func main() {       

    imagick.Initialize() 
    defer imagick.Terminate()   

        myMux := http.NewServeMux()
        myMux.HandleFunc("/", serveHTTP)       

      if err := http.ListenAndServe(":8082", myMux); err != nil {
        logFatal("Error when starting or running http server: %v", err)
    }      

}

func serveHTTP(w http.ResponseWriter, r *http.Request) {

        var isMaster bool = true        
    var desiredAspectRatio float64 = 1.77   

    if r.RequestURI == "/favicon.ico" {
        w.WriteHeader(http.StatusNotFound)
        return
    }

    if len(strings.TrimSpace(r.URL.Query().Get("ar"))) != 0 {
        desiredAspectRatio, _ = strconv.ParseFloat(r.URL.Query().Get("ar"), 64)
    }

    if len(strings.TrimSpace(r.URL.Query().Get("ism"))) != 0 {      
        isMaster, _ = strconv.ParseBool(r.URL.Query().Get("ism"))
    }   

    imageUrl := strings.ToLower(r.URL.Path[1:])     

    isProcessed := CreateMaster(imageUrl, desiredAspectRatio, isMaster)     

    if isProcessed == false {
        w.WriteHeader(http.StatusNotFound) 
        return
    }

    if !sendResponse(w, r, imageUrl) {
        // w.WriteHeader() is skipped intentionally here, since the response may be already partially created.
        return
    }


    logRequestMessage(r, "SUCCESS")     
}


func sendResponse(w http.ResponseWriter, r *http.Request, imageUrl string) bool {

    w.Header().Set("Content-Type", "text/plain")        

    if _, err := w.Write([]byte("master created")); err != nil {
        logRequestError(r, "Cannot send image from imageUrl=%v to client: %v", imageUrl, err)
        return false
    }
    return true
}


func CreateMaster(keyName string, desiredAspectRatio float64, isMaster bool) bool {             

    s3Client := s3.New(session.New(), &aws.Config{Region: aws.String(region)})
        params := &s3.GetObjectInput{
        Bucket: aws.String(bucketName),
        Key: aws.String(keyName),
        }

    fmt.Println(" Master creation request for key : " + keyName)
    out, err := s3Client.GetObject(params)

    if err != nil { 
        return false                                       
    }

    defer out.Body.Close()  
    img, err := ioutil.ReadAll(out.Body)

    if err != nil { 
            return false      
    }                   

    mw := imagick.NewMagickWand()
    defer mw.Destroy()

    err = mw.ReadImageBlob(img)
    if err != nil {  
        return false                   
    }


    if isMaster == false {
        paramsPut := &s3.PutObjectInput{
                    Bucket:         aws.String(masterBucketName),
                    Key:            aws.String(keyName),
                    Body:         bytes.NewReader(mw.GetImageBlob()),
            }

        _, err = s3Client.PutObject(paramsPut)
        if err != nil {
            log.Printf("Couldn't put the image on s3 : " + keyName + "%s
", err)       
        }

        return true
    }


        originalWidth := float64(mw.GetImageWidth())
        originalHeight := float64(mw.GetImageHeight())

    imageAspectRatio  := originalWidth / originalHeight
        masterWidth := cwMasterWidth
        masterHeight := cwMasterHeight
        masterAspectRatio := math.Trunc((cwMasterWidth / cwMasterHeight) * 100)/100

    if masterAspectRatio != desiredAspectRatio {           
                    masterAspectRatio = desiredAspectRatio
                }


    pwm := imagick.NewPixelWand()
    defer pwm.Destroy()

    tx := imagick.NewMagickWand()
    defer tx.Destroy()  

    if isMaster == true {               

            var w, h uint = 0, 0
            size := fmt.Sprintf("%dx%d^+0+0", w, h) 
                    if imageAspectRatio <= masterAspectRatio {                
                        // trim the height
            w = uint(originalWidth)
            h = (uint(originalWidth / masterAspectRatio))
               size = fmt.Sprintf("%dx%d^+0+0", w, h)
                     } else { 
                        // trim the width
            w = uint(originalHeight * masterAspectRatio)
            h = uint(originalHeight)
            size = fmt.Sprintf("%dx%d^+0+0", w, h)
                     }

            tx = mw.TransformImage("", size)        
            tx.SetImageGravity(imagick.GRAVITY_CENTER)
            offsetX := -(int(w) - int(tx.GetImageWidth())) / 2
            offsetY := -(int(h) - int(tx.GetImageHeight())) / 2
            err := tx.ExtentImage(w, h, offsetX, offsetY)

                    if float64(tx.GetImageWidth()) > masterWidth && float64(tx.GetImageHeight()) > masterHeight  {                                        
                        err = tx.ResizeImage(uint(masterWidth), uint(masterHeight), imagick.FILTER_BOX, 1)
            if err != nil {
                log.Printf("Inside CreateMaster function Couldn't resize the image : " + keyName + "%s
", err) 
                return false                 
            }                                                           
                    }                                       
                }       

     paramsPut := &s3.PutObjectInput{
                    Bucket:         aws.String(masterBucketName),
                    Key:            aws.String(keyName),
                    Body:         bytes.NewReader(tx.GetImageBlob()),
            }

    _, err = s3Client.PutObject(paramsPut)
    if err != nil {
        log.Printf("Inside CreateMaster function Couldn't put the image on s3 : " + keyName + "%s
", err)  
        return false        
    }

    return true
}
  • 点赞
  • 写回答
  • 关注问题
  • 收藏
  • 复制链接分享
  • 邀请回答

1条回答

  • drh47606 drh47606 5年前

    You are leaking a wand.

    Here, you allocate a new wand and defer it to destroy:

    tx := imagick.NewMagickWand()
    defer tx.Destroy()  
    

    But then further down, in an "if" block, you replace it with the wand that is returned from the call to TransformImage():

            tx = mw.TransformImage("", size)        
            tx.SetImageGravity(imagick.GRAVITY_CENTER)
    

    If you completely get rid of the first allocation of that new magick wand, and simply make sure to Destroy() that new wand returned from TransformImage(), the leak goes away.

    Ref the issue tracker, #72, for details

    点赞 评论 复制链接分享

为你推荐