I had designed and implemented the file rotation of file in golang.
As per design I am rotating the file based on filesize >= FileSizeThreshold(50000bytes) or file duration >= FileDurationThreshold(1 minute) (whichever is first).
Following is the implementation in golang.
package main
import (
"os"
"path/filepath"
"time"
"log"
"strings"
"flag"
"os/exec"
)
type FileStruct struct{
Filename string
CreatedAt time.Time
}
type FileRotate struct {
Dir string
File chan FileStruct
}
const(
MAX_FILE_SIZE = 50000
MAX_FILE_DURATION = time.Minute * 1
filename_time_format = "20060102150405000"
MAX_TRY = 5
)
var blockingChan chan int
func main(){
path := flag.String("dir", "", "absolute path of dir ")
flag.Parse()
if strings.Contains(*path, "./") {
log.Fatalln("ERROR: please give absolute path")
}
if info, err := os.Stat(*path); err == nil{
if ! info.IsDir(){
log.Fatalln(*path," is not a directory")
}
log.Println("directory found..")
} else {
if os.IsNotExist(err){
log.Println("directory not found..")
log.Println("creating the directory..",*path)
if err := exec.Command("mkdir","-p",*path).Run(); err != nil{
log.Fatalln("failed to create the directory ERROR:",err)
}
log.Println("directory created successfully")
}
}
filerotate := &FileRotate{*path,make(chan FileStruct,1)}
go filerotate.FileOperationsRoutine()
log.Println("generating file name struct..")
filerotate.File <- GetFileStruct()
<- blockingChan
}
func (rotate *FileRotate) FileOperationsRoutine(){
try := 0
var f *os.File
for{
if file, ok := <- rotate.File; ok{
if f == nil {
log.Println("WARN: file ptr is nil")
}
filePath := filepath.Join(rotate.Dir, file.Filename)
fileInfo, err := os.Stat(filePath)
if err != nil && os.IsNotExist(err) {
log.Println("file:", filePath, " does not exist...creating file")
_, err = os.Create(filePath)
if err != nil {
log.Println("failed to create the file ERROR:",err)
try++
if try == MAX_TRY {
log.Println("tried creating the file ",MAX_TRY," times. No luck")
time.Sleep(time.Second * 3)
continue
}
rotate.File <- file
continue
}
log.Println("file:", filePath, " created successfully")
fileInfo,err = os.Stat(filePath)
}
sizeCheck := fileInfo.Size() >= MAX_FILE_SIZE
durationCheck := time.Now().After(file.CreatedAt.Add(MAX_FILE_DURATION))
if sizeCheck || durationCheck {
log.Println("filesize of ",filePath," is ",fileInfo.Size(),"..filesizeCheck=",sizeCheck)
log.Println("fileDurationCheck=",durationCheck)
log.Println("rotating the file..")
f.Close()
f = nil
go ZipAndSendRoutine(filePath)
rotate.File <- GetFileStruct()
}else{
if f == nil {
f, err = os.OpenFile(filePath, os.O_RDWR | os.O_APPEND, 0644)
if err != nil {
log.Println("failed to open the file ERROR:", err)
try++
if try == MAX_TRY {
log.Println("tried opening the file ", MAX_TRY, " times. No luck")
time.Sleep(time.Second * 3)
continue
}
rotate.File <- file
continue
}
log.Println("file opened in append mode")
}
rotate.File <- file
}
}
}
}
func GetFileStruct() FileStruct{
current_time := time.Now()
log.Println("returning the filestruct..")
return FileStruct{"example_" + current_time.Format(filename_time_format),current_time}
}
func ZipAndSendRoutine(file string){
log.Println("zipping and sending the file:",file,"to remote server")
}
Execution log :
root@workstation:/media/sf_golang# ./bin/file_rotation -dir "/tmp/file_rotaion"
2017/01/16 15:05:03 directory found..
2017/01/16 15:05:03 starting file operations routine...
2017/01/16 15:05:03 generating file name struct..
2017/01/16 15:05:03 returning the filestruct..
2017/01/16 15:05:03 WARN: file ptr is nil
2017/01/16 15:05:03 file: /tmp/file_rotaion/example_20170116150503000 does not exist...creating file
2017/01/16 15:05:03 file: /tmp/file_rotaion/example_20170116150503000 created successfully
2017/01/16 15:05:03 file opened in append mode
2017/01/16 15:06:03 filesize of /tmp/file_rotaion/example_20170116150503000 is 0 ..filesizeCheck= false ...fileDurationCheck= true
2017/01/16 15:06:03 rotating the file..
2017/01/16 15:06:03 returning the filestruct..
2017/01/16 15:06:03 WARN: file ptr is nil
2017/01/16 15:06:03 file: /tmp/file_rotaion/example_20170116150603000 does not exist...creating file
2017/01/16 15:06:03 file: /tmp/file_rotaion/example_20170116150603000 created successfully
2017/01/16 15:06:03 file opened in append mode
2017/01/16 15:06:03 zipping and sending the file: /tmp/file_rotaion/example_20170116150503000 to remote server
2017/01/16 15:07:03 filesize of /tmp/file_rotaion/example_20170116150603000 is 0 ..filesizeCheck= false ...fileDurationCheck= true
2017/01/16 15:07:03 rotating the file..
2017/01/16 15:07:03 returning the filestruct..
2017/01/16 15:07:03 WARN: file ptr is nil
2017/01/16 15:07:03 file: /tmp/file_rotaion/example_20170116150703000 does not exist...creating file
2017/01/16 15:07:03 file: /tmp/file_rotaion/example_20170116150703000 created successfully
2017/01/16 15:07:03 file opened in append mode
2017/01/16 15:07:03 zipping and sending the file: /tmp/file_rotaion/example_20170116150603000 to remote server
As seen from the logs, the utility is working as expected. But during the execution of this utility, CPU usage was almost 100%
After stopping the utility..
I have identified the cause of this:
FileOperations goroutine is running indefinitely and within this routine I am sending the file pointer on rotate.File channel
I am stuck at this point and not sure how to optimize this further. Could anyone tell me how should I optimize CPU utilization for this utility?
