I'm trying to read all the files (+10.000) in a directory, but when i've processed about 1400 files I get a 'too many open files' error. I've added an explicit call to the garbage collector, but that doesn't seem to do much for my problem. I checked the source for ioutil package and ReadFile uses defer file.Close()
internally (as expected). So what is going wrong here?
const TEMPLATE = `{ "source_db" : "CDARCHIEF", "doc_type" : "%s", "referentie" : "%s", "bestandsnaam" : "%s", "tekst" : "%s" }`
const MAPPING = `{ ... }`
var DIR, DOCTYPE, URL string
func init() {
flag.StringVar(&DIR, "d", "./", "de directory met de ge-ocrde bestanden")
flag.StringVar(&DOCTYPE, "t", "AG", "document type [ AG, CO, NN ]")
flag.StringVar(&URL, "url", "...", "url voor de juiste index")
}
func main() {
flag.Parse()
fmt.Println("CD Archive Importer")
importDocuments()
}
func importDocuments() {
logfile, _ := os.Create("./importer.log")
defer logfile.Close()
files, _ := ioutil.ReadDir(DIR)
error_counter := 0
for i, file := range files {
if math.Mod(float64(i), 400.0) == 0.0 {
runtime.GC()
fmt.Println("Running garbage collector")
}
fmt.Printf("Importing ( %d / %d ) [ errors: %d ]", i+1, len(files), error_counter)
contents, err := ioutil.ReadFile(DIR + "/" + file.Name())
if err != nil {
error_counter = error_counter + 1
logfile.WriteString(fmt.Sprintf("[ERROR/IO] : %s | %s
", file.Name(), err))
continue
}
contents_string := strings.Replace(string(contents), "
", " ", -1)
contents_string = strings.Replace(contents_string, "\"", " ", -1)
contents_string = strings.Replace(contents_string, "\\", " ", -1)
referentie := strings.Trim(file.Name(), ".txt")
message := strings.NewReader(fmt.Sprintf(TEMPLATE, DOCTYPE, referentie, file.Name(), contents_string))
resp, error := http.Post(URL, "application/json", message)
if error != nil {
error_counter = error_counter + 1
logfile.WriteString(fmt.Sprintf("[ERROR/NET] : %s | %s | %s
", file.Name(), resp.Status, error))
continue
}
defer resp.Body.Close()
if resp.StatusCode != 201 {
body, _ := ioutil.ReadAll(resp.Body)
error_counter = error_counter + 1
logfile.WriteString(fmt.Sprintf("[ERROR/ES] : %s | %s | %s
", file.Name(), resp.Status, string(body)))
}
}
fmt.Println("
Done!")
}
I know that there is a somewhat similar question from about 2 years ago, but that didn't have a useful answer for my issue.