I have a memory problem when I try to send a large array with Echo (and Gin too). After the request, memory is not free.
package main
import (
"net/http"
"strconv"
"github.com/labstack/echo"
)
type User struct {
Username string
Password string
Lastname string
Firstname string
}
func main() {
e := echo.New()
e.GET("/", func(c echo.Context) error {
var user User
users := make([]User, 0)
for i := 0; i < 100000; i++ {
user = User{
Username: "ffgfgfghhfghfhgfgfhgfghfghfhgfhgfh" + strconv.Itoa(i),
Password: "gjgjghjgjhgjhghjfrserhkhjhklljjkbhjvftxersgdghjjkhkljkbhftd",
Lastname: "njuftydfhgjkjlkjlkjlkhjkhu",
Firstname: "jkggkjkl,,lm,kljkvgf"}
users = append(users, user)
}
defer func() {
users = nil
}()
return c.JSON(http.StatusOK, users)
})
e.Logger.Fatal(e.Start(":1323"))
}
To test, I run request in parallel and I have these results :
- 1 request : 300Mo
- 5 requests : 1.5Go
- 10 requests : 3.1Go
- more : my PC freeze :)
How can I reduce memory consumption?
EDIT
It works well if I don't have to process the data.
If, for example, I get 100,000 lines from the database.
And then I need to process them to return a JSON with several levels. In this case, I am obliged to create an array or map.
But the problem is that memory is never released. And it increases with each request and it is even worse in the case of parallel requests.
Here an example:
import (
"strconv"
"time"
)
type sqlDataType struct {
ApplicationID int
ApplicationName string
ApplicationCreatedAt time.Time
ApplicationUpdatedAt time.Time
ModuleID int
ModuleName string
ModuleCreatedAt time.Time
ModuleUpdatedAt time.Time
ActionID int
ActionName string
ActionCreatedAt time.Time
ActionUpdatedAt time.Time
}
type DataApplicationType struct {
Name string
CreatedAt time.Time
UpdatedAt time.Time
Modules map[int]dataModuleType
}
type dataModuleType struct {
Name string
CreatedAt time.Time
UpdatedAt time.Time
Actions map[int]dataActionType
}
type dataActionType struct {
Name string
CreatedAt time.Time
UpdatedAt time.Time
}
// InitData inits data for test
func InitData() map[int]DataApplicationType {
data := make(map[int]DataApplicationType)
const nbApplications = 10
const nbModules = 1000
const nbActions = 100000
sqlData := make([]sqlDataType, 0)
for i := 0; i < nbActions; i++ {
line := sqlDataType{
ApplicationID: (i % nbApplications) + 1,
ApplicationName: "Application " + strconv.Itoa((i%nbApplications)+1),
ApplicationCreatedAt: time.Now(),
ApplicationUpdatedAt: time.Now(),
ModuleID: (i % nbModules) + 1,
ModuleName: "Module " + strconv.Itoa((i%nbModules)+1),
ModuleCreatedAt: time.Now(),
ModuleUpdatedAt: time.Now(),
ActionID: i + 1,
ActionName: "Action " + strconv.Itoa(i+1),
ActionCreatedAt: time.Now(),
ActionUpdatedAt: time.Now(),
}
sqlData = append(sqlData, line)
}
nbData := len(sqlData)
for i := 0; i < nbData; i++ {
if _, ok := data[sqlData[i].ApplicationID]; !ok {
dac := new(dataActionType)
dac.Name = sqlData[i].ActionName
dac.CreatedAt = sqlData[i].ActionCreatedAt
dac.UpdatedAt = sqlData[i].ActionUpdatedAt
dmo := new(dataModuleType)
dmo.Name = sqlData[i].ModuleName
dmo.CreatedAt = sqlData[i].ModuleCreatedAt
dmo.UpdatedAt = sqlData[i].ModuleUpdatedAt
dmo.Actions = make(map[int]dataActionType)
dmo.Actions[sqlData[i].ActionID] = *dac
dap := new(DataApplicationType)
dap.Name = sqlData[i].ApplicationName
dap.CreatedAt = sqlData[i].ApplicationCreatedAt
dap.UpdatedAt = sqlData[i].ApplicationUpdatedAt
dap.Modules = make(map[int]dataModuleType)
dap.Modules[sqlData[i].ModuleID] = *dmo
data[sqlData[i].ApplicationID] = *dap
}
if _, ok := data[sqlData[i].ApplicationID].Modules[sqlData[i].ModuleID]; !ok {
dac := new(dataActionType)
dac.Name = sqlData[i].ActionName
dac.CreatedAt = sqlData[i].ActionCreatedAt
dac.UpdatedAt = sqlData[i].ActionUpdatedAt
dmo := new(dataModuleType)
dmo.Name = sqlData[i].ModuleName
dmo.CreatedAt = sqlData[i].ModuleCreatedAt
dmo.UpdatedAt = sqlData[i].ModuleUpdatedAt
dmo.Actions = make(map[int]dataActionType)
dmo.Actions[sqlData[i].ActionID] = *dac
data[sqlData[i].ApplicationID].Modules[sqlData[i].ModuleID] = *dmo
}
if _, ok := data[sqlData[i].ApplicationID].Modules[sqlData[i].ModuleID].Actions[sqlData[i].ActionID]; !ok {
dac := new(dataActionType)
dac.Name = sqlData[i].ActionName
dac.CreatedAt = sqlData[i].ActionCreatedAt
dac.UpdatedAt = sqlData[i].ActionUpdatedAt
data[sqlData[i].ApplicationID].Modules[sqlData[i].ModuleID].Actions[sqlData[i].ActionID] = *dac
}
}
return data
}
In the main.go:
func main() {
// Lancement de Cobra
// commands.Execute()
go issues.InitData()
go issues.InitData()
go issues.InitData()
go issues.InitData()
go issues.InitData()
time.Sleep(60 * time.Second)
}
This script needs around 500 Mo of memory and does not release it whereas the map hasn't even been transformed into JSON.
How can I reduce memory consumption and/or I can have a stable memory consumption state with many calls?
Thanks for you help