用指针序列化结构

Question

Having a struct heirarchy like:

type DomainStore struct {
    Domains []*Domain
    Users []*User
}

type Domain struct {
    Name    string
    Records []*Record
    Owner   *User
}

type User struct {
    Name      string
    Email     string
    Domains []*Domain
}

type Record struct {
    Name      string
    Host      string
}

With a single DomainStore having a list of Domain and Users with pointer between Domain and User.

I'm looking for a way to serialize/deserialize to/from file. I have been trying to use gob, but the pointers is not (by design) serialized correct (its flattened).

Thinking about giving each object a unique id and making a func to serialize/deserialize each type, but it seems much work/boilerplate. Any suggestions for a strategy?

I would like to keep the whole DomainStore in memory, and just serialize to file on user request.

The main problem: How to serialise/deserialize and keep the pointers pointing to the same object and not different copies of the same object

Both gob and json seems to "just" copy the value of the object and afted deserializasion I end up with multiple independent copies of objects.

Using gob ang json this is what happens:

Before, A & C both points to B:

A -> B <- C

After deserialization with json/gob:

A -> B1 , C -> B2

A & C points to to different object, with the same values. But, if i change B1 it's not changed in B2.

--- Update ---

When marshalling i can obtain the memory location of the object and use it as an ID:

func (u *User) MarshalJSON() ([]byte, error) {
    return json.Marshal(&JsonUser{
        ID:       fmt.Sprintf("%p", u),
        Name:     u.Name,
        Email:    u.Email,
    })
}

And when marshalling the Domain I can replace the

func (d *Domain) MarshalJSON() ([]byte, error) {
    return json.Marshal(&struct {
        ID       string `json:"id"`
        Name     string `json:"name"`
        User     string `json:"user"`
    }{
        ID:       fmt.Sprintf("%p", d),
        Name:     d.Name,
        User:     fmt.Sprintf("%p", d.User),
    })
}

Now I just need to be able to unmarshal this which gives me a problem in the UnmarshalJSON need to access a map of id's and their respective objects.

func (u *User) UnmarshalJSON(data []byte) error {
  // need acces to a map shared by all UnmarshalJSON functions
}

Answer 1

It can be done using the following method:

1. All the objects are placed in maps in a State object.
2. When the objects in a State object is marshalled, all objects refered to using pointers is replaced with the memory location of the object.
3. When unmarshalled pointers are restored using a global list of previously read objects.

The code will run, and is just to illustrate the method, I'm new to Go, so bear with me.

package main

import (
    "encoding/json"
    "errors"
    "fmt"
    "log"
    "strings"
)

type User struct {
    Name  string
    Email string
}
type JsonUser struct {
    ID    string `json:"id"`
    Name  string `json:"name"`
    Email string `json:"email"`
}

func (u *User) Print(level int) {
    ident := strings.Repeat("-", level)
    log.Println(ident, "Username:", u.Name, u.Email)
}
func (u *User) Id() string {
    return fmt.Sprintf("%p", u)
}
func (u *User) MarshalJSON() ([]byte, error) {
    return json.Marshal(&JsonUser{
        ID:    u.Id(),
        Name:  u.Name,
        Email: u.Email,
    })
}
func (u *User) UnmarshalJSON(data []byte) error {
    aux := &JsonUser{}
    if err := json.Unmarshal(data, &aux); err != nil {
        return err
    }
    u.Name = aux.Name
    u.Email = aux.Email
    load_helper[aux.ID] = u
    log.Println("Added user with id ", aux.ID, u.Name)
    return nil
}

type Record struct {
    Type     string // MX / A / CNAME / TXT / REDIR / SVR
    Name     string // @ / www
    Host     string // IP / address
    Priority int    // Used for MX
    Port     int    // Used for SVR
}
type JsonRecord struct {
    ID       string
    Type     string
    Name     string
    Host     string
    Priority int
    Port     int
}

func (r *Record) Print(level int) {
    ident := strings.Repeat("-", level)
    log.Println(ident, "", r.Type, r.Name, r.Host)
}
func (r *Record) Id() string {
    return fmt.Sprintf("%p", r)
}
func (r *Record) MarshalJSON() ([]byte, error) {
    return json.Marshal(&JsonRecord{
        ID:       r.Id(),
        Name:     r.Name,
        Type:     r.Type,
        Host:     r.Host,
        Priority: r.Priority,
        Port:     r.Port,
    })
}
func (r *Record) UnmarshalJSON(data []byte) error {
    aux := &JsonRecord{}
    if err := json.Unmarshal(data, &aux); err != nil {
        return err
    }
    r.Name = aux.Name
    r.Type = aux.Type
    r.Host = aux.Host
    r.Priority = aux.Priority
    r.Port = aux.Port
    load_helper[aux.ID] = r
    log.Println("Added record with id ", aux.ID, r.Name)
    return nil
}

type Domain struct {
    Name    string
    User    *User     // User ID
    Records []*Record // Record ID's
}
type JsonDomain struct {
    ID      string   `json:"id"`
    Name    string   `json:"name"`
    User    string   `json:"user"`
    Records []string `json:"records"`
}

func (d *Domain) Print(level int) {
    ident := strings.Repeat("-", level)
    log.Println(ident, "Domain:", d.Name)
    d.User.Print(level + 1)
    log.Println(ident, " Records:")
    for _, r := range d.Records {
        r.Print(level + 2)
    }
}
func (d *Domain) Id() string {
    return fmt.Sprintf("%p", d)
}
func (d *Domain) MarshalJSON() ([]byte, error) {
    var record_ids []string
    for _, r := range d.Records {
        record_ids = append(record_ids, r.Id())
    }
    return json.Marshal(JsonDomain{
        ID:      d.Id(),
        Name:    d.Name,
        User:    d.User.Id(),
        Records: record_ids,
    })
}
func (d *Domain) UnmarshalJSON(data []byte) error {
    log.Println("UnmarshalJSON domain")
    aux := &JsonDomain{}
    if err := json.Unmarshal(data, &aux); err != nil {
        return err
    }
    d.Name = aux.Name
    d.User = load_helper[aux.User].(*User) // restore pointer to domains user
    for _, record_id := range aux.Records {
        d.Records = append(d.Records, load_helper[record_id].(*Record))
    }
    return nil
}

type State struct {
    Users   map[string]*User
    Records map[string]*Record
    Domains map[string]*Domain
}

func NewState() *State {
    s := &State{}
    s.Users = make(map[string]*User)
    s.Domains = make(map[string]*Domain)
    s.Records = make(map[string]*Record)
    return s
}
func (s *State) Print() {
    log.Println("State:")
    log.Println("Users:")
    for _, u := range s.Users {
        u.Print(1)
    }
    log.Println("Domains:")
    for _, d := range s.Domains {
        d.Print(1)
    }
}
func (s *State) NewUser(name string, email string) *User {
    u := &User{Name: name, Email: email}
    id := fmt.Sprintf("%p", u)
    s.Users[id] = u
    return u
}
func (s *State) NewDomain(user *User, name string) *Domain {
    d := &Domain{Name: name, User: user}
    s.Domains[d.Id()] = d
    return d
}
func (s *State) NewMxRecord(d *Domain, rtype string, name string, host string, priority int) *Record {
    r := &Record{Type: rtype, Name: name, Host: host, Priority: priority}
    d.Records = append(d.Records, r)
    s.Records[r.Id()] = r
    return r
}
func (s *State) FindDomain(name string) (*Domain, error) {
    for _, v := range s.Domains {
        if v.Name == name {
            return v, nil
        }
    }
    return nil, errors.New("Not found")
}
func Save(s *State) (string, error) {
    b, err := json.MarshalIndent(s, "", "    ")
    if err == nil {
        return string(b), nil
    } else {
        log.Println(err)
        return "", err
    }
}

var load_helper map[string]interface{}

func Load(s *State, blob string) {
    load_helper = make(map[string]interface{})
    if err := json.Unmarshal([]byte(blob), s); err != nil {
        log.Println(err)
    } else {
        log.Println("OK")
    }
}

func test_state() {

    s := NewState()
    u := s.NewUser("Ownername", "[e-mail]")
    d := s.NewDomain(u, "somedomain.com")
    s.NewMxRecord(d, "MX", "@", "192.168.1.1", 10)
    s.NewMxRecord(d, "A", "www", "192.168.1.1", 0)

    s.Print()

    x, _ := Save(s) // Saved to json string

    log.Println("State saved, the json string is:")
    log.Println(x)

    s2 := NewState() // Create a new empty State
    Load(s2, x)
    s2.Print()

    d, err := s2.FindDomain("somedomain.com")
    if err == nil {
        d.User.Name = "Changed"
    } else {
        log.Println("Error:", err)
    }
    s2.Print()
}

func main() {
    test_state()
}

This is quite a lot of code and there are to much coupling between the objects and the serialization. Also the global var load_helper is bad. Ideas to improve will be appreciated.

Another approch would be to use reflection to make a more generic solution. Here is an example using this method:

package main

import (
    "encoding/json"
    "fmt"
    "log"
    "strings"
    "reflect"
)

func pprint(x interface{}) {
    b, err := json.MarshalIndent(x, "", "  ")
    if err != nil {
        fmt.Println("error:", err)
    }
    fmt.Println(string(b))  
}


var typeRegistry = make(map[string]reflect.Type)

// Register a type to make it possible for the Save/Load functions
// to serialize it.
func Register(v interface{}) {
    t := reflect.TypeOf(v)
    n := t.Name()
    fmt.Println("Register type",n)
    typeRegistry[n] = reflect.TypeOf(v)
}

// Make an instance of a type from the string name of the type.
func makeInstance(name string) reflect.Value {
    v := reflect.New(typeRegistry[name]).Elem()
    return v
}

// Translate a string type name tpo a real type.
func getTypeFromString(name string) reflect.Type {
    return typeRegistry[name]
}


// Serializeable interface must be supported by all objects passed to the Load / Save functions.
type Serializeable interface {
    Id() string
}

// GenericSave saves the object d
func GenericSave(d interface{}) (string, error) {
    r := make(map[string]interface{})
    v := reflect.ValueOf(d)
    t := reflect.TypeOf(d)
    if t.Kind()==reflect.Ptr {
        t=t.Elem()
        v=v.Elem()
    }
    r["_TYPE"]=t.Name()
    r["_ID"]=fmt.Sprintf("%p", d)
    for i := 0; i < t.NumField(); i++ {
        f := t.Field(i)
        name := f.Name
        vf := v.FieldByName(name)
//      fmt.Println("Field", i+1, "name is", name, "type is", f.Type.Name(), "and kind is", f.Type.Kind())      
//      fmt.Println("V:", vf)
        if f.Tag != "" {
            store:=strings.Split(f.Tag.Get("store"),",")
            switch store[1] {
            case "v":
                switch t.Field(i).Type.Name() {
                case "string":
                    r[store[0]]=vf.String()
                case "int":
                    r[store[0]]=vf.Int()
                }
            case "p":
                vals:=vf.MethodByName("Id").Call([]reflect.Value{})
                r[store[0]]=vals[0].String()
            case "lp":
                tr:=[]string{}
                for j := 0; j < vf.Len(); j++ {
                    vals:=vf.Index(j).MethodByName("Id").Call([]reflect.Value{})
                    tr=append(tr,vals[0].String())
                }
                r[store[0]]=tr
            }
        }
    }   
    m,_:=json.Marshal(r)
    return string(m),nil
}

// Save saves the list of objects.
func Save(objects []Serializeable) []byte {
    lst:=[]string{}
    for _,o := range(objects) {
        os,_:= GenericSave(o) // o.Save()
        lst=append(lst,os)
    }
    m,_:=json.Marshal(lst)
    return m
}


func toStructPtr(obj interface{}) interface{} {
    vp := reflect.New(reflect.TypeOf(obj))
    vp.Elem().Set(reflect.ValueOf(obj))
    return vp.Interface()
}

// Load creates a list of serializeable objects from json blob
func Load(blob []byte) []Serializeable {
    objects := []Serializeable{}
    loadHelper := make(map[string]interface{})
    var olist []interface{}
    if err := json.Unmarshal(blob, &olist); err != nil {
        log.Println(err)
    } else {
        for _,o := range(olist) {

            var omap map[string]interface{}
            json.Unmarshal([]byte(o.(string)), &omap)

            t:= getTypeFromString(omap["_TYPE"].(string))
            obj := reflect.New(t).Elem() 

            for i := 0; i < t.NumField(); i++ {
//              n:=t.Field(i).Name
//              fmt.Println(i,n,t.Field(i).Type.Name())

                if t.Field(i).Tag != "" {
                    store:=strings.Split(t.Field(i).Tag.Get("store"),",")
//                  fmt.Println(store)
                    switch store[1] {
                    case "v":
                        switch t.Field(i).Type.Name() {
                        case "string":
                            obj.FieldByIndex([]int{i}).SetString(omap[store[0]].(string))
                        case "int":
                            obj.FieldByIndex([]int{i}).SetInt(int64(omap[store[0]].(float64)))
                        }
                    case "p":
                        nObj:=loadHelper[omap[store[0]].(string)]
                        obj.FieldByIndex([]int{i}).Set(reflect.ValueOf(nObj.(*User)))
                    case "lp":
                        ptrItemType:=t.Field(i).Type.Elem()
                        slice := reflect.Zero(reflect.SliceOf(  ptrItemType /* reflect.TypeOf( &Record{} ) */  ))//.Interface()
                        for _, pID := range(omap[store[0]].([]interface{})) {
                            nObj:=loadHelper[pID.(string)]
                            slice=reflect.Append(slice,  reflect.ValueOf(nObj)  )
                        }
                        obj.FieldByIndex([]int{i}).Set(slice)                       
                    }
                }
            }
            oi:=toStructPtr(obj.Interface())
            oip:=oi.(Serializeable)
            objects=append(objects,oip)
            loadHelper[omap["_ID"].(string)]=oip
        }
    }
    return objects

}



/* Application data structures */

type User struct {
    Name  string `store:"name,v"`
    Email string `store:"email,v"`
}
func (u *User) Id() string {
    return fmt.Sprintf("%p", u)
}
func (u *User) Save() (string, error) {
    return GenericSave(u)
}
func (u *User) Print() {
    fmt.Println("User:",u.Name)
}


type Record struct {
    Type     string `store:"type,v"`// MX / A / CNAME / TXT / REDIR / SVR
    Name     string `store:"name,v"`// @ / www
    Host     string `store:"host,v"`// IP / address
    Priority int    `store:"priority,v"`// Used for MX
    Port     int    `store:"port,v"`// Used for SVR
}
func (r *Record) Id() string {
    return fmt.Sprintf("%p", r)
}
func (r *Record) Save() (string, error) {
    return GenericSave(r)
}
func (r *Record) Print() {
    fmt.Println("Record:",r.Type,r.Name,r.Host)
}


type Domain struct {
    Name    string    `store:"name,v"`
    User    *User     `store:"user,p"`    // User ID
    Records []*Record `store:"record,lp"` // Record ID's
}
func (d *Domain) Id() string {
    return fmt.Sprintf("%p", d)
}
func (d *Domain) Save() (string, error) {
    return GenericSave(d)
}
func (d *Domain) Print() {
    fmt.Println("Domain:",d.Name)
    d.User.Print()
    fmt.Println("Records:")
    for _, r := range d.Records {
        r.Print()
    }
}


type DBM struct {
    Domains []*Domain
    Users []*User
    Records []*Record
}
func (dbm *DBM) AddDomain(d *Domain) {
    dbm.Domains=append(dbm.Domains,d)
}
func (dbm *DBM) AddUser(u *User) {
    dbm.Users=append(dbm.Users,u)
}
func (dbm *DBM) AddRecord(r *Record) {
    dbm.Records=append(dbm.Records,r)
}
func (dbm *DBM) GetObjects() []Serializeable {
    objects:=[]Serializeable{}
    for _,r := range(dbm.Records) {
        objects=append(objects, r)
    }
    for _,u := range(dbm.Users) {
        objects=append(objects, u)
    }
    for _,d := range(dbm.Domains) {
        objects=append(objects, d)
    }
    return objects
}
func (dbm *DBM) SetObjects(objects []Serializeable) {
    for _,o := range(objects) {
        switch o.(type) {
        case *Record:
            fmt.Println("record")
            dbm.AddRecord(o.(*Record))
        case *User:
            fmt.Println("record")
            dbm.AddUser(o.(*User))
        case *Domain:
            fmt.Println("record")
            dbm.AddDomain(o.(*Domain))
        }
    }
}


func testState() {

    Register(User{})
    Register(Domain{})
    Register(Record{})

    dbm:=DBM{}

    u := &User{Name: "Martin", Email: "[e-mail]"}
    dbm.AddUser(u)

    r1 := &Record{Name: "@", Type: "MX", Host: "mail.ishost.dk"}
    r2 := &Record{Name: "@", Type: "MX", Host: "mail.infoserv.dk"}
    dbm.AddRecord(r1)
    dbm.AddRecord(r2)

    d := &Domain{User:u, Name: "Martin", Records: []*Record{r1, r2}}
    dbm.AddDomain(d)

    x:=Save(dbm.GetObjects())

    fmt.Println("== Saved objects")
//  fmt.Println(string(x))

    fmt.Println("== Loading")

    dbm2:=DBM{}
    dbm2.SetObjects(Load(x))


    u2:=dbm2.Users[0]
    u2.Print()
    u2.Name="KURT"
    u2.Print()

    d2:=dbm2.Domains[0]
    d2.Print()
    d2.User.Name="ZIG"
    u2.Print()

}

func main() {
    testState()
}