donglu9898 2017-02-02 19:40
浏览 320
已采纳

如何使用Golang将数据导入Elasticsearch

I am using gopkg.in/olivere/elastic.v5 and I am trying to import data from a json file to elasticsearch DB using golang. This is my code

package main

import(
    "gopkg.in/olivere/elastic.v5"
    "golang.org/x/net/context"
    "log"
    "os"
    "encoding/json"
)

type people struct{
    Firstname string `json:"firstname"`
    Lastname string `json:"lastname"`
    Institution string `json:"institution"`
    Email string `json:"email"`
}

type item struct{
    Id string `json:"id"`
    Title string `json:"title"`
    Journal  string `json:"journal"`
    Volume int `json:"volume"`
    Number int `json:"number"`
    Pages string `json:"pages"`
    Year int `json:"year"`
    Authors []people `json:"authors"`
    Abstract string `json:"abstract"`
    Link string `json:"link"`
    Keywords []string `json:"keywords"`
    Body string `json:"body"`
}

var client *elastic.Client
var err error
func init(){
    client,err = elastic.NewClient()
    if err!=nil{
        log.Fatal(err)
    }
}

func main() {
    var data []item

    file,err := os.Open("data.json")
    if err!=nil{
        log.Fatal(err)
    }
    defer file.Close()

    jsonDeocder :=  json.NewDecoder(file)
    if err := jsonDeocder.Decode(&data); err!=nil{
        log.Fatal("Decode: ",err)
    }

    bulkIndex("library","article",data)
}

func bulkIndex(index string,typ string ,data []item){
    ctx := context.Background()
    for _,item := range data{
        _,err := client.Index().Index(index).Type(typ).BodyJson(item).Do(ctx)   
        if err !=nil{
            log.Fatal(err)
        }
    }   
}

The package documentation is huge and I am not sure if I have gone the right way. This compiles fine but after running this, when I check my elasticsearch DB on kibana using GET /library/article/575084573a2404eec25acdcd?pretty (575084573a2404eec25acdcd is the correct id from my json file), I am getting the following response

{
  "_index": "library",
  "_type": "article",
  "_id": "575084573a2404eec25acdcd",
  "found": false
}

How do I import my data?

EDIT: This is what I get on doing GET /library?pretty on kibana

{
  "library": {
    "aliases": {},
    "mappings": {
      "article": {
        "properties": {
          "abstract": {
            "type": "text",
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "authors": {
            "properties": {
              "email": {
                "type": "text",
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "firstname": {
                "type": "text",
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "institution": {
                "type": "text",
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "lastname": {
                "type": "text",
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              }
            }
          },
          "body": {
            "type": "text",
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "id": {
            "type": "text",
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "journal": {
            "type": "text",
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "keywords": {
            "type": "text",
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "link": {
            "type": "text",
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "number": {
            "type": "long"
          },
          "pages": {
            "type": "text",
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "title": {
            "type": "text",
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "volume": {
            "type": "long"
          },
          "year": {
            "type": "long"
          }
        }
      }
    },
    "settings": {
      "index": {
        "creation_date": "1486063182258",
        "number_of_shards": "5",
        "number_of_replicas": "1",
        "uuid": "_SLeDWb4QPinFcSwOCUtCw",
        "version": {
          "created": "5020099"
        },
        "provided_name": "library"
      }
    }
  }
}
  • 写回答

1条回答 默认 最新

  • douxianliu6756 2017-02-02 21:33
    关注

    Ok, I got it. I should have specified the Id for my item as well instead of just specifying the index and type.

    The correct statement should be

    _,err := client.Index().Index(index).Type(typ).Id(item.Id).BodyJson(item).Do(ctx)
    
    本回答被题主选为最佳回答 , 对您是否有帮助呢?
    评论

报告相同问题?

悬赏问题

  • ¥15 MATLAB动图的问题
  • ¥15 求差集那个函数有问题,有无佬可以解决
  • ¥15 【提问】基于Invest的水源涵养
  • ¥20 微信网友居然可以通过vx号找到我绑的手机号
  • ¥15 寻一个支付宝扫码远程授权登录的软件助手app
  • ¥15 解riccati方程组
  • ¥15 display:none;样式在嵌套结构中的已设置了display样式的元素上不起作用?
  • ¥15 使用rabbitMQ 消息队列作为url源进行多线程爬取时,总有几个url没有处理的问题。
  • ¥15 Ubuntu在安装序列比对软件STAR时出现报错如何解决
  • ¥50 树莓派安卓APK系统签名