donglu9898 2017-02-02 19:40
浏览 320
已采纳

如何使用Golang将数据导入Elasticsearch

I am using gopkg.in/olivere/elastic.v5 and I am trying to import data from a json file to elasticsearch DB using golang. This is my code

package main

import(
    "gopkg.in/olivere/elastic.v5"
    "golang.org/x/net/context"
    "log"
    "os"
    "encoding/json"
)

type people struct{
    Firstname string `json:"firstname"`
    Lastname string `json:"lastname"`
    Institution string `json:"institution"`
    Email string `json:"email"`
}

type item struct{
    Id string `json:"id"`
    Title string `json:"title"`
    Journal  string `json:"journal"`
    Volume int `json:"volume"`
    Number int `json:"number"`
    Pages string `json:"pages"`
    Year int `json:"year"`
    Authors []people `json:"authors"`
    Abstract string `json:"abstract"`
    Link string `json:"link"`
    Keywords []string `json:"keywords"`
    Body string `json:"body"`
}

var client *elastic.Client
var err error
func init(){
    client,err = elastic.NewClient()
    if err!=nil{
        log.Fatal(err)
    }
}

func main() {
    var data []item

    file,err := os.Open("data.json")
    if err!=nil{
        log.Fatal(err)
    }
    defer file.Close()

    jsonDeocder :=  json.NewDecoder(file)
    if err := jsonDeocder.Decode(&data); err!=nil{
        log.Fatal("Decode: ",err)
    }

    bulkIndex("library","article",data)
}

func bulkIndex(index string,typ string ,data []item){
    ctx := context.Background()
    for _,item := range data{
        _,err := client.Index().Index(index).Type(typ).BodyJson(item).Do(ctx)   
        if err !=nil{
            log.Fatal(err)
        }
    }   
}

The package documentation is huge and I am not sure if I have gone the right way. This compiles fine but after running this, when I check my elasticsearch DB on kibana using GET /library/article/575084573a2404eec25acdcd?pretty (575084573a2404eec25acdcd is the correct id from my json file), I am getting the following response

{
  "_index": "library",
  "_type": "article",
  "_id": "575084573a2404eec25acdcd",
  "found": false
}

How do I import my data?

EDIT: This is what I get on doing GET /library?pretty on kibana

{
  "library": {
    "aliases": {},
    "mappings": {
      "article": {
        "properties": {
          "abstract": {
            "type": "text",
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "authors": {
            "properties": {
              "email": {
                "type": "text",
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "firstname": {
                "type": "text",
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "institution": {
                "type": "text",
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "lastname": {
                "type": "text",
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              }
            }
          },
          "body": {
            "type": "text",
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "id": {
            "type": "text",
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "journal": {
            "type": "text",
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "keywords": {
            "type": "text",
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "link": {
            "type": "text",
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "number": {
            "type": "long"
          },
          "pages": {
            "type": "text",
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "title": {
            "type": "text",
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "volume": {
            "type": "long"
          },
          "year": {
            "type": "long"
          }
        }
      }
    },
    "settings": {
      "index": {
        "creation_date": "1486063182258",
        "number_of_shards": "5",
        "number_of_replicas": "1",
        "uuid": "_SLeDWb4QPinFcSwOCUtCw",
        "version": {
          "created": "5020099"
        },
        "provided_name": "library"
      }
    }
  }
}
  • 写回答

1条回答 默认 最新

  • douxianliu6756 2017-02-02 21:33
    关注

    Ok, I got it. I should have specified the Id for my item as well instead of just specifying the index and type.

    The correct statement should be

    _,err := client.Index().Index(index).Type(typ).Id(item.Id).BodyJson(item).Do(ctx)
    
    本回答被题主选为最佳回答 , 对您是否有帮助呢?
    评论

报告相同问题?

悬赏问题

  • ¥15 drone 推送镜像时候 purge: true 推送完毕后没有删除对应的镜像,手动拷贝到服务器执行结果正确在样才能让指令自动执行成功删除对应镜像,如何解决?
  • ¥15 求daily translation(DT)偏差订正方法的代码
  • ¥15 js调用html页面需要隐藏某个按钮
  • ¥15 ads仿真结果在圆图上是怎么读数的
  • ¥20 Cotex M3的调试和程序执行方式是什么样的?
  • ¥20 java项目连接sqlserver时报ssl相关错误
  • ¥15 一道python难题3
  • ¥15 牛顿斯科特系数表表示
  • ¥15 arduino 步进电机
  • ¥20 程序进入HardFault_Handler