标签：mysql elasticsearch elasticsearch导入数据 elasticsearch-jdbc
Elasticsearch一个分布式多用户能力的全文搜索引擎，基于RESTful web接口。ElasticSearch也支持存储，查询，排序，分页等数据库的功能。Elasticsearch的数据就存储在硬盘中。当我们的访问日志非常大时，kabana绘制图形的时候会非常缓慢。而且硬盘空间有限，不可能保存所有的日志文件。如果我们想获取站点每天的重要数据信息，比如每天的访问量并希望能图像化的形式显示该如何做呢？
当然首先我们要把你想要的信息从Elasticsearch导出，然在整理后再导入Elasticsearch。
下面介绍下如何把数据导入Elasticsearch
第一种方法：手动导入
1、cat test.json
{"index":{"_index":"stuff_orders","_type":"order_list","_id":903713}}
{"real_name":"刘备","user_id":48430,"address_province":"上海","address_city":"浦东新区","address_district":null,"address_street":"上海市浦东新区广兰路1弄2号345室","price":30.0,"carriage":6.0,"state":"canceled","created_at":"2013-10-24T09:09:28.000Z","payed_at":null,"goods":["营养早餐：火腿麦满分"],"position":[121.53,31.22],"weight":70.0,"height":172.0,"sex_type":"female","birthday":"1988-01-01"}2、导入elasticsearch
[root@ELKServer opt]# curl -XPOST ‘localhost:9200/stuff_orders/_bulk?pretty‘ --data-binary @test.json
{
"took" : 600,
"errors" : false,
"items" : [ {
"index" : {
"_index" : "stuff_orders",
"_type" : "order_list",
"_id" : "903713",
"_version" : 1,
"_shards" : {
"total" : 2,
"successful" : 1,
"failed" : 0
},
"status" : 201
}
} ]
}3、查看elasticsearch是否存在数据
[root@ELKServer opt]# curl localhost:9200/stuff_orders/order_list/903713?pretty
{
"_index" : "stuff_orders",
"_type" : "order_list",
"_id" : "903713",
"_version" : 1,
"found" : true,
"_source" : {
"real_name" : "刘备",
"user_id" : 48430,
"address_province" : "上海",
"address_city" : "浦东新区",
"address_district" : null,
"address_street" : "上海市浦东新区广兰路1弄2号345室",
"price" : 30.0,
"carriage" : 6.0,
"state" : "canceled",
"created_at" : "2013-10-24T09:09:28.000Z",
"payed_at" : null,
"goods" : [ "营养早餐：火腿麦满分" ],
"position" : [ 121.53, 31.22 ],
"weight" : 70.0,
"height" : 172.0,
"sex_type" : "female",
"birthday" : "1988-01-01"
}
}第二种方法：从数据库中导入
参考：http://blog.csdn.net/laoyang360/article/details/51694519
1、下载安装插件elasticsearch-jdbc-2.3.4.0
weget http://xbib.org/repository/org/xbib/elasticsearch/importer/elasticsearch-jdbc/2.3.4.0/elasticsearch-jdbc-2.3.4.0-dist.zip
elasticsearch-jdbc-2.3.4.0-dist.zip的版本要和你安装的elasticsearch对应。
unzip elasticsearch-jdbc-2.3.4.0-dist.zip mv elasticsearch-jdbc-2.3.4.0 /usr/local/ cd /usr/local/elasticsearch-jdbc-2.3.4.0/
2、配置脚本
vim import.sh
#!/bin/sh
JDBC_IMPORTER_HOME=/usr/local/elasticsearch-jdbc-2.3.4.0
bin=$JDBC_IMPORTER_HOME/bin
lib=$JDBC_IMPORTER_HOME/lib
echo ‘{
"type" : "jdbc",
"jdbc": {
"elasticsearch.autodiscover":true,
"elasticsearch.cluster":"my-application", #簇名 详见：/usr/local/elasticsearch/config/elasticsearch.yml
"url":"jdbc:mysql://localhost:3306/test", #mysql数据库地址
"user":"test", #mysql用户名
"password":"1234", #mysql密码
"sql":"select *,id as _id from workers_info",
"elasticsearch" : {
"host" : "192.168.10.49",
"port" : 9300
},
"index" : "myindex", #新的index
"type" : "mytype" #新的type
}
}‘| java -cp "${lib}/*" -Dlog4j.configurationFile=${bin}/log4j2.xml org.xbib.tools.Runner org.xbib.tools.JDBCImporterchmod + import.sh
sh import.sh
3、查看数据是否导入elasticsearch
[root@ELKServer bin]# curl -XGET ‘http://localhost:9200/myindex/mytype/_search?pretty‘
{
"took" : 15,
"timed_out" : false,
"_shards" : {
"total" : 5,
"successful" : 5,
"failed" : 0
},
"hits" : {
"total" : 1,
"max_score" : 1.0,
"hits" : [ {
"_index" : "myindex",
"_type" : "mytype",
"_id" : "AVZyXCReGHjmX33dpJi3",
"_score" : 1.0,
"_source" : {
"id" : 1,
"workername" : "xing",
"salary" : 10000,
"tel" : "1598232123",
"mailbox" : "xing@qq.com",
"department" : "yanfa",
"sex" : "F",
"qq" : 736019646,
"EmployedDates" : "2012-12-21T00:00:00.000+08:00"
}
} ]
}
}
本文出自 “宁静致远” 博客，请务必保留此出处http://irow10.blog.51cto.com/2425361/1840034
标签：mysql elasticsearch elasticsearch导入数据 elasticsearch-jdbc
原文地址：http://irow10.blog.51cto.com/2425361/1840034
