elasticsearch 教程
安装、启动、测试
elasticsearch下载地址:https://www.elastic.co/cn/downloads/elasticsearch
kibana下载地址:https://www.elastic.co/cn/downloads/kibana
启动
./elasticsearch-7.5.1/bin/elasticsearch #浏览器访问：localhost:9200
{
"name" : "jiaopandeMacBook-Pro.local",
"cluster_name" : "elasticsearch",
"cluster_uuid" : "5vku_94_SjevMGVYIo3RYg",
"version" : {
"number" : "7.5.1",
"build_flavor" : "default",
"build_type" : "tar",
"build_hash" : "3ae9ac9a93c95bd0cdc054951cf95d88e1e18d96",
"build_date" : "2019-12-16T22:57:37.835892Z",
"build_snapshot" : false,
"lucene_version" : "8.3.0",
"minimum_wire_compatibility_version" : "6.8.0",
"minimum_index_compatibility_version" : "6.0.0-beta1"
},
"tagline" : "You Know, for Search"
}
配置kibana-7.5.1-darwin-x86_64/config/kibana.yml
server.host: "localhost"
elasticsearch.hosts: ["http://localhost:9200"]
./kibana-7.5.1-darwin-x86_64/bin/kibana #浏览器访问：localhost:5601
测试
kibana Dev Tools
index
创建索引
PUT http://127.0.0.1:9200/test_index
删除索引
DELETE http://127.0.0.1:9200/test_index
索引别名
POST _aliases
{
"actions": [
{ "add" : { "index" : "test_index", "alias" : "test" } }
]
}
索引迁移
POST _reindex
{
"source": {
"index": "test_index"
},
"dest": {
"index": "test_index_v2"
}
}
POST _aliases
{
"actions": [
{ "remove" : { "index" : "test_index", "alias" : "test" } },
{ "add" : { "index" : "test_index_v2", "alias" : "test_v2" } }
]
}
索引mapping设置
PUT /dyw_index_v2
{
"settings": {
"analysis": {
"analyzer": {
"default": {
"type": "ik_max_word"
}
}
}
},
"mappings": {
"properties": {
"type":{
"type": "keyword"
},
"tags":{
"type": "text"
},
"content":{
"type": "text",
"boost": 2
},
"suggest" : {
"type" : "completion",
"analyzer": "ik_max_word"
},
"update_time":{
"type": "date",
"format": ["yyyy-MM-dd HH:mm:ss"]
},
"location":{
"type": "geo_point"
},
"join":{
"type": "object",
"properties": {
"id":{
"type":"keyword"
},
"content":{
"type":"text"
}
}
},
"extra":{
"type": "object",
"properties": {
"scene":{
"type": "nested",
"properties": {
"id":{
"type": "keyword"
},
"position":{
"type": "integer"
}
}
}
}
}
}
}
}
document
https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html
安装中文分词
./bin/elasticsearch-plugin install https://github.com/medcl/elasticsearch-analysis-ik/releases/download/v7.5.1/elasticsearch-analysis-ik-7.5.1.zip
Java API
pom依赖
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>7.5.1</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-client</artifactId>
<version>7.5.1</version>
</dependency>
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
<version>7.5.1</version>
</dependency>
客户端
/**
* @author 作者 jiaopan:
* @version 创建时间：Feb 10, 2020 11:09:05 AM
* 类说明
*/
public class ESClient {
private volatile static RestHighLevelClient client;
public static RestHighLevelClient instance(EsConfigValue config) {
if (client == null) {
synchronized (RestHighLevelClient.class) {
if (client == null) {
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(config.getUsername(),config.getPassword()));
client = new RestHighLevelClient(RestClient.builder(new HttpHost(config.getIp(),config.getPort(), "http"))
.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
@Override
public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
httpClientBuilder.disableAuthCaching();
return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
}
}));
}
}
}
return client;
}
/*
public static RestHighLevelClient instance(EsConfigValue config) {
if (client == null) {
synchronized (RestHighLevelClient.class) {
if (client == null) {
client = new RestHighLevelClient(RestClient.builder(new HttpHost(config.getIp(),config.getPort(), "http")));
}
}
}
return client;
}
*/
}
base apis
public Boolean isExist(String index, String id,String routing) {
RestHighLevelClient client = ESClient.instance(esConfig);
GetRequest request = new GetRequest(index).id(id);
request.fetchSourceContext(new FetchSourceContext(false));
request.storedFields("_none_");
request.routing(routing);
try {
return client.exists(request, RequestOptions.DEFAULT);
} catch (IOException e) {
return false;
}
}
public Map<String, Object> get(String index, String id,String routing) throws IOException {
RestHighLevelClient client = ESClient.instance(esConfig);
GetRequest request = new GetRequest(esConfig.getIndex(), id);
request.routing(routing);
GetResponse getResponse = client.get(request, RequestOptions.DEFAULT);
Map<String, Object> doc = new HashMap<String, Object>();
if(getResponse.isExists()) {
doc = getResponse.getSourceAsMap();
doc.put("id",getResponse.getId());
}
return doc;
}
public SearchResponse getByIds(List<String> ids) throws IOException {
RestHighLevelClient client = ESClient.instance(esConfig);
SearchRequest searchRequest = new SearchRequest(esConfig.getIndex());
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
BoolQueryBuilder booleanQueryBuilder = QueryBuilders.boolQuery();
IdsQueryBuilder idsQueryBuilder = QueryBuilders.idsQuery();
idsQueryBuilder.ids().addAll(ids);
booleanQueryBuilder.must(idsQueryBuilder);
/*构建查询*/
searchSourceBuilder.from(0);
searchSourceBuilder.size(ids.size());
searchSourceBuilder.query(booleanQueryBuilder);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
return searchResponse;
}
public void createOrUpdate(String index, String id,String routing,JSONObject item,Boolean bulk) throws IOException {
RestHighLevelClient client = ESClient.instance(esConfig);
GetRequest request = new GetRequest(esConfig.getIndex(), id);
request.routing(routing);
JSONObject doc = new JSONObject();
GetResponse getResponse = client.get(request, RequestOptions.DEFAULT);
if(getResponse.isExists()) {
Map<String, Object> source = getResponse.getSourceAsMap();
JSONObject content = JSONObject.parseObject(source.get("content").toString());
item.keySet().forEach(key->{
content.put(key,item.get(key));
});
doc = this.createDocSource(content,routing,true);
}
else
doc = this.createDocSource(item,routing,false);
if(bulk)
this.bulkUpdate(index, id, doc, true, routing);
else
this.update(index, id, doc, true,routing);
}
public void delete(String index, String id,String routing) {
RestHighLevelClient client = ESClient.instance(esConfig);
DeleteRequest request = new DeleteRequest(index, id);
request.routing(routing);
client.deleteAsync(request, RequestOptions.DEFAULT, new ActionListener<DeleteResponse>() {
@Override
public void onResponse(DeleteResponse response) {
if (response.getResult() == DocWriteResponse.Result.DELETED) {
logger.info("delete>>>"+"{\"id\":\""+id+"\",\"type\":\""+routing+"\"}");
}
}
@Override
public void onFailure(Exception e) {
logger.info("delete fail>>>"+"{\"id\":\""+id+"\",\"type\":\""+routing+"\"}");
}
});
}
public void update(String index, String id, JSONObject item,Boolean upsert,String routing) {
if(item == null) {
return;
}
RestHighLevelClient client = ESClient.instance(esConfig);
UpdateRequest request = new UpdateRequest(index,id).doc(item);
request.docAsUpsert(upsert);
request.retryOnConflict(3);
request.routing(routing);
client.updateAsync(request, RequestOptions.DEFAULT, new ActionListener<UpdateResponse>() {
@Override
public void onResponse(UpdateResponse response) {
if (response.getResult() == DocWriteResponse.Result.CREATED) {
logger.info("created>>>"+"{\"id\":\""+id+"\",\"type\":\""+routing+"\"}");
}
else if (response.getResult() == DocWriteResponse.Result.UPDATED) {
logger.info("updated>>>"+"{\"id\":\""+id+"\",\"type\":\""+routing+"\"}");
}
}
@Override
public void onFailure(Exception e) {
logger.warn("not_store_record("+e.getMessage()+")>>>"+item.toJSONString());
}
});
}
public void bulkUpdate(String index, String id, JSONObject item,Boolean upsert,String routing) {
if(item == null) {
return;
}
UpdateRequest request = new UpdateRequest(index,id).doc(item);
request.docAsUpsert(upsert);
request.retryOnConflict(3);
request.routing(routing);
queue.offer(request);
}
public void bulk(ConcurrentLinkedQueue<UpdateRequest> queue, RestHighLevelClient client) {
BulkRequest bulk = new BulkRequest();
for (int i = 0; i < esConfig.getBulk(); i++) {
if(queue.isEmpty()) break;
bulk.add(queue.poll());
}
if(bulk.numberOfActions() == 0) return;
logger.info(DateUtils.getNowDate()+":starting handle doc queue,actions:"+bulk.numberOfActions());
client.bulkAsync(bulk, RequestOptions.DEFAULT, new ActionListener<BulkResponse>() {
@Override
public void onResponse(BulkResponse response) {
if(response.hasFailures())
logger.info(response.buildFailureMessage());
}
@Override
public void onFailure(Exception e) {
logger.warn("bulk handle docs error:"+e.getMessage());
}
});
}
Common apis
public void locationSearch(SearchSourceBuilder searchSourceBuilder, BoolQueryBuilder booleanQueryBuilder,
String location,String lockey,Double distance,Boolean sort) throws FontFormatException {
String[] latlon = location.split(",");
if(latlon.length != 2) {
throw new NumberFormatException("经纬度参数不正确.location参数以英文逗号(,)连接的字符串.正确示例:“纬度(latitude),经度(longitude)\"");
}
Double lat = Double.parseDouble(latlon[0]);
Double lon = Double.parseDouble(latlon[1]);
GeoDistanceQueryBuilder geoQueryBuilder = QueryBuilders.geoDistanceQuery(lockey).point(lat, lon)
.distance(distance == null?esConfig.getDistance():distance, DistanceUnit.KILOMETERS)
.geoDistance(GeoDistance.PLANE);
if(sort) {
GeoDistanceSortBuilder distanceSortBuilder = SortBuilders.geoDistanceSort(lockey,lat,lon);
distanceSortBuilder.order(SortOrder.ASC);
distanceSortBuilder.point(lat, lon);
searchSourceBuilder.sort(distanceSortBuilder);
}
booleanQueryBuilder.must(geoQueryBuilder);
}
public void fullTextSearch(BoolQueryBuilder booleanQueryBuilder, String keyword,List<String> fields,Boolean must) {
DisMaxQueryBuilder disMaxQueryBuilder = QueryBuilders.disMaxQuery();
fields.forEach(item->{
disMaxQueryBuilder.add(QueryBuilders.matchQuery(item,keyword));
});
if(must) booleanQueryBuilder.must(disMaxQueryBuilder);
else booleanQueryBuilder.should(disMaxQueryBuilder);
}
public void phraseTextSearch(BoolQueryBuilder booleanQueryBuilder, String keyword,List<String> fields,Boolean must) {
DisMaxQueryBuilder disMaxQueryBuilder = QueryBuilders.disMaxQuery();
List<String> words = Arrays.asList(keyword.split(" "));
fields.forEach(item->{
words.forEach(word->{
disMaxQueryBuilder.add(QueryBuilders.matchPhraseQuery(item,word));
});
});
if(must) booleanQueryBuilder.must(disMaxQueryBuilder);
else booleanQueryBuilder.should(disMaxQueryBuilder);
}
public void textSearch(BoolQueryBuilder booleanQueryBuilder, String keyword, List<String> fields, Boolean phrase,Boolean must) {
if(phrase) phraseTextSearch(booleanQueryBuilder, keyword, fields,must);
else fullTextSearch(booleanQueryBuilder, keyword, fields,must);
}
public void typeSearch(BoolQueryBuilder booleanQueryBuilder,String key,List<String> typeArray, List<String> mustNot) {
DisMaxQueryBuilder disMaxQueryBuilder = QueryBuilders.disMaxQuery();
for (int i = 0; i < typeArray.size(); i++) {
disMaxQueryBuilder.add(QueryBuilders.termQuery(key,typeArray.get(i)));
}
booleanQueryBuilder.must(disMaxQueryBuilder);
List<QueryBuilder> mustNotQuerys = new ArrayList<QueryBuilder>();
for (int i = 0; i < mustNot.size(); i++) {
QueryBuilder query = QueryBuilders.termQuery(key,mustNot.get(i));
mustNotQuerys.add(query);
}
booleanQueryBuilder.mustNot().addAll(mustNotQuerys);
}
public void typeSearch(BoolQueryBuilder booleanQueryBuilder,String key, List<String> typeArray) {
DisMaxQueryBuilder disMaxQueryBuilder = QueryBuilders.disMaxQuery();
for (int i = 0; i < typeArray.size(); i++) {
disMaxQueryBuilder.add(QueryBuilders.termQuery(key,typeArray.get(i)));
}
booleanQueryBuilder.must(disMaxQueryBuilder);
}
public void createQuery(SearchRequest searchRequest, SearchSourceBuilder searchSourceBuilder,Integer from, Integer size, QueryBuilder queryBuilder) {
searchSourceBuilder.from(from == null?esConfig.getFrom():from);
searchSourceBuilder.size(size == null?esConfig.getSize():size);
searchSourceBuilder.query(queryBuilder);
searchRequest.source(searchSourceBuilder);
}
public void createQuery(SearchRequest searchRequest, SearchSourceBuilder searchSourceBuilder,Integer from, Integer size, QueryBuilder queryBuilder, QueryBuilder postFilter) {
searchSourceBuilder.from(from == null?esConfig.getFrom():from);
searchSourceBuilder.size(size == null?esConfig.getSize():size);
searchSourceBuilder.query(queryBuilder);
searchSourceBuilder.postFilter(postFilter);
searchRequest.source(searchSourceBuilder);
}
public void sort(SearchSourceBuilder searchSourceBuilder, int sortField) {
String field = dict.getSortField().get(sortField);
String order = dict.getSortType().get(field);
searchSourceBuilder.sort(new FieldSortBuilder(field).order(SortOrder.fromString(order)));
}
public void searchMustNot(BoolQueryBuilder booleanQueryBuilder,String field,String values) {
List<String> mustNot = Arrays.asList(values.split(","));
mustNot.forEach(item->{
booleanQueryBuilder.mustNot(QueryBuilders.termQuery(field,item));
});
}
search
public List<Map<String, Object>> search(String keyword,String scopes, Integer from, Integer size,String location, Double distance,Boolean matchPhrase,Boolean scoreSort) throws IOException, FontFormatException {
RestHighLevelClient client = ESClient.instance(esConfig);
SearchRequest searchRequest = new SearchRequest(esConfig.getIndex());
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
BoolQueryBuilder booleanQueryBuilder = QueryBuilders.boolQuery();
/*type检索*/
if(!StringUtils.equals(scopes,"all")) {
List<String> scopeArray = new ArrayList<String>();
scopeArray = Arrays.asList(scopes.split(","));
engineBaseMethod.typeSearch(booleanQueryBuilder,"type",scopeArray);
}
String notTypes = StringUtils.join(dict.getMustNot(), ",");
engineBaseMethod.searchMustNot(booleanQueryBuilder, "type",notTypes);
booleanQueryBuilder.must(QueryBuilders.termQuery("extra.up","1"));
/*多字段检索*/
if(StringUtils.isNotBlank(keyword)) {
//List<String> fieldList = Arrays.asList("title,cn_name,suggest".split(","));
List<String> fieldList = dict.getSearchMatch();
engineBaseMethod.textSearch(booleanQueryBuilder, keyword, fieldList, matchPhrase,true);
}
/*排序*/
engineBaseMethod.sort(searchSourceBuilder,0);
engineBaseMethod.sort(searchSourceBuilder,1);
/*地理位置范围内搜索并就近排序*/
if(StringUtils.isNotBlank(location)) {
engineBaseMethod.locationSearch(searchSourceBuilder, booleanQueryBuilder, location,"location",distance,true);
}
int index = 2 + (int)(Math.random() * ((3 - 2) + 1));
if(scoreSort) searchSourceBuilder.sort(new ScoreSortBuilder().order(SortOrder.DESC));
else engineBaseMethod.sort(searchSourceBuilder,index);
/*构建查询*/
engineBaseMethod.createQuery(searchRequest, searchSourceBuilder, from, size, booleanQueryBuilder);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
/*解析结果并返回*/
List<Map<String, Object>> result = new ArrayList<Map<String,Object>>();
engineBaseMethod.searchResultAnalyse(result, searchResponse);
return result;
}
public List<String> searchTips(String keyword, String type) throws IOException, FontFormatException {
RestHighLevelClient client = ESClient.instance(esConfig);
//前缀搜索建议
SearchRequest searchRequest = new SearchRequest(esConfig.getIndex());
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
CompletionSuggestionBuilder suggestionBuilder = new CompletionSuggestionBuilder("suggest");
suggestionBuilder.prefix(keyword);
suggestionBuilder.size(esConfig.getSize());
suggestionBuilder.skipDuplicates(true);
SuggestBuilder suggestBuilder = new SuggestBuilder();
suggestBuilder.addSuggestion("search_tips", suggestionBuilder);
searchSourceBuilder.suggest(suggestBuilder);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
//匹配搜索建议
SearchSourceBuilder querySourceBuilder = new SearchSourceBuilder();
SearchRequest queryRequest = new SearchRequest(esConfig.getIndex());
BoolQueryBuilder booleanQueryBuilder = QueryBuilders.boolQuery();
if(StringUtils.isNotBlank(keyword)) booleanQueryBuilder.should(QueryBuilders.matchQuery("title",keyword));
String notTypes = StringUtils.join(dict.getMustNot(), ",");
engineBaseMethod.searchMustNot(booleanQueryBuilder, "type",notTypes);
querySourceBuilder.query(booleanQueryBuilder);
queryRequest.source(querySourceBuilder);
SearchResponse queryResponse = client.search(queryRequest, RequestOptions.DEFAULT);
/*解析结果并返回*/
List<String> result = new ArrayList<String>();
Suggest suggest = searchResponse.getSuggest();
CompletionSuggestion completionSuggestion = suggest.getSuggestion("search_tips");
for(CompletionSuggestion.Entry entry : completionSuggestion.getEntries()) {
for (CompletionSuggestion.Entry.Option option : entry) {
String optionType = option.getHit().getSourceAsMap().get("type").toString();
if(StringUtils.equals(type, "all") || StringUtils.equals(type,optionType)) result.add(option.getText().string());
}
}
for(SearchHit hit:queryResponse.getHits()) {
String text = (String) hit.getSourceAsMap().get("title");
if(!result.contains(text)) result.add(text);
}
return result;
}
集群安装
master 配置
# ---------------------------------- Cluster -----------------------------------
cluster.name: es-cluster
# ------------------------------------ Node ------------------------------------
node.name: node1-rec
node.master: true
node.data: false
# ----------------------------------- Paths ------------------------------------
path.data: /data/es/data
path.logs: /data/es/logs
# ----------------------------------- Memory -----------------------------------
bootstrap.memory_lock: false
# ---------------------------------- Network -----------------------------------
network.host: 0.0.0.0
network.publish_host: 10.2.64.110
http.port: 9200
# --------------------------------- Discovery ----------------------------------
discovery.seed_hosts: ["10.2.64.110", "10.2.64.111", "10.2.64.112"]
cluster.initial_master_nodes: ["10.2.64.110"]
discovery.zen.fd.ping_timeout: 1m
discovery.zen.fd.ping_retries: 5
http.cors.allow-origin: "*"
http.cors.enabled: true
bootstrap.system_call_filter: false
xpack.license.self_generated.type: basic
node 配置
cluster.name: es-cluster
# ------------------------------------ Node ------------------------------------
node.name: node2-rec
node.master: false
node.data: true
# ----------------------------------- Paths ------------------------------------
path.data: /data/es/data
path.logs: /data/es/logs
# ----------------------------------- Memory -----------------------------------
bootstrap.memory_lock: false
# ---------------------------------- Network -----------------------------------
network.host: 0.0.0.0
network.publish_host: 10.2.64.111
http.port: 9200
# --------------------------------- Discovery ----------------------------------
discovery.seed_hosts: ["10.2.64.110", "10.2.64.111", "10.2.64.112"]
cluster.initial_master_nodes: ["10.2.64.110"]
discovery.zen.fd.ping_timeout: 1m
discovery.zen.fd.ping_retries: 5
http.cors.allow-origin: "*"
http.cors.enabled: true
bootstrap.system_call_filter: false
#
xpack.license.self_generated.type: basic
master kibana 配置
server.port: 5601
server.host: "0.0.0.0"
server.name: "node1-rec"
elasticsearch.hosts: ["http://10.2.64.110:9200","http://10.2.64.111:9200","http://10.2.64.112:9200"]
Kibana查看集群状态
GET _cluster/health?pretty
{
"cluster_name" : "es-cluster",
"status" : "green",
"timed_out" : false,
"number_of_nodes" : 3,
"number_of_data_nodes" : 2,
"active_primary_shards" : 7,
"active_shards" : 14,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 0,
"delayed_unassigned_shards" : 0,
"number_of_pending_tasks" : 0,
"number_of_in_flight_fetch" : 0,
"task_max_waiting_in_queue_millis" : 0,
"active_shards_percent_as_number" : 100.0
}
本博客所有文章除特别声明外，均采用 CC BY-SA 3.0协议 。转载请注明出处！