1、 Es基本指令

  1. index,document 索引、类型
  1. #POST 提交的时候会检查数据对比，如果一直_seq_num 和version 都不会变 需要带上_update
  2. #如果不带update会导致version和如果一直_seq_num都加1
  3. POST /customer/external/1/_update
  4. {
  5.     "doc":{
  6.       "name":"xuwei"
  7.     }
  8. }
  10. #PUT 会直接替换
  11. PUT /customer/external/1/
  12. {
  13.   "name": "xuwei"
  14. }

2、导入批量数据

  1. POST /customer/external/_bulk
  2. #https://github.com/elastic/elasticsearch/blob/master/docs/src/test/resources/accounts.json
  3. #https://segmentfault.com/a/1190000037572389

3、 请求的两种方式

  1. 1.Request Api URl
  2. 2.Query DSL （常用）
  6. GET /bank/_search
  7. {
  8.   "query": {
  9.     "match_all": {}
  10.   },
  11.   "sort": [
  12.     {
  13.       "account_number": {
  14.         "order": "desc"
  15.       }
  16.       ,
  17.       "balance": {
  18.         "order": "asc"
  19.       }
  20.     }
  21.   ],
  22.   "_source": ["balance","firstname"] #不使用默认是select *
  24. }

3.1 Query

1、 match

  1. #match_phrase精准匹配  match会分词匹配
  2. GET bank/_search
  3. {
  4.   "query": {
  5.     "match_phrase": {
  6.       "address": "kings Hwy"
  7.     }
  8.   }
  9. }
  11. #multi_match 多熟悉查询包含 也支持分词
  12. GET bank/_search
  13. {
  14.   "query": {
  15.     "multi_match": {
  16.       "query": "mill Movico",
  17.       "fields": ["address","city"]
  18.     }
  19.   }
  20. }

2、 符合查询 Bool

  1. //Must
  2. GET bank/_search
  3. {
  4.   "query": {
  5.     "bool": {
  6.       "must": [
  7.         {
  8.           "match": {
  9.             "gender": "F"
  10.           }
  11.         },
  12.         {
  13.           "match": {
  14.             "address": "mill"
  15.           }
  16.         }
  17.       ]
  18.     }
  19.   }
  20. }
  22. //must_not
  23. GET bank/_search
  24. {
  25.   "query": {
  26.     "bool": {
  27.       "must": [
  28.         {
  29.           "match": {
  30.             "gender": "F"
  31.           }
  32.         },
  33.         {
  34.           "match": {
  35.             "address": "mill Street"
  36.           }
  37.         }
  38.       ],
  39.       "must_not": [
  40.         {
  41.           "match": {
  42.             "age": "32"
  43.           }
  44.         }
  45.       ]
  46.     }
  47.   }
  48. }
  50. //should 对得分影响
  51. GET bank/_search
  52. {
  53.   "query": {
  54.     "bool": {
  55.       "must": [
  56.         {
  57.           "match": {
  58.             "gender": "F"
  59.           }
  60.         },
  61.         {
  62.           "match": {
  63.             "address": "mill Street"
  64.           }
  65.         }
  66.       ],
  68.       "should": [
  69.         {
  70.           "match": {
  71.             "lastname": "Bates"
  72.           }
  73.         }
  74.       ]
  75.     }
  76.   }
  77. }
  78. // must 会提供相关性得分 filter不会提供相关性得分
  79. GET bank/_search
  80. {
  81.   "query": {
  82.     "bool": {
  83.       "filter": [
  84.         {
  85.           "range": {
  86.             "age": {
  87.               "gte": 10,
  88.               "lte": 30
  89.             }
  90.           }
  91.         }
  92.       ]
  93.     }
  94.   }
  95. }
  97. GET bank/_search
  98. {
  99.   "query": {
  100.     "bool": {
  101.       "must": [
  102.         {
  103.           "range": {
  104.             "age": {
  105.               "gte": 10,
  106.               "lte": 30
  107.             }
  108.           }
  109.         }
  110.       ]
  111.     }
  112.   }
  113. }
  115. //term 推荐搜索准确的值对text不要使用 match_pahse也可以精确查询，可以短语匹配
  116. #全文检索用match(match下有keyword)    非text检索用term
  117. GET /bank/_search
  118. {
  119.   "query": {
  120.     "term": {
  121.       "age": {
  122.         "value": "28"
  123.       }
  124.     }
  125.   }
  126. }
  128. //精确匹配 keyword
  129. GET /bank/_search
  130. {
  131.   "query": {
  132.     "match": {
  133.       "address.keyword": "302 Howard Place"
  134.     }
  135.   }
  136. }

2、数据分析分组（aggs）

  1. //搜索address中所有mill的年龄分布和平均年龄 子聚合
  3. GET bank/_search
  4. {
  5.   "query": {
  6.     "match_all": {
  7.     }
  8.   },
  9.   "aggs": {
  10.     "ageFAgg": {
  11.       "terms": {
  12.         "field": "age",
  13.         "size": 10
  14.       },
  15.       "aggs": {
  16.         "ageAgg": {
  17.           "avg": {
  18.             "field": "age"
  19.           }
  20.         }
  21.       }
  22.     },
  23.     "banlanceAgg":{
  24.       "avg": {
  25.         "field": "balance"
  26.       }
  27.     }
  28.   }
  29. }
  34. #搜索所有年龄分布下的性别分布的平均年龄 如：M Count: 24  avg:32,F Count: 24  avg:32
  36. GET bank/_search
  37. {
  38.   "query": {
  39.     "match_all": {}
  40.   },
  41.   "aggs": {
  42.     "ageFAgg": {
  43.       "terms": {
  44.         "field": "age",
  45.         "size": 10
  46.       },
  47.       "aggs": {
  48.         "genderAgg": {
  49.           "terms": {
  50.             "field": "gender.keyword"
  51.           },
  52.           "aggs": {
  53.             "avg2Agg": {
  54.               "avg": {
  55.                 "field": "age"
  56.               }
  57.             }
  58.           }
  59.         }
  60.       }
  61.     }
  62.   }
  63. }

3、属性映射

  1. #创建所以制定映射
  2. PUT /my_index
  3. {
  4.   "mappings": {
  5.     "properties": {
  6.       "age":{"type": "integer"}, //普通类型
  7.       "email":{"type": "keyword"},//全词精确查找
  8.       "name":{"type": "text"}//自动分词检索
  9.     }
  10.   }
  11. }
  13. #修改映射 可以添加属性
  14. PUT /my_index/_mapping
  15. {
  16.   "properties": {
  17.     "employee-id": {
  18.       "type": "keyword",
  19.       "index": false, //不需要索引  默认是true 可以被索引
  20.        "doc_values": false //不会被用作排序啥的  冗余处处、存储可以使用这两个字段
  21.     }
  22.   }
  23. }

4、数据迁移/映射修改

  1. #新映射规则
  2. PUT newbank
  3. {
  4.   "mappings": {
  5.     "properties": {
  6.       "account_number": {
  7.         "type": "long"
  8.       },
  9.       "address": {
  10.         "type": "text"
  11.       },
  12.       "age": {
  13.         "type": "long"
  14.       },
  15.       "balance": {
  16.         "type": "long"
  17.       },
  18.       "city": {
  19.         "type": "text"
  20.       },
  21.       "email": {
  22.         "type": "keyword"
  23.       },
  24.       "employer": {
  25.         "type": "keyword"
  26.       },
  27.       "firstname": {
  28.         "type": "text"
  29.       },
  30.       "gender": {
  31.         "type": "text"
  32.       },
  33.       "lastname": {
  34.         "type": "text",
  35.         "fields": {
  36.           "keyword": {
  37.             "type": "keyword",
  38.             "ignore_above": 256
  39.           }
  40.         }
  41.       },
  42.       "state": {
  43.         "type": "keyword"
  44.       }
  45.     }
  46.   }
  47. }
  49. #迁移 index/type/doc
  50. POST _reindex
  51. {
  52.   "source": {
  53.     "index": "bank",
  54.     "type": "count" #如果是存在type的源数据 加上type ，否则不需要加
  55.   },
  56.   "dest": {
  57.     "index": "newbank"
  58.   }
  59. }

4、分词器

  1. #安装 就是下载一个压缩包解压到plugins
  2. chmod +R 777 /mydata/elasticsearch/plugins/ik
  3. cd /mydata/elasticsearch/plugins/ik
  4. wget https://github.com/medcl/elasticsearch-analysis-ik/releases/download/v7.6.2/elasticsearch-analysis-ik-7.6.2.zip
  5. unzip elasticsearch-analysis-ik-7.6.2.zip
  7. #使用 支持中文分词
  8. POST _analyze
  9. {
  10.   "analyzer": "ik_smart",
  11.   "text": "我是中国人"
  12. }
  14. POST _analyze
  15. {
  16.   "analyzer": "ik_max_word",
  17.   "text": "我是中国人"
  18. }
  22. #建立索引的时候制定默认的IK
  23. PUT /my_idnex
  24. {
  26. }

5、 RestHighLevelClient

  1. # es config
  2. @Configuration
  3. public class EsConfig {
  5.     public static final RequestOptions COMMON_OPTIONS;
  7.     static {
  8.         RequestOptions.Builder builder = RequestOptions.DEFAULT.toBuilder();
  9.         //后期需要加权限认证等
  11. //        builder.addHeader("Authorization", "Bearer " + TOKEN);
  12. //        builder.setHttpAsyncResponseConsumerFactory(
  13. //                new HttpAsyncResponseConsumerFactory
  14. //                        .HeapBufferedResponseConsumerFactory(30 * 1024 * 1024 * 1024));
  15.         COMMON_OPTIONS = builder.build();
  16.     }
  19.     @Bean
  20.     public RestHighLevelClient restHighLevelClient() {
  21.         RestHighLevelClient restHighLevelClient = new RestHighLevelClient(RestClient.builder
  22.                 (new HttpHost("t.freefish.info", 9200, "http")));
  24.         return restHighLevelClient;
  25.     }
  27. }
  1. DSL查询语句
  2. GET bank/_search
  3. {
  4.   "query": {
  5.     "match": {
  6.       "address": "mill"
  7.     }
  8.   },
  9.   "aggs": {
  10.     "ageAgg": {
  11.       "terms": {
  12.         "field": "age",
  13.         "size": 10
  14.       },
  15.       "aggs": {
  16.         "avgAgg": {
  17.           "avg": {
  18.             "field": "age"
  19.           }
  20.         },
  21.         "balanceAgg": {
  22.           "avg": {
  23.             "field": "balance"
  24.           }
  25.         }
  26.       }
  27.     }
  28.   }
  29. }
  31. DSL直接查询数据   
  32. {
  33.   "took" : 1,
  34.   "timed_out" : false,
  35.   "_shards" : {
  36.     "total" : 1,
  37.     "successful" : 1,
  38.     "skipped" : 0,
  39.     "failed" : 0
  40.   },
  41.   "hits" : {
  42.     "total" : {
  43.       "value" : 4,
  44.       "relation" : "eq"
  45.     },
  46.     "max_score" : 5.4032025,
  47.     "hits" : [
  48.       {
  49.         "_index" : "bank",
  50.         "_type" : "count",
  51.         "_id" : "970",
  52.         "_score" : 5.4032025,
  53.         "_source" : {
  54.           "account_number" : 970,
  55.           "balance" : 19648,
  56.           "firstname" : "Forbes",
  57.           "lastname" : "Wallace",
  58.           "age" : 28,
  59.           "gender" : "M",
  60.           "address" : "990 Mill Road",
  61.           "employer" : "Pheast",
  62.           "email" : "forbeswallace@pheast.com",
  63.           "city" : "Lopezo",
  64.           "state" : "AK"
  65.         }
  66.       },
  67.       {
  68.         "_index" : "bank",
  69.         "_type" : "count",
  70.         "_id" : "136",
  71.         "_score" : 5.4032025,
  72.         "_source" : {
  73.           "account_number" : 136,
  74.           "balance" : 45801,
  75.           "firstname" : "Winnie",
  76.           "lastname" : "Holland",
  77.           "age" : 38,
  78.           "gender" : "M",
  79.           "address" : "198 Mill Lane",
  80.           "employer" : "Neteria",
  81.           "email" : "winnieholland@neteria.com",
  82.           "city" : "Urie",
  83.           "state" : "IL"
  84.         }
  85.       },
  86.       {
  87.         "_index" : "bank",
  88.         "_type" : "count",
  89.         "_id" : "345",
  90.         "_score" : 5.4032025,
  91.         "_source" : {
  92.           "account_number" : 345,
  93.           "balance" : 9812,
  94.           "firstname" : "Parker",
  95.           "lastname" : "Hines",
  96.           "age" : 38,
  97.           "gender" : "M",
  98.           "address" : "715 Mill Avenue",
  99.           "employer" : "Baluba",
  100.           "email" : "parkerhines@baluba.com",
  101.           "city" : "Blackgum",
  102.           "state" : "KY"
  103.         }
  104.       },
  105.       {
  106.         "_index" : "bank",
  107.         "_type" : "count",
  108.         "_id" : "472",
  109.         "_score" : 5.4032025,
  110.         "_source" : {
  111.           "account_number" : 472,
  112.           "balance" : 25571,
  113.           "firstname" : "Lee",
  114.           "lastname" : "Long",
  115.           "age" : 32,
  116.           "gender" : "F",
  117.           "address" : "288 Mill Street",
  118.           "employer" : "Comverges",
  119.           "email" : "leelong@comverges.com",
  120.           "city" : "Movico",
  121.           "state" : "MT"
  122.         }
  123.       }
  124.     ]
  125.   },
  126.   "aggregations" : {
  127.     "ageAgg" : {
  128.       "doc_count_error_upper_bound" : 0,
  129.       "sum_other_doc_count" : 0,
  130.       "buckets" : [
  131.         {
  132.           "key" : 38,
  133.           "doc_count" : 2,
  134.           "balanceAgg" : {
  135.             "value" : 27806.5
  136.           },
  137.           "avgAgg" : {
  138.             "value" : 38.0
  139.           }
  140.         },
  141.         {
  142.           "key" : 28,
  143.           "doc_count" : 1,
  144.           "balanceAgg" : {
  145.             "value" : 19648.0
  146.           },
  147.           "avgAgg" : {
  148.             "value" : 28.0
  149.           }
  150.         },
  151.         {
  152.           "key" : 32,
  153.           "doc_count" : 1,
  154.           "balanceAgg" : {
  155.             "value" : 25571.0
  156.           },
  157.           "avgAgg" : {
  158.             "value" : 32.0
  159.           }
  160.         }
  161.       ]
  162.     }
  163.   }
  164. }
  166. //API
  167. @Test
  168.     public void SearchData() throws IOException {
  169.         SearchRequest search = new SearchRequest("bank");
  170.         //知道DSL
  171.         SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
  172.         SearchSourceBuilder query = searchSourceBuilder.query(QueryBuilders.matchQuery("address", "mill"));
  175.         //年龄分布聚合
  176.         TermsAggregationBuilder ageAgg = AggregationBuilders.terms("ageAgg").field("age").size(5);
  178.         //年龄分布子聚合 avgAgg
  179.         ageAgg.subAggregation(AggregationBuilders.avg("avgAgg").field("age"));
  180.         //年龄分布子聚合 balanceAgg
  181.         ageAgg.subAggregation(AggregationBuilders.avg("balanceAgg").field("balance"));
  184.         query.aggregation(ageAgg);
  185.         search.source(searchSourceBuilder);
  187.         log.info("检索条件{}", query);
  189.         SearchResponse ret = restHighLevelClient.search(search, RequestOptions.DEFAULT);
  190.         log.info("检索结果{}", ret);
  191.         SearchHit[] hits = ret.getHits().getHits();
  192.         for (int i = 0; i < hits.length; i++) {
  193.             System.out.println(JSON.parseObject(hits[i].getSourceAsString(), Acount.class));
  194.         }
  196.         //获取聚合信息
  197.         Aggregations aggregations = ret.getAggregations();
  198.         Terms ageAgg_ret = aggregations.get("ageAgg");
  200.                 //获取buckets
  201.         List<? extends Terms.Bucket> buckets = ageAgg_ret.getBuckets();
  203.             //每个buckets包含agg信息
  204.         buckets.forEach(p -> {
  205.             System.out.println("bucket:" + p.getKeyAsString());
  206.             Aggregations aggs = p.getAggregations();
  207.             Avg avgAgg = aggs.get("avgAgg");
  208.             Avg balanceAgg = aggs.get("balanceAgg");
  209.             System.out.println("avgAgg.getValue()" + avgAgg.getValue());
  210.             System.out.println("balanceAgg.getValue()" + balanceAgg.getValue());
  211.         });
  214.     }

6、数组扁平处理

  1. #默认Es会扁平化处理数据 nested 嵌入式的
  2. PUT my_index/my_type/1
  3. {
  4.   "group" : "fans",
  5.   "user" : [ 
  6.     {
  7.       "first" : "John",
  8.       "last" :  "Smith"
  9.     },
  10.     {
  11.       "first" : "Alice",
  12.       "last" :  "White"
  13.     }
  14.   ]
  15. }
  16. #实际存储结构为
  17. {
  18.   "group" :        "fans",
  19.   "user.first" : [ "alice", "john" ],
  20.   "user.last" :  [ "smith", "white" ]
  21. }
  23. #查询 不应该查到的数据被查到了
  24. GET my_index/_search
  25. {
  26.   "query": {
  27.     "bool": {
  28.       "must": [
  29.         {
  30.           "match": {
  31.             "user.first": "Alice"
  32.           }
  33.         },
  34.         {
  35.           "match": {
  36.             "user.last": "Smith"
  37.           }
  38.         }
  39.       ]
  40.     }
  41.   }
  42. }

6、正确的方式

  1. //创建nested索引
  2. PUT my_index
  3. {
  4.   "mappings": {
  5.     "properties": {
  6.       "user":{
  7.         "type": "nested"
  8.       }
  9.     }
  10.   }
  11. }
  13. //存入数据
  14. PUT my_index/my_type/1
  15. {
  16.   "group" : "fans",
  17.   "user" : [ 
  18.     {
  19.       "first" : "John",
  20.       "last" :  "Smith"
  21.     },
  22.     {
  23.       "first" : "Alice",
  24.       "last" :  "White"
  25.     }
  26.   ]
  27. }
  28. //此时正确不可查询到数据
  29. GET my_index/_search
  30. {
  31.   "query": {
  32.     "bool": {
  33.       "must": [
  34.         {
  35.           "match": {
  36.             "user.first": "Alice"
  37.           }
  38.         },
  39.         {
  40.           "match": {
  41.             "user.last": "Smith"
  42.           }
  43.         }
  44.       ]
  45.     }
  46.   }
  47. }