elasticsearch学习
1.是什么
- 分布式的实时文件存储,每个字段都被索引并可被搜索
- 分布式的实时分析搜索引擎
- 可以扩展到上百台服务器,处理PB级结构化或非结构化数据
2.名词解释
index
类似于MySQL的数据库type
类似于MySQL的表document
类似于MySQL的行
3.简单入门
- 依赖
```xml
cn.hutool hutool-all 4.0.12 org.elasticsearch elasticsearch 6.2.4 org.elasticsearch.client transport 6.2.4 org.elasticsearch elasticsearch com.alibaba fastjson 1.2.47
2.配置文件1.application.properties```pr# Elasticsearch# 9200端口是用来让HTTP REST API来访问ElasticSearch,而9300端口是传输层监听的默认端口elasticsearch.ip=127.0.0.1elasticsearch.port=9300# 默认值是核心线程数*4elasticsearch.pool=5elasticsearch.cluster.name=qiudx_es_study
- es配置文件ElasticsearchConfig ```java
/**
- 用于定义配置类,可替换xml配置文件 *
- @author qiudx
@version $Id ElasticsearchConfig.java, v 0.1 2018-06-13 13:26 qiudx Exp $$ */ @Configuration public class ElasticsearchConfig {
private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchConfig.class);
/**
elk集群地址 */ @Value(“${elasticsearch.ip}”) private String hostName;
/**
端口 */ @Value(“${elasticsearch.port}”) private String port;
/**
集群名称 */ @Value(“${elasticsearch.cluster.name}”) private String clusterName;
/**
连接池 */ @Value(“${elasticsearch.pool}”) private String poolSize;
/**
- Bean name default 函数名字
*/
@Bean(name = “transportClient”)
public TransportClient transportClient() {
LOGGER.info(“Elasticsearch初始化开始。。。。。”);
TransportClient transportClient = null;
try {
} catch (Exception e) {// 配置信息Settings esSetting = Settings.builder()//集群名字.put("cluster.name", clusterName)//增加嗅探机制,找到ES集群.put("client.transport.sniff", true)//增加线程池个数,暂时设为5.put("thread_pool.search.size", Integer.parseInt(poolSize)).build();//配置信息Settings自定义transportClient = new PreBuiltTransportClient(esSetting);TransportAddress transportAddress = new TransportAddress(InetAddress.getByName(hostName), Integer.valueOf(port));transportClient.addTransportAddresses(transportAddress);
} return transportClient; }LOGGER.error("elasticsearch TransportClient create error!!", e);
}
3.工具类1.对es操作进行封装```java/*** @author qiudx* @version $Id ElasticsearchUtil.java, v 0.1 2018-06-13 13:29 qiudx Exp $$*/@Componentpublic class ElasticsearchUtil {private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchUtil.class);private final TransportClient transportClient;private static TransportClient client;@Autowiredpublic ElasticsearchUtil(TransportClient transportClient) {this.transportClient = transportClient;}/*** spring容器初始化的时候执行该方法*/@PostConstructpublic void init() {client = this.transportClient;}/*** 创建索引*/public static boolean createIndex(String index) {if (!isIndexExist(index)) {LOGGER.info("Index is not exits!");}CreateIndexResponse indexresponse = client.admin().indices().prepareCreate(index).execute().actionGet();LOGGER.info("执行建立成功?" + indexresponse.isAcknowledged());return indexresponse.isAcknowledged();}/*** 删除索引*/public static boolean deleteIndex(String index) {if (!isIndexExist(index)) {LOGGER.info("Index is not exits!");}DeleteIndexResponse dResponse = client.admin().indices().prepareDelete(index).execute().actionGet();if (dResponse.isAcknowledged()) {LOGGER.info("delete index " + index + " successfully!");} else {LOGGER.info("Fail to delete index " + index);}return dResponse.isAcknowledged();}/*** 判断索引是否存在*/public static boolean isIndexExist(String index) {IndicesExistsResponse inExistsResponse = client.admin().indices().exists(new IndicesExistsRequest(index)).actionGet();if (inExistsResponse.isExists()) {LOGGER.info("Index [" + index + "] is exist!");} else {LOGGER.info("Index [" + index + "] is not exist!");}return inExistsResponse.isExists();}/*** 数据添加,正定ID** @param jsonObject 要增加的数据* @param index 索引,类似数据库* @param type 类型,类似表* @param id 数据ID*/public static String addData(JSONObject jsonObject, String index, String type, String id) {IndexResponse response = client.prepareIndex(index, type, id).setSource(jsonObject).get();LOGGER.info("addData response status:{},id:{}", response.status().getStatus(), response.getId());return response.getId();}/*** 数据添加** @param jsonObject 要增加的数据* @param index 索引,类似数据库* @param type 类型,类似表*/public static String addData(JSONObject jsonObject, String index, String type) {return addData(jsonObject, index, type, UUID.randomUUID().toString().replaceAll("-", "").toUpperCase());}/*** 通过ID删除数据** @param index 索引,类似数据库* @param type 类型,类似表* @param id 数据ID*/public static void deleteDataById(String index, String type, String id) {DeleteResponse response = client.prepareDelete(index, type, id).execute().actionGet();LOGGER.info("deleteDataById response status:{},id:{}", response.status().getStatus(), response.getId());}/*** 通过ID 更新数据** @param jsonObject 要增加的数据* @param index 索引,类似数据库* @param type 类型,类似表* @param id 数据ID*/public static void updateDataById(JSONObject jsonObject, String index, String type, String id) {UpdateRequest updateRequest = new UpdateRequest();updateRequest.index(index).type(type).id(id).doc(jsonObject);client.update(updateRequest);}/*** 通过ID获取数据** @param index 索引,类似数据库* @param type 类型,类似表* @param id 数据ID* @param fields 需要显示的字段,逗号分隔(缺省为全部字段)*/public static Map<String, Object> searchDataById(String index, String type, String id, String fields) {GetRequestBuilder getRequestBuilder = client.prepareGet(index, type, id);if (StrUtil.isNotEmpty(fields)) {getRequestBuilder.setFetchSource(fields.split(","), null);}GetResponse getResponse = getRequestBuilder.execute().actionGet();return getResponse.getSource();}/*** 使用分词查询,并分页** @param index 索引名称* @param type 类型名称,可传入多个type逗号分隔* @param startPage 当前页* @param pageSize 每页显示条数* @param query 查询条件* @param fields 需要显示的字段,逗号分隔(缺省为全部字段)* @param sortField 排序字段* @param highlightField 高亮字段*/public static EsPage searchDataPage(String index, String type, int startPage, int pageSize, QueryBuilder query, String fields, String sortField, String highlightField) {SearchRequestBuilder searchRequestBuilder = client.prepareSearch(index);if (StrUtil.isNotEmpty(type)) {searchRequestBuilder.setTypes(type.split(","));}searchRequestBuilder.setSearchType(SearchType.QUERY_THEN_FETCH);// 需要显示的字段,逗号分隔(缺省为全部字段)if (StrUtil.isNotEmpty(fields)) {searchRequestBuilder.setFetchSource(fields.split(","), null);}//排序字段if (StrUtil.isNotEmpty(sortField)) {searchRequestBuilder.addSort(sortField, SortOrder.DESC);}// 高亮(xxx=111,aaa=222)if (StrUtil.isNotEmpty(highlightField)) {HighlightBuilder highlightBuilder = new HighlightBuilder();//highlightBuilder.preTags("<span style='color:red' >");//设置前缀//highlightBuilder.postTags("</span>");//设置后缀// 设置高亮字段highlightBuilder.field(highlightField);searchRequestBuilder.highlighter(highlightBuilder);}//searchRequestBuilder.setQuery(QueryBuilders.matchAllQuery());searchRequestBuilder.setQuery(query);// 分页应用searchRequestBuilder.setFrom(startPage).setSize(pageSize);// 设置是否按查询匹配度排序searchRequestBuilder.setExplain(true);//打印的内容 可以在 Elasticsearch head 和 Kibana 上执行查询LOGGER.info("\n{}", searchRequestBuilder);// 执行搜索,返回搜索响应信息SearchResponse searchResponse = searchRequestBuilder.execute().actionGet();long totalHits = searchResponse.getHits().totalHits;long length = searchResponse.getHits().getHits().length;LOGGER.debug("共查询到[{}]条数据,处理数据条数[{}]", totalHits, length);if (searchResponse.status().getStatus() == 200) {// 解析对象List<Map<String, Object>> sourceList = setSearchResponse(searchResponse, highlightField);return new EsPage(startPage, pageSize, (int) totalHits, sourceList);}return null;}/*** 使用分词查询** @param index 索引名称* @param type 类型名称,可传入多个type逗号分隔* @param query 查询条件* @param size 文档大小限制* @param fields 需要显示的字段,逗号分隔(缺省为全部字段)* @param sortField 排序字段* @param highlightField 高亮字段*/public static List<Map<String, Object>> searchListData(String index, String type, QueryBuilder query, Integer size, String fields, String sortField, String highlightField) {SearchRequestBuilder searchRequestBuilder = client.prepareSearch(index);if (StrUtil.isNotEmpty(type)) {searchRequestBuilder.setTypes(type.split(","));}if (StrUtil.isNotEmpty(highlightField)) {HighlightBuilder highlightBuilder = new HighlightBuilder();// 设置高亮字段highlightBuilder.field(highlightField);searchRequestBuilder.highlighter(highlightBuilder);}searchRequestBuilder.setQuery(query);if (StrUtil.isNotEmpty(fields)) {searchRequestBuilder.setFetchSource(fields.split(","), null);}searchRequestBuilder.setFetchSource(true);if (StrUtil.isNotEmpty(sortField)) {searchRequestBuilder.addSort(sortField, SortOrder.DESC);}if (size != null && size > 0) {searchRequestBuilder.setSize(size);}//打印的内容 可以在 Elasticsearch head 和 Kibana 上执行查询LOGGER.info("\n{}", searchRequestBuilder);SearchResponse searchResponse = searchRequestBuilder.execute().actionGet();long totalHits = searchResponse.getHits().totalHits;long length = searchResponse.getHits().getHits().length;LOGGER.info("共查询到[{}]条数据,处理数据条数[{}]", totalHits, length);if (searchResponse.status().getStatus() == 200) {// 解析对象return setSearchResponse(searchResponse, highlightField);}return null;}/*** 高亮结果集 特殊处理*/private static List<Map<String, Object>> setSearchResponse(SearchResponse searchResponse, String highlightField) {List<Map<String, Object>> sourceList = new ArrayList<>();StringBuilder stringBuffer = new StringBuilder();for (SearchHit searchHit : searchResponse.getHits().getHits()) {searchHit.getSourceAsMap().put("id", searchHit.getId());if (StrUtil.isNotEmpty(highlightField)) {System.out.println("遍历 高亮结果集,覆盖 正常结果集" + searchHit.getSourceAsMap());Text[] text = searchHit.getHighlightFields().get(highlightField).getFragments();if (text != null) {for (Text str : text) {stringBuffer.append(str.string());}//遍历 高亮结果集,覆盖 正常结果集searchHit.getSourceAsMap().put(highlightField, stringBuffer.toString());}}sourceList.add(searchHit.getSourceAsMap());}return sourceList;}}
分页工具类 ```java /**
- @author qiudx
- @version $Id EsPage.java, v 0.1 2018-06-13 13:32 qiudx Exp $$ */ public class EsPage {
/**
- 当前页 / private int currentPage; /*
- 每页显示多少条 */ private int pageSize;
/**
- 总记录数 / private int recordCount; /*
- 本页的数据列表 */ private List
/**
- 总页数 / private int pageCount; /*
- 页码列表的开始索引(包含) / private int beginPageIndex; /*
- 页码列表的结束索引(包含) */ private int endPageIndex;
/**
只接受前4个必要的属性,会自动的计算出其他3个属性的值 */ public EsPage(int currentPage, int pageSize, int recordCount, List
// 计算总页码 pageCount = (recordCount + pageSize - 1) / pageSize;
// 计算 beginPageIndex 和 endPageIndex // >> 总页数不多于10页,则全部显示 if (pageCount <= 10) {
beginPageIndex = 1;endPageIndex = pageCount;
} // >> 总页数多于10页,则显示当前页附近的共10个页码 else { // 当前页附近的共10个页码(前4个 + 当前页 + 后5个)
beginPageIndex = currentPage - 4;endPageIndex = currentPage + 5;// 当前面的页码不足4个时,则显示前10个页码if (beginPageIndex < 1) {beginPageIndex = 1;endPageIndex = 10;}
// 当后面的页码不足5个时,则显示后10个页码
if (endPageIndex > pageCount) {endPageIndex = pageCount;beginPageIndex = pageCount - 10 + 1;}
} }
public int getCurrentPage() { return currentPage; }
public void setCurrentPage(int currentPage) { this.currentPage = currentPage; }
public int getPageSize() { return pageSize; }
public void setPageSize(int pageSize) { this.pageSize = pageSize; }
public int getRecordCount() { return recordCount; }
public void setRecordCount(int recordCount) { this.recordCount = recordCount; }
public List
public void setRecordList(List
public int getPageCount() { return pageCount; }
public void setPageCount(int pageCount) { this.pageCount = pageCount; }
public int getBeginPageIndex() { return beginPageIndex; }
public void setBeginPageIndex(int beginPageIndex) { this.beginPageIndex = beginPageIndex; }
public int getEndPageIndex() { return endPageIndex; }
public void setEndPageIndex(int endPageIndex) { this.endPageIndex = endPageIndex; }
}
4.实体```java/*** @author qiudx* @version $Id EsModel.java, v 0.1 2018-06-13 13:40 qiudx Exp $$*/public class EsModel {private String id;private String name;private int age;private Date date;/*** Getter method for property <tt>id</tt>.** @return property value of id*/public String getId() {return id;}/*** Setter method for property <tt>id</tt>.** @param id value to be assigned to property id*/public void setId(String id) {this.id = id;}/*** Getter method for property <tt>name</tt>.** @return property value of name*/public String getName() {return name;}/*** Setter method for property <tt>name</tt>.** @param name value to be assigned to property name*/public void setName(String name) {this.name = name;}/*** Getter method for property <tt>age</tt>.** @return property value of age*/public int getAge() {return age;}/*** Setter method for property <tt>age</tt>.** @param age value to be assigned to property age*/public void setAge(int age) {this.age = age;}/*** Getter method for property <tt>date</tt>.** @return property value of date*/public Date getDate() {return date;}/*** Setter method for property <tt>date</tt>.** @param date value to be assigned to property date*/public void setDate(Date date) {this.date = date;}}
测试
/*** @author qiudx* @version $Id EsController.java, v 0.1 2018-06-13 13:36 qiudx Exp $$*/@RestController@EnableAutoConfiguration@RequestMapping("/es")public class EsController {/*** 测试索引*/private String indexName = "test_index";/*** 类型*/private String esType = "external";/*** http://127.0.0.1:8080/es/createIndex* 创建索引*/@RequestMapping("/createIndex")public String createIndex() {if (!ElasticsearchUtil.isIndexExist(indexName)) {ElasticsearchUtil.createIndex(indexName);} else {return "索引已经存在";}return "索引创建成功";}/*** 插入记录*/@RequestMapping("/insertJson")public String insertJson() {JSONObject jsonObject = new JSONObject();jsonObject.put("id", DateUtil.formatDate(new Date()));jsonObject.put("age", 25);jsonObject.put("name", "j-" + new Random(100).nextInt());jsonObject.put("date", new Date());return ElasticsearchUtil.addData(jsonObject, indexName, esType, jsonObject.getString("id"));}/*** 插入记录*/@RequestMapping("/insertModel")public String insertModel() {EsModel esModel = new EsModel();esModel.setId(DateUtil.formatDate(new Date()));esModel.setName("m-" + new Random(100).nextInt());esModel.setAge(30);esModel.setDate(new Date());JSONObject jsonObject = (JSONObject) JSONObject.toJSON(esModel);return ElasticsearchUtil.addData(jsonObject, indexName, esType, jsonObject.getString("id"));}/*** 删除记录*/@RequestMapping("/delete")public String delete(String id) {if (StrUtil.isNotBlank(id)) {ElasticsearchUtil.deleteDataById(indexName, esType, id);return "删除id=" + id;} else {return "id为空";}}/*** 更新数据*/@RequestMapping("/update")public String update(String id) {if (StrUtil.isNotBlank(id)) {JSONObject jsonObject = new JSONObject();jsonObject.put("id", id);jsonObject.put("age", 31);jsonObject.put("name", "修改");jsonObject.put("date", new Date());ElasticsearchUtil.updateDataById(jsonObject, indexName, esType, id);return "id=" + id;} else {return "id为空";}}/*** 获取数据* http://127.0.0.1:8080/es/getData?id=2018-04-25%2016:33:44*/@RequestMapping("/getData")public String getData(String id) {if (StrUtil.isNotBlank(id)) {Map<String, Object> map = ElasticsearchUtil.searchDataById(indexName, esType, id, null);return JSONObject.toJSONString(map);} else {return "id为空";}}/*** 查询数据* 模糊查询*/@RequestMapping("/queryMatchData")@ResponseBodypublic String queryMatchData() {BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();boolQuery.must(QueryBuilders.matchQuery("name", "修"));List<Map<String, Object>> list = ElasticsearchUtil.searchListData(indexName, esType, boolQuery, 10, null, null, null);return JSONObject.toJSONString(list);}/*** 通配符查询数据* 通配符查询 ?用来匹配1个任意字符,*用来匹配零个或者多个字符*/@RequestMapping("/queryWildcardData")public String queryWildcardData() {QueryBuilder queryBuilder = QueryBuilders.wildcardQuery("name.keyword", "修*");List<Map<String, Object>> list = ElasticsearchUtil.searchListData(indexName, esType, queryBuilder, 10, null, null, null);return JSONObject.toJSONString(list);}/*** 正则查询*/@RequestMapping("/queryRegexpData")public String queryRegexpData() {QueryBuilder queryBuilder = QueryBuilders.regexpQuery("name.keyword", "j--[0-9]{1,11}");List<Map<String, Object>> list = ElasticsearchUtil.searchListData(indexName, esType, queryBuilder, 10, null, null, null);return JSONObject.toJSONString(list);}/*** 查询数字范围数据*/@RequestMapping("/queryIntRangeData")public String queryIntRangeData() {BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();boolQuery.must(QueryBuilders.rangeQuery("age").from(31).to(32));List<Map<String, Object>> list = ElasticsearchUtil.searchListData(indexName, esType, boolQuery, 10, null, null, null);return JSONObject.toJSONString(list);}/*** 查询日期范围数据*/@RequestMapping("/queryDateRangeData")public String queryDateRangeData() {BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();boolQuery.must(QueryBuilders.rangeQuery("date").from("2018-04-25T08:33:44.840Z").to("2018-04-25T10:03:08.081Z"));List<Map<String, Object>> list = ElasticsearchUtil.searchListData(indexName, esType, boolQuery, 10, null, null, null);return JSONObject.toJSONString(list);}/*** 查询分页** @param startPage 第几条记录开始* 从0开始* 第1页 :http://127.0.0.1:8080/es/queryPage?startPage=0&pageSize=2* 第2页 :http://127.0.0.1:8080/es/queryPage?startPage=2&pageSize=2* @param pageSize 每页大小*/@RequestMapping("/queryPage")public String queryPage(String startPage, String pageSize) {if (StrUtil.isNotBlank(startPage) && StrUtil.isNotBlank(pageSize)) {BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();boolQuery.must(QueryBuilders.rangeQuery("date").from("2018-04-25T08:33:44.840Z").to("2018-04-25T10:03:08.081Z"));EsPage list = ElasticsearchUtil.searchDataPage(indexName, esType, Integer.parseInt(startPage), Integer.parseInt(pageSize), boolQuery, null, null, null);return JSONObject.toJSONString(list);} else {return "startPage或者pageSize缺失";}}}
