elk 单机版 docker 快速安装
ik 分词插件
kibana dev tools 常用操作
ES 7.6 java API
问题
- 1. The mapping definition cannot be nested under a type [_doc]

elk 单机版 docker 快速安装

Docker Hub官网：https://hub.docker.com/r/sebp/elk/
Docker ELK使用文档：http://elk-docker.readthedocs.io/

Docker至少得分配3GB的内存；
Elasticsearch至少需要单独2G的内存；

# 1. 修改内核参数
vim /etc/sysctl.conf
  vm.max_map_count = 655360
  vm.swappiness = 1
# 2. 安装
docker run -p 5601:5601 -p 9200:9200 -p 5044:5044 \
-e ES_MIN_MEM=128m  \
-e ES_MAX_MEM=1024m \
-v /data/es/plugins/:/opt/elasticsearch/plugins/  \
-it -d --name elk sebp/elk

ik 分词插件

IK分词器github地址: https://github.com/medcl/elasticsearch-analysis-ik

kibana dev tools 常用操作

#########  查看原数据信息  ##########
# 查看集群的健康状态
GET _cat/health?v
# 查看集群的 index
GET _cat/indices?v
GET _cat/plugins?v
##### ik 分词测试 ###
GET _analyze
{
  "analyzer": "ik_max_word"
  , "text": ["学习近平平安归来"]
}
#########   数据的 PUT, POST, GET   ###############
DELETE student
PUT student
{
  "mappings": {
    "properties": {
      "name": {
        "type": "keyword"
      },
      "age": {
        "type": "integer"
      },
      "sex": {
        "type": "keyword"
      },
      "favo": {
        "type": "text",
        "analyzer": "ik_max_word"
      }
    }
  }
}
# PUT 操作
PUT student/_doc/001
{
  "name": "ylb",
  "age": "99",
  "sex": "男",
  "favo": "学习近平"
}
PUT student/_doc/002
{
  "name": "y002",
  "age": "12",
  "sex": "女",
  "favo": "吃饭睡觉打豆豆..."
}
PUT student/_doc/003
{
  "name": "y003",
  "age": "13",
  "sex": "女",
  "favo": "学习旅游行走"
}
PUT student/_doc/004
{
  "name": "y004",
  "age": "13",
  "sex": "男",
  "favo": "学习旅行跑"
}
###   GET 查看 student 中的数据
GET student/_search
# POST 操作
POST student/_doc
{
  "name": "p004",
  "age": "14",
  "sex": "女",
  "favo": "敲代码"
}
#########   检索    ############
# 全文档检索
GET student/_search
# 
GET student/_search
{
  "query": {
    "bool": {
      "filter": {
        "term": {
          "favo": "学习旅行"
        }
      }
    }
  }
}
# match:  where ... or ...
GET student/_search
{
  "query": {
    "match": {
      "favo": "学习旅行"
    }
  }
}
# match:  where ... and ...
GET student/_search
{
  "query": {
    "match": {
      "favo": {
        "query": "学习旅行",
        "operator": "and"
      }
    }
  }
}
# 多条件匹配:   filter ... must ...
GET student/_search
{
  "query": {
    "bool": {
      "filter": {
        "term": {
          "sex": "男"
        }
      },
      "must": [
        {
          "match": {
            "favo": "学习"
          }
        },
        {
          "match": {
            "age": "13"
          }
        }
      ]
    }
  }
}
# 模糊匹配:  fuzzy  ,主要用来匹配英文
GET student/_search
{
  "query": {
    "fuzzy": {
      "name": {
        "value": "y00"
      }
    }
  },
  "from": 0,
  "size": 2
}
# 聚合操作:  aggs
GET student/_search
{
  "aggs": {
    "group_by_sex": {
      "terms": {
        "field": "sex",
        "size": 2
      }
    },
    "group_age":{
      "max": {
        "field": "age"
      }
    }
  }
}
# 分页显示操作:   from
## 显示第0页的2条数据
GET student/_search
{
  "from": 0,
  "size": 2
}
## 综合练习 ##
## 统计 "学习" 女 有多少人,最大年龄的是多少
GET student/_search
{
  "query": {
    "bool": {
      "filter": {
        "term": {
          "favo": "学习"
        }
      },
      "must": [
        {"match": {
          "sex": "男"
        }}
      ]
    }
  },
  "aggs": {
    "group_age": {
      "max": {
        "field": "age"
      }
    }
  },
  "from": 0,
  "size": 20
}

ES 7.6 java API

先决条件

## 在 kibana dev tools 中 执行如下:
DELETE test01
PUT test01
{
  "mappings": {
    "properties": {
      "name": {
        "type": "keyword"
      },
      "age": {
        "type": "integer"
      }
    }
  }
}

es 依赖

                <!-- ES 依赖  -->
                <dependency>
            <groupId>org.elasticsearch.client</groupId>
            <artifactId>elasticsearch-rest-client</artifactId>
            <version>7.6.0</version>
        </dependency>
        <dependency>
            <groupId>org.elasticsearch.client</groupId>
            <artifactId>elasticsearch-rest-high-level-client</artifactId>
            <version>7.6.0</version>
        </dependency>
        <dependency>
            <groupId>org.elasticsearch</groupId>
            <artifactId>elasticsearch</artifactId>
            <version>7.6.0</version>
        </dependency>
<!-- maven 打包依赖  -->
<build>
    <plugins>
        <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-shade-plugin</artifactId>
            <version>3.1.0</version>
            <executions>
                <execution>
                    <phase>package</phase>
                    <goals><goal>shade</goal></goals>
                    <configuration>
                        <relocations>
                            <relocation>
                                <pattern>org.apache.http</pattern>
                                <shadedPattern>hidden.org.apache.http</shadedPattern>
                            </relocation>
                            <relocation>
                                <pattern>org.apache.logging</pattern>
                                <shadedPattern>hidden.org.apache.logging</shadedPattern>
                            </relocation>
                            <relocation>
                                <pattern>org.apache.commons.codec</pattern>
                                <shadedPattern>hidden.org.apache.commons.codec</shadedPattern>
                            </relocation>
                            <relocation>
                                <pattern>org.apache.commons.logging</pattern>
                                <shadedPattern>hidden.org.apache.commons.logging</shadedPattern>
                            </relocation>
                        </relocations>
                    </configuration>
                </execution>
            </executions>
        </plugin>
    </plugins>
</build>

Java API 操作

官方文档: https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/java-rest-high-document-index.html

Put (Index)

     /**
     * 向Index 中 put 数据
     * @param hostname
     * @param port
     * @param scheme
     * @param index
     * @throws IOException
     */
    public static void PutIndex(String hostname, Integer port, String scheme, String index) throws IOException {
        // 创建 client
        RestHighLevelClient client = new RestHighLevelClient(
                RestClient.builder(
                        new HttpHost(hostname,port,scheme)
                ));
// ----------------     方法4     --------------------
//        IndexRequest indexRequest = new IndexRequest(index)
//                .id("005")
//                .source("name", "qq003",
//                        "age", 24
//                );
// ---------------- 方法3  ----------------------------
//        XContentBuilder builder = XContentFactory.jsonBuilder();
//        builder.startObject();
//        {
//            builder.field("name", "W003");
//            builder.field("age", 23);
//        }
//        builder.endObject();
//        IndexRequest indexRequest = new IndexRequest(index)
//                .id("004").source(builder);
// ---------------- 方法2  ----------------------------
//        Map<String, Object> jsonMap = new HashMap<>();
//        jsonMap.put("name", "W001");
//        jsonMap.put("age", 22);
//
//        IndexRequest indexRequest = new IndexRequest(index)
//                .id("002").source(jsonMap);
// -----------------    方法1     ----------------------
        IndexRequest request = new IndexRequest(index);
        request.id("003");
        String jsonString = "{\n" +
                "  \"name\": \"N001\",\n" +
                "  \"age\": \"12\"\n" +
                "}";
        IndexRequest indexRequest = request.source(jsonString, XContentType.JSON);
        // 执行
        client.index(indexRequest,RequestOptions.DEFAULT);
        // close client
        client.close();
    }

Bulk

    /**
     *
     * @param hostname
     * @param port
     * @param scheme
     * @param index
     * @throws IOException
     */
    public static void Bulk(String hostname, Integer port, String scheme, String index) throws IOException {
        List<test01> list = new ArrayList<test01>();
        list.add(new test01(010,"aa01",111));
        list.add(new test01(011,"bb02", 222));
        // 创建 client
        RestHighLevelClient client = new RestHighLevelClient(
                RestClient.builder(new HttpHost(hostname, port, scheme))
        );
        // bulk request
        BulkRequest bulkRequest = new BulkRequest();
        for (test01 test01 : list) {
            // 将 test01对象 转为 json 对象
            String jsonString = JSON.toJSONString(test01);
            bulkRequest.add(new IndexRequest(index).id(test01.getId().toString()).source(jsonString,XContentType.JSON));
        }
        // 执行
        client.bulk(bulkRequest,RequestOptions.DEFAULT);
        // client close
        client.close();
    }

Get ( index for id)

   /**
     * @function  Get Index 中的　某个　id 的数据
     * @param hostname
     * @param port
     * @param scheme
     * @param index
     * @return
     * @throws IOException
     */
    public static GetResponse GetIndex(String hostname, Integer port, String scheme, String index) throws IOException {
        // 创建　client
        final RestHighLevelClient client = new RestHighLevelClient(
                RestClient.builder(new HttpHost(hostname, port, scheme)));
        GetRequest getRequest = new GetRequest(
                index,
                "003");
        // 执行
        GetResponse getResponse = client.get(getRequest, RequestOptions.DEFAULT);
        // client close
        client.close();
        return getResponse;
    }

search ( 高级搜索 )

dev tools 执行如下:

# 统计 "学习" 女 有多少人,最大年龄的是多少
GET student/_search
{
  "query": {
    "bool": {
      "filter": {
        "term": {
          "favo": "学习"
        }
      },
      "must": [
        {"match": {
          "sex": "男"
        }}
      ]
    }
  },
  "aggs": {
    "count_by_name": {
      "terms": {
        "field": "name",
        "size": 10
      }
    },
    "group_age": {
      "max": {
        "field": "age"
      }
    }
  },
  "from": 0,
  "size": 20
}

java API 代码

    public static void  Search(String hostname, Integer port, String scheme, String index) throws IOException {
        // 创建　client
        final RestHighLevelClient client = new RestHighLevelClient(
                RestClient.builder(new HttpHost(hostname, port, scheme)));
        final SearchRequest searchRequest = new SearchRequest(index);
        final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        // 查询条件
        // query
        BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
        // filter
        boolQueryBuilder.filter(new TermQueryBuilder("favo","学习"));
        // must
        boolQueryBuilder.must(new MatchQueryBuilder("sex", "男"));
        searchSourceBuilder.query(boolQueryBuilder);
        // aggs
        TermsAggregationBuilder groupName = new TermsAggregationBuilder("groupName", ValueType.LONG);
        groupName.field("name");
        groupName.size(20);
        searchSourceBuilder.aggregation(groupName);
        MaxAggregationBuilder maxAge = new MaxAggregationBuilder("maxAge");
        maxAge.field("age");
        searchSourceBuilder.aggregation(maxAge);
        // 分页
        searchSourceBuilder.from(0);
        searchSourceBuilder.size(2);
        searchRequest.source(searchSourceBuilder);
        // 执行
        SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
        // 打印结果
        System.out.println("----------  原始数据  -------------");
        System.out.println(searchResponse);
        TotalHits hits = searchResponse.getHits().getTotalHits();
        float maxScore = searchResponse.getHits().getMaxScore();
        final SearchHit[] searchHits = searchResponse.getHits().getHits();
        // 遍历 hits 的数据
        System.out.println("-----------     遍历 hits 的数据 ----------------");
        for (SearchHit searchHit : searchHits) {
            final Map<String, Object> sourceAsMap = searchHit.getSourceAsMap();
            final JSONObject jsonObject = new JSONObject();
            for (Object o: sourceAsMap.keySet()) {
                jsonObject.put((String) o, sourceAsMap.get(o));
            }
            jsonObject.put("index",searchHit.getIndex());
            jsonObject.put("id", searchHit.getId());
            System.out.println(jsonObject.toString());
        }
        // 解析聚合数据
        Aggregations aggregation = searchResponse.getAggregations();
        // 获取最大年龄
//        Aggregation ma = aggregation.get("maxAge");
//        System.out.println(ma);    // 先看下 ma 的类型, 下面再定义类型
        ParsedMax MaxAge = aggregation.get("maxAge");
        // 获取班级分组
        System.out.println("---- 班级分组 ----");
        ParsedStringTerms groupN = aggregation.get("groupName");
        List<? extends Terms.Bucket> buckets = groupN.getBuckets();
        for (Terms.Bucket bucket : buckets) {
            System.out.println(bucket.getKey() + " ---> " + bucket.getDocCount());
        }
        System.out.printf("共查询出 %s 条数据,\n最高分数 %s ,\n最大年龄为 %s ,\n",
                hits,maxScore,MaxAge.getValue());
        // client close
        client.close();
    }

问题

1. The mapping definition cannot be nested under a type [_doc]

在 ES6.X 中执行正常, 在7.x 中有错误。

PUT student
{
  "mappings": {
    "_doc": {
      "properties": {
        "name": {
          "type": "keyword"
        },
        "age": {
          "type": "integer"
        },
        "sex": {
          "type": "keyword"
        },
        "favo": {
          "type": "text",
          "analyzer": "ik_max_word"
        }
      }
    }
  }
}

报错如下:

{
  "error" : {
    "root_cause" : [
      {
        "type" : "illegal_argument_exception",
        "reason" : "The mapping definition cannot be nested under a type [_doc] unless include_type_name is set to true."
      }
    ],
    "type" : "illegal_argument_exception",
    "reason" : "The mapping definition cannot be nested under a type [_doc] unless include_type_name is set to true."
  },
  "status" : 400
}

解决方案:
默认不在支持指定索引类型。
尽管官方说, include_type_name is set to true , 但是这里不建议使用, 因为 ES8.x 中将移除。
把 _doc 类型 去掉即可。

devops

docker install ELK