LogStash7.11.1

1、Point

  1. input {
  2.   jdbc {
  3.     jdbc_driver_library => "D:/elk/logstash-7.11.1/test/postgresql-42.2.19.jar"
  4.     jdbc_driver_class => "org.postgresql.Driver"
  5.     jdbc_connection_string => "jdbc:postgresql://localhost:5432/waterway?characterEncoding=utf8&serverTimezone=GMT%2B8"
  6.     jdbc_user => "postgres"
  7.     jdbc_password => "953598751"
  8.     schedule => "*/5 * * * * *"
  9.     statement => "select gid id,ST_AsGeoJSON(ST_Force2D(geom)) geojson,name from polylines where gid = 134"
  10.     tracking_column => "id"
  11.     tracking_column_type => "numeric"
  12.     id => "geotest"
  13.     use_column_value => true
  14.     jdbc_default_timezone =>"Asia/Shanghai"
  15.   }
  16. }
  17. filter {
  18.   json {
  19.     source => "geojson"
  20.   }
  21.   mutate {
  22.      add_field => { "location" => "%{[coordinates][1]},%{[coordinates][0]}"}
  23.    }
  24.   mutate {
  25.      remove_field => ["@version","geojson","coordinates","@timestamp"]
  26.   }
  27. }
  28. output {
  29.   stdout {
  30.     codec => rubydebug
  31.   }
  32.   # elasticsearch {
  33.   #   hosts => ["localhost:9200"]
  34.   #   index => "example"
  35.   #   document_id => "%{id}"
  36.   #   user => "elastic"
  37.   #   password => "953598751"
  38.   # }
  39. }
  42. ############ ES ######
  43. PUT index_point
  44. {
  45.   "mappings": {
  46.     "properties": {
  47.       "location": {
  48.         "type": "geo_point"
  49.       }
  50.     }
  51.   }
  52. }

2、通用

  1. input {
  2.   jdbc {
  3.     jdbc_driver_library => "D:/elk/logstash-7.11.1/test/postgresql-42.2.19.jar"
  4.     jdbc_driver_class => "org.postgresql.Driver"
  5.     jdbc_connection_string => "jdbc:postgresql://localhost:5432/waterway?characterEncoding=utf8&serverTimezone=GMT%2B8"
  6.     jdbc_user => "postgres"
  7.     jdbc_password => "953598751"
  8.     schedule => "*/5 * * * * *"
  9.     statement => "select gid id,st_astext(ST_Force2D(geom)) locations,name from polylines where gid = 134"
  10.     tracking_column => "id"
  11.     tracking_column_type => "numeric"
  12.     id => "geotest"
  13.     use_column_value => true
  14.     jdbc_default_timezone =>"Asia/Shanghai"
  15.     type => "polylines"
  16.   }
  17. }
  18. filter {
  19.   # json {
  20.   #   source => "geojson"
  21.   # }
  22.   # mutate {
  23.   #   add_field => { "location" => "" }
  24.   # }
  25.   mutate {
  26.     remove_field => ["@version","@timestamp"]
  27.   }
  28. }
  29. output {
  30.   stdout {
  31.     codec => rubydebug
  32.   }
  33.   # elasticsearch {
  34.   #   hosts => ["localhost:9200"]
  35.   #   index => "example"
  36.   #   document_id => "%{id}"
  37.   #   user => "elastic"
  38.   #   password => "953598751"
  39.   # }
  40. }

3、Output ES Template

  • order：表示模板的优先级，当一个索引匹配到多个模板时，order值越大匹配优先级越高。order值默认为0

  • settings： 指定index的配置信息, 比如分片数、副本数, tranlog同步条件、refresh策略等信息; | 属性 | 说明 | | :—- | —- | | number_of_replicas | 副本数 | | number_of_shards | 分片数 | | index.refresh_interval | fsync频率 |

  • mappings: 指定index的内部构建信息, 主要有

    • _all: All Field字段, 如果开启, _all字段就会把所有字段的内容都包含进来,检索的时候可以不用指定字段查询 —— 会检索多个字段, 设置方式: "_all": {"enabled": true},在ES 6.0开始, _all字段被禁用了, 作为替换, 可以通过copy_to自定义实现all字段的功能

    • _source: Source Field字段, ES为每个文档都保存一份源数据, 如果不开启, 也就是"_source": {"enabled": false}, 查询的时候就只会返回文档的ID, 其他的文档内容需要通过Fields字段到索引中再次获取, 效率很低. 但若开启, 索引的体积会更大, 此时就可以通过Compress进行压缩, 并通过inclueds、excludes等方式在field上进行限制 —— 指定义允许哪些字段存储到_source中, 哪些不存储

    • _default_ 文档类型，可以根据实际情况自定义。表示对应文档类型下field的数据类型。如果想要一个索引中的所有文档类型都使用这个mappings配置，名称可配置为 _default_

    • dynamic 在索引文档时，对于不在mapping中的field，elastcisearch会使用什么策略进行动态映射，可以通过这个属性进行配置，只可以使用elastcisearch中约定的值，如下 | dynamic | 说明 | | —- | —- | | true | 在索引一个文档时，如果文档中有field不在mapping中，会自动映射类型，添加到mapping，并索引该字段 | | false | 在索引一个文档时，如果文档中有field不在mapping中，不会添加到mapping，也不会索引该字段，但是会保存下来，在_source可以看到该字段，但该字段不能被搜索 | | strict | 在索引一个文档时，如果文档中有field不在mapping中，会直接抛出异常，拒绝索引 |

3.1静态模板

  1. ########## template.json ##########
  2. {
  3.     "index_patterns": "index_*",
  4.     "order" : 1000,
  5.     "settings": {
  6.         "number_of_shards": "5",
  7.         "number_of_replicas": "0"
  8.     },
  9.     "mappings": {
  10.         "properties": {
  11.             "locations": {
  12.                 "type": "geo_shape"
  13.             },
  14.             "id": {
  15.                 "type": "keyword"
  16.             },
  17.             "type": {
  18.                 "type": "text"
  19.             },
  20.             "name": {
  21.                 "type": "text",
  22.                 "analyzer": "ik_max_word"
  23.             },
  24.             "username": {
  25.                 "type": "text",
  26.                 "analyzer": "ik_max_word"
  27.             }
  28.         }
  29.     },
  30.     "aliases": {}
  31. }
  34. ########## config.conf ##########
  35. output {
  36.   stdout {
  37.     codec => rubydebug
  38.   }
  39.   elasticsearch {
  40.     hosts => ["localhost:9200"]
  41.     index => "index_my"
  42.     document_id => "%{id}"
  43.     template =>"D:/elk/logstash-7.11.1/test/template/template.json"
  44.     template_name => "index_my"
  45.     template_overwrite => true
  46.     user => "elastic"
  47.     password => "953598751"
  48.   }
  49. }

3.2 拼音分词和IK分词的结合

  1. ########## template.json ##########
  2. {
  3.     "index_patterns": "index_*",
  4.     "order": 1000,
  5.     "settings": {
  6.         "number_of_shards": "5",
  7.         "number_of_replicas": "0",
  8.         "analysis": {
  9.             "analyzer": {
  10.                 "my_analyzer": {
  11.                     "type": "custom",
  12.                     "tokenizer": "ik_max_word",
  13.                     "filter": [
  14.                         "lowercase",
  15.                         "my_pinyin"
  16.                     ]
  17.                 }
  18.             },
  19.             "filter": {
  20.                 "my_pinyin": {
  21.                     "type": "pinyin",
  22.                     "keep_separate_first_letter": true,
  23.                     "keep_full_pinyin": true,
  24.                     "keep_first_letter":true,
  25.                     "keep_original": false,
  26.                     "limit_first_letter_length": 10,
  27.                     "lowercase": true,
  28.                     "remove_duplicated_term": true
  29.                 }
  30.             }
  31.         }
  32.     },
  33.     "mappings": {
  34.         "_source": {
  35.             "enabled": true
  36.         },
  37.         "properties": {
  38.             "locations": {
  39.                 "type": "geo_shape"
  40.             },
  41.             "id": {
  42.                 "type": "keyword"
  43.             },
  44.             "type": {
  45.                 "type": "text"
  46.             },
  47.             "name": {
  48.                 "type": "text",
  49.                 "analyzer": "my_analyzer"
  50.             }
  51.         }
  52.     },
  53.     "aliases": {}
  54. }
  55. ########## config.conf ##########
  56. output {
  57.   stdout {
  58.     codec => rubydebug
  59.   }
  60.   elasticsearch {
  61.     hosts => ["localhost:9200"]
  62.     index => "index_my"
  63.     document_id => "%{id}"
  64.     template =>"D:/elk/logstash-7.11.1/test/template/template.json"
  65.     template_name => "index_my"
  66.     template_overwrite => true
  67.     user => "elastic"
  68.     password => "953598751"
  69.   }
  70. }