https://www.elastic.co/guide/en/beats/filebeat/current/processor-script.html

    1. ###################### Filebeat Configuration Example #########################
    3. # This file is an example configuration file highlighting only the most common
    4. # options. The filebeat.reference.yml file from the same directory contains all the
    5. # supported options with more comments. You can use it as a reference.
    6. #
    7. # You can find the full configuration reference here:
    8. # https://www.elastic.co/guide/en/beats/filebeat/index.html
    10. # For more available modules and options, please see the filebeat.reference.yml sample
    11. # configuration file.
    13. #=========================== Filebeat inputs =============================
    15. filebeat.inputs:
    17. # Each - is an input. Most options can be set at the input level, so
    18. # you can use different inputs for various configurations.
    19. # Below are the input specific configurations.
    21. - type: log
    23.   # Change to true to enable this input configuration.
    24.   enabled: false
    26.   # Paths that should be crawled and fetched. Glob based paths.
    27.   paths:
    28.     - /var/log/*.log
    29.     #- c:\programdata\elasticsearch\logs\*
    31.   # Exclude lines. A list of regular expressions to match. It drops the lines that are
    32.   # matching any regular expression from the list.
    33.   #exclude_lines: ['^DBG']
    35.   # Include lines. A list of regular expressions to match. It exports the lines that are
    36.   # matching any regular expression from the list.
    37.   #include_lines: ['^ERR', '^WARN']
    39.   # Exclude files. A list of regular expressions to match. Filebeat drops the files that
    40.   # are matching any regular expression from the list. By default, no files are dropped.
    41.   #exclude_files: ['.gz$']
    43.   # Optional additional fields. These fields can be freely picked
    44.   # to add additional information to the crawled log files for filtering
    45.   #fields:
    46.   #  level: debug
    47.   #  review: 1
    49.   ### Multiline options
    51.   # Multiline can be used for log messages spanning multiple lines. This is common
    52.   # for Java Stack Traces or C-Line Continuation
    54.   # The regexp Pattern that has to be matched. The example pattern matches all lines starting with [
    55.   #multiline.pattern: ^\[
    57.   # Defines if the pattern set under pattern should be negated or not. Default is false.
    58.   #multiline.negate: false
    60.   # Match can be set to "after" or "before". It is used to define if lines should be append to a pattern
    61.   # that was (not) matched before or after or as long as a pattern is not matched based on negate.
    62.   # Note: After is the equivalent to previous and before is the equivalent to to next in Logstash
    63.   #multiline.match: after
    66. #多行
    67. - type: log
    68.   enabled: true
    69.   paths:
    70.     - /logs/cave2/cave2_all_*/*.log
    71.   multiline.pattern: '^\[[0-9][0-9]'
    72.   multiline.negate: true
    73.   multiline.max_lines: 20000
    74.   multiline.match: after
    75.   multiline.timeout: 300s
    76.   fields:
    77.     AvalonIndexKey: co-ap-server
    80. #============================= Filebeat modules ===============================
    82. filebeat.config.modules:
    83.   # Glob pattern for configuration loading
    84.   path: ${path.config}/modules.d/*.yml
    86.   # Set to true to enable config reloading
    87.   reload.enabled: false
    89.   # Period on which files under path should be checked for changes
    90.   #reload.period: 10s
    92. #==================== Elasticsearch template setting ==========================
    94. setup.template.settings:
    95.   index.number_of_shards: 1
    96.   #index.codec: best_compression
    97.   #_source.enabled: false
    99. #================================ General =====================================
    101. # The name of the shipper that publishes the network data. It can be used to group
    102. # all the transactions sent by a single shipper in the web interface.
    103. #name:
    105. # The tags of the shipper are included in their own field with each
    106. # transaction published.
    107. #tags: ["service-X", "web-tier"]
    109. # Optional fields that you can specify to add additional information to the
    110. # output.
    111. #fields:
    112. #  env: staging
    115. #============================== Dashboards =====================================
    116. # These settings control loading the sample dashboards to the Kibana index. Loading
    117. # the dashboards is disabled by default and can be enabled either by setting the
    118. # options here or by using the `setup` command.
    119. #setup.dashboards.enabled: false
    121. # The URL from where to download the dashboards archive. By default this URL
    122. # has a value which is computed based on the Beat name and version. For released
    123. # versions, this URL points to the dashboard archive on the artifacts.elastic.co
    124. # website.
    125. #setup.dashboards.url:
    127. #============================== Kibana =====================================
    129. # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API.
    130. # This requires a Kibana endpoint configuration.
    131. setup.kibana:
    133.   # Kibana Host
    134.   # Scheme and port can be left out and will be set to the default (http and 5601)
    135.   # In case you specify and additional path, the scheme is required: http://localhost:5601/path
    136.   # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
    137.   #host: "localhost:5601"
    139.   # Kibana Space ID
    140.   # ID of the Kibana Space into which the dashboards should be loaded. By default,
    141.   # the Default Space will be used.
    142.   #space.id:
    144. #============================= Elastic Cloud ==================================
    146. # These settings simplify using Filebeat with the Elastic Cloud (https://cloud.elastic.co/).
    148. # The cloud.id setting overwrites the `output.elasticsearch.hosts` and
    149. # `setup.kibana.host` options.
    150. # You can find the `cloud.id` in the Elastic Cloud web UI.
    151. #cloud.id:
    153. # The cloud.auth setting overwrites the `output.elasticsearch.username` and
    154. # `output.elasticsearch.password` settings. The format is `<user>:<pass>`.
    155. #cloud.auth:
    157. #================================ Outputs =====================================
    159. # Configure what output to use when sending the data collected by the beat.
    161. #-------------------------- Elasticsearch output ------------------------------
    162. #output.elasticsearch:
    163.   # Array of hosts to connect to.
    164.   # hosts: ["localhost:9200"]
    166.   # Optional protocol and basic auth credentials.
    167.   #protocol: "https"
    168.   #username: "elastic"
    169.   #password: "changeme"
    171. #----------------------------- Logstash output --------------------------------
    172. #output.logstash:
    173.   # The Logstash hosts
    174.   #hosts: ["localhost:5044"]
    176.   # Optional SSL. By default is off.
    177.   # List of root certificates for HTTPS server verifications
    178.   #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
    180.   # Certificate for SSL client authentication
    181.   #ssl.certificate: "/etc/pki/client/cert.pem"
    183.   # Client Certificate Key
    184.   #ssl.key: "/etc/pki/client/cert.key"
    186. #------------------------------- Redis output ----------------------------------
    187. output.redis:
    188.   hosts: ["172.16.0.61:6379"]
    189.   #password: ""
    190.   db: 1
    191.   timeout: 10
    192.   key: "%{[fields.AvalonIndexKey]}"
    195. #================================ Processors =====================================
    197. # Configure processors to enhance or manipulate events generated by the beat.
    199. processors:
    200.   - script:
    201.       lang: javascript
    202.       id: my_filter
    203.       tag: enable
    204.       # 切割message字段 获取start_time
    205.       source: >
    206.         function process(event) {
    207.             var str= event.Get("message");
    208.             var time =str.split(" ").slice(0,2).join(" ").replace('[','').replace(']','');
    209.             event.Put("start_time",time);
    210.         }
    212.   - timestamp:
    213.       # 格式化时间值 给 时间戳 
    214.       field: start_time
    215.       layouts:
    216.         - '01-02 15:04:05'
    217.         - '01-02 15:04:05.999'
    218.       test:
    219.         - '06-22 16:33:51'
    220.   #- add_host_metadata: ~
    221.   #- add_cloud_metadata: ~
    222.   #- add_docker_metadata: ~
    223.   #- add_kubernetes_metadata: ~
    224.   - drop_fields:
    225.       fields: ["agent", "cloud","ecs","host.os","log.offset","tags"]
    227. #================================ Logging =====================================
    229. # Sets log level. The default log level is info.
    230. # Available log levels are: error, warning, info, debug
    231. #logging.level: debug
    233. # At debug level, you can selectively enable logging only for some components.
    234. # To enable all selectors use ["*"]. Examples of other selectors are "beat",
    235. # "publish", "service".
    236. #logging.selectors: ["*"]
    238. #============================== X-Pack Monitoring ===============================
    239. # filebeat can export internal metrics to a central Elasticsearch monitoring
    240. # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The
    241. # reporting is disabled by default.
    243. # Set to true to enable the monitoring reporter.
    244. #monitoring.enabled: false
    246. # Sets the UUID of the Elasticsearch cluster under which monitoring data for this
    247. # Filebeat instance will appear in the Stack Monitoring UI. If output.elasticsearch
    248. # is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch.
    249. #monitoring.cluster_uuid:
    251. # Uncomment to send the metrics to Elasticsearch. Most settings from the
    252. # Elasticsearch output are accepted here as well.
    253. # Note that the settings should point to your Elasticsearch *monitoring* cluster.
    254. # Any setting that is not set is automatically inherited from the Elasticsearch
    255. # output configuration, so if you have the Elasticsearch output configured such
    256. # that it is pointing to your Elasticsearch monitoring cluster, you can simply
    257. # uncomment the following line.
    258. #monitoring.elasticsearch:
    260. #================================= Migration ==================================
    262. # This allows to enable 6.7 migration aliases
    263. #migration.6_to_7.enabled: true