date: 2020-09-10title: flume配置文件 #标题
tags: flume #标签
categories: elastic stack # 分类

记录下flume配置文件。

  1. [root@host-33 conf]# cat flume-iov-sjjh.conf
  2. ########<<-------------RDC-CLOUD-RTC------------->>########
  3. IOV-SJJH-F2KA.sources=mysource_00 mysource_01 mysource_02 mysource_03 mysource_04 mysource_05 mysource_06 mysource_07 mysource_08
  4. IOV-SJJH-F2KA.channels=mychannel_00 mychannel_01 mychannel_02 mychannel_03 mychannel_04 mychannel_05 mychannel_06 mychannel_07 mychannel_08
  5. IOV-SJJH-F2KA.sinks=mysink_00 mysink_01 mysink_02 mysink_03 mysink_04 mysink_05 mysink_06 mysink_07 mysink_08
  7. ##閾捐矾鏃ュ織
  9. IOV-SJJH-F2KA.sources.mysource_00.type=TAILDIR
  10. IOV-SJJH-F2KA.sources.mysource_00.channels=mychannel_00
  11. IOV-SJJH-F2KA.sources.mysource_00.positionFile = /apps/var/log/.FlumE/taildir_position_00.json
  12. IOV-SJJH-F2KA.sources.mysource_00.filegroups = f1
  13. IOV-SJJH-F2KA.sources.mysource_00.filegroups.f1 = /apps/var/jakarta/logs/^sleuth.*.log
  14. IOV-SJJH-F2KA.sources.mysource_00.headers.f1.headerKey1 = VALUE1
  15. IOV-SJJH-F2KA.sources.mysource_00.fileHeader = true
  17. IOV-SJJH-F2KA.sinks.mysink_00.channel=mychannel_00
  18. IOV-SJJH-F2KA.sinks.mysink_00.type=org.apache.flume.sink.kafka.KafkaSink
  19. IOV-SJJH-F2KA.sinks.mysink_00.kafka.bootstrap.servers=192.168.1.26:9092
  21. IOV-SJJH-F2KA.sinks.mysink_00.kafka.topic=iov_sjjh_trace_online
  22. IOV-SJJH-F2KA.sinks.mysink_00.kafka.batchSize=1
  23. #IOV-SJJH-F2KA.sinks.mysink_00.kafka.producer.requiredAcks=1
  24. IOV-SJJH-F2KA.sinks.mysink_00.kafka.producer.linger.ms = 1
  25. IOV-SJJH-F2KA.sinks.mysink_00.kafka.producer.max.request.size = 4194304
  27. IOV-SJJH-F2KA.channels.mychannel_00.type=memory
  28. IOV-SJJH-F2KA.channels.mychannel_00.capacity=30000
  29. IOV-SJJH-F2KA.channels.mychannel_00.transactionCapacity=100
  31. ##璁块棶鏃ュ織
  33. IOV-SJJH-F2KA.sources.mysource_01.interceptors=i1 i2
  34. IOV-SJJH-F2KA.sources.mysource_01.interceptors.i1.type=regex_filter
  35. IOV-SJJH-F2KA.sources.mysource_01.interceptors.i1.regex=("URI":"/actuator)|("URI":"/api/v2/spans)|(/favicon.ico)|(/v2/api-docs)|(/swagger-resources)|(/swagger-ui.html)
  36. IOV-SJJH-F2KA.sources.mysource_01.interceptors.i1.excludeEvents = true
  37. IOV-SJJH-F2KA.sources.mysource_01.interceptors.i2.type=regex_filter
  38. IOV-SJJH-F2KA.sources.mysource_01.interceptors.i2.regex=(/webjars/springfox-swagger-ui/)|(/csrf)
  39. IOV-SJJH-F2KA.sources.mysource_01.interceptors.i2.excludeEvents = true
  41. IOV-SJJH-F2KA.sources.mysource_01.type=TAILDIR
  42. IOV-SJJH-F2KA.sources.mysource_01.channels=mychannel_01
  43. #IOV-SJJH-F2KA.sources.mysource_01.inputCharset = UTF-8
  44. IOV-SJJH-F2KA.sources.mysource_01.positionFile = /apps/var/log/.FlumE/taildir_position_01.json
  45. IOV-SJJH-F2KA.sources.mysource_01.filegroups = f1
  46. IOV-SJJH-F2KA.sources.mysource_01.filegroups.f1 = /apps/var/jakarta/logs/^access.*.log
  47. IOV-SJJH-F2KA.sources.mysource_01.headers.f1.headerKey1 = VALUE1
  48. IOV-SJJH-F2KA.sources.mysource_01.fileHeader = true
  50. IOV-SJJH-F2KA.sinks.mysink_01.channel=mychannel_01
  51. IOV-SJJH-F2KA.sinks.mysink_01.type=org.apache.flume.sink.kafka.KafkaSink
  52. IOV-SJJH-F2KA.sinks.mysink_01.kafka.bootstrap.servers=192.168.1.26:9092
  54. IOV-SJJH-F2KA.sinks.mysink_01.kafka.topic=iov_sjjh_access_online
  55. IOV-SJJH-F2KA.sinks.mysink_01.kafka.batchSize=1
  56. #IOV-SJJH-F2KA.sinks.mysink_01.custom.encoding=UTF-8
  57. #IOV-SJJH-F2KA.sinks.mysink_01.kafka.producer.requiredAcks=1
  58. IOV-SJJH-F2KA.sinks.mysink_01.kafka.producer.linger.ms = 1
  59. IOV-SJJH-F2KA.sinks.mysink_01.kafka.producer.max.request.size = 4194304
  61. IOV-SJJH-F2KA.channels.mychannel_01.type=memory
  62. IOV-SJJH-F2KA.channels.mychannel_01.capacity=30000
  63. IOV-SJJH-F2KA.channels.mychannel_01.transactionCapacity=100
  65. ## 閿欒鏃ュ織
  67. IOV-SJJH-F2KA.sources.mysource_02.type=TAILDIR
  68. IOV-SJJH-F2KA.sources.mysource_02.channels=mychannel_02
  69. IOV-SJJH-F2KA.sources.mysource_02.positionFile = /apps/var/log/.FlumE/taildir_position_02.json
  70. IOV-SJJH-F2KA.sources.mysource_02.filegroups = f1
  71. IOV-SJJH-F2KA.sources.mysource_02.filegroups.f1 = /apps/var/jakarta/logs/^error.*.log
  72. IOV-SJJH-F2KA.sources.mysource_02.headers.f1.headerKey1 = VALUE1
  73. IOV-SJJH-F2KA.sources.mysource_02.fileHeader = true
  75. IOV-SJJH-F2KA.sinks.mysink_02.channel=mychannel_02
  76. IOV-SJJH-F2KA.sinks.mysink_02.type=org.apache.flume.sink.kafka.KafkaSink
  77. IOV-SJJH-F2KA.sinks.mysink_02.kafka.bootstrap.servers=192.168.1.26:9092
  79. IOV-SJJH-F2KA.sinks.mysink_02.kafka.topic=iov_sjjh_error_online
  80. IOV-SJJH-F2KA.sinks.mysink_02.kafka.batchSize=1
  81. #IOV-SJJH-F2KA.sinks.mysink_02.kafka.producer.requiredAcks=1
  82. IOV-SJJH-F2KA.sinks.mysink_02.kafka.producer.linger.ms = 1
  83. IOV-SJJH-F2KA.sinks.mysink_02.kafka.producer.max.request.size = 4194304
  85. IOV-SJJH-F2KA.channels.mychannel_02.type=memory
  86. IOV-SJJH-F2KA.channels.mychannel_02.capacity=30000
  87. IOV-SJJH-F2KA.channels.mychannel_02.transactionCapacity=100
  90. ## API缃戝叧璁块棶鏃ュ織
  92. IOV-SJJH-F2KA.sources.mysource_03.interceptors=i1 i2
  93. IOV-SJJH-F2KA.sources.mysource_03.interceptors.i1.type=regex_filter
  94. IOV-SJJH-F2KA.sources.mysource_03.interceptors.i1.regex=("URI":"/actuator)|("URI":"/api/v2/spans)|(/favicon.ico)|(/v2/api-docs)|(/swagger-resources)|(/swagger-ui.html)
  95. IOV-SJJH-F2KA.sources.mysource_03.interceptors.i1.excludeEvents = true
  96. IOV-SJJH-F2KA.sources.mysource_03.interceptors.i2.type=regex_filter
  97. IOV-SJJH-F2KA.sources.mysource_03.interceptors.i2.regex=(/webjars/springfox-swagger-ui/)|(/csrf)
  98. IOV-SJJH-F2KA.sources.mysource_03.interceptors.i2.excludeEvents = true
  100. IOV-SJJH-F2KA.sources.mysource_03.type=TAILDIR
  101. IOV-SJJH-F2KA.sources.mysource_03.channels=mychannel_03
  102. IOV-SJJH-F2KA.sources.mysource_03.positionFile = /apps/var/log/.FlumE/taildir_position_03.json
  103. IOV-SJJH-F2KA.sources.mysource_03.filegroups = f1
  104. IOV-SJJH-F2KA.sources.mysource_03.filegroups.f1 = /apps/var/jakarta/logs/^access.*._iov-sjjh-gateway_.*.log
  105. IOV-SJJH-F2KA.sources.mysource_03.headers.f1.headerKey1 = VALUE1
  106. IOV-SJJH-F2KA.sources.mysource_03.fileHeader = true
  108. IOV-SJJH-F2KA.sinks.mysink_03.channel=mychannel_03
  109. IOV-SJJH-F2KA.sinks.mysink_03.type=org.apache.flume.sink.kafka.KafkaSink
  110. IOV-SJJH-F2KA.sinks.mysink_03.kafka.bootstrap.servers=192.168.1.26:9092
  112. IOV-SJJH-F2KA.sinks.mysink_03.kafka.topic=iov_sjjh_access_iov-sjjh-gateway_online
  113. IOV-SJJH-F2KA.sinks.mysink_03.kafka.batchSize=1
  114. IOV-SJJH-F2KA.sinks.mysink_03.kafka.producer.linger.ms = 1
  115. IOV-SJJH-F2KA.sinks.mysink_03.kafka.producer.max.request.size = 4194304
  117. IOV-SJJH-F2KA.channels.mychannel_03.type=memory
  118. IOV-SJJH-F2KA.channels.mychannel_03.capacity=30000
  119. IOV-SJJH-F2KA.channels.mychannel_03.transactionCapacity=100
  121. ##API浜岀骇缃戝叧璁块棶鏃ュ織
  123. IOV-SJJH-F2KA.sources.mysource_04.interceptors=i1 i2
  124. IOV-SJJH-F2KA.sources.mysource_04.interceptors.i1.type=regex_filter
  125. IOV-SJJH-F2KA.sources.mysource_04.interceptors.i1.regex=("URI":"/actuator)|("URI":"/api/v2/spans)|(/favicon.ico)|(/v2/api-docs)|(/swagger-resources)|(/swagger-ui.html)
  126. IOV-SJJH-F2KA.sources.mysource_04.interceptors.i1.excludeEvents = true
  127. IOV-SJJH-F2KA.sources.mysource_04.interceptors.i2.type=regex_filter
  128. IOV-SJJH-F2KA.sources.mysource_04.interceptors.i2.regex=(/webjars/springfox-swagger-ui/)|(/csrf)
  129. IOV-SJJH-F2KA.sources.mysource_04.interceptors.i2.excludeEvents = true
  131. IOV-SJJH-F2KA.sources.mysource_04.type=TAILDIR
  132. IOV-SJJH-F2KA.sources.mysource_04.channels=mychannel_04
  133. IOV-SJJH-F2KA.sources.mysource_04.positionFile = /apps/var/log/.FlumE/taildir_position_04.json
  134. IOV-SJJH-F2KA.sources.mysource_04.filegroups = f1
  135. IOV-SJJH-F2KA.sources.mysource_04.filegroups.f1 = /apps/var/jakarta/logs/^access.*._iov-sjjh-gateway-ct_.*.log
  136. IOV-SJJH-F2KA.sources.mysource_04.headers.f1.headerKey1 = VALUE1
  137. IOV-SJJH-F2KA.sources.mysource_04.fileHeader = true
  139. IOV-SJJH-F2KA.sinks.mysink_04.channel=mychannel_04
  140. IOV-SJJH-F2KA.sinks.mysink_04.type=org.apache.flume.sink.kafka.KafkaSink
  141. IOV-SJJH-F2KA.sinks.mysink_04.kafka.bootstrap.servers=192.168.1.26:9092
  143. IOV-SJJH-F2KA.sinks.mysink_04.kafka.topic=iov_sjjh_access_iov-sjjh-gateway-ct_online
  144. IOV-SJJH-F2KA.sinks.mysink_04.kafka.batchSize=1
  145. IOV-SJJH-F2KA.sinks.mysink_04.kafka.producer.linger.ms = 1
  146. IOV-SJJH-F2KA.sinks.mysink_04.kafka.producer.max.request.size = 4194304
  148. IOV-SJJH-F2KA.channels.mychannel_04.type=memory
  149. IOV-SJJH-F2KA.channels.mychannel_04.capacity=30000
  150. IOV-SJJH-F2KA.channels.mychannel_04.transactionCapacity=100
  152. ##鍚庡彴缃叧块棶鏃ュ織
  154. IOV-SJJH-F2KA.sources.mysource_05.interceptors=i1 i2
  155. IOV-SJJH-F2KA.sources.mysource_05.interceptors.i1.type=regex_filter
  156. IOV-SJJH-F2KA.sources.mysource_05.interceptors.i1.regex=("URI":"/actuator)|("URI":"/api/v2/spans)|(/favicon.ico)|(/v2/api-docs)|(/swagger-resources)|(/swagger-ui.html)
  157. IOV-SJJH-F2KA.sources.mysource_05.interceptors.i1.excludeEvents = true
  158. IOV-SJJH-F2KA.sources.mysource_05.interceptors.i2.type=regex_filter
  159. IOV-SJJH-F2KA.sources.mysource_05.interceptors.i2.regex=(/webjars/springfox-swagger-ui/)|(/csrf)
  160. IOV-SJJH-F2KA.sources.mysource_05.interceptors.i2.excludeEvents = true
  162. IOV-SJJH-F2KA.sources.mysource_05.type=TAILDIR
  163. IOV-SJJH-F2KA.sources.mysource_05.channels=mychannel_05
  164. IOV-SJJH-F2KA.sources.mysource_05.positionFile = /apps/var/log/.FlumE/taildir_position_05.json
  165. IOV-SJJH-F2KA.sources.mysource_05.filegroups = f1
  166. IOV-SJJH-F2KA.sources.mysource_05.filegroups.f1 = /apps/var/jakarta/logs/^access.*._iov-sjjh-gateway-portal_.*.log
  167. IOV-SJJH-F2KA.sources.mysource_05.headers.f1.headerKey1 = VALUE1
  168. IOV-SJJH-F2KA.sources.mysource_05.fileHeader = true
  170. IOV-SJJH-F2KA.sinks.mysink_05.channel=mychannel_05
  171. IOV-SJJH-F2KA.sinks.mysink_05.type=org.apache.flume.sink.kafka.KafkaSink
  172. IOV-SJJH-F2KA.sinks.mysink_05.kafka.bootstrap.servers=192.168.1.26:9092
  174. IOV-SJJH-F2KA.sinks.mysink_05.kafka.topic=iov_sjjh_access_iov-sjjh-gateway-portal_online
  175. IOV-SJJH-F2KA.sinks.mysink_05.kafka.batchSize=1
  176. IOV-SJJH-F2KA.sinks.mysink_05.kafka.producer.linger.ms = 1
  177. IOV-SJJH-F2KA.sinks.mysink_05.kafka.producer.max.request.size = 4194304
  179. IOV-SJJH-F2KA.channels.mychannel_05.type=memory
  180. IOV-SJJH-F2KA.channels.mychannel_05.capacity=30000
  181. IOV-SJJH-F2KA.channels.mychannel_05.transactionCapacity=100
  184. ## API涓€绾х綉鍏冲搷搴旀棩蹇
  186. IOV-SJJH-F2KA.sources.mysource_06.interceptors=i1 i2
  187. IOV-SJJH-F2KA.sources.mysource_06.interceptors.i1.type=regex_filter
  188. IOV-SJJH-F2KA.sources.mysource_06.interceptors.i1.regex=("URI":"/actuator)|("URI":"/api/v2/spans)|(/favicon.ico)|(/v2/api-docs)|(/swagger-resources)|(/swagger-ui.html)
  189. IOV-SJJH-F2KA.sources.mysource_06.interceptors.i1.excludeEvents = true
  190. IOV-SJJH-F2KA.sources.mysource_06.interceptors.i2.type=regex_filter
  191. IOV-SJJH-F2KA.sources.mysource_06.interceptors.i2.regex=(/webjars/springfox-swagger-ui/)|(/csrf)
  192. IOV-SJJH-F2KA.sources.mysource_06.interceptors.i2.excludeEvents = true
  194. IOV-SJJH-F2KA.sources.mysource_06.type=TAILDIR
  195. IOV-SJJH-F2KA.sources.mysource_06.channels=mychannel_06
  196. IOV-SJJH-F2KA.sources.mysource_06.positionFile = /apps/var/log/.FlumE/taildir_position_06.json
  197. IOV-SJJH-F2KA.sources.mysource_06.filegroups = f1
  198. IOV-SJJH-F2KA.sources.mysource_06.filegroups.f1 = /apps/var/jakarta/logs/^request.*._iov-sjjh-gateway_.*.log
  199. IOV-SJJH-F2KA.sources.mysource_06.headers.f1.headerKey1 = VALUE1
  200. IOV-SJJH-F2KA.sources.mysource_06.fileHeader = true
  202. IOV-SJJH-F2KA.sinks.mysink_06.channel=mychannel_06
  203. IOV-SJJH-F2KA.sinks.mysink_06.type=org.apache.flume.sink.kafka.KafkaSink
  204. IOV-SJJH-F2KA.sinks.mysink_06.kafka.bootstrap.servers=192.168.1.26:9092
  206. IOV-SJJH-F2KA.sinks.mysink_06.kafka.topic=iov_sjjh_request_online
  207. IOV-SJJH-F2KA.sinks.mysink_06.kafka.batchSize=1
  208. IOV-SJJH-F2KA.sinks.mysink_06.kafka.producer.linger.ms = 1
  209. IOV-SJJH-F2KA.sinks.mysink_06.kafka.producer.max.request.size = 4194304
  211. IOV-SJJH-F2KA.channels.mychannel_06.type=memory
  212. IOV-SJJH-F2KA.channels.mychannel_06.capacity=30000
  213. IOV-SJJH-F2KA.channels.mychannel_06.transactionCapacity=100
  216. ## yysj 涓€绾х綉鍏冲搷搴旀棩蹇
  218. IOV-SJJH-F2KA.sources.mysource_07.interceptors=i1 i2
  219. IOV-SJJH-F2KA.sources.mysource_07.interceptors.i1.type=regex_filter
  220. IOV-SJJH-F2KA.sources.mysource_07.interceptors.i1.regex=("URI":"/actuator)|("URI":"/api/v2/spans)|(/favicon.ico)|(/v2/api-docs)|(/swagger-resources)|(/swagger-ui.html)
  221. IOV-SJJH-F2KA.sources.mysource_07.interceptors.i1.excludeEvents = true
  222. IOV-SJJH-F2KA.sources.mysource_07.interceptors.i2.type=regex_filter
  223. IOV-SJJH-F2KA.sources.mysource_07.interceptors.i2.regex=(/webjars/springfox-swagger-ui/)|(/csrf)
  224. IOV-SJJH-F2KA.sources.mysource_07.interceptors.i2.excludeEvents = true
  226. IOV-SJJH-F2KA.sources.mysource_07.type=TAILDIR
  227. IOV-SJJH-F2KA.sources.mysource_07.channels=mychannel_07
  228. IOV-SJJH-F2KA.sources.mysource_07.positionFile = /apps/var/log/.FlumE/taildir_position_07.json
  229. IOV-SJJH-F2KA.sources.mysource_07.filegroups = f1
  230. IOV-SJJH-F2KA.sources.mysource_07.filegroups.f1 = /apps/var/jakarta/logs/^yysjaccess.*._cloud-yysj-check-info.*.log
  231. IOV-SJJH-F2KA.sources.mysource_07.headers.f1.headerKey1 = VALUE1
  232. IOV-SJJH-F2KA.sources.mysource_07.fileHeader = true
  234. IOV-SJJH-F2KA.sinks.mysink_07.channel=mychannel_07
  235. IOV-SJJH-F2KA.sinks.mysink_07.type=org.apache.flume.sink.kafka.KafkaSink
  236. IOV-SJJH-F2KA.sinks.mysink_07.kafka.bootstrap.servers=192.168.1.26:9092
  238. IOV-SJJH-F2KA.sinks.mysink_07.kafka.topic=cloud_yysj_check_info_online
  239. IOV-SJJH-F2KA.sinks.mysink_07.kafka.batchSize=1
  240. IOV-SJJH-F2KA.sinks.mysink_07.kafka.producer.linger.ms = 1
  241. IOV-SJJH-F2KA.sinks.mysink_07.kafka.producer.max.request.size = 4194304
  243. IOV-SJJH-F2KA.channels.mychannel_07.type=memory
  244. IOV-SJJH-F2KA.channels.mychannel_07.capacity=30000
  245. IOV-SJJH-F2KA.channels.mychannel_07.transactionCapacity=100
  250. ## yysj 浜岀骇缃戝叧鍝嶅簲鏃ュ織
  252. IOV-SJJH-F2KA.sources.mysource_08.interceptors=i1 i2
  253. IOV-SJJH-F2KA.sources.mysource_08.interceptors.i1.type=regex_filter
  254. IOV-SJJH-F2KA.sources.mysource_08.interceptors.i1.regex=("URI":"/actuator)|("URI":"/api/v2/spans)|(/favicon.ico)|(/v2/api-docs)|(/swagger-resources)|(/swagger-ui.html)
  255. IOV-SJJH-F2KA.sources.mysource_08.interceptors.i1.excludeEvents = true
  256. IOV-SJJH-F2KA.sources.mysource_08.interceptors.i2.type=regex_filter
  257. IOV-SJJH-F2KA.sources.mysource_08.interceptors.i2.regex=(/webjars/springfox-swagger-ui/)|(/csrf)
  258. IOV-SJJH-F2KA.sources.mysource_08.interceptors.i2.excludeEvents = true
  260. IOV-SJJH-F2KA.sources.mysource_08.type=TAILDIR
  261. IOV-SJJH-F2KA.sources.mysource_08.channels=mychannel_08
  262. IOV-SJJH-F2KA.sources.mysource_08.positionFile = /apps/var/log/.FlumE/taildir_position_08.json
  263. IOV-SJJH-F2KA.sources.mysource_08.filegroups = f1
  264. IOV-SJJH-F2KA.sources.mysource_08.filegroups.f1 = /apps/var/jakarta/logs/^yysjaccess.*._cloud-yysj-product-check.*.log
  265. IOV-SJJH-F2KA.sources.mysource_08.headers.f1.headerKey1 = VALUE1
  266. IOV-SJJH-F2KA.sources.mysource_08.fileHeader = true
  268. IOV-SJJH-F2KA.sinks.mysink_08.channel=mychannel_08
  269. IOV-SJJH-F2KA.sinks.mysink_08.type=org.apache.flume.sink.kafka.KafkaSink
  270. IOV-SJJH-F2KA.sinks.mysink_08.kafka.bootstrap.servers=192.168.1.26:9092
  272. IOV-SJJH-F2KA.sinks.mysink_08.kafka.topic=cloud_yysj_product_check_online
  273. IOV-SJJH-F2KA.sinks.mysink_08.kafka.batchSize=1
  274. IOV-SJJH-F2KA.sinks.mysink_08.kafka.producer.linger.ms = 1
  275. IOV-SJJH-F2KA.sinks.mysink_08.kafka.producer.max.request.size = 4194304
  277. IOV-SJJH-F2KA.channels.mychannel_08.type=memory
  278. IOV-SJJH-F2KA.channels.mychannel_08.capacity=30000
  279. IOV-SJJH-F2KA.channels.mychannel_08.transactionCapacity=100