1.    根据资源规划，Flume安装在节点LTSR005（**agent2**）、LTSR006（**agent3**）、LTSR008（**agent1**）上。

解压缩

  1. cd ~/software
  2. tar -zxvf apache-flume-1.6.0-bin.tar.gz -C ~/modules/
  3. cd ~/modules/apache-flume-1.6.0-bin
  4. rm -rf docs/*  ##清理doc文档

创建相关目录

  1. cd ~/modules/apache-flume-1.6.0-bin/conf
  2. cp flume-env.sh.template flume-env.sh
  3. cp flume-conf.properties.template  flume-conf.properties

配置flume-env.sh

  1. export JAVA_HOME=/home/lonton/modules/jdk1.8.0_221
  2. export JAVA_OPTS="-Xms100m -Xmx200m -Dcom.sun.management.jmxremote"
  3. export HADOOP_HOME=/home/lonton/modules/hadoop-2.5.0

分发

  1. cd ~/modules
  2. scp -r apache-flume-1.6.0-bin lonton@LTSR005:/home/lonton/modules/
  3. scp -r apache-flume-1.6.0-bin lonton@LTSR006:/home/lonton/modules/

配置flume-conf.properties

1.agent2:LTSR005（从文件采集数据）

  1. agent2.sources = r1
  2. agent2.channels = c1
  3. agent2.sinks = k1
  5. # sources
  6. agent2.sources.r1.type = exec
  7. agent2.sources.r1.command = tail -F /home/lonton/modules/weblog-flume.log
  8. agent2.sources.r1.channels = c1
  10. # channels
  11. agent2.channels.c1.type = memory
  12. agent2.channels.c1.capacity = 10000
  13. agent2.channels.c1.transactionCapacity = 10000
  14. agent2.channels.c1.keep-alive = 5
  16. # sinks
  17. agent2.sinks.k1.type = avro
  18. agent2.sinks.k1.channel = c1
  19. agent2.sinks.k1.hostname = LTSR008
  20. agent2.sinks.k1.port = 5555

2.agent3:LTSR006（从文件采集数据）

  1. agent3.sources = r1
  2. agent3.channels = c1
  3. agent3.sinks = k1
  5. # sources
  6. agent3.sources.r1.type = exec
  7. agent3.sources.r1.command = tail -F /home/lonton/modules/weblog-flume.log
  8. agent3.sources.r1.channels = c1
  10. # channels
  11. agent3.channels.c1.type = memory
  12. agent3.channels.c1.capacity = 10000
  13. agent3.channels.c1.transactionCapacity = 10000
  14. agent3.channels.c1.keep-alive = 5
  16. # sinks
  17. agent3.sinks.k1.type = avro
  18. agent3.sinks.k1.channel = c1
  19. agent3.sinks.k1.hostname = LTSR008
  20. agent3.sinks.k1.port = 5555

3.agent1: LTSR008（聚合agent2、agent3，输出到控制台）

  1. agent1.sources = r1
  2. agent1.channels = testC
  3. agent1.sinks = testS
  5. agent1.sources.r1.type = avro
  6. agent1.sources.r1.channels = testC
  7. agent1.sources.r1.bind = LTSR008
  8. agent1.sources.r1.port = 5555
  9. agent1.sources.r1.threads = 5
  11. agent1.channels.testC.type = memory
  12. agent1.channels.testC.capacity = 100000
  13. agent1.channels.testC.transactionCapacity = 100000
  14. agent1.channels.testC.keep-alive = 20
  16. agent1.sinks.testS.channel = testC
  17. agent1.sinks.testS.type = logger

启动Flume

  1. cd ~/modules/apache-flume-1.6.0-bin
  2. bin/flume-ng agent --conf conf -f conf/flume-conf.properties -n agent2 -Dflume.root.logger=INFO,console
  3. bin/flume-ng agent --conf conf -f conf/flume-conf.properties -n agent3 -Dflume.root.logger=INFO,console
  4. bin/flume-ng agent --conf conf -f conf/flume-conf.properties -n agent1 -Dflume.root.logger=INFO,console

验证

追加测试文件数据

  1. cd ~/modules/
  2. for i in {1..105}
  3. > do
  4. > echo "exec tail$i">> weblog-flume.log
  5. > done

读取成功后LTSR008的控制台将可以获取采集数据。