image.png

    1. 使用Flume-1监控文件变动，Flume-1将变动内容传递给Flume-2，Flume-2负责存储到HDFS。同时Flume-1将变动内容传递给Flume-3，Flume-3负责输出到Local FileSystem。
    2. mkdir -p /opt/egg/apache-flume-1.7.0-bin/group1
    4. 配置1个接收日志文件的source和两个channel、两个sink，分别输送给flume-flume-hdfs和flume-flume-dir。
    5. vim flume-file-flume.conf
    6. # Name the components on this agent
    7. a1.sources = r1
    8. a1.sinks = k1 k2
    9. a1.channels = c1 c2
    10. # 将数据流复制给所有channel
    11. a1.sources.r1.selector.type = replicating
    13. # Describe/configure the source
    14. a1.sources.r1.type = exec
    15. a1.sources.r1.command = tail -F /opt/ha/hadoop-2.7.2/logs/hadoop-root-namenode-hadoop1.log
    16. a1.sources.r1.shell = /bin/bash -c
    18. # Describe the sink
    19. # sink端的avro是一个数据发送者
    20. a1.sinks.k1.type = avro
    21. a1.sinks.k1.hostname = hadoop1
    22. a1.sinks.k1.port = 4141
    24. a1.sinks.k2.type = avro
    25. a1.sinks.k2.hostname = hadoop1
    26. a1.sinks.k2.port = 4142
    28. # Describe the channel
    29. a1.channels.c1.type = memory
    30. a1.channels.c1.capacity = 1000
    31. a1.channels.c1.transactionCapacity = 100
    33. a1.channels.c2.type = memory
    34. a1.channels.c2.capacity = 1000
    35. a1.channels.c2.transactionCapacity = 100
    37. # Bind the source and sink to the channel
    38. a1.sources.r1.channels = c1 c2
    39. a1.sinks.k1.channel = c1
    40. a1.sinks.k2.channel = c2
    46. 输出是到HDFS的Sink
    47. vim flume-flume-hdfs.conf
    48. # Name the components on this agent
    49. a2.sources = r1
    50. a2.sinks = k1
    51. a2.channels = c1
    53. # Describe/configure the source
    54. # source端的avro是一个数据接收服务
    55. a2.sources.r1.type = avro
    56. a2.sources.r1.bind = hadoop1
    57. a2.sources.r1.port = 4141
    59. # Describe the sink
    60. a2.sinks.k1.type = hdfs
    61. a2.sinks.k1.hdfs.path = hdfs://hadoop1:9000/flume2/%Y%m%d/%H
    62. #上传文件的前缀
    63. a2.sinks.k1.hdfs.filePrefix = flume2-
    64. #是否按照时间滚动文件夹
    65. a2.sinks.k1.hdfs.round = true
    66. #多少时间单位创建一个新的文件夹
    67. a2.sinks.k1.hdfs.roundValue = 1
    68. #重新定义时间单位
    69. a2.sinks.k1.hdfs.roundUnit = hour
    70. #是否使用本地时间戳
    71. a2.sinks.k1.hdfs.useLocalTimeStamp = true
    72. #积攒多少个Event才flush到HDFS一次
    73. a2.sinks.k1.hdfs.batchSize = 100
    74. #设置文件类型，可支持压缩
    75. a2.sinks.k1.hdfs.fileType = DataStream
    76. #多久生成一个新的文件
    77. a2.sinks.k1.hdfs.rollInterval = 600
    78. #设置每个文件的滚动大小大概是128M
    79. a2.sinks.k1.hdfs.rollSize = 134217700
    80. #文件的滚动与Event数量无关
    81. a2.sinks.k1.hdfs.rollCount = 0
    83. # Describe the channel
    84. a2.channels.c1.type = memory
    85. a2.channels.c1.capacity = 1000
    86. a2.channels.c1.transactionCapacity = 100
    88. # Bind the source and sink to the channel
    89. a2.sources.r1.channels = c1
    90. a2.sinks.k1.channel = c1
    95. 输出是到本地目录的Sink:输出的本地目录必须是已经存在的目录，如果该目录不存在，并不会创建新的目录
    96. vim flume-flume-dir.conf
    97. # Name the components on this agent
    98. a3.sources = r1
    99. a3.sinks = k1
    100. a3.channels = c2
    102. # Describe/configure the source
    103. a3.sources.r1.type = avro
    104. a3.sources.r1.bind = hadoop1
    105. a3.sources.r1.port = 4142
    107. # Describe the sink
    108. a3.sinks.k1.type = file_roll
    109. a3.sinks.k1.sink.directory = /opt/egg/apache-flume-1.7.0-bin/group1
    111. # Describe the channel
    112. a3.channels.c2.type = memory
    113. a3.channels.c2.capacity = 1000
    114. a3.channels.c2.transactionCapacity = 100
    116. # Bind the source and sink to the channel
    117. a3.sources.r1.channels = c2
    118. a3.sinks.k1.channel = c2
    122. 启动flume
    123. bin/flume-ng agent --conf conf/ --name a3 --conf-file job/group1/flume-flume-dir.conf
    124. bin/flume-ng agent --conf conf/ --name a2 --conf-file job/group1/flume-flume-hdfs.conf
    125. bin/flume-ng agent --conf conf/ --name a1 --conf-file job/group1/flume-file-flume.conf
    129. 制造一些hive日志
    130. sbin/start-dfs.sh
    131. sbin/start-yarn.sh
    132. bin/hive