image.png

    1. 使用Flume1监控一个端口，其sink组中的sink分别对接Flume2和Flume3，采用FailoverSinkProcessor，实现故障转移的功能
    2. mkdir -p /opt/egg/apache-flume-1.7.0-bin/group2
    5. 1个sink group（2个sink），分别输送给flume-flume-console1和flume-flume-console2
    6. vim flume-netcat-flume.conf
    7. # Name the components on this agent
    8. a1.sources = r1
    9. a1.channels = c1
    10. a1.sinkgroups = g1
    11. a1.sinks = k1 k2
    13. # Describe/configure the source
    14. a1.sources.r1.type = netcat
    15. a1.sources.r1.bind = localhost
    16. a1.sources.r1.port = 44444
    18. a1.sinkgroups.g1.processor.type = failover
    19. a1.sinkgroups.g1.processor.priority.k1 = 5
    20. a1.sinkgroups.g1.processor.priority.k2 = 10
    21. a1.sinkgroups.g1.processor.maxpenalty = 10000
    23. # Describe the sink
    24. a1.sinks.k1.type = avro
    25. a1.sinks.k1.hostname = hadoop1
    26. a1.sinks.k1.port = 4141
    28. a1.sinks.k2.type = avro
    29. a1.sinks.k2.hostname = hadoop1
    30. a1.sinks.k2.port = 4142
    32. # Describe the channel
    33. a1.channels.c1.type = memory
    34. a1.channels.c1.capacity = 1000
    35. a1.channels.c1.transactionCapacity = 100
    37. # Bind the source and sink to the channel
    38. a1.sources.r1.channels = c1
    39. a1.sinkgroups.g1.sinks = k1 k2
    40. a1.sinks.k1.channel = c1
    41. a1.sinks.k2.channel = c1
    45. 输出是到本地控制台
    46. vim flume-flume-console1.conf
    47. # Name the components on this agent
    48. a2.sources = r1
    49. a2.sinks = k1
    50. a2.channels = c1
    52. # Describe/configure the source
    53. a2.sources.r1.type = avro
    54. a2.sources.r1.bind = hadoop1
    55. a2.sources.r1.port = 4141
    57. # Describe the sink
    58. a2.sinks.k1.type = logger
    60. # Describe the channel
    61. a2.channels.c1.type = memory
    62. a2.channels.c1.capacity = 1000
    63. a2.channels.c1.transactionCapacity = 100
    65. # Bind the source and sink to the channel
    66. a2.sources.r1.channels = c1
    67. a2.sinks.k1.channel = c1
    70. 输出是到本地控制台。
    71. vim flume-flume-console2.conf
    72. # Name the components on this agent
    73. a3.sources = r1
    74. a3.sinks = k1
    75. a3.channels = c2
    77. # Describe/configure the source
    78. a3.sources.r1.type = avro
    79. a3.sources.r1.bind = hadoop1
    80. a3.sources.r1.port = 4142
    82. # Describe the sink
    83. a3.sinks.k1.type = logger
    85. # Describe the channel
    86. a3.channels.c2.type = memory
    87. a3.channels.c2.capacity = 1000
    88. a3.channels.c2.transactionCapacity = 100
    90. # Bind the source and sink to the channel
    91. a3.sources.r1.channels = c2
    92. a3.sinks.k1.channel = c2
    96. 启动flume
    97. bin/flume-ng agent --conf conf/ --name a3 --conf-file job/group2/flume-flume-console2.conf -Dflume.root.logger=INFO,console
    98. bin/flume-ng agent --conf conf/ --name a2 --conf-file job/group2/flume-flume-console1.conf -Dflume.root.logger=INFO,console
    99. bin/flume-ng agent --conf conf/ --name a1 --conf-file job/group2/flume-netcat-flume.conf
    101. 向44444端口发送字符串
    102. nc localhost 44444