image.png

    1. Flume要想将数据输出到HDFS，必须持有Hadoop相关jar包
    2. commons-configuration-1.6.jar
    3. hadoop-auth-2.7.2.jar
    4. hadoop-common-2.7.2.jar
    5. hadoop-hdfs-2.7.2.jar
    6. commons-io-2.4.jar
    7. htrace-core-3.1.0-incubating.jar
    8. 拷贝到/opt/egg/flume/lib文件夹下。
    11. 创建本次案例的配置文件
    12. vim flume-file-hdfs.conf
    13. # Name the components on this agent
    14. a2.sources = r2
    15. a2.sinks = k2
    16. a2.channels = c2
    18. # Describe/configure the source
    19. a2.sources.r2.type = exec
    20. #选择监控的文件
    21. a2.sources.r2.command = tail -F /opt/ha/hadoop-2.7.2/logs/hadoop-root-namenode-hadoop1.log
    22. #执行脚本的绝对路径
    23. a2.sources.r2.shell = /bin/bash -c
    25. # Describe the sink
    26. #sink类型
    27. a2.sinks.k2.type = hdfs
    28. a2.sinks.k2.hdfs.path = hdfs://hadoop1:9000/flume/%Y%m%d/%H
    29. #上传文件的前缀
    30. a2.sinks.k2.hdfs.filePrefix = logs-
    31. #是否按照时间滚动文件夹
    32. a2.sinks.k2.hdfs.round = true
    33. #多少时间单位创建一个新的文件夹
    34. a2.sinks.k2.hdfs.roundValue = 1
    35. #重新定义时间单位
    36. a2.sinks.k2.hdfs.roundUnit = hour
    37. #是否使用本地时间戳
    38. a2.sinks.k2.hdfs.useLocalTimeStamp = true
    39. #积攒多少个Event才flush到HDFS一次
    40. a2.sinks.k2.hdfs.batchSize = 1000
    41. #设置文件类型，可支持压缩
    42. a2.sinks.k2.hdfs.fileType = DataStream
    43. #多久生成一个新的文件
    44. a2.sinks.k2.hdfs.rollInterval = 60
    45. #设置每个文件的滚动大小
    46. a2.sinks.k2.hdfs.rollSize = 134217700
    47. #文件的滚动与Event数量无关
    48. a2.sinks.k2.hdfs.rollCount = 0
    50. # Use a channel which buffers events in memory
    51. a2.channels.c2.type = file
    52. a2.channels.c2.checkpointDir = /opt
    53. a2.channels.c2.dataDirs = /opt
    54. a2.channels.c2.capacity = 1000
    55. a2.channels.c2.transactionCapacity = 100
    57. # Bind the source and sink to the channel
    58. a2.sources.r2.channels = c2
    59. a2.sinks.k2.channel = c2
    62. 运行flume
    63. bin/flume-ng agent --conf conf/ --name a2 --conf-file job/flume-file-hdfs.conf
    65. 开启Hadoop和Hive并操作Hive产生日志
    66. sbin/start-dfs.sh
    67. sbin/start-yarn.sh
    68. bin/hive
    70. 在hdfs上查看文件
    71. hadoop fs -ls /flume/20190901/20
    72. drwxr-xr-x   - root supergroup          0 2019-09-01 16:01 /flume/20190901/16
    73. -rw-r--r--   3 root supergroup     307944 2019-09-01 16:01 /flume/20190901/16/logs-.1567324803944
    74. -rw-r--r--   3 root supergroup          0 2019-09-01 16:01 /flume/20190901/16/logs-.1567324914795.tmp