image.png

    1. vim flume-dir-hdfs.conf
    2. a3.sources = r3
    3. a3.sinks = k3
    4. a3.channels = c3
    6. # Describe/configure the source
    7. #source类型
    8. a3.sources.r3.type = spooldir
    9. #监控的路径
    10. a3.sources.r3.spoolDir = /opt/egg/apache-flume-1.7.0-bin/upload
    11. #定义文件上传完后生成的后缀(linux 的文件上传完会加这个后缀)
    12. a3.sources.r3.fileSuffix = .COMPLETED
    13. #是否有文件头
    14. a3.sources.r3.fileHeader = true
    15. #忽略所有以.tmp结尾的文件，不上传
    16. a3.sources.r3.ignorePattern = ([^ ]*\.tmp)
    18. # Describe the sink
    19. a3.sinks.k3.type = hdfs
    20. a3.sinks.k3.hdfs.path = hdfs://hadoop1:9000/flume/upload/%Y%m%d/%H
    21. #上传文件的前缀
    22. a3.sinks.k3.hdfs.filePrefix = upload-
    23. #是否按照时间滚动文件夹
    24. a3.sinks.k3.hdfs.round = true
    25. #多少时间单位创建一个新的文件夹
    26. a3.sinks.k3.hdfs.roundValue = 1
    27. #重新定义时间单位
    28. a3.sinks.k3.hdfs.roundUnit = hour
    29. #是否使用本地时间戳
    30. a3.sinks.k3.hdfs.useLocalTimeStamp = true
    31. #积攒多少个Event才flush到HDFS一次
    32. a3.sinks.k3.hdfs.batchSize = 100
    33. #设置文件类型，可支持压缩
    34. a3.sinks.k3.hdfs.fileType = DataStream
    35. #多久生成一个新的文件
    36. a3.sinks.k3.hdfs.rollInterval = 60
    37. #设置每个文件的滚动大小大概是128M
    38. a3.sinks.k3.hdfs.rollSize = 134217700
    39. #文件的滚动与Event数量无关
    40. a3.sinks.k3.hdfs.rollCount = 0
    42. # Use a channel which buffers events in memory
    43. a3.channels.c3.type = memory
    44. a3.channels.c3.capacity = 1000
    45. a3.channels.c3.transactionCapacity = 100
    47. # Bind the source and sink to the channel
    48. a3.sources.r3.channels = c3
    49. a3.sinks.k3.channel = c3
    52. 运行flume
    53. bin/flume-ng agent --conf conf/ --name a3 --conf-file job/flume-dir-hdfs.conf
    56. 实验
    57. 往目录touch文件 包括tmp结尾的和txt结尾的
    58. echo "hha" > log3.log
    59. echo "haha" > log4.tmp
    61. drwxr-xr-x   - root supergroup          0 2019-09-01 16:19 /flume/upload
    62. drwxr-xr-x   - root supergroup          0 2019-09-01 16:19 /flume/upload/20190901
    63. drwxr-xr-x   - root supergroup          0 2019-09-01 16:20 /flume/upload/20190901/16
    64. -rw-r--r--   3 root supergroup          8 2019-09-01 16:20 /flume/upload/20190901/16/upload-.1567325951362