image.png

    1. Taildir Source维护了一个json格式的position File，其会定期的往position File中更新每个文件读取到的最新的位置，因此能够实现断点续传
    2. vim flume-taildir-hdfs.conf
    3. a3.sources = r3
    4. a3.sinks = k3
    5. a3.channels = c3
    7. # Describe/configure the source
    8. a3.sources.r3.type = TAILDIR
    9. #指定position_file位置
    10. a3.sources.r3.positionFile = /opt/egg/apache-flume-1.7.0-bin/upload/tail_dir.json
    11. #定义监控目录文件
    12. a3.sources.r3.filegroups = f1 f2
    13. a3.sources.r3.filegroups.f1 = /opt/egg/apache-flume-1.7.0-bin/upload/.*file.*
    14. a3.sources.r3.filegroups.f2 = /opt/module/flume/files/.*log.*
    15. # Describe the sink
    16. a3.sinks.k3.type = hdfs
    17. a3.sinks.k3.hdfs.path = hdfs://hadoop1:9000/flume/upload/%Y%m%d/%H
    18. #上传文件的前缀
    19. a3.sinks.k3.hdfs.filePrefix = tail-
    20. #是否按照时间滚动文件夹
    21. a3.sinks.k3.hdfs.round = true
    22. #多少时间单位创建一个新的文件夹
    23. a3.sinks.k3.hdfs.roundValue = 1
    24. #重新定义时间单位
    25. a3.sinks.k3.hdfs.roundUnit = hour
    26. #是否使用本地时间戳
    27. a3.sinks.k3.hdfs.useLocalTimeStamp = true
    28. #积攒多少个Event才flush到HDFS一次
    29. a3.sinks.k3.hdfs.batchSize = 100
    30. #设置文件类型，可支持压缩
    31. a3.sinks.k3.hdfs.fileType = DataStream
    32. #多久生成一个新的文件
    33. a3.sinks.k3.hdfs.rollInterval = 60
    34. #设置每个文件的滚动大小大概是128M
    35. a3.sinks.k3.hdfs.rollSize = 134217700
    36. #文件的滚动与Event数量无关
    37. a3.sinks.k3.hdfs.rollCount = 0
    39. # Use a channel which buffers events in memory
    40. a3.channels.c3.type = memory
    41. a3.channels.c3.capacity = 1000
    42. a3.channels.c3.transactionCapacity = 100
    44. # Bind the source and sink to the channel
    45. a3.sources.r3.channels = c3
    46. a3.sinks.k3.channel = c3
    49. 启动flume
    50. bin/flume-ng agent --conf conf/ --name a3 --conf-file job/flume-taildir-hdfs.conf
    52. /opt/egg/apache-flume-1.7.0-bin/upload
    53. echo hello >> file1.txt
    54. echo atguigu >> file2.txt
    56. 查看hdfs会看到两个linux 文件输出到了一个hdfs文件里
    57. -rw-r--r--   3 root supergroup         28 2019-09-01 16:29 /flume/upload/20190901/16/tail-.1567326496953