flink-conf.yaml

flink-conf.yaml文件存放目录在/opt/flink/conf

  1. # jobManager 的IP地址
  2. jobmanager.rpc.address: localhost
  4. # jobManager 的端口号
  5. jobmanager.rpc.port: 6123
  7. # jobManager JVM heap 内存大小
  8. jobmanager.heap.size: 1024m
  10. # taskManager JVM heap 内存大小
  11. taskmanager.heap.size: 1024m
  13. # 每个taskManager提供任务的slots数量大小
  14. taskmanager.numberOfTaskSlots: 1
  16. #程序默认的并行计算的个数
  17. parallelism.default: 1
  19. #==============================================================================
  20. # High Availability
  21. #==============================================================================
  22. # 可以选择'NONE'或者'zookeeper'
  23. # The high-availability mode. Possible options are 'NONE' or 'zookeeper'.
  24. #
  25. # high-availability: zookeeper
  27. # The path where metadata for master recovery is persisted. While ZooKeeper stores
  28. # the small ground truth for checkpoint and leader election, this location stores
  29. # the larger objects, like persisted dataflow graphs.
  30. # 
  31. # Must be a durable file system that is accessible from all nodes
  32. # (like HDFS, S3, Ceph, nfs, ...) 
  33. # 文件系统路径，让Flink在高可用设置中持久保存元数据
  34. # high-availability.storageDir: hdfs:///flink/ha/
  36. # The list of ZooKeeper quorum peers that coordinate the high-availability
  37. # setup. This must be a list of the form:
  38. # "host1:clientPort,host2:clientPort,..." (default clientPort: 2181)
  39. # zookeeper集群中仲裁者的机器ip和port端口号
  40. # high-availability.zookeeper.quorum: localhost:2181
  43. # ACL options are based on https://zookeeper.apache.org/doc/r3.1.2/zookeeperProgrammers.html#sc_BuiltinACLSchemes
  44. # It can be either "creator" (ZOO_CREATE_ALL_ACL) or "open" (ZOO_OPEN_ACL_UNSAFE)
  45. # The default value is "open" and it can be changed to "creator" if ZK security is enabled
  46. # 默认是open，如果zookeeper security启用了该值会更改成 creator
  47. # high-availability.zookeeper.client.acl: open
  49. #==============================================================================
  50. # Fault tolerance and checkpointing 容错和检查点
  51. #==============================================================================
  53. # The backend that will be used to store operator state checkpoints if
  54. # checkpointing is enabled.
  55. #
  56. # Supported backends are 'jobmanager', 'filesystem', 'rocksdb', or the
  57. # <class-name-of-factory>.
  58. # 用于存储和检查点状态
  59. # state.backend: filesystem
  61. # Directory for checkpoints filesystem, when using any of the default bundled
  62. # state backends.
  63. # 存储检查点的数据文件和元数据的默认目录
  64. # state.checkpoints.dir: hdfs://namenode-host:port/flink-checkpoints
  66. # Default target directory for savepoints, optional.
  67. # savepoints 的默认目标目录(可选)
  68. # state.savepoints.dir: hdfs://namenode-host:port/flink-checkpoints
  70. # Flag to enable/disable incremental checkpoints for backends that
  71. # support incremental checkpoints (like the RocksDB state backend). 
  72. # 用于启用/禁用增量 checkpoints 的标志
  73. # state.backend.incremental: false
  75. #==============================================================================
  76. # Web Frontend
  77. #==============================================================================
  79. # The address under which the web-based runtime monitor listens.
  80. # 基于web的运行时监视器侦听的地址
  81. #jobmanager.web.address: 0.0.0.0
  83. # web的运行时监视器端口
  84. rest.port: 8081
  86. # 是否从基于web的jobManager启用作业提交
  87. #jobmanager.web.submit.enable: false
  89. #==============================================================================
  90. # Advanced 高级配置
  91. #==============================================================================
  93. # 是否应在TaskManager启动时预先分配TaskManager管理的内存
  94. # taskmanager.memory.preallocate: false
  96. # The classloading resolve order. Possible values are 'child-first' (Flink's default)
  97. # and 'parent-first' (Java's default).
  98. #
  99. # Child first classloading allows users to use different dependency/library
  100. # versions in their application than those in the classpath. Switching back
  101. # to 'parent-first' may help with debugging dependency issues.
  102. # 类加载解析顺序，是先检查用户代码jar('child-first')还是应用程序类路径('parent-first')。默认设置指示首先从用户代码jar加载类
  103. # classloader.resolve-order: child-first
  105. # The amount of memory going to the network stack. These numbers usually need 
  106. # no tuning. Adjusting them may be necessary in case of an "Insufficient number
  107. # of network buffers" error. The default min is 64MB, teh default max is 1GB.
  108. # 用于网络缓冲区的JVM内存的分数。这决定了 TaskManager 可以同时拥有多少流数据交换通道以及通道缓冲的程度。
  109. # 如果作业被拒绝或者您收到系统没有足够缓冲区的警告，请增加此值或下面的最小/最大值。
  110. # 另外请注意'taskmanager.network.memory.min'和'taskmanager.network.memory.max'可能会覆盖此分数
  111. # taskmanager.network.memory.fraction: 0.1
  112. # taskmanager.network.memory.min: 67108864
  113. # taskmanager.network.memory.max: 1073741824
  115. #==============================================================================
  116. # Flink Cluster Security Configuration 集群的安全配置
  117. #==============================================================================
  118. # 
  119. # Kerberos authentication for various components - Hadoop, ZooKeeper, and connectors -
  120. # may be enabled in four steps:
  121. # 1. configure the local krb5.conf file
  122. # 2. provide Kerberos credentials (either a keytab or a ticket cache w/ kinit)
  123. # 3. make the credentials available to various JAAS login contexts
  124. # 4. configure the connector to use JAAS/SASL
  126. # The below configure how Kerberos credentials are provided. A keytab will be used instead of
  127. # a ticket cache if the keytab path and principal are set.
  128. # 提示是否从 Kerberos ticket 缓存中读取
  129. # security.kerberos.login.use-ticket-cache: true
  130. #
  131. # 包含用户凭据的 Kerberos 秘钥表文件的绝对路径
  132. # security.kerberos.login.keytab: /path/to/kerberos/keytab
  133. # 与keytab 关联的 Kerberos 主体名称
  134. # security.kerberos.login.principal: flink-user
  136. # The configuration below defines which JAAS login contexts
  137. # 以逗号分隔的登录上下文列表，用于提供 Kerberos 凭据 (例如：'Client , KafkaClient' 使用凭证进行 zookeeper 身份验证和 kafka 的身份验证)
  138. # security.kerberos.login.contexts: Client,KafkaClient
  140. #==============================================================================
  141. # ZK Security Configuration zookeeper 安全配置
  142. #==============================================================================
  144. # Below configurations are applicable if ZK ensemble is configured for security
  146. # Override below configuration to provide custom ZK service name if configured
  147. # 覆盖以下配置以提供自定义 zookeeper 名字
  148. # zookeeper.sasl.service-name: zookeeper
  150. # The configuration below must match one of the values set in "security.kerberos.login.contexts"
  151. # 该配置必须匹配'security.kerberos.login.contexts'中的列表(含有一个)
  152. # zookeeper.sasl.login-context-name: Client
  154. #==============================================================================
  155. # HistoryServer
  156. #==============================================================================
  158. # 可以通过 bin/historyserver.sh (start|stop) 命令启动和关闭 HistoryServer
  159. # The HistoryServer is started and stopped via bin/historyserver.sh (start|stop)
  161. # Directory to upload completed jobs to. Add this directory to the list of
  162. # monitored directories of the HistoryServer as well (see below).
  163. # 将已经完成的作业上传到的目录
  164. #jobmanager.archive.fs.dir: hdfs:///completed-jobs/
  166. # The address under which the web-based HistoryServer listens.
  167. # 基于 Web 的 HistoryServer 的地址
  168. #historyserver.web.address: 0.0.0.0
  170. # The port under which the web-based HistoryServer listens.
  171. # 基于 Web 的 HistoryServer 的端口号
  172. #historyserver.web.port: 8082
  174. # Comma separated list of directories to monitor for completed jobs.
  175. # 以逗号分隔的目录列表，用于监视已完成的作业
  176. #historyserver.archive.fs.dir: hdfs:///completed-jobs/
  178. # Interval in milliseconds for refreshing the monitored directories.
  179. # 刷新受监控目录的时间间隔(以毫秒为单位)
  180. #historyserver.archive.fs.refresh-interval: 10000

zoo.cfg

  1. # 每个 tick 的毫秒数
  2. tickTime=2000
  4. # 初始同步阶段可以采用的 tick 数
  5. initLimit=10
  7. # 在发送请求和获取确认之间可以传递的 tick 数
  8. syncLimit=5
  10. # 存储快照的目录
  11. # dataDir=/tmp/zookeeper
  13. # 客户端将连接的端口
  14. clientPort=2181
  16. # ZooKeeper quorum peers
  17. server.1=localhost:2888:3888
  18. # server.2=host:peer-port:leader-port