配置Spark Metrics文件

  1. master.source.jvm.class=org.apache.spark.metrics.source.JvmSource
  2. worker.source.jvm.class=org.apache.spark.metrics.source.JvmSource
  3. driver.source.jvm.class=org.apache.spark.metrics.source.JvmSource
  4. executor.source.jvm.class=org.apache.spark.metrics.source.JvmSource
  5. *.source.jvm.class=org.apache.spark.metrics.source.JvmSource # Optional JVM metrics
  7. *.sink.graphite.class=org.apache.spark.metrics.sink.GraphiteSink
  8. *.sink.graphite.host=192.168.9.155
  9. *.sink.graphite.port=9109
  10. *.sink.graphite.period=5
  11. *.sink.graphite.unit=seconds

安装graphite_exporter

配置映射文件

  1. mappings:
  2. - match: '*.*.executor.filesystem.*.*'
  3.   name: filesystem_usage
  4.   labels:
  5.     application: $1
  6.     executor_id: $2
  7.     fs_type: $3
  8.     qty: $4
  10. - match: '*.*.jvm.*.*'
  11.   name: jvm_memory_usage
  12.   labels:
  13.     application: $1
  14.     executor_id: $2
  15.     mem_type: $3
  16.     qty: $4
  18. - match: '*.*.executor.jvmGCTime.count'
  19.   name: jvm_gcTime_count
  20.   labels:
  21.     application: $1
  22.     executor_id: $2
  24. - match: '*.*.jvm.pools.*.*'
  25.   name: jvm_memory_pools
  26.   labels:
  27.     application: $1
  28.     executor_id: $2
  29.     mem_type: $3
  30.     qty: $4
  32. - match: '*.*.executor.threadpool.*'
  33.   name: executor_tasks
  34.   labels:
  35.     application: $1
  36.     executor_id: $2
  37.     qty: $3
  39. - match: '*.*.BlockManager.*.*'
  40.   name: block_manager
  41.   labels:
  42.     application: $1
  43.     executor_id: $2
  44.     type: $3
  45.     qty: $4
  47. - match: DAGScheduler.*.*
  48.   name: DAG_scheduler
  49.   labels:
  50.     type: $1
  51.     qty: $2

启动命令

  1. ssh 192.168.9.155
  2. cd /home/graphite_exporter
  3. nohup ./graphite_exporter --graphite.mapping-config=graphite_exporter_spark_mapping &

安装Prometheus

配置文件

  1. # my global config
  2. global:
  3.   scrape_interval:     15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
  4.   evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
  5.   # scrape_timeout is set to the global default (10s).
  7. # Alertmanager configuration
  8. alerting:
  9.   alertmanagers:
  10.   - static_configs:
  11.     - targets:
  12.       # - alertmanager:9093
  14. # Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
  15. rule_files:
  16.   # - "first_rules.yml"
  17.   # - "second_rules.yml"
  19. # A scrape configuration containing exactly one endpoint to scrape:
  20. # Here it's Prometheus itself.
  21. #scrape_configs:
  22. #  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
  23. #  - job_name: 'prometheus'
  24. #
  25. #    # metrics_path defaults to '/metrics'
  26. #    # scheme defaults to 'http'.
  27. #
  28. #    static_configs:
  29. #    - targets: ['192.168.9.155:9090']
  32. scrape_configs:
  33.   - job_name: 'spark'
  34.     static_configs:
  35.     - targets: ['192.168.9.155:9108']

启动命令

  1. ssh 192.168.9.155
  2. cd /home/prometheus/prometheus-2.8.1.linux-amd64
  3. nohup ./prometheus --config.file=prometheus.yml &

Docker安装Grafana

访问地址

admin/cluster
http://192.168.9.155:3001/d/yJ1JPHTiz/spark-prometheus-metrics-json?orgId=1

配置监控大屏

  1. {
  2.   "annotations": {
  3.     "list": [
  4.       {
  5.         "builtIn": 1,
  6.         "datasource": "-- Grafana --",
  7.         "enable": true,
  8.         "hide": true,
  9.         "iconColor": "rgba(0, 211, 255, 1)",
  10.         "name": "Annotations & Alerts",
  11.         "type": "dashboard"
  12.       }
  13.     ]
  14.   },
  15.   "editable": true,
  16.   "gnetId": null,
  17.   "graphTooltip": 0,
  18.   "id": 4,
  19.   "iteration": 1644894489986,
  20.   "links": [],
  21.   "panels": [
  22.     {
  23.       "collapsed": false,
  24.       "datasource": null,
  25.       "gridPos": {
  26.         "h": 1,
  27.         "w": 24,
  28.         "x": 0,
  29.         "y": 0
  30.       },
  31.       "id": 34,
  32.       "panels": [],
  33.       "title": "执行器内存指标",
  34.       "type": "row"
  35.     },
  36.     {
  37.       "aliasColors": {},
  38.       "bars": false,
  39.       "dashLength": 10,
  40.       "dashes": false,
  41.       "datasource": "Prometheus",
  42.       "fieldConfig": {
  43.         "defaults": {
  44.           "custom": {}
  45.         },
  46.         "overrides": []
  47.       },
  48.       "fill": 1,
  49.       "fillGradient": 0,
  50.       "gridPos": {
  51.         "h": 7,
  52.         "w": 12,
  53.         "x": 0,
  54.         "y": 1
  55.       },
  56.       "hiddenSeries": false,
  57.       "id": 14,
  58.       "legend": {
  59.         "avg": false,
  60.         "current": true,
  61.         "max": false,
  62.         "min": false,
  63.         "show": true,
  64.         "total": false,
  65.         "values": true
  66.       },
  67.       "lines": true,
  68.       "linewidth": 1,
  69.       "links": [],
  70.       "nullPointMode": "null",
  71.       "options": {
  72.         "alertThreshold": true
  73.       },
  74.       "percentage": false,
  75.       "pluginVersion": "7.3.6",
  76.       "pointradius": 5,
  77.       "points": false,
  78.       "renderer": "flot",
  79.       "seriesOverrides": [
  80.         {
  81.           "alias": "heap_max",
  82.           "yaxis": 1
  83.         }
  84.       ],
  85.       "spaceLength": 10,
  86.       "stack": false,
  87.       "steppedLine": false,
  88.       "targets": [
  89.         {
  90.           "expr": "jvm_memory_usage{mem_type=\"heap\", qty=\"used\", application=\"$application_ID\", executor_id!~\"driver\"}",
  91.           "format": "time_series",
  92.           "hide": false,
  93.           "interval": "",
  94.           "intervalFactor": 1,
  95.           "legendFormat": "{{executor_id}}",
  96.           "refId": "A"
  97.         },
  98.         {
  99.           "expr": "max(jvm_memory_usage{mem_type=\"heap\", qty=\"max\", application=\"$application_ID\", executor_id!~\"driver\"})",
  100.           "format": "time_series",
  101.           "hide": false,
  102.           "intervalFactor": 1,
  103.           "legendFormat": "heap_max",
  104.           "refId": "B"
  105.         }
  106.       ],
  107.       "thresholds": [],
  108.       "timeFrom": null,
  109.       "timeRegions": [],
  110.       "timeShift": null,
  111.       "title": "执行器堆内存使用量",
  112.       "tooltip": {
  113.         "shared": true,
  114.         "sort": 0,
  115.         "value_type": "individual"
  116.       },
  117.       "type": "graph",
  118.       "xaxis": {
  119.         "buckets": null,
  120.         "mode": "time",
  121.         "name": null,
  122.         "show": true,
  123.         "values": []
  124.       },
  125.       "yaxes": [
  126.         {
  127.           "format": "decbytes",
  128.           "label": null,
  129.           "logBase": 1,
  130.           "max": null,
  131.           "min": null,
  132.           "show": true
  133.         },
  134.         {
  135.           "format": "short",
  136.           "label": null,
  137.           "logBase": 1,
  138.           "max": null,
  139.           "min": null,
  140.           "show": true
  141.         }
  142.       ],
  143.       "yaxis": {
  144.         "align": false,
  145.         "alignLevel": null
  146.       }
  147.     },
  148.     {
  149.       "aliasColors": {},
  150.       "bars": false,
  151.       "dashLength": 10,
  152.       "dashes": false,
  153.       "datasource": null,
  154.       "decimals": 2,
  155.       "fieldConfig": {
  156.         "defaults": {
  157.           "custom": {}
  158.         },
  159.         "overrides": []
  160.       },
  161.       "fill": 1,
  162.       "fillGradient": 0,
  163.       "gridPos": {
  164.         "h": 7,
  165.         "w": 12,
  166.         "x": 12,
  167.         "y": 1
  168.       },
  169.       "hiddenSeries": false,
  170.       "id": 18,
  171.       "legend": {
  172.         "avg": false,
  173.         "current": false,
  174.         "max": true,
  175.         "min": false,
  176.         "show": true,
  177.         "total": false,
  178.         "values": true
  179.       },
  180.       "lines": true,
  181.       "linewidth": 1,
  182.       "links": [],
  183.       "nullPointMode": "null",
  184.       "options": {
  185.         "alertThreshold": true
  186.       },
  187.       "percentage": false,
  188.       "pluginVersion": "7.3.6",
  189.       "pointradius": 5,
  190.       "points": false,
  191.       "renderer": "flot",
  192.       "seriesOverrides": [],
  193.       "spaceLength": 10,
  194.       "stack": false,
  195.       "steppedLine": false,
  196.       "targets": [
  197.         {
  198.           "expr": "max(jvm_memory_usage{mem_type=\"heap\", qty=\"usage\", application=\"$application_ID\", executor_id!~\"driver\"})",
  199.           "format": "time_series",
  200.           "intervalFactor": 1,
  201.           "legendFormat": "max_usage",
  202.           "refId": "A"
  203.         }
  204.       ],
  205.       "thresholds": [],
  206.       "timeFrom": null,
  207.       "timeRegions": [],
  208.       "timeShift": null,
  209.       "title": "执行器最大堆内存使用量",
  210.       "tooltip": {
  211.         "shared": true,
  212.         "sort": 0,
  213.         "value_type": "individual"
  214.       },
  215.       "type": "graph",
  216.       "xaxis": {
  217.         "buckets": null,
  218.         "mode": "time",
  219.         "name": null,
  220.         "show": true,
  221.         "values": []
  222.       },
  223.       "yaxes": [
  224.         {
  225.           "format": "short",
  226.           "label": null,
  227.           "logBase": 1,
  228.           "max": null,
  229.           "min": null,
  230.           "show": true
  231.         },
  232.         {
  233.           "format": "short",
  234.           "label": null,
  235.           "logBase": 1,
  236.           "max": null,
  237.           "min": null,
  238.           "show": true
  239.         }
  240.       ],
  241.       "yaxis": {
  242.         "align": false,
  243.         "alignLevel": null
  244.       }
  245.     },
  246.     {
  247.       "aliasColors": {},
  248.       "bars": false,
  249.       "dashLength": 10,
  250.       "dashes": false,
  251.       "datasource": "Prometheus",
  252.       "fieldConfig": {
  253.         "defaults": {
  254.           "custom": {}
  255.         },
  256.         "overrides": []
  257.       },
  258.       "fill": 1,
  259.       "fillGradient": 0,
  260.       "gridPos": {
  261.         "h": 7,
  262.         "w": 8,
  263.         "x": 0,
  264.         "y": 8
  265.       },
  266.       "hiddenSeries": false,
  267.       "id": 16,
  268.       "legend": {
  269.         "avg": false,
  270.         "current": true,
  271.         "max": false,
  272.         "min": false,
  273.         "show": true,
  274.         "total": false,
  275.         "values": true
  276.       },
  277.       "lines": true,
  278.       "linewidth": 1,
  279.       "links": [],
  280.       "nullPointMode": "null",
  281.       "options": {
  282.         "alertThreshold": true
  283.       },
  284.       "percentage": false,
  285.       "pluginVersion": "7.3.6",
  286.       "pointradius": 5,
  287.       "points": false,
  288.       "renderer": "flot",
  289.       "seriesOverrides": [],
  290.       "spaceLength": 10,
  291.       "stack": false,
  292.       "steppedLine": false,
  293.       "targets": [
  294.         {
  295.           "expr": "jvm_gcTime_count{application=\"$application_ID\"} / 1000",
  296.           "format": "time_series",
  297.           "hide": false,
  298.           "intervalFactor": 1,
  299.           "legendFormat": "{{executor_id}} ",
  300.           "refId": "A"
  301.         }
  302.       ],
  303.       "thresholds": [],
  304.       "timeFrom": null,
  305.       "timeRegions": [],
  306.       "timeShift": null,
  307.       "title": "执行器GC总时长",
  308.       "tooltip": {
  309.         "shared": true,
  310.         "sort": 0,
  311.         "value_type": "individual"
  312.       },
  313.       "type": "graph",
  314.       "xaxis": {
  315.         "buckets": null,
  316.         "mode": "time",
  317.         "name": null,
  318.         "show": true,
  319.         "values": []
  320.       },
  321.       "yaxes": [
  322.         {
  323.           "format": "short",
  324.           "label": null,
  325.           "logBase": 1,
  326.           "max": null,
  327.           "min": null,
  328.           "show": true
  329.         },
  330.         {
  331.           "format": "short",
  332.           "label": null,
  333.           "logBase": 1,
  334.           "max": null,
  335.           "min": null,
  336.           "show": true
  337.         }
  338.       ],
  339.       "yaxis": {
  340.         "align": false,
  341.         "alignLevel": null
  342.       }
  343.     },
  344.     {
  345.       "aliasColors": {},
  346.       "bars": false,
  347.       "dashLength": 10,
  348.       "dashes": false,
  349.       "datasource": "Prometheus",
  350.       "editable": true,
  351.       "error": false,
  352.       "fieldConfig": {
  353.         "defaults": {
  354.           "custom": {}
  355.         },
  356.         "overrides": []
  357.       },
  358.       "fill": 0,
  359.       "fillGradient": 0,
  360.       "grid": {},
  361.       "gridPos": {
  362.         "h": 7,
  363.         "w": 8,
  364.         "x": 8,
  365.         "y": 8
  366.       },
  367.       "hiddenSeries": false,
  368.       "id": 9,
  369.       "legend": {
  370.         "avg": false,
  371.         "current": false,
  372.         "max": false,
  373.         "min": false,
  374.         "show": true,
  375.         "total": false,
  376.         "values": false
  377.       },
  378.       "lines": true,
  379.       "linewidth": 1,
  380.       "links": [],
  381.       "nullPointMode": "connected",
  382.       "options": {
  383.         "alertThreshold": true
  384.       },
  385.       "percentage": false,
  386.       "pluginVersion": "7.3.6",
  387.       "pointradius": 5,
  388.       "points": false,
  389.       "renderer": "flot",
  390.       "seriesOverrides": [],
  391.       "spaceLength": 10,
  392.       "stack": false,
  393.       "steppedLine": false,
  394.       "targets": [
  395.         {
  396.           "expr": "jvm_memory_pools{executor_id!~\"driver\", application=\"$application_ID\",qty=\"usage\", mem_type=\"PS-Old-Gen\"}",
  397.           "format": "time_series",
  398.           "intervalFactor": 2,
  399.           "legendFormat": "{{executor_id}}",
  400.           "metric": "jvm_memory_pools",
  401.           "refId": "A",
  402.           "step": 2,
  403.           "target": ""
  404.         }
  405.       ],
  406.       "thresholds": [],
  407.       "timeFrom": null,
  408.       "timeRegions": [],
  409.       "timeShift": null,
  410.       "title": "执行器老年代使用量",
  411.       "tooltip": {
  412.         "shared": true,
  413.         "sort": 0,
  414.         "value_type": "cumulative"
  415.       },
  416.       "type": "graph",
  417.       "xaxis": {
  418.         "buckets": null,
  419.         "mode": "time",
  420.         "name": null,
  421.         "show": true,
  422.         "values": []
  423.       },
  424.       "yaxes": [
  425.         {
  426.           "format": "short",
  427.           "logBase": 1,
  428.           "max": null,
  429.           "min": null,
  430.           "show": true
  431.         },
  432.         {
  433.           "format": "short",
  434.           "logBase": 1,
  435.           "max": null,
  436.           "min": null,
  437.           "show": true
  438.         }
  439.       ],
  440.       "yaxis": {
  441.         "align": false,
  442.         "alignLevel": null
  443.       }
  444.     },
  445.     {
  446.       "aliasColors": {},
  447.       "bars": false,
  448.       "dashLength": 10,
  449.       "dashes": false,
  450.       "datasource": "Prometheus",
  451.       "editable": true,
  452.       "error": false,
  453.       "fieldConfig": {
  454.         "defaults": {
  455.           "custom": {}
  456.         },
  457.         "overrides": []
  458.       },
  459.       "fill": 0,
  460.       "fillGradient": 0,
  461.       "grid": {},
  462.       "gridPos": {
  463.         "h": 7,
  464.         "w": 8,
  465.         "x": 16,
  466.         "y": 8
  467.       },
  468.       "hiddenSeries": false,
  469.       "id": 7,
  470.       "legend": {
  471.         "avg": false,
  472.         "current": false,
  473.         "max": false,
  474.         "min": false,
  475.         "show": false,
  476.         "total": false,
  477.         "values": false
  478.       },
  479.       "lines": true,
  480.       "linewidth": 1,
  481.       "links": [],
  482.       "nullPointMode": "connected",
  483.       "options": {
  484.         "alertThreshold": true
  485.       },
  486.       "percentage": false,
  487.       "pluginVersion": "7.3.6",
  488.       "pointradius": 5,
  489.       "points": false,
  490.       "renderer": "flot",
  491.       "seriesOverrides": [],
  492.       "spaceLength": 10,
  493.       "stack": false,
  494.       "steppedLine": false,
  495.       "targets": [
  496.         {
  497.           "expr": "jvm_memory_pools{executor_id!~\"driver\", application=\"$application_ID\",qty=\"usage\", mem_type=\"PS-Eden-Space\"}",
  498.           "format": "time_series",
  499.           "hide": false,
  500.           "intervalFactor": 2,
  501.           "legendFormat": "{{executor_id}}",
  502.           "metric": "jvm_memory_pools",
  503.           "refId": "A",
  504.           "step": 2,
  505.           "target": ""
  506.         }
  507.       ],
  508.       "thresholds": [],
  509.       "timeFrom": null,
  510.       "timeRegions": [],
  511.       "timeShift": null,
  512.       "title": "执行器年轻代使用量",
  513.       "tooltip": {
  514.         "shared": true,
  515.         "sort": 0,
  516.         "value_type": "cumulative"
  517.       },
  518.       "type": "graph",
  519.       "xaxis": {
  520.         "buckets": null,
  521.         "mode": "time",
  522.         "name": null,
  523.         "show": true,
  524.         "values": []
  525.       },
  526.       "yaxes": [
  527.         {
  528.           "format": "short",
  529.           "logBase": 1,
  530.           "max": null,
  531.           "min": null,
  532.           "show": true
  533.         },
  534.         {
  535.           "format": "short",
  536.           "logBase": 1,
  537.           "max": null,
  538.           "min": null,
  539.           "show": true
  540.         }
  541.       ],
  542.       "yaxis": {
  543.         "align": false,
  544.         "alignLevel": null
  545.       }
  546.     },
  547.     {
  548.       "collapsed": false,
  549.       "datasource": null,
  550.       "gridPos": {
  551.         "h": 1,
  552.         "w": 24,
  553.         "x": 0,
  554.         "y": 15
  555.       },
  556.       "id": 32,
  557.       "panels": [],
  558.       "title": "驱动器内存指标",
  559.       "type": "row"
  560.     },
  561.     {
  562.       "aliasColors": {},
  563.       "bars": false,
  564.       "dashLength": 10,
  565.       "dashes": false,
  566.       "datasource": "Prometheus",
  567.       "decimals": null,
  568.       "fieldConfig": {
  569.         "defaults": {
  570.           "custom": {}
  571.         },
  572.         "overrides": []
  573.       },
  574.       "fill": 1,
  575.       "fillGradient": 0,
  576.       "gridPos": {
  577.         "h": 7,
  578.         "w": 12,
  579.         "x": 0,
  580.         "y": 16
  581.       },
  582.       "hiddenSeries": false,
  583.       "id": 12,
  584.       "legend": {
  585.         "avg": false,
  586.         "current": true,
  587.         "hideEmpty": false,
  588.         "hideZero": false,
  589.         "max": false,
  590.         "min": false,
  591.         "show": true,
  592.         "total": false,
  593.         "values": true
  594.       },
  595.       "lines": true,
  596.       "linewidth": 1,
  597.       "links": [],
  598.       "nullPointMode": "null",
  599.       "options": {
  600.         "alertThreshold": true
  601.       },
  602.       "percentage": false,
  603.       "pluginVersion": "7.3.6",
  604.       "pointradius": 5,
  605.       "points": false,
  606.       "renderer": "flot",
  607.       "seriesOverrides": [
  608.         {
  609.           "alias": "max",
  610.           "yaxis": 1
  611.         },
  612.         {
  613.           "alias": "max_usage",
  614.           "yaxis": 2
  615.         },
  616.         {
  617.           "alias": "usage",
  618.           "yaxis": 2
  619.         }
  620.       ],
  621.       "spaceLength": 10,
  622.       "stack": false,
  623.       "steppedLine": false,
  624.       "targets": [
  625.         {
  626.           "expr": "jvm_memory_usage{mem_type=\"heap\", qty=\"used\", application=\"$application_ID\", executor_id=\"driver\"}",
  627.           "format": "time_series",
  628.           "hide": false,
  629.           "instant": false,
  630.           "intervalFactor": 1,
  631.           "legendFormat": "{{qty}}",
  632.           "refId": "A"
  633.         },
  634.         {
  635.           "expr": "jvm_memory_usage{mem_type=\"heap\", qty=\"max\", application=\"$application_ID\", executor_id=\"driver\"}",
  636.           "format": "time_series",
  637.           "hide": false,
  638.           "instant": false,
  639.           "interval": "",
  640.           "intervalFactor": 1,
  641.           "legendFormat": "{{qty}}",
  642.           "refId": "B"
  643.         }
  644.       ],
  645.       "thresholds": [],
  646.       "timeFrom": null,
  647.       "timeRegions": [],
  648.       "timeShift": null,
  649.       "title": "驱动器堆内存使用量",
  650.       "tooltip": {
  651.         "shared": true,
  652.         "sort": 0,
  653.         "value_type": "individual"
  654.       },
  655.       "type": "graph",
  656.       "xaxis": {
  657.         "buckets": null,
  658.         "mode": "time",
  659.         "name": null,
  660.         "show": true,
  661.         "values": []
  662.       },
  663.       "yaxes": [
  664.         {
  665.           "format": "decbytes",
  666.           "label": null,
  667.           "logBase": 1,
  668.           "max": null,
  669.           "min": null,
  670.           "show": true
  671.         },
  672.         {
  673.           "format": "short",
  674.           "label": null,
  675.           "logBase": 1,
  676.           "max": null,
  677.           "min": null,
  678.           "show": true
  679.         }
  680.       ],
  681.       "yaxis": {
  682.         "align": false,
  683.         "alignLevel": null
  684.       }
  685.     },
  686.     {
  687.       "aliasColors": {},
  688.       "bars": false,
  689.       "dashLength": 10,
  690.       "dashes": false,
  691.       "datasource": "Prometheus",
  692.       "editable": true,
  693.       "error": false,
  694.       "fieldConfig": {
  695.         "defaults": {
  696.           "custom": {}
  697.         },
  698.         "overrides": []
  699.       },
  700.       "fill": 1,
  701.       "fillGradient": 0,
  702.       "grid": {},
  703.       "gridPos": {
  704.         "h": 7,
  705.         "w": 12,
  706.         "x": 12,
  707.         "y": 16
  708.       },
  709.       "hiddenSeries": false,
  710.       "id": 5,
  711.       "legend": {
  712.         "avg": false,
  713.         "current": false,
  714.         "max": false,
  715.         "min": false,
  716.         "show": true,
  717.         "total": false,
  718.         "values": false
  719.       },
  720.       "lines": true,
  721.       "linewidth": 2,
  722.       "links": [],
  723.       "nullPointMode": "connected",
  724.       "options": {
  725.         "alertThreshold": true
  726.       },
  727.       "percentage": false,
  728.       "pluginVersion": "7.3.6",
  729.       "pointradius": 5,
  730.       "points": false,
  731.       "renderer": "flot",
  732.       "seriesOverrides": [],
  733.       "spaceLength": 10,
  734.       "stack": false,
  735.       "steppedLine": false,
  736.       "targets": [
  737.         {
  738.           "expr": "jvm_memory_pools{executor_id=\"driver\", application=\"$application_ID\",qty=\"used\"}",
  739.           "format": "time_series",
  740.           "intervalFactor": 2,
  741.           "legendFormat": "{{mem_type}}",
  742.           "metric": "jvm_memory_pools",
  743.           "refId": "A",
  744.           "step": 2,
  745.           "target": ""
  746.         }
  747.       ],
  748.       "thresholds": [],
  749.       "timeFrom": null,
  750.       "timeRegions": [],
  751.       "timeShift": null,
  752.       "title": "驱动器Java虚拟机初始化内存（JVM Memory Pools）",
  753.       "tooltip": {
  754.         "shared": true,
  755.         "sort": 0,
  756.         "value_type": "cumulative"
  757.       },
  758.       "type": "graph",
  759.       "xaxis": {
  760.         "buckets": null,
  761.         "mode": "time",
  762.         "name": null,
  763.         "show": true,
  764.         "values": []
  765.       },
  766.       "yaxes": [
  767.         {
  768.           "format": "decbytes",
  769.           "logBase": 1,
  770.           "max": null,
  771.           "min": null,
  772.           "show": true
  773.         },
  774.         {
  775.           "format": "short",
  776.           "logBase": 1,
  777.           "max": null,
  778.           "min": null,
  779.           "show": true
  780.         }
  781.       ],
  782.       "yaxis": {
  783.         "align": false,
  784.         "alignLevel": null
  785.       }
  786.     },
  787.     {
  788.       "collapsed": false,
  789.       "datasource": null,
  790.       "gridPos": {
  791.         "h": 1,
  792.         "w": 24,
  793.         "x": 0,
  794.         "y": 23
  795.       },
  796.       "id": 28,
  797.       "panels": [],
  798.       "title": "HDFS 读取和写入速率",
  799.       "type": "row"
  800.     },
  801.     {
  802.       "aliasColors": {},
  803.       "bars": false,
  804.       "dashLength": 10,
  805.       "dashes": false,
  806.       "datasource": "Prometheus",
  807.       "editable": true,
  808.       "error": false,
  809.       "fieldConfig": {
  810.         "defaults": {
  811.           "custom": {}
  812.         },
  813.         "overrides": []
  814.       },
  815.       "fill": 0,
  816.       "fillGradient": 0,
  817.       "grid": {},
  818.       "gridPos": {
  819.         "h": 7,
  820.         "w": 12,
  821.         "x": 0,
  822.         "y": 24
  823.       },
  824.       "hiddenSeries": false,
  825.       "id": 3,
  826.       "legend": {
  827.         "avg": false,
  828.         "current": false,
  829.         "max": false,
  830.         "min": false,
  831.         "show": true,
  832.         "total": false,
  833.         "values": false
  834.       },
  835.       "lines": true,
  836.       "linewidth": 1,
  837.       "links": [],
  838.       "nullPointMode": "connected",
  839.       "options": {
  840.         "alertThreshold": true
  841.       },
  842.       "percentage": false,
  843.       "pluginVersion": "7.3.6",
  844.       "pointradius": 5,
  845.       "points": false,
  846.       "renderer": "flot",
  847.       "seriesOverrides": [
  848.         {
  849.           "alias": "total",
  850.           "linewidth": 4
  851.         }
  852.       ],
  853.       "spaceLength": 10,
  854.       "stack": false,
  855.       "steppedLine": false,
  856.       "targets": [
  857.         {
  858.           "expr": "rate(filesystem_usage{qty=\"read_bytes\", fs_type=\"hdfs\", application=\"$application_ID\"}[1m])",
  859.           "format": "time_series",
  860.           "intervalFactor": 1,
  861.           "legendFormat": "{{executor_id}}",
  862.           "metric": "filesystem_usage",
  863.           "refId": "A",
  864.           "step": 1,
  865.           "target": ""
  866.         },
  867.         {
  868.           "expr": "sum(rate(filesystem_usage{qty=\"read_bytes\", fs_type=\"hdfs\", application=\"$application_ID\"}[1m]))",
  869.           "format": "time_series",
  870.           "hide": false,
  871.           "intervalFactor": 1,
  872.           "legendFormat": "total",
  873.           "metric": "filesystem_usage",
  874.           "refId": "B",
  875.           "step": 1,
  876.           "target": ""
  877.         }
  878.       ],
  879.       "thresholds": [],
  880.       "timeFrom": null,
  881.       "timeRegions": [],
  882.       "timeShift": null,
  883.       "title": "HDFS读取速率",
  884.       "tooltip": {
  885.         "shared": true,
  886.         "sort": 0,
  887.         "value_type": "cumulative"
  888.       },
  889.       "type": "graph",
  890.       "xaxis": {
  891.         "buckets": null,
  892.         "mode": "time",
  893.         "name": null,
  894.         "show": true,
  895.         "values": []
  896.       },
  897.       "yaxes": [
  898.         {
  899.           "format": "Bps",
  900.           "logBase": 1,
  901.           "max": null,
  902.           "min": null,
  903.           "show": true
  904.         },
  905.         {
  906.           "format": "short",
  907.           "logBase": 1,
  908.           "max": null,
  909.           "min": null,
  910.           "show": true
  911.         }
  912.       ],
  913.       "yaxis": {
  914.         "align": false,
  915.         "alignLevel": null
  916.       }
  917.     },
  918.     {
  919.       "aliasColors": {},
  920.       "bars": false,
  921.       "dashLength": 10,
  922.       "dashes": false,
  923.       "datasource": "Prometheus",
  924.       "editable": true,
  925.       "error": false,
  926.       "fieldConfig": {
  927.         "defaults": {
  928.           "custom": {}
  929.         },
  930.         "overrides": []
  931.       },
  932.       "fill": 0,
  933.       "fillGradient": 0,
  934.       "grid": {},
  935.       "gridPos": {
  936.         "h": 7,
  937.         "w": 12,
  938.         "x": 12,
  939.         "y": 24
  940.       },
  941.       "hiddenSeries": false,
  942.       "id": 30,
  943.       "legend": {
  944.         "avg": false,
  945.         "current": false,
  946.         "max": false,
  947.         "min": false,
  948.         "show": true,
  949.         "total": false,
  950.         "values": false
  951.       },
  952.       "lines": true,
  953.       "linewidth": 1,
  954.       "links": [],
  955.       "nullPointMode": "connected",
  956.       "options": {
  957.         "alertThreshold": true
  958.       },
  959.       "percentage": false,
  960.       "pluginVersion": "7.3.6",
  961.       "pointradius": 5,
  962.       "points": false,
  963.       "renderer": "flot",
  964.       "seriesOverrides": [
  965.         {
  966.           "alias": "total",
  967.           "linewidth": 4,
  968.           "yaxis": 1
  969.         }
  970.       ],
  971.       "spaceLength": 10,
  972.       "stack": false,
  973.       "steppedLine": false,
  974.       "targets": [
  975.         {
  976.           "expr": "rate(filesystem_usage{qty=\"write_bytes\", fs_type=\"hdfs\", application=\"$application_ID\"}[1m])",
  977.           "format": "time_series",
  978.           "intervalFactor": 1,
  979.           "legendFormat": "{{executor_id}}",
  980.           "metric": "filesystem_usage",
  981.           "refId": "A",
  982.           "step": 1,
  983.           "target": ""
  984.         },
  985.         {
  986.           "expr": "sum(rate(filesystem_usage{qty=\"write_bytes\", fs_type=\"hdfs\", application=\"$application_ID\"}[1m]))",
  987.           "format": "time_series",
  988.           "hide": false,
  989.           "intervalFactor": 1,
  990.           "legendFormat": "total",
  991.           "metric": "filesystem_usage",
  992.           "refId": "B",
  993.           "step": 1,
  994.           "target": ""
  995.         }
  996.       ],
  997.       "thresholds": [],
  998.       "timeFrom": null,
  999.       "timeRegions": [],
  1000.       "timeShift": null,
  1001.       "title": "HDFS写入速率",
  1002.       "tooltip": {
  1003.         "shared": true,
  1004.         "sort": 0,
  1005.         "value_type": "cumulative"
  1006.       },
  1007.       "type": "graph",
  1008.       "xaxis": {
  1009.         "buckets": null,
  1010.         "mode": "time",
  1011.         "name": null,
  1012.         "show": true,
  1013.         "values": []
  1014.       },
  1015.       "yaxes": [
  1016.         {
  1017.           "format": "Bps",
  1018.           "logBase": 1,
  1019.           "max": null,
  1020.           "min": null,
  1021.           "show": true
  1022.         },
  1023.         {
  1024.           "format": "short",
  1025.           "logBase": 1,
  1026.           "max": null,
  1027.           "min": null,
  1028.           "show": true
  1029.         }
  1030.       ],
  1031.       "yaxis": {
  1032.         "align": false,
  1033.         "alignLevel": null
  1034.       }
  1035.     },
  1036.     {
  1037.       "collapsed": false,
  1038.       "datasource": null,
  1039.       "gridPos": {
  1040.         "h": 1,
  1041.         "w": 24,
  1042.         "x": 0,
  1043.         "y": 31
  1044.       },
  1045.       "id": 22,
  1046.       "panels": [],
  1047.       "title": "HDFS每个执行器读取和写入速率",
  1048.       "type": "row"
  1049.     },
  1050.     {
  1051.       "aliasColors": {
  1052.         "total": "#6ed0e0"
  1053.       },
  1054.       "bars": false,
  1055.       "dashLength": 10,
  1056.       "dashes": false,
  1057.       "datasource": "Prometheus",
  1058.       "editable": true,
  1059.       "error": false,
  1060.       "fieldConfig": {
  1061.         "defaults": {
  1062.           "custom": {}
  1063.         },
  1064.         "overrides": []
  1065.       },
  1066.       "fill": 1,
  1067.       "fillGradient": 0,
  1068.       "grid": {},
  1069.       "gridPos": {
  1070.         "h": 7,
  1071.         "w": 12,
  1072.         "x": 0,
  1073.         "y": 32
  1074.       },
  1075.       "hiddenSeries": false,
  1076.       "id": 1,
  1077.       "legend": {
  1078.         "avg": false,
  1079.         "current": false,
  1080.         "max": false,
  1081.         "min": false,
  1082.         "show": true,
  1083.         "total": false,
  1084.         "values": false
  1085.       },
  1086.       "lines": true,
  1087.       "linewidth": 1,
  1088.       "links": [],
  1089.       "nullPointMode": "connected",
  1090.       "options": {
  1091.         "alertThreshold": true
  1092.       },
  1093.       "percentage": false,
  1094.       "pluginVersion": "7.3.6",
  1095.       "pointradius": 5,
  1096.       "points": false,
  1097.       "renderer": "flot",
  1098.       "seriesOverrides": [
  1099.         {
  1100.           "alias": "total",
  1101.           "linewidth": 4
  1102.         }
  1103.       ],
  1104.       "spaceLength": 10,
  1105.       "stack": false,
  1106.       "steppedLine": false,
  1107.       "targets": [
  1108.         {
  1109.           "expr": "filesystem_usage{qty=\"read_bytes\", fs_type=\"hdfs\", application=\"$application_ID\"}",
  1110.           "format": "time_series",
  1111.           "hide": false,
  1112.           "instant": false,
  1113.           "intervalFactor": 1,
  1114.           "legendFormat": "{{executor_id}}",
  1115.           "metric": "filesystem_usage",
  1116.           "refId": "A",
  1117.           "step": 2,
  1118.           "target": ""
  1119.         },
  1120.         {
  1121.           "expr": "sum(filesystem_usage{qty=\"read_bytes\", fs_type=\"hdfs\", application=\"$application_ID\"})",
  1122.           "format": "time_series",
  1123.           "intervalFactor": 1,
  1124.           "legendFormat": "total",
  1125.           "refId": "B"
  1126.         }
  1127.       ],
  1128.       "thresholds": [],
  1129.       "timeFrom": null,
  1130.       "timeRegions": [],
  1131.       "timeShift": null,
  1132.       "title": "执行器HDFS读取速率",
  1133.       "tooltip": {
  1134.         "shared": true,
  1135.         "sort": 0,
  1136.         "value_type": "cumulative"
  1137.       },
  1138.       "type": "graph",
  1139.       "xaxis": {
  1140.         "buckets": null,
  1141.         "mode": "time",
  1142.         "name": null,
  1143.         "show": true,
  1144.         "values": []
  1145.       },
  1146.       "yaxes": [
  1147.         {
  1148.           "format": "decbytes",
  1149.           "logBase": 1,
  1150.           "max": null,
  1151.           "min": 0,
  1152.           "show": true
  1153.         },
  1154.         {
  1155.           "format": "short",
  1156.           "logBase": 1,
  1157.           "max": null,
  1158.           "min": null,
  1159.           "show": true
  1160.         }
  1161.       ],
  1162.       "yaxis": {
  1163.         "align": false,
  1164.         "alignLevel": null
  1165.       }
  1166.     },
  1167.     {
  1168.       "aliasColors": {},
  1169.       "bars": false,
  1170.       "dashLength": 10,
  1171.       "dashes": false,
  1172.       "datasource": "Prometheus",
  1173.       "editable": true,
  1174.       "error": false,
  1175.       "fieldConfig": {
  1176.         "defaults": {
  1177.           "custom": {}
  1178.         },
  1179.         "overrides": []
  1180.       },
  1181.       "fill": 1,
  1182.       "fillGradient": 0,
  1183.       "grid": {},
  1184.       "gridPos": {
  1185.         "h": 7,
  1186.         "w": 12,
  1187.         "x": 12,
  1188.         "y": 32
  1189.       },
  1190.       "hiddenSeries": false,
  1191.       "id": 26,
  1192.       "legend": {
  1193.         "avg": false,
  1194.         "current": false,
  1195.         "max": false,
  1196.         "min": false,
  1197.         "show": true,
  1198.         "total": false,
  1199.         "values": false
  1200.       },
  1201.       "lines": true,
  1202.       "linewidth": 1,
  1203.       "links": [],
  1204.       "nullPointMode": "connected",
  1205.       "options": {
  1206.         "alertThreshold": true
  1207.       },
  1208.       "percentage": false,
  1209.       "pluginVersion": "7.3.6",
  1210.       "pointradius": 5,
  1211.       "points": false,
  1212.       "renderer": "flot",
  1213.       "seriesOverrides": [
  1214.         {
  1215.           "alias": "total",
  1216.           "linewidth": 4,
  1217.           "yaxis": 1
  1218.         }
  1219.       ],
  1220.       "spaceLength": 10,
  1221.       "stack": false,
  1222.       "steppedLine": false,
  1223.       "targets": [
  1224.         {
  1225.           "expr": "filesystem_usage{qty=\"write_bytes\", fs_type=\"hdfs\", application=\"$application_ID\"}",
  1226.           "format": "time_series",
  1227.           "hide": false,
  1228.           "intervalFactor": 2,
  1229.           "legendFormat": "{{executor_id}}",
  1230.           "metric": "filesystem_usage",
  1231.           "refId": "A",
  1232.           "step": 2,
  1233.           "target": ""
  1234.         },
  1235.         {
  1236.           "expr": "sum(filesystem_usage{qty=\"write_bytes\", fs_type=\"hdfs\", application=\"$application_ID\"})",
  1237.           "format": "time_series",
  1238.           "intervalFactor": 1,
  1239.           "legendFormat": "total",
  1240.           "refId": "B"
  1241.         }
  1242.       ],
  1243.       "thresholds": [],
  1244.       "timeFrom": null,
  1245.       "timeRegions": [],
  1246.       "timeShift": null,
  1247.       "title": "执行器HDFS写入速度",
  1248.       "tooltip": {
  1249.         "shared": true,
  1250.         "sort": 0,
  1251.         "value_type": "cumulative"
  1252.       },
  1253.       "type": "graph",
  1254.       "xaxis": {
  1255.         "buckets": null,
  1256.         "mode": "time",
  1257.         "name": null,
  1258.         "show": true,
  1259.         "values": []
  1260.       },
  1261.       "yaxes": [
  1262.         {
  1263.           "format": "decbytes",
  1264.           "logBase": 1,
  1265.           "max": null,
  1266.           "min": 0,
  1267.           "show": true
  1268.         },
  1269.         {
  1270.           "format": "short",
  1271.           "logBase": 1,
  1272.           "max": null,
  1273.           "min": null,
  1274.           "show": true
  1275.         }
  1276.       ],
  1277.       "yaxis": {
  1278.         "align": false,
  1279.         "alignLevel": null
  1280.       }
  1281.     },
  1282.     {
  1283.       "collapsed": false,
  1284.       "datasource": null,
  1285.       "gridPos": {
  1286.         "h": 1,
  1287.         "w": 24,
  1288.         "x": 0,
  1289.         "y": 39
  1290.       },
  1291.       "id": 36,
  1292.       "panels": [],
  1293.       "title": "任务完成指标",
  1294.       "type": "row"
  1295.     },
  1296.     {
  1297.       "aliasColors": {},
  1298.       "bars": false,
  1299.       "dashLength": 10,
  1300.       "dashes": false,
  1301.       "datasource": null,
  1302.       "fieldConfig": {
  1303.         "defaults": {
  1304.           "custom": {}
  1305.         },
  1306.         "overrides": []
  1307.       },
  1308.       "fill": 1,
  1309.       "fillGradient": 0,
  1310.       "gridPos": {
  1311.         "h": 7,
  1312.         "w": 12,
  1313.         "x": 0,
  1314.         "y": 40
  1315.       },
  1316.       "hiddenSeries": false,
  1317.       "id": 38,
  1318.       "legend": {
  1319.         "avg": false,
  1320.         "current": false,
  1321.         "max": false,
  1322.         "min": false,
  1323.         "show": true,
  1324.         "total": false,
  1325.         "values": false
  1326.       },
  1327.       "lines": true,
  1328.       "linewidth": 1,
  1329.       "links": [],
  1330.       "nullPointMode": "null",
  1331.       "options": {
  1332.         "alertThreshold": true
  1333.       },
  1334.       "percentage": false,
  1335.       "pluginVersion": "7.3.6",
  1336.       "pointradius": 5,
  1337.       "points": false,
  1338.       "renderer": "flot",
  1339.       "seriesOverrides": [],
  1340.       "spaceLength": 10,
  1341.       "stack": false,
  1342.       "steppedLine": false,
  1343.       "targets": [
  1344.         {
  1345.           "expr": "executor_tasks{application=\"$application_ID\",qty=\"completeTasks\"}",
  1346.           "format": "time_series",
  1347.           "intervalFactor": 1,
  1348.           "legendFormat": "{{executor_id}}",
  1349.           "refId": "A"
  1350.         }
  1351.       ],
  1352.       "thresholds": [],
  1353.       "timeFrom": null,
  1354.       "timeRegions": [],
  1355.       "timeShift": null,
  1356.       "title": "每个执行器完成的任务",
  1357.       "tooltip": {
  1358.         "shared": true,
  1359.         "sort": 0,
  1360.         "value_type": "individual"
  1361.       },
  1362.       "type": "graph",
  1363.       "xaxis": {
  1364.         "buckets": null,
  1365.         "mode": "time",
  1366.         "name": null,
  1367.         "show": true,
  1368.         "values": []
  1369.       },
  1370.       "yaxes": [
  1371.         {
  1372.           "format": "short",
  1373.           "label": null,
  1374.           "logBase": 1,
  1375.           "max": null,
  1376.           "min": null,
  1377.           "show": true
  1378.         },
  1379.         {
  1380.           "format": "short",
  1381.           "label": null,
  1382.           "logBase": 1,
  1383.           "max": null,
  1384.           "min": null,
  1385.           "show": true
  1386.         }
  1387.       ],
  1388.       "yaxis": {
  1389.         "align": false,
  1390.         "alignLevel": null
  1391.       }
  1392.     },
  1393.     {
  1394.       "aliasColors": {},
  1395.       "bars": false,
  1396.       "dashLength": 10,
  1397.       "dashes": false,
  1398.       "datasource": null,
  1399.       "fieldConfig": {
  1400.         "defaults": {
  1401.           "custom": {}
  1402.         },
  1403.         "overrides": []
  1404.       },
  1405.       "fill": 1,
  1406.       "fillGradient": 0,
  1407.       "gridPos": {
  1408.         "h": 7,
  1409.         "w": 12,
  1410.         "x": 12,
  1411.         "y": 40
  1412.       },
  1413.       "hiddenSeries": false,
  1414.       "id": 40,
  1415.       "legend": {
  1416.         "avg": false,
  1417.         "current": false,
  1418.         "max": false,
  1419.         "min": false,
  1420.         "show": true,
  1421.         "total": false,
  1422.         "values": false
  1423.       },
  1424.       "lines": true,
  1425.       "linewidth": 1,
  1426.       "links": [],
  1427.       "nullPointMode": "null",
  1428.       "options": {
  1429.         "alertThreshold": true
  1430.       },
  1431.       "percentage": false,
  1432.       "pluginVersion": "7.3.6",
  1433.       "pointradius": 5,
  1434.       "points": false,
  1435.       "renderer": "flot",
  1436.       "seriesOverrides": [],
  1437.       "spaceLength": 10,
  1438.       "stack": false,
  1439.       "steppedLine": false,
  1440.       "targets": [
  1441.         {
  1442.           "expr": "rate(executor_tasks{application=\"$application_ID\",qty=\"completeTasks\"}[1m])",
  1443.           "format": "time_series",
  1444.           "intervalFactor": 1,
  1445.           "legendFormat": "{{executor_id}}",
  1446.           "refId": "A"
  1447.         }
  1448.       ],
  1449.       "thresholds": [],
  1450.       "timeFrom": null,
  1451.       "timeRegions": [],
  1452.       "timeShift": null,
  1453.       "title": "每分钟完成的任务",
  1454.       "tooltip": {
  1455.         "shared": true,
  1456.         "sort": 0,
  1457.         "value_type": "individual"
  1458.       },
  1459.       "type": "graph",
  1460.       "xaxis": {
  1461.         "buckets": null,
  1462.         "mode": "time",
  1463.         "name": null,
  1464.         "show": true,
  1465.         "values": []
  1466.       },
  1467.       "yaxes": [
  1468.         {
  1469.           "format": "short",
  1470.           "label": null,
  1471.           "logBase": 1,
  1472.           "max": null,
  1473.           "min": null,
  1474.           "show": true
  1475.         },
  1476.         {
  1477.           "format": "short",
  1478.           "label": null,
  1479.           "logBase": 1,
  1480.           "max": null,
  1481.           "min": null,
  1482.           "show": true
  1483.         }
  1484.       ],
  1485.       "yaxis": {
  1486.         "align": false,
  1487.         "alignLevel": null
  1488.       }
  1489.     }
  1490.   ],
  1491.   "refresh": false,
  1492.   "schemaVersion": 26,
  1493.   "style": "dark",
  1494.   "tags": [],
  1495.   "templating": {
  1496.     "list": [
  1497.       {
  1498.         "allFormat": "glob",
  1499.         "allValue": null,
  1500.         "current": {
  1501.           "selected": false,
  1502.           "text": "application_1644566142502_0017",
  1503.           "value": "application_1644566142502_0017"
  1504.         },
  1505.         "datasource": "Prometheus",
  1506.         "definition": "",
  1507.         "error": null,
  1508.         "hide": 0,
  1509.         "includeAll": false,
  1510.         "label": "",
  1511.         "multi": false,
  1512.         "multiFormat": "glob",
  1513.         "name": "application_ID",
  1514.         "options": [],
  1515.         "query": "label_values(application)",
  1516.         "refresh": 1,
  1517.         "refresh_on_load": false,
  1518.         "regex": "",
  1519.         "skipUrlSync": false,
  1520.         "sort": 2,
  1521.         "tagValuesQuery": "",
  1522.         "tags": [],
  1523.         "tagsQuery": "",
  1524.         "type": "query",
  1525.         "useTags": false
  1526.       }
  1527.     ]
  1528.   },
  1529.   "time": {
  1530.     "from": "now-5m",
  1531.     "to": "now"
  1532.   },
  1533.   "timepicker": {
  1534.     "now": true,
  1535.     "refresh_intervals": [
  1536.       "5s",
  1537.       "10s",
  1538.       "30s",
  1539.       "1m",
  1540.       "5m",
  1541.       "15m",
  1542.       "30m",
  1543.       "1h",
  1544.       "2h",
  1545.       "1d"
  1546.     ],
  1547.     "time_options": [
  1548.       "5m",
  1549.       "15m",
  1550.       "1h",
  1551.       "6h",
  1552.       "12h",
  1553.       "24h",
  1554.       "2d",
  1555.       "7d",
  1556.       "30d"
  1557.     ]
  1558.   },
  1559.   "timezone": "browser",
  1560.   "title": "Spark Prometheus Metrics Json",
  1561.   "uid": "yJ1JPHTiz",
  1562.   "version": 7
  1563. }