部署


环境（Thanos Query）
[root@LF-MYSQL-136-130 thanos]# /root/k8s.sh sh01admin -n kvass get pod -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE
kvass-coordinator 1/1 Running 0 108m 11.80.14.238 11.80.0.51
kvass-metrics-b75b6466d-b4t5p 1/1 Running 0 142m 11.80.18.99 11.80.0.108
kvass-metrics-b75b6466d-b64x5 1/1 Running 0 142m 11.80.12.230 11.80.0.115
kvass-metrics-b75b6466d-dhgzx 1/1 Running 0 142m 11.80.31.30 11.80.0.29
kvass-metrics-b75b6466d-l7r8p 1/1 Running 0 142m 11.80.30.123 11.80.0.4
kvass-metrics-b75b6466d-mbrv8 1/1 Running 0 142m 11.80.14.226 11.80.0.34
kvass-metrics-b75b6466d-mshgn 1/1 Running 0 142m 11.80.22.163 11.80.0.112
kvass-prometheus1 1/1 Running 0 108m 11.80.31.36 11.80.0.72
kvass-prometheus2 1/1 Running 0 108m 11.80.22.179 11.80.0.112
kvass-prometheus3 1/1 Running 0 108m 11.80.22.188 11.80.0.116
kvass-thanos-query 1/1 Running 0 109m 11.80.14.230 11.80.0.34

prometheus.yaml
[root@kvass-prometheus1 config]# more prometheus.yaml
global:
scrapeinterval: 15s
evaluationinterval: 15s
externallabels:
cluster: custom
scrape_configs:
- job_name: ‘metrics-test’
tls_config:
insecure_skip_verify: true
kubernetes_sd_configs:
- api_server: “xxx”
role: pod
namespaces:
names: [“kvass”]
basic_auth:
username: “xxx”
password: “xxx”
tls_config:
insecure_skip_verify: true
relabel_configs:
- source_labels: [meta_kubernetes_pod_label_app_kubernetes_io_name]
regex: metrics
action: keep
- source_labels: [meta_kubernetes_pod_ip]
action: replace
regex: (.*)
replacement: ${1}:9091
target_label: __address
- source_labels:
- meta_kubernetes_pod_name
target_label: pod

coordinator
./kvass coordinator \
—shard.max-series=30000 \
—shard.selector=”component=kvass-prometheus” \
—shard.namespace=kvass \
—config.file=/export/prometheus/config_out/prometheus.yaml

thanos sidecar
./thanos sidecar \
—tsdb.path=/export/prometheus/data \
—prometheus.url=http://localhost:8080 \
—reloader.config-file=/export/prometheus/config/prometheus.yaml \
—reloader.config-envsubst-file=/export/prometheus/config_out/prometheus.env.yaml

kvass sidecar
./kvass sidecar \
—config.file=/export/prometheus/config_out/prometheus.env.yaml \
—config.output-file=/export/prometheus/config_out/prometheus_injected.yaml

prometheus_injected.yaml
[root@kvass-prometheus1 config_out]# more prometheus_injected.yaml
global:
scrape_interval: 15s
scrape_timeout: 10s
evaluation_interval: 15s
external_labels:
cluster: custom
scrape_configs:
- job_name: metrics-test
honor_timestamps: true
scrape_interval: 15s
scrape_timeout: 10s
metrics_path: /metrics
scheme: http
proxy_url: http://127.0.0.1:8008
relabel_configs:
- separator: ;
regex: invalid_label(.+)
replacement: $1
action: labelmap

prometheus
./prometheus \
—storage.tsdb.path=/export/prometheus/data \
—storage.tsdb.retention.time=3h \
—web.enable-lifecycle \
—storage.tsdb.no-lockfile \
—storage.tsdb.max-block-duration=2h \
—storage.tsdb.min-block-duration=2h \
—config.file=/export/prometheus/configout/prometheusinjected.yaml

thanos query
./thanos query \
—log.level=debug \
—query.auto-downsampling \
—grpc-address=0.0.0.0:10901 \
—http-address=0.0.0.0:80 \
—query.partial-response \
—query.replica-label=prometheusreplica \
—query.replica-label=rulereplica \
—store=11.80.31.36:10901 \
—store=11.80.22.179:10901 \
—store=11.80.22.188:10901


环境
[root@LF-MYSQL-136-130 kvass]# /root/k8s.sh sh01admin -n kvass get all -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE
pod/kvass-coordinator-6cb6b9f49c-d9zgg 2/2 Running 0 22m 11.80.31.36 11.80.0.51
pod/metrics-b75b6466d-4njhf 1/1 Running 0 78m 11.80.23.32 11.80.0.15
pod/metrics-b75b6466d-5lkvt 1/1 Running 0 78m 11.80.22.188 11.80.0.112
pod/metrics-b75b6466d-pr2sj 1/1 Running 0 78m 11.80.30.123 11.80.0.115
pod/metrics-b75b6466d-q96vd 1/1 Running 0 78m 11.80.22.179 11.80.0.51
pod/metrics-b75b6466d-qp4xr 1/1 Running 0 78m 11.80.23.66 11.80.0.108
pod/metrics-b75b6466d-sr8rx 1/1 Running 0 78m 11.80.23.63 11.80.0.4
pod/prometheus-rep-0-0 3/3 Running 0 11m 11.80.31.40 11.80.0.51
pod/prometheus-rep-0-1 3/3 Running 0 11m 11.80.22.158 11.80.0.112
pod/prometheus-rep-0-2 3/3 Running 0 11m 11.80.14.230 11.80.0.108
pod/thanos-query-5bd67644dc-2hk64 0/1 Running 0 3m42s 11.80.22.159 11.80.0.115

NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR
service/kvass-coordinator ClusterIP 10.0.236.229 11.80.15.115,11.80.18.99 9090/TCP 22m app.kubernetes.io/name=kvass-coordinator
service/prometheus ClusterIP None 8080/TCP,10901/TCP 15m app.kubernetes.io/name=prometheus
service/thanos-query ClusterIP 10.0.215.140 11.80.15.117,11.80.15.116 10901/TCP,9090/TCP 3m42s app.kubernetes.io/name=thanos-query

NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE CONTAINERS IMAGES SELECTOR
deployment.apps/kvass-coordinator 1 1 1 1 22m config-reload,kvass is.ht1.n.jd.local/vitesss/coreos-prometheus-config-reloader:v20210105190044,is.ht1.n.jd.local/vitesss/kvass:v20210105183443 app.kubernetes.io/name=kvass-coordinator
deployment.apps/metrics 6 6 6 6 78m metrics is.ht1.n.jd.local/vitesss/metrics-gen:latest app.kubernetes.io/name=metrics
deployment.apps/thanos-query 1 1 1 0 3m42s thanos-query is.ht1.n.jd.local/vitesss/thanos:v20210105185701 app.kubernetes.io/name=thanos-query

NAME DESIRED CURRENT READY AGE CONTAINERS IMAGES SELECTOR
replicaset.apps/kvass-coordinator-6cb6b9f49c 1 1 1 22m config-reload,kvass is.ht1.n.jd.local/vitesss/coreos-prometheus-config-reloader:v20210105190044,is.ht1.n.jd.local/vitesss/kvass:v20210105183443 app.kubernetes.io/name=kvass-coordinator,pod-template-hash=6cb6b9f49c
replicaset.apps/metrics-b75b6466d 6 6 6 78m metrics is.ht1.n.jd.local/vitesss/metrics-gen:latest app.kubernetes.io/name=metrics,pod-template-hash=b75b6466d
replicaset.apps/thanos-query-5bd67644dc 1 1 0 3m42s thanos-query is.ht1.n.jd.local/vitesss/thanos:v20210105185701 app.kubernetes.io/name=thanos-query,pod-template-hash=5bd67644dc

NAME DESIRED CURRENT AGE CONTAINERS IMAGES
statefulset.apps/prometheus-rep-0 3 3 11m thanos,kvass,prometheus is.ht1.n.jd.local/vitesss/thanos:v20210105185701,is.ht1.n.jd.local/vitesss/kvass:v20210105183443,is.ht1.n.jd.local/vitesss/prometheus:v20210105200056
[root@LF-MYSQL-136-130 kvass]#


kvass部署 - 图1


kvass部署 - 图2

kvass部署 - 图3

#coordinator
/kvass coordinator —shard.max-series=30000 —shard.selector=app.kubernetes.io/name=prometheus —shard.namespace=kvass —config.file=/etc/prometheus/configout/prometheus.env.yaml
/bin/prometheus-config-reloader —reload-url=http://localhost:9090/-/reload —config-file=/etc/prometheus/config/prometheus.yml —config-envsubst-file=/etc/prometheus/config_out/prometheus.env.yaml

#thanos query
/bin/thanos query —log.level=debug —query.auto-downsampling —grpc-address=0.0.0.0:10901 —http-address=0.0.0.0:80 —query.partial-response —query.replica-label=prometheus_replica —query.replica-label=rule_replica —store=11.80.31.40:10901 —store=11.80.22.158:10901 —store=11.80.14.230:10901

#prometheus+thanos+kvass
/bin/thanos sidecar —tsdb.path=/prometheus —prometheus.url=http://localhost:8080 —reloader.config-file=/etc/prometheus/config/prometheus.yml —reloader.config-envsubst-file=/etc/prometheus/config_out/prometheus.env.yaml

/kvass sidecar

/bin/prometheus —storage.tsdb.path=/prometheus —storage.tsdb.retention.time=3h —web.enable-lifecycle —storage.tsdb.no-lockfile —storage.tsdb.max-block-duration=2h —storage.tsdb.min-block-duration=2h —config.file=/etc/prometheus/config_out/prometheus_injected.yaml
/etc/prometheus/config/prometheus.yml
global:
scrape_interval: 15s
evaluation_interval: 15s
external_labels:
cluster: custom
scrape_configs:
- job_name: ‘metrics-test’
kubernetes_sd_configs:
- api_server: “https://api.sh01.n.jd.local:6443“
role: pod
basic_auth:
username: “admin”
password: “oPFKSoYXh8cJ4hbleJzh9A==”
tls_config:
insecure_skip_verify: true
relabel_configs:
- source_labels: [meta_kubernetes_pod_label_app_kubernetes_io_name]
regex: metrics
action: keep
- source_labels: [meta_kubernetes_pod_ip]
action: replace
regex: (.*)
replacement: ${1}:9091
target_label: __address
- source_labels:
- meta_kubernetes_pod_name
target_label: pod

/etc/prometheus/config_out/prometheus.env.yaml
global:
scrape_interval: 15s
evaluation_interval: 15s
external_labels:
cluster: custom
scrape_configs:
- job_name: ‘metrics-test’
kubernetes_sd_configs:
- api_server: “https://api.sh01.n.jd.local:6443“
role: pod
basic_auth:
username: “admin”
password: “oPFKSoYXh8cJ4hbleJzh9A==”
tls_config:
insecure_skip_verify: true
relabel_configs:
- source_labels: [meta_kubernetes_pod_label_app_kubernetes_io_name]
regex: metrics
action: keep
- source_labels: [meta_kubernetes_pod_ip]
action: replace
regex: (.*)
replacement: ${1}:9091
target_label: address
- source_labels:
- meta_kubernetes_pod_name
target_label: pod

/etc/prometheus/config_out/prometheus_injected.yaml
global:
scrape_interval: 15s
scrape_timeout: 10s
evaluation_interval: 15s
external_labels:
cluster: custom
scrape_configs:
- job_name: metrics-test
honor_timestamps: true
scrape_interval: 15s
scrape_timeout: 10s
metrics_path: /metrics
scheme: http
proxy_url: http://127.0.0.1:8008
relabel_configs:
- separator: ;
regex: invalid_label(.+)
replacement: $1
action: labelmap
staticconfigs:
- targets:
- 11.80.22.179:9091
labels:
address: 11.80.22.179:9091
metrics_path: /metrics
paramhash: “9893488292205083802”
paramjobName: metrics-test
paramscheme: http
scheme: http
instance: 11.80.22.179:9091
job: metrics-test
pod: metrics-b75b6466d-q96vd
- targets:
- 11.80.22.188:9091
labels:
address: 11.80.22.188:9091
metrics_path: /metrics
paramhash: “3709920721341630930”
paramjobName: metrics-test
paramscheme: http
_scheme: http
instance: 11.80.22.188:9091
job: metrics-test
pod: metrics-b75b6466d-5lkvt