Rancher 中文文档 - 28 etcd 集群 - 《Rancher》

rancher
kubeadm
etcd 命令帮助

rancher

https://rancher.com/docs/rancher/v2.x/en/troubleshooting/kubernetes-components/etcd/
[

](https://rancher.com/docs/rancher/v2.5/en/troubleshooting/kubernetes-components/etcd/)
https://github.com/rancher/rke/issues/2512

https://rancher.com/docs/rancher/v2.5/en/troubleshooting/kubernetes-components/etcd/

[rancher@rmaster01 ~]$ docker ps -a -f=name=etcd$
CONTAINER ID        IMAGE                                  COMMAND                  CREATED             STATUS              PORTS               NAMES
cf3240f66065        rancher/coreos-etcd:v3.4.14-rancher1   "/usr/local/bin/etcd…"   8 weeks ago         Up 6 minutes                            etcd
[rancher@rmaster01 ~]$ docker exec -e ETCDCTL_ENDPOINTS=$(docker exec etcd /bin/sh -c "etcdctl member list | cut -d, -f5 | sed -e 's/ //g' | paste -sd ','") etcd etcdctl endpoint status --write-out table
+----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
|          ENDPOINT          |        ID        | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| https://192.168.6.102:2379 | 48c8f6426a43d1f1 |  3.4.14 |   92 MB |     false |      false |        49 |   18502729 |           18502729 |   |
| https://192.168.6.101:2379 | e5f985c597c36724 |  3.4.14 |   91 MB |     false |      false |        49 |   18502729 |           18502729 |   |
| https://192.168.6.100:2379 | f4e7ca4943b7b405 |  3.4.14 |   91 MB |      true |      false |        49 |   18502729 |           18502729 |   |
+----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
[rancher@rmaster01 ~]$ docker exec -e ETCDCTL_ENDPOINTS=$(docker exec etcd /bin/sh -c "etcdctl member list | cut -d, -f5 | sed -e 's/ //g' | paste -sd ','") etcd etcdctl member list
48c8f6426a43d1f1, started, etcd-rmaster03, https://192.168.6.102:2380, https://192.168.6.102:2379, false
e5f985c597c36724, started, etcd-rmaster02, https://192.168.6.101:2380, https://192.168.6.101:2379, false
f4e7ca4943b7b405, started, etcd-rmaster01, https://192.168.6.100:2380, https://192.168.6.100:2379, false
[rancher@rmaster01 ~]$

证书

[rancher@rmaster01 ssl]$ pwd
/etc/kubernetes/ssl
[rancher@rmaster01 ssl]$ ls
certs                                         kubecfg-kube-controller-manager.yaml  kube-etcd-192-168-11-102.pem
kube-apiserver-key.pem                        kubecfg-kube-node.yaml                kube-node-key.pem
kube-apiserver.pem                            kubecfg-kube-proxy.yaml               kube-node.pem
kube-apiserver-proxy-client-key.pem           kubecfg-kube-scheduler.yaml           kube-proxy-key.pem
kube-apiserver-proxy-client.pem               kube-controller-manager-key.pem       kube-proxy.pem
kube-apiserver-requestheader-ca-key.pem       kube-controller-manager.pem           kube-scheduler-key.pem
kube-apiserver-requestheader-ca.pem           kube-etcd-192-168-11-100-key.pem      kube-scheduler.pem
kube-ca-key.pem                               kube-etcd-192-168-11-100.pem          kube-service-account-token-key.pem
kube-ca.pem                                   kube-etcd-192-168-11-101-key.pem      kube-service-account-token.pem
kubecfg-kube-apiserver-proxy-client.yaml      kube-etcd-192-168-11-101.pem
kubecfg-kube-apiserver-requestheader-ca.yaml  kube-etcd-192-168-11-102-key.pem
[rancher@rmaster01 ssl]$

[rancher@rmaster01 ~]$ docker ps |grep etcd 
c93210bbcacc        rancher/rke-tools:v0.1.72              "/docker-entrypoint.…"   3 weeks ago         Up 6 days                               etcd-rolling-snapshots
e073d4c5266b        rancher/coreos-etcd:v3.4.13-rancher1   "/usr/local/bin/etcd…"   3 weeks ago         Up 6 days                               etcd
[rancher@rmaster01 ~]$ 
[rancher@rmaster01 ~]$ docker image ls |grep etcd 
rancher/coreos-etcd                 v3.4.13-rancher1    7a8adaf3e7ad        7 months ago        83.8MB
[rancher@rmaster01 ~]$

[rancher@rmaster01 ~]$ docker exec -it  c93 bash
bash-5.0#
bash-5.0# etcdctl -v
etcdctl version: 3.3.10
API version: 2
bash-5.0#

[rancher@rmaster01 ~]$ docker exec -it e073 sh 
# export ETCDCTL_API=3
# etcdctl member list
48c8f6426a43d1f1, started, etcd-rmaster03, https://192.168.6.102:2380, https://192.168.6.102:2379, false
e5f985c597c36724, started, etcd-rmaster02, https://192.168.6.101:2380, https://192.168.6.101:2379, false
f4e7ca4943b7b405, started, etcd-rmaster01, https://192.168.6.100:2380, https://192.168.6.100:2379, false
#

[root@rmaster01 ~]# docker exec -it  e073d4c5266b sh
# etcdctl endpoint health 
https://127.0.0.1:2379 is healthy: successfully committed proposal: took = 31.811534ms
# etcdctl member list
48c8f6426a43d1f1, started, etcd-rmaster03, https://192.168.6.102:2380, https://192.168.6.102:2379, false
e5f985c597c36724, started, etcd-rmaster02, https://192.168.6.101:2380, https://192.168.6.101:2379, false
f4e7ca4943b7b405, started, etcd-rmaster01, https://192.168.6.100:2380, https://192.168.6.100:2379, false

# etcdctl snapshot save snapshotdb
{"level":"info","ts":1616891288.2478633,"caller":"snapshot/v3_snapshot.go:119","msg":"created temporary db file","path":"snapshotdb.part"}
{"level":"info","ts":"2021-03-28T00:28:08.280Z","caller":"clientv3/maintenance.go:200","msg":"opened snapshot stream; downloading"}
{"level":"info","ts":1616891288.2803595,"caller":"snapshot/v3_snapshot.go:127","msg":"fetching snapshot","endpoint":"https://127.0.0.1:2379"}
{"level":"info","ts":"2021-03-28T00:28:09.231Z","caller":"clientv3/maintenance.go:208","msg":"completed snapshot read; closing"}
{"level":"info","ts":1616891289.5301962,"caller":"snapshot/v3_snapshot.go:142","msg":"fetched snapshot","endpoint":"https://127.0.0.1:2379","size":"23 MB","took":1.282092525}
{"level":"info","ts":1616891289.5304568,"caller":"snapshot/v3_snapshot.go:152","msg":"saved","path":"snapshotdb"}
Snapshot saved at snapshotdb
#

kubeadm

ETCDCTL_API=3 etcdctl --endpoints 127.0.0.1:2379 \
  --cert=/etc/kubernetes/pki/etcd/server.crt \
  --key=/etc/kubernetes/pki/etcd/server.key \
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
  member list
ETCDCTL_API=3 etcdctl --endpoints 127.0.0.1:2379 \
  --cert=/etc/kubernetes/pki/etcd/server.crt \
  --key=/etc/kubernetes/pki/etcd/server.key \
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
 snapshot save /var/lib/backup/etcd-snapshot.db
ETCDCTL_API=3 etcdctl --endpoints 127.0.0.1:2379 \
  --cert=/etc/kubernetes/pki/etcd/server.crt \
  --key=/etc/kubernetes/pki/etcd/server.key \
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
 --write-out=table snapshot status /var/lib/backup/etcd-snapshot.db
ETCDCTL_API=3 etcdctl --endpoints 127.0.0.1:2379 \
  --cert=/etc/kubernetes/pki/etcd/server.crt \
  --key=/etc/kubernetes/pki/etcd/server.key \
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
snapshot restore  /var/lib/backup/etcd-snapshot.db
ETCDCTL_API=3 etcdctl --endpoints 127.0.0.1:2379 \
  --cert=/etc/kubernetes/pki/etcd/server.crt \
  --key=/etc/kubernetes/pki/etcd/server.key \
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
  --name master \
  --data-dir=/var/lib/etcd \
  --skip-hash-check \
  --initial-advertise-peer-urls=https://127.0.0.1:2380 \
  --initial-cluster=master=https://127.0.0.1:2380 \
snapshot restore /var/lib/backup/etcd-snapshot.db
[root@master ~]# kubectl -n kube-system  exec -it  etcd-master sh
# ETCDCTL_API=3 etcdctl --endpoints 127.0.0.1:2379 \
  --cert=/etc/kubernetes/pki/etcd/server.crt \
  --key=/etc/kubernetes/pki/etcd/server.key \
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
  member list> > > > 
3ced1f2dde846660, started, master, https://192.168.11.90:2380, https://192.168.11.90:2379, false
#
# ETCDCTL_API=3 etcdctl --endpoints 127.0.0.1:2379 \
  --cert=/etc/kubernetes/pki/etcd/server.crt \
  --key=/etc/kubernetes/pki/etcd/server.key \
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
 --write-out=table snapshot status /var/lib/backup/etcd-snapshot.db> > > > 
+----------+----------+------------+------------+
|   HASH   | REVISION | TOTAL KEYS | TOTAL SIZE |
+----------+----------+------------+------------+
| bbf56722 |    34835 |       1349 |     4.5 MB |
+----------+----------+------------+------------+
#

[root@master ~]# cat /etc/kubernetes/manifests/etcd.yaml 
apiVersion: v1
kind: Pod
metadata:
  annotations:
    kubeadm.kubernetes.io/etcd.advertise-client-urls: https://192.168.11.90:2379
  creationTimestamp: null
  labels:
    component: etcd
    tier: control-plane
  name: etcd
  namespace: kube-system
spec:
  containers:
  - command:
    - etcd
    - --advertise-client-urls=https://192.168.11.90:2379
    - --cert-file=/etc/kubernetes/pki/etcd/server.crt
    - --client-cert-auth=true
    - --data-dir=/var/lib/etcd
    - --initial-advertise-peer-urls=https://192.168.11.90:2380
    - --initial-cluster=master=https://192.168.11.90:2380
    - --key-file=/etc/kubernetes/pki/etcd/server.key
    - --listen-client-urls=https://127.0.0.1:2379,https://192.168.11.90:2379
    - --listen-metrics-urls=http://127.0.0.1:2381
    - --listen-peer-urls=https://192.168.11.90:2380
    - --name=master
    - --peer-cert-file=/etc/kubernetes/pki/etcd/peer.crt
    - --peer-client-cert-auth=true
    - --peer-key-file=/etc/kubernetes/pki/etcd/peer.key
    - --peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt
    - --snapshot-count=10000
    - --trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt
    image: registry.aliyuncs.com/google_containers/etcd:3.4.3-0
    imagePullPolicy: IfNotPresent
    livenessProbe:
      failureThreshold: 8
      httpGet:
        host: 127.0.0.1
        path: /health
        port: 2381
        scheme: HTTP
      initialDelaySeconds: 15
      timeoutSeconds: 15
    name: etcd
    resources: {}
    volumeMounts:
    - mountPath: /var/lib/etcd
      name: etcd-data
    - mountPath: /etc/kubernetes/pki/etcd
      name: etcd-certs
  hostNetwork: true
  priorityClassName: system-cluster-critical
  volumes:
  - hostPath:
      path: /etc/kubernetes/pki/etcd
      type: DirectoryOrCreate
    name: etcd-certs
  - hostPath:
      path: /var/lib/etcd
      type: DirectoryOrCreate
    name: etcd-data
status: {}
[root@master ~]#

etcd 命令帮助

# etcdctl -h
NAME:
    etcdctl - A simple command line client for etcd3.
USAGE:
    etcdctl [flags]
VERSION:
    3.4.3
API VERSION:
    3.4
COMMANDS:
    alarm disarm        Disarms all alarms
    alarm list        Lists all alarms
    auth disable        Disables authentication
    auth enable        Enables authentication
    check datascale        Check the memory usage of holding data for different workloads on a given server endpoint.
    check perf        Check the performance of the etcd cluster
    compaction        Compacts the event history in etcd
    defrag            Defragments the storage of the etcd members with given endpoints
    del            Removes the specified key or range of keys [key, range_end)
    elect            Observes and participates in leader election
    endpoint hashkv        Prints the KV history hash for each endpoint in --endpoints
    endpoint health        Checks the healthiness of endpoints specified in `--endpoints` flag
    endpoint status        Prints out the status of endpoints specified in `--endpoints` flag
    get            Gets the key or a range of keys
    help            Help about any command
    lease grant        Creates leases
    lease keep-alive    Keeps leases alive (renew)
    lease list        List all active leases
    lease revoke        Revokes leases
    lease timetolive    Get lease information
    lock            Acquires a named lock
    make-mirror        Makes a mirror at the destination etcd cluster
    member add        Adds a member into the cluster
    member list        Lists all members in the cluster
    member promote        Promotes a non-voting member in the cluster
    member remove        Removes a member from the cluster
    member update        Updates a member in the cluster
    migrate            Migrates keys in a v2 store to a mvcc store
    move-leader        Transfers leadership to another etcd cluster member.
    put            Puts the given key into the store
    role add        Adds a new role
    role delete        Deletes a role
    role get        Gets detailed information of a role
    role grant-permission    Grants a key to a role
    role list        Lists all roles
    role revoke-permission    Revokes a key from a role
    snapshot restore    Restores an etcd member snapshot to an etcd directory
    snapshot save        Stores an etcd node backend snapshot to a given file
    snapshot status        Gets backend snapshot status of a given file
    txn            Txn processes all the requests in one transaction
    user add        Adds a new user
    user delete        Deletes a user
    user get        Gets detailed information of a user
    user grant-role        Grants a role to a user
    user list        Lists all users
    user passwd        Changes password of user
    user revoke-role    Revokes a role from a user
    version            Prints the version of etcdctl
    watch            Watches events stream on keys or prefixes
OPTIONS:
      --cacert=""                verify certificates of TLS-enabled secure servers using this CA bundle
      --cert=""                    identify secure client using this TLS certificate file
      --command-timeout=5s            timeout for short running command (excluding dial timeout)
      --debug[=false]                enable client-side debug logging
      --dial-timeout=2s                dial timeout for client connections
  -d, --discovery-srv=""            domain name to query for SRV records describing cluster endpoints
      --discovery-srv-name=""            service name to query when using DNS discovery
      --endpoints=[127.0.0.1:2379]        gRPC endpoints
  -h, --help[=false]                help for etcdctl
      --hex[=false]                print byte strings as hex encoded strings
      --insecure-discovery[=true]        accept insecure SRV records describing cluster endpoints
      --insecure-skip-tls-verify[=false]    skip server certificate verification
      --insecure-transport[=true]        disable transport security for client connections
      --keepalive-time=2s            keepalive time for client connections
      --keepalive-timeout=6s            keepalive timeout for client connections
      --key=""                    identify secure client using this TLS key file
      --password=""                password for authentication (if this option is used, --user option shouldn't include password)
      --user=""                    username[:password] for authentication (prompt if password is not supplied)
  -w, --write-out="simple"            set the output format (fields, json, protobuf, simple, table)