镜像离线打包脚本

    1. docker pull rancher/coreos-etcd:v3.4.3-rancher1
    2. docker pull rancher/rke-tools:v0.1.59
    3. docker pull rancher/k8s-dns-kube-dns:1.15.2
    4. docker pull rancher/k8s-dns-dnsmasq-nanny:1.15.2
    5. docker pull rancher/k8s-dns-sidecar:1.15.2
    6. docker pull rancher/cluster-proportional-autoscaler:1.7.1
    7. docker pull rancher/coredns-coredns:1.6.9
    8. docker pull rancher/k8s-dns-node-cache:1.15.7
    9. docker pull rancher/hyperkube:v1.18.6-rancher1
    10. docker pull rancher/coreos-flannel:v0.12.0
    11. docker pull rancher/flannel-cni:v0.3.0-rancher6
    12. docker pull rancher/calico-node:v3.13.4
    13. docker pull rancher/calico-cni:v3.13.4
    14. docker pull rancher/calico-kube-controllers:v3.13.4
    15. docker pull rancher/calico-ctl:v3.13.4
    16. docker pull rancher/calico-pod2daemon-flexvol:v3.13.4
    17. docker pull weaveworks/weave-kube:2.6.4
    18. docker pull weaveworks/weave-npc:2.6.4
    19. docker pull rancher/pause:3.1
    20. docker pull rancher/nginx-ingress-controller:nginx-0.32.0-rancher1
    21. docker pull rancher/nginx-ingress-controller-defaultbackend:1.5-rancher1
    22. docker pull rancher/metrics-server:v0.3.6
    1. [root@uat-rancher-node01 rancher]# ll *.tar
    2. -rw-rw-r-- 1 root root  226441216 Aug 29 20:53 calico-cni.tar
    3. -rw-rw-r-- 1 root root   48206848 Aug 29 20:51 calico-ctl.tar
    4. -rw-rw-r-- 1 root root   56617984 Aug 29 20:50 calico-kube-controllers.tar
    5. -rw-rw-r-- 1 root root  265684992 Aug 29 20:55 calico-node.tar
    6. -rw-rw-r-- 1 root root  114141696 Aug 29 20:54 calico-pod2daemon-flexvol.tar
    7. -rw-rw-r-- 1 root root   41323520 Aug 29 20:42 cluster-proportional-autoscaler.tar
    8. -rw-rw-r-- 1 root root   43297792 Aug 29 20:47 coredns-coredns.tar
    9. -rw-rw-r-- 1 root root   85169664 Aug 29 20:46 coreos-etcd.tar
    10. -rw-rw-r-- 1 root root 1552846336 Aug 29 20:59 hyperkube.tar
    11. -rw-rw-r-- 1 root root   40140288 Aug 29 20:34 k8s-dns-dnsmasq-nanny.tar
    12. -rw-rw-r-- 1 root root   88547328 Aug 29 20:38 k8s-dns-kube-dns.tar
    13. -rw-rw-r-- 1 root root   92754944 Aug 29 20:43 k8s-dns-node-cache.tar
    14. -rw-rw-r-- 1 root root   80885760 Aug 29 20:39 k8s-dns-sidecar.tar
    15. -rw-rw-r-- 1 root root   41199616 Aug 29 20:44 metrics-server.tar
    16. -rw-rw-r-- 1 root root    5144064 Aug 29 20:34 nginx-ingress-controller-defaultbackend.tar
    17. -rw-rw-r-- 1 root root  331629056 Aug 29 20:48 nginx-ingress-controller.tar
    18. -rw-rw-r-- 1 root root     754176 Aug 29 20:34 pause.tar
    19. -rw-rw-r-- 1 root root  135227392 Aug 29 20:56 rke-tools.tar
    1. [root@uat-rancher-node01 rancher]# ll *.tar|awk '{print $NF}'
    2. calico-cni.tar
    3. calico-ctl.tar
    4. calico-kube-controllers.tar
    5. calico-node.tar
    6. calico-pod2daemon-flexvol.tar
    7. cluster-proportional-autoscaler.tar
    8. coredns-coredns.tar
    9. coreos-etcd.tar
    10. hyperkube.tar
    11. k8s-dns-dnsmasq-nanny.tar
    12. k8s-dns-kube-dns.tar
    13. k8s-dns-node-cache.tar
    14. k8s-dns-sidecar.tar
    15. metrics-server.tar
    16. nginx-ingress-controller-defaultbackend.tar
    17. nginx-ingress-controller.tar
    18. pause.tar
    19. rke-tools.tar
    20. [root@uat-rancher-node01 rancher]# 
    21. [root@uat-rancher-node01 rancher]# ll *.tar|awk '{print $NF}'|sed -r 's#(.*)#docker load -i \1#' |bash
    22. b76aa58f4c23: Loading layer [==================================================>]  107.4MB/107.4MB
    23. 566d20c1ccdf: Loading layer [==================================================>]  20.48kB/20.48kB
    24. f3e35332f964: Loading layer [==================================================>]  101.2MB/101.2MB
    25. 109a0c66209a: Loading layer [==================================================>]  10.24kB/10.24kB
    26. 6ac59be9ed64: Loading layer [==================================================>]   2.56kB/2.56kB
    27. 3b481276ac88: Loading layer [==================================================>]  17.79MB/17.79MB
    28. 11703ffeceb1: Loading layer [==================================================>]  13.82kB/13.82kB
    29. Loaded image: rancher/calico-cni:v3.13.4
    30. 1b3ee35aacca: Loading layer [==================================================>]   5.84MB/5.84MB
    31. 419eaad88244: Loading layer [==================================================>]  42.35MB/42.35MB
    32. Loaded image: rancher/calico-ctl:v3.13.4
    33. 7bd4affc29eb: Loading layer [==================================================>]  13.82kB/13.82kB
    34. 523c4550fd32: Loading layer [==================================================>]  53.52MB/53.52MB
    35. b5dadf89acf5: Loading layer [==================================================>]   3.07MB/3.07MB
    36. Loaded image: rancher/calico-kube-controllers:v3.13.4
    37. f80c95f61fff: Loading layer [==================================================>]  108.5MB/108.5MB
    38. eddba477a8ae: Loading layer [==================================================>]  20.48kB/20.48kB
    39. 76224ad063b6: Loading layer [==================================================>]  2.781MB/2.781MB
    40. 2ca638bbce84: Loading layer [==================================================>]  3.298MB/3.298MB
    41. 62b8adc82952: Loading layer [==================================================>]  3.072kB/3.072kB
    42. 2e1e28e9c135: Loading layer [==================================================>]  75.24MB/75.24MB
    43. 1fa8cad5d0e3: Loading layer [==================================================>]  4.096kB/4.096kB
    44. af19fd058de9: Loading layer [==================================================>]  7.185MB/7.185MB
    45. 4ba854688443: Loading layer [==================================================>]  6.052MB/6.052MB
    46. db5b88cbf87f: Loading layer [==================================================>]  2.048kB/2.048kB
    47. 9fcde00c5e9a: Loading layer [==================================================>]  13.82kB/13.82kB
    48. 5f956aaa317c: Loading layer [==================================================>]  61.32MB/61.32MB
    49. 151cbe937db5: Loading layer [==================================================>]  1.251MB/1.251MB
    50. Loaded image: rancher/calico-node:v3.13.4
    51. 724362325411: Loading layer [==================================================>]  2.048kB/2.048kB
    52. 689becca0610: Loading layer [==================================================>]  13.82kB/13.82kB
    53. 1140aa2f2fa9: Loading layer [==================================================>]   5.12kB/5.12kB
    54. 6c83a0e86620: Loading layer [==================================================>]  5.606MB/5.606MB
    55. Loaded image: rancher/calico-pod2daemon-flexvol:v3.13.4
    56. 932da5156413: Loading layer [==================================================>]  3.062MB/3.062MB
    57. 7b00adda7217: Loading layer [==================================================>]  38.25MB/38.25MB
    58. Loaded image: rancher/cluster-proportional-autoscaler:1.7.1
    59. 225df95e717c: Loading layer [==================================================>]  336.4kB/336.4kB
    60. 8762ba1e4767: Loading layer [==================================================>]  42.95MB/42.95MB
    61. Loaded image: rancher/coredns-coredns:1.6.9
    62. fe9a8b4f1dcc: Loading layer [==================================================>]  43.87MB/43.87MB
    63. 816dcf8208f7: Loading layer [==================================================>]  23.72MB/23.72MB
    64. 4da29af72f7f: Loading layer [==================================================>]  17.55MB/17.55MB
    65. e94602b7c460: Loading layer [==================================================>]   2.56kB/2.56kB
    66. e74140cc410f: Loading layer [==================================================>]  3.072kB/3.072kB
    67. 0c356e885c8a: Loading layer [==================================================>]  3.072kB/3.072kB
    68. Loaded image: rancher/coreos-etcd:v3.4.3-rancher1
    69. 82a5cde9d9a9: Loading layer [==================================================>]  53.87MB/53.87MB
    70. a2b38eae1b39: Loading layer [==================================================>]  21.62MB/21.62MB
    71. f378e9487360: Loading layer [==================================================>]  5.168MB/5.168MB
    72. a35a0b8b55f5: Loading layer [==================================================>]  4.608kB/4.608kB
    73. dea351e760ec: Loading layer [==================================================>]  8.192kB/8.192kB
    74. d57a645c2b0c: Loading layer [==================================================>]  8.704kB/8.704kB
    75. 80c8b272a31c: Loading layer [==================================================>]  9.728kB/9.728kB
    76. 588868021aa0: Loading layer [==================================================>]  1.824MB/1.824MB
    77. 0a514e5f8343: Loading layer [==================================================>]   5.12kB/5.12kB
    78. e46a9dd0cf8f: Loading layer [==================================================>]  23.04kB/23.04kB
    79. ea3e975f63d4: Loading layer [==================================================>]  349.1MB/349.1MB
    80. 488428ef6ca2: Loading layer [==================================================>]  72.17MB/72.17MB
    81. 8cff9f524c8b: Loading layer [==================================================>]  469.4MB/469.4MB
    82. 9105b0079c7f: Loading layer [==================================================>]  3.584kB/3.584kB
    83. 03a3faf297ee: Loading layer [==================================================>]  579.6MB/579.6MB
    84. Loaded image: rancher/hyperkube:v1.18.6-rancher1
    85. d9ff549177a9: Loading layer [==================================================>]  4.671MB/4.671MB
    86. 257e31c9cb28: Loading layer [==================================================>]   2.56kB/2.56kB
    87. e3418a4c0703: Loading layer [==================================================>]    362kB/362kB
    88. 39fe80c5a89b: Loading layer [==================================================>]  3.584kB/3.584kB
    89. 295a83faf517: Loading layer [==================================================>]  35.08MB/35.08MB
    90. Loaded image: rancher/k8s-dns-dnsmasq-nanny:1.15.2
    91. 47d8bc5560bb: Loading layer [==================================================>]  44.66MB/44.66MB
    92. Loaded image: rancher/k8s-dns-kube-dns:1.15.2
    93. 35c35a973795: Loading layer [==================================================>]  2.276MB/2.276MB
    94. d71bb3de1e76: Loading layer [==================================================>]  46.59MB/46.59MB
    95. Loaded image: rancher/k8s-dns-node-cache:1.15.7
    96. e3b0318787d0: Loading layer [==================================================>]     37MB/37MB
    97. Loaded image: rancher/k8s-dns-sidecar:1.15.2
    98. 7bf3709d22bb: Loading layer [==================================================>]  38.13MB/38.13MB
    99. Loaded image: rancher/metrics-server:v0.3.6
    100. b108d4968233: Loading layer [==================================================>]  5.134MB/5.134MB
    101. Loaded image: rancher/nginx-ingress-controller-defaultbackend:1.5-rancher1
    102. beee9f30bc1f: Loading layer [==================================================>]  5.862MB/5.862MB
    103. 378129e7fefc: Loading layer [==================================================>]  161.4MB/161.4MB
    104. 2c4876c55341: Loading layer [==================================================>]  6.144kB/6.144kB
    105. a5cd644ea51c: Loading layer [==================================================>]  30.43MB/30.43MB
    106. f99824ffc859: Loading layer [==================================================>]  16.91MB/16.91MB
    107. e816d879e6f5: Loading layer [==================================================>]  4.096kB/4.096kB
    108. d43abedb29d5: Loading layer [==================================================>]  8.042MB/8.042MB
    109. bfa0001530d1: Loading layer [==================================================>]  50.41MB/50.41MB
    110. 4becb331f71a: Loading layer [==================================================>]  6.656kB/6.656kB
    111. b2a014433f54: Loading layer [==================================================>]  37.57MB/37.57MB
    112. 7863f1fb0c1e: Loading layer [==================================================>]   20.9MB/20.9MB
    113. 909651a2178f: Loading layer [==================================================>]  6.656kB/6.656kB
    114. Loaded image: rancher/nginx-ingress-controller:nginx-0.32.0-rancher1
    115. e17133b79956: Loading layer [==================================================>]  744.4kB/744.4kB
    116. Loaded image: rancher/pause:3.1
    117. f1b5933fe4b5: Loading layer [==================================================>]  5.796MB/5.796MB
    118. 2bdf88b2699d: Loading layer [==================================================>]  17.72MB/17.72MB
    119. 519b820c8bb3: Loading layer [==================================================>]  1.603MB/1.603MB
    120. eed905e10fc1: Loading layer [==================================================>]  24.45MB/24.45MB
    121. 126480a1531c: Loading layer [==================================================>]  3.072kB/3.072kB
    122. 021ed1072f21: Loading layer [==================================================>]  56.38MB/56.38MB
    123. 25f3c3b9821e: Loading layer [==================================================>]  3.509MB/3.509MB
    124. b5449afc16e1: Loading layer [==================================================>]  16.08MB/16.08MB
    125. 8122015cea19: Loading layer [==================================================>]  4.096kB/4.096kB
    126. 06002c2e3403: Loading layer [==================================================>]  4.096kB/4.096kB
    127. 3f81d91d9bbb: Loading layer [==================================================>]  5.632kB/5.632kB
    128. c6382345d3ee: Loading layer [==================================================>]   12.8kB/12.8kB
    129. 4ab3470e2725: Loading layer [==================================================>]  9.601MB/9.601MB
    130. Loaded image: rancher/rke-tools:v0.1.59
    131. [root@uat-rancher-node01 rancher]# docker image ls 
    132. REPOSITORY                                        TAG                     IMAGE ID            CREATED             SIZE
    133. rancher/hyperkube                                 v1.18.6-rancher1        5a1e9f24e782        6 weeks ago         1.51GB
    134. rancher/rke-tools                                 v0.1.59                 904d2afa34c8        7 weeks ago         132MB
    135. rancher/calico-node                               v3.13.4                 c91d49e6f044        3 months ago        261MB
    136. rancher/calico-pod2daemon-flexvol                 v3.13.4                 c5dca18c0346        3 months ago        112MB
    137. rancher/calico-cni                                v3.13.4                 9e1176a74e85        3 months ago        225MB
    138. rancher/calico-ctl                                v3.13.4                 cbd105686d60        3 months ago        47.9MB
    139. rancher/calico-kube-controllers                   v3.13.4                 f9f70a2e922f        3 months ago        56.6MB
    140. rancher/nginx-ingress-controller                  nginx-0.32.0-rancher1   eda78cfd6f9d        3 months ago        328MB
    141. rancher/coredns-coredns                           1.6.9                   4e797b323460        5 months ago        43.2MB
    142. rancher/coreos-etcd                               v3.4.3-rancher1         a0b920cf970d        10 months ago       83.6MB
    143. rancher/metrics-server                            v0.3.6                  9dd718864ce6        10 months ago       39.9MB
    144. rancher/k8s-dns-node-cache                        1.15.7                  ce4f91502e1b        10 months ago       91MB
    145. rancher/cluster-proportional-autoscaler           1.7.1                   14afc47fd5af        12 months ago       40.1MB
    146. rancher/k8s-dns-sidecar                           1.15.2                  ffc7ccc8fded        16 months ago       79.3MB
    147. rancher/k8s-dns-kube-dns                          1.15.2                  4ad5e24b1ad2        16 months ago       87MB
    148. rancher/k8s-dns-dnsmasq-nanny                     1.15.2                  c4d9bb9e5ff0        16 months ago       39.8MB
    149. rancher/nginx-ingress-controller-defaultbackend   1.5-rancher1            b5af743e5984        23 months ago       5.13MB
    150. rancher/pause                                     3.1                     da86e6ba6ca1        2 years ago         742kB
    151. [root@uat-rancher-node01 rancher]#

    批量下载镜像并打包推送到harbor镜像仓库

    1. [root@salt-master ~]# docker push 10.182.220.240:80/rancher/calico-pod2daemon-flexvol:v3.13.4 
    2. The push refers to a repository [10.182.220.240:80/rancher/calico-pod2daemon-flexvol]
    3. 6c83a0e86620: Pushed 
    4. 1140aa2f2fa9: Pushed 
    5. 689becca0610: Pushed 
    6. 724362325411: Pushed 
    7. eddba477a8ae: Mounted from rancher/calico-node 
    8. f80c95f61fff: Mounted from rancher/calico-node 
    9. v3.13.4: digest: sha256:3a12c023e964104ebf8af330bc74fa25831e961c871f8024bd6917c1357a57a6 size: 1571
    10. [root@salt-master ~]#

    image.png

    Harbor私有镜像仓库部署
    [root@harbor harbor]# vim harbor.yml
    image.png

    harbor证书配置

    1. [root@harbor harbor]# cat key.sh 
    2. #!/bin/bash
    4. # 在该目录下操作生成证书，正好供harbor.yml使用
    5. #mkdir -p /data/cert
    6. cd /app/docker-compose/harbor/key
    8. openssl genrsa -out ca.key 4096
    9. openssl req -x509 -new -nodes -sha512 -days 3650 -subj "/C=CN/ST=Beijing/L=Beijing/O=example/OU=Personal/CN=www.harbor.mobi" -key ca.key -out ca.crt
    10. openssl genrsa -out www.harbor.mobi.key 4096
    11. openssl req -sha512 -new -subj "/C=CN/ST=Beijing/L=Beijing/O=example/OU=Personal/CN=www.harbor.mobi" -key www.harbor.mobi.key -out www.harbor.mobi.csr
    13. cat > v3.ext <<-EOF
    14. authorityKeyIdentifier=keyid,issuer
    15. basicConstraints=CA:FALSE
    16. keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment
    17. extendedKeyUsage = serverAuth
    18. subjectAltName = @alt_names
    20. [alt_names]
    21. DNS.1=www.harbor.mobi
    22. DNS.2=harbor
    23. DNS.3=ks-allinone
    24. EOF
    26. openssl x509 -req -sha512 -days 3650 -extfile v3.ext -CA ca.crt -CAkey ca.key -CAcreateserial -in www.harbor.mobi.csr -out www.harbor.mobi.crt
    28. openssl x509 -inform PEM -in www.harbor.mobi.crt -out www.harbor.mobi.cert
    30. cp www.harbor.mobi.crt /etc/pki/ca-trust/source/anchors/www.harbor.mobi.crt 
    31. update-ca-trust
    32. [root@harbor harbor]#
    1. [root@harbor harbor]# tree /app/docker-compose/harbor/key/
    2. /app/docker-compose/harbor/key/
    3. ├── ca.crt
    4. ├── ca.key
    5. ├── ca.srl
    6. ├── v3.ext
    7. ├── www.harbor.mobi.cert
    8. ├── www.harbor.mobi.crt
    9. ├── www.harbor.mobi.csr
    10. └── www.harbor.mobi.key
    12. 0 directories, 8 files
    13. [root@harbor harbor]#

    新建一个目录
    www.harbor.mobi
    复制到各个dockers节点的www.harbor.mobi

    1. [rancher@uat-rancher-node01 ~]$ tree /etc/docker/certs.d/www.harbor.mobi/
    2. /etc/docker/certs.d/www.harbor.mobi/
    3. ├── ca.crt
    4. ├── www.harbor.mobi.cert
    5. └── www.harbor.mobi.key
    7. 0 directories, 3 files
    8. [rancher@uat-rancher-node01 ~]$

    image.png

    //“registry-mirrors”: [“https://10.182.220.240“],
    “insecure-registries”: [“https://10.182.220.240“],

    1. [root@uat-rancher-node04 ~]# docker login https://10.182.220.240
    2. Username: liwm
    3. Password: 
    4. WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
    5. Configure a credential helper to remove this warning. See
    6. https://docs.docker.com/engine/reference/commandline/login/#credentials-store
    8. Login Succeeded
    9. [root@uat-rancher-node04 ~]# docker login 10.182.220.242
    10. Username: admin
    11. Password: 
    12. WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
    13. Configure a credential helper to remove this warning. See
    14. https://docs.docker.com/engine/reference/commandline/login/#credentials-store
    16. Login Succeeded
    17. [root@uat-rancher-node04 ~]# cat /etc/docker/daemon.json 
    18. {
    19. "graph": "/app/docker",
    20. "max-concurrent-downloads": 3,
    21. "max-concurrent-uploads": 5,
    22. "insecure-registries": ["https://10.182.220.240","http://10.182.220.242"],
    23. "storage-driver": "overlay2",
    24. "storage-opts": ["overlay2.override_kernel_check=true"],
    25. "log-driver": "json-file",
    26. "log-opts": {"max-size": "100m","max-file": "3"}
    27. }
    28. [root@uat-rancher-node04 ~]#

    image.png
    配置多个私有仓库

    1. [rancher@uat-rancher-node01 ~]$ cat /etc/docker/daemon.json 
    2. {
    3. "graph": "/app/docker",
    4. "max-concurrent-downloads": 3,
    5. "max-concurrent-uploads": 5,
    6. "insecure-registries": ["https://10.182.220.240"],
    7. "storage-driver": "overlay2",
    8. "storage-opts": ["overlay2.override_kernel_check=true"],
    9. "log-driver": "json-file",
    10. "log-opts": {"max-size": "100m","max-file": "3"}
    11. }
    12. [rancher@uat-rancher-node01 ~]$

    image.png
    image.png
    image.png

    rke部署kubernetes集群
    ##配置私有仓库地址（需要配置证书）

    修改rke cluster.yml部署配置文件

    1. [rancher@uat-rancher-node01 ~]$ cat cluster.yml 
    2. nodes:
    3.   - address: 10.182.220.241
    4.     hostname_override: uat-rancher-node01
    5.     internal_address:
    6.     user: rancher
    7.     role: [controlplane,etcd]
    8.   - address: 10.182.220.242
    9.     hostname_override: uat-rancher-node02
    10.     internal_address:
    11.     user: rancher
    12.     role: [controlplane,etcd]
    13.   - address: 10.182.220.243
    14.     hostname_override: uat-rancher-node03
    15.     internal_address:
    16.     user: rancher
    17.     role: [controlplane,etcd]
    18.   - address: 10.182.220.244
    19.     hostname_override: uat-rancher-node04
    20.     internal_address:
    21.     user: rancher
    22.     role: [worker]
    23.   - address: 10.182.220.245
    24.     hostname_override: uat-rancher-node05
    25.     internal_address:
    26.     user: rancher
    27.     role: [worker]
    29. # 定义kubernetes版本
    30. kubernetes_version: v1.18.6-rancher1-2
    32. # 如果要使用私有仓库中的镜像，配置以下参数来指定默认私有仓库地址需要启用证书。
    33. private_registries:
    34.     - url: 10.182.220.240
    35.       user: liwm
    36.       password: !Q2w3e4r
    37.       is_default: true
    39. services:
    40.   etcd:
    41.     # 扩展参数
    42.     extra_args:
    43.       # 240个小时后自动清理磁盘碎片,通过auto-compaction-retention对历史数据压缩后，后端数据库可能会出现内部碎片。内部碎片是指空闲状态的，能被后端使用但是仍然消耗存储空间，碎片整理过程将此存储空间释放回文>件系统
    44.       auto-compaction-retention: 240 #(单位小时)
    45.       # 修改空间配额为6442450944，默认2G,最大8G
    46.       quota-backend-bytes: '6442450944'
    47.     # 自动备份
    48.     snapshot: true
    49.     creation: 5m0s
    50.     retention: 24h
    51.   kubelet:
    52.     extra_args:
    53.       # 支持静态Pod。在主机/etc/kubernetes/目录下创建manifest目录，Pod YAML文件放在/etc/kubernetes/manifest/目录下
    54.       pod-manifest-path: "/etc/kubernetes/manifest/"
    55. # 有几个网络插件可以选择：flannel、canal、calico，Rancher2默认canal
    56. network:
    57.   plugin: canal
    58.   options:
    59.     flannel_backend_type: "vxlan"
    60. # 可以设置provider: none来禁用ingress controller
    61. ingress:
    62.   provider: nginx
    63.   node_selector:
    64.     app: ingress
    65. [rancher@uat-rancher-node01 ~]$

    image.png

    helm 部署 rancher

    helm证书配置

    1. [rancher@uat-rancher-node01 ~]$ cat helm.sh 
    2. #!/bin/bash -e
    4. # * 为必改项
    5. # * 服务器FQDN或颁发者名(更换为你自己的域名)
    6. CN='rancher'
    8. # 扩展信任IP或域名
    10. ## 一般ssl证书只信任域名的访问请求，有时候需要使用ip去访问server，那么需要给ssl证书添加扩展IP，用逗号隔开。配置节点ip和lb的ip。
    11. SSL_IP='10.182.220.241,10.182.220.242,10.182.220.243,10.182.220.244,10.182.220.245,10.182.220.246'
    12. SSL_DNS=''
    14. # 国家名(2个字母的代号)
    15. C=CN
    16. # 证书加密位数
    17. SSL_SIZE=2048
    19. # 证书有效期
    20. DATE=${DATE:-3650}
    22. # 配置文件
    23. SSL_CONFIG='openssl.cnf'
    25. if [[ -z $SILENT ]]; then
    26. echo "----------------------------"
    27. echo "| SSL Cert Generator |"
    28. echo "----------------------------"
    29. echo
    30. fi
    32. export CA_KEY=${CA_KEY-"cakey.pem"}
    33. export CA_CERT=${CA_CERT-"cacerts.pem"}
    34. export CA_SUBJECT=ca-$CN
    35. export CA_EXPIRE=${DATE}
    37. export SSL_CONFIG=${SSL_CONFIG}
    38. export SSL_KEY=$CN.key
    39. export SSL_CSR=$CN.csr
    40. export SSL_CERT=$CN.crt
    41. export SSL_EXPIRE=${DATE}
    43. export SSL_SUBJECT=${CN}
    44. export SSL_DNS=${SSL_DNS}
    45. export SSL_IP=${SSL_IP}
    47. export K8S_SECRET_COMBINE_CA=${K8S_SECRET_COMBINE_CA:-'true'}
    49. [[ -z $SILENT ]] && echo "--> Certificate Authority"
    51. if [[ -e ./${CA_KEY} ]]; then
    52.     [[ -z $SILENT ]] && echo "====> Using existing CA Key ${CA_KEY}"
    53. else
    54.     [[ -z $SILENT ]] && echo "====> Generating new CA key ${CA_KEY}"
    55.     openssl genrsa -out ${CA_KEY} ${SSL_SIZE} > /dev/null
    56. fi
    58. if [[ -e ./${CA_CERT} ]]; then
    59.     [[ -z $SILENT ]] && echo "====> Using existing CA Certificate ${CA_CERT}"
    60. else
    61.     [[ -z $SILENT ]] && echo "====> Generating new CA Certificate ${CA_CERT}"
    62.     openssl req -x509 -sha256 -new -nodes -key ${CA_KEY} -days ${CA_EXPIRE} -out ${CA_CERT} -subj "/CN=${CA_SUBJECT}" > /dev/null || exit 1
    63. fi
    65. echo "====> Generating new config file ${SSL_CONFIG}"
    66. cat > ${SSL_CONFIG} <<EOM
    67. [req]
    68. req_extensions = v3_req
    69. distinguished_name = req_distinguished_name
    70. [req_distinguished_name]
    71. [ v3_req ]
    72. basicConstraints = CA:FALSE
    73. keyUsage = nonRepudiation, digitalSignature, keyEncipherment
    74. extendedKeyUsage = clientAuth, serverAuth
    75. EOM
    77. if [[ -n ${SSL_DNS} || -n ${SSL_IP} ]]; then
    78.     cat >> ${SSL_CONFIG} <<EOM
    79. subjectAltName = @alt_names
    80. [alt_names]
    81. EOM
    82.     IFS=","
    83.     dns=(${SSL_DNS})
    84.     dns+=(${SSL_SUBJECT})
    85.     for i in "${!dns[@]}"; do
    86.       echo DNS.$((i+1)) = ${dns[$i]} >> ${SSL_CONFIG}
    87.     done
    89.     if [[ -n ${SSL_IP} ]]; then
    90.         ip=(${SSL_IP})
    91.         for i in "${!ip[@]}"; do
    92.           echo IP.$((i+1)) = ${ip[$i]} >> ${SSL_CONFIG}
    93.         done
    94.     fi
    95. fi
    97. [[ -z $SILENT ]] && echo "====> Generating new SSL KEY ${SSL_KEY}"
    98. openssl genrsa -out ${SSL_KEY} ${SSL_SIZE} > /dev/null || exit 1
    100. [[ -z $SILENT ]] && echo "====> Generating new SSL CSR ${SSL_CSR}"
    101. openssl req -sha256 -new -key ${SSL_KEY} -out ${SSL_CSR} -subj "/CN=${SSL_SUBJECT}" -config ${SSL_CONFIG} > /dev/null || exit 1
    103. [[ -z $SILENT ]] && echo "====> Generating new SSL CERT ${SSL_CERT}"
    104. openssl x509 -sha256 -req -in ${SSL_CSR} -CA ${CA_CERT} -CAkey ${CA_KEY} -CAcreateserial -out ${SSL_CERT} \
    105.     -days ${SSL_EXPIRE} -extensions v3_req -extfile ${SSL_CONFIG} > /dev/null || exit 1
    107. if [[ -z $SILENT ]]; then
    108. echo "====> Complete"
    109. echo "keys can be found in volume mapped to $(pwd)"
    110. echo
    111. echo "====> Output results as YAML"
    112. echo "---"
    113. echo "ca_key: |"
    114. cat $CA_KEY | sed 's/^/  /'
    115. echo
    116. echo "ca_cert: |"
    117. cat $CA_CERT | sed 's/^/  /'
    118. echo
    119. echo "ssl_key: |"
    120. cat $SSL_KEY | sed 's/^/  /'
    121. echo
    122. echo "ssl_csr: |"
    123. cat $SSL_CSR | sed 's/^/  /'
    124. echo
    125. echo "ssl_cert: |"
    126. cat $SSL_CERT | sed 's/^/  /'
    127. echo
    128. fi
    130. if [[ -n $K8S_SECRET_NAME ]]; then
    132.   if [[ -n $K8S_SECRET_COMBINE_CA ]]; then
    133.     [[ -z $SILENT ]] && echo "====> Adding CA to Cert file"
    134.     cat ${CA_CERT} >> ${SSL_CERT}
    135.   fi
    137.   [[ -z $SILENT ]] && echo "====> Creating Kubernetes secret: $K8S_SECRET_NAME"
    138.   kubectl delete secret $K8S_SECRET_NAME --ignore-not-found
    140.   if [[ -n $K8S_SECRET_SEPARATE_CA ]]; then
    141.     kubectl create secret generic \
    142.     $K8S_SECRET_NAME \
    143.     --from-file="tls.crt=${SSL_CERT}" \
    144.     --from-file="tls.key=${SSL_KEY}" \
    145.     --from-file="ca.crt=${CA_CERT}"
    146.   else
    147.     kubectl create secret tls \
    148.     $K8S_SECRET_NAME \
    149.     --cert=${SSL_CERT} \
    150.     --key=${SSL_KEY}
    151.   fi
    153.   if [[ -n $K8S_SECRET_LABELS ]]; then
    154.     [[ -z $SILENT ]] && echo "====> Labeling Kubernetes secret"
    155.     IFS=$' \n\t' # We have to reset IFS or label secret will misbehave on some systems
    156.     kubectl label secret \
    157.       $K8S_SECRET_NAME \
    158.       $K8S_SECRET_LABELS
    159.   fi
    160. fi
    162. echo "4. 重命名服务证书"
    163. mv ${CN}.key tls.key
    164. mv ${CN}.crt tls.crt
    167. # 把生成的证书作为密文导入K8S
    169. ## * 指定K8S配置文件路径
    171. kubeconfig=/home/rancher/.kube/config
    172. kubectl --kubeconfig=$kubeconfig create namespace cattle-system
    173. kubectl --kubeconfig=$kubeconfig -n cattle-system create secret tls tls-rancher-ingress --cert=./tls.crt --key=./tls.key
    174. kubectl --kubeconfig=$kubeconfig -n cattle-system create secret generic tls-ca --from-file=cacerts.pem

    配置仓库地址
    image.png

    1. [rancher@uat-rancher-node01 rancher]$ cat values.yaml 
    3. # Additional Trusted CAs.
    4. # Enable this flag and add your CA certs as a secret named tls-ca-additional in the namespace.
    5. # See README.md for details.
    6. additionalTrustedCAs: false
    8. antiAffinity: preferred
    10. # Audit Logs https://rancher.com/docs/rancher/v2.x/en/installation/api-auditing/
    11. # The audit log is piped to the console of the rancher-audit-log container in the rancher pod.
    12. # https://rancher.com/docs/rancher/v2.x/en/installation/api-auditing/
    13. # destination stream to sidecar container console or hostPath volume
    14. # level: Verbosity of logs, 0 to 3. 0 is off 3 is a lot.
    15. auditLog:
    16.   destination: sidecar
    17.   hostPath: /var/log/rancher/audit/
    18.   level: 0
    19.   maxAge: 1
    20.   maxBackup: 1
    21.   maxSize: 100
    22.   fluentbitImage: 10.182.220.240/cnrancher/rancher-auditlog-fluentbit
    23.   fluentbitImageTag: v1.0.0
    25. # The Mysql should be deployed manually and create the user and database schema for auditlog server.
    26. # You should use nonadministrative account and a high strength password to connect to the Mysql.
    27. auditLogServer:
    28.   image: 10.182.220.240/cnrancher/rancher-auditlog-server
    29.   imageTag: v1.0.0
    30.   replicas: 1
    31.   antiAffinity: preferred
    32.   serverPort: 9000
    33.   DBHost: localhost
    34.   DBPort: 3306
    35.   DBUser: root
    36.   DBPassword: password
    37.   DBName: rancher
    39. # Have Rancher detect and import the "local" Rancher server cluster
    40. # Adding the "local" cluster available in the GUI can be convenient, but any user with access to this cluster has "root" on any of the clusters that Rancher manages.
    41. # options; "auto", "false". (auto pretty much means true)
    42. addLocal: "auto"
    44. # Image for collecting rancher audit logs.
    45. # Important: update pkg/image/export/main.go when this default image is changed, so that it's reflected accordingly in rancher-images.txt generated for air-gapped setups.
    46. busyboxImage: 10.182.220.240/dev/busybox
    48. # Add debug flag to Rancher server
    49. debug: false
    51. # Extra environment variables passed to the rancher pods.
    52. # extraEnv:
    53. # - name: CATTLE_TLS_MIN_VERSION
    54. #   value: "1.0"
    56. # Fully qualified name to reach your Rancher server
    57. # hostname: rancher.my.org
    59. ## Optional array of imagePullSecrets containing private registry credentials
    60. ## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
    61. imagePullSecrets: []
    62. # - name: secretName
    64. ### ingress ###
    65. # Readme for details and instruction on adding tls secrets.
    66. ingress:
    67.   extraAnnotations:
    68.     nginx.ingress.kubernetes.io/proxy-connect-timeout: "30"
    69.     nginx.ingress.kubernetes.io/proxy-read-timeout: "1800"
    70.     nginx.ingress.kubernetes.io/proxy-send-timeout: "1800"
    72.   # configurationSnippet - Add additional Nginx configuration. This example statically sets a header on the ingress.
    73.   # configurationSnippet: |
    74.   #   more_set_input_headers "X-Forwarded-Host: {{ .Values.hostname }}";
    75.   configurationSnippet: |
    76.     more_clear_headers Server;
    78.   tls:
    79.     # options: rancher, letsEncrypt, secret
    80.     source: rancher
    82. ### LetsEncrypt config ###
    83. # ProTip: The production environment only allows you to register a name 5 times a week.
    84. #         Use staging until you have your config right.
    85. letsEncrypt:
    86.   # email: none@example.com
    87.   environment: production
    89. # If you are using certs signed by a private CA set to 'true' and set the 'tls-ca'
    90. # in the 'rancher-system' namespace. See the README.md for details
    91. privateCA: false
    93. # http[s] proxy server passed into rancher server.
    94. # proxy: http://<username>@<password>:<url>:<port>
    96. # comma separated list of domains or ip addresses that will not use the proxy
    97. noProxy: 127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16
    99. # Override rancher image location for Air Gap installs
    100. rancherImage: 10.182.220.240/cnrancher/rancher
    101. ## set Air Gap registry for rancher
    102. # rancherRegistry: harbor.cnrancher.com
    103. systemDefaultRegistry: ""
    105. # rancher/rancher image tag. https://hub.docker.com/r/rancher/rancher/tags/
    106. # Defaults to .Chart.appVersion
    107. # rancherImageTag: v2.0.7
    109. # Override imagePullPolicy for rancher server images
    110. # options: Always, Never, IfNotPresent
    111. # Defaults to IfNotPresent
    112. # rancherImagePullPolicy: <pullPolicy>
    114. # Number of Rancher server replicas.
    115. replicas: 3
    117. # Set pod resource requests/limits for Rancher.
    118. resources: {}
    120. #
    121. # tls
    122. #   Where to offload the TLS/SSL encryption
    123. # - ingress (default)
    124. # - external
    125. tls: ingress
    127. # Set to use the packaged system charts
    128. useBundledSystemChart: false
    130. service:
    131.   type: ClusterIP
    132.   ports:
    133.     nodePort: 30443
    135. # Certmanager version compatibility
    136. certmanager:
    137.   version: ""
    138. [rancher@uat-rancher-node01 rancher]$

    image.png

    1. helm install rancher rancher/   --namespace cattle-system --set rancherImage=cnrancher/rancher --set service.type=NodePort --set service.ports.nodePort=30001 --set tls=internal --set privateCA=true

    image.png

    image.png