阿里云 ACK集群文档，容器服务ACK在基因计算、AI大数据等领域提供了高度集成的解决方案，结合IaaS高性能计算、网络能力，发挥容器的最佳性能。在多云混合云领域，容器服务ACK提供了多集群统一管理能力，您可在容器服务控制台，统一管理来自线下IDC，或者其他云上的Kubernetes集群。

购买集群、节点池

  • 购买地址，托管版本集群
  • 选择节点池，并放在同一个VPC网络 -> 交换机** **下面。
  • 购买单台服务架设VPN
  • 购买2个公网IP
  • NAT 网关 负责SNAT DNAT

集群拓架构图

阿里云 Kubernetes - 图1

安装必要组件

阿里云安装的集群托管版，是有一些必要的组件是需要手动安装的

Ingress

安装

在右上角搜索 ingress 体验原汁原味的K8S 选择 Nginx Ingress Controller 进行安装

image.png

安装参数默认参数；点击确定即可

image.png

配置

修改 kube-system 命名空间配置下的 nginx-ingress-controller 配置; 调度修改成0

新增 DaemonSet守护进程，保证每个宿主机节点必须有一个Ingress入网口

  1. apiVersion: apps/v1
  2. kind: DaemonSet
  3. metadata:
  4. annotations:
  5. component.revision: '2'
  6. component.version: 1.1.0
  7. deployment.kubernetes.io/revision: '3'
  8. deprecated.daemonset.template.generation: '1'
  9. kubectl.kubernetes.io/last-applied-configuration: >
  10. {"apiVersion":"apps/v1","kind":"Deployment","metadata":{"annotations":{"component.revision":"2","component.version":"1.1.0"},"labels":{"app":"ingress-nginx"},"name":"nginx-ingress-controller","namespace":"kube-system"},"spec":{"minReadySeconds":0,"replicas":2,"revisionHistoryLimit":10,"selector":{"matchLabels":{"app":"ingress-nginx"}},"template":{"metadata":{"annotations":{"prometheus.io/port":"10254","prometheus.io/scrape":"true"},"labels":{"app":"ingress-nginx"}},"spec":{"affinity":{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchExpressions":[{"key":"type","operator":"NotIn","values":["virtual-kubelet"]},{"key":"k8s.aliyun.com","operator":"NotIn","values":["true"]}]}]}},"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app","operator":"In","values":["ingress-nginx"]}]},"topologyKey":"kubernetes.io/hostname"},"weight":100}]}},"containers":[{"args":["/nginx-ingress-controller","--election-id=ingress-controller-leader-nginx","--ingress-class=nginx","--watch-ingress-without-class","--controller-class=k8s.io/ingress-nginx","--configmap=$(POD_NAMESPACE)/nginx-configuration","--tcp-services-configmap=$(POD_NAMESPACE)/tcp-services","--udp-services-configmap=$(POD_NAMESPACE)/udp-services","--annotations-prefix=nginx.ingress.kubernetes.io","--publish-service=$(POD_NAMESPACE)/nginx-ingress-lb","--validating-webhook=:8443","--validating-webhook-certificate=/usr/local/certificates/cert","--validating-webhook-key=/usr/local/certificates/key","--v=2"],"env":[{"name":"POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"POD_NAMESPACE","valueFrom":{"fieldRef":{"fieldPath":"metadata.namespace"}}},{"name":"LD_PRELOAD","value":"/usr/local/lib/libmimalloc.so"}],"image":"registry-vpc.cn-shanghai.aliyuncs.com/acs/aliyun-ingress-controller:v1.1.0-aliyun.2","imagePullPolicy":"IfNotPresent","lifecycle":{"preStop":{"exec":{"command":["/wait-shutdown"]}}},"livenessProbe":{"failureThreshold":5,"httpGet":{"path":"/healthz","port":10254,"scheme":"HTTP"},"initialDelaySeconds":10,"periodSeconds":10,"successThreshold":1,"timeoutSeconds":1},"name":"nginx-ingress-controller","ports":[{"containerPort":80,"name":"http","protocol":"TCP"},{"containerPort":443,"name":"https","protocol":"TCP"},{"containerPort":8443,"name":"webhook","protocol":"TCP"}],"readinessProbe":{"failureThreshold":3,"httpGet":{"path":"/healthz","port":10254,"scheme":"HTTP"},"initialDelaySeconds":10,"periodSeconds":10,"successThreshold":1,"timeoutSeconds":1},"resources":{"requests":{"cpu":"90m","memory":"100Mi"}},"securityContext":{"allowPrivilegeEscalation":true,"capabilities":{"add":["NET_BIND_SERVICE"],"drop":["ALL"]},"runAsUser":101},"volumeMounts":[{"mountPath":"/usr/local/certificates/","name":"webhook-cert","readOnly":true},{"mountPath":"/etc/localtime","name":"localtime","readOnly":true}]}],"dnsPolicy":"ClusterFirst","hostNetwork":false,"initContainers":[{"command":["/bin/sh","-c","mount
  11. -o remount rw /proc/sys\nsysctl -w net.core.somaxconn=65535\nsysctl -w
  12. net.ipv4.ip_local_port_range=\"1024 65535\"\nsysctl -w
  13. kernel.core_uses_pid=0\n"],"image":"registry-vpc.cn-shanghai.aliyuncs.com/acs/busybox:v1.29.2","name":"init-sysctl","securityContext":{"capabilities":{"add":["SYS_ADMIN"],"drop":["ALL"]}}}],"nodeSelector":{"kubernetes.io/os":"linux"},"priorityClassName":"system-node-critical","serviceAccountName":"ingress-nginx","terminationGracePeriodSeconds":300,"volumes":[{"name":"webhook-cert","secret":{"secretName":"ingress-nginx-admission"}},{"hostPath":{"path":"/etc/localtime","type":"File"},"name":"localtime"}]}}}}
  14. creationTimestamp: '2022-04-08T09:58:33Z'
  15. generation: 1
  16. labels:
  17. app: ingress-nginx
  18. managedFields:
  19. - apiVersion: apps/v1
  20. fieldsType: FieldsV1
  21. fieldsV1:
  22. 'f:spec':
  23. 'f:progressDeadlineSeconds': {}
  24. 'f:replicas': {}
  25. 'f:strategy':
  26. 'f:rollingUpdate':
  27. .: {}
  28. 'f:maxSurge': {}
  29. 'f:maxUnavailable': {}
  30. 'f:type': {}
  31. manager: rc
  32. operation: Update
  33. time: '2022-03-10T06:14:09Z'
  34. - apiVersion: apps/v1
  35. fieldsType: FieldsV1
  36. fieldsV1:
  37. 'f:metadata':
  38. 'f:annotations':
  39. .: {}
  40. 'f:component.revision': {}
  41. 'f:component.version': {}
  42. 'f:deployment.kubernetes.io/revision': {}
  43. 'f:deprecated.daemonset.template.generation': {}
  44. 'f:kubectl.kubernetes.io/last-applied-configuration': {}
  45. 'f:labels':
  46. .: {}
  47. 'f:app': {}
  48. 'f:spec':
  49. 'f:revisionHistoryLimit': {}
  50. 'f:selector': {}
  51. 'f:template':
  52. 'f:metadata':
  53. 'f:annotations':
  54. .: {}
  55. 'f:prometheus.io/port': {}
  56. 'f:prometheus.io/scrape': {}
  57. 'f:labels':
  58. .: {}
  59. 'f:app': {}
  60. 'f:spec':
  61. 'f:affinity':
  62. .: {}
  63. 'f:podAntiAffinity':
  64. .: {}
  65. 'f:preferredDuringSchedulingIgnoredDuringExecution': {}
  66. 'f:containers':
  67. 'k:{"name":"nginx-ingress-controller"}':
  68. .: {}
  69. 'f:args': {}
  70. 'f:env':
  71. .: {}
  72. 'k:{"name":"LD_PRELOAD"}':
  73. .: {}
  74. 'f:name': {}
  75. 'f:value': {}
  76. 'k:{"name":"POD_NAME"}':
  77. .: {}
  78. 'f:name': {}
  79. 'f:valueFrom':
  80. .: {}
  81. 'f:fieldRef': {}
  82. 'k:{"name":"POD_NAMESPACE"}':
  83. .: {}
  84. 'f:name': {}
  85. 'f:valueFrom':
  86. .: {}
  87. 'f:fieldRef': {}
  88. 'f:image': {}
  89. 'f:imagePullPolicy': {}
  90. 'f:lifecycle':
  91. .: {}
  92. 'f:preStop':
  93. .: {}
  94. 'f:exec':
  95. .: {}
  96. 'f:command': {}
  97. 'f:livenessProbe':
  98. .: {}
  99. 'f:failureThreshold': {}
  100. 'f:httpGet':
  101. .: {}
  102. 'f:path': {}
  103. 'f:port': {}
  104. 'f:scheme': {}
  105. 'f:initialDelaySeconds': {}
  106. 'f:periodSeconds': {}
  107. 'f:successThreshold': {}
  108. 'f:timeoutSeconds': {}
  109. 'f:name': {}
  110. 'f:ports':
  111. .: {}
  112. 'k:{"containerPort":443,"protocol":"TCP"}':
  113. .: {}
  114. 'f:containerPort': {}
  115. 'f:name': {}
  116. 'f:protocol': {}
  117. 'k:{"containerPort":80,"protocol":"TCP"}':
  118. .: {}
  119. 'f:containerPort': {}
  120. 'f:name': {}
  121. 'f:protocol': {}
  122. 'k:{"containerPort":8443,"protocol":"TCP"}':
  123. .: {}
  124. 'f:containerPort': {}
  125. 'f:name': {}
  126. 'f:protocol': {}
  127. 'f:readinessProbe':
  128. .: {}
  129. 'f:failureThreshold': {}
  130. 'f:httpGet':
  131. .: {}
  132. 'f:path': {}
  133. 'f:port': {}
  134. 'f:scheme': {}
  135. 'f:initialDelaySeconds': {}
  136. 'f:periodSeconds': {}
  137. 'f:successThreshold': {}
  138. 'f:timeoutSeconds': {}
  139. 'f:resources':
  140. .: {}
  141. 'f:requests':
  142. .: {}
  143. 'f:cpu': {}
  144. 'f:memory': {}
  145. 'f:securityContext':
  146. .: {}
  147. 'f:allowPrivilegeEscalation': {}
  148. 'f:capabilities':
  149. .: {}
  150. 'f:add': {}
  151. 'f:drop': {}
  152. 'f:runAsUser': {}
  153. 'f:terminationMessagePath': {}
  154. 'f:terminationMessagePolicy': {}
  155. 'f:volumeMounts':
  156. .: {}
  157. 'k:{"mountPath":"/etc/localtime"}':
  158. .: {}
  159. 'f:mountPath': {}
  160. 'f:name': {}
  161. 'f:readOnly': {}
  162. 'k:{"mountPath":"/usr/local/certificates/"}':
  163. .: {}
  164. 'f:mountPath': {}
  165. 'f:name': {}
  166. 'f:readOnly': {}
  167. 'f:dnsPolicy': {}
  168. 'f:initContainers':
  169. .: {}
  170. 'k:{"name":"init-sysctl"}':
  171. .: {}
  172. 'f:command': {}
  173. 'f:image': {}
  174. 'f:imagePullPolicy': {}
  175. 'f:name': {}
  176. 'f:resources': {}
  177. 'f:securityContext':
  178. .: {}
  179. 'f:capabilities':
  180. .: {}
  181. 'f:add': {}
  182. 'f:drop': {}
  183. 'f:terminationMessagePath': {}
  184. 'f:terminationMessagePolicy': {}
  185. 'f:nodeSelector': {}
  186. 'f:priorityClassName': {}
  187. 'f:restartPolicy': {}
  188. 'f:schedulerName': {}
  189. 'f:securityContext': {}
  190. 'f:serviceAccount': {}
  191. 'f:serviceAccountName': {}
  192. 'f:terminationGracePeriodSeconds': {}
  193. 'f:volumes':
  194. .: {}
  195. 'k:{"name":"localtime"}':
  196. .: {}
  197. 'f:hostPath':
  198. .: {}
  199. 'f:path': {}
  200. 'f:type': {}
  201. 'f:name': {}
  202. 'k:{"name":"webhook-cert"}':
  203. .: {}
  204. 'f:name': {}
  205. 'f:secret':
  206. .: {}
  207. 'f:defaultMode': {}
  208. 'f:secretName': {}
  209. 'f:updateStrategy':
  210. 'f:rollingUpdate':
  211. .: {}
  212. 'f:maxSurge': {}
  213. 'f:maxUnavailable': {}
  214. 'f:type': {}
  215. manager: ACK-Console Apache-HttpClient
  216. operation: Update
  217. time: '2022-04-08T09:58:33Z'
  218. - apiVersion: apps/v1
  219. fieldsType: FieldsV1
  220. fieldsV1:
  221. 'f:status':
  222. 'f:availableReplicas': {}
  223. 'f:conditions':
  224. .: {}
  225. 'k:{"type":"Available"}':
  226. .: {}
  227. 'f:lastTransitionTime': {}
  228. 'f:lastUpdateTime': {}
  229. 'f:message': {}
  230. 'f:reason': {}
  231. 'f:status': {}
  232. 'f:type': {}
  233. 'k:{"type":"Progressing"}':
  234. .: {}
  235. 'f:lastTransitionTime': {}
  236. 'f:lastUpdateTime': {}
  237. 'f:message': {}
  238. 'f:reason': {}
  239. 'f:status': {}
  240. 'f:type': {}
  241. 'f:currentNumberScheduled': {}
  242. 'f:desiredNumberScheduled': {}
  243. 'f:numberAvailable': {}
  244. 'f:numberMisscheduled': {}
  245. 'f:numberReady': {}
  246. 'f:observedGeneration': {}
  247. 'f:readyReplicas': {}
  248. 'f:replicas': {}
  249. 'f:updatedNumberScheduled': {}
  250. 'f:updatedReplicas': {}
  251. manager: kube-controller-manager
  252. operation: Update
  253. subresource: status
  254. time: '2022-04-10T09:21:25Z'
  255. name: nginx-ingress-controller
  256. namespace: kube-system
  257. resourceVersion: '47048176'
  258. uid: 78293e59-c8ea-4b19-9307-d4a097095fc5
  259. spec:
  260. revisionHistoryLimit: 10
  261. selector:
  262. matchLabels:
  263. app: ingress-nginx
  264. template:
  265. metadata:
  266. annotations:
  267. prometheus.io/port: '10254'
  268. prometheus.io/scrape: 'true'
  269. labels:
  270. app: ingress-nginx
  271. spec:
  272. affinity:
  273. podAntiAffinity:
  274. preferredDuringSchedulingIgnoredDuringExecution:
  275. - podAffinityTerm:
  276. labelSelector:
  277. matchExpressions:
  278. - key: app
  279. operator: In
  280. values:
  281. - ingress-nginx
  282. topologyKey: kubernetes.io/hostname
  283. weight: 100
  284. containers:
  285. - args:
  286. - /nginx-ingress-controller
  287. - '--election-id=ingress-controller-leader-nginx'
  288. - '--ingress-class=nginx'
  289. - '--watch-ingress-without-class'
  290. - '--controller-class=k8s.io/ingress-nginx'
  291. - '--configmap=$(POD_NAMESPACE)/nginx-configuration'
  292. - '--tcp-services-configmap=$(POD_NAMESPACE)/tcp-services'
  293. - '--udp-services-configmap=$(POD_NAMESPACE)/udp-services'
  294. - '--annotations-prefix=nginx.ingress.kubernetes.io'
  295. - '--publish-service=$(POD_NAMESPACE)/nginx-ingress-lb'
  296. - '--validating-webhook=:8443'
  297. - '--validating-webhook-certificate=/usr/local/certificates/cert'
  298. - '--validating-webhook-key=/usr/local/certificates/key'
  299. - '--v=2'
  300. env:
  301. - name: POD_NAME
  302. valueFrom:
  303. fieldRef:
  304. apiVersion: v1
  305. fieldPath: metadata.name
  306. - name: POD_NAMESPACE
  307. valueFrom:
  308. fieldRef:
  309. apiVersion: v1
  310. fieldPath: metadata.namespace
  311. - name: LD_PRELOAD
  312. value: /usr/local/lib/libmimalloc.so
  313. image: >-
  314. registry-vpc.cn-shanghai.aliyuncs.com/acs/aliyun-ingress-controller:v1.1.0-aliyun.2
  315. imagePullPolicy: IfNotPresent
  316. lifecycle:
  317. preStop:
  318. exec:
  319. command:
  320. - /wait-shutdown
  321. livenessProbe:
  322. failureThreshold: 5
  323. httpGet:
  324. path: /healthz
  325. port: 10254
  326. scheme: HTTP
  327. initialDelaySeconds: 10
  328. periodSeconds: 10
  329. successThreshold: 1
  330. timeoutSeconds: 1
  331. name: nginx-ingress-controller
  332. ports:
  333. - containerPort: 80
  334. name: http
  335. protocol: TCP
  336. - containerPort: 443
  337. name: https
  338. protocol: TCP
  339. - containerPort: 8443
  340. name: webhook
  341. protocol: TCP
  342. readinessProbe:
  343. failureThreshold: 3
  344. httpGet:
  345. path: /healthz
  346. port: 10254
  347. scheme: HTTP
  348. initialDelaySeconds: 10
  349. periodSeconds: 10
  350. successThreshold: 1
  351. timeoutSeconds: 1
  352. resources:
  353. requests:
  354. cpu: 90m
  355. memory: 100Mi
  356. securityContext:
  357. allowPrivilegeEscalation: true
  358. capabilities:
  359. add:
  360. - NET_BIND_SERVICE
  361. drop:
  362. - ALL
  363. runAsUser: 101
  364. terminationMessagePath: /dev/termination-log
  365. terminationMessagePolicy: File
  366. volumeMounts:
  367. - mountPath: /usr/local/certificates/
  368. name: webhook-cert
  369. readOnly: true
  370. - mountPath: /etc/localtime
  371. name: localtime
  372. readOnly: true
  373. dnsPolicy: ClusterFirst
  374. initContainers:
  375. - command:
  376. - /bin/sh
  377. - '-c'
  378. - |
  379. mount -o remount rw /proc/sys
  380. sysctl -w net.core.somaxconn=65535
  381. sysctl -w net.ipv4.ip_local_port_range="1024 65535"
  382. sysctl -w kernel.core_uses_pid=0
  383. image: 'registry-vpc.cn-shanghai.aliyuncs.com/acs/busybox:v1.29.2'
  384. imagePullPolicy: IfNotPresent
  385. name: init-sysctl
  386. resources: {}
  387. securityContext:
  388. capabilities:
  389. add:
  390. - SYS_ADMIN
  391. drop:
  392. - ALL
  393. terminationMessagePath: /dev/termination-log
  394. terminationMessagePolicy: File
  395. nodeSelector:
  396. kubernetes.io/os: linux
  397. priorityClassName: system-node-critical
  398. restartPolicy: Always
  399. schedulerName: default-scheduler
  400. securityContext: {}
  401. serviceAccount: ingress-nginx
  402. serviceAccountName: ingress-nginx
  403. terminationGracePeriodSeconds: 300
  404. volumes:
  405. - name: webhook-cert
  406. secret:
  407. defaultMode: 420
  408. secretName: ingress-nginx-admission
  409. - hostPath:
  410. path: /etc/localtime
  411. type: File
  412. name: localtime
  413. updateStrategy:
  414. rollingUpdate:
  415. maxSurge: 0
  416. maxUnavailable: 1
  417. type: RollingUpdate
  418. status:
  419. currentNumberScheduled: 3
  420. desiredNumberScheduled: 3
  421. numberAvailable: 3
  422. numberMisscheduled: 0
  423. numberReady: 3
  424. observedGeneration: 1
  425. updatedNumberScheduled: 3

修改 kube-system 命名空间配置下的Service配置 nginx-ingress-lb

  • 外部流量策略：一定要选择 Local 否则在Cluster 集群模式下会重新转发一次请求网关，导致源IP会丢失
  • 节点端口：我这边不采用负载均衡，成本过高，采用节点端口``4层网络暴露给SLB

image.png

  1. apiVersion: v1
  2. kind: Service
  3. metadata:
  4.   annotations:
  5.     kubectl.kubernetes.io/last-applied-configuration: >
  6.       {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"service.beta.kubernetes.io/alibaba-cloud-loadbalancer-resource-group-id":"rg-aekzhqulpipexny"},"labels":{"app":"nginx-ingress-lb"},"name":"nginx-ingress-lb","namespace":"kube-system"},"spec":{"externalTrafficPolicy":"Local","ipFamilyPolicy":"SingleStack","ports":[{"name":"http","port":80,"targetPort":80},{"name":"https","port":443,"targetPort":443}],"selector":{"app":"ingress-nginx"},"type":"LoadBalancer"}}
  7.     service.beta.kubernetes.io/alibaba-cloud-loadbalancer-resource-group-id: rg-aekzhqulpipexny
  8.   creationTimestamp: '2022-03-10T06:14:09Z'
  9.   labels:
  10.     app: nginx-ingress-lb
  11.   managedFields:
  12.     - apiVersion: v1
  13.       fieldsType: FieldsV1
  14.       fieldsV1:
  15.         'f:metadata':
  16.           'f:annotations':
  17.             .: {}
  18.             'f:kubectl.kubernetes.io/last-applied-configuration': {}
  19.             'f:service.beta.kubernetes.io/alibaba-cloud-loadbalancer-resource-group-id': {}
  20.           'f:labels':
  21.             .: {}
  22.             'f:app': {}
  23.         'f:spec':
  24.           'f:allocateLoadBalancerNodePorts': {}
  25.           'f:internalTrafficPolicy': {}
  26.           'f:ipFamilyPolicy': {}
  27.           'f:ports':
  28.             .: {}
  29.             'k:{"port":443,"protocol":"TCP"}':
  30.               .: {}
  31.               'f:name': {}
  32.               'f:port': {}
  33.               'f:protocol': {}
  34.               'f:targetPort': {}
  35.             'k:{"port":80,"protocol":"TCP"}':
  36.               .: {}
  37.               'f:name': {}
  38.               'f:port': {}
  39.               'f:protocol': {}
  40.               'f:targetPort': {}
  41.           'f:selector': {}
  42.           'f:sessionAffinity': {}
  43.       manager: rc
  44.       operation: Update
  45.       time: '2022-03-10T06:14:09Z'
  46.     - apiVersion: v1
  47.       fieldsType: FieldsV1
  48.       fieldsV1:
  49.         'f:spec':
  50.           'f:externalTrafficPolicy': {}
  51.           'f:ports':
  52.             'k:{"port":443,"protocol":"TCP"}':
  53.               'f:nodePort': {}
  54.             'k:{"port":80,"protocol":"TCP"}':
  55.               'f:nodePort': {}
  56.           'f:type': {}
  57.       manager: ACK-Console Apache-HttpClient
  58.       operation: Update
  59.       time: '2022-03-28T07:51:54Z'
  60.   name: nginx-ingress-lb
  61.   namespace: kube-system
  62.   resourceVersion: '42014734'
  63.   uid: 38f86a16-1cd1-430a-9a9d-5bb913b92ebb
  64. spec:
  65.   clusterIP: 172.16.161.244
  66.   clusterIPs:
  67.     - 172.16.161.244
  68.   externalTrafficPolicy: Cluster
  69.   internalTrafficPolicy: Cluster
  70.   ipFamilies:
  71.     - IPv4
  72.   ipFamilyPolicy: SingleStack
  73.   ports:
  74.     - name: http
  75.       nodePort: 31080
  76.       port: 80
  77.       protocol: TCP
  78.       targetPort: 80
  79.     - name: https
  80.       nodePort: 31443
  81.       port: 443
  82.       protocol: TCP
  83.       targetPort: 443
  84.   selector:
  85.     app: ingress-nginx
  86.   sessionAffinity: None
  87.   type: NodePort
  88. status:
  89.   loadBalancer: {}