环境准备

IP Hostname 内核 CPU Memory
10.240.0.3 master 3.10.0-1062 2 4G
10.240.0.4 node01 3.10.0-1062 2 4G
10.240.0.5 node02 3.10.0-1062 2 4G
10.240.0.6 node01 3.10.0-1062 2 4G

准备k8s集群

image.png

部署eureka

  1. cat > eureka.yaml << EOF
  2. apiVersion: v1
  3. kind: Service
  4. metadata:
  5.   name: eureka
  6.   labels:
  7.     app: eureka
  8. spec:
  9.   ports:
  10.     - port: 8761
  11.       name: eureka
  12.   clusterIP: None
  13.   selector:
  14.     app: eureka
  15. ---
  16. apiVersion: v1
  17. kind: Service
  18. metadata:
  19.   name: eureka-nodeport
  20.   labels:
  21.     app: eureka-nodeport
  22. spec:
  23.   type: NodePort
  24.   ports:
  25.     - port: 8761
  26.       name: eureka
  27.       nodePort: 31820
  28.   selector:
  29.     app: eureka
  30. ---
  31. apiVersion: apps/v1
  32. kind: StatefulSet
  33. metadata:
  34.   name: eureka
  35. spec:
  36.   serviceName: "eureka"
  37.   replicas: 3
  38.   selector:
  39.     matchLabels:
  40.       app: eureka
  41.   template:
  42.     metadata:
  43.       labels:
  44.         app: eureka
  45.     spec:
  46.       containers:
  47.         - name: eureka
  48.           image: freemanliu/eureka:v1.0.0
  49.           ports:
  50.             - containerPort: 8761
  51.           resources:
  52.             limits:
  53.               # jvm会自动发现该限制
  54.               memory: 2Gi
  55.           env:
  56.             - name: MY_POD_NAME
  57.               valueFrom:
  58.                 fieldRef:
  59.                   fieldPath: metadata.name
  60.             - name: JAVA_OPTS
  61.               value: -XX:+UnlockExperimentalVMOptions
  62.                 -XX:+UseCGroupMemoryLimitForHeap
  63.                 -XX:MaxRAMFraction=2
  64.                 -XX:CICompilerCount=8
  65.                 -XX:ActiveProcessorCount=8
  66.                 -XX:+UseG1GC
  67.                 -XX:+AggressiveOpts
  68.                 -XX:+UseFastAccessorMethods
  69.                 -XX:+UseStringDeduplication
  70.                 -XX:+UseCompressedOops
  71.                 -XX:+OptimizeStringConcat
  72.             - name: EUREKA_SERVER
  73.               value: "http://eureka-0.eureka:8761/eureka/,http://eureka-1.eureka:8761/eureka/,http://eureka-2.eureka:8761/eureka/"
  74.             - name: EUREKA_INSTANCE_HOSTNAME
  75.               value: ${MY_POD_NAME}.eureka
  76.   podManagementPolicy: "Parallel"
  77. EOF
  1. kubectl  apply -f eureka.yaml

image.png
访问ip:nodeport可以看到
image.png

部署apollo

下载git包

  1. git clone https://github.com/ctripcorp/apollo.git

image.png

创建数据库

直接在master上部署数据库

  1. wget -c https://dev.mysql.com/get/mysql80-community-release-el7-1.noarch.rpm
  2. rpm -ivh mysql80-community-release-el7-1.noarch.rpm
  3. yum -y install yum-utils
  4. yum-config-manager --disable mysql80-community
  5. yum-config-manager --enable mysql57-community
  6. yum install mysql-community-server -y
  7. # 启动mysql
  8. systemctl start mysqld
  9. # 开机启动
  10. systemctl enable mysqld
  11. # 查看root临时密码
  12. grep 'temporary password' /var/log/mysqld.log
  13. # 使用mysql临时登录，修改root密码
  14. ALTER USER 'root'@'localhost' IDENTIFIED BY '123456@Abc';

配置mysql 参数
在/etc/my.cnf添加以下参数

  1. sql_mode=STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION
  2. skip-name-resolve
  3. lower_case_table_names = 1
  4. max_connections = 4000
  5. character-set-server = utf8mb4  
  6. collation-server = utf8mb4_unicode_ci  
  7. init_connect='SET NAMES utf8mb4'  
  8. skip-character-set-client-handshake = true  
  9. [mysql]  
  10. default-character-set = utf8mb4

重启mysql

  1. systemctl restart mysqld

建库建表

  1. mysql> source /root/apollo/scripts/sql/apolloconfigdb.sql
  1. source /root/apollo/scripts/sql/apolloportaldb.sql
  1. use apolloconfigdb;
  2. update ServerConfig set Value='http://eureka-0.eureka:8761/eureka/' where id=1;

修改configservice配置

  1. vim /root/apollo/apollo-configservice/src/main/config/apollo-configservice.conf
  1. LOG_FOLDER=/opt/logs/config/

只修改下面内容，调整日志目录
image.png

  1. vim /root/apollo/apollo-configservice/src/main/config/application-github.properties
  1. apollo.eureka.server.enabled=false

增加如下配置，禁用apollo的eureka，使用上面独立eureka集群
image.png

  1. vim /root/apollo/apollo-configservice/src/main/resources/application.yml
  1. file: /opt/logs/config/apollo-configservice.log

修改日志目录为config/apollo-configservice.conf中配置目录，指定文件名
image.png

  1. vim  /root/apollo/apollo-configservice/src/main/resources/bootstrap.yml
  1. eureka:
  2.   instance:
  3.     homePageUrl: http://k8s-node-ip:30003
  4.     # 修改此配置如下
  5.     preferIpAddress: false
  7. # 增加此属性， 禁用apollo自带的eureka
  8. apollo:
  9.   eureka:
  10.     server:
  11.       enabled: false

只修改图vim /root/apollo/apollo-configservice/src/main/resources/bootstrap.yml中内容
image.png

  1. vim  /root/apollo/apollo-configservice/src/main/resources/configservice.properties
  1. logging.file= /opt/logs/config/apollo-configservice.log

修改日志路径
image.png

  1. vim  /root/apollo/apollo-configservice/src/main/scripts/startup.sh
  1. LOG_DIR=/opt/logs/config

image.png

修改adminservier配置

  1. vim  /root/apollo/apollo-adminservice/src/main/config/apollo-adminservice.conf
  1. LOG_FOLDER=/opt/logs/admin/

修改日志路径
image.png

  1. vim  /root/apollo/apollo-adminservice/src/main/config/application-github.properties
  1. apollo.eureka.server.enabled=false

增加如下配置，禁用apollo的eureka，使用上面独立eureka集群
image.png

  1. vim  /root/apollo/apollo-adminservice/src/main/resources/adminservice.properties
  1. logging.file= /opt/logs/admin/apollo-adminservice.log

image.png

  1. vim  /root/apollo/apollo-adminservice/src/main/resources/application.yml
  1.  file: /opt/logs/admin/apollo-adminservice.log

image.png

  1. vim  /root/apollo/apollo-adminservice/src/main/resources/bootstrap.yml
  1. eureka:
  2.   instance:
  3.     # 修改此配置如下 28091端口号在第十步中说明 k8s-node-ip替换为自己k8s节点ip
  4.     homePageUrl: http://k8s-node-ip:28091
  5.     # 修改此配置如下
  6.     preferIpAddress: false
  8. # 增加此属性， 禁用apollo自带的eureka
  9. apollo:
  10.   eureka:
  11.     server:
  12.       enabled: false

image.png

  1. vim  /root/apollo/apollo-adminservice/src/main/scripts/startup.sh
  1. LOG_DIR=/opt/logs/admin

image.png

修改portal配置

  1. vim  /root/apollo/apollo-portal/src/main/config/apollo-portal.conf
  1. LOG_FOLDER=/opt/logs/portal/

image.png

  1. vim  /root/apollo/apollo-portal/src/main/resources/application.yml
  1. file: ~opt/logs/portal/apollo-portal.log

image.png

  1.  vim  /root/apollo/apollo-portal/src/main/scripts/startup.sh
  1. LOG_DIR=/opt/logs/portal

image.png

修改Dockerfile内容

  1. vim  /root/apollo/scripts/apollo-on-kubernetes/apollo-config-server/Dockerfile
  1. ENV APOLLO_ADMIN_SERVICE_NAME=“service-apollo-admin-server.apollo”

image.png

  1. vim  /root/apollo/scripts/apollo-on-kubernetes/apollo-config-server/scripts/startup-kubernetes.sh
  1. LOG_DIR=/opt/logs/config

image.png

  1. vim  /root/apollo/scripts/apollo-on-kubernetes/apollo-admin-server/Dockerfile
  1. ENV APOLLO_PORTAL_SERVICE_NAME=“service-apollo-portal-server.apollo”

image.png

  1. vim  /root/apollo/scripts/apollo-on-kubernetes/apollo-admin-server/scripts/startup-kubernetes.sh
  1. LOG_DIR=/opt/logs/admin

image.png

  1. vim  /root/apollo/scripts/apollo-on-kubernetes/apollo-portal-server/Dockerfile
  1. ENV APOLLO_PORTAL_SERVICE_NAME=“service-apollo-portal-server.apollo”

image.png

  1. vim  /root/apollo/scripts/apollo-on-kubernetes/apollo-portal-server/scripts/startup-kubernetes.sh
  1. LOG_DIR=/opt/logs/portal

image.png

执行编译脚本

  1. sh /root/apollo/scripts/build.sh

复制jar包

  1. unzip  /root/apollo/apollo-adminservice/target/apollo-adminservice-1.8.0-SNAPSHOT-github.zip   -d   /root/apollo-adminservice
  1. cp /root/apollo-adminservice/apollo-adminservice-1.8.0-SNAPSHOT.jar   /root/apollo/scripts/apollo-on-kubernetes/apollo-admin-server/apollo-adminservice.jar
  1. unzip  /root/apollo/apollo-configservice/target/apollo-configservice-1.8.0-SNAPSHOT-github.zip  -d  /root/apollo-configservice
  1. cp /root/apollo-configservice/apollo-configservice-1.8.0-SNAPSHOT.jar   /root/apollo/scripts/apollo-on-kubernetes/apollo-config-server/apollo-configservice.jar
  1.  unzip /root/apollo/apollo-portal/target/apollo-portal-1.8.0-SNAPSHOT-github.zip  -d /root/apollo-portal
  1. cp /root/apollo-portal/apollo-portal-1.8.0-SNAPSHOT.jar /root/apollo/scripts/apollo-on-kubernetes/apollo-portal-server/apollo-portal.jar

编译成镜像并push到dockerhub

  1. cd /root/apollo/scripts/apollo-on-kubernetes/apollo-admin-server/
  1. docker build -t tanmgweiwow/apollo-adminservice:v1.0 .
  1. docker push tanmgweiwow/apollo-adminservice:v1.0
  1. cd /root/apollo/scripts/apollo-on-kubernetes/apollo-config-server/
  1. docker build -t tanmgweiwow/apollo-configservice:v1.0 .
  1. docker push tanmgweiwow/apollo-configservice:v1.0
  1. cd /root/apollo/scripts/apollo-on-kubernetes/apollo-portal-server/
  1. docker build -t tanmgweiwow/apollo-portal:v1.0 .
  1. docker push  tanmgweiwow/apollo-portal:v1.0

编写k8s yaml文件

  1. cat > apollo-config-server.yaml << EOF
  2. kind: ConfigMap
  3. apiVersion: v1
  4. metadata:
  5.   namespace: apollo                                                 
  6.   name: configmap-apollo-config-server-dev
  7. data:        
  8.   application-github.properties: |
  9.     spring.datasource.url = jdbc:mysql://35.201.132.33:3306/apolloconfigdb?characterEncoding=utf8   
  10.     spring.datasource.username = root
  11.     spring.datasource.password = 123456@Abc
  12.     eureka.service.url = http://eureka-0.eureka.default:8761/eureka/,http://eureka-1.eureka.default:8761/eureka/,http://eureka-2.eureka.default:8761/eureka/
  13. ---
  14. kind: Service
  15. apiVersion: v1
  16. metadata:
  17.   namespace: apollo
  18.   name: service-apollo-meta-server-dev
  19.   labels:
  20.     app: service-apollo-meta-server-dev
  21. spec:
  22.   ports:
  23.     - protocol: TCP
  24.       port: 8080                          # 设定Serivce对外提供服务的端口
  25.       targetPort: 8080                    # 设定容器(Pod)的端口,即Pod网络的端口
  26.       nodePort: 30003                      # 将meta-server服务对节点外开发，让开发机器也能访问到k8s中的apollo
  27.   selector:
  28.     app: pod-apollo-config-server-dev
  29.   type: NodePort
  30.   sessionAffinity: ClientIP
  31. ---
  32. kind: StatefulSet
  33. apiVersion: apps/v1
  34. metadata:
  35.   namespace: apollo
  36.   name: statefulset-apollo-config-server-dev
  37.   labels:
  38.     app: statefulset-apollo-config-server-dev
  39. spec:
  40.   serviceName: service-apollo-meta-server-dev               # 声明它属于哪个Headless Service.
  41.   replicas: 1
  42.   selector:
  43.     matchLabels:
  44.       app: pod-apollo-config-server-dev                     # has to match .spec.template.metadata.labels
  45.   updateStrategy:
  46.     type: RollingUpdate
  47.   template:
  48.     metadata:
  49.       labels:
  50.         app: pod-apollo-config-server-dev                   # has to match .spec.selector.matchLabels
  51.     spec:
  52.       volumes:
  53.         - name: volume-configmap-apollo-config-server-dev
  54.           configMap:
  55.             name: configmap-apollo-config-server-dev
  56.             items:
  57.               - key: application-github.properties
  58.                 path: application-github.properties
  59.       containers:
  60.         - image: tanmgweiwow/apollo-configservice:v1.0   # 这里需要改成自己打的镜像
  61.           securityContext:
  62.             privileged: true
  63.           imagePullPolicy: IfNotPresent
  64.           name: container-apollo-config-server-dev
  65.           ports:
  66.             - protocol: TCP
  67.               containerPort: 8080
  68.           resources:
  69.             limits:
  70.               memory: 512Mi  #限制内存为512M
  71.           volumeMounts:
  72.             - name: volume-configmap-apollo-config-server-dev
  73.               mountPath: /apollo-config-server/config/application-github.properties
  74.               subPath: application-github.properties
  76.           env:
  77.             - name: APOLLO_CONFIG_SERVICE_NAME
  78.               value: "service-apollo-config-server-dev.apollo"
  79.           readinessProbe:
  80.             tcpSocket:
  81.               port: 8080
  82.             initialDelaySeconds: 10
  83.             periodSeconds: 5
  84.           livenessProbe:
  85.             tcpSocket:
  86.               port: 8080
  87.             initialDelaySeconds:  120
  88.             periodSeconds: 10
  89.       dnsPolicy: ClusterFirst
  90.       restartPolicy: Always
  91. EOF
  1. cat > apollo-admin-server.yaml << EOF
  2. kind: ConfigMap
  3. apiVersion: v1
  4. metadata:
  5.   namespace: apollo                                    
  6.   name: configmap-apollo-admin-server-dev
  7. data:        
  8.   application-github.properties: |
  9.     spring.datasource.url = jdbc:mysql://35.201.132.33:3306/apolloconfigdb?characterEncoding=utf8   
  10.     spring.datasource.username = root
  11.     spring.datasource.password = 123456@Abc
  12.     eureka.service.url = http://eureka-0.eureka.default:8761/eureka/,http://eureka-1.eureka.default:8761/eureka/,http://eureka-2.eureka.default:8761/eureka/
  13. ---
  14. kind: Service
  15. apiVersion: v1
  16. metadata:
  17.   namespace: apollo
  18.   name: service-apollo-admin-server-dev
  19.   labels:
  20.     app: service-apollo-admin-server-dev
  21. spec:
  22.   ports:
  23.     - protocol: TCP
  24.       port: 8090                          # 设定Serivce对外提供服务的端口
  25.       targetPort: 8090                    # 设定容器(Pod)的端口,即Pod网络的端口
  26.       nodePort: 28091
  27.   selector:
  28.     app: pod-apollo-admin-server-dev
  29.   type: NodePort
  30.   sessionAffinity: ClientIP
  31. ---
  32. kind: Deployment
  33. apiVersion: apps/v1
  34. metadata:
  35.   namespace: apollo
  36.   name: deployment-apollo-admin-server-dev
  37.   labels:
  38.     app: deployment-apollo-admin-server-dev
  39. spec:
  40.   replicas: 1
  41.   selector:
  42.     matchLabels:
  43.       app: pod-apollo-admin-server-dev                     # has to match .spec.template.metadata.labels
  44.   strategy:
  45.     rollingUpdate:
  46.       maxSurge: 1
  47.       maxUnavailable: 1
  48.     type: RollingUpdate
  49.   template:
  50.     metadata:
  51.       labels:
  52.         app: pod-apollo-admin-server-dev                   # has to match .spec.selector.matchLabels
  53.     spec:
  54.       volumes:
  55.         - name: volume-configmap-apollo-admin-server-dev
  56.           configMap:
  57.             name: configmap-apollo-admin-server-dev
  58.             items:
  59.               - key: application-github.properties
  60.                 path: application-github.properties
  61.       containers:
  62.         - image: tanmgweiwow/apollo-adminservice:v1.0
  63.           securityContext:
  64.             privileged: true
  65.           imagePullPolicy: IfNotPresent
  66.           name: container-apollo-admin-server-dev
  67.           ports:
  68.             - protocol: TCP
  69.               containerPort: 8090
  70.           resources:
  71.             limits:
  72.               memory: 512Mi
  73.           volumeMounts:
  74.             - name: volume-configmap-apollo-admin-server-dev
  75.               mountPath: /apollo-admin-server/config/application-github.properties
  76.               subPath: application-github.properties
  77.           env:
  78.             - name: APOLLO_CONFIG_SERVICE_NAME
  79.               value: "service-apollo-admin-server-dev.apollo" 
  80.           readinessProbe:
  81.             tcpSocket:
  82.               port: 8090
  83.             initialDelaySeconds: 10
  84.             periodSeconds: 5 
  85.           livenessProbe:
  86.             tcpSocket:
  87.               port: 8090
  88.             initialDelaySeconds:  120
  89.             periodSeconds: 10 
  90.       dnsPolicy: ClusterFirst
  91.       restartPolicy: Always
  92. EOF
  1. cat > apollo-portal-server.yaml << EOF
  2. ---
  3. # configmap for apollo-portal-server
  4. kind: ConfigMap
  5. apiVersion: v1
  6. metadata:
  7.   namespace: apollo
  8.   name: configmap-apollo-portal-server
  9. data:
  10.   # 数据库通过ip连接 【改成自己的数据库连接】
  11.   application-github.properties: |
  12.     spring.datasource.url = jdbc:mysql://35.201.132.33:3306/apolloportaldb?characterEncoding=utf8   
  13.     spring.datasource.username = root
  14.     spring.datasource.password = 123456@Abc
  15.   apollo-env.properties: |
  16.     dev.meta=http://statefulset-apollo-config-server-dev-0.service-apollo-meta-server-dev:8080
  17. #   资源限制，现在只使用dev环境
  18. #   fat.meta=http://statefulset-apollo-config-server-fat-0.service-apollo-meta-server-fat:28080,http://statefulset-apollo-config-server-fat-1.service-apollo-meta-server-fat:28080,http://statefulset-apollo-config-server-fat-2.service-apollo-meta-server-fat:28080
  19. #   uat.meta=http://service-apollo-config-server-test-beta.sre:8080
  20. #   pro.meta=http://service-apollo-config-server-prod.sre:8080
  22. ---
  23. kind: Service
  24. apiVersion: v1
  25. metadata:
  26.   namespace: apollo
  27.   name: service-apollo-portal-server
  28.   labels:
  29.     app: service-apollo-portal-server
  30. spec:
  31.   ports:
  32.     - protocol: TCP
  33.       port: 8070
  34.       targetPort: 8070
  35.       nodePort: 30001
  36.   selector:
  37.     app: pod-apollo-portal-server
  38.   type: NodePort
  39.   sessionAffinity: ClientIP
  41. # 这里使用Deployment
  42. ---
  43. kind: Deployment
  44. apiVersion: apps/v1
  45. metadata:
  46.   namespace: apollo
  47.   name: deployment-apollo-portal-server
  48.   labels:
  49.     app: deployment-apollo-portal-server
  50. spec:
  51.   replicas: 1
  52.   selector:
  53.     matchLabels:
  54.       app: pod-apollo-portal-server
  55.   strategy:
  56.     rollingUpdate:
  57.       maxSurge: 1
  58.       maxUnavailable: 1
  59.     type: RollingUpdate
  60.   template:
  61.     metadata:
  62.       labels:
  63.         app: pod-apollo-portal-server
  64.     spec:    
  65.       volumes:
  66.         - name: volume-configmap-apollo-portal-server
  67.           configMap:
  68.             name: configmap-apollo-portal-server
  69.             items:
  70.               - key: application-github.properties
  71.                 path: application-github.properties
  72.               - key: apollo-env.properties
  73.                 path: apollo-env.properties
  74.       containers:
  75.         - image: tanmgweiwow/apollo-portal:v1.0   # 更改为你的 docker registry 下的 image
  76.           securityContext:
  77.             privileged: true
  78.           imagePullPolicy: IfNotPresent
  79.           name: container-apollo-portal-server
  80.           ports:
  81.             - protocol: TCP
  82.               containerPort: 8070
  83.           resources:
  84.             limits:
  85.               memory: 512Mi          
  86.           volumeMounts:
  87.             - name: volume-configmap-apollo-portal-server
  88.               mountPath: /apollo-portal-server/config/application-github.properties
  89.               subPath: application-github.properties
  90.             - name: volume-configmap-apollo-portal-server
  91.               mountPath: /apollo-portal-server/config/apollo-env.properties
  92.               subPath: apollo-env.properties
  94.           env:
  95.             - name: APOLLO_PORTAL_SERVICE_NAME
  96.               value: "service-apollo-portal-server.apollo"
  98.           readinessProbe:
  99.             tcpSocket:
  100.               port: 8070
  101.             initialDelaySeconds: 10
  102.             periodSeconds: 5
  104.           livenessProbe:
  105.             tcpSocket:
  106.               port: 8070
  107.             initialDelaySeconds: 120
  108.             periodSeconds: 15
  109.       dnsPolicy: ClusterFirst
  110.       restartPolicy: Always
  111. EOF
  1. kubectl create ns apollo
  2. kubectl apply -f apollo-config-server.yaml
  3. kubectl apply -f apollo-admin-server.yaml
  4. kubectl apply -f apollo-portal-server.yaml

image.png

访问portal

nodeip:30001
账号密码 apollo/admin
image.png