1. 制作Jenkins镜像
2. 准备NFS共享存储
3. 部署Jenkins
3.6. 安装blue ocean插件
4. 安装Maven到Jenkins
- 4.1. 场景一: maven需求的jdk版本和jenkins一致
- 4.2. 场景二: maven使用 jdk 1.8.131版本时候

1. 制作Jenkins镜像

[root@hdss7-200 ~]# docker pull jenkins/jenkins:2.190.3 
[root@hdss7-200 ~]# docker tag  jenkins/jenkins:2.190.3 harbor.odl.com/public/jenkins:v2.190.3
[root@hdss7-200 ~]# docker push harbor.odl.com/public/jenkins:v2.190.3
# 准备Dockerfile自定义镜像的相关文件
[root@hdss7-200 ~]# mkdir /opt/dockerfile ; cd /opt/dockerfile
# 准备密钥对
[root@hdss7-200 dockerfile]# ssh-keygen -t rsa -b 2048 -C "694188152@qq.com" -N "" -f id_rsa
# Docker的登录信息
[root@hdss7-200 dockerfile]# cp ~/.docker/config.json .
# 准备安装docker-ce的脚本文件
[root@hdss7-200 dockerfile]# wget -O get-docker.sh https://get.docker.com 
# 创建Dockerfile文件
[root@hdss7-200 dockerfile]# vim Dockerfile 
#### 修改默认的 Jenkins 镜像, 请注意 docker有时候会安装失败, 如无法安装,请手动做镜像
FROM harbor.odl.com/public/jenkins:v2.190.3
USER root 
ADD id_rsa /root/.ssh/id_rsa
ADD config.json /root/.docker/config.json
ADD get-docker.sh /get-docker.sh
RUN /bin/cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && echo 'Asia/Shanghai' >/etc/timezone && echo " StrictHostKeyChecking no" >> /etc/ssh/ssh_config && sh /get-docker.sh ; rm -f get-docker.sh
# 在harbor新建一个私有仓库 "infra"(infrastructure 基础设置)
# 构建镜像  国内环境通过脚本安装docker很慢
[root@hdss7-200 dockerfile]# docker build -t harbor.odl.com/infra/jenkins:v2.190.3 .
# 上传镜像至harbor
[root@hdss7-200 dockerfile]# docker push harbor.odl.com/infra/jenkins:v2.190.3

2. 准备NFS共享存储

NFS-Server 部署在hdss7-200上用于存储Jenkins持久化文件所有node节点和hdss7-200 都需要安装 nfs-utils

# 三个节点都需要
[root@hdss7-21 ~]# yum install -y nfs-utils
[root@hdss7-21 ~]# rpm -qa nfs-utils
# hdss7-200
[root@hdss7-200 ~]# systemctl start nfs ; systemctl enable nfs

# 创建共享目录
[root@hdss7-200 ~]# mkdir -p /data/nfs-volume/jenkins_home
# 编写nfs目录配置文件
[root@hdss7-200 ~]# vim /etc/exports
/data/nfs-volume  10.4.7.0/24(rw,sync,no_root_squash)
[root@hdss7-200 ~]# systemctl reload nfs
[root@hdss7-200 ~]# showmount -e 
Export list for hdss7-200.host.com:
/data/nfs-volume 10.4.7.0/24

3. 部署Jenkins

3.1 配置jenkins资源清单

在hdss7-200新建/data/k8s-yaml 新建devops/jenkins目录

[root@hdss7-200 ~]# mkdir -p /data/k8s-yaml/devops/jenkins

3.1.1. deployment.yaml

kind: Deployment
apiVersion: extensions/v1beta1
metadata:
  name: jenkins
  namespace: infra
  labels: 
    name: jenkins
spec:
  replicas: 1
  selector:
    matchLabels: 
      name: jenkins
  template:
    metadata:
      labels: 
        app: jenkins 
        name: jenkins
    spec:
      volumes:
      - name: data
        nfs: 
          server: hdss7-200
          path: /data/nfs-volume/jenkins_home
      - name: docker
        hostPath: 
          path: /run/docker.sock
          type: ''
      containers:
      - name: jenkins
        image: harbor.odl.com/infra/jenkins:v2.190.3
        ports:
        - containerPort: 8080
          protocol: TCP
        env:
        - name: JAVA_OPTS
          value: -Xmx512m -Xms512m
        volumeMounts:
        - name: data
          mountPath: /var/jenkins_home
        - name: docker
          mountPath: /run/docker.sock
      imagePullSecrets:
      - name: harbor
      securityContext: 
        runAsUser: 0
  strategy:
    type: RollingUpdate
    rollingUpdate: 
      maxUnavailable: 1
      maxSurge: 0
  revisionHistoryLimit: 7
  progressDeadlineSeconds: 600

3.1.2. service.yaml

kind: Service
apiVersion: v1
metadata: 
  name: jenkins
  namespace: infra
spec:
  ports:
  - protocol: TCP
    port: 80
    targetPort: 8080
  selector:
    app: jenkins

3.1.3. ingress.yaml

当service资源使用NodePort暴露端口,可忽略此资源

kind: Ingress
apiVersion: extensions/v1beta1
metadata: 
  name: jenkins
  namespace: infra
spec:
  rules:
  - host: jenkins.od.com
    http:
      paths:
      - path: /
        backend: 
          serviceName: jenkins
          servicePort: 80

3.1.4. secret.yaml

# 需要hdss7-200登录过harbor, 登录信息记录在config.json文件中
# 对config.json文件进行base64编码,然后把编码后的内容复制到.dockerconfigjson字段
[root@hdss7-200 ~]# base64 ~/.docker/config.json
ewoJImF1dGhzIjogewoJCSJoYXJib3Iub2RsLmNvbSI6IHsKCQkJImF1dGgiOiAiWVdSdGFXNDZT
R0Z5WW05eU1USXpORFU9IgoJCX0KCX0sCgkiSHR0cEhlYWRlcnMiOiB7CgkJIlVzZXItQWdlbnQi
OiAiRG9ja2VyLUNsaWVudC8xOS4wMy4xMyAobGludXgpIgoJfQp9

[root@hdss7-200 ~]# vim /data/k8s-yaml/devops/jenkins/secret.yaml
apiVersion: v1
kind: Secret
metadata:
  name: harbor
  namespace: infra
type: kubernetes.io/dockerconfigjson
data:
  .dockerconfigjson: ewoJImF1dGhzIjogewoJCSJoYXJib3Iub2RsLmNvbSI6IHsKCQkJImF1dGgiOiAiWVdSdGFXNDZTR0Z5WW05eU1USXpORFU9IgoJCX0KCX0sCgkiSHR0cEhlYWRlcnMiOiB7CgkJIlVzZXItQWdlbnQi
OiAiRG9ja2VyLUNsaWVudC8xOS4wMy4xMyAobGludXgpIgoJfQp9

3.2. Jenkins交付至k8s

# 创建infra命名空间
[root@hdss7-21 ~]# kubectl create namespace infra
namespace/infra created
[root@hdss7-21 ~]# kubectl apply -f http://k8s-yaml.odl.com/devops/jenkins/secret.yaml
[root@hdss7-21 ~]# kubectl apply -f http://k8s-yaml.odl.com/devops/jenkins/deployment.yaml
[root@hdss7-21 ~]# kubectl apply -f http://k8s-yaml.odl.com/devops/jenkins/service.yaml
[root@hdss7-21 ~]# kubectl apply -f http://k8s-yaml.odl.com/devops/jenkins/ingress.yaml

3.3. 配置DNS解析

当service资源使用NodePort暴露端口,可忽略此资源

[root@hdss7-11 ~]# vim /var/named/odl.com.zone
$ORIGIN odl.com.
$TTL 600  ; 10 minutes
@       IN SOA  dns.odl.com. dnsadmin.odl.com. (
        2020091711 ; serial
        10800      ; refresh (3 hours)
        900        ; retry (15 minutes)
        604800     ; expire (1 week)
        86400      ; minimum (1 day)
        )
        NS   dns.odl.com.
$TTL 60 ; 1 minute
dns                A    10.4.7.11
harbor             A    10.4.7.200
k8s-yaml           A    10.4.7.200
traefik            A    10.4.7.10
dashboard          A    10.4.7.10
zk1                A    10.4.7.11
zk2                A    10.4.7.12
zk3                A    10.4.7.21
jenkins            A    10.4.7.10

[root@hdss7-11 ~]# systemctl restart named

3.4. 登录Jenkins

[root@hdss7-21 ~]# kubectl get pod -n infra
NAME                       READY   STATUS    RESTARTS   AGE
jenkins-67f57b4ddf-r8ntt   1/1     Running   0          3m

# 获取Jenkins初始密码,也可以查看jenkins数据目录 secrets/initialAdminPassword
[root@hdss7-21 ~]# kubectl log -f jenkins-67f57b4ddf-r8ntt   -n infra
Jenkins initial setup is required. An admin user has been created and a password generated.
Please use the following password to proceed to installation:
d2b85d80ea5e4633880104b0b69b19f1
[root@hdss7-200 ~]# cat /data/nfs-volume/jenkins_home/secrets/initialAdminPassword 
d2b85d80ea5e4633880104b0b69b19f1

检查pod 运行是否为root 时区是否正确是否连接到本地的docker-server


[root@hdss7-21 ~]# kubectl exec -it   jenkins-67f57b4ddf-r8ntt  /bin/bash -n infra
root@jenkins-67f57b4ddf-r8ntt:/# ps aux|grep jenkins|grep -v grep
root          1  0.0  0.0   1136     0 ?        Ss   19:34   0:00 /sbin/tini -- /usr/local/bin/jenkins.sh
root          6  7.5  9.5 3039012 385992 ?      Sl   19:34   0:32 java -Duser.home=/var/jenkins_home -Xmx512m -Xms512m -Djenkins.model.Jenkins.slaveAgentPort=50000 -jar /usr/share/jenkins/jenkins.war
root@jenkins-67f57b4ddf-r8ntt:/# date
Wed Dec  2 19:41:56 CST 2020

3.5 问题

3.5.1. 登录后页面显示空白界面

解决办法:

登录页面
网址输入 http://jenkins.odl.com/pluginManager/advanced
页面下拉至”升级站点” ,修改为https://mirrors.tuna.tsinghua.edu.cn/jenkins/updates/update-center.json
重启Jenkins 可以 http://jenkins.odl.com/restart 点击确认

3.5.2. jenkins显示英文 , 修改成中文

解决办法:

安装插件localization: chinese (Simplified)
在系统管理配置语言 (并无该选项但重启后显示了中文)
重启jenkins

修改下载地址

[root@hdss7-200 ~]# cd /data/nfs-volume/jenkins_home/updates
[root@hdss7-200 updates]# sed -i 's#http://updates.jenkins-ci.org/download#https://mirrors.tuna.tsinghua.edu.cn/jenkins#g' default.json
[root@hdss7-200 updates]# sed -i  's#http://www.google.com#https://www.baidu.com#g' default.json

3.6. 安装blue ocean插件

等待10+分钟以上依赖的插件较多

4. 安装Maven到Jenkins

Maven是提供给Jenkins使用，需要放到Jenkins的持久化目录中，直接将二进制包形式的Maven拷贝到Jenkins目录最方便
本次安装直接在 hdss7-200 操作。
不同的项目对编译的JDK版本和Maven可能不同，可能需要多个版本的JDK和Maven组合使用，因此Maven目录名称就使用 maven-${maven_versin}-${jdk_version}格式。
Maven的 bin/mvn 文件中可以定义 JAVA_HOME环境变量的值，不同的Maven可以使用不同的 JAVA_HOME 值。

4.1. 场景一: maven需求的jdk版本和jenkins一致

不需要定义 bin/mvn中的JAVA_HOME变量

# 查看Jenkins中jdk版本
[root@hdss7-21 ~]# kubectl exec -it jenkins-67f57b4ddf-f7dxh -n infra -- java -version
openjdk version "1.8.0_232"
OpenJDK Runtime Environment (build 1.8.0_232-b09)
OpenJDK 64-Bit Server VM (build 25.232-b09, mixed mode)

# 获取maven安装包
[root@hdss7-200 ~]# wget https://archive.apache.org/dist/maven/maven-3/3.6.2/binaries/apache-maven-3.6.2-bin.tar.gz
[root@hdss7-200 ~]# tar -xf apache-maven-3.6.2-bin.tar.gz
# 8u232是jenkins的jdk地址
[root@hdss7-200 ~]# mv apache-maven-3.6.2 /data/nfs-volume/jenkins_home/maven-3.6.2-8u232 
# settings.xml 中 <mirrors></mirrors>标签中添加国内源
[root@hdss7-200 ~]# vim /data/nfs-volume/jenkins_home/maven-3.6.2-8u232/conf/settings.xml
<mirror>
  <id>nexus-aliyun</id>
  <mirrorOf>*</mirrorOf>
  <name>Nexus aliyun</name>
  <url>http://maven.aliyun.com/nexus/content/groups/public</url>
</mirror>

4.2. 场景二: maven使用 jdk 1.8.131版本时候

[root@hdss7-200 ~]# mkdir /data/nfs-volume/jenkins_home/jdk_versions
[root@hdss7-200 ~]# tar -xf jdk-8u131-linux-x64.tar.gz -C /data/nfs-volume/jenkins_home/jdk_versions/
[root@hdss7-200 ~]# cp -r /data/nfs-volume/jenkins_home/maven-3.6.2-8u232 /data/nfs-volume/jenkins_home/maven-3.6.2-8u131
# 使用jenkins中绝对路径,并非宿主机路径
[root@hdss7-200 ~]# vim /data/nfs-volume/jenkins_home/maven-3.6.2-8u131/bin/mvn 
JAVA_HOME='/var/jenkins_home/jdk_versions/jdk1.8.0_131'

kubernetes部署Jenkins