一、自建k8s集群
可参考本文:k8s双master双Node部署
或现有k8s集群也可
二、自建NFS
使k8s集群运行的deployment、pod可以持久化存储,例如jenkins连接信息、工作空间,重建pod也能恢复到上一个状态。
三、基于k8s搭建jenkins-master & agent
3.1 部署jenkins-master
3.1.1 创建svc
kind: Service
apiVersion: v1
metadata:
labels:
k8s-app: jenkins-master
name: jenkins-master
namespace: devops
spec:
type: NodePort
ports:
- name: web
port: 8080
targetPort: 8080
nodePort: 30080
- name: slave
port: 50000
targetPort: 50000
nodePort: 30081
selector:
k8s-app: jenkins-master
3.1.2 创建Role
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: jenkins-master
namespace: devops
rules:
- apiGroups: [""]
resources: ["pods","configmaps","namespaces"]
verbs: ["create","delete","get","list","patch","update","watch"]
- apiGroups: [""]
resources: ["pods/exec"]
verbs: ["create","delete","get","list","patch","update","watch"]
- apiGroups: [""]
resources: ["pods/log"]
verbs: ["get","list","watch"]
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get"]
3.1.3 创建RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: jenkins-master
namespace: devops
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: jenkins-master
subjects:
- kind: ServiceAccount
name: jenkins-master
namespace: devops
3.1.4 创建pv
apiVersion: v1
kind: PersistentVolume
metadata:
name: jenkins-master-pv
labels:
app: jenkins-master-storage
spec:
capacity:
storage: 10Gi
accessModes:
- ReadWriteMany
nfs:
server: 172.16.10.130
path: /data/nfs-data/jenkins-master
persistentVolumeReclaimPolicy: Retain
3.1.5 创建pvc
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: jenkins-master-pvc
namespace: devops
spec:
accessModes:
- ReadWriteMany
resources:
requests:
storage: 10Gi
selector:
matchLabels:
app: jenkins-master-storage
3.1.6 创建deployment
kind: Deployment
apiVersion: apps/v1
metadata:
labels:
k8s-app: jenkins-master
name: jenkins-master
namespace: devops
spec:
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
k8s-app: jenkins-master
template:
metadata:
labels:
k8s-app: jenkins-master
namespace: devops
name: jenkins-master
spec:
containers:
- name: jenkins-master
image: myharbor.com/jenkins/jenkins:latest
imagePullPolicy: IfNotPresent
ports:
- containerPort: 8080
name: web
protocol: TCP
- containerPort: 50000
name: agent
protocol: TCP
resources:
limits:
cpu: 1000m
memory: 2Gi
requests:
cpu: 500m
memory: 512Mi
livenessProbe:
httpGet:
path: /login
port: 8080
initialDelaySeconds: 60
timeoutSeconds: 5
failureThreshold: 12
readinessProbe:
httpGet:
path: /login
port: 8080
initialDelaySeconds: 60
timeoutSeconds: 5
failureThreshold: 12
volumeMounts:
- name: jenkins-home
mountPath: /var/lib/jenkins
- name: localtime
mountPath: /etc/localtime
env:
- name: JENKINS_HOME
value: /var/lib/jenkins
- name: JENKINS_OPTS
value: >
--httpPort=8080
-Dhudson.remoting.LiteRemoteChannel.disableXInstanceIdentity=true
- name: JENKINS_SLAVE_AGENT_PORT
value: "50000"
volumes:
- name: jenkins-home
persistentVolumeClaim:
claimName: jenkins-master-pvc
- name: localtime
hostPath:
path: /etc/localtime
serviceAccountName: jenkins-master
最后都应用完成后,查看jenkins的运行日志,包含首次登录jenkins的密码
四、创建jenkins动态slave(推荐)
4.1 创建cloud云平台
可以参考此博文配置cloud云平台
配置为实际参数即可,以下均已博文参数一致
PS:不配置cloud,jenkins无连接k8s集群的方式,会报如下错误
4.2 通过pipeline创建动态slave
4.2.1 测试多层容器执行命令
新建一个流水线demo
仅测试,内容只需要pipeline script,其他暂不用配置。Jenkinsfile定义了三个容器:jenkins/inbound-agent本身+docker+kubectl,所以运行pod模板,会包含三个容器
def label = "java-agent"
podTemplate(label: label, containers: [
containerTemplate(name: 'docker', image: 'docker:latest', command: 'cat', ttyEnabled: true),
containerTemplate(name: 'kubectl', image: 'cnych/kubectl', command: 'cat', ttyEnabled: true)
], serviceAccount: 'jenkins-master', volumes: [
hostPathVolume(mountPath: '/var/run/docker.sock', hostPath: '/var/run/docker.sock')
]) {
node(label) {
stage('docker测试') {
container('docker') {
echo "1. 检测docker命令并获取本地资源"
sh "docker ps"
}
}
stage(' Kubectl测试') {
withCredentials([file(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) {
container('kubectl') {
echo "1.kubectl查看jenkins应用"
sh 'kubectl --kubeconfig=$KUBECONFIG get pod -n devops'
}
}
}
stage('测试完成') {
sh "echo 测试流程正常"
}
}
}
立即构建,随后会生成控制台日志,指引:运行流水线
查看devops命名空间发现,动态slave已经生成,各容器执行每个阶段的命令
查看控制台日志,docker ps + kubectl --kubeconfig=${scret file} get pod -n devops。均成功运行
当流水线执行完之后,动态slave会自动删除
4.3 模拟测试环境自动化部署(devops)
4.3.1 部署需求
开始 >>拉取java代码 >> mvn编译可运行jar >> docker构建jar镜像 >> 推送镜像至harbor仓库 >> kubectl一键部署应用 >> 结束
4.3.2 Jenkinsfile内容
官方推荐使用Pipeline script from SCM,需要把Jenkinsfile整合到gitlab对应的代码仓库里面
配置参考:创建SCM
Jenkinsfile如下
def label = "jenkins-agent"
podTemplate(label: label, containers: [
containerTemplate(name: 'maven', image: 'maven:3.6.0-jdk-8-alpine', command: 'cat', ttyEnabled: true),
containerTemplate(name: 'docker', image: 'docker:latest', command: 'cat', ttyEnabled: true),
containerTemplate(name: 'kubectl', image: 'cnych/kubectl', command: 'cat', ttyEnabled: true)],
serviceAccount: 'jenkins-master',
volumes: [
hostPathVolume(mountPath: '/var/run/docker.sock', hostPath: '/var/run/docker.sock'),
hostPathVolume(mountPath: '/root/.m2', hostPath: '/data/nfs-data/mvn')])
//hostPathVolume(mountPath: '/home/jenkins/agent/workspace', hostPath: '/data/nfs-data/jenkins_share')])
{
node(label) {
def myRepo = checkout scm
// 获取 git commit id 作为镜像标签
def imageTag = sh(script: "git rev-parse --short HEAD", returnStdout: true).trim()
// 仓库地址
def registryUrl = "myharbor.com"
def imageEndpoint = "devops/devopsimage-demo"
// 镜像
def image = "${registryUrl}/${imageEndpoint}:${imageTag}"
stage('mvn构建jar包') {
container('maven') {
echo "1.mvn构建"
sh "mvn -B -q clean install -DskipTests"
}
}
stage('构建镜像推送harbor仓库') {
withCredentials([usernamePassword(
credentialsId: 'harbor_auth',
passwordVariable: 'HARBOR_PASSWORD',
usernameVariable: 'HARBOR_USER')]) {
container('docker') {
echo "1. 检测docker命令并获取本地资源"
sh """
docker login ${registryUrl} -u ${HARBOR_USER} -p ${HARBOR_PASSWORD}
docker --version
docker build -t ${image} .
docker push ${image}
"""
}
}
}
stage('kubectl一键部署') {
withCredentials([file(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) {
container('kubectl') {
echo "1.kubectl查看jenkins应用"
sh """
kubectl --kubeconfig=$KUBECONFIG set image deployment/micro micro=${image} -n devops"
sleep 10"
kubectl --kubeconfig=$KUBECONFIG get pod -n devops |grep micro|grep Running
"""
}
}
}
stage('kubectl一键部署') {
sh "echo 测试流程正常"
}
}
}
4.3.3 Dockerfile内容
FROM openjdk:8-jre-alpine
WORKDIR /app
COPY springboot/target/*.jar /app/app.jar
# 启动命令
ENTRYPOINT ["java", "-jar", "/app/app.jar"]
4.3.3 执行流水线demo
执行后查看所有阶段日志
Started by user jamking
Obtained Jenkinsfile from git http://172.16.10.142/newtestgroup/newtest.git
[Pipeline] Start of Pipeline
[Pipeline] podTemplate
[Pipeline] {
[Pipeline] node
Created Pod: devops-kubernetes devops/java-agent-0gxhm-3kkr0
[PodInfo] devops/java-agent-0gxhm-3kkr0
Container [docker] waiting [ContainerCreating] No message
Container [jnlp] waiting [ContainerCreating] No message
Container [kubectl] waiting [ContainerCreating] No message
Container [maven] waiting [ContainerCreating] No message
Pod [Pending][ContainersNotReady] containers with unready status: [maven kubectl docker jnlp]
[PodInfo] devops/java-agent-0gxhm-3kkr0
Container [docker] waiting [ContainerCreating] No message
Container [jnlp] waiting [ContainerCreating] No message
Container [kubectl] waiting [ContainerCreating] No message
Container [maven] waiting [ContainerCreating] No message
Pod [Pending][ContainersNotReady] containers with unready status: [maven kubectl docker jnlp]
Agent java-agent-0gxhm-3kkr0 is provisioned from template java-agent-0gxhm
---
apiVersion: "v1"
kind: "Pod"
metadata:
annotations:
kubernetes.jenkins.io/last-refresh: "1755676521367"
buildUrl: "http://jenkins-master.devops.svc.cluster.local:8080/job/test-demo/29/"
runUrl: "job/test-demo/29/"
labels:
jenkins: "slave"
jenkins/label-digest: "b0bf52919934ffbb3665469466121a62f7004086"
jenkins/label: "java-agent"
kubernetes.jenkins.io/controller: "http___jenkins-master_devops_svc_cluster_local_8080x"
name: "java-agent-0gxhm-3kkr0"
namespace: "devops"
spec:
containers:
- command:
- "cat"
image: "maven:3.6.0-jdk-8-alpine"
imagePullPolicy: "IfNotPresent"
name: "maven"
resources: {}
tty: true
volumeMounts:
- mountPath: "/root/.m2"
name: "volume-1"
readOnly: false
- mountPath: "/var/run/docker.sock"
name: "volume-0"
readOnly: false
- mountPath: "/home/jenkins/agent"
name: "workspace-volume"
readOnly: false
- command:
- "cat"
image: "cnych/kubectl"
imagePullPolicy: "IfNotPresent"
name: "kubectl"
resources: {}
tty: true
volumeMounts:
- mountPath: "/root/.m2"
name: "volume-1"
readOnly: false
- mountPath: "/var/run/docker.sock"
name: "volume-0"
readOnly: false
- mountPath: "/home/jenkins/agent"
name: "workspace-volume"
readOnly: false
- command:
- "cat"
image: "docker:latest"
imagePullPolicy: "IfNotPresent"
name: "docker"
resources: {}
tty: true
volumeMounts:
- mountPath: "/root/.m2"
name: "volume-1"
readOnly: false
- mountPath: "/var/run/docker.sock"
name: "volume-0"
readOnly: false
- mountPath: "/home/jenkins/agent"
name: "workspace-volume"
readOnly: false
- env:
- name: "JENKINS_SECRET"
value: "********"
- name: "REMOTING_OPTS"
value: "-noReconnectAfter 1d"
- name: "JENKINS_AGENT_NAME"
value: "java-agent-0gxhm-3kkr0"
- name: "JENKINS_NAME"
value: "java-agent-0gxhm-3kkr0"
- name: "JENKINS_AGENT_WORKDIR"
value: "/home/jenkins/agent"
- name: "JENKINS_URL"
value: "http://jenkins-master.devops.svc.cluster.local:8080/"
image: "jenkins/inbound-agent:3309.v27b_9314fd1a_4-1"
name: "jnlp"
resources:
requests:
memory: "256Mi"
cpu: "100m"
volumeMounts:
- mountPath: "/root/.m2"
name: "volume-1"
readOnly: false
- mountPath: "/var/run/docker.sock"
name: "volume-0"
readOnly: false
- mountPath: "/home/jenkins/agent"
name: "workspace-volume"
readOnly: false
nodeSelector:
kubernetes.io/os: "linux"
restartPolicy: "Never"
serviceAccountName: "jenkins-master"
volumes:
- hostPath:
path: "/var/run/docker.sock"
name: "volume-0"
- hostPath:
path: "/data/nfs-data/mvn"
name: "volume-1"
- emptyDir:
medium: ""
name: "workspace-volume"
Running on java-agent-0gxhm-3kkr0 in /home/jenkins/agent/workspace/test-demo
[Pipeline] {
[Pipeline] checkout
The recommended git tool is: NONE
using credential gitlab_user
Cloning the remote Git repository
Cloning repository http://172.16.10.142/newtestgroup/newtest.git
> git init /home/jenkins/agent/workspace/test-demo # timeout=10
Fetching upstream changes from http://172.16.10.142/newtestgroup/newtest.git
> git --version # timeout=10
> git --version # 'git version 2.39.5'
using GIT_ASKPASS to set credentials 流水线代码访问账密
> git fetch --tags --force --progress -- http://172.16.10.142/newtestgroup/newtest.git +refs/heads/*:refs/remotes/origin/* # timeout=10
Avoid second fetch
Checking out Revision e44c1d1f38f7dceb15dd2e36d738a94a9174879b (refs/remotes/origin/branch_dev)
> git config remote.origin.url http://172.16.10.142/newtestgroup/newtest.git # timeout=10
> git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10
> git rev-parse refs/remotes/origin/branch_dev^{commit} # timeout=10
> git config core.sparsecheckout # timeout=10
> git checkout -f e44c1d1f38f7dceb15dd2e36d738a94a9174879b # timeout=10
Commit message: "Update Jenkinsfile"
> git rev-list --no-walk 76c444df57f09d5f978dab8da05054a60ba4c1e8 # timeout=10
[Pipeline] sh
+ git rev-parse --short HEAD
[Pipeline] stage
[Pipeline] { (mvn构建jar包)
[Pipeline] container
[Pipeline] {
[Pipeline] echo
1.mvn构建
[Pipeline] sh
+ mvn -B -q clean install -DskipTests
[Pipeline] }
[Pipeline] // container
[Pipeline] }
[Pipeline] // stage
[Pipeline] stage
[Pipeline] { (构建镜像推送harbor仓库)
[Pipeline] withCredentials
Masking supported pattern matches of $HARBOR_PASSWORD
[Pipeline] {
[Pipeline] container
[Pipeline] {
[Pipeline] echo
1. 检测docker命令并获取本地资源
[Pipeline] sh
Warning: A secret was passed to "sh" using Groovy String interpolation, which is insecure.
Affected argument(s) used the following variable(s): [HARBOR_PASSWORD]
See https://jenkins.io/redirect/groovy-string-interpolation for details.
+ docker login myharbor.com -u admin -p ****
WARNING! Using --password via the CLI is insecure. Use --password-stdin.
WARNING! Your credentials are stored unencrypted in '/root/.docker/config.json'.
Configure a credential helper to remove this warning. See
https://docs.docker.com/go/credential-store/
Login Succeeded
+ docker --version
Docker version 28.3.3, build 980b856
+ docker build -t myharbor.com/devops/devopsimage-demo:e44c1d1f .
#0 building with "default" instance using docker driver
#1 [internal] load build definition from Dockerfile
#1 transferring dockerfile: 257B 0.0s done
#1 DONE 0.0s
#2 [internal] load metadata for docker.io/library/openjdk:8-jre-alpine
#2 DONE 0.0s
#3 [internal] load .dockerignore
#3 transferring context: 2B done
#3 DONE 0.0s
#4 [1/3] FROM docker.io/library/openjdk:8-jre-alpine
#4 DONE 0.0s
#5 [internal] load build context
#5 transferring context: 166.11MB 0.9s done
#5 DONE 0.9s
#6 [2/3] WORKDIR /app
#6 CACHED
#7 [3/3] COPY springboot/target/MicroCommunityBoot.jar /app/app.jar
#7 DONE 0.4s
#8 exporting to image
#8 exporting layers
#8 exporting layers 0.6s done
#8 writing image sha256:d43564e5cd6da8ceceae9c0b8c5b97844f7071389db88ac4cec6b48cd26cef0c done
#8 naming to myharbor.com/devops/devopsimage-demo:e44c1d1f done
#8 DONE 0.6s
WARNING: current commit information was not captured by the build: failed to read current commit information with git rev-parse --is-inside-work-tree
+ docker push myharbor.com/devops/devopsimage-demo:e44c1d1f
The push refers to repository [myharbor.com/devops/devopsimage-demo]
b5f20a7d3bcd: Preparing
bbbec76424e1: Preparing
edd61588d126: Preparing
9b9b7f3d56a0: Preparing
f1b5933fe4b5: Preparing
9b9b7f3d56a0: Layer already exists
edd61588d126: Layer already exists
bbbec76424e1: Layer already exists
f1b5933fe4b5: Layer already exists
b5f20a7d3bcd: Pushed
e44c1d1f: digest: sha256:cbcbef5f59d33a0bd16ff55e21613ea5b8beba80ed7939414c0c69e0dcd72d1d size: 1367
[Pipeline] }
[Pipeline] // container
[Pipeline] }
[Pipeline] // withCredentials
[Pipeline] }
[Pipeline] // stage
[Pipeline] stage
[Pipeline] { (kubectl一键部署)
[Pipeline] withCredentials
Masking supported pattern matches of $KUBECONFIG
[Pipeline] {
[Pipeline] container
[Pipeline] {
[Pipeline] echo
1.kubectl查看jenkins应用
[Pipeline] sh
Warning: A secret was passed to "sh" using Groovy String interpolation, which is insecure.
Affected argument(s) used the following variable(s): [KUBECONFIG]
See https://jenkins.io/redirect/groovy-string-interpolation for details.
+ kubectl '--kubeconfig=****' set image deployment/micro 'micro=myharbor.com/devops/devopsimage-demo:e44c1d1f' -n devops
deployment.apps/micro image updated
[Pipeline] sh
+ sleep 10
[Pipeline] sh
+ kubectl '--kubeconfig=****' get pod -n devops
+ grep micro
+ grep Running
micro-76cdf45444-xrknh 1/1 Running 0 11s
[Pipeline] }
[Pipeline] // container
[Pipeline] }
[Pipeline] // withCredentials
[Pipeline] }
[Pipeline] // stage
[Pipeline] stage
[Pipeline] { (kubectl一键部署)
[Pipeline] sh
+ echo 测试流程正常
测试流程正常
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
[Pipeline] // node
[Pipeline] }
[Pipeline] // podTemplate
[Pipeline] End of Pipeline
Finished: SUCCESS
最后打印【
Finished: SUCCESS
】才代表流水线运行正常,查看部署的服务Running和日志
五、创建jenkins静态slave(可选)
建议采取动态slave的方法,动态slave完成job自动释放资源,保证工作空间代码不外泄,相对安全可靠。但根据实际情况选择,多一种学习或使用方式。
5.1 创建流程
流程如下
新建agent名称,这里我已新建,所以会爆红
填好这些关键信息,名字、进程job数、工作目录、标签。最后【保存】
用法如下,因为只有一个节点,和部署agent的方式是通过web连接的,因此启动方式选择java web
5.1.1 获取连接master的secret(重要)
从新建的agent端点击
复制此串secret并保存,在3.3.3的yaml的env.JENKINS_SECRET.value并填上
5.2 部署jenkins-agent
5.2.1 创建pv
apiVersion: v1
kind: PersistentVolume
metadata:
name: jenkins-agent-pv
labels:
app: jenkins-agent-storage
spec:
capacity:
storage: 10Gi
accessModes:
- ReadWriteMany
nfs:
server: 172.16.10.130
path: /data/nfs-data/jenkins-agent
persistentVolumeReclaimPolicy: Retain
5.2.2 创建pvc
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: jenkins-agent-pvc
namespace: devops
spec:
accessModes:
- ReadWriteMany
resources:
requests:
storage: 10Gi
selector:
matchLabels:
app: jenkins-agent-storage
5.2.3 创建deployment
apiVersion: v1
kind: ConfigMap
metadata:
name: scripts
namespace: devops
data:
start.sh: |
#!/bin/bash
set -e
echo "=== 启动master连接 ==="
curl -sO ${JENKINS_URL}/jnlpJars/agent.jar
java -jar agent.jar -url ${JENKINS_URL}/ -secret ${JENKINS_SECRET} -name "${JENKINS_AGENT_NAME}" -webSocket -workDir "${JENKINS_AGENT_WORKDIR}"
echo "=== 连接master成功 ==="
---
kind: Deployment
apiVersion: apps/v1
metadata:
labels:
k8s-app: jenkinsi-agent
name: jenkins-agent
namespace: devops
spec:
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
k8s-app: jenkins-agent
template:
metadata:
labels:
k8s-app: jenkins-agent
namespace: devops
name: jenkins-agent
spec:
containers:
- name: jenkins-agent
image: myharbor.com/jenkins/inbound-agent:latest
securityContext:
privileged: true
imagePullPolicy: IfNotPresent
resources:
limits:
cpu: 500m
memory: 1Gi
requests:
cpu: 250m
memory: 512Mi
volumeMounts:
- name: agent-home
mountPath: /home/jenkins
- name: dockersock
mountPath: "/var/run/docker.sock"
- name: dockercmd
mountPath: /usr/bin/docker
#- name: kubectlconfig
# mountPath: /home/jenkins/.kube/config
- name: kubectlcmd
mountPath: /usr/bin/kubectl
#- name: jenkins-agent-workdir
# mountPath: /home/jenkins/workspace
- name: script-mount
mountPath: /home/jenkins/scripts
# - name: localtime
# mountPath: /etc/localtime
env:
- name: JENKINS_URL
value: http://jenkins-master.devops.svc.cluster.local:8080
- name: JENKINS_SECRET
value: 0049c76bc98ad0403be866f79115f05b185e6a05c28075d830fea8da4173d193
- name: JENKINS_AGENT_NAME
value: k8s-agent
- name: JENKINS_AGENT_WORKDIR
value: /home/jenkins/workspace
- name: JENKINS_TUNNEL
value: http://jenkins-master.devops.svc.cluster.local:50000
command: ["/bin/bash", "/home/jenkins/scripts/start.sh"]
#command: ["/bin/bash", "-c"]
#args:
# - |
# echo ${JENKINS_URL}
# curl -sO ${JENKINS_URL}/jnlpJars/agent.jar && java -jar agent.jar -url ${JENKINS_URL}/ -secret ${JENKINS_SECRET} -name "${JENKINS_AGENT_NAME}" -webSocket -workDir "${JENKINS_AGENT_WORKDIR}"
volumes:
# - name: buildtools
# hostPath:
# path: /data/Jenkins/buildtools
# type: Directory
# - name: kubectlconfig
# hostPath:
# path: /data/.kube/config
- name: kubectlcmd
hostPath:
path: /usr/bin/kubectl
- name: dockersock
hostPath:
path: /var/run/docker.sock
- name: dockercmd
hostPath:
path: /usr/bin/docker
#- name: jenkins-agent-workdir
# hostPath:
# path: /data/Jenkins/workspace
- name: script-mount
configMap:
name: scripts
defaultMode: 0755
- name: agent-home
persistentVolumeClaim:
claimName: jenkins-agent-pvc
#- name: localtime
# hostPath:
# path: /etc/localtime
当如上yaml均执行后,容器会自动执行configmap内的连接master的shell脚本
当连接Connected,节点列表的k8s-agent的PC图标会正常,未正常连接的话,PC内会红×
5.3 流水线实践
步骤和本文4.3章节类似,不再赘述
评论区