Jenkins K3s集群中启动带systemd的Rocky Linux容器失败求助

问题背景 #

在K3s集群中使用Jenkins Pipeline运行带systemd服务的Rocky Linux 8.6容器（因8.7及以上版本镜像移除systemd服务，故改用8.6），但执行时出现错误，即使注释部分步骤仍启动失败。

原Pipeline脚本 #

pipeline {
  parameters {
    choice(name: 'ROCKY_LINUX_IMAGE', choices: ['rockylinux:8.6'], description: 'Rocky Linux image to use.')
  }
  options {
    buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10'))
  }
  agent {
    kubernetes {
      yaml(
        'apiVersion: v1\n' +
        'kind: Pod\n' +
        'spec:\n' +
        '  containers:\n' +
        '  - name: rockylinux\n' +
        '    image: ' + params.ROCKY_LINUX_IMAGE + '\n' +
        '    command:\n' +
        '    - "/usr/lib/systemd/systemd"\n' +
//        '    - "/usr/bin/dbus-daemon --system"\n' +
//        '    - "/usr/sbin/init"\n' +
        '    imagePullPolicy: IfNotPresent\n' +
        '    resources:\n' +
        '      limits:\n' +
        '        memory: "64Gi"\n' +
        '        cpu: "8"\n' +
        '    securityContext:\n' +
        '      privileged: true\n' +
//        '      capabilities:\n' +
//        '        add:\n' +
//        '        - CAP_SYS_ADMIN\n' +
        '  volumes:\n' +
        '  - name: systemd-units\n' +
        '    hostPath:\n' +
        '      path: /etc/systemd/system\n' +
        '  volumeMounts:\n' +
        '  - name: systemd-config\n' +
        '    mountPath: /etc/systemd/system\n'
      )
    }
  }
  stages {
    stage('Starting Container') {
      steps {
        script {
          echo "\n=================================\nNode Name: ${NODE_NAME}\nBuild Number: ${env.BUILD_NUMBER}"
        }
        container('rockylinux') {
          sh "echo Hello World from ${params.ROCKY_LINUX_IMAGE}"
          sh 'whoami'
          script {
            echo 'Installing missing OS packages for CONTAINER'
            //sh 'modprobe fq_codel'
            sh '''
              export DBUS_SYSTEM_BUS_ADDRESS=unix:path=/host/run/dbus/system_bus_socket
              yum -y install sudo passwd systemd firewalld dbus
            '''
            //sh 'systemctl start dbus'
            echo 'Installing openssh'
            sh 'sudo yum -y install openssh-server openssh-clients'
          }
        }

      }
    }
  }
  
}

错误信息 #

Also:   org.jenkinsci.plugins.workflow.actions.ErrorAction$ErrorId: adeaafd3-fcc3-468b-886e-15dbd1646bf7
groovy.lang.MissingPropertyException: No such property: sh for class: groovy.lang.Binding
at groovy.lang.Binding.getVariable(Binding.java:63)
at org.jenkinsci.plugins.scriptsecurity.sandbox.groovy.SandboxInterceptor.onGetProperty(SandboxInterceptor.java:285)
at org.kohsuke.groovy.sandbox.impl.Checker$7.call(Checker.java:375)
at org.kohsuke.groovy.sandbox.impl.Checker.checkedGetProperty(Checker.java:379)
at org.kohsuke.groovy.sandbox.impl.Checker.checkedGetProperty(Checker.java:355)
at org.kohsuke.groovy.sandbox.impl.Checker.checkedGetProperty(Checker.java:355)
at org.kohsuke.groovy.sandbox.impl.Checker.checkedGetProperty(Checker.java:355)
at org.kohsuke.groovy.sandbox.impl.Checker.checkedGetProperty(Checker.java:355)
at com.cloudbees.groovy.cps.sandbox.SandboxInvoker.getProperty(SandboxInvoker.java:29)
at org.jenkinsci.plugins.workflow.cps.LoggingInvoker.getProperty(LoggingInvoker.java:121)
at com.cloudbees.groovy.cps.impl.PropertyAccessBlock.rawGet(PropertyAccessBlock.java:20)
at WorkflowScript.run(WorkflowScript:241)
at **cps.transform**(Native Method)
at com.cloudbees.groovy.cps.impl.PropertyishBlock$ContinuationImpl.get(PropertyishBlock.java:73)
at com.cloudbees.groovy.cps.LValueBlock$GetAdapter.receive(LValueBlock.java:30)
at com.cloudbees.groovy.cps.impl.PropertyishBlock$ContinuationImpl.fixName(PropertyishBlock.java:65)
at jdk.internal.reflect.GeneratedMethodAccessor504.invoke(Unknown Source)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:568)
at com.cloudbees.groovy.cps.impl.ContinuationPtr$ContinuationImpl.receive(ContinuationPtr.java:72)
at com.cloudbees.groovy.cps.impl.ConstantBlock.eval(ConstantBlock.java:21)
at com.cloudbees.groovy.cps.Next.step(Next.java:83)
at com.cloudbees.groovy.cps.Continuable$1.call(Continuable.java:152)
at com.cloudbees.groovy.cps.Continuable$1.call(Continuable.java:146)
at org.codehaus.groovy.runtime.GroovyCategorySupport$ThreadCategoryInfo.use(GroovyCategorySupport.java:136)
at org.codehaus.groovy.runtime.GroovyCategorySupport.use(GroovyCategorySupport.java:275)
at com.cloudbees.groovy.cps.Continuable.run0(Continuable.java:146)
at org.jenkinsci.plugins.workflow.cps.SandboxContinuable.access$001(SandboxContinuable.java:18)
at org.jenkinsci.plugins.workflow.cps.SandboxContinuable.run0(SandboxContinuable.java:51)
at org.jenkinsci.plugins.workflow.cps.CpsThread.runNextChunk(CpsThread.java:187)
at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup.run(CpsThreadGroup.java:423)
at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup$2.call(CpsThreadGroup.java:331)
at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup$2.call(CpsThreadGroup.java:295)
at org.jenkinsci.plugins.workflow.cps.CpsVmExecutorService$2.call(CpsVmExecutorService.java:97)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at hudson.remoting.SingleLaneExecutorService$1.run(SingleLaneExecutorService.java:139)
at jenkins.util.ContextResettingExecutorService$1.run(ContextResettingExecutorService.java:28)
at jenkins.security.ImpersonatingExecutorService$1.run(ImpersonatingExecutorService.java:68)
at jenkins.util.ErrorLoggingExecutorService.lambda$wrap$0(ErrorLoggingExecutorService.java:51)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:833)
Finished: FAILURE

环境信息 #

kubectl version
Client Version: v1.29.0+k3s1
Kustomize Version: v5.0.4-0.20230601165947-6ce0bf390ce3
Server Version: v1.29.0+k3s1

问题分析与解决方案 #

1. 核心错误原因 #

• Pod YAML语法错误：volumeMounts配置放在了spec根层级，正确位置应该是容器配置块内部；同时卷名称定义为systemd-units，但挂载时用了systemd-config，名称不匹配，导致Pod创建失败，Jenkins无法进入容器上下文执行sh命令，触发No such property: sh错误。
• Systemd运行依赖缺失：未挂载/sys/fs/cgroup卷，且未开启tty和stdin，导致systemd无法正常初始化。

2. 修改后的Pipeline脚本 #

pipeline {
  parameters {
    choice(name: 'ROCKY_LINUX_IMAGE', choices: ['rockylinux:8.6'], description: 'Rocky Linux image to use.')
  }
  options {
    buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10'))
  }
  agent {
    kubernetes {
      yaml(
        'apiVersion: v1\n' +
        'kind: Pod\n' +
        'spec:\n' +
        '  containers:\n' +
        '  - name: rockylinux\n' +
        '    image: ' + params.ROCKY_LINUX_IMAGE + '\n' +
        '    command: ["/usr/lib/systemd/systemd"]\n' +
        '    args: ["--system"]\n' +
        '    imagePullPolicy: IfNotPresent\n' +
        '    resources:\n' +
        '      limits:\n' +
        '        memory: "64Gi"\n' +
        '        cpu: "8"\n' +
        '    securityContext:\n' +
        '      privileged: true\n' +
        '    tty: true\n' +
        '    stdin: true\n' +
        '    volumeMounts:\n' +
        '    - name: systemd-units\n' +
        '      mountPath: /etc/systemd/system\n' +
        '    - name: cgroup\n' +
        '      mountPath: /sys/fs/cgroup\n' +
        '  volumes:\n' +
        '  - name: systemd-units\n' +
        '    hostPath:\n' +
        '      path: /etc/systemd/system\n' +
        '  - name: cgroup\n' +
        '    hostPath:\n' +
        '      path: /sys/fs/cgroup\n' +
        '      type: Directory\n'
      )
    }
  }
  stages {
    stage('Starting Container') {
      steps {
        script {
          echo "\n=================================\nNode Name: ${NODE_NAME}\nBuild Number: ${env.BUILD_NUMBER}"
        }
        container('rockylinux') {
          sh "echo Hello World from ${params.ROCKY_LINUX_IMAGE}"
          sh 'whoami'
          script {
            echo 'Installing missing OS packages for CONTAINER'
            sh '''
              yum -y install sudo passwd systemd firewalld dbus openssh-server openssh-clients
              systemctl enable --now dbus
            '''
          }
        }
      }
    }
  }
}

3. 关键修改点说明 #

• 把volumeMounts移到容器配置块内部，修正卷名称匹配问题
• 添加/sys/fs/cgroup卷挂载，满足systemd对cgroup的依赖
• 开启tty: true和stdin: true，保证systemd正常运行
• 合并yum安装命令，简化步骤；添加systemctl enable --now dbus确保服务启动
• 给systemd添加--system参数，明确运行模式

内容的提问来源于stack exchange，提问作者Drasius