使用mock client-go时Replica状态值未同步的原因排查

背景信息 #

目标是编写Go语言单元测试，验证指定命名空间内的Deployment是否就绪，使用k8s.io/client-go/kubernetes/fake库实现。

计划执行以下步骤：

• 使用client-go创建mock Kubernetes API：

kubectl := &KubeClient{
        testclient.NewSimpleClientset(),
}

• 创建初始副本数为1的mock Deployment：

deploymentSpec := &appsv1.Deployment{
    ObjectMeta: metav1.ObjectMeta{
        Name:      RESOURCE_NAME,
        Namespace: NAMESPACE,
        Labels:    resourceLabel,
    },
    Spec: appsv1.DeploymentSpec{
        Replicas: &replicas,
        Template: v1.PodTemplateSpec{
            Spec: v1.PodSpec{
                Containers: []v1.Container{
                    {
                    Name:  RESOURCE_NAME,
                    Image: image,
                    },
                },
            },
        },
    },
}

• 使用client-go的Update API将副本数更新为3：

var updateReplicas int32 = 3
replicaCountPtr := &updateReplicas
result.Spec.Replicas = replicaCountPtr

_, err = kubectl.AppsV1().Deployments(NAMESPACE).Update(context.TODO(), result, metav1.UpdateOptions{})

• 运行goroutine检查Deployment是否就绪。

问题 #

当通过kubectl.AppsV1().Deployments(NAMESPACE).Get(...).Status.Replicas等方式查询Replicas、ReadyReplicas、AvailableReplicas、UpdatedReplicas时，这些值始终返回0而非3。

为何副本状态值未更新？使用k8s.io/client-go/kubernetes/fake是否是问题根源？

测试代码 #

package main

import (
    "context"
    "log"
    "time"

    appsv1 "k8s.io/api/apps/v1"
    v1 "k8s.io/api/core/v1"
    metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    "k8s.io/client-go/kubernetes"
    testclient "k8s.io/client-go/kubernetes/fake"
)

type KubeClient struct {
    kubernetes.Interface
}

const NAMESPACE = "default"
const RESOURCE_NAME = "TestResource"

func main() {
    var replicas int32 = 1
    image := NAMESPACE
    resourceLabel := map[string]string{
        "env": "test",
    }

    kubectl := &KubeClient{
        testclient.NewSimpleClientset(),
    }

    // 创建Deployment
    deploymentSpec := &appsv1.Deployment{
        ObjectMeta: metav1.ObjectMeta{
            Name:      RESOURCE_NAME,
            Namespace: NAMESPACE,
            Labels:    resourceLabel,
        },
        Spec: appsv1.DeploymentSpec{
            Replicas: &replicas,
            Template: v1.PodTemplateSpec{
                Spec: v1.PodSpec{
                    Containers: []v1.Container{
                        {
                            Name:  RESOURCE_NAME,
                            Image: image,
                        },
                    },
                },
            },
        },
    }

    deployResult, err := kubectl.AppsV1().Deployments(NAMESPACE).Create(context.TODO(), deploymentSpec, metav1.CreateOptions{})
    if err != nil {
        panic(err)
    }
    log.Printf("Created deployment %q in namespace '%s'\n", deployResult.GetObjectMeta().GetName(), deployResult.GetNamespace())

    result, err := kubectl.AppsV1().Deployments(NAMESPACE).Get(context.TODO(), RESOURCE_NAME, metav1.GetOptions{})
    if err != nil {
        log.Printf("Failed to fetch deployment %s in the namespace %s with error: %s\n", RESOURCE_NAME, NAMESPACE, err.Error())
    }

    var updateReplicas int32 = 3
    replicaCountPtr := &updateReplicas
    result.Spec.Replicas = replicaCountPtr

    _, err = kubectl.AppsV1().Deployments(NAMESPACE).Update(context.TODO(), result, metav1.UpdateOptions{})
    if err != nil {
        log.Printf("Failed to update replica value: %v\n", err)
    }
    time.Sleep(1 * time.Second)
    log.Printf("Updated replica count of deployment %q in namespace %q : (%d)\n", RESOURCE_NAME, NAMESPACE, &replicas)

    var status bool
    channel := make(chan bool, 1)
    go func(bool) {
        for {
            getResult, getErr := kubectl.AppsV1().Deployments(NAMESPACE).Get(context.TODO(), RESOURCE_NAME, metav1.GetOptions{})
            if getErr != nil {
                log.Printf("Failed to fetch deployment %s in the namespace %s with error: %s\n", RESOURCE_NAME, NAMESPACE, getErr.Error())
                status = false
                break
            }

            log.Printf("Replicas status: %d\n", getResult.Status.ReadyReplicas)
            log.Printf("ReadyReplicas status: %d\n", getResult.Status.Replicas)
            log.Printf("AvailableReplicas status: %d\n", getResult.Status.AvailableReplicas)
            log.Printf("UpdatedReplicas status: %d\n", getResult.Status.UpdatedReplicas)
            log.Printf("Replicas spec: %d\n", *(getResult.Spec.Replicas))
            if getResult.Status.ReadyReplicas == *(getResult.Spec.Replicas) {
                status = true
                break
            }
            status = false
        }
        channel <- (status)
    }(status)

    select {
    case <-channel:
        if status {
            log.Printf("Deployment '%s' is up", RESOURCE_NAME)
        } else {
            log.Printf("Deployment '%s' is NOT up", RESOURCE_NAME)
        }
    case <-time.After(1 * time.Second):
        log.Printf("Deployment '%s' timed out", RESOURCE_NAME)
    }
}

日志 #

...
2022/11/25 17:46:55 UpdatedReplicas status: 0
2022/11/25 17:46:55 Replicas spec: 3
2022/11/25 17:46:55 Replicas status: 0
2022/11/25 17:46:55 ReadyReplicas status: 0
2022/11/25 17:46:55 AvailableReplicas status: 0
2022/11/25 17:46:55 UpdatedReplicas status: 0
2022/11/25 17:46:55 Replicas spec: 3
2022/11/25 17:46:55 Replicas status: 0
2022/11/25 17:46:55 ReadyReplicas status: 0
2022/11/25 17:46:55 AvailableReplicas status: 0
2022/11/25 17:46:55 UpdatedReplicas status: 0
2022/11/25 17:46:55 Replicas spec: 3
2022/11/25 17:46:55 Deployment 'TestResource' timed out
2022/11/25 17:46:55 Replicas status: 0
2022/11/25 17:46:55 ReadyReplicas status: 0
2022/11/25 17:46:55 AvailableReplicas status: 0
2022/11/25 17:46:55 UpdatedReplicas status: 0
2022/11/25 17:46:55 Replicas spec: 3

内容的提问来源于stack exchange，提问作者kelsonic