K8s部署NestJS微服务配置NGINX Ingress返回404问题求助

我正在将基于Node.js与MySQL构建的微服务应用迁移至Kubernetes平台，接入NGINX Ingress Controller以实现弹性扩缩容能力。目前所有服务已打包为容器镜像并以Pod形式部署，架构中包含一个网关负责将TCP流量转发至各后端微服务。

此前通过分别执行以下端口转发命令暴露服务时，应用可以正常访问：

• kubectl port-forward my-gateway --address 192.168.18.x 3000
• kubectl port-forward my-adm-contact --address 192.168.18.x 8181

由于单条port-forward命令无法同时转发多个Pod端口，我通过Helm安装了NGINX Ingress Controller统一暴露服务，但访问页面时始终返回 404 Not Found 错误。

网关服务代码（NestJS） #

import { Module } from '@nestjs/common';
import { AppController } from './app.controller';
import { ClientsModule, Transport } from '@nestjs/microservices';
import { AppService } from './app.service';

@Module({
  imports: [
    ClientsModule.register([
      {
        name: 'SERVICE_A',
        transport: Transport.TCP,
        options: {
          host: "192.168.18.x",
          port: 8888,
        },
      },
      {
        name: 'SERVICE_B',
        transport: Transport.TCP,
        options: {
          host: "192.168.18.x",
          port: 8889,
        },
      },
      {
        name: 'USER',
        transport: Transport.TCP,
        options: {
          host: "192.168.18.x",
          port: 8887,
        },
      },
      {
        name: 'USER_LOGIN',
        transport: Transport.TCP,
        options: {
          host: "192.168.18.x",
          port: 8886,
        },
      },
      {
        name: 'USER_CREATE',
        transport: Transport.TCP,
        options: {
          host: "192.168.18.x",
          port: 8885,
        },
      },
      {
        name: 'USER_UPDATE',
        transport: Transport.TCP,
        options: {
          host: "192.168.18.x",
          port: 8884,
        },
      },
      {
        name: 'CATEGORY',
        transport: Transport.TCP,
        options: {
          host: "192.168.18.x",
          port: 8883,
        },
      },
      {
        name: 'CATEGORY_BUSCA',
        transport: Transport.TCP,
        options: {
          host: "192.168.18.x",
          port: 8882,
        },
      },
      {
        name: 'CATEGORY_PRODUCT',
        transport: Transport.TCP,
        options: {
          host: "192.168.18.x",
          port: 8881,
        },
      },
      {
        name: 'USER_SENHA',
        transport: Transport.TCP,
        options: {
          host: "192.168.18.x",
          port: 8880,
        },
      },
      {
        name: 'ADM_CONTACT',
        transport: Transport.TCP,
        options: {
          host: "192.168.18.x",
          port: 8181,
        },
      },
      {
        name: 'LOCATION',
        transport: Transport.TCP,
        options: {
          host: "192.168.18.x",
          port: 8878,
        },
      },
      {
        name: 'PRODUCT_STAR',
        transport: Transport.TCP,
        options: {
          host: "192.168.18.x",
          port: 8877,
        },
      },
      {
        name: 'PRODUCT_SINGLE',
        transport: Transport.TCP,
        options: {
          host: "192.168.18.x",
          port: 8876,
        },
      },
      {
        name: 'PRODUCT_GET_STAR',
        transport: Transport.TCP,
        options: {
          host: "192.168.18.x",
          port: 8875,
        },
      },
      {
        name: 'PURCHASE_CREATE',
        transport: Transport.TCP,
        options: {
          host: "192.168.18.x",
          port: 8874,
        },
      },
      {
        name: 'PURCHASE_GET_CART',
        transport: Transport.TCP,
        options: {
          host: "192.168.18.x",
          port: 8873,
        },
      },
      {
        name: 'PURCHASE_GET',
        transport: Transport.TCP,
        options: {
          host: "192.168.18.x",
          port: 8870,
        },
      }
    ]),
  ],
  controllers: [AppController],
  providers: [AppService],
})
export class AppModule {}

contact微服务启动代码 #

import { NestFactory } from '@nestjs/core';
import { Transport } from '@nestjs/microservices';
import { AppModule } from './app.module';
import { Logger } from '@nestjs/common';

const logger = new Logger();

async function bootstrap() {
  const app = await NestFactory.createMicroservice(AppModule, {
    transport: Transport.TCP,
    options: {
      host: "0.0.0.0",
      port: 8181,
    },
  });
  app.listen(() => logger.log('Microservice ADM CONTACT is listening'));
}
bootstrap();

contact服务Kubernetes配置 #

Service配置 #

apiVersion: v1
kind: Service
metadata:
  name: my-adm-contact-service
  namespace: default
spec:
  ports:
  - port: 8181
    protocol: TCP
    targetPort: 8181

Pod配置 #

apiVersion: v1
kind: Pod
metadata:
  name: my-adm-contact
  labels:
    app: my-adm-contact
    name: my-adm-contact
spec:
  containers:
  - image: my-adm-contact
    imagePullPolicy: Never
    name: my-adm-contact
    ports:
    - containerPort: 8181
      protocol: TCP
  restartPolicy: Always

Endpoints配置 #

apiVersion: v1
kind: Endpoints
metadata:
  name: my-adm-contact-service
subsets:
  - addresses:
      - ip: 192.168.18.x
    ports:
      - port: 8181

网关服务Kubernetes配置 #

Service配置 #

apiVersion: v1
kind: Service
metadata:
  name:  my-gateway-service
  namespace: default
spec:
  ports:
  - port: 3000
    protocol: TCP
    targetPort: 3000

Pod配置 #

apiVersion: v1
kind: Pod
metadata:
  name: my-gateway
  labels:
    app: my-gateway
    name: my-gateway
spec:
  containers:
  - image: api-gateway
    imagePullPolicy: Never
    name: my-gateway
    ports:
    - containerPort: 3000
      protocol: TCP
  restartPolicy: Always

Endpoints配置 #

apiVersion: v1
kind: Endpoints
metadata:
  name: my-gateway-service
subsets:
  - addresses:
      - ip: 192.168.18.x
    ports:
      - port: 3000

Ingress资源配置 #

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: web-ingress
  namespace: default
spec:
  ingressClassName: nginx
  rules:
  - host: host.docker.internal
  - http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: my-gateway-service
            port:
              number: 3000
      - path: /adm-contact
        pathType: Prefix
        backend:
          service:
            name: my-adm-contact-service
            port:
              number: 8181

ingress-nginx-tcp ConfigMap配置 #

apiVersion: v1
data:
  "3000": default/my-gateway-service:3000
  "8181": default/my-adm-contact-service:8181
kind: ConfigMap
metadata:
  annotations:
    meta.helm.sh/release-name: ingress-nginx
    meta.helm.sh/release-namespace: ingress-nginx
  creationTimestamp: "2022-06-28T00:00:28Z"
  labels:
    app.kubernetes.io/component: controller
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
    app.kubernetes.io/version: 1.2.1
    helm.sh/chart: ingress-nginx-4.1.4
  name: ingress-nginx-tcp
  namespace: ingress-nginx
  resourceVersion: "264565"
  uid: 2284f6eb-53f9-4d8c-9f62-6ad303120f63

返回404的核心问题来自4处配置错误：

1. Ingress规则缩进错误：当前配置中- host: host.docker.internal和- http:是两个平级的独立规则条目，意味着匹配host.docker.internal域名的规则没有配置任何HTTP转发路径，所有携带该Host头的请求都会直接返回404。需要将http块嵌套到对应host规则下，修正后格式如下：

rules:
- host: host.docker.internal
  http:
    paths:
    - path: /
      pathType: Prefix
      backend:
        service:
          name: my-gateway-service
          port:
            number: 3000

2. 错误给纯TCP微服务配置HTTP路由：contact服务是通过NestFactory.createMicroservice启动的原生TCP服务，没有HTTP协议监听能力，Ingress的HTTP路径规则无法转发TCP原生流量，直接将/adm-contact路径指向该服务必然返回404。所有内部TCP微服务不需要配置Ingress规则，仅需让网关通过集群内部网络访问即可，不需要暴露到Ingress层。
3. 服务发现配置不符合Kubernetes规范：网关代码中所有后端微服务地址都硬编码为节点固定IP，Service资源也没有配置selector自动关联Pod端点，手动维护Endpoints的方式在Pod重建、弹性扩缩容场景下会直接失效。需要将所有后端服务的host改为Kubernetes内部服务域名，例如ADM_CONTACT的host改为my-adm-contact-service.default.svc.cluster.local，同时给Service添加对应label selector自动关联Pod，无需手动创建Endpoints资源。
4. TCP端口转发配置不完整：虽然创建了tcp-services ConfigMap，但Helm安装的ingress-nginx默认不会自动读取该ConfigMap暴露TCP端口，需要在ingress-nginx的Helm values配置中开启tcp参数关联该ConfigMap，同时给ingress-nginx的Service添加对应3000、8181端口映射才能生效。实际场景下仅需暴露HTTP网关服务即可，内部TCP微服务走集群内部通信，无需对外暴露。

内容的提问来源于stack exchange，提问作者Rafael Souza