Cloudwatch Subscription Filter无法向Kinesis Data Stream传输日志求助

核心问题点分析 #

你的配置存在几个关键疏漏，导致日志无法正常转发：

• Kinesis数据流缺少资源策略：仅给IAM角色配置Kinesis权限不够，必须在Kinesis资源上添加信任策略，明确允许logs.amazonaws.com写入数据流。
• 空过滤模式的潜在问题：虽然AWS允许空过滤模式，但部分场景下可能无法匹配所有日志，建议使用?来捕获任意内容的日志。
• 信任策略条件可优化：CloudWatch ingestion角色的信任策略中aws:SourceArn条件过于宽泛，限定为具体日志组ARN能提升安全性。

修正后的Terraform代码 #

resource "aws_iam_role" "sample_lambda_role" {
  name = "sample_lambda_role"

  assume_role_policy = <<EOF
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Action": "sts:AssumeRole",
      "Principal": {
        "Service": "lambda.amazonaws.com"
      },
      "Effect": "Allow",
      "Sid": ""
    }
  ]
}
EOF
}

resource "aws_iam_policy" "sample_lambda_policy" {
  name        = "sample_lambda_policy"
  path        = "/"
  description = "AWS IAM Policy for managing aws lambda role"
  policy      = <<EOF
{
 "Version": "2012-10-17",
 "Statement": [
   {
     "Action": [
       "logs:CreateLogGroup",
       "logs:CreateLogStream",
       "logs:PutLogEvents"
     ],
     "Resource": "arn:aws:logs:*:*:*",
     "Effect": "Allow"
   }
 ]
}
EOF
}

resource "aws_iam_role_policy_attachment" "sample_lambda_attach_iam_policy_to_iam_role" {
  role       = aws_iam_role.sample_lambda_role.name
  policy_arn = aws_iam_policy.sample_lambda_policy.arn
}

data "archive_file" "lambda_app_zip" {
  type        = "zip"
  source_dir  = "${path.module}/sample-lambda"
  output_path = "${path.module}/sample-lambda.zip"
}

resource "aws_lambda_function" "sample_lambda" {
  filename         = "${path.module}/sample-lambda.zip"
  function_name    = "sample-lambda"
  role             = aws_iam_role.sample_lambda_role.arn
  handler          = "index.handler"
  source_code_hash = data.archive_file.lambda_app_zip.output_base64sha256
  runtime          = "nodejs14.x"
  depends_on       = [aws_iam_role_policy_attachment.sample_lambda_attach_iam_policy_to_iam_role]
}

resource "aws_cloudwatch_log_group" "sample_lambda_function_log_group" {
  name              = "/aws/lambda/${aws_lambda_function.sample_lambda.function_name}"
  retention_in_days = 1
  lifecycle {
    prevent_destroy = false
  }
}

resource "aws_kinesis_stream" "log_stream" {
  name             = "terraform-kinesis-test"
  shard_count      = 1
  retention_period = 24

  shard_level_metrics = [
    "IncomingBytes",
    "OutgoingBytes",
  ]

  # 添加Kinesis资源策略，允许CloudWatch日志写入
  resource_policy = <<POLICY
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Effect": "Allow",
      "Principal": {
        "Service": "logs.amazonaws.com"
      },
      "Action": "kinesis:PutRecord",
      "Resource": "${aws_kinesis_stream.log_stream.arn}",
      "Condition": {
        "ArnLike": {
          "aws:SourceArn": "${aws_cloudwatch_log_group.sample_lambda_function_log_group.arn}"
        }
      }
    }
  ]
}
POLICY
}

resource "aws_iam_role" "cloudwatch_ingestion_role" {
  name = "cloudwatch_ingestion_role"

  assume_role_policy = <<EOF
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Action": "sts:AssumeRole",
      "Principal": {
        "Service": "logs.amazonaws.com"
      },
      "Effect": "Allow",
      "Sid": "",
      "Condition": { 
        "ArnLike": { "aws:SourceArn": "${aws_cloudwatch_log_group.sample_lambda_function_log_group.arn}" } 
      }
    }
  ]
}
EOF
}

resource "aws_iam_policy" "cloudwatch_ingestion_policy" {
  name        = "cloudwatch_ingestion_policy"
  path        = "/"
  description = "AWS IAM Policy for cloudwatch logs ingestion"
  policy      = <<EOF
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Action": "kinesis:PutRecord",
      "Resource": "${aws_kinesis_stream.log_stream.arn}",
      "Effect": "Allow"
    }
  ]
}
EOF
}

resource "aws_iam_role_policy_attachment" "cloudwatch_ingestion_attach_iam_policy_to_iam_role" {
  role       = aws_iam_role.cloudwatch_ingestion_role.name
  policy_arn = aws_iam_policy.cloudwatch_ingestion_policy.arn
}

resource "aws_cloudwatch_log_subscription_filter" "sample_lambda_function_logfilter" {
  name            = "sample_lambda_function_logfilter"
  role_arn        = aws_iam_role.cloudwatch_ingestion_role.arn
  log_group_name  = aws_cloudwatch_log_group.sample_lambda_function_log_group.name
  filter_pattern  = "?" # 匹配所有日志内容
  destination_arn = aws_kinesis_stream.log_stream.arn
  distribution    = "ByLogStream"
}

验证步骤 #

1. 重新部署修正后的Terraform堆栈
2. 触发Lambda函数生成新日志
3. 查看Kinesis数据流的IncomingBytes指标，确认有数据流入
4. 通过Kinesis Data Analytics或Lambda读取数据流，验证日志内容是否正确转发

内容的提问来源于stack exchange，提问作者user203687