使用@aws-sdk/client-dynamodb的ScanCommand无法获取DynamoDB全量记录

我有一个由API Gateway触发的Lambda服务（模块类型），用于从DynamoDB表中获取记录。使用ScanCommand读取所有记录时，DynamoDB控制台显示表中有1903条记录，但该命令仅返回1350条，缺失了30%以上的数据。日志中无报错信息，数据也不存在特殊字符等异常情况。

相关代码 #

Lambda函数代码 #

import { ScanCommand } from "@aws-sdk/client-dynamodb";
import { marshall, unmarshall } from "@aws-sdk/util-dynamodb";
import { dbClient } from "./db.js";

const tableName = "restaurants";

const getAllRestaurants = async () => {
  console.log("getAllRestaurants");
  try {
    const params = {
      TableName: tableName,
    };

    const { Items } = await dbClient.send(new ScanCommand(params));

    console.log(Items);
    return Items ? Items.map((item) => unmarshall(item)) : {};
  } catch (e) {
    console.error(e);
    throw e;
  }
};


export const handler = async function (event) {
  console.log("request:", JSON.stringify(event, undefined, 2));
  let body;

  try {
    switch (event.httpMethod) {
      case "GET":
        if (event.pathParameters !== null) {
          body = await getRestaurant(event);
        } else {
          body = await getAllRestaurants();
        }
        break;
      default:
        throw new Error(`Unsupported route: \"${event.httpMethod}\"`);
    }
    console.log(body);

    return {
      statusCode: 200,
      headers: {
        "Access-Control-Allow-Headers": "*",
        "Access-Control-Allow-Origin": "*",
      },
      body: JSON.stringify({
        message: `Successfully finished Restaurant operation: \"${event.httpMethod}\"`,
        body,
      }),
    };
  } catch (e) {
    console.error(e);
    return {
      statusCode: 500,
      body: JSON.stringify({
        message: "Failed to perform Restaurant operation.",
        errorMsg: e.message,
        errorStack: e.stack,
      }),
    };
  }
};

DB客户端代码 #

import { DynamoDBClient } from "@aws-sdk/client-dynamodb"
const REGION = "us-east-2"
const dbClient = new DynamoDBClient({ region: REGION })
export { dbClient }

package.json依赖 #

"dependencies": {
    "@aws-sdk/client-dynamodb": "^3.352.0",
    "@aws-sdk/util-dynamodb": "^3.352.0"
  }

未被获取的示例记录 #

{
  "restaurantId": {
    "S": "5cb7449b-7a11-46f1-830b-82fc5b5a99a0"
  },
  "accessability": {
    "BOOL": true
  },
  "city": {
    "S": "Walnut Creek"
  },
  "coordinates": {
    "M": {
      "latitude": {
        "N": "37.90086769999999"
      },
      "longitude": {
        "N": "-122.0612685"
      }
    }
  },
  "cuisine": {
    "S": "Burmese"
  },
  "dateAdded": {
    "S": "Sat Jan 20 2024 21:25:11 GMT-0800"
  },
  "dateUpdated": {
    "S": "Thu Jul 04 2024 03:53:10 GMT+0000 (Coordinated Universal Time)"
  },
  "delivery": {
    "BOOL": true
  },
  "description": {
    "S": ""
  },
  "formattedAddress": {
    "S": "1616 N Main St, Walnut Creek, CA 94596, USA"
  },
  "formattedPhone": {
    "S": "(925) 261-8580"
  },
  "name": {
    "S": "Burma 2"
  },
  "openHours": {
    "L": [
      {
        "S": "Monday: 11:30 AM – 2:30 PM, 5:00 – 9:00 PM"
      },
      {
        "S": "Tuesday: 11:30 AM – 2:30 PM, 5:00 – 9:00 PM"
      },
      {
        "S": "Wednesday: 11:30 AM – 2:30 PM, 5:00 – 9:00 PM"
      },
      {
        "S": "Thursday: 11:30 AM – 2:30 PM, 5:00 – 9:00 PM"
      },
      {
        "S": "Friday: 11:30 AM – 2:30 PM, 5:00 – 10:00 PM"
      },
      {
        "S": "Saturday: 11:30 AM – 10:00 PM"
      },
      {
        "S": "Sunday: 11:30 AM – 9:00 PM"
      }
    ]
  },
  "overallScore": {
    "N": "89"
  },
  "priceGuide": {
    "N": "2"
  },
  "vegeterian": {
    "BOOL": true
  },
  "website": {
    "S": "https://www.burma2.com/"
  }
}

问题根源 #

DynamoDB的Scan操作单次返回的数据量上限为1MB（或达到指定的Limit值），当表中数据总大小超过这个限制时，只会返回部分结果，并在响应中包含LastEvaluatedKey，用于标记下一次扫描的起始位置。你的代码仅执行了一次Scan，因此只获取了第一页的记录，导致缺失大量数据。

修改后的代码 #

更新getAllRestaurants函数，通过循环调用Scan并传入LastEvaluatedKey，直到获取所有数据：

import { ScanCommand } from "@aws-sdk/client-dynamodb";
import { unmarshall } from "@aws-sdk/util-dynamodb";
import { dbClient } from "./db.js";

const tableName = "restaurants";

const getAllRestaurants = async () => {
  console.log("getAllRestaurants");
  try {
    let allItems = [];
    let params = {
      TableName: tableName,
    };

    do {
      const response = await dbClient.send(new ScanCommand(params));
      if (response.Items) {
        allItems = [...allItems, ...response.Items];
      }
      // 带上上次扫描的起始键，继续获取剩余数据
      params.ExclusiveStartKey = response.LastEvaluatedKey;
    } while (params.ExclusiveStartKey); // 没有起始键时表示所有数据已获取

    console.log(`Total items retrieved: ${allItems.length}`);
    return allItems.map((item) => unmarshall(item));
  } catch (e) {
    console.error(e);
    throw e;
  }
};

说明 #

• 循环中每次执行Scan后，将返回的Items追加到总列表中
• 使用response.LastEvaluatedKey更新扫描参数，确保下一次从上次结束的位置继续
• 当LastEvaluatedKey不存在时，说明所有数据已获取完成，终止循环

这样修改后，你的Lambda函数就能获取到DynamoDB表中的全部1903条记录了。

内容的提问来源于stack exchange，提问作者Clarkie