You need to enable JavaScript to run this app.
导航

断点续传下载(Node.js SDK)

最近更新时间2024.02.26 17:05:19

首次发布时间2022.05.13 18:28:15

下载大文件时,可以使用 downloadFile 断点续传下载接口。断点续传下载使用了分片下载将内容写入到临时文件中,如果出现部分分片成功部分失败,可以记录下载成功和失败分片的信息,保存到 Checkpoint 文件中。再次下载相同对象时,只下载失败的分片。当分片全部成功时将临时文件重命名为正式文件,并删除 Checkpoint 文件。

注意事项

  • 下载对象前,您必须具有 tos:GetObject 权限,具体操作,请参见权限配置指南
  • 对于开启多版本的桶,下载指定版本对象时,您必须具有 tos:GetObjectVersion 权限,具体操作,请参见权限配置指南
  • 如果应用程序会在同一时刻大量下载同一个对象,您的访问速度会受到 TOS 带宽及地域的限制。建议您使用 CDN 产品,提升性能的同时也能降低您的成本。通过 CDN 访问 TOS 的详细信息,请参见使用 CDN 加速访问 TOS 资源
  • SDK 会将上传的状态信息记录在 Checkpoint 文件中,所以程序需要对 Checkpoint 文件有写权限。
  • 使用断点续传下载时,文件下载的进度信息会记录在 Checkpoint 文件中,如果下载过程中某一分片下载失败,再次下载时会从 Checkpoint 文件中记录的点继续下载,从而达到断点续传的效果。下载完成后, Checkpoint 文件会被删除。
  • 如果下载过程中文件的 ETag 发生变化、Part 丢失或被修改,则重新下载文件。

示例代码

断点续传下载

以下代码用于使用断点续传的方式下载文件。

// 导入 SDK, 当 TOS Node.JS SDK 版本小于 2.5.2 请把下方 TosClient 改成 TOS 导入
import { TosClient, TosClientError, TosServerError } from '@volcengine/tos-sdk';
import fsp from 'fs/promises';

// 创建客户端
const client = new TosClient({
  accessKeyId: process.env['TOS_ACCESS_KEY'],
  accessKeySecret: process.env['TOS_SECRET_KEY'],
  region: "Provide your region", // 填写 Bucket 所在地域。以华北2(北京)为例,则 "Provide your region" 填写为 cn-beijing。
  endpoint: "Provide your endpoint", // 填写域名地址
});

function handleError(error) {
  if (error instanceof TosClientError) {
    console.log('Client Err Msg:', error.message);
    console.log('Client Err Stack:', error.stack);
  } else if (error instanceof TosServerError) {
    console.log('Request ID:', error.requestId);
    console.log('Response Status Code:', error.statusCode);
    console.log('Response Header:', error.headers);
    console.log('Response Err Code:', error.code);
    console.log('Response Err Msg:', error.message);
  } else {
    console.log('unexpected exception, message: ', error);
  }
}

async function main() {
  try {
    const bucketName = 'node-sdk-test-bucket'; // 目标桶
    const objectName = 'example_dir/example.txt'; // 目标对象
    // 下载的文件路径
    const filePath = './example_dir/example.txt';
    await client.downloadFile({
      bucket: bucketName,
      key: objectName,
      // 下载的文件路径
      filePath,
      // 分片下载任务并发数量
      taskNum: 5,
      // 指定断点续传临时文件路径
      checkpoint: './example_dir/example.checkpoint',
    });

    const fileStat = await fsp.stat(filePath);
    console.log('fileSize: ', fileStat.size);
  } catch (error) {
    handleError(error);
  }
}

main();

配置进度条

断点续传下载时可通过 dataTransferStatusChange 参数接收下载进度,代码示例如下。

// 导入 SDK, 当 TOS Node.JS SDK 版本小于 2.5.2 请把下方 TosClient 改成 TOS 导入
import { DataTransferType, TosClient, TosClientError, TosServerError } from '@volcengine/tos-sdk';
import fsp from 'fs/promises';

// 创建客户端
const client = new TosClient({
  accessKeyId: process.env['TOS_ACCESS_KEY'],
  accessKeySecret: process.env['TOS_SECRET_KEY'],
  region: "Provide your region", // 填写 Bucket 所在地域。以华北2(北京)为例,则 "Provide your region" 填写为 cn-beijing。
  endpoint: "Provide your endpoint", // 填写域名地址
});

function handleError(error) {
  if (error instanceof TosClientError) {
    console.log('Client Err Msg:', error.message);
    console.log('Client Err Stack:', error.stack);
  } else if (error instanceof TosServerError) {
    console.log('Request ID:', error.requestId);
    console.log('Response Status Code:', error.statusCode);
    console.log('Response Header:', error.headers);
    console.log('Response Err Code:', error.code);
    console.log('Response Err Msg:', error.message);
  } else {
    console.log('unexpected exception, message: ', error);
  }
}

async function main() {
  try {
    const bucketName = 'node-sdk-test-bucket'; // 目标桶
    const objectName = 'example_dir/example.txt'; // 目标对象
    // 下载的文件路径
    const filePath = './example_dir/example.txt';
    await client.downloadFile({
      bucket: bucketName,
      key: objectName,
      // 下载的文件路径
      filePath,
      // 下载时指定分片大小 10M
      partSize: 10 * 1024 * 1024,
      // 分片下载任务并发数量
      taskNum: 5,
      // 指定断点续传临时文件路径
      checkpoint: './example_dir/example.checkpoint',
      // 通过自定义方式设置回调函数查看下载进度
      dataTransferStatusChange: (event) => {
        if (event.type === DataTransferType.Started) {
          console.log('Data Transfer Started');
        } else if (event.type === DataTransferType.Rw) {
          const percent = ((event.consumedBytes / event.totalBytes) * 100).toFixed(2);
          console.log(`Once Read:${event.rwOnceBytes},ConsumerBytes/TotalBytes: ${event.consumedBytes}/${event.totalBytes},${percent}%`);
        } else if (event.type === DataTransferType.Succeed) {
          const percent = ((event.consumedBytes / event.totalBytes) * 100).toFixed(2);
          console.log(`Data Transfer Succeed, ConsumerBytes/TotalBytes:${event.consumedBytes}/${event.totalBytes},${percent}%`);
        } else if (event.type === DataTransferType.Failed) {
          console.log('Data Transfer Failed');
        }
      },
    });

    const fileStat = await fsp.stat(filePath);
    console.log('fileSize: ', fileStat.size);
  } catch (error) {
    handleError(error);
  }
}

main();

配置事件回调

以下代码用于自定义断点续传下载回调函数。

// 导入 SDK, 当 TOS Node.JS SDK 版本小于 2.5.2 请把下方 TosClient 改成 TOS 导入
import { DataTransferType, DownloadEventType, TosClient, TosClientError, TosServerError } from '@volcengine/tos-sdk';
import fsp from 'fs/promises';

// 创建客户端
const client = new TosClient({
  accessKeyId: process.env['TOS_ACCESS_KEY'],
  accessKeySecret: process.env['TOS_SECRET_KEY'],
  region: "Provide your region", // 填写 Bucket 所在地域。以华北2(北京)为例,则 "Provide your region" 填写为 cn-beijing。
  endpoint: "Provide your endpoint", // 填写域名地址
});

function handleError(error) {
  if (error instanceof TosClientError) {
    console.log('Client Err Msg:', error.message);
    console.log('Client Err Stack:', error.stack);
  } else if (error instanceof TosServerError) {
    console.log('Request ID:', error.requestId);
    console.log('Response Status Code:', error.statusCode);
    console.log('Response Header:', error.headers);
    console.log('Response Err Code:', error.code);
    console.log('Response Err Msg:', error.message);
  } else {
    console.log('unexpected exception, message: ', error);
  }
}

async function main() {
  try {
    const bucketName = 'node-sdk-test-bucket'; // 目标桶
    const objectName = 'example_dir/example.txt'; // 目标对象
    // 下载的文件路径
    const filePath = './example_dir/example.txt';
    await client.downloadFile({
      bucket: bucketName,
      key: objectName,
      // 下载的文件路径
      filePath,
      // 下载时指定分片大小 10M
      partSize: 10 * 1024 * 1024,
      // 分片下载任务并发数量
      taskNum: 5,
      // 指定断点续传临时文件路径
      checkpoint: './example_dir/example.checkpoint',
      // 通过自定义方式设置回调函数查看下载进度
      dataTransferStatusChange: (event) => {
        if (event.type === DataTransferType.Started) {
          console.log('Data Transfer Started');
        } else if (event.type === DataTransferType.Rw) {
          const percent = ((event.consumedBytes / event.totalBytes) * 100).toFixed(2);
          console.log(`Once Read:${event.rwOnceBytes},ConsumerBytes/TotalBytes: ${event.consumedBytes}/${event.totalBytes},${percent}%`);
        } else if (event.type === DataTransferType.Succeed) {
          const percent = ((event.consumedBytes / event.totalBytes) * 100).toFixed(2);
          console.log(`Data Transfer Succeed, ConsumerBytes/TotalBytes:${event.consumedBytes}/${event.totalBytes},${percent}%`);
        } else if (event.type === DataTransferType.Failed) {
          console.log('Data Transfer Failed');
        }
      },
      // 事件监听回调
      downloadEventChange: (event) => {
        switch (event.type) {
          case DownloadEventType.CreateTempFileSucceed: {
            console.log('Download File Create Temp File Success.');
            break;
          }
          case DownloadEventType.CreateTempFileFailed: {
            console.log('Download File Create Temp File Fail.');
            break;
          }
          case DownloadEventType.DownloadPartSucceed: {
            console.log(`Download ${event.bucket} ${event.key} part success, DownloadPartInfo:%o`, event.downloadPartInfo);
            break;
          }
          case DownloadEventType.DownloadPartAborted: {
            console.log(`Download ${event.bucket} ${event.key} part aborted.`);
            break;
          }
          case DownloadEventType.DownloadPartFailed: {
            console.log(`Download ${event.bucket} ${event.key} part fail, err:%o`, event.err);
            break;
          }
          case DownloadEventType.RenameTempFileSucceed: {
            console.log(`Download ${event.bucket} ${event.key} success.`);
            break;
          }
          case DownloadEventType.CreateTempFileFailed: {
            console.log(`Download ${event.bucket} ${event.key} fail, err:%o`, event.err);
            break;
          }
        }
      },
    });

    const fileStat = await fsp.stat(filePath);
    console.log('fileSize: ', fileStat.size);
  } catch (error) {
    handleError(error);
  }
}

main();

配置客户端限速

断点续传下载时可以通过客户端使用 rateLimiter 参数对下载数据所占用的带宽进行限制,代码如下所示。

// 导入 SDK, 当 TOS Node.JS SDK 版本小于 2.5.2 请把下方 TosClient 改成 TOS 导入
import { DataTransferType, createDefaultRateLimiter, TosClient, TosClientError, TosServerError } from '@volcengine/tos-sdk';
import fsp from 'fs/promises';

// 创建客户端
const client = new TosClient({
  accessKeyId: process.env['TOS_ACCESS_KEY'],
  accessKeySecret: process.env['TOS_SECRET_KEY'],
  region: "Provide your region", // 填写 Bucket 所在地域。以华北2(北京)为例,则 "Provide your region" 填写为 cn-beijing。
  endpoint: "Provide your endpoint", // 填写域名地址
});

function handleError(error) {
  if (error instanceof TosClientError) {
    console.log('Client Err Msg:', error.message);
    console.log('Client Err Stack:', error.stack);
  } else if (error instanceof TosServerError) {
    console.log('Request ID:', error.requestId);
    console.log('Response Status Code:', error.statusCode);
    console.log('Response Header:', error.headers);
    console.log('Response Err Code:', error.code);
    console.log('Response Err Msg:', error.message);
  } else {
    console.log('unexpected exception, message: ', error);
  }
}

async function main() {
  try {
    const bucketName = 'node-sdk-test-bucket'; // 目标桶
    const objectName = 'example_dir/example.txt'; // 目标对象
    // 下载的文件路径
    const filePath = './example_dir/example.txt';
    const rateLimit1M = 1024 * 1024;
    await client.downloadFile({
      bucket: bucketName,
      key: objectName,
      // 下载的文件路径
      filePath,
      // 下载时指定分片大小 10M
      partSize: 10 * 1024 * 1024,
      // 分片下载任务并发数量
      taskNum: 5,
      // 指定断点续传临时文件路径
      checkpoint: './example_dir/example.checkpoint',
      // 通过自定义方式设置回调函数查看下载进度
      dataTransferStatusChange: (event) => {
        if (event.type === DataTransferType.Started) {
          console.log('Data Transfer Started');
        } else if (event.type === DataTransferType.Rw) {
          const percent = ((event.consumedBytes / event.totalBytes) * 100).toFixed(2);
          console.log(`Once Read:${event.rwOnceBytes},ConsumerBytes/TotalBytes: ${event.consumedBytes}/${event.totalBytes},${percent}%`);
        } else if (event.type === DataTransferType.Succeed) {
          const percent = ((event.consumedBytes / event.totalBytes) * 100).toFixed(2);
          console.log(`Data Transfer Succeed, ConsumerBytes/TotalBytes:${event.consumedBytes}/${event.totalBytes},${percent}%`);
        } else if (event.type === DataTransferType.Failed) {
          console.log('Data Transfer Failed');
        }
      },
      // 下载对象并在客户端限制下载速度为 1M/s
      rateLimiter: createDefaultRateLimiter(rateLimit1M, rateLimit1M),
    });

    const fileStat = await fsp.stat(filePath);
    console.log('fileSize: ', fileStat.size);
  } catch (error) {
    handleError(error);
  }
}

main();

取消机制

以下代码用于在运行时取消正在执行的断点续传下载任务。

// 导入 SDK, 当 TOS Node.JS SDK 版本小于 2.5.2 请把下方 TosClient 改成 TOS 导入
import { DataTransferType, CancelToken, TosClient, TosClientError, TosServerError, isCancel } from '@volcengine/tos-sdk';
import fsp from 'fs/promises';

// 创建客户端
const client = new TosClient({
  accessKeyId: process.env['TOS_ACCESS_KEY'],
  accessKeySecret: process.env['TOS_SECRET_KEY'],
  region: "Provide your region", // 填写 Bucket 所在地域。以华北2(北京)为例,则 "Provide your region" 填写为 cn-beijing。
  endpoint: "Provide your endpoint", // 填写域名地址
});

function handleError(error) {
  if (error instanceof TosClientError) {
    console.log('Client Err Msg:', error.message);
    console.log('Client Err Stack:', error.stack);
  } else if (error instanceof TosServerError) {
    console.log('Request ID:', error.requestId);
    console.log('Response Status Code:', error.statusCode);
    console.log('Response Header:', error.headers);
    console.log('Response Err Code:', error.code);
    console.log('Response Err Msg:', error.message);
  } else {
    console.log('unexpected exception, message: ', error);
  }
}

async function main() {
  // checkpoint 文件路径
  const checkpointFilePath = './example_dir/example.checkpoint';

  try {
    const bucketName = 'node-sdk-test-bucket'; // 目标桶
    const objectName = 'example_dir/example.txt'; // 目标对象
    // 下载的文件路径
    const filePath = './example_dir/example.txt';

    try {
      // 生成 cancelTokenSource
      const cancelTokenSource = CancelToken.source();
      // 1 秒后取消任务
      setTimeout(() => {
        cancelTokenSource.cancel();
        console.log('cancel downloadFile');
      }, 1_000);

      await client.downloadFile({
        bucket: bucketName,
        key: objectName,
        // 下载的文件路径
        filePath,
        // 下载时指定分片大小 1M
        partSize: 1 * 1024 * 1024,
        // 分片下载任务并发数量
        taskNum: 5,
        // 指定断点续传临时文件路径
        checkpoint: './example_dir/example.checkpoint',
        // 通过自定义方式设置回调函数查看下载进度
        dataTransferStatusChange: (event) => {
          if (event.type === DataTransferType.Started) {
            console.log('Data Transfer Started');
          } else if (event.type === DataTransferType.Rw) {
            const percent = ((event.consumedBytes / event.totalBytes) * 100).toFixed(2);
            console.log(`Once Read:${event.rwOnceBytes},ConsumerBytes/TotalBytes: ${event.consumedBytes}/${event.totalBytes},${percent}%`);
          } else if (event.type === DataTransferType.Succeed) {
            const percent = ((event.consumedBytes / event.totalBytes) * 100).toFixed(2);
            console.log(`Data Transfer Succeed, ConsumerBytes/TotalBytes:${event.consumedBytes}/${event.totalBytes},${percent}%`);
          } else if (event.type === DataTransferType.Failed) {
            console.log('Data Transfer Failed');
          }
        },
        cancelToken: cancelTokenSource.token,
      });
    } catch (error) {
      if (!isCancel(error)) {
        throw error;
      }

      const checkpointFileContent = await fsp.readFile(checkpointFilePath, 'utf-8');
      console.log(`checkpoint file's content after cancel: `, checkpointFileContent);

      console.log('Continue to download');
      // 可以通过 checkpoint File 断点续传下载
      await client.downloadFile({
        bucket: bucketName,
        key: objectName,
        // 下载的文件路径
        filePath,
        // 下载时指定分片大小 1M
        partSize: 1 * 1024 * 1024,
        // 分片下载任务并发数量
        taskNum: 5,
        // 指定断点续传临时文件路径
        checkpoint: './example_dir/example.checkpoint',
        // 通过自定义方式设置回调函数查看下载进度
        dataTransferStatusChange: (event) => {
          if (event.type === DataTransferType.Started) {
            console.log('Data Transfer Started');
          } else if (event.type === DataTransferType.Rw) {
            const percent = ((event.consumedBytes / event.totalBytes) * 100).toFixed(2);
            console.log(`Once Read:${event.rwOnceBytes},ConsumerBytes/TotalBytes: ${event.consumedBytes}/${event.totalBytes},${percent}%`);
          } else if (event.type === DataTransferType.Succeed) {
            const percent = ((event.consumedBytes / event.totalBytes) * 100).toFixed(2);
            console.log(`Data Transfer Succeed, ConsumerBytes/TotalBytes:${event.consumedBytes}/${event.totalBytes},${percent}%`);
          } else if (event.type === DataTransferType.Failed) {
            console.log('Data Transfer Failed');
          }
        },
      });

      const fileStat = await fsp.stat(filePath);
      console.log('fileSize: ', fileStat.size);
    }
  } catch (error) {
    handleError(error);
  }
}

main();