下载大文件时,可以使用 downloadFile
断点续传下载接口。断点续传下载使用了分片下载将内容写入到临时文件中,如果出现部分分片成功部分失败,可以记录下载成功和失败分片的信息,保存到 Checkpoint 文件中。再次下载相同对象时,只下载失败的分片。当分片全部成功时将临时文件重命名为正式文件,并删除 Checkpoint 文件。
tos:GetObject
权限,具体操作,请参见权限配置指南。tos:GetObjectVersion
权限,具体操作,请参见权限配置指南。Checkpoint
文件中,所以程序需要对 Checkpoint
文件有写权限。Checkpoint
文件中,如果下载过程中某一分片下载失败,再次下载时会从 Checkpoint
文件中记录的点继续下载,从而达到断点续传的效果。下载完成后, Checkpoint
文件会被删除。以下代码用于使用断点续传的方式下载文件。
// 导入 SDK, 当 TOS Node.JS SDK 版本小于 2.5.2 请把下方 TosClient 改成 TOS 导入 import { TosClient, TosClientError, TosServerError } from '@volcengine/tos-sdk'; import fsp from 'fs/promises'; // 创建客户端 const client = new TosClient({ accessKeyId: process.env['TOS_ACCESS_KEY'], accessKeySecret: process.env['TOS_SECRET_KEY'], region: "Provide your region", // 填写 Bucket 所在地域。以华北2(北京)为例,则 "Provide your region" 填写为 cn-beijing。 endpoint: "Provide your endpoint", // 填写域名地址 }); function handleError(error) { if (error instanceof TosClientError) { console.log('Client Err Msg:', error.message); console.log('Client Err Stack:', error.stack); } else if (error instanceof TosServerError) { console.log('Request ID:', error.requestId); console.log('Response Status Code:', error.statusCode); console.log('Response Header:', error.headers); console.log('Response Err Code:', error.code); console.log('Response Err Msg:', error.message); } else { console.log('unexpected exception, message: ', error); } } async function main() { try { const bucketName = 'node-sdk-test-bucket'; // 目标桶 const objectName = 'example_dir/example.txt'; // 目标对象 // 下载的文件路径 const filePath = './example_dir/example.txt'; await client.downloadFile({ bucket: bucketName, key: objectName, // 下载的文件路径 filePath, // 分片下载任务并发数量 taskNum: 5, // 指定断点续传临时文件路径 checkpoint: './example_dir/example.checkpoint', }); const fileStat = await fsp.stat(filePath); console.log('fileSize: ', fileStat.size); } catch (error) { handleError(error); } } main();
断点续传下载时可通过 dataTransferStatusChange
参数接收下载进度,代码示例如下。
// 导入 SDK, 当 TOS Node.JS SDK 版本小于 2.5.2 请把下方 TosClient 改成 TOS 导入 import { DataTransferType, TosClient, TosClientError, TosServerError } from '@volcengine/tos-sdk'; import fsp from 'fs/promises'; // 创建客户端 const client = new TosClient({ accessKeyId: process.env['TOS_ACCESS_KEY'], accessKeySecret: process.env['TOS_SECRET_KEY'], region: "Provide your region", // 填写 Bucket 所在地域。以华北2(北京)为例,则 "Provide your region" 填写为 cn-beijing。 endpoint: "Provide your endpoint", // 填写域名地址 }); function handleError(error) { if (error instanceof TosClientError) { console.log('Client Err Msg:', error.message); console.log('Client Err Stack:', error.stack); } else if (error instanceof TosServerError) { console.log('Request ID:', error.requestId); console.log('Response Status Code:', error.statusCode); console.log('Response Header:', error.headers); console.log('Response Err Code:', error.code); console.log('Response Err Msg:', error.message); } else { console.log('unexpected exception, message: ', error); } } async function main() { try { const bucketName = 'node-sdk-test-bucket'; // 目标桶 const objectName = 'example_dir/example.txt'; // 目标对象 // 下载的文件路径 const filePath = './example_dir/example.txt'; await client.downloadFile({ bucket: bucketName, key: objectName, // 下载的文件路径 filePath, // 下载时指定分片大小 10M partSize: 10 * 1024 * 1024, // 分片下载任务并发数量 taskNum: 5, // 指定断点续传临时文件路径 checkpoint: './example_dir/example.checkpoint', // 通过自定义方式设置回调函数查看下载进度 dataTransferStatusChange: (event) => { if (event.type === DataTransferType.Started) { console.log('Data Transfer Started'); } else if (event.type === DataTransferType.Rw) { const percent = ((event.consumedBytes / event.totalBytes) * 100).toFixed(2); console.log(`Once Read:${event.rwOnceBytes},ConsumerBytes/TotalBytes: ${event.consumedBytes}/${event.totalBytes},${percent}%`); } else if (event.type === DataTransferType.Succeed) { const percent = ((event.consumedBytes / event.totalBytes) * 100).toFixed(2); console.log(`Data Transfer Succeed, ConsumerBytes/TotalBytes:${event.consumedBytes}/${event.totalBytes},${percent}%`); } else if (event.type === DataTransferType.Failed) { console.log('Data Transfer Failed'); } }, }); const fileStat = await fsp.stat(filePath); console.log('fileSize: ', fileStat.size); } catch (error) { handleError(error); } } main();
以下代码用于自定义断点续传下载回调函数。
// 导入 SDK, 当 TOS Node.JS SDK 版本小于 2.5.2 请把下方 TosClient 改成 TOS 导入 import { DataTransferType, DownloadEventType, TosClient, TosClientError, TosServerError } from '@volcengine/tos-sdk'; import fsp from 'fs/promises'; // 创建客户端 const client = new TosClient({ accessKeyId: process.env['TOS_ACCESS_KEY'], accessKeySecret: process.env['TOS_SECRET_KEY'], region: "Provide your region", // 填写 Bucket 所在地域。以华北2(北京)为例,则 "Provide your region" 填写为 cn-beijing。 endpoint: "Provide your endpoint", // 填写域名地址 }); function handleError(error) { if (error instanceof TosClientError) { console.log('Client Err Msg:', error.message); console.log('Client Err Stack:', error.stack); } else if (error instanceof TosServerError) { console.log('Request ID:', error.requestId); console.log('Response Status Code:', error.statusCode); console.log('Response Header:', error.headers); console.log('Response Err Code:', error.code); console.log('Response Err Msg:', error.message); } else { console.log('unexpected exception, message: ', error); } } async function main() { try { const bucketName = 'node-sdk-test-bucket'; // 目标桶 const objectName = 'example_dir/example.txt'; // 目标对象 // 下载的文件路径 const filePath = './example_dir/example.txt'; await client.downloadFile({ bucket: bucketName, key: objectName, // 下载的文件路径 filePath, // 下载时指定分片大小 10M partSize: 10 * 1024 * 1024, // 分片下载任务并发数量 taskNum: 5, // 指定断点续传临时文件路径 checkpoint: './example_dir/example.checkpoint', // 通过自定义方式设置回调函数查看下载进度 dataTransferStatusChange: (event) => { if (event.type === DataTransferType.Started) { console.log('Data Transfer Started'); } else if (event.type === DataTransferType.Rw) { const percent = ((event.consumedBytes / event.totalBytes) * 100).toFixed(2); console.log(`Once Read:${event.rwOnceBytes},ConsumerBytes/TotalBytes: ${event.consumedBytes}/${event.totalBytes},${percent}%`); } else if (event.type === DataTransferType.Succeed) { const percent = ((event.consumedBytes / event.totalBytes) * 100).toFixed(2); console.log(`Data Transfer Succeed, ConsumerBytes/TotalBytes:${event.consumedBytes}/${event.totalBytes},${percent}%`); } else if (event.type === DataTransferType.Failed) { console.log('Data Transfer Failed'); } }, // 事件监听回调 downloadEventChange: (event) => { switch (event.type) { case DownloadEventType.CreateTempFileSucceed: { console.log('Download File Create Temp File Success.'); break; } case DownloadEventType.CreateTempFileFailed: { console.log('Download File Create Temp File Fail.'); break; } case DownloadEventType.DownloadPartSucceed: { console.log(`Download ${event.bucket} ${event.key} part success, DownloadPartInfo:%o`, event.downloadPartInfo); break; } case DownloadEventType.DownloadPartAborted: { console.log(`Download ${event.bucket} ${event.key} part aborted.`); break; } case DownloadEventType.DownloadPartFailed: { console.log(`Download ${event.bucket} ${event.key} part fail, err:%o`, event.err); break; } case DownloadEventType.RenameTempFileSucceed: { console.log(`Download ${event.bucket} ${event.key} success.`); break; } case DownloadEventType.CreateTempFileFailed: { console.log(`Download ${event.bucket} ${event.key} fail, err:%o`, event.err); break; } } }, }); const fileStat = await fsp.stat(filePath); console.log('fileSize: ', fileStat.size); } catch (error) { handleError(error); } } main();
断点续传下载时可以通过客户端使用 rateLimiter
参数对下载数据所占用的带宽进行限制,代码如下所示。
// 导入 SDK, 当 TOS Node.JS SDK 版本小于 2.5.2 请把下方 TosClient 改成 TOS 导入 import { DataTransferType, createDefaultRateLimiter, TosClient, TosClientError, TosServerError } from '@volcengine/tos-sdk'; import fsp from 'fs/promises'; // 创建客户端 const client = new TosClient({ accessKeyId: process.env['TOS_ACCESS_KEY'], accessKeySecret: process.env['TOS_SECRET_KEY'], region: "Provide your region", // 填写 Bucket 所在地域。以华北2(北京)为例,则 "Provide your region" 填写为 cn-beijing。 endpoint: "Provide your endpoint", // 填写域名地址 }); function handleError(error) { if (error instanceof TosClientError) { console.log('Client Err Msg:', error.message); console.log('Client Err Stack:', error.stack); } else if (error instanceof TosServerError) { console.log('Request ID:', error.requestId); console.log('Response Status Code:', error.statusCode); console.log('Response Header:', error.headers); console.log('Response Err Code:', error.code); console.log('Response Err Msg:', error.message); } else { console.log('unexpected exception, message: ', error); } } async function main() { try { const bucketName = 'node-sdk-test-bucket'; // 目标桶 const objectName = 'example_dir/example.txt'; // 目标对象 // 下载的文件路径 const filePath = './example_dir/example.txt'; const rateLimit1M = 1024 * 1024; await client.downloadFile({ bucket: bucketName, key: objectName, // 下载的文件路径 filePath, // 下载时指定分片大小 10M partSize: 10 * 1024 * 1024, // 分片下载任务并发数量 taskNum: 5, // 指定断点续传临时文件路径 checkpoint: './example_dir/example.checkpoint', // 通过自定义方式设置回调函数查看下载进度 dataTransferStatusChange: (event) => { if (event.type === DataTransferType.Started) { console.log('Data Transfer Started'); } else if (event.type === DataTransferType.Rw) { const percent = ((event.consumedBytes / event.totalBytes) * 100).toFixed(2); console.log(`Once Read:${event.rwOnceBytes},ConsumerBytes/TotalBytes: ${event.consumedBytes}/${event.totalBytes},${percent}%`); } else if (event.type === DataTransferType.Succeed) { const percent = ((event.consumedBytes / event.totalBytes) * 100).toFixed(2); console.log(`Data Transfer Succeed, ConsumerBytes/TotalBytes:${event.consumedBytes}/${event.totalBytes},${percent}%`); } else if (event.type === DataTransferType.Failed) { console.log('Data Transfer Failed'); } }, // 下载对象并在客户端限制下载速度为 1M/s rateLimiter: createDefaultRateLimiter(rateLimit1M, rateLimit1M), }); const fileStat = await fsp.stat(filePath); console.log('fileSize: ', fileStat.size); } catch (error) { handleError(error); } } main();
以下代码用于在运行时取消正在执行的断点续传下载任务。
// 导入 SDK, 当 TOS Node.JS SDK 版本小于 2.5.2 请把下方 TosClient 改成 TOS 导入 import { DataTransferType, CancelToken, TosClient, TosClientError, TosServerError, isCancel } from '@volcengine/tos-sdk'; import fsp from 'fs/promises'; // 创建客户端 const client = new TosClient({ accessKeyId: process.env['TOS_ACCESS_KEY'], accessKeySecret: process.env['TOS_SECRET_KEY'], region: "Provide your region", // 填写 Bucket 所在地域。以华北2(北京)为例,则 "Provide your region" 填写为 cn-beijing。 endpoint: "Provide your endpoint", // 填写域名地址 }); function handleError(error) { if (error instanceof TosClientError) { console.log('Client Err Msg:', error.message); console.log('Client Err Stack:', error.stack); } else if (error instanceof TosServerError) { console.log('Request ID:', error.requestId); console.log('Response Status Code:', error.statusCode); console.log('Response Header:', error.headers); console.log('Response Err Code:', error.code); console.log('Response Err Msg:', error.message); } else { console.log('unexpected exception, message: ', error); } } async function main() { // checkpoint 文件路径 const checkpointFilePath = './example_dir/example.checkpoint'; try { const bucketName = 'node-sdk-test-bucket'; // 目标桶 const objectName = 'example_dir/example.txt'; // 目标对象 // 下载的文件路径 const filePath = './example_dir/example.txt'; try { // 生成 cancelTokenSource const cancelTokenSource = CancelToken.source(); // 1 秒后取消任务 setTimeout(() => { cancelTokenSource.cancel(); console.log('cancel downloadFile'); }, 1_000); await client.downloadFile({ bucket: bucketName, key: objectName, // 下载的文件路径 filePath, // 下载时指定分片大小 1M partSize: 1 * 1024 * 1024, // 分片下载任务并发数量 taskNum: 5, // 指定断点续传临时文件路径 checkpoint: './example_dir/example.checkpoint', // 通过自定义方式设置回调函数查看下载进度 dataTransferStatusChange: (event) => { if (event.type === DataTransferType.Started) { console.log('Data Transfer Started'); } else if (event.type === DataTransferType.Rw) { const percent = ((event.consumedBytes / event.totalBytes) * 100).toFixed(2); console.log(`Once Read:${event.rwOnceBytes},ConsumerBytes/TotalBytes: ${event.consumedBytes}/${event.totalBytes},${percent}%`); } else if (event.type === DataTransferType.Succeed) { const percent = ((event.consumedBytes / event.totalBytes) * 100).toFixed(2); console.log(`Data Transfer Succeed, ConsumerBytes/TotalBytes:${event.consumedBytes}/${event.totalBytes},${percent}%`); } else if (event.type === DataTransferType.Failed) { console.log('Data Transfer Failed'); } }, cancelToken: cancelTokenSource.token, }); } catch (error) { if (!isCancel(error)) { throw error; } const checkpointFileContent = await fsp.readFile(checkpointFilePath, 'utf-8'); console.log(`checkpoint file's content after cancel: `, checkpointFileContent); console.log('Continue to download'); // 可以通过 checkpoint File 断点续传下载 await client.downloadFile({ bucket: bucketName, key: objectName, // 下载的文件路径 filePath, // 下载时指定分片大小 1M partSize: 1 * 1024 * 1024, // 分片下载任务并发数量 taskNum: 5, // 指定断点续传临时文件路径 checkpoint: './example_dir/example.checkpoint', // 通过自定义方式设置回调函数查看下载进度 dataTransferStatusChange: (event) => { if (event.type === DataTransferType.Started) { console.log('Data Transfer Started'); } else if (event.type === DataTransferType.Rw) { const percent = ((event.consumedBytes / event.totalBytes) * 100).toFixed(2); console.log(`Once Read:${event.rwOnceBytes},ConsumerBytes/TotalBytes: ${event.consumedBytes}/${event.totalBytes},${percent}%`); } else if (event.type === DataTransferType.Succeed) { const percent = ((event.consumedBytes / event.totalBytes) * 100).toFixed(2); console.log(`Data Transfer Succeed, ConsumerBytes/TotalBytes:${event.consumedBytes}/${event.totalBytes},${percent}%`); } else if (event.type === DataTransferType.Failed) { console.log('Data Transfer Failed'); } }, }); const fileStat = await fsp.stat(filePath); console.log('fileSize: ', fileStat.size); } } catch (error) { handleError(error); } } main();