S3-查找格式不正确的parquet文件

1.使用Python模块pyarrow和boto3连接S3 Bucket, 并下载一个测试文件

import boto3
import pyarrow.parquet as pq
from io import BytesIO

s3client = boto3.client('s3')
s3resource = boto3.resource('s3')

bucket_name = "bucket_name"
object_key = "parquet_folder/test_file.parquet"
obj = s3resource.Object(bucket_name=bucket_name, key=object_key).get()
memoryfile = BytesIO(obj['Body'].read())

data = pq.ParquetFile(memoryfile).read()
print(data)

2.检测Parquet文件是否可以读取

def is_parquet_file_valid(parquet_file):
    try:
        pq.ParquetFile(parquet_file).read()
        return True
    except:
        return False

3.列出S3 Bucket中所有的Parquet文件，并将格式不正确的文件标记出来

def list_all_parquet_files(bucket_name):
    s3 = boto3.client('s3')
    incomplete_files = []
    continuation_token = None

    while True:
        list_objects_response = s3.list_objects_v2(
            Bucket=bucket_name,
            ContinuationToken=continuation_token,
        )
        for content in list_objects_response.get("Contents", []):
            if content['Key'].endswith('.parquet') and not is_parquet_file_valid(s3.get_object(Bucket=bucket_name,
                                                                                               Key=content['Key'])['Body']):
                incomplete_files.append(content['Key'])

        if not list_objects_response.get("IsTruncated"):
            break
        continuation_token = list_objects_response.get("NextContinuationToken")

    return incomplete_files

4.执行以上代码后，输出不可读取Parquet文件的文件名列表