保存Pandas DataFrame至Parquet文件时遇ValueError问题求助

将Pandas DataFrame保存为Parquet文件时遇到报错，使用的代码如下：

import pandas as pd
import yfinance as yf

start_date = "2022-08-06"
end_date = "2024-08-05"

ticker = 'RELIANCE.NS'
data = yf.download(tickers=ticker, start=start_date, end=end_date, interval="1h")

data.reset_index(inplace=True)

data['Date'] = data['Datetime'].dt.date
data['Time'] = data['Datetime'].dt.time

data.to_parquet('./RELIANCE.parquet')

运行后报错：

ValueError: Can't infer object conversion type: 0

详细错误堆栈信息：

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Cell In[1], line 15
     12 data['Date'] = data['Datetime'].dt.date
     13 data['Time'] = data['Datetime'].dt.time
---> 15 data.to_parquet('./RELIANCE.parquet')

File ~/python_venv/lib/python3.10/site-packages/pandas/util/_decorators.py:333, in deprecate_nonkeyword_arguments.<locals>.decorate.<locals>.wrapper(*args, **kwargs)
    327 if len(args) > num_allow_args:
    328     warnings.warn(
    329         msg.format(arguments=_format_argument_list(allow_args)),
    330         FutureWarning,
    331         stacklevel=find_stack_level(),
    332     )
--> 333 return func(*args, **kwargs)

File ~/python_venv/lib/python3.10/site-packages/pandas/core/frame.py:3113, in DataFrame.to_parquet(self, path, engine, compression, index, partition_cols, storage_options, **kwargs)
   3032 """
   3033 Write a DataFrame to the binary parquet format.
   3034 
   (...)
   3109 >>> content = f.read()
   3110 """
   3111 from pandas.io.parquet import to_parquet
-> 3113 return to_parquet(
   3114     self,
   3115     path,
   3116     engine,
   3117     compression=compression,
   3118     index=index,
   3119     partition_cols=partition_cols,
   3120     storage_options=storage_options,
   3121     **kwargs,
   3122 )

File ~/python_venv/lib/python3.10/site-packages/pandas/io/parquet.py:480, in to_parquet(df, path, engine, compression, index, storage_options, partition_cols, filesystem, **kwargs)
    476 impl = get_engine(engine)
    478 path_or_buf: FilePath | WriteBuffer[bytes] = io.BytesIO() if path is None else path
--> 480 impl.write(
    481     df,
    482     path_or_buf,
    483     compression=compression,
    484     index=index,
    485     partition_cols=partition_cols,
    486     storage_options=storage_options,
    487     filesystem=filesystem,
    488     **kwargs,
    489 )
    491 if path is None:
    492     assert isinstance(path_or_buf, io.BytesIO)

File ~/python_venv/lib/python3.10/site-packages/pandas/io/parquet.py:349, in FastParquetImpl.write(self, df, path, compression, index, partition_cols, storage_options, filesystem, **kwargs)
    344     raise ValueError(
    345         "storage_options passed with file object or non-fsspec file path"
    346     )
    348 with catch_warnings(record=True):
--> 349     self.api.write(
    350         path,
    351         df,
    352         compression=compression,
    353         write_index=index,
    354         partition_on=partition_cols,
    355         **kwargs,
    356     )

File ~/python_venv/lib/python3.10/site-packages/fastparquet/writer.py:1304, in write(filename, data, row_group_offsets, compression, file_scheme, open_with, mkdirs, has_nulls, write_index, partition_on, fixed_text, append, object_encoding, times, custom_metadata, stats)
   1301 check_column_names(data.columns, partition_on, fixed_text,
   1302                    object_encoding, has_nulls)
   1303 ignore = partition_on if file_scheme != 'simple' else []
-> 1304 fmd = make_metadata(data, has_nulls=has_nulls, ignore_columns=ignore,
   1305                     fixed_text=fixed_text,
   1306                     object_encoding=object_encoding,
   1307                     times=times, index_cols=index_cols,
   1308                     partition_cols=partition_on, cols_dtype=cols_dtype)
   1309 if custom_metadata:
   1310     kvm = fmd.key_value_metadata or []

File ~/python_venv/lib/python3.10/site-packages/fastparquet/writer.py:904, in make_metadata(data, has_nulls, ignore_columns, fixed_text, object_encoding, times, index_cols, partition_cols, cols_dtype)
    902     se.name = column
    903 else:
--> 904     se, type = find_type(data[column], fixed_text=fixed,
    905                          object_encoding=oencoding, times=times,
    906                          is_index=is_index)
    907 col_has_nulls = has_nulls
    908 if has_nulls is None:

File ~/python_venv/lib/python3.10/site-packages/fastparquet/writer.py:122, in find_type(data, fixed_text, object_encoding, times, is_index)
    120 elif dtype == "O":
    121     if object_encoding == 'infer':
--> 122         object_encoding = infer_object_encoding(data)
    124     if object_encoding == 'utf8':
    125         type, converted_type, width = (parquet_thrift.Type.BYTE_ARRAY,
    126                                        parquet_thrift.ConvertedType.UTF8,
    127                                        None)

File ~/python_venv/lib/python3.10/site-packages/fastparquet/writer.py:357, in infer_object_encoding(data)
    355     s += 1
    356 else:
--> 357     raise ValueError("Can't infer object conversion type: %s" % data)
    358 if s > 10:
    359     break

ValueError: Can't infer object conversion type: 0       2022-08-08
1       2022-08-08
2       2022-08-08
3       2022-08-08
4       2022-08-08
           ...    
3398    2024-08-02
3399    2024-08-02
3400    2024-08-02
3401    2024-08-02
3402    2024-08-02
Name: Date, Length: 3403, dtype: object

报错原因是生成的Date和Time列是Python原生的date和time对象类型，这类不属于Parquet标准支持的数据类型，fastparquet引擎无法自动推断转换格式，因此抛出错误。以下是三种修复方式：

方式一：将Date和Time列转为字符串类型 #

把原生date/time对象转换为字符串，Parquet可以完美处理字符串类型：

import pandas as pd
import yfinance as yf

start_date = "2022-08-06"
end_date = "2024-08-05"

ticker = 'RELIANCE.NS'
data = yf.download(tickers=ticker, start=start_date, end=end_date, interval="1h")

data.reset_index(inplace=True)

# 转换为字符串格式
data['Date'] = data['Datetime'].dt.date.astype(str)
data['Time'] = data['Datetime'].dt.time.astype(str)

data.to_parquet('./RELIANCE.parquet')

方式二：保留原Datetime列，不拆分日期时间 #

如果不需要单独的Date和Time列，直接保留原始的Datetime列即可——它是pandas的datetime64类型，Parquet完全支持：

import pandas as pd
import yfinance as yf

start_date = "2022-08-06"
end_date = "2024-08-05"

ticker = 'RELIANCE.NS'
data = yf.download(tickers=ticker, start=start_date, end=end_date, interval="1h")

data.reset_index(inplace=True)

# 移除拆分Date和Time的代码
data.to_parquet('./RELIANCE.parquet')

方式三：切换到pyarrow引擎 #

如果必须保留原生date/time对象，可以切换到pyarrow引擎，它对Python原生日期时间类型的支持更友好：

import pandas as pd
import yfinance as yf

start_date = "2022-08-06"
end_date = "2024-08-05"

ticker = 'RELIANCE.NS'
data = yf.download(tickers=ticker, start=start_date, end=end_date, interval="1h")

data.reset_index(inplace=True)

data['Date'] = data['Datetime'].dt.date
data['Time'] = data['Datetime'].dt.time

# 指定使用pyarrow引擎
data.to_parquet('./RELIANCE.parquet', engine='pyarrow')

注意：若未安装pyarrow，需先执行pip install pyarrow完成安装。

内容的提问来源于stack exchange，提问作者pkj