自定义Transformer的fit_transform报错：Series真值判断模糊求排查

我基于BaseEstimator和TransformerMixin实现了两个自定义数据转换器：FillOutAverage（按类别填充depth、height、width字段的缺失值）与DataCompute（处理old_price字段格式），构建Pipeline调用fit_transform时触发「truth value of a Series is ambiguous」错误，多次尝试无法定位问题，请求排查代码问题。

原代码如下：

import pandas as pd
import numpy as np
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.pipeline import Pipeline, make_pipeline
from sklearn.preprocessing import StandardScaler, LabelEncoder, ColumnTransformer

class FillOutAverage(BaseEstimator, TransformerMixin):
    def __init__(self):
        print("inside the filloutaverage")

    def fit(self, df,y=None):
        print("inside fillout fit method")
        return self


    def transform(self, df):
        print("got into fillout transform method")
        median_depth = df.groupby(['category'])['depth'].median().reset_index()
        median_depth.columns = ['category', 'MedianDepth']

        # 计算每个类别的height中位数
        median_height = df.groupby(['category'])['height'].median().reset_index()
        median_height.columns = ['category', 'MedianHeight']

        # 计算每个类别的width中位数
        median_width = df.groupby(['category'])['width'].median().reset_index()
        median_width.columns = ['category', 'MedianWidth']

        # 合并中位数数据
        median_size = pd.merge(pd.merge(median_depth,median_height,on='category'),median_width,on='category')

        df = pd.merge(df, median_size, on="category")
        df['depth'] = df['depth'].fillna(df["MedianDepth"])
        df["height"] = df["height"].fillna(df["MedianHeight"])
        df["width"] = df["width"].fillna(df["MedianWidth"])

        print("completed fillout out method")

        return df


class DataCompute(BaseEstimator, TransformerMixin):
    def __init__(self):
        print("inside datacomputation")

    def fit(self,df=None, y=None):
        print("the length of the data columns",len(df.columns))
        return self

    def transform(self, df=None):
        print("Got into datacompute transform method")

        if df["old_price"] == "No old price" or str(df["old_price"])[-4:] == "pack":
            df["old_price"] = "0"
            df["old_price"] = df["old_price"].strip()
        if str(df["old_price"])[:2] == "SR":
            df["old_price"] = str(df["old_price"]).replace("SR ","").strip()
        if "," in str(df["old_price"]):
            df["old_price"] = str(df["old_price"]).replace(",","").strip()
        else:
            df["old_price"] = str(df["old_price"]).strip()
        
        df["old_price"] = np.where("0", df["old_price"], int(df["price"]))
        
        print(df["old_price"].unique())
        print("completed datacompute out method")

        return df

# 假设df是已加载的数据集
# df = pd.read_csv("your_data.csv")

pipe1 = Pipeline([("fill_avg", FillOutAverage()), ("datacompute",DataCompute())])
cat_cols = df.select_dtypes(include='object')
num_cols = df.select_dtypes(exclude="object")
num_pipeline = Pipeline([("scaling", StandardScaler())])
cat_pipeline = Pipeline([("encoding", LabelEncoder())])
columns_transformer = ColumnTransformer(transformers= [("cat_cols",cat_pipeline, cat_cols),("num_cols",num_pipeline, num_cols)])
pipe2 = make_pipeline(pipe1, columns_transformer)
pipe2.fit_transform(df)

错误原因分析 #

• Series直接用于if条件判断：df["old_price"] == "No old price"返回布尔值Series，if语句无法处理多值布尔结果，直接触发「truth value of a Series is ambiguous」错误。
• np.where参数逻辑错误：np.where("0", ...)中第一个参数应为布尔条件，字符串"0"会被强制转为True，逻辑完全失效；int(df["price"])试图将整个Series转为整数，而非逐元素转换，引发类型错误。
• ColumnTransformer列参数错误：传入的cat_cols和num_cols是DataFrame，但ColumnTransformer需要列名列表，无法正确识别目标列。
• LabelEncoder不适用于多列：LabelEncoder仅支持单列数据编码，多列分类数据应使用OrdinalEncoder，否则会报错。

修复方案 #

1. 重构DataCompute.transform方法 #

使用向量化字符串操作处理old_price字段，避免逐行判断：

def transform(self, df):
    print("Got into datacompute transform method")
    df = df.copy()
    
    # 处理"No old price"或结尾为"pack"的情况
    mask = (df["old_price"] == "No old price") | df["old_price"].str.endswith("pack", na=False)
    df.loc[mask, "old_price"] = "0"
    
    # 移除前缀"SR "
    df["old_price"] = df["old_price"].str.replace(r'^SR ', '', regex=True)
    
    # 移除逗号并去除空格
    df["old_price"] = df["old_price"].str.replace(',', '').str.strip()
    
    # 处理old_price为0的情况，替换为price的整数值
    df["old_price"] = np.where(df["old_price"] == "0", df["price"].astype(int), df["old_price"].astype(int))
    
    print(df["old_price"].unique())
    print("completed datacompute out method")
    return df

2. 修正ColumnTransformer的列参数 #

提取列名列表传入：

cat_cols = df.select_dtypes(include='object').columns.tolist()
num_cols = df.select_dtypes(exclude='object').columns.tolist()

3. 替换LabelEncoder为OrdinalEncoder #

from sklearn.preprocessing import OrdinalEncoder

cat_pipeline = Pipeline([("encoding", OrdinalEncoder())])

4. 优化FillOutAverage的transform方法 #

合并groupby操作，减少重复代码：

def transform(self, df):
    print("got into fillout transform method")
    df = df.copy()
    
    # 一次性计算所有列的中位数
    median_size = df.groupby('category')[['depth', 'height', 'width']].median().reset_index()
    median_size.columns = ['category', 'MedianDepth', 'MedianHeight', 'MedianWidth']
    
    df = pd.merge(df, median_size, on="category")
    df['depth'] = df['depth'].fillna(df["MedianDepth"])
    df["height"] = df["height"].fillna(df["MedianHeight"])
    df["width"] = df["width"].fillna(df["MedianWidth"])
    
    # 移除临时中位数列
    df = df.drop(['MedianDepth', 'MedianHeight', 'MedianWidth'], axis=1)
    
    print("completed fillout out method")
    return df

完整修复后的代码 #

import pandas as pd
import numpy as np
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.pipeline import Pipeline, make_pipeline
from sklearn.preprocessing import StandardScaler, OrdinalEncoder, ColumnTransformer

class FillOutAverage(BaseEstimator, TransformerMixin):
    def __init__(self):
        print("inside the filloutaverage")

    def fit(self, df, y=None):
        print("inside fillout fit method")
        return self

    def transform(self, df):
        print("got into fillout transform method")
        df = df.copy()
        
        # 一次性计算类别维度中位数
        median_size = df.groupby('category')[['depth', 'height', 'width']].median().reset_index()
        median_size.columns = ['category', 'MedianDepth', 'MedianHeight', 'MedianWidth']
        
        df = pd.merge(df, median_size, on="category")
        df['depth'] = df['depth'].fillna(df["MedianDepth"])
        df["height"] = df["height"].fillna(df["MedianHeight"])
        df["width"] = df["width"].fillna(df["MedianWidth"])
        
        # 清理临时列
        df = df.drop(['MedianDepth', 'MedianHeight', 'MedianWidth'], axis=1)
        
        print("completed fillout out method")
        return df


class DataCompute(BaseEstimator, TransformerMixin):
    def __init__(self):
        print("inside datacomputation")

    def fit(self, df, y=None):
        print("the length of the data columns", len(df.columns))
        return self

    def transform(self, df):
        print("Got into datacompute transform method")
        df = df.copy()
        
        # 处理特殊值为0
        mask = (df["old_price"] == "No old price") | df["old_price"].str.endswith("pack", na=False)
        df.loc[mask, "old_price"] = "0"
        
        # 移除SR前缀
        df["old_price"] = df["old_price"].str.replace(r'^SR ', '', regex=True)
        
        # 清理格式
        df["old_price"] = df["old_price"].str.replace(',', '').str.strip()
        
        # 转换数值类型，0值替换为price的整数
        df["old_price"] = np.where(
            df["old_price"] == "0",
            df["price"].astype(int),
            df["old_price"].astype(int)
        )
        
        print(df["old_price"].unique())
        print("completed datacompute out method")
        return df

# 假设df是你的数据集
# df = pd.read_csv("your_dataset.csv")

pipe1 = Pipeline([("fill_avg", FillOutAverage()), ("datacompute", DataCompute())])
cat_cols = df.select_dtypes(include='object').columns.tolist()
num_cols = df.select_dtypes(exclude='object').columns.tolist()

num_pipeline = Pipeline([("scaling", StandardScaler())])
cat_pipeline = Pipeline([("encoding", OrdinalEncoder())])

columns_transformer = ColumnTransformer(
    transformers=[
        ("cat_cols", cat_pipeline, cat_cols),
        ("num_cols", num_pipeline, num_cols)
    ]
)

pipe2 = make_pipeline(pipe1, columns_transformer)
pipe2.fit_transform(df)

内容的提问来源于stack exchange，提问作者Boggu Maheshbabu