Pandas子类化中__init__重复执行问题的原因分析与解决求助

问题现象 #

• 短列名数据实例化子类并调用print输出时，__init__仅执行一次，符合预期；
• 长列名数据实例化子类并调用print输出时，父类AFrame的__init__会重复执行；
• 直接用AFrame实例化长列名数据并print时，__init__执行四次；
• 移除print调用后，所有场景下__init__仅执行一次。

复现代码 #

from pandas import DataFrame, Series


class ASeries(Series):
    @property
    def _constructor(self):
        return ASeries

    @property
    def _constructor_expanddim(self):
        return AFrame


class AFrame(DataFrame):
    a_count = 0
    thing = 'a'

    @property
    def _constructor(self):
        return AFrame

    @property
    def _constructor_sliced(self):
        return ASeries

    def __init__(self, data, *args, **kwargs):
        super(AFrame, self).__init__(data, *args, **kwargs)
        self.__class__.a_count += 1
        print('a', self.__class__.a_count)


class BSeries(Series):
    @property
    def _constructor(self):
        return BSeries

    @property
    def _constructor_expanddim(self):
        return BFrame


class BFrame(AFrame):
    b_count = 0
    thing = 'b'

    @property
    def _constructor(self):
        return BFrame

    @property
    def _constructor_sliced(self):
        return BSeries

    def __init__(self, data, *args, **kwargs):
        super(BFrame, self).__init__(data, *args, **kwargs)
        self.__class__.b_count += 1
        print('b', self.__class__.b_count)

测试场景与输出 #

测试场景1：短列名数据 #

b = BFrame(
    [
        {
            'a': 'a',
            'b': 'Something',
            'c': 1,
            'd': 'aa',
            'e': 'Somethings',
        },
        {
            'a': 'a',
            'b': 'Something else',
            'c': 1,
            'd': 'aa',
            'e': 'Somethings',
        },
    ],
)
print(b, type(b))

输出：

a 1
b 1
   a               b  c   d           e
0  a       Something  1  aa  Somethings
1  a  Something else  1  aa  Somethings <class '__main__.BFrame'>

测试场景2：长列名数据 #

b = BFrame(
    [
        {
            'aaaaaaaaaaaaaaaa': 'a',
            'bbbbbbbbbbbbbbbb': 'Something',
            'cccccccccccccccc': 1,
            'dddddddddddddddd': 'aa',
            'e': 'Somethings',
        },
        {
            'aaaaaaaaaaaaaaaa': 'a',
            'bbbbbbbbbbbbbbbb': 'Something else',
            'cccccccccccccccc': 1,
            'dddddddddddddddd': 'aa',
            'e': 'Somethings',
        },
    ],
)
print(b, type(b))

输出：

a 1
b 1
a 1
a 2
a 3
  aaaaaaaaaaaaaaaa bbbbbbbbbbbbbbbb  ...  dddddddddddddddd           e
0                a        Something  ...                aa  Somethings
1                a   Something else  ...                aa  Somethings

[2 rows x 5 columns] <class '__main__.BFrame'>

测试场景3：直接使用AFrame #

a = AFrame(
    [
        {
            'aaaaaaaaaaaaaaaa': 'a',
            'bbbbbbbbbbbbbbbb': 'Something',
            'cccccccccccccccc': 1,
            'dddddddddddddddd': 'aa',
            'e': 'Somethings',
        },
        {
            'aaaaaaaaaaaaaaaa': 'a',
            'bbbbbbbbbbbbbbbb': 'Something else',
            'cccccccccccccccc': 1,
            'dddddddddddddddd': 'aa',
            'e': 'Somethings',
        },
    ],
)
print(a, type(a))

输出：

a 1
a 2
a 3
a 4
  aaaaaaaaaaaaaaaa bbbbbbbbbbbbbbbb  ...  dddddddddddddddd           e
0                a        Something  ...                aa  Somethings
1                a   Something else  ...                aa  Somethings

[2 rows x 5 columns] <class '__main__.AFrame'>

根本原因 #

问题出在Pandas的打印渲染逻辑上：
当DataFrame的列名过长或列数过多时，Pandas为了生成友好的折叠式输出（用...省略中间列），会内部调用子类的_constructor方法创建临时的DataFrame对象，用于预览表头、内容片段等。而我们自定义的子类中，_constructor返回的是自身类，所以每次创建临时对象都会触发__init__方法执行，导致计数和打印重复出现。

短列名场景下，Pandas不需要折叠输出，直接渲染原对象，因此不会创建临时对象，__init__仅执行一次。移除print后，没有触发渲染逻辑，自然也不会有重复调用。

解决方法 #

方案1：修改_constructor，区分内部调用与用户实例化 #

通过在_constructor中传递标识参数，让__init__仅在用户主动实例化时执行自定义逻辑：

from pandas import DataFrame, Series


class ASeries(Series):
    @property
    def _constructor(self):
        return ASeries

    @property
    def _constructor_expanddim(self):
        return AFrame


class AFrame(DataFrame):
    a_count = 0
    thing = 'a'

    @property
    def _constructor(self):
        # 内部构造时传递internal=True参数
        def _constructor(data, *args, **kwargs):
            return AFrame(data, *args, internal=True, **kwargs)
        return _constructor

    @property
    def _constructor_sliced(self):
        return ASeries

    def __init__(self, data, *args, internal=False, **kwargs):
        super(AFrame, self).__init__(data, *args, **kwargs)
        # 仅在用户实例化时执行计数与打印
        if not internal:
            self.__class__.a_count += 1
            print('a', self.__class__.a_count)


class BSeries(Series):
    @property
    def _constructor(self):
        return BSeries

    @property
    def _constructor_expanddim(self):
        return BFrame


class BFrame(AFrame):
    b_count = 0
    thing = 'b'

    @property
    def _constructor(self):
        def _constructor(data, *args, **kwargs):
            return BFrame(data, *args, internal=True, **kwargs)
        return _constructor

    @property
    def _constructor_sliced(self):
        return BSeries

    def __init__(self, data, *args, internal=False, **kwargs):
        super(BFrame, self).__init__(data, *args, internal=internal, **kwargs)
        if not internal:
            self.__class__.b_count += 1
            print('b', self.__class__.b_count)

方案2：使用Pandas推荐的__finalize__方法替代__init__ #

Pandas官方推荐子类化时使用__finalize__而非直接重写__init__，该方法用于对象创建后复制元数据，可通过method参数判断是否为内部构造：

from pandas import DataFrame, Series


class ASeries(Series):
    @property
    def _constructor(self):
        return ASeries

    @property
    def _constructor_expanddim(self):
        return AFrame


class AFrame(DataFrame):
    a_count = 0
    thing = 'a'

    @property
    def _constructor(self):
        return AFrame

    @property
    def _constructor_sliced(self):
        return ASeries

    def __finalize__(self, other, method=None, **kwargs):
        super().__finalize__(other, method=method, **kwargs)
        # 非内部构造场景下执行计数逻辑
        if method != 'construct':
            self.__class__.a_count += 1
            print('a', self.__class__.a_count)
        return self


class BSeries(Series):
    @property
    def _constructor(self):
        return BSeries

    @property
    def _constructor_expanddim(self):
        return BFrame


class BFrame(AFrame):
    b_count = 0
    thing = 'b'

    @property
    def _constructor(self):
        return BFrame

    @property
    def _constructor_sliced(self):
        return BSeries

    def __finalize__(self, other, method=None, **kwargs):
        super().__finalize__(other, method=method, **kwargs)
        if method != 'construct':
            self.__class__.b_count += 1
            print('b', self.__class__.b_count)
        return self

两种方案都能避免打印时的__init__重复调用，推荐方案2，更符合Pandas子类化的设计规范。

内容的提问来源于stack exchange，提问作者Reg