音频分类脚本中ESC50Data类无len()方法报错排查

问题场景 #

在Jupyter Notebook中运行PyTorch音频分类脚本，创建DataLoader时触发TypeError: object of type 'ESC50Data' has no len()错误。

原代码如下：

from torch.utils.data import Dataset, DataLoader

class ESC50Data(Dataset):
      def __init__(self, base, df, in_col, out_col):
        self.df = df
        self.data = []
        self.labels = []
        self.c2i={}
        self.i2c={}
        self.categories = sorted(df[out_col].unique())
        for i, category in enumerate(self.categories):
            self.c2i[category]=i
            self.i2c[i]=category
        for ind in tqdm(range(len(df))):
            row = df.iloc[ind]
            file_path = os.path.join(base,row[in_col])
            self.data.append(spec_to_image(get_melspectrogram(file_path))[np.newaxis,...])
            self.labels.append(self.c2i[row['category']])
            def __len__(self):
                                    return len(self.data)
            def __getitem__(self, idx):
                                                return self.data[idx], self.labels[idx]

train_data = ESC50Data('audio', train, 'filename', 'category')
valid_data = ESC50Data('audio', valid, 'filename', 'category')

train_loader = DataLoader(train_data, batch_size=16, shuffle=True)
valid_loader = DataLoader(valid_data, batch_size=16, shuffle=True)

报错堆栈：

TypeError                                 Traceback (most recent call last)
Input In [47], in <cell line: 1>()
----> 1 train_loader = DataLoader(train_data, batch_size=16, shuffle=True)
      2 valid_loader = DataLoader(valid_data, batch_size=16, shuffle=True)

File ~/opt/anaconda3/lib/python3.9/site-packages/torch/utils/data/dataloader.py:353, in DataLoader.__init__(self, dataset, batch_size, shuffle, sampler, batch_sampler, num_workers, collate_fn, pin_memory, drop_last, timeout, worker_init_fn, multiprocessing_context, generator, prefetch_factor, persistent_workers, pin_memory_device)
    351 else:  # map-style
    352     if shuffle:
---> 353         sampler = RandomSampler(dataset, generator=generator)  # type: ignore[arg-type]
    354     else:
    355         sampler = SequentialSampler(dataset)  # type: ignore[arg-type]

File ~/opt/anaconda3/lib/python3.9/site-packages/torch/utils/data/sampler.py:106, in RandomSampler.__init__(self, data_source, replacement, num_samples, generator)
    102 if not isinstance(self.replacement, bool):
    103     raise TypeError("replacement should be a boolean value, but got "
    104                     "replacement={}".format(self.replacement))
---> 106 if not isinstance(self.num_samples, int) or self.num_samples <= 0:
    107     raise ValueError("num_samples should be a positive integer "
    108                      "value, but got num_samples={}".format(self.num_samples))

File ~/opt/anaconda3/lib/python3.9/site-packages/torch/utils/data/sampler.py:114, in RandomSampler.num_samples(self)
    110 @property
    111 def num_samples(self) -> int:
    112     # dataset size might change at runtime
    113     if self._num_samples is None:
---> 114         return len(self.data_source)
    115     return self._num_samples

TypeError: object of type 'ESC50Data' has no len()

错误原因 #

核心问题是缩进错误：__len__和__getitem__方法被错误定义在__init__方法内部的for循环中，导致这两个方法不是ESC50Data类的成员方法，而是for循环内的局部函数。PyTorch的Dataset子类要求必须实现__len__和__getitem__作为类的公开方法，否则DataLoader无法获取数据集长度，从而触发报错。

修正后的代码 #

将__len__和__getitem__的缩进调整到与__init__同级，作为类的成员方法：

from torch.utils.data import Dataset, DataLoader

class ESC50Data(Dataset):
    def __init__(self, base, df, in_col, out_col):
        self.df = df
        self.data = []
        self.labels = []
        self.c2i={}
        self.i2c={}
        self.categories = sorted(df[out_col].unique())
        for i, category in enumerate(self.categories):
            self.c2i[category]=i
            self.i2c[i]=category
        for ind in tqdm(range(len(df))):
            row = df.iloc[ind]
            file_path = os.path.join(base,row[in_col])
            self.data.append(spec_to_image(get_melspectrogram(file_path))[np.newaxis,...])
            self.labels.append(self.c2i[row['category']])
    
    # 调整缩进，作为类方法
    def __len__(self):
        return len(self.data)
    
    # 调整缩进，作为类方法
    def __getitem__(self, idx):
        return self.data[idx], self.labels[idx]

train_data = ESC50Data('audio', train, 'filename', 'category')
valid_data = ESC50Data('audio', valid, 'filename', 'category')

train_loader = DataLoader(train_data, batch_size=16, shuffle=True)
valid_loader = DataLoader(valid_data, batch_size=16, shuffle=True)

内容的提问来源于stack exchange，提问作者Adil Hydari