Docker构建失败：Cython尝试将编译文件复制到错误路径

我使用Cython 3.0a7在Docker镜像中构建分发库文件，近期Docker构建流程失败，原因是Cython尝试将一个.so文件复制到不存在的路径。

setup.py 代码 #

# coding: utf-8
import os
import fnmatch
import sysconfig

from setuptools import setup, find_packages
from setuptools.command.build_py import build_py as _build_py

from Cython.Build import cythonize


EXCLUDE_FILES = [
    # "pkgname/utils/server/server.py",
    # "pkgname/bleep.py",
]

for root, dirs, files in os.walk("pkgname/"):
    for filename in files:
        if filename == "__init__.py":
            EXCLUDE_FILES.append(os.path.join(root, filename))

def get_ext_paths(root_dir, exclude_files):
    """get filepaths for compilation"""
    paths = []

    for root, dirs, files in os.walk(root_dir):
        for filename in files:
            if os.path.splitext(filename)[1] != '.py':
                continue

            file_path = os.path.join(root, filename)
            if file_path in exclude_files:
                continue

            paths.append(file_path)
    return paths


# noinspection PyPep8Naming
class build_py(_build_py):

    def find_package_modules(self, package, package_dir):
        ext_suffix = sysconfig.get_config_var('EXT_SUFFIX')
        modules = super().find_package_modules(package, package_dir)
        filtered_modules = []
        non_check_paths = set()#{'pkgname/__main__.py', 'pkgname/__init__.py', 'pkgname/bleep.py'}
        for (pkg, mod, filepath) in modules:
            if filepath not in non_check_paths and os.path.exists(filepath.replace('.py', ext_suffix)):
                continue
            filtered_modules.append((pkg, mod, filepath, ))
        return filtered_modules


setup(
    name='pkgname',
    version='0.1.0',
    packages=find_packages(),
    ext_modules=cythonize(
        get_ext_paths('pkgname', EXCLUDE_FILES),
        compiler_directives={'language_level': 3}
    ),
    cmdclass={
        'build_py': build_py
    },
    include_package_data=True
)

错误日志 #

#55 175.8 building 'pkgname.core.trainers.utils' extension
#55 175.8 creating build/temp.linux-x86_64-3.8/pkgname/core/trainers
#55 175.8 gcc -pthread -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/usr/local/include/python3.8 -c pkgname/core/trainers/utils.c -o build/temp.linux-x86_64-3.8/pkgname/core/trainers/utils.o
#55 176.6 creating build/lib.linux-x86_64-3.8/pkgname/core/trainers
#55 176.6 gcc -pthread -shared build/temp.linux-x86_64-3.8/pkgname/core/trainers/utils.o -L/usr/local/lib -o build/lib.linux-x86_64-3.8/pkgname/core/trainers/utils.cpython-38-x86_64-linux-gnu.so
...
#55 203.3 copying build/lib.linux-x86_64-3.8/pkgname/bleep.cpython-38-x86_64-linux-gnu.so -> pkgname
#55 203.3 copying build/lib.linux-x86_64-3.8/pkgname/download.cpython-38-x86_64-linux-gnu.so -> pkgname
#55 203.3 copying build/lib.linux-x86_64-3.8/pkgname/__main__.cpython-38-x86_64-linux-gnu.so -> pkgname
#55 203.3 copying build/lib.linux-x86_64-3.8/pkgname/_meta.cpython-38-x86_64-linux-gnu.so -> pkgname
#55 203.3 copying build/lib.linux-x86_64-3.8/pkgname/utils/alice/request_parameters.cpython-38-x86_64-linux-gnu.so -> pkgname/utils/alice
#55 203.3 copying build/lib.linux-x86_64-3.8/pkgname/utils/alice/server.cpython-38-x86_64-linux-gnu.so -> pkgname/utils/alice
...
#55 203.3 copying build/lib.linux-x86_64-3.8/abcd/abcd_lib.cpython-38-x86_64-linux-gnu.so -> abcd    ### 目标路径应为pkgname/somepath/abcd
#55 203.3 error: could not create 'abcd/abcd_lib.cpython-38-x86_64-linux-gnu.so': No such file or directory
------
process "/bin/sh -c python setup.py build_ext --inplace" did not complete successfully: exit code: 1

解决方案 #

问题核心是abcd/abcd_lib.py实际位于pkgname/somepath/abcd/目录下，但Cython错误地将其识别为根目录下的独立包，导致复制路径偏差。以下是具体修复步骤：

1. 修复包识别问题 #

find_packages()默认仅识别包含__init__.py的目录为合法包。如果pkgname/somepath/abcd/下缺少该文件，会被排除在包结构外，导致Cython解析路径错误。

• 在pkgname/somepath/abcd/目录中添加空的__init__.py文件，让find_packages()正确识别它为pkgname.somepath.abcd子包。

2. 限制路径遍历范围 #

当前get_ext_paths()函数可能因目录软链接或结构异常跳出pkgname/范围，需添加路径校验确保只处理目标目录下的文件：

def get_ext_paths(root_dir, exclude_files):
    """get filepaths for compilation"""
    paths = []
    root_dir = os.path.abspath(root_dir)
    
    for root, dirs, files in os.walk(root_dir):
        # 确保只遍历root_dir下的子目录
        if not os.path.abspath(root).startswith(root_dir):
            continue
        
        for filename in files:
            if os.path.splitext(filename)[1] != '.py':
                continue
            
            file_path = os.path.join(root, filename)
            if file_path in exclude_files:
                continue
            
            paths.append(file_path)
    return paths

3. 明确包根目录 #

在setup()函数中指定package_dir参数，消除路径解析歧义：

setup(
    name='pkgname',
    version='0.1.0',
    packages=find_packages(),
    package_dir={'': '.'},  # 明确包根目录为当前工作目录
    ext_modules=cythonize(
        get_ext_paths('pkgname', EXCLUDE_FILES),
        compiler_directives={'language_level': 3}
    ),
    cmdclass={
        'build_py': build_py
    },
    include_package_data=True
)

4. 手动指定模块路径（备选） #

如果上述方法无效，可手动指定abcd_lib.py的正确路径，避免Cython错误解析：

• 将pkgname/somepath/abcd/abcd_lib.py加入EXCLUDE_FILES，然后手动添加到ext_modules列表：

ext_modules=cythonize(
    get_ext_paths('pkgname', EXCLUDE_FILES) + ['pkgname/somepath/abcd/abcd_lib.py'],
    compiler_directives={'language_level': 3}
)

内容的提问来源于stack exchange，提问作者icedwater