Mystic diffev2并行报cannot pickle 'PyCapsule' object错误咨询

针对一款第三方软件开展输入参数优化工作：该软件先通过多个独立可执行程序生成系列输入文件，再将其合并为主执行程序使用的单份输入文件；本次优化目标为钢垫下方弹簧的二维排布位置，约束条件为弹簧之间无重叠、所有弹簧始终位于钢垫覆盖范围内，相关优化逻辑与待优化目标函数均封装在自定义类中。

# This function will optimize the position of the current springs in a thrust bearing using GENMAT using mystic
def SpringPositionAngGAPmystic_Opt_Par(self, InitialSpringList):
    # Import local packages
    import datetime
    from random import randint, random
    # Import contrains, penalties, and solvers
    from mystic.symbolic import generate_constraint, generate_solvers, solve
    from mystic.symbolic import generate_penalty, generate_conditions
    from mystic.solvers import DifferentialEvolutionSolver2,diffev2
    # if available, use a pathos worker pool
    try:
        from pathos.multiprocessing import ProcessingPool
        from multiprocessing import current_process
    except ImportError:
        from mystic.pools import SerialPool as ProcessingPool
        print('Performing Serial Analysis')
    # tools
    from mystic.termination import NormalizedChangeOverGeneration as NCOG
    from mystic.monitors import VerboseMonitor

    # This function will read in and run the spring executable with spring replacement
    def SingleSpringPositionRun(x0):
        # Declare the global variables
        caseID = str(datetime.datetime.now()).replace('.','p').replace(r' ', '_').replace(':', 'c').replace('-', 'd')
        caseID = caseID + '_' + str(current_process().pid)
        # Create temporary work directory
        WorkDir = WorkDir_glob + '\\' + caseID + '\\'
        if not os.path.exists(WorkDir):
            os.makedirs(WorkDir)
        # Record the spring locations
        AppendData(self.DatFile[:-4]+'SpLoc.txt', [x0])
        # Calculate the number of springs
        NumSprings = int(len(x0)/2)
        # Unpack the spring position inputs
        SpringArray = [[x0[i], x0[i + NumSprings]] for i in range(0, NumSprings)]
        # Create spring class
        SpringIns = Spring(SpringXLIn_glob, GENMATDir_glob, WorkDir, SpringRunFile_glob, SpringRunInputFile_glob)
        # Read in the spring geometry
        SpringIns.readSpringExcel()
        # Replace the old springs with the input springs
        SpringIns.replaceSprings(SpringArray)
        # Create the spring parameters
        SpringIns.createSpringParams()
        # Run the spring executable
        SpringIns.RunSpringEXE()
        # Create general class and run it
        GeneralIns = General(GeneralXLIn_glob, GENMATDir_glob, WorkDir, GeneralRunFile_glob, GeneralRunInputFile_glob)
        GeneralIns.FullGENERALRun()
        (NumLoadCases, _, _, _, _) = GeneralIns.getLoads()
        # Create the gmdata file
        self.CreateGMDATAfile(WorkDir)
        # If a trailing edge recess is included
        if (self.TERecessFlag == 1):
            # Extract the number of springs
            self.NumSprings = SpringIns.GetNumSprings()
            # Extract Pad Geometry information
            [self.brgOD, self.brgID, self.PadAngle, self.GSpringDia] = SpringIns.GetPadSpringProp()
            # Extract mesh ingormation
            (self.OilFilmGridRadial, self.OilFilmGridCircum, self.GridThruFilm,
            self.GridThruPad) = GeneralIns.getMeshData()
            # Add the trailing edge recess
            self.AddGeneralTERecess(self.TERecessInfo)
        # Save a backup of the gmdata file
        self.BackupGMDATA(caseID)
        # Run GENMAT
        self.RunGENMAT(WorkDir)
        # Post process the results
        [success, MinFilmThick] = PostProcessGENMATSpringPos(caseID, WorkDir)
        # If a read error occurs marke the case a failure
        if not success:
            MinFilmThick = -mm.FailValue1
        # Remove the remporary working directory
        shutil.rmtree(WorkDir)

        return 1000.0-MinFilmThick

    # Initialize the data file
    self.InitializeDataFile()
    # Initialize the spring location file
    InitializeFile(self.DatFile[:-4]+'SpLoc.txt', 'Spring Locations [x0, x1,..., xn, y0, y1,..., yn]\n')
    # Define global variables for creation based on GENMAT class
    SpringXLIn_glob = self.SpringXLIn
    GENMATDir_glob = self.GENMATDir
    WorkDir_glob = self.workDir
    SpringRunFile_glob = self.SpringRunFile
    SpringRunInputFile_glob = self.SpringRunInputFile
    GeneralXLIn_glob = self.GenXLIn
    GeneralRunFile_glob = self.GeneralRunFile
    GeneralRunInputFile_glob = self.GeneralRunInputFile
    # Unpack initial spring list
    initSprings = [float(InitialSpringList[i][j])  for j in range(0, 2) for i in range(0, len(InitialSpringList))]
    # Define derived variables used in the optimization
    self.innerRad = self.brgID/2
    self.outerRad = self.brgOD/2
    self.innerCord = 2*(self.brgID/2)*mm.sind(self.PadAngle/2)
    self.RadInSq = self.innerRad**2
    self.RadOutSq = self.outerRad**2
    # Define constants for nonlinear edge constraints
    self.edgeConst1 = (self.PadAngle/2+90)*np.pi/180
    self.edgeConst2 = (90-self.PadAngle/2)*np.pi/180
    # Initialize constraint array
    constArray = []
    # Add the nonlinear constraint for all the spring-spring relationships
    constArray = constraintsSp2Sp(constArray, len(InitialSpringList))
    # Add the nonlinear constraint for all the spring-edge relationships
    constArray = constraintsSp2Edge(constArray, len(InitialSpringList))
    # Convert the constraints string array to constraints
    constStr='\n'
    for constraint in constArray:
        constStr = constStr + constraint + '\n'
    pf = generate_penalty(generate_conditions(constStr), k=1e12)
    # dimensional information
    #from mystic.tools import random_seed
    #random_seed(123)
    ndim = len(initSprings)
    nbins = 8 #[2,1,2,1,2,1,2,1,1]
    # configure monitor
    stepmon = VerboseMonitor(10)
    # Run the optimization
    pool = ProcessingPool(nodes=4)
    #springList = []
    #for i in range(0, 4):
    #    springList.append([])
    #    for j in range(0, len(initSprings)):
    #         springList[i].append(initSprings[j] + (random() - 0.5))
    #print(pool.map(SingleSpringPositionRun, springList))
    res = diffev2(SingleSpringPositionRun, x0=initSprings, penalty=pf, map=pool.map, gtol = 20,
                  intermon=stepmon, npop = 10, disp=True, full_output=True)
    print(res[0]+1000.0)
    return

调用mystic库的diffev2差分进化求解器、结合pathos多进程池开展并行优化时，程序抛出如下报错：
cannot pickle 'PyCapsule' object
但将代码中注释的直接调用pool.map并行执行目标函数的测试代码取消注释后，并行逻辑可正常运行，无同类序列化报错。需要明确两种并行调用方式的底层差异，以及仅diffev2求解器触发pickle序列化错误的原因。

两种调用方式的核心差异是多进程序列化的对象范围完全不同：

• 直接手动调用pool.map(SingleSpringPositionRun, springList)时，pathos需要序列化传输给子进程的内容只有两部分：嵌套定义的目标函数SingleSpringPositionRun、传入的候选参数列表springList。这部分闭包捕获的上下文不包含mystic生成的求解相关对象，只要自定义类实例self的普通属性可被dill序列化，就能正常运行，不会触碰到不可序列化的底层对象。
• 传入map=pool.map调用diffev2时，mystic内部在分发每一代种群的并行评估任务时，会把全量求解上下文打包序列化传给子进程，不止目标函数和候选解参数：包括传入的惩罚函数对象pf、迭代监控器stepmon、求解器终止条件对象、闭包连带捕获的外层类实例self的所有绑定属性，都会被纳入序列化范围。

PyCapsule是Python为C扩展模块暴露的底层API对象，本身不支持pickle序列化，报错触发的核心原因是全量序列化时纳入了带这类对象的资源：

• 一类来源是mystic组件：通过generate_penalty生成的惩罚函数内部持有符号计算生成的C层表达式对象，VerboseMonitor持有标准输出相关的C层IO句柄，这些对象在手动测试pool.map时完全没有进入序列化流程，自然不会报错。
• 另一类来源是自定义类实例：优化类实例self上如果绑定了第三方软件接口的C扩展对象、未释放的文件句柄、C层内存指针，手动测试时dill可以跳过闭包未直接引用的属性，但diffev2打包全量上下文时会把这些不可序列化的PyCapsule对象一并拉取，最终触发报错。

本质上手动测试的并行逻辑和diffev2内部调用的并行逻辑没有区别，只是前者序列化的对象集合更小，没有覆盖到不可序列化的资源而已。

内容的提问来源于stack exchange，提问作者Michael Branagan