Python多进程运行两函数并终止其一遇AttributeError问题

问题描述 #

我搭建了一套运行Blender渲染任务并测量GPU负载的环境，使用multiprocessing启动两个独立函数：一个运行Blender渲染，另一个用nvidia-smi监控GPU。我想在Blender渲染结束后终止GPU监控进程，于是把subprocess.Popen()返回的对象存在全局变量里，但执行时出现错误：

import os
import multiprocessing
import subprocess

nvidia_cmd = None

blender_cmd_to_run = '/home/mickey/dev/blender-4.2.1-linux-x64/blender -b /home/mickey/dev/elastic_shared_memory/benchmark_scenes/bmw27/bmw27.blend -f 10 -- --cycles-device CUDA'
nvidia_cmd_to_run = 'nvidia-smi --query-gpu=gpu_bus_id,memory.used --format=csv -l 1'

def run_blender():
    blender_cmd = subprocess.Popen(blender_cmd_to_run, shell=True, stdout=subprocess.PIPE)
    for line in blender_cmd.stdout:
        print(line.decode().strip())
    nvidia_cmd.terminate()

def run_nvidia():
    global nvidia_cmd
    nvidia_cmd = subprocess.Popen(nvidia_cmd_to_run, shell=True, stdout=subprocess.PIPE)
    for line in nvidia_cmd.stdout:
        print(line.decode().strip())


if __name__ == '__main__':
    blender_process = multiprocessing.Process(target=run_blender)
    nvidia_process = multiprocessing.Process(target=run_nvidia)

    nvidia_process.start()
    blender_process.start()

错误信息：

AttributeError: 'NoneType' object has no attribute 'terminate'

问题原因 #

Python的multiprocessing创建的子进程拥有独立的内存空间：

• 主进程初始化nvidia_cmd = None后，每个子进程都会复制这个初始状态到自己的内存中。
• run_nvidia函数在自己的子进程里修改了nvidia_cmd，但这个修改只存在于该子进程的内存中，run_blender所在的子进程里的nvidia_cmd仍然是初始的None。
• 当Blender渲染结束后调用nvidia_cmd.terminate()时，自然会触发NoneType的属性错误。

解决方案 #

方案1：主进程统一管理终止逻辑（最简单） #

让主进程等待Blender进程结束，然后主动终止GPU监控进程，无需跨进程共享变量：

import os
import multiprocessing
import subprocess

blender_cmd_to_run = '/home/mickey/dev/blender-4.2.1-linux-x64/blender -b /home/mickey/dev/elastic_shared_memory/benchmark_scenes/bmw27/bmw27.blend -f 10 -- --cycles-device CUDA'
nvidia_cmd_to_run = 'nvidia-smi --query-gpu=gpu_bus_id,memory.used --format=csv -l 1'

def run_blender():
    blender_cmd = subprocess.Popen(blender_cmd_to_run, shell=True, stdout=subprocess.PIPE)
    for line in blender_cmd.stdout:
        print(line.decode().strip())

def run_nvidia():
    nvidia_cmd = subprocess.Popen(nvidia_cmd_to_run, shell=True, stdout=subprocess.PIPE)
    for line in nvidia_cmd.stdout:
        print(line.decode().strip())

if __name__ == '__main__':
    blender_process = multiprocessing.Process(target=run_blender)
    nvidia_process = multiprocessing.Process(target=run_nvidia)

    nvidia_process.start()
    blender_process.start()
    
    # 等待Blender渲染完成
    blender_process.join()
    # 终止GPU监控进程
    nvidia_process.terminate()
    nvidia_process.join()

方案2：用multiprocessing.Manager共享进程PID #

通过Manager创建跨进程共享的字典，存储GPU监控进程的PID，Blender进程结束后通过PID终止该进程：

import os
import multiprocessing
import subprocess

blender_cmd_to_run = '/home/mickey/dev/blender-4.2.1-linux-x64/blender -b /home/mickey/dev/elastic_shared_memory/benchmark_scenes/bmw27/bmw27.blend -f 10 -- --cycles-device CUDA'
nvidia_cmd_to_run = 'nvidia-smi --query-gpu=gpu_bus_id,memory.used --format=csv -l 1'

def run_blender(shared_dict):
    blender_cmd = subprocess.Popen(blender_cmd_to_run, shell=True, stdout=subprocess.PIPE)
    for line in blender_cmd.stdout:
        print(line.decode().strip())
    # 通过PID终止GPU监控进程
    if 'nvidia_pid' in shared_dict:
        os.kill(shared_dict['nvidia_pid'], 15)  # 发送SIGTERM信号

def run_nvidia(shared_dict):
    nvidia_cmd = subprocess.Popen(nvidia_cmd_to_run, shell=True, stdout=subprocess.PIPE)
    shared_dict['nvidia_pid'] = nvidia_cmd.pid
    for line in nvidia_cmd.stdout:
        print(line.decode().strip())

if __name__ == '__main__':
    with multiprocessing.Manager() as manager:
        shared_dict = manager.dict()
        blender_process = multiprocessing.Process(target=run_blender, args=(shared_dict,))
        nvidia_process = multiprocessing.Process(target=run_nvidia, args=(shared_dict,))

        nvidia_process.start()
        blender_process.start()
        blender_process.join()
        nvidia_process.join()

方案3：用multiprocessing.Pipe传递终止信号 #

通过管道在两个子进程间传递终止指令，GPU监控进程监听管道信号，收到后自行终止：

import os
import multiprocessing
import subprocess
import select

blender_cmd_to_run = '/home/mickey/dev/blender-4.2.1-linux-x64/blender -b /home/mickey/dev/elastic_shared_memory/benchmark_scenes/bmw27/bmw27.blend -f 10 -- --cycles-device CUDA'
nvidia_cmd_to_run = 'nvidia-smi --query-gpu=gpu_bus_id,memory.used --format=csv -l 1'

def run_blender(conn):
    blender_cmd = subprocess.Popen(blender_cmd_to_run, shell=True, stdout=subprocess.PIPE)
    for line in blender_cmd.stdout:
        print(line.decode().strip())
    # 发送终止信号
    conn.send('terminate')
    conn.close()

def run_nvidia(conn):
    nvidia_cmd = subprocess.Popen(nvidia_cmd_to_run, shell=True, stdout=subprocess.PIPE)
    while True:
        # 同时监听stdout输出和管道信号
        ready, _, _ = select.select([nvidia_cmd.stdout, conn], [], [], 0.1)
        if nvidia_cmd.stdout in ready:
            line = nvidia_cmd.stdout.readline()
            if not line:
                break
            print(line.decode().strip())
        if conn in ready:
            signal = conn.recv()
            if signal == 'terminate':
                nvidia_cmd.terminate()
                break

if __name__ == '__main__':
    parent_conn, child_conn = multiprocessing.Pipe()
    blender_process = multiprocessing.Process(target=run_blender, args=(parent_conn,))
    nvidia_process = multiprocessing.Process(target=run_nvidia, args=(child_conn,))

    nvidia_process.start()
    blender_process.start()
    
    blender_process.join()
    nvidia_process.join()

内容的提问来源于stack exchange，提问作者easythrees