Python中无法启动独立内存空间的子进程问题排查

我尝试通过Flask服务器为每个Web连接启动一个运行AutoGPT的Python子进程，目前进程可正常启动，但所有AutoGPT进程似乎共享同一内存空间：给第一个进程命名TOTO、第二个命名TITI后，第一个进程的名称变为了TITI。以下是我的代码：

from subprocess import Popen, call, PIPE
import sys
import re
import io

class AutoGPT:
    cache_running_process = []
    def __getCache__(self, sid):
        for e in AutoGPT.cache_running_process:
            if e.sid == sid:
                return e
        return None

    def __init__(self, socketio, sid):
        self.socketio = socketio
        cache = self.__getCache__(sid)
        if cache is not None:
            self.sid = cache.sid
            self.proc = cache.proc
        else:
            self.proc = Popen(["./run.sh"], stdin=PIPE, stdout=PIPE, stderr=PIPE, cwd="./Auto-GPT", start_new_session=True)
            self.sid = sid
            print('current sid :'+sid)
            self.cache_running_process.append(self)

    def check_tag_at_the_end(self, line_target):
        line_splitted = line_target.split()
        if len(line_splitted) < 2:
            return None
        return re.search(r": \]$", line_splitted[-2] + " " + line_splitted[-1])

    def read_until_it_finishes(self):
        line = ""
        colorama = False
        while self.proc.poll() is None:
            char =  self.proc.stdout.read(1)
            char_utf8 = char.decode('utf-8', 'ignore')
            line += char_utf8
            if self.check_tag_at_the_end(line) is not None:
                break
            if colorama == True and char == bytes('m', 'utf-8'):
                colorama = False
            else:
                if colorama == False and char != bytes('\x1b', 'utf-8'):
                    if char == bytes('\n', 'utf-8'):
                        self.socketio.emit('my_response', 
                            {'data': "<br />"}, namespace='/test')
                    else:
                        self.socketio.emit('my_response', 
                            {'data': char_utf8}, namespace='/test')
                else:
                    colorama = True

    def write(self, message):
        self.proc.stdin.write(bytes(message + "\n", 'utf-8'))
        self.proc.stdin.flush()
        self.read_until_it_finishes()

    def close(self):
        self.proc.terminate()
        cache = self.__getCache__(self.sid)
        index = self.cache_running_process.index(cache)
        del self.cache_running_process[index]

核心问题 #

所有AutoGPT进程共享了同一个工作目录，导致运行时生成的配置文件、状态文件被不同进程互相覆盖，出现名称被修改的现象。

修复步骤 #

1. 为每个进程创建独立工作目录 #

启动子进程前，为每个sid生成专属的工作目录，复制基础的Auto-GPT文件到该目录，让每个进程在独立环境中运行：

from subprocess import Popen, call, PIPE
import sys
import re
import io
import os
import shutil

class AutoGPT:
    cache_running_process = []
    def __getCache__(self, sid):
        for e in AutoGPT.cache_running_process:
            if e.sid == sid:
                return e
        return None

    def __init__(self, socketio, sid):
        self.socketio = socketio
        cache = self.__getCache__(sid)
        if cache is not None:
            self.sid = cache.sid
            self.proc = cache.proc
        else:
            # 生成独立工作目录
            worker_dir = f"./Auto-GPT-{sid}"
            if not os.path.exists(worker_dir):
                # 复制基础Auto-GPT文件到独立目录
                shutil.copytree("./Auto-GPT", worker_dir)
            # 指定独立目录启动子进程
            self.proc = Popen(["./run.sh"], stdin=PIPE, stdout=PIPE, stderr=PIPE, cwd=worker_dir, start_new_session=True)
            self.sid = sid
            print('current sid :'+sid)
            self.cache_running_process.append(self)

    # 其余方法保持不变...

    def close(self):
        self.proc.terminate()
        cache = self.__getCache__(self.sid)
        index = self.cache_running_process.index(cache)
        del self.cache_running_process[index]
        # 清理独立工作目录（可选）
        worker_dir = f"./Auto-GPT-{self.sid}"
        if os.path.exists(worker_dir):
            shutil.rmtree(worker_dir)

2. 检查run.sh脚本 #

确保run.sh中没有硬编码的文件路径，所有依赖文件的引用都基于当前工作目录，避免跨进程共享状态文件。

内容的提问来源于stack exchange，提问作者Alex A.