多进程执行仿真命令时,每个子进程加载Shell文件后程序卡住的问题排查
多进程执行仿真命令时,每个子进程加载Shell文件后程序卡住的问题排查
我正在编写一段代码,目的是收集仿真命令并通过多进程执行,每个子进程在运行仿真命令前会先加载一个shell文件。我在另一个函数中把这些命令收集到字典里,供下面的函数使用:
def source_shell_script(self, script_path: str) -> dict: """ Sources a shell script and updates the environment for each subprocess. """ if not os.path.exists(script_path): raise FileNotFoundError(f"Script not found: {script_path}") # source the script and output environment variables command = f"bash -c 'source {script_path} && env'" try: # run the command and capture the output result = subprocess.run(command, shell=True, stdout=subprocess.PIPE, text=True, check=True) # parse the environment variables from the command output env_vars = dict( line.split("=", 1) for line in result.stdout.splitlines() if "=" in line ) return env_vars except subprocess.CalledProcessError as e: raise RuntimeError(f"Failed to source script: {script_path}. Error: {e}")
def run_cmd(self, cmd_px: tuple) -> None: cmd, px_src_path = cmd_px try: # get environment variables env_vars = self.source_shell_script(px_src_path) print(f"Executing command: {cmd}") result = subprocess.run(cmd, shell=True, env=env_vars, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) print(f"Output: {result.stdout}") if result.returncode != 0: print(f"Error: {result.stderr}") except Exception as e: print(f"Failed to execute command: {cmd}. Error: {e}") def exec_sim(self) -> None: """ Execute all simulations in parallel using multiprocessing. """ # create a list of (command, px_src_path) tuples for each pixel configuration run_queue = [(cmd, self.sim_dict[px_key]["px_src_path"]) for px_key in self.sim_dict for cmd in self.sim_dict[px_key]["px_ddsim_cmds"]] num_workers = os.cpu_count() # number of processes with multiprocessing.Pool(num_workers) as pool: pool.map(self.run_cmd, run_queue)
if __name__ == "__main__": # initialize program eic_object = HandleEIC() eic_object.init_path_var() pixel_sizes = eic_object.setup_json() eic_object.pixel_sizes = pixel_sizes os.chmod(eic_object.execution_path, 0o777) eic_object.setup_sim() print("Simulation dictionary:", eic_object.sim_dict) eic_object.exec_sim() # create backup for simulation eic_object.mk_sim_backup()
但程序并没有正常运行,而是卡住了,控制台只打印出了命令:
控制台输出:
Simulation dictionary: {'2.0_0.1': {'px_epic_path': '/data/user/Analysis_epic_new/simulations/simEvents/2.0x0.1px/epic', 'px_compact_path': '/data/user/Analysis_epic_new/simulations/simEvents/2.0x0.1px/epic/install/share/epic/compact', 'px_ip6_path': '/data/user/Analysis_epic_new/simulations/simEvents/2.0x0.1px/epic/install/share/epic/epic_ip6_extended.xml', 'px_src_path': '/data/user/Analysis_epic_new/simulations/simEvents/2.0x0.1px/epic/install/bin/thisepic.sh', 'px_out_path': '/data/user/Analysis_epic_new/simulations/simEvents/2.0x0.1px', 'px_ddsim_cmds': ['ddsim --inputFiles /data/user/Analysis_epic_new/simulations/genEvents/results/beamEffectsElectrons_20.hepmc --outputFile /data/user/Analysis_epic_new/simulations/simEvents/2.0x0.1px/output_20edm4hep.root --compactFile /data/user/Analysis_epic_new/simulations/simEvents/2.0x0.1px/epic/install/share/epic/epic_ip6_extended.xml -N 5']}, '0.1_0.1': {'px_epic_path': '/data/user/Analysis_epic_new/simulations/simEvents/0.1x0.1px/epic', 'px_compact_path': '/data/user/Analysis_epic_new/simulations/simEvents/0.1x0.1px/epic/install/share/epic/compact', 'px_ip6_path': '/data/user/Analysis_epic_new/simulations/simEvents/0.1x0.1px/epic/install/share/epic/epic_ip6_extended.xml', 'px_src_path': '/data/user/Analysis_epic_new/simulations/simEvents/0.1x0.1px/epic/install/bin/thisepic.sh', 'px_out_path': '/data/user/Analysis_epic_new/simulations/simEvents/0.1x0.1px', 'px_ddsim_cmds': ['ddsim --inputFiles /data/user/Analysis_epic_new/simulations/genEvents/results/beamEffectsElectrons_20.hepmc --outputFile /data/user/Analysis_epic_new/simulations/simEvents/0.1x0.1px/output_20edm4hep.root --compactFile /data/user/Analysis_epic_new/simulations/simEvents/0.1x0.1px/epic/install/share/epic/epic_ip6_extended.xml -N 5']}} Executing command: ddsim --inputFiles /data/user/Analysis_epic_new/simulations/genEvents/results/beamEffectsElectrons_20.hepmc --outputFile /data/user/Analysis_epic_new/simulations/simEvents/0.1x0.1px/output_20edm4hep.root --compactFile /data/user/Analysis_epic_new/simulations/simEvents/0.1x0.1px/epic/install/share/epic/epic_ip6_extended.xml -N 5 Executing command: ddsim --inputFiles /data/user/Analysis_epic_new/simulations/genEvents/results/beamEffectsElectrons_20.hepmc --outputFile /data/user/Analysis_epic_new/simulations/simEvents/2.0x0.1px/output_20edm4hep.root --compactFile /data/user/Analysis_epic_new/simulations/simEvents/2.0x0.1px/epic/install/share/epic/epic_ip6_extended.xml -N 5
为什么我的代码没有按预期运行?我哪里做错了?
备注:内容来源于stack exchange,提问作者Niandra Lades
相关产品推荐
相关产品推荐

