Python多进程比单进程运行更慢的原因探究

我生成了200万个边界框内均匀分布的点，用NumPy做计算并按规则分区。单进程版本运行耗时约0.09秒，其中最耗时的是四个分区计算逻辑。改用multiprocessing实现多进程后，耗时反而增至约0.15秒。设备是11代酷睿i5笔记本，原本预期多进程能利用多核提速，想知道为什么反而更慢？

单进程代码 #

import numpy as np
from draw import draw
import time

X = 0
Y = 1
N = 2000000

max_x = -100000
max_y = -100000
min_x = 100000
min_y = 100000

points = np.random.uniform(-10, 10, (N, 2))

start = time.time()
max_x_index = np.argmax(points[:, X])
max_y_index = np.argmax(points[:, Y])
min_x_index = np.argmin(points[:, X])
min_y_index = np.argmin(points[:, Y])

p_right = points[max_x_index]
p_top = points[max_y_index]
p_left = points[min_x_index]
p_bottom = points[min_y_index]

top_right = points[
points[:, X] > ((points[:, Y] - p_top[Y]) / (p_right[Y] - p_top[Y])) * (p_right[X] - p_top[X]) + p_top[X]]
top_left = points[
points[:, X] < ((points[:, Y] - p_top[Y]) / (p_left[Y] - p_top[Y])) * (p_left[X] - p_top[X]) + p_top[X]]
bottom_right = points[
points[:, X] > ((points[:, Y] - p_bottom[Y]) / (p_right[Y] - p_bottom[Y])) * (p_right[X] - p_bottom[X]) + p_bottom[
X]]
bottom_left = points[
points[:, X] < ((points[:, Y] - p_bottom[Y]) / (p_left[Y] - p_bottom[Y])) * (p_left[X] - p_bottom[X]) + p_bottom[X]]

end = time.time()
print(end - start)

多进程代码 #

import numpy as np
from draw import draw
import time
import multiprocessing

N = 2000000
X = 0
Y = 1
points = np.random.uniform(-10, 10, (N, 2))

max_x = -100000
max_y = -100000
min_x = 100000
min_y = 100000

manager = multiprocessing.Manager()
top_right = manager.list()
top_left = manager.list()
bottom_right = manager.list()
bottom_left = manager.list()


def set_top_right():
    global X, Y, points, p_top, p_right, top_right
    top_right.extend(points[
        points[:, X] > ((points[:, Y] - p_top[Y]) / (p_right[Y] - p_top[Y])) * (p_right[X] - p_top[X]) + p_top[X]])


def set_top_left():
    global X, Y, points, p_top, p_left, top_left
    top_left.extend(points[
        points[:, X] < ((points[:, Y] - p_top[Y]) / (p_left[Y] - p_top[Y])) * (p_left[X] - p_top[X]) + p_top[X]])


def set_bottom_right():
    global X, Y, points, p_bottom, p_right, bottom_right
    bottom_right.extend(points[
        points[:, X] > ((points[:, Y] - p_bottom[Y]) / (p_right[Y] - p_bottom[Y])) * (p_right[X] - p_bottom[X]) +
        p_bottom[X]])


def set_bottom_left():
    global X, Y, points, p_bottom, p_left, bottom_left
    bottom_left.extend(points[
        points[:, X] < ((points[:, Y] - p_bottom[Y]) / (p_left[Y] - p_bottom[Y])) * (p_left[X] - p_bottom[X]) +
        p_bottom[X]])


start = time.time()
max_x_index = np.argmax(points[:, X])
max_y_index = np.argmax(points[:, Y])
min_x_index = np.argmin(points[:, X])
min_y_index = np.argmin(points[:, Y])

p_right = points[max_x_index]
p_top = points[max_y_index]
p_left = points[min_x_index]
p_bottom = points[min_y_index]

p1 = multiprocessing.Process(target=set_top_right)
p2 = multiprocessing.Process(target=set_top_left)
p3 = multiprocessing.Process(target=set_bottom_right)
p4 = multiprocessing.Process(target=set_bottom_left)

p1.start()
p2.start()
p3.start()
p4.start()

p1.join()
p2.join()
p3.join()
p4.join()

end = time.time()

print(end - start)

问题分析与解决 #

为什么多进程反而更慢？ #

1. 进程启动与内存拷贝开销
   每个multiprocessing.Process启动时，会复制父进程的整个内存空间——包括你的200万条数据的points数组。这部分内存拷贝的时间，远超过四个分区计算本身的耗时，直接抵消了并行的收益。

2. 共享内存的通信成本
   你用multiprocessing.Manager()创建的共享列表，进程间传递数据需要跨进程通信（IPC）。extend操作要把NumPy数组序列化后传递，再反序列化为列表，这个过程的开销极大，尤其是处理大数组时。

3. NumPy本身已经做了多线程优化
   NumPy的核心运算（比如布尔索引、数组广播计算）依赖底层的BLAS/LAPACK库，这些库默认已经开启了多线程并行。单进程下你的代码已经在利用多核CPU的算力，再套多进程反而会导致CPU资源竞争，增加进程切换的开销。

4. 任务粒度太小
   四个分区计算总耗时才0.09秒，属于典型的细粒度任务。多进程的启动、调度、通信开销完全盖过了并行执行节省的时间。

优化建议 #

1. 优化单进程NumPy代码
   优先从代码本身入手，比如预计算直线方程的斜率和截距，避免重复计算相同的表达式，减少运算量：

   import numpy as np
   import time
   
   X = 0
   Y = 1
   N = 2000000
   
   points = np.random.uniform(-10, 10, (N, 2))
   
   start = time.time()
   # 计算极值点
   max_x_idx = np.argmax(points[:, X])
   max_y_idx = np.argmax(points[:, Y])
   min_x_idx = np.argmin(points[:, X])
   min_y_idx = np.argmin(points[:, Y])
   
   p_right = points[max_x_idx]
   p_top = points[max_y_idx]
   p_left = points[min_x_idx]
   p_bottom = points[min_y_idx]
   
   # 预计算每个分区的直线参数 y = kx + b → x > k*y + b 形式
   # 右上分区
   k_tr = (p_right[X] - p_top[X]) / (p_right[Y] - p_top[Y])
   b_tr = p_top[X] - k_tr * p_top[Y]
   top_right = points[points[:, X] > k_tr * points[:, Y] + b_tr]
   
   # 左上分区
   k_tl = (p_left[X] - p_top[X]) / (p_left[Y] - p_top[Y])
   b_tl = p_top[X] - k_tl * p_top[Y]
   top_left = points[points[:, X] < k_tl * points[:, Y] + b_tl]
   
   # 右下分区
   k_br = (p_right[X] - p_bottom[X]) / (p_right[Y] - p_bottom[Y])
   b_br = p_bottom[X] - k_br * p_bottom[Y]
   bottom_right = points[points[:, X] > k_br * points[:, Y] + b_br]
   
   # 左下分区
   k_bl = (p_left[X] - p_bottom[X]) / (p_left[Y] - p_bottom[Y])
   b_bl = p_bottom[X] - k_bl * p_bottom[Y]
   bottom_left = points[points[:, X] < k_bl * points[:, Y] + b_bl]
   
   end = time.time()
   print(end - start)
   

2. 合理使用多进程（如果必须）
   如果你一定要用多进程，要避免全量数组拷贝：

   • 使用multiprocessing.shared_memory共享points数组，让子进程直接访问父进程的内存，避免复制。
   • 用multiprocessing.Pool配合分块数据，把大数组拆分成小块分配给不同进程处理，减少单进程的数据量。

3. 用JIT编译加速
   考虑用numba对核心计算逻辑做JIT编译，能在单进程下进一步提升性能，比多进程更高效。

内容的提问来源于stack exchange，提问作者M a m a D