Node.js通过stdio pipe向Python传输大图片数据失败求助

原方案问题分析 #

直接将图片Buffer序列化为JSON通过stdio传输的方式，会把二进制数据转成冗长的数组格式，不仅序列化/反序列化开销极大，大文件还容易触发管道缓冲区限制，导致数据传输不完整或接收失败。下面是几种更适合大文件传输的替代方案：

方案1：临时文件中转 #

这是实现成本最低的方案，利用本地文件系统传递数据，避免管道传输的大小限制：

Node.js 代码 #

const fs = require('fs');
const { spawn } = require('child_process');
const path = require('path');
const os = require('os');

app.get("/", (request, response) => {
  let dataToSend;
  // 生成唯一临时文件路径，避免冲突
  const tempPath1 = path.join(os.tmpdir(), `temp_img_${Date.now()}_1.jpg`);
  const tempPath2 = path.join(os.tmpdir(), `temp_img_${Date.now()}_2.jpg`);
  
  // 写入图片到临时文件
  fs.writeFileSync(tempPath1, fs.readFileSync('jeff (1).jpg'));
  fs.writeFileSync(tempPath2, fs.readFileSync('falsified2 (1).jpg'));

  // 把临时文件路径作为参数传给Python
  const python = spawn('python', ['script1.py', tempPath1, tempPath2]);

  python.stdout.on('data', (data) => {
    dataToSend = data.toString();
  });
  
  python.stderr.on('data', (data) => {
    console.error(`stderr: ${data}`);
  });
  
  python.on('exit', (code) => {
    console.log(dataToSend);
    // 任务完成后清理临时文件
    fs.unlinkSync(tempPath1);
    fs.unlinkSync(tempPath2);
  });
});

Python 代码 #

import sys
import os

def main():
    print('python begins')
    # 从命令行参数获取临时文件路径
    img_path1 = sys.argv[1]
    img_path2 = sys.argv[2]
    
    # 读取图片二进制数据
    with open(img_path1, 'rb') as f:
        img_data1 = f.read()
    with open(img_path2, 'rb') as f:
        img_data2 = f.read()
    
    print("Data received from Node.js:")
    print(f"Image 1 size: {len(img_data1)} bytes")
    print(f"Image 2 size: {len(img_data2)} bytes")

if __name__ == "__main__":
    main()

方案2：二进制流分块传输（优化stdio） #

如果不想依赖文件系统，可以直接通过管道传输二进制数据，先传递数据长度再传内容，避免JSON序列化的损耗：

Node.js 代码 #

const fs = require('fs');
const { spawn } = require('child_process');

app.get("/", (request, response) => {
  let dataToSend;
  const image3v = fs.readFileSync('jeff (1).jpg');
  const image4v = fs.readFileSync('falsified2 (1).jpg');

  const python = spawn('python', ['script1.py'], { stdio: ['pipe', 'pipe', 'inherit'] });

  // 先发送第一个图片的长度（4字节大端格式）
  const lenBuf1 = Buffer.alloc(4);
  lenBuf1.writeUInt32BE(image3v.length, 0);
  python.stdin.write(lenBuf1);
  // 发送第一个图片的二进制数据
  python.stdin.write(image3v);
  
  // 同理发送第二个图片
  const lenBuf2 = Buffer.alloc(4);
  lenBuf2.writeUInt32BE(image4v.length, 0);
  python.stdin.write(lenBuf2);
  python.stdin.write(image4v);
  
  python.stdin.end();

  python.stdout.on('data', (data) => {
    dataToSend = data.toString();
  });
  
  python.stderr.on('data', (data) => {
    console.error(`stderr: ${data}`);
  });
  
  python.on('exit', (code) => {
    console.log(dataToSend);
  });
});

Python 代码 #

import sys
import struct

def read_exact_bytes(n):
    """读取指定长度的二进制数据，确保不会中途中断"""
    received = b''
    while len(received) < n:
        chunk = sys.stdin.read(n - len(received))
        if not chunk:
            raise EOFError("Unexpected end of input stream")
        received += chunk
    return received

print('python begins')

# 读取第一个图片的长度（4字节转无符号整数）
len_buf1 = read_exact_bytes(4)
img1_len = struct.unpack('!I', len_buf1)[0]
# 读取对应长度的图片数据
img_data1 = read_exact_bytes(img1_len)

# 读取第二个图片
len_buf2 = read_exact_bytes(4)
img2_len = struct.unpack('!I', len_buf2)[0]
img_data2 = read_exact_bytes(img2_len)

print("Data received from Node.js:")
print(f"Image 1 size: {len(img_data1)} bytes")
print(f"Image 2 size: {len(img_data2)} bytes")

方案3：本地HTTP服务传输 #

适合需要更灵活数据交互的场景，通过临时HTTP服务传递图片：

Node.js 代码 #

const fs = require('fs');
const { spawn } = require('child_process');
const http = require('http');
const port = 3001; // 临时使用的本地端口

app.get("/", (request, response) => {
  let dataToSend;
  
  // 创建临时HTTP服务器
  const server = http.createServer((req, res) => {
    if (req.url === '/img1') {
      res.writeHead(200, {'Content-Type': 'image/jpeg'});
      res.end(fs.readFileSync('jeff (1).jpg'));
    } else if (req.url === '/img2') {
      res.writeHead(200, {'Content-Type': 'image/jpeg'});
      res.end(fs.readFileSync('falsified2 (1).jpg'));
    }
  });

  server.listen(port, () => {
    // 启动Python脚本，传递服务器地址
    const python = spawn('python', ['script1.py', `http://localhost:${port}`]);

    python.stdout.on('data', (data) => {
      dataToSend = data.toString();
    });
    
    python.stderr.on('data', (data) => {
      console.error(`stderr: ${data}`);
    });
    
    python.on('exit', (code) => {
      console.log(dataToSend);
      server.close(); // 任务完成关闭临时服务器
    });
  });
});

Python 代码 #

import sys
import requests

def main():
    print('python begins')
    base_url = sys.argv[1]
    
    # 通过HTTP请求获取图片数据
    resp1 = requests.get(f"{base_url}/img1")
    img_data1 = resp1.content
    resp2 = requests.get(f"{base_url}/img2")
    img_data2 = resp2.content
    
    print("Data received from Node.js:")
    print(f"Image 1 size: {len(img_data1)} bytes")
    print(f"Image 2 size: {len(img_data2)} bytes")

if __name__ == "__main__":
    main()

内容的提问来源于stack exchange，提问作者blueberrr