如何在WebAssembly中处理约3GB的大文件？

不想使用已停止维护的stdweb，当前接收的参数是web_sys::File类型，但希望能像处理本地文件系统的std::fs::File那样处理它，比如用BufReader按行读取解析mbox文件。

当前接收参数的函数：

use web_sys::{File as WebFile};

pub fn process_emails(file: WebFile, json_file_path: &str, json_headers_file_path: &str) {
   // ...
}

期望的使用方式：

let file = File::open(mbox_file_path).expect("Failed to open mbox file");

以下是处理本地文件的完整函数（省略导入和相关变量）：

pub fn process_emails(mbox_file_path: &str, json_file_path: &str, json_headers_file_path: &str) {
    let file = File::open(mbox_file_path).expect("Failed to open mbox file");
    let reader = BufReader::new(file);
    let mut emails = Vec::new();
    let mut email_headers = Vec::new();
    let mut current_email = Vec::new();
    let mut id_counter = 0;

    for line in reader.lines() {
        let line = line.expect("Failed to read line");
        if line.starts_with("From ") && !current_email.is_empty() {
            extract_and_process_email(&current_email, &mut emails, &mut email_headers, id_counter);
            current_email.clear();
            id_counter += 1;
        }
        current_email.push(line);
    }

    if !current_email.is_empty() {
        extract_and_process_email(&current_email, &mut emails, &mut email_headers, id_counter);
    }

    write_to_json(json_file_path, &emails).expect("Failed to write JSON file");
    write_to_json(json_headers_file_path, &email_headers).expect("Failed to write JSON file");

    println!("Conversion complete. JSON data saved to {} and {}", json_file_path, json_headers_file_path);
}

vite.config.ts #

import {defineConfig} from 'vite'
import react from '@vitejs/plugin-react-swc'
import wasm from "vite-plugin-wasm";
import topLevelAwait from "vite-plugin-top-level-await";

// https://vitejs.dev/config/
export default defineConfig({
    plugins: [
        react(),
        wasm(),
        topLevelAwait()
    ],
    build: {
        target: 'esnext',
    },
    optimizeDeps: {
        esbuildOptions: {
            target: 'esnext',
        },
    },
})

src/App.tsx #

import './App.css'
import * as wasm from '../pkg';
import {useState} from "react";

function App() {
    const [result, setResult] = useState('');

    const handleFileChange = (e) => {
        const files = e.target.files;
        if (files.length === 0) return;
        const file = files[0];
        const result = wasm.read_file(file);
        setResult(result);
    };


    return (
        <div className="App">
            <h1>hi</h1>
            <input type="file" onChange={handleFileChange}/>
            {result && <div>Result: {result}</div>}
        </div>
    );
}

export default App;

Rust代码 #

use wasm_bindgen::prelude::*;
use wasm_bindgen::JsCast;
use web_sys::{File};

#[wasm_bindgen]
pub fn read_file(file: File) -> String {
    let blob_result = file.slice();
    let len = match blob_result {
        Ok(blob) => blob.size() as f64,
        Err(_) => 0.0,
    };

    let result = format!("Processed {} bytes", len);
    result
}

目前Wasm在React中的使用没问题，处理2.8GB文件也能正常运行。

核心思路是把web_sys::File的内容读取成字节数组，再用std::io::Cursor包装，让它实现std::io::Read trait，这样就能和本地文件一样用BufReader处理。同时要处理Web环境的异步特性。

步骤1：添加依赖 #

在Cargo.toml中添加：

[dependencies]
wasm-bindgen-futures = "0.4"
futures = "0.3"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"

步骤2：改造process_emails函数 #

让它接受任意实现Read的类型，同时把本地文件写入逻辑改为返回处理结果（Web环境无法直接写本地文件，需由JS处理输出）：

use std::io::{Read, BufRead, BufReader};
use serde::Serialize;

// 假设Email和EmailHeader是已定义的序列化结构体
#[derive(Serialize)]
pub struct Email { /* 字段定义 */ }
#[derive(Serialize)]
pub struct EmailHeader { /* 字段定义 */ }

pub fn process_emails<R: Read>(reader: R) -> (Vec<Email>, Vec<EmailHeader>) {
    let reader = BufReader::new(reader);
    let mut emails = Vec::new();
    let mut email_headers = Vec::new();
    let mut current_email = Vec::new();
    let mut id_counter = 0;

    for line in reader.lines() {
        let line = line.expect("Failed to read line");
        if line.starts_with("From ") && !current_email.is_empty() {
            extract_and_process_email(&current_email, &mut emails, &mut email_headers, id_counter);
            current_email.clear();
            id_counter += 1;
        }
        current_email.push(line);
    }

    if !current_email.is_empty() {
        extract_and_process_email(&current_email, &mut emails, &mut email_headers, id_counter);
    }

    (emails, email_headers)
}

步骤3：实现异步读取WebFile并处理 #

修改Rust的read_file为异步函数：

use wasm_bindgen::prelude::*;
use wasm_bindgen_futures::JsFuture;
use web_sys::{File, Blob};
use std::io::Cursor;

#[wasm_bindgen]
pub async fn read_file(file: File) -> Result<JsValue, JsValue> {
    // 将File转为Blob并读取ArrayBuffer
    let blob: Blob = file.into();
    let array_buffer = JsFuture::from(blob.array_buffer()).await?;
    
    // 把ArrayBuffer转为Vec<u8>
    let bytes = js_sys::Uint8Array::new(&array_buffer).to_vec();
    
    // 用Cursor包装字节数组，实现Read trait
    let cursor = Cursor::new(bytes);
    
    // 处理邮件
    let (emails, headers) = process_emails(cursor);
    
    // 序列化结果为JsValue返回给JS
    let result = serde_json::json!({
        "emails": emails,
        "headers": headers
    });
    
    Ok(result.into())
}

步骤4：修改JS代码处理异步 #

更新App.tsx中的handleFileChange为异步函数：

const handleFileChange = async (e) => {
    const files = e.target.files;
    if (files.length === 0) return;
    const file = files[0];
    try {
        const result = await wasm.read_file(file);
        setResult(JSON.stringify(result, null, 2));
    } catch (err) {
        setResult(`Error: ${err}`);
    }
};

关键点说明 #

• Web环境无法直接写入本地文件，需将处理后的JSON结果返回给JS，由JS负责下载或展示。
• Cursor<Vec<u8>>实现了Read trait，完美适配原有按行读取的逻辑。
• 异步读取是Web API的要求，必须用wasm_bindgen_futures处理Promise。

内容的提问来源于stack exchange，提问作者Jacob Marshall