如何在C#中将超过2GB的文件加载到byte数组中？

你的核心问题是一次性将整个文件加载到byte数组，这不仅受限于.NET数组的2GB上限（32位进程），还会占用大量内存，完全无法处理大文件。同时原代码的匹配算法和写入方式效率极低，必须重构为流式处理，边读边匹配边写入，无需加载整个文件到内存。

原代码的问题分析 #

1. 内存限制：File.ReadAllBytes会把整个文件加载到内存，超过2GB直接抛出IO异常，且大文件会导致内存溢出
2. 匹配效率极低：PatternAt中的Skip(i).Take(pattern.Length).SequenceEqual是暴力匹配，每次都要重新枚举字节，大文件下耗时指数级增长
3. 写入效率极差：每次替换后调用File.WriteAllBytes，重复写入整个文件，IO开销巨大

解决方案：流式处理+高效匹配算法 #

采用KMP字符串匹配算法（比暴力匹配效率高几个数量级），结合文件流边读边匹配，同时将结果写入临时文件，最后替换原文件。

1. 实现KMP算法的辅助方法 #

KMP算法通过预处理模式串生成最长前缀后缀数组（LPS），避免重复匹配，大幅提升效率：

private static int[] ComputeLps(byte[] pattern)
{
    int[] lps = new int[pattern.Length];
    int len = 0;
    int i = 1;
    while (i < pattern.Length)
    {
        if (pattern[i] == pattern[len])
        {
            len++;
            lps[i] = len;
            i++;
        }
        else
        {
            if (len != 0)
            {
                len = lps[len - 1];
            }
            else
            {
                lps[i] = 0;
                i++;
            }
        }
    }
    return lps;
}

2. 查找替换（查找与替换长度相同） #

这种情况无需修改文件总长度，处理最简单：

public static void ReplacePatternInLargeFile(string filePath, byte[] findPattern, byte[] replacePattern)
{
    if (findPattern.Length != replacePattern.Length)
    {
        throw new ArgumentException("查找和替换的字节长度必须相同，如需处理不同长度请使用下方重载方法");
    }
    if (findPattern.Length == 0) return;

    string tempFilePath = Path.GetTempFileName();

    using (var readStream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read))
    using (var writeStream = new FileStream(tempFilePath, FileMode.Create, FileAccess.Write, FileShare.None))
    {
        int[] lps = ComputeLps(findPattern);
        byte[] buffer = new byte[4096]; // 4KB缓冲区，可根据内存调整大小
        int bufferOffset = 0;
        int readBytes;
        int patternIndex = 0;

        while ((readBytes = readStream.Read(buffer, bufferOffset, buffer.Length - bufferOffset)) > 0)
        {
            int totalBytesInBuffer = bufferOffset + readBytes;
            int i = 0;
            while (i < totalBytesInBuffer)
            {
                if (buffer[i] == findPattern[patternIndex])
                {
                    i++;
                    patternIndex++;
                    if (patternIndex == findPattern.Length)
                    {
                        // 找到匹配，写入替换字节
                        writeStream.Write(replacePattern, 0, replacePattern.Length);
                        patternIndex = lps[patternIndex - 1];
                        // 移动缓冲区剩余内容到开头，继续匹配
                        int remaining = totalBytesInBuffer - i;
                        if (remaining > 0)
                        {
                            Array.Copy(buffer, i, buffer, 0, remaining);
                            bufferOffset = remaining;
                            i = 0;
                        }
                        else
                        {
                            bufferOffset = 0;
                            break;
                        }
                    }
                }
                else
                {
                    if (patternIndex != 0)
                    {
                        patternIndex = lps[patternIndex - 1];
                    }
                    else
                    {
                        // 无匹配，写入当前字节
                        writeStream.WriteByte(buffer[i]);
                        i++;
                    }
                }
            }

            // 写入缓冲区中未匹配的剩余字节（仅当没有部分匹配时）
            if (bufferOffset > 0 && patternIndex == 0)
            {
                writeStream.Write(buffer, 0, bufferOffset);
                bufferOffset = 0;
            }
        }

        // 写入最后剩余的缓冲区内容
        if (bufferOffset > 0)
        {
            writeStream.Write(buffer, 0, bufferOffset);
        }
    }

    // 替换原文件
    File.Delete(filePath);
    File.Move(tempFilePath, filePath);
}

3. 查找替换（查找与替换长度不同） #

如果替换的字节长度和查找的不同，需要调整文件长度，此时需先收集所有匹配位置，从后往前修改文件，避免覆盖未处理的内容：

private static IEnumerable<long> FindPatternPositions(Stream stream, byte[] pattern)
{
    if (pattern.Length == 0) yield break;

    int[] lps = ComputeLps(pattern);
    byte[] buffer = new byte[4096];
    int bufferOffset = 0;
    int readBytes;
    int patternIndex = 0;
    long fileOffset = 0;

    while ((readBytes = stream.Read(buffer, bufferOffset, buffer.Length - bufferOffset)) > 0)
    {
        int totalBytes = bufferOffset + readBytes;
        int i = 0;
        while (i < totalBytes)
        {
            if (buffer[i] == pattern[patternIndex])
            {
                i++;
                patternIndex++;
                if (patternIndex == pattern.Length)
                {
                    yield return fileOffset + i - pattern.Length;
                    patternIndex = lps[patternIndex - 1];
                }
            }
            else
            {
                if (patternIndex != 0)
                {
                    patternIndex = lps[patternIndex - 1];
                }
                else
                {
                    i++;
                    fileOffset++;
                }
            }
        }

        // 保留缓冲区中可能的部分匹配内容
        int remaining = totalBytes - i + patternIndex;
        if (remaining > 0)
        {
            Array.Copy(buffer, i - patternIndex, buffer, 0, remaining);
            bufferOffset = remaining;
            fileOffset += i - patternIndex;
        }
        else
        {
            bufferOffset = 0;
            fileOffset += totalBytes;
        }
    }
}

public static void ReplacePatternWithDifferentLength(string filePath, byte[] findPattern, byte[] replacePattern)
{
    if (findPattern.Length == 0) return;

    // 先收集所有匹配位置
    List<long> matchPositions = new List<long>();
    using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read))
    {
        foreach (long pos in FindPatternPositions(stream, findPattern))
        {
            matchPositions.Add(pos);
        }
    }

    if (matchPositions.Count == 0) return;

    // 从后往前处理，避免覆盖未修改的内容
    matchPositions.Reverse();
    long lengthDelta = replacePattern.Length - findPattern.Length;
    long currentFileLength = new FileInfo(filePath).Length;

    using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.ReadWrite, FileShare.None))
    {
        foreach (long pos in matchPositions)
        {
            stream.Position = pos + findPattern.Length;
            long remainingBytes = currentFileLength - stream.Position;

            // 处理长度变化：扩展或收缩文件
            if (lengthDelta > 0)
            {
                stream.SetLength(currentFileLength + lengthDelta);
                byte[] buffer = new byte[4096];
                long writePos = stream.Position + lengthDelta;
                while (remainingBytes > 0)
                {
                    int read = stream.Read(buffer, 0, (int)Math.Min(remainingBytes, buffer.Length));
                    stream.Position = writePos;
                    stream.Write(buffer, 0, read);
                    stream.Position = pos + findPattern.Length + (remainingBytes - read);
                    remainingBytes -= read;
                    writePos += read;
                }
            }
            else if (lengthDelta < 0)
            {
                byte[] buffer = new byte[4096];
                long readPos = stream.Position;
                stream.Position = pos + replacePattern.Length;
                while (remainingBytes > 0)
                {
                    int read = stream.Read(buffer, 0, (int)Math.Min(remainingBytes, buffer.Length));
                    stream.Position = readPos;
                    stream.Write(buffer, 0, read);
                    readPos += read;
                    stream.Position = readPos;
                    remainingBytes -= read;
                }
                stream.SetLength(currentFileLength + lengthDelta);
            }

            // 写入替换内容
            stream.Position = pos;
            stream.Write(replacePattern, 0, replacePattern.Length);
            currentFileLength += lengthDelta;
        }
    }
}

使用示例 #

// 示例：查找0x12 0x34 0x56，替换为0xAB 0xCD 0xEF
byte[] find = new byte[] { 0x12, 0x34, 0x56 };
byte[] replace = new byte[] { 0xAB, 0xCD, 0xEF };

// 长度相同的情况调用
ReplacePatternInLargeFile("example.bin", find, replace);

// 长度不同的情况调用（比如替换为2个字节）
// ReplacePatternWithDifferentLength("example.bin", find, new byte[] {0x00, 0x01});

内容的提问来源于stack exchange，提问作者Bohdan