使用Goroutines构建Zip归档时触发切片越界Panic问题排查

尝试通过goroutine并行处理文件来优化Zip归档构建函数，目标目录包含190个文件（总计500MB），但运行时触发panic：

panic: runtime error: slice bounds out of range [4126:4096]

错误发生在io.Copy(bundleFile, fsFile)调用处，以下是完整函数代码、错误位置及堆栈跟踪：

函数代码 #

func BuildArchive() error {                                                         
                                                                                   
    var files []string                                                              
    err := filepath.Walk("/tmp/dir-to-zip", func(filePath string, info os.FileInfo, err error) error {
        if info.IsDir() {                                                          
            return nil                                                             
        }                                                                           
        if err != nil {                                                            
            fmt.Println(err)                                                       
            return err                                                             
        }                                                                           
        files = append(files, filePath)                                            
                                                                                   
        return nil                                                                
    })                                                                             
    if err != nil {                                                                
        return err                                                                
    }                                                                              
                                                                                   
    bundle, err := os.Create("/tmp/archive.zip")                                   
    if err != nil {                                                                
        return err                                                                
    }                                                                              
    bundleWriter := zip.NewWriter(bundle)                                          
                                                                                   
    var wg sync.WaitGroup                                                          
    wg.Add(len(files))                                                             
                                                                                   
    for _, filePath := range files {                                               
        go func(filePath string) {                                                 
            defer wg.Done()                                                        
            relPath := strings.TrimPrefix(filePath, fmt.Sprintf("%v/", filepath.Dir("/tmp/dir-to-zip")))
                                                                                   
            bundleFile, err := bundleWriter.Create(relPath)                        
            if err != nil {                                                        
                fmt.Println(err)                                                   
            }                                                                      
            fsFile, err := os.Open(filePath)                                       
            if err != nil {                                                        
                fmt.Println(err)                                                   
            }                                                                      
            _, err = io.Copy(bundleFile, fsFile)                                   
            if err != nil {                                                        
                fmt.Println(err)                                                   
            }                                                                      
                                                                                   
        }(filePath)                                                                
    }                                                                              
    wg.Wait()                                                                      
                                                                                   
    err = bundleWriter.Close()                                                     
    if err != nil {                                                                
        return err                                                                
    }                                                                              
    return nil                                                                    
} 

错误发生位置 #

_, err = io.Copy(bundleFile, fsFile)                                   
if err != nil {                                                        
    fmt.Println(err)                                                   
}   

堆栈跟踪 #

goroutine 48 [running]:                                                            
bufio.(*Writer).Write(0xc00002a100, {0xc00041a400?, 0x3d?, 0xc00041a400?})          
    /usr/local/go/src/bufio/bufio.go:670 +0x1c8                                    
archive/zip.(*countWriter).Write(0xc00000c138, {0xc00041a400?, 0x3d?, 0x4afa20?})
    /usr/local/go/src/archive/zip/writer.go:601 +0x2e                                
io.WriteString({0x4e7538, 0xc00000c138}, {0xc0000212c9, 0x3d})                     
    /usr/local/go/src/io/io.go:314 +0x91                                             
archive/zip.writeHeader({0x4e7538, 0xc00000c138}, 0xc000220090)                      
    /usr/local/go/src/archive/zip/writer.go:422 +0x5ec                               
archive/zip.(*Writer).CreateHeader(0xc0000760a0, 0xc00021e1b0)                       
    /usr/local/go/src/archive/zip/writer.go:378 +0x797                               
archive/zip.(*Writer).Create(0x4e7698?, {0xc0000212c9, 0x3d})                        
    /usr/local/go/src/archive/zip/writer.go:223 +0x6c                                
main.BuildArchive.func2({0xc0000212c0, 0x46})                                        
    /home/simba/go/src/foobar/main.go:79 +0x1c5                                      
created by main.BuildArchive                                                         
    /home/simba/go/src/foobar/main.go:73 +0x5aa                                      
panic: runtime error: slice bounds out of range [:4126] with capacity 4096          
                                                                                   
goroutine 6 [running]:                                                               
bufio.(*Writer).Flush(0xc00002a100)                                                  
    /usr/local/go/src/bufio/bufio.go:634 +0x171                                      
bufio.(*Writer).Write(0xc00002a100, {0xc0001b4200?, 0xc000199b20?, 0xc000199b20?})
    /usr/local/go/src/bufio/bufio.go:672 +0xd8                                       
archive/zip.(*countWriter).Write(0xc00000c138, {0xc0001b4200?, 0x0?, 0xc000199b40?})
    /usr/local/go/src/archive/zip/writer.go:601 +0x2e                                
archive/zip.(*countWriter).Write(0xc000220018, {0xc0001b4200?, 0xc0001b02c0?, 0xc0001b02f0?})
    /usr/local/go/src/archive/zip/writer.go:601 +0x2e                                
compress/flate.(*huffmanBitWriter).write(...)                                        
    /usr/local/go/src/compress/flate/huffman_bit_writer.go:136                       
compress/flate.(*huffmanBitWriter).writeCode(0xc0001b41e0?, {0x6000?, 0x22?})    
    /usr/local/go/src/compress/flate/huffman_bit_writer.go:347 +0xe5                
compress/flate.(*huffmanBitWriter).writeTokens(0xc0001b41e0, {0xc002558000, 0x4001, 0x403f800000403f?}, {0xc0001aa900, 0x11e, 0x108129000000000f?}, {0xc0001ac100, 0x1e, 0x1e})
    /usr/local/go/src/compress/flate/huffman_bit_writer.go:583 +0xb9                
compress/flate.(*huffmanBitWriter).writeBlock(0xc0001b41e0, {0xc002558000?, 0x20?, 0xd79?}, 0x0, {0x0, 0x0, 0x0})
    /usr/local/go/src/compress/flate/huffman_bit_writer.go:495 +0x490               
compress/flate.(*compressor).writeBlock(0xc0005a2000, {0xc002558000?, 0xc000032f00?, 0xc000199d28?}, 0x47739b?)
    /usr/local/go/src/compress/flate/deflate.go:170 +0x9c                            
compress/flate.(*compressor).deflate(0xc0005a2000)                                   
    /usr/local/go/src/compress/flate/deflate.go:509 +0x59b                           
compress/flate.(*compressor).write(0xc0005a2000, {0xc00256a000?, 0x8000, 0xf311b6fd?})
    /usr/local/go/src/compress/flate/deflate.go:554 +0x82                            
compress/flate.(*Writer).Write(...)                                                  
    /usr/local/go/src/compress/flate/deflate.go:712                                  
archive/zip.(*pooledFlateWriter).Write(0xc00020c040?, {0xc00256a000?, 0x8000?, 0x4af140?})
    /usr/local/go/src/archive/zip/register.go:51 +0xc5                               
archive/zip.(*countWriter).Write(...)                                                
    /usr/local/go/src/archive/zip/writer.go:601                                      
archive/zip.(*fileWriter).Write(0xc000222000, {0xc00256a000, 0x8000, 0x8000})    
    /usr/local/go/src/archive/zip/writer.go:533 +0x97                                
io.copyBuffer({0x4e7558, 0xc000222000}, {0x4e7678, 0xc0001f8008}, {0x0, 0x0, 0x0})
    /usr/local/go/src/io/io.go:428 +0x204                                            
io.Copy(...)                                                                       
    /usr/local/go/src/io/io.go:385                                                 
main.BuildArchive.func2({0xc00001c0c0, 0x35})                                        
    /home/simba/go/src/foobar/main.go:89 +0x385                                      
created by main.BuildArchive                                                         
    /home/simba/go/src/foobar/main.go:73 +0x5aa                                      
exit status 2

核心原因 #

archive/zip.Writer不是并发安全的，它内部维护了共享的缓冲区和状态信息（比如全局的bufio.Writer），当多个goroutine同时调用Create方法或写入文件内容时，会导致并发写冲突，破坏内部数据结构，最终触发slice越界的panic。

从堆栈跟踪也能看出，panic发生在bufio.Writer的Flush和Write操作中，这是多个goroutine同时操作同一个缓冲区导致的数据竞争。

修复方案 #

方案1：对zip.Writer的操作加互斥锁 #

在goroutine中，所有涉及bundleWriter的操作（Create、写入文件内容）都需要加锁，确保同一时间只有一个goroutine操作zip.Writer：

修改后的代码：

func BuildArchive() error {                                                         
    var files []string                                                              
    err := filepath.Walk("/tmp/dir-to-zip", func(filePath string, info os.FileInfo, err error) error {
        if info.IsDir() {                                                          
            return nil                                                             
        }                                                                           
        if err != nil {                                                            
            fmt.Println(err)                                                       
            return err                                                             
        }                                                                           
        files = append(files, filePath)                                            
        return nil                                                                
    })                                                                             
    if err != nil {                                                                
        return err                                                                
    }                                                                              
                                                                                   
    bundle, err := os.Create("/tmp/archive.zip")                                   
    if err != nil {                                                                
        return err                                                                
    }                                                                              
    defer bundle.Close() // 增加defer关闭文件，避免句柄泄漏
                                                                                   
    bundleWriter := zip.NewWriter(bundle)                                          
    defer bundleWriter.Close() // 用defer简化关闭逻辑
                                                                                   
    var wg sync.WaitGroup                                                          
    var mu sync.Mutex // 新增互斥锁，保护zip.Writer的并发操作
                                                                                   
    wg.Add(len(files))                                                             
                                                                                   
    for _, filePath := range files {                                               
        go func(filePath string) {                                                 
            defer wg.Done()                                                        
            relPath := strings.TrimPrefix(filePath, fmt.Sprintf("%v/", filepath.Dir("/tmp/dir-to-zip")))
            
            // 打开文件放在锁外，减少锁占用时间
            fsFile, err := os.Open(filePath)                                       
            if err != nil {                                                        
                fmt.Println(err)                                                   
                return
            }                                                                      
            defer fsFile.Close() // 新增defer关闭文件
            
            mu.Lock() // 加锁保护zip.Writer的操作
            defer mu.Unlock()
            
            bundleFile, err := bundleWriter.Create(relPath)                        
            if err != nil {                                                        
                fmt.Println(err)                                                   
                return
            }                                                                      
            _, err = io.Copy(bundleFile, fsFile)                                   
            if err != nil {                                                        
                fmt.Println(err)                                                   
            }                                                                      
        }(filePath)                                                                
    }                                                                              
    wg.Wait()                                                                      
                                                                                   
    return nil                                                                    
} 

方案2：并行读取文件，串行写入Zip #

如果想最大化并行效率，可以先在goroutine中读取文件内容到内存，然后串行写入Zip。这种方式适合小文件场景，大文件会占用过多内存，需要根据实际情况权衡：

func BuildArchive() error {
    // 前面的文件遍历逻辑不变
    var files []string                                                              
    err := filepath.Walk("/tmp/dir-to-zip", func(filePath string, info os.FileInfo, err error) error {
        if info.IsDir() {                                                          
            return nil                                                             
        }                                                                           
        if err != nil {                                                            
            fmt.Println(err)                                                       
            return err                                                             
        }                                                                           
        files = append(files, filePath)                                            
        return nil                                                                
    })                                                                             
    if err != nil {                                                                
        return err                                                                
    }                                                                              

    bundle, err := os.Create("/tmp/archive.zip")
    if err != nil {
        return err
    }
    defer bundle.Close()

    bundleWriter := zip.NewWriter(bundle)
    defer bundleWriter.Close()

    // 定义结构体存储文件路径、内容和读取错误
    type fileData struct {
        relPath string
        content []byte
        err     error
    }

    ch := make(chan fileData, len(files))
    var wg sync.WaitGroup

    // 并行读取文件
    wg.Add(len(files))
    for _, filePath := range files {
        go func(filePath string) {
            defer wg.Done()
            relPath := strings.TrimPrefix(filePath, fmt.Sprintf("%v/", filepath.Dir("/tmp/dir-to-zip")))
            
            content, err := os.ReadFile(filePath)
            ch <- fileData{relPath: relPath, content: content, err: err}
        }(filePath)
    }

    // 所有文件读取完成后关闭通道
    go func() {
        wg.Wait()
        close(ch)
    }()

    // 串行写入Zip归档
    for fd := range ch {
        if fd.err != nil {
            fmt.Println(fd.err)
            continue
        }
        bundleFile, err := bundleWriter.Create(fd.relPath)
        if err != nil {
            fmt.Println(err)
            continue
        }
        _, err = bundleFile.Write(fd.content)
        if err != nil {
            fmt.Println(err)
        }
    }

    return nil
}

额外注意事项 #

• 原代码未关闭fsFile，会导致文件句柄泄漏，修复时需添加defer fsFile.Close()。
• 并行处理Zip归档的性能提升有限，因为磁盘IO是串行瓶颈；若开启压缩，CPU也可能成为瓶颈，需根据实际场景选择是否并行。

内容的提问来源于stack exchange，提问作者SimbaPK