Golang LZ4压缩后前端JS解压报无效幻数错误求助

我搭建了一个Gin API，使用LZ4帧压缩字符串后返回给前端。前端通过fetch请求该API，使用pierrec的LZ4包解压数据时，出现Invalid magic number错误。

后端代码 #

package main

import (
    "bytes"
    "fmt"
    "io"

    "github.com/pierrec/lz4"

    "github.com/gin-gonic/gin"
)

var fileContent = `CompressBlock compresses the source buffer starting at soffet into the destination one. This is the fast version of LZ4 compression and also the default one. The size of the compressed data is returned. If it is 0 and no error, then the data is incompressible. An error is returned if the destination buffer is too small.`

func main() {
    router := gin.Default()

    // Define the API endpoint
    router.POST("/api/endpoint", func(c *gin.Context) {
        toCompress := []byte(fileContent)
        // compressed := make([]byte, len(toCompress))

        result, err := compress(toCompress)
        if err != nil {
            fmt.Println(err)
        }

        c.JSON(200, gin.H{"result": result})
    })

    // Start the server
    router.Run(":8000")
}

func compress(in []byte) ([]byte, error) {
    r := bytes.NewReader(in)
    w := &bytes.Buffer{}
    zw := lz4.NewWriter(w)
    _, err := io.Copy(zw, r)
    if err != nil {
        return nil, err
    }
    // Closing is very important
    if err := zw.Close(); err != nil {
        return nil, err
    }
    return w.Bytes(), nil
}

前端代码 #

import fetch from 'node-fetch';
import LZ4 from 'lz4';

const apiUrl = 'http://localhost:8000/api/endpoint';

const postData = async () => {
  try {
    const response = await fetch(apiUrl, {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
      },
      body: JSON.stringify({}),
    });

    if (!response.ok) {
      throw new Error('Request failed with status ' + response.status);
    }
    const result = await response.json();
    // const decoded = base64ToArrayBuffer(result.result);
    const buffer = Buffer.from(result.result);
    const decompressed = LZ4.decode(buffer);
    const decodedBuffer = Buffer.from(decompressed);
    console.log(decodedBuffer.toString());
  } catch (error) {
    console.error(error);
  }
};

postData();

错误信息 #

Error: Invalid magic number: 4D466941 @0
    at Decoder.emit_Error (/home/anand/go/src/github.com/anand-dotworld/test/client/node_modules/lz4/lib/decoder_stream.js:64:22)
    at Decoder.read_MagicNumber (/home/anand/go/src/github.com/anand-dotworld/test/client/node_modules/lz4/lib/decoder_stream.js:93:8)
    at Decoder._main (/home/anand/go/src/github.com/anand-dotworld/test/client/node_modules/lz4/lib/decoder_stream.js:289:25)
    at Decoder._transform (/home/anand/go/src/github.com/anand-dotworld/test/client/node_modules/lz4/lib/decoder_stream.js:60:7)
    at Decoder.Transform._write (node:internal/streams/transform:184:23)
    at writeOrBuffer (node:internal/streams/writable:389:12)
    at _write (node:internal/streams/writable:330:10)
    at Decoder.Writable.end (node:internal/streams/writable:609:17)
    at Object.LZ4_uncompress [as decode] (/home/anand/go/src/github.com/anand-dotworld/test/client/node_modules/lz4/lib/decoder.js:14:10)
    at postData (file:///home/anand/go/src/github.com/anand-dotworld/test/client/index.js:24:30)

问题原因 #

1. JSON编码二进制数据的问题：后端用c.JSON返回二进制压缩数据时，JSON会自动将二进制数组编码为Base64字符串，但前端直接用Buffer.from(result.result)处理，相当于把Base64字符串当成原始字节流解析，导致解压时无法识别LZ4的魔数。
2. LZ4格式不匹配：后端使用lz4.NewWriter生成的是LZ4帧格式（带魔数和头部），但前端用LZ4.decode（默认处理块格式）解压，导致格式不兼容。

修复方案 #

1. 后端修改：显式将压缩数据转为Base64返回 #

避免JSON自动编码的不确定性，手动将二进制数据转为Base64字符串后返回：

package main

import (
    "bytes"
    "encoding/base64"
    "fmt"
    "io"

    "github.com/pierrec/lz4"
    "github.com/gin-gonic/gin"
)

// ... 省略fileContent定义 ...

func main() {
    router := gin.Default()

    router.POST("/api/endpoint", func(c *gin.Context) {
        toCompress := []byte(fileContent)
        result, err := compress(toCompress)
        if err != nil {
            fmt.Println(err)
            c.JSON(500, gin.H{"error": err.Error()})
            return
        }
        // 手动转Base64字符串
        base64Result := base64.StdEncoding.EncodeToString(result)
        c.JSON(200, gin.H{"result": base64Result})
    })

    router.Run(":8000")
}

// ... 省略compress函数 ...

2. 前端修改：解码Base64并使用帧格式解压 #

先将返回的Base64字符串解码为Buffer，再用LZ4.decodeFrame处理帧格式数据：

import fetch from 'node-fetch';
import LZ4 from 'lz4';

const apiUrl = 'http://localhost:8000/api/endpoint';

const postData = async () => {
  try {
    const response = await fetch(apiUrl, {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
      },
      body: JSON.stringify({}),
    });

    if (!response.ok) {
      throw new Error('Request failed with status ' + response.status);
    }
    const result = await response.json();
    // 解码Base64为Buffer
    const buffer = Buffer.from(result.result, 'base64');
    // 使用帧格式解压
    const decompressed = LZ4.decodeFrame(buffer);
    console.log(decompressed.toString());
  } catch (error) {
    console.error(error);
  }
};

postData();

内容的提问来源于stack exchange，提问作者Anand Samuel