升级Redis npm包至4.6.13后Protobuf反序列化Redis数据报错

问题描述 #

将Node.js应用中的redis npm包从3.1.2升级至4.6.13（同时Redis服务器升级至7.x）后，存储在Redis中的Protobuf数据取出反序列化时失败。测试发现，只要在setApplicationInfo函数中取消注释applicationDetailMessage.setResidencelengthinyears(data.durationOfResidenceYear);这行代码，就会抛出Error: Failure: Invalid wire type: %s (at position %s)错误。需注意：Protobuf数据存入Redis前的反序列化完全正常，仅从Redis取出后反序列化时报错。

测试代码 #

设置Proto消息 #

const setApplicationInfo = (data) => {
    try {
       const applicationDetailMessage = new ApplicationDetails();
       applicationDetailMessage.setEmail(data.email);
       applicationDetailMessage.setHomephonenumber('');
       applicationDetailMessage.setMobilephonenumber(data.primaryPhoneNumber);
       applicationDetailMessage.setYearofbirth(data.yearOfBirth);
       applicationDetailMessage.setFirstname(data.firstName);
       applicationDetailMessage.setMiddlename(data.middleName);
       applicationDetailMessage.setLastname(data.lastName);
       applicationDetailMessage.setMonthofbirth(data.monthOfBirth);
       applicationDetailMessage.setDayofbirth(data.dayOfBirth);
       applicationDetailMessage.setSsn(data.ssn);
       applicationDetailMessage.setStreetaddress(data.streetAddress1);
       applicationDetailMessage.setCity(data.city);
       applicationDetailMessage.setState(data.stateCode);
       applicationDetailMessage.setZipcode(data.zipCode);
       applicationDetailMessage.setResidencetype(data.residenceType);
       // applicationDetailMessage.setResidencelengthinyears(data.durationOfResidenceYear);
       // applicationDetailMessage.setResidencelengthinmonths(data.durationOfResidenceMonth);
       const serializedData = applicationDetailMessage.serializeBinary();
       return Buffer.from(serializedData);
    } catch (err) {
        throw new Error(err)
    }
};

获取Redis连接 #

const getRedisConnection  = async () => {
    try {
         const client = await createClient({
              url: `redis://${process.env.REDIS_USER}:${process.env.REDIS_PASS}@${process.env.REDIS_HOST}:${process.env.REDIS_PORT}`,
           //    legacyMode: true
         })
         client.on('error', err => console.log('Redis Client Error', err))
         await client.connect();
         await client.ping();
         return client;
    } catch (err) {
         throw err;
    }
}

主函数 #

(async () => {
    try {
        const redisConn = await getRedisConnection();
        const id  = '1';
        const data = {
            email: 'shrawanlakhey@gmail.com',
            primaryPhoneNumber: '4078790329',
            dayOfBirth: 11,
            monthOfBirth: 12,
            yearOfBirth: 1996,
            firstName: 'Vernon',
            middleName: '',
            lastName: 'McClelland',
            ssn: '228412571',
            streetAddress1: '4311 Grand Avenue',
            city: 'Orlando',
            stateCode: 'FL',
            zipCode: '32810',
            residenceType: 1,
            durationOfResidenceYear: 2,
            durationOfResidenceMonth: 1,
        }
        const message = setApplicationInfo(data)
        const deserializedApplicationObj = ApplicationDetails.deserializeBinary(new Uint8Array(message)).toObject();
        console.log('deserializedApplicationObj', deserializedApplicationObj);
        await redisConn.set(`application_detail_${id}`, message);

        const applicationRes = await redisConn.get(`application_detail_${id}`);
        const applicationBufferData = Buffer.from(applicationRes, 'utf-8');
        console.log('applicationBufferData',  applicationBufferData);
        if (applicationBufferData && applicationBufferData.length > 0) {
            const applicationObj = ApplicationDetails.deserializeBinary(new Uint8Array(applicationBufferData)).toObject();
            console.log('applicationObj', applicationObj);
        }
    } catch (err) {
        throw err;
    }
})()

报错信息 #

jspb.asserts.fail=function(a,b){for(var c=[],d=1;d<arguments.length;++d)c[d-1]=arguments[d];throw Error("Failure"+(a?": "+a:""),c);};jspb.asserts.assertInstanceof=function(a,b,c,d){for(var e=[],f=3;f<arguments.length;++f)e[f-3]=arguments[f];a instanceof b||jspb.asserts.doAssertFailure("Expected instanceof %s but got %s.",[jspb.asserts.getType(b),jspb.asserts.getType(a)],c,e);return a};
                                                                                                  ^

Error: Failure: Invalid wire type: %s (at position %s)
    at jspb.asserts.fail (protos/node_modules/google-protobuf/google-protobuf.js:90:99)
    at jspb.BinaryReader.nextField (/protos/node_modules/google-protobuf/google-protobuf.js:384:536)
    at proto.demo.entities.application_pb.ApplicationDetails.deserializeBinaryFromReader (protos/build/application_pb.js:24459:17)
    at proto.demo.entities.application_pb.ApplicationDetails.deserializeBinary (protos/build/application_pb.js:24447:66)
    at /del_v2/redis/testredis.js:93:53
    at process.processTicksAndRejections (node:internal/process/task_queues:95:5)

Node.js v20.9.0

错误原因 #

核心问题在于Redis客户端v4版本对二进制数据的处理逻辑发生了变化：

• Redis v3.x客户端默认会将Buffer以二进制格式写入Redis；而v4.x客户端默认把Buffer当作UTF-8字符串处理，直接存储为文本格式。
• 当Protobuf序列化后的二进制数据包含非UTF-8兼容的字节（比如数值字段序列化后的某些高位字节），存储为UTF-8字符串时会出现字节损坏——Redis会将无法解析的字节替换为�（替换字符），导致取出的Buffer与原始数据不一致。
• 注释掉那两行代码时，序列化后的二进制恰好全部是UTF-8兼容字节，因此未出现问题；一旦加入residencelengthinyears这类数值字段，序列化后产生非UTF-8字节，存储时被破坏，取出后反序列化自然会触发wire type错误（Protobuf的wire type是二进制格式的关键标识，字节损坏后无法正确识别）。

解决方法 #

以下两种方式任选其一即可修复：

方式1：显式指定二进制存储/读取 #

调用set和get时，明确指定编码为buffer：

// 存储时
await redisConn.set(`application_detail_${id}`, message, { encoding: 'buffer' });

// 读取时
const applicationRes = await redisConn.get(`application_detail_${id}`, { encoding: 'buffer' });
// 此时applicationRes已经是Buffer，无需再用Buffer.from转换
const applicationObj = ApplicationDetails.deserializeBinary(new Uint8Array(applicationRes)).toObject();

方式2：开启legacy兼容模式 #

创建Redis客户端时启用legacyMode: true，让v4客户端兼容v3版本的二进制处理逻辑：

const client = await createClient({
    url: `redis://${process.env.REDIS_USER}:${process.env.REDIS_PASS}@${process.env.REDIS_HOST}:${process.env.REDIS_PORT}`,
    legacyMode: true
})

开启后，get方法会直接返回Buffer，无需额外转换。

内容的提问来源于stack exchange，提问作者shrawan_lakhe