OpenSSL AES-CTR+HMAC-SHA256解密后文本乱码问题求助

问题背景 #

在Linux Ubuntu环境下，使用C语言结合OpenSSL实现AES-CTR加密+HMAC-SHA256认证功能，加密解密过程无报错，但解密生成的decrypted.txt内容为乱码。操作指令如下：
加密：

./cryp enc -key shared.key -in original.txt -out encrypted.txt -tag encrypted.tag

解密：

./cryp dec -key shared.key -in encrypted.txt -tag encrypted.tag -out decrypted.txt

原代码与配置 #

C代码 #

#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <openssl/sha.h>
#include <openssl/evp.h>
#include <openssl/hmac.h>

char* readFile(char* fileName){
    int size;
    int count;
    FILE *fp = fopen(fileName, "r");
    fseek(fp, 0, SEEK_END);
    size = ftell(fp);
    char* buffer = malloc(size+1);
    char* result = malloc(size+1);
    memset(buffer, 0, size+1);
    memset(result, 0, size+1);
    fseek(fp, 0, SEEK_SET);
    while(1){
        char* pStr = fgets(buffer, size+1, fp);
        if(pStr==NULL)break;
        strcat(result, pStr);
    }
    fclose(fp);
    free(buffer);
    return result;
}

int main(int argc, char* argv[]){

    if(argc != 10){
        printf("ERROR\n");
        exit(2);
    }
    int ikey, iin, iout, itag;
    for(int i=2; i < 9;i+=2){
        if(!strcmp(argv[i], "-key")) ikey = i+1;
        else if(!strcmp(argv[i], "-in")) iin = i+1;
        else if(!strcmp(argv[i], "-out")) iout = i+1;
        else if(!strcmp(argv[i], "-tag")) itag = i+1;
    }
    if(ikey+iin+iout+itag != 24){
        printf("ERROR\n");
        exit(2);
    }
    char* key = readFile(argv[ikey]);
    char *iv = "0123456789012345";



    if(!strcmp(argv[1], "enc")){
        FILE *inFp = fopen(argv[iin],"rb");
        FILE *outFp = fopen(argv[iout], "wb");
        int inLen, outLen;
        char inBuf[BUFSIZ], outBuf[BUFSIZ+EVP_MAX_BLOCK_LENGTH];
        EVP_CIPHER_CTX* ctx = EVP_CIPHER_CTX_new();
        EVP_EncryptInit_ex(ctx, EVP_aes_256_ctr(), NULL, key, iv);
        while((inLen=fread(inBuf, 1, sizeof(inBuf), inFp))>0){

            if(!EVP_EncryptUpdate(ctx, outBuf, &outLen, inBuf, inLen)){
                printf("ERROR\n");
                EVP_CIPHER_CTX_cleanup(ctx);
                exit(2);
            }
            fwrite(outBuf, 1, outLen, outFp);
        }
        if(!EVP_EncryptFinal_ex(ctx, outBuf, &outLen)){
            printf("ERROR\n");
            EVP_CIPHER_CTX_cleanup(ctx);
            exit(2);
        }
        fwrite(outBuf, 1, outLen, outFp);
        EVP_CIPHER_CTX_cleanup(ctx);

        fclose(inFp);
        fclose(outFp);
        char* cipher = readFile(argv[iout]);
        char* hashVal;
        hashVal = HMAC(EVP_sha256(), key,strlen((char*)key), cipher, strlen((char*)cipher), NULL, NULL);
        FILE *tagFp = fopen(argv[itag], "w");
        fwrite(hashVal, 1, strlen(hashVal), tagFp);
        exit(0);
    }
    else if(!strcmp(argv[1], "dec")){
        FILE *inFp = fopen(argv[iin],"rb");
        char* cipher = readFile(argv[iin]);
        char* hashVal;
        hashVal = HMAC(EVP_sha256(), key,strlen(key), cipher, strlen(cipher), NULL, NULL);
        char* compareVal = readFile(argv[itag]);
        if(strcmp(compareVal, hashVal) != 0){
            printf("VERIFICATION FAILURE\n");
            exit(1);
        }

        FILE *outFp = fopen(argv[iout], "wb");
        int inLen, outLen;
        char inBuf[BUFSIZ], outBuf[BUFSIZ+EVP_MAX_BLOCK_LENGTH];
        EVP_CIPHER_CTX* ctx = EVP_CIPHER_CTX_new();
        EVP_DecryptInit_ex(ctx, EVP_aes_256_ctr(), NULL, key, iv);
        while((inLen=fread(inBuf, 1, sizeof(inBuf), inFp))>0){
            if(!EVP_DecryptUpdate(ctx, outBuf, &outLen, inBuf, inLen)){
                printf("ERROR\n");
                EVP_CIPHER_CTX_cleanup(ctx);
                exit(2);
            }
            fwrite(outBuf, 1, outLen, outFp);
        }
        if(!EVP_DecryptFinal_ex(ctx, outBuf, &outLen)){
            printf("ERROR\n");
            EVP_CIPHER_CTX_cleanup(ctx);
            exit(2);
        }
        fwrite(outBuf, 1, outLen, outFp);
        EVP_CIPHER_CTX_cleanup(ctx);
        fclose(inFp);
        fclose(outFp);
        exit(0);
    }
    exit(0);
}

Makefile #

cryp: cryp.o
    gcc -o $@ cryp.o -lssl -lcrypto

cryp.o: cryp.c

错误原因分析 #

1. 文本模式读取二进制文件：readFile用"r"文本模式读取加密文件（二进制）和密钥文件，会自动转换换行符，且fgets/strcat依赖\0结尾，二进制数据中存在的\0会导致读取内容不完整，直接破坏HMAC计算和加密解密流程。
2. HMAC处理错误：HMAC返回的是32字节二进制哈希值，用strlen获取长度会因为哈希中可能存在\0而得到错误长度；写入标签文件用"w"文本模式，会损坏二进制哈希数据。
3. 解密时文件指针位置错误：解密阶段先调用readFile读取加密文件，导致文件指针移动到末尾，后续fread无法读取任何数据，解密内容为空或乱码。
4. 流模式错误调用Final函数：AES-CTR是流加密模式，不需要填充，调用EVP_EncryptFinal_ex/EVP_DecryptFinal_ex无意义，虽不会直接报错，但可能引入异常数据。
5. 密钥读取异常：readFile会读取密钥文件的换行符，导致实际密钥长度超过AES-256要求的32字节，OpenSSL会自动截断，但这会导致加密解密使用的密钥不一致。

修复方案 #

1. 修复文件读取函数 #

改用二进制模式读取，直接一次性读取整个文件，避免文本模式转换和字符串操作的限制：

char* readFile(char* fileName, size_t *fileSize) {
    FILE *fp = fopen(fileName, "rb");
    if (!fp) {
        perror("Failed to open file");
        exit(2);
    }
    fseek(fp, 0, SEEK_END);
    *fileSize = ftell(fp);
    fseek(fp, 0, SEEK_SET);

    char* buffer = malloc(*fileSize);
    if (!buffer) {
        perror("Failed to allocate memory");
        fclose(fp);
        exit(2);
    }

    size_t readBytes = fread(buffer, 1, *fileSize, fp);
    if (readBytes != *fileSize) {
        perror("Failed to read file");
        free(buffer);
        fclose(fp);
        exit(2);
    }

    fclose(fp);
    return buffer;
}

2. 修正HMAC处理逻辑 #

使用二进制模式写入标签文件，用固定的SHA256_DIGEST_LENGTH（32字节）作为哈希长度：

// 加密时生成标签
size_t cipherSize;
char* cipher = readFile(argv[iout], &cipherSize);
unsigned char hashVal[SHA256_DIGEST_LENGTH];
unsigned int hashLen;
HMAC(EVP_sha256(), key, strlen(key), (unsigned char*)cipher, cipherSize, hashVal, &hashLen);

FILE *tagFp = fopen(argv[itag], "wb");
fwrite(hashVal, 1, hashLen, tagFp);
fclose(tagFp);
free(cipher);

3. 修复解密时的文件读取问题 #

解密时先读取加密文件到内存完成HMAC验证，再用内存中的密文数据进行解密，避免文件指针位置冲突：

else if(!strcmp(argv[1], "dec")){
    size_t cipherSize, tagSize;
    char* cipher = readFile(argv[iin], &cipherSize);
    unsigned char* tag = (unsigned char*)readFile(argv[itag], &tagSize);

    // 验证标签长度
    if (tagSize != SHA256_DIGEST_LENGTH) {
        printf("INVALID TAG SIZE\n");
        free(cipher);
        free(tag);
        exit(1);
    }

    // 计算HMAC
    unsigned char hashVal[SHA256_DIGEST_LENGTH];
    unsigned int hashLen;
    HMAC(EVP_sha256(), key, strlen(key), (unsigned char*)cipher, cipherSize, hashVal, &hashLen);

    // 比较标签（用memcmp而非strcmp，因为是二进制数据）
    if(memcmp(tag, hashVal, SHA256_DIGEST_LENGTH) != 0){
        printf("VERIFICATION FAILURE\n");
        free(cipher);
        free(tag);
        exit(1);
    }
    free(tag);

    // 解密内存中的密文
    FILE *outFp = fopen(argv[iout], "wb");
    int outLen;
    char outBuf[BUFSIZ+EVP_MAX_BLOCK_LENGTH];
    EVP_CIPHER_CTX* ctx = EVP_CIPHER_CTX_new();
    EVP_DecryptInit_ex(ctx, EVP_aes_256_ctr(), NULL, (unsigned char*)key, (unsigned char*)iv);
    
    // 分块解密内存中的数据
    size_t offset = 0;
    while (offset < cipherSize) {
        size_t chunkSize = (cipherSize - offset) > BUFSIZ ? BUFSIZ : (cipherSize - offset);
        if(!EVP_DecryptUpdate(ctx, (unsigned char*)outBuf, &outLen, (unsigned char*)cipher + offset, chunkSize)){
            printf("ERROR\n");
            EVP_CIPHER_CTX_cleanup(ctx);
            free(cipher);
            fclose(outFp);
            exit(2);
        }
        fwrite(outBuf, 1, outLen, outFp);
        offset += chunkSize;
    }

    EVP_CIPHER_CTX_cleanup(ctx);
    fclose(outFp);
    free(cipher);
    exit(0);
}

4. 移除流模式下的Final函数调用 #

AES-CTR是流模式，无需填充，删除EVP_EncryptFinal_ex和EVP_DecryptFinal_ex相关代码。

5. 确保密钥正确性 #

读取密钥后，手动去除末尾的换行符（如果存在），保证密钥长度为32字节：

size_t keySize;
char* key = readFile(argv[ikey], &keySize);
// 去除末尾的换行符或回车符
if (keySize > 0 && (key[keySize-1] == '\n' || key[keySize-1] == '\r')) {
    keySize--;
    key[keySize] = '\0';
}
// 验证密钥长度
if (keySize != 32) {
    printf("AES-256 requires 32-byte key\n");
    free(key);
    exit(2);
}

6. Makefile优化 #

添加编译警告，帮助提前发现问题：

CC = gcc
CFLAGS = -Wall -Wextra -O2

cryp: cryp.o
    $(CC) -o $@ cryp.o -lssl -lcrypto

cryp.o: cryp.c
    $(CC) $(CFLAGS) -c $<

clean:
    rm -f cryp cryp.o

验证步骤 #

1. 确保shared.key是32字节的二进制密钥文件（无换行符），可通过head -c 32 /dev/urandom > shared.key生成。
2. 重新编译：make clean && make
3. 执行加密解密指令，对比original.txt和decrypted.txt内容：diff original.txt decrypted.txt，无输出则表示内容一致。

内容的提问来源于stack exchange，提问作者Hysperion