使用XOR加密Payload未生效的问题求助（Visual Studio 2022）

我正在开发一个基于XOR的简易Payload加密项目，已将Shellcode加载到.rsrc节，但在Visual Studio 2022调试时发现pTmpBuffer中的Shellcode未被加密，保持原样。实现代码如下：

#include <Windows.h>
#include <stdio.h>
#include "resource.h"

VOID XorByOneKey(IN PBYTE pShellCode, IN SIZE_T sShellCodeSize, IN BYTE bKey) {
    for (size_t i = 0; i < sShellCodeSize; i++) {
        pShellCode[i] = pShellCode[i] ^ bKey;
    }
}

int main() {
    HRSRC hRsrc = NULL;
    HGLOBAL hGlobal = NULL;
    PVOID pPayloadAddress = NULL;
    SIZE_T sPayloadSize = NULL;

    // Get the location to the data stored in .rsrc by its id *IDR_RCDATA1*
    hRsrc = FindResourceW(NULL, MAKEINTRESOURCEW(IDR_RCDATA1), RT_RCDATA);
    if (hRsrc == NULL) {
        // in case of function failure
        printf("[!] FindResourceW Failed with Error : %d\n", GetLastError());
        return EXIT_FAILURE;
    }

    // Get the handle of the specified resource data since it is required to lock resource later
    hGlobal = LoadResource(NULL, hRsrc);
    if (hGlobal == NULL) {
        // in case of function failure
        printf("[!] LoadResource Failed with Error : %d\n", GetLastError());
        return EXIT_FAILURE;
    }

    // Get the address of our payload in .rsrc section
    pPayloadAddress = LockResource(hGlobal);
    if (pPayloadAddress == NULL) {
        // in case of function failure
        printf("[!] LockResource Failed With Error : %d\n", GetLastError());
        return EXIT_FAILURE;
    }

    // Get the size of our payload in .rsrc section
    sPayloadSize = SizeofResource(NULL, hRsrc);
    if (sPayloadSize == NULL) {
        // in case of function failure
        printf("[!] SizeofResource Failed with Error : %d\n ", GetLastError());
        return EXIT_FAILURE;
    }

    PVOID pTmpBuffer = HeapAlloc(GetProcessHeap(), 0, sPayloadSize);
    if (pTmpBuffer != NULL) {
        // copying the payload from the resource section
        memcpy(pTmpBuffer, pPayloadAddress, sPayloadSize);
    }
    int i = 0;
    XorByOneKey((PBYTE)pTmpBuffer, sPayloadSize, i);

    // Printing the base address of the buffer
    printf("[i] pTmpBuffer var : 0x%p \n", pTmpBuffer);

    // Printing pointer and size to screen
    printf("[i] sPayloadAddress var : 0x%p \n", pPayloadAddress);
    printf("[i] sPayloadSize var : 0x%ld \n", sPayloadSize);
    getchar();

    return EXIT_SUCCESS;
}

问题根源 #

1. XOR密钥为0，导致加密无效果
   代码中使用int i = 0;作为XOR密钥，而XOR运算的特性是：任何值与0异或结果等于原值，因此加密操作完全没有改变Shellcode内容。

2. SIZE_T类型变量初始化/判断逻辑错误
   SIZE_T是无符号整数类型，不应使用NULL初始化或判断，正确的初始值应为0。

3. 缺失HeapAlloc失败的错误处理
   如果内存分配失败，pTmpBuffer会是NULL，后续调用XorByOneKey会触发未定义行为。

修复后的代码 #

#include <Windows.h>
#include <stdio.h>
#include "resource.h"

VOID XorByOneKey(IN PBYTE pShellCode, IN SIZE_T sShellCodeSize, IN BYTE bKey) {
    for (size_t i = 0; i < sShellCodeSize; i++) {
        pShellCode[i] ^= bKey; // 简化写法，效果与原代码一致
    }
}

int main() {
    HRSRC hRsrc = NULL;
    HGLOBAL hGlobal = NULL;
    PVOID pPayloadAddress = NULL;
    SIZE_T sPayloadSize = 0; // 无符号类型初始化为0

    // 查找资源
    hRsrc = FindResourceW(NULL, MAKEINTRESOURCEW(IDR_RCDATA1), RT_RCDATA);
    if (hRsrc == NULL) {
        printf("[!] FindResourceW Failed with Error : %d\n", GetLastError());
        return EXIT_FAILURE;
    }

    // 加载资源
    hGlobal = LoadResource(NULL, hRsrc);
    if (hGlobal == NULL) {
        printf("[!] LoadResource Failed with Error : %d\n", GetLastError());
        return EXIT_FAILURE;
    }

    // 锁定资源获取地址
    pPayloadAddress = LockResource(hGlobal);
    if (pPayloadAddress == NULL) {
        printf("[!] LockResource Failed With Error : %d\n", GetLastError());
        return EXIT_FAILURE;
    }

    // 获取资源大小
    sPayloadSize = SizeofResource(NULL, hRsrc);
    if (sPayloadSize == 0) { // 无符号类型判断是否为0
        printf("[!] SizeofResource Failed with Error : %d\n ", GetLastError());
        return EXIT_FAILURE;
    }

    // 分配内存并检查结果
    PVOID pTmpBuffer = HeapAlloc(GetProcessHeap(), 0, sPayloadSize);
    if (pTmpBuffer == NULL) {
        printf("[!] HeapAlloc Failed with Error : %d\n", GetLastError());
        return EXIT_FAILURE;
    }

    // 复制Payload到临时缓冲区
    memcpy(pTmpBuffer, pPayloadAddress, sPayloadSize);

    // 使用非0密钥执行XOR加密
    BYTE bKey = 0x41; // 自定义非0密钥，可根据需求修改
    XorByOneKey((PBYTE)pTmpBuffer, sPayloadSize, bKey);

    // 打印基础信息
    printf("[i] pTmpBuffer var : 0x%p \n", pTmpBuffer);
    printf("[i] sPayloadAddress var : 0x%p \n", pPayloadAddress);
    printf("[i] sPayloadSize var : 0x%llu \n", sPayloadSize); // 64位系统用%llu适配SIZE_T

    // 可选：打印加密前后的前4字节，验证加密效果
    printf("\n[*] Original first 4 bytes: 0x%02X 0x%02X 0x%02X 0x%02X\n",
           ((PBYTE)pPayloadAddress)[0], ((PBYTE)pPayloadAddress)[1],
           ((PBYTE)pPayloadAddress)[2], ((PBYTE)pPayloadAddress)[3]);
    printf("[*] Encrypted first 4 bytes: 0x%02X 0x%02X 0x%02X 0x%02X\n",
           ((PBYTE)pTmpBuffer)[0], ((PBYTE)pTmpBuffer)[1],
           ((PBYTE)pTmpBuffer)[2], ((PBYTE)pTmpBuffer)[3]);

    getchar();

    // 释放内存，避免泄漏
    HeapFree(GetProcessHeap(), 0, pTmpBuffer);

    return EXIT_SUCCESS;
}

关键修复点说明 #

• 更换为非0的BYTE类型密钥，确保XOR操作能修改Shellcode内容；
• 修正SIZE_T变量的初始化和判断逻辑，符合无符号整数的使用规范；
• 添加HeapAlloc的错误处理，避免空指针操作；
• 增加加密前后字节对比的打印代码，可直观验证加密是否生效；
• 添加内存释放逻辑，避免内存泄漏。

内容的提问来源于stack exchange，提问作者koiboi