多进程安装全局WH_KEYBOARD钩子致首个进程失效的解决方案咨询

需求背景 #

• 开发一款通过USB键盘类配件发送命令的产品，不同配件按键组不同，需根据发送按键的配件做出对应响应，同时拦截这些配件的按键事件，不让系统其他部分接收。

当前实现方案 #

• 采用Raw Input + Windows Hooks组合方案：C++编写WH_KEYBOARD全局钩子（满足全局钩子的原生语言要求），C#处理Raw Input和业务逻辑判断。
• 托管应用加载原生DLL，安装WH_KEYBOARD钩子并指定基于WM_APP的自定义消息码。
• 钩子回调向托管应用发送消息，应用通过Raw Input识别键盘来源，判断是否拦截该按键：需要拦截则返回1，否则调用CallNextHookEx传递事件。
• 单进程单配件场景下运行正常。

多进程问题 #

启动第二个进程（仅自定义消息码不同）后，首个进程无法接收钩子发来的WM_APP + MessageId消息，仅能接收WM_INPUT；第二个进程可正常接收消息。根据Windows钩子文档，同类钩子应加入钩子链，各进程回调都能处理事件，但当前不符合预期。

相关代码 #

原生代码（NativeHook.dll） #

LRESULT CALLBACK KeyboardProc(int nCode, WPARAM wParam, LPARAM lParam)
{
    fprintf(stdout, "proc\n");
    if (nCode < 0)
    {
        return CallNextHookEx(hookHandle, nCode, wParam, lParam);
    }
    USHORT keyPressed = lParam & 0x80000000 ? 0 : 1;
    if (SendMessage(windowHandle, MSGID, wParam, keyPressed))
    {
        fprintf(stdout, "block\n");
        return TRUE;
    }
    fprintf(stdout, "next\n");

    return CallNextHookEx(hookHandle, nCode, wParam, lParam);
}


BOOL HookUp(HWND handle, UINT msgid, DWORD threadId)
{
    if (windowHandle != NULL)
    {
        // already hooked
        return 2;
    }

    hookHandle = SetWindowsHookEx(WH_KEYBOARD, (HOOKPROC)KeyboardProc, dllInstance, threadId);
    if (hookHandle == NULL)
    {
        DWORD errorCode = GetLastError();
        fprintf(stdout, "hooking error code: %d\n", errorCode);
        return errorCode;
    }

    MSGID = msgid;
    windowHandle = handle;
    fprintf(stdout, "hook handle: %p; window handle: %p; MSGID: %d\n", hookHandle, windowHandle, MSGID);
    
    return TRUE;
}

BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved)
{
    switch (fdwReason)
    {
    case DLL_PROCESS_ATTACH:
        dllInstance = hinstDLL;
        AllocConsole(); // Enable the console
        freopen_s((FILE**)stdout, "CONOUT$", "w", stdout);
        freopen_s((FILE**)stdin, "CONIN$", "r", stdin);
        fprintf(stdout, "%p\n", hinstDLL);
        break;
    case DLL_PROCESS_DETACH:
        UnhookWindowsHookEx(hookHandle);
        fprintf(stdout, "%p detached (Window: %p)\n", dllInstance, windowHandle);
        break;
    default:
        break;
    }

    return TRUE;
}

托管代码相关部分 #

[DllImport("user32.dll", SetLastError = true)]
public static extern int GetWindowThreadProcessId(IntPtr hWnd, IntPtr processId);

[DllImport("NativeHook.dll", EntryPoint = "HookUp", SetLastError = true, CallingConvention = CallingConvention.Cdecl)]
public static extern int HookUp(IntPtr appHandle, uint msgid, uint threadId);

// zero for the window handle and the process ID will install the hook globally 
int threadId = NativeResources.GetWindowThreadProcessId(IntPtr.Zero, IntPtr.Zero);

var hooked = NativeResources.HookUp(this.Handle, WM_APP + (uint)MessageId, (uint)threadId);

// ...

protected override void WndProc(ref Message m)
{
    if (m.Msg == WM_INPUT)
    {
        // raw input will allow the identification of which keyboard generated the input
        base.WndProc(ref m);
    }
    // MessageId is a property initialized based on the device to be monitored
    else if (m.Msg == WM_APP + MessageId)
    {
        IntPtr intercept = new IntPtr(1);
        // the value for _block will depend on a logic that will check if there's a raw input of the same key AND if that key was pressed from the desired keyboard
        if (_block)
        {
            // setting an IntPtr with 1 means block to the hook callback on the native code
            m.Result = intercept;
        }
        else
        {
            // calling base.WndProc means that the hook callback should call CallNextHook
            base.WndProc(ref m);
        }
    }
    else
    {
        base.WndProc(ref m);
    }
}

问题确认 #

即使移除托管代码中的逻辑判断，仅对两类消息调用base.WndProc，启动第二个进程后，首个进程仍无法获取WM_APP + MessageId消息，仅能接收WM_INPUT。

提问 #

需要对钩子安装做哪些特殊处理，才能让钩子在多进程环境下正常运行，实现多键盘/配件的同时监控？

内容的提问来源于stack exchange，提问作者allw