WINUI 3中Hook WndProc遇SystemEngineException问题求解决方案

问题背景 #

本人此前在VB3至C# WinForms环境中Hook WndProc均无问题，但在WINUI 3+C#环境下遇到模糊的SystemEngineException（错误码-2146233082），无详细信息，需要处理WM_COPYDATA消息。

用户原有代码 #

DLL导入代码 #

/// <summary>
/// Sets window data.
/// </summary>
/// <param name="hWnd">The handle of the window to set.</param>
/// <param name="nIndex">The index of the item to set.</param>
/// <param name="dwNewLong">The new value.</param>
/// <returns></returns>
[DllImport("user32.dll",
    EntryPoint = "SetWindowLongPtr",
    CallingConvention = CallingConvention.Cdecl)]
private static extern IntPtr SetWindowLongPtr64(HandleRef hWnd, int nIndex, IntPtr dwNewLong);

// Defines our window WndProcDelegate.
private delegate IntPtr WndProcDelegate(IntPtr hwnd, uint message, IntPtr wParam, IntPtr lParam);

// This handles calling the underlying base WndProc.
[DllImport("user32.dll", 
    CallingConvention = CallingConvention.Cdecl,
    CharSet=CharSet.Auto)]
static extern IntPtr CallWindowProc(WndProcDelegate lpPrevWndFunc, IntPtr hWnd, uint Msg, IntPtr wParam, IntPtr lParam);

窗口子类化与取消子类化实现代码 #

public IntPtr GetHwnd()
{
    IntPtr handle = WinRT.Interop.WindowNative.GetWindowHandle(this.m_MainWindow);
    return handle;
}

// Subclass the window.
public void Subclass(object window)
{
    if (this.m_isSubclassed)
    {
        return;
    }

    this.m_MainWindow = window;
    this.m_NewWndProcDelegate = new WndProcDelegate(this.ReplacementWndProc);
    IntPtr hWnd = this.GetHwnd();
    HandleRef handleRef = new(this, hWnd);
    IntPtr newDelegatePtr = Marshal.GetFunctionPointerForDelegate(this.m_NewWndProcDelegate);
    IntPtr oldDelegatePtr = SetWindowLongPtr64(handleRef, GWLP_WNDPROC, newDelegatePtr);
    this.m_OldWndProcDelegate = (WndProcDelegate)Marshal.GetDelegateForFunctionPointer(
        oldDelegatePtr,
        typeof(WndProcDelegate));
    this.m_isSubclassed = true;
}

// Unsubclass the window.
public void Unsubclass()
{
    if (!this.m_isSubclassed)
    {
        return;
    }
    IntPtr hWnd = this.GetHwnd();
    SetWindowLongPtr64(new HandleRef(this, hWnd), GWLP_WNDPROC, Marshal.GetFunctionPointerForDelegate(this.m_OldWndProcDelegate));
    this.m_isSubclassed = false;
}

public bool IsSubclassed()
{
    return this.m_isSubclassed;
}


/// <summary>
/// This is the replacement WndProc.
/// </summary>
/// <param name="hwnd"></param>
/// <param name="message"></param>
/// <param name="wParam"></param>
/// <param name="lParam"></param>
/// <returns></returns>
private IntPtr ReplacementWndProc(IntPtr hwnd, uint message, IntPtr wParam, IntPtr lParam)
{
    // Our custom Windows messages code...
    /*if (message == WM_COPYDATA)
    {
        CopyDataStruct copyStruct = (CopyDataStruct)Marshal.PtrToStructure(lParam, typeof(CopyDataStruct));
        string messageText = Utf8PtrToString(copyStruct.lpData);
        int cbData = copyStruct.cbData;
        // Do something with the messageText and cbData.
    }*/

    // Finally, call the original WndProc.
    return CallWindowProc(this.m_OldWndProcDelegate, hwnd, message, wParam, lParam);
}

经过验证的WinUI 3实现方法 #

WinUI 3中不推荐直接用SetWindowLongPtr子类化窗口，推荐使用SetWindowSubclass和RemoveWindowSubclassAPI，这些API专门用于安全的窗口子类化，能避免委托被GC回收、调用约定不匹配等问题。

完整实现代码 #

using System;
using System.Runtime.InteropServices;
using WinRT.Interop;

public class WindowMessageHandler
{
    private const uint WM_COPYDATA = 0x004A;
    private const uint WM_NCDESTROY = 0x0082;

    // 子类化相关API
    [DllImport("comctl32.dll", CharSet = CharSet.Unicode)]
    private static extern bool SetWindowSubclass(IntPtr hWnd, SubclassProcDelegate pfnSubclass, uint uIdSubclass, IntPtr dwRefData);

    [DllImport("comctl32.dll", CharSet = CharSet.Unicode)]
    private static extern bool RemoveWindowSubclass(IntPtr hWnd, SubclassProcDelegate pfnSubclass, uint uIdSubclass);

    [DllImport("comctl32.dll", CharSet = CharSet.Unicode)]
    private static extern IntPtr DefSubclassProc(IntPtr hWnd, uint uMsg, IntPtr wParam, IntPtr lParam);

    private delegate IntPtr SubclassProcDelegate(IntPtr hWnd, uint uMsg, IntPtr wParam, IntPtr lParam, uint uIdSubclass, IntPtr dwRefData);

    private SubclassProcDelegate _subclassProc;
    private IntPtr _windowHandle;
    private bool _isSubclassed;
    private const uint _subclassId = 1; // 自定义子类ID

    public void Subclass(object window)
    {
        if (_isSubclassed) return;

        _windowHandle = WindowNative.GetWindowHandle(window);
        _subclassProc = new SubclassProcDelegate(SubclassProc);
        SetWindowSubclass(_windowHandle, _subclassProc, _subclassId, IntPtr.Zero);
        _isSubclassed = true;
    }

    public void Unsubclass()
    {
        if (!_isSubclassed) return;

        RemoveWindowSubclass(_windowHandle, _subclassProc, _subclassId);
        _isSubclassed = false;
    }

    public bool IsSubclassed() => _isSubclassed;

    private IntPtr SubclassProc(IntPtr hWnd, uint uMsg, IntPtr wParam, IntPtr lParam, uint uIdSubclass, IntPtr dwRefData)
    {
        switch (uMsg)
        {
            case WM_COPYDATA:
                // 处理WM_COPYDATA消息
                COPYDATASTRUCT copyData = Marshal.PtrToStructure<COPYDATASTRUCT>(lParam);
                string message = Marshal.PtrToStringUni(copyData.lpData);
                // 此处添加自定义处理逻辑
                break;
            case WM_NCDESTROY:
                // 窗口销毁时自动取消子类化，避免内存泄漏
                Unsubclass();
                break;
        }

        // 调用默认子类处理流程
        return DefSubclassProc(hWnd, uMsg, wParam, lParam);
    }

    [StructLayout(LayoutKind.Sequential)]
    private struct COPYDATASTRUCT
    {
        public IntPtr dwData;
        public int cbData;
        public IntPtr lpData;
    }
}

关键改进点 #

• 用SetWindowSubclass替代SetWindowLongPtr：该API由ComCtl32提供，专门用于安全的窗口子类化，内部会管理委托引用，避免被GC回收。
• 自动清理资源：在WM_NCDESTROY消息中自动取消子类化，确保窗口销毁时释放资源，防止内存泄漏。
• 使用DefSubclassProc替代CallWindowProc：符合子类化标准流程，避免调用约定不匹配问题。

内容的提问来源于stack exchange，提问作者Jason Coulls