如何查找Electron应用iframe内部的指定UI元素？

问题描述 #

我使用了UIA查找UI元素示例中相似的代码，用于获取窗口内的特定元素，但若元素位于Electron应用内部的iframe中时，代码无法正常工作。
但我运行Accessibility Insights手动检查Slack、Teams等Electron应用后，再执行代码就能准确找到iframe内的目标元素。
我的代码如下：

int main(int argc, char *argv[])
{
    HRESULT hr;
    int ret = 0;
    IUIAutomationElement *pTargetElement = NULL;

    CoInitializeEx(NULL, COINIT_MULTITHREADED);

    IUIAutomationElement *pRootElement;

    std::string elementName = "Missed";
    CComBSTR temp(elementName.c_str());

    std::string inputWindowTitle = "Microsoft Teams";
    CA2T wt (inputWindowTitle.c_str());
    HWND hwnd = ::FindWindow(NULL, wt);

    IUIAutomation *pAutomation = NULL;
    hr = CoCreateInstance(__uuidof(CUIAutomation), NULL, CLSCTX_INPROC_SERVER, __uuidof(IUIAutomation), (void **)&pAutomation);
    if (FAILED(hr) || pAutomation == NULL)
    {
        ret = 1;
        goto cleanup;
    }

    PROPERTYID pPIDProperties[] = {UIA_BoundingRectanglePropertyId};

    IUIAutomationCondition *pCondition;
    VARIANT varProperty;
    VariantInit(&varProperty);
    varProperty.vt = VT_BSTR;
    varProperty.bstrVal = temp;

    SetForegroundWindow(hwnd);
    SetFocus(hwnd);
    SetActiveWindow(hwnd);
    
    if (SUCCEEDED(pAutomation->ElementFromHandle(hwnd, &pRootElement)))
    {
        if (SUCCEEDED(pAutomation->CreatePropertyCondition(UIA_NamePropertyId, varProperty, &pCondition)))
        {
            if (SUCCEEDED(pRootElement->FindFirst(TreeScope_Descendants, pCondition, &pTargetElement)))
            {

                if (pTargetElement != NULL)
                {
                    BSTR nn;
                    hr = pTargetElement->get_CurrentName(&nn);
                    wprintf(L">> ELEMENT FOUND (%s)! ", nn);

                } else {
                    wprintf(L">> ELEMENT IS NULL");
                    goto cleanup;
                }
            } else {
                wprintf(L">> ELEMENT NOT FOUND! ");
                goto cleanup;
            }
        } else {
            wprintf(L">> SOMETHING WRONG WITH CONDITION ");
            goto cleanup;
        }
    }
    else {
        wprintf(L">> NO WINDOW ");
        goto cleanup;
    }
cleanup:
    if (pTargetElement != NULL)
        pTargetElement->Release();

    if (pAutomation != NULL)
        pAutomation->Release();

    CoUninitialize();
    return ret;
}

请问应该如何正确解决该问题？为什么经过Accessibility Insights交互后就可以找到对应元素，背后是否存在缓存机制？

原因说明 #

该现象由Electron的UIA提供器懒加载机制导致，和系统级UIA缓存无关：
Electron默认仅暴露顶层窗口的基础UIA节点，内部Web内容（包含iframe）的UIA节点不会随程序启动默认构建，仅当检测到UIA客户端主动扫描当前窗口后代节点时，才会触发Web内容到UIA节点的转换。
Accessibility Insights扫描目标窗口时会全量遍历所有UIA后代节点，触发了Electron的UIA节点构建逻辑，后续代码查询时节点已存在，因此可以正常命中目标元素。

解决方案 #

可通过以下修改修复问题：

• 新增UIA节点预热逻辑：执行FindFirst查询前，先主动遍历一次根节点下的子节点，唤醒Electron的UIA构建逻辑
• 配置CacheRequest强制拉取最新节点：主动声明需要查询的属性，绕过默认的UIA缓存逻辑，强制获取实时节点数据
• 增加轻量重试逻辑：首次查询不到时，间隔100ms重试，最多重试3次，兼容Electron节点构建的异步延迟

修改后的参考代码 #

int main(int argc, char *argv[])
{
    HRESULT hr;
    int ret = 0;
    IUIAutomationElement *pTargetElement = NULL;

    CoInitializeEx(NULL, COINIT_MULTITHREADED);

    IUIAutomationElement *pRootElement;

    std::string elementName = "Missed";
    CComBSTR temp(elementName.c_str());

    std::string inputWindowTitle = "Microsoft Teams";
    CA2T wt (inputWindowTitle.c_str());
    HWND hwnd = ::FindWindow(NULL, wt);

    IUIAutomation *pAutomation = NULL;
    hr = CoCreateInstance(__uuidof(CUIAutomation), NULL, CLSCTX_INPROC_SERVER, __uuidof(IUIAutomation), (void **)&pAutomation);
    if (FAILED(hr) || pAutomation == NULL)
    {
        ret = 1;
        goto cleanup;
    }

    VARIANT varProperty;
    VariantInit(&varProperty);
    varProperty.vt = VT_BSTR;
    varProperty.bstrVal = temp;

    SetForegroundWindow(hwnd);
    SetFocus(hwnd);
    SetActiveWindow(hwnd);
    
    if (SUCCEEDED(pAutomation->ElementFromHandle(hwnd, &pRootElement)))
    {
        // 新增：创建CacheRequest强制拉取最新节点
        IUIAutomationCacheRequest *pCacheReq = NULL;
        pAutomation->CreateCacheRequest(&pCacheReq);
        pCacheReq->AddProperty(UIA_NamePropertyId);
        pCacheReq->put_TreeScope(TreeScope_Descendants);
        pCacheReq->Push();

        // 新增：预热节点，主动触发一次子节点遍历
        IUIAutomationElementArray *pChildren = NULL;
        pRootElement->FindAll(TreeScope_Children, pAutomation->CreateTrueCondition(), &pChildren);
        if (pChildren) pChildren->Release();

        // 新增：最多重试3次查询
        int retryCnt = 0;
        IUIAutomationCondition *pCondition = NULL;
        while (retryCnt < 3 && pTargetElement == NULL) {
            if (SUCCEEDED(pAutomation->CreatePropertyCondition(UIA_NamePropertyId, varProperty, &pCondition)))
            {
                pRootElement->FindFirst(TreeScope_Descendants, pCondition, &pTargetElement);
                pCondition->Release();
            }
            if (!pTargetElement) {
                Sleep(100);
                retryCnt++;
            }
        }

        pCacheReq->Pop();
        pCacheReq->Release();

        if (pTargetElement != NULL)
        {
            BSTR nn;
            hr = pTargetElement->get_CurrentName(&nn);
            wprintf(L">> ELEMENT FOUND (%s)! ", nn);
            SysFreeString(nn);
        } else {
            wprintf(L">> ELEMENT NOT FOUND! ");
            ret = 1;
            goto cleanup;
        }
    }
    else {
        wprintf(L">> NO WINDOW ");
        ret = 1;
        goto cleanup;
    }
cleanup:
    VariantClear(&varProperty);
    if (pTargetElement != NULL)
        pTargetElement->Release();
    if (pRootElement != NULL)
        pRootElement->Release();
    if (pAutomation != NULL)
        pAutomation->Release();

    CoUninitialize();
    return ret;
}

内容的提问来源于stack exchange，提问作者N.Sto