如何在C#中绕过Shadow-Root层爬取VISA货币计算器页面？

Shadow-DOM会隔离页面内部元素，Selenium原生API无法直接定位，必须通过JavaScript执行脚本穿透Shadow层访问元素。以下是针对VISA汇率计算器页面的完整解决方案：

核心思路 #

通过IJavaScriptExecutor执行DOM脚本，先获取dm-calculator组件的shadowRoot，再定位内部的输入框、下拉选单等元素，完成填充和交互操作。

修正后的完整代码 #

using OpenQA.Selenium;
using OpenQA.Selenium.Chrome;
using OpenQA.Selenium.Support.UI;
using System;
using System.Collections.Generic;
using System.Threading;

class Program
{
    static void Main(string[] args)
    {
        ChromeOptions options = new ChromeOptions();
        options.AddArgument("--disable-infobars");
        options.AddArgument("--disable-extensions");
        options.AddArgument("--disable-gpu");
        options.AddArgument("--no-sandbox");
        options.AddArgument("--disable-dev-shm-usage");
        options.AddArgument("--start-maximized");

        using (var driver = new ChromeDriver(options))
        {
            driver.Navigate().GoToUrl("https://www.visa.co.uk/support/consumer/travel-support/exchange-rate-calculator.html");
            var wait = new WebDriverWait(driver, TimeSpan.FromSeconds(10));

            // 等待dm-calculator组件加载完成
            wait.Until(d => (IWebElement)d.ExecuteScript("return document.querySelector('dm-calculator')"));

            // 1. 填充交易金额
            var amountInput = (IWebElement)driver.ExecuteScript(
                "return document.querySelector('dm-calculator').shadowRoot.querySelector('#amount')");
            amountInput.Clear();
            amountInput.SendKeys("100");

            // 2. 设置"From"货币（示例设为USD）
            var fromCurrencySelect = (IWebElement)driver.ExecuteScript(
                "return document.querySelector('dm-calculator').shadowRoot.querySelector('#fromCurrency')");
            new SelectElement(fromCurrencySelect).SelectByValue("USD");

            // 3. 遍历目标货币，逐个设置"To"货币并获取结果
            List<string> targetCurrencies = new List<string>
            {
                "AFN", "ALL", "DZD", "AOA", "ARS", "AMD", "AWG", "AUD", "AZN", "BSD",
                "BHD", "BDT", "BBD", "BYN", "BZD", "BMD", "BTN", "BOB", "BAM", "BWP",
                "BRL", "BND", "BGN", "BIF", "KHR", "CAD", "CVE", "KYD", "XOF", "XAF",
                "XPF", "CLP", "CNY", "COP", "KMF", "CDF", "CRC", "CUP", "CZK", "DKK",
                "DJF", "DOP", "XCD", "EGP", "SVC", "ETB", "EUR", "FKP", "FJD", "GMD",
                "GEL", "GHS", "GIP", "GBP", "GTQ", "GNF", "GYD", "HTG", "HNL", "HKD",
                "HUF", "ISK", "INR", "IDR", "IQD", "ILS", "JMD", "JPY", "JOD", "KZT",
                "KES", "KWD", "KGS", "LAK", "LBP", "LSL", "LRD", "LYD", "MOP", "MKD",
                "MGA", "MWK", "MYR", "MVR", "MRU", "MUR", "MXN", "MDL", "MNT", "MAD",
                "MZN", "MMK", "NAD", "NPR", "ANG", "NZD", "NIO", "NGN", "NOK", "OMR",
                "PKR", "PAB", "PGK", "PYG", "PEN", "PHP", "PLN", "QAR", "RON", "RUB",
                "RWF", "SHP", "WST", "STN", "SAR", "RSD", "SCR", "SLE", "SGD", "SBD",
                "SOS", "ZAR", "KRW", "SSP", "LKR", "SDG", "SRD", "SZL", "SEK", "CHF",
                "TWD", "TJS", "TZS", "THB", "TOP", "TTD", "TND", "TRY", "TMT", "UGX",
                "UAH", "AED", "USD", "UYU", "UZS", "VUV", "VES", "VND", "YER", "ZMW",
                "ZWL"
            };

            foreach (string currency in targetCurrencies)
            {
                try
                {
                    // 设置"To"货币
                    var toCurrencySelect = (IWebElement)driver.ExecuteScript(
                        "return document.querySelector('dm-calculator').shadowRoot.querySelector('#toCurrency')");
                    new SelectElement(toCurrencySelect).SelectByValue(currency);

                    // 等待结果加载并获取汇率转换结果
                    Thread.Sleep(500); // 简单等待，也可用WebDriverWait等待结果元素出现
                    var resultText = (string)driver.ExecuteScript(
                        "return document.querySelector('dm-calculator').shadowRoot.querySelector('.result-value').textContent");
                    Console.WriteLine($"100 USD = {resultText} {currency}");
                }
                catch (Exception ex)
                {
                    Console.WriteLine($"处理货币 {currency} 时出错: {ex.Message}");
                }
            }

            driver.Quit();
        }
    }
}

关键说明 #

• 等待元素加载：使用WebDriverWait确保dm-calculator组件加载完成，避免因页面异步渲染导致的元素找不到问题。
• Shadow-DOM元素定位：通过document.querySelector('dm-calculator').shadowRoot获取Shadow根节点，再用常规CSS选择器定位内部元素。
• 下拉选单处理：将获取到的下拉框元素转为SelectElement，直接通过SelectByValue设置目标货币，比模拟点击更高效。
• 结果获取：转换后等待短时间（或用WebDriverWait等待结果元素），再通过脚本提取结果文本。

内容的提问来源于stack exchange，提问作者N._.coDev