使用WebKit获取Instagram Reel封面图：Swift实现遇阻求助

我用WebKit尝试获取Instagram Reel的封面图，能正常获取个人头像，但拿不到Reel的图片。已经确认每个Reel有唯一URI标识符，在浏览器控制台写的JavaScript代码能成功拿到图片URL，但对应的Swift代码始终失败。用JS控制台和Python+Selenium都没问题，Swift版本就是不行，求技术帮助。

成功运行的JavaScript代码（对应Reel链接：https://www.instagram.com/p/CyzX0mCJtmn/） #

(function() {
    var anchorElement = document.querySelector('a[href="/p/CyzX0mCJtmn/"]');
    if (anchorElement) {
        var imgElement = anchorElement.querySelector('div._aagu img');
        return imgElement ? imgElement.getAttribute('src') : "Image not found";
    }
    return "No anchor element found with identifier: CyzX0mCJtmn";
})();

当前使用的Swift代码 #

import SwiftUI
import WebKit

struct ContentView: View {
    @State private var urlInput: String = ""
    @ObservedObject var webViewStore = WebViewStore()
    var body: some View {
        VStack {
            Spacer() // Push content to the bottom
            HStack(alignment: .top) {
                if let uiImage = webViewStore.fetchedUIImage {
                    Image(uiImage: uiImage)
                        .resizable()
                        .scaledToFit()
                        .frame(width: 100, height: 100)
                }
            }
            Spacer()
            Divider()

            HStack {
                TextField("Enter URL", text: $urlInput)
                    .padding()
                    .textFieldStyle(RoundedBorderTextFieldStyle())

                Button("Load") {
                    if let validURL = URL(string: urlInput) {
                        webViewStore.load(url: validURL)
                    }
                }
            }
        }
        .padding()
    }
}

class WebViewStore: NSObject, ObservableObject, WKNavigationDelegate {
    
    @Published var fetchedUIImage: UIImage?
    var uniqueIdentifier: String?  // Store the unique identifier
    var webView: WKWebView = {
        let webConfiguration = WKWebViewConfiguration()
        let webView = WKWebView(frame: .zero, configuration: webConfiguration)
        return webView
    }()
    
    override init() {
        super.init()
        webView.navigationDelegate = self
    }
    
    func load(url: URL) {
        // Extract the unique identifier from the given URL
        let components = url.absoluteString.components(separatedBy: "/")
        if components.count > 4 {
            uniqueIdentifier = components[4]
        }
        
        let request = URLRequest(url: url)
        webView.load(request)
    }
    
    func webView(_ webView: WKWebView, didFinish navigation: WKNavigation!) {
        // Wait for a few seconds to give the webpage additional time to load before attempting to fetch the image URL
        DispatchQueue.global().asyncAfter(deadline: .now() + 7.0) {
            self.attemptFetchingDescription(retryCount: 3)
        }
    }
    
    private func attemptFetchingDescription(retryCount: Int) {
        if retryCount <= 0 {
            print("Max retry count reached.")
            return
        }
        
        let checkScript = "document.body.innerHTML.includes('Instagram')"
        
        DispatchQueue.main.async {
            self.webView.evaluateJavaScript(checkScript) { (result, error) in
                if let isPresent = result as? Bool, isPresent {
                    if let urlStr = self.webView.url?.absoluteString {
                        self.fetchImageURL(from: urlStr, retryCount: retryCount)
                    }

                } else {
                    print("Did not find 'Instagram' in the body. Retrying... Remaining retries: \(retryCount-1)")
                    DispatchQueue.global().asyncAfter(deadline: .now() + 1.0) {
                        self.attemptFetchingDescription(retryCount: retryCount - 1)
                    }
                }
            }
        }
    }
    private func fetchImageURL(from url: String, retryCount: Int) {
        let identifier = url.components(separatedBy: "/").filter { !$0.isEmpty }.last ?? ""
        let fetchImageScript = """
        (function() {
            var anchorElement = document.querySelector('a[href="/p/\(identifier)/"]');
            if (anchorElement) {
                var imgElement = anchorElement.querySelector('div._aagu img');
                return imgElement ? imgElement.getAttribute('src') : "Image not found";
            }
            return "No anchor element found with identifier: \(identifier)";
        })();
        """
        self.webView.evaluateJavaScript(fetchImageScript) { (fetchResult, fetchError) in
            if let link = fetchResult as? String, link != "Image not found", let validURL = URL(string: link) {
                print("Successfully fetched image URL: \(link)")
                self.loadImage(from: validURL)
            } else {
                print("Failed to fetch image URL with error: \(fetchResult ?? "Unknown error")")
                if retryCount > 0 {
                    self.retryFetching(retryCount: retryCount - 1)
                } else {
                    print("Max retry attempts reached.")
                }
            }
        }
    }
    private func retryFetching(retryCount: Int) {
        print("Retrying... Remaining retries: \(retryCount-1).")
        DispatchQueue.global().asyncAfter(deadline: .now() + 3.0) {
            self.attemptFetchingDescription(retryCount: retryCount - 1)
        }
    }
    private func loadImage(from url: URL) {
        URLSession.shared.dataTask(with: url) { (data, response, error) in
            if let imageData = data, let uiImage = UIImage(data: imageData) {
                DispatchQueue.main.async {
                    self.fetchedUIImage = uiImage
                }
            }
        }.resume()
    }
}

问题排查与修复方案 #

核心问题分析 #

1. DOM结构匹配差异：浏览器控制台是在已完全加载的桌面端页面环境执行JS，而WKWebView默认触发Instagram移动端布局，DOM结构和桌面端不同，导致选择器失效。
2. 标识符提取逻辑错误：fetchImageURL从当前URL取最后一段作为标识符，页面跳转（如重定向到移动端路径）时会拿到错误值。
3. 硬编码等待时间不可靠：固定7秒等待无法适配不同网络环境，可能元素未渲染就执行JS，或等待时间过长浪费资源。

修复步骤 #

1. 模拟桌面浏览器环境

修改WKWebView初始化代码，设置桌面端User-Agent和内容模式，确保DOM结构与浏览器控制台一致：

var webView: WKWebView = {
    let webConfiguration = WKWebViewConfiguration()
    let userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
    webConfiguration.defaultWebpagePreferences.preferredContentMode = .desktop
    let webView = WKWebView(frame: .zero, configuration: webConfiguration)
    webView.customUserAgent = userAgent
    return webView
}()

2. 修正标识符提取逻辑

直接使用load方法中已提取的uniqueIdentifier，避免从可能变化的当前URL获取：

// 替换原fetchImageURL方法
private func fetchImageURL(retryCount: Int) {
    guard let identifier = uniqueIdentifier else {
        print("No valid identifier extracted")
        return
    }
    let fetchImageScript = """
    (function() {
        var anchorElement = document.querySelector('a[href="/p/\(identifier)/"]');
        if (anchorElement) {
            var imgElement = anchorElement.querySelector('div._aagu img');
            return imgElement ? imgElement.getAttribute('src') : "Image not found";
        }
        return "No anchor element found with identifier: \(identifier)";
    })();
    """
    self.webView.evaluateJavaScript(fetchImageScript) { (fetchResult, fetchError) in
        if let link = fetchResult as? String, link != "Image not found", let validURL = URL(string: link) {
            print("Successfully fetched image URL: \(link)")
            self.loadImage(from: validURL)
        } else {
            print("Failed to fetch image URL with error: \(fetchResult ?? "Unknown error")")
            if retryCount > 0 {
                DispatchQueue.global().asyncAfter(deadline: .now() + 3.0) {
                    self.fetchImageURL(retryCount: retryCount - 1)
                }
            } else {
                print("Max retry attempts reached.")
            }
        }
    }
}

同时修改attemptFetchingDescription中的调用：

// 替换原self.fetchImageURL(from: urlStr, retryCount: retryCount)
self.fetchImageURL(retryCount: retryCount)

3. 用MutationObserver替代硬等待

移除固定等待时间，通过监听DOM变化确保元素渲染完成后再执行JS：

// 替换原webView(_:didFinish:)方法
func webView(_ webView: WKWebView, didFinish navigation: WKNavigation!) {
    guard let identifier = uniqueIdentifier else { return }
    let observerScript = """
    (function() {
        return new Promise((resolve) => {
            const targetSelector = 'a[href="/p/\(identifier)/"] div._aagu img';
            let element = document.querySelector(targetSelector);
            if (element) {
                resolve(element.getAttribute('src'));
                return;
            }
            const observer = new MutationObserver((mutations) => {
                element = document.querySelector(targetSelector);
                if (element) {
                    observer.disconnect();
                    resolve(element.getAttribute('src'));
                }
            });
            observer.observe(document.body, { childList: true, subtree: true });
            setTimeout(() => {
                observer.disconnect();
                resolve("Timeout waiting for element");
            }, 10000);
        });
    })();
    """
    webView.evaluateJavaScript(observerScript) { result, error in
        if let link = result as? String, link != "Timeout waiting for element", let validURL = URL(string: link) {
            print("Successfully fetched image URL: \(link)")
            self.loadImage(from: validURL)
        } else {
            print("Failed to fetch image: \(result ?? error?.localizedDescription ?? "Unknown error")")
        }
    }
}

此时可移除attemptFetchingDescription和retryFetching方法，简化逻辑。

4. 处理图片URL重定向

Instagram图片URL可能存在302重定向，修改loadImage方法处理重定向：

private func loadImage(from url: URL) {
    var request = URLRequest(url: url)
    request.httpShouldHandleCookies = true
    let task = URLSession.shared.dataTask(with: request) { data, response, error in
        if let httpResponse = response as? HTTPURLResponse, (300...399).contains(httpResponse.statusCode), 
           let redirectURLStr = httpResponse.allHeaderFields["Location"] as? String, 
           let validRedirectURL = URL(string: redirectURLStr) {
            self.loadImage(from: validRedirectURL)
            return
        }
        if let imageData = data, let uiImage = UIImage(data: imageData) {
            DispatchQueue.main.async {
                self.fetchedUIImage = uiImage
            }
        }
    }
    task.resume()
}

内容的提问来源于stack exchange，提问作者P_n