批量Webscraping实现：一次完成多关键词Google搜索并返回所有结果

Hey there! Your current script works great for grabbing results when you load the "one" search page, but it doesn't actively trigger multiple searches. Let's fix that—we'll use GM_xmlhttpRequest wrapped in Promises to run all three searches in parallel, then collect and process all results once everything's done.

The Core Idea #

• Wrap GM_xmlhttpRequest in a Promise so we can use async/await to manage multiple asynchronous requests
• Use Promise.all to fire off all three search requests at the same time (way faster than running them one after another)
• Parse the HTML response from each search to extract the results you care about (like titles, links, snippets, etc.)

Full Example Script #

// ==UserScript==
// @name Google Batch Search Tool
// @match https://www.google.com/*
// @version 1.0
// @require https://ajax.googleapis.com/ajax/libs/jquery/3.3.1/jquery.min.js
// @grant GM_xmlhttpRequest
// @run-at document-end
// ==/UserScript==

// Your list of keywords to search
const searchTerms = ["one", "two", "three"];

// Helper function to run a single Google search and return results as a Promise
function performSearch(keyword) {
  return new Promise((resolve, reject) => {
    // Encode the keyword to handle spaces/special characters
    const searchUrl = `https://www.google.com/search?q=${encodeURIComponent(keyword)}`;

    GM_xmlhttpRequest({
      method: "GET",
      url: searchUrl,
      onload: (response) => {
        if (response.status === 200) {
          // Parse the HTML response with jQuery to extract results
          const $page = $(response.responseText);
          const results = $page.find(".g").map((index, element) => {
            return {
              keyword: keyword,
              title: $(element).find("h3").text().trim(),
              link: $(element).find("a").attr("href"),
              snippet: $(element).find(".VwiC3b").text().trim()
            };
          }).get(); // Convert jQuery object to plain array
          resolve(results);
        } else {
          reject(new Error(`Failed to search for ${keyword} (Status: ${response.status})`));
        }
      },
      onerror: (error) => {
        reject(new Error(`Request error for ${keyword}: ${error}`));
      }
    });
  });
}

// Main function to run all searches and handle results
async function runBatchSearch() {
  try {
    console.log("Starting batch searches...");
    // Run all searches in parallel
    const allResults = await Promise.all(searchTerms.map(term => performSearch(term)));
    // Flatten the array of arrays into a single list of results
    const combinedResults = allResults.flat();

    console.log("All search results:", combinedResults);
    // Show results directly on the page for easy access
    displayResults(combinedResults);
  } catch (err) {
    console.error("Batch search failed:", err);
  }
}

// Helper to display results in a floating window on the page
function displayResults(results) {
  const resultPanel = $("<div>").css({
    position: "fixed",
    top: "20px",
    right: "20px",
    width: "450px",
    maxHeight: "700px",
    overflowY: "auto",
    background: "#fff",
    border: "1px solid #ddd",
    borderRadius: "8px",
    padding: "16px",
    boxShadow: "0 2px 10px rgba(0,0,0,0.1)",
    zIndex: 99999
  });

  resultPanel.append($("<h2>").text("Batch Search Results").css({ margin: "0 0 16px 0", color: "#1a73e8" }));

  results.forEach(result => {
    const resultItem = $("<div>").css({ marginBottom: "12px", paddingBottom: "12px", borderBottom: "1px solid #eee" });
    const keywordBadge = $("<span>").text(`[${result.keyword}]`).css({
      background: "#e8f0fe",
      color: "#1a73e8",
      padding: "2px 6px",
      borderRadius: "4px",
      fontSize: "12px",
      fontWeight: "bold"
    });
    const titleLink = $("<a>")
      .attr("href", result.link)
      .attr("target", "_blank")
      .text(result.title)
      .css({ display: "block", marginTop: "6px", color: "#1a0dab", textDecoration: "none" });
    const snippet = $("<p>").text(result.snippet).css({ margin: "6px 0 0 0", fontSize: "13px", color: "#545454" });

    resultItem.append(keywordBadge).append(titleLink).append(snippet);
    resultPanel.append(resultItem);
  });

  $("body").append(resultPanel);
}

// Run the batch search when the page loads (you could also add a button to trigger it manually!)
runBatchSearch();

Key Notes #

• Parallel Requests: Promise.all lets us run all three searches at once, which is much more efficient than sequential requests.
• Result Parsing: The script extracts titles, links, and snippets from each result—you can adjust the jQuery selectors if you need different data.
• Anti-Crawl Heads-Up: Google has anti-scraping measures, so don't run this too frequently or with too many keywords, otherwise you might get blocked temporarily.
• Flexible Trigger: Right now it runs automatically when you load Google, but you could add a button to the page that triggers runBatchSearch() instead if you prefer manual control.

内容的提问来源于stack exchange，提问作者frosty