UrlFetchApp.fetch()超时无响应求助：谷歌表格域名爬取异常

问题背景 #

我在谷歌表格里维护了一份域名列表，用脚本抓取页面的title、description、H1和H2写回表格。调整请求头后99%的域名都能正常处理，但以下域名会无报错超时：

• accuweather.com
• nasdaq.com
• haberaybasti.com
• militarycupid.com
• wetter.de

以下域名可正常运行：

• potv.bg
• actualno.com
• honeymoments.de
• rosenheim24.de
• mamaliefde.nl
• i-diakopes.gr
• liteapks.com
• kreuzwortraetsellexikon.de

当前脚本的try/catch无法捕获这类超时，没法跳过这些域名并在表格标记状态，求解决方案。原代码如下：

function fetchData(url) {
  var headers = {
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
    'Accept-Encoding': 'gzip, deflate, br, zstd',
    'Accept-Language': 'de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7',
    'Priority': 'u=0, i',
    'Sec-Ch-Ua': '"Not/A)Brand";v="8", "Chromium";v="126", "Google Chrome";v="126"',
    'Sec-Ch-Ua-Mobile': '?0',
    'Sec-Ch-Ua-Platform': '"macOS"',
    'Sec-Fetch-Dest': 'document',
    'Sec-Fetch-Mode': 'navigate',
    'Sec-Fetch-Site': 'none',
    'Sec-Fetch-User': '?1',
    'Upgrade-Insecure-Requests': '1',
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',
    'X-Amzn-Trace-Id': 'Root=1-669d8ffa-6c30c4d90649d665390140a4'
  };
  var options = {
    'followRedirects': true,
    'muteHttpExceptions': false,
    'headers': headers,
    'timeout': 10000 // Timeout set to 10 seconds
  };
  try {
    var response = UrlFetchApp.fetch(url, options);
    var responseCode = response.getResponseCode();
    
    if (responseCode === 200) {
      var content = response.getContentText();
      
      var titleMatch = content.match(/<title>([^<]+)<\/title>/);
      var title = titleMatch && titleMatch[1] ? titleMatch[1] : "-";
      
      var metaDescriptionMatch = content.match(/<meta\s+name="description"\s+content="([^"]*)"/i);
      var metaDescription = metaDescriptionMatch && metaDescriptionMatch[1] ? metaDescriptionMatch[1] : "-";
      
      var h1Match = content.match(/<h1[^>]*>([^<]+)<\/h1>/i);
      var h1 = h1Match && h1Match[1] ? h1Match[1] : "-";
      
      var h2Match = content.match(/<h2[^>]*>([^<]+)<\/h2>/i);
      var h2 = h2Match && h2Match[1] ? h2Match[1] : "-";
      
      return {
        title: title,
        metaDescription: metaDescription,
        h1: h1,
        h2: h2,
        success: true
      };
    } else {
      return {
        title: "Failed to fetch",
        metaDescription: "Failed to fetch",
        h1: "Failed to fetch",
        h2: "Failed to fetch",
        success: false
      };
    }
  } catch (e) {
    Logger.log('Error fetching URL: ' + url + ' - ' + e.message);
    return {
      title: 'DNS Error',
      metaDescription: 'DNS Error',
      h1: 'DNS Error',
      h2: 'DNS Error',
      success: false
    };
  }
}

function processUrlsIndividually() {
  var ss = SpreadsheetApp.openById("SHEETNAMEINTHERE");
  var sheet = ss.getSheetByName("check_plmWebsites");
  var range = sheet.getDataRange();
  var values = range.getValues();

  var lastFetchedIndex = getLastFetchedIndex(values);
  var startIndex = lastFetchedIndex + 1;

  Logger.log('Starting processing from index: ' + startIndex);

  for (var i = startIndex; i < values.length; i++) {
    if (isCloseToTimeout()) {
      Logger.log('Approaching timeout, stopping script execution.');
      break;
    }

    var url = values[i][0]; // Placement column (A)
    if (!url) continue; // Skip if URL is empty or undefined

    var urlData = fetchData(url);

    if (urlData.success) {
      sheet.getRange(i + 1, 9, 1, 4).setValues([[urlData.title, urlData.metaDescription, urlData.h1, urlData.h2]]); // Columns I, J, K, L
      sheet.getRange(i + 1, 13).setValue(new Date()); // Column M (fetched)
      Logger.log('Successfully fetched URL: ' + url);
    } else {
      sheet.getRange(i + 1, 13).setValue('Skipped due to timeout'); // Column M (fetched)
      Logger.log('Failed to fetch URL: ' + url + ' - Skipped due to timeout.');
    }

    // Sleep for 1 second between requests to avoid hitting rate limits
    Utilities.sleep(1000);
  }

  Logger.log('Processing completed up to index: ' + (i - 1));
}

function getLastFetchedIndex(values) {
  for (var i = values.length - 1; i >= 0; i--) {
    if (values[i][12]) { // Column M (fetched)
      return i;
    }
  }
  return 0;
}

function isCloseToTimeout() {
  var runtime = new Date().getTime() - SCRIPT_START_TIME.getTime();
  return runtime > 5 * 60 * 1000; // Stop execution after 5 minutes to be safe
}

var SCRIPT_START_TIME = new Date();

解决方案 #

UrlFetchApp的内置超时在部分场景下不会触发常规异常，需要结合手动超时监控和请求配置调整解决：

关键修改点 #

1. 启用muteHttpExceptions: true，让所有HTTP异常（包括超时）进入catch块
2. 添加手动超时检测，通过请求前后的时间差判断是否超时
3. 细化错误类型，区分超时、DNS错误、HTTP状态码错误
4. 在表格中标记具体错误原因，替代笼统的"Skipped"

修改后的完整代码 #

function fetchData(url) {
  const headers = {
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
    'Accept-Encoding': 'gzip, deflate, br, zstd',
    'Accept-Language': 'de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7',
    'Priority': 'u=0, i',
    'Sec-Ch-Ua': '"Not/A)Brand";v="8", "Chromium";v="126", "Google Chrome";v="126"',
    'Sec-Ch-Ua-Mobile': '?0',
    'Sec-Ch-Ua-Platform': '"macOS"',
    'Sec-Fetch-Dest': 'document',
    'Sec-Fetch-Mode': 'navigate',
    'Sec-Fetch-Site': 'none',
    'Sec-Fetch-User': '?1',
    'Upgrade-Insecure-Requests': '1',
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36'
  };
  const timeoutMs = 10000; // 10秒超时阈值
  const options = {
    'followRedirects': true,
    'muteHttpExceptions': true, // 核心：让所有HTTP异常可捕获
    'headers': headers,
    'timeout': timeoutMs
  };

  const startTime = Date.now();
  try {
    const response = UrlFetchApp.fetch(url, options);
    const responseCode = response.getResponseCode();
    
    // 手动校验是否超时，防止内置超时失效
    if (Date.now() - startTime > timeoutMs) {
      throw new Error('Request timed out');
    }

    if (responseCode === 200) {
      const content = response.getContentText();
      
      const titleMatch = content.match(/<title>([^<]+)<\/title>/);
      const title = titleMatch?.[1] || "-";
      
      const metaDescriptionMatch = content.match(/<meta\s+name="description"\s+content="([^"]*)"/i);
      const metaDescription = metaDescriptionMatch?.[1] || "-";
      
      const h1Match = content.match(/<h1[^>]*>([^<]+)<\/h1>/i);
      const h1 = h1Match?.[1] || "-";
      
      const h2Match = content.match(/<h2[^>]*>([^<]+)<\/h2>/i);
      const h2 = h2Match?.[1] || "-";
      
      return {
        title: title,
        metaDescription: metaDescription,
        h1: h1,
        h2: h2,
        success: true,
        error: null
      };
    } else {
      return {
        title: "HTTP Error",
        metaDescription: `Status Code: ${responseCode}`,
        h1: "HTTP Error",
        h2: "HTTP Error",
        success: false,
        error: `HTTP ${responseCode}`
      };
    }
  } catch (e) {
    let errorType = "Unknown Error";
    // 区分不同错误类型
    if (e.message.includes("timed out") || Date.now() - startTime > timeoutMs) {
      errorType = "Timeout";
    } else if (e.message.includes("DNS")) {
      errorType = "DNS Error";
    } else if (e.message.includes("HTTP")) {
      errorType = "HTTP Error";
    }
    
    Logger.log(`Error fetching ${url}: ${errorType} - ${e.message}`);
    return {
      title: errorType,
      metaDescription: errorType,
      h1: errorType,
      h2: errorType,
      success: false,
      error: errorType
    };
  }
}

function processUrlsIndividually() {
  const ss = SpreadsheetApp.openById("SHEETNAMEINTHERE");
  const sheet = ss.getSheetByName("check_plmWebsites");
  const range = sheet.getDataRange();
  const values = range.getValues();

  const lastFetchedIndex = getLastFetchedIndex(values);
  const startIndex = lastFetchedIndex + 1;

  Logger.log(`Starting processing from index: ${startIndex}`);

  for (let i = startIndex; i < values.length; i++) {
    if (isCloseToTimeout()) {
      Logger.log('Approaching timeout, stopping script execution.');
      break;
    }

    const url = values[i][0];
    if (!url) continue;

    const urlData = fetchData(url);

    if (urlData.success) {
      sheet.getRange(i + 1, 9, 1, 4).setValues([[urlData.title, urlData.metaDescription, urlData.h1, urlData.h2]]);
      sheet.getRange(i + 1, 13).setValue(new Date());
      Logger.log(`Successfully fetched ${url}`);
    } else {
      sheet.getRange(i + 1, 13).setValue(`Skipped: ${urlData.error}`);
      Logger.log(`Failed to fetch ${url} - ${urlData.error}`);
    }

    Utilities.sleep(1000);
  }

  Logger.log(`Processing completed up to index: ${i - 1}`);
}

function getLastFetchedIndex(values) {
  for (let i = values.length - 1; i >= 0; i--) {
    if (values[i][12]) {
      return i;
    }
  }
  return 0;
}

function isCloseToTimeout() {
  const runtime = Date.now() - SCRIPT_START_TIME.getTime();
  return runtime > 5 * 60 * 1000;
}

const SCRIPT_START_TIME = new Date();

说明 #

• muteHttpExceptions: true是核心设置，它让UrlFetchApp不再直接终止脚本，而是将所有HTTP相关错误转化为可捕获的异常
• 手动超时检测作为兜底，避免内置超时机制失效的情况
• 错误类型细分后，表格中会明确标记"Timeout"、"DNS Error"等状态，便于后续排查
• 用ES6可选链?.简化了正则匹配后的取值逻辑，代码更简洁

内容的提问来源于Stack Exchange，提问作者marketom