使用Google Apps Script获取数据时的分页错误问题

我尝试从PhilGEPS的招标机会表格中获取可点击标题对应的URL数据，当前脚本能正确获取第一页数据，但无法获取第二页。尝试在URL中添加页码参数无效，需要解决指导。

示例脚本 #

function scrapeTitlesData() {
        var url = "https://notices.philgeps.gov.ph/GEPSNONPILOT/Tender/SplashOpportunitiesSearchUI.aspx?menuIndex=3&BusCatID=157&type=category&ClickFrom=OpenOpp";

        let cookieResponse = UrlFetchApp.fetch(url);
        let cookie = cookieResponse.getAllHeaders()['Set-Cookie'].map(c => c.split(';')[0]).join(';');
    
        let options = {
         method: 'get',
         headers: { cookie },
    };

        var response = UrlFetchApp.fetch(url,options).getContentText();
        extractAndModifyUrls(response)
  }

function extractAndModifyUrls(html) {
      // Regex pattern to match the required URLs
      var regex = /SplashBidNoticeAbstractUI\.aspx\?menuIndex=3&refID=\d+&[^"]+/g;
    
      // Initialize an empty array to store the modified URLs
      var modifiedUrls = [];
      var match;
      
      // Find all matches in the HTML
      while ((match = regex.exec(html)) !== null) {
        var newUrl = 'https://notices.philgeps.gov.ph/GEPSNONPILOT/Tender/' + match[0];
        newUrl = newUrl.replace(/&/g, '&');
        
        // Add the new URL to the array
        modifiedUrls.push(newUrl);
      }
    
      console.log(modifiedUrls);
}

问题原因 #

这类基于ASP.NET构建的网站，分页逻辑通常不依赖URL参数，而是通过POST请求提交表单参数来实现，核心依赖__VIEWSTATE、__VIEWSTATEGENERATOR、__EVENTVALIDATION等ASP.NET特有字段，以及分页按钮的触发参数。单纯修改URL无法触发分页跳转。

解决步骤 #

1. 分析分页请求参数 #

打开浏览器开发者工具（F12），切换到Network标签：

• 点击页面的「下一页」按钮
• 观察请求类型（应为POST）
• 记录请求中携带的所有表单参数，重点关注：
  • __VIEWSTATE、__VIEWSTATEGENERATOR、__EVENTVALIDATION
  • 分页按钮对应的参数（如ctl00$ContentPlaceHolder1$gvOpportunity$ctl23$ctl01，值通常为「下一页」）

2. 修改脚本实现多页爬取 #

以下是调整后的完整脚本，支持循环爬取多页数据：

function scrapeAllPages() {
  const baseUrl = "https://notices.philgeps.gov.ph/GEPSNONPILOT/Tender/SplashOpportunitiesSearchUI.aspx?menuIndex=3&BusCatID=157&type=category&ClickFrom=OpenOpp";
  let cookie = "";
  let viewState = "";
  let viewStateGenerator = "";
  let eventValidation = "";
  let currentPage = 1;
  let hasNextPage = true;

  // 首次请求获取Cookie和初始页面参数
  const initialResponse = UrlFetchApp.fetch(baseUrl);
  cookie = initialResponse.getAllHeaders()['Set-Cookie'].map(c => c.split(';')[0]).join(';');
  const initialHtml = initialResponse.getContentText();
  
  // 提取ASP.NET核心参数
  viewState = extractAspParam(initialHtml, '__VIEWSTATE');
  viewStateGenerator = extractAspParam(initialHtml, '__VIEWSTATEGENERATOR');
  eventValidation = extractAspParam(initialHtml, '__EVENTVALIDATION');

  // 处理第一页数据
  extractAndModifyUrls(initialHtml);

  while (hasNextPage) {
    console.log(`正在爬取第 ${currentPage + 1} 页`);
    
    // 构造分页POST请求参数
    const postPayload = {
      '__VIEWSTATE': viewState,
      '__VIEWSTATEGENERATOR': viewStateGenerator,
      '__EVENTVALIDATION': eventValidation,
      'ctl00$ContentPlaceHolder1$gvOpportunity$ctl23$ctl01': '下一页' // 替换为你在开发者工具中看到的分页按钮参数
    };

    const requestOptions = {
      method: 'post',
      headers: { 
        'Cookie': cookie,
        'Content-Type': 'application/x-www-form-urlencoded'
      },
      payload: postPayload
    };

    try {
      const nextPageResponse = UrlFetchApp.fetch(baseUrl, requestOptions);
      const nextPageHtml = nextPageResponse.getContentText();

      // 更新核心参数，用于下一次请求
      viewState = extractAspParam(nextPageHtml, '__VIEWSTATE');
      viewStateGenerator = extractAspParam(nextPageHtml, '__VIEWSTATEGENERATOR');
      eventValidation = extractAspParam(nextPageHtml, '__EVENTVALIDATION');

      // 检查是否还有下一页（通过判断分页按钮是否存在）
      hasNextPage = nextPageHtml.includes('ctl00$ContentPlaceHolder1$gvOpportunity$ctl23$ctl01');
      currentPage++;

      // 处理当前页数据
      extractAndModifyUrls(nextPageHtml);
    } catch (error) {
      console.log(`爬取第 ${currentPage + 1} 页失败: ${error.message}`);
      hasNextPage = false;
    }
  }
}

// 提取ASP.NET隐藏字段参数
function extractAspParam(html, paramName) {
  const regex = new RegExp(`<input type="hidden" name="${paramName}" id="${paramName}" value="([^"]+)" />`);
  const match = html.match(regex);
  return match ? match[1] : '';
}

function extractAndModifyUrls(html) {
  const regex = /SplashBidNoticeAbstractUI\.aspx\?menuIndex=3&amp;amp;refID=\d+&amp;[^"]+/g;
  const modifiedUrls = [];
  let match;
  
  while ((match = regex.exec(html)) !== null) {
    let newUrl = 'https://notices.philgeps.gov.ph/GEPSNONPILOT/Tender/' + match[0];
    newUrl = newUrl.replace(/&amp;amp;/g, '&amp;');
    modifiedUrls.push(newUrl);
  }

  console.log(`当前页获取到 ${modifiedUrls.length} 个URL`, modifiedUrls);
}

注意事项 #

• 分页按钮的参数（如ctl00$ContentPlaceHolder1$gvOpportunity$ctl23$ctl01）可能会随网站更新变化，需要定期通过开发者工具确认
• 控制请求频率，避免因频繁请求触发网站反爬机制导致IP被封禁
• 若网站参数加密或有其他反爬措施，需进一步分析请求逻辑

内容的提问来源于stack exchange，提问作者EagleEye