使用Jsoup抓取含__doPostBack的ASP.NET网站分页数据失败求助

安全阿岳

2026-6-20

问题

尝试用Java的Jsoup框架抓取https://pad.minem.gob.pe/REINFO_WEB/Index.aspx页面中名为stdregistro的表格数据并保存到数据库，但仅能获取第一页数据，无法获取后续分页数据。当前实现代码如下：

public class apiscraping {

    private static final String URL = "https://pad.minem.gob.pe/REINFO_WEB/Index.aspx";
    private static final String JDBC_URL = "jdbc:sqlserver://xxxxx:1433;databaseName=xxxxx;user=xxxx;password=xxxxx";
    
    public static void main(String[] args) {
        try {
            scrapeAndStore();
        } catch (Exception e) {
            System.out.println(e.getMessage());
            e.printStackTrace();
        }
    }
    
    private static void scrapeAndStore() throws Exception {
        CloseableHttpClient httpClient = HttpClients.createDefault();
        int pageNumber = 1;
        boolean hasNextPage = true;

        while (hasNextPage) {
            Document doc = Jsoup.connect(URL).get();
            Elements rows = doc.select("#stdregistro tr");

            for (Element row : rows) {
                Elements columns = row.select("td");
                if (columns.size() > 0) {
                    String ruc = columns.get(2).text();
                    String mineroFormal = columns.get(3).text();
                    String codigoUnico = columns.get(4).text();
                    String nombre = columns.get(5).text();
                    String departamento = columns.get(6).text();
                    String provincia = columns.get(7).text();
                    String distrito = columns.get(8).text();
                    String estado = columns.get(9).text();

                    saveToDatabase(ruc, mineroFormal, codigoUnico, nombre, departamento, provincia, distrito, estado);
                }
            }

            hasNextPage = navigateToNextPage(httpClient, doc);
            System.out.println("Processed page: " + pageNumber);
            pageNumber++;
        }

        httpClient.close();
    }

    private static boolean navigateToNextPage(CloseableHttpClient httpClient, Document doc) throws IOException {
        Element nextButton = doc.select("#ImgBtnSiguiente").first();
        if (nextButton == null || nextButton.hasAttr("disabled")) {
            return false;
        }

        String viewState = doc.select("input[name=__VIEWSTATE]").val();
        String eventValidation = doc.select("input[name=__EVENTVALIDATION]").val();
        String viewStateGenerator = doc.select("input[name=__VIEWSTATEGENERATOR]").val();
        String txtpagina = doc.select("input[name=txtpagina]").val();
        int nextPage = Integer.parseInt(txtpagina) + 1;

        HttpPost post = new HttpPost(URL);
        List<BasicNameValuePair> params = new ArrayList<>();
        params.add(new BasicNameValuePair("__VIEWSTATE", viewState));
        params.add(new BasicNameValuePair("__EVENTVALIDATION", eventValidation));
        params.add(new BasicNameValuePair("__VIEWSTATEGENERATOR", viewStateGenerator));
        params.add(new BasicNameValuePair("txtpagina", String.valueOf(nextPage)));
        params.add(new BasicNameValuePair("ImgBtnSiguiente.x", "1"));
        params.add(new BasicNameValuePair("ImgBtnSiguiente.y", "1"));

        post.setEntity(new UrlEncodedFormEntity(params));
        CloseableHttpResponse response = httpClient.execute(post);

        Document nextDoc = Jsoup.parse(response.getEntity().getContent(), "UTF-8", URL);
        response.close();

        return nextDoc != null;
    }

    private static void saveToDatabase(String ruc, String mineroFormal, String codigoUnico, String nombre, String departamento, String provincia, String distrito, String estado) {
        String sql = "INSERT INTO OPE_REINFO (OPE_RUC, OPE_MINEROFORMAL, OPE_CODIGOUNICO, OPE_NOMBRE, OPE_DEPARTAMENTO, OPE_PROVINCIA, OPE_DISTRITO, OPE_ESTADO) VALUES (?, ?, ?, ?, ?, ?, ?, ?)";

        try (Connection conn = DriverManager.getConnection(JDBC_URL);
             PreparedStatement pstmt = conn.prepareStatement(sql)) {

            pstmt.setString(1, ruc);
            pstmt.setString(2, mineroFormal);
            pstmt.setString(3, codigoUnico);
            pstmt.setString(4, nombre);
            pstmt.setString(5, departamento);
            pstmt.setString(6, provincia);
            pstmt.setString(7, distrito);
            pstmt.setString(8, estado);

            pstmt.executeUpdate();
            System.out.println("Inserted record: RUC = " + ruc + ", Minero = " + mineroFormal + ", Código Único = " + codigoUnico);
        } catch (SQLException e) {
            System.out.println("Error inserting record: " + e.getMessage());
        }
    }
}

作为爬虫新手，认为问题出在表格数据的获取方式上，寻求解决建议与帮助。

解决方案

核心问题是每次循环都重新发起GET请求获取第一页，没有使用分页请求后的页面文档解析下一页数据。以下是具体修复方案：

1. 维护当前页面的文档对象

在scrapeAndStore方法中，初始获取第一页文档后，每次分页请求成功后更新当前文档，而非重复从原始URL拉取数据。

2. 修正分页逻辑的返回值

将navigateToNextPage方法改为返回下一页的Document对象（空值表示无下一页），替代原有的布尔值返回，确保后续循环能基于新页面解析数据。

3. 完善分页按钮的判断逻辑

部分ASP.NET页面的禁用按钮不会直接标记disabled属性，可能通过父元素或CSS类控制，需补充相关判断。

修改后的完整代码

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.NameValuePair;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.io.IOException;

public class apiscraping {

    private static final String URL = "https://pad.minem.gob.pe/REINFO_WEB/Index.aspx";
    private static final String JDBC_URL = "jdbc:sqlserver://xxxxx:1433;databaseName=xxxxx;user=xxxx;password=xxxxx";
    
    public static void main(String[] args) {
        try {
            scrapeAndStore();
        } catch (Exception e) {
            System.out.println(e.getMessage());
            e.printStackTrace();
        }
    }
    
    private static void scrapeAndStore() throws Exception {
        CloseableHttpClient httpClient = HttpClients.createDefault();
        int pageNumber = 1;
        Document currentDoc = Jsoup.connect(URL).get();
        boolean hasNextPage = true;

        while (hasNextPage) {
            // 解析当前页面的表格数据
            Elements rows = currentDoc.select("#stdregistro tr");
            for (Element row : rows) {
                Elements columns = row.select("td");
                if (columns.size() > 0) {
                    String ruc = columns.get(2).text();
                    String mineroFormal = columns.get(3).text();
                    String codigoUnico = columns.get(4).text();
                    String nombre = columns.get(5).text();
                    String departamento = columns.get(6).text();
                    String provincia = columns.get(7).text();
                    String distrito = columns.get(8).text();
                    String estado = columns.get(9).text();

                    saveToDatabase(ruc, mineroFormal, codigoUnico, nombre, departamento, provincia, distrito, estado);
                }
            }

            // 尝试获取下一页文档
            Document nextDoc = navigateToNextPage(httpClient, currentDoc);
            hasNextPage = nextDoc != null;
            currentDoc = nextDoc;
            
            System.out.println("Processed page: " + pageNumber);
            pageNumber++;
        }

        httpClient.close();
    }

    private static Document navigateToNextPage(CloseableHttpClient httpClient, Document doc) throws IOException {
        Element nextButton = doc.select("#ImgBtnSiguiente").first();
        // 检查按钮是否存在，以及是否可点击（兼容父元素禁用的情况）
        if (nextButton == null || nextButton.hasAttr("disabled") || nextButton.parent().hasAttr("disabled")) {
            return null;
        }

        // 获取ASP.NET表单所需的隐藏字段
        String viewState = doc.select("input[name=__VIEWSTATE]").val();
        String eventValidation = doc.select("input[name=__EVENTVALIDATION]").val();
        String viewStateGenerator = doc.select("input[name=__VIEWSTATEGENERATOR]").val();
        String txtpagina = doc.select("input[name=txtpagina]").val();
        int nextPage = Integer.parseInt(txtpagina) + 1;

        // 构造POST请求参数，随机x/y坐标更贴近真实用户操作
        HttpPost post = new HttpPost(URL);
        List<NameValuePair> params = new ArrayList<>();
        params.add(new BasicNameValuePair("__VIEWSTATE", viewState));
        params.add(new BasicNameValuePair("__EVENTVALIDATION", eventValidation));
        params.add(new BasicNameValuePair("__VIEWSTATEGENERATOR", viewStateGenerator));
        params.add(new BasicNameValuePair("txtpagina", String.valueOf(nextPage)));
        params.add(new BasicNameValuePair("ImgBtnSiguiente.x", String.valueOf((int)(Math.random()*100))));
        params.add(new BasicNameValuePair("ImgBtnSiguiente.y", String.valueOf((int)(Math.random()*100))));

        post.setEntity(new UrlEncodedFormEntity(params, "UTF-8"));
        CloseableHttpResponse response = httpClient.execute(post);

        // 解析下一页文档
        Document nextDoc = Jsoup.parse(response.getEntity().getContent(), "UTF-8", URL);
        response.close();

        // 验证下一页有效性（检查表格是否存在）
        return nextDoc.select("#stdregistro").size() > 0 ? nextDoc : null;
    }

    private static void saveToDatabase(String ruc, String mineroFormal, String codigoUnico, String nombre, String departamento, String provincia, String distrito, String estado) {
        String sql = "INSERT INTO OPE_REINFO (OPE_RUC, OPE_MINEROFORMAL, OPE_CODIGOUNICO, OPE_NOMBRE, OPE_DEPARTAMENTO, OPE_PROVINCIA, OPE_DISTRITO, OPE_ESTADO) VALUES (?, ?, ?, ?, ?, ?, ?, ?)";

        try (Connection conn = DriverManager.getConnection(JDBC_URL);
             PreparedStatement pstmt = conn.prepareStatement(sql)) {

            pstmt.setString(1, ruc);
            pstmt.setString(2, mineroFormal);
            pstmt.setString(3, codigoUnico);
            pstmt.setString(4, nombre);
            pstmt.setString(5, departamento);
            pstmt.setString(6, provincia);
            pstmt.setString(7, distrito);
            pstmt.setString(8, estado);

            pstmt.executeUpdate();
            System.out.println("Inserted record: RUC = " + ruc + ", Minero = " + mineroFormal + ", Código Único = " + codigoUnico);
        } catch (SQLException e) {
            System.out.println("Error inserting record: " + e.getMessage());
        }
    }
}

额外建议

添加请求延迟：在分页请求之间添加1-2秒延迟，避免给服务器造成压力，降低被反爬拦截的风险。
处理重复数据：给数据库中OPE_RUC或OPE_CODIGOUNICO字段添加唯一约束，避免重复插入。
替换日志方式：使用SLF4J等日志框架替代System.out.println，便于后续排查问题。

内容的提问来源于stack exchange，提问作者DrGun Gun

使用Jsoup抓取含__doPostBack的ASP.NET网站分页数据失败求助

解决方案 #

1. 维护当前页面的文档对象 #

2. 修正分页逻辑的返回值 #

3. 完善分页按钮的判断逻辑 #

修改后的完整代码 #