PHP cURL循环爬取零件信息时始终返回首个URL数据问题求解

问题根因 #

• 你声明的$articles数组在整个循环周期内只会持续追加数据，不会自动重置。第一次循环存入的零件号WE04X10010永远是数组的第一个元素，后续所有循环你都固定读取$articles[0]的内容，所以输出的零件号永远是第一次的结果。
• 额外冗余逻辑：你把登录请求放在了循环内，每次遍历零件号都会重复执行一次登录，完全没有必要，还会增加请求耗时和被目标站点封禁的风险。
• 潜在风险：simple_html_dom对象使用后没有清理内存，多次循环后可能出现解析异常；没有判断搜索接口是否返回了有效结果，直接取下标0的元素会在无搜索结果时报错。

修复代码 #

include('./simple_html_dom.php');
require './includes/config.inc.php';
require './includes/Database.inc.php';

$db = new Database();
$db->query('SELECT part_num FROM GE LIMIT 4');
$results = $db->resultset();

// 登录逻辑移到循环外，只执行一次
$curlPost['accountNumber']="*****";
$curlPost['password']="******";
$curlPost['contactForm']="https://www.*****.com/images/layout/input/submit.gif";

$ch = curl_init();
curl_setopt($ch, CURLOPT_URL , "https://www.*****.com/login");
curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.A.B.C Safari/525.13");
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, $curlPost);
curl_setopt($ch, CURLOPT_HEADER, true);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); 
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
curl_setopt($ch, CURLOPT_COOKIEFILE, "cookies.txt");
curl_setopt($ch, CURLOPT_COOKIEJAR, "cookies.txt");
$response = curl_exec ($ch);

foreach($results as $result){
    // 每次循环先清空结果数组
    $articles = [];
    $startingPN = trim($result['part_num']);

    curl_setopt($ch, CURLOPT_URL, 'https://www.******.com/search/?part='.$startingPN);
    $exec = curl_exec($ch);

    $html = new simple_html_dom();
    $html->load($exec);

    foreach($html->find('table.searchWholesale') as $article) {
        $item['part_num'] = $article->find('span.purpleBold', 0)->plaintext;
        $item['description'] = $article->find('span.twelve', 1)->plaintext;
        $item['cost'] = $article->find('span.fourteen', 1)->plaintext;
        $item['retail'] = $article->find('span.fourteen', 2)->plaintext;
        $item['brand'] = $article->find('td.leftColumn',0);
        $item['count'] = count($article->find('div.productHide'));
        $item['productHide1'] = $article->find('div.productHide', 0)->plaintext;
        $item['productHide2'] = $article->find('div.productHide', 1)->plaintext;
        $articles[] = $item;
    }
    // 清理dom对象释放内存
    $html->clear();
    unset($html);

    // 增加判空逻辑避免报错
    if(empty($articles)){
        echo '<br>'.$startingPN.'<br>无搜索结果';
        continue;
    }
    
    $last_checked = date("U");
    $part_num = $articles[0]['part_num'];
    $title = ucwords(strtolower($articles[0]['description']));
    $cost = floatval(ltrim($articles[0]['cost'], '$'));
    $retail = floatval(ltrim($articles[0]['retail'], '$'));
    $brand = $articles[0]['brand'];
    $brand->children(0)->outertext = "";
    $brand->children(1)->outertext = "";
    $brand->children(2)->outertext = "";
    $brand->children(3)->outertext = "";
    $brand->children(4)->outertext = "";
    $brand->children(5)->outertext = "";
    $brand->children(6)->outertext = "";

    $brand_qty = explode(" ", $brand);
    $name_brand =  $brand_qty[15];
    $qty_avail = $brand_qty[21];

    echo '<br>'.$startingPN . '<br>' . $part_num . '<br>' . $title . '<br>' . $cost . '<br>' . $retail  . '<br>' . $name_brand;
}
// 循环结束后关闭curl句柄
curl_close($ch);

内容的提问来源于stack exchange，提问作者user3154948