Python爬取Instagram个人资料遇404错误，求技术支持

我有一批目标Instagram个人资料的URL，想要爬取粉丝数、关注人数等数据。在浏览器中输入这些URL可正常访问，但使用Python的requests库请求时始终返回'Not found'。尝试编写了Selenium代码，仍存在问题，希望帮忙排查。

原代码 #

import pandas as pd
import numpy as np
import requests 
from bs4 import BeautifulSoup
from selenium import webdriver
import time
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

data = pd.read_csv('HashTagCSV')
dataframe = pd.DataFrame(data)

posturl = dataframe.iloc[:, 0]
username = dataframe.iloc[:, 2]
commentcount = dataframe.iloc[:, 4]
likecount = dataframe.iloc[:, 5]

final_data_frame = dataframe.iloc[:, [1, 4, 5]]
final_data_frame.dropna(inplace=True)
print(final_data_frame)

final_data_frame['commentCount'] = pd.to_numeric(final_data_frame['commentCount'], errors='coerce')
final_data_frame['likeCount'] = pd.to_numeric(final_data_frame['likeCount'], errors='coerce')
final_data_frame['commentCount'] = final_data_frame['commentCount'].replace(np.nan, 0)
final_data_frame['likeCount'] = final_data_frame['likeCount'].replace(np.nan, 0)

dictionary_list = []

for row in final_data_frame.itertuples():
    
        if int(row.commentCount) > 50 and int(row.likeCount) > 200:
            
            dictionary = {'url': row.profileUrl, 'comments': row.commentCount, 'likes': row.likeCount}
            
            dictionary_list.append(dictionary)
            
new_data_frame = pd.DataFrame.from_dict(dictionary_list)
#print(new_data_frame)  
new_data_frame = new_data_frame.iloc[:, 0]
#print(new_data_frame)
data_frame = pd.DataFrame(new_data_frame)
print(data_frame)     
profileurl_list = data_frame['url'].values.tolist()
print(profileurl_list)     

#for url in profileurl_list:
    #url = 'https://www.instagram.com/'+username
 
    
#for names in profileurl_list:  

driver = webdriver.Chrome(ChromeDriverManager().install())
driver.get('https://www.instagram.com/jojajovana11')


#pop_up_box = driver.find_element(By.XPATH, '//*[@id=\"mount_0_0_Hp\"]/div/div/div/div[2]/div/div/div[1]/div/div[2]/div/div/div/div/div/div/div/div')

try:
    WebDriverWait(driver, 5).until(EC.element_to_be_clickable((By.XPATH, '//*[@id=\"mount_0_0_OO\"]/div/div/div/div[2]/div/div/div[1]/div/div[2]/div/div/div/div/div/div/button[2]'))).click()
except:
    driver.close()
    
#cookie_button = driver.find_element(By.XPATH,'//*[@id=\"mount_0_0_OO\"]/div/div/div/div[2]/div/div/div[1]/div/div[2]/div/div/div/div/div/div/button[2]')
#cookie_button.click()

问题分析与修复方案 #

1. requests请求失败原因 #

Instagram对未携带浏览器身份标识的HTTP请求做了反爬拦截，直接返回404。必须模拟完整的浏览器请求头，或者使用Selenium这类工具模拟真实浏览器行为。

2. 当前Selenium代码的核心问题 #

• 硬编码测试URL，未遍历目标profileurl_list列表
• 使用动态生成的XPATH（如mount_0_0_OO），页面加载后ID会变化，导致元素定位失败
• 未处理Instagram强制登录弹窗，未登录无法查看用户公开资料
• 缺少提取粉丝数、关注人数的逻辑

3. 修正后的代码 #

import pandas as pd
import numpy as np
from selenium import webdriver
import time
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

# 读取并处理CSV数据
data = pd.read_csv('HashTagCSV')
final_data_frame = data.iloc[:, [1, 4, 5]].dropna()

# 转换数值类型，填充空值
final_data_frame['commentCount'] = pd.to_numeric(final_data_frame['commentCount'], errors='coerce').replace(np.nan, 0)
final_data_frame['likeCount'] = pd.to_numeric(final_data_frame['likeCount'], errors='coerce').replace(np.nan, 0)

# 筛选符合条件的账号
dictionary_list = []
for row in final_data_frame.itertuples():
    if int(row.commentCount) > 50 and int(row.likeCount) > 200:
        dictionary_list.append({'url': row.profileUrl, 'comments': row.commentCount, 'likes': row.likeCount})

profileurl_list = [item['url'] for item in dictionary_list]

# 初始化浏览器
options = webdriver.ChromeOptions()
# 添加无头模式（可选，后台运行）
# options.add_argument('--headless=new')
options.add_argument('--disable-blink-features=AutomationControlled')
options.add_argument('--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36')

driver = webdriver.Chrome(ChromeDriverManager().install(), options=options)
wait = WebDriverWait(driver, 10)

# 遍历所有目标URL
for url in profileurl_list:
    try:
        driver.get(url)
        time.sleep(2)  # 等待页面加载

        # 处理Cookie弹窗
        try:
            cookie_btn = wait.until(EC.element_to_be_clickable((By.XPATH, "//button[contains(text(), '允许必要和可选 Cookie')]")))
            cookie_btn.click()
        except:
            pass  # 无弹窗则跳过

        # 处理登录提示弹窗
        try:
            close_login_btn = wait.until(EC.element_to_be_clickable((By.XPATH, "//div[contains(@role, 'button') and contains(@aria-label, '关闭')]")))
            close_login_btn.click()
        except:
            pass

        # 提取粉丝、关注、帖子数
        stats = wait.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, "div._aa_c > span._ac2a")))
        if len(stats) >=3:
            post_count = stats[0].text
            follower_count = stats[1].text
            following_count = stats[2].text
            print(f"URL: {url}")
            print(f"帖子数: {post_count}, 粉丝数: {follower_count}, 关注数: {following_count}\n")

    except Exception as e:
        print(f"处理{url}时出错: {str(e)}")
        continue

driver.quit()

关键优化点 #

• 使用--disable-blink-features=AutomationControlled绕过Selenium检测
• 采用稳定的CSS_SELECTOR和文本匹配定位元素，避免动态ID问题
• 添加用户代理，模拟真实浏览器
• 处理Cookie和登录弹窗，确保页面正常加载
• 遍历目标URL列表，批量提取数据

内容的提问来源于stack exchange，提问作者Stephen