如何从NOAA采暖度日链接解析数据并按日期拼接为DataFrame？

原有爬虫代码修正 #

原代码匹配txt文件的正则存在偏差，实际文件链接的href为带URL编码空格的格式（如Mar%2020,%202021.txt），原正则无法正确匹配，同时补充链接解码处理避免拼接错误，修正后的爬虫代码：

from bs4 import BeautifulSoup as bs
import requests
import re
from urllib.parse import unquote, urljoin

topdir = 'https://ftp.cpc.ncep.noaa.gov/htdocs/products/analysis_monitoring/cdus/degree_days/archives/Heating%20degree%20Days/weekly%20states/'
res = requests.get(topdir)
soup = bs(res.text, 'html.parser')

toplinks = []
midlinks = []
txtlinks = []

# 匹配年份目录
for link in soup.find_all('a'):
    href = link.get('href', '')
    if re.fullmatch(r'2\d{3}/', href):
        toplinks.append(href)

# 拼接年份目录的完整链接
for href in toplinks:
    midlinks.append(urljoin(topdir, href))

# 遍历年份目录匹配所有txt文件链接
for mlink in midlinks:
    mid_res = requests.get(mlink)
    msoup = bs(mid_res.text, 'html.parser')
    for link in msoup.find_all('a'):
        href = link.get('href', '')
        # 修正正则：匹配 Mar%2020,%202021.txt 这类格式
        if re.fullmatch(r'[a-zA-Z]{3}%20\d{1,2},%20\d{4}\.txt', href):
            txtlinks.append(urljoin(mlink, href))

核心数据提取逻辑 #

• 日期提取：从每个txt链接的文件名部分解码得到日期字符串，转换为pandas datetime格式作为DataFrame的行索引。
• 文本定位：逐行遍历txt内容，定位第一个独立的REGION标记行，后续连续有效行直到遇到空行或UNITED STATES行即停止，仅保留该区间内容，避免混入后面的燃气用户加权REGION模块。
• 字段提取：对区间内每一行拆分出区域名称和第一个数值列（即需求的WEEK TOTAL列），组装为单条临时数据。
• 数据集合并：将所有单条数据按日期纵向拼接，最后按日期索引排序即可得到结构化数据集。

编码与pd.read_csv使用说明 #

• 编码确定：该站点所有txt文件仅包含英文、数字、符号，为ASCII兼容编码，直接使用utf-8或latin-1读取即可，不会出现乱码，无需额外使用第三方库探测编码。
• pd.read_csv适用方式：如果需要用pandas读取提取到的REGION模块内容，需先将纯文本转为可解析的流对象，指定空白符分割、仅取需要的列，示例：

from io import StringIO
import pandas as pd

# region_text为提取到的首个REGION模块的纯文本
df = pd.read_csv(
    StringIO(region_text),
    sep=r'\s+',  # 按任意数量的空格/制表符分割
    header=None,
    usecols=[0, 1],  # 第0列为区域名，第1列为WEEK TOTAL
    names=['region', 'week_total'],
    encoding='utf-8'
)

完整可运行代码 #

from bs4 import BeautifulSoup as bs
import requests
import re
from urllib.parse import unquote, urljoin
from io import StringIO
import pandas as pd
from datetime import datetime

# 1. 爬取所有txt文件链接
topdir = 'https://ftp.cpc.ncep.noaa.gov/htdocs/products/analysis_monitoring/cdus/degree_days/archives/Heating%20degree%20Days/weekly%20states/'
res = requests.get(topdir)
soup = bs(res.text, 'html.parser')

toplinks = []
midlinks = []
txtlinks = []

for link in soup.find_all('a'):
    href = link.get('href', '')
    if re.fullmatch(r'2\d{3}/', href):
        toplinks.append(href)

for href in toplinks:
    midlinks.append(urljoin(topdir, href))

for mlink in midlinks:
    mid_res = requests.get(mlink)
    msoup = bs(mid_res.text, 'html.parser')
    for link in msoup.find_all('a'):
        href = link.get('href', '')
        if re.fullmatch(r'[a-zA-Z]{3}%20\d{1,2},%20\d{4}\.txt', href):
            txtlinks.append(urljoin(mlink, href))

# 2. 遍历每个txt文件提取数据
all_data = []
for txt_url in txtlinks:
    # 从链接提取并解析日期
    filename = unquote(txt_url.split('/')[-1].replace('.txt', ''))
    date = datetime.strptime(filename, '%b %d, %Y')
    
    # 读取txt内容
    txt_res = requests.get(txt_url)
    txt_res.encoding = 'utf-8'
    lines = txt_res.text.splitlines()
    
    # 定位首个REGION模块的边界
    region_start = None
    region_end = None
    for i, line in enumerate(lines):
        stripped_line = line.strip()
        if stripped_line == 'REGION' and region_start is None:
            region_start = i + 1
            continue
        if region_start is not None:
            # 遇到空行或UNITED STATES行即停止
            if stripped_line == '' or stripped_line == 'UNITED STATES':
                region_end = i
                break
    
    if region_start is None or region_end is None:
        continue
    
    # 提取模块内的文本
    region_lines = lines[region_start:region_end]
    region_text = '\n'.join(region_lines)
    
    # 用pd.read_csv读取需要的列
    df_region = pd.read_csv(
        StringIO(region_text),
        sep=r'\s+',
        header=None,
        usecols=[0, 1],
        names=['region', 'week_total'],
        encoding='utf-8'
    )
    
    # 转为以区域为列、值为WEEK TOTAL的单行Series
    row = pd.Series(
        data=df_region['week_total'].values,
        index=df_region['region'].values,
        name=date
    )
    all_data.append(row)

# 3. 合并为最终数据集并排序
final_df = pd.DataFrame(all_data).sort_index()

内容的提问来源于stack exchange，提问作者poorpractice