Python解析XML RSS feed：清除无用标签、自定义输出NITF格式问题咨询

前置依赖 #

需要先安装HTML解析库处理内嵌的HTML标签，执行以下命令安装：

pip install beautifulsoup4

修改后完整代码 #

import os
import requests
import xml.etree.ElementTree as ET
from bs4 import BeautifulSoup
from datetime import datetime

def loadRSS():
    url = "https://timesofindia.indiatimes.com/toirssfeed/-2128936835.cms"
    resp = requests.get(url)
    with open('topnewsfeed.xml', 'wb') as f:
        f.write(resp.content)

def clean_html(raw_html):
    # 清除HTML标签，返回纯文本
    soup = BeautifulSoup(raw_html, 'html.parser')
    return soup.get_text(separator='\n', strip=True)

def sanitize_filename(filename):
    # 清除文件名中的非法字符，避免存储报错
    illegal_chars = ['<', '>', ':', '"', '/', '\\', '|', '?', '*']
    for char in illegal_chars:
        filename = filename.replace(char, '_')
    return filename[:100] # 限制文件名长度避免过长

def build_nitf_xml(title, content, pub_date=None):
    # 构造NITF格式XML
    nitf = ET.Element('nitf')
    # 构造head部分
    head = ET.SubElement(nitf, 'head')
    ET.SubElement(head, 'title').text = title
    # 构造iim部分，可根据实际业务需求修改ds字段的值
    iim = ET.SubElement(head, 'iim', attrib={'ver': '3'})
    # 示例iim字段，可根据原RSS的实际信息填充
    current_date = datetime.now().strftime('%Y%m%d')
    current_time = datetime.now().strftime('%H%M%S+0000')
    ds_list = [
        ('1:20', ''),
        ('1:30', 'Times of India'),
        ('1:40', ''),
        ('1:50', 'TOI'),
        ('1:70', pub_date.strftime('%Y%m%d') if pub_date else current_date),
        ('1:80', pub_date.strftime('%H%M%S+0000') if pub_date else current_time),
        ('2:05', f"TOI.{current_date}_{title.replace(' ', '-')}.XML"),
        ('2:07', ''),
        ('2:10', '3'),
        ('2:15', 'OEC'),
        ('2:20', ''),
        ('2:22', ''),
        ('2:25', ''),
        ('2:50', ''),
        ('2:55', pub_date.strftime('%Y%m%d') if pub_date else current_date),
        ('2:80', ''),
        ('2:85', ''),
        ('2:90', ''),
        ('2:95', ''),
        ('2:101', ''),
        ('2:105', title),
        ('2:110', 'Times of India'),
        ('2:115', 'Times of India'),
        ('2:120', content.split('\n')[0] if content else '')
    ]
    for num, value in ds_list:
        ET.SubElement(iim, 'ds', attrib={'num': num, 'value': value})
    # 构造body部分
    body = ET.SubElement(nitf, 'body')
    body_content = ET.SubElement(body, 'body.content')
    # 按换行拆分内容为多个p标签
    for para in content.split('\n'):
        if para.strip():
            ET.SubElement(body_content, 'p').text = para.strip()
    # 生成XML字符串，添加声明
    xml_str = b'<?xml version="1.0" encoding="UTF-8"?>\n' + ET.tostring(nitf, encoding='utf-8')
    return xml_str

def wire_xml(filename):
    # 自动创建xml存储目录
    if not os.path.exists('./xml'):
        os.makedirs('./xml')
    context = ET.iterparse(filename, events=('end', ))
    for event, elem in context:
        if elem.tag == 'article':
            # 提取原字段
            title = elem.find('headline').text.strip()
            raw_content = elem.find('content').text
            # 处理HTML标签得到纯文本
            clean_content = clean_html(raw_content)
            # 可以提取原文章的发布时间，这里示例如果有pubDate字段就取
            pub_date = None
            pub_date_node = elem.find('pubDate')
            if pub_date_node is not None and pub_date_node.text:
                try:
                    pub_date = datetime.strptime(pub_date_node.text, '%a, %d %b %Y %H:%M:%S %Z')
                except:
                    pass
            # 构造NITF XML
            nitf_xml = build_nitf_xml(title, clean_content, pub_date)
            # 处理文件名
            out_filename = sanitize_filename(title) + '.xml'
            with open(os.path.join('./xml', out_filename), 'wb') as f:
                f.write(nitf_xml)
            # 清理节点释放内存
            elem.clear()

def main():
    loadRSS()
    wire_xml('topnewsfeed.xml') 
      
if __name__ == "__main__":
    main()

修改说明 #

• 新增clean_html函数，使用BeautifulSoup提取content字段的纯文本，自动清除所有内嵌的<div>、<br/>等HTML标签
• 新增build_nitf_xml函数，完全按照要求的NITF结构构造输出XML，iim下的ds字段可根据实际业务需求调整取值，当前默认填充了Times of India相关的来源信息
• 新增文件名非法字符处理、目录自动创建逻辑，避免运行时出现存储报错
• 新增节点清理逻辑，避免处理大量XML时内存占用过高

内容的提问来源于stack exchange，提问作者Aleem