Python使用BeautifulSoup提取EML文件URL遇换行等号问题求助

我尝试读取EML文件并提取其中所有URL，编写了两个方法：body_to_text()通过BytesParser或BeautifulSoup获取邮件正文，find_links()用正则匹配URL。多数样本运行正常，但处理非多部分邮件时，若内容含行尾等号，BeautifulSoup解析会出现异常，导致提取的URL不完整。更换html.parser、html5lib解析器均无效，请问是否是目标邮件的编码问题？

代码示例：

def body_to_text():
        with open("email.eml", "rb") as email_file:
            email_message = email.message_from_binary_file(email_file)

        if email_message.is_multipart():
            with open(self.email, 'rb') as fp:
                msg = BytesParser(policy=policy.default).parse(fp)

            try:
                body_text = msg.get_body(preferencelist=('plain')).get_content().strip()
            except AttributeError:
                print("No body found")
            else:
                body_text = body_text.replace("\n", "")
                
                if body_text == "":
                    print("No body found")
                else:
                    self.find_links(body_text)

        else:            
            body_html = email_message.get_payload()
            soup = BeautifulSoup(body_html, "lxml")
            find_links(soup)

def find_links(scan_text):
        WEB_URL_REGEX = r"""(?i)\b((?:https?:(?:/{1,3}|[a-z0-9%])|[a-z0-9.\-]+[.](?:com|net|org|edu|gov|mil|aero|asia|biz|cat|coop|info|int|jobs|mobi|museum|name|post|pro|tel|travel|xxx|ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cs|cu|cv|cx|cy|cz|dd|de|dj|dk|dm|do|dz|ec|ee|eg|eh|er|es|et|eu|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|mc|md|me|mg|mh|mk|ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|om|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ro|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|Ja|sk|sl|sm|sn|so|sr|ss|st|su|sv|sx|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|uk|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|yu|za|zm|zw)/)(?:[^\s()<>{}[\]]+|\([^\s()]*?\([^\s()]+\)[^\s()]*?\)|\([^\s]+?\))+(?:\([^\s()]*?\([^\s()]+\)[^\s()]*?\)|\([^\s]+?\)|[^\s`!()\[\]{};:'".,<>?«»“”‘’])|(?:(?<!@)[a-z0-9]+(?:[.\-][a-z0-9]+)*[.](?:com|net|org|edu|gov|mil|aero|asia|biz|cat|coop|info|int|jobs|mobi|museum|name|post|pro|tel|travel|xxx|ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cs|cu|cv|cx|cy|cz|dd|de|dj|dk|dm|do|dz|ec|ee|eg|eh|er|es|et|eu|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|mc|md|me|mg|mh|mk|ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|om|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ro|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|Ja|sk|sl|sm|sn|so|sr|ss|st|su|sv|sx|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|uk|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|yu|za|zm|zw)\b/?(?!@)))"""
        links = re.findall(WEB_URL_REGEX, str(scan_text))

        links = list(dict.fromkeys(self.links))

        print(f"{len(self.links)} links found")
        print(links)

异常情况：
print(body_html)输出：

<a href=3D"http://fawper.xyz/corruptly/676197486/trout/gen=
eralizing/1683814388/upgather/disjoin" style=3D"-webkit-text-size-adjust:no=
ne;text-decoration:none;"> <font style=3D"-webkit-text-size-adjust:none;fon=
t-size:15px;

print(soup)输出：

href='3D"http://fawper.xyz/corruptly/676197486/trout/gen=' ne="" style='3D"-webkit-text-size-adjust:no='> <font style='3D"-webkit-text-size-adjust:none;fon=' t-size:15px=""

• 实际提取的URL：'http://fawper.xyz/corruptly/676197486/trout/gen='
• 期望提取的URL：'http://fawper.xyz/corruptly/676197486/trout/generalizing/1683814388/upgather/disjoin'

这不是解析器的问题，而是你没有对邮件内容做Quoted-Printable解码。从输出的=3D和行尾的=可以看出，邮件正文采用了Quoted-Printable编码：

• =3D是=字符的Quoted-Printable编码形式
• 行尾的单个=表示该行内容是续行，需要和下一行拼接
  直接将未解码的原始内容传给BeautifulSoup，会导致解析器把编码后的字符当成HTML的一部分，从而破坏标签结构，最终提取出不完整的URL。

1. 正确解码非多部分邮件的内容 #

不要直接调用email_message.get_payload()，而是使用get_content()方法自动处理编码，或者手动进行Quoted-Printable解码：

else:            
    # 自动处理编码，获取解码后的HTML内容
    body_html = email_message.get_content()
    # 若自动解码失效，可手动解码
    # import quopri
    # body_html = quopri.decodestring(email_message.get_payload()).decode('utf-8')
    soup = BeautifulSoup(body_html, "lxml")
    self.find_links(soup)

2. 优化URL提取逻辑 #

直接解析BeautifulSoup对象提取a标签的href属性，比转成字符串用正则更可靠：

def find_links(scan_text):
    links = []
    if isinstance(scan_text, BeautifulSoup):
        # 从HTML中提取所有合法的a标签href属性
        for a_tag in scan_text.find_all('a', href=True):
            href = a_tag['href'].strip()
            if re.match(r'https?://', href, re.IGNORECASE):
                links.append(href)
    else:
        # 处理纯文本内容的正则匹配
        WEB_URL_REGEX = r"""(?i)\b((?:https?:(?:/{1,3}|[a-z0-9%])|[a-z0-9.\-]+[.](?:com|net|org|edu|gov|mil|aero|asia|biz|cat|coop|info|int|jobs|mobi|museum|name|post|pro|tel|travel|xxx|ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cs|cu|cv|cx|cy|cz|dd|de|dj|dk|dm|do|dz|ec|ee|eg|eh|er|es|et|eu|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|mc|md|me|mg|mh|mk|ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|om|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ro|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|Ja|sk|sl|sm|sn|so|sr|ss|st|su|sv|sx|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|uk|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|yu|za|zm|zw)/)(?:[^\s()<>{}[\]]+|\([^\s()]*?\([^\s()]+\)[^\s()]*?\)|\([^\s]+?\))+(?:\([^\s()]*?\([^\s()]+\)[^\s()]*?\)|\([^\s]+?\)|[^\s`!()\[\]{};:'".,<>?«»“”‘’])|(?:(?<!@)[a-z0-9]+(?:[.\-][a-z0-9]+)*[.](?:com|net|org|edu|gov|mil|aero|asia|biz|cat|coop|info|int|jobs|mobi|museum|name|post|pro|tel|travel|xxx|ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cs|cu|cv|cx|cy|cz|dd|de|dj|dk|dm|do|dz|ec|ee|eg|eh|er|es|et|eu|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|mc|md|me|mg|mh|mk|ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|om|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ro|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|Ja|sk|sl|sm|sn|so|sr|ss|st|su|sv|sx|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|uk|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|yu|za|zm|zw)\b/?(?!@)))"""
        links = re.findall(WEB_URL_REGEX, scan_text)
    
    # 去重
    links = list(dict.fromkeys(links))
    print(f"{len(links)} links found")
    print(links)

3. 统一多部分与非多部分邮件的处理逻辑 #

建议统一使用BytesParser处理所有邮件，避免重复代码：

def body_to_text():
    with open("email.eml", "rb") as email_file:
        msg = BytesParser(policy=policy.default).parse(email_file)
    
    # 优先获取纯文本正文，没有则获取HTML
    body_part = msg.get_body(preferencelist=('plain', 'html'))
    if not body_part:
        print("No body found")
        return
    
    body_content = body_part.get_content().strip()
    if not body_content:
        print("No body found")
        return
    
    if body_part.get_content_type() == 'text/html':
        soup = BeautifulSoup(body_content, "lxml")
        self.find_links(soup)
    else:
        self.find_links(body_content)

内容的提问来源于stack exchange，提问作者Cowx