如何修正句子分词代码避免引号迁移至后续句子？

我正在把PDF里的文本抓取到Excel，需要把故事文本分成句子。现在基本实现了，但有个问题：当句子以闭合引号结尾，且下一句在新段落或新页面时，闭合引号会跑到下一句里，而不是留在前句。比如文本里“makeve!”后面的引号会被带到下一页的句子中。请帮忙修改代码解决这个问题。

现有代码 #

import pdfplumber
import pandas as pd
import nltk
import re
import glob

def exclude_page_numbers(page_text, page_elements, exclusion_regions):
    filtered_text = ""

    for element in page_elements:
        text = element["text"]
        x, y = element["x0"], element["top"]  # 获取元素的x、y坐标

        # 检查元素是否在排除区域内
        exclude = False
        for region in exclusion_regions:
            x_min, y_min, x_max, y_max = region
            if x_min <= x <= x_max and y_min <= y <= y_max:
                exclude = True
                break

        if not exclude and not any(char.isdigit() for char in text) and not text.strip().isdigit():
            filtered_text += text + " "  # 每个元素后加空格

    return filtered_text


def extract_sentences(text):
    # 基于.!?分句，同时考虑句子内的引号配对
    sentences = re.split(r'(?<=[.!?])(?=(?:[^"]*"[^"]*")*[^"]*$)', text)
    sentences = [sentence.strip() for sentence in sentences]

    # 尝试将闭合引号合并到前一句（当前逻辑未生效）
    merged_sentences = []
    i = 0
    while i < len(sentences):
        if sentences[i].endswith('"'):
            if i + 1 < len(sentences):
                merged_sentence = sentences[i] + ' ' + sentences[i + 1]
                merged_sentences.append(merged_sentence)
                i += 2
            else:
                merged_sentences.append(sentences[i])
                i += 1
        else:
            merged_sentences.append(sentences[i])
            i += 1

    # 使用nltk对合并后的句子做二次分句
    tokenized_sentences = []
    for sentence in merged_sentences:
        tokenized_sentences.extend(nltk.sent_tokenize(sentence))

    return tokenized_sentences

def extract_text_from_pdf(pdf_file_path):
    with pdfplumber.open(pdf_file_path) as pdf:
        num_pages = len(pdf.pages)
        text = ""
        for page in range(3, num_pages - 2):
            page_obj = pdf.pages[page]
            page_text = page_obj.extract_text()
            page_elements = page_obj.extract_words()  # 获取文本元素及其位置
            filtered_text = exclude_page_numbers(page_text, page_elements, exclusion_regions)
            text += filtered_text

        return text

def scrape_sentences_to_excel(pdf_file_paths, excel_file_path):
    sentences = []
    for pdf_file_path in pdf_file_paths:
        # 提取PDF文本
        pdf_text = extract_text_from_pdf(pdf_file_path)

        # 分句处理
        extracted_sentences = extract_sentences(pdf_text)
        sentences.extend(extracted_sentences)

    # 创建DataFrame并写入Excel
    df = pd.DataFrame({'Sentences': sentences})
    df.to_excel(excel_file_path, index=False)

    print("句子提取并保存到Excel成功！")


pdf_folder_path = 'my_pdf_file_path'  # PDF文件夹路径
excel_file_path = 'me_excel_file_path'  # 输出Excel路径

# 定义排除区域（x_min, y_min, x_max, y_max）
exclusion_regions = [(42, 558, 359, 559)]  # 添加你的排除区域

# 获取文件夹内所有PDF文件
pdf_file_paths = glob.glob(pdf_folder_path + '/*.pdf')

# 执行提取并保存
scrape_sentences_to_excel(pdf_file_paths, excel_file_path)

问题根源是PDF跨页/跨段落提取时，闭合引号和前一句被拆分，后续分句逻辑未处理这种边界情况。可以从以下两个关键环节修改：

1. 优化PDF文本拼接逻辑，提前处理跨页引号 #

在extract_text_from_pdf函数中，拼接页面文本时检查当前页开头是否为闭合引号，若是则将其移至上一页文本末尾：

def extract_text_from_pdf(pdf_file_path):
    with pdfplumber.open(pdf_file_path) as pdf:
        num_pages = len(pdf.pages)
        text = ""
        for page in range(3, num_pages - 2):
            page_obj = pdf.pages[page]
            page_elements = page_obj.extract_words()
            filtered_text = exclude_page_numbers(page_obj.extract_text(), page_elements, exclusion_regions)
            
            # 处理跨页的闭合引号：如果当前页开头是引号，移到上一页末尾
            if text and filtered_text.startswith('"'):
                filtered_text = filtered_text[1:].strip()
                text = text.rstrip() + '" '
            
            text += filtered_text + " "

        return text.strip()

2. 修正分句逻辑，确保闭合引号归属正确 #

原extract_sentences函数的合并逻辑方向错误，应检查下一句是否以引号开头，而非当前句以引号结尾：

def extract_sentences(text):
    # 先清理多余空格，避免干扰分句
    text = re.sub(r'\s+', ' ', text).strip()
    
    # 基于标点分句，同时考虑引号配对
    sentences = re.split(r'(?<=[.!?])(?=(?:[^"]*"[^"]*")*[^"]*$)', text)
    sentences = [s.strip() for s in sentences if s.strip()]

    # 处理被拆分到下一句的闭合引号
    merged_sentences = []
    i = 0
    while i < len(sentences):
        current_sent = sentences[i]
        # 检查下一句是否以闭合引号开头，且当前句是完整句子
        if i + 1 < len(sentences) and sentences[i+1].startswith('"'):
            merged_sent = f"{current_sent}\""
            merged_sentences.append(merged_sent)
            i += 2
        else:
            merged_sentences.append(current_sent)
            i += 1
    
    # 用nltk做二次分句优化
    final_sentences = []
    for sent in merged_sentences:
        final_sentences.extend(nltk.sent_tokenize(sent))
    
    # 清理空句子
    final_sentences = [s.strip() for s in final_sentences if s.strip()]
    
    return final_sentences

3. 额外准备：加载nltk分句模型 #

第一次运行前需下载分句模型：

nltk.download('punkt')

内容的提问来源于stack exchange，提问作者Paige Cox