如何修复PDF分句时对话与引述语句被拆分的问题？

我正在将PDF中的句子提取至Excel表格，多数场景处理正常，但当对话以?或!结尾且后续带有类似“he said”的引述表述时，引号内容与引述语句会被拆分为两个独立句子。例如祖鲁语例句：

"Ingaba ibhasi isishiyile?" babebuza.
（意为：“‘公交车已经开走了吗？’他们问道。”）
会被拆分为 "Ingaba ibhasi isishiyile?" 和 "babebuza."。该问题存在于所有引号以?或!结尾的场景。我曾尝试合并以?"或!"结尾且后续句子以小写字母开头的语句，但未达预期效果，当前代码如下：

import pdfplumber
import pandas as pd
import nltk
import re
import glob

def exclude_page_numbers(page_text, page_elements, exclusion_regions):
    filtered_text = ""

    for element in page_elements:
        text = element["text"]
        x, y = element["x0"], element["top"]  # 获取元素的x、y坐标

        # 检查元素是否在排除区域内
        exclude = False
        for region in exclusion_regions:
            x_min, y_min, x_max, y_max = region
            if x_min <= x <= x_max and y_min <= y <= y_max:
                exclude = True
                break

        if not exclude and not any(char.isdigit() for char in text) and not text.strip().isdigit():
            filtered_text += text + " "  # 每个元素后加空格

    return filtered_text


def extract_sentences(text):
    # 将中文引号替换为英文标准引号
    text = text.replace("“", '"').replace("”", '"')
    text = text.replace("‘", "'").replace("’", "'")

    # 按.、!、?拆分句子，同时考虑引号内的内容
    sentences = re.split(r'(?<=[.!?])(?=(?:[^"]*"[^"]*")*[^"]*$)', text)
    sentences = [sentence.strip() for sentence in sentences]

    # 将闭合引号与前一句合并
    merged_sentences = []
    i = 0
    while i < len(sentences):
        if sentences[i].endswith('"'):
            if i + 1 < len(sentences):
                merged_sentence = sentences[i] + ' ' + sentences[i + 1]
                merged_sentences.append(merged_sentence)
                i += 2
            else:
                merged_sentences.append(sentences[i])
                i += 1
        else:
            merged_sentences.append(sentences[i])
            i += 1

    # 若前一句以?"或!"结尾且下一句以小写开头，则合并相邻句子
    final_sentences = []
    i = 0
    while i < len(merged_sentences):
        if i + 1 < len(merged_sentences):
            first_sentence = merged_sentences[i]
            second_sentence = merged_sentences[i + 1]
            if re.search(r'[?!]"?\s*$', first_sentence) and re.search(r'^[a-z]', second_sentence):
                joined_sentence = first_sentence + ' ' + second_sentence
                final_sentences.append(joined_sentence)
                i += 2
                continue
        final_sentences.append(merged_sentences[i])
        i += 1

    # 使用nltk对最终句子进行分句
    tokenized_sentences = []
    for sentence in final_sentences:
        tokenized_sentences.extend(nltk.sent_tokenize(sentence))

    return tokenized_sentences

def extract_text_from_pdf(pdf_file_path):
    with pdfplumber.open(pdf_file_path) as pdf:
        num_pages = len(pdf.pages)
        text = ""
        for page in range(3, num_pages - 2):
            page_obj = pdf.pages[page]
            page_text = page_obj.extract_text()
            page_elements = page_obj.extract_words()  # 获取文本元素及其位置
            filtered_text = exclude_page_numbers(page_text, page_elements, exclusion_regions)

def scrape_sentences_to_excel(pdf_file_paths, excel_file_path):
    sentences = []
    for pdf_file_path in pdf_file_paths:
        # 从PDF提取文本
        pdf_text = extract_text_from_pdf(pdf_file_path)

        # 从文本中提取句子
        extracted_sentences = extract_sentences(pdf_text)
        sentences.extend(extracted_sentences)

    # 创建包含句子列的DataFrame
    df = pd.DataFrame({'Sentences': sentences})

    # 将DataFrame写入Excel文件
    df.to_excel(excel_file_path, index=False)

    print("句子提取并保存至Excel成功！")

pdf_folder_path = 'C:/Users/Paige Cox/Desktop/Little Zebra Books/pdfs'  # PDF文件夹路径
excel_file_path = 'C:/Users/Paige Cox/Desktop/Little Zebra Books/pdfs/sentence_corpus_xhosa.xlsx'  # 输出Excel路径

# 指定排除区域（x_min, y_min, x_max, y_max）
exclusion_regions = [(42, 558, 359, 559)]  # 添加你的排除区域

# 获取文件夹中的所有PDF文件
pdf_file_paths = glob.glob(pdf_folder_path + '/*.pdf')

# 从PDF提取句子并保存到Excel
scrape_sentences_to_excel(pdf_file_paths, excel_file_path)

问题根源有三点：初始正则拆分逻辑会误拆分引号内的?/!、后续的nltk.sent_tokenize会再次拆分已合并的句子、extract_text_from_pdf存在语法错误。以下是针对性修复：

1. 修正文本提取函数的语法错误 #

原函数未将每页过滤后的文本拼接并返回，修改后：

def extract_text_from_pdf(pdf_file_path):
    with pdfplumber.open(pdf_file_path) as pdf:
        num_pages = len(pdf.pages)
        text = ""
        for page in range(3, num_pages - 2):
            page_obj = pdf.pages[page]
            page_elements = page_obj.extract_words()
            filtered_text = exclude_page_numbers(page_obj.extract_text(), page_elements, exclusion_regions)
            text += filtered_text + " "
    return text

2. 调整句子拆分规则 #

修改正则，避免在引号内的?/!后拆分，仅当引号后是大写字母或文本结束时才拆分：

def extract_sentences(text):
    # 替换中文引号
    text = text.replace("“", '"').replace("”", '"')
    text = text.replace("‘", "'").replace("’", "'")

    # 修改拆分规则：跳过引号内的?/!拆分，仅在引号后是大写/结尾时拆分
    sentences = re.split(r'(?<=[.!?])(?!")(?=(?:[^"]*"[^"]*")*[^"]*$)|(?<=[.!?]")(?=[A-Z]|\s*$)', text)
    sentences = [sentence.strip() for sentence in sentences if sentence.strip()]

    # 合并引号结尾+小写开头的引述语句
    final_sentences = []
    i = 0
    while i < len(sentences):
        current = sentences[i]
        if i + 1 < len(sentences) and re.search(r'[?!]"$', current) and re.match(r'^[a-z]', sentences[i+1]):
            merged = f"{current} {sentences[i+1]}"
            final_sentences.append(merged)
            i += 2
        else:
            final_sentences.append(current)
            i += 1

    return final_sentences

3. 移除多余的nltk分句步骤 #

nltk.sent_tokenize会重新拆分已合并的句子，直接删除该部分逻辑即可。

修改后，引号结尾的对话与后续引述语句会被正确合并为一个句子，不会被拆分。

内容的提问来源于stack exchange，提问作者Paige Cox