如何基于OpenAI、Pinecone、Langchain实现多PDF文档问答？

已实现基于OpenAI、Pinecone和Langchain的单PDF文档问答功能，现有代码可正常处理单个PDF的问答需求，但不清楚如何将其扩展为支持多PDF文档的问答系统，原代码如下：

# process_message.py
from flask import request
import pinecone
# from PyPDF2 import PdfReader
from langchain.embeddings.openai import OpenAIEmbeddings
from langchain.text_splitter import CharacterTextSplitter
from langchain.vectorstores import ElasticVectorSearch, Pinecone, Weaviate, FAISS
from langchain.chains.question_answering import load_qa_chain
from langchain.llms import OpenAI
import os
import json
# from constants.company import file_company_id_column, file_location_column, file_name_column
from services.files import FileFireStorage
from middleware.auth import check_authorization
import configparser
from langchain.document_loaders import UnstructuredPDFLoader, OnlinePDFLoader, PyPDFLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter


def process_message():
    
    # Create a ConfigParser object and read the config.ini file
    config = configparser.ConfigParser()
    config.read('config.ini')
    # Retrieve the value of OPENAI_API_KEY
    openai_key = config.get('openai', 'OPENAI_API_KEY')
    pinecone_env_key = config.get('pinecone', 'PINECONE_ENVIRONMENT')
    pinecone_api_key = config.get('pinecone', 'PINECONE_API_KEY')


    loader = PyPDFLoader("docs/ops.pdf")
    data = loader.load()
    # data = body['data'][1]['name']
    # Print information about the loaded data
    print(f"You have {len(data)} document(s) in your data")
    print(f"There are {len(data[30].page_content)} characters in your document")

    # Chunk your data up into smaller documents
    text_splitter = RecursiveCharacterTextSplitter(chunk_size=2000, chunk_overlap=0)
    texts = text_splitter.split_documents(data)
   

    embeddings = OpenAIEmbeddings(openai_api_key=openai_key)

    pinecone.init(api_key=pinecone_api_key, environment=pinecone_env_key)
    index_name = "pdf-chatbot"  # Put in the name of your Pinecone index here

    docsearch = Pinecone.from_texts([t.page_content for t in texts], embeddings, index_name=index_name)
    # Query those docs to get your answer back
    llm = OpenAI(temperature=0, openai_api_key=openai_key)
    chain = load_qa_chain(llm, chain_type="stuff")

    query = "Are there any other documents listed in this document?"
    docs = docsearch.similarity_search(query)
    answer = chain.run(input_documents=docs, question=query)
    print(answer)

    return answer

核心修改步骤 #

1. 批量加载目录下的所有PDF文件 #

替换原代码中固定加载单个PDF的逻辑，改为遍历指定目录下的所有.pdf文件，逐个加载并合并文档数据。

2. 保留文档元数据（可选但推荐） #

在加载每个PDF时，将文件名作为元数据添加到文档对象中，后续问答时可以追踪答案来源，提升结果可信度。

3. 优化向量库写入逻辑 #

避免每次调用都重新写入Pinecone索引，可选择一次性合并所有文档片段后写入，或增量添加新文档（需处理重复问题）。

4. 可选：增强问答结果的来源标识 #

修改问答链的输出，将匹配到的文档元数据（如文件名、页码）附加到答案中。

修改后的完整代码示例 #

# process_message.py
from flask import request
import pinecone
import os
import configparser
from langchain.embeddings.openai import OpenAIEmbeddings
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.vectorstores import Pinecone
from langchain.chains.question_answering import load_qa_chain
from langchain.llms import OpenAI
from langchain.document_loaders import PyPDFLoader


def process_message():
    # 读取配置
    config = configparser.ConfigParser()
    config.read('config.ini')
    openai_key = config.get('openai', 'OPENAI_API_KEY')
    pinecone_env_key = config.get('pinecone', 'PINECONE_ENVIRONMENT')
    pinecone_api_key = config.get('pinecone', 'PINECONE_API_KEY')

    # -------------------------- 1. 批量加载多PDF文件 --------------------------
    pdf_dir = "docs/"  # 存放PDF的目录
    all_data = []
    
    # 遍历目录下所有PDF文件
    for filename in os.listdir(pdf_dir):
        if filename.endswith(".pdf"):
            file_path = os.path.join(pdf_dir, filename)
            loader = PyPDFLoader(file_path)
            docs = loader.load()
            
            # -------------------------- 2. 添加文档元数据 --------------------------
            # 为每个页面添加文件名元数据
            for doc in docs:
                doc.metadata["source"] = filename
            
            all_data.extend(docs)
    
    print(f"已加载 {len(all_data)} 个页面（来自 {len([f for f in os.listdir(pdf_dir) if f.endswith('.pdf')])} 个PDF文件）")

    # 分割文档片段
    text_splitter = RecursiveCharacterTextSplitter(chunk_size=2000, chunk_overlap=0)
    texts = text_splitter.split_documents(all_data)

    # 初始化向量模型和Pinecone
    embeddings = OpenAIEmbeddings(openai_api_key=openai_key)
    pinecone.init(api_key=pinecone_api_key, environment=pinecone_env_key)
    index_name = "pdf-chatbot"

    # -------------------------- 3. 写入Pinecone向量库 --------------------------
    # 注意：如果索引已存在，from_texts会追加数据；若需清空重建，先调用pinecone.delete_index(index_name)再重建
    docsearch = Pinecone.from_texts(
        [t.page_content for t in texts],
        embeddings,
        index_name=index_name,
        metadatas=[t.metadata for t in texts]  # 传入元数据
    )

    # 问答逻辑
    llm = OpenAI(temperature=0, openai_api_key=openai_key)
    chain = load_qa_chain(llm, chain_type="stuff")

    # 示例查询（可替换为请求传入的查询）
    query = "Are there any other documents listed in this document?"
    docs = docsearch.similarity_search(query)
    
    # -------------------------- 4. 可选：附加来源信息到答案 --------------------------
    # 收集匹配到的文档来源
    sources = set(doc.metadata["source"] for doc in docs)
    answer = chain.run(input_documents=docs, question=query)
    answer_with_sources = f"{answer}\n\n参考来源：{', '.join(sources)}"
    
    print(answer_with_sources)
    return answer_with_sources

额外注意事项 #

• 索引重复数据问题：如果多次运行该函数，会重复向Pinecone索引添加数据。若需避免，可在写入前检查索引是否存在，或使用Pinecone.from_existing_index加载已有索引并增量添加新文档。
• 目录权限：确保程序对pdf_dir目录有读取权限，若使用外部存储（如FireStorage），需修改加载逻辑从存储服务获取文件路径。
• 性能优化：对于大量PDF文件，建议将文档加载和向量写入逻辑独立为定时任务，避免每次问答请求都重复执行加载和写入操作。

内容的提问来源于stack exchange，提问作者user17281101