Telegram机器人中pdfminer+Langchain做PDF提取的依赖问题

我正在开发一款支持用户上传PDF文件的Telegram机器人，计划用pdfminer提取PDF文本，结合Langchain响应用户查询。原代码依赖OpenAI API密钥，现在想移除这个依赖，只用pdfminer和Langchain处理，已经更新了create_index.py和conversation.py，但运行时遇到一系列导入错误：

• 首先出现ImportError：无法从langchain_community.chains导入ConversationalRetrievalChain
• 尝试安装Langchain 0.0.181版本后，又出现ImportError：无法从pydantic导入field_validator
• 更新pydantic版本后，该错误仍未解决

更新后的create_index.py代码 #

import os
from langchain_community.vectorstores.chroma import Chroma
from langchain.text_splitter import CharacterTextSplitter
from langchain_community.document_loaders import DirectoryLoader, TextLoader
from pdfminer.high_level import extract_text  # Import pdfminer to extract text from PDFs

from config import config

def create_index(file_path: str) -> None:
    # Use pdfminer to extract text from the PDF file
    text = extract_text(file_path)

    # Save extracted text to a file
    with open(f'{config.OUTPUT_DIR}/output.txt', 'w') as file:
        file.write(text)

    loader = DirectoryLoader(
        config.OUTPUT_DIR,
        glob='**/*.txt',
        loader_cls=TextLoader
    )

    documents = loader.load()

    text_splitter = CharacterTextSplitter(
        separator='\n',
        chunk_size=1024,
        chunk_overlap=128
    )

    texts = text_splitter.split_documents(documents)

    # Instead of embeddings, we can create an index directly using the text
    persist_directory = config.DB_DIR

    vectordb = Chroma.from_documents(
        documents=texts,
        persist_directory=persist_directory
    )

    vectordb.persist()

更新后的conversation.py代码 #

from langchain_community.vectorstores import Chroma
from langchain_community.document_loaders import DirectoryLoader, TextLoader
from langchain_community.chat_models import ChatOpenAI
from langchain_community.chains import ConversationalRetrievalChain
from langchain_community.memory import ConversationBufferMemory

from config import config

def create_conversation() -> ConversationalRetrievalChain:
    persist_directory = config.DB_DIR

    db = Chroma(
        persist_directory=persist_directory
    )

    memory = ConversationBufferMemory(
        memory_key='chat_history',
        return_messages=False
    )

    qa = ConversationalRetrievalChain.from_llm(
        llm=ChatOpenAI(),
        chain_type='stuff',
        retriever=db.as_retriever(),
        memory=memory,
        get_chat_history=lambda h: h,
        verbose=True
    )

    return qa

第一轮错误栈 #

Traceback (most recent call last):

File "C:\Users\User\Telegramv1\Telegramv1jeeva\run.py", line 1, in <module>
  from document_gpt.src.main import app
File "C:\Users\User\Telegramv1\Telegramv1jeeva\document_gpt\src\main.py", line 3, in <module>
  from document_gpt.helper.utils import process_telegram_data, generate_text_response, generate_file_response
File "C:\Users\User\Telegramv1\Telegramv1jeeva\document_gpt\helper\utils.py", line 191, in <module>
  from document_gpt.helper.conversation import create_conversation
File "C:\Users\User\Telegramv1\Telegramv1jeeva\document_gpt\helper\conversation.py", line 77, in <module>
  from langchain_community.chains import ConversationalRetrievalChain
ImportError: cannot import name 'ConversationalRetrievalChain' from 'langchain_community.chains' (C:\Users\User\Telegramv1\Telegramv1jeeva\venv\Lib\site-packages\langchain_community\chains\__init__.py)

降级Langchain后的错误栈 #

Traceback (most recent call last):
File "C:\Users\User\Telegramv1\Telegramv1jeeva\run.py", line 1, in <module>
  from document_gpt.src.main import app
File "C:\Users\User\Telegramv1\Telegramv1jeeva\document_gpt\src\main.py", line 3, in <module>
  from document_gpt.helper.utils import process_telegram_data, generate_text_response, generate_file_response
File "C:\Users\User\Telegramv1\Telegramv1jeeva\document_gpt\helper\utils.py", line 191, in <module>
  from document_gpt.helper.conversation import create_conversation
File "C:\Users\User\Telegramv1\Telegramv1jeeva\document_gpt\helper\conversation.py", line 74, in <module>
  from langchain_community.vectorstores import Chroma
File "C:\Users\User\Telegramv1\Telegramv1jeeva\venv\Lib\site-packages\langchain_community\vectorstores\__init__.py", line 524, in __getattr__
  module = importlib.import_module(module_lookup[name])
  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\Edify Admin\AppData\Local\Programs\Python\Python311\Lib\importlib\__init__.py", line 126, in import_module
  return _bootstrap._gcd_import(name[level:], package, level)
  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\User\Telegramv1\Telegramv1jeeva\venv\Lib\site-packages\langchain_community\vectorstores\chroma.py", line 20, in <module>
  from langchain_core.documents import Document
File "C:\Users\User\Telegramv1\Telegramv1jeeva\venv\Lib\site-packages\langchain_core\documents\__init__.py", line 6, in <module>
  from langchain_core.documents.base import Document
File "C:\Users\User\Telegramv1\Telegramv1jeeva\venv\Lib\site-packages\langchain_core\documents\base.py", line 10, in <module>
  from pydantic import ConfigDict, Field, field_validator, model_validator
ImportError: cannot import name 'field_validator' from 'pydantic' (C:\Users\User\Telegramv1\Telegramv1jeeva\venv\Lib\site-packages\pydantic\__init__.cp311-win_amd64.pyd)

更新pydantic后的错误栈 #

Traceback (most recent call last):
File "C:\Users\User\Telegramv1\Telegramv1jeeva\run.py", line 1, in <module>
  from document_gpt.src.main import app
File "C:\Users\User\Telegramv1\Telegramv1jeeva\document_gpt\src\main.py", line 3, in <module>
  from document_gpt.helper.utils import process_telegram_data, generate_text_response, generate_file_response
File "C:\Users\User\Telegramv1\Telegramv1jeeva\document_gpt\helper\utils.py", line 191, in <module>
  from document_gpt.helper.conversation import create_conversation
File "C:\Users\User\Telegramv1\Telegramv1jeeva\document_gpt\helper\conversation.py", line 74, in <module>
  from langchain_community.vectorstores import Chroma
File "C:\Users\User\Telegramv1\Telegramv1jeeva\venv\Lib\site-packages\langchain_community\vectorstores\__init__.py", line 524, in __getattr__
  module = importlib.import_module(module_lookup[name])
  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\Edify Admin\AppData\Local\Programs\Python\Python311\Lib\importlib\__init__.py", line 126, in import_module
  return _bootstrap._gcd_import(name[level:], package, level)
  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\User\Telegramv1\Telegramv1jeeva\venv\Lib\site-packages\langchain_community\vectorstores\chroma.py", line 20, in <module>
  from langchain_core.documents import Document
File "C:\Users\User\Telegramv1\Telegramv1jeeva\venv\Lib\site-packages\langchain_core\documents\__init__.py", line 6, in <module>
  from langchain_core.documents.base import Document
File "C:\Users\User\Telegramv1\Telegramv1jeeva\venv\Lib\site-packages\langchain_core\documents\base.py", line 10, in <module>
  from pydantic import ConfigDict, Field, field_validator, model_validator
ImportError: cannot import name 'field_validator' from 'pydantic' (C:\Users\User\Telegramv1\Telegramv1jeeva\venv\Lib\site-packages\pydantic\__init__.cp311-win_amd64.pyd)

内容的提问来源于stack exchange，提问作者Jeeva