diff --git a/.env.example b/.env.example index fdc5c72..c50a162 100644 --- a/.env.example +++ b/.env.example @@ -21,4 +21,7 @@ MAX_FILE_SIZE=10485760 # 日志配置 LOG_LEVEL=INFO -LOG_DIR=./logs \ No newline at end of file +LOG_DIR=./logs + +# Tokenizers 配置 +TOKENIZERS_PARALLELISM=false \ No newline at end of file diff --git a/config.py b/config.py index a79027a..6671221 100644 --- a/config.py +++ b/config.py @@ -32,16 +32,23 @@ class Config: LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO") LOG_DIR = os.getenv("LOG_DIR", "./logs") + # Tokenizers 配置 + TOKENIZERS_PARALLELISM = os.getenv("TOKENIZERS_PARALLELISM", "false") + @classmethod def validate(cls): """验证配置""" if not cls.OPENAI_API_KEY: raise ValueError("OPENAI_API_KEY 环境变量未设置") + # 设置 tokenizers 并行化配置 + os.environ["TOKENIZERS_PARALLELISM"] = cls.TOKENIZERS_PARALLELISM + # 创建必要的目录 os.makedirs(cls.UPLOAD_DIR, exist_ok=True) os.makedirs(cls.LOG_DIR, exist_ok=True) os.makedirs(cls.CHROMA_PERSIST_DIRECTORY, exist_ok=True) + # 创建配置实例 config = Config() diff --git a/requirements.txt b/requirements.txt index d25d9c0..11b240b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,6 +6,7 @@ langchain==0.1.0 langchain-community==0.0.10 langchain-openai==0.0.2 chromadb==0.4.22 +posthog==3.0.1 sentence-transformers==2.2.2 huggingface-hub==0.16.4 PyPDF2==3.0.1 diff --git a/services/vector_store.py b/services/vector_store.py index baf746b..98457ad 100644 --- a/services/vector_store.py +++ b/services/vector_store.py @@ -3,18 +3,17 @@ import os # 在导入 chromadb 之前设置环境变量 os.environ["CHROMA_TELEMETRY"] = "false" os.environ["ANONYMIZED_TELEMETRY"] = "false" -import chromadb + from typing import List, Dict, Any import asyncio - +import chromadb from chromadb.config import Settings from sentence_transformers import SentenceTransformer from langchain.text_splitter import RecursiveCharacterTextSplitter import uuid from datetime import datetime - class AsyncVectorStore: """异步向量存储服务"""