feat: 切分检测
This commit is contained in:
parent
51c83ff4cc
commit
f6a4e43220
|
@ -84,11 +84,12 @@ async def test_advanced_functionality():
|
|||
print("🚀 高级多格式文档和图片内容测试")
|
||||
print("=" * 60)
|
||||
|
||||
# 清理向量数据库
|
||||
db_path = Path("/Users/liruwei/Documents/code/project/demo/base_rag/storage/chroma_db/ad_test")
|
||||
if db_path.exists():
|
||||
shutil.rmtree(db_path)
|
||||
print("🧹 已清理向量数据库")
|
||||
# 清理数据
|
||||
for p in ["/Users/liruwei/Documents/code/project/demo/base_rag/storage/chroma_db/ad_test", "/Users/liruwei/Documents/code/project/demo/base_rag/storage/status_db"]:
|
||||
p_obj = Path(p)
|
||||
if p_obj.exists():
|
||||
shutil.rmtree(p_obj)
|
||||
print("🧹 已清理数据")
|
||||
|
||||
# 创建RAG实例 - 启用图片处理
|
||||
rag = AdvancedTestRAG(
|
||||
|
|
|
@ -39,7 +39,12 @@ class FileManager:
|
|||
def __init__(self, storage_dir: str = "./documents", db_path: str = "./file_status.db"):
|
||||
self.storage_dir = Path(storage_dir)
|
||||
self.db_path = db_path
|
||||
self.storage_dir.mkdir(exist_ok=True)
|
||||
|
||||
# 确保存储目录存在
|
||||
Path(storage_dir).mkdir(parents=True, exist_ok=True)
|
||||
# 确保数据库目录存在
|
||||
Path(db_path).parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
self._init_lock = asyncio.Lock()
|
||||
self._db_initialized = False
|
||||
|
||||
|
@ -832,7 +837,7 @@ class BaseRAG(ABC):
|
|||
return splitter.split_documents(documents)
|
||||
|
||||
split_docs = await loop.run_in_executor(None, _split_docs)
|
||||
|
||||
# print(split_docs)
|
||||
# 7. 为每个切分的文档添加元数据
|
||||
for doc in split_docs:
|
||||
doc.metadata.update({
|
||||
|
|
File diff suppressed because one or more lines are too long
Binary file not shown.
Binary file not shown.
|
@ -0,0 +1,6 @@
|
|||
日期,产品,销售额,数量,客户类型,销售员
|
||||
2024-01-01,笔记本电脑,8500,5,企业,张三
|
||||
2024-01-02,台式机,6200,4,个人,李四
|
||||
2024-01-03,平板电脑,3200,8,学生,王五
|
||||
2024-01-04,智能手机,4500,9,个人,张三
|
||||
2024-01-05,耳机,280,12,学生,李四
|
|
Binary file not shown.
Loading…
Reference in New Issue