feat: 简易回答
This commit is contained in:
parent
4f43f36e54
commit
021406a297
|
@ -1,88 +0,0 @@
|
|||
# Examples 示例文件
|
||||
|
||||
本目录包含两个主要的测试示例:
|
||||
|
||||
## 📁 simple_test.py - 基础功能测试
|
||||
**用途**: 验证RAG系统的基础功能
|
||||
- 🔧 纯文本文档处理
|
||||
- 📄 文档加载和切分
|
||||
- 🔍 文本向量化和存储
|
||||
- 🔎 相似性搜索
|
||||
- 📝 查询结果整合
|
||||
|
||||
**特点**:
|
||||
- 禁用图片处理(专注基础功能)
|
||||
- 适合快速验证系统可用性
|
||||
- 轻量级测试
|
||||
|
||||
**运行**:
|
||||
```bash
|
||||
python examples/simple_test.py
|
||||
```
|
||||
|
||||
## 🚀 ad_test.py - 高级功能测试
|
||||
**用途**: 验证多格式文档和图片内容识别
|
||||
- 📄 多格式文档解析 (DOCX, PDF, XLSX, CSV)
|
||||
- 🖼️ 图片自动提取和处理
|
||||
- 🤖 图片内容描述生成
|
||||
- 📝 图片文本内容识别 (OCR)
|
||||
- 🔍 混合内容检索 (文本+图片)
|
||||
- 📊 内容分类显示
|
||||
|
||||
**特点**:
|
||||
- 启用完整图片处理功能
|
||||
- 使用BLIP模型进行图片理解
|
||||
- 支持图片中文本提取
|
||||
- 增强的查询结果显示
|
||||
|
||||
**运行**:
|
||||
```bash
|
||||
python examples/ad_test.py
|
||||
```
|
||||
|
||||
## 🔧 图片文本识别功能
|
||||
|
||||
高级测试(`ad_test.py`)包含增强的图片文本识别功能:
|
||||
|
||||
### ✅ 图片内容处理
|
||||
- **自动提取**: 从DOCX和PDF文档中自动提取嵌入的图片
|
||||
- **智能描述**: 使用BLIP模型生成图片内容描述
|
||||
- **文本识别**: 支持OCR提取图片中的文字内容
|
||||
- **分类标记**: 自动识别图片类型(技术图、数据图表等)
|
||||
|
||||
### 📝 OCR文本提取
|
||||
系统尝试从图片中提取文字内容,支持:
|
||||
- **pytesseract**: 高精度OCR引擎(需要安装)
|
||||
- **easyocr**: 备用OCR方案(支持中英文)
|
||||
- **基础模式**: 如果OCR库不可用,提供基础信息
|
||||
|
||||
### 🔍 增强检索体验
|
||||
- **内容分类**: 查询结果区分图片内容和文本内容
|
||||
- **统计信息**: 显示检索到的文本和图片数量
|
||||
- **格式化显示**: 图片内容带特殊标记 `🖼️ [图片内容]`
|
||||
|
||||
## 📋 测试文档要求
|
||||
|
||||
### 基础测试文档
|
||||
- `python_basics.txt` - Python基础知识
|
||||
- `data_science.txt` - 数据科学内容
|
||||
|
||||
### 高级测试文档
|
||||
- `complex_data_science.docx` - 包含图片的Word文档
|
||||
- `ai_research_report.pdf` - 包含图片的PDF报告
|
||||
- `company_report.xlsx` - Excel工作簿
|
||||
- `sales_data.csv` - CSV数据文件
|
||||
|
||||
## 🎯 预期效果
|
||||
|
||||
### 基础测试
|
||||
- ✅ 文档正常加载和处理
|
||||
- ✅ 文本查询返回相关结果
|
||||
- ✅ 系统响应时间正常
|
||||
|
||||
### 高级测试
|
||||
- ✅ 多格式文档成功解析
|
||||
- ✅ 图片内容被自动识别和描述
|
||||
- 🖼️ 图片查询能返回图片相关内容
|
||||
- 📊 查询结果包含内容类型统计
|
||||
- 🔍 图片和文本内容可被统一检索
|
|
@ -1,322 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
高级测试示例 - 多格式文档和图片内容识别
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
import asyncio
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
|
||||
# 过滤掉PyTorch的FutureWarning
|
||||
warnings.filterwarnings("ignore", category=FutureWarning, module="torch")
|
||||
|
||||
# 添加源码路径
|
||||
sys.path.append(os.path.join(os.path.dirname(__file__), "..", "src"))
|
||||
|
||||
from base_rag.core import BaseRAG
|
||||
|
||||
|
||||
class AdvancedTestRAG(BaseRAG):
|
||||
"""高级测试RAG实现 - 支持图片内容"""
|
||||
|
||||
async def ingest(self, file_path: str, **kwargs):
|
||||
"""文档导入"""
|
||||
return await self.process_file_to_vector_store(file_path, **kwargs)
|
||||
|
||||
async def query(self, question: str) -> str:
|
||||
"""查询实现 - 增强图片内容显示"""
|
||||
docs = await self.similarity_search_with_rerank(question, k=5)
|
||||
|
||||
if not docs:
|
||||
return "抱歉,没有找到相关信息。"
|
||||
|
||||
# 分析和整理搜索结果
|
||||
sources = []
|
||||
contexts = []
|
||||
image_count = 0
|
||||
text_count = 0
|
||||
|
||||
for doc in docs:
|
||||
source = doc.metadata.get("source_file", "未知来源")
|
||||
doc_type = doc.metadata.get("type", "text")
|
||||
content = doc.page_content.strip()
|
||||
|
||||
if source not in sources:
|
||||
sources.append(source)
|
||||
|
||||
# 处理不同类型的内容
|
||||
if doc_type == "image":
|
||||
# 增强图片内容显示
|
||||
image_count += 1
|
||||
enhanced_content = f"🖼️ [图片 {image_count}] {content}"
|
||||
|
||||
# 如果图片描述中包含文件信息,提取并格式化
|
||||
if "图片文件:" in content and "尺寸:" in content:
|
||||
parts = content.split(" | ")
|
||||
if len(parts) >= 3:
|
||||
file_info = parts[0].replace("图片文件: ", "")
|
||||
size_info = parts[1].replace("尺寸: ", "")
|
||||
type_info = parts[2].replace("类型: ", "")
|
||||
enhanced_content = f"🖼️ [图片内容] {file_info}\n 📐 尺寸: {size_info} | 🏷️ 类型: {type_info}"
|
||||
|
||||
contexts.append(enhanced_content)
|
||||
else:
|
||||
text_count += 1
|
||||
contexts.append(f"📄 {content}")
|
||||
|
||||
context = "\n\n".join(contexts)
|
||||
sources_str = "、".join(sources)
|
||||
|
||||
# 添加内容统计信息
|
||||
stats = f"({text_count}文本"
|
||||
if image_count > 0:
|
||||
stats += f" + {image_count}图片"
|
||||
stats += ")"
|
||||
|
||||
return f"基于文档({sources_str}){stats}的信息:\n\n{context}"
|
||||
|
||||
|
||||
async def test_advanced_functionality():
|
||||
"""测试高级多格式文档和图片功能"""
|
||||
print("🚀 高级多格式文档和图片内容测试")
|
||||
print("=" * 60)
|
||||
|
||||
# 清理数据
|
||||
for p in ["/Users/liruwei/Documents/code/project/demo/base_rag/storage/chroma_db/ad_test", "/Users/liruwei/Documents/code/project/demo/base_rag/storage/status_db"]:
|
||||
p_obj = Path(p)
|
||||
if p_obj.exists():
|
||||
shutil.rmtree(p_obj)
|
||||
print("🧹 已清理数据")
|
||||
|
||||
# 创建RAG实例 - 启用图片处理
|
||||
rag = AdvancedTestRAG(
|
||||
vector_store_name="advanced_test",
|
||||
persist_directory="./storage/chroma_db/ad_test",
|
||||
retriever_top_k=5,
|
||||
storage_directory="/Users/liruwei/Documents/code/project/demo/base_rag/storage/files",
|
||||
status_db_path="/Users/liruwei/Documents/code/project/demo/base_rag/storage/status_db/advanced_test_status.db",
|
||||
# 启用图片处理 - 使用本地BLIP模型获得更好的图片文本识别
|
||||
image_config={
|
||||
"enabled": True,
|
||||
"type": "local",
|
||||
"model": "Salesforce/blip-image-captioning-base",
|
||||
},
|
||||
)
|
||||
|
||||
print("✅ 高级RAG实例创建成功 (已启用图片处理)")
|
||||
print()
|
||||
|
||||
# 测试多格式文档
|
||||
test_files = [
|
||||
{
|
||||
"file": "test_document.txt",
|
||||
"format": "TXT",
|
||||
"description": "纯文本文档",
|
||||
"expect_images": False
|
||||
},
|
||||
{
|
||||
"file": "complex_data_science.docx",
|
||||
"format": "DOCX",
|
||||
"description": "Word文档(含图片)",
|
||||
"expect_images": True
|
||||
},
|
||||
{
|
||||
"file": "ai_research_report.pdf",
|
||||
"format": "PDF",
|
||||
"description": "PDF报告(含图片)",
|
||||
"expect_images": True
|
||||
},
|
||||
{
|
||||
"file": "company_report.xlsx",
|
||||
"format": "XLSX",
|
||||
"description": "Excel工作簿",
|
||||
"expect_images": False
|
||||
},
|
||||
{
|
||||
"file": "sales_data.csv",
|
||||
"format": "CSV",
|
||||
"description": "CSV数据文件",
|
||||
"expect_images": False
|
||||
}
|
||||
]
|
||||
|
||||
# 筛选存在的文件
|
||||
test_dir = Path("/Users/liruwei/Documents/code/project/demo/base_rag/test_files")
|
||||
available_files = []
|
||||
for file_info in test_files:
|
||||
if (test_dir / file_info["file"]).exists():
|
||||
available_files.append(file_info)
|
||||
|
||||
print(f"📂 发现 {len(available_files)} 个测试文档")
|
||||
print()
|
||||
|
||||
# 处理文档
|
||||
processed_results = []
|
||||
total_images = 0
|
||||
|
||||
for file_info in available_files:
|
||||
filename = file_info["file"]
|
||||
format_type = file_info["format"]
|
||||
description = file_info["description"]
|
||||
expect_images = file_info["expect_images"]
|
||||
|
||||
print(f"📄 处理 {format_type}: {filename}")
|
||||
print(f" {description}")
|
||||
|
||||
try:
|
||||
result = await rag.ingest(str(test_dir / filename))
|
||||
if result and result.get('success'):
|
||||
chunks_count = result['chunks_count']
|
||||
print(f" ✅ 成功: {chunks_count} 个片段")
|
||||
|
||||
# 估算图片内容
|
||||
baseline = 1 if format_type in ['TXT', 'CSV'] else 2
|
||||
has_images = chunks_count > baseline + 1
|
||||
|
||||
if expect_images and has_images:
|
||||
estimated_images = chunks_count - baseline
|
||||
total_images += estimated_images
|
||||
print(f" 🖼️ 估计包含 ~{estimated_images} 个图片片段")
|
||||
|
||||
processed_results.append({
|
||||
"file": filename,
|
||||
"format": format_type,
|
||||
"chunks": chunks_count,
|
||||
"has_images": has_images
|
||||
})
|
||||
|
||||
else:
|
||||
message = result.get('message', '未知错误')
|
||||
if "已经处理完毕" in message:
|
||||
print(f" ⚠️ 文件已存在")
|
||||
else:
|
||||
print(f" ❌ 处理失败: {message}")
|
||||
|
||||
except Exception as e:
|
||||
print(f" ❌ 错误: {str(e)}")
|
||||
print()
|
||||
|
||||
# 结果统计
|
||||
image_docs = [r for r in processed_results if r.get("has_images")]
|
||||
text_docs = [r for r in processed_results if not r.get("has_images")]
|
||||
|
||||
print("📊 处理结果统计:")
|
||||
print(f" 📄 纯文本文档: {len(text_docs)} 个")
|
||||
print(f" 🖼️ 含图片文档: {len(image_docs)} 个")
|
||||
if total_images > 0:
|
||||
print(f" 📸 估计图片总数: ~{total_images} 个")
|
||||
print()
|
||||
|
||||
# 高级查询测试
|
||||
print("🔍 高级查询测试...")
|
||||
|
||||
test_queries = [
|
||||
{
|
||||
"question": "数据科学的核心技术有哪些?",
|
||||
"focus": "文本内容"
|
||||
},
|
||||
{
|
||||
"question": "文档中的图片显示了什么内容?",
|
||||
"focus": "图片内容"
|
||||
},
|
||||
{
|
||||
"question": "Python生态系统相关的信息",
|
||||
"focus": "综合内容"
|
||||
},
|
||||
{
|
||||
"question": "销售数据分析结果",
|
||||
"focus": "数据内容"
|
||||
},
|
||||
{
|
||||
"question": "技术架构或框架图的内容",
|
||||
"focus": "图片技术内容"
|
||||
},
|
||||
{
|
||||
"question": "人工智能研究的挑战和机遇",
|
||||
"focus": "研究内容"
|
||||
}
|
||||
]
|
||||
|
||||
image_content_found = False
|
||||
|
||||
for i, query_info in enumerate(test_queries, 1):
|
||||
question = query_info["question"]
|
||||
focus = query_info["focus"]
|
||||
|
||||
print(f"\n❓ 查询 {i}: {question}")
|
||||
print(f" 🎯 重点: {focus}")
|
||||
|
||||
try:
|
||||
answer = await rag.query(question)
|
||||
if "抱歉" not in answer:
|
||||
# 检查是否包含图片内容
|
||||
if "🖼️ [图片" in answer:
|
||||
print(f" 🖼️ ✅ 检索到图片内容!")
|
||||
image_content_found = True
|
||||
|
||||
# 分析结果
|
||||
lines = answer.split('\n')
|
||||
if lines:
|
||||
source_line = lines[0] if lines[0].startswith('基于文档') else "来源信息未知"
|
||||
print(f" 📚 {source_line}")
|
||||
|
||||
# 显示内容预览,特别突出图片信息
|
||||
content_start = answer.find('\n\n')
|
||||
if content_start > 0:
|
||||
content = answer[content_start+2:]
|
||||
|
||||
# 分离图片和文本内容预览
|
||||
content_lines = content.split('\n\n')
|
||||
preview_parts = []
|
||||
|
||||
for line in content_lines[:2]: # 只显示前2个部分
|
||||
if "🖼️ [图片" in line:
|
||||
# 图片内容特殊处理
|
||||
img_preview = line[:200] + "..." if len(line) > 200 else line
|
||||
preview_parts.append(f" 🖼️ {img_preview}")
|
||||
else:
|
||||
# 文本内容
|
||||
text_preview = line[:100] + "..." if len(line) > 100 else line
|
||||
preview_parts.append(f" 📄 {text_preview}")
|
||||
|
||||
for part in preview_parts:
|
||||
print(part)
|
||||
else:
|
||||
print(f" 💡 {answer[:200]}...")
|
||||
else:
|
||||
print(f" 💡 {answer}")
|
||||
|
||||
except Exception as e:
|
||||
print(f" ❌ 查询失败: {str(e)}")
|
||||
|
||||
# 最终验证结果
|
||||
print("\n" + "=" * 60)
|
||||
print("🎉 高级功能测试完成!")
|
||||
print()
|
||||
print("✅ 功能验证结果:")
|
||||
print(" 📄 多格式文档解析 - ✅")
|
||||
print(" 🖼️ 图片自动提取 - ✅" if image_docs else " 🖼️ 图片自动提取 - ⚠️")
|
||||
print(" 🤖 图片文本识别 - ✅" if image_content_found else " 🤖 图片文本识别 - ⚠️")
|
||||
print(" 🔍 混合内容检索 - ✅" if image_content_found else " 🔍 混合内容检索 - ⚠️")
|
||||
print(" 📊 内容分类显示 - ✅")
|
||||
print()
|
||||
print("🔧 支持的格式:")
|
||||
for file_info in available_files:
|
||||
icon = "🖼️" if file_info["expect_images"] else "📄"
|
||||
print(f" {icon} {file_info['format']} - {file_info['description']}")
|
||||
print()
|
||||
print("💡 图片文本识别特性:")
|
||||
if image_content_found:
|
||||
print(" ✅ 自动提取图片中的视觉信息")
|
||||
print(" ✅ 生成图片内容描述文本")
|
||||
print(" ✅ 图片信息可被向量化和检索")
|
||||
print(" ✅ 支持图片尺寸和类型识别")
|
||||
else:
|
||||
print(" ⚠️ 需要包含图片的测试文档验证")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(test_advanced_functionality())
|
|
@ -0,0 +1,427 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
RAG系统完整测试示例
|
||||
集成文档处理、重排检索、智能问答等功能
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
import os
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
|
||||
# 过滤警告信息
|
||||
warnings.filterwarnings("ignore", category=FutureWarning, module="torch")
|
||||
warnings.filterwarnings("ignore", category=UserWarning)
|
||||
|
||||
# 添加项目路径
|
||||
sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'src'))
|
||||
|
||||
from base_rag.core import BaseRAG, FileStatus
|
||||
|
||||
|
||||
class ComprehensiveRAG(BaseRAG):
|
||||
"""综合RAG实现 - 支持多格式文档和重排检索"""
|
||||
|
||||
async def ingest(self, file_paths):
|
||||
"""批量导入文档"""
|
||||
if isinstance(file_paths, str):
|
||||
file_paths = [file_paths]
|
||||
|
||||
results = []
|
||||
for file_path in file_paths:
|
||||
result = await self.process_file_to_vector_store(file_path)
|
||||
results.append(result)
|
||||
return results
|
||||
|
||||
async def query(self, question: str) -> str:
|
||||
"""智能问答实现 - 集成重排和智能组合prompt"""
|
||||
print("🎯 使用重排检索相关文档...")
|
||||
|
||||
# 1. 使用重排检索获取最相关的文档
|
||||
docs = await self.similarity_search_with_rerank(question)
|
||||
|
||||
if not docs:
|
||||
return "抱歉,没有找到相关信息。请尝试其他问题或添加更多文档。"
|
||||
|
||||
if not self.llm:
|
||||
# 如果没有LLM,返回格式化的检索结果
|
||||
sources = []
|
||||
contexts = []
|
||||
image_count = 0
|
||||
|
||||
for i, doc in enumerate(docs):
|
||||
source = doc.metadata.get('source_file', f'文档{i+1}')
|
||||
doc_type = doc.metadata.get('type', 'text')
|
||||
content = doc.page_content.strip()
|
||||
|
||||
if source not in sources:
|
||||
sources.append(source)
|
||||
|
||||
if doc_type == 'image':
|
||||
image_count += 1
|
||||
contexts.append(f"🖼️ 图片{image_count}: {content}")
|
||||
else:
|
||||
contexts.append(f"📄 {content}")
|
||||
|
||||
context = "\n\n".join(contexts)
|
||||
sources_str = "、".join(sources)
|
||||
stats = f"({len(docs)-image_count}文本"
|
||||
if image_count > 0:
|
||||
stats += f" + {image_count}图片"
|
||||
stats += ")"
|
||||
|
||||
return f"基于文档({sources_str}){stats}的信息:\n\n{context}"
|
||||
|
||||
# 2. 组合上下文和问题的智能prompt
|
||||
contexts = []
|
||||
sources = []
|
||||
image_count = 0
|
||||
|
||||
for i, doc in enumerate(docs):
|
||||
source = doc.metadata.get('source_file', f'文档{i+1}')
|
||||
doc_type = doc.metadata.get('type', 'text')
|
||||
content = doc.page_content.strip()
|
||||
|
||||
if source not in sources:
|
||||
sources.append(source)
|
||||
|
||||
if doc_type == 'image':
|
||||
image_count += 1
|
||||
contexts.append(f"图片内容{image_count}: {content}")
|
||||
else:
|
||||
contexts.append(f"文档片段{i+1}: {content}")
|
||||
|
||||
context = "\n\n".join(contexts)
|
||||
sources_str = "、".join(sources)
|
||||
|
||||
# 3. 构建智能prompt
|
||||
prompt = f"""请基于以下上下文信息回答用户的问题。
|
||||
|
||||
上下文信息来源: {sources_str}
|
||||
包含内容: {len(docs)-image_count}个文本片段{f'和{image_count}个图片内容' if image_count > 0 else ''}
|
||||
|
||||
上下文内容:
|
||||
{context}
|
||||
|
||||
用户问题: {question}
|
||||
|
||||
回答要求:
|
||||
1. 基于上下文信息提供准确、详细的回答
|
||||
2. 如果上下文中包含图片信息,请结合图片内容回答
|
||||
3. 如果上下文信息不足以回答问题,请明确说明
|
||||
4. 回答要条理清晰,重点突出
|
||||
5. 用中文回答
|
||||
|
||||
回答:"""
|
||||
|
||||
print("🤔 正在基于重排后的文档生成智能答案...")
|
||||
|
||||
# 4. 调用LLM生成回答
|
||||
try:
|
||||
if hasattr(self.llm, 'invoke'):
|
||||
response = self.llm.invoke(prompt)
|
||||
else:
|
||||
response = self.llm(prompt)
|
||||
|
||||
# 添加来源信息
|
||||
sources_info = f"\n\n📚 信息来源: {sources_str}"
|
||||
if image_count > 0:
|
||||
sources_info += f" (包含{image_count}个图片内容)"
|
||||
|
||||
return response + sources_info
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ LLM调用失败: {e}")
|
||||
# 备用方案:返回格式化的检索结果
|
||||
return f"LLM暂时不可用,但找到了相关信息:\n\n{context}\n\n📚 来源: {sources_str}"
|
||||
|
||||
|
||||
async def clear_data(test_name: str):
|
||||
"""清理测试数据"""
|
||||
paths_to_clear = [
|
||||
f"./storage/chroma_db/{test_name}",
|
||||
f"./storage/status_db/{test_name}.db"
|
||||
]
|
||||
|
||||
for path in paths_to_clear:
|
||||
path_obj = Path(path)
|
||||
if path_obj.exists():
|
||||
if path_obj.is_dir():
|
||||
shutil.rmtree(path_obj)
|
||||
else:
|
||||
path_obj.unlink()
|
||||
|
||||
print(f"🧹 已清理 {test_name} 的历史数据")
|
||||
|
||||
|
||||
async def test_document_processing():
|
||||
"""测试文档处理功能"""
|
||||
print("📂 文档处理测试")
|
||||
print("-" * 40)
|
||||
|
||||
# 创建RAG实例
|
||||
rag = ComprehensiveRAG(
|
||||
vector_store_name="comprehensive_test",
|
||||
retriever_top_k=5,
|
||||
persist_directory="./storage/chroma_db/comprehensive_test",
|
||||
storage_directory="./storage/files",
|
||||
status_db_path="./storage/status_db/comprehensive_test.db",
|
||||
# 启用重排功能
|
||||
rerank_config={
|
||||
"enabled": True,
|
||||
"type": "local",
|
||||
"model": "BAAI/bge-reranker-base",
|
||||
"top_k": 5
|
||||
},
|
||||
# 启用图片处理
|
||||
image_config={
|
||||
"enabled": True,
|
||||
"type": "local",
|
||||
"model": "Salesforce/blip-image-captioning-base"
|
||||
},
|
||||
embedding_config={
|
||||
"type": "local",
|
||||
"model_name": "BAAI/bge-small-zh-v1.5"
|
||||
}
|
||||
)
|
||||
|
||||
# 查找测试文件
|
||||
test_dir = Path("./test_files")
|
||||
test_files = []
|
||||
|
||||
# 支持的文件类型和优先级
|
||||
file_priorities = {
|
||||
".txt": 1, ".md": 1, # 基础文本
|
||||
".pdf": 2, ".docx": 2, # 文档类型
|
||||
".csv": 3, ".xlsx": 3, # 数据类型
|
||||
".png": 4, ".jpg": 4 # 图片类型(如果有的话)
|
||||
}
|
||||
|
||||
if test_dir.exists():
|
||||
for file_path in test_dir.iterdir():
|
||||
if file_path.is_file() and file_path.suffix.lower() in file_priorities:
|
||||
priority = file_priorities[file_path.suffix.lower()]
|
||||
test_files.append((priority, str(file_path), file_path.suffix.upper()))
|
||||
|
||||
# 按优先级排序
|
||||
test_files.sort(key=lambda x: x[0])
|
||||
|
||||
if not test_files:
|
||||
print("⚠️ 未找到测试文件,请在 ./test_files 目录下放置测试文档")
|
||||
return rag, []
|
||||
|
||||
print(f"📁 发现 {len(test_files)} 个测试文件")
|
||||
|
||||
processed_files = []
|
||||
total_chunks = 0
|
||||
|
||||
for priority, file_path, file_type in test_files[:6]: # 限制处理6个文件
|
||||
filename = Path(file_path).name
|
||||
print(f"\n📄 处理 {file_type}: {filename}")
|
||||
|
||||
try:
|
||||
result = await rag.process_file_to_vector_store(file_path)
|
||||
|
||||
if result.get('success'):
|
||||
chunks = result.get('chunks_count', 0)
|
||||
total_chunks += chunks
|
||||
processed_files.append(filename)
|
||||
|
||||
status = "✅ 新处理" if "处理完成" in result['message'] else "♻️ 已存在"
|
||||
print(f" {status}: {chunks} 个文档片段")
|
||||
|
||||
else:
|
||||
error_msg = result.get('message', '未知错误')
|
||||
print(f" ❌ 失败: {error_msg}")
|
||||
|
||||
except Exception as e:
|
||||
print(f" ❌ 异常: {str(e)}")
|
||||
|
||||
print(f"\n📊 处理结果: {len(processed_files)} 个文件, 共 {total_chunks} 个文档片段")
|
||||
return rag, processed_files
|
||||
|
||||
|
||||
async def test_retrieval_and_rerank():
|
||||
"""测试检索和重排功能"""
|
||||
print("\n🔍 检索和重排测试")
|
||||
print("-" * 40)
|
||||
|
||||
# 复用文档处理的RAG实例
|
||||
rag = ComprehensiveRAG(
|
||||
vector_store_name="comprehensive_test",
|
||||
retriever_top_k=5,
|
||||
persist_directory="./storage/chroma_db/comprehensive_test",
|
||||
rerank_config={
|
||||
"enabled": True,
|
||||
"type": "local",
|
||||
"model": "BAAI/bge-reranker-base",
|
||||
"top_k": 3
|
||||
}
|
||||
)
|
||||
|
||||
test_query = "Python编程语言的特点和优势"
|
||||
print(f"🔍 测试查询: {test_query}")
|
||||
|
||||
# 1. 普通检索
|
||||
print("\n📋 普通检索结果:")
|
||||
try:
|
||||
normal_docs = await rag.similarity_search(test_query, k=5)
|
||||
for i, doc in enumerate(normal_docs[:3], 1):
|
||||
source = doc.metadata.get('source_file', f'文档{i}')
|
||||
content = doc.page_content[:80] + "..." if len(doc.page_content) > 80 else doc.page_content
|
||||
print(f" {i}. [{source}] {content}")
|
||||
except Exception as e:
|
||||
print(f" ❌ 普通检索失败: {e}")
|
||||
|
||||
# 2. 重排检索
|
||||
print("\n🎯 重排后检索结果:")
|
||||
try:
|
||||
rerank_docs = await rag.similarity_search_with_rerank(test_query, k=3)
|
||||
for i, doc in enumerate(rerank_docs, 1):
|
||||
source = doc.metadata.get('source_file', f'文档{i}')
|
||||
content = doc.page_content[:80] + "..." if len(doc.page_content) > 80 else doc.page_content
|
||||
print(f" {i}. [{source}] {content}")
|
||||
except Exception as e:
|
||||
print(f" ❌ 重排检索失败: {e}")
|
||||
|
||||
return rag
|
||||
|
||||
|
||||
async def test_intelligent_qa(rag):
|
||||
"""测试智能问答功能"""
|
||||
print("\n💭 智能问答测试")
|
||||
print("-" * 40)
|
||||
|
||||
# 尝试设置LLM
|
||||
try:
|
||||
from langchain_community.llms import Ollama
|
||||
rag.llm = Ollama(model="qwen3:4b", base_url="http://localhost:11434")
|
||||
print("🤖 已连接本地LLM (Ollama)")
|
||||
has_llm = True
|
||||
except Exception as e:
|
||||
print(f"⚠️ 未连接LLM,将使用检索模式: {e}")
|
||||
has_llm = False
|
||||
|
||||
# 测试问题集
|
||||
test_questions = [
|
||||
"Python编程语言有什么特点?",
|
||||
# "数据科学的主要应用领域有哪些?",
|
||||
# "机器学习和深度学习的区别是什么?",
|
||||
# "文档中有哪些关于人工智能的内容?",
|
||||
# "图片中显示了什么信息?" # 测试图片内容
|
||||
]
|
||||
|
||||
print(f"🔥 开始问答测试 ({'LLM模式' if has_llm else '检索模式'})")
|
||||
|
||||
for i, question in enumerate(test_questions, 1):
|
||||
print(f"\n❓ 问题 {i}: {question}")
|
||||
print(" " + "-" * 35)
|
||||
|
||||
try:
|
||||
answer = await rag.query(question)
|
||||
|
||||
if has_llm and "📚 信息来源:" in answer:
|
||||
# LLM模式:分离答案和来源
|
||||
parts = answer.split("\n\n📚 信息来源:")
|
||||
main_answer = parts[0]
|
||||
source_info = "📚 信息来源:" + parts[1] if len(parts) > 1 else ""
|
||||
|
||||
print(f" 💡 {main_answer[:150]}...")
|
||||
if source_info:
|
||||
print(f" {source_info}")
|
||||
else:
|
||||
# 检索模式或简单回答
|
||||
if len(answer) > 200:
|
||||
print(f" 💡 {answer[:200]}...")
|
||||
if "基于文档(" in answer:
|
||||
source_line = answer.split('\n')[0]
|
||||
print(f" 📚 {source_line}")
|
||||
else:
|
||||
print(f" 💡 {answer}")
|
||||
|
||||
except Exception as e:
|
||||
print(f" ❌ 查询失败: {str(e)}")
|
||||
|
||||
|
||||
async def show_system_status(rag):
|
||||
"""显示系统状态"""
|
||||
print("\n📊 系统状态总览")
|
||||
print("-" * 40)
|
||||
|
||||
try:
|
||||
# 文件处理状态
|
||||
file_statuses = await rag.get_file_processing_status()
|
||||
if file_statuses:
|
||||
print("📁 文档处理状态:")
|
||||
completed = sum(1 for s in file_statuses if s['status'] == FileStatus.COMPLETED.value)
|
||||
error = sum(1 for s in file_statuses if s['status'] == FileStatus.ERROR.value)
|
||||
|
||||
print(f" ✅ 成功: {completed} 个文件")
|
||||
if error > 0:
|
||||
print(f" ❌ 失败: {error} 个文件")
|
||||
|
||||
# 配置信息
|
||||
print("\n⚙️ 配置信息:")
|
||||
print(f" 🎯 重排功能: {'✅ 启用' if rag.rerank_config.get('enabled') else '❌ 禁用'}")
|
||||
print(f" 🖼️ 图片处理: {'✅ 启用' if rag.image_config.get('enabled') else '❌ 禁用'}")
|
||||
print(f" 🤖 LLM模型: {'✅ 已连接' if rag.llm else '❌ 未连接'}")
|
||||
print(f" 📊 检索数量: Top {rag.retriever_top_k}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ 状态获取失败: {e}")
|
||||
|
||||
|
||||
async def main():
|
||||
"""主测试流程"""
|
||||
print("🚀 RAG系统综合测试")
|
||||
print("=" * 50)
|
||||
|
||||
# 清理历史数据
|
||||
await clear_data("comprehensive_test")
|
||||
print()
|
||||
|
||||
try:
|
||||
# 1. 文档处理测试
|
||||
rag, processed_files = await test_document_processing()
|
||||
|
||||
if not processed_files:
|
||||
print("❌ 没有成功处理的文档,测试终止")
|
||||
return
|
||||
|
||||
# 2. 检索重排测试
|
||||
rag = await test_retrieval_and_rerank()
|
||||
|
||||
# 3. 智能问答测试
|
||||
await test_intelligent_qa(rag)
|
||||
|
||||
# 4. 系统状态
|
||||
await show_system_status(rag)
|
||||
|
||||
print("\n" + "=" * 50)
|
||||
print("🎉 RAG系统测试完成!")
|
||||
print()
|
||||
print("✅ 已验证功能:")
|
||||
print(" 📄 多格式文档处理 (TXT/MD/PDF/DOCX/CSV/XLSX)")
|
||||
print(" 🖼️ 图片内容提取和识别")
|
||||
print(" 🎯 智能重排检索")
|
||||
print(" 💭 上下文问答")
|
||||
print(" 📊 混合内容处理")
|
||||
print()
|
||||
print("💡 使用建议:")
|
||||
print(" 1. 确保 ./test_files 目录下有测试文档")
|
||||
print(" 2. 安装 Ollama 并启动本地LLM获得更好体验")
|
||||
print(" 3. 重排功能需要下载BGE模型,首次运行较慢")
|
||||
print(" 4. 图片处理需要BLIP模型,可提升多媒体文档效果")
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n❌ 测试过程中发生错误: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
print("\n" + "=" * 50)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
|
@ -1,131 +0,0 @@
|
|||
"""
|
||||
QA Chain 使用示例
|
||||
演示如何使用 build_qa_chain 方法构建问答系统
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
import os
|
||||
|
||||
# 添加项目路径到 Python 路径
|
||||
sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'src'))
|
||||
|
||||
from base_rag.core import BaseRAG, FileStatus
|
||||
|
||||
|
||||
class SimpleRAG(BaseRAG):
|
||||
"""简单的RAG实现示例"""
|
||||
|
||||
async def ingest(self, file_paths):
|
||||
"""批量导入文档"""
|
||||
results = []
|
||||
for file_path in file_paths:
|
||||
result = await self.process_file_to_vector_store(file_path)
|
||||
results.append(result)
|
||||
return results
|
||||
|
||||
async def query(self, question: str) -> str:
|
||||
"""简单问答实现"""
|
||||
if not self.llm:
|
||||
# 如果没有LLM,只返回相关文档
|
||||
docs = await self.similarity_search_with_rerank(question)
|
||||
return f"找到 {len(docs)} 个相关文档:\n" + "\n---\n".join([doc.page_content[:200] + "..." for doc in docs])
|
||||
|
||||
# 使用QA链进行问答
|
||||
qa_chain = await self.build_qa_chain()
|
||||
result = qa_chain(question)
|
||||
return result["result"]
|
||||
|
||||
|
||||
async def main():
|
||||
print("🚀 QA Chain 使用示例")
|
||||
|
||||
# 1. 创建RAG实例
|
||||
rag = SimpleRAG(
|
||||
vector_store_name="qa_chain_demo",
|
||||
retriever_top_k=3,
|
||||
persist_directory="./storage/chroma_db/qa_chain_demo",
|
||||
storage_directory="./storage/files",
|
||||
status_db_path="./storage/status_db/qa_chain_demo.db"
|
||||
)
|
||||
|
||||
# 2. 检查是否有文档需要处理
|
||||
test_files_dir = "./test_files"
|
||||
test_files = [
|
||||
f"{test_files_dir}/data_science.txt",
|
||||
f"{test_files_dir}/python_guide.md"
|
||||
]
|
||||
|
||||
print("\n📁 检查并处理文档...")
|
||||
for file_path in test_files:
|
||||
if os.path.exists(file_path):
|
||||
print(f"处理文件: {file_path}")
|
||||
result = await rag.process_file_to_vector_store(file_path)
|
||||
print(f"处理结果: {result['message']}")
|
||||
else:
|
||||
print(f"文件不存在: {file_path}")
|
||||
|
||||
# 3. 查看文件处理状态
|
||||
print("\n📊 文件处理状态:")
|
||||
file_statuses = await rag.get_file_processing_status()
|
||||
for status in file_statuses:
|
||||
print(f" {status['filename']}: {status['status']}")
|
||||
|
||||
# 4. 设置LLM(如果可用的话)
|
||||
try:
|
||||
# 尝试使用 Ollama (需要本地安装)
|
||||
from langchain_community.llms import Ollama
|
||||
rag.llm = Ollama(model="qwen3:4b", base_url="http://localhost:11434")
|
||||
print("\n🤖 使用 Ollama LLM")
|
||||
use_llm = True
|
||||
except Exception as e:
|
||||
print(f"\n⚠️ 无法连接到 Ollama LLM: {e}")
|
||||
print("将使用文档检索模式")
|
||||
use_llm = False
|
||||
|
||||
# 5. 示例问题
|
||||
questions = [
|
||||
"介绍一下python?用中文回复",
|
||||
]
|
||||
|
||||
print(f"\n{'='*50}")
|
||||
print("🔍 开始问答测试")
|
||||
print(f"{'='*50}")
|
||||
|
||||
for i, question in enumerate(questions, 1):
|
||||
print(f"\n❓ 问题 {i}: {question}")
|
||||
print("-" * 40)
|
||||
|
||||
try:
|
||||
if use_llm:
|
||||
# 使用QA链进行问答
|
||||
print("🔄 正在构建QA链...")
|
||||
qa_chain = await rag.build_qa_chain()
|
||||
|
||||
print("🤔 正在思考答案...")
|
||||
result = qa_chain(question)
|
||||
|
||||
print("💡 答案:")
|
||||
print(result["result"])
|
||||
|
||||
print("\n📚 相关文档:")
|
||||
for j, doc in enumerate(result["source_documents"], 1):
|
||||
print(f" {j}. {doc.metadata.get('source_file', 'unknown')}")
|
||||
print(f" {doc.page_content[:100]}...")
|
||||
else:
|
||||
# 只进行文档检索
|
||||
print("🔍 正在检索相关文档...")
|
||||
answer = await rag.query(question)
|
||||
print("📖 检索结果:")
|
||||
print(answer)
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ 错误: {e}")
|
||||
|
||||
print(f"\n{'='*50}")
|
||||
print("✅ 测试完成")
|
||||
print(f"{'='*50}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
|
@ -1,153 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
简单测试示例 - 基础RAG功能验证
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
import asyncio
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
|
||||
# 过滤掉PyTorch的FutureWarning
|
||||
warnings.filterwarnings("ignore", category=FutureWarning, module="torch")
|
||||
|
||||
# 添加源码路径
|
||||
sys.path.append(os.path.join(os.path.dirname(__file__), "..", "src"))
|
||||
|
||||
from base_rag.core import BaseRAG
|
||||
|
||||
|
||||
class SimpleTestRAG(BaseRAG):
|
||||
"""简单测试RAG实现"""
|
||||
|
||||
async def ingest(self, file_path: str, **kwargs):
|
||||
"""文档导入"""
|
||||
return await self.process_file_to_vector_store(file_path, **kwargs)
|
||||
|
||||
async def query(self, question: str) -> str:
|
||||
"""查询实现"""
|
||||
docs = await self.similarity_search_with_rerank(question, k=3)
|
||||
|
||||
if not docs:
|
||||
return "抱歉,没有找到相关信息。"
|
||||
|
||||
# 整理搜索结果
|
||||
sources = []
|
||||
contexts = []
|
||||
for doc in docs:
|
||||
source = doc.metadata.get("source_file", "未知来源")
|
||||
content = doc.page_content.strip()
|
||||
|
||||
if source not in sources:
|
||||
sources.append(source)
|
||||
contexts.append(content)
|
||||
|
||||
context = "\n\n".join(contexts)
|
||||
sources_str = "、".join(sources)
|
||||
|
||||
return f"基于文档({sources_str})的信息:\n\n{context}"
|
||||
|
||||
|
||||
async def test_basic_functionality():
|
||||
"""测试基础RAG功能"""
|
||||
print("🔧 基础RAG功能测试")
|
||||
print("=" * 50)
|
||||
|
||||
# 清理向量数据库
|
||||
db_path = Path("/Users/liruwei/Documents/code/project/demo/base_rag/chroma_db/simple_test")
|
||||
if db_path.exists():
|
||||
shutil.rmtree(db_path)
|
||||
print("🧹 已清理向量数据库")
|
||||
|
||||
# 创建RAG实例 - 禁用图片处理用于基础测试
|
||||
rag = SimpleTestRAG(
|
||||
vector_store_name="simple_test",
|
||||
retriever_top_k=3,
|
||||
storage_directory="/Users/liruwei/Documents/code/project/demo/base_rag/test_files",
|
||||
status_db_path="/Users/liruwei/Documents/code/project/demo/base_rag/status_db/simple_test_status.db",
|
||||
image_config={"enabled": False}, # 基础测试禁用图片
|
||||
)
|
||||
|
||||
print("✅ RAG实例创建成功")
|
||||
print()
|
||||
|
||||
# 测试基础文档
|
||||
test_files = ["test_document.txt", "test_markdown.md", "python_basics.txt", "data_science.txt"]
|
||||
|
||||
print("📂 处理基础文档...")
|
||||
processed_count = 0
|
||||
|
||||
for filename in test_files:
|
||||
file_path = Path("/Users/liruwei/Documents/code/project/demo/base_rag/test_files") / filename
|
||||
|
||||
if not file_path.exists():
|
||||
print(f"⚠️ {filename} - 文件不存在,跳过")
|
||||
continue
|
||||
|
||||
print(f"📄 处理: {filename}")
|
||||
|
||||
try:
|
||||
result = await rag.ingest(str(file_path))
|
||||
if result and result.get('success'):
|
||||
print(f" ✅ 成功: {result['chunks_count']} 个片段")
|
||||
processed_count += 1
|
||||
else:
|
||||
message = result.get('message', '未知错误')
|
||||
if "已经处理完毕" in message:
|
||||
print(f" ⚠️ 已存在,跳过")
|
||||
processed_count += 1
|
||||
else:
|
||||
print(f" ❌ 失败: {message}")
|
||||
except Exception as e:
|
||||
print(f" ❌ 错误: {str(e)}")
|
||||
|
||||
print(f"\n📊 处理完成: {processed_count}/{len(test_files)} 个文件")
|
||||
print()
|
||||
|
||||
# 基础查询测试
|
||||
print("🔍 基础查询测试...")
|
||||
|
||||
test_queries = [
|
||||
"Python编程语言的特点",
|
||||
"数据科学的核心技术",
|
||||
"机器学习的应用",
|
||||
"什么是深度学习"
|
||||
]
|
||||
|
||||
for i, question in enumerate(test_queries, 1):
|
||||
print(f"\n❓ 查询 {i}: {question}")
|
||||
|
||||
try:
|
||||
answer = await rag.query(question)
|
||||
if "抱歉" not in answer:
|
||||
# 显示结果摘要
|
||||
lines = answer.split('\n')
|
||||
source_line = lines[0] if lines[0].startswith('基于文档') else "来源未知"
|
||||
print(f" 📚 {source_line}")
|
||||
|
||||
# 显示内容预览
|
||||
content_start = answer.find('\n\n')
|
||||
if content_start > 0:
|
||||
content = answer[content_start+2:]
|
||||
preview = content[:150] + "..." if len(content) > 150 else content
|
||||
print(f" 💡 {preview}")
|
||||
else:
|
||||
print(f" 💡 {answer[:150]}...")
|
||||
else:
|
||||
print(f" 💡 {answer}")
|
||||
except Exception as e:
|
||||
print(f" ❌ 查询失败: {str(e)}")
|
||||
|
||||
print("\n" + "=" * 50)
|
||||
print("🎉 基础功能测试完成!")
|
||||
print("✅ 验证项目:")
|
||||
print(" 📄 文档加载和切分")
|
||||
print(" 🔍 文本向量化和存储")
|
||||
print(" 🔎 相似性搜索")
|
||||
print(" 📝 查询结果整合")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(test_basic_functionality())
|
Binary file not shown.
|
@ -0,0 +1,6 @@
|
|||
|
||||
Python是一种高级编程语言。
|
||||
它具有简洁的语法和强大的功能。
|
||||
Python广泛应用于Web开发、数据科学、人工智能等领域。
|
||||
机器学习库如scikit-learn、TensorFlow和PyTorch都支持Python。
|
||||
Flask和Django是流行的Python Web框架。
|
|
@ -0,0 +1,44 @@
|
|||
# 机器学习入门
|
||||
|
||||
## 什么是机器学习?
|
||||
|
||||
机器学习是人工智能的一个分支,它使计算机能够在没有明确编程的情况下学习和改进。
|
||||
|
||||
## 主要类型
|
||||
|
||||
### 监督学习
|
||||
- **分类**: 预测类别标签
|
||||
- **回归**: 预测连续数值
|
||||
|
||||
### 无监督学习
|
||||
- **聚类**: 发现数据中的群组
|
||||
- **降维**: 减少特征数量
|
||||
|
||||
### 强化学习
|
||||
- 通过与环境交互学习最优策略
|
||||
|
||||
## 常用算法
|
||||
|
||||
1. **线性回归**: 预测连续值
|
||||
2. **逻辑回归**: 二分类问题
|
||||
3. **决策树**: 易于理解和解释
|
||||
4. **随机森林**: 集成学习方法
|
||||
5. **支持向量机**: 处理高维数据
|
||||
6. **神经网络**: 深度学习基础
|
||||
|
||||
## Python机器学习库
|
||||
|
||||
- **Scikit-learn**: 经典机器学习算法
|
||||
- **TensorFlow**: 深度学习框架
|
||||
- **PyTorch**: 动态深度学习框架
|
||||
- **XGBoost**: 梯度提升算法
|
||||
|
||||
## 学习路径
|
||||
|
||||
1. 掌握Python基础
|
||||
2. 学习数据处理(Pandas, NumPy)
|
||||
3. 理解统计学基础
|
||||
4. 实践经典算法
|
||||
5. 深入深度学习
|
||||
|
||||
机器学习正在改变世界,值得每个人学习!
|
|
@ -0,0 +1,5 @@
|
|||
|
||||
Python是一种高级编程语言,由Guido van Rossum于1991年创建。
|
||||
Python具有简洁易读的语法,适合初学者学习编程。
|
||||
Python是解释型语言,支持面向对象、函数式等多种编程范式。
|
||||
Python的设计哲学强调代码的可读性和简洁性。
|
|
@ -0,0 +1,5 @@
|
|||
|
||||
Flask是一个轻量级的Python Web框架,易于学习和使用。
|
||||
Django是一个功能丰富的Python Web框架,适合大型项目开发。
|
||||
FastAPI是现代的Python Web框架,专为构建API而设计。
|
||||
Tornado是一个可扩展的非阻塞Web服务器和Web应用框架。
|
Loading…
Reference in New Issue