base_rag/create_test_files.py

322 lines
12 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
"""
创建复杂格式的测试文件 - DOCX、PDF、Excel、CSV
"""
import asyncio
from docx import Document
from docx.shared import Inches
import pandas as pd
import matplotlib.pyplot as plt
from reportlab.lib.pagesizes import letter
from reportlab.pdfgen import canvas
from reportlab.lib.units import inch
import io
import os
async def create_complex_docx():
"""创建包含表格和图片的复杂 DOCX 文件"""
doc = Document()
# 添加标题
doc.add_heading('数据科学技术报告', 0)
# 添加段落
doc.add_paragraph('本报告全面介绍了数据科学领域的核心技术和应用场景。')
# 添加子标题
doc.add_heading('1. 数据科学概述', level=1)
# 添加详细内容
doc.add_paragraph(
'数据科学是一个跨学科领域,结合了统计学、计算机科学和领域专业知识。'
'它使用科学方法、流程、算法和系统从结构化和非结构化数据中提取知识和洞察。'
)
# 添加表格
doc.add_heading('2. 核心技术对比', level=1)
table = doc.add_table(rows=1, cols=4)
table.style = 'Table Grid'
# 表头
hdr_cells = table.rows[0].cells
hdr_cells[0].text = '技术'
hdr_cells[1].text = '应用场景'
hdr_cells[2].text = '优势'
hdr_cells[3].text = '难度等级'
# 数据行
technologies = [
('机器学习', '预测分析、分类', '自动化决策', '中等'),
('深度学习', '图像识别、NLP', '高精度', '困难'),
('数据挖掘', '模式发现', '洞察发现', '简单'),
('大数据分析', '海量数据处理', '可扩展性', '中等'),
('统计分析', '假设检验', '科学严谨', '简单')
]
for tech, scenario, advantage, difficulty in technologies:
row_cells = table.add_row().cells
row_cells[0].text = tech
row_cells[1].text = scenario
row_cells[2].text = advantage
row_cells[3].text = difficulty
# 添加工具和库部分
doc.add_heading('3. 常用工具和库', level=1)
# Python工具
doc.add_heading('Python生态系统', level=2)
python_tools = [
'NumPy - 数值计算基础库',
'Pandas - 数据操作和分析',
'Matplotlib/Seaborn - 数据可视化',
'Scikit-learn - 机器学习',
'TensorFlow/PyTorch - 深度学习',
'Jupyter Notebook - 交互式开发环境'
]
for tool in python_tools:
doc.add_paragraph(tool, style='List Bullet')
# R工具
doc.add_heading('R生态系统', level=2)
r_tools = [
'dplyr - 数据操作',
'ggplot2 - 数据可视化',
'caret - 机器学习',
'shiny - Web应用开发'
]
for tool in r_tools:
doc.add_paragraph(tool, style='List Bullet')
# 添加流程图说明
doc.add_heading('4. 数据科学流程', level=1)
flow_steps = [
'1. 问题定义:明确业务目标和分析需求',
'2. 数据收集:获取相关的内部和外部数据源',
'3. 数据清洗:处理缺失值、异常值和数据质量问题',
'4. 探索性数据分析:理解数据分布和特征关系',
'5. 特征工程:创建和选择有价值的特征',
'6. 模型建立:选择和训练合适的算法',
'7. 模型评估:验证模型性能和泛化能力',
'8. 模型部署:将模型集成到生产环境',
'9. 监控和维护:持续跟踪模型性能'
]
for step in flow_steps:
doc.add_paragraph(step, style='List Number')
# 添加挑战和趋势
doc.add_heading('5. Python在数据科学中的应用', level=1)
doc.add_paragraph(
'Python已成为数据科学领域最受欢迎的编程语言之一。'
'下图展示了Python的生态系统'
)
# 添加图片
try:
doc.add_picture('/Users/liruwei/Documents/code/project/demo/base_rag/test_files/python.png',
width=Inches(4))
doc.add_paragraph('图1: Python生态系统', style='Caption')
except Exception as e:
print(f"警告无法添加图片到DOCX: {e}")
doc.add_paragraph('[此处应显示Python生态系统图片]')
doc.add_heading('6. 行业挑战与未来趋势', level=1)
doc.add_paragraph(
'数据科学领域面临着数据隐私、算法偏见、可解释性等挑战。'
'未来趋势包括自动化机器学习(AutoML)、边缘计算、'
'联邦学习和可解释AI等技术的发展。'
)
# 保存文档
doc.save('/Users/liruwei/Documents/code/project/demo/base_rag/test_files/complex_data_science.docx')
print("已创建复杂的 DOCX 文件: complex_data_science.docx")
async def create_test_csv():
"""创建CSV测试文件"""
# 销售数据
sales_data = {
'日期': ['2024-01-01', '2024-01-02', '2024-01-03', '2024-01-04', '2024-01-05'],
'产品': ['笔记本电脑', '台式机', '平板电脑', '智能手机', '耳机'],
'销售额': [8500, 6200, 3200, 4500, 280],
'数量': [5, 4, 8, 9, 12],
'客户类型': ['企业', '个人', '学生', '个人', '学生'],
'销售员': ['张三', '李四', '王五', '张三', '李四']
}
df = pd.DataFrame(sales_data)
df.to_csv('/Users/liruwei/Documents/code/project/demo/base_rag/test_files/sales_data.csv',
index=False, encoding='utf-8')
print("已创建 CSV 文件: sales_data.csv")
async def create_test_excel():
"""创建Excel测试文件"""
# 创建多个工作表的Excel文件
with pd.ExcelWriter('/Users/liruwei/Documents/code/project/demo/base_rag/test_files/company_report.xlsx',
engine='openpyxl') as writer:
# 销售数据表
sales_data = {
'月份': ['1月', '2月', '3月', '4月', '5月', '6月'],
'销售额(万元)': [120, 135, 158, 142, 167, 189],
'利润率(%)': [15.2, 16.8, 18.3, 16.9, 19.1, 20.5],
'客户数': [856, 923, 1047, 978, 1156, 1289],
'新客户': [45, 67, 124, 55, 178, 133]
}
pd.DataFrame(sales_data).to_excel(writer, sheet_name='销售数据', index=False)
# 员工信息表
employee_data = {
'姓名': ['张三', '李四', '王五', '赵六', '钱七'],
'部门': ['销售部', '技术部', '市场部', '人事部', '财务部'],
'职位': ['销售经理', '高级工程师', '市场专员', 'HR主管', '会计师'],
'入职年份': [2020, 2019, 2021, 2018, 2022],
'年薪(万元)': [18, 25, 12, 16, 14]
}
pd.DataFrame(employee_data).to_excel(writer, sheet_name='员工信息', index=False)
# 产品分析表
product_data = {
'产品类别': ['电子产品', '服装', '食品', '图书', '家居'],
'销售占比(%)': [35.2, 28.6, 15.8, 12.4, 8.0],
'平均客单价': [1280, 320, 85, 45, 560],
'库存周转率': [4.2, 6.8, 12.5, 8.3, 3.9],
'客户满意度': [4.3, 4.1, 4.5, 4.2, 4.0]
}
pd.DataFrame(product_data).to_excel(writer, sheet_name='产品分析', index=False)
print("已创建 Excel 文件: company_report.xlsx")
async def create_test_pdf():
"""创建PDF测试文件"""
filename = '/Users/liruwei/Documents/code/project/demo/base_rag/test_files/ai_research_report.pdf'
c = canvas.Canvas(filename, pagesize=letter)
width, height = letter
# 标题
c.setFont("Helvetica-Bold", 20)
c.drawString(50, height - 50, "Artificial Intelligence Research Report")
# 副标题
c.setFont("Helvetica", 14)
c.drawString(50, height - 80, "A Comprehensive Study on Modern AI Technologies")
# 内容
c.setFont("Helvetica", 12)
y_position = height - 120
content = [
"1. Introduction",
"",
"Artificial Intelligence (AI) has become one of the most transformative",
"technologies of the 21st century. This report examines the current state",
"of AI research and its applications across various industries.",
"",
"2. Machine Learning Fundamentals",
"",
"Machine Learning is a subset of AI that enables computers to learn",
"without being explicitly programmed. Key approaches include:",
"- Supervised Learning: Learning from labeled data",
"- Unsupervised Learning: Finding patterns in unlabeled data",
"- Reinforcement Learning: Learning through interaction",
"",
"3. Deep Learning Revolution",
"",
"Deep Learning has revolutionized AI by enabling:",
"- Image Recognition: Achieving human-level accuracy",
"- Natural Language Processing: Understanding human language",
"- Speech Recognition: Converting speech to text",
"- Autonomous Systems: Self-driving cars and robots",
"",
"4. Applications in Industry",
"",
"Healthcare: AI assists in medical diagnosis and drug discovery",
"Finance: Fraud detection and algorithmic trading",
"Transportation: Autonomous vehicles and traffic optimization",
"Entertainment: Recommendation systems and content generation",
"",
"5. Python in AI Development",
"",
"Python has become the dominant language for AI development due to",
"its simplicity and rich ecosystem of libraries.",
"",
"[Python Ecosystem Image - see below]",
"",
"6. Challenges and Future Directions",
"",
"Current challenges include data privacy, algorithmic bias,",
"and the need for explainable AI. Future research focuses on",
"artificial general intelligence and quantum machine learning.",
]
for line in content:
if y_position < 50: # 新页面
c.showPage()
y_position = height - 50
c.setFont("Helvetica", 12)
if line.startswith(("1.", "2.", "3.", "4.", "5.")):
c.setFont("Helvetica-Bold", 12)
else:
c.setFont("Helvetica", 12)
c.drawString(50, y_position, line)
y_position -= 20
# 添加图片到PDF
try:
# 在新页面添加图片
if y_position < 200: # 确保有足够空间
c.showPage()
y_position = height - 50
# 添加图片标题
c.setFont("Helvetica-Bold", 12)
c.drawString(50, y_position, "Figure 1: Python Ecosystem")
y_position -= 30
# 添加图片
img_path = '/Users/liruwei/Documents/code/project/demo/base_rag/test_files/python.png'
if os.path.exists(img_path):
c.drawImage(img_path, 50, y_position - 200, width=300, height=150)
y_position -= 220
else:
c.setFont("Helvetica", 10)
c.drawString(50, y_position, "[Python ecosystem image would be displayed here]")
y_position -= 20
except Exception as e:
print(f"警告无法添加图片到PDF: {e}")
c.setFont("Helvetica", 10)
c.drawString(50, y_position, "[Image placeholder - python.png]")
y_position -= 20
c.save()
print("已创建 PDF 文件: ai_research_report.pdf")
async def main():
"""创建所有测试文件"""
print("🔨 创建多格式测试文件...")
# 确保目录存在
os.makedirs('/Users/liruwei/Documents/code/project/demo/base_rag/test_files', exist_ok=True)
await create_complex_docx()
await create_test_csv()
await create_test_excel()
await create_test_pdf()
print("\n✅ 所有测试文件创建完成!")
print("📁 文件列表:")
print(" - complex_data_science.docx (带表格的复杂Word文档)")
print(" - sales_data.csv (销售数据CSV)")
print(" - company_report.xlsx (多工作表Excel)")
print(" - ai_research_report.pdf (AI研究报告PDF)")
if __name__ == "__main__":
asyncio.run(main())