From f9550c867439b542f4feaea9a372e928bab2a882 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E5=A6=82=E5=A8=81?= Date: Mon, 19 Jan 2026 15:44:58 +0800 Subject: [PATCH] init --- .gitignore | 38 +++ README.md | 84 +++++++ download_manager.py | 592 ++++++++++++++++++++++++++++++++++++++++++++ show_status.sh | 106 ++++++++ sources.txt | 23 ++ start.sh | 64 +++++ stop.sh | 50 ++++ 7 files changed, 957 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 download_manager.py create mode 100755 show_status.sh create mode 100644 sources.txt create mode 100755 start.sh create mode 100755 stop.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..25f24ae --- /dev/null +++ b/.gitignore @@ -0,0 +1,38 @@ +# 运行时生成的文件 +download_manager.pid +download_state.json +download_manager.log + +# 下载目录 +download/ + +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +*.egg +*.egg-info/ +dist/ +build/ + +# 系统文件 +.DS_Store +.DS_Store? +._* +.Spotlight-V100 +.Trashes +ehthumbs.db +Thumbs.db + +# 编辑器 +*.swp +*.swo +*~ +.vscode/ +.idea/ +*.sublime-* + +# 日志 +*.log diff --git a/README.md b/README.md new file mode 100644 index 0000000..cd54741 --- /dev/null +++ b/README.md @@ -0,0 +1,84 @@ +# 文件下载管理器 + +简单的Python文件下载工具,支持后台运行和Web界面监控。 + +## 功能特点 + +- ✅ 顺序下载,一个完成后才开始下一个 +- ✅ 自动从URL或响应头获取文件名 +- ✅ Web界面实时查看下载状态 +- ✅ 后台运行,记录PID和日志 +- ✅ 使用Python3内置库,无需额外安装 + +## 使用方法 + +### 1. 配置下载地址 + +编辑 `sources.txt` 文件,一行一个URL: + +``` +# 注释行以 # 开头 +https://example.com/file1.zip +https://example.com/file2.pdf +``` + +### 2. 启动服务 + +```bash +./start.sh +``` + +启动成功后会显示: +- PID(进程ID) +- Web访问地址:http://localhost:8888 +- 日志文件路径 +- 下载目录路径 +- **所有下载任务的状态统计和详情** + +### 3. 查看状态 + +```bash +./show_status.sh +``` + +显示所有任务的: +- 状态统计(等待/下载/完成/失败) +- 每个任务的详情(URL、文件名、大小、重试次数、错误信息) + +或者访问 Web 界面:http://localhost:8888 + +### 4. 停止服务 + +```bash +./stop.sh +``` + +停止前会显示当前任务状态。 + +## 文件说明 + +| 文件 | 说明 | +|------|------| +| `sources.txt` | 下载资源列表(一行一个URL) | +| `download_manager.py` | 主程序 | +| `start.sh` | 启动脚本 | +| `stop.sh` | 停止脚本 | +| `show_status.sh` | 状态查看脚本 | +| `download/` | 下载文件保存目录 | +| `download_state.json` | 任务状态保存文件 | +| `download_manager.log` | 运行日志 | +| `download_manager.pid` | 进程PID文件 | + +## 常见问题 + +**Q: 如何修改端口?** +A: 修改 `download_manager.py` 中的 `PORT = 8888` + +**Q: 下载中断后如何恢复?** +A: 重新运行 `./start.sh`,会自动从中断处继续 + +**Q: 如何重新下载失败的文件?** +A: 删除 `download_state.json` 后重启 + +**Q: 查看日志?** +A: `tail -f download_manager.log` diff --git a/download_manager.py b/download_manager.py new file mode 100644 index 0000000..a1704bd --- /dev/null +++ b/download_manager.py @@ -0,0 +1,592 @@ +#!/usr/bin/env python3 +""" +简单的文件下载管理器 +支持后台运行,通过Web界面查看下载状态 +""" +import http.server +import socketserver +import urllib.request +import urllib.error +import os +import sys +import json +import time +import logging +from pathlib import Path +from datetime import datetime + +# 配置 +PORT = 8888 +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +DOWNLOAD_DIR = os.path.join(SCRIPT_DIR, "download") +STATE_FILE = os.path.join(SCRIPT_DIR, "download_state.json") +LOG_FILE = os.path.join(SCRIPT_DIR, "download_manager.log") +PID_FILE = os.path.join(SCRIPT_DIR, "download_manager.pid") +MAX_RETRIES = 2 # 最大重试次数 + +# 数据源配置 +SOURCES_FILE = os.path.join(SCRIPT_DIR, "sources.txt") + + +def load_sources(): + """从 sources.txt 文件加载下载资源""" + sources = [] + if os.path.exists(SOURCES_FILE): + with open(SOURCES_FILE, 'r', encoding='utf-8') as f: + for line in f: + url = line.strip() + # 跳过空行和注释行 + if url and not url.startswith('#'): + sources.append(url) + return sources + + +SOURCES = load_sources() + + +class DownloadManager: + """下载管理器""" + + def __init__(self): + self.tasks = [] + self.load_state() + self.setup_logging() + + def setup_logging(self): + """设置日志""" + logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s', + handlers=[ + logging.FileHandler(LOG_FILE, encoding='utf-8'), + logging.StreamHandler(sys.stdout) + ] + ) + self.logger = logging.getLogger(__name__) + + def load_state(self): + """加载状态""" + # 每次启动都重新加载 sources.txt,检测是否有新增或删除的 URL + current_sources = load_sources() + + if os.path.exists(STATE_FILE): + try: + with open(STATE_FILE, 'r', encoding='utf-8') as f: + self.tasks = json.load(f) + + # 检查 sources 是否有变化,有变化则重新初始化 + existing_urls = {task['url'] for task in self.tasks} + if set(current_sources) != existing_urls: + print("检测到 sources.txt 有变化,重新初始化任务列表") + self.tasks = [{"url": url, "status": "pending", "filename": "", "size": 0, "error": "", "retry": 0} for url in current_sources] + self.save_state() + except Exception as e: + print(f"加载状态失败: {e}") + self.tasks = [{"url": url, "status": "pending", "filename": "", "size": 0, "error": "", "retry": 0} for url in current_sources] + else: + # 初始化任务列表 + self.tasks = [{"url": url, "status": "pending", "filename": "", "size": 0, "error": "", "retry": 0} for url in current_sources] + + def save_state(self): + """保存状态""" + try: + with open(STATE_FILE, 'w', encoding='utf-8') as f: + json.dump(self.tasks, f, ensure_ascii=False, indent=2) + except Exception as e: + self.logger.error(f"保存状态失败: {e}") + + def get_filename_from_url(self, url, response): + """从URL或响应头获取文件名""" + # 尝试从Content-Disposition获取 + content_disp = response.headers.get('Content-Disposition', '') + if content_disp: + import re + match = re.search(r'filename[*]?=["\']?([^"\';\s]+)["\']?', content_disp) + if match: + return match.group(1) + + # 从URL路径提取 + from urllib.parse import unquote, urlparse + path = urlparse(url).path + filename = os.path.basename(unquote(path)) + if filename: + return filename + + # 默认文件名 + return f"file_{int(time.time())}" + + def download_file(self, task): + """下载单个文件(带重试机制)""" + url = task['url'] + retry_count = task.get('retry', 0) + + try: + self.logger.info(f"开始下载: {url}" + (f" (重试 {retry_count}/{MAX_RETRIES})" if retry_count > 0 else "")) + + # 创建下载目录 + os.makedirs(DOWNLOAD_DIR, exist_ok=True) + + # 先获取文件名和预期大小 + req = urllib.request.Request(url, method='HEAD') + try: + with urllib.request.urlopen(req) as response: + filename = self.get_filename_from_url(url, response) + except: + # HEAD请求失败,使用GET请求获取文件名 + with urllib.request.urlopen(url) as response: + filename = self.get_filename_from_url(url, response) + + filepath = os.path.join(DOWNLOAD_DIR, filename) + + # 检查文件是否已存在且完整 + if self.check_file_complete(filepath, url): + task['status'] = 'completed' + task['filename'] = filename + task['size'] = os.path.getsize(filepath) + task['error'] = '' + self.logger.info(f"文件已存在且完整,跳过下载: {filename}") + self.save_state() + return True + + # 下载文件 + task['status'] = 'downloading' + task['filename'] = filename + self.save_state() + + with urllib.request.urlopen(url) as response: + # 获取预期文件大小 + expected_size = response.headers.get('Content-Length') + if expected_size: + expected_size = int(expected_size) + + downloaded = 0 + with open(filepath, 'wb') as f: + while True: + chunk = response.read(8192) + if not chunk: + break + f.write(chunk) + downloaded += len(chunk) + + # 验证文件完整性 + if expected_size and downloaded != expected_size: + os.remove(filepath) + raise IOError(f"文件不完整: 预期 {expected_size} 字节,实际 {downloaded} 字节") + + task['size'] = downloaded + task['status'] = 'completed' + task['error'] = '' + self.logger.info(f"下载完成: {filename} ({downloaded} bytes)") + self.save_state() + return True + + except Exception as e: + retry_count = task.get('retry', 0) + 1 + task['retry'] = retry_count + error_msg = str(e) + + if retry_count <= MAX_RETRIES: + task['status'] = 'retrying' + task['error'] = f'{error_msg} (准备第 {retry_count} 次重试)' + self.logger.warning(f"下载失败: {url} - {error_msg},准备第 {retry_count} 次重试") + self.save_state() + time.sleep(2) # 等待2秒后重试 + return self.download_file(task) + else: + task['status'] = 'failed' + task['error'] = f'{error_msg} (已重试 {MAX_RETRIES} 次)' + self.logger.error(f"下载失败: {url} - {error_msg} (已重试 {MAX_RETRIES} 次)") + self.save_state() + return False + + def check_file_complete(self, filepath, url): + """检查文件是否已存在且完整""" + if not os.path.exists(filepath): + return False + + # 获取已下载文件的大小 + local_size = os.path.getsize(filepath) + + # 尝试获取远程文件大小 + try: + req = urllib.request.Request(url, method='HEAD') + with urllib.request.urlopen(req, timeout=10) as response: + remote_size = response.headers.get('Content-Length') + if remote_size: + remote_size = int(remote_size) + # 文件存在且大小一致 + if local_size == remote_size: + return True + return False + except: + # 无法获取远程文件大小,检查本地文件是否大于0 + return local_size > 0 + + return local_size > 0 + + def run_downloads(self): + """执行所有下载任务""" + self.logger.info("=" * 50) + self.logger.info("开始下载任务") + + for task in self.tasks: + if task['status'] in ['pending', 'downloading', 'retrying']: + self.download_file(task) + + self.logger.info("所有下载任务完成") + self.logger.info("=" * 50) + + +class WebHandler(http.server.SimpleHTTPRequestHandler): + """Web请求处理器""" + + def __init__(self, *args, manager=None, **kwargs): + self.manager = manager + super().__init__(*args, **kwargs) + + def do_GET(self): + """处理GET请求""" + if self.path == '/' or self.path == '/status': + self.send_html() + elif self.path == '/api/tasks': + self.send_json(self.manager.tasks) + elif self.path.startswith('/api/log'): + self.send_log() + else: + super().do_GET() + + def send_html(self): + """发送HTML页面""" + html = """ + + + + + 下载管理器 + + + +
+
+

📥 文件下载管理器

+
+
+ 自动刷新 + +
+
+
+ 3秒 +
+ +
+
+
+
+ 下载目录: + """ + os.path.basename(DOWNLOAD_DIR) + """ +
+
+ 日志: + """ + os.path.basename(LOG_FILE) + """ +
+
+
+
+ + + + + + + + + + + + + +
#URL状态重试文件名大小错误信息
+
+
+
+

📋 最近日志

+ 更新于: --:--:-- +
+
加载中...
+
+
+ + +""" + self.send_response(200) + self.send_header('Content-Type', 'text/html; charset=utf-8') + self.end_headers() + self.wfile.write(html.encode('utf-8')) + + def send_json(self, data): + """发送JSON响应""" + self.send_response(200) + self.send_header('Content-Type', 'application/json; charset=utf-8') + self.end_headers() + self.wfile.write(json.dumps(data, ensure_ascii=False).encode('utf-8')) + + def send_log(self): + """发送日志内容""" + try: + if os.path.exists(LOG_FILE): + with open(LOG_FILE, 'r', encoding='utf-8') as f: + lines = f.readlines() + # 返回最后50行 + log_content = ''.join(lines[-50:]) + else: + log_content = '日志文件不存在' + except Exception as e: + log_content = f'读取日志失败: {e}' + + self.send_response(200) + self.send_header('Content-Type', 'text/plain; charset=utf-8') + self.end_headers() + self.wfile.write(log_content.encode('utf-8')) + + def log_message(self, format, *args): + """禁用默认日志""" + pass + + +def save_pid(pid): + """保存PID""" + with open(PID_FILE, 'w') as f: + f.write(str(pid)) + + +def run_server(manager): + """运行Web服务器""" + def handler(*args, **kwargs): + return WebHandler(*args, manager=manager, **kwargs) + + with socketserver.TCPServer(("", PORT), handler) as httpd: + print(f"\n🌐 Web服务器启动: http://localhost:{PORT}") + print(f"📁 下载目录: {os.path.abspath(DOWNLOAD_DIR)}") + print(f"📋 日志文件: {os.path.abspath(LOG_FILE)}") + print(f"📄 PID文件: {os.path.abspath(PID_FILE)}\n") + httpd.serve_forever() + + +def main(): + """主函数""" + # 检查是否已有进程运行 + if os.path.exists(PID_FILE): + try: + with open(PID_FILE, 'r') as f: + old_pid = int(f.read().strip()) + # 检查进程是否存在 + try: + os.kill(old_pid, 0) + print(f"⚠️ 进程已在运行 (PID: {old_pid})") + print("如需重启,请先运行: ./stop.sh") + return + except OSError: + # 进程不存在,删除旧的PID文件 + os.remove(PID_FILE) + except: + pass + + # 保存当前PID + save_pid(os.getpid()) + + # 创建管理器 + manager = DownloadManager() + + # 在后台线程执行下载 + import threading + download_thread = threading.Thread(target=manager.run_downloads, daemon=True) + download_thread.start() + + # 启动Web服务器 + run_server(manager) + + +if __name__ == "__main__": + main() diff --git a/show_status.sh b/show_status.sh new file mode 100755 index 0000000..73de466 --- /dev/null +++ b/show_status.sh @@ -0,0 +1,106 @@ +#!/bin/bash +# 显示下载任务状态 + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd "$SCRIPT_DIR" + +STATE_FILE="download_state.json" + +# 检查状态文件是否存在 +if [ ! -f "$STATE_FILE" ]; then + echo "⚠️ 状态文件不存在,服务可能尚未初始化" + exit 0 +fi + +# 使用 Python 解析 JSON 并显示状态 +python3 << 'EOF' +import json +import os + +STATE_FILE = "download_state.json" + +try: + with open(STATE_FILE, 'r', encoding='utf-8') as f: + tasks = json.load(f) + + if not tasks: + print("📋 暂无下载任务") + exit(0) + + # 统计状态 + status_count = {} + for task in tasks: + status = task.get('status', 'unknown') + status_count[status] = status_count.get(status, 0) + 1 + + # 显示统计 + total = len(tasks) + print(f"📊 总计: {total} 个任务") + print() + + status_map = { + 'pending': ('⏳ 等待中', 'yellow'), + 'downloading': ('⬇️ 下载中', 'blue'), + 'retrying': ('🔄 重试中', 'orange'), + 'completed': ('✅ 已完成', 'green'), + 'failed': ('❌ 失败', 'red'), + } + + for status, (label, _) in status_map.items(): + count = status_count.get(status, 0) + if count > 0: + print(f" {label}: {count}") + + print() + print("📋 任务详情:") + print("-" * 80) + + # 显示每个任务 + for i, task in enumerate(tasks, 1): + url = task.get('url', '') + status = task.get('status', 'unknown') + filename = task.get('filename', '') + size = task.get('size', 0) + error = task.get('error', '') + retry = task.get('retry', 0) + + # 状态图标 + status_icons = { + 'pending': '⏳', + 'downloading': '⬇️ ', + 'retrying': '🔄', + 'completed': '✅', + 'failed': '❌', + } + icon = status_icons.get(status, '❓') + + # 格式化大小 + def format_size(bytes_size): + if bytes_size == 0: + return '-' + for unit in ['B', 'KB', 'MB', 'GB']: + if bytes_size < 1024: + return f"{bytes_size:.2f} {unit}" + bytes_size /= 1024 + return f"{bytes_size:.2f} TB" + + # URL 缩短显示 + short_url = url if len(url) <= 60 else url[:57] + "..." + + print(f"{i:2d}. {icon} {status}") + print(f" URL: {short_url}") + if filename: + print(f" 文件: {filename} ({format_size(size)})") + if retry > 0: + print(f" 重试: {retry} 次") + if error: + print(f" 错误: {error}") + print() + +except FileNotFoundError: + print("⚠️ 状态文件不存在") +except json.JSONDecodeError: + print("⚠️ 状态文件格式错误") +except Exception as e: + print(f"⚠️ 读取状态失败: {e}") +EOF diff --git a/sources.txt b/sources.txt new file mode 100644 index 0000000..35b5dc1 --- /dev/null +++ b/sources.txt @@ -0,0 +1,23 @@ +# 下载资源列表 +# 一行一个URL,以 # 开头的行会被忽略(注释) + +# 测试用的小文件(来自 httpbin.org) +https://httpbin.org/json +https://httpbin.org/uuid +https://httpbin.org/ip +https://httpbin.org/user-agent +https://httpbin.org/headers +https://httpbin.org/get +https://httpbin.org/brotli +https://httpbin.org/status/200 +https://httpbin.org/encoding/utf8 +https://httpbin.org/gzip +https://httpbin.org/deflate +https://httpbin.org/robots.txt +https://httpbin.org/delay/0 +https://httpbin.org/deny +https://httpbin.org/html +https://httpbin.org/xml +https://httpbin.org/image/png +https://httpbin.org/image/jpeg +https://httpbin.org/image/webp diff --git a/start.sh b/start.sh new file mode 100755 index 0000000..a8ff48a --- /dev/null +++ b/start.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# 启动下载管理器 + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd "$SCRIPT_DIR" + +echo "================================" +echo " 启动下载管理器" +echo "================================" + +# 检查Python3 +if ! command -v python3 &> /dev/null; then + echo "❌ 错误: 未找到 python3" + exit 1 +fi + +# 检查是否已运行 +if [ -f "download_manager.pid" ]; then + OLD_PID=$(cat download_manager.pid) + if ps -p "$OLD_PID" > /dev/null 2>&1; then + echo "⚠️ 下载管理器已在运行 (PID: $OLD_PID)" + echo "" + ./show_status.sh + echo "" + echo "如需重启,请先运行: ./stop.sh" + exit 1 + else + echo "清理旧的PID文件..." + rm -f download_manager.pid + fi +fi + +# 启动服务 +echo "启动服务中..." +nohup python3 download_manager.py > /dev/null 2>&1 & + +# 等待启动 +sleep 2 + +# 检查是否成功 +if [ -f "download_manager.pid" ]; then + PID=$(cat download_manager.pid) + if ps -p "$PID" > /dev/null 2>&1; then + echo "✅ 启动成功!" + echo "" + echo " PID: $PID" + echo " Web: http://localhost:8888" + echo " 日志: download_manager.log" + echo " 下载目录: download/" + echo "" + echo "================================" + echo " 下载任务状态" + echo "================================" + ./show_status.sh + echo "" + echo "使用 ./stop.sh 停止服务" + else + echo "❌ 启动失败,请查看日志: download_manager.log" + exit 1 + fi +else + echo "❌ 启动失败,请检查 download_manager.log" + exit 1 +fi diff --git a/stop.sh b/stop.sh new file mode 100755 index 0000000..df5eb7b --- /dev/null +++ b/stop.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# 停止下载管理器 + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd "$SCRIPT_DIR" + +echo "================================" +echo " 停止下载管理器" +echo "================================" + +PID_FILE="download_manager.pid" + +if [ ! -f "$PID_FILE" ]; then + echo "⚠️ PID文件不存在,服务可能未运行" + exit 0 +fi + +PID=$(cat "$PID_FILE") + +# 检查进程是否存在 +if ! ps -p "$PID" > /dev/null 2>&1; then + echo "⚠️ 进程 $PID 不存在" + rm -f "$PID_FILE" + exit 0 +fi + +# 显示停止前的状态 +echo "" +./show_status.sh +echo "" + +# 停止进程 +echo "停止进程 $PID ..." +kill "$PID" + +# 等待进程结束 +for i in {1..10}; do + if ! ps -p "$PID" > /dev/null 2>&1; then + echo "✅ 已成功停止" + rm -f "$PID_FILE" + exit 0 + fi + sleep 1 +done + +# 强制结束 +echo "强制停止进程..." +kill -9 "$PID" +rm -f "$PID_FILE" +echo "✅ 已强制停止"