download_app/download_manager.py

606 lines
24 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
"""
简单的文件下载管理器
支持后台运行通过Web界面查看下载状态
"""
import http.server
import socketserver
import urllib.request
import urllib.error
import os
import sys
import json
import time
import logging
from pathlib import Path
from datetime import datetime
# 配置
PORT = 8888
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
DOWNLOAD_DIR = os.path.join(SCRIPT_DIR, "download")
STATE_FILE = os.path.join(SCRIPT_DIR, "download_state.json")
LOG_FILE = os.path.join(SCRIPT_DIR, "download_manager.log")
PID_FILE = os.path.join(SCRIPT_DIR, "download_manager.pid")
MAX_RETRIES = 2 # 最大重试次数
# 数据源配置
SOURCES_FILE = os.path.join(SCRIPT_DIR, "sources.txt")
def load_sources():
"""从 sources.txt 文件加载下载资源"""
sources = []
if os.path.exists(SOURCES_FILE):
with open(SOURCES_FILE, 'r', encoding='utf-8') as f:
for line in f:
url = line.strip()
# 跳过空行和注释行
if url and not url.startswith('#'):
sources.append(url)
return sources
SOURCES = load_sources()
class DownloadManager:
"""下载管理器"""
def __init__(self):
self.tasks = []
self.load_state()
self.setup_logging()
def setup_logging(self):
"""设置日志"""
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s',
handlers=[
logging.FileHandler(LOG_FILE, encoding='utf-8'),
logging.StreamHandler(sys.stdout)
]
)
self.logger = logging.getLogger(__name__)
def load_state(self):
"""加载状态"""
# 每次启动都重新加载 sources.txt检测是否有新增或删除的 URL
current_sources = load_sources()
if os.path.exists(STATE_FILE):
try:
with open(STATE_FILE, 'r', encoding='utf-8') as f:
self.tasks = json.load(f)
# 检查 sources 是否有变化,有变化则重新初始化
existing_urls = {task['url'] for task in self.tasks}
if set(current_sources) != existing_urls:
print("检测到 sources.txt 有变化,重新初始化任务列表")
self.tasks = [{"url": url, "status": "pending", "filename": "", "size": 0, "error": "", "retry": 0} for url in current_sources]
self.save_state()
except Exception as e:
print(f"加载状态失败: {e}")
self.tasks = [{"url": url, "status": "pending", "filename": "", "size": 0, "error": "", "retry": 0} for url in current_sources]
else:
# 初始化任务列表
self.tasks = [{"url": url, "status": "pending", "filename": "", "size": 0, "error": "", "retry": 0} for url in current_sources]
def save_state(self):
"""保存状态"""
try:
with open(STATE_FILE, 'w', encoding='utf-8') as f:
json.dump(self.tasks, f, ensure_ascii=False, indent=2)
except Exception as e:
self.logger.error(f"保存状态失败: {e}")
def get_filename_from_url(self, url, response):
"""从URL或响应头获取文件名"""
from urllib.parse import unquote, urlparse
# 优先从 URL 的 ?n= 参数获取文件名
if '?n=' in url:
try:
file_name = unquote(url.split('?n=')[1].split('&')[0])
if file_name:
# 确保有 .zip 后缀(如果文件本身没有扩展名)
if '.' not in os.path.basename(file_name):
file_name = file_name + '.zip'
return file_name
except:
pass
# 尝试从Content-Disposition获取
content_disp = response.headers.get('Content-Disposition', '')
if content_disp:
import re
match = re.search(r'filename[*]?=["\']?([^"\';\s]+)["\']?', content_disp)
if match:
return match.group(1)
# 从URL路径提取
path = urlparse(url).path
filename = os.path.basename(unquote(path))
if filename:
return filename
# 默认文件名
return f"file_{int(time.time())}"
def download_file(self, task):
"""下载单个文件(带重试机制)"""
url = task['url']
retry_count = task.get('retry', 0)
try:
self.logger.info(f"开始下载: {url}" + (f" (重试 {retry_count}/{MAX_RETRIES})" if retry_count > 0 else ""))
# 创建下载目录
os.makedirs(DOWNLOAD_DIR, exist_ok=True)
# 先获取文件名和预期大小
req = urllib.request.Request(url, method='HEAD')
try:
with urllib.request.urlopen(req) as response:
filename = self.get_filename_from_url(url, response)
except:
# HEAD请求失败使用GET请求获取文件名
with urllib.request.urlopen(url) as response:
filename = self.get_filename_from_url(url, response)
filepath = os.path.join(DOWNLOAD_DIR, filename)
# 检查文件是否已存在且完整
if self.check_file_complete(filepath, url):
task['status'] = 'completed'
task['filename'] = filename
task['size'] = os.path.getsize(filepath)
task['error'] = ''
self.logger.info(f"文件已存在且完整,跳过下载: {filename}")
self.save_state()
return True
# 下载文件
task['status'] = 'downloading'
task['filename'] = filename
self.save_state()
with urllib.request.urlopen(url) as response:
# 获取预期文件大小
expected_size = response.headers.get('Content-Length')
if expected_size:
expected_size = int(expected_size)
downloaded = 0
with open(filepath, 'wb') as f:
while True:
chunk = response.read(8192)
if not chunk:
break
f.write(chunk)
downloaded += len(chunk)
# 验证文件完整性
if expected_size and downloaded != expected_size:
os.remove(filepath)
raise IOError(f"文件不完整: 预期 {expected_size} 字节,实际 {downloaded} 字节")
task['size'] = downloaded
task['status'] = 'completed'
task['error'] = ''
self.logger.info(f"下载完成: {filename} ({downloaded} bytes)")
self.save_state()
return True
except Exception as e:
retry_count = task.get('retry', 0) + 1
task['retry'] = retry_count
error_msg = str(e)
if retry_count <= MAX_RETRIES:
task['status'] = 'retrying'
task['error'] = f'{error_msg} (准备第 {retry_count} 次重试)'
self.logger.warning(f"下载失败: {url} - {error_msg},准备第 {retry_count} 次重试")
self.save_state()
time.sleep(2) # 等待2秒后重试
return self.download_file(task)
else:
task['status'] = 'failed'
task['error'] = f'{error_msg} (已重试 {MAX_RETRIES} 次)'
self.logger.error(f"下载失败: {url} - {error_msg} (已重试 {MAX_RETRIES} 次)")
self.save_state()
return False
def check_file_complete(self, filepath, url):
"""检查文件是否已存在且完整"""
if not os.path.exists(filepath):
return False
# 获取已下载文件的大小
local_size = os.path.getsize(filepath)
# 尝试获取远程文件大小
try:
req = urllib.request.Request(url, method='HEAD')
with urllib.request.urlopen(req, timeout=10) as response:
remote_size = response.headers.get('Content-Length')
if remote_size:
remote_size = int(remote_size)
# 文件存在且大小一致
if local_size == remote_size:
return True
return False
except:
# 无法获取远程文件大小检查本地文件是否大于0
return local_size > 0
return local_size > 0
def run_downloads(self):
"""执行所有下载任务"""
self.logger.info("=" * 50)
self.logger.info("开始下载任务")
for task in self.tasks:
if task['status'] in ['pending', 'downloading', 'retrying']:
self.download_file(task)
self.logger.info("所有下载任务完成")
self.logger.info("=" * 50)
class WebHandler(http.server.SimpleHTTPRequestHandler):
"""Web请求处理器"""
def __init__(self, *args, manager=None, **kwargs):
self.manager = manager
super().__init__(*args, **kwargs)
def do_GET(self):
"""处理GET请求"""
if self.path == '/' or self.path == '/status':
self.send_html()
elif self.path == '/api/tasks':
self.send_json(self.manager.tasks)
elif self.path.startswith('/api/log'):
self.send_log()
else:
super().do_GET()
def send_html(self):
"""发送HTML页面"""
html = """<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>下载管理器</title>
<style>
* { margin: 0; padding: 0; box-sizing: border-box; }
body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); min-height: 100vh; padding: 20px; }
.container { max-width: 1400px; margin: 0 auto; background: white; border-radius: 12px; box-shadow: 0 10px 40px rgba(0,0,0,0.2); overflow: hidden; }
.header { background: linear-gradient(135deg, #4CAF50 0%, #45a049 100%); color: white; padding: 25px 30px; display: flex; justify-content: space-between; align-items: center; }
.header h1 { font-size: 24px; font-weight: 600; }
.header-controls { display: flex; gap: 10px; align-items: center; }
.btn { padding: 8px 16px; border: none; border-radius: 6px; cursor: pointer; font-size: 14px; font-weight: 500; transition: all 0.3s; }
.btn:hover { transform: translateY(-2px); box-shadow: 0 4px 12px rgba(0,0,0,0.15); }
.btn-primary { background: white; color: #4CAF50; }
.btn-secondary { background: rgba(255,255,255,0.2); color: white; }
.btn-secondary:hover { background: rgba(255,255,255,0.3); }
.refresh-badge { display: inline-flex; align-items: center; gap: 6px; padding: 6px 12px; background: rgba(255,255,255,0.2); border-radius: 20px; font-size: 13px; }
.refresh-badge.active { background: rgba(76, 175, 80, 0.9); }
.refresh-badge.paused { background: rgba(255, 152, 0, 0.9); }
.spinner { width: 14px; height: 14px; border: 2px solid rgba(255,255,255,0.3); border-top-color: white; border-radius: 50%; animation: spin 1s linear infinite; }
.paused .spinner { animation: none; border: 2px solid rgba(255,255,255,0.5); }
@keyframes spin { to { transform: rotate(360deg); } }
.info-bar { background: #f8f9fa; padding: 15px 30px; border-bottom: 1px solid #e9ecef; display: flex; flex-wrap: wrap; gap: 20px; align-items: center; font-size: 14px; }
.info-item { display: flex; align-items: center; gap: 8px; }
.info-item code { background: #e9ecef; padding: 3px 8px; border-radius: 4px; font-size: 12px; }
.stats { display: flex; gap: 20px; margin-left: auto; }
.stat-badge { padding: 4px 12px; border-radius: 20px; font-size: 12px; font-weight: 600; }
.stat-badge.pending { background: #fff3cd; color: #856404; }
.stat-badge.downloading { background: #cce5ff; color: #004085; }
.stat-badge.retrying { background: #ffe8cc; color: #d97706; }
.stat-badge.completed { background: #d4edda; color: #155724; }
.stat-badge.failed { background: #f8d7da; color: #721c24; }
.table-container { overflow-x: auto; }
table { width: 100%; border-collapse: collapse; }
th { background: #f8f9fa; padding: 15px; text-align: left; font-weight: 600; font-size: 13px; color: #495057; border-bottom: 2px solid #dee2e6; white-space: nowrap; }
td { padding: 12px 15px; border-bottom: 1px solid #e9ecef; font-size: 14px; }
tr:hover { background: #f8f9fa; }
.status { padding: 4px 10px; border-radius: 4px; font-size: 12px; font-weight: 600; display: inline-block; }
.status.pending { background: #fff3cd; color: #856404; }
.status.downloading { background: #cce5ff; color: #004085; animation: pulse 2s infinite; }
.status.retrying { background: #ffe8cc; color: #d97706; animation: pulse 2s infinite; }
.status.completed { background: #d4edda; color: #155724; }
.status.failed { background: #f8d7da; color: #721c24; }
@keyframes pulse { 0%, 100% { opacity: 1; } 50% { opacity: 0.7; } }
.error { color: #dc3545; font-size: 12px; max-width: 300px; display: block; }
.url-link { color: #4CAF50; text-decoration: none; max-width: 400px; display: inline-block; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
.url-link:hover { text-decoration: underline; }
.log-section { padding: 20px 30px; background: #f8f9fa; border-top: 1px solid #e9ecef; }
.log-header { display: flex; justify-content: space-between; align-items: center; margin-bottom: 10px; }
.log-header h3 { font-size: 16px; color: #495057; }
.last-update { font-size: 12px; color: #6c757d; }
pre { background: #2d2d2d; color: #f8f8f2; padding: 15px; border-radius: 8px; overflow-x: auto; max-height: 300px; font-size: 12px; line-height: 1.5; }
.auto-refresh-toggle { display: flex; align-items: center; gap: 8px; font-size: 14px; }
.toggle-switch { position: relative; width: 44px; height: 24px; }
.toggle-switch input { opacity: 0; width: 0; height: 0; }
.slider { position: absolute; cursor: pointer; top: 0; left: 0; right: 0; bottom: 0; background-color: #ccc; transition: 0.3s; border-radius: 24px; }
.slider:before { position: absolute; content: ""; height: 18px; width: 18px; left: 3px; bottom: 3px; background-color: white; transition: 0.3s; border-radius: 50%; }
input:checked + .slider { background-color: #4CAF50; }
input:checked + .slider:before { transform: translateX(20px); }
</style>
</head>
<body>
<div class="container">
<div class="header">
<h1>📥 文件下载管理器</h1>
<div class="header-controls">
<div class="auto-refresh-toggle">
<span>自动刷新</span>
<label class="toggle-switch">
<input type="checkbox" id="autoRefresh" checked>
<span class="slider"></span>
</label>
</div>
<div class="refresh-badge active" id="refreshBadge">
<div class="spinner"></div>
<span id="refreshStatus">3秒</span>
</div>
<button class="btn btn-primary" onclick="refreshNow()">🔄 立即刷新</button>
</div>
</div>
<div class="info-bar">
<div class="info-item">
<strong>下载目录:</strong>
<code>""" + os.path.basename(DOWNLOAD_DIR) + """</code>
</div>
<div class="info-item">
<strong>日志:</strong>
<code>""" + os.path.basename(LOG_FILE) + """</code>
</div>
<div class="stats" id="stats"></div>
</div>
<div class="table-container">
<table>
<thead>
<tr>
<th>#</th>
<th>URL</th>
<th>状态</th>
<th>重试</th>
<th>文件名</th>
<th>大小</th>
<th>错误信息</th>
</tr>
</thead>
<tbody id="tasklist"></tbody>
</table>
</div>
<div class="log-section">
<div class="log-header">
<h3>📋 最近日志</h3>
<span class="last-update">更新于: <span id="lastUpdate">--:--:--</span></span>
</div>
<pre id="log">加载中...</pre>
</div>
</div>
<script>
const MAX_RETRIES = """ + str(MAX_RETRIES) + """;
const REFRESH_INTERVAL = 3000;
let autoRefresh = true;
let refreshTimer = null;
let countdownTimer = null;
let countdown = REFRESH_INTERVAL / 1000;
function formatSize(bytes) {
if (bytes === 0) return '-';
const units = ['B', 'KB', 'MB', 'GB'];
const i = Math.floor(Math.log(bytes) / Math.log(1024));
return (bytes / Math.pow(1024, i)).toFixed(2) + ' ' + units[i];
}
function getStatusClass(status) {
return 'status ' + status;
}
function getStatusText(status) {
const map = {
'pending': '⏳ 等待中',
'downloading': '⬇️ 下载中',
'retrying': '🔄 重试中',
'completed': '✅ 完成',
'failed': '❌ 失败',
'skipped': '⏭️ 已跳过'
};
return map[status] || status;
}
function updateStats(tasks) {
const stats = { pending: 0, downloading: 0, retrying: 0, completed: 0, failed: 0 };
tasks.forEach(t => stats[t.status] = (stats[t.status] || 0) + 1);
const statsHtml = Object.entries(stats)
.filter(([_, count]) => count > 0)
.map(([status, count]) => `<span class="stat-badge ${status}">${getStatusText(status)}: ${count}</span>`)
.join('');
document.getElementById('stats').innerHTML = statsHtml;
}
function loadTasks() {
return fetch('/api/tasks')
.then(r => r.json())
.then(tasks => {
const tbody = document.getElementById('tasklist');
tbody.innerHTML = tasks.map((t, i) => `
<tr>
<td>${i + 1}</td>
<td><a href="${t.url}" target="_blank" class="url-link">${t.url}</a></td>
<td><span class="${getStatusClass(t.status)}">${getStatusText(t.status)}</span></td>
<td>${(t.retry || 0) > 0 ? '<span style="color:#d97706;font-weight:bold;">' + (t.retry || 0) + ' / ' + MAX_RETRIES + '</span>' : '-'}</td>
<td>${t.filename || '-'}</td>
<td>${formatSize(t.size)}</td>
<td>${t.error ? `<span class="error" title="${t.error}">${t.error}</span>` : '-'}</td>
</tr>
`).join('');
updateStats(tasks);
});
}
function loadLog() {
return fetch('/api/log')
.then(r => r.text())
.then(log => {
document.getElementById('log').textContent = log || '暂无日志';
});
}
function updateLastUpdateTime() {
const now = new Date();
const time = now.toLocaleTimeString('zh-CN', { hour12: false });
document.getElementById('lastUpdate').textContent = time;
}
function refreshNow() {
loadTasks();
loadLog();
updateLastUpdateTime();
countdown = REFRESH_INTERVAL / 1000;
}
function startAutoRefresh() {
if (refreshTimer) clearInterval(refreshTimer);
if (countdownTimer) clearInterval(countdownTimer);
countdownTimer = setInterval(() => {
countdown--;
if (countdown <= 0) countdown = REFRESH_INTERVAL / 1000;
document.getElementById('refreshStatus').textContent = countdown + '';
}, 1000);
refreshTimer = setInterval(() => {
if (autoRefresh) {
refreshNow();
}
}, REFRESH_INTERVAL);
}
function stopAutoRefresh() {
if (refreshTimer) clearInterval(refreshTimer);
if (countdownTimer) clearInterval(countdownTimer);
}
document.getElementById('autoRefresh').addEventListener('change', function(e) {
autoRefresh = e.target.checked;
const badge = document.getElementById('refreshBadge');
if (autoRefresh) {
badge.classList.remove('paused');
badge.classList.add('active');
startAutoRefresh();
} else {
badge.classList.remove('active');
badge.classList.add('paused');
stopAutoRefresh();
document.getElementById('refreshStatus').textContent = '已暂停';
}
});
// 初始加载
refreshNow();
startAutoRefresh();
</script>
</body>
</html>"""
self.send_response(200)
self.send_header('Content-Type', 'text/html; charset=utf-8')
self.end_headers()
self.wfile.write(html.encode('utf-8'))
def send_json(self, data):
"""发送JSON响应"""
self.send_response(200)
self.send_header('Content-Type', 'application/json; charset=utf-8')
self.end_headers()
self.wfile.write(json.dumps(data, ensure_ascii=False).encode('utf-8'))
def send_log(self):
"""发送日志内容"""
try:
if os.path.exists(LOG_FILE):
with open(LOG_FILE, 'r', encoding='utf-8') as f:
lines = f.readlines()
# 返回最后50行
log_content = ''.join(lines[-50:])
else:
log_content = '日志文件不存在'
except Exception as e:
log_content = f'读取日志失败: {e}'
self.send_response(200)
self.send_header('Content-Type', 'text/plain; charset=utf-8')
self.end_headers()
self.wfile.write(log_content.encode('utf-8'))
def log_message(self, format, *args):
"""禁用默认日志"""
pass
def save_pid(pid):
"""保存PID"""
with open(PID_FILE, 'w') as f:
f.write(str(pid))
def run_server(manager):
"""运行Web服务器"""
def handler(*args, **kwargs):
return WebHandler(*args, manager=manager, **kwargs)
with socketserver.TCPServer(("", PORT), handler) as httpd:
print(f"\n🌐 Web服务器启动: http://localhost:{PORT}")
print(f"📁 下载目录: {os.path.abspath(DOWNLOAD_DIR)}")
print(f"📋 日志文件: {os.path.abspath(LOG_FILE)}")
print(f"📄 PID文件: {os.path.abspath(PID_FILE)}\n")
httpd.serve_forever()
def main():
"""主函数"""
# 检查是否已有进程运行
if os.path.exists(PID_FILE):
try:
with open(PID_FILE, 'r') as f:
old_pid = int(f.read().strip())
# 检查进程是否存在
try:
os.kill(old_pid, 0)
print(f"⚠️ 进程已在运行 (PID: {old_pid})")
print("如需重启,请先运行: ./stop.sh")
return
except OSError:
# 进程不存在删除旧的PID文件
os.remove(PID_FILE)
except:
pass
# 保存当前PID
save_pid(os.getpid())
# 创建管理器
manager = DownloadManager()
# 在后台线程执行下载
import threading
download_thread = threading.Thread(target=manager.run_downloads, daemon=True)
download_thread.start()
# 启动Web服务器
run_server(manager)
if __name__ == "__main__":
main()