spider
/
eh-v2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

update

Browse Source
main
Jack 2 months ago
parent f9bf9826b4
commit f307496333
2 changed files with 389 additions and 91 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 473
      main.py
  2. 9
      post_eh_data.js

473
main.py
Unescape View File

@ -4,9 +4,11 @@ import json
import logging
from pathlib import Path
from typing import Dict, Any, List
import asyncio
import httpx
import aiofiles
from fastapi import FastAPI, HTTPException
from fastapi import FastAPI, HTTPException, BackgroundTasks
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import HTMLResponse
from fastapi.staticfiles import StaticFiles
@ -24,10 +26,15 @@ logger = logging.getLogger(__name__)
DOWNLOADS_DIR = "downloads"
MAX_FILENAME_LENGTH = 100
INVALID_FILENAME_CHARS = '<>:"/\\|?*'
MAX_CONCURRENT_DOWNLOADS = 5 # 最大并发下载数
DOWNLOAD_TIMEOUT = 30 # 下载超时时间（秒）
# FastAPI应用
app = FastAPI(title="eh-v2")
# 全局变量用于跟踪下载状态
download_status: Dict[str, Dict[str, Any]] = {}
# 数据模型
class SaveDataRequest(BaseModel):
url: str
@ -41,6 +48,13 @@ class GalleryInfo(BaseModel):
total_images: int
downloaded_images: int
class DownloadStatusResponse(BaseModel):
status: str
message: str
downloaded: int
total: int
current_progress: float
# 工具函数
def setup_downloads_directory() -> Path:
"""创建并返回下载目录路径"""
@ -109,42 +123,178 @@ def get_all_galleries() -> List[GalleryInfo]:
return galleries
# 初始化
downloads_path = setup_downloads_directory()
async def download_single_image(client: httpx.AsyncClient, url: str, file_path: Path, semaphore: asyncio.Semaphore) -> bool:
"""下载单张图片"""
async with semaphore:
try:
logger.info(f"开始下载: {url} -> {file_path}")
# 如果文件已存在，跳过下载
if file_path.exists():
logger.info(f"文件已存在，跳过: {file_path}")
return True
# 发送请求下载图片
async with client.stream('GET', url, timeout=DOWNLOAD_TIMEOUT) as response:
response.raise_for_status()
# 异步写入文件
async with aiofiles.open(file_path, 'wb') as f:
async for chunk in response.aiter_bytes():
await f.write(chunk)
logger.info(f"下载完成: {file_path}")
return True
except httpx.TimeoutException:
logger.error(f"下载超时: {url}")
return False
except httpx.HTTPStatusError as e:
logger.error(f"HTTP错误 {e.response.status_code}: {url}")
return False
except Exception as e:
logger.error(f"下载失败 {url}: {e}")
return False
async def download_gallery_images(title: str) -> DownloadStatusResponse:
"""下载指定画廊的所有图片"""
safe_title = sanitize_filename(title)
gallery_path = downloads_path / safe_title
data_file = gallery_path / "data.json"
if not data_file.exists():
return DownloadStatusResponse(
status="error",
message="画廊数据文件不存在",
downloaded=0,
total=0,
current_progress=0.0
)
# API路由
@app.post("/save_url")
async def save_url(data: SaveDataRequest):
"""保存URL数据到文件系统"""
try:
logger.info("收到保存数据请求")
logger.info(f"标题: {data.title}, URL: {data.url}, 图片数量: {data.total_images}")
# 读取画廊数据
async with aiofiles.open(data_file, 'r', encoding='utf-8') as f:
content = await f.read()
data = json.loads(content)
all_images = data.get('all_images', {})
total_images = len(all_images)
if total_images == 0:
return DownloadStatusResponse(
status="error",
message="没有可下载的图片",
downloaded=0,
total=0,
current_progress=0.0
)
# 初始化下载状态
download_status[title] = {
"downloaded": 0,
"total": total_images,
"status": "downloading"
}
# 创建标题目录
title_dir = create_title_directory(downloads_path, data.title)
logger.info(f"开始下载画廊 '{title}'，共 {total_images} 张图片")
# 创建信号量限制并发数
semaphore = asyncio.Semaphore(MAX_CONCURRENT_DOWNLOADS)
# 使用异步HTTP客户端
async with httpx.AsyncClient(
headers={
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
},
follow_redirects=True
) as client:
# 准备下载任务
tasks = []
for filename, url in all_images.items():
image_path = gallery_path / filename
# 如果图片已存在，跳过下载但计入完成数量
if image_path.exists():
download_status[title]["downloaded"] += 1
continue
task = download_single_image(client, url, image_path, semaphore)
tasks.append(task)
# 批量执行下载任务
if tasks:
results = await asyncio.gather(*tasks, return_exceptions=True)
# 统计成功下载的数量
successful_downloads = sum(1 for result in results if result is True)
download_status[title]["downloaded"] += successful_downloads
# 更新最终状态
downloaded_count = download_status[title]["downloaded"]
progress = (downloaded_count / total_images) * 100
if downloaded_count == total_images:
download_status[title]["status"] = "completed"
message = f"下载完成！共下载 {downloaded_count}/{total_images} 张图片"
logger.info(f"画廊 '{title}' {message}")
else:
download_status[title]["status"] = "partial"
message = f"部分完成！下载 {downloaded_count}/{total_images} 张图片"
logger.warning(f"画廊 '{title}' {message}")
return DownloadStatusResponse(
status="success",
message=message,
downloaded=downloaded_count,
total=total_images,
current_progress=progress
)
# 数据文件路径
data_file = title_dir / "data.json"
except Exception as e:
logger.error(f"下载画廊 '{title}' 时发生错误: {e}")
download_status[title] = {
"status": "error",
"message": str(e)
}
return DownloadStatusResponse(
status="error",
message=f"下载失败: {str(e)}",
downloaded=0,
total=0,
current_progress=0.0
)
async def download_all_pending_galleries():
"""下载所有未完成的画廊"""
galleries = get_all_galleries()
pending_galleries = [g for g in galleries if g.downloaded_images < g.total_images]
# 异步保存数据
await save_data_to_file(data_file, data.dict())
if not pending_galleries:
logger.info("没有待下载的画廊")
return
logger.info(f"数据已保存到: {data_file}")
logger.info(f"开始批量下载 {len(pending_galleries)} 个画廊")
return {
"status": "success",
"message": "数据保存成功",
"file_path": str(data_file),
"title": data.title,
"total_images": data.total_images
}
for gallery in pending_galleries:
if gallery.downloaded_images < gallery.total_images:
logger.info(f"开始下载画廊: {gallery.title}")
result = await download_gallery_images(gallery.title)
except Exception as e:
error_msg = f"保存数据时出错: {str(e)}"
logger.error(error_msg)
logger.exception("详细错误信息:")
raise HTTPException(status_code=500, detail=error_msg)
if result.status == "success":
logger.info(f"画廊 '{gallery.title}' 下载完成: {result.message}")
else:
logger.error(f"画廊 '{gallery.title}' 下载失败: {result.message}")
# 添加延迟避免请求过于频繁
await asyncio.sleep(1)
logger.info("批量下载任务完成")
# 初始化
downloads_path = setup_downloads_directory()
# API路由
@app.get("/", response_class=HTMLResponse)
async def read_gallery_manager():
"""画廊管理页面"""
@ -233,6 +383,14 @@ async def read_gallery_manager():
background: #c82333;
transform: translateY(-2px);
}
.btn-warning {
background: #ffc107;
color: #212529;
}
.btn-warning:hover {
background: #e0a800;
transform: translateY(-2px);
}
.btn:disabled {
background: #6c757d;
cursor: not-allowed;
@ -249,7 +407,7 @@ async def read_gallery_manager():
margin-bottom: 15px;
transition: all 0.3s ease;
display: flex;
justify-content: space-between;
justify-content: between;
align-items: center;
}
.gallery-item:hover {
@ -271,12 +429,8 @@ async def read_gallery_manager():
color: #6c757d;
font-size: 0.9em;
}
.gallery-actions {
display: flex;
gap: 10px;
}
.progress-bar {
width: 200px;
width: 100%;
height: 8px;
background: #e9ecef;
border-radius: 4px;
@ -288,6 +442,29 @@ async def read_gallery_manager():
background: linear-gradient(90deg, #28a745, #20c997);
transition: width 0.3s ease;
}
.completed .progress-fill {
background: linear-gradient(90deg, #007bff, #0056b3);
}
.status-badge {
display: inline-block;
padding: 4px 12px;
border-radius: 20px;
font-size: 0.8em;
font-weight: 600;
margin-left: 10px;
}
.status-downloading {
background: #fff3cd;
color: #856404;
}
.status-completed {
background: #d1ecf1;
color: #0c5460;
}
.status-error {
background: #f8d7da;
color: #721c24;
}
.empty-state {
text-align: center;
padding: 60px 20px;
@ -297,6 +474,25 @@ async def read_gallery_manager():
margin-bottom: 10px;
font-size: 1.5em;
}
.stats-summary {
background: #f8f9fa;
padding: 15px 20px;
border-bottom: 1px solid #e9ecef;
display: flex;
justify-content: space-between;
align-items: center;
font-size: 0.9em;
color: #6c757d;
}
.gallery-actions {
display: flex;
gap: 10px;
}
.download-progress {
margin-top: 10px;
font-size: 0.9em;
color: #6c757d;
}
</style>
</head>
<body>
@ -306,21 +502,30 @@ async def read_gallery_manager():
<p>管理您的画廊下载任务</p>
</div>
<div class="stats-summary" id="statsSummary">
<span>总计: <strong id="totalGalleries">0</strong> 个画廊</span>
<span>待下载: <strong id="pendingGalleries">0</strong> 个</span>
<span>已完成: <strong id="completedGalleries">0</strong> 个</span>
</div>
<div class="controls">
<button class="btn btn-primary" onclick="loadGalleries()">
📁 读取文件夹
</button>
<button class="btn btn-success" onclick="startDownload()" id="downloadBtn">
⬇️ 开始下载
⬇️ 开始下载所有未完成
</button>
<button class="btn btn-warning" onclick="downloadSelected()" id="downloadSelectedBtn">
🎯 下载选中画廊
</button>
<button class="btn btn-danger" onclick="deleteJsonFiles()">
🗑️ 删除JSON文件
🗑️ 删除所有JSON文件
</button>
</div>
<div class="gallery-list" id="galleryList">
<div class="empty-state">
<h3>暂无画廊数据</h3>
<h3>暂无待下载任务</h3>
<p>点击"读取文件夹"按钮加载数据</p>
</div>
</div>
@ -328,6 +533,7 @@ async def read_gallery_manager():
<script>
let currentGalleries = [];
let selectedGalleries = new Set();
async function loadGalleries() {
try {
@ -335,6 +541,7 @@ async def read_gallery_manager():
const galleries = await response.json();
currentGalleries = galleries;
displayGalleries(galleries);
updateStats(galleries);
} catch (error) {
alert('读取文件夹失败: ' + error);
}
@ -347,35 +554,82 @@ async def read_gallery_manager():
galleryList.innerHTML = `
<div class="empty-state">
<h3>暂无画廊数据</h3>
<p>未找到任何画廊数据文件</p>
<p>请先添加画廊数据文件</p>
</div>
`;
return;
}
galleryList.innerHTML = galleries.map(gallery => `
<div class="gallery-item">
<div class="gallery-info">
<div class="gallery-title">${gallery.title}</div>
<div class="gallery-stats">
<span>总图片: ${gallery.total_images}</span>
<span>已下载: ${gallery.downloaded_images}</span>
<span>进度: ${Math.round((gallery.downloaded_images / gallery.total_images) * 100)}%</span>
</div>
<div class="progress-bar">
<div class="progress-fill" style="width: ${(gallery.downloaded_images / gallery.total_images) * 100}%"></div>
</div>
// 过滤掉已完成的画廊（已下载数量等于总数量）
const pendingGalleries = galleries.filter(gallery =>
gallery.downloaded_images < gallery.total_images
);
if (pendingGalleries.length === 0) {
galleryList.innerHTML = `
<div class="empty-state">
<h3>🎉 所有任务已完成！</h3>
<p>没有待下载的画廊任务</p>
</div>
<div class="gallery-actions">
<button class="btn btn-primary" onclick="downloadGallery('${gallery.title}')">
下载
</button>
<button class="btn btn-danger" onclick="deleteGallery('${gallery.title}')">
删除
</button>
`;
return;
}
galleryList.innerHTML = pendingGalleries.map(gallery => {
const progress = (gallery.downloaded_images / gallery.total_images) * 100;
const isCompleted = gallery.downloaded_images === gallery.total_images;
const isSelected = selectedGalleries.has(gallery.title);
return `
<div class="gallery-item ${isCompleted ? 'completed' : ''} ${isSelected ? 'selected' : ''}"
onclick="toggleGallerySelection('${gallery.title}')"
style="cursor: pointer; ${isSelected ? 'border-color: #007bff; background-color: #f8f9fa;' : ''}">
<div class="gallery-info">
<div class="gallery-title">
<input type="checkbox" ${isSelected ? 'checked' : ''}
onclick="event.stopPropagation(); toggleGallerySelection('${gallery.title}')">
${gallery.title}
${isCompleted ?
'<span class="status-badge status-completed">已完成</span>' :
'<span class="status-badge status-downloading">待下载</span>'
}
</div>
<div class="gallery-stats">
<span>总图片: ${gallery.total_images}</span>
<span>已下载: ${gallery.downloaded_images}</span>
<span>进度: ${Math.round(progress)}%</span>
</div>
<div class="progress-bar">
<div class="progress-fill" style="width: ${progress}%"></div>
</div>
<div class="gallery-actions">
<button class="btn btn-primary btn-sm" onclick="event.stopPropagation(); downloadSingleGallery('${gallery.title}')">
单独下载
</button>
</div>
</div>
</div>
</div>
`).join('');
`;
}).join('');
}
function toggleGallerySelection(title) {
if (selectedGalleries.has(title)) {
selectedGalleries.delete(title);
} else {
selectedGalleries.add(title);
}
displayGalleries(currentGalleries);
}
function updateStats(galleries) {
const total = galleries.length;
const completed = galleries.filter(g => g.downloaded_images === g.total_images).length;
const pending = total - completed;
document.getElementById('totalGalleries').textContent = total;
document.getElementById('pendingGalleries').textContent = pending;
document.getElementById('completedGalleries').textContent = completed;
}
async function startDownload() {
@ -384,23 +638,54 @@ async def read_gallery_manager():
btn.innerHTML = '⏳ 下载中...';
try {
// 这里可以添加批量下载逻辑
for (const gallery of currentGalleries) {
if (gallery.downloaded_images < gallery.total_images) {
await downloadGallery(gallery.title);
}
const response = await fetch('/api/download/all', {
method: 'POST'
});
const result = await response.json();
if (result.status === 'success') {
alert('批量下载任务已开始！请查看控制台了解进度。');
// 定期刷新状态
setTimeout(loadGalleries, 3000);
} else {
alert('下载失败: ' + result.message);
}
alert('所有下载任务已完成！');
} catch (error) {
alert('下载请求失败: ' + error);
} finally {
btn.disabled = false;
btn.innerHTML = '⬇️ 开始下载所有未完成';
}
}
async function downloadSelected() {
if (selectedGalleries.size === 0) {
alert('请先选择要下载的画廊！');
return;
}
const btn = document.getElementById('downloadSelectedBtn');
btn.disabled = true;
btn.innerHTML = '⏳ 下载中...';
try {
for (const title of selectedGalleries) {
await downloadSingleGallery(title);
// 添加延迟避免请求过于频繁
await new Promise(resolve => setTimeout(resolve, 1000));
}
alert('选中的画廊下载任务已完成！');
selectedGalleries.clear();
await loadGalleries();
} catch (error) {
alert('下载失败: ' + error);
} finally {
btn.disabled = false;
btn.innerHTML = '⬇️ 开始下载';
await loadGalleries(); // 刷新列表
btn.innerHTML = '🎯 下载选中画廊';
}
}
async function downloadGallery(title) {
async function downloadSingleGallery(title) {
try {
const response = await fetch(`/api/download/${encodeURIComponent(title)}`, {
method: 'POST'
@ -408,8 +693,10 @@ async def read_gallery_manager():
const result = await response.json();
if (result.status === 'success') {
console.log(`开始下载: ${title}`);
alert(`开始下载: ${title}`);
// 这里可以添加实时进度更新
// 刷新状态
setTimeout(loadGalleries, 2000);
} else {
alert(`下载失败: ${result.message}`);
}
@ -435,23 +722,6 @@ async def read_gallery_manager():
}
}
async function deleteGallery(title) {
if (!confirm(`确定要删除画廊"${title}"吗？此操作不可恢复！`)) {
return;
}
try {
const response = await fetch(`/api/galleries/${encodeURIComponent(title)}`, {
method: 'DELETE'
});
const result = await response.json();
alert(result.message);
await loadGalleries(); // 刷新列表
} catch (error) {
alert('删除失败: ' + error);
}
}
// 页面加载时自动读取
document.addEventListener('DOMContentLoaded', loadGalleries);
</script>
@ -461,16 +731,17 @@ async def read_gallery_manager():
@app.get("/api/galleries")
async def get_galleries():
"""获取所有画廊信息"""
"""获取所有画廊信息（包括已完成和未完成的）"""
galleries = get_all_galleries()
return galleries
@app.post("/api/download/{title}")
async def download_gallery(title: str):
async def download_gallery(title: str, background_tasks: BackgroundTasks):
"""开始下载指定画廊的图片"""
try:
# 这里实现图片下载逻辑
# 遍历 all_images 字典，下载每个图片
# 使用后台任务执行下载，避免阻塞请求
background_tasks.add_task(download_gallery_images, title)
return {
"status": "success",
"message": f"开始下载画廊: {title}",
@ -479,6 +750,26 @@ async def download_gallery(title: str):
except Exception as e:
raise HTTPException(status_code=500, detail=f"下载失败: {str(e)}")
@app.post("/api/download/all")
async def download_all_galleries(background_tasks: BackgroundTasks):
"""开始下载所有未完成的画廊"""
try:
# 使用后台任务执行批量下载
background_tasks.add_task(download_all_pending_galleries)
return {
"status": "success",
"message": "开始批量下载所有未完成的画廊"
}
except Exception as e:
raise HTTPException(status_code=500, detail=f"批量下载失败: {str(e)}")
@app.get("/api/download/status/{title}")
async def get_download_status(title: str):
"""获取指定画廊的下载状态"""
status = download_status.get(title, {})
return status
@app.delete("/api/cleanup")
async def cleanup_json_files():
"""删除所有JSON文件（保留图片）"""
@ -512,6 +803,8 @@ async def delete_gallery(title: str):
# 删除整个画廊目录
import shutil
shutil.rmtree(gallery_path)
# 清除下载状态
download_status.pop(title, None)
return {
"status": "success",
"message": f"已删除画廊: {title}"

9
post_eh_data.js
Unescape View File

@ -72,8 +72,7 @@
// 发送数据到后端的函数
function sendDataToBackend(data) {
console.log('准备发送的数据:', data);
console.log('数据类型:', typeof data);
console.log('字符串化后的数据:', JSON.stringify(data));
console.log('后端地址:', `http://${BACKEND_IP}:${BACKEND_PORT}/save_url`);
return new Promise((resolve, reject) => {
GM_xmlhttpRequest({
@ -86,6 +85,7 @@
onload: function(response) {
console.log('后端响应状态:', response.status);
console.log('后端响应内容:', response.responseText);
console.log('响应头:', response.responseHeaders);
if (response.status === 200) {
resolve(response);
} else {
@ -93,7 +93,12 @@
}
},
onerror: function(error) {
console.error('请求错误详情:', error);
reject(error);
},
ontimeout: function() {
console.error('请求超时');
reject(new Error('请求超时'));
}
});
});

Write Preview
Loading…
Cancel
Save