diff --git a/main.py b/main.py
index 1eb6087..eda065b 100644
--- a/main.py
+++ b/main.py
@@ -26,8 +26,8 @@ logger = logging.getLogger(__name__)
DOWNLOADS_DIR = "downloads"
MAX_FILENAME_LENGTH = 100
INVALID_FILENAME_CHARS = '<>:"/\\|?*'
-MAX_CONCURRENT_DOWNLOADS = 5 # 最大并发下载数
-DOWNLOAD_TIMEOUT = 30 # 下载超时时间（秒）
+MAX_CONCURRENT_DOWNLOADS = 5
+DOWNLOAD_TIMEOUT = 30
# FastAPI应用
app = FastAPI(title="eh-v2")
@@ -57,39 +57,29 @@ class DownloadStatusResponse(BaseModel):
# 工具函数
def setup_downloads_directory() -> Path:
- """创建并返回下载目录路径"""
downloads_path = Path(DOWNLOADS_DIR)
downloads_path.mkdir(exist_ok=True)
- logger.info(f"下载目录已准备: {downloads_path.absolute()}")
return downloads_path
def sanitize_filename(filename: str) -> str:
- """清理文件名，移除非法字符并限制长度"""
sanitized = filename
for char in INVALID_FILENAME_CHARS:
sanitized = sanitized.replace(char, '_')
-
- # 限制文件名长度
if len(sanitized) > MAX_FILENAME_LENGTH:
sanitized = sanitized[:MAX_FILENAME_LENGTH]
-
return sanitized
def create_title_directory(base_path: Path, title: str) -> Path:
- """创建标题对应的目录"""
safe_title = sanitize_filename(title)
title_dir = base_path / safe_title
title_dir.mkdir(exist_ok=True)
- logger.info(f"创建标题目录: {title_dir}")
return title_dir
async def save_data_to_file(file_path: Path, data: Dict[str, Any]) -> None:
- """异步保存数据到JSON文件"""
async with aiofiles.open(file_path, 'w', encoding='utf-8') as f:
await f.write(json.dumps(data, ensure_ascii=False, indent=2))
def get_all_galleries() -> List[GalleryInfo]:
- """获取所有画廊信息"""
galleries = []
downloads_path = Path(DOWNLOADS_DIR)
@@ -104,7 +94,6 @@ def get_all_galleries() -> List[GalleryInfo]:
with open(data_file, 'r', encoding='utf-8') as f:
data = json.load(f)
- # 计算已下载的图片数量
downloaded_count = 0
if 'all_images' in data:
for filename, url in data['all_images'].items():
@@ -124,38 +113,32 @@ def get_all_galleries() -> List[GalleryInfo]:
return galleries
async def download_single_image(client: httpx.AsyncClient, url: str, file_path: Path, semaphore: asyncio.Semaphore) -> bool:
- """下载单张图片 - 精简版"""
async with semaphore:
try:
- logger.info(f"开始下载: {url}")
-
- if file_path.exists():
- logger.info(f"文件已存在: {file_path}")
- return True
-
- # 第一步：获取中间页面
+ # 先获取图片后缀
response = await client.get(url, timeout=DOWNLOAD_TIMEOUT)
response.raise_for_status()
- # 第二步：提取真实图片URL
import re
match = re.search(r'img id="img" src="(.*?)"', response.text)
if not match:
- logger.error(f"无法提取图片URL: {url}")
return False
real_img_url = match.group(1)
- logger.info(f"真实URL: {real_img_url}")
+ suffix = real_img_url.split('.')[-1]
+
+ # 创建带后缀的文件路径
+ file_path_with_suffix = file_path.with_suffix('.' + suffix)
+
+ if file_path_with_suffix.exists():
+ return True
- # 第三步：下载图片
img_response = await client.get(real_img_url, timeout=DOWNLOAD_TIMEOUT)
img_response.raise_for_status()
- # 保存图片
- async with aiofiles.open(file_path, 'wb') as f:
+ async with aiofiles.open(file_path_with_suffix, 'wb') as f:
await f.write(img_response.content)
- logger.info(f"下载完成: {file_path}")
return True
except Exception as e:
@@ -163,7 +146,6 @@ async def download_single_image(client: httpx.AsyncClient, url: str, file_path:
return False
async def download_gallery_images(title: str) -> DownloadStatusResponse:
- """下载指定画廊的所有图片"""
safe_title = sanitize_filename(title)
gallery_path = downloads_path / safe_title
data_file = gallery_path / "data.json"
@@ -178,7 +160,6 @@ async def download_gallery_images(title: str) -> DownloadStatusResponse:
)
try:
- # 读取画廊数据
async with aiofiles.open(data_file, 'r', encoding='utf-8') as f:
content = await f.read()
data = json.loads(content)
@@ -195,7 +176,6 @@ async def download_gallery_images(title: str) -> DownloadStatusResponse:
current_progress=0.0
)
- # 初始化下载状态
download_status[title] = {
"downloaded": 0,
"total": total_images,
@@ -204,10 +184,8 @@ async def download_gallery_images(title: str) -> DownloadStatusResponse:
logger.info(f"开始下载画廊 '{title}'，共 {total_images} 张图片")
- # 创建信号量限制并发数
semaphore = asyncio.Semaphore(MAX_CONCURRENT_DOWNLOADS)
- # 使用异步HTTP客户端
async with httpx.AsyncClient(
headers={
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
@@ -215,12 +193,10 @@ async def download_gallery_images(title: str) -> DownloadStatusResponse:
follow_redirects=True
) as client:
- # 准备下载任务
tasks = []
for filename, url in all_images.items():
image_path = gallery_path / filename
- # 如果图片已存在，跳过下载但计入完成数量
if image_path.exists():
download_status[title]["downloaded"] += 1
continue
@@ -228,26 +204,20 @@ async def download_gallery_images(title: str) -> DownloadStatusResponse:
task = download_single_image(client, url, image_path, semaphore)
tasks.append(task)
- # 批量执行下载任务
if tasks:
results = await asyncio.gather(*tasks, return_exceptions=True)
-
- # 统计成功下载的数量
successful_downloads = sum(1 for result in results if result is True)
download_status[title]["downloaded"] += successful_downloads
- # 更新最终状态
downloaded_count = download_status[title]["downloaded"]
progress = (downloaded_count / total_images) * 100
if downloaded_count == total_images:
download_status[title]["status"] = "completed"
message = f"下载完成！共下载 {downloaded_count}/{total_images} 张图片"
- logger.info(f"画廊 '{title}' {message}")
else:
download_status[title]["status"] = "partial"
message = f"部分完成！下载 {downloaded_count}/{total_images} 张图片"
- logger.warning(f"画廊 '{title}' {message}")
return DownloadStatusResponse(
status="success",
@@ -272,28 +242,25 @@ async def download_gallery_images(title: str) -> DownloadStatusResponse:
)
async def download_all_pending_galleries():
- """下载所有未完成的画廊"""
galleries = get_all_galleries()
pending_galleries = [g for g in galleries if g.downloaded_images < g.total_images]
+ logger.info(f"找到 {len(pending_galleries)} 个待下载画廊")
+
if not pending_galleries:
logger.info("没有待下载的画廊")
return
- logger.info(f"开始批量下载 {len(pending_galleries)} 个画廊")
-
for gallery in pending_galleries:
- if gallery.downloaded_images < gallery.total_images:
- logger.info(f"开始下载画廊: {gallery.title}")
- result = await download_gallery_images(gallery.title)
-
- if result.status == "success":
- logger.info(f"画廊 '{gallery.title}' 下载完成: {result.message}")
- else:
- logger.error(f"画廊 '{gallery.title}' 下载失败: {result.message}")
-
- # 添加延迟避免请求过于频繁
- await asyncio.sleep(1)
+ logger.info(f"开始下载画廊: {gallery.title}")
+ result = await download_gallery_images(gallery.title)
+
+ if result.status == "success":
+ logger.info(f"画廊 '{gallery.title}' 下载完成: {result.message}")
+ else:
+ logger.error(f"画廊 '{gallery.title}' 下载失败: {result.message}")
+
+ await asyncio.sleep(1)
logger.info("批量下载任务完成")
@@ -301,9 +268,35 @@ async def download_all_pending_galleries():
downloads_path = setup_downloads_directory()
# API路由
+@app.post("/save_url")
+@app.options("/save_url")
+async def save_url_data(request: SaveDataRequest = None):
+ if not request:
+ return {"status": "ok"}
+
+ try:
+ title_dir = create_title_directory(downloads_path, request.title)
+ data_file = title_dir / "data.json"
+ await save_data_to_file(data_file, {
+ "url": request.url,
+ "title": request.title,
+ "all_images": request.all_images,
+ "total_images": request.total_images
+ })
+
+ logger.info(f"成功保存数据: {request.title}")
+ return {
+ "status": "success",
+ "message": f"数据保存成功，共 {request.total_images} 张图片",
+ "path": str(title_dir)
+ }
+
+ except Exception as e:
+ logger.error(f"保存数据失败: {e}")
+ raise HTTPException(status_code=500, detail=f"保存失败: {str(e)}")
+
@app.get("/", response_class=HTMLResponse)
async def read_gallery_manager():
- """画廊管理页面"""
return """
@@ -312,193 +305,24 @@ async def read_gallery_manager():
画廊下载管理器
@@ -508,25 +332,9 @@ async def read_gallery_manager():
管理您的画廊下载任务
-
- 总计: 0 个画廊
- 待下载: 0 个
- 已完成: 0 个
-
-
-
-
-
-
+
+
@@ -538,16 +346,11 @@ async def read_gallery_manager():
@@ -737,92 +420,28 @@ async def read_gallery_manager():
@app.get("/api/galleries")
async def get_galleries():
- """获取所有画廊信息（包括已完成和未完成的）"""
galleries = get_all_galleries()
return galleries
-@app.post("/api/download/{title}")
-async def download_gallery(title: str, background_tasks: BackgroundTasks):
- """开始下载指定画廊的图片"""
- try:
- # 使用后台任务执行下载，避免阻塞请求
- background_tasks.add_task(download_gallery_images, title)
-
- return {
- "status": "success",
- "message": f"开始下载画廊: {title}",
- "title": title
- }
- except Exception as e:
- raise HTTPException(status_code=500, detail=f"下载失败: {str(e)}")
-
@app.post("/api/download/all")
async def download_all_galleries(background_tasks: BackgroundTasks):
- """开始下载所有未完成的画廊"""
- try:
- # 使用后台任务执行批量下载
- background_tasks.add_task(download_all_pending_galleries)
-
- return {
- "status": "success",
- "message": "开始批量下载所有未完成的画廊"
- }
- except Exception as e:
- raise HTTPException(status_code=500, detail=f"批量下载失败: {str(e)}")
-
-@app.get("/api/download/status/{title}")
-async def get_download_status(title: str):
- """获取指定画廊的下载状态"""
- status = download_status.get(title, {})
- return status
+ background_tasks.add_task(download_all_pending_galleries)
+ return {
+ "status": "success",
+ "message": "开始批量下载所有未完成的画廊"
+ }
-@app.delete("/api/cleanup")
-async def cleanup_json_files():
- """删除所有JSON文件（保留图片）"""
- try:
- deleted_count = 0
- downloads_path = Path(DOWNLOADS_DIR)
-
- for gallery_dir in downloads_path.iterdir():
- if gallery_dir.is_dir():
- data_file = gallery_dir / "data.json"
- if data_file.exists():
- data_file.unlink()
- deleted_count += 1
-
- return {
- "status": "success",
- "message": f"已删除 {deleted_count} 个JSON文件",
- "deleted_count": deleted_count
- }
- except Exception as e:
- raise HTTPException(status_code=500, detail=f"清理失败: {str(e)}")
-
-@app.delete("/api/galleries/{title}")
-async def delete_gallery(title: str):
- """删除指定画廊的所有文件"""
- try:
- safe_title = sanitize_filename(title)
- gallery_path = downloads_path / safe_title
-
- if gallery_path.exists():
- # 删除整个画廊目录
- import shutil
- shutil.rmtree(gallery_path)
- # 清除下载状态
- download_status.pop(title, None)
- return {
- "status": "success",
- "message": f"已删除画廊: {title}"
- }
- else:
- raise HTTPException(status_code=404, detail="画廊不存在")
- except Exception as e:
- raise HTTPException(status_code=500, detail=f"删除失败: {str(e)}")
+@app.post("/api/download/{title}")
+async def download_gallery(title: str, background_tasks: BackgroundTasks):
+ background_tasks.add_task(download_gallery_images, title)
+ return {
+ "status": "success",
+ "message": f"开始下载画廊: {title}",
+ "title": title
+ }
@app.get("/health")
async def health_check():
- """健康检查端点"""
return {"status": "healthy"}
if __name__ == "__main__":
diff --git a/post_eh_data.js b/post_eh_data.js
index de5d150..f3bb539 100644
--- a/post_eh_data.js
+++ b/post_eh_data.js
@@ -14,7 +14,7 @@
// 全局配置 - 请根据实际情况修改这些值
const BACKEND_IP = '127.0.0.1';
const BACKEND_PORT = '5100';
- const BUTTON_LOCATION_SELECTOR = 'body';
+ const BUTTON_LOCATION_SELECTOR = '#gd5 > p:nth-child(5)';
const DATA_LIST_SELECTOR = '#gdt a'; // 修改为a标签的选择器
const ALL_IMG_DATA = {}; // 用于储存每一页的图片url, 格式为 {"0001": "https://example001.jpg", "0002": "https://example002.jpg"}, 最高支持4位数至9999
@@ -72,8 +72,9 @@
// 发送数据到后端的函数
function sendDataToBackend(data) {
console.log('准备发送的数据:', data);
- console.log('后端地址:', `http://${BACKEND_IP}:${BACKEND_PORT}/save_url`);
-
+ console.log('数据类型:', typeof data);
+ console.log('字符串化后的数据:', JSON.stringify(data));
+
return new Promise((resolve, reject) => {
GM_xmlhttpRequest({
method: "POST",
@@ -85,7 +86,6 @@
onload: function(response) {
console.log('后端响应状态:', response.status);
console.log('后端响应内容:', response.responseText);
- console.log('响应头:', response.responseHeaders);
if (response.status === 200) {
resolve(response);
} else {
@@ -93,12 +93,7 @@
}
},
onerror: function(error) {
- console.error('请求错误详情:', error);
reject(error);
- },
- ontimeout: function() {
- console.error('请求超时');
- reject(new Error('请求超时'));
}
});
});
@@ -163,7 +158,7 @@
// 从响应中提取图片链接
const pageImages = extractImagesFromPage(response.responseText);
-
+
if (pageImages.length === 0) {
console.log(`第${page}页没有图片，可能是最后一页`);
return false; // 没有图片，可能是最后一页
@@ -182,7 +177,7 @@
});
console.log(`第${page}页采集完成，获取到${pageImages.length}个图片链接，新增${hasNewImage ? '有新图片' : '全是重复图片'}`);
-
+
return hasNewImage; // 返回是否有新图片
} catch (error) {
@@ -195,15 +190,15 @@
let shouldContinue = true;
for (let page = 0; page <= 100; page++) {
if (!shouldContinue) break;
-
+
const hasNewImages = await processPage(page);
-
+
// 如果没有新图片，说明可能是最后一页了
if (!hasNewImages && page > 0) {
console.log(`第${page}页没有新图片，可能已到最后一页，停止采集`);
shouldContinue = false;
}
-
+
// 如果图片数量达到上限也停止
if (img_count > 2200) {
console.log('图片数量达到上限2200，停止采集');
@@ -222,7 +217,7 @@
// 显示结果并发送到后端
console.log('采集完成的所有数据:', data);
console.log('后端地址:', BACKEND_IP + ':' + BACKEND_PORT);
-
+
try {
await sendDataToBackend(data);
alert(`数据采集完成并已保存到后端！\n标题: ${pageTitle}\n总图片链接数量: ${Object.keys(ALL_IMG_DATA).length}`);