diff --git a/downloader.py b/downloader.py new file mode 100644 index 0000000..48932d2 --- /dev/null +++ b/downloader.py @@ -0,0 +1,73 @@ +from fastapi import APIRouter, BackgroundTasks +from pydantic import BaseModel +import uuid +import os +from pathlib import Path + +router = APIRouter(prefix="/api/v1", tags=["downloader"]) + +# 存储任务状态 +tasks = {} + +class CrawlRequest(BaseModel): + url: str + cookies: str + timestamp: str + +class TaskStatus(BaseModel): + status: str # 'running', 'completed', 'failed' + result: dict = None + error: str = None + +@router.post("/start-crawl") +async def start_crawl(request: CrawlRequest, background_tasks: BackgroundTasks): + task_id = str(uuid.uuid4()) + tasks[task_id] = {'status': 'running', 'result': None, 'error': None} + + # 在后台运行爬虫任务 + background_tasks.add_task(run_crawler, task_id, request) + + return {"task_id": task_id, "status": "started"} + +@router.get("/task-status/{task_id}") +async def get_task_status(task_id: str): + task = tasks.get(task_id) + if not task: + return {"status": "not_found"} + return task + +async def run_crawler(task_id: str, request: CrawlRequest): + try: + # 这里执行您的爬虫逻辑，模拟长时间运行 + # 例如：time.sleep(300) # 5分钟 + + # 确保 downloads 目录存在（双重保障） + downloads_dir = Path("downloads") + downloads_dir.mkdir(exist_ok=True) + + # 模拟下载文件到 downloads 目录 + filename = f"download_{task_id}.txt" + filepath = downloads_dir / filename + + with open(filepath, 'w', encoding='utf-8') as f: + f.write(f"URL: {request.url}\n") + f.write(f"Cookies: {request.cookies}\n") + f.write(f"Timestamp: {request.timestamp}\n") + f.write("Download completed successfully\n") + + # 爬虫完成后更新状态 + tasks[task_id] = { + 'status': 'completed', + 'result': { + 'message': '爬虫完成', + 'data': '您的爬虫结果', + 'download_path': str(filepath) + }, + 'error': None + } + except Exception as e: + tasks[task_id] = { + 'status': 'failed', + 'result': None, + 'error': str(e) + } diff --git a/main.py b/main.py new file mode 100644 index 0000000..acf47f5 --- /dev/null +++ b/main.py @@ -0,0 +1,43 @@ +from fastapi import FastAPI +from contextlib import asynccontextmanager +import uvicorn +from pathlib import Path +from downloader import router as downloader_router + +# 检查并创建 downloads 目录 +def ensure_downloads_dir(): + downloads_dir = Path("downloads") + downloads_dir.mkdir(exist_ok=True) + print(f"确保 downloads 目录存在: {downloads_dir.absolute()}") + +# lifespan 事件处理器 +@asynccontextmanager +async def lifespan(app: FastAPI): + # 启动时执行 + ensure_downloads_dir() + print("应用启动完成！") + yield + # 关闭时执行（可选） + print("应用正在关闭...") + +app = FastAPI( + title="下载器API", + description="一个基于FastAPI的异步下载器服务", + version="1.0.0", + lifespan=lifespan +) + +# 注册路由 +app.include_router(downloader_router) + +@app.get("/") +async def root(): + return {"message": "下载器服务运行中", "status": "healthy"} + +if __name__ == "__main__": + uvicorn.run( + "main:app", + host="0.0.0.0", + port=5100, + reload=True # 开发时自动重载 + ) diff --git a/post_eh_data.js b/post_eh_data.js new file mode 100644 index 0000000..0173e1a --- /dev/null +++ b/post_eh_data.js @@ -0,0 +1,200 @@ +// ==UserScript== +// @name 数据发送工具 +// @namespace http://tampermonkey.net/ +// @version 1.0 +// @description 向本地后端发送当前页面的URL和Cookies +// @author You +// @match *://*/* +// @grant GM_xmlhttpRequest +// @connect 127.0.0.1 +// @connect localhost +// ==/UserScript== + +(function() { + 'use strict'; + + // 配置：您可以修改这些变量来自定义行为 + const TARGET_SELECTOR = 'body'; // 按钮插入位置的选择器 + const BACKEND_IP = '127.0.0.1'; // 后端IP地址 + const BACKEND_PORT = '5100'; // 后端端口号 + + // 构建后端基础URL + const BACKEND_BASE_URL = `http://${BACKEND_IP}:${BACKEND_PORT}`; + + function addButton() { + if (document.getElementById('data-sender-button')) { + return; + } + + const button = document.createElement('button'); + button.id = 'data-sender-button'; + button.textContent = "send data"; + button.style.position = "fixed"; + button.style.top = "12.5%"; + button.style.right = "1%"; + button.style.transform = "translateY(-50%)"; + button.style.padding = "3px 8px"; + button.style.fontSize = "10px"; + button.style.backgroundColor = "#007baf"; + button.style.color = "#fff"; + button.style.border = "none"; + button.style.borderRadius = "5px"; + button.style.cursor = "pointer"; + button.style.zIndex = "10000"; + + button.addEventListener('click', function() { + sendDataToBackend(); + }); + + const targetElement = document.querySelector(TARGET_SELECTOR); + + if (targetElement && TARGET_SELECTOR !== 'body') { + const buttonContainer = document.createElement('div'); + buttonContainer.style.display = 'inline-block'; + buttonContainer.style.marginLeft = '10px'; + + button.style.position = 'relative'; + button.style.top = 'auto'; + button.style.right = 'auto'; + button.style.transform = 'none'; + button.style.margin = '0'; + + buttonContainer.appendChild(button); + + if (targetElement.nextSibling) { + targetElement.parentNode.insertBefore(buttonContainer, targetElement.nextSibling); + } else { + targetElement.parentNode.appendChild(buttonContainer); + } + } else { + document.body.appendChild(button); + } + } + + function sendDataToBackend() { + const currentUrl = window.location.href; + const cookies = document.cookie; + + const data = { + url: currentUrl, + cookies: cookies, + timestamp: new Date().toISOString() + }; + + // 禁用按钮防止重复点击 + const button = document.getElementById('data-sender-button'); + if (button) { + button.disabled = true; + button.textContent = "任务进行中..."; + button.style.backgroundColor = "#6c757d"; + } + + // 发送任务请求 + GM_xmlhttpRequest({ + method: "POST", + url: `${BACKEND_BASE_URL}/start-crawl`, + headers: { + "Content-Type": "application/json" + }, + data: JSON.stringify(data), + onload: function(response) { + if (response.status === 200) { + const result = JSON.parse(response.responseText); + if (result.task_id) { + alert("爬虫任务已启动！任务ID: " + result.task_id); + // 开始轮询任务状态 + pollTaskStatus(result.task_id); + } else { + alert("任务启动失败: " + (result.message || "未知错误")); + resetButton(); + } + } else { + alert("请求失败，状态码: " + response.status); + resetButton(); + } + }, + onerror: function(error) { + console.error("数据发送失败:", error); + alert("数据发送失败，请检查后端服务是否运行"); + resetButton(); + } + }); + } + + function pollTaskStatus(taskId) { + let pollCount = 0; + const maxPolls = 300; // 最多轮询300次（5分钟，每秒一次） + + const pollInterval = setInterval(() => { + pollCount++; + + GM_xmlhttpRequest({ + method: "GET", + url: `${BACKEND_BASE_URL}/task-status/${taskId}`, + onload: function(response) { + if (response.status === 200) { + const result = JSON.parse(response.responseText); + + // 更新按钮状态显示进度 + const button = document.getElementById('data-sender-button'); + if (button) { + button.textContent = `任务中...${pollCount}s`; + } + + if (result.status === 'completed') { + clearInterval(pollInterval); + alert("爬虫任务完成！\n结果: " + JSON.stringify(result.result, null, 2)); + resetButton(); + } else if (result.status === 'failed') { + clearInterval(pollInterval); + alert("爬虫任务失败: " + result.error); + resetButton(); + } + // 如果状态是 'running'，继续轮询 + } else { + console.error("获取任务状态失败:", response.status); + } + }, + onerror: function(error) { + console.error("轮询任务状态失败:", error); + } + }); + + // 超过最大轮询次数，停止轮询 + if (pollCount >= maxPolls) { + clearInterval(pollInterval); + alert("任务超时，请稍后手动检查结果"); + resetButton(); + } + }, 1000); // 每秒轮询一次 + } + + function resetButton() { + const button = document.getElementById('data-sender-button'); + if (button) { + button.disabled = false; + button.textContent = "send data"; + button.style.backgroundColor = "#007baf"; + } + } + + // 初始尝试添加按钮 + addButton(); + + // 使用MutationObserver监听DOM变化 + const observer = new MutationObserver(function(mutations) { + addButton(); + }); + + observer.observe(document.body, { + childList: true, + subtree: true + }); + + if (document.readyState === 'loading') { + document.addEventListener('DOMContentLoaded', addButton); + } else { + addButton(); + } + +})(); \ No newline at end of file