spider/ImagesDownloader
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

++

Browse Source
This commit is contained in:
jack
2025-12-17 08:53:20 +08:00
parent 990963cd70
commit 6ada6d0efb
16 changed files with 697 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+5 -2
View File
@@ -54,8 +54,7 @@ coverage.xml
# Django stuff:
*.log
# Sphinx documentation
docs/_build/
build
# PyBuilder
target/
@@ -63,3 +62,7 @@ target/
other/split_clash_config/split_config
ai_news/save_data
daily/*.txt
*.exe
*.db
vendor/
*.out
api/hd4k-downloader.api
-48
View File
@@ -1,48 +0,0 @@
syntax = "v1"
info(
title: "HD4K漫画下载服务"
desc: "下载HD4K漫画图片的API服务"
author: "hd4k-downloader"
version: "v1.0.0"
)
type (
// 下载请求
DownloadRequest {
Title string `json:"title"`
Imgs map[string]string `json:"imgs"`
}
// 下载进度详情
ProgressDetail {
Key string `json:"key"`
URL string `json:"url"`
Status string `json:"status"`
Message string `json:"message"`
SavedAs string `json:"saved_as,optional"`
}
// 下载响应
DownloadResponse {
Success bool `json:"success"`
Message string `json:"message"`
Title string `json:"title"`
Folder string `json:"folder"`
JsonPath string `json:"json_path"`
Total int `json:"total"`
Saved int `json:"saved"`
Skipped int `json:"skipped"`
Failed int `json:"failed"`
Details []ProgressDetail `json:"details"`
}
)
@server(
middleware: Cors
timeout: 300000ms # 添加时间单位ms
)
service hd4k_downloader {
@handler SaveImages
post /api/save_imgs (DownloadRequest) returns (DownloadResponse)
}
build.sh
+106
View File
@@ -0,0 +1,106 @@
#!/bin/bash
# HD4K下载器 - 跨平台构建脚本
# 构建 Windows 和 macOS 版本到 build 文件夹
echo "========================================"
echo "HD4K下载器 - 跨平台构建"
echo "========================================"
# 设置变量
APP_NAME="hd4k-downloader"
VERSION="1.0.0"
BUILD_DIR="./build"
echo "创建构建目录..."
# 创建build目录（如果不存在）
mkdir -p "${BUILD_DIR}"
echo "清理旧文件..."
# 清理build目录中的旧文件
rm -f "${BUILD_DIR}/${APP_NAME}-windows.exe"
rm -f "${BUILD_DIR}/${APP_NAME}-macos"
rm -f "${BUILD_DIR}/${APP_NAME}-macos-arm64"
echo ""
echo "开始构建 Windows 版本 (amd64)..."
echo "----------------------------------------"
GOOS=windows GOARCH=amd64 go build -o "${BUILD_DIR}/${APP_NAME}-windows.exe" main.go handler.go downloader.go
if [ $? -eq 0 ]; then
echo "✅ Windows 版本构建成功"
echo " 文件: ${BUILD_DIR}/${APP_NAME}-windows.exe"
echo " 大小:" $(ls -lh "${BUILD_DIR}/${APP_NAME}-windows.exe" | awk '{print $5}')
else
echo "❌ Windows 版本构建失败"
fi
echo ""
echo "开始构建 macOS 版本 (Intel amd64)..."
echo "----------------------------------------"
GOOS=darwin GOARCH=amd64 go build -o "${BUILD_DIR}/${APP_NAME}-macos-intel" main.go handler.go downloader.go
if [ $? -eq 0 ]; then
echo "✅ macOS Intel 版本构建成功"
echo " 文件: ${BUILD_DIR}/${APP_NAME}-macos-intel"
echo " 大小:" $(ls -lh "${BUILD_DIR}/${APP_NAME}-macos-intel" | awk '{print $5}')
chmod +x "${BUILD_DIR}/${APP_NAME}-macos-intel"
else
echo "❌ macOS Intel 版本构建失败"
fi
echo ""
echo "开始构建 macOS 版本 (Apple Silicon arm64)..."
echo "----------------------------------------"
GOOS=darwin GOARCH=arm64 go build -o "${BUILD_DIR}/${APP_NAME}-macos-arm" main.go handler.go downloader.go
if [ $? -eq 0 ]; then
echo "✅ macOS Apple Silicon 版本构建成功"
echo " 文件: ${BUILD_DIR}/${APP_NAME}-macos-arm"
echo " 大小:" $(ls -lh "${BUILD_DIR}/${APP_NAME}-macos-arm" | awk '{print $5}')
chmod +x "${BUILD_DIR}/${APP_NAME}-macos-arm"
else
echo "❌ macOS Apple Silicon 版本构建失败"
fi
echo ""
echo "========================================"
echo "构建完成！"
echo "========================================"
echo ""
echo "生成的文件在 build/ 目录："
echo "----------------------------------------"
ls -lh "${BUILD_DIR}/" | grep -v "^total"
echo ""
echo "使用方法："
echo "----------------------------------------"
echo ""
echo "Windows:"
echo " 双击 ${APP_NAME}-windows.exe"
echo " 或命令行: ${BUILD_DIR}/${APP_NAME}-windows.exe -port=8888 -dir=./downloads"
echo ""
echo "macOS (Intel芯片):"
echo " 终端执行: ${BUILD_DIR}/${APP_NAME}-macos-intel -port=8888 -dir=./downloads"
echo ""
echo "macOS (Apple Silicon M1/M2/M3芯片):"
echo " 终端执行: ${BUILD_DIR}/${APP_NAME}-macos-arm -port=8888 -dir=./downloads"
echo ""
echo "常用参数："
echo " -port=端口号 指定服务端口（默认: 8888）"
echo " -dir=目录路径 指定下载目录（默认: ./downloads）"
echo ""
echo "示例："
echo " # 使用自定义端口"
echo " ${BUILD_DIR}/${APP_NAME}-macos-intel -port=9999"
echo ""
echo " # 使用自定义下载目录"
echo " ${BUILD_DIR}/${APP_NAME}-windows.exe -dir=D:\\下载"
echo ""
echo " # 完整示例"
echo " ${BUILD_DIR}/${APP_NAME}-macos-arm -port=9000 -dir=~/Downloads/"
echo ""
echo "服务启动后访问："
echo " API接口: http://127.0.0.1:端口号/api/save_imgs"
echo " 测试页面: http://127.0.0.1:端口号/index.html"
echo "========================================"
downloader.go
+104
View File
@@ -0,0 +1,104 @@
package main
import (
"crypto/tls"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"sync"
"time"
)
func downloadImages(images map[string]string, dir string) (map[string]bool, map[string]error) {
var mu sync.Mutex
var wg sync.WaitGroup
success := make(map[string]bool)
errors := make(map[string]error)
// 并发控制：最多同时下载5张图片
semaphore := make(chan struct{}, 5)
for key, url := range images {
wg.Add(1)
go func(key, url string) {
defer wg.Done()
// 获取信号量
semaphore <- struct{}{}
defer func() { <-semaphore }()
// 下载图片
err := downloadImage(url, dir, key)
mu.Lock()
if err != nil {
errors[key] = err
} else {
success[key] = true
}
mu.Unlock()
}(key, url)
}
wg.Wait()
return success, errors
}
func downloadImage(url, dir, filename string) error {
// 创建HTTP客户端
client := &http.Client{
Timeout: 30 * time.Second,
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
},
}
// 创建请求
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return fmt.Errorf("创建请求失败: %v", err)
}
// 设置请求头（模拟浏览器）
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36")
req.Header.Set("Accept", "image/webp,image/apng,image/*,*/*;q=0.8")
req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8")
req.Header.Set("Referer", "https://hd4k.com/")
// 发送请求
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("请求失败: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("HTTP错误: %s", resp.Status)
}
// 获取文件扩展名
ext := getFileExtension(url)
fullFilename := filename + ext
filePath := filepath.Join(dir, fullFilename)
// 创建文件
file, err := os.Create(filePath)
if err != nil {
return fmt.Errorf("创建文件失败: %v", err)
}
defer file.Close()
// 下载并保存
_, err = io.Copy(file, resp.Body)
if err != nil {
// 删除可能已损坏的文件
os.Remove(filePath)
return fmt.Errorf("保存文件失败: %v", err)
}
return nil
}
etc/hd4kdownloader.yaml
-3
View File
@@ -1,3 +0,0 @@
Name: hd4k_downloader
Host: 0.0.0.0
Port: 8888
go.mod
+1 -3
View File
@@ -1,5 +1,3 @@
module hd4k-downloader
go 1.24.7
require github.com/zeromicro/go-zero v1.9.3 // indirect
go 1.25.1
go.sum
-2
View File
@@ -1,2 +0,0 @@
github.com/zeromicro/go-zero v1.9.3 h1:dJ568uUoRJY0RUxo4aH4htSglbEUF60WiM1MZVkTK9A=
github.com/zeromicro/go-zero v1.9.3/go.mod h1:JBAtfXQvErk+V7pxzcySR0mW6m2I4KPhNQZGASltDRQ=
handler.go
+237
View File
@@ -0,0 +1,237 @@
package main
import (
"encoding/json"
"fmt"
"net/http"
"os"
"path/filepath"
"strings"
"time"
)
// 请求结构
type SaveImagesRequest struct {
Title string `json:"title"`
Imgs map[string]string `json:"imgs"`
}
// 下载详情
type DownloadDetail struct {
Key string `json:"key"`
URL string `json:"url"`
Status string `json:"status"` // success, skipped, failed
Message string `json:"message"`
SavedAs string `json:"saved_as,omitempty"`
}
// 响应结构
type SaveImagesResponse struct {
Success bool `json:"success"`
Message string `json:"message"`
Title string `json:"title"`
Folder string `json:"folder"`
JsonPath string `json:"json_path"`
Total int `json:"total"`
Saved int `json:"saved"`
Skipped int `json:"skipped"`
Failed int `json:"failed"`
Details []DownloadDetail `json:"details"`
}
func saveImagesHandler(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(w, "只支持POST请求", http.StatusMethodNotAllowed)
return
}
// 解析请求
var req SaveImagesRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
http.Error(w, "请求格式错误: "+err.Error(), http.StatusBadRequest)
return
}
// 验证参数
if req.Title == "" {
http.Error(w, "标题不能为空", http.StatusBadRequest)
return
}
if len(req.Imgs) == 0 {
http.Error(w, "图片列表不能为空", http.StatusBadRequest)
return
}
// 处理下载
resp, err := processDownload(&req)
if err != nil {
http.Error(w, "处理失败: "+err.Error(), http.StatusInternalServerError)
return
}
// 返回响应
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(resp)
}
func processDownload(req *SaveImagesRequest) (*SaveImagesResponse, error) {
// 清理标题（移除非法字符）
cleanTitle := cleanFilename(req.Title)
// 创建目录
comicDir := filepath.Join(*downloadDir, cleanTitle)
if err := os.MkdirAll(comicDir, 0755); err != nil {
return nil, fmt.Errorf("创建目录失败: %v", err)
}
// JSON文件路径
jsonFilename := cleanTitle + ".json"
jsonPath := filepath.Join(comicDir, jsonFilename)
// 检查现有文件
existingFiles := make(map[string]bool)
files, _ := os.ReadDir(comicDir)
for _, file := range files {
if !file.IsDir() {
filename := file.Name()
if strings.HasSuffix(filename, ".json") {
continue
}
// 提取序号（去掉扩展名）
ext := filepath.Ext(filename)
key := strings.TrimSuffix(filename, ext)
existingFiles[key] = true
}
}
// 准备下载
details := make([]DownloadDetail, 0, len(req.Imgs))
needDownload := make(map[string]string)
for key, url := range req.Imgs {
// 获取文件扩展名
ext := getFileExtension(url)
filename := key + ext
if existingFiles[key] {
// 文件已存在，跳过
details = append(details, DownloadDetail{
Key: key,
URL: url,
Status: "skipped",
Message: "文件已存在",
SavedAs: filename,
})
continue
}
// 需要下载
needDownload[key] = url
details = append(details, DownloadDetail{
Key: key,
URL: url,
Status: "pending",
Message: "等待下载",
})
}
// 下载图片
var saved, failed int
if len(needDownload) > 0 {
successMap, errors := downloadImages(needDownload, comicDir)
// 使用 successMap 来检查哪些下载成功
// 虽然我们不直接使用 successMap，但它用于 downloadImages 函数的返回值
_ = successMap // 明确标记为使用（避免未使用警告）
// 更新下载详情
for i, detail := range details {
if detail.Status == "pending" {
if err, ok := errors[detail.Key]; ok {
details[i].Status = "failed"
details[i].Message = err.Error()
failed++
} else {
ext := getFileExtension(detail.URL)
filename := detail.Key + ext
details[i].Status = "success"
details[i].Message = "下载成功"
details[i].SavedAs = filename
saved++
}
}
}
}
// 保存JSON文件
skipped := len(req.Imgs) - saved - failed
finalImgs := make(map[string]string)
for key, url := range req.Imgs {
finalImgs[key] = url
}
saveJSON(comicDir, jsonFilename, req.Title, finalImgs)
// 构建响应
return &SaveImagesResponse{
Success: failed == 0,
Message: fmt.Sprintf("下载完成: 新增%d张, 跳过%d张, 失败%d张", saved, skipped, failed),
Title: req.Title,
Folder: comicDir,
JsonPath: jsonPath,
Total: len(req.Imgs),
Saved: saved,
Skipped: skipped,
Failed: failed,
Details: details,
}, nil
}
func saveJSON(dir, filename, title string, imgs map[string]string) error {
data := map[string]interface{}{
"title": title,
"imgs": imgs,
"created_at": time.Now().Format("2006-01-02 15:04:05"),
"updated_at": time.Now().Format("2006-01-02 15:04:05"),
}
jsonData, err := json.MarshalIndent(data, "", " ")
if err != nil {
return err
}
path := filepath.Join(dir, filename)
return os.WriteFile(path, jsonData, 0644)
}
func cleanFilename(filename string) string {
illegalChars := []string{"/", "\\", ":", "*", "?", "\"", "<", ">", "|", "\n", "\r", "\t"}
result := filename
for _, char := range illegalChars {
result = strings.ReplaceAll(result, char, "_")
}
if len(result) > 200 {
result = result[:200]
}
return strings.TrimSpace(result)
}
func getFileExtension(url string) string {
// 默认扩展名
ext := ".jpg"
// 从URL提取扩展名
if idx := strings.LastIndex(url, "."); idx != -1 {
fileExt := strings.ToLower(url[idx:])
// 检查是否为常见图片格式
validExts := []string{".jpg", ".jpeg", ".png", ".webp", ".gif", ".bmp", ".tiff"}
for _, validExt := range validExts {
if strings.HasPrefix(fileExt, validExt) {
ext = validExt
break
}
}
}
return ext
}
hd4kdownloader.go
-34
View File
@@ -1,34 +0,0 @@
// Code scaffolded by goctl. Safe to edit.
// goctl 1.9.2
package main
import (
"flag"
"fmt"
"hd4k-downloader/internal/config"
"hd4k-downloader/internal/handler"
"hd4k-downloader/internal/svc"
"github.com/zeromicro/go-zero/core/conf"
"github.com/zeromicro/go-zero/rest"
)
var configFile = flag.String("f", "etc/hd4kdownloader.yaml", "the config file")
func main() {
flag.Parse()
var c config.Config
conf.MustLoad(*configFile, &c)
server := rest.MustNewServer(c.RestConf)
defer server.Stop()
ctx := svc.NewServiceContext(c)
handler.RegisterHandlers(server, ctx)
fmt.Printf("Starting server at %s:%d...\n", c.Host, c.Port)
server.Start()
}
hdk4_downloader.js
View File
index.html
+181
View File
@@ -0,0 +1,181 @@
<!DOCTYPE html>
<html>
<head>
<title>HD4K下载测试</title>
<style>
body {
font-family: Arial, sans-serif;
max-width: 800px;
margin: 0 auto;
padding: 20px;
}
.form-group {
margin-bottom: 15px;
}
label {
display: block;
margin-bottom: 5px;
font-weight: bold;
}
input, textarea {
width: 100%;
padding: 8px;
border: 1px solid #ddd;
border-radius: 4px;
}
textarea {
height: 200px;
font-family: monospace;
}
button {
background-color: #007bff;
color: white;
border: none;
padding: 10px 20px;
border-radius: 4px;
cursor: pointer;
font-size: 16px;
}
button:hover {
background-color: #0056b3;
}
.result {
margin-top: 20px;
padding: 15px;
border-radius: 4px;
background-color: #f8f9fa;
}
.success {
color: #28a745;
}
.error {
color: #dc3545;
}
.status-item {
margin: 5px 0;
padding: 5px;
border-left: 3px solid #ddd;
}
.status-success {
border-left-color: #28a745;
}
.status-skipped {
border-left-color: #ffc107;
}
.status-failed {
border-left-color: #dc3545;
}
</style>
</head>
<body>
<h1>HD4K下载测试</h1>
<div class="form-group">
<label for="title">标题:</label>
<input type="text" id="title" value="测试">
</div>
<div class="form-group">
<label for="jsonData">JSON数据:</label>
<textarea id="jsonData">{
"title": "测试",
"imgs": {
"0001": "https://i.imgur.com/3dV1KnX1.jpg",
"0002": "https://i.imgur.com/3dV1KnX2.jpg",
"0003": "https://i.imgur.com/3dV1KnX3.jpg",
"0004": "https://i.imgur.com/3dV1KnX4.jpg"
}
}</textarea>
</div>
<button onclick="downloadComic()">开始下载</button>
<div id="result" class="result" style="display: none;"></div>
<script>
async function downloadComic() {
const title = document.getElementById('title').value;
const jsonData = document.getElementById('jsonData').value;
let imgs;
try {
imgs = JSON.parse(jsonData);
} catch (e) {
showResult('error', 'JSON格式错误: ' + e.message);
return;
}
const data = {
title: title,
imgs: imgs.imgs || imgs
};
try {
const response = await fetch('http://127.0.0.1:8888/api/save_imgs', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify(data)
});
const result = await response.json();
showResult('success', result);
} catch (error) {
showResult('error', '请求失败: ' + error.message);
}
}
function showResult(type, result) {
const resultDiv = document.getElementById('result');
resultDiv.style.display = 'block';
if (type === 'error') {
resultDiv.innerHTML = `<div class="error"><strong>错误:</strong> ${result}</div>`;
return;
}
let html = `<div class="success"><strong>下载完成!</strong></div>`;
html += `<p><strong>标题:</strong> ${result.title}</p>`;
html += `<p><strong>文件夹:</strong> ${result.folder}</p>`;
html += `<p><strong>总计:</strong> ${result.total} 张</p>`;
html += `<p><strong>新增:</strong> ${result.saved} 张</p>`;
html += `<p><strong>跳过:</strong> ${result.skipped} 张</p>`;
html += `<p><strong>失败:</strong> ${result.failed} 张</p>`;
html += `<p><strong>消息:</strong> ${result.message}</p>`;
if (result.details && result.details.length > 0) {
html += `<h3>下载详情:</h3>`;
result.details.forEach(detail => {
let statusClass = '';
if (detail.status === 'success') statusClass = 'status-success';
else if (detail.status === 'skipped') statusClass = 'status-skipped';
else if (detail.status === 'failed') statusClass = 'status-failed';
html += `
<div class="status-item ${statusClass}">
<strong>${detail.key}:</strong> ${detail.status} - ${detail.message}
${detail.saved_as ? ` (保存为: ${detail.saved_as})` : ''}
</div>
`;
});
}
resultDiv.innerHTML = html;
}
</script>
</body>
</html>
internal/config/config.go
-10
View File
@@ -1,10 +0,0 @@
// Code scaffolded by goctl. Safe to edit.
// goctl 1.9.2
package config
import "github.com/zeromicro/go-zero/rest"
type Config struct {
rest.RestConf
}
internal/svc/servicecontext.go
-22
View File
@@ -1,22 +0,0 @@
// Code scaffolded by goctl. Safe to edit.
// goctl 1.9.2
package svc
import (
"github.com/zeromicro/go-zero/rest"
"hd4k-downloader/internal/config"
"hd4k-downloader/internal/middleware"
)
type ServiceContext struct {
Config config.Config
Cors rest.Middleware
}
func NewServiceContext(c config.Config) *ServiceContext {
return &ServiceContext{
Config: c,
Cors: middleware.NewCorsMiddleware().Handle,
}
}
internal/types/types.go
-30
View File
@@ -1,30 +0,0 @@
// Code generated by goctl. DO NOT EDIT.
// goctl 1.9.2
package types
type DownloadRequest struct {
Title string `json:"title"`
Imgs map[string]string `json:"imgs"`
}
type DownloadResponse struct {
Success bool `json:"success"`
Message string `json:"message"`
Title string `json:"title"`
Folder string `json:"folder"`
JsonPath string `json:"json_path"`
Total int `json:"total"`
Saved int `json:"saved"`
Skipped int `json:"skipped"`
Failed int `json:"failed"`
Details []ProgressDetail `json:"details"`
}
type ProgressDetail struct {
Key string `json:"key"`
URL string `json:"url"`
Status string `json:"status"`
Message string `json:"message"`
SavedAs string `json:"saved_as,optional"`
}
main.go
+54
View File
@@ -0,0 +1,54 @@
package main
import (
"flag"
"fmt"
"log"
"net/http"
"os"
)
var (
downloadDir = flag.String("dir", "./downloads", "下载目录")
port = flag.String("port", "55830", "服务端口")
)
func main() {
flag.Parse()
// 创建下载目录
if err := os.MkdirAll(*downloadDir, 0755); err != nil {
log.Fatal("创建下载目录失败:", err)
}
// 设置路由
http.HandleFunc("/api/save_imgs", saveImagesHandler)
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
http.ServeFile(w, r, "index.html")
})
// 允许跨域
http.HandleFunc("/api/", enableCORS(saveImagesHandler))
fmt.Printf("HD4K下载服务启动\n")
fmt.Printf("下载目录: %s\n", *downloadDir)
fmt.Printf("API地址: http://127.0.0.1:%s/api/save_imgs\n", *port)
fmt.Printf("按 Ctrl+C 停止服务\n")
log.Fatal(http.ListenAndServe(":"+*port, nil))
}
func enableCORS(next http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Access-Control-Allow-Origin", "*")
w.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS")
w.Header().Set("Access-Control-Allow-Headers", "Content-Type")
if r.Method == "OPTIONS" {
w.WriteHeader(http.StatusOK)
return
}
next(w, r)
}
}
test.txt
+9
View File
@@ -0,0 +1,9 @@
curl -X POST http://127.0.0.1:8888/api/save_imgs \
-H "Content-Type: application/json" \
-d '{
"title": "测试",
"imgs": {
"0001": "https://icon-icons.com/images/menu_photos.png",
"0002": "https://icon-icons.com/images/flags/zh.webp"
}
}'