You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
ImagesDownloader/main.go

857 lines
22 KiB

package main
import (
"crypto/tls"
"encoding/json"
"flag"
"fmt"
"io"
"log"
"net/http"
"os"
"path/filepath"
"strings"
"sync"
"time"
)
var (
downloadDir = flag.String("dir", "./downloads", "下载目录")
port = flag.String("port", "55830", "服务端口")
)
type SaveImagesRequest struct {
Title string `json:"title"`
Imgs map[string]string `json:"imgs"`
}
// 下载详情
type DownloadDetail struct {
Key string `json:"key"`
URL string `json:"url"`
Status string `json:"status"` // success, skipped, failed
Message string `json:"message"`
SavedAs string `json:"saved_as,omitempty"`
}
// 响应结构
type SaveImagesResponse struct {
Success bool `json:"success"`
Message string `json:"message"`
Title string `json:"title"`
Folder string `json:"folder"`
JsonPath string `json:"json_path"`
Total int `json:"total"`
Saved int `json:"saved"`
Skipped int `json:"skipped"`
Failed int `json:"failed"`
Details []DownloadDetail `json:"details"`
}
// 文件夹信息结构
type FolderInfo struct {
Title string `json:"title"`
Folder string `json:"folder"`
JsonPath string `json:"json_path"`
Total int `json:"total"`
Downloaded int `json:"downloaded"`
Progress float64 `json:"progress"`
}
// 加载JSON数据
type JsonData struct {
Title string `json:"title"`
Imgs map[string]string `json:"imgs"`
}
func saveImagesHandler(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusMethodNotAllowed)
json.NewEncoder(w).Encode(map[string]interface{}{
"success": false,
"message": "只支持POST请求",
})
return
}
// 解析请求
var req SaveImagesRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusBadRequest)
json.NewEncoder(w).Encode(map[string]interface{}{
"success": false,
"message": "请求格式错误: " + err.Error(),
})
return
}
// 验证参数
if req.Title == "" {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusBadRequest)
json.NewEncoder(w).Encode(map[string]interface{}{
"success": false,
"message": "标题不能为空",
})
return
}
if len(req.Imgs) == 0 {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusBadRequest)
json.NewEncoder(w).Encode(map[string]interface{}{
"success": false,
"message": "图片列表不能为空",
})
return
}
// 处理下载
resp, err := processDownload(&req)
if err != nil {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusInternalServerError)
json.NewEncoder(w).Encode(map[string]interface{}{
"success": false,
"message": "处理失败: " + err.Error(),
})
return
}
// 返回响应
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(resp)
}
func reloadFoldersHandler(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusMethodNotAllowed)
json.NewEncoder(w).Encode(map[string]interface{}{
"success": false,
"message": "只支持GET请求",
})
return
}
folders, err := loadFolders()
if err != nil {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusInternalServerError)
json.NewEncoder(w).Encode(map[string]interface{}{
"success": false,
"message": "加载文件夹失败: " + err.Error(),
})
return
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(map[string]interface{}{
"success": true,
"folders": folders,
})
}
func downloadMissingHandler(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(w, `{"success": false, "message": "只支持POST请求"}`, http.StatusMethodNotAllowed)
return
}
var req struct {
Folder string `json:"folder"`
Title string `json:"title"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusBadRequest)
json.NewEncoder(w).Encode(map[string]interface{}{
"success": false,
"message": "请求格式错误: " + err.Error(),
})
return
}
if req.Folder == "" || req.Title == "" {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusBadRequest)
json.NewEncoder(w).Encode(map[string]interface{}{
"success": false,
"message": "参数不能为空",
})
return
}
resp, err := processMissingDownload(req.Folder, req.Title)
if err != nil {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusInternalServerError)
json.NewEncoder(w).Encode(map[string]interface{}{
"success": false,
"message": "处理失败: " + err.Error(),
})
return
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(resp)
}
func cleanupJsonHandler(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusMethodNotAllowed)
json.NewEncoder(w).Encode(map[string]interface{}{
"success": false,
"message": "只支持POST请求",
})
return
}
count, err := cleanupJsonFiles()
if err != nil {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusInternalServerError)
json.NewEncoder(w).Encode(map[string]interface{}{
"success": false,
"message": "清理失败: " + err.Error(),
})
return
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(map[string]interface{}{
"success": true,
"message": fmt.Sprintf("已删除 %d 个JSON文件", count),
"deleted": count,
})
}
// 查找文件夹中的JSON文件(支持多种方式)
func findJsonFiles(folderPath string) ([]string, error) {
jsonFiles := []string{}
// 方法1: 标准glob
pattern1 := filepath.Join(folderPath, "*.json")
files1, _ := filepath.Glob(pattern1)
jsonFiles = append(jsonFiles, files1...)
// 方法2: 不区分大小写
pattern2 := filepath.Join(folderPath, "*.JSON")
files2, _ := filepath.Glob(pattern2)
jsonFiles = append(jsonFiles, files2...)
// 方法3: 使用ReadDir手动查找(最可靠)
files3, err := os.ReadDir(folderPath)
if err != nil {
return jsonFiles, err
}
for _, file := range files3 {
if !file.IsDir() {
name := strings.ToLower(file.Name())
if strings.HasSuffix(name, ".json") {
fullPath := filepath.Join(folderPath, file.Name())
// 去重
found := false
for _, existing := range jsonFiles {
if existing == fullPath {
found = true
break
}
}
if !found {
jsonFiles = append(jsonFiles, fullPath)
}
}
}
}
return jsonFiles, nil
}
func loadFolders() ([]FolderInfo, error) {
folders := []FolderInfo{}
fmt.Println("=== 开始扫描下载目录 ===")
fmt.Println("下载目录绝对路径:", *downloadDir)
// 获取绝对路径
absDownloadDir, _ := filepath.Abs(*downloadDir)
fmt.Println("下载目录绝对路径:", absDownloadDir)
entries, err := os.ReadDir(absDownloadDir)
if err != nil {
fmt.Printf("读取目录失败: %v\n", err)
return nil, err
}
for i, entry := range entries {
if entry.IsDir() {
folderPath := filepath.Join(absDownloadDir, entry.Name())
fmt.Printf(" 处理文件夹: %s\n", folderPath)
// 使用通用函数查找JSON文件
jsonFiles, err := findJsonFiles(folderPath)
if err != nil {
fmt.Printf(" 查找JSON文件失败: %v\n", err)
continue
}
for j, jsonFile := range jsonFiles {
fmt.Printf(" [%d] %s\n", j, jsonFile)
}
if len(jsonFiles) == 0 {
fmt.Println(" 没有JSON文件,跳过")
continue
}
// 读取JSON文件
for _, jsonFile := range jsonFiles {
fmt.Printf(" 处理JSON文件: %s\n", jsonFile)
// 检查文件是否存在
if _, err := os.Stat(jsonFile); os.IsNotExist(err) {
fmt.Printf(" ❌ JSON文件不存在\n")
continue
}
data, err := os.ReadFile(jsonFile)
if err != nil {
fmt.Printf(" 读取JSON文件失败: %v\n", err)
continue
}
fmt.Printf(" 成功读取JSON文件,大小: %d bytes\n", len(data))
var jsonData JsonData
if err := json.Unmarshal(data, &jsonData); err != nil {
fmt.Printf(" 解析JSON失败: %v\n", err)
fmt.Printf(" 文件内容前100字节: %s\n", string(data[:min(100, len(data))]))
continue
}
fmt.Printf(" JSON解析成功: title='%s', imgs数量=%d\n", jsonData.Title, len(jsonData.Imgs))
if len(jsonData.Imgs) == 0 {
fmt.Println(" JSON中没有图片数据,跳过")
continue
}
// 统计实际已下载的图片数量(与JSON中的key匹配)
downloaded := 0
files, _ := os.ReadDir(folderPath)
fmt.Printf(" 文件夹中总文件数: %d\n", len(files))
// 创建已存在文件的map
existingFiles := make(map[string]bool)
for _, file := range files {
if !file.IsDir() && !strings.HasSuffix(strings.ToLower(file.Name()), ".json") {
// 提取序号(去掉扩展名)
ext := filepath.Ext(file.Name())
key := strings.TrimSuffix(file.Name(), ext)
existingFiles[key] = true
fmt.Printf(" 文件: %s -> key: %s\n", file.Name(), key)
}
}
fmt.Println(" 检查JSON中的key匹配情况:")
// 检查JSON中的每个key是否都有对应的文件
missingKeys := []string{}
for key := range jsonData.Imgs {
if existingFiles[key] {
downloaded++
fmt.Printf(" ✓ 存在: key=%s\n", key)
} else {
missingKeys = append(missingKeys, key)
fmt.Printf(" ✗ 缺失: key=%s\n", key)
}
}
// 计算进度
total := len(jsonData.Imgs)
progress := 0.0
if total > 0 {
progress = float64(downloaded) / float64(total) * 100
}
fmt.Printf(" 统计结果: 总=%d, 已下载=%d, 缺失=%d, 进度=%.2f%%\n",
total, downloaded, len(missingKeys), progress)
// 显示所有未完成的任务(包括部分完成的)
folders = append(folders, FolderInfo{
Title: jsonData.Title,
Folder: folderPath,
JsonPath: jsonFile,
Total: total,
Downloaded: downloaded,
Progress: progress,
})
fmt.Printf(" 添加到列表: title='%s', progress=%.2f%%\n", jsonData.Title, progress)
}
}
}
fmt.Printf("\n=== 扫描完成,共找到 %d 个文件夹 ===\n", len(folders))
return folders, nil
}
func processMissingDownload(folderPath, title string) (*SaveImagesResponse, error) {
fmt.Printf("=== 开始处理缺失文件下载 ===\n")
fmt.Printf("文件夹: %s\n", folderPath)
fmt.Printf("标题: %s\n", title)
// 使用通用函数查找JSON文件
jsonFiles, err := findJsonFiles(folderPath)
if err != nil || len(jsonFiles) == 0 {
fmt.Printf("未找到JSON文件,错误: %v\n", err)
return nil, fmt.Errorf("未找到JSON文件")
}
fmt.Printf("找到JSON文件: %s\n", jsonFiles[0])
// 读取JSON文件
data, err := os.ReadFile(jsonFiles[0])
if err != nil {
fmt.Printf("读取JSON文件失败: %v\n", err)
return nil, fmt.Errorf("读取JSON文件失败: %v", err)
}
var jsonData JsonData
if err := json.Unmarshal(data, &jsonData); err != nil {
fmt.Printf("解析JSON失败: %v\n", err)
return nil, fmt.Errorf("解析JSON失败: %v", err)
}
fmt.Printf("JSON解析成功: title='%s', imgs数量=%d\n", jsonData.Title, len(jsonData.Imgs))
// 检查现有文件 - 修正为只统计与JSON key匹配的文件
existingFiles := make(map[string]bool)
files, _ := os.ReadDir(folderPath)
fmt.Printf("文件夹中总文件数: %d\n", len(files))
for _, file := range files {
if !file.IsDir() && !strings.HasSuffix(strings.ToLower(file.Name()), ".json") {
// 提取序号(去掉扩展名)
ext := filepath.Ext(file.Name())
key := strings.TrimSuffix(file.Name(), ext)
existingFiles[key] = true
}
}
// 准备下载缺失的文件
needDownload := make(map[string]string)
missingCount := 0
fmt.Println("检查缺失文件:")
for key, url := range jsonData.Imgs {
if !existingFiles[key] {
needDownload[key] = url
missingCount++
fmt.Printf(" ✗ 缺失: key=%s\n", key)
} else {
fmt.Printf(" ✓ 存在: key=%s\n", key)
}
}
// 计算已下载数量(基于JSON中实际存在的key)
downloadedCount := 0
for key := range jsonData.Imgs {
if existingFiles[key] {
downloadedCount++
}
}
fmt.Printf("统计: 总文件=%d, 已下载=%d, 缺失=%d\n",
len(jsonData.Imgs), downloadedCount, missingCount)
if len(needDownload) == 0 {
fmt.Println("所有文件都已存在,无需下载")
return &SaveImagesResponse{
Success: true,
Message: "所有图片已下载完成",
Title: title,
Folder: folderPath,
JsonPath: jsonFiles[0],
Total: len(jsonData.Imgs),
Saved: 0,
Skipped: len(jsonData.Imgs),
Failed: 0,
}, nil
}
fmt.Printf("开始下载 %d 个缺失文件...\n", len(needDownload))
// 下载缺失图片
saved := 0
failed := 0
details := make([]DownloadDetail, 0, len(needDownload))
for key, url := range needDownload {
details = append(details, DownloadDetail{
Key: key,
URL: url,
Status: "pending",
Message: "等待下载",
})
}
successMap, errors := downloadImages(needDownload, folderPath)
_ = successMap
for i, detail := range details {
if detail.Status == "pending" {
if err, ok := errors[detail.Key]; ok {
details[i].Status = "failed"
details[i].Message = err.Error()
failed++
fmt.Printf("下载失败: key=%s, error=%v\n", detail.Key, err)
} else {
ext := getFileExtension(detail.URL)
filename := detail.Key + ext
details[i].Status = "success"
details[i].Message = "下载成功"
details[i].SavedAs = filename
saved++
fmt.Printf("下载成功: key=%s\n", detail.Key)
}
}
}
skipped := len(jsonData.Imgs) - saved - failed
fmt.Printf("下载完成: 新增=%d, 跳过=%d, 失败=%d\n", saved, skipped, failed)
return &SaveImagesResponse{
Success: failed == 0,
Message: fmt.Sprintf("下载完成: 新增%d张, 跳过%d张, 失败%d张", saved, skipped, failed),
Title: title,
Folder: folderPath,
JsonPath: jsonFiles[0],
Total: len(jsonData.Imgs),
Saved: saved,
Skipped: skipped,
Failed: failed,
Details: details,
}, nil
}
func cleanupJsonFiles() (int, error) {
count := 0
err := filepath.Walk(*downloadDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() && strings.HasSuffix(strings.ToLower(info.Name()), ".json") {
if err := os.Remove(path); err == nil {
count++
}
}
return nil
})
return count, err
}
func processDownload(req *SaveImagesRequest) (*SaveImagesResponse, error) {
// 清理标题(移除非法字符)
cleanTitle := cleanFilename(req.Title)
// 创建目录
comicDir := filepath.Join(*downloadDir, cleanTitle)
if err := os.MkdirAll(comicDir, 0755); err != nil {
return nil, fmt.Errorf("创建目录失败: %v", err)
}
// JSON文件路径
jsonFilename := cleanTitle + ".json"
jsonPath := filepath.Join(comicDir, jsonFilename)
// 检查现有文件
existingFiles := make(map[string]bool)
files, _ := os.ReadDir(comicDir)
for _, file := range files {
if !file.IsDir() {
filename := file.Name()
if strings.HasSuffix(filename, ".json") {
continue
}
// 提取序号(去掉扩展名)
ext := filepath.Ext(filename)
key := strings.TrimSuffix(filename, ext)
existingFiles[key] = true
}
}
// 准备下载
details := make([]DownloadDetail, 0, len(req.Imgs))
needDownload := make(map[string]string)
for key, url := range req.Imgs {
// 获取文件扩展名
ext := getFileExtension(url)
filename := key + ext
if existingFiles[key] {
// 文件已存在,跳过
details = append(details, DownloadDetail{
Key: key,
URL: url,
Status: "skipped",
Message: "文件已存在",
SavedAs: filename,
})
continue
}
// 需要下载
needDownload[key] = url
details = append(details, DownloadDetail{
Key: key,
URL: url,
Status: "pending",
Message: "等待下载",
})
}
// 下载图片
var saved, failed int
if len(needDownload) > 0 {
successMap, errors := downloadImages(needDownload, comicDir)
// 使用 successMap 来检查哪些下载成功
// 虽然我们不直接使用 successMap,但它用于 downloadImages 函数的返回值
_ = successMap // 明确标记为使用(避免未使用警告)
// 更新下载详情
for i, detail := range details {
if detail.Status == "pending" {
if err, ok := errors[detail.Key]; ok {
details[i].Status = "failed"
details[i].Message = err.Error()
failed++
} else {
ext := getFileExtension(detail.URL)
filename := detail.Key + ext
details[i].Status = "success"
details[i].Message = "下载成功"
details[i].SavedAs = filename
saved++
}
}
}
}
// 保存JSON文件
skipped := len(req.Imgs) - saved - failed
finalImgs := make(map[string]string)
for key, url := range req.Imgs {
finalImgs[key] = url
}
saveJSON(comicDir, jsonFilename, req.Title, finalImgs)
// 构建响应
return &SaveImagesResponse{
Success: failed == 0,
Message: fmt.Sprintf("下载完成: 新增%d张, 跳过%d张, 失败%d张", saved, skipped, failed),
Title: req.Title,
Folder: comicDir,
JsonPath: jsonPath,
Total: len(req.Imgs),
Saved: saved,
Skipped: skipped,
Failed: failed,
Details: details,
}, nil
}
func saveJSON(dir, filename, title string, imgs map[string]string) error {
data := map[string]interface{}{
"title": title,
"imgs": imgs,
"created_at": time.Now().Format("2006-01-02 15:04:05"),
"updated_at": time.Now().Format("2006-01-02 15:04:05"),
}
jsonData, err := json.MarshalIndent(data, "", " ")
if err != nil {
return err
}
path := filepath.Join(dir, filename)
return os.WriteFile(path, jsonData, 0644)
}
func cleanFilename(filename string) string {
illegalChars := []string{"/", "\\", ":", "*", "?", "\"", "<", ">", "|", "\n", "\r", "\t"}
result := filename
for _, char := range illegalChars {
result = strings.ReplaceAll(result, char, "_")
}
if len(result) > 200 {
result = result[:200]
}
return strings.TrimSpace(result)
}
func getFileExtension(url string) string {
// 默认扩展名
ext := ".jpg"
// 从URL提取扩展名
if idx := strings.LastIndex(url, "."); idx != -1 {
fileExt := strings.ToLower(url[idx:])
// 检查是否为常见图片格式
validExts := []string{".jpg", ".jpeg", ".png", ".webp", ".gif", ".bmp", ".tiff"}
for _, validExt := range validExts {
if strings.HasPrefix(fileExt, validExt) {
ext = validExt
break
}
}
}
return ext
}
func downloadImages(images map[string]string, dir string) (map[string]bool, map[string]error) {
var mu sync.Mutex
var wg sync.WaitGroup
success := make(map[string]bool)
errors := make(map[string]error)
// 并发控制:最多同时下载5张图片
semaphore := make(chan struct{}, 5)
for key, url := range images {
wg.Add(1)
go func(key, url string) {
defer wg.Done()
// 获取信号量
semaphore <- struct{}{}
defer func() { <-semaphore }()
// 下载图片
err := downloadImage(url, dir, key)
mu.Lock()
if err != nil {
errors[key] = err
} else {
success[key] = true
}
mu.Unlock()
}(key, url)
}
wg.Wait()
return success, errors
}
func downloadImage(url, dir, filename string) error {
// 创建HTTP客户端
client := &http.Client{
Timeout: 30 * time.Second,
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
},
}
// 创建请求
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return fmt.Errorf("创建请求失败: %v", err)
}
// 设置请求头(模拟浏览器)
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36")
req.Header.Set("Accept", "image/webp,image/apng,image/*,*/*;q=0.8")
req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8")
req.Header.Set("Referer", "https://hd4k.com/")
// 发送请求
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("请求失败: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("HTTP错误: %s", resp.Status)
}
// 获取文件扩展名
ext := getFileExtension(url)
fullFilename := filename + ext
filePath := filepath.Join(dir, fullFilename)
// 创建文件
file, err := os.Create(filePath)
if err != nil {
return fmt.Errorf("创建文件失败: %v", err)
}
defer file.Close()
// 下载并保存
_, err = io.Copy(file, resp.Body)
if err != nil {
// 删除可能已损坏的文件
os.Remove(filePath)
return fmt.Errorf("保存文件失败: %v", err)
}
return nil
}
func main() {
flag.Parse()
// 创建下载目录
if err := os.MkdirAll(*downloadDir, 0755); err != nil {
log.Fatal("创建下载目录失败:", err)
}
// 设置路由
http.HandleFunc("/api/save_imgs", enableCORS(saveImagesHandler))
http.HandleFunc("/api/reload_folders", enableCORS(reloadFoldersHandler))
http.HandleFunc("/api/download_missing", enableCORS(downloadMissingHandler))
http.HandleFunc("/api/cleanup_json", enableCORS(cleanupJsonHandler))
// 静态文件服务
http.Handle("/", http.FileServer(http.Dir(".")))
fmt.Printf("HD4K下载服务启动\n")
fmt.Printf("下载目录: %s\n", *downloadDir)
fmt.Printf("API地址: http://127.0.0.1:%s/api/save_imgs\n", *port)
fmt.Printf("管理页面: http://127.0.0.1:%s/\n", *port)
fmt.Printf("按 Ctrl+C 停止服务\n")
log.Fatal(http.ListenAndServe(":"+*port, nil))
}
func enableCORS(next http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Access-Control-Allow-Origin", "*")
w.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS")
w.Header().Set("Access-Control-Allow-Headers", "Content-Type")
if r.Method == "OPTIONS" {
w.WriteHeader(http.StatusOK)
return
}
next(w, r)
}
}