jack
1 year ago
commit
defbba01fa
@ -0,0 +1,421 @@ |
|||||||
|
package main |
||||||
|
|
||||||
|
import ( |
||||||
|
"bufio" |
||||||
|
"context" |
||||||
|
"encoding/json" |
||||||
|
"fmt" |
||||||
|
"io" |
||||||
|
"log" |
||||||
|
"net/http" |
||||||
|
"net/url" |
||||||
|
"os" |
||||||
|
"path/filepath" |
||||||
|
"regexp" |
||||||
|
"strings" |
||||||
|
"sync" |
||||||
|
"time" |
||||||
|
|
||||||
|
"fyne.io/fyne/v2/widget" |
||||||
|
"golang.org/x/net/html" |
||||||
|
"golang.org/x/sync/semaphore" |
||||||
|
) |
||||||
|
|
||||||
|
// -------------------- 全局变量 --------------------
|
||||||
|
var ( |
||||||
|
illegalChars = regexp.MustCompile(`[<>:"/\\|?*\x00-\x1F]`) |
||||||
|
imgLinkRegex = regexp.MustCompile(`<a href="(.*?)"`) |
||||||
|
) |
||||||
|
|
||||||
|
// -------------------- 数据结构 --------------------
|
||||||
|
type GalleryResult struct { |
||||||
|
URL string |
||||||
|
OK bool |
||||||
|
Error error |
||||||
|
} |
||||||
|
|
||||||
|
type CrawlStats struct { |
||||||
|
Total int |
||||||
|
Success int |
||||||
|
Failed int |
||||||
|
Skipped int |
||||||
|
mu sync.Mutex |
||||||
|
} |
||||||
|
|
||||||
|
// -------------------- 工具函数 --------------------
|
||||||
|
func cleanFolderName(title string) string { |
||||||
|
// 移除非法字符
|
||||||
|
clean := illegalChars.ReplaceAllString(title, "_") |
||||||
|
// 移除空格和下划线
|
||||||
|
clean = strings.ReplaceAll(clean, " ", "") |
||||||
|
clean = strings.ReplaceAll(clean, "_", "") |
||||||
|
clean = strings.TrimSpace(clean) |
||||||
|
|
||||||
|
if clean == "" { |
||||||
|
return "gallery" |
||||||
|
} |
||||||
|
return clean |
||||||
|
} |
||||||
|
|
||||||
|
func loadTargets() ([]string, error) { |
||||||
|
file, err := os.Open(TargetsFile) |
||||||
|
if err != nil { |
||||||
|
if os.IsNotExist(err) { |
||||||
|
// 创建空文件
|
||||||
|
file, err := os.Create(TargetsFile) |
||||||
|
if err != nil { |
||||||
|
return nil, fmt.Errorf("创建目标文件失败: %v", err) |
||||||
|
} |
||||||
|
file.Close() |
||||||
|
return nil, fmt.Errorf("目标文件不存在，已自动创建，请先填写URL") |
||||||
|
} |
||||||
|
return nil, err |
||||||
|
} |
||||||
|
defer file.Close() |
||||||
|
|
||||||
|
var targets []string |
||||||
|
seen := make(map[string]bool) |
||||||
|
scanner := bufio.NewScanner(file) |
||||||
|
for scanner.Scan() { |
||||||
|
line := strings.TrimSpace(scanner.Text()) |
||||||
|
if line != "" && !seen[line] { |
||||||
|
targets = append(targets, line) |
||||||
|
seen[line] = true |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
if err := scanner.Err(); err != nil { |
||||||
|
return nil, err |
||||||
|
} |
||||||
|
|
||||||
|
if len(targets) == 0 { |
||||||
|
return nil, fmt.Errorf("目标文件为空，请先填写URL") |
||||||
|
} |
||||||
|
|
||||||
|
return targets, nil |
||||||
|
} |
||||||
|
|
||||||
|
func loadFailedUrl() ([]string, error) { |
||||||
|
data, err := os.ReadFile(FailedRecordUrl) |
||||||
|
if err != nil { |
||||||
|
if os.IsNotExist(err) { |
||||||
|
return []string{}, nil |
||||||
|
} |
||||||
|
return nil, err |
||||||
|
} |
||||||
|
|
||||||
|
var failed []string |
||||||
|
err = json.Unmarshal(data, &failed) |
||||||
|
if err != nil { |
||||||
|
return nil, err |
||||||
|
} |
||||||
|
|
||||||
|
return failed, nil |
||||||
|
} |
||||||
|
|
||||||
|
func saveFailedUrl(keys []string) error { |
||||||
|
data, err := json.MarshalIndent(keys, "", " ") |
||||||
|
if err != nil { |
||||||
|
return err |
||||||
|
} |
||||||
|
|
||||||
|
return os.WriteFile(FailedRecordUrl, data, 0644) |
||||||
|
} |
||||||
|
|
||||||
|
func ensureDownloadsDir() error { |
||||||
|
return os.MkdirAll(DownloadsDir, 0755) |
||||||
|
} |
||||||
|
|
||||||
|
func fetchPage(client *http.Client, url string) (string, error) { |
||||||
|
var lastErr error |
||||||
|
|
||||||
|
for attempt := 1; attempt <= RetryPerPage; attempt++ { |
||||||
|
resp, err := client.Get(url) |
||||||
|
if err != nil { |
||||||
|
lastErr = err |
||||||
|
log.Printf("[%d/%d] 请求失败 %s -> %v", attempt, RetryPerPage, url, err) |
||||||
|
time.Sleep(time.Duration(1<<uint(attempt)) * time.Second) |
||||||
|
continue |
||||||
|
} |
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK { |
||||||
|
resp.Body.Close() |
||||||
|
lastErr = fmt.Errorf("HTTP %d", resp.StatusCode) |
||||||
|
log.Printf("[%d/%d] 请求失败 %s -> %s", attempt, RetryPerPage, url, resp.Status) |
||||||
|
time.Sleep(time.Duration(1<<uint(attempt)) * time.Second) |
||||||
|
continue |
||||||
|
} |
||||||
|
|
||||||
|
body, err := io.ReadAll(resp.Body) |
||||||
|
resp.Body.Close() |
||||||
|
if err != nil { |
||||||
|
lastErr = err |
||||||
|
log.Printf("[%d/%d] 读取响应失败 %s -> %v", attempt, RetryPerPage, url, err) |
||||||
|
time.Sleep(time.Duration(1<<uint(attempt)) * time.Second) |
||||||
|
continue |
||||||
|
} |
||||||
|
|
||||||
|
return string(body), nil |
||||||
|
} |
||||||
|
|
||||||
|
return "", lastErr |
||||||
|
} |
||||||
|
|
||||||
|
// -------------------- HTML 解析 --------------------
|
||||||
|
func extractTitle(htmlContent string) string { |
||||||
|
doc, err := html.Parse(strings.NewReader(htmlContent)) |
||||||
|
if err != nil { |
||||||
|
return "gallery" |
||||||
|
} |
||||||
|
|
||||||
|
var title string |
||||||
|
var findTitle func(*html.Node) |
||||||
|
findTitle = func(n *html.Node) { |
||||||
|
if n.Type == html.ElementNode && n.Data == "title" { |
||||||
|
if n.FirstChild != nil { |
||||||
|
title = n.FirstChild.Data |
||||||
|
return |
||||||
|
} |
||||||
|
} |
||||||
|
for c := n.FirstChild; c != nil; c = c.NextSibling { |
||||||
|
findTitle(c) |
||||||
|
} |
||||||
|
} |
||||||
|
findTitle(doc) |
||||||
|
|
||||||
|
if title == "" { |
||||||
|
return "gallery" |
||||||
|
} |
||||||
|
return title |
||||||
|
} |
||||||
|
|
||||||
|
func extractImageLinks(htmlContent string) []string { |
||||||
|
var links []string |
||||||
|
matches := imgLinkRegex.FindAllStringSubmatch(htmlContent, -1) |
||||||
|
for _, match := range matches { |
||||||
|
if len(match) > 1 { |
||||||
|
links = append(links, match[1]) |
||||||
|
} |
||||||
|
} |
||||||
|
return links |
||||||
|
} |
||||||
|
|
||||||
|
// -------------------- 画廊爬取 --------------------
|
||||||
|
func crawlSingleGallery(client *http.Client, sem *semaphore.Weighted, galleryURL string, stats *CrawlStats) GalleryResult { |
||||||
|
// 获取信号量
|
||||||
|
ctx := context.Background() |
||||||
|
if err := sem.Acquire(ctx, 1); err != nil { |
||||||
|
return GalleryResult{URL: galleryURL, OK: false, Error: err} |
||||||
|
} |
||||||
|
defer sem.Release(1) |
||||||
|
|
||||||
|
// 解析基础URL和key
|
||||||
|
baseURL := strings.TrimRight(galleryURL, "/") |
||||||
|
parsed, err := url.Parse(baseURL) |
||||||
|
if err != nil { |
||||||
|
return GalleryResult{URL: galleryURL, OK: false, Error: err} |
||||||
|
} |
||||||
|
|
||||||
|
pathParts := strings.Split(parsed.Path, "/") |
||||||
|
key := pathParts[len(pathParts)-1] |
||||||
|
jsonName := key + ".json" |
||||||
|
|
||||||
|
var folderPath string |
||||||
|
jsonData := make(map[string]string) |
||||||
|
imgCount := 1 |
||||||
|
lastPage := false |
||||||
|
|
||||||
|
for page := 0; page < MaxPage && !lastPage; page++ { |
||||||
|
pageURL := baseURL |
||||||
|
if page > 0 { |
||||||
|
pageURL = fmt.Sprintf("%s?p=%d", baseURL, page) |
||||||
|
} |
||||||
|
|
||||||
|
htmlContent, err := fetchPage(client, pageURL) |
||||||
|
if err != nil { |
||||||
|
log.Printf("获取页面失败 %s: %v", pageURL, err) |
||||||
|
continue |
||||||
|
} |
||||||
|
|
||||||
|
// 提取标题和创建文件夹
|
||||||
|
title := extractTitle(htmlContent) |
||||||
|
cleanTitle := cleanFolderName(title) |
||||||
|
folderPath = filepath.Join(DownloadsDir, cleanTitle) |
||||||
|
|
||||||
|
if err := os.MkdirAll(folderPath, 0755); err != nil { |
||||||
|
return GalleryResult{URL: galleryURL, OK: false, Error: err} |
||||||
|
} |
||||||
|
|
||||||
|
// 检查JSON文件是否已存在
|
||||||
|
jsonPath := filepath.Join(folderPath, jsonName) |
||||||
|
if _, err := os.Stat(jsonPath); err == nil { |
||||||
|
stats.mu.Lock() |
||||||
|
stats.Skipped++ |
||||||
|
stats.mu.Unlock() |
||||||
|
log.Printf("%s 已存在，跳过", jsonName) |
||||||
|
return GalleryResult{URL: galleryURL, OK: true} |
||||||
|
} |
||||||
|
|
||||||
|
log.Printf("当前页码：%d %s", page+1, pageURL) |
||||||
|
|
||||||
|
// 提取图片链接
|
||||||
|
links := extractImageLinks(htmlContent) |
||||||
|
if len(links) == 0 { |
||||||
|
log.Printf("本页无图片入口，视为最后一页") |
||||||
|
lastPage = true |
||||||
|
continue |
||||||
|
} |
||||||
|
|
||||||
|
// 处理图片链接
|
||||||
|
for _, link := range links { |
||||||
|
// 检查是否重复（简单的重复检测）
|
||||||
|
isDuplicate := false |
||||||
|
for _, existingLink := range jsonData { |
||||||
|
if existingLink == link { |
||||||
|
isDuplicate = true |
||||||
|
lastPage = true |
||||||
|
break |
||||||
|
} |
||||||
|
} |
||||||
|
if isDuplicate { |
||||||
|
break |
||||||
|
} |
||||||
|
|
||||||
|
jsonData[fmt.Sprintf("%04d", imgCount)] = link |
||||||
|
imgCount++ |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// 保存JSON文件
|
||||||
|
if len(jsonData) > 0 { |
||||||
|
jsonPath := filepath.Join(folderPath, jsonName) |
||||||
|
data, err := json.MarshalIndent(jsonData, "", " ") |
||||||
|
if err != nil { |
||||||
|
return GalleryResult{URL: galleryURL, OK: false, Error: err} |
||||||
|
} |
||||||
|
|
||||||
|
if err := os.WriteFile(jsonPath, data, 0644); err != nil { |
||||||
|
return GalleryResult{URL: galleryURL, OK: false, Error: err} |
||||||
|
} |
||||||
|
|
||||||
|
log.Printf("保存成功 -> %s (%d 张)", jsonPath, len(jsonData)) |
||||||
|
stats.mu.Lock() |
||||||
|
stats.Success++ |
||||||
|
stats.mu.Unlock() |
||||||
|
return GalleryResult{URL: galleryURL, OK: true} |
||||||
|
} else { |
||||||
|
log.Printf("%s 未解析到任何图片链接", key) |
||||||
|
stats.mu.Lock() |
||||||
|
stats.Failed++ |
||||||
|
stats.mu.Unlock() |
||||||
|
return GalleryResult{URL: galleryURL, OK: false, Error: fmt.Errorf("未解析到图片链接")} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// -------------------- 主流程 --------------------
|
||||||
|
func UrlDownloader(ip, port string, output *widget.Entry) { |
||||||
|
log.SetFlags(log.LstdFlags | log.Lshortfile) |
||||||
|
|
||||||
|
// 确保下载目录存在
|
||||||
|
if err := ensureDownloadsDir(); err != nil { |
||||||
|
log.Fatalf("创建下载目录失败: %v", err) |
||||||
|
} |
||||||
|
|
||||||
|
// 加载目标URL
|
||||||
|
targets, err := loadTargets() |
||||||
|
if err != nil { |
||||||
|
log.Fatal(err) |
||||||
|
} |
||||||
|
|
||||||
|
// 加载失败记录
|
||||||
|
failed, err := loadFailedUrl() |
||||||
|
if err != nil { |
||||||
|
log.Printf("加载失败记录失败: %v", err) |
||||||
|
failed = []string{} |
||||||
|
} |
||||||
|
|
||||||
|
// 合并URL列表（去重）
|
||||||
|
allURLs := make([]string, 0) |
||||||
|
seen := make(map[string]bool) |
||||||
|
|
||||||
|
// 优先添加失败记录
|
||||||
|
if len(failed) > 0 { |
||||||
|
log.Printf("优先重试上次失败画廊: %d 个", len(failed)) |
||||||
|
for _, url := range failed { |
||||||
|
if !seen[url] { |
||||||
|
allURLs = append(allURLs, url) |
||||||
|
seen[url] = true |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// 添加新目标
|
||||||
|
for _, url := range targets { |
||||||
|
if !seen[url] { |
||||||
|
allURLs = append(allURLs, url) |
||||||
|
seen[url] = true |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
if len(allURLs) == 0 { |
||||||
|
log.Println("没有需要处理的URL") |
||||||
|
return |
||||||
|
} |
||||||
|
|
||||||
|
log.Printf("开始处理 %d 个画廊", len(allURLs)) |
||||||
|
|
||||||
|
// 创建HTTP客户端
|
||||||
|
proxy := ip + port |
||||||
|
client := createHTTPClient(proxy) |
||||||
|
|
||||||
|
// 创建信号量控制并发
|
||||||
|
sem := semaphore.NewWeighted(int64(Concurrency)) |
||||||
|
stats := &CrawlStats{Total: len(allURLs)} |
||||||
|
|
||||||
|
// 使用WaitGroup等待所有任务完成
|
||||||
|
var wg sync.WaitGroup |
||||||
|
results := make(chan GalleryResult, len(allURLs)) |
||||||
|
|
||||||
|
// 启动所有爬取任务
|
||||||
|
for _, galleryURL := range allURLs { |
||||||
|
wg.Add(1) |
||||||
|
go func(url string) { |
||||||
|
defer wg.Done() |
||||||
|
result := crawlSingleGallery(client, sem, url, stats) |
||||||
|
results <- result |
||||||
|
}(galleryURL) |
||||||
|
} |
||||||
|
|
||||||
|
// 等待所有任务完成
|
||||||
|
wg.Wait() |
||||||
|
close(results) |
||||||
|
|
||||||
|
// 收集失败结果
|
||||||
|
var newFailed []string |
||||||
|
for result := range results { |
||||||
|
if !result.OK { |
||||||
|
newFailed = append(newFailed, result.URL) |
||||||
|
log.Printf("画廊处理失败 %s: %v", result.URL, result.Error) |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// 处理失败记录
|
||||||
|
if len(newFailed) > 0 { |
||||||
|
if err := saveFailedUrl(newFailed); err != nil { |
||||||
|
log.Printf("保存失败记录失败: %v", err) |
||||||
|
} else { |
||||||
|
log.Printf("本轮仍有 %d 个画廊失败，已写入 %s", len(newFailed), FailedRecordUrl) |
||||||
|
} |
||||||
|
} else { |
||||||
|
// 删除失败记录文件
|
||||||
|
if err := os.Remove(FailedRecordUrl); err != nil && !os.IsNotExist(err) { |
||||||
|
log.Printf("删除失败记录文件失败: %v", err) |
||||||
|
} else { |
||||||
|
log.Println("全部画廊抓取完成！") |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// 输出统计信息
|
||||||
|
log.Printf("统计信息: 总计=%d, 成功=%d, 失败=%d, 跳过=%d", |
||||||
|
stats.Total, stats.Success, stats.Failed, stats.Skipped) |
||||||
|
} |
||||||
@ -0,0 +1,316 @@ |
|||||||
|
package main |
||||||
|
|
||||||
|
import ( |
||||||
|
"encoding/json" |
||||||
|
"fmt" |
||||||
|
"io" |
||||||
|
"log" |
||||||
|
"net/http" |
||||||
|
"os" |
||||||
|
"path/filepath" |
||||||
|
"regexp" |
||||||
|
"strings" |
||||||
|
"sync" |
||||||
|
"time" |
||||||
|
|
||||||
|
"fyne.io/fyne/v2/widget" |
||||||
|
"github.com/schollz/progressbar/v3" |
||||||
|
) |
||||||
|
|
||||||
|
// -------------------- 全局变量 --------------------
|
||||||
|
var ( |
||||||
|
imgURLRegex = regexp.MustCompile(`<img id="img" src="(.*?)"`) |
||||||
|
extRegex = regexp.MustCompile(`\.(jpg|jpeg|png|gif|webp)$`) |
||||||
|
logger = log.New(os.Stdout, "", log.LstdFlags|log.Lmsgprefix) |
||||||
|
) |
||||||
|
|
||||||
|
// -------------------- 数据结构 --------------------
|
||||||
|
type DownloadTask struct { |
||||||
|
ImgPath string `json:"img_path"` |
||||||
|
ImgURL string `json:"img_url"` |
||||||
|
} |
||||||
|
|
||||||
|
// -------------------- 工具函数 --------------------
|
||||||
|
func loadFailedImg() []DownloadTask { |
||||||
|
if _, err := os.Stat(FailedRecordImg); os.IsNotExist(err) { |
||||||
|
return []DownloadTask{} |
||||||
|
} |
||||||
|
|
||||||
|
data, err := os.ReadFile(FailedRecordImg) |
||||||
|
if err != nil { |
||||||
|
logger.Printf("加载失败记录失败 -> %v", err) |
||||||
|
return []DownloadTask{} |
||||||
|
} |
||||||
|
|
||||||
|
var tasks []DownloadTask |
||||||
|
if err := json.Unmarshal(data, &tasks); err != nil { |
||||||
|
logger.Printf("解析失败记录失败 -> %v", err) |
||||||
|
return []DownloadTask{} |
||||||
|
} |
||||||
|
|
||||||
|
return tasks |
||||||
|
} |
||||||
|
|
||||||
|
func saveFailedImg(failed []DownloadTask) { |
||||||
|
data, err := json.MarshalIndent(failed, "", " ") |
||||||
|
if err != nil { |
||||||
|
logger.Printf("序列化失败记录失败 -> %v", err) |
||||||
|
return |
||||||
|
} |
||||||
|
|
||||||
|
if err := os.WriteFile(FailedRecordImg, data, 0644); err != nil { |
||||||
|
logger.Printf("保存失败记录失败 -> %v", err) |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
func fileExists(path string) bool { |
||||||
|
_, err := os.Stat(path) |
||||||
|
return err == nil |
||||||
|
} |
||||||
|
|
||||||
|
func getFileExtension(url string) string { |
||||||
|
match := extRegex.FindStringSubmatch(strings.ToLower(url)) |
||||||
|
if len(match) > 1 { |
||||||
|
return match[1] |
||||||
|
} |
||||||
|
return "jpg" // 默认扩展名
|
||||||
|
} |
||||||
|
|
||||||
|
// -------------------- 下载核心 --------------------
|
||||||
|
func downloadOne(client *http.Client, sem chan struct{}, wg *sync.WaitGroup, task DownloadTask, bar *progressbar.ProgressBar) bool { |
||||||
|
defer wg.Done() |
||||||
|
defer func() { <-sem }() |
||||||
|
|
||||||
|
imgPath, imgURL := task.ImgPath, task.ImgURL |
||||||
|
|
||||||
|
for attempt := 1; attempt <= RetryPerImg; attempt++ { |
||||||
|
success := func() bool { |
||||||
|
// 1. 获取详情页
|
||||||
|
resp, err := client.Get(imgURL) |
||||||
|
if err != nil { |
||||||
|
logger.Printf("[ERROR] %s -> %v (尝试 %d/%d)", imgURL, err, attempt, RetryPerImg) |
||||||
|
return false |
||||||
|
} |
||||||
|
defer resp.Body.Close() |
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK { |
||||||
|
if resp.StatusCode == http.StatusTooManyRequests { |
||||||
|
wait := 1 << (attempt - 1) // 指数退避
|
||||||
|
logger.Printf("[429] 等待 %ds 后重试（%d/%d）", wait, attempt, RetryPerImg) |
||||||
|
time.Sleep(time.Duration(wait) * time.Second) |
||||||
|
return false |
||||||
|
} |
||||||
|
logger.Printf("[HTTP %d] %s", resp.StatusCode, imgURL) |
||||||
|
return false |
||||||
|
} |
||||||
|
|
||||||
|
// 读取响应内容
|
||||||
|
body, err := io.ReadAll(resp.Body) |
||||||
|
if err != nil { |
||||||
|
logger.Printf("[ERROR] 读取响应失败 %s -> %v", imgURL, err) |
||||||
|
return false |
||||||
|
} |
||||||
|
|
||||||
|
// 解析真实图片链接
|
||||||
|
realURLMatch := imgURLRegex.FindStringSubmatch(string(body)) |
||||||
|
if len(realURLMatch) < 2 { |
||||||
|
logger.Printf("未解析到真实图片链接: %s", imgURL) |
||||||
|
return false |
||||||
|
} |
||||||
|
realURL := realURLMatch[1] |
||||||
|
|
||||||
|
// 2. 下载真实图片
|
||||||
|
ext := getFileExtension(realURL) |
||||||
|
finalPath := strings.TrimSuffix(imgPath, filepath.Ext(imgPath)) + "." + ext |
||||||
|
|
||||||
|
// 检查文件是否已存在
|
||||||
|
if fileExists(finalPath) { |
||||||
|
logger.Printf("已存在，跳过: %s", filepath.Base(finalPath)) |
||||||
|
bar.Add(1) |
||||||
|
return true |
||||||
|
} |
||||||
|
|
||||||
|
// 创建目录
|
||||||
|
if err := os.MkdirAll(filepath.Dir(finalPath), 0755); err != nil { |
||||||
|
logger.Printf("[ERROR] 创建目录失败 %s -> %v", filepath.Dir(finalPath), err) |
||||||
|
return false |
||||||
|
} |
||||||
|
|
||||||
|
// 下载图片
|
||||||
|
imgResp, err := client.Get(realURL) |
||||||
|
if err != nil { |
||||||
|
logger.Printf("[ERROR] 下载图片失败 %s -> %v", realURL, err) |
||||||
|
return false |
||||||
|
} |
||||||
|
defer imgResp.Body.Close() // 修复：应该是 Body.Close()
|
||||||
|
|
||||||
|
if imgResp.StatusCode != http.StatusOK { |
||||||
|
logger.Printf("[HTTP %d] %s", imgResp.StatusCode, realURL) |
||||||
|
return false |
||||||
|
} |
||||||
|
|
||||||
|
// 创建文件
|
||||||
|
file, err := os.Create(finalPath) |
||||||
|
if err != nil { |
||||||
|
logger.Printf("[ERROR] 创建文件失败 %s -> %v", finalPath, err) |
||||||
|
return false |
||||||
|
} |
||||||
|
defer file.Close() |
||||||
|
|
||||||
|
// 写入文件
|
||||||
|
_, err = io.Copy(file, imgResp.Body) |
||||||
|
if err != nil { |
||||||
|
logger.Printf("[ERROR] 写入文件失败 %s -> %v", finalPath, err) |
||||||
|
return false |
||||||
|
} |
||||||
|
|
||||||
|
logger.Printf("[OK] %s", filepath.Base(finalPath)) |
||||||
|
bar.Add(1) |
||||||
|
return true |
||||||
|
}() |
||||||
|
|
||||||
|
if success { |
||||||
|
return true |
||||||
|
} |
||||||
|
|
||||||
|
if attempt < RetryPerImg { |
||||||
|
time.Sleep(time.Second) // 重试前等待
|
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
return false |
||||||
|
} |
||||||
|
|
||||||
|
// -------------------- 扫描待下载列表 --------------------
|
||||||
|
func scanTasks() ([]DownloadTask, error) { |
||||||
|
var tasks []DownloadTask |
||||||
|
|
||||||
|
err := filepath.Walk(DownloadDir, func(path string, info os.FileInfo, err error) error { |
||||||
|
if err != nil { |
||||||
|
return err |
||||||
|
} |
||||||
|
|
||||||
|
if info.IsDir() || filepath.Ext(path) != ".json" { |
||||||
|
return nil |
||||||
|
} |
||||||
|
|
||||||
|
// 读取JSON文件
|
||||||
|
data, err := os.ReadFile(path) |
||||||
|
if err != nil { |
||||||
|
logger.Printf("读取JSON文件失败 %s -> %v", path, err) |
||||||
|
return nil |
||||||
|
} |
||||||
|
|
||||||
|
var urlMap map[string]string |
||||||
|
if err := json.Unmarshal(data, &urlMap); err != nil { |
||||||
|
logger.Printf("解析JSON失败 %s -> %v", path, err) |
||||||
|
return nil |
||||||
|
} |
||||||
|
|
||||||
|
dir := filepath.Dir(path) |
||||||
|
for imgName, imgURL := range urlMap { |
||||||
|
imgPathWithoutExt := filepath.Join(dir, imgName) |
||||||
|
|
||||||
|
// 检查文件是否已存在（任意扩展名）
|
||||||
|
exists := false |
||||||
|
for _, ext := range []string{".jpg", ".jpeg", ".png", ".gif", ".webp"} { |
||||||
|
if fileExists(imgPathWithoutExt + ext) { |
||||||
|
exists = true |
||||||
|
break |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
if !exists { |
||||||
|
tasks = append(tasks, DownloadTask{ |
||||||
|
ImgPath: imgPathWithoutExt, |
||||||
|
ImgURL: imgURL, |
||||||
|
}) |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
return nil |
||||||
|
}) |
||||||
|
|
||||||
|
return tasks, err |
||||||
|
} |
||||||
|
|
||||||
|
// -------------------- 主流程 --------------------
|
||||||
|
func ImgDownloader(ip, port string, output *widget.Entry) { |
||||||
|
logger.SetPrefix("[INFO] ") |
||||||
|
|
||||||
|
// 1. 优先重试上次失败
|
||||||
|
failedTasks := loadFailedImg() |
||||||
|
if len(failedTasks) > 0 { |
||||||
|
logger.Printf("优先重试上次失败任务: %d 张", len(failedTasks)) |
||||||
|
} |
||||||
|
|
||||||
|
// 2. 扫描新任务
|
||||||
|
newTasks, err := scanTasks() |
||||||
|
if err != nil { |
||||||
|
logger.Printf("扫描任务失败: %v", err) |
||||||
|
return |
||||||
|
} |
||||||
|
|
||||||
|
// 合并任务
|
||||||
|
tasks := append(failedTasks, newTasks...) |
||||||
|
if len(tasks) == 0 { |
||||||
|
logger.Println("没有需要下载的图片，收工！") |
||||||
|
return |
||||||
|
} |
||||||
|
|
||||||
|
logger.Printf("开始下载 %d 张图片", len(tasks)) |
||||||
|
|
||||||
|
// 3. 创建HTTP客户端
|
||||||
|
proxy := ip + port |
||||||
|
client := createHTTPClient(proxy) |
||||||
|
|
||||||
|
// 4. 创建进度条
|
||||||
|
bar := progressbar.NewOptions(len(tasks), |
||||||
|
progressbar.OptionSetDescription("Downloading"), |
||||||
|
progressbar.OptionSetWriter(os.Stderr), |
||||||
|
progressbar.OptionShowCount(), |
||||||
|
progressbar.OptionShowIts(), |
||||||
|
progressbar.OptionSetWidth(30), |
||||||
|
progressbar.OptionThrottle(100*time.Millisecond), |
||||||
|
progressbar.OptionOnCompletion(func() { |
||||||
|
fmt.Fprint(os.Stderr, "\n") |
||||||
|
}), |
||||||
|
) |
||||||
|
|
||||||
|
// 5. 并发下载
|
||||||
|
var wg sync.WaitGroup |
||||||
|
sem := make(chan struct{}, Concurrency) |
||||||
|
results := make([]bool, len(tasks)) // 修复：使用slice存储结果
|
||||||
|
|
||||||
|
for i, task := range tasks { |
||||||
|
wg.Add(1) |
||||||
|
sem <- struct{}{} // 获取信号量
|
||||||
|
|
||||||
|
go func(idx int, t DownloadTask) { |
||||||
|
results[idx] = downloadOne(client, sem, &wg, t, bar) |
||||||
|
}(i, task) |
||||||
|
} |
||||||
|
|
||||||
|
wg.Wait() |
||||||
|
|
||||||
|
// 6. 统计结果
|
||||||
|
var failedAgain []DownloadTask |
||||||
|
successCount := 0 |
||||||
|
|
||||||
|
for i, success := range results { |
||||||
|
if !success { |
||||||
|
failedAgain = append(failedAgain, tasks[i]) |
||||||
|
} else { |
||||||
|
successCount++ |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// 7. 保存失败记录
|
||||||
|
if len(failedAgain) > 0 { |
||||||
|
saveFailedImg(failedAgain) |
||||||
|
logger.Printf("本轮仍有 %d 张下载失败，已写入 %s", len(failedAgain), FailedRecordImg) |
||||||
|
} else { |
||||||
|
os.Remove(FailedRecordImg) |
||||||
|
logger.Printf("全部下载完成！成功 %d 张", successCount) |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,49 @@ |
|||||||
|
module eh-crawler |
||||||
|
|
||||||
|
go 1.22 |
||||||
|
|
||||||
|
toolchain go1.22.2 |
||||||
|
|
||||||
|
require ( |
||||||
|
fyne.io/fyne/v2 v2.6.3 |
||||||
|
github.com/schollz/progressbar/v3 v3.18.0 |
||||||
|
golang.org/x/net v0.35.0 |
||||||
|
golang.org/x/sync v0.11.0 |
||||||
|
) |
||||||
|
|
||||||
|
require ( |
||||||
|
fyne.io/systray v1.11.0 // indirect |
||||||
|
github.com/BurntSushi/toml v1.4.0 // indirect |
||||||
|
github.com/davecgh/go-spew v1.1.1 // indirect |
||||||
|
github.com/fredbi/uri v1.1.0 // indirect |
||||||
|
github.com/fsnotify/fsnotify v1.9.0 // indirect |
||||||
|
github.com/fyne-io/gl-js v0.2.0 // indirect |
||||||
|
github.com/fyne-io/glfw-js v0.3.0 // indirect |
||||||
|
github.com/fyne-io/image v0.1.1 // indirect |
||||||
|
github.com/fyne-io/oksvg v0.1.0 // indirect |
||||||
|
github.com/go-gl/gl v0.0.0-20231021071112-07e5d0ea2e71 // indirect |
||||||
|
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20240506104042-037f3cc74f2a // indirect |
||||||
|
github.com/go-text/render v0.2.0 // indirect |
||||||
|
github.com/go-text/typesetting v0.2.1 // indirect |
||||||
|
github.com/godbus/dbus/v5 v5.1.0 // indirect |
||||||
|
github.com/hack-pad/go-indexeddb v0.3.2 // indirect |
||||||
|
github.com/hack-pad/safejs v0.1.0 // indirect |
||||||
|
github.com/jeandeaual/go-locale v0.0.0-20250612000132-0ef82f21eade // indirect |
||||||
|
github.com/jsummers/gobmp v0.0.0-20230614200233-a9de23ed2e25 // indirect |
||||||
|
github.com/kr/text v0.1.0 // indirect |
||||||
|
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect |
||||||
|
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect |
||||||
|
github.com/nicksnyder/go-i18n/v2 v2.5.1 // indirect |
||||||
|
github.com/pmezard/go-difflib v1.0.0 // indirect |
||||||
|
github.com/rivo/uniseg v0.4.7 // indirect |
||||||
|
github.com/rymdport/portal v0.4.1 // indirect |
||||||
|
github.com/srwiley/oksvg v0.0.0-20221011165216-be6e8873101c // indirect |
||||||
|
github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef // indirect |
||||||
|
github.com/stretchr/testify v1.10.0 // indirect |
||||||
|
github.com/yuin/goldmark v1.7.8 // indirect |
||||||
|
golang.org/x/image v0.24.0 // indirect |
||||||
|
golang.org/x/sys v0.30.0 // indirect |
||||||
|
golang.org/x/term v0.29.0 // indirect |
||||||
|
golang.org/x/text v0.22.0 // indirect |
||||||
|
gopkg.in/yaml.v3 v3.0.1 // indirect |
||||||
|
) |
||||||
@ -0,0 +1,94 @@ |
|||||||
|
fyne.io/fyne/v2 v2.6.3 h1:cvtM2KHeRuH+WhtHiA63z5wJVBkQ9+Ay0UMl9PxFHyA= |
||||||
|
fyne.io/fyne/v2 v2.6.3/go.mod h1:NGSurpRElVoI1G3h+ab2df3O5KLGh1CGbsMMcX0bPIs= |
||||||
|
fyne.io/systray v1.11.0 h1:D9HISlxSkx+jHSniMBR6fCFOUjk1x/OOOJLa9lJYAKg= |
||||||
|
fyne.io/systray v1.11.0/go.mod h1:RVwqP9nYMo7h5zViCBHri2FgjXF7H2cub7MAq4NSoLs= |
||||||
|
github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0= |
||||||
|
github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= |
||||||
|
github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM= |
||||||
|
github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY= |
||||||
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= |
||||||
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= |
||||||
|
github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g= |
||||||
|
github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw= |
||||||
|
github.com/fredbi/uri v1.1.0 h1:OqLpTXtyRg9ABReqvDGdJPqZUxs8cyBDOMXBbskCaB8= |
||||||
|
github.com/fredbi/uri v1.1.0/go.mod h1:aYTUoAXBOq7BLfVJ8GnKmfcuURosB1xyHDIfWeC/iW4= |
||||||
|
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= |
||||||
|
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= |
||||||
|
github.com/fyne-io/gl-js v0.2.0 h1:+EXMLVEa18EfkXBVKhifYB6OGs3HwKO3lUElA0LlAjs= |
||||||
|
github.com/fyne-io/gl-js v0.2.0/go.mod h1:ZcepK8vmOYLu96JoxbCKJy2ybr+g1pTnaBDdl7c3ajI= |
||||||
|
github.com/fyne-io/glfw-js v0.3.0 h1:d8k2+Y7l+zy2pc7wlGRyPfTgZoqDf3AI4G+2zOWhWUk= |
||||||
|
github.com/fyne-io/glfw-js v0.3.0/go.mod h1:Ri6te7rdZtBgBpxLW19uBpp3Dl6K9K/bRaYdJ22G8Jk= |
||||||
|
github.com/fyne-io/image v0.1.1 h1:WH0z4H7qfvNUw5l4p3bC1q70sa5+YWVt6HCj7y4VNyA= |
||||||
|
github.com/fyne-io/image v0.1.1/go.mod h1:xrfYBh6yspc+KjkgdZU/ifUC9sPA5Iv7WYUBzQKK7JM= |
||||||
|
github.com/fyne-io/oksvg v0.1.0 h1:7EUKk3HV3Y2E+qypp3nWqMXD7mum0hCw2KEGhI1fnBw= |
||||||
|
github.com/fyne-io/oksvg v0.1.0/go.mod h1:dJ9oEkPiWhnTFNCmRgEze+YNprJF7YRbpjgpWS4kzoI= |
||||||
|
github.com/go-gl/gl v0.0.0-20231021071112-07e5d0ea2e71 h1:5BVwOaUSBTlVZowGO6VZGw2H/zl9nrd3eCZfYV+NfQA= |
||||||
|
github.com/go-gl/gl v0.0.0-20231021071112-07e5d0ea2e71/go.mod h1:9YTyiznxEY1fVinfM7RvRcjRHbw2xLBJ3AAGIT0I4Nw= |
||||||
|
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20240506104042-037f3cc74f2a h1:vxnBhFDDT+xzxf1jTJKMKZw3H0swfWk9RpWbBbDK5+0= |
||||||
|
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20240506104042-037f3cc74f2a/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= |
||||||
|
github.com/go-text/render v0.2.0 h1:LBYoTmp5jYiJ4NPqDc2pz17MLmA3wHw1dZSVGcOdeAc= |
||||||
|
github.com/go-text/render v0.2.0/go.mod h1:CkiqfukRGKJA5vZZISkjSYrcdtgKQWRa2HIzvwNN5SU= |
||||||
|
github.com/go-text/typesetting v0.2.1 h1:x0jMOGyO3d1qFAPI0j4GSsh7M0Q3Ypjzr4+CEVg82V8= |
||||||
|
github.com/go-text/typesetting v0.2.1/go.mod h1:mTOxEwasOFpAMBjEQDhdWRckoLLeI/+qrQeBCTGEt6M= |
||||||
|
github.com/go-text/typesetting-utils v0.0.0-20241103174707-87a29e9e6066 h1:qCuYC+94v2xrb1PoS4NIDe7DGYtLnU2wWiQe9a1B1c0= |
||||||
|
github.com/go-text/typesetting-utils v0.0.0-20241103174707-87a29e9e6066/go.mod h1:DDxDdQEnB70R8owOx3LVpEFvpMK9eeH1o2r0yZhFI9o= |
||||||
|
github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= |
||||||
|
github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= |
||||||
|
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd h1:1FjCyPC+syAzJ5/2S8fqdZK1R22vvA0J7JZKcuOIQ7Y= |
||||||
|
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= |
||||||
|
github.com/hack-pad/go-indexeddb v0.3.2 h1:DTqeJJYc1usa45Q5r52t01KhvlSN02+Oq+tQbSBI91A= |
||||||
|
github.com/hack-pad/go-indexeddb v0.3.2/go.mod h1:QvfTevpDVlkfomY498LhstjwbPW6QC4VC/lxYb0Kom0= |
||||||
|
github.com/hack-pad/safejs v0.1.0 h1:qPS6vjreAqh2amUqj4WNG1zIw7qlRQJ9K10eDKMCnE8= |
||||||
|
github.com/hack-pad/safejs v0.1.0/go.mod h1:HdS+bKF1NrE72VoXZeWzxFOVQVUSqZJAG0xNCnb+Tio= |
||||||
|
github.com/jeandeaual/go-locale v0.0.0-20250612000132-0ef82f21eade h1:FmusiCI1wHw+XQbvL9M+1r/C3SPqKrmBaIOYwVfQoDE= |
||||||
|
github.com/jeandeaual/go-locale v0.0.0-20250612000132-0ef82f21eade/go.mod h1:ZDXo8KHryOWSIqnsb/CiDq7hQUYryCgdVnxbj8tDG7o= |
||||||
|
github.com/jsummers/gobmp v0.0.0-20230614200233-a9de23ed2e25 h1:YLvr1eE6cdCqjOe972w/cYF+FjW34v27+9Vo5106B4M= |
||||||
|
github.com/jsummers/gobmp v0.0.0-20230614200233-a9de23ed2e25/go.mod h1:kLgvv7o6UM+0QSf0QjAse3wReFDsb9qbZJdfexWlrQw= |
||||||
|
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= |
||||||
|
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= |
||||||
|
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= |
||||||
|
github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= |
||||||
|
github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= |
||||||
|
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= |
||||||
|
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= |
||||||
|
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= |
||||||
|
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= |
||||||
|
github.com/nicksnyder/go-i18n/v2 v2.5.1 h1:IxtPxYsR9Gp60cGXjfuR/llTqV8aYMsC472zD0D1vHk= |
||||||
|
github.com/nicksnyder/go-i18n/v2 v2.5.1/go.mod h1:DrhgsSDZxoAfvVrBVLXoxZn/pN5TXqaDbq7ju94viiQ= |
||||||
|
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= |
||||||
|
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= |
||||||
|
github.com/pkg/profile v1.7.0 h1:hnbDkaNWPCLMO9wGLdBFTIZvzDrDfBM2072E1S9gJkA= |
||||||
|
github.com/pkg/profile v1.7.0/go.mod h1:8Uer0jas47ZQMJ7VD+OHknK4YDY07LPUC6dEvqDjvNo= |
||||||
|
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= |
||||||
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= |
||||||
|
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= |
||||||
|
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= |
||||||
|
github.com/rymdport/portal v0.4.1 h1:2dnZhjf5uEaeDjeF/yBIeeRo6pNI2QAKm7kq1w/kbnA= |
||||||
|
github.com/rymdport/portal v0.4.1/go.mod h1:kFF4jslnJ8pD5uCi17brj/ODlfIidOxlgUDTO5ncnC4= |
||||||
|
github.com/schollz/progressbar/v3 v3.18.0 h1:uXdoHABRFmNIjUfte/Ex7WtuyVslrw2wVPQmCN62HpA= |
||||||
|
github.com/schollz/progressbar/v3 v3.18.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec= |
||||||
|
github.com/srwiley/oksvg v0.0.0-20221011165216-be6e8873101c h1:km8GpoQut05eY3GiYWEedbTT0qnSxrCjsVbb7yKY1KE= |
||||||
|
github.com/srwiley/oksvg v0.0.0-20221011165216-be6e8873101c/go.mod h1:cNQ3dwVJtS5Hmnjxy6AgTPd0Inb3pW05ftPSX7NZO7Q= |
||||||
|
github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef h1:Ch6Q+AZUxDBCVqdkI8FSpFyZDtCVBc2VmejdNrm5rRQ= |
||||||
|
github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef/go.mod h1:nXTWP6+gD5+LUJ8krVhhoeHjvHTutPxMYl5SvkcnJNE= |
||||||
|
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= |
||||||
|
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= |
||||||
|
github.com/yuin/goldmark v1.7.8 h1:iERMLn0/QJeHFhxSt3p6PeN9mGnvIKSpG9YYorDMnic= |
||||||
|
github.com/yuin/goldmark v1.7.8/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E= |
||||||
|
golang.org/x/image v0.24.0 h1:AN7zRgVsbvmTfNyqIbbOraYL8mSwcKncEj8ofjgzcMQ= |
||||||
|
golang.org/x/image v0.24.0/go.mod h1:4b/ITuLfqYq1hqZcjofwctIhi7sZh2WaCjvsBNjjya8= |
||||||
|
golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8= |
||||||
|
golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk= |
||||||
|
golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= |
||||||
|
golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= |
||||||
|
golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= |
||||||
|
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= |
||||||
|
golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU= |
||||||
|
golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= |
||||||
|
golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= |
||||||
|
golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= |
||||||
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= |
||||||
|
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= |
||||||
|
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= |
||||||
|
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= |
||||||
|
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= |
||||||
@ -0,0 +1,85 @@ |
|||||||
|
package main |
||||||
|
|
||||||
|
import ( |
||||||
|
"fmt" |
||||||
|
|
||||||
|
"fyne.io/fyne/v2" |
||||||
|
"fyne.io/fyne/v2/app" |
||||||
|
"fyne.io/fyne/v2/container" |
||||||
|
"fyne.io/fyne/v2/widget" |
||||||
|
) |
||||||
|
|
||||||
|
func main() { |
||||||
|
myApp := app.New() |
||||||
|
myWindow := myApp.NewWindow("下载工具") |
||||||
|
myWindow.Resize(fyne.NewSize(800, 600)) |
||||||
|
|
||||||
|
// 创建输入框
|
||||||
|
ipEntry := widget.NewEntry() |
||||||
|
ipEntry.SetText("127.0.0.1") |
||||||
|
ipEntry.Resize(fyne.NewSize(200, ipEntry.MinSize().Height)) |
||||||
|
portEntry := widget.NewEntry() |
||||||
|
portEntry.SetText("7890") |
||||||
|
ipEntry.Resize(fyne.NewSize(150, ipEntry.MinSize().Height)) |
||||||
|
|
||||||
|
// 创建输出框
|
||||||
|
outputText := widget.NewMultiLineEntry() |
||||||
|
outputText.SetPlaceHolder("输出将显示在这里...") |
||||||
|
outputText.Wrapping = fyne.TextWrapWord |
||||||
|
|
||||||
|
// 创建按钮
|
||||||
|
urlButton := widget.NewButton("下载URL", func() { |
||||||
|
ip := ipEntry.Text |
||||||
|
port := portEntry.Text |
||||||
|
// 在输出框中显示信息
|
||||||
|
outputText.SetText(fmt.Sprintf("开始下载URL - IP: %s, Port: %s\n%s", ip, port, outputText.Text)) |
||||||
|
// 调用UrlDownloader函数
|
||||||
|
UrlDownloader(ip, port, outputText) |
||||||
|
}) |
||||||
|
|
||||||
|
imgButton := widget.NewButton("下载图片", func() { |
||||||
|
ip := ipEntry.Text |
||||||
|
port := portEntry.Text |
||||||
|
// 在输出框中显示信息
|
||||||
|
outputText.SetText(fmt.Sprintf("开始下载图片 - IP: %s, Port: %s\n%s", ip, port, outputText.Text)) |
||||||
|
// 调用ImgDownloader函数
|
||||||
|
ImgDownloader(ip, port, outputText) |
||||||
|
}) |
||||||
|
|
||||||
|
clearButton := widget.NewButton("清除输出", func() { |
||||||
|
outputText.SetText("") |
||||||
|
}) |
||||||
|
|
||||||
|
// 创建滚动容器用于输出框
|
||||||
|
scrollContainer := container.NewScroll(outputText) |
||||||
|
scrollContainer.SetMinSize(fyne.NewSize(400, 400)) |
||||||
|
|
||||||
|
// 布局
|
||||||
|
// 左侧按钮容器
|
||||||
|
leftPanel := container.NewVBox( |
||||||
|
urlButton, |
||||||
|
imgButton, |
||||||
|
clearButton, |
||||||
|
widget.NewLabel(""), // 空标签用于间隔
|
||||||
|
) |
||||||
|
|
||||||
|
// 顶部输入框容器
|
||||||
|
inputPanel := container.NewHBox( |
||||||
|
widget.NewLabel("IP:"), |
||||||
|
ipEntry, |
||||||
|
widget.NewLabel("端口:"), |
||||||
|
portEntry, |
||||||
|
) |
||||||
|
|
||||||
|
// 主布局：顶部是输入框，中间是左右分栏
|
||||||
|
content := container.NewBorder( |
||||||
|
inputPanel, // 顶部
|
||||||
|
nil, // 底部
|
||||||
|
leftPanel, // 左侧
|
||||||
|
nil, // 右侧
|
||||||
|
scrollContainer, // 中间
|
||||||
|
) |
||||||
|
|
||||||
|
myWindow.SetContent(content) |
||||||
|
myWindow.ShowAndRun() |
||||||
|
} |
||||||
@ -0,0 +1 @@ |
|||||||
|
https://e-hentai.org/g/3550066/47d6393550 |
||||||
@ -0,0 +1,46 @@ |
|||||||
|
package main |
||||||
|
|
||||||
|
import ( |
||||||
|
"log" |
||||||
|
"net/http" |
||||||
|
"net/url" |
||||||
|
"time" |
||||||
|
) |
||||||
|
|
||||||
|
// -------------------- 配置常量 --------------------
|
||||||
|
const ( |
||||||
|
Concurrency = 20 // 并发页数
|
||||||
|
MaxPage = 100 // 单专辑最大翻页
|
||||||
|
RetryPerPage = 5 // 单页重试次数
|
||||||
|
Timeout = 10 // 请求超时（秒）
|
||||||
|
ImgSelector = "#gdt" // 图片入口区域选择器（用于参考，实际用正则）
|
||||||
|
FailedRecordUrl = "failed_keys.json" |
||||||
|
DownloadsDir = "downloads" |
||||||
|
TargetsFile = "targets.txt" |
||||||
|
RetryPerImg = 5 // 单图重试次数
|
||||||
|
FailedRecordImg = "failed_downloads.json" |
||||||
|
DownloadDir = "downloads" |
||||||
|
) |
||||||
|
|
||||||
|
// -------------------- HTTP客户端 --------------------
|
||||||
|
func createHTTPClient(proxy string) *http.Client { |
||||||
|
if proxy == "" { |
||||||
|
return &http.Client{ |
||||||
|
Timeout: time.Duration(Timeout) * time.Second, |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
proxyURL, err := url.Parse(proxy) |
||||||
|
if err != nil { |
||||||
|
log.Fatalf("解析代理地址失败: %v", err) |
||||||
|
} |
||||||
|
|
||||||
|
transport := &http.Transport{ |
||||||
|
Proxy: http.ProxyURL(proxyURL), |
||||||
|
} |
||||||
|
|
||||||
|
return &http.Client{ |
||||||
|
Transport: transport, |
||||||
|
Timeout: time.Duration(Timeout) * time.Second, |
||||||
|
} |
||||||
|
} |
||||||
Loading…
Reference in new issue