commit defbba01fa1417884d609c4aafed356a2eb24549 Author: jack Date: Fri Sep 26 18:03:58 2025 +0800 update diff --git a/1step.go b/1step.go new file mode 100644 index 0000000..b64bcca --- /dev/null +++ b/1step.go @@ -0,0 +1,421 @@ +package main + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "net/url" + "os" + "path/filepath" + "regexp" + "strings" + "sync" + "time" + + "fyne.io/fyne/v2/widget" + "golang.org/x/net/html" + "golang.org/x/sync/semaphore" +) + +// -------------------- 全局变量 -------------------- +var ( + illegalChars = regexp.MustCompile(`[<>:"/\\|?*\x00-\x1F]`) + imgLinkRegex = regexp.MustCompile(` %v", attempt, RetryPerPage, url, err) + time.Sleep(time.Duration(1< %s", attempt, RetryPerPage, url, resp.Status) + time.Sleep(time.Duration(1< %v", attempt, RetryPerPage, url, err) + time.Sleep(time.Duration(1< 1 { + links = append(links, match[1]) + } + } + return links +} + +// -------------------- 画廊爬取 -------------------- +func crawlSingleGallery(client *http.Client, sem *semaphore.Weighted, galleryURL string, stats *CrawlStats) GalleryResult { + // 获取信号量 + ctx := context.Background() + if err := sem.Acquire(ctx, 1); err != nil { + return GalleryResult{URL: galleryURL, OK: false, Error: err} + } + defer sem.Release(1) + + // 解析基础URL和key + baseURL := strings.TrimRight(galleryURL, "/") + parsed, err := url.Parse(baseURL) + if err != nil { + return GalleryResult{URL: galleryURL, OK: false, Error: err} + } + + pathParts := strings.Split(parsed.Path, "/") + key := pathParts[len(pathParts)-1] + jsonName := key + ".json" + + var folderPath string + jsonData := make(map[string]string) + imgCount := 1 + lastPage := false + + for page := 0; page < MaxPage && !lastPage; page++ { + pageURL := baseURL + if page > 0 { + pageURL = fmt.Sprintf("%s?p=%d", baseURL, page) + } + + htmlContent, err := fetchPage(client, pageURL) + if err != nil { + log.Printf("获取页面失败 %s: %v", pageURL, err) + continue + } + + // 提取标题和创建文件夹 + title := extractTitle(htmlContent) + cleanTitle := cleanFolderName(title) + folderPath = filepath.Join(DownloadsDir, cleanTitle) + + if err := os.MkdirAll(folderPath, 0755); err != nil { + return GalleryResult{URL: galleryURL, OK: false, Error: err} + } + + // 检查JSON文件是否已存在 + jsonPath := filepath.Join(folderPath, jsonName) + if _, err := os.Stat(jsonPath); err == nil { + stats.mu.Lock() + stats.Skipped++ + stats.mu.Unlock() + log.Printf("%s 已存在，跳过", jsonName) + return GalleryResult{URL: galleryURL, OK: true} + } + + log.Printf("当前页码：%d %s", page+1, pageURL) + + // 提取图片链接 + links := extractImageLinks(htmlContent) + if len(links) == 0 { + log.Printf("本页无图片入口，视为最后一页") + lastPage = true + continue + } + + // 处理图片链接 + for _, link := range links { + // 检查是否重复（简单的重复检测） + isDuplicate := false + for _, existingLink := range jsonData { + if existingLink == link { + isDuplicate = true + lastPage = true + break + } + } + if isDuplicate { + break + } + + jsonData[fmt.Sprintf("%04d", imgCount)] = link + imgCount++ + } + } + + // 保存JSON文件 + if len(jsonData) > 0 { + jsonPath := filepath.Join(folderPath, jsonName) + data, err := json.MarshalIndent(jsonData, "", " ") + if err != nil { + return GalleryResult{URL: galleryURL, OK: false, Error: err} + } + + if err := os.WriteFile(jsonPath, data, 0644); err != nil { + return GalleryResult{URL: galleryURL, OK: false, Error: err} + } + + log.Printf("保存成功 -> %s (%d 张)", jsonPath, len(jsonData)) + stats.mu.Lock() + stats.Success++ + stats.mu.Unlock() + return GalleryResult{URL: galleryURL, OK: true} + } else { + log.Printf("%s 未解析到任何图片链接", key) + stats.mu.Lock() + stats.Failed++ + stats.mu.Unlock() + return GalleryResult{URL: galleryURL, OK: false, Error: fmt.Errorf("未解析到图片链接")} + } +} + +// -------------------- 主流程 -------------------- +func UrlDownloader(ip, port string, output *widget.Entry) { + log.SetFlags(log.LstdFlags | log.Lshortfile) + + // 确保下载目录存在 + if err := ensureDownloadsDir(); err != nil { + log.Fatalf("创建下载目录失败: %v", err) + } + + // 加载目标URL + targets, err := loadTargets() + if err != nil { + log.Fatal(err) + } + + // 加载失败记录 + failed, err := loadFailedUrl() + if err != nil { + log.Printf("加载失败记录失败: %v", err) + failed = []string{} + } + + // 合并URL列表（去重） + allURLs := make([]string, 0) + seen := make(map[string]bool) + + // 优先添加失败记录 + if len(failed) > 0 { + log.Printf("优先重试上次失败画廊: %d 个", len(failed)) + for _, url := range failed { + if !seen[url] { + allURLs = append(allURLs, url) + seen[url] = true + } + } + } + + // 添加新目标 + for _, url := range targets { + if !seen[url] { + allURLs = append(allURLs, url) + seen[url] = true + } + } + + if len(allURLs) == 0 { + log.Println("没有需要处理的URL") + return + } + + log.Printf("开始处理 %d 个画廊", len(allURLs)) + + // 创建HTTP客户端 + proxy := ip + port + client := createHTTPClient(proxy) + + // 创建信号量控制并发 + sem := semaphore.NewWeighted(int64(Concurrency)) + stats := &CrawlStats{Total: len(allURLs)} + + // 使用WaitGroup等待所有任务完成 + var wg sync.WaitGroup + results := make(chan GalleryResult, len(allURLs)) + + // 启动所有爬取任务 + for _, galleryURL := range allURLs { + wg.Add(1) + go func(url string) { + defer wg.Done() + result := crawlSingleGallery(client, sem, url, stats) + results <- result + }(galleryURL) + } + + // 等待所有任务完成 + wg.Wait() + close(results) + + // 收集失败结果 + var newFailed []string + for result := range results { + if !result.OK { + newFailed = append(newFailed, result.URL) + log.Printf("画廊处理失败 %s: %v", result.URL, result.Error) + } + } + + // 处理失败记录 + if len(newFailed) > 0 { + if err := saveFailedUrl(newFailed); err != nil { + log.Printf("保存失败记录失败: %v", err) + } else { + log.Printf("本轮仍有 %d 个画廊失败，已写入 %s", len(newFailed), FailedRecordUrl) + } + } else { + // 删除失败记录文件 + if err := os.Remove(FailedRecordUrl); err != nil && !os.IsNotExist(err) { + log.Printf("删除失败记录文件失败: %v", err) + } else { + log.Println("全部画廊抓取完成！") + } + } + + // 输出统计信息 + log.Printf("统计信息: 总计=%d, 成功=%d, 失败=%d, 跳过=%d", + stats.Total, stats.Success, stats.Failed, stats.Skipped) +} diff --git a/2step.go b/2step.go new file mode 100644 index 0000000..b459760 --- /dev/null +++ b/2step.go @@ -0,0 +1,316 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "os" + "path/filepath" + "regexp" + "strings" + "sync" + "time" + + "fyne.io/fyne/v2/widget" + "github.com/schollz/progressbar/v3" +) + +// -------------------- 全局变量 -------------------- +var ( + imgURLRegex = regexp.MustCompile(` %v", err) + return []DownloadTask{} + } + + var tasks []DownloadTask + if err := json.Unmarshal(data, &tasks); err != nil { + logger.Printf("解析失败记录失败 -> %v", err) + return []DownloadTask{} + } + + return tasks +} + +func saveFailedImg(failed []DownloadTask) { + data, err := json.MarshalIndent(failed, "", " ") + if err != nil { + logger.Printf("序列化失败记录失败 -> %v", err) + return + } + + if err := os.WriteFile(FailedRecordImg, data, 0644); err != nil { + logger.Printf("保存失败记录失败 -> %v", err) + } +} + +func fileExists(path string) bool { + _, err := os.Stat(path) + return err == nil +} + +func getFileExtension(url string) string { + match := extRegex.FindStringSubmatch(strings.ToLower(url)) + if len(match) > 1 { + return match[1] + } + return "jpg" // 默认扩展名 +} + +// -------------------- 下载核心 -------------------- +func downloadOne(client *http.Client, sem chan struct{}, wg *sync.WaitGroup, task DownloadTask, bar *progressbar.ProgressBar) bool { + defer wg.Done() + defer func() { <-sem }() + + imgPath, imgURL := task.ImgPath, task.ImgURL + + for attempt := 1; attempt <= RetryPerImg; attempt++ { + success := func() bool { + // 1. 获取详情页 + resp, err := client.Get(imgURL) + if err != nil { + logger.Printf("[ERROR] %s -> %v (尝试 %d/%d)", imgURL, err, attempt, RetryPerImg) + return false + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + if resp.StatusCode == http.StatusTooManyRequests { + wait := 1 << (attempt - 1) // 指数退避 + logger.Printf("[429] 等待 %ds 后重试（%d/%d）", wait, attempt, RetryPerImg) + time.Sleep(time.Duration(wait) * time.Second) + return false + } + logger.Printf("[HTTP %d] %s", resp.StatusCode, imgURL) + return false + } + + // 读取响应内容 + body, err := io.ReadAll(resp.Body) + if err != nil { + logger.Printf("[ERROR] 读取响应失败 %s -> %v", imgURL, err) + return false + } + + // 解析真实图片链接 + realURLMatch := imgURLRegex.FindStringSubmatch(string(body)) + if len(realURLMatch) < 2 { + logger.Printf("未解析到真实图片链接: %s", imgURL) + return false + } + realURL := realURLMatch[1] + + // 2. 下载真实图片 + ext := getFileExtension(realURL) + finalPath := strings.TrimSuffix(imgPath, filepath.Ext(imgPath)) + "." + ext + + // 检查文件是否已存在 + if fileExists(finalPath) { + logger.Printf("已存在，跳过: %s", filepath.Base(finalPath)) + bar.Add(1) + return true + } + + // 创建目录 + if err := os.MkdirAll(filepath.Dir(finalPath), 0755); err != nil { + logger.Printf("[ERROR] 创建目录失败 %s -> %v", filepath.Dir(finalPath), err) + return false + } + + // 下载图片 + imgResp, err := client.Get(realURL) + if err != nil { + logger.Printf("[ERROR] 下载图片失败 %s -> %v", realURL, err) + return false + } + defer imgResp.Body.Close() // 修复：应该是 Body.Close() + + if imgResp.StatusCode != http.StatusOK { + logger.Printf("[HTTP %d] %s", imgResp.StatusCode, realURL) + return false + } + + // 创建文件 + file, err := os.Create(finalPath) + if err != nil { + logger.Printf("[ERROR] 创建文件失败 %s -> %v", finalPath, err) + return false + } + defer file.Close() + + // 写入文件 + _, err = io.Copy(file, imgResp.Body) + if err != nil { + logger.Printf("[ERROR] 写入文件失败 %s -> %v", finalPath, err) + return false + } + + logger.Printf("[OK] %s", filepath.Base(finalPath)) + bar.Add(1) + return true + }() + + if success { + return true + } + + if attempt < RetryPerImg { + time.Sleep(time.Second) // 重试前等待 + } + } + + return false +} + +// -------------------- 扫描待下载列表 -------------------- +func scanTasks() ([]DownloadTask, error) { + var tasks []DownloadTask + + err := filepath.Walk(DownloadDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if info.IsDir() || filepath.Ext(path) != ".json" { + return nil + } + + // 读取JSON文件 + data, err := os.ReadFile(path) + if err != nil { + logger.Printf("读取JSON文件失败 %s -> %v", path, err) + return nil + } + + var urlMap map[string]string + if err := json.Unmarshal(data, &urlMap); err != nil { + logger.Printf("解析JSON失败 %s -> %v", path, err) + return nil + } + + dir := filepath.Dir(path) + for imgName, imgURL := range urlMap { + imgPathWithoutExt := filepath.Join(dir, imgName) + + // 检查文件是否已存在（任意扩展名） + exists := false + for _, ext := range []string{".jpg", ".jpeg", ".png", ".gif", ".webp"} { + if fileExists(imgPathWithoutExt + ext) { + exists = true + break + } + } + + if !exists { + tasks = append(tasks, DownloadTask{ + ImgPath: imgPathWithoutExt, + ImgURL: imgURL, + }) + } + } + + return nil + }) + + return tasks, err +} + +// -------------------- 主流程 -------------------- +func ImgDownloader(ip, port string, output *widget.Entry) { + logger.SetPrefix("[INFO] ") + + // 1. 优先重试上次失败 + failedTasks := loadFailedImg() + if len(failedTasks) > 0 { + logger.Printf("优先重试上次失败任务: %d 张", len(failedTasks)) + } + + // 2. 扫描新任务 + newTasks, err := scanTasks() + if err != nil { + logger.Printf("扫描任务失败: %v", err) + return + } + + // 合并任务 + tasks := append(failedTasks, newTasks...) + if len(tasks) == 0 { + logger.Println("没有需要下载的图片，收工！") + return + } + + logger.Printf("开始下载 %d 张图片", len(tasks)) + + // 3. 创建HTTP客户端 + proxy := ip + port + client := createHTTPClient(proxy) + + // 4. 创建进度条 + bar := progressbar.NewOptions(len(tasks), + progressbar.OptionSetDescription("Downloading"), + progressbar.OptionSetWriter(os.Stderr), + progressbar.OptionShowCount(), + progressbar.OptionShowIts(), + progressbar.OptionSetWidth(30), + progressbar.OptionThrottle(100*time.Millisecond), + progressbar.OptionOnCompletion(func() { + fmt.Fprint(os.Stderr, "\n") + }), + ) + + // 5. 并发下载 + var wg sync.WaitGroup + sem := make(chan struct{}, Concurrency) + results := make([]bool, len(tasks)) // 修复：使用slice存储结果 + + for i, task := range tasks { + wg.Add(1) + sem <- struct{}{} // 获取信号量 + + go func(idx int, t DownloadTask) { + results[idx] = downloadOne(client, sem, &wg, t, bar) + }(i, task) + } + + wg.Wait() + + // 6. 统计结果 + var failedAgain []DownloadTask + successCount := 0 + + for i, success := range results { + if !success { + failedAgain = append(failedAgain, tasks[i]) + } else { + successCount++ + } + } + + // 7. 保存失败记录 + if len(failedAgain) > 0 { + saveFailedImg(failedAgain) + logger.Printf("本轮仍有 %d 张下载失败，已写入 %s", len(failedAgain), FailedRecordImg) + } else { + os.Remove(FailedRecordImg) + logger.Printf("全部下载完成！成功 %d 张", successCount) + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..c281362 --- /dev/null +++ b/go.mod @@ -0,0 +1,49 @@ +module eh-crawler + +go 1.22 + +toolchain go1.22.2 + +require ( + fyne.io/fyne/v2 v2.6.3 + github.com/schollz/progressbar/v3 v3.18.0 + golang.org/x/net v0.35.0 + golang.org/x/sync v0.11.0 +) + +require ( + fyne.io/systray v1.11.0 // indirect + github.com/BurntSushi/toml v1.4.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/fredbi/uri v1.1.0 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/fyne-io/gl-js v0.2.0 // indirect + github.com/fyne-io/glfw-js v0.3.0 // indirect + github.com/fyne-io/image v0.1.1 // indirect + github.com/fyne-io/oksvg v0.1.0 // indirect + github.com/go-gl/gl v0.0.0-20231021071112-07e5d0ea2e71 // indirect + github.com/go-gl/glfw/v3.3/glfw v0.0.0-20240506104042-037f3cc74f2a // indirect + github.com/go-text/render v0.2.0 // indirect + github.com/go-text/typesetting v0.2.1 // indirect + github.com/godbus/dbus/v5 v5.1.0 // indirect + github.com/hack-pad/go-indexeddb v0.3.2 // indirect + github.com/hack-pad/safejs v0.1.0 // indirect + github.com/jeandeaual/go-locale v0.0.0-20250612000132-0ef82f21eade // indirect + github.com/jsummers/gobmp v0.0.0-20230614200233-a9de23ed2e25 // indirect + github.com/kr/text v0.1.0 // indirect + github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect + github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect + github.com/nicksnyder/go-i18n/v2 v2.5.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/rymdport/portal v0.4.1 // indirect + github.com/srwiley/oksvg v0.0.0-20221011165216-be6e8873101c // indirect + github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef // indirect + github.com/stretchr/testify v1.10.0 // indirect + github.com/yuin/goldmark v1.7.8 // indirect + golang.org/x/image v0.24.0 // indirect + golang.org/x/sys v0.30.0 // indirect + golang.org/x/term v0.29.0 // indirect + golang.org/x/text v0.22.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..488534b --- /dev/null +++ b/go.sum @@ -0,0 +1,94 @@ +fyne.io/fyne/v2 v2.6.3 h1:cvtM2KHeRuH+WhtHiA63z5wJVBkQ9+Ay0UMl9PxFHyA= +fyne.io/fyne/v2 v2.6.3/go.mod h1:NGSurpRElVoI1G3h+ab2df3O5KLGh1CGbsMMcX0bPIs= +fyne.io/systray v1.11.0 h1:D9HISlxSkx+jHSniMBR6fCFOUjk1x/OOOJLa9lJYAKg= +fyne.io/systray v1.11.0/go.mod h1:RVwqP9nYMo7h5zViCBHri2FgjXF7H2cub7MAq4NSoLs= +github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0= +github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= +github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM= +github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g= +github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw= +github.com/fredbi/uri v1.1.0 h1:OqLpTXtyRg9ABReqvDGdJPqZUxs8cyBDOMXBbskCaB8= +github.com/fredbi/uri v1.1.0/go.mod h1:aYTUoAXBOq7BLfVJ8GnKmfcuURosB1xyHDIfWeC/iW4= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fyne-io/gl-js v0.2.0 h1:+EXMLVEa18EfkXBVKhifYB6OGs3HwKO3lUElA0LlAjs= +github.com/fyne-io/gl-js v0.2.0/go.mod h1:ZcepK8vmOYLu96JoxbCKJy2ybr+g1pTnaBDdl7c3ajI= +github.com/fyne-io/glfw-js v0.3.0 h1:d8k2+Y7l+zy2pc7wlGRyPfTgZoqDf3AI4G+2zOWhWUk= +github.com/fyne-io/glfw-js v0.3.0/go.mod h1:Ri6te7rdZtBgBpxLW19uBpp3Dl6K9K/bRaYdJ22G8Jk= +github.com/fyne-io/image v0.1.1 h1:WH0z4H7qfvNUw5l4p3bC1q70sa5+YWVt6HCj7y4VNyA= +github.com/fyne-io/image v0.1.1/go.mod h1:xrfYBh6yspc+KjkgdZU/ifUC9sPA5Iv7WYUBzQKK7JM= +github.com/fyne-io/oksvg v0.1.0 h1:7EUKk3HV3Y2E+qypp3nWqMXD7mum0hCw2KEGhI1fnBw= +github.com/fyne-io/oksvg v0.1.0/go.mod h1:dJ9oEkPiWhnTFNCmRgEze+YNprJF7YRbpjgpWS4kzoI= +github.com/go-gl/gl v0.0.0-20231021071112-07e5d0ea2e71 h1:5BVwOaUSBTlVZowGO6VZGw2H/zl9nrd3eCZfYV+NfQA= +github.com/go-gl/gl v0.0.0-20231021071112-07e5d0ea2e71/go.mod h1:9YTyiznxEY1fVinfM7RvRcjRHbw2xLBJ3AAGIT0I4Nw= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20240506104042-037f3cc74f2a h1:vxnBhFDDT+xzxf1jTJKMKZw3H0swfWk9RpWbBbDK5+0= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20240506104042-037f3cc74f2a/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-text/render v0.2.0 h1:LBYoTmp5jYiJ4NPqDc2pz17MLmA3wHw1dZSVGcOdeAc= +github.com/go-text/render v0.2.0/go.mod h1:CkiqfukRGKJA5vZZISkjSYrcdtgKQWRa2HIzvwNN5SU= +github.com/go-text/typesetting v0.2.1 h1:x0jMOGyO3d1qFAPI0j4GSsh7M0Q3Ypjzr4+CEVg82V8= +github.com/go-text/typesetting v0.2.1/go.mod h1:mTOxEwasOFpAMBjEQDhdWRckoLLeI/+qrQeBCTGEt6M= +github.com/go-text/typesetting-utils v0.0.0-20241103174707-87a29e9e6066 h1:qCuYC+94v2xrb1PoS4NIDe7DGYtLnU2wWiQe9a1B1c0= +github.com/go-text/typesetting-utils v0.0.0-20241103174707-87a29e9e6066/go.mod h1:DDxDdQEnB70R8owOx3LVpEFvpMK9eeH1o2r0yZhFI9o= +github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= +github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/google/pprof v0.0.0-20211214055906-6f57359322fd h1:1FjCyPC+syAzJ5/2S8fqdZK1R22vvA0J7JZKcuOIQ7Y= +github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= +github.com/hack-pad/go-indexeddb v0.3.2 h1:DTqeJJYc1usa45Q5r52t01KhvlSN02+Oq+tQbSBI91A= +github.com/hack-pad/go-indexeddb v0.3.2/go.mod h1:QvfTevpDVlkfomY498LhstjwbPW6QC4VC/lxYb0Kom0= +github.com/hack-pad/safejs v0.1.0 h1:qPS6vjreAqh2amUqj4WNG1zIw7qlRQJ9K10eDKMCnE8= +github.com/hack-pad/safejs v0.1.0/go.mod h1:HdS+bKF1NrE72VoXZeWzxFOVQVUSqZJAG0xNCnb+Tio= +github.com/jeandeaual/go-locale v0.0.0-20250612000132-0ef82f21eade h1:FmusiCI1wHw+XQbvL9M+1r/C3SPqKrmBaIOYwVfQoDE= +github.com/jeandeaual/go-locale v0.0.0-20250612000132-0ef82f21eade/go.mod h1:ZDXo8KHryOWSIqnsb/CiDq7hQUYryCgdVnxbj8tDG7o= +github.com/jsummers/gobmp v0.0.0-20230614200233-a9de23ed2e25 h1:YLvr1eE6cdCqjOe972w/cYF+FjW34v27+9Vo5106B4M= +github.com/jsummers/gobmp v0.0.0-20230614200233-a9de23ed2e25/go.mod h1:kLgvv7o6UM+0QSf0QjAse3wReFDsb9qbZJdfexWlrQw= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= +github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= +github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= +github.com/nicksnyder/go-i18n/v2 v2.5.1 h1:IxtPxYsR9Gp60cGXjfuR/llTqV8aYMsC472zD0D1vHk= +github.com/nicksnyder/go-i18n/v2 v2.5.1/go.mod h1:DrhgsSDZxoAfvVrBVLXoxZn/pN5TXqaDbq7ju94viiQ= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/pkg/profile v1.7.0 h1:hnbDkaNWPCLMO9wGLdBFTIZvzDrDfBM2072E1S9gJkA= +github.com/pkg/profile v1.7.0/go.mod h1:8Uer0jas47ZQMJ7VD+OHknK4YDY07LPUC6dEvqDjvNo= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rymdport/portal v0.4.1 h1:2dnZhjf5uEaeDjeF/yBIeeRo6pNI2QAKm7kq1w/kbnA= +github.com/rymdport/portal v0.4.1/go.mod h1:kFF4jslnJ8pD5uCi17brj/ODlfIidOxlgUDTO5ncnC4= +github.com/schollz/progressbar/v3 v3.18.0 h1:uXdoHABRFmNIjUfte/Ex7WtuyVslrw2wVPQmCN62HpA= +github.com/schollz/progressbar/v3 v3.18.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec= +github.com/srwiley/oksvg v0.0.0-20221011165216-be6e8873101c h1:km8GpoQut05eY3GiYWEedbTT0qnSxrCjsVbb7yKY1KE= +github.com/srwiley/oksvg v0.0.0-20221011165216-be6e8873101c/go.mod h1:cNQ3dwVJtS5Hmnjxy6AgTPd0Inb3pW05ftPSX7NZO7Q= +github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef h1:Ch6Q+AZUxDBCVqdkI8FSpFyZDtCVBc2VmejdNrm5rRQ= +github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef/go.mod h1:nXTWP6+gD5+LUJ8krVhhoeHjvHTutPxMYl5SvkcnJNE= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/yuin/goldmark v1.7.8 h1:iERMLn0/QJeHFhxSt3p6PeN9mGnvIKSpG9YYorDMnic= +github.com/yuin/goldmark v1.7.8/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E= +golang.org/x/image v0.24.0 h1:AN7zRgVsbvmTfNyqIbbOraYL8mSwcKncEj8ofjgzcMQ= +golang.org/x/image v0.24.0/go.mod h1:4b/ITuLfqYq1hqZcjofwctIhi7sZh2WaCjvsBNjjya8= +golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8= +golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk= +golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= +golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU= +golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= +golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main b/main new file mode 100755 index 0000000..f016704 Binary files /dev/null and b/main differ diff --git a/main.go b/main.go new file mode 100644 index 0000000..76e8907 --- /dev/null +++ b/main.go @@ -0,0 +1,85 @@ +package main + +import ( + "fmt" + + "fyne.io/fyne/v2" + "fyne.io/fyne/v2/app" + "fyne.io/fyne/v2/container" + "fyne.io/fyne/v2/widget" +) + +func main() { + myApp := app.New() + myWindow := myApp.NewWindow("下载工具") + myWindow.Resize(fyne.NewSize(800, 600)) + + // 创建输入框 + ipEntry := widget.NewEntry() + ipEntry.SetText("127.0.0.1") + ipEntry.Resize(fyne.NewSize(200, ipEntry.MinSize().Height)) + portEntry := widget.NewEntry() + portEntry.SetText("7890") + ipEntry.Resize(fyne.NewSize(150, ipEntry.MinSize().Height)) + + // 创建输出框 + outputText := widget.NewMultiLineEntry() + outputText.SetPlaceHolder("输出将显示在这里...") + outputText.Wrapping = fyne.TextWrapWord + + // 创建按钮 + urlButton := widget.NewButton("下载URL", func() { + ip := ipEntry.Text + port := portEntry.Text + // 在输出框中显示信息 + outputText.SetText(fmt.Sprintf("开始下载URL - IP: %s, Port: %s\n%s", ip, port, outputText.Text)) + // 调用UrlDownloader函数 + UrlDownloader(ip, port, outputText) + }) + + imgButton := widget.NewButton("下载图片", func() { + ip := ipEntry.Text + port := portEntry.Text + // 在输出框中显示信息 + outputText.SetText(fmt.Sprintf("开始下载图片 - IP: %s, Port: %s\n%s", ip, port, outputText.Text)) + // 调用ImgDownloader函数 + ImgDownloader(ip, port, outputText) + }) + + clearButton := widget.NewButton("清除输出", func() { + outputText.SetText("") + }) + + // 创建滚动容器用于输出框 + scrollContainer := container.NewScroll(outputText) + scrollContainer.SetMinSize(fyne.NewSize(400, 400)) + + // 布局 + // 左侧按钮容器 + leftPanel := container.NewVBox( + urlButton, + imgButton, + clearButton, + widget.NewLabel(""), // 空标签用于间隔 + ) + + // 顶部输入框容器 + inputPanel := container.NewHBox( + widget.NewLabel("IP:"), + ipEntry, + widget.NewLabel("端口:"), + portEntry, + ) + + // 主布局：顶部是输入框，中间是左右分栏 + content := container.NewBorder( + inputPanel, // 顶部 + nil, // 底部 + leftPanel, // 左侧 + nil, // 右侧 + scrollContainer, // 中间 + ) + + myWindow.SetContent(content) + myWindow.ShowAndRun() +} diff --git a/targets.txt b/targets.txt new file mode 100644 index 0000000..24bd4ae --- /dev/null +++ b/targets.txt @@ -0,0 +1 @@ +https://e-hentai.org/g/3550066/47d6393550 \ No newline at end of file diff --git a/utils.go b/utils.go new file mode 100644 index 0000000..52bc616 --- /dev/null +++ b/utils.go @@ -0,0 +1,46 @@ +package main + +import ( + "log" + "net/http" + "net/url" + "time" +) + +// -------------------- 配置常量 -------------------- +const ( + Concurrency = 20 // 并发页数 + MaxPage = 100 // 单专辑最大翻页 + RetryPerPage = 5 // 单页重试次数 + Timeout = 10 // 请求超时（秒） + ImgSelector = "#gdt" // 图片入口区域选择器（用于参考，实际用正则） + FailedRecordUrl = "failed_keys.json" + DownloadsDir = "downloads" + TargetsFile = "targets.txt" + RetryPerImg = 5 // 单图重试次数 + FailedRecordImg = "failed_downloads.json" + DownloadDir = "downloads" +) + +// -------------------- HTTP客户端 -------------------- +func createHTTPClient(proxy string) *http.Client { + if proxy == "" { + return &http.Client{ + Timeout: time.Duration(Timeout) * time.Second, + } + } + + proxyURL, err := url.Parse(proxy) + if err != nil { + log.Fatalf("解析代理地址失败: %v", err) + } + + transport := &http.Transport{ + Proxy: http.ProxyURL(proxyURL), + } + + return &http.Client{ + Transport: transport, + Timeout: time.Duration(Timeout) * time.Second, + } +}