toor
7243f73d33
用户反馈：点开始下载后 6 个画廊（都是 2000 张级）一直显示「解析中 0/?」， 看起来像卡死。排查发现两个问题： 1. 连接泄漏（真 bug） httpClient() 每次请求都新建 http.Transport，用完丢弃。Transport 被丢弃后 它持有的 keep-alive 连接不会立刻释放（要等 IdleConnTimeout=90s）， 于是每发一个请求留下一条僵尸连接。 现场实测：91 条 Established 连接，而解析是串行的，正常应该只有 1 条。 修复：Transport 全局复用，只在代理配置变化时重建， 重建时 CloseIdleConnections 关掉旧连接；新增 Client.Close()。 实测：20 次请求从 20 条连接降到 1 条；100 次并发请求只用 22 条。 2. 解析阶段毫无反馈 2000 张 = 100 屏串行翻页，整个解析阶段界面都显示「解析中 0/?」， 完全没有进度，用户只能干等。 修复：FetchGalleryPage 增加 ParseProgress 回调，每翻完一屏上报一次 （已解析张数/总张数/当前屏），界面显示「已解析 340/2000 张」。 另外：用户设置里请求间隔是 1000ms（1 秒），全局限流让 5 个画廊共享 1 请求/秒 —— 500 次解析请求需要约 8 分钟。这是配置导致的慢，不是 bug， 但可以在界面上看到进度后就不会以为卡死了。 测试从 74 增至 80，新增： - TestTransportIsReused / TestTransportReusedUnderConcurrency（连接复用） - TestSetProxyRebuildsTransport（改代理要重建） - TestParseProgressCallback（进度回调，含最后一屏）
701 lines
19 KiB
Go
701 lines
19 KiB
Go
package eh
|
||
|
||
// E-Hentai 抓取核心（2026 版协议）
|
||
//
|
||
// 已实测的链路：
|
||
// 1) GET https://e-hentai.org/g/{gid}/{token}/ 拿全部页的 /s/ 链接 + 原文件名
|
||
// 2) GET https://e-hentai.org/s/{key}/{gid}-{page} 拿该页 hath 全图 URL（key 必须与 page 配对）
|
||
// 3) GET <hath 图 URL> 下载图片本身
|
||
//
|
||
// 老协议（var gmetadata / /img/{gid}/{gid}/{gid}-{page}.jpg / api.php gdata）全部失效。
|
||
//
|
||
// 关于「图片直链」：hath 的 URL 是 随机子域 + 随机端口 + keystamp 时间戳签名，
|
||
// 不存在可推算的固定直链，只能每次实时从 /s/ 页面解析。所以这里没有「解码」，
|
||
// 只有「从 HTML 里提取」—— 见 ResolveImageURL。
|
||
|
||
import (
|
||
"context"
|
||
"errors"
|
||
"fmt"
|
||
"io"
|
||
"net"
|
||
"net/http"
|
||
"net/url"
|
||
"os"
|
||
"regexp"
|
||
"strconv"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
|
||
"golang.org/x/net/proxy"
|
||
)
|
||
|
||
const (
|
||
// Referer 不是必需的（实测裸请求也能 200），但带上更稳。
|
||
defaultUA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " +
|
||
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||
)
|
||
|
||
// EHBase 站点根地址（测试里会临时替换成 mock 服务器）
|
||
var EHBase = "https://e-hentai.org"
|
||
|
||
// ---------------------------------------------------------------- 输入解析
|
||
|
||
// GalleryRef 是一个待下载的画廊（gid + token 缺一不可，只有 gid 会 404）
|
||
type GalleryRef struct {
|
||
GID string `json:"gid"`
|
||
Token string `json:"token"`
|
||
Raw string `json:"raw"`
|
||
Title string `json:"title"`
|
||
}
|
||
|
||
var (
|
||
reGalleryURL = regexp.MustCompile(`e-hentai\.org/g/(\d+)/([0-9a-f]+)`)
|
||
reBareGID = regexp.MustCompile(`\b(\d{6,8})\b`)
|
||
reSplitter = regexp.MustCompile(`[\s,|;]+`)
|
||
)
|
||
|
||
// ParseGalleryInput 从粘贴的文本里提取画廊列表（去重，保持原顺序）
|
||
func ParseGalleryInput(text string) []GalleryRef {
|
||
var out []GalleryRef
|
||
seen := make(map[string]bool)
|
||
|
||
for _, rawLine := range strings.FieldsFunc(text, func(r rune) bool { return r == '\n' || r == '\r' }) {
|
||
line := strings.TrimSpace(rawLine)
|
||
if line == "" || strings.HasPrefix(line, "#") {
|
||
continue
|
||
}
|
||
for _, piece := range reSplitter.Split(line, -1) {
|
||
m := reGalleryURL.FindStringSubmatch(piece)
|
||
if m == nil {
|
||
continue
|
||
}
|
||
key := m[1] + "/" + m[2]
|
||
if seen[key] {
|
||
continue
|
||
}
|
||
seen[key] = true
|
||
out = append(out, GalleryRef{GID: m[1], Token: m[2], Raw: piece})
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
// ExtractGIDs 兜底：只给了裸数字的情况
|
||
func ExtractGIDs(text string) []string {
|
||
seen := map[string]bool{}
|
||
var out []string
|
||
for _, m := range reBareGID.FindAllStringSubmatch(text, -1) {
|
||
if !seen[m[1]] {
|
||
seen[m[1]] = true
|
||
out = append(out, m[1])
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
var reNumericEntity = regexp.MustCompile(`&#(x?[0-9a-fA-F]+);`)
|
||
|
||
// DecodeEntities 解 HTML 实体
|
||
func DecodeEntities(s string) string {
|
||
s = strings.ReplaceAll(s, "&", "&")
|
||
s = strings.ReplaceAll(s, "<", "<")
|
||
s = strings.ReplaceAll(s, ">", ">")
|
||
s = strings.ReplaceAll(s, """, `"`)
|
||
s = strings.ReplaceAll(s, "'", "'")
|
||
s = strings.ReplaceAll(s, " ", " ")
|
||
// 数字实体 &#NNN; 与 &#xHH;
|
||
s = reNumericEntity.ReplaceAllStringFunc(s, func(m string) string {
|
||
n := m[2 : len(m)-1]
|
||
base := 10
|
||
if strings.HasPrefix(strings.ToLower(n), "x") {
|
||
base = 16
|
||
n = n[1:]
|
||
}
|
||
if v, err := strconv.ParseInt(n, base, 32); err == nil && v > 0 && v < 0x110000 {
|
||
return string(rune(v))
|
||
}
|
||
return m
|
||
})
|
||
return s
|
||
}
|
||
|
||
// ---------------------------------------------------------------- 元信息
|
||
|
||
// GalleryMeta 画廊元信息，对应前端 {artist}/{group} 等占位符
|
||
type GalleryMeta struct {
|
||
Artist string `json:"artist"`
|
||
Group string `json:"group"`
|
||
Series string `json:"series"`
|
||
Category string `json:"category"`
|
||
Lang string `json:"lang"`
|
||
Uploader string `json:"uploader"`
|
||
Tags []string `json:"tags"`
|
||
ByNs map[string][]string `json:"byNs"`
|
||
}
|
||
|
||
// GalleryPageInfo 是「某页 -> /s/ 路径 + 原文件名」
|
||
type GalleryPageInfo struct {
|
||
Page int `json:"page"`
|
||
SPath string `json:"sPath"`
|
||
Filename string `json:"filename"`
|
||
}
|
||
|
||
// Gallery 是 FetchGalleryPage 的返回值
|
||
type Gallery struct {
|
||
GID string `json:"gid"`
|
||
Token string `json:"token"`
|
||
Title string `json:"title"`
|
||
TotalPages int `json:"totalPages"`
|
||
Pages []GalleryPageInfo `json:"pages"`
|
||
Meta GalleryMeta `json:"meta"`
|
||
}
|
||
|
||
var (
|
||
reTitleTag = regexp.MustCompile(`(?is)<title>(.*?)</title>`)
|
||
reMetaDesc = regexp.MustCompile(`(?i)<meta\s+name="description"\s+content="([^"]*)"`)
|
||
reTitleSuffix = regexp.MustCompile(`(?i)\s*-\s*E-Hentai Galleries\s*$`)
|
||
reTotal = regexp.MustCompile(`(?i)Showing\s+\d+\s*-\s*\d+\s+of\s+(\d+)\s+images`)
|
||
// <a href="/s/xxx"><div title="Page 1: 原名"
|
||
reThumbLink = regexp.MustCompile(`(?is)<a\s+href="((?:https://e-hentai\.org)?/s/[^"]+)"[^>]*>\s*<div\s+title="Page\s+(\d+):\s*([^"]*)"`)
|
||
reUploader = regexp.MustCompile(`(?i)/uploader/([^"']+)`)
|
||
reArtistBracket = regexp.MustCompile(`^\s*\[([^\]]{1,60})\]\s*`)
|
||
reGroupParen = regexp.MustCompile(`\(([^)]{1,60})\)`)
|
||
reCatFromDesc = regexp.MustCompile(`(?i)Hentai\s+([A-Za-z]+)\s+Gallery`)
|
||
reTitleFromDesc = regexp.MustCompile(`Gallery:\s*([^:]+):`)
|
||
reRestricted = regexp.MustCompile(`(?i)Login Required|restricted|Sad Panda`)
|
||
|
||
// #taglist 的每一行：<td class="tc">artist:</td><td><div class="gtl"><a ...>Noxe</a>...</td>
|
||
reTagRow = regexp.MustCompile(`(?is)<td\s+class="tc">\s*([a-z_]+):?\s*</td>\s*<td>(.*?)</td>`)
|
||
reTagLink = regexp.MustCompile(`(?is)<a\b[^>]*>([^<]+)</a>`)
|
||
|
||
// hath 全图 URL：随机子域 + 随机端口 + keystamp 签名
|
||
reHath = regexp.MustCompile(`(?i)https://[a-z0-9-]+(?:\.[a-z0-9-]+)*\.hath\.network(?::\d+)?/[^\s"'<>)\]]+`)
|
||
// 兜底：CDN 换域名时，从页面里抓第一个看起来像图片的 URL
|
||
reImgAny = regexp.MustCompile(`(?i)https?://[^\s"'<>)\]]+\.(?:png|jpe?g|webp|gif)(?:\?[^\s"'<>)\]]*)?`)
|
||
)
|
||
|
||
// parseMeta 从画廊页 HTML 的 #taglist 表格解析 艺术家/社团/分类/标签
|
||
func parseMeta(html, title string) GalleryMeta {
|
||
meta := GalleryMeta{
|
||
Tags: []string{},
|
||
ByNs: map[string][]string{},
|
||
}
|
||
|
||
if dm := reMetaDesc.FindStringSubmatch(html); dm != nil {
|
||
d := DecodeEntities(dm[1])
|
||
if cm := reCatFromDesc.FindStringSubmatch(d); cm != nil {
|
||
meta.Category = strings.ToLower(cm[1])
|
||
}
|
||
}
|
||
|
||
i := strings.Index(html, `id="taglist"`)
|
||
if i < 0 {
|
||
return finishMeta(meta, title, html)
|
||
}
|
||
seg := html[i:]
|
||
if len(seg) > 30000 {
|
||
seg = seg[:30000]
|
||
}
|
||
|
||
for _, row := range reTagRow.FindAllStringSubmatch(seg, -1) {
|
||
ns := strings.ToLower(row[1])
|
||
var names []string
|
||
for _, a := range reTagLink.FindAllStringSubmatch(row[2], -1) {
|
||
n := strings.TrimSpace(DecodeEntities(a[1]))
|
||
if n == "" || strings.HasPrefix(n, "tag:") {
|
||
continue
|
||
}
|
||
if strings.EqualFold(n, "more") || strings.EqualFold(n, "less") {
|
||
continue
|
||
}
|
||
names = append(names, n)
|
||
}
|
||
if len(names) == 0 {
|
||
continue
|
||
}
|
||
if _, ok := meta.ByNs[ns]; !ok {
|
||
meta.ByNs[ns] = names
|
||
}
|
||
|
||
switch ns {
|
||
case "artist":
|
||
if meta.Artist == "" {
|
||
meta.Artist = names[0]
|
||
}
|
||
case "group":
|
||
if meta.Group == "" {
|
||
meta.Group = names[0]
|
||
}
|
||
case "parody":
|
||
if meta.Series == "" {
|
||
meta.Series = names[0]
|
||
}
|
||
case "language":
|
||
if meta.Lang == "" {
|
||
meta.Lang = names[0]
|
||
}
|
||
default:
|
||
meta.Tags = append(meta.Tags, names...)
|
||
}
|
||
}
|
||
|
||
return finishMeta(meta, title, html)
|
||
}
|
||
|
||
// finishMeta 做三级兜底：EH 标题约定很可靠 —— [作者] 标题 (社团)
|
||
func finishMeta(meta GalleryMeta, title, html string) GalleryMeta {
|
||
if meta.Artist == "" && title != "" {
|
||
if m := reArtistBracket.FindStringSubmatch(title); m != nil {
|
||
meta.Artist = strings.TrimSpace(m[1])
|
||
}
|
||
}
|
||
if meta.Group == "" && title != "" {
|
||
if m := reGroupParen.FindStringSubmatch(title); m != nil {
|
||
meta.Group = strings.TrimSpace(m[1])
|
||
}
|
||
}
|
||
if up := reUploader.FindStringSubmatch(html); up != nil {
|
||
meta.Uploader = strings.TrimSpace(DecodeEntities(up[1]))
|
||
}
|
||
if meta.Artist == "" {
|
||
meta.Artist = meta.Uploader
|
||
}
|
||
if meta.Tags == nil {
|
||
meta.Tags = []string{}
|
||
}
|
||
return meta
|
||
}
|
||
|
||
// ---------------------------------------------------------------- Client
|
||
|
||
// Client 是 EH 协议客户端，并发安全
|
||
type Client struct {
|
||
mu sync.RWMutex
|
||
timeout time.Duration
|
||
proxy string
|
||
proxyMode string // off | http | socks5
|
||
xres string // auto | org | 数字(长边上限)
|
||
|
||
transport *http.Transport // 全局共用，只有代理配置变了才重建
|
||
limiter *Limiter
|
||
}
|
||
|
||
// NewClient 建客户端
|
||
func NewClient(timeout time.Duration, proxyURL, proxyMode, xres string) *Client {
|
||
if timeout <= 0 {
|
||
timeout = 30 * time.Second
|
||
}
|
||
if xres == "" {
|
||
xres = "auto"
|
||
}
|
||
return &Client{
|
||
timeout: timeout,
|
||
proxy: proxyURL,
|
||
proxyMode: proxyMode,
|
||
xres: xres,
|
||
limiter: NewLimiter(0),
|
||
}
|
||
}
|
||
|
||
// SetRequestInterval 设置全局请求间隔（0 = 不限流）
|
||
func (c *Client) SetRequestInterval(d time.Duration) {
|
||
if d < 0 {
|
||
d = 0
|
||
}
|
||
c.limiter.SetInterval(d)
|
||
}
|
||
|
||
// RequestInterval 当前请求间隔
|
||
func (c *Client) RequestInterval() time.Duration {
|
||
return c.limiter.Interval()
|
||
}
|
||
|
||
// SetProxy 改代理配置
|
||
func (c *Client) SetProxy(p, mode string) {
|
||
c.mu.Lock()
|
||
c.proxy = p
|
||
if mode != "" {
|
||
c.proxyMode = mode
|
||
}
|
||
c.mu.Unlock()
|
||
c.rebuildTransport() // 代理变了，Transport 必须重建
|
||
}
|
||
|
||
// SetXres 改分辨率策略
|
||
func (c *Client) SetXres(x string) {
|
||
c.mu.Lock()
|
||
c.xres = x
|
||
c.mu.Unlock()
|
||
}
|
||
|
||
// SetTimeout 改超时
|
||
func (c *Client) SetTimeout(d time.Duration) {
|
||
c.mu.Lock()
|
||
c.timeout = d
|
||
c.mu.Unlock()
|
||
}
|
||
|
||
func (c *Client) snapshot() (string, string, time.Duration) {
|
||
c.mu.RLock()
|
||
defer c.mu.RUnlock()
|
||
return c.proxy, c.proxyMode, c.timeout
|
||
}
|
||
|
||
// httpClient 取可用的 http.Client。
|
||
//
|
||
// ⚠️ Transport 必须复用：如果每次请求都新建 Transport，旧的那个会被丢弃，
|
||
// 但它持有的 keep-alive 连接不会立刻释放（要等 IdleConnTimeout），
|
||
// 于是每请求一条僵尸连接堆积起来 —— 实测能到 91 条，
|
||
// 既吃代理的连接配额，又让新请求变慢。
|
||
// 所以这里只在「代理配置变化」时才重建 Transport，平时全局共用一个。
|
||
func (c *Client) httpClient(timeout time.Duration) *http.Client {
|
||
c.mu.RLock()
|
||
tr := c.transport
|
||
defTimeout := c.timeout
|
||
c.mu.RUnlock()
|
||
|
||
if tr == nil {
|
||
tr = c.rebuildTransport()
|
||
}
|
||
|
||
if timeout <= 0 {
|
||
timeout = defTimeout
|
||
}
|
||
return &http.Client{Transport: tr, Timeout: timeout}
|
||
}
|
||
|
||
// rebuildTransport 按当前代理配置重建 Transport，并关掉旧连接的空闲池
|
||
func (c *Client) rebuildTransport() *http.Transport {
|
||
proxyURL, mode, _ := c.snapshot()
|
||
|
||
tr := &http.Transport{
|
||
MaxIdleConns: 64,
|
||
MaxIdleConnsPerHost: 16,
|
||
IdleConnTimeout: 90 * time.Second,
|
||
TLSHandshakeTimeout: 10 * time.Second,
|
||
}
|
||
|
||
if proxyURL != "" && mode != "off" && mode != "" {
|
||
switch mode {
|
||
case "socks5":
|
||
raw := strings.TrimPrefix(strings.TrimPrefix(proxyURL, "socks5://"), "socks5h://")
|
||
var auth *proxy.Auth
|
||
if u, err := url.Parse("socks5://" + raw); err == nil && u.User != nil {
|
||
pw, _ := u.User.Password()
|
||
auth = &proxy.Auth{User: u.User.Username(), Password: pw}
|
||
}
|
||
if dialer, err := proxy.SOCKS5("tcp", raw, auth, proxy.Direct); err == nil {
|
||
tr.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||
return dialer.Dial(network, addr)
|
||
}
|
||
}
|
||
default: // http
|
||
if p, err := url.Parse(proxyURL); err == nil {
|
||
tr.Proxy = http.ProxyURL(p)
|
||
}
|
||
}
|
||
}
|
||
|
||
c.mu.Lock()
|
||
old := c.transport
|
||
c.transport = tr
|
||
c.mu.Unlock()
|
||
|
||
if old != nil {
|
||
old.CloseIdleConnections()
|
||
}
|
||
return tr
|
||
}
|
||
|
||
// Close 关闭空闲连接（退出时调用）
|
||
func (c *Client) Close() {
|
||
c.mu.Lock()
|
||
tr := c.transport
|
||
c.transport = nil
|
||
c.mu.Unlock()
|
||
if tr != nil {
|
||
tr.CloseIdleConnections()
|
||
}
|
||
}
|
||
|
||
func (c *Client) get(ctx context.Context, u string, timeout time.Duration) (*http.Response, error) {
|
||
// 全局限流：所有请求（含 /s/ 解析与画廊页）统一排队
|
||
if err := c.limiter.Wait(ctx); err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
req.Header.Set("User-Agent", defaultUA)
|
||
req.Header.Set("Accept", "*/*")
|
||
|
||
resp, err := c.httpClient(timeout).Do(req)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if resp.StatusCode < 200 || resp.StatusCode >= 400 {
|
||
resp.Body.Close()
|
||
return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, u)
|
||
}
|
||
return resp, nil
|
||
}
|
||
|
||
func (c *Client) getText(ctx context.Context, u string) (string, error) {
|
||
resp, err := c.get(ctx, u, 0)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
defer resp.Body.Close()
|
||
b, err := io.ReadAll(resp.Body)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
return string(b), nil
|
||
}
|
||
|
||
// ---------------------------------------------------------------- 1) 画廊页
|
||
|
||
// ParseProgress 解析进度回调
|
||
//
|
||
// 大画廊要翻很多页（2000 张 = 100 页），没有进度反馈的话界面会一直显示
|
||
// 「解析中 0/?」，看着像卡死了。
|
||
//
|
||
// done = 已收集到的图片数
|
||
// total = 站点声明的总图数（第一页才知道，可能为 0）
|
||
// page = 当前翻到第几屏（每屏 20 张）
|
||
type ParseProgress func(done, total, page int)
|
||
|
||
// FetchGalleryPage 取画廊页。注意：EH 每屏只列 20 张缩略图，
|
||
// 必须翻 ?p=N 逐屏收集，否则静默只下到前 20 张且不报错。
|
||
func (c *Client) FetchGalleryPage(ctx context.Context, gid, token string, onProgress ParseProgress) (*Gallery, error) {
|
||
var (
|
||
pages []GalleryPageInfo
|
||
seen = map[int]bool{}
|
||
total int
|
||
title string
|
||
lastHTML string
|
||
firstHTML string
|
||
)
|
||
|
||
for p := 0; p < 500; p++ {
|
||
if err := ctx.Err(); err != nil {
|
||
return nil, err
|
||
}
|
||
u := EHBase + "/g/" + gid + "/" + token + "/"
|
||
if p > 0 {
|
||
u += "?p=" + strconv.Itoa(p)
|
||
}
|
||
html, err := c.getText(ctx, u)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("解析画廊页失败: %w", err)
|
||
}
|
||
lastHTML = html
|
||
|
||
if p == 0 {
|
||
if len(html) < 2000 && strings.Contains(html, "404 Not Found") {
|
||
return nil, fmt.Errorf("画廊不存在或 token 失效: %s", gid)
|
||
}
|
||
if m := reTitleTag.FindStringSubmatch(html); m != nil {
|
||
title = strings.TrimSpace(reTitleSuffix.ReplaceAllString(DecodeEntities(m[1]), ""))
|
||
}
|
||
if m := reMetaDesc.FindStringSubmatch(html); m != nil {
|
||
if nm := reTitleFromDesc.FindStringSubmatch(DecodeEntities(m[1])); nm != nil {
|
||
title = strings.TrimSpace(nm[1])
|
||
}
|
||
}
|
||
if m := reTotal.FindStringSubmatch(html); m != nil {
|
||
total, _ = strconv.Atoi(m[1])
|
||
}
|
||
firstHTML = html // 标签/作者等元信息只在第一页有
|
||
}
|
||
|
||
added := 0
|
||
for _, m := range reThumbLink.FindAllStringSubmatch(html, -1) {
|
||
page, _ := strconv.Atoi(m[2])
|
||
if seen[page] {
|
||
continue
|
||
}
|
||
seen[page] = true
|
||
added++
|
||
|
||
sPath := m[1]
|
||
if !strings.HasPrefix(sPath, "http") {
|
||
sPath = EHBase + sPath
|
||
}
|
||
name := strings.TrimSpace(DecodeEntities(m[3]))
|
||
if name == "" {
|
||
name = gid + "-" + strconv.Itoa(page) + ".webp"
|
||
}
|
||
pages = append(pages, GalleryPageInfo{Page: page, SPath: sPath, Filename: name})
|
||
}
|
||
|
||
if added == 0 {
|
||
break // 这屏没新东西 -> 到头了
|
||
}
|
||
|
||
// 报告进度：大画廊翻几十上百屏时，界面上要看得见在动
|
||
// （放在 break 之前，保证最后一屏也会报告）
|
||
if onProgress != nil {
|
||
onProgress(len(pages), total, p+1)
|
||
}
|
||
|
||
if total > 0 && len(pages) >= total {
|
||
break // 收齐
|
||
}
|
||
}
|
||
|
||
if len(pages) == 0 {
|
||
if reRestricted.MatchString(lastHTML) {
|
||
return nil, fmt.Errorf("画廊受限或不存在（无 cookie 访问不了）: %s", gid)
|
||
}
|
||
return nil, fmt.Errorf("页面结构无法解析（EH 可能又改版）: %s", gid)
|
||
}
|
||
|
||
// 按页码排序（插入排序，页数通常几十到几百）
|
||
for i := 1; i < len(pages); i++ {
|
||
for j := i; j > 0 && pages[j].Page < pages[j-1].Page; j-- {
|
||
pages[j], pages[j-1] = pages[j-1], pages[j]
|
||
}
|
||
}
|
||
|
||
if title == "" {
|
||
title = gid
|
||
}
|
||
if total == 0 {
|
||
total = len(pages)
|
||
}
|
||
|
||
return &Gallery{
|
||
GID: gid,
|
||
Token: token,
|
||
Title: title,
|
||
TotalPages: total,
|
||
Pages: pages,
|
||
Meta: parseMeta(firstHTML, title),
|
||
}, nil
|
||
}
|
||
|
||
// ---------------------------------------------------------------- 2) 图片直链
|
||
|
||
// ResolveImageURL 把 /s/ 链接解析成 hath 全图 URL。
|
||
//
|
||
// 这一步就是所谓「图片直链」的全部：GET 一次 /s/ 页面，
|
||
// 用正则从 HTML 里抓出那条 hath.network URL。没有签名计算、没有解码算法，
|
||
// 因为 URL 每次都不一样（随机子域+随机端口+keystamp 时间戳），只能实时取。
|
||
func (c *Client) ResolveImageURL(ctx context.Context, sPath string) (string, error) {
|
||
html, err := c.getText(ctx, sPath)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
if m := reHath.FindString(html); m != "" {
|
||
return c.applyXres(DecodeEntities(m)), nil
|
||
}
|
||
// CDN 换域名时的兜底：抓页面上第一个图片 URL
|
||
if m := reImgAny.FindString(html); m != "" {
|
||
return c.applyXres(DecodeEntities(m)), nil
|
||
}
|
||
return "", fmt.Errorf("未能从 /s/ 页面解析出图片地址: %s", sPath)
|
||
}
|
||
|
||
var (
|
||
reXresNum = regexp.MustCompile(`xres=\d+`)
|
||
reXresAny = regexp.MustCompile(`xres=[^/]+`)
|
||
)
|
||
|
||
// applyXres 改写 URL 里的 xres 参数（数字=长边上限，org=原图）
|
||
func (c *Client) applyXres(u string) string {
|
||
c.mu.RLock()
|
||
x := c.xres
|
||
c.mu.RUnlock()
|
||
|
||
if x == "" || x == "auto" {
|
||
return u
|
||
}
|
||
if _, err := strconv.Atoi(x); err == nil {
|
||
return reXresNum.ReplaceAllString(u, "xres="+x)
|
||
}
|
||
if x == "org" {
|
||
return reXresAny.ReplaceAllString(u, "xres=org")
|
||
}
|
||
return u
|
||
}
|
||
|
||
// ---------------------------------------------------------------- 3) 下载
|
||
|
||
// DownloadImage 流式下载到 destPath（先写 .part 再 rename，保证原子性）
|
||
func (c *Client) DownloadImage(ctx context.Context, imgURL, destPath string, onProgress func(received, total int64)) (int64, error) {
|
||
_, _, baseTimeout := c.snapshot()
|
||
|
||
// 图片本体也要限流
|
||
if err := c.limiter.Wait(ctx); err != nil {
|
||
return 0, err
|
||
}
|
||
|
||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, imgURL, nil)
|
||
if err != nil {
|
||
return 0, err
|
||
}
|
||
req.Header.Set("User-Agent", defaultUA)
|
||
req.Header.Set("Referer", EHBase+"/")
|
||
|
||
resp, err := c.httpClient(baseTimeout * 2).Do(req)
|
||
if err != nil {
|
||
return 0, err
|
||
}
|
||
defer resp.Body.Close()
|
||
if resp.StatusCode < 200 || resp.StatusCode >= 400 {
|
||
return 0, fmt.Errorf("HTTP %d 下载图片失败", resp.StatusCode)
|
||
}
|
||
|
||
tmp := destPath + ".part"
|
||
f, err := os.Create(tmp)
|
||
if err != nil {
|
||
return 0, err
|
||
}
|
||
|
||
received, copyErr := io.Copy(f, &progressReader{
|
||
r: resp.Body, total: resp.ContentLength, onProgress: onProgress,
|
||
})
|
||
closeErr := f.Close()
|
||
if copyErr != nil {
|
||
os.Remove(tmp)
|
||
return 0, copyErr
|
||
}
|
||
if closeErr != nil {
|
||
os.Remove(tmp)
|
||
return 0, closeErr
|
||
}
|
||
if received == 0 {
|
||
os.Remove(tmp)
|
||
return 0, errors.New("下载内容为空")
|
||
}
|
||
if err := os.Rename(tmp, destPath); err != nil {
|
||
os.Remove(tmp)
|
||
return 0, err
|
||
}
|
||
return received, nil
|
||
}
|
||
|
||
// progressReader 包一层统计下载进度
|
||
type progressReader struct {
|
||
r io.Reader
|
||
total int64
|
||
received int64
|
||
onProgress func(received, total int64)
|
||
}
|
||
|
||
func (p *progressReader) Read(b []byte) (int, error) {
|
||
n, err := p.r.Read(b)
|
||
if n > 0 {
|
||
p.received += int64(n)
|
||
if p.onProgress != nil {
|
||
p.onProgress(p.received, p.total)
|
||
}
|
||
}
|
||
return n, err
|
||
}
|