meta description 的真实格式是 "Free Hentai Misc Gallery: <title> - Tags: a, b, c" 旧正则 `Gallery:\s*([^:]+):` 里的 [^:]+ 是贪婪的, 一路吃到标签段自己的冒号为止,把 " - Tags" 也吞进了标题, 导致下载目录/文件名结尾凭空多出 " - Tags"。 改为显式认 " - Tags:" 这个分隔符(Tags 段可选,无标签时认行尾), reTitleSuffix 同时认 Tags 结尾,提取后再兜底剥一次。 线上实测(g/4239272):标题由 "Hinata l Boruto (CriticalHAI) [AI Generated] - Tags" 变为 "Hinata l Boruto (CriticalHAI) [AI Generated]" 附带 remove-tags-suffix.bat/.ps1:递归去掉已有压缩包文件名结尾的 " - Tags",覆盖 zip/rar/7z/cbz/cbr/cb7/tar/gz,幂等。
752 lines
22 KiB
Go
752 lines
22 KiB
Go
package eh
|
||
|
||
// E-Hentai 抓取核心(2026 版协议)
|
||
//
|
||
// 已实测的链路:
|
||
// 1) GET https://e-hentai.org/g/{gid}/{token}/ 拿全部页的 /s/ 链接 + 原文件名
|
||
// 2) GET https://e-hentai.org/s/{key}/{gid}-{page} 拿该页 hath 全图 URL(key 必须与 page 配对)
|
||
// 3) GET <hath 图 URL> 下载图片本身
|
||
//
|
||
// 老协议(var gmetadata / /img/{gid}/{gid}/{gid}-{page}.jpg / api.php gdata)全部失效。
|
||
//
|
||
// 关于「图片直链」:hath 的 URL 是 随机子域 + 随机端口 + keystamp 时间戳签名,
|
||
// 不存在可推算的固定直链,只能每次实时从 /s/ 页面解析。所以这里没有「解码」,
|
||
// 只有「从 HTML 里提取」—— 见 ResolveImageURL。
|
||
|
||
import (
|
||
"context"
|
||
"errors"
|
||
"fmt"
|
||
"io"
|
||
"net"
|
||
"net/http"
|
||
"net/http/cookiejar"
|
||
"net/url"
|
||
"os"
|
||
"regexp"
|
||
"strconv"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
|
||
"golang.org/x/net/proxy"
|
||
)
|
||
|
||
const (
|
||
// Referer 不是必需的(实测裸请求也能 200),但带上更稳。
|
||
defaultUA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " +
|
||
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||
)
|
||
|
||
// EHBase 站点根地址(测试里会临时替换成 mock 服务器)
|
||
var EHBase = "https://e-hentai.org"
|
||
|
||
// ---------------------------------------------------------------- 输入解析
|
||
|
||
// GalleryRef 是一个待下载的画廊(gid + token 缺一不可,只有 gid 会 404)
|
||
type GalleryRef struct {
|
||
GID string `json:"gid"`
|
||
Token string `json:"token"`
|
||
Raw string `json:"raw"`
|
||
Title string `json:"title"`
|
||
}
|
||
|
||
var (
|
||
reGalleryURL = regexp.MustCompile(`e-hentai\.org/g/(\d+)/([0-9a-f]+)`)
|
||
reBareGID = regexp.MustCompile(`\b(\d{6,8})\b`)
|
||
reSplitter = regexp.MustCompile(`[\s,|;]+`)
|
||
)
|
||
|
||
// ParseGalleryInput 从粘贴的文本里提取画廊列表(去重,保持原顺序)
|
||
func ParseGalleryInput(text string) []GalleryRef {
|
||
var out []GalleryRef
|
||
seen := make(map[string]bool)
|
||
|
||
for _, rawLine := range strings.FieldsFunc(text, func(r rune) bool { return r == '\n' || r == '\r' }) {
|
||
line := strings.TrimSpace(rawLine)
|
||
if line == "" || strings.HasPrefix(line, "#") {
|
||
continue
|
||
}
|
||
for _, piece := range reSplitter.Split(line, -1) {
|
||
m := reGalleryURL.FindStringSubmatch(piece)
|
||
if m == nil {
|
||
continue
|
||
}
|
||
key := m[1] + "/" + m[2]
|
||
if seen[key] {
|
||
continue
|
||
}
|
||
seen[key] = true
|
||
out = append(out, GalleryRef{GID: m[1], Token: m[2], Raw: piece})
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
// ExtractGIDs 兜底:只给了裸数字的情况
|
||
func ExtractGIDs(text string) []string {
|
||
seen := map[string]bool{}
|
||
var out []string
|
||
for _, m := range reBareGID.FindAllStringSubmatch(text, -1) {
|
||
if !seen[m[1]] {
|
||
seen[m[1]] = true
|
||
out = append(out, m[1])
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
var reNumericEntity = regexp.MustCompile(`&#(x?[0-9a-fA-F]+);`)
|
||
|
||
// DecodeEntities 解 HTML 实体
|
||
func DecodeEntities(s string) string {
|
||
s = strings.ReplaceAll(s, "&", "&")
|
||
s = strings.ReplaceAll(s, "<", "<")
|
||
s = strings.ReplaceAll(s, ">", ">")
|
||
s = strings.ReplaceAll(s, """, `"`)
|
||
s = strings.ReplaceAll(s, "'", "'")
|
||
s = strings.ReplaceAll(s, " ", " ")
|
||
// 数字实体 &#NNN; 与 &#xHH;
|
||
s = reNumericEntity.ReplaceAllStringFunc(s, func(m string) string {
|
||
n := m[2 : len(m)-1]
|
||
base := 10
|
||
if strings.HasPrefix(strings.ToLower(n), "x") {
|
||
base = 16
|
||
n = n[1:]
|
||
}
|
||
if v, err := strconv.ParseInt(n, base, 32); err == nil && v > 0 && v < 0x110000 {
|
||
return string(rune(v))
|
||
}
|
||
return m
|
||
})
|
||
return s
|
||
}
|
||
|
||
// ---------------------------------------------------------------- 元信息
|
||
|
||
// GalleryMeta 画廊元信息,对应前端 {artist}/{group} 等占位符
|
||
type GalleryMeta struct {
|
||
Artist string `json:"artist"`
|
||
Group string `json:"group"`
|
||
Series string `json:"series"`
|
||
Category string `json:"category"`
|
||
Lang string `json:"lang"`
|
||
Uploader string `json:"uploader"`
|
||
Tags []string `json:"tags"`
|
||
ByNs map[string][]string `json:"byNs"`
|
||
}
|
||
|
||
// GalleryPageInfo 是「某页 -> /s/ 路径 + 原文件名」
|
||
type GalleryPageInfo struct {
|
||
Page int `json:"page"`
|
||
SPath string `json:"sPath"`
|
||
Filename string `json:"filename"`
|
||
}
|
||
|
||
// Gallery 是 FetchGalleryPage 的返回值
|
||
type Gallery struct {
|
||
GID string `json:"gid"`
|
||
Token string `json:"token"`
|
||
Title string `json:"title"`
|
||
TotalPages int `json:"totalPages"`
|
||
Pages []GalleryPageInfo `json:"pages"`
|
||
Meta GalleryMeta `json:"meta"`
|
||
}
|
||
|
||
var (
|
||
reTitleTag = regexp.MustCompile(`(?is)<title>(.*?)</title>`)
|
||
reMetaDesc = regexp.MustCompile(`(?i)<meta\s+name="description"\s+content="([^"]*)"`)
|
||
reTitleSuffix = regexp.MustCompile(`(?i)\s*-\s*(?:E-Hentai Galleries|Tags)\s*$`)
|
||
reTotal = regexp.MustCompile(`(?i)Showing\s+\d+\s*-\s*\d+\s+of\s+(\d+)\s+images`)
|
||
// <a href="/s/xxx"><div title="Page 1: 原名"
|
||
reThumbLink = regexp.MustCompile(`(?is)<a\s+href="((?:https://e-hentai\.org)?/s/[^"]+)"[^>]*>\s*<div\s+title="Page\s+(\d+):\s*([^"]*)"`)
|
||
reUploader = regexp.MustCompile(`(?i)/uploader/([^"']+)`)
|
||
reArtistBracket = regexp.MustCompile(`^\s*\[([^\]]{1,60})\]\s*`)
|
||
reGroupParen = regexp.MustCompile(`\(([^)]{1,60})\)`)
|
||
reCatFromDesc = regexp.MustCompile(`(?i)Hentai\s+([A-Za-z]+)\s+Gallery`)
|
||
// meta description 形如:
|
||
// "Free Hentai Misc Gallery: <title> - Tags: a, b, c"
|
||
// 标题的结束边界优先认 " - Tags:"(标签段);没有 Tags 段时认行尾。
|
||
// ⚠️ 别用 `Gallery:\s*([^:]+):` —— 那个会把 " - Tags" 一起吞进标题。
|
||
reTitleFromDesc = regexp.MustCompile(`(?is)Gallery:\s*(.*?)(?:\s*-\s*Tags\s*:|\s*$|[.:]\s+\d+\s+images\b)`)
|
||
reRestricted = regexp.MustCompile(`(?i)Login Required|restricted|Sad Panda`)
|
||
|
||
// #taglist 的每一行:<td class="tc">artist:</td><td><div class="gtl"><a ...>Noxe</a>...</td>
|
||
reTagRow = regexp.MustCompile(`(?is)<td\s+class="tc">\s*([a-z_]+):?\s*</td>\s*<td>(.*?)</td>`)
|
||
reTagLink = regexp.MustCompile(`(?is)<a\b[^>]*>([^<]+)</a>`)
|
||
|
||
// hath 全图 URL:随机子域 + 随机端口 + keystamp 签名
|
||
reHath = regexp.MustCompile(`(?i)https://[a-z0-9-]+(?:\.[a-z0-9-]+)*\.hath\.network(?::\d+)?/[^\s"'<>)\]]+`)
|
||
// 兜底:CDN 换域名时,从页面里抓第一个看起来像图片的 URL
|
||
reImgAny = regexp.MustCompile(`(?i)https?://[^\s"'<>)\]]+\.(?:png|jpe?g|webp|gif)(?:\?[^\s"'<>)\]]*)?`)
|
||
)
|
||
|
||
// parseMeta 从画廊页 HTML 的 #taglist 表格解析 艺术家/社团/分类/标签
|
||
func parseMeta(html, title string) GalleryMeta {
|
||
meta := GalleryMeta{
|
||
Tags: []string{},
|
||
ByNs: map[string][]string{},
|
||
}
|
||
|
||
if dm := reMetaDesc.FindStringSubmatch(html); dm != nil {
|
||
d := DecodeEntities(dm[1])
|
||
if cm := reCatFromDesc.FindStringSubmatch(d); cm != nil {
|
||
meta.Category = strings.ToLower(cm[1])
|
||
}
|
||
}
|
||
|
||
i := strings.Index(html, `id="taglist"`)
|
||
if i < 0 {
|
||
return finishMeta(meta, title, html)
|
||
}
|
||
seg := html[i:]
|
||
if len(seg) > 30000 {
|
||
seg = seg[:30000]
|
||
}
|
||
|
||
for _, row := range reTagRow.FindAllStringSubmatch(seg, -1) {
|
||
ns := strings.ToLower(row[1])
|
||
var names []string
|
||
for _, a := range reTagLink.FindAllStringSubmatch(row[2], -1) {
|
||
n := strings.TrimSpace(DecodeEntities(a[1]))
|
||
if n == "" || strings.HasPrefix(n, "tag:") {
|
||
continue
|
||
}
|
||
if strings.EqualFold(n, "more") || strings.EqualFold(n, "less") {
|
||
continue
|
||
}
|
||
names = append(names, n)
|
||
}
|
||
if len(names) == 0 {
|
||
continue
|
||
}
|
||
if _, ok := meta.ByNs[ns]; !ok {
|
||
meta.ByNs[ns] = names
|
||
}
|
||
|
||
switch ns {
|
||
case "artist":
|
||
if meta.Artist == "" {
|
||
meta.Artist = names[0]
|
||
}
|
||
case "group":
|
||
if meta.Group == "" {
|
||
meta.Group = names[0]
|
||
}
|
||
case "parody":
|
||
if meta.Series == "" {
|
||
meta.Series = names[0]
|
||
}
|
||
case "language":
|
||
if meta.Lang == "" {
|
||
meta.Lang = names[0]
|
||
}
|
||
default:
|
||
meta.Tags = append(meta.Tags, names...)
|
||
}
|
||
}
|
||
|
||
return finishMeta(meta, title, html)
|
||
}
|
||
|
||
// finishMeta 做三级兜底:EH 标题约定很可靠 —— [作者] 标题 (社团)
|
||
func finishMeta(meta GalleryMeta, title, html string) GalleryMeta {
|
||
if meta.Artist == "" && title != "" {
|
||
if m := reArtistBracket.FindStringSubmatch(title); m != nil {
|
||
meta.Artist = strings.TrimSpace(m[1])
|
||
}
|
||
}
|
||
if meta.Group == "" && title != "" {
|
||
if m := reGroupParen.FindStringSubmatch(title); m != nil {
|
||
meta.Group = strings.TrimSpace(m[1])
|
||
}
|
||
}
|
||
if up := reUploader.FindStringSubmatch(html); up != nil {
|
||
meta.Uploader = strings.TrimSpace(DecodeEntities(up[1]))
|
||
}
|
||
if meta.Artist == "" {
|
||
meta.Artist = meta.Uploader
|
||
}
|
||
if meta.Tags == nil {
|
||
meta.Tags = []string{}
|
||
}
|
||
return meta
|
||
}
|
||
|
||
// ---------------------------------------------------------------- Client
|
||
|
||
// Client 是 EH 协议客户端,并发安全
|
||
type Client struct {
|
||
mu sync.RWMutex
|
||
timeout time.Duration
|
||
proxy string
|
||
proxyMode string // off | http | socks5
|
||
xres string // auto | org | 数字(长边上限)
|
||
|
||
transport *http.Transport // 全局共用,只有代理配置变了才重建
|
||
limiter *Limiter
|
||
|
||
// jar 必须有:EH 对部分画廊先弹「Content Warning」拦截页,
|
||
// 放行靠的是它下发的 nw=1 Cookie。没有 jar 的话每次请求都是
|
||
// 无状态连接,Cookie 下发即丢,于是永远过不去(2026-10 修)。
|
||
jar http.CookieJar
|
||
}
|
||
|
||
// NewClient 建客户端
|
||
func NewClient(timeout time.Duration, proxyURL, proxyMode, xres string) *Client {
|
||
if timeout <= 0 {
|
||
timeout = 30 * time.Second
|
||
}
|
||
if xres == "" {
|
||
xres = "auto"
|
||
}
|
||
jar, _ := cookiejar.New(nil) // cookiejar.New 只在 nil options 时出错
|
||
return &Client{
|
||
timeout: timeout,
|
||
proxy: proxyURL,
|
||
proxyMode: proxyMode,
|
||
xres: xres,
|
||
limiter: NewLimiter(0),
|
||
jar: jar,
|
||
}
|
||
}
|
||
|
||
// SetRequestInterval 设置全局请求间隔(0 = 不限流)
|
||
func (c *Client) SetRequestInterval(d time.Duration) {
|
||
if d < 0 {
|
||
d = 0
|
||
}
|
||
c.limiter.SetInterval(d)
|
||
}
|
||
|
||
// RequestInterval 当前请求间隔
|
||
func (c *Client) RequestInterval() time.Duration {
|
||
return c.limiter.Interval()
|
||
}
|
||
|
||
// SetProxy 改代理配置
|
||
func (c *Client) SetProxy(p, mode string) {
|
||
c.mu.Lock()
|
||
c.proxy = p
|
||
if mode != "" {
|
||
c.proxyMode = mode
|
||
}
|
||
c.mu.Unlock()
|
||
c.rebuildTransport() // 代理变了,Transport 必须重建
|
||
}
|
||
|
||
// SetXres 改分辨率策略
|
||
func (c *Client) SetXres(x string) {
|
||
c.mu.Lock()
|
||
c.xres = x
|
||
c.mu.Unlock()
|
||
}
|
||
|
||
// SetTimeout 改超时
|
||
func (c *Client) SetTimeout(d time.Duration) {
|
||
c.mu.Lock()
|
||
c.timeout = d
|
||
c.mu.Unlock()
|
||
}
|
||
|
||
func (c *Client) snapshot() (string, string, time.Duration) {
|
||
c.mu.RLock()
|
||
defer c.mu.RUnlock()
|
||
return c.proxy, c.proxyMode, c.timeout
|
||
}
|
||
|
||
// httpClient 取可用的 http.Client。
|
||
//
|
||
// ⚠️ Transport 必须复用:如果每次请求都新建 Transport,旧的那个会被丢弃,
|
||
// 但它持有的 keep-alive 连接不会立刻释放(要等 IdleConnTimeout),
|
||
// 于是每请求一条僵尸连接堆积起来 —— 实测能到 91 条,
|
||
// 既吃代理的连接配额,又让新请求变慢。
|
||
// 所以这里只在「代理配置变化」时才重建 Transport,平时全局共用一个。
|
||
func (c *Client) httpClient(timeout time.Duration) *http.Client {
|
||
c.mu.RLock()
|
||
tr := c.transport
|
||
defTimeout := c.timeout
|
||
c.mu.RUnlock()
|
||
|
||
if tr == nil {
|
||
tr = c.rebuildTransport()
|
||
}
|
||
|
||
if timeout <= 0 {
|
||
timeout = defTimeout
|
||
}
|
||
// Jar 挂在 Client 上(Go 1.26 起 http.Transport 已经没有 Jar 字段了),
|
||
// 少一个 jar 就会退回无状态模式,EH 的 nw=1 记不住。
|
||
return &http.Client{Transport: tr, Timeout: timeout, Jar: c.jar}
|
||
}
|
||
|
||
// rebuildTransport 按当前代理配置重建 Transport,并关掉旧连接的空闲池
|
||
func (c *Client) rebuildTransport() *http.Transport {
|
||
proxyURL, mode, _ := c.snapshot()
|
||
|
||
tr := &http.Transport{
|
||
MaxIdleConns: 64,
|
||
MaxIdleConnsPerHost: 16,
|
||
IdleConnTimeout: 90 * time.Second,
|
||
TLSHandshakeTimeout: 10 * time.Second,
|
||
}
|
||
|
||
if proxyURL != "" && mode != "off" && mode != "" {
|
||
switch mode {
|
||
case "socks5":
|
||
raw := strings.TrimPrefix(strings.TrimPrefix(proxyURL, "socks5://"), "socks5h://")
|
||
var auth *proxy.Auth
|
||
if u, err := url.Parse("socks5://" + raw); err == nil && u.User != nil {
|
||
pw, _ := u.User.Password()
|
||
auth = &proxy.Auth{User: u.User.Username(), Password: pw}
|
||
}
|
||
if dialer, err := proxy.SOCKS5("tcp", raw, auth, proxy.Direct); err == nil {
|
||
tr.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||
return dialer.Dial(network, addr)
|
||
}
|
||
}
|
||
default: // http
|
||
if p, err := url.Parse(proxyURL); err == nil {
|
||
tr.Proxy = http.ProxyURL(p)
|
||
}
|
||
}
|
||
}
|
||
|
||
c.mu.Lock()
|
||
old := c.transport
|
||
c.transport = tr
|
||
c.mu.Unlock()
|
||
|
||
if old != nil {
|
||
old.CloseIdleConnections()
|
||
}
|
||
return tr
|
||
}
|
||
|
||
// Close 关闭空闲连接(退出时调用)
|
||
func (c *Client) Close() {
|
||
c.mu.Lock()
|
||
tr := c.transport
|
||
c.transport = nil
|
||
c.mu.Unlock()
|
||
if tr != nil {
|
||
tr.CloseIdleConnections()
|
||
}
|
||
}
|
||
|
||
func (c *Client) get(ctx context.Context, u string, timeout time.Duration) (*http.Response, error) {
|
||
// 全局限流:所有请求(含 /s/ 解析与画廊页)统一排队
|
||
if err := c.limiter.Wait(ctx); err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
req.Header.Set("User-Agent", defaultUA)
|
||
req.Header.Set("Accept", "*/*")
|
||
|
||
resp, err := c.httpClient(timeout).Do(req)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if resp.StatusCode < 200 || resp.StatusCode >= 400 {
|
||
resp.Body.Close()
|
||
return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, u)
|
||
}
|
||
return resp, nil
|
||
}
|
||
|
||
func (c *Client) getText(ctx context.Context, u string) (string, error) {
|
||
resp, err := c.get(ctx, u, 0)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
defer resp.Body.Close()
|
||
b, err := io.ReadAll(resp.Body)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
return string(b), nil
|
||
}
|
||
|
||
// reContentWarning EH 的「Content Warning」拦截页特征。
|
||
//
|
||
// 这类画廊(多被标成 Offensive For Everyone)第一次访问只会拿到一个
|
||
// 3~4KB 的黄框页,标题和 meta 都在,但**一个 /s/ 链接都没有**,
|
||
// 于是旧代码直接报「页面结构无法解析(EH 可能又改版)」——纯误导。
|
||
var reContentWarning = regexp.MustCompile(`(?i)<h1>\s*Content Warning\s*</h1>|nw=always`)
|
||
|
||
func isContentWarning(html string) bool {
|
||
return reContentWarning.MatchString(html)
|
||
}
|
||
|
||
// getTextAutoWarn 抓文本;撞上内容警告页就自动放行一次再重试。
|
||
//
|
||
// 放行方式是请求一次 ?nw=always —— EH 会种下 nw=1 Cookie,
|
||
// 而 jar 会把它记住,之后所有画廊页、/s/ 页都不用再警告。
|
||
func (c *Client) getTextAutoWarn(ctx context.Context, u string) (string, error) {
|
||
html, err := c.getText(ctx, u)
|
||
if err != nil || !isContentWarning(html) {
|
||
return html, err
|
||
}
|
||
|
||
sep := "?"
|
||
if strings.Contains(u, "?") {
|
||
sep = "&"
|
||
}
|
||
if _, werr := c.getText(ctx, u+sep+"nw=always"); werr != nil {
|
||
return html, nil // 放行失败就原样返回,交给调用方判断
|
||
}
|
||
return c.getText(ctx, u)
|
||
}
|
||
|
||
// ---------------------------------------------------------------- 1) 画廊页
|
||
|
||
// ParseProgress 解析进度回调
|
||
//
|
||
// 大画廊要翻很多页(2000 张 = 100 页),没有进度反馈的话界面会一直显示
|
||
// 「解析中 0/?」,看着像卡死了。
|
||
//
|
||
// done = 已收集到的图片数
|
||
// total = 站点声明的总图数(第一页才知道,可能为 0)
|
||
// page = 当前翻到第几屏(每屏 20 张)
|
||
type ParseProgress func(done, total, page int)
|
||
|
||
// FetchGalleryPage 取画廊页。注意:EH 每屏只列 20 张缩略图,
|
||
// 必须翻 ?p=N 逐屏收集,否则静默只下到前 20 张且不报错。
|
||
func (c *Client) FetchGalleryPage(ctx context.Context, gid, token string, onProgress ParseProgress) (*Gallery, error) {
|
||
var (
|
||
pages []GalleryPageInfo
|
||
seen = map[int]bool{}
|
||
total int
|
||
title string
|
||
lastHTML string
|
||
firstHTML string
|
||
)
|
||
|
||
for p := 0; p < 500; p++ {
|
||
if err := ctx.Err(); err != nil {
|
||
return nil, err
|
||
}
|
||
u := EHBase + "/g/" + gid + "/" + token + "/"
|
||
if p > 0 {
|
||
u += "?p=" + strconv.Itoa(p)
|
||
}
|
||
html, err := c.getTextAutoWarn(ctx, u)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("解析画廊页失败: %w", err)
|
||
}
|
||
lastHTML = html
|
||
|
||
if p == 0 {
|
||
if len(html) < 2000 && strings.Contains(html, "404 Not Found") {
|
||
return nil, fmt.Errorf("画廊不存在或 token 失效: %s", gid)
|
||
}
|
||
if m := reTitleTag.FindStringSubmatch(html); m != nil {
|
||
title = strings.TrimSpace(reTitleSuffix.ReplaceAllString(DecodeEntities(m[1]), ""))
|
||
}
|
||
if m := reMetaDesc.FindStringSubmatch(html); m != nil {
|
||
if nm := reTitleFromDesc.FindStringSubmatch(DecodeEntities(m[1])); nm != nil {
|
||
// 保险:再剥一次尾部 " - Tags"(有些页面 desc 里 Tags 前无空格)
|
||
title = strings.TrimSpace(reTitleSuffix.ReplaceAllString(nm[1], ""))
|
||
}
|
||
}
|
||
if m := reTotal.FindStringSubmatch(html); m != nil {
|
||
total, _ = strconv.Atoi(m[1])
|
||
}
|
||
firstHTML = html // 标签/作者等元信息只在第一页有
|
||
}
|
||
|
||
added := 0
|
||
for _, m := range reThumbLink.FindAllStringSubmatch(html, -1) {
|
||
page, _ := strconv.Atoi(m[2])
|
||
if seen[page] {
|
||
continue
|
||
}
|
||
seen[page] = true
|
||
added++
|
||
|
||
sPath := m[1]
|
||
if !strings.HasPrefix(sPath, "http") {
|
||
sPath = EHBase + sPath
|
||
}
|
||
name := strings.TrimSpace(DecodeEntities(m[3]))
|
||
if name == "" {
|
||
name = gid + "-" + strconv.Itoa(page) + ".webp"
|
||
}
|
||
pages = append(pages, GalleryPageInfo{Page: page, SPath: sPath, Filename: name})
|
||
}
|
||
|
||
if added == 0 {
|
||
break // 这屏没新东西 -> 到头了
|
||
}
|
||
|
||
// 报告进度:大画廊翻几十上百屏时,界面上要看得见在动
|
||
// (放在 break 之前,保证最后一屏也会报告)
|
||
if onProgress != nil {
|
||
onProgress(len(pages), total, p+1)
|
||
}
|
||
|
||
if total > 0 && len(pages) >= total {
|
||
break // 收齐
|
||
}
|
||
}
|
||
|
||
if len(pages) == 0 {
|
||
if reRestricted.MatchString(lastHTML) {
|
||
return nil, fmt.Errorf("画廊受限或不存在(无 cookie 访问不了): %s", gid)
|
||
}
|
||
if isContentWarning(lastHTML) {
|
||
return nil, fmt.Errorf(
|
||
"画廊被 EH 标记为「Offensive For Everyone」,自动放行没成功"+
|
||
"(请在浏览器里打开一次并点 Never Warn Me Again): %s", gid)
|
||
}
|
||
return nil, fmt.Errorf("页面结构无法解析(EH 可能又改版): %s", gid)
|
||
}
|
||
|
||
// 按页码排序(插入排序,页数通常几十到几百)
|
||
for i := 1; i < len(pages); i++ {
|
||
for j := i; j > 0 && pages[j].Page < pages[j-1].Page; j-- {
|
||
pages[j], pages[j-1] = pages[j-1], pages[j]
|
||
}
|
||
}
|
||
|
||
if title == "" {
|
||
title = gid
|
||
}
|
||
if total == 0 {
|
||
total = len(pages)
|
||
}
|
||
|
||
return &Gallery{
|
||
GID: gid,
|
||
Token: token,
|
||
Title: title,
|
||
TotalPages: total,
|
||
Pages: pages,
|
||
Meta: parseMeta(firstHTML, title),
|
||
}, nil
|
||
}
|
||
|
||
// ---------------------------------------------------------------- 2) 图片直链
|
||
|
||
// ResolveImageURL 把 /s/ 链接解析成 hath 全图 URL。
|
||
//
|
||
// 这一步就是所谓「图片直链」的全部:GET 一次 /s/ 页面,
|
||
// 用正则从 HTML 里抓出那条 hath.network URL。没有签名计算、没有解码算法,
|
||
// 因为 URL 每次都不一样(随机子域+随机端口+keystamp 时间戳),只能实时取。
|
||
func (c *Client) ResolveImageURL(ctx context.Context, sPath string) (string, error) {
|
||
html, err := c.getText(ctx, sPath)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
if m := reHath.FindString(html); m != "" {
|
||
return c.applyXres(DecodeEntities(m)), nil
|
||
}
|
||
// CDN 换域名时的兜底:抓页面上第一个图片 URL
|
||
if m := reImgAny.FindString(html); m != "" {
|
||
return c.applyXres(DecodeEntities(m)), nil
|
||
}
|
||
return "", fmt.Errorf("未能从 /s/ 页面解析出图片地址: %s", sPath)
|
||
}
|
||
|
||
var (
|
||
reXresNum = regexp.MustCompile(`xres=\d+`)
|
||
reXresAny = regexp.MustCompile(`xres=[^/]+`)
|
||
)
|
||
|
||
// applyXres 改写 URL 里的 xres 参数(数字=长边上限,org=原图)
|
||
func (c *Client) applyXres(u string) string {
|
||
c.mu.RLock()
|
||
x := c.xres
|
||
c.mu.RUnlock()
|
||
|
||
if x == "" || x == "auto" {
|
||
return u
|
||
}
|
||
if _, err := strconv.Atoi(x); err == nil {
|
||
return reXresNum.ReplaceAllString(u, "xres="+x)
|
||
}
|
||
if x == "org" {
|
||
return reXresAny.ReplaceAllString(u, "xres=org")
|
||
}
|
||
return u
|
||
}
|
||
|
||
// ---------------------------------------------------------------- 3) 下载
|
||
|
||
// DownloadImage 流式下载到 destPath(先写 .part 再 rename,保证原子性)
|
||
func (c *Client) DownloadImage(ctx context.Context, imgURL, destPath string, onProgress func(received, total int64)) (int64, error) {
|
||
_, _, baseTimeout := c.snapshot()
|
||
|
||
// 图片本体也要限流
|
||
if err := c.limiter.Wait(ctx); err != nil {
|
||
return 0, err
|
||
}
|
||
|
||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, imgURL, nil)
|
||
if err != nil {
|
||
return 0, err
|
||
}
|
||
req.Header.Set("User-Agent", defaultUA)
|
||
req.Header.Set("Referer", EHBase+"/")
|
||
|
||
resp, err := c.httpClient(baseTimeout * 2).Do(req)
|
||
if err != nil {
|
||
return 0, err
|
||
}
|
||
defer resp.Body.Close()
|
||
if resp.StatusCode < 200 || resp.StatusCode >= 400 {
|
||
return 0, fmt.Errorf("HTTP %d 下载图片失败", resp.StatusCode)
|
||
}
|
||
|
||
tmp := destPath + ".part"
|
||
f, err := os.Create(tmp)
|
||
if err != nil {
|
||
return 0, err
|
||
}
|
||
|
||
received, copyErr := io.Copy(f, &progressReader{
|
||
r: resp.Body, total: resp.ContentLength, onProgress: onProgress,
|
||
})
|
||
closeErr := f.Close()
|
||
if copyErr != nil {
|
||
os.Remove(tmp)
|
||
return 0, copyErr
|
||
}
|
||
if closeErr != nil {
|
||
os.Remove(tmp)
|
||
return 0, closeErr
|
||
}
|
||
if received == 0 {
|
||
os.Remove(tmp)
|
||
return 0, errors.New("下载内容为空")
|
||
}
|
||
if err := os.Rename(tmp, destPath); err != nil {
|
||
os.Remove(tmp)
|
||
return 0, err
|
||
}
|
||
return received, nil
|
||
}
|
||
|
||
// progressReader 包一层统计下载进度
|
||
type progressReader struct {
|
||
r io.Reader
|
||
total int64
|
||
received int64
|
||
onProgress func(received, total int64)
|
||
}
|
||
|
||
func (p *progressReader) Read(b []byte) (int, error) {
|
||
n, err := p.r.Read(b)
|
||
if n > 0 {
|
||
p.received += int64(n)
|
||
if p.onProgress != nil {
|
||
p.onProgress(p.received, p.total)
|
||
}
|
||
}
|
||
return n, err
|
||
}
|