Files
eh-downloader/internal/eh/e2e_test.go
T
toor 7243f73d33 fix: 解析大画廊时连接泄漏 + 解析阶段无进度反馈
用户反馈:点开始下载后 6 个画廊(都是 2000 张级)一直显示「解析中 0/?」,
看起来像卡死。排查发现两个问题:

1. 连接泄漏(真 bug)
   httpClient() 每次请求都新建 http.Transport,用完丢弃。Transport 被丢弃后
   它持有的 keep-alive 连接不会立刻释放(要等 IdleConnTimeout=90s),
   于是每发一个请求留下一条僵尸连接。
   现场实测:91 条 Established 连接,而解析是串行的,正常应该只有 1 条。

   修复:Transport 全局复用,只在代理配置变化时重建,
   重建时 CloseIdleConnections 关掉旧连接;新增 Client.Close()。

   实测:20 次请求从 20 条连接降到 1 条;100 次并发请求只用 22 条。

2. 解析阶段毫无反馈
   2000 张 = 100 屏串行翻页,整个解析阶段界面都显示「解析中 0/?」,
   完全没有进度,用户只能干等。

   修复:FetchGalleryPage 增加 ParseProgress 回调,每翻完一屏上报一次
   (已解析张数/总张数/当前屏),界面显示「已解析 340/2000 张」。

另外:用户设置里请求间隔是 1000ms(1 秒),全局限流让 5 个画廊共享
1 请求/秒 —— 500 次解析请求需要约 8 分钟。这是配置导致的慢,不是 bug,
但可以在界面上看到进度后就不会以为卡死了。

测试从 74 增至 80,新增:
- TestTransportIsReused / TestTransportReusedUnderConcurrency(连接复用)
- TestSetProxyRebuildsTransport(改代理要重建)
- TestParseProgressCallback(进度回调,含最后一屏)
2026-10-09 10:02:50 +08:00

294 lines
8.7 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package eh
// 端到端协议链路测试:用 httptest 模拟 EH 服务器,
// 验证「画廊页分页 → /s/ → hath 直链 → 下载」整条链路,
// 以及断点续传时「文件已存在就跳过」的行为。
//
// 这些 HTML 样本按真实 EH 页面结构构造(字段名、缩略图 title 属性、
// Showing X - Y of N images 等都照抄线上结构)。
import (
"context"
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
// 模拟画廊:45 张图 = 3 屏(20 + 20 + 5),用来验证 ?p=N 分页
func mockGalleryHTML(t *testing.T, gid, token string, totalImages int) map[int]string {
pages := map[int]string{} // p 序号 -> html
for p := 0; p < 3; p++ {
from := p*20 + 1
to := from + 19
if to > totalImages {
to = totalImages
}
var sb strings.Builder
if p == 0 {
sb.WriteString(`<!DOCTYPE html><html><head>`)
sb.WriteString(`<title>[Noxe] Test Gallery - E-Hentai Galleries</title>`)
sb.WriteString(`<meta name="description" content="Free Hentai Doujinshi Gallery: [Noxe] Test Gallery: 45 images (ratelimit)" />`)
sb.WriteString(`</head><body><div class="gtb"><table><tr><td>`)
sb.WriteString(`<div id="taglist"><table>`)
sb.WriteString(`<tr><td class="tc">artist:</td><td><div class="gtl"><a href="/tag/artist:noxe">Noxe</a></div></td></tr>`)
sb.WriteString(`<tr><td class="tc">group:</td><td><div class="gtl"><a href="/tag/group:criticalhai">CriticalHAI</a></div></td></tr>`)
sb.WriteString(`<tr><td class="tc">other:</td><td><div class="gtl"><a href="/tag:qingxiao">qingxiao</a><a href="/tag:rover">rover</a></div></td></tr>`)
sb.WriteString(`</table></div><a href="/uploader/velius11">up</a>`)
}
for page := from; page <= to; page++ {
key := fmt.Sprintf("key%02d", page)
sb.WriteString(fmt.Sprintf(
`<a href="/s/%s/%s-%d"><div title="Page %d: %03d.png" class="lazy"></div></a>`,
key, gid, page, page, page))
}
if p == 0 {
sb.WriteString(`<div class="gtb">`)
}
sb.WriteString(fmt.Sprintf(`Showing %d - %d of %d images`, from, to, totalImages))
if p == 0 {
sb.WriteString(`</div></body></html>`)
}
pages[p] = sb.String()
}
return pages
}
func TestFetchGalleryPagePagination(t *testing.T) {
const (
gid = "4239250"
tok = "58703925db"
total = 45
)
galleryPages := mockGalleryHTML(t, gid, tok, total)
mux := http.NewServeMux()
mux.HandleFunc("/g/"+gid+"/"+tok+"/", func(w http.ResponseWriter, r *http.Request) {
p := 0
if q := r.URL.Query().Get("p"); q != "" {
fmt.Sscanf(q, "%d", &p)
}
html, ok := galleryPages[p]
if !ok {
// 超过范围的屏:EH 会返回空列表页
w.Write([]byte(`<html>Showing 46 - 45 of 45 images</html>`))
return
}
w.Write([]byte(html))
})
mux.HandleFunc("/s/", func(w http.ResponseWriter, r *http.Request) {
// /s/{key}/{gid}-{page}
page := strings.TrimPrefix(r.URL.Path, "/s/")
parts := strings.Split(page, "/")
if len(parts) != 2 {
w.WriteHeader(404)
return
}
idx := strings.LastIndex(parts[1], "-")
pno := parts[1][idx+1:]
// 每页一个不同的 hath URL(模拟随机子域+端口+keystamp)
w.Write([]byte(fmt.Sprintf(
`<html><img src="https://sub%d.gyktyewfonyf.hath.network:%d/h/abc/keystamp=1700000%d;fileindex=%d;xres=800/pic.png"/></html>`,
len(pno)%10, 39000+len(pno), len(pno), len(pno))))
})
srv := httptest.NewServer(mux)
defer srv.Close()
// 把 EHBase 换到测试服务器
oldBase := EHBase
EHBase = srv.URL
defer func() { EHBase = oldBase }()
c := NewClient(5*time.Second, "", "off", "auto")
g, err := c.FetchGalleryPage(context.Background(), gid, tok, nil)
if err != nil {
t.Fatalf("FetchGalleryPage 失败: %v", err)
}
// 关键断言:45 张必须全部拿到(分页生效)
if len(g.Pages) != total {
t.Errorf("应拿到 %d 页,实际 %d 页 —— 分页可能没生效", total, len(g.Pages))
}
if g.TotalPages != total {
t.Errorf("TotalPages 不对: %d", g.TotalPages)
}
// 页码连续且有序
for i, p := range g.Pages {
if p.Page != i+1 {
t.Errorf("第 %d 项页码应为 %d,实际 %d", i, i+1, p.Page)
}
}
// key 必须与页码配对(每页的 key 不同);SPath 会被补全成绝对地址
if !strings.HasSuffix(g.Pages[0].SPath, "/s/key01/4239250-1") {
t.Errorf("第 1 页路径不对: %s", g.Pages[0].SPath)
}
if !strings.HasSuffix(g.Pages[1].SPath, "/s/key02/4239250-2") {
t.Errorf("第 2 页路径不对(key 与 page 必须配对): %s", g.Pages[1].SPath)
}
// 文件名
if g.Pages[0].Filename != "001.png" {
t.Errorf("文件名不对: %q", g.Pages[0].Filename)
}
// 元信息只在第一页解析
if g.Title != "[Noxe] Test Gallery" {
t.Errorf("标题不对: %q", g.Title)
}
if g.Meta.Artist != "Noxe" {
t.Errorf("作者不对: %q", g.Meta.Artist)
}
if g.Meta.Category != "doujinshi" {
t.Errorf("分类不对: %q", g.Meta.Category)
}
if len(g.Meta.Tags) != 2 {
t.Errorf("标签不对: %v", g.Meta.Tags)
}
}
// 解析 hath 直链 + xres 改写
func TestResolveImageURLEndToEnd(t *testing.T) {
mux := http.NewServeMux()
mux.HandleFunc("/s/", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(`<html><body>
<img src="https://oclzqqk.gyktyewfonyf.hath.network:39656/h/4e9af9f-44168-707-754-wbp/keystamp=1791448200-8b38ea8f6d;fileindex=270107377;xres=800/NH.webp">
</body></html>`))
})
srv := httptest.NewServer(mux)
defer srv.Close()
oldBase := EHBase
EHBase = srv.URL
defer func() { EHBase = oldBase }()
c := NewClient(5*time.Second, "", "off", "auto")
got, err := c.ResolveImageURL(context.Background(), srv.URL+"/s/abc/123-1")
if err != nil {
t.Fatalf("解析失败: %v", err)
}
if !strings.HasPrefix(got, "https://oclzqqk.gyktyewfonyf.hath.network:39656/") {
t.Errorf("直链不对: %s", got)
}
if !strings.Contains(got, "xres=800") {
t.Errorf("应保留站点给的 xres: %s", got)
}
if strings.Contains(got, `"`) || strings.Contains(got, ">") {
t.Errorf("URL 里混进了 HTML 符号: %s", got)
}
// 切到 org 应该改写
c.SetXres("org")
got2, _ := c.ResolveImageURL(context.Background(), srv.URL+"/s/abc/123-1")
if !strings.Contains(got2, "xres=org") {
t.Errorf("xres=org 改写失败: %s", got2)
}
}
// 断点续传:文件已存在且非空就该跳过
func TestDownloadImageResume(t *testing.T) {
payload := []byte("fake webp bytes for testing resume")
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Write(payload)
}))
defer srv.Close()
dir := t.TempDir()
c := NewClient(5*time.Second, "", "off", "auto")
dest := filepath.Join(dir, "001.webp")
n, err := c.DownloadImage(context.Background(), srv.URL+"/pic.webp", dest, nil)
if err != nil {
t.Fatalf("下载失败: %v", err)
}
if n != int64(len(payload)) {
t.Errorf("下载字节数不对: %d", n)
}
got, err := os.ReadFile(dest)
if err != nil {
t.Fatal(err)
}
if string(got) != string(payload) {
t.Errorf("内容不对")
}
// .part 文件不该残留
if _, err := os.Stat(dest + ".part"); err == nil {
t.Error("临时 .part 文件没清理")
}
// 进度回调
var received int64
c2 := NewClient(5*time.Second, "", "off", "auto")
dest2 := filepath.Join(dir, "002.webp")
c2.DownloadImage(context.Background(), srv.URL+"/pic.webp", dest2, func(recv, total int64) {
received = recv
})
if received != int64(len(payload)) {
t.Errorf("进度回调不对: %d", received)
}
}
// 取消要能真的打断
func TestDownloadImageCancel(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// 慢慢吐
w.Write([]byte("start"))
if f, ok := w.(http.Flusher); ok {
f.Flush()
}
time.Sleep(3 * time.Second)
}))
defer srv.Close()
ctx, cancel := context.WithCancel(context.Background())
go func() {
time.Sleep(200 * time.Millisecond)
cancel()
}()
c := NewClient(5*time.Second, "", "off", "auto")
dest := filepath.Join(t.TempDir(), "x.webp")
_, err := c.DownloadImage(ctx, srv.URL+"/slow.webp", dest, nil)
if err == nil {
t.Error("取消后应该返回错误")
}
// 临时文件要被清理
if _, err := os.Stat(dest + ".part"); err == nil {
t.Error("取消后 .part 没清理")
}
}
// 受限画廊要给出明确错误
func TestFetchGalleryRestricted(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(`<html><body>Login Required - restricted gallery</body></html>`))
}))
defer srv.Close()
oldBase := EHBase
EHBase = srv.URL
defer func() { EHBase = oldBase }()
c := NewClient(5*time.Second, "", "off", "auto")
_, err := c.FetchGalleryPage(context.Background(), "12345678", "abc", nil)
if err == nil {
t.Fatal("受限画廊应该报错")
}
if !strings.Contains(err.Error(), "受限") {
t.Errorf("错误信息不明确: %v", err)
}
}