用户反馈:点开始下载后 6 个画廊(都是 2000 张级)一直显示「解析中 0/?」, 看起来像卡死。排查发现两个问题: 1. 连接泄漏(真 bug) httpClient() 每次请求都新建 http.Transport,用完丢弃。Transport 被丢弃后 它持有的 keep-alive 连接不会立刻释放(要等 IdleConnTimeout=90s), 于是每发一个请求留下一条僵尸连接。 现场实测:91 条 Established 连接,而解析是串行的,正常应该只有 1 条。 修复:Transport 全局复用,只在代理配置变化时重建, 重建时 CloseIdleConnections 关掉旧连接;新增 Client.Close()。 实测:20 次请求从 20 条连接降到 1 条;100 次并发请求只用 22 条。 2. 解析阶段毫无反馈 2000 张 = 100 屏串行翻页,整个解析阶段界面都显示「解析中 0/?」, 完全没有进度,用户只能干等。 修复:FetchGalleryPage 增加 ParseProgress 回调,每翻完一屏上报一次 (已解析张数/总张数/当前屏),界面显示「已解析 340/2000 张」。 另外:用户设置里请求间隔是 1000ms(1 秒),全局限流让 5 个画廊共享 1 请求/秒 —— 500 次解析请求需要约 8 分钟。这是配置导致的慢,不是 bug, 但可以在界面上看到进度后就不会以为卡死了。 测试从 74 增至 80,新增: - TestTransportIsReused / TestTransportReusedUnderConcurrency(连接复用) - TestSetProxyRebuildsTransport(改代理要重建) - TestParseProgressCallback(进度回调,含最后一屏)
294 lines
8.7 KiB
Go
294 lines
8.7 KiB
Go
package eh
|
||
|
||
// 端到端协议链路测试:用 httptest 模拟 EH 服务器,
|
||
// 验证「画廊页分页 → /s/ → hath 直链 → 下载」整条链路,
|
||
// 以及断点续传时「文件已存在就跳过」的行为。
|
||
//
|
||
// 这些 HTML 样本按真实 EH 页面结构构造(字段名、缩略图 title 属性、
|
||
// Showing X - Y of N images 等都照抄线上结构)。
|
||
|
||
import (
|
||
"context"
|
||
"fmt"
|
||
"net/http"
|
||
"net/http/httptest"
|
||
"os"
|
||
"path/filepath"
|
||
"strings"
|
||
"testing"
|
||
"time"
|
||
)
|
||
|
||
// 模拟画廊:45 张图 = 3 屏(20 + 20 + 5),用来验证 ?p=N 分页
|
||
func mockGalleryHTML(t *testing.T, gid, token string, totalImages int) map[int]string {
|
||
pages := map[int]string{} // p 序号 -> html
|
||
|
||
for p := 0; p < 3; p++ {
|
||
from := p*20 + 1
|
||
to := from + 19
|
||
if to > totalImages {
|
||
to = totalImages
|
||
}
|
||
var sb strings.Builder
|
||
|
||
if p == 0 {
|
||
sb.WriteString(`<!DOCTYPE html><html><head>`)
|
||
sb.WriteString(`<title>[Noxe] Test Gallery - E-Hentai Galleries</title>`)
|
||
sb.WriteString(`<meta name="description" content="Free Hentai Doujinshi Gallery: [Noxe] Test Gallery: 45 images (ratelimit)" />`)
|
||
sb.WriteString(`</head><body><div class="gtb"><table><tr><td>`)
|
||
sb.WriteString(`<div id="taglist"><table>`)
|
||
sb.WriteString(`<tr><td class="tc">artist:</td><td><div class="gtl"><a href="/tag/artist:noxe">Noxe</a></div></td></tr>`)
|
||
sb.WriteString(`<tr><td class="tc">group:</td><td><div class="gtl"><a href="/tag/group:criticalhai">CriticalHAI</a></div></td></tr>`)
|
||
sb.WriteString(`<tr><td class="tc">other:</td><td><div class="gtl"><a href="/tag:qingxiao">qingxiao</a><a href="/tag:rover">rover</a></div></td></tr>`)
|
||
sb.WriteString(`</table></div><a href="/uploader/velius11">up</a>`)
|
||
}
|
||
|
||
for page := from; page <= to; page++ {
|
||
key := fmt.Sprintf("key%02d", page)
|
||
sb.WriteString(fmt.Sprintf(
|
||
`<a href="/s/%s/%s-%d"><div title="Page %d: %03d.png" class="lazy"></div></a>`,
|
||
key, gid, page, page, page))
|
||
}
|
||
|
||
if p == 0 {
|
||
sb.WriteString(`<div class="gtb">`)
|
||
}
|
||
sb.WriteString(fmt.Sprintf(`Showing %d - %d of %d images`, from, to, totalImages))
|
||
if p == 0 {
|
||
sb.WriteString(`</div></body></html>`)
|
||
}
|
||
pages[p] = sb.String()
|
||
}
|
||
return pages
|
||
}
|
||
|
||
func TestFetchGalleryPagePagination(t *testing.T) {
|
||
const (
|
||
gid = "4239250"
|
||
tok = "58703925db"
|
||
total = 45
|
||
)
|
||
|
||
galleryPages := mockGalleryHTML(t, gid, tok, total)
|
||
|
||
mux := http.NewServeMux()
|
||
mux.HandleFunc("/g/"+gid+"/"+tok+"/", func(w http.ResponseWriter, r *http.Request) {
|
||
p := 0
|
||
if q := r.URL.Query().Get("p"); q != "" {
|
||
fmt.Sscanf(q, "%d", &p)
|
||
}
|
||
html, ok := galleryPages[p]
|
||
if !ok {
|
||
// 超过范围的屏:EH 会返回空列表页
|
||
w.Write([]byte(`<html>Showing 46 - 45 of 45 images</html>`))
|
||
return
|
||
}
|
||
w.Write([]byte(html))
|
||
})
|
||
mux.HandleFunc("/s/", func(w http.ResponseWriter, r *http.Request) {
|
||
// /s/{key}/{gid}-{page}
|
||
page := strings.TrimPrefix(r.URL.Path, "/s/")
|
||
parts := strings.Split(page, "/")
|
||
if len(parts) != 2 {
|
||
w.WriteHeader(404)
|
||
return
|
||
}
|
||
idx := strings.LastIndex(parts[1], "-")
|
||
pno := parts[1][idx+1:]
|
||
// 每页一个不同的 hath URL(模拟随机子域+端口+keystamp)
|
||
w.Write([]byte(fmt.Sprintf(
|
||
`<html><img src="https://sub%d.gyktyewfonyf.hath.network:%d/h/abc/keystamp=1700000%d;fileindex=%d;xres=800/pic.png"/></html>`,
|
||
len(pno)%10, 39000+len(pno), len(pno), len(pno))))
|
||
})
|
||
|
||
srv := httptest.NewServer(mux)
|
||
defer srv.Close()
|
||
|
||
// 把 EHBase 换到测试服务器
|
||
oldBase := EHBase
|
||
EHBase = srv.URL
|
||
defer func() { EHBase = oldBase }()
|
||
|
||
c := NewClient(5*time.Second, "", "off", "auto")
|
||
g, err := c.FetchGalleryPage(context.Background(), gid, tok, nil)
|
||
if err != nil {
|
||
t.Fatalf("FetchGalleryPage 失败: %v", err)
|
||
}
|
||
|
||
// 关键断言:45 张必须全部拿到(分页生效)
|
||
if len(g.Pages) != total {
|
||
t.Errorf("应拿到 %d 页,实际 %d 页 —— 分页可能没生效", total, len(g.Pages))
|
||
}
|
||
if g.TotalPages != total {
|
||
t.Errorf("TotalPages 不对: %d", g.TotalPages)
|
||
}
|
||
|
||
// 页码连续且有序
|
||
for i, p := range g.Pages {
|
||
if p.Page != i+1 {
|
||
t.Errorf("第 %d 项页码应为 %d,实际 %d", i, i+1, p.Page)
|
||
}
|
||
}
|
||
|
||
// key 必须与页码配对(每页的 key 不同);SPath 会被补全成绝对地址
|
||
if !strings.HasSuffix(g.Pages[0].SPath, "/s/key01/4239250-1") {
|
||
t.Errorf("第 1 页路径不对: %s", g.Pages[0].SPath)
|
||
}
|
||
if !strings.HasSuffix(g.Pages[1].SPath, "/s/key02/4239250-2") {
|
||
t.Errorf("第 2 页路径不对(key 与 page 必须配对): %s", g.Pages[1].SPath)
|
||
}
|
||
|
||
// 文件名
|
||
if g.Pages[0].Filename != "001.png" {
|
||
t.Errorf("文件名不对: %q", g.Pages[0].Filename)
|
||
}
|
||
|
||
// 元信息只在第一页解析
|
||
if g.Title != "[Noxe] Test Gallery" {
|
||
t.Errorf("标题不对: %q", g.Title)
|
||
}
|
||
if g.Meta.Artist != "Noxe" {
|
||
t.Errorf("作者不对: %q", g.Meta.Artist)
|
||
}
|
||
if g.Meta.Category != "doujinshi" {
|
||
t.Errorf("分类不对: %q", g.Meta.Category)
|
||
}
|
||
if len(g.Meta.Tags) != 2 {
|
||
t.Errorf("标签不对: %v", g.Meta.Tags)
|
||
}
|
||
}
|
||
|
||
// 解析 hath 直链 + xres 改写
|
||
func TestResolveImageURLEndToEnd(t *testing.T) {
|
||
mux := http.NewServeMux()
|
||
mux.HandleFunc("/s/", func(w http.ResponseWriter, r *http.Request) {
|
||
w.Write([]byte(`<html><body>
|
||
<img src="https://oclzqqk.gyktyewfonyf.hath.network:39656/h/4e9af9f-44168-707-754-wbp/keystamp=1791448200-8b38ea8f6d;fileindex=270107377;xres=800/NH.webp">
|
||
</body></html>`))
|
||
})
|
||
srv := httptest.NewServer(mux)
|
||
defer srv.Close()
|
||
|
||
oldBase := EHBase
|
||
EHBase = srv.URL
|
||
defer func() { EHBase = oldBase }()
|
||
|
||
c := NewClient(5*time.Second, "", "off", "auto")
|
||
got, err := c.ResolveImageURL(context.Background(), srv.URL+"/s/abc/123-1")
|
||
if err != nil {
|
||
t.Fatalf("解析失败: %v", err)
|
||
}
|
||
if !strings.HasPrefix(got, "https://oclzqqk.gyktyewfonyf.hath.network:39656/") {
|
||
t.Errorf("直链不对: %s", got)
|
||
}
|
||
if !strings.Contains(got, "xres=800") {
|
||
t.Errorf("应保留站点给的 xres: %s", got)
|
||
}
|
||
if strings.Contains(got, `"`) || strings.Contains(got, ">") {
|
||
t.Errorf("URL 里混进了 HTML 符号: %s", got)
|
||
}
|
||
|
||
// 切到 org 应该改写
|
||
c.SetXres("org")
|
||
got2, _ := c.ResolveImageURL(context.Background(), srv.URL+"/s/abc/123-1")
|
||
if !strings.Contains(got2, "xres=org") {
|
||
t.Errorf("xres=org 改写失败: %s", got2)
|
||
}
|
||
}
|
||
|
||
// 断点续传:文件已存在且非空就该跳过
|
||
func TestDownloadImageResume(t *testing.T) {
|
||
payload := []byte("fake webp bytes for testing resume")
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
w.Write(payload)
|
||
}))
|
||
defer srv.Close()
|
||
|
||
dir := t.TempDir()
|
||
c := NewClient(5*time.Second, "", "off", "auto")
|
||
|
||
dest := filepath.Join(dir, "001.webp")
|
||
n, err := c.DownloadImage(context.Background(), srv.URL+"/pic.webp", dest, nil)
|
||
if err != nil {
|
||
t.Fatalf("下载失败: %v", err)
|
||
}
|
||
if n != int64(len(payload)) {
|
||
t.Errorf("下载字节数不对: %d", n)
|
||
}
|
||
|
||
got, err := os.ReadFile(dest)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if string(got) != string(payload) {
|
||
t.Errorf("内容不对")
|
||
}
|
||
|
||
// .part 文件不该残留
|
||
if _, err := os.Stat(dest + ".part"); err == nil {
|
||
t.Error("临时 .part 文件没清理")
|
||
}
|
||
|
||
// 进度回调
|
||
var received int64
|
||
c2 := NewClient(5*time.Second, "", "off", "auto")
|
||
dest2 := filepath.Join(dir, "002.webp")
|
||
c2.DownloadImage(context.Background(), srv.URL+"/pic.webp", dest2, func(recv, total int64) {
|
||
received = recv
|
||
})
|
||
if received != int64(len(payload)) {
|
||
t.Errorf("进度回调不对: %d", received)
|
||
}
|
||
}
|
||
|
||
// 取消要能真的打断
|
||
func TestDownloadImageCancel(t *testing.T) {
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
// 慢慢吐
|
||
w.Write([]byte("start"))
|
||
if f, ok := w.(http.Flusher); ok {
|
||
f.Flush()
|
||
}
|
||
time.Sleep(3 * time.Second)
|
||
}))
|
||
defer srv.Close()
|
||
|
||
ctx, cancel := context.WithCancel(context.Background())
|
||
go func() {
|
||
time.Sleep(200 * time.Millisecond)
|
||
cancel()
|
||
}()
|
||
|
||
c := NewClient(5*time.Second, "", "off", "auto")
|
||
dest := filepath.Join(t.TempDir(), "x.webp")
|
||
_, err := c.DownloadImage(ctx, srv.URL+"/slow.webp", dest, nil)
|
||
if err == nil {
|
||
t.Error("取消后应该返回错误")
|
||
}
|
||
// 临时文件要被清理
|
||
if _, err := os.Stat(dest + ".part"); err == nil {
|
||
t.Error("取消后 .part 没清理")
|
||
}
|
||
}
|
||
|
||
// 受限画廊要给出明确错误
|
||
func TestFetchGalleryRestricted(t *testing.T) {
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
w.Write([]byte(`<html><body>Login Required - restricted gallery</body></html>`))
|
||
}))
|
||
defer srv.Close()
|
||
|
||
oldBase := EHBase
|
||
EHBase = srv.URL
|
||
defer func() { EHBase = oldBase }()
|
||
|
||
c := NewClient(5*time.Second, "", "off", "auto")
|
||
_, err := c.FetchGalleryPage(context.Background(), "12345678", "abc", nil)
|
||
if err == nil {
|
||
t.Fatal("受限画廊应该报错")
|
||
}
|
||
if !strings.Contains(err.Error(), "受限") {
|
||
t.Errorf("错误信息不明确: %v", err)
|
||
}
|
||
}
|