Files
hd4k-downloader/internal/hd4k/hd4k_test.go
T
2026-10-11 12:56:38 +08:00

586 lines
18 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package hd4k
// 协议层测试
//
// ⚠️ 这里的夹具是**真实抓下来的页面**(testdata/gallery-p{1,2,3}.html,
// 2026-10-11 从 migmg.ssuu.uk 实抓),不是手写的 mock。
//
// 这么做的原因:站点哪天改版,正则失效,靠手写 mock 是发现不了的 ——
// mock 只验证「代码符合我写 mock 时的想象」。真实样本能真正当回归基线:
// 解析结果对不上这三个文件里的确切数字,就说明站点变了或代码退化了。
import (
"context"
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"sync"
"testing"
)
func loadFixture(t *testing.T, name string) string {
t.Helper()
data, err := os.ReadFile(filepath.Join("testdata", name))
if err != nil {
t.Fatalf("读夹具 %s 失败: %v", name, err)
}
return string(data)
}
// ---------------------------------------------------------------- 真实样本回归
func TestParseRealGalleryPages(t *testing.T) {
// 实测基线:53 张图,分 3 页,每页 20/20/13
cases := []struct {
file string
wantImgs int
wantPages int
firstFile string
lastFile string
}{
{"gallery-p1.html", 20, 3, "tbox-vol52-4khd.com-001.webp", "tbox-vol52-4khd.com-020.webp"},
{"gallery-p2.html", 20, 3, "tbox-vol52-4khd.com-021.webp", "tbox-vol52-4khd.com-040.webp"},
{"gallery-p3.html", 13, 3, "tbox-vol52-4khd.com-041.webp", "tbox-vol52-4khd.com-053.webp"},
}
for _, c := range cases {
t.Run(c.file, func(t *testing.T) {
p := ParseGalleryPageHTML(loadFixture(t, c.file))
if len(p.Images) != c.wantImgs {
t.Errorf("图片数 = %d, 想要 %d", len(p.Images), c.wantImgs)
}
if p.TotalPages != c.wantPages {
t.Errorf("总页数 = %d, 想要 %d", p.TotalPages, c.wantPages)
}
if len(p.Images) == 0 {
return
}
if got := p.Images[0].Filename; got != c.firstFile {
t.Errorf("首图文件名 = %q, 想要 %q", got, c.firstFile)
}
if got := p.Images[len(p.Images)-1].Filename; got != c.lastFile {
t.Errorf("尾图文件名 = %q, 想要 %q", got, c.lastFile)
}
})
}
}
// 推荐位图片(4KHD-beautifulGirls / ?resize= / w1090-h1500-p-k-no-rw)
// 三页各出现 13 次 —— 如果被当成正文图混进去,数量会明显超标。
func TestRecommendImagesExcluded(t *testing.T) {
for _, f := range []string{"gallery-p1.html", "gallery-p2.html", "gallery-p3.html"} {
p := ParseGalleryPageHTML(loadFixture(t, f))
for _, im := range p.Images {
if isRecommendImage(im.URL) {
t.Errorf("%s: 推荐位图片混进来了: %s", f, im.URL)
}
}
}
}
// 图片 URL 必须是**源站直链**,不能是 Photon 代理。
//
// 2026-10-11 实测:批量下载走 i0.wp.com 会打爆 WordPress 免费 CDN 的额度,
// 被封后连裸请求都返回 429("Access has been disabled due to excessive
// usage"),同一时刻源站 20 并发却零失败。所以解析阶段就转成源站。
//
// 注意源站路径里的 w1300-rw 段必须保留 —— 那是原站自己的缩略图目录,
// 不是 Photon 的参数;Photon 的 ?w= 查询串则要去掉(源站不认)。
func TestImageURLOriginNotPhoton(t *testing.T) {
for _, f := range []string{"gallery-p1.html", "gallery-p2.html", "gallery-p3.html"} {
p := ParseGalleryPageHTML(loadFixture(t, f))
if len(p.Images) == 0 {
t.Fatalf("%s: 一张图都没解析到", f)
}
for _, im := range p.Images {
if IsPhoton(im.URL) {
t.Errorf("%s: 还是 Photon 代理,会被 429 封: %s", f, im.URL)
}
if !strings.Contains(im.URL, "/pic.4khd.com/") {
t.Errorf("%s: 不是源站 pic.4khd.com: %s", f, im.URL)
}
if !strings.Contains(im.URL, "w1300-rw/") {
t.Errorf("%s: 丢了 w1300-rw 缩略图目录: %s", f, im.URL)
}
if strings.Contains(im.URL, "?") {
t.Errorf("%s: 不该带查询串(源站不认 Photon 的 resize 参数): %s", f, im.URL)
}
}
}
}
// OriginURL 转换本身的行为
func TestOriginURL(t *testing.T) {
cases := []struct{ in, want string }{
// Photon 代理 → 源站,查询串去掉
{
"https://i0.wp.com/pic.4khd.com/-a/b/c/w1300-rw/x.webp?w=1300",
"https://pic.4khd.com/-a/b/c/w1300-rw/x.webp",
},
{
"https://i1.wp.com/pic.4khd.com/p/q?resize=1,300",
"https://pic.4khd.com/p/q",
},
// 非 Photon:原样返回,不能乱改
{
"https://pic.4khd.com/-a/b/c/w1300-rw/x.webp?w=1300",
"https://pic.4khd.com/-a/b/c/w1300-rw/x.webp?w=1300",
},
{
"https://migmg.ssuu.uk/content/09/x.html",
"https://migmg.ssuu.uk/content/09/x.html",
},
// 路径里恰好含 wp.com 但不是代理
{
"https://pic.4khd.com/i2.wp.com/keep/me.webp",
"https://pic.4khd.com/i2.wp.com/keep/me.webp",
},
}
for _, c := range cases {
if got := OriginURL(c.in); got != c.want {
t.Errorf("OriginURL(%q)\n = %q\n 想要 %q", c.in, got, c.want)
}
}
}
func TestIsPhoton(t *testing.T) {
if !IsPhoton("https://i0.wp.com/pic.4khd.com/x.webp?w=1300") {
t.Error("i0.wp.com 应识别为 Photon")
}
if IsPhoton("https://pic.4khd.com/i0.wp.com/x.webp") {
t.Error("路径里的 i0.wp.com 不该算 Photon")
}
if IsPhoton("https://migmg.ssuu.uk/") {
t.Error("普通站点不该算 Photon")
}
}
// 标题:h3 里是双空格,<title> 里被 WordPress 规范化成单空格。
// 折叠空白后两者必须一致,否则同一画廊在不同路径上会算出两个目录名。
func TestTitleExtraction(t *testing.T) {
p := ParseGalleryPageHTML(loadFixture(t, "gallery-p1.html"))
want := "Myu_a(뮤아) SWEETBOX Vol.52[190MB-53photos]"
if p.Title != want {
t.Errorf("标题 = %q\n 想要 %q", p.Title, want)
}
if strings.Contains(p.Title, " - 4KHD") {
t.Errorf("标题没去掉站点后缀: %q", p.Title)
}
// 下划线/方括号必须是原样 ASCII,不能被转义(实测源文件里就没有反斜杠)
if !strings.Contains(p.Title, "Myu_a") {
t.Errorf("标题里的下划线被破坏了: %q", p.Title)
}
if !strings.Contains(p.Title, "[190MB-53photos]") {
t.Errorf("标题里的方括号被破坏了: %q", p.Title)
}
}
func TestTitleFromTitleTagMatchesH3(t *testing.T) {
html := loadFixture(t, "gallery-p1.html")
viaH3 := extractTitle(html)
viaTitle := cleanText(reTitle.FindStringSubmatch(html)[1])
viaTitle = stripTitleSuffix(viaTitle)
if viaH3 != viaTitle {
t.Errorf("h3 与 <title> 提取结果不一致(空白折叠没做对):\n h3 = %q\n title = %q", viaH3, viaTitle)
}
}
func TestTotalFromTitle(t *testing.T) {
// 标题里的 [190MB-53photos] 应能取到 53,与 20+20+13 吻合
if n := TotalFromTitle("Myu_a SWEETBOX Vol.52[190MB-53photos]"); n != 53 {
t.Errorf("TotalFromTitle = %d, 想要 53", n)
}
}
// 分页链:p1/p2 有 next,p3 没有
func TestNextLink(t *testing.T) {
p1 := ParseGalleryPageHTML(loadFixture(t, "gallery-p1.html"))
if !strings.HasSuffix(p1.Next, "/2") {
t.Errorf("p1 的 next 应指向 /2, 实际 %q", p1.Next)
}
p2 := ParseGalleryPageHTML(loadFixture(t, "gallery-p2.html"))
if !strings.HasSuffix(p2.Next, "/3") {
t.Errorf("p2 的 next 应指向 /3, 实际 %q", p2.Next)
}
p3 := ParseGalleryPageHTML(loadFixture(t, "gallery-p3.html"))
if p3.Next != "" {
t.Errorf("p3 是末页,不该有 next, 实际 %q", p3.Next)
}
}
// 三页合起来应正好 53 张,无重复
func TestNoDuplicateAcrossPages(t *testing.T) {
seen := map[string]bool{}
total := 0
for _, f := range []string{"gallery-p1.html", "gallery-p2.html", "gallery-p3.html"} {
p := ParseGalleryPageHTML(loadFixture(t, f))
for _, im := range p.Images {
if seen[im.URL] {
t.Errorf("%s 里出现重复图片: %s", f, im.URL)
}
seen[im.URL] = true
total++
}
}
if total != 53 {
t.Errorf("三页合计 %d 张, 想要 53", total)
}
}
// ---------------------------------------------------------------- URL 解析
func TestStripPageSuffix(t *testing.T) {
cases := []struct{ in, want string }{
{"/content/09/x.html", "/content/09/x.html"},
{"/content/09/x.html/", "/content/09/x.html"},
{"/content/09/x.html/2", "/content/09/x.html"},
{"/content/09/x.html/12", "/content/09/x.html"},
{"/content/09/x.html/2/", "/content/09/x.html"},
// 非数字段不是分页,原样保留(宁可后续报「不是画廊链接」,
// 也不要盲目砍掉路径的一部分)
{"/content/09/x.html/abc", "/content/09/x.html/abc"},
}
for _, c := range cases {
if got := StripPageSuffix(c.in); got != c.want {
t.Errorf("StripPageSuffix(%q) = %q, 想要 %q", c.in, got, c.want)
}
}
}
func TestParseGalleryURL(t *testing.T) {
sites := DefaultSites()
base := "https://migmg.ssuu.uk"
cases := []struct {
in string
wantID string
wantPath string
wantSlug string
}{
{
in: base + "/content/09/myua-sweetbox-vol52.html",
wantID: "migmg.ssuu.uk/content/09/myua-sweetbox-vol52.html",
wantPath: "/content/09/myua-sweetbox-vol52.html",
wantSlug: "myua-sweetbox-vol52.html",
},
{
// 裸 host
in: "migmg.ssuu.uk/content/09/myua-sweetbox-vol52.html",
wantID: "migmg.ssuu.uk/content/09/myua-sweetbox-vol52.html",
wantPath: "/content/09/myua-sweetbox-vol52.html",
wantSlug: "myua-sweetbox-vol52.html",
},
{
// 第 2 页应归一到第 1 页(否则同一画廊会变成两个任务)
in: base + "/content/09/myua-sweetbox-vol52.html/2",
wantID: "migmg.ssuu.uk/content/09/myua-sweetbox-vol52.html",
wantPath: "/content/09/myua-sweetbox-vol52.html",
wantSlug: "myua-sweetbox-vol52.html",
},
{
// 尾斜杠
in: base + "/content/09/myua-sweetbox-vol52.html/",
wantID: "migmg.ssuu.uk/content/09/myua-sweetbox-vol52.html",
wantPath: "/content/09/myua-sweetbox-vol52.html",
wantSlug: "myua-sweetbox-vol52.html",
},
}
for _, c := range cases {
ref, err := ParseGalleryURL(c.in, sites)
if err != nil {
t.Errorf("ParseGalleryURL(%q) 报错: %v", c.in, err)
continue
}
if ref.ID != c.wantID {
t.Errorf("ParseGalleryURL(%q).ID = %q, 想要 %q", c.in, ref.ID, c.wantID)
}
if ref.Path != c.wantPath {
t.Errorf("ParseGalleryURL(%q).Path = %q, 想要 %q", c.in, ref.Path, c.wantPath)
}
if ref.Slug != c.wantSlug {
t.Errorf("ParseGalleryURL(%q).Slug = %q, 想要 %q", c.in, ref.Slug, c.wantSlug)
}
}
}
// 不同域名下的同一 slug 必须是两个不同的任务
func TestSameSlugDifferentHost(t *testing.T) {
// 两个**互不相关**的站点(都不在默认别名表里)
sites := []Site{
{Host: "aaa.example", Enabled: true},
{Host: "bbb.example", Enabled: true},
}
a, err := ParseGalleryURL("https://aaa.example/content/09/x.html", sites)
if err != nil {
t.Fatal(err)
}
b, err := ParseGalleryURL("https://bbb.example/content/09/x.html", sites)
if err != nil {
t.Fatal(err)
}
if a.ID == b.ID {
t.Errorf("不同站点下的同 slug 被当成同一个画廊了: %s", a.ID)
}
}
// 别名域名必须归一到主域名。
//
// 这是实测逼出来的:migmg.ssuu.uk 首页里 90 个 /content/ 链接全都指向
// www.4khd.com,而那个域名已经 404。不归一的话,同一个画廊会因为
// 「从哪个域名复制的链接」变成两个不同任务,后端还会去请求死域名。
func TestAliasNormalizesToPrimaryHost(t *testing.T) {
sites := DefaultSites()
path := "/content/09/myua-sweetbox-vol52.html"
want := "migmg.ssuu.uk" + path
for _, in := range []string{
"https://migmg.ssuu.uk" + path,
"https://www.4khd.com" + path,
"https://4khd.com" + path,
"https://m.4khd.com" + path,
} {
ref, err := ParseGalleryURL(in, sites)
if err != nil {
t.Errorf("ParseGalleryURL(%q) 报错: %v", in, err)
continue
}
if ref.ID != want {
t.Errorf("ParseGalleryURL(%q).ID = %q, 想要 %q", in, ref.ID, want)
}
if ref.Host != "migmg.ssuu.uk" {
t.Errorf("ParseGalleryURL(%q).Host = %q, 想要归一到 migmg.ssuu.uk", in, ref.Host)
}
if !strings.Contains(ref.URL, "https://migmg.ssuu.uk/") {
t.Errorf("ParseGalleryURL(%q).URL = %q, 应指向主域名", in, ref.URL)
}
}
}
func TestAllHostsIncludesAliases(t *testing.T) {
hosts := AllHosts(DefaultSites())
for _, want := range []string{"migmg.ssuu.uk", "www.4khd.com", "4khd.com", "m.4khd.com"} {
found := false
for _, h := range hosts {
if h == want {
found = true
break
}
}
if !found {
t.Errorf("AllHosts 缺了 %q: %v", want, hosts)
}
}
}
func TestParseGalleryURLRejectsNonGallery(t *testing.T) {
sites := DefaultSites()
bad := []string{
"",
"https://migmg.ssuu.uk/",
"https://migmg.ssuu.uk/pages/popular",
"https://example.com",
"随便什么",
}
for _, s := range bad {
if ref, err := ParseGalleryURL(s, sites); err == nil {
t.Errorf("ParseGalleryURL(%q) 应该报错, 却解析成了 %s", s, ref.ID)
}
}
}
func TestParseGalleryInput(t *testing.T) {
sites := DefaultSites()
text := `
# 注释行
https://migmg.ssuu.uk/content/09/a.html
https://migmg.ssuu.uk/content/09/b.html
https://migmg.ssuu.uk/content/09/a.html
// 另一个注释
https://migmg.ssuu.uk/content/09/c.html
`
refs := ParseGalleryInput(text, sites)
if len(refs) != 3 {
t.Fatalf("解析出 %d 个画廊, 想要 3(应去重、忽略注释与空行)", len(refs))
}
}
func TestNormalizeHost(t *testing.T) {
cases := []struct{ in, want string }{
{"https://migmg.ssuu.uk/", "migmg.ssuu.uk"},
{"http://MIGMG.SSUU.UK", "migmg.ssuu.uk"},
{"migmg.ssuu.uk", "migmg.ssuu.uk"},
{"migmg.ssuu.uk/content/09/x.html", "migmg.ssuu.uk"},
// 默认端口砍掉,非默认端口保留(本地 mock / 自建站要用)
{"https://migmg.ssuu.uk:443", "migmg.ssuu.uk"},
{"http://migmg.ssuu.uk:80", "migmg.ssuu.uk"},
{"https://migmg.ssuu.uk:8443", "migmg.ssuu.uk:8443"},
{"http://127.0.0.1:9000", "127.0.0.1:9000"},
}
for _, c := range cases {
if got := NormalizeHost(c.in); got != c.want {
t.Errorf("NormalizeHost(%q) = %q, 想要 %q", c.in, got, c.want)
}
}
}
// ---------------------------------------------------------------- 实体 / 文件名
func TestDecodeEntities(t *testing.T) {
cases := []struct{ in, want string }{
{"&#8211;", "–"},
{"&#038;", "&"},
{"&raquo;", "»"},
{"&amp;", "&"},
{"&lt;tag&gt;", "<tag>"},
{"&#x41;", "A"},
{"Myu_a [190MB]", "Myu_a [190MB]"}, // 下划线/方括号原样不动
}
for _, c := range cases {
if got := DecodeEntities(c.in); got != c.want {
t.Errorf("DecodeEntities(%q) = %q, 想要 %q", c.in, got, c.want)
}
}
}
func TestFilenameFromURL(t *testing.T) {
cases := []struct{ in, want string }{
{"https://i0.wp.com/pic.4khd.com/a/b/c/d/w1300-rw/x-001.webp?w=1300", "x-001.webp"},
{"https://i0.wp.com/pic.4khd.com/a/b/c/d/w1300-rw/x-001.webp", "x-001.webp"},
{"", ""},
}
for _, c := range cases {
if got := filenameFromURL(c.in); got != c.want {
t.Errorf("filenameFromURL(%q) = %q, 想要 %q", c.in, got, c.want)
}
}
}
func TestRenameUniqueHandlesDuplicateNames(t *testing.T) {
imgs := []ImageInfo{
{URL: "https://x/1.webp", Filename: "a.webp"},
{URL: "https://x/2.webp", Filename: "a.webp"},
{URL: "https://x/3.webp", Filename: "a.webp"},
}
renameUnique(imgs)
if imgs[0].Filename != "a.webp" {
t.Errorf("第 1 个应保持 a.webp, 实际 %q", imgs[0].Filename)
}
if imgs[1].Filename != "a(2).webp" {
t.Errorf("第 2 个应改名 a(2).webp, 实际 %q", imgs[1].Filename)
}
if imgs[2].Filename != "a(3).webp" {
t.Errorf("第 3 个应改名 a(3).webp, 实际 %q", imgs[2].Filename)
}
seen := map[string]bool{}
for _, im := range imgs {
if seen[im.Filename] {
t.Errorf("文件名仍重复: %s", im.Filename)
}
seen[im.Filename] = true
}
}
// ---------------------------------------------------------------- 端到端
// TestFetchGalleryEndToEnd 用 mock 服务器跑完整链路:
// 分页抓取 → 去重 → 排序 → 标题。响应体用真实夹具。
//
// 夹具里的 rel="next" / .page-links 指向 www.4khd.com(真实站点),
// 这里统一替换成 @@BASE@@ 占位符,在 handler 内部再替换成 mock 地址 ——
// 这样既能跑通全链路,又不会误请求真实站点。
func TestFetchGalleryEndToEnd(t *testing.T) {
paths := []string{
"/content/09/myua-sweetbox-vol52.html",
"/content/09/myua-sweetbox-vol52.html/2",
"/content/09/myua-sweetbox-vol52.html/3",
}
files := []string{"gallery-p1.html", "gallery-p2.html", "gallery-p3.html"}
// 路径 -> 已替换占位符的响应体
loaded := map[string]string{}
for i := range paths {
loaded[paths[i]] = strings.ReplaceAll(
loadFixture(t, files[i]), "https://www.4khd.com", "@@BASE@@")
}
var mu sync.Mutex
var gotPaths []string
var srvURL string // handler 内部才读,NewServer 返回后才赋值
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
mu.Lock()
gotPaths = append(gotPaths, r.URL.Path)
mu.Unlock()
body, ok := loaded[r.URL.Path]
if !ok {
http.NotFound(w, r)
return
}
w.Header().Set("Content-Type", "text/html; charset=utf-8")
fmt.Fprint(w, strings.ReplaceAll(body, "@@BASE@@", srvURL))
}))
defer srv.Close()
srvURL = srv.URL
ref, err := ParseGalleryURL(srv.URL+"/content/09/myua-sweetbox-vol52.html", DefaultSites())
if err != nil {
t.Fatal(err)
}
c := NewClient(0, "", "off", DefaultSites())
g, err := c.FetchGallery(context.Background(), ref, nil)
if err != nil {
t.Fatalf("FetchGallery 失败: %v", err)
}
defer c.Close()
if g.Total != 53 {
t.Errorf("图片总数 = %d, 想要 53", g.Total)
}
if g.TotalPages != 3 {
t.Errorf("总页数 = %d, 想要 3", g.TotalPages)
}
if !strings.Contains(g.Title, "SWEETBOX Vol.52") {
t.Errorf("标题不对: %q", g.Title)
}
for i, im := range g.Images {
if im.Index != i+1 {
t.Fatalf("第 %d 张的序号 = %d, 想要 %d", i, im.Index, i+1)
}
}
if g.Images[0].Filename != "tbox-vol52-4khd.com-001.webp" {
t.Errorf("首图文件名 = %q", g.Images[0].Filename)
}
if g.Images[52].Filename != "tbox-vol52-4khd.com-053.webp" {
t.Errorf("尾图文件名 = %q", g.Images[52].Filename)
}
mu.Lock()
seenPaths := append([]string(nil), gotPaths...)
mu.Unlock()
if len(seenPaths) != 3 {
t.Errorf("请求了 %d 次, 想要 3 次: %v", len(seenPaths), seenPaths)
}
}
func TestPageURL(t *testing.T) {
base := "https://migmg.ssuu.uk/content/09/x.html"
if got := PageURL(base, 1); got != base {
t.Errorf("第 1 页应等于原地址, 实际 %q", got)
}
if got := PageURL(base, 3); got != base+"/3" {
t.Errorf("第 3 页 = %q, 想要 %q", got, base+"/3")
}
}