first commit
This commit is contained in:
@@ -0,0 +1,468 @@
|
||||
package core
|
||||
|
||||
// 引擎的端到端集成测试：起一个本地 HTTP 服务器同时扮演
|
||||
// 「画廊站点」和「图片 CDN」，把 解析 → 下载 → 写清单 → 状态机
|
||||
// 整条链路跑通，并验证重启后能靠清单零请求续传。
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"hd4k-downloader/internal/hd4k"
|
||||
)
|
||||
|
||||
// mockSite 一个假的 4KHD 站点。
|
||||
//
|
||||
// 页面形态照抄真实站点：正文图片在 entry-content 里、每张图既是 <a href>
|
||||
// 又是 <img src>（所以真实抓取必须去重，否则数量翻倍），
|
||||
// 推荐位图片混在里面且带 w1090-h1500 特征。
|
||||
type mockSite struct {
|
||||
mu sync.Mutex
|
||||
imgHits map[string]int // 每张图被请求了几次
|
||||
pageHits map[string]int // 每个画廊页被请求了几次
|
||||
imgData map[string][]byte
|
||||
}
|
||||
|
||||
func newMockSite() *mockSite {
|
||||
return &mockSite{
|
||||
imgHits: map[string]int{},
|
||||
pageHits: map[string]int{},
|
||||
imgData: map[string][]byte{},
|
||||
}
|
||||
}
|
||||
|
||||
const mockSlug = "myua-sweetbox-vol52"
|
||||
const mockPath = "/content/09/" + mockSlug + ".html"
|
||||
|
||||
// galleryHTML 造一页画廊 HTML：2 张正文图 + 1 张推荐图 + 1 张重复出现的正文图
|
||||
//
|
||||
// 构造器。注意：httptest server 的并发 handler 会同时调用它，
|
||||
// 所有共享状态的读写都必须走 m.mu。
|
||||
func (m *mockSite) galleryHTML(base, baseURL string, page, totalPages, firstIdx int) string {
|
||||
var b strings.Builder
|
||||
img := func(i int) string {
|
||||
name := fmt.Sprintf("%s-%03d.webp", mockSlug, i)
|
||||
m.mu.Lock()
|
||||
m.imgData[name] = []byte(fmt.Sprintf("WEBP-DATA-%03d-%s", i, strings.Repeat("x", 500+i)))
|
||||
m.mu.Unlock()
|
||||
return baseURL + "/img/" + name
|
||||
}
|
||||
|
||||
b.WriteString(`<!DOCTYPE html><html><head><title>Test Gallery - 4KHD</title>`)
|
||||
b.WriteString(`<link rel="canonical" href="https://www.4khd.com` + mockPath + `" />`)
|
||||
b.WriteString(`</head><body><h1>4KHD</h1>`)
|
||||
b.WriteString(`<h3 class="wp-block-post-title">Myu_a SWEETBOX Vol.52[190MB-53photos]</h3>`)
|
||||
b.WriteString(`<div class="entry-content">`)
|
||||
|
||||
for i := 0; i < 2; i++ {
|
||||
u := img(firstIdx + i)
|
||||
// 真实站点里 href 和 src 是同一个 URL → 解析必须去重
|
||||
fmt.Fprintf(&b, `<p><a href="%s"><img src="%s" width="1300" height="1500"/></a><br /></p>`, u, u)
|
||||
}
|
||||
// 推荐位图片：必须被过滤掉
|
||||
b.WriteString(`<p><a href="` + baseURL + `/pic.4khd.com/recommend/4KHD-beautifulGirls.webp?resize=1,300">`)
|
||||
b.WriteString(`<img src="` + baseURL + `/pic.4khd.com/recommend/4KHD-beautifulGirls.webp?resize=1,300"/></a></p>`)
|
||||
|
||||
b.WriteString(`</div>`)
|
||||
|
||||
// 分页盒。形态照抄真实页面：**每个页码是独立的 <li class="numpages">**
|
||||
// （实测真实 HTML 是这样，解析器取所有 li 里的最大页码）。
|
||||
// 之前这个 mock 把所有页码塞进一个 li，又不留分隔符，
|
||||
// 去标签后变成 "12"，害得差点去改本来正确的解析逻辑。
|
||||
b.WriteString(`<ul class="page-links">`)
|
||||
for i := 1; i <= totalPages; i++ {
|
||||
if i == page {
|
||||
fmt.Fprintf(&b, `<li class="numpages current"><span>%d</span></li>`, i)
|
||||
} else {
|
||||
fmt.Fprintf(&b, `<li class="numpages"><a class="page-numbers" href="%s%s/%d">%d</a></li>`,
|
||||
base, mockPath, i, i)
|
||||
}
|
||||
}
|
||||
b.WriteString(`</ul>`)
|
||||
|
||||
// rel=next 指向已死的 4khd.com —— 引擎绝不能跟这条链，否则会去请求死域名
|
||||
b.WriteString(`<link rel="next" href="https://www.4khd.com` + mockPath + `/2" />`)
|
||||
b.WriteString(`</body></html>`)
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func (m *mockSite) start(t *testing.T) *httptest.Server {
|
||||
t.Helper()
|
||||
var srv *httptest.Server
|
||||
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
p := r.URL.Path
|
||||
|
||||
if strings.HasPrefix(p, "/img/") {
|
||||
name := filepath.Base(p)
|
||||
m.mu.Lock()
|
||||
m.imgHits[name]++
|
||||
data, ok := m.imgData[name]
|
||||
m.mu.Unlock()
|
||||
if !ok {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "image/webp")
|
||||
w.Write(data)
|
||||
return
|
||||
}
|
||||
|
||||
if p == mockPath || strings.HasPrefix(p, mockPath+"/") {
|
||||
page := 1
|
||||
if seg := strings.TrimPrefix(p, mockPath); seg != "" {
|
||||
if n, err := strconv.Atoi(strings.Trim(seg, "/")); err == nil {
|
||||
page = n
|
||||
}
|
||||
}
|
||||
m.mu.Lock()
|
||||
m.pageHits[fmt.Sprint(page)]++
|
||||
m.mu.Unlock()
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
fmt.Fprint(w, m.galleryHTML(srv.URL, srv.URL, page, 2, (page-1)*2+1))
|
||||
return
|
||||
}
|
||||
|
||||
http.NotFound(w, r)
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
return srv
|
||||
}
|
||||
|
||||
func (m *mockSite) counts() (imgs, pages map[string]int) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
imgs = map[string]int{}
|
||||
for k, v := range m.imgHits {
|
||||
imgs[k] = v
|
||||
}
|
||||
pages = map[string]int{}
|
||||
for k, v := range m.pageHits {
|
||||
pages[k] = v
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- 测试
|
||||
|
||||
func TestEngineEndToEndDownload(t *testing.T) {
|
||||
site := newMockSite()
|
||||
srv := site.start(t)
|
||||
root := t.TempDir()
|
||||
|
||||
s := newTestStore(t)
|
||||
s.UpdateSettings(map[string]interface{}{
|
||||
"downloadRoot": root,
|
||||
"galleryConcurrency": 1,
|
||||
"pageConcurrency": 2,
|
||||
"timeout": 15000,
|
||||
"requestIntervalMs": 0,
|
||||
"sites": []hd4k.Site{{Host: "127.0.0.1", Scheme: "http", Enabled: true}},
|
||||
})
|
||||
|
||||
ref, err := hd4k.ParseGalleryURL(srv.URL+mockPath, s.Settings().Sites)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.AddTasks([]hd4k.GalleryRef{ref})
|
||||
|
||||
done := make(chan string, 16)
|
||||
eng := NewEngine(s, &testSink{done: done}, s.Settings())
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
eng.Enqueue(s.Tasks())
|
||||
go eng.Start(ctx)
|
||||
|
||||
select {
|
||||
case status := <-done:
|
||||
tk := s.FindTask(ref.ID)
|
||||
if status != "done" {
|
||||
t.Fatalf("引擎状态 = %q, 想要 done (err=%s done=%d/%d failed=%d list=%+v)",
|
||||
status, tk.Error, tk.Done, tk.Total, tk.Failed, tk.FailedList)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
t.Fatal("引擎超时")
|
||||
}
|
||||
|
||||
task := s.FindTask(ref.ID)
|
||||
if task.Status != "done" {
|
||||
t.Errorf("任务状态 = %q, 想要 done (err=%s, done=%d, failed=%d, failedList=%+v)",
|
||||
task.Status, task.Error, task.Done, task.Failed, task.FailedList)
|
||||
}
|
||||
if task.Done != 4 || task.Failed != 0 {
|
||||
t.Errorf("下载计数 = %d 成 / %d 败, 想要 4/0", task.Done, task.Failed)
|
||||
}
|
||||
|
||||
// 推荐位图片绝不能落盘
|
||||
dir := filepath.Join(root, "Myu_a SWEETBOX Vol.52[190MB-53photos]")
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("目标目录不存在: %v", err)
|
||||
}
|
||||
var files []string
|
||||
for _, e := range entries {
|
||||
if !strings.HasPrefix(e.Name(), ".") {
|
||||
files = append(files, e.Name())
|
||||
}
|
||||
}
|
||||
if len(files) != 4 {
|
||||
names := strings.Join(files, ", ")
|
||||
t.Errorf("落盘文件数 = %d, 想要 4: %s", len(files), names)
|
||||
}
|
||||
for _, f := range files {
|
||||
if strings.Contains(f, "beautifulGirls") {
|
||||
t.Errorf("推荐位图片被下载了: %s", f)
|
||||
}
|
||||
}
|
||||
|
||||
// 文件内容要和服务器给的一致（确认没写半截 / 没串页）
|
||||
for _, f := range files {
|
||||
got, err := os.ReadFile(filepath.Join(dir, f))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want, ok := site.imgData[f]
|
||||
if !ok {
|
||||
t.Errorf("落盘了未知文件: %s", f)
|
||||
continue
|
||||
}
|
||||
if string(got) != string(want) {
|
||||
t.Errorf("%s 内容不对: 拿到 %d 字节, 想要 %d", f, len(got), len(want))
|
||||
}
|
||||
}
|
||||
|
||||
// 清单必须落盘，否则重启无法续传
|
||||
if _, err := os.Stat(filepath.Join(dir, ManifestFileName)); err != nil {
|
||||
t.Errorf("清单没写成: %v", err)
|
||||
}
|
||||
|
||||
// 请求次数：每页 1 次、每图 1 次。绝不能去追 rel=next 指向的死域名。
|
||||
imgs, pages := site.counts()
|
||||
if len(pages) != 2 {
|
||||
t.Errorf("请求了 %d 个画廊页, 想要 2: %v", len(pages), pages)
|
||||
}
|
||||
for name, n := range imgs {
|
||||
if n != 1 {
|
||||
t.Errorf("图片 %s 被请求了 %d 次, 想要 1", name, n)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type testSink struct{ done chan string }
|
||||
|
||||
func (s *testSink) TaskUpdate(t Task) {
|
||||
if t.Status == "done" || t.Status == "failed" || t.Status == "partial" {
|
||||
select {
|
||||
case s.done <- t.Status:
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
func (s *testSink) Log(level, msg string) {}
|
||||
func (s *testSink) Done() {}
|
||||
|
||||
// 断点续传：清单存在时，重跑必须**一个网络请求都不发**。
|
||||
// 这是 4KHD 相对 EH 的最大优势 —— 直链是稳定的 URL 哈希，
|
||||
// 不像 EH 的 hath 签名链接会过期。
|
||||
func TestEngineResumeFromManifestZeroRequests(t *testing.T) {
|
||||
site := newMockSite()
|
||||
srv := site.start(t)
|
||||
root := t.TempDir()
|
||||
|
||||
s := newTestStore(t)
|
||||
s.UpdateSettings(map[string]interface{}{
|
||||
"downloadRoot": root,
|
||||
"sites": []hd4k.Site{{Host: "127.0.0.1", Scheme: "http", Enabled: true}},
|
||||
"timeout": 15000,
|
||||
})
|
||||
|
||||
ref, err := hd4k.ParseGalleryURL(srv.URL+mockPath, s.Settings().Sites)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
g := &hd4k.Gallery{
|
||||
ID: ref.ID, URL: ref.URL, Host: ref.Host, Path: ref.Path, Slug: ref.Slug,
|
||||
Title: "Myu_a SWEETBOX Vol.52[190MB-53photos]", TotalPages: 2, Total: 4,
|
||||
}
|
||||
|
||||
// 假装第一轮已经下完 4 张，清单也写好了
|
||||
dir := BuildDir(DefaultSettings().DirTemplate, g)
|
||||
full := filepath.Join(root, dir)
|
||||
if err := os.MkdirAll(full, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pageMap := map[string]ManifestPage{}
|
||||
n := 0
|
||||
for i := 1; i <= 4; i++ {
|
||||
name := fmt.Sprintf("%s-%03d.webp", mockSlug, i)
|
||||
site.imgData[name] = []byte("DATA")
|
||||
if err := os.WriteFile(filepath.Join(full, name), []byte("DATA"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
n++
|
||||
pageMap[fmt.Sprint(i)] = ManifestPage{Page: i, File: name, Finished: true,
|
||||
URL: "https://i0.wp.com/pic.4khd.com/x/" + name + "?w=1300"}
|
||||
}
|
||||
meta := g.Meta
|
||||
if err := WriteManifest(full, Manifest{
|
||||
ID: g.ID, URL: g.URL, Host: g.Host, Path: g.Path, Slug: g.Slug,
|
||||
Title: g.Title, TotalPages: 2, Total: 4,
|
||||
DonePages: []int{1, 2, 3, 4}, PageMap: pageMap, Meta: &meta,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
s.AddTasks([]hd4k.GalleryRef{ref})
|
||||
s.UpsertTask(Task{ID: ref.ID, Title: g.Title, Total: 4, Status: "failed", Done: 2})
|
||||
|
||||
done := make(chan string, 8)
|
||||
eng := NewEngine(s, &testSink{done: done}, s.Settings())
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
eng.Enqueue(s.Tasks())
|
||||
go eng.Start(ctx)
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("续传超时")
|
||||
}
|
||||
|
||||
imgs, pages := site.counts()
|
||||
if len(imgs) != 0 || len(pages) != 0 {
|
||||
t.Errorf("靠清单续传时不该发任何请求，实际 图片=%v 页面=%v", imgs, pages)
|
||||
}
|
||||
if got := s.FindTask(ref.ID).Status; got != "done" {
|
||||
t.Errorf("续传后状态 = %q, 想要 done", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 清单里信息不全（比如旧版本只存了文件名没存直链）时，
|
||||
// 必须老实回退到重新解析，而不是拿空 URL 去下载。
|
||||
func TestEngineManifestWithoutURLFallsBackToParsing(t *testing.T) {
|
||||
site := newMockSite()
|
||||
srv := site.start(t)
|
||||
root := t.TempDir()
|
||||
|
||||
s := newTestStore(t)
|
||||
s.UpdateSettings(map[string]interface{}{
|
||||
"downloadRoot": root,
|
||||
"sites": []hd4k.Site{{Host: "127.0.0.1", Scheme: "http", Enabled: true}},
|
||||
"timeout": 15000,
|
||||
})
|
||||
|
||||
ref, err := hd4k.ParseGalleryURL(srv.URL+mockPath, s.Settings().Sites)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
g := &hd4k.Gallery{
|
||||
ID: ref.ID, URL: ref.URL, Host: ref.Host, Path: ref.Path, Slug: ref.Slug,
|
||||
Title: "Myu_a SWEETBOX Vol.52[190MB-53photos]", TotalPages: 2, Total: 4,
|
||||
}
|
||||
full := filepath.Join(root, BuildDir(DefaultSettings().DirTemplate, g))
|
||||
if err := os.MkdirAll(full, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// PageMap 里没有 URL —— 不可用
|
||||
if err := WriteManifest(full, Manifest{
|
||||
ID: g.ID, Title: g.Title, Total: 4,
|
||||
PageMap: map[string]ManifestPage{"1": {Page: 1, File: "x.webp", Finished: true}},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
s.AddTasks([]hd4k.GalleryRef{ref})
|
||||
done := make(chan string, 8)
|
||||
eng := NewEngine(s, &testSink{done: done}, s.Settings())
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
eng.Enqueue(s.Tasks())
|
||||
go eng.Start(ctx)
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("回退路径超时")
|
||||
}
|
||||
|
||||
// 清单不可用 → 必须去站点重新解析（页面有请求）
|
||||
_, pages := site.counts()
|
||||
if len(pages) == 0 {
|
||||
t.Error("清单信息不全时应回退到重新抓页面，但一个请求都没发")
|
||||
}
|
||||
if got := s.FindTask(ref.ID).Status; got != "done" {
|
||||
t.Errorf("回退后状态 = %q, 想要 done", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 改目录模板后重跑，不该重下已有的图。
|
||||
// 引擎要先在老目录（标题名）里找到已下好的文件。
|
||||
func TestEngineDirTemplateChangeDoesNotRedownload(t *testing.T) {
|
||||
site := newMockSite()
|
||||
srv := site.start(t)
|
||||
root := t.TempDir()
|
||||
|
||||
s := newTestStore(t)
|
||||
s.UpdateSettings(map[string]interface{}{
|
||||
"downloadRoot": root,
|
||||
"dirTemplate": "{title}",
|
||||
"sites": []hd4k.Site{{Host: "127.0.0.1", Scheme: "http", Enabled: true}},
|
||||
"timeout": 15000,
|
||||
})
|
||||
|
||||
ref, err := hd4k.ParseGalleryURL(srv.URL+mockPath, s.Settings().Sites)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// 第一轮用 {title} 模板全量下载
|
||||
s.AddTasks([]hd4k.GalleryRef{ref})
|
||||
done1 := make(chan string, 8)
|
||||
eng1 := NewEngine(s, &testSink{done: done1}, s.Settings())
|
||||
ctx1, cancel1 := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel1()
|
||||
eng1.Enqueue(s.Tasks())
|
||||
go eng1.Start(ctx1)
|
||||
select {
|
||||
case <-done1:
|
||||
case <-ctx1.Done():
|
||||
t.Fatal("第一轮超时")
|
||||
}
|
||||
|
||||
before, _ := site.counts()
|
||||
|
||||
// 换成 {category}/{name} 模板，清空请求计数后重跑
|
||||
s.UpdateSettings(map[string]interface{}{"dirTemplate": "{category}/{name}"})
|
||||
site.mu.Lock()
|
||||
site.imgHits = map[string]int{}
|
||||
site.pageHits = map[string]int{}
|
||||
site.mu.Unlock()
|
||||
|
||||
s.UpsertTask(Task{ID: ref.ID, Title: "Myu_a SWEETBOX Vol.52[190MB-53photos]", Total: 4, Status: "failed", Done: 1})
|
||||
done2 := make(chan string, 8)
|
||||
eng2 := NewEngine(s, &testSink{done: done2}, s.Settings())
|
||||
ctx2, cancel2 := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel2()
|
||||
eng2.Enqueue(s.Tasks())
|
||||
go eng2.Start(ctx2)
|
||||
select {
|
||||
case <-done2:
|
||||
case <-ctx2.Done():
|
||||
t.Fatal("第二轮超时")
|
||||
}
|
||||
|
||||
imgs2, _ := site.counts()
|
||||
if len(imgs2) != 0 {
|
||||
t.Errorf("换模板后重下��� %d 张图, 想要 0（应复用老目录里的文件）: %v", len(imgs2), imgs2)
|
||||
}
|
||||
_ = before
|
||||
}
|
||||
Reference in New Issue
Block a user