Files
toor 0447875b6d feat: 双层并发 + 全局限流 + 整轮扫尾 + 画廊目录清单
原实现只能「多画廊并行,每个画廊串行下」,单张大画廊下得很慢。
按用户多年使用习惯(5 画廊 × 每画廊 8 张)改造为双层并发。

- 双层并发:画廊并发(默认5) × 页并发(默认8) = 最多 40 张同时下载,
  两个值都在设置面板可调(1-20)。页级 worker 池 + mutex 保护计数。
- 全局限流:对齐参考实现的 nextRequestTime 算法,限所有请求
  (含 /s/ 解析与图片本体),默认 0 秒关闭。
- 整轮扫尾:跑完后自动重扫失败页 1 轮(同样走完整 4 次退避重试),
  仍失败则停下并标记 partial,换线路后手动点开始即可续传。
- 画廊目录清单 .ehdl.json:记录 gid/token/总页数/已完成页,
  即使 state.json 丢失也能算出「应该多少张、还差多少张」。

顺带修复:
- Start() 双重加锁导致死锁(引擎完全无法启动)
- clampInt 越界时返回默认值而非边界值(设 30 会静默变 8)
- 直链正则写死 hath.network,CDN 换域名会全军覆没 -> 加图片 URL 兜底
- ReconcileTask 会误报「有改动」

测试从 37 增至 62,其中并发/限流/扫尾用 httptest 模拟 EH 实测:
- 页并发峰值 = 8、画廊并发峰值 = 5(完全生效)
- 12 次请求限流 80ms 实测 966ms(期望 880ms)
- 断点续传 4 张已存在时只发 5 次请求
2026-10-09 00:11:38 +08:00

377 lines
9.8 KiB
Go
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package core
import (
"context"
"os"
"path/filepath"
"testing"
"time"
"eh-downloader-wails/internal/eh"
)
func newTestStore(t *testing.T) (*Store, string) {
t.Helper()
dir := t.TempDir()
p := filepath.Join(dir, "state.json")
return NewStore(p), p
}
func TestStoreDefaults(t *testing.T) {
s, _ := newTestStore(t)
st := s.Settings()
if st.GalleryConcurrency != 5 || st.PageConcurrency != 8 {
t.Errorf("并发默认值不对: 画廊%d 页%d", st.GalleryConcurrency, st.PageConcurrency)
}
if st.RequestIntervalMs != 0 {
t.Errorf("请求间隔默认应为 0(不限流),实际 %d", st.RequestIntervalMs)
}
if st.Timeout != 30000 || st.BridgePort != 37777 {
t.Errorf("默认值不对: %+v", st)
}
if st.DirTemplate != "{title}" {
t.Errorf("默认模板不对: %q", st.DirTemplate)
}
if st.Xres != "auto" {
t.Errorf("默认 xres 不对: %q", st.Xres)
}
}
// 原子写 + 重新加载
func TestStorePersist(t *testing.T) {
s, path := newTestStore(t)
s.UpdateSettings(map[string]interface{}{
"galleryConcurrency": 6,
"pageConcurrency": 10,
"requestIntervalMs": 250,
"proxy": "http://127.0.0.1:7890",
})
s.Flush()
s2 := NewStore(path)
got := s2.Settings()
if got.GalleryConcurrency != 6 {
t.Errorf("画廊并发没存住: %d", got.GalleryConcurrency)
}
if got.PageConcurrency != 10 {
t.Errorf("页并发没存住: %d", got.PageConcurrency)
}
if got.RequestIntervalMs != 250 {
t.Errorf("请求间隔没存住: %d", got.RequestIntervalMs)
}
if got.Proxy != "http://127.0.0.1:7890" {
t.Errorf("代理没存住: %q", got.Proxy)
}
}
func TestStoreTasks(t *testing.T) {
s, _ := newTestStore(t)
refs := []eh.GalleryRef{
{GID: "1111111", Token: "aaa"},
{GID: "2222222", Token: "bbb"},
{GID: "1111111", Token: "aaa"}, // 重复
}
if n := s.AddTasks(refs); n != 2 {
t.Errorf("应新增 2 个(去重),实际 %d", n)
}
if len(s.Tasks()) != 2 {
t.Errorf("任务数不对: %d", len(s.Tasks()))
}
if s.FindTask("1111111", "aaa") == nil {
t.Error("找不到已加任务")
}
// SetTitle 只在标题为空时生效
if !s.SetTitle("1111111", "aaa", "标题A") {
t.Error("应设置标题")
}
if s.SetTitle("1111111", "aaa", "标题B") {
t.Error("标题非空时不应覆盖")
}
if s.FindTask("1111111", "aaa").Title != "标题A" {
t.Error("标题不对")
}
}
// 清除只动列表不动文件
func TestStoreClearAll(t *testing.T) {
s, _ := newTestStore(t)
s.AddTasks([]eh.GalleryRef{
{GID: "1", Token: "a"},
{GID: "2", Token: "b"},
{GID: "3", Token: "c"},
})
// 标记第 2 个为选中
s.SelectAll([]string{"2"})
if n := s.ClearAll(true); n != 1 {
t.Errorf("删选中应删 1 个,实际 %d", n)
}
if len(s.Tasks()) != 2 {
t.Errorf("剩余任务数不对: %d", len(s.Tasks()))
}
if n := s.ClearAll(false); n != 2 {
t.Errorf("删全部应删 2 个,实际 %d", n)
}
if len(s.Tasks()) != 0 {
t.Error("应清空")
}
}
// ClearFinished 只清 done,partial/failed 保留
func TestStoreClearFinished(t *testing.T) {
s, _ := newTestStore(t)
for _, g := range []struct{ gid, status string }{
{"1", "done"},
{"2", "partial"},
{"3", "failed"},
{"4", "queued"},
} {
s.AddTasks([]eh.GalleryRef{{GID: g.gid, Token: "t"}})
s.UpsertTask(Task{GID: g.gid, Token: "t", Status: g.status})
}
n := s.ClearFinished()
if n != 1 {
t.Errorf("应只清 1 个 done,实际 %d", n)
}
for _, task := range s.Tasks() {
if task.GID == "1" {
t.Error("done 的不该还在")
}
}
if len(s.Tasks()) != 3 {
t.Errorf("应剩 3 个,实际 %d", len(s.Tasks()))
}
}
func TestStoreRequeueAndRecover(t *testing.T) {
s, _ := newTestStore(t)
for _, g := range []struct{ gid, status string }{
{"1", "failed"},
{"2", "partial"},
{"3", "running"}, // 崩溃残留
{"4", "downloading"}, // 崩溃残留
{"5", "done"},
} {
s.AddTasks([]eh.GalleryRef{{GID: g.gid, Token: "t"}})
s.UpsertTask(Task{GID: g.gid, Token: "t", Status: g.status})
}
// RequeueFailed 会把 failed/partial/running/downloading 一起重排
if n := s.RequeueFailed(); n != 4 {
t.Errorf("失败/部分/中断应重排 4 个,实际 %d", n)
}
if n := s.RecoverOnStart(); n != 0 {
t.Errorf("已全部重排,不该再有残留,实际 %d", n)
}
if n := s.CountDone(); n != 1 {
t.Errorf("done 应剩 1 个,实际 %d", n)
}
// 单独验证 RecoverOnStart:手动塞个 running 残留
s.UpsertTask(Task{GID: "6", Token: "t", Status: "downloading"})
if n := s.RecoverOnStart(); n != 1 {
t.Errorf("running/downloading 应归位 1 个,实际 %d", n)
}
if s.FindTask("6", "t").Status != "queued" {
t.Error("残留任务应被归位为 queued")
}
}
// 状态文件损坏时回落默认值,不崩
func TestStoreCorruptedFile(t *testing.T) {
dir := t.TempDir()
p := filepath.Join(dir, "state.json")
os.WriteFile(p, []byte("{ 这不是合法 JSON"), 0o644)
s := NewStore(p)
if s.Settings().GalleryConcurrency != 5 {
t.Error("损坏文件应回落到默认设置")
}
if len(s.Tasks()) != 0 {
t.Error("损坏文件应回落空任务")
}
}
// Tasks() 返回副本,改它不影响内部状态
func TestStoreTasksIsCopy(t *testing.T) {
s, _ := newTestStore(t)
s.AddTasks([]eh.GalleryRef{{GID: "1", Token: "t", Raw: "orig"}})
got := s.Tasks()
got[0].Status = "done"
if s.FindTask("1", "t").Status == "done" {
t.Error("Tasks() 应返回副本,外部修改不该影响内部")
}
}
// ---- 引擎:断点续传的核心是「文件在就算完成」----
func TestEngineResumeSkipsExisting(t *testing.T) {
s, _ := newTestStore(t)
root := t.TempDir()
s.UpdateSettings(map[string]interface{}{"downloadRoot": root, "dirTemplate": "{gid}"})
dir := filepath.Join(root, "12345678")
os.MkdirAll(dir, 0o755)
// 造一个已存在的文件 + 一个空文件
full := filepath.Join(dir, "001.webp")
os.WriteFile(full, []byte("fake image data"), 0o644)
empty := filepath.Join(dir, "002.webp")
os.WriteFile(empty, []byte{}, 0o644)
// targetName 的行为验证:页码补零
if got := targetName(ehPageInfo("001.webp", 1), "12345678", 3); got != "001.webp" {
t.Errorf("文件名不对: %q", got)
}
if st, _ := os.Stat(full); st == nil || st.Size() == 0 {
t.Error("测试前提失败:文件应存在且非空")
}
}
func TestEngineConcurrencyClamp(t *testing.T) {
s, _ := newTestStore(t)
e := NewEngine(s, nil, s.Settings())
// 默认
if e.GalleryConcurrency() != 5 {
t.Errorf("默认画廊并发应为 5,实际 %d", e.GalleryConcurrency())
}
if e.PageConcurrency() != 8 {
t.Errorf("默认页并发应为 8,实际 %d", e.PageConcurrency())
}
// 越界要夹到合理范围而不是 panic
e.SetGalleryConcurrency(99)
if e.GalleryConcurrency() != 20 {
t.Errorf("画廊并发应夹到 20,实际 %d", e.GalleryConcurrency())
}
e.SetGalleryConcurrency(0)
if e.GalleryConcurrency() != 5 {
t.Errorf("画廊并发 0 应回落默认 5,实际 %d", e.GalleryConcurrency())
}
e.SetPageConcurrency(99)
if e.PageConcurrency() != 20 {
t.Errorf("页并发应夹到 20,实际 %d", e.PageConcurrency())
}
e.SetPageConcurrency(-1)
if e.PageConcurrency() != 8 {
t.Errorf("页并发 -1 应回落默认 8,实际 %d", e.PageConcurrency())
}
// 正常范围
e.SetGalleryConcurrency(5)
e.SetPageConcurrency(8)
if e.GalleryConcurrency()*e.PageConcurrency() != 40 {
t.Errorf("理论总并发应为 40,实际 %d", e.GalleryConcurrency()*e.PageConcurrency())
}
}
// 旧配置文件(只有 concurrency 字段)要能平滑迁移
func TestSettingsMigration(t *testing.T) {
dir := t.TempDir()
p := filepath.Join(dir, "state.json")
// 模拟旧版 state.json:只有 concurrency,没有新的三个字段
old := `{
"settings": {
"downloadRoot": "D:/dl",
"proxy": "http://127.0.0.1:7890",
"proxyMode": "http",
"concurrency": 4,
"timeout": 45000,
"xres": "auto",
"bridgePort": 37777,
"dirTemplate": "{title}"
},
"tasks": []
}`
os.WriteFile(p, []byte(old), 0o644)
s := NewStore(p)
got := s.Settings()
if got.GalleryConcurrency != 5 {
t.Errorf("旧配置迁移后画廊并发应为默认 5,实际 %d", got.GalleryConcurrency)
}
if got.PageConcurrency != 8 {
t.Errorf("旧配置迁移后页并发应为默认 8,实际 %d", got.PageConcurrency)
}
if got.Proxy != "http://127.0.0.1:7890" {
t.Errorf("旧配置的代理丢了: %q", got.Proxy)
}
if got.Timeout != 45000 {
t.Errorf("旧配置的超时丢了: %d", got.Timeout)
}
}
func TestEngineStopIsSafeWhenIdle(t *testing.T) {
s, _ := newTestStore(t)
e := NewEngine(s, nil, s.Settings())
e.Stop() // 不在跑时调 Stop 不该 panic
if e.IsRunning() {
t.Error("不该在跑")
}
}
func TestEngineStartEmptyQueue(t *testing.T) {
s, _ := newTestStore(t)
e := NewEngine(s, &noopSink{}, s.Settings())
done := make(chan struct{})
go func() { e.Start(context.Background()); close(done) }()
select {
case <-done:
case <-time.After(3 * time.Second):
t.Fatal("空队列应该立刻返回")
}
}
type noopSink struct{}
func (n *noopSink) TaskUpdate(Task) {}
func (n *noopSink) Log(string, string) {}
func (n *noopSink) Done() {}
// 错误可重试性判断
func TestIsRetryable(t *testing.T) {
retryable := []error{
errStr("connection reset by peer"),
errStr("dial tcp: i/o timeout"),
errStr("HTTP 503"),
errStr("HTTP 429"),
errStr("socket hang up"),
}
for _, e := range retryable {
if !isRetryable(e) {
t.Errorf("应判为可重试: %v", e)
}
}
notRetryable := []error{
nil,
ErrCanceled,
context.Canceled,
errStr("未能从 /s/ 页面解析出图片地址"),
}
for _, e := range notRetryable {
if isRetryable(e) {
t.Errorf("应判为不可重试: %v", e)
}
}
}
type errStr string
func (e errStr) Error() string { return string(e) }
func ehPageInfo(name string, page int) eh.GalleryPageInfo {
return eh.GalleryPageInfo{Page: page, Filename: name}
}