feat: 双层并发 + 全局限流 + 整轮扫尾 + 画廊目录清单
原实现只能「多画廊并行,每个画廊串行下」,单张大画廊下得很慢。 按用户多年使用习惯(5 画廊 × 每画廊 8 张)改造为双层并发。 - 双层并发:画廊并发(默认5) × 页并发(默认8) = 最多 40 张同时下载, 两个值都在设置面板可调(1-20)。页级 worker 池 + mutex 保护计数。 - 全局限流:对齐参考实现的 nextRequestTime 算法,限所有请求 (含 /s/ 解析与图片本体),默认 0 秒关闭。 - 整轮扫尾:跑完后自动重扫失败页 1 轮(同样走完整 4 次退避重试), 仍失败则停下并标记 partial,换线路后手动点开始即可续传。 - 画廊目录清单 .ehdl.json:记录 gid/token/总页数/已完成页, 即使 state.json 丢失也能算出「应该多少张、还差多少张」。 顺带修复: - Start() 双重加锁导致死锁(引擎完全无法启动) - clampInt 越界时返回默认值而非边界值(设 30 会静默变 8) - 直链正则写死 hath.network,CDN 换域名会全军覆没 -> 加图片 URL 兜底 - ReconcileTask 会误报「有改动」 测试从 37 增至 62,其中并发/限流/扫尾用 httptest 模拟 EH 实测: - 页并发峰值 = 8、画廊并发峰值 = 5(完全生效) - 12 次请求限流 80ms 实测 966ms(期望 880ms) - 断点续传 4 张已存在时只发 5 次请求
This commit is contained in:
@@ -191,8 +191,14 @@ func (a *App) SetSettings(patch map[string]interface{}) core.Settings {
|
||||
next := a.store.UpdateSettings(patch)
|
||||
|
||||
// 同步到引擎
|
||||
if _, ok := patch["concurrency"]; ok {
|
||||
a.engine.SetConcurrency(next.Concurrency)
|
||||
if _, ok := patch["galleryConcurrency"]; ok {
|
||||
a.engine.SetGalleryConcurrency(next.GalleryConcurrency)
|
||||
}
|
||||
if _, ok := patch["pageConcurrency"]; ok {
|
||||
a.engine.SetPageConcurrency(next.PageConcurrency)
|
||||
}
|
||||
if _, ok := patch["requestIntervalMs"]; ok {
|
||||
a.engine.SetRequestInterval(next.RequestIntervalMs)
|
||||
}
|
||||
if _, ok := patch["timeout"]; ok {
|
||||
a.engine.Client().SetTimeout(time.Duration(next.Timeout) * time.Millisecond)
|
||||
@@ -241,6 +247,69 @@ func (a *App) Placeholders() []string {
|
||||
return core.ListPlaceholders()
|
||||
}
|
||||
|
||||
// EngineInfo 引擎当前的并发配置(前端用来显示"总上限 = 画廊 × 页")
|
||||
type EngineInfo struct {
|
||||
Running bool `json:"running"`
|
||||
Queued int `json:"queued"`
|
||||
GalleryConcurrency int `json:"galleryConcurrency"`
|
||||
PageConcurrency int `json:"pageConcurrency"`
|
||||
MaxConcurrent int `json:"maxConcurrent"`
|
||||
RequestIntervalMs int `json:"requestIntervalMs"`
|
||||
}
|
||||
|
||||
// EngineInfo 取引擎状态和并发配置
|
||||
func (a *App) EngineInfo() EngineInfo {
|
||||
g := a.engine.GalleryConcurrency()
|
||||
p := a.engine.PageConcurrency()
|
||||
return EngineInfo{
|
||||
Running: a.engine.IsRunning(),
|
||||
Queued: a.engine.QueueLength(),
|
||||
GalleryConcurrency: g,
|
||||
PageConcurrency: p,
|
||||
MaxConcurrent: g * p,
|
||||
RequestIntervalMs: int(a.engine.Client().RequestInterval().Milliseconds()),
|
||||
}
|
||||
}
|
||||
|
||||
// ScanManifestsResult 扫描下载目录的结果
|
||||
type ScanManifestsResult struct {
|
||||
Found int `json:"found"`
|
||||
Adopted int `json:"adopted"`
|
||||
Root string `json:"root"`
|
||||
Note string `json:"note"`
|
||||
}
|
||||
|
||||
// ScanDownloadDir 扫描下载目录里的画廊清单,把 state.json 里缺的信息补上。
|
||||
// 场景:用户删了 state.json,靠清单重建"这个画廊应该有多少张"的认知。
|
||||
func (a *App) ScanDownloadDir() ScanManifestsResult {
|
||||
root := a.store.Settings().DownloadRoot
|
||||
if root == "" {
|
||||
root = core.DefaultDownloadRoot()
|
||||
}
|
||||
|
||||
manifests := core.ScanManifests(root)
|
||||
res := ScanManifestsResult{Root: root, Found: len(manifests)}
|
||||
if len(manifests) == 0 {
|
||||
res.Note = "下载目录里没找到画廊清单(.ehdl.json)"
|
||||
return res
|
||||
}
|
||||
|
||||
for _, m := range manifests {
|
||||
// 找到对应任务(按 gid 匹配,token 可能有差异所以只按 gid)
|
||||
t := a.store.FindTaskByGID(m.GID)
|
||||
if t == nil {
|
||||
continue
|
||||
}
|
||||
if core.ReconcileTask(t, m) {
|
||||
a.store.UpsertTask(*t)
|
||||
res.Adopted++
|
||||
}
|
||||
}
|
||||
|
||||
res.Note = fmt.Sprintf("扫描到 %d 个画廊清单,补全了 %d 个任务", res.Found, res.Adopted)
|
||||
return res
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- 任务
|
||||
|
||||
// ListTasks 全部任务
|
||||
@@ -368,7 +437,16 @@ func (a *App) Start(onlyGids []string) StartResult {
|
||||
}
|
||||
go a.engine.Start(ctx)
|
||||
|
||||
a.emitLog("info", fmt.Sprintf("开始下载 %d 个画廊(并发 %d)", len(targets), len(targets)))
|
||||
g := a.engine.GalleryConcurrency()
|
||||
p := a.engine.PageConcurrency()
|
||||
iv := a.engine.Client().RequestInterval().Milliseconds()
|
||||
msg := fmt.Sprintf("开始下载 %d 个画廊(画廊并发 %d × 页并发 %d,最多 %d 张同时下载)",
|
||||
len(targets), g, p, g*p)
|
||||
if iv > 0 {
|
||||
msg += fmt.Sprintf(",请求间隔 %dms", iv)
|
||||
}
|
||||
a.emitLog("info", msg)
|
||||
|
||||
return StartResult{Started: true, Count: len(targets)}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user