feat: 双层并发 + 全局限流 + 整轮扫尾 + 画廊目录清单
原实现只能「多画廊并行，每个画廊串行下」，单张大画廊下得很慢。 按用户多年使用习惯（5 画廊 × 每画廊 8 张）改造为双层并发。 - 双层并发：画廊并发(默认5) × 页并发(默认8) = 最多 40 张同时下载， 两个值都在设置面板可调（1-20）。页级 worker 池 + mutex 保护计数。 - 全局限流：对齐参考实现的 nextRequestTime 算法，限所有请求 （含 /s/ 解析与图片本体），默认 0 秒关闭。 - 整轮扫尾：跑完后自动重扫失败页 1 轮（同样走完整 4 次退避重试）， 仍失败则停下并标记 partial，换线路后手动点开始即可续传。 - 画廊目录清单 .ehdl.json：记录 gid/token/总页数/已完成页， 即使 state.json 丢失也能算出「应该多少张、还差多少张」。 顺带修复： - Start() 双重加锁导致死锁（引擎完全无法启动） - clampInt 越界时返回默认值而非边界值（设 30 会静默变 8） - 直链正则写死 hath.network，CDN 换域名会全军覆没 -> 加图片 URL 兜底 - ReconcileTask 会误报「有改动」 测试从 37 增至 62，其中并发/限流/扫尾用 httptest 模拟 EH 实测： - 页并发峰值 = 8、画廊并发峰值 = 5（完全生效） - 12 次请求限流 80ms 实测 966ms（期望 880ms） - 断点续传 4 张已存在时只发 5 次请求
This commit is contained in:
+327
-82
@@ -49,47 +49,90 @@ type Engine struct {
|
||||
client *eh.Client
|
||||
sink EventSink
|
||||
|
||||
mu sync.Mutex
|
||||
concurrency int
|
||||
running bool
|
||||
cancel context.CancelFunc
|
||||
queue []Task
|
||||
mu sync.Mutex
|
||||
galleryWorkers int
|
||||
pageConcurrency int
|
||||
running bool
|
||||
cancel context.CancelFunc
|
||||
queue []Task
|
||||
}
|
||||
|
||||
// NewEngine 建引擎
|
||||
func NewEngine(store *Store, sink EventSink, settings Settings) *Engine {
|
||||
e := &Engine{
|
||||
store: store,
|
||||
sink: sink,
|
||||
client: eh.NewClient(time.Duration(settings.Timeout)*time.Millisecond,
|
||||
settings.Proxy, settings.ProxyMode, settings.Xres),
|
||||
concurrency: settings.Concurrency,
|
||||
}
|
||||
if e.concurrency < 1 {
|
||||
e.concurrency = 3
|
||||
}
|
||||
if e.concurrency > 8 {
|
||||
e.concurrency = 8
|
||||
store: store,
|
||||
sink: sink,
|
||||
client: eh.NewClient(time.Duration(settings.Timeout)*time.Millisecond, settings.Proxy, settings.ProxyMode, settings.Xres),
|
||||
galleryWorkers: clampInt(settings.GalleryConcurrency, 1, 20, 5),
|
||||
pageConcurrency: clampInt(settings.PageConcurrency, 1, 20, 8),
|
||||
}
|
||||
e.client.SetRequestInterval(time.Duration(settings.RequestIntervalMs) * time.Millisecond)
|
||||
return e
|
||||
}
|
||||
|
||||
// clampInt 把值夹到 [lo, hi]。0 或负数视为「没设置」，用默认值。
|
||||
func clampInt(v, lo, hi, def int) int {
|
||||
if v <= 0 {
|
||||
return def
|
||||
}
|
||||
if v < lo {
|
||||
return lo
|
||||
}
|
||||
if v > hi {
|
||||
return hi
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// Client 暴露给 app 层改配置用
|
||||
func (e *Engine) Client() *eh.Client { return e.client }
|
||||
|
||||
// SetConcurrency 运行时改并发
|
||||
func (e *Engine) SetConcurrency(n int) {
|
||||
if n < 1 {
|
||||
n = 1
|
||||
}
|
||||
if n > 8 {
|
||||
n = 8
|
||||
}
|
||||
e.mu.Lock()
|
||||
e.concurrency = n
|
||||
e.galleryWorkers = clampInt(n, 1, 20, 5)
|
||||
e.mu.Unlock()
|
||||
}
|
||||
|
||||
// SetGalleryConcurrency 改画廊并发
|
||||
func (e *Engine) SetGalleryConcurrency(n int) { e.SetConcurrency(n) }
|
||||
|
||||
// SetPageConcurrency 改单画廊页并发
|
||||
func (e *Engine) SetPageConcurrency(n int) {
|
||||
e.mu.Lock()
|
||||
e.pageConcurrency = clampInt(n, 1, 20, 8)
|
||||
e.mu.Unlock()
|
||||
}
|
||||
|
||||
// SetRequestInterval 改全局请求间隔（毫秒）
|
||||
func (e *Engine) SetRequestInterval(ms int) {
|
||||
if ms < 0 {
|
||||
ms = 0
|
||||
}
|
||||
e.client.SetRequestInterval(time.Duration(ms) * time.Millisecond)
|
||||
}
|
||||
|
||||
// GalleryConcurrency 当前画廊并发
|
||||
func (e *Engine) GalleryConcurrency() int {
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
return e.galleryWorkers
|
||||
}
|
||||
|
||||
// PageConcurrency 当前页并发
|
||||
func (e *Engine) PageConcurrency() int {
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
return e.pageConcurrency
|
||||
}
|
||||
|
||||
// emitTask 把任务状态推给 UI
|
||||
func (e *Engine) emitTask(t Task) {
|
||||
if e.sink != nil {
|
||||
e.sink.TaskUpdate(t)
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) IsRunning() bool {
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
@@ -138,9 +181,12 @@ func (e *Engine) Start(ctx context.Context) {
|
||||
return
|
||||
}
|
||||
e.running = true
|
||||
e.mu.Unlock()
|
||||
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
e.mu.Lock()
|
||||
e.cancel = cancel
|
||||
n := e.concurrency
|
||||
n := e.galleryWorkers
|
||||
e.mu.Unlock()
|
||||
|
||||
defer func() {
|
||||
@@ -256,59 +302,266 @@ func (e *Engine) runTask(ctx context.Context, task Task) {
|
||||
}
|
||||
e.save(task)
|
||||
|
||||
// ② ③ 逐页：解析 URL → 下载
|
||||
total := len(gallery.Pages)
|
||||
for _, p := range gallery.Pages {
|
||||
if ctx.Err() != nil {
|
||||
e.markCanceled(task)
|
||||
// ② ③ 页级并发下载
|
||||
st := &pageState{
|
||||
task: &task,
|
||||
dir: dir,
|
||||
pages: gallery.Pages,
|
||||
gid: task.GID,
|
||||
token: task.Token,
|
||||
title: gallery.Title,
|
||||
total: len(gallery.Pages),
|
||||
engine: e,
|
||||
}
|
||||
|
||||
// 先做一遍快速扫描：已存在的直接算完成，剩下的才进队列
|
||||
st.scanExisting()
|
||||
st.saveManifest()
|
||||
|
||||
// 页级 worker 池
|
||||
n := e.pageConcurrency
|
||||
if n > len(st.pending) {
|
||||
n = len(st.pending)
|
||||
}
|
||||
if n > 0 {
|
||||
var wg sync.WaitGroup
|
||||
work := make(chan eh.GalleryPageInfo, len(st.pending))
|
||||
for _, p := range st.pending {
|
||||
work <- p
|
||||
}
|
||||
close(work)
|
||||
|
||||
for i := 0; i < n; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for p := range work {
|
||||
st.downloadPage(ctx, p)
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
// ---- 整轮扫尾：把刚失败的页再扫一遍 ----
|
||||
if st.failedCount() > 0 && ctx.Err() == nil {
|
||||
e.log("info", fmt.Sprintf("画廊 %s：%d 页首轮失败，开始扫尾重试", task.GID, st.failedCount()))
|
||||
|
||||
retry := st.collectFailed()
|
||||
// 扫尾时清掉 failedList，让结果反映最终状态
|
||||
st.clearFailures()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
work := make(chan eh.GalleryPageInfo, len(retry))
|
||||
for _, p := range retry {
|
||||
work <- p
|
||||
}
|
||||
close(work)
|
||||
|
||||
n := e.pageConcurrency
|
||||
if n > len(retry) {
|
||||
n = len(retry)
|
||||
}
|
||||
for i := 0; i < n; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for p := range work {
|
||||
st.downloadPage(ctx, p)
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
if st.failedCount() > 0 {
|
||||
e.log("warn", fmt.Sprintf(
|
||||
"画廊 %s：扫尾后仍有 %d 页失败（换个线路再点「开始下载」可续传）",
|
||||
task.GID, st.failedCount()))
|
||||
} else {
|
||||
e.log("ok", fmt.Sprintf("画廊 %s：扫尾重试全部成功", task.GID))
|
||||
}
|
||||
}
|
||||
|
||||
st.saveManifest()
|
||||
|
||||
if ctx.Err() != nil {
|
||||
e.markCanceled(task)
|
||||
return
|
||||
}
|
||||
|
||||
t := st.snapshot()
|
||||
if t.Failed > 0 {
|
||||
t.Status = "partial"
|
||||
} else {
|
||||
t.Status = "done"
|
||||
}
|
||||
e.save(t)
|
||||
}
|
||||
|
||||
// pageState 是「一个画廊的页级下载状态」，所有字段都由 mutex 保护，
|
||||
// 因为页级 worker 是并发的。
|
||||
type pageState struct {
|
||||
mu sync.Mutex
|
||||
engine *Engine
|
||||
|
||||
task *Task
|
||||
dir string
|
||||
gid string
|
||||
token string
|
||||
title string
|
||||
total int
|
||||
|
||||
pages []eh.GalleryPageInfo
|
||||
pending []eh.GalleryPageInfo
|
||||
|
||||
done map[int]bool
|
||||
failed map[int]string // 页码 -> 错误
|
||||
failedL []FailedItem
|
||||
}
|
||||
|
||||
// scanExisting 扫描已有文件（断点续传的第一依据：文件在且非空 = 完成）
|
||||
func (s *pageState) scanExisting() {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
s.done = map[int]bool{}
|
||||
s.failed = map[int]string{}
|
||||
s.failedL = nil
|
||||
|
||||
if s.task.Pages == nil {
|
||||
s.task.Pages = map[string]string{}
|
||||
}
|
||||
|
||||
for _, p := range s.pages {
|
||||
fname := targetName(p, s.gid, s.total)
|
||||
dest := filepath.Join(s.dir, fname)
|
||||
if st, err := os.Stat(dest); err == nil && st.Size() > 0 {
|
||||
s.done[p.Page] = true
|
||||
s.task.Pages[strconv.Itoa(p.Page)] = "done"
|
||||
} else {
|
||||
s.pending = append(s.pending, p)
|
||||
}
|
||||
}
|
||||
s.task.Done = len(s.done)
|
||||
}
|
||||
|
||||
// downloadPage 下单页（解析 URL → 下载），内部含 4 次退避重试
|
||||
func (s *pageState) downloadPage(ctx context.Context, p eh.GalleryPageInfo) {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
|
||||
fname := targetName(p, s.gid, s.total)
|
||||
dest := filepath.Join(s.dir, fname)
|
||||
|
||||
err := s.engine.withRetry(ctx, func() error {
|
||||
imgURL, e := s.engine.client.ResolveImageURL(ctx, p.SPath)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
_, e = s.engine.client.DownloadImage(ctx, imgURL, dest, nil)
|
||||
return e
|
||||
})
|
||||
|
||||
s.mu.Lock()
|
||||
k := strconv.Itoa(p.Page)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil || errors.Is(err, ErrCanceled) {
|
||||
s.mu.Unlock()
|
||||
s.markCanceled()
|
||||
return
|
||||
}
|
||||
|
||||
fname := targetName(p, task.GID, total)
|
||||
dest := filepath.Join(dir, fname)
|
||||
|
||||
// 断点续传：文件在就算完成（文件系统即真相，状态文件丢了也能续）
|
||||
if st, err := os.Stat(dest); err == nil && st.Size() > 0 {
|
||||
task.Done++
|
||||
task.Pages[strconv.Itoa(p.Page)] = "done"
|
||||
e.progress(task)
|
||||
continue
|
||||
}
|
||||
|
||||
page := p.Page
|
||||
err := e.withRetry(ctx, func() error {
|
||||
imgURL, err := e.client.ResolveImageURL(ctx, p.SPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = e.client.DownloadImage(ctx, imgURL, dest, nil)
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
if ctx.Err() != nil || errors.Is(err, ErrCanceled) {
|
||||
e.markCanceled(task)
|
||||
return
|
||||
}
|
||||
task.Failed++
|
||||
task.Pages[strconv.Itoa(page)] = "failed"
|
||||
task.FailedList = append(task.FailedList, FailedItem{
|
||||
Page: page, File: fname, Error: err.Error(),
|
||||
})
|
||||
e.progress(task)
|
||||
continue
|
||||
}
|
||||
|
||||
task.Done++
|
||||
task.Pages[strconv.Itoa(page)] = "done"
|
||||
e.progress(task)
|
||||
}
|
||||
|
||||
if task.Failed > 0 {
|
||||
task.Status = "partial"
|
||||
s.failed[p.Page] = err.Error()
|
||||
s.failedL = append(s.failedL, FailedItem{Page: p.Page, File: fname, Error: err.Error()})
|
||||
s.task.Pages[k] = "failed"
|
||||
} else {
|
||||
task.Status = "done"
|
||||
s.done[p.Page] = true
|
||||
s.task.Pages[k] = "done"
|
||||
}
|
||||
e.save(task)
|
||||
s.syncCountsLocked()
|
||||
snap := s.snapshotLocked()
|
||||
s.mu.Unlock()
|
||||
|
||||
s.engine.emitTask(snap)
|
||||
}
|
||||
|
||||
func (s *pageState) syncCountsLocked() {
|
||||
s.task.Done = len(s.done)
|
||||
s.task.Failed = len(s.failed)
|
||||
s.task.FailedList = append([]FailedItem(nil), s.failedL...)
|
||||
}
|
||||
|
||||
func (s *pageState) snapshotLocked() Task {
|
||||
s.syncCountsLocked()
|
||||
t := *s.task
|
||||
return t
|
||||
}
|
||||
|
||||
func (s *pageState) snapshot() Task {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.snapshotLocked()
|
||||
}
|
||||
|
||||
func (s *pageState) failedCount() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return len(s.failed)
|
||||
}
|
||||
|
||||
func (s *pageState) clearFailures() {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.failed = map[int]string{}
|
||||
s.failedL = nil
|
||||
s.task.Failed = 0
|
||||
s.task.FailedList = []FailedItem{}
|
||||
}
|
||||
|
||||
// collectFailed 收集失败页（清空失败表前先取出）
|
||||
func (s *pageState) collectFailed() []eh.GalleryPageInfo {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
failed := intSet(keysOf(s.failed))
|
||||
var out []eh.GalleryPageInfo
|
||||
for _, p := range s.pages {
|
||||
if failed[p.Page] {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// markCanceled 任一页发现取消就整体标记
|
||||
func (s *pageState) markCanceled() {
|
||||
s.engine.markCanceled(s.snapshot())
|
||||
}
|
||||
|
||||
func (s *pageState) saveManifest() {
|
||||
s.mu.Lock()
|
||||
done := keysOf(s.done)
|
||||
failed := keysOf(s.failed)
|
||||
t := *s.task
|
||||
s.mu.Unlock()
|
||||
|
||||
WriteManifest(s.dir, Manifest{
|
||||
GID: s.gid,
|
||||
Token: s.token,
|
||||
Title: s.title,
|
||||
TotalPages: s.total,
|
||||
DonePages: done,
|
||||
FailedPage: failed,
|
||||
CreatedAt: t.CreatedAt,
|
||||
})
|
||||
}
|
||||
|
||||
func keysOf[V any](m map[int]V) []int {
|
||||
out := make([]int, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// targetName 页码补零，保证排序
|
||||
@@ -395,17 +648,9 @@ func (e *Engine) fail(task Task, err error) {
|
||||
e.log("error", fmt.Sprintf("画廊 %s 失败：%s", task.GID, msg))
|
||||
}
|
||||
|
||||
func (e *Engine) progress(task Task) {
|
||||
if e.sink != nil {
|
||||
e.sink.TaskUpdate(task)
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) save(task Task) {
|
||||
task = e.store.UpsertTask(task)
|
||||
if e.sink != nil {
|
||||
e.sink.TaskUpdate(task)
|
||||
}
|
||||
e.emitTask(task)
|
||||
}
|
||||
|
||||
func (e *Engine) log(level, msg string) {
|
||||
|
||||
Reference in New Issue
Block a user