diff --git a/internal/core/crash_test.go b/internal/core/crash_test.go new file mode 100644 index 0000000..3af99df --- /dev/null +++ b/internal/core/crash_test.go @@ -0,0 +1,271 @@ +package core + +// 崩溃复现测试 +// +// 曾经的 bug:snapshotLocked 用浅拷贝把 Task 里的 Pages map 直接交出去, +// 而其他页 worker 还在写同一个 map —— 序列化时撞上写入, +// Go runtime 抛 fatal error: concurrent map read and map write, +// 不可 recover,整个进程静默退出。 +// +// 症状:图片多的画廊 + 高页并发时才偶发,因为并发写窗口更长。 + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strconv" + "sync" + "testing" + "time" + + "eh-downloader-wails/internal/eh" +) + +// marshalSink 模拟前端事件通道:拿到 Task 就做 JSON 序列化(等价于 EventsEmit) +type marshalSink struct{} + +func (marshalSink) TaskUpdate(t Task) { + // Wails 内部就是 json.Marshal(task),这里复刻同一件事 + if _, err := json.Marshal(t); err != nil { + panic(err) + } +} + +func (marshalSink) Log(string, string) {} +func (marshalSink) Done() {} + +// 快照必须与原 task 完全隔离:改快照不能影响原始,反之亦然 +func TestSnapshotDoesNotSharePagesMap(t *testing.T) { + st := &pageState{ + task: &Task{GID: "1", Token: "t", Pages: map[string]string{}}, + dir: t.TempDir(), + gid: "1", + token: "t", + total: 3, + done: map[int]bool{}, + failed: map[int]string{}, + } + st.pages = []eh.GalleryPageInfo{ + {Page: 1, Filename: "001.png"}, + {Page: 2, Filename: "002.png"}, + {Page: 3, Filename: "003.png"}, + } + st.scanExisting() + + // 模拟下载完 1 页 + st.mu.Lock() + st.done[1] = true + st.task.Pages["1"] = "done" + snap := st.snapshotLocked() + st.mu.Unlock() + + // 立刻序列化快照(Wails 会做的事) + if _, err := json.Marshal(snap); err != nil { + t.Fatal(err) + } + + // 原 task 继续被改(模拟其他 worker 同时在下载) + st.mu.Lock() + st.done[2] = true + st.task.Pages["2"] = "done" + st.mu.Unlock() + + // 快照不该跟着长 —— 共享同一块 map 的话快照也会看到 "2" + if _, leaked := snap.Pages["2"]; leaked { + t.Error("快照的 Pages map 与原 task 共享内存(浅拷贝),并发下会崩") + } +} + +// FailedList 也必须深拷贝 +func TestSnapshotDoesNotShareFailedList(t *testing.T) { + st := &pageState{ + task: &Task{GID: "1", Token: "t", Pages: map[string]string{}}, + dir: t.TempDir(), + gid: "1", + total: 3, + done: map[int]bool{}, + failed: map[int]string{}, + } + st.scanExisting() + + st.mu.Lock() + st.failed[1] = "boom" + st.failedL = append(st.failedL, FailedItem{Page: 1, File: "001.png", Error: "boom"}) + snap := st.snapshotLocked() + st.mu.Unlock() + + st.mu.Lock() + st.failed[2] = "boom2" + st.failedL = append(st.failedL, FailedItem{Page: 2, File: "002.png", Error: "boom2"}) + st.mu.Unlock() + + if len(snap.FailedList) != 1 { + t.Errorf("快照的 FailedList 跟着原切片增长了(应深拷贝): %+v", snap.FailedList) + } +} + +// 真实并发压测:200 页 + 页并发 20 + 每页都触发事件序列化。 +// 若仍有共享 map 问题,这个用例会崩进程或被 -race 抓到。 +func TestConcurrentPageEventsNoCrash(t *testing.T) { + const total = 200 // 大画廊 + + m := newMockEH(total) + defer m.Close() + restore := m.installBase() + defer restore() + + root := t.TempDir() + s, _ := newTestStore(t) + s.UpdateSettings(map[string]interface{}{ + "downloadRoot": root, + "dirTemplate": "{gid}", + "galleryConcurrency": 1, + "pageConcurrency": 20, + "timeout": 10000, + }) + e := NewEngine(s, &marshalSink{}, s.Settings()) + s.AddTasks([]eh.GalleryRef{{GID: "777777", Token: "tok"}}) + + e.Enqueue(s.Tasks()) + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + + done := make(chan struct{}) + go func() { e.Start(ctx); close(done) }() + + select { + case <-done: + case <-time.After(110 * time.Second): + t.Fatal("大画廊下载超时") + } + + task := s.FindTask("777777", "tok") + if task == nil { + t.Fatal("任务不见了") + } + if task.Status != "done" { + t.Errorf("应全部下完,实际 %s (done=%d failed=%d)", task.Status, task.Done, task.Failed) + } + if task.Done != total { + t.Errorf("应下 %d 张,实际 %d", total, task.Done) + } + + for _, p := range []int{1, 50, 100, 200} { + name := filepath.Join(root, "777777", fmt.Sprintf("%03d.png", p)) + if st, err := os.Stat(name); err != nil || st.Size() == 0 { + t.Errorf("第 %d 张图缺失", p) + } + } +} + +// 精准复现崩溃窗口:一路持续写快照里的 map(原 bug 的形态), +// 另一路持续序列化快照。若快照与原 task 共享内存,这里会直接 fatal。 +// +// 这是用户报告的崩溃场景:大画廊 + 高页并发时偶发闪退。 +func TestSnapshotRaceWindowClosed(t *testing.T) { + st := &pageState{ + task: &Task{GID: "1", Token: "t", Pages: map[string]string{}}, + dir: t.TempDir(), + gid: "1", + total: 200, + done: map[int]bool{}, + failed: map[int]string{}, + } + for i := 1; i <= 200; i++ { + st.pages = append(st.pages, eh.GalleryPageInfo{ + Page: i, Filename: fmt.Sprintf("%03d.png", i), + }) + } + st.scanExisting() + + // 快照(模拟发给 UI 的事件) + snap := st.snapshot() + + // 一路持续写原 task(模拟页 worker) + stop := make(chan struct{}) + var writerDone sync.WaitGroup + writerDone.Add(1) + go func() { + defer writerDone.Done() + for { + select { + case <-stop: + return + default: + } + st.mu.Lock() + for i := 1; i <= 200; i++ { + st.task.Pages[strconv.Itoa(i)] = "done" + } + st.mu.Unlock() + } + }() + + // 一路持续序列化快照 + var marshalerDone sync.WaitGroup + marshalerDone.Add(1) + go func() { + defer marshalerDone.Done() + for i := 0; i < 3000; i++ { + // 修复前:这里会撞上并发写 -> fatal error + if _, err := json.Marshal(snap); err != nil { + t.Errorf("序列化失败: %v", err) + return + } + } + }() + + marshalerDone.Wait() + close(stop) + writerDone.Wait() +} + +// 多画廊高并发压测(更容易撞出共享状态问题) +func TestConcurrentMultiGalleryNoCrash(t *testing.T) { + const ( + galleries = 5 + pages = 60 + ) + + m := newMockEH(pages) + defer m.Close() + restore := m.installBase() + defer restore() + + root := t.TempDir() + s, _ := newTestStore(t) + s.UpdateSettings(map[string]interface{}{ + "downloadRoot": root, + "dirTemplate": "{gid}", + "galleryConcurrency": galleries, + "pageConcurrency": 16, + "timeout": 10000, + }) + e := NewEngine(s, &marshalSink{}, s.Settings()) + + refs := make([]eh.GalleryRef, 0, galleries) + for i := 1; i <= galleries; i++ { + gid := strconv.Itoa(800000 + i) + refs = append(refs, eh.GalleryRef{GID: gid, Token: "tok"}) + } + s.AddTasks(refs) + + e.Enqueue(s.Tasks()) + ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second) + defer cancel() + e.Start(ctx) + + for i := 1; i <= galleries; i++ { + gid := strconv.Itoa(800000 + i) + task := s.FindTask(gid, "tok") + if task == nil || task.Status != "done" { + t.Errorf("画廊 %s 没下完: %v", gid, task) + continue + } + if task.Done != pages { + t.Errorf("画廊 %s 应下 %d 张,实际 %d", gid, pages, task.Done) + } + } +} \ No newline at end of file diff --git a/internal/core/engine.go b/internal/core/engine.go index 83b4188..0c2d061 100644 --- a/internal/core/engine.go +++ b/internal/core/engine.go @@ -491,9 +491,27 @@ func (s *pageState) syncCountsLocked() { s.task.FailedList = append([]FailedItem(nil), s.failedL...) } +// snapshotLocked 生成任务快照。 +// +// ⚠️ 必须深拷贝 Pages / FailedList —— 这两个是 map 和 slice, +// 浅拷贝会把底层内存一起交出去。而快照是要被送到 UI 做 JSON 序列化的, +// 其他页 worker 此刻还在写同一份数据 —— 共享内存会导致 +// fatal error: concurrent map read and map write(不可 recover,整个进程闪退)。 +// 症状:图片多的画廊 + 高页并发时偶发,因为并发写窗口更长。 func (s *pageState) snapshotLocked() Task { s.syncCountsLocked() t := *s.task + + // 深拷贝 map + pages := make(map[string]string, len(s.task.Pages)) + for k, v := range s.task.Pages { + pages[k] = v + } + t.Pages = pages + + // 深拷贝 slice + t.FailedList = append([]FailedItem(nil), s.task.FailedList...) + return t } diff --git a/internal/core/store.go b/internal/core/store.go index b41adee..8fa14bb 100644 --- a/internal/core/store.go +++ b/internal/core/store.go @@ -350,40 +350,41 @@ func truncate(s string, n int) string { return string(r[:n]) } -// UpsertTask 更新/插入任务 +// UpsertTask 更新/插入任务。 +// +// ⚠️ 存进去前必须 clone() —— 否则外部(页级 worker)继续改同一个 map, +// 而 store 的 Flush() 会在后台 json.Marshal 整个 state —— 撞上就是 +// fatal error: concurrent map write(不可 recover,进程闪退)。 func (s *Store) UpsertTask(t Task) Task { s.mu.Lock() defer s.mu.Unlock() t.UpdatedAt = time.Now().UnixMilli() - for i := range s.state.Tasks { - if s.state.Tasks[i].GID == t.GID && s.state.Tasks[i].Token == t.Token { - if t.Pages == nil { - t.Pages = map[string]string{} - } - if t.FailedList == nil { - t.FailedList = []FailedItem{} - } - if t.CreatedAt == 0 { - t.CreatedAt = s.state.Tasks[i].CreatedAt - } - s.state.Tasks[i] = t - s.startFlushLocked() - return t.clone() - } - } if t.Pages == nil { t.Pages = map[string]string{} } if t.FailedList == nil { t.FailedList = []FailedItem{} } + + for i := range s.state.Tasks { + if s.state.Tasks[i].GID == t.GID && s.state.Tasks[i].Token == t.Token { + if t.CreatedAt == 0 { + t.CreatedAt = s.state.Tasks[i].CreatedAt + } + s.state.Tasks[i] = t.clone() + s.startFlushLocked() + return t.clone() + } + } + if t.CreatedAt == 0 { t.CreatedAt = t.UpdatedAt } - s.state.Tasks = append([]Task{t}, s.state.Tasks...) + stored := t.clone() + s.state.Tasks = append([]Task{stored}, s.state.Tasks...) s.startFlushLocked() - return t.clone() + return stored.clone() } // RemoveTask 删单个任务