现象:点「开始下载」后,单个画廊图片较多时整个程序无报错直接退出。
只在图片多的画廊上偶发。
根因:pageState.snapshotLocked 用浅拷贝把 Task 交给 UI:
t := *s.task // Pages map 仍是同一块内存
快照被 EventsEmit 做 JSON 序列化时,其他页 worker 正在写同一个 map,
撞上就抛 Go runtime 的致命错误:
fatal error: concurrent map iteration and map write
这是 runtime 级 fatal error,不可 recover,直接杀进程 —— 所以没有任何
报错提示。图片越多、页并发越高,并发写窗口越长,越容易触发。
同一问题还有第二处:Store.UpsertTask 存的是原值(含活的 map),
而 Flush() 会在后台 json.Marshal 整个 state,同样会撞上。
修复:
- snapshotLocked 深拷贝 Pages map 与 FailedList slice
- UpsertTask 存前先 clone(),让 store 完全拥有自己的数据
- 另加一个兜底 recover,防止将来再有 panic 直接杀掉进程
测试:
- TestSnapshotDoesNotSharePagesMap:验证快照隔离(修复前失败)
- TestSnapshotRaceWindowClosed:持续写原 task + 持续序列化快照,
精准复现崩溃窗口 —— 实测修复前 fatal error 崩进程,修复后通过
- 大画廊压测:200 页 × 页并发 20,以及 5 画廊 × 60 页 × 并发 16
测试从 62 增至 67。
271 lines
6.8 KiB
Go
271 lines
6.8 KiB
Go
package core
|
||
|
||
// 崩溃复现测试
|
||
//
|
||
// 曾经的 bug:snapshotLocked 用浅拷贝把 Task 里的 Pages map 直接交出去,
|
||
// 而其他页 worker 还在写同一个 map —— 序列化时撞上写入,
|
||
// Go runtime 抛 fatal error: concurrent map read and map write,
|
||
// 不可 recover,整个进程静默退出。
|
||
//
|
||
// 症状:图片多的画廊 + 高页并发时才偶发,因为并发写窗口更长。
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"fmt"
|
||
"os"
|
||
"path/filepath"
|
||
"strconv"
|
||
"sync"
|
||
"testing"
|
||
"time"
|
||
|
||
"eh-downloader-wails/internal/eh"
|
||
)
|
||
|
||
// marshalSink 模拟前端事件通道:拿到 Task 就做 JSON 序列化(等价于 EventsEmit)
|
||
type marshalSink struct{}
|
||
|
||
func (marshalSink) TaskUpdate(t Task) {
|
||
// Wails 内部就是 json.Marshal(task),这里复刻同一件事
|
||
if _, err := json.Marshal(t); err != nil {
|
||
panic(err)
|
||
}
|
||
}
|
||
|
||
func (marshalSink) Log(string, string) {}
|
||
func (marshalSink) Done() {}
|
||
|
||
// 快照必须与原 task 完全隔离:改快照不能影响原始,反之亦然
|
||
func TestSnapshotDoesNotSharePagesMap(t *testing.T) {
|
||
st := &pageState{
|
||
task: &Task{GID: "1", Token: "t", Pages: map[string]string{}},
|
||
dir: t.TempDir(),
|
||
gid: "1",
|
||
token: "t",
|
||
total: 3,
|
||
done: map[int]bool{},
|
||
failed: map[int]string{},
|
||
}
|
||
st.pages = []eh.GalleryPageInfo{
|
||
{Page: 1, Filename: "001.png"},
|
||
{Page: 2, Filename: "002.png"},
|
||
{Page: 3, Filename: "003.png"},
|
||
}
|
||
st.scanExisting()
|
||
|
||
// 模拟下载完 1 页
|
||
st.mu.Lock()
|
||
st.done[1] = true
|
||
st.task.Pages["1"] = "done"
|
||
snap := st.snapshotLocked()
|
||
st.mu.Unlock()
|
||
|
||
// 立刻序列化快照(Wails 会做的事)
|
||
if _, err := json.Marshal(snap); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
|
||
// 原 task 继续被改(模拟其他 worker 同时在下载)
|
||
st.mu.Lock()
|
||
st.done[2] = true
|
||
st.task.Pages["2"] = "done"
|
||
st.mu.Unlock()
|
||
|
||
// 快照不该跟着长 —— 共享同一块 map 的话快照也会看到 "2"
|
||
if _, leaked := snap.Pages["2"]; leaked {
|
||
t.Error("快照的 Pages map 与原 task 共享内存(浅拷贝),并发下会崩")
|
||
}
|
||
}
|
||
|
||
// FailedList 也必须深拷贝
|
||
func TestSnapshotDoesNotShareFailedList(t *testing.T) {
|
||
st := &pageState{
|
||
task: &Task{GID: "1", Token: "t", Pages: map[string]string{}},
|
||
dir: t.TempDir(),
|
||
gid: "1",
|
||
total: 3,
|
||
done: map[int]bool{},
|
||
failed: map[int]string{},
|
||
}
|
||
st.scanExisting()
|
||
|
||
st.mu.Lock()
|
||
st.failed[1] = "boom"
|
||
st.failedL = append(st.failedL, FailedItem{Page: 1, File: "001.png", Error: "boom"})
|
||
snap := st.snapshotLocked()
|
||
st.mu.Unlock()
|
||
|
||
st.mu.Lock()
|
||
st.failed[2] = "boom2"
|
||
st.failedL = append(st.failedL, FailedItem{Page: 2, File: "002.png", Error: "boom2"})
|
||
st.mu.Unlock()
|
||
|
||
if len(snap.FailedList) != 1 {
|
||
t.Errorf("快照的 FailedList 跟着原切片增长了(应深拷贝): %+v", snap.FailedList)
|
||
}
|
||
}
|
||
|
||
// 真实并发压测:200 页 + 页并发 20 + 每页都触发事件序列化。
|
||
// 若仍有共享 map 问题,这个用例会崩进程或被 -race 抓到。
|
||
func TestConcurrentPageEventsNoCrash(t *testing.T) {
|
||
const total = 200 // 大画廊
|
||
|
||
m := newMockEH(total)
|
||
defer m.Close()
|
||
restore := m.installBase()
|
||
defer restore()
|
||
|
||
root := t.TempDir()
|
||
s, _ := newTestStore(t)
|
||
s.UpdateSettings(map[string]interface{}{
|
||
"downloadRoot": root,
|
||
"dirTemplate": "{gid}",
|
||
"galleryConcurrency": 1,
|
||
"pageConcurrency": 20,
|
||
"timeout": 10000,
|
||
})
|
||
e := NewEngine(s, &marshalSink{}, s.Settings())
|
||
s.AddTasks([]eh.GalleryRef{{GID: "777777", Token: "tok"}})
|
||
|
||
e.Enqueue(s.Tasks())
|
||
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
|
||
defer cancel()
|
||
|
||
done := make(chan struct{})
|
||
go func() { e.Start(ctx); close(done) }()
|
||
|
||
select {
|
||
case <-done:
|
||
case <-time.After(110 * time.Second):
|
||
t.Fatal("大画廊下载超时")
|
||
}
|
||
|
||
task := s.FindTask("777777", "tok")
|
||
if task == nil {
|
||
t.Fatal("任务不见了")
|
||
}
|
||
if task.Status != "done" {
|
||
t.Errorf("应全部下完,实际 %s (done=%d failed=%d)", task.Status, task.Done, task.Failed)
|
||
}
|
||
if task.Done != total {
|
||
t.Errorf("应下 %d 张,实际 %d", total, task.Done)
|
||
}
|
||
|
||
for _, p := range []int{1, 50, 100, 200} {
|
||
name := filepath.Join(root, "777777", fmt.Sprintf("%03d.png", p))
|
||
if st, err := os.Stat(name); err != nil || st.Size() == 0 {
|
||
t.Errorf("第 %d 张图缺失", p)
|
||
}
|
||
}
|
||
}
|
||
|
||
// 精准复现崩溃窗口:一路持续写快照里的 map(原 bug 的形态),
|
||
// 另一路持续序列化快照。若快照与原 task 共享内存,这里会直接 fatal。
|
||
//
|
||
// 这是用户报告的崩溃场景:大画廊 + 高页并发时偶发闪退。
|
||
func TestSnapshotRaceWindowClosed(t *testing.T) {
|
||
st := &pageState{
|
||
task: &Task{GID: "1", Token: "t", Pages: map[string]string{}},
|
||
dir: t.TempDir(),
|
||
gid: "1",
|
||
total: 200,
|
||
done: map[int]bool{},
|
||
failed: map[int]string{},
|
||
}
|
||
for i := 1; i <= 200; i++ {
|
||
st.pages = append(st.pages, eh.GalleryPageInfo{
|
||
Page: i, Filename: fmt.Sprintf("%03d.png", i),
|
||
})
|
||
}
|
||
st.scanExisting()
|
||
|
||
// 快照(模拟发给 UI 的事件)
|
||
snap := st.snapshot()
|
||
|
||
// 一路持续写原 task(模拟页 worker)
|
||
stop := make(chan struct{})
|
||
var writerDone sync.WaitGroup
|
||
writerDone.Add(1)
|
||
go func() {
|
||
defer writerDone.Done()
|
||
for {
|
||
select {
|
||
case <-stop:
|
||
return
|
||
default:
|
||
}
|
||
st.mu.Lock()
|
||
for i := 1; i <= 200; i++ {
|
||
st.task.Pages[strconv.Itoa(i)] = "done"
|
||
}
|
||
st.mu.Unlock()
|
||
}
|
||
}()
|
||
|
||
// 一路持续序列化快照
|
||
var marshalerDone sync.WaitGroup
|
||
marshalerDone.Add(1)
|
||
go func() {
|
||
defer marshalerDone.Done()
|
||
for i := 0; i < 3000; i++ {
|
||
// 修复前:这里会撞上并发写 -> fatal error
|
||
if _, err := json.Marshal(snap); err != nil {
|
||
t.Errorf("序列化失败: %v", err)
|
||
return
|
||
}
|
||
}
|
||
}()
|
||
|
||
marshalerDone.Wait()
|
||
close(stop)
|
||
writerDone.Wait()
|
||
}
|
||
|
||
// 多画廊高并发压测(更容易撞出共享状态问题)
|
||
func TestConcurrentMultiGalleryNoCrash(t *testing.T) {
|
||
const (
|
||
galleries = 5
|
||
pages = 60
|
||
)
|
||
|
||
m := newMockEH(pages)
|
||
defer m.Close()
|
||
restore := m.installBase()
|
||
defer restore()
|
||
|
||
root := t.TempDir()
|
||
s, _ := newTestStore(t)
|
||
s.UpdateSettings(map[string]interface{}{
|
||
"downloadRoot": root,
|
||
"dirTemplate": "{gid}",
|
||
"galleryConcurrency": galleries,
|
||
"pageConcurrency": 16,
|
||
"timeout": 10000,
|
||
})
|
||
e := NewEngine(s, &marshalSink{}, s.Settings())
|
||
|
||
refs := make([]eh.GalleryRef, 0, galleries)
|
||
for i := 1; i <= galleries; i++ {
|
||
gid := strconv.Itoa(800000 + i)
|
||
refs = append(refs, eh.GalleryRef{GID: gid, Token: "tok"})
|
||
}
|
||
s.AddTasks(refs)
|
||
|
||
e.Enqueue(s.Tasks())
|
||
ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second)
|
||
defer cancel()
|
||
e.Start(ctx)
|
||
|
||
for i := 1; i <= galleries; i++ {
|
||
gid := strconv.Itoa(800000 + i)
|
||
task := s.FindTask(gid, "tok")
|
||
if task == nil || task.Status != "done" {
|
||
t.Errorf("画廊 %s 没下完: %v", gid, task)
|
||
continue
|
||
}
|
||
if task.Done != pages {
|
||
t.Errorf("画廊 %s 应下 %d 张,实际 %d", gid, pages, task.Done)
|
||
}
|
||
}
|
||
} |