feat: 双层并发 + 全局限流 + 整轮扫尾 + 画廊目录清单
原实现只能「多画廊并行，每个画廊串行下」，单张大画廊下得很慢。 按用户多年使用习惯（5 画廊 × 每画廊 8 张）改造为双层并发。 - 双层并发：画廊并发(默认5) × 页并发(默认8) = 最多 40 张同时下载， 两个值都在设置面板可调（1-20）。页级 worker 池 + mutex 保护计数。 - 全局限流：对齐参考实现的 nextRequestTime 算法，限所有请求 （含 /s/ 解析与图片本体），默认 0 秒关闭。 - 整轮扫尾：跑完后自动重扫失败页 1 轮（同样走完整 4 次退避重试）， 仍失败则停下并标记 partial，换线路后手动点开始即可续传。 - 画廊目录清单 .ehdl.json：记录 gid/token/总页数/已完成页， 即使 state.json 丢失也能算出「应该多少张、还差多少张」。 顺带修复： - Start() 双重加锁导致死锁（引擎完全无法启动） - clampInt 越界时返回默认值而非边界值（设 30 会静默变 8） - 直链正则写死 hath.network，CDN 换域名会全军覆没 -> 加图片 URL 兜底 - ReconcileTask 会误报「有改动」 测试从 37 增至 62，其中并发/限流/扫尾用 httptest 模拟 EH 实测： - 页并发峰值 = 8、画廊并发峰值 = 5（完全生效） - 12 次请求限流 80ms 实测 966ms（期望 880ms） - 断点续传 4 张已存在时只发 5 次请求
This commit is contained in:
+17
-3
@@ -157,9 +157,23 @@
|
||||
</label>
|
||||
|
||||
<label class="field">
|
||||
<span>并发数：<b id="set-conc-v">3</b></span>
|
||||
<input type="range" id="set-conc" min="1" max="8" step="1" value="3" />
|
||||
<small>每个画廊要发 2N+1 次请求，建议 2~4。调太高容易被限流。</small>
|
||||
<span>画廊并发：<b id="set-gconc-v">5</b> 个</span>
|
||||
<input type="range" id="set-gconc" min="1" max="20" step="1" value="5" />
|
||||
<small>同时处理几个画廊。</small>
|
||||
</label>
|
||||
|
||||
<label class="field">
|
||||
<span>单画廊页并发：<b id="set-pconc-v">8</b> 张</span>
|
||||
<input type="range" id="set-pconc" min="1" max="20" step="1" value="8" />
|
||||
<small>每个画廊内部同时下几张图。<br>
|
||||
总并发 = 画廊并发 × 页并发，默认 <b id="set-max">40</b> 张同时下载。</small>
|
||||
</label>
|
||||
|
||||
<label class="field">
|
||||
<span>请求间隔：<b id="set-interval-v">0</b> 秒</span>
|
||||
<input type="range" id="set-interval" min="0" max="50" step="1" value="0" />
|
||||
<small>所有请求之间的最小间隔（全局生效，含 /s/ 解析与图片下载）。0 = 不限流。<br>
|
||||
高并发下如果频繁遇到 429，可以调到 1~2 秒。</small>
|
||||
</label>
|
||||
|
||||
<label class="field">
|
||||
|
||||
@@ -30,12 +30,16 @@ window.eh = {
|
||||
|
||||
// ---- 引擎 ----
|
||||
status: async () => requireGo().Status(),
|
||||
engineInfo: async () => requireGo().EngineInfo(),
|
||||
start: async (payload) => {
|
||||
const gids = (payload && payload.onlyGids) || [];
|
||||
return requireGo().Start(gids);
|
||||
},
|
||||
stop: async () => requireGo().Stop(),
|
||||
|
||||
// ---- 扫描下载目录（用 .ehdl.json 清单补全丢失的任务信息）----
|
||||
scanDownloadDir: async () => requireGo().ScanDownloadDir(),
|
||||
|
||||
// ---- 系统 ----
|
||||
pickDir: async () => requireGo().PickDir(),
|
||||
exportFailed: async (gid) => {
|
||||
|
||||
+32
-5
@@ -134,17 +134,28 @@ async function loadSettings() {
|
||||
$('set-dir').value = settings.downloadRoot || '';
|
||||
$('set-proxy').value = settings.proxy || '';
|
||||
$('set-proxymode').value = settings.proxyMode || 'off';
|
||||
$('set-conc').value = settings.concurrency || 3;
|
||||
$('set-conc-v').textContent = settings.concurrency || 3;
|
||||
$('set-gconc').value = settings.galleryConcurrency || 5;
|
||||
$('set-gconc-v').textContent = settings.galleryConcurrency || 5;
|
||||
$('set-pconc').value = settings.pageConcurrency || 8;
|
||||
$('set-pconc-v').textContent = settings.pageConcurrency || 8;
|
||||
$('set-interval').value = Math.round((settings.requestIntervalMs || 0) / 1000);
|
||||
$('set-interval-v').textContent = Math.round((settings.requestIntervalMs || 0) / 1000);
|
||||
$('set-to').value = Math.round((settings.timeout || 30000) / 1000);
|
||||
$('set-to-v').textContent = Math.round((settings.timeout || 30000) / 1000);
|
||||
$('set-xres').value = String(settings.xres || 'auto');
|
||||
$('set-bridgeport').value = String(settings.bridgePort || 37777);
|
||||
$('set-dirtpl').value = settings.dirTemplate || '{title}';
|
||||
updateMaxConcurrent();
|
||||
renderPresets();
|
||||
refreshPreview();
|
||||
}
|
||||
|
||||
function updateMaxConcurrent() {
|
||||
const g = parseInt($('set-gconc').value, 10) || 5;
|
||||
const p = parseInt($('set-pconc').value, 10) || 8;
|
||||
$('set-max').textContent = g * p;
|
||||
}
|
||||
|
||||
const DIR_PRESETS = [
|
||||
['{title}', '仅标题'],
|
||||
['{artist}/{title}', '按作者'],
|
||||
@@ -208,7 +219,15 @@ function bind() {
|
||||
if (d) $('set-dir').value = d;
|
||||
};
|
||||
|
||||
$('set-conc').oninput = (e) => { $('set-conc-v').textContent = e.target.value; };
|
||||
$('set-gconc').oninput = (e) => {
|
||||
$('set-gconc-v').textContent = e.target.value;
|
||||
updateMaxConcurrent();
|
||||
};
|
||||
$('set-pconc').oninput = (e) => {
|
||||
$('set-pconc-v').textContent = e.target.value;
|
||||
updateMaxConcurrent();
|
||||
};
|
||||
$('set-interval').oninput = (e) => { $('set-interval-v').textContent = e.target.value; };
|
||||
$('set-to').oninput = (e) => { $('set-to-v').textContent = e.target.value; };
|
||||
$('set-dirtpl').oninput = () => { renderPresets(); refreshPreview(); };
|
||||
|
||||
@@ -293,7 +312,9 @@ async function saveSettings() {
|
||||
proxy: $('set-proxy').value.trim(),
|
||||
proxyMode: $('set-proxymode').value,
|
||||
xres: $('set-xres').value,
|
||||
concurrency: parseInt($('set-conc').value, 10),
|
||||
galleryConcurrency: parseInt($('set-gconc').value, 10),
|
||||
pageConcurrency: parseInt($('set-pconc').value, 10),
|
||||
requestIntervalMs: parseInt($('set-interval').value, 10) * 1000,
|
||||
timeout: parseInt($('set-to').value, 10) * 1000,
|
||||
bridgePort: parseInt($('set-bridgeport').value, 10) || 37777,
|
||||
dirTemplate: $('set-dirtpl').value.trim() || '{title}',
|
||||
@@ -301,7 +322,13 @@ async function saveSettings() {
|
||||
await window.eh.setSettings(patch);
|
||||
$('set-msg').textContent = '';
|
||||
closeModal('modal-settings');
|
||||
pushLog({ level: 'ok', msg: '设置已保存', at: Date.now() });
|
||||
const max = patch.galleryConcurrency * patch.pageConcurrency;
|
||||
pushLog({
|
||||
level: 'ok',
|
||||
msg: `设置已保存 · 画廊并发 ${patch.galleryConcurrency} × 页并发 ${patch.pageConcurrency}（最多 ${max} 张同时下载）` +
|
||||
(patch.requestIntervalMs ? ` · 请求间隔 ${patch.requestIntervalMs / 1000} 秒` : ''),
|
||||
at: Date.now(),
|
||||
});
|
||||
}
|
||||
|
||||
async function doStart() {
|
||||
|
||||
Vendored
+4
@@ -11,6 +11,8 @@ export function ClearFinished():Promise<Record<string, any>>;
|
||||
|
||||
export function CountDone():Promise<number>;
|
||||
|
||||
export function EngineInfo():Promise<main.EngineInfo>;
|
||||
|
||||
export function ExportFailed(arg1:string):Promise<string>;
|
||||
|
||||
export function GetSettings():Promise<core.Settings>;
|
||||
@@ -29,6 +31,8 @@ export function RemoveTask(arg1:string,arg2:string):Promise<Array<core.Task>>;
|
||||
|
||||
export function RequeueFailed():Promise<Array<core.Task>>;
|
||||
|
||||
export function ScanDownloadDir():Promise<main.ScanManifestsResult>;
|
||||
|
||||
export function SelectAll(arg1:Array<string>):Promise<Array<core.Task>>;
|
||||
|
||||
export function SetSettings(arg1:Record<string, any>):Promise<core.Settings>;
|
||||
|
||||
@@ -18,6 +18,10 @@ export function CountDone() {
|
||||
return window['go']['main']['App']['CountDone']();
|
||||
}
|
||||
|
||||
export function EngineInfo() {
|
||||
return window['go']['main']['App']['EngineInfo']();
|
||||
}
|
||||
|
||||
export function ExportFailed(arg1) {
|
||||
return window['go']['main']['App']['ExportFailed'](arg1);
|
||||
}
|
||||
@@ -54,6 +58,10 @@ export function RequeueFailed() {
|
||||
return window['go']['main']['App']['RequeueFailed']();
|
||||
}
|
||||
|
||||
export function ScanDownloadDir() {
|
||||
return window['go']['main']['App']['ScanDownloadDir']();
|
||||
}
|
||||
|
||||
export function SelectAll(arg1) {
|
||||
return window['go']['main']['App']['SelectAll'](arg1);
|
||||
}
|
||||
|
||||
@@ -20,7 +20,9 @@ export namespace core {
|
||||
downloadRoot: string;
|
||||
proxy: string;
|
||||
proxyMode: string;
|
||||
concurrency: number;
|
||||
galleryConcurrency: number;
|
||||
pageConcurrency: number;
|
||||
requestIntervalMs: number;
|
||||
timeout: number;
|
||||
xres: string;
|
||||
bridgePort: number;
|
||||
@@ -36,7 +38,9 @@ export namespace core {
|
||||
this.downloadRoot = source["downloadRoot"];
|
||||
this.proxy = source["proxy"];
|
||||
this.proxyMode = source["proxyMode"];
|
||||
this.concurrency = source["concurrency"];
|
||||
this.galleryConcurrency = source["galleryConcurrency"];
|
||||
this.pageConcurrency = source["pageConcurrency"];
|
||||
this.requestIntervalMs = source["requestIntervalMs"];
|
||||
this.timeout = source["timeout"];
|
||||
this.xres = source["xres"];
|
||||
this.bridgePort = source["bridgePort"];
|
||||
@@ -139,6 +143,46 @@ export namespace main {
|
||||
this.error = source["error"];
|
||||
}
|
||||
}
|
||||
export class EngineInfo {
|
||||
running: boolean;
|
||||
queued: number;
|
||||
galleryConcurrency: number;
|
||||
pageConcurrency: number;
|
||||
maxConcurrent: number;
|
||||
requestIntervalMs: number;
|
||||
|
||||
static createFrom(source: any = {}) {
|
||||
return new EngineInfo(source);
|
||||
}
|
||||
|
||||
constructor(source: any = {}) {
|
||||
if ('string' === typeof source) source = JSON.parse(source);
|
||||
this.running = source["running"];
|
||||
this.queued = source["queued"];
|
||||
this.galleryConcurrency = source["galleryConcurrency"];
|
||||
this.pageConcurrency = source["pageConcurrency"];
|
||||
this.maxConcurrent = source["maxConcurrent"];
|
||||
this.requestIntervalMs = source["requestIntervalMs"];
|
||||
}
|
||||
}
|
||||
export class ScanManifestsResult {
|
||||
found: number;
|
||||
adopted: number;
|
||||
root: string;
|
||||
note: string;
|
||||
|
||||
static createFrom(source: any = {}) {
|
||||
return new ScanManifestsResult(source);
|
||||
}
|
||||
|
||||
constructor(source: any = {}) {
|
||||
if ('string' === typeof source) source = JSON.parse(source);
|
||||
this.found = source["found"];
|
||||
this.adopted = source["adopted"];
|
||||
this.root = source["root"];
|
||||
this.note = source["note"];
|
||||
}
|
||||
}
|
||||
export class StartResult {
|
||||
started: boolean;
|
||||
reason: string;
|
||||
|
||||
Reference in New Issue
Block a user