This commit is contained in:
Stardream
+290
@@ -0,0 +1,290 @@
|
||||
'use strict';
|
||||
|
||||
const { SUBJECT_TYPES, COLLECTION_TYPES, SUBJECT_LABEL, STATUS } = require('./constants');
|
||||
const { sleep, fetchMe, fetchCollectionAll, fetchCollectionRecent, fetchSubjectDetail, setApiBase } = require('./bgm');
|
||||
const { nowISO, normalizeCollection, applySubjectDetail, makeSnapshot, collectionFieldDiff } = require('./normalize');
|
||||
const { cacheCoversForState } = require('./covers');
|
||||
|
||||
let syncStatus = {
|
||||
running: false, mode: null, trigger: null, phase: 'idle', percent: 0,
|
||||
processed: 0, total: 0, startedAt: null, finishedAt: null, message: '', lastResult: null, error: null
|
||||
};
|
||||
let lock = false;
|
||||
|
||||
function getStatus() { return syncStatus; }
|
||||
|
||||
function emptyState() { return { schema: 9, settings: {}, collections: {}, history: [], runs: [] }; }
|
||||
|
||||
// Previous cron fire time before `from` (scans backward minute by minute, capped). Used by the
|
||||
// "interval" auto-sync range mode: the lookback window is one scheduling period.
|
||||
function cronPrevRun(expr, from) {
|
||||
const d = new Date(from.getTime());
|
||||
d.setSeconds(0, 0);
|
||||
d.setMinutes(d.getMinutes() - 1);
|
||||
const limit = 45 * 24 * 60; // ~45 days covers weekly/monthly schedules
|
||||
for (let i = 0; i < limit; i++) {
|
||||
if (cronMatches(expr, d)) return new Date(d.getTime());
|
||||
d.setMinutes(d.getMinutes() - 1);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// Auto-sync "fixed" range cutoff = now − (autoSyncRangeValue × autoSyncRangeUnit).
|
||||
function fixedCutoff(state) {
|
||||
const unit = state.settings.autoSyncRangeUnit || 'month';
|
||||
const val = Math.max(1, Number(state.settings.autoSyncRangeValue != null ? state.settings.autoSyncRangeValue : (state.settings.recentMonths || 3)) || 1);
|
||||
const d = new Date();
|
||||
if (unit === 'minute') d.setMinutes(d.getMinutes() - val);
|
||||
else if (unit === 'hour') d.setHours(d.getHours() - val);
|
||||
else if (unit === 'day') d.setDate(d.getDate() - val);
|
||||
else d.setMonth(d.getMonth() - val);
|
||||
return d;
|
||||
}
|
||||
|
||||
function computeCutoff(state, mode, trigger, recentMonthsOverride) {
|
||||
if (mode !== 'recent') return null;
|
||||
const months = recentMonthsOverride != null ? recentMonthsOverride : Number(state.settings.recentMonths || 3);
|
||||
const recentMonths = Math.max(1, Math.min(120, months));
|
||||
const monthsCutoff = () => { const d = new Date(); d.setMonth(d.getMonth() - recentMonths); return d; };
|
||||
if (trigger === 'auto') {
|
||||
if (state.settings.autoSyncRangeMode === 'interval') {
|
||||
const cron = state.settings.autoSyncCron || legacyToCron(state.settings) || '0 2 * * *';
|
||||
const prev = cronPrevRun(cron, new Date()); // ≈ now − one scheduling interval
|
||||
if (prev) return prev;
|
||||
// fall through to the fixed range if no previous fire was found
|
||||
}
|
||||
return fixedCutoff(state); // value + unit (minute/hour/day/month)
|
||||
}
|
||||
return monthsCutoff(); // manual sync still uses months (from the dialog / recentMonths)
|
||||
}
|
||||
|
||||
async function runSync({ store, mode, trigger, opts = {}, onProgress }) {
|
||||
const report = (p) => { try { onProgress && onProgress(p); } catch {} };
|
||||
// Per-run options (manual sync passes these; auto sync falls back to settings).
|
||||
const refreshSubjects = opts.refreshSubjects !== false;
|
||||
|
||||
const state = (await store.getState()) || emptyState();
|
||||
state.settings = state.settings || {};
|
||||
state.collections = state.collections || {};
|
||||
state.history = state.history || [];
|
||||
state.runs = state.runs || [];
|
||||
|
||||
setApiBase(state.settings.bgmApiBase); // custom API base / mirror (empty = default api.bgm.tv)
|
||||
|
||||
const token = state.settings.token;
|
||||
if (!token) throw new Error('未设置 Access Token，无法同步');
|
||||
|
||||
let username = state.settings.username;
|
||||
if (!username) {
|
||||
report({ phase: 'auth', percent: 2, message: '读取用户名…' });
|
||||
const me = await fetchMe(token);
|
||||
username = me.username || me.name;
|
||||
if (!username) throw new Error('无法自动获取用户名，请在设置中手动填写 Bangumi 用户名');
|
||||
state.settings.username = username;
|
||||
}
|
||||
|
||||
const effectiveMonths = mode === 'recent'
|
||||
? Math.max(1, Math.min(120, opts.recentMonths != null ? opts.recentMonths : Number(state.settings.recentMonths || 3)))
|
||||
: null;
|
||||
const cacheCovers = opts.cacheCovers !== undefined ? opts.cacheCovers : state.settings.autoCacheCovers;
|
||||
const cutoff = computeCutoff(state, mode, trigger, effectiveMonths);
|
||||
const getAll = () => Object.values(state.collections);
|
||||
|
||||
// --- phase 1: fetch collections ---
|
||||
report({ phase: 'fetching', percent: 4, message: `开始${mode === 'recent' ? '增量' : '完整'}同步 @${username}` });
|
||||
const remote = [];
|
||||
const totalTasks = SUBJECT_TYPES.length * COLLECTION_TYPES.length;
|
||||
let taskDone = 0;
|
||||
for (const st of SUBJECT_TYPES) {
|
||||
for (const ct of COLLECTION_TYPES) {
|
||||
const onPage = (page, count) => report({
|
||||
phase: 'fetching', percent: 4 + Math.round(taskDone / totalTasks * 36),
|
||||
processed: taskDone, total: totalTasks,
|
||||
message: `${SUBJECT_LABEL[st] || st} / ${STATUS[ct] || ct}：第 ${page} 页，累计 ${count}`
|
||||
});
|
||||
const got = mode === 'recent'
|
||||
? await fetchCollectionRecent(username, st, ct, token, cutoff, onPage)
|
||||
: await fetchCollectionAll(username, st, ct, token, onPage);
|
||||
remote.push(...got);
|
||||
taskDone++;
|
||||
report({ phase: 'fetching', percent: 4 + Math.round(taskDone / totalTasks * 36), processed: taskDone, total: totalTasks });
|
||||
await sleep(mode === 'recent' ? 80 : 160);
|
||||
}
|
||||
}
|
||||
|
||||
// --- phase 2: subject details (skipped when refreshSubjects is off) ---
|
||||
const subjectDetails = new Map();
|
||||
if (refreshSubjects) {
|
||||
report({ phase: 'details', percent: 40, processed: 0, total: remote.length, message: `远端 ${remote.length} 条，获取条目详情…` });
|
||||
for (let i = 0; i < remote.length; i++) {
|
||||
const sid = remote[i].subject_id || remote[i].subject?.id;
|
||||
if (!sid) continue;
|
||||
const detail = await fetchSubjectDetail(sid, token);
|
||||
if (detail) subjectDetails.set(String(sid), detail);
|
||||
if ((i + 1) % 10 === 0 || i === remote.length - 1) {
|
||||
report({ phase: 'details', percent: 40 + Math.round((i + 1) / Math.max(1, remote.length) * 45), processed: i + 1, total: remote.length, message: `条目详情 ${i + 1}/${remote.length}` });
|
||||
}
|
||||
await sleep(110);
|
||||
}
|
||||
} else {
|
||||
report({ phase: 'details', percent: 85, message: '已跳过条目详情刷新' });
|
||||
}
|
||||
|
||||
// --- phase 3: normalize + compare ---
|
||||
report({ phase: 'merging', percent: 86, message: '比对本地…' });
|
||||
const seen = new Map();
|
||||
for (const item of remote) {
|
||||
let n = normalizeCollection(item, item.__fallback);
|
||||
const sid = String(n.subject_id);
|
||||
if (subjectDetails.has(sid)) n = applySubjectDetail(n, subjectDetails.get(sid));
|
||||
const snap = makeSnapshot(n);
|
||||
if (snap.subject_id) seen.set(sid, snap);
|
||||
}
|
||||
|
||||
// safety threshold (full only)
|
||||
if (mode === 'full') {
|
||||
const activeLocal = getAll().filter(c => c.status !== 'confirmed_deleted' && c.status !== 'protected_local');
|
||||
const missingPreview = activeLocal.filter(c => !seen.has(String(c.subject_id)));
|
||||
if (activeLocal.length && missingPreview.length > Math.max(10, Math.floor(activeLocal.length * 0.03))) {
|
||||
const run = { id: Date.now().toString(), mode, trigger, started_at: nowISO(), finished_at: nowISO(), success: false, audit_only: true, fetched_total: seen.size, missing_preview: missingPreview.length, error: '远端缺失数量超过安全阈值，本次不写入本地' };
|
||||
state.runs.push(run);
|
||||
await store.persistSyncResult({ collections: state.collections, history: state.history, runs: state.runs, lastRunDate: new Date().toLocaleDateString('sv') });
|
||||
return { aborted: true, reason: 'safety', missing_preview: missingPreview.length, active: activeLocal.length };
|
||||
}
|
||||
}
|
||||
|
||||
let added = 0, modified = 0, restored = 0, missing = 0;
|
||||
const runId = Date.now().toString(); const started_at = nowISO();
|
||||
for (const [id, n] of seen) {
|
||||
const old = state.collections[id];
|
||||
if (!old) {
|
||||
const commentTime = String(n.comment || '').trim() ? (n.bangumi_updated_at || n.collection_created_at || started_at) : '';
|
||||
state.collections[id] = { ...n, status: 'active', marked_at: n.collection_created_at || '', mark_changed_at: n.bangumi_updated_at || n.collection_created_at || started_at, first_seen_at: started_at, last_seen_at: started_at, last_synced_at: started_at, comment_updated_at: commentTime, missing_count: 0, last_sync_run_id: runId };
|
||||
added++;
|
||||
state.history.push({ subject_id: n.subject_id, change_type: 'added', old: null, new: structuredClone(state.collections[id]), created_at: started_at, sync_run_id: runId, note: mode === 'recent' ? '最近同步首次备份' : '首次备份' });
|
||||
} else {
|
||||
if (old.status === 'missing_candidate' || old.status === 'protected_local' || old.status === 'confirmed_deleted') {
|
||||
restored++;
|
||||
state.history.push({ subject_id: n.subject_id, change_type: 'restored', old: structuredClone(old), new: structuredClone(n), changes: collectionFieldDiff(old, n), created_at: started_at, sync_run_id: runId, note: '远端重新出现' });
|
||||
}
|
||||
if (old.content_hash !== n.content_hash) {
|
||||
const changes = collectionFieldDiff(old, n); modified++;
|
||||
state.history.push({ subject_id: n.subject_id, change_type: 'modified', old: structuredClone(old), new: structuredClone(n), changes, created_at: started_at, sync_run_id: runId, note: '收藏字段、吐槽、标签、评分、进度或条目资料变化' });
|
||||
}
|
||||
const statusChanged = old.collection_type !== n.collection_type;
|
||||
const markChangedAt = statusChanged ? (n.bangumi_updated_at || started_at) : (old.mark_changed_at || n.bangumi_updated_at || old.collection_created_at || '');
|
||||
state.collections[id] = { ...old, ...n, status: 'active', mark_changed_at: markChangedAt, first_seen_at: old.first_seen_at || started_at, last_seen_at: started_at, last_synced_at: started_at, missing_count: 0, last_sync_run_id: runId };
|
||||
}
|
||||
}
|
||||
|
||||
if (mode === 'full') {
|
||||
for (const c of getAll()) {
|
||||
const id = String(c.subject_id); if (seen.has(id)) continue;
|
||||
if (c.status === 'confirmed_deleted' || c.status === 'protected_local') continue;
|
||||
if (c.status !== 'missing_candidate') {
|
||||
state.history.push({ subject_id: c.subject_id, change_type: 'missing', old: structuredClone(c), new: null, created_at: started_at, sync_run_id: runId, note: '远端完整同步后未出现；未删除，仅进入安全箱' });
|
||||
}
|
||||
c.status = 'missing_candidate'; c.missing_count = (c.missing_count || 0) + 1; c.last_sync_run_id = runId; missing++;
|
||||
}
|
||||
}
|
||||
|
||||
state.runs.push({ id: runId, mode, trigger, recent_months: effectiveMonths, started_at, finished_at: nowISO(), success: true, fetched_total: seen.size, added, modified, restored, missing });
|
||||
|
||||
// --- phase 4: covers ---
|
||||
// Only cache covers for items fetched in THIS sync (the `seen` set), not the whole library.
|
||||
// So a recent sync caches just the recently-updated items; a full sync caches everything fetched.
|
||||
if (cacheCovers) {
|
||||
report({ phase: 'covers', percent: 95, message: '缓存封面…' });
|
||||
const seenColl = {};
|
||||
for (const id of seen.keys()) { if (state.collections[id]) seenColl[id] = state.collections[id]; }
|
||||
await cacheCoversForState(seenColl, (done, total) => {
|
||||
report({ phase: 'covers', percent: 95 + Math.round(done / Math.max(1, total) * 4), processed: done, total, message: `封面 ${done}/${total}` });
|
||||
}, state.settings.imageProxy);
|
||||
}
|
||||
|
||||
report({ phase: 'saving', percent: 99, message: '写入数据库…' });
|
||||
await store.persistSyncResult({ collections: state.collections, history: state.history, runs: state.runs, lastRunDate: new Date().toLocaleDateString('sv') });
|
||||
|
||||
return { added, modified, restored, missing, fetched_total: seen.size };
|
||||
}
|
||||
|
||||
function startSync(store, mode, trigger, opts = {}) {
|
||||
if (lock) return false;
|
||||
lock = true;
|
||||
syncStatus = {
|
||||
running: true, mode, trigger, phase: 'starting', percent: 0, processed: 0, total: 0,
|
||||
startedAt: new Date().toISOString(), finishedAt: null, message: '准备同步…', lastResult: null, error: null
|
||||
};
|
||||
runSync({ store, mode, trigger, opts, onProgress: p => Object.assign(syncStatus, p) })
|
||||
.then(r => { syncStatus.lastResult = r; syncStatus.phase = r?.aborted ? 'aborted' : 'done'; syncStatus.percent = 100; syncStatus.message = r?.aborted ? '安全保护触发，本次未覆盖本地' : '同步完成'; })
|
||||
.catch(e => { syncStatus.error = String(e?.message || e); syncStatus.phase = 'error'; syncStatus.message = '同步失败：' + syncStatus.error; })
|
||||
.finally(() => { syncStatus.running = false; syncStatus.finishedAt = new Date().toISOString(); lock = false; });
|
||||
return true;
|
||||
}
|
||||
|
||||
// --- cron scheduling (no external deps) ---
|
||||
|
||||
// Parse one cron field into a Set of allowed values. Supports '*', '*/n', 'a', 'a-b', 'a-b/n', and comma lists.
|
||||
function parseCronField(field, min, max) {
|
||||
const out = new Set();
|
||||
for (const part of String(field).split(',')) {
|
||||
const m = part.trim().match(/^(\*|\d+)(?:-(\d+))?(?:\/(\d+))?$/);
|
||||
if (!m) continue;
|
||||
const step = m[3] ? Number(m[3]) : 1;
|
||||
if (step < 1) continue;
|
||||
let lo, hi;
|
||||
if (m[1] === '*') { lo = min; hi = max; }
|
||||
else { lo = Number(m[1]); hi = m[2] !== undefined ? Number(m[2]) : (m[3] ? max : lo); }
|
||||
for (let v = lo; v <= hi; v += step) { if (v >= min && v <= max) out.add(v); }
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// Match a 5-field cron expression (min hour day-of-month month day-of-week) against a Date (local time).
|
||||
function cronMatches(expr, date) {
|
||||
const parts = String(expr).trim().split(/\s+/);
|
||||
if (parts.length !== 5) return false;
|
||||
const [mi, ho, dom, mon, dow] = parts;
|
||||
if (!parseCronField(mi, 0, 59).has(date.getMinutes())) return false;
|
||||
if (!parseCronField(ho, 0, 23).has(date.getHours())) return false;
|
||||
if (!parseCronField(mon, 1, 12).has(date.getMonth() + 1)) return false;
|
||||
const domRestricted = dom.trim() !== '*';
|
||||
const dowRestricted = dow.trim() !== '*';
|
||||
const domOk = parseCronField(dom, 1, 31).has(date.getDate());
|
||||
const dowOk = parseCronField(dow, 0, 6).has(date.getDay());
|
||||
// Standard cron: when both DOM and DOW are restricted, match if either matches.
|
||||
if (domRestricted && dowRestricted) return domOk || dowOk;
|
||||
if (domRestricted) return domOk;
|
||||
if (dowRestricted) return dowOk;
|
||||
return true;
|
||||
}
|
||||
|
||||
// Fallback: derive a cron from the legacy autoSyncTime/autoSyncDays settings.
|
||||
function legacyToCron(s) {
|
||||
if (!s.autoSyncTime && !Array.isArray(s.autoSyncDays)) return null;
|
||||
const [h, m] = String(s.autoSyncTime || '02:00').split(':').map(Number);
|
||||
const days = Array.isArray(s.autoSyncDays) ? s.autoSyncDays : [];
|
||||
const dow = (days.length === 0 || days.length >= 7) ? '*' : [...days].sort((a, b) => a - b).join(',');
|
||||
return `${m || 0} ${h || 0} * * ${dow}`;
|
||||
}
|
||||
|
||||
let lastAutoRunKey = '';
|
||||
|
||||
async function checkSchedule(store) {
|
||||
if (lock) return;
|
||||
let state;
|
||||
try { state = await store.getState(); } catch { return; }
|
||||
const s = (state && state.settings) || {};
|
||||
if (s.autoSyncEnabled !== true && s.autoSyncEnabled !== 'true') return; // accept boolean or legacy string
|
||||
const cron = s.autoSyncCron || legacyToCron(s) || '0 2 * * *';
|
||||
const now = new Date();
|
||||
const key = now.toLocaleString('sv').slice(0, 16); // local 'YYYY-MM-DD HH:MM'
|
||||
if (lastAutoRunKey === key) return; // in-memory dedup (this minute already handled)
|
||||
if (s.autoSyncLastRunKey === key) return; // persisted dedup (survives container restart)
|
||||
if (!cronMatches(cron, now)) return;
|
||||
lastAutoRunKey = key;
|
||||
try { await store.setStateSettings({ autoSyncLastRunKey: key }); } catch {}
|
||||
startSync(store, s.autoSyncMode || 'recent', 'auto');
|
||||
}
|
||||
|
||||
module.exports = { runSync, startSync, getStatus, checkSchedule, cronMatches, parseCronField };
|
||||
Reference in New Issue
Block a user