Skip to content

fix(sync): (Codex) Sync digest consistency / eventual consistency #1393

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
CodFrm merged 2 commits into from
May 6, 2026
Merged

fix(sync): (Codex) Sync digest consistency / eventual consistency #1393

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
68f3606
fix(sync): Sync digest consistency / eventual consistency
cyfung1031 May 1, 2026
1b4e05c
fix(sync): preserve cloud-native digest in updateFileDigest
CodFrm May 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
97 changes: 97 additions & 0 deletions src/app/service/service_worker/synchronize.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { initTestEnv } from "@Tests/utils";
import type FileSystem from "@Packages/filesystem/filesystem";
import type { CloudSyncConfig } from "@App/pkg/config/config";
import { stackAsyncTask } from "@App/pkg/utils/async_queue";
import { md5OfText } from "@App/pkg/utils/crypto";

initTestEnv();

Expand Down Expand Up @@ -397,6 +398,101 @@ console.log("ok");`
expect(order.indexOf("push:end")).toBeLessThan(order.indexOf("digest:list"));
});

it("keeps pushed script digest when cloud list is stale after upload", async () => {
const scriptCode = "// code";
const script = {
uuid: "push-uuid",
name: "push",
origin: "origin",
downloadUrl: "download-url",
checkUpdateUrl: "check-update-url",
updatetime: 1,
createtime: 1,
status: 1,
sort: 0,
metadata: {},
};
const fs = createFs({
list: vi.fn().mockResolvedValueOnce([]).mockResolvedValueOnce([]),
});
const service = new SynchronizeService(
{} as any,
{} as any,
{} as any,
{} as any,
{} as any,
{} as any,
{} as any,
{
scriptCodeDAO: {
get: vi.fn().mockResolvedValue({ code: scriptCode }),
},
all: vi.fn().mockResolvedValue([script]),
} as any
);

await service.syncOnce({ ...syncConfig, syncStatus: false }, fs);

const metaJson = JSON.stringify({
uuid: script.uuid,
origin: script.origin,
downloadUrl: script.downloadUrl,
checkUpdateUrl: script.checkUpdateUrl,
});
await expect((service as any).storage.get("file_digest")).resolves.toEqual({
"push-uuid.user.js": md5OfText(scriptCode),
"push-uuid.meta.json": md5OfText(metaJson),
});
});

it("preserves cloud-native digest and does not overwrite with pushed md5", async () => {
// 各后端 digest 格式不一致（webdav/onedrive 是 etag、dropbox 是 content_hash 等），
// 上传后再次 list 已经能拿到原生 digest 时，必须保留它，不能被本地 md5 覆盖，
// 否则下次同步比对会因格式不一致而把未变动的脚本判定为已变动并触发不必要的拉取/推送
const scriptCode = "// code";
const script = {
uuid: "push-uuid",
name: "push",
origin: "origin",
downloadUrl: "download-url",
checkUpdateUrl: "check-update-url",
updatetime: 1,
createtime: 1,
status: 1,
sort: 0,
metadata: {},
};
const cloudListAfterPush = [
{ name: "push-uuid.user.js", digest: "etag-user-js", updatetime: 1 },
{ name: "push-uuid.meta.json", digest: "etag-meta-json", updatetime: 1 },
];
const fs = createFs({
list: vi.fn().mockResolvedValueOnce([]).mockResolvedValueOnce(cloudListAfterPush),
});
const service = new SynchronizeService(
{} as any,
{} as any,
{} as any,
{} as any,
{} as any,
{} as any,
{} as any,
{
scriptCodeDAO: {
get: vi.fn().mockResolvedValue({ code: scriptCode }),
},
all: vi.fn().mockResolvedValue([script]),
} as any
);

await service.syncOnce({ ...syncConfig, syncStatus: false }, fs);

await expect((service as any).storage.get("file_digest")).resolves.toEqual({
"push-uuid.user.js": "etag-user-js",
"push-uuid.meta.json": "etag-meta-json",
});
});

it("scriptInstall enters cloud_sync queue and updates digest after push", async () => {
let releaseSync!: () => void;
const syncGate = new Promise<void>((resolve) => {
Expand Down Expand Up @@ -439,6 +535,7 @@ console.log("ok");`
vi.spyOn(service as any, "buildFileSystem").mockResolvedValue(installFs);
vi.spyOn(service, "pushScript").mockImplementation(async () => {
order.push("install:push");
return {};
});
const realUpdateDigest = service.updateFileDigest.bind(service);
vi.spyOn(service, "updateFileDigest").mockImplementation(async (fs) => {
Expand Down
68 changes: 44 additions & 24 deletions src/app/service/service_worker/synchronize.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ import { dayFormat } from "@App/pkg/utils/day_format";
import i18n, { i18nName } from "@App/locales/locales";
import { InfoNotification } from "./utils";
import { stackAsyncTask } from "@App/pkg/utils/async_queue";
import { md5OfText } from "@App/pkg/utils/crypto";

// type SynchronizeTarget = "local";

Expand Down Expand Up @@ -67,6 +68,10 @@ type ScriptcatSyncStatus = {

type PushScriptParam = TInstallScriptParams;

type FileDigestMap = {
[key: string]: string;
};

const SYNC_SERVICE_TASK_KEY = "cloud_sync_queue";

export class SynchronizeService {
Expand Down Expand Up @@ -348,10 +353,7 @@ export class SynchronizeService {
// 根据文件名生成一个map
const uuidMap = new Map<string, Partial<SyncFiles>>();
// 储存文件摘要,用于检测文件是否有变化
const fileDigestMap =
((await this.storage.get("file_digest")) as {
[key: string]: string;
}) || {};
const fileDigestMap = ((await this.storage.get("file_digest")) as FileDigestMap) || {};

for (const file of list) {
if (file.name.endsWith(".user.js")) {
Expand Down Expand Up @@ -397,7 +399,7 @@ export class SynchronizeService {
}

// 对比脚本列表和文件列表,进行同步
const result: Promise<void>[] = [];
const result: Promise<FileDigestMap | void>[] = [];
const updateScript: Map<string, boolean> = new Map();
// 记录被跳过的孤儿云端脚本（仅 .user.js 无 .meta.json）
// 避免本机回写 scriptcat-sync.json 时丢失对应 uuid 的云端 status
Expand Down Expand Up @@ -426,7 +428,7 @@ export class SynchronizeService {
} else {
// 否则认为是一个无效的.meta文件，进行删除，并进行同步
await fs.delete(file.meta!.name);
await this.pushScript(fs, script);
return await this.pushScript(fs, script);
}
})()
);
Expand Down Expand Up @@ -469,7 +471,13 @@ export class SynchronizeService {
result.push(this.pushScript(fs, script));
});
// 忽略错误
await Promise.allSettled(result);
const syncResults = await Promise.allSettled(result);
const pushedFileDigestMap: FileDigestMap = {};
syncResults.forEach((ret) => {
if (ret.status === "fulfilled" && ret.value) {
Object.assign(pushedFileDigestMap, ret.value);
}
});
// 同步状态
if (syncConfig.syncStatus) {
const scriptlist = await this.scriptDAO.all();
Expand Down Expand Up @@ -533,17 +541,25 @@ export class SynchronizeService {
}
// 重新获取文件列表,保存文件摘要
this.logger.info("update file digest");
await this.updateFileDigest(fs);
await this.updateFileDigest(fs, pushedFileDigestMap);
this.logger.info("sync complete");
return;
}

async updateFileDigest(fs: FileSystem) {
async updateFileDigest(fs: FileSystem, knownFileDigestMap: FileDigestMap = {}) {
const newList = await fs.list();
const newFileDigestMap: { [key: string]: string } = {};
const newFileDigestMap: FileDigestMap = {};
for (const file of newList) {
newFileDigestMap[file.name] = file.digest;
}
// 各后端 digest 格式不一（WebDAV/OneDrive/S3 是 etag、Dropbox 是 content_hash、Zip 为空，
// 仅 GoogleDrive/Baidu 是 md5），只在云端列表暂时漏掉刚上传的文件时用本地 md5 兜底，
// 不能覆盖 fs.list 已返回的原生 digest，否则下次同步比对会因格式不一致而误判
for (const name in knownFileDigestMap) {
if (!(name in newFileDigestMap)) {
newFileDigestMap[name] = knownFileDigestMap[name];
}
}
await this.storage.set("file_digest", newFileDigestMap);
return;
}
Expand Down Expand Up @@ -581,8 +597,9 @@ export class SynchronizeService {
}

// 上传脚本
async pushScript(fs: FileSystem, script: PushScriptParam) {
async pushScript(fs: FileSystem, script: PushScriptParam): Promise<FileDigestMap> {
const filename = `${script.uuid}.user.js`;
const metaFilename = `${script.uuid}.meta.json`;
const logger = this.logger.with({
uuid: script.uuid,
name: script.name,
Expand All @@ -592,22 +609,25 @@ export class SynchronizeService {
const w = await fs.create(filename);
// 获取脚本代码
const code = await this.scriptCodeDAO.get(script.uuid);
await w.write(code!.code);
const meta = await fs.create(`${script.uuid}.meta.json`);
await meta.write(
JSON.stringify(<SyncMeta>{
uuid: script.uuid,
origin: script.origin,
downloadUrl: script.downloadUrl,
checkUpdateUrl: script.checkUpdateUrl,
})
);
const scriptCode = code!.code;
await w.write(scriptCode);
const meta = await fs.create(metaFilename);
const metaJson = JSON.stringify(<SyncMeta>{
uuid: script.uuid,
origin: script.origin,
downloadUrl: script.downloadUrl,
checkUpdateUrl: script.checkUpdateUrl,
});
await meta.write(metaJson);
logger.info("push script success");
return {
[filename]: md5OfText(scriptCode),
[metaFilename]: md5OfText(metaJson),
};
} catch (e) {
logger.error("push script error", Logger.E(e));
throw e;
}
return;
}

async pullScript(fs: FileSystem, file: SyncFiles, status: ScriptcatSyncStatus | undefined, existingScript?: Script) {
Expand Down Expand Up @@ -703,8 +723,8 @@ export class SynchronizeService {
if (config.enable) {
stackAsyncTask(SYNC_SERVICE_TASK_KEY, async () => {
const fs = await this.buildFileSystem(config);
await this.pushScript(fs, params.script);
await this.updateFileDigest(fs);
const pushedFileDigestMap = await this.pushScript(fs, params.script);
await this.updateFileDigest(fs, pushedFileDigestMap);
}).catch((e) => {
this.logger.error("push script on install error", Logger.E(e));
});
Expand Down
Loading