Skip to content

Commit b780d50

Browse files
CopetanCopetan
committed
update tool.ts script
1 parent a57e861 commit b780d50

1 file changed

Lines changed: 72 additions & 45 deletions

File tree

‎src/tool.ts‎

Lines changed: 72 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,21 @@ import { crypto } from "jsr:@std/crypto";
55
//import { MainManifest, VersionData, OmniarchiveMainManifest, OmniVersionManifest } from './types.d.ts';
66

77
const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
8+
const getFileSha1 = async (path: string): Promise<string> => encodeHex((await crypto.subtle.digest('SHA-1', (await Deno.open(path, { read: true })).readable)));
9+
10+
async function fileExists(path: string) {
11+
try {
12+
return await Deno.lstat(path) as unknown as boolean;
13+
} catch (error) {
14+
if(!(error instanceof Deno.errors.NotFound)) throw error;
15+
return false;
16+
}
17+
}
818

919
async function readLocalVersionJsons() {
1020
const versionsMap: Map<string, VersionManifest> = new Map();
1121
for await (const file of Deno.readDir(('data/version/manifest'))) {
12-
const version: VersionManifest = JSON.parse(await Deno.readTextFile('data/version/manifest/' + file.name));
22+
const version: VersionManifest = JSON.parse(await Deno.readTextFile(`data/version/manifest/${file.name}`));
1323
versionsMap.set(version.id, version);
1424
}
1525
return versionsMap;
@@ -19,20 +29,29 @@ async function readLocalDetailsJsons() {
1929
const detailsMap: Map<string, VersionData> = new Map();
2030
for await (const file of Deno.readDir('data/version')) {
2131
if (file.isDirectory) continue;
22-
const versionDetails = JSON.parse(await Deno.readTextFile('data/version/' + file.name));
32+
const versionDetails = JSON.parse(await Deno.readTextFile(`data/version/${file.name}`));
2333
detailsMap.set(versionDetails.id, versionDetails);
2434
}
2535
return detailsMap;
2636
}
2737

28-
async function readAndCacheExternalVersionJsons(remoteManifestJson: OmniarchiveMainManifest) {
38+
async function updateAndCacheExternalVersionJsons(remoteManifestJson: OmniarchiveMainManifest) {
2939
const versionsMap: Map<string, OmniVersionManifest> = new Map();
3040
for (const version of remoteManifestJson.versions) {
31-
const versionJson: OmniVersionManifest = await (await fetch(version.url)).json();
41+
let versionJson: OmniVersionManifest;
42+
43+
const versionJsonPath = `external_manifests/ugly/${version.id}.json`;
44+
if (await fileExists(versionJsonPath) && version.sha1 === await getFileSha1(versionJsonPath)) {
45+
versionJson = JSON.parse(await Deno.readTextFile(versionJsonPath));
46+
console.log(`${version.id}.json is already up to date`);
47+
} else {
48+
versionJson = await (await fetch(version.url)).json();
3249

33-
console.log(`Writing ${versionJson.id}.json`);
34-
await Deno.writeTextFile(`external_manifests/${versionJson.id}.json`, JSON.stringify(versionJson, null, 2));
35-
await sleep(1000);
50+
console.log(`Writing ${version.id}.json`);
51+
await Deno.writeTextFile(versionJsonPath, `${JSON.stringify(versionJson)}\n`);
52+
await Deno.writeTextFile(`external_manifests/${version.id}`, JSON.stringify(versionJson, null, 2));
53+
await sleep(1000);
54+
}
3655

3756
versionsMap.set(version.id, versionJson);
3857
}
@@ -41,17 +60,15 @@ async function readAndCacheExternalVersionJsons(remoteManifestJson: OmniarchiveM
4160
}
4261

4362
async function readCachedExternalVersionJsons() {
44-
try {
45-
await Deno.lstat('external_manifests');
46-
} catch (error) {
47-
if(!(error instanceof Deno.errors.NotFound)) throw error;
63+
const cacheDir = 'external_manifests/ugly';
64+
if (!(await fileExists(cacheDir))) {
65+
await Deno.mkdir(cacheDir);
4866
return null;
4967
}
50-
const cacheDirectory = Deno.readDir('external_manifests');
5168

5269
const versionsMap: Map<string, OmniVersionManifest> = new Map();
53-
for await (const file of cacheDirectory) {
54-
const versionManifest = JSON.parse(await Deno.readTextFile('external_manifests/' + file.name));
70+
for await (const file of Deno.readDir(cacheDir)) {
71+
const versionManifest = JSON.parse(await Deno.readTextFile(`${cacheDir}/${file.name}`));
5572
versionsMap.set(versionManifest.id, versionManifest);
5673
}
5774
return versionsMap;
@@ -235,44 +252,58 @@ const mergedMirrorMap: Map<MirrorId, [client: LocalOriginalId, server: LocalOrig
235252
["14w11b", ["14w11b-1650", "14w11b-1640"]],
236253
["1.7.7", ["1.7.7-101331", "1.7.7-091529"]]
237254
]);
238-
// Versions that had disambiguation added to the local manifests, but which don't exist as is in the Omniarchive manifest
255+
// Versions that had disambiguation added to the local manifests, but which don't exist as is in the Omniarchive manifest (with server versions that are unavailable in the Omniarchive manifest)
239256
const weirdMergeMap: Map<LocalId, [client: ExternalOriginalId, server: IndexOriginalId]> = new Map([
240257
["13w22a-1434", ["13w22a", "13w22a-1434"]],
241258
["1.6.4-201309191549", ["1.6.4", "1.6.4-201309191549"]],
242259
["13w39a-1511", ["13w39a", "13w39a-1511"]],
243-
// TODO super weird merge
260+
// Since only the jar got reuploaded, Omniarchive's manifest doesn't have the jar version, but *does* contain the Windows server exe
244261
["14w04a-1526", ["14w04a", "14w04a-1526"]],
245262
["1.7.5-02260922", ["1.7.5", "1.7.5-02260922"]]
246263
]);
247264

265+
const shouldSkip = (key: string): boolean => !(standaloneSevers.includes(key) || orphanServers.includes(key) ||
266+
mirrorlessRenameMap.has(key) || mirrorMap.has(key) || reverseMirrorMap.has(key) || mergedMirrorMap.has(key) || weirdMergeMap.has(key));
267+
248268
export { standaloneSevers, renamedStandaloneServers, orphanServers, renamedOrphanServers, mirrorlessRenameMap, mirrorMap, reverseMirrorMap, mergedMirrorMap, weirdMergeMap }
249269

250270
function compareLocalWithOmniarchive(localVersionsMap: Map<string, VersionManifest>, remoteVersionsMap: Map<string, OmniVersionManifest>) {
251-
const missing = [];
271+
const missingExternal = [];
272+
252273
localVersionsMap.forEach((_value, key) => {
253-
if (standaloneSevers.includes(key)) {
254-
console.log(`${key} is a standalone server`);
255-
} else if (orphanServers.includes(key)) {
256-
console.log(`${key} is an orphan server`);
257-
} else if (mirrorlessRenameMap.has(key)) {
258-
console.log(`${key} is a renamed version for omniarchive's ${mirrorlessRenameMap.get(key)} without a mirror`);
259-
} else if (mirrorMap.has(key)) {
260-
console.log(`${key} is a mirror version for ${mirrorMap.get(key)}`);
261-
} else if(reverseMirrorMap.has(key)) {
262-
console.log(`${key} is a reverse mirror version for ${mirrorMap.get(key)}`);
263-
} else if (mergedMirrorMap.has(key)) {
264-
const [client, server] = mergedMirrorMap.get(key)!;
265-
console.log(`${key} is a merged mirror version for ${client} client and ${server} server`);
266-
} else if (weirdMergeMap.has(key)) {
267-
const [client, server] = weirdMergeMap.get(key)!;
268-
console.log(`${key} is a weird merged version for Omniarchive's ${client} client and ${server} orphaned server`);
269-
} else if (!remoteVersionsMap.has(key)) {
274+
if (shouldSkip(key) && !remoteVersionsMap.has(key)) {
270275
console.log(`%cOmniarchive manifest is missing ${key}`, 'color: red');
271-
missing.push(key);
276+
missingExternal.push(key);
277+
}
278+
});
279+
280+
console.log(`Skipped ${standaloneSevers.length + orphanServers.length + mirrorlessRenameMap.size + mirrorMap.size + reverseMirrorMap.size + mergedMirrorMap.size + weirdMergeMap.size} versions
281+
- ${standaloneSevers.length} server versions included in most closely related client in the Omniarchive manifest
282+
- ${orphanServers.length} server versions that are not included in the Omniarchive manifest at all
283+
- ${mirrorlessRenameMap.size} versions (that are not copies of another local manifest) with a disambiguation removed when compared to its respective Omniarchive manifest (i.e. Omniarchive's 1.7.1-pre is just 1.7.1 in this manifest)
284+
- ${mirrorMap.size} versions that are copies of a version with a disambiguation suffix (e.g. 14w04b for 14w04b-1554)
285+
- ${reverseMirrorMap.size} versions that are the reverse of the previous ones
286+
- ${mergedMirrorMap.size} versions without disambiguation that contain downloads of two different versions (one client and one server)
287+
- ${weirdMergeMap.size} versions that had a disambiguation added in this manifest to distinguish them from Omniarchive versions without disambiguation and other local versions`);
288+
console.log(`\nOmniarchive manifest is missing ${missingExternal.length} versions`);
289+
290+
// const test = remoteVersionsMap.keys().toArray().sort();
291+
console.log(/*test*/);
292+
293+
const missingLocal = [];
294+
const iterable = remoteVersionsMap.entries().filter((entry) =>
295+
entry[0] <= "1.14.4-pre7" || (entry[0] >= "1.2-pre" && entry[0] <= "1.2.5") || (entry[0] >= "1.3-pre-1249" && entry[0] <= "19w36a")
296+
|| entry[0].startsWith("2.0-") || (entry[0] >= "3D Shareware v1.34" && entry[0] <= "combat3") || entry[0] >= "in-20091223-1459");
297+
const filteredRemoteMap = new Map(iterable);
298+
299+
filteredRemoteMap.forEach((_value, key) => {
300+
if (!localVersionsMap.has(key) && !mirrorlessRenameMap.values().toArray().includes(key)) {
301+
console.log(`%cThe local manifest is missing ${key}`, 'color: red');
302+
missingLocal.push(key);
272303
}
273304
});
274305

275-
console.log(`\nOmniarchive manifest is missing ${missing.length} versions`);
306+
console.log(`\nLocal manifest is missing ${missingLocal.length} versions`);
276307
}
277308

278309
function checkLocalManifestEntries(manifest: MainManifest, versionJsons: Map<string, VersionManifest>, detailsJsons: Map<string, VersionData>) {
@@ -288,7 +319,7 @@ function checkLocalManifestEntries(manifest: MainManifest, versionJsons: Map<str
288319
}
289320

290321
// log any references to unknown versions in the next and previous fields of details jsons
291-
detailsJsons.forEach((details, id) => {
322+
detailsJsons.forEach((details) => {
292323
if (details.previous != null) {
293324
for (let i = 0; i < details.previous.length; i++) {
294325
const prevId = details.previous[i];
@@ -328,12 +359,8 @@ function checkLocalManifestEntries(manifest: MainManifest, versionJsons: Map<str
328359
async function updateLocalManifestHashes(manifest: MainManifest) {
329360
for (let i = 0; i < manifest.versions.length; i++) {
330361
const version = manifest.versions[i];
331-
const manifestFileStream = (await Deno.open(`data/${version.url}`, { read: true })).readable;
332-
const detailsFileStream = (await Deno.open(`data/${version.details}`, { read: true })).readable;
333-
const manifestHashBuffer = await crypto.subtle.digest('SHA-1', manifestFileStream);
334-
const detailsHashBuffer = await crypto.subtle.digest('SHA-1', detailsFileStream);
335-
manifest.versions[i].sha1 = encodeHex(manifestHashBuffer);
336-
manifest.versions[i].detailsSha1 = encodeHex(detailsHashBuffer);
362+
manifest.versions[i].sha1 = await getFileSha1(`data/${version.url}`);
363+
manifest.versions[i].detailsSha1 = await getFileSha1(`data/${version.details}`);
337364
}
338365
await Deno.writeTextFile('data/version_manifest.json', JSON.stringify(manifest, null, 2));
339366
console.log('Hashes updated');
@@ -349,7 +376,7 @@ if (import.meta.main) (async () => {
349376
const cached = await readCachedExternalVersionJsons();
350377
if (cached && cached.size > 0) return cached;
351378
console.log('Cache does not exist! Creating...\n');
352-
return await readAndCacheExternalVersionJsons(remoteManifestJson);
379+
return await updateAndCacheExternalVersionJsons(remoteManifestJson);
353380
})();
354381

355382
console.log('Locally stored manifests loaded');
@@ -377,7 +404,7 @@ if (import.meta.main) (async () => {
377404
case '4': {
378405
const confirmation = confirm('Are you sure? This will take a while');
379406
console.log();
380-
if (confirmation) remoteVersionJsonsMap = await readAndCacheExternalVersionJsons(remoteManifestJson);
407+
if (confirmation) remoteVersionJsonsMap = await updateAndCacheExternalVersionJsons(remoteManifestJson);
381408
break;
382409
}
383410
case 'e':

0 commit comments

Comments
 (0)