From 9590a34b5d98aaea838fee85dff0c61308f96b2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=B9=A4=E4=BB=99?= Date: Fri, 9 Jan 2026 13:36:57 +0800 Subject: [PATCH 1/5] check-duplicate --- .gitignore | 3 +++ package.json | 12 ++++++++++++ scripts/check-duplicate.js | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+) create mode 100644 package.json create mode 100644 scripts/check-duplicate.js diff --git a/.gitignore b/.gitignore index 715a9758c0..88ab148b2b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ /libretro-super .DS_Store +node_modules +pnpm-lock.yaml +package-lock.json \ No newline at end of file diff --git a/package.json b/package.json new file mode 100644 index 0000000000..9d833e33f5 --- /dev/null +++ b/package.json @@ -0,0 +1,12 @@ +{ + "private": true, + "type": "module", + "scripts": { + "check-duplicate": "node scripts/check-duplicate.js" + }, + "devDependencies": { + "@retrobrainz/dat": "^1.0.1", + "chalk": "^5.6.2", + "glob": "^13.0.0" + } +} diff --git a/scripts/check-duplicate.js b/scripts/check-duplicate.js new file mode 100644 index 0000000000..d14b74e6d6 --- /dev/null +++ b/scripts/check-duplicate.js @@ -0,0 +1,34 @@ +import { parse } from "@retrobrainz/dat"; +import chalk from "chalk"; +import { glob, readFile } from "node:fs/promises"; + +for await (const datFile of glob("**/*.dat")) { + console.log(datFile); + const datContent = await readFile(datFile, "utf8"); + const dat = parse(datContent); + const crcMap = {}; + const serialMap = {}; + for (const game of dat) { + if (game.$class !== "game" || game.$entries?.length !== 1) { + continue; + } + const crc = game.$entries?.[0].crc; + const serial = game.$entries?.[0].serial; + if (crc) { + crcMap[crc] = (crcMap[crc] || 0) + 1; + } + if (serial) { + serialMap[serial] = (serialMap[serial] || 0) + 1; + } + } + for (const [crc, count] of Object.entries(crcMap)) { + if (count > 1) { + console.log(chalk.red(` duplicate crc: ${crc} (${count})`)); + } + } + for (const [serial, count] of Object.entries(serialMap)) { + if (count > 1) { + console.log(chalk.red(` duplicate serial: ${serial} (${count})`)); + } + } +} From e0ec0d9b45e8157aafca385bc1c42fdeaf19b43a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=B9=A4=E4=BB=99?= Date: Fri, 9 Jan 2026 13:39:07 +0800 Subject: [PATCH 2/5] check-duplicate --- scripts/check-duplicate.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/check-duplicate.js b/scripts/check-duplicate.js index d14b74e6d6..3911d5ebdf 100644 --- a/scripts/check-duplicate.js +++ b/scripts/check-duplicate.js @@ -9,7 +9,7 @@ for await (const datFile of glob("**/*.dat")) { const crcMap = {}; const serialMap = {}; for (const game of dat) { - if (game.$class !== "game" || game.$entries?.length !== 1) { + if (game.$class !== "game" || game.$entries?.length !== 1 || game.$entries?.[0].$class !== 'rom') { continue; } const crc = game.$entries?.[0].crc; From 7c5a0d8188af0eaa5882a9ff1939256e47c13a8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=B9=A4=E4=BB=99?= Date: Fri, 9 Jan 2026 13:50:28 +0800 Subject: [PATCH 3/5] check-duplicate --- scripts/check-duplicate.js | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/scripts/check-duplicate.js b/scripts/check-duplicate.js index 3911d5ebdf..300254f262 100644 --- a/scripts/check-duplicate.js +++ b/scripts/check-duplicate.js @@ -6,29 +6,24 @@ for await (const datFile of glob("**/*.dat")) { console.log(datFile); const datContent = await readFile(datFile, "utf8"); const dat = parse(datContent); - const crcMap = {}; - const serialMap = {}; + const crcSerialMap = {}; for (const game of dat) { - if (game.$class !== "game" || game.$entries?.length !== 1 || game.$entries?.[0].$class !== 'rom') { + if ( + game.$class !== "game" || + game.$entries?.length !== 1 || + game.$entries?.[0].$class !== "rom" + ) { continue; } const crc = game.$entries?.[0].crc; - const serial = game.$entries?.[0].serial; - if (crc) { - crcMap[crc] = (crcMap[crc] || 0) + 1; - } - if (serial) { - serialMap[serial] = (serialMap[serial] || 0) + 1; - } - } - for (const [crc, count] of Object.entries(crcMap)) { - if (count > 1) { - console.log(chalk.red(` duplicate crc: ${crc} (${count})`)); - } + const serial = game.$entries?.[0].serial || game.serial; + // if both crc and serial are the same, mark as duplicate + const key = `${crc || ""}-${serial || ""}`; + crcSerialMap[key] = (crcSerialMap[key] || 0) + 1; } - for (const [serial, count] of Object.entries(serialMap)) { + for (const [key, count] of Object.entries(crcSerialMap)) { if (count > 1) { - console.log(chalk.red(` duplicate serial: ${serial} (${count})`)); + console.log(chalk.red(` duplicate: crc(${key.split("-")[0]}), serial(${key.split("-")[1]}), count(${count})`)); } } } From 3e4d7be806ecd5a28b0e1eb672e2696b36f883ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=B9=A4=E4=BB=99?= Date: Fri, 9 Jan 2026 13:58:31 +0800 Subject: [PATCH 4/5] check-duplicate --- package.json | 3 +-- scripts/check-duplicate.js | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 9d833e33f5..bbaab38f0e 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,6 @@ }, "devDependencies": { "@retrobrainz/dat": "^1.0.1", - "chalk": "^5.6.2", - "glob": "^13.0.0" + "chalk": "^5.6.2" } } diff --git a/scripts/check-duplicate.js b/scripts/check-duplicate.js index 300254f262..d2789dc306 100644 --- a/scripts/check-duplicate.js +++ b/scripts/check-duplicate.js @@ -2,7 +2,7 @@ import { parse } from "@retrobrainz/dat"; import chalk from "chalk"; import { glob, readFile } from "node:fs/promises"; -for await (const datFile of glob("**/*.dat")) { +for await (const datFile of glob(["dat/**/*.dat", "metadat/**/*.dat"], { exclude: ["dat/DOS.dat"] })) { console.log(datFile); const datContent = await readFile(datFile, "utf8"); const dat = parse(datContent); From 5c604324e5501b4119ec16c49e5c2844609e415f Mon Sep 17 00:00:00 2001 From: Guo Yunhe Date: Sat, 17 Jan 2026 14:47:59 +0800 Subject: [PATCH 5/5] fix script --- scripts/check-duplicate.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/check-duplicate.js b/scripts/check-duplicate.js index d2789dc306..3cb798d119 100644 --- a/scripts/check-duplicate.js +++ b/scripts/check-duplicate.js @@ -18,12 +18,13 @@ for await (const datFile of glob(["dat/**/*.dat", "metadat/**/*.dat"], { exclude const crc = game.$entries?.[0].crc; const serial = game.$entries?.[0].serial || game.serial; // if both crc and serial are the same, mark as duplicate - const key = `${crc || ""}-${serial || ""}`; + const key = JSON.stringify({crc, serial}); crcSerialMap[key] = (crcSerialMap[key] || 0) + 1; } for (const [key, count] of Object.entries(crcSerialMap)) { if (count > 1) { - console.log(chalk.red(` duplicate: crc(${key.split("-")[0]}), serial(${key.split("-")[1]}), count(${count})`)); + const {crc, serial} = JSON.parse(key); + console.log(chalk.red(` duplicate: crc(${crc}), serial(${serial}), count(${count})`)); } } }