From 2ee3294bf49975c6f2da74e9ae951a52ea295232 Mon Sep 17 00:00:00 2001 From: Peter Hirn Date: Sat, 3 May 2025 11:56:51 +0200 Subject: [PATCH 1/4] wip: element table --- src/elementTable.test.ts | 20 ++++++ src/elementTable.ts | 133 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 153 insertions(+) create mode 100644 src/elementTable.test.ts create mode 100644 src/elementTable.ts diff --git a/src/elementTable.test.ts b/src/elementTable.test.ts new file mode 100644 index 0000000..90b0e6e --- /dev/null +++ b/src/elementTable.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, test } from "vitest"; + +import { elementTable } from "./elementTable.js"; +import { exampleFile } from "./node.test.js"; + +describe("element table", () => { + test("empty-2025", async () => { + const file = await exampleFile("empty-2025.rvt"); + const table = await elementTable(file); + + expect(table.size).toEqual(2413); + }); + + test("2026", async () => { + const file = await exampleFile("racbasicsamplefamily-2026.rfa"); + const table = await elementTable(file); + + expect(table.size).toEqual(1992); + }); +}); diff --git a/src/elementTable.ts b/src/elementTable.ts new file mode 100644 index 0000000..ce09af4 --- /dev/null +++ b/src/elementTable.ts @@ -0,0 +1,133 @@ +import { writeFileSync } from "node:fs"; + +import { Cfb } from "./cfb/index.js"; +import * as array from "./utils/array.js"; + +export interface ElementEntry { + id: bigint; + unknown1: number; + unknown2: number; + unknown3: number; + id2: bigint; + unknown4: bigint; + unknown5: number; +} + +export interface ElementTable { + fileVersion: number; + size: number; +} + +export const parseElementTable = async (data: Uint8Array): Promise => { + const header = data.subarray(0, 8); + if (!array.isZero(header)) + throw Error(`Unexpected element table compressed header non-zero [${header}]`); + + const stream = new Blob([data.subarray(8)]).stream(); + const decompressedStream = stream.pipeThrough(new DecompressionStream("gzip")); + + const chunks = []; + for await (const chunk of decompressedStream) { + chunks.push(chunk); + } + + const blob = new Blob(chunks); + const buffer = await blob.arrayBuffer(); + const decompressed = new Uint8Array(buffer); + + const fileVersion = decompressed[0]; + + if (decompressed[1] !== 0x05) + throw Error(`Unexpected element table decompressed header [${decompressed[1]}]`); + + const view = new DataView( + decompressed.buffer, + decompressed.byteOffset, + decompressed.byteLength + ); + + const size = view.getInt32(2, true); + + const ids: ElementEntry[] = []; + for (let i = 0; i < size; i++) { + const offset = i * 40 + 6; + const id = view.getBigInt64(offset, true); + const unknown1 = view.getInt32(offset + 8, true); + const unknown2 = view.getInt32(offset + 12, true); + const unknown3 = view.getInt32(offset + 16, true); + const id2 = view.getBigInt64(offset + 20, true); + const unknown4 = view.getBigInt64(offset + 28, true); + const unknown5 = view.getInt32(offset + 36, true); + ids.push({ + id, + unknown1, + unknown2, + unknown3, + id2, + unknown4, + unknown5 + }); + } + + console.log(size); + //const missing = [1521n, 1957n, 1967n, 1976n, 2283n]; + const missing = [17n, 23n, 24n, 25n, 29n, 31n, 32n, 33n, 5702n, 5701n, 7199n, 7200n]; + //console.log(ids.slice(0, 100).map((e) => e.id)); + + console.log(ids.filter((e) => e.id === 0n)); + /* + console.log(ids.filter((e) => e.id === 1n)); + console.log(ids.filter((e) => e.id === 1519n)); + console.log(ids.filter((e) => e.id === 1520n)); + */ + console.log(ids.filter((e) => missing.includes(e.id))); + + console.log(ids.filter((e) => e.id > e.unknown4).length); + //console.log(ids.filter((e) => e.unknown2 !== 74).length); + + console.log( + ids.filter( + (e) => + !( + e.unknown1 === 74 && + e.unknown2 === 74 && + e.unknown3 === 74 && + e.id > e.unknown4 + ) + ).length + ); + + //const foo = ids.filter((e) => e.id > e.unknown4); + //writeFileSync("ids.txt", foo.map((e) => e.id).join(",")); + + return { fileVersion, size }; +}; + +export interface ElementTableSuccess { + ok: true; + data: ElementTable; + error?: never; +} + +export interface ElementTableError { + ok: false; + data?: never; + error: string; +} + +export type ElementTableResult = ElementTableSuccess | ElementTableError; + +export const elementTable = async (cfb: Cfb): Promise => { + const entry = cfb.findEntry("ElemTable"); + if (!entry) throw Error("ElemTable not found"); + return await parseElementTable(await cfb.entryData(entry)); +}; + +export const tryElementTable = async (cfb: Cfb): Promise => { + try { + return { ok: true, data: await elementTable(cfb) }; + } catch (e) { + if (e instanceof Error) return { ok: false, error: e.message }; + throw e; + } +}; From b2ab9846c0b487b7a48c74b96f5edd1e461b4223 Mon Sep 17 00:00:00 2001 From: Peter Hirn Date: Sat, 3 May 2025 13:14:40 +0200 Subject: [PATCH 2/4] wip: add empty project example files --- examples/empty-2025.rvt | 3 +++ examples/empty-2026.rvt | 3 +++ src/elementTable.test.ts | 11 +++++++++-- src/elementTable.ts | 41 ++++++++++++++++++++++++++-------------- src/node.test.ts | 3 +++ 5 files changed, 45 insertions(+), 16 deletions(-) create mode 100644 examples/empty-2025.rvt create mode 100644 examples/empty-2026.rvt diff --git a/examples/empty-2025.rvt b/examples/empty-2025.rvt new file mode 100644 index 0000000..b9ecd03 --- /dev/null +++ b/examples/empty-2025.rvt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c49c82adbee1bf2ecc8cd7d42a751462bdd39aca4c76b5726e6a7abdf6bf752a +size 454656 diff --git a/examples/empty-2026.rvt b/examples/empty-2026.rvt new file mode 100644 index 0000000..5ab7af0 --- /dev/null +++ b/examples/empty-2026.rvt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:948e82ae3f73a314551b0eb87baffc7a2caa77a0ba28d975bd8a21a6c5644765 +size 479232 diff --git a/src/elementTable.test.ts b/src/elementTable.test.ts index 90b0e6e..d13949a 100644 --- a/src/elementTable.test.ts +++ b/src/elementTable.test.ts @@ -1,7 +1,7 @@ import { describe, expect, test } from "vitest"; import { elementTable } from "./elementTable.js"; -import { exampleFile } from "./node.test.js"; +import { adskExampleFile, exampleFile } from "./node.test.js"; describe("element table", () => { test("empty-2025", async () => { @@ -11,8 +11,15 @@ describe("element table", () => { expect(table.size).toEqual(2413); }); + test("empty-2026", async () => { + const file = await exampleFile("empty-2026.rvt"); + const table = await elementTable(file); + + expect(table.size).toEqual(2426); + }); + test("2026", async () => { - const file = await exampleFile("racbasicsamplefamily-2026.rfa"); + const file = await adskExampleFile("racbasicsamplefamily-2026.rfa"); const table = await elementTable(file); expect(table.size).toEqual(1992); diff --git a/src/elementTable.ts b/src/elementTable.ts index ce09af4..c87d7c2 100644 --- a/src/elementTable.ts +++ b/src/elementTable.ts @@ -1,5 +1,3 @@ -import { writeFileSync } from "node:fs"; - import { Cfb } from "./cfb/index.js"; import * as array from "./utils/array.js"; @@ -8,9 +6,7 @@ export interface ElementEntry { unknown1: number; unknown2: number; unknown3: number; - id2: bigint; unknown4: bigint; - unknown5: number; } export interface ElementTable { @@ -18,12 +14,14 @@ export interface ElementTable { size: number; } -export const parseElementTable = async (data: Uint8Array): Promise => { - const header = data.subarray(0, 8); - if (!array.isZero(header)) - throw Error(`Unexpected element table compressed header non-zero [${header}]`); - - const stream = new Blob([data.subarray(8)]).stream(); +/* + * TODO: quick and dirty decompress hack + * - should be streaming + * - underlying CFB should stream too + * - `for await` doesn't work in Safari + */ +const decompress = async (data: Uint8Array): Promise => { + const stream = new Blob([data]).stream(); const decompressedStream = stream.pipeThrough(new DecompressionStream("gzip")); const chunks = []; @@ -33,8 +31,15 @@ export const parseElementTable = async (data: Uint8Array): Promise const blob = new Blob(chunks); const buffer = await blob.arrayBuffer(); - const decompressed = new Uint8Array(buffer); + return new Uint8Array(buffer); +}; + +export const parseElementTable = async (data: Uint8Array): Promise => { + const header = data.subarray(0, 8); + if (!array.isZero(header)) + throw Error(`Unexpected element table compressed header non-zero [${header}]`); + const decompressed = await decompress(data.subarray(8)); const fileVersion = decompressed[0]; if (decompressed[1] !== 0x05) @@ -58,14 +63,16 @@ export const parseElementTable = async (data: Uint8Array): Promise const id2 = view.getBigInt64(offset + 20, true); const unknown4 = view.getBigInt64(offset + 28, true); const unknown5 = view.getInt32(offset + 36, true); + + if (id !== id2) throw Error(`Id mismatch (${id} != ${id2})`); + if (unknown5 !== 0) throw Error(`Unknown5 is not zero (${id})`); + ids.push({ id, unknown1, unknown2, unknown3, - id2, - unknown4, - unknown5 + unknown4 }); } @@ -97,6 +104,12 @@ export const parseElementTable = async (data: Uint8Array): Promise ).length ); + //console.log(ids.filter((e) => e.unknown4 !== -1n).length); + + console.log(ids.filter((e) => e.id <= e.unknown4)); + console.log(ids.find((e) => e.id === 1530n)); + console.log(ids.find((e) => e.id === 1527n)); + //const foo = ids.filter((e) => e.id > e.unknown4); //writeFileSync("ids.txt", foo.map((e) => e.id).join(",")); diff --git a/src/node.test.ts b/src/node.test.ts index 2ae63e6..90dd1ba 100644 --- a/src/node.test.ts +++ b/src/node.test.ts @@ -10,6 +10,9 @@ import { thumbnail } from "./thumbnail.js"; export const examplePath = (...paths: string[]) => join(import.meta.dirname, "..", "examples", ...paths); +export const exampleFile = (fileName: string): Promise => + openPath(examplePath(fileName)); + export const adskExamplePath = (fileName: string): string => join(examplePath("Autodesk", fileName)); From 7dec0944d2cf7bc3fa63ca482fff28ccdf299466 Mon Sep 17 00:00:00 2001 From: Peter Hirn Date: Mon, 5 May 2025 17:37:28 +0200 Subject: [PATCH 3/4] ci(renovate): disable git submodule checking --- renovate.json | 3 --- 1 file changed, 3 deletions(-) diff --git a/renovate.json b/renovate.json index 8a82297..414069a 100644 --- a/renovate.json +++ b/renovate.json @@ -6,9 +6,6 @@ "group:monorepos", "group:recommended" ], - "git-submodules": { - "enabled": true - }, "postUpdateOptions": ["pnpmDedupe"], "packageRules": [ { "matchDepNames": ["node", "@types/node"], "ignoreUnstable": false }, From 476d42e1b1039718acb9920dcda68c440c6e3543 Mon Sep 17 00:00:00 2001 From: Peter Hirn Date: Mon, 5 May 2025 17:38:49 +0200 Subject: [PATCH 4/4] wip: more element table hacking --- src/elementTable.test.ts | 21 +++++- src/elementTable.ts | 144 ++++++++++++++++++++++++++++++--------- 2 files changed, 128 insertions(+), 37 deletions(-) diff --git a/src/elementTable.test.ts b/src/elementTable.test.ts index d13949a..cb90aec 100644 --- a/src/elementTable.test.ts +++ b/src/elementTable.test.ts @@ -1,27 +1,42 @@ import { describe, expect, test } from "vitest"; import { elementTable } from "./elementTable.js"; +import { openPath } from "./node.js"; import { adskExampleFile, exampleFile } from "./node.test.js"; describe("element table", () => { - test("empty-2025", async () => { + test.skip("empty-2025", async () => { const file = await exampleFile("empty-2025.rvt"); const table = await elementTable(file); expect(table.size).toEqual(2413); }); - test("empty-2026", async () => { + test.skip("empty-2026", async () => { const file = await exampleFile("empty-2026.rvt"); const table = await elementTable(file); expect(table.size).toEqual(2426); }); - test("2026", async () => { + test.skip("2021", async () => { + const file = await adskExampleFile("racbasicsamplefamily-2021.rfa"); + const table = await elementTable(file); + + expect(table.size).toEqual(1814); + }); + + test.skip("2026", async () => { const file = await adskExampleFile("racbasicsamplefamily-2026.rfa"); const table = await elementTable(file); expect(table.size).toEqual(1992); }); + + test.skip("large-proprietary", async () => { + const file = await openPath("/home/peter/rdp/008_YYYY_MOD_F10_Z01-2021.rvt"); + const table = await elementTable(file); + + expect(table.size).toEqual(1992); + }); }); diff --git a/src/elementTable.ts b/src/elementTable.ts index c87d7c2..b780e4b 100644 --- a/src/elementTable.ts +++ b/src/elementTable.ts @@ -3,10 +3,11 @@ import * as array from "./utils/array.js"; export interface ElementEntry { id: bigint; - unknown1: number; - unknown2: number; - unknown3: number; - unknown4: bigint; + u1: number; + u2: number; + u3: number; + u4: bigint; + u5: number; } export interface ElementTable { @@ -25,13 +26,98 @@ const decompress = async (data: Uint8Array): Promise => { const decompressedStream = stream.pipeThrough(new DecompressionStream("gzip")); const chunks = []; - for await (const chunk of decompressedStream) { - chunks.push(chunk); + try { + for await (const chunk of decompressedStream) { + chunks.push(chunk); + } + } catch (e) { + if (e instanceof Error && (e.cause as { errno: number })?.errno === -3) { + console.warn("Expected invalid gzip data in ElemTable"); + } else { + throw e; + } } const blob = new Blob(chunks); const buffer = await blob.arrayBuffer(); - return new Uint8Array(buffer); + //return new Uint8Array(buffer); + const foo = new Uint8Array(buffer); + //console.log("ASDFASF", foo.length); + return foo; + /* + const stream = new DecompressionStream("gzip"); + try { + await stream.writable.getWriter().write(data); + } catch (e) { + console.log("ASDF", e); + } + + const result = await stream.readable.getReader().read(); + if (!result.value) throw Error("SDAF"); + + console.log("ASDFASF", result.value.length); + + return result.value; + */ + + //console.log(await stream.readable.getReader().read()); // "Hello World" +}; + +const elementIdSize = (flag: number): 32 | 64 => { + switch (flag) { + case 0x04: + return 32; + case 0x05: + return 64; + } + + throw Error(`Unexpected element table id size (${flag})`); +}; + +const tableEntry32 = (view: DataView, offset: number): ElementEntry => { + const id = view.getInt32(offset, true); + const id2 = view.getInt32(offset + 4, true); + const u1 = view.getInt32(offset + 8, true); + const u2 = view.getInt32(offset + 12, true); + const u3 = view.getInt32(offset + 16, true); + const u4 = view.getInt32(offset + 20, true); + const u5 = view.getInt32(offset + 24, true); + + // NOTE: found a large 2021 project where this throws + if (id !== id2) throw Error(`Id mismatch (${id} != ${id2})`); + // NOTE: found a large 2021 project where this throws + if (u4 !== 0) throw Error(`Unknown4 is not zero (${id}) ${u5}`); + + return { + id: BigInt(id), + u1, + u2, + u3, + u4: BigInt(u4), + u5 + }; +}; + +const tableEntry64 = (view: DataView, offset: number): ElementEntry => { + const id = view.getBigInt64(offset, true); + const u1 = view.getInt32(offset + 8, true); + const u2 = view.getInt32(offset + 12, true); + const u3 = view.getInt32(offset + 16, true); + const id2 = view.getBigInt64(offset + 20, true); + const u4 = view.getBigInt64(offset + 28, true); + const u5 = view.getInt32(offset + 36, true); + + if (id !== id2) throw Error(`Id mismatch (${id} != ${id2})`); + if (u5 !== 0) throw Error(`Unknown5 is not zero (${id})`); + + return { + id, + u1, + u2, + u3, + u4, + u5 + }; }; export const parseElementTable = async (data: Uint8Array): Promise => { @@ -42,8 +128,7 @@ export const parseElementTable = async (data: Uint8Array): Promise const decompressed = await decompress(data.subarray(8)); const fileVersion = decompressed[0]; - if (decompressed[1] !== 0x05) - throw Error(`Unexpected element table decompressed header [${decompressed[1]}]`); + const idSize = elementIdSize(decompressed[1]); const view = new DataView( decompressed.buffer, @@ -52,41 +137,31 @@ export const parseElementTable = async (data: Uint8Array): Promise ); const size = view.getInt32(2, true); + console.log("Size", size); + console.log("View", view.buffer.byteLength); + + const chunkSize = idSize === 32 ? 28 : 40; + console.log("chunks", (view.buffer.byteLength - 6) / chunkSize); const ids: ElementEntry[] = []; for (let i = 0; i < size; i++) { - const offset = i * 40 + 6; - const id = view.getBigInt64(offset, true); - const unknown1 = view.getInt32(offset + 8, true); - const unknown2 = view.getInt32(offset + 12, true); - const unknown3 = view.getInt32(offset + 16, true); - const id2 = view.getBigInt64(offset + 20, true); - const unknown4 = view.getBigInt64(offset + 28, true); - const unknown5 = view.getInt32(offset + 36, true); - - if (id !== id2) throw Error(`Id mismatch (${id} != ${id2})`); - if (unknown5 !== 0) throw Error(`Unknown5 is not zero (${id})`); - - ids.push({ - id, - unknown1, - unknown2, - unknown3, - unknown4 - }); + const offset = i * chunkSize + 6; + + const entry = idSize === 32 ? tableEntry32(view, offset) : tableEntry64(view, offset); + ids.push(entry); + //if (entry.id === BigInt(size)) break; + //throw Error("ASDFASFASDF"); } - console.log(size); + console.log(ids.slice(0, 200)); + + /* + console.log(ids.slice(0, 100).map((e) => e.id)); //const missing = [1521n, 1957n, 1967n, 1976n, 2283n]; const missing = [17n, 23n, 24n, 25n, 29n, 31n, 32n, 33n, 5702n, 5701n, 7199n, 7200n]; //console.log(ids.slice(0, 100).map((e) => e.id)); console.log(ids.filter((e) => e.id === 0n)); - /* - console.log(ids.filter((e) => e.id === 1n)); - console.log(ids.filter((e) => e.id === 1519n)); - console.log(ids.filter((e) => e.id === 1520n)); - */ console.log(ids.filter((e) => missing.includes(e.id))); console.log(ids.filter((e) => e.id > e.unknown4).length); @@ -109,6 +184,7 @@ export const parseElementTable = async (data: Uint8Array): Promise console.log(ids.filter((e) => e.id <= e.unknown4)); console.log(ids.find((e) => e.id === 1530n)); console.log(ids.find((e) => e.id === 1527n)); + */ //const foo = ids.filter((e) => e.id > e.unknown4); //writeFileSync("ids.txt", foo.map((e) => e.id).join(","));