Lazy source transforms for DataTable

src/lib/data-table/combine.ts

@@ -1,4 +1,7 @@
 import { Column, DataTable, TypedArray } from './data-table';
+import { convertToSpace } from './transform';
+import { logger } from '../utils/logger';
+import { Transform } from '../utils/math';
 
 /**
  * Combines multiple DataTables into a single DataTable.
@@ -7,6 +10,9 @@ import { Column, DataTable, TypedArray } from './data-table';
  * columns that don't exist in all tables will have undefined values for
  * rows from tables lacking that column.
  *
+ * If tables have differing source transforms, all data is first converted
+ * to engine coordinate space (identity transform) before combining.
+ *
  * @param dataTables - Array of DataTables to combine.
  * @returns A new DataTable containing all rows from all input tables.
  *
@@ -18,10 +24,22 @@ import { Column, DataTable, TypedArray } from './data-table';
  */
 const combine = (dataTables: DataTable[]) : DataTable => {
     if (dataTables.length === 1) {
-        // nothing to combine
         return dataTables[0];
     }
 
+    // Check if all transforms match the first table's transform
+    const refTransform = dataTables[0].transform;
+    const allMatch = dataTables.every(dt => dt.transform.equals(refTransform));
+
+    let tables = dataTables;
+    let resultTransform = refTransform;
+
+    if (!allMatch) {
+        logger.warn('Combining DataTables with different source transforms; converting to engine space.');
+        tables = dataTables.map(dt => convertToSpace(dt, Transform.IDENTITY));
+        resultTransform = Transform.IDENTITY;
+    }
+
     const findMatchingColumn = (columns: Column[], column: Column) => {
         for (let i = 0; i < columns.length; ++i) {
             if (columns[i].name === column.name &&
@@ -32,10 +50,10 @@ const combine = (dataTables: DataTable[]) : DataTable => {
         return null;
     };
 
-    // make unique list of columns where name and type much match
-    const columns = dataTables[0].columns.slice();
-    for (let i = 1; i < dataTables.length; ++i) {
-        const dataTable = dataTables[i];
+    // make unique list of columns where name and type must match
+    const columns = tables[0].columns.slice();
+    for (let i = 1; i < tables.length; ++i) {
+        const dataTable = tables[i];
         for (let j = 0; j < dataTable.columns.length; ++j) {
             if (!findMatchingColumn(columns, dataTable.columns[j])) {
                 columns.push(dataTable.columns[j]);
@@ -44,19 +62,19 @@ const combine = (dataTables: DataTable[]) : DataTable => {
     }
 
     // count total number of rows
-    const totalRows = dataTables.reduce((sum, dataTable) => sum + dataTable.numRows, 0);
+    const totalRows = tables.reduce((sum, dataTable) => sum + dataTable.numRows, 0);
 
     // construct output dataTable
     const resultColumns = columns.map((column) => {
         const constructor = column.data.constructor as new (length: number) => TypedArray;
         return new Column(column.name, new constructor(totalRows));
     });
-    const result = new DataTable(resultColumns);
+    const result = new DataTable(resultColumns, resultTransform);
 
     // copy data
     let rowOffset = 0;
-    for (let i = 0; i < dataTables.length; ++i) {
-        const dataTable = dataTables[i];
+    for (let i = 0; i < tables.length; ++i) {
+        const dataTable = tables[i];
 
         for (let j = 0; j < dataTable.columns.length; ++j) {
             const column = dataTable.columns[j];

src/lib/data-table/data-table.ts

@@ -1,3 +1,5 @@
+import { Transform } from '../utils/math';
+
 /**
  * Union of all typed array types supported for column data.
  */
@@ -84,8 +86,9 @@ type Row = {
  */
 class DataTable {
     columns: Column[];
+    transform: Transform;
 
-    constructor(columns: Column[]) {
+    constructor(columns: Column[], transform?: Transform) {
         if (columns.length === 0) {
             throw new Error('DataTable must have at least one column');
         }
@@ -98,6 +101,7 @@ class DataTable {
         }
 
         this.columns = columns;
+        this.transform = transform ? transform.clone() : new Transform();
     }
 
     // rows
@@ -215,13 +219,13 @@ class DataTable {
         }
 
         if (!rows) {
-            return new DataTable(srcColumns.map(c => c.clone()));
+            return new DataTable(srcColumns.map(c => c.clone()), this.transform);
         }
 
         const result = new DataTable(srcColumns.map((c) => {
             const constructor = c.data.constructor as new (length: number) => TypedArray;
             return new Column(c.name, new constructor(rows.length));
-        }));
+        }), this.transform);
 
         for (let i = 0; i < result.numColumns; ++i) {
             const src = srcColumns[i].data;

src/lib/data-table/decimate.ts

@@ -777,7 +777,7 @@ const simplifyGaussians = (dataTable: DataTable, targetCount: number): DataTable
             const c = cols[ci];
             newColumns.push(new Column(c.name, new (c.data.constructor as any)(outCount)));
         }
-        const newTable = new DataTable(newColumns);
+        const newTable = new DataTable(newColumns, dataTable.transform);
 
         // Copy unmerged splats
         let dst = 0;

src/lib/data-table/transform.ts

@@ -1,85 +1,209 @@
-import { Mat3, Mat4, Quat, Vec3 } from 'playcanvas';
+import { Mat3, Quat, Vec3 } from 'playcanvas';
 
-import { DataTable } from './data-table';
+import { Column, DataTable, TypedArray } from './data-table';
+import { Transform } from '../utils/math';
 import { RotateSH } from '../utils/rotate-sh';
 
 const shNames = new Array(45).fill('').map((_, i) => `f_rest_${i}`);
 
-const v = new Vec3();
-const q = new Quat();
+const _v = new Vec3();
+const _q = new Quat();
+
+// -- Helpers for on-demand column generation --
 
 /**
- * Applies a spatial transformation to splat data in-place.
- *
- * Transforms position, rotation, scale, and spherical harmonics data.
- * The transformation is applied as: scale first, then rotation, then translation.
- *
- * @param dataTable - The DataTable to transform (modified in-place).
- * @param t - Translation vector.
- * @param r - Rotation quaternion.
- * @param s - Uniform scale factor.
+ * Computes the delta transform needed to convert raw data from its current
+ * coordinate system into the output format's coordinate system.
  *
- * @example
- * ```ts
- * import { Vec3, Quat } from 'playcanvas';
+ * @param transform - The DataTable's current source transform.
+ * @param outputFormatTransform - The output format's expected transform.
+ * @returns The delta transform to apply to raw data, or null if it is identity.
+ */
+const computeWriteTransform = (transform: Transform, outputFormatTransform: Transform): Transform | null => {
+    const delta = outputFormatTransform.clone().invert().mul(transform);
+    return delta.isIdentity() ? null : delta;
+};
+
+/**
+ * Detects how many SH bands (0-3) the DataTable has.
+ * @ignore
+ */
+const detectSHBands = (dataTable: DataTable): number => {
+    return ({ '9': 1, '24': 2, '-1': 3 } as Record<string, number>)[String(shNames.findIndex(n => !dataTable.hasColumn(n)))] ?? 0;
+};
+
+/**
+ * Generates transformed typed arrays for requested columns, applying the given
+ * transform. Columns unaffected by the transform return references to the
+ * original arrays (zero copy).
  *
- * // Scale by 2x, rotate 90° around Y, translate up
- * transform(dataTable, new Vec3(0, 5, 0), new Quat().setFromEulerAngles(0, 90, 0), 2.0);
- * ```
+ * @param dataTable - The source DataTable.
+ * @param columnNames - Which columns to produce.
+ * @param delta - The transform to apply. If identity or null, original arrays are returned.
+ * @returns A map of column name to typed array.
  */
-const transform = (dataTable: DataTable, t: Vec3, r: Quat, s: number): void => {
-    const mat = new Mat4().setTRS(t, r, new Vec3(s, s, s));
-    const mat3 = new Mat3().setFromQuat(r);
-    const rotateSH = new RotateSH(mat3);
-
-    const hasTranslation = ['x', 'y', 'z'].every(c => dataTable.hasColumn(c));
-    const hasRotation = ['rot_0', 'rot_1', 'rot_2', 'rot_3'].every(c => dataTable.hasColumn(c));
-    const hasScale = ['scale_0', 'scale_1', 'scale_2'].every(c => dataTable.hasColumn(c));
-    const shBands = { '9': 1, '24': 2, '-1': 3 }[shNames.findIndex(v => !dataTable.hasColumn(v))] ?? 0;
-    const shCoeffs = new Float32Array([0, 3, 8, 15][shBands]);
-
-    const row: any = {};
-    for (let i = 0; i < dataTable.numRows; ++i) {
-        dataTable.getRow(i, row);
-
-        if (hasTranslation) {
-            v.set(row.x, row.y, row.z);
-            mat.transformPoint(v, v);
-            row.x = v.x;
-            row.y = v.y;
-            row.z = v.z;
+const transformColumns = (dataTable: DataTable, columnNames: string[], delta: Transform | null): Map<string, TypedArray> => {
+    const result = new Map<string, TypedArray>();
+
+    if (!delta || delta.isIdentity()) {
+        for (const name of columnNames) {
+            const col = dataTable.getColumnByName(name);
+            if (col) result.set(name, col.data);
+        }
+        return result;
+    }
+
+    const numRows = dataTable.numRows;
+    const r = delta.rotation;
+    const s = delta.scale;
+
+    // Categorize requested columns
+    const posNames = ['x', 'y', 'z'];
+    const rotNames = ['rot_0', 'rot_1', 'rot_2', 'rot_3'];
+    const scaleNames = ['scale_0', 'scale_1', 'scale_2'];
+
+    const hasPos = posNames.every(n => dataTable.hasColumn(n));
+    const needPos = hasPos && posNames.some(n => columnNames.includes(n));
+    const needRot = rotNames.every(n => columnNames.includes(n) && dataTable.hasColumn(n));
+    const needScale = scaleNames.some(n => columnNames.includes(n) && dataTable.hasColumn(n)) && s !== 1;
+
+    const shBands = detectSHBands(dataTable);
+    const shCoeffsPerChannel = [0, 3, 8, 15][shBands];
+    const rotIsIdentity = Math.abs(Math.abs(r.w) - 1) < 1e-6;
+    const requestedSH = shBands > 0 && !rotIsIdentity && shNames.slice(0, shCoeffsPerChannel * 3).some(n => columnNames.includes(n));
+
+    // Position columns
+    if (needPos) {
+        const srcX = dataTable.getColumnByName('x')!.data;
+        const srcY = dataTable.getColumnByName('y')!.data;
+        const srcZ = dataTable.getColumnByName('z')!.data;
+        const dstX = new Float32Array(numRows);
+        const dstY = new Float32Array(numRows);
+        const dstZ = new Float32Array(numRows);
+
+        for (let i = 0; i < numRows; ++i) {
+            _v.set(srcX[i], srcY[i], srcZ[i]);
+            delta.transformPoint(_v, _v);
+            dstX[i] = _v.x;
+            dstY[i] = _v.y;
+            dstZ[i] = _v.z;
+        }
+
+        if (columnNames.includes('x')) result.set('x', dstX);
+        if (columnNames.includes('y')) result.set('y', dstY);
+        if (columnNames.includes('z')) result.set('z', dstZ);
+    }
+
+    // Rotation columns
+    if (needRot) {
+        const src0 = dataTable.getColumnByName('rot_0')!.data;
+        const src1 = dataTable.getColumnByName('rot_1')!.data;
+        const src2 = dataTable.getColumnByName('rot_2')!.data;
+        const src3 = dataTable.getColumnByName('rot_3')!.data;
+        const dst0 = new Float32Array(numRows);
+        const dst1 = new Float32Array(numRows);
+        const dst2 = new Float32Array(numRows);
+        const dst3 = new Float32Array(numRows);
+
+        for (let i = 0; i < numRows; ++i) {
+            _q.set(src1[i], src2[i], src3[i], src0[i]).mul2(r, _q);
+            dst0[i] = _q.w;
+            dst1[i] = _q.x;
+            dst2[i] = _q.y;
+            dst3[i] = _q.z;
         }
 
-        if (hasRotation) {
-            q.set(row.rot_1, row.rot_2, row.rot_3, row.rot_0).mul2(r, q);
-            row.rot_0 = q.w;
-            row.rot_1 = q.x;
-            row.rot_2 = q.y;
-            row.rot_3 = q.z;
+        result.set('rot_0', dst0);
+        result.set('rot_1', dst1);
+        result.set('rot_2', dst2);
+        result.set('rot_3', dst3);
+    }
+
+    // Scale columns (only affected when uniform scale != 1)
+    if (needScale) {
+        const logS = Math.log(s);
+        for (const name of scaleNames) {
+            if (!columnNames.includes(name) || !dataTable.hasColumn(name)) continue;
+            const src = dataTable.getColumnByName(name)!.data;
+            const dst = new Float32Array(numRows);
+            for (let i = 0; i < numRows; ++i) {
+                dst[i] = src[i] + logS;
+            }
+            result.set(name, dst);
         }
+    }
+
+    // SH columns
+    if (requestedSH) {
+        const mat3 = new Mat3().setFromQuat(r);
+        const rotateSH = new RotateSH(mat3);
+        const shCoeffs = new Float32Array(shCoeffsPerChannel);
 
-        if (hasScale && s !== 1) {
-            row.scale_0 = Math.log(Math.exp(row.scale_0) * s);
-            row.scale_1 = Math.log(Math.exp(row.scale_1) * s);
-            row.scale_2 = Math.log(Math.exp(row.scale_2) * s);
+        const shSrc: Float32Array[][] = [];
+        const shDst: Float32Array[][] = [];
+        for (let j = 0; j < 3; ++j) {
+            const src: Float32Array[] = [];
+            const dst: Float32Array[] = [];
+            for (let k = 0; k < shCoeffsPerChannel; ++k) {
+                const name = shNames[k + j * shCoeffsPerChannel];
+                src.push(dataTable.getColumnByName(name)!.data as Float32Array);
+                dst.push(new Float32Array(numRows));
+            }
+            shSrc.push(src);
+            shDst.push(dst);
         }
 
-        if (shBands > 0) {
+        for (let i = 0; i < numRows; ++i) {
             for (let j = 0; j < 3; ++j) {
-                for (let k = 0; k < shCoeffs.length; ++k) {
-                    shCoeffs[k] = row[shNames[k + j * shCoeffs.length]];
+                for (let k = 0; k < shCoeffsPerChannel; ++k) {
+                    shCoeffs[k] = shSrc[j][k][i];
                 }
-
                 rotateSH.apply(shCoeffs);
+                for (let k = 0; k < shCoeffsPerChannel; ++k) {
+                    shDst[j][k][i] = shCoeffs[k];
+                }
+            }
+        }
 
-                for (let k = 0; k < shCoeffs.length; ++k) {
-                    row[shNames[k + j * shCoeffs.length]] = shCoeffs[k];
+        for (let j = 0; j < 3; ++j) {
+            for (let k = 0; k < shCoeffsPerChannel; ++k) {
+                const name = shNames[k + j * shCoeffsPerChannel];
+                if (columnNames.includes(name)) {
+                    result.set(name, shDst[j][k]);
                 }
             }
         }
+    }
 
-        dataTable.setRow(i, row);
+    // All remaining requested columns: return original array references
+    for (const name of columnNames) {
+        if (!result.has(name)) {
+            const col = dataTable.getColumnByName(name);
+            if (col) result.set(name, col.data);
+        }
     }
+
+    return result;
 };
 
-export { transform };
+/**
+ * Returns a new DataTable with column data converted to the target coordinate
+ * space. If the DataTable is already in that space, returns it unchanged.
+ *
+ * @param dataTable - The source DataTable.
+ * @param targetTransform - The desired coordinate-space transform.
+ * @returns A DataTable whose raw data is in the target coordinate space.
+ */
+const convertToSpace = (dataTable: DataTable, targetTransform: Transform): DataTable => {
+    const delta = computeWriteTransform(dataTable.transform, targetTransform);
+    if (!delta) return dataTable;
+    const allNames = dataTable.columnNames;
+    const cols = transformColumns(dataTable, allNames, delta);
+    return new DataTable(allNames.map(name => new Column(name, cols.get(name)!)), targetTransform);
+};
+
+export {
+    transformColumns,
+    computeWriteTransform,
+    convertToSpace
+};