significantly re-optimized cursor predicate layer

Author: magiccodingman

Magic.IndexedDb/wwwroot/utilities/cursorEngine.js

@@ -53,21 +53,33 @@ async function processCursorRecords(db, table, predicateTree, yieldedPrimaryKeys
     const now = Date.now();
     let shouldLogWarning = !lastCursorWarningTime || now - lastCursorWarningTime > 10 * 60 * 1000;
 
-    // Dynamically extract all required properties from the predicate tree
     const requiredPropertiesFiltered = new Set();
-    collectPropertiesFromTree(predicateTree, requiredPropertiesFiltered);
+
+    // Only collect properties if we actually have a filter tree with children
+    const hasConditions =
+        predicateTree &&
+        (
+            predicateTree.nodeType === "condition" ||
+            (predicateTree.children && predicateTree.children.length > 0)
+        );
+
+    if (hasConditions) {
+        collectPropertiesFromTree(predicateTree, requiredPropertiesFiltered);
+    }
 
     await db.transaction('r', table, async () => {
         await table.orderBy(compoundKeys[0]).each((record) => {
-            // Skip if required property is missing
-            for (const prop of requiredPropertiesFiltered) {
-                if (record[prop] === undefined) {
-                    if (shouldLogWarning) {
-                        console.warn(`[IndexedDB Cursor Warning] Skipping record due to missing property: ${prop}`);
-                        lastCursorWarningTime = now;
-                        shouldLogWarning = false;
+            // Still apply property checks *only* if we have any to check
+            if (requiredPropertiesFiltered.size > 0) {
+                for (const prop of requiredPropertiesFiltered) {
+                    if (record[prop] === undefined) {
+                        if (shouldLogWarning) {
+                            console.warn(`[IndexedDB Cursor Warning] Skipping record due to missing property: ${prop}`);
+                            lastCursorWarningTime = now;
+                            shouldLogWarning = false;
+                        }
+                        return;
                     }
-                    return;
                 }
             }
 
@@ -76,48 +88,77 @@ async function processCursorRecords(db, table, predicateTree, yieldedPrimaryKeys
                 return;
             }
 
-            // Evaluate the structured predicate tree
-            if (!evaluatePredicateTree(predicateTree, record)) return;
+            // Only evaluate if we actually have predicate logic
+            if (hasConditions && !evaluatePredicateTree(predicateTree, record))
+                return;
 
             recordHandler(record, recordKey);
         });
     });
 }
 
+
 function collectPropertiesFromTree(node, propertySet) {
     if (node.nodeType === "condition") {
         propertySet.add(node.condition.property);
         return;
     }
-    for (const child of node.children) {
+    for (const child of node.children ?? []) {
         collectPropertiesFromTree(child, propertySet);
     }
 }
 
 function evaluatePredicateTree(node, record) {
     if (node.nodeType === "condition") {
-        const condition = optimizeSingleCondition(node.condition);
-        return applyCondition(record, condition);
+        if (!node.optimized) {
+            node.optimized = optimizeSingleCondition(node.condition);
+        }
+        return applyCondition(record, node.optimized);
     }
 
-    const results = node.children.map(child => evaluatePredicateTree(child, record));
+    const results = (node.children ?? []).map(child => evaluatePredicateTree(child, record));
     return node.operator === "And"
         ? results.every(r => r)
         : results.some(r => r);
 }
 
 function optimizeSingleCondition(condition) {
-    let optimized = { ...condition };
+    if (condition.value === -Infinity || condition.value === Infinity) {
+        return condition; // Already a no-op filter, skip transformation
+    }
 
+    const optimized = { ...condition };
+
+    // Lowercase normalization for string values if not case-sensitive
     if (!condition.caseSensitive && typeof condition.value === "string") {
         optimized.value = condition.value.toLowerCase();
     }
 
+    // Edge case: GETDAY sanity check
+    if (
+        condition.operation === QUERY_OPERATIONS.GETDAY &&
+        typeof condition.value !== "number"
+    ) {
+        console.warn(`[IndexedDB Warning] GETDAY expects a numeric day (1–31). Got:`, condition.value);
+    }
+
+    // Generic numeric validation for date-based derived operations
+    if (
+        [QUERY_OPERATIONS.GET_DAY_OF_WEEK, QUERY_OPERATIONS.GET_DAY_OF_YEAR, QUERY_OPERATIONS.GETDAY]
+            .includes(condition.operation)
+    ) {
+        if (typeof condition.value !== "number") {
+            console.warn(`[IndexedDB Warning] ${condition.operation} expects a numeric value. Got:`, condition.value);
+        }
+    }
+
     optimized.comparisonFunction = getComparisonFunction(condition.operation);
     return optimized;
 }
 
 
+
+
 /**
  * Directly retrieves records that match the conditions without metadata processing.
  */
@@ -158,12 +199,14 @@ async function runMetaDataCursorQuery(db, table, conditions, queryAdditions, yie
     let requiredProperties = new Set();
     let magicOrder = 0;
 
-    for (const andGroup of conditions) {
-        for (const condition of andGroup) {
-            if (condition.property) requiredProperties.add(condition.property);
-        }
+    if (conditions?.nodeType === "logical" && !conditions.children) {
+        // No conditions — grab everything
+        debugLog("Detected no-op predicate. All records will be evaluated.");
+    } else {
+        collectPropertiesFromTree(conditions, requiredProperties);
     }
 
+
     for (const addition of queryAdditions) {
         if ((addition.additionFunction === QUERY_ADDITIONS.ORDER_BY
             || addition.additionFunction === QUERY_ADDITIONS.ORDER_BY_DESCENDING) &&
@@ -209,33 +252,6 @@ async function runMetaDataCursorQuery(db, table, conditions, queryAdditions, yie
     return primaryKeyList.slice(0, resultIndex);
 }
 
-
-function optimizeConditions(conditions) {
-    return conditions.map(condition => {
-        let optimizedCondition = { ...condition };
-
-        if (!condition.caseSensitive && typeof condition.value === "string") {
-            optimizedCondition.value = condition.value.toLowerCase();
-        }
-
-        // Edge case: GETDAY sanity check
-        if (condition.operation === QUERY_OPERATIONS.GETDAY &&
-            typeof condition.value !== "number") {
-            console.warn(`[IndexedDB Warning] GETDAY expects a numeric day (1–31). Got:`, condition.value);
-        }
-
-        if ([QUERY_OPERATIONS.GET_DAY_OF_WEEK, QUERY_OPERATIONS.GET_DAY_OF_YEAR, QUERY_OPERATIONS.GETDAY].includes(condition.operation)) {
-            if (typeof condition.value !== "number") {
-                console.warn(`[IndexedDB Warning] ${condition.operation} expects a numeric value. Got:`, condition.value);
-            }
-        }
-
-        optimizedCondition.comparisonFunction = getComparisonFunction(condition.operation);
-        return optimizedCondition;
-    });
-}
-
-
 function getComparisonFunction(operation) {
     const operations = {
         [QUERY_OPERATIONS.EQUAL]: (recordValue, queryValue) => recordValue === queryValue,

Magic.IndexedDb/wwwroot/utilities/rebuildNestedPredicate.js

@@ -1,69 +1,131 @@
 export function rebuildCursorConditionsToPredicateTree(flattened) {
-    // Step 1: Normalize to OR of ANDs
-    const orRoot = {
-        nodeType: "logical",
-        operator: "Or",
-        children: flattened.map(andArray => ({
+    const orGroups = [];
+
+    for (const andSet of flattened) {
+        const filteredConditions = andSet.filter(cond => !isDummyCondition(cond));
+        if (filteredConditions.length === 0) continue; // Skip this group if all conditions were dummy
+
+        const andGroup = {
             nodeType: "logical",
             operator: "And",
-            children: andArray.map(cond => ({
+            children: filteredConditions.map(condition => ({
                 nodeType: "condition",
                 condition: {
-                    property: cond.property,
-                    operation: cond.operation,
-                    value: cond.value,
-                    isString: cond.isString ?? false,
-                    caseSensitive: cond.caseSensitive ?? false
+                    property: condition.property,
+                    operation: condition.operation,
+                    value: condition.value,
+                    isString: condition.isString ?? false,
+                    caseSensitive: condition.caseSensitive ?? false
                 }
             }))
-        }))
-    };
+        };
 
-    // Step 2: Simplify redundant ANDs if possible
-    const simplified = fixpointSimplify(orRoot);
+        orGroups.push(andGroup);
+    }
+
+    // Handle the case where ALL conditions were dummy: this means "match everything"
+    if (orGroups.length === 0) {
+        return fixpointSimplify({
+            nodeType: "logical",
+            operator: "Or",
+            children: []
+        });
+    }
 
-    return simplified;
+    const collapsed = collapseOrGroups(orGroups);
+
+    return fixpointSimplify({
+        nodeType: "logical",
+        operator: "Or",
+        children: collapsed
+    });
 }
 
+function collapseOrGroups(orChildren) {
+    const buckets = new Map();
+    const results = [];
+
+    for (const node of orChildren) {
+        if (node.operator !== "And") {
+            results.push(node);
+            continue;
+        }
 
-function extractDynamicIntentGroups(branches) {
-    const conditionMap = new Map();
+        const simpleKeys = [];
+        const complexNodes = [];
 
-    for (const group of branches) {
-        for (const conditionNode of group) {
-            const { property, operation, value } = conditionNode.condition;
-            const key = `${property}||${operation}`;
-            if (!conditionMap.has(key)) conditionMap.set(key, new Set());
-            conditionMap.get(key).add(JSON.stringify(conditionNode.condition));
+        for (const child of node.children) {
+            if (child.nodeType === "condition") {
+                const key = `${child.condition.property}||${child.condition.operation}`;
+                simpleKeys.push({ key, node: child });
+            } else {
+                complexNodes.push(child);
+            }
         }
-    }
 
-    // Build OR groups per property+operation
-    const groupedByProperty = new Map();
+        const groupKey = simpleKeys.map(s => s.key).sort().join("&&");
 
-    for (const [key, valueSet] of conditionMap.entries()) {
-        const [property, op] = key.split("||");
-        if (!groupedByProperty.has(property)) groupedByProperty.set(property, []);
-        groupedByProperty.get(property).push({
-            operation: op,
-            values: [...valueSet].map(json => ({
-                nodeType: "condition",
-                condition: JSON.parse(json)
-            }))
-        });
+        if (!buckets.has(groupKey)) {
+            buckets.set(groupKey, []);
+        }
+
+        buckets.get(groupKey).push(node);
     }
 
-    const resultGroups = [];
-    for (const [property, ops] of groupedByProperty.entries()) {
-        const propertyGroup = {
+    // Build optimized OR sets per bucket
+    for (const group of buckets.values()) {
+        if (group.length === 1) {
+            results.push(group[0]);
+            continue;
+        }
+
+        // Find which conditions are shared across all ANDs
+        const conditionMatrix = group.map(g => g.children);
+        const transposed = transpose(conditionMatrix);
+
+        const collapsedAnd = {
             nodeType: "logical",
-            operator: "Or",
-            children: ops.flatMap(o => o.values)
+            operator: "And",
+            children: []
         };
-        resultGroups.push(propertyGroup);
+
+        for (const col of transposed) {
+            // If all columns refer to the same property/operator but different values
+            const base = col[0];
+            const allSamePO = col.every(c =>
+                c.nodeType === "condition" &&
+                c.condition.property === base.condition.property &&
+                c.condition.operation === base.condition.operation
+            );
+
+            if (allSamePO) {
+                // Build OR group of all values
+                const orGroup = {
+                    nodeType: "logical",
+                    operator: "Or",
+                    children: col
+                };
+                collapsedAnd.children.push(orGroup);
+            } else {
+                // They differ structurally, preserve individually
+                collapsedAnd.children.push(...col);
+            }
+        }
+
+        results.push(collapsedAnd);
     }
 
-    return resultGroups;
+    return results;
+}
+
+
+function transpose(matrix) {
+    if (!matrix.length) return [];
+    const len = Math.max(...matrix.map(row => row.length));
+    const transposed = Array.from({ length: len }, (_, i) =>
+        matrix.map(row => row[i]).filter(Boolean)
+    );
+    return transposed;
 }
 
 function fixpointSimplify(node) {
@@ -89,6 +151,7 @@ function simplify(node) {
             flattened.push(child);
         }
     }
+
     node.children = flattened;
 
     const seen = new Set();
@@ -104,4 +167,15 @@ function simplify(node) {
     }
 
     return node;
-}
+}
+
+
+function isDummyCondition(cond) {
+    if (typeof cond.value === "number") {
+        if ((cond.value === Infinity || cond.value === -Infinity) &&
+            ["GreaterThanOrEqual", "LessThanOrEqual", "GreaterThan", "LessThan"].includes(cond.operation)) {
+            return true;
+        }
+    }
+    return false;
+}