whole new optimization system and universal translation layer that is significantly more performant than framework wrapping layers.

Author: magiccodingman

Magic.IndexedDb/LinqTranslation/Extensions/UniversalExpressionBuilder.cs

@@ -184,14 +184,40 @@ private FilterNode ParseNotExpression(UnaryExpression notExpr)
         {
             var inner = notExpr.Operand;
 
-            return inner switch
+            if (inner is BinaryExpression bin)
             {
-                BinaryExpression bin => HandleNotBinary(bin),
-                MethodCallExpression call => HandleNotMethod(call),
-                _ => throw new InvalidOperationException($"Unsupported NOT expression: {inner}")
-            };
+                if (bin.NodeType == ExpressionType.OrElse || bin.NodeType == ExpressionType.AndAlso)
+                {
+                    // De Morgan’s Law: !(A || B) => !A && !B,  !(A && B) => !A || !B
+                    var invertedOperator = bin.NodeType == ExpressionType.OrElse
+                        ? FilterLogicalOperator.And
+                        : FilterLogicalOperator.Or;
+
+                    return new FilterNode
+                    {
+                        NodeType = FilterNodeType.Logical,
+                        Operator = invertedOperator,
+                        Children = new List<FilterNode>
+                {
+                    ParseNotExpression(Expression.Not(bin.Left)),
+                    ParseNotExpression(Expression.Not(bin.Right))
+                }
+                    };
+                }
+                else
+                {
+                    return HandleNotBinary(bin);
+                }
+            }
+            else if (inner is MethodCallExpression call)
+            {
+                return HandleNotMethod(call);
+            }
+
+            throw new InvalidOperationException($"Unsupported NOT expression: {inner}");
         }
 
+
         private FilterNode HandleNotBinary(BinaryExpression bin)
         {
             string invertedOp = bin.NodeType switch

Magic.IndexedDb/wwwroot/magicLinqToIndexedDb.js

@@ -33,17 +33,17 @@ export async function* magicQueryYield(db, table, universalSerializedPredicate,
         throw new Error("A valid Dexie table instance must be provided.");
     }
 
-    console.log('universal serialized predicate');
-    console.log(universalSerializedPredicate);
+    debugLog('universal serialized predicate');
+    debugLog(universalSerializedPredicate);
     const { nestedOrFilterUnclean, isUniversalTrue, isUniversalFalse } = flattenUniversalPredicate(universalSerializedPredicate);
 
     if (isUniversalFalse === true) {
         debugLog("Universal False, sending back no data");
         return;
     }
 
-    console.log('flattened serialized predicate');
-    console.log(nestedOrFilterUnclean);
+    debugLog('flattened serialized predicate');
+    debugLog(nestedOrFilterUnclean);
 
     debugLog("Starting where function", { nestedOrFilterUnclean, queryAdditions });
 
@@ -69,6 +69,9 @@ export async function* magicQueryYield(db, table, universalSerializedPredicate,
         return;
     }
 
+    debugLog('Flattened or groups');
+    debugLog(nestedOrFilter);
+
     let { indexedQueries, compoundIndexQueries, cursorConditions } =
         partitionQueryConditions(nestedOrFilter, queryAdditions, indexCache, forceCursor);
 

Magic.IndexedDb/wwwroot/utilities/FlattenFilterNode.js

@@ -4,7 +4,10 @@ import {
     resolveNodeType,
     resolveLogicalOperator
 } from "./predicateEnumHelpers.js";
-
+import { QUERY_OPERATIONS } from "./queryConstants.js";
+import { areConditionsCompatible } from "./areConditionsCompatible.js";
+import { advancedOptimizeNestedOrFilter } from "./optimizeConditions.js";
+import { optimizeOrGroupStructure } from "./optimizeOrGroupStructure.js";
 /**
  * Flattens a Universal Predicate Tree into a nestedOrFilter structure.
  * This creates `orGroups -> andGroups -> conditions[]`
@@ -34,9 +37,10 @@ export function flattenUniversalPredicate(rootNode) {
         ]
     }));
 
+    let orGroupsOptimized = optimizeOrGroupStructure(advancedOptimizeNestedOrFilter(orGroups));
     const result = {
         nestedOrFilterUnclean: {
-            orGroups
+            orGroups: orGroupsOptimized
         },
         isUniversalTrue: detection.isUniversalTrue,
         isUniversalFalse: detection.isUniversalFalse
@@ -48,60 +52,54 @@ export function flattenUniversalPredicate(rootNode) {
 
 function flattenNodeToAndGroups(node) {
     const nodeType = resolveNodeType(node.nodeType);
-
-    if (!nodeType) {
-        debugLog("Skipping node with null/invalid nodeType", node);
-        return [];
-    }
+    if (!nodeType) return [];
 
     if (nodeType === "Condition") {
-        if (!node.condition) {
-            debugLog("Skipping condition node with null condition", node);
-            return [];
-        }
-        return [[node.condition]];
+        if (!node.condition) return [];
+        const normalized = normalizeCondition(node.condition);
+        return [[normalized]];
     }
 
     if (nodeType === "Logical") {
         const operator = resolveLogicalOperator(node.operator);
         const { children } = node;
 
-        if (!operator || !Array.isArray(children)) {
-            debugLog("Skipping logical node with null operator or children", node);
-            return [];
-        }
+        if (!operator || !Array.isArray(children)) return [];
 
         if (operator === "And") {
             let result = [[]];
+
             for (const child of children) {
-                const childFlattened = flattenNodeToAndGroups(child);
+                const childGroups = flattenNodeToAndGroups(child);
                 const newResult = [];
 
-                for (const resGroup of result) {
-                    for (const childGroup of childFlattened) {
-                        newResult.push([...resGroup, ...childGroup]);
+                for (const groupA of result) {
+                    for (const groupB of childGroups) {
+                        if (areConditionsCompatible(groupA, groupB)) {
+                            newResult.push([...groupA, ...groupB]);
+                        } else {
+                            debugLog("Skipping incompatible group merge", { groupA, groupB });
+                        }
                     }
                 }
 
                 result = newResult;
             }
+
             return result;
         }
 
         if (operator === "Or") {
             let result = [];
             for (const child of children) {
-                const childFlattened = flattenNodeToAndGroups(child);
-                result = result.concat(childFlattened);
+                result = result.concat(flattenNodeToAndGroups(child));
             }
             return result;
         }
 
-        debugLog("Unknown operator encountered during flattening", operator);
         return [];
     }
 
-    debugLog("Unknown node type encountered during flattening", nodeType);
     return [];
 }
 
@@ -126,3 +124,27 @@ function detectUniversalTruth(node) {
         isUniversalFalse: value === false
     };
 }
+
+
+/**
+ * Normalizes a condition before execution.
+ * Handles special logic like null-equality conversion, case handling, etc.
+ */
+function normalizeCondition(condition) {
+    const normalized = { ...condition };
+
+    if (
+        (condition.operation === QUERY_OPERATIONS.EQUAL || condition.operation === QUERY_OPERATIONS.NOT_EQUAL) &&
+        (condition.value === null || condition.value === undefined)
+    ) {
+        normalized.operation = condition.operation === QUERY_OPERATIONS.EQUAL
+            ? QUERY_OPERATIONS.IS_NULL
+            : QUERY_OPERATIONS.IS_NOT_NULL;
+
+        // Leave value in place to pass validation
+        normalized.value = null;
+    }
+
+    return normalized;
+}
+