patching cursor engine meta data information grabs for better ordering. Also added StableOrdering() functionality.

Author: magiccodingman

Magic.IndexedDb/LinqTranslation/Interfaces/IMagicCursor.cs

@@ -31,4 +31,13 @@ public interface IMagicCursor<T> : IMagicExecute<T> where T : class
     /// <param name="predicate"></param>
     /// <returns></returns>
     IMagicCursorStage<T> OrderByDescending(Expression<Func<T, object>> predicate);
+
+    /// <summary>
+    /// Removes ordering of any indexed columns. Only orders by 
+    /// what you dictate then by the row insert order. This creates 
+    /// a much more predictable and stable ordering. Only needs 
+    /// to be applied once. 
+    /// </summary>
+    /// <returns></returns>
+    IMagicCursorStage<T> StableOrdering();
 }

Magic.IndexedDb/LinqTranslation/Models/MagicCursor.cs

@@ -1,4 +1,5 @@
 using Magic.IndexedDb.LinqTranslation.Extensions;
+using Magic.IndexedDb.Models;
 using System.Linq.Expressions;
 using System.Runtime.CompilerServices;
 
@@ -35,6 +36,15 @@ public IMagicCursorStage<T> OrderBy(Expression<Func<T, object>> predicate)
     public IMagicCursorStage<T> OrderByDescending(Expression<Func<T, object>> predicate)
         => new MagicCursorExtension<T>(MagicQuery).OrderByDescending(predicate);
 
+    public IMagicCursorStage<T> StableOrdering()
+    {
+        var _MagicQuery = new MagicQuery<T>(this.MagicQuery);
+        StoredMagicQuery smq = new StoredMagicQuery();
+        smq.additionFunction = MagicQueryFunctions.StableOrdering;
+        _MagicQuery.StoredMagicQueries.Add(smq);
+        return new MagicCursorExtension<T>(_MagicQuery);
+    }
+
     public async IAsyncEnumerable<T> AsAsyncEnumerable(
         [EnumeratorCancellation] CancellationToken cancellationToken = default)
     {

Magic.IndexedDb/LinqTranslation/Models/MagicQuery.cs

@@ -40,6 +40,7 @@ public MagicQuery(MagicQuery<T> _MagicQuery)
         StoredMagicQueries = new List<StoredMagicQuery>(_MagicQuery.StoredMagicQueries); // Deep copy
         ResultsUnique = _MagicQuery.ResultsUnique;
         Predicates = new List<Expression<Func<T, bool>>>(_MagicQuery.Predicates); // Deep copy
+        ForceCursorMode = _MagicQuery.ForceCursorMode;
     }
 
     public IMagicQueryStaging<T> Where(Expression<Func<T, bool>> predicate)

Magic.IndexedDb/Models/StoredMagicQuery.cs

@@ -10,6 +10,7 @@ internal struct MagicQueryFunctions
     public const string Reverse = "reverse";
     public const string First = "first";
     public const string Last = "last";
+    public const string StableOrdering = "stableOrdering";
 
 }
 public class StoredMagicQuery

Magic.IndexedDb/wwwroot/magicLinqToIndexedDb.js

@@ -34,6 +34,11 @@ export async function* magicQueryYield(db, table, universalSerializedPredicate,
         throw new Error("A valid Dexie table instance must be provided.");
     }
 
+    const stableOrderingRequested = hasStableOrdering(queryAdditions);
+    if (stableOrderingRequested) {
+        forceCursor = true;
+    }
+
     debugLog('universal serialized predicate');
     debugLog(universalSerializedPredicate);
     const { nestedOrFilterUnclean, isUniversalTrue, isUniversalFalse } = flattenUniversalPredicate(universalSerializedPredicate);
@@ -116,6 +121,10 @@ export async function* magicQueryYield(db, table, universalSerializedPredicate,
 
 }
 
+function hasStableOrdering(queryAdditions) {
+    return queryAdditions?.some(q => q.additionFunction === QUERY_ADDITIONS.STABLE_ORDERING);
+}
+
 async function runIndexedQueries(db, table, universalQueries,
     queryAdditions, primaryKeys, yieldedPrimaryKeys) {
     if (universalQueries.length === 0) {
@@ -281,6 +290,8 @@ function runIndexedQuery(table, indexedConditions, queryAdditions = []) {
                     return query.first();
                 case QUERY_ADDITIONS.LAST:
                     return query.last();
+                case QUERY_ADDITIONS.STABLE_ORDERING:
+                    break; // do nothing
                 default:
                     throw new Error(`Unsupported query addition: ${addition.additionFunction}`);
             }

Magic.IndexedDb/wwwroot/utilities/cursorEngine.js

@@ -24,10 +24,20 @@ export async function runCursorQuery(db, table, conditions, queryAdditions, yiel
     );
 
     if (requiresMetaProcessing) {
-        // **Metadata Path: Extract primary keys and sorting properties**
-        let primaryKeyList = await runMetaDataCursorQuery(db, table, structuredPredicateTree, queryAdditions, yieldedPrimaryKeys, compoundKeys);
 
-        const indexOrderProps = detectIndexOrderProperties(structuredPredicateTree, table);
+        let stableOrdering = hasStableOrdering(queryAdditions);
+
+        let indexOrderProps = [];
+
+        if (!stableOrdering) {
+            indexOrderProps = detectIndexOrderProperties(structuredPredicateTree, table);
+        }
+        else {
+            debugLog("Stable Ordering detected. Disabling any ordering by indexed queries.");
+        }
+
+        // **Metadata Path: Extract primary keys and sorting properties**
+        let primaryKeyList = await runMetaDataCursorQuery(db, table, structuredPredicateTree, queryAdditions, yieldedPrimaryKeys, compoundKeys, indexOrderProps);
 
         // **Apply sorting, take, and skip operations**
         let finalPrimaryKeys = applyCursorQueryAdditions(primaryKeyList, queryAdditions, compoundKeys, true, indexOrderProps);
@@ -43,6 +53,11 @@ export async function runCursorQuery(db, table, conditions, queryAdditions, yiel
     }
 }
 
+function hasStableOrdering(queryAdditions) {
+    return queryAdditions?.some(q => q.additionFunction === QUERY_ADDITIONS.STABLE_ORDERING);
+}
+
+
 function detectIndexOrderProperties(predicateTree, table) {
     const indexedProps = new Set();
 
@@ -183,7 +198,7 @@ function optimizeSingleCondition(condition) {
     // Lowercase normalization for string values if not case-sensitive
     if (!condition.caseSensitive && typeof condition.value === "string") {
         optimized.value = condition.value.toLowerCase();
-    }    
+    }
 
     optimized.comparisonFunction = getComparisonFunction(condition.operation);
     return optimized;
@@ -226,7 +241,7 @@ async function runDirectCursorQuery(db, table, conditions, yieldedPrimaryKeys, c
 /**
  * Extracts only necessary metadata using a Dexie cursor in a transaction.
  */
-async function runMetaDataCursorQuery(db, table, conditions, queryAdditions, yieldedPrimaryKeys, compoundKeys) {
+async function runMetaDataCursorQuery(db, table, conditions, queryAdditions, yieldedPrimaryKeys, compoundKeys, detectedIndexOrderProperties = []) {
     debugLog("Extracting Metadata for Cursor Query", { conditions, queryAdditions });
 
     let requiredProperties = new Set();
@@ -252,6 +267,12 @@ async function runMetaDataCursorQuery(db, table, conditions, queryAdditions, yie
         requiredProperties.add(key);
     }
 
+    // Include all indexable props that affect ordering
+    for (const prop of detectedIndexOrderProperties) {
+        requiredProperties.add(prop);
+    }
+
+
     requiredProperties.add("_MagicOrderId");
 
     let estimatedSize = await table.count();
@@ -820,6 +841,9 @@ function applyCursorQueryAdditions(
                 primaryKeyList = primaryKeyList.length > 0 ? [primaryKeyList[primaryKeyList.length - 1]] : [];
                 break;
 
+            case QUERY_ADDITIONS.STABLE_ORDERING:                
+                break; // skip this
+
             default:
                 throw new Error(`Unsupported query addition: ${addition.additionFunction}`);
         }

Magic.IndexedDb/wwwroot/utilities/linqValidation.js

@@ -4,8 +4,14 @@
 import { QUERY_OPERATIONS, QUERY_ADDITIONS, QUERY_COMBINATION_RULES, QUERY_ADDITION_RULES } from "./queryConstants.js";
 import { debugLog } from "./utilityHelpers.js";
 
-export function validateQueryAdditions(queryAdditions, indexCache) {
-    queryAdditions = queryAdditions || []; // Ensure it's always an array
+export function validateQueryAdditions(queryAdditionsPre, indexCache) {
+    queryAdditionsPre = queryAdditionsPre || []; // Ensure it's always an array
+
+    // Filter out non-actionable additions like STABLE_ORDERING
+    const queryAdditions = queryAdditionsPre.filter(q =>
+        q.additionFunction !== QUERY_ADDITIONS.STABLE_ORDERING
+    );
+
     let seenAdditions = new Set();
     let requiresCursor = false;
 
@@ -151,6 +157,11 @@ export function isValidQueryAdditions(arr) {
             return;
         }
 
+        // Skip intValue and property validation for STABLE_ORDERING
+        if (obj.additionFunction === QUERY_ADDITIONS.STABLE_ORDERING) {
+            return;
+        }
+
         if (typeof obj.additionFunction !== 'string') {
             console.error(`Error at index ${index}: additionFunction must be a string but got:`, obj.additionFunction);
             isValid = false;

Magic.IndexedDb/wwwroot/utilities/queryConstants.js

@@ -119,6 +119,7 @@ export const QUERY_ADDITIONS = {
     SKIP: "skip",
     TAKE: "take",
     TAKE_LAST: "takeLast",
+    STABLE_ORDERING: "stableOrdering",
 };
 
 //  Query Combination Ruleset (What Can Be Combined in AND `&&`)

TestWasm/Share/PanelCustomTests.razor

@@ -91,10 +91,13 @@
                 StateHasChanged();
 
 
-                RunTest("Weird IndexedDB order by test",
+                RunTest("Indexed Column in predicate order by",
                                await personQuery.Where(x => x.TestInt > 2).Take(2).ToListAsync(),
                                allPeople.Where(x => x.TestInt > 2).OrderBy(x => x.TestInt).ThenBy(x => x._Id).Take(2));
 
+                RunTest("Indexed Column in predicate order by StableOrderBy",
+                await personQuery.Cursor(x => x.TestInt > 2).StableOrdering().Take(2).ToListAsync(),
+                allPeople.Where(x => x.TestInt > 2).OrderBy(x => x._Id).Take(2));
 
                 RunTest("Date Equal",
                     await personQuery.Where(x => x.DateOfBirth.Value.Date == new DateTime(2020, 2, 10)).ToListAsync(),