From fb7b9912fd94daf67961611b5970089444f102db Mon Sep 17 00:00:00 2001 From: Tomi Tuhkanen Date: Tue, 10 Oct 2023 22:26:48 +0300 Subject: [PATCH] feat: replace Newtonsoft.Json with System.Text.Json --- JsonFlatFileDataStore.Benchmark/Program.cs | 8 +- .../CollectionModificationTests.cs | 120 ++--- .../CollectionQueryTests.cs | 14 +- .../CopyPropertiesTests.cs | 64 +-- .../DataStoreDisposeTests.cs | 67 +++ JsonFlatFileDataStore.Test/DataStoreTests.cs | 29 +- .../FileContentTests.cs | 26 +- JsonFlatFileDataStore.Test/SingleItemTests.cs | 97 +++- JsonFlatFileDataStore/CommitActionHandler.cs | 5 +- JsonFlatFileDataStore/DataStore.cs | 450 ++++++++++++++---- JsonFlatFileDataStore/DocumentCollection.cs | 13 +- .../ExpandoObjectConverter.cs | 283 +++++++++++ JsonFlatFileDataStore/GlobalUsings.cs | 3 +- JsonFlatFileDataStore/IDataStore.cs | 2 +- JsonFlatFileDataStore/IDocumentCollection.cs | 2 +- .../JsonFlatFileDataStore.csproj | 10 +- .../NewtonsoftDateTimeConverter.cs | 68 +++ JsonFlatFileDataStore/ObjectExtensions.cs | 88 +++- README.md | 114 ++++- 19 files changed, 1201 insertions(+), 262 deletions(-) create mode 100644 JsonFlatFileDataStore/ExpandoObjectConverter.cs create mode 100644 JsonFlatFileDataStore/NewtonsoftDateTimeConverter.cs diff --git a/JsonFlatFileDataStore.Benchmark/Program.cs b/JsonFlatFileDataStore.Benchmark/Program.cs index cdb4b64..1cba4e1 100644 --- a/JsonFlatFileDataStore.Benchmark/Program.cs +++ b/JsonFlatFileDataStore.Benchmark/Program.cs @@ -6,10 +6,10 @@ private static void Main(string[] args) { var switcher = new BenchmarkSwitcher(new[] { - typeof(TypedCollectionBenchmark), - typeof(DynamicCollectionBenchmark), - typeof(ObjectExtensionsBenchmark) - }); + typeof(TypedCollectionBenchmark), + typeof(DynamicCollectionBenchmark), + typeof(ObjectExtensionsBenchmark) + }); switcher.Run(args); } diff --git a/JsonFlatFileDataStore.Test/CollectionModificationTests.cs b/JsonFlatFileDataStore.Test/CollectionModificationTests.cs index 941275c..7aeb944 100644 --- a/JsonFlatFileDataStore.Test/CollectionModificationTests.cs +++ b/JsonFlatFileDataStore.Test/CollectionModificationTests.cs @@ -1,7 +1,7 @@ -using System.Collections.Generic; -using System.Dynamic; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; +using System.Dynamic; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Threading.Tasks; namespace JsonFlatFileDataStore.Test; @@ -142,7 +142,7 @@ public async Task UpdateOneAsync_TypedUser_WrongCase() var collection2 = store2.GetCollection("users2"); await collection2.UpdateOneAsync(x => x.Id == 0, new { name = "new value" }); - await collection2.UpdateOneAsync(x => x.Id == 1, JToken.Parse("{ name: \"new value 2\"} ")); + await collection2.UpdateOneAsync(x => x.Id == 1, JsonNode.Parse("{ \"name\": \"new value 2\"} ")); var store3 = new DataStore(newFilePath); @@ -203,9 +203,9 @@ public async Task UpdateOneAsync_TypedModel_InnerSimpleArray() Id = Guid.NewGuid().ToString(), Type = "empty", Fragments = new List - { - Guid.NewGuid().ToString() - } + { + Guid.NewGuid().ToString() + } }; var insertResult = collection.InsertOne(newModel); @@ -220,10 +220,10 @@ public async Task UpdateOneAsync_TypedModel_InnerSimpleArray() { Type = "filled", Fragments = new List - { - Guid.NewGuid().ToString(), - Guid.NewGuid().ToString() - } + { + Guid.NewGuid().ToString(), + Guid.NewGuid().ToString() + } }; await collection2.UpdateOneAsync(e => e.Id == newModel.Id, updateData); @@ -254,9 +254,9 @@ public async Task UpdateOneAsync_TypedModel_InnerSimpleIntArray() Id = Guid.NewGuid().ToString(), Type = "empty", Fragments = new List - { - 1 - } + { + 1 + } }; var insertResult = collection.InsertOne(newModel); @@ -271,10 +271,10 @@ public async Task UpdateOneAsync_TypedModel_InnerSimpleIntArray() { Type = "filled", Fragments = new List - { - 2, - 3 - } + { + 2, + 3 + } }; await collection2.UpdateOneAsync(e => e.Id == newModel.Id, updateData); @@ -319,10 +319,10 @@ public async Task UpdateOneAsync_TypedModel_NestedArrays() { Type = "filled", NestedLists = new List> - { - null, - new List { 4 }, - } + { + null, + new List { 4 }, + } }; await collection2.UpdateOneAsync(e => e.Id == newModel.Id, updateData); @@ -407,10 +407,10 @@ public async Task UpdateManyAsync_DynamicUser() var newUsers = new[] { - new { id = 20, name = "A1", age = 55 }, - new { id = 21, name = "A2", age = 55 }, - new { id = 22, name = "A3", age = 55 } - }; + new { id = 20, name = "A1", age = 55 }, + new { id = 21, name = "A2", age = 55 }, + new { id = 22, name = "A3", age = 55 } + }; await collection.InsertManyAsync(newUsers); @@ -445,19 +445,20 @@ public async Task UpdateManyAsync_JsonUser() Assert.Equal(3, collection.Count); var newUsersJson = @" - [ - { 'id': 20, 'name': 'A1', 'age': 55 }, - { 'id': 21, 'name': 'A2', 'age': 55 }, - { 'id': 22, 'name': 'A3', 'age': 55 } - ] - "; + [ + { ""id"": 20, ""name"": ""A1"", ""age"": 55 }, + { ""id"": 21, ""name"": ""A2"", ""age"": 55 }, + { ""id"": 22, ""name"": ""A3"", ""age"": 55 } + ] + "; - var newUsers = JToken.Parse(newUsersJson); + var newUsersArray = JsonNode.Parse(newUsersJson).AsArray(); + var newUsers = newUsersArray.Select(n => n as dynamic); await collection.InsertManyAsync(newUsers); - var newUserJson = "{ 'id': 23, 'name': 'A4', 'age': 22 }"; - var newUser = JToken.Parse(newUserJson); + var newUserJson = "{ \"id\": 23, \"name\": \"A4\", \"age\": 22 }"; + var newUser = JsonNode.Parse(newUserJson); await collection.InsertOneAsync(newUser); @@ -494,10 +495,10 @@ public void UpdateMany_TypedUser() var newUsers = new[] { - new User { Id = 20, Name = "A1", Age = 55 }, - new User { Id = 21, Name = "A2", Age = 55 }, - new User { Id = 22, Name = "A3", Age = 55 } - }; + new User { Id = 20, Name = "A1", Age = 55 }, + new User { Id = 21, Name = "A2", Age = 55 }, + new User { Id = 22, Name = "A3", Age = 55 } + }; collection.InsertMany(newUsers); @@ -623,7 +624,7 @@ public void ReplaceOne_Upsert_DynamicWithInnerData() var collection = store.GetCollection("sensor"); var success = collection.ReplaceOne(e => e.id == 11, - JToken.Parse("{ 'id': 11, 'mac': 'F4:A5:74:89:16:57', 'data': { 'temperature': 20.5 } }"), + JsonNode.Parse("{ \"id\": 11, \"mac\": \"F4:A5:74:89:16:57\", \"data\": { \"temperature\": 20.5 } }"), true); Assert.True(success); @@ -884,13 +885,14 @@ public void UpdateOne_InnerExpandos() collection.InsertOne(user); var patchData = new Dictionary - { - { "Age", 41 }, - { "name", "James" }, - { "Work", new Dictionary { { "Name", "ACME" } } } - }; - var jobject = JObject.FromObject(patchData); - dynamic patchExpando = JsonConvert.DeserializeObject(jobject.ToString()); + { + { "Age", 41 }, + { "name", "James" }, + { "Work", new Dictionary { { "Name", "ACME" } } } + }; + var jsonString = JsonSerializer.Serialize(patchData); + var options = new JsonSerializerOptions { Converters = { new SystemExpandoObjectConverter() } }; + dynamic patchExpando = JsonSerializer.Deserialize(jsonString, options); collection.UpdateOne(i => i.Id == 4, patchExpando as object); @@ -1043,25 +1045,29 @@ public async Task UpdateComplexObject_Dynamic() var collection = store.GetCollection("employee"); - var ja = new JArray { "Hello World!" }; - - var jObj = new JObject() + var data = new { - ["custom_id"] = 11, - ["nestedArray"] = new JArray { ja }, + custom_id = 11, + nestedArray = new[] + { + new[] { "Hello World!" } + } }; - await collection.InsertOneAsync(jObj); + await collection.InsertOneAsync(data); var original = collection.Find(e => e.custom_id == 11).First(); Assert.Equal(0, original.id); Assert.Equal(11, original.custom_id); Assert.Equal("Hello World!", original.nestedArray[0][0]); - var update = new JObject() + var update = new { - ["custom_id"] = 12, - ["nestedArray"] = new JArray { new JArray { "Other text" } }, + custom_id = 12, + nestedArray = new[] + { + new[] { "Other text" } + } }; await collection.UpdateOneAsync(e => e.custom_id == 11, update); diff --git a/JsonFlatFileDataStore.Test/CollectionQueryTests.cs b/JsonFlatFileDataStore.Test/CollectionQueryTests.cs index 12dc2cf..487ccd2 100644 --- a/JsonFlatFileDataStore.Test/CollectionQueryTests.cs +++ b/JsonFlatFileDataStore.Test/CollectionQueryTests.cs @@ -1,4 +1,4 @@ -using Newtonsoft.Json.Linq; +using System.Text.Json.Nodes; namespace JsonFlatFileDataStore.Test; @@ -132,22 +132,22 @@ public void GetNextIdValue_StringType_JToken() var collection = store.GetCollection("collectionWithStringId"); // Insert seed value with upsert - collection.ReplaceOne(e => e, JToken.Parse("{ 'myId': 'test1' }"), true); + collection.ReplaceOne(e => e, JsonNode.Parse("{ \"myId\": \"test1\" }"), true); var nextId = collection.GetNextIdValue(); Assert.Equal("test2", nextId); - var nextUpdate = JToken.Parse("{ 'myId': 'somethingWrong2' }"); + var nextUpdate = JsonNode.Parse("{ \"myId\": \"somethingWrong2\" }"); collection.InsertOne(nextUpdate); - Assert.Equal(nextId, nextUpdate["myId"]); + Assert.Equal(nextId, nextUpdate["myId"].ToString()); nextId = collection.GetNextIdValue(); Assert.Equal("test3", nextId); - nextUpdate = JToken.Parse("{ 'xxx': 111 }"); + nextUpdate = JsonNode.Parse("{ \"xxx\": 111 }"); collection.InsertOne(nextUpdate); - Assert.Equal(nextId, nextUpdate["myId"]); - Assert.Equal(111, nextUpdate["xxx"]); + Assert.Equal(nextId, nextUpdate["myId"].GetValue()); + Assert.Equal(111, nextUpdate["xxx"].GetValue()); nextId = collection.GetNextIdValue(); Assert.Equal("test4", nextId); diff --git a/JsonFlatFileDataStore.Test/CopyPropertiesTests.cs b/JsonFlatFileDataStore.Test/CopyPropertiesTests.cs index 3bade4c..08e3f89 100644 --- a/JsonFlatFileDataStore.Test/CopyPropertiesTests.cs +++ b/JsonFlatFileDataStore.Test/CopyPropertiesTests.cs @@ -1,8 +1,8 @@ using System.Collections.Generic; using System.Collections.ObjectModel; using System.Dynamic; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; +using System.Text.Json; +using Xunit; namespace JsonFlatFileDataStore.Test; @@ -73,10 +73,10 @@ public void CopyProperties_TypedFamily() var family = new Family { Parents = new List - { - new Parent { Name = "Jim", Age = 52 }, - new Parent { Name = "Theodor", Age = 14 } - }, + { + new Parent { Name = "Jim", Age = 52 }, + new Parent { Name = "Theodor", Age = 14 } + }, Address = new Address { City = "Helsinki" } }; @@ -123,10 +123,10 @@ public void CopyProperties_DynamicFamily() sParent.Age = 14; family.Parents = new List - { - fParent, - sParent, - }; + { + fParent, + sParent, + }; family.Address = new ExpandoObject(); family.Address.City = "Helsinki"; @@ -228,9 +228,9 @@ public void CopyProperties_TypedFamilyParents() var family = new Family { Parents = new List - { - new Parent { Name = "Jim", Age = 52 } - }, + { + new Parent { Name = "Jim", Age = 52 } + }, Address = new Address { City = "Helsinki" } }; @@ -251,13 +251,14 @@ public void CopyProperties_DynamicWithInnerExpandos() user.work = work; var patchData = new Dictionary - { - { "age", 41 }, - { "name", "James" }, - { "work", new Dictionary { { "name", "ACME" } } } - }; - var jobject = JObject.FromObject(patchData); - dynamic patchExpando = JsonConvert.DeserializeObject(jobject.ToString()); + { + { "age", 41 }, + { "name", "James" }, + { "work", new Dictionary { { "name", "ACME" } } } + }; + var jsonString = JsonSerializer.Serialize(patchData); + var options = new JsonSerializerOptions { Converters = { new SystemExpandoObjectConverter() } }; + dynamic patchExpando = JsonSerializer.Deserialize(jsonString, options); ObjectExtensions.CopyProperties(patchExpando, user); Assert.Equal("James", user.name); @@ -296,10 +297,10 @@ public void CopyProperties_DynamicEmptyWithInnerDictionary() sensor.mac = "F4:A5:74:89:16:57"; sensor.timestamp = null; sensor.data = new Dictionary - { - { "temperature", 24.3 }, - { "identifier", null } - }; + { + { "temperature", 24.3 }, + { "identifier", null } + }; ObjectExtensions.CopyProperties(sensor, destination); @@ -319,13 +320,14 @@ public void CopyProperties_TypedWithInnerExpandos() }; var patchData = new Dictionary - { - { "Age", 41 }, - { "Name", "James" }, - { "Work", new Dictionary { { "Name", "ACME" } } } - }; - var jobject = JObject.FromObject(patchData); - dynamic patchExpando = JsonConvert.DeserializeObject(jobject.ToString()); + { + { "Age", 41 }, + { "Name", "James" }, + { "Work", new Dictionary { { "Name", "ACME" } } } + }; + var jsonString = JsonSerializer.Serialize(patchData); + var options = new JsonSerializerOptions { Converters = { new SystemExpandoObjectConverter() } }; + dynamic patchExpando = JsonSerializer.Deserialize(jsonString, options); ObjectExtensions.CopyProperties(patchExpando, user); Assert.Equal("James", user.Name); diff --git a/JsonFlatFileDataStore.Test/DataStoreDisposeTests.cs b/JsonFlatFileDataStore.Test/DataStoreDisposeTests.cs index 7101352..08a675c 100644 --- a/JsonFlatFileDataStore.Test/DataStoreDisposeTests.cs +++ b/JsonFlatFileDataStore.Test/DataStoreDisposeTests.cs @@ -73,4 +73,71 @@ private void RunDataStore(out WeakReference storeRef, string newFilePath, int co store = null; } + + [Fact] + public void VerifyJsonDocumentDisposal_NoMemoryLeaks() + { + // This test verifies that JsonDocument instances are properly disposed + // to prevent memory leaks when using ConvertToJsonElement and related methods + var newFilePath = UTHelpers.Up(); + var store = new DataStore(newFilePath); + + // Insert many items to stress test the resource management + for (int i = 0; i < 100; i++) + { + var item = new + { + id = i, + name = $"User{i}", + data = new + { + value = i * 10, + timestamp = DateTime.UtcNow + } + }; + + store.InsertItem($"item{i}", item); + } + + // Force garbage collection to see if any disposed documents cause issues + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + + // Verify all items can still be read correctly + for (int i = 0; i < 100; i++) + { + var retrieved = store.GetItem($"item{i}"); + Assert.NotNull(retrieved); + Assert.Equal(i, retrieved.id); + Assert.Equal($"User{i}", retrieved.name); + Assert.Equal(i * 10, retrieved.data.value); + } + + // Delete items to test RemoveJsonDataElement disposal + for (int i = 0; i < 50; i++) + { + var deleted = store.DeleteItem($"item{i}"); + Assert.True(deleted); + } + + // Force GC again + GC.Collect(); + GC.WaitForPendingFinalizers(); + + // Verify deleted items are gone and others remain + for (int i = 0; i < 50; i++) + { + Assert.Null(store.GetItem($"item{i}")); + } + + for (int i = 50; i < 100; i++) + { + var retrieved = store.GetItem($"item{i}"); + Assert.NotNull(retrieved); + Assert.Equal(i, retrieved.id); + } + + UTHelpers.Down(newFilePath); + } } \ No newline at end of file diff --git a/JsonFlatFileDataStore.Test/DataStoreTests.cs b/JsonFlatFileDataStore.Test/DataStoreTests.cs index 9cba265..0213f79 100644 --- a/JsonFlatFileDataStore.Test/DataStoreTests.cs +++ b/JsonFlatFileDataStore.Test/DataStoreTests.cs @@ -1,7 +1,10 @@ using System.Dynamic; using System.IO; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Nodes; using System.Text.RegularExpressions; -using Newtonsoft.Json.Linq; +using System.Threading.Tasks; using NSubstitute; namespace JsonFlatFileDataStore.Test; @@ -15,7 +18,7 @@ public void UpdateAll() var store = new DataStore(newFilePath); - store.UpdateAll("{ 'tasks': [ { 'id': 0, 'task': 'Commit'} ] }"); + store.UpdateAll("{ \"tasks\": [ { \"id\": 0, \"task\": \"Commit\"} ] }"); var collection = store.GetCollection("tasks"); Assert.Equal(1, collection.Count); @@ -182,7 +185,7 @@ public async Task Readme_Example2() }; // Example with JSON object - var employeeJson = JToken.Parse("{ 'id': 2, 'name': 'Raymond', 'age': 32 }"); + var employeeJson = JsonNode.Parse("{ \"id\": 2, \"name\": \"Raymond\", \"age\": 32 }"); // Example with JSON object var employeeDict = new Dictionary @@ -208,7 +211,7 @@ public async Task Readme_Example2() var updateData = new { name = "John Doe" }; await collection.UpdateOneAsync(e => e.id == employee.id, updateData); - var updateJson = JObject.Parse("{ 'name': 'Raymond Doe' }"); + var updateJson = JsonNode.Parse("{ \"name\": \"Raymond Doe\" }"); await collection.UpdateOneAsync(e => e.id == 1, updateJson); var updateDict = new Dictionary { ["name"] = "Andy Doe" }; @@ -243,7 +246,7 @@ public async Task Insert_CorrectIdWithDynamic() }; // Example with JSON object - var employeeJson = JToken.Parse("{ 'id': 200, 'name': 'Raymond', 'age': 32 }"); + var employeeJson = JsonNode.Parse("{ \"id\": 200, \"name\": \"Raymond\", \"age\": 32 }"); // Example with JSON object var employeeDict = new Dictionary @@ -266,7 +269,7 @@ public async Task Insert_CorrectIdWithDynamic() await collection.InsertOneAsync(employeeExpando); Assert.Equal(20, employee.id); - Assert.Equal(21, employeeJson["id"]); + Assert.Equal(21, employeeJson["id"].GetValue()); Assert.Equal(22, employeeDict["id"]); Assert.Equal(23, ((IDictionary)employeeExpando)["id"]); @@ -290,7 +293,7 @@ public async Task Insert_CorrectIdWithDynamic_No_InitialId() }; // Example with JSON object - var employeeJson = JToken.Parse("{ 'name': 'Raymond', 'age': 32 }"); + var employeeJson = JsonNode.Parse("{ \"name\": \"Raymond\", \"age\": 32 }"); // Example with JSON object var employeeDict = new Dictionary @@ -310,7 +313,10 @@ public async Task Insert_CorrectIdWithDynamic_No_InitialId() await collection.InsertOneAsync(employeeDict); await collection.InsertOneAsync(employeeExpando); - Assert.Equal(1, employeeJson["acc"]); + // System.Text.Json: JsonNode indexer returns JsonNode, requires explicit GetValue() + // Newtonsoft.Json: JToken had implicit conversion operators, allowed direct comparison + Assert.Equal(1, employeeJson["acc"].GetValue()); + // Dictionary and ExpandoObject return 'object' (boxed int), which Assert.Equal handles directly Assert.Equal(2, employeeDict["acc"]); Assert.Equal(3, ((IDictionary)employeeExpando)["acc"]); @@ -335,7 +341,7 @@ public async Task Insert_CorrectIdWithDynamic_With_InitialId() }; // Example with JSON object - var employeeJson = JToken.Parse("{ 'name': 'Raymond', 'age': 32 }"); + var employeeJson = JsonNode.Parse("{ \"name\": \"Raymond\", \"age\": 32 }"); // Example with JSON object var employeeDict = new Dictionary @@ -356,7 +362,10 @@ public async Task Insert_CorrectIdWithDynamic_With_InitialId() await collection.InsertOneAsync(employeeExpando); Assert.Equal("hello", employee.acc); - Assert.Equal("hello0", employeeJson["acc"]); + // System.Text.Json: JsonNode indexer returns JsonNode, requires explicit GetValue() + // Newtonsoft.Json: JToken had implicit conversion operators, allowed direct comparison + Assert.Equal("hello0", employeeJson["acc"].GetValue()); + // Dictionary and ExpandoObject return 'object' (boxed string), which Assert.Equal handles directly Assert.Equal("hello1", employeeDict["acc"]); Assert.Equal("hello2", ((IDictionary)employeeExpando)["acc"]); diff --git a/JsonFlatFileDataStore.Test/FileContentTests.cs b/JsonFlatFileDataStore.Test/FileContentTests.cs index 1afe391..2e02eec 100644 --- a/JsonFlatFileDataStore.Test/FileContentTests.cs +++ b/JsonFlatFileDataStore.Test/FileContentTests.cs @@ -23,10 +23,10 @@ public void FileNotFound_CreateNewFile() } [Theory] - [InlineData(true, true, new[] { 40 })] - [InlineData(false, true, new[] { 40 })] - [InlineData(true, false, new[] { 81, 74 })] - [InlineData(false, false, new[] { 81, 74 })] + [InlineData(true, true, new[] { 40, 38 })] // Minified, lowerCamelCase: Newtonsoft=40, System.Text.Json=38 + [InlineData(false, true, new[] { 40, 38 })] // Minified, UpperCamelCase: Newtonsoft=40, System.Text.Json=38 + [InlineData(true, false, new[] { 81, 79, 74, 72 })] // Formatted, lowerCamelCase: Newtonsoft=81(Win)/74(Unix), System.Text.Json=79(Win)/72(Unix) + [InlineData(false, false, new[] { 81, 79, 74, 72 })] // Formatted, UpperCamelCase: Newtonsoft=81(Win)/74(Unix), System.Text.Json=79(Win)/72(Unix) public async Task AllFormats_CorrectLength(bool useLowerCamelCase, bool useMinifiedJson, int[] allowedLengths) { var path = UTHelpers.GetFullFilePath($"AllFormats_CorrectLength_{DateTime.UtcNow.Ticks}"); @@ -37,13 +37,19 @@ public async Task AllFormats_CorrectLength(bool useLowerCamelCase, bool useMinif var content = UTHelpers.GetFileContent(path); - // NOTE: File format is different depending on used OS. Windows uses \r\n and Linux/macOS \r - // - "{\r\n \"movie\": [\r\n {\r\n \"name\": \"Test\",\r\n \"rating\": 5.0\r\n }\r\n ]\r\n}", - // - "{\r \"movie\": [\r {\r \"name\": \"Test\",\r \"rating\": 5.0\r }\r ]\r}" - // Length on Windows is 81 and on Linux/macOS 74 + // NOTE: File format is different depending on used OS and serializer: // - // Minified length: 40 - // - "{\"movie\":\"name\":\"Test\",\"rating\":5.0}]}" + // Newtonsoft.Json (formatted with "rating": 5.0): + // - Windows (CRLF): 81 bytes + // - Linux/macOS (LF): 74 bytes + // + // System.Text.Json (formatted with "rating": 5): + // - Windows (CRLF): 79 bytes + // - Linux/macOS (LF): 72 bytes + // + // Minified (both serializers): + // - Newtonsoft.Json: 40 bytes (with 5.0) + // - System.Text.Json: 38 bytes (with 5) Assert.Contains(allowedLengths, i => i == content.Length); diff --git a/JsonFlatFileDataStore.Test/SingleItemTests.cs b/JsonFlatFileDataStore.Test/SingleItemTests.cs index ad48d4e..454e71b 100644 --- a/JsonFlatFileDataStore.Test/SingleItemTests.cs +++ b/JsonFlatFileDataStore.Test/SingleItemTests.cs @@ -75,15 +75,24 @@ public void GetItem_DynamicAndTyped_DateType(string encryptionPassword) var test = DateTime.Now.ToShortDateString(); - var itemDynamic = store.GetItem("myDate_string"); + // Typed: System.Text.Json deserializes date strings to DateTime var itemTyped = store.GetItem("myDate_string"); Assert.Equal(2009, itemTyped.Year); - var itemDynamic2 = store.GetItem("myDate_date"); + // Dynamic: Now automatically parses date strings to DateTime (Newtonsoft.Json compatibility) + var itemDynamic = store.GetItem("myDate_string"); + Assert.IsType(itemDynamic); + Assert.Equal(2009, itemDynamic.Year); + + // Typed: System.Text.Json deserializes ISO date strings to DateTime var itemTyped2 = store.GetItem("myDate_date"); - Assert.Equal(2015, itemDynamic2.Year); Assert.Equal(2015, itemTyped2.Year); + // Dynamic: Now automatically parses ISO date strings to DateTime (Newtonsoft.Json compatibility) + var itemDynamic2 = store.GetItem("myDate_date"); + Assert.IsType(itemDynamic2); + Assert.Equal(2015, itemDynamic2.Year); + UTHelpers.Down(newFilePath); } @@ -408,4 +417,86 @@ public void DeleteItem_DynamicUser(string encryptionPassword) UTHelpers.Down(newFilePath); } + + [Theory] + [InlineData(null)] + [InlineData(EncryptionPassword)] + public void InsertItem_ComplexTypes_VerifiesJsonElementSerialization(string encryptionPassword) + { + // This test verifies that SetJsonDataElement and RemoveJsonDataElement work correctly + // with Dictionary serialization/deserialization in System.Text.Json + var (newFilePath, store) = InitializeFileAndStore(encryptionPassword); + + // Test 1: Insert nested object + var nestedUser = new + { + id = 100, + name = "Complex User", + metadata = new + { + tags = new[] { "admin", "developer" }, + settings = new Dictionary + { + { "theme", "dark" }, + { "notifications", true }, + { "maxItems", 50 } + } + } + }; + + var result1 = store.InsertItem("complexUser", nestedUser); + Assert.True(result1); + + var retrieved1 = store.GetItem("complexUser"); + Assert.Equal("Complex User", retrieved1.name); + Assert.Equal("admin", retrieved1.metadata.tags[0]); + Assert.Equal("dark", retrieved1.metadata.settings.theme); + Assert.True(retrieved1.metadata.settings.notifications); + + // Test 2: Insert array + var arrayData = new[] { 1, 2, 3, 4, 5 }; + var result2 = store.InsertItem("numbers", arrayData); + Assert.True(result2); + + var retrieved2 = store.GetItem("numbers"); + Assert.Equal(5, retrieved2.Count); + Assert.Equal(3, (int)retrieved2[2]); + + // Test 3: Insert mixed type object + var mixedData = new + { + stringVal = "test", + intVal = 42, + doubleVal = 3.14, + boolVal = true, + nullVal = (string)null, + dateVal = new DateTime(2023, 1, 15) + }; + + var result3 = store.InsertItem("mixedTypes", mixedData); + Assert.True(result3); + + var retrieved3 = store.GetItem("mixedTypes"); + Assert.Equal("test", retrieved3.stringVal); + Assert.Equal(42, retrieved3.intVal); + Assert.Equal(3.14, (double)retrieved3.doubleVal); + Assert.True(retrieved3.boolVal); + Assert.Null(retrieved3.nullVal); + Assert.Equal(2023, ((DateTime)retrieved3.dateVal).Year); + + // Test 4: Delete items (tests RemoveJsonDataElement) + var deleteResult1 = store.DeleteItem("complexUser"); + Assert.True(deleteResult1); + Assert.Null(store.GetItem("complexUser")); + + var deleteResult2 = store.DeleteItem("numbers"); + Assert.True(deleteResult2); + Assert.Null(store.GetItem("numbers")); + + var deleteResult3 = store.DeleteItem("mixedTypes"); + Assert.True(deleteResult3); + Assert.Null(store.GetItem("mixedTypes")); + + UTHelpers.Down(newFilePath); + } } \ No newline at end of file diff --git a/JsonFlatFileDataStore/CommitActionHandler.cs b/JsonFlatFileDataStore/CommitActionHandler.cs index 71b35bc..f2e6871 100644 --- a/JsonFlatFileDataStore/CommitActionHandler.cs +++ b/JsonFlatFileDataStore/CommitActionHandler.cs @@ -1,7 +1,6 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Threading; -using Newtonsoft.Json.Linq; namespace JsonFlatFileDataStore; @@ -42,7 +41,9 @@ internal static void HandleStoreCommitActions(CancellationToken token, BlockingC foreach (var action in batch) { - var (actionSuccess, updatedJson) = action.HandleAction(JObject.Parse(jsonText)); + using var jsonDocument = JsonDocument.Parse(jsonText); + var rootElement = jsonDocument.RootElement.Clone(); + var (actionSuccess, updatedJson) = action.HandleAction(rootElement); callbacks.Enqueue((action, actionSuccess)); diff --git a/JsonFlatFileDataStore/DataStore.cs b/JsonFlatFileDataStore/DataStore.cs index 9dd5e15..6347db7 100644 --- a/JsonFlatFileDataStore/DataStore.cs +++ b/JsonFlatFileDataStore/DataStore.cs @@ -3,12 +3,9 @@ using System.Dynamic; using System.Globalization; using System.Linq; +using System.Text.Json.Serialization; using System.Threading; using System.Threading.Tasks; -using Newtonsoft.Json; -using Newtonsoft.Json.Converters; -using Newtonsoft.Json.Linq; -using Newtonsoft.Json.Serialization; namespace JsonFlatFileDataStore; @@ -17,19 +14,23 @@ public class DataStore : IDataStore private readonly string _filePath; private readonly string _keyProperty; private readonly bool _reloadBeforeGetCollection; - private readonly Func _toJsonFunc; + private readonly Func _toJsonFunc; private readonly Func _convertPathToCorrectCamelCase; private readonly BlockingCollection _updates = new BlockingCollection(); private readonly CancellationTokenSource _cts = new CancellationTokenSource(); - private readonly ExpandoObjectConverter _converter = new ExpandoObjectConverter(); + private readonly JsonSerializerOptions _options = new JsonSerializerOptions + { + Converters = { new NewtonsoftDateTimeConverter(), new SystemExpandoObjectConverter() }, + PropertyNameCaseInsensitive = true + }; - private readonly JsonSerializerSettings _serializerSettings = new JsonSerializerSettings() - { ContractResolver = new CamelCasePropertyNamesContractResolver() }; + private readonly JsonSerializerOptions _serializerOptions; private readonly Func _encryptJson; private readonly Func _decryptJson; - private JObject _jsonData; + private JsonDocument _jsonData; + private readonly object _jsonDataLock = new object(); private bool _executingJsonUpdate; public DataStore(string path, bool useLowerCamelCase = true, string keyProperty = null, bool reloadBeforeGetCollection = false, @@ -38,17 +39,28 @@ public DataStore(string path, bool useLowerCamelCase = true, string keyProperty _filePath = path; var useEncryption = !string.IsNullOrWhiteSpace(encryptionKey); - var usedFormatting = minifyJson || useEncryption ? Formatting.None : Formatting.Indented; + var writeIntended = !minifyJson && !useEncryption; // Set to `true` if not minifying or encrypting + + _serializerOptions = useLowerCamelCase ? new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = writeIntended + } : new JsonSerializerOptions + { + WriteIndented = writeIntended + }; _toJsonFunc = useLowerCamelCase - ? new Func(data => + ? new Func(data => { - // Serializing JObject ignores SerializerSettings, so we have to first deserialize to ExpandoObject and then serialize - // http://json.codeplex.com/workitem/23853 - var jObject = JsonConvert.DeserializeObject(data.ToString()); - return JsonConvert.SerializeObject(jObject, usedFormatting, _serializerSettings); + // Deserialize to ExpandoObject to allow flexible serialization settings + var expandoObject = JsonSerializer.Deserialize(data.GetRawText(), _options); + + // Serialize back to JSON with camel casing and indentation options applied + // Case-insensitive property matching is handled in ObjectExtensions.CopyProperties + return JsonSerializer.Serialize(expandoObject, _serializerOptions); }) - : (s => s.ToString(usedFormatting)); + : (s => JsonSerializer.Serialize(s, _serializerOptions)); _convertPathToCorrectCamelCase = useLowerCamelCase ? new Func(s => string.Concat(s.Select((x, i) => i == 0 ? char.ToLower(x).ToString() : x.ToString()))) @@ -70,7 +82,7 @@ public DataStore(string path, bool useLowerCamelCase = true, string keyProperty _decryptJson = (json => json); } - _jsonData = GetJsonObjectFromFile(); + SetJsonData(GetJsonObjectFromFile()); // Run updates on a background thread and use BlockingCollection to prevent multiple updates to run simultaneously Task.Run(() => @@ -80,9 +92,9 @@ public DataStore(string path, bool useLowerCamelCase = true, string keyProperty executionState => _executingJsonUpdate = executionState, jsonText => { - lock (_jsonData) + lock (_jsonDataLock) { - _jsonData = JObject.Parse(jsonText); + SetJsonData(Parse(jsonText)); } return FileAccess.WriteJsonToFile(_filePath, _encryptJson, jsonText); @@ -102,15 +114,20 @@ public void Dispose() { _cts.Cancel(); } + + // Dispose the JsonDocument to free unmanaged resources + _jsonData?.Dispose(); } + + public bool IsUpdating => _updates.Count > 0 || _executingJsonUpdate; public void UpdateAll(string jsonData) { - lock (_jsonData) + lock (_jsonDataLock) { - _jsonData = JObject.Parse(jsonData); + SetJsonData(Parse(jsonData)); } FileAccess.WriteJsonToFile(_filePath, _encryptJson, jsonData); @@ -118,9 +135,9 @@ public void UpdateAll(string jsonData) public void Reload() { - lock (_jsonData) + lock (_jsonDataLock) { - _jsonData = GetJsonObjectFromFile(); + SetJsonData(GetJsonObjectFromFile()); } } @@ -129,12 +146,12 @@ public T GetItem(string key) if (_reloadBeforeGetCollection) { // This might be a bad idea especially if the file is in use, as this can take a long time - _jsonData = GetJsonObjectFromFile(); + SetJsonData(GetJsonObjectFromFile()); } var convertedKey = _convertPathToCorrectCamelCase(key); - var token = _jsonData[convertedKey]; + var token = TryGetElement(_jsonData.RootElement, convertedKey); if (token == null) { @@ -146,7 +163,7 @@ public T GetItem(string key) throw new KeyNotFoundException(); } - return token.ToObject(); + return ConvertJsonElementToObject(token.Value); } public dynamic GetItem(string key) @@ -154,17 +171,17 @@ public dynamic GetItem(string key) if (_reloadBeforeGetCollection) { // This might be a bad idea especially if the file is in use, as this can take a long time - _jsonData = GetJsonObjectFromFile(); + SetJsonData(GetJsonObjectFromFile()); } var convertedKey = _convertPathToCorrectCamelCase(key); - var token = _jsonData[convertedKey]; + var token = TryGetElement(_jsonData.RootElement, convertedKey); if (token == null) return null; - return SingleDynamicItemReadConverter(token); + return SingleDynamicItemReadConverter(token.Value); } public bool InsertItem(string key, T item) => Insert(key, item).Result; @@ -175,13 +192,15 @@ private Task Insert(string key, T item, bool isAsync = false) { var convertedKey = _convertPathToCorrectCamelCase(key); - (bool, JObject) UpdateAction() + (bool, JsonElement) UpdateAction() { - if (_jsonData[convertedKey] != null) - return (false, _jsonData); + var data = TryGetElement(_jsonData.RootElement, convertedKey); + if (data.HasValue) + return (false, data.Value); - _jsonData[convertedKey] = JToken.FromObject(item); - return (true, _jsonData); + var newElement = ConvertToJsonElement(item); + SetJsonData(SetJsonDataElement(_jsonData.RootElement, convertedKey, newElement)); + return (true, _jsonData.RootElement); } return CommitItem(UpdateAction, isAsync); @@ -195,13 +214,15 @@ private Task Replace(string key, T item, bool upsert = false, bool isAs { var convertedKey = _convertPathToCorrectCamelCase(key); - (bool, JObject) UpdateAction() + (bool, JsonElement) UpdateAction() { - if (_jsonData[convertedKey] == null && upsert == false) - return (false, _jsonData); + var data = TryGetElement(_jsonData.RootElement, convertedKey); + if (data == null && upsert == false) + return (false, _jsonData.RootElement); - _jsonData[convertedKey] = JToken.FromObject(item); - return (true, _jsonData); + var newElement = ConvertToJsonElement(item); + SetJsonData(SetJsonDataElement(_jsonData.RootElement, convertedKey, newElement)); + return (true, _jsonData.RootElement); } return CommitItem(UpdateAction, isAsync); @@ -215,24 +236,27 @@ private Task Update(string key, dynamic item, bool isAsync = false) { var convertedKey = _convertPathToCorrectCamelCase(key); - (bool, JObject) UpdateAction() + (bool, JsonElement) UpdateAction() { - if (_jsonData[convertedKey] == null) - return (false, _jsonData); + var data = TryGetElement(_jsonData.RootElement, convertedKey); + if (data == null) + return (false, _jsonData.RootElement); - var toUpdate = SingleDynamicItemReadConverter(_jsonData[convertedKey]); + var toUpdate = SingleDynamicItemReadConverter(data.Value); if (ObjectExtensions.IsReferenceType(item) && ObjectExtensions.IsReferenceType(toUpdate)) { ObjectExtensions.CopyProperties(item, toUpdate); - _jsonData[convertedKey] = JToken.FromObject(toUpdate); + var newElement = ConvertToJsonElement(toUpdate); + _jsonData = SetJsonDataElement(_jsonData.RootElement, convertedKey, newElement); } else { - _jsonData[convertedKey] = JToken.FromObject(item); + var newElement = ConvertToJsonElement(item); + _jsonData = SetJsonDataElement(_jsonData.RootElement, convertedKey, newElement); } - return (true, _jsonData); + return (true, _jsonData.RootElement); } return CommitItem(UpdateAction, isAsync); @@ -246,10 +270,9 @@ private Task Delete(string key, bool isAsync = false) { var convertedKey = _convertPathToCorrectCamelCase(key); - (bool, JObject) UpdateAction() + (bool, JsonElement) UpdateAction() { - var result = _jsonData.Remove(convertedKey); - return (result, _jsonData); + return RemoveJsonDataElement(_jsonData.RootElement, convertedKey); } return CommitItem(UpdateAction, isAsync); @@ -257,8 +280,14 @@ private Task Delete(string key, bool isAsync = false) public IDocumentCollection GetCollection(string name = null) where T : class { - // NOTE 27.6.2017: Should this be new Func(e => e.ToObject())? - var readConvert = new Func(e => JsonConvert.DeserializeObject(e.ToString())); + // Deserialize JsonElement to T + // Uses NewtonsoftDateTimeConverter for backward compatibility with Newtonsoft.Json DateTime formats + var readConvert = new Func(e => e.Deserialize( + new JsonSerializerOptions + { + PropertyNameCaseInsensitive = true, + Converters = { new JsonStringEnumConverter(JsonNamingPolicy.CamelCase, true), new NewtonsoftDateTimeConverter() } + })); var insertConvert = new Func(e => e); var createNewInstance = new Func(() => Activator.CreateInstance()); @@ -267,9 +296,13 @@ public IDocumentCollection GetCollection(string name = null) where T : cla public IDocumentCollection GetCollection(string name) { - // As we don't want to return JObject when using dynamic, JObject will be converted to ExpandoObject - var readConvert = new Func(e => JsonConvert.DeserializeObject(e.ToString(), _converter) as dynamic); - var insertConvert = new Func(e => JsonConvert.DeserializeObject(JsonConvert.SerializeObject(e), _converter)); + // Deserialize JsonElement to ExpandoObject for dynamic handling + var readConvert = new Func(e => e.Deserialize(_options)); + var insertConvert = new Func(e => + { + var json = JsonSerializer.Serialize(e, _serializerOptions); + return JsonSerializer.Deserialize(json, _options); + }); var createNewInstance = new Func(() => new ExpandoObject()); return GetCollection(name, readConvert, insertConvert, createNewInstance); @@ -277,57 +310,66 @@ public IDocumentCollection GetCollection(string name) public IDictionary GetKeys(ValueType? typeToGet = null) { - bool IsCollection(JToken c) => c.Children().FirstOrDefault() is JArray && c.Children().FirstOrDefault().Any() == false - || c.Children().FirstOrDefault()?.FirstOrDefault()?.Type == JTokenType.Object; + bool IsCollection(JsonElement property) => + property.ValueKind == JsonValueKind.Array && + (!property.EnumerateArray().Any() || property.EnumerateArray().First().ValueKind == JsonValueKind.Object); - bool IsItem(JToken c) => c.Children().FirstOrDefault().GetType() != typeof(JArray) - || (c.Children().FirstOrDefault() is JArray - && c.Children().FirstOrDefault().Any() // Empty array is considered as a collection - && c.Children().FirstOrDefault()?.FirstOrDefault()?.Type != JTokenType.Object); + bool IsItem(JsonElement property) => + property.ValueKind != JsonValueKind.Array || + (property.EnumerateArray().Any() && property.EnumerateArray().First().ValueKind != JsonValueKind.Object); - lock (_jsonData) + lock (_jsonDataLock) { - switch (typeToGet) + var result = new Dictionary(); + + if (_jsonData.RootElement.ValueKind == JsonValueKind.Object) { - case null: - return _jsonData.Children() - .ToDictionary(c => c.Path, c => IsCollection(c) ? ValueType.Collection : ValueType.Item); - - case ValueType.Collection: - return _jsonData.Children() - .Where(IsCollection) - .ToDictionary(c => c.Path, c => ValueType.Collection); - - case ValueType.Item: - return _jsonData.Children() - .Where(IsItem) - .ToDictionary(c => c.Path, c => ValueType.Item); - - default: - throw new NotSupportedException(); + foreach (var property in _jsonData.RootElement.EnumerateObject()) + { + bool isCollection = IsCollection(property.Value); + bool isItem = IsItem(property.Value); + + switch (typeToGet) + { + case null: + result[property.Name] = isCollection ? ValueType.Collection : ValueType.Item; + break; + case ValueType.Collection when isCollection: + result[property.Name] = ValueType.Collection; + break; + case ValueType.Item when isItem: + result[property.Name] = ValueType.Item; + break; + } + } } + + return result; } } - private IDocumentCollection GetCollection(string path, Func readConvert, Func insertConvert, Func createNewInstance) + private IDocumentCollection GetCollection(string path, Func readConvert, Func insertConvert, Func createNewInstance) { var pathWithConfiguredCase = _convertPathToCorrectCamelCase(path); var data = new Lazy>(() => { - lock (_jsonData) + lock (_jsonDataLock) { if (_reloadBeforeGetCollection) { // This might be a bad idea especially if the file is in use, as this can take a long time - _jsonData = GetJsonObjectFromFile(); + SetJsonData(GetJsonObjectFromFile()); } - return _jsonData[pathWithConfiguredCase]? - .Children() + var data = TryGetElement(_jsonData.RootElement, pathWithConfiguredCase); + + if (data.HasValue == false) + return new List(); + + return GetChildren(data.Value) .Select(e => readConvert(e)) - .ToList() - ?? new List(); + .ToList(); } }); @@ -340,7 +382,7 @@ private IDocumentCollection GetCollection(string path, Func rea createNewInstance); } - private async Task CommitItem(Func<(bool, JObject)> commitOperation, bool isOperationAsync) + private async Task CommitItem(Func<(bool, JsonElement)> commitOperation, bool isOperationAsync) { var commitAction = new CommitAction(); @@ -353,7 +395,7 @@ private async Task CommitItem(Func<(bool, JObject)> commitOperation, bool return await InnerCommit(isOperationAsync, commitAction); } - private async Task Commit(string dataPath, Func, bool> commitOperation, bool isOperationAsync, Func readConvert) + private async Task Commit(string dataPath, Func, bool> commitOperation, bool isOperationAsync, Func readConvert) { var commitAction = new CommitAction(); @@ -361,17 +403,19 @@ private async Task Commit(string dataPath, Func, bool> commitOp { var updatedJson = string.Empty; - var selectedData = currentJson[dataPath]? - .Children() + var data = TryGetElement(currentJson, dataPath); + + var selectedData = (data.HasValue) ? GetChildren(data.Value) .Select(e => readConvert(e)) .ToList() - ?? new List(); + : new List(); var success = commitOperation(selectedData); if (success) { - currentJson[dataPath] = JArray.FromObject(selectedData); + var newElement = ConvertToJsonElement(selectedData); + currentJson = SetJsonDataElement(currentJson, dataPath, newElement).RootElement; updatedJson = _toJsonFunc(currentJson); } @@ -410,35 +454,231 @@ private async Task InnerCommit(bool isOperationAsync, CommitAction commitA return actionSuccess; } - private dynamic SingleDynamicItemReadConverter(JToken e) + private dynamic SingleDynamicItemReadConverter(JsonElement e) { - switch (e) + switch (e.ValueKind) { - case var objToken when e.Type == JTokenType.Object: - //As we don't want to return JObject when using dynamic, JObject will be converted to ExpandoObject - // JToken.ToString() is not culture invariant, so need to use string.Format - var content = string.Format(CultureInfo.InvariantCulture, "{0}", objToken); - return JsonConvert.DeserializeObject(content, _converter) as dynamic; + case JsonValueKind.Object: + // Convert JsonElement to ExpandoObject for a dynamic structure + var content = e.GetRawText(); // Get the JSON as a raw string + return JsonSerializer.Deserialize(content, _options) as dynamic; + + case JsonValueKind.Array: + // Convert JsonElement array to a List + var list = new List(); + foreach (var item in e.EnumerateArray()) + { + list.Add(SingleDynamicItemReadConverter(item)); // Recursively handle each item + } + return list; + + case JsonValueKind.String: + // Try to parse as DateTime to maintain Newtonsoft.Json compatibility + // Performance Note: DateTime.TryParse() is expensive, so we use a heuristic + // to quickly filter out obvious non-date strings before attempting to parse. + var strValue = e.GetString(); + if (!string.IsNullOrEmpty(strValue)) + { + // Quick heuristic: likely a date if it starts with a digit and contains date separators + // This filters out most non-date strings very quickly + // Common date formats: "2015-11-23T00:00:00", "6/15/2009", "2023-01-15" + if (strValue.Length >= 8 && // Minimum reasonable date length (e.g., "1/1/2023") + char.IsDigit(strValue[0]) && + (strValue.IndexOf('-') >= 0 || strValue.IndexOf('/') >= 0 || strValue.IndexOf('T') >= 0)) + { + // Try InvariantCulture first (for ISO formats like "2015-11-23T00:00:00") + if (DateTime.TryParse(strValue, CultureInfo.InvariantCulture, DateTimeStyles.None, out DateTime dateTime)) + { + return dateTime; + } + // Fallback to current culture (for locale-specific formats like "6/15/2009") + if (DateTime.TryParse(strValue, CultureInfo.CurrentCulture, DateTimeStyles.None, out dateTime)) + { + return dateTime; + } + } + } + return strValue; + + case JsonValueKind.Number: + return e.TryGetInt64(out long l) ? l : e.GetDouble(); - case var arrayToken when e.Type == JTokenType.Array: - return e.ToObject>(); + case JsonValueKind.True: + case JsonValueKind.False: + return e.GetBoolean(); - case JValue jv when e is JValue: - return jv.Value; + case JsonValueKind.Null: + return null; default: - return e.ToObject(); + return e.GetRawText(); // Return as string for unknown types } } + + private void SetJsonData(JsonDocument newData) + { + // Safely replaces _jsonData by disposing the old JsonDocument before assigning the new one. + // This prevents memory leaks by ensuring JsonDocument resources are properly released. + var oldData = _jsonData; + _jsonData = newData; + oldData?.Dispose(); + } private string GetJsonTextFromFile() => FileAccess.ReadJsonFromFile(_filePath, _encryptJson, _decryptJson); - private JObject GetJsonObjectFromFile() => JObject.Parse(GetJsonTextFromFile()); + private JsonDocument GetJsonObjectFromFile() + { + var jsonText = GetJsonTextFromFile(); + return JsonDocument.Parse(jsonText); + } + + private JsonElement? TryGetElement(JsonElement element, string key) + { + if (element.TryGetProperty(key, out JsonElement childElement)) + { + return childElement; + } + else + { + return null; + } + } + + private JsonDocument Parse(string json) + { + return JsonDocument.Parse(json); + } + + private T ConvertJsonElementToObject(JsonElement token) + { + // Special handling for DateTime to support Newtonsoft.Json format + if (typeof(T) == typeof(DateTime) && token.ValueKind == JsonValueKind.String) + { + var dateString = token.GetString(); + var converter = new NewtonsoftDateTimeConverter(); + var reader = new Utf8JsonReader(System.Text.Encoding.UTF8.GetBytes($"\"{dateString}\"")); + reader.Read(); // Advance to the string token + return (T)(object)converter.Read(ref reader, typeof(DateTime), _options); + } + + return JsonSerializer.Deserialize(token.GetRawText(), _options); + } + + private JsonElement ConvertToJsonElement(object item) + { + // Serialize the object to JSON with proper naming policy and parse it as a JsonDocument + var json = JsonSerializer.Serialize(item, _serializerOptions); + using var jsonDocument = JsonDocument.Parse(json); + + // Clone the root element so it doesn't reference the disposed JsonDocument + // JsonElement is a struct that holds a reference to its parent document's buffer, + // so we must clone it to create a copy that's independent of the document's lifetime + return jsonDocument.RootElement.Clone(); + } + + private JsonDocument SetJsonDataElement(JsonElement original, string key, object item) + { + // Convert _jsonData to a Dictionary to make modifications + var jsonDataDict = JsonSerializer.Deserialize>(original.GetRawText()); + + // Convert the item to JsonElement + var newElement = ConvertToJsonElement(item); + + // Set or update the element in the dictionary + jsonDataDict[key] = newElement; + + // Serialize back to JsonElement + var modifiedJson = JsonSerializer.Serialize(jsonDataDict); + return JsonDocument.Parse(modifiedJson); + } + + public (bool, JsonElement) RemoveJsonDataElement(JsonElement original, string key) + { + // Deserialize _jsonData to a dictionary for modification + var jsonDataDict = JsonSerializer.Deserialize>(original.GetRawText()); + + // Remove the specified key + var removed = jsonDataDict.Remove(key); + + // Serialize the updated dictionary back to a JsonElement + var modifiedJson = JsonSerializer.Serialize(jsonDataDict); + using var jsonDocument = JsonDocument.Parse(modifiedJson); + + // Clone the root element so it doesn't reference the disposed JsonDocument + return (removed, jsonDocument.RootElement.Clone()); + } + + private IEnumerable GetChildren(JsonElement element) + { + if (element.ValueKind == JsonValueKind.Object) + { + // If element is an object, return its property values + // JsonElement is a struct - no disposal needed + return element.EnumerateObject().Select(p => p.Value); + } + else if (element.ValueKind == JsonValueKind.Array) + { + // If element is an array, return the array items + // JsonElement is a struct - no disposal needed + return element.EnumerateArray(); + } + + // If it's neither an object nor an array, return an empty sequence + return Enumerable.Empty(); + } + + public static string GetJsonPath(JsonElement root, JsonElement target) + { + return FindPath(root, target); + } + + private static string FindPath(JsonElement element, JsonElement target, string currentPath = "") + { + if (element.Equals(target)) + { + return currentPath; + } + + switch (element.ValueKind) + { + case JsonValueKind.Object: + foreach (var property in element.EnumerateObject()) + { + var propertyPath = string.IsNullOrEmpty(currentPath) + ? property.Name + : $"{currentPath}.{property.Name}"; + + var result = FindPath(property.Value, target, propertyPath); + if (result != null) + { + return result; + } + } + break; + + case JsonValueKind.Array: + int index = 0; + foreach (var item in element.EnumerateArray()) + { + var arrayPath = $"{currentPath}[{index}]"; + + var result = FindPath(item, target, arrayPath); + if (result != null) + { + return result; + } + index++; + } + break; + } + + return null; // Target not found in this branch + } internal class CommitAction { public Action Ready { get; set; } - public Func HandleAction { get; set; } + public Func HandleAction { get; set; } } } \ No newline at end of file diff --git a/JsonFlatFileDataStore/DocumentCollection.cs b/JsonFlatFileDataStore/DocumentCollection.cs index d436068..b239a53 100644 --- a/JsonFlatFileDataStore/DocumentCollection.cs +++ b/JsonFlatFileDataStore/DocumentCollection.cs @@ -2,8 +2,6 @@ using System.Dynamic; using System.Linq; using System.Threading.Tasks; -using Newtonsoft.Json; -using Newtonsoft.Json.Converters; namespace JsonFlatFileDataStore; @@ -410,12 +408,21 @@ private bool ExecuteLocked(Func, bool> func, List data) if (keyValue is Int64) return (int)keyValue + 1; + if (keyValue is Int32) + return (int)keyValue + 1; + return ParseNextIntegerToKeyValue(keyValue.ToString()); } private dynamic GetFieldValue(T item, string fieldName) { - var expando = JsonConvert.DeserializeObject(JsonConvert.SerializeObject(item), new ExpandoObjectConverter()); + var options = new JsonSerializerOptions + { + Converters = { new SystemExpandoObjectConverter() }, + PropertyNameCaseInsensitive = true // Optional: make property name matching case-insensitive + }; + + var expando = JsonSerializer.Deserialize(JsonSerializer.Serialize(item), options); // Problem here is if we have typed data with upper camel case properties but lower camel case in JSON, so need to use OrdinalIgnoreCase string comparer var expandoAsIgnoreCase = new Dictionary(expando, StringComparer.OrdinalIgnoreCase); diff --git a/JsonFlatFileDataStore/ExpandoObjectConverter.cs b/JsonFlatFileDataStore/ExpandoObjectConverter.cs new file mode 100644 index 0000000..d75b6c5 --- /dev/null +++ b/JsonFlatFileDataStore/ExpandoObjectConverter.cs @@ -0,0 +1,283 @@ +using System.Collections.Generic; +using System.Dynamic; +using System.Text.Json.Serialization; + +namespace JsonFlatFileDataStore; + +public class SystemExpandoObjectConverter : JsonConverter +{ + public override ExpandoObject Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + using (JsonDocument doc = JsonDocument.ParseValue(ref reader)) + { + var root = doc.RootElement; + + if (root.ValueKind == JsonValueKind.Object) + { + var expando = new ExpandoObject(); + var dictionary = (IDictionary)expando; + + foreach (var property in root.EnumerateObject()) + { + AddPropertyToExpando(dictionary, property.Name, property.Value); + } + + return expando; + } + + throw new JsonException("Invalid JSON: Expected JSON object."); + } + } + + public override void Write(Utf8JsonWriter writer, ExpandoObject value, JsonSerializerOptions options) + { + JsonSerializer.Serialize(writer, (object)value, options); + } + + private static void AddPropertyToExpando(IDictionary expando, string propertyName, JsonElement propertyValue) + { + switch (propertyValue.ValueKind) + { + case JsonValueKind.Undefined: + case JsonValueKind.Null: + expando[propertyName] = null; + break; + + case JsonValueKind.False: + expando[propertyName] = false; + break; + + case JsonValueKind.True: + expando[propertyName] = true; + break; + + case JsonValueKind.Number: + if (propertyValue.TryGetInt32(out int intValue)) + { + expando[propertyName] = intValue; + } + else if (propertyValue.TryGetInt64(out long longValue)) + { + expando[propertyName] = longValue; + } + else if (propertyValue.TryGetDouble(out double doubleValue)) + { + expando[propertyName] = doubleValue; + } + else if (propertyValue.TryGetDecimal(out decimal decimalValue)) + { + expando[propertyName] = decimalValue; + } + else + { + // TODO: Handle other numeric types as needed + throw new JsonException("Unsupported numeric type"); + } + break; + + case JsonValueKind.String: + // Try to parse as DateTime to maintain Newtonsoft.Json compatibility + // Newtonsoft.Json automatically parsed strings that looked like dates + expando[propertyName] = TryParseDateTime(propertyValue.GetString()); + break; + + case JsonValueKind.Object: + var nestedExpando = new ExpandoObject(); + var nestedDictionary = (IDictionary)nestedExpando; + foreach (var nestedProperty in propertyValue.EnumerateObject()) + { + AddPropertyToExpando(nestedDictionary, nestedProperty.Name, nestedProperty.Value); + } + expando[propertyName] = nestedExpando; + break; + + case JsonValueKind.Array: + var arrayValues = new List(); + foreach (var arrayElement in propertyValue.EnumerateArray()) + { + switch (arrayElement.ValueKind) + { + case JsonValueKind.Undefined: + case JsonValueKind.Null: + arrayValues.Add(null); + break; + + case JsonValueKind.False: + arrayValues.Add(false); + break; + + case JsonValueKind.True: + arrayValues.Add(true); + break; + + case JsonValueKind.Number: + if (arrayElement.TryGetInt32(out int arrayElementIntValue)) + { + arrayValues.Add(arrayElementIntValue); + } + else if (arrayElement.TryGetInt64(out long longValue)) + { + arrayValues.Add(longValue); + } + else if (arrayElement.TryGetDouble(out double doubleValue)) + { + arrayValues.Add(doubleValue); + } + else if (arrayElement.TryGetDecimal(out decimal decimalValue)) + { + arrayValues.Add(decimalValue); + } + else + { + throw new JsonException("Unsupported numeric type"); + } + break; + + case JsonValueKind.String: + // Try to parse as DateTime to maintain Newtonsoft.Json compatibility + arrayValues.Add(TryParseDateTime(arrayElement.GetString())); + break; + + case JsonValueKind.Object: + var nestedExpandoInArray = new ExpandoObject(); + var nestedDictionaryInArray = (IDictionary)nestedExpandoInArray; + foreach (var nestedPropertyInArray in arrayElement.EnumerateObject()) + { + AddPropertyToExpando(nestedDictionaryInArray, nestedPropertyInArray.Name, nestedPropertyInArray.Value); + } + arrayValues.Add(nestedExpandoInArray); + break; + + case JsonValueKind.Array: + // Recursively handle nested arrays + var nestedArray = new List(); + foreach (var nestedArrayElement in arrayElement.EnumerateArray()) + { + switch (nestedArrayElement.ValueKind) + { + case JsonValueKind.Undefined: + case JsonValueKind.Null: + nestedArray.Add(null); + break; + + case JsonValueKind.False: + nestedArray.Add(false); + break; + + case JsonValueKind.True: + nestedArray.Add(true); + break; + + case JsonValueKind.Number: + if (nestedArrayElement.TryGetInt32(out int nestedArrayElementIntValue)) + { + nestedArray.Add(nestedArrayElementIntValue); + } + else if (nestedArrayElement.TryGetInt64(out long longValue)) + { + nestedArray.Add(longValue); + } + else if (nestedArrayElement.TryGetDouble(out double doubleValue)) + { + nestedArray.Add(doubleValue); + } + else if (nestedArrayElement.TryGetDecimal(out decimal decimalValue)) + { + nestedArray.Add(decimalValue); + } + else + { + throw new JsonException("Unsupported numeric type"); + } + break; + + case JsonValueKind.String: + // Try to parse as DateTime to maintain Newtonsoft.Json compatibility + nestedArray.Add(TryParseDateTime(nestedArrayElement.GetString())); + break; + + case JsonValueKind.Object: + + var nestedExpandoInNestedArray = new ExpandoObject(); + var nestedDictionaryInNestedArray = (IDictionary)nestedExpandoInNestedArray; + foreach (var nestedPropertyInNestedArray in nestedArrayElement.EnumerateObject()) + { + AddPropertyToExpando(nestedDictionaryInNestedArray, + nestedPropertyInNestedArray.Name, + nestedPropertyInNestedArray.Value); + } + nestedArray.Add(nestedExpandoInNestedArray); + break; + + case JsonValueKind.Array: + // Recursively handle deeper nested arrays + nestedArray.Add(ProcessNestedArray(nestedArrayElement)); + break; + + } + } + arrayValues.Add(nestedArray); + break; + } + } + expando[propertyName] = arrayValues; + break; + } + } + + private static object ProcessNestedArray(JsonElement arrayElement) + { + var arrayValues = new List(); + + foreach (var element in arrayElement.EnumerateArray()) + { + if (element.ValueKind == JsonValueKind.Object) + { + var nestedExpando = new ExpandoObject(); + var nestedDictionary = (IDictionary)nestedExpando; + foreach (var nestedProperty in element.EnumerateObject()) + { + AddPropertyToExpando(nestedDictionary, nestedProperty.Name, nestedProperty.Value); + } + arrayValues.Add(nestedExpando); + } + else if (element.ValueKind == JsonValueKind.Array) + { + // Recursively handle further nested arrays + arrayValues.Add(ProcessNestedArray(element)); + } + else + { + arrayValues.Add(element.ToString()); + } + } + + return arrayValues; + } + + /// + /// Try to parse a string as DateTime to maintain backward compatibility with Newtonsoft.Json + /// Newtonsoft.Json automatically parsed strings that looked like dates + /// + /// Performance Note: This method is called for every string value when deserializing + /// dynamic/ExpandoObject data. DateTime.TryParse() has a performance cost, but it's + /// necessary to maintain backward compatibility. For performance-critical scenarios, + /// consider using strongly-typed collections (GetCollection<T>()) which don't + /// require this parsing step. + /// + private static object TryParseDateTime(string value) + { + if (string.IsNullOrEmpty(value)) + return value; + + // Try to parse as DateTime using standard formats + // This includes ISO 8601, RFC 1123, and common date formats + if (DateTime.TryParse(value, out DateTime dateTime)) + { + return dateTime; + } + + // If not a valid date, return as string + return value; + } +} \ No newline at end of file diff --git a/JsonFlatFileDataStore/GlobalUsings.cs b/JsonFlatFileDataStore/GlobalUsings.cs index 76d1fbe..f3616cf 100644 --- a/JsonFlatFileDataStore/GlobalUsings.cs +++ b/JsonFlatFileDataStore/GlobalUsings.cs @@ -1 +1,2 @@ -global using System; \ No newline at end of file +global using System; +global using System.Text.Json; \ No newline at end of file diff --git a/JsonFlatFileDataStore/IDataStore.cs b/JsonFlatFileDataStore/IDataStore.cs index 46b28c9..c387c35 100644 --- a/JsonFlatFileDataStore/IDataStore.cs +++ b/JsonFlatFileDataStore/IDataStore.cs @@ -1,8 +1,8 @@ using System.Collections.Generic; +using System.Dynamic; using System.Threading.Tasks; namespace JsonFlatFileDataStore; - /// /// JSON data store /// diff --git a/JsonFlatFileDataStore/IDocumentCollection.cs b/JsonFlatFileDataStore/IDocumentCollection.cs index 5afe6da..5bac2f3 100644 --- a/JsonFlatFileDataStore/IDocumentCollection.cs +++ b/JsonFlatFileDataStore/IDocumentCollection.cs @@ -1,8 +1,8 @@ using System.Collections.Generic; +using System.Dynamic; using System.Threading.Tasks; namespace JsonFlatFileDataStore; - /// /// Collection of items /// diff --git a/JsonFlatFileDataStore/JsonFlatFileDataStore.csproj b/JsonFlatFileDataStore/JsonFlatFileDataStore.csproj index ef57f0c..6119c7f 100644 --- a/JsonFlatFileDataStore/JsonFlatFileDataStore.csproj +++ b/JsonFlatFileDataStore/JsonFlatFileDataStore.csproj @@ -8,7 +8,7 @@ https://github.com/ttu/json-flatfile-datastore git 2.4.2 - + Simple JSON flat file data store json flat file data store database linq True @@ -31,13 +31,13 @@ - - + + - - + + \ No newline at end of file diff --git a/JsonFlatFileDataStore/NewtonsoftDateTimeConverter.cs b/JsonFlatFileDataStore/NewtonsoftDateTimeConverter.cs new file mode 100644 index 0000000..5872cba --- /dev/null +++ b/JsonFlatFileDataStore/NewtonsoftDateTimeConverter.cs @@ -0,0 +1,68 @@ +using System.Globalization; +using System.Text.Json.Serialization; + +namespace JsonFlatFileDataStore; +/// +/// Custom DateTime converter for System.Text.Json that provides compatibility with Newtonsoft.Json DateTime formats. +/// +/// This converter was created during the migration from Newtonsoft.Json to System.Text.Json to handle +/// DateTime strings that were serialized by Newtonsoft.Json, particularly the format "yyyy-MM-ddTHH:mm:ss" +/// without timezone designators, which System.Text.Json's default converter doesn't support. +/// +/// Supported formats: +/// - "yyyy-MM-ddTHH:mm:ss.FFFFFFFK" (full precision with timezone) +/// - "yyyy-MM-ddTHH:mm:ss.FFFFFFF" (full precision without timezone) +/// - "yyyy-MM-ddTHH:mm:ssK" (seconds with timezone) +/// - "yyyy-MM-ddTHH:mm:ss" (seconds without timezone - Newtonsoft.Json default) +/// - "yyyy-MM-dd" (date only) +/// +public class NewtonsoftDateTimeConverter : JsonConverter +{ + private readonly string _defaultFormat = "yyyy'-'MM'-'dd'T'HH':'mm':'ss.FFFFFFFK"; // ISO 8601 format + + public override DateTime Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType == JsonTokenType.String) + { + string dateString = reader.GetString(); + + // Common formats to try + string[] formats = new[] + { + "yyyy-MM-ddTHH:mm:ss.FFFFFFFK", // Full precision with timezone + "yyyy-MM-ddTHH:mm:ss.FFFFFFF", // Full precision without timezone + "yyyy-MM-ddTHH:mm:ssK", // Seconds with timezone + "yyyy-MM-ddTHH:mm:ss", // Seconds without timezone (Newtonsoft default) + "yyyy-MM-dd" // Date only + }; + + // Try parsing with various formats + foreach (var format in formats) + { + // Try with RoundtripKind first (for formats with timezone) + if (DateTime.TryParseExact(dateString, format, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out DateTime date)) + { + return date; + } + // Try without RoundtripKind for formats without timezone + if (DateTime.TryParseExact(dateString, format, CultureInfo.InvariantCulture, DateTimeStyles.None, out date)) + { + return date; + } + } + + // Last resort: try general parsing + if (DateTime.TryParse(dateString, CultureInfo.InvariantCulture, DateTimeStyles.None, out DateTime parsedDate)) + { + return parsedDate; + } + } + throw new JsonException($"Invalid date format: {reader.GetString()}"); + } + + public override void Write(Utf8JsonWriter writer, DateTime value, JsonSerializerOptions options) + { + // Write DateTime in ISO 8601 format (default for Newtonsoft.Json) + writer.WriteStringValue(value.ToString(_defaultFormat, CultureInfo.InvariantCulture)); + } +} \ No newline at end of file diff --git a/JsonFlatFileDataStore/ObjectExtensions.cs b/JsonFlatFileDataStore/ObjectExtensions.cs index 37a7409..c6df095 100644 --- a/JsonFlatFileDataStore/ObjectExtensions.cs +++ b/JsonFlatFileDataStore/ObjectExtensions.cs @@ -4,9 +4,7 @@ using System.Dynamic; using System.Linq; using System.Reflection; -using Newtonsoft.Json; -using Newtonsoft.Json.Converters; -using Newtonsoft.Json.Linq; +using System.Text.Json.Nodes; namespace JsonFlatFileDataStore; @@ -22,8 +20,15 @@ internal static void CopyProperties(object source, object destination) if (source == null || destination == null) throw new Exception("source or/and destination objects are null"); - if (source is JToken || IsDictionary(source.GetType())) - source = JsonConvert.DeserializeObject(JsonConvert.SerializeObject(source), new ExpandoObjectConverter()); + if (source is JsonNode || IsDictionary(source.GetType())) + { + var options = new JsonSerializerOptions + { + Converters = { new SystemExpandoObjectConverter() }, + PropertyNameCaseInsensitive = true + }; + source = JsonSerializer.Deserialize(JsonSerializer.Serialize(source), options); + } if (destination is ExpandoObject) HandleExpando(source, destination); @@ -33,10 +38,13 @@ internal static void CopyProperties(object source, object destination) internal static void AddDataToField(object item, string fieldName, dynamic data) { - if (item is JToken) + if (item is JsonNode jsonNode) { - dynamic jTokenItem = item; - jTokenItem[fieldName] = data; + // JsonNode from System.Text.Json + if (jsonNode is JsonObject jsonObject) + { + jsonObject[fieldName] = JsonValue.Create(data); + } } else if (item is ExpandoObject) { @@ -288,7 +296,23 @@ private static void HandleExpando(object source, object destination) } else { - ((IDictionary)destination)[srcProp.Name] = GetValue(source, srcProp); + // Find the property key with case-insensitive comparison + // Newtonsoft.Json: Had built-in case-insensitive property matching + // System.Text.Json: Requires custom implementation for backward compatibility + // This ensures updating "Age" correctly updates "age" in the destination + var destDict = (IDictionary)destination; + var existingKey = destDict.Keys.FirstOrDefault(k => string.Equals(k, srcProp.Name, StringComparison.OrdinalIgnoreCase)); + + if (existingKey != null) + { + // Update the existing property (preserving the original casing) + destDict[existingKey] = GetValue(source, srcProp); + } + else + { + // Add new property with the source property name + destDict[srcProp.Name] = GetValue(source, srcProp); + } } } } @@ -297,22 +321,30 @@ private static void HandleExpandoEnumerable(object source, object destination, d { var destExpandoDict = ((IDictionary)destination); - if (!destExpandoDict.ContainsKey(srcProp.Name)) - destExpandoDict.Add(srcProp.Name, CreateInstance(srcProp.PropertyType)); + // Find existing key with case-insensitive comparison + // Newtonsoft.Json: Built-in case-insensitive matching + // System.Text.Json: Custom implementation for backward compatibility + var existingKey = destExpandoDict.Keys.FirstOrDefault(k => string.Equals(k, srcProp.Name, StringComparison.OrdinalIgnoreCase)); - var targetArray = (IList)destExpandoDict[srcProp.Name]; + if (existingKey == null) + { + existingKey = srcProp.Name; + destExpandoDict.Add(existingKey, CreateInstance(srcProp.PropertyType)); + } + + var targetArray = (IList)destExpandoDict[existingKey]; var sourceArray = (IList)GetValue(source, srcProp); if (sourceArray == null) { - destExpandoDict[srcProp.Name] = null; + destExpandoDict[existingKey] = null; return; } if (targetArray == null) { targetArray = CreateInstance(srcProp.PropertyType); - destExpandoDict[srcProp.Name] = targetArray; + destExpandoDict[existingKey] = targetArray; } Type GetTypeFromTargetItem(IList target, int index) @@ -361,21 +393,37 @@ private static void HandleExpandoObject(object source, object destination, dynam { var destExpandoDict = ((IDictionary)destination); - if (!destExpandoDict.ContainsKey(srcProp.Name)) - destExpandoDict.Add(srcProp.Name, CreateInstance(srcProp.PropertyType)); + // Find existing key with case-insensitive comparison + // Newtonsoft.Json: Built-in case-insensitive matching + // System.Text.Json: Custom implementation for backward compatibility + var existingKey = destExpandoDict.Keys.FirstOrDefault(k => string.Equals(k, srcProp.Name, StringComparison.OrdinalIgnoreCase)); + + if (existingKey == null) + { + existingKey = srcProp.Name; + destExpandoDict.Add(existingKey, CreateInstance(srcProp.PropertyType)); + } var sourceValue = GetValue(source, srcProp); - HandleExpando(sourceValue, destExpandoDict[srcProp.Name]); + HandleExpando(sourceValue, destExpandoDict[existingKey]); } private static void HandleExpandoDictionary(object source, object destination, dynamic srcProp) { var destExpandoDict = ((IDictionary)destination); - if (!destExpandoDict.ContainsKey(srcProp.Name)) - destExpandoDict.Add(srcProp.Name, CreateInstance(srcProp.PropertyType)); + // Find existing key with case-insensitive comparison + // Newtonsoft.Json: Built-in case-insensitive matching + // System.Text.Json: Custom implementation for backward compatibility + var existingKey = destExpandoDict.Keys.FirstOrDefault(k => string.Equals(k, srcProp.Name, StringComparison.OrdinalIgnoreCase)); + + if (existingKey == null) + { + existingKey = srcProp.Name; + destExpandoDict.Add(existingKey, CreateInstance(srcProp.PropertyType)); + } - var targetDict = (IDictionary)destExpandoDict[srcProp.Name]; + var targetDict = (IDictionary)destExpandoDict[existingKey]; var sourceDict = (IDictionary)GetValue(source, srcProp); targetDict.Clear(); diff --git a/README.md b/README.md index 307cac9..e261ea2 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,10 @@ A lightweight, JSON-based data storage solution, ideal for small applications an * .NET implementation & version support: [.NET Standard 2.0](https://docs.microsoft.com/en-us/dotnet/standard/net-standard?tabs=net-standard-2-0#select-net-standard-version) * For example, .NET 6, .NET Core 2.0, .NET Framework 4.6.1 +**Major Version Changes** + +* Version 3.0: Uses `System.Text.Json` instead of `Newtonsoft.Json` for JSON serialization and deserialization. This change reduces dependencies, as `System.Text.Json` is part of the .NET runtime. + **Docs Website** [https://ttu.github.io/json-flatfile-datastore/](https://ttu.github.io/json-flatfile-datastore/) @@ -671,9 +675,111 @@ collection2.ReplaceOne(e => e.id == 11, dynamicUser as object); collection2.ReplaceOne((Predicate)(e => e.id == 11), dynamicUser); ``` -## C# Language Version +## System.Text.Json vs Newtonsoft.Json -The main library (`JsonFlatFileDataStore`) uses C# 10 language features and targets .NET Standard 2.0. Most C# 10 features are compiler-only and work with .NET Standard 2.0. However, some C# 10 features require runtime support and are not compatible with .NET Standard 2.0. +Starting from version 3.x, JSON Flat File Data Store uses `System.Text.Json` instead of `Newtonsoft.Json` for JSON serialization and deserialization. This change reduces external dependencies, as `System.Text.Json` is part of the .NET runtime. + +**Note**: To maintain backward compatibility with Newtonsoft.Json behavior, custom parsing logic has been implemented for: +- **Automatic date parsing** for dynamic types (parsing date strings to DateTime) +- **Case-insensitive property matching** when updating ExpandoObjects (e.g., updating `Age` correctly updates `age`) + +These features were built-in to Newtonsoft.Json but required custom implementation in System.Text.Json. This may have a slight performance impact compared to using System.Text.Json without custom converters. For performance-critical applications, consider using strongly-typed collections. + +### Key Differences for Users + +#### 1. Working with Dynamic JSON Objects + +**Newtonsoft.Json (versions < 3.0):** +```csharp +// JToken/JObject had implicit conversion operators +var employeeJson = JToken.Parse("{ \"id\": 1, \"name\": \"John\" }"); +await collection.InsertOneAsync(employeeJson); + +// Direct comparison worked due to implicit conversions +Assert.Equal(1, employeeJson["id"]); +``` + +**System.Text.Json (version 3.x+):** +```csharp +// JsonNode is the equivalent of JToken +var employeeJson = JsonNode.Parse("{ \"id\": 1, \"name\": \"John\" }"); +await collection.InsertOneAsync(employeeJson); + +// Explicit conversion required using GetValue() +Assert.Equal(1, employeeJson["id"].GetValue()); +Assert.Equal("John", employeeJson["name"].GetValue()); +``` + +#### 2. Dynamic Data Types Support + +Both versions support the same dynamic data types: +* `Anonymous types` +* `ExpandoObject` +* `Dictionary` +* JSON objects: + * versions < 3.0: `JToken`, `JObject`, `JArray` (Newtonsoft.Json) + * version 3.x+: `JsonNode`, `JsonObject`, `JsonArray` (System.Text.Json) + +#### 3. Creating Update Data + +**Newtonsoft.Json (versions < 3.0):** +```csharp +var patchData = new Dictionary +{ + { "Age", 41 }, + { "Name", "James" } +}; +var jobject = JObject.FromObject(patchData); +dynamic patchExpando = JsonConvert.DeserializeObject(jobject.ToString()); + +await collection.UpdateOneAsync(e => e.Id == 12, patchExpando); +``` + +**System.Text.Json (version 3.x+):** +```csharp +var patchData = new Dictionary +{ + { "Age", 41 }, + { "Name", "James" } +}; +var jsonString = JsonSerializer.Serialize(patchData); +var options = new JsonSerializerOptions { Converters = { new SystemExpandoObjectConverter() } }; +dynamic patchExpando = JsonSerializer.Deserialize(jsonString, options); + +await collection.UpdateOneAsync(e => e.Id == 12, patchExpando); +``` + +#### 4. Date Handling with Dynamic Types + +Both versions automatically parse date strings when using dynamic types: + +```csharp +// Both Newtonsoft.Json and System.Text.Json automatically parse date strings +dynamic itemDynamic = store.GetItem("myDate"); +// itemDynamic is automatically converted to DateTime +int year = itemDynamic.Year; // Works in both versions! + +// Typed retrieval also works as expected +var itemTyped = store.GetItem("myDate"); +int year = itemTyped.Year; // Works! +``` + +**Performance Note**: In version 3.x, to maintain backward compatibility with Newtonsoft.Json, custom logic has been implemented for: +1. **Automatic date parsing**: For **dynamic types only**, attempts to parse all string values as dates using `DateTime.TryParse()`. This performance impact **only affects dynamic/ExpandoObject collections** - typed collections (`GetCollection()`) only parse strings that are actually mapped to DateTime properties, making them more efficient. +2. **Case-insensitive property matching**: When updating ExpandoObjects, finds existing properties using case-insensitive comparison + +These features were built-in to Newtonsoft.Json but require custom implementation in System.Text.Json. The date parsing performance impact **only affects dynamic types** - if performance is critical, use strongly-typed collections (`GetCollection()`) where possible. + +### Migration Notes + +* **API remains the same**: Public APIs for collections and data store operations are unchanged. +* **JSON parsing**: Replace `JToken.Parse()` with `JsonNode.Parse()`. +* **Value extraction**: Use `.GetValue()` when accessing values from `JsonNode`. +* **Date handling**: Both versions automatically parse date strings to DateTime for dynamic types (backward compatible). +* **Converters**: If you were using custom Newtonsoft.Json converters, you'll need to rewrite them for System.Text.Json. +* **Case-insensitive property matching**: Both versions support case-insensitive property updates (e.g., updating `Age` will correctly update `age`). +* **Performance considerations**: Custom parsing logic for backward compatibility (automatic date parsing and case-insensitive property matching) may impact performance with large dynamic datasets. Use strongly-typed collections (`GetCollection()`) for better performance when possible. +* **Implementation note**: Features like automatic date parsing and case-insensitive property updates were built-in to Newtonsoft.Json but required custom implementation for System.Text.Json to maintain backward compatibility. ## Unit Tests & Benchmarks @@ -688,6 +794,10 @@ Run benchmarks from the command line: $ dotnet run --configuration Release --project JsonFlatFileDataStore.Benchmark\JsonFlatFileDataStore.Benchmark.csproj ``` +## C# Language Version + +The main library (`JsonFlatFileDataStore`) uses C# 10 language features and targets .NET Standard 2.0. Most C# 10 features are compiler-only and work with .NET Standard 2.0. However, some C# 10 features require runtime support and are not compatible with .NET Standard 2.0. + ## API API is heavily influenced by MongoDB's C# API, so switching to the MongoDB or [DocumentDB](https://docs.microsoft.com/en-us/azure/documentdb/documentdb-protocol-mongodb) might be easy.