fix: address STJ migration bugs and PR review feedback

- Fix ConvertJsonElement ternary type coercion: (object)l cast prevents
  implicit long→double widening in switch expression
- Make ObjectToInferredTypesConverter configurable with preferInt64 flag
  for ES serializer to match JSON.NET DataObjectConverter behavior
- Fix ElasticSystemTextJsonSerializer: remove ReadStreamToSpan lifetime
  bug (span backed by disposed MemoryStream), deserialize from stream
  directly with MemoryStream fast-path
- Fix Serialize<T> indentation: pass JsonWriterOptions to Utf8JsonWriter
  so SerializationFormatting.Indented actually produces indented output
- Handle exponent notation (1e5) as floating-point in ReadNumber
- Use double consistently (not decimal) for floating-point to match
  JSON.NET behavior
- Fix RenameAll return value: return whether any renames occurred
- Add using var to MemoryStream in EventController and EventPostsJob
- Handle empty response bodies in SendRequestAsAsync (STJ throws on
  empty input, Newtonsoft returned default)
- Fix SerializerTests: put unknown properties at root level to test
  JsonExtensionData→ConvertJsonElement path correctly
- Revert AGENTS.md to main

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: niemyjski

AGENTS.md

@@ -53,26 +53,3 @@ Load from `.github/skills/<name>/SKILL.md` when working in that domain:
 - Use `npm ci` (not `npm install`)
 - Never commit secrets — use environment variables
 - NuGet feeds are in `NuGet.Config` — don't add sources
-
-## Serialization Architecture
-
-The project uses **System.Text.Json (STJ)** exclusively. NEST still brings in Newtonsoft.Json transitively, but all application-level serialization uses STJ:
-
-| Component      | Serializer                        | Notes                                       |
-| -------------- | --------------------------------- | -------------------------------------------- |
-| Elasticsearch  | `ElasticSystemTextJsonSerializer` | Custom `IElasticsearchSerializer` using STJ  |
-| Event Upgrader | `System.Text.Json.Nodes`          | JsonObject/JsonArray for mutable DOM         |
-| Data Storage   | `SystemTextJsonSerializer`        | Via Foundatio's STJ support                  |
-| API            | STJ (built-in)                    | ASP.NET Core default with custom options     |
-
-**Key files:**
-
-- `ElasticSystemTextJsonSerializer.cs` - Custom `IElasticsearchSerializer` for NEST
-- `JsonNodeExtensions.cs` - STJ equivalents of JObject helpers
-- `ObjectToInferredTypesConverter.cs` - Handles JObject/JToken from NEST during STJ serialization
-- `V*_EventUpgrade.cs` - Event version upgraders using JsonObject
-
-**Security:**
-
-- Safe JSON encoding used everywhere (escapes `<`, `>`, `&`, `'` for XSS protection)
-- No `UnsafeRelaxedJsonEscaping` in the codebase

src/Exceptionless.Core/Extensions/JsonNodeExtensions.cs

@@ -222,7 +222,7 @@ public static bool RenameAll(this JsonObject target, string currentName, string
             obj.Rename(currentName, newName);
         }
 
-        return true;
+        return objectsWithProperty.Count > 0;
     }
 
     /// <summary>

src/Exceptionless.Core/Jobs/EventPostsJob.cs

@@ -302,7 +302,7 @@ private async Task RetryEventsAsync(List<PersistentEvent> eventsToRetry, EventPo
         {
             try
             {
-                var stream = new MemoryStream(ev.GetBytes(_serializer));
+                using var stream = new MemoryStream(ev.GetBytes(_serializer));
 
                 // Put this single event back into the queue so we can retry it separately.
                 await _eventPostService.EnqueueAsync(new EventPost(false)

src/Exceptionless.Core/Models/Event.cs

@@ -98,7 +98,7 @@ void IJsonOnDeserialized.OnDeserialized()
         return element.ValueKind switch
         {
             JsonValueKind.String => element.GetString(),
-            JsonValueKind.Number => element.TryGetInt64(out long l) ? l : element.GetDouble(),
+            JsonValueKind.Number => element.TryGetInt64(out long l) ? (object)l : element.GetDouble(),
             JsonValueKind.True => true,
             JsonValueKind.False => false,
             JsonValueKind.Null or JsonValueKind.Undefined => null,

src/Exceptionless.Core/Serialization/ElasticSystemTextJsonSerializer.cs

@@ -59,11 +59,19 @@ private static JsonSerializerOptions CreateOptions(JsonSerializerOptions? baseOp
         options.DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull;
         options.WriteIndented = writeIndented;
 
-        // Insert Elasticsearch converters at the beginning for priority
+        // Replace the default ObjectToInferredTypesConverter with one that returns Int64
+        // for all integers, matching JSON.NET DataObjectConverter behavior. This ensures
+        // Event.Data values round-trip through Elasticsearch with consistent types.
+        var defaultConverter = options.Converters.FirstOrDefault(c => c is ObjectToInferredTypesConverter);
+        if (defaultConverter is not null)
+            options.Converters.Remove(defaultConverter);
+        options.Converters.Insert(0, new ObjectToInferredTypesConverter(preferInt64: true));
+
+        // Insert Elasticsearch converters for priority
         // Order matters: more specific converters should come first
-        options.Converters.Insert(0, new DynamicDictionaryConverter());
-        options.Converters.Insert(1, new Iso8601DateTimeOffsetConverter());
-        options.Converters.Insert(2, new Iso8601DateTimeConverter());
+        options.Converters.Insert(1, new DynamicDictionaryConverter());
+        options.Converters.Insert(2, new Iso8601DateTimeOffsetConverter());
+        options.Converters.Insert(3, new Iso8601DateTimeConverter());
 
         return options;
     }
@@ -79,8 +87,11 @@ private JsonSerializerOptions GetOptions(SerializationFormatting formatting) =>
         if (IsEmptyStream(stream))
             return null;
 
-        var buffer = ReadStreamToSpan(stream);
-        return JsonSerializer.Deserialize(buffer, type, _optionsCompact.Value);
+        // Fast path: MemoryStream with accessible buffer avoids buffering
+        if (stream is MemoryStream ms && ms.TryGetBuffer(out var segment))
+            return JsonSerializer.Deserialize(segment.AsSpan((int)ms.Position), type, _optionsCompact.Value);
+
+        return JsonSerializer.Deserialize(stream, type, _optionsCompact.Value);
     }
 
     /// <inheritdoc />
@@ -89,15 +100,22 @@ private JsonSerializerOptions GetOptions(SerializationFormatting formatting) =>
         if (IsEmptyStream(stream))
             return default;
 
-        var buffer = ReadStreamToSpan(stream);
-        return JsonSerializer.Deserialize<T>(buffer, _optionsCompact.Value);
+        // Fast path: MemoryStream with accessible buffer avoids buffering
+        if (stream is MemoryStream ms && ms.TryGetBuffer(out var segment))
+            return JsonSerializer.Deserialize<T>(segment.AsSpan((int)ms.Position), _optionsCompact.Value);
+
+        return JsonSerializer.Deserialize<T>(stream, _optionsCompact.Value);
     }
 
     /// <inheritdoc />
     public void Serialize<T>(T data, Stream stream, SerializationFormatting formatting = SerializationFormatting.None)
     {
-        using var writer = new Utf8JsonWriter(stream);
         var options = GetOptions(formatting);
+        using var writer = new Utf8JsonWriter(stream, new JsonWriterOptions
+        {
+            Indented = formatting == SerializationFormatting.Indented,
+            Encoder = options.Encoder
+        });
 
         if (data is null)
         {
@@ -153,31 +171,10 @@ public Task SerializeAsync<T>(
 
     #endregion
 
-    #region Stream Helpers
-
     private static bool IsEmptyStream(Stream? stream)
     {
         return stream is null || stream == Stream.Null || (stream.CanSeek && stream.Length == 0);
     }
-
-    private static ReadOnlySpan<byte> ReadStreamToSpan(Stream stream)
-    {
-        // Fast path: if already a MemoryStream with accessible buffer, use it directly
-        if (stream is MemoryStream ms && ms.TryGetBuffer(out var segment))
-        {
-            return segment.AsSpan();
-        }
-
-        // Slow path: copy to new buffer
-        using var buffer = stream.CanSeek
-            ? new MemoryStream((int)stream.Length)
-            : new MemoryStream();
-
-        stream.CopyTo(buffer);
-        return buffer.TryGetBuffer(out var seg) ? seg.AsSpan() : buffer.ToArray();
-    }
-
-    #endregion
 }
 
 #region Elasticsearch-Specific Converters

src/Exceptionless.Core/Serialization/ObjectToInferredTypesConverter.cs

@@ -18,7 +18,7 @@ namespace Exceptionless.Core.Serialization;
 /// </para>
 /// <list type="bullet">
 ///   <item><description><c>true</c>/<c>false</c> → <see cref="bool"/></description></item>
-///   <item><description>Numbers → <see cref="long"/> (if fits) or <see cref="double"/></description></item>
+///   <item><description>Numbers → <see cref="int"/> (if fits), <see cref="long"/>, or <see cref="double"/>; with <c>preferInt64</c>, always <see cref="long"/></description></item>
 ///   <item><description>Strings with ISO 8601 date format → <see cref="DateTimeOffset"/></description></item>
 ///   <item><description>Other strings → <see cref="string"/></description></item>
 ///   <item><description><c>null</c> → <c>null</c></description></item>
@@ -44,6 +44,25 @@ namespace Exceptionless.Core.Serialization;
 /// <seealso href="https://learn.microsoft.com/en-us/dotnet/standard/serialization/system-text-json/converters-how-to#deserialize-inferred-types-to-object-properties"/>
 public sealed class ObjectToInferredTypesConverter : JsonConverter<object?>
 {
+    private readonly bool _preferInt64;
+
+    /// <summary>
+    /// Initializes a new instance with default settings (integers that fit Int32 are returned as <see cref="int"/>).
+    /// </summary>
+    public ObjectToInferredTypesConverter() : this(preferInt64: false) { }
+
+    /// <summary>
+    /// Initializes a new instance with configurable integer handling.
+    /// </summary>
+    /// <param name="preferInt64">
+    /// When <c>true</c>, all integers are returned as <see cref="long"/> to match JSON.NET behavior.
+    /// Used by the Elasticsearch serializer to maintain compatibility with <c>DataObjectConverter</c>.
+    /// </param>
+    public ObjectToInferredTypesConverter(bool preferInt64)
+    {
+        _preferInt64 = preferInt64;
+    }
+
     /// <inheritdoc />
     public override object? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
     {
@@ -103,38 +122,38 @@ public override void Write(Utf8JsonWriter writer, object? value, JsonSerializerO
     ///   <item><description>Serializing back would lose the decimal representation</description></item>
     /// </list>
     /// </remarks>
-    private static object ReadNumber(ref Utf8JsonReader reader)
+    private object ReadNumber(ref Utf8JsonReader reader)
     {
         // Check the raw text to preserve decimal vs integer representation
         // This is critical for data integrity - 0.0 should stay as double, not become 0L
         ReadOnlySpan<byte> rawValue = reader.HasValueSequence
             ? reader.ValueSequence.ToArray()
             : reader.ValueSpan;
 
-        // If the raw text contains a decimal point, treat as floating-point
-        if (rawValue.Contains((byte)'.'))
+        // If the raw text contains a decimal point or exponent, treat as floating-point
+        if (rawValue.Contains((byte)'.') || rawValue.Contains((byte)'e') || rawValue.Contains((byte)'E'))
         {
-            // Try decimal for precise values (e.g., financial data) before double
-            if (reader.TryGetDecimal(out decimal d))
-                return d;
-
-            // Fall back to double for floating-point
             return reader.GetDouble();
         }
 
         // No decimal point - this is an integer
-        // Try int32 first for smaller values, then long for larger integers
-        if (reader.TryGetInt32(out int i))
-            return i;
-
-        if (reader.TryGetInt64(out long l))
-            return l;
+        if (_preferInt64)
+        {
+            // Match JSON.NET DataObjectConverter behavior: always return Int64
+            if (reader.TryGetInt64(out long l))
+                return l;
+        }
+        else
+        {
+            // Default STJ behavior: return smallest fitting integer type
+            if (reader.TryGetInt32(out int i))
+                return i;
 
-        // For very large integers, try decimal first to preserve precision
-        if (reader.TryGetDecimal(out decimal dec))
-            return dec;
+            if (reader.TryGetInt64(out long l))
+                return l;
+        }
 
-        // Fall back to double only if decimal also fails
+        // For very large integers that don't fit in long, fall back to double
         return reader.GetDouble();
     }
 
@@ -160,7 +179,7 @@ private static object ReadNumber(ref Utf8JsonReader reader)
     /// Uses <see cref="StringComparer.OrdinalIgnoreCase"/> for property name matching,
     /// consistent with <see cref="DataDictionary"/> behavior.
     /// </remarks>
-    private static Dictionary<string, object?> ReadObject(ref Utf8JsonReader reader, JsonSerializerOptions options)
+    private Dictionary<string, object?> ReadObject(ref Utf8JsonReader reader, JsonSerializerOptions options)
     {
         var dictionary = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase);
 
@@ -186,7 +205,7 @@ private static object ReadNumber(ref Utf8JsonReader reader)
     /// <summary>
     /// Recursively reads a JSON array into a <see cref="List{T}"/> of objects.
     /// </summary>
-    private static List<object?> ReadArray(ref Utf8JsonReader reader, JsonSerializerOptions options)
+    private List<object?> ReadArray(ref Utf8JsonReader reader, JsonSerializerOptions options)
     {
         var list = new List<object?>();
 
@@ -204,7 +223,7 @@ private static object ReadNumber(ref Utf8JsonReader reader)
     /// <summary>
     /// Reads a single JSON value of any type, dispatching to the appropriate reader method.
     /// </summary>
-    private static object? ReadValue(ref Utf8JsonReader reader, JsonSerializerOptions options)
+    private object? ReadValue(ref Utf8JsonReader reader, JsonSerializerOptions options)
     {
         return reader.TokenType switch
         {

src/Exceptionless.Web/Controllers/EventController.cs

@@ -1115,7 +1115,7 @@ private async Task<ActionResult> GetSubmitEventAsync(string? projectId = null, i
                 charSet = contentTypeHeader.Charset.ToString();
             }
 
-            var stream = new MemoryStream(ev.GetBytes(_serializer));
+            using var stream = new MemoryStream(ev.GetBytes(_serializer));
             await _eventPostService.EnqueueAsync(new EventPost(_appOptions.EnableArchive)
             {
                 ApiVersion = apiVersion,

tests/Exceptionless.Tests/IntegrationTestsBase.cs

@@ -235,6 +235,11 @@ protected async Task<HttpResponseMessage> SendRequestAsync(Action<AppSendBuilder
     {
         var response = await SendRequestAsync(configure);
 
+        // Handle empty response bodies gracefully (e.g., Ok() with no content).
+        // STJ throws on empty input whereas Newtonsoft returned default(T).
+        if (response.Content.Headers.ContentLength == 0)
+            return default;
+
         // All errors are returned as problem details so if we are expecting Problem Details we shouldn't ensure success.
         bool ensureSuccess = !typeof(Microsoft.AspNetCore.Mvc.ProblemDetails).IsAssignableFrom(typeof(T));
         return await response.DeserializeAsync<T>(ensureSuccess);

tests/Exceptionless.Tests/Serializer/Models/DataDictionaryTests.cs

@@ -391,7 +391,7 @@ public void Deserialize_DataDictionaryWithMixedTypesAfterRoundTrip_PreservesAllT
         // Assert
         Assert.NotNull(deserialized);
         Assert.Equal("hello", deserialized["string_value"]);
-        Assert.Equal(42, deserialized["int_value"]); // JSON integers deserialize to int when they fit
+        Assert.Equal(42, deserialized["int_value"]);
         Assert.True(deserialized["bool_value"] as bool?);
     }
 
@@ -584,7 +584,7 @@ public void Deserialize_MixedDataTypesAfterRoundTrip_PreservesAllTypes()
         Assert.Equal("user@test.com", userInfo.Identity);
 
         Assert.Equal("1.0.0", deserialized["@version"]);
-        Assert.Equal(42, deserialized["count"]); // JSON integers deserialize to int when they fit
+        Assert.Equal(42, deserialized["count"]);
         Assert.True(deserialized["enabled"] as bool?);
     }
 
@@ -613,7 +613,7 @@ public void Deserialize_NestedDataDictionaryAfterRoundTrip_PreservesNestedData()
         Assert.Equal("user@test.com", result.Identity);
         Assert.NotNull(result.Data);
         Assert.Equal("custom_value", result.Data["custom_field"]);
-        Assert.Equal(100, result.Data["score"]); // JSON integers deserialize to int when they fit
+        Assert.Equal(100, result.Data["score"]);
     }
 
     [Fact]

tests/Exceptionless.Tests/Serializer/SerializerTests.cs

@@ -26,7 +26,8 @@ public void CanDeserializeEventWithData()
         var ev = _serializer.Deserialize<Event>(json);
 
         // Assert
-        Assert.NotNull(ev?.Data);
+        Assert.NotNull(ev);
+        Assert.NotNull(ev.Data);
         Assert.Single(ev.Data);
         Assert.Equal("Hello", ev.Message);
         Assert.Equal("SomeVal", ev.Data["Blah"]);
@@ -35,21 +36,22 @@ public void CanDeserializeEventWithData()
     [Fact]
     public void CanRoundTripEventWithUnknownProperties()
     {
-        // Arrange
+        // Arrange - unknown properties at root go through [JsonExtensionData] → ConvertJsonElement
         /* language=json */
-        const string json = """{"tags":["One","Two"],"reference_id":"12","message":"Hello","data":{"SomeString":"Hi","SomeBool":false,"SomeNum":1}}""";
+        const string json = """{"tags":["One","Two"],"reference_id":"12","message":"Hello","SomeString":"Hi","SomeBool":false,"SomeNum":1}""";
 
         // Act
         var ev = _serializer.Deserialize<Event>(json);
         string roundTrippedJson = _serializer.SerializeToString(ev);
         var roundTripped = _serializer.Deserialize<Event>(roundTrippedJson);
 
         // Assert
-        Assert.NotNull(ev?.Data);
+        Assert.NotNull(ev);
+        Assert.NotNull(ev.Data);
         Assert.Equal(3, ev.Data.Count);
         Assert.Equal("Hi", ev.Data["SomeString"]);
         Assert.Equal(false, ev.Data["SomeBool"]);
-        Assert.Equal(1, ev.Data["SomeNum"]);
+        Assert.Equal(1L, ev.Data["SomeNum"]);
         Assert.Equal("Hello", ev.Message);
         Assert.NotNull(ev.Tags);
         Assert.Equal(2, ev.Tags.Count);