Detect large strings, and don't grow the buffer when processing them (#397)

Currently, large strings will grow the internal buffer. Instead, have a largeStringMax limit, split strings greater than this limit into segments and write those flushing [but not growing] when needed.

Author: rbygrave

json-core/src/main/java/io/avaje/json/stream/core/JGenerator.java

@@ -56,6 +56,7 @@ class JGenerator implements JsonGenerator {
   private static final int OP_END = 4;
 
   private final Grisu3.FastDtoaBuilder doubleBuilder = new Grisu3.FastDtoaBuilder();
+  private final int largeStringMax;
   private byte[] buffer;
   private JsonOutput target;
   private int lastOp;
@@ -77,6 +78,8 @@ class JGenerator implements JsonGenerator {
 
   JGenerator(final byte[] buffer) {
     this.buffer = buffer;
+    // each char can take up to 6 bytes when Unicode escaped, round down 1/8 number of chars
+    this.largeStringMax = buffer.length >> 3;
   }
 
   @Override
@@ -132,26 +135,50 @@ private void writeByte(final byte value) {
 
   private void writeString(final String value) {
     final int len = value.length();
+    if (len > largeStringMax) {
+      writeLargeString(value);
+      return;
+    }
     if (position + (len << 2) + (len << 1) + 2 >= buffer.length) {
       enlargeOrFlush(position, (len << 2) + (len << 1) + 2);
     }
-    final byte[] _result = buffer;
-    _result[position] = QUOTE;
-    int cur = position + 1;
-    for (int i = 0; i < len; i++) {
+    buffer[position++] = QUOTE;
+    writeStringSegment(value, 0, len);
+    buffer[position++] = QUOTE;
+  }
+
+  private void writeStringSegment(String value, int i, int end) {
+    int cur = position;
+    for (;i < end; i++) {
       final char c = value.charAt(i);
       if (c > 31 && c != '"' && c != '\\' && c < 126) {
-        _result[cur++] = (byte) c;
+        buffer[cur++] = (byte) c;
       } else {
-        writeQuotedString(value, i, cur, len);
+        writeStringEscape(value, i, cur, end);
         return;
       }
     }
-    _result[cur] = QUOTE;
-    position = cur + 1;
+    position = cur;
+  }
+
+  /** Break a large string into segments and flush when necessary */
+  private void writeLargeString(String text) {
+    writeByte(QUOTE);
+    int left = text.length();
+    int offset = 0;
+    while (left > 0) {
+      final int len = Math.min(largeStringMax, left);
+      if (position + (len << 2) + (len << 1) + 2 >= buffer.length) {
+        enlargeOrFlush(position, 0); // just flush
+      }
+      writeStringSegment(text, offset, offset + len);
+      offset += len;
+      left -= len;
+    }
+    writeByte(QUOTE);
   }
 
-  private void writeQuotedString(final String str, int i, int cur, final int len) {
+  private void writeStringEscape(final String str, int i, int cur, final int len) {
     final byte[] _result = this.buffer;
     for (; i < len; i++) {
       final char c = str.charAt(i);
@@ -327,8 +354,7 @@ private void writeQuotedString(final String str, int i, int cur, final int len)
         }
       }
     }
-    _result[cur] = QUOTE;
-    position = cur + 1;
+    position = cur;
   }
 
   @SuppressWarnings("deprecation")

json-core/src/test/java/io/avaje/json/stream/core/JsonWriterTest.java

@@ -141,6 +141,55 @@ void using_names() {
 
     String asJson = os.toString();
     assertThat(asJson).isEqualTo("[{\"one\":\"hello\",\"size\":43},{\"one\":\"another\",\"active\":true,\"flags\":[42,43]}]");
+  }
+
+  @Test
+  void largeString() {
+    ByteArrayOutputStream os = new ByteArrayOutputStream();
+    JGenerator dJsonWriter = new JGenerator(200);
+    dJsonWriter.prepare(JsonOutput.ofStream(os));
+
+    JsonWriteAdapter fw = new JsonWriteAdapter(dJsonWriter, HybridBufferRecycler.shared(), true, true);
+
+    String largeValue = "_123456789_123456789_123456789_123456789_123456789".repeat(11);
+
+    fw.beginObject();
+    fw.name("key");
+    fw.value(largeValue);
+    fw.endObject();
+    fw.close();
+
+    String jsonResult = new String(os.toByteArray(), 0, os.toByteArray().length);
+    assertThat(jsonResult).isEqualTo("{\"key\":\"" + largeValue + "\"}");
+
+    byte[] effectiveBufferSize = dJsonWriter.ensureCapacity(0);
+    assertThat(effectiveBufferSize.length)
+      .describedAs("internal buffer should not grow")
+      .isEqualTo(200);
+  }
+
+  @Test
+  void largeStringUnicode() {
+    ByteArrayOutputStream os = new ByteArrayOutputStream();
+    JGenerator dJsonWriter = new JGenerator(100);
+    dJsonWriter.prepare(JsonOutput.ofStream(os));
+
+    JsonWriteAdapter fw = new JsonWriteAdapter(dJsonWriter, HybridBufferRecycler.shared(), true, true);
+
+    String largeValue = "_12£45Ã789ǣ123456789_123456789Ŕ123456789_123456789".repeat(11);
+
+    fw.beginObject();
+    fw.name("key");
+    fw.value(largeValue);
+    fw.endObject();
+    fw.close();
+
+    String jsonResult = new String(os.toByteArray(), 0, os.toByteArray().length);
+    assertThat(jsonResult).isEqualTo("{\"key\":\"" + largeValue + "\"}");
 
+    byte[] effectiveBufferSize = dJsonWriter.ensureCapacity(0);
+    assertThat(effectiveBufferSize.length)
+      .describedAs("internal buffer should not grow")
+      .isEqualTo(100);
   }
 }