feat: Remove string truncation for events (#5095)

Remove the default string limit applied when serializing envelopes.

The `max_value_length` parameter now accepts `None`, which disables string truncation. The default value is changed to `None`.

The change follows on from raising the limit in #4632.

Replaces `DEFAULT_MAX_VALUE_LENGTH` with the value 1024 in tests, which was the original limit.

Closes #4988

Author: alexander-alderman-webb

sentry_sdk/client.py

@@ -34,7 +34,6 @@
 from sentry_sdk.transport import BaseHttpTransport, make_transport
 from sentry_sdk.consts import (
     SPANDATA,
-    DEFAULT_MAX_VALUE_LENGTH,
     DEFAULT_OPTIONS,
     INSTRUMENTER,
     VERSION,
@@ -584,7 +583,7 @@ def _prepare_event(
                                     "include_local_variables", True
                                 ),
                                 max_value_length=self.options.get(
-                                    "max_value_length", DEFAULT_MAX_VALUE_LENGTH
+                                    "max_value_length", None
                                 ),
                             ),
                             "crashed": False,

sentry_sdk/consts.py

@@ -2,12 +2,6 @@
 from enum import Enum
 from typing import TYPE_CHECKING
 
-# up top to prevent circular import due to integration import
-# This is more or less an arbitrary large-ish value for now, so that we allow
-# pretty long strings (like LLM prompts), but still have *some* upper limit
-# until we verify that removing the trimming completely is safe.
-DEFAULT_MAX_VALUE_LENGTH = 100_000
-
 DEFAULT_MAX_STACK_FRAMES = 100
 DEFAULT_ADD_FULL_STACK = False
 
@@ -996,7 +990,7 @@ def __init__(
         ],  # type: Optional[Sequence[str]]
         functions_to_trace=[],  # type: Sequence[Dict[str, str]]  # noqa: B006
         event_scrubber=None,  # type: Optional[sentry_sdk.scrubber.EventScrubber]
-        max_value_length=DEFAULT_MAX_VALUE_LENGTH,  # type: int
+        max_value_length=None,  # type: Optional[int]
         enable_backpressure_handling=True,  # type: bool
         error_sampler=None,  # type: Optional[Callable[[Event, Hint], Union[float, bool]]]
         enable_db_query_source=True,  # type: bool

sentry_sdk/serializer.py

@@ -119,7 +119,7 @@ def serialize(event, **kwargs):
     * Annotating the payload with the _meta field whenever trimming happens.
 
     :param max_request_body_size: If set to "always", will never trim request bodies.
-    :param max_value_length: The max length to strip strings to, defaults to sentry_sdk.consts.DEFAULT_MAX_VALUE_LENGTH
+    :param max_value_length: The max length to strip strings to, or None to disable string truncation. Defaults to None.
     :param is_vars: If we're serializing vars early, we want to repr() things that are JSON-serializable to make their type more apparent. For example, it's useful to see the difference between a unicode-string and a bytestring when viewing a stacktrace.
     :param custom_repr: A custom repr function that runs before safe_repr on the object to be serialized. If it returns None or throws internally, we will fallback to safe_repr.
 

sentry_sdk/utils.py

@@ -29,7 +29,6 @@
 from sentry_sdk.consts import (
     DEFAULT_ADD_FULL_STACK,
     DEFAULT_MAX_STACK_FRAMES,
-    DEFAULT_MAX_VALUE_LENGTH,
     EndpointType,
 )
 from sentry_sdk._types import Annotated, AnnotatedValue, SENSITIVE_DATA_SUBSTITUTE
@@ -730,7 +729,7 @@ def single_exception_from_error_tuple(
     if client_options is None:
         include_local_variables = True
         include_source_context = True
-        max_value_length = DEFAULT_MAX_VALUE_LENGTH  # fallback
+        max_value_length = None  # fallback
         custom_repr = None
     else:
         include_local_variables = client_options["include_local_variables"]
@@ -1211,7 +1210,7 @@ def strip_string(value, max_length=None):
         return value
 
     if max_length is None:
-        max_length = DEFAULT_MAX_VALUE_LENGTH
+        return value
 
     byte_size = _get_size_in_bytes(value)
     text_size = len(value)

tests/integrations/bottle/test_bottle.py

@@ -5,7 +5,6 @@
 from io import BytesIO
 from bottle import Bottle, debug as set_debug, abort, redirect, HTTPResponse
 from sentry_sdk import capture_message
-from sentry_sdk.consts import DEFAULT_MAX_VALUE_LENGTH
 from sentry_sdk.integrations.bottle import BottleIntegration
 from sentry_sdk.serializer import MAX_DATABAG_BREADTH
 
@@ -121,10 +120,17 @@ def index():
     assert event["exception"]["values"][0]["mechanism"]["handled"] is False
 
 
-def test_large_json_request(sentry_init, capture_events, app, get_client):
-    sentry_init(integrations=[BottleIntegration()], max_request_body_size="always")
+@pytest.mark.parametrize("max_value_length", [1024, None])
+def test_large_json_request(
+    sentry_init, capture_events, app, get_client, max_value_length
+):
+    sentry_init(
+        integrations=[BottleIntegration()],
+        max_request_body_size="always",
+        max_value_length=max_value_length,
+    )
 
-    data = {"foo": {"bar": "a" * (DEFAULT_MAX_VALUE_LENGTH + 10)}}
+    data = {"foo": {"bar": "a" * (1034)}}
 
     @app.route("/", method="POST")
     def index():
@@ -144,15 +150,17 @@ def index():
     assert response[1] == "200 OK"
 
     (event,) = events
-    assert event["_meta"]["request"]["data"]["foo"]["bar"] == {
-        "": {
-            "len": DEFAULT_MAX_VALUE_LENGTH + 10,
-            "rem": [
-                ["!limit", "x", DEFAULT_MAX_VALUE_LENGTH - 3, DEFAULT_MAX_VALUE_LENGTH]
-            ],
+
+    if max_value_length:
+        assert event["_meta"]["request"]["data"]["foo"]["bar"] == {
+            "": {
+                "len": 1034,
+                "rem": [["!limit", "x", 1021, 1024]],
+            }
         }
-    }
-    assert len(event["request"]["data"]["foo"]["bar"]) == DEFAULT_MAX_VALUE_LENGTH
+        assert len(event["request"]["data"]["foo"]["bar"]) == 1024
+    else:
+        assert len(event["request"]["data"]["foo"]["bar"]) == 1034
 
 
 @pytest.mark.parametrize("data", [{}, []], ids=["empty-dict", "empty-list"])
@@ -179,10 +187,17 @@ def index():
     assert event["request"]["data"] == data
 
 
-def test_medium_formdata_request(sentry_init, capture_events, app, get_client):
-    sentry_init(integrations=[BottleIntegration()], max_request_body_size="always")
+@pytest.mark.parametrize("max_value_length", [1024, None])
+def test_medium_formdata_request(
+    sentry_init, capture_events, app, get_client, max_value_length
+):
+    sentry_init(
+        integrations=[BottleIntegration()],
+        max_request_body_size="always",
+        max_value_length=max_value_length,
+    )
 
-    data = {"foo": "a" * (DEFAULT_MAX_VALUE_LENGTH + 10)}
+    data = {"foo": "a" * (1034)}
 
     @app.route("/", method="POST")
     def index():
@@ -199,15 +214,17 @@ def index():
     assert response[1] == "200 OK"
 
     (event,) = events
-    assert event["_meta"]["request"]["data"]["foo"] == {
-        "": {
-            "len": DEFAULT_MAX_VALUE_LENGTH + 10,
-            "rem": [
-                ["!limit", "x", DEFAULT_MAX_VALUE_LENGTH - 3, DEFAULT_MAX_VALUE_LENGTH]
-            ],
+
+    if max_value_length:
+        assert event["_meta"]["request"]["data"]["foo"] == {
+            "": {
+                "len": 1034,
+                "rem": [["!limit", "x", 1021, 1024]],
+            }
         }
-    }
-    assert len(event["request"]["data"]["foo"]) == DEFAULT_MAX_VALUE_LENGTH
+        assert len(event["request"]["data"]["foo"]) == 1024
+    else:
+        assert len(event["request"]["data"]["foo"]) == 1034
 
 
 @pytest.mark.parametrize("input_char", ["a", b"a"])
@@ -241,11 +258,16 @@ def index():
     assert not event["request"]["data"]
 
 
-def test_files_and_form(sentry_init, capture_events, app, get_client):
-    sentry_init(integrations=[BottleIntegration()], max_request_body_size="always")
+@pytest.mark.parametrize("max_value_length", [1024, None])
+def test_files_and_form(sentry_init, capture_events, app, get_client, max_value_length):
+    sentry_init(
+        integrations=[BottleIntegration()],
+        max_request_body_size="always",
+        max_value_length=max_value_length,
+    )
 
     data = {
-        "foo": "a" * (DEFAULT_MAX_VALUE_LENGTH + 10),
+        "foo": "a" * (1034),
         "file": (BytesIO(b"hello"), "hello.txt"),
     }
 
@@ -266,15 +288,16 @@ def index():
     assert response[1] == "200 OK"
 
     (event,) = events
-    assert event["_meta"]["request"]["data"]["foo"] == {
-        "": {
-            "len": DEFAULT_MAX_VALUE_LENGTH + 10,
-            "rem": [
-                ["!limit", "x", DEFAULT_MAX_VALUE_LENGTH - 3, DEFAULT_MAX_VALUE_LENGTH]
-            ],
+    if max_value_length:
+        assert event["_meta"]["request"]["data"]["foo"] == {
+            "": {
+                "len": 1034,
+                "rem": [["!limit", "x", 1021, 1024]],
+            }
         }
-    }
-    assert len(event["request"]["data"]["foo"]) == DEFAULT_MAX_VALUE_LENGTH
+        assert len(event["request"]["data"]["foo"]) == 1024
+    else:
+        assert len(event["request"]["data"]["foo"]) == 1034
 
     assert event["_meta"]["request"]["data"]["file"] == {
         "": {

tests/integrations/falcon/test_falcon.py

@@ -5,7 +5,6 @@
 import falcon
 import falcon.testing
 import sentry_sdk
-from sentry_sdk.consts import DEFAULT_MAX_VALUE_LENGTH
 from sentry_sdk.integrations.falcon import FalconIntegration
 from sentry_sdk.integrations.logging import LoggingIntegration
 from sentry_sdk.utils import parse_version
@@ -207,10 +206,15 @@ def on_get(self, req, resp):
     assert len(events) == 0
 
 
-def test_falcon_large_json_request(sentry_init, capture_events):
-    sentry_init(integrations=[FalconIntegration()], max_request_body_size="always")
+@pytest.mark.parametrize("max_value_length", [1024, None])
+def test_falcon_large_json_request(sentry_init, capture_events, max_value_length):
+    sentry_init(
+        integrations=[FalconIntegration()],
+        max_request_body_size="always",
+        max_value_length=max_value_length,
+    )
 
-    data = {"foo": {"bar": "a" * (DEFAULT_MAX_VALUE_LENGTH + 10)}}
+    data = {"foo": {"bar": "a" * (1034)}}
 
     class Resource:
         def on_post(self, req, resp):
@@ -228,15 +232,16 @@ def on_post(self, req, resp):
     assert response.status == falcon.HTTP_200
 
     (event,) = events
-    assert event["_meta"]["request"]["data"]["foo"]["bar"] == {
-        "": {
-            "len": DEFAULT_MAX_VALUE_LENGTH + 10,
-            "rem": [
-                ["!limit", "x", DEFAULT_MAX_VALUE_LENGTH - 3, DEFAULT_MAX_VALUE_LENGTH]
-            ],
+    if max_value_length:
+        assert event["_meta"]["request"]["data"]["foo"]["bar"] == {
+            "": {
+                "len": 1034,
+                "rem": [["!limit", "x", 1021, 1024]],
+            }
         }
-    }
-    assert len(event["request"]["data"]["foo"]["bar"]) == DEFAULT_MAX_VALUE_LENGTH
+        assert len(event["request"]["data"]["foo"]["bar"]) == 1024
+    else:
+        assert len(event["request"]["data"]["foo"]["bar"]) == 1034
 
 
 @pytest.mark.parametrize("data", [{}, []], ids=["empty-dict", "empty-list"])