SL-19707 throw an error if we exceed 200 depth in formatting or parsing

Author: roxanneskelly

llsd/base.py

@@ -31,7 +31,7 @@
 
 ALL_CHARS = str(bytearray(range(256))) if PY2 else bytes(range(256))
 
-
+MAX_FORMAT_DEPTH = 200
 class _LLSD:
     __metaclass__ = abc.ABCMeta
 

llsd/serde_binary.py

@@ -5,7 +5,7 @@
 import uuid
 
 from llsd.base import (_LLSD, LLSDBaseParser, LLSDSerializationError, BINARY_HEADER,
-                       _str_to_bytes, binary, is_integer, is_string, uri)
+                       MAX_FORMAT_DEPTH,_str_to_bytes, binary, is_integer, is_string, uri)
 
 
 try:
@@ -15,7 +15,6 @@
     # Python 3: 'range()' is already lazy
     pass
 
-
 class LLSDBinaryParser(LLSDBaseParser):
     """
     Parse application/llsd+binary to a python object.
@@ -164,15 +163,19 @@ def format_binary(something):
 
 def write_binary(stream, something):
     stream.write(b'<?llsd/binary?>\n')
-    _write_binary_recurse(stream, something)
+    _write_binary_recurse(stream, something, 0)
 
 
-def _write_binary_recurse(stream, something):
+def _write_binary_recurse(stream, something, depth):
     "Binary formatter workhorse."
+
+    if depth > MAX_FORMAT_DEPTH:
+        raise LLSDSerializationError("Cannot serialize depth of more than %d" % MAX_FORMAT_DEPTH)
+
     if something is None:
         stream.write(b'!')
     elif isinstance(something, _LLSD):
-        _write_binary_recurse(stream, something.thing)
+        _write_binary_recurse(stream, something.thing, depth)
     elif isinstance(something, bool):
         stream.write(b'1' if something else b'0')
     elif is_integer(something):
@@ -202,27 +205,27 @@ def _write_binary_recurse(stream, something):
         seconds_since_epoch = calendar.timegm(something.timetuple())
         stream.writelines([b'd', struct.pack('<d', seconds_since_epoch)])
     elif isinstance(something, (list, tuple)):
-        _write_list(stream, something)
+        _write_list(stream, something, depth)
     elif isinstance(something, dict):
         stream.writelines([b'{', struct.pack('!i', len(something))])
         for key, value in something.items():
             key = _str_to_bytes(key)
             stream.writelines([b'k', struct.pack('!i', len(key)), key])
-            _write_binary_recurse(stream, value)
+            _write_binary_recurse(stream, value, depth+1)
         stream.write(b'}')
     else:
         try:
-            return _write_list(stream, list(something))
+            return _write_list(stream, list(something), depth)
         except TypeError:
             raise LLSDSerializationError(
                 "Cannot serialize unknown type: %s (%s)" %
                 (type(something), something))
 
 
-def _write_list(stream, something):
+def _write_list(stream, something, depth):
     stream.writelines([b'[', struct.pack('!i', len(something))])
     for item in something:
-        _write_binary_recurse(stream, item)
+        _write_binary_recurse(stream, item, depth+1)
     stream.write(b']')
 
 

llsd/serde_notation.py

@@ -4,7 +4,7 @@
 import uuid
 
 from llsd.base import (_LLSD, B, LLSDBaseFormatter, LLSDBaseParser, NOTATION_HEADER,
-                       LLSDParseError, LLSDSerializationError, UnicodeType,
+                       MAX_FORMAT_DEPTH, LLSDParseError, LLSDSerializationError, UnicodeType,
                        _format_datestr, _parse_datestr, _str_to_bytes, binary, uri)
 
 
@@ -411,6 +411,11 @@ class LLSDNotationFormatter(LLSDBaseFormatter):
 
     See http://wiki.secondlife.com/wiki/LLSD#Notation_Serialization
     """
+
+    def __init__(self):
+        super(LLSDNotationFormatter, self).__init__()
+        self._depth = 0
+
     def _LLSD(self, v):
         return self._generate(v.thing)
     def _UNDEF(self, v):
@@ -443,18 +448,22 @@ def _DATE(self, v):
     def _ARRAY(self, v):
         self.stream.write(b'[')
         delim = b''
+        self._depth = self._depth + 1
         for item in v:
             self.stream.write(delim)
             self._generate(item)
             delim = b','
+        self._depth = self._depth - 1
         self.stream.write(b']')
     def _MAP(self, v):
         self.stream.write(b'{')
         delim = b''
+        self._depth = self._depth + 1
         for key, value in v.items():
             self.stream.writelines([delim, b"'", self._esc(UnicodeType(key)), b"':"])
             self._generate(value)
             delim = b','
+        self._depth = self._depth - 1
         self.stream.write(b'}')
 
     def _esc(self, data, quote=b"'"):
@@ -466,6 +475,9 @@ def _generate(self, something):
 
         :param something: a python object (typically a dict) to be serialized.
         """
+        if self._depth > MAX_FORMAT_DEPTH:
+            raise LLSDSerializationError("Cannot serialize depth of more than %d" % MAX_FORMAT_DEPTH)
+
         t = type(something)
         handler = self.type_map.get(t)
         if handler:

llsd/serde_xml.py

@@ -3,7 +3,7 @@
 import re
 
 from llsd.base import (_LLSD, ALL_CHARS, LLSDBaseParser, LLSDBaseFormatter, XML_HEADER,
-                       LLSDParseError, LLSDSerializationError, UnicodeType,
+                       MAX_FORMAT_DEPTH, LLSDParseError, LLSDSerializationError, UnicodeType,
                        _format_datestr, _str_to_bytes, _to_python, is_unicode, PY2)
 from llsd.fastest_elementtree import ElementTreeError, fromstring, parse as _parse
 
@@ -24,7 +24,6 @@
     for x in INVALID_XML_BYTES:
         XML_ESC_TRANS[x] = None
 
-
 def remove_invalid_xml_bytes(b):
     """
     Remove characters that aren't allowed in xml.
@@ -76,7 +75,7 @@ def __init__(self, indent_atom = None):
         super(LLSDXMLFormatter, self).__init__()
         self._indent_atom = b''
         self._eol = b''
-        self._indent_level = 0
+        self._depth = 1
 
     def _indent(self):
         pass
@@ -115,15 +114,15 @@ def _DATE(self, v):
         self.stream.writelines([b'<date>', _format_datestr(v), b'</date>', self._eol])
     def _ARRAY(self, v):
         self.stream.writelines([b'<array>', self._eol])
-        self._indent_level = self._indent_level + 1
+        self._depth = self._depth + 1
         for item in v:
             self._indent()
             self._generate(item)
-        self._indent_level = self._indent_level - 1
+        self._depth = self._depth - 1
         self.stream.writelines([b'</array>', self._eol])
     def _MAP(self, v):
         self.stream.writelines([b'<map>', self._eol])
-        self._indent_level = self._indent_level + 1
+        self._depth = self._depth + 1
         for key, value in v.items():
             self._indent()
             if PY2:     # pragma: no cover
@@ -138,12 +137,14 @@ def _MAP(self, v):
                                         self._eol])
             self._indent()
             self._generate(value)
-        self._indent_level = self._indent_level - 1
+        self._depth = self._depth - 1
         self._indent()
         self.stream.writelines([b'</map>', self._eol])
 
     def _generate(self, something):
         "Generate xml from a single python object."
+        if self._depth - 1 > MAX_FORMAT_DEPTH:
+            raise LLSDSerializationError("Cannot serialize depth of more than %d" % MAX_FORMAT_DEPTH)
         t = type(something)
         if t in self.type_map:
             return self.type_map[t](something)
@@ -183,13 +184,12 @@ def __init__(self, indent_atom = b'  '):
         super(LLSDXMLPrettyFormatter, self).__init__()
 
         # Private data used for indentation.
-        self._indent_level = 1
         self._indent_atom = indent_atom
         self._eol = b'\n'
 
     def _indent(self):
         "Write an indentation based on the atom and indentation level."
-        self.stream.writelines([self._indent_atom] * self._indent_level)
+        self.stream.writelines([self._indent_atom] * self._depth)
 
 
 def format_pretty_xml(something):

tests/bench.py

@@ -84,7 +84,7 @@ def binary_stream():
 def build_deep_xml():
     deep_data = {}
     curr_data = deep_data
-    for i in range(250):
+    for i in range(198):
         curr_data["curr_data"] = {}
         curr_data["integer"] = 7
         curr_data["string"] = "string"

tests/llsd_test.py

@@ -527,6 +527,26 @@ def testParseNotationHalfTruncatedHex(self):
     def testParseNotationInvalidHex(self):
         self.assertRaises(llsd.LLSDParseError, self.llsd.parse, b"'\\xzz'")
 
+    def testDeepMap(self):
+        """
+        Test formatting of a deeply nested map
+        """
+
+        test_map = {"foo":"bar", "depth":0}
+        max_depth = 199
+        for depth in range(max_depth):
+            test_map = {"foo":"bar", "depth":depth, "next":test_map}
+
+        # this should not throw an exception.
+        test_notation_out = self.llsd.as_notation(test_map)
+
+        test_notation_parsed = self.llsd.parse(io.BytesIO(test_notation_out))
+        self.assertEqual(test_map, test_notation_parsed)
+
+        test_map = {"foo":"bar", "depth":depth, "next":test_map}
+        # this should throw an exception.
+        self.assertRaises(llsd.LLSDSerializationError, self.llsd.as_notation, test_map)
+
 
 class LLSDBinaryUnitTest(unittest.TestCase):
     """
@@ -964,6 +984,26 @@ def testParseDelimitedString(self):
 
         self.assertEqual('\t\x07\x08\x0c\n\r\t\x0b\x0fp', llsd.parse(delimited_string))
 
+    def testDeepMap(self):
+        """
+        Test formatting of a deeply nested map
+        """
+
+        test_map = {"foo":"bar", "depth":0}
+        max_depth = 199
+        for depth in range(max_depth):
+            test_map = {"foo":"bar", "depth":depth, "next":test_map}
+
+        # this should not throw an exception.
+        test_binary_out = self.llsd.as_binary(test_map)
+
+        test_binary_parsed = self.llsd.parse(io.BytesIO(test_binary_out))
+        self.assertEqual(test_map, test_binary_parsed)
+
+        test_map = {"foo":"bar", "depth":depth, "next":test_map}
+        # this should throw an exception.
+        self.assertRaises(llsd.LLSDSerializationError, self.llsd.as_binary, test_map)
+
 
 
 class LLSDPythonXMLUnitTest(unittest.TestCase):
@@ -1345,20 +1385,6 @@ def testMap(self):
                                   map_within_map_xml)
         self.assertXMLRoundtrip({}, blank_map_xml)
 
-    def testDeepMap(self):
-        """
-        Test that formatting a deeply nested map does not cause a RecursionError
-        """
-
-        test_map = {"foo":"bar", "depth":0, "next":None}
-        max_depth = 200
-        for depth in range(max_depth):
-            test_map = {"foo":"bar", "depth":depth, "next":test_map}
-
-        # this should not throw an exception.
-        test_xml = self.llsd.as_xml(test_map)
-
-
     def testBinary(self):
         """
         Test the parse and serialization of input type : binary.
@@ -1493,6 +1519,26 @@ def testFormatPrettyXML(self):
         self.assertEqual(result[result.find(b"?>") + 2: len(result)],
                          format_xml_result[format_xml_result.find(b"?>") + 2: len(format_xml_result)])
 
+    def testDeepMap(self):
+        """
+        Test formatting of a deeply nested map
+        """
+
+        test_map = {"foo":"bar", "depth":0}
+        max_depth = 199
+        for depth in range(max_depth):
+            test_map = {"foo":"bar", "depth":depth, "next":test_map}
+
+        # this should not throw an exception.
+        test_xml_out = self.llsd.as_xml(test_map)
+
+        test_xml_parsed = self.llsd.parse(io.BytesIO(test_xml_out))
+        self.assertEqual(test_map, test_xml_parsed)
+
+        test_map = {"foo":"bar", "depth":depth, "next":test_map}
+        # this should throw an exception.
+        self.assertRaises(llsd.LLSDSerializationError, self.llsd.as_xml, test_map)
+
     def testLLSDSerializationFailure(self):
         """
         Test serialization function as_xml with an object of non-supported type.