[IMP] spreadsheet: support array literals in formula tokenizer

The tokenizer now recognizes array literals (e.g. {1,2;3,4})
by handling braces and array row separators.
This allows formulas using array literals to be properly tokenized.

Task: 4735250

Author: laa-odoo

src/spreadsheet/tests/test_spreadsheet_tokenizer.py

@@ -184,3 +184,34 @@ def test_wrong_references(self):
             tokenize("=''!A1"),
             [("OPERATOR", "="), ("SYMBOL", "''!A1")],
         )
+
+    def test_literal_array(self):
+        self.assertEqual(
+            tokenize("={1,2;3,4}"),
+            [
+                ("OPERATOR", "="),
+                ("LEFT_BRACE", "{"),
+                ("NUMBER", "1"),
+                ("ARG_SEPARATOR", ","),
+                ("NUMBER", "2"),
+                ("ARRAY_ROW_SEPARATOR", ";"),
+                ("NUMBER", "3"),
+                ("ARG_SEPARATOR", ","),
+                ("NUMBER", "4"),
+                ("RIGHT_BRACE", "}"),
+            ],
+        )
+        self.assertEqual(
+            tokenize("=SUM({1,2})"),
+            [
+                ("OPERATOR", "="),
+                ("SYMBOL", "SUM"),
+                ("LEFT_PAREN", "("),
+                ("LEFT_BRACE", "{"),
+                ("NUMBER", "1"),
+                ("ARG_SEPARATOR", ","),
+                ("NUMBER", "2"),
+                ("RIGHT_BRACE", "}"),
+                ("RIGHT_PAREN", ")"),
+            ],
+        )

src/util/spreadsheet/tokenizer.py

@@ -87,7 +87,9 @@ def tokenize(string, locale=DEFAULT_LOCALE):
         while not chars.is_over():
             token = (
                 tokenize_space(chars)
+                or tokenize_array_row_separator(chars, locale)
                 or tokenize_args_separator(chars, locale)
+                or tokenize_braces(chars)
                 or tokenize_parenthesis(chars)
                 or tokenize_operator(chars)
                 or tokenize_string(chars)
@@ -123,6 +125,17 @@ def tokenize_parenthesis(chars):
     return None
 
 
+braces = {"{": ("LEFT_BRACE", "{"), "}": ("RIGHT_BRACE", "}")}
+
+
+def tokenize_braces(chars):
+    value = chars.current
+    if value in braces:
+        chars.shift()
+        return braces[value]
+    return None
+
+
 def tokenize_args_separator(chars, locale):
     if chars.current == locale["formulaArgSeparator"]:
         value = chars.shift()
@@ -141,6 +154,16 @@ def tokenize_operator(chars):
 FIRST_POSSIBLE_NUMBER_CHARS = set("0123456789")
 
 
+def tokenize_array_row_separator(chars, locale):
+    row_separator = "\\" if locale["formulaArgSeparator"] == ";" else ";"
+    if not row_separator:
+        return None
+    if chars.current == row_separator:
+        chars.shift()
+        return "ARRAY_ROW_SEPARATOR", row_separator
+    return None
+
+
 def tokenize_number(chars, locale):
     if chars.current not in FIRST_POSSIBLE_NUMBER_CHARS and chars.current != locale["decimalSeparator"]:
         return None