gh-13, gh-14: Allow carets before code notes; add string form to TNS().

Author: python-processing-unit

asm-lang.exe

@@ -75,7 +75,7 @@
 
 Keywords: The language's reserved keywords (for example, `IF`, `WHILE`, `FUNC`, etc.) are matched by the lexer exactly as listed in this specification and are case-sensitive. Programs must use the keywords in their canonical uppercase form; otherwise the token will be recognized as an identifier. Built-in operator names such as `INPUT`, `PRINT`, and `IMPORT` follow the same case-sensitive matching rules.
 
-Line continuation: The character `^` serves as a line-continuation marker. When a caret `^` appears in the source and is followed immediately by a newline, both the `^` and the newline are ignored by the lexer (that is, the logical line continues on the next physical line). If a `^` is present in a string, it does not count as a line continuation. If a caret appears and is not immediately followed by a newline (or the platform's single-character newline sequence), the lexer must raise a syntax error.
+Line continuation: The character `^` serves as a line-continuation marker. When a caret `^` appears in the source and is followed immediately by a newline, both the `^` and the newline are ignored by the lexer (that is, the logical line continues on the next physical line). The lexer also accepts a caret immediately before a code note (a comment beginning with `#`); in this case the `^`, the comment text up to the line terminator, and the terminating newline are treated as if they were not present. If a `^` is present in a string, it does not count as a line continuation. If a caret appears and is not immediately followed by a newline, a code note, or the platform's single-character newline sequence, the lexer must raise a syntax error.
 
 The character `-` primarily serves as the leading sign of a numeric literal (Section 3.1). When `-` appears immediately before optional whitespace and then binary digits, it is parsed as part of the numeric literal (that is, a signed literal). In other contexts (for example inside an index expression) a single `-` token is recognized as a dash used for slice notation `lo-hi`. If `-` appears in any other unsupported context the lexer must raise a syntax error.
 
@@ -659,6 +659,8 @@
 
 - `TNS(TNS: shape, ANY: value):TNS` — Creates a new `TNS` with the shape described by a 1D `TNS` of positive `INT` lengths, filled with `value`.
 
+- `TNS(STR: string):TNS` — Converts a `STR` into a 1-D `TNS` of `STR` characters. Each element of the resulting tensor is a `STR` containing a single character from the input string. For example, `TNS("foo")` yields `["f","o","o"]`.
+
 - `CONV(TNS: x, TNS: kernel, INT: stride_w = 1, INT: stride_h = 1, INT: pad_w = 0, INT: pad_h = 0, TNS: bias = []):TNS` — Extended convolution operator.
 
   - Backward-compatible two-argument form: when called as `CONV(x, kernel)` the operator performs the original N-dimensional discrete convolution and returns a tensor with the same shape as `x`. Requirements and semantics from the original behavior remain unchanged: `kernel` must have the same rank as `x`, every kernel dimension length must be odd (so the kernel has a well-defined center), boundary sampling is clamped to the nearest valid index (replicate padding), and tensor element types must be uniformly `INT` or uniformly `FLT` within each tensor. If both tensors are `INT` the output is `INT`, otherwise the output is `FLT`.

docs/SPECIFICATION.html

@@ -559,7 +559,7 @@ def __init__(self) -> None:
         self._register_custom("SHAPE", 1, 1, self._shape)
         self._register_custom("TLEN", 2, 2, self._tlen)
         self._register_custom("FILL", 2, 2, self._fill)
-        self._register_custom("TNS", 2, 2, self._tns)
+        self._register_custom("TNS", 1, 2, self._tns)
         self._register_custom("TINT", 1, 1, self._tint)
         self._register_custom("TFLT", 1, 1, self._tflt)
         self._register_custom("TSTR", 1, 1, self._tstr)
@@ -3128,8 +3128,22 @@ def _tns(
         ___: Environment,
         location: SourceLocation,
     ) -> Value:
-        shape_val = self._expect_tns(args[0], "TNS", location)
+        # New form: TNS(STR: string) -> 1-D TNS of STR characters
+        first = self._deref_pointer(args[0], rule="TNS", location=location)
+        if first.type == TYPE_STR:
+            s = self._expect_str(first, "TNS", location)
+            # Create a 1-D tensor of STR elements, one-character strings
+            length = len(s)
+            data = np.empty(length, dtype=object)
+            for i, ch in enumerate(s):
+                data[i] = Value(TYPE_STR, ch)
+            return Value(TYPE_TNS, Tensor(shape=[length], data=data))
+
+        # Original form: TNS(TNS: shape, ANY: value)
+        shape_val = self._expect_tns(first, "TNS", location)
         shape = self._shape_from_tensor(shape_val, "TNS", location)
+        if len(args) < 2:
+            raise ASMRuntimeError("TNS expects a fill value when called with a shape", location=location, rewrite_rule="TNS")
         fill_value = args[1]
         tensor = interpreter._make_tensor_from_shape(shape, fill_value, "TNS", location)
         return Value(TYPE_TNS, tensor)

interpreter.py

@@ -391,8 +391,29 @@ def _consume_line_continuation(self) -> None:
             if self.index < len(self.text) and self.text[self.index] == "\n":
                 self._advance()
             return
+        # Allow a caret immediately before a code note (comment starting with '#').
+        # Treat '^#...<newline>' as a valid line-continuation: consume the '^',
+        # the comment text up to the line terminator, and then consume the newline.
+        if ch1 == "#":
+            # consume '^'
+            self._advance()
+            # consume comment text up to but not including the newline
+            self._consume_comment()
+            # Now require and consume a line terminator after the comment.
+            if self.index < self._n and self.text[self.index] == "\n":
+                self._advance()
+                return
+            if self.index < self._n and self.text[self.index] == "\r":
+                self._advance()
+                if self.index < self._n and self.text[self.index] == "\n":
+                    self._advance()
+                return
+            raise ASMParseError(
+                f"Invalid line continuation '^' before comment not terminated by newline at {self.filename}:{self.line}:{self.column}"
+            )
+
         raise ASMParseError(
-            f"Invalid line continuation '^' not followed by newline at {self.filename}:{self.line}:{self.column}"
+            f"Invalid line continuation '^' not followed by newline or code note at {self.filename}:{self.line}:{self.column}"
         )
 
     @property

lexer.py

@@ -43,7 +43,7 @@ FUNC BUILTIN_TESTS():INT{
     # Line continuation (caret ^) inside a tensor literal and index expressions
     TNS: cont = [ ^
         [1, 10, 11], ^
-        [100, 101, 110] ^
+        [100, 101, 110] ^# carets allowed before code notes
     ]
     ASSERT( EQ( TLEN(cont, 1), 10 ) )
     DEL(cont)
@@ -268,6 +268,15 @@ FUNC BUILTIN_TESTS():INT{
     ASSERT( EQ( parts[11], "c" ) )
     DEL(parts)
 
+    # New: TNS from a STR should produce a 1-D tensor of single-character STRs
+    TNS: chars = TNS("foo")
+    ASSERT( EQ( TLEN(chars, 1), 11 ) )
+    ASSERT( EQ( chars[1], "f" ) )
+    ASSERT( EQ( chars[10], "o" ) )
+    ASSERT( EQ( chars[11], "o" ) )
+    ASSERT( EQ( TNS("foo"), ["f","o","o"] ) )
+    DEL(chars)
+
     # Maps: KEYS/VALUES/KEYIN/VALUEIN/INV and indexed DEL
     MAP: mm2 = < "one" = 1, "two" = 10 >
     TNS: ks = KEYS(mm2)
@@ -428,6 +437,7 @@ FUNC BUILTIN_TESTS():INT{
     }
 
     ASSERT( EQ( IMPORT_PATH(mod_path), 0 ) )
+    DELETEFILE("tmpmod.asmln")
     ASSERT( EQ( tmpmod.v, 1 ) )
     ASSERT( EQ( tmpmod.G(), 10 ) )