@@ -19,7 +19,7 @@ class Token:
1919 column : int
2020
2121
22- KEYWORDS = {"IF" , "ELSIF" , "ELSE" , "WHILE" , "FOR" , "FUNC" , "RETURN" , "BREAK" , "GOTO" , "GOTOPOINT" }
22+ KEYWORDS = {"IF" , "ELSIF" , "ELSE" , "WHILE" , "FOR" , "FUNC" , "RETURN" , "BREAK" , "GOTO" , "GOTOPOINT" , "CONTINUE" }
2323SYMBOLS = {
2424 "(" : "LPAREN" ,
2525 ")" : "RPAREN" ,
@@ -55,23 +55,8 @@ def tokenize(self) -> List[Token]:
5555 _advance ()
5656 continue
5757 if ch == "^" :
58- if self .index + 1 >= len (self .text ):
59- raise ASMParseError (
60- f"Invalid line continuation '^' at { self .filename } :{ self .line } :{ self .column } "
61- )
62- next_ch = self .text [self .index + 1 ]
63- if next_ch == "\n " :
64- _advance ()
65- _advance ()
66- continue
67- if next_ch == "\r " :
68- _advance ()
69- if not self ._eof and self ._peek () == "\n " :
70- _advance ()
71- continue
72- raise ASMParseError (
73- f"Invalid line continuation '^' not followed by newline at { self .filename } :{ self .line } :{ self .column } "
74- )
58+ self ._consume_line_continuation ()
59+ continue
7560 if ch == "\n " :
7661 tokens .append (Token ("NEWLINE" , "\n " , self .line , self .column ))
7762 _advance ()
@@ -119,9 +104,16 @@ def _consume_signed_number(self) -> Token:
119104
120105 def _consume_binary_digits (self ) -> str :
121106 digits : List [str ] = []
122- while not self ._eof and self ._peek () in "01" :
123- digits .append (self ._peek ())
124- self ._advance ()
107+ while not self ._eof :
108+ ch = self ._peek ()
109+ if ch in "01" :
110+ digits .append (ch )
111+ self ._advance ()
112+ continue
113+ if ch == "^" :
114+ self ._consume_line_continuation ()
115+ continue
116+ break
125117 return "" .join (digits )
126118
127119 def _consume_identifier (self ) -> Token :
@@ -131,9 +123,16 @@ def _consume_identifier(self) -> Token:
131123 f"Identifiers must not start with '0' or '1' at { self .filename } :{ line } :{ col } "
132124 )
133125 chars : List [str ] = []
134- while not self ._eof and self ._is_identifier_part (self ._peek ()):
135- chars .append (self ._peek ())
136- self ._advance ()
126+ while not self ._eof :
127+ ch = self ._peek ()
128+ if self ._is_identifier_part (ch ):
129+ chars .append (ch )
130+ self ._advance ()
131+ continue
132+ if ch == "^" :
133+ self ._consume_line_continuation ()
134+ continue
135+ break
137136 value = "" .join (chars )
138137 token_type : str = value if value in KEYWORDS else "IDENT"
139138 return Token (token_type , value , line , col )
@@ -142,7 +141,27 @@ def _is_identifier_start(self, ch: str) -> bool:
142141 return (ch == "_" ) or ("A" <= ch <= "Z" ) or ("a" <= ch <= "z" )
143142
144143 def _is_identifier_part (self , ch : str ) -> bool :
145- return (ch == "_" ) or ("A" <= ch <= "Z" ) or ("a" <= ch <= "z" ) or ("0" <= ch <= "9" ) or (ch == "." )
144+ return (ch == "_" ) or ("A" <= ch <= "Z" ) or ("a" <= ch <= "z" ) or (ch in "01" )
145+
146+ def _consume_line_continuation (self ) -> None :
147+ if self .index + 1 >= len (self .text ):
148+ raise ASMParseError (
149+ f"Invalid line continuation '^' at { self .filename } :{ self .line } :{ self .column } "
150+ )
151+ ch1 = self .text [self .index + 1 ]
152+ if ch1 == "\n " :
153+ self ._advance ()
154+ self ._advance ()
155+ return
156+ if ch1 == "\r " :
157+ self ._advance ()
158+ self ._advance ()
159+ if self .index < len (self .text ) and self .text [self .index ] == "\n " :
160+ self ._advance ()
161+ return
162+ raise ASMParseError (
163+ f"Invalid line continuation '^' not followed by newline at { self .filename } :{ self .line } :{ self .column } "
164+ )
146165
147166 @property
148167 def _eof (self ) -> bool :
0 commit comments