Lot of large changes breaking compatibility, though strictly required.

Python 3 support has been implemented, Python 2 support has been dropped.
Migrated from dumb strings to `ast`, which has greatly improved the structure of the code and allowed further improvements. The cost is that the comments and whitespaces are not preserved, since there are no nodes for them. Transforming asts into stri

ngs is done using `astor` library. Optionally (if installed) the result is postprocessed by `black`.
Added types, reformatted source. Now we use tabs (a tab is now considered to be completion to 4 spaces). Enable `editorconfig` in your favorite editor if you want to see the sources right.
Implemented python code detection. Instead of custom and incomplete parsing as was done earlier we now use python parser. The first syntax error in python code is considered to be an end of python code and continuation of CoCo/R grammar.
Implemented optional typing for parameters. Typing uses Java/C++/C# syntax (type name), not python one (name: type) for compatibility
Moved the components used by every parser into `CoCoRuntime` module. It provides abstract base classes with some props that must be populated in generated code
Started eliminating global state. Stuff earlier saved into classes now is saved into instances.
Dropped frame files support. Use object oriented programming instead.
`Copyright.frame` is not used anymore (in fact it has been never used in CoCoPy, but CoCo/R manual mentions it). Instead put copyright header into `__copyright__` variable, and it will be inserted in both `Scanner` and `Parser`
Dropped generation of drivers. They were not really generated, just copied.
(Non)terminals enum values are now stored in a single shared `IntEnum` rather than in parser and lexer classes themselves.
Added a mode to use lookups from the enum instead of raw numbers in generated code. Should improve readability and simplify debugging and diffing the generated code
Now we use `setup.cfg` and `pyproject.toml` for fetching versions from git. Installation from the sources other than git is deprecated.
Refactored keywords lookup. Now they are not matched using elif ladders, instead they are stored in a map
Optimized error messages storage. Since their numbers are sequential, now they are stored in a `list` rather than a `dict`.
Dropped numeric values (that initially were not in a enum, just untidy mess) mixing node types, symbol types and state types. Now we have a dedicated class for each.
Optionally optimized keywords lookup using prefix trees using `datrie` library. Automatically enabled, if `datrie` is installed.

Author: KOLANICH

.editorconfig

@@ -1,7 +1,12 @@
-root = true
-[*]
-charset = utf-8
-indent_style = tab
-indent_size = 4
-insert_final_newline = true
-newline = lf
+root = true
+
+[*]
+charset = utf-8
+indent_style = tab
+indent_size = 4
+insert_final_newline = true
+end_of_line = lf
+
+[*.{yml,yaml}]
+indent_style = space
+indent_size = 2

.gitattributes

@@ -0,0 +1 @@
+*.pdf filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -1,8 +1,13 @@
 __pycache__
 *.pyc
 *.pyo
+/monkeytype.sqlite3
 /CocoPy.egg-info
 /build
 /dist
 /.eggs
 /testSuite/tmp
+/*.srctrlbm
+/*.srctrldb
+/.pytest_cache
+/.hypothesis

CoCoPy.srctrlprj

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<config>
+	<source_groups>
+		<source_group_ffffffff-ffff-ffff-ffff-ffffffffffff>
+			<name>Python Source Group</name>
+			<source_extensions>
+				<source_extension>.py</source_extension>
+			</source_extensions>
+			<source_paths>
+				<source_path>./Coco</source_path>
+				<source_path>./CoCoRuntime</source_path>
+			</source_paths>
+			<status>enabled</status>
+			<type>Python Source Group</type>
+		</source_group_ffffffff-ffff-ffff-ffff-ffffffffffff>
+	</source_groups>
+	<version>8</version>
+</config>

Coco/frames/__init__.py CoCoRuntime/__init__.pyCoco/frames/__init__.py renamed to CoCoRuntime/__init__.py

@@ -0,0 +1,137 @@
+"""self.py -- Error handling routines"""
+
+__copyright__ = """
+Compiler Generator Coco/R,
+Copyright (c) 1990, 2004 Hanspeter Moessenboeck, University of Linz
+extended by M. Loeberbauer & A. Woess, Univ. of Linz
+ported from Java to Python by Ronald Longo
+improved and refactored by KOLANICH
+
+This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+As an exception, it is allowed to write an extension of Coco/R that is used as a plugin in non-free software.
+
+If not otherwise stated, any source code generated by Coco/R (other than Coco/R itself) does not fall under the GNU General Public License.
+"""  # pylint: disable=duplicate-code
+
+import typing
+import sys
+from pathlib import Path
+from io import StringIO
+
+from .scanner import Buffer
+
+
+class ErrorRec:
+	__slots__ = ("line", "col", "num", "str")
+
+	def __init__(self, line: int, col: int, s: str) -> None:
+		assert isinstance(line, int), line
+		assert isinstance(col, int), col
+		assert isinstance(s, str), repr(s)
+
+		self.line = line
+		self.col = col
+		self.num = 0
+		self.str = s
+
+
+class Errors:
+	errMsgFormat = "file %(file)s : (%(line)d, %(col)d) %(text)s\n"
+	minErrDist = 2
+	errDist = minErrDist
+	# A function with prototype: f( errorNum=None ) where errorNum is a
+	# predefined error number.  f returns a tuple, ( line, column, message )
+	# such that line and column refer to the location in the
+	# source file most recently parsed.  message is the error
+	# message corresponging to errorNum.
+
+	def __init__(self) -> None:
+		self.errors = []
+		self.warnings = []
+		self.mergedList = None  # PrintWriter
+		self.listName = ""
+
+	def storeError(self, line: int, col: int, s: str) -> None:
+		self.errors.append(ErrorRec(line, col, s))
+
+	def storeWarning(self, line: int, col: int, s: str) -> None:
+		self.warnings.append(ErrorRec(line, col, s))
+
+	@property
+	def count(self) -> int:
+		return len(self.errors)
+
+	def Warn(self, line: int, col: int, errMsg: str) -> None:
+		self.storeWarning(line, col, errMsg)
+
+	@staticmethod
+	def Exception(errMsg):
+		print(errMsg)
+		sys.exit(1)
+
+	@staticmethod
+	def printMsg(fileName: str, line: int, column: int, msg: str) -> None:
+		vals = {"file": fileName, "line": line, "col": column, "text": msg}
+		sys.stdout.write(__class__.errMsgFormat % vals)
+
+	@staticmethod
+	def display(s, e, resBuff):
+		resBuff.write("**** ")
+		for c in range(1, e.col):
+			if s[c - 1] == "\t":
+				resBuff.write("\t")
+			else:
+				resBuff.write(" ")
+		resBuff.write("^ " + e.str + "\n")
+
+	def Summarize(self, sourceBuffer: Buffer) -> str:
+		# Initialize the line iterator
+		srcLineIter = iter(sourceBuffer)
+		srcLineStr = next(srcLineIter, None)
+		srcLineNum = 1
+
+		resBuff = StringIO()
+
+		# Initialize the error iterator
+		errIter = iter(sorted(self.errors, key=lambda v: (v.line, v.col)))
+		warnIter = iter(sorted(self.warnings, key=lambda v: (v.line, v.col)))
+
+		errRec = next(errIter, None)
+		warnRec = next(warnIter, None)
+
+		while errRec and errRec.line < 0:
+			print(errRec.str, file=resBuff)
+			errRec = next(errIter, None)
+
+		while warnRec and warnRec.line < 0:
+			print(warnRec.str, file=resBuff)
+			warnRec = next(warnIter, None)
+
+		while errRec or warnRec:
+			# Advance to the source line of the next error
+			while srcLineStr is not None and (errRec is None or srcLineNum < errRec.line) and (warnRec is None or srcLineNum < warnRec.line):
+				srcLineStr = next(srcLineIter, None)
+				srcLineNum += 1
+
+			resBuff.write("%4d %s\n" % (srcLineNum, srcLineStr))
+			# Write out all errors for the current source line
+			while errRec and errRec.line == srcLineNum:
+				self.display(srcLineStr, errRec, resBuff)
+				errRec = next(errIter, None)
+
+			while warnRec and warnRec.line == srcLineNum:
+				self.display(srcLineStr, warnRec, resBuff)
+				warnRec = next(warnIter, None)
+
+			errRec = next(errIter, None)
+			warnRec = next(warnIter, None)
+
+		resBuff.write("%d errors and %d warnings detected\n" % (self.count, len(self.warnings)))
+		if self.count > 0:
+			resBuff.write("see " + self.listName + "\n")
+		return resBuff.getvalue()

CoCoRuntime/errors.py

@@ -0,0 +1,145 @@
+"""Parser.py -- ATG parser runtime"""
+
+__copyright__ = """
+Compiler Generator Coco/R,
+Copyright (c) 1990, 2004 Hanspeter Moessenboeck, University of Linz
+extended by M. Loeberbauer & A. Woess, Univ. of Linz
+ported from Java to Python by Ronald Longo
+improved and refactored by KOLANICH
+
+This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+As an exception, it is allowed to write an extension of Coco/R that is used as a plugin in non-free software.
+
+If not otherwise stated, any source code generated by Coco/R (other than Coco/R itself) does not fall under the GNU General Public License.
+"""  # pylint: disable=duplicate-code
+
+import typing
+from abc import ABC, abstractmethod
+
+from .errors import Errors
+from .scanner import Scanner, Token
+
+
+class Parser(ABC):
+	ENUM = None
+	__main_production_name__ = None  # type: str
+	__EOF_sym__ = None  # type: typing.Type["ScannerEnum"]
+
+	@abstractmethod
+	def pragmas(self):
+		raise NotImplementedError
+
+	set = None
+	errorMessages = None
+	minErrDist = 2
+
+	# -->declarations
+
+	def __init__(self) -> None:
+		self.scanner = None
+		self.token = None  # last recognized token
+		self.la = None  # lookahead token
+		self.genScanner = False
+		self.tokenString = ""  # used in declarations of literal tokens
+		self.noString = "-none-"  # used in declarations of literal tokens
+		self.errDist = self.__class__.minErrDist
+		self.errors = Errors()
+
+	def getParsingPos(self) -> typing.Tuple[int, int]:
+		return self.la.line, self.la.col
+
+	def SynErr(self, errNum: int) -> None:
+		if self.errDist >= self.__class__.minErrDist:
+			line, col = self.getParsingPos()
+			self.errors.storeError(line, col, self.__class__.errorMessages[errNum])
+
+		self.errDist = 0
+
+	def SemErr(self, msg: str) -> None:
+		if self.errDist >= self.__class__.minErrDist:
+			line, col = self.getParsingPos()
+			self.errors.storeError(line, col, msg)
+
+		self.errDist = 0
+
+	def Warning(self, msg):
+		if self.errDist >= self.__class__.minErrDist:
+			self.errors.Warn(msg)
+
+		self.errDist = 0
+
+	@staticmethod
+	def Successful():
+		return self.errors.count == 0
+
+	def LexString(self):
+		return self.token.val
+
+	def LookAheadString(self):
+		return self.la.val
+
+	def Get(self) -> None:
+		while True:
+			self.token = self.la
+			self.la = self.scanner.Scan()
+			if self.la.kind <= self.__class__.ENUM.maxT:
+				self.errDist += 1
+				break
+			self.pragmas()
+			self.la = self.token
+
+	def Expect(self, n: "ScannerEnum") -> None:
+		if self.la.kind == n:
+			self.Get()
+		else:
+			self.SynErr(n)
+
+	def StartOf(self, s: int) -> bool:
+		return self._StartOf(s, self.la.kind)
+
+	@classmethod
+	def _StartOf(cls, s: int, kind: "ScannerEnum") -> bool:
+		if kind >= 0:
+			return cls.set[s][kind]
+		return False
+
+	def ExpectWeak(self, n: "ScannerEnum", follow: int) -> None:
+		if self.la.kind == n:
+			self.Get()
+		else:
+			self.SynErr(n)
+			while not self.StartOf(follow):
+				self.Get()
+
+	def WeakSeparator(self, n: "ScannerEnum", syFol: int, repFol: int) -> bool:
+		s = [False for i in range(self.__class__.ENUM.maxT + 1)]
+		if self.la.kind == n:
+			self.Get()
+			return True
+
+		if self.StartOf(repFol):
+			return False
+
+		for i in range(self.__class__.ENUM.maxT):
+			s[i] = self._StartOf(syFol, i) or self._StartOf(repFol, i) or self._StartOf(0, i)
+		self.SynErr(n)
+		while not s[self.la.kind]:
+			self.Get()
+		return self.StartOf(syFol)
+
+	@property
+	def __main_production__(self) -> typing.Callable[[], None]:
+		return getattr(self, self.__class__.__main_production_name__)
+
+	def Parse(self, scanner: Scanner) -> None:
+		self.scanner = scanner
+		self.la = Token()
+		self.la.val = ""
+		self.Get()
+		self.__main_production__()
+		self.Expect(self.__class__.ENUM.EOF_SYM)