From 211c8194106a6e762a73212623d92a3c0bd36bf8 Mon Sep 17 00:00:00 2001 From: DetachHead Date: Thu, 23 Nov 2023 01:00:09 +1000 Subject: [PATCH 01/10] robot to python converter script --- .gitignore | 1 + .vscode/launch.json | 15 +++ pdm.lock | 16 ++- pyproject.toml | 5 +- pytest_robotframework/_internal/utils.py | 13 ++ pytest_robotframework/robot2python.py | 125 ++++++++++++++++++ .../test_one_test_with_one_keyword/foo.robot | 3 + .../test_foo.py | 2 + tests/test_robot2python.py | 31 +++++ tests/utils.py | 2 + 10 files changed, 211 insertions(+), 2 deletions(-) create mode 100644 pytest_robotframework/robot2python.py create mode 100644 tests/fixtures/test_robot2python/test_one_test_with_one_keyword/foo.robot create mode 100644 tests/fixtures/test_robot2python/test_one_test_with_one_keyword/test_foo.py create mode 100644 tests/test_robot2python.py diff --git a/.gitignore b/.gitignore index c5b7a256..37a6661d 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ output.xml log.html report.html +robot2python_output \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index 0be14c79..2ba6ac6f 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -4,6 +4,15 @@ // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ + { + "name": "robot2python", + "type": "python", + "request": "launch", + "program": "pytest_robotframework/robot2python.py", + "args": ["${input:robot2pythonInput}", "robot2python_output"], + "console": "integratedTerminal", + "justMyCode": true + }, { "name": "pytest justMyCode disabled", "type": "python", @@ -19,6 +28,12 @@ "type": "promptString", "description": "args for pytest", "default": "" + }, + { + "id": "robot2pythonInput", + "type": "promptString", + "description": "robot test suite to convert to python", + "default": "" } ] } diff --git a/pdm.lock b/pdm.lock index e1f68cbf..0d439e2b 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "lint", "test", "docs"] strategy = ["cross_platform"] lock_version = "4.4" -content_hash = "sha256:cb3bfc2ccb1a4d37dfcd96508f2e6c467dc2ea87a161fefa377ae64922222a93" +content_hash = "sha256:582b5a5affc621a8cda43b791a7a21fe823af1d8d33c0482a6e3c5045c8974aa" [[package]] name = "astroid" @@ -707,6 +707,20 @@ files = [ {file = "tomlkit-0.12.1.tar.gz", hash = "sha256:38e1ff8edb991273ec9f6181244a6a391ac30e9f5098e7535640ea6be97a7c86"}, ] +[[package]] +name = "typer" +version = "0.9.0" +requires_python = ">=3.6" +summary = "Typer, build great CLIs. Easy to code. Based on Python type hints." +dependencies = [ + "click<9.0.0,>=7.1.1", + "typing-extensions>=3.7.4.3", +] +files = [ + {file = "typer-0.9.0-py3-none-any.whl", hash = "sha256:5d96d986a21493606a358cae4461bd8cdf83cbf33a5aa950ae629ca3b51467ee"}, + {file = "typer-0.9.0.tar.gz", hash = "sha256:50922fd79aea2f4751a8e0408ff10d2662bd0c8bbfa84755a699f3bada2978b2"}, +] + [[package]] name = "typing-extensions" version = "4.8.0" diff --git a/pyproject.toml b/pyproject.toml index 9d8f13ae..ff230df7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,6 +10,8 @@ dependencies = [ "robotframework<7.0.0,>=6.1.1", "deepmerge<2.0.0,>=1.1.0", "basedtyping<0.2,>=0.1.0", + "typer>=0.9.0", + "astunparse>=1.6.3", ] requires-python = ">=3.8,<4.0" readme = "README.md" @@ -284,13 +286,14 @@ no_implicit_reexport = false ignore_missing_py_typed = true # https://github.com/robotframework/robotframework/issues/4822 [[tool.mypy.overrides]] -module = ['deepmerge.*'] +module = ['deepmerge.*', 'astunparse.*'] ignore_missing_py_typed = true [[tool.mypy.overrides]] module = ['pytest_robotframework.*', 'tests.*'] default_return = true + [tool.pyright] pythonVersion = "3.8" pythonPlatform = "All" diff --git a/pytest_robotframework/_internal/utils.py b/pytest_robotframework/_internal/utils.py index 06b931e4..3227ae45 100644 --- a/pytest_robotframework/_internal/utils.py +++ b/pytest_robotframework/_internal/utils.py @@ -1,5 +1,6 @@ from __future__ import annotations +import sys from contextlib import AbstractContextManager from functools import wraps from typing import TYPE_CHECKING, Callable, Generic, Type, Union, cast @@ -11,6 +12,7 @@ if TYPE_CHECKING: from abc import abstractmethod + from ast import AST from types import TracebackType from pytest import Item, Session, StashKey @@ -84,6 +86,17 @@ class ContextManager(Generic[out_T], AbstractContextManager): pass +# ast.unparse doesn't exist in 3.8 +if sys.version_info < (3, 9): + from astunparse import unparse as unparse_ + + def unparse(ast: AST) -> str: + return cast(str, unparse_(ast)) # type:ignore[no-untyped-call] + +else: + from ast import unparse as unparse + + def init_stash( # pylint:disable=missing-param-doc key: StashKey[T], initializer: Callable[[], T], diff --git a/pytest_robotframework/robot2python.py b/pytest_robotframework/robot2python.py new file mode 100644 index 00000000..8c3c06bd --- /dev/null +++ b/pytest_robotframework/robot2python.py @@ -0,0 +1,125 @@ +from __future__ import annotations + +from ast import Call, Constant, Expr, FunctionDef, Module, Name, stmt +from pathlib import Path +from typing import TYPE_CHECKING, cast + +from robot.api import SuiteVisitor +from robot.run import RobotFramework +from typer import run +from typing_extensions import override + +from pytest_robotframework._internal.errors import InternalError, UserError +from pytest_robotframework._internal.utils import unparse + +if TYPE_CHECKING: + from robot import model + + +def _pythonify_name(name: str) -> str: + return name.lower().replace(" ", "_") + + +def _pytestify_name(name: str) -> str: + return f"test_{_pythonify_name(name)}" + + +def _robot_file(suite: model.TestSuite) -> Path | None: + if suite.source is None: + raise InternalError(f"ayo whyyo suite aint got no path 💀 ({suite.name})") + suite_path = Path(suite.source) + return suite_path if suite_path.suffix == ".robot" else None + + +class Robot2Python(SuiteVisitor): + def __init__(self, output_dir: Path) -> None: + self.modules: dict[Path, Module] = {} + self.output_dir = output_dir + self.module_stack: list[Module] = [] + self.statement_stack: list[stmt] = [] + + @override + def start_suite(self, suite: model.TestSuite): + robot_file = _robot_file(suite) + if robot_file is None: + return + module = Module( + body=[], type_ignores=[] # type:ignore[no-any-expr] + ) + self.module_stack.append(module) + self.modules[ + # cringe + Path( + str(robot_file.parent.resolve()).replace( + str(Path.cwd()), str(self.output_dir) + ) + ) + / f"{_pytestify_name(robot_file.stem)}.py" + ] = module + + @override + def end_suite(self, suite: model.TestSuite): + # make sure no tests are actually executed once this is done + suite.tests.clear() # type:ignore[no-untyped-call] + if _robot_file(suite) is not None: + self.module_stack.pop() + + @override + def start_test(self, test: model.TestCase): + function = FunctionDef( + name=_pytestify_name(test.name), + args=[], # type:ignore[no-any-expr] + decorator_list=[], # type:ignore[no-any-expr] + body=[], # type:ignore[no-any-expr] + lineno=-1, + ) + self.statement_stack.append(function) + self.module_stack[-1].body.append(function) + + @override + def end_test(self, test: model.TestCase): + self.statement_stack.pop() + + @override + def start_keyword(self, keyword: model.Keyword): + function = cast(FunctionDef, self.module_stack[-1].body[-1]) + self.statement_stack.append(function) + if not keyword.name: + raise UserError("why yo keyword aint got no name") + function.body.append( + Expr( + Call( + func=Name(id=_pythonify_name(keyword.name)), + args=[ + Constant(value=arg) + for arg in keyword.args # type:ignore[no-any-expr] + ], + keywords=[], # type:ignore[no-any-expr] + ) + ) + ) + + @override + def end_keyword(self, keyword: model.Keyword): + self.statement_stack.pop() + + +def _convert(suite: Path, output: Path) -> dict[Path, str]: + robot_2_python = Robot2Python(output) + RobotFramework().main( # type:ignore[no-untyped-call] + [suite], # type:ignore[no-any-expr] + prerunmodifier=robot_2_python, + runemptysuite=True, + ) + return {path: unparse(module) for path, module in robot_2_python.modules.items()} + + +def main(suite: Path, output: Path): + for path, module_text in _convert(suite, output).items(): + if not path.parent.exists(): + path.parent.mkdir(parents=True) + path.write_text(module_text) + + +if __name__ == "__main__": + run(main) diff --git a/tests/fixtures/test_robot2python/test_one_test_with_one_keyword/foo.robot b/tests/fixtures/test_robot2python/test_one_test_with_one_keyword/foo.robot new file mode 100644 index 00000000..e98af1b4 --- /dev/null +++ b/tests/fixtures/test_robot2python/test_one_test_with_one_keyword/foo.robot @@ -0,0 +1,3 @@ +*** Test Cases *** +Foo + No Operation diff --git a/tests/fixtures/test_robot2python/test_one_test_with_one_keyword/test_foo.py b/tests/fixtures/test_robot2python/test_one_test_with_one_keyword/test_foo.py new file mode 100644 index 00000000..c205544b --- /dev/null +++ b/tests/fixtures/test_robot2python/test_one_test_with_one_keyword/test_foo.py @@ -0,0 +1,2 @@ +def test_foo(): + no_operation() diff --git a/tests/test_robot2python.py b/tests/test_robot2python.py new file mode 100644 index 00000000..bb612992 --- /dev/null +++ b/tests/test_robot2python.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from ast import parse +from os import listdir +from pathlib import Path + +from pytest import mark + +from pytest_robotframework._internal.utils import unparse +from pytest_robotframework.robot2python import _convert + +current_file = Path(__file__) +fixtures_folder = current_file.parent / f"fixtures/{current_file.stem}" + + +def format_code(code: str) -> str: + """hacky way to format code however the ast unparser does. would use black but it's a pain to + run programmatically""" + return unparse(parse(code)) + + +@mark.parametrize( + "suite", [fixtures_folder / suite for suite in listdir(fixtures_folder)] +) +def test_robot2python(suite: Path): + converted_tests = _convert(suite, suite) + for expected_python_file, actual_python_code in converted_tests.items(): + assert format_code(expected_python_file.read_text()) == format_code( + actual_python_code + ) + assert len(converted_tests) == len(list(suite.glob("**/*.py"))) diff --git a/tests/utils.py b/tests/utils.py index 2526d422..18d1c52a 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,5 +1,7 @@ from __future__ import annotations +import sys +from ast import AST from typing import TYPE_CHECKING, Dict, cast from lxml.etree import XML From ec8414d15187bbbb705f402a5869c15083a6419b Mon Sep 17 00:00:00 2001 From: detachhead Date: Thu, 23 Nov 2023 17:19:42 +1000 Subject: [PATCH 02/10] convert `log` to `logger.info` --- pyproject.toml | 4 + pytest_robotframework/robot2python.py | 116 +++++++++++++----- .../fixtures/test_robot2python/log/foo.robot | 3 + .../test_robot2python/log/test_foo.py | 7 ++ .../foo.robot | 0 .../no_operation/test_foo.py | 4 + .../test_foo.py | 2 - tests/test_robot2python.py | 10 +- tests/utils.py | 2 - 9 files changed, 110 insertions(+), 38 deletions(-) create mode 100644 tests/fixtures/test_robot2python/log/foo.robot create mode 100644 tests/fixtures/test_robot2python/log/test_foo.py rename tests/fixtures/test_robot2python/{test_one_test_with_one_keyword => no_operation}/foo.robot (100%) create mode 100644 tests/fixtures/test_robot2python/no_operation/test_foo.py delete mode 100644 tests/fixtures/test_robot2python/test_one_test_with_one_keyword/test_foo.py diff --git a/pyproject.toml b/pyproject.toml index ff230df7..a6644f17 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -383,6 +383,10 @@ fixture-parentheses = false "tests/**/*.py" = [ "S101", # Use of assert detected (pytest uses assert statements) ] +"tests/fixtures/test_robot2python/**/*.py" = [ + "INP001", # implicit-namespace-package +] + [tool.ruff.isort] combine-as-imports = true required-imports = ["from __future__ import annotations"] diff --git a/pytest_robotframework/robot2python.py b/pytest_robotframework/robot2python.py index 8c3c06bd..20d0199c 100644 --- a/pytest_robotframework/robot2python.py +++ b/pytest_robotframework/robot2python.py @@ -1,10 +1,22 @@ from __future__ import annotations -from ast import Call, Constant, Expr, FunctionDef, Module, Name, stmt +from ast import ( + Call, + Constant, + Expr, + FunctionDef, + ImportFrom, + Module, + Name, + alias, + expr, + stmt, +) +from contextlib import contextmanager from pathlib import Path -from typing import TYPE_CHECKING, cast +from typing import TYPE_CHECKING, Iterator, cast -from robot.api import SuiteVisitor +from robot.api import SuiteVisitor, logger from robot.run import RobotFramework from typer import run from typing_extensions import override @@ -13,6 +25,8 @@ from pytest_robotframework._internal.utils import unparse if TYPE_CHECKING: + from types import ModuleType + from robot import model @@ -35,18 +49,46 @@ class Robot2Python(SuiteVisitor): def __init__(self, output_dir: Path) -> None: self.modules: dict[Path, Module] = {} self.output_dir = output_dir - self.module_stack: list[Module] = [] + self.current_module: Module self.statement_stack: list[stmt] = [] + @property + def context(self) -> stmt: + return self.statement_stack[-1] + + @contextmanager + def _stack_frame(self, statement: stmt) -> Iterator[None]: + self.statement_stack.append(statement) + try: + yield + finally: + self.statement_stack.pop() + + def _add_import(self, module: ModuleType, names: list[str]): + # insert it at the top of the file but not before the `__future__` import + self.current_module.body.insert( + 1, + ImportFrom( + module=module.__name__, + names=[alias(name=name) for name in names], # type:ignore[no-any-expr] + ), + ) + @override def start_suite(self, suite: model.TestSuite): robot_file = _robot_file(suite) if robot_file is None: return module = Module( - body=[], type_ignores=[] # type:ignore[no-any-expr] + body=[ + ImportFrom( + module="__future__", + names=[alias(name="annotations")], # type:ignore[no-any-expr] + ) + ], + type_ignores=[], # type:ignore[no-any-expr] ) - self.module_stack.append(module) + self.current_module = module self.modules[ # cringe Path( @@ -62,10 +104,10 @@ def end_suite(self, suite: model.TestSuite): # make sure no tests are actually executed once this is done suite.tests.clear() # type:ignore[no-untyped-call] if _robot_file(suite) is not None: - self.module_stack.pop() + del self.current_module @override - def start_test(self, test: model.TestCase): + def visit_test(self, test: model.TestCase): function = FunctionDef( name=_pytestify_name(test.name), args=[], # type:ignore[no-any-expr] @@ -73,35 +115,49 @@ def start_test(self, test: model.TestCase): body=[], # type:ignore[no-any-expr] lineno=-1, ) - self.statement_stack.append(function) - self.module_stack[-1].body.append(function) + self.current_module.body.append(function) + with self._stack_frame(function): + super().visit_test(test) - @override - def end_test(self, test: model.TestCase): - self.statement_stack.pop() - - @override - def start_keyword(self, keyword: model.Keyword): - function = cast(FunctionDef, self.module_stack[-1].body[-1]) - self.statement_stack.append(function) + # eventually this will add imports to self.current_module + def _resolve_call(self, keyword: model.Keyword) -> expr: if not keyword.name: raise UserError("why yo keyword aint got no name") - function.body.append( - Expr( - Call( - func=Name(id=_pythonify_name(keyword.name)), - args=[ - Constant(value=arg) - for arg in keyword.args # type:ignore[no-any-expr] - ], - keywords=[], # type:ignore[no-any-expr] - ) + python_name = _pythonify_name(keyword.name) + + def create_call(module: ModuleType, function: str) -> Call: + self._add_import(module, [function]) + return Call( + func=Name(id=function), + args=[ + Constant(value=arg) + for arg in keyword.args # type:ignore[no-any-expr] + ], + keywords=[], # type:ignore[no-any-expr] ) + + if python_name == "no_operation": + return Constant(value=...) + if python_name == "log": + return create_call(logger, "info") + return Call( + func=Name(id="run_keyword"), + args=[ + Constant(value=keyword.name), + *( + Constant(value=arg) + for arg in keyword.args # type:ignore[no-any-expr] + ), + ], + keywords=[], # type:ignore[no-any-expr] ) @override - def end_keyword(self, keyword: model.Keyword): - self.statement_stack.pop() + def visit_keyword(self, keyword: model.Keyword): + function = cast(FunctionDef, self.current_module.body[-1]) + function.body.append(Expr(self._resolve_call(keyword))) + with self._stack_frame(function): + super().visit_keyword(keyword) def _convert(suite: Path, output: Path) -> dict[Path, str]: diff --git a/tests/fixtures/test_robot2python/log/foo.robot b/tests/fixtures/test_robot2python/log/foo.robot new file mode 100644 index 00000000..5ac25ebf --- /dev/null +++ b/tests/fixtures/test_robot2python/log/foo.robot @@ -0,0 +1,3 @@ +*** Test Cases *** +Foo + Log hi diff --git a/tests/fixtures/test_robot2python/log/test_foo.py b/tests/fixtures/test_robot2python/log/test_foo.py new file mode 100644 index 00000000..c7b42dca --- /dev/null +++ b/tests/fixtures/test_robot2python/log/test_foo.py @@ -0,0 +1,7 @@ +from __future__ import annotations + +from robot.api.logger import info + + +def test_foo(): + info("hi") # type:ignore[no-untyped-call] diff --git a/tests/fixtures/test_robot2python/test_one_test_with_one_keyword/foo.robot b/tests/fixtures/test_robot2python/no_operation/foo.robot similarity index 100% rename from tests/fixtures/test_robot2python/test_one_test_with_one_keyword/foo.robot rename to tests/fixtures/test_robot2python/no_operation/foo.robot diff --git a/tests/fixtures/test_robot2python/no_operation/test_foo.py b/tests/fixtures/test_robot2python/no_operation/test_foo.py new file mode 100644 index 00000000..4431ef63 --- /dev/null +++ b/tests/fixtures/test_robot2python/no_operation/test_foo.py @@ -0,0 +1,4 @@ +from __future__ import annotations + + +def test_foo(): ... diff --git a/tests/fixtures/test_robot2python/test_one_test_with_one_keyword/test_foo.py b/tests/fixtures/test_robot2python/test_one_test_with_one_keyword/test_foo.py deleted file mode 100644 index c205544b..00000000 --- a/tests/fixtures/test_robot2python/test_one_test_with_one_keyword/test_foo.py +++ /dev/null @@ -1,2 +0,0 @@ -def test_foo(): - no_operation() diff --git a/tests/test_robot2python.py b/tests/test_robot2python.py index bb612992..baad5cf1 100644 --- a/tests/test_robot2python.py +++ b/tests/test_robot2python.py @@ -9,9 +9,6 @@ from pytest_robotframework._internal.utils import unparse from pytest_robotframework.robot2python import _convert -current_file = Path(__file__) -fixtures_folder = current_file.parent / f"fixtures/{current_file.stem}" - def format_code(code: str) -> str: """hacky way to format code however the ast unparser does. would use black but it's a pain to @@ -19,8 +16,13 @@ def format_code(code: str) -> str: return unparse(parse(code)) +current_file = Path(__file__) +fixtures_folder = current_file.parent / f"fixtures/{current_file.stem}" +suite_names = listdir(fixtures_folder) + + @mark.parametrize( - "suite", [fixtures_folder / suite for suite in listdir(fixtures_folder)] + "suite", [fixtures_folder / suite for suite in suite_names], ids=suite_names ) def test_robot2python(suite: Path): converted_tests = _convert(suite, suite) diff --git a/tests/utils.py b/tests/utils.py index 18d1c52a..2526d422 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,7 +1,5 @@ from __future__ import annotations -import sys -from ast import AST from typing import TYPE_CHECKING, Dict, cast from lxml.etree import XML From 789e88f12486679603206ab573b1917ef39e7c4c Mon Sep 17 00:00:00 2001 From: detachhead Date: Fri, 24 Nov 2023 14:37:05 +1000 Subject: [PATCH 03/10] fix output path --- pytest_robotframework/robot2python.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pytest_robotframework/robot2python.py b/pytest_robotframework/robot2python.py index 20d0199c..ef3ba7a2 100644 --- a/pytest_robotframework/robot2python.py +++ b/pytest_robotframework/robot2python.py @@ -90,12 +90,8 @@ def start_suite(self, suite: model.TestSuite): ) self.current_module = module self.modules[ - # cringe - Path( - str(robot_file.parent.resolve()).replace( - str(Path.cwd()), str(self.output_dir) - ) - ) + self.output_dir + / (robot_file.parent).relative_to(self.output_dir) / f"{_pytestify_name(robot_file.stem)}.py" ] = module @@ -161,6 +157,8 @@ def visit_keyword(self, keyword: model.Keyword): def _convert(suite: Path, output: Path) -> dict[Path, str]: + suite = suite.resolve() + output = output.resolve() robot_2_python = Robot2Python(output) RobotFramework().main( # type:ignore[no-untyped-call] [suite], # type:ignore[no-any-expr] From 28e82264dc96ccd473143fcd80ed72ddf6b0f1b3 Mon Sep 17 00:00:00 2001 From: detachhead Date: Fri, 24 Nov 2023 15:03:32 +1000 Subject: [PATCH 04/10] fix ast unparse crash on <=3.9 --- pytest_robotframework/robot2python.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pytest_robotframework/robot2python.py b/pytest_robotframework/robot2python.py index ef3ba7a2..d10e0250 100644 --- a/pytest_robotframework/robot2python.py +++ b/pytest_robotframework/robot2python.py @@ -71,6 +71,7 @@ def _add_import(self, module: ModuleType, names: list[str]): ImportFrom( module=module.__name__, names=[alias(name=name) for name in names], # type:ignore[no-any-expr] + level=0, ), ) @@ -84,6 +85,7 @@ def start_suite(self, suite: model.TestSuite): ImportFrom( module="__future__", names=[alias(name="annotations")], # type:ignore[no-any-expr] + level=0, ) ], type_ignores=[], # type:ignore[no-any-expr] From e84956b539b79a892603fb14e2b47dc7d3b09032 Mon Sep 17 00:00:00 2001 From: detachhead Date: Fri, 24 Nov 2023 15:18:36 +1000 Subject: [PATCH 05/10] fix unparse crashes on 3.8 --- pytest_robotframework/robot2python.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pytest_robotframework/robot2python.py b/pytest_robotframework/robot2python.py index d10e0250..25683f9b 100644 --- a/pytest_robotframework/robot2python.py +++ b/pytest_robotframework/robot2python.py @@ -70,7 +70,9 @@ def _add_import(self, module: ModuleType, names: list[str]): 1, ImportFrom( module=module.__name__, - names=[alias(name=name) for name in names], # type:ignore[no-any-expr] + names=[ + alias(name=name, asname=None) for name in names + ], # type:ignore[no-any-expr] level=0, ), ) @@ -84,7 +86,9 @@ def start_suite(self, suite: model.TestSuite): body=[ ImportFrom( module="__future__", - names=[alias(name="annotations")], # type:ignore[no-any-expr] + names=[ + alias(name="annotations", asname=None) + ], # type:ignore[no-any-expr] level=0, ) ], @@ -128,7 +132,7 @@ def create_call(module: ModuleType, function: str) -> Call: return Call( func=Name(id=function), args=[ - Constant(value=arg) + Constant(value=arg, kind=None) for arg in keyword.args # type:ignore[no-any-expr] ], keywords=[], # type:ignore[no-any-expr] From 39ee3502a89fecfdf7e5468989b61a85e255b722 Mon Sep 17 00:00:00 2001 From: DetachHead Date: Mon, 27 Nov 2023 20:08:58 +1000 Subject: [PATCH 06/10] disable `unused-wildcard-import` pylint rule that's covered by ruff --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a6644f17..2907559d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -223,7 +223,6 @@ enable = [ "unnecessary-ellipsis", "unreachable", "unused-private-member", - "unused-wildcard-import", "useless-param-doc", "useless-parent-delegation", "useless-type-doc", From 886dc7015e31f2d1eed3b08f2d09f86265c0c376 Mon Sep 17 00:00:00 2001 From: DetachHead Date: Mon, 27 Nov 2023 20:09:56 +1000 Subject: [PATCH 07/10] dupport imports --- pytest_robotframework/robot2python.py | 66 +++++++++---------- .../function_from_library/foo.robot | 7 ++ .../function_from_library/test_foo.py | 7 ++ 3 files changed, 45 insertions(+), 35 deletions(-) create mode 100644 tests/fixtures/test_robot2python/function_from_library/foo.robot create mode 100644 tests/fixtures/test_robot2python/function_from_library/test_foo.py diff --git a/pytest_robotframework/robot2python.py b/pytest_robotframework/robot2python.py index 25683f9b..eaa73665 100644 --- a/pytest_robotframework/robot2python.py +++ b/pytest_robotframework/robot2python.py @@ -16,7 +16,9 @@ from pathlib import Path from typing import TYPE_CHECKING, Iterator, cast +from robot import running from robot.api import SuiteVisitor, logger +from robot.model.itemlist import ItemList from robot.run import RobotFramework from typer import run from typing_extensions import override @@ -64,52 +66,49 @@ def _stack_frame(self, statement: stmt) -> Iterator[None]: finally: self.statement_stack.pop() - def _add_import(self, module: ModuleType, names: list[str]): - # insert it at the top of the file but not before the `__future__` import + def _add_import(self, module: ModuleType | str, names: list[str] | None = None): + module_name = module if isinstance(module, str) else module.__name__ + if names is None: + names = ["*"] self.current_module.body.insert( - 1, + # __future__ imports need to go first + (0 if module_name == "__future__" else 1), ImportFrom( - module=module.__name__, - names=[ + module=module_name, + names=([ # type:ignore[no-any-expr] alias(name=name, asname=None) for name in names - ], # type:ignore[no-any-expr] + ]), level=0, ), ) @override - def start_suite(self, suite: model.TestSuite): + def start_suite(self, suite: running.TestSuite): robot_file = _robot_file(suite) if robot_file is None: return module = Module( - body=[ - ImportFrom( - module="__future__", - names=[ - alias(name="annotations", asname=None) - ], # type:ignore[no-any-expr] - level=0, - ) - ], - type_ignores=[], # type:ignore[no-any-expr] + body=[], type_ignores=[] # type:ignore[no-any-expr] ) self.current_module = module + self._add_import("__future__", ["annotations"]) self.modules[ self.output_dir / (robot_file.parent).relative_to(self.output_dir) / f"{_pytestify_name(robot_file.stem)}.py" ] = module + for module in cast(ItemList[running.model.Import], suite.resource.imports): + self._add_import(module.name) @override - def end_suite(self, suite: model.TestSuite): + def end_suite(self, suite: running.TestSuite): # make sure no tests are actually executed once this is done suite.tests.clear() # type:ignore[no-untyped-call] if _robot_file(suite) is not None: del self.current_module @override - def visit_test(self, test: model.TestCase): + def visit_test(self, test: running.TestCase): function = FunctionDef( name=_pytestify_name(test.name), args=[], # type:ignore[no-any-expr] @@ -122,40 +121,33 @@ def visit_test(self, test: model.TestCase): super().visit_test(test) # eventually this will add imports to self.current_module - def _resolve_call(self, keyword: model.Keyword) -> expr: + def _resolve_call(self, keyword: running.Keyword) -> expr: if not keyword.name: raise UserError("why yo keyword aint got no name") python_name = _pythonify_name(keyword.name) - def create_call(module: ModuleType, function: str) -> Call: - self._add_import(module, [function]) + def create_call(function: str, module: ModuleType | None = None) -> Call: + if module: + self._add_import(module, [function]) return Call( func=Name(id=function), args=[ Constant(value=arg, kind=None) for arg in keyword.args # type:ignore[no-any-expr] ], + # TODO keywords=[], # type:ignore[no-any-expr] ) if python_name == "no_operation": return Constant(value=...) if python_name == "log": - return create_call(logger, "info") - return Call( - func=Name(id="run_keyword"), - args=[ - Constant(value=keyword.name), - *( - Constant(value=arg) - for arg in keyword.args # type:ignore[no-any-expr] - ), - ], - keywords=[], # type:ignore[no-any-expr] - ) + return create_call("info", logger) + return create_call(python_name) @override - def visit_keyword(self, keyword: model.Keyword): + # https://github.com/robotframework/robotframework/issues/4940 + def visit_keyword(self, keyword: running.Keyword): # type:ignore[override] function = cast(FunctionDef, self.current_module.body[-1]) function.body.append(Expr(self._resolve_call(keyword))) with self._stack_frame(function): @@ -170,6 +162,10 @@ def _convert(suite: Path, output: Path) -> dict[Path, str]: [suite], # type:ignore[no-any-expr] prerunmodifier=robot_2_python, runemptysuite=True, + report=None, + output=None, + log=None, + exitonerror=True, ) return {path: unparse(module) for path, module in robot_2_python.modules.items()} diff --git a/tests/fixtures/test_robot2python/function_from_library/foo.robot b/tests/fixtures/test_robot2python/function_from_library/foo.robot new file mode 100644 index 00000000..c7aecb7f --- /dev/null +++ b/tests/fixtures/test_robot2python/function_from_library/foo.robot @@ -0,0 +1,7 @@ +*** Settings *** +Library typing + + +*** Test Cases *** +Foo + Cast str asdf diff --git a/tests/fixtures/test_robot2python/function_from_library/test_foo.py b/tests/fixtures/test_robot2python/function_from_library/test_foo.py new file mode 100644 index 00000000..050a8cd2 --- /dev/null +++ b/tests/fixtures/test_robot2python/function_from_library/test_foo.py @@ -0,0 +1,7 @@ +from __future__ import annotations + +from typing import * # noqa: F403 + + +def test_foo(): + cast("str", "asdf") # noqa: F405 From b7d59077324b40c4c16997fc8364f5219fd10a2f Mon Sep 17 00:00:00 2001 From: DetachHead Date: Mon, 27 Nov 2023 20:25:55 +1000 Subject: [PATCH 08/10] make robot2python a script when installing the package --- pyproject.toml | 3 +++ .../_internal/scripts/__init__.py | 0 .../{ => _internal/scripts}/robot2python.py | 17 ++++++++--------- tests/test_robot2python.py | 2 +- 4 files changed, 12 insertions(+), 10 deletions(-) create mode 100644 pytest_robotframework/_internal/scripts/__init__.py rename pytest_robotframework/{ => _internal/scripts}/robot2python.py (95%) diff --git a/pyproject.toml b/pyproject.toml index 2907559d..201772fb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,9 @@ repository = "https://github.com/detachhead/pytest-robotframework" [project.entry-points.pytest11] robotframework = "pytest_robotframework._internal.plugin" +[project.scripts] +robot2python = "pytest_robotframework._internal.scripts.robot2python:main" + [tool.pyprojectx] pdm = "pdm==2.10.4" diff --git a/pytest_robotframework/_internal/scripts/__init__.py b/pytest_robotframework/_internal/scripts/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pytest_robotframework/robot2python.py b/pytest_robotframework/_internal/scripts/robot2python.py similarity index 95% rename from pytest_robotframework/robot2python.py rename to pytest_robotframework/_internal/scripts/robot2python.py index eaa73665..392a239c 100644 --- a/pytest_robotframework/robot2python.py +++ b/pytest_robotframework/_internal/scripts/robot2python.py @@ -170,12 +170,11 @@ def _convert(suite: Path, output: Path) -> dict[Path, str]: return {path: unparse(module) for path, module in robot_2_python.modules.items()} -def main(suite: Path, output: Path): - for path, module_text in _convert(suite, output).items(): - if not path.parent.exists(): - path.parent.mkdir(parents=True) - path.write_text(module_text) - - -if __name__ == "__main__": - run(main) +def main(): + def inner(suite: Path, output: Path): + for path, module_text in _convert(suite, output).items(): + if not path.parent.exists(): + path.parent.mkdir(parents=True) + path.write_text(module_text) + + run(inner) diff --git a/tests/test_robot2python.py b/tests/test_robot2python.py index baad5cf1..46d9238d 100644 --- a/tests/test_robot2python.py +++ b/tests/test_robot2python.py @@ -6,8 +6,8 @@ from pytest import mark +from pytest_robotframework._internal.scripts.robot2python import _convert from pytest_robotframework._internal.utils import unparse -from pytest_robotframework.robot2python import _convert def format_code(code: str) -> str: From 909502d33a906a345c0c9c35b22b8bde5e5faefd Mon Sep 17 00:00:00 2001 From: detachhead Date: Tue, 28 Nov 2023 10:45:23 +1000 Subject: [PATCH 09/10] add documentation about robot2python --- README.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/README.md b/README.md index c214f33b..ba7084de 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,30 @@ def test_bar(): ... ``` +## automatically convert `.robot` tests to `.py` + +pytest-robotframework comes with a script to automatically convert tests from robot to python: + +``` +robot2python ./foo.robot ./tests +``` + +this will convert the `foo.robot` file to an equivalent `test_foo.py` file and output it to the `tests` directory. + +### limitations + +note that the script is not perfect and you will probably have to make manual changes to tests converted with it. + +- because the script doesn't know what exactly what keywords are being imported from each library: + + - outside of some special cased keywords in the robot `BuiltIn` library, keywords are currently not resolved, so robot `Library` imports are converted to star-imports (ie. `Library library_name` -> `from library_name import *`). + + star imports [should not be used](https://docs.astral.sh/ruff/rules/undefined-local-with-import-star/), so you should manually update them to import individual keywords explicitly. + + - robot converters are not yet used, so all arguments to keywords are assumed to be strings in the converted python code. + +- some of the control flows possible in robot aren't able to be accurately converted to python (see [here](#continuable-failures-dont-work)). the script attempts to convert what it can but you will probably have to rewrite parts of your tests that use continuable failures + ## setup/teardown and other hooks to define a function that runs for each test at setup or teardown, create a `conftest.py` with a `pytest_runtest_setup` and/or `pytest_runtest_teardown` function: From 21d0a8cabed6cbbac5ee49dce47323fe6c8f2c00 Mon Sep 17 00:00:00 2001 From: detachhead Date: Tue, 28 Nov 2023 17:24:01 +1000 Subject: [PATCH 10/10] why cant i inspect keyword args :( --- README.md | 9 +- pyproject.toml | 4 +- .../_internal/scripts/robot2python.py | 156 +++++++++++++----- .../function_from_library/test_foo.py | 4 +- .../test_robot2python/robot_keyword/foo.robot | 9 + .../robot_keyword/test_foo.py | 14 ++ 6 files changed, 141 insertions(+), 55 deletions(-) create mode 100644 tests/fixtures/test_robot2python/robot_keyword/foo.robot create mode 100644 tests/fixtures/test_robot2python/robot_keyword/test_foo.py diff --git a/README.md b/README.md index ba7084de..9d035451 100644 --- a/README.md +++ b/README.md @@ -95,14 +95,7 @@ this will convert the `foo.robot` file to an equivalent `test_foo.py` file and o note that the script is not perfect and you will probably have to make manual changes to tests converted with it. -- because the script doesn't know what exactly what keywords are being imported from each library: - - - outside of some special cased keywords in the robot `BuiltIn` library, keywords are currently not resolved, so robot `Library` imports are converted to star-imports (ie. `Library library_name` -> `from library_name import *`). - - star imports [should not be used](https://docs.astral.sh/ruff/rules/undefined-local-with-import-star/), so you should manually update them to import individual keywords explicitly. - - - robot converters are not yet used, so all arguments to keywords are assumed to be strings in the converted python code. - +- robot converters are not yet used, so all arguments to keywords are assumed to be strings in the converted python code. - some of the control flows possible in robot aren't able to be accurately converted to python (see [here](#continuable-failures-dont-work)). the script attempts to convert what it can but you will probably have to rewrite parts of your tests that use continuable failures ## setup/teardown and other hooks diff --git a/pyproject.toml b/pyproject.toml index 201772fb..84f8578d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -278,9 +278,9 @@ xfail_strict = true enable_assertion_pass_hook = true [tool.mypy] -allow_redefinition = true +allow_redefinition = false default_return = false -cache_dir = 'nul' # disable cache because it sucks +cache_dir = 'nul' # disable cache because it sucks [[tool.mypy.overrides]] module = ['robot.*'] diff --git a/pytest_robotframework/_internal/scripts/robot2python.py b/pytest_robotframework/_internal/scripts/robot2python.py index 392a239c..68d92b82 100644 --- a/pytest_robotframework/_internal/scripts/robot2python.py +++ b/pytest_robotframework/_internal/scripts/robot2python.py @@ -9,27 +9,37 @@ Module, Name, alias, + arg, + arguments, expr, + parse, stmt, ) -from contextlib import contextmanager +from contextlib import contextmanager, nullcontext from pathlib import Path +from tempfile import TemporaryDirectory from typing import TYPE_CHECKING, Iterator, cast -from robot import running from robot.api import SuiteVisitor, logger -from robot.model.itemlist import ItemList +from robot.api.interfaces import ( + ListenerV2, + StartKeywordAttributes, + StartSuiteAttributes, +) +from robot.api.parsing import ModelVisitor +from robot.libraries import STDLIBS from robot.run import RobotFramework from typer import run from typing_extensions import override +import pytest_robotframework from pytest_robotframework._internal.errors import InternalError, UserError from pytest_robotframework._internal.utils import unparse if TYPE_CHECKING: from types import ModuleType - from robot import model + from robot import model, result def _pythonify_name(name: str) -> str: @@ -40,6 +50,10 @@ def _pytestify_name(name: str) -> str: return f"test_{_pythonify_name(name)}" +def _module_name(module: ModuleType | str) -> str: + return module if isinstance(module, str) else module.__name__ + + def _robot_file(suite: model.TestSuite) -> Path | None: if suite.source is None: raise InternalError(f"ayo whyyo suite aint got no path 💀 ({suite.name})") @@ -47,6 +61,12 @@ def _robot_file(suite: model.TestSuite) -> Path | None: return suite_path if suite_path.suffix == ".robot" else None +class Robot2PythonListener(ListenerV2): + @override + def start_keyword(self, name: str, attributes: StartKeywordAttributes): + super().start_keyword(name, attributes) + + class Robot2Python(SuiteVisitor): def __init__(self, output_dir: Path) -> None: self.modules: dict[Path, Module] = {} @@ -67,9 +87,17 @@ def _stack_frame(self, statement: stmt) -> Iterator[None]: self.statement_stack.pop() def _add_import(self, module: ModuleType | str, names: list[str] | None = None): - module_name = module if isinstance(module, str) else module.__name__ + module_name = _module_name(module) if names is None: names = ["*"] + if [ + expression + for expression in self.current_module.body + if isinstance(expression, ImportFrom) + and expression.module == module_name + and set(expression.names) & {"*", *names} + ]: + return self.current_module.body.insert( # __future__ imports need to go first (0 if module_name == "__future__" else 1), @@ -82,8 +110,13 @@ def _add_import(self, module: ModuleType | str, names: list[str] | None = None): ), ) + def _name(self, name: str, *, module: ModuleType | str | None) -> Name: + if module: + self._add_import(module, [name]) + return Name(id=name) + @override - def start_suite(self, suite: running.TestSuite): + def start_suite(self, suite: result.TestSuite): robot_file = _robot_file(suite) if robot_file is None: return @@ -97,60 +130,93 @@ def start_suite(self, suite: running.TestSuite): / (robot_file.parent).relative_to(self.output_dir) / f"{_pytestify_name(robot_file.stem)}.py" ] = module - for module in cast(ItemList[running.model.Import], suite.resource.imports): - self._add_import(module.name) @override - def end_suite(self, suite: running.TestSuite): - # make sure no tests are actually executed once this is done - suite.tests.clear() # type:ignore[no-untyped-call] + def end_suite(self, suite: result.TestSuite): if _robot_file(suite) is not None: del self.current_module @override - def visit_test(self, test: running.TestCase): - function = FunctionDef( + def visit_test(self, test: result.TestCase): + test_function = FunctionDef( name=_pytestify_name(test.name), args=[], # type:ignore[no-any-expr] decorator_list=[], # type:ignore[no-any-expr] body=[], # type:ignore[no-any-expr] lineno=-1, ) - self.current_module.body.append(function) - with self._stack_frame(function): + self.current_module.body.append(test_function) + with self._stack_frame(test_function): super().visit_test(test) - # eventually this will add imports to self.current_module - def _resolve_call(self, keyword: running.Keyword) -> expr: + @override + # https://github.com/robotframework/robotframework/issues/4940 + def visit_keyword(self, keyword: result.Keyword): # type:ignore[override] if not keyword.name: raise UserError("why yo keyword aint got no name") - python_name = _pythonify_name(keyword.name) + strcomputer = "." + call: expr | None = None - def create_call(function: str, module: ModuleType | None = None) -> Call: - if module: - self._add_import(module, [function]) + def create_call(function: str, module: ModuleType | str | None = None) -> Call: return Call( - func=Name(id=function), + func=self._name(function, module=module), args=[ - Constant(value=arg, kind=None) + # CRINGE: i cbf figureing out how to properly convert robot variables for now so + # just turn them all into fstrings in the jankiest way possible + parse(f"f{arg.replace('${{', '{{')!r}").body[0] for arg in keyword.args # type:ignore[no-any-expr] ], # TODO keywords=[], # type:ignore[no-any-expr] ) - if python_name == "no_operation": - return Constant(value=...) - if python_name == "log": - return create_call("info", logger) - return create_call(python_name) + library_name, _, keyword_name = keyword.name.rpartition(strcomputer) + function_name = _pythonify_name(keyword_name) + module_name: str | None = None + if strcomputer in keyword.name: + # if there's a . that means it was imported from some other module: + if library_name in STDLIBS: + if function_name == "no_operation": + call = Constant(value=...) + elif function_name == "log": + call = create_call("info", logger) + module_name = f"robot.libraries.{library_name}" + else: + module_name = _pythonify_name(library_name) + keyword_function = None + else: + # otherwise assume it was defined in this robot file + keyword_function = FunctionDef( + name=_pythonify_name(function_name), + args=arguments( + # TODO: is there a way to get the positional arg names? + args=[ + arg(arg=f"arg{index}") + for index, _ in enumerate( + keyword.args + ) # type:ignore[no-any-expr] + ] + ), + decorator_list=[ + self._name("keyword", module=pytest_robotframework) + ], # type:ignore[no-any-expr] + body=[], # type:ignore[no-any-expr] + lineno=-1, + ) + self.current_module.body.insert( + # insert the keyword before the first test + next( + index + for index, statement in enumerate(self.current_module.body) + if isinstance(statement, FunctionDef) + ), + keyword_function, + ) + if not call: + call = create_call(function_name, module_name) + cast(FunctionDef, self.context).body.append(Expr(call)) - @override - # https://github.com/robotframework/robotframework/issues/4940 - def visit_keyword(self, keyword: running.Keyword): # type:ignore[override] - function = cast(FunctionDef, self.current_module.body[-1]) - function.body.append(Expr(self._resolve_call(keyword))) - with self._stack_frame(function): + with self._stack_frame(keyword_function) if keyword_function else nullcontext(): super().visit_keyword(keyword) @@ -158,15 +224,19 @@ def _convert(suite: Path, output: Path) -> dict[Path, str]: suite = suite.resolve() output = output.resolve() robot_2_python = Robot2Python(output) - RobotFramework().main( # type:ignore[no-untyped-call] - [suite], # type:ignore[no-any-expr] - prerunmodifier=robot_2_python, - runemptysuite=True, - report=None, - output=None, - log=None, - exitonerror=True, - ) + # ideally we'd set output and log to None since they aren't used, but theyu're needed for the + # prerebotmodifier to run: + with TemporaryDirectory() as output_dir: + RobotFramework().main( # type:ignore[no-untyped-call] + [suite], # type:ignore[no-any-expr] + dryrun=True, + listener=Robot2PythonListener(), + prerebotmodifier=robot_2_python, + runemptysuite=True, + outputdir=output_dir, + report=None, + exitonerror=True, + ) return {path: unparse(module) for path, module in robot_2_python.modules.items()} diff --git a/tests/fixtures/test_robot2python/function_from_library/test_foo.py b/tests/fixtures/test_robot2python/function_from_library/test_foo.py index 050a8cd2..890e9dd5 100644 --- a/tests/fixtures/test_robot2python/function_from_library/test_foo.py +++ b/tests/fixtures/test_robot2python/function_from_library/test_foo.py @@ -1,7 +1,7 @@ from __future__ import annotations -from typing import * # noqa: F403 +from typing import cast def test_foo(): - cast("str", "asdf") # noqa: F405 + cast("str", "asdf") diff --git a/tests/fixtures/test_robot2python/robot_keyword/foo.robot b/tests/fixtures/test_robot2python/robot_keyword/foo.robot new file mode 100644 index 00000000..b9f1ebee --- /dev/null +++ b/tests/fixtures/test_robot2python/robot_keyword/foo.robot @@ -0,0 +1,9 @@ +*** Test Cases *** +Foo + Bar hi + + +*** Keywords *** +Bar + [Arguments] ${a} + Log ${a} diff --git a/tests/fixtures/test_robot2python/robot_keyword/test_foo.py b/tests/fixtures/test_robot2python/robot_keyword/test_foo.py new file mode 100644 index 00000000..34aff2c9 --- /dev/null +++ b/tests/fixtures/test_robot2python/robot_keyword/test_foo.py @@ -0,0 +1,14 @@ +from __future__ import annotations + +from robot.api.logger import info + +from pytest_robotframework import keyword + + +@keyword +def bar(arg1: str): + info(f"{arg1}") # type:ignore[no-untyped-call] + + +def test_foo(): + bar(f"hi")