From a68639dc3b13570a4424468acfdd1a24464d6f30 Mon Sep 17 00:00:00 2001 From: Sam Waseda Date: Tue, 24 Feb 2026 11:03:07 +0100 Subject: [PATCH 01/11] add import parser to deal with local imports --- flowrep/crawler.py | 15 ++++++- flowrep/models/parsers/import_parser.py | 54 +++++++++++++++++++++++++ flowrep/models/parsers/object_scope.py | 5 ++- tests/unit/test_crawler.py | 14 +++++++ 4 files changed, 84 insertions(+), 4 deletions(-) create mode 100644 flowrep/models/parsers/import_parser.py diff --git a/flowrep/crawler.py b/flowrep/crawler.py index aa721376..3fa36c70 100644 --- a/flowrep/crawler.py +++ b/flowrep/crawler.py @@ -4,7 +4,7 @@ from pyiron_snippets import versions -from flowrep.models.parsers import object_scope, parser_helpers +from flowrep.models.parsers import object_scope, parser_helpers, import_parser CallDependencies = dict[versions.VersionInfo, Callable] @@ -44,10 +44,11 @@ def get_call_dependencies( return call_dependencies visited.add(func_fqn) - scope = object_scope.get_scope(func) tree = parser_helpers.get_ast_function_node(func) collector = CallCollector() collector.visit(tree) + local_modules = import_parser.build_scope(collector.imports, collector.import_froms) + scope = object_scope.get_scope(func, extra_modules=local_modules) for call in collector.calls: try: @@ -94,7 +95,17 @@ def split_by_version_availability( class CallCollector(ast.NodeVisitor): def __init__(self): self.calls: list[ast.expr] = [] + self.imports: list[ast.Import] = [] + self.import_froms: list[ast.ImportFrom] = [] def visit_Call(self, node: ast.Call) -> None: self.calls.append(node.func) self.generic_visit(node) + + def visit_Import(self, node: ast.Import) -> None: + self.imports.append(node) + self.generic_visit(node) + + def visit_ImportFrom(self, node: ast.ImportFrom) -> None: + self.import_froms.append(node) + self.generic_visit(node) diff --git a/flowrep/models/parsers/import_parser.py b/flowrep/models/parsers/import_parser.py new file mode 100644 index 00000000..fd5be547 --- /dev/null +++ b/flowrep/models/parsers/import_parser.py @@ -0,0 +1,54 @@ +import ast +import importlib + + +def build_scope(imports: list | None = None, import_froms: list | None = None) -> dict: + """ + Build a scope dictionary from a list of `import` and `from ... import ...` statements. + + Args: + imports (list | None): A list of `ast.Import` nodes. + import_froms (list | None): A list of `ast.ImportFrom` nodes. + + Returns: + dict: A dictionary representing the scope with imported modules and objects. + """ + scope = {} + + imports = imports or [] + import_froms = import_froms or [] + + # Handle `import` statements + for imp in imports: + for alias in imp.names: + module_name = alias.name + asname = alias.asname or module_name + try: + # Dynamically import the module + module = importlib.import_module(module_name) + scope[asname] = module + except ImportError: + print(f"Warning: Could not import module '{module_name}'") + + # Handle `from ... import ...` statements + for imp_from in import_froms: + module_name = imp_from.module + level = imp_from.level + try: + # Dynamically import the module (absolute or relative) + module = importlib.import_module( + module_name, package=None if level == 0 else "." + ) + for alias in imp_from.names: + name = alias.name + asname = alias.asname or name + try: + # Get the specific object from the module + obj = getattr(module, name) + scope[asname] = obj + except AttributeError: + print(f"Warning: Module '{module_name}' has no attribute '{name}'") + except ImportError: + print(f"Warning: Could not import module '{module_name}'") + + return scope diff --git a/flowrep/models/parsers/object_scope.py b/flowrep/models/parsers/object_scope.py index bcae785f..30e4ce15 100644 --- a/flowrep/models/parsers/object_scope.py +++ b/flowrep/models/parsers/object_scope.py @@ -20,8 +20,9 @@ def __getattr__(self, name: str): raise AttributeError(name) from None -def get_scope(func: FunctionType) -> ScopeProxy: - return ScopeProxy(inspect.getmodule(func).__dict__ | vars(builtins)) +def get_scope(func: FunctionType, extra_modules: dict | None = None) -> ScopeProxy: + extra_modules = extra_modules or {} + return ScopeProxy(inspect.getmodule(func).__dict__ | vars(builtins) | extra_modules) def resolve_symbol_to_object( diff --git a/tests/unit/test_crawler.py b/tests/unit/test_crawler.py index cf7606fa..e204ecc3 100644 --- a/tests/unit/test_crawler.py +++ b/tests/unit/test_crawler.py @@ -75,6 +75,14 @@ def _fqns(deps: crawler.CallDependencies) -> set[str]: return {info.fully_qualified_name for info in deps} +def _local_imports(x): + import sys + from math import sqrt + + a = sys.getsizeof(x) + return sqrt(a) + + class TestGetCallDependencies(unittest.TestCase): """Tests for :func:`crawler.get_call_dependencies`.""" @@ -121,6 +129,12 @@ def test_returns_dict_type(self): deps = crawler.get_call_dependencies(_leaf) self.assertIsInstance(deps, dict) + def test_local_imports_included(self): + deps = crawler.get_call_dependencies(_local_imports) + fqns = _fqns(deps) + self.assertIn("sys.getsizeof", fqns) + self.assertIn("math.sqrt", fqns) + class TestSplitByVersionAvailability(unittest.TestCase): """Tests for :func:`crawler.split_by_version_availability`.""" From 9e662ada38366f52c9f6fad98c892c1d9c8075f8 Mon Sep 17 00:00:00 2001 From: Sam Waseda Date: Tue, 24 Feb 2026 11:06:59 +0100 Subject: [PATCH 02/11] Make it more complicated --- tests/unit/test_crawler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_crawler.py b/tests/unit/test_crawler.py index e204ecc3..5a42e45e 100644 --- a/tests/unit/test_crawler.py +++ b/tests/unit/test_crawler.py @@ -76,10 +76,10 @@ def _fqns(deps: crawler.CallDependencies) -> set[str]: def _local_imports(x): - import sys + import sys as s from math import sqrt - a = sys.getsizeof(x) + a = s.getsizeof(x) return sqrt(a) From ded7935b23c2cf76fcdec387c29e85da775b4575 Mon Sep 17 00:00:00 2001 From: Sam Waseda Date: Tue, 24 Feb 2026 12:29:01 +0100 Subject: [PATCH 03/11] ruff --- flowrep/crawler.py | 2 +- flowrep/models/parsers/import_parser.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/flowrep/crawler.py b/flowrep/crawler.py index 3fa36c70..5596fc35 100644 --- a/flowrep/crawler.py +++ b/flowrep/crawler.py @@ -4,7 +4,7 @@ from pyiron_snippets import versions -from flowrep.models.parsers import object_scope, parser_helpers, import_parser +from flowrep.models.parsers import import_parser, object_scope, parser_helpers CallDependencies = dict[versions.VersionInfo, Callable] diff --git a/flowrep/models/parsers/import_parser.py b/flowrep/models/parsers/import_parser.py index fd5be547..b46055db 100644 --- a/flowrep/models/parsers/import_parser.py +++ b/flowrep/models/parsers/import_parser.py @@ -1,4 +1,3 @@ -import ast import importlib From 68cb97d0abc13696f55b63d382671e4fc8be5881 Mon Sep 17 00:00:00 2001 From: Sam Dareska <37879103+samwaseda@users.noreply.github.com> Date: Tue, 24 Feb 2026 14:45:28 +0100 Subject: [PATCH 04/11] Update flowrep/models/parsers/import_parser.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- flowrep/models/parsers/import_parser.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/flowrep/models/parsers/import_parser.py b/flowrep/models/parsers/import_parser.py index b46055db..6fda8bf1 100644 --- a/flowrep/models/parsers/import_parser.py +++ b/flowrep/models/parsers/import_parser.py @@ -1,7 +1,11 @@ import importlib +import ast -def build_scope(imports: list | None = None, import_froms: list | None = None) -> dict: +def build_scope( + imports: list[ast.Import] | None = None, + import_froms: list[ast.ImportFrom] | None = None, +) -> dict: """ Build a scope dictionary from a list of `import` and `from ... import ...` statements. From 0c8a01adfebe420785c3e0f688bfd5d66ec8924e Mon Sep 17 00:00:00 2001 From: Sam Waseda Date: Tue, 24 Feb 2026 15:00:03 +0100 Subject: [PATCH 05/11] ruff mypy --- flowrep/models/parsers/import_parser.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/flowrep/models/parsers/import_parser.py b/flowrep/models/parsers/import_parser.py index 6fda8bf1..3a5b6023 100644 --- a/flowrep/models/parsers/import_parser.py +++ b/flowrep/models/parsers/import_parser.py @@ -1,5 +1,5 @@ -import importlib import ast +import importlib def build_scope( @@ -24,23 +24,21 @@ def build_scope( # Handle `import` statements for imp in imports: for alias in imp.names: - module_name = alias.name - asname = alias.asname or module_name + asname = alias.asname or alias.name try: # Dynamically import the module - module = importlib.import_module(module_name) + module = importlib.import_module(alias.name) scope[asname] = module except ImportError: - print(f"Warning: Could not import module '{module_name}'") + print(f"Warning: Could not import module '{alias.name}'") # Handle `from ... import ...` statements for imp_from in import_froms: - module_name = imp_from.module level = imp_from.level try: # Dynamically import the module (absolute or relative) module = importlib.import_module( - module_name, package=None if level == 0 else "." + imp_from.module, package=None if level == 0 else "." ) for alias in imp_from.names: name = alias.name @@ -50,8 +48,8 @@ def build_scope( obj = getattr(module, name) scope[asname] = obj except AttributeError: - print(f"Warning: Module '{module_name}' has no attribute '{name}'") + print(f"Warning: Module '{imp_from.module}' has no attribute '{name}'") except ImportError: - print(f"Warning: Could not import module '{module_name}'") + print(f"Warning: Could not import module '{imp_from.module}'") return scope From 19f69d00cf2cec822031565c1d022efc7c0218c6 Mon Sep 17 00:00:00 2001 From: Sam Waseda Date: Tue, 24 Feb 2026 15:01:32 +0100 Subject: [PATCH 06/11] black --- flowrep/models/parsers/import_parser.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flowrep/models/parsers/import_parser.py b/flowrep/models/parsers/import_parser.py index 3a5b6023..897520e5 100644 --- a/flowrep/models/parsers/import_parser.py +++ b/flowrep/models/parsers/import_parser.py @@ -48,7 +48,9 @@ def build_scope( obj = getattr(module, name) scope[asname] = obj except AttributeError: - print(f"Warning: Module '{imp_from.module}' has no attribute '{name}'") + print( + f"Warning: Module '{imp_from.module}' has no attribute '{name}'" + ) except ImportError: print(f"Warning: Could not import module '{imp_from.module}'") From bd628d70a9a62733543c95f88304628cb4667334 Mon Sep 17 00:00:00 2001 From: Sam Waseda Date: Tue, 24 Feb 2026 15:10:07 +0100 Subject: [PATCH 07/11] Fail hard when importlib fails --- flowrep/models/parsers/import_parser.py | 37 +++++++++---------------- 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/flowrep/models/parsers/import_parser.py b/flowrep/models/parsers/import_parser.py index 897520e5..e624b7af 100644 --- a/flowrep/models/parsers/import_parser.py +++ b/flowrep/models/parsers/import_parser.py @@ -25,33 +25,22 @@ def build_scope( for imp in imports: for alias in imp.names: asname = alias.asname or alias.name - try: - # Dynamically import the module - module = importlib.import_module(alias.name) - scope[asname] = module - except ImportError: - print(f"Warning: Could not import module '{alias.name}'") + module = importlib.import_module(alias.name) + scope[asname] = module # Handle `from ... import ...` statements for imp_from in import_froms: level = imp_from.level - try: - # Dynamically import the module (absolute or relative) - module = importlib.import_module( - imp_from.module, package=None if level == 0 else "." - ) - for alias in imp_from.names: - name = alias.name - asname = alias.asname or name - try: - # Get the specific object from the module - obj = getattr(module, name) - scope[asname] = obj - except AttributeError: - print( - f"Warning: Module '{imp_from.module}' has no attribute '{name}'" - ) - except ImportError: - print(f"Warning: Could not import module '{imp_from.module}'") + # Dynamically import the module (absolute or relative) + if imp_from.module is None: + raise ValueError("The module attribute of imp_from cannot be None") + module = importlib.import_module( + imp_from.module, package=None if level == 0 else "." + ) + for alias in imp_from.names: + name = alias.name + asname = alias.asname or name + obj = getattr(module, name) + scope[asname] = obj return scope From e058027db2689bc9424ee1dfa631296e1541a420 Mon Sep 17 00:00:00 2001 From: Sam Waseda Date: Sun, 1 Mar 2026 15:38:59 +0100 Subject: [PATCH 08/11] correct import statement --- tests/unit/models/parsers/test_dependency_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/models/parsers/test_dependency_parser.py b/tests/unit/models/parsers/test_dependency_parser.py index b4d92c36..2031de44 100644 --- a/tests/unit/models/parsers/test_dependency_parser.py +++ b/tests/unit/models/parsers/test_dependency_parser.py @@ -180,7 +180,7 @@ def test_non_callable_resolved_symbol_is_skipped(self): self.assertIsInstance(deps, dict) def test_local_imports_included(self): - deps = crawler.get_call_dependencies(_local_imports) + deps = dependency_parser.get_call_dependencies(_local_imports) fqns = _fqns(deps) self.assertIn("sys.getsizeof", fqns) self.assertIn("math.sqrt", fqns) From 3ede4350c421740a91f25f97f32a9b15cb515859 Mon Sep 17 00:00:00 2001 From: Sam Dareska <37879103+samwaseda@users.noreply.github.com> Date: Thu, 19 Mar 2026 19:08:15 +0100 Subject: [PATCH 09/11] Update flowrep/models/parsers/import_parser.py Co-authored-by: Liam Huber --- flowrep/models/parsers/import_parser.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/flowrep/models/parsers/import_parser.py b/flowrep/models/parsers/import_parser.py index e624b7af..09890cb3 100644 --- a/flowrep/models/parsers/import_parser.py +++ b/flowrep/models/parsers/import_parser.py @@ -32,11 +32,12 @@ def build_scope( for imp_from in import_froms: level = imp_from.level # Dynamically import the module (absolute or relative) - if imp_from.module is None: - raise ValueError("The module attribute of imp_from cannot be None") - module = importlib.import_module( - imp_from.module, package=None if level == 0 else "." - ) + if imp_from.module is None or node.level > 0: + raise ValueError( + f"Relative imports are not supported in dependency parsing. " + f"Encountered importing from {imp_from.module}." + ) + module = importlib.import_module(imp_from.module) for alias in imp_from.names: name = alias.name asname = alias.asname or name From 10700213ab1acb95d8939d82db19d65ef25395b4 Mon Sep 17 00:00:00 2001 From: Sam Waseda Date: Thu, 19 Mar 2026 19:22:46 +0100 Subject: [PATCH 10/11] Implement Liam's suggestion (also corecting typo) --- flowrep/models/parsers/import_parser.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/flowrep/models/parsers/import_parser.py b/flowrep/models/parsers/import_parser.py index e624b7af..b9c4c86c 100644 --- a/flowrep/models/parsers/import_parser.py +++ b/flowrep/models/parsers/import_parser.py @@ -32,11 +32,12 @@ def build_scope( for imp_from in import_froms: level = imp_from.level # Dynamically import the module (absolute or relative) - if imp_from.module is None: - raise ValueError("The module attribute of imp_from cannot be None") - module = importlib.import_module( - imp_from.module, package=None if level == 0 else "." - ) + if imp_from.module is None or level > 0: + raise ValueError( + f"Relative imports are not supported in dependency parsing. " + f"Encountered importing from {imp_from.module}." + ) + module = importlib.import_module(imp_from.module) for alias in imp_from.names: name = alias.name asname = alias.asname or name From b16ffabc3e7e19bd9b78eb8088ce4a29248f6aff Mon Sep 17 00:00:00 2001 From: liamhuber Date: Thu, 19 Mar 2026 12:07:43 -0700 Subject: [PATCH 11/11] Test exception branch Signed-off-by: liamhuber --- tests/unit/models/parsers/test_dependency_parser.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/unit/models/parsers/test_dependency_parser.py b/tests/unit/models/parsers/test_dependency_parser.py index 2031de44..7814d566 100644 --- a/tests/unit/models/parsers/test_dependency_parser.py +++ b/tests/unit/models/parsers/test_dependency_parser.py @@ -94,6 +94,13 @@ def _local_imports(x): return sqrt(a) +def _import_from_sibling(x, y): + from .test_for_parser import pair + + a, b = pair(x, y) + return a, b + + class TestGetCallDependencies(unittest.TestCase): """Tests for :func:`dependency_parser.get_call_dependencies`.""" @@ -185,6 +192,12 @@ def test_local_imports_included(self): self.assertIn("sys.getsizeof", fqns) self.assertIn("math.sqrt", fqns) + def test_relative_import_raises(self): + with self.assertRaises(ValueError) as ctx: + dependency_parser.get_call_dependencies(_import_from_sibling) + self.assertIn("Relative imports are not supported", str(ctx.exception)) + self.assertIn("test_for_parser", str(ctx.exception)) + class TestSplitByVersionAvailability(unittest.TestCase): """Tests for :func:`dependency_parser.split_by_version_availability`."""