From 9b5011eeb3be6eed413f104c089fc99b55dd27b0 Mon Sep 17 00:00:00 2001 From: refai06 Date: Tue, 10 Jun 2025 09:24:37 +0530 Subject: [PATCH 1/3] Initial commit - update code Signed-off-by: refai06 --- .../workflow/notebooktools/code_analyzer.py | 67 +++++++++++++++++++ openfl/utilities/workspace.py | 3 + 2 files changed, 70 insertions(+) diff --git a/openfl/experimental/workflow/notebooktools/code_analyzer.py b/openfl/experimental/workflow/notebooktools/code_analyzer.py index 9fed1e7426..84b204f947 100644 --- a/openfl/experimental/workflow/notebooktools/code_analyzer.py +++ b/openfl/experimental/workflow/notebooktools/code_analyzer.py @@ -4,6 +4,7 @@ import ast import inspect import re +import shutil import sys from importlib import import_module from pathlib import Path @@ -45,6 +46,8 @@ def __init__(self, notebook_path: Path, output_path: Path) -> None: ) ).resolve() self.requirements = self._get_requirements() + user_imports = self.__extract_user_defined_imports(notebook_path) + self.__copy_user_defined_modules(user_imports, notebook_path) self.__modify_experiment_script() def __get_exp_name(self, notebook_path: Path) -> str: @@ -86,6 +89,70 @@ def __convert_to_python(self, notebook_path: Path, output_path: Path, export_fil return Path(output_path).joinpath(export_filename).resolve() + def __extract_user_defined_imports(self, notebook_path) -> List[str]: + """ + Extract user-defined imports, excluding inbuild and third-party module + + Args: + notebook_path: Path to Jupyter notebook. + + """ + with open(self.script_path, "r") as file: + code = "".join(line for line in file if not line.lstrip().startswith(("!", "%"))) + + tree = ast.parse(code) + user_imports = set() + + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + module_name = alias.name.split(".")[0] + if self._is_user_defined_module(module_name, notebook_path): + user_imports.add(module_name) + + elif isinstance(node, ast.ImportFrom) and node.module and node.level == 0: + module_name = node.module.split(".")[0] + if self._is_user_defined_module(module_name, notebook_path): + user_imports.add(module_name) + + return list(user_imports) + + def _is_user_defined_module(self, module_name: str, notebook_path: Path) -> bool: + """ + Check if a module is user-defined + + Args: + notebook_path: Path to Jupyter notebook. + """ + notebook_dir = notebook_path.parent + module_path = notebook_dir / f"{module_name}.py" + + module_dir = notebook_dir / module_name + + if (module_path.exists() and module_path.is_file()) or module_dir.exists(): + return True + + return False + + def __copy_user_defined_modules(self, module_names: List[str], notebook_path: Path) -> None: + """ + Copies user-defined modules/packages to the workspace's src directory + + Args: + module_name: List of module name to copy. + notebook_path: Path to Jupyter notebook. + """ + src_dir = self.script_path.parent + for module_name in module_names: + module_file = notebook_path.parent / f"{module_name}.py" + module_dir = notebook_path.parent / module_name + if module_file.exists() and module_file.is_file(): + shutil.copy(module_file, src_dir) + print(f"Copied used-defined module: {module_name}.py") + elif module_dir.exists() and module_dir.is_dir(): + shutil.copytree(module_dir, src_dir / module_name, dirs_exist_ok=True) + print(f"Copied used-defined directory: {module_name}/") + def __modify_experiment_script(self) -> None: """Modifies the given python script by commenting out following code: - occurences of flflow.run() diff --git a/openfl/utilities/workspace.py b/openfl/utilities/workspace.py index 15e7a3a339..358aa36dc0 100644 --- a/openfl/utilities/workspace.py +++ b/openfl/utilities/workspace.py @@ -106,6 +106,7 @@ def __enter__(self): # This is needed for python module finder sys.path.append(str(self.experiment_work_dir)) + sys.path.append(str(self.experiment_work_dir / "src")) def __exit__(self, exc_type, exc_value, traceback): """Remove the workspace.""" @@ -113,6 +114,8 @@ def __exit__(self, exc_type, exc_value, traceback): shutil.rmtree(self.experiment_work_dir, ignore_errors=True) if str(self.experiment_work_dir) in sys.path: sys.path.remove(str(self.experiment_work_dir)) + if str(self.experiment_work_dir / "src") in sys.path: + sys.path.remove(str(self.experiment_work_dir / "src")) if self.remove_archive: logger.debug( From 9857c2a030db27b7ad47fdc64ce4721a0b36526d Mon Sep 17 00:00:00 2001 From: refai06 Date: Thu, 12 Jun 2025 14:57:30 +0530 Subject: [PATCH 2/3] Incorporate internal-review comment Signed-off-by: refai06 --- .../workflow/notebooktools/code_analyzer.py | 60 +++++++++++-------- openfl/utilities/workspace.py | 6 +- 2 files changed, 39 insertions(+), 27 deletions(-) diff --git a/openfl/experimental/workflow/notebooktools/code_analyzer.py b/openfl/experimental/workflow/notebooktools/code_analyzer.py index 84b204f947..36a4798a69 100644 --- a/openfl/experimental/workflow/notebooktools/code_analyzer.py +++ b/openfl/experimental/workflow/notebooktools/code_analyzer.py @@ -8,7 +8,7 @@ import sys from importlib import import_module from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Tuple import nbformat from nbdev.export import nb_export @@ -117,41 +117,23 @@ def __extract_user_defined_imports(self, notebook_path) -> List[str]: return list(user_imports) - def _is_user_defined_module(self, module_name: str, notebook_path: Path) -> bool: - """ - Check if a module is user-defined - - Args: - notebook_path: Path to Jupyter notebook. - """ - notebook_dir = notebook_path.parent - module_path = notebook_dir / f"{module_name}.py" - - module_dir = notebook_dir / module_name - - if (module_path.exists() and module_path.is_file()) or module_dir.exists(): - return True - - return False - def __copy_user_defined_modules(self, module_names: List[str], notebook_path: Path) -> None: """ Copies user-defined modules/packages to the workspace's src directory Args: - module_name: List of module name to copy. + module_names: List of module names. notebook_path: Path to Jupyter notebook. """ src_dir = self.script_path.parent for module_name in module_names: - module_file = notebook_path.parent / f"{module_name}.py" - module_dir = notebook_path.parent / module_name - if module_file.exists() and module_file.is_file(): - shutil.copy(module_file, src_dir) - print(f"Copied used-defined module: {module_name}.py") + module_path, module_dir = self._get_module_paths(module_name, notebook_path) + if module_path.exists() and module_path.is_file(): + shutil.copy(module_path, src_dir) + print(f"Copied user-defined module: {module_name}.py") elif module_dir.exists() and module_dir.is_dir(): shutil.copytree(module_dir, src_dir / module_name, dirs_exist_ok=True) - print(f"Copied used-defined directory: {module_name}/") + print(f"Copied user-defined directory: {module_name}/") def __modify_experiment_script(self) -> None: """Modifies the given python script by commenting out following code: @@ -361,6 +343,34 @@ def _clean_value(self, value: str) -> str: value = value.lstrip("[").rstrip("]") return value + def _is_user_defined_module(self, module_name: str, notebook_path: Path) -> bool: + """ + Check if a module is user-defined + + Args: + module_name: Name of the module. + notebook_path: Path to Jupyter notebook. + """ + if not isinstance(module_name, str) or not module_name.strip(): + return False + + module_path, module_dir = self._get_module_paths(module_name, notebook_path) + + return (module_path.exists() and module_path.is_file()) or module_dir.exists() + + def _get_module_paths(self, module_name: str, notebook_path: Path) -> Tuple: + """ + Get the file and directory paths for a user-defined module + + Args: + module_name: Name of the module. + notebook_path: Path to the Jupyter notebook. + """ + notebook_dir = notebook_path.parent + module_path = notebook_dir / f"{module_name}.py" + module_dir = notebook_dir / module_name + return module_path, module_dir + def _get_requirements(self) -> List[str]: """Extract pip libraries from the script diff --git a/openfl/utilities/workspace.py b/openfl/utilities/workspace.py index 358aa36dc0..6971bf09ba 100644 --- a/openfl/utilities/workspace.py +++ b/openfl/utilities/workspace.py @@ -105,8 +105,10 @@ def __enter__(self): os.chdir(self.experiment_work_dir) # This is needed for python module finder - sys.path.append(str(self.experiment_work_dir)) - sys.path.append(str(self.experiment_work_dir / "src")) + for path in [self.experiment_work_dir, self.experiment_work_dir / "src"]: + path_str = str(path) + if path_str not in sys.path: + sys.path.append(path_str) def __exit__(self, exc_type, exc_value, traceback): """Remove the workspace.""" From e40a28fa1c7267eba84fbef3cf8bdd0faf5aa0e8 Mon Sep 17 00:00:00 2001 From: refai06 Date: Mon, 16 Jun 2025 11:40:41 +0530 Subject: [PATCH 3/3] Incorporate comments Signed-off-by: refai06 --- .../workflow/notebooktools/code_analyzer.py | 54 ++++++++++++------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/openfl/experimental/workflow/notebooktools/code_analyzer.py b/openfl/experimental/workflow/notebooktools/code_analyzer.py index 36a4798a69..2fb15c844f 100644 --- a/openfl/experimental/workflow/notebooktools/code_analyzer.py +++ b/openfl/experimental/workflow/notebooktools/code_analyzer.py @@ -89,13 +89,16 @@ def __convert_to_python(self, notebook_path: Path, output_path: Path, export_fil return Path(output_path).joinpath(export_filename).resolve() - def __extract_user_defined_imports(self, notebook_path) -> List[str]: + def __extract_user_defined_imports(self, notebook_path: Path) -> List[str]: """ - Extract user-defined imports, excluding inbuild and third-party module + Extract user-defined module imports from the notebook script, + excluding standard library and third-party modules. Args: - notebook_path: Path to Jupyter notebook. - + notebook_path (Path): Path to the Jupyter notebook. + + Returns: + List[str]: A list of user-defined module names used in the notebook. """ with open(self.script_path, "r") as file: code = "".join(line for line in file if not line.lstrip().startswith(("!", "%"))) @@ -119,21 +122,24 @@ def __extract_user_defined_imports(self, notebook_path) -> List[str]: def __copy_user_defined_modules(self, module_names: List[str], notebook_path: Path) -> None: """ - Copies user-defined modules/packages to the workspace's src directory + Copies user-defined modules/packages to the generated workspace's src directory Args: - module_names: List of module names. - notebook_path: Path to Jupyter notebook. + module_names (List[str]): A list of user-defined module names + notebook_path (Path): Path to Jupyter notebook. """ src_dir = self.script_path.parent for module_name in module_names: - module_path, module_dir = self._get_module_paths(module_name, notebook_path) - if module_path.exists() and module_path.is_file(): - shutil.copy(module_path, src_dir) - print(f"Copied user-defined module: {module_name}.py") - elif module_dir.exists() and module_dir.is_dir(): - shutil.copytree(module_dir, src_dir / module_name, dirs_exist_ok=True) - print(f"Copied user-defined directory: {module_name}/") + try: + module_path, module_dir = self._get_module_paths(module_name, notebook_path) + if module_path.exists() and module_path.is_file(): + shutil.copy(module_path, src_dir) + print(f"Copied user-defined module: {module_name}.py") + elif module_dir.exists() and module_dir.is_dir(): + shutil.copytree(module_dir, src_dir / module_name, dirs_exist_ok=True) + print(f"Copied user-defined directory: {module_name}/") + except Exception as e: + print(f"[WARNING] Failed to copy '{module_name}':{e}") def __modify_experiment_script(self) -> None: """Modifies the given python script by commenting out following code: @@ -345,26 +351,34 @@ def _clean_value(self, value: str) -> str: def _is_user_defined_module(self, module_name: str, notebook_path: Path) -> bool: """ - Check if a module is user-defined + Determine whether a given module is user-defined. Args: - module_name: Name of the module. - notebook_path: Path to Jupyter notebook. + module_name (str): Name of the module. + notebook_path (Path): Path to Jupyter notebook using the module. + + Return: + bool: True if the module is user-defined, False otherwise. """ + # Reject empty or non-string module names if not isinstance(module_name, str) or not module_name.strip(): return False + # Expected file path or directory path of the module module_path, module_dir = self._get_module_paths(module_name, notebook_path) return (module_path.exists() and module_path.is_file()) or module_dir.exists() - def _get_module_paths(self, module_name: str, notebook_path: Path) -> Tuple: + def _get_module_paths(self, module_name: str, notebook_path: Path) -> Tuple[Path, Path]: """ Get the file and directory paths for a user-defined module Args: - module_name: Name of the module. - notebook_path: Path to the Jupyter notebook. + module_name (str): Name of the module. + notebook_path (Path): Path to the Jupyter notebook. + + Returns: + Tuple[Path, Path]: (module_file_path, module_directory_path) """ notebook_dir = notebook_path.parent module_path = notebook_dir / f"{module_name}.py"