From c0341c2d34fea6ddb2204495888096f459d1d97f Mon Sep 17 00:00:00 2001 From: guillaume-gricourt Date: Thu, 16 Oct 2025 13:31:45 +0200 Subject: [PATCH 1/4] feat(knime): allow online and local install from Zenodo --- README.md | 12 +- retropath2_wrapper/knime.py | 253 ++++++++++++++++++++++++------------ 2 files changed, 178 insertions(+), 87 deletions(-) diff --git a/README.md b/README.md index f0028f1..1755e49 100644 --- a/README.md +++ b/README.md @@ -117,13 +117,13 @@ To run functional tests, the environment variable `RP2_FUNCTIONAL=TRUE` is requi ### Knime dependencies -Availbale options: +Available options: 1. You provide a path of the Knime root directory through the argument `--kinstall`. You need to have the following libraries installed: `org.knime.features.chem.types.feature.group`, `org.knime.features.datageneration.feature.group`, `org.knime.features.python.feature.group`, `org.rdkit.knime.feature.feature.group` 2. `retropath2_wrapper` will install Knime for you, at the root of the python package, by downloading the softwares available on Zenodo. You can choose a version among `4.6.4` or `4.7.0`. Optionally, you can locate a path for the installation. If an executable is found in the path, Knime will not be reinstalled. ```bash -python -m retropath2_wrapper.knime \ +python -m retropath2_wrapper.knime online \ --kinstall "/path/to/knime/root/dir" \ --kver {4.6.4,4.7.0} ``` @@ -134,6 +134,14 @@ Knime software and packages are available at: - [KNIME v4.6.4 - Zenodo](https://zenodo.org/record/7515771) - [KNIME v4.7.0 - Zenodo](https://zenodo.org/record/7564938) +All files from these repositories can be downloaded through the `Download all` button: + +```bash +python -m retropath2_wrapper.knime local \ + --kinstall "/path/to/knime/root/dir" \ + --zenodo-zip "/path/to/7515771.zip" +``` + ## Known issues 1. Could not load native RDKit library, libfreetype.so.6: cannot open shared object file diff --git a/retropath2_wrapper/knime.py b/retropath2_wrapper/knime.py index 7bfc5fe..3e5cf6f 100644 --- a/retropath2_wrapper/knime.py +++ b/retropath2_wrapper/knime.py @@ -1,6 +1,8 @@ import argparse import glob import os +import re +from zipfile import ZipFile import requests import shutil import subprocess @@ -20,9 +22,8 @@ from subprocess import PIPE as sp_PIPE from brs_utils import ( - download_and_extract_tar_gz, + extract_tar_gz, download, - download_and_unzip, chown_r, subprocess_call, ) @@ -33,8 +34,26 @@ from retropath2_wrapper.preference import Preference +# Patch for brs-utils: https://github.com/brsynth/brs-utils/issues/6 +def unzip( + file: str, + dir: str +) -> None: + '''Unzip the given file. + + Parameters + ---------- + file: str + Filename to unzip + dir: str + Directory to unzip into + ''' + with ZipFile(file, 'r') as zip_ref: + zip_ref.extractall(dir) + class Knime(object): """Knime is useful to install executable, install packages or commandline. + http://download.knime.org/analytics-platform/ Attributes ---------- @@ -44,13 +63,12 @@ class Knime(object): path of the Knime workflow """ ZENODO_API = "https://zenodo.org/api/" - KNIME_URL = "http://download.knime.org/analytics-platform/" ZENODO = { "4.6.4": "7515771", "4.7.0": "7564938", } DEFAULT_VERSION = "4.6.4" - PLUGINGS = [ + PLUGINS = [ "org.eclipse.equinox.preferences", "org.knime.chem.base", "org.knime.datageneration", @@ -120,7 +138,7 @@ def find_executable(cls, path: str) -> str: for file in files: path_file = os.path.join(root, file) if os.access(path_file, os.X_OK) and os.path.isfile(path_file): - if file.lower() == "knime": + if "knime" in os.path.basename(file.lower()): return os.path.abspath(path_file) return "" @@ -140,71 +158,102 @@ def collect_top_level_dirs(cls, path) -> Set: names.add(p.name) return names - def install(self, kver: str, logger: Logger = getLogger(__name__)) -> bool: + @classmethod + def download_from_zenodo(cls, path: str, kver: str, logger: Logger = getLogger(__name__)): + """ + Download files from a Zenodo repository + + Parameters + ---------- + path: str + An empty directory where files will be downloaded + kver: str + 4.6.4 or 4.7.0 + """ data = Knime.zenodo_show_repo(kver=kver) - # Install Knime - dirs_before = Knime.collect_top_level_dirs(path=self.kinstall) for file in data["files"]: basename = file["key"] url = file["links"]["self"] + foutput = os.path.join(path, basename) + logger.info(f"Download: {url} to {foutput}") + download(url, foutput) + + def install(self, path: str, logger: Logger = getLogger(__name__)) -> bool: + """ + Install Knime + + Parameters + ---------- + path: str + Directory with all files from Zenodo, such as: + - knime_4.6.4.app.macosx.cocoa.x86_64.dmg + - knime_4.6.4.linux.gtk.x86_64.tar.gz + - knime_4.6.4.win32.win32.x86_64.zip + - org.knime.update.analytics-platform_4.6.4.zip + - TrustedCommunityContributions_4.6_202212212136.zip + """ + knime_files = glob.glob(os.path.join(path, "*")) + # Install Knime + dirs_before = Knime.collect_top_level_dirs(path=self.kinstall) + for file in knime_files: + basename = os.path.basename(file) if "linux" in basename and sys.platform == "linux": - download_and_extract_tar_gz(url, self.kinstall) + extract_tar_gz(file, self.kinstall) chown_r(self.kinstall, getuser()) # chown_r(kinstall, geteuid(), getegid()) break elif "macosx" in basename and sys.platform == "darwin": - with tempfile.NamedTemporaryFile() as tempf: - download(url, tempf.name) - app_path = f'{self.kinstall}/KNIME_{kver}.app' - if os.path.exists(app_path): - shutil.rmtree(app_path) - with tempfile.TemporaryDirectory() as tempd: - cmd = f'hdiutil mount -noverify {tempf.name} -mountpoint {tempd}/KNIME' - subprocess_call(cmd, logger=logger) - shutil.copytree( - f'{tempd}/KNIME/KNIME {kver}.app', - app_path - ) - cmd = f'hdiutil unmount {tempd}/KNIME' - subprocess_call(cmd, logger=logger) + match = re.search(r"\d+\.\d+\.\d+", basename) + if match: + kver = match.group() + else: + raise ValueError(f"Version can not be guessed from filename: {file}") + app_path = f'{self.kinstall}/KNIME_{kver}.app' + if os.path.exists(app_path): + shutil.rmtree(app_path) + with tempfile.TemporaryDirectory() as tempd: + cmd = f'hdiutil mount -noverify {file} -mountpoint {tempd}/KNIME' + subprocess_call(cmd, logger=logger) + shutil.copytree( + f'{tempd}/KNIME/KNIME {kver}.app', + app_path + ) + cmd = f'hdiutil unmount {tempd}/KNIME' + subprocess_call(cmd, logger=logger) break elif "win32" in basename and sys.platform == "win32": - download_and_unzip(url, self.kinstall) + unzip(file, self.kinstall) break dirs_after = Knime.collect_top_level_dirs(path=self.kinstall) dirs_only_after = dirs_after - dirs_before assert len(dirs_only_after) == 1, dirs_only_after # Download Plugins - tempdir = tempfile.mkdtemp() - try: - path_plugins = [] - for file in data["files"]: - basename = file["key"] - url = file["links"]["self"] - if "org.knime.update" in basename or "TrustedCommunity" in basename: - path_plugin = os.path.join(tempdir, basename) - download(url=url, file=path_plugin) - path_plugins.append(path_plugin) - - # Install Plugins - self.kexec = Knime.find_executable(path=self.kinstall) - p2_dir = Knime.find_p2_dir(path=self.kinstall) - args = [f"{self.kexec}"] - args += ["-nosplash", "-consoleLog"] - args += ["-application", "org.eclipse.equinox.p2.director"] - args += ["-repository", ",".join([f"jar:file:{path_plugin}!/" for path_plugin in path_plugins])] - args += ["-bundlepool", p2_dir] - args += ["-destination", os.path.abspath(os.path.join(self.kinstall, dirs_only_after.pop()))] - args += ["-i", ",".join(Knime.PLUGINGS)] + path_plugins = [] + for file in knime_files: + basename = os.path.basename(file) + if "org.knime.update" in basename or "TrustedCommunity" in basename: + path_plugins.append(os.path.abspath(file)) - CPE = subprocess.run(args) - logger.debug(CPE) - except Exception as error: - logger.error(error) - finally: - # Clean up - shutil.rmtree(tempdir) + # Install Plugins + self.kexec = Knime.find_executable(path=self.kinstall) + p2_dir = Knime.find_p2_dir(path=self.kinstall) + + if not self.kexec or not os.path.exists(self.kexec): + raise FileNotFoundError(f"KNIME executable not found under {self.kinstall}") + if not p2_dir: + raise RuntimeError("p2 directory not found after installation.") + + args = [f"{self.kexec}"] + args += ["-nosplash", "-consoleLog"] + args += ["-application", "org.eclipse.equinox.p2.director"] + args += ["-repository", ",".join([f"jar:file:{path_plugin}!/" for path_plugin in path_plugins])] + args += ["-bundlepool", p2_dir] + args += ["-destination", os.path.abspath(os.path.join(self.kinstall, dirs_only_after.pop()))] + args += ["-i", ",".join(Knime.PLUGINS)] + CPE = subprocess.run(args) + logger.debug(CPE) + return True def call( @@ -234,6 +283,9 @@ def call( StreamHandler.terminator = "" logger.info('{attr1}Running KNIME...{attr2}'.format(attr1=attr('bold'), attr2=attr('reset'))) + if not self.kexec or not os.path.exists(self.kexec): + raise FileNotFoundError(f"KNIME executable not found under {self.kinstall}") + args = [self.kexec] args += ["-nosplash"] args += ["-nosave"] @@ -255,7 +307,6 @@ def call( args += ['-workflow.variable=output.dir,"%s",String' % (self.standardize_path(files['outdir']),)] args += ['-workflow.variable=output.solutionfile,"%s",String' % (self.standardize_path(files['results']),)] args += ['-workflow.variable=output.sourceinsinkfile,"%s",String' % (self.standardize_path(files['src-in-sk']),)] - print("Hydrogen:", params["std_hydrogen"]) args += ['-workflow.variable=input.std_mode,"%s",String' % (params["std_hydrogen"],)] if preference and preference.is_init(): preference.to_file() @@ -263,29 +314,19 @@ def call( logger.debug(" ".join(args)) + old_ld = os.environ.get("LD_LIBRARY_PATH") try: printout = open(os.devnull, 'wb') if logger.level > 10 else None # Hack to link libGraphMolWrap.so (RDKit) against libfreetype.so.6 (from conda) - is_ld_path_modified = False - if "CONDA_PREFIX" in os.environ.keys(): - os.environ['LD_LIBRARY_PATH'] = os.environ.get( - 'LD_LIBRARY_PATH', - '' - ) + ':' + os.path.join( - os.environ['CONDA_PREFIX'], - "lib" - ) + ':' + os.path.join( - os.environ['CONDA_PREFIX'], - "x86_64-conda-linux-gnu/sysroot/usr/lib64" - ) - is_ld_path_modified = True + if "CONDA_PREFIX" in os.environ: + extra = [ + os.path.join(os.environ["CONDA_PREFIX"], "lib"), + os.path.join(os.environ["CONDA_PREFIX"], "x86_64-conda-linux-gnu/sysroot/usr/lib64"), + ] + os.environ["LD_LIBRARY_PATH"] = ":".join(filter(None, [old_ld, *extra])) CPE = subprocess.run(args) logger.debug(CPE) - if is_ld_path_modified: - os.environ['LD_LIBRARY_PATH'] = ':'.join( - os.environ['LD_LIBRARY_PATH'].split(':')[:-1] - ) StreamHandler.terminator = "\n" logger.info(' {bold}OK{reset}'.format(bold=attr('bold'), reset=attr('reset'))) @@ -294,28 +335,70 @@ def call( except OSError as e: logger.error(e) return RETCODES['OSError'] + finally: + if old_ld is None: + os.environ.pop("LD_LIBRARY_PATH", None) + else: + os.environ["LD_LIBRARY_PATH"] = old_ld +def install_online(args, logger: Logger = getLogger(__name__)): + path_knime = args.kinstall + kver = args.kver + + tempdir = tempfile.mkdtemp() + os.makedirs(path_knime, exist_ok=True) + try: + Knime.download_from_zenodo(path=tempdir, kver=kver, logger=logger) + knime = Knime(kinstall=path_knime) + knime.install(path=tempdir) + except Exception as e: + logger.error(e) + finally: + shutil.rmtree(tempdir) + +def install_local(args, logger: Logger = getLogger(__name__)): + path_knime = args.kinstall + path_zenodo = args.zenodo_zip + + tempdir = tempfile.mkdtemp() + os.makedirs(path_knime, exist_ok=True) + try: + unzip( + file=path_zenodo, + dir=tempdir, + ) + knime = Knime(kinstall=path_knime) + knime.install(path=tempdir) + except Exception as e: + logger.error(e) + finally: + shutil.rmtree(tempdir) + if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument( + subparsers = parser.add_subparsers(required=True) + + # Online + par_onl = subparsers.add_parser("online") + par_onl.add_argument( "--kinstall", required=True, help="Path to install Knime" ) - parser.add_argument( + par_onl.add_argument( "--kver", default="4.6.4", choices=["4.6.4", "4.7.0"], help="Knime version" ) - parser.add_argument( - "--overwrite", action="store_true", help="Install even if the executable is not present" + par_onl.set_defaults(func=install_online) + + # Local + par_loc = subparsers.add_parser("local") + par_loc.add_argument( + "--kinstall", required=True, help="Path to install Knime" ) - args = parser.parse_args() - - path_knime = args.kinstall - kver = args.kver - to_overwrite = True if args.overwrite else False - - os.makedirs(path_knime, exist_ok=True) - knime = Knime(kinstall=path_knime) + zenodo_files = ", ".join([x + ".zip" for x in Knime.ZENODO.values()]) + par_loc.add_argument( + "--zenodo-zip", required=True, help=f"Zenodo file obtained from \"Download all\" button from Zenodo, such as {zenodo_files}" + ) + par_loc.set_defaults(func=install_local) - if to_overwrite or not knime.kexec: - knime.install(kver=kver) - \ No newline at end of file + args = parser.parse_args() + args.func(args) \ No newline at end of file From a2e202b42830d83483a0c0c9ec01646e38e71c58 Mon Sep 17 00:00:00 2001 From: guillaume-gricourt Date: Thu, 16 Oct 2025 13:45:10 +0200 Subject: [PATCH 2/4] feat(knime): raise error if installation is failed, install Knime online in the code --- README.md | 1 - retropath2_wrapper/Args.py | 1 - retropath2_wrapper/RetroPath2.py | 9 +++++++-- retropath2_wrapper/knime.py | 8 ++++---- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 1755e49..290c342 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,6 @@ Executions can be timed out using the `timeout` arguments (in minutes). - 2: Running the RetroPath2.0 Knime program produced an OSError - 3: InChI is malformated - 4: Sink file is malformed -- 5: Knime installation returns an error - 10: Source has been found in the sink (warning) - 11: No solution is found (warning) diff --git a/retropath2_wrapper/Args.py b/retropath2_wrapper/Args.py index 20cbb00..1e68b6a 100644 --- a/retropath2_wrapper/Args.py +++ b/retropath2_wrapper/Args.py @@ -30,7 +30,6 @@ 'OSError': 2, 'InChI': 3, 'SinkFileMalformed': 4, - 'KnimeInstallationError': 5, } diff --git a/retropath2_wrapper/RetroPath2.py b/retropath2_wrapper/RetroPath2.py index cbeb51c..f170833 100644 --- a/retropath2_wrapper/RetroPath2.py +++ b/retropath2_wrapper/RetroPath2.py @@ -19,6 +19,7 @@ from filetype import guess from tempfile import TemporaryDirectory from typing import Dict, Tuple +from types import SimpleNamespace from logging import ( Logger, getLogger @@ -32,6 +33,7 @@ DEFAULTS, RETCODES, ) +from retropath2_wrapper import knime as knime_module from retropath2_wrapper.knime import Knime from retropath2_wrapper.preference import Preference @@ -78,8 +80,11 @@ def retropath2( ) if knime.kexec == "": # Install KNIME - if not knime.install(kver=Knime.DEFAULT_VERSION, logger=logger): - return RETCODES["KnimeInstallationError"] + args_knime = SimpleNamespace( + kinstall=knime.kinstall, + kver=Knime.DEFAULT_VERSION, + ) + knime_module.install_online(args_knime, logger=logger) logger.debug('knime: ' + str(knime)) # Store RetroPath2 params into a dictionary diff --git a/retropath2_wrapper/knime.py b/retropath2_wrapper/knime.py index 3e5cf6f..c51242c 100644 --- a/retropath2_wrapper/knime.py +++ b/retropath2_wrapper/knime.py @@ -351,9 +351,9 @@ def install_online(args, logger: Logger = getLogger(__name__)): try: Knime.download_from_zenodo(path=tempdir, kver=kver, logger=logger) knime = Knime(kinstall=path_knime) - knime.install(path=tempdir) + knime.install(path=tempdir, logger=logger) except Exception as e: - logger.error(e) + raise RuntimeError(e) finally: shutil.rmtree(tempdir) @@ -368,10 +368,10 @@ def install_local(args, logger: Logger = getLogger(__name__)): file=path_zenodo, dir=tempdir, ) - knime = Knime(kinstall=path_knime) + knime = Knime(kinstall=path_knime, logger=logger) knime.install(path=tempdir) except Exception as e: - logger.error(e) + raise RuntimeError(e) finally: shutil.rmtree(tempdir) From 6cb6a6eef00d6018e884d0c8e50f989de8acc819 Mon Sep 17 00:00:00 2001 From: guillaume-gricourt Date: Fri, 17 Oct 2025 11:50:44 +0200 Subject: [PATCH 3/4] fix(knime): mv logger arg to another function --- retropath2_wrapper/knime.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/retropath2_wrapper/knime.py b/retropath2_wrapper/knime.py index c51242c..06e442b 100644 --- a/retropath2_wrapper/knime.py +++ b/retropath2_wrapper/knime.py @@ -251,6 +251,8 @@ def install(self, path: str, logger: Logger = getLogger(__name__)) -> bool: args += ["-bundlepool", p2_dir] args += ["-destination", os.path.abspath(os.path.join(self.kinstall, dirs_only_after.pop()))] args += ["-i", ",".join(Knime.PLUGINS)] + logger.info("Command line to install Knime plugins") + logger.info(" ".join(args)) CPE = subprocess.run(args) logger.debug(CPE) @@ -364,12 +366,14 @@ def install_local(args, logger: Logger = getLogger(__name__)): tempdir = tempfile.mkdtemp() os.makedirs(path_knime, exist_ok=True) try: + logger.info(f"Unzip: {path_zenodo}") unzip( file=path_zenodo, dir=tempdir, ) - knime = Knime(kinstall=path_knime, logger=logger) - knime.install(path=tempdir) + knime = Knime(kinstall=path_knime) + logger.info(f"Install to: {path_knime}") + knime.install(path=tempdir, logger=logger) except Exception as e: raise RuntimeError(e) finally: From c7774dad5273d2c9403c859bbc8979bebffbdbcf Mon Sep 17 00:00:00 2001 From: guillaume-gricourt Date: Sun, 19 Oct 2025 21:31:04 +0200 Subject: [PATCH 4/4] fix(knime): trigger self.kexec of Knime --- retropath2_wrapper/RetroPath2.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/retropath2_wrapper/RetroPath2.py b/retropath2_wrapper/RetroPath2.py index f170833..ff94dd0 100644 --- a/retropath2_wrapper/RetroPath2.py +++ b/retropath2_wrapper/RetroPath2.py @@ -85,6 +85,11 @@ def retropath2( kver=Knime.DEFAULT_VERSION, ) knime_module.install_online(args_knime, logger=logger) + # Init Knime again to set Knime.kexec + knime = Knime( + kinstall=knime.kinstall, + workflow=knime.workflow, + ) logger.debug('knime: ' + str(knime)) # Store RetroPath2 params into a dictionary