diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..f017d6d --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,42 @@ +# Code of Conduct + +## Our Commitment + +We are dedicated to providing a welcoming, inclusive, and respectful +environment for all contributors. Whether you are contributing code, +documentation, or feedback, we expect everyone to interact in a manner that +is constructive and respectful. + +## Standards of Behavior + +- **Be Respectful**: Treat everyone with kindness and respect. +- **Be Collaborative**: Value diverse perspectives and work together toward + solutions. +- **Be Professional**: Avoid personal attacks, harassment, or inappropriate + language. +- **Be Open**: Encourage questions and contributions, regardless of experience + level or background. + +## Unacceptable Behavior + +The following actions are not tolerated: +- Harassment or discrimination based on gender, race, sexual orientation, + religion, or other personal characteristics. +- Personal attacks, threats, or intimidation. +- Inappropriate or offensive language. + +## Reporting Issues + +If you experience or witness unacceptable behavior, please report it to the +repository maintainers at [email@example.com]. All reports will be handled +confidentially. + +## Consequences + +Participants who violate this Code of Conduct may be removed from the project +and banned from future contributions. + +## Acknowledgements + +This Code of Conduct is adapted from the +[Contributor Covenant](https://www.contributor-covenant.org/), version 2.1. diff --git a/README.md b/README.md index 7f4669d..657d1fc 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,182 @@ -# data-analysis-tools -Public data analysis tools for freva +# Freva Tool Configuration with `tool.toml` + +Freva allows users to define data analysis tools in a structured and +reproducible way using `tool.toml` configuration files. These files provide +a standardized interface for defining tool metadata, parameters, dependencies, +and execution commands. Freva parses these files to automatically create a +user interface (via command-line or web UI) for applying the tools in a +reproducible manner. + +## Define your tool via a `tool.toml` file + +The `tool.toml` file simplifies the process of: +1. **Tool Definition**: + - Specify metadata such as name, version, and description. + - Define input parameters, execution commands, and dependencies. +2. **Reproducibility**: + - Document dependencies and build processes for consistent execution. + - Capture all required information in a single, portable file. +3. **User Interface Integration**: + - Automatically generate a CLI or web interface for the tool based on + the `tool.toml` configuration. + +### Writing the `tool.toml` File + +The `tool.toml` file is structured into several sections: + +#### **1. General Information** +The `[tool]` section provides metadata about the tool. + +```toml +[tool] +name = "example-tool" # Unique name for the tool +version = "v1.0.0" # Semantic version +authors = ["Author 1 ", "Author 2 "] +summary = "A brief description of what this tool does." +title = "A catchy title for the tool (optional)" +description = """ +A detailed explanation of the tool's purpose, functionality, and usage. +""" +``` + +#### **2. Execution Settings** +The [tool.run] section defines how the tool is executed. + +```toml +[tool.run] +command = "python script.py" # Command to run the tool +dependencies = ["python=3.10", "numpy"] # Conda-Forge dependencies +``` + +If the tool requires compilation or installation steps, include a build.sh script, +and define build-time dependencies in the `[tool.build]` section: + +```toml +[tool.build] +dependencies = ["rust"] # Build-specific dependencies +``` + +#### **3. Input Parameters** + +The `[tool.input_parameters]` section specifies the parameters the tool accepts. + +Each parameter is defined with: + +- `title`: The name or description of the parameter. +- `type`: The expected data type (e.g., string, integer, float). +- `mandatory`: Whether the parameter is required. +- `default`: The default value (if optional). +- `help`: A detailed explanation of the parameter's purpose. + + +```toml +[tool.input_parameters.parameter_1] +title = "Input File" +type = "string" +mandatory = true +help = "The path to the input file for the analysis." + +[tool.input_parameters.parameter_2] +title = "Verbose Mode" +type = "bool" +default = false +help = "Enable verbose output during execution." +``` + +#### **4. Advanced Features** + +Freva supports advanced parameter types, such as databrowser integration or ranges, for more complex use cases. +Databrowser Parameters + +These allow integration with a databrowser search interface: + +##### Databrowser Parameters + +These allow integration with a databrowser search interface: + +```toml +[tool.input_parameters.parameter_db] +title = "Search Parameter" +type = "databrowser" +search_key = "variable" +default = "tas" +help = """ +Integrates with the databrowser to search for data based on the specified key. +""" +``` + +##### Range Parameters + +Define ranges for numerical inputs: + +```toml +[tool.input_parameters.parameter_range] +title = "Range Example" +type = "range" +default = [0, 10, 1] +help = "Specify a numerical range in the format [start, end, increment]." +``` + +## Tool Execution Workflow + +1. Define the tool.toml File: + - Create the tool.toml file with the required sections. +1. Parse with Freva: + - Freva parses the tool.toml file to generate a CLI or web interface. +1. Run the Tool: + - Users can apply the tool through the Freva interface, providing input parameters interactively or via scripts. +1. Reproducibility: + - Freva ensures all executions are logged with version and parameter details for reproducibility. + + +## How to Contribute + +We encourage users to contribute their tools to this repository. Follow these +steps to add your tool: + +1. Navigate to the repository page and click the **Fork** button. +1. Clone your fork to your local machine: + ```console + git clone https://github.com/your-username/freva-tools.git + cd freva-tools + ``` +1. Create a new branch for your tool: + ```console + git checkout -b add-your-tool-name + ``` +1. Create a new folder in the `tools/` directory with a descriptive name for your tool: + ```console + mkdir tools/your-tool-name + ``` +1. Add your tool files to this folder: + - `tool.toml`: Defines your tool's metadata, parameters, and execution logic. + - `build.sh`: (if applicable): Handles build or installation steps. + - Source code files (e.g., Python scripts, shell scripts or other source). + +1. Include a `LICENSE` file to specify how others can use your tool. For scientific tools, + consider using a license that encourages proper attribution or citation (e.g., BSD 3-Clause License). + +1. Add and commit your changes: + ```console + git add tools/your-tool-name + git commit -m "Add your-tool-name" + ``` + +1. Push your branch to your fork: + ```console + git push origin add-your-tool-name + ``` +1. Navigate to the original repository and open a pull request. + + +## Best Practices + +- Semantic Versioning: Use clear versioning for tools (e.g., v1.0.0) to track changes. +- Dependencies: List all required dependencies explicitly in the tool.run.dependencies or tool.build.dependencies sections. +- Parameters: Provide descriptive help messages for all parameters to guide users. +- Reproducibility: Document any additional setup steps (e.g., build.sh) and ensure they are included in the tool's environment. + +## Additional Resources + +- [TOML Syntax Documentation](https://toml.io) +- [Conda forge](https://conda-forge.org/) diff --git a/create_environment.py b/create_environment.py new file mode 100644 index 0000000..b1b1f2f --- /dev/null +++ b/create_environment.py @@ -0,0 +1,594 @@ +#!/usr/bin/env python3 +import argparse +import json +import logging +import os +import platform +import re +import shutil +import subprocess +import sys +import tarfile +import urllib.request +from pathlib import Path +from tempfile import TemporaryDirectory + + +def pip_install(package): + """ + Install a Python package using pip. + + Parameters + ---------- + package : str + The name of the Python package to install. + """ + + subprocess.check_call( + [os.path.join(sys.exec_prefix, "bin", "python"), "-m", "ensurepip"] + ) + subprocess.check_call( + [ + os.path.join(sys.exec_prefix, "bin", "python"), + "-m", + "pip", + "install", + package, + ] + ) + + +try: + import yaml +except ImportError: + pip_install("pyyaml") + import yaml + +try: + import tomllib as toml +except ImportError: + try: + import tomli as toml + except ImportError: + pip_install("tomli") + import tomli as toml + + +try: + from packaging.specifiers import SpecifierSet + from packaging.version import Version +except ImportError: + pip_install("packaging") + from packaging.specifiers import SpecifierSet + from packaging.version import Version + +logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(message)s", level=logging.ERROR +) + +logger = logging.getLogger("create-conda-env") + + +def get_download_url(): + """ + Determine the appropriate Micromamba download URL based on the current system's + architecture and operating system. + + Returns + ------- + str: + The Micromamba download URL. + """ + system = platform.system().lower() + arch = platform.machine().lower() + + logger.debug("Checking arch and getting mamba url") + + if system == "linux": + if arch == "x86_64": + return "https://micro.mamba.pm/api/micromamba/linux-64/latest" + if arch == "aarch64": + return "https://micro.mamba.pm/api/micromamba/linux-aarch64/latest" + if arch == "ppc64le": + return "https://micro.mamba.pm/api/micromamba/linux-ppc64le/latest" + elif system == "darwin": + if arch == "x86_64": + return "https://micro.mamba.pm/api/micromamba/osx-64/latest" + if arch == "arm64": + return "https://micro.mamba.pm/api/micromamba/osx-arm64/latest" + + raise ValueError(f"Unsupported system or architecture: {system}-{arch}") + + +def download_with_progress(url, output_path): + """ + Download a file from a URL with a visual progress bar. + + Parameters + ---------- + url (str): + The URL to download from. + output_path (str): + The path where the downloaded file will be saved. + """ + logger.debug("Downloading micromamba") + with urllib.request.urlopen(url) as response: + total_size = int(response.getheader("Content-Length", 0)) + downloaded_size = 0 + chunk_size = 8192 + + with open(output_path, "wb") as file: + while chunk := response.read(chunk_size): + file.write(chunk) + downloaded_size += len(chunk) + done = int(50 * downloaded_size / total_size) + sys.stdout.write( + f"\rDownloading: [{'=' * done}{' ' * (50 - done)}] " + f"{downloaded_size // 1024} KB / {total_size // 1024} KB" + ) + sys.stdout.flush() + print("\nDownload complete.") + + +def extract_micromamba(tar_path, extract_dir): + """ + Extract the Micromamba binary from the tar file. + + Parameters + ----------- + tar_path (str): + The path to the tar file. + extract_dir (str): + The directory to extract to. + """ + logger.debug("Extracting micromamba") + with tarfile.open(tar_path, "r") as tar: + for member in tar.getmembers(): + if "bin/micromamba" in member.name: + print(f"Extracting {member.name} to {extract_dir}...") + tar.extract(member, path=extract_dir) + break + logger.debug("Extraction complete.") + + +def parse_args(argv=None): + """Parse the command line arguments.""" + parser = argparse.ArgumentParser("create-conda-env") + parser.add_argument( + "input_dir", + default=".", + help="The path to the tool definition.", + type=Path, + ) + parser.add_argument( + "-d", + "--dev", + action="store_true", + default=False, + help="Use development mode for any installation.", + ) + parser.add_argument( + "-f", + "--force", + action="store_true", + default=False, + help="Force recreation of the environment.", + ) + parser.add_argument( + "-p", + "--prefix", + help=("The install prefix where the environment should" " be installed"), + type=Path, + default=os.getenv("INSTALL_PREFIX", "~/.tools/"), + ) + parser.add_argument("-v", "--verbose", action="count", default=0) + args = parser.parse_args(argv) + logger.setLevel(max(logging.DEBUG, logger.level - (10 * args.verbose))) + return args + + +def create_environment(mamba_dir, tool_dir, env_dir, tool_config): + """Create the the tool conda environments. + + Parameters + ---------- + mamba_dir : Path + Path to the Micromamba binary directory. + tool_dir : Path + Path to the tool definition directory. + env_dir : Path + Path where the Conda environment will be created. + tool_config : dict + The parsed configuration of the tool from the TOML file. + """ + + requ_file = tool_dir.expanduser().absolute() / "requirements.txt" + pip_cmd = [] + env_file = tool_dir / "environment.yml" + if env_dir.is_dir(): + shutil.rmtree(env_dir) + if requ_file.is_file(): + pip_cmd = ["-r", str(requ_file)] + for file in ("setup.py", "setup.cfg", "pyproject.toml"): + if (tool_dir / file).is_file(): + pip_cmd = [str(tool_dir)] + break + logger.debug( + "Creating environment for %s", + tool_config["tool"]["run"].get("dependencies", []), + ) + deps = tool_config["tool"]["run"].get("dependencies", []) + [ + "jq", + "mamba", + "websockets", + ] + if not env_file.is_file(): + env = os.environ.copy() + subprocess.check_call( + [ + str(mamba_dir / "bin" / "micromamba"), + "create", + "-c", + "conda-forge", + "-p", + str(env_dir), + "-y", + "--strict-channel-priority", + ] + + deps, + ) + env["PATH"] = str(env_dir / "bin") + os.pathsep + env["PATH"] + create_environment_file(env_dir, env, env_file) + else: + logger.debug("Recrating principal environment") + subprocess.check_call( + [ + str(mamba_dir / "bin" / "micromamba"), + "env", + "create", + "-y", + "-p", + str(env_dir), + "--file", + str(env_file), + ], + ) + if pip_cmd: + logger.debug("Installing additional packages via pip") + subprocess.check_call( + [ + str(env_dir / "bin" / "python3"), + "-m", + "pip", + "install", + ] + + pip_cmd + ) + + +def set_version(conda_dir, version, new=True): + """Set the version in the version file. + + Parameters + ---------- + conda_dir: Path + The parent path of all conda environments for this tool + version: str + The of the tool + new: + Wheather or not an entire conda environment was created. + """ + version_file = conda_dir.parent / ".versions.json" + if version_file.is_file(): + content = json.loads(version_file.read_text()) + else: + version_file.parent.mkdir(exist_ok=True, parents=True) + new = True + content = {"latest": str(conda_dir.absolute())} + if new: + content[version] = str(conda_dir.absolute()) + content["latest"] = content[version] + else: + content[version] = content["latest"] + version_file.write_text(json.dumps(content, indent=3)) + + +def copy_all(input_path, target_path): + """ + Recursively copy all files and subdirectories from input_dir/* to target_dir/. + + Parameters + ---------- + input_dir (str): + Path to the source directory. + target_dir (str): + Path to the destination directory. + """ + + if not input_path.exists(): + raise FileNotFoundError(f"Input directory '{input_path}' does not exist.") + shutil.rmtree(target_path) + # Ensure the target directory exists + target_path.mkdir(parents=True, exist_ok=True) + + # Iterate over all files and directories in input_path + for item in input_path.rglob("*"): + target_item = target_path / item.relative_to(input_path) + + if item.is_dir(): + # Creae the corresponding directory in the target path + target_item.mkdir(parents=True, exist_ok=True) + else: + # Copy the file to the corresponding target path + shutil.copy(item, target_item) + logger.debug("Copied: %s -> %s", item, target_item) + + +def create_environment_file(path, env, env_file): + """Create an environment file of a conda environment.""" + out = subprocess.check_output( + ["mamba", "env", "export", "-p", str(path)], + env=env, + ) + conda_env = yaml.safe_load(out.decode()) + with (env_file).open("w", encoding="utf-8") as stream: + stream.write( + yaml.safe_dump( + { + "dependencies": conda_env["dependencies"], + "channels": ["conda-forge"], + } + ) + ) + + +def build(env_dir, build_dir, env_file, build_config): + """Build the tool if it has to be built.""" + conda_dir = json.loads((env_dir / ".versions.json").read_text())["latest"] + build_script = build_dir.expanduser().absolute() / "build.sh" + env = os.environ.copy() + if not build_config and not build_script.is_file(): + return + with TemporaryDirectory() as temp_dir: + temp_path = str(Path(temp_dir) / "bin") + env["PATH"] = ( + temp_path + + os.pathsep + + str(conda_dir) + + os.pathsep + + os.getenv("PATH") + ) + + deps = build_config.get("dependencies", []) + if deps: + if not env_file.is_file(): + subprocess.check_call( + [ + "mamba", + "create", + "-c", + "conda-forge", + "-p", + temp_dir, + "-y", + "--strict-channel-priority", + ] + + deps, + env=env, + ) + create_environment_file(temp_dir, env, env_file) + else: + logger.debug("Recrating build environment") + subprocess.check_call( + [ + "mamba", + "env", + "create", + "-p", + temp_dir, + "--file", + str(env_file), + ], + env=env, + ) + + if build_script.is_file(): + build_script.chmod(0o755) + subprocess.check_call( + [str(build_script)], env=env, cwd=str(build_script.parent) + ) + + +def load_config(input_dir): + """ + Load the tool configuration from a TOML or pyproject.toml file. + + Parameters + ---------- + input_dir : Path + Path to the directory containing the configuration file. + + Returns + ------- + dict + The parsed configuration data. + + Raises + ------ + ValueError + f no valid configuration file is found in the directory. + """ + if input_dir.is_dir(): + for file in ("tool.toml", "pyproject.toml"): + if (input_dir / file).is_file(): + return toml.loads((input_dir / file).read_text()) + elif input_dir.is_file(): + return toml.loads((input_dir).read_text()) + + raise ValueError( + "Your tool must be defined in either a tool.toml or pyproject.toml file" + ) + + +def parse_dependency(dependency: str): + """ + Parse a dependency string into a package name and version constraint. + + Parameters + ---------- + dependency : str + The dependency string (e.g., "foo>3"). + + Returns + ------- + tuple + A tuple of (package_name, version_constraint). + If no version constraint is provided, returns (package_name, ""). + """ + match = re.match(r"^([a-zA-Z0-9_\-]+)([<>=!~^].*)?$", dependency) + if not match: + raise ValueError(f"Invalid dependency format: {dependency}") + package_name, version_constraint = match.groups() + return package_name, version_constraint or "" + + +def check_for_environment_creation(source_dir, env_dir, dependencies): + """Check the dependencies and decide about a (re)creation of the environment. + + Parameters + ---------- + source_dir: Path + The source-code directory + env_dir: Path + The conda env of all versions of the tool + dependencies: List[str] + defined dependencies of the tool + + Returns + ------- + bool: whether or not a complete new environment needs to be created. + """ + + env_file = source_dir / "environment.yml" + for file in (env_file, env_dir, env_dir / ".versions.json"): + if not file.exists(): + return True + deps_lock = {} + + try: + deps_lock = yaml.safe_load(env_file.read_text()) + except Exception: + logger.warning("Could not read environment.yml file") + env_file.unlink() + return True + dependency_specs = {} + for dep in deps_lock["dependencies"]: + dep_vers = dep.split("=") + dependency_specs[dep_vers[0]] = dep_vers[1:] + try: + versions = json.loads((env_dir / ".versions.json").read_text()) + if not (Path(versions["latest"]) / "bin").exists(): + recreate = True + else: + recreate = False + except Exception: + recreate = True + for p in dependencies: + package_name, constraint = parse_dependency(p) + if package_name not in dependency_specs: + # We do have a requested package that is not installed yet. + deps_lock["dependencies"].append(p) + recreate = True + else: + + installed_version = Version(dependency_specs[package_name][0]) + if constraint and installed_version not in SpecifierSet(constraint): + recreate = True + # We need another version + deps_lock["dependencies"] = [ + d + for d in deps_lock["dependencies"] + if not d.startswith(package_name) + ] + deps_lock["dependencies"].append(p) + if recreate: + deps_lock["dependencies"].sort() + env_file.write_text(yaml.safe_dump(deps_lock)) + return recreate + + +def main(input_dir, prefix_dir, force=False): + """ + Main function to create the Conda environment for a tool. + + Parameters + ---------- + input_dir : Path + Path to the tool definition directory. + prefix_dir : Path + Installation prefix for the Conda environment. + force : bool, optional + Whether to force recreation of the environment, by default False. + """ + + mamba_url = get_download_url() + input_dir = input_dir.expanduser().absolute() + try: + config = load_config(input_dir) + except toml.TOMLDecodeError as error: + raise ValueError(f"Invalid toml file: {error}") + if input_dir.is_file(): + # We probaply got the path to the tool definition. + input_dir = input_dir.parent + version = config["tool"].get( + "version", config.get("project", {}).get("version") + ) + try: + version = str(Version(version)) + except Exception: + raise ValueError("You need to define a valid version.") from None + env_dir = ( + prefix_dir.expanduser().absolute() + / config["tool"]["name"] + / version.lower().strip("v") + ) + if force is True or check_for_environment_creation( + input_dir, env_dir.parent, config["tool"]["run"].get("dependencies", []) + ): + with TemporaryDirectory() as temp_dir: + tar_path = Path(temp_dir) / "micromamba.tar.bz2" + download_with_progress(mamba_url, tar_path) + extract_micromamba(tar_path, temp_dir) + micromamba_path = Path(temp_dir) / "bin" / "micromamba" + if not micromamba_path.is_file: + raise ValueError( + "Micromamba binary was not found after extraction." + ) + create_environment(Path(temp_dir), input_dir, env_dir, config) + set_version(env_dir, version, new=True) + else: + set_version(env_dir, version, new=False) + share_dir = env_dir / "share" / "tool" / config["tool"]["name"] + share_dir.mkdir(exist_ok=True, parents=True) + build_env_file = input_dir / "build-environment.yml" + copy_all(input_dir, share_dir) + try: + build( + env_dir.parent, + share_dir, + build_env_file, + config["tool"].get("build", {}), + ) + except Exception as error: + logger.error(error) + shutil.rmtree(env_dir) + raise ValueError("Failed to create environment.") + print("The tool was successfully deployed in:", share_dir) + + +if __name__ == "__main__": + app = parse_args() + try: + main(app.input_dir, app.prefix, force=app.force) + except ValueError as error: + raise SystemExit(error) from None diff --git a/examples/fix-me/README.md b/examples/fix-me/README.md new file mode 100644 index 0000000..17a6133 --- /dev/null +++ b/examples/fix-me/README.md @@ -0,0 +1,4 @@ +# Fix me +The folder contains various definition of tools that wouldn't work. + +Can you spot the errors and correct them? diff --git a/examples/fix-me/tool-1.toml b/examples/fix-me/tool-1.toml new file mode 100644 index 0000000..87e4543 --- /dev/null +++ b/examples/fix-me/tool-1.toml @@ -0,0 +1,42 @@ +# This is a template configuration file for your data analysis tool. +# Use this file as a starting point by copying and editing it to fit your +# requirements. The configuration file follows TOML syntax. For more details, +# visit: https://toml.io. +# +# IMPORTANT: Remove all comments (lines starting with '#') before using this +# configuration. + +[tool] +# General information about your tool. +authors = [ + "Author 1 ", + "Author 2 " +] # List of authors and their contact information. +version = "v0.0.1" # Semantic version of the tool. +summary = "A brief description of what this tool does." # Short overview. +title = "A catchy title for the tool (optional)" # Optional title for flair. +description = """ +A detailed description of the tool's purpose and functionality. This section +supports multi-line text for extensive details. +""" # Extended multi-line description. + +[tool.run] +# The specified command will receive a single JSON input containing all input +# parameters. Ensure your script processes this JSON file. Refer to the +# example script for guidance. +dependencies = ["python=3.13", "ffmpeg"] # Conda-forge packages needed. + +[tool.build] +# Configuration for building the tool. +dependencies = ["python=3.13", "mamba"] # Optional build-specific deps. +# NOTE: If a 'build.sh' script is provided, it will be executed during build. + +[tool.input_parameters.parameter] +# Parameter 1: Example of a string parameter with a default value. +# NOTE: Setting a default value and marking a parameter as mandatory are +# typically mutually exclusive. If a parameter has a default value, users +# are not required to provide it explicitly. +type = "string" # Data type of the parameter. +mandatory = false # Indicates whether this parameter is required. +default = "default_value" # Default value for the parameter. +help = "A help message for parameter-1, which is a string parameter." diff --git a/examples/fix-me/tool-2.toml b/examples/fix-me/tool-2.toml new file mode 100644 index 0000000..00d98d4 --- /dev/null +++ b/examples/fix-me/tool-2.toml @@ -0,0 +1,35 @@ +# This is a template configuration file for your data analysis tool. +# Use this file as a starting point by copying and editing it to fit your +# requirements. The configuration file follows TOML syntax. For more details, +# visit: https://toml.io. +# +# IMPORTANT: Remove all comments (lines starting with '#') before using this +# configuration. + +[tool] +# General information about your tool. +authors = [ + "Author 1 ", + "Author 2 " +] # List of authors and their contact information. +version = "two" # Semantic version of the tool. +summary = "A brief description of what this tool does." # Short overview. +title = "A catchy title for the tool (optional)" # Optional title for flair. +description = """ +A detailed description of the tool's purpose and functionality. This section +supports multi-line text for extensive details. +""" # Extended multi-line description. + +[tool.run] +# Defines how your tool is executed. This section specifies the command to +# run, such as a script, compiled binary, or Jupyter notebook. +command = "echo hello world" # Command to execute the tool. +# The specified command will receive a single JSON input containing all input +# parameters. Ensure your script processes this JSON file. Refer to the +# example script for guidance. +dependencies = ["python=3.13", "ffmpeg"] # Conda-forge packages needed. + +[tool.build] +# Configuration for building the tool. +dependencies = ["python=3.13", "mamba"] # Optional build-specific deps. +# NOTE: If a 'build.sh' script is provided, it will be executed during build. diff --git a/examples/fix-me/tool-3.toml b/examples/fix-me/tool-3.toml new file mode 100644 index 0000000..b68ef96 --- /dev/null +++ b/examples/fix-me/tool-3.toml @@ -0,0 +1,35 @@ +# This is a template configuration file for your data analysis tool. +# Use this file as a starting point by copying and editing it to fit your +# requirements. The configuration file follows TOML syntax. For more details, +# visit: https://toml.io. +# +# IMPORTANT: Remove all comments (lines starting with '#') before using this +# configuration. + +[tool] +# General information about your tool. +authors = [ + "Author 1 ", + "Author 2 " +] # List of authors and their contact information. +version = "v0.0.1" # Semantic version of the tool. +summary = "A brief description of what this tool does." # Short overview. +title = "A catchy title for the tool (optional)" # Optional title for flair. +description = """ +A detailed description of the tool's purpose and functionality. This section +supports multi-line text for extensive details. +""" # Extended multi-line description. + +[tool.run] +# Defines how your tool is executed. This section specifies the command to +# run, such as a script, compiled binary, or Jupyter notebook. +command = echo hello world # Command to execute the tool. +# The specified command will receive a single JSON input containing all input +# parameters. Ensure your script processes this JSON file. Refer to the +# example script for guidance. +dependencies = ["python=3.13", "ffmpeg"] # Conda-forge packages needed. + +[tool.build] +# Configuration for building the tool. +dependencies = ["python=3.13", "mamba"] # Optional build-specific deps. +# NOTE: If a 'build.sh' script is provided, it will be executed during build. diff --git a/examples/fix-me/tool-4.toml b/examples/fix-me/tool-4.toml new file mode 100644 index 0000000..4fe4d1b --- /dev/null +++ b/examples/fix-me/tool-4.toml @@ -0,0 +1,91 @@ +# This is a template configuration file for your data analysis tool. +# Use this file as a starting point by copying and editing it to fit your +# requirements. The configuration file follows TOML syntax. For more details, +# visit: https://toml.io. +# +# IMPORTANT: Remove all comments (lines starting with '#') before using this +# configuration. + +[tools] +# General information about your tool. +name = "example-shell-script" # The name of your tool. +authors = [ + "Author 1 ", + "Author 2 " +] # List of authors and their contact information. +version = "v0.0.1" # Semantic version of the tool. +summary = "A brief description of what this tool does." # Short overview. +title = "A catchy title for the tool (optional)" # Optional title for flair. +description = """ +A detailed description of the tool's purpose and functionality. This section +supports multi-line text for extensive details. +""" # Extended multi-line description. + +[tools.run] +# Defines how your tool is executed. This section specifies the command to +# run, such as a script, compiled binary, or Jupyter notebook. +command = "/bin/bash example.sh" # Command to execute the tool. +# The specified command will receive a single JSON input containing all input +# parameters. Ensure your script processes this JSON file. Refer to the +# example script for guidance. +dependencies = ["python=3.13", "ffmpeg"] # Conda-forge packages needed. + +[tools.build] +# Configuration for building the tool. +dependencies = ["python=3.13", "mamba"] # Optional build-specific deps. +# NOTE: If a 'build.sh' script is provided, it will be executed during build. + +[tools.input_parameters.parameter_1] +# Parameter 1: Example of a string parameter with a default value. +# NOTE: Setting a default value and marking a parameter as mandatory are +# typically mutually exclusive. If a parameter has a default value, users +# are not required to provide it explicitly. +title = "Parameter 1" +type = "string" # Data type of the parameter. +mandatory = false # Indicates whether this parameter is required. +default = "default_value" # Default value for the parameter. +help = "A help message for parameter-1, which is a string parameter." + +[tools.input_parameters.parameter_2] +# Parameter 2: Example of a mandatory integer parameter. +# NOTE: Mandatory parameters should not have default values. +title = "Parameter 2" +type = "integer" # Data type: integer. +mandatory = true # Indicates this parameter is required. +help = "A help message for parameter-2, which is an integer." + +[tools.input_parameters.parameter_3] +# Parameter 3: Example of a float parameter with a default value. +title = "Parameter 3" +type = "float" # Data type: floating-point number. +mandatory = false # This parameter is optional since it has a default value. +default = 2.0 # Default value. +help = "A help message for parameter-3, which is a float." + +[tools.input_parameters.parameter_4] +# Parameter 4: Example of a boolean parameter with a default value. +title = "Parameter 4" +type = "bool" # Data type: boolean. +default = true # Default value is true. +help = "A help message for parameter-4, which is a boolean flag." + +[tools.input_parameters.parameter_5] +# Parameter 5: Example of a datetime parameter. +title = "Parameter 5" +type = "datetime" # Data type: datetime in ISO 8601 format. +default = "2000-01-01T00:00:00Z" # Default value. +help = "A help message for parameter-5, which is a datetime parameter." + +[tools.input_parameters.parameter_6] +# Parameter 6: Example of a databrowser parameter for search functionality. +title = "Parameter for Variable" +type = "databrowser" # Special parameter type for databrowser integration. +help = """ +Allows integration with a databrowser search. Specify a `search_key` and +optional constraints for filtering search results. +""" +search_key = "variable" # Key used in the databrowser search. +constraint.time_frequency = "1hr" # Example constraint: time frequency. +constraint.product = "cmip" # Example constraint: product type. +default = "tas" # Default value for the search key. +search_result = "input_files" # Resolves this search to actual data. diff --git a/examples/gnuR-script/README.md b/examples/gnuR-script/README.md new file mode 100644 index 0000000..0e43d09 --- /dev/null +++ b/examples/gnuR-script/README.md @@ -0,0 +1,56 @@ +# Example R Script Tool + +This example demonstrates how to define a tool in `tool.toml` to set up a +**GNU R** environment using Conda-Forge. The tool executes an R script +(`example.R`) which takes a single command-line argument: a `JSON` file +containing all the parsed parameters defined in the `tool.toml` file. + +## Purpose + +This configuration allows you to: +1. Define a reproducible Conda-Forge environment specifically tailored for + running R scripts. +2. Specify and document input parameters directly in the `tool.toml` file. +3. Provide flexibility by parameterizing the R script through a `JSON` file + generated from the tool's input configuration. + +## How It Works + +1. **Environment Setup**: + - The `tool.toml` file specifies all dependencies required to run the + R script. For this example, the `r-base` package is included to provide + the R environment. + +2. **Parameter Input**: + - Input parameters, along with their metadata (e.g., type, default value, + and description), are defined in the `tool.input_parameters` section of + the `tool.toml` file. + +3. **Script Execution**: + - When the tool runs, it generates a `JSON` file containing the parsed + parameters from the `tool.toml` configuration. + - This `JSON` file is passed as an argument to the R script (`example.R`), + allowing the script to dynamically handle user-defined parameters. + +## Example Use Case + +- **Data Analysis**: + Run an R script that processes datasets dynamically, based on user-specified + parameters such as file paths, thresholds, or analysis methods. + +- **Visualization**: + Generate plots or dashboards in R by passing custom parameters for data + sources, filters, and styles through the `tool.toml` configuration. + +## Notes + +- Ensure that all dependencies (e.g., `r-base`, `jsonlite`) are specified in + the `tool.run.dependencies` section of `tool.toml`. +- For more details on TOML syntax, visit [https://toml.io](https://toml.io). + +## Additional Resources + +- [GNU R on Conda-Forge](https://conda-forge.org/feedstocks/r-base) +- [JSON Parsing in R (jsonlite)](https://cran.r-project.org/web/packages/jsonlite/index.html) +- [TOML Syntax Documentation](https://toml.io) + diff --git a/examples/gnuR-script/config.json b/examples/gnuR-script/config.json new file mode 100644 index 0000000..33a6009 --- /dev/null +++ b/examples/gnuR-script/config.json @@ -0,0 +1,8 @@ +{ + "parameter_1": "example_value_1", + "parameter_2": 42, + "parameter_3": "/path/to/resource", + "parameter_4": true, + "parameter_5": "2024-11-19T10:00:00Z" +} + diff --git a/examples/gnuR-script/environment.yml b/examples/gnuR-script/environment.yml new file mode 100644 index 0000000..683e656 --- /dev/null +++ b/examples/gnuR-script/environment.yml @@ -0,0 +1,153 @@ +channels: +- conda-forge +dependencies: +- _libgcc_mutex=0.1=conda_forge +- _openmp_mutex=4.5=2_gnu +- _r-mutex=1.0.1=anacondar_1 +- aom=3.9.1=hac33072_0 +- binutils_impl_linux-64=2.43=h4bf12b8_2 +- bwidget=1.9.14=ha770c72_1 +- bzip2=1.0.8=h4bc722e_7 +- c-ares=1.34.3=hb9d3cd8_1 +- ca-certificates=2024.8.30=hbcca054_0 +- cairo=1.18.0=hebfffa5_3 +- cpp-expected=1.1.0=hf52228f_0 +- curl=8.10.1=hbbe4b11_0 +- dav1d=1.2.1=hd590300_0 +- ffmpeg=7.1.0=gpl_heed6883_705 +- fmt=11.0.2=h434a139_0 +- font-ttf-dejavu-sans-mono=2.37=hab24e00_0 +- font-ttf-inconsolata=3.000=h77eed37_0 +- font-ttf-source-code-pro=2.038=h77eed37_0 +- font-ttf-ubuntu=0.83=h77eed37_3 +- fontconfig=2.15.0=h7e30c49_1 +- fonts-conda-ecosystem=1=0 +- fonts-conda-forge=1=0 +- freetype=2.12.1=h267a509_2 +- fribidi=1.0.10=h36c2ea0_0 +- gcc_impl_linux-64=14.2.0=h6b349bd_1 +- gdk-pixbuf=2.42.12=hb9ae30d_0 +- gfortran_impl_linux-64=14.2.0=hc73f493_1 +- gmp=6.3.0=hac33072_2 +- graphite2=1.3.13=h59595ed_1003 +- gsl=2.7=he838d99_0 +- gxx_impl_linux-64=14.2.0=h2c03514_1 +- harfbuzz=9.0.0=hda332d3_1 +- icu=75.1=he02047a_0 +- jq=1.7.1=hd590300_0 +- kernel-headers_linux-64=3.10.0=he073ed8_18 +- keyutils=1.6.1=h166bdaf_0 +- krb5=1.21.3=h659f571_0 +- lame=3.100=h166bdaf_1003 +- ld_impl_linux-64=2.43=h712a8e2_2 +- lerc=4.0.0=h27087fc_0 +- libabseil=20240722.0=cxx17_h5888daf_1 +- libarchive=3.7.4=hfca40fe_0 +- libass=0.17.3=h1dc1e6a_0 +- libblas=3.9.0=25_linux64_openblas +- libcblas=3.9.0=25_linux64_openblas +- libcurl=8.10.1=hbbe4b11_0 +- libdeflate=1.22=hb9d3cd8_0 +- libdrm=2.4.123=hb9d3cd8_0 +- libedit=3.1.20191231=he28a2e2_2 +- libegl=1.7.0=ha4b6fd6_2 +- libev=4.33=hd590300_2 +- libexpat=2.6.4=h5888daf_0 +- libffi=3.4.2=h7f98852_5 +- libgcc=14.2.0=h77fa898_1 +- libgcc-devel_linux-64=14.2.0=h41c2201_101 +- libgcc-ng=14.2.0=h69a702a_1 +- libgfortran=14.2.0=h69a702a_1 +- libgfortran-ng=14.2.0=h69a702a_1 +- libgfortran5=14.2.0=hd5240d6_1 +- libgl=1.7.0=ha4b6fd6_2 +- libglib=2.82.2=h2ff4ddf_0 +- libglvnd=1.7.0=ha4b6fd6_2 +- libglx=1.7.0=ha4b6fd6_2 +- libgomp=14.2.0=h77fa898_1 +- libhwloc=2.11.2=default_h0d58e46_1001 +- libiconv=1.17=hd590300_2 +- libjpeg-turbo=3.0.0=hd590300_1 +- liblapack=3.9.0=25_linux64_openblas +- libmamba=2.0.4=hf72d635_0 +- libnghttp2=1.64.0=h161d5f1_0 +- libopenblas=0.3.28=pthreads_h94d23a6_1 +- libopenvino=2024.4.0=hac27bb2_2 +- libopenvino-auto-batch-plugin=2024.4.0=h4d9b6c2_2 +- libopenvino-auto-plugin=2024.4.0=h4d9b6c2_2 +- libopenvino-hetero-plugin=2024.4.0=h3f63f65_2 +- libopenvino-intel-cpu-plugin=2024.4.0=hac27bb2_2 +- libopenvino-intel-gpu-plugin=2024.4.0=hac27bb2_2 +- libopenvino-intel-npu-plugin=2024.4.0=hac27bb2_2 +- libopenvino-ir-frontend=2024.4.0=h3f63f65_2 +- libopenvino-onnx-frontend=2024.4.0=h5c8f2c3_2 +- libopenvino-paddle-frontend=2024.4.0=h5c8f2c3_2 +- libopenvino-pytorch-frontend=2024.4.0=h5888daf_2 +- libopenvino-tensorflow-frontend=2024.4.0=h6481b9d_2 +- libopenvino-tensorflow-lite-frontend=2024.4.0=h5888daf_2 +- libopus=1.3.1=h7f98852_1 +- libpciaccess=0.18=hd590300_0 +- libpng=1.6.44=hadc24fc_0 +- libprotobuf=5.28.2=h5b01275_0 +- librsvg=2.58.4=hc0ffecb_0 +- libsanitizer=14.2.0=h2a3dede_1 +- libsolv=0.7.30=h3509ff9_0 +- libssh2=1.11.1=hf672d98_0 +- libstdcxx=14.2.0=hc0a3c3a_1 +- libstdcxx-devel_linux-64=14.2.0=h41c2201_101 +- libstdcxx-ng=14.2.0=h4852527_1 +- libtiff=4.7.0=he137b08_1 +- libuuid=2.38.1=h0b41bf4_0 +- libva=2.22.0=h8a09558_1 +- libvpx=1.14.1=hac33072_0 +- libwebp-base=1.4.0=hd590300_0 +- libxcb=1.17.0=h8a09558_0 +- libxml2=2.13.5=hb346dea_0 +- libzlib=1.3.1=hb9d3cd8_2 +- lz4-c=1.9.4=hcb278e6_0 +- lzo=2.10=hd590300_1001 +- make=4.4.1=hb9d3cd8_2 +- mamba=2.0.4=hfdd0a45_0 +- ncurses=6.5=he02047a_1 +- nlohmann_json=3.11.3=he02047a_1 +- ocl-icd=2.3.2=hd590300_1 +- oniguruma=6.9.9=hd590300_0 +- openh264=2.5.0=hf92e6e3_0 +- openssl=3.4.0=hb9d3cd8_0 +- pango=1.54.0=h4c5309f_1 +- pcre2=10.44=hba22ea6_2 +- pixman=0.43.2=h59595ed_0 +- pthread-stubs=0.4=hb9d3cd8_1002 +- pugixml=1.14=h59595ed_0 +- r-base=4.4.2=h64c9cd0_0 +- readline=8.2=h8228510_1 +- reproc=14.2.5.post0=hb9d3cd8_0 +- reproc-cpp=14.2.5.post0=h5888daf_0 +- sed=4.8=he412f7d_0 +- simdjson=3.10.1=h84d6215_0 +- snappy=1.2.1=ha2e4443_0 +- spdlog=1.14.1=hed91bc2_1 +- svt-av1=2.3.0=h5888daf_0 +- sysroot_linux-64=2.17=h4a8ded7_18 +- tbb=2022.0.0=hceb3a55_0 +- tk=8.6.13=noxft_h4845f30_101 +- tktable=2.10=h8bc8fbc_6 +- tzdata=2024b=hc8b5060_0 +- wayland=1.23.1=h3e06ad9_0 +- wayland-protocols=1.37=hd8ed1ab_0 +- x264=1!164.3095=h166bdaf_2 +- x265=3.5=h924138e_3 +- xorg-libice=1.1.1=hb9d3cd8_1 +- xorg-libsm=1.2.4=he73a12e_1 +- xorg-libx11=1.8.10=h4f16b4b_0 +- xorg-libxau=1.0.11=hb9d3cd8_1 +- xorg-libxdmcp=1.1.5=hb9d3cd8_0 +- xorg-libxext=1.3.6=hb9d3cd8_0 +- xorg-libxfixes=6.0.1=hb9d3cd8_0 +- xorg-libxrender=0.9.11=hb9d3cd8_1 +- xorg-libxt=1.3.1=hb9d3cd8_0 +- xorg-xorgproto=2024.1=hb9d3cd8_1 +- xz=5.2.6=h166bdaf_0 +- yaml-cpp=0.8.0=h59595ed_0 +- zlib=1.3.1=hb9d3cd8_2 +- zstd=1.5.6=ha6fb4c9_0 diff --git a/examples/gnuR-script/example.R b/examples/gnuR-script/example.R new file mode 100644 index 0000000..0b9dcd8 --- /dev/null +++ b/examples/gnuR-script/example.R @@ -0,0 +1,110 @@ +#!/usr/bin/env Rscript + +# Load necessary libraries +if (!requireNamespace("jsonlite", quietly = TRUE)) { + stop("Please install the 'jsonlite' package to run this script.") +} +if (!requireNamespace("futile.logger", quietly = TRUE)) { + stop("Please install the 'futile.logger' package to run this script.") +} + +library(jsonlite) +library(futile.logger) + +# Configure logging +flog.appender(appender.console(), name = "default") +flog.threshold(INFO, name = "default") + +load_config <- function(config_file) { + #' Load and parse the configuration JSON file. + #' + #' @param config_file Path to the configuration JSON file. + #' @return A list representing the parsed configuration. + #' @throws Error if the file does not exist or JSON is invalid. + + if (!file.exists(config_file)) { + flog.error("Error: File '%s' does not exist.", config_file) + stop("Configuration file not found.") + } + + tryCatch( + { + config <- fromJSON(config_file) + return(config) + }, + error = function(e) { + flog.error("Error: Invalid JSON format in '%s': %s", config_file, e$message) + stop("Failed to parse JSON configuration.") + } + ) +} + +main <- function(config_file) { + #' Main function to evaluate the configuration file and perform actions. + #' + #' @param config_file Path to the configuration JSON file. + + # Load the configuration + config <- tryCatch( + load_config(config_file), + error = function(e) { + stop("Exiting due to error: ", e$message) + } + ) + + flog.info("Evaluating configuration from '%s'...", config_file) + + # Extract specific parameters with defaults + parameter_1 <- ifelse("parameter_1" %in% names(config), config$parameter_1, "default_value") + parameter_2 <- ifelse("parameter_2" %in% names(config), config$parameter_2, NULL) + parameter_3 <- ifelse("parameter_3" %in% names(config), config$parameter_3, "/path/to/default") + parameter_4 <- ifelse("parameter_4" %in% names(config), config$parameter_4, FALSE) + parameter_5 <- ifelse("parameter_5" %in% names(config), config$parameter_5, "2000-01-01T00:00:00Z") + + # Handle mandatory parameter checks + if (is.null(parameter_2)) { + flog.error("Error: 'parameter_2' is mandatory but is not provided in the config.") + stop("Mandatory parameter missing.") + } + + # Log extracted parameters + flog.info("Extracted Parameters:") + flog.info(" Parameter 1: %s", parameter_1) + flog.info(" Parameter 2: %s", parameter_2) + flog.info(" Parameter 3: %s", parameter_3) + flog.info(" Parameter 4: %s", parameter_4) + flog.info(" Parameter 5: %s", parameter_5) + + # Perform actions based on the parameters + flog.info("Performing actions based on the configuration...") + + # Example: Conditional logic based on a boolean parameter + if (parameter_4) { + flog.info("Boolean parameter is true, taking appropriate action.") + } + + # Example: Simulating action based on a parameter + if (!is.null(parameter_3)) { + flog.info("Simulating action with parameter 3: %s", parameter_3) + } + + flog.info("Configuration evaluation completed successfully.") +} + +# Script entry point +if (!interactive()) { + args <- commandArgs(trailingOnly = TRUE) + if (length(args) != 1) { + flog.error("Usage: Rscript %s ", commandArgs()[1]) + stop("Invalid number of arguments.") + } + + config_file_path <- args[1] + tryCatch( + main(config_file_path), + error = function(e) { + flog.error("Script terminated with error: %s", e$message) + quit(status = 1) + } + ) +} diff --git a/examples/gnuR-script/tool.toml b/examples/gnuR-script/tool.toml new file mode 100644 index 0000000..ab4e6de --- /dev/null +++ b/examples/gnuR-script/tool.toml @@ -0,0 +1,143 @@ +# This is a template configuration file for your data analysis tool. +# Use this file as a starting point by copying and editing it to fit your +# requirements. The configuration file follows TOML syntax. For more details, +# visit: https://toml.io. +# +# IMPORTANT: Remove all comments (lines starting with '#') before using this +# configuration. + +[tool] +# General information about your tool. +name = "example-r-script" # The name of your tool. +authors = [ + "Author 1 ", + "Author 2 " +] # List of authors and their contact information. +version = "v0.0.1" # Semantic version of the tool. +summary = "A brief description of what this tool does." # Short overview. +title = "A catchy title for the tool (optional)" # Optional title for flair. +description = """ +A detailed description of the tool's purpose and functionality. This section +supports multi-line text for extensive details. +""" # Extended multi-line description. + +[tool.run] +# Defines how your tool is executed. This section specifies the command to +# run, such as a script, compiled binary, or Jupyter notebook. +command = "Rscript example.R" # Command to execute the tool. +# The specified command will receive a single JSON input containing all input +# parameters. Ensure your script processes this JSON file. Refer to the +# example script for guidance. +dependencies = ["r-base", "ffmpeg"] # Conda-forge packages needed. + +[tool.input_parameters.parameter_1] +# Parameter 1: Example of a string parameter with a default value. +# NOTE: Setting a default value and marking a parameter as mandatory are +# typically mutually exclusive. If a parameter has a default value, users +# are not required to provide it explicitly. +title = "Parameter 1" +type = "string" # Data type of the parameter. +mandatory = false # Indicates whether this parameter is required. +default = "default_value" # Default value for the parameter. +help = "A help message for parameter-1, which is a string parameter." + +[tool.input_parameters.parameter_2] +# Parameter 2: Example of a mandatory integer parameter. +# NOTE: Mandatory parameters should not have default values. +title = "Parameter 2" +type = "integer" # Data type: integer. +mandatory = true # Indicates this parameter is required. +help = "A help message for parameter-2, which is an integer." + +[tool.input_parameters.parameter_3] +# Parameter 3: Example of a float parameter with a default value. +title = "Parameter 3" +type = "float" # Data type: floating-point number. +mandatory = false # This parameter is optional since it has a default value. +default = 2.0 # Default value. +help = "A help message for parameter-3, which is a float." + +[tool.input_parameters.parameter_4] +# Parameter 4: Example of a boolean parameter with a default value. +title = "Parameter 4" +type = "bool" # Data type: boolean. +default = true # Default value is true. +help = "A help message for parameter-4, which is a boolean flag." + +[tool.input_parameters.parameter_5] +# Parameter 5: Example of a datetime parameter. +title = "Parameter 5" +type = "datetime" # Data type: datetime in ISO 8601 format. +default = "2000-01-01T00:00:00Z" # Default value. +help = "A help message for parameter-5, which is a datetime parameter." + +[tool.input_parameters.parameter_6] +# Parameter 6: Example of a databrowser parameter for search functionality. +title = "Parameter for Variable" +type = "databrowser" # Special parameter type for databrowser integration. +help = """Allows integration with a databrowser search. Specify a `search_key` and +optional constraints for filtering search results. +""" +search_key = "variable" # Key used in the databrowser search. +constraint.time_frequency = "1hr" # Example constraint: time frequency. +constraint.product = "cmip" # Example constraint: product type. +default = "tas" # Default value for the search key. +search_result = "input_files" # Resolves this search to actual data. + +[tool.input_parameters.parameter_6a] +# Additional databrowser parameter for combining search queries. +title = "Parameter for Experiment" +type = "databrowser" +help = """ +Defines the "experiment" search key for querying data. Combines with other +databrowser parameters to refine search results. +""" +search_key = "experiment" +constraint.time_frequency = "1hr" +constraint.product = "cmip" +default = "amip" +search_result = "input_files" + +[tool.input_parameters.parameter_6b] +# Separate databrowser parameter for a new search result key. +title = "Another Parameter" +type = "databrowser" +help = """ +Creates a new search query and stores results under the key 'other_files'. +""" +search_key = "experiment" +constraint.time_frequency = "1hr" +constraint.product = "cmip" +default = "amip" +search_result = "other_files" + +[tool.input_parameters.parameter_6c] +# Databrowser parameter without a resolved search. +title = "Databrowser Parameter without a Search Query" +type = "databrowser" +help = """ +Passes the databrowser parameter directly without applying a search. Omit +the `search_result` key to prevent a search query. +""" +search_key = "time_frequency" +default = "1hr" + +[tool.input_parameters.parameter_7] +# Parameter 7: Example of a range parameter. +title = "Parameter 7" +type = "range" # Range type parameter. +help = """ +Specifies a range in the format [start, end, increment]. Supports shorthand +forms: [end] (equivalent to [0, end, 1]) or [start, end]. +""" +default = [0.0, 10.5, 0.1] # Default range. +mandatory = false + +[tool.input_parameters.parameter_8] +# Parameter 8: Example of a list parameter. +title = "List Parameter" +type = "integer" # List of integers. +help = """ +Accepts a list of values where each item matches the specified data type. +""" +default = [10, 1, -1] # Default list of integers. diff --git a/examples/python-lib/LICENSE b/examples/python-lib/LICENSE new file mode 100644 index 0000000..7092cf1 --- /dev/null +++ b/examples/python-lib/LICENSE @@ -0,0 +1,28 @@ +BSD 3-Clause License + +Copyright (c) 2024, Climate Informatics and Technologies (CLINT) + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/examples/python-lib/README.md b/examples/python-lib/README.md new file mode 100644 index 0000000..1176341 --- /dev/null +++ b/examples/python-lib/README.md @@ -0,0 +1,59 @@ +# Using a Tool by Installing a Python Package + +This example demonstrates how to define a tool as a Python package that can be +installed via standard Python packaging tools. The package is defined using a +`pyproject.toml`, `setup.cfg`, or `setup.py` file. The recommended approach is +to use `pyproject.toml`, which allows you to merge the `tool.toml` file into a +single configuration. + +## Purpose + +This configuration allows you to: +1. Define a Python package that can be deployed and installed using standard + Python packaging tools. +2. Merge the `tool.toml` configuration into the `pyproject.toml` file for a + streamlined setup. +3. Leverage the `tool.run.command` section to execute an entry-point script + defined in your Python package. + +## How It Works + +1. **Package Definition**: + - The tool is defined as a Python package using `pyproject.toml`, + `setup.cfg`, or `setup.py`. + - The `pyproject.toml` file is the recommended format, as it supports + modern Python packaging standards and allows merging with the + `tool.toml` file. + +2. **Command Configuration**: + - After the package is installed, the `tool.run.command` section of the + `pyproject.toml` file should reference the entry-point script defined in + the package. + +3. **Version Management**: + - When merging `tool.toml` into `pyproject.toml`, the `version` entry + should only be defined in the `[project]` section of `pyproject.toml`. + +4. **Tool Execution**: + - The installed Python package provides entry points for the tool, allowing + users to execute commands via the entry-point script. + +## Example Use Case + +- **Data Analysis Library**: + Create a Python library for analyzing datasets, where the main tool logic is + exposed through an entry-point script. + +- **Custom CLI Tool**: + Develop a command-line tool as a Python package, enabling users to install + it via `pip` and execute commands using a defined entry-point. + +## Notes + +- Ensure your `pyproject.toml` file adheres to the + [PEP 621](https://peps.python.org/pep-0621/) standard for defining project + metadata. +- Clearly document the entry-point script in the `[project.scripts]` section + of `pyproject.toml`. +- The `tool.run.command` in the merged `pyproject.toml` file should point to + the entry-point defined in the `[project.scripts]` section. diff --git a/examples/python-lib/environment.yml b/examples/python-lib/environment.yml new file mode 100644 index 0000000..c566e56 --- /dev/null +++ b/examples/python-lib/environment.yml @@ -0,0 +1,133 @@ +channels: +- conda-forge +dependencies: +- _libgcc_mutex=0.1=conda_forge +- _openmp_mutex=4.5=2_gnu +- aom=3.9.1=hac33072_0 +- bzip2=1.0.8=h4bc722e_7 +- c-ares=1.34.3=heb4867d_0 +- ca-certificates=2024.8.30=hbcca054_0 +- cairo=1.18.0=hebfffa5_3 +- cpp-expected=1.1.0=hf52228f_0 +- dav1d=1.2.1=hd590300_0 +- ffmpeg=7.1.0=gpl_hde54019_704 +- fmt=11.0.2=h434a139_0 +- font-ttf-dejavu-sans-mono=2.37=hab24e00_0 +- font-ttf-inconsolata=3.000=h77eed37_0 +- font-ttf-source-code-pro=2.038=h77eed37_0 +- font-ttf-ubuntu=0.83=h77eed37_3 +- fontconfig=2.15.0=h7e30c49_1 +- fonts-conda-ecosystem=1=0 +- fonts-conda-forge=1=0 +- freetype=2.12.1=h267a509_2 +- fribidi=1.0.10=h36c2ea0_0 +- gdk-pixbuf=2.42.12=hb9ae30d_0 +- gmp=6.3.0=hac33072_2 +- graphite2=1.3.13=h59595ed_1003 +- harfbuzz=9.0.0=hda332d3_1 +- icu=75.1=he02047a_0 +- jq=1.7.1=hd590300_0 +- keyutils=1.6.1=h166bdaf_0 +- krb5=1.21.3=h659f571_0 +- lame=3.100=h166bdaf_1003 +- ld_impl_linux-64=2.43=h712a8e2_2 +- lerc=4.0.0=h27087fc_0 +- libabseil=20240722.0=cxx17_h5888daf_1 +- libarchive=3.7.4=hfca40fe_0 +- libass=0.17.3=h1dc1e6a_0 +- libcurl=8.10.1=hbbe4b11_0 +- libdeflate=1.22=hb9d3cd8_0 +- libdrm=2.4.123=hb9d3cd8_0 +- libedit=3.1.20191231=he28a2e2_2 +- libegl=1.7.0=ha4b6fd6_2 +- libev=4.33=hd590300_2 +- libexpat=2.6.4=h5888daf_0 +- libffi=3.4.2=h7f98852_5 +- libgcc=14.2.0=h77fa898_1 +- libgcc-ng=14.2.0=h69a702a_1 +- libgl=1.7.0=ha4b6fd6_2 +- libglib=2.82.2=h2ff4ddf_0 +- libglvnd=1.7.0=ha4b6fd6_2 +- libglx=1.7.0=ha4b6fd6_2 +- libgomp=14.2.0=h77fa898_1 +- libhwloc=2.11.2=default_h0d58e46_1001 +- libiconv=1.17=hd590300_2 +- libjpeg-turbo=3.0.0=hd590300_1 +- libmamba=2.0.3=hf72d635_0 +- libmpdec=4.0.0=h4bc722e_0 +- libnghttp2=1.64.0=h161d5f1_0 +- libopenvino=2024.4.0=hac27bb2_2 +- libopenvino-auto-batch-plugin=2024.4.0=h4d9b6c2_2 +- libopenvino-auto-plugin=2024.4.0=h4d9b6c2_2 +- libopenvino-hetero-plugin=2024.4.0=h3f63f65_2 +- libopenvino-intel-cpu-plugin=2024.4.0=hac27bb2_2 +- libopenvino-intel-gpu-plugin=2024.4.0=hac27bb2_2 +- libopenvino-intel-npu-plugin=2024.4.0=hac27bb2_2 +- libopenvino-ir-frontend=2024.4.0=h3f63f65_2 +- libopenvino-onnx-frontend=2024.4.0=h5c8f2c3_2 +- libopenvino-paddle-frontend=2024.4.0=h5c8f2c3_2 +- libopenvino-pytorch-frontend=2024.4.0=h5888daf_2 +- libopenvino-tensorflow-frontend=2024.4.0=h6481b9d_2 +- libopenvino-tensorflow-lite-frontend=2024.4.0=h5888daf_2 +- libopus=1.3.1=h7f98852_1 +- libpciaccess=0.18=hd590300_0 +- libpng=1.6.44=hadc24fc_0 +- libprotobuf=5.28.2=h5b01275_0 +- librsvg=2.58.4=hc0ffecb_0 +- libsolv=0.7.30=h3509ff9_0 +- libsqlite=3.47.0=hadc24fc_1 +- libssh2=1.11.0=h0841786_0 +- libstdcxx=14.2.0=hc0a3c3a_1 +- libstdcxx-ng=14.2.0=h4852527_1 +- libtiff=4.7.0=he137b08_1 +- libuuid=2.38.1=h0b41bf4_0 +- libva=2.22.0=h8a09558_1 +- libvpx=1.14.1=hac33072_0 +- libwebp-base=1.4.0=hd590300_0 +- libxcb=1.17.0=h8a09558_0 +- libxml2=2.13.5=hb346dea_0 +- libzlib=1.3.1=hb9d3cd8_2 +- lz4-c=1.9.4=hcb278e6_0 +- lzo=2.10=hd590300_1001 +- mamba=2.0.3=hfdd0a45_0 +- ncurses=6.5=he02047a_1 +- nlohmann_json=3.11.3=he02047a_1 +- ocl-icd=2.3.2=hd590300_1 +- oniguruma=6.9.9=hd590300_0 +- openh264=2.5.0=hf92e6e3_0 +- openssl=3.4.0=hb9d3cd8_0 +- pango=1.54.0=h4c5309f_1 +- pcre2=10.44=hba22ea6_2 +- pip=24.3.1=pyh145f28c_0 +- pixman=0.43.2=h59595ed_0 +- pthread-stubs=0.4=hb9d3cd8_1002 +- pugixml=1.14=h59595ed_0 +- python=3.13.0=h9ebbce0_100_cp313 +- python_abi=3.13=5_cp313 +- readline=8.2=h8228510_1 +- reproc=14.2.5.post0=hb9d3cd8_0 +- reproc-cpp=14.2.5.post0=h5888daf_0 +- simdjson=3.10.1=h84d6215_0 +- snappy=1.2.1=ha2e4443_0 +- spdlog=1.14.1=hed91bc2_1 +- svt-av1=2.3.0=h5888daf_0 +- tbb=2022.0.0=hceb3a55_0 +- tk=8.6.13=noxft_h4845f30_101 +- tzdata=2024b=hc8b5060_0 +- wayland=1.23.1=h3e06ad9_0 +- wayland-protocols=1.37=hd8ed1ab_0 +- x264=1!164.3095=h166bdaf_2 +- x265=3.5=h924138e_3 +- xorg-libice=1.1.1=hb9d3cd8_1 +- xorg-libsm=1.2.4=he73a12e_1 +- xorg-libx11=1.8.10=h4f16b4b_0 +- xorg-libxau=1.0.11=hb9d3cd8_1 +- xorg-libxdmcp=1.1.5=hb9d3cd8_0 +- xorg-libxext=1.3.6=hb9d3cd8_0 +- xorg-libxfixes=6.0.1=hb9d3cd8_0 +- xorg-libxrender=0.9.11=hb9d3cd8_1 +- xorg-xorgproto=2024.1=hb9d3cd8_1 +- xz=5.2.6=h166bdaf_0 +- yaml-cpp=0.8.0=h59595ed_0 +- zlib=1.3.1=hb9d3cd8_2 +- zstd=1.5.6=ha6fb4c9_0 diff --git a/examples/python-lib/pyproject.toml b/examples/python-lib/pyproject.toml new file mode 100644 index 0000000..00f0cc6 --- /dev/null +++ b/examples/python-lib/pyproject.toml @@ -0,0 +1,176 @@ +# This is a template configuration file for your data analysis tool. +# You can configure your tool within a pyproject.toml file. This +# Example file outlines how. + + +[build-system] +requires = ["flit_core >=3.2"] +build-backend = "flit_core.buildapi" + +[project] +name = "my-tool" +authors = [{name = "Author", email = "author@email.de"}] +readme = "README.md" +license = {file = "LICENSE"} +classifiers = [ + "Development Status :: 3 - Alpha", + "Environment :: Console", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: BSD License", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3", +] +version = "v0.0.1" # If the tool definition is defined in a pyproject.toml file + # you should add the version to the `project` section. +description = "A tool that can be installed with pip" +requires-python = ">=3.8" +dependencies = [ "argparse"] +[project.urls] +Documentation = "https://some-documentation.org" +Issues = "https://github.com/author/my_tool/issues" +Source = "https://github.com/author/mu_tool" +Home = "https://some-documentation.org" + +[project.scripts] +my-tool = "my_tool:cli" + + +[package-data] +my_tool = ["py.typed"] + +[tool] +# General information about your tool. +name = "example-python-lib" # The name of your tool. +authors = [ + "Author 1 ", + "Author 2 " +] # List of authors and their contact information. +summary = "A brief description of what this tool does." # Short overview. +title = "A catchy title for the tool (optional)" # Optional title for flair. +description = """ +A detailed description of the tool's purpose and functionality. This section +supports multi-line text for extensive details. +""" # Extended multi-line description. + +[tool.run] +# Defines how your tool is executed. This section specifies the command to +# run, such as a script, compiled binary, or Jupyter notebook. +command = "my-tool" # Command to execute the tool. +# The specified command will receive a single JSON input containing all input +# parameters. Ensure your script processes this JSON file. Refer to the +# example script for guidance. +dependencies = ["python=3.13", "ffmpeg"] # Conda-forge packages needed. + +[tool.input_parameters.parameter_1] +# Parameter 1: Example of a string parameter with a default value. +# NOTE: Setting a default value and marking a parameter as mandatory are +# typically mutually exclusive. If a parameter has a default value, users +# are not required to provide it explicitly. +title = "Parameter 1" +type = "string" # Data type of the parameter. +mandatory = false # Indicates whether this parameter is required. +default = "default_value" # Default value for the parameter. +help = "A help message for parameter-1, which is a string parameter." + +[tool.input_parameters.parameter_2] +# Parameter 2: Example of a mandatory integer parameter. +# NOTE: Mandatory parameters should not have default values. +title = "Parameter 2" +type = "integer" # Data type: integer. +mandatory = true # Indicates this parameter is required. +help = "A help message for parameter-2, which is an integer." + +[tool.input_parameters.parameter_3] +# Parameter 3: Example of a float parameter with a default value. +title = "Parameter 3" +type = "float" # Data type: floating-point number. +mandatory = false # This parameter is optional since it has a default value. +default = 2.0 # Default value. +help = "A help message for parameter-3, which is a float." + +[tool.input_parameters.parameter_4] +# Parameter 4: Example of a boolean parameter with a default value. +title = "Parameter 4" +type = "bool" # Data type: boolean. +default = true # Default value is true. +help = "A help message for parameter-4, which is a boolean flag." + +[tool.input_parameters.parameter_5] +# Parameter 5: Example of a datetime parameter. +title = "Parameter 5" +type = "datetime" # Data type: datetime in ISO 8601 format. +default = "2000-01-01T00:00:00Z" # Default value. +help = "A help message for parameter-5, which is a datetime parameter." + +[tool.input_parameters.parameter_6] +# Parameter 6: Example of a databrowser parameter for search functionality. +title = "Parameter for Variable" +type = "databrowser" # Special parameter type for databrowser integration. +help = """ +Allows integration with a databrowser search. Specify a `search_key` and +optional constraints for filtering search results. +""" +search_key = "variable" # Key used in the databrowser search. +constraint.time_frequency = "1hr" # Example constraint: time frequency. +constraint.product = "cmip" # Example constraint: product type. +default = "tas" # Default value for the search key. +search_result = "input_files" # Resolves this search to actual data. + +[tool.input_parameters.parameter_6a] +# Additional databrowser parameter for combining search queries. +title = "Parameter for Experiment" +type = "databrowser" +help = """ +Defines the "experiment" search key for querying data. Combines with other +databrowser parameters to refine search results. +""" +search_key = "experiment" +constraint.time_frequency = "1hr" +constraint.product = "cmip" +default = "amip" +search_result = "input_files" + +[tool.input_parameters.parameter_6b] +# Separate databrowser parameter for a new search result key. +title = "Another Parameter" +type = "databrowser" +help = """ +Creates a new search query and stores results under the key 'other_files'. +""" +search_key = "experiment" +constraint.time_frequency = "1hr" +constraint.product = "cmip" +default = "amip" +search_result = "other_files" + +[tool.input_parameters.parameter_6c] +# Databrowser parameter without a resolved search. +title = "Databrowser Parameter without a Search Query" +type = "databrowser" +help = """ +Passes the databrowser parameter directly without applying a search. Omit +the `search_result` key to prevent a search query. +""" +search_key = "time_frequency" +default = "1hr" + +[tool.input_parameters.parameter_7] +# Parameter 7: Example of a range parameter. +title = "Parameter 7" +type = "range" # Range type parameter. +help = """ +Specifies a range in the format [start, end, increment]. Supports shorthand +forms: [end] (equivalent to [0, end, 1]) or [start, end]. +""" +default = [0.0, 10.5, 0.1] # Default range. +mandatory = false + +[tool.input_parameters.parameter_8] +# Parameter 8: Example of a list parameter. +title = "List Parameter" +type = "integer" # List of integers. +help = """ +Accepts a list of values where each item matches the specified data type. +""" +default = [10, 1, -1] # Default list of integers. diff --git a/examples/python-lib/src/my_tool/__init__.py b/examples/python-lib/src/my_tool/__init__.py new file mode 100644 index 0000000..d5fabab --- /dev/null +++ b/examples/python-lib/src/my_tool/__init__.py @@ -0,0 +1,103 @@ +"""A mock package.""" + +import argparse +import json +import logging +import sys +from pathlib import Path +from typing import Any, Dict + +# Configure logging +logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(message)s", level=logging.INFO +) + + +def load_config(config_file: str) -> Dict[str, Any]: + """ + Load and parse the configuration JSON file. + + Parameters + ----------- + config_file (str): + Path to the configuration JSON file. + + Returns + ------- + Dict[str, Any]: + Parsed configuration as a dictionary. + + Raises + ------- + FileNotFoundError: + If the file does not exist. + json.JSONDecodeError: + If the JSON is not properly formatted. + """ + try: + with open(config_file, "r") as file: + return json.load(file) + except FileNotFoundError as error: + logging.error(f"Error: File '{config_file}' does not exist.") + raise error + except json.JSONDecodeError as error: + logging.error( + f"Error: Invalid JSON format in '{config_file}': {error}" + ) + raise error + + +def main(config_file: str) -> None: + """ + Main function to evaluate the configuration file and perform actions. + + Parameters + ----------- + config_file (str): + Path to the configuration JSON file. + """ + config = load_config(config_file) + + logging.info("Evaluating configuration from '%s'...", config_file) + + # Extract specific parameters with defaults + parameter_1 = config.get("parameter_1", "default_value") + parameter_2 = config.get("parameter_2") + parameter_3 = config.get("parameter_3", "/path/to/default") + parameter_4 = config.get("parameter_4", False) + parameter_5 = config.get("parameter_5", "2000-01-01T00:00:00Z") + + # Handle mandatory parameter checks + if parameter_2 is None: + logging.error( + "'parameter_2' is mandatory but is not provided in the config." + ) + sys.exit(1) + + # Log extracted parameters + logging.info("Extracted Parameters:") + logging.info(" Parameter 1: %s", parameter_1) + logging.info(" Parameter 2: %s", parameter_2) + logging.info(" Parameter 3: %s", parameter_3) + logging.info(" Parameter 4: %s", parameter_4) + logging.info(" Parameter 5: %s", parameter_5) + + # Perform actions based on the parameters + logging.info("Performing actions based on the configuration...") + + # Example: Conditional logic based on a boolean parameter + if parameter_4: + logging.info("Boolean parameter is true, taking appropriate action.") + + # Example: Using a parameter in a simulated action + if parameter_3: + logging.info("Simulating action with parameter 3: %s", parameter_3) + + logging.info("Configuration evaluation completed successfully.") + + +def cli(): + parser = argparse.ArgumentParser("tool-command") + parser.add_argument("config", type=Path, help="The config file.") + args = parser.parse_args() + main(args.config) diff --git a/examples/python-script/README.md b/examples/python-script/README.md new file mode 100644 index 0000000..908e6d6 --- /dev/null +++ b/examples/python-script/README.md @@ -0,0 +1,60 @@ +# Example Python Script Tool + +This example demonstrates how to define a tool in `tool.toml` to set up a +**Python** environment using Conda-Forge. The tool executes a Python script +(`example.py`) which takes a single command-line argument: a `JSON` file +containing all the parsed parameters defined in the `tool.toml` file. + +## Purpose + +This configuration allows you to: +1. Define a reproducible Conda-Forge environment specifically tailored for + running Python scripts. +2. Specify and document input parameters directly in the `tool.toml` file. +3. Provide flexibility by parameterizing the Python script through a `JSON` + file generated from the tool's input configuration. + +## How It Works + +1. **Environment Setup**: + - The `tool.toml` file specifies all dependencies required to run the + Python script. For this example, the `python=3.13` package is included + to provide the Python environment. + - Additional pip based dependencies can be added into the [requirements.txt] + file. Those dependencies will be added automatically to the conda environment. + +2. **Parameter Input**: + - Input parameters, along with their metadata (e.g., type, default value, + and description), are defined in the `tool.input_parameters` section of + the `tool.toml` file. + +3. **Script Execution**: + - When the tool runs, it generates a `JSON` file containing the parsed + parameters from the `tool.toml` configuration. + - This `JSON` file is passed as an argument to the Python script + (`example.py`), allowing the script to dynamically handle user-defined + parameters. + +## Example Use Case + +- **Data Processing**: + Run a Python script to process data dynamically, based on user-specified + parameters such as file paths, thresholds, or processing options. + +- **Machine Learning**: + Train or evaluate machine learning models with parameters like hyper- + parameters, datasets, and evaluation metrics passed through the + `tool.toml` configuration. + +## Notes + +- Ensure that all dependencies (e.g., `python`, `ffmpeg`) are specified in the + `tool.run.dependencies` section of `tool.toml`. +- For more details on TOML syntax, visit [https://toml.io](https://toml.io). + +## Additional Resources + +- [Conda-Forge Python](https://conda-forge.org/feedstocks/python) +- [JSON Parsing in Python](https://docs.python.org/3/library/json.html) +- [TOML Syntax Documentation](https://toml.io) + diff --git a/examples/python-script/environment.yml b/examples/python-script/environment.yml new file mode 100644 index 0000000..c566e56 --- /dev/null +++ b/examples/python-script/environment.yml @@ -0,0 +1,133 @@ +channels: +- conda-forge +dependencies: +- _libgcc_mutex=0.1=conda_forge +- _openmp_mutex=4.5=2_gnu +- aom=3.9.1=hac33072_0 +- bzip2=1.0.8=h4bc722e_7 +- c-ares=1.34.3=heb4867d_0 +- ca-certificates=2024.8.30=hbcca054_0 +- cairo=1.18.0=hebfffa5_3 +- cpp-expected=1.1.0=hf52228f_0 +- dav1d=1.2.1=hd590300_0 +- ffmpeg=7.1.0=gpl_hde54019_704 +- fmt=11.0.2=h434a139_0 +- font-ttf-dejavu-sans-mono=2.37=hab24e00_0 +- font-ttf-inconsolata=3.000=h77eed37_0 +- font-ttf-source-code-pro=2.038=h77eed37_0 +- font-ttf-ubuntu=0.83=h77eed37_3 +- fontconfig=2.15.0=h7e30c49_1 +- fonts-conda-ecosystem=1=0 +- fonts-conda-forge=1=0 +- freetype=2.12.1=h267a509_2 +- fribidi=1.0.10=h36c2ea0_0 +- gdk-pixbuf=2.42.12=hb9ae30d_0 +- gmp=6.3.0=hac33072_2 +- graphite2=1.3.13=h59595ed_1003 +- harfbuzz=9.0.0=hda332d3_1 +- icu=75.1=he02047a_0 +- jq=1.7.1=hd590300_0 +- keyutils=1.6.1=h166bdaf_0 +- krb5=1.21.3=h659f571_0 +- lame=3.100=h166bdaf_1003 +- ld_impl_linux-64=2.43=h712a8e2_2 +- lerc=4.0.0=h27087fc_0 +- libabseil=20240722.0=cxx17_h5888daf_1 +- libarchive=3.7.4=hfca40fe_0 +- libass=0.17.3=h1dc1e6a_0 +- libcurl=8.10.1=hbbe4b11_0 +- libdeflate=1.22=hb9d3cd8_0 +- libdrm=2.4.123=hb9d3cd8_0 +- libedit=3.1.20191231=he28a2e2_2 +- libegl=1.7.0=ha4b6fd6_2 +- libev=4.33=hd590300_2 +- libexpat=2.6.4=h5888daf_0 +- libffi=3.4.2=h7f98852_5 +- libgcc=14.2.0=h77fa898_1 +- libgcc-ng=14.2.0=h69a702a_1 +- libgl=1.7.0=ha4b6fd6_2 +- libglib=2.82.2=h2ff4ddf_0 +- libglvnd=1.7.0=ha4b6fd6_2 +- libglx=1.7.0=ha4b6fd6_2 +- libgomp=14.2.0=h77fa898_1 +- libhwloc=2.11.2=default_h0d58e46_1001 +- libiconv=1.17=hd590300_2 +- libjpeg-turbo=3.0.0=hd590300_1 +- libmamba=2.0.3=hf72d635_0 +- libmpdec=4.0.0=h4bc722e_0 +- libnghttp2=1.64.0=h161d5f1_0 +- libopenvino=2024.4.0=hac27bb2_2 +- libopenvino-auto-batch-plugin=2024.4.0=h4d9b6c2_2 +- libopenvino-auto-plugin=2024.4.0=h4d9b6c2_2 +- libopenvino-hetero-plugin=2024.4.0=h3f63f65_2 +- libopenvino-intel-cpu-plugin=2024.4.0=hac27bb2_2 +- libopenvino-intel-gpu-plugin=2024.4.0=hac27bb2_2 +- libopenvino-intel-npu-plugin=2024.4.0=hac27bb2_2 +- libopenvino-ir-frontend=2024.4.0=h3f63f65_2 +- libopenvino-onnx-frontend=2024.4.0=h5c8f2c3_2 +- libopenvino-paddle-frontend=2024.4.0=h5c8f2c3_2 +- libopenvino-pytorch-frontend=2024.4.0=h5888daf_2 +- libopenvino-tensorflow-frontend=2024.4.0=h6481b9d_2 +- libopenvino-tensorflow-lite-frontend=2024.4.0=h5888daf_2 +- libopus=1.3.1=h7f98852_1 +- libpciaccess=0.18=hd590300_0 +- libpng=1.6.44=hadc24fc_0 +- libprotobuf=5.28.2=h5b01275_0 +- librsvg=2.58.4=hc0ffecb_0 +- libsolv=0.7.30=h3509ff9_0 +- libsqlite=3.47.0=hadc24fc_1 +- libssh2=1.11.0=h0841786_0 +- libstdcxx=14.2.0=hc0a3c3a_1 +- libstdcxx-ng=14.2.0=h4852527_1 +- libtiff=4.7.0=he137b08_1 +- libuuid=2.38.1=h0b41bf4_0 +- libva=2.22.0=h8a09558_1 +- libvpx=1.14.1=hac33072_0 +- libwebp-base=1.4.0=hd590300_0 +- libxcb=1.17.0=h8a09558_0 +- libxml2=2.13.5=hb346dea_0 +- libzlib=1.3.1=hb9d3cd8_2 +- lz4-c=1.9.4=hcb278e6_0 +- lzo=2.10=hd590300_1001 +- mamba=2.0.3=hfdd0a45_0 +- ncurses=6.5=he02047a_1 +- nlohmann_json=3.11.3=he02047a_1 +- ocl-icd=2.3.2=hd590300_1 +- oniguruma=6.9.9=hd590300_0 +- openh264=2.5.0=hf92e6e3_0 +- openssl=3.4.0=hb9d3cd8_0 +- pango=1.54.0=h4c5309f_1 +- pcre2=10.44=hba22ea6_2 +- pip=24.3.1=pyh145f28c_0 +- pixman=0.43.2=h59595ed_0 +- pthread-stubs=0.4=hb9d3cd8_1002 +- pugixml=1.14=h59595ed_0 +- python=3.13.0=h9ebbce0_100_cp313 +- python_abi=3.13=5_cp313 +- readline=8.2=h8228510_1 +- reproc=14.2.5.post0=hb9d3cd8_0 +- reproc-cpp=14.2.5.post0=h5888daf_0 +- simdjson=3.10.1=h84d6215_0 +- snappy=1.2.1=ha2e4443_0 +- spdlog=1.14.1=hed91bc2_1 +- svt-av1=2.3.0=h5888daf_0 +- tbb=2022.0.0=hceb3a55_0 +- tk=8.6.13=noxft_h4845f30_101 +- tzdata=2024b=hc8b5060_0 +- wayland=1.23.1=h3e06ad9_0 +- wayland-protocols=1.37=hd8ed1ab_0 +- x264=1!164.3095=h166bdaf_2 +- x265=3.5=h924138e_3 +- xorg-libice=1.1.1=hb9d3cd8_1 +- xorg-libsm=1.2.4=he73a12e_1 +- xorg-libx11=1.8.10=h4f16b4b_0 +- xorg-libxau=1.0.11=hb9d3cd8_1 +- xorg-libxdmcp=1.1.5=hb9d3cd8_0 +- xorg-libxext=1.3.6=hb9d3cd8_0 +- xorg-libxfixes=6.0.1=hb9d3cd8_0 +- xorg-libxrender=0.9.11=hb9d3cd8_1 +- xorg-xorgproto=2024.1=hb9d3cd8_1 +- xz=5.2.6=h166bdaf_0 +- yaml-cpp=0.8.0=h59595ed_0 +- zlib=1.3.1=hb9d3cd8_2 +- zstd=1.5.6=ha6fb4c9_0 diff --git a/examples/python-script/example.py b/examples/python-script/example.py new file mode 100644 index 0000000..4811bf1 --- /dev/null +++ b/examples/python-script/example.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 + +import json +import logging +import sys +from typing import Any, Dict + +# Configure logging +logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(message)s", level=logging.INFO +) + + +def load_config(config_file: str) -> Dict[str, Any]: + """ + Load and parse the configuration JSON file. + + Parameters + ----------- + config_file (str): + Path to the configuration JSON file. + + Returns + ------- + Dict[str, Any]: + Parsed configuration as a dictionary. + + Raises + ------- + FileNotFoundError: + If the file does not exist. + json.JSONDecodeError: + If the JSON is not properly formatted. + """ + try: + with open(config_file, "r") as file: + return json.load(file) + except FileNotFoundError as error: + logging.error(f"Error: File '{config_file}' does not exist.") + raise error + except json.JSONDecodeError as error: + logging.error( + f"Error: Invalid JSON format in '{config_file}': {error}" + ) + raise error + + +def main(config_file: str) -> None: + """ + Main function to evaluate the configuration file and perform actions. + + Parameters + ----------- + config_file (str): + Path to the configuration JSON file. + """ + config = load_config(config_file) + + logging.info("Evaluating configuration from '%s'...", config_file) + + # Extract specific parameters with defaults + parameter_1 = config.get("parameter_1", "default_value") + parameter_2 = config.get("parameter_2") + parameter_3 = config.get("parameter_3", "/path/to/default") + parameter_4 = config.get("parameter_4", False) + parameter_5 = config.get("parameter_5", "2000-01-01T00:00:00Z") + + # Handle mandatory parameter checks + if parameter_2 is None: + logging.error( + "'parameter_2' is mandatory but is not provided in the config." + ) + sys.exit(1) + + # Log extracted parameters + logging.info("Extracted Parameters:") + logging.info(" Parameter 1: %s", parameter_1) + logging.info(" Parameter 2: %s", parameter_2) + logging.info(" Parameter 3: %s", parameter_3) + logging.info(" Parameter 4: %s", parameter_4) + logging.info(" Parameter 5: %s", parameter_5) + + # Perform actions based on the parameters + logging.info("Performing actions based on the configuration...") + + # Example: Conditional logic based on a boolean parameter + if parameter_4: + logging.info("Boolean parameter is true, taking appropriate action.") + + # Example: Using a parameter in a simulated action + if parameter_3: + logging.info("Simulating action with parameter 3: %s", parameter_3) + + logging.info("Configuration evaluation completed successfully.") + + +if __name__ == "__main__": + if len(sys.argv) != 2: + raise SystemExit(f"Usage: {sys.argv[0]} ") + + config_file_path = sys.argv[1] + main(config_file_path) diff --git a/examples/python-script/requirements.txt b/examples/python-script/requirements.txt new file mode 100644 index 0000000..f45cd03 --- /dev/null +++ b/examples/python-script/requirements.txt @@ -0,0 +1,3 @@ +# You can define additional pip installable dependencies in a +# requirements.txt file +- pyyaml diff --git a/examples/python-script/tool.toml b/examples/python-script/tool.toml new file mode 100644 index 0000000..3ae8889 --- /dev/null +++ b/examples/python-script/tool.toml @@ -0,0 +1,144 @@ +# This is a template configuration file for your data analysis tool. +# Use this file as a starting point by copying and editing it to fit your +# requirements. The configuration file follows TOML syntax. For more details, +# visit: https://toml.io. +# +# IMPORTANT: Remove all comments (lines starting with '#') before using this +# configuration. + +[tool] +# General information about your tool. +name = "example-python-script" # The name of your tool. +authors = [ + "Author 1 ", + "Author 2 " +] # List of authors and their contact information. +version = "v0.0.1" # Semantic version of the tool. +summary = "A brief description of what this tool does." # Short overview. +title = "A catchy title for the tool (optional)" # Optional title for flair. +description = """ +A detailed description of the tool's purpose and functionality. This section +supports multi-line text for extensive details. +""" # Extended multi-line description. + +[tool.run] +# Defines how your tool is executed. This section specifies the command to +# run, such as a script, compiled binary, or Jupyter notebook. +command = "python example.py" # Command to execute the tool. +# The specified command will receive a single JSON input containing all input +# parameters. Ensure your script processes this JSON file. Refer to the +# example script for guidance. +dependencies = ["python=3.13", "ffmpeg"] # Conda-forge packages needed. + +[tool.input_parameters.parameter_1] +# Parameter 1: Example of a string parameter with a default value. +# NOTE: Setting a default value and marking a parameter as mandatory are +# typically mutually exclusive. If a parameter has a default value, users +# are not required to provide it explicitly. +title = "Parameter 1" +type = "string" # Data type of the parameter. +mandatory = false # Indicates whether this parameter is required. +default = "default_value" # Default value for the parameter. +help = "A help message for parameter-1, which is a string parameter." + +[tool.input_parameters.parameter_2] +# Parameter 2: Example of a mandatory integer parameter. +# NOTE: Mandatory parameters should not have default values. +title = "Parameter 2" +type = "integer" # Data type: integer. +mandatory = true # Indicates this parameter is required. +help = "A help message for parameter-2, which is an integer." + +[tool.input_parameters.parameter_3] +# Parameter 3: Example of a float parameter with a default value. +title = "Parameter 3" +type = "float" # Data type: floating-point number. +mandatory = false # This parameter is optional since it has a default value. +default = 2.0 # Default value. +help = "A help message for parameter-3, which is a float." + +[tool.input_parameters.parameter_4] +# Parameter 4: Example of a boolean parameter with a default value. +title = "Parameter 4" +type = "bool" # Data type: boolean. +default = true # Default value is true. +help = "A help message for parameter-4, which is a boolean flag." + +[tool.input_parameters.parameter_5] +# Parameter 5: Example of a datetime parameter. +title = "Parameter 5" +type = "datetime" # Data type: datetime in ISO 8601 format. +default = "2000-01-01T00:00:00Z" # Default value. +help = "A help message for parameter-5, which is a datetime parameter." + +[tool.input_parameters.parameter_6] +# Parameter 6: Example of a databrowser parameter for search functionality. +title = "Parameter for Variable" +type = "databrowser" # Special parameter type for databrowser integration. +help = """ +Allows integration with a databrowser search. Specify a `search_key` and +optional constraints for filtering search results. +""" +search_key = "variable" # Key used in the databrowser search. +constraint.time_frequency = "1hr" # Example constraint: time frequency. +constraint.product = "cmip" # Example constraint: product type. +default = "tas" # Default value for the search key. +search_result = "input_files" # Resolves this search to actual data. + +[tool.input_parameters.parameter_6a] +# Additional databrowser parameter for combining search queries. +title = "Parameter for Experiment" +type = "databrowser" +help = """ +Defines the "experiment" search key for querying data. Combines with other +databrowser parameters to refine search results. +""" +search_key = "experiment" +constraint.time_frequency = "1hr" +constraint.product = "cmip" +default = "amip" +search_result = "input_files" + +[tool.input_parameters.parameter_6b] +# Separate databrowser parameter for a new search result key. +title = "Another Parameter" +type = "databrowser" +help = """ +Creates a new search query and stores results under the key 'other_files'. +""" +search_key = "experiment" +constraint.time_frequency = "1hr" +constraint.product = "cmip" +default = "amip" +search_result = "other_files" + +[tool.input_parameters.parameter_6c] +# Databrowser parameter without a resolved search. +title = "Databrowser Parameter without a Search Query" +type = "databrowser" +help = """ +Passes the databrowser parameter directly without applying a search. Omit +the `search_result` key to prevent a search query. +""" +search_key = "time_frequency" +default = "1hr" + +[tool.input_parameters.parameter_7] +# Parameter 7: Example of a range parameter. +title = "Parameter 7" +type = "range" # Range type parameter. +help = """ +Specifies a range in the format [start, end, increment]. Supports shorthand +forms: [end] (equivalent to [0, end, 1]) or [start, end]. +""" +default = [0.0, 10.5, 0.1] # Default range. +mandatory = false + +[tool.input_parameters.parameter_8] +# Parameter 8: Example of a list parameter. +title = "List Parameter" +type = "integer" # List of integers. +help = """ +Accepts a list of values where each item matches the specified data type. +""" +default = [10, 1, -1] # Default list of integers. diff --git a/examples/rust-build/Cargo.lock b/examples/rust-build/Cargo.lock new file mode 100644 index 0000000..81187b2 --- /dev/null +++ b/examples/rust-build/Cargo.lock @@ -0,0 +1,284 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "env_logger" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd405aab171cb85d6735e5c8d9db038c17d3ca007a4d2c25f337935c3d90580" +dependencies = [ + "humantime", + "is-terminal", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "example" +version = "0.1.0" +dependencies = [ + "env_logger", + "log", + "serde", + "serde_json", +] + +[[package]] +name = "hermit-abi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "is-terminal" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "itoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" + +[[package]] +name = "libc" +version = "0.2.164" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "433bfe06b8c75da9b2e3fbea6e5329ff87748f0b144ef75306e674c3f6f7c13f" + +[[package]] +name = "log" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "proc-macro2" +version = "1.0.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + +[[package]] +name = "serde" +version = "1.0.215" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.215" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.133" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "syn" +version = "2.0.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "unicode-ident" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" + +[[package]] +name = "winapi-util" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" diff --git a/examples/rust-build/Cargo.toml b/examples/rust-build/Cargo.toml new file mode 100644 index 0000000..5dccd38 --- /dev/null +++ b/examples/rust-build/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "example" +version = "0.1.0" +edition = "2021" + +[dependencies] +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +log = "0.4" +env_logger = "0.10" diff --git a/examples/rust-build/README.md b/examples/rust-build/README.md new file mode 100644 index 0000000..5f29dfe --- /dev/null +++ b/examples/rust-build/README.md @@ -0,0 +1,43 @@ +# Example Rust Binary Tool + +This example demonstrates how to define a tool in `tool.toml` to set up a +**Rust binary execution environment** using Conda-Forge. The tool compiles +and executes a Rust binary (`example`) which takes a single command-line +argument: a `JSON` file containing all the parsed parameters defined in the +`tool.toml` file. + +## Purpose + +This configuration allows you to: +1. Define a reproducible Conda-Forge environment with Rust as the primary + language. +2. Automate the build process for compiling the Rust binary. +3. Specify and document input parameters directly in the `tool.toml` file. + +## How It Works + +1. **Environment Setup**: + - The `tool.toml` file specifies all dependencies required to build and + execute the Rust binary. For this example, the `rust` package is included + to provide the Rust compiler. + +2. **Build Process**: + - A `build.sh` script is required to handle all compilation or additional + commands needed to prepare the binary for execution. + - This script is automatically executed during the build process. + +3. **Parameter Input**: + - Input parameters, along with their metadata (e.g., type, default value, + and description), are defined in the `tool.input_parameters` section of + the `tool.toml` file. + +4. **Binary Execution**: + - After the build process completes, the compiled binary is executed + using the command defined in `tool.run.command`. It dynamically handles + user-defined parameters provided through a `JSON` file. + +## Build Script (`build.sh`) + +The `build.sh` script is essential for tools requiring compilation. It allows +you to define custom build steps. For this Rust example, the script compiles +the binary and places it in the `target/release` directory. diff --git a/examples/rust-build/build-environment.yml b/examples/rust-build/build-environment.yml new file mode 100644 index 0000000..d3eba2a --- /dev/null +++ b/examples/rust-build/build-environment.yml @@ -0,0 +1,20 @@ +--- +channels: + - conda-forge +dependencies: + - _libgcc_mutex=0.1=conda_forge + - _openmp_mutex=4.5=2_gnu + - binutils_impl_linux-64=2.43=h4bf12b8_2 + - gcc_impl_linux-64=14.2.0=h6b349bd_1 + - kernel-headers_linux-64=3.10.0=he073ed8_18 + - ld_impl_linux-64=2.43=h712a8e2_2 + - libgcc=14.2.0=h77fa898_1 + - libgcc-devel_linux-64=14.2.0=h41c2201_101 + - libgomp=14.2.0=h77fa898_1 + - libsanitizer=14.2.0=h2a3dede_1 + - libstdcxx=14.2.0=hc0a3c3a_1 + - libzlib=1.3.1=hb9d3cd8_2 + - rust=1.81.0=h1a8d7c4_0 + - rust-std-x86_64-unknown-linux-gnu=1.81.0=h2c6d0dc_0 + - sysroot_linux-64=2.17=h4a8ded7_18 + - tzdata=2024b=hc8b5060_0 diff --git a/examples/rust-build/build.sh b/examples/rust-build/build.sh new file mode 100755 index 0000000..088e09d --- /dev/null +++ b/examples/rust-build/build.sh @@ -0,0 +1,23 @@ +#!/bin/sh + +# Exit immediately if a command exits with a non-zero status +set -e + +# Check if Cargo is installed +if ! command -v cargo &> /dev/null +then + echo "Error: Cargo is not installed. Please install Cargo to build this project." + exit 1 +fi + +# Build the Rust project +echo "Building the Rust project..." +cargo build --release + +# Confirm the build succeeded +if [ -f "./target/release/example" ]; then + echo "Build succeeded. Binary is located at ./target/release/example" +else + echo "Error: Build failed." + exit 1 +fi diff --git a/examples/rust-build/config.json b/examples/rust-build/config.json new file mode 100644 index 0000000..33a6009 --- /dev/null +++ b/examples/rust-build/config.json @@ -0,0 +1,8 @@ +{ + "parameter_1": "example_value_1", + "parameter_2": 42, + "parameter_3": "/path/to/resource", + "parameter_4": true, + "parameter_5": "2024-11-19T10:00:00Z" +} + diff --git a/examples/rust-build/environment.yml b/examples/rust-build/environment.yml new file mode 100644 index 0000000..0bea4b4 --- /dev/null +++ b/examples/rust-build/environment.yml @@ -0,0 +1,43 @@ +channels: +- conda-forge +dependencies: +- _libgcc_mutex=0.1=conda_forge +- _openmp_mutex=4.5=2_gnu +- bzip2=1.0.8=h4bc722e_7 +- c-ares=1.34.3=heb4867d_0 +- ca-certificates=2024.8.30=hbcca054_0 +- cpp-expected=1.1.0=hf52228f_0 +- fmt=11.0.2=h434a139_0 +- jq=1.7.1=hd590300_0 +- keyutils=1.6.1=h166bdaf_0 +- krb5=1.21.3=h659f571_0 +- libarchive=3.7.4=hfca40fe_0 +- libcurl=8.10.1=hbbe4b11_0 +- libedit=3.1.20191231=he28a2e2_2 +- libev=4.33=hd590300_2 +- libgcc=14.2.0=h77fa898_1 +- libgcc-ng=14.2.0=h69a702a_1 +- libgomp=14.2.0=h77fa898_1 +- libiconv=1.17=hd590300_2 +- libmamba=2.0.3=hf72d635_0 +- libnghttp2=1.64.0=h161d5f1_0 +- libsolv=0.7.30=h3509ff9_0 +- libssh2=1.11.0=h0841786_0 +- libstdcxx=14.2.0=hc0a3c3a_1 +- libstdcxx-ng=14.2.0=h4852527_1 +- libxml2=2.13.5=h064dc61_0 +- libzlib=1.3.1=hb9d3cd8_2 +- lz4-c=1.9.4=hcb278e6_0 +- lzo=2.10=hd590300_1001 +- mamba=2.0.3=hfdd0a45_0 +- ncurses=6.5=he02047a_1 +- nlohmann_json=3.11.3=he02047a_1 +- oniguruma=6.9.9=hd590300_0 +- openssl=3.4.0=hb9d3cd8_0 +- reproc=14.2.5.post0=hb9d3cd8_0 +- reproc-cpp=14.2.5.post0=h5888daf_0 +- simdjson=3.10.1=h84d6215_0 +- spdlog=1.14.1=hed91bc2_1 +- xz=5.2.6=h166bdaf_0 +- yaml-cpp=0.8.0=h59595ed_0 +- zstd=1.5.6=ha6fb4c9_0 diff --git a/examples/rust-build/src/main.rs b/examples/rust-build/src/main.rs new file mode 100644 index 0000000..adca829 --- /dev/null +++ b/examples/rust-build/src/main.rs @@ -0,0 +1,57 @@ +use std::env; +use std::fs::File; +use std::io::Read; +use serde_json::Value; +use log::{info, error}; + +fn main() { + // Initialize logging + env_logger::init(); + + // Get the configuration file path from the command-line arguments + let args: Vec = env::args().collect(); + if args.len() != 2 { + error!("Usage: {} ", args[0]); + std::process::exit(1); + } + let config_file = &args[1]; + + // Read and parse the configuration JSON file + let config = match load_config(config_file) { + Ok(config) => config, + Err(e) => { + error!("Failed to load configuration: {}", e); + std::process::exit(1); + } + }; + + // Assign a long-lived default value + let default_parameter_1 = Value::String("default_value".to_string()); + let parameter_1 = config.get("parameter_1").unwrap_or(&default_parameter_1); + + let parameter_2 = config.get("parameter_2"); + + // Check mandatory parameter + if parameter_2.is_none() { + error!("Error: 'parameter_2' is mandatory but is not provided."); + std::process::exit(1); + } + + // Log extracted parameters + info!("Parameter 1: {}", parameter_1); + info!("Parameter 2: {}", parameter_2.unwrap()); + + // Perform actions based on the parameters + info!("Performing actions based on the configuration..."); + // Add logic for processing parameters + + info!("Configuration evaluation completed successfully."); +} + +fn load_config(config_file: &str) -> Result> { + let mut file = File::open(config_file)?; + let mut contents = String::new(); + file.read_to_string(&mut contents)?; + let json: Value = serde_json::from_str(&contents)?; + Ok(json) +} diff --git a/examples/rust-build/tool.toml b/examples/rust-build/tool.toml new file mode 100644 index 0000000..349dbcb --- /dev/null +++ b/examples/rust-build/tool.toml @@ -0,0 +1,155 @@ +# This is a template configuration file for your data analysis tool. +# Use this file as a starting point by copying and editing it to fit your +# requirements. The configuration file follows TOML syntax. For more details, +# visit: https://toml.io. +# +# IMPORTANT: Remove all comments (lines starting with '#') before using this +# configuration. + +[tool] +# General information about your tool. +name = "example-rust-build" # The name of your tool. +authors = [ + "Author 1 ", + "Author 2 " +] # List of authors and their contact information. +version = "v0.0.2" # Semantic version of the tool. +summary = "A brief description of what this tool does." # Short overview. +description = """ +A detailed description of the tool's purpose and functionality. This section +supports multi-line text for extensive details. +""" # Extended multi-line description. + +[tool.run] +# Defines how your tool is executed. This section specifies the command to +# run, such as a script, compiled binary, or Jupyter notebook. +command = "./target/release/example" # Command to execute the tool. +# The specified command will receive a single JSON input containing all input +# parameters. Ensure your command processes this JSON file. Refer to the +# example script for guidance. + +[tool.build] +# Configuration for building the tool. +dependencies = ["rust"] # Optional build-specific deps. +# NOTE: If a 'build.sh' script is provided, it will be executed during build. + +[tool.input_parameters.parameter_1] +# Parameter 1: Example of a string parameter with a default value. +# NOTE: Setting a default value and marking a parameter as mandatory are +# typically mutually exclusive. If a parameter has a default value, users +# are not required to provide it explicitly. +title = "Parameter 1" +type = "string" # Data type of the parameter. +mandatory = false # Indicates whether this parameter is required. +default = "default_value" # Default value for the parameter. +help = "A help message for parameter-1, which is a string parameter." + +[tool.input_parameters.parameter_2] +# Parameter 2: Example of a mandatory integer parameter. +# NOTE: Mandatory parameters should not have default values. +title = "Parameter 2" +type = "integer" # Data type: integer. +mandatory = true # Indicates this parameter is required. +help = "A help message for parameter-2, which is an integer." + +[tool.input_parameters.parameter_3] +# Parameter 3: Example of a float parameter with a default value. +title = "Parameter 3" +type = "float" # Data type: floating-point number. +mandatory = false # This parameter is optional since it has a default value. +default = 2.0 # Default value. +help = "A help message for parameter-3, which is a float." + +[tool.input_parameters.parameter_4] +# Parameter 4: Example of a boolean parameter with a default value. +type = "bool" # Data type: boolean. +default = true # Default value is true. +help = "A help message for parameter-4, which is a boolean flag." + +[tool.input_parameters.parameter_5] +# Parameter 5: Example of a datetime parameter. +title = "Parameter 5" +type = "datetime" # Data type: datetime in ISO 8601 format. +default = "2000-01-01T00:00:00Z" # Default value. +help = "A help message for parameter-5, which is a datetime parameter." + +[tool.input_parameters.parameter_6] +# Parameter 6: Example of a databrowser parameter for search functionality. +title = "Parameter for Variable" +type = "databrowser" # Special parameter type for databrowser integration. +help = """ +Allows integration with a databrowser search. Specify a `search_key` and +optional constraints for filtering search results. +""" +search_key = "variable" # Key used in the databrowser search. +constraint.time_frequency = "1hr" # Example constraint: time frequency. +constraint.product = "cmip" # Example constraint: product type. +default = "tas" # Default value for the search key. +search_result = "input_files" # Resolves this search to actual data. + +[tool.input_parameters.parameter_6a] +# Additional databrowser parameter for combining search queries. +title = "Parameter for Experiment" +type = "databrowser" +help = """ +Defines the "experiment" search key for querying data. Combines with other +databrowser parameters to refine search results. +""" +search_key = "experiment" +constraint.time_frequency = "1hr" +constraint.product = "cmip" +default = "amip" +search_result = "input_files" + +[tool.input_parameters.parameter_6b] +# Separate databrowser parameter for a new search result key. +title = "Another Parameter" +type = "databrowser" +help = """ +Creates a new search query and stores results under the key 'other_files'. +""" +search_key = "experiment" +constraint.time_frequency = "1hr" +constraint.product = "cmip" +default = "amip" +search_result = "other_files" + +[tool.input_parameters.parameter_6c] +# Databrowser parameter without a resolved search. +title = "Databrowser Parameter without a Search Query" +type = "databrowser" +help = """ +Passes the databrowser parameter directly without applying a search. Omit +the `search_result` key to prevent a search query. +""" +search_key = "time_frequency" +default = "1hr" + +[tool.input_parameters.parameter_7] +# Parameter 7: Example of a range parameter. +title = "Parameter 7" +type = "range" # Range type parameter. +help = """ +Specifies a range in the format [start, end, increment]. Supports shorthand +forms: [end] (equivalent to [0, end, 1]) or [start, end]. +""" +default = [0.0, 10.5, 0.1] # Default range. +mandatory = false + +[tool.input_parameters.parameter_8] +# Parameter 8: Example of a list parameter. +title = "List Parameter" +type = "integer" # List of integers. +help = """ +Accepts a list of values where each item matches the specified data type. +""" +default = [10, 1, -1] # Default list of integers. + +[tool.input_parameters.parameter_9] +# Parameter 9: Example of a path parameter. +title = "Parameter 9" +type = "path" # Data type of the parameter. +mandatory = true # Indicates whether this parameter is required. +help = "A help message for parameter-9, which is a path parameter." + + diff --git a/examples/shell-script/README.md b/examples/shell-script/README.md new file mode 100644 index 0000000..c350831 --- /dev/null +++ b/examples/shell-script/README.md @@ -0,0 +1,59 @@ +# Example Shell Script Tool + +This example demonstrates how to define a tool in `tool.toml` to set up a +**bash shell script** execution environment using Conda-Forge. The tool +executes a shell script (`example.sh`) which takes a single command-line +argument: a `JSON` file containing all the parsed parameters defined in the +`tool.toml` file. + +## Purpose + +This configuration allows you to: +1. Define a reproducible Conda-Forge environment specifically tailored for + running shell scripts. +2. Specify and document input parameters directly in the `tool.toml` file. +3. Provide flexibility by parameterizing the shell script through a `JSON` + file generated from the tool's input configuration. + +## How It Works + +1. **Environment Setup**: + - The `tool.toml` file specifies all dependencies required to run the + shell script. For this example, dependencies such as `python=3.13` are + included to support auxiliary functionality like parsing JSON. + +2. **Parameter Input**: + - Input parameters, along with their metadata (e.g., type, default value, + and description), are defined in the `tool.input_parameters` section of + the `tool.toml` file. + +3. **Script Execution**: + - When the tool runs, it generates a `JSON` file containing the parsed + parameters from the `tool.toml` configuration. + - This `JSON` file is passed as an argument to the shell script + (`example.sh`), allowing the script to dynamically handle user-defined + parameters. + +## Example Use Case + +- **File Processing**: + Execute a shell script that processes files dynamically based on user- + specified parameters such as file paths, processing methods, or filters. + +- **System Automation**: + Automate tasks like backups, deployments, or data migrations by defining + parameters for paths, thresholds, or other criteria in the `tool.toml` + configuration. + +## Notes + +- Ensure that all dependencies required by the shell script (e.g., `bash`, + `python`, `ffmpeg`) are specified in the `tool.run.dependencies` section + of `tool.toml`. +- For more details on TOML syntax, visit [https://toml.io](https://toml.io). + +## Additional Resources + +- [Conda-Forge Bash](https://github.com/conda-forge/bash-feedstock) +- [JSON Parsing in Bash](https://stedolan.github.io/jq/) +- [TOML Syntax Documentation](https://toml.io) diff --git a/examples/shell-script/example.sh b/examples/shell-script/example.sh new file mode 100644 index 0000000..1ae2327 --- /dev/null +++ b/examples/shell-script/example.sh @@ -0,0 +1,55 @@ +#!/bin/sh +# This is an example script that should demonstrate how your data-analysis +# shell based tool can handle the input. + +# Check if the required input JSON file is provided +if [ "$#" -ne 1 ]; then + echo "Usage: $0 " + exit 1 +fi + +# Read the JSON file +CONFIG_FILE="$1" + +# Check if the file exists +if [ ! -f "$CONFIG_FILE" ]; then + echo "Error: File '$CONFIG_FILE' does not exist." + exit 1 +fi + +# Parse and extract values from the JSON file using jq +# By default jq (https://stedolan.github.io/jq/) is installed into the tools +# conda-forge environment. +echo "Evaluating configuration from '$CONFIG_FILE'..." + +# Example: Extract specific parameters +PARAMETER_1=$(jq -r '.parameter_1' "$CONFIG_FILE") +PARAMETER_2=$(jq -r '.parameter_2' "$CONFIG_FILE") +PARAMETER_3=$(jq -r '.parameter_3' "$CONFIG_FILE") +PARAMETER_4=$(jq -r '.parameter_4' "$CONFIG_FILE") +PARAMETER_5=$(jq -r '.parameter_5' "$CONFIG_FILE") + +# Handle default values or mandatory checks if needed +if [ -z "$PARAMETER_2" ]; then + echo "Error: 'parameter_2' is mandatory but is not provided in the config." + exit 1 +fi + +# Log extracted parameters +echo "Parameter 1: $PARAMETER_1" +echo "Parameter 2: $PARAMETER_2" +echo "Parameter 3: $PARAMETER_3" +echo "Parameter 4: $PARAMETER_4" +echo "Parameter 5: $PARAMETER_5" + +# Perform actions based on the parameters +echo "Performing actions based on the configuration..." + +# Example of conditional logic +if [ "$PARAMETER_4" = "true" ]; then + echo "Boolean parameter is true, taking appropriate action." +fi + +# Final message +echo "Configuration evaluation completed successfully." +exit 0 diff --git a/examples/shell-script/tool.toml b/examples/shell-script/tool.toml new file mode 100644 index 0000000..ca0ea43 --- /dev/null +++ b/examples/shell-script/tool.toml @@ -0,0 +1,149 @@ +# This is a template configuration file for your data analysis tool. +# Use this file as a starting point by copying and editing it to fit your +# requirements. The configuration file follows TOML syntax. For more details, +# visit: https://toml.io. +# +# IMPORTANT: Remove all comments (lines starting with '#') before using this +# configuration. + +[tool] +# General information about your tool. +name = "example-shell-script" # The name of your tool. +authors = [ + "Author 1 ", + "Author 2 " +] # List of authors and their contact information. +version = "v0.0.1" # Semantic version of the tool. +summary = "A brief description of what this tool does." # Short overview. +title = "A catchy title for the tool (optional)" # Optional title for flair. +description = """ +A detailed description of the tool's purpose and functionality. This section +supports multi-line text for extensive details. +""" # Extended multi-line description. + +[tool.run] +# Defines how your tool is executed. This section specifies the command to +# run, such as a script, compiled binary, or Jupyter notebook. +command = "/bin/bash example.sh" # Command to execute the tool. +# The specified command will receive a single JSON input containing all input +# parameters. Ensure your script processes this JSON file. Refer to the +# example script for guidance. +dependencies = ["python=3.13", "ffmpeg"] # Conda-forge packages needed. + +[tool.build] +# Configuration for building the tool. +dependencies = ["python=3.13", "mamba"] # Optional build-specific deps. +# NOTE: If a 'build.sh' script is provided, it will be executed during build. + +[tool.input_parameters.parameter_1] +# Parameter 1: Example of a string parameter with a default value. +# NOTE: Setting a default value and marking a parameter as mandatory are +# typically mutually exclusive. If a parameter has a default value, users +# are not required to provide it explicitly. +title = "Parameter 1" +type = "string" # Data type of the parameter. +mandatory = false # Indicates whether this parameter is required. +default = "default_value" # Default value for the parameter. +help = "A help message for parameter-1, which is a string parameter." + +[tool.input_parameters.parameter_2] +# Parameter 2: Example of a mandatory integer parameter. +# NOTE: Mandatory parameters should not have default values. +title = "Parameter 2" +type = "integer" # Data type: integer. +mandatory = true # Indicates this parameter is required. +help = "A help message for parameter-2, which is an integer." + +[tool.input_parameters.parameter_3] +# Parameter 3: Example of a float parameter with a default value. +title = "Parameter 3" +type = "float" # Data type: floating-point number. +mandatory = false # This parameter is optional since it has a default value. +default = 2.0 # Default value. +help = "A help message for parameter-3, which is a float." + +[tool.input_parameters.parameter_4] +# Parameter 4: Example of a boolean parameter with a default value. +title = "Parameter 4" +type = "bool" # Data type: boolean. +default = true # Default value is true. +help = "A help message for parameter-4, which is a boolean flag." + +[tool.input_parameters.parameter_5] +# Parameter 5: Example of a datetime parameter. +title = "Parameter 5" +type = "datetime" # Data type: datetime in ISO 8601 format. +default = "2000-01-01T00:00:00Z" # Default value. +help = "A help message for parameter-5, which is a datetime parameter." + +[tool.input_parameters.parameter_6] +# Parameter 6: Example of a databrowser parameter for search functionality. +title = "Parameter for Variable" +type = "databrowser" # Special parameter type for databrowser integration. +help = """ +Allows integration with a databrowser search. Specify a `search_key` and +optional constraints for filtering search results. +""" +search_key = "variable" # Key used in the databrowser search. +constraint.time_frequency = "1hr" # Example constraint: time frequency. +constraint.product = "cmip" # Example constraint: product type. +default = "tas" # Default value for the search key. +search_result = "input_files" # Resolves this search to actual data. + +[tool.input_parameters.parameter_6a] +# Additional databrowser parameter for combining search queries. +title = "Parameter for Experiment" +type = "databrowser" +help = """ +Defines the "experiment" search key for querying data. Combines with other +databrowser parameters to refine search results. +""" +search_key = "experiment" +constraint.time_frequency = "1hr" +constraint.product = "cmip" +default = "amip" +search_result = "input_files" + +[tool.input_parameters.parameter_6b] +# Separate databrowser parameter for a new search result key. +title = "Another Parameter" +type = "databrowser" +help = """ +Creates a new search query and stores results under the key 'other_files'. +""" +search_key = "experiment" +constraint.time_frequency = "1hr" +constraint.product = "cmip" +default = "amip" +search_result = "other_files" + +[tool.input_parameters.parameter_6c] +# Databrowser parameter without a resolved search. +title = "Databrowser Parameter without a Search Query" +type = "databrowser" +help = """ +Passes the databrowser parameter directly without applying a search. Omit +the `search_result` key to prevent a search query. +""" +search_key = "time_frequency" +default = "1hr" + +[tool.input_parameters.parameter_7] +# Parameter 7: Example of a range parameter. +title = "Parameter 7" +type = "range" # Range type parameter. +help = """ +Specifies a range in the format [start, end, increment]. Supports shorthand +forms: [end] (equivalent to [0, end, 1]) or [start, end]. +""" +default = [0.0, 10.5, 0.1] # Default range. +mandatory = false + +[tool.input_parameters.parameter_8] +# Parameter 8: Example of a list parameter. +title = "List Parameter" +type = "integer" # List of integers. +help = """ +Accepts a list of values where each item matches the specified data type. +""" +default = [10, 1, -1] # Default list of integers. diff --git a/tools/.keep b/tools/.keep new file mode 100644 index 0000000..e69de29