From 46f551d99586964c63eb5e146009f43066f203d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5kon=20H=C3=A6gland?= Date: Wed, 8 Jan 2025 15:53:46 +0100 Subject: [PATCH 1/5] Add script to download the docstring files This script can be used to download the docstring files and the dune.module file if you want to build the documentation locally. --- README.md | 2 ++ scripts/download_files.py | 73 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100755 scripts/download_files.py diff --git a/README.md b/README.md index c39a40d..98987be 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,8 @@ ## Building the documentation locally Follow the commands in .github/workflows/python_sphinx_docs.yml for your local setup! +See the script [scripts/download_files.py](https://github.com/OPM/opm-python-documentation/blob/master/scripts/download_files.py) for more information. + ## Building the documentation online on your fork - Turn on github actions at `https://github.com//opm-python-documentation/actions` - Push any changes to a branch of your fork, this should trigger a build of the documentation, where the built documentation is pushed to the branch `gh-pages-` diff --git a/scripts/download_files.py b/scripts/download_files.py new file mode 100755 index 0000000..2e27ae1 --- /dev/null +++ b/scripts/download_files.py @@ -0,0 +1,73 @@ +#! /usr/bin/env python3 + +import logging +import requests +from pathlib import Path + +URL_SIMULATORS = "https://raw.githubusercontent.com/OPM/opm-simulators/master/python/docstrings_simulators.json" +URL_COMMON = "https://raw.githubusercontent.com/OPM/opm-common/master/python/docstrings_common.json" +URL_DUNE_MODULE = "https://raw.githubusercontent.com/OPM/opm-simulators/master/dune.module" + +def get_script_dir(): + """Return the directory of the script.""" + script_path = Path(__file__).resolve() + script_dir = script_path.parent + return script_dir + +def convert_pr_to_commit_hash(repo: str, pr_number: str) -> str: + """Convert a PR number to a commit hash.""" + url = f"https://api.github.com/repos/OPM/{repo}/pulls/{pr_number}" + response = requests.get(url) + response.raise_for_status() + commit_hash = response.json()["head"]["sha"] + return commit_hash + +def download_docstring_file(url: str) -> None: + """Download a docstrings file from a URL (either opm-simulators or opm-common).""" + # Ask command line user question if to use master or PR branch + if "opm-simulators" in url: + repo = "opm-simulators" + filename = "docstrings_simulators.json" + else: + repo = "opm-common" + filename = "docstrings_common.json" + print(f"Downloading docstrings file from {repo} repository. " + "Should we use the master branch or a PR branch?") + branch = input("Enter 'master' or a PR number: ") + if branch == "master": + url = url.replace("master", branch) + else: + pr_number = branch + if not pr_number.isdigit(): + print("Invalid PR number.") + return + commit_hash = convert_pr_to_commit_hash(repo, pr_number) + url = url.replace("master", commit_hash) + logging.info(f"Downloading docstrings file from {url}") + response = requests.get(url) + response.raise_for_status() # Raises 404 if the file is not found + script_dir = get_script_dir() + save_path = script_dir.parent / "python" / filename + with open(save_path, "wb") as file: + file.write(response.content) + logging.info(f"Saved docstrings file to {save_path}") + +def download_dune_module() -> None: + """Download the dune.module file from the opm-simulators repository.""" + logging.info("Downloading dune.module file") + response = requests.get(URL_DUNE_MODULE) + response.raise_for_status() + script_dir = get_script_dir() + save_path = script_dir.parent / "dune.module" + with open(save_path, "wb") as file: + file.write(response.content) + logging.info(f"Saved dune.module file to {save_path}") + +def main() -> None: + download_docstring_file(URL_SIMULATORS) + download_docstring_file(URL_COMMON) + download_dune_module() + +if __name__ == '__main__': + logging.basicConfig(level=logging.INFO) + main() From 053593362862ff9c3d5b8b4d77de2910744ca8a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5kon=20H=C3=A6gland?= Date: Wed, 8 Jan 2025 16:07:17 +0100 Subject: [PATCH 2/5] Remove unecessary replace --- scripts/download_files.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scripts/download_files.py b/scripts/download_files.py index 2e27ae1..436ad9c 100755 --- a/scripts/download_files.py +++ b/scripts/download_files.py @@ -34,15 +34,13 @@ def download_docstring_file(url: str) -> None: print(f"Downloading docstrings file from {repo} repository. " "Should we use the master branch or a PR branch?") branch = input("Enter 'master' or a PR number: ") - if branch == "master": - url = url.replace("master", branch) - else: + if branch != "master": pr_number = branch if not pr_number.isdigit(): print("Invalid PR number.") return commit_hash = convert_pr_to_commit_hash(repo, pr_number) - url = url.replace("master", commit_hash) + url = url.replace("/master/", f"/{commit_hash}/") logging.info(f"Downloading docstrings file from {url}") response = requests.get(url) response.raise_for_status() # Raises 404 if the file is not found From 22c80d05b4994a78f833c679b640d5b5168e74b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5kon=20H=C3=A6gland?= Date: Wed, 8 Jan 2025 18:35:29 +0100 Subject: [PATCH 3/5] Move script to python folder --- python/sphinx_docs/README.md | 14 +++++++- python/sphinx_docs/docs/conf.py | 4 +-- python/sphinx_docs/pyproject.toml | 9 +++-- .../src/opm_python_docs}/download_files.py | 34 ++++++++++++------- .../sphinx_ext_docstrings.py | 0 5 files changed, 43 insertions(+), 18 deletions(-) rename {scripts => python/sphinx_docs/src/opm_python_docs}/download_files.py (71%) rename python/sphinx_docs/src/{opm_simulators_docs => opm_python_docs}/sphinx_ext_docstrings.py (100%) diff --git a/python/sphinx_docs/README.md b/python/sphinx_docs/README.md index 599a5db..24c2712 100644 --- a/python/sphinx_docs/README.md +++ b/python/sphinx_docs/README.md @@ -1,4 +1,4 @@ -# Python scripts for building opm-simulators sphinx documentation +# Python scripts for building opm-simulators and opm-common sphinx documentation ## Installation of the python scripts - Requires python3 >= 3.10 @@ -21,3 +21,15 @@ $ python -m venv .venv $ source .venv/bin/activate $ pip install . ``` + +### Scripts + +After installation, you can run the following scripts: + +``` +# Downloads docstrings JSON files and dune.module file before building the documentation locally +$ opmdoc-download-files +# Generate the documentation +$ make +# View the generated documentation in the browser +``` diff --git a/python/sphinx_docs/docs/conf.py b/python/sphinx_docs/docs/conf.py index 110fb80..5bc7600 100644 --- a/python/sphinx_docs/docs/conf.py +++ b/python/sphinx_docs/docs/conf.py @@ -34,7 +34,7 @@ def extract_opm_simulators_release(): # opm.simulators.BlackOilSimulator Python module will also have access to the docstrings.) sys.path.insert(0, os.path.abspath("../src")) # Our sphinx extension that will use the docstrings.json file to generate documentation -extensions = ["opm_simulators_docs.sphinx_ext_docstrings"] +extensions = ["opm_python_docs.sphinx_ext_docstrings"] # Path to docstrings.json opm_simulators_docstrings_path = os.path.abspath('../../docstrings_simulators.json') opm_common_docstrings_path = os.path.abspath('../../docstrings_common.json') @@ -50,7 +50,7 @@ def extract_opm_simulators_release(): html_context = { "display_github": True, "github_user": "OPM", - "github_repo": "opm-simulators", + "github_repo": "opm-python-documentation", "github_version": "master", "conf_py_path": "/python/docs/", } diff --git a/python/sphinx_docs/pyproject.toml b/python/sphinx_docs/pyproject.toml index 3e22ef6..30111aa 100644 --- a/python/sphinx_docs/pyproject.toml +++ b/python/sphinx_docs/pyproject.toml @@ -1,11 +1,11 @@ [tool.poetry] -name = "opm-simulators-docs" +name = "opm-python-docs" version = "0.1.0" description = """Helper scripts for generating sphinx documentation for the \ - opm-simulators Python bindings""" + opm-simulators and opm-common Python bindings""" authors = ["Håkon Hægland "] readme = "README.md" -packages = [{ include = "opm_simulators_docs", from = "src"}] +packages = [{ include = "opm_python_docs", from = "src"}] license = "GPL3" repository = "https://github.com/OPM/opm-simulators" @@ -16,6 +16,9 @@ sphinx-rtd-theme = "^1.3.0" click = "^8.1.7" sphinx-versioned-docs = "^1.3.1" +[tool.poetry.scripts] +opmdoc-download-files = "opm_python_docs.download_files:main" + [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" diff --git a/scripts/download_files.py b/python/sphinx_docs/src/opm_python_docs/download_files.py similarity index 71% rename from scripts/download_files.py rename to python/sphinx_docs/src/opm_python_docs/download_files.py index 436ad9c..c43a8ba 100755 --- a/scripts/download_files.py +++ b/python/sphinx_docs/src/opm_python_docs/download_files.py @@ -2,17 +2,28 @@ import logging import requests +import subprocess from pathlib import Path URL_SIMULATORS = "https://raw.githubusercontent.com/OPM/opm-simulators/master/python/docstrings_simulators.json" URL_COMMON = "https://raw.githubusercontent.com/OPM/opm-common/master/python/docstrings_common.json" URL_DUNE_MODULE = "https://raw.githubusercontent.com/OPM/opm-simulators/master/dune.module" -def get_script_dir(): - """Return the directory of the script.""" - script_path = Path(__file__).resolve() - script_dir = script_path.parent - return script_dir +def get_git_root() -> Path: + """Return the absolute path of the opm-python-documentation repository's root.""" + try: + output = subprocess.check_output( + ["git", "rev-parse", "--show-toplevel"], + stderr=subprocess.STDOUT + ) + except subprocess.CalledProcessError: + # Handle the error if we're not inside a Git repo, etc. + raise RuntimeError("Not a valid Git repository or other error occurred.") + # Check that the parent directory is the opm-python-documentation repository + root = output.decode("utf-8").strip() + if not root.endswith("opm-python-documentation"): + raise RuntimeError("Not in the opm-python-documentation repository.") + return Path(root) def convert_pr_to_commit_hash(repo: str, pr_number: str) -> str: """Convert a PR number to a commit hash.""" @@ -24,7 +35,6 @@ def convert_pr_to_commit_hash(repo: str, pr_number: str) -> str: def download_docstring_file(url: str) -> None: """Download a docstrings file from a URL (either opm-simulators or opm-common).""" - # Ask command line user question if to use master or PR branch if "opm-simulators" in url: repo = "opm-simulators" filename = "docstrings_simulators.json" @@ -44,9 +54,9 @@ def download_docstring_file(url: str) -> None: logging.info(f"Downloading docstrings file from {url}") response = requests.get(url) response.raise_for_status() # Raises 404 if the file is not found - script_dir = get_script_dir() - save_path = script_dir.parent / "python" / filename - with open(save_path, "wb") as file: + git_root_dir = get_git_root() + save_path = git_root_dir / "python" / filename + with open(str(save_path), "wb") as file: file.write(response.content) logging.info(f"Saved docstrings file to {save_path}") @@ -55,17 +65,17 @@ def download_dune_module() -> None: logging.info("Downloading dune.module file") response = requests.get(URL_DUNE_MODULE) response.raise_for_status() - script_dir = get_script_dir() - save_path = script_dir.parent / "dune.module" + git_root_dir = get_git_root() + save_path = git_root_dir / "dune.module" with open(save_path, "wb") as file: file.write(response.content) logging.info(f"Saved dune.module file to {save_path}") def main() -> None: + logging.basicConfig(level=logging.INFO) download_docstring_file(URL_SIMULATORS) download_docstring_file(URL_COMMON) download_dune_module() if __name__ == '__main__': - logging.basicConfig(level=logging.INFO) main() diff --git a/python/sphinx_docs/src/opm_simulators_docs/sphinx_ext_docstrings.py b/python/sphinx_docs/src/opm_python_docs/sphinx_ext_docstrings.py similarity index 100% rename from python/sphinx_docs/src/opm_simulators_docs/sphinx_ext_docstrings.py rename to python/sphinx_docs/src/opm_python_docs/sphinx_ext_docstrings.py From 89d336349b9b86cf742ef72dca171f662362a782 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5kon=20H=C3=A6gland?= Date: Wed, 8 Jan 2025 18:42:13 +0100 Subject: [PATCH 4/5] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 98987be..62942e4 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ # Repository to host the Python documentation for OPM Flow ## Building the documentation locally -Follow the commands in .github/workflows/python_sphinx_docs.yml for your local setup! +Follow the commands in `.github/workflows/python_sphinx_docs.yml` for your local setup! -See the script [scripts/download_files.py](https://github.com/OPM/opm-python-documentation/blob/master/scripts/download_files.py) for more information. +See also the script [opmdoc-download-files](https://github.com/OPM/opm-python-documentation/blob/master/python/sphinx_docs/README.md) for more information. ## Building the documentation online on your fork - Turn on github actions at `https://github.com//opm-python-documentation/actions` From 5cd9f9d990a2333c0de867ee8bf4e502ff3701fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5kon=20H=C3=A6gland?= Date: Wed, 8 Jan 2025 20:45:41 +0100 Subject: [PATCH 5/5] Convert to CLI and click app --- .../src/opm_python_docs/download_files.py | 46 +++++++++++++------ 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/python/sphinx_docs/src/opm_python_docs/download_files.py b/python/sphinx_docs/src/opm_python_docs/download_files.py index c43a8ba..30f51c5 100755 --- a/python/sphinx_docs/src/opm_python_docs/download_files.py +++ b/python/sphinx_docs/src/opm_python_docs/download_files.py @@ -5,6 +5,8 @@ import subprocess from pathlib import Path +import click + URL_SIMULATORS = "https://raw.githubusercontent.com/OPM/opm-simulators/master/python/docstrings_simulators.json" URL_COMMON = "https://raw.githubusercontent.com/OPM/opm-common/master/python/docstrings_common.json" URL_DUNE_MODULE = "https://raw.githubusercontent.com/OPM/opm-simulators/master/dune.module" @@ -25,7 +27,7 @@ def get_git_root() -> Path: raise RuntimeError("Not in the opm-python-documentation repository.") return Path(root) -def convert_pr_to_commit_hash(repo: str, pr_number: str) -> str: +def convert_pr_to_commit_hash(repo: str, pr_number: int) -> str: """Convert a PR number to a commit hash.""" url = f"https://api.github.com/repos/OPM/{repo}/pulls/{pr_number}" response = requests.get(url) @@ -33,7 +35,7 @@ def convert_pr_to_commit_hash(repo: str, pr_number: str) -> str: commit_hash = response.json()["head"]["sha"] return commit_hash -def download_docstring_file(url: str) -> None: +def download_docstring_file(url: str, pr_number: int|None) -> None: """Download a docstrings file from a URL (either opm-simulators or opm-common).""" if "opm-simulators" in url: repo = "opm-simulators" @@ -41,14 +43,7 @@ def download_docstring_file(url: str) -> None: else: repo = "opm-common" filename = "docstrings_common.json" - print(f"Downloading docstrings file from {repo} repository. " - "Should we use the master branch or a PR branch?") - branch = input("Enter 'master' or a PR number: ") - if branch != "master": - pr_number = branch - if not pr_number.isdigit(): - print("Invalid PR number.") - return + if pr_number is not None: commit_hash = convert_pr_to_commit_hash(repo, pr_number) url = url.replace("/master/", f"/{commit_hash}/") logging.info(f"Downloading docstrings file from {url}") @@ -71,10 +66,35 @@ def download_dune_module() -> None: file.write(response.content) logging.info(f"Saved dune.module file to {save_path}") -def main() -> None: +# CLI command: opmdoc-download-files +# +# SHELL USAGE: +# +# opmdoc-download-files --opm-simulators --opm-common +# +# DESCRIPTION: +# +# Downloads the docstring JSON files from opm-simulators and opm-common. Also downloads +# the dune.module from opm-simulators. By default, the files are downloaded from the +# master branches. If a PR number is provided, the files are downloaded from the corresponding +# PR branch. +# +# EXAMPLES: +# +# opmdoc-download-files # Downloads the docstrings files and dune.module file from master branches +# +# opmdoc-download-files \ +# --opm-simulators 1234 \ +# --opm-common 5678 # Downloads the docstrings files from PR 1234 and 5678 and dune.module from master +# +# +@click.command() +@click.option("--opm-simulators", type=int, help="PR number for opm-simulators") +@click.option("--opm-common", type=int, help="PR number for opm-common") +def main(opm_simulators: int|None, opm_common: int|None) -> None: logging.basicConfig(level=logging.INFO) - download_docstring_file(URL_SIMULATORS) - download_docstring_file(URL_COMMON) + download_docstring_file(URL_SIMULATORS, pr_number=opm_simulators) + download_docstring_file(URL_COMMON, pr_number=opm_common) download_dune_module() if __name__ == '__main__':