From 5f82cbca3563b26f99a26e2f1ef30ea6b3c0a858 Mon Sep 17 00:00:00 2001 From: Christian Beilschmidt Date: Thu, 6 Nov 2025 11:20:02 +0100 Subject: [PATCH 1/9] ci: add notebook execution to coverage report --- .github/workflows/test-python.yml | 16 ++++++------- examples/data_usage.ipynb | 31 +++++++++++++----------- test_all_notebooks.py | 10 +++++--- test_notebook.py | 40 ++++++++++++++++++++++++------- 4 files changed, 63 insertions(+), 34 deletions(-) diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index 0ce0ccbf..5f26c55e 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -103,6 +103,14 @@ jobs: env: GEOENGINE_TEST_CODE_PATH: ${{ github.workspace }}/backend GEOENGINE_TEST_BUILD_TYPE: "release" + - name: Examples + run: | + ${{ steps.vars.outputs.VENV_CALL }} + ${{ steps.vars.outputs.PIP_INSTALL }} -e .[examples] + python test_all_notebooks.py + env: + GEOENGINE_TEST_CODE_PATH: ${{ github.workspace }}/backend + GEOENGINE_TEST_BUILD_TYPE: "release" - name: Report coverage to Coveralls if: ${{ inputs.coverage }} # 1. We need to adjust the paths in the lcov file to match the repository structure. @@ -114,11 +122,3 @@ jobs: working-directory: library/geoengine env: COVERALLS_REPO_TOKEN: ${{ github.token }} - - name: Examples - run: | - ${{ steps.vars.outputs.VENV_CALL }} - ${{ steps.vars.outputs.PIP_INSTALL }} -e .[examples] - python test_all_notebooks.py - env: - GEOENGINE_TEST_CODE_PATH: ${{ github.workspace }}/backend - GEOENGINE_TEST_BUILD_TYPE: "release" diff --git a/examples/data_usage.ipynb b/examples/data_usage.ipynb index 163c7e7f..04944fb0 100644 --- a/examples/data_usage.ipynb +++ b/examples/data_usage.ipynb @@ -16,7 +16,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -329,21 +329,24 @@ } ], "source": [ - "df = ge.data_usage_summary(ge.UsageSummaryGranularity.MINUTES)\n", - "if df.empty:\n", - " print(\"No data usage found\")\n", - " exit()\n", - "df[\"timestamp\"] = pd.to_datetime(df[\"timestamp\"]).dt.tz_localize(None)\n", + "def plot_summary():\n", + " df = ge.data_usage_summary(ge.UsageSummaryGranularity.MINUTES)\n", + " if df.empty:\n", + " print(\"No data usage found\")\n", + " return\n", + " df[\"timestamp\"] = pd.to_datetime(df[\"timestamp\"]).dt.tz_localize(None)\n", + "\n", + " pivot_df = df.pivot(index=\"timestamp\", columns=\"data\", values=\"count\").fillna(0)\n", + " pivot_df.plot(kind=\"bar\", figsize=(10, 6))\n", "\n", - "pivot_df = df.pivot(index=\"timestamp\", columns=\"data\", values=\"count\").fillna(0)\n", - "pivot_df.plot(kind=\"bar\", figsize=(10, 6))\n", + " plt.title(\"Data Usage by Data over time\")\n", + " plt.xlabel(\"Timestamp\")\n", + " plt.ylabel(\"Count\")\n", + " plt.xticks(rotation=45)\n", + " plt.legend(title=\"Data\")\n", + " plt.show()\n", "\n", - "plt.title(\"Data Usage by Data over time\")\n", - "plt.xlabel(\"Timestamp\")\n", - "plt.ylabel(\"Count\")\n", - "plt.xticks(rotation=45)\n", - "plt.legend(title=\"Data\")\n", - "plt.show()" + "plot_summary()" ] } ], diff --git a/test_all_notebooks.py b/test_all_notebooks.py index e10a7bb8..0f19e6ee 100755 --- a/test_all_notebooks.py +++ b/test_all_notebooks.py @@ -16,13 +16,17 @@ def eprint(*args, **kwargs): def run_test_notebook(notebook_path) -> bool: """Run test_notebook.py for the given notebook.""" - python_bin = shutil.which("python3") + pytest_bin = shutil.which("pytest") - if python_bin is None: + if pytest_bin is None: raise RuntimeError("Python 3 not found") result = subprocess.run( - [python_bin, "test_notebook.py", notebook_path], + [pytest_bin, "--ignore=test", "--cov", "--cov-append", "test_notebook.py"], + env={ + **os.environ, + "INPUT_FILE": notebook_path, + }, capture_output=True, text=True, check=False, diff --git a/test_notebook.py b/test_notebook.py index aa9a82be..cc892a47 100755 --- a/test_notebook.py +++ b/test_notebook.py @@ -4,6 +4,7 @@ import argparse import ast +import os import sys import warnings @@ -46,7 +47,8 @@ def convert_to_python(input_file: str) -> str: def run_script(script: str) -> bool: """Run the script.""" - code = compile(script, "", "exec", flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT) + code = compile(script, "", "exec", + flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT) try: # prevent interactive backend to pop up @@ -55,6 +57,8 @@ def run_script(script: str) -> bool: with warnings.catch_warnings(record=True): # pylint: disable-next=exec-used exec(code, {}) + # pytest.main(["-p", "no:warnings", "-c", "pytest.ini", + # "--tb=short", "-"], plugins=[], args=[], obj=code) eprint("SUCCESS") return True @@ -64,22 +68,40 @@ def run_script(script: str) -> bool: return False +def setup_geoengine_and_run_script(input_file: str) -> bool: + """Setup Geo Engine test instance and run the script.""" + python_script = convert_to_python(input_file) + + eprint(f"Running script `{input_file}`", end=": ") + + with GeoEngineTestInstance(port=3030) as ge_instance: + ge_instance.wait_for_ready() + + return run_script(python_script) + + def main(): """Main entry point.""" input_file = parse_args() - python_script = convert_to_python(input_file) + if setup_geoengine_and_run_script(input_file): + sys.exit(0) + else: + sys.exit(1) - eprint(f"Running script `{input_file}`", end=": ") - with GeoEngineTestInstance(port=3030) as ge_instance: - ge_instance.wait_for_ready() +def test_main(): + """Run main function with pytest""" + input_file = os.getenv("INPUT_FILE") + + if not input_file: + assert False, "INPUT_FILE environment variable not set" - if run_script(python_script): - sys.exit(0) - else: - sys.exit(1) + if setup_geoengine_and_run_script(input_file): + assert True, "Notebook ran successfully" + else: + assert False, "Notebook failed" if __name__ == "__main__": From 452f145a85865de6144c70453c6a566441cf43d4 Mon Sep 17 00:00:00 2001 From: Christian Beilschmidt Date: Thu, 6 Nov 2025 11:28:06 +0100 Subject: [PATCH 2/9] format --- examples/data_usage.ipynb | 1 + test_notebook.py | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/data_usage.ipynb b/examples/data_usage.ipynb index 04944fb0..76455e12 100644 --- a/examples/data_usage.ipynb +++ b/examples/data_usage.ipynb @@ -346,6 +346,7 @@ " plt.legend(title=\"Data\")\n", " plt.show()\n", "\n", + "\n", "plot_summary()" ] } diff --git a/test_notebook.py b/test_notebook.py index cc892a47..0d7cb4f7 100755 --- a/test_notebook.py +++ b/test_notebook.py @@ -47,8 +47,7 @@ def convert_to_python(input_file: str) -> str: def run_script(script: str) -> bool: """Run the script.""" - code = compile(script, "", "exec", - flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT) + code = compile(script, "", "exec", flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT) try: # prevent interactive backend to pop up From 6b3c3b33ecd9960da30019c7381964dbddfe81fe Mon Sep 17 00:00:00 2001 From: Christian Beilschmidt Date: Thu, 6 Nov 2025 11:31:38 +0100 Subject: [PATCH 3/9] lints --- test_notebook.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test_notebook.py b/test_notebook.py index 0d7cb4f7..899b9c9e 100755 --- a/test_notebook.py +++ b/test_notebook.py @@ -47,7 +47,8 @@ def convert_to_python(input_file: str) -> str: def run_script(script: str) -> bool: """Run the script.""" - code = compile(script, "", "exec", flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT) + code = compile(script, "", "exec", + flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT) try: # prevent interactive backend to pop up @@ -95,12 +96,12 @@ def test_main(): input_file = os.getenv("INPUT_FILE") if not input_file: - assert False, "INPUT_FILE environment variable not set" + raise AssertionError("INPUT_FILE environment variable not set") if setup_geoengine_and_run_script(input_file): assert True, "Notebook ran successfully" else: - assert False, "Notebook failed" + raise AssertionError("Notebook failed") if __name__ == "__main__": From 3f7ca17a052338cd60ddb3ac9e4ba7c6457df811 Mon Sep 17 00:00:00 2001 From: Christian Beilschmidt Date: Thu, 6 Nov 2025 11:33:38 +0100 Subject: [PATCH 4/9] format --- test_notebook.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test_notebook.py b/test_notebook.py index 899b9c9e..1bede3b9 100755 --- a/test_notebook.py +++ b/test_notebook.py @@ -47,8 +47,7 @@ def convert_to_python(input_file: str) -> str: def run_script(script: str) -> bool: """Run the script.""" - code = compile(script, "", "exec", - flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT) + code = compile(script, "", "exec", flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT) try: # prevent interactive backend to pop up From 9b54bf6315cac1b7621c062c7e15a1c329bb3c6b Mon Sep 17 00:00:00 2001 From: Christian Beilschmidt Date: Thu, 6 Nov 2025 11:45:03 +0100 Subject: [PATCH 5/9] use correct ci script --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 51323ed4..a88e0aff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ on: jobs: check: - uses: geo-engine/geoengine-python/.github/workflows/test-python.yml@main + uses: geo-engine/geoengine-python/.github/workflows/test-python.yml@notebooks-in-coverage strategy: fail-fast: false @@ -28,7 +28,7 @@ jobs: # Checks the library using minimum version resolution # `uv` has this feature built-in, c.f. https://github.com/astral-sh/uv check-min-version: - uses: geo-engine/geoengine-python/.github/workflows/test-python.yml@main + uses: geo-engine/geoengine-python/.github/workflows/test-python.yml@notebooks-in-coverage with: python-version: "3.10" From 04f18281955baabd5d4c5132033a126bf0c894db Mon Sep 17 00:00:00 2001 From: Christian Beilschmidt Date: Thu, 6 Nov 2025 13:55:25 +0100 Subject: [PATCH 6/9] don't run test_*_notebook files on pytest --- pyproject.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 6c86b5f7..89a3b7e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,3 +96,8 @@ select = [ "tests/__init__.py" = [ "F401", # module imported but unused ] + +[tool.pytest.ini_options] +testpaths = [ + "tests", +] From 5422cdf79b5069608ffba586dffb7bc937d73feb Mon Sep 17 00:00:00 2001 From: Christian Beilschmidt Date: Thu, 6 Nov 2025 14:55:14 +0100 Subject: [PATCH 7/9] coverage args as env var --- .github/workflows/test-python.yml | 1 + test_all_notebooks.py | 22 +++++++++++++++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index 5f26c55e..e4632473 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -111,6 +111,7 @@ jobs: env: GEOENGINE_TEST_CODE_PATH: ${{ github.workspace }}/backend GEOENGINE_TEST_BUILD_TYPE: "release" + COVERAGE_COMMAND: ${{ steps.vars.outputs.COVERAGE_COMMAND }} - name: Report coverage to Coveralls if: ${{ inputs.coverage }} # 1. We need to adjust the paths in the lcov file to match the repository structure. diff --git a/test_all_notebooks.py b/test_all_notebooks.py index 0f19e6ee..4b802660 100755 --- a/test_all_notebooks.py +++ b/test_all_notebooks.py @@ -7,13 +7,15 @@ import subprocess import sys +COVERAGE_COMMAND_ENV_VAR = "COVERAGE_COMMAND" + def eprint(*args, **kwargs): """Print to stderr.""" print(*args, file=sys.stderr, **kwargs) -def run_test_notebook(notebook_path) -> bool: +def run_test_notebook(notebook_path: str, coverage_args: list[str]) -> bool: """Run test_notebook.py for the given notebook.""" pytest_bin = shutil.which("pytest") @@ -22,7 +24,7 @@ def run_test_notebook(notebook_path) -> bool: raise RuntimeError("Python 3 not found") result = subprocess.run( - [pytest_bin, "--ignore=test", "--cov", "--cov-append", "test_notebook.py"], + [pytest_bin, "--ignore=test", *coverage_args, "test_notebook.py"], env={ **os.environ, "INPUT_FILE": notebook_path, @@ -40,6 +42,14 @@ def run_test_notebook(notebook_path) -> bool: return False +def parse_coverage_command() -> list[str]: + """Get coverage command from environment variable.""" + coverage_cmd = os.getenv(COVERAGE_COMMAND_ENV_VAR) + if not coverage_cmd: + return [] + return [*coverage_cmd.split(), "--cov-append"] + + def main() -> int: """Run all Jupyter Notebooks and check for errors.""" @@ -49,13 +59,19 @@ def main() -> int: eprint(f"The folder {example_folder} does not exist.") return -1 + coverage_args = parse_coverage_command() + if coverage_args: + eprint(f"Using coverage args: {' '.join(coverage_args)}") + else: + eprint(f"No coverage args in env {COVERAGE_COMMAND_ENV_VAR} provided.") + for root, _dirs, files in os.walk(example_folder): for file in files: if not file.endswith(".ipynb"): eprint(f"Skipping non-notebook file {file}") continue notebook_path = os.path.join(root, file) - if not run_test_notebook(notebook_path): + if not run_test_notebook(notebook_path, coverage_args): return -1 break # skip subdirectories From 72ba85deb03c6080280e5455cdc4bd75045211f4 Mon Sep 17 00:00:00 2001 From: Christian Beilschmidt Date: Mon, 10 Nov 2025 10:55:01 +0100 Subject: [PATCH 8/9] point CI back to main --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a88e0aff..51323ed4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ on: jobs: check: - uses: geo-engine/geoengine-python/.github/workflows/test-python.yml@notebooks-in-coverage + uses: geo-engine/geoengine-python/.github/workflows/test-python.yml@main strategy: fail-fast: false @@ -28,7 +28,7 @@ jobs: # Checks the library using minimum version resolution # `uv` has this feature built-in, c.f. https://github.com/astral-sh/uv check-min-version: - uses: geo-engine/geoengine-python/.github/workflows/test-python.yml@notebooks-in-coverage + uses: geo-engine/geoengine-python/.github/workflows/test-python.yml@main with: python-version: "3.10" From c3380e98f0adc5fbd57b98ce8afef8f7df2d5113 Mon Sep 17 00:00:00 2001 From: Christian Beilschmidt Date: Mon, 10 Nov 2025 10:55:34 +0100 Subject: [PATCH 9/9] remove comments --- test_notebook.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test_notebook.py b/test_notebook.py index 1bede3b9..35c84767 100755 --- a/test_notebook.py +++ b/test_notebook.py @@ -56,8 +56,6 @@ def run_script(script: str) -> bool: with warnings.catch_warnings(record=True): # pylint: disable-next=exec-used exec(code, {}) - # pytest.main(["-p", "no:warnings", "-c", "pytest.ini", - # "--tb=short", "-"], plugins=[], args=[], obj=code) eprint("SUCCESS") return True