fix ci

.github/frictionless-validate-failure.md

@@ -0,0 +1,4 @@
+---
+title: Frictionless validate failure on {{ date | date() }}
+labels: bug
+---

.github/workflows/pypi-publish.yml

@@ -0,0 +1,33 @@
+# NOTE-1: This action builds the py pkg as per a pixi command `pixi run build`
+#         and publishes it to PyPI. Requires org / repo / action / env setup
+#         as a trusted publisher.
+# NOTE-2: this can be copy and pasted between repos without edit
+#         provided that the repo name and pypi pkg name are the same
+name: Publish to PyPI
+
+on:
+  push:
+    tags:
+      - '*'
+      - '!rc*'  # Exclude release candidates
+
+jobs:
+
+  pypi-publish:
+    name: Upload release to PyPI
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: prefix-dev/setup-pixi@v0.9.3
+        with:
+            pixi-version: v0.59.0
+            cache: true
+            cache-write: ${{ github.event_name == 'push' && github.ref_name == 'main' }}
+      - run: pixi run build
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+    environment:
+      name: pypi
+      url: https://pypi.org/p/${{ github.action.repository }}
+    permissions:
+      id-token: write  # IMPORTANT: this permission is mandatory for trusted publishing

.github/workflows/test-and-validate.yml

@@ -0,0 +1,31 @@
+name: package
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  pypi-publish:
+    name: Run tests and validate data
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: prefix-dev/setup-pixi@v0.9.3
+        with:
+          pixi-version: v0.59.0
+          cache: true
+          cache-write: ${{ github.event_name == 'push' && github.ref_name == 'main' }}
+      - run: pixi run tests
+      - run: pixi run frictionless-validate
+      # TODO
+      # - name: Workflow failure notification
+      #   uses: JasonEtco/create-an-issue@v2
+      #   env:
+      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      #   if: failure()
+      #   with:
+      #     filename: .github/frictionless-validate-failure.md

.gitignore

@@ -1,3 +1,7 @@
 # pixi environments
 .pixi/*
 !.pixi/config.toml
+src/netzero_metrics_reference_data/data
+src/netzero_metrics_reference_data/__pycache__
+tests/__pycache__
+dist

.vscode/settings.json

@@ -1,4 +1,10 @@
 {
-    "python-envs.defaultEnvManager": "ms-python.python:system",
-    "python-envs.pythonProjects": []
-}
+  "python-envs.defaultEnvManager": "renan-r-santos.pixi-code:pixi",
+  "python-envs.pythonProjects": [],
+  "python-envs.defaultPackageManager": "renan-r-santos.pixi-code:pixi",
+  "python.testing.pytestArgs": [
+    "tests"
+  ],
+  "python.testing.unittestEnabled": false,
+  "python.testing.pytestEnabled": true
+}

README.md

@@ -0,0 +1,9 @@
+# NetZero Metrics Reference Data
+
+This repository provides reference data and resources for NetZero metrics calculations.
+
+The data is provided as a [datapackage](https://datapackage.org/) directly from the root of this repo. It has also been packaged as a Python package, which can be installed via pip:
+
+```bash
+pip install netzero-metrics-reference-data
+```

pyproject.toml

@@ -0,0 +1,33 @@
+[project]
+authors = [{name = "jgunstone", email = "j.gunstone@maxfordham.com"}]
+dependencies = ["frictionless>=5.18.1,<6"]
+name = "netzero-metrics-reference-data"
+requires-python = ">= 3.11"
+version = "0.1.0"
+
+[build-system]
+build-backend = "hatchling.build"
+requires = ["hatchling"]
+
+[tool.pixi.workspace]
+channels = ["conda-forge"]
+platforms = ["linux-64"]
+
+[tool.pixi.pypi-dependencies]
+netzero_metrics_reference_data = { path = ".", editable = true }
+
+[tool.pixi.dependencies]
+stringcase = ">=1.2.0,<2"
+openpyxl = ">=3.1.5,<4"
+pandas = ">=2.3,<3"
+pytest = ">=7.2.2,<8"
+hatch = ">=1.0.0,<2"
+casefy = ">=1.0.0,<2"
+
+[tool.pixi.tasks]
+get-eui = { cmd = "python get_eui.py", cwd = "scripts" }
+create-datapackage = { cmd = "python create_datapackage.py", cwd = "scripts" }
+cp-data = { cmd = "python scripts/cp_data.py"} #, cwd = "scripts"
+tests = {cmd = "pytest", depends-on = ["cp-data"] }
+build = "hatch build"
+frictionless-validate = { cmd = "frictionless validate datapackage.yaml" }

scripts/cp_data.py

@@ -0,0 +1,18 @@
+import shutil
+from pathlib import Path
+
+# Define source and destination directories
+repo_root = Path(__file__).resolve().parent.parent
+destination_dir = repo_root / "src" / "netzero_metrics_reference_data" / "data"
+
+# Ensure destination directory exists
+destination_dir.mkdir(parents=True, exist_ok=True)
+
+# List of file extensions to copy
+extensions = [".csv", ".json", ".yaml", ".txt"]
+
+# Copy matching files from repo root to destination
+for file_path in repo_root.iterdir():
+    if file_path.is_file() and any(file_path.name.endswith(ext) for ext in extensions):
+        dst_path = destination_dir / file_path.name
+        shutil.copy2(file_path, dst_path)

scripts/get_eui.py

@@ -1,13 +1,14 @@
 import pandas as pd
 import pathlib
-import stringcase
+import casefy
 
 CWD = pathlib.Path(__file__).parent.parent
 PTH_EUI_IN = CWD / "scripts" / "energy-use-intensity.xlsx"
-DIR_OUT = CWD 
+DIR_OUT = CWD
 PTH_EUI = DIR_OUT / "energy-use-intensity.csv"
 PTH_BUILDING_TYPES = DIR_OUT / "building-types.txt"
 
+
 def get_sheet_data(PTH, sheet_name):
     construction_deleivery_type = sheet_name
     meta = pd.read_excel(PTH, sheet_name=sheet_name, nrows=2)
@@ -16,7 +17,9 @@ def get_sheet_data(PTH, sheet_name):
         .T.reset_index(drop=False)
         .rename(columns={"year": "building-type", "index": "building-type-shorthand"})
     )
-    col_index_w_gia = [0] + [n+1 for n,x in enumerate(meta.unit) if "GIA" in x] # +1 as yr is first column
+    col_index_w_gia = [0] + [
+        n + 1 for n, x in enumerate(meta.unit) if "GIA" in x
+    ]  # +1 as yr is first column
     data = pd.read_excel(PTH, sheet_name=sheet_name, skiprows=2)
     data = data.iloc[:, col_index_w_gia]
     if not data.columns.is_unique:
@@ -26,29 +29,37 @@ def get_sheet_data(PTH, sheet_name):
         data, id_vars="year", var_name="building-type", value_name="benchmark-target"
     )
 
-    data = data.join(meta.set_index('building-type'), on="building-type")
+    data = data.join(meta.set_index("building-type"), on="building-type")
     data["construction-delivery-type"] = construction_deleivery_type
-    cols = ["building-type","building-type-shorthand","unit","construction-delivery-type"]
+    cols = [
+        "building-type",
+        "building-type-shorthand",
+        "unit",
+        "construction-delivery-type",
+    ]
     for x in cols:
         data[x] = data[x].str.strip()
 
     for x in [" (GIA)", " (NIA)"]:
         data["building-type"] = data["building-type"].str.replace(x, "")
 
     li = [" ".join(x.split("-")) for x in data.columns]
-    data.columns = [stringcase.pascalcase(stringcase.snakecase(x)) for x in li]
+    data.columns = [casefy.pascalcase(casefy.snakecase(x)) for x in li]
     return data
 
 
 def get_eui_data():
     sheet_names = ["newbuild", "retrofit-in-one-go", "retrofit-stepped"]
-    return pd.concat([get_sheet_data(PTH_EUI_IN, sheet_name=s) for s in sheet_names], axis=0) 
+    return pd.concat(
+        [get_sheet_data(PTH_EUI_IN, sheet_name=s) for s in sheet_names], axis=0
+    )
+
 
 def get_building_types(df_eui):
     return list(df_eui["BuildingType"].unique())
 
+
 df_eui = get_eui_data()
 df_eui.to_csv(PTH_EUI, index=False)
 
 PTH_BUILDING_TYPES.write_text("\n".join(get_building_types(df_eui)))
-

src/netzero_metrics_reference_data/__init__.py

@@ -0,0 +1,11 @@
+from importlib.resources import files
+from frictionless import Package
+
+PTH_PKG = (
+    files("netzero_metrics_reference_data")
+    .joinpath("data")
+    .joinpath("datapackage.yaml")
+)
+nzm_pkg = Package(PTH_PKG)
+
+__all__ = ["nzm_pkg"]

tests/test_netzero_metrics_reference_data.py

@@ -0,0 +1,16 @@
+from netzero_metrics_reference_data import nzm_pkg
+
+
+def test_nzm_pkg():
+    assert nzm_pkg is not None
+    li = [
+        "energy-use-intensity",
+        "building-types",
+        "life-cycle-modules",
+        "rics-building-element-category",
+        "color-energy-end-use",
+        "color-fuel-type",
+    ]
+    assert nzm_pkg.resource_names == li
+
+    print("done")