Refactor the data publication pipeline for KG to the cloud pipeline

libs/matrix-gcp-datasets/src/matrix_gcp_datasets/huggingface.py

@@ -7,6 +7,7 @@
 
 import requests
 from datasets import load_dataset
+from huggingface_hub import create_tag
 from kedro.io import AbstractDataset
 from pydantic import BaseModel, Field, ValidationError
 from pyspark.sql import SparkSession
@@ -232,7 +233,6 @@ def _verify_stream_load(self, dataset_id, revision="main", token: str | None = N
 
     def _create_tag(self, dataset_id: str, revision: str, tag: str, token: str | None = None) -> None:
         """Create a Git tag on the HuggingFace Hub dataset repository."""
-        from huggingface_hub import create_tag
 
         log.info(f"Creating tag '{tag}' on {dataset_id} at {revision}")
         create_tag(

pipelines/matrix/conf/base/data_publication/catalog.yml → pipelines/matrix/conf/cloud/data_publication/catalog.yml

@@ -5,10 +5,12 @@ data_publication.prm.kg_edges_hf_published:
   split: train
   dataframe_type: spark
   data_dir: data/edges
+  tag: ${globals:versions.release}
 
 data_publication.prm.kg_nodes_hf_published:
   type: matrix_gcp_datasets.huggingface.HFIterableDataset
   repo_id: everycure/kg-nodes
   split: train
   dataframe_type: spark
   data_dir: data/nodes
+  tag: ${globals:versions.release}