Add data platform toolbox

pyproject.toml

@@ -38,6 +38,11 @@ dependencies = [
     "matplotlib>=3.8,<4.0",
     "dp-sdk",
     "aiocache",
+    "grpcio>=1.50.0",
+    "grpcio-tools>=1.50.0",
+    "grpc-requests>=0.1.17",
+    "betterproto>=2.0.0b7",
+    "pytest-asyncio>=1.3.0",
 ]
 
 [project.optional-dependencies]
@@ -65,13 +70,14 @@ dev-dependencies = [
     "pytest",
     "pytest-cov",
     "ruff",
+    "testcontainers>=4.9.0",
 ]
 index-url = "https://download.pytorch.org/whl/cpu"
 extra-index-url = ["https://pypi.org/simple"]
 
 [tool.uv.sources]
-dp-sdk = { url = "https://github.com/openclimatefix/data-platform/releases/download/v0.13.2/dp_sdk-0.13.2-py3-none-any.whl" }
 
+dp-sdk = { url = "https://github.com/openclimatefix/data-platform/releases/download/v0.16.0/dp_sdk-0.16.0-py3-none-any.whl" }
 [tool.pytest.ini_options]
 testpaths = ["tests"]
 python_files = ["test_*.py"]

src/dataplatform/toolbox/location.py

@@ -0,0 +1,178 @@
+"""Location management section for the Data Platform Toolbox"""
+
+import streamlit as st
+import json
+from dp_sdk.ocf import dp
+from grpclib.exceptions import GRPCError
+
+
+async def locations_section(data_client):
+    """Location management section."""
+
+    # Energy source and location type mappings
+    ENERGY_SOURCES = {
+        "UNSPECIFIED": dp.EnergySource.UNSPECIFIED,
+        "SOLAR": dp.EnergySource.SOLAR,
+        "WIND": dp.EnergySource.WIND,
+    }
+
+    LOCATION_TYPES = {
+        "UNSPECIFIED": dp.LocationType.UNSPECIFIED,
+        "SITE": dp.LocationType.SITE,
+        "GSP": dp.LocationType.GSP,
+        "DNO": dp.LocationType.DNO,
+        "NATION": dp.LocationType.NATION,
+        "STATE": dp.LocationType.STATE,
+        "COUNTY": dp.LocationType.COUNTY,
+        "CITY": dp.LocationType.CITY,
+        "PRIMARY SUBSTATION": dp.LocationType.PRIMARY_SUBSTATION,
+    }
+
+    # List Locations
+    st.markdown(
+        '<h2 style="color:#63BCAF;font-size:32px;">List Locations</h2>',
+        unsafe_allow_html=True,
+    )
+    with st.expander("Filter options"):
+        energy_source_filter = st.selectbox(
+            "Energy Source", list(ENERGY_SOURCES.keys()), key="list_loc_energy"
+        )
+        location_type_filter = st.selectbox(
+            "Location Type", list(LOCATION_TYPES.keys()), key="list_loc_type"
+        )
+        user_filter = st.text_input(
+            "Filter by User OAuth ID (optional)",
+            key="list_loc_user",
+            help="Leave empty to show all locations",
+        )
+    if st.button("List Locations", key="list_locations_button"):
+        if not data_client:
+            st.error("❌ Could not connect to Data Platform")
+        else:
+            try:
+                request = dp.ListLocationsRequest()
+                if energy_source_filter != "UNSPECIFIED":
+                    request.energy_source_filter = ENERGY_SOURCES[energy_source_filter]
+                if location_type_filter != "UNSPECIFIED":
+                    request.location_type_filter = LOCATION_TYPES[location_type_filter]
+                if user_filter:
+                    request.user_oauth_id_filter = user_filter
+
+                response = await data_client.list_locations(request)
+                locations = response.locations
+
+                if locations:
+                    st.success(f"✅ Found {len(locations)} location(s)")
+                    loc_dicts = [loc.to_dict() for loc in locations]
+                    st.write(loc_dicts)
+                else:
+                    st.info("No locations found with the specified filters")
+            except GRPCError as e:
+                st.error(
+                    f"❌ gRPC Error: {e.message}"
+                )
+            except Exception as e:
+                st.error(f"❌ Error listing locations: {str(e)}")
+
+    # Get Location Details
+    st.markdown(
+        '<h2 style="color:#63BCAF;font-size: 32px;">Get Location Details</h2>',
+        unsafe_allow_html=True,
+    )
+    loc_uuid = st.text_input("Location UUID", key="get_loc_uuid")
+    loc_energy = st.selectbox(
+        # to avoid defaulting to UNSPECIFIED
+        "Energy Source", list(ENERGY_SOURCES.keys())[1:], key="get_loc_energy"
+    )
+    include_geometry = st.checkbox("Include Geometry", key="get_loc_geom")
+    if st.button("Get Location Details", key="get_location_button"):
+        if not loc_uuid.strip():
+            st.warning("⚠️ Please enter a location UUID")
+        elif not data_client:
+            st.error("❌ Could not connect to Data Platform")
+        else:
+            try:
+                response = await data_client.get_location(
+                    dp.GetLocationRequest(
+                        location_uuid=loc_uuid,
+                        energy_source=ENERGY_SOURCES[loc_energy],
+                        include_geometry=include_geometry,
+                    )
+                )
+                response_dict = response.to_dict()
+                st.success(f"✅ Found location: {loc_uuid}")
+                st.write(response_dict)
+            except GRPCError as e:
+                st.error(
+                    f"❌ gRPC Error: {e.message}"
+                )
+            except Exception as e:
+                st.error(f"❌ Error fetching location: {str(e)}")
+
+    # Create Location
+    st.markdown(
+        '<h2 style="color:#7bcdf3;font-size:32px;">Create Location</h2>',
+        unsafe_allow_html=True,
+    )
+    with st.expander("Create new location"):
+        loc_name = st.text_input("Location Name *", key="create_loc_name")
+        loc_energy_src = st.selectbox(
+            # to avoid defaulting to UNSPECIFIED
+            "Energy Source *", list(ENERGY_SOURCES.keys())[1:], key="create_loc_energy"
+        )
+        loc_type = st.selectbox(
+            # to avoid defaulting to UNSPECIFIED
+            "Location Type *", list(LOCATION_TYPES.keys())[1:], key="create_loc_type"
+        )
+        geometry_wkt = st.text_input(
+            "Geometry (WKT) *",
+            placeholder="POINT(-0.127 51.507)",
+            key="create_loc_geom",
+            help="Enter location geometry in WKT format (e.g., POINT(lon lat))",
+        )
+        capacity_watts = st.number_input(
+            "Effective Capacity (Watts) *",
+            min_value=0,
+            key="create_loc_cap",
+            help="Enter the effective capacity in watts",
+        )
+        loc_metadata = st.text_area(
+            "Metadata (JSON)",
+            value="{}",
+            key="create_loc_metadata",
+            help="Enter valid JSON for location metadata",
+        )
+
+        if st.button("Create Location", key="create_location_button"):
+            if not data_client:
+                st.error("❌ Could not connect to Data Platform")
+            elif (
+                not loc_name.strip() or not geometry_wkt.strip() or capacity_watts <= 0
+            ):
+                st.warning("⚠️ Please fill in all required fields (*)")
+            else:
+                try:
+                    # Parse metadata JSON
+                    metadata = json.loads(loc_metadata) if loc_metadata.strip() else {}
+                    response = await data_client.create_location(
+                        dp.CreateLocationRequest(
+                            location_name=loc_name,
+                            energy_source=ENERGY_SOURCES.get(loc_energy_src, 1),
+                            location_type=LOCATION_TYPES.get(loc_type, 1),
+                            geometry_wkt=geometry_wkt,
+                            effective_capacity_watts=int(capacity_watts),
+                            metadata=metadata,
+                        )
+                    )
+                    response_dict = response.to_dict()
+                    st.success(f"✅ Location '{loc_name}' created successfully!")
+                    st.write(response_dict)
+
+                except json.JSONDecodeError:
+                    st.error("❌ Invalid JSON in metadata field")
+                except GRPCError as e:
+                    st.error(
+                        f"❌ gRPC Error: {e.message}"
+                    )
+                except Exception as e:
+                    st.error(f"❌ Error creating location: {str(e)}")

src/dataplatform/toolbox/main.py

@@ -0,0 +1,69 @@
+"""Data Platform Toolbox Streamlit Page Main Code."""
+
+import asyncio
+from grpclib.client import Channel
+import streamlit as st
+from dataplatform.toolbox.organisation import organisation_section
+from dataplatform.toolbox.users import users_section
+from dataplatform.toolbox.user_organisation import user_organisation_section
+from dataplatform.toolbox.location import locations_section
+from dataplatform.toolbox.policy import policies_section
+from dp_sdk.ocf import dp
+import os
+
+# Color scheme (matching existing toolbox)
+# teal:  #63BCAF (Get operations)
+# blue: #7bcdf3 (Create operations)
+# yellow: #ffd053 (Update operations)
+# red: #E63946 (Delete operations)
+# orange: #FF9736 (Info sections)
+
+
+def dataplatform_toolbox_page() -> None:
+    """Wrapper function that is not async to call the main async function."""
+    asyncio.run(async_dataplatform_toolbox_page())
+
+
+async def async_dataplatform_toolbox_page():
+    """Async Main function for the Data Platform Toolbox Streamlit page."""
+    host = os.environ.get("DATA_PLATFORM_HOST", "localhost")
+    port = os.environ.get("DATA_PLATFORM_PORT", "50051")
+    async with Channel(host=host, port=int(port)) as channel:
+        admin_client = dp.DataPlatformAdministrationServiceStub(channel)
+        data_client = dp.DataPlatformDataServiceStub(channel)
+
+        st.markdown(
+            '<h1 style="color:#63BCAF;font-size:48px;">Data Platform Toolbox</h1>',
+            unsafe_allow_html=True,
+        )
+
+        # Create tabs for different sections
+        tab1, tab2, tab3, tab4, tab5 = st.tabs(
+            [
+                "🏢 Organisations",
+                "👤 Users",
+                "🔗 User + Organisation",
+                "📍 Locations",
+                "📋 Policies",
+            ]
+        )
+
+        with tab1:
+            await organisation_section(admin_client)
+
+        with tab2:
+            await users_section(admin_client)
+
+        with tab3:
+            await user_organisation_section(admin_client)
+
+        with tab4:
+            await locations_section(data_client)
+
+        with tab5:
+            await policies_section(admin_client, data_client)
+
+
+# Required for the tests to run this as a script
+if __name__ == "__main__":
+    dataplatform_toolbox_page()

src/dataplatform/toolbox/organisation.py

@@ -0,0 +1,119 @@
+"""Organisation management section for the Data Platform Toolbox."""
+
+import streamlit as st
+import json
+from dp_sdk.ocf import dp
+from grpclib.exceptions import GRPCError
+
+
+async def organisation_section(admin_client):
+    """Organisation management section."""
+
+    # Get Organisation Details
+    st.markdown(
+        '<h2 style="color:#63BCAF;font-size: 32px;">Get Organisation Details</h2>',
+        unsafe_allow_html=True,
+    )
+    org_name = st.text_input("Organisation Name", key="get_org_name")
+    if st.button("Get Organisation Details", key="get_org_button"):
+        if not admin_client:
+            st.error("❌ Could not connect to Data Platform")
+        elif not org_name.strip():
+            st.warning("⚠️ Please enter an organisation name")
+        else:
+            try:
+                response = await admin_client.get_organisation(
+                    dp.GetOrganisationRequest(org_name=org_name)
+                )
+                response_dict = response.to_dict()
+                st.success(f"✅ Found organisation: {org_name}")
+                st.write(response_dict)
+
+            except GRPCError as e:
+                st.error(
+                    f"❌ gRPC Error: {e.message}"
+                )
+            except Exception as e:
+                st.error(f"❌ Error fetching organisation: {str(e)}")
+
+    # Create Organisation
+    st.markdown(
+        '<h2 style="color:#7bcdf3;font-size: 32px;">Create Organisation</h2>',
+        unsafe_allow_html=True,
+    )
+    with st.expander("Create new organisation"):
+        new_org_name = st.text_input("New Organisation Name", key="create_org_name")
+        metadata_json = st.text_area(
+            "Metadata (JSON)",
+            value="{}",
+            key="create_org_metadata",
+            help="Enter valid JSON for organisation metadata",
+        )
+
+        if st.button("Create Organisation", key="create_org_button"):
+            if not admin_client:
+                st.error("❌ Could not connect to Data Platform")
+            elif not new_org_name.strip():
+                st.warning("⚠️ Please enter an organisation name")
+            else:
+                try:
+                    # Parse metadata JSON
+                    metadata = (
+                        json.loads(metadata_json) if metadata_json.strip() else {}
+                    )
+                    response = await admin_client.create_organisation(
+                        dp.CreateOrganisationRequest(
+                            org_name=new_org_name, metadata=metadata
+                        )
+                    )
+                    response_dict = response.to_dict()
+                    st.success(
+                        f"✅ Organisation '{new_org_name}' created successfully!"
+                    )
+                    st.write(response_dict)
+
+                except json.JSONDecodeError:
+                    st.error("❌ Invalid JSON in metadata field")
+                except GRPCError as e:
+                    st.error(
+                        f"❌ gRPC Error: {e.message}"
+                    )
+                except Exception as e:
+                    st.error(f"❌ Error creating organisation: {str(e)}")
+
+    # Delete Organisation
+    st.markdown(
+        '<h2 style="color:#E63946;font-size:32px;">Delete Organisation</h2>',
+        unsafe_allow_html=True,
+    )
+    with st.expander("Delete organisation"):
+        del_org_name = st.text_input(
+            "Organisation Name to Delete", key="delete_org_name"
+        )
+        st.warning("⚠️ This action cannot be undone!")
+        confirm_delete = st.checkbox(
+            "I understand this will permanently delete the organisation",
+            key="confirm_delete_org",
+        )
+        if st.button("Delete Organisation", key="delete_org_button"):
+            if not admin_client:
+                st.error("❌ Could not connect to Data Platform")
+            elif not del_org_name.strip():
+                st.warning("⚠️ Please enter an organisation name")
+            elif not confirm_delete:
+                st.warning("⚠️ Please confirm deletion by checking the box above")
+            else:
+                try:
+                    await admin_client.delete_organisation(
+                        dp.DeleteOrganisationRequest(org_name=del_org_name)
+                    )
+                    st.success(
+                        f"✅ Organisation '{del_org_name}' deleted successfully!"
+                    )
+
+                except GRPCError as e:
+                    st.error(
+                        f"❌ gRPC Error: {e.message}"
+                    )
+                except Exception as e:
+                    st.error(f"❌ Error deleting organisation: {str(e)}")