Skip to content

Commit 80dc0f6

Browse files
katyalmohitkatyalmohit
authored
Merge pull request #390 from openclimatefix/dataplatform-toolbox
Add data platform toolbox
2 parents e814ee6 + 66ca071 commit 80dc0f6

14 files changed

Lines changed: 1633 additions & 2 deletions

‎pyproject.toml‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,11 @@ dependencies = [
3838
"matplotlib>=3.8,<4.0",
3939
"dp-sdk",
4040
"aiocache",
41+
"grpcio>=1.50.0",
42+
"grpcio-tools>=1.50.0",
43+
"grpc-requests>=0.1.17",
44+
"betterproto>=2.0.0b7",
45+
"pytest-asyncio>=1.3.0",
4146
]
4247

4348
[project.optional-dependencies]
@@ -65,13 +70,14 @@ dev-dependencies = [
6570
"pytest",
6671
"pytest-cov",
6772
"ruff",
73+
"testcontainers>=4.9.0",
6874
]
6975
index-url = "https://download.pytorch.org/whl/cpu"
7076
extra-index-url = ["https://pypi.org/simple"]
7177

7278
[tool.uv.sources]
73-
dp-sdk = { url = "https://github.com/openclimatefix/data-platform/releases/download/v0.13.2/dp_sdk-0.13.2-py3-none-any.whl" }
7479

80+
dp-sdk = { url = "https://github.com/openclimatefix/data-platform/releases/download/v0.16.0/dp_sdk-0.16.0-py3-none-any.whl" }
7581
[tool.pytest.ini_options]
7682
testpaths = ["tests"]
7783
python_files = ["test_*.py"]

‎src/dataplatform/toolbox/location.py‎

Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
"""Location management section for the Data Platform Toolbox"""
2+
3+
import streamlit as st
4+
import json
5+
from dp_sdk.ocf import dp
6+
from grpclib.exceptions import GRPCError
7+
8+
9+
async def locations_section(data_client):
10+
"""Location management section."""
11+
12+
# Energy source and location type mappings
13+
ENERGY_SOURCES = {
14+
"UNSPECIFIED": dp.EnergySource.UNSPECIFIED,
15+
"SOLAR": dp.EnergySource.SOLAR,
16+
"WIND": dp.EnergySource.WIND,
17+
}
18+
19+
LOCATION_TYPES = {
20+
"UNSPECIFIED": dp.LocationType.UNSPECIFIED,
21+
"SITE": dp.LocationType.SITE,
22+
"GSP": dp.LocationType.GSP,
23+
"DNO": dp.LocationType.DNO,
24+
"NATION": dp.LocationType.NATION,
25+
"STATE": dp.LocationType.STATE,
26+
"COUNTY": dp.LocationType.COUNTY,
27+
"CITY": dp.LocationType.CITY,
28+
"PRIMARY SUBSTATION": dp.LocationType.PRIMARY_SUBSTATION,
29+
}
30+
31+
# List Locations
32+
st.markdown(
33+
'<h2 style="color:#63BCAF;font-size:32px;">List Locations</h2>',
34+
unsafe_allow_html=True,
35+
)
36+
with st.expander("Filter options"):
37+
energy_source_filter = st.selectbox(
38+
"Energy Source", list(ENERGY_SOURCES.keys()), key="list_loc_energy"
39+
)
40+
location_type_filter = st.selectbox(
41+
"Location Type", list(LOCATION_TYPES.keys()), key="list_loc_type"
42+
)
43+
user_filter = st.text_input(
44+
"Filter by User OAuth ID (optional)",
45+
key="list_loc_user",
46+
help="Leave empty to show all locations",
47+
)
48+
if st.button("List Locations", key="list_locations_button"):
49+
if not data_client:
50+
st.error("❌ Could not connect to Data Platform")
51+
else:
52+
try:
53+
request = dp.ListLocationsRequest()
54+
if energy_source_filter != "UNSPECIFIED":
55+
request.energy_source_filter = ENERGY_SOURCES[energy_source_filter]
56+
if location_type_filter != "UNSPECIFIED":
57+
request.location_type_filter = LOCATION_TYPES[location_type_filter]
58+
if user_filter:
59+
request.user_oauth_id_filter = user_filter
60+
61+
response = await data_client.list_locations(request)
62+
locations = response.locations
63+
64+
if locations:
65+
st.success(f"✅ Found {len(locations)} location(s)")
66+
loc_dicts = [loc.to_dict() for loc in locations]
67+
st.write(loc_dicts)
68+
else:
69+
st.info("No locations found with the specified filters")
70+
except GRPCError as e:
71+
st.error(
72+
f"❌ gRPC Error: {e.message}"
73+
)
74+
except Exception as e:
75+
st.error(f"❌ Error listing locations: {str(e)}")
76+
77+
# Get Location Details
78+
st.markdown(
79+
'<h2 style="color:#63BCAF;font-size: 32px;">Get Location Details</h2>',
80+
unsafe_allow_html=True,
81+
)
82+
loc_uuid = st.text_input("Location UUID", key="get_loc_uuid")
83+
loc_energy = st.selectbox(
84+
# to avoid defaulting to UNSPECIFIED
85+
"Energy Source", list(ENERGY_SOURCES.keys())[1:], key="get_loc_energy"
86+
)
87+
include_geometry = st.checkbox("Include Geometry", key="get_loc_geom")
88+
if st.button("Get Location Details", key="get_location_button"):
89+
if not loc_uuid.strip():
90+
st.warning("⚠️ Please enter a location UUID")
91+
elif not data_client:
92+
st.error("❌ Could not connect to Data Platform")
93+
else:
94+
try:
95+
response = await data_client.get_location(
96+
dp.GetLocationRequest(
97+
location_uuid=loc_uuid,
98+
energy_source=ENERGY_SOURCES[loc_energy],
99+
include_geometry=include_geometry,
100+
)
101+
)
102+
response_dict = response.to_dict()
103+
st.success(f"✅ Found location: {loc_uuid}")
104+
st.write(response_dict)
105+
except GRPCError as e:
106+
st.error(
107+
f"❌ gRPC Error: {e.message}"
108+
)
109+
except Exception as e:
110+
st.error(f"❌ Error fetching location: {str(e)}")
111+
112+
# Create Location
113+
st.markdown(
114+
'<h2 style="color:#7bcdf3;font-size:32px;">Create Location</h2>',
115+
unsafe_allow_html=True,
116+
)
117+
with st.expander("Create new location"):
118+
loc_name = st.text_input("Location Name *", key="create_loc_name")
119+
loc_energy_src = st.selectbox(
120+
# to avoid defaulting to UNSPECIFIED
121+
"Energy Source *", list(ENERGY_SOURCES.keys())[1:], key="create_loc_energy"
122+
)
123+
loc_type = st.selectbox(
124+
# to avoid defaulting to UNSPECIFIED
125+
"Location Type *", list(LOCATION_TYPES.keys())[1:], key="create_loc_type"
126+
)
127+
geometry_wkt = st.text_input(
128+
"Geometry (WKT) *",
129+
placeholder="POINT(-0.127 51.507)",
130+
key="create_loc_geom",
131+
help="Enter location geometry in WKT format (e.g., POINT(lon lat))",
132+
)
133+
capacity_watts = st.number_input(
134+
"Effective Capacity (Watts) *",
135+
min_value=0,
136+
key="create_loc_cap",
137+
help="Enter the effective capacity in watts",
138+
)
139+
loc_metadata = st.text_area(
140+
"Metadata (JSON)",
141+
value="{}",
142+
key="create_loc_metadata",
143+
help="Enter valid JSON for location metadata",
144+
)
145+
146+
if st.button("Create Location", key="create_location_button"):
147+
if not data_client:
148+
st.error("❌ Could not connect to Data Platform")
149+
elif (
150+
not loc_name.strip() or not geometry_wkt.strip() or capacity_watts <= 0
151+
):
152+
st.warning("⚠️ Please fill in all required fields (*)")
153+
else:
154+
try:
155+
# Parse metadata JSON
156+
metadata = json.loads(loc_metadata) if loc_metadata.strip() else {}
157+
response = await data_client.create_location(
158+
dp.CreateLocationRequest(
159+
location_name=loc_name,
160+
energy_source=ENERGY_SOURCES.get(loc_energy_src, 1),
161+
location_type=LOCATION_TYPES.get(loc_type, 1),
162+
geometry_wkt=geometry_wkt,
163+
effective_capacity_watts=int(capacity_watts),
164+
metadata=metadata,
165+
)
166+
)
167+
response_dict = response.to_dict()
168+
st.success(f"✅ Location '{loc_name}' created successfully!")
169+
st.write(response_dict)
170+
171+
except json.JSONDecodeError:
172+
st.error("❌ Invalid JSON in metadata field")
173+
except GRPCError as e:
174+
st.error(
175+
f"❌ gRPC Error: {e.message}"
176+
)
177+
except Exception as e:
178+
st.error(f"❌ Error creating location: {str(e)}")

‎src/dataplatform/toolbox/main.py‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
"""Data Platform Toolbox Streamlit Page Main Code."""
2+
3+
import asyncio
4+
from grpclib.client import Channel
5+
import streamlit as st
6+
from dataplatform.toolbox.organisation import organisation_section
7+
from dataplatform.toolbox.users import users_section
8+
from dataplatform.toolbox.user_organisation import user_organisation_section
9+
from dataplatform.toolbox.location import locations_section
10+
from dataplatform.toolbox.policy import policies_section
11+
from dp_sdk.ocf import dp
12+
import os
13+
14+
# Color scheme (matching existing toolbox)
15+
# teal: #63BCAF (Get operations)
16+
# blue: #7bcdf3 (Create operations)
17+
# yellow: #ffd053 (Update operations)
18+
# red: #E63946 (Delete operations)
19+
# orange: #FF9736 (Info sections)
20+
21+
22+
def dataplatform_toolbox_page() -> None:
23+
"""Wrapper function that is not async to call the main async function."""
24+
asyncio.run(async_dataplatform_toolbox_page())
25+
26+
27+
async def async_dataplatform_toolbox_page():
28+
"""Async Main function for the Data Platform Toolbox Streamlit page."""
29+
host = os.environ.get("DATA_PLATFORM_HOST", "localhost")
30+
port = os.environ.get("DATA_PLATFORM_PORT", "50051")
31+
async with Channel(host=host, port=int(port)) as channel:
32+
admin_client = dp.DataPlatformAdministrationServiceStub(channel)
33+
data_client = dp.DataPlatformDataServiceStub(channel)
34+
35+
st.markdown(
36+
'<h1 style="color:#63BCAF;font-size:48px;">Data Platform Toolbox</h1>',
37+
unsafe_allow_html=True,
38+
)
39+
40+
# Create tabs for different sections
41+
tab1, tab2, tab3, tab4, tab5 = st.tabs(
42+
[
43+
"🏢 Organisations",
44+
"👤 Users",
45+
"🔗 User + Organisation",
46+
"📍 Locations",
47+
"📋 Policies",
48+
]
49+
)
50+
51+
with tab1:
52+
await organisation_section(admin_client)
53+
54+
with tab2:
55+
await users_section(admin_client)
56+
57+
with tab3:
58+
await user_organisation_section(admin_client)
59+
60+
with tab4:
61+
await locations_section(data_client)
62+
63+
with tab5:
64+
await policies_section(admin_client, data_client)
65+
66+
67+
# Required for the tests to run this as a script
68+
if __name__ == "__main__":
69+
dataplatform_toolbox_page()

‎src/dataplatform/toolbox/organisation.py‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
"""Organisation management section for the Data Platform Toolbox."""
2+
3+
import streamlit as st
4+
import json
5+
from dp_sdk.ocf import dp
6+
from grpclib.exceptions import GRPCError
7+
8+
9+
async def organisation_section(admin_client):
10+
"""Organisation management section."""
11+
12+
# Get Organisation Details
13+
st.markdown(
14+
'<h2 style="color:#63BCAF;font-size: 32px;">Get Organisation Details</h2>',
15+
unsafe_allow_html=True,
16+
)
17+
org_name = st.text_input("Organisation Name", key="get_org_name")
18+
if st.button("Get Organisation Details", key="get_org_button"):
19+
if not admin_client:
20+
st.error("❌ Could not connect to Data Platform")
21+
elif not org_name.strip():
22+
st.warning("⚠️ Please enter an organisation name")
23+
else:
24+
try:
25+
response = await admin_client.get_organisation(
26+
dp.GetOrganisationRequest(org_name=org_name)
27+
)
28+
response_dict = response.to_dict()
29+
st.success(f"✅ Found organisation: {org_name}")
30+
st.write(response_dict)
31+
32+
except GRPCError as e:
33+
st.error(
34+
f"❌ gRPC Error: {e.message}"
35+
)
36+
except Exception as e:
37+
st.error(f"❌ Error fetching organisation: {str(e)}")
38+
39+
# Create Organisation
40+
st.markdown(
41+
'<h2 style="color:#7bcdf3;font-size: 32px;">Create Organisation</h2>',
42+
unsafe_allow_html=True,
43+
)
44+
with st.expander("Create new organisation"):
45+
new_org_name = st.text_input("New Organisation Name", key="create_org_name")
46+
metadata_json = st.text_area(
47+
"Metadata (JSON)",
48+
value="{}",
49+
key="create_org_metadata",
50+
help="Enter valid JSON for organisation metadata",
51+
)
52+
53+
if st.button("Create Organisation", key="create_org_button"):
54+
if not admin_client:
55+
st.error("❌ Could not connect to Data Platform")
56+
elif not new_org_name.strip():
57+
st.warning("⚠️ Please enter an organisation name")
58+
else:
59+
try:
60+
# Parse metadata JSON
61+
metadata = (
62+
json.loads(metadata_json) if metadata_json.strip() else {}
63+
)
64+
response = await admin_client.create_organisation(
65+
dp.CreateOrganisationRequest(
66+
org_name=new_org_name, metadata=metadata
67+
)
68+
)
69+
response_dict = response.to_dict()
70+
st.success(
71+
f"✅ Organisation '{new_org_name}' created successfully!"
72+
)
73+
st.write(response_dict)
74+
75+
except json.JSONDecodeError:
76+
st.error("❌ Invalid JSON in metadata field")
77+
except GRPCError as e:
78+
st.error(
79+
f"❌ gRPC Error: {e.message}"
80+
)
81+
except Exception as e:
82+
st.error(f"❌ Error creating organisation: {str(e)}")
83+
84+
# Delete Organisation
85+
st.markdown(
86+
'<h2 style="color:#E63946;font-size:32px;">Delete Organisation</h2>',
87+
unsafe_allow_html=True,
88+
)
89+
with st.expander("Delete organisation"):
90+
del_org_name = st.text_input(
91+
"Organisation Name to Delete", key="delete_org_name"
92+
)
93+
st.warning("⚠️ This action cannot be undone!")
94+
confirm_delete = st.checkbox(
95+
"I understand this will permanently delete the organisation",
96+
key="confirm_delete_org",
97+
)
98+
if st.button("Delete Organisation", key="delete_org_button"):
99+
if not admin_client:
100+
st.error("❌ Could not connect to Data Platform")
101+
elif not del_org_name.strip():
102+
st.warning("⚠️ Please enter an organisation name")
103+
elif not confirm_delete:
104+
st.warning("⚠️ Please confirm deletion by checking the box above")
105+
else:
106+
try:
107+
await admin_client.delete_organisation(
108+
dp.DeleteOrganisationRequest(org_name=del_org_name)
109+
)
110+
st.success(
111+
f"✅ Organisation '{del_org_name}' deleted successfully!"
112+
)
113+
114+
except GRPCError as e:
115+
st.error(
116+
f"❌ gRPC Error: {e.message}"
117+
)
118+
except Exception as e:
119+
st.error(f"❌ Error deleting organisation: {str(e)}")

0 commit comments

Comments
 (0)