From 22f1e2889ba8ba68a4e83d296a2e26d8683a8336 Mon Sep 17 00:00:00 2001 From: Kurt Greaves Date: Mon, 5 May 2025 12:36:06 +1000 Subject: [PATCH 1/4] Update scripts for endeavour feeders Signed-off-by: Kurt Greaves --- setup.py | 5 +- .../examples/fetching_network_hierarchy.py | 8 +- src/zepben/examples/fetching_network_model.py | 2 +- .../examples/studies/dist_tx_polygons.py | 150 ++++++++++++++++++ .../tracing_conductor_type_by_lv_circuit.py | 29 ++-- src/zepben/examples/tracing_example.py | 2 +- 6 files changed, 173 insertions(+), 23 deletions(-) create mode 100644 src/zepben/examples/studies/dist_tx_polygons.py diff --git a/setup.py b/setup.py index d25b3a6..3ea57f6 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,10 @@ "zepben.evolve==0.48.0", "numba==0.60.0", "geojson==2.5.0", - "gql[requests]==3.4.1" + "gql[requests]==3.4.1", + "geopandas", + "pandas", + "shapely" ], extras_require={ "test": test_deps, diff --git a/src/zepben/examples/fetching_network_hierarchy.py b/src/zepben/examples/fetching_network_hierarchy.py index e6f2af4..392eb27 100644 --- a/src/zepben/examples/fetching_network_hierarchy.py +++ b/src/zepben/examples/fetching_network_hierarchy.py @@ -23,13 +23,13 @@ async def main(): print("Network hierarchy:") for gr in network_hierarchy.result.geographical_regions.values(): - print(f"- {gr.name}") + print(f"- GeographicalRegion mRID: {gr.mrid} name: {gr.name}") for sgr in gr.sub_geographical_regions: - print(f" - {sgr.name}") + print(f" - SubgeographicalRegion mRID: {sgr.mrid} name: {sgr.name}") for sub in sgr.substations: - print(f" - {sub.name}") + print(f" - Substation mRID: {sub.mrid} name: {sub.name}") for fdr in sub.feeders: - print(f" - {fdr.name}") + print(f" - Feeder mRID: {fdr.mrid} name: {fdr.name}") if __name__ == "__main__": diff --git a/src/zepben/examples/fetching_network_model.py b/src/zepben/examples/fetching_network_model.py index 2216e1f..0e3143d 100644 --- a/src/zepben/examples/fetching_network_model.py +++ b/src/zepben/examples/fetching_network_model.py @@ -21,7 +21,7 @@ async def main(): with open("config.json") as f: c = json.loads(f.read()) channel = connect_with_token(host=c["host"], access_token=c["access_token"], rpc_port=c["rpc_port"]) - feeder_mrid = "WD24" + feeder_mrid = "RW1292" print(f"Fetching {feeder_mrid}") # Note you should create a new client for each Feeder you retrieve # There is also a NetworkConsumerClient that is asyncio compatible, with the same API. diff --git a/src/zepben/examples/studies/dist_tx_polygons.py b/src/zepben/examples/studies/dist_tx_polygons.py new file mode 100644 index 0000000..c67939e --- /dev/null +++ b/src/zepben/examples/studies/dist_tx_polygons.py @@ -0,0 +1,150 @@ +# Copyright 2025 Zeppelin Bend Pty Ltd +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. +import asyncio +import json +import pandas as pd +from shapely.geometry import Polygon +import geopandas as gpd + +from zepben.protobuf.nc.nc_requests_pb2 import IncludedEnergizedContainers + +from zepben.evolve import connect_with_token, NetworkConsumerClient, Switch, Site, LvFeeder +from zepben.eas.client.eas_client import EasClient +from zepben.eas.client.study import Study, Result, GeoJsonOverlay + +with open("../config.json") as f: + config = json.load(f) + + +async def connect(): + channel = connect_with_token(host=config["host"], rpc_port=config["rpc_port"], access_token=config["access_token"], ca_filename=config["ca_path"]) + + feeder = "RW1292" + print(f"Processing feeder {feeder}") + geojson_features = [] + await process_feeder(feeder, channel, geojson_features) + + print("Uploading study") + await upload_study({"type": "FeatureCollection", "features": geojson_features}) + + +async def process_feeder(feeder_mrid: str, channel, geojson_features: list): + print(f"Fetching {feeder_mrid}") + network_client = NetworkConsumerClient(channel=channel) + network_service = network_client.service + + # Fetches the feeder plus all the LV feeders (dist txs and LV circuits) + (await network_client.get_equipment_container(feeder_mrid, include_energized_containers=IncludedEnergizedContainers.INCLUDE_ENERGIZED_LV_FEEDERS)).throw_on_error() + + counter = 0 + for lvf in network_service.objects(LvFeeder): + print(f"Processing {lvf.name}...") + points = [] + + # Get all the coordinates from the LvFeeder + for psr in lvf.equipment: + if psr.location is not None: + for pp in psr.location.points: + points.append((pp.x_position, pp.y_position)) + + # Only care about feeders that had more than 3 points - this just excludes anything empty + # and stops the algorithm from failing + if len(points) > 3: + # Build a concave hull of the points + p = Polygon(points) + df = pd.DataFrame({'hull': [1]}) + df['geometry'] = p + + gdf = gpd.GeoDataFrame(df, crs='EPSG:4326', geometry='geometry') + geojson = json.loads(gdf.concave_hull(0.30).to_json()) + feature = geojson["features"][0] + feature["properties"]["pen"] = counter % 14 + # Add this to the list of features to upload in the study - there should be one feature per zone substation + geojson_features.append(feature) + counter += 1 + + +async def upload_study(geojson): + protocol = config.get("eas_protocol", "https") + eas_client = EasClient( + host=config["eas_host"], + port=config["eas_port"], + protocol=config.get("eas_protocol", "https"), + access_token=config["access_token"] if protocol == "https" else None, + ca_filename=config["ca_path"], + verify_certificate=False + ) + + styles = [ + { + "id": "dist tx boundaries", + "name": "Distribution Transformer Boundaries", + "type": "line", + "paint": { + "line-color": "rgb(0,0,0)", + "line-width": 3 + }, + "maxzoom": 24, + }, + { + "id": "feedersfill", + "name": "boundaryfill", + "type": "fill", + "paint": { + 'fill-color': [ + "match", + ["get", "pen"], + 0,"#3388FF", + 1,"#8800FF", + 2,"#AAAA00", + 3,"#00AA00", + 4,"#FF00AA", + 5,"#0000AA", + 6,"#AAAAAA", + 7,"#AA0000", + 8,"#00AAFF", + 9,"#AA00AA", + 10,"#CC8800", + 11,"#00AAAA", + 12,"#0000FF", + 13,"#666666", + 14,"#e30707", + "#cccccc" + ], + 'fill-opacity': 0.5 + }, + "maxzoom": 24, + } + + ] + + result = await eas_client.async_upload_study( + Study( + name="Dist TX polygons", + description="Distribution Transformer polygons", + tags=["tx polygons distribution"], + results=[ + Result( + name="Dist TX Boundaries", + geo_json_overlay=GeoJsonOverlay( + data=geojson, + styles=['feeders', 'feedersfill'] + ) + ) + ], + styles=styles + ) + ) + print(f"EAS upload result: {result}") + await eas_client.aclose() + + +if __name__ == "__main__": + asyncio.run(connect()) diff --git a/src/zepben/examples/tracing_conductor_type_by_lv_circuit.py b/src/zepben/examples/tracing_conductor_type_by_lv_circuit.py index b767c3a..80b6f62 100644 --- a/src/zepben/examples/tracing_conductor_type_by_lv_circuit.py +++ b/src/zepben/examples/tracing_conductor_type_by_lv_circuit.py @@ -11,7 +11,7 @@ from typing import Any, List, Union from zepben.evolve import NetworkConsumerClient, PhaseStep, PhaseCode, AcLineSegment, \ - Switch, normal_downstream_trace, FeederDirection, connect_with_token + Switch, normal_downstream_trace, FeederDirection, connect_with_token, PowerTransformer from zepben.evolve.services.network.tracing.phases.phase_step import start_at from zepben.protobuf.nc.nc_requests_pb2 import IncludedEnergizedContainers @@ -27,15 +27,17 @@ async def main(): result = (await client.get_network_hierarchy()).throw_on_error().result print("Connection Established") - switch_to_line_type: dict[str, tuple[list[Any], bool]] = {} + tx_to_line_type: dict[str, tuple[list[Any], bool]] = {} os.makedirs("csvs", exist_ok=True) for feeder in result.feeders.values(): + if feeder.mrid != "J503": + continue print(f"Fetching {feeder.mrid}") if not (network := await get_feeder_network(channel, feeder.mrid)): # Skip feeders that fail to pull down print(f"Failed to retrieve feeder {feeder.mrid}") continue - for io in network.objects(Switch): + for io in network.objects(PowerTransformer): _loop = False for t in io.terminals: @@ -43,29 +45,24 @@ async def main(): if t_dir == FeederDirection.BOTH: _loop = True - sw_name = io.name - sw_id = io.mrid - - # Currently using switch with the following name as a marker for LV circuit heads - if "Circuit Head Switch" in sw_name: - switch_to_line_type[sw_id] = ( - await get_downstream_trace(start_at(io, PhaseCode.ABCN)), - loop - ) - await save_to_csv(switch_to_line_type, feeder.mrid) + tx_to_line_type[io.mrid] = ( + await get_downstream_trace(start_at(io, PhaseCode.ABCN)), + loop + ) + await save_to_csv(tx_to_line_type, feeder.mrid) async def save_to_csv(data: dict[str, tuple[list[Any], bool]], feeder_mrid): filename = f"csvs/conductor_types_{feeder_mrid}.csv" with open(filename, mode='w', newline='') as file: writer = csv.writer(file) - writer.writerow(["Feeder", "Switch", "Line", "Line Type", "Length", "Loop"]) + writer.writerow(["Feeder", "Transformer", "Line", "Line Type", "Length", "Loop"]) - for switch, (values, loop) in data.items(): + for transformer, (values, loop) in data.items(): for i in range(0, len(values), 3): line_type = values[i + 1] if i + 1 < len(values) else "" length = values[i + 2] if i + 2 < len(values) else "" - switch_data = [feeder_mrid, switch, values[i], line_type, length, loop] + switch_data = [feeder_mrid, transformer, values[i], line_type, length, loop] writer.writerow(switch_data) print(f"Data saved to {filename}") diff --git a/src/zepben/examples/tracing_example.py b/src/zepben/examples/tracing_example.py index 6f3628b..4aa23f5 100644 --- a/src/zepben/examples/tracing_example.py +++ b/src/zepben/examples/tracing_example.py @@ -27,7 +27,7 @@ async def main(): print("Connection Established") for feeder in result.feeders.values(): - if feeder.mrid != "WD24": + if feeder.mrid != "RW1292": continue print(f"\nFetching {feeder.mrid}") network = await get_feeder_network(channel, feeder.mrid) From 204f6e96378685a1b5062a803ab6934f8d3b5ebf Mon Sep 17 00:00:00 2001 From: Kurt Greaves Date: Thu, 26 Jun 2025 10:05:50 +1000 Subject: [PATCH 2/4] Add script for dumping transformer ID -> name mapping to a CSV for all substations and feeders Signed-off-by: Kurt Greaves --- src/zepben/examples/tx_id_to_name.py | 82 ++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 src/zepben/examples/tx_id_to_name.py diff --git a/src/zepben/examples/tx_id_to_name.py b/src/zepben/examples/tx_id_to_name.py new file mode 100644 index 0000000..7c6d735 --- /dev/null +++ b/src/zepben/examples/tx_id_to_name.py @@ -0,0 +1,82 @@ +# Copyright 2025 Zeppelin Bend Pty Ltd +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. +import asyncio +import json +import os.path +from dataclasses import dataclass +import pandas as pd + +from zepben.protobuf.nc.nc_requests_pb2 import IncludedEnergizedContainers + +from zepben.evolve import NetworkConsumerClient, connect_with_token, PowerTransformer + +OUTPUT_FILE = "transformer_id_mapping.csv" +HEADER = True + +with open("./config.json") as f: + c = json.loads(f.read()) + + +async def connect(): + channel = connect_with_token(host=c["host"], rpc_port=c["rpc_port"], access_token=c["access_token"], ca_filename=c["ca_path"]) + network_client = NetworkConsumerClient(channel=channel) + + if os.path.exists(OUTPUT_FILE): + print(f"Output file {OUTPUT_FILE} already exists, please delete it if you would like to regenerate.") + return + + network_hierarchy = (await network_client.get_network_hierarchy()).throw_on_error().value + + print("Network hierarchy:") + for gr in network_hierarchy.geographical_regions.values(): + print(f"- Geographical region: {gr.name}") + for sgr in gr.sub_geographical_regions: + print(f" - Subgeographical region: {sgr.name}") + for sub in sgr.substations: + print(f" - Zone Substation: {sub.name}") + await process_nodes(sub.mrid, channel) + for fdr in sub.feeders: + print(f" - Processing Feeder: {fdr.name}") + await process_nodes(fdr.mrid, channel) + return # Only process the first zone... + + +@dataclass +class NetworkObject(object): + dist_tx_id: str + dist_tx_name: str + container: str + container_mrid: str + + +async def process_nodes(container_mrid: str, channel): + global HEADER + print("Fetching from server ...") + network_client = NetworkConsumerClient(channel=channel) + network_service = network_client.service + (await network_client.get_equipment_container(container_mrid)).throw_on_error() + container = network_service.get(container_mrid) + container_name = container.name + + print("Processing equipment ...") + network_objects = [] + for equip in network_service.objects(PowerTransformer): + no = NetworkObject(equip.mrid, equip.name, container_name, container_mrid) + network_objects.append(no) + + network_objects = pd.DataFrame(network_objects) + network_objects.to_csv(OUTPUT_FILE, index=False, mode='a', header=HEADER) + print(f"Finished processing {container_mrid}") + if HEADER: + HEADER = False + + +if __name__ == "__main__": + asyncio.run(connect()) From 48ef9ff74d2996012f8fad7f3d634916b08f27c5 Mon Sep 17 00:00:00 2001 From: Max Chesterfield Date: Tue, 22 Jul 2025 13:51:21 +1000 Subject: [PATCH 3/4] Fixed tracing_conductor_type_by_lv_circuit Signed-off-by: Max Chesterfield --- .../tracing_conductor_type_by_lv_circuit.py | 47 +++++++++---------- 1 file changed, 21 insertions(+), 26 deletions(-) diff --git a/src/zepben/examples/tracing_conductor_type_by_lv_circuit.py b/src/zepben/examples/tracing_conductor_type_by_lv_circuit.py index 80b6f62..12aa4db 100644 --- a/src/zepben/examples/tracing_conductor_type_by_lv_circuit.py +++ b/src/zepben/examples/tracing_conductor_type_by_lv_circuit.py @@ -8,12 +8,13 @@ import csv import json import os -from typing import Any, List, Union +from typing import List, Union, Tuple, Optional, Dict -from zepben.evolve import NetworkConsumerClient, PhaseStep, PhaseCode, AcLineSegment, \ - Switch, normal_downstream_trace, FeederDirection, connect_with_token, PowerTransformer -from zepben.evolve.services.network.tracing.phases.phase_step import start_at -from zepben.protobuf.nc.nc_requests_pb2 import IncludedEnergizedContainers +from zepben.evolve import NetworkConsumerClient, PhaseCode, AcLineSegment, \ + FeederDirection, connect_with_token, Tracing, downstream, NetworkTraceStep, ConductingEquipment, PowerTransformer +from zepben.protobuf.nc.nc_requests_pb2 import INCLUDE_ENERGIZED_LV_FEEDERS + +LineInfo = Tuple[str, str, Optional[Union[int, float]]] async def main(): @@ -27,7 +28,7 @@ async def main(): result = (await client.get_network_hierarchy()).throw_on_error().result print("Connection Established") - tx_to_line_type: dict[str, tuple[list[Any], bool]] = {} + tx_to_line_type: Dict[str, Tuple[List[LineInfo], bool]] = {} os.makedirs("csvs", exist_ok=True) for feeder in result.feeders.values(): @@ -38,6 +39,7 @@ async def main(): print(f"Failed to retrieve feeder {feeder.mrid}") continue for io in network.objects(PowerTransformer): + print(io) _loop = False for t in io.terminals: @@ -45,24 +47,21 @@ async def main(): if t_dir == FeederDirection.BOTH: _loop = True - tx_to_line_type[io.mrid] = ( - await get_downstream_trace(start_at(io, PhaseCode.ABCN)), - loop - ) + + tx_to_line_type[io.mrid] = (await get_downstream_trace(io, PhaseCode.ABCN), _loop) await save_to_csv(tx_to_line_type, feeder.mrid) -async def save_to_csv(data: dict[str, tuple[list[Any], bool]], feeder_mrid): +async def save_to_csv(data: Dict[str, Tuple[List[LineInfo], bool]], feeder_mrid): filename = f"csvs/conductor_types_{feeder_mrid}.csv" with open(filename, mode='w', newline='') as file: writer = csv.writer(file) writer.writerow(["Feeder", "Transformer", "Line", "Line Type", "Length", "Loop"]) for transformer, (values, loop) in data.items(): - for i in range(0, len(values), 3): - line_type = values[i + 1] if i + 1 < len(values) else "" - length = values[i + 2] if i + 2 < len(values) else "" - switch_data = [feeder_mrid, transformer, values[i], line_type, length, loop] + for value in values: + line, line_type, length = value + switch_data = [feeder_mrid, transformer, line, line_type, length, loop] writer.writerow(switch_data) print(f"Data saved to {filename}") @@ -82,22 +81,18 @@ async def get_feeder_network(channel, feeder_mrid): return client.service -async def get_downstream_trace(ce: ConductingEquipment, phase_code: PhaseCode) -> list[Union[str, float]]: - l_type: List[Union[str, float]] = [] - - def collect_eq_in(): - async def add_eq(ps: NetworkTraceStep, _): - equip = ps.path.to_equipment - if isinstance(equip, AcLineSegment): - nonlocal l_type - l_type.extend((equip.mrid, equip.asset_info.name, equip.length or 0)) +async def get_downstream_trace(ce: ConductingEquipment, phase_code: PhaseCode) -> List[LineInfo]: + l_type: List[LineInfo] = [] - return add_eq + def collect_eq_in(step: NetworkTraceStep, _): + if isinstance(equip := step.path.to_equipment, AcLineSegment): + nonlocal l_type + l_type.append((equip.mrid, equip.asset_info.name, equip.length or 0)) await ( Tracing.network_trace() .add_condition(downstream()) - .add_step_action(collect_eq_in()) + .add_step_action(collect_eq_in) ).run(start=ce, phases=phase_code) return l_type From 571e5a1c6b9f059b84c6333e89eb21318a7c013a Mon Sep 17 00:00:00 2001 From: Max Chesterfield Date: Tue, 22 Jul 2025 14:17:26 +1000 Subject: [PATCH 4/4] small fixes, syntax/formatting Signed-off-by: Max Chesterfield --- src/zepben/examples/connecting_to_grpc_service.py | 2 +- src/zepben/examples/current_state_manipulations.py | 13 +++++++------ src/zepben/examples/examining_connectivity.py | 2 +- src/zepben/examples/export_open_dss_model.py | 2 +- src/zepben/examples/tx_id_to_name.py | 2 -- 5 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/zepben/examples/connecting_to_grpc_service.py b/src/zepben/examples/connecting_to_grpc_service.py index d6fdf3d..63ba393 100644 --- a/src/zepben/examples/connecting_to_grpc_service.py +++ b/src/zepben/examples/connecting_to_grpc_service.py @@ -114,7 +114,7 @@ async def connect_using_token(): from zepben.evolve import connect_with_token, NetworkConsumerClient with open("config.json") as f: - c = json.loads(f.read()) + c = json.load(f) print("Connecting to EWB..") channel = connect_with_token( diff --git a/src/zepben/examples/current_state_manipulations.py b/src/zepben/examples/current_state_manipulations.py index 6502167..e6c7237 100644 --- a/src/zepben/examples/current_state_manipulations.py +++ b/src/zepben/examples/current_state_manipulations.py @@ -5,12 +5,13 @@ # file, You can obtain one at https://mozilla.org/MPL/2.0/. import asyncio +import json import sys from typing import List, Set from zepben.evolve import ( Feeder, PowerTransformer, Switch, Tracing, NetworkConsumerClient, connect_with_password, Terminal, - BusbarSection, ConductingEquipment, Breaker, EquipmentContainer, StepContext, NetworkTraceStep + BusbarSection, ConductingEquipment, Breaker, EquipmentContainer, StepContext, NetworkTraceStep, connect_with_token ) from zepben.protobuf.nc.nc_requests_pb2 import INCLUDE_ENERGIZED_FEEDERS, INCLUDE_ENERGIZING_FEEDERS @@ -48,8 +49,8 @@ async def fetch_zone_feeders(client: NetworkConsumerClient): await client.get_equipment_container( feeder.mrid, Feeder, - include_energizing_containers=INCLUDE_ENERGIZED_FEEDERS, - include_energized_containers=INCLUDE_ENERGIZING_FEEDERS + include_energizing_containers=INCLUDE_ENERGIZING_FEEDERS, + include_energized_containers=INCLUDE_ENERGIZED_FEEDERS ) print("CPM feeders fetched.") @@ -244,11 +245,11 @@ def log_txs(desc: str, feeders: Set[Feeder]): async def main(): - if len(sys.argv) != 6: - raise TypeError("you must provided the CLIENT_ID, username, password, host and port to connect") # noinspection PyTypeChecker - async with connect_with_password(*sys.argv[1:]) as secure_channel: + with open('config.json') as f: + config = json.load(f) + async with connect_with_token(**config) as secure_channel: await run_simple(NetworkConsumerClient(secure_channel)) await run_swap_feeder(NetworkConsumerClient(secure_channel)) diff --git a/src/zepben/examples/examining_connectivity.py b/src/zepben/examples/examining_connectivity.py index 53b1b7c..dd67de6 100644 --- a/src/zepben/examples/examining_connectivity.py +++ b/src/zepben/examples/examining_connectivity.py @@ -26,7 +26,7 @@ def build_network() -> NetworkService: # We create the objects, and their Terminals _es = EnergySource(mrid="es", terminals=[ - Terminal(mrid="es-t") + Terminal(mrid="es_t") ]) _hv_line = AcLineSegment(mrid="hv_line", terminals=[ diff --git a/src/zepben/examples/export_open_dss_model.py b/src/zepben/examples/export_open_dss_model.py index 5154987..efc0921 100644 --- a/src/zepben/examples/export_open_dss_model.py +++ b/src/zepben/examples/export_open_dss_model.py @@ -7,7 +7,7 @@ from datetime import datetime from zepben.eas.client.opendss import OpenDssConfig -from zepben.eas.client.work_package import GeneratorConfig, ModelConfig, LoadPlacement, FeederScenarioAllocationStrategy, SolveConfig, RawResultsConfig, \ +from zepben.eas.client.work_package import GeneratorConfig, ModelConfig, FeederScenarioAllocationStrategy, SolveConfig, RawResultsConfig, \ MeterPlacementConfig, SwitchMeterPlacementConfig, SwitchClass from zepben.eas import EasClient, TimePeriod from time import sleep diff --git a/src/zepben/examples/tx_id_to_name.py b/src/zepben/examples/tx_id_to_name.py index 7c6d735..07bb962 100644 --- a/src/zepben/examples/tx_id_to_name.py +++ b/src/zepben/examples/tx_id_to_name.py @@ -13,8 +13,6 @@ from dataclasses import dataclass import pandas as pd -from zepben.protobuf.nc.nc_requests_pb2 import IncludedEnergizedContainers - from zepben.evolve import NetworkConsumerClient, connect_with_token, PowerTransformer OUTPUT_FILE = "transformer_id_mapping.csv"