diff --git a/.gencode_hash.txt b/.gencode_hash.txt index fe83115014..f34f7089b8 100644 --- a/.gencode_hash.txt +++ b/.gencode_hash.txt @@ -7,15 +7,15 @@ a65e8177ca59cd51c4a8ff63ecaa194897f7e22b82afb14708d63efbd7b96a84 gencode/docs/c 11b21f73b6a4065102968a4c09979639b8a7ea6efb20e40d52cd21b2a60167bb gencode/docs/configuration_pod.html b34c136cee32cb88f32a427ff400c3898ed49168f6dcaca1bc9ba65365bc5ae4 gencode/docs/configuration_pubber.html 1057fa40fb7a31a23bb2773d21c38cf4590a935bd8b5ea4218e695c6204f5dd9 gencode/docs/data_template.html -ea06d489d98f96f8ac0134388fb4172b1fca6d924aba895c6e3119c19b0c7dc0 gencode/docs/events.html +7335338a462cbd476bfacba8be66bbc3c0e2be6eaf9c9169d8d7455a9be3074f gencode/docs/events.html 70e57ad6ef39330d958727ebf9dcd61ef6ea30e4c8653eac412bf1867fdb3a70 gencode/docs/events_alarmset.html -feb4138c5acee9b3626e9c2e541711cec304c0e67c2999e6d713ee2e7144ef53 gencode/docs/events_discovery.html +05caeef731ef8f7e039350e19c7c2769fcaf4c45e08b1523560de918245c97f6 gencode/docs/events_discovery.html 808ad1cad37e9f4bf08ea3631162a663998ce60fe8935cbd0ca5e548c3b6df2a gencode/docs/events_mapping.html eafcc3c48189f605f114cde051fea9d13fc5f1a3e395d64fb0a91cb53d4c9aeb gencode/docs/events_pointset.html cac253f57c5c92ef32e2a5f91b6cec8229e8db1dcffcc96a58f06da068e741e7 gencode/docs/events_system.html 151c1b62db35e84e51d5ff2a7464f61ced4d7fb0c7eb795715c245ee0a1b3436 gencode/docs/events_udmi.html 73dbe799e7943ec20ac58b544998e986a39539d4ef0cb4f5023e92e7634d3124 gencode/docs/events_validation.html -0af72961d68e952c511f3edff1fb9d7c94ba1aadfa31fbe89128f8dae7f9703c gencode/docs/metadata.html +a6d4c6d1c291054590d82b8c581ff1189641be31a5e0a3ddc56587dbd789fd15 gencode/docs/metadata.html c86682715d348bd3dd971fa5bd925a8a3d0f3c2944c65a47c4b64fe1a5ccdea2 gencode/docs/monitoring.html 474ca16edc9f3cad2bb3ab40b6993cbced90263f762f66ee6cd246a6c4a0d18f gencode/docs/persistent_device.html e11595fd11477947a27461f8ef4fb6facb5f60e2abd6212193f7581ab123ff84 gencode/docs/properties.html @@ -116,7 +116,7 @@ d48a48484965295327ef553a4f8552c44b9345224d7dbb847877c68c1e1b307e gencode/java/u 34704328b6074f96dbf6ce2f926c2e5bafcde5f1412ea50da74e9fa294de32ea gencode/java/udmi/schema/PodConfiguration.java 750ff98b9049990b8a56989cd41e19ce3216c5ed668719f5e1fce019db4b8bf0 gencode/java/udmi/schema/PointPointsetConfig.java 43844df7c477fcb51e0bbf4f8a5e8b68631efa6b9cc4dee58bef82c619e64cc9 gencode/java/udmi/schema/PointPointsetEvents.java -90a0a42def3a0e3a3b5ed1f01010324d1e4a9023aa48a5f56d24e7672441998c gencode/java/udmi/schema/PointPointsetModel.java +1c059256fd5f1ae9b517314588b542bea35d1bade2542a5a384fbd9985390aa8 gencode/java/udmi/schema/PointPointsetModel.java 88ea66c1a8a1db71ee805a8facabafa0954690d0afd8a559758e50ec9624ed94 gencode/java/udmi/schema/PointPointsetState.java 66c33eb0cc692fb4d0938967ac7eeacd6400c3ce31274f39aa7c26a65b884b41 gencode/java/udmi/schema/PointsetConfig.java f4735049f0c27c3f0669d8cf2d6dda6e25dc7d20cef598d5677e3a72943ff019 gencode/java/udmi/schema/PointsetEvents.java @@ -229,7 +229,7 @@ c799ce667100201d81f02f0a5d22a4d0ef7db01035e7f08219a76a04c4507a15 gencode/python bdec4b1e8091db8420e813e22a57106acbdf77e7354e6b1ca38f3fd5fcff1141 gencode/python/udmi/schema/model_localnet.py e727422e0690d5cb6b45eb1e6771f6f676a429b2b9708af84490eddb6287b518 gencode/python/udmi/schema/model_localnet_family.py 7a252256cce946af6750c644e91b0f09363c8e28b03dbb41c8ae3e5d0758d615 gencode/python/udmi/schema/model_pointset.py -b27f7187d5aa69be9900bbb652dfbb09602dbde77f368f20523305636d18aea6 gencode/python/udmi/schema/model_pointset_point.py +3f8ccf5035f1870e72f0f13d92102fe1d6d5e0a931def5ea437d7354be406d9c gencode/python/udmi/schema/model_pointset_point.py f65ab38c968d1dc2ffb2d3eaf632f401e2b722ff0d6fe69abf20227f694c0d33 gencode/python/udmi/schema/model_policy.py 53cc53278a4d387bd2f6f9d564be0b80dd470aae5d32bd79d0019fc6ec8dc138 gencode/python/udmi/schema/model_relationships.py 241c702f6a877f41c0ebd997c7e431ba8e6d6c0d1b21bc95a9fc57592ff094a7 gencode/python/udmi/schema/model_system.py diff --git a/.gitignore b/.gitignore index 196d992335..57bc16fe8e 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ tmp/ var/ .gradle/ .firebase/ +.wincolor udmis/functions/node_modules/ udmis/public/deploy_version.js firebase-debug.log diff --git a/bin/dbo_extract b/bin/dbo_extract deleted file mode 100755 index 878d6116c2..0000000000 --- a/bin/dbo_extract +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" -ROOT_DIR="$( cd "$DIR/.." &> /dev/null && pwd )" - -# Activate the venv created by bin/setup_base -if [ -f "$ROOT_DIR/venv/bin/activate" ]; then - source "$ROOT_DIR/venv/bin/activate" -fi - -python3 "$ROOT_DIR/bin/dbo_extract.py" "$1" "$2" diff --git a/bin/dbo_extract.py b/bin/dbo_extract.py index 59488ea884..0c45a1d765 100755 --- a/bin/dbo_extract.py +++ b/bin/dbo_extract.py @@ -3,19 +3,45 @@ import sys from pathlib import Path import yaml +import re + +def sanitize_id(id_str: str) -> str: + """Sanitizes a string for use as a UDMI device_id.""" + # Replace non-alphanumeric with underscores, strip leading/trailing underscores + s = re.sub(r'[^a-zA-Z0-9_-]+', '_', id_str) + return s.strip('_') def extract_dbo_config(site_model_dir: Path) -> dict: """Extracts DBO configurations to a dict.""" building_config = {} + + ancillary_path = site_model_dir / "ancillary.json" + if ancillary_path.exists(): + with open(ancillary_path, "r", encoding="utf-8") as f: + building_config = json.load(f) + + device_id_to_guid = {} - # We will assume a static CONFIG_METADATA for this basic example - building_config["CONFIG_METADATA"] = { - "operation": "INITIALIZE" - } + # Build mapping for FACILITIES from ancillary data + used_ids = set() + # Sort keys to ensure deterministic collision handling if any + for guid in sorted(building_config.keys()): + if guid == "CONFIG_METADATA": continue + entity = building_config[guid] + code = entity.get("code") + base_id = sanitize_id(code) if code else guid + dev_id = base_id + counter = 2 + while dev_id in used_ids: + dev_id = f"{base_id}_{counter}" + counter += 1 + device_id_to_guid[dev_id] = guid + used_ids.add(dev_id) devices_dir = site_model_dir / "devices" + if not devices_dir.is_dir(): + return building_config - device_id_to_guid = {} metadata_map = {} # Pass 1: Build the mapping of device ID -> GUID @@ -49,10 +75,7 @@ def extract_dbo_config(site_model_dir: Path) -> dict: device_entry = {} - if "label" in dbo_external: - device_entry["code"] = dbo_external["label"] - else: - device_entry["code"] = device_id + device_entry["code"] = dbo_external.get("label", device_id) if "cloud" in metadata and "num_id" in metadata["cloud"]: device_entry["cloud_device_id"] = str(metadata["cloud"]["num_id"]) @@ -63,18 +86,10 @@ def extract_dbo_config(site_model_dir: Path) -> dict: for target_id, relation in metadata["relationships"].items(): target_guid = device_id_to_guid.get(target_id, target_id) if isinstance(relation, dict): - conn_list = [] - for rel_type, rel_instances in relation.items(): - for inst in rel_instances: - conn_obj = inst.copy() - conn_obj["type"] = rel_type - conn_list.append(conn_obj) - - # If it's just one type without tags, flatten it. - if len(conn_list) == 1 and list(conn_list[0].keys()) == ["type"]: - connections[target_guid] = conn_list[0]["type"] - else: - connections[target_guid] = conn_list + if relation: + # Match string enum format expected by schema + rel_type = list(relation.keys())[0] + connections[target_guid] = rel_type device_entry["connections"] = connections @@ -83,32 +98,33 @@ def extract_dbo_config(site_model_dir: Path) -> dict: translations = {} links_dbo = {} for point_name, point_info in pointset.items(): + # Reconstruct DBO translation from standard UDMI fields pt_dbo = {} - if "ref" in point_info: - pt_dbo["present_value"] = point_info["ref"] + pt_dbo["present_value"] = point_info.get("ref", f"points.{point_name}.present_value") + if "units" in point_info: - u_map = { - "degC": "degrees_celsius", - "L/s": "liters_per_second" - } - pt_dbo["units"] = { - "key": f"pointset.points.{point_name}.units", - "values": { - u_map.get( - point_info["units"], - point_info["units"]): point_info["units"] - } - } - - if pt_dbo: - translations[point_name] = pt_dbo - - if "link" in point_info: - link_val = point_info["link"] + u_map = { + "degC": "degrees_celsius", + "L/s": "liters_per_second" + } + pt_dbo["units"] = { + "key": f"pointset.points.{point_name}.units", + "values": { + u_map.get(point_info["units"], point_info["units"]): point_info["units"] + } + } + + if "value_map" in point_info: + pt_dbo["states"] = point_info["value_map"] + + if len(pt_dbo) > 1 or pt_dbo["present_value"] != f"points.{point_name}.present_value": + translations[point_name] = pt_dbo + + if "expr" in point_info: + link_val = point_info["expr"] if ":" in link_val: remote_device_id, remote_pt = link_val.split(":", 1) - remote_guid = device_id_to_guid.get( - remote_device_id, remote_device_id) + remote_guid = device_id_to_guid.get(remote_device_id, remote_device_id) if remote_guid not in links_dbo: links_dbo[remote_guid] = {} # DBO links are local_point: remote_point @@ -120,6 +136,12 @@ def extract_dbo_config(site_model_dir: Path) -> dict: if links_dbo: device_entry["links"] = links_dbo + if "etag" in dbo_external: + device_entry["etag"] = dbo_external["etag"] + + if "system" in metadata and "name" in metadata["system"]: + device_entry["display_name"] = metadata["system"]["name"] + if "type" in dbo_external: device_entry["type"] = dbo_external["type"] @@ -133,6 +155,8 @@ def extract_dbo_config(site_model_dir: Path) -> dict: sys.exit(1) site_dir = Path(sys.argv[1]) + if (site_dir / "site_model").is_dir(): + site_dir = site_dir / "site_model" out_yaml = Path(sys.argv[2]) config_result = extract_dbo_config(site_dir) @@ -140,6 +164,6 @@ def extract_dbo_config(site_model_dir: Path) -> dict: with open(out_yaml, "w", encoding="utf-8") as f_out: # Avoid aliases in YAML output to match standard formatting yaml.Dumper.ignore_aliases = lambda *args: True - yaml.dump(config_result, f_out, sort_keys=False, default_flow_style=False) + yaml.dump(config_result, f_out, sort_keys=True, default_flow_style=False) print(f"Extracted building config to {out_yaml}") diff --git a/bin/dbo_merge.py b/bin/dbo_merge.py index 15f1ae2ccc..258d73e03b 100755 --- a/bin/dbo_merge.py +++ b/bin/dbo_merge.py @@ -1,11 +1,37 @@ #!/usr/bin/env python3 """Script to merge a DBO config into a UDMI site model.""" import json +import os import sys +import csv +import re from datetime import datetime, timezone from pathlib import Path import yaml +def sanitize_id(id_str: str) -> str: + """Sanitizes a string for use as a UDMI device_id.""" + # Replace non-alphanumeric with underscores, strip leading/trailing underscores + s = re.sub(r'[^a-zA-Z0-9_-]+', '_', id_str) + return s.strip('_') + +def load_csv_map(site_model_dir: Path): + """Loads device_num_id -> device_id map from discovery.csv if it exists.""" + csv_map = {} + csv_path = site_model_dir / "discovery.csv" + if not csv_path.exists(): + csv_path = site_model_dir.parent / "discovery.csv" + + if csv_path.exists(): + with open(csv_path, "r", encoding="utf-8") as f: + reader = csv.DictReader(f) + for row in reader: + num_id = row.get("device_num_id") + dev_id = row.get("device_id") + if num_id and dev_id: + csv_map[num_id] = dev_id + return csv_map + def merge_dbo_config(yaml_file: Path, site_model_dir: Path): """Merges a DBO config file into the given UDMI site model.""" with open(yaml_file, "r", encoding="utf-8") as f: @@ -14,21 +40,55 @@ def merge_dbo_config(yaml_file: Path, site_model_dir: Path): devices_dir = site_model_dir / "devices" devices_dir.mkdir(parents=True, exist_ok=True) - # First pass: map guids to device codes - guid_to_code = {} + csv_map = load_csv_map(site_model_dir) + + # First pass: map every GUID to a unique ID + guid_to_id = {} + used_ids = set() + + # Priority 1: CSV-mapped devices (don't sanitize these, assume they are authoritative) for guid, entity in config.items(): - if guid == "CONFIG_METADATA": - continue - code = entity.get("code", guid) - guid_to_code[guid] = code + if guid == "CONFIG_METADATA": continue + cloud_id = str(entity.get("cloud_device_id", "")) + if cloud_id in csv_map: + dev_id = csv_map[cloud_id] + guid_to_id[guid] = dev_id + used_ids.add(dev_id) + + # Priority 2: Code (with sanitization and collision handling) + # This now applies to ALL entities including FACILITIES + for guid, entity in config.items(): + if guid == "CONFIG_METADATA" or guid in guid_to_id: continue + + code = entity.get("code") + if code: + base_id = sanitize_id(code) + else: + base_id = guid + + dev_id = base_id + counter = 2 + while dev_id in used_ids: + dev_id = f"{base_id}_{counter}" + counter += 1 + + guid_to_id[guid] = dev_id + used_ids.add(dev_id) + + ancillary = {} # Second pass: merge data for guid, entity in config.items(): if guid == "CONFIG_METADATA": continue - code = entity.get("code", guid) - device_dir = devices_dir / code + etype = entity.get("type", "") + if etype.startswith("FACILITIES/"): + ancillary[guid] = entity + continue + + device_id = guid_to_id[guid] + device_dir = devices_dir / device_id device_dir.mkdir(parents=True, exist_ok=True) metadata_path = device_dir / "metadata.json" @@ -36,18 +96,27 @@ def merge_dbo_config(yaml_file: Path, site_model_dir: Path): with open(metadata_path, "r", encoding="utf-8") as f: metadata = json.load(f) else: - timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + timestamp = os.environ.get("UDMI_TEST_TIMESTAMP", datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")) metadata = { "timestamp": timestamp, "version": "1.5.3", } + # System + system = metadata.setdefault("system", {}) + if "display_name" in entity: + system["name"] = entity["display_name"] + # Externals externals = metadata.setdefault("externals", {}) dbo = externals.setdefault("dbo", {}) dbo["ext_id"] = guid if "type" in entity: dbo["type"] = entity["type"] + if "etag" in entity: + dbo["etag"] = entity["etag"] + if "code" in entity: + dbo["label"] = entity["code"] # Cloud if "cloud_device_id" in entity: @@ -58,18 +127,21 @@ def merge_dbo_config(yaml_file: Path, site_model_dir: Path): if "connections" in entity: relationships = metadata.setdefault("relationships", {}) for target_guid, rel_data in entity["connections"].items(): - target_code = guid_to_code.get(target_guid, target_guid) + target_id = guid_to_id.get(target_guid, target_guid) if isinstance(rel_data, str): - relationships[target_code] = {rel_data: [{}]} + relationships[target_id] = {rel_data: [{}]} elif isinstance(rel_data, list): rel_dict = {} for item in rel_data: - item_copy = item.copy() - if "type" in item_copy: - rel_type = item_copy.pop("type") - rel_dict.setdefault(rel_type, []).append(item_copy) - relationships[target_code] = rel_dict + if isinstance(item, str): + rel_dict.setdefault(item, []).append({}) + elif isinstance(item, dict): + item_copy = item.copy() + if "type" in item_copy: + rel_type = item_copy.pop("type") + rel_dict.setdefault(rel_type, []).append(item_copy) + relationships[target_id] = rel_dict # Pointset Translation and Links if "translation" in entity or "links" in entity: @@ -80,23 +152,32 @@ def merge_dbo_config(yaml_file: Path, site_model_dir: Path): for pt_name, pt_dbo in entity["translation"].items(): pt_udmi = points.setdefault(pt_name, {}) if "present_value" in pt_dbo: - pt_udmi["ref"] = pt_dbo["present_value"] + pv = pt_dbo["present_value"] + if pv != "present_value" and pv != f"points.{pt_name}.present_value": + pt_udmi["ref"] = pv if "units" in pt_dbo and "values" in pt_dbo["units"]: - # Extract the first value from the values dictionary udmi_unit = list(pt_dbo["units"]["values"].values())[0] pt_udmi["units"] = udmi_unit + if "states" in pt_dbo: + pt_udmi["value_map"] = pt_dbo["states"] if "links" in entity: for target_guid, link_map in entity["links"].items(): - target_code = guid_to_code.get(target_guid, target_guid) + target_id = guid_to_id.get(target_guid, target_guid) for local_pt, remote_pt in link_map.items(): pt_udmi = points.setdefault(local_pt, {}) - pt_udmi["link"] = f"{target_code}:{remote_pt}" + pt_udmi["expr"] = f"{target_id}:{remote_pt}" with open(metadata_path, "w", encoding="utf-8") as f: json.dump(metadata, f, indent=2) f.write("\n") + if ancillary: + ancillary_path = site_model_dir / "ancillary.json" + with open(ancillary_path, "w", encoding="utf-8") as f: + json.dump(ancillary, f, indent=2) + f.write("\n") + if __name__ == "__main__": if len(sys.argv) != 3: print( diff --git a/bin/run_tests b/bin/run_tests index 2a96b05737..bacbdf43ff 100755 --- a/bin/run_tests +++ b/bin/run_tests @@ -70,6 +70,7 @@ case "$1" in registrar_tests) test_wrap bin/test_registrar test_wrap bin/test_sites + test_wrap bin/test_dbo_extract ;; all_tests) run_wrap install_dependencies diff --git a/bin/test_dbo_extract b/bin/test_dbo_extract new file mode 100755 index 0000000000..4d63d2a4f7 --- /dev/null +++ b/bin/test_dbo_extract @@ -0,0 +1,83 @@ +#!/bin/bash + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +ROOT_DIR="$( cd "$DIR/.." &> /dev/null && pwd )" + +# Activate the venv created by bin/setup_base +if [ -f "$ROOT_DIR/venv/bin/activate" ]; then + source "$ROOT_DIR/venv/bin/activate" +fi + +function run_test { + TEST_DIR=$1 + + GOLD_FILE="$TEST_DIR/building_config.yaml" + if [ ! -f "$GOLD_FILE" ]; then + echo "Gold file not found: $GOLD_FILE" + return 1 + fi + + if head -n 1 "$GOLD_FILE" | grep -q "device_registry_id,"; then + echo "Skipping site_model generation: $GOLD_FILE appears to be CSV, not YAML." + echo "Test passed: $TEST_DIR (skipped execution)" + return 0 + fi + + BEFORE_DIR="$TEST_DIR/site_model_before" + AFTER_DIR="$TEST_DIR/site_model_after" + + if [ ! -d "$BEFORE_DIR" ]; then + echo "Required directory missing: $BEFORE_DIR" + return 1 + fi + + echo "Running DBO merge test for $TEST_DIR..." + rm -rf "$AFTER_DIR" + mkdir -p "$AFTER_DIR" + + if [ -d "$BEFORE_DIR/devices" ]; then + cp -r "$BEFORE_DIR/devices" "$AFTER_DIR/" + fi + + export UDMI_TEST_TIMESTAMP="2026-01-01T00:00:00Z" + python3 "$ROOT_DIR/bin/dbo_merge.py" "$GOLD_FILE" "$AFTER_DIR" + + # General test: extract from merged output and compare with GOLD_FILE + OUTPUT_FILE="$AFTER_DIR/extracted_config.yaml" + echo "Running extraction test for $TEST_DIR..." + + python3 "$ROOT_DIR/bin/dbo_extract.py" "$AFTER_DIR" "$OUTPUT_FILE" + + if ! diff "$GOLD_FILE" "$OUTPUT_FILE"; then + echo "Extraction mismatch for $TEST_DIR" + return 1 + fi + + EXPECTED_DIR="$TEST_DIR/site_model_expected" + if [ -d "$EXPECTED_DIR" ]; then + echo "Comparing $AFTER_DIR with $EXPECTED_DIR..." + if ! diff -r "$EXPECTED_DIR" "$AFTER_DIR"; then + echo "Site model mismatch for $TEST_DIR" + return 1 + fi + fi + + echo "Test passed: $TEST_DIR" + return 0 +} + +SITES=("$@") +if [ ${#SITES[@]} -eq 0 ]; then + SITES=($ROOT_DIR/tests/dbo/*) +fi + +RET=0 +for SITE in "${SITES[@]}"; do + if [ -d "$SITE" ]; then + run_test "$SITE" || RET=1 + else + echo "Skipping non-directory: $SITE" + fi +done + +exit $RET diff --git a/gencode/docs/events.html b/gencode/docs/events.html index 337a590403..b0d2fe71c6 100644 --- a/gencode/docs/events.html +++ b/gencode/docs/events.html @@ -16364,18 +16364,18 @@

-