From 8bc4ae69bb5f7eb999dfce640dc18898494b19f9 Mon Sep 17 00:00:00 2001 From: msweier Date: Wed, 13 May 2026 15:27:28 -0500 Subject: [PATCH 1/3] add tests for NWO --- tests/commands/fixtures/MRBWM_GAPT.in | 14 +++++++ .../test_shef_import_infile_dry_run.py | 39 ++++++++++++++++++- 2 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 tests/commands/fixtures/MRBWM_GAPT.in diff --git a/tests/commands/fixtures/MRBWM_GAPT.in b/tests/commands/fixtures/MRBWM_GAPT.in new file mode 100644 index 0000000..1c53dd5 --- /dev/null +++ b/tests/commands/fixtures/MRBWM_GAPT.in @@ -0,0 +1,14 @@ +# Missouri River Basin Water Management GRFT Forecast +# Extract CWMS into SHEF for MVS and MBRFC and post to LDM +# +debug 1 +type .E +ti t t+31d +system eng +tz z +DB LOCAL + +PE Flow-Out.*=QT +FORMAT QT(5.1) +ts Fcst-MRBWM-GRFT_TW=FX +GAPT.Flow-Out.Inst.6Hours.0.Fcst-MRBWM-GRFT_TW diff --git a/tests/commands/test_shef_import_infile_dry_run.py b/tests/commands/test_shef_import_infile_dry_run.py index e82f50d..ce00cf2 100644 --- a/tests/commands/test_shef_import_infile_dry_run.py +++ b/tests/commands/test_shef_import_infile_dry_run.py @@ -7,7 +7,7 @@ import pytest from click.testing import CliRunner -from cwmscli.commands.shef.import_infile import import_shef_infile +from cwmscli.commands.shef.import_infile import import_shef_infile, parse_in_file @pytest.fixture @@ -200,3 +200,40 @@ def test_import_shef_infile_parses_location_mappings(fake_cwms, caplog): # Verify that location mappings were recognized (GENW3, LYNW3, GTTI4) # These are the SHEF location IDs from the LOCATION directives assert "GENW3" in output or "LYNW3" in output or "GTTI4" in output + + +# --------------------------------------------------------------------------- +# Regression tests: both .in file styles +# --------------------------------------------------------------------------- + +_NWO_FIXTURE = Path(__file__).parent / "fixtures" / "MRBWM_GAPT.in" + + +def test_nwo_style_version_send_code(): + """ts VersionName=SEND_CODE maps a specific TSID version to a send code.""" + if not _NWO_FIXTURE.exists(): + pytest.skip(f"Fixture file not found: {_NWO_FIXTURE}") + _, entries = parse_in_file(_NWO_FIXTURE) + assert len(entries) == 1 + assert entries[0]["send_code"] == "FX" + + +def test_nwo_style_location_fallback(): + """Without a LOCATION directive the CWMS location part is used as the SHEF ID.""" + if not _NWO_FIXTURE.exists(): + pytest.skip(f"Fixture file not found: {_NWO_FIXTURE}") + _, entries = parse_in_file(_NWO_FIXTURE) + assert len(entries) == 1 + assert entries[0]["shef_loc"] == "GAPT" + assert entries[0]["tsid"] == "GAPT.Flow-Out.Inst.6Hours.0.Fcst-MRBWM-GRFT_TW" + + +def test_existing_style_location_and_wildcard_send_code(): + """Existing style: LOCATION directives + TS * = SEND_CODE still works.""" + fixture_path = Path(__file__).parent / "fixtures" / "exportShef_CWMS_LD8-10.in" + if not fixture_path.exists(): + pytest.skip(f"Fixture file not found: {fixture_path}") + _, entries = parse_in_file(fixture_path) + assert len(entries) > 0 + genw3 = [e for e in entries if e["shef_loc"].startswith("GENW3")] + assert genw3, "Expected LOCATION mapping GENW3 for LockDam_08*" From b5633eba1c094e7216dd2e897ff35daa94973c40 Mon Sep 17 00:00:00 2001 From: msweier Date: Thu, 14 May 2026 08:52:47 -0500 Subject: [PATCH 2/3] add fall back for shef infile import if tsid doesn't exist --- cwmscli/commands/shef/import_infile.py | 64 ++++++++++++++++++++++++-- 1 file changed, 61 insertions(+), 3 deletions(-) diff --git a/cwmscli/commands/shef/import_infile.py b/cwmscli/commands/shef/import_infile.py index 518434a..6fd8db6 100644 --- a/cwmscli/commands/shef/import_infile.py +++ b/cwmscli/commands/shef/import_infile.py @@ -588,13 +588,32 @@ def build_group_json( # --------------------------------------------------------------------------- -def store_group( +def _filter_existing_tsids( + entries: list[dict], + office_id: str, +) -> list[dict]: + """Probe each TSID via cwms.get_timeseries_identifier and drop missing ones.""" + valid: list[dict] = [] + for e in entries: + try: + cwms.get_timeseries_identifier(ts_id=e["tsid"], office_id=office_id) + valid.append(e) + except Exception as exc: + log.warning( + "Skipping missing TSID '%s' (%s)", + e["tsid"], + type(exc).__name__, + ) + return valid + + +def _save( group_json: dict, group_id: str, office_id: str, fail_if_exists: bool, ) -> None: - """POST (create) or PATCH (update) the timeseries group in CWMS.""" + """Single store/update attempt with no validation fallback.""" try: cwms.store_timeseries_groups(group_json, fail_if_exists=fail_if_exists) log.info("SUCCESS — group stored via store_timeseries_groups.") @@ -606,7 +625,6 @@ def store_group( exc, ) - # Fallback: update, replacing all assigned timeseries cwms.update_timeseries_groups( data=group_json, group_id=group_id, @@ -616,6 +634,44 @@ def store_group( log.info("SUCCESS — group updated via update_timeseries_groups.") +def store_group( + group_json: dict, + group_id: str, + office_id: str, + fail_if_exists: bool, + entries: Optional[list[dict]] = None, + category_id: str = DEFAULT_CATEGORY, +) -> None: + """POST (create) or PATCH (update) the timeseries group in CWMS. + + On failure, validate each TSID individually, drop any that don't exist in + CWMS, and retry the save with the surviving entries. + """ + try: + _save(group_json, group_id, office_id, fail_if_exists) + return + except Exception as exc: + if entries is None: + raise + log.warning( + "Group save failed (%s: %s) — validating each TSID and retrying ...", + type(exc).__name__, + exc, + ) + + valid = _filter_existing_tsids(entries, office_id) + if not valid: + log.error("No valid TSIDs remain after validation — nothing to store.") + return + if len(valid) == len(entries): + log.error("All TSIDs validated but save still failed; re-raising.") + raise + + log.info("Retrying save with %d/%d valid TSIDs.", len(valid), len(entries)) + retry_json = build_group_json(valid, group_id, office_id, category_id) + _save(retry_json, group_id, office_id, fail_if_exists=False) + + # --------------------------------------------------------------------------- # Importable function for CLI integration # --------------------------------------------------------------------------- @@ -713,6 +769,8 @@ def import_shef_infile( group_id=group_name, office_id=office_id, fail_if_exists=fail_if_exists, + entries=entries, + category_id=category_id, ) log.info( From a072cdd9495717c2883844349a4b9142c2ddcf61 Mon Sep 17 00:00:00 2001 From: msweier Date: Thu, 14 May 2026 08:53:05 -0500 Subject: [PATCH 3/3] update tests and docs --- cwmscli/commands/commands_cwms.py | 10 ++- .../test_shef_import_infile_dry_run.py | 74 +++++++++++++++++++ 2 files changed, 83 insertions(+), 1 deletion(-) diff --git a/cwmscli/commands/commands_cwms.py b/cwmscli/commands/commands_cwms.py index e74af87..b076d02 100644 --- a/cwmscli/commands/commands_cwms.py +++ b/cwmscli/commands/commands_cwms.py @@ -283,7 +283,15 @@ def shef_import_crit(filename, office, api_root, api_key, api_key_loc, dry_run): # ================================================================================ @shef_group.command( "import_infile", - help="Import SHEF .in file into timeseries group for SHEF file processing", + help=( + "Import a legacy exportShef .in configuration file into a CWMS " + "timeseries group (default category 'SHEF Export'). Each entry " + "becomes a group member whose alias-id encodes the SHEF location, " + "PE code, send code, duration, and (optionally) units. " + "If the bulk save fails, each TSID is validated via " + "get_timeseries_identifier; missing TSIDs are logged and skipped, " + "and the save is retried with the surviving entries." + ), ) @click.option( "-f", diff --git a/tests/commands/test_shef_import_infile_dry_run.py b/tests/commands/test_shef_import_infile_dry_run.py index ce00cf2..615ae60 100644 --- a/tests/commands/test_shef_import_infile_dry_run.py +++ b/tests/commands/test_shef_import_infile_dry_run.py @@ -228,6 +228,80 @@ def test_nwo_style_location_fallback(): assert entries[0]["tsid"] == "GAPT.Flow-Out.Inst.6Hours.0.Fcst-MRBWM-GRFT_TW" +def test_missing_tsid_falls_back_to_validated_save(tmp_path, monkeypatch, caplog): + """If save fails, validate each TSID and retry with only the valid ones.""" + import logging + + from cwmscli.commands.shef import import_infile as mod + + caplog.set_level(logging.INFO) + + in_file = tmp_path / "missing.in" + in_file.write_text( + "PE Flow.*=QR\n" + "ts Best-MRBWM=RG\n" + "loc AAA=AAA\nAAA.Flow.Inst.1Hour.0.Best-MRBWM\n" + "loc BBB=BBB\nBBB.Flow.Inst.1Hour.0.Best-MRBWM\n" + "loc CCC=CCC\nCCC.Flow.Inst.1Hour.0.Best-MRBWM\n", + encoding="utf-8", + ) + + missing_tsid = "BBB.Flow.Inst.1Hour.0.Best-MRBWM" + store_calls: list[dict] = [] + + def fake_store(payload, fail_if_exists): + store_calls.append(payload) + # First call (with all 3 entries) fails; retry (with 2) succeeds. + if any(m["timeseries-id"] == missing_tsid for m in payload["members"]): + raise RuntimeError(f"DOES_NOT_EXIST: {missing_tsid}") + + def fake_update(data, group_id, office_id, replace_assigned_ts): + # Force the inner _save fallback to also fail so store_group enters + # the per-TSID validation branch. + raise RuntimeError(f"DOES_NOT_EXIST: {missing_tsid}") + + def fake_get_tsid(ts_id, office_id): + if ts_id == missing_tsid: + raise RuntimeError("404 Not Found") + return {"office-id": office_id, "timeseries-id": ts_id} + + def fake_init_session(api_root, api_key=None, token=None): + pass + + def fake_df_to_json( + data, group_id, group_office_id, category_office_id, category_id + ): + return { + "group-id": group_id, + "office-id": group_office_id, + "category-id": category_id, + "members": data.to_dict("records"), + } + + monkeypatch.setattr(mod.cwms, "store_timeseries_groups", fake_store) + monkeypatch.setattr(mod.cwms, "update_timeseries_groups", fake_update) + monkeypatch.setattr(mod.cwms, "get_timeseries_identifier", fake_get_tsid) + monkeypatch.setattr(mod.cwms, "timeseries_group_df_to_json", fake_df_to_json) + monkeypatch.setattr(mod.cwms_api, "init_session", fake_init_session) + + import_shef_infile( + in_file=str(in_file), + group_name="Test Group", + office_id="MVP", + api_root="https://test.example.com/", + api_key="test-key", + ) + + # First store had 3 entries (failed), retry store had 2 (succeeded). + assert len(store_calls) == 2 + assert len(store_calls[0]["members"]) == 3 + assert len(store_calls[1]["members"]) == 2 + retried_ids = {m["timeseries-id"] for m in store_calls[1]["members"]} + assert missing_tsid not in retried_ids + assert "Skipping missing TSID" in caplog.text + assert "Retrying save with 2/3 valid TSIDs" in caplog.text + + def test_existing_style_location_and_wildcard_send_code(): """Existing style: LOCATION directives + TS * = SEND_CODE still works.""" fixture_path = Path(__file__).parent / "fixtures" / "exportShef_CWMS_LD8-10.in"