From 18a3b84ff116b3272703c4a238d268ef3bc146b0 Mon Sep 17 00:00:00 2001 From: Samuel Johnson Date: Tue, 19 May 2026 15:56:32 -0400 Subject: [PATCH 1/2] logs & capture --- .github/scripts/run_validation.sh | 2 +- Published/CORE-000929/negative/01/data/.env | 1 - .../negative/01/results/results.csv | 9 +-- Published/CORE-000929/negative/02/data/.env | 1 - Published/CORE-000929/positive/01/data/.env | 1 - test.py | 81 ++++++++++++++++--- 6 files changed, 76 insertions(+), 19 deletions(-) diff --git a/.github/scripts/run_validation.sh b/.github/scripts/run_validation.sh index 30359cdfb..a9644b403 100644 --- a/.github/scripts/run_validation.sh +++ b/.github/scripts/run_validation.sh @@ -33,7 +33,7 @@ fi echo "Rule file: $RULE_YML" # --------------------------------------------------------------------------- -# Initialise report +# Initialize report # --------------------------------------------------------------------------- { echo "# Rule Validation Report" diff --git a/Published/CORE-000929/negative/01/data/.env b/Published/CORE-000929/negative/01/data/.env index c2dd61172..db94f54f8 100644 --- a/Published/CORE-000929/negative/01/data/.env +++ b/Published/CORE-000929/negative/01/data/.env @@ -1,4 +1,3 @@ PRODUCT=SDTMIG VERSION=3-3 -CT=sdtmct2020-12-18 DEFINE_XML=define_negative1.xml diff --git a/Published/CORE-000929/negative/01/results/results.csv b/Published/CORE-000929/negative/01/results/results.csv index 7ba2b1f15..b03aedf17 100644 --- a/Published/CORE-000929/negative/01/results/results.csv +++ b/Published/CORE-000929/negative/01/results/results.csv @@ -1,6 +1,5 @@ Dataset,Record,Variable,Value -zb.csv,,, -fa.csv,2,$domain_lib_ccode,"['C49563', 'C49562', 'C117755', 'C147168', 'C147169', 'C147170', 'C132354', 'C147171', 'C147172', 'C147173', 'C147174', 'C147175', 'C147176', 'C147177', 'C147178', 'C147179', 'C111282', 'C111138', 'C111137', 'C85441', 'C49568', 'C49569', 'C102605', 'C49578', 'C95087', 'C102616', 'C102618', 'C49572', 'C102619', 'C102620', 'C49576', 'C102621', 'C102622', 'C49585', 'C102617', 'C117466', 'C102630', 'C49626', 'C49587', 'C85442', 'C117756', 'C102640', 'C102641', 'C117757', 'C61536', 'C102651', 'C112320', 'C49592', 'C49602', 'C49603', 'C95095', 'C102674', 'C49604', 'C102671', 'C61531', 'C102677', 'C147180', 'C161329', 'C106552', 'C49606', 'C49608', 'C111289', 'C61529', 'C147181', 'C49607', 'C102700', 'C49609', 'C95098', 'C147182', 'C147183', 'C102707', 'C107097', 'C106571', 'C49610', 'C49616', 'C147184', 'C112420', 'C117655', 'C49615', 'C147185', 'C147186', 'C147187', 'C147189', 'C147191', 'C147193', 'C147195', 'C147196', 'C147197', 'C147198', 'C147200', 'C147201', 'C147203', 'C147204', 'C147205', 'C147206', 'C147207', 'C147208', 'C147209', 'C147210', 'C147211', 'C147212', 'C147216', 'C147218', 'C147219', 'C147220', 'C147222', 'C147223', 'C147224', 'C147225', 'C147226', 'C165871', 'C147227', 'C147228', 'C147229', 'C147232', 'C147233', 'C147234', 'C147235', 'C147236', 'C147238', 'C147239', 'C147240', 'C147241', 'C147242', 'C147243', 'C147244', 'C147245', 'C147246', 'C147249', 'C147250', 'C147251', 'C147252', 'C147261', 'C147264', 'C147267', 'C147268', 'C147269', 'C49617', 'C49618', 'C117699', 'C49619', 'C49620', 'C147270', 'C106578', 'C53483', 'C106577', 'C49621', 'C95103', 'C102726', 'C49622']" -fa.csv,2,define_variable_codelist_coded_codes,['C00002'] -vs.csv,2,$domain_lib_ccode,"['C49563', 'C49562', 'C117755', 'C147168', 'C147169', 'C147170', 'C132354', 'C147171', 'C147172', 'C147173', 'C147174', 'C147175', 'C147176', 'C147177', 'C147178', 'C147179', 'C111282', 'C111138', 'C111137', 'C85441', 'C49568', 'C49569', 'C102605', 'C49578', 'C95087', 'C102616', 'C102618', 'C49572', 'C102619', 'C102620', 'C49576', 'C102621', 'C102622', 'C49585', 'C102617', 'C117466', 'C102630', 'C49626', 'C49587', 'C85442', 'C117756', 'C102640', 'C102641', 'C117757', 'C61536', 'C102651', 'C112320', 'C49592', 'C49602', 'C49603', 'C95095', 'C102674', 'C49604', 'C102671', 'C61531', 'C102677', 'C147180', 'C161329', 'C106552', 'C49606', 'C49608', 'C111289', 'C61529', 'C147181', 'C49607', 'C102700', 'C49609', 'C95098', 'C147182', 'C147183', 'C102707', 'C107097', 'C106571', 'C49610', 'C49616', 'C147184', 'C112420', 'C117655', 'C49615', 'C147185', 'C147186', 'C147187', 'C147189', 'C147191', 'C147193', 'C147195', 'C147196', 'C147197', 'C147198', 'C147200', 'C147201', 'C147203', 'C147204', 'C147205', 'C147206', 'C147207', 'C147208', 'C147209', 'C147210', 'C147211', 'C147212', 'C147216', 'C147218', 'C147219', 'C147220', 'C147222', 'C147223', 'C147224', 'C147225', 'C147226', 'C165871', 'C147227', 'C147228', 'C147229', 'C147232', 'C147233', 'C147234', 'C147235', 'C147236', 'C147238', 'C147239', 'C147240', 'C147241', 'C147242', 'C147243', 'C147244', 'C147245', 'C147246', 'C147249', 'C147250', 'C147251', 'C147252', 'C147261', 'C147264', 'C147267', 'C147268', 'C147269', 'C49617', 'C49618', 'C117699', 'C49619', 'C49620', 'C147270', 'C106578', 'C53483', 'C106577', 'C49621', 'C95103', 'C102726', 'C49622']" -vs.csv,2,define_variable_codelist_coded_codes,['C00004'] +FA,2,$domain_lib_ccode,"['C49563', 'C49562', 'C117755', 'C147168', 'C147169', 'C147170', 'C132354', 'C147171', 'C147172', 'C147173', 'C147174', 'C147175', 'C147176', 'C147177', 'C147178', 'C147179', 'C111282', 'C111138', 'C111137', 'C85441', 'C49568', 'C49569', 'C102605', 'C49578', 'C95087', 'C102616', 'C102618', 'C49572', 'C102619', 'C102620', 'C49576', 'C102621', 'C102622', 'C49585', 'C102617', 'C117466', 'C102630', 'C49626', 'C49587', 'C85442', 'C117756', 'C102640', 'C102641', 'C117757', 'C61536', 'C102651', 'C112320', 'C49592', 'C49602', 'C49603', 'C95095', 'C102674', 'C49604', 'C102671', 'C61531', 'C102677', 'C147180', 'C161329', 'C106552', 'C49606', 'C49608', 'C111289', 'C61529', 'C147181', 'C49607', 'C102700', 'C49609', 'C95098', 'C147182', 'C147183', 'C102707', 'C107097', 'C106571', 'C49610', 'C49616', 'C147184', 'C112420', 'C117655', 'C49615', 'C147185', 'C147186', 'C147187', 'C147189', 'C147191', 'C147193', 'C147195', 'C147196', 'C147197', 'C147198', 'C147200', 'C147201', 'C147203', 'C147204', 'C147205', 'C147206', 'C147207', 'C147208', 'C147209', 'C147210', 'C147211', 'C147212', 'C147216', 'C147218', 'C147219', 'C147220', 'C147222', 'C147223', 'C147224', 'C147225', 'C147226', 'C165871', 'C147227', 'C147228', 'C147229', 'C147232', 'C147233', 'C147234', 'C147235', 'C147236', 'C147238', 'C147239', 'C147240', 'C147241', 'C147242', 'C147243', 'C147244', 'C147245', 'C147246', 'C147249', 'C147250', 'C147251', 'C147252', 'C147261', 'C147264', 'C147267', 'C147268', 'C147269', 'C49617', 'C49618', 'C117699', 'C49619', 'C49620', 'C147270', 'C106578', 'C53483', 'C106577', 'C49621', 'C95103', 'C102726', 'C49622']" +FA,2,define_variable_codelist_coded_codes,['C00002'] +VS,2,$domain_lib_ccode,"['C49563', 'C49562', 'C117755', 'C147168', 'C147169', 'C147170', 'C132354', 'C147171', 'C147172', 'C147173', 'C147174', 'C147175', 'C147176', 'C147177', 'C147178', 'C147179', 'C111282', 'C111138', 'C111137', 'C85441', 'C49568', 'C49569', 'C102605', 'C49578', 'C95087', 'C102616', 'C102618', 'C49572', 'C102619', 'C102620', 'C49576', 'C102621', 'C102622', 'C49585', 'C102617', 'C117466', 'C102630', 'C49626', 'C49587', 'C85442', 'C117756', 'C102640', 'C102641', 'C117757', 'C61536', 'C102651', 'C112320', 'C49592', 'C49602', 'C49603', 'C95095', 'C102674', 'C49604', 'C102671', 'C61531', 'C102677', 'C147180', 'C161329', 'C106552', 'C49606', 'C49608', 'C111289', 'C61529', 'C147181', 'C49607', 'C102700', 'C49609', 'C95098', 'C147182', 'C147183', 'C102707', 'C107097', 'C106571', 'C49610', 'C49616', 'C147184', 'C112420', 'C117655', 'C49615', 'C147185', 'C147186', 'C147187', 'C147189', 'C147191', 'C147193', 'C147195', 'C147196', 'C147197', 'C147198', 'C147200', 'C147201', 'C147203', 'C147204', 'C147205', 'C147206', 'C147207', 'C147208', 'C147209', 'C147210', 'C147211', 'C147212', 'C147216', 'C147218', 'C147219', 'C147220', 'C147222', 'C147223', 'C147224', 'C147225', 'C147226', 'C165871', 'C147227', 'C147228', 'C147229', 'C147232', 'C147233', 'C147234', 'C147235', 'C147236', 'C147238', 'C147239', 'C147240', 'C147241', 'C147242', 'C147243', 'C147244', 'C147245', 'C147246', 'C147249', 'C147250', 'C147251', 'C147252', 'C147261', 'C147264', 'C147267', 'C147268', 'C147269', 'C49617', 'C49618', 'C117699', 'C49619', 'C49620', 'C147270', 'C106578', 'C53483', 'C106577', 'C49621', 'C95103', 'C102726', 'C49622']" +VS,2,define_variable_codelist_coded_codes,['C00004'] diff --git a/Published/CORE-000929/negative/02/data/.env b/Published/CORE-000929/negative/02/data/.env index b29d60c7b..c21b28051 100644 --- a/Published/CORE-000929/negative/02/data/.env +++ b/Published/CORE-000929/negative/02/data/.env @@ -1,4 +1,3 @@ PRODUCT=SDTMIG VERSION=3-4 -CT=sdtmct2020-12-18 DEFINE_XML=define_negative2.xml diff --git a/Published/CORE-000929/positive/01/data/.env b/Published/CORE-000929/positive/01/data/.env index f3ff11581..a34647d8a 100644 --- a/Published/CORE-000929/positive/01/data/.env +++ b/Published/CORE-000929/positive/01/data/.env @@ -1,4 +1,3 @@ PRODUCT=SDTMIG VERSION=3-3 -CT=sdtmct2020-12-18 DEFINE_XML=define_positive1.xml diff --git a/test.py b/test.py index 416fed954..33990c229 100644 --- a/test.py +++ b/test.py @@ -5,6 +5,7 @@ and writes results.json into each case's results/ directory. """ +import os import sys import subprocess from pathlib import Path @@ -13,6 +14,8 @@ ENGINE_DIR = Path("engine") CONVERT_SCRIPT = Path(".github/scripts/convert_results.py") +LOG_LEVELS = ["info", "debug", "error", "critical", "disabled", "warn"] + # --------------------------------------------------------------------------- # Rule folder helpers # --------------------------------------------------------------------------- @@ -83,7 +86,13 @@ def next_results_path(results_dir: Path) -> Path: # Engine invocation # --------------------------------------------------------------------------- -def run_engine(rule_yml: Path, data_dir: Path, output_path: Path) -> tuple[bool, str]: +def run_engine( + rule_yml: Path, + data_dir: Path, + output_path: Path, + log_level: str, + capture_logs: bool, +) -> tuple[bool, str]: env_file = find_env_file(data_dir) if env_file is None: return False, f"No .env file found in {data_dir}" @@ -96,11 +105,36 @@ def run_engine(rule_yml: Path, data_dir: Path, output_path: Path) -> tuple[bool, "-of", "JSON", "-o", str(output_path.resolve()), "-p", "disabled", + "-l", log_level, ] try: - proc = subprocess.run(cmd, cwd=ENGINE_DIR, capture_output=True, text=True) - output = (proc.stdout + proc.stderr).strip() + env = os.environ.copy() + + stream_output = log_level != "disabled" + lines: list[str] = [] + with subprocess.Popen( + cmd, + cwd=ENGINE_DIR, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + ) as proc: + for line in proc.stdout: + if stream_output: + print(f" {line}", end="") + lines.append(line) + proc.wait() + + output = "".join(lines).strip() + + if capture_logs and output: + log_path = output_path.parent / f"{output_path.name}_engine.log.txt" + log_path.write_text(output, encoding="utf-8") + print(f" Log captured — {log_path}") + return proc.returncode == 0, output except Exception as e: return False, str(e) @@ -111,7 +145,12 @@ def run_engine(rule_yml: Path, data_dir: Path, output_path: Path) -> tuple[bool, # --------------------------------------------------------------------------- -def run_rule(rule_path: Path, specific_case: Optional[str] = None): +def run_rule( + rule_path: Path, + specific_case: Optional[str], + log_level: str, + capture_logs: bool, +): rule_yml = find_rule_yml(rule_path) all_cases = get_test_cases(rule_path) @@ -134,7 +173,7 @@ def run_rule(rule_path: Path, specific_case: Optional[str] = None): output_path = next_results_path(case["results_dir"]) print(f"\n Running {test_type}/{case_id}...") - ok, output = run_engine(rule_yml, data_dir, output_path) + ok, output = run_engine(rule_yml, data_dir, output_path, log_level, capture_logs) json_path = Path(str(output_path) + ".json") if ok and json_path.exists(): @@ -212,15 +251,37 @@ def prompt_case(cases: Dict[str, List[dict]]) -> Optional[str]: print("Invalid — try again.") +def prompt_log_level() -> str: + print("\nLog level options:") + for i, level in enumerate(LOG_LEVELS, 1): + print(f" {i}. {level}") + while True: + choice = input("Select log level (default: disabled): ").strip().lower() + if not choice: + return "disabled" + if choice in LOG_LEVELS: + return choice + if choice.isdigit() and 1 <= int(choice) <= len(LOG_LEVELS): + return LOG_LEVELS[int(choice) - 1] + print("Invalid — try again.") + + +def prompt_capture_logs() -> bool: + choice = input("Capture engine logs to results folder? (y/N): ").strip().lower() + return choice == "y" + + # --------------------------------------------------------------------------- # Entry point # --------------------------------------------------------------------------- def main(): - rule_path = prompt_rule_path() - cases = get_test_cases(rule_path) - specific = prompt_case(cases) - run_rule(rule_path, specific) + rule_path = prompt_rule_path() + cases = get_test_cases(rule_path) + specific = prompt_case(cases) + log_level = prompt_log_level() + capture_logs = prompt_capture_logs() + run_rule(rule_path, specific, log_level, capture_logs) if __name__ == "__main__": @@ -228,4 +289,4 @@ def main(): main() except KeyboardInterrupt: print("\nInterrupted.") - sys.exit(1) + sys.exit(1) \ No newline at end of file From 2d4761456c93af91c87af4c221861ab75626f24c Mon Sep 17 00:00:00 2001 From: Samuel Johnson Date: Tue, 19 May 2026 15:56:39 -0400 Subject: [PATCH 2/2] logs & capture --- README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index b82af8dc6..339278d96 100644 --- a/README.md +++ b/README.md @@ -152,7 +152,7 @@ Unpublished/ **`.env`** Specifies the standard and version to validate against. `PRODUCT` and `VERSION` are required; `SUBSTANDARD`, `USE_CASE`, `CT`, and `DEFINE_XML` are optional depending on the rule. - + See [`.env.example`](./engine/.env.example) for example env environment variables. ``` PRODUCT=TIG VERSION=1-0 @@ -179,14 +179,14 @@ Unpublished/ ``` dataset,variable,label,type,length - cm,STUDYID,Study Identifier,Char,50 - cm,DOMAIN,Domain Abbreviation,Char,50 - cm,USUBJID,Unique Subject Identifier,Char,50 - cm,CMTRT,"Reported Name of Drug, Med, or Therapy",Char,50 - dm,STUDYID,Study Identifier,Char,50 - dm,DOMAIN,Domain Abbreviation,Char,50 - dm,USUBJID,Unique Subject Identifier,Char,50 - dm,AGE,Age,Num,8 + CM,STUDYID,Study Identifier,Char,50 + CM,DOMAIN,Domain Abbreviation,Char,50 + CM,USUBJID,Unique Subject Identifier,Char,50 + CM,CMTRT,"Reported Name of Drug, Med, or Therapy",Char,50 + DM,STUDYID,Study Identifier,Char,50 + DM,DOMAIN,Domain Abbreviation,Char,50 + DM,USUBJID,Unique Subject Identifier,Char,50 + DM,AGE,Age,Num,8 ... ```