From 4ef65d9874148625b6f00edfc42c90aea2d5194d Mon Sep 17 00:00:00 2001 From: EMontandon Date: Tue, 26 May 2026 15:50:53 +0200 Subject: [PATCH] fix(RR dataelements): wrong indicator names + ref to NER file --- ...snt_dhis2_reporting_rate_dataelement.ipynb | 128 ++++++++++-------- 1 file changed, 70 insertions(+), 58 deletions(-) diff --git a/pipelines/snt_dhis2_reporting_rate_dataelement/code/snt_dhis2_reporting_rate_dataelement.ipynb b/pipelines/snt_dhis2_reporting_rate_dataelement/code/snt_dhis2_reporting_rate_dataelement.ipynb index 68468ba..10ff9c3 100644 --- a/pipelines/snt_dhis2_reporting_rate_dataelement/code/snt_dhis2_reporting_rate_dataelement.ipynb +++ b/pipelines/snt_dhis2_reporting_rate_dataelement/code/snt_dhis2_reporting_rate_dataelement.ipynb @@ -14,7 +14,7 @@ "tags": [] }, "source": [ - "# Calcualate Reporting Rate (method: \"Data Element\")\n", + "# Calculate Reporting Rate (method: \"Data Element\")\n", "Based on facilities reporting data for malaria-related indicators" ] }, @@ -78,33 +78,31 @@ }, { "cell_type": "markdown", - "id": "a7a15634-4623-40f2-8e2d-3fa47203aa6e", + "id": "7dedcc32-c531-498d-90b9-89b0ee9fb9be", "metadata": { "papermill": { - "duration": 0.00011, - "end_time": "2026-01-16T10:23:56.165873", + "duration": 9.5e-05, + "end_time": "2026-01-16T10:23:56.200231", "exception": false, - "start_time": "2026-01-16T10:23:56.165763", + "start_time": "2026-01-16T10:23:56.200136", "status": "completed" }, "tags": [] }, "source": [ - "### 1.1. Fallback parameters values\n", - "This parameters are injected by papermill when running in OH via pipeline run interface.
\n", - "The code cell below here provides fallback paramater values needed when running this notebook locally." + "### 1.2. Load and check `snt config` file" ] }, { "cell_type": "code", "execution_count": null, - "id": "b17f7685-5291-4e5d-9eec-2d1f9435fccb", + "id": "5b6d29ea-91f3-4c53-b95e-4b485f88383f", "metadata": { "papermill": { - "duration": 0.033954, - "end_time": "2026-01-16T10:23:56.199937", + "duration": 0.521572, + "end_time": "2026-01-16T10:23:56.721932", "exception": false, - "start_time": "2026-01-16T10:23:56.165983", + "start_time": "2026-01-16T10:23:56.200360", "status": "completed" }, "tags": [], @@ -114,41 +112,51 @@ }, "outputs": [], "source": [ - "if (!exists(\"ROUTINE_FILE\")) {ROUTINE_FILE <- \"NER_routine_outliers-mean_imputed.parquet\"}\n", + "# Load SNT config\n", + "config_json <- tryCatch({ jsonlite::fromJSON(file.path(CONFIG_PATH, \"SNT_config.json\")) },\n", + " error = function(e) {\n", + " msg <- paste0(\"[ERROR] Error while loading configuration\", conditionMessage(e)) \n", + " cat(msg) \n", + " stop(msg) \n", + " })\n", "\n", - "if (!exists(\"DATAELEMENT_METHOD_DENOMINATOR\")) {DATAELEMENT_METHOD_DENOMINATOR <- \"ROUTINE_ACTIVE_FACILITIES\"} # or \"PYRAMID_OPEN_FACILITIES\"\n", - "if (!exists(\"ACTIVITY_INDICATORS\")) {ACTIVITY_INDICATORS <- c(\"CONF\", \"PRES\", \"SUSP\")} \n", - "if (!exists(\"VOLUME_ACTIVITY_INDICATORS\")) {VOLUME_ACTIVITY_INDICATORS <- c(\"CONF\", \"PRES\")}\n", - "if (!exists(\"USE_WEIGHTED_REPORTING_RATES\")) {USE_WEIGHTED_REPORTING_RATES <- FALSE}" + "log_msg(paste0(\"SNT configuration loaded from : \", file.path(CONFIG_PATH, \"SNT_config.json\")))\n", + "\n", + "# Configuration settings\n", + "COUNTRY_CODE <- config_json$SNT_CONFIG$COUNTRY_CODE\n", + "ADMIN_1 <- toupper(config_json$SNT_CONFIG$DHIS2_ADMINISTRATION_1)\n", + "ADMIN_2 <- toupper(config_json$SNT_CONFIG$DHIS2_ADMINISTRATION_2)" ] }, { "cell_type": "markdown", - "id": "7dedcc32-c531-498d-90b9-89b0ee9fb9be", + "id": "a7a15634-4623-40f2-8e2d-3fa47203aa6e", "metadata": { "papermill": { - "duration": 0.000095, - "end_time": "2026-01-16T10:23:56.200231", + "duration": 0.00011, + "end_time": "2026-01-16T10:23:56.165873", "exception": false, - "start_time": "2026-01-16T10:23:56.200136", + "start_time": "2026-01-16T10:23:56.165763", "status": "completed" }, "tags": [] }, "source": [ - "### 1.2. Load and check `snt config` file" + "### 1.3. Fallback parameters values\n", + "This parameters are injected by papermill when running in OH via pipeline run interface.
\n", + "The code cell below here provides fallback paramater values needed when running this notebook locally." ] }, { "cell_type": "code", "execution_count": null, - "id": "5b6d29ea-91f3-4c53-b95e-4b485f88383f", + "id": "b17f7685-5291-4e5d-9eec-2d1f9435fccb", "metadata": { "papermill": { - "duration": 0.521572, - "end_time": "2026-01-16T10:23:56.721932", + "duration": 0.033954, + "end_time": "2026-01-16T10:23:56.199937", "exception": false, - "start_time": "2026-01-16T10:23:56.200360", + "start_time": "2026-01-16T10:23:56.165983", "status": "completed" }, "tags": [], @@ -158,46 +166,42 @@ }, "outputs": [], "source": [ - "# Load SNT config\n", - "config_json <- tryCatch({ jsonlite::fromJSON(file.path(CONFIG_PATH, \"SNT_config.json\")) },\n", - " error = function(e) {\n", - " msg <- paste0(\"[ERROR] Error while loading configuration\", conditionMessage(e)) \n", - " cat(msg) \n", - " stop(msg) \n", - " })\n", - "\n", - "log_msg(paste0(\"SNT configuration loaded from : \", file.path(CONFIG_PATH, \"SNT_config.json\")))" + "if (!exists(\"ROUTINE_FILE\")) {ROUTINE_FILE <- glue(\"{COUNTRY_CODE}_routine_outliers_imputed.parquet\")}\n", + "if (!exists(\"DATAELEMENT_METHOD_DENOMINATOR\")) {DATAELEMENT_METHOD_DENOMINATOR <- \"ROUTINE_ACTIVE_FACILITIES\"} # or \"PYRAMID_OPEN_FACILITIES\"\n", + "if (!exists(\"ACTIVITY_INDICATORS\")) {ACTIVITY_INDICATORS <- c(\"CONF\", \"PRES\", \"SUSP\")} \n", + "if (!exists(\"VOLUME_ACTIVITY_INDICATORS\")) {VOLUME_ACTIVITY_INDICATORS <- c(\"CONF\", \"PRES\")}\n", + "if (!exists(\"USE_WEIGHTED_REPORTING_RATES\")) {USE_WEIGHTED_REPORTING_RATES <- FALSE}" ] }, { - "cell_type": "code", - "execution_count": null, - "id": "c26c981c-dadd-48ac-ae35-613b8ba61a82", + "cell_type": "markdown", + "id": "2285e5ea-72da-415c-adf7-49ce49a3ab63", "metadata": { "papermill": { - "duration": 0.033003, - "end_time": "2026-01-16T10:23:56.755117", + "duration": 0.00011, + "end_time": "2026-01-16T10:23:56.165873", "exception": false, - "start_time": "2026-01-16T10:23:56.722114", + "start_time": "2026-01-16T10:23:56.165763", "status": "completed" }, - "tags": [], - "vscode": { - "languageId": "r" - } + "tags": [] }, + "source": [ + "### 1.4. Set parameters\n", + "Set paramater values needed to run this notebook." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c340cdb0-1b68-4255-9095-27006d0f9b0b", + "metadata": {}, "outputs": [], "source": [ - "# Configuration settings\n", - "COUNTRY_CODE <- config_json$SNT_CONFIG$COUNTRY_CODE\n", - "ADMIN_1 <- toupper(config_json$SNT_CONFIG$DHIS2_ADMINISTRATION_1)\n", - "ADMIN_2 <- toupper(config_json$SNT_CONFIG$DHIS2_ADMINISTRATION_2)\n", - "\n", "# DHIS2_INDICATORS <- names(config_json$DHIS2_DATA_DEFINITIONS$DHIS2_INDICATOR_DEFINITIONS) \n", "# Only consider the core 4 indicators \n", "# (Referring to SNT_config.json might be problematic if the indicators are set but empty. i.e., with no valid de id!)\n", - "DHIS2_INDICATORS <- c(\"CONF\", \"PRES\", \"SUSP\", \"TESTS\") \n", - "\n", + "DHIS2_INDICATORS <- c(\"CONF\", \"PRES\", \"SUSP\", \"TEST\")\n", "ACTIVITY_INDICATORS <- unlist(ACTIVITY_INDICATORS)\n", "VOLUME_ACTIVITY_INDICATORS <- unlist(VOLUME_ACTIVITY_INDICATORS)\n", "fixed_cols <- c('PERIOD', 'YEAR', 'MONTH', 'ADM1_ID', 'ADM2_ID', 'OU_ID')\n", @@ -209,7 +213,7 @@ "id": "e44ae2ab-4af7-475a-8cbe-6d669895a18b", "metadata": { "papermill": { - "duration": 0.000093, + "duration": 9.3e-05, "end_time": "2026-01-16T10:23:56.779812", "exception": false, "start_time": "2026-01-16T10:23:56.779719", @@ -226,7 +230,7 @@ "id": "39e2add7-bbc7-4312-9a6f-9886d675f532", "metadata": { "papermill": { - "duration": 0.000069, + "duration": 6.9e-05, "end_time": "2026-01-16T10:23:56.779987", "exception": false, "start_time": "2026-01-16T10:23:56.779918", @@ -388,7 +392,7 @@ "id": "bcbd3a9f-5e45-4ae5-8671-e23155236295", "metadata": { "papermill": { - "duration": 0.000091, + "duration": 9.1e-05, "end_time": "2026-01-16T10:23:59.817949", "exception": false, "start_time": "2026-01-16T10:23:59.817858", @@ -657,7 +661,7 @@ "id": "160c08ec-cc9a-4e1a-99ec-f703db83a71d", "metadata": { "papermill": { - "duration": 0.000098, + "duration": 9.8e-05, "end_time": "2026-01-16T10:24:05.948452", "exception": false, "start_time": "2026-01-16T10:24:05.948354", @@ -869,7 +873,7 @@ "id": "c75f2249", "metadata": { "papermill": { - "duration": 0.000057, + "duration": 5.7e-05, "end_time": "2026-01-16T10:24:07.310743", "exception": false, "start_time": "2026-01-16T10:24:07.310686", @@ -1079,7 +1083,7 @@ "id": "2866816a-7015-4c5c-b904-f553f3b4790d", "metadata": { "papermill": { - "duration": 0.000088, + "duration": 8.8e-05, "end_time": "2026-01-16T10:24:09.592563", "exception": false, "start_time": "2026-01-16T10:24:09.592475", @@ -1122,6 +1126,14 @@ "write.csv(reporting_rate_dataelement, file_path, row.names = FALSE)\n", "log_msg(glue(\"Exported : {file_path}\"))" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8ebed94f-2a13-4f90-b7da-5a4b14352bd9", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": {