From 13e252169b9463ade7bab3a793ac5d90947171c5 Mon Sep 17 00:00:00 2001 From: hdpriest Date: Fri, 5 Dec 2025 09:54:11 -0600 Subject: [PATCH 1/3] final changes for deployment to icrn dev; kernel; catalog discovery now based on location of icrn_manager script; primary method of environment discovery is the location of the icrn_manager script; kernel discovery is relative --- Deploy.md | 23 ++++++++++++++++++++--- icrn_manager | 5 ++++- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/Deploy.md b/Deploy.md index 23677f8..f1fd73d 100644 --- a/Deploy.md +++ b/Deploy.md @@ -2,9 +2,24 @@ ## get the most recent changes moved to campus cluster +#### ONCE +Move or re-create the central repo located at `/sw/icrn/jupyter/icrn_ncsa_resources/Kernels/` to +`/sw/icrn/dev/kernels/` -## Changes needed to ICRN containers in dev +copy tools to path loc (dev paths): +```sh +cp ./icrn_manager /sw/icrn/dev/bin/ +cp ./update_r_libs.sh /sw/icrn/dev/bin/ +chmod +x /sw/icrn/dev/bin/icrn_manager +chmod +x /sw/icrn/dev/bin/update_r_libs.sh +``` +Change Kernel Central Repo Path in icrn_manager to reflect environment +-- what is the signal in icrn that we are in prod/dev? -- + + +## Changes needed to ICRN containers in dev +JQ installed on container ## running the kernel index process @@ -18,6 +33,9 @@ Tested on campus cluster via apptainer, this indexes the central repository of k (base) [hdpriest@cc-login2 icrn_manager]$ apptainer pull docker://hdpriest0uiuc/icrn-kernel-indexer (base) [hdpriest@cc-login2 icrn_manager]$ apptainer run --bind /sw/icrn/jupyter/icrn_ncsa_resources/Kernels:/sw/icrn/jupyter/icrn_ncsa_resources/Kernels icrn-kernel-indexer_latest.sif + +#### DEV +(base) [hdpriest@cc-login2 icrn_manager]$ apptainer run --bind /sw/icrn/dev/kernels:/sw/icrn/dev/kernels icrn-kernel-indexer_latest.sif ## ... output Collated manifest written to: /sw/icrn/jupyter/icrn_ncsa_resources/Kernels/collated_manifests.json ## ... output @@ -36,8 +54,7 @@ docker build -t icrn-web -f web/Dockerfile web/ Assuming you have a built container (obtained from dockerhub, or built locally), the container expects bind mounts to the location where the manifests/index are kept, and so is flexible for its back-end storage as long as it has access to the same disk mount as the index job: ```bash docker run -d -p 8080:80 --name icrn-web \ - -v /sw/icrn/jupyter/icrn_ncsa_resources/Kernels/collated_manifests.json:/app/data/collated_manifests.json \ - -v /sw/icrn/jupyter/icrn_ncsa_resources/Kernels/package_index.json:/app/data/package_index.json \ + -v /sw/icrn/dev/kernels:/app/data:ro \ icrn-web ``` diff --git a/icrn_manager b/icrn_manager index d568647..09d64bd 100755 --- a/icrn_manager +++ b/icrn_manager @@ -13,7 +13,10 @@ fi icrn_base=".icrn" icrn_kernels="icrn_kernels" -central_catalog_default="/sw/icrn/jupyter/icrn_ncsa_resources/Kernels" +APP_FOLDER=$(dirname $0) +KERNEL_FOLDER=${APP_FOLDER}"/../kernels" +if [ ! -d ${KERNEL_FOLDER} ]; then echo "Could not determine location of kernel respository - contact administrator"; exit -1 ; fi +central_catalog_default=${KERNEL_FOLDER} ICRN_USER_BASE=${ICRN_USER_BASE:-${HOME}/${icrn_base}} ICRN_MANAGER_CONFIG=${ICRN_MANAGER_CONFIG:-${ICRN_USER_BASE}/manager_config.json} From 3c6f5a3a62451ed4a8b189e4f7ec548cfc43eb61 Mon Sep 17 00:00:00 2001 From: hdpriest Date: Fri, 5 Dec 2025 11:35:42 -0600 Subject: [PATCH 2/3] changed command line behavior of icrn_manager to respect a passed 'KERNEL_FOLDER' env var, and fall-back to 'APP_FOLDER/../kernels' otherwise (useful for leveraging icrn_manager script not-located in standard dev deployment location) --- icrn_manager | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/icrn_manager b/icrn_manager index 09d64bd..3d59f59 100755 --- a/icrn_manager +++ b/icrn_manager @@ -14,7 +14,7 @@ fi icrn_base=".icrn" icrn_kernels="icrn_kernels" APP_FOLDER=$(dirname $0) -KERNEL_FOLDER=${APP_FOLDER}"/../kernels" +KERNEL_FOLDER=${KERNEL_FOLDER:-${APP_FOLDER}/../kernels} if [ ! -d ${KERNEL_FOLDER} ]; then echo "Could not determine location of kernel respository - contact administrator"; exit -1 ; fi central_catalog_default=${KERNEL_FOLDER} From 43d4cbdbc8b33eefa98ab58c98b8144e9f886c9c Mon Sep 17 00:00:00 2001 From: Henry Priest Date: Fri, 5 Dec 2025 12:36:48 -0600 Subject: [PATCH 3/3] added kernel_root override based on CLI passed env vars. --- Deploy.md | 8 +++++++- kernel-indexer/README.md | 6 +++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/Deploy.md b/Deploy.md index f1fd73d..af33533 100644 --- a/Deploy.md +++ b/Deploy.md @@ -33,9 +33,15 @@ Tested on campus cluster via apptainer, this indexes the central repository of k (base) [hdpriest@cc-login2 icrn_manager]$ apptainer pull docker://hdpriest0uiuc/icrn-kernel-indexer (base) [hdpriest@cc-login2 icrn_manager]$ apptainer run --bind /sw/icrn/jupyter/icrn_ncsa_resources/Kernels:/sw/icrn/jupyter/icrn_ncsa_resources/Kernels icrn-kernel-indexer_latest.sif +``` + +For dev and prod environments, the kernel index job needs to be run using a custom kernel root env variable (see directly below) +this will obviously be different based on the container runtime +```sh #### DEV -(base) [hdpriest@cc-login2 icrn_manager]$ apptainer run --bind /sw/icrn/dev/kernels:/sw/icrn/dev/kernels icrn-kernel-indexer_latest.sif +(base) [hdpriest@cc-login2 icrn_manager]$ +apptainer run --env "KERNEL_ROOT=/sw/icrn/dev/kernels" --bind /sw/icrn/dev/kernels:/sw/icrn/dev/kernels icrn-kernel-indexer_latest.sif ## ... output Collated manifest written to: /sw/icrn/jupyter/icrn_ncsa_resources/Kernels/collated_manifests.json ## ... output diff --git a/kernel-indexer/README.md b/kernel-indexer/README.md index b4b8d55..df9d04d 100644 --- a/kernel-indexer/README.md +++ b/kernel-indexer/README.md @@ -30,7 +30,7 @@ docker build -t icrn-kernel-indexer:latest . ```bash docker run --rm \ - -v /sw/icrn/jupyter/icrn_ncsa_resources/Kernels:/sw/icrn/jupyter/icrn_ncsa_resources/Kernels \ + -v /sw/icrn/dev/kernels:/sw/icrn/dev/kernels \ icrn-kernel-indexer:latest ``` @@ -38,8 +38,8 @@ docker run --rm \ ```bash docker run --rm \ - -v /sw/icrn/jupyter/icrn_ncsa_resources/Kernels:/sw/icrn/jupyter/icrn_ncsa_resources/Kernels \ - -e KERNEL_ROOT=/sw/icrn/jupyter/icrn_ncsa_resources/Kernels \ + -v /sw/icrn/dev/kernels:/sw/icrn/dev/kernels \ + -e KERNEL_ROOT=/sw/icrn/dev/kernels \ -e LANGUAGE_FILTER=Python \ -e LOG_LEVEL=DEBUG \ icrn-kernel-indexer:latest