Skip to content

NASA-GISS/docker

BranchesTags

Repository files navigation

Table of contents

Docker

  • Instructions on using Docker to run ModelE
  • The Docker container contains all the tools needed to compile and run ModelE such as gfortran, gdb, netcdf, etc.
  • ModelE source code and ModelE_Support (input files) are assumed to be on the host computer, and accessed via mount points.

Build container

docker build -t modele -f Dockerfile.ubuntu_2204 . # Ubuntu 22.04 base image
# docker tag local-image:tagname new-repo:tagname
docker tag modele mankoff/modele:ubuntu_2204

docker build -t modele -f Dockerfile.oracle_9 .  # Oracle 9 base image
docker tag modele mankoff/modele:oracle_9

docker build -t modele -f Dockerfile.debian_trixie .  # Debian Trixie
docker tag modele mankoff/modele:debian_trixie

# Deploy for sharing
# Note: TEST (see below) before push
docker login -u "user" -p "pass" docker.io
docker push mankoff/modele:ubuntu_2204
docker push mankoff/modele:oracle_9
docker push mankoff/modele:debian_trixie

Test

Basic test

# run it with default CMD
docker run -it mankoff/modele:debian_trixie

# run it with custom command
docker run -it mankoff/modele:debian_trixie gfortran --version

# run interactively
docker run -it mankoff/modele:debian_trixie bash
exit

ModelE environment

  • Set ${MODELE_SRC} to your /path/to/modelE source code directory.
  • Set ${MODELE_SUPPORT} to your /path/to/ModelE_Support.
  • Set ${GCMSEARCHPATH} to your /path/to/prod_input_files (E2) or /path/to/hier_input_files (E3)
  • Place your ${MODELERC} config file in this source code directory (normally found in your home directory)
    • Name it modelErc (no leading dot)
    • Paths should expect:
      • Source at /modelE
      • ModelE_Support at /ModelE_Support
      • input files at /prod_input_files (E2) or /hier_input_files (E3)

Example modelErc file follows. Swap GCMSEARCHPATH if running E2 vs E3.

DECKS_REPOSITORY=/ModelE_Support/prod_decks
CMRUNDIR=/ModelE_Support/prod_runs
#GCMSEARCHPATH=/prod_input_files
GCMSEARCHPATH=/hier_input_files
EXECDIR=/ModelE_Support/exec
SAVEDISK=/ModelE_Support/huge_space
NETCDFHOME=/usr
NETCDFLIBDIR=/usr/lib/x86_64-linux-gnu

PNETCDFHOME=/usr
PNETCDFLIBDIR=/usr/lib/x86_64-linux-gnu
# FFLAGS="-I/usr/include/x86_64-linux-gnu" and OPTS_dd2d = NC_IO=PNETCDF in .R

MPI=YES
MPIDISTR=openmpi
MPIDIR=/usr
MPILIBDIR=/usr/lib/x86_64-linux-gnu/openmpi/lib
MPIINCLUDEDIR=/usr/lib/x86_64-linux-gnu/openmpi/include/
ABI=64
COMPILER=gfortran
MAILTO=
UMASK=022
OVERWRITE=NO
OUTPUT_TO_FILES=YES
VERBOSE_OUTPUT=YES

MODELE_SRC=${HOME}/projects/GISS/E3
MODELE_SUPPORT=${HOME}/data/ModelE_Support
GCMSEARCHPATH=${HOME}/data/ModelE_Support/hier_input_files

# on the host computer, copy your ${{MODELERC} file to the source folder where the container expects it
cp ~/.modelErc ${MODELE_SRC}/modelErc

#       --mount type=bind,src=${HOME},dst=/home/user \
#       --mount type=bind,src=${GCMSEARCHPATH},dst=/prod_input_files E2
#       --mount type=bind,src=${GCMSEARCHPATH},dst=/hier_input_files E3

docker run -it --user $(id -u):$(id -g) \
       --mount type=bind,src=${MODELE_SRC},dst=/modelE \
       --mount type=bind,src=${MODELE_SUPPORT},dst=/ModelE_Support \
       --mount type=bind,src=${GCMSEARCHPATH},dst=/hier_input_files \
       mankoff/modele:debian_trixie bash
ls -l /modelE
ls -l /ModelE_Support
echo $MODELERC
ls -l $MODELERC
exit

Compile and run modelE

The example below are for local testing, off discover, where docker is installed. See section on Singularity for running on discover

E2.1_branch

Enter container

MODELE_SRC=${HOME}/projects/GISS/E2.1_branch
MODELE_SUPPORT=${HOME}/data/ModelE_Support
GCMSEARCHPATH=${HOME}/data/ModelE_Support/prod_input_files

docker run -it \
       --mount type=bind,src=${MODELE_SRC},dst=/modelE \
       --mount type=bind,src=${MODELE_SUPPORT},dst=/ModelE_Support \
       --mount type=bind,src=${GCMSEARCHPATH},dst=/prod_input_files \
       --user $(id -u):$(id -g) \
       mankoff/modele:debian_trixie \
       bash

Compile

cd /modelE/decks/

# 8x10 lat x lon; no ocean; model month takes O(minute)
RUNNAME=docker_test01
make rundeck RUNSRC=E1oM20 RUN=${RUNNAME} OVERWRITE=YES
# fetch input files for this run
../exec/get_input_data -w ${RUNNAME} /ModelE_Support/prod_input_files/

# # 4x5 lat x lon; ocean; model month takes 10x longer
# RUNNAME=docker_test02
# make rundeck RUNSRC=E1oM20 RUN=${RUNNAME}

make clean RUN=${RUNNAME}

# compile for debugging

make -j setup RUN=${RUNNAME} COMPILE_WITH_TRAPS=YES EXTRA_FFLAGS="-g -O0 -fallow-argument-mismatch -I/usr/include/x86_64-linux-gnu " EXTRA_LFLAGS="-g -O0" > >(tee ${RUNNAME}.compile.out) 2> >(tee ${RUNNAME}.compile.err >&2); mv ${RUNNAME}.compile.{out,err} ${RUNNAME}/

# compile for fast run
make -j setup RUN=${RUNNAME} COMPILE_WITH_TRAPS=NO EXTRA_FFLAGS="-O2 -fallow-argument-mismatch -I/usr/include/x86_64-linux-gnu -I/usr/include/openmpi-x86_64 " EXTRA_LFLAGS="-O2" > >(tee ${RUNNAME}.compile.out) 2> >(tee ${RUNNAME}.compile.err >&2); mv ${RUNNAME}.compile.{out,err} ${RUNNAME}/

Run

../exec/runE ${RUNNAME} -cold-restart -np 2
# Should see: Run docker_test01 completed successfully

# debug
(cd ${RUNNAME}; ./${RUNNAME}ln; gdb --args ${RUNNAME}.bin -i I -cold-restart) # debug
# br model/solvers/TRIDIAG.f:40

../exec/runE ${RUNNAME} -np 8

E3

Enter container

MODELE_SRC=${HOME}/projects/GISS/E3
MODELE_SUPPORT=${HOME}/data/ModelE_Support
GCMSEARCHPATH=${HOME}/data/ModelE_Support/hier_input_files

docker run -it \
       --mount type=bind,src=${MODELE_SRC},dst=/modelE \
       --mount type=bind,src=${MODELE_SUPPORT},dst=/ModelE_Support \
       --mount type=bind,src=${GCMSEARCHPATH},dst=/hier_input_files \
       --user $(id -u):$(id -g) \
       mankoff/modele:debian_trixie \
       bash

Compile

cd /modelE/decks/

# compile rdex
(cd ../rdex && gfortran rdex.f90 -o rdex && cp rdex /ModelE_Support/exec)

RUNNAME=docker_E3_001

make rundeck RUNSEEDS=AOLite,year2000 RUN=${RUNNAME} OVERWRITE=YES

# fetch input files for this run
# WARNING: This step not automated or scriptable yet.
# ../exec/get_input_data -w ${RUNNAME} /ModelE_Support/prod_input_files/

make clean RUN=${RUNNAME}

# compile for debugging

make -j setup RUN=${RUNNAME} COMPILE_WITH_TRAPS=YES EXTRA_FFLAGS="-g -O0 -fallow-argument-mismatch -I/usr/include/x86_64-linux-gnu " EXTRA_LFLAGS="-g -O0"  > >(tee ${RUNNAME}.compile.out) 2> >(tee ${RUNNAME}.compile.err >&2); mv ${RUNNAME}.compile.{out,err} ${RUNNAME}/

# compile for fast run
make -j setup RUN=${RUNNAME} COMPILE_WITH_TRAPS=NO EXTRA_FFLAGS="-O2 -fallow-argument-mismatch -I/usr/include/x86_64-linux-gnu -I/usr/include/openmpi-x86_64 " EXTRA_LFLAGS="-O2" > >(tee ${RUNNAME}.compile.out) 2> >(tee ${RUNNAME}.compile.err >&2); mv ${RUNNAME}.compile.{out,err} ${RUNNAME}/

Run

../exec/runE ${RUNNAME} -cold-restart -np 2
# Should see: Run docker_E3_001 completed successfully

# debug
(cd ${RUNNAME}; ./${RUNNAME}ln; gdb --args ${RUNNAME}.bin -i I -cold-restart) # debug
# br model/solvers/TRIDIAG.f:40

../exec/runE ${RUNNAME} -np 8

Singularity

Pulling down VMs can take some space, don’t do it in ~/

cd ${NOBACKUP}
export SINGULARITY_CACHEDIR="${NOBACKUP}/.singularity"
export SINGULARITY_TMPDIR="${NOBACKUP}/.singularity"

One time setup and test

On a login node.

WARNING: When mounting /hier_input_files, links therein need to be resolved inside the singularity container. So a file that appears as /hier_input_files/foo but is actually a link to /discover/nobackup/user/path/to/bar.nc needs to have /discover/nobackup/user/path/to accessible from inside the container. Hence, several hard-coded bind options are used below.

module load singularity/4.3.0

mkdir -p local/singularity
singularity build --sandbox local/singularity/modelE_debian_trixie docker://mankoff/modele:debian_trixie
# singularity build --sandbox local/singularity/modelE_ubuntu2204 docker://mankoff/modele:ubuntu_2204

MODELE_SRC=${HOME}/projects/GISS/E3.1
MODELE_SUPPORT=${HOME}/data/ModelE_Support
GCMSEARCHPATH=${MODELE_SUPPORT}/hier_input_files/

cp ~/modelErc.singularity ${MODELE_SRC}/modelErc

singularity exec \
  --compat \
  --bind ${HOME}/projects/GISS/tmp/:/gpfsm \
  --bind ${MODELE_SRC}:/modelE \
  --bind ${MODELE_SUPPORT}:/ModelE_Support \
  --bind ${GCMSEARCHPATH}/:/hier_input_files/ \
  --bind /discover/nobackup/mkelley5/prod_input_files:/discover/nobackup/mkelley5/prod_input_files \
  --bind /discover/nobackup/projects/giss/prod_input_files/:/discover/nobackup/projects/giss/prod_input_files/ \
  --bind /discover/nobackup/projects/giss_ana/pub/Ent_TBM/EntGVSD/entgvsd_v1/entgvsd_v1.1/2HX2/soilalbedo/fill/:/discover/nobackup/projects/giss_ana/pub/Ent_TBM/EntGVSD/entgvsd_v1/entgvsd_v1.1/2HX2/soilalbedo/fill/ \
  $(pwd)/local/singularity/modelE_debian_trixie \
  bash

cd /modelE/decks
RUNNAME=singularity_E3_001

make rundeck RUNSEEDS=AOLite,year2000 RUN=${RUNNAME} OVERWRITE=YES

make clean RUN=${RUNNAME}

make -j setup RUN=${RUNNAME} COMPILE_WITH_TRAPS=NO EXTRA_FFLAGS="-g -O0 -fallow-argument-mismatch -I/usr/include/x86_64-linux-gnu " EXTRA_LFLAGS="-g -O0" > >(tee ${RUNNAME}.compile.out) 2> >(tee ${RUNNAME}.compile.err >&2); mv ${RUNNAME}.compile.{out,err} ${RUNNAME}/

# run...
../exec/runE ${RUNNAME} -cold-restart -np 2 # login node, only 1 hour and 2 cores.

# debug
(cd ${RUNNAME}; ./${RUNNAME}ln; gdb --args ${RUNNAME}.bin -i I -cold-restart)

Example ${modelErc} file:

DECKS_REPOSITORY=/ModelE_Support/prod_decks
CMRUNDIR=/ModelE_Support/prod_runs
#GCMSEARCHPATH=/prod_input_files
GCMSEARCHPATH=/hier_input_files         
EXECDIR=/ModelE_Support/exec         
SAVEDISK=/ModelE_Support/huge_space

NETCDFHOME=/usr         
NETCDFLIBDIR=/usr/lib/x86_64-linux-gnu         

MPI=YES         
MPIDISTR=openmpi         
MPIDIR=/usr         
MPILIBDIR=/usr/lib/x86_64-linux-gnu/openmpi/lib         
MPIINCLUDEDIR=/usr/lib/x86_64-linux-gnu/openmpi/include/         
ABI=64         
COMPILER=gfortran         
MAILLOT=          
UMASK=022         
OVERWRITE=YES         
OUTPUT_TO_FILES=YES         
VERBOSE_OUTPUT=YES

Running a simulation

See One time setup and test first.

TODO

xalloc_debug # enter an interactive debug node

module load singularity/4.3.0

mkdir -p local/singularity
singularity build --sandbox local/singularity/modelE_debian_trixie docker://mankoff/modele:debian_trixie
# singularity build --sandbox local/singularity/modelE_ubuntu2204 docker://mankoff/modele:ubuntu_2204

MODELE_SRC=${HOME}/projects/GISS/E3.1
MODELE_SUPPORT=${HOME}/data/ModelE_Support
GCMSEARCHPATH=${MODELE_SUPPORT}/hier_input_files/

cp ~/modelErc.singularity ${MODELE_SRC}/modelErc

singularity exec \
  --compat \
  --bind ${HOME}/projects/GISS/tmp/:/gpfsm \
  --bind ${MODELE_SRC}:/modelE \
  --bind ${MODELE_SUPPORT}:/ModelE_Support \
  --bind ${GCMSEARCHPATH}/:/hier_input_files/ \
  --bind /discover/nobackup/mkelley5/prod_input_files:/discover/nobackup/mkelley5/prod_input_files \
  --bind /discover/nobackup/projects/giss/prod_input_files/:/discover/nobackup/projects/giss/prod_input_files/ \
  --bind /discover/nobackup/projects/giss_ana/pub/Ent_TBM/EntGVSD/entgvsd_v1/entgvsd_v1.1/2HX2/soilalbedo/fill/:/discover/nobackup/projects/giss_ana/pub/Ent_TBM/EntGVSD/entgvsd_v1/entgvsd_v1.1/2HX2/soilalbedo/fill/ \
  $(pwd)/local/singularity/modelE_debian_trixie \
  bash

cd /modelE/decks
RUNNAME=singularity_E3_001

make rundeck RUNSEEDS=AOLite,year2000 RUN=${RUNNAME} OVERWRITE=YES

make clean RUN=${RUNNAME}

make -j setup RUN=${RUNNAME} COMPILE_WITH_TRAPS=NO EXTRA_FFLAGS="-g -O0 -fallow-argument-mismatch -I/usr/include/x86_64-linux-gnu " EXTRA_LFLAGS="-g -O0" > >(tee ${RUNNAME}.compile.out) 2> >(tee ${RUNNAME}.compile.err >&2); mv ${RUNNAME}.compile.{out,err} ${RUNNAME}/

# run...
../exec/runE ${RUNNAME} -cold-restart -np 88
../exec/runE ${RUNNAME} -np 88

About

Docker containers for ModelE

Resources

Readme
Activity
Custom properties

Stars

0 stars

Watchers

0 watching

Forks

0 forks
Report repository

Releases

No releases published

Packages

No packages published