Skip to content

Commit 8f4b50e

Browse files
oj-toothoj-tooth
committed
Update functional tests for OceanDataStore CLI.
1 parent d465a20 commit 8f4b50e

9 files changed

Lines changed: 241 additions & 32 deletions

‎tests/test_send_domain_to_icechunk.slurm‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
#!/bin/bash
2+
#SBATCH --job-name=test_send_to_icechunk
3+
#SBATCH --partition=test
4+
#SBATCH --time=00:20:00
5+
#SBATCH --ntasks-per-core=1
6+
#SBATCH --ntasks-per-node=64
7+
#SBATCH --ntasks-per-socket=32
8+
#SBATCH --nodes=1
9+
10+
# ==============================================================
11+
# test_send_domain_to_icechunk.slurm
12+
#
13+
# Description: SLURM script to test send_to_icechunk using the
14+
# eORCA1 ERA-5 v1 monthly-mean output variables.
15+
#
16+
# Created By: Ollie Tooth (oliver.tooth@noc.ac.uk)
17+
# Created On: 2025-09-16
18+
#
19+
# ==============================================================
20+
# -- Input arguments to msm-os -- #
21+
# Filepath to eORCA1 ancillary file:
22+
filepath_domain=/dssgfs01/scratch/npd/simulations/Domains/eORCA1/subbasins_CMIP6.nc
23+
24+
# Filepath to JASMIN OS credentials:
25+
store_credentials_json=.../credentials/jasmin_os_credentials.json
26+
27+
# Filepath to Dask Configuration file:
28+
dask_config_json=.../OceanDataStore/tests/dask_config.json
29+
30+
# Bucket and object prefix:
31+
bucket=npd-eorca1-era5v1
32+
33+
# -- Python Environment -- #
34+
# Activate miniconda environment:
35+
source /home/otooth/miniconda3/etc/profile.d/conda.sh
36+
conda activate env_jasmin_os
37+
38+
# -- Send eORCA1 ERA-5 v1 subbasins to JASMIN OS -- #
39+
prefix=test/domain_icechunk
40+
41+
echo "In Progress: Sending eORCA1 ERA-5 v1 subbasins to JASMIN object store without Dask..."
42+
ods send_to_icechunk -f $filepath_domain -c $store_credentials_json -b $bucket -p $prefix \
43+
-cs '{"x":360,"y":331}' || { echo "Error: ods send_to_icechunk failed."; exit 1; }
44+
45+
echo "Completed: Sent eORCA1 ERA-5 v1 subbasins to JASMIN object store without Dask."
46+
47+
# -- Send eORCA1 ERA-5 v1 subbasins to JASMIN OS -- #
48+
prefix=test/domain_icechunk_dask
49+
50+
echo "In Progress: Sending eORCA1 ERA-5 v1 subbasins to JASMIN object store with Dask..."
51+
ods send_to_icechunk -f $filepath_domain -c $store_credentials_json -b $bucket -p $prefix \
52+
-cs '{"x":360,"y":331}' -dc $dask_config_json || { echo "Error: ods send_to_icechunk failed."; exit 1; }
53+
54+
echo "Completed: Sent eORCA1 ERA-5 v1 subbasins to JASMIN object store with Dask."

‎tests/test_send_domain_to_zarr.slurm‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
#!/bin/bash
2+
#SBATCH --job-name=test_send_to_zarr
3+
#SBATCH --partition=test
4+
#SBATCH --time=00:20:00
5+
#SBATCH --ntasks-per-core=1
6+
#SBATCH --ntasks-per-node=64
7+
#SBATCH --ntasks-per-socket=32
8+
#SBATCH --nodes=1
9+
10+
# ==============================================================
11+
# test_send_domain_to_zarr.slurm
12+
#
13+
# Description: SLURM script to test send_to_zarr using the eORCA1
14+
# ERA-5 v1 monthly-mean output variables.
15+
#
16+
# Created By: Ollie Tooth (oliver.tooth@noc.ac.uk)
17+
# Created On: 2025-09-16
18+
#
19+
# ==============================================================
20+
# -- Input arguments to msm-os -- #
21+
# Filepath to eORCA1 ancillary file:
22+
filepath_domain=/dssgfs01/scratch/npd/simulations/Domains/eORCA1/subbasins_CMIP6.nc
23+
24+
# Filepath to JASMIN OS credentials:
25+
store_credentials_json=.../credentials/jasmin_os_credentials.json
26+
27+
# Filepath to Dask Configuration file:
28+
dask_config_json=.../OceanDataStore/tests/dask_config.json
29+
30+
# Bucket and object prefix:
31+
bucket=npd-eorca1-era5v1
32+
33+
# -- Python Environment -- #
34+
# Activate miniconda environment:
35+
source /home/otooth/miniconda3/etc/profile.d/conda.sh
36+
conda activate env_jasmin_os
37+
38+
# -- Send eORCA1 ERA-5 v1 subbasins to JASMIN OS -- #
39+
prefix=test/domain_zarr
40+
41+
echo "In Progress: Sending eORCA1 ERA-5 v1 subbasins to JASMIN object store without Dask..."
42+
ods send_to_zarr -f $filepath_domain -c $store_credentials_json -b $bucket -p $prefix \
43+
-cs '{"x":360,"y":331}' || { echo "Error: ods send failed."; exit 1; }
44+
45+
echo "Completed: Sent eORCA1 ERA-5 v1 subbasins to JASMIN object store without Dask."
46+
47+
# -- Send eORCA1 ERA-5 v1 subbasins to JASMIN OS -- #
48+
prefix=test/domain_zarr_dask
49+
50+
echo "In Progress: Sending eORCA1 ERA-5 v1 subbasins to JASMIN object store with Dask..."
51+
ods send_to_zarr -f $filepath_domain -c $store_credentials_json -b $bucket -p $prefix \
52+
-cs '{"x":360,"y":331}' -dc $dask_config_json || { echo "Error: ods send failed."; exit 1; }
53+
54+
echo "Completed: Sent eORCA1 ERA-5 v1 subbasins to JASMIN object store with Dask."

tests/test_update_vs_icechunk.slurm renamed to tests/test_send_to_icechunk_vc_w_dask.slurm

Lines changed: 11 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#!/bin/bash
2-
#SBATCH --job-name=test_update_icechunk
2+
#SBATCH --job-name=test_send_to_icechunk
33
#SBATCH --partition=test
44
#SBATCH --time=00:20:00
55
#SBATCH --ntasks-per-core=1
@@ -8,9 +8,9 @@
88
#SBATCH --nodes=1
99

1010
# ==============================================================
11-
# test_update_icechunk.slurm
11+
# test_send_to_icechunk.slurm
1212
#
13-
# Description: SLURM script to test update_icechunk using the
13+
# Description: SLURM script to test send_to_icechunk using the
1414
# eORCA1 ERA-5 v1 monthly-mean output variables.
1515
#
1616
# Created By: Ollie Tooth (oliver.tooth@noc.ac.uk)
@@ -31,34 +31,25 @@ dask_config_json=.../OceanDataStore/tests/dask_config.json
3131

3232
# Bucket and object prefix:
3333
bucket=npd-eorca1-era5v1
34-
prefix=test/U1m_update_icechunk
34+
prefix=test/U1m_icechunk_vc_dask
3535

3636
# Define the append dimension:
3737
append_dim=time_counter
3838

3939
# Define variable:
40-
variables="uo umo time_counter_bounds"
40+
variables="time_counter_bounds uo umo"
4141

4242
# -- Python Environment -- #
4343
# Activate miniconda environment:
4444
source /home/otooth/miniconda3/etc/profile.d/conda.sh
45-
conda activate env_testing
45+
conda activate env_jasmin_os
4646

4747
# -- Send eORCA1 ERA-5 v1 monthly mean outputs to JASMIN OS -- #
48-
echo "In Progress: Sending eORCA1 ERA-5 v1 U1m variables to JASMIN object store..."
49-
filepath_gridU=$filedir/eORCA1_ERA5_1m_grid_U_197601-197601.nc
48+
echo "In Progress: Sending eORCA1 ERA-5 v1 U1m variables to JASMIN object store without Dask..."
49+
filepath_gridU=$filedir/eORCA1_ERA5_1m_grid_U_197*.nc
5050
ods send_to_icechunk -f $filepath_gridU -c $store_credentials_json -b $bucket -p $prefix \
5151
-gf $filepath_domain -uc '{"nav_lon":"glamu", "nav_lat":"gphiu"}' \
52-
-cs '{"x":360,"y":331,"depthu":25}' -v $variables \
53-
-dc $dask_config_json -vs || { echo "Error: ods send_to_icechunk failed."; exit 1; }
52+
-cs '{"x":360,"y":331,"depthu":25}' -dc $dask_config_json -vc \
53+
-v $variables || { echo "Error: ods send_to_icechunk failed."; exit 1; }
5454

55-
echo "Completed: Sent eORCA1 ERA-5 v1 monthly mean fields to JASMIN object store."
56-
57-
echo "In Progress: Updating eORCA1 ERA-5 v1 U1m variables to JASMIN object store..."
58-
filepath_gridU=$filedir/eORCA1_ERA5_1m_grid_U_1976*.nc
59-
ods update_icechunk -f $filepath_gridU -c $store_credentials_json -b $bucket -p $prefix \
60-
-gf $filepath_domain -uc '{"nav_lon":"glamu", "nav_lat":"gphiu"}' \
61-
-ad $append_dim -cs '{"x":360,"y":331,"depthu":25}' -v $variables \
62-
-dc $dask_config_json -vs || { echo "Error: ods update_icechunk failed."; exit 1; }
63-
64-
echo "Completed: Updated eORCA1 ERA-5 v1 monthly mean fields in JASMIN object store."
55+
echo "Completed: Sent eORCA1 ERA-5 v1 monthly mean fields to JASMIN object store without Dask."

‎tests/test_send_to_icechunk_w_dask.slurm‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
#!/bin/bash
2+
#SBATCH --job-name=test_send_to_icechunk
3+
#SBATCH --partition=test
4+
#SBATCH --time=00:20:00
5+
#SBATCH --ntasks-per-core=1
6+
#SBATCH --ntasks-per-node=64
7+
#SBATCH --ntasks-per-socket=32
8+
#SBATCH --nodes=1
9+
10+
# ==============================================================
11+
# test_send_to_icechunk.slurm
12+
#
13+
# Description: SLURM script to test send_to_icechunk using the
14+
# eORCA1 ERA-5 v1 monthly-mean output variables.
15+
#
16+
# Created By: Ollie Tooth (oliver.tooth@noc.ac.uk)
17+
# Created On: 2025-05-15
18+
#
19+
# ==============================================================
20+
# -- Input arguments to msm-os -- #
21+
# Filepath to eORCA1 ancillary file:
22+
filepath_domain=/dssgfs01/scratch/npd/simulations/Domains/eORCA1/domain_cfg.nc
23+
# Filepath to eORCA1 monthly mean output files:
24+
filedir=/dssgfs01/scratch/npd/simulations/eORCA1_ERA5_v1
25+
26+
# Filepath to JASMIN OS credentials:
27+
store_credentials_json=.../credentials/jasmin_os_credentials.json
28+
29+
# Filepath to Dask Configuration file:
30+
dask_config_json=.../OceanDataStore/tests/dask_config.json
31+
32+
# Bucket and object prefix:
33+
bucket=npd-eorca1-era5v1
34+
prefix=test/U1m_icechunk_dask
35+
36+
# Define the append dimension:
37+
append_dim=time_counter
38+
39+
# Define variable:
40+
variables="time_counter_bounds uo umo"
41+
42+
# -- Python Environment -- #
43+
# Activate miniconda environment:
44+
source /home/otooth/miniconda3/etc/profile.d/conda.sh
45+
conda activate env_jasmin_os
46+
47+
# -- Send eORCA1 ERA-5 v1 monthly mean outputs to JASMIN OS -- #
48+
echo "In Progress: Sending eORCA1 ERA-5 v1 U1m variables to JASMIN object store without Dask..."
49+
filepath_gridU=$filedir/eORCA1_ERA5_1m_grid_U_197*.nc
50+
ods send_to_icechunk -f $filepath_gridU -c $store_credentials_json -b $bucket -p $prefix \
51+
-gf $filepath_domain -uc '{"nav_lon":"glamu", "nav_lat":"gphiu"}' \
52+
-cs '{"x":360,"y":331,"depthu":25}' -dc $dask_config_json \
53+
-v $variables || { echo "Error: ods send_to_icechunk failed."; exit 1; }
54+
55+
echo "Completed: Sent eORCA1 ERA-5 v1 monthly mean fields to JASMIN object store without Dask."

tests/test_send_to_icechunk.slurm renamed to tests/test_send_to_icechunk_wo_dask.slurm

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ variables="time_counter_bounds uo umo"
4242
# -- Python Environment -- #
4343
# Activate miniconda environment:
4444
source /home/otooth/miniconda3/etc/profile.d/conda.sh
45-
conda activate env_testing
45+
conda activate env_jasmin_os
4646

4747
# -- Send eORCA1 ERA-5 v1 monthly mean outputs to JASMIN OS -- #
4848
echo "In Progress: Sending eORCA1 ERA-5 v1 U1m variables to JASMIN object store without Dask..."

‎tests/test_send_to_zarr_w_dask.slurm‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
#!/bin/bash
2+
#SBATCH --job-name=test_send_to_zarr
3+
#SBATCH --partition=test
4+
#SBATCH --time=00:20:00
5+
#SBATCH --ntasks-per-core=1
6+
#SBATCH --ntasks-per-node=64
7+
#SBATCH --ntasks-per-socket=32
8+
#SBATCH --nodes=1
9+
10+
# ==============================================================
11+
# test_send_to_zarr.slurm
12+
#
13+
# Description: SLURM script to test send_to_zarr using the eORCA1
14+
# ERA-5 v1 monthly-mean output variables.
15+
#
16+
# Created By: Ollie Tooth (oliver.tooth@noc.ac.uk)
17+
# Created On: 2025-05-15
18+
#
19+
# ==============================================================
20+
# -- Input arguments to msm-os -- #
21+
# Filepath to eORCA1 ancillary file:
22+
filepath_domain=/dssgfs01/scratch/npd/simulations/Domains/eORCA1/domain_cfg.nc
23+
# Filepath to eORCA1 monthly mean output files:
24+
filedir=/dssgfs01/scratch/npd/simulations/eORCA1_ERA5_v1
25+
26+
# Filepath to JASMIN OS credentials:
27+
store_credentials_json=.../credentials/jasmin_os_credentials.json
28+
29+
# Filepath to Dask Configuration file:
30+
dask_config_json=.../OceanDataStore/tests/dask_config.json
31+
32+
# Bucket and object prefix:
33+
bucket=npd-eorca1-era5v1
34+
prefix=test/U1m_zarr_dask
35+
36+
# Define the append dimension:
37+
append_dim=time_counter
38+
39+
# Define variable:
40+
variables="time_counter_bounds uo umo"
41+
42+
# -- Python Environment -- #
43+
# Activate miniconda environment:
44+
source /home/otooth/miniconda3/etc/profile.d/conda.sh
45+
conda activate env_jasmin_os
46+
47+
# -- Send eORCA1 ERA-5 v1 monthly mean outputs to JASMIN OS -- #
48+
echo "In Progress: Sending eORCA1 ERA-5 v1 U1m variables to JASMIN object store..."
49+
filepath_gridU=$filedir/eORCA1_ERA5_1m_grid_U_197*.nc
50+
ods send_to_zarr -f $filepath_gridU -c $store_credentials_json -b $bucket -p $prefix \
51+
-gf $filepath_domain -uc '{"nav_lon":"glamu", "nav_lat":"gphiu"}' \
52+
-cs '{"x":360,"y":331,"depthu":25}' -v $variables \
53+
-dc $dask_config_json || { echo "Error: ods send failed."; exit 1; }
54+
55+
echo "Completed: Sent eORCA1 ERA-5 v1 monthly mean fields to JASMIN object store."

tests/test_send_to_zarr.slurm renamed to tests/test_send_to_zarr_wo_dask.slurm

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,14 +42,14 @@ variables="time_counter_bounds uo umo"
4242
# -- Python Environment -- #
4343
# Activate miniconda environment:
4444
source /home/otooth/miniconda3/etc/profile.d/conda.sh
45-
conda activate env_testing
45+
conda activate env_jasmin_os
4646

4747
# -- Send eORCA1 ERA-5 v1 monthly mean outputs to JASMIN OS -- #
4848
echo "In Progress: Sending eORCA1 ERA-5 v1 U1m variables to JASMIN object store..."
4949
filepath_gridU=$filedir/eORCA1_ERA5_1m_grid_U_197*.nc
5050
ods send_to_zarr -f $filepath_gridU -c $store_credentials_json -b $bucket -p $prefix \
5151
-gf $filepath_domain -uc '{"nav_lon":"glamu", "nav_lat":"gphiu"}' \
52-
-cs '{"x":360,"y":331,"depthu":25}' -v $variables -vs \
52+
-cs '{"x":360,"y":331,"depthu":25}' -v $variables \
5353
-dc $dask_config_json || { echo "Error: ods send failed."; exit 1; }
5454

5555
echo "Completed: Sent eORCA1 ERA-5 v1 monthly mean fields to JASMIN object store."

tests/test_update_vc_icechunk.slurm renamed to tests/test_update_vc_icechunk_w_dask.slurm

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ dask_config_json=.../OceanDataStore/tests/dask_config.json
3131

3232
# Bucket and object prefix:
3333
bucket=npd-eorca1-era5v1
34-
prefix=test/U1m
34+
prefix=test/U1m_icechunk_vc_dask
3535

3636
# Define the append dimension:
3737
append_dim=time_counter
@@ -42,7 +42,7 @@ variables="time_counter_bounds uo"
4242
# -- Python Environment -- #
4343
# Activate miniconda environment:
4444
source /home/otooth/miniconda3/etc/profile.d/conda.sh
45-
conda activate env_testing
45+
conda activate env_jasmin_os
4646

4747
# -- Send eORCA1 ERA-5 v1 monthly mean outputs to JASMIN OS -- #
4848
echo "In Progress: Sending eORCA1 ERA-5 v1 U1m variables to JASMIN object store..."
@@ -55,7 +55,7 @@ ods send_to_icechunk -f $filepath_gridU -c $store_credentials_json -b $bucket -p
5555
echo "Completed: Sent eORCA1 ERA-5 v1 monthly mean fields to JASMIN object store."
5656

5757
echo "In Progress: Updating eORCA1 ERA-5 v1 U1m variables to JASMIN object store..."
58-
variables="umo"
58+
filepath_gridU=$filedir/eORCA1_ERA5_1m_grid_U_1976*.nc
5959
ods update_icechunk -f $filepath_gridU -c $store_credentials_json -b $bucket -p $prefix \
6060
-gf $filepath_domain -uc '{"nav_lon":"glamu", "nav_lat":"gphiu"}' \
6161
-ad $append_dim -cs '{"x":360,"y":331,"depthu":25}' -v $variables \

tests/test_update_zarr.slurm renamed to tests/test_update_zarr_w_dask.slurm

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -32,35 +32,35 @@ dask_config_json=.../OceanDataStore/tests/dask_config.json
3232

3333
# Bucket and object prefix:
3434
bucket=npd-eorca1-era5v1
35-
prefix=test/U1m_zarr_update
35+
prefix=test/U1m_zarr_dask
3636

3737
# Define the append dimension:
3838
append_dim=time_counter
3939

4040
# Define variable:
41-
variables="uo umo time_counter_bounds"
41+
variables="time_counter_bounds uo"
4242

4343
# -- Python Environment -- #
4444
# Activate miniconda environment:
4545
source /home/otooth/miniconda3/etc/profile.d/conda.sh
46-
conda activate env_testing
46+
conda activate env_jasmin_os
4747

4848
# -- Send eORCA1 ERA-5 v1 monthly mean outputs to JASMIN OS -- #
4949
echo "In Progress: Sending eORCA1 ERA-5 v1 U1m variables to JASMIN object store..."
5050
filepath_gridU=$filedir/eORCA1_ERA5_1m_grid_U_197601-197601.nc
5151
ods send_to_zarr -f $filepath_gridU -c $store_credentials_json -b $bucket -p $prefix \
5252
-gf $filepath_domain -uc '{"nav_lon":"glamu", "nav_lat":"gphiu"}' \
5353
-ad $append_dim -cs '{"x":360,"y":331,"depthu":25}' -v $variables \
54-
-dc $dask_config_json -vs || { echo "Error: ods send_to_zarr failed."; exit 1; }
54+
-dc $dask_config_json|| { echo "Error: ods send_to_zarr failed."; exit 1; }
5555

5656
echo "Completed: Sent eORCA1 ERA-5 v1 monthly mean fields to JASMIN object store."
5757

5858
# -- Send eORCA1 ERA-5 v1 monthly mean outputs to JASMIN OS -- #
5959
echo "In Progress: Updating eORCA1 ERA-5 v1 U1m variables to JASMIN object store..."
60-
filepath_gridU=$filedir/eORCA1_ERA5_1m_grid_U_1976*.nc
60+
filepath_gridU=$filedir/eORCA1_ERA5_1m_grid_U_197*.nc
6161
ods update_zarr -f $filepath_gridU -c $store_credentials_json -b $bucket -p $prefix \
6262
-gf $filepath_domain -uc '{"nav_lon":"glamu", "nav_lat":"gphiu"}' \
6363
-ad $append_dim -cs '{"x":360,"y":331,"depthu":25}' -v $variables \
64-
-dc $dask_config_json -vs || { echo "Error: ods update_zarr failed."; exit 1; }
64+
-dc $dask_config_json || { echo "Error: ods update_zarr failed."; exit 1; }
6565

6666
echo "Completed: Updated eORCA1 ERA-5 v1 monthly mean fields to JASMIN object store."

0 commit comments

Comments
 (0)