Skip to content

Collective access when extending a file #2016

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jedwards4b merged 4 commits into from
Nov 28, 2025
Merged

Collective access when extending a file #2016

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
60b7159
when writing to a netcdf4 file we need to set var_par_access to colle…
jedwards4b Nov 26, 2025
06e7937
Merge branch 'main' into fix/netcdf4_var_par_access
jedwards4b Nov 26, 2025
2e6f3ea
add a test and confirm that the test passes with the change and fails…
jedwards4b Nov 28, 2025
559bf63
Merge branch 'fix/netcdf4_var_par_access' of github.com:NCAR/Parallel…
jedwards4b Nov 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion src/clib/pio_getput_int.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1081,7 +1081,6 @@ PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Offset
else
fake_stride = (PIO_Offset *)stride;
}

#ifdef _PNETCDF
if (file->iotype == PIO_IOTYPE_PNETCDF)
{
Expand Down Expand Up @@ -1137,6 +1136,15 @@ PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Offset
{
PLOG((2, "PIOc_put_vars_tc calling netcdf function file->iotype = %d",
file->iotype));
#ifdef _NETCDF4
if (file->iotype == PIO_IOTYPE_NETCDF4P)
{
PLOG((2, "Setting NC_COLLECTIVE mode"));
if ((ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE)))
return pio_err(ios, file, ierr, __FILE__, __LINE__);
}

#endif
switch(xtype)
{
case NC_BYTE:
Expand Down
3 changes: 2 additions & 1 deletion src/clib/pioc_support.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2720,7 +2720,7 @@ PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename,
#ifdef _MPISERIAL
ierr = nc_open(filename, mode, &file->fh);
#else
imode = mode | NC_MPIIO;
imode = mode | NC_MPIIO | NC_NETCDF4;
if ((ierr = nc_open_par(filename, imode, ios->io_comm, ios->info,
&file->fh))){
PLOG((2, "%d: PIOc_openfile_retry nc_open_par ierr=%d",__LINE__,ierr));
Expand All @@ -2736,6 +2736,7 @@ PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename,
&pio_type_size, &mpi_type,
&mpi_type_size, &ndims)))
break;

PLOG((2, "PIOc_openfile_retry:nc_open_par filename = %s mode = %d "
"imode = %d ierr = %d", filename, mode, imode, ierr));
#endif
Expand Down
38 changes: 31 additions & 7 deletions tests/cunit/test_pioc_putget.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1767,7 +1767,7 @@ int putget_read_vars_nt(int ncid, int *varid, PIO_Offset *start, PIO_Offset *cou
* @returns 0 for success, error code otherwise.
*/
int create_putget_file(int iosysid, int access, int unlim, int flavor, int *dim_len,
int *varid, const char *filename, int *ncidp)
int *varid, const char *filename, int *ncidp, int *tsvarid)
{
int dimids[NDIM]; /* The dimension IDs. */
int num_vars = NUM_CLASSIC_TYPES + 1;
Expand All @@ -1786,7 +1786,9 @@ int create_putget_file(int iosysid, int access, int unlim, int flavor, int *dim_
return ret;

/* Are we using unlimited dimension? */
if (!unlim)
if (unlim)
dim_len[0] = NC_UNLIMITED;
else
dim_len[0] = NUM_TIMESTEPS;

/* Define netCDF dimensions and variable. */
Expand All @@ -1812,7 +1814,14 @@ int create_putget_file(int iosysid, int access, int unlim, int flavor, int *dim_
if ((ret = PIOc_def_var(ncid, var_name, my_type, NDIM, dimids, &varid[v])))
return ret;
}

if(unlim){
/* create an additional variable with only the unlimdim dimension */
if ((ret = PIOc_def_var(ncid, "timestep", PIO_INT, 1, dimids, tsvarid)))
{
printf("Defined a timestep variable %d\n", ret);
return ret;
}
}
/* For the first access, also test attributes. */
if (access == 0)
if ((ret = test_write_atts(ncid, varid, flavor)))
Expand Down Expand Up @@ -1908,7 +1917,8 @@ int test_putget(int iosysid, int num_flavors, int *flavor, int my_rank,
MPI_Comm test_comm)
{
int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN};

int tsvarid;

#define NUM_ACCESS 8
for (int unlim = 0; unlim < 2; unlim++)
for (int access = 0; access < NUM_ACCESS; access++)
Expand All @@ -1931,15 +1941,15 @@ int test_putget(int iosysid, int num_flavors, int *flavor, int my_rank,

/* Create test file with dims and vars defined. */
if ((ret = create_putget_file(iosysid, access, unlim, flavor[fmt], dim_len, varid,
filename, &ncid)))
filename, &ncid, &tsvarid)))
return ret;

/* Write some data. */
PIO_Offset index[NDIM] = {0, 0, 0};
PIO_Offset start[NDIM] = {0, 0, 0};
PIO_Offset count[NDIM] = {1, X_DIM_LEN, Y_DIM_LEN};
PIO_Offset stride[NDIM] = {1, 1, 1};

PIO_Offset index1d = 0;
switch (access)
{
case 0:
Expand Down Expand Up @@ -2003,7 +2013,21 @@ int test_putget(int iosysid, int num_flavors, int *flavor, int my_rank,
/* Check contents of the file. */
if ((ret = check_file(access, ncid, varid, flavor[fmt], index, start, count, stride, unlim)))
return ret;

if(unlim > 0) {
/* try advancing the file */
index1d = 0;
if ((ret = PIOc_put_var1_int(ncid, tsvarid, &index1d, dim_len)))
{
printf("extend file time test %d\n",ret);
return ret;
}
index1d = 1;
if ((ret = PIOc_put_var1_int(ncid, tsvarid, &index1d, dim_len+1)))
{
printf("extend file time test %d\n",ret);
return ret;
}
}
/* Close the netCDF file. */
if ((ret = PIOc_closefile(ncid)))
ERR(ret);
Expand Down
8 changes: 7 additions & 1 deletion tests/fncint/ftst_pio.f90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ program ftst_pio
integer, dimension(3) :: var_dim
integer :: maplen
integer :: decompid, iosysid
integer :: varid, i
integer :: varid, i, tsvarid
integer :: ierr

! Set up MPI.
Expand Down Expand Up @@ -75,13 +75,19 @@ program ftst_pio
! Define a data variable.
ierr = nf_def_var(ncid, VAR_NAME, NF_INT, NDIM3, var_dim, varid)
if (ierr .ne. nf_noerr) call handle_err(ierr)

ierr = nf_def_var(ncid, 'timestep', NF_INT, 1, var_dim(3), tsvarid)

ierr = nf_enddef(ncid)
if (ierr .ne. nf_noerr) call handle_err(ierr)

! Write 1st record with distributed arrays.
ierr = nf_put_vard_int(ncid, varid, decompid, 1, data_buffer)
if (ierr .ne. nf_noerr) call handle_err(ierr)

ierr = nf_put_var(ncid, tsvarid, 2, 2)
if (ierr .ne. nf_noerr) call handle_err(ierr)

! Close the file.
ierr = nf_close(ncid)
if (ierr .ne. nf_noerr) call handle_err(ierr)
Expand Down
65 changes: 65 additions & 0 deletions tests/general/ncdf_simple_tests.F90.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,69 @@ PIO_TF_AUTO_TEST_SUB_BEGIN test_def_var

PIO_TF_AUTO_TEST_SUB_END test_def_var

PIO_TF_AUTO_TEST_SUB_BEGIN test_extend_unlimited_var
use ncdf_simple_tests_tgv
Implicit none
type(file_desc_t) :: pio_file
type(var_desc_t) :: pio_var
integer :: pio_dim
integer :: ret, i
integer, parameter :: initial_len = 5, extend_len = 3
integer, dimension(initial_len + extend_len) :: data_out, data_in
character(len=PIO_TF_MAX_STR_LEN), parameter :: fname = "pio_test_extend_unlim.nc"

! Create file with unlimited dimension and write initial data
ret = PIO_createfile(pio_tf_iosystem_, pio_file, tgv_iotype, fname, PIO_CLOBBER)
PIO_TF_CHECK_ERR(ret, "Failed to create:" // trim(fname))

ret = PIO_def_dim(pio_file, 'unlim_dim', PIO_UNLIMITED, pio_dim) ! -1 for unlimited
PIO_TF_CHECK_ERR(ret, "Failed to define unlimited dim:" // trim(fname))

ret = PIO_def_var(pio_file, 'unlim_var', PIO_int, (/pio_dim/), pio_var)
PIO_TF_CHECK_ERR(ret, "Failed to define var:" // trim(fname))

ret = PIO_enddef(pio_file)
PIO_TF_CHECK_ERR(ret, "Failed to enddef:" // trim(fname))

do i = 1, initial_len
data_out(i) = i
end do
ret = PIO_put_var(pio_file, pio_var, data_out(1:initial_len))
PIO_TF_CHECK_ERR(ret, "Failed to write initial data:" // trim(fname))

call PIO_closefile(pio_file)

! Reopen file and extend variable
ret = PIO_openfile(pio_tf_iosystem_, pio_file, tgv_iotype, fname, PIO_write)
PIO_TF_CHECK_ERR(ret, "Failed to reopen:" // trim(fname))

do i = 1, extend_len
data_out(initial_len + i) = 100 + i
end do
ret = PIO_put_var(pio_file, pio_var, start=(/initial_len/), count=(/extend_len/), &
ival=data_out(initial_len+1:initial_len+extend_len))
PIO_TF_CHECK_ERR(ret, "Failed to extend data:" // trim(fname))

call PIO_syncfile(pio_file)

! Optionally, read back and verify
ret = PIO_get_var(pio_file, pio_var, data_in)
PIO_TF_CHECK_ERR(ret, "Failed to read back data:" // trim(fname))
do i = 1, initial_len
if (data_in(i) /= i) then
print *, "ERROR: Initial data mismatch at ", i, ":", data_in(i), "!=", i
end if
end do
do i = 1, extend_len
if (data_in(initial_len + i) /= 100 + i) then
print *, "ERROR: Extended data mismatch at ", initial_len + i, ":", data_in(initial_len + i), "!=", 100 + i
end if
end do

call PIO_closefile(pio_file)
call PIO_deletefile(pio_tf_iosystem_, fname)

PIO_TF_AUTO_TEST_SUB_END test_extend_unlimited_var
PIO_TF_TEMPLATE<PIO_TF_PREDEF_TYPENAME PIO_TF_DATA_TYPE, PIO_TF_PREDEF_TYPENAME PIO_TF_FC_DATA_TYPE>
PIO_TF_AUTO_TEST_SUB_BEGIN test_data_conversion
use ncdf_simple_tests_tgv
Expand Down Expand Up @@ -190,6 +253,8 @@ PIO_TF_TEST_DRIVER_BEGIN

! Make sure that global variables are set correctly before running the tests
PIO_TF_AUTO_TESTS_RUN(trim(iotype_descs(i)))
PRINT *, "PIO_TF: Starting test_extend_unlimited_var for ", trim(iotype_descs(i))
CALL test_extend_unlimited_var()

call PIO_deletefile(pio_tf_iosystem_, tgv_fname)
end do
Expand Down