From 633182a6158aa4548e42c200887d62737a574b07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eivind=20Lyche=20Melv=C3=A6r?= Date: Wed, 10 Jul 2019 21:16:39 +0200 Subject: [PATCH 01/19] Initial implementation --- include/netcdf.h | 1 + libhdf5/hdf5create.c | 24 ++++++++++++++++++---- libhdf5/hdf5open.c | 16 ++++++++++++++- libhdf5/hdf5var.c | 5 +++++ nc_test4/tst_files.c | 48 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 89 insertions(+), 5 deletions(-) diff --git a/include/netcdf.h b/include/netcdf.h index 6ba8ba48be..8729e14151 100644 --- a/include/netcdf.h +++ b/include/netcdf.h @@ -159,6 +159,7 @@ Use this in mode flags for both nc_create() and nc_open(). */ #define NC_PERSIST 0x4000 /**< Save diskless contents to disk. Mode flag for nc_open() or nc_create() */ #define NC_INMEMORY 0x8000 /**< Read from memory. Mode flag for nc_open() or nc_create() */ +#define NC_HDF5_SWMR 0x2000 /** Reuse deprecated MPIIO flag for SWMR **/ #define NC_MAX_MAGIC_NUMBER_LEN 8 /**< Max len of user-defined format magic number. */ diff --git a/libhdf5/hdf5create.c b/libhdf5/hdf5create.c index 66fa0ac230..f39cac86b1 100644 --- a/libhdf5/hdf5create.c +++ b/libhdf5/hdf5create.c @@ -50,6 +50,10 @@ nc4_create_file(const char *path, int cmode, size_t initialsz, NC_HDF5_FILE_INFO_T *hdf5_info; NC_HDF5_GRP_INFO_T *hdf5_grp; +#ifdef HAVE_H5PSET_LIBVER_BOUNDS + H5F_libver_t low, high; +#endif + #ifdef USE_PARALLEL4 NC_MPI_INFO *mpiinfo = NULL; MPI_Comm comm; @@ -157,12 +161,18 @@ nc4_create_file(const char *path, int cmode, size_t initialsz, #ifdef HAVE_H5PSET_LIBVER_BOUNDS #if H5_VERSION_GE(1,10,2) - if (H5Pset_libver_bounds(fapl_id, H5F_LIBVER_EARLIEST, H5F_LIBVER_V18) < 0) + low = H5F_LIBVER_EARLIEST; + high = H5F_LIBVER_V18; + if ((cmode & NC_HDF5_SWMR)) { + low = H5F_LIBVER_LATEST; + high = H5F_LIBVER_LATEST; + } #else - if (H5Pset_libver_bounds(fapl_id, H5F_LIBVER_EARLIEST, - H5F_LIBVER_LATEST) < 0) + low = H5F_LIBVER_EARLIEST; + high = H5F_LIBVER_LATEST; #endif - BAIL(NC_EHDFERR); + if (H5Pset_libver_bounds(fapl_id, low, high) < 0) + BAIL(NC_EHDFERR); #endif /* Create the property list. */ @@ -237,6 +247,12 @@ nc4_create_file(const char *path, int cmode, size_t initialsz, if ((retval = NC4_new_provenance(nc4_info))) BAIL(retval); + if ((cmode & NC_HDF5_SWMR)) { + /* Prepare for single writer multiple readers */ + if ((retval = H5Fstart_swmr_write(hdf5_info->hdfid))) + BAIL(retval); + } + return NC_NOERR; exit: /*failure exit*/ diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c index 1b11855290..b8a6bd70c5 100644 --- a/libhdf5/hdf5open.c +++ b/libhdf5/hdf5open.c @@ -652,7 +652,14 @@ nc4_open_file(const char *path, int mode, void* parameters, NC *nc) LOG((3, "%s: path %s mode %d", __func__, path, mode)); assert(path && nc); - flags = (mode & NC_WRITE) ? H5F_ACC_RDWR : H5F_ACC_RDONLY; + if((mode & NC_WRITE)) { + flags = H5F_ACC_RDWR; + } else { + flags = H5F_ACC_RDONLY; + if((mode & NC_HDF5_SWMR)) { + flags |= H5F_ACC_SWMR_READ; + } + } /* Add necessary structs to hold netcdf-4 file data. */ if ((retval = nc4_nc4f_list_add(nc, path, mode))) @@ -829,6 +836,13 @@ nc4_open_file(const char *path, int mode, void* parameters, NC *nc) if (H5Pclose(fapl_id) < 0) BAIL(NC_EHDFERR); + /* Prepare for single writer multiple reader. */ + if (mode & NC_WRITE && mode & NC_HDF5_SWMR) { + if ((retval = H5Fstart_swmr_write(h5->hdfid))) { + BAIL(retval); + } + } + return NC_NOERR; exit: diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c index 3ff19c4f62..e518e8d30e 100644 --- a/libhdf5/hdf5var.c +++ b/libhdf5/hdf5var.c @@ -1764,6 +1764,11 @@ NC4_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp, return retval; if (range_error) return NC_ERANGE; + + /* Flush data for SWMR */ + if (H5Dflush(hdf5_var->hdf_datasetid) < 0) + BAIL(NC_EHDFERR); + return NC_NOERR; } diff --git a/nc_test4/tst_files.c b/nc_test4/tst_files.c index d8927d8ecb..d79f8be5a4 100644 --- a/nc_test4/tst_files.c +++ b/nc_test4/tst_files.c @@ -292,6 +292,54 @@ main(int argc, char **argv) if (nc_close(ncid)) ERR; } SUMMARIZE_ERR; + printf("*** testing HDF5 SWMR..."); + { +#define DATA_LEN 3 + + int ncid, ncid2, ncid3, varid, dimids[2]; + int ndims, nvars, natts, unlimdimid; + int dimids_var[1], var_type; + size_t time_len, beam_len; + int i; + int values[DATA_LEN]; + size_t start[2] = {0,0}, count[2] = {1, DATA_LEN}; + + /* Initialize some phony data. */ + for (i = 0; i < DATA_LEN; i++) + values[i] = DATA_LEN*2 - i; + + /* Create a file in SWMR mode for writing, create structure and close. */ + if (nc_create("tst_swmr.nc", NC_NETCDF4|NC_HDF5_SWMR, &ncid)) ERR; + if (nc_def_dim(ncid, "time", NC_UNLIMITED, &dimids[0])) ERR; + if (nc_def_dim(ncid, "beam", NC_UNLIMITED, &dimids[1])) ERR; + if (nc_def_var(ncid, "depth", NC_INT, 2, dimids, &varid)) ERR; + if (nc_close(ncid)) ERR; + + /* Open the file for SWMR reading and close. */ + if (nc_open("tst_swmr.nc", NC_HDF5_SWMR, &ncid)) ERR; + if (nc_close(ncid)) ERR; + + /* Open the file for SWMR writing, append data, and close. */ + if (nc_open("tst_swmr.nc", NC_WRITE|NC_HDF5_SWMR, &ncid)) ERR; + if (nc_inq_varid(ncid, "depth", &varid)) ERR; + if (nc_put_vara_int(ncid, varid, start, count, values)) ERR; + if (nc_inq_dimlen(ncid, dimids[0], &time_len)) ERR; + if (time_len != 1) ERR; + if (nc_inq_dimlen(ncid, dimids[1], &beam_len)) ERR; + if (beam_len != DATA_LEN) ERR; + if (nc_close(ncid)) ERR; + + /* Open the file for SWMR reading, verify data, and close. */ + if (nc_open("tst_swmr.nc", NC_HDF5_SWMR, &ncid)) ERR; + if (nc_inq_varid(ncid, "depth", &varid)) ERR; + if (nc_put_vara_int(ncid, varid, start, count, values) == 0) ERR; // Writing should fail + if (nc_inq_dimlen(ncid, dimids[0], &time_len)) ERR; + if (time_len != 1) ERR; + if (nc_inq_dimlen(ncid, dimids[1], &beam_len)) ERR; + if (beam_len != DATA_LEN) ERR; + if (nc_close(ncid)) ERR; + } + SUMMARIZE_ERR; printf("*** testing CLASSIC_MODEL flag with classic formats..."); { int ncid; From 757207f53567cdfadb69fe77c7c1029cb3fe87cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eivind=20Lyche=20Melv=C3=A6r?= Date: Wed, 31 Jul 2019 10:07:45 +0200 Subject: [PATCH 02/19] Move up call to flush --- libhdf5/hdf5var.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c index e518e8d30e..fe5b2fc9ce 100644 --- a/libhdf5/hdf5var.c +++ b/libhdf5/hdf5var.c @@ -1736,6 +1736,10 @@ NC4_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp, mem_spaceid, file_spaceid, xfer_plistid, bufr) < 0) BAIL(NC_EHDFERR); + /* Flush data for SWMR */ + if (H5Dflush(hdf5_var->hdf_datasetid) < 0) + BAIL(NC_EHDFERR); + /* Remember that we have written to this var so that Fill Value * can't be set for it. */ if (!var->written_to) @@ -1765,10 +1769,6 @@ NC4_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp, if (range_error) return NC_ERANGE; - /* Flush data for SWMR */ - if (H5Dflush(hdf5_var->hdf_datasetid) < 0) - BAIL(NC_EHDFERR); - return NC_NOERR; } From 932d6e079c6a11478b30ac48476e420925ac5199 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eivind=20Lyche=20Melv=C3=A6r?= Date: Wed, 31 Jul 2019 15:24:01 +0200 Subject: [PATCH 03/19] Expand test --- nc_test4/tst_files.c | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/nc_test4/tst_files.c b/nc_test4/tst_files.c index d79f8be5a4..5004f21fc5 100644 --- a/nc_test4/tst_files.c +++ b/nc_test4/tst_files.c @@ -296,9 +296,7 @@ main(int argc, char **argv) { #define DATA_LEN 3 - int ncid, ncid2, ncid3, varid, dimids[2]; - int ndims, nvars, natts, unlimdimid; - int dimids_var[1], var_type; + int ncid, ncid2, varid, dimids[2]; size_t time_len, beam_len; int i; int values[DATA_LEN]; @@ -309,18 +307,18 @@ main(int argc, char **argv) values[i] = DATA_LEN*2 - i; /* Create a file in SWMR mode for writing, create structure and close. */ - if (nc_create("tst_swmr.nc", NC_NETCDF4|NC_HDF5_SWMR, &ncid)) ERR; + if (nc_create(FILE_NAME, NC_NETCDF4|NC_HDF5_SWMR, &ncid)) ERR; if (nc_def_dim(ncid, "time", NC_UNLIMITED, &dimids[0])) ERR; if (nc_def_dim(ncid, "beam", NC_UNLIMITED, &dimids[1])) ERR; if (nc_def_var(ncid, "depth", NC_INT, 2, dimids, &varid)) ERR; if (nc_close(ncid)) ERR; /* Open the file for SWMR reading and close. */ - if (nc_open("tst_swmr.nc", NC_HDF5_SWMR, &ncid)) ERR; + if (nc_open(FILE_NAME, NC_HDF5_SWMR, &ncid)) ERR; if (nc_close(ncid)) ERR; /* Open the file for SWMR writing, append data, and close. */ - if (nc_open("tst_swmr.nc", NC_WRITE|NC_HDF5_SWMR, &ncid)) ERR; + if (nc_open(FILE_NAME, NC_WRITE|NC_HDF5_SWMR, &ncid)) ERR; if (nc_inq_varid(ncid, "depth", &varid)) ERR; if (nc_put_vara_int(ncid, varid, start, count, values)) ERR; if (nc_inq_dimlen(ncid, dimids[0], &time_len)) ERR; @@ -330,7 +328,7 @@ main(int argc, char **argv) if (nc_close(ncid)) ERR; /* Open the file for SWMR reading, verify data, and close. */ - if (nc_open("tst_swmr.nc", NC_HDF5_SWMR, &ncid)) ERR; + if (nc_open(FILE_NAME, NC_HDF5_SWMR, &ncid)) ERR; if (nc_inq_varid(ncid, "depth", &varid)) ERR; if (nc_put_vara_int(ncid, varid, start, count, values) == 0) ERR; // Writing should fail if (nc_inq_dimlen(ncid, dimids[0], &time_len)) ERR; @@ -338,6 +336,31 @@ main(int argc, char **argv) if (nc_inq_dimlen(ncid, dimids[1], &beam_len)) ERR; if (beam_len != DATA_LEN) ERR; if (nc_close(ncid)) ERR; + + /* Append data to the file from one writer (ncid1) and verify from a reader (ncid2) */ + if (nc_open(FILE_NAME, NC_WRITE|NC_HDF5_SWMR, &ncid)) ERR; + if (nc_open(FILE_NAME, NC_HDF5_SWMR, &ncid2)) ERR; + + // Verify length of time dimension == 1 in both reader and writer + if (nc_inq_dimlen(ncid, dimids[0], &time_len)) ERR; + if (time_len != 1) ERR; + if (nc_inq_dimlen(ncid2, dimids[0], &time_len)) ERR; + if (time_len != 1) ERR; + + // Append data + start[0] = 1; start[1] = 0; + if (nc_inq_varid(ncid, "depth", &varid)) ERR; + if (nc_put_vara_int(ncid, varid, start, count, values)) ERR; + + // Verify length of time dimension == 2 in both reader and writer + if (nc_inq_dimlen(ncid, dimids[0], &time_len)) ERR; + if (time_len != 2) ERR; + if (nc_inq_dimlen(ncid2, dimids[0], &time_len)) ERR; + if (time_len != 2) ERR; + + if (nc_close(ncid)) ERR; + if (nc_close(ncid2)) ERR; + } SUMMARIZE_ERR; printf("*** testing CLASSIC_MODEL flag with classic formats..."); From f0f3543d3ff170f62755e03f3e4f9fb7483bc32c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eivind=20Lyche=20Melv=C3=A6r?= Date: Wed, 31 Jul 2019 16:23:28 +0200 Subject: [PATCH 04/19] Add flag H5F_ACC_SWMR_WRITE to H5Fcreate if we do not have H5P_set_libver_bounds --- libhdf5/hdf5create.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/libhdf5/hdf5create.c b/libhdf5/hdf5create.c index f39cac86b1..7afee69b52 100644 --- a/libhdf5/hdf5create.c +++ b/libhdf5/hdf5create.c @@ -108,6 +108,11 @@ nc4_create_file(const char *path, int cmode, size_t initialsz, else flags = H5F_ACC_TRUNC; +#ifndef HAVE_H5PSET_LIBVER_BOUNDS + if (cmode & NC_HDF5_SWMR) + flags |= H5F_ACC_SWMR_WRITE; +#endif + /* If this file already exists, and NC_NOCLOBBER is specified, return an error (unless diskless|inmemory) */ if (!nc4_info->mem.diskless && !nc4_info->mem.inmemory) { From cd4cd05a7fcbaad411aab5996e0c276931dfadce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eivind=20Lyche=20Melv=C3=A6r?= Date: Tue, 6 Aug 2019 10:47:16 +0200 Subject: [PATCH 05/19] Protect usage of swmr by ifdefs --- CMakeLists.txt | 9 +++++++++ libhdf5/hdf5create.c | 6 ++++++ libhdf5/hdf5open.c | 4 ++++ libhdf5/hdf5var.c | 2 ++ nc_test4/tst_files.c | 2 ++ 5 files changed, 23 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3aa2ef73fb..c2d91a88ca 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -496,6 +496,8 @@ IF(ENABLE_HDF4) ENDIF() ENDIF() +OPTION(ENABLE_HDF5_SWMR "Support SWMR in HDF5. This requires HDF version 1.10 or later" OFF) + # Option to Build DLL IF(WIN32) OPTION(ENABLE_DLL "Build a Windows DLL." ${BUILD_SHARED_LIBS}) @@ -629,6 +631,10 @@ IF(USE_HDF5 OR ENABLE_NETCDF_4) # Assert HDF5 version meets minimum required version. ## SET(HDF5_VERSION_REQUIRED 1.8.10) + IF(ENABLE_HDF5_SWMR_SUPPORT) + SET(HDF5_VERSION_REQUIRED 1.10) + MESSAGE(STATUS "HDF5 SWMR is enabled. This implies that HDF5 version at least ${HDF5_VERSION_REQUIRED} is required.") + ENDIF() IF(HDF5_VERSION_STRING AND NOT HDF5_VERSION) SET(HDF5_VERSION ${HDF5_VERSION_STRING}) @@ -642,6 +648,9 @@ IF(USE_HDF5 OR ENABLE_NETCDF_4) "netCDF requires at least HDF5 ${HDF5_VERSION_REQUIRED}. Found ${HDF5_VERSION}.") ELSE() MESSAGE(STATUS "Found HDF5 libraries version ${HDF5_VERSION}") + IF(ENABLE_HDF5_SWMR_SUPPORT) + ADD_DEFINITIONS(-DHDF5_HAS_SWMR) + ENDIF() ENDIF() ENDIF() diff --git a/libhdf5/hdf5create.c b/libhdf5/hdf5create.c index 7afee69b52..e92e553b35 100644 --- a/libhdf5/hdf5create.c +++ b/libhdf5/hdf5create.c @@ -108,9 +108,11 @@ nc4_create_file(const char *path, int cmode, size_t initialsz, else flags = H5F_ACC_TRUNC; +#ifdef HDF5_HAS_SWMR #ifndef HAVE_H5PSET_LIBVER_BOUNDS if (cmode & NC_HDF5_SWMR) flags |= H5F_ACC_SWMR_WRITE; +#endif #endif /* If this file already exists, and NC_NOCLOBBER is specified, @@ -168,10 +170,12 @@ nc4_create_file(const char *path, int cmode, size_t initialsz, #if H5_VERSION_GE(1,10,2) low = H5F_LIBVER_EARLIEST; high = H5F_LIBVER_V18; +#ifdef HDF5_HAS_SWMR if ((cmode & NC_HDF5_SWMR)) { low = H5F_LIBVER_LATEST; high = H5F_LIBVER_LATEST; } +#endif /* HDF5_HAS_SWMR */ #else low = H5F_LIBVER_EARLIEST; high = H5F_LIBVER_LATEST; @@ -252,11 +256,13 @@ nc4_create_file(const char *path, int cmode, size_t initialsz, if ((retval = NC4_new_provenance(nc4_info))) BAIL(retval); +#ifdef HDF5_HAS_SWMR if ((cmode & NC_HDF5_SWMR)) { /* Prepare for single writer multiple readers */ if ((retval = H5Fstart_swmr_write(hdf5_info->hdfid))) BAIL(retval); } +#endif return NC_NOERR; diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c index b8a6bd70c5..f3ad84767a 100644 --- a/libhdf5/hdf5open.c +++ b/libhdf5/hdf5open.c @@ -656,9 +656,11 @@ nc4_open_file(const char *path, int mode, void* parameters, NC *nc) flags = H5F_ACC_RDWR; } else { flags = H5F_ACC_RDONLY; +#ifdef HDF5_HAS_SWMR if((mode & NC_HDF5_SWMR)) { flags |= H5F_ACC_SWMR_READ; } +#endif } /* Add necessary structs to hold netcdf-4 file data. */ @@ -836,12 +838,14 @@ nc4_open_file(const char *path, int mode, void* parameters, NC *nc) if (H5Pclose(fapl_id) < 0) BAIL(NC_EHDFERR); +#ifdef HDF5_HAS_SWMR /* Prepare for single writer multiple reader. */ if (mode & NC_WRITE && mode & NC_HDF5_SWMR) { if ((retval = H5Fstart_swmr_write(h5->hdfid))) { BAIL(retval); } } +#endif return NC_NOERR; diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c index 09748c8ecd..c18ec552af 100644 --- a/libhdf5/hdf5var.c +++ b/libhdf5/hdf5var.c @@ -1736,9 +1736,11 @@ NC4_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp, mem_spaceid, file_spaceid, xfer_plistid, bufr) < 0) BAIL(NC_EHDFERR); +#ifdef HDF5_HAS_SWMR /* Flush data for SWMR */ if (H5Dflush(hdf5_var->hdf_datasetid) < 0) BAIL(NC_EHDFERR); +#endif /* Remember that we have written to this var so that Fill Value * can't be set for it. */ diff --git a/nc_test4/tst_files.c b/nc_test4/tst_files.c index 5004f21fc5..c7eea558d0 100644 --- a/nc_test4/tst_files.c +++ b/nc_test4/tst_files.c @@ -292,6 +292,7 @@ main(int argc, char **argv) if (nc_close(ncid)) ERR; } SUMMARIZE_ERR; +#ifdef HDF5_HAS_SWMR printf("*** testing HDF5 SWMR..."); { #define DATA_LEN 3 @@ -363,6 +364,7 @@ main(int argc, char **argv) } SUMMARIZE_ERR; +#endif /* HDF_HAS_SWMR */ printf("*** testing CLASSIC_MODEL flag with classic formats..."); { int ncid; From 31c2dddef3d05ba800aa6bf3f01175cf532e2c45 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Mon, 6 Mar 2023 11:37:50 +0000 Subject: [PATCH 06/19] Fix typo in option name --- CMakeLists.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 56e6f1b369..0a12cb56c8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -756,7 +756,7 @@ IF(USE_HDF5) # Assert HDF5 version meets minimum required version. ## SET(HDF5_VERSION_REQUIRED 1.8.10) - IF(ENABLE_HDF5_SWMR_SUPPORT) + IF(ENABLE_HDF5_SWMR) SET(HDF5_VERSION_REQUIRED 1.10) MESSAGE(STATUS "HDF5 SWMR is enabled. This implies that HDF5 version at least ${HDF5_VERSION_REQUIRED} is required.") ENDIF() @@ -780,7 +780,8 @@ IF(USE_HDF5) unset(_hdf5_version) unset(_hdf5_version_lines) MESSAGE(STATUS "Found HDF5 libraries version ${HDF5_VERSION}") - IF(ENABLE_HDF5_SWMR_SUPPORT) + + IF(ENABLE_HDF5_SWMR) ADD_DEFINITIONS(-DHDF5_HAS_SWMR) ENDIF() ENDIF() From 8f0a2fcf89e9ecefd81daf6724bf0a6048f54342 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Mon, 6 Mar 2023 14:09:45 +0000 Subject: [PATCH 07/19] Add preprocessor define for HDF5 SWMR to `config.h` --- CMakeLists.txt | 7 ++----- config.h.cmake.in | 3 +++ 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0a12cb56c8..3be433d3eb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -780,10 +780,6 @@ IF(USE_HDF5) unset(_hdf5_version) unset(_hdf5_version_lines) MESSAGE(STATUS "Found HDF5 libraries version ${HDF5_VERSION}") - - IF(ENABLE_HDF5_SWMR) - ADD_DEFINITIONS(-DHDF5_HAS_SWMR) - ENDIF() ENDIF() ELSE() SET(HDF5_VERSION ${HDF5_VERSION} CACHE STRING "") @@ -806,7 +802,8 @@ IF(USE_HDF5) MESSAGE(FATAL_ERROR "netCDF requires at least HDF5 ${HDF5_VERSION_REQUIRED}. Found ${HDF5_VERSION}.") ENDIF() - + SET(HDF5_HAS_SWMR ${ENABLE_HDF5_SWMR}) + MESSAGE(STATUS "HDF5 SWMR support: ${HDF5_HAS_SWMR}") ## # Include the HDF5 include directory. diff --git a/config.h.cmake.in b/config.h.cmake.in index 7273971f31..fcbb72af3e 100644 --- a/config.h.cmake.in +++ b/config.h.cmake.in @@ -441,6 +441,9 @@ with zip */ /* if true, HDF5 is at least version 1.10.5 and supports UTF8 paths */ #cmakedefine HDF5_UTF8_PATHS 1 +/* if true, HDF5 is at least version 1.10 and supports single-writer multiple-reader IO */ +#cmakedefine HDF5_HAS_SWMR 1 + /* if true, backtrace support will be used. */ #cmakedefine HAVE_EXECINFO_H 1 From 0579c475cf3e6cd69246f0d92e77837690aa3850 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Tue, 7 Mar 2023 09:56:05 +0000 Subject: [PATCH 08/19] Add `--enable-hdf5-swmr` argument to `configure.ac` --- configure.ac | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 94bf84b4d7..d14c8401eb 100644 --- a/configure.ac +++ b/configure.ac @@ -156,6 +156,14 @@ if test "x$enable_netcdf4" = xno ; then enable_hdf5=no ; fi # disable-netcdf4 is synonym for disable-hdf5 AC_MSG_RESULT([$enable_hdf5]) +AC_MSG_CHECKING([whether we should build with HDF5 SWMR support]) +AC_ARG_ENABLE([hdf5_swmr], [AS_HELP_STRING([--enable-hdf5-swmr], + [build with HDF5 SWMR support (requires HDF5 version 1.10 or later)])]) +test "x$enable_hdf5_swmr" = xyes || enable_hdf5_swmr=no +if test "x$enable_hdf5" = xno ; then enable_hdf5_swmr=no ; fi +# Don't enable HDF5 SWMR if we're not using HDF5 +AC_MSG_RESULT([$enable_hdf5_swmr]) + # Check whether we want to enable CDF5 support. AC_MSG_CHECKING([whether CDF5 support should be disabled]) AC_ARG_ENABLE([cdf5], @@ -1481,6 +1489,7 @@ hdf5_parallel=no hdf5_supports_par_filters=no enable_hdf5_szip=no has_ross3=no +hdf5_swmr=no if test "x$enable_hdf5" = xyes; then @@ -1506,7 +1515,7 @@ if test "x$enable_hdf5" = xyes; then # H5Pset_fapl_mpiposix and H5Pget_fapl_mpiposix have been removed since HDF5 1.8.12. # Use H5Pset_fapl_mpio and H5Pget_fapl_mpio, instead. - AC_CHECK_FUNCS([H5Pget_fapl_mpio H5Pset_deflate H5Z_SZIP H5Pset_all_coll_metadata_ops H5Literate]) + AC_CHECK_FUNCS([H5Pget_fapl_mpio H5Pset_deflate H5Z_SZIP H5Pset_all_coll_metadata_ops H5Literate H5Fstart_swmr_write]) # Check to see if HDF5 library has collective metadata APIs, (HDF5 >= 1.10.0) if test "x$ac_cv_func_H5Pset_all_coll_metadata_ops" = xyes; then @@ -1568,6 +1577,13 @@ if test "x$enable_hdf5" = xyes; then AC_DEFINE([HDF5_UTF8_PATHS], [1], [if true, HDF5 paths can be utf-8]) fi + if test "x$enable_hdf5_swmr" = xyes && test "x$ac_cv_func_H5Fstart_swmr_write" = xyes; then + hdf5_swmr=yes + AC_DEFINE([HDF5_HAS_SWMR], [1], [if true, HDF5 supports single-writer multiple-reader]) + fi + AC_MSG_CHECKING([whether HDF5 supports SWMR]) + AC_MSG_RESULT([$hdf5_swmr]) + fi AM_CONDITIONAL(ENABLE_NCDUMPCHUNKS, [test "x$has_readchunks" = xyes ]) From ea1d617e2de61129cf604dd7438e09e87b746f0a Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Tue, 7 Mar 2023 09:58:32 +0000 Subject: [PATCH 09/19] Remove use of deleted `HAVE_H5PSET_LIBVER_BOUNDS` macro --- libhdf5/hdf5create.c | 7 ------- 1 file changed, 7 deletions(-) diff --git a/libhdf5/hdf5create.c b/libhdf5/hdf5create.c index 6268223ae5..cfbebfeaba 100644 --- a/libhdf5/hdf5create.c +++ b/libhdf5/hdf5create.c @@ -48,10 +48,7 @@ nc4_create_file(const char *path, int cmode, size_t initialsz, NC_FILE_INFO_T *nc4_info; NC_HDF5_FILE_INFO_T *hdf5_info; NC_HDF5_GRP_INFO_T *hdf5_grp; - -#ifdef HAVE_H5PSET_LIBVER_BOUNDS H5F_libver_t low, high; -#endif #ifdef USE_PARALLEL4 NC_MPI_INFO *mpiinfo = NULL; @@ -108,10 +105,8 @@ nc4_create_file(const char *path, int cmode, size_t initialsz, flags = H5F_ACC_TRUNC; #ifdef HDF5_HAS_SWMR -#ifndef HAVE_H5PSET_LIBVER_BOUNDS if (cmode & NC_HDF5_SWMR) flags |= H5F_ACC_SWMR_WRITE; -#endif #endif /* If this file already exists, and NC_NOCLOBBER is specified, @@ -172,7 +167,6 @@ nc4_create_file(const char *path, int cmode, size_t initialsz, gs->chunkcache.preemption)); } -#ifdef HAVE_H5PSET_LIBVER_BOUNDS #if H5_VERSION_GE(1,10,2) low = H5F_LIBVER_EARLIEST; high = H5F_LIBVER_V18; @@ -188,7 +182,6 @@ nc4_create_file(const char *path, int cmode, size_t initialsz, #endif if (H5Pset_libver_bounds(fapl_id, low, high) < 0) BAIL(NC_EHDFERR); -#endif { NCglobalstate* gs = NC_getglobalstate(); From cbbb87308af59cc3da0f6486cdfa296dd6ae2552 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Tue, 7 Mar 2023 13:26:32 +0000 Subject: [PATCH 10/19] Call `H5Pset_libver_bounds` when opening HDF5 file for SWMR --- libhdf5/hdf5open.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c index 1c458a5cb0..7196043b79 100644 --- a/libhdf5/hdf5open.c +++ b/libhdf5/hdf5open.c @@ -9,6 +9,7 @@ * @author Ed Hartnett */ +#include "H5version.h" #include "config.h" #include "hdf5internal.h" #include "hdf5err.h" @@ -834,6 +835,25 @@ nc4_open_file(const char *path, int mode, void* parameters, int ncid) } } + { + H5F_libver_t low, high; +#if H5_VERSION_GE(1,10,2) + low = H5F_LIBVER_EARLIEST; + high = H5F_LIBVER_V18; +#ifdef HDF5_HAS_SWMR + if ((mode & NC_HDF5_SWMR)) { + low = H5F_LIBVER_LATEST; + high = H5F_LIBVER_LATEST; + } +#endif /* HDF5_HAS_SWMR */ +#else + low = H5F_LIBVER_EARLIEST; + high = H5F_LIBVER_LATEST; +#endif + if (H5Pset_libver_bounds(fapl_id, low, high) < 0) + BAIL(NC_EHDFERR); + } + /* Set HDF5 format compatibility in the FILE ACCESS property list. * Compatibility is transient and must be reselected every time * a file is opened for writing. */ From ccdc9ff4452b959dd85eda8a00a02cf00b9f65c1 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Tue, 7 Mar 2023 13:27:59 +0000 Subject: [PATCH 11/19] Rely on file open flag to enable SWMR rather than manual start --- libhdf5/hdf5create.c | 8 -------- libhdf5/hdf5open.c | 14 +++++--------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/libhdf5/hdf5create.c b/libhdf5/hdf5create.c index cfbebfeaba..9024f8590b 100644 --- a/libhdf5/hdf5create.c +++ b/libhdf5/hdf5create.c @@ -280,14 +280,6 @@ nc4_create_file(const char *path, int cmode, size_t initialsz, if ((retval = NC4_new_provenance(nc4_info))) BAIL(retval); -#ifdef HDF5_HAS_SWMR - if ((cmode & NC_HDF5_SWMR)) { - /* Prepare for single writer multiple readers */ - if ((retval = H5Fstart_swmr_write(hdf5_info->hdfid))) - BAIL(retval); - } -#endif - return NC_NOERR; exit: /*failure exit*/ diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c index 7196043b79..dc2cba1ad8 100644 --- a/libhdf5/hdf5open.c +++ b/libhdf5/hdf5open.c @@ -711,6 +711,11 @@ nc4_open_file(const char *path, int mode, void* parameters, int ncid) /* Determine the HDF5 open flag to use. */ if((mode & NC_WRITE)) { flags = H5F_ACC_RDWR; +#ifdef HDF5_HAS_SWMR + if((mode & NC_HDF5_SWMR)) { + flags |= H5F_ACC_SWMR_WRITE; + } +#endif } else { flags = H5F_ACC_RDONLY; #ifdef HDF5_HAS_SWMR @@ -1003,15 +1008,6 @@ nc4_open_file(const char *path, int mode, void* parameters, int ncid) if (H5Pclose(fapl_id) < 0) BAIL(NC_EHDFERR); -#ifdef HDF5_HAS_SWMR - /* Prepare for single writer multiple reader. */ - if (mode & NC_WRITE && mode & NC_HDF5_SWMR) { - if ((retval = H5Fstart_swmr_write(h5->hdfid))) { - BAIL(retval); - } - } -#endif - return NC_NOERR; exit: From 2509d845fcb35eadeb9f752712392d7af889b512 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Tue, 7 Mar 2023 16:00:44 +0000 Subject: [PATCH 12/19] Set feature flags for HDF5 SWMR support in netcdf_meta.h/nc-config --- CMakeLists.txt | 1 + configure.ac | 2 ++ include/netcdf_meta.h.in | 1 + nc-config.cmake.in | 13 +++++++++++++ nc-config.in | 7 +++++++ netCDFConfig.cmake.in | 1 + 6 files changed, 25 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3be433d3eb..92d7f24aef 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2541,6 +2541,7 @@ is_enabled(ENABLE_V2_API HAS_NC2) is_enabled(ENABLE_NETCDF_4 HAS_NC4) is_enabled(ENABLE_HDF4 HAS_HDF4) is_enabled(USE_HDF5 HAS_HDF5) +is_enabled(HDF5_HAS_SWMR HAS_HDF5_SWMR) is_enabled(OFF HAS_BENCHMARKS) is_enabled(STATUS_PNETCDF HAS_PNETCDF) is_enabled(STATUS_PARALLEL HAS_PARALLEL) diff --git a/configure.ac b/configure.ac index d14c8401eb..53219e940b 100644 --- a/configure.ac +++ b/configure.ac @@ -1932,6 +1932,7 @@ AC_SUBST(HAS_CDF5,[$enable_cdf5]) AC_SUBST(HAS_HDF4,[$enable_hdf4]) AC_SUBST(HAS_BENCHMARKS,[$enable_benchmarks]) AC_SUBST(HAS_HDF5,[$enable_hdf5]) +AC_SUBST(HAS_HDF5_SWMR,[$hdf5_swmr]) AC_SUBST(HAS_PNETCDF,[$enable_pnetcdf]) AC_SUBST(HAS_LOGGING, [$enable_logging]) AC_SUBST(HAS_PARALLEL,[$enable_parallel]) @@ -2060,6 +2061,7 @@ AX_SET_META([NC_HAS_NC4],[$enable_netcdf_4],[yes]) AX_SET_META([NC_HAS_HDF4],[$enable_hdf4],[yes]) AX_SET_META([NC_HAS_BENCHMARKS],[$enable_benchmarks],[yes]) AX_SET_META([NC_HAS_HDF5],[$enable_hdf5],[yes]) +AX_SET_META([NC_HAS_HDF5_SWMR],[$enable_hdf5_swmr],[yes]) AX_SET_META([NC_HAS_DAP2],[$enable_dap],[yes]) AX_SET_META([NC_HAS_DAP4],[$enable_dap4],[yes]) AX_SET_META([NC_HAS_DISKLESS],[yes],[yes]) diff --git a/include/netcdf_meta.h.in b/include/netcdf_meta.h.in index 705cb6a526..94d319e92c 100644 --- a/include/netcdf_meta.h.in +++ b/include/netcdf_meta.h.in @@ -43,6 +43,7 @@ #define NC_HAS_NC4 @NC_HAS_NC4@ /*!< API version 4 support. */ #define NC_HAS_HDF4 @NC_HAS_HDF4@ /*!< HDF4 support. */ #define NC_HAS_HDF5 @NC_HAS_HDF5@ /*!< HDF5 support. */ +#define NC_HAS_HDF5_SWMR @NC_HAS_HDF5_SWMR@ /*!< HDF5 single-writer multiple reader support. */ #define NC_HAS_SZIP @NC_HAS_SZIP@ /*!< szip support */ #define NC_HAS_SZIP_WRITE @NC_HAS_SZIP@ /*!< szip write support */ #define NC_HAS_DAP2 @NC_HAS_DAP2@ /*!< DAP2 support. */ diff --git a/nc-config.cmake.in b/nc-config.cmake.in index e9dda9999d..c4dd85b1f3 100644 --- a/nc-config.cmake.in +++ b/nc-config.cmake.in @@ -74,6 +74,13 @@ else has_hdf5="yes" fi +has_hdf5_swmr="@HDF5_HAS_SWMR@" +if [ -z "$has_hdf5_swmr" -o "$has_hdf5_swmr" = "OFF" ]; then + has_hdf5_swmr="no" +else + has_hdf5_swmr="yes" +fi + has_szlib="@USE_SZIP@" if [ -z "$has_szlib" -o "$has_szlib" = "OFF" ]; then has_szlib="no" @@ -157,6 +164,7 @@ Available values for OPTION include: --has-nc2 whether NetCDF-2 API is enabled --has-nc4 whether NetCDF-4/HDF-5 is enabled in this build --has-hdf5 whether HDF5 is used in build (always the same as --has-nc4) + --has-hdf5-swmr whether HDF5 single-writer multiple-reader mode is supported --has-hdf4 whether HDF4 was used in build --has-logging whether logging is enabled with --enable-logging. --has-pnetcdf whether PnetCDF was used in build @@ -202,6 +210,7 @@ all() echo " --has-nc2 -> $has_nc2" echo " --has-nc4 -> $has_nc4" echo " --has-hdf5 -> $has_hdf5" + echo " --has-hdf5-swmr -> $has_hdf5_swmr" echo " --has-hdf4 -> $has_hdf4" echo " --has-logging -> $has_logging" echo " --has-pnetcdf -> $has_pnetcdf" @@ -293,6 +302,10 @@ while test $# -gt 0; do echo $has_hdf5 ;; + --has-hdf5-swmr) + echo $has_hdf5_swmr + ;; + --has-hdf4) echo $has_hdf4 ;; diff --git a/nc-config.in b/nc-config.in index 66913a1ac5..e46b67f689 100644 --- a/nc-config.in +++ b/nc-config.in @@ -27,6 +27,7 @@ has_nc4="@HAS_NC4@" has_hdf4="@HAS_HDF4@" has_pnetcdf="@HAS_PNETCDF@" has_hdf5="@HAS_HDF5@" +has_hdf5_swmr="@HAS_HDF5_SWMR@" has_logging="@HAS_LOGGING@" has_cdf5="@HAS_CDF5@" has_szlib="@HAS_SZLIB@" @@ -59,6 +60,7 @@ Available values for OPTION include: --has-nc2 whether NetCDF-2 API is enabled --has-nc4 whether NetCDF-4/HDF-5 is enabled in this build --has-hdf5 whether HDF5 is used in build (always the same as --has-nc4) + --has-hdf5-swmr whether HDF5 single-writer multiple-reader mode is supported --has-hdf4 whether HDF4 was used in build --has-logging whether logging is enabled with --enable-logging. --has-pnetcdf whether PnetCDF was used in build @@ -101,6 +103,7 @@ all() echo " --has-nc2 -> $has_nc2" echo " --has-nc4 -> $has_nc4" echo " --has-hdf5 -> $has_hdf5" + echo " --has-hdf5-swmr -> $has_hdf5_swmr" echo " --has-hdf4 -> $has_hdf4" echo " --has-logging -> $has_logging" echo " --has-pnetcdf -> $has_pnetcdf" @@ -193,6 +196,10 @@ while test $# -gt 0; do echo $has_hdf5 ;; + --has-hdf5-swmr) + echo $has_hdf5_swmr + ;; + --has-hdf4) echo $has_hdf4 ;; diff --git a/netCDFConfig.cmake.in b/netCDFConfig.cmake.in index 9d68eec5a2..82c2a3ac5c 100644 --- a/netCDFConfig.cmake.in +++ b/netCDFConfig.cmake.in @@ -32,6 +32,7 @@ set(netCDF_HAS_NC2 @HAS_NC2@) set(netCDF_HAS_NC4 @HAS_NC4@) set(netCDF_HAS_HDF4 @HAS_HDF4@) set(netCDF_HAS_HDF5 @HAS_HDF5@) +set(netCDF_HAS_HDF5_SWMR @HAS_HDF5_SWMR@) set(netCDF_HAS_PNETCDF @HAS_PNETCDF@) set(netCDF_HAS_PARALLEL @HAS_PARALLEL@) set(netCDF_HAS_DAP @HAS_DAP2@) From c8a85c4d5c47ceadab926aa0b678d3e12694d2ef Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Mon, 20 Nov 2023 17:09:24 +0000 Subject: [PATCH 13/19] CMake: Fix detecting HDF5 SWMR support Previously, setting some HDF5 CMake config flags directly would skip both the check for HDF5 1.10 and setting the `HDF5_HAS_SWMR` flag --- CMakeLists.txt | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 291aeb0606..1bd900f690 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -655,7 +655,10 @@ IF(USE_HDF5) # Assert HDF5 version meets minimum required version. ## SET(HDF5_VERSION_REQUIRED 1.8.10) - + IF(ENABLE_HDF5_SWMR) + SET(HDF5_VERSION_REQUIRED 1.10) + MESSAGE(STATUS "HDF5 SWMR is enabled. This implies that HDF5 version at least ${HDF5_VERSION_REQUIRED} is required.") + ENDIF() ## # Accommodate developers who have hdf5 libraries and @@ -735,15 +738,6 @@ IF(USE_HDF5) # HDF5 variables we need to munge. ## - ## - # Assert HDF5 version meets minimum required version. - ## - SET(HDF5_VERSION_REQUIRED 1.8.10) - IF(ENABLE_HDF5_SWMR) - SET(HDF5_VERSION_REQUIRED 1.10) - MESSAGE(STATUS "HDF5 SWMR is enabled. This implies that HDF5 version at least ${HDF5_VERSION_REQUIRED} is required.") - ENDIF() - # Some versions of HDF5 set HDF5_VERSION_STRING instead of HDF5_VERSION IF(HDF5_VERSION_STRING AND NOT HDF5_VERSION) SET(HDF5_VERSION ${HDF5_VERSION_STRING}) @@ -785,9 +779,6 @@ IF(USE_HDF5) MESSAGE(FATAL_ERROR "netCDF requires at least HDF5 ${HDF5_VERSION_REQUIRED}. Found ${HDF5_VERSION}.") ENDIF() - SET(HDF5_HAS_SWMR ${ENABLE_HDF5_SWMR}) - MESSAGE(STATUS "HDF5 SWMR support: ${HDF5_HAS_SWMR}") - ## # Include the HDF5 include directory. ## @@ -852,6 +843,11 @@ IF(USE_HDF5) ENDIF(HDF5_C_LIBRARY AND HDF5_HL_LIBRARY AND HDF5_INCLUDE_DIR) + # Requesting SWMR support sets the minimum version to 1.10, so if + # we've got this far, we've got SWMR support + set(HDF5_HAS_SWMR ${ENABLE_HDF5_SWMR}) + message(STATUS "HDF5 SWMR support: ${HDF5_HAS_SWMR}") + FIND_PACKAGE(Threads) # There is a missing case in the above code so default it From cfdb7ef41a81bcd7a203839b6af896bb554aa966 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Mon, 20 Nov 2023 17:19:58 +0000 Subject: [PATCH 14/19] Refresh dataset metadata before reading, if using HDF5 SWMR --- libhdf5/hdf5var.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c index bc0b70d39a..3906d54354 100644 --- a/libhdf5/hdf5var.c +++ b/libhdf5/hdf5var.c @@ -1905,6 +1905,15 @@ NC4_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp, LOG((3, "%s: var->hdr.name %s mem_nc_type %d", __func__, var->hdr.name, mem_nc_type)); +#ifdef HDF5_HAS_SWMR + /* Refresh dataset metadata, required if opened in SWMR mode */ + if (h5->cmode & NC_HDF5_SWMR) + { + if (H5Drefresh(hdf5_var->hdf_datasetid) < 0) + BAIL(NC_EHDFERR); + } +#endif + /* Check some stuff about the type and the file. Also end define * mode, if needed. */ if ((retval = check_for_vara(&mem_nc_type, var, h5))) From 2d1725ba4f1a2a81776e15468712b6d1af7931fa Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Mon, 20 Nov 2023 17:20:29 +0000 Subject: [PATCH 15/19] Only auto-flush data if in SWMR mode --- libhdf5/hdf5var.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c index 3906d54354..ab177434ad 100644 --- a/libhdf5/hdf5var.c +++ b/libhdf5/hdf5var.c @@ -1798,8 +1798,11 @@ NC4_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp, #ifdef HDF5_HAS_SWMR /* Flush data for SWMR */ - if (H5Dflush(hdf5_var->hdf_datasetid) < 0) - BAIL(NC_EHDFERR); + if (h5->cmode & NC_HDF5_SWMR) + { + if (H5Dflush(hdf5_var->hdf_datasetid) < 0) + BAIL(NC_EHDFERR); + } #endif /* Remember that we have written to this var so that Fill Value From cae96c99fad06cdc2f95e63f2d7017807ef2b4fc Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Thu, 23 Nov 2023 14:57:44 +0000 Subject: [PATCH 16/19] Don't set libver bounds for SWMR mode Libver bounds already taken care of elsewhere, and merely opening the file in SWMR mode will correctly set format to at least v110 --- libhdf5/hdf5create.c | 16 ---------------- libhdf5/hdf5open.c | 19 ------------------- 2 files changed, 35 deletions(-) diff --git a/libhdf5/hdf5create.c b/libhdf5/hdf5create.c index 9024f8590b..f3f39cffeb 100644 --- a/libhdf5/hdf5create.c +++ b/libhdf5/hdf5create.c @@ -167,22 +167,6 @@ nc4_create_file(const char *path, int cmode, size_t initialsz, gs->chunkcache.preemption)); } -#if H5_VERSION_GE(1,10,2) - low = H5F_LIBVER_EARLIEST; - high = H5F_LIBVER_V18; -#ifdef HDF5_HAS_SWMR - if ((cmode & NC_HDF5_SWMR)) { - low = H5F_LIBVER_LATEST; - high = H5F_LIBVER_LATEST; - } -#endif /* HDF5_HAS_SWMR */ -#else - low = H5F_LIBVER_EARLIEST; - high = H5F_LIBVER_LATEST; -#endif - if (H5Pset_libver_bounds(fapl_id, low, high) < 0) - BAIL(NC_EHDFERR); - { NCglobalstate* gs = NC_getglobalstate(); if(gs->alignment.defined) { diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c index 044cc3c786..9c0f114d46 100644 --- a/libhdf5/hdf5open.c +++ b/libhdf5/hdf5open.c @@ -856,25 +856,6 @@ nc4_open_file(const char *path, int mode, void* parameters, int ncid) } } - { - H5F_libver_t low, high; -#if H5_VERSION_GE(1,10,2) - low = H5F_LIBVER_EARLIEST; - high = H5F_LIBVER_V18; -#ifdef HDF5_HAS_SWMR - if ((mode & NC_HDF5_SWMR)) { - low = H5F_LIBVER_LATEST; - high = H5F_LIBVER_LATEST; - } -#endif /* HDF5_HAS_SWMR */ -#else - low = H5F_LIBVER_EARLIEST; - high = H5F_LIBVER_LATEST; -#endif - if (H5Pset_libver_bounds(fapl_id, low, high) < 0) - BAIL(NC_EHDFERR); - } - /* Set HDF5 format compatibility in the FILE ACCESS property list. * Compatibility is transient and must be reselected every time * a file is opened for writing. */ From da731ec85023ef5835cc0b11e0c812989246c5f0 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Thu, 23 Nov 2023 15:11:17 +0000 Subject: [PATCH 17/19] Use unique flag for SWMR mode --- include/netcdf.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/netcdf.h b/include/netcdf.h index a6b0330305..62dfb8652a 100644 --- a/include/netcdf.h +++ b/include/netcdf.h @@ -161,7 +161,7 @@ Use this in mode flags for both nc_create() and nc_open(). */ #define NC_PERSIST 0x4000 /**< Save diskless contents to disk. Mode flag for nc_open() or nc_create() */ #define NC_INMEMORY 0x8000 /**< Read from memory. Mode flag for nc_open() or nc_create() */ -#define NC_HDF5_SWMR 0x2000 /** Reuse deprecated MPIIO flag for SWMR **/ +#define NC_HDF5_SWMR 0x10000 /**< Enable HDF5's Single Writer Multiple Reader mode **/ /* Upper 16 bits */ #define NC_NOATTCREORD 0x20000 /**< Disable the netcdf-4 (hdf5) attribute creation order tracking */ From 69359301ffa73554f46580ebc5dd37f4659e475b Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Thu, 23 Nov 2023 15:58:55 +0000 Subject: [PATCH 18/19] Refresh variable metadata when getting dimension length for SWMR --- libhdf5/nc4hdf.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/libhdf5/nc4hdf.c b/libhdf5/nc4hdf.c index a84abd0223..387c73c217 100644 --- a/libhdf5/nc4hdf.c +++ b/libhdf5/nc4hdf.c @@ -221,6 +221,16 @@ nc4_open_var_grp2(NC_GRP_INFO_T *grp, int varid, hid_t *dataset) if ((hdf5_var->hdf_datasetid = H5Dopen2(hdf5_grp->hdf_grpid, var->hdr.name, H5P_DEFAULT)) < 0) return NC_ENOTVAR; + } else { +#ifdef HDF5_HAS_SWMR + /* If file is opened in SWMR mode, we need to refresh the + * dataset's metadata */ + if (grp->nc4_info->cmode & NC_HDF5_SWMR) + { + if (H5Drefresh(hdf5_var->hdf_datasetid) < 0) + return NC_EHDFERR; + } +#endif } *dataset = hdf5_var->hdf_datasetid; From ff7bde58f7dac0055c4e783b9dd2085f5c216908 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Thu, 23 Nov 2023 16:02:21 +0000 Subject: [PATCH 19/19] Add test that properly exercises SWMR mode --- nc_test4/CMakeLists.txt | 6 ++++ nc_test4/test_hdf5_swmr.sh | 35 ++++++++++++++++++++ nc_test4/test_hdf5_swmr_reader.c | 45 ++++++++++++++++++++++++++ nc_test4/test_hdf5_swmr_writer.c | 53 ++++++++++++++++++++++++++++++ nc_test4/test_swmr_common.h | 55 ++++++++++++++++++++++++++++++++ nc_test4/tst_files.c | 2 ++ 6 files changed, 196 insertions(+) create mode 100755 nc_test4/test_hdf5_swmr.sh create mode 100644 nc_test4/test_hdf5_swmr_reader.c create mode 100644 nc_test4/test_hdf5_swmr_writer.c create mode 100644 nc_test4/test_swmr_common.h diff --git a/nc_test4/CMakeLists.txt b/nc_test4/CMakeLists.txt index 117277e6c3..a8f7b42fde 100644 --- a/nc_test4/CMakeLists.txt +++ b/nc_test4/CMakeLists.txt @@ -71,6 +71,12 @@ IF(${HDF5_VERSION} VERSION_GREATER "1.10.0") SET(NC4_TESTS ${NC4_TESTS} tst_virtual_datasets) ENDIF(${HDF5_VERSION} VERSION_GREATER "1.10.0") +if (ENABLE_HDF5_SWMR) + build_bin_test(test_hdf5_swmr_writer) + build_bin_test(test_hdf5_swmr_reader) + add_sh_test(nc_test4 test_hdf5_swmr) +endif() + ## # The shell script, run_empty_vlen_test.sh, # depends on the 'tst_empty_vlen_unlim' binary. diff --git a/nc_test4/test_hdf5_swmr.sh b/nc_test4/test_hdf5_swmr.sh new file mode 100755 index 0000000000..3de060b0d7 --- /dev/null +++ b/nc_test4/test_hdf5_swmr.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# Test ability to use HDF5's SWMR functionality to simultaneously +# write to and read from a file in different processes. +# +# 1. Create file, enabling SWMR mode; create all variables +# 2. Close and reopen file for writing in SWMR mode +# 3. Then open file in a separate process for SWMR reading +# +# Note that these *must* happen sequentially -- the reader process +# *must not* start until the writer process in step 2 has started. +# +# At this point we can perform some tests on the file. + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +set -e + +# Clean up any test files +rm -f ./test_hdf5_swmr_file.nc + +echo " *** Testing SWMR mode" + +# Launch the writer process in the background +${execdir}/test_hdf5_swmr_writer & + +# Pause briefly here to ensure writer process has both created file +# and then *reopened it* +sleep 0.5 + +# Now we're safe to launch the reader process +${execdir}/test_hdf5_swmr_reader + +echo " *** Pass: SWMR mode" diff --git a/nc_test4/test_hdf5_swmr_reader.c b/nc_test4/test_hdf5_swmr_reader.c new file mode 100644 index 0000000000..e583b9c600 --- /dev/null +++ b/nc_test4/test_hdf5_swmr_reader.c @@ -0,0 +1,45 @@ +/* This program reads a file currently being modified by another + * process. See test_hdf5_swmr_writer.c for details +*/ + +#include "test_swmr_common.h" +#include "netcdf.h" +#include +#include + + +int main(int argc, char** argv) +{ + int ncid, varid; + int values[1]; + size_t start[NUM_DIMS] = {0}, count[NUM_DIMS] = {1}; + + /* Open the file for SWMR reading, verify data, and close. */ + if ((status = nc_open(FILE_NAME, NC_HDF5_SWMR, &ncid))) ERR_MSG; + if ((status = nc_inq_varid(ncid, "depth", &varid))) ERR_MSG; + + int time_dim; + if ((status = nc_inq_dimid(ncid, TIME_DIM_NAME, &time_dim))) ERR_MSG; + + size_t time_dim_len_start; + if ((status = nc_inq_dimlen(ncid, time_dim, &time_dim_len_start))) ERR_MSG; + + // Read initial slice + start[0] = time_dim_len_start - 1; + if ((status = nc_get_vara_int(ncid, varid, start, count, values))) ERR_MSG; + if (values[0] != (int)time_dim_len_start - 1) ERR; + + // Wait a little bit to be sure the writer process has written a few more slices + sleep(1); + + size_t time_dim_len_end; + if ((status = nc_inq_dimlen(ncid, time_dim, &time_dim_len_end))) ERR_MSG; + + if (time_dim_len_end <= time_dim_len_start) ERR; + + start[0] = time_dim_len_end - 1; + if ((status = nc_get_vara_int(ncid, varid, start, count, values))) ERR_MSG; + if (values[0] != (int)time_dim_len_end - 1) ERR; + + if ((status = nc_close(ncid))) ERR_MSG; +} diff --git a/nc_test4/test_hdf5_swmr_writer.c b/nc_test4/test_hdf5_swmr_writer.c new file mode 100644 index 0000000000..a66c066cf5 --- /dev/null +++ b/nc_test4/test_hdf5_swmr_writer.c @@ -0,0 +1,53 @@ +/* This program uses the HDF5 SWMR mode to write to a file while a + * second process simultaneously reads from it. As there are some + * strict ordering requirements, this is orchestrated by an + * accompanying shell script. + * + * Here, we create the file, create our dimensions and variables, then + * close and reopen it to write data. We write half the data, pause, + * then write the other half. This gives us the opportunity to see the + * data get *updated* in the reader process. +*/ + +#include "test_swmr_common.h" +#include "netcdf.h" +#include + + +int main(int argc, char** argv) +{ + int ncid, varid, dimids[NUM_DIMS]; + size_t i; + int values[1]; + size_t start[NUM_DIMS] = {0}, count[NUM_DIMS] = {1}; + + /* Create a file in SWMR mode for writing, create structure and close. */ + if (nc_create(FILE_NAME, NC_NETCDF4|NC_HDF5_SWMR, &ncid)) ERR; + if (nc_def_dim(ncid, TIME_DIM_NAME, NC_UNLIMITED, &dimids[0])) ERR; + if (nc_def_var(ncid, DEPTH_VAR_NAME, NC_INT, NUM_DIMS, dimids, &varid)) ERR; + if (nc_close(ncid)) ERR; + + /* Open the file for SWMR writing, append data, and close. */ + if (nc_open(FILE_NAME, NC_WRITE|NC_HDF5_SWMR, &ncid)) ERR; + if (nc_inq_varid(ncid, "depth", &varid)) ERR; + + /* Write a bunch of initial values */ + for (i = 0; i < TOTAL_LENGTH / 2; i++) { + start[0] = i; + values[0] = (int)i; + if (nc_put_vara_int(ncid, varid, start, count, values)) ERR; + } + + /* Pause here briefly so reader process has a chance to start */ + sleep(1); + + /* At this point, the reader process should've opened and read the + * first half of the data so we can write the second half */ + for (i = TOTAL_LENGTH / 2; i < TOTAL_LENGTH; i++) { + start[0] = i; + values[0] = (int)i; + if (nc_put_vara_int(ncid, varid, start, count, values)) ERR; + } + + if (nc_close(ncid)) ERR; +} diff --git a/nc_test4/test_swmr_common.h b/nc_test4/test_swmr_common.h new file mode 100644 index 0000000000..c0f361ce62 --- /dev/null +++ b/nc_test4/test_swmr_common.h @@ -0,0 +1,55 @@ +// Some common defines for the HDF5 SWMR mode tests + +#ifndef NC_TEST_SWMR_COMMON_H +#define NC_TEST_SWMR_COMMON_H + +#include "config.h" + +#ifdef HAVE_UNISTD_H +#include "unistd.h" +#endif +#ifdef _WIN32 +#include +#endif + +#include + +// Portability macro for sleep in seconds +#ifdef _WIN32 +#define SLEEP(x) Sleep((x) * 1000) +#else +#define SLEEP(x) sleep((x)) +#endif + +#define FILE_NAME "test_hdf5_swmr_file.nc" +#define TIME_DIM_NAME "time" +#define BEAM_DIM_NAME "beam" +#define DEPTH_VAR_NAME "depth" +#define NUM_DIMS 1 +#define TOTAL_LENGTH 10000 + +// netCDF function error code +static int status = 0; + +/* This macro prints an error message with line number and name of + * test program. */ +#define ERR do { \ + fflush(stdout); /* Make sure our stdout is synced with stderr. */ \ + fprintf(stderr, "Sorry! Unexpected result, %s, line: %d\n", \ + __FILE__, __LINE__); \ + fflush(stderr); \ + return 2; \ + } while (0) + +/* This macro prints an error message with line number and name of + * test program, along with netcdf error message */ +#define ERR_MSG do { \ + fflush(stdout); /* Make sure our stdout is synced with stderr. */ \ + fprintf(stderr, "Failed: %s, %s, line: %d\n", \ + nc_strerror(status), __FILE__, __LINE__); \ + fflush(stderr); \ + return 2; \ + } while (0) + + +#endif /* NC_TEST_SWMR_COMMON_H */ diff --git a/nc_test4/tst_files.c b/nc_test4/tst_files.c index 0220c254ef..68f27339cc 100644 --- a/nc_test4/tst_files.c +++ b/nc_test4/tst_files.c @@ -293,6 +293,8 @@ main(int argc, char **argv) } SUMMARIZE_ERR; #ifdef HDF5_HAS_SWMR + // Note that this test is really only a *very* rough test. See + // `test_hdf5_swmr.sh` for a more complete test printf("*** testing HDF5 SWMR..."); { #define DATA_LEN 3