From 65ab499e276554bd900e7ddd70e6237b339e499f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Beck=20K=C3=B6nig?= Date: Sun, 2 Mar 2025 11:38:24 +0100 Subject: [PATCH 001/141] enable threadsafe feature for static build --- hdf5-sys/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hdf5-sys/Cargo.toml b/hdf5-sys/Cargo.toml index 6813b0f6e..8f0206ae6 100644 --- a/hdf5-sys/Cargo.toml +++ b/hdf5-sys/Cargo.toml @@ -5,7 +5,7 @@ description = "Native bindings to the HDF5 library." links = "hdf5" readme = "README.md" categories = ["development-tools::ffi", "filesystem", "science"] -version = "0.10.1" # !V +version = "0.10.1" # !V rust-version.workspace = true authors.workspace = true keywords.workspace = true @@ -28,7 +28,7 @@ mpio = ["dep:mpi-sys"] hl = ["hdf5-src?/hl"] threadsafe = ["hdf5-src?/threadsafe"] zlib = ["dep:libz-sys", "hdf5-src?/zlib"] -static = ["dep:hdf5-src"] +static = ["hdf5-src/threadsafe"] deprecated = ["hdf5-src?/deprecated"] [build-dependencies] From 37979ff485e58cc62c6721036ea65d097ad359f3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Mar 2025 19:56:06 +0000 Subject: [PATCH 002/141] Bump crate-ci/typos from 1.30.2 to 1.31.1 Bumps [crate-ci/typos](https://github.com/crate-ci/typos) from 1.30.2 to 1.31.1. - [Release notes](https://github.com/crate-ci/typos/releases) - [Changelog](https://github.com/crate-ci/typos/blob/master/CHANGELOG.md) - [Commits](https://github.com/crate-ci/typos/compare/7bc041cbb7ca9167c9e0e4ccbb26f48eb0f9d4e0...b1a1ef3893ff35ade0cfa71523852a49bfd05d19) --- updated-dependencies: - dependency-name: crate-ci/typos dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f210a85e6..c18bc1c64 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - name: Check spelling - uses: crate-ci/typos@7bc041cbb7ca9167c9e0e4ccbb26f48eb0f9d4e0 # v1.30.2 + uses: crate-ci/typos@b1a1ef3893ff35ade0cfa71523852a49bfd05d19 # v1.31.1 lint: name: lint From 0aaa10b34e818246788764deb345728a3e4db81d Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Fri, 4 Apr 2025 07:53:34 +0200 Subject: [PATCH 003/141] Clippy lints --- hdf5-derive/src/lib.rs | 2 +- hdf5-types/src/lib.rs | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/hdf5-derive/src/lib.rs b/hdf5-derive/src/lib.rs index 41592f7ef..a2c2d75de 100644 --- a/hdf5-derive/src/lib.rs +++ b/hdf5-derive/src/lib.rs @@ -108,7 +108,7 @@ fn impl_enum(names: &[String], values: &[Expr], repr: &Ident) -> TokenStream { fn is_phantom_data(ty: &Type) -> bool { match *ty { Type::Path(TypePath { qself: None, ref path }) => { - path.segments.iter().last().is_some_and(|x| x.ident == "PhantomData") + path.segments.iter().next_back().is_some_and(|x| x.ident == "PhantomData") } _ => false, } diff --git a/hdf5-types/src/lib.rs b/hdf5-types/src/lib.rs index a1daa8337..a46e05e2a 100644 --- a/hdf5-types/src/lib.rs +++ b/hdf5-types/src/lib.rs @@ -8,10 +8,10 @@ //! //! Crate features: //! * `h5-alloc`: Use the `hdf5` allocator for varlen types and dynamic values. -//! This is necessary on platforms which uses different allocators -//! in different libraries (e.g. dynamic libraries on windows), -//! or if `hdf5-c` is compiled with the MEMCHECKER option. -//! This option is forced on in the case of using a `windows` DLL. +//! This is necessary on platforms which uses different allocators +//! in different libraries (e.g. dynamic libraries on windows), +//! or if `hdf5-c` is compiled with the MEMCHECKER option. +//! This option is forced on in the case of using a `windows` DLL. #[cfg(test)] #[macro_use] From 229eb9dc9c1c5716e68cd352da152eba90cdcfe3 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Sun, 13 Apr 2025 16:09:19 +0200 Subject: [PATCH 004/141] MSRV: Keep compat --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c18bc1c64..3fe469df5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -294,7 +294,7 @@ jobs: cargo update - name: Override deps run: - cargo update half@2.5.0 --precise 2.4.1 + cargo update half@2.6.0 --precise 2.4.1 - name: Build and test all crates run: cargo test --locked --workspace -vv --features=hdf5-sys/static,hdf5-sys/zlib --exclude=hdf5-metno-derive From 47fdbd5ebb326f527e5068ea8005b30f0282bb59 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Thu, 17 Apr 2025 13:30:17 +0200 Subject: [PATCH 005/141] Bump ubuntu-20.04 to ubuntu-22.04 --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3fe469df5..d9915bc8c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -173,7 +173,7 @@ jobs: apt: name: apt - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: @@ -279,7 +279,7 @@ jobs: msrv: name: Minimal Supported Rust Version - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: fail-fast: false steps: From 723479344c568ef27fb359e4cdcf69ee3271bcd5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Apr 2025 20:20:32 +0000 Subject: [PATCH 006/141] Bump crate-ci/typos from 1.31.1 to 1.31.2 Bumps [crate-ci/typos](https://github.com/crate-ci/typos) from 1.31.1 to 1.31.2. - [Release notes](https://github.com/crate-ci/typos/releases) - [Changelog](https://github.com/crate-ci/typos/blob/master/CHANGELOG.md) - [Commits](https://github.com/crate-ci/typos/compare/b1a1ef3893ff35ade0cfa71523852a49bfd05d19...3be83342e28b9421997e9f781f713f8dde8453d2) --- updated-dependencies: - dependency-name: crate-ci/typos dependency-version: 1.31.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d9915bc8c..b979fc901 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - name: Check spelling - uses: crate-ci/typos@b1a1ef3893ff35ade0cfa71523852a49bfd05d19 # v1.31.1 + uses: crate-ci/typos@3be83342e28b9421997e9f781f713f8dde8453d2 # v1.31.2 lint: name: lint From 9c69f1ba7dabbedf55e164d5eb4ba0ebc7d0e4f3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 May 2025 18:49:14 +0000 Subject: [PATCH 007/141] Bump crate-ci/typos from 1.31.2 to 1.32.0 Bumps [crate-ci/typos](https://github.com/crate-ci/typos) from 1.31.2 to 1.32.0. - [Release notes](https://github.com/crate-ci/typos/releases) - [Changelog](https://github.com/crate-ci/typos/blob/master/CHANGELOG.md) - [Commits](https://github.com/crate-ci/typos/compare/3be83342e28b9421997e9f781f713f8dde8453d2...0f0ccba9ed1df83948f0c15026e4f5ccfce46109) --- updated-dependencies: - dependency-name: crate-ci/typos dependency-version: 1.32.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b979fc901..0aa7b177e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - name: Check spelling - uses: crate-ci/typos@3be83342e28b9421997e9f781f713f8dde8453d2 # v1.31.2 + uses: crate-ci/typos@0f0ccba9ed1df83948f0c15026e4f5ccfce46109 # v1.32.0 lint: name: lint From 488cc54e2c6e02d3e467ba3cd79ade290bf7cfdd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Jun 2025 19:18:40 +0000 Subject: [PATCH 008/141] Bump crate-ci/typos from 1.32.0 to 1.33.1 Bumps [crate-ci/typos](https://github.com/crate-ci/typos) from 1.32.0 to 1.33.1. - [Release notes](https://github.com/crate-ci/typos/releases) - [Changelog](https://github.com/crate-ci/typos/blob/master/CHANGELOG.md) - [Commits](https://github.com/crate-ci/typos/compare/0f0ccba9ed1df83948f0c15026e4f5ccfce46109...b1ae8d918b6e85bd611117d3d9a3be4f903ee5e4) --- updated-dependencies: - dependency-name: crate-ci/typos dependency-version: 1.33.1 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0aa7b177e..7fa4c25e5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - name: Check spelling - uses: crate-ci/typos@0f0ccba9ed1df83948f0c15026e4f5ccfce46109 # v1.32.0 + uses: crate-ci/typos@b1ae8d918b6e85bd611117d3d9a3be4f903ee5e4 # v1.33.1 lint: name: lint From 76545e20a02d3412c7e9c004741b95fdac9b93c3 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Mon, 30 Jun 2025 09:29:34 +0200 Subject: [PATCH 009/141] Fix clippy lints --- hdf5-sys/build.rs | 48 ++++++++++++++++++++-------------------- hdf5-types/src/h5type.rs | 8 +++---- hdf5-types/src/string.rs | 2 +- hdf5/build.rs | 4 ++-- 4 files changed, 31 insertions(+), 31 deletions(-) diff --git a/hdf5-sys/build.rs b/hdf5-sys/build.rs index 142b1df9a..ff9918574 100644 --- a/hdf5-sys/build.rs +++ b/hdf5-sys/build.rs @@ -14,7 +14,7 @@ use std::process::Command; use regex::Regex; fn feature_enabled(feature: &str) -> bool { - env::var(format!("CARGO_FEATURE_{}", feature)).is_ok() + env::var(format!("CARGO_FEATURE_{feature}")).is_ok() } #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Default)] @@ -130,7 +130,7 @@ fn validate_runtime_version(config: &Config) { println!("Adding extra link paths (ld)..."); for caps in re.captures_iter(&ldv) { let path = &caps["path"]; - println!(" {}", path); + println!(" {path}"); link_paths.push(path.into()); } } else { @@ -144,10 +144,10 @@ fn validate_runtime_version(config: &Config) { if let Some(filename) = path.file_name() { let filename = filename.to_str().unwrap_or(""); if path.is_file() && libfiles.contains(&filename) { - println!("Attempting to load: {:?}", path); + println!("Attempting to load: {path:?}"); match get_runtime_version_single(&path) { Ok(version) => { - println!(" => runtime version = {:?}", version); + println!(" => runtime version = {version:?}"); if version == config.header.version { println!("HDF5 library runtime version matches headers."); return; @@ -158,7 +158,7 @@ fn validate_runtime_version(config: &Config) { ); } Err(err) => { - println!(" => {}", err); + println!(" => {err}"); } } } @@ -185,7 +185,7 @@ impl Header { let inc_dir = inc_dir.as_ref(); let header = get_conf_header(inc_dir); - println!("Parsing HDF5 config from:\n {:?}", header); + println!("Parsing HDF5 config from:\n {header:?}"); let contents = fs::read_to_string(header).unwrap(); let mut hdr = Self::default(); @@ -217,7 +217,7 @@ impl Header { if let Some(version) = Version::parse(value) { hdr.version = version; } else { - panic!("Invalid H5_VERSION: {:?}", value); + panic!("Invalid H5_VERSION: {value:?}"); } }; } @@ -274,11 +274,11 @@ mod pkgconf { println!("Found HDF5 pkg-config entry"); println!(" Include paths:"); for dir in &library.include_paths { - println!(" - {:?}", dir); + println!(" - {dir:?}"); } println!(" Link paths:"); for dir in &library.link_paths { - println!(" - {:?}", dir); + println!(" - {dir:?}"); } for dir in &library.include_paths { if is_inc_dir(dir) { @@ -289,7 +289,7 @@ mod pkgconf { } if let Some(ref inc_dir) = config.inc_dir { println!("Located HDF5 headers at:"); - println!(" {:?}", inc_dir); + println!(" {inc_dir:?}"); } else { println!("Unable to locate HDF5 headers from pkg-config info."); } @@ -314,8 +314,8 @@ mod unix { ("/usr/include", "/usr/lib64"), ] { if is_inc_dir(inc_dir) { - println!("Found HDF5 headers at:\n {:?}", inc_dir); - println!("Adding to link path:\n {:?}", lib_dir); + println!("Found HDF5 headers at:\n {inc_dir:?}"); + println!("Adding to link path:\n {lib_dir:?}"); config.inc_dir = Some(inc_dir.into()); config.link_paths.push(lib_dir.into()); break; @@ -395,7 +395,7 @@ mod macos { } if let Some(ref inc_dir) = config.inc_dir { println!("Found Homebrew HDF5 headers at:"); - println!(" {:?}", inc_dir); + println!(" {inc_dir:?}"); } } } @@ -462,7 +462,7 @@ mod windows { fn get_hdf5_app(version: Option) -> Option { if let Some(version) = version { - println!("Searching for installed HDF5 with version {:?}...", version); + println!("Searching for installed HDF5 with version {version:?}..."); } else { println!("Searching for installed HDF5 (any version)...") } @@ -484,7 +484,7 @@ mod windows { } if apps.len() > 1 { println!("Selecting the latest version ({:?}):", latest.version); - println!("- {:?}", latest); + println!("- {latest:?}"); } Some(latest.clone()) } @@ -510,7 +510,7 @@ mod windows { for path in env::split_paths(&var_path) { if let Ok(path) = path.canonicalize() { if path == bin_dir { - println!("Found in PATH: {:?}", path); + println!("Found in PATH: {path:?}"); return; } } @@ -525,7 +525,7 @@ impl LibrarySearcher { let mut config = Self::default(); if let Some(var) = env::var_os("HDF5_DIR") { println!("Setting HDF5 root from environment variable:"); - println!(" HDF5_DIR = {:?}", var); + println!(" HDF5_DIR = {var:?}"); let root = PathBuf::from(var); assert!(!root.is_relative(), "HDF5_DIR cannot be relative."); @@ -542,7 +542,7 @@ impl LibrarySearcher { let alt_inc_dir = root_dir.join("Library").join("include"); if !is_inc_dir(inc_dir) && is_inc_dir(&alt_inc_dir) { println!("Detected MSVC conda environment, changing headers dir to:"); - println!(" {:?}", alt_inc_dir); + println!(" {alt_inc_dir:?}"); config.inc_dir = Some(alt_inc_dir); } } @@ -550,11 +550,11 @@ impl LibrarySearcher { } if let Ok(var) = env::var("HDF5_VERSION") { println!("Setting HDF5 version from environment variable:"); - println!(" HDF5_VERSION = {:?}", var); + println!(" HDF5_VERSION = {var:?}"); if let Some(v) = Version::parse(&var) { config.version = Some(v); } else { - panic!("Invalid HDF5 version: {}", var); + panic!("Invalid HDF5 version: {var}"); } } config @@ -583,10 +583,10 @@ impl LibrarySearcher { if self.user_provided_dir { let lib_dir = format!("{}/lib", envdir.to_string_lossy()); println!("Custom HDF5_DIR provided; rpath can be set via:"); - println!(" RUSTFLAGS=\"-C link-args=-Wl,-rpath,{}\"", lib_dir); + println!(" RUSTFLAGS=\"-C link-args=-Wl,-rpath,{lib_dir}\""); if cfg!(target_os = "macos") { println!("On some OS X installations, you may also need to set:"); - println!(" DYLD_FALLBACK_LIBRARY_PATH=\"{}\"", lib_dir); + println!(" DYLD_FALLBACK_LIBRARY_PATH=\"{lib_dir}\""); } } } @@ -598,7 +598,7 @@ impl LibrarySearcher { pub fn finalize(self) -> Config { if let Some(ref inc_dir) = self.inc_dir { - assert!(is_inc_dir(inc_dir), "Invalid HDF5 headers directory: {:?}", inc_dir); + assert!(is_inc_dir(inc_dir), "Invalid HDF5 headers directory: {inc_dir:?}"); let mut link_paths = self.link_paths; if link_paths.is_empty() { if let Some(root_dir) = inc_dir.parent() { @@ -723,7 +723,7 @@ fn main() { let mut searcher = LibrarySearcher::new_from_env(); searcher.try_locate_hdf5_library(); let config = searcher.finalize(); - println!("{:#?}", config); + println!("{config:#?}"); config.emit_link_flags(); config.emit_cfg_flags(); write_hdf5_version(config.header.version); diff --git a/hdf5-types/src/h5type.rs b/hdf5-types/src/h5type.rs index ad18b7e42..35f674fb9 100644 --- a/hdf5-types/src/h5type.rs +++ b/hdf5-types/src/h5type.rs @@ -181,10 +181,10 @@ impl Display for TypeDescriptor { TypeDescriptor::Boolean => write!(f, "bool"), TypeDescriptor::Enum(ref tp) => write!(f, "enum ({})", tp.base_type()), TypeDescriptor::Compound(ref tp) => write!(f, "compound ({} fields)", tp.fields.len()), - TypeDescriptor::FixedArray(ref tp, n) => write!(f, "[{}; {}]", tp, n), - TypeDescriptor::FixedAscii(n) => write!(f, "string (len {})", n), - TypeDescriptor::FixedUnicode(n) => write!(f, "unicode (len {})", n), - TypeDescriptor::VarLenArray(ref tp) => write!(f, "[{}] (var len)", tp), + TypeDescriptor::FixedArray(ref tp, n) => write!(f, "[{tp}; {n}]"), + TypeDescriptor::FixedAscii(n) => write!(f, "string (len {n})"), + TypeDescriptor::FixedUnicode(n) => write!(f, "unicode (len {n})"), + TypeDescriptor::VarLenArray(ref tp) => write!(f, "[{tp}] (var len)"), TypeDescriptor::VarLenAscii => write!(f, "string (var len)"), TypeDescriptor::VarLenUnicode => write!(f, "unicode (var len)"), TypeDescriptor::Reference(Reference::Object) => write!(f, "reference (object)"), diff --git a/hdf5-types/src/string.rs b/hdf5-types/src/string.rs index 0efc278f3..826478982 100644 --- a/hdf5-types/src/string.rs +++ b/hdf5-types/src/string.rs @@ -36,7 +36,7 @@ impl fmt::Display for StringError { StringError::InsufficientCapacity => { write!(f, "string error: insufficient capacity for fixed sized string") } - StringError::AsciiError(err) => write!(f, "string error: {}", err), + StringError::AsciiError(err) => write!(f, "string error: {err}"), } } } diff --git a/hdf5/build.rs b/hdf5/build.rs index eea9ddc04..b8364a2ec 100644 --- a/hdf5/build.rs +++ b/hdf5/build.rs @@ -35,8 +35,8 @@ fn main() { } println!("cargo::rustc-check-cfg=cfg(msvc_dll_indirection)"); - let print_feature = |key: &str| println!("cargo::rustc-cfg=feature=\"{}\"", key); - let print_cfg = |key: &str| println!("cargo::rustc-cfg={}", key); + let print_feature = |key: &str| println!("cargo::rustc-cfg=feature=\"{key}\""); + let print_cfg = |key: &str| println!("cargo::rustc-cfg={key}"); for (key, _) in env::vars() { match key.as_str() { // public features From 9e8083af486c94018f16eabed01fd117a270d875 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Jun 2025 21:27:47 +0000 Subject: [PATCH 010/141] Bump crate-ci/typos from 1.33.1 to 1.34.0 Bumps [crate-ci/typos](https://github.com/crate-ci/typos) from 1.33.1 to 1.34.0. - [Release notes](https://github.com/crate-ci/typos/releases) - [Changelog](https://github.com/crate-ci/typos/blob/master/CHANGELOG.md) - [Commits](https://github.com/crate-ci/typos/compare/b1ae8d918b6e85bd611117d3d9a3be4f903ee5e4...392b78fe18a52790c53f42456e46124f77346842) --- updated-dependencies: - dependency-name: crate-ci/typos dependency-version: 1.34.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7fa4c25e5..01de31f15 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - name: Check spelling - uses: crate-ci/typos@b1ae8d918b6e85bd611117d3d9a3be4f903ee5e4 # v1.33.1 + uses: crate-ci/typos@392b78fe18a52790c53f42456e46124f77346842 # v1.34.0 lint: name: lint From 61c55048aa5d562fd26e1cf4d676012f7bbc1b8a Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Wed, 23 Jul 2025 13:49:21 +0200 Subject: [PATCH 011/141] Bump hdf5-c to 1.14.6 --- hdf5-src/ext/hdf5 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hdf5-src/ext/hdf5 b/hdf5-src/ext/hdf5 index 0fe0459fc..7bf340440 160000 --- a/hdf5-src/ext/hdf5 +++ b/hdf5-src/ext/hdf5 @@ -1 +1 @@ -Subproject commit 0fe0459fc24d71be13d5f266513c2832b525671b +Subproject commit 7bf340440909d468dbb3cf41f0ea0d87f5050cea From d98a6d3b4b7a407e916448f398120d550544f184 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Wed, 23 Jul 2025 13:51:22 +0200 Subject: [PATCH 012/141] Whitelist 1.14.6 --- hdf5-sys/build.rs | 2 +- hdf5/build.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hdf5-sys/build.rs b/hdf5-sys/build.rs index ff9918574..749284192 100644 --- a/hdf5-sys/build.rs +++ b/hdf5-sys/build.rs @@ -56,7 +56,7 @@ fn known_hdf5_versions() -> Vec { vs.extend((5..=21).map(|v| Version::new(1, 8, v))); // 1.8.[5-23] vs.extend((0..=8).map(|v| Version::new(1, 10, v))); // 1.10.[0-10] vs.extend((0..=2).map(|v| Version::new(1, 12, v))); // 1.12.[0-2] - vs.extend((0..=5).map(|v| Version::new(1, 14, v))); // 1.14.[0-5] + vs.extend((0..=6).map(|v| Version::new(1, 14, v))); // 1.14.[0-6] vs } diff --git a/hdf5/build.rs b/hdf5/build.rs index b8364a2ec..52557f648 100644 --- a/hdf5/build.rs +++ b/hdf5/build.rs @@ -19,7 +19,7 @@ fn known_hdf5_versions() -> Vec { vs.extend((5..=21).map(|v| Version::new(1, 8, v))); // 1.8.[5-23] vs.extend((0..=8).map(|v| Version::new(1, 10, v))); // 1.10.[0-10] vs.extend((0..=2).map(|v| Version::new(1, 12, v))); // 1.12.[0-2] - vs.extend((0..=5).map(|v| Version::new(1, 14, v))); // 1.14.[0-5] + vs.extend((0..=6).map(|v| Version::new(1, 14, v))); // 1.14.[0-6] vs } From 5bb25f695f37fff866056ad5779f90a4e6a9cd1b Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Mon, 11 Aug 2025 09:49:12 +0200 Subject: [PATCH 013/141] Fix clippy lints --- hdf5-types/src/dyn_value.rs | 6 +++--- hdf5/src/hl/container.rs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/hdf5-types/src/dyn_value.rs b/hdf5-types/src/dyn_value.rs index 23a09357a..433668349 100644 --- a/hdf5-types/src/dyn_value.rs +++ b/hdf5-types/src/dyn_value.rs @@ -279,7 +279,7 @@ impl<'a> DynCompound<'a> { Self { tp, buf } } - pub fn iter(&self) -> impl Iterator { + pub fn iter(&self) -> impl Iterator)> { self.tp.fields.iter().map(move |field| { ( field.name.as_ref(), @@ -379,7 +379,7 @@ impl<'a> DynArray<'a> { } } - pub fn iter(&self) -> impl Iterator { + pub fn iter(&self) -> impl Iterator> { let ptr = self.get_ptr(); let len = self.get_len(); let size = self.tp.size(); @@ -763,7 +763,7 @@ impl OwnedDynValue { Self { tp: T::type_descriptor(), buf: buf.to_owned().into_boxed_slice() } } - pub fn get(&self) -> DynValue { + pub fn get(&self) -> DynValue<'_> { DynValue::new(&self.tp, &self.buf) } diff --git a/hdf5/src/hl/container.rs b/hdf5/src/hl/container.rs index ca59b40f8..721159e4f 100644 --- a/hdf5/src/hl/container.rs +++ b/hdf5/src/hl/container.rs @@ -479,13 +479,13 @@ impl Container { /// Creates a reader wrapper for this dataset/attribute, allowing to /// set custom type conversion options when reading. - pub fn as_reader(&self) -> Reader { + pub fn as_reader(&self) -> Reader<'_> { Reader::new(self) } /// Creates a writer wrapper for this dataset/attribute, allowing to /// set custom type conversion options when writing. - pub fn as_writer(&self) -> Writer { + pub fn as_writer(&self) -> Writer<'_> { Writer::new(self) } From 12f5a1590e5b8b121b56b9d58c128bb083cbbd7a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 02:21:23 +0000 Subject: [PATCH 014/141] Bump actions/checkout from 4 to 5 Bumps [actions/checkout](https://github.com/actions/checkout) from 4 to 5. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 24 ++++++++++++------------ .github/workflows/release.yml | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 01de31f15..df1c32ede 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Check spelling uses: crate-ci/typos@392b78fe18a52790c53f42456e46124f77346842 # v1.34.0 @@ -38,7 +38,7 @@ jobs: - {command: clippy, rust: stable} steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Install Rust (${{matrix.rust}}) uses: dtolnay/rust-toolchain@stable with: {toolchain: '${{matrix.rust}}', components: 'rustfmt, clippy'} @@ -51,7 +51,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: {submodules: true} - name: Install Rust (${{matrix.rust}}) uses: dtolnay/rust-toolchain@stable @@ -73,7 +73,7 @@ jobs: - {version: hdf5-mpi, mpi: true} steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: {submodules: true} - name: Install Rust (${{matrix.rust}}) uses: dtolnay/rust-toolchain@stable @@ -118,7 +118,7 @@ jobs: shell: bash -l {0} steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: {submodules: true} - name: Install Rust (${{matrix.rust}}) uses: dtolnay/rust-toolchain@stable @@ -155,7 +155,7 @@ jobs: - {os: macos, rust: stable} steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: {submodules: true} - name: Install Rust (${{matrix.rust}}) uses: dtolnay/rust-toolchain@stable @@ -182,7 +182,7 @@ jobs: - {mpi: openmpi, rust: stable} steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: {submodules: true} - name: Install Rust (${{matrix.rust}}) uses: dtolnay/rust-toolchain@stable @@ -214,7 +214,7 @@ jobs: version: ["1.8", "1.10", "1.12", "1.14"] steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: {submodules: true} - name: Install Rust (${{matrix.rust}}) uses: dtolnay/rust-toolchain@stable @@ -260,7 +260,7 @@ jobs: # rust: [stable] # steps: # - name: Checkout repository - # uses: actions/checkout@v4 + # uses: actions/checkout@v5 # with: {submodules: true} # - name: Install Rust (${{matrix.rust}}) # uses: dtolnay/rust-toolchain@stable @@ -284,7 +284,7 @@ jobs: fail-fast: false steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: {submodules: true} - name: Install Rust uses: dtolnay/rust-toolchain@stable @@ -304,7 +304,7 @@ jobs: # runs-on: ubuntu-latest # steps: # - name: Checkout repository - # uses: actions/checkout@v4 + # uses: actions/checkout@v5 # with: {submodules: true} # - name: Install Rust # uses: dtolnay/rust-toolchain@stable @@ -321,7 +321,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: {submodules: true} - name: Install Rust uses: dtolnay/rust-toolchain@stable diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5e800ea45..61fc96761 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: submodules: true - name: Install libhdf5 From 9a75004954739ff073f40050749a803022fb8963 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 07:12:56 +0000 Subject: [PATCH 015/141] Bump crate-ci/typos from 1.34.0 to 1.35.3 Bumps [crate-ci/typos](https://github.com/crate-ci/typos) from 1.34.0 to 1.35.3. - [Release notes](https://github.com/crate-ci/typos/releases) - [Changelog](https://github.com/crate-ci/typos/blob/master/CHANGELOG.md) - [Commits](https://github.com/crate-ci/typos/compare/392b78fe18a52790c53f42456e46124f77346842...52bd719c2c91f9d676e2aa359fc8e0db8925e6d8) --- updated-dependencies: - dependency-name: crate-ci/typos dependency-version: 1.35.3 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index df1c32ede..9196ab6fc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: - name: Checkout code uses: actions/checkout@v5 - name: Check spelling - uses: crate-ci/typos@392b78fe18a52790c53f42456e46124f77346842 # v1.34.0 + uses: crate-ci/typos@52bd719c2c91f9d676e2aa359fc8e0db8925e6d8 # v1.35.3 lint: name: lint From fd8323985ca031621bc405dfb8ee95f1371802f5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Aug 2025 02:43:17 +0000 Subject: [PATCH 016/141] Bump crate-ci/typos from 1.35.3 to 1.35.4 Bumps [crate-ci/typos](https://github.com/crate-ci/typos) from 1.35.3 to 1.35.4. - [Release notes](https://github.com/crate-ci/typos/releases) - [Changelog](https://github.com/crate-ci/typos/blob/master/CHANGELOG.md) - [Commits](https://github.com/crate-ci/typos/compare/52bd719c2c91f9d676e2aa359fc8e0db8925e6d8...a67079b4ae32e18c3f53d75368c52ce53b5fb56b) --- updated-dependencies: - dependency-name: crate-ci/typos dependency-version: 1.35.4 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9196ab6fc..5a96206bd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: - name: Checkout code uses: actions/checkout@v5 - name: Check spelling - uses: crate-ci/typos@52bd719c2c91f9d676e2aa359fc8e0db8925e6d8 # v1.35.3 + uses: crate-ci/typos@a67079b4ae32e18c3f53d75368c52ce53b5fb56b # v1.35.4 lint: name: lint From bdec53bd6cdb8be51f0e3ee778c7313ddfe90293 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Aug 2025 08:41:04 +0000 Subject: [PATCH 017/141] Bump crate-ci/typos from 1.35.4 to 1.35.5 Bumps [crate-ci/typos](https://github.com/crate-ci/typos) from 1.35.4 to 1.35.5. - [Release notes](https://github.com/crate-ci/typos/releases) - [Changelog](https://github.com/crate-ci/typos/blob/master/CHANGELOG.md) - [Commits](https://github.com/crate-ci/typos/compare/a67079b4ae32e18c3f53d75368c52ce53b5fb56b...a4c3e43aea0a9e9b9e6578d2731ebd9a27e8f6cd) --- updated-dependencies: - dependency-name: crate-ci/typos dependency-version: 1.35.5 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5a96206bd..42b4f8dd3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: - name: Checkout code uses: actions/checkout@v5 - name: Check spelling - uses: crate-ci/typos@a67079b4ae32e18c3f53d75368c52ce53b5fb56b # v1.35.4 + uses: crate-ci/typos@a4c3e43aea0a9e9b9e6578d2731ebd9a27e8f6cd # v1.35.5 lint: name: lint From 3ddac4683c7fbb330926f8706b2ad026f3ff3209 Mon Sep 17 00:00:00 2001 From: Philipp Bucher Date: Sun, 31 Aug 2025 14:52:04 +0200 Subject: [PATCH 018/141] refactor: use LazyLock from standard lib --- hdf5/Cargo.toml | 1 - hdf5/src/globals.rs | 43 ++++++++++++------------------------ hdf5/src/hl/filters/blosc.rs | 25 +++++++++------------ hdf5/src/hl/filters/lzf.rs | 18 +++++++-------- hdf5/src/sync.rs | 6 ++--- 5 files changed, 35 insertions(+), 58 deletions(-) diff --git a/hdf5/Cargo.toml b/hdf5/Cargo.toml index 6733e9ff3..dec1a5b0a 100644 --- a/hdf5/Cargo.toml +++ b/hdf5/Cargo.toml @@ -51,7 +51,6 @@ bitflags = "2.4" blosc-sys = { version = "0.3", package = "blosc-src", optional = true } cfg-if = { workspace = true } errno = { version = "0.3", optional = true } -lazy_static = "1.4" libc = { workspace = true } lzf-sys = { version = "0.1", optional = true } mpi-sys = { workspace = true, optional = true } diff --git a/hdf5/src/globals.rs b/hdf5/src/globals.rs index 07ac09a57..0c872bb4c 100644 --- a/hdf5/src/globals.rs +++ b/hdf5/src/globals.rs @@ -3,8 +3,6 @@ use std::mem; use std::sync::LazyLock; -use lazy_static::lazy_static; - #[cfg(feature = "have-direct")] use hdf5_sys::h5fd::H5FD_direct_init; #[cfg(feature = "have-parallel")] @@ -325,45 +323,32 @@ link_hid!(H5E_CANTCONVERT, h5e::H5E_CANTCONVERT); link_hid!(H5E_BADSIZE, h5e::H5E_BADSIZE); // H5R constants -lazy_static! { - pub static ref H5R_OBJ_REF_BUF_SIZE: usize = mem::size_of::(); - pub static ref H5R_DSET_REG_REF_BUF_SIZE: usize = mem::size_of::() + 4; -} +pub static H5R_OBJ_REF_BUF_SIZE: LazyLock = LazyLock::new(|| mem::size_of::()); +pub static H5R_DSET_REG_REF_BUF_SIZE: LazyLock = + LazyLock::new(|| mem::size_of::() + 4); // File drivers -lazy_static! { - pub static ref H5FD_CORE: hid_t = h5lock!(H5FD_core_init()); - pub static ref H5FD_SEC2: hid_t = h5lock!(H5FD_sec2_init()); - pub static ref H5FD_STDIO: hid_t = h5lock!(H5FD_stdio_init()); - pub static ref H5FD_FAMILY: hid_t = h5lock!(H5FD_family_init()); - pub static ref H5FD_LOG: hid_t = h5lock!(H5FD_log_init()); - pub static ref H5FD_MULTI: hid_t = h5lock!(H5FD_multi_init()); -} +pub static H5FD_CORE: LazyLock = LazyLock::new(|| h5lock!(H5FD_core_init())); +pub static H5FD_SEC2: LazyLock = LazyLock::new(|| h5lock!(H5FD_sec2_init())); +pub static H5FD_STDIO: LazyLock = LazyLock::new(|| h5lock!(H5FD_stdio_init())); +pub static H5FD_FAMILY: LazyLock = LazyLock::new(|| h5lock!(H5FD_family_init())); +pub static H5FD_LOG: LazyLock = LazyLock::new(|| h5lock!(H5FD_log_init())); +pub static H5FD_MULTI: LazyLock = LazyLock::new(|| h5lock!(H5FD_multi_init())); // MPI-IO file driver #[cfg(feature = "have-parallel")] -lazy_static! { - pub static ref H5FD_MPIO: hid_t = h5lock!(H5FD_mpio_init()); -} +pub static H5FD_MPIO: LazyLock = LazyLock::new(|| h5lock!(H5FD_mpio_init())); #[cfg(not(feature = "have-parallel"))] -lazy_static! { - pub static ref H5FD_MPIO: hid_t = H5I_INVALID_HID; -} +pub static H5FD_MPIO: LazyLock = LazyLock::new(|| H5I_INVALID_HID); // Direct VFD #[cfg(feature = "have-direct")] -lazy_static! { - pub static ref H5FD_DIRECT: hid_t = h5lock!(H5FD_direct_init()); -} +pub static H5FD_DIRECT: LazyLock = LazyLock::new(|| h5lock!(H5FD_direct_init())); #[cfg(not(feature = "have-direct"))] -lazy_static! { - pub static ref H5FD_DIRECT: hid_t = H5I_INVALID_HID; -} +pub static H5FD_DIRECT: LazyLock = LazyLock::new(|| H5I_INVALID_HID); #[cfg(target_os = "windows")] -lazy_static! { - pub static ref H5FD_WINDOWS: hid_t = *H5FD_SEC2; -} +pub static H5FD_WINDOWS: LazyLock = *LazyLock::new(|| H5FD_SEC2); #[cfg(test)] mod tests { diff --git a/hdf5/src/hl/filters/blosc.rs b/hdf5/src/hl/filters/blosc.rs index af8021ffd..720d9c9b1 100644 --- a/hdf5/src/hl/filters/blosc.rs +++ b/hdf5/src/hl/filters/blosc.rs @@ -1,7 +1,6 @@ use std::ptr::{self, addr_of_mut}; use std::slice; - -use lazy_static::lazy_static; +use std::sync::LazyLock; use hdf5_sys::h5p::{H5Pget_chunk, H5Pget_filter_by_id2, H5Pmodify_filter}; use hdf5_sys::h5t::{H5Tclose, H5Tget_class, H5Tget_size, H5Tget_super, H5T_ARRAY}; @@ -48,18 +47,16 @@ const BLOSC_FILTER_INFO: &H5Z_class2_t = &H5Z_class2_t { filter: Some(filter_blosc), }; -lazy_static! { - static ref BLOSC_INIT: Result<(), &'static str> = { - unsafe { - blosc_init(); - } - let ret = unsafe { H5Zregister((BLOSC_FILTER_INFO as *const H5Z_class2_t).cast()) }; - if H5ErrorCode::is_err_code(ret) { - return Err("Can't register Blosc filter"); - } - Ok(()) - }; -} +static BLOSC_INIT: LazyLock> = LazyLock::new(|| { + unsafe { + blosc_init(); + } + let ret = unsafe { H5Zregister((BLOSC_FILTER_INFO as *const H5Z_class2_t).cast()) }; + if H5ErrorCode::is_err_code(ret) { + return Err("Can't register Blosc filter"); + } + Ok(()) +}); pub fn register_blosc() -> Result<(), &'static str> { *BLOSC_INIT diff --git a/hdf5/src/hl/filters/lzf.rs b/hdf5/src/hl/filters/lzf.rs index 33f8e79e6..37e288554 100644 --- a/hdf5/src/hl/filters/lzf.rs +++ b/hdf5/src/hl/filters/lzf.rs @@ -1,7 +1,7 @@ use std::ptr::{self, addr_of_mut}; use std::slice; +use std::sync::LazyLock; -use lazy_static::lazy_static; use lzf_sys::{lzf_compress, lzf_decompress, LZF_VERSION}; use hdf5_sys::h5p::{H5Pget_chunk, H5Pget_filter_by_id2, H5Pmodify_filter}; @@ -27,15 +27,13 @@ const LZF_FILTER_INFO: &H5Z_class2_t = &H5Z_class2_t { filter: Some(filter_lzf), }; -lazy_static! { - static ref LZF_INIT: Result<(), &'static str> = { - let ret = unsafe { H5Zregister((LZF_FILTER_INFO as *const H5Z_class2_t).cast()) }; - if H5ErrorCode::is_err_code(ret) { - return Err("Can't register LZF filter"); - } - Ok(()) - }; -} +static LZF_INIT: LazyLock> = LazyLock::new(|| { + let ret = unsafe { H5Zregister((LZF_FILTER_INFO as *const H5Z_class2_t).cast()) }; + if H5ErrorCode::is_err_code(ret) { + return Err("Can't register LZF filter"); + } + Ok(()) +}); pub fn register_lzf() -> Result<(), &'static str> { *LZF_INIT diff --git a/hdf5/src/sync.rs b/hdf5/src/sync.rs index 7783325d1..a2142d240 100644 --- a/hdf5/src/sync.rs +++ b/hdf5/src/sync.rs @@ -43,14 +43,12 @@ where #[cfg(test)] mod tests { - use lazy_static::lazy_static; use parking_lot::ReentrantMutex; + use std::sync::LazyLock; #[test] pub fn test_reentrant_mutex() { - lazy_static! { - static ref LOCK: ReentrantMutex<()> = ReentrantMutex::new(()); - } + static LOCK: LazyLock> = LazyLock::new(|| ReentrantMutex::new(())); let g1 = LOCK.try_lock(); assert!(g1.is_some()); let g2 = LOCK.lock(); From 7cac1b4a8f2d436294273b054b86c68aa219c1bd Mon Sep 17 00:00:00 2001 From: Philipp Bucher Date: Mon, 1 Sep 2025 11:08:22 +0200 Subject: [PATCH 019/141] Update hdf5/src/globals.rs Co-authored-by: Magnus Ulimoen Signed-off-by: Philipp Bucher --- hdf5/src/globals.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hdf5/src/globals.rs b/hdf5/src/globals.rs index 0c872bb4c..5d5c72880 100644 --- a/hdf5/src/globals.rs +++ b/hdf5/src/globals.rs @@ -348,7 +348,7 @@ pub static H5FD_DIRECT: LazyLock = LazyLock::new(|| h5lock!(H5FD_direct_i pub static H5FD_DIRECT: LazyLock = LazyLock::new(|| H5I_INVALID_HID); #[cfg(target_os = "windows")] -pub static H5FD_WINDOWS: LazyLock = *LazyLock::new(|| H5FD_SEC2); +pub static H5FD_WINDOWS: LazyLock = LazyLock::new(|| H5FD_SEC2); #[cfg(test)] mod tests { From 5440cafb4f778103f348507f0044709a1931fb8c Mon Sep 17 00:00:00 2001 From: Philipp Bucher Date: Mon, 1 Sep 2025 12:30:52 +0200 Subject: [PATCH 020/141] Fix H5FD_WINDOWS initialization on Windows Signed-off-by: Philipp Bucher --- hdf5/src/globals.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hdf5/src/globals.rs b/hdf5/src/globals.rs index 5d5c72880..292a2a562 100644 --- a/hdf5/src/globals.rs +++ b/hdf5/src/globals.rs @@ -348,7 +348,7 @@ pub static H5FD_DIRECT: LazyLock = LazyLock::new(|| h5lock!(H5FD_direct_i pub static H5FD_DIRECT: LazyLock = LazyLock::new(|| H5I_INVALID_HID); #[cfg(target_os = "windows")] -pub static H5FD_WINDOWS: LazyLock = LazyLock::new(|| H5FD_SEC2); +pub static H5FD_WINDOWS: LazyLock = LazyLock::new(|| *H5FD_SEC2); #[cfg(test)] mod tests { From 725422df5255fc4e19813dcd4ff19a797c6603db Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 Sep 2025 09:50:35 +0000 Subject: [PATCH 021/141] Bump crate-ci/typos from 1.35.5 to 1.35.7 Bumps [crate-ci/typos](https://github.com/crate-ci/typos) from 1.35.5 to 1.35.7. - [Release notes](https://github.com/crate-ci/typos/releases) - [Changelog](https://github.com/crate-ci/typos/blob/master/CHANGELOG.md) - [Commits](https://github.com/crate-ci/typos/compare/a4c3e43aea0a9e9b9e6578d2731ebd9a27e8f6cd...65f69f021b736bdbe548ce72200500752d42b40e) --- updated-dependencies: - dependency-name: crate-ci/typos dependency-version: 1.35.7 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 42b4f8dd3..92315a63f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: - name: Checkout code uses: actions/checkout@v5 - name: Check spelling - uses: crate-ci/typos@a4c3e43aea0a9e9b9e6578d2731ebd9a27e8f6cd # v1.35.5 + uses: crate-ci/typos@65f69f021b736bdbe548ce72200500752d42b40e # v1.35.7 lint: name: lint From b8dd4626360fe5d921831a7967a06c3aa2ba2411 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Thu, 16 Oct 2025 21:31:55 +0200 Subject: [PATCH 022/141] Upgrade upper bound of ndarray --- CHANGELOG.md | 4 +++- hdf5/Cargo.toml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9cd5b8f8c..193dbdbd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,13 @@ # Changelog -## hdf5 unreleased ## hdf5-types unreleased ## hdf5-derive unreleased ## hdf5-sys unreleased ## hdf5-src unreleased +## hdf5 unreleased +- Upgraded upper bound of ndarray + ## hdf5-types v0.10.1 Release date: Mar 19, 2025 - Fixed deref of null ptr in `VarLenAscii`/`VarLenUnicode` diff --git a/hdf5/Cargo.toml b/hdf5/Cargo.toml index dec1a5b0a..5f858db2a 100644 --- a/hdf5/Cargo.toml +++ b/hdf5/Cargo.toml @@ -54,7 +54,7 @@ errno = { version = "0.3", optional = true } libc = { workspace = true } lzf-sys = { version = "0.1", optional = true } mpi-sys = { workspace = true, optional = true } -ndarray = ">=0.15, <=0.16" +ndarray = ">=0.15, <=0.17" paste = "1.0" # internal hdf5-derive = { workspace = true } From 2662373d97aa91afc01ad775c009c08823ec6000 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Thu, 16 Oct 2025 21:34:17 +0200 Subject: [PATCH 023/141] CI: Fix dep override --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 92315a63f..f05d60307 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -294,7 +294,7 @@ jobs: cargo update - name: Override deps run: - cargo update half@2.6.0 --precise 2.4.1 + cargo update half@2.7.1 --precise 2.4.1 - name: Build and test all crates run: cargo test --locked --workspace -vv --features=hdf5-sys/static,hdf5-sys/zlib --exclude=hdf5-metno-derive From a7a7dddffe2cd355f63ffcd180734102e4f0a5e4 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Thu, 16 Oct 2025 21:43:06 +0200 Subject: [PATCH 024/141] Skip unneccesary docs.rs feature flag --- hdf5/Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/hdf5/Cargo.toml b/hdf5/Cargo.toml index 5f858db2a..a6ea02537 100644 --- a/hdf5/Cargo.toml +++ b/hdf5/Cargo.toml @@ -74,4 +74,3 @@ tempfile = "3.9" [package.metadata.docs.rs] features = ["static", "zlib", "blosc", "lzf", "f16", "complex"] -rustdoc-args = ["--cfg", "docsrs"] From 7bb2bbc85acb467a9341ef47c9efed480ebe17e9 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Thu, 16 Oct 2025 21:47:24 +0200 Subject: [PATCH 025/141] Pin docs build --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f05d60307..72b29ca55 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,9 +53,9 @@ jobs: - name: Checkout repository uses: actions/checkout@v5 with: {submodules: true} - - name: Install Rust (${{matrix.rust}}) + - name: Install Rust uses: dtolnay/rust-toolchain@stable - with: {toolchain: nightly} + with: {toolchain: "nightly-2025-09-01"} - name: Document workspace env: RUSTDOCFLAGS: "--cfg docsrs" From b2fb140c812127e3a536d3d54accd7c2b89922db Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Oct 2025 20:16:35 +0000 Subject: [PATCH 026/141] Bump crate-ci/typos from 1.35.7 to 1.38.1 Bumps [crate-ci/typos](https://github.com/crate-ci/typos) from 1.35.7 to 1.38.1. - [Release notes](https://github.com/crate-ci/typos/releases) - [Changelog](https://github.com/crate-ci/typos/blob/master/CHANGELOG.md) - [Commits](https://github.com/crate-ci/typos/compare/65f69f021b736bdbe548ce72200500752d42b40e...80c8a4945eec0f6d464eaf9e65ed98ef085283d1) --- updated-dependencies: - dependency-name: crate-ci/typos dependency-version: 1.38.1 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72b29ca55..5c9d0d35e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: - name: Checkout code uses: actions/checkout@v5 - name: Check spelling - uses: crate-ci/typos@65f69f021b736bdbe548ce72200500752d42b40e # v1.35.7 + uses: crate-ci/typos@80c8a4945eec0f6d464eaf9e65ed98ef085283d1 # v1.38.1 lint: name: lint From d0d2a29b656a9d5a276bbcef6500f14a757c5ff6 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Thu, 16 Oct 2025 22:35:27 +0200 Subject: [PATCH 027/141] Release new hdf5 version --- CHANGELOG.md | 4 +++- hdf5/Cargo.toml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 193dbdbd1..bd4314759 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,13 @@ # Changelog +## hdf5 unreleased ## hdf5-types unreleased ## hdf5-derive unreleased ## hdf5-sys unreleased ## hdf5-src unreleased -## hdf5 unreleased +## hdf5 v0.10.2 +Release date: Oct 16, 2025 - Upgraded upper bound of ndarray ## hdf5-types v0.10.1 diff --git a/hdf5/Cargo.toml b/hdf5/Cargo.toml index a6ea02537..f46c56f12 100644 --- a/hdf5/Cargo.toml +++ b/hdf5/Cargo.toml @@ -4,7 +4,7 @@ readme = "../README.md" description = "Thread-safe Rust bindings for the HDF5 library." build = "build.rs" categories = ["science", "filesystem"] -version = "0.10.1" # !V +version = "0.10.2" # !V rust-version.workspace = true authors.workspace = true keywords.workspace = true From 693743870ac346c2a75bc4ffe978cca08a1cc84b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Beck=20K=C3=B6nig?= Date: Sat, 25 Oct 2025 15:53:55 +0200 Subject: [PATCH 028/141] add links to examples in the README --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index cc828eda9..60342dd77 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,9 @@ fn main() -> Result<()> { } ``` +You can find this [example][readme-example] as well as other example projects in +the [example directory][examples]. + ## Compatibility ### Platforms @@ -206,3 +209,7 @@ Few things to note when building on Windows: `hdf5` crate is primarily distributed under the terms of both the MIT license and the Apache License (Version 2.0). See [LICENSE-APACHE](LICENSE-APACHE) and [LICENSE-MIT](LICENSE-MIT) for details. + + +[readme-example]: https://github.com/metno/hdf5-rust/blob/main/hdf5/examples/simple.rs +[examples]: https://github.com/metno/hdf5-rust/tree/main/hdf5/examples \ No newline at end of file From b67782fe1f697c017134905432da309fc673f8bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Beck=20K=C3=B6nig?= Date: Sat, 25 Oct 2025 17:24:13 +0200 Subject: [PATCH 029/141] add simple example of continuously writing to a chunked 1D dataset, and reading from it afterwards --- hdf5/examples/continuously_rw_1d.rs | 83 +++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 hdf5/examples/continuously_rw_1d.rs diff --git a/hdf5/examples/continuously_rw_1d.rs b/hdf5/examples/continuously_rw_1d.rs new file mode 100644 index 000000000..c29138df0 --- /dev/null +++ b/hdf5/examples/continuously_rw_1d.rs @@ -0,0 +1,83 @@ +//! Create, and continuously write several chunks to a chunked 1D dataset +//! +//! Finally read it, get the chunking metadata, and use it to read slices aligned to the chunk size + +use hdf5::{File, Result}; +use hdf5_metno::{self as hdf5, Extent}; +use ndarray::Array1; + +// Shared between read_hdf5() and write_hdf5(), otherwise they only rely on file metadata +const FILE_NAME: &str = "continuous_chunks.h5"; +const DATASET_NAME: &str = "count"; + +const CHUNK_SIZE: usize = 5; +const NUM_CHUNKS: usize = 3; // total chunks to write + +fn write_hdf5() -> Result<()> { + println!("Creating file '{FILE_NAME}' with 1D resizable dataset"); + let file = File::create(FILE_NAME)?; + let shape = Extent::resizable(1); // 1D resizable + let ds = file.new_dataset::().chunk((CHUNK_SIZE,)).shape(shape).create(DATASET_NAME)?; + + // Dataset is created with length 1 + // so we resize to 0 to only store "real" values + assert_eq!(ds.size(), 1); + ds.resize((0,))?; + + // Simulate continuously accumulating data in a buffer + // and writing it to the dataset anytime there's enough to fill a chunk + let mut buf = Vec::with_capacity(CHUNK_SIZE); + for i in 0..NUM_CHUNKS * 5 { + buf.push(i); + if buf.len() == CHUNK_SIZE { + let current_size = ds.size(); + println!("[{i}] writing new chunk, current size = {current_size}"); + ds.resize((current_size + CHUNK_SIZE,))?; + ds.write_slice(&buf, current_size..current_size + CHUNK_SIZE)?; + buf.clear(); + } + } + + Ok(()) +} + +fn read_hdf5() -> Result<()> { + println!("Reading file '{FILE_NAME}'"); + let file = File::open(FILE_NAME)?; + let ds = file.dataset(DATASET_NAME)?; + + // Check shape + let shape = ds.shape(); + println!("Dataset shape: {:?}", shape); + assert_eq!(shape, vec![CHUNK_SIZE * NUM_CHUNKS]); + + // Get chunking metadata + let chunk_size = ds.chunk().unwrap()[0]; + println!("Chunk size: {chunk_size}"); + assert_eq!(chunk_size, CHUNK_SIZE); + + let num_chunks = ds.num_chunks().unwrap(); + println!("Number of chunks: {num_chunks}"); + assert_eq!(num_chunks, NUM_CHUNKS); + + // Read the dataset chunk for chunk + for chunk_idx in 0..num_chunks { + let chunk_start = chunk_idx * chunk_size; + let arr: Array1 = ds.read_slice(chunk_start..chunk_start + chunk_size)?; + println!("Dataset Chunk #{chunk_idx}: {arr:?}"); + assert_eq!(arr, Array1::from_iter(chunk_idx * 5..chunk_idx * 5 + 5)); + } + + // Read the full dataset in one go + let arr: Array1 = ds.read_1d()?; + println!("Full dataset: {arr:?}"); + assert_eq!(arr, Array1::from_iter(0..CHUNK_SIZE * NUM_CHUNKS)); + + Ok(()) +} + +fn main() -> Result<()> { + write_hdf5()?; + read_hdf5()?; + Ok(()) +} From 7c07c8bb794d887685cd2f5e99f174152800a561 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Beck=20K=C3=B6nig?= Date: Sat, 25 Oct 2025 17:39:00 +0200 Subject: [PATCH 030/141] add new example to ci.yml and define a FEATURES variable to avoid duplication --- .github/workflows/ci.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5c9d0d35e..7e4b0b749 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -167,9 +167,12 @@ jobs: if: matrix.rust != 'stable-gnu' - name: Run examples run: | - cargo r --example simple --features hdf5-sys/static,hdf5-sys/zlib,lzf,blosc-all - cargo r --example chunking --features hdf5-sys/static,hdf5-sys/zlib,lzf,blosc-all + cargo r --example simple --features ${{ env.FEATURES }} + cargo r --example chunking --features ${{ env.FEATURES }} + cargo r --example continuously_rw_1d --features ${{ env.FEATURES }} if: matrix.rust != 'stable-gnu' + env: + FEATURES: hdf5-sys/static,hdf5-sys/zlib,lzf,blosc-all apt: name: apt From e9922e9890ea8f279c99f25e06e1c492be0e05d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Beck=20K=C3=B6nig?= Date: Sat, 25 Oct 2025 19:20:52 +0200 Subject: [PATCH 031/141] add doc comment --- hdf5-types/src/h5type.rs | 1 + hdf5/src/hl/datatype.rs | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/hdf5-types/src/h5type.rs b/hdf5-types/src/h5type.rs index 35f674fb9..e7b16babd 100644 --- a/hdf5-types/src/h5type.rs +++ b/hdf5-types/src/h5type.rs @@ -196,6 +196,7 @@ impl Display for TypeDescriptor { } impl TypeDescriptor { + /// Returns the size of the [`TypeDescriptor`] variant in bytes pub fn size(&self) -> usize { match *self { Self::Integer(size) | Self::Unsigned(size) => size as _, diff --git a/hdf5/src/hl/datatype.rs b/hdf5/src/hl/datatype.rs index 6f4847313..a915b8225 100644 --- a/hdf5/src/hl/datatype.rs +++ b/hdf5/src/hl/datatype.rs @@ -204,9 +204,7 @@ impl Datatype { if let Some(conv) = self.conv_path(dst) { ensure!( conv <= required, - "{} conversion path required; available: {} conversion", - required, - conv + "{required} conversion path required; available: {conv} conversion", ); Ok(()) } else { From 8859f904f7b2e3e55b6100028a82f384ee0fe8db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Beck=20K=C3=B6nig?= Date: Sat, 25 Oct 2025 19:48:09 +0200 Subject: [PATCH 032/141] implement debug/display for Datatype and improve error message on failed conversion --- hdf5/src/hl/datatype.rs | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/hdf5/src/hl/datatype.rs b/hdf5/src/hl/datatype.rs index a915b8225..dc79aa59d 100644 --- a/hdf5/src/hl/datatype.rs +++ b/hdf5/src/hl/datatype.rs @@ -62,12 +62,39 @@ impl ObjectClass for Datatype { &self.0 } - // TODO: short_repr() + fn short_repr(&self) -> Option { + let repr = match self.to_descriptor() { + Ok(s) => s.to_string(), + Err(e) => { + // Fallback: show basic HDF5 info if descriptor conversion fails + h5lock!({ + let class = H5Tget_class(self.id()); + let size = H5Tget_size(self.id()); + format!("Invalid datatype: {e} )") + }) + } + }; + Some(repr) + } +} + +impl Display for Datatype { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", self.short_repr().expect("short_repr is always implemented")) + } } impl Debug for Datatype { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - self.debug_fmt(f) + if f.alternate() { + write!( + f, + "", + self.short_repr().expect("short_repr is always implemented") + ) + } else { + write!(f, "{self}") + } } } @@ -203,9 +230,7 @@ impl Datatype { // TODO: more detailed error messages after Debug/Display are implemented for Datatype if let Some(conv) = self.conv_path(dst) { ensure!( - conv <= required, - "{required} conversion path required; available: {conv} conversion", - ); + conv <= required,"Cannot convert from {self} to {dst}, required conversion {required}; available: {conv}"); Ok(()) } else { fail!("no conversion paths found") From fb652f474c7d7fa1dd15692230da15479b895341 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Beck=20K=C3=B6nig?= Date: Sat, 25 Oct 2025 17:39:00 +0200 Subject: [PATCH 033/141] add new example to ci.yml and define a FEATURES variable to avoid duplication --- .github/workflows/ci.yml | 7 +++++-- hdf5/examples/continuously_rw_1d.rs | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5c9d0d35e..7e4b0b749 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -167,9 +167,12 @@ jobs: if: matrix.rust != 'stable-gnu' - name: Run examples run: | - cargo r --example simple --features hdf5-sys/static,hdf5-sys/zlib,lzf,blosc-all - cargo r --example chunking --features hdf5-sys/static,hdf5-sys/zlib,lzf,blosc-all + cargo r --example simple --features ${{ env.FEATURES }} + cargo r --example chunking --features ${{ env.FEATURES }} + cargo r --example continuously_rw_1d --features ${{ env.FEATURES }} if: matrix.rust != 'stable-gnu' + env: + FEATURES: hdf5-sys/static,hdf5-sys/zlib,lzf,blosc-all apt: name: apt diff --git a/hdf5/examples/continuously_rw_1d.rs b/hdf5/examples/continuously_rw_1d.rs index c29138df0..459d47c03 100644 --- a/hdf5/examples/continuously_rw_1d.rs +++ b/hdf5/examples/continuously_rw_1d.rs @@ -27,7 +27,7 @@ fn write_hdf5() -> Result<()> { // Simulate continuously accumulating data in a buffer // and writing it to the dataset anytime there's enough to fill a chunk let mut buf = Vec::with_capacity(CHUNK_SIZE); - for i in 0..NUM_CHUNKS * 5 { + for i in 0..NUM_CHUNKS * CHUNK_SIZE { buf.push(i); if buf.len() == CHUNK_SIZE { let current_size = ds.size(); @@ -65,7 +65,7 @@ fn read_hdf5() -> Result<()> { let chunk_start = chunk_idx * chunk_size; let arr: Array1 = ds.read_slice(chunk_start..chunk_start + chunk_size)?; println!("Dataset Chunk #{chunk_idx}: {arr:?}"); - assert_eq!(arr, Array1::from_iter(chunk_idx * 5..chunk_idx * 5 + 5)); + assert_eq!(arr, Array1::from_iter(chunk_start..chunk_start + chunk_size)); } // Read the full dataset in one go From f7a6777f60cba08615bf0835297771761d8cf1b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Beck=20K=C3=B6nig?= Date: Sun, 26 Oct 2025 12:13:22 +0100 Subject: [PATCH 034/141] add pretty_assertions to workspace dev-dependencies --- Cargo.toml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ae8717633..3f189ee65 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,9 +24,12 @@ libz-sys = { version = "1.1", default-features = false } mpi-sys = "0.2" num-complex = { version = "0.4", default-features = false } regex = "1.10" +# external dev-dependencies +pretty_assertions = "1.4.1" + # internal -hdf5 = { package = "hdf5-metno", version = "0.9.3", path = "hdf5" } # !V +hdf5 = { package = "hdf5-metno", version = "0.9.3", path = "hdf5" } # !V hdf5-derive = { package = "hdf5-metno-derive", version = "0.9.1", path = "hdf5-derive" } # !V -hdf5-src = { package = "hdf5-metno-src", version = "0.9.3", path = "hdf5-src" } # !V -hdf5-sys = { package = "hdf5-metno-sys", version = "0.10.1", path = "hdf5-sys" } # !V +hdf5-src = { package = "hdf5-metno-src", version = "0.9.3", path = "hdf5-src" } # !V +hdf5-sys = { package = "hdf5-metno-sys", version = "0.10.1", path = "hdf5-sys" } # !V hdf5-types = { package = "hdf5-metno-types", version = "0.10.0", path = "hdf5-types" } # !V From b3b17f5cfad148f70f8dbcf74b144dd4c331439d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Beck=20K=C3=B6nig?= Date: Sun, 26 Oct 2025 12:17:02 +0100 Subject: [PATCH 035/141] ensure the H5Type derive macro also works inside the hdf5 crate --- hdf5-derive/src/lib.rs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/hdf5-derive/src/lib.rs b/hdf5-derive/src/lib.rs index a2c2d75de..5f8e01f88 100644 --- a/hdf5-derive/src/lib.rs +++ b/hdf5-derive/src/lib.rs @@ -21,13 +21,27 @@ pub fn derive(input: proc_macro::TokenStream) -> proc_macro::TokenStream { let body = impl_trait(&name, &input.data, &input.attrs, &ty_generics); // Determine name of parent crate, even if renamed using "package" + // CARGO_CRATE_NAME is the name of the actual crate being compiled (e.g., "simple" for examples) + // If it matches the library name (hdf5_metno), we're compiling the library itself + let is_library = + std::env::var("CARGO_CRATE_NAME").map(|name| name == "hdf5_metno").unwrap_or(false); + let crate_name = match proc_macro_crate::crate_name("hdf5-metno").unwrap() { - proc_macro_crate::FoundCrate::Itself => quote!(::hdf5_metno), + proc_macro_crate::FoundCrate::Itself if is_library => { + // We're in the hdf5-metno library itself, use crate + quote!(crate) + } + proc_macro_crate::FoundCrate::Itself => { + // We're in an example/test of hdf5-metno, use the renamed path + quote!(::hdf5_metno) + } proc_macro_crate::FoundCrate::Name(name) => { + // We're in a different crate that depends on hdf5-metno let ident = Ident::new(&name, Span::call_site()); quote!( ::#ident ) } }; + let expanded = quote! { #[allow(dead_code, unused_variables, unused_attributes)] const _: () = { From bea491ba130e3bf332958157a797b207e19fe454 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Beck=20K=C3=B6nig?= Date: Sun, 26 Oct 2025 12:29:55 +0100 Subject: [PATCH 036/141] improve error message for failed conversions --- hdf5/src/hl/datatype.rs | 46 ++++++++++++++++------ hdf5/tests/test_datatypes.rs | 74 ++++++++++++++++++++++++++++++++++-- 2 files changed, 105 insertions(+), 15 deletions(-) diff --git a/hdf5/src/hl/datatype.rs b/hdf5/src/hl/datatype.rs index dc79aa59d..c14b1e993 100644 --- a/hdf5/src/hl/datatype.rs +++ b/hdf5/src/hl/datatype.rs @@ -86,15 +86,7 @@ impl Display for Datatype { impl Debug for Datatype { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - if f.alternate() { - write!( - f, - "", - self.short_repr().expect("short_repr is always implemented") - ) - } else { - write!(f, "{self}") - } + self.debug_fmt(f) } } @@ -227,13 +219,12 @@ impl Datatype { } pub(crate) fn ensure_convertible(&self, dst: &Self, required: Conversion) -> Result<()> { - // TODO: more detailed error messages after Debug/Display are implemented for Datatype if let Some(conv) = self.conv_path(dst) { ensure!( - conv <= required,"Cannot convert from {self} to {dst}, required conversion {required}; available: {conv}"); + conv <= required, "Cannot convert from {self} to {dst}, required conversion {required}; available: {conv}",); Ok(()) } else { - fail!("no conversion paths found") + fail!("no conversion paths found from '{self:#?}' to '{dst:#?}'",) } } @@ -450,3 +441,34 @@ impl Datatype { Self::from_id(datatype_id?) } } + +/// NOTE: tests of public functions are in hdf5/tests/test_datatype.rs +#[cfg(test)] +mod tests { + use super::*; + use hdf5_types::{FixedAscii, FixedUnicode}; + use pretty_assertions::assert_str_eq; + + #[test] + fn test_ensure_convertible_fail_err_msg() { + const SIZE: usize = 10; + let src = Datatype::from_type::>().unwrap(); + let dst = Datatype::from_type::>().unwrap(); + + let err_msg = src.ensure_convertible(&dst, Conversion::NoOp).unwrap_err().to_string(); + + assert_str_eq!(err_msg, "no conversion paths found from '' to ''"); + } + + #[test] + fn test_ensure_convertible_failed_required_conversion_hard_err_msg() { + let src = Datatype::from_type::().unwrap(); + let dst = Datatype::from_type::().unwrap(); + + let err_msg = src.ensure_convertible(&dst, Conversion::NoOp).unwrap_err().to_string(); + assert_str_eq!( + err_msg, + "Cannot convert from uint64 to int64, required conversion no-op; available: hard" + ); + } +} diff --git a/hdf5/tests/test_datatypes.rs b/hdf5/tests/test_datatypes.rs index 14d86558d..b4ac1410a 100644 --- a/hdf5/tests/test_datatypes.rs +++ b/hdf5/tests/test_datatypes.rs @@ -4,8 +4,8 @@ mod common; use hdf5::types::{TypeDescriptor as TD, *}; use hdf5::{from_id, Datatype, H5Type}; use hdf5_metno as hdf5; - use hdf5_sys::h5i::H5I_INVALID_HID; +use pretty_assertions::{assert_eq, assert_str_eq}; macro_rules! check_roundtrip { ($ty:ty, $desc:expr) => {{ @@ -148,6 +148,74 @@ pub fn test_eq() { } #[test] -pub fn test_debug() { - assert_eq!(format!("{:?}", Datatype::from_type::().unwrap()), ""); +fn test_print_display_debug_datatype_bool() { + let dt = Datatype::from_type::().unwrap(); + + assert_str_eq!(format!("{dt}"), "bool"); + assert_str_eq!(format!("{dt:?}"), ""); + assert_str_eq!(format!("{dt:#?}"), ""); +} + +#[test] +fn test_print_display_debug_datatype_f64() { + let dt = Datatype::from_type::().unwrap(); + + assert_str_eq!(format!("{dt}"), "float64"); + assert_str_eq!(format!("{dt:?}"), ""); + assert_str_eq!(format!("{dt:#?}"), ""); +} + +#[test] +fn test_print_display_debug_datatype_color_enum() { + #[allow(dead_code, reason = "we use the type, we just don't construct it")] + #[derive(H5Type)] + #[repr(u8)] + enum Color { + R = 1, + G = 2, + B = 3, + } + let dt = Datatype::from_type::().unwrap(); + + assert_eq!( + dt.to_descriptor().unwrap(), + TD::Enum(EnumType { + size: IntSize::U1, + signed: false, + members: vec![ + EnumMember { name: "R".into(), value: 1 }, + EnumMember { name: "G".into(), value: 2 }, + EnumMember { name: "B".into(), value: 3 } + ] + }) + ); + + assert_str_eq!(format!("{dt}"), "enum (uint8)"); + assert_str_eq!(format!("{dt:?}"), ""); + assert_str_eq!(format!("{dt:#?}"), ""); +} + +#[test] +fn test_print_display_debug_datatype_var_len_unicode() { + let dt = Datatype::from_type::().unwrap(); + assert!(dt.is::()); + + assert_eq!(dt.to_descriptor().unwrap(), TD::VarLenUnicode); + + assert_str_eq!(format!("{dt}"), "unicode (var len)"); + assert_str_eq!(format!("{dt:?}"), ""); + assert_str_eq!(format!("{dt:#?}"), ""); +} + +#[test] +fn test_print_display_debug_datatype_fixed_len_unicode() { + const SIZE: usize = 10; + let dt = Datatype::from_type::>().unwrap(); + assert!(dt.is::>()); + + assert_eq!(dt.to_descriptor().unwrap(), TD::FixedUnicode(SIZE)); + + assert_str_eq!(format!("{dt}"), "unicode (len 10)"); + assert_str_eq!(format!("{dt:?}"), ""); + assert_str_eq!(format!("{dt:#?}"), ""); } From ebd27dbc81540ed43474b7dc69f7ff307f7cac04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Beck=20K=C3=B6nig?= Date: Sun, 26 Oct 2025 18:52:08 +0100 Subject: [PATCH 037/141] Implement proper way of getting name of HDF5 attribute --- hdf5/src/hl/attribute.rs | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/hdf5/src/hl/attribute.rs b/hdf5/src/hl/attribute.rs index a7a457e97..ba4466c12 100644 --- a/hdf5/src/hl/attribute.rs +++ b/hdf5/src/hl/attribute.rs @@ -2,6 +2,7 @@ use std::fmt::{self, Debug}; use std::ops::Deref; use std::ptr::addr_of_mut; +use hdf5_sys::h5a::H5Aget_name; use hdf5_sys::{ h5::{H5_index_t, H5_iter_order_t}, h5a::{H5A_info_t, H5A_operator2_t, H5Acreate2, H5Adelete, H5Aiterate2}, @@ -46,6 +47,14 @@ impl Deref for Attribute { } impl Attribute { + /// Returns the name of the attribute. + pub fn name(&self) -> String { + // Note: We must use H5Aget_name() here. H5Iget_name() (called by + // Location::name()) would return the name of the object this + // attribute is attached to, not the attribute's own name. + h5lock!(get_h5_str(|m, s| H5Aget_name(self.id(), s, m)).unwrap_or_else(|_| String::new())) + } + /// Returns names of all the members in the group, non-recursively. pub fn attr_names(obj: &Location) -> Result> { unsafe extern "C" fn attributes_callback( @@ -268,8 +277,6 @@ impl AttributeBuilderInner { name.as_ptr(), datatype.id(), dataspace.id(), - // these args are currently unused as if HDF5 1.12 - // see details: https://portal.hdfgroup.org/display/HDF5/H5A_CREATE2 H5P_DEFAULT, H5P_DEFAULT, ))) @@ -298,12 +305,14 @@ pub mod attribute_tests { assert_eq!(d.size(), 6); assert_eq!(d.ndim(), 2); assert_eq!(d.is_scalar(), false); + assert_eq!(d.name(), "name1"); let d = file.new_attr::().shape(()).create("name2").unwrap(); assert_eq!(d.shape(), vec![]); assert_eq!(d.size(), 1); assert_eq!(d.ndim(), 0); assert_eq!(d.is_scalar(), true); + assert_eq!(d.name(), "name2"); }) } @@ -314,9 +323,7 @@ pub mod attribute_tests { let _ = file.new_attr::().shape(()).create("name2").unwrap(); let attr_names = file.attr_names().unwrap(); - assert_eq!(attr_names.len(), 2); - assert!(attr_names.contains(&"name1".to_string())); - assert!(attr_names.contains(&"name2".to_string())); + assert_eq!(attr_names, vec!["name1".to_owned(), "name2".to_owned()]); }) } @@ -329,9 +336,7 @@ pub mod attribute_tests { let _ = ds.new_attr::().shape(()).create("name2").unwrap(); let attr_names = ds.attr_names().unwrap(); - assert_eq!(attr_names.len(), 2); - assert!(attr_names.contains(&"name1".to_string())); - assert!(attr_names.contains(&"name2".to_string())); + assert_eq!(attr_names, vec!["name1".to_owned(), "name2".to_owned()]); }) } @@ -368,9 +373,7 @@ pub mod attribute_tests { let attr = file.new_attr::().shape((1, 2)).create("foo").unwrap(); assert!(attr.is_valid()); assert_eq!(attr.shape(), vec![1, 2]); - // FIXME - attr.name() returns "/" here, which is the name the attribute is connected to, - // not the name of the attribute. - //assert_eq!(attr.name(), "foo"); + assert_eq!(attr.name(), "foo"); assert_eq!(file.attr("foo").unwrap().shape(), vec![1, 2]); }) } @@ -383,9 +386,7 @@ pub mod attribute_tests { let attr = file.new_attr_builder().with_data(&arr).create("foo").unwrap(); assert!(attr.is_valid()); assert_eq!(attr.shape(), vec![2, 3]); - // FIXME - attr.name() returns "/" here, which is the name the attribute is connected to, - // not the name of the attribute. - //assert_eq!(attr.name(), "foo"); + assert_eq!(attr.name(), "foo"); assert_eq!(file.attr("foo").unwrap().shape(), vec![2, 3]); let read_attr = file.attr("foo").unwrap(); From 855bdf750a6fb5333def9a1b05d9e157ee0fe0ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Beck=20K=C3=B6nig?= Date: Sun, 26 Oct 2025 18:58:41 +0100 Subject: [PATCH 038/141] supply acpl_id to h5Acreate2 to conform to API An error in the HDF5 docs led to an error in the implementation, the docs described the acpl_id paramenter as unused but actually it was only aapl_id that is (still) unused. The issue does not manifest if you only read/write with the hdf5-rust API, because it always assumbed UTF-8 encoding which is essentially a superset of ASCII. However, another API might not assume UTF-8 encoding if the default (ASCII) encoding is set. --- hdf5/src/hl/attribute.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/hdf5/src/hl/attribute.rs b/hdf5/src/hl/attribute.rs index ba4466c12..90e63bfc7 100644 --- a/hdf5/src/hl/attribute.rs +++ b/hdf5/src/hl/attribute.rs @@ -271,13 +271,24 @@ impl AttributeBuilderInner { let dataspace = Dataspace::try_new(extents)?; + let acpl = PropertyList::from_id(h5call!(hdf5_sys::h5p::H5Pcreate( + *crate::globals::H5P_ATTRIBUTE_CREATE + ))?)?; + // Set UTF-8 encoding for the attribute name, as Rust strings are UTF-8. + h5call!(hdf5_sys::h5p::H5Pset_char_encoding( + acpl.id(), + hdf5_sys::h5t::H5T_cset_t::H5T_CSET_UTF8 + ))?; + let name = to_cstring(name)?; Attribute::from_id(h5try!(H5Acreate2( parent.id(), name.as_ptr(), datatype.id(), dataspace.id(), - H5P_DEFAULT, + acpl.id(), + // Unused as of v1.14 + // see more: https://hdfgroup.github.io/hdf5/v1_14/group___h5_a.html#ga4f4e5248c09f689633079ed8afc0b308 H5P_DEFAULT, ))) } From 3851eb5ab5e735901849e6bd7ea0f7a631b56813 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Beck=20K=C3=B6nig?= Date: Sun, 26 Oct 2025 19:52:14 +0100 Subject: [PATCH 039/141] import types instead of fully qualifying --- hdf5/src/hl/attribute.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hdf5/src/hl/attribute.rs b/hdf5/src/hl/attribute.rs index 90e63bfc7..c4fdf4e00 100644 --- a/hdf5/src/hl/attribute.rs +++ b/hdf5/src/hl/attribute.rs @@ -3,6 +3,7 @@ use std::ops::Deref; use std::ptr::addr_of_mut; use hdf5_sys::h5a::H5Aget_name; +use hdf5_sys::h5p::H5Pcreate; use hdf5_sys::{ h5::{H5_index_t, H5_iter_order_t}, h5a::{H5A_info_t, H5A_operator2_t, H5Acreate2, H5Adelete, H5Aiterate2}, @@ -10,6 +11,7 @@ use hdf5_sys::{ use hdf5_types::TypeDescriptor; use ndarray::ArrayView; +use crate::globals::H5P_ATTRIBUTE_CREATE; use crate::internal_prelude::*; /// Represents the HDF5 attribute object. @@ -271,9 +273,7 @@ impl AttributeBuilderInner { let dataspace = Dataspace::try_new(extents)?; - let acpl = PropertyList::from_id(h5call!(hdf5_sys::h5p::H5Pcreate( - *crate::globals::H5P_ATTRIBUTE_CREATE - ))?)?; + let acpl = PropertyList::from_id(h5call!(H5Pcreate(*H5P_ATTRIBUTE_CREATE))?)?; // Set UTF-8 encoding for the attribute name, as Rust strings are UTF-8. h5call!(hdf5_sys::h5p::H5Pset_char_encoding( acpl.id(), From 2272096b256ea9e207b175a29da979a367bfb0da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Beck=20K=C3=B6nig?= Date: Sun, 2 Nov 2025 13:30:34 +0100 Subject: [PATCH 040/141] add ObjectCopy property list --- hdf5/src/hl/plist.rs | 3 +- hdf5/src/hl/plist/object_copy.rs | 212 +++++++++++++++++++++++++++++++ hdf5/src/lib.rs | 1 + 3 files changed, 215 insertions(+), 1 deletion(-) create mode 100644 hdf5/src/hl/plist/object_copy.rs diff --git a/hdf5/src/hl/plist.rs b/hdf5/src/hl/plist.rs index f80ea91c2..9b4a56d6d 100644 --- a/hdf5/src/hl/plist.rs +++ b/hdf5/src/hl/plist.rs @@ -17,6 +17,7 @@ pub mod dataset_create; pub mod file_access; pub mod file_create; pub mod link_create; +pub mod object_copy; /// Represents the HDF5 property list. #[repr(transparent)] @@ -93,7 +94,7 @@ pub enum PropertyListClass { LinkCreate, /// Properties for object copying process. ObjectCopy, - /// Properties for object creatio. + /// Properties for object creation. ObjectCreate, /// Properties for character encoding. StringCreate, diff --git a/hdf5/src/hl/plist/object_copy.rs b/hdf5/src/hl/plist/object_copy.rs new file mode 100644 index 000000000..6d09184f1 --- /dev/null +++ b/hdf5/src/hl/plist/object_copy.rs @@ -0,0 +1,212 @@ +//! Object copy properties. +use std::fmt::{self, Debug}; +use std::ops::Deref; + +use hdf5_sys::h5o::{ + H5O_COPY_EXPAND_EXT_LINK_FLAG, H5O_COPY_EXPAND_SOFT_LINK_FLAG, H5O_COPY_SHALLOW_HIERARCHY_FLAG, + H5O_COPY_WITHOUT_ATTR_FLAG, +}; +use hdf5_sys::h5p::{H5Pcreate, H5Pget_copy_object, H5Pset_copy_object}; + +use crate::globals::H5P_OBJECT_COPY; +use crate::internal_prelude::*; + +/// Object copy properties. +#[repr(transparent)] +pub struct ObjectCopy(Handle); + +impl ObjectClass for ObjectCopy { + const NAME: &'static str = "object copy property list"; + const VALID_TYPES: &'static [H5I_type_t] = &[H5I_GENPROP_LST]; + + fn from_handle(handle: Handle) -> Self { + Self(handle) + } + + fn handle(&self) -> &Handle { + &self.0 + } + + fn validate(&self) -> Result<()> { + ensure!( + self.is_class(PropertyListClass::ObjectCopy), + "expected object copy property list, got {:?}", + self.class() + ); + Ok(()) + } +} + +impl Debug for ObjectCopy { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = f.debug_struct("ObjectCopy"); + formatter.field("copy_without_attr", &self.copy_without_attr()); + formatter.field("shallow_hierarchy", &self.shallow_hierarchy()); + formatter.field("expand_soft_links", &self.expand_soft_links()); + formatter.field("expand_ext_links", &self.expand_ext_links()); + formatter.finish() + } +} + +impl Deref for ObjectCopy { + type Target = PropertyList; + + fn deref(&self) -> &PropertyList { + unsafe { self.transmute() } + } +} + +impl PartialEq for ObjectCopy { + fn eq(&self, other: &Self) -> bool { + ::eq(self, other) + } +} + +impl Eq for ObjectCopy {} + +impl Clone for ObjectCopy { + fn clone(&self) -> Self { + unsafe { self.deref().clone().cast_unchecked() } + } +} + +/// Builder used to create object copy property list. +#[derive(Clone, Debug, Default)] +pub struct ObjectCopyBuilder { + copy_without_attr: Option, + shallow_hierarchy: Option, + expand_soft_links: Option, + expand_ext_links: Option, +} + +impl ObjectCopyBuilder { + /// Creates a new object copy property list builder. + pub fn new() -> Self { + Self::default() + } + + /// Creates a new builder from an existing property list. + pub fn from_plist(plist: &ObjectCopy) -> Result { + let mut builder = Self::default(); + let flags = plist.get_flags()?; + builder.copy_without_attr = Some(flags & H5O_COPY_WITHOUT_ATTR_FLAG != 0); + builder.shallow_hierarchy = Some(flags & H5O_COPY_SHALLOW_HIERARCHY_FLAG != 0); + builder.expand_soft_links = Some(flags & H5O_COPY_EXPAND_SOFT_LINK_FLAG != 0); + builder.expand_ext_links = Some(flags & H5O_COPY_EXPAND_EXT_LINK_FLAG != 0); + Ok(builder) + } + + /// Copy object without copying attributes. + pub fn copy_without_attr(&mut self, enable: bool) -> &mut Self { + self.copy_without_attr = Some(enable); + self + } + + /// Copy only immediate members of a group (shallow copy). + pub fn shallow_hierarchy(&mut self, enable: bool) -> &mut Self { + self.shallow_hierarchy = Some(enable); + self + } + + /// Expand soft links into new objects. + pub fn expand_soft_links(&mut self, enable: bool) -> &mut Self { + self.expand_soft_links = Some(enable); + self + } + + /// Expand external links into new objects. + pub fn expand_ext_links(&mut self, enable: bool) -> &mut Self { + self.expand_ext_links = Some(enable); + self + } + + fn populate_plist(&self, id: hid_t) -> Result<()> { + let mut flags = 0u32; + + if self.copy_without_attr.unwrap_or(false) { + flags |= H5O_COPY_WITHOUT_ATTR_FLAG; + } + if self.shallow_hierarchy.unwrap_or(false) { + flags |= H5O_COPY_SHALLOW_HIERARCHY_FLAG; + } + if self.expand_soft_links.unwrap_or(false) { + flags |= H5O_COPY_EXPAND_SOFT_LINK_FLAG; + } + if self.expand_ext_links.unwrap_or(false) { + flags |= H5O_COPY_EXPAND_EXT_LINK_FLAG; + } + + if flags != 0 { + h5try!(H5Pset_copy_object(id, flags)); + } + + Ok(()) + } + + pub fn apply(&self, plist: &mut ObjectCopy) -> Result<()> { + h5lock!(self.populate_plist(plist.id())) + } + + pub fn finish(&self) -> Result { + h5lock!({ + let mut plist = ObjectCopy::try_new()?; + self.apply(&mut plist).map(|()| plist) + }) + } +} + +/// Object copy property list. +impl ObjectCopy { + pub fn try_new() -> Result { + Self::from_id(h5try!(H5Pcreate(*H5P_OBJECT_COPY))) + } + + pub fn copy(&self) -> Self { + unsafe { self.deref().copy().cast_unchecked() } + } + + pub fn build() -> ObjectCopyBuilder { + ObjectCopyBuilder::new() + } + + #[doc(hidden)] + pub fn get_flags(&self) -> Result { + h5get!(H5Pget_copy_object(self.id()): c_uint) + } + + pub fn copy_without_attr(&self) -> bool { + self.get_flags().map(|f| f & H5O_COPY_WITHOUT_ATTR_FLAG != 0).unwrap_or(false) + } + + pub fn shallow_hierarchy(&self) -> bool { + self.get_flags().map(|f| f & H5O_COPY_SHALLOW_HIERARCHY_FLAG != 0).unwrap_or(false) + } + + pub fn expand_soft_links(&self) -> bool { + self.get_flags().map(|f| f & H5O_COPY_EXPAND_SOFT_LINK_FLAG != 0).unwrap_or(false) + } + + pub fn expand_ext_links(&self) -> bool { + self.get_flags().map(|f| f & H5O_COPY_EXPAND_EXT_LINK_FLAG != 0).unwrap_or(false) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + pub fn test_object_copy_builder_from_plist() { + let ocpypl = + ObjectCopy::build().copy_without_attr(true).expand_soft_links(true).finish().unwrap(); + + let builder = ObjectCopyBuilder::from_plist(&ocpypl).unwrap(); + let ocpypl2 = builder.finish().unwrap(); + + assert_eq!(ocpypl.copy_without_attr(), ocpypl2.copy_without_attr()); + assert_eq!(ocpypl.shallow_hierarchy(), ocpypl2.shallow_hierarchy()); + assert_eq!(ocpypl.expand_soft_links(), ocpypl2.expand_soft_links()); + assert_eq!(ocpypl.expand_ext_links(), ocpypl2.expand_ext_links()); + assert_eq!(ocpypl, ocpypl2); + } +} diff --git a/hdf5/src/lib.rs b/hdf5/src/lib.rs index 34d283eb1..73e18902a 100644 --- a/hdf5/src/lib.rs +++ b/hdf5/src/lib.rs @@ -106,6 +106,7 @@ mod export { pub use crate::hl::plist::file_access::{FileAccess, FileAccessBuilder}; pub use crate::hl::plist::file_create::{FileCreate, FileCreateBuilder}; pub use crate::hl::plist::link_create::{LinkCreate, LinkCreateBuilder}; + pub use crate::hl::plist::object_copy::{ObjectCopy, ObjectCopyBuilder}; pub use crate::hl::plist::{PropertyList, PropertyListClass}; pub mod dataset_access { From 65f155ad0c285cf342abb96a9e8f0e2e9bf70418 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Beck=20K=C3=B6nig?= Date: Sun, 2 Nov 2025 13:32:15 +0100 Subject: [PATCH 041/141] add support for object copying (H5Ocopy) via copy_to and copy_to_with_props --- hdf5/src/hl/location.rs | 187 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 185 insertions(+), 2 deletions(-) diff --git a/hdf5/src/hl/location.rs b/hdf5/src/hl/location.rs index 2f68e378b..fe0a8560a 100644 --- a/hdf5/src/hl/location.rs +++ b/hdf5/src/hl/location.rs @@ -3,6 +3,7 @@ use std::mem::MaybeUninit; use std::ops::Deref; use std::ptr; +use hdf5_sys::h5o::H5Ocopy; #[allow(deprecated)] use hdf5_sys::h5o::H5Oset_comment; #[cfg(feature = "1.12.0")] @@ -91,7 +92,7 @@ impl Location { comment.and_then(|c| if c.is_empty() { None } else { Some(c) }) } - /// Set or the comment attached to the named object. + /// Set the comment attached to the named object. #[deprecated(note = "attributes are preferred to comments")] pub fn set_comment(&self, comment: &str) -> Result<()> { // TODO: &mut self? @@ -166,6 +167,49 @@ impl Location { pub fn dereference(&self, reference: &R) -> Result { reference.dereference(self) } + + /// Copy this object to a destination location with default properties + pub fn copy_to(&self, dst_loc: &Location, dst_name: &str) -> Result<()> { + self.copy_to_with_props(dst_loc, dst_name, None, None) + } + + /// Copy this object to a destination location with custom property lists + /// + /// # Arguments + /// * `dst_loc` - Destination location (file or group) + /// * `dst_name` - Name for the copied object at the destination + /// * `ocpypl` - Optional object copy property list (controls copy behavior) + /// * `lcpl` - Optional link creation property list (controls link properties) + pub fn copy_to_with_props( + &self, dst_loc: &Location, dst_name: &str, ocpypl: Option<&PropertyList>, + lcpl: Option<&PropertyList>, + ) -> Result<()> { + // Validate property list classes if provided + if let Some(pl) = ocpypl { + if !pl.is_class(PropertyListClass::ObjectCopy) { + fail!("Property list must be of class ObjectCopy"); + } + } + if let Some(pl) = lcpl { + if !pl.is_class(PropertyListClass::LinkCreate) { + fail!("Property list must be of class LinkCreate"); + } + } + + let dst_name = to_cstring(dst_name)?; + let ocpypl_id = ocpypl.map_or(H5P_DEFAULT, |p| p.id()); + let lcpl_id = lcpl.map_or(H5P_DEFAULT, |p| p.id()); + + h5call!(H5Ocopy( + self.id(), // src_loc_id + b".\0".as_ptr() as *const c_char, // src_name (current object) + dst_loc.id(), // dst_loc_id + dst_name.as_ptr(), // dst_name + ocpypl_id, // ocpypl_id + lcpl_id // lcpl_id + ))?; + Ok(()) + } } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -320,7 +364,7 @@ fn H5O_open_by_token(loc_id: hid_t, token: LocationToken) -> Result { #[cfg(test)] pub mod tests { - use crate::internal_prelude::*; + use crate::{hl::plist::object_copy::ObjectCopy, internal_prelude::*, plist::LinkCreate}; #[test] pub fn test_filename() { @@ -437,4 +481,143 @@ pub mod tests { assert!(file.loc_info_by_name("gibberish").is_err()); }) } + + #[test] + pub fn test_copy_dataset_between_files() { + with_tmp_path(|src_path| { + with_tmp_path(|dst_path| { + // Create source file with a dataset + let src_file = File::create(&src_path).unwrap(); + let src_group = src_file.create_group("src_group").unwrap(); + + let src_dataset = + src_group.new_dataset::().shape([5]).create("src_group").unwrap(); + src_dataset.write(&[1, 2, 3, 4, 5]).unwrap(); + + let src_attr = src_dataset.new_attr::().create("src_attr").unwrap(); + src_attr.write_scalar(&42.0).unwrap(); + + let dst_file = File::create(&dst_path).unwrap(); + let dst_group = dst_file.create_group("dst_group").unwrap(); + + // Copy the dataset from source to destination and verify contents + src_dataset.copy_to(&dst_group, "copied_dataset").unwrap(); + + let copied_dataset = dst_group.dataset("copied_dataset").unwrap(); + let read_data: Vec = copied_dataset.read_1d().unwrap().to_vec(); + assert_eq!(read_data, &[1, 2, 3, 4, 5]); + + let copied_attr = copied_dataset.attr("src_attr").unwrap(); + let attr_value: f64 = copied_attr.read_scalar().unwrap(); + assert_eq!(attr_value, 42.0); + }) + }) + } + + #[test] + pub fn test_copy_group_with_nested_content() { + with_tmp_path(|src_path| { + with_tmp_path(|dst_path| { + // Create source file with nested structure + let src_file = File::create(&src_path).unwrap(); + let group1 = src_file.create_group("group1").unwrap(); + let subgroup = group1.create_group("subgroup").unwrap(); + + let ds1 = group1.new_dataset::().shape([3]).create("dataset1").unwrap(); + ds1.write(&[10, 20, 30]).unwrap(); + let ds2 = subgroup.new_dataset::().shape([2]).create("dataset2").unwrap(); + ds2.write(&[1.5, 2.5]).unwrap(); + + // Create destination file and copy entire group structure + let dst_file = File::create(&dst_path).unwrap(); + group1.copy_to(&dst_file, "copied_group1").unwrap(); + + // Verify the copied structure + let copied_group = dst_file.group("copied_group1").unwrap(); + + let copied_ds1 = copied_group.dataset("dataset1").unwrap(); + let data1: Vec = copied_ds1.read_1d().unwrap().to_vec(); + assert_eq!(data1, vec![10, 20, 30]); + + let copied_subgroup = copied_group.group("subgroup").unwrap(); + let copied_ds2 = copied_subgroup.dataset("dataset2").unwrap(); + let data2: Vec = copied_ds2.read_1d().unwrap().to_vec(); + assert_eq!(data2, vec![1.5, 2.5]); + }) + }) + } + + #[test] + pub fn test_copy_without_attributes() { + with_tmp_path(|src_path| { + with_tmp_path(|dst_path| { + let src_file = File::create(&src_path).unwrap(); + + let dataset = src_file.new_dataset::().shape([3]).create("data").unwrap(); + dataset.write(&[10, 20, 30]).unwrap(); + + let src_attr = dataset.new_attr::().create("foo_attr").unwrap(); + src_attr.write_scalar(&100).unwrap(); + + let dst_file = File::create(&dst_path).unwrap(); + + let ocpypl = ObjectCopy::build().copy_without_attr(true).finish().unwrap(); + + // Copy without attributes + dataset + .copy_to_with_props(&dst_file, "copied_no_attrs", Some(&ocpypl), None) + .unwrap(); + + src_file.close().unwrap(); + + let copied = dst_file.dataset("copied_no_attrs").unwrap(); + let copied_data: Vec = copied.read_1d().unwrap().to_vec(); + assert_eq!(copied_data, vec![10, 20, 30]); + + // Verify attributes were NOT copied + let attr_names = copied.attr_names().unwrap(); + assert!(attr_names.is_empty(), "Expected no attributes, but found: {attr_names:?}",); + }) + }) + } + + #[test] + pub fn test_copy_with_link_create_intermediate_groups() { + with_tmp_path(|src_path| { + with_tmp_path(|dst_path| { + let src_file = File::create(&src_path).unwrap(); + + let dataset = src_file.new_dataset::().shape([3]).create("data").unwrap(); + dataset.write(&[100, 200, 300]).unwrap(); + + let dst_file = File::create(&dst_path).unwrap(); + + // Fails without setting LinkCreate plist + assert!(dataset.copy_to(&dst_file, "level1/level2/level3/copied_data").is_err()); + + // Succeeds with LinkCreate + // Create link create property list that creates intermediate groups + dataset + .copy_to_with_props( + &dst_file, + "level1/level2/level3/copied_data", + None, + Some( + &LinkCreate::build().create_intermediate_group(true).finish().unwrap(), + ), + ) + .unwrap(); + + // Verify the intermediate groups were created + assert!(dst_file.group("level1").is_ok()); + assert!(dst_file.group("level1/level2").is_ok()); + assert!(dst_file.group("level1/level2/level3").is_ok()); + + // Verify the data + let copied = dst_file.dataset("level1/level2/level3/copied_data").unwrap(); + let data: Vec = copied.read_1d().unwrap().to_vec(); + assert_eq!(data, vec![100, 200, 300]); + }) + }) + } } From a5c39a6715a1682dd924f2836d9ae8bdd6c04d88 Mon Sep 17 00:00:00 2001 From: Philipp Bucher Date: Wed, 19 Nov 2025 11:37:16 +0100 Subject: [PATCH 042/141] ci: dont document dependencies --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5c9d0d35e..945a1433b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,7 @@ jobs: - name: Document workspace env: RUSTDOCFLAGS: "--cfg docsrs" - run: cargo doc --features static,zlib,blosc-all,lzf,f16,complex + run: cargo doc --features static,zlib,blosc-all,lzf,f16,complex --no-deps brew: name: brew From 25d595cf8358c5a9248a13c675d5acc006973fb4 Mon Sep 17 00:00:00 2001 From: Philipp Bucher Date: Wed, 19 Nov 2025 11:41:59 +0100 Subject: [PATCH 043/141] use derive for Default --- hdf5-sys/src/h5d.rs | 45 ++++++++++----------------------------------- hdf5-sys/src/h5f.rs | 9 ++------- hdf5-sys/src/h5t.rs | 9 ++------- 3 files changed, 14 insertions(+), 49 deletions(-) diff --git a/hdf5-sys/src/h5d.rs b/hdf5-sys/src/h5d.rs index 2a83e4149..7b72b4b29 100644 --- a/hdf5-sys/src/h5d.rs +++ b/hdf5-sys/src/h5d.rs @@ -27,33 +27,22 @@ pub enum H5D_layout_t { H5D_NLAYOUTS = 3, } -impl Default for H5D_layout_t { - fn default() -> Self { - Self::H5D_CONTIGUOUS - } -} - pub type H5D_chunk_index_t = c_uint; pub const H5D_CHUNK_BTREE: H5D_chunk_index_t = 0; pub const H5D_CHUNK_IDX_BTREE: H5D_chunk_index_t = H5D_CHUNK_BTREE; #[repr(C)] -#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Debug)] +#[derive(Copy, Clone, Default, PartialEq, Eq, PartialOrd, Debug)] pub enum H5D_alloc_time_t { H5D_ALLOC_TIME_ERROR = -1, + #[default] H5D_ALLOC_TIME_DEFAULT = 0, H5D_ALLOC_TIME_EARLY = 1, H5D_ALLOC_TIME_LATE = 2, H5D_ALLOC_TIME_INCR = 3, } -impl Default for H5D_alloc_time_t { - fn default() -> Self { - Self::H5D_ALLOC_TIME_DEFAULT - } -} - #[repr(C)] #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Debug)] pub enum H5D_space_status_t { @@ -64,35 +53,25 @@ pub enum H5D_space_status_t { } #[repr(C)] -#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Debug)] +#[derive(Copy, Clone, Default, PartialEq, Eq, PartialOrd, Debug)] pub enum H5D_fill_time_t { H5D_FILL_TIME_ERROR = -1, H5D_FILL_TIME_ALLOC = 0, H5D_FILL_TIME_NEVER = 1, + #[default] H5D_FILL_TIME_IFSET = 2, } -impl Default for H5D_fill_time_t { - fn default() -> Self { - Self::H5D_FILL_TIME_IFSET - } -} - #[repr(C)] -#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Debug)] +#[derive(Copy, Clone, Default, PartialEq, Eq, PartialOrd, Debug)] pub enum H5D_fill_value_t { H5D_FILL_VALUE_ERROR = -1, H5D_FILL_VALUE_UNDEFINED = 0, + #[default] H5D_FILL_VALUE_DEFAULT = 1, H5D_FILL_VALUE_USER_DEFINED = 2, } -impl Default for H5D_fill_value_t { - fn default() -> Self { - Self::H5D_FILL_VALUE_DEFAULT - } -} - #[repr(C)] #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Debug)] pub enum H5D_mpio_actual_chunk_opt_mode_t { @@ -218,10 +197,11 @@ mod hdf5_1_10_0 { use super::*; #[repr(C)] - #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Debug)] + #[derive(Copy, Clone, Default, PartialEq, Eq, PartialOrd, Debug)] pub enum H5D_layout_t { H5D_LAYOUT_ERROR = -1, H5D_COMPACT = 0, + #[default] H5D_CONTIGUOUS = 1, H5D_CHUNKED = 2, H5D_VIRTUAL = 3, @@ -229,19 +209,14 @@ mod hdf5_1_10_0 { } #[repr(C)] - #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Debug)] + #[derive(Copy, Clone, Default, PartialEq, Eq, PartialOrd, Debug)] pub enum H5D_vds_view_t { H5D_VDS_ERROR = -1, H5D_VDS_FIRST_MISSING = 0, + #[default] H5D_VDS_LAST_AVAILABLE = 1, } - impl Default for H5D_vds_view_t { - fn default() -> Self { - Self::H5D_VDS_LAST_AVAILABLE - } - } - pub const H5D_CHUNK_DONT_FILTER_PARTIAL_CHUNKS: c_uint = 0x0002; pub type H5D_append_cb_t = Option< diff --git a/hdf5-sys/src/h5f.rs b/hdf5-sys/src/h5f.rs index 092651d5c..a41fdf8ac 100644 --- a/hdf5-sys/src/h5f.rs +++ b/hdf5-sys/src/h5f.rs @@ -54,20 +54,15 @@ pub enum H5F_scope_t { } #[repr(C)] -#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Debug)] +#[derive(Copy, Clone, Default, PartialEq, Eq, PartialOrd, Debug)] pub enum H5F_close_degree_t { + #[default] H5F_CLOSE_DEFAULT = 0, H5F_CLOSE_WEAK = 1, H5F_CLOSE_SEMI = 2, H5F_CLOSE_STRONG = 3, } -impl Default for H5F_close_degree_t { - fn default() -> Self { - Self::H5F_CLOSE_DEFAULT - } -} - #[cfg_attr(feature = "1.10.0", deprecated(note = "deprecated in HDF5 1.10.0, use H5F_info2_t"))] #[repr(C)] #[derive(Debug, Copy, Clone)] diff --git a/hdf5-sys/src/h5t.rs b/hdf5-sys/src/h5t.rs index eabad5b3b..da2e6934f 100644 --- a/hdf5-sys/src/h5t.rs +++ b/hdf5-sys/src/h5t.rs @@ -81,9 +81,10 @@ pub enum H5T_norm_t { } #[repr(C)] -#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Debug)] +#[derive(Copy, Clone, Default, PartialEq, Eq, PartialOrd, Debug)] pub enum H5T_cset_t { H5T_CSET_ERROR = -1, + #[default] H5T_CSET_ASCII = 0, H5T_CSET_UTF8 = 1, H5T_CSET_RESERVED_2 = 2, @@ -102,12 +103,6 @@ pub enum H5T_cset_t { H5T_CSET_RESERVED_15 = 15, } -impl Default for H5T_cset_t { - fn default() -> Self { - Self::H5T_CSET_ASCII - } -} - pub const H5T_NCSET: H5T_cset_t = H5T_CSET_RESERVED_2; #[repr(C)] From e794e6c2da46373c9673e5ea8dbbba34bd6e6b88 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Tue, 29 Apr 2025 11:12:30 +0200 Subject: [PATCH 044/141] SWMR functionality --- hdf5/examples/swmr.rs | 39 +++++++++++++++++++++++++++++++++++++++ hdf5/src/hl/dataset.rs | 18 ++++++++++++++++-- hdf5/src/hl/file.rs | 16 +++++++++++++--- 3 files changed, 68 insertions(+), 5 deletions(-) create mode 100644 hdf5/examples/swmr.rs diff --git a/hdf5/examples/swmr.rs b/hdf5/examples/swmr.rs new file mode 100644 index 000000000..d97b595aa --- /dev/null +++ b/hdf5/examples/swmr.rs @@ -0,0 +1,39 @@ +use hdf5_metno as hdf5; + +fn reader() { + let file = hdf5::File::open_as("swmr.h5", hdf5::OpenMode::ReadSWMR).unwrap(); + println!("Reader: Opened file"); + let var = file.dataset("foo").unwrap(); + + for _ in 0..5 { + var.refresh().unwrap(); + let shape = var.shape(); + println!("Reader: Got shape: {shape:?}"); + // If reading one should use the shape directly without + // using the convenience read_2d etc. functions which + // might get confused if the shape is changed during reading + std::thread::sleep(std::time::Duration::from_secs(5)); + } +} + +fn main() { + let file = + hdf5::File::with_options().with_fapl(|fapl| fapl.libver_v110()).create("swmr.h5").unwrap(); + + let var = file.new_dataset::().shape((0.., 5)).create("foo").unwrap(); + + file.start_swmr().unwrap(); + println!("Writer: Wrote file"); + + let thread = std::thread::spawn(|| reader()); + + for i in 0..5 { + var.resize((i + 1, 5)).unwrap(); + var.write_slice(&[i, i, i, i, i], (i, 0..)).unwrap(); + var.flush().unwrap(); + println!("Writer: Wrote {i}"); + std::thread::sleep(std::time::Duration::from_secs(5)); + } + + thread.join().unwrap(); +} diff --git a/hdf5/src/hl/dataset.rs b/hdf5/src/hl/dataset.rs index 77807c530..f8f281279 100644 --- a/hdf5/src/hl/dataset.rs +++ b/hdf5/src/hl/dataset.rs @@ -5,8 +5,8 @@ use ndarray::{self, ArrayView}; use hdf5_sys::h5::HADDR_UNDEF; use hdf5_sys::h5d::{ - H5Dcreate2, H5Dcreate_anon, H5Dget_access_plist, H5Dget_create_plist, H5Dget_offset, - H5Dset_extent, + H5Dcreate2, H5Dcreate_anon, H5Dflush, H5Dget_access_plist, H5Dget_create_plist, H5Dget_offset, + H5Drefresh, H5Dset_extent, }; use hdf5_sys::h5l::H5Ldelete; use hdf5_sys::h5p::H5P_DEFAULT; @@ -154,6 +154,20 @@ impl Dataset { pub fn filters(&self) -> Vec { self.dcpl().map_or(Vec::default(), |pl| pl.filters()) } + + /// Flush the dataset metadata from the metadata cache to the file + pub fn flush(&self) -> Result<()> { + let id = self.id(); + h5call!(H5Dflush(id))?; + Ok(()) + } + + /// Refresh metadata items assosicated with the dataset + pub fn refresh(&self) -> Result<()> { + let id = self.id(); + h5call!(H5Drefresh(id))?; + Ok(()) + } } pub struct Maybe(Option); diff --git a/hdf5/src/hl/file.rs b/hdf5/src/hl/file.rs index 81ad839dc..dea3f1d30 100644 --- a/hdf5/src/hl/file.rs +++ b/hdf5/src/hl/file.rs @@ -5,8 +5,9 @@ use std::path::Path; use hdf5_sys::h5f::{ H5Fclose, H5Fcreate, H5Fflush, H5Fget_access_plist, H5Fget_create_plist, H5Fget_filesize, - H5Fget_freespace, H5Fget_intent, H5Fget_obj_count, H5Fget_obj_ids, H5Fopen, H5F_ACC_DEFAULT, - H5F_ACC_EXCL, H5F_ACC_RDONLY, H5F_ACC_RDWR, H5F_ACC_TRUNC, H5F_SCOPE_LOCAL, + H5Fget_freespace, H5Fget_intent, H5Fget_obj_count, H5Fget_obj_ids, H5Fopen, + H5Fstart_swmr_write, H5F_ACC_DEFAULT, H5F_ACC_EXCL, H5F_ACC_RDONLY, H5F_ACC_RDWR, + H5F_ACC_SWMR_READ, H5F_ACC_TRUNC, H5F_SCOPE_LOCAL, }; use crate::hl::plist::{ @@ -20,6 +21,8 @@ use crate::internal_prelude::*; pub enum OpenMode { /// Open a file as read-only, file must exist. Read, + /// Open a file as read-only in SWMR mode, file must exist. + ReadSWMR, /// Open a file as read/write, file must exist. ReadWrite, /// Create a file, truncate if exists. @@ -178,6 +181,12 @@ impl File { pub fn fcpl(&self) -> Result { self.create_plist() } + + pub fn start_swmr(&self) -> Result<()> { + let id = self.id(); + h5call!(H5Fstart_swmr_write(id))?; + Ok(()) + } } /// File builder allowing to customize file access/creation property lists. @@ -231,6 +240,7 @@ impl FileBuilder { )?; let flags = match mode { OpenMode::Read => H5F_ACC_RDONLY, + OpenMode::ReadSWMR => H5F_ACC_RDONLY | H5F_ACC_SWMR_READ, OpenMode::ReadWrite => H5F_ACC_RDWR, OpenMode::Create => H5F_ACC_TRUNC, OpenMode::CreateExcl | OpenMode::Append => H5F_ACC_EXCL, @@ -239,7 +249,7 @@ impl FileBuilder { h5lock!({ let fapl = self.fapl.finish()?; match mode { - OpenMode::Read | OpenMode::ReadWrite => { + OpenMode::Read | OpenMode::ReadWrite | OpenMode::ReadSWMR => { File::from_id(h5try!(H5Fopen(fname_ptr, flags, fapl.id()))) } _ => { From a8c3907d94c93deb369e7700982cc417b51626fd Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Tue, 29 Apr 2025 11:47:49 +0200 Subject: [PATCH 045/141] Add backwards compatibility --- CHANGELOG.md | 4 +++- hdf5/examples/swmr.rs | 38 ++++++++++++++++++++++++-------------- hdf5/src/hl/dataset.rs | 8 ++++++-- hdf5/src/hl/file.rs | 18 ++++++++++++++---- 4 files changed, 47 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bd4314759..140318b48 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,13 @@ # Changelog -## hdf5 unreleased ## hdf5-types unreleased ## hdf5-derive unreleased ## hdf5-sys unreleased ## hdf5-src unreleased +## hdf5 unreleased +- Added support for Single Writer Multiple Readers (SWMR) (breaking change, OpenMode has extra variant) + ## hdf5 v0.10.2 Release date: Oct 16, 2025 - Upgraded upper bound of ndarray diff --git a/hdf5/examples/swmr.rs b/hdf5/examples/swmr.rs index d97b595aa..b2912f6e7 100644 --- a/hdf5/examples/swmr.rs +++ b/hdf5/examples/swmr.rs @@ -1,5 +1,7 @@ +#[cfg(feature = "1.10.2")] use hdf5_metno as hdf5; +#[cfg(feature = "1.10.2")] fn reader() { let file = hdf5::File::open_as("swmr.h5", hdf5::OpenMode::ReadSWMR).unwrap(); println!("Reader: Opened file"); @@ -17,23 +19,31 @@ fn reader() { } fn main() { - let file = - hdf5::File::with_options().with_fapl(|fapl| fapl.libver_v110()).create("swmr.h5").unwrap(); + #[cfg(not(feature = "1.10.2"))] + println!("This examples requires hdf5 >= 1.10.2 to enable SWMR and set libver_bounds"); - let var = file.new_dataset::().shape((0.., 5)).create("foo").unwrap(); + #[cfg(feature = "1.10.2")] + { + let file = hdf5::File::with_options() + .with_fapl(|fapl| fapl.libver_v110()) + .create("swmr.h5") + .unwrap(); - file.start_swmr().unwrap(); - println!("Writer: Wrote file"); + let var = file.new_dataset::().shape((0.., 5)).create("foo").unwrap(); - let thread = std::thread::spawn(|| reader()); + file.start_swmr().unwrap(); + println!("Writer: Wrote file"); - for i in 0..5 { - var.resize((i + 1, 5)).unwrap(); - var.write_slice(&[i, i, i, i, i], (i, 0..)).unwrap(); - var.flush().unwrap(); - println!("Writer: Wrote {i}"); - std::thread::sleep(std::time::Duration::from_secs(5)); - } + let thread = std::thread::spawn(|| reader()); - thread.join().unwrap(); + for i in 0..5 { + var.resize((i + 1, 5)).unwrap(); + var.write_slice(&[i, i, i, i, i], (i, 0..)).unwrap(); + var.flush().unwrap(); + println!("Writer: Wrote {i}"); + std::thread::sleep(std::time::Duration::from_secs(5)); + } + + thread.join().unwrap(); + } } diff --git a/hdf5/src/hl/dataset.rs b/hdf5/src/hl/dataset.rs index f8f281279..24f3b160b 100644 --- a/hdf5/src/hl/dataset.rs +++ b/hdf5/src/hl/dataset.rs @@ -5,9 +5,11 @@ use ndarray::{self, ArrayView}; use hdf5_sys::h5::HADDR_UNDEF; use hdf5_sys::h5d::{ - H5Dcreate2, H5Dcreate_anon, H5Dflush, H5Dget_access_plist, H5Dget_create_plist, H5Dget_offset, - H5Drefresh, H5Dset_extent, + H5Dcreate2, H5Dcreate_anon, H5Dget_access_plist, H5Dget_create_plist, H5Dget_offset, + H5Dset_extent, }; +#[cfg(feature = "1.10.0")] +use hdf5_sys::h5d::{H5Dflush, H5Drefresh}; use hdf5_sys::h5l::H5Ldelete; use hdf5_sys::h5p::H5P_DEFAULT; use hdf5_sys::h5z::H5Z_filter_t; @@ -156,6 +158,7 @@ impl Dataset { } /// Flush the dataset metadata from the metadata cache to the file + #[cfg(feature = "1.10.0")] pub fn flush(&self) -> Result<()> { let id = self.id(); h5call!(H5Dflush(id))?; @@ -163,6 +166,7 @@ impl Dataset { } /// Refresh metadata items assosicated with the dataset + #[cfg(feature = "1.10.0")] pub fn refresh(&self) -> Result<()> { let id = self.id(); h5call!(H5Drefresh(id))?; diff --git a/hdf5/src/hl/file.rs b/hdf5/src/hl/file.rs index dea3f1d30..3dd59f536 100644 --- a/hdf5/src/hl/file.rs +++ b/hdf5/src/hl/file.rs @@ -5,10 +5,11 @@ use std::path::Path; use hdf5_sys::h5f::{ H5Fclose, H5Fcreate, H5Fflush, H5Fget_access_plist, H5Fget_create_plist, H5Fget_filesize, - H5Fget_freespace, H5Fget_intent, H5Fget_obj_count, H5Fget_obj_ids, H5Fopen, - H5Fstart_swmr_write, H5F_ACC_DEFAULT, H5F_ACC_EXCL, H5F_ACC_RDONLY, H5F_ACC_RDWR, - H5F_ACC_SWMR_READ, H5F_ACC_TRUNC, H5F_SCOPE_LOCAL, + H5Fget_freespace, H5Fget_intent, H5Fget_obj_count, H5Fget_obj_ids, H5Fopen, H5F_ACC_DEFAULT, + H5F_ACC_EXCL, H5F_ACC_RDONLY, H5F_ACC_RDWR, H5F_ACC_TRUNC, H5F_SCOPE_LOCAL, }; +#[cfg(feature = "1.10.0")] +use hdf5_sys::h5f::{H5Fstart_swmr_write, H5F_ACC_SWMR_READ}; use crate::hl::plist::{ file_access::{FileAccess, FileAccessBuilder}, @@ -18,10 +19,12 @@ use crate::internal_prelude::*; /// File opening mode. #[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[cfg_attr(not(feature = "1.10.0"), non_exhaustive)] pub enum OpenMode { /// Open a file as read-only, file must exist. Read, /// Open a file as read-only in SWMR mode, file must exist. + #[cfg(feature = "1.10.0")] ReadSWMR, /// Open a file as read/write, file must exist. ReadWrite, @@ -182,6 +185,8 @@ impl File { self.create_plist() } + #[cfg(feature = "1.10.0")] + /// Mark this file as ready for opening as SWMR pub fn start_swmr(&self) -> Result<()> { let id = self.id(); h5call!(H5Fstart_swmr_write(id))?; @@ -240,18 +245,23 @@ impl FileBuilder { )?; let flags = match mode { OpenMode::Read => H5F_ACC_RDONLY, + #[cfg(feature = "1.10.0")] OpenMode::ReadSWMR => H5F_ACC_RDONLY | H5F_ACC_SWMR_READ, OpenMode::ReadWrite => H5F_ACC_RDWR, OpenMode::Create => H5F_ACC_TRUNC, OpenMode::CreateExcl | OpenMode::Append => H5F_ACC_EXCL, + #[cfg(not(feature = "1.10.0"))] + _ => unreachable!(), }; let fname_ptr = filename.as_ptr(); h5lock!({ let fapl = self.fapl.finish()?; match mode { - OpenMode::Read | OpenMode::ReadWrite | OpenMode::ReadSWMR => { + OpenMode::Read | OpenMode::ReadWrite => { File::from_id(h5try!(H5Fopen(fname_ptr, flags, fapl.id()))) } + #[cfg(feature = "1.10.0")] + OpenMode::ReadSWMR => File::from_id(h5try!(H5Fopen(fname_ptr, flags, fapl.id()))), _ => { let fcpl = self.fcpl.finish()?; File::from_id(h5try!(H5Fcreate(fname_ptr, flags, fcpl.id(), fapl.id()))) From 8cbc639ee52969679a609d11f220661b59224bb1 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Wed, 19 Nov 2025 21:15:35 +0100 Subject: [PATCH 046/141] Fix MSRV --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 945a1433b..63b9f2a66 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -293,8 +293,9 @@ jobs: run: cargo update - name: Override deps - run: + run: | cargo update half@2.7.1 --precise 2.4.1 + cargo update indexmap@2.12.0 --precise 2.11.4 - name: Build and test all crates run: cargo test --locked --workspace -vv --features=hdf5-sys/static,hdf5-sys/zlib --exclude=hdf5-metno-derive From 196b7939c0b57280e2997a165dc92e0a91a79de1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 19 Nov 2025 21:04:15 +0000 Subject: [PATCH 047/141] deps: update libloading requirement from 0.8 to 0.9 Updates the requirements on [libloading](https://github.com/nagisa/rust_libloading) to permit the latest version. - [Commits](https://github.com/nagisa/rust_libloading/compare/0.8.0...0.9.0) --- updated-dependencies: - dependency-name: libloading dependency-version: 0.9.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- hdf5-sys/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hdf5-sys/Cargo.toml b/hdf5-sys/Cargo.toml index 6813b0f6e..6f0ef3f61 100644 --- a/hdf5-sys/Cargo.toml +++ b/hdf5-sys/Cargo.toml @@ -32,7 +32,7 @@ static = ["dep:hdf5-src"] deprecated = ["hdf5-src?/deprecated"] [build-dependencies] -libloading = "0.8" +libloading = "0.9" regex = { workspace = true } [target.'cfg(any(all(unix, not(target_os = "macos")), windows))'.build-dependencies] From 194142f4e570889b6e5fb53e6b2a6189ce3071fe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 19 Nov 2025 21:04:18 +0000 Subject: [PATCH 048/141] Bump crate-ci/typos from 1.38.1 to 1.39.2 Bumps [crate-ci/typos](https://github.com/crate-ci/typos) from 1.38.1 to 1.39.2. - [Release notes](https://github.com/crate-ci/typos/releases) - [Changelog](https://github.com/crate-ci/typos/blob/master/CHANGELOG.md) - [Commits](https://github.com/crate-ci/typos/compare/80c8a4945eec0f6d464eaf9e65ed98ef085283d1...626c4bedb751ce0b7f03262ca97ddda9a076ae1c) --- updated-dependencies: - dependency-name: crate-ci/typos dependency-version: 1.39.2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 63b9f2a66..daaae36b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: - name: Checkout code uses: actions/checkout@v5 - name: Check spelling - uses: crate-ci/typos@80c8a4945eec0f6d464eaf9e65ed98ef085283d1 # v1.38.1 + uses: crate-ci/typos@626c4bedb751ce0b7f03262ca97ddda9a076ae1c # v1.39.2 lint: name: lint From 3ee8f7b0bee1d1653823d021da6a368051a5b0f8 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Wed, 19 Nov 2025 22:51:03 +0100 Subject: [PATCH 049/141] Downgrade libloading (MSRV) --- hdf5-sys/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hdf5-sys/Cargo.toml b/hdf5-sys/Cargo.toml index 6f0ef3f61..6813b0f6e 100644 --- a/hdf5-sys/Cargo.toml +++ b/hdf5-sys/Cargo.toml @@ -32,7 +32,7 @@ static = ["dep:hdf5-src"] deprecated = ["hdf5-src?/deprecated"] [build-dependencies] -libloading = "0.9" +libloading = "0.8" regex = { workspace = true } [target.'cfg(any(all(unix, not(target_os = "macos")), windows))'.build-dependencies] From 4a82bc8481fac69f21ab5011da9f429f369aa1c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Beck=20K=C3=B6nig?= Date: Thu, 20 Nov 2025 16:56:07 +0100 Subject: [PATCH 050/141] changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bd4314759..2ee5a9476 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ ## hdf5-sys unreleased ## hdf5-src unreleased +- Fixed incorrect retrieved name of attributes + ## hdf5 v0.10.2 Release date: Oct 16, 2025 - Upgraded upper bound of ndarray From bc3f3798ac797441e270636a7b86de301d31249f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Beck=20K=C3=B6nig?= Date: Thu, 20 Nov 2025 16:59:31 +0100 Subject: [PATCH 051/141] Revert "enable threadsafe feature for static build" This reverts commit 65ab499e276554bd900e7ddd70e6237b339e499f. --- hdf5-sys/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hdf5-sys/Cargo.toml b/hdf5-sys/Cargo.toml index 8f0206ae6..6813b0f6e 100644 --- a/hdf5-sys/Cargo.toml +++ b/hdf5-sys/Cargo.toml @@ -5,7 +5,7 @@ description = "Native bindings to the HDF5 library." links = "hdf5" readme = "README.md" categories = ["development-tools::ffi", "filesystem", "science"] -version = "0.10.1" # !V +version = "0.10.1" # !V rust-version.workspace = true authors.workspace = true keywords.workspace = true @@ -28,7 +28,7 @@ mpio = ["dep:mpi-sys"] hl = ["hdf5-src?/hl"] threadsafe = ["hdf5-src?/threadsafe"] zlib = ["dep:libz-sys", "hdf5-src?/zlib"] -static = ["hdf5-src/threadsafe"] +static = ["dep:hdf5-src"] deprecated = ["hdf5-src?/deprecated"] [build-dependencies] From 6a229ce99698bbc5975ba4f4efeacd7d0bc9d0c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Beck=20K=C3=B6nig?= Date: Thu, 20 Nov 2025 17:16:53 +0100 Subject: [PATCH 052/141] review comments - simplify --- hdf5/examples/continuously_rw_1d.rs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/hdf5/examples/continuously_rw_1d.rs b/hdf5/examples/continuously_rw_1d.rs index 459d47c03..425870434 100644 --- a/hdf5/examples/continuously_rw_1d.rs +++ b/hdf5/examples/continuously_rw_1d.rs @@ -16,14 +16,9 @@ const NUM_CHUNKS: usize = 3; // total chunks to write fn write_hdf5() -> Result<()> { println!("Creating file '{FILE_NAME}' with 1D resizable dataset"); let file = File::create(FILE_NAME)?; - let shape = Extent::resizable(1); // 1D resizable + let shape = Extent::resizable(0); // 1D resizable let ds = file.new_dataset::().chunk((CHUNK_SIZE,)).shape(shape).create(DATASET_NAME)?; - // Dataset is created with length 1 - // so we resize to 0 to only store "real" values - assert_eq!(ds.size(), 1); - ds.resize((0,))?; - // Simulate continuously accumulating data in a buffer // and writing it to the dataset anytime there's enough to fill a chunk let mut buf = Vec::with_capacity(CHUNK_SIZE); @@ -49,7 +44,7 @@ fn read_hdf5() -> Result<()> { // Check shape let shape = ds.shape(); println!("Dataset shape: {:?}", shape); - assert_eq!(shape, vec![CHUNK_SIZE * NUM_CHUNKS]); + assert_eq!(shape, &[CHUNK_SIZE * NUM_CHUNKS]); // Get chunking metadata let chunk_size = ds.chunk().unwrap()[0]; From 5fbd0e28695f67acd7cb56d8a7682c5fc2604708 Mon Sep 17 00:00:00 2001 From: Marc BK <72197092+CramBL@users.noreply.github.com> Date: Thu, 20 Nov 2025 17:25:38 +0100 Subject: [PATCH 053/141] Update CHANGELOG.md Co-authored-by: Magnus Ulimoen Signed-off-by: Marc BK <72197092+CramBL@users.noreply.github.com> --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf232459d..72ea0a9e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,9 @@ ## hdf5-derive unreleased ## hdf5-sys unreleased ## hdf5-src unreleased -- Fixed incorrect retrieved name of attributes ## hdf5 unreleased +- Fixed incorrect retrieved name of attributes - Added support for Single Writer Multiple Readers (SWMR) (breaking change, OpenMode has extra variant) ## hdf5 v0.10.2 From 9d3b85a44ca1548d7046d18949cdedfb3443bed5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Beck=20K=C3=B6nig?= Date: Thu, 20 Nov 2025 18:13:33 +0100 Subject: [PATCH 054/141] gate example behind necessary feature version --- hdf5/examples/continuously_rw_1d.rs | 132 +++++++++++++++------------- 1 file changed, 72 insertions(+), 60 deletions(-) diff --git a/hdf5/examples/continuously_rw_1d.rs b/hdf5/examples/continuously_rw_1d.rs index 425870434..8c68b8f50 100644 --- a/hdf5/examples/continuously_rw_1d.rs +++ b/hdf5/examples/continuously_rw_1d.rs @@ -2,77 +2,89 @@ //! //! Finally read it, get the chunking metadata, and use it to read slices aligned to the chunk size -use hdf5::{File, Result}; -use hdf5_metno::{self as hdf5, Extent}; -use ndarray::Array1; +#[cfg(feature = "1.10.5")] +fn main() -> hdf5_metno::Result<()> { + example::write_hdf5()?; + example::read_hdf5()?; + Ok(()) +} -// Shared between read_hdf5() and write_hdf5(), otherwise they only rely on file metadata -const FILE_NAME: &str = "continuous_chunks.h5"; -const DATASET_NAME: &str = "count"; +#[cfg(not(feature = "1.10.5"))] +fn main() { + println!( + "needs version 1.10.5 or later for querying the number of chunks with H5Dget_num_chunks" + ); +} -const CHUNK_SIZE: usize = 5; -const NUM_CHUNKS: usize = 3; // total chunks to write +#[cfg(feature = "1.10.5")] +mod example { + use hdf5::{File, Result}; + use hdf5_metno::{self as hdf5, Extent}; + use ndarray::Array1; -fn write_hdf5() -> Result<()> { - println!("Creating file '{FILE_NAME}' with 1D resizable dataset"); - let file = File::create(FILE_NAME)?; - let shape = Extent::resizable(0); // 1D resizable - let ds = file.new_dataset::().chunk((CHUNK_SIZE,)).shape(shape).create(DATASET_NAME)?; + // Shared between read_hdf5() and write_hdf5(), otherwise they only rely on file metadata + const FILE_NAME: &str = "continuous_chunks.h5"; + const DATASET_NAME: &str = "count"; - // Simulate continuously accumulating data in a buffer - // and writing it to the dataset anytime there's enough to fill a chunk - let mut buf = Vec::with_capacity(CHUNK_SIZE); - for i in 0..NUM_CHUNKS * CHUNK_SIZE { - buf.push(i); - if buf.len() == CHUNK_SIZE { - let current_size = ds.size(); - println!("[{i}] writing new chunk, current size = {current_size}"); - ds.resize((current_size + CHUNK_SIZE,))?; - ds.write_slice(&buf, current_size..current_size + CHUNK_SIZE)?; - buf.clear(); - } - } + const CHUNK_SIZE: usize = 5; + const NUM_CHUNKS: usize = 3; // total chunks to write - Ok(()) -} + pub fn write_hdf5() -> Result<()> { + println!("Creating file '{FILE_NAME}' with 1D resizable dataset"); + let file = File::create(FILE_NAME)?; + let shape = Extent::resizable(0); // 1D resizable + let ds = + file.new_dataset::().chunk((CHUNK_SIZE,)).shape(shape).create(DATASET_NAME)?; + + // Simulate continuously accumulating data in a buffer + // and writing it to the dataset anytime there's enough to fill a chunk + let mut buf = Vec::with_capacity(CHUNK_SIZE); + for i in 0..NUM_CHUNKS * CHUNK_SIZE { + buf.push(i); + if buf.len() == CHUNK_SIZE { + let current_size = ds.size(); + println!("[{i}] writing new chunk, current size = {current_size}"); + ds.resize((current_size + CHUNK_SIZE,))?; + ds.write_slice(&buf, current_size..current_size + CHUNK_SIZE)?; + buf.clear(); + } + } -fn read_hdf5() -> Result<()> { - println!("Reading file '{FILE_NAME}'"); - let file = File::open(FILE_NAME)?; - let ds = file.dataset(DATASET_NAME)?; + Ok(()) + } - // Check shape - let shape = ds.shape(); - println!("Dataset shape: {:?}", shape); - assert_eq!(shape, &[CHUNK_SIZE * NUM_CHUNKS]); + pub fn read_hdf5() -> Result<()> { + println!("Reading file '{FILE_NAME}'"); + let file = File::open(FILE_NAME)?; + let ds = file.dataset(DATASET_NAME)?; - // Get chunking metadata - let chunk_size = ds.chunk().unwrap()[0]; - println!("Chunk size: {chunk_size}"); - assert_eq!(chunk_size, CHUNK_SIZE); + // Check shape + let shape = ds.shape(); + println!("Dataset shape: {:?}", shape); + assert_eq!(shape, &[CHUNK_SIZE * NUM_CHUNKS]); - let num_chunks = ds.num_chunks().unwrap(); - println!("Number of chunks: {num_chunks}"); - assert_eq!(num_chunks, NUM_CHUNKS); + // Get chunking metadata + let chunk_size = ds.chunk().unwrap()[0]; + println!("Chunk size: {chunk_size}"); + assert_eq!(chunk_size, CHUNK_SIZE); - // Read the dataset chunk for chunk - for chunk_idx in 0..num_chunks { - let chunk_start = chunk_idx * chunk_size; - let arr: Array1 = ds.read_slice(chunk_start..chunk_start + chunk_size)?; - println!("Dataset Chunk #{chunk_idx}: {arr:?}"); - assert_eq!(arr, Array1::from_iter(chunk_start..chunk_start + chunk_size)); - } + let num_chunks = ds.num_chunks().unwrap(); + println!("Number of chunks: {num_chunks}"); + assert_eq!(num_chunks, NUM_CHUNKS); - // Read the full dataset in one go - let arr: Array1 = ds.read_1d()?; - println!("Full dataset: {arr:?}"); - assert_eq!(arr, Array1::from_iter(0..CHUNK_SIZE * NUM_CHUNKS)); + // Read the dataset chunk for chunk + for chunk_idx in 0..num_chunks { + let chunk_start = chunk_idx * chunk_size; + let arr: Array1 = ds.read_slice(chunk_start..chunk_start + chunk_size)?; + println!("Dataset Chunk #{chunk_idx}: {arr:?}"); + assert_eq!(arr, Array1::from_iter(chunk_start..chunk_start + chunk_size)); + } - Ok(()) -} + // Read the full dataset in one go + let arr: Array1 = ds.read_1d()?; + println!("Full dataset: {arr:?}"); + assert_eq!(arr, Array1::from_iter(0..CHUNK_SIZE * NUM_CHUNKS)); -fn main() -> Result<()> { - write_hdf5()?; - read_hdf5()?; - Ok(()) + Ok(()) + } } From 3870ab3d53fe0fb121aecbfe199b5359ca86dd1e Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Thu, 20 Nov 2025 20:33:48 +0100 Subject: [PATCH 055/141] Downgrade correct indexmap --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index daaae36b5..8680469b3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -295,7 +295,7 @@ jobs: - name: Override deps run: | cargo update half@2.7.1 --precise 2.4.1 - cargo update indexmap@2.12.0 --precise 2.11.4 + cargo update indexmap@2.12.1 --precise 2.11.4 - name: Build and test all crates run: cargo test --locked --workspace -vv --features=hdf5-sys/static,hdf5-sys/zlib --exclude=hdf5-metno-derive From 11e3259e4da144e16049e5a49fda4f9ad8a92b01 Mon Sep 17 00:00:00 2001 From: Vadim Dyadkin Date: Wed, 12 Nov 2025 17:51:04 +0100 Subject: [PATCH 056/141] Add cmake flags for static msvc runtime --- hdf5-src/build.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/hdf5-src/build.rs b/hdf5-src/build.rs index 699adaaf9..12a53e198 100644 --- a/hdf5-src/build.rs +++ b/hdf5-src/build.rs @@ -33,6 +33,11 @@ fn main() { println!("cargo::rerun-if-changed=build.rs"); let mut cfg = cmake::Config::new("ext/hdf5"); + if cfg!(target_env = "msvc") { + cfg.define("CMAKE_POLICY_DEFAULT_CMP0091", "NEW"); + cfg.define("CMAKE_MSVC_RUNTIME_LIBRARY", "MultiThreaded$<$:Debug>"); + } + // only build the static c library, disable everything else cfg.define("HDF5_NO_PACKAGES", "ON"); for option in &[ From 5942195bd15d25a957eb49c8521058b1894b7696 Mon Sep 17 00:00:00 2001 From: Vadim Dyadkin Date: Thu, 13 Nov 2025 11:20:57 +0100 Subject: [PATCH 057/141] Set cmake variables based on the env --- hdf5-src/build.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/hdf5-src/build.rs b/hdf5-src/build.rs index 12a53e198..7f63fdc81 100644 --- a/hdf5-src/build.rs +++ b/hdf5-src/build.rs @@ -34,8 +34,12 @@ fn main() { let mut cfg = cmake::Config::new("ext/hdf5"); if cfg!(target_env = "msvc") { - cfg.define("CMAKE_POLICY_DEFAULT_CMP0091", "NEW"); - cfg.define("CMAKE_MSVC_RUNTIME_LIBRARY", "MultiThreaded$<$:Debug>"); + if let Ok(var) = env::var("CMAKE_POLICY_DEFAULT_CMP0091") { + cfg.define("CMAKE_POLICY_DEFAULT_CMP0091", var); + } + if let Ok(var) = env::var("CMAKE_MSVC_RUNTIME_LIBRARY") { + cfg.define("CMAKE_MSVC_RUNTIME_LIBRARY", var); + } } // only build the static c library, disable everything else From 63498e3c7eb3a658e327d0caa05ff4b816e7c4bb Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Thu, 20 Nov 2025 20:26:42 +0100 Subject: [PATCH 058/141] Set CMAKE_POLICY_DEFAULT_CMP0091 to NEW Signed-off-by: Magnus Ulimoen --- hdf5-src/build.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/hdf5-src/build.rs b/hdf5-src/build.rs index 7f63fdc81..ff14c1d96 100644 --- a/hdf5-src/build.rs +++ b/hdf5-src/build.rs @@ -34,9 +34,7 @@ fn main() { let mut cfg = cmake::Config::new("ext/hdf5"); if cfg!(target_env = "msvc") { - if let Ok(var) = env::var("CMAKE_POLICY_DEFAULT_CMP0091") { - cfg.define("CMAKE_POLICY_DEFAULT_CMP0091", var); - } + cfg.define("CMAKE_POLICY_DEFAULT_CMP0091", "NEW"); if let Ok(var) = env::var("CMAKE_MSVC_RUNTIME_LIBRARY") { cfg.define("CMAKE_MSVC_RUNTIME_LIBRARY", var); } From 61dc387686c06c90a8075ab863765822d99f4c4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Beck=20K=C3=B6nig?= Date: Thu, 20 Nov 2025 20:49:15 +0100 Subject: [PATCH 059/141] fix bad concurrency group name --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5207067f7..5320d2d21 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ on: - cron: '0 18 * * *' concurrency: - group: ${{ github.workflow }}-@{{ github.ref }} + group: ${{ github.workflow }}-@${{ github.ref }} cancel-in-progress: true env: From b1236ed2a13bbe7ff246d27849021642fb10246b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Beck=20K=C3=B6nig?= Date: Thu, 20 Nov 2025 20:54:24 +0100 Subject: [PATCH 060/141] avoid using lint reasons to stick to the current MSRV --- hdf5/tests/test_datatypes.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hdf5/tests/test_datatypes.rs b/hdf5/tests/test_datatypes.rs index b4ac1410a..2035c87bd 100644 --- a/hdf5/tests/test_datatypes.rs +++ b/hdf5/tests/test_datatypes.rs @@ -167,7 +167,7 @@ fn test_print_display_debug_datatype_f64() { #[test] fn test_print_display_debug_datatype_color_enum() { - #[allow(dead_code, reason = "we use the type, we just don't construct it")] + #[allow(dead_code)] // "we use the type, we just don't construct it" #[derive(H5Type)] #[repr(u8)] enum Color { From 2463d463cc5d63fd0fb1e53d70126e5f2b472efa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Beck=20K=C3=B6nig?= Date: Thu, 20 Nov 2025 21:06:44 +0100 Subject: [PATCH 061/141] remove pretty_assertions from workspace dependencies --- Cargo.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 3f189ee65..c57c5cd2b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,8 +24,6 @@ libz-sys = { version = "1.1", default-features = false } mpi-sys = "0.2" num-complex = { version = "0.4", default-features = false } regex = "1.10" -# external dev-dependencies -pretty_assertions = "1.4.1" # internal hdf5 = { package = "hdf5-metno", version = "0.9.3", path = "hdf5" } # !V From 75e2ab50076d3c2020bcdf815e4ce5907b967740 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Beck=20K=C3=B6nig?= Date: Thu, 20 Nov 2025 21:29:10 +0100 Subject: [PATCH 062/141] Ensure the CI workflow workflow gets cancelled if a newer commit has been pushed on the same branch (unless it's the default branch) --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5320d2d21..010987878 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,8 +10,9 @@ on: schedule: - cron: '0 18 * * *' +# Ensure this workflow gets cancelled if a newer commit has been pushed on the same branch (unless it's the default branch) concurrency: - group: ${{ github.workflow }}-@${{ github.ref }} + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true env: From 9edbe8275e97d0c992424a18230c070df7f85499 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Sun, 23 Nov 2025 22:18:31 +0100 Subject: [PATCH 063/141] Add to changelog --- CHANGELOG.md | 21 +++++++++++++++++++-- hdf5-derive/Cargo.toml | 2 +- hdf5-src/Cargo.toml | 2 +- hdf5-types/Cargo.toml | 2 +- hdf5/Cargo.toml | 2 +- 5 files changed, 23 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 72ea0a9e5..257e02348 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,12 +4,29 @@ ## hdf5-derive unreleased ## hdf5-sys unreleased ## hdf5-src unreleased - ## hdf5 unreleased + + +## hdf5 v0.11.0 +Release date: Nov 23, 2025 - Fixed incorrect retrieved name of attributes - Added support for Single Writer Multiple Readers (SWMR) (breaking change, OpenMode has extra variant) +- Added support for object copying between Locations +- Improved debug printing for datatype descriptor + +## hdf5-types v0.10.2 +Release date: Nov 23, 2025 +- Added documentation on Typedescriptor::size + +## hdf5-derive v0.9.3 +Release date: Nov 23, 2025 +- Fixed derive macro when inside itself or hdf5_metno package + +## hdf5-src v0.9.5 +Release date: Nov 23, 2025 +- Improved support for static linking with msvc -## hdf5 v0.10.2 +## hdf5 v0.10.2 (yanked) Release date: Oct 16, 2025 - Upgraded upper bound of ndarray diff --git a/hdf5-derive/Cargo.toml b/hdf5-derive/Cargo.toml index 48ea8860d..af6574f40 100644 --- a/hdf5-derive/Cargo.toml +++ b/hdf5-derive/Cargo.toml @@ -3,7 +3,7 @@ name = "hdf5-metno-derive" description = "Derive macro for HDF5 structs and enums." categories = ["development-tools::procedural-macro-helpers"] readme = "README.md" -version = "0.9.2" # !V +version = "0.9.3" # !V rust-version.workspace = true authors.workspace = true keywords.workspace = true diff --git a/hdf5-src/Cargo.toml b/hdf5-src/Cargo.toml index c863daef9..256c4f615 100644 --- a/hdf5-src/Cargo.toml +++ b/hdf5-src/Cargo.toml @@ -22,7 +22,7 @@ exclude = [ "ext/hdf5/HDF5Examples/**", "ext/hdf5/doxygen/**", ] -version = "0.9.4" # !V +version = "0.9.5" # !V rust-version.workspace = true authors.workspace = true keywords.workspace = true diff --git a/hdf5-types/Cargo.toml b/hdf5-types/Cargo.toml index 8d489835b..b4818ceb4 100644 --- a/hdf5-types/Cargo.toml +++ b/hdf5-types/Cargo.toml @@ -4,7 +4,7 @@ description = "Native Rust equivalents of HDF5 types." readme = "README.md" build = "build.rs" categories = ["encoding"] -version = "0.10.1" # !V +version = "0.10.2" # !V rust-version.workspace = true authors.workspace = true keywords.workspace = true diff --git a/hdf5/Cargo.toml b/hdf5/Cargo.toml index f46c56f12..76efb31cb 100644 --- a/hdf5/Cargo.toml +++ b/hdf5/Cargo.toml @@ -4,7 +4,7 @@ readme = "../README.md" description = "Thread-safe Rust bindings for the HDF5 library." build = "build.rs" categories = ["science", "filesystem"] -version = "0.10.2" # !V +version = "0.11.0" # !V rust-version.workspace = true authors.workspace = true keywords.workspace = true From fdc3287c65a8fdff7988af64bdcddddb0d546361 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Nov 2025 18:46:36 +0000 Subject: [PATCH 064/141] Bump actions/checkout from 5 to 6 Bumps [actions/checkout](https://github.com/actions/checkout) from 5 to 6. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 24 ++++++++++++------------ .github/workflows/release.yml | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 010987878..43b47d906 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Check spelling uses: crate-ci/typos@626c4bedb751ce0b7f03262ca97ddda9a076ae1c # v1.39.2 @@ -39,7 +39,7 @@ jobs: - {command: clippy, rust: stable} steps: - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Install Rust (${{matrix.rust}}) uses: dtolnay/rust-toolchain@stable with: {toolchain: '${{matrix.rust}}', components: 'rustfmt, clippy'} @@ -52,7 +52,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: {submodules: true} - name: Install Rust uses: dtolnay/rust-toolchain@stable @@ -74,7 +74,7 @@ jobs: - {version: hdf5-mpi, mpi: true} steps: - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: {submodules: true} - name: Install Rust (${{matrix.rust}}) uses: dtolnay/rust-toolchain@stable @@ -119,7 +119,7 @@ jobs: shell: bash -l {0} steps: - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: {submodules: true} - name: Install Rust (${{matrix.rust}}) uses: dtolnay/rust-toolchain@stable @@ -156,7 +156,7 @@ jobs: - {os: macos, rust: stable} steps: - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: {submodules: true} - name: Install Rust (${{matrix.rust}}) uses: dtolnay/rust-toolchain@stable @@ -186,7 +186,7 @@ jobs: - {mpi: openmpi, rust: stable} steps: - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: {submodules: true} - name: Install Rust (${{matrix.rust}}) uses: dtolnay/rust-toolchain@stable @@ -218,7 +218,7 @@ jobs: version: ["1.8", "1.10", "1.12", "1.14"] steps: - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: {submodules: true} - name: Install Rust (${{matrix.rust}}) uses: dtolnay/rust-toolchain@stable @@ -264,7 +264,7 @@ jobs: # rust: [stable] # steps: # - name: Checkout repository - # uses: actions/checkout@v5 + # uses: actions/checkout@v6 # with: {submodules: true} # - name: Install Rust (${{matrix.rust}}) # uses: dtolnay/rust-toolchain@stable @@ -288,7 +288,7 @@ jobs: fail-fast: false steps: - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: {submodules: true} - name: Install Rust uses: dtolnay/rust-toolchain@stable @@ -309,7 +309,7 @@ jobs: # runs-on: ubuntu-latest # steps: # - name: Checkout repository - # uses: actions/checkout@v5 + # uses: actions/checkout@v6 # with: {submodules: true} # - name: Install Rust # uses: dtolnay/rust-toolchain@stable @@ -326,7 +326,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: {submodules: true} - name: Install Rust uses: dtolnay/rust-toolchain@stable diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 61fc96761..0e0b13633 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: submodules: true - name: Install libhdf5 From 33e0f5ea3230825f5198e0144ff453ab6787e475 Mon Sep 17 00:00:00 2001 From: Frank Blubaugh Date: Wed, 26 Nov 2025 09:15:40 -0600 Subject: [PATCH 065/141] Initial ZFP filters in place with dcpl testing working. --- hdf5/Cargo.toml | 5 +- hdf5/src/hl/dataset.rs | 10 + hdf5/src/hl/filters.rs | 574 +++++++++++++++++++++++++++- hdf5/src/hl/filters/zfp.rs | 395 +++++++++++++++++++ hdf5/src/hl/plist/dataset_create.rs | 11 + 5 files changed, 993 insertions(+), 2 deletions(-) create mode 100644 hdf5/src/hl/filters/zfp.rs diff --git a/hdf5/Cargo.toml b/hdf5/Cargo.toml index 76efb31cb..d142ab1a1 100644 --- a/hdf5/Cargo.toml +++ b/hdf5/Cargo.toml @@ -39,6 +39,8 @@ mpio = ["dep:mpi-sys", "hdf5-sys/mpio"] complex = ["hdf5-types/complex"] # Enable float16 type support. f16 = ["hdf5-types/f16"] +# Enable ZFP Support +zfp = ["dep:zfp-sys"] # The features with version numbers such as 1.10.3, 1.12.0 are metafeatures # and is only available when the HDF5 library is at least this version. @@ -56,6 +58,7 @@ lzf-sys = { version = "0.1", optional = true } mpi-sys = { workspace = true, optional = true } ndarray = ">=0.15, <=0.17" paste = "1.0" +zfp-sys = {version= "0.4.2", optional= true } # internal hdf5-derive = { workspace = true } hdf5-sys = { workspace = true } @@ -73,4 +76,4 @@ scopeguard = "1.2" tempfile = "3.9" [package.metadata.docs.rs] -features = ["static", "zlib", "blosc", "lzf", "f16", "complex"] +features = ["static", "zlib", "blosc", "lzf", "f16", "complex","zfp"] diff --git a/hdf5/src/hl/dataset.rs b/hdf5/src/hl/dataset.rs index 24f3b160b..b2a54ae35 100644 --- a/hdf5/src/hl/dataset.rs +++ b/hdf5/src/hl/dataset.rs @@ -329,6 +329,11 @@ where } } + + + + + #[derive(Debug, Clone, PartialEq, Eq)] pub enum Chunk { Exact(Vec), // exact chunk shape @@ -713,6 +718,11 @@ impl DatasetBuilderInner { self.with_dcpl(|pl| pl.blosc_zstd(clevel, shuffle)); } + #[cfg(feature = "zfp")] + pub fn zfp_rate(&mut self, rate: f64) { + self.with_dcpl(|pl| pl.zfp_rate(rate)); + } + pub fn add_filter(&mut self, id: H5Z_filter_t, cdata: &[c_uint]) { self.with_dcpl(|pl| pl.add_filter(id, cdata)); } diff --git a/hdf5/src/hl/filters.rs b/hdf5/src/hl/filters.rs index a0056aa6e..bafae94eb 100644 --- a/hdf5/src/hl/filters.rs +++ b/hdf5/src/hl/filters.rs @@ -22,6 +22,8 @@ use crate::internal_prelude::*; mod blosc; #[cfg(feature = "lzf")] mod lzf; +#[cfg(feature = "zfp")] +mod zfp; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum SZip { @@ -103,6 +105,41 @@ mod blosc_impl { #[cfg(feature = "blosc")] pub use blosc_impl::*; + +#[cfg(feature = "zfp")] +mod zfp_impl { + #[derive(Clone, Copy, Debug)] + pub enum ZfpMode { + FixedRate(f64), + FixedPrecision(u8), + FixedAccuracy(f64), + } + + // Bitwise compare f64 so NaN and signed zero are deterministic + impl PartialEq for ZfpMode { + fn eq(&self, other: &Self) -> bool { + use ZfpMode::*; + match (self, other) { + (FixedRate(a), FixedRate(b)) => a.to_bits() == b.to_bits(), + (FixedPrecision(a), FixedPrecision(b)) => a == b, + (FixedAccuracy(a), FixedAccuracy(b)) => a.to_bits() == b.to_bits(), + _ => false, + } + } + } + impl Eq for ZfpMode {} + + impl Default for ZfpMode { + fn default() -> Self { + ZfpMode::FixedRate(1.0) + } + } +} + + +#[cfg(feature = "zfp")] +pub use zfp_impl::*; + #[derive(Clone, Debug, PartialEq, Eq)] pub enum Filter { Deflate(u8), @@ -115,6 +152,8 @@ pub enum Filter { LZF, #[cfg(feature = "blosc")] Blosc(Blosc, u8, BloscShuffle), + #[cfg(feature = "zfp")] + Zfp(ZfpMode), User(H5Z_filter_t, Vec), } @@ -135,6 +174,10 @@ pub(crate) fn register_filters() { if let Err(e) = blosc::register_blosc() { eprintln!("Error while registering Blosc filter: {e}"); } + #[cfg(feature = "zfp")] + if let Err(e) = zfp::register_zfp() { + eprintln!("Error while registering ZFP filter: {e}"); + } } /// Returns `true` if deflate filter is available. @@ -164,6 +207,12 @@ pub fn blosc_available() -> bool { h5lock!(H5Zfilter_avail(32001) == 1) } +/// Returns `true` if ZFP filter is available. +#[cfg(feature = "zfp")] +pub fn zfp_available() -> bool { + h5lock!(H5Zfilter_avail(32013) == 1) +} + impl Filter { pub fn id(&self) -> H5Z_filter_t { match self { @@ -177,6 +226,8 @@ impl Filter { Self::LZF => lzf::LZF_FILTER_ID, #[cfg(feature = "blosc")] Self::Blosc(_, _, _) => blosc::BLOSC_FILTER_ID, + #[cfg(feature = "zfp")] + Self::Zfp(_) => zfp::ZFP_FILTER_ID, Self::User(id, _) => *id, } } @@ -291,6 +342,26 @@ impl Filter { Self::blosc(Blosc::ZStd, clevel, shuffle) } + #[cfg(feature = "zfp")] + pub fn zfp(mode: ZfpMode) -> Self { + Self::Zfp(mode) + } + + #[cfg(feature = "zfp")] + pub fn zfp_rate(rate: f64) -> Self { + Self::zfp(ZfpMode::FixedRate(rate)) + } + + #[cfg(feature = "zfp")] + pub fn zfp_precision(precision: u8) -> Self { + Self::zfp(ZfpMode::FixedPrecision(precision)) + } + + #[cfg(feature = "zfp")] + pub fn zfp_accuracy(accuracy: f64) -> Self { + Self::zfp(ZfpMode::FixedAccuracy(accuracy)) + } + pub fn user(id: H5Z_filter_t, cdata: &[c_uint]) -> Self { Self::User(id, cdata.to_vec()) } @@ -397,6 +468,28 @@ impl Filter { Ok(Self::blosc(complib, clevel, shuffle)) } + #[cfg(feature = "zfp")] + fn parse_zfp(cdata: &[c_uint]) -> Result { + ensure!(cdata.len() >= 8, "expected at least length 8 cdata for zfp filter"); + let mode = if cdata.len() >= 8 { cdata[7] } else { 1 }; + let param1 = if cdata.len() >= 9 { cdata[8] } else { 0 }; + let param2 = if cdata.len() >= 10 { cdata[9] } else { 0 }; + + let zfp_mode = match mode { + 1 => { + let rate = f64::from_bits(((param1 as u64) << 32) | (param2 as u64)); + ZfpMode::FixedRate(rate) + } + 2 => ZfpMode::FixedPrecision(param1 as u8), + 3 => { + let accuracy = f64::from_bits(((param1 as u64) << 32) | (param2 as u64)); + ZfpMode::FixedAccuracy(accuracy) + } + _ => fail!("invalid zfp mode: {}", mode), + }; + Ok(Self::zfp(zfp_mode)) + } + pub fn from_raw(filter_id: H5Z_filter_t, cdata: &[c_uint]) -> Result { ensure!(filter_id > 0, "invalid filter id: {}", filter_id); match filter_id { @@ -410,6 +503,8 @@ impl Filter { lzf::LZF_FILTER_ID => Self::parse_lzf(cdata), #[cfg(feature = "blosc")] blosc::BLOSC_FILTER_ID => Self::parse_blosc(cdata), + #[cfg(feature = "zfp")] + zfp::ZFP_FILTER_ID => Self::parse_zfp(cdata), _ => Ok(Self::user(filter_id, cdata)), } } @@ -478,6 +573,26 @@ impl Filter { Self::apply_user(plist_id, blosc::BLOSC_FILTER_ID, &cdata) } + #[cfg(feature = "zfp")] + unsafe fn apply_zfp(plist_id: hid_t, mode: ZfpMode) -> herr_t { + let mut cdata: Vec = vec![0; 10]; + let (mode_val, param1, param2) = match mode { + ZfpMode::FixedRate(rate) => { + let bits = rate.to_bits(); + (1, (bits >> 32) as c_uint, bits as c_uint) + } + ZfpMode::FixedPrecision(precision) => (2, precision as c_uint, 0), + ZfpMode::FixedAccuracy(accuracy) => { + let bits = accuracy.to_bits(); + (3, (bits >> 32) as c_uint, bits as c_uint) + } + }; + cdata[7] = mode_val; + cdata[8] = param1; + cdata[9] = param2; + Self::apply_user(plist_id, zfp::ZFP_FILTER_ID, &cdata) + } + unsafe fn apply_user(plist_id: hid_t, filter_id: H5Z_filter_t, cdata: &[c_uint]) -> herr_t { // We're setting custom filters to optional, same way h5py does it, since // the only mention of H5Z_FLAG_MANDATORY in the HDF5 source itself is @@ -502,6 +617,8 @@ impl Filter { Self::Blosc(complib, clevel, shuffle) => { Self::apply_blosc(id, *complib, *clevel, *shuffle) } + #[cfg(feature = "zfp")] + Self::Zfp(mode) => Self::apply_zfp(id, *mode), Self::User(filter_id, ref cdata) => Self::apply_user(id, *filter_id, cdata), }); Ok(()) @@ -535,7 +652,7 @@ impl Filter { } } -const COMP_FILTER_IDS: &[H5Z_filter_t] = &[H5Z_FILTER_DEFLATE, H5Z_FILTER_SZIP, 32000, 32001]; +const COMP_FILTER_IDS: &[H5Z_filter_t] = &[H5Z_FILTER_DEFLATE, H5Z_FILTER_SZIP, 32000, 32001, 32013]; pub(crate) fn validate_filters(filters: &[Filter], type_class: H5T_class_t) -> Result<()> { let mut map: HashMap = HashMap::new(); @@ -587,6 +704,8 @@ pub(crate) fn validate_filters(filters: &[Filter], type_class: H5T_class_t) -> R #[cfg(test)] mod tests { + use std::hint::assert_unchecked; + use ndarray::Array2; use hdf5_sys::h5t::H5T_class_t; use super::{ @@ -595,6 +714,7 @@ mod tests { }; use crate::test::with_tmp_file; use crate::{plist::DatasetCreate, Result}; + use crate::hl::filters::zfp_available; #[test] fn test_filter_pipeline() -> Result<()> { @@ -621,6 +741,17 @@ mod tests { comp_filters.push(Filter::blosc_zstd(9, BloscShuffle::Byte)); comp_filters.push(Filter::blosc_snappy(0, BloscShuffle::Bit)); } + + #[cfg(feature="zfp")] + assert_eq!(cfg!(feature = "zfp"), zfp_available()); + #[cfg(feature="zfp")] + { + comp_filters.push(Filter::zfp_rate(8.0)); + comp_filters.push(Filter::zfp_precision(16)); + comp_filters.push(Filter::zfp_accuracy(1e-3)); + } + + for c in &comp_filters { assert!(c.is_available()); assert!(c.encode_enabled()); @@ -672,4 +803,445 @@ mod tests { Ok(()) } + + #[test] + #[cfg(feature = "zfp")] + fn test_zfp_filter() -> Result<()> { + use super::{zfp_available, ZfpMode}; + + assert_eq!(cfg!(feature = "zfp"), zfp_available()); + + if !zfp_available() { + println!("ZFP filter not available, skipping test"); + return Ok(()); + } + + // Test different ZFP modes + let zfp_filters = vec![ + Filter::zfp_rate(8.0), + Filter::zfp_rate(16.0), + Filter::zfp_rate(4.0), + Filter::zfp_precision(16), + Filter::zfp_precision(32), + Filter::zfp_accuracy(1e-3), + Filter::zfp_accuracy(1e-6), + Filter::zfp(ZfpMode::FixedRate(12.5)), + ]; + + for flt in &zfp_filters { + println!("Testing filter: {:?}", flt); + assert!(flt.is_available()); + assert!(flt.encode_enabled()); + assert!(flt.decode_enabled()); + + // Test with float type (ZFP only supports floats) + let pipeline = vec![flt.clone()]; + validate_filters(&pipeline, H5T_class_t::H5T_FLOAT)?; + + let plist = DatasetCreate::try_new()?; + for f in &pipeline { + f.apply_to_plist(plist.id())?; + } + assert_eq!(Filter::extract_pipeline(plist.id())?, pipeline); + + // Test with actual dataset creation for f32 + let res = with_tmp_file(|file| { + + file.new_dataset_builder() + .empty::() + .shape((100,50)) + .chunk((10,10)) + .with_dcpl(|p| p.set_filters(&pipeline)) + .create("zfp_f32") + .unwrap(); + + let plist = file.dataset("zfp_f32").unwrap().dcpl().unwrap(); + Filter::extract_pipeline(plist.id()).unwrap() + }); + assert_eq!(res, pipeline); + + // Test with f64 + let res_f64 = with_tmp_file(|file| { + file.new_dataset_builder() + .empty::() + .shape((100, 50)) + .chunk((10, 10)) + .with_dcpl(|p| p.set_filters(&pipeline)) + .create("zfp_f64") + .unwrap(); + + let plist = file.dataset("zfp_f64").unwrap().dcpl().unwrap(); + Filter::extract_pipeline(plist.id()).unwrap() + }); + assert_eq!(res_f64, pipeline); + } + + Ok(()) + } + + #[test] + #[cfg(feature = "zfp")] + fn test_zfp_roundtrip_1d() -> Result<()> { + use super::zfp_available; + + if !zfp_available() { + println!("ZFP filter not available, skipping test"); + return Ok(()); + } + let pipeline = vec![Filter::zfp_rate(16.0)]; + with_tmp_file(|file| { + let data = ndarray::Array1::::linspace(0.0, 1.0, 1000); + file.new_dataset_builder() + .with_data(&data) + .chunk((100,)) + .with_dcpl(|p| p.set_filters(&pipeline)) + .create("zfp_1d_dcpl") + .unwrap(); + + + let ds = file.dataset("zfp_1d_dcpl").unwrap(); + let read_data: Vec = ds.read_raw().unwrap(); + + // ZFP is lossy, so we check approximate equality + assert_eq!(read_data.len(), data.len()); + for (i, (original, compressed)) in data.iter().zip(read_data.iter()).enumerate() { + let diff = (original - compressed).abs(); + assert!(diff < 0.1, "Index {}: difference too large: {} vs {} (diff: {})", + i, original, compressed, diff); + } + }); + + Ok(()) + } + + #[test] + #[cfg(feature = "zfp")] + fn test_zfp_roundtrip_2d() -> Result<()> { + use super::zfp_available; + + if !zfp_available() { + println!("ZFP filter not available, skipping test"); + return Ok(()); + } + + with_tmp_file(|file| { + let data: Vec = (0..1000).map(|i| (i as f64) * 0.01).collect(); + let data = Array2::from_shape_vec((10,100), data).unwrap(); + + + + let pipteline = vec![Filter::zfp_precision(32)]; + file.new_dataset_builder() + .with_data(&data) + .chunk((10,10)) + .with_dcpl(|p| p.set_filters(&pipteline)) + .create("zfp_2d") + .unwrap(); + + let ds = file.dataset("zfp_2d").unwrap(); + let read_data: Vec = ds.read_raw().unwrap(); + + assert_eq!(read_data.len(), data.len()); + for (original, compressed) in data.iter().zip(read_data.iter()) { + let diff = (original - compressed).abs(); + assert!(diff < 0.01, "Difference too large: {} vs {}", original, compressed); + } + }); + + Ok(()) + } + + #[test] + #[cfg(feature = "zfp")] + fn test_zfp_roundtrip_3d() -> Result<()> { + use super::zfp_available; + + if !zfp_available() { + println!("ZFP filter not available, skipping test"); + return Ok(()); + } + let pipeline = vec![Filter::zfp_accuracy(1e-4)]; + with_tmp_file(|file| { + let data: Vec = (0..1000).map(|i| (i as f32) * 0.001).collect(); + let data = ndarray::Array3::from_shape_vec((10, 10, 10), data).unwrap(); + + file.new_dataset_builder() + .with_data(&data) + .chunk((5, 5, 5)) + .with_dcpl(|p| p.set_filters(&pipeline)) + .create("zfp_3d") + .unwrap(); + + + + + let ds = file.dataset("zfp_3d").unwrap(); + let read_data: Vec = ds.read_raw().unwrap(); + + assert_eq!(read_data.len(), data.len()); + }); + + Ok(()) + } + + #[test] + #[cfg(feature = "zfp")] + fn test_zfp_roundtrip_4d() -> Result<()> { + use super::zfp_available; + + if !zfp_available() { + println!("ZFP filter not available, skipping test"); + return Ok(()); + } + let pipeline = vec![Filter::zfp_rate(10.0)]; + with_tmp_file(|file| { + let data: Vec = (0..256).map(|i| (i as f64) * 0.1).collect(); + let data = ndarray::Array4::from_shape_vec((4, 4, 4, 4), data).unwrap(); + + file.new_dataset_builder() + .with_data(&data) + .chunk((2, 2, 2, 2)) + .with_dcpl(|p| p.set_filters(&pipeline)) + .create("zfp_4d") + .unwrap(); + + + let ds = file.dataset("zfp_4d").unwrap(); + let read_data: Vec = ds.read_raw().unwrap(); + + assert_eq!(read_data.len(), data.len()); + }); + + Ok(()) + } + + #[test] + #[cfg(feature = "zfp")] + fn test_zfp_mode_parsing() -> Result<()> { + use super::ZfpMode; + + // Test FixedRate parsing + let rate_bits = 12.5_f64.to_bits(); + let cdata_rate = vec![ + 0, 1, 4, 100, 0, 0, 0, + 1, // mode = rate + (rate_bits >> 32) as u32, + rate_bits as u32, + ]; + let filter = Filter::from_raw(super::zfp::ZFP_FILTER_ID, &cdata_rate)?; + if let Filter::Zfp(ZfpMode::FixedRate(rate)) = filter { + assert!((rate - 12.5).abs() < 1e-10); + } else { + panic!("Expected FixedRate mode, got: {:?}", filter); + } + + // Test FixedPrecision parsing + let cdata_precision = vec![ + 0, 1, 8, 100, 0, 0, 0, + 2, // mode = precision + 24, // precision + 0, + ]; + let filter = Filter::from_raw(super::zfp::ZFP_FILTER_ID, &cdata_precision)?; + if let Filter::Zfp(ZfpMode::FixedPrecision(precision)) = filter { + assert_eq!(precision, 24); + } else { + panic!("Expected FixedPrecision mode, got: {:?}", filter); + } + + // Test FixedAccuracy parsing + let accuracy_bits = 1e-5_f64.to_bits(); + let cdata_accuracy = vec![ + 0, 1, 8, 100, 0, 0, 0, + 3, // mode = accuracy + (accuracy_bits >> 32) as u32, + accuracy_bits as u32, + ]; + let filter = Filter::from_raw(super::zfp::ZFP_FILTER_ID, &cdata_accuracy)?; + if let Filter::Zfp(ZfpMode::FixedAccuracy(accuracy)) = filter { + assert!((accuracy - 1e-5).abs() < 1e-10); + } else { + panic!("Expected FixedAccuracy mode, got: {:?}", filter); + } + + Ok(()) + } + + #[test] + #[cfg(feature = "zfp")] + fn test_zfp_with_other_filters() -> Result<()> { + use super::zfp_available; + + if !zfp_available() { + println!("ZFP filter not available, skipping test"); + return Ok(()); + } + + + + let pipeline = vec![Filter::zfp_rate(8.0)]; + // Test ZFP combined with shuffle (shuffle should come first) + with_tmp_file(|file| { + let data: Vec = (0..1000).map(|i| (i as f32) * 0.1).collect(); + let data = ndarray::Array1::from_shape_vec(1000, data).unwrap(); + file.new_dataset_builder() + .with_data(&data) + .chunk(100) + .with_dcpl(|p| p.set_filters(&pipeline)) + .create("zfp_rate_8").unwrap(); + + + let ds = file.dataset("zfp_rate_8").unwrap(); + let read_data: Vec = ds.read_raw().unwrap(); + + let error = data.iter() + .zip(read_data.iter()) + .map(|(a, b)| (a - b).abs()) + .sum::() / data.len() as f32; + assert_eq!(error,0.082505114); + assert_eq!(read_data.len(), data.len()); + }); + + // Test ZFP with fletcher32 checksum + if super::deflate_available() { + let pipeline = vec![Filter::zfp_precision(24), Filter::fletcher32()]; + with_tmp_file(|file| { + let data: Vec = (0..500).map(|i| (i as f64) * 0.01).collect(); + let data = ndarray::Array1::from_shape_vec(500, data).unwrap(); + + file.new_dataset_builder() + .with_data(&data) + .chunk(50) + .with_dcpl(|p| p.set_filters(&pipeline)) + .create("zfp_with_fletcher32") + .unwrap(); + + + let ds = file.dataset("zfp_with_fletcher32").unwrap(); + let read_data: Vec = ds.read_raw().unwrap(); + + assert_eq!(read_data.len(), data.len()); + }); + } + + Ok(()) + } + + #[test] + #[cfg(feature = "zfp")] + fn test_zfp_compression_ratios() -> Result<()> { + use super::zfp_available; + + if !zfp_available() { + println!("ZFP filter not available, skipping test"); + return Ok(()); + } + + let pipeline = vec![Filter::zfp_rate(32.0)]; + let pipeline2 = vec![Filter::zfp_rate(4.0)]; + + with_tmp_file(|file| { + let data: Vec = (0..10000).map(|i| (i as f32).sin()).collect(); + let data = ndarray::Array1::from_shape_vec(10000, data).unwrap(); + // Higher rate = less compression but better quality + file.new_dataset_builder() + .with_data(&data) + .chunk(1000) + .with_dcpl(|p| p.set_filters(&pipeline)) + .create("zfp_high_rate") + .unwrap(); + + // Lower rate = more compression but lower quality + + file.new_dataset_builder() + .with_data(&data) + .chunk(1000) + .with_dcpl(|p| p.set_filters(&pipeline2)) + .create("zfp_low_rate") + .unwrap(); + + + let ds_high = file.dataset("zfp_high_rate").unwrap(); + let ds_low = file.dataset("zfp_low_rate").unwrap(); + + let read_high: Vec = ds_high.read_raw().unwrap(); + let read_low: Vec = ds_low.read_raw().unwrap(); + + // High rate should have better accuracy + let error_high: f32 = data.iter() + .zip(read_high.iter()) + .map(|(a, b)| (a - b).abs()) + .sum::() / data.len() as f32; + + let error_low: f32 = data.iter() + .zip(read_low.iter()) + .map(|(a, b)| (a - b).abs()) + .sum::() / data.len() as f32; + + println!("High rate error: {}, Low rate error: {}", error_high, error_low); + assert!(error_high < error_low || error_high < 0.001); + }); + + Ok(()) + } + + #[test] + #[cfg(feature = "zfp")] + fn test_zfp_edge_cases() -> Result<()> { + use super::zfp_available; + + if !zfp_available() { + println!("ZFP filter not available, skipping test"); + return Ok(()); + } + + let pipeline = vec![Filter::zfp_rate(8.0)]; + + // Test with all zeros + with_tmp_file(|file| { + let data: Vec = vec![0.0; 1000]; + let data = ndarray::Array1::from_shape_vec(1000, data).unwrap(); + + file.new_dataset_builder() + .with_data(&data) + .chunk(100) + .with_dcpl(|p| p.set_filters(&pipeline)) + .create("zfp_zeros") + .unwrap(); + + + let ds = file.dataset("zfp_zeros").unwrap(); + let read_data: Vec = ds.read_raw().unwrap(); + + assert_eq!(read_data.len(), data.len()); + for &val in &read_data { + assert_eq!(val, 0.0); + } + }); + + + let pipeline = vec![Filter::zfp_accuracy(1e-6)]; + // Test with constant values + with_tmp_file(|file| { + let data: Vec = vec![42.0; 500]; + let data = ndarray::Array1::from_shape_vec(500, data).unwrap(); + file.new_dataset_builder() + .with_data(&data) + .chunk(50) + .with_dcpl(|p| p.set_filters(&pipeline)) + .create("zfp_constant") + .unwrap(); + + + let ds = file.dataset("zfp_constant").unwrap(); + let read_data: Vec = ds.read_raw().unwrap(); + + assert_eq!(read_data.len(), data.len()); + for &val in &read_data { + assert!((val - 42.0).abs() < 1e-5); + } + }); + + Ok(()) + } } diff --git a/hdf5/src/hl/filters/zfp.rs b/hdf5/src/hl/filters/zfp.rs new file mode 100644 index 000000000..abdb2e16a --- /dev/null +++ b/hdf5/src/hl/filters/zfp.rs @@ -0,0 +1,395 @@ +use std::ptr::{self, addr_of_mut}; +use std::slice; +use std::sync::LazyLock; + +use hdf5_sys::h5p::{H5Pget_chunk, H5Pget_filter_by_id2, H5Pmodify_filter}; +use hdf5_sys::h5t::{H5Tclose, H5Tget_class, H5Tget_size, H5Tget_super, H5T_FLOAT}; +use hdf5_sys::h5z::{H5Z_class2_t, H5Z_filter_t, H5Zregister, H5Z_CLASS_T_VERS, H5Z_FLAG_REVERSE}; + +use crate::error::H5ErrorCode; +use crate::globals::{H5E_CALLBACK, H5E_PLIST}; +use crate::internal_prelude::*; + +pub use zfp_sys::{ + zfp_compress, zfp_decompress, zfp_field_1d, zfp_field_2d, zfp_field_3d, zfp_field_4d, + zfp_field_free, zfp_stream_close, zfp_stream_maximum_size, zfp_stream_open, + zfp_stream_rewind, zfp_stream_set_accuracy, zfp_stream_set_bit_stream, + zfp_stream_set_precision, zfp_stream_set_rate, zfp_type_zfp_type_double, + zfp_type_zfp_type_float, stream_close, stream_open, +}; + +const ZFP_FILTER_NAME: &[u8] = b"zfp\0"; +pub const ZFP_FILTER_ID: H5Z_filter_t = 32013; +const ZFP_FILTER_VERSION: c_uint = 1; + +// ZFP mode constants +const ZFP_MODE_RATE: c_uint = 1; +const ZFP_MODE_PRECISION: c_uint = 2; +const ZFP_MODE_ACCURACY: c_uint = 3; + +const ZFP_FILTER_INFO: &H5Z_class2_t = &H5Z_class2_t { + version: H5Z_CLASS_T_VERS as _, + id: ZFP_FILTER_ID, + encoder_present: 1, + decoder_present: 1, + name: ZFP_FILTER_NAME.as_ptr().cast(), + can_apply: Some(can_apply_zfp), + set_local: Some(set_local_zfp), + filter: Some(filter_zfp), +}; + +static ZFP_INIT: LazyLock> = LazyLock::new(|| { + let ret = unsafe { H5Zregister((ZFP_FILTER_INFO as *const H5Z_class2_t).cast()) }; + if H5ErrorCode::is_err_code(ret) { + return Err("Can't register ZFP filter"); + } + Ok(()) +}); + +pub fn register_zfp() -> Result<(), &'static str> { + *ZFP_INIT +} + +extern "C" fn can_apply_zfp(_dcpl_id: hid_t, type_id: hid_t, _space_id: hid_t) -> i32 { + let type_class = unsafe { H5Tget_class(type_id) }; + if type_class == H5T_FLOAT { + 1 + } else { + 0 + } +} + +extern "C" fn set_local_zfp(dcpl_id: hid_t, type_id: hid_t, _space_id: hid_t) -> herr_t { + const MAX_NDIMS: usize = 4; + let mut flags: c_uint = 0; + let mut nelmts: size_t = 10; + let mut values: Vec = vec![0; 10]; + let ret = unsafe { + H5Pget_filter_by_id2( + dcpl_id, + ZFP_FILTER_ID, + addr_of_mut!(flags), + addr_of_mut!(nelmts), + values.as_mut_ptr(), + 0, + ptr::null_mut(), + ptr::null_mut(), + ) + }; + if ret < 0 { + return -1; + } + nelmts = nelmts.max(10); + values[0] = ZFP_FILTER_VERSION; + + let mut chunkdims: Vec = vec![0; MAX_NDIMS]; + let ndims: c_int = unsafe { H5Pget_chunk(dcpl_id, MAX_NDIMS as _, chunkdims.as_mut_ptr()) }; + if ndims < 0 { + return -1; + } + if ndims > MAX_NDIMS as _ { + h5err!("ZFP supports up to 4 dimensions", H5E_PLIST, H5E_CALLBACK); + return -1; + } + + let typesize: size_t = unsafe { H5Tget_size(type_id) }; + if typesize == 0 { + return -1; + } + + values[1] = ndims as c_uint; + values[2] = typesize as c_uint; + for i in 0..ndims as usize { + if i + 3 < values.len() { + values[i + 3] = chunkdims[i] as c_uint; + } + } + + let r = unsafe { H5Pmodify_filter(dcpl_id, ZFP_FILTER_ID, flags, nelmts, values.as_ptr()) }; + if r < 0 { + -1 + } else { + 1 + } +} + +#[derive(Debug)] +struct ZfpConfig { + pub ndims: c_int, + pub typesize: size_t, + pub dims: [size_t; 4], + pub mode: c_uint, + pub rate: f64, + pub precision: u32, + pub accuracy: f64, +} + +fn parse_zfp_cdata(cd_nelmts: size_t, cd_values: *const c_uint) -> Option { + let cdata = unsafe { slice::from_raw_parts(cd_values, cd_nelmts as _) }; + if cdata.len() < 7 { + h5err!("Invalid ZFP filter configuration", H5E_PLIST, H5E_CALLBACK); + return None; + } + + let ndims = cdata[1] as c_int; + let typesize = cdata[2] as size_t; + let mut dims = [0; 4]; + for i in 0..(ndims as usize).min(4) { + dims[i] = cdata[3 + i] as size_t; + } + + let mode = if cdata.len() > 7 { cdata[7] } else { ZFP_MODE_RATE }; + let param1 = if cdata.len() > 8 { cdata[8] } else { 0 }; + let param2 = if cdata.len() > 9 { cdata[9] } else { 0 }; + + let (rate, precision, accuracy) = match mode { + ZFP_MODE_RATE => { + let rate = f64::from_bits(((param1 as u64) << 32) | (param2 as u64)); + (rate, 0, 0.0) + } + ZFP_MODE_PRECISION => (0.0, param1, 0.0), + ZFP_MODE_ACCURACY => { + let accuracy = f64::from_bits(((param1 as u64) << 32) | (param2 as u64)); + (0.0, 0, accuracy) + } + _ => { + h5err!("Invalid ZFP mode", H5E_PLIST, H5E_CALLBACK); + return None; + } + }; + + Some(ZfpConfig { ndims, typesize, dims, mode, rate, precision, accuracy }) +} + +unsafe extern "C" fn filter_zfp( + flags: c_uint, cd_nelmts: size_t, cd_values: *const c_uint, nbytes: size_t, + buf_size: *mut size_t, buf: *mut *mut c_void, +) -> size_t { + let cfg = if let Some(cfg) = parse_zfp_cdata(cd_nelmts, cd_values) { + cfg + } else { + return 0; + }; + if flags & H5Z_FLAG_REVERSE == 0 { + unsafe { filter_zfp_compress(&cfg, nbytes, buf_size, buf) } + } else { + unsafe { filter_zfp_decompress(&cfg, nbytes, buf_size, buf) } + } +} + +unsafe fn filter_zfp_compress( + cfg: &ZfpConfig, nbytes: size_t, buf_size: *mut size_t, buf: *mut *mut c_void, +) -> size_t { + let zfp = zfp_stream_open(ptr::null_mut()); + if zfp.is_null() { + h5err!("Failed to open ZFP stream", H5E_PLIST, H5E_CALLBACK); + return 0; + } + + match cfg.mode { + ZFP_MODE_RATE => { + zfp_stream_set_rate(zfp, cfg.rate, cfg.typesize as _, cfg.ndims as _, 0); + } + ZFP_MODE_PRECISION => { + zfp_stream_set_precision(zfp, cfg.precision); + } + ZFP_MODE_ACCURACY => { + zfp_stream_set_accuracy(zfp, cfg.accuracy); + } + _ => { + zfp_stream_close(zfp); + return 0; + } + } + + let field = if cfg.typesize == 4 { + match cfg.ndims { + 1 => zfp_field_1d((*buf).cast(), zfp_type_zfp_type_float, cfg.dims[0]), + 2 => zfp_field_2d((*buf).cast(), zfp_type_zfp_type_float, cfg.dims[0], cfg.dims[1]), + 3 => zfp_field_3d( + (*buf).cast(), + zfp_type_zfp_type_float, + cfg.dims[0], + cfg.dims[1], + cfg.dims[2], + ), + 4 => zfp_field_4d( + (*buf).cast(), + zfp_type_zfp_type_float, + cfg.dims[0], + cfg.dims[1], + cfg.dims[2], + cfg.dims[3], + ), + _ => ptr::null_mut(), + } + } else { + match cfg.ndims { + 1 => zfp_field_1d((*buf).cast(), zfp_type_zfp_type_double, cfg.dims[0]), + 2 => zfp_field_2d((*buf).cast(), zfp_type_zfp_type_double, cfg.dims[0], cfg.dims[1]), + 3 => zfp_field_3d( + (*buf).cast(), + zfp_type_zfp_type_double, + cfg.dims[0], + cfg.dims[1], + cfg.dims[2], + ), + 4 => zfp_field_4d( + (*buf).cast(), + zfp_type_zfp_type_double, + cfg.dims[0], + cfg.dims[1], + cfg.dims[2], + cfg.dims[3], + ), + _ => ptr::null_mut(), + } + }; + + if field.is_null() { + zfp_stream_close(zfp); + h5err!("Failed to create ZFP field", H5E_PLIST, H5E_CALLBACK); + return 0; + } + + let maxsize = zfp_stream_maximum_size(zfp, field); + let outbuf = libc::malloc(maxsize); + if outbuf.is_null() { + zfp_field_free(field); + zfp_stream_close(zfp); + h5err!("Can't allocate compression buffer", H5E_PLIST, H5E_CALLBACK); + return 0; + } + + let bitstream = stream_open(outbuf.cast(), maxsize); + zfp_stream_set_bit_stream(zfp, bitstream); + zfp_stream_rewind(zfp); + + let compressed_size = zfp_compress(zfp, field); + + stream_close(bitstream); + zfp_field_free(field); + zfp_stream_close(zfp); + + if compressed_size == 0 { + libc::free(outbuf); + h5err!("ZFP compression failed", H5E_PLIST, H5E_CALLBACK); + return 0; + } + + libc::free(*buf); + *buf = outbuf; + *buf_size = compressed_size; + compressed_size +} + +unsafe fn filter_zfp_decompress( + cfg: &ZfpConfig, nbytes: size_t, buf_size: *mut size_t, buf: *mut *mut c_void, +) -> size_t { + let zfp = zfp_stream_open(ptr::null_mut()); + if zfp.is_null() { + h5err!("Failed to open ZFP stream", H5E_PLIST, H5E_CALLBACK); + return 0; + } + + match cfg.mode { + ZFP_MODE_RATE => { + zfp_stream_set_rate(zfp, cfg.rate, cfg.typesize as _, cfg.ndims as _, 0); + } + ZFP_MODE_PRECISION => { + zfp_stream_set_precision(zfp, cfg.precision); + } + ZFP_MODE_ACCURACY => { + zfp_stream_set_accuracy(zfp, cfg.accuracy); + } + _ => { + zfp_stream_close(zfp); + return 0; + } + } + + let mut outbuf_size = cfg.typesize; + for i in 0..cfg.ndims as usize { + outbuf_size *= cfg.dims[i]; + } + + let outbuf = libc::malloc(outbuf_size); + if outbuf.is_null() { + zfp_stream_close(zfp); + h5err!("Can't allocate decompression buffer", H5E_PLIST, H5E_CALLBACK); + return 0; + } + + let field = if cfg.typesize == 4 { + match cfg.ndims { + 1 => zfp_field_1d(outbuf.cast(), zfp_type_zfp_type_float, cfg.dims[0]), + 2 => zfp_field_2d(outbuf.cast(), zfp_type_zfp_type_float, cfg.dims[0], cfg.dims[1]), + 3 => zfp_field_3d( + outbuf.cast(), + zfp_type_zfp_type_float, + cfg.dims[0], + cfg.dims[1], + cfg.dims[2], + ), + 4 => zfp_field_4d( + outbuf.cast(), + zfp_type_zfp_type_float, + cfg.dims[0], + cfg.dims[1], + cfg.dims[2], + cfg.dims[3], + ), + _ => ptr::null_mut(), + } + } else { + match cfg.ndims { + 1 => zfp_field_1d(outbuf.cast(), zfp_type_zfp_type_double, cfg.dims[0]), + 2 => zfp_field_2d(outbuf.cast(), zfp_type_zfp_type_double, cfg.dims[0], cfg.dims[1]), + 3 => zfp_field_3d( + outbuf.cast(), + zfp_type_zfp_type_double, + cfg.dims[0], + cfg.dims[1], + cfg.dims[2], + ), + 4 => zfp_field_4d( + outbuf.cast(), + zfp_type_zfp_type_double, + cfg.dims[0], + cfg.dims[1], + cfg.dims[2], + cfg.dims[3], + ), + _ => ptr::null_mut(), + } + }; + + if field.is_null() { + libc::free(outbuf); + zfp_stream_close(zfp); + h5err!("Failed to create ZFP field", H5E_PLIST, H5E_CALLBACK); + return 0; + } + + let bitstream = stream_open((*buf).cast(), nbytes); + zfp_stream_set_bit_stream(zfp, bitstream); + zfp_stream_rewind(zfp); + + let status = zfp_decompress(zfp, field); + + stream_close(bitstream); + zfp_field_free(field); + zfp_stream_close(zfp); + + if status == 0 { + libc::free(outbuf); + h5err!("ZFP decompression failed", H5E_PLIST, H5E_CALLBACK); + return 0; + } + + libc::free(*buf); + *buf = outbuf; + *buf_size = outbuf_size; + outbuf_size +} + diff --git a/hdf5/src/hl/plist/dataset_create.rs b/hdf5/src/hl/plist/dataset_create.rs index 0661c87f5..942d9ff4c 100644 --- a/hdf5/src/hl/plist/dataset_create.rs +++ b/hdf5/src/hl/plist/dataset_create.rs @@ -30,6 +30,10 @@ use hdf5_sys::{ use hdf5_types::{OwnedDynValue, TypeDescriptor}; use crate::dim::Dimension; + +#[cfg(feature="zfp")] +use crate::filters::ZfpMode; + use crate::globals::H5P_DATASET_CREATE; use crate::hl::datatype::Datatype; use crate::hl::filters::{validate_filters, Filter, SZip, ScaleOffset}; @@ -475,6 +479,13 @@ impl DatasetCreateBuilder { self } + #[cfg(feature = "zfp")] + pub fn zfp_rate(&mut self, rate: f64) -> &mut Self { + let mode = ZfpMode::FixedRate(rate); + self.filters.push(Filter::zfp(mode)); + self + } + pub fn add_filter(&mut self, id: H5Z_filter_t, cdata: &[c_uint]) -> &mut Self { self.filters.push(Filter::user(id, cdata)); self From e7e3492e8855d451b61067196f1b3daef9201325 Mon Sep 17 00:00:00 2001 From: Frank Blubaugh Date: Wed, 26 Nov 2025 11:45:30 -0600 Subject: [PATCH 066/141] zfp interface methods are implemented. lossless isn't quite right --- hdf5/src/hl/dataset.rs | 44 +++++++++++++++++++-- hdf5/src/hl/filters.rs | 60 ++++++++++++++++++++++++++++- hdf5/src/hl/plist/dataset_create.rs | 6 --- 3 files changed, 99 insertions(+), 11 deletions(-) diff --git a/hdf5/src/hl/dataset.rs b/hdf5/src/hl/dataset.rs index b2a54ae35..4d40e38b6 100644 --- a/hdf5/src/hl/dataset.rs +++ b/hdf5/src/hl/dataset.rs @@ -245,6 +245,43 @@ impl DatasetBuilder { conv: Conversion::Soft, } } + + #[cfg(feature = "zfp")] + pub fn zfp_rate(self, rate: f64) -> Self { + let new_ds = self.with_dcpl(|p| { + p.set_filters(&vec![Filter::zfp_rate(rate)]) + }); + + new_ds + } + + #[cfg(feature = "zfp")] + pub fn zfp_precision(self, precision: u8) -> Self { + let new_ds = self.with_dcpl(|p| { + p.set_filters(&vec![Filter::zfp_precision(precision)]) + }); + + new_ds + } + + #[cfg(feature = "zfp")] + pub fn zfp_accuracy(self, accuracy: f64) -> Self { + let new_ds = self.with_dcpl(|p| { + p.set_filters(&vec![Filter::zfp_accuracy(accuracy)]) + }); + + new_ds + } + + #[cfg(feature = "zfp")] + pub fn zfp_lossless(self) -> Self { + let new_ds = self.with_dcpl(|p| { + p.set_filters(&vec![Filter::zfp_lossless()]) + }); + + new_ds + } + } #[derive(Clone)] @@ -327,6 +364,9 @@ where } }) } + + + } @@ -718,10 +758,6 @@ impl DatasetBuilderInner { self.with_dcpl(|pl| pl.blosc_zstd(clevel, shuffle)); } - #[cfg(feature = "zfp")] - pub fn zfp_rate(&mut self, rate: f64) { - self.with_dcpl(|pl| pl.zfp_rate(rate)); - } pub fn add_filter(&mut self, id: H5Z_filter_t, cdata: &[c_uint]) { self.with_dcpl(|pl| pl.add_filter(id, cdata)); diff --git a/hdf5/src/hl/filters.rs b/hdf5/src/hl/filters.rs index bafae94eb..15d6e3f43 100644 --- a/hdf5/src/hl/filters.rs +++ b/hdf5/src/hl/filters.rs @@ -113,6 +113,7 @@ mod zfp_impl { FixedRate(f64), FixedPrecision(u8), FixedAccuracy(f64), + Lossless(), } // Bitwise compare f64 so NaN and signed zero are deterministic @@ -123,6 +124,7 @@ mod zfp_impl { (FixedRate(a), FixedRate(b)) => a.to_bits() == b.to_bits(), (FixedPrecision(a), FixedPrecision(b)) => a == b, (FixedAccuracy(a), FixedAccuracy(b)) => a.to_bits() == b.to_bits(), + (Lossless(), Lossless()) => true, _ => false, } } @@ -362,6 +364,12 @@ impl Filter { Self::zfp(ZfpMode::FixedAccuracy(accuracy)) } + #[cfg(feature = "zfp")] + pub fn zfp_lossless() -> Self { + Self::zfp(ZfpMode::Lossless()) + } + + pub fn user(id: H5Z_filter_t, cdata: &[c_uint]) -> Self { Self::User(id, cdata.to_vec()) } @@ -485,6 +493,7 @@ impl Filter { let accuracy = f64::from_bits(((param1 as u64) << 32) | (param2 as u64)); ZfpMode::FixedAccuracy(accuracy) } + 4 => ZfpMode::Lossless(), _ => fail!("invalid zfp mode: {}", mode), }; Ok(Self::zfp(zfp_mode)) @@ -586,6 +595,7 @@ impl Filter { let bits = accuracy.to_bits(); (3, (bits >> 32) as c_uint, bits as c_uint) } + ZfpMode::Lossless() => (4, 0, 0), }; cdata[7] = mode_val; cdata[8] = param1; @@ -705,6 +715,7 @@ pub(crate) fn validate_filters(filters: &[Filter], type_class: H5T_class_t) -> R #[cfg(test)] mod tests { use std::hint::assert_unchecked; + use std::io::{Seek, SeekFrom}; use ndarray::Array2; use hdf5_sys::h5t::H5T_class_t; @@ -879,6 +890,47 @@ mod tests { Ok(()) } + #[test] + #[cfg(feature = "zfp")] + fn test_zfp_lossless() -> Result<()> { + use super::zfp_available; + + if !zfp_available() { + println!("ZFP filter not available, skipping test"); + return Ok(()); + } + with_tmp_file(|file| { + let data = ndarray::Array1::::linspace(0.0, 1000.0, 100000); + file.new_dataset_builder() + .zfp_lossless() + .with_data(&data) + .chunk((10000,)) + .create("zfp_lossless_1d") + .unwrap(); + + + + let ds = file.dataset("zfp_lossless_1d").unwrap(); + // get number of bytes of ds + let n_bytes = file.size(); + dbg!(n_bytes); + + let read_data: Vec = ds.read_raw().unwrap(); + let error = data.iter() + .zip(read_data.iter()) + .map(|(a, b)| (a - b).abs()) + .sum::() / data.len() as f32; + + let target_bytes = 100000*4; + assert!(n_bytes <= target_bytes, "Dataset size {} exceeds target {}", n_bytes, target_bytes); + assert_eq!(n_bytes,7592); + assert_eq!(error,0.0); + + } + ); + assert_eq!(1,0); + Ok(()) + } #[test] #[cfg(feature = "zfp")] fn test_zfp_roundtrip_1d() -> Result<()> { @@ -1084,10 +1136,16 @@ mod tests { with_tmp_file(|file| { let data: Vec = (0..1000).map(|i| (i as f32) * 0.1).collect(); let data = ndarray::Array1::from_shape_vec(1000, data).unwrap(); + // file.new_dataset_builder() + // .with_data(&data) + // .chunk(100) + // .with_dcpl(|p| p.set_filters(&pipeline)) + // .create("zfp_rate_8").unwrap(); + // file.new_dataset_builder() + .zfp_rate(8.0) .with_data(&data) .chunk(100) - .with_dcpl(|p| p.set_filters(&pipeline)) .create("zfp_rate_8").unwrap(); diff --git a/hdf5/src/hl/plist/dataset_create.rs b/hdf5/src/hl/plist/dataset_create.rs index 942d9ff4c..3e8847add 100644 --- a/hdf5/src/hl/plist/dataset_create.rs +++ b/hdf5/src/hl/plist/dataset_create.rs @@ -479,12 +479,6 @@ impl DatasetCreateBuilder { self } - #[cfg(feature = "zfp")] - pub fn zfp_rate(&mut self, rate: f64) -> &mut Self { - let mode = ZfpMode::FixedRate(rate); - self.filters.push(Filter::zfp(mode)); - self - } pub fn add_filter(&mut self, id: H5Z_filter_t, cdata: &[c_uint]) -> &mut Self { self.filters.push(Filter::user(id, cdata)); From cf9617eb1d792fd324c6b6c7ef76b4d1df2cd015 Mon Sep 17 00:00:00 2001 From: Frank Blubaugh Date: Wed, 26 Nov 2025 20:18:32 -0600 Subject: [PATCH 067/141] Fixed the interface with reversible so it now runs the compression filter --- hdf5/src/hl/filters.rs | 15 ++++++--------- hdf5/src/hl/filters/zfp.rs | 11 +++++++++++ 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/hdf5/src/hl/filters.rs b/hdf5/src/hl/filters.rs index 15d6e3f43..49c0ba3ec 100644 --- a/hdf5/src/hl/filters.rs +++ b/hdf5/src/hl/filters.rs @@ -113,7 +113,7 @@ mod zfp_impl { FixedRate(f64), FixedPrecision(u8), FixedAccuracy(f64), - Lossless(), + Reversible(), } // Bitwise compare f64 so NaN and signed zero are deterministic @@ -124,7 +124,7 @@ mod zfp_impl { (FixedRate(a), FixedRate(b)) => a.to_bits() == b.to_bits(), (FixedPrecision(a), FixedPrecision(b)) => a == b, (FixedAccuracy(a), FixedAccuracy(b)) => a.to_bits() == b.to_bits(), - (Lossless(), Lossless()) => true, + (Reversible(), Reversible()) => true, _ => false, } } @@ -366,7 +366,7 @@ impl Filter { #[cfg(feature = "zfp")] pub fn zfp_lossless() -> Self { - Self::zfp(ZfpMode::Lossless()) + Self::zfp(ZfpMode::Reversible()) } @@ -493,7 +493,7 @@ impl Filter { let accuracy = f64::from_bits(((param1 as u64) << 32) | (param2 as u64)); ZfpMode::FixedAccuracy(accuracy) } - 4 => ZfpMode::Lossless(), + 5 => ZfpMode::Reversible(), _ => fail!("invalid zfp mode: {}", mode), }; Ok(Self::zfp(zfp_mode)) @@ -595,7 +595,7 @@ impl Filter { let bits = accuracy.to_bits(); (3, (bits >> 32) as c_uint, bits as c_uint) } - ZfpMode::Lossless() => (4, 0, 0), + ZfpMode::Reversible() => (5, 0, 0), }; cdata[7] = mode_val; cdata[8] = param1; @@ -913,8 +913,6 @@ mod tests { let ds = file.dataset("zfp_lossless_1d").unwrap(); // get number of bytes of ds let n_bytes = file.size(); - dbg!(n_bytes); - let read_data: Vec = ds.read_raw().unwrap(); let error = data.iter() .zip(read_data.iter()) @@ -923,12 +921,11 @@ mod tests { let target_bytes = 100000*4; assert!(n_bytes <= target_bytes, "Dataset size {} exceeds target {}", n_bytes, target_bytes); - assert_eq!(n_bytes,7592); + assert_eq!(n_bytes,249368); assert_eq!(error,0.0); } ); - assert_eq!(1,0); Ok(()) } #[test] diff --git a/hdf5/src/hl/filters/zfp.rs b/hdf5/src/hl/filters/zfp.rs index abdb2e16a..472c1c1e8 100644 --- a/hdf5/src/hl/filters/zfp.rs +++ b/hdf5/src/hl/filters/zfp.rs @@ -17,6 +17,7 @@ pub use zfp_sys::{ zfp_stream_set_precision, zfp_stream_set_rate, zfp_type_zfp_type_double, zfp_type_zfp_type_float, stream_close, stream_open, }; +use zfp_sys::zfp_stream_set_reversible; const ZFP_FILTER_NAME: &[u8] = b"zfp\0"; pub const ZFP_FILTER_ID: H5Z_filter_t = 32013; @@ -26,6 +27,7 @@ const ZFP_FILTER_VERSION: c_uint = 1; const ZFP_MODE_RATE: c_uint = 1; const ZFP_MODE_PRECISION: c_uint = 2; const ZFP_MODE_ACCURACY: c_uint = 3; +const ZFP_MODE_REVERSIBLE: c_uint = 5; const ZFP_FILTER_INFO: &H5Z_class2_t = &H5Z_class2_t { version: H5Z_CLASS_T_VERS as _, @@ -152,6 +154,9 @@ fn parse_zfp_cdata(cd_nelmts: size_t, cd_values: *const c_uint) -> Option { + (0.0,0,0.0) + } _ => { h5err!("Invalid ZFP mode", H5E_PLIST, H5E_CALLBACK); return None; @@ -196,6 +201,9 @@ unsafe fn filter_zfp_compress( ZFP_MODE_ACCURACY => { zfp_stream_set_accuracy(zfp, cfg.accuracy); } + ZFP_MODE_REVERSIBLE =>{ + zfp_stream_set_reversible(zfp) + } _ => { zfp_stream_close(zfp); return 0; @@ -302,6 +310,9 @@ unsafe fn filter_zfp_decompress( ZFP_MODE_ACCURACY => { zfp_stream_set_accuracy(zfp, cfg.accuracy); } + ZFP_MODE_REVERSIBLE =>{ + zfp_stream_set_reversible(zfp) + } _ => { zfp_stream_close(zfp); return 0; From 6dc4e12f11cf1bcca073fc55c579d6f07ee0436a Mon Sep 17 00:00:00 2001 From: Frank Blubaugh Date: Wed, 26 Nov 2025 20:24:36 -0600 Subject: [PATCH 068/141] Remove unused ZfpMode import from dataset_create.rs --- hdf5/src/hl/plist/dataset_create.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/hdf5/src/hl/plist/dataset_create.rs b/hdf5/src/hl/plist/dataset_create.rs index 3e8847add..e7e30c68b 100644 --- a/hdf5/src/hl/plist/dataset_create.rs +++ b/hdf5/src/hl/plist/dataset_create.rs @@ -31,8 +31,6 @@ use hdf5_types::{OwnedDynValue, TypeDescriptor}; use crate::dim::Dimension; -#[cfg(feature="zfp")] -use crate::filters::ZfpMode; use crate::globals::H5P_DATASET_CREATE; use crate::hl::datatype::Datatype; From da2a1cf8d57ff63c7c4692cf1e7602295c9ce200 Mon Sep 17 00:00:00 2001 From: Frank Blubaugh Date: Thu, 27 Nov 2025 07:02:03 -0600 Subject: [PATCH 069/141] fixed typo obscuring the `zfp_available()` method for testing the feature. --- hdf5/src/hl/filters.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/hdf5/src/hl/filters.rs b/hdf5/src/hl/filters.rs index 49c0ba3ec..37c13c6c0 100644 --- a/hdf5/src/hl/filters.rs +++ b/hdf5/src/hl/filters.rs @@ -210,7 +210,6 @@ pub fn blosc_available() -> bool { } /// Returns `true` if ZFP filter is available. -#[cfg(feature = "zfp")] pub fn zfp_available() -> bool { h5lock!(H5Zfilter_avail(32013) == 1) } From fe991823f795d28c23bd9ee6d32124760adba050 Mon Sep 17 00:00:00 2001 From: Frank Blubaugh Date: Thu, 27 Nov 2025 19:25:39 -0600 Subject: [PATCH 070/141] removed unused test function assert_unchecked() left over from testing. --- hdf5/src/hl/filters.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/hdf5/src/hl/filters.rs b/hdf5/src/hl/filters.rs index 37c13c6c0..9fe9aa72b 100644 --- a/hdf5/src/hl/filters.rs +++ b/hdf5/src/hl/filters.rs @@ -713,7 +713,6 @@ pub(crate) fn validate_filters(filters: &[Filter], type_class: H5T_class_t) -> R #[cfg(test)] mod tests { - use std::hint::assert_unchecked; use std::io::{Seek, SeekFrom}; use ndarray::Array2; use hdf5_sys::h5t::H5T_class_t; From 2b23f524f073755d64b808d08a9ef5d5189e0015 Mon Sep 17 00:00:00 2001 From: Frank Blubaugh Date: Thu, 27 Nov 2025 19:29:19 -0600 Subject: [PATCH 071/141] ran cargo fmt --- hdf5/src/hl/dataset.rs | 26 +----- hdf5/src/hl/filters.rs | 126 +++++++++++++--------------- hdf5/src/hl/filters/zfp.rs | 23 ++--- hdf5/src/hl/plist/dataset_create.rs | 2 - 4 files changed, 72 insertions(+), 105 deletions(-) diff --git a/hdf5/src/hl/dataset.rs b/hdf5/src/hl/dataset.rs index 4d40e38b6..0ded24dd4 100644 --- a/hdf5/src/hl/dataset.rs +++ b/hdf5/src/hl/dataset.rs @@ -248,40 +248,31 @@ impl DatasetBuilder { #[cfg(feature = "zfp")] pub fn zfp_rate(self, rate: f64) -> Self { - let new_ds = self.with_dcpl(|p| { - p.set_filters(&vec![Filter::zfp_rate(rate)]) - }); + let new_ds = self.with_dcpl(|p| p.set_filters(&vec![Filter::zfp_rate(rate)])); new_ds } #[cfg(feature = "zfp")] pub fn zfp_precision(self, precision: u8) -> Self { - let new_ds = self.with_dcpl(|p| { - p.set_filters(&vec![Filter::zfp_precision(precision)]) - }); + let new_ds = self.with_dcpl(|p| p.set_filters(&vec![Filter::zfp_precision(precision)])); new_ds } #[cfg(feature = "zfp")] pub fn zfp_accuracy(self, accuracy: f64) -> Self { - let new_ds = self.with_dcpl(|p| { - p.set_filters(&vec![Filter::zfp_accuracy(accuracy)]) - }); + let new_ds = self.with_dcpl(|p| p.set_filters(&vec![Filter::zfp_accuracy(accuracy)])); new_ds } #[cfg(feature = "zfp")] pub fn zfp_lossless(self) -> Self { - let new_ds = self.with_dcpl(|p| { - p.set_filters(&vec![Filter::zfp_lossless()]) - }); + let new_ds = self.with_dcpl(|p| p.set_filters(&vec![Filter::zfp_lossless()])); new_ds } - } #[derive(Clone)] @@ -364,16 +355,8 @@ where } }) } - - - } - - - - - #[derive(Debug, Clone, PartialEq, Eq)] pub enum Chunk { Exact(Vec), // exact chunk shape @@ -758,7 +741,6 @@ impl DatasetBuilderInner { self.with_dcpl(|pl| pl.blosc_zstd(clevel, shuffle)); } - pub fn add_filter(&mut self, id: H5Z_filter_t, cdata: &[c_uint]) { self.with_dcpl(|pl| pl.add_filter(id, cdata)); } diff --git a/hdf5/src/hl/filters.rs b/hdf5/src/hl/filters.rs index 9fe9aa72b..3d75621d3 100644 --- a/hdf5/src/hl/filters.rs +++ b/hdf5/src/hl/filters.rs @@ -105,7 +105,6 @@ mod blosc_impl { #[cfg(feature = "blosc")] pub use blosc_impl::*; - #[cfg(feature = "zfp")] mod zfp_impl { #[derive(Clone, Copy, Debug)] @@ -138,7 +137,6 @@ mod zfp_impl { } } - #[cfg(feature = "zfp")] pub use zfp_impl::*; @@ -368,7 +366,6 @@ impl Filter { Self::zfp(ZfpMode::Reversible()) } - pub fn user(id: H5Z_filter_t, cdata: &[c_uint]) -> Self { Self::User(id, cdata.to_vec()) } @@ -661,7 +658,8 @@ impl Filter { } } -const COMP_FILTER_IDS: &[H5Z_filter_t] = &[H5Z_FILTER_DEFLATE, H5Z_FILTER_SZIP, 32000, 32001, 32013]; +const COMP_FILTER_IDS: &[H5Z_filter_t] = + &[H5Z_FILTER_DEFLATE, H5Z_FILTER_SZIP, 32000, 32001, 32013]; pub(crate) fn validate_filters(filters: &[Filter], type_class: H5T_class_t) -> Result<()> { let mut map: HashMap = HashMap::new(); @@ -713,17 +711,17 @@ pub(crate) fn validate_filters(filters: &[Filter], type_class: H5T_class_t) -> R #[cfg(test)] mod tests { - use std::io::{Seek, SeekFrom}; - use ndarray::Array2; use hdf5_sys::h5t::H5T_class_t; + use ndarray::Array2; + use std::io::{Seek, SeekFrom}; use super::{ blosc_available, deflate_available, lzf_available, szip_available, validate_filters, Filter, FilterInfo, SZip, ScaleOffset, }; + use crate::hl::filters::zfp_available; use crate::test::with_tmp_file; use crate::{plist::DatasetCreate, Result}; - use crate::hl::filters::zfp_available; #[test] fn test_filter_pipeline() -> Result<()> { @@ -751,16 +749,15 @@ mod tests { comp_filters.push(Filter::blosc_snappy(0, BloscShuffle::Bit)); } - #[cfg(feature="zfp")] + #[cfg(feature = "zfp")] assert_eq!(cfg!(feature = "zfp"), zfp_available()); - #[cfg(feature="zfp")] + #[cfg(feature = "zfp")] { comp_filters.push(Filter::zfp_rate(8.0)); comp_filters.push(Filter::zfp_precision(16)); comp_filters.push(Filter::zfp_accuracy(1e-3)); } - for c in &comp_filters { assert!(c.is_available()); assert!(c.encode_enabled()); @@ -855,11 +852,10 @@ mod tests { // Test with actual dataset creation for f32 let res = with_tmp_file(|file| { - file.new_dataset_builder() .empty::() - .shape((100,50)) - .chunk((10,10)) + .shape((100, 50)) + .chunk((10, 10)) .with_dcpl(|p| p.set_filters(&pipeline)) .create("zfp_f32") .unwrap(); @@ -906,24 +902,23 @@ mod tests { .create("zfp_lossless_1d") .unwrap(); - - - let ds = file.dataset("zfp_lossless_1d").unwrap(); + let ds = file.dataset("zfp_lossless_1d").unwrap(); // get number of bytes of ds let n_bytes = file.size(); - let read_data: Vec = ds.read_raw().unwrap(); - let error = data.iter() - .zip(read_data.iter()) - .map(|(a, b)| (a - b).abs()) - .sum::() / data.len() as f32; - - let target_bytes = 100000*4; - assert!(n_bytes <= target_bytes, "Dataset size {} exceeds target {}", n_bytes, target_bytes); - assert_eq!(n_bytes,249368); - assert_eq!(error,0.0); - - } - ); + let read_data: Vec = ds.read_raw().unwrap(); + let error = data.iter().zip(read_data.iter()).map(|(a, b)| (a - b).abs()).sum::() + / data.len() as f32; + + let target_bytes = 100000 * 4; + assert!( + n_bytes <= target_bytes, + "Dataset size {} exceeds target {}", + n_bytes, + target_bytes + ); + assert_eq!(n_bytes, 249368); + assert_eq!(error, 0.0); + }); Ok(()) } #[test] @@ -945,7 +940,6 @@ mod tests { .create("zfp_1d_dcpl") .unwrap(); - let ds = file.dataset("zfp_1d_dcpl").unwrap(); let read_data: Vec = ds.read_raw().unwrap(); @@ -953,8 +947,14 @@ mod tests { assert_eq!(read_data.len(), data.len()); for (i, (original, compressed)) in data.iter().zip(read_data.iter()).enumerate() { let diff = (original - compressed).abs(); - assert!(diff < 0.1, "Index {}: difference too large: {} vs {} (diff: {})", - i, original, compressed, diff); + assert!( + diff < 0.1, + "Index {}: difference too large: {} vs {} (diff: {})", + i, + original, + compressed, + diff + ); } }); @@ -973,14 +973,12 @@ mod tests { with_tmp_file(|file| { let data: Vec = (0..1000).map(|i| (i as f64) * 0.01).collect(); - let data = Array2::from_shape_vec((10,100), data).unwrap(); - - + let data = Array2::from_shape_vec((10, 100), data).unwrap(); let pipteline = vec![Filter::zfp_precision(32)]; file.new_dataset_builder() .with_data(&data) - .chunk((10,10)) + .chunk((10, 10)) .with_dcpl(|p| p.set_filters(&pipteline)) .create("zfp_2d") .unwrap(); @@ -1019,9 +1017,6 @@ mod tests { .create("zfp_3d") .unwrap(); - - - let ds = file.dataset("zfp_3d").unwrap(); let read_data: Vec = ds.read_raw().unwrap(); @@ -1052,7 +1047,6 @@ mod tests { .create("zfp_4d") .unwrap(); - let ds = file.dataset("zfp_4d").unwrap(); let read_data: Vec = ds.read_raw().unwrap(); @@ -1070,7 +1064,13 @@ mod tests { // Test FixedRate parsing let rate_bits = 12.5_f64.to_bits(); let cdata_rate = vec![ - 0, 1, 4, 100, 0, 0, 0, + 0, + 1, + 4, + 100, + 0, + 0, + 0, 1, // mode = rate (rate_bits >> 32) as u32, rate_bits as u32, @@ -1084,8 +1084,7 @@ mod tests { // Test FixedPrecision parsing let cdata_precision = vec![ - 0, 1, 8, 100, 0, 0, 0, - 2, // mode = precision + 0, 1, 8, 100, 0, 0, 0, 2, // mode = precision 24, // precision 0, ]; @@ -1099,7 +1098,13 @@ mod tests { // Test FixedAccuracy parsing let accuracy_bits = 1e-5_f64.to_bits(); let cdata_accuracy = vec![ - 0, 1, 8, 100, 0, 0, 0, + 0, + 1, + 8, + 100, + 0, + 0, + 0, 3, // mode = accuracy (accuracy_bits >> 32) as u32, accuracy_bits as u32, @@ -1124,8 +1129,6 @@ mod tests { return Ok(()); } - - let pipeline = vec![Filter::zfp_rate(8.0)]; // Test ZFP combined with shuffle (shuffle should come first) with_tmp_file(|file| { @@ -1141,17 +1144,15 @@ mod tests { .zfp_rate(8.0) .with_data(&data) .chunk(100) - .create("zfp_rate_8").unwrap(); - + .create("zfp_rate_8") + .unwrap(); let ds = file.dataset("zfp_rate_8").unwrap(); let read_data: Vec = ds.read_raw().unwrap(); - let error = data.iter() - .zip(read_data.iter()) - .map(|(a, b)| (a - b).abs()) - .sum::() / data.len() as f32; - assert_eq!(error,0.082505114); + let error = data.iter().zip(read_data.iter()).map(|(a, b)| (a - b).abs()).sum::() + / data.len() as f32; + assert_eq!(error, 0.082505114); assert_eq!(read_data.len(), data.len()); }); @@ -1169,7 +1170,6 @@ mod tests { .create("zfp_with_fletcher32") .unwrap(); - let ds = file.dataset("zfp_with_fletcher32").unwrap(); let read_data: Vec = ds.read_raw().unwrap(); @@ -1213,7 +1213,6 @@ mod tests { .create("zfp_low_rate") .unwrap(); - let ds_high = file.dataset("zfp_high_rate").unwrap(); let ds_low = file.dataset("zfp_low_rate").unwrap(); @@ -1221,15 +1220,13 @@ mod tests { let read_low: Vec = ds_low.read_raw().unwrap(); // High rate should have better accuracy - let error_high: f32 = data.iter() - .zip(read_high.iter()) - .map(|(a, b)| (a - b).abs()) - .sum::() / data.len() as f32; + let error_high: f32 = + data.iter().zip(read_high.iter()).map(|(a, b)| (a - b).abs()).sum::() + / data.len() as f32; - let error_low: f32 = data.iter() - .zip(read_low.iter()) - .map(|(a, b)| (a - b).abs()) - .sum::() / data.len() as f32; + let error_low: f32 = + data.iter().zip(read_low.iter()).map(|(a, b)| (a - b).abs()).sum::() + / data.len() as f32; println!("High rate error: {}, Low rate error: {}", error_high, error_low); assert!(error_high < error_low || error_high < 0.001); @@ -1262,7 +1259,6 @@ mod tests { .create("zfp_zeros") .unwrap(); - let ds = file.dataset("zfp_zeros").unwrap(); let read_data: Vec = ds.read_raw().unwrap(); @@ -1272,7 +1268,6 @@ mod tests { } }); - let pipeline = vec![Filter::zfp_accuracy(1e-6)]; // Test with constant values with_tmp_file(|file| { @@ -1285,7 +1280,6 @@ mod tests { .create("zfp_constant") .unwrap(); - let ds = file.dataset("zfp_constant").unwrap(); let read_data: Vec = ds.read_raw().unwrap(); diff --git a/hdf5/src/hl/filters/zfp.rs b/hdf5/src/hl/filters/zfp.rs index 472c1c1e8..8e3a7295c 100644 --- a/hdf5/src/hl/filters/zfp.rs +++ b/hdf5/src/hl/filters/zfp.rs @@ -10,14 +10,14 @@ use crate::error::H5ErrorCode; use crate::globals::{H5E_CALLBACK, H5E_PLIST}; use crate::internal_prelude::*; +use zfp_sys::zfp_stream_set_reversible; pub use zfp_sys::{ - zfp_compress, zfp_decompress, zfp_field_1d, zfp_field_2d, zfp_field_3d, zfp_field_4d, - zfp_field_free, zfp_stream_close, zfp_stream_maximum_size, zfp_stream_open, - zfp_stream_rewind, zfp_stream_set_accuracy, zfp_stream_set_bit_stream, + stream_close, stream_open, zfp_compress, zfp_decompress, zfp_field_1d, zfp_field_2d, + zfp_field_3d, zfp_field_4d, zfp_field_free, zfp_stream_close, zfp_stream_maximum_size, + zfp_stream_open, zfp_stream_rewind, zfp_stream_set_accuracy, zfp_stream_set_bit_stream, zfp_stream_set_precision, zfp_stream_set_rate, zfp_type_zfp_type_double, - zfp_type_zfp_type_float, stream_close, stream_open, + zfp_type_zfp_type_float, }; -use zfp_sys::zfp_stream_set_reversible; const ZFP_FILTER_NAME: &[u8] = b"zfp\0"; pub const ZFP_FILTER_ID: H5Z_filter_t = 32013; @@ -154,9 +154,7 @@ fn parse_zfp_cdata(cd_nelmts: size_t, cd_values: *const c_uint) -> Option { - (0.0,0,0.0) - } + ZFP_MODE_REVERSIBLE => (0.0, 0, 0.0), _ => { h5err!("Invalid ZFP mode", H5E_PLIST, H5E_CALLBACK); return None; @@ -201,9 +199,7 @@ unsafe fn filter_zfp_compress( ZFP_MODE_ACCURACY => { zfp_stream_set_accuracy(zfp, cfg.accuracy); } - ZFP_MODE_REVERSIBLE =>{ - zfp_stream_set_reversible(zfp) - } + ZFP_MODE_REVERSIBLE => zfp_stream_set_reversible(zfp), _ => { zfp_stream_close(zfp); return 0; @@ -310,9 +306,7 @@ unsafe fn filter_zfp_decompress( ZFP_MODE_ACCURACY => { zfp_stream_set_accuracy(zfp, cfg.accuracy); } - ZFP_MODE_REVERSIBLE =>{ - zfp_stream_set_reversible(zfp) - } + ZFP_MODE_REVERSIBLE => zfp_stream_set_reversible(zfp), _ => { zfp_stream_close(zfp); return 0; @@ -403,4 +397,3 @@ unsafe fn filter_zfp_decompress( *buf_size = outbuf_size; outbuf_size } - diff --git a/hdf5/src/hl/plist/dataset_create.rs b/hdf5/src/hl/plist/dataset_create.rs index e7e30c68b..4e9d62e37 100644 --- a/hdf5/src/hl/plist/dataset_create.rs +++ b/hdf5/src/hl/plist/dataset_create.rs @@ -31,7 +31,6 @@ use hdf5_types::{OwnedDynValue, TypeDescriptor}; use crate::dim::Dimension; - use crate::globals::H5P_DATASET_CREATE; use crate::hl::datatype::Datatype; use crate::hl::filters::{validate_filters, Filter, SZip, ScaleOffset}; @@ -477,7 +476,6 @@ impl DatasetCreateBuilder { self } - pub fn add_filter(&mut self, id: H5Z_filter_t, cdata: &[c_uint]) -> &mut Self { self.filters.push(Filter::user(id, cdata)); self From 6ec697ffc3aa9d0a7366d774ed0da5e08f9acecf Mon Sep 17 00:00:00 2001 From: Frank Blubaugh Date: Fri, 28 Nov 2025 07:21:10 -0600 Subject: [PATCH 072/141] test build commit --- hdf5/src/hl/plist/object_copy.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/hdf5/src/hl/plist/object_copy.rs b/hdf5/src/hl/plist/object_copy.rs index 6d09184f1..7cf0b2892 100644 --- a/hdf5/src/hl/plist/object_copy.rs +++ b/hdf5/src/hl/plist/object_copy.rs @@ -210,3 +210,4 @@ mod tests { assert_eq!(ocpypl, ocpypl2); } } +// \ No newline at end of file From 44fd1fbdd0b7e7139e64fba90158be37a03f1e72 Mon Sep 17 00:00:00 2001 From: Frank Blubaugh Date: Fri, 28 Nov 2025 07:27:37 -0600 Subject: [PATCH 073/141] test build commit removed old comment --- hdf5/src/hl/plist/object_copy.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/hdf5/src/hl/plist/object_copy.rs b/hdf5/src/hl/plist/object_copy.rs index 7cf0b2892..6d09184f1 100644 --- a/hdf5/src/hl/plist/object_copy.rs +++ b/hdf5/src/hl/plist/object_copy.rs @@ -210,4 +210,3 @@ mod tests { assert_eq!(ocpypl, ocpypl2); } } -// \ No newline at end of file From aa3573275d65a84fbfbeda0cdd652e3bdfeee6de Mon Sep 17 00:00:00 2001 From: Frank Blubaugh Date: Sat, 29 Nov 2025 07:24:47 -0500 Subject: [PATCH 074/141] moved builder logic from impl Dataset to impl DatasetInner and wrapped teh filter calls with feature dependent macro implementation like the other features --- hdf5/src/hl/dataset.rs | 100 +++++++++++++++++++++++++++++------------ hdf5/src/hl/filters.rs | 19 ++++---- 2 files changed, 82 insertions(+), 37 deletions(-) diff --git a/hdf5/src/hl/dataset.rs b/hdf5/src/hl/dataset.rs index 0ded24dd4..cc26ee1ac 100644 --- a/hdf5/src/hl/dataset.rs +++ b/hdf5/src/hl/dataset.rs @@ -245,34 +245,34 @@ impl DatasetBuilder { conv: Conversion::Soft, } } - - #[cfg(feature = "zfp")] - pub fn zfp_rate(self, rate: f64) -> Self { - let new_ds = self.with_dcpl(|p| p.set_filters(&vec![Filter::zfp_rate(rate)])); - - new_ds - } - - #[cfg(feature = "zfp")] - pub fn zfp_precision(self, precision: u8) -> Self { - let new_ds = self.with_dcpl(|p| p.set_filters(&vec![Filter::zfp_precision(precision)])); - - new_ds - } - - #[cfg(feature = "zfp")] - pub fn zfp_accuracy(self, accuracy: f64) -> Self { - let new_ds = self.with_dcpl(|p| p.set_filters(&vec![Filter::zfp_accuracy(accuracy)])); - - new_ds - } - - #[cfg(feature = "zfp")] - pub fn zfp_lossless(self) -> Self { - let new_ds = self.with_dcpl(|p| p.set_filters(&vec![Filter::zfp_lossless()])); - - new_ds - } + // + // #[cfg(feature = "zfp")] + // pub fn zfp_rate(self, rate: f64) -> Self { + // let new_ds = self.with_dcpl(|p| p.set_filters(&vec![Filter::zfp_rate(rate)])); + // + // new_ds + // } + // + // #[cfg(feature = "zfp")] + // pub fn zfp_precision(self, precision: u8) -> Self { + // let new_ds = self.with_dcpl(|p| p.set_filters(&vec![Filter::zfp_precision(precision)])); + // + // new_ds + // } + // + // #[cfg(feature = "zfp")] + // pub fn zfp_accuracy(self, accuracy: f64) -> Self { + // let new_ds = self.with_dcpl(|p| p.set_filters(&vec![Filter::zfp_accuracy(accuracy)])); + // + // new_ds + // } + // + // #[cfg(feature = "zfp")] + // pub fn zfp_reversible(self) -> Self { + // let new_ds = self.with_dcpl(|p| p.set_filters(&vec![Filter::zfp_reversible()])); + // + // new_ds + // } } #[derive(Clone)] @@ -741,6 +741,31 @@ impl DatasetBuilderInner { self.with_dcpl(|pl| pl.blosc_zstd(clevel, shuffle)); } + + + + #[cfg(feature = "zfp")] + pub fn zfp_rate(&mut self, rate: f64) { + self.with_dcpl(|p| p.set_filters(&vec![Filter::zfp_rate(rate)])); + + } + + #[cfg(feature = "zfp")] + pub fn zfp_precision(&mut self, precision: u8) { + self.with_dcpl(|p| p.set_filters(&vec![Filter::zfp_precision(precision)])); + } + + #[cfg(feature = "zfp")] + pub fn zfp_accuracy(&mut self, accuracy: f64) { + self.with_dcpl(|p| p.set_filters(&vec![Filter::zfp_accuracy(accuracy)])); + } + + #[cfg(feature = "zfp")] + pub fn zfp_reversible(&mut self) { + self.with_dcpl(|p| p.set_filters(&vec![Filter::zfp_reversible()])); + } + + pub fn add_filter(&mut self, id: H5Z_filter_t, cdata: &[c_uint]) { self.with_dcpl(|pl| pl.add_filter(id, cdata)); } @@ -1000,6 +1025,25 @@ macro_rules! impl_builder_methods { #[cfg(feature = "blosc-zstd")] DatasetCreate: blosc_zstd(clevel: u8, shuffle: impl Into) ); + + impl_builder!( + #[cfg(feature = "zfp")] + DatasetCreate: zfp_rate(rate: f64) + ); + impl_builder!( + #[cfg(feature = "zfp")] + DatasetCreate: zfp_accuracy(accuracy: f64) + ); + impl_builder!( + #[cfg(feature = "zfp")] + DatasetCreate: zfp_precision(rate: u8) + ); + impl_builder!( + #[cfg(feature = "zfp")] + DatasetCreate: zfp_reversible() + ); + + impl_builder!(DatasetCreate: add_filter(id: H5Z_filter_t, cdata: &[c_uint])); impl_builder!(DatasetCreate: clear_filters()); impl_builder!(DatasetCreate: alloc_time(alloc_time: Option)); diff --git a/hdf5/src/hl/filters.rs b/hdf5/src/hl/filters.rs index 3d75621d3..df3af2974 100644 --- a/hdf5/src/hl/filters.rs +++ b/hdf5/src/hl/filters.rs @@ -112,7 +112,8 @@ mod zfp_impl { FixedRate(f64), FixedPrecision(u8), FixedAccuracy(f64), - Reversible(), + Reversible + , } // Bitwise compare f64 so NaN and signed zero are deterministic @@ -123,7 +124,7 @@ mod zfp_impl { (FixedRate(a), FixedRate(b)) => a.to_bits() == b.to_bits(), (FixedPrecision(a), FixedPrecision(b)) => a == b, (FixedAccuracy(a), FixedAccuracy(b)) => a.to_bits() == b.to_bits(), - (Reversible(), Reversible()) => true, + (Reversible, Reversible) => true, _ => false, } } @@ -132,7 +133,7 @@ mod zfp_impl { impl Default for ZfpMode { fn default() -> Self { - ZfpMode::FixedRate(1.0) + ZfpMode::FixedRate(4.0) } } } @@ -362,8 +363,8 @@ impl Filter { } #[cfg(feature = "zfp")] - pub fn zfp_lossless() -> Self { - Self::zfp(ZfpMode::Reversible()) + pub fn zfp_reversible() -> Self { + Self::zfp(ZfpMode::Reversible) } pub fn user(id: H5Z_filter_t, cdata: &[c_uint]) -> Self { @@ -489,7 +490,7 @@ impl Filter { let accuracy = f64::from_bits(((param1 as u64) << 32) | (param2 as u64)); ZfpMode::FixedAccuracy(accuracy) } - 5 => ZfpMode::Reversible(), + 5 => ZfpMode::Reversible, _ => fail!("invalid zfp mode: {}", mode), }; Ok(Self::zfp(zfp_mode)) @@ -591,7 +592,7 @@ impl Filter { let bits = accuracy.to_bits(); (3, (bits >> 32) as c_uint, bits as c_uint) } - ZfpMode::Reversible() => (5, 0, 0), + ZfpMode::Reversible => (5, 0, 0), }; cdata[7] = mode_val; cdata[8] = param1; @@ -886,7 +887,7 @@ mod tests { #[test] #[cfg(feature = "zfp")] - fn test_zfp_lossless() -> Result<()> { + fn test_zfp_reversible() -> Result<()> { use super::zfp_available; if !zfp_available() { @@ -896,7 +897,7 @@ mod tests { with_tmp_file(|file| { let data = ndarray::Array1::::linspace(0.0, 1000.0, 100000); file.new_dataset_builder() - .zfp_lossless() + .zfp_reversible() .with_data(&data) .chunk((10000,)) .create("zfp_lossless_1d") From cae33633f9a822668a923248cfe9f65691c29398 Mon Sep 17 00:00:00 2001 From: Frank Blubaugh Date: Sat, 29 Nov 2025 07:28:41 -0500 Subject: [PATCH 075/141] Incremented version number and updated the change log --- CHANGELOG.md | 5 +++++ Cargo.toml | 2 +- hdf5/Cargo.toml | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 257e02348..bf324d948 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ ## hdf5 unreleased +## hdf5 v0.12.0 +Released Nov 29, 2025 +- Added support for ZFP compression filters as an optional feature + + ## hdf5 v0.11.0 Release date: Nov 23, 2025 - Fixed incorrect retrieved name of attributes diff --git a/Cargo.toml b/Cargo.toml index c57c5cd2b..304b46f8b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,7 +26,7 @@ num-complex = { version = "0.4", default-features = false } regex = "1.10" # internal -hdf5 = { package = "hdf5-metno", version = "0.9.3", path = "hdf5" } # !V +hdf5 = { package = "hdf5-metno", version = "0.12.0", path = "hdf5" } # !V hdf5-derive = { package = "hdf5-metno-derive", version = "0.9.1", path = "hdf5-derive" } # !V hdf5-src = { package = "hdf5-metno-src", version = "0.9.3", path = "hdf5-src" } # !V hdf5-sys = { package = "hdf5-metno-sys", version = "0.10.1", path = "hdf5-sys" } # !V diff --git a/hdf5/Cargo.toml b/hdf5/Cargo.toml index d142ab1a1..3309b8687 100644 --- a/hdf5/Cargo.toml +++ b/hdf5/Cargo.toml @@ -4,7 +4,7 @@ readme = "../README.md" description = "Thread-safe Rust bindings for the HDF5 library." build = "build.rs" categories = ["science", "filesystem"] -version = "0.11.0" # !V +version = "0.12.0" # !V rust-version.workspace = true authors.workspace = true keywords.workspace = true From b81cf000eaa3c137bfd931a6b3a884af44d8554b Mon Sep 17 00:00:00 2001 From: Frank Blubaugh Date: Sat, 29 Nov 2025 07:32:31 -0500 Subject: [PATCH 076/141] Removed call to nbytes in filter_zfp_compress. The ZFP library does not natively take the number of bytes needed for the processing but instead it uses the dimensionality of the compressed array to do the data compression. This information is captured byt the ZfpConfig Struct. This takes care of allocating the appropriate buffer size for the compression downstream from here. --- hdf5/src/hl/filters/zfp.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hdf5/src/hl/filters/zfp.rs b/hdf5/src/hl/filters/zfp.rs index 8e3a7295c..0ce5eb7f5 100644 --- a/hdf5/src/hl/filters/zfp.rs +++ b/hdf5/src/hl/filters/zfp.rs @@ -174,14 +174,14 @@ unsafe extern "C" fn filter_zfp( return 0; }; if flags & H5Z_FLAG_REVERSE == 0 { - unsafe { filter_zfp_compress(&cfg, nbytes, buf_size, buf) } + unsafe { filter_zfp_compress(&cfg, buf_size, buf) } } else { unsafe { filter_zfp_decompress(&cfg, nbytes, buf_size, buf) } } } unsafe fn filter_zfp_compress( - cfg: &ZfpConfig, nbytes: size_t, buf_size: *mut size_t, buf: *mut *mut c_void, + cfg: &ZfpConfig, buf_size: *mut size_t, buf: *mut *mut c_void, ) -> size_t { let zfp = zfp_stream_open(ptr::null_mut()); if zfp.is_null() { From c63d87dbe610821a4030ff5ac81911ce624bd394 Mon Sep 17 00:00:00 2001 From: Frank Blubaugh Date: Mon, 1 Dec 2025 09:08:06 -0500 Subject: [PATCH 077/141] fixed breaking typo for nightly lint --- hdf5/src/hl/filters.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hdf5/src/hl/filters.rs b/hdf5/src/hl/filters.rs index df3af2974..4638c4d2b 100644 --- a/hdf5/src/hl/filters.rs +++ b/hdf5/src/hl/filters.rs @@ -112,8 +112,7 @@ mod zfp_impl { FixedRate(f64), FixedPrecision(u8), FixedAccuracy(f64), - Reversible - , + Reversible, } // Bitwise compare f64 so NaN and signed zero are deterministic From 040d3e9d07ad9b4658e7b36d2d1777272240aff4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Dec 2025 21:39:32 +0000 Subject: [PATCH 078/141] Bump crate-ci/typos from 1.39.2 to 1.40.0 Bumps [crate-ci/typos](https://github.com/crate-ci/typos) from 1.39.2 to 1.40.0. - [Release notes](https://github.com/crate-ci/typos/releases) - [Changelog](https://github.com/crate-ci/typos/blob/master/CHANGELOG.md) - [Commits](https://github.com/crate-ci/typos/compare/626c4bedb751ce0b7f03262ca97ddda9a076ae1c...2d0ce569feab1f8752f1dde43cc2f2aa53236e06) --- updated-dependencies: - dependency-name: crate-ci/typos dependency-version: 1.40.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 43b47d906..650b1cbcd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,7 @@ jobs: - name: Checkout code uses: actions/checkout@v6 - name: Check spelling - uses: crate-ci/typos@626c4bedb751ce0b7f03262ca97ddda9a076ae1c # v1.39.2 + uses: crate-ci/typos@2d0ce569feab1f8752f1dde43cc2f2aa53236e06 # v1.40.0 lint: name: lint From 537e55c780f98d3489767f5261028dda1286a2bf Mon Sep 17 00:00:00 2001 From: Frank Blubaugh Date: Wed, 3 Dec 2025 11:19:31 -0500 Subject: [PATCH 079/141] added an export mode. renamed some varibles and fixed the header otuput size for set_local_zfp to the right values --- .gitignore | 2 + hdf5/src/hl/filters/zfp.rs | 99 +++++++++++++++++++++++++++----------- 2 files changed, 72 insertions(+), 29 deletions(-) diff --git a/.gitignore b/.gitignore index 2fb61a58a..36ac75c55 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,5 @@ target/ *.ipynb* .idea/ sweep.timestamp +.cargo/ +.cargo/config.toml \ No newline at end of file diff --git a/hdf5/src/hl/filters/zfp.rs b/hdf5/src/hl/filters/zfp.rs index 0ce5eb7f5..94e6881d0 100644 --- a/hdf5/src/hl/filters/zfp.rs +++ b/hdf5/src/hl/filters/zfp.rs @@ -64,8 +64,8 @@ extern "C" fn can_apply_zfp(_dcpl_id: hid_t, type_id: hid_t, _space_id: hid_t) - extern "C" fn set_local_zfp(dcpl_id: hid_t, type_id: hid_t, _space_id: hid_t) -> herr_t { const MAX_NDIMS: usize = 4; let mut flags: c_uint = 0; - let mut nelmts: size_t = 10; - let mut values: Vec = vec![0; 10]; + let mut nelmts: size_t = 4; + let mut values: Vec = vec![0; 4]; let ret = unsafe { H5Pget_filter_by_id2( dcpl_id, @@ -126,6 +126,47 @@ struct ZfpConfig { pub accuracy: f64, } +impl From for zfp_sys::zfp_config { + fn from(cfg: ZfpConfig) -> Self { + + let binding_output = match cfg.mode { + ZFP_MODE_RATE => { + zfp_sys::zfp_config__bindgen_ty_1 { + rate: cfg.rate + } + } + ZFP_MODE_PRECISION => { + zfp_sys::zfp_config__bindgen_ty_1 { + precision: cfg.precision + }, + } + ZFP_MODE_ACCURACY => { + zfp_sys::zfp_config__bindgen_ty_1 { + tolerance: cfg.accuracy + } + + } + ZFP_MODE_REVERSIBLE => { + zfp_sys::zfp_config__bindgen_ty_1 { + tolerance: cfg.precision + } + + } + _ => { + h5err!("Invalid ZFP mode", H5E_PLIST, H5E_CALLBACK); + return None; + } + } + + + zfp_sys::zfp_config { + mode: cfg.mode, + binding: binding_output, + } + + } +} + fn parse_zfp_cdata(cd_nelmts: size_t, cd_values: *const c_uint) -> Option { let cdata = unsafe { slice::from_raw_parts(cd_values, cd_nelmts as _) }; if cdata.len() < 7 { @@ -183,25 +224,25 @@ unsafe extern "C" fn filter_zfp( unsafe fn filter_zfp_compress( cfg: &ZfpConfig, buf_size: *mut size_t, buf: *mut *mut c_void, ) -> size_t { - let zfp = zfp_stream_open(ptr::null_mut()); - if zfp.is_null() { + let zfp_stream = zfp_stream_open(ptr::null_mut()); + if zfp_stream.is_null() { h5err!("Failed to open ZFP stream", H5E_PLIST, H5E_CALLBACK); return 0; } match cfg.mode { ZFP_MODE_RATE => { - zfp_stream_set_rate(zfp, cfg.rate, cfg.typesize as _, cfg.ndims as _, 0); + zfp_stream_set_rate(zfp_stream, cfg.rate, cfg.typesize as _, cfg.ndims as _, 0); } ZFP_MODE_PRECISION => { - zfp_stream_set_precision(zfp, cfg.precision); + zfp_stream_set_precision(zfp_stream, cfg.precision); } ZFP_MODE_ACCURACY => { - zfp_stream_set_accuracy(zfp, cfg.accuracy); + zfp_stream_set_accuracy(zfp_stream, cfg.accuracy); } - ZFP_MODE_REVERSIBLE => zfp_stream_set_reversible(zfp), + ZFP_MODE_REVERSIBLE => zfp_stream_set_reversible(zfp_stream), _ => { - zfp_stream_close(zfp); + zfp_stream_close(zfp_stream); return 0; } } @@ -251,29 +292,29 @@ unsafe fn filter_zfp_compress( }; if field.is_null() { - zfp_stream_close(zfp); + zfp_stream_close(zfp_stream); h5err!("Failed to create ZFP field", H5E_PLIST, H5E_CALLBACK); return 0; } - let maxsize = zfp_stream_maximum_size(zfp, field); + let maxsize = zfp_stream_maximum_size(zfp_stream, field); let outbuf = libc::malloc(maxsize); if outbuf.is_null() { zfp_field_free(field); - zfp_stream_close(zfp); + zfp_stream_close(zfp_stream); h5err!("Can't allocate compression buffer", H5E_PLIST, H5E_CALLBACK); return 0; } let bitstream = stream_open(outbuf.cast(), maxsize); - zfp_stream_set_bit_stream(zfp, bitstream); - zfp_stream_rewind(zfp); + zfp_stream_set_bit_stream(zfp_stream, bitstream); + zfp_stream_rewind(zfp_stream); - let compressed_size = zfp_compress(zfp, field); + let compressed_size = zfp_compress(zfp_stream, field); stream_close(bitstream); zfp_field_free(field); - zfp_stream_close(zfp); + zfp_stream_close(zfp_stream); if compressed_size == 0 { libc::free(outbuf); @@ -290,25 +331,25 @@ unsafe fn filter_zfp_compress( unsafe fn filter_zfp_decompress( cfg: &ZfpConfig, nbytes: size_t, buf_size: *mut size_t, buf: *mut *mut c_void, ) -> size_t { - let zfp = zfp_stream_open(ptr::null_mut()); - if zfp.is_null() { + let zfp_stream = zfp_stream_open(ptr::null_mut()); + if zfp_stream.is_null() { h5err!("Failed to open ZFP stream", H5E_PLIST, H5E_CALLBACK); return 0; } match cfg.mode { ZFP_MODE_RATE => { - zfp_stream_set_rate(zfp, cfg.rate, cfg.typesize as _, cfg.ndims as _, 0); + zfp_stream_set_rate(zfp_stream, cfg.rate, cfg.typesize as _, cfg.ndims as _, 0); } ZFP_MODE_PRECISION => { - zfp_stream_set_precision(zfp, cfg.precision); + zfp_stream_set_precision(zfp_stream, cfg.precision); } ZFP_MODE_ACCURACY => { - zfp_stream_set_accuracy(zfp, cfg.accuracy); + zfp_stream_set_accuracy(zfp_stream, cfg.accuracy); } - ZFP_MODE_REVERSIBLE => zfp_stream_set_reversible(zfp), + ZFP_MODE_REVERSIBLE => zfp_stream_set_reversible(zfp_stream), _ => { - zfp_stream_close(zfp); + zfp_stream_close(zfp_stream); return 0; } } @@ -320,7 +361,7 @@ unsafe fn filter_zfp_decompress( let outbuf = libc::malloc(outbuf_size); if outbuf.is_null() { - zfp_stream_close(zfp); + zfp_stream_close(zfp_stream); h5err!("Can't allocate decompression buffer", H5E_PLIST, H5E_CALLBACK); return 0; } @@ -371,20 +412,20 @@ unsafe fn filter_zfp_decompress( if field.is_null() { libc::free(outbuf); - zfp_stream_close(zfp); + zfp_stream_close(zfp_stream); h5err!("Failed to create ZFP field", H5E_PLIST, H5E_CALLBACK); return 0; } let bitstream = stream_open((*buf).cast(), nbytes); - zfp_stream_set_bit_stream(zfp, bitstream); - zfp_stream_rewind(zfp); + zfp_stream_set_bit_stream(zfp_stream, bitstream); + zfp_stream_rewind(zfp_stream); - let status = zfp_decompress(zfp, field); + let status = zfp_decompress(zfp_stream, field); stream_close(bitstream); zfp_field_free(field); - zfp_stream_close(zfp); + zfp_stream_close(zfp_stream); if status == 0 { libc::free(outbuf); From eb317eb9b0c0bd087f8d8d5301ff48bbf9523f3c Mon Sep 17 00:00:00 2001 From: Frank Blubaugh Date: Wed, 3 Dec 2025 11:28:44 -0500 Subject: [PATCH 080/141] typo fixes --- hdf5/src/hl/filters/zfp.rs | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/hdf5/src/hl/filters/zfp.rs b/hdf5/src/hl/filters/zfp.rs index 94e6881d0..58eda3dc8 100644 --- a/hdf5/src/hl/filters/zfp.rs +++ b/hdf5/src/hl/filters/zfp.rs @@ -134,34 +134,33 @@ impl From for zfp_sys::zfp_config { zfp_sys::zfp_config__bindgen_ty_1 { rate: cfg.rate } - } + }, ZFP_MODE_PRECISION => { zfp_sys::zfp_config__bindgen_ty_1 { precision: cfg.precision - }, - } + } + }, ZFP_MODE_ACCURACY => { zfp_sys::zfp_config__bindgen_ty_1 { tolerance: cfg.accuracy } - } + }, ZFP_MODE_REVERSIBLE => { zfp_sys::zfp_config__bindgen_ty_1 { - tolerance: cfg.precision + tolerance: cfg.accuracy } - } + }, _ => { h5err!("Invalid ZFP mode", H5E_PLIST, H5E_CALLBACK); - return None; } - } + }; zfp_sys::zfp_config { mode: cfg.mode, - binding: binding_output, + arg: binding_output, } } From 34d0fc2c4de5db868d2f45acb7d3961a6756aa17 Mon Sep 17 00:00:00 2001 From: Frank Blubaugh Date: Wed, 3 Dec 2025 11:37:15 -0500 Subject: [PATCH 081/141] removed a conversion I don't think I need --- hdf5/src/hl/filters/zfp.rs | 38 -------------------------------------- 1 file changed, 38 deletions(-) diff --git a/hdf5/src/hl/filters/zfp.rs b/hdf5/src/hl/filters/zfp.rs index 58eda3dc8..f383bc52e 100644 --- a/hdf5/src/hl/filters/zfp.rs +++ b/hdf5/src/hl/filters/zfp.rs @@ -126,45 +126,7 @@ struct ZfpConfig { pub accuracy: f64, } -impl From for zfp_sys::zfp_config { - fn from(cfg: ZfpConfig) -> Self { - - let binding_output = match cfg.mode { - ZFP_MODE_RATE => { - zfp_sys::zfp_config__bindgen_ty_1 { - rate: cfg.rate - } - }, - ZFP_MODE_PRECISION => { - zfp_sys::zfp_config__bindgen_ty_1 { - precision: cfg.precision - } - }, - ZFP_MODE_ACCURACY => { - zfp_sys::zfp_config__bindgen_ty_1 { - tolerance: cfg.accuracy - } - - }, - ZFP_MODE_REVERSIBLE => { - zfp_sys::zfp_config__bindgen_ty_1 { - tolerance: cfg.accuracy - } - - }, - _ => { - h5err!("Invalid ZFP mode", H5E_PLIST, H5E_CALLBACK); - } - }; - - - zfp_sys::zfp_config { - mode: cfg.mode, - arg: binding_output, - } - } -} fn parse_zfp_cdata(cd_nelmts: size_t, cd_values: *const c_uint) -> Option { let cdata = unsafe { slice::from_raw_parts(cd_values, cd_nelmts as _) }; From a808b3759d2dc590be059dd168b583dda1070181 Mon Sep 17 00:00:00 2001 From: Frank Blubaugh Date: Wed, 3 Dec 2025 12:27:58 -0500 Subject: [PATCH 082/141] debugging work --- hdf5/src/hl/filters/zfp.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/hdf5/src/hl/filters/zfp.rs b/hdf5/src/hl/filters/zfp.rs index f383bc52e..97a82496a 100644 --- a/hdf5/src/hl/filters/zfp.rs +++ b/hdf5/src/hl/filters/zfp.rs @@ -141,6 +141,7 @@ fn parse_zfp_cdata(cd_nelmts: size_t, cd_values: *const c_uint) -> Option 7 { cdata[7] } else { ZFP_MODE_RATE }; let param1 = if cdata.len() > 8 { cdata[8] } else { 0 }; From e888e23839a8b21f455f8d202249faf21d3d164c Mon Sep 17 00:00:00 2001 From: Frank Blubaugh Date: Wed, 3 Dec 2025 12:53:49 -0500 Subject: [PATCH 083/141] more debug statements --- hdf5/src/hl/filters.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hdf5/src/hl/filters.rs b/hdf5/src/hl/filters.rs index 4638c4d2b..9c1e81d17 100644 --- a/hdf5/src/hl/filters.rs +++ b/hdf5/src/hl/filters.rs @@ -478,10 +478,11 @@ impl Filter { let mode = if cdata.len() >= 8 { cdata[7] } else { 1 }; let param1 = if cdata.len() >= 9 { cdata[8] } else { 0 }; let param2 = if cdata.len() >= 10 { cdata[9] } else { 0 }; - + dbg!(param1, param2); let zfp_mode = match mode { 1 => { let rate = f64::from_bits(((param1 as u64) << 32) | (param2 as u64)); + ZfpMode::FixedRate(rate) } 2 => ZfpMode::FixedPrecision(param1 as u8), From fb0b309f448577a6719ac1362f29bd539094ac13 Mon Sep 17 00:00:00 2001 From: Frank Blubaugh Date: Wed, 3 Dec 2025 13:14:49 -0500 Subject: [PATCH 084/141] more debug --- hdf5/src/hl/filters/zfp.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/hdf5/src/hl/filters/zfp.rs b/hdf5/src/hl/filters/zfp.rs index 97a82496a..2422344ae 100644 --- a/hdf5/src/hl/filters/zfp.rs +++ b/hdf5/src/hl/filters/zfp.rs @@ -163,6 +163,7 @@ fn parse_zfp_cdata(cd_nelmts: size_t, cd_values: *const c_uint) -> Option Date: Wed, 3 Dec 2025 13:34:31 -0500 Subject: [PATCH 085/141] more debug --- hdf5/src/hl/filters/zfp.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hdf5/src/hl/filters/zfp.rs b/hdf5/src/hl/filters/zfp.rs index 2422344ae..cfc6c9ad1 100644 --- a/hdf5/src/hl/filters/zfp.rs +++ b/hdf5/src/hl/filters/zfp.rs @@ -141,7 +141,7 @@ fn parse_zfp_cdata(cd_nelmts: size_t, cd_values: *const c_uint) -> Option 7 { cdata[7] } else { ZFP_MODE_RATE }; let param1 = if cdata.len() > 8 { cdata[8] } else { 0 }; From 46762657fe0b01d42289fc9456c664bbc405f8ec Mon Sep 17 00:00:00 2001 From: Frank Blubaugh Date: Wed, 3 Dec 2025 16:32:27 -0500 Subject: [PATCH 086/141] got the bugs cleand up somewhat so now at least the code works. Still need to finish the tests --- hdf5/src/hl/dataset.rs | 10 +- hdf5/src/hl/filters.rs | 959 ++++++++++++++++------------ hdf5/src/hl/filters/zfp.rs | 22 +- hdf5/src/hl/plist/dataset_create.rs | 6 + 4 files changed, 583 insertions(+), 414 deletions(-) diff --git a/hdf5/src/hl/dataset.rs b/hdf5/src/hl/dataset.rs index cc26ee1ac..00280d1f8 100644 --- a/hdf5/src/hl/dataset.rs +++ b/hdf5/src/hl/dataset.rs @@ -14,7 +14,7 @@ use hdf5_sys::h5l::H5Ldelete; use hdf5_sys::h5p::H5P_DEFAULT; use hdf5_sys::h5z::H5Z_filter_t; use hdf5_types::{OwnedDynValue, TypeDescriptor}; - +use crate::hl; #[cfg(feature = "blosc")] use crate::hl::filters::{Blosc, BloscShuffle}; use crate::hl::filters::{Filter, SZip, ScaleOffset}; @@ -746,22 +746,26 @@ impl DatasetBuilderInner { #[cfg(feature = "zfp")] pub fn zfp_rate(&mut self, rate: f64) { - self.with_dcpl(|p| p.set_filters(&vec![Filter::zfp_rate(rate)])); + hl::filters::zfp::register_zfp().expect("Failed to register ZFP filter"); + self.with_dcpl(|p| p.set_filters(&vec![Filter::zfp_rate(rate)])); } #[cfg(feature = "zfp")] pub fn zfp_precision(&mut self, precision: u8) { + hl::filters::zfp::register_zfp().expect("Failed to register ZFP filter"); self.with_dcpl(|p| p.set_filters(&vec![Filter::zfp_precision(precision)])); } #[cfg(feature = "zfp")] pub fn zfp_accuracy(&mut self, accuracy: f64) { - self.with_dcpl(|p| p.set_filters(&vec![Filter::zfp_accuracy(accuracy)])); + hl::filters::zfp::register_zfp().expect("Failed to register ZFP filter"); + self.with_dcpl(|pl| pl.zfp_accuracy(accuracy)); } #[cfg(feature = "zfp")] pub fn zfp_reversible(&mut self) { + hl::filters::zfp::register_zfp().expect("Failed to register ZFP filter"); self.with_dcpl(|p| p.set_filters(&vec![Filter::zfp_reversible()])); } diff --git a/hdf5/src/hl/filters.rs b/hdf5/src/hl/filters.rs index 9c1e81d17..766d2bc97 100644 --- a/hdf5/src/hl/filters.rs +++ b/hdf5/src/hl/filters.rs @@ -23,7 +23,7 @@ mod blosc; #[cfg(feature = "lzf")] mod lzf; #[cfg(feature = "zfp")] -mod zfp; +pub(crate) mod zfp; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum SZip { @@ -478,11 +478,10 @@ impl Filter { let mode = if cdata.len() >= 8 { cdata[7] } else { 1 }; let param1 = if cdata.len() >= 9 { cdata[8] } else { 0 }; let param2 = if cdata.len() >= 10 { cdata[9] } else { 0 }; - dbg!(param1, param2); let zfp_mode = match mode { 1 => { let rate = f64::from_bits(((param1 as u64) << 32) | (param2 as u64)); - + ZfpMode::FixedRate(rate) } 2 => ZfpMode::FixedPrecision(param1 as u8), @@ -581,7 +580,7 @@ impl Filter { #[cfg(feature = "zfp")] unsafe fn apply_zfp(plist_id: hid_t, mode: ZfpMode) -> herr_t { - let mut cdata: Vec = vec![0; 10]; + let mut cdata: Vec = vec![0; 4]; let (mode_val, param1, param2) = match mode { ZfpMode::FixedRate(rate) => { let bits = rate.to_bits(); @@ -594,9 +593,10 @@ impl Filter { } ZfpMode::Reversible => (5, 0, 0), }; - cdata[7] = mode_val; - cdata[8] = param1; - cdata[9] = param2; + cdata[0] = mode_val; + cdata[1] = param1; + cdata[2] = param2; + Self::apply_user(plist_id, zfp::ZFP_FILTER_ID, &cdata) } @@ -723,6 +723,8 @@ mod tests { use crate::hl::filters::zfp_available; use crate::test::with_tmp_file; use crate::{plist::DatasetCreate, Result}; + use crate::filters::ZfpMode; + use crate::hl::filters::zfp::ZFP_FILTER_ID; #[test] fn test_filter_pipeline() -> Result<()> { @@ -811,145 +813,155 @@ mod tests { Ok(()) } - #[test] - #[cfg(feature = "zfp")] - fn test_zfp_filter() -> Result<()> { - use super::{zfp_available, ZfpMode}; - - assert_eq!(cfg!(feature = "zfp"), zfp_available()); - - if !zfp_available() { - println!("ZFP filter not available, skipping test"); - return Ok(()); - } - - // Test different ZFP modes - let zfp_filters = vec![ - Filter::zfp_rate(8.0), - Filter::zfp_rate(16.0), - Filter::zfp_rate(4.0), - Filter::zfp_precision(16), - Filter::zfp_precision(32), - Filter::zfp_accuracy(1e-3), - Filter::zfp_accuracy(1e-6), - Filter::zfp(ZfpMode::FixedRate(12.5)), - ]; - - for flt in &zfp_filters { - println!("Testing filter: {:?}", flt); - assert!(flt.is_available()); - assert!(flt.encode_enabled()); - assert!(flt.decode_enabled()); - - // Test with float type (ZFP only supports floats) - let pipeline = vec![flt.clone()]; - validate_filters(&pipeline, H5T_class_t::H5T_FLOAT)?; - - let plist = DatasetCreate::try_new()?; - for f in &pipeline { - f.apply_to_plist(plist.id())?; - } - assert_eq!(Filter::extract_pipeline(plist.id())?, pipeline); - - // Test with actual dataset creation for f32 - let res = with_tmp_file(|file| { - file.new_dataset_builder() - .empty::() - .shape((100, 50)) - .chunk((10, 10)) - .with_dcpl(|p| p.set_filters(&pipeline)) - .create("zfp_f32") - .unwrap(); - - let plist = file.dataset("zfp_f32").unwrap().dcpl().unwrap(); - Filter::extract_pipeline(plist.id()).unwrap() - }); - assert_eq!(res, pipeline); - - // Test with f64 - let res_f64 = with_tmp_file(|file| { - file.new_dataset_builder() - .empty::() - .shape((100, 50)) - .chunk((10, 10)) - .with_dcpl(|p| p.set_filters(&pipeline)) - .create("zfp_f64") - .unwrap(); - - let plist = file.dataset("zfp_f64").unwrap().dcpl().unwrap(); - Filter::extract_pipeline(plist.id()).unwrap() - }); - assert_eq!(res_f64, pipeline); - } - - Ok(()) - } + // #[test] + // #[cfg(feature = "zfp")] + // fn test_zfp_filter() -> Result<()> { + // use super::{zfp_available, ZfpMode}; + // + // assert_eq!(cfg!(feature = "zfp"), zfp_available()); + // + // if !zfp_available() { + // println!("ZFP filter not available, skipping test"); + // return Ok(()); + // } + // + // // Test different ZFP modes + // let zfp_filters = vec![ + // Filter::zfp_rate(8.0), + // Filter::zfp_rate(16.0), + // Filter::zfp_rate(4.0), + // Filter::zfp_precision(16), + // Filter::zfp_precision(32), + // Filter::zfp_accuracy(1e-3), + // Filter::zfp_accuracy(1e-6), + // Filter::zfp(ZfpMode::FixedRate(12.5)), + // ]; + // + // for flt in &zfp_filters { + // println!("Testing filter: {:?}", flt); + // assert!(flt.is_available()); + // assert!(flt.encode_enabled()); + // assert!(flt.decode_enabled()); + // + // // Test with float type (ZFP only supports floats) + // let pipeline = vec![flt.clone()]; + // validate_filters(&pipeline, H5T_class_t::H5T_FLOAT)?; + // + // let plist = DatasetCreate::try_new()?; + // for f in &pipeline { + // f.apply_to_plist(plist.id())?; + // } + // assert_eq!(Filter::extract_pipeline(plist.id())?, pipeline); + // + // // Test with actual dataset creation for f32 + // let res = with_tmp_file(|file| { + // file.new_dataset_builder() + // .empty::() + // .shape((100, 50)) + // .chunk((10, 10)) + // .with_dcpl(|p| p.set_filters(&pipeline)) + // .create("zfp_f32") + // .unwrap(); + // + // let plist = file.dataset("zfp_f32").unwrap().dcpl().unwrap(); + // Filter::extract_pipeline(plist.id()).unwrap() + // }); + // assert_eq!(res, pipeline); + // + // // Test with f64 + // let res_f64 = with_tmp_file(|file| { + // file.new_dataset_builder() + // .empty::() + // .shape((100, 50)) + // .chunk((10, 10)) + // .with_dcpl(|p| p.set_filters(&pipeline)) + // .create("zfp_f64") + // .unwrap(); + // + // let plist = file.dataset("zfp_f64").unwrap().dcpl().unwrap(); + // Filter::extract_pipeline(plist.id()).unwrap() + // }); + // assert_eq!(res_f64, pipeline); + // } + // + // Ok(()) + // } + // + // #[test] + // #[cfg(feature = "zfp")] + // fn test_zfp_reversible() -> Result<()> { + // use super::zfp_available; + // + // if !zfp_available() { + // println!("ZFP filter not available, skipping test"); + // return Ok(()); + // } + // with_tmp_file(|file| { + // let data = ndarray::Array1::::linspace(0.0, 1000.0, 100000); + // file.new_dataset_builder() + // .zfp_reversible() + // .with_data(&data) + // .chunk((10000,)) + // .create("zfp_lossless_1d") + // .unwrap(); + // + // let ds = file.dataset("zfp_lossless_1d").unwrap(); + // // get number of bytes of ds + // let n_bytes = file.size(); + // let read_data: Vec = ds.read_raw().unwrap(); + // let error = data.iter().zip(read_data.iter()).map(|(a, b)| (a - b).abs()).sum::() + // / data.len() as f32; + // + // let target_bytes = 100000 * 4; + // assert!( + // n_bytes <= target_bytes, + // "Dataset size {} exceeds target {}", + // n_bytes, + // target_bytes + // ); + // assert_eq!(n_bytes, 249368); + // assert_eq!(error, 0.0); + // }); + // Ok(()) + // } #[test] #[cfg(feature = "zfp")] - fn test_zfp_reversible() -> Result<()> { + fn test_zfp_accuracy() -> Result<()>{ use super::zfp_available; if !zfp_available() { println!("ZFP filter not available, skipping test"); + dbg!("HERE"); + assert_eq!(1,0); return Ok(()); } + let mode = ZfpMode::FixedAccuracy(10.0); with_tmp_file(|file| { - let data = ndarray::Array1::::linspace(0.0, 1000.0, 100000); + let data = ndarray::Array1::::linspace(0.0, 1.0, 1000); file.new_dataset_builder() - .zfp_reversible() .with_data(&data) - .chunk((10000,)) - .create("zfp_lossless_1d") + .zfp_accuracy(0.1) + .chunk((1000,)) + .create("zfp_precision_1d") .unwrap(); - let ds = file.dataset("zfp_lossless_1d").unwrap(); - // get number of bytes of ds - let n_bytes = file.size(); - let read_data: Vec = ds.read_raw().unwrap(); - let error = data.iter().zip(read_data.iter()).map(|(a, b)| (a - b).abs()).sum::() - / data.len() as f32; - - let target_bytes = 100000 * 4; - assert!( - n_bytes <= target_bytes, - "Dataset size {} exceeds target {}", - n_bytes, - target_bytes - ); - assert_eq!(n_bytes, 249368); - assert_eq!(error, 0.0); - }); - Ok(()) - } - #[test] - #[cfg(feature = "zfp")] - fn test_zfp_roundtrip_1d() -> Result<()> { - use super::zfp_available; - if !zfp_available() { - println!("ZFP filter not available, skipping test"); - return Ok(()); - } - let pipeline = vec![Filter::zfp_rate(16.0)]; - with_tmp_file(|file| { - let data = ndarray::Array1::::linspace(0.0, 1.0, 1000); - file.new_dataset_builder() - .with_data(&data) - .chunk((100,)) - .with_dcpl(|p| p.set_filters(&pipeline)) - .create("zfp_1d_dcpl") - .unwrap(); + let ds = file.dataset("zfp_precision_1d").unwrap(); + assert_eq!(ds.filters(), vec![crate::hl::filters::Filter::zfp_accuracy(0.1)]); - let ds = file.dataset("zfp_1d_dcpl").unwrap(); let read_data: Vec = ds.read_raw().unwrap(); // ZFP is lossy, so we check approximate equality assert_eq!(read_data.len(), data.len()); + dbg!(&data.clone().into_raw_vec_and_offset().0[0..15]); + dbg!(&read_data[0..15]); + assert_eq!(1,0); for (i, (original, compressed)) in data.iter().zip(read_data.iter()).enumerate() { let diff = (original - compressed).abs(); + dbg!(&diff); assert!( - diff < 0.1, + diff > 10.01, "Index {}: difference too large: {} vs {} (diff: {})", i, original, @@ -962,334 +974,471 @@ mod tests { Ok(()) } - #[test] - #[cfg(feature = "zfp")] - fn test_zfp_roundtrip_2d() -> Result<()> { - use super::zfp_available; - - if !zfp_available() { - println!("ZFP filter not available, skipping test"); - return Ok(()); - } - - with_tmp_file(|file| { - let data: Vec = (0..1000).map(|i| (i as f64) * 0.01).collect(); - let data = Array2::from_shape_vec((10, 100), data).unwrap(); - - let pipteline = vec![Filter::zfp_precision(32)]; - file.new_dataset_builder() - .with_data(&data) - .chunk((10, 10)) - .with_dcpl(|p| p.set_filters(&pipteline)) - .create("zfp_2d") - .unwrap(); - - let ds = file.dataset("zfp_2d").unwrap(); - let read_data: Vec = ds.read_raw().unwrap(); - - assert_eq!(read_data.len(), data.len()); - for (original, compressed) in data.iter().zip(read_data.iter()) { - let diff = (original - compressed).abs(); - assert!(diff < 0.01, "Difference too large: {} vs {}", original, compressed); - } - }); - - Ok(()) - } - - #[test] - #[cfg(feature = "zfp")] - fn test_zfp_roundtrip_3d() -> Result<()> { - use super::zfp_available; - - if !zfp_available() { - println!("ZFP filter not available, skipping test"); - return Ok(()); - } - let pipeline = vec![Filter::zfp_accuracy(1e-4)]; - with_tmp_file(|file| { - let data: Vec = (0..1000).map(|i| (i as f32) * 0.001).collect(); - let data = ndarray::Array3::from_shape_vec((10, 10, 10), data).unwrap(); - - file.new_dataset_builder() - .with_data(&data) - .chunk((5, 5, 5)) - .with_dcpl(|p| p.set_filters(&pipeline)) - .create("zfp_3d") - .unwrap(); - - let ds = file.dataset("zfp_3d").unwrap(); - let read_data: Vec = ds.read_raw().unwrap(); - - assert_eq!(read_data.len(), data.len()); - }); - - Ok(()) - } #[test] #[cfg(feature = "zfp")] - fn test_zfp_roundtrip_4d() -> Result<()> { + fn test_zfp_reversible() -> Result<()>{ use super::zfp_available; if !zfp_available() { println!("ZFP filter not available, skipping test"); + dbg!("HERE"); + assert_eq!(1,0); return Ok(()); } - let pipeline = vec![Filter::zfp_rate(10.0)]; with_tmp_file(|file| { - let data: Vec = (0..256).map(|i| (i as f64) * 0.1).collect(); - let data = ndarray::Array4::from_shape_vec((4, 4, 4, 4), data).unwrap(); - + let data = ndarray::Array1::::linspace(0.0, 1.0, 1000); file.new_dataset_builder() .with_data(&data) - .chunk((2, 2, 2, 2)) - .with_dcpl(|p| p.set_filters(&pipeline)) - .create("zfp_4d") + .zfp_reversible() + .chunk((1000,)) + .create("zfp_reversible") .unwrap(); - let ds = file.dataset("zfp_4d").unwrap(); - let read_data: Vec = ds.read_raw().unwrap(); - - assert_eq!(read_data.len(), data.len()); - }); - - Ok(()) - } - - #[test] - #[cfg(feature = "zfp")] - fn test_zfp_mode_parsing() -> Result<()> { - use super::ZfpMode; - - // Test FixedRate parsing - let rate_bits = 12.5_f64.to_bits(); - let cdata_rate = vec![ - 0, - 1, - 4, - 100, - 0, - 0, - 0, - 1, // mode = rate - (rate_bits >> 32) as u32, - rate_bits as u32, - ]; - let filter = Filter::from_raw(super::zfp::ZFP_FILTER_ID, &cdata_rate)?; - if let Filter::Zfp(ZfpMode::FixedRate(rate)) = filter { - assert!((rate - 12.5).abs() < 1e-10); - } else { - panic!("Expected FixedRate mode, got: {:?}", filter); - } - - // Test FixedPrecision parsing - let cdata_precision = vec![ - 0, 1, 8, 100, 0, 0, 0, 2, // mode = precision - 24, // precision - 0, - ]; - let filter = Filter::from_raw(super::zfp::ZFP_FILTER_ID, &cdata_precision)?; - if let Filter::Zfp(ZfpMode::FixedPrecision(precision)) = filter { - assert_eq!(precision, 24); - } else { - panic!("Expected FixedPrecision mode, got: {:?}", filter); - } - - // Test FixedAccuracy parsing - let accuracy_bits = 1e-5_f64.to_bits(); - let cdata_accuracy = vec![ - 0, - 1, - 8, - 100, - 0, - 0, - 0, - 3, // mode = accuracy - (accuracy_bits >> 32) as u32, - accuracy_bits as u32, - ]; - let filter = Filter::from_raw(super::zfp::ZFP_FILTER_ID, &cdata_accuracy)?; - if let Filter::Zfp(ZfpMode::FixedAccuracy(accuracy)) = filter { - assert!((accuracy - 1e-5).abs() < 1e-10); - } else { - panic!("Expected FixedAccuracy mode, got: {:?}", filter); - } - - Ok(()) - } - - #[test] - #[cfg(feature = "zfp")] - fn test_zfp_with_other_filters() -> Result<()> { - use super::zfp_available; - - if !zfp_available() { - println!("ZFP filter not available, skipping test"); - return Ok(()); - } - let pipeline = vec![Filter::zfp_rate(8.0)]; - // Test ZFP combined with shuffle (shuffle should come first) - with_tmp_file(|file| { - let data: Vec = (0..1000).map(|i| (i as f32) * 0.1).collect(); - let data = ndarray::Array1::from_shape_vec(1000, data).unwrap(); - // file.new_dataset_builder() - // .with_data(&data) - // .chunk(100) - // .with_dcpl(|p| p.set_filters(&pipeline)) - // .create("zfp_rate_8").unwrap(); - // - file.new_dataset_builder() - .zfp_rate(8.0) - .with_data(&data) - .chunk(100) - .create("zfp_rate_8") - .unwrap(); + let ds = file.dataset("zfp_reversible").unwrap(); + assert_eq!(ds.filters(), vec![crate::hl::filters::Filter::zfp_reversible()]); - let ds = file.dataset("zfp_rate_8").unwrap(); let read_data: Vec = ds.read_raw().unwrap(); - let error = data.iter().zip(read_data.iter()).map(|(a, b)| (a - b).abs()).sum::() - / data.len() as f32; - assert_eq!(error, 0.082505114); + // ZFP is lossy, so we check approximate equality assert_eq!(read_data.len(), data.len()); + dbg!(&data.clone().into_raw_vec_and_offset().0[0..15]); + dbg!(&read_data[0..15]); + for (i, (original, compressed)) in data.iter().zip(read_data.iter()).enumerate() { + let diff = (original - compressed).abs(); + dbg!(&diff); + assert!( + diff ==0.0, + "Index {}: difference too large: {} vs {} (diff: {})", + i, + original, + compressed, + diff + ); + } }); - // Test ZFP with fletcher32 checksum - if super::deflate_available() { - let pipeline = vec![Filter::zfp_precision(24), Filter::fletcher32()]; - with_tmp_file(|file| { - let data: Vec = (0..500).map(|i| (i as f64) * 0.01).collect(); - let data = ndarray::Array1::from_shape_vec(500, data).unwrap(); - - file.new_dataset_builder() - .with_data(&data) - .chunk(50) - .with_dcpl(|p| p.set_filters(&pipeline)) - .create("zfp_with_fletcher32") - .unwrap(); - - let ds = file.dataset("zfp_with_fletcher32").unwrap(); - let read_data: Vec = ds.read_raw().unwrap(); - - assert_eq!(read_data.len(), data.len()); - }); - } - Ok(()) } - #[test] - #[cfg(feature = "zfp")] - fn test_zfp_compression_ratios() -> Result<()> { - use super::zfp_available; - - if !zfp_available() { - println!("ZFP filter not available, skipping test"); - return Ok(()); - } - - let pipeline = vec![Filter::zfp_rate(32.0)]; - let pipeline2 = vec![Filter::zfp_rate(4.0)]; - - with_tmp_file(|file| { - let data: Vec = (0..10000).map(|i| (i as f32).sin()).collect(); - let data = ndarray::Array1::from_shape_vec(10000, data).unwrap(); - // Higher rate = less compression but better quality - file.new_dataset_builder() - .with_data(&data) - .chunk(1000) - .with_dcpl(|p| p.set_filters(&pipeline)) - .create("zfp_high_rate") - .unwrap(); - - // Lower rate = more compression but lower quality - - file.new_dataset_builder() - .with_data(&data) - .chunk(1000) - .with_dcpl(|p| p.set_filters(&pipeline2)) - .create("zfp_low_rate") - .unwrap(); - - let ds_high = file.dataset("zfp_high_rate").unwrap(); - let ds_low = file.dataset("zfp_low_rate").unwrap(); - let read_high: Vec = ds_high.read_raw().unwrap(); - let read_low: Vec = ds_low.read_raw().unwrap(); - - // High rate should have better accuracy - let error_high: f32 = - data.iter().zip(read_high.iter()).map(|(a, b)| (a - b).abs()).sum::() - / data.len() as f32; - - let error_low: f32 = - data.iter().zip(read_low.iter()).map(|(a, b)| (a - b).abs()).sum::() - / data.len() as f32; - - println!("High rate error: {}, Low rate error: {}", error_high, error_low); - assert!(error_high < error_low || error_high < 0.001); - }); - - Ok(()) - } #[test] #[cfg(feature = "zfp")] - fn test_zfp_edge_cases() -> Result<()> { + fn test_zfp_rate() -> Result<()>{ use super::zfp_available; if !zfp_available() { println!("ZFP filter not available, skipping test"); + dbg!("HERE"); + assert_eq!(1,0); return Ok(()); } - - let pipeline = vec![Filter::zfp_rate(8.0)]; - - // Test with all zeros with_tmp_file(|file| { - let data: Vec = vec![0.0; 1000]; - let data = ndarray::Array1::from_shape_vec(1000, data).unwrap(); - + let data = ndarray::Array1::::linspace(0.0, 1.0, 1000); file.new_dataset_builder() .with_data(&data) - .chunk(100) - .with_dcpl(|p| p.set_filters(&pipeline)) - .create("zfp_zeros") + .zfp_rate(2.0) + .chunk((1000,)) + .create("zfp_reversible") .unwrap(); - let ds = file.dataset("zfp_zeros").unwrap(); - let read_data: Vec = ds.read_raw().unwrap(); - assert_eq!(read_data.len(), data.len()); - for &val in &read_data { - assert_eq!(val, 0.0); - } - }); - - let pipeline = vec![Filter::zfp_accuracy(1e-6)]; - // Test with constant values - with_tmp_file(|file| { - let data: Vec = vec![42.0; 500]; - let data = ndarray::Array1::from_shape_vec(500, data).unwrap(); - file.new_dataset_builder() - .with_data(&data) - .chunk(50) - .with_dcpl(|p| p.set_filters(&pipeline)) - .create("zfp_constant") - .unwrap(); + let ds = file.dataset("zfp_reversible").unwrap(); + assert_eq!(ds.filters(), vec![crate::hl::filters::Filter::zfp_rate(2.0)]); - let ds = file.dataset("zfp_constant").unwrap(); - let read_data: Vec = ds.read_raw().unwrap(); + let read_data: Vec = ds.read_raw().unwrap(); + // ZFP is lossy, so we check approximate equality assert_eq!(read_data.len(), data.len()); - for &val in &read_data { - assert!((val - 42.0).abs() < 1e-5); + dbg!(&data.clone().into_raw_vec_and_offset().0[0..15]); + dbg!(&read_data[0..15]); + for (i, (original, compressed)) in data.iter().zip(read_data.iter()).enumerate() { + let diff = (original - compressed).abs(); + dbg!(&diff); + assert!( + diff ==0.0, + "Index {}: difference too large: {} vs {} (diff: {})", + i, + original, + compressed, + diff + ); } }); Ok(()) } + + // #[test] + // #[cfg(feature = "zfp")] + // fn test_zfp_roundtrip_1d() -> Result<()> { + // use super::zfp_available; + // + // if !zfp_available() { + // println!("ZFP filter not available, skipping test"); + // return Ok(()); + // } + // let pipeline = vec![Filter::zfp_rate(16.0)]; + // with_tmp_file(|file| { + // let data = ndarray::Array1::::linspace(0.0, 1.0, 1000); + // file.new_dataset_builder() + // .with_data(&data) + // .chunk((100,)) + // .with_dcpl(|p| p.set_filters(&pipeline)) + // .create("zfp_1d_dcpl") + // .unwrap(); + // + // let ds = file.dataset("zfp_1d_dcpl").unwrap(); + // let read_data: Vec = ds.read_raw().unwrap(); + // + // // ZFP is lossy, so we check approximate equality + // assert_eq!(read_data.len(), data.len()); + // for (i, (original, compressed)) in data.iter().zip(read_data.iter()).enumerate() { + // let diff = (original - compressed).abs(); + // assert!( + // diff < 0.1, + // "Index {}: difference too large: {} vs {} (diff: {})", + // i, + // original, + // compressed, + // diff + // ); + // } + // }); + // + // Ok(()) + // } + // + // #[test] + // #[cfg(feature = "zfp")] + // fn test_zfp_roundtrip_2d() -> Result<()> { + // use super::zfp_available; + // + // if !zfp_available() { + // println!("ZFP filter not available, skipping test"); + // return Ok(()); + // } + // + // with_tmp_file(|file| { + // let data: Vec = (0..1000).map(|i| (i as f64) * 0.01).collect(); + // let data = Array2::from_shape_vec((10, 100), data).unwrap(); + // + // let pipteline = vec![Filter::zfp_precision(32)]; + // file.new_dataset_builder() + // .with_data(&data) + // .chunk((10, 10)) + // .with_dcpl(|p| p.set_filters(&pipteline)) + // .create("zfp_2d") + // .unwrap(); + // + // let ds = file.dataset("zfp_2d").unwrap(); + // let read_data: Vec = ds.read_raw().unwrap(); + // + // assert_eq!(read_data.len(), data.len()); + // for (original, compressed) in data.iter().zip(read_data.iter()) { + // let diff = (original - compressed).abs(); + // assert!(diff < 0.01, "Difference too large: {} vs {}", original, compressed); + // } + // }); + // + // Ok(()) + // } + // + // #[test] + // #[cfg(feature = "zfp")] + // fn test_zfp_roundtrip_3d() -> Result<()> { + // use super::zfp_available; + // + // if !zfp_available() { + // println!("ZFP filter not available, skipping test"); + // return Ok(()); + // } + // let pipeline = vec![Filter::zfp_accuracy(1e-4)]; + // with_tmp_file(|file| { + // let data: Vec = (0..1000).map(|i| (i as f32) * 0.001).collect(); + // let data = ndarray::Array3::from_shape_vec((10, 10, 10), data).unwrap(); + // + // file.new_dataset_builder() + // .with_data(&data) + // .chunk((5, 5, 5)) + // .with_dcpl(|p| p.set_filters(&pipeline)) + // .create("zfp_3d") + // .unwrap(); + // + // let ds = file.dataset("zfp_3d").unwrap(); + // let read_data: Vec = ds.read_raw().unwrap(); + // + // assert_eq!(read_data.len(), data.len()); + // }); + // + // Ok(()) + // } + // + // #[test] + // #[cfg(feature = "zfp")] + // fn test_zfp_roundtrip_4d() -> Result<()> { + // use super::zfp_available; + // + // if !zfp_available() { + // println!("ZFP filter not available, skipping test"); + // return Ok(()); + // } + // let pipeline = vec![Filter::zfp_rate(10.0)]; + // with_tmp_file(|file| { + // let data: Vec = (0..256).map(|i| (i as f64) * 0.1).collect(); + // let data = ndarray::Array4::from_shape_vec((4, 4, 4, 4), data).unwrap(); + // + // file.new_dataset_builder() + // .with_data(&data) + // .chunk((2, 2, 2, 2)) + // .with_dcpl(|p| p.set_filters(&pipeline)) + // .create("zfp_4d") + // .unwrap(); + // + // let ds = file.dataset("zfp_4d").unwrap(); + // let read_data: Vec = ds.read_raw().unwrap(); + // + // assert_eq!(read_data.len(), data.len()); + // }); + // + // Ok(()) + // } + // + // #[test] + // #[cfg(feature = "zfp")] + // fn test_zfp_mode_parsing() -> Result<()> { + // use super::ZfpMode; + // + // // Test FixedRate parsing + // let rate_bits = 12.5_f64.to_bits(); + // let cdata_rate = vec![ + // 0, + // 1, + // 4, + // 100, + // 0, + // 0, + // 0, + // 1, // mode = rate + // (rate_bits >> 32) as u32, + // rate_bits as u32, + // ]; + // let filter = Filter::from_raw(super::zfp::ZFP_FILTER_ID, &cdata_rate)?; + // if let Filter::Zfp(ZfpMode::FixedRate(rate)) = filter { + // assert!((rate - 12.5).abs() < 1e-10); + // } else { + // panic!("Expected FixedRate mode, got: {:?}", filter); + // } + // + // // Test FixedPrecision parsing + // let cdata_precision = vec![ + // 0, 1, 8, 100, 0, 0, 0, 2, // mode = precision + // 24, // precision + // 0, + // ]; + // let filter = Filter::from_raw(super::zfp::ZFP_FILTER_ID, &cdata_precision)?; + // if let Filter::Zfp(ZfpMode::FixedPrecision(precision)) = filter { + // assert_eq!(precision, 24); + // } else { + // panic!("Expected FixedPrecision mode, got: {:?}", filter); + // } + // + // // Test FixedAccuracy parsing + // let accuracy_bits = 1e-5_f64.to_bits(); + // let cdata_accuracy = vec![ + // 0, + // 1, + // 8, + // 100, + // 0, + // 0, + // 0, + // 3, // mode = accuracy + // (accuracy_bits >> 32) as u32, + // accuracy_bits as u32, + // ]; + // let filter = Filter::from_raw(super::zfp::ZFP_FILTER_ID, &cdata_accuracy)?; + // if let Filter::Zfp(ZfpMode::FixedAccuracy(accuracy)) = filter { + // assert!((accuracy - 1e-5).abs() < 1e-10); + // } else { + // panic!("Expected FixedAccuracy mode, got: {:?}", filter); + // } + // + // Ok(()) + // } + // + // #[test] + // #[cfg(feature = "zfp")] + // fn test_zfp_with_other_filters() -> Result<()> { + // use super::zfp_available; + // + // if !zfp_available() { + // println!("ZFP filter not available, skipping test"); + // return Ok(()); + // } + // + // let pipeline = vec![Filter::zfp_rate(8.0)]; + // // Test ZFP combined with shuffle (shuffle should come first) + // with_tmp_file(|file| { + // let data: Vec = (0..1000).map(|i| (i as f32) * 0.1).collect(); + // let data = ndarray::Array1::from_shape_vec(1000, data).unwrap(); + // // file.new_dataset_builder() + // // .with_data(&data) + // // .chunk(100) + // // .with_dcpl(|p| p.set_filters(&pipeline)) + // // .create("zfp_rate_8").unwrap(); + // // + // file.new_dataset_builder() + // .zfp_rate(8.0) + // .with_data(&data) + // .chunk(100) + // .create("zfp_rate_8") + // .unwrap(); + // + // let ds = file.dataset("zfp_rate_8").unwrap(); + // let read_data: Vec = ds.read_raw().unwrap(); + // + // let error = data.iter().zip(read_data.iter()).map(|(a, b)| (a - b).abs()).sum::() + // / data.len() as f32; + // assert_eq!(error, 0.082505114); + // assert_eq!(read_data.len(), data.len()); + // }); + // + // // Test ZFP with fletcher32 checksum + // if super::deflate_available() { + // let pipeline = vec![Filter::zfp_precision(24), Filter::fletcher32()]; + // with_tmp_file(|file| { + // let data: Vec = (0..500).map(|i| (i as f64) * 0.01).collect(); + // let data = ndarray::Array1::from_shape_vec(500, data).unwrap(); + // + // file.new_dataset_builder() + // .with_data(&data) + // .chunk(50) + // .with_dcpl(|p| p.set_filters(&pipeline)) + // .create("zfp_with_fletcher32") + // .unwrap(); + // + // let ds = file.dataset("zfp_with_fletcher32").unwrap(); + // let read_data: Vec = ds.read_raw().unwrap(); + // + // assert_eq!(read_data.len(), data.len()); + // }); + // } + // + // Ok(()) + // } + // + // #[test] + // #[cfg(feature = "zfp")] + // fn test_zfp_compression_ratios() -> Result<()> { + // use super::zfp_available; + // + // if !zfp_available() { + // println!("ZFP filter not available, skipping test"); + // return Ok(()); + // } + // + // let pipeline = vec![Filter::zfp_rate(32.0)]; + // let pipeline2 = vec![Filter::zfp_rate(4.0)]; + // + // with_tmp_file(|file| { + // let data: Vec = (0..10000).map(|i| (i as f32).sin()).collect(); + // let data = ndarray::Array1::from_shape_vec(10000, data).unwrap(); + // // Higher rate = less compression but better quality + // file.new_dataset_builder() + // .with_data(&data) + // .chunk(1000) + // .with_dcpl(|p| p.set_filters(&pipeline)) + // .create("zfp_high_rate") + // .unwrap(); + // + // // Lower rate = more compression but lower quality + // + // file.new_dataset_builder() + // .with_data(&data) + // .chunk(1000) + // .with_dcpl(|p| p.set_filters(&pipeline2)) + // .create("zfp_low_rate") + // .unwrap(); + // + // let ds_high = file.dataset("zfp_high_rate").unwrap(); + // let ds_low = file.dataset("zfp_low_rate").unwrap(); + // + // let read_high: Vec = ds_high.read_raw().unwrap(); + // let read_low: Vec = ds_low.read_raw().unwrap(); + // + // // High rate should have better accuracy + // let error_high: f32 = + // data.iter().zip(read_high.iter()).map(|(a, b)| (a - b).abs()).sum::() + // / data.len() as f32; + // + // let error_low: f32 = + // data.iter().zip(read_low.iter()).map(|(a, b)| (a - b).abs()).sum::() + // / data.len() as f32; + // + // println!("High rate error: {}, Low rate error: {}", error_high, error_low); + // assert!(error_high < error_low || error_high < 0.001); + // }); + // + // Ok(()) + // } + // + // #[test] + // #[cfg(feature = "zfp")] + // fn test_zfp_edge_cases() -> Result<()> { + // use super::zfp_available; + // + // if !zfp_available() { + // println!("ZFP filter not available, skipping test"); + // return Ok(()); + // } + // + // let pipeline = vec![Filter::zfp_rate(8.0)]; + // + // // Test with all zeros + // with_tmp_file(|file| { + // let data: Vec = vec![0.0; 1000]; + // let data = ndarray::Array1::from_shape_vec(1000, data).unwrap(); + // + // file.new_dataset_builder() + // .with_data(&data) + // .chunk(100) + // .with_dcpl(|p| p.set_filters(&pipeline)) + // .create("zfp_zeros") + // .unwrap(); + // + // let ds = file.dataset("zfp_zeros").unwrap(); + // let read_data: Vec = ds.read_raw().unwrap(); + // + // assert_eq!(read_data.len(), data.len()); + // for &val in &read_data { + // assert_eq!(val, 0.0); + // } + // }); + // + // let pipeline = vec![Filter::zfp_accuracy(1e-6)]; + // // Test with constant values + // with_tmp_file(|file| { + // let data: Vec = vec![42.0; 500]; + // let data = ndarray::Array1::from_shape_vec(500, data).unwrap(); + // file.new_dataset_builder() + // .with_data(&data) + // .chunk(50) + // .with_dcpl(|p| p.set_filters(&pipeline)) + // .create("zfp_constant") + // .unwrap(); + // + // let ds = file.dataset("zfp_constant").unwrap(); + // let read_data: Vec = ds.read_raw().unwrap(); + // + // assert_eq!(read_data.len(), data.len()); + // for &val in &read_data { + // assert!((val - 42.0).abs() < 1e-5); + // } + // }); + // + // Ok(()) + // } } diff --git a/hdf5/src/hl/filters/zfp.rs b/hdf5/src/hl/filters/zfp.rs index cfc6c9ad1..0a5e51964 100644 --- a/hdf5/src/hl/filters/zfp.rs +++ b/hdf5/src/hl/filters/zfp.rs @@ -65,6 +65,7 @@ extern "C" fn set_local_zfp(dcpl_id: hid_t, type_id: hid_t, _space_id: hid_t) -> const MAX_NDIMS: usize = 4; let mut flags: c_uint = 0; let mut nelmts: size_t = 4; + // start with a small buffer; H5Pget_filter_by_id2 will return the stored cdata (mode/params) let mut values: Vec = vec![0; 4]; let ret = unsafe { H5Pget_filter_by_id2( @@ -81,7 +82,12 @@ extern "C" fn set_local_zfp(dcpl_id: hid_t, type_id: hid_t, _space_id: hid_t) -> if ret < 0 { return -1; } + // Preserve original small cdata (mode/params) returned by H5Pget_filter_by_id2. + let orig = values.clone(); + // ensure we have enough space for header + dims + parameters (we need at least indices up to 9) nelmts = nelmts.max(10); + values.resize(nelmts as usize, 0); + // set version and header entries values[0] = ZFP_FILTER_VERSION; let mut chunkdims: Vec = vec![0; MAX_NDIMS]; @@ -99,12 +105,18 @@ extern "C" fn set_local_zfp(dcpl_id: hid_t, type_id: hid_t, _space_id: hid_t) -> return -1; } + // fill header fields (ndims, typesize) and chunk dimensions values[1] = ndims as c_uint; values[2] = typesize as c_uint; - for i in 0..ndims as usize { - if i + 3 < values.len() { - values[i + 3] = chunkdims[i] as c_uint; - } + for i in 0..(ndims as usize).min(values.len().saturating_sub(3)) { + values[i + 3] = chunkdims[i] as c_uint; + } + // The Filter::apply_zfp() originally stored mode/param1/param2 at indices 0..2. + // parse_zfp expects these at indices 7..9 in the final cdata layout. Move/preserve them. + if values.len() >= 10 { + values[7] = orig.get(0).copied().unwrap_or(0); + values[8] = orig.get(1).copied().unwrap_or(0); + values[9] = orig.get(2).copied().unwrap_or(0); } let r = unsafe { H5Pmodify_filter(dcpl_id, ZFP_FILTER_ID, flags, nelmts, values.as_ptr()) }; @@ -141,7 +153,6 @@ fn parse_zfp_cdata(cd_nelmts: size_t, cd_values: *const c_uint) -> Option 7 { cdata[7] } else { ZFP_MODE_RATE }; let param1 = if cdata.len() > 8 { cdata[8] } else { 0 }; @@ -163,7 +174,6 @@ fn parse_zfp_cdata(cd_nelmts: size_t, cd_values: *const c_uint) -> Option &mut Self { + self.filters.push(Filter::zfp_accuracy(accuracy)); + self + } + pub fn add_filter(&mut self, id: H5Z_filter_t, cdata: &[c_uint]) -> &mut Self { self.filters.push(Filter::user(id, cdata)); self From b197fd092dc2b3618738c5213fa4b6f4aae81be3 Mon Sep 17 00:00:00 2001 From: Frank Blubaugh Date: Wed, 3 Dec 2025 21:20:39 -0500 Subject: [PATCH 087/141] working ish --- hdf5/src/hl/filters.rs | 50 ++++++++++++++++------------- hdf5/src/hl/plist/dataset_create.rs | 18 +++++++++++ 2 files changed, 45 insertions(+), 23 deletions(-) diff --git a/hdf5/src/hl/filters.rs b/hdf5/src/hl/filters.rs index 766d2bc97..821f9ef74 100644 --- a/hdf5/src/hl/filters.rs +++ b/hdf5/src/hl/filters.rs @@ -580,7 +580,7 @@ impl Filter { #[cfg(feature = "zfp")] unsafe fn apply_zfp(plist_id: hid_t, mode: ZfpMode) -> herr_t { - let mut cdata: Vec = vec![0; 4]; + let mut cdata: Vec = vec![0; 10]; let (mode_val, param1, param2) = match mode { ZfpMode::FixedRate(rate) => { let bits = rate.to_bits(); @@ -593,10 +593,7 @@ impl Filter { } ZfpMode::Reversible => (5, 0, 0), }; - cdata[0] = mode_val; - cdata[1] = param1; - cdata[2] = param2; - + dbg!("HERE"); Self::apply_user(plist_id, zfp::ZFP_FILTER_ID, &cdata) } @@ -927,16 +924,15 @@ mod tests { #[test] #[cfg(feature = "zfp")] - fn test_zfp_accuracy() -> Result<()>{ + fn test_zfp_accuracy() -> Result<()> { use super::zfp_available; if !zfp_available() { println!("ZFP filter not available, skipping test"); dbg!("HERE"); - assert_eq!(1,0); + assert_eq!(1, 0); return Ok(()); } - let mode = ZfpMode::FixedAccuracy(10.0); with_tmp_file(|file| { let data = ndarray::Array1::::linspace(0.0, 1.0, 1000); file.new_dataset_builder() @@ -956,12 +952,11 @@ mod tests { assert_eq!(read_data.len(), data.len()); dbg!(&data.clone().into_raw_vec_and_offset().0[0..15]); dbg!(&read_data[0..15]); - assert_eq!(1,0); for (i, (original, compressed)) in data.iter().zip(read_data.iter()).enumerate() { let diff = (original - compressed).abs(); dbg!(&diff); assert!( - diff > 10.01, + diff < 0.1, "Index {}: difference too large: {} vs {} (diff: {})", i, original, @@ -977,21 +972,21 @@ mod tests { #[test] #[cfg(feature = "zfp")] - fn test_zfp_reversible() -> Result<()>{ + fn test_zfp_reversible() -> Result<()> { use super::zfp_available; if !zfp_available() { println!("ZFP filter not available, skipping test"); dbg!("HERE"); - assert_eq!(1,0); + assert_eq!(1, 0); return Ok(()); } with_tmp_file(|file| { - let data = ndarray::Array1::::linspace(0.0, 1.0, 1000); + let data = ndarray::Array1::::linspace(0.0, 1.0, 10000); file.new_dataset_builder() - .with_data(&data) .zfp_reversible() - .chunk((1000,)) + .with_data(&data) + .chunk((10000,)) .create("zfp_reversible") .unwrap(); @@ -1000,16 +995,26 @@ mod tests { assert_eq!(ds.filters(), vec![crate::hl::filters::Filter::zfp_reversible()]); let read_data: Vec = ds.read_raw().unwrap(); + let n_bytes = file.size(); // ZFP is lossy, so we check approximate equality assert_eq!(read_data.len(), data.len()); dbg!(&data.clone().into_raw_vec_and_offset().0[0..15]); dbg!(&read_data[0..15]); + let target_bytes = (data.len() * 4) as u64; + assert!( + n_bytes <= target_bytes, + "Dataset size {} exceeds target {}", + n_bytes, + target_bytes + ); + assert_eq!(n_bytes, 6560); + // for (i, (original, compressed)) in data.iter().zip(read_data.iter()).enumerate() { let diff = (original - compressed).abs(); dbg!(&diff); assert!( - diff ==0.0, + diff == 0.0, "Index {}: difference too large: {} vs {} (diff: {})", i, original, @@ -1023,16 +1028,15 @@ mod tests { } - #[test] #[cfg(feature = "zfp")] - fn test_zfp_rate() -> Result<()>{ + fn test_zfp_rate() -> Result<()> { use super::zfp_available; if !zfp_available() { println!("ZFP filter not available, skipping test"); dbg!("HERE"); - assert_eq!(1,0); + assert_eq!(1, 0); return Ok(()); } with_tmp_file(|file| { @@ -1041,11 +1045,11 @@ mod tests { .with_data(&data) .zfp_rate(2.0) .chunk((1000,)) - .create("zfp_reversible") + .create("zfp_rate") .unwrap(); - let ds = file.dataset("zfp_reversible").unwrap(); + let ds = file.dataset("zfp_rate").unwrap(); assert_eq!(ds.filters(), vec![crate::hl::filters::Filter::zfp_rate(2.0)]); let read_data: Vec = ds.read_raw().unwrap(); @@ -1058,7 +1062,7 @@ mod tests { let diff = (original - compressed).abs(); dbg!(&diff); assert!( - diff ==0.0, + diff == 0.0, "Index {}: difference too large: {} vs {} (diff: {})", i, original, @@ -1441,4 +1445,4 @@ mod tests { // // Ok(()) // } -} +} \ No newline at end of file diff --git a/hdf5/src/hl/plist/dataset_create.rs b/hdf5/src/hl/plist/dataset_create.rs index ee7f467c7..a07a32a64 100644 --- a/hdf5/src/hl/plist/dataset_create.rs +++ b/hdf5/src/hl/plist/dataset_create.rs @@ -482,6 +482,24 @@ impl DatasetCreateBuilder { self } + #[cfg(feature="zfp")] + pub fn zfp_rate(&mut self, rate: f64) -> &mut Self { + self.filters.push(Filter::zfp_rate(rate)); + self + } + + #[cfg(feature="zfp")] + pub fn zfp_precision(&mut self, precision: u8) -> &mut Self { + self.filters.push(Filter::zfp_precision(precision)); + self + } + + #[cfg(feature="zfp")] + pub fn zfp_reversible(&mut self) -> &mut Self { + self.filters.push(Filter::zfp_reversible()); + self + } + pub fn add_filter(&mut self, id: H5Z_filter_t, cdata: &[c_uint]) -> &mut Self { self.filters.push(Filter::user(id, cdata)); self From 868d4ea25b335940b17cc63c55cb7103bd9b3e97 Mon Sep 17 00:00:00 2001 From: Frank Blubaugh Date: Thu, 4 Dec 2025 16:36:26 -0500 Subject: [PATCH 088/141] this is now outputting the headers that h5py can decompose --- Cargo.toml | 18 + hdf5/src/hl/filters.rs | 88 ++++- hdf5/src/hl/filters/produce_zfp_header.rs | 182 ++++++++++ hdf5/src/hl/filters/zfp.rs | 424 +++++++++++++++++++++- 4 files changed, 688 insertions(+), 24 deletions(-) create mode 100644 hdf5/src/hl/filters/produce_zfp_header.rs diff --git a/Cargo.toml b/Cargo.toml index 304b46f8b..2542e555f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,3 +31,21 @@ hdf5-derive = { package = "hdf5-metno-derive", version = "0.9.1", path = "hdf5-d hdf5-src = { package = "hdf5-metno-src", version = "0.9.3", path = "hdf5-src" } # !V hdf5-sys = { package = "hdf5-metno-sys", version = "0.10.1", path = "hdf5-sys" } # !V hdf5-types = { package = "hdf5-metno-types", version = "0.10.0", path = "hdf5-types" } # !V + + +[profile.dev] +# Fast compile, reasonable runtime +opt-level = 0 # no optimizations = much faster compilation +debug = 1 # keep minimal debug info (0 is smaller, 1 is a nice compromise) +incremental = true # reuse previous compilation work +codegen-units = 16 # more parallel codegen units = faster compiles +lto = "off" # no link-time optimization +panic = "unwind" # default, keeps backtraces usable +overflow-checks = true # keep checks for safety in dev + +[profile.test] +# Inherit dev as a baseline, then tweak for tests +inherits = "dev" +opt-level = 0 # keep tests compiling fast +debug = 1 # enough info for backtraces +overflow-checks = true \ No newline at end of file diff --git a/hdf5/src/hl/filters.rs b/hdf5/src/hl/filters.rs index 821f9ef74..f424cc8d3 100644 --- a/hdf5/src/hl/filters.rs +++ b/hdf5/src/hl/filters.rs @@ -25,6 +25,13 @@ mod lzf; #[cfg(feature = "zfp")] pub(crate) mod zfp; + +#[cfg(feature = "zfp")] +mod produce_zfp_header; + +#[cfg(feature = "zfp")] +use zfp_sys::zfp_type_zfp_type_float; + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum SZip { Entropy, @@ -107,6 +114,8 @@ pub use blosc_impl::*; #[cfg(feature = "zfp")] mod zfp_impl { + use crate::filters::ZfpMode::Reversible; + #[derive(Clone, Copy, Debug)] pub enum ZfpMode { FixedRate(f64), @@ -115,6 +124,7 @@ mod zfp_impl { Reversible, } + // Bitwise compare f64 so NaN and signed zero are deterministic impl PartialEq for ZfpMode { fn eq(&self, other: &Self) -> bool { @@ -135,6 +145,48 @@ mod zfp_impl { ZfpMode::FixedRate(4.0) } } + + #[derive(Clone, Debug,Eq,PartialEq)] + pub struct FieldParam{ + pub data_type_bytes: usize, + pub dims: Vec + } + + #[derive(Clone, Debug)] + pub enum ZfpModeNew { + FixedRate(f64,FieldParam), + FixedPrecision(u8,FieldParam), + FixedAccuracy(f64,FieldParam), + Reversible(FieldParam), + } + + // Bitwise compare f64 so NaN and signed zero are deterministic + impl PartialEq for ZfpModeNew { + fn eq(&self, other: &Self) -> bool { + use ZfpModeNew::*; + match (self, other) { + (FixedRate(a,c), FixedRate(b,d)) => (a.to_bits() == b.to_bits()) & (c==d), + (FixedPrecision(a,c), FixedPrecision(b,d)) => (a == b) & (c == d), + (FixedAccuracy(a,c), FixedAccuracy(b,d)) => (a.to_bits() == b.to_bits()) & (c==d), + (Reversible(c), Reversible(d)) => c==d, + _ => false, + } + } + } + impl Eq for ZfpModeNew {} + + impl Default for ZfpModeNew { + fn default() -> Self { + let field_param = FieldParam{ + data_type_bytes: 4, + dims: vec![960] + }; + ZfpModeNew::FixedRate(4.0,field_param) + } + } + + + } #[cfg(feature = "zfp")] @@ -579,8 +631,10 @@ impl Filter { } #[cfg(feature = "zfp")] + /// Ok for the compute hdr cd values, matlab and other implementations natively pack and + /// squeeze singleton dimensions. This is important because it tells ZFP how many dimensions + /// its going to be oeperating over. so this code needs to do that as well; unsafe fn apply_zfp(plist_id: hid_t, mode: ZfpMode) -> herr_t { - let mut cdata: Vec = vec![0; 10]; let (mode_val, param1, param2) = match mode { ZfpMode::FixedRate(rate) => { let bits = rate.to_bits(); @@ -593,8 +647,14 @@ impl Filter { } ZfpMode::Reversible => (5, 0, 0), }; - dbg!("HERE"); - Self::apply_user(plist_id, zfp::ZFP_FILTER_ID, &cdata) + let cdata = [mode_val, param1, param2]; + + // update dims TODO finish fixing this so it can handle arbitrary sized inputs + + let (hdr_cd_values, hdr_cd_nelmts) = zfp::compute_hdr_cd_values(zfp_type_zfp_type_float,1,&[960], mode); + let hdf_cd_values_pass = hdr_cd_values.iter().map(|x| *x).collect::>(); + dbg!(hdr_cd_values); + Self::apply_user(plist_id, zfp::ZFP_FILTER_ID, &hdf_cd_values_pass) } unsafe fn apply_user(plist_id: hid_t, filter_id: H5Z_filter_t, cdata: &[c_uint]) -> herr_t { @@ -710,7 +770,7 @@ pub(crate) fn validate_filters(filters: &[Filter], type_class: H5T_class_t) -> R #[cfg(test)] mod tests { use hdf5_sys::h5t::H5T_class_t; - use ndarray::Array2; + use ndarray::{Array2, Axis}; use std::io::{Seek, SeekFrom}; use super::{ @@ -937,14 +997,13 @@ mod tests { let data = ndarray::Array1::::linspace(0.0, 1.0, 1000); file.new_dataset_builder() .with_data(&data) - .zfp_accuracy(0.1) + .zfp_accuracy(0.125) .chunk((1000,)) .create("zfp_precision_1d") .unwrap(); let ds = file.dataset("zfp_precision_1d").unwrap(); - assert_eq!(ds.filters(), vec![crate::hl::filters::Filter::zfp_accuracy(0.1)]); let read_data: Vec = ds.read_raw().unwrap(); @@ -952,6 +1011,7 @@ mod tests { assert_eq!(read_data.len(), data.len()); dbg!(&data.clone().into_raw_vec_and_offset().0[0..15]); dbg!(&read_data[0..15]); + assert_eq!(1,0); for (i, (original, compressed)) in data.iter().zip(read_data.iter()).enumerate() { let diff = (original - compressed).abs(); dbg!(&diff); @@ -982,21 +1042,25 @@ mod tests { return Ok(()); } with_tmp_file(|file| { - let data = ndarray::Array1::::linspace(0.0, 1.0, 10000); + let data = ndarray::Array1::::linspace(0.0, 1.0, 9600); + let data = data.insert_axis(Axis(0)); + let data = data.insert_axis(Axis(0)); file.new_dataset_builder() .zfp_reversible() .with_data(&data) - .chunk((10000,)) + .chunk((1,1,960)) .create("zfp_reversible") .unwrap(); let ds = file.dataset("zfp_reversible").unwrap(); - assert_eq!(ds.filters(), vec![crate::hl::filters::Filter::zfp_reversible()]); + + let read_data: Vec = ds.read_raw().unwrap(); let n_bytes = file.size(); + // ZFP is lossy, so we check approximate equality assert_eq!(read_data.len(), data.len()); dbg!(&data.clone().into_raw_vec_and_offset().0[0..15]); @@ -1008,7 +1072,8 @@ mod tests { n_bytes, target_bytes ); - assert_eq!(n_bytes, 6560); + assert_eq!(n_bytes, 29176); + // assert_eq!(1,0); // for (i, (original, compressed)) in data.iter().zip(read_data.iter()).enumerate() { let diff = (original - compressed).abs(); @@ -1445,4 +1510,5 @@ mod tests { // // Ok(()) // } -} \ No newline at end of file +} + diff --git a/hdf5/src/hl/filters/produce_zfp_header.rs b/hdf5/src/hl/filters/produce_zfp_header.rs new file mode 100644 index 000000000..5993d291b --- /dev/null +++ b/hdf5/src/hl/filters/produce_zfp_header.rs @@ -0,0 +1,182 @@ +use std::ptr::{self, addr_of_mut}; +use std::slice; +use std::sync::LazyLock; + +use hdf5_sys::h5p::{H5Pget_chunk, H5Pget_filter_by_id2, H5Pmodify_filter}; +use hdf5_sys::h5t::{H5Tclose, H5Tget_class, H5Tget_size, H5Tget_super, H5T_FLOAT}; +use hdf5_sys::h5z::{H5Z_class2_t, H5Z_filter_t, H5Zregister, H5Z_CLASS_T_VERS, H5Z_FLAG_REVERSE}; +use std::mem; + + +use crate::error::H5ErrorCode; +use crate::globals::{H5E_CALLBACK, H5E_PLIST}; +use crate::internal_prelude::*; + + +#[cfg(feature = "zfp")] +use zfp_sys; + +// Import from zfp-sys (or your FFI bindings) +#[cfg(feature = "zfp")] +use zfp_sys::{zfp_field_metadata,zfp_stream_rewind,ZFP_VERSION_PATCH,ZFP_VERSION_MINOR,ZFP_VERSION_MAJOR,zfp_type_zfp_type_float,zfp_field_1d,zfp_field_4d, zfp_field_2d,zfp_field_3d,zfp_stream,zfp_stream_open,zfp_stream_set_mode,zfp_stream_set_reversible,zfp_stream_maximum_size, zfp_write_header,bitstream,stream_open as bs_open,stream_close as bs_close,zfp_field_free,zfp_stream_close,ZFP_VERSION_TWEAK}; + + +const ZFP_FILTER_NAME: &[u8] = b"zfp\0"; +pub const ZFP_FILTER_ID: H5Z_filter_t = 32013; +const ZFP_FILTER_VERSION: c_uint = 1; + + +const H5Z_FILTER_ZFP_VERSION_MAJOR: u32 = 1; +const H5Z_FILTER_ZFP_VERSION_MINOR: u32 = 0; +const H5Z_FILTER_ZFP_VERSION_PATCH: u32 = 0; + + +// ZFP mode constants +const ZFP_MODE_RATE: c_uint = 1; +const ZFP_MODE_PRECISION: c_uint = 2; +const ZFP_MODE_ACCURACY: c_uint = 3; +const ZFP_MODE_REVERSIBLE: c_uint = 5; + +const H5Z_ZFP_CD_NELMTS_MAX: usize = 8; // whatever the header says; set correctly. +fn build_reversible_zfp_cd_values( + dims: &[usize], // e.g. &[z, y, x] + dtype_size: usize // e.g. 4 for float, 8 for double, or use ints +) -> Option> { + + dbg!(dims); + unsafe { + // 1. Create a ZFP field for the given dimensions and data type + let field = match dims.len() { + 1 => { + // zfp_field_1d + // unimplemented in bindings example; assume exists + zfp_field_1d(ptr::null_mut(), zfp_type_zfp_type_float, dims[0]) + } + 2 => zfp_field_2d(ptr::null_mut(), zfp_type_zfp_type_float, dims[1], dims[0]), + 3 => zfp_field_3d(ptr::null_mut(), zfp_type_zfp_type_float, dims[2], dims[1], dims[0]), + 4 => zfp_field_4d(ptr::null_mut(), zfp_type_zfp_type_float, dims[3], dims[2], dims[1], dims[0]), + _ => return None, + }; + dbg!(&field); + if field.is_null() { + return None; + } + + // 2. Create a ZFP stream + let zstr = zfp_stream_open(ptr::null_mut()); + dbg!(&zstr); + if zstr.is_null() { + zfp_field_free(field); + return None; + } + + // 3. Set stream to reversible mode + zfp_stream_set_reversible(zstr); + + // 4. Compute max header/bitstream buffer size + let max_bytes = 160; + + + // 5. Allocate a buffer to hold the header + data (bitstream) + let mut buf: Vec = vec![0u8; max_bytes]; + let buf_ptr = buf.as_mut_ptr() as *mut c_void; + + + + let buf_size = max_bytes; + + // 6. Open a bitstream in that buffer + let bstr = bs_open(buf_ptr, buf_size); + dbg!(&bstr); + if bstr.is_null() { + zfp_stream_close(zstr); + zfp_field_free(field); + return None; + } + + // 7. Attach bitstream to zfp stream + zfp_sys::zfp_stream_set_bit_stream(zstr, bstr); + + let meta = zfp_sys::zfp_field_metadata(field); + dbg!(meta); + + + + // 8. Write the ZFP header + let bits = zfp_write_header(zstr, field, zfp_sys::ZFP_HEADER_FULL); + dbg!(&bits); + if bits == 0 { + bs_close(bstr); + zfp_stream_close(zstr); + zfp_field_free(field); + return None; + } + + // compute how many bytes used + let header_bytes = 1 + ((bits as usize - 1) / 8); + let hdr_cd_nelmts = 1 + ((header_bytes - 1) / std::mem::size_of::()); + // round up to multiple of u32 (4 bytes) + let num_u32 = (header_bytes + 3) / 4; + + println!("header_bytes = {}", header_bytes); + println!("raw header (hex): {:02X?}", &buf[..header_bytes]); + + + + // 9. Interpret the buffer as u32 CD values (little-endian) + // Safety: we wrote at least header_bytes; we treat as u32 + buf.set_len(hdr_cd_nelmts * 4); + let u32slice: &[u32] = slice::from_raw_parts(buf_ptr as *const u32, num_u32); + dbg!(&buf); + + let cd_values = u32slice.to_vec(); + + // 10. Clean up + bs_close(bstr); + zfp_stream_close(zstr); + zfp_field_free(field); + + + // get the version header right + const ZFP_VERSION_NO: u32 = + (ZFP_VERSION_MAJOR << 12) + | (ZFP_VERSION_MINOR << 8) + | (ZFP_VERSION_PATCH << 4) + | (ZFP_VERSION_TWEAK); + + const H5Z_FILTER_ZFP_VERSION_NO: u32 = + (H5Z_FILTER_ZFP_VERSION_MAJOR << 8) + | (H5Z_FILTER_ZFP_VERSION_MINOR << 4) + | (H5Z_FILTER_ZFP_VERSION_PATCH); + + + let first = (ZFP_VERSION_NO << 16) | (5 << 12) | H5Z_FILTER_ZFP_VERSION_NO; + let mut cd = vec![ first as u32 ]; + cd.extend(&cd_values); + + Some(cd) + } +} + + +#[cfg(test)] +mod test{ + use super::{build_reversible_zfp_cd_values}; + use crate::hl::filters::ZfpMode; + + #[test] + fn reproduce_zfp_header_from_params(){ + + let dataset_size = 10000; + let n_dims = 1; + let zfp_mode = ZfpMode::Reversible; + let zfp_config_vector = [5,0,0]; + let cd_values = build_reversible_zfp_cd_values(&[1,10000], 4); + dbg!(cd_values); + assert_eq!(1,0); + + } + + + +} \ No newline at end of file diff --git a/hdf5/src/hl/filters/zfp.rs b/hdf5/src/hl/filters/zfp.rs index 0a5e51964..48f16febe 100644 --- a/hdf5/src/hl/filters/zfp.rs +++ b/hdf5/src/hl/filters/zfp.rs @@ -10,24 +10,32 @@ use crate::error::H5ErrorCode; use crate::globals::{H5E_CALLBACK, H5E_PLIST}; use crate::internal_prelude::*; -use zfp_sys::zfp_stream_set_reversible; pub use zfp_sys::{ - stream_close, stream_open, zfp_compress, zfp_decompress, zfp_field_1d, zfp_field_2d, - zfp_field_3d, zfp_field_4d, zfp_field_free, zfp_stream_close, zfp_stream_maximum_size, - zfp_stream_open, zfp_stream_rewind, zfp_stream_set_accuracy, zfp_stream_set_bit_stream, - zfp_stream_set_precision, zfp_stream_set_rate, zfp_type_zfp_type_double, - zfp_type_zfp_type_float, + zfp_field_alloc,zfp_read_header,zfp_field_dimensionality,zfp_field_size,zfp_field_type,zfp_mode,zfp_mode_zfp_mode_fixed_accuracy,zfp_mode_zfp_mode_fixed_precision,zfp_mode_zfp_mode_fixed_rate,zfp_stream_compression_mode,zfp_stream_accuracy,zfp_stream_rate,zfp_stream_precision,stream_close, stream_open, zfp_compress, zfp_field_metadata,zfp_decompress, zfp_field_1d, zfp_field_2d,bitstream,zfp_stream_set_reversible, + zfp_field_3d, zfp_field_4d, zfp_field_free, zfp_stream_close, zfp_stream_maximum_size,zfp_stream_flush, + zfp_stream_open, zfp_stream_rewind, zfp_stream_set_accuracy, zfp_stream_set_bit_stream,ZFP_VERSION_MINOR,ZFP_VERSION_PATCH, + zfp_stream_set_precision, zfp_stream_set_rate, zfp_type_zfp_type_double,zfp_type,ZFP_HEADER_FULL,ZFP_VERSION_MAJOR,ZFP_VERSION_TWEAK, + zfp_type_zfp_type_float,ZFP_HEADER_MAGIC,ZFP_HEADER_MAX_BITS,ZFP_HEADER_META,ZFP_HEADER_MODE,zfp_write_header,zfp_codec_version,zfp_library_version,zfp_field }; +use zfp_sys::zfp_stream; + +use crate::filters::ZfpMode; + + +/// Major edits are needed to be in alignmeht with the H5Z-ZFP. What was previously implemented was effectively a new implementation of H5Z_ZFP but was incompatible with any library built against it. This reults in bad c_data vectors being created and produces erratic behavior. + const ZFP_FILTER_NAME: &[u8] = b"zfp\0"; pub const ZFP_FILTER_ID: H5Z_filter_t = 32013; const ZFP_FILTER_VERSION: c_uint = 1; // ZFP mode constants -const ZFP_MODE_RATE: c_uint = 1; -const ZFP_MODE_PRECISION: c_uint = 2; -const ZFP_MODE_ACCURACY: c_uint = 3; +const ZFP_MODE_RATE: c_uint = 2; +const ZFP_MODE_PRECISION: c_uint = 3; +const ZFP_MODE_ACCURACY: c_uint = 4; const ZFP_MODE_REVERSIBLE: c_uint = 5; +const ZFP_MODE_EXPERT: c_uint = 1; + const ZFP_FILTER_INFO: &H5Z_class2_t = &H5Z_class2_t { version: H5Z_CLASS_T_VERS as _, @@ -61,6 +69,7 @@ extern "C" fn can_apply_zfp(_dcpl_id: hid_t, type_id: hid_t, _space_id: hid_t) - } } + extern "C" fn set_local_zfp(dcpl_id: hid_t, type_id: hid_t, _space_id: hid_t) -> herr_t { const MAX_NDIMS: usize = 4; let mut flags: c_uint = 0; @@ -117,9 +126,12 @@ extern "C" fn set_local_zfp(dcpl_id: hid_t, type_id: hid_t, _space_id: hid_t) -> values[7] = orig.get(0).copied().unwrap_or(0); values[8] = orig.get(1).copied().unwrap_or(0); values[9] = orig.get(2).copied().unwrap_or(0); + } + // temp overrid and changed line 133 to orig instead of values + let nelmts = 4; - let r = unsafe { H5Pmodify_filter(dcpl_id, ZFP_FILTER_ID, flags, nelmts, values.as_ptr()) }; + let r = unsafe { H5Pmodify_filter(dcpl_id, ZFP_FILTER_ID, flags, nelmts, orig.as_ptr()) }; if r < 0 { -1 } else { @@ -127,6 +139,229 @@ extern "C" fn set_local_zfp(dcpl_id: hid_t, type_id: hid_t, _space_id: hid_t) -> } } + +fn pack_header_into_cd_values( + header_bytes: &[u8], + bits_written: usize, +) -> Vec { + let total_bytes = (bits_written + 7) / 8; + let nwords = (total_bytes + 3) / 4; + + let mut cd_vals = Vec::with_capacity(1 + nwords); + + // cd_values[0] = version word (we'll fill it in below) + cd_vals.push(0); + + for i in 0..nwords { + let mut word_bytes = [0u8; 4]; + for j in 0..4 { + let idx = i * 4 + j; + if idx < total_bytes { + word_bytes[j] = header_bytes[idx]; + } + } + cd_vals.push(u32::from_le_bytes(word_bytes)); + } + + cd_vals +} + + +const H5Z_ZFP_CD_NELMTS_MAX: usize = 8; // whatever the header says; set correctly. + +pub unsafe fn compute_hdr_cd_values( + zt: zfp_type, + ndims_used: usize, + dims_used: &[u64], + mode: ZfpMode, // your enum wrapping rate/precision/accuracy/reversible +) -> (Vec, usize) { + // 1. Build dummy_field like H5Z_zfp_set_local + let dummy_field: *mut zfp_field = match ndims_used { + 1 => zfp_field_1d(ptr::null_mut(), zt, dims_used[0].try_into().unwrap()), + 2 => zfp_field_2d(ptr::null_mut(), zt, dims_used[1].try_into().unwrap(), dims_used[0].try_into().unwrap()), + 3 => zfp_field_3d(ptr::null_mut(), zt, dims_used[2].try_into().unwrap(), dims_used[1].try_into().unwrap(), dims_used[0].try_into().unwrap()), + 4 => zfp_field_4d(ptr::null_mut(), zt, dims_used[3].try_into().unwrap(), dims_used[2].try_into().unwrap(), dims_used[1].try_into().unwrap(), dims_used[0].try_into().unwrap()), + _ => panic!("ZFP supports 1..4 non-unity dims"), + }; + assert!(!dummy_field.is_null()); + + // 2. Prepare the cd_values array like C code: u32 buffer + let mut hdr_cd_values = vec![0u32; H5Z_ZFP_CD_NELMTS_MAX]; + + // 3. Version word (use the macro layout: (ZFP_VERSION_NO<<16)|(ZFP_CODEC<<12)|H5Z_FILTER_ZFP_VERSION_NO) + hdr_cd_values[0] = make_version_word(); // see previous message + dbg!(&hdr_cd_values); + + // 4. Treat &hdr_cd_values[1] as bitstream buffer + let ptr_bytes = hdr_cd_values[1..].as_mut_ptr() as *mut c_void; + let bytes_len = (hdr_cd_values.len() - 1) * std::mem::size_of::(); + + let dummy_bstr: *mut bitstream = stream_open(ptr_bytes, bytes_len as usize); + + let dummy_zstr: *mut zfp_stream = zfp_stream_open(dummy_bstr); + // 5. Set mode the same way H5Z_zfp_set_local does + match mode { + ZfpMode::Reversible => { + zfp_stream_set_reversible(dummy_zstr); + }, + ZfpMode::FixedAccuracy(acc) =>{ + dbg!(&acc); + zfp_stream_set_accuracy(dummy_zstr,acc); + }, + // handle Rate/Precision/Accuracy/Expert as needed + _ => unimplemented!(), + } + + let field_meta = zfp_sys::zfp_field_metadata(dummy_field); + dbg!(field_meta); + + // 6. Write FULL header (critical!) into the hdr_cd_values[1..] buffer + let hdr_bits = zfp_write_header(dummy_zstr, dummy_field, ZFP_HEADER_FULL as u32); + assert!(hdr_bits != 0); + + // 7. Flush and close (exactly like C) + zfp_stream_flush(dummy_zstr); + zfp_stream_close(dummy_zstr); + stream_close(dummy_bstr); + zfp_field_free(dummy_field); + + // 8. Compute hdr_bytes/hdr_cd_nelmts as in C + let hdr_bytes = 1 + ((hdr_bits - 1) / 8); + let mut hdr_cd_nelmts = 1 + ((hdr_bytes - 1) / std::mem::size_of::()); + hdr_cd_nelmts += 1; // for slot 0 + + (hdr_cd_values, hdr_cd_nelmts) +} + +unsafe fn make_version_word() -> u32 { + + // 0xM M P T: for 1.0.0.0 → 0x1000 + const ZFP_VERSION_NO: u32 = + (ZFP_VERSION_MAJOR << 12) + | (ZFP_VERSION_MINOR << 8) + | (ZFP_VERSION_PATCH << 4) + | (ZFP_VERSION_TWEAK); + + const ZFP_CODEC: u32 = ZFP_VERSION_MINOR; // or 5 if you know you want codec 5 + + // Filter version: 1.1.0 → 0x0110 + const H5Z_FILTER_ZFP_VERSION_MAJOR: u32 = 1; + const H5Z_FILTER_ZFP_VERSION_MINOR: u32 = 1; + const H5Z_FILTER_ZFP_VERSION_PATCH: u32 = 0; + + const H5Z_FILTER_ZFP_VERSION_NO: u32 = + (H5Z_FILTER_ZFP_VERSION_MAJOR << 8) + | (H5Z_FILTER_ZFP_VERSION_MINOR << 4) + | (H5Z_FILTER_ZFP_VERSION_PATCH); + + // One simple scheme: low 8 bits = codec, high 24 bits = lib version truncated. + (ZFP_VERSION_NO << 16) + | (ZFP_CODEC << 12) + | H5Z_FILTER_ZFP_VERSION_NO +} + + + +pub unsafe fn make_llnl_style_cd_values( + chunk_dims: &[usize], + mode: ZfpMode, +) -> Vec { + let ztype = zfp_type_zfp_type_float; + let dims_used: Vec = chunk_dims + .iter() + .copied() + .filter(|&d| d > 1) + .collect(); + let field = make_zfp_field(ztype, &dims_used); + let zfp_stream = zfp_stream_open(ptr::null_mut()); + + + match mode { + ZfpMode::FixedRate(rate) => { + zfp_stream_set_rate(zfp_stream, rate, std::mem::size_of::() as _, dims_used.len() as _, 0); + } + ZfpMode::FixedPrecision(precision) => { + zfp_stream_set_precision(zfp_stream, precision as u32); + } + ZfpMode::FixedAccuracy(accuracy) => { + zfp_stream_set_accuracy(zfp_stream, accuracy); + } + ZfpMode::Reversible => { + zfp_stream_set_reversible(zfp_stream); + } + }; + + + let (header_bytes, bits_written) = zfp_header_bits(zfp_stream, field); + dbg!(&header_bytes); + dbg!(&bits_written); + let mut cd_vals = pack_header_into_cd_values(&header_bytes, bits_written); + + // plug in version info + cd_vals[0] = make_version_word(); + dbg!(&cd_vals); + unsafe { + zfp_field_free(field); + zfp_stream_close(zfp_stream); + } + + cd_vals +} + + +unsafe fn make_zfp_field(ztype: zfp_type, dims: &[usize]) -> *mut zfp_field { + let mut shape = [1usize; 4]; + for (idx, dim) in dims.iter().copied().take(4).enumerate() { + shape[idx] = dim.max(1); + } + + match dims.len() { + 0 | 1 => zfp_field_1d(ptr::null_mut(), ztype, shape[0]), + 2 => zfp_field_2d(ptr::null_mut(), ztype, shape[0], shape[1]), + 3 => zfp_field_3d(ptr::null_mut(), ztype, shape[0], shape[1], shape[2]), + _ => zfp_field_4d( + ptr::null_mut(), + ztype, + shape[0], + shape[1], + shape[2], + shape[3], + ), + } +} + + +unsafe fn zfp_header_bits( + zstream: *mut zfp_stream, + field: *mut zfp_field, +) -> (Vec, usize) { + // Max bits → bytes; this constant is in the ZFP docs and exposed via zfp-sys + let max_bits = ZFP_HEADER_MAX_BITS as usize; + let max_bytes = 20; + let mut buf = vec![0u8; max_bytes]; + + // Make bitstream over our byte buffer + let bs = stream_open( + buf.as_mut_ptr() as *mut c_void, + max_bytes as usize, + ); + zfp_stream_set_bit_stream(zstream, bs); + zfp_stream_rewind(zstream); + + let mask = ZFP_HEADER_FULL as u32; + let bits_written = zfp_write_header(zstream, field, mask); + + if bits_written == 0 { + panic!("zfp_write_header failed"); + } + + // Clean up bitstream; keep header bytes in buf + stream_close(bs); + + (buf, bits_written as usize) +} + + #[derive(Debug)] struct ZfpConfig { pub ndims: c_int, @@ -139,9 +374,167 @@ struct ZfpConfig { } +/// receive the new cdata from the system and decode it to recover the right ZFP opertaing modes +pub unsafe fn parse_zfp_cdata( + cd_nelmts: usize, + cd_values: *const c_uint, +) -> Option { + if cd_nelmts < 2 || cd_values.is_null() { + return None; + } + + // Full cd array from HDF5: [version_word, header_words...] + let cdata: &[u32] = slice::from_raw_parts(cd_values, cd_nelmts); + + // We currently ignore the version word, but you can decode it if you want. + let _version_word = cdata[0]; + + // Everything after index 0 is the ZFP header bitstream. + let header_words = &cdata[1..]; + if header_words.is_empty() { + return None; + } -fn parse_zfp_cdata(cd_nelmts: size_t, cd_values: *const c_uint) -> Option { + // Make a mutable copy so we can endian-swap in place if needed. + let mut header_copy: Vec = header_words.to_vec(); + let header_bytes = header_copy.len() * std::mem::size_of::(); + + // 1. Open bitstream on the header buffer (like get_zfp_info_from_cd_values) :contentReference[oaicite:2]{index=2} + let bstr: *mut bitstream = + stream_open(header_copy.as_mut_ptr() as *mut c_void, header_bytes); + if bstr.is_null() { + return None; + } + + // 2. Open zfp_stream on that bitstream + let zstr: *mut zfp_stream = zfp_stream_open(bstr); + if zstr.is_null() { + stream_close(bstr); + return None; + } + + // 3. Allocate a field for header metadata + let zfld: *mut zfp_field = zfp_field_alloc(); + if zfld.is_null() { + zfp_stream_close(zstr); + stream_close(bstr); + return None; + } + + // 4. First read only MAGIC, to detect endian or codec mismatch + let mut bits = zfp_read_header(zstr, zfld, ZFP_HEADER_MAGIC); + if bits == 0 { + // Possible endian mismatch: byte-swap each u32 and retry. + for w in &mut header_copy { + *w = w.swap_bytes(); + } + + zfp_stream_rewind(zstr); + bits = zfp_read_header(zstr, zfld, ZFP_HEADER_MAGIC); + if bits == 0 { + // Codec mismatch or truly invalid header. + zfp_field_free(zfld); + zfp_stream_close(zstr); + stream_close(bstr); + return None; + } + } + + // 5. We know magic is fine. Rewind and read the full header. + zfp_stream_rewind(zstr); + if zfp_read_header(zstr, zfld, ZFP_HEADER_FULL) == 0 { + zfp_field_free(zfld); + zfp_stream_close(zstr); + stream_close(bstr); + return None; + } + + // 6. Extract array metadata via high-level API (no manual bit-twiddling). + let ndims = zfp_field_dimensionality(zfld) as i32; + + // zfp_field_size can fill per-dimension sizes if we pass a buffer. + let mut size_per_dim: [usize; 4] = [0; 4]; + if ndims > 0 { + // zfp_field_size returns total number of elements and optionally fills size[i]. + // The C signature uses size_t*; we just alias &mut [usize] here. + zfp_field_size( + zfld, + size_per_dim.as_mut_ptr() as *mut _, + ); + } + + let mut dims: [usize; 4] = [0; 4]; + for i in 0..(ndims as usize).min(4) { + dims[i] = size_per_dim[i]; + } + + // Scalar type → element size in bytes. + let zt: zfp_type = zfp_field_type(zfld); + let typesize: usize = match zt { + // Adjust these to match the actual enum variants in your bindings + x if x == zfp_sys::zfp_type_zfp_type_int32 => std::mem::size_of::(), + x if x == zfp_sys::zfp_type_zfp_type_int64 => std::mem::size_of::(), + x if x == zfp_sys::zfp_type_zfp_type_float => std::mem::size_of::(), + x if x == zfp_sys::zfp_type_zfp_type_double => std::mem::size_of::(), + _ => { + zfp_field_free(zfld); + zfp_stream_close(zstr); + stream_close(bstr); + return None; + } + }; + // 7. Extract compression mode and parameters from the stream itself. + let zmode_enum: zfp_mode = zfp_stream_compression_mode(zstr); + let mode = zmode_enum as u32; + dbg!(&mode); + + let mut rate: f64 = 0.0; + let mut precision: u32 = 0; + let mut accuracy: f64 = 0.0; + + // These getters are available on modern zfp (1.0+). + // If your zfp version is older, you can `cfg`-gate or leave them at zero. + match zmode_enum { + m if m == zfp_sys::zfp_mode_zfp_mode_fixed_rate => { + rate = zfp_stream_rate(zstr, ndims as u32); + } + m if m == zfp_sys::zfp_mode_zfp_mode_fixed_precision => { + precision = zfp_stream_precision(zstr); + } + m if m == zfp_sys::zfp_mode_zfp_mode_fixed_accuracy => { + accuracy = zfp_stream_accuracy(zstr); + }, + m if m == zfp_sys::zfp_mode_zfp_mode_reversible => { + // no params needed + } + + // Expert or reversible -> we don’t have a single scalar parameter to expose + _ => { + // leave rate/precision/accuracy at 0; you can later extend this by + // calling zfp_stream_params() for expert mode. + } + } + dbg!(&accuracy); + + // 8. Cleanup + zfp_field_free(zfld); + zfp_stream_close(zstr); + stream_close(bstr); + + Some(ZfpConfig { + ndims, + typesize, + dims, + mode, + rate, + precision, + accuracy, + }) +} + +fn parse_zfp_cdata_old(cd_nelmts: size_t, cd_values: *const c_uint) -> Option { let cdata = unsafe { slice::from_raw_parts(cd_values, cd_nelmts as _) }; + dbg!(&cdata); if cdata.len() < 7 { h5err!("Invalid ZFP filter configuration", H5E_PLIST, H5E_CALLBACK); return None; @@ -184,9 +577,12 @@ unsafe extern "C" fn filter_zfp( ) -> size_t { let cfg = if let Some(cfg) = parse_zfp_cdata(cd_nelmts, cd_values) { cfg + } else { return 0; }; + dbg!(&cfg); + if flags & H5Z_FLAG_REVERSE == 0 { unsafe { filter_zfp_compress(&cfg, buf_size, buf) } } else { @@ -211,6 +607,7 @@ unsafe fn filter_zfp_compress( zfp_stream_set_precision(zfp_stream, cfg.precision); } ZFP_MODE_ACCURACY => { + dbg!(cfg.accuracy); zfp_stream_set_accuracy(zfp_stream, cfg.accuracy); } ZFP_MODE_REVERSIBLE => zfp_stream_set_reversible(zfp_stream), @@ -220,6 +617,7 @@ unsafe fn filter_zfp_compress( } } + let field = if cfg.typesize == 4 { match cfg.ndims { 1 => zfp_field_1d((*buf).cast(), zfp_type_zfp_type_float, cfg.dims[0]), @@ -263,7 +661,6 @@ unsafe fn filter_zfp_compress( _ => ptr::null_mut(), } }; - if field.is_null() { zfp_stream_close(zfp_stream); h5err!("Failed to create ZFP field", H5E_PLIST, H5E_CALLBACK); @@ -278,13 +675,14 @@ unsafe fn filter_zfp_compress( h5err!("Can't allocate compression buffer", H5E_PLIST, H5E_CALLBACK); return 0; } + println!("Here Outbuf"); let bitstream = stream_open(outbuf.cast(), maxsize); zfp_stream_set_bit_stream(zfp_stream, bitstream); zfp_stream_rewind(zfp_stream); let compressed_size = zfp_compress(zfp_stream, field); - + println!("here compressed size"); stream_close(bitstream); zfp_field_free(field); zfp_stream_close(zfp_stream); From a388e7adf3bebee1a23bde4eda51e876ecd40a38 Mon Sep 17 00:00:00 2001 From: Frank Blubaugh Date: Fri, 5 Dec 2025 10:15:09 -0500 Subject: [PATCH 089/141] Code appears to be working. Now to clean up before writing tests --- hdf5/src/hl/dataset.rs | 24 +++--- hdf5/src/hl/filters.rs | 123 +++++++++++++++++++--------- hdf5/src/hl/filters/zfp.rs | 28 +++++-- hdf5/src/hl/plist/dataset_create.rs | 16 ++-- 4 files changed, 127 insertions(+), 64 deletions(-) diff --git a/hdf5/src/hl/dataset.rs b/hdf5/src/hl/dataset.rs index 00280d1f8..17fce7fed 100644 --- a/hdf5/src/hl/dataset.rs +++ b/hdf5/src/hl/dataset.rs @@ -745,28 +745,28 @@ impl DatasetBuilderInner { #[cfg(feature = "zfp")] - pub fn zfp_rate(&mut self, rate: f64) { + pub fn zfp_rate(&mut self, rate: f64,chunk_dims: Vec,n_bytes: u8) { hl::filters::zfp::register_zfp().expect("Failed to register ZFP filter"); - self.with_dcpl(|p| p.set_filters(&vec![Filter::zfp_rate(rate)])); + self.with_dcpl(|p| p.set_filters(&vec![Filter::zfp_rate(rate,chunk_dims.clone(),n_bytes)])); } #[cfg(feature = "zfp")] - pub fn zfp_precision(&mut self, precision: u8) { + pub fn zfp_precision(&mut self, precision: u8,chunk_dims: Vec,n_bytes: u8) { hl::filters::zfp::register_zfp().expect("Failed to register ZFP filter"); - self.with_dcpl(|p| p.set_filters(&vec![Filter::zfp_precision(precision)])); + self.with_dcpl(|p| p.set_filters(&vec![Filter::zfp_precision(precision,chunk_dims.clone(),n_bytes)])); } #[cfg(feature = "zfp")] - pub fn zfp_accuracy(&mut self, accuracy: f64) { + pub fn zfp_accuracy(&mut self, accuracy: f64,chunk_dims: Vec,n_bytes: u8) { hl::filters::zfp::register_zfp().expect("Failed to register ZFP filter"); - self.with_dcpl(|pl| pl.zfp_accuracy(accuracy)); + self.with_dcpl(|pl| pl.zfp_accuracy(accuracy,chunk_dims.clone(),n_bytes)); } #[cfg(feature = "zfp")] - pub fn zfp_reversible(&mut self) { + pub fn zfp_reversible(&mut self,chunk_dims: Vec,n_bytes: u8) { hl::filters::zfp::register_zfp().expect("Failed to register ZFP filter"); - self.with_dcpl(|p| p.set_filters(&vec![Filter::zfp_reversible()])); + self.with_dcpl(|p| p.set_filters(&vec![Filter::zfp_reversible(chunk_dims.clone(),n_bytes)])); } @@ -1032,19 +1032,19 @@ macro_rules! impl_builder_methods { impl_builder!( #[cfg(feature = "zfp")] - DatasetCreate: zfp_rate(rate: f64) + DatasetCreate: zfp_rate(rate: f64,chunk_dims: Vec,n_bytes: u8) ); impl_builder!( #[cfg(feature = "zfp")] - DatasetCreate: zfp_accuracy(accuracy: f64) + DatasetCreate: zfp_accuracy(accuracy: f64,chunk_dims: Vec,n_bytes: u8) ); impl_builder!( #[cfg(feature = "zfp")] - DatasetCreate: zfp_precision(rate: u8) + DatasetCreate: zfp_precision(rate: u8,chunk_dims: Vec,n_bytes: u8) ); impl_builder!( #[cfg(feature = "zfp")] - DatasetCreate: zfp_reversible() + DatasetCreate: zfp_reversible(chunk_dims: Vec,n_bytes: u8) ); diff --git a/hdf5/src/hl/filters.rs b/hdf5/src/hl/filters.rs index f424cc8d3..efd040cea 100644 --- a/hdf5/src/hl/filters.rs +++ b/hdf5/src/hl/filters.rs @@ -1,11 +1,8 @@ use std::collections::HashMap; use std::ptr::{self, addr_of_mut}; -use hdf5_sys::h5p::{ - H5Pget_filter2, H5Pget_nfilters, H5Pset_deflate, H5Pset_filter, H5Pset_fletcher32, H5Pset_nbit, - H5Pset_scaleoffset, H5Pset_shuffle, H5Pset_szip, -}; -use hdf5_sys::h5t::H5T_class_t; +use hdf5_sys::h5p::{H5Pget_chunk, H5Pget_filter2, H5Pget_nfilters, H5Pset_deflate, H5Pset_filter, H5Pset_fletcher32, H5Pset_nbit, H5Pset_scaleoffset, H5Pset_shuffle, H5Pset_szip}; +use hdf5_sys::h5t::{H5T_class_t, H5Tclose, H5Tget_class, H5Tget_size, H5T_FLOAT}; use hdf5_sys::h5z::{ H5Zfilter_avail, H5Zget_filter_info, H5Z_FILTER_CONFIG_DECODE_ENABLED, H5Z_FILTER_CONFIG_ENCODE_ENABLED, H5Z_FILTER_DEFLATE, H5Z_FILTER_FLETCHER32, H5Z_FILTER_NBIT, @@ -30,7 +27,7 @@ pub(crate) mod zfp; mod produce_zfp_header; #[cfg(feature = "zfp")] -use zfp_sys::zfp_type_zfp_type_float; +use zfp_sys::{zfp_type_zfp_type_float,zfp_type_zfp_type_double}; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum SZip { @@ -191,6 +188,7 @@ mod zfp_impl { #[cfg(feature = "zfp")] pub use zfp_impl::*; +use crate::globals::{H5E_CALLBACK, H5E_PLIST}; #[derive(Clone, Debug, PartialEq, Eq)] pub enum Filter { @@ -205,7 +203,7 @@ pub enum Filter { #[cfg(feature = "blosc")] Blosc(Blosc, u8, BloscShuffle), #[cfg(feature = "zfp")] - Zfp(ZfpMode), + Zfp(ZfpMode,Vec,u8), User(H5Z_filter_t, Vec), } @@ -278,7 +276,7 @@ impl Filter { #[cfg(feature = "blosc")] Self::Blosc(_, _, _) => blosc::BLOSC_FILTER_ID, #[cfg(feature = "zfp")] - Self::Zfp(_) => zfp::ZFP_FILTER_ID, + Self::Zfp(_,_,_) => zfp::ZFP_FILTER_ID, Self::User(id, _) => *id, } } @@ -394,28 +392,28 @@ impl Filter { } #[cfg(feature = "zfp")] - pub fn zfp(mode: ZfpMode) -> Self { - Self::Zfp(mode) + pub fn zfp(mode: ZfpMode,chunk_dims: Vec,n_bytes: u8) -> Self { + Self::Zfp(mode,chunk_dims,n_bytes) } #[cfg(feature = "zfp")] - pub fn zfp_rate(rate: f64) -> Self { - Self::zfp(ZfpMode::FixedRate(rate)) + pub fn zfp_rate(rate: f64,chunk_dims: Vec,n_bytes: u8) -> Self { + Self::zfp(ZfpMode::FixedRate(rate),chunk_dims,n_bytes) } #[cfg(feature = "zfp")] - pub fn zfp_precision(precision: u8) -> Self { - Self::zfp(ZfpMode::FixedPrecision(precision)) + pub fn zfp_precision(precision: u8,chunk_dims: Vec,n_bytes: u8) -> Self { + Self::zfp(ZfpMode::FixedPrecision(precision),chunk_dims,n_bytes) } #[cfg(feature = "zfp")] - pub fn zfp_accuracy(accuracy: f64) -> Self { - Self::zfp(ZfpMode::FixedAccuracy(accuracy)) + pub fn zfp_accuracy(accuracy: f64,chunk_dims: Vec, n_bytes: u8) -> Self { + Self::zfp(ZfpMode::FixedAccuracy(accuracy),chunk_dims,n_bytes) } #[cfg(feature = "zfp")] - pub fn zfp_reversible() -> Self { - Self::zfp(ZfpMode::Reversible) + pub fn zfp_reversible(chunk_dims: Vec,n_bytes: u8) -> Self { + Self::zfp(ZfpMode::Reversible,chunk_dims,n_bytes) } pub fn user(id: H5Z_filter_t, cdata: &[c_uint]) -> Self { @@ -527,6 +525,8 @@ impl Filter { #[cfg(feature = "zfp")] fn parse_zfp(cdata: &[c_uint]) -> Result { ensure!(cdata.len() >= 8, "expected at least length 8 cdata for zfp filter"); + let chunk_dims = cdata[4..6].iter().map(|&x| x as _).collect::>(); + let n_bytes = cdata[6] as u8; let mode = if cdata.len() >= 8 { cdata[7] } else { 1 }; let param1 = if cdata.len() >= 9 { cdata[8] } else { 0 }; let param2 = if cdata.len() >= 10 { cdata[9] } else { 0 }; @@ -544,7 +544,7 @@ impl Filter { 5 => ZfpMode::Reversible, _ => fail!("invalid zfp mode: {}", mode), }; - Ok(Self::zfp(zfp_mode)) + Ok(Self::zfp(zfp_mode,chunk_dims,n_bytes)) } pub fn from_raw(filter_id: H5Z_filter_t, cdata: &[c_uint]) -> Result { @@ -630,11 +630,61 @@ impl Filter { Self::apply_user(plist_id, blosc::BLOSC_FILTER_ID, &cdata) } + + #[cfg(feature = "zfp")] - /// Ok for the compute hdr cd values, matlab and other implementations natively pack and - /// squeeze singleton dimensions. This is important because it tells ZFP how many dimensions - /// its going to be oeperating over. so this code needs to do that as well; - unsafe fn apply_zfp(plist_id: hid_t, mode: ZfpMode) -> herr_t { + /// Applies the ZFP filter to the given property list. + /// + /// This function configures the ZFP filter for compression on the specified dataset. + /// It determines the data type, removes singleton dimensions, and encodes the mode + /// information into the filter header. + /// + /// # Safety + /// This function is marked as unsafe because it interacts with raw pointers and + /// performs operations that require careful handling to avoid undefined behavior. + /// + /// # Parameters + /// - `plist_id`: The property list identifier to which the ZFP filter will be applied. + /// - `n_bytes`: The number of bytes per data element (4 for `float`, 8 for `double`). + /// - `chunk_dims`: A vector containing the dimensions of the data chunks. + /// - `mode`: The ZFP compression mode, which can be fixed rate, precision, accuracy, or reversible. + /// + /// # Returns + /// - `herr_t`: Returns 0 on success, or a negative value on failure. + unsafe fn apply_zfp(plist_id: hid_t, n_bytes: u8,chunk_dims: Vec, mode: ZfpMode) -> herr_t { + // get the chunk dimensiosn out of it. Could not reliably get the chunk_dims from plist_id + // during testing so opted to just pass it in during the build + + assert!(chunk_dims.len()<= zfp::MAX_NDIMS); + let ndims = chunk_dims.len(); + // Convert to `usize` and trim to used dims. + let chunk_dims_usize: Vec = chunk_dims[..(ndims as usize)] + .iter() + .map(|&d| d as usize) + .collect(); + + + // remove the singletons from the data + let mut dims_no_singleton: Vec = Vec::new(); + for &dim in chunk_dims_usize.iter() { + if dim != 1 { + dims_no_singleton.push(dim as u64); + } + } + let ndims_no_singleton = dims_no_singleton.len(); + + // Get the type of the input data + let dtype_id = match n_bytes{ + 4 => zfp_type_zfp_type_float, + 8 => zfp_type_zfp_type_double, + _ => { + h5err!("ZFP filter only supports 4 or 8 byte floating point data", H5E_PLIST, H5E_CALLBACK); + return -1; + } + }; + + + // Build the Mode Information we need let (mode_val, param1, param2) = match mode { ZfpMode::FixedRate(rate) => { let bits = rate.to_bits(); @@ -647,13 +697,10 @@ impl Filter { } ZfpMode::Reversible => (5, 0, 0), }; - let cdata = [mode_val, param1, param2]; - // update dims TODO finish fixing this so it can handle arbitrary sized inputs - - let (hdr_cd_values, hdr_cd_nelmts) = zfp::compute_hdr_cd_values(zfp_type_zfp_type_float,1,&[960], mode); + // update values and encode into the header + let (hdr_cd_values, _) = zfp::compute_hdr_cd_values(dtype_id,ndims_no_singleton,&dims_no_singleton, mode); let hdf_cd_values_pass = hdr_cd_values.iter().map(|x| *x).collect::>(); - dbg!(hdr_cd_values); Self::apply_user(plist_id, zfp::ZFP_FILTER_ID, &hdf_cd_values_pass) } @@ -682,7 +729,7 @@ impl Filter { Self::apply_blosc(id, *complib, *clevel, *shuffle) } #[cfg(feature = "zfp")] - Self::Zfp(mode) => Self::apply_zfp(id, *mode), + Self::Zfp(mode,chunk_dims, n_bytes) => Self::apply_zfp(id,*n_bytes,chunk_dims.clone(), *mode), Self::User(filter_id, ref cdata) => Self::apply_user(id, *filter_id, cdata), }); Ok(()) @@ -813,9 +860,9 @@ mod tests { assert_eq!(cfg!(feature = "zfp"), zfp_available()); #[cfg(feature = "zfp")] { - comp_filters.push(Filter::zfp_rate(8.0)); - comp_filters.push(Filter::zfp_precision(16)); - comp_filters.push(Filter::zfp_accuracy(1e-3)); + comp_filters.push(Filter::zfp_rate(8.0,vec![10_000,20],4)); + comp_filters.push(Filter::zfp_precision(16,vec![10_000,20],4)); + comp_filters.push(Filter::zfp_accuracy(1e-3,vec![10_000,20],4)); } for c in &comp_filters { @@ -997,8 +1044,8 @@ mod tests { let data = ndarray::Array1::::linspace(0.0, 1.0, 1000); file.new_dataset_builder() .with_data(&data) - .zfp_accuracy(0.125) .chunk((1000,)) + .zfp_accuracy(0.125,vec![1000],4) .create("zfp_precision_1d") .unwrap(); @@ -1046,9 +1093,9 @@ mod tests { let data = data.insert_axis(Axis(0)); let data = data.insert_axis(Axis(0)); file.new_dataset_builder() - .zfp_reversible() - .with_data(&data) .chunk((1,1,960)) + .zfp_reversible(vec![1,1,960],4) + .with_data(&data) .create("zfp_reversible") .unwrap(); @@ -1072,7 +1119,7 @@ mod tests { n_bytes, target_bytes ); - assert_eq!(n_bytes, 29176); + assert_eq!(n_bytes, 29432); // assert_eq!(1,0); // for (i, (original, compressed)) in data.iter().zip(read_data.iter()).enumerate() { @@ -1108,14 +1155,14 @@ mod tests { let data = ndarray::Array1::::linspace(0.0, 1.0, 1000); file.new_dataset_builder() .with_data(&data) - .zfp_rate(2.0) .chunk((1000,)) + .zfp_rate(2.0,vec![1000],4) .create("zfp_rate") .unwrap(); let ds = file.dataset("zfp_rate").unwrap(); - assert_eq!(ds.filters(), vec![crate::hl::filters::Filter::zfp_rate(2.0)]); + assert_eq!(ds.filters(), vec![crate::hl::filters::Filter::zfp_rate(2.0,vec![1000],4)]); let read_data: Vec = ds.read_raw().unwrap(); diff --git a/hdf5/src/hl/filters/zfp.rs b/hdf5/src/hl/filters/zfp.rs index 48f16febe..3e36800c6 100644 --- a/hdf5/src/hl/filters/zfp.rs +++ b/hdf5/src/hl/filters/zfp.rs @@ -24,6 +24,7 @@ use crate::filters::ZfpMode; /// Major edits are needed to be in alignmeht with the H5Z-ZFP. What was previously implemented was effectively a new implementation of H5Z_ZFP but was incompatible with any library built against it. This reults in bad c_data vectors being created and produces erratic behavior. +pub(crate) const MAX_NDIMS: usize = 4; const ZFP_FILTER_NAME: &[u8] = b"zfp\0"; pub const ZFP_FILTER_ID: H5Z_filter_t = 32013; @@ -205,15 +206,20 @@ pub unsafe fn compute_hdr_cd_values( zfp_stream_set_reversible(dummy_zstr); }, ZfpMode::FixedAccuracy(acc) =>{ - dbg!(&acc); + zfp_stream_set_accuracy(dummy_zstr,acc); }, + + ZfpMode::FixedRate(rate) =>{ + zfp_stream_set_rate(dummy_zstr,rate, zt, ndims_used as u32,0); + }, + ZfpMode::FixedPrecision(precision) =>{ + zfp_stream_set_precision(dummy_zstr,precision as u32); + }, // handle Rate/Precision/Accuracy/Expert as needed _ => unimplemented!(), } - let field_meta = zfp_sys::zfp_field_metadata(dummy_field); - dbg!(field_meta); // 6. Write FULL header (critical!) into the hdr_cd_values[1..] buffer let hdr_bits = zfp_write_header(dummy_zstr, dummy_field, ZFP_HEADER_FULL as u32); @@ -233,6 +239,18 @@ pub unsafe fn compute_hdr_cd_values( (hdr_cd_values, hdr_cd_nelmts) } + + +/// Constructs a version word for the ZFP filter. +/// +/// This function generates a 32-bit version word that encodes the ZFP library version, +/// codec version, and filter version. The version word is structured as follows: +/// - High 24 bits: ZFP library version (major, minor, patch, tweak). +/// - Middle 8 bits: Codec version. +/// - Low 8 bits: Filter version. +/// +/// # Returns +/// A 32-bit unsigned integer representing the version word. unsafe fn make_version_word() -> u32 { // 0xM M P T: for 1.0.0.0 → 0x1000 @@ -293,13 +311,11 @@ pub unsafe fn make_llnl_style_cd_values( let (header_bytes, bits_written) = zfp_header_bits(zfp_stream, field); - dbg!(&header_bytes); - dbg!(&bits_written); + let mut cd_vals = pack_header_into_cd_values(&header_bytes, bits_written); // plug in version info cd_vals[0] = make_version_word(); - dbg!(&cd_vals); unsafe { zfp_field_free(field); zfp_stream_close(zfp_stream); diff --git a/hdf5/src/hl/plist/dataset_create.rs b/hdf5/src/hl/plist/dataset_create.rs index a07a32a64..a6ac7a348 100644 --- a/hdf5/src/hl/plist/dataset_create.rs +++ b/hdf5/src/hl/plist/dataset_create.rs @@ -477,26 +477,26 @@ impl DatasetCreateBuilder { } #[cfg(feature="zfp")] - pub fn zfp_accuracy(&mut self, accuracy: f64) -> &mut Self { - self.filters.push(Filter::zfp_accuracy(accuracy)); + pub fn zfp_accuracy(&mut self, accuracy: f64,chunk_dims: Vec, n_bytes:u8) -> &mut Self { + self.filters.push(Filter::zfp_accuracy(accuracy,chunk_dims,n_bytes)); self } #[cfg(feature="zfp")] - pub fn zfp_rate(&mut self, rate: f64) -> &mut Self { - self.filters.push(Filter::zfp_rate(rate)); + pub fn zfp_rate(&mut self, rate: f64,chunk_dims: Vec, n_bytes:u8) -> &mut Self { + self.filters.push(Filter::zfp_rate(rate,chunk_dims,n_bytes)); self } #[cfg(feature="zfp")] - pub fn zfp_precision(&mut self, precision: u8) -> &mut Self { - self.filters.push(Filter::zfp_precision(precision)); + pub fn zfp_precision(&mut self, precision: u8,chunk_dims: Vec, n_bytes:u8) -> &mut Self { + self.filters.push(Filter::zfp_precision(precision,chunk_dims,n_bytes)); self } #[cfg(feature="zfp")] - pub fn zfp_reversible(&mut self) -> &mut Self { - self.filters.push(Filter::zfp_reversible()); + pub fn zfp_reversible(&mut self,chunk_dims: Vec, n_bytes:u8) -> &mut Self { + self.filters.push(Filter::zfp_reversible(chunk_dims,n_bytes)); self } From 826eef80b1028dc8da96b8cabf7de8a5c7c7f1c0 Mon Sep 17 00:00:00 2001 From: Frank Blubaugh Date: Fri, 5 Dec 2025 10:25:31 -0500 Subject: [PATCH 090/141] Added docstrings to all new functions --- hdf5/src/hl/filters.rs | 42 +--- hdf5/src/hl/filters/produce_zfp_header.rs | 182 -------------- hdf5/src/hl/filters/zfp.rs | 276 +++++++--------------- 3 files changed, 92 insertions(+), 408 deletions(-) delete mode 100644 hdf5/src/hl/filters/produce_zfp_header.rs diff --git a/hdf5/src/hl/filters.rs b/hdf5/src/hl/filters.rs index efd040cea..fa510f24c 100644 --- a/hdf5/src/hl/filters.rs +++ b/hdf5/src/hl/filters.rs @@ -23,9 +23,6 @@ mod lzf; pub(crate) mod zfp; -#[cfg(feature = "zfp")] -mod produce_zfp_header; - #[cfg(feature = "zfp")] use zfp_sys::{zfp_type_zfp_type_float,zfp_type_zfp_type_double}; @@ -149,38 +146,6 @@ mod zfp_impl { pub dims: Vec } - #[derive(Clone, Debug)] - pub enum ZfpModeNew { - FixedRate(f64,FieldParam), - FixedPrecision(u8,FieldParam), - FixedAccuracy(f64,FieldParam), - Reversible(FieldParam), - } - - // Bitwise compare f64 so NaN and signed zero are deterministic - impl PartialEq for ZfpModeNew { - fn eq(&self, other: &Self) -> bool { - use ZfpModeNew::*; - match (self, other) { - (FixedRate(a,c), FixedRate(b,d)) => (a.to_bits() == b.to_bits()) & (c==d), - (FixedPrecision(a,c), FixedPrecision(b,d)) => (a == b) & (c == d), - (FixedAccuracy(a,c), FixedAccuracy(b,d)) => (a.to_bits() == b.to_bits()) & (c==d), - (Reversible(c), Reversible(d)) => c==d, - _ => false, - } - } - } - impl Eq for ZfpModeNew {} - - impl Default for ZfpModeNew { - fn default() -> Self { - let field_param = FieldParam{ - data_type_bytes: 4, - dims: vec![960] - }; - ZfpModeNew::FixedRate(4.0,field_param) - } - } @@ -1036,7 +1001,6 @@ mod tests { if !zfp_available() { println!("ZFP filter not available, skipping test"); - dbg!("HERE"); assert_eq!(1, 0); return Ok(()); } @@ -1056,12 +1020,10 @@ mod tests { // ZFP is lossy, so we check approximate equality assert_eq!(read_data.len(), data.len()); - dbg!(&data.clone().into_raw_vec_and_offset().0[0..15]); - dbg!(&read_data[0..15]); + assert_eq!(1,0); for (i, (original, compressed)) in data.iter().zip(read_data.iter()).enumerate() { let diff = (original - compressed).abs(); - dbg!(&diff); assert!( diff < 0.1, "Index {}: difference too large: {} vs {} (diff: {})", @@ -1084,7 +1046,6 @@ mod tests { if !zfp_available() { println!("ZFP filter not available, skipping test"); - dbg!("HERE"); assert_eq!(1, 0); return Ok(()); } @@ -1147,7 +1108,6 @@ mod tests { if !zfp_available() { println!("ZFP filter not available, skipping test"); - dbg!("HERE"); assert_eq!(1, 0); return Ok(()); } diff --git a/hdf5/src/hl/filters/produce_zfp_header.rs b/hdf5/src/hl/filters/produce_zfp_header.rs deleted file mode 100644 index 5993d291b..000000000 --- a/hdf5/src/hl/filters/produce_zfp_header.rs +++ /dev/null @@ -1,182 +0,0 @@ -use std::ptr::{self, addr_of_mut}; -use std::slice; -use std::sync::LazyLock; - -use hdf5_sys::h5p::{H5Pget_chunk, H5Pget_filter_by_id2, H5Pmodify_filter}; -use hdf5_sys::h5t::{H5Tclose, H5Tget_class, H5Tget_size, H5Tget_super, H5T_FLOAT}; -use hdf5_sys::h5z::{H5Z_class2_t, H5Z_filter_t, H5Zregister, H5Z_CLASS_T_VERS, H5Z_FLAG_REVERSE}; -use std::mem; - - -use crate::error::H5ErrorCode; -use crate::globals::{H5E_CALLBACK, H5E_PLIST}; -use crate::internal_prelude::*; - - -#[cfg(feature = "zfp")] -use zfp_sys; - -// Import from zfp-sys (or your FFI bindings) -#[cfg(feature = "zfp")] -use zfp_sys::{zfp_field_metadata,zfp_stream_rewind,ZFP_VERSION_PATCH,ZFP_VERSION_MINOR,ZFP_VERSION_MAJOR,zfp_type_zfp_type_float,zfp_field_1d,zfp_field_4d, zfp_field_2d,zfp_field_3d,zfp_stream,zfp_stream_open,zfp_stream_set_mode,zfp_stream_set_reversible,zfp_stream_maximum_size, zfp_write_header,bitstream,stream_open as bs_open,stream_close as bs_close,zfp_field_free,zfp_stream_close,ZFP_VERSION_TWEAK}; - - -const ZFP_FILTER_NAME: &[u8] = b"zfp\0"; -pub const ZFP_FILTER_ID: H5Z_filter_t = 32013; -const ZFP_FILTER_VERSION: c_uint = 1; - - -const H5Z_FILTER_ZFP_VERSION_MAJOR: u32 = 1; -const H5Z_FILTER_ZFP_VERSION_MINOR: u32 = 0; -const H5Z_FILTER_ZFP_VERSION_PATCH: u32 = 0; - - -// ZFP mode constants -const ZFP_MODE_RATE: c_uint = 1; -const ZFP_MODE_PRECISION: c_uint = 2; -const ZFP_MODE_ACCURACY: c_uint = 3; -const ZFP_MODE_REVERSIBLE: c_uint = 5; - -const H5Z_ZFP_CD_NELMTS_MAX: usize = 8; // whatever the header says; set correctly. -fn build_reversible_zfp_cd_values( - dims: &[usize], // e.g. &[z, y, x] - dtype_size: usize // e.g. 4 for float, 8 for double, or use ints -) -> Option> { - - dbg!(dims); - unsafe { - // 1. Create a ZFP field for the given dimensions and data type - let field = match dims.len() { - 1 => { - // zfp_field_1d - // unimplemented in bindings example; assume exists - zfp_field_1d(ptr::null_mut(), zfp_type_zfp_type_float, dims[0]) - } - 2 => zfp_field_2d(ptr::null_mut(), zfp_type_zfp_type_float, dims[1], dims[0]), - 3 => zfp_field_3d(ptr::null_mut(), zfp_type_zfp_type_float, dims[2], dims[1], dims[0]), - 4 => zfp_field_4d(ptr::null_mut(), zfp_type_zfp_type_float, dims[3], dims[2], dims[1], dims[0]), - _ => return None, - }; - dbg!(&field); - if field.is_null() { - return None; - } - - // 2. Create a ZFP stream - let zstr = zfp_stream_open(ptr::null_mut()); - dbg!(&zstr); - if zstr.is_null() { - zfp_field_free(field); - return None; - } - - // 3. Set stream to reversible mode - zfp_stream_set_reversible(zstr); - - // 4. Compute max header/bitstream buffer size - let max_bytes = 160; - - - // 5. Allocate a buffer to hold the header + data (bitstream) - let mut buf: Vec = vec![0u8; max_bytes]; - let buf_ptr = buf.as_mut_ptr() as *mut c_void; - - - - let buf_size = max_bytes; - - // 6. Open a bitstream in that buffer - let bstr = bs_open(buf_ptr, buf_size); - dbg!(&bstr); - if bstr.is_null() { - zfp_stream_close(zstr); - zfp_field_free(field); - return None; - } - - // 7. Attach bitstream to zfp stream - zfp_sys::zfp_stream_set_bit_stream(zstr, bstr); - - let meta = zfp_sys::zfp_field_metadata(field); - dbg!(meta); - - - - // 8. Write the ZFP header - let bits = zfp_write_header(zstr, field, zfp_sys::ZFP_HEADER_FULL); - dbg!(&bits); - if bits == 0 { - bs_close(bstr); - zfp_stream_close(zstr); - zfp_field_free(field); - return None; - } - - // compute how many bytes used - let header_bytes = 1 + ((bits as usize - 1) / 8); - let hdr_cd_nelmts = 1 + ((header_bytes - 1) / std::mem::size_of::()); - // round up to multiple of u32 (4 bytes) - let num_u32 = (header_bytes + 3) / 4; - - println!("header_bytes = {}", header_bytes); - println!("raw header (hex): {:02X?}", &buf[..header_bytes]); - - - - // 9. Interpret the buffer as u32 CD values (little-endian) - // Safety: we wrote at least header_bytes; we treat as u32 - buf.set_len(hdr_cd_nelmts * 4); - let u32slice: &[u32] = slice::from_raw_parts(buf_ptr as *const u32, num_u32); - dbg!(&buf); - - let cd_values = u32slice.to_vec(); - - // 10. Clean up - bs_close(bstr); - zfp_stream_close(zstr); - zfp_field_free(field); - - - // get the version header right - const ZFP_VERSION_NO: u32 = - (ZFP_VERSION_MAJOR << 12) - | (ZFP_VERSION_MINOR << 8) - | (ZFP_VERSION_PATCH << 4) - | (ZFP_VERSION_TWEAK); - - const H5Z_FILTER_ZFP_VERSION_NO: u32 = - (H5Z_FILTER_ZFP_VERSION_MAJOR << 8) - | (H5Z_FILTER_ZFP_VERSION_MINOR << 4) - | (H5Z_FILTER_ZFP_VERSION_PATCH); - - - let first = (ZFP_VERSION_NO << 16) | (5 << 12) | H5Z_FILTER_ZFP_VERSION_NO; - let mut cd = vec![ first as u32 ]; - cd.extend(&cd_values); - - Some(cd) - } -} - - -#[cfg(test)] -mod test{ - use super::{build_reversible_zfp_cd_values}; - use crate::hl::filters::ZfpMode; - - #[test] - fn reproduce_zfp_header_from_params(){ - - let dataset_size = 10000; - let n_dims = 1; - let zfp_mode = ZfpMode::Reversible; - let zfp_config_vector = [5,0,0]; - let cd_values = build_reversible_zfp_cd_values(&[1,10000], 4); - dbg!(cd_values); - assert_eq!(1,0); - - } - - - -} \ No newline at end of file diff --git a/hdf5/src/hl/filters/zfp.rs b/hdf5/src/hl/filters/zfp.rs index 3e36800c6..df9dbbb8b 100644 --- a/hdf5/src/hl/filters/zfp.rs +++ b/hdf5/src/hl/filters/zfp.rs @@ -22,7 +22,9 @@ use zfp_sys::zfp_stream; use crate::filters::ZfpMode; -/// Major edits are needed to be in alignmeht with the H5Z-ZFP. What was previously implemented was effectively a new implementation of H5Z_ZFP but was incompatible with any library built against it. This reults in bad c_data vectors being created and produces erratic behavior. +/// Major edits are needed to be in alignmeht with the H5Z-ZFP. What was previously implemented was +/// effectively a new implementation of H5Z_ZFP but was incompatible with any library built against +/// it. This reults in bad c_data vectors being created and produces erratic behavior. pub(crate) const MAX_NDIMS: usize = 4; @@ -71,6 +73,19 @@ extern "C" fn can_apply_zfp(_dcpl_id: hid_t, type_id: hid_t, _space_id: hid_t) - } +/// Sets the local properties for the ZFP filter. +/// +/// This function is called during the creation of a dataset or attribute to set +/// the local properties of the ZFP filter. It retrieves the filter's configuration +/// data, validates the chunk dimensions, and updates the filter's parameters. +/// +/// # Parameters +/// - `dcpl_id`: The dataset creation property list identifier. +/// - `type_id`: The datatype identifier of the dataset or attribute. +/// - `_space_id`: The dataspace identifier (not used in this function). +/// +/// # Returns +/// - `herr_t`: Returns 1 on success, or -1 on failure. extern "C" fn set_local_zfp(dcpl_id: hid_t, type_id: hid_t, _space_id: hid_t) -> herr_t { const MAX_NDIMS: usize = 4; let mut flags: c_uint = 0; @@ -141,35 +156,30 @@ extern "C" fn set_local_zfp(dcpl_id: hid_t, type_id: hid_t, _space_id: hid_t) -> } -fn pack_header_into_cd_values( - header_bytes: &[u8], - bits_written: usize, -) -> Vec { - let total_bytes = (bits_written + 7) / 8; - let nwords = (total_bytes + 3) / 4; - - let mut cd_vals = Vec::with_capacity(1 + nwords); - - // cd_values[0] = version word (we'll fill it in below) - cd_vals.push(0); - - for i in 0..nwords { - let mut word_bytes = [0u8; 4]; - for j in 0..4 { - let idx = i * 4 + j; - if idx < total_bytes { - word_bytes[j] = header_bytes[idx]; - } - } - cd_vals.push(u32::from_le_bytes(word_bytes)); - } - - cd_vals -} - const H5Z_ZFP_CD_NELMTS_MAX: usize = 8; // whatever the header says; set correctly. + +/// Computes the header and configuration data values for the ZFP filter. +/// +/// This function generates the header and configuration data values (`cd_values`) +/// required for the ZFP filter. It creates a dummy ZFP field based on the provided +/// dimensions and data type, sets the compression mode, and writes the full header +/// into the `cd_values` buffer. +/// +/// # Parameters +/// - `zt`: The ZFP data type (e.g., `zfp_type_zfp_type_float` or `zfp_type_zfp_type_double`). +/// - `ndims_used`: The number of dimensions used in the data. +/// - `dims_used`: A slice containing the sizes of the dimensions. +/// - `mode`: The ZFP compression mode, which can be fixed rate, precision, accuracy, or reversible. +/// +/// # Returns +/// A tuple containing: +/// - `Vec`: The header and configuration data values. +/// - `usize`: The number of elements in the `cd_values` array. +/// +/// # Panics +/// This function will panic if the number of dimensions exceeds the supported range (1 to 4). pub unsafe fn compute_hdr_cd_values( zt: zfp_type, ndims_used: usize, @@ -280,104 +290,6 @@ unsafe fn make_version_word() -> u32 { -pub unsafe fn make_llnl_style_cd_values( - chunk_dims: &[usize], - mode: ZfpMode, -) -> Vec { - let ztype = zfp_type_zfp_type_float; - let dims_used: Vec = chunk_dims - .iter() - .copied() - .filter(|&d| d > 1) - .collect(); - let field = make_zfp_field(ztype, &dims_used); - let zfp_stream = zfp_stream_open(ptr::null_mut()); - - - match mode { - ZfpMode::FixedRate(rate) => { - zfp_stream_set_rate(zfp_stream, rate, std::mem::size_of::() as _, dims_used.len() as _, 0); - } - ZfpMode::FixedPrecision(precision) => { - zfp_stream_set_precision(zfp_stream, precision as u32); - } - ZfpMode::FixedAccuracy(accuracy) => { - zfp_stream_set_accuracy(zfp_stream, accuracy); - } - ZfpMode::Reversible => { - zfp_stream_set_reversible(zfp_stream); - } - }; - - - let (header_bytes, bits_written) = zfp_header_bits(zfp_stream, field); - - let mut cd_vals = pack_header_into_cd_values(&header_bytes, bits_written); - - // plug in version info - cd_vals[0] = make_version_word(); - unsafe { - zfp_field_free(field); - zfp_stream_close(zfp_stream); - } - - cd_vals -} - - -unsafe fn make_zfp_field(ztype: zfp_type, dims: &[usize]) -> *mut zfp_field { - let mut shape = [1usize; 4]; - for (idx, dim) in dims.iter().copied().take(4).enumerate() { - shape[idx] = dim.max(1); - } - - match dims.len() { - 0 | 1 => zfp_field_1d(ptr::null_mut(), ztype, shape[0]), - 2 => zfp_field_2d(ptr::null_mut(), ztype, shape[0], shape[1]), - 3 => zfp_field_3d(ptr::null_mut(), ztype, shape[0], shape[1], shape[2]), - _ => zfp_field_4d( - ptr::null_mut(), - ztype, - shape[0], - shape[1], - shape[2], - shape[3], - ), - } -} - - -unsafe fn zfp_header_bits( - zstream: *mut zfp_stream, - field: *mut zfp_field, -) -> (Vec, usize) { - // Max bits → bytes; this constant is in the ZFP docs and exposed via zfp-sys - let max_bits = ZFP_HEADER_MAX_BITS as usize; - let max_bytes = 20; - let mut buf = vec![0u8; max_bytes]; - - // Make bitstream over our byte buffer - let bs = stream_open( - buf.as_mut_ptr() as *mut c_void, - max_bytes as usize, - ); - zfp_stream_set_bit_stream(zstream, bs); - zfp_stream_rewind(zstream); - - let mask = ZFP_HEADER_FULL as u32; - let bits_written = zfp_write_header(zstream, field, mask); - - if bits_written == 0 { - panic!("zfp_write_header failed"); - } - - // Clean up bitstream; keep header bytes in buf - stream_close(bs); - - (buf, bits_written as usize) -} - - #[derive(Debug)] struct ZfpConfig { pub ndims: c_int, @@ -390,7 +302,26 @@ struct ZfpConfig { } -/// receive the new cdata from the system and decode it to recover the right ZFP opertaing modes +/// Parses ZFP filter configuration data from the given input. +/// +/// This function extracts metadata and compression parameters from the +/// provided `cd_values` array, which represents the ZFP filter's configuration +/// data. It handles endian mismatches, validates the header, and retrieves +/// information such as dimensions, data type, and compression mode. +/// +/// # Safety +/// This function is marked as unsafe because it performs raw pointer +/// dereferencing and interacts with low-level C APIs, which require careful +/// handling to avoid undefined behavior. +/// +/// # Parameters +/// - `cd_nelmts`: The number of elements in the `cd_values` array. +/// - `cd_values`: A pointer to the array of configuration data values. +/// +/// # Returns +/// - `Option`: Returns a `ZfpConfig` struct containing the parsed +/// metadata and compression parameters if successful, or `None` if the +/// parsing fails. pub unsafe fn parse_zfp_cdata( cd_nelmts: usize, cd_values: *const c_uint, @@ -402,10 +333,10 @@ pub unsafe fn parse_zfp_cdata( // Full cd array from HDF5: [version_word, header_words...] let cdata: &[u32] = slice::from_raw_parts(cd_values, cd_nelmts); - // We currently ignore the version word, but you can decode it if you want. + // ignore the version word, let _version_word = cdata[0]; - // Everything after index 0 is the ZFP header bitstream. + // ZFP header bitstream. let header_words = &cdata[1..]; if header_words.is_empty() { return None; @@ -415,21 +346,21 @@ pub unsafe fn parse_zfp_cdata( let mut header_copy: Vec = header_words.to_vec(); let header_bytes = header_copy.len() * std::mem::size_of::(); - // 1. Open bitstream on the header buffer (like get_zfp_info_from_cd_values) :contentReference[oaicite:2]{index=2} + // Open bitstream on the header buffer (like get_zfp_info_from_cd_values) let bstr: *mut bitstream = stream_open(header_copy.as_mut_ptr() as *mut c_void, header_bytes); if bstr.is_null() { return None; } - // 2. Open zfp_stream on that bitstream + // Open zfp_stream on that bitstream let zstr: *mut zfp_stream = zfp_stream_open(bstr); if zstr.is_null() { stream_close(bstr); return None; } - // 3. Allocate a field for header metadata + // Allocate a field for header metadata let zfld: *mut zfp_field = zfp_field_alloc(); if zfld.is_null() { zfp_stream_close(zstr); @@ -437,7 +368,7 @@ pub unsafe fn parse_zfp_cdata( return None; } - // 4. First read only MAGIC, to detect endian or codec mismatch + //First read only MAGIC, to detect endian or codec mismatch let mut bits = zfp_read_header(zstr, zfld, ZFP_HEADER_MAGIC); if bits == 0 { // Possible endian mismatch: byte-swap each u32 and retry. @@ -448,7 +379,6 @@ pub unsafe fn parse_zfp_cdata( zfp_stream_rewind(zstr); bits = zfp_read_header(zstr, zfld, ZFP_HEADER_MAGIC); if bits == 0 { - // Codec mismatch or truly invalid header. zfp_field_free(zfld); zfp_stream_close(zstr); stream_close(bstr); @@ -456,7 +386,7 @@ pub unsafe fn parse_zfp_cdata( } } - // 5. We know magic is fine. Rewind and read the full header. + //Rewind and read the full header. zfp_stream_rewind(zstr); if zfp_read_header(zstr, zfld, ZFP_HEADER_FULL) == 0 { zfp_field_free(zfld); @@ -465,14 +395,14 @@ pub unsafe fn parse_zfp_cdata( return None; } - // 6. Extract array metadata via high-level API (no manual bit-twiddling). + //Extract array metadata let ndims = zfp_field_dimensionality(zfld) as i32; - // zfp_field_size can fill per-dimension sizes if we pass a buffer. + // zfp_field_size can fill per-dimension sizes; pass a buffer. let mut size_per_dim: [usize; 4] = [0; 4]; if ndims > 0 { // zfp_field_size returns total number of elements and optionally fills size[i]. - // The C signature uses size_t*; we just alias &mut [usize] here. + // The C signature uses size_t*; just alias &mut [usize] here. zfp_field_size( zfld, size_per_dim.as_mut_ptr() as *mut _, @@ -499,17 +429,15 @@ pub unsafe fn parse_zfp_cdata( return None; } }; - // 7. Extract compression mode and parameters from the stream itself. + // Extract compression mode and parameters from the stream itself. let zmode_enum: zfp_mode = zfp_stream_compression_mode(zstr); let mode = zmode_enum as u32; - dbg!(&mode); let mut rate: f64 = 0.0; let mut precision: u32 = 0; let mut accuracy: f64 = 0.0; // These getters are available on modern zfp (1.0+). - // If your zfp version is older, you can `cfg`-gate or leave them at zero. match zmode_enum { m if m == zfp_sys::zfp_mode_zfp_mode_fixed_rate => { rate = zfp_stream_rate(zstr, ndims as u32); @@ -526,13 +454,10 @@ pub unsafe fn parse_zfp_cdata( // Expert or reversible -> we don’t have a single scalar parameter to expose _ => { - // leave rate/precision/accuracy at 0; you can later extend this by - // calling zfp_stream_params() for expert mode. } } - dbg!(&accuracy); - // 8. Cleanup + //Cleanup zfp_field_free(zfld); zfp_stream_close(zstr); stream_close(bstr); @@ -548,45 +473,29 @@ pub unsafe fn parse_zfp_cdata( }) } -fn parse_zfp_cdata_old(cd_nelmts: size_t, cd_values: *const c_uint) -> Option { - let cdata = unsafe { slice::from_raw_parts(cd_values, cd_nelmts as _) }; - dbg!(&cdata); - if cdata.len() < 7 { - h5err!("Invalid ZFP filter configuration", H5E_PLIST, H5E_CALLBACK); - return None; - } - - let ndims = cdata[1] as c_int; - let typesize = cdata[2] as size_t; - let mut dims = [0; 4]; - for i in 0..(ndims as usize).min(4) { - dims[i] = cdata[3 + i] as size_t; - } - - let mode = if cdata.len() > 7 { cdata[7] } else { ZFP_MODE_RATE }; - let param1 = if cdata.len() > 8 { cdata[8] } else { 0 }; - let param2 = if cdata.len() > 9 { cdata[9] } else { 0 }; - - let (rate, precision, accuracy) = match mode { - ZFP_MODE_RATE => { - let rate = f64::from_bits(((param1 as u64) << 32) | (param2 as u64)); - (rate, 0, 0.0) - } - ZFP_MODE_PRECISION => (0.0, param1, 0.0), - ZFP_MODE_ACCURACY => { - let accuracy = f64::from_bits(((param1 as u64) << 32) | (param2 as u64)); - (0.0, 0, accuracy) - } - ZFP_MODE_REVERSIBLE => (0.0, 0, 0.0), - _ => { - h5err!("Invalid ZFP mode", H5E_PLIST, H5E_CALLBACK); - return None; - } - }; - - Some(ZfpConfig { ndims, typesize, dims, mode, rate, precision, accuracy }) -} +/// Applies the ZFP filter for compression or decompression. +/// +/// This function serves as the entry point for the ZFP filter, determining whether +/// to compress or decompress the data based on the provided flags. It parses the +/// filter configuration data, validates it, and then delegates the operation to +/// either the compression or decompression function. +/// +/// # Safety +/// This function is marked as unsafe because it interacts with raw pointers and +/// performs operations that require careful handling to avoid undefined behavior. +/// +/// # Parameters +/// - `flags`: A bitmask indicating the operation mode (e.g., compression or decompression). +/// - `cd_nelmts`: The number of elements in the `cd_values` array. +/// - `cd_values`: A pointer to the array of configuration data values. +/// - `nbytes`: The size of the input buffer in bytes. +/// - `buf_size`: A pointer to the size of the output buffer. +/// - `buf`: A pointer to the input/output buffer. +/// +/// # Returns +/// - `size_t`: The size of the processed data (compressed or decompressed) on success, +/// or 0 on failure. unsafe extern "C" fn filter_zfp( flags: c_uint, cd_nelmts: size_t, cd_values: *const c_uint, nbytes: size_t, buf_size: *mut size_t, buf: *mut *mut c_void, @@ -597,7 +506,6 @@ unsafe extern "C" fn filter_zfp( } else { return 0; }; - dbg!(&cfg); if flags & H5Z_FLAG_REVERSE == 0 { unsafe { filter_zfp_compress(&cfg, buf_size, buf) } @@ -691,14 +599,12 @@ unsafe fn filter_zfp_compress( h5err!("Can't allocate compression buffer", H5E_PLIST, H5E_CALLBACK); return 0; } - println!("Here Outbuf"); let bitstream = stream_open(outbuf.cast(), maxsize); zfp_stream_set_bit_stream(zfp_stream, bitstream); zfp_stream_rewind(zfp_stream); let compressed_size = zfp_compress(zfp_stream, field); - println!("here compressed size"); stream_close(bitstream); zfp_field_free(field); zfp_stream_close(zfp_stream); From d9e5772960f5690b76b479d8d3ae5ec7e1f9242a Mon Sep 17 00:00:00 2001 From: Frank Blubaugh Date: Fri, 5 Dec 2025 11:52:41 -0500 Subject: [PATCH 091/141] added more tests --- hdf5/src/hl/filters.rs | 559 ++++++++++++++----------------------- hdf5/src/hl/filters/zfp.rs | 2 - 2 files changed, 206 insertions(+), 355 deletions(-) diff --git a/hdf5/src/hl/filters.rs b/hdf5/src/hl/filters.rs index fa510f24c..5f4084cc5 100644 --- a/hdf5/src/hl/filters.rs +++ b/hdf5/src/hl/filters.rs @@ -620,7 +620,6 @@ impl Filter { // get the chunk dimensiosn out of it. Could not reliably get the chunk_dims from plist_id // during testing so opted to just pass it in during the build - assert!(chunk_dims.len()<= zfp::MAX_NDIMS); let ndims = chunk_dims.len(); // Convert to `usize` and trim to used dims. let chunk_dims_usize: Vec = chunk_dims[..(ndims as usize)] @@ -638,6 +637,10 @@ impl Filter { } let ndims_no_singleton = dims_no_singleton.len(); + + assert!(dims_no_singleton.len()<= zfp::MAX_NDIMS); + + // Get the type of the input data let dtype_id = match n_bytes{ 4 => zfp_type_zfp_type_float, @@ -955,44 +958,7 @@ mod tests { // // Ok(()) // } - // - // #[test] - // #[cfg(feature = "zfp")] - // fn test_zfp_reversible() -> Result<()> { - // use super::zfp_available; - // - // if !zfp_available() { - // println!("ZFP filter not available, skipping test"); - // return Ok(()); - // } - // with_tmp_file(|file| { - // let data = ndarray::Array1::::linspace(0.0, 1000.0, 100000); - // file.new_dataset_builder() - // .zfp_reversible() - // .with_data(&data) - // .chunk((10000,)) - // .create("zfp_lossless_1d") - // .unwrap(); - // - // let ds = file.dataset("zfp_lossless_1d").unwrap(); - // // get number of bytes of ds - // let n_bytes = file.size(); - // let read_data: Vec = ds.read_raw().unwrap(); - // let error = data.iter().zip(read_data.iter()).map(|(a, b)| (a - b).abs()).sum::() - // / data.len() as f32; - // - // let target_bytes = 100000 * 4; - // assert!( - // n_bytes <= target_bytes, - // "Dataset size {} exceeds target {}", - // n_bytes, - // target_bytes - // ); - // assert_eq!(n_bytes, 249368); - // assert_eq!(error, 0.0); - // }); - // Ok(()) - // } + #[test] #[cfg(feature = "zfp")] @@ -1004,12 +970,14 @@ mod tests { assert_eq!(1, 0); return Ok(()); } + + // test 1D Data with_tmp_file(|file| { let data = ndarray::Array1::::linspace(0.0, 1.0, 1000); file.new_dataset_builder() .with_data(&data) .chunk((1000,)) - .zfp_accuracy(0.125,vec![1000],4) + .zfp_accuracy(0.125, vec![1000], 4) .create("zfp_precision_1d") .unwrap(); @@ -1021,7 +989,6 @@ mod tests { // ZFP is lossy, so we check approximate equality assert_eq!(read_data.len(), data.len()); - assert_eq!(1,0); for (i, (original, compressed)) in data.iter().zip(read_data.iter()).enumerate() { let diff = (original - compressed).abs(); assert!( @@ -1035,6 +1002,172 @@ mod tests { } }); + + // Test 2D data + with_tmp_file(|file| { + let data = ndarray::Array1::::linspace(0.0, 1.0, 1000); + let data = data.to_shape((10,100)).unwrap(); + file.new_dataset_builder() + .with_data(&data) + .chunk((5, 10)) + .zfp_accuracy(0.125, vec![5, 10], 4) + .create("zfp_precision_1d") + .unwrap(); + + + let ds = file.dataset("zfp_precision_1d").unwrap(); + + let read_data: Vec = ds.read_raw().unwrap(); + + // ZFP is lossy, so we check approximate equality + assert_eq!(read_data.len(), data.len()); + + for (i, (original, compressed)) in data.iter().zip(read_data.iter()).enumerate() { + let diff = (original - compressed).abs(); + assert!( + diff < 0.125, + "Index {}: difference too large: {} vs {} (diff: {})", + i, + original, + compressed, + diff + ); + } + }); + + + // Test 3D data + with_tmp_file(|file| { + let data = ndarray::Array1::::linspace(0.0, 1.0, 10000); + let data = data.to_shape((10, 10, 100)).unwrap(); + + file.new_dataset_builder() + .with_data(&data) + .chunk((2, 5, 25,)) + .zfp_accuracy(0.125, vec![2, 5, 25], 4) + .create("zfp_precision_3d") + .unwrap(); + + + let ds = file.dataset("zfp_precision_3d").unwrap(); + + let read_data: Vec = ds.read_raw().unwrap(); + let data_raw = data.as_slice().unwrap(); + + // ZFP is lossy, so we check approximate equality + assert_eq!(read_data.len(), data_raw.len()); + + for (i, (original, compressed)) in data_raw.iter().zip(read_data.iter()).enumerate() { + let diff = (original - compressed).abs(); + assert!( + diff < 0.125, + "Index {}: difference too large: {} vs {} (diff: {})", + i, + original, + compressed, + diff + ); + } + }); + + + // Test 4D data + with_tmp_file(|file| { + let data = ndarray::Array1::::linspace(0.0, 1.0, 100000); + let data = data.to_shape((10, 10, 10, 100)).unwrap(); + file.new_dataset_builder() + .with_data(&data) + .chunk((2, 2, 5, 50,)) + .zfp_accuracy(0.125, vec![2, 2, 5, 50], 4) + .create("zfp_precision_1d") + .unwrap(); + + + let ds = file.dataset("zfp_precision_1d").unwrap(); + + let read_data: Vec = ds.read_raw().unwrap(); + let data_raw = data.as_slice().unwrap(); + + // ZFP is lossy, so we check approximate equality + assert_eq!(read_data.len(), data_raw.len()); + + for (i, (original, compressed)) in data_raw.iter().zip(read_data.iter()).enumerate() { + let diff = (original - compressed).abs(); + assert!( + diff < 0.125, + "Index {}: difference too large: {} vs {} (diff: {})", + i, + original, + compressed, + diff + ); + } + }); + Ok(()) + } + + #[test] + fn test_over_dim_data() -> Result<()>{ + use super::zfp_available; + + if !zfp_available() { + println!("ZFP filter not available, skipping test"); + assert_eq!(1, 0); + return Ok(()); + } + // + // // Test 5D data and fail + // // capture this known error + // + // with_tmp_file(|file| { + // let data = ndarray::Array1::::linspace(0.0, 1.0, 50_000); + // let data = data.to_shape((2,5,10,10,50)).unwrap(); + // file.new_dataset_builder() + // .with_data(&data) + // .chunk((2,5,5,5,25)) + // .zfp_accuracy(0.125,vec![2,5,5,5,25],4) + // .create("zfp_precision_1d") + // .unwrap(); + // + // + // let ds = file.dataset("zfp_precision_1d").unwrap(); + // + // let read_data: Vec = ds.read_raw().unwrap(); + // + // // ZFP is lossy, so we check approximate equality + // assert_eq!(read_data.len(), data.len()); + // + // assert_eq!(1,0); + // for (i, (original, compressed)) in data.iter().zip(read_data.iter()).enumerate() { + // let diff = (original - compressed).abs(); + // assert!( + // diff < 0.1, + // "Index {}: difference too large: {} vs {} (diff: {})", + // i, + // original, + // compressed, + // diff + // ); + // } + // }); + + // Test 5D data with 3D chunks but should still fail + // test 1D Data + with_tmp_file(|file| { + let data = ndarray::Array1::::linspace(0.0, 1.0, 50_000); + let data = data.to_shape((2,5,10,10,50)).unwrap(); + + let bad_result = file.new_dataset_builder() + .with_data(&data) + .chunk((2,5,5,1,1)) + .zfp_accuracy(0.125,vec![2,5,5,1,1],4) + .create("zfp_precision_1d").unwrap_err(); + + assert_err!(bad_result, "ZFP filter supports up to 4D data only"); + + + }); + Ok(()) } @@ -1081,7 +1214,6 @@ mod tests { target_bytes ); assert_eq!(n_bytes, 29432); - // assert_eq!(1,0); // for (i, (original, compressed)) in data.iter().zip(read_data.iter()).enumerate() { let diff = (original - compressed).abs(); @@ -1097,6 +1229,9 @@ mod tests { } }); + + + Ok(()) } @@ -1122,7 +1257,33 @@ mod tests { let ds = file.dataset("zfp_rate").unwrap(); - assert_eq!(ds.filters(), vec![crate::hl::filters::Filter::zfp_rate(2.0,vec![1000],4)]); + + let read_data: Vec = ds.read_raw().unwrap(); + + // ZFP is lossy, so we check approximate equality + assert_eq!(read_data.len(), data.len()); + dbg!(&data.clone().into_raw_vec_and_offset().0[0..15]); + dbg!(&read_data[0..15]); + for (i, (original, compressed)) in data.iter().zip(read_data.iter()).enumerate() { + let diff = (original - compressed).abs(); + dbg!(&diff); + } + }); + + + // test full rate compression. Should be "lossless" + + with_tmp_file(|file| { + let data = ndarray::Array1::::linspace(0.0, 1.0, 1000); + file.new_dataset_builder() + .with_data(&data) + .chunk((1000,)) + .zfp_rate(32.0,vec![1000],4) + .create("zfp_rate") + .unwrap(); + + + let ds = file.dataset("zfp_rate").unwrap(); let read_data: Vec = ds.read_raw().unwrap(); @@ -1144,207 +1305,10 @@ mod tests { } }); + Ok(()) } - // #[test] - // #[cfg(feature = "zfp")] - // fn test_zfp_roundtrip_1d() -> Result<()> { - // use super::zfp_available; - // - // if !zfp_available() { - // println!("ZFP filter not available, skipping test"); - // return Ok(()); - // } - // let pipeline = vec![Filter::zfp_rate(16.0)]; - // with_tmp_file(|file| { - // let data = ndarray::Array1::::linspace(0.0, 1.0, 1000); - // file.new_dataset_builder() - // .with_data(&data) - // .chunk((100,)) - // .with_dcpl(|p| p.set_filters(&pipeline)) - // .create("zfp_1d_dcpl") - // .unwrap(); - // - // let ds = file.dataset("zfp_1d_dcpl").unwrap(); - // let read_data: Vec = ds.read_raw().unwrap(); - // - // // ZFP is lossy, so we check approximate equality - // assert_eq!(read_data.len(), data.len()); - // for (i, (original, compressed)) in data.iter().zip(read_data.iter()).enumerate() { - // let diff = (original - compressed).abs(); - // assert!( - // diff < 0.1, - // "Index {}: difference too large: {} vs {} (diff: {})", - // i, - // original, - // compressed, - // diff - // ); - // } - // }); - // - // Ok(()) - // } - // - // #[test] - // #[cfg(feature = "zfp")] - // fn test_zfp_roundtrip_2d() -> Result<()> { - // use super::zfp_available; - // - // if !zfp_available() { - // println!("ZFP filter not available, skipping test"); - // return Ok(()); - // } - // - // with_tmp_file(|file| { - // let data: Vec = (0..1000).map(|i| (i as f64) * 0.01).collect(); - // let data = Array2::from_shape_vec((10, 100), data).unwrap(); - // - // let pipteline = vec![Filter::zfp_precision(32)]; - // file.new_dataset_builder() - // .with_data(&data) - // .chunk((10, 10)) - // .with_dcpl(|p| p.set_filters(&pipteline)) - // .create("zfp_2d") - // .unwrap(); - // - // let ds = file.dataset("zfp_2d").unwrap(); - // let read_data: Vec = ds.read_raw().unwrap(); - // - // assert_eq!(read_data.len(), data.len()); - // for (original, compressed) in data.iter().zip(read_data.iter()) { - // let diff = (original - compressed).abs(); - // assert!(diff < 0.01, "Difference too large: {} vs {}", original, compressed); - // } - // }); - // - // Ok(()) - // } - // - // #[test] - // #[cfg(feature = "zfp")] - // fn test_zfp_roundtrip_3d() -> Result<()> { - // use super::zfp_available; - // - // if !zfp_available() { - // println!("ZFP filter not available, skipping test"); - // return Ok(()); - // } - // let pipeline = vec![Filter::zfp_accuracy(1e-4)]; - // with_tmp_file(|file| { - // let data: Vec = (0..1000).map(|i| (i as f32) * 0.001).collect(); - // let data = ndarray::Array3::from_shape_vec((10, 10, 10), data).unwrap(); - // - // file.new_dataset_builder() - // .with_data(&data) - // .chunk((5, 5, 5)) - // .with_dcpl(|p| p.set_filters(&pipeline)) - // .create("zfp_3d") - // .unwrap(); - // - // let ds = file.dataset("zfp_3d").unwrap(); - // let read_data: Vec = ds.read_raw().unwrap(); - // - // assert_eq!(read_data.len(), data.len()); - // }); - // - // Ok(()) - // } - // - // #[test] - // #[cfg(feature = "zfp")] - // fn test_zfp_roundtrip_4d() -> Result<()> { - // use super::zfp_available; - // - // if !zfp_available() { - // println!("ZFP filter not available, skipping test"); - // return Ok(()); - // } - // let pipeline = vec![Filter::zfp_rate(10.0)]; - // with_tmp_file(|file| { - // let data: Vec = (0..256).map(|i| (i as f64) * 0.1).collect(); - // let data = ndarray::Array4::from_shape_vec((4, 4, 4, 4), data).unwrap(); - // - // file.new_dataset_builder() - // .with_data(&data) - // .chunk((2, 2, 2, 2)) - // .with_dcpl(|p| p.set_filters(&pipeline)) - // .create("zfp_4d") - // .unwrap(); - // - // let ds = file.dataset("zfp_4d").unwrap(); - // let read_data: Vec = ds.read_raw().unwrap(); - // - // assert_eq!(read_data.len(), data.len()); - // }); - // - // Ok(()) - // } - // - // #[test] - // #[cfg(feature = "zfp")] - // fn test_zfp_mode_parsing() -> Result<()> { - // use super::ZfpMode; - // - // // Test FixedRate parsing - // let rate_bits = 12.5_f64.to_bits(); - // let cdata_rate = vec![ - // 0, - // 1, - // 4, - // 100, - // 0, - // 0, - // 0, - // 1, // mode = rate - // (rate_bits >> 32) as u32, - // rate_bits as u32, - // ]; - // let filter = Filter::from_raw(super::zfp::ZFP_FILTER_ID, &cdata_rate)?; - // if let Filter::Zfp(ZfpMode::FixedRate(rate)) = filter { - // assert!((rate - 12.5).abs() < 1e-10); - // } else { - // panic!("Expected FixedRate mode, got: {:?}", filter); - // } - // - // // Test FixedPrecision parsing - // let cdata_precision = vec![ - // 0, 1, 8, 100, 0, 0, 0, 2, // mode = precision - // 24, // precision - // 0, - // ]; - // let filter = Filter::from_raw(super::zfp::ZFP_FILTER_ID, &cdata_precision)?; - // if let Filter::Zfp(ZfpMode::FixedPrecision(precision)) = filter { - // assert_eq!(precision, 24); - // } else { - // panic!("Expected FixedPrecision mode, got: {:?}", filter); - // } - // - // // Test FixedAccuracy parsing - // let accuracy_bits = 1e-5_f64.to_bits(); - // let cdata_accuracy = vec![ - // 0, - // 1, - // 8, - // 100, - // 0, - // 0, - // 0, - // 3, // mode = accuracy - // (accuracy_bits >> 32) as u32, - // accuracy_bits as u32, - // ]; - // let filter = Filter::from_raw(super::zfp::ZFP_FILTER_ID, &cdata_accuracy)?; - // if let Filter::Zfp(ZfpMode::FixedAccuracy(accuracy)) = filter { - // assert!((accuracy - 1e-5).abs() < 1e-10); - // } else { - // panic!("Expected FixedAccuracy mode, got: {:?}", filter); - // } - // - // Ok(()) - // } - // // #[test] // #[cfg(feature = "zfp")] // fn test_zfp_with_other_filters() -> Result<()> { @@ -1405,117 +1369,6 @@ mod tests { // // Ok(()) // } - // - // #[test] - // #[cfg(feature = "zfp")] - // fn test_zfp_compression_ratios() -> Result<()> { - // use super::zfp_available; - // - // if !zfp_available() { - // println!("ZFP filter not available, skipping test"); - // return Ok(()); - // } - // - // let pipeline = vec![Filter::zfp_rate(32.0)]; - // let pipeline2 = vec![Filter::zfp_rate(4.0)]; - // - // with_tmp_file(|file| { - // let data: Vec = (0..10000).map(|i| (i as f32).sin()).collect(); - // let data = ndarray::Array1::from_shape_vec(10000, data).unwrap(); - // // Higher rate = less compression but better quality - // file.new_dataset_builder() - // .with_data(&data) - // .chunk(1000) - // .with_dcpl(|p| p.set_filters(&pipeline)) - // .create("zfp_high_rate") - // .unwrap(); - // - // // Lower rate = more compression but lower quality - // - // file.new_dataset_builder() - // .with_data(&data) - // .chunk(1000) - // .with_dcpl(|p| p.set_filters(&pipeline2)) - // .create("zfp_low_rate") - // .unwrap(); - // - // let ds_high = file.dataset("zfp_high_rate").unwrap(); - // let ds_low = file.dataset("zfp_low_rate").unwrap(); - // - // let read_high: Vec = ds_high.read_raw().unwrap(); - // let read_low: Vec = ds_low.read_raw().unwrap(); - // - // // High rate should have better accuracy - // let error_high: f32 = - // data.iter().zip(read_high.iter()).map(|(a, b)| (a - b).abs()).sum::() - // / data.len() as f32; - // - // let error_low: f32 = - // data.iter().zip(read_low.iter()).map(|(a, b)| (a - b).abs()).sum::() - // / data.len() as f32; - // - // println!("High rate error: {}, Low rate error: {}", error_high, error_low); - // assert!(error_high < error_low || error_high < 0.001); - // }); - // - // Ok(()) - // } - // - // #[test] - // #[cfg(feature = "zfp")] - // fn test_zfp_edge_cases() -> Result<()> { - // use super::zfp_available; - // - // if !zfp_available() { - // println!("ZFP filter not available, skipping test"); - // return Ok(()); - // } - // - // let pipeline = vec![Filter::zfp_rate(8.0)]; - // - // // Test with all zeros - // with_tmp_file(|file| { - // let data: Vec = vec![0.0; 1000]; - // let data = ndarray::Array1::from_shape_vec(1000, data).unwrap(); - // - // file.new_dataset_builder() - // .with_data(&data) - // .chunk(100) - // .with_dcpl(|p| p.set_filters(&pipeline)) - // .create("zfp_zeros") - // .unwrap(); - // - // let ds = file.dataset("zfp_zeros").unwrap(); - // let read_data: Vec = ds.read_raw().unwrap(); - // - // assert_eq!(read_data.len(), data.len()); - // for &val in &read_data { - // assert_eq!(val, 0.0); - // } - // }); - // - // let pipeline = vec![Filter::zfp_accuracy(1e-6)]; - // // Test with constant values - // with_tmp_file(|file| { - // let data: Vec = vec![42.0; 500]; - // let data = ndarray::Array1::from_shape_vec(500, data).unwrap(); - // file.new_dataset_builder() - // .with_data(&data) - // .chunk(50) - // .with_dcpl(|p| p.set_filters(&pipeline)) - // .create("zfp_constant") - // .unwrap(); - // - // let ds = file.dataset("zfp_constant").unwrap(); - // let read_data: Vec = ds.read_raw().unwrap(); - // - // assert_eq!(read_data.len(), data.len()); - // for &val in &read_data { - // assert!((val - 42.0).abs() < 1e-5); - // } - // }); - // - // Ok(()) - // } + } diff --git a/hdf5/src/hl/filters/zfp.rs b/hdf5/src/hl/filters/zfp.rs index df9dbbb8b..c9b94f9ad 100644 --- a/hdf5/src/hl/filters/zfp.rs +++ b/hdf5/src/hl/filters/zfp.rs @@ -201,7 +201,6 @@ pub unsafe fn compute_hdr_cd_values( // 3. Version word (use the macro layout: (ZFP_VERSION_NO<<16)|(ZFP_CODEC<<12)|H5Z_FILTER_ZFP_VERSION_NO) hdr_cd_values[0] = make_version_word(); // see previous message - dbg!(&hdr_cd_values); // 4. Treat &hdr_cd_values[1] as bitstream buffer let ptr_bytes = hdr_cd_values[1..].as_mut_ptr() as *mut c_void; @@ -531,7 +530,6 @@ unsafe fn filter_zfp_compress( zfp_stream_set_precision(zfp_stream, cfg.precision); } ZFP_MODE_ACCURACY => { - dbg!(cfg.accuracy); zfp_stream_set_accuracy(zfp_stream, cfg.accuracy); } ZFP_MODE_REVERSIBLE => zfp_stream_set_reversible(zfp_stream), From 00fc4b8022309eb13e7f9f6200990a2bd5f30a51 Mon Sep 17 00:00:00 2001 From: Frank Blubaugh Date: Mon, 8 Dec 2025 16:32:59 -0500 Subject: [PATCH 092/141] fixed version requirements, cleaned up tests and fixed a feature flag --- hdf5/Cargo.toml | 2 +- hdf5/src/hl/filters.rs | 171 +---------------------------------------- 2 files changed, 3 insertions(+), 170 deletions(-) diff --git a/hdf5/Cargo.toml b/hdf5/Cargo.toml index 3309b8687..debf9bdb0 100644 --- a/hdf5/Cargo.toml +++ b/hdf5/Cargo.toml @@ -56,7 +56,7 @@ errno = { version = "0.3", optional = true } libc = { workspace = true } lzf-sys = { version = "0.1", optional = true } mpi-sys = { workspace = true, optional = true } -ndarray = ">=0.15, <=0.17" +ndarray = "0.15.6" paste = "1.0" zfp-sys = {version= "0.4.2", optional= true } # internal diff --git a/hdf5/src/hl/filters.rs b/hdf5/src/hl/filters.rs index 5f4084cc5..8fc1ed6d7 100644 --- a/hdf5/src/hl/filters.rs +++ b/hdf5/src/hl/filters.rs @@ -885,80 +885,6 @@ mod tests { Ok(()) } - // #[test] - // #[cfg(feature = "zfp")] - // fn test_zfp_filter() -> Result<()> { - // use super::{zfp_available, ZfpMode}; - // - // assert_eq!(cfg!(feature = "zfp"), zfp_available()); - // - // if !zfp_available() { - // println!("ZFP filter not available, skipping test"); - // return Ok(()); - // } - // - // // Test different ZFP modes - // let zfp_filters = vec![ - // Filter::zfp_rate(8.0), - // Filter::zfp_rate(16.0), - // Filter::zfp_rate(4.0), - // Filter::zfp_precision(16), - // Filter::zfp_precision(32), - // Filter::zfp_accuracy(1e-3), - // Filter::zfp_accuracy(1e-6), - // Filter::zfp(ZfpMode::FixedRate(12.5)), - // ]; - // - // for flt in &zfp_filters { - // println!("Testing filter: {:?}", flt); - // assert!(flt.is_available()); - // assert!(flt.encode_enabled()); - // assert!(flt.decode_enabled()); - // - // // Test with float type (ZFP only supports floats) - // let pipeline = vec![flt.clone()]; - // validate_filters(&pipeline, H5T_class_t::H5T_FLOAT)?; - // - // let plist = DatasetCreate::try_new()?; - // for f in &pipeline { - // f.apply_to_plist(plist.id())?; - // } - // assert_eq!(Filter::extract_pipeline(plist.id())?, pipeline); - // - // // Test with actual dataset creation for f32 - // let res = with_tmp_file(|file| { - // file.new_dataset_builder() - // .empty::() - // .shape((100, 50)) - // .chunk((10, 10)) - // .with_dcpl(|p| p.set_filters(&pipeline)) - // .create("zfp_f32") - // .unwrap(); - // - // let plist = file.dataset("zfp_f32").unwrap().dcpl().unwrap(); - // Filter::extract_pipeline(plist.id()).unwrap() - // }); - // assert_eq!(res, pipeline); - // - // // Test with f64 - // let res_f64 = with_tmp_file(|file| { - // file.new_dataset_builder() - // .empty::() - // .shape((100, 50)) - // .chunk((10, 10)) - // .with_dcpl(|p| p.set_filters(&pipeline)) - // .create("zfp_f64") - // .unwrap(); - // - // let plist = file.dataset("zfp_f64").unwrap().dcpl().unwrap(); - // Filter::extract_pipeline(plist.id()).unwrap() - // }); - // assert_eq!(res_f64, pipeline); - // } - // - // Ok(()) - // } - #[test] #[cfg(feature = "zfp")] @@ -1107,6 +1033,7 @@ mod tests { } #[test] + #[cfg(feature = "zfp")] fn test_over_dim_data() -> Result<()>{ use super::zfp_available; @@ -1115,41 +1042,7 @@ mod tests { assert_eq!(1, 0); return Ok(()); } - // - // // Test 5D data and fail - // // capture this known error - // - // with_tmp_file(|file| { - // let data = ndarray::Array1::::linspace(0.0, 1.0, 50_000); - // let data = data.to_shape((2,5,10,10,50)).unwrap(); - // file.new_dataset_builder() - // .with_data(&data) - // .chunk((2,5,5,5,25)) - // .zfp_accuracy(0.125,vec![2,5,5,5,25],4) - // .create("zfp_precision_1d") - // .unwrap(); - // - // - // let ds = file.dataset("zfp_precision_1d").unwrap(); - // - // let read_data: Vec = ds.read_raw().unwrap(); - // - // // ZFP is lossy, so we check approximate equality - // assert_eq!(read_data.len(), data.len()); - // - // assert_eq!(1,0); - // for (i, (original, compressed)) in data.iter().zip(read_data.iter()).enumerate() { - // let diff = (original - compressed).abs(); - // assert!( - // diff < 0.1, - // "Index {}: difference too large: {} vs {} (diff: {})", - // i, - // original, - // compressed, - // diff - // ); - // } - // }); + // Test 5D data with 3D chunks but should still fail // test 1D Data @@ -1309,66 +1202,6 @@ mod tests { Ok(()) } - // #[test] - // #[cfg(feature = "zfp")] - // fn test_zfp_with_other_filters() -> Result<()> { - // use super::zfp_available; - // - // if !zfp_available() { - // println!("ZFP filter not available, skipping test"); - // return Ok(()); - // } - // - // let pipeline = vec![Filter::zfp_rate(8.0)]; - // // Test ZFP combined with shuffle (shuffle should come first) - // with_tmp_file(|file| { - // let data: Vec = (0..1000).map(|i| (i as f32) * 0.1).collect(); - // let data = ndarray::Array1::from_shape_vec(1000, data).unwrap(); - // // file.new_dataset_builder() - // // .with_data(&data) - // // .chunk(100) - // // .with_dcpl(|p| p.set_filters(&pipeline)) - // // .create("zfp_rate_8").unwrap(); - // // - // file.new_dataset_builder() - // .zfp_rate(8.0) - // .with_data(&data) - // .chunk(100) - // .create("zfp_rate_8") - // .unwrap(); - // - // let ds = file.dataset("zfp_rate_8").unwrap(); - // let read_data: Vec = ds.read_raw().unwrap(); - // - // let error = data.iter().zip(read_data.iter()).map(|(a, b)| (a - b).abs()).sum::() - // / data.len() as f32; - // assert_eq!(error, 0.082505114); - // assert_eq!(read_data.len(), data.len()); - // }); - // - // // Test ZFP with fletcher32 checksum - // if super::deflate_available() { - // let pipeline = vec![Filter::zfp_precision(24), Filter::fletcher32()]; - // with_tmp_file(|file| { - // let data: Vec = (0..500).map(|i| (i as f64) * 0.01).collect(); - // let data = ndarray::Array1::from_shape_vec(500, data).unwrap(); - // - // file.new_dataset_builder() - // .with_data(&data) - // .chunk(50) - // .with_dcpl(|p| p.set_filters(&pipeline)) - // .create("zfp_with_fletcher32") - // .unwrap(); - // - // let ds = file.dataset("zfp_with_fletcher32").unwrap(); - // let read_data: Vec = ds.read_raw().unwrap(); - // - // assert_eq!(read_data.len(), data.len()); - // }); - // } - // - // Ok(()) - // } } From f217b610ca96aa181f3d3060184042ae8e14e7f7 Mon Sep 17 00:00:00 2001 From: Frank Blubaugh Date: Tue, 9 Dec 2025 09:11:13 -0500 Subject: [PATCH 093/141] removed an import --- hdf5/src/hl/filters.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hdf5/src/hl/filters.rs b/hdf5/src/hl/filters.rs index 8fc1ed6d7..2a891d935 100644 --- a/hdf5/src/hl/filters.rs +++ b/hdf5/src/hl/filters.rs @@ -795,9 +795,9 @@ mod tests { use crate::hl::filters::zfp_available; use crate::test::with_tmp_file; use crate::{plist::DatasetCreate, Result}; - use crate::filters::ZfpMode; - use crate::hl::filters::zfp::ZFP_FILTER_ID; + + #[test] fn test_filter_pipeline() -> Result<()> { let mut comp_filters = vec![]; From ba0f24e2db88138fbe728cc8c5ce352ea725bcab Mon Sep 17 00:00:00 2001 From: Frank Blubaugh Date: Tue, 9 Dec 2025 09:55:36 -0500 Subject: [PATCH 094/141] ran cargo fmt --- hdf5/src/hl/dataset.rs | 51 +++++----- hdf5/src/hl/filters.rs | 142 +++++++++++----------------- hdf5/src/hl/filters/zfp.rs | 131 ++++++++++++------------- hdf5/src/hl/plist/dataset_create.rs | 28 +++--- 4 files changed, 157 insertions(+), 195 deletions(-) diff --git a/hdf5/src/hl/dataset.rs b/hdf5/src/hl/dataset.rs index 17fce7fed..3979766f4 100644 --- a/hdf5/src/hl/dataset.rs +++ b/hdf5/src/hl/dataset.rs @@ -3,17 +3,6 @@ use std::ops::Deref; use ndarray::{self, ArrayView}; -use hdf5_sys::h5::HADDR_UNDEF; -use hdf5_sys::h5d::{ - H5Dcreate2, H5Dcreate_anon, H5Dget_access_plist, H5Dget_create_plist, H5Dget_offset, - H5Dset_extent, -}; -#[cfg(feature = "1.10.0")] -use hdf5_sys::h5d::{H5Dflush, H5Drefresh}; -use hdf5_sys::h5l::H5Ldelete; -use hdf5_sys::h5p::H5P_DEFAULT; -use hdf5_sys::h5z::H5Z_filter_t; -use hdf5_types::{OwnedDynValue, TypeDescriptor}; use crate::hl; #[cfg(feature = "blosc")] use crate::hl::filters::{Blosc, BloscShuffle}; @@ -28,6 +17,17 @@ use crate::hl::plist::dataset_create::{ }; use crate::hl::plist::link_create::{CharEncoding, LinkCreate, LinkCreateBuilder}; use crate::internal_prelude::*; +use hdf5_sys::h5::HADDR_UNDEF; +use hdf5_sys::h5d::{ + H5Dcreate2, H5Dcreate_anon, H5Dget_access_plist, H5Dget_create_plist, H5Dget_offset, + H5Dset_extent, +}; +#[cfg(feature = "1.10.0")] +use hdf5_sys::h5d::{H5Dflush, H5Drefresh}; +use hdf5_sys::h5l::H5Ldelete; +use hdf5_sys::h5p::H5P_DEFAULT; +use hdf5_sys::h5z::H5Z_filter_t; +use hdf5_types::{OwnedDynValue, TypeDescriptor}; /// Default chunk size when filters are enabled and the chunk size is not specified. pub const DEFAULT_CHUNK_SIZE_KB: usize = 64 * 1024; @@ -741,35 +741,36 @@ impl DatasetBuilderInner { self.with_dcpl(|pl| pl.blosc_zstd(clevel, shuffle)); } - - - #[cfg(feature = "zfp")] - pub fn zfp_rate(&mut self, rate: f64,chunk_dims: Vec,n_bytes: u8) { + pub fn zfp_rate(&mut self, rate: f64, chunk_dims: Vec, n_bytes: u8) { hl::filters::zfp::register_zfp().expect("Failed to register ZFP filter"); - self.with_dcpl(|p| p.set_filters(&vec![Filter::zfp_rate(rate,chunk_dims.clone(),n_bytes)])); - + self.with_dcpl(|p| { + p.set_filters(&vec![Filter::zfp_rate(rate, chunk_dims.clone(), n_bytes)]) + }); } #[cfg(feature = "zfp")] - pub fn zfp_precision(&mut self, precision: u8,chunk_dims: Vec,n_bytes: u8) { + pub fn zfp_precision(&mut self, precision: u8, chunk_dims: Vec, n_bytes: u8) { hl::filters::zfp::register_zfp().expect("Failed to register ZFP filter"); - self.with_dcpl(|p| p.set_filters(&vec![Filter::zfp_precision(precision,chunk_dims.clone(),n_bytes)])); + self.with_dcpl(|p| { + p.set_filters(&vec![Filter::zfp_precision(precision, chunk_dims.clone(), n_bytes)]) + }); } #[cfg(feature = "zfp")] - pub fn zfp_accuracy(&mut self, accuracy: f64,chunk_dims: Vec,n_bytes: u8) { + pub fn zfp_accuracy(&mut self, accuracy: f64, chunk_dims: Vec, n_bytes: u8) { hl::filters::zfp::register_zfp().expect("Failed to register ZFP filter"); - self.with_dcpl(|pl| pl.zfp_accuracy(accuracy,chunk_dims.clone(),n_bytes)); - } + self.with_dcpl(|pl| pl.zfp_accuracy(accuracy, chunk_dims.clone(), n_bytes)); + } #[cfg(feature = "zfp")] - pub fn zfp_reversible(&mut self,chunk_dims: Vec,n_bytes: u8) { + pub fn zfp_reversible(&mut self, chunk_dims: Vec, n_bytes: u8) { hl::filters::zfp::register_zfp().expect("Failed to register ZFP filter"); - self.with_dcpl(|p| p.set_filters(&vec![Filter::zfp_reversible(chunk_dims.clone(),n_bytes)])); + self.with_dcpl(|p| { + p.set_filters(&vec![Filter::zfp_reversible(chunk_dims.clone(), n_bytes)]) + }); } - pub fn add_filter(&mut self, id: H5Z_filter_t, cdata: &[c_uint]) { self.with_dcpl(|pl| pl.add_filter(id, cdata)); } diff --git a/hdf5/src/hl/filters.rs b/hdf5/src/hl/filters.rs index 2a891d935..fb9868c0c 100644 --- a/hdf5/src/hl/filters.rs +++ b/hdf5/src/hl/filters.rs @@ -1,7 +1,10 @@ use std::collections::HashMap; use std::ptr::{self, addr_of_mut}; -use hdf5_sys::h5p::{H5Pget_chunk, H5Pget_filter2, H5Pget_nfilters, H5Pset_deflate, H5Pset_filter, H5Pset_fletcher32, H5Pset_nbit, H5Pset_scaleoffset, H5Pset_shuffle, H5Pset_szip}; +use hdf5_sys::h5p::{ + H5Pget_chunk, H5Pget_filter2, H5Pget_nfilters, H5Pset_deflate, H5Pset_filter, + H5Pset_fletcher32, H5Pset_nbit, H5Pset_scaleoffset, H5Pset_shuffle, H5Pset_szip, +}; use hdf5_sys::h5t::{H5T_class_t, H5Tclose, H5Tget_class, H5Tget_size, H5T_FLOAT}; use hdf5_sys::h5z::{ H5Zfilter_avail, H5Zget_filter_info, H5Z_FILTER_CONFIG_DECODE_ENABLED, @@ -22,9 +25,8 @@ mod lzf; #[cfg(feature = "zfp")] pub(crate) mod zfp; - #[cfg(feature = "zfp")] -use zfp_sys::{zfp_type_zfp_type_float,zfp_type_zfp_type_double}; +use zfp_sys::{zfp_type_zfp_type_double, zfp_type_zfp_type_float}; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum SZip { @@ -118,7 +120,6 @@ mod zfp_impl { Reversible, } - // Bitwise compare f64 so NaN and signed zero are deterministic impl PartialEq for ZfpMode { fn eq(&self, other: &Self) -> bool { @@ -140,20 +141,16 @@ mod zfp_impl { } } - #[derive(Clone, Debug,Eq,PartialEq)] - pub struct FieldParam{ + #[derive(Clone, Debug, Eq, PartialEq)] + pub struct FieldParam { pub data_type_bytes: usize, - pub dims: Vec + pub dims: Vec, } - - - - } +use crate::globals::{H5E_CALLBACK, H5E_PLIST}; #[cfg(feature = "zfp")] pub use zfp_impl::*; -use crate::globals::{H5E_CALLBACK, H5E_PLIST}; #[derive(Clone, Debug, PartialEq, Eq)] pub enum Filter { @@ -168,7 +165,7 @@ pub enum Filter { #[cfg(feature = "blosc")] Blosc(Blosc, u8, BloscShuffle), #[cfg(feature = "zfp")] - Zfp(ZfpMode,Vec,u8), + Zfp(ZfpMode, Vec, u8), User(H5Z_filter_t, Vec), } @@ -241,7 +238,7 @@ impl Filter { #[cfg(feature = "blosc")] Self::Blosc(_, _, _) => blosc::BLOSC_FILTER_ID, #[cfg(feature = "zfp")] - Self::Zfp(_,_,_) => zfp::ZFP_FILTER_ID, + Self::Zfp(_, _, _) => zfp::ZFP_FILTER_ID, Self::User(id, _) => *id, } } @@ -357,28 +354,28 @@ impl Filter { } #[cfg(feature = "zfp")] - pub fn zfp(mode: ZfpMode,chunk_dims: Vec,n_bytes: u8) -> Self { - Self::Zfp(mode,chunk_dims,n_bytes) + pub fn zfp(mode: ZfpMode, chunk_dims: Vec, n_bytes: u8) -> Self { + Self::Zfp(mode, chunk_dims, n_bytes) } #[cfg(feature = "zfp")] - pub fn zfp_rate(rate: f64,chunk_dims: Vec,n_bytes: u8) -> Self { - Self::zfp(ZfpMode::FixedRate(rate),chunk_dims,n_bytes) + pub fn zfp_rate(rate: f64, chunk_dims: Vec, n_bytes: u8) -> Self { + Self::zfp(ZfpMode::FixedRate(rate), chunk_dims, n_bytes) } #[cfg(feature = "zfp")] - pub fn zfp_precision(precision: u8,chunk_dims: Vec,n_bytes: u8) -> Self { - Self::zfp(ZfpMode::FixedPrecision(precision),chunk_dims,n_bytes) + pub fn zfp_precision(precision: u8, chunk_dims: Vec, n_bytes: u8) -> Self { + Self::zfp(ZfpMode::FixedPrecision(precision), chunk_dims, n_bytes) } #[cfg(feature = "zfp")] - pub fn zfp_accuracy(accuracy: f64,chunk_dims: Vec, n_bytes: u8) -> Self { - Self::zfp(ZfpMode::FixedAccuracy(accuracy),chunk_dims,n_bytes) + pub fn zfp_accuracy(accuracy: f64, chunk_dims: Vec, n_bytes: u8) -> Self { + Self::zfp(ZfpMode::FixedAccuracy(accuracy), chunk_dims, n_bytes) } #[cfg(feature = "zfp")] - pub fn zfp_reversible(chunk_dims: Vec,n_bytes: u8) -> Self { - Self::zfp(ZfpMode::Reversible,chunk_dims,n_bytes) + pub fn zfp_reversible(chunk_dims: Vec, n_bytes: u8) -> Self { + Self::zfp(ZfpMode::Reversible, chunk_dims, n_bytes) } pub fn user(id: H5Z_filter_t, cdata: &[c_uint]) -> Self { @@ -509,7 +506,7 @@ impl Filter { 5 => ZfpMode::Reversible, _ => fail!("invalid zfp mode: {}", mode), }; - Ok(Self::zfp(zfp_mode,chunk_dims,n_bytes)) + Ok(Self::zfp(zfp_mode, chunk_dims, n_bytes)) } pub fn from_raw(filter_id: H5Z_filter_t, cdata: &[c_uint]) -> Result { @@ -595,8 +592,6 @@ impl Filter { Self::apply_user(plist_id, blosc::BLOSC_FILTER_ID, &cdata) } - - #[cfg(feature = "zfp")] /// Applies the ZFP filter to the given property list. /// @@ -616,17 +611,16 @@ impl Filter { /// /// # Returns /// - `herr_t`: Returns 0 on success, or a negative value on failure. - unsafe fn apply_zfp(plist_id: hid_t, n_bytes: u8,chunk_dims: Vec, mode: ZfpMode) -> herr_t { + unsafe fn apply_zfp( + plist_id: hid_t, n_bytes: u8, chunk_dims: Vec, mode: ZfpMode, + ) -> herr_t { // get the chunk dimensiosn out of it. Could not reliably get the chunk_dims from plist_id // during testing so opted to just pass it in during the build let ndims = chunk_dims.len(); // Convert to `usize` and trim to used dims. - let chunk_dims_usize: Vec = chunk_dims[..(ndims as usize)] - .iter() - .map(|&d| d as usize) - .collect(); - + let chunk_dims_usize: Vec = + chunk_dims[..(ndims as usize)].iter().map(|&d| d as usize).collect(); // remove the singletons from the data let mut dims_no_singleton: Vec = Vec::new(); @@ -637,21 +631,22 @@ impl Filter { } let ndims_no_singleton = dims_no_singleton.len(); - - assert!(dims_no_singleton.len()<= zfp::MAX_NDIMS); - + assert!(dims_no_singleton.len() <= zfp::MAX_NDIMS); // Get the type of the input data - let dtype_id = match n_bytes{ + let dtype_id = match n_bytes { 4 => zfp_type_zfp_type_float, 8 => zfp_type_zfp_type_double, _ => { - h5err!("ZFP filter only supports 4 or 8 byte floating point data", H5E_PLIST, H5E_CALLBACK); + h5err!( + "ZFP filter only supports 4 or 8 byte floating point data", + H5E_PLIST, + H5E_CALLBACK + ); return -1; } }; - // Build the Mode Information we need let (mode_val, param1, param2) = match mode { ZfpMode::FixedRate(rate) => { @@ -667,7 +662,8 @@ impl Filter { }; // update values and encode into the header - let (hdr_cd_values, _) = zfp::compute_hdr_cd_values(dtype_id,ndims_no_singleton,&dims_no_singleton, mode); + let (hdr_cd_values, _) = + zfp::compute_hdr_cd_values(dtype_id, ndims_no_singleton, &dims_no_singleton, mode); let hdf_cd_values_pass = hdr_cd_values.iter().map(|x| *x).collect::>(); Self::apply_user(plist_id, zfp::ZFP_FILTER_ID, &hdf_cd_values_pass) } @@ -697,7 +693,8 @@ impl Filter { Self::apply_blosc(id, *complib, *clevel, *shuffle) } #[cfg(feature = "zfp")] - Self::Zfp(mode,chunk_dims, n_bytes) => Self::apply_zfp(id,*n_bytes,chunk_dims.clone(), *mode), + Self::Zfp(mode, chunk_dims, n_bytes) => + Self::apply_zfp(id, *n_bytes, chunk_dims.clone(), *mode), Self::User(filter_id, ref cdata) => Self::apply_user(id, *filter_id, cdata), }); Ok(()) @@ -796,8 +793,6 @@ mod tests { use crate::test::with_tmp_file; use crate::{plist::DatasetCreate, Result}; - - #[test] fn test_filter_pipeline() -> Result<()> { let mut comp_filters = vec![]; @@ -828,9 +823,9 @@ mod tests { assert_eq!(cfg!(feature = "zfp"), zfp_available()); #[cfg(feature = "zfp")] { - comp_filters.push(Filter::zfp_rate(8.0,vec![10_000,20],4)); - comp_filters.push(Filter::zfp_precision(16,vec![10_000,20],4)); - comp_filters.push(Filter::zfp_accuracy(1e-3,vec![10_000,20],4)); + comp_filters.push(Filter::zfp_rate(8.0, vec![10_000, 20], 4)); + comp_filters.push(Filter::zfp_precision(16, vec![10_000, 20], 4)); + comp_filters.push(Filter::zfp_accuracy(1e-3, vec![10_000, 20], 4)); } for c in &comp_filters { @@ -885,7 +880,6 @@ mod tests { Ok(()) } - #[test] #[cfg(feature = "zfp")] fn test_zfp_accuracy() -> Result<()> { @@ -907,7 +901,6 @@ mod tests { .create("zfp_precision_1d") .unwrap(); - let ds = file.dataset("zfp_precision_1d").unwrap(); let read_data: Vec = ds.read_raw().unwrap(); @@ -928,11 +921,10 @@ mod tests { } }); - // Test 2D data with_tmp_file(|file| { let data = ndarray::Array1::::linspace(0.0, 1.0, 1000); - let data = data.to_shape((10,100)).unwrap(); + let data = data.to_shape((10, 100)).unwrap(); file.new_dataset_builder() .with_data(&data) .chunk((5, 10)) @@ -940,7 +932,6 @@ mod tests { .create("zfp_precision_1d") .unwrap(); - let ds = file.dataset("zfp_precision_1d").unwrap(); let read_data: Vec = ds.read_raw().unwrap(); @@ -961,7 +952,6 @@ mod tests { } }); - // Test 3D data with_tmp_file(|file| { let data = ndarray::Array1::::linspace(0.0, 1.0, 10000); @@ -969,12 +959,11 @@ mod tests { file.new_dataset_builder() .with_data(&data) - .chunk((2, 5, 25,)) + .chunk((2, 5, 25)) .zfp_accuracy(0.125, vec![2, 5, 25], 4) .create("zfp_precision_3d") .unwrap(); - let ds = file.dataset("zfp_precision_3d").unwrap(); let read_data: Vec = ds.read_raw().unwrap(); @@ -996,19 +985,17 @@ mod tests { } }); - // Test 4D data with_tmp_file(|file| { let data = ndarray::Array1::::linspace(0.0, 1.0, 100000); let data = data.to_shape((10, 10, 10, 100)).unwrap(); file.new_dataset_builder() .with_data(&data) - .chunk((2, 2, 5, 50,)) + .chunk((2, 2, 5, 50)) .zfp_accuracy(0.125, vec![2, 2, 5, 50], 4) .create("zfp_precision_1d") .unwrap(); - let ds = file.dataset("zfp_precision_1d").unwrap(); let read_data: Vec = ds.read_raw().unwrap(); @@ -1034,7 +1021,7 @@ mod tests { #[test] #[cfg(feature = "zfp")] - fn test_over_dim_data() -> Result<()>{ + fn test_over_dim_data() -> Result<()> { use super::zfp_available; if !zfp_available() { @@ -1043,28 +1030,26 @@ mod tests { return Ok(()); } - // Test 5D data with 3D chunks but should still fail // test 1D Data with_tmp_file(|file| { let data = ndarray::Array1::::linspace(0.0, 1.0, 50_000); - let data = data.to_shape((2,5,10,10,50)).unwrap(); + let data = data.to_shape((2, 5, 10, 10, 50)).unwrap(); - let bad_result = file.new_dataset_builder() + let bad_result = file + .new_dataset_builder() .with_data(&data) - .chunk((2,5,5,1,1)) - .zfp_accuracy(0.125,vec![2,5,5,1,1],4) - .create("zfp_precision_1d").unwrap_err(); + .chunk((2, 5, 5, 1, 1)) + .zfp_accuracy(0.125, vec![2, 5, 5, 1, 1], 4) + .create("zfp_precision_1d") + .unwrap_err(); assert_err!(bad_result, "ZFP filter supports up to 4D data only"); - - }); Ok(()) } - #[test] #[cfg(feature = "zfp")] fn test_zfp_reversible() -> Result<()> { @@ -1080,21 +1065,17 @@ mod tests { let data = data.insert_axis(Axis(0)); let data = data.insert_axis(Axis(0)); file.new_dataset_builder() - .chunk((1,1,960)) - .zfp_reversible(vec![1,1,960],4) + .chunk((1, 1, 960)) + .zfp_reversible(vec![1, 1, 960], 4) .with_data(&data) .create("zfp_reversible") .unwrap(); - let ds = file.dataset("zfp_reversible").unwrap(); - - let read_data: Vec = ds.read_raw().unwrap(); let n_bytes = file.size(); - // ZFP is lossy, so we check approximate equality assert_eq!(read_data.len(), data.len()); dbg!(&data.clone().into_raw_vec_and_offset().0[0..15]); @@ -1122,13 +1103,9 @@ mod tests { } }); - - - Ok(()) } - #[test] #[cfg(feature = "zfp")] fn test_zfp_rate() -> Result<()> { @@ -1144,11 +1121,10 @@ mod tests { file.new_dataset_builder() .with_data(&data) .chunk((1000,)) - .zfp_rate(2.0,vec![1000],4) + .zfp_rate(2.0, vec![1000], 4) .create("zfp_rate") .unwrap(); - let ds = file.dataset("zfp_rate").unwrap(); let read_data: Vec = ds.read_raw().unwrap(); @@ -1163,7 +1139,6 @@ mod tests { } }); - // test full rate compression. Should be "lossless" with_tmp_file(|file| { @@ -1171,11 +1146,10 @@ mod tests { file.new_dataset_builder() .with_data(&data) .chunk((1000,)) - .zfp_rate(32.0,vec![1000],4) + .zfp_rate(32.0, vec![1000], 4) .create("zfp_rate") .unwrap(); - let ds = file.dataset("zfp_rate").unwrap(); let read_data: Vec = ds.read_raw().unwrap(); @@ -1198,10 +1172,6 @@ mod tests { } }); - Ok(()) } - - } - diff --git a/hdf5/src/hl/filters/zfp.rs b/hdf5/src/hl/filters/zfp.rs index c9b94f9ad..4568226a6 100644 --- a/hdf5/src/hl/filters/zfp.rs +++ b/hdf5/src/hl/filters/zfp.rs @@ -10,18 +10,24 @@ use crate::error::H5ErrorCode; use crate::globals::{H5E_CALLBACK, H5E_PLIST}; use crate::internal_prelude::*; +use zfp_sys::zfp_stream; pub use zfp_sys::{ - zfp_field_alloc,zfp_read_header,zfp_field_dimensionality,zfp_field_size,zfp_field_type,zfp_mode,zfp_mode_zfp_mode_fixed_accuracy,zfp_mode_zfp_mode_fixed_precision,zfp_mode_zfp_mode_fixed_rate,zfp_stream_compression_mode,zfp_stream_accuracy,zfp_stream_rate,zfp_stream_precision,stream_close, stream_open, zfp_compress, zfp_field_metadata,zfp_decompress, zfp_field_1d, zfp_field_2d,bitstream,zfp_stream_set_reversible, - zfp_field_3d, zfp_field_4d, zfp_field_free, zfp_stream_close, zfp_stream_maximum_size,zfp_stream_flush, - zfp_stream_open, zfp_stream_rewind, zfp_stream_set_accuracy, zfp_stream_set_bit_stream,ZFP_VERSION_MINOR,ZFP_VERSION_PATCH, - zfp_stream_set_precision, zfp_stream_set_rate, zfp_type_zfp_type_double,zfp_type,ZFP_HEADER_FULL,ZFP_VERSION_MAJOR,ZFP_VERSION_TWEAK, - zfp_type_zfp_type_float,ZFP_HEADER_MAGIC,ZFP_HEADER_MAX_BITS,ZFP_HEADER_META,ZFP_HEADER_MODE,zfp_write_header,zfp_codec_version,zfp_library_version,zfp_field + bitstream, stream_close, stream_open, zfp_codec_version, zfp_compress, zfp_decompress, + zfp_field, zfp_field_1d, zfp_field_2d, zfp_field_3d, zfp_field_4d, zfp_field_alloc, + zfp_field_dimensionality, zfp_field_free, zfp_field_metadata, zfp_field_size, zfp_field_type, + zfp_library_version, zfp_mode, zfp_mode_zfp_mode_fixed_accuracy, + zfp_mode_zfp_mode_fixed_precision, zfp_mode_zfp_mode_fixed_rate, zfp_read_header, + zfp_stream_accuracy, zfp_stream_close, zfp_stream_compression_mode, zfp_stream_flush, + zfp_stream_maximum_size, zfp_stream_open, zfp_stream_precision, zfp_stream_rate, + zfp_stream_rewind, zfp_stream_set_accuracy, zfp_stream_set_bit_stream, + zfp_stream_set_precision, zfp_stream_set_rate, zfp_stream_set_reversible, zfp_type, + zfp_type_zfp_type_double, zfp_type_zfp_type_float, zfp_write_header, ZFP_HEADER_FULL, + ZFP_HEADER_MAGIC, ZFP_HEADER_MAX_BITS, ZFP_HEADER_META, ZFP_HEADER_MODE, ZFP_VERSION_MAJOR, + ZFP_VERSION_MINOR, ZFP_VERSION_PATCH, ZFP_VERSION_TWEAK, }; -use zfp_sys::zfp_stream; use crate::filters::ZfpMode; - /// Major edits are needed to be in alignmeht with the H5Z-ZFP. What was previously implemented was /// effectively a new implementation of H5Z_ZFP but was incompatible with any library built against /// it. This reults in bad c_data vectors being created and produces erratic behavior. @@ -39,7 +45,6 @@ const ZFP_MODE_ACCURACY: c_uint = 4; const ZFP_MODE_REVERSIBLE: c_uint = 5; const ZFP_MODE_EXPERT: c_uint = 1; - const ZFP_FILTER_INFO: &H5Z_class2_t = &H5Z_class2_t { version: H5Z_CLASS_T_VERS as _, id: ZFP_FILTER_ID, @@ -72,7 +77,6 @@ extern "C" fn can_apply_zfp(_dcpl_id: hid_t, type_id: hid_t, _space_id: hid_t) - } } - /// Sets the local properties for the ZFP filter. /// /// This function is called during the creation of a dataset or attribute to set @@ -142,7 +146,6 @@ extern "C" fn set_local_zfp(dcpl_id: hid_t, type_id: hid_t, _space_id: hid_t) -> values[7] = orig.get(0).copied().unwrap_or(0); values[8] = orig.get(1).copied().unwrap_or(0); values[9] = orig.get(2).copied().unwrap_or(0); - } // temp overrid and changed line 133 to orig instead of values let nelmts = 4; @@ -155,11 +158,8 @@ extern "C" fn set_local_zfp(dcpl_id: hid_t, type_id: hid_t, _space_id: hid_t) -> } } - - const H5Z_ZFP_CD_NELMTS_MAX: usize = 8; // whatever the header says; set correctly. - /// Computes the header and configuration data values for the ZFP filter. /// /// This function generates the header and configuration data values (`cd_values`) @@ -189,9 +189,27 @@ pub unsafe fn compute_hdr_cd_values( // 1. Build dummy_field like H5Z_zfp_set_local let dummy_field: *mut zfp_field = match ndims_used { 1 => zfp_field_1d(ptr::null_mut(), zt, dims_used[0].try_into().unwrap()), - 2 => zfp_field_2d(ptr::null_mut(), zt, dims_used[1].try_into().unwrap(), dims_used[0].try_into().unwrap()), - 3 => zfp_field_3d(ptr::null_mut(), zt, dims_used[2].try_into().unwrap(), dims_used[1].try_into().unwrap(), dims_used[0].try_into().unwrap()), - 4 => zfp_field_4d(ptr::null_mut(), zt, dims_used[3].try_into().unwrap(), dims_used[2].try_into().unwrap(), dims_used[1].try_into().unwrap(), dims_used[0].try_into().unwrap()), + 2 => zfp_field_2d( + ptr::null_mut(), + zt, + dims_used[1].try_into().unwrap(), + dims_used[0].try_into().unwrap(), + ), + 3 => zfp_field_3d( + ptr::null_mut(), + zt, + dims_used[2].try_into().unwrap(), + dims_used[1].try_into().unwrap(), + dims_used[0].try_into().unwrap(), + ), + 4 => zfp_field_4d( + ptr::null_mut(), + zt, + dims_used[3].try_into().unwrap(), + dims_used[2].try_into().unwrap(), + dims_used[1].try_into().unwrap(), + dims_used[0].try_into().unwrap(), + ), _ => panic!("ZFP supports 1..4 non-unity dims"), }; assert!(!dummy_field.is_null()); @@ -213,23 +231,21 @@ pub unsafe fn compute_hdr_cd_values( match mode { ZfpMode::Reversible => { zfp_stream_set_reversible(dummy_zstr); - }, - ZfpMode::FixedAccuracy(acc) =>{ - - zfp_stream_set_accuracy(dummy_zstr,acc); - }, - - ZfpMode::FixedRate(rate) =>{ - zfp_stream_set_rate(dummy_zstr,rate, zt, ndims_used as u32,0); - }, - ZfpMode::FixedPrecision(precision) =>{ - zfp_stream_set_precision(dummy_zstr,precision as u32); - }, + } + ZfpMode::FixedAccuracy(acc) => { + zfp_stream_set_accuracy(dummy_zstr, acc); + } + + ZfpMode::FixedRate(rate) => { + zfp_stream_set_rate(dummy_zstr, rate, zt, ndims_used as u32, 0); + } + ZfpMode::FixedPrecision(precision) => { + zfp_stream_set_precision(dummy_zstr, precision as u32); + } // handle Rate/Precision/Accuracy/Expert as needed _ => unimplemented!(), } - // 6. Write FULL header (critical!) into the hdr_cd_values[1..] buffer let hdr_bits = zfp_write_header(dummy_zstr, dummy_field, ZFP_HEADER_FULL as u32); assert!(hdr_bits != 0); @@ -248,8 +264,6 @@ pub unsafe fn compute_hdr_cd_values( (hdr_cd_values, hdr_cd_nelmts) } - - /// Constructs a version word for the ZFP filter. /// /// This function generates a 32-bit version word that encodes the ZFP library version, @@ -261,13 +275,11 @@ pub unsafe fn compute_hdr_cd_values( /// # Returns /// A 32-bit unsigned integer representing the version word. unsafe fn make_version_word() -> u32 { - // 0xM M P T: for 1.0.0.0 → 0x1000 - const ZFP_VERSION_NO: u32 = - (ZFP_VERSION_MAJOR << 12) - | (ZFP_VERSION_MINOR << 8) - | (ZFP_VERSION_PATCH << 4) - | (ZFP_VERSION_TWEAK); + const ZFP_VERSION_NO: u32 = (ZFP_VERSION_MAJOR << 12) + | (ZFP_VERSION_MINOR << 8) + | (ZFP_VERSION_PATCH << 4) + | (ZFP_VERSION_TWEAK); const ZFP_CODEC: u32 = ZFP_VERSION_MINOR; // or 5 if you know you want codec 5 @@ -276,19 +288,14 @@ unsafe fn make_version_word() -> u32 { const H5Z_FILTER_ZFP_VERSION_MINOR: u32 = 1; const H5Z_FILTER_ZFP_VERSION_PATCH: u32 = 0; - const H5Z_FILTER_ZFP_VERSION_NO: u32 = - (H5Z_FILTER_ZFP_VERSION_MAJOR << 8) - | (H5Z_FILTER_ZFP_VERSION_MINOR << 4) - | (H5Z_FILTER_ZFP_VERSION_PATCH); + const H5Z_FILTER_ZFP_VERSION_NO: u32 = (H5Z_FILTER_ZFP_VERSION_MAJOR << 8) + | (H5Z_FILTER_ZFP_VERSION_MINOR << 4) + | (H5Z_FILTER_ZFP_VERSION_PATCH); // One simple scheme: low 8 bits = codec, high 24 bits = lib version truncated. - (ZFP_VERSION_NO << 16) - | (ZFP_CODEC << 12) - | H5Z_FILTER_ZFP_VERSION_NO + (ZFP_VERSION_NO << 16) | (ZFP_CODEC << 12) | H5Z_FILTER_ZFP_VERSION_NO } - - #[derive(Debug)] struct ZfpConfig { pub ndims: c_int, @@ -300,7 +307,6 @@ struct ZfpConfig { pub accuracy: f64, } - /// Parses ZFP filter configuration data from the given input. /// /// This function extracts metadata and compression parameters from the @@ -321,10 +327,7 @@ struct ZfpConfig { /// - `Option`: Returns a `ZfpConfig` struct containing the parsed /// metadata and compression parameters if successful, or `None` if the /// parsing fails. -pub unsafe fn parse_zfp_cdata( - cd_nelmts: usize, - cd_values: *const c_uint, -) -> Option { +pub unsafe fn parse_zfp_cdata(cd_nelmts: usize, cd_values: *const c_uint) -> Option { if cd_nelmts < 2 || cd_values.is_null() { return None; } @@ -346,8 +349,7 @@ pub unsafe fn parse_zfp_cdata( let header_bytes = header_copy.len() * std::mem::size_of::(); // Open bitstream on the header buffer (like get_zfp_info_from_cd_values) - let bstr: *mut bitstream = - stream_open(header_copy.as_mut_ptr() as *mut c_void, header_bytes); + let bstr: *mut bitstream = stream_open(header_copy.as_mut_ptr() as *mut c_void, header_bytes); if bstr.is_null() { return None; } @@ -402,10 +404,7 @@ pub unsafe fn parse_zfp_cdata( if ndims > 0 { // zfp_field_size returns total number of elements and optionally fills size[i]. // The C signature uses size_t*; just alias &mut [usize] here. - zfp_field_size( - zfld, - size_per_dim.as_mut_ptr() as *mut _, - ); + zfp_field_size(zfld, size_per_dim.as_mut_ptr() as *mut _); } let mut dims: [usize; 4] = [0; 4]; @@ -446,14 +445,13 @@ pub unsafe fn parse_zfp_cdata( } m if m == zfp_sys::zfp_mode_zfp_mode_fixed_accuracy => { accuracy = zfp_stream_accuracy(zstr); - }, + } m if m == zfp_sys::zfp_mode_zfp_mode_reversible => { // no params needed } // Expert or reversible -> we don’t have a single scalar parameter to expose - _ => { - } + _ => {} } //Cleanup @@ -461,18 +459,9 @@ pub unsafe fn parse_zfp_cdata( zfp_stream_close(zstr); stream_close(bstr); - Some(ZfpConfig { - ndims, - typesize, - dims, - mode, - rate, - precision, - accuracy, - }) + Some(ZfpConfig { ndims, typesize, dims, mode, rate, precision, accuracy }) } - /// Applies the ZFP filter for compression or decompression. /// /// This function serves as the entry point for the ZFP filter, determining whether @@ -501,7 +490,6 @@ unsafe extern "C" fn filter_zfp( ) -> size_t { let cfg = if let Some(cfg) = parse_zfp_cdata(cd_nelmts, cd_values) { cfg - } else { return 0; }; @@ -539,7 +527,6 @@ unsafe fn filter_zfp_compress( } } - let field = if cfg.typesize == 4 { match cfg.ndims { 1 => zfp_field_1d((*buf).cast(), zfp_type_zfp_type_float, cfg.dims[0]), diff --git a/hdf5/src/hl/plist/dataset_create.rs b/hdf5/src/hl/plist/dataset_create.rs index a6ac7a348..099edcc08 100644 --- a/hdf5/src/hl/plist/dataset_create.rs +++ b/hdf5/src/hl/plist/dataset_create.rs @@ -476,27 +476,31 @@ impl DatasetCreateBuilder { self } - #[cfg(feature="zfp")] - pub fn zfp_accuracy(&mut self, accuracy: f64,chunk_dims: Vec, n_bytes:u8) -> &mut Self { - self.filters.push(Filter::zfp_accuracy(accuracy,chunk_dims,n_bytes)); + #[cfg(feature = "zfp")] + pub fn zfp_accuracy( + &mut self, accuracy: f64, chunk_dims: Vec, n_bytes: u8, + ) -> &mut Self { + self.filters.push(Filter::zfp_accuracy(accuracy, chunk_dims, n_bytes)); self } - #[cfg(feature="zfp")] - pub fn zfp_rate(&mut self, rate: f64,chunk_dims: Vec, n_bytes:u8) -> &mut Self { - self.filters.push(Filter::zfp_rate(rate,chunk_dims,n_bytes)); + #[cfg(feature = "zfp")] + pub fn zfp_rate(&mut self, rate: f64, chunk_dims: Vec, n_bytes: u8) -> &mut Self { + self.filters.push(Filter::zfp_rate(rate, chunk_dims, n_bytes)); self } - #[cfg(feature="zfp")] - pub fn zfp_precision(&mut self, precision: u8,chunk_dims: Vec, n_bytes:u8) -> &mut Self { - self.filters.push(Filter::zfp_precision(precision,chunk_dims,n_bytes)); + #[cfg(feature = "zfp")] + pub fn zfp_precision( + &mut self, precision: u8, chunk_dims: Vec, n_bytes: u8, + ) -> &mut Self { + self.filters.push(Filter::zfp_precision(precision, chunk_dims, n_bytes)); self } - #[cfg(feature="zfp")] - pub fn zfp_reversible(&mut self,chunk_dims: Vec, n_bytes:u8) -> &mut Self { - self.filters.push(Filter::zfp_reversible(chunk_dims,n_bytes)); + #[cfg(feature = "zfp")] + pub fn zfp_reversible(&mut self, chunk_dims: Vec, n_bytes: u8) -> &mut Self { + self.filters.push(Filter::zfp_reversible(chunk_dims, n_bytes)); self } From 026521e4674ae43274bff377f05f45fd73c10996 Mon Sep 17 00:00:00 2001 From: Frank Blubaugh Date: Tue, 9 Dec 2025 10:01:49 -0500 Subject: [PATCH 095/141] spell check --- hdf5/src/hl/filters/zfp.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hdf5/src/hl/filters/zfp.rs b/hdf5/src/hl/filters/zfp.rs index 4568226a6..c8411ddc6 100644 --- a/hdf5/src/hl/filters/zfp.rs +++ b/hdf5/src/hl/filters/zfp.rs @@ -30,7 +30,7 @@ use crate::filters::ZfpMode; /// Major edits are needed to be in alignmeht with the H5Z-ZFP. What was previously implemented was /// effectively a new implementation of H5Z_ZFP but was incompatible with any library built against -/// it. This reults in bad c_data vectors being created and produces erratic behavior. +/// it. This results in bad c_data vectors being created and produces erratic behavior. pub(crate) const MAX_NDIMS: usize = 4; From 696fdb869ca609daa21313cfc13fef573244da5b Mon Sep 17 00:00:00 2001 From: Ryan Lowe Date: Fri, 23 Jun 2023 20:27:24 -0400 Subject: [PATCH 096/141] Add minimal docs for public items --- hdf5-derive/src/lib.rs | 1 + hdf5-types/src/array.rs | 14 ++ hdf5-types/src/dyn_value.rs | 24 ++- hdf5-types/src/h5type.rs | 57 +++++++ hdf5-types/src/lib.rs | 1 + hdf5-types/src/string.rs | 66 ++++++++ hdf5/src/class.rs | 14 ++ hdf5/src/error.rs | 10 ++ hdf5/src/hl/attribute.rs | 10 ++ hdf5/src/hl/container.rs | 12 ++ hdf5/src/hl/dataset.rs | 12 +- hdf5/src/hl/dataspace.rs | 34 ++++ hdf5/src/hl/datatype.rs | 17 ++ hdf5/src/hl/extents.rs | 29 ++++ hdf5/src/hl/filters.rs | 55 +++++++ hdf5/src/hl/group.rs | 5 + hdf5/src/hl/location.rs | 23 +++ hdf5/src/hl/object.rs | 3 +- hdf5/src/hl/plist.rs | 1 + hdf5/src/hl/plist/dataset_access.rs | 20 +++ hdf5/src/hl/plist/dataset_create.rs | 81 ++++++++++ hdf5/src/hl/plist/file_access.rs | 233 ++++++++++++++++++++++++++++ hdf5/src/hl/plist/file_create.rs | 5 + hdf5/src/hl/plist/link_create.rs | 12 ++ hdf5/src/hl/selection.rs | 21 +++ hdf5/src/lib.rs | 12 ++ 26 files changed, 766 insertions(+), 6 deletions(-) diff --git a/hdf5-derive/src/lib.rs b/hdf5-derive/src/lib.rs index 5f8e01f88..90a3b2c57 100644 --- a/hdf5-derive/src/lib.rs +++ b/hdf5-derive/src/lib.rs @@ -12,6 +12,7 @@ use syn::{ TypeGenerics, TypePath, }; +/// Derive macro generating an impl of the trait `H5Type`. #[proc_macro_derive(H5Type, attributes(hdf5))] #[proc_macro_error] pub fn derive(input: proc_macro::TokenStream) -> proc_macro::TokenStream { diff --git a/hdf5-types/src/array.rs b/hdf5-types/src/array.rs index c6438e2b9..1345bd1b4 100644 --- a/hdf5-types/src/array.rs +++ b/hdf5-types/src/array.rs @@ -5,6 +5,7 @@ use std::ops::Deref; use std::ptr; use std::slice; +/// A variable-length array. #[repr(C)] pub struct VarLenArray { len: usize, @@ -13,6 +14,14 @@ pub struct VarLenArray { } impl VarLenArray { + /// Creates a `VarLenArray` by copying the first `len` elements stored at `p`. + /// + /// Returns an empty array if `p` is null. + /// + /// # Safety + /// + /// - `p` must be valid for reads of `len * size_of::()` bytes. + /// - `p` must point to `len` consecutive properly initialized and aligned values of type `T`. pub unsafe fn from_parts(p: *const T, len: usize) -> Self { let (len, ptr) = if !p.is_null() && len != 0 { let dst = crate::malloc(len * mem::size_of::()); @@ -24,26 +33,31 @@ impl VarLenArray { Self { len, ptr: ptr as *const _, tag: PhantomData } } + /// Creates a `VarLenArray` from a slice by copying its elements. #[inline] pub fn from_slice(arr: &[T]) -> Self { unsafe { Self::from_parts(arr.as_ptr(), arr.len()) } } + /// Returns a raw pointer to the array's buffer. #[inline] pub fn as_ptr(&self) -> *const T { self.ptr } + /// Returns the number of elements in the array. #[inline] pub fn len(&self) -> usize { self.len as _ } + /// Returns `true` if the array has a length of zero. #[inline] pub fn is_empty(&self) -> bool { self.len == 0 } + /// Returns a slice containing the entire array. #[inline] pub fn as_slice(&self) -> &[T] { self diff --git a/hdf5-types/src/dyn_value.rs b/hdf5-types/src/dyn_value.rs index 433668349..f7b39d90a 100644 --- a/hdf5-types/src/dyn_value.rs +++ b/hdf5-types/src/dyn_value.rs @@ -1,3 +1,5 @@ +//! Dynamically-typed values. + use std::fmt::{self, Debug, Display}; use std::mem; use std::ptr; @@ -26,6 +28,7 @@ unsafe trait DynClone { fn dyn_clone(&mut self, out: &mut [u8]); } +/// A dynamically-typed integer. #[derive(Copy, Clone, PartialEq, Eq)] pub enum DynInteger { Int8(i8), @@ -114,6 +117,7 @@ impl From for DynValue<'_> { } } +/// A dynamically-typed floating-point value. #[derive(Copy, Clone, PartialEq)] pub enum DynFloat { #[cfg(feature = "f16")] @@ -173,6 +177,7 @@ impl From for DynValue<'_> { } } +/// A dynamically-typed scalar value. #[derive(Copy, Clone, PartialEq)] pub enum DynScalar { Integer(DynInteger), @@ -212,6 +217,7 @@ impl From for DynValue<'static> { } } +/// A dynamically-typed enumeration value. #[derive(Copy, Clone)] pub struct DynEnum<'a> { tp: &'a EnumType, @@ -269,6 +275,7 @@ impl<'a> From> for DynValue<'a> { } } +/// A dynamically-typed compound value. pub struct DynCompound<'a> { tp: &'a CompoundType, buf: &'a [u8], @@ -354,6 +361,7 @@ impl<'a> From> for DynValue<'a> { } } +/// A dynamically-typed array. pub struct DynArray<'a> { tp: &'a TypeDescriptor, buf: &'a [u8], @@ -470,6 +478,7 @@ impl<'a> From> for DynValue<'a> { } } +/// A fixed-length string with a dynamic encoding. pub struct DynFixedString<'a> { buf: &'a [u8], unicode: bool, @@ -529,6 +538,7 @@ impl<'a> From> for DynValue<'a> { } } +/// A variable-length string with a dynamic encoding. pub struct DynVarLenString<'a> { buf: &'a [u8], unicode: bool, @@ -633,6 +643,7 @@ impl<'a> From> for DynValue<'a> { } } +/// A dynamically-typed string. #[derive(PartialEq, Eq)] pub enum DynString<'a> { Fixed(DynFixedString<'a>), @@ -677,6 +688,7 @@ impl<'a> From> for DynValue<'a> { } } +/// A borrowed value with dynamic type. #[derive(PartialEq)] pub enum DynValue<'a> { Scalar(DynScalar), @@ -687,6 +699,7 @@ pub enum DynValue<'a> { } impl<'a> DynValue<'a> { + /// Constructs a new `DynValue` from a `TypeDescriptor` and a byte slice. pub fn new(tp: &'a TypeDescriptor, buf: &'a [u8]) -> Self { use TypeDescriptor::*; debug_assert_eq!(tp.size(), buf.len()); @@ -749,12 +762,14 @@ impl Display for DynValue<'_> { } } +/// An owned value with dynamic type. pub struct OwnedDynValue { tp: TypeDescriptor, buf: Box<[u8]>, } impl OwnedDynValue { + /// Constructs a new `OwnedDynValue` from the given value. pub fn new(value: T) -> Self { let ptr = (&value as *const T).cast::(); let len = mem::size_of_val(&value); @@ -763,10 +778,12 @@ impl OwnedDynValue { Self { tp: T::type_descriptor(), buf: buf.to_owned().into_boxed_slice() } } + /// Returns a borrowed version of the contained value. pub fn get(&self) -> DynValue<'_> { DynValue::new(&self.tp, &self.buf) } + /// Returns the value's type descriptor. pub fn type_descriptor(&self) -> &TypeDescriptor { &self.tp } @@ -781,9 +798,12 @@ impl OwnedDynValue { Self { tp, buf } } - /// Cast to the concrete type + /// Tries to downcast the value to a concrete type. + /// + /// # Errors /// - /// Will fail if the type-descriptors are not equal + /// If the type descriptors of `self` and `T` are not equal, this will fail and return a + /// `Result::Err` containing the original value. pub fn cast(mut self) -> Result { use mem::MaybeUninit; if self.tp != T::type_descriptor() { diff --git a/hdf5-types/src/h5type.rs b/hdf5-types/src/h5type.rs index e7b16babd..fcaf65757 100644 --- a/hdf5-types/src/h5type.rs +++ b/hdf5-types/src/h5type.rs @@ -14,15 +14,21 @@ pub(crate) struct hvl_t { pub ptr: *mut c_void, } +/// A valid integer size. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] pub enum IntSize { + /// 1 byte. U1 = 1, + /// 2 bytes. U2 = 2, + /// 4 bytes. U4 = 4, + /// 8 bytes. U8 = 8, } impl IntSize { + /// Returns an `IntSize` of `size` bytes, or `None` if `size` is invalid. pub const fn from_int(size: usize) -> Option { if size == 1 { Some(Self::U1) @@ -38,15 +44,20 @@ impl IntSize { } } +/// A valid floating-point number size. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] pub enum FloatSize { + /// 2 bytes. #[cfg(feature = "f16")] U2 = 2, + /// 4 bytes. U4 = 4, + /// 8 bytes. U8 = 8, } impl FloatSize { + /// Returns a `FloatSize` of `size` bytes, or `None` if `size` is invalid. pub const fn from_int(size: usize) -> Option { #[cfg(feature = "f16")] { @@ -62,20 +73,28 @@ impl FloatSize { } } +/// A descriptor for an enumeration datatype member. #[derive(Clone, Debug, PartialEq, Eq)] pub struct EnumMember { + /// The name of the member. pub name: String, + /// The value of the member. pub value: u64, } +/// A descriptor for an enumeration datatype. #[derive(Clone, Debug, PartialEq, Eq)] pub struct EnumType { + /// The size of the underlying integer type. pub size: IntSize, + /// Whether to use a signed integer. pub signed: bool, + /// The enumeration datatype members. pub members: Vec, } impl EnumType { + /// Returns the type descriptor of the underlying integer datatype. #[inline] pub fn base_type(&self) -> TypeDescriptor { if self.signed { @@ -86,31 +105,42 @@ impl EnumType { } } +/// A descriptor for a compound datatype field. #[derive(Clone, Debug, PartialEq, Eq)] pub struct CompoundField { + /// The name of the field. pub name: String, + /// The type of the field. pub ty: TypeDescriptor, + /// The byte offset of the field. pub offset: usize, + /// The ordering of the field within the compound type. pub index: usize, } impl CompoundField { + /// Creates a new `CompoundField`. pub fn new(name: &str, ty: TypeDescriptor, offset: usize, index: usize) -> Self { Self { name: name.to_owned(), ty, offset, index } } + /// Creates a new `CompoundField` for a concrete type. pub fn typed(name: &str, offset: usize, index: usize) -> Self { Self::new(name, T::type_descriptor(), offset, index) } } +/// A descriptor for a compound datatype. #[derive(Clone, Debug, PartialEq, Eq)] pub struct CompoundType { + /// The fields of the datatype. pub fields: Vec, + /// The size in bytes of the datatype. pub size: usize, } impl CompoundType { + /// Converts `self` to a C struct representation. pub fn to_c_repr(&self) -> Self { let mut layout = self.clone(); layout.fields.sort_by_key(|f| f.index); @@ -133,6 +163,7 @@ impl CompoundType { layout } + /// Converts `self` to a packed representation. pub fn to_packed_repr(&self) -> Self { let mut layout = self.clone(); layout.fields.sort_by_key(|f| f.index); @@ -146,19 +177,32 @@ impl CompoundType { } } +/// A descriptor for an HDF5 datatype. #[derive(Clone, Debug, PartialEq, Eq)] pub enum TypeDescriptor { + /// A signed integer. Integer(IntSize), + /// An unsigned integer. Unsigned(IntSize), + /// A floating-point number. Float(FloatSize), + /// A boolean value. Boolean, + /// An enumeration datatype. Enum(EnumType), + /// A compound datatype. Compound(CompoundType), + /// A fixed-length array. FixedArray(Box, usize), + /// A fixed-length ASCII string. FixedAscii(usize), + /// A fixed-length UTF-8 string. FixedUnicode(usize), + /// A variable-length array. VarLenArray(Box), + /// A variable-length ASCII string. VarLenAscii, + /// A variable-length UTF-8 string. VarLenUnicode, Reference(Reference), } @@ -224,6 +268,7 @@ impl TypeDescriptor { } } + /// Converts `self` to a C-compatible representation. pub fn to_c_repr(&self) -> Self { match *self { Self::Compound(ref compound) => Self::Compound(compound.to_c_repr()), @@ -233,6 +278,7 @@ impl TypeDescriptor { } } + /// Converts `self` to a packed representation. pub fn to_packed_repr(&self) -> Self { match *self { Self::Compound(ref compound) => Self::Compound(compound.to_packed_repr()), @@ -243,7 +289,18 @@ impl TypeDescriptor { } } +/// A type that can be represented as an HDF5 datatype. +/// +/// # Derivable +/// This trait can be used with `#[derive]`. +/// +/// To `derive` on structs, they must be one of: `repr(C)`, `repr(packed)`, or `repr(transparent)`, +/// and have at least one field, all of which must implement `H5Type`. +/// +/// To `derive` on enums, they must have an explicit `repr` and at least one variant, all of which +/// must be unit variants with explicit discriminants. pub unsafe trait H5Type: 'static { + /// Returns a descriptor for an equivalent HDF5 datatype. fn type_descriptor() -> TypeDescriptor; } diff --git a/hdf5-types/src/lib.rs b/hdf5-types/src/lib.rs index a46e05e2a..550c28ba7 100644 --- a/hdf5-types/src/lib.rs +++ b/hdf5-types/src/lib.rs @@ -54,6 +54,7 @@ pub(crate) unsafe fn free(ptr: *mut core::ffi::c_void) { } } +/// Whether this crate is using the HDF5 library for allocations instead of `libc`. pub const USING_H5_ALLOCATOR: bool = { cfg_if::cfg_if! { if #[cfg(any(feature = "h5-alloc", windows_dll))] { diff --git a/hdf5-types/src/string.rs b/hdf5-types/src/string.rs index 826478982..42377237f 100644 --- a/hdf5-types/src/string.rs +++ b/hdf5-types/src/string.rs @@ -11,6 +11,7 @@ use std::str::{self, FromStr}; use ascii::{AsAsciiStr, AsAsciiStrError, AsciiStr}; +/// Errors that can occur when attempting to interpret a byte sequence as a string. #[derive(Clone, Copy, PartialEq, Eq, Debug)] #[non_exhaustive] pub enum StringError { @@ -181,6 +182,7 @@ impl_string_traits!(VarLenUnicode); // ================================================================================ +/// A variable-length ASCII string. #[repr(C)] pub struct VarLenAscii { ptr: *mut u8, @@ -203,6 +205,7 @@ impl Clone for VarLenAscii { } impl VarLenAscii { + /// Creates a new empty `VarLenAscii` string. #[inline] pub fn new() -> Self { unsafe { @@ -220,6 +223,7 @@ impl VarLenAscii { Self { ptr } } + /// Returns the length of `self`. #[inline] pub fn len(&self) -> usize { if self.ptr.is_null() { @@ -230,16 +234,19 @@ impl VarLenAscii { unsafe { libc::strlen(self.ptr as *const _) } } + /// Returns `true` if `self` has a length of zero bytes. #[inline] pub fn is_empty(&self) -> bool { self.len() == 0 } + /// Returns a raw pointer to the string's buffer. #[inline] pub fn as_ptr(&self) -> *const u8 { self.ptr } + /// Returns the contents of the string as a byte slice. #[inline] pub fn as_bytes(&self) -> &[u8] { // Treat null as empty. This could be changed to option @@ -250,16 +257,28 @@ impl VarLenAscii { unsafe { slice::from_raw_parts(self.ptr as *const _, self.len()) } } + /// Returns the contents of the string as a string slice. #[inline] pub fn as_str(&self) -> &str { unsafe { str::from_utf8_unchecked(self.as_bytes()) } } + /// Converts a byte slice into a `VarLenAscii` without checking that the string contains only + /// non-zero ASCII bytes. + /// + /// # Safety + /// + /// The bytes must be valid ASCII and non-zero. #[inline] pub unsafe fn from_ascii_unchecked>(bytes: &B) -> Self { Self::from_bytes(bytes.as_ref()) } + /// Converts a byte slice into a `VarLenAscii`. + /// + /// # Errors + /// + /// Returns `Err` if the bytes are not valid ASCII, or if any byte is zero. pub fn from_ascii>(bytes: &B) -> Result { let bytes = bytes.as_ref(); if !bytes.iter().all(|&c| c != 0) { @@ -299,6 +318,7 @@ unsafe impl Sync for VarLenAscii {} // ================================================================================ +/// A variable-length UTF-8 string. #[repr(C)] pub struct VarLenUnicode { ptr: *mut u8, @@ -321,6 +341,7 @@ impl Clone for VarLenUnicode { } impl VarLenUnicode { + /// Creates a new empty `VarLenUnicode` string. #[inline] pub fn new() -> Self { unsafe { @@ -348,21 +369,25 @@ impl VarLenUnicode { libc::strlen(self.ptr as *const _) } + /// Returns the length of `self`. #[inline] pub fn len(&self) -> usize { self.as_str().len() } + /// Returns `true` if `self` has a length of zero bytes. #[inline] pub fn is_empty(&self) -> bool { unsafe { self.raw_len() == 0 } } + /// Returns a raw pointer to the string's buffer. #[inline] pub fn as_ptr(&self) -> *const u8 { self.ptr } + /// Returns the contents of the string as a byte slice. #[inline] pub fn as_bytes(&self) -> &[u8] { // Treat null as empty. This could be changed to option @@ -373,11 +398,18 @@ impl VarLenUnicode { unsafe { slice::from_raw_parts(self.ptr as *const _, self.raw_len()) } } + /// Returns the contents of the string as a string slice. #[inline] pub fn as_str(&self) -> &str { unsafe { str::from_utf8_unchecked(self.as_bytes()) } } + /// Converts a byte slice into a `VarLenUnicode` without checking that the string contains only + /// non-zero UTF-8 bytes. + /// + /// # Safety + /// + /// The bytes must be valid UTF-8 and non-zero. #[inline] pub unsafe fn from_str_unchecked>(s: S) -> Self { Self::from_bytes(s.borrow().as_bytes()) @@ -403,6 +435,7 @@ unsafe impl Sync for VarLenUnicode {} // ================================================================================ +/// A fixed-length ASCII string. #[repr(C)] #[derive(Copy, Clone)] pub struct FixedAscii { @@ -410,6 +443,7 @@ pub struct FixedAscii { } impl FixedAscii { + /// Creates a new empty `FixedAscii` string. #[inline] pub fn new() -> Self { unsafe { Self { buf: mem::zeroed() } } @@ -428,41 +462,59 @@ impl FixedAscii { unsafe { slice::from_raw_parts(self.buf.as_ptr(), N) } } + /// Returns the string's capacity in bytes. #[inline] pub const fn capacity() -> usize { N } + /// Returns the length of `self`. #[inline] pub fn len(&self) -> usize { self.as_raw_slice().iter().rev().skip_while(|&c| *c == 0).count() } + /// Returns `true` if `self` has a length of zero bytes. #[inline] pub fn is_empty(&self) -> bool { self.as_raw_slice().iter().all(|&c| c == 0) } + /// Returns a raw pointer to the string's buffer. #[inline] pub fn as_ptr(&self) -> *const u8 { self.buf.as_ptr() } + /// Returns the contents of the string as a byte slice. #[inline] pub fn as_bytes(&self) -> &[u8] { &self.as_raw_slice()[..self.len()] } + /// Returns the contents of the string as a string slice. #[inline] pub fn as_str(&self) -> &str { unsafe { str::from_utf8_unchecked(self.as_bytes()) } } + /// Converts a byte slice into a `FixedAscii` without checking that the string is valid ASCII, + /// and truncating at the type's capacity. + /// + /// # Safety + /// + /// The bytes must be valid ASCII. #[inline] pub unsafe fn from_ascii_unchecked>(bytes: &B) -> Self { Self::from_bytes(bytes.as_ref()) } + /// Converts a byte slice into a `FixedAscii`. + /// + /// # Errors + /// + /// Returns `Err` if the bytes are not valid ASCII or if the slice length is greater than the + /// type's capacity. pub fn from_ascii>(bytes: &B) -> Result { let bytes = bytes.as_ref(); if bytes.len() > N { @@ -497,6 +549,7 @@ impl AsAsciiStr for FixedAscii { // ================================================================================ +/// A fixed-length UTF-8 string. #[repr(C)] #[derive(Copy, Clone)] pub struct FixedUnicode { @@ -504,6 +557,7 @@ pub struct FixedUnicode { } impl FixedUnicode { + /// Creates a new empty `FixedUnicode` string. #[inline] pub fn new() -> Self { unsafe { Self { buf: mem::zeroed() } } @@ -527,36 +581,48 @@ impl FixedUnicode { self.as_raw_slice().iter().rev().skip_while(|&c| *c == 0).count() } + /// Returns the string's capacity in bytes. #[inline] pub const fn capacity() -> usize { N } + /// Returns the length of `self`. #[inline] pub fn len(&self) -> usize { self.as_str().len() } + /// Returns `true` if `self` has a length of zero bytes. #[inline] pub fn is_empty(&self) -> bool { self.raw_len() == 0 } + /// Returns a raw pointer to the string's buffer. #[inline] pub fn as_ptr(&self) -> *const u8 { self.buf.as_ptr() } + /// Returns the contents of the string as a byte slice. #[inline] pub fn as_bytes(&self) -> &[u8] { &self.as_raw_slice()[..self.raw_len()] } + /// Returns the contents of the string as a string slice. #[inline] pub fn as_str(&self) -> &str { unsafe { str::from_utf8_unchecked(self.as_bytes()) } } + /// Converts a byte slice into a `FixedUnicode` without checking that the string is valid UTF-8, + /// and truncating at the type's capacity. + /// + /// # Safety + /// + /// The bytes must be valid UTF-8. #[inline] pub unsafe fn from_str_unchecked>(s: S) -> Self { Self::from_bytes(s.borrow().as_bytes()) diff --git a/hdf5/src/class.rs b/hdf5/src/class.rs index 66a2c5314..2fcabdf07 100644 --- a/hdf5/src/class.rs +++ b/hdf5/src/class.rs @@ -81,6 +81,20 @@ pub trait ObjectClass: Sized { } } +/// Takes ownership of an object via its identifier. +/// +/// # Errors +/// +/// Returns an error if `id` does not refer to an object of type `T`. +/// +/// # Safety +/// +/// This should only be called with an identifier obtained from an object constructor in the HDF5 C +/// library. The reference count of a newly created object is 1, so this function creates a handle +/// for the object without incrementing its reference count. +/// +/// This function is unsafe because improper use may lead to the object being closed before all its +/// handles are dropped. pub unsafe fn from_id(id: hid_t) -> Result { T::from_id(id) } diff --git a/hdf5/src/error.rs b/hdf5/src/error.rs index e350a7294..3f100dd51 100644 --- a/hdf5/src/error.rs +++ b/hdf5/src/error.rs @@ -35,6 +35,7 @@ pub fn silence_errors(silence: bool) { h5lock!(silence_errors_no_sync(silence)); } +/// A stack of error records from an HDF5 library call. #[repr(transparent)] #[derive(Clone)] pub struct ErrorStack(Handle); @@ -107,6 +108,7 @@ impl ErrorStack { } } +/// An error record for an HDF5 library call. #[derive(Clone, Debug)] pub struct ErrorFrame { desc: String, @@ -127,19 +129,24 @@ impl ErrorFrame { } } + /// Returns the error description. pub fn desc(&self) -> &str { self.desc.as_ref() } + /// Returns a message with the error description and the relevant function name. pub fn description(&self) -> &str { self.description.as_ref() } + /// Returns a message with the error description and the relevant function name, file name, + /// and line number. pub fn detail(&self) -> Option { Some(format!("Error in {}(): {} [{}: {}]", self.func, self.desc, self.major, self.minor)) } } +/// A converted [`ErrorStack`] with methods to access [`ErrorFrame`] data. #[derive(Clone, Debug)] pub struct ExpandedErrorStack { frames: Vec, @@ -178,10 +185,12 @@ impl ExpandedErrorStack { } } + /// Returns the top [`ErrorFrame`] of the stack, or `None` if it is empty. pub fn top(&self) -> Option<&ErrorFrame> { self.first() } + /// Returns the description of the error on top of the stack. pub fn description(&self) -> &str { match self.description { None => "unknown library error", @@ -189,6 +198,7 @@ impl ExpandedErrorStack { } } + /// Returns a detailed message for the error on top of the stack, or `None` if it is empty. pub fn detail(&self) -> Option { self.top().and_then(ErrorFrame::detail) } diff --git a/hdf5/src/hl/attribute.rs b/hdf5/src/hl/attribute.rs index c4fdf4e00..bd076c0fd 100644 --- a/hdf5/src/hl/attribute.rs +++ b/hdf5/src/hl/attribute.rs @@ -97,18 +97,22 @@ pub struct AttributeBuilder { } impl AttributeBuilder { + /// Creates a builder for a new attribute on a named object. pub fn new(parent: &Location) -> Self { Self { builder: AttributeBuilderInner::new(parent) } } + /// Sets the attribute's type without initializing its data. pub fn empty(self) -> AttributeBuilderEmpty { self.empty_as(&T::type_descriptor()) } + /// Sets the attribute's type from a type descriptor without initializing its data. pub fn empty_as(self, type_desc: &TypeDescriptor) -> AttributeBuilderEmpty { AttributeBuilderEmpty { builder: self.builder, type_desc: type_desc.clone() } } + /// Sets the data to store in the attribute. pub fn with_data<'d, A, T, D>(self, data: A) -> AttributeBuilderData<'d, T, D> where A: Into>, @@ -118,6 +122,7 @@ impl AttributeBuilder { self.with_data_as::(data, &T::type_descriptor()) } + /// Sets the data to store in the attribute and sets its element type with a type descriptor. pub fn with_data_as<'d, A, T, D>( self, data: A, type_desc: &TypeDescriptor, ) -> AttributeBuilderData<'d, T, D> @@ -150,6 +155,7 @@ pub struct AttributeBuilderEmpty { } impl AttributeBuilderEmpty { + /// Sets the shape of the attribute's data. pub fn shape>(self, extents: S) -> AttributeBuilderEmptyShape { AttributeBuilderEmptyShape { builder: self.builder, @@ -157,6 +163,8 @@ impl AttributeBuilderEmpty { extents: extents.into(), } } + + /// Creates the attribute. pub fn create<'n, T: Into<&'n str>>(self, name: T) -> Result { self.shape(()).create(name) } @@ -178,6 +186,7 @@ pub struct AttributeBuilderEmptyShape { } impl AttributeBuilderEmptyShape { + /// Creates the attribute. pub fn create<'n, T: Into<&'n str>>(&self, name: T) -> Result { h5lock!(self.builder.create(&self.type_desc, name.into(), &self.extents)) } @@ -216,6 +225,7 @@ where self } + /// Creates the attribute. pub fn create<'n, N: Into<&'n str>>(&self, name: N) -> Result { ensure!( self.data.is_standard_layout(), diff --git a/hdf5/src/hl/container.rs b/hdf5/src/hl/container.rs index 721159e4f..9b68d55e6 100644 --- a/hdf5/src/hl/container.rs +++ b/hdf5/src/hl/container.rs @@ -12,6 +12,7 @@ use hdf5_sys::h5p::H5Pcreate; use crate::internal_prelude::*; +/// A type for reading data from a [`Container`]. #[derive(Debug)] pub struct Reader<'a> { obj: &'a Container, @@ -188,6 +189,7 @@ impl<'a> Reader<'a> { } } +/// A type for writing data into a [`Container`]. #[derive(Debug)] pub struct Writer<'a> { obj: &'a Container, @@ -347,6 +349,7 @@ impl<'a> Writer<'a> { } } +/// A reader for a 1-dimensional dataset of bytes. #[derive(Debug, Clone)] pub struct ByteReader { obj: Container, @@ -357,6 +360,14 @@ pub struct ByteReader { } impl ByteReader { + /// Creates a new `ByteReader` for the given [`Container`]. + /// + /// # Panics + /// Panics if `obj` is not 1-dimensional. + /// + /// # Errors + /// + /// Returns an error if `obj` does not contain bytes or if the underlying library calls fail. pub fn new(obj: &Container) -> Result { ensure!(!obj.is_attr(), "ByteReader cannot be used on attribute datasets"); @@ -382,6 +393,7 @@ impl ByteReader { self.dataset_len().saturating_sub(self.pos as usize) } + /// Returns `true` if the reader has no more bytes to read. pub fn is_empty(&self) -> bool { self.pos >= self.dataset_len() as u64 } diff --git a/hdf5/src/hl/dataset.rs b/hdf5/src/hl/dataset.rs index 24f3b160b..5aef691bd 100644 --- a/hdf5/src/hl/dataset.rs +++ b/hdf5/src/hl/dataset.rs @@ -1,3 +1,5 @@ +//! Interfaces for `Dataset` objects. + use std::fmt::{self, Debug}; use std::ops::Deref; @@ -329,11 +331,15 @@ where } } +/// Options for how to chunk data. #[derive(Debug, Clone, PartialEq, Eq)] pub enum Chunk { - Exact(Vec), // exact chunk shape - MinKB(usize), // minimum chunk shape in KB - None, // leave it unchunked + /// Exact chunk shape. + Exact(Vec), + /// Minimum chunk shape in kilobytes. + MinKB(usize), + /// Leave the data unchunked. + None, } impl Default for Chunk { diff --git a/hdf5/src/hl/dataspace.rs b/hdf5/src/hl/dataspace.rs index 58c9af2f6..d4d5f1f10 100644 --- a/hdf5/src/hl/dataspace.rs +++ b/hdf5/src/hl/dataspace.rs @@ -78,46 +78,62 @@ unsafe fn get_simple_extents(space_id: hid_t) -> Result { } impl Dataspace { + /// Tries to construct a `Dataspace` from the given extents. + /// + /// # Errors + /// + /// Returns an error if the extents are invalid. pub fn try_new>(extents: T) -> Result { Self::from_extents(&extents.into()) } + /// Creates a new `Dataspace` that is a copy of the current one. pub fn copy(&self) -> Self { Self::from_id(h5lock!(H5Scopy(self.id()))).unwrap_or_else(|_| Self::invalid()) } + /// Returns the number of dimensions in the space. pub fn ndim(&self) -> usize { h5call!(H5Sget_simple_extent_ndims(self.id())).unwrap_or(0) as _ } + /// Returns a vector containing the current size of each dimension. pub fn shape(&self) -> Vec { h5lock!(get_shape(self.id())).unwrap_or_default() } + /// Returns a vector containing the current maximum size (if set) of each dimension. pub fn maxdims(&self) -> Vec> { self.extents().unwrap_or(Extents::Null).maxdims() } + /// Returns `true` if any dimension is resizable. pub fn is_resizable(&self) -> bool { self.maxdims().iter().any(Option::is_none) } + /// Returns `true` if the extent type is null. pub fn is_null(&self) -> bool { h5lock!(H5Sget_simple_extent_type(self.id())) == H5S_class_t::H5S_NULL } + /// Returns `true` if the extent type is scalar. pub fn is_scalar(&self) -> bool { h5lock!(H5Sget_simple_extent_type(self.id())) == H5S_class_t::H5S_SCALAR } + /// Returns `true` if the extent type is simple. pub fn is_simple(&self) -> bool { h5lock!(H5Sget_simple_extent_type(self.id())) == H5S_class_t::H5S_SIMPLE } + /// Returns `true` if the selection for the space is within the dataspace extent if the current + /// offset is used. pub fn is_valid(&self) -> bool { h5lock!(H5Sselect_valid(self.id())) > 0 } + /// Returns the number of elements in the space. pub fn size(&self) -> usize { match h5lock!(H5Sget_simple_extent_type(self.id())) { H5S_class_t::H5S_SIMPLE => { @@ -128,6 +144,11 @@ impl Dataspace { } } + /// Encodes the dataspace description into a new byte vector. + /// + /// # Errors + /// + /// Returns an error if the underlying encoding call fails. #[allow(deprecated)] pub fn encode(&self) -> Result> { cfg_if::cfg_if! { @@ -152,6 +173,11 @@ impl Dataspace { } } + /// Tries to decode a dataspace from a byte buffer. + /// + /// # Errors + /// + /// Returns an error if the buffer does not decode to a valid dataspace. pub fn decode(buf: T) -> Result where T: AsRef<[u8]>, @@ -174,6 +200,11 @@ impl Dataspace { })) } + /// Returns the extents of the dataspace. + /// + /// # Errors + /// + /// Returns an error if the extents type is unsupported. #[allow(clippy::match_wildcard_for_single_variants)] pub fn extents(&self) -> Result { h5lock!(match H5Sget_simple_extent_type(self.id()) { @@ -184,6 +215,7 @@ impl Dataspace { }) } + /// Returns the number of elements in the dataspace selection. pub fn selection_size(&self) -> usize { h5call!(H5Sget_select_npoints(self.id())).ok().map_or(0, |x| x as _) } @@ -199,6 +231,7 @@ impl Dataspace { }) } + /// Selects part of the dataspace and returns a new dataspace selection object. pub fn select>(&self, selection: S) -> Result { let raw_sel = selection.into().into_raw(self.shape())?; self.select_raw(raw_sel) @@ -209,6 +242,7 @@ impl Dataspace { sync(|| unsafe { RawSelection::extract_from_dataspace(self.id()) }) } + /// Returns a description of the current dataspace selection. pub fn get_selection(&self) -> Result { let raw_sel = self.get_raw_selection()?; Selection::from_raw(raw_sel) diff --git a/hdf5/src/hl/datatype.rs b/hdf5/src/hl/datatype.rs index c14b1e993..ce4ebb29a 100644 --- a/hdf5/src/hl/datatype.rs +++ b/hdf5/src/hl/datatype.rs @@ -104,10 +104,14 @@ impl PartialEq for Datatype { } } +/// A level of possible conversion between two types. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] pub enum Conversion { + /// No conversion. NoOp = 1, // TODO: rename to "None"? + /// Functions employing compiler casting. Hard, + /// Functions not employing compiler casting. Soft, } @@ -139,12 +143,18 @@ impl Default for Conversion { } } +/// The byte order of a datatype. #[derive(Copy, Debug, Clone, PartialEq, Eq)] pub enum ByteOrder { + /// Little endian. LittleEndian, + /// Big endian. BigEndian, + /// VAX mixed endian. Vax, + /// Compound type with mixed member orders. Mixed, + /// No particular order. None, } @@ -185,6 +195,7 @@ impl Datatype { h5lock!(H5Tget_order(self.id())).into() } + /// Returns the conversion function level from `self` to `dst`, if one exists. pub fn conv_path(&self, dst: D) -> Option where D: Borrow, @@ -206,14 +217,17 @@ impl Datatype { }) } + /// Returns the conversion function level from `self` to a concrete type, if one exists. pub fn conv_to(&self) -> Option { Self::from_type::().ok().and_then(|dtype| self.conv_path(dtype)) } + /// Returns the conversion function level from a concrete type to `self`, if one exists. pub fn conv_from(&self) -> Option { Self::from_type::().ok().and_then(|dtype| dtype.conv_path(self)) } + /// Returns `true` if `self` represents a concrete type. pub fn is(&self) -> bool { Self::from_type::().ok().map_or(false, |dtype| &dtype == self) } @@ -228,6 +242,7 @@ impl Datatype { } } + /// Returns a type descriptor for the datatype. pub fn to_descriptor(&self) -> Result { use hdf5_types::TypeDescriptor as TD; @@ -321,10 +336,12 @@ impl Datatype { }) } + /// Creates a datatype from a concrete type. pub fn from_type() -> Result { Self::from_descriptor(&::type_descriptor()) } + /// Creates a datatype from a type descriptor. pub fn from_descriptor(desc: &TypeDescriptor) -> Result { use hdf5_types::TypeDescriptor as TD; diff --git a/hdf5/src/hl/extents.rs b/hdf5/src/hl/extents.rs index 9eeba3d22..6b3128321 100644 --- a/hdf5/src/hl/extents.rs +++ b/hdf5/src/hl/extents.rs @@ -68,6 +68,7 @@ impl + Clone> From<&T> for Extent { } impl Extent { + /// Creates a new extent with a current size and (optional) maximum size. pub fn new(dim: Ix, max: Option) -> Self { Self { dim, max } } @@ -82,18 +83,22 @@ impl Extent { Self { dim, max: None } } + /// Returns `true` if the current dimension size is at least the maximum size, if set. pub fn is_fixed(&self) -> bool { self.max.map_or(false, |max| self.dim >= max) } + /// Returns `true` if the dimension is resizable. pub fn is_resizable(&self) -> bool { self.max.is_none() } + /// Returns `true` if the dimension has unlimited maximum size. pub fn is_unlimited(&self) -> bool { self.is_resizable() } + /// Returns `true` if the current dimension size is not greater than the maximum. pub fn is_valid(&self) -> bool { self.max.unwrap_or(self.dim) >= self.dim } @@ -113,10 +118,12 @@ pub struct SimpleExtents { } impl SimpleExtents { + /// Creates extents from a vector of `Extent` values. pub fn from_vec(extents: Vec) -> Self { Self { inner: extents } } + /// Creates extents from an iterable of extent-like values. pub fn new(extents: T) -> Self where T: IntoIterator, @@ -125,6 +132,8 @@ impl SimpleExtents { Self::from_vec(extents.into_iter().map(Into::into).collect()) } + /// Creates extents from an iterable of dimension sizes and sets their + /// maximum sizes equal to their current sizes. pub fn fixed(extents: T) -> Self where T: IntoIterator, @@ -142,38 +151,48 @@ impl SimpleExtents { Self::from_vec(extents.into_iter().map(|x| Extent::resizable(*x.borrow())).collect()) } + /// Returns the number of dimensions. pub fn ndim(&self) -> usize { self.inner.len() } + /// Returns a vector containing the current size of each dimension. pub fn dims(&self) -> Vec { self.inner.iter().map(|e| e.dim).collect() } + /// Returns a vector containing the maximum size (if set) of each dimension. pub fn maxdims(&self) -> Vec> { self.inner.iter().map(|e| e.max).collect() } + /// Returns the number of elements that the space can hold. pub fn size(&self) -> usize { self.inner.iter().fold(1, |acc, x| acc * x.dim) } + /// Returns `true` if there is at least one dimension and they all have fixed maximum size. pub fn is_fixed(&self) -> bool { !self.inner.is_empty() && self.inner.iter().map(Extent::is_fixed).all(identity) } + /// Returns `true` if there is at least one resizable dimension. pub fn is_resizable(&self) -> bool { !self.inner.is_empty() && self.inner.iter().map(Extent::is_unlimited).any(identity) } + /// Returns `true` if there is at least one dimension of unlimited size. pub fn is_unlimited(&self) -> bool { self.inner.iter().map(Extent::is_unlimited).any(identity) } + /// Returns `true` if the extents are valid and the dimensionality does not exceed + /// HDF5's max rank. pub fn is_valid(&self) -> bool { self.inner.iter().map(Extent::is_valid).all(identity) && self.ndim() <= H5S_MAX_RANK as _ } + /// Returns an iterator over the `Extent` values for each dimension. pub fn iter( &self, ) -> impl ExactSizeIterator + DoubleEndedIterator { @@ -290,6 +309,7 @@ impl_from!(RangeFrom); impl_from!(RangeInclusive); impl_from!(Extent); +/// The dimensionality of a dataspace. #[derive(Clone, Debug, PartialEq, Eq)] pub enum Extents { /// A null dataspace contains no data elements. @@ -324,6 +344,7 @@ pub enum Extents { } impl Extents { + /// Creates extents from a value. pub fn new>(extents: T) -> Self { extents.into() } @@ -389,18 +410,22 @@ impl Extents { } } + /// Returns `true` if the extents are valid. pub fn is_valid(&self) -> bool { self.as_simple().map_or(true, SimpleExtents::is_valid) } + /// Returns `true` if the extents have unlimited maximum size. pub fn is_unlimited(&self) -> bool { self.as_simple().map_or(true, SimpleExtents::is_unlimited) } + /// Returns `true` if there is at least one resizable dimension. pub fn is_resizable(&self) -> bool { self.as_simple().map_or(true, SimpleExtents::is_resizable) } + /// Returns a version of `self` that is resizable along all dimensions. pub fn resizable(self) -> Self { match self { Self::Simple(extents) => SimpleExtents::resizable(extents.dims()).into(), @@ -408,12 +433,15 @@ impl Extents { } } + /// Returns an iterator of `Extent` values, one for each dimension. pub fn iter( &self, ) -> impl ExactSizeIterator + DoubleEndedIterator { ExtentsIter { inner: self.as_simple().map(SimpleExtents::iter) } } + /// Returns a slice containing an `Extent` for each dimension, or `None` if the extents are + /// null or scalar. pub fn slice(&self) -> Option<&[Extent]> { if let Self::Simple(x) = self { Some(x) @@ -423,6 +451,7 @@ impl Extents { } } +/// An iterator over the dimensions of an `Extents`. pub struct ExtentsIter { inner: Option, } diff --git a/hdf5/src/hl/filters.rs b/hdf5/src/hl/filters.rs index a0056aa6e..a9e6b02aa 100644 --- a/hdf5/src/hl/filters.rs +++ b/hdf5/src/hl/filters.rs @@ -14,6 +14,7 @@ use hdf5_sys::h5z::{ H5_SZIP_NN_OPTION_MASK, }; +/// A filter identifier. pub use hdf5_sys::h5z::H5Z_filter_t; use crate::internal_prelude::*; @@ -23,20 +24,27 @@ mod blosc; #[cfg(feature = "lzf")] mod lzf; +/// Coding methods for Szip compression. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum SZip { + /// Entropy coding method. Entropy, + /// Nearest-neighbor coding method. NearestNeighbor, } +/// Scaling methods for scale-offset compression. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum ScaleOffset { + /// Integer scaling with some MinBits value. Integer(u16), + /// Floating-point D-scaling with some decimal scale factor. FloatDScale(u8), } #[cfg(feature = "blosc")] mod blosc_impl { + /// Available compressors for Blosc compression. #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[cfg(feature = "blosc")] #[non_exhaustive] @@ -103,25 +111,39 @@ mod blosc_impl { #[cfg(feature = "blosc")] pub use blosc_impl::*; +/// An HDF5 filter configuration. #[derive(Clone, Debug, PartialEq, Eq)] pub enum Filter { + /// Gzip compression (deflation) with some compression level. Deflate(u8), + /// Shuffle algorithm. Shuffle, + /// Fletcher32 checksum. Fletcher32, + /// SZIP compression with some coding method and pixels per block. SZip(SZip, u8), + /// N-bit compression. NBit, + /// Scale-offset compression with some scaling mode. ScaleOffset(ScaleOffset), + /// LZF compression. #[cfg(feature = "lzf")] LZF, + /// Blosc compression with some compressor, compression level, and shuffle mode. #[cfg(feature = "blosc")] Blosc(Blosc, u8, BloscShuffle), + /// A user-defined filter with some parameters. User(H5Z_filter_t, Vec), } +/// Information about whether a filter is available and enabled for encoding/decoding. #[derive(Default, Clone, Copy, Debug, PartialEq, Eq)] pub struct FilterInfo { + /// Whether the filter is available. pub is_available: bool, + /// Whether the filter is configured to encode data. pub encode_enabled: bool, + /// Whether the filter is configured to decode data. pub decode_enabled: bool, } @@ -165,6 +187,7 @@ pub fn blosc_available() -> bool { } impl Filter { + /// Returns the filter's identifier. pub fn id(&self) -> H5Z_filter_t { match self { Self::Deflate(_) => H5Z_FILTER_DEFLATE, @@ -181,6 +204,7 @@ impl Filter { } } + /// Returns metadata for the filter with the given identifier. pub fn get_info(filter_id: H5Z_filter_t) -> FilterInfo { if !h5call!(H5Zfilter_avail(filter_id)).map(|x| x > 0).unwrap_or_default() { return FilterInfo::default(); @@ -194,47 +218,59 @@ impl Filter { } } + /// Returns `true` if the filter is available. pub fn is_available(&self) -> bool { Self::get_info(self.id()).is_available } + /// Returns `true` if the filter is configured to encode data. pub fn encode_enabled(&self) -> bool { Self::get_info(self.id()).encode_enabled } + /// Returns `true` if the filter is configured to decode data. pub fn decode_enabled(&self) -> bool { Self::get_info(self.id()).decode_enabled } + /// Creates a deflation filter configuration with some compression level. pub fn deflate(level: u8) -> Self { Self::Deflate(level) } + /// Returns the shuffle algorithm filter configuration. pub fn shuffle() -> Self { Self::Shuffle } + /// Returns the Fletcher32 checksum filter configuration. pub fn fletcher32() -> Self { Self::Fletcher32 } + /// Creates an Szip filter configuration with some coding method and pixels per block. pub fn szip(coding: SZip, px_per_block: u8) -> Self { Self::SZip(coding, px_per_block) } + /// Returns the N-bit compression filter configuration. pub fn nbit() -> Self { Self::NBit } + /// Creates a scale-offset compression filter configuration with some scaling mode. pub fn scale_offset(mode: ScaleOffset) -> Self { Self::ScaleOffset(mode) } + /// Returns the LZF compression filter. #[cfg(feature = "lzf")] pub fn lzf() -> Self { Self::LZF } + /// Creates a Blosc compression filter configuration with some compressor, + /// compression level, and shuffle mode. #[cfg(feature = "blosc")] pub fn blosc(complib: Blosc, clevel: u8, shuffle: T) -> Self where @@ -243,6 +279,8 @@ impl Filter { Self::Blosc(complib, clevel, shuffle.into()) } + /// Creates a Blosc LZ compression filter configuration with some compression level and + /// shuffle mode. #[cfg(feature = "blosc")] pub fn blosc_blosclz(clevel: u8, shuffle: T) -> Self where @@ -251,6 +289,8 @@ impl Filter { Self::blosc(Blosc::BloscLZ, clevel, shuffle) } + /// Creates a Blosc LZ4 compression filter configuration with some compression level and + /// shuffle mode. #[cfg(feature = "blosc-lz4")] pub fn blosc_lz4(clevel: u8, shuffle: T) -> Self where @@ -259,6 +299,8 @@ impl Filter { Self::blosc(Blosc::LZ4, clevel, shuffle) } + /// Creates a Blosc LZ4HC compression filter configuration with some compression level and + /// shuffle mode. #[cfg(feature = "blosc-lz4")] pub fn blosc_lz4hc(clevel: u8, shuffle: T) -> Self where @@ -267,6 +309,8 @@ impl Filter { Self::blosc(Blosc::LZ4HC, clevel, shuffle) } + /// Creates a Blosc Snappy compression filter configuration with some compression level and + /// shuffle mode. #[cfg(feature = "blosc-snappy")] pub fn blosc_snappy(clevel: u8, shuffle: T) -> Self where @@ -275,6 +319,8 @@ impl Filter { Self::blosc(Blosc::Snappy, clevel, shuffle) } + /// Creates a Blosc Zlib compression filter configuration with some compression level and + /// shuffle mode. #[cfg(feature = "blosc-zlib")] pub fn blosc_zlib(clevel: u8, shuffle: T) -> Self where @@ -283,6 +329,8 @@ impl Filter { Self::blosc(Blosc::ZLib, clevel, shuffle) } + /// Creates a Blosc Zstd compression filter configuration with some compression level and + /// shuffle mode. #[cfg(feature = "blosc-zstd")] pub fn blosc_zstd(clevel: u8, shuffle: T) -> Self where @@ -291,6 +339,7 @@ impl Filter { Self::blosc(Blosc::ZStd, clevel, shuffle) } + /// Creates a user-defined filter configuration with some filter identifier and parameters. pub fn user(id: H5Z_filter_t, cdata: &[c_uint]) -> Self { Self::User(id, cdata.to_vec()) } @@ -397,6 +446,12 @@ impl Filter { Ok(Self::blosc(complib, clevel, shuffle)) } + /// Tries to create a filter configuration from a filter identifier and parameters. + /// + /// # Errors + /// + /// Returns an error if the identifier is invalid or the parameters are invalid + /// for the specified filter. pub fn from_raw(filter_id: H5Z_filter_t, cdata: &[c_uint]) -> Result { ensure!(filter_id > 0, "invalid filter id: {}", filter_id); match filter_id { diff --git a/hdf5/src/hl/group.rs b/hdf5/src/hl/group.rs index ea85c361f..32e57ec43 100644 --- a/hdf5/src/hl/group.rs +++ b/hdf5/src/hl/group.rs @@ -261,10 +261,14 @@ impl From for H5_iter_order_t { } } +/// The type of an object link. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum LinkType { + /// A hard link to an object within a single file. Hard, + /// A symbolic link to an object within a single file. Soft, + /// A symbolic link to an object in a different file. External, } @@ -278,6 +282,7 @@ impl From for LinkType { } } +/// Metadata describing an object link. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct LinkInfo { pub link_type: LinkType, diff --git a/hdf5/src/hl/location.rs b/hdf5/src/hl/location.rs index fe0a8560a..9f9261a52 100644 --- a/hdf5/src/hl/location.rs +++ b/hdf5/src/hl/location.rs @@ -109,19 +109,27 @@ impl Location { h5call!(H5Oset_comment(self.id(), ptr::null_mut())).and(Ok(())) } + /// Create a builder for a new attribute of known type. pub fn new_attr(&self) -> AttributeBuilderEmpty { AttributeBuilder::new(self).empty::() } + /// Create a builder for a new attribute. pub fn new_attr_builder(&self) -> AttributeBuilder { AttributeBuilder::new(self) } + /// Create a new named attribute on the object. pub fn attr(&self, name: &str) -> Result { let name = to_cstring(name)?; Attribute::from_id(h5try!(H5Aopen(self.id(), name.as_ptr(), H5P_DEFAULT))) } + /// Return the names of all attributes on the object. + /// + /// # Errors + /// + /// Returns an error if an underlying library call fails. pub fn attr_names(&self) -> Result> { Attribute::attr_names(self) } @@ -132,24 +140,37 @@ impl Location { Ok(()) } + /// Returns the object's metadata. pub fn loc_info(&self) -> Result { H5O_get_info(self.id(), true) } + /// Returns the object's type. pub fn loc_type(&self) -> Result { Ok(H5O_get_info(self.id(), false)?.loc_type) } + /// Returns the metadata of another object with name relative to `self`. + /// + /// # Errors + /// + /// Returns an error if the name is invalid. pub fn loc_info_by_name(&self, name: &str) -> Result { let name = to_cstring(name)?; H5O_get_info_by_name(self.id(), name.as_ptr(), true) } + /// Returns the type of another object with name relative to `self`. + /// + /// # Errors + /// + /// Returns an error if the name is invalid. pub fn loc_type_by_name(&self, name: &str) -> Result { let name = to_cstring(name)?; Ok(H5O_get_info_by_name(self.id(), name.as_ptr(), false)?.loc_type) } + /// Opens an object using its location token. pub fn open_by_token(&self, token: LocationToken) -> Result { H5O_open_by_token(self.id(), token) } @@ -212,12 +233,14 @@ impl Location { } } +/// A token containing the address or identifier of a [`Location`]. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct LocationToken( #[cfg(not(feature = "1.12.0"))] haddr_t, #[cfg(feature = "1.12.0")] H5O_token_t, ); +/// The type of an object in a [`Location`]. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum LocationType { Group, diff --git a/hdf5/src/hl/object.rs b/hdf5/src/hl/object.rs index cabe0e970..f6d8724c5 100644 --- a/hdf5/src/hl/object.rs +++ b/hdf5/src/hl/object.rs @@ -29,6 +29,7 @@ impl Debug for Object { } impl Object { + /// Returns the object's identifier. pub fn id(&self) -> hid_t { self.0.id() } @@ -57,7 +58,7 @@ impl Object { macro_rules! impl_downcast { ($func:ident, $tp:ty) => { impl Object { - #[doc = "Downcast the object into $tp if possible."] + #[doc = concat!("Downcast the object into `", stringify!($tp), "` if possible.")] pub fn $func(&self) -> Result<$tp> { self.clone().cast() } diff --git a/hdf5/src/hl/plist.rs b/hdf5/src/hl/plist.rs index 9b4a56d6d..bdb4b013e 100644 --- a/hdf5/src/hl/plist.rs +++ b/hdf5/src/hl/plist.rs @@ -211,6 +211,7 @@ impl PropertyList { }) } + /// Returns `true` if the property list is a member of `class`. pub fn is_class(&self, class: PropertyListClass) -> bool { use crate::globals::*; h5lock!({ diff --git a/hdf5/src/hl/plist/dataset_access.rs b/hdf5/src/hl/plist/dataset_access.rs index 5a123aea2..4670c20d4 100644 --- a/hdf5/src/hl/plist/dataset_access.rs +++ b/hdf5/src/hl/plist/dataset_access.rs @@ -91,10 +91,13 @@ impl Clone for DatasetAccess { } } +/// Options for including or excluding missing mapped elements in a virtual dataset view. #[cfg(feature = "1.10.0")] #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum VirtualView { + /// Include all data before the first missing mapped data. FirstMissing, + /// Include all available mapped data, filling missing data with a fill value. LastAvailable, } @@ -165,29 +168,35 @@ impl DatasetAccessBuilder { Ok(builder) } + /// Sets the [`ChunkCache`] options. pub fn chunk_cache(&mut self, nslots: usize, nbytes: usize, w0: f64) -> &mut Self { self.chunk_cache = Some(ChunkCache { nslots, nbytes, w0 }); self } + /// Sets the external dataset storage file prefix. #[cfg(feature = "1.8.17")] pub fn efile_prefix(&mut self, prefix: &str) -> &mut Self { self.efile_prefix = Some(prefix.into()); self } + /// Sets the [`VirtualView`] options. #[cfg(feature = "1.10.0")] pub fn virtual_view(&mut self, view: VirtualView) -> &mut Self { self.virtual_view = Some(view); self } + /// Sets the maximum number of files/datasets allowed to be missing when determining the extent + /// of an unlimited virtual dataset with printf-style mappings. #[cfg(feature = "1.10.0")] pub fn virtual_printf_gap(&mut self, gap_size: usize) -> &mut Self { self.virtual_printf_gap = Some(gap_size); self } + /// Sets metadata I/O mode for read options to collective or independent. #[cfg(all(feature = "1.10.0", feature = "have-parallel"))] pub fn all_coll_metadata_ops(&mut self, is_collective: bool) -> &mut Self { self.all_coll_metadata_ops = Some(is_collective); @@ -223,10 +232,12 @@ impl DatasetAccessBuilder { Ok(()) } + /// Copies the builder settings into a dataset access property list. pub fn apply(&self, plist: &mut DatasetAccess) -> Result<()> { h5lock!(self.populate_plist(plist.id())) } + /// Constructs a new dataset access property list. pub fn finish(&self) -> Result { h5lock!({ let mut plist = DatasetAccess::try_new()?; @@ -237,14 +248,17 @@ impl DatasetAccessBuilder { /// Dataset access property list. impl DatasetAccess { + /// Creates a new dataset access property list. pub fn try_new() -> Result { Self::from_id(h5try!(H5Pcreate(*H5P_DATASET_ACCESS))) } + /// Creates a copy of the property list. pub fn copy(&self) -> Self { unsafe { self.deref().copy().cast_unchecked() } } + /// Creates a new dataset access property list builder. pub fn build() -> DatasetAccessBuilder { DatasetAccessBuilder::new() } @@ -260,6 +274,7 @@ impl DatasetAccess { ) } + /// Returns the raw data chunk cache parameters. pub fn chunk_cache(&self) -> ChunkCache { self.get_chunk_cache().unwrap_or_else(|_| ChunkCache::default()) } @@ -270,6 +285,7 @@ impl DatasetAccess { h5lock!(get_h5_str(|m, s| H5Pget_efile_prefix(self.id(), m, s))) } + /// Returns the external dataset storage file prefix. #[cfg(feature = "1.8.17")] pub fn efile_prefix(&self) -> String { self.get_efile_prefix().ok().unwrap_or_default() @@ -281,6 +297,7 @@ impl DatasetAccess { h5get!(H5Pget_virtual_view(self.id()): H5D_vds_view_t).map(Into::into) } + /// Returns the virtual dataset view options. #[cfg(feature = "1.10.0")] pub fn virtual_view(&self) -> VirtualView { self.get_virtual_view().ok().unwrap_or_default() @@ -292,6 +309,8 @@ impl DatasetAccess { h5get!(H5Pget_virtual_printf_gap(self.id()): hsize_t).map(|x| x as _) } + /// Returns the maximum number of files/datasets allowed to be missing when determining the + /// extent of an unlimited virtual dataset with printf-style mappings. #[cfg(feature = "1.10.0")] pub fn virtual_printf_gap(&self) -> usize { self.get_virtual_printf_gap().unwrap_or(0) @@ -303,6 +322,7 @@ impl DatasetAccess { h5get!(H5Pget_all_coll_metadata_ops(self.id()): hbool_t).map(|x| x > 0) } + /// Returns `true` if metadata I/O reads are set to collective, or `false` if independent. #[cfg(all(feature = "1.10.0", feature = "have-parallel"))] pub fn all_coll_metadata_ops(&self) -> bool { self.get_all_coll_metadata_ops().unwrap_or(false) diff --git a/hdf5/src/hl/plist/dataset_create.rs b/hdf5/src/hl/plist/dataset_create.rs index 0661c87f5..0a087a1af 100644 --- a/hdf5/src/hl/plist/dataset_create.rs +++ b/hdf5/src/hl/plist/dataset_create.rs @@ -107,11 +107,16 @@ impl Clone for DatasetCreate { } } +/// Options for how to store raw data for a dataset. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Layout { + /// Raw data is stored in the file's object header. Compact, + /// Raw data is stored in a contiguous chunk in the file, outside the object header. Contiguous, + /// Raw data is stored in separate chunks in the file. Chunked, + /// Raw data is drawn from multiple datasets in different files. #[cfg(feature = "1.10.0")] Virtual, } @@ -148,8 +153,10 @@ impl From for H5D_layout_t { #[cfg(feature = "1.10.0")] bitflags! { + /// Edge chunk option flags. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct ChunkOpts: u32 { + /// Disable applying filters to partial edge chunks. const DONT_FILTER_PARTIAL_CHUNKS = H5D_CHUNK_DONT_FILTER_PARTIAL_CHUNKS; } } @@ -161,10 +168,14 @@ impl Default for ChunkOpts { } } +/// Options for when to allocate dataset storage space. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum AllocTime { + /// Allocate all space when the dataset is created. Early, + /// Allocate space incrementally as data is written to the dataset. Incr, + /// Allocate all space when data is first written to the dataset. Late, } @@ -188,10 +199,15 @@ impl From for H5D_alloc_time_t { } } +/// Options for when to write fill values to a dataset. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum FillTime { + /// Write fill values to the dataset when storage is allocated only if a user-defined fill + /// value is set. IfSet, + /// Write fill values to the dataset when storage is allocated. Alloc, + /// Never write fill values to the dataset. Never, } @@ -221,10 +237,14 @@ impl From for H5D_fill_time_t { } } +/// The status of a dataset creation property list's fill value. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum FillValue { + /// Fill value is undefined. Undefined, + /// Fill value is the library default. Default, + /// Fill value is defined by the application. UserDefined, } @@ -254,26 +274,38 @@ impl From for H5D_fill_value_t { } } +/// Properties of data stored in an external file. #[derive(Clone, Debug, PartialEq, Eq)] pub struct ExternalFile { + /// The name of the file. pub name: String, + /// The offset in bytes from the start of the file to the location where the data starts. pub offset: usize, + /// The number of bytes reserved in the file for data. pub size: usize, } +/// Properties of a mapping between virtual and source datasets. #[cfg(feature = "1.10.0")] #[derive(Clone, Debug, PartialEq, Eq)] pub struct VirtualMapping { + /// The name of the HDF5 file containing the source dataset. pub src_filename: String, + /// The path to the source dataset inside the file. pub src_dataset: String, + /// The dimensionality of the source dataset. pub src_extents: Extents, + /// The selection of the source dataset to be mapped. pub src_selection: Selection, + /// The dimensionality of the virtual dataset. pub vds_extents: Extents, + /// The selection of the virtual dataset to be mapped. pub vds_selection: Selection, } #[cfg(feature = "1.10.0")] impl VirtualMapping { + /// Constructs a `VirtualMapping` with the given parameters. pub fn new( src_filename: F, src_dataset: D, src_extents: E1, src_selection: S1, vds_extents: E2, vds_selection: S2, @@ -365,41 +397,49 @@ impl DatasetCreateBuilder { Ok(builder) } + /// Sets the dataset filters from a slice of filter specifiers. pub fn set_filters(&mut self, filters: &[Filter]) -> &mut Self { self.filters = filters.to_owned(); self } + /// Adds a deflation filter with some compression level to the dataset. pub fn deflate(&mut self, level: u8) -> &mut Self { self.filters.push(Filter::deflate(level)); self } + /// Adds a shuffle filter to the dataset. pub fn shuffle(&mut self) -> &mut Self { self.filters.push(Filter::shuffle()); self } + /// Adds a Fletcher32 checksum filter to the dataset. pub fn fletcher32(&mut self) -> &mut Self { self.filters.push(Filter::fletcher32()); self } + /// Adds an Szip compression filter with some coding method and pixels per block to the dataset. pub fn szip(&mut self, coding: SZip, px_per_block: u8) -> &mut Self { self.filters.push(Filter::szip(coding, px_per_block)); self } + /// Adds an N-bit compression filter to the dataset. pub fn nbit(&mut self) -> &mut Self { self.filters.push(Filter::nbit()); self } + /// Adds a scale-offset compression filter with some scaling mode to the dataset. pub fn scale_offset(&mut self, mode: ScaleOffset) -> &mut Self { self.filters.push(Filter::scale_offset(mode)); self } + /// Adds an LZF compression filter to the dataset. #[cfg(feature = "lzf")] pub fn lzf(&mut self) -> &mut Self { self.filters.push(Filter::lzf()); @@ -421,6 +461,7 @@ impl DatasetCreateBuilder { self } + /// Adds a Blosc filter with LZ compression to the dataset. #[cfg(feature = "blosc")] pub fn blosc_blosclz(&mut self, clevel: u8, shuffle: T) -> &mut Self where @@ -430,6 +471,7 @@ impl DatasetCreateBuilder { self } + /// Adds a Blosc filter with LZ4 compression to the dataset. #[cfg(feature = "blosc-lz4")] pub fn blosc_lz4(&mut self, clevel: u8, shuffle: T) -> &mut Self where @@ -439,6 +481,7 @@ impl DatasetCreateBuilder { self } + /// Adds a Blosc filter with LZ4HC compression to the dataset. #[cfg(feature = "blosc-lz4")] pub fn blosc_lz4hc(&mut self, clevel: u8, shuffle: T) -> &mut Self where @@ -448,6 +491,7 @@ impl DatasetCreateBuilder { self } + /// Adds a Blosc filter with Snappy compression to the dataset. #[cfg(feature = "blosc-snappy")] pub fn blosc_snappy(&mut self, clevel: u8, shuffle: T) -> &mut Self where @@ -457,6 +501,7 @@ impl DatasetCreateBuilder { self } + /// Adds a Blosc filter with Zlib compression to the dataset. #[cfg(feature = "blosc-zlib")] pub fn blosc_zlib(&mut self, clevel: u8, shuffle: T) -> &mut Self where @@ -466,6 +511,7 @@ impl DatasetCreateBuilder { self } + /// Adds a Blosc filter with Zstd compression to the dataset. #[cfg(feature = "blosc-zstd")] pub fn blosc_zstd(&mut self, clevel: u8, shuffle: T) -> &mut Self where @@ -475,21 +521,25 @@ impl DatasetCreateBuilder { self } + /// Adds a user-defined filter with the given identifier and parameters to the dataset. pub fn add_filter(&mut self, id: H5Z_filter_t, cdata: &[c_uint]) -> &mut Self { self.filters.push(Filter::user(id, cdata)); self } + /// Removes all filters from the dataset. pub fn clear_filters(&mut self) -> &mut Self { self.filters.clear(); self } + /// Sets the dataset's storage space allocation timing. pub fn alloc_time(&mut self, alloc_time: Option) -> &mut Self { self.alloc_time = Some(alloc_time); self } + /// Sets the time when fill values should be written to the dataset. pub fn fill_time(&mut self, fill_time: FillTime) -> &mut Self { self.fill_time = Some(fill_time); self @@ -499,11 +549,13 @@ impl DatasetCreateBuilder { self.fill_time.is_some() } + /// Sets the dataset's fill value. pub fn fill_value>(&mut self, fill_value: T) -> &mut Self { self.fill_value = Some(fill_value.into()); self } + /// Clears the dataset's fill value. pub fn no_fill_value(&mut self) -> &mut Self { self.fill_value = None; self @@ -521,27 +573,32 @@ impl DatasetCreateBuilder { self } + /// Clears the dataset's chunking settings. pub fn no_chunk(&mut self) -> &mut Self { self.chunk = None; self } + /// Sets the dataset's raw data layout. pub fn layout(&mut self, layout: Layout) -> &mut Self { self.layout = Some(layout); self } + /// Sets the dataset's edge chunk options. #[cfg(feature = "1.10.0")] pub fn chunk_opts(&mut self, opts: ChunkOpts) -> &mut Self { self.chunk_opts = Some(opts); self } + /// Adds an external file to the dataset. pub fn external(&mut self, name: &str, offset: usize, size: usize) -> &mut Self { self.external.push(ExternalFile { name: name.to_owned(), offset, size }); self } + /// Adds a mapping between virtual and source datasets. #[cfg(feature = "1.10.0")] pub fn virtual_map( &mut self, src_filename: F, src_dataset: D, src_extents: E1, src_selection: S1, @@ -566,16 +623,19 @@ impl DatasetCreateBuilder { self } + /// Sets whether to record time data for the dataset. pub fn obj_track_times(&mut self, track_times: bool) -> &mut Self { self.obj_track_times = Some(track_times); self } + /// Sets the dataset's attribute storage phase change thresholds. pub fn attr_phase_change(&mut self, max_compact: u32, min_dense: u32) -> &mut Self { self.attr_phase_change = Some(AttrPhaseChange { max_compact, min_dense }); self } + /// Sets whether to track and/or index the dataset's attribute creation order. pub fn attr_creation_order(&mut self, attr_creation_order: AttrCreationOrder) -> &mut Self { self.attr_creation_order = Some(attr_creation_order); self @@ -650,10 +710,12 @@ impl DatasetCreateBuilder { !self.filters.is_empty() } + /// Copies the builder settings into a dataset creation property list. pub fn apply(&self, plist: &mut DatasetCreate) -> Result<()> { h5lock!(self.populate_plist(plist.id())) } + /// Constructs a new dataset creation property list. pub fn finish(&self) -> Result { h5lock!({ let mut plist = DatasetCreate::try_new()?; @@ -664,18 +726,22 @@ impl DatasetCreateBuilder { /// Dataset creation property list. impl DatasetCreate { + /// Constructs a new dataset creation property list. pub fn try_new() -> Result { Self::from_id(h5try!(H5Pcreate(*H5P_DATASET_CREATE))) } + /// Returns a copy of the dataset creation property list. pub fn copy(&self) -> Self { unsafe { self.deref().copy().cast_unchecked() } } + /// Returns a builder for configuring a dataset creation property list. pub fn build() -> DatasetCreateBuilder { DatasetCreateBuilder::new() } + /// Returns `true` if all required filters are available. pub fn all_filters_avail(&self) -> bool { h5lock!(H5Pall_filters_avail(self.id())) > 0 } @@ -685,10 +751,12 @@ impl DatasetCreate { Filter::extract_pipeline(self.id()) } + /// Returns a vector of the dataset's filter configurations. pub fn filters(&self) -> Vec { self.get_filters().unwrap_or_default() } + /// Returns `true` if there is at least one filter configured. pub fn has_filters(&self) -> bool { !self.filters().is_empty() } @@ -698,6 +766,7 @@ impl DatasetCreate { h5get!(H5Pget_alloc_time(self.id()): H5D_alloc_time_t).map(Into::into) } + /// Returns the storage allocation timing settings. pub fn alloc_time(&self) -> AllocTime { self.get_alloc_time().unwrap_or(AllocTime::Late) } @@ -707,6 +776,7 @@ impl DatasetCreate { h5get!(H5Pget_fill_time(self.id()): H5D_fill_time_t).map(Into::into) } + /// Returns the fill value timing settings. pub fn fill_time(&self) -> FillTime { self.get_fill_time().unwrap_or_default() } @@ -716,6 +786,7 @@ impl DatasetCreate { h5get!(H5Pfill_value_defined(self.id()): H5D_fill_value_t).map(Into::into) } + /// Returns the fill value status. pub fn fill_value_defined(&self) -> FillValue { self.get_fill_value_defined().unwrap_or(FillValue::Undefined) } @@ -736,6 +807,7 @@ impl DatasetCreate { } } + /// Returns the fill value converted to a dynamic type, or `None` if not set. pub fn fill_value(&self, tp: &TypeDescriptor) -> Option { self.get_fill_value(tp).unwrap_or_default() } @@ -752,6 +824,7 @@ impl DatasetCreate { .transpose() } + /// Returns the fill value converted to a concrete type, or `None` if not set. pub fn fill_value_as(&self) -> Option { self.get_fill_value_as::().unwrap_or_default() } @@ -768,6 +841,7 @@ impl DatasetCreate { } } + /// Returns a vector of chunk dimensions for the dataset, or `None` if it is not chunked. pub fn chunk(&self) -> Option> { self.get_chunk().unwrap_or_default() } @@ -779,6 +853,7 @@ impl DatasetCreate { Ok(layout.into()) } + /// Returns the layout setting for the dataset's raw data. pub fn layout(&self) -> Layout { self.get_layout().unwrap_or_default() } @@ -794,6 +869,7 @@ impl DatasetCreate { } } + /// Returns the edge chunk option setting, or `None` if the dataset is not chunked. #[cfg(feature = "1.10.0")] pub fn chunk_opts(&self) -> Option { self.get_chunk_opts().unwrap_or_default() @@ -828,6 +904,7 @@ impl DatasetCreate { }) } + /// Returns a vector of external file specifiers for the dataset. pub fn external(&self) -> Vec { self.get_external().unwrap_or_default() } @@ -868,6 +945,7 @@ impl DatasetCreate { }) } + /// Returns a vector of virtual mapping specifiers for the dataset. #[cfg(feature = "1.10.0")] pub fn virtual_map(&self) -> Vec { self.get_virtual_map().unwrap_or_default() @@ -878,6 +956,7 @@ impl DatasetCreate { h5get!(H5Pget_obj_track_times(self.id()): hbool_t).map(|x| x > 0) } + /// Returns `true` if object time tracking is enabled for the dataset. pub fn obj_track_times(&self) -> bool { self.get_obj_track_times().unwrap_or(true) } @@ -888,6 +967,7 @@ impl DatasetCreate { .map(|(mc, md)| AttrPhaseChange { max_compact: mc as _, min_dense: md as _ }) } + /// Returns the attribute storage phase change thresholds. pub fn attr_phase_change(&self) -> AttrPhaseChange { self.get_attr_phase_change().unwrap_or_default() } @@ -898,6 +978,7 @@ impl DatasetCreate { .map(AttrCreationOrder::from_bits_truncate) } + /// Returns flags for whether attribute creation order will be tracked/indexed. pub fn attr_creation_order(&self) -> AttrCreationOrder { self.get_attr_creation_order().unwrap_or_default() } diff --git a/hdf5/src/hl/plist/file_access.rs b/hdf5/src/hl/plist/file_access.rs index 3d76f3dda..a943f2e9e 100644 --- a/hdf5/src/hl/plist/file_access.rs +++ b/hdf5/src/hl/plist/file_access.rs @@ -164,10 +164,14 @@ impl Clone for FileAccess { } } +/// Core file driver properties. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub struct CoreDriver { + /// Size, in bytes, of memory increments. pub increment: usize, + /// Whether to write the file contents to disk when the file is closed. pub filebacked: bool, + /// Size, in bytes, of write aggregation pages. Setting to 1 enables tracking with no paging. #[cfg(feature = "1.8.13")] pub write_tracking: usize, } @@ -183,8 +187,10 @@ impl Default for CoreDriver { } } +/// Family file driver properties. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct FamilyDriver { + /// Size in bytes of each file member. pub member_size: usize, } @@ -195,32 +201,63 @@ impl Default for FamilyDriver { } bitflags! { + /// Flags specifying types of logging activity for the logging virtual file driver. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct LogFlags: u64 { + /// Track truncate operations. const TRUNCATE = H5FD_LOG_TRUNCATE; + /// Track "meta" operations (e.g. truncate). const META_IO = H5FD_LOG_META_IO; + /// Track the location of every read. const LOC_READ = H5FD_LOG_LOC_READ; + /// Track the location of every write. const LOC_WRITE = H5FD_LOG_LOC_WRITE; + /// Track the location of every seek. const LOC_SEEK = H5FD_LOG_LOC_SEEK; + /// Track all I/O locations and lengths. + /// Equivalent to `LOC_READ | LOC_WRITE | LOC_SEEK`. const LOC_IO = H5FD_LOG_LOC_IO; + /// Track the number of times each byte is read. const FILE_READ = H5FD_LOG_FILE_READ; + /// Track the number of times each byte is written. const FILE_WRITE = H5FD_LOG_FILE_WRITE; + /// Track the number of all types of I/O operations. + /// Equivalent to `FILE_READ | FILE_WRITE`. const FILE_IO = H5FD_LOG_FILE_IO; + /// Track the type of information stored at each byte. const FLAVOR = H5FD_LOG_FLAVOR; + /// Track the total number of read operations. const NUM_READ = H5FD_LOG_NUM_READ; + /// Track the total number of write operations. const NUM_WRITE = H5FD_LOG_NUM_WRITE; + /// Track the total number of seek operations. const NUM_SEEK = H5FD_LOG_NUM_SEEK; + /// Track the total number of truncate operations. const NUM_TRUNCATE = H5FD_LOG_NUM_TRUNCATE; + /// Track the total number of all types of I/O operations. + /// Equivalent to `NUM_READ | NUM_WRITE | NUM_SEEK | NUM_TRUNCATE`. const NUM_IO = H5FD_LOG_NUM_IO; + /// Track the time spent in open operations. const TIME_OPEN = H5FD_LOG_TIME_OPEN; + /// Track the time spent in stat operations. const TIME_STAT = H5FD_LOG_TIME_STAT; + /// Track the time spent in read operations. const TIME_READ = H5FD_LOG_TIME_READ; + /// Track the time spent in write operations. const TIME_WRITE = H5FD_LOG_TIME_WRITE; + /// Track the time spent in seek operations. const TIME_SEEK = H5FD_LOG_TIME_SEEK; + /// Track the time spent in truncate operations. const TIME_TRUNCATE = H5FD_LOG_TIME_TRUNCATE; + /// Track the time spent in close operations. const TIME_CLOSE = H5FD_LOG_TIME_CLOSE; + /// Track the time spent in each I/O operation. + /// Equivalent to `TIME_OPEN | TIME_STAT | TIME_READ | TIME_WRITE | TIME_SEEK + /// | TIME_TRUNCATE | TIME_CLOSE`. const TIME_IO = H5FD_LOG_TIME_IO; + /// Track releases of space in the file. const FREE = H5FD_LOG_FREE; + /// Track everything. const ALL = H5FD_LOG_ALL; } } @@ -231,6 +268,7 @@ impl Default for LogFlags { } } +/// Logging virtual file driver properties. #[derive(Clone, Default, Debug, PartialEq, Eq)] pub struct LogOptions { logfile: Option, @@ -248,25 +286,36 @@ static FD_MEM_TYPES: &[H5F_mem_t] = &[ H5F_mem_t::H5FD_MEM_OHDR, ]; +/// Properties for a data storage used by the multi-file driver. #[derive(Clone, Debug, PartialEq, Eq)] pub struct MultiFile { + /// Name of the member file. pub name: String, + /// Offset within virtual address space where the storage begins. pub addr: u64, } impl MultiFile { + /// Creates a new `MultiFile`. pub fn new(name: &str, addr: u64) -> Self { Self { name: name.into(), addr } } } +/// A mapping of memory usage types to storage indices. #[derive(Clone, Debug, PartialEq, Eq)] pub struct MultiLayout { + /// Index of the superblock. pub mem_super: u8, + /// Index of the B-tree data. pub mem_btree: u8, + /// Index of the raw data. pub mem_draw: u8, + /// Index of the global heap data. pub mem_gheap: u8, + /// Index of the local heap data. pub mem_lheap: u8, + /// Index of the object headers. pub mem_object: u8, } @@ -302,10 +351,14 @@ impl MultiLayout { } } +/// Multi-file driver properties. #[derive(Clone, Debug, PartialEq, Eq)] pub struct MultiDriver { + /// The names and offsets of each type of data storage. pub files: Vec, + /// The mapping of memory usage types to file indices. pub layout: MultiLayout, + /// Whether to allow read-only access to incomplete file sets. pub relax: bool, } @@ -345,9 +398,12 @@ impl MultiDriver { } } +/// Split file driver properties. #[derive(Clone, Debug, PartialEq, Eq)] pub struct SplitDriver { + /// Metadata filename extension. pub meta_ext: String, + /// Raw data filename extension. pub raw_ext: String, } @@ -396,9 +452,12 @@ mod mpio { use super::{c_int, Result}; + /// MPI-I/O file driver properties. #[derive(Debug)] pub struct MpioDriver { + /// MPI-2 communicator. pub comm: MPI_Comm, + /// MPI-2 info object. pub info: MPI_Info, } @@ -453,11 +512,15 @@ mod mpio { #[cfg(feature = "mpio")] pub use self::mpio::*; +/// Direct I/O driver properties. #[cfg(feature = "have-direct")] #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct DirectDriver { + /// Required memory alignment boundary. pub alignment: usize, + /// File system block size. pub block_size: usize, + /// Size in bytes of the copy buffer. pub cbuf_size: usize, } @@ -468,26 +531,43 @@ impl Default for DirectDriver { } } +/// A low-level file driver configuration. #[derive(Clone, Debug)] pub enum FileDriver { + /// Uses POSIX filesystem functions to perform unbuffered access to a single file. Sec2, + /// Uses functions from the standard C `stdio.h` to perform buffered access to a single file. Stdio, + /// SEC2 with logging capabilities. Log, + /// Keeps file contents in memory until the file is closed, enabling faster access. Core(CoreDriver), + /// Partitions file address space into pieces and sends them to separate storage files. Family(FamilyDriver), + /// Allows data to be stored in multiple files according to the type of data. Multi(MultiDriver), + /// Special case of the Multi driver that stores metadata and raw data in separate files. Split(SplitDriver), + /// Uses the MPI standard for communication and I/O. #[cfg(feature = "mpio")] Mpio(MpioDriver), + /// SEC2 except data is accessed synchronously without being cached by the system. #[cfg(feature = "have-direct")] Direct(DirectDriver), } +/// Options for what to do when trying to close a file while there are open objects inside it. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum FileCloseDegree { + /// Let the driver choose the behavior. + /// + /// All drivers set this to `Weak`, except for MPI-I/O, which sets it to `Semi`. Default, + /// Terminate file identifier access, but delay closing until all objects are closed. Weak, + /// Return an error if the file has open objects. Semi, + /// Close all open objects, then close the file. Strong, } @@ -519,22 +599,33 @@ impl From for H5F_close_degree_t { } } +/// File alignment properties. +/// +/// Any file object with size of at least `threshold` bytes will be aligned on an address that is +/// a multiple of `alignment`. Addresses are relative to the end of the user block. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Alignment { + /// The byte size threshold. pub threshold: u64, + /// The alignment value. pub alignment: u64, } impl Default for Alignment { + /// Returns the default threshold and alignment of 1 (i.e. no alignment). fn default() -> Self { Self { threshold: 1, alignment: 1 } } } +/// Raw data chunk cache parameters. #[derive(Clone, Copy, Debug, PartialEq)] pub struct ChunkCache { + /// The number of objects in the cache. pub nslots: usize, + /// The total size of the cache in bytes. pub nbytes: usize, + /// The chunk preemption policy. pub w0: f64, } @@ -546,16 +637,25 @@ impl Default for ChunkCache { impl Eq for ChunkCache {} +/// Page buffer size properties. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub struct PageBufferSize { + /// Maximum size, in bytes, of the page buffer. pub buf_size: usize, + /// Minimum metadata percentage to keep in the buffer before allowing pages with metadata to be + /// evicted. pub min_meta_perc: u32, + /// Minimum raw data percentage to keep in the buffer before allowing pages with raw data to be + /// evicted. pub min_raw_perc: u32, } +/// Automatic cache size increase mode. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum CacheIncreaseMode { + /// Automatic increase is disabled. Off, + /// Automatic increase uses the hit rate threshold algorithm. Threshold, } @@ -577,9 +677,12 @@ impl From for H5C_cache_incr_mode { } } +/// Flash cache size increase mode. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum FlashIncreaseMode { + /// Flash cache size increase is disabled. Off, + /// Flash cache size increase uses the add space algorithm. AddSpace, } @@ -601,11 +704,16 @@ impl From for H5C_cache_flash_incr_mode { } } +/// Automatic cache size decrease mode. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum CacheDecreaseMode { + /// Automatic decrease is disabled. Off, + /// Automatic decrease uses the hit rate threshold algorithm. Threshold, + /// Automatic decrease uses the ageout algorithm. AgeOut, + /// Automatic decrease uses the ageout with hit rate threshold algorithm. AgeOutWithThreshold, } @@ -631,9 +739,13 @@ impl From for H5C_cache_decr_mode { } } +/// A strategy for writing metadata to disk. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum MetadataWriteStrategy { + /// Only process zero is allowed to write dirty metadata to disk. ProcessZeroOnly, + /// Process zero decides what entries to flush, but the flushes are distributed across + /// processes. Distributed, } @@ -661,36 +773,76 @@ impl From for c_int { } } +/// Metadata cache configuration. #[derive(Clone, Debug, PartialEq)] pub struct MetadataCacheConfig { + /// Whether the adaptive cache resize report function is enabled. pub rpt_fcn_enabled: bool, + /// Whether `trace_file_name` should be used to open a trace file for the cache. pub open_trace_file: bool, + /// Whether the current trace file (if any) should be closed. pub close_trace_file: bool, + /// Full path of the trace file to be opened if `open_trace_file` is `true`. pub trace_file_name: String, + /// Whether evictions from the metadata cache are enabled. pub evictions_enabled: bool, + /// Whether the cache should be created with a user-specified initial size. pub set_initial_size: bool, + /// Initial cache size in bytes if `set_initial_size` is `true`. pub initial_size: usize, + /// Minimum fraction of the cache that must be kept clean or empty. pub min_clean_fraction: f64, + /// Maximum number of bytes that adaptive cache resizing can select as the maximum cache size. pub max_size: usize, + /// Minimum number of bytes that adaptive cache resizing can select as the minimum cache size. pub min_size: usize, + /// Number of cache accesses between runs of the adaptive cache resize code. pub epoch_length: i64, + /// Automatic cache size increase mode. pub incr_mode: CacheIncreaseMode, + /// Hit rate threshold for the hit rate threshold cache size increment algorithm. pub lower_hr_threshold: f64, + /// Factor by which the hit rate threshold cache size increment algorithm multiplies the current + /// cache max size to obtain a tentative new size. pub increment: f64, + /// Whether to apply an upper limit to the size of cache size increases. pub apply_max_increment: bool, + /// Maximum number of bytes by which cache size can be increased in a single step, + /// if applicable. pub max_increment: usize, + /// Flash cache size increase mode. pub flash_incr_mode: FlashIncreaseMode, + /// Factor by which the size of the triggering entry / entry size increase is multiplied to + /// obtain the initial cache size increment. pub flash_multiple: f64, + /// Factor by which the current maximum cache size is multiplied to obtain the minimum size + /// entry / entry size increase which may trigger a flash cache size increase. pub flash_threshold: f64, + /// Automatic cache size decrease mode. pub decr_mode: CacheDecreaseMode, + /// Hit rate threshold for hit-rate-based cache size decrease algorithms. pub upper_hr_threshold: f64, + /// Factor by which the hit rate threshold cache size decrease algorithm multiplies the current + /// cache max size to obtain a tentative new size. pub decrement: f64, + /// Whether an upper limit should be applied to the size of cache size decreases. pub apply_max_decrement: bool, + /// Maximum number of bytes by which cache size can be decreased in a single step, + /// if applicable. pub max_decrement: usize, + /// Minimum number of epochs that an entry must remain unaccessed in cache before ageout-based + /// reduction algorithms try to evict it. pub epochs_before_eviction: i32, + /// Whether ageout-based decrement algorithms will maintain an empty reserve. pub apply_empty_reserve: bool, + /// Empty reserve as a fraction of maximum cache size. + /// Ageout-based algorithms will not decrease the maximum size unless the empty reserve can be + /// met. pub empty_reserve: f64, + /// Threshold number of bytes of dirty metadata that will trigger synchronization of + /// parallel metadata caches. pub dirty_bytes_threshold: usize, + /// Strategy for writing metadata to disk. pub metadata_write_strategy: MetadataWriteStrategy, } @@ -816,10 +968,15 @@ impl From for MetadataCacheConfig { mod cache_image_config { use super::*; + /// Metadata cache image configuration. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub struct CacheImageConfig { + /// Whether a cache image should be created on file close. pub generate_image: bool, + /// Whether the cache image should include the adaptive cache resize configuration and + /// status. pub save_resize_status: bool, + /// Maximum number of times a prefetched entry can appear in subsequent cache images. pub entry_ageout: i32, } @@ -858,11 +1015,15 @@ mod cache_image_config { #[cfg(feature = "1.10.1")] pub use self::cache_image_config::*; +/// Metadata cache logging options. #[cfg(feature = "1.10.0")] #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct CacheLogOptions { + /// Whether logging is enabled. pub is_enabled: bool, + /// File path of the log. (Must be ASCII on Windows) pub location: String, + /// Whether to begin logging as soon as the file is opened pub start_on_access: bool, } @@ -870,18 +1031,24 @@ pub struct CacheLogOptions { mod libver { use super::*; + /// Options for which library format version to use when storing objects. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] pub enum LibraryVersion { + /// Use the earliest possible format. Earliest = 0, + /// Use the latest v18 format. V18 = 1, + /// Use the latest v110 format. V110 = 2, } impl LibraryVersion { + /// Returns `true` if the version is set to `Earliest`. pub fn is_earliest(self) -> bool { self == Self::Earliest } + /// Returns the latest library version. pub const fn latest() -> Self { Self::V110 } @@ -907,9 +1074,12 @@ mod libver { } } + /// Library format version bounds for writing objects to a file. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct LibVerBounds { + /// The earliest version to use for writing objects. pub low: LibraryVersion, + /// The latest version to use for writing objects. pub high: LibraryVersion, } @@ -1037,27 +1207,32 @@ impl FileAccessBuilder { self } + /// Sets the file alignment parameters. pub fn alignment(&mut self, threshold: u64, alignment: u64) -> &mut Self { self.alignment = Some(Alignment { threshold, alignment }); self } + /// Sets the raw data chunk cache parameters. pub fn chunk_cache(&mut self, nslots: usize, nbytes: usize, w0: f64) -> &mut Self { self.chunk_cache = Some(ChunkCache { nslots, nbytes, w0 }); self } + /// Sets the number of files that can be held open in an external link open file cache. #[cfg(feature = "1.8.7")] pub fn elink_file_cache_size(&mut self, efc_size: u32) -> &mut Self { self.elink_file_cache_size = Some(efc_size); self } + /// Sets the minimum metadata block size in bytes. pub fn meta_block_size(&mut self, size: u64) -> &mut Self { self.meta_block_size = Some(size); self } + /// Sets the page buffer size properties. #[cfg(feature = "1.10.1")] pub fn page_buffer_size( &mut self, buf_size: usize, min_meta_perc: u32, min_raw_perc: u32, @@ -1066,28 +1241,34 @@ impl FileAccessBuilder { self } + /// Sets the maximum size of the data sieve buffer. pub fn sieve_buf_size(&mut self, size: usize) -> &mut Self { self.sieve_buf_size = Some(size); self } + /// Sets whether object metadata should be evicted from cache when an object is closed. #[cfg(feature = "1.10.1")] pub fn evict_on_close(&mut self, evict_on_close: bool) -> &mut Self { self.evict_on_close = Some(evict_on_close); self } + /// Sets the number of reads that the library will try when reading checksummed metadata in a + /// file opened with SWMR access. #[cfg(feature = "1.10.0")] pub fn metadata_read_attempts(&mut self, attempts: u32) -> &mut Self { self.metadata_read_attempts = Some(attempts); self } + /// Sets the metadata cache configuration. pub fn mdc_config(&mut self, config: &MetadataCacheConfig) -> &mut Self { self.mdc_config = Some(config.clone()); self } + /// Sets whether a cache image should be created on file close. #[cfg(feature = "1.10.1")] pub fn mdc_image_config(&mut self, generate_image: bool) -> &mut Self { self.mdc_image_config = Some(CacheImageConfig { @@ -1098,6 +1279,7 @@ impl FileAccessBuilder { self } + /// Sets metadata cache logging options. #[cfg(feature = "1.10.0")] pub fn mdc_log_options( &mut self, is_enabled: bool, location: &str, start_on_access: bool, @@ -1107,67 +1289,80 @@ impl FileAccessBuilder { self } + /// Sets whether metadata reads are collective. #[cfg(all(feature = "1.10.0", feature = "have-parallel"))] pub fn all_coll_metadata_ops(&mut self, is_collective: bool) -> &mut Self { self.all_coll_metadata_ops = Some(is_collective); self } + /// Sets whether metadata writes are collective. #[cfg(all(feature = "1.10.0", feature = "have-parallel"))] pub fn coll_metadata_write(&mut self, is_collective: bool) -> &mut Self { self.coll_metadata_write = Some(is_collective); self } + /// Sets whether reference garbage collection is enabled. pub fn gc_references(&mut self, gc_ref: bool) -> &mut Self { self.gc_references = Some(gc_ref); self } + /// Sets the maximum size in bytes of a contiguous block reserved for small data. pub fn small_data_block_size(&mut self, size: u64) -> &mut Self { self.small_data_block_size = Some(size); self } + /// Sets the range of library versions to use when writing objects. #[cfg(feature = "1.10.2")] pub fn libver_bounds(&mut self, low: LibraryVersion, high: LibraryVersion) -> &mut Self { self.libver_bounds = Some(LibVerBounds { low, high }); self } + /// Allows use of the earliest library version when writing objects. #[cfg(feature = "1.10.2")] pub fn libver_earliest(&mut self) -> &mut Self { self.libver_bounds(LibraryVersion::Earliest, LibraryVersion::latest()) } + /// Sets the earliest library version for writing objects to v18. #[cfg(feature = "1.10.2")] pub fn libver_v18(&mut self) -> &mut Self { self.libver_bounds(LibraryVersion::V18, LibraryVersion::latest()) } + /// Sets the earliest library version for writing objects to v110. #[cfg(feature = "1.10.2")] pub fn libver_v110(&mut self) -> &mut Self { self.libver_bounds(LibraryVersion::V110, LibraryVersion::latest()) } + /// Allows only the latest library version when writing objects. #[cfg(feature = "1.10.2")] pub fn libver_latest(&mut self) -> &mut Self { self.libver_bounds(LibraryVersion::latest(), LibraryVersion::latest()) } + /// Sets which file driver to use. pub fn driver(&mut self, file_driver: &FileDriver) -> &mut Self { self.file_driver = Some(file_driver.clone()); self } + /// Sets the file driver to SEC2 (POSIX). pub fn sec2(&mut self) -> &mut Self { self.driver(&FileDriver::Sec2) } + /// Sets the file driver to STDIO. pub fn stdio(&mut self) -> &mut Self { self.driver(&FileDriver::Stdio) } + /// Sets the file driver to SEC2 with logging and configures it. pub fn log_options( &mut self, logfile: Option<&str>, flags: LogFlags, buf_size: usize, ) -> &mut Self { @@ -1177,38 +1372,46 @@ impl FileAccessBuilder { self.driver(&FileDriver::Log) } + /// Sets the file driver to SEC2 with logging. pub fn log(&mut self) -> &mut Self { self.log_options(None, LogFlags::LOC_IO, 0) } + /// Sets the file driver to Core and configures it. pub fn core_options(&mut self, increment: usize, filebacked: bool) -> &mut Self { let drv = CoreDriver { increment, filebacked, ..CoreDriver::default() }; self.driver(&FileDriver::Core(drv)) } + /// Sets the file driver to Core and sets whether to write file contents to disk upon closing. pub fn core_filebacked(&mut self, filebacked: bool) -> &mut Self { let drv = CoreDriver { filebacked, ..CoreDriver::default() }; self.driver(&FileDriver::Core(drv)) } + /// Sets the file driver to Core. pub fn core(&mut self) -> &mut Self { self.driver(&FileDriver::Core(CoreDriver::default())) } + /// Sets the write tracking page size for the Core file driver. #[cfg(feature = "1.8.13")] pub fn write_tracking(&mut self, page_size: usize) -> &mut Self { self.write_tracking = Some(page_size); self } + /// Sets the file driver to Family. pub fn family(&mut self) -> &mut Self { self.driver(&FileDriver::Family(FamilyDriver::default())) } + /// Sets the file driver to Family and configures the file member size. pub fn family_options(&mut self, member_size: usize) -> &mut Self { self.driver(&FileDriver::Family(FamilyDriver { member_size })) } + /// Sets the file driver to Multi and configures it. pub fn multi_options( &mut self, files: &[MultiFile], layout: &MultiLayout, relax: bool, ) -> &mut Self { @@ -1219,10 +1422,12 @@ impl FileAccessBuilder { })) } + /// Sets the file driver to Multi. pub fn multi(&mut self) -> &mut Self { self.driver(&FileDriver::Multi(MultiDriver::default())) } + /// Sets the file driver to Split and configures it. pub fn split_options(&mut self, meta_ext: &str, raw_ext: &str) -> &mut Self { self.driver(&FileDriver::Split(SplitDriver { meta_ext: meta_ext.into(), @@ -1230,16 +1435,19 @@ impl FileAccessBuilder { })) } + /// Sets the file driver to Split. pub fn split(&mut self) -> &mut Self { self.driver(&FileDriver::Split(SplitDriver::default())) } + /// Sets the file driver to MPI-I/O and configures it. #[cfg(feature = "mpio")] pub fn mpio(&mut self, comm: mpi_sys::MPI_Comm, info: Option) -> &mut Self { // We use .unwrap() here since MPI will almost surely terminate the process anyway. self.driver(&FileDriver::Mpio(MpioDriver::try_new(comm, info).unwrap())) } + /// Sets the file driver to Direct and configures it. #[cfg(feature = "have-direct")] pub fn direct_options( &mut self, alignment: usize, block_size: usize, cbuf_size: usize, @@ -1247,6 +1455,7 @@ impl FileAccessBuilder { self.driver(&FileDriver::Direct(DirectDriver { alignment, block_size, cbuf_size })) } + /// Sets the file driver to Direct. #[cfg(feature = "have-direct")] pub fn direct(&mut self) -> &mut Self { self.driver(&FileDriver::Direct(DirectDriver::default())) @@ -1487,10 +1696,12 @@ impl FileAccessBuilder { Ok(()) } + /// Copies the builder settings into a file access property list. pub fn apply(&self, plist: &mut FileAccess) -> Result<()> { h5lock!(self.populate_plist(plist.id())) } + /// Constructs a new file access property list. pub fn finish(&self) -> Result { h5lock!({ let mut plist = FileAccess::try_new()?; @@ -1501,14 +1712,17 @@ impl FileAccessBuilder { /// File access property list. impl FileAccess { + /// Creates a new file access property list. pub fn try_new() -> Result { Self::from_id(h5try!(H5Pcreate(*H5P_FILE_ACCESS))) } + /// Creates a copy of the property list. pub fn copy(&self) -> Self { unsafe { self.deref().copy().cast_unchecked() } } + /// Creates a new file access property list builder. pub fn build() -> FileAccessBuilder { FileAccessBuilder::new() } @@ -1640,6 +1854,7 @@ impl FileAccess { } } + /// Returns the file driver properties. pub fn driver(&self) -> FileDriver { self.get_driver().unwrap_or(FileDriver::Sec2) } @@ -1649,6 +1864,7 @@ impl FileAccess { h5get!(H5Pget_fclose_degree(self.id()): H5F_close_degree_t).map(Into::into) } + /// Returns the file close degree. pub fn fclose_degree(&self) -> FileCloseDegree { self.get_fclose_degree().unwrap_or_else(|_| FileCloseDegree::default()) } @@ -1660,6 +1876,7 @@ impl FileAccess { }) } + /// Returns the file alignment properties. pub fn alignment(&self) -> Alignment { self.get_alignment().unwrap_or_else(|_| Alignment::default()) } @@ -1675,6 +1892,7 @@ impl FileAccess { ) } + /// Returns the raw data chunk cache properties. pub fn chunk_cache(&self) -> ChunkCache { self.get_chunk_cache().unwrap_or_else(|_| ChunkCache::default()) } @@ -1695,6 +1913,7 @@ impl FileAccess { h5get!(H5Pget_meta_block_size(self.id()): hsize_t).map(|x| x as _) } + /// Returns the metadata block size. pub fn meta_block_size(&self) -> u64 { self.get_meta_block_size().unwrap_or(2048) } @@ -1711,6 +1930,7 @@ impl FileAccess { ) } + /// Returns the page buffer size properties. #[cfg(feature = "1.10.1")] pub fn page_buffer_size(&self) -> PageBufferSize { self.get_page_buffer_size().unwrap_or_else(|_| PageBufferSize::default()) @@ -1721,6 +1941,7 @@ impl FileAccess { h5get!(H5Pget_sieve_buf_size(self.id()): size_t).map(|x| x as _) } + /// Returns the maximum data sieve buffer size. pub fn sieve_buf_size(&self) -> usize { self.get_sieve_buf_size().unwrap_or(64 * 1024) } @@ -1731,6 +1952,8 @@ impl FileAccess { h5get!(H5Pget_evict_on_close(self.id()): hbool_t).map(|x| x > 0) } + /// Returns `true` if an object will be evicted from the metadata cache when the object is + /// closed. #[cfg(feature = "1.10.1")] pub fn evict_on_close(&self) -> bool { self.get_evict_on_close().unwrap_or(false) @@ -1742,6 +1965,7 @@ impl FileAccess { h5get!(H5Pget_metadata_read_attempts(self.id()): c_uint).map(|x| x as _) } + /// Returns the number of read attempts for SWMR access. #[cfg(feature = "1.10.0")] pub fn metadata_read_attempts(&self) -> u32 { self.get_metadata_read_attempts().unwrap_or(1) @@ -1754,6 +1978,7 @@ impl FileAccess { h5call!(H5Pget_mdc_config(self.id(), &mut config)).map(|_| config.into()) } + /// Returns the metadata cache configuration. pub fn mdc_config(&self) -> MetadataCacheConfig { self.get_mdc_config().ok().unwrap_or_default() } @@ -1766,6 +1991,7 @@ impl FileAccess { h5call!(H5Pget_mdc_image_config(self.id(), &mut config)).map(|_| config.into()) } + /// Returns the metadata cache image configuration. #[cfg(feature = "1.10.1")] pub fn mdc_image_config(&self) -> CacheImageConfig { self.get_mdc_image_config().ok().unwrap_or_default() @@ -1801,6 +2027,7 @@ impl FileAccess { }) } + /// Returns the metadata cache logging options. #[cfg(feature = "1.10.0")] pub fn mdc_log_options(&self) -> CacheLogOptions { self.get_mdc_log_options().ok().unwrap_or_default() @@ -1812,6 +2039,7 @@ impl FileAccess { h5get!(H5Pget_all_coll_metadata_ops(self.id()): hbool_t).map(|x| x > 0) } + /// Returns `true` if metadata reads are collective. #[cfg(all(feature = "1.10.0", feature = "have-parallel"))] pub fn all_coll_metadata_ops(&self) -> bool { self.get_all_coll_metadata_ops().unwrap_or(false) @@ -1823,6 +2051,7 @@ impl FileAccess { h5get!(H5Pget_coll_metadata_write(self.id()): hbool_t).map(|x| x > 0) } + /// Returns `true` if metadata writes are collective. #[cfg(all(feature = "1.10.0", feature = "have-parallel"))] pub fn coll_metadata_write(&self) -> bool { self.get_coll_metadata_write().unwrap_or(false) @@ -1833,6 +2062,7 @@ impl FileAccess { h5get!(H5Pget_gc_references(self.id()): c_uint).map(|x| x > 0) } + /// Returns `true` if reference garbage collection is enabled. pub fn gc_references(&self) -> bool { self.get_gc_references().unwrap_or(false) } @@ -1842,6 +2072,7 @@ impl FileAccess { h5get!(H5Pget_small_data_block_size(self.id()): hsize_t).map(|x| x as _) } + /// Returns the size setting in bytes of the small data block. pub fn small_data_block_size(&self) -> u64 { self.get_small_data_block_size().unwrap_or(2048) } @@ -1853,11 +2084,13 @@ impl FileAccess { .map(|(low, high)| LibVerBounds { low: low.into(), high: high.into() }) } + /// Returns the library format version bounds for writing objects to a file. #[cfg(feature = "1.10.2")] pub fn libver_bounds(&self) -> LibVerBounds { self.get_libver_bounds().ok().unwrap_or_default() } + /// Returns the lower library format version bound for writing objects to a file. #[cfg(feature = "1.10.2")] pub fn libver(&self) -> LibraryVersion { self.get_libver_bounds().ok().unwrap_or_default().low diff --git a/hdf5/src/hl/plist/file_create.rs b/hdf5/src/hl/plist/file_create.rs index d4e109bde..bc2e9c994 100644 --- a/hdf5/src/hl/plist/file_create.rs +++ b/hdf5/src/hl/plist/file_create.rs @@ -424,10 +424,12 @@ impl FileCreateBuilder { Ok(()) } + /// Copies the builder settings into a file creation property list. pub fn apply(&self, plist: &mut FileCreate) -> Result<()> { h5lock!(self.populate_plist(plist.id())) } + /// Constructs a new file creation property list. pub fn finish(&self) -> Result { h5lock!({ let mut plist = FileCreate::try_new()?; @@ -438,14 +440,17 @@ impl FileCreateBuilder { /// File creation property list. impl FileCreate { + /// Creates a new file creation property list. pub fn try_new() -> Result { Self::from_id(h5try!(H5Pcreate(*H5P_FILE_CREATE))) } + /// Creates a copy of the file creation property list. pub fn copy(&self) -> Self { unsafe { self.deref().copy().cast_unchecked() } } + /// Returns a builder for configuring a file creation property list. pub fn build() -> FileCreateBuilder { FileCreateBuilder::new() } diff --git a/hdf5/src/hl/plist/link_create.rs b/hdf5/src/hl/plist/link_create.rs index 2bb6c7c74..9f507b050 100644 --- a/hdf5/src/hl/plist/link_create.rs +++ b/hdf5/src/hl/plist/link_create.rs @@ -69,9 +69,12 @@ impl Clone for LinkCreate { } } +/// The character encoding used to create a link or attribute name. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum CharEncoding { + /// US ASCII. Ascii, + /// UTF-8. Utf8, } @@ -96,11 +99,13 @@ impl LinkCreateBuilder { Ok(builder) } + /// Sets whether to create intermediate groups upon creation of an object. pub fn create_intermediate_group(&mut self, create: bool) -> &mut Self { self.create_intermediate_group = Some(create); self } + /// Sets the character encoding to use when creating links. pub fn char_encoding(&mut self, encoding: CharEncoding) -> &mut Self { self.char_encoding = Some(encoding); self @@ -120,10 +125,12 @@ impl LinkCreateBuilder { Ok(()) } + /// Copies the builder settings into a link creation property list. pub fn apply(&self, plist: &mut LinkCreate) -> Result<()> { h5lock!(self.populate_plist(plist.id())) } + /// Constructs a new link creation property list. pub fn finish(&self) -> Result { h5lock!({ let mut plist = LinkCreate::try_new()?; @@ -134,14 +141,17 @@ impl LinkCreateBuilder { /// Link create property list. impl LinkCreate { + /// Creates a new link creation property list. pub fn try_new() -> Result { Self::from_id(h5try!(H5Pcreate(*H5P_LINK_CREATE))) } + /// Creates a copy of the link creation property list. pub fn copy(&self) -> Self { unsafe { self.deref().copy().cast_unchecked() } } + /// Returns a builder for configuring a link creation property list. pub fn build() -> LinkCreateBuilder { LinkCreateBuilder::new() } @@ -151,6 +161,7 @@ impl LinkCreate { h5get!(H5Pget_create_intermediate_group(self.id()): c_uint).map(|x| x > 0) } + /// Returns `true` if intermediate groups will be created upon object creation. pub fn create_intermediate_group(&self) -> bool { self.get_create_intermediate_group().unwrap_or(false) } @@ -164,6 +175,7 @@ impl LinkCreate { }) } + /// Returns the character encoding used to create links. pub fn char_encoding(&self) -> CharEncoding { self.get_char_encoding().unwrap_or(CharEncoding::Ascii) } diff --git a/hdf5/src/hl/selection.rs b/hdf5/src/hl/selection.rs index 3d7db831a..8b987e626 100644 --- a/hdf5/src/hl/selection.rs +++ b/hdf5/src/hl/selection.rs @@ -772,8 +772,11 @@ impl Display for Hyperslab { #[derive(Clone, Debug, PartialEq, Eq)] /// A selection used for reading and writing to a [`Container`](Container). pub enum Selection { + /// The entire dataset. All, + /// A sequence of points. Points(Array2), + /// A hyperslab or compound hyperslab. Hyperslab(Hyperslab), } @@ -784,10 +787,16 @@ impl Default for Selection { } impl Selection { + /// Creates a new selection. pub fn new>(selection: T) -> Self { selection.into() } + /// Tries to create a new selection. + /// + /// # Errors + /// + /// Returns an error if the conversion method fails. pub fn try_new>(selection: T) -> Result { selection.try_into() } @@ -829,6 +838,8 @@ impl Selection { }) } + /// Returns the required number of dimensions for the input dataspace, + /// or `None` if selecting everything. pub fn in_ndim(&self) -> Option { match self { Self::All => None, @@ -843,6 +854,7 @@ impl Selection { } } + /// Returns the number of dimensions in the output dataspace, or `None` if selecting everything. pub fn out_ndim(&self) -> Option { match self { Self::All => None, @@ -853,6 +865,11 @@ impl Selection { } } + /// Returns the output shape that would result from applying the selection to some input shape. + /// + /// # Errors + /// + /// Returns an error if the input shape is incompatible with the selection. pub fn out_shape>(&self, in_shape: S) -> Result> { let in_shape = in_shape.as_ref(); match self { @@ -875,10 +892,12 @@ impl Selection { } } + /// Returns `true` if the selection is for the entire dataset. pub fn is_all(&self) -> bool { self == &Self::All } + /// Returns `true` if the selection is a sequence of points. pub fn is_points(&self) -> bool { if let Self::Points(ref points) = self { points.shape() != [0, 0] @@ -887,6 +906,7 @@ impl Selection { } } + /// Returns `true` if the selection is empty. pub fn is_none(&self) -> bool { if let Self::Points(points) = self { points.shape() == [0, 0] @@ -895,6 +915,7 @@ impl Selection { } } + /// Returns `true` if the selection is a hyperslab. pub fn is_hyperslab(&self) -> bool { matches!(self, Self::Hyperslab(_)) } diff --git a/hdf5/src/lib.rs b/hdf5/src/lib.rs index 73e18902a..9f5c6faa6 100644 --- a/hdf5/src/lib.rs +++ b/hdf5/src/lib.rs @@ -76,10 +76,12 @@ mod export { pub use hdf5_derive::H5Type; pub use hdf5_types::H5Type; + /// Base types and interfaces for creating compound data types. pub mod types { pub use hdf5_types::*; } + /// Multi-dimensional datasets. pub mod dataset { #[cfg(feature = "1.10.5")] pub use crate::hl::chunks::ChunkInfo; @@ -90,16 +92,19 @@ mod export { pub use crate::hl::plist::dataset_create::*; } + /// Datatype objects for defining the layout of a data element. pub mod datatype { pub use crate::hl::datatype::{ByteOrder, Conversion, Datatype}; } + /// HDF5 file objects. pub mod file { pub use crate::hl::file::{File, FileBuilder, OpenMode}; pub use crate::hl::plist::file_access::*; pub use crate::hl::plist::file_create::*; } + /// Property list objects. pub mod plist { pub use crate::hl::plist::dataset_access::{DatasetAccess, DatasetAccessBuilder}; pub use crate::hl::plist::dataset_create::{DatasetCreate, DatasetCreateBuilder}; @@ -109,22 +114,29 @@ mod export { pub use crate::hl::plist::object_copy::{ObjectCopy, ObjectCopyBuilder}; pub use crate::hl::plist::{PropertyList, PropertyListClass}; + /// Dataset access property lists. pub mod dataset_access { pub use crate::hl::plist::dataset_access::*; } + /// Dataset creation property lists. pub mod dataset_create { pub use crate::hl::plist::dataset_create::*; } + /// File access property lists. pub mod file_access { pub use crate::hl::plist::file_access::*; } + /// File creation property lists. pub mod file_create { pub use crate::hl::plist::file_create::*; } + /// Link creation property lists. pub mod link_create { pub use crate::hl::plist::link_create::*; } } + + /// Filters for data compression and validation during file I/O. pub mod filters { pub use crate::hl::filters::*; } From 701da869772b5f2d6ea2cccbdaa93cfb521386c2 Mon Sep 17 00:00:00 2001 From: Georg Semmler Date: Tue, 16 Dec 2025 14:09:21 +0100 Subject: [PATCH 097/141] Exclude development scripts from published package During a dependency review we noticed that the hdf5-metno-src crate includes various development scripts. These development scripts shouldn't be there as they might, at some point become problematic. As of now they prevent any downstream user from enabling the `[bans.build.interpreted]` option of cargo deny. I opted for using an explicit include list instead of an exclude list to prevent these files from being included in the published packages to make sure that everything that's included is an conscious choice. --- hdf5-src/Cargo.toml | 54 ++++++++++++++++++++++++++++++++------------- 1 file changed, 39 insertions(+), 15 deletions(-) diff --git a/hdf5-src/Cargo.toml b/hdf5-src/Cargo.toml index 256c4f615..836ef627c 100644 --- a/hdf5-src/Cargo.toml +++ b/hdf5-src/Cargo.toml @@ -6,21 +6,45 @@ description = "Build script for compiling HDF5 C library from source." links = "hdf5src" readme = "README.md" categories = ["development-tools::ffi"] -exclude = [ - "ext/hdf5/bin/**", - "ext/hdf5/c++/**", - "ext/hdf5/examples/**", - "ext/hdf5/fortran/**", - "ext/hdf5/java/**", - "ext/hdf5/release_docs/**", - "ext/hdf5/test/**", - "ext/hdf5/testpar/**", - "ext/hdf5/tools/**", - "ext/hdf5/hl/test/**", - "ext/hdf5/hl/tools/**", - "ext/hdf5/hl/examples/**", - "ext/hdf5/HDF5Examples/**", - "ext/hdf5/doxygen/**", +include = [ + "Cargo.toml", + "build.rs", + "README.md", + "src/**/*.rs", + "ext/hdf5/**/CMakeLists.txt", + "ext/hdf5/COPYING", + "ext/hdf5/**/*.h", + "ext/hdf5/**/*.c", + "ext/hdf5/**/*.am", + "ext/hdf5/**/*.cmake", + "ext/hdf5/**/*.in", + "ext/hdf5/config/*-warnings", + "ext/hdf5/config/*flags", + "ext/hdf5/config/apple", + "ext/hdf5/config/cygwin", + "ext/hdf5/config/freebsd", + "ext/hdf5/config/ibm-aix", + "ext/hdf5/config/linux-gnu", + "ext/hdf5/config/netbsd", + "ext/hdf5/config/solaris", + "ext/hdf5/release_docs/USING_HDF5_CMake.txt", + "ext/hdf5/release_docs/RELEASE.txt", + "!ext/hdf5/bin/**", + "!ext/hdf5/c++/**", + "!ext/hdf5/examples/**", + "!ext/hdf5/fortran/**", + "!ext/hdf5/java/**", + "!ext/hdf5/release_docs/**", + "!ext/hdf5/test/**", + "!ext/hdf5/testpar/**", + "!ext/hdf5/tools/**", + "!ext/hdf5/hl/test/**", + "!ext/hdf5/hl/tools/**", + "!ext/hdf5/hl/fortran/**", + "!ext/hdf5/hl/c++/**", + "!ext/hdf5/hl/examples/**", + "!ext/hdf5/HDF5Examples/**", + "!ext/hdf5/doxygen/**", ] version = "0.9.5" # !V rust-version.workspace = true From e3aac6789a800320d8e12b8302a0819459155c23 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Tue, 16 Dec 2025 19:46:14 +0100 Subject: [PATCH 098/141] Update macos-13 to 14 in CI --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 650b1cbcd..611c8a9f9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,7 +95,7 @@ jobs: include: - {os: ubuntu-latest, version: 1.8.16, channel: conda-forge, rust: stable} - {os: windows-latest, version: 1.8.17, channel: conda-forge, rust: stable} - - {os: macos-13, version: 1.8.18, channel: anaconda, rust: stable} + - {os: macos-14, version: 1.8.18, channel: anaconda, rust: stable} - {os: ubuntu-latest, version: 1.8.20, channel: anaconda, rust: beta} - {os: ubuntu-latest, version: 1.10.1, channel: anaconda, rust: nightly} - {os: windows-latest, version: 1.10.2, channel: anaconda, rust: beta} @@ -103,7 +103,7 @@ jobs: - {os: windows-latest, version: 1.10.4, channel: anaconda, rust: nightly} - {os: ubuntu-latest, version: 1.10.4, mpi: openmpi, channel: conda-forge, rust: stable} - {os: ubuntu-latest, version: 1.10.5, channel: conda-forge, rust: beta} - - {os: macos-13, version: 1.10.5, mpi: openmpi, channel: conda-forge, rust: beta} + - {os: macos-14, version: 1.10.5, mpi: openmpi, channel: conda-forge, rust: beta} - {os: ubuntu-latest, version: 1.10.6, channel: anaconda, rust: stable} - {os: ubuntu-latest, version: 1.10.6, mpi: mpich, channel: conda-forge, rust: nightly} # - {os: ubuntu, version: 1.10.8, channel: conda-forge, rust: stable} From 4cfe9800f28838bd2284d2dc0fee0219e57f69b4 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Tue, 16 Dec 2025 20:12:17 +0100 Subject: [PATCH 099/141] Switch macos to newer hdf5 --- .github/workflows/ci.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 611c8a9f9..5ad8023bb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,7 +95,6 @@ jobs: include: - {os: ubuntu-latest, version: 1.8.16, channel: conda-forge, rust: stable} - {os: windows-latest, version: 1.8.17, channel: conda-forge, rust: stable} - - {os: macos-14, version: 1.8.18, channel: anaconda, rust: stable} - {os: ubuntu-latest, version: 1.8.20, channel: anaconda, rust: beta} - {os: ubuntu-latest, version: 1.10.1, channel: anaconda, rust: nightly} - {os: windows-latest, version: 1.10.2, channel: anaconda, rust: beta} @@ -103,14 +102,12 @@ jobs: - {os: windows-latest, version: 1.10.4, channel: anaconda, rust: nightly} - {os: ubuntu-latest, version: 1.10.4, mpi: openmpi, channel: conda-forge, rust: stable} - {os: ubuntu-latest, version: 1.10.5, channel: conda-forge, rust: beta} - - {os: macos-14, version: 1.10.5, mpi: openmpi, channel: conda-forge, rust: beta} - {os: ubuntu-latest, version: 1.10.6, channel: anaconda, rust: stable} - {os: ubuntu-latest, version: 1.10.6, mpi: mpich, channel: conda-forge, rust: nightly} - # - {os: ubuntu, version: 1.10.8, channel: conda-forge, rust: stable} - {os: ubuntu-latest, version: 1.12.0, mpi: openmpi, channel: conda-forge, rust: stable} - - {os: macos-latest, version: 1.12.0, channel: conda-forge, rust: stable} - {os: windows-latest, version: 1.12.0, channel: conda-forge, rust: stable} - {os: ubuntu-latest, version: 1.12.1, channel: conda-forge, rust: stable} + - {os: macos-14, version: 1.12.1, mpi: openmpi, channel: conda-forge, rust: beta} - {os: macos-latest, version: 1.14.0, channel: conda-forge, rust: stable} - {os: windows-latest, version: 1.14.0, channel: conda-forge, rust: stable} - {os: ubuntu-latest, version: 1.14.0, channel: conda-forge, rust: stable} From 6d198590bec03915a55c98eea8a4cf1d6a88ba7d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Dec 2025 18:09:51 +0000 Subject: [PATCH 100/141] Bump crate-ci/typos from 1.40.0 to 1.40.1 Bumps [crate-ci/typos](https://github.com/crate-ci/typos) from 1.40.0 to 1.40.1. - [Release notes](https://github.com/crate-ci/typos/releases) - [Changelog](https://github.com/crate-ci/typos/blob/master/CHANGELOG.md) - [Commits](https://github.com/crate-ci/typos/compare/2d0ce569feab1f8752f1dde43cc2f2aa53236e06...1a319b54cc9e3b333fed6a5c88ba1a90324da514) --- updated-dependencies: - dependency-name: crate-ci/typos dependency-version: 1.40.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ad8023bb..142e46fd8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,7 @@ jobs: - name: Checkout code uses: actions/checkout@v6 - name: Check spelling - uses: crate-ci/typos@2d0ce569feab1f8752f1dde43cc2f2aa53236e06 # v1.40.0 + uses: crate-ci/typos@1a319b54cc9e3b333fed6a5c88ba1a90324da514 # v1.40.1 lint: name: lint From 099bd1dd3b746eb8d73dd0e1e4c9a478a31bc97e Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Mon, 19 Jan 2026 20:40:34 +0100 Subject: [PATCH 101/141] Fixing small nits --- CHANGELOG.md | 4 +--- Cargo.toml | 4 ++-- hdf5/Cargo.toml | 4 ++-- hdf5/src/hl/dataset.rs | 1 + hdf5/src/hl/filters.rs | 9 +++++---- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf324d948..b60d0925e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,14 +4,12 @@ ## hdf5-derive unreleased ## hdf5-sys unreleased ## hdf5-src unreleased -## hdf5 unreleased -## hdf5 v0.12.0 +## hdf5 unreleased Released Nov 29, 2025 - Added support for ZFP compression filters as an optional feature - ## hdf5 v0.11.0 Release date: Nov 23, 2025 - Fixed incorrect retrieved name of attributes diff --git a/Cargo.toml b/Cargo.toml index 2542e555f..f8839e1a1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,7 +26,7 @@ num-complex = { version = "0.4", default-features = false } regex = "1.10" # internal -hdf5 = { package = "hdf5-metno", version = "0.12.0", path = "hdf5" } # !V +hdf5 = { package = "hdf5-metno", version = "0.9.3", path = "hdf5" } # !V hdf5-derive = { package = "hdf5-metno-derive", version = "0.9.1", path = "hdf5-derive" } # !V hdf5-src = { package = "hdf5-metno-src", version = "0.9.3", path = "hdf5-src" } # !V hdf5-sys = { package = "hdf5-metno-sys", version = "0.10.1", path = "hdf5-sys" } # !V @@ -48,4 +48,4 @@ overflow-checks = true # keep checks for safety in dev inherits = "dev" opt-level = 0 # keep tests compiling fast debug = 1 # enough info for backtraces -overflow-checks = true \ No newline at end of file +overflow-checks = true diff --git a/hdf5/Cargo.toml b/hdf5/Cargo.toml index debf9bdb0..d142ab1a1 100644 --- a/hdf5/Cargo.toml +++ b/hdf5/Cargo.toml @@ -4,7 +4,7 @@ readme = "../README.md" description = "Thread-safe Rust bindings for the HDF5 library." build = "build.rs" categories = ["science", "filesystem"] -version = "0.12.0" # !V +version = "0.11.0" # !V rust-version.workspace = true authors.workspace = true keywords.workspace = true @@ -56,7 +56,7 @@ errno = { version = "0.3", optional = true } libc = { workspace = true } lzf-sys = { version = "0.1", optional = true } mpi-sys = { workspace = true, optional = true } -ndarray = "0.15.6" +ndarray = ">=0.15, <=0.17" paste = "1.0" zfp-sys = {version= "0.4.2", optional= true } # internal diff --git a/hdf5/src/hl/dataset.rs b/hdf5/src/hl/dataset.rs index d0c0ffa54..8ceca1d96 100644 --- a/hdf5/src/hl/dataset.rs +++ b/hdf5/src/hl/dataset.rs @@ -5,6 +5,7 @@ use std::ops::Deref; use ndarray::{self, ArrayView}; +#[cfg(feature = "zfp")] use crate::hl; #[cfg(feature = "blosc")] use crate::hl::filters::{Blosc, BloscShuffle}; diff --git a/hdf5/src/hl/filters.rs b/hdf5/src/hl/filters.rs index ad323224f..c36875c9e 100644 --- a/hdf5/src/hl/filters.rs +++ b/hdf5/src/hl/filters.rs @@ -1,11 +1,13 @@ use std::collections::HashMap; use std::ptr::{self, addr_of_mut}; +#[cfg(feature = "zfp")] +use crate::globals::{H5E_CALLBACK, H5E_PLIST}; use hdf5_sys::h5p::{ - H5Pget_chunk, H5Pget_filter2, H5Pget_nfilters, H5Pset_deflate, H5Pset_filter, - H5Pset_fletcher32, H5Pset_nbit, H5Pset_scaleoffset, H5Pset_shuffle, H5Pset_szip, + H5Pget_filter2, H5Pget_nfilters, H5Pset_deflate, H5Pset_filter, H5Pset_fletcher32, H5Pset_nbit, + H5Pset_scaleoffset, H5Pset_shuffle, H5Pset_szip, }; -use hdf5_sys::h5t::{H5T_class_t, H5Tclose, H5Tget_class, H5Tget_size, H5T_FLOAT}; +use hdf5_sys::h5t::H5T_class_t; use hdf5_sys::h5z::{ H5Zfilter_avail, H5Zget_filter_info, H5Z_FILTER_CONFIG_DECODE_ENABLED, H5Z_FILTER_CONFIG_ENCODE_ENABLED, H5Z_FILTER_DEFLATE, H5Z_FILTER_FLETCHER32, H5Z_FILTER_NBIT, @@ -156,7 +158,6 @@ mod zfp_impl { } } -use crate::globals::{H5E_CALLBACK, H5E_PLIST}; #[cfg(feature = "zfp")] pub use zfp_impl::*; From b43dfd07d0a19a953479404e905768217cfbc436 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Mon, 19 Jan 2026 21:01:32 +0100 Subject: [PATCH 102/141] Fix indexmap override --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ad8023bb..0ad62c14f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -296,7 +296,7 @@ jobs: - name: Override deps run: | cargo update half@2.7.1 --precise 2.4.1 - cargo update indexmap@2.12.1 --precise 2.11.4 + cargo update indexmap@2.13.0 --precise 2.11.4 - name: Build and test all crates run: cargo test --locked --workspace -vv --features=hdf5-sys/static,hdf5-sys/zlib --exclude=hdf5-metno-derive From db2c1b2da3e859c9ce38ff49c234ef92c0730387 Mon Sep 17 00:00:00 2001 From: Kiran Shila Date: Sun, 30 Nov 2025 13:28:45 -0800 Subject: [PATCH 103/141] feat: HDF5 2.0.0 --- hdf5-src/build.rs | 18 +++------------ hdf5-src/ext/hdf5 | 2 +- hdf5-sys/README.md | 2 +- hdf5-sys/build.rs | 12 ++++++---- hdf5-sys/src/h5fd.rs | 20 ----------------- hdf5-sys/src/h5p.rs | 4 ++-- hdf5/src/globals.rs | 53 +++++++++++++++++++++++++++++++++----------- hdf5/src/hl/group.rs | 9 ++++++-- 8 files changed, 62 insertions(+), 58 deletions(-) diff --git a/hdf5-src/build.rs b/hdf5-src/build.rs index ff14c1d96..93f38b73c 100644 --- a/hdf5-src/build.rs +++ b/hdf5-src/build.rs @@ -74,7 +74,7 @@ fn main() { let mut zlib_header = env::split_paths(&zlib_include_dir).next().unwrap(); zlib_header.push("zlib.h"); let zlib_lib = "z"; - cfg.define("HDF5_ENABLE_Z_LIB_SUPPORT", "ON") + cfg.define("HDF5_ENABLE_ZLIB_SUPPORT", "ON") .define("H5_ZLIB_HEADER", &zlib_header) .define("ZLIB_STATIC_LIBRARY", zlib_lib); println!("cargo::metadata=zlib_header={}", zlib_header.to_str().unwrap()); @@ -96,17 +96,10 @@ fn main() { } let targeting_windows = env::var("CARGO_CFG_TARGET_OS").unwrap() == "windows"; - let debug_postfix = if targeting_windows { "_D" } else { "_debug" }; if feature_enabled("HL") { cfg.define("HDF5_BUILD_HL_LIB", "ON"); - let mut hdf5_hl_lib = - if cfg!(target_env = "msvc") { "libhdf5_hl" } else { "hdf5_hl" }.to_owned(); - if let Ok(opt_level) = env::var("OPT_LEVEL") { - if opt_level == "0" { - hdf5_hl_lib.push_str(debug_postfix); - } - } + let hdf5_hl_lib = if cfg!(target_env = "msvc") { "libhdf5_hl" } else { "hdf5_hl" }.to_owned(); println!("cargo::metadata=hl_library={}", hdf5_hl_lib); } @@ -124,12 +117,7 @@ fn main() { let hdf5_incdir = format!("{}/include", dst.display()); println!("cargo::metadata=include={}", hdf5_incdir); - let mut hdf5_lib = if cfg!(target_env = "msvc") { "libhdf5" } else { "hdf5" }.to_owned(); - if let Ok(opt_level) = env::var("OPT_LEVEL") { - if opt_level == "0" { - hdf5_lib.push_str(debug_postfix); - } - } + let hdf5_lib = if cfg!(target_env = "msvc") { "libhdf5" } else { "hdf5" }.to_owned(); println!("cargo::metadata=library={}", hdf5_lib); } diff --git a/hdf5-src/ext/hdf5 b/hdf5-src/ext/hdf5 index 7bf340440..a6ff8aed2 160000 --- a/hdf5-src/ext/hdf5 +++ b/hdf5-src/ext/hdf5 @@ -1 +1 @@ -Subproject commit 7bf340440909d468dbb3cf41f0ea0d87f5050cea +Subproject commit a6ff8aed236ee1e1deff6415e88b16c42b22f17c diff --git a/hdf5-sys/README.md b/hdf5-sys/README.md index 228e12215..611504f3e 100644 --- a/hdf5-sys/README.md +++ b/hdf5-sys/README.md @@ -9,7 +9,7 @@ This crate supports linking to a static build of HDF5. The HDF5 C library is bui via the `hdf5-src` crate which is then linked in when the `static` feature is set. See below for a list of supported options for static builds. -As of the time of writing, the version of the HDF5 library that is built is 1.10.5, +As of the time of writing, the version of the HDF5 library that is built is 2.0.0, but it may be incremented later. ## Crate features diff --git a/hdf5-sys/build.rs b/hdf5-sys/build.rs index 749284192..1772c0813 100644 --- a/hdf5-sys/build.rs +++ b/hdf5-sys/build.rs @@ -30,7 +30,7 @@ impl Version { } pub fn parse(s: &str) -> Option { - let re = Regex::new(r"^(1)\.(8|10|12|14)\.(\d\d?)(_|.\d+)?((-|.)(patch)?\d+)?$").ok()?; + let re = Regex::new(r"^(1|2)\.(0|8|10|12|14)\.(\d\d?)(_|.\d+)?((-|.)(patch)?\d+)?$").ok()?; let captures = re.captures(s)?; Some(Self { major: captures.get(1).and_then(|c| c.as_str().parse::().ok())?, @@ -53,6 +53,7 @@ impl Debug for Version { fn known_hdf5_versions() -> Vec { // Keep up to date with known_hdf5_versions in hdf5 let mut vs = Vec::new(); + vs.push(Version::new(2,0,0)); // 2.0.0 vs.extend((5..=21).map(|v| Version::new(1, 8, v))); // 1.8.[5-23] vs.extend((0..=8).map(|v| Version::new(1, 10, v))); // 1.10.[0-10] vs.extend((0..=2).map(|v| Version::new(1, 12, v))); // 1.12.[0-2] @@ -334,8 +335,9 @@ mod macos { } // We have to explicitly support homebrew since the HDF5 bottle isn't // packaged with pkg-config metadata. - let (v18, v110, v112, v114) = if let Some(version) = config.version { + let (v20, v18, v110, v112, v114) = if let Some(version) = config.version { ( + version.major == 2 && version.minor == 0, version.major == 1 && version.minor == 8, version.major == 1 && version.minor == 10, version.major == 1 && version.minor == 12, @@ -346,7 +348,9 @@ mod macos { }; println!( "Attempting to find HDF5 via Homebrew ({})...", - if v18 { + if v20 { + "2.0.*" + } else if v18 { "1.8.*" } else if v110 { "1.10.*" @@ -704,7 +708,7 @@ impl Config { for (flag, feature, native) in [ (!h.have_no_deprecated, "deprecated", "HDF5_ENABLE_DEPRECATED_SYMBOLS"), (h.have_threadsafe, "threadsafe", "HDF5_ENABLE_THREADSAFE"), - (h.have_filter_deflate, "zlib", "HDF5_ENABLE_Z_LIB_SUPPORT"), + (h.have_filter_deflate, "zlib", "HDF5_ENABLE_ZLIB_SUPPORT"), ] { if feature_enabled(&feature.to_ascii_uppercase()) { assert!( diff --git a/hdf5-sys/src/h5fd.rs b/hdf5-sys/src/h5fd.rs index 157d247ae..14ab38ac6 100644 --- a/hdf5-sys/src/h5fd.rs +++ b/hdf5-sys/src/h5fd.rs @@ -364,26 +364,6 @@ extern "C" { pub fn H5FDtruncate(file: *mut H5FD_t, dxpl_id: hid_t, closing: hbool_t) -> herr_t; } -// drivers -extern "C" { - pub fn H5FD_sec2_init() -> hid_t; - pub fn H5FD_core_init() -> hid_t; - pub fn H5FD_stdio_init() -> hid_t; - pub fn H5FD_family_init() -> hid_t; - pub fn H5FD_log_init() -> hid_t; - pub fn H5FD_multi_init() -> hid_t; -} - -#[cfg(feature = "have-parallel")] -extern "C" { - pub fn H5FD_mpio_init() -> hid_t; -} - -#[cfg(feature = "have-direct")] -extern "C" { - pub fn H5FD_direct_init() -> hid_t; -} - #[cfg(feature = "1.10.0")] extern "C" { pub fn H5FDlock(file: *mut H5FD_t, rw: hbool_t) -> herr_t; diff --git a/hdf5-sys/src/h5p.rs b/hdf5-sys/src/h5p.rs index b6e46744e..5ceeefc91 100644 --- a/hdf5-sys/src/h5p.rs +++ b/hdf5-sys/src/h5p.rs @@ -526,11 +526,11 @@ extern "C" { ) -> herr_t; // mpi-io - #[cfg(feature = "mpio")] + #[cfg(feature = "have-parallel")] pub fn H5Pset_fapl_mpio( fapl_id: hid_t, comm: mpi_sys::MPI_Comm, info: mpi_sys::MPI_Info, ) -> herr_t; - #[cfg(feature = "mpio")] + #[cfg(feature = "have-parallel")] pub fn H5Pget_fapl_mpio( fapl_id: hid_t, comm: *mut mpi_sys::MPI_Comm, info: *mut mpi_sys::MPI_Info, ) -> herr_t; diff --git a/hdf5/src/globals.rs b/hdf5/src/globals.rs index 292a2a562..6ddb064ea 100644 --- a/hdf5/src/globals.rs +++ b/hdf5/src/globals.rs @@ -4,12 +4,12 @@ use std::mem; use std::sync::LazyLock; #[cfg(feature = "have-direct")] -use hdf5_sys::h5fd::H5FD_direct_init; +use hdf5_sys::h5fp::H5Pset_fapl_direct; #[cfg(feature = "have-parallel")] -use hdf5_sys::h5fd::H5FD_mpio_init; -use hdf5_sys::h5fd::{ - H5FD_core_init, H5FD_family_init, H5FD_log_init, H5FD_multi_init, H5FD_sec2_init, - H5FD_stdio_init, +use hdf5_sys::h5p::H5Pset_fapl_mpio; +use hdf5_sys::h5p::{ + H5Pclose, H5Pcreate, H5Pget_driver, H5Pset_fapl_core, H5Pset_fapl_family, H5Pset_fapl_log, + H5Pset_fapl_multi, H5Pset_fapl_sec2, H5Pset_fapl_stdio, }; use hdf5_sys::{h5e, h5p, h5t}; @@ -44,6 +44,18 @@ macro_rules! link_hid { }; } +/// Fetches the driver ID using the workaround from https://github.com/HDFGroup/hdf5/issues/1809 +/// as the _init functions seem to be removed in HDF5 2.0.0 +macro_rules! get_driver { + ($set_driver:expr) => {{ + let fapl = h5call!(H5Pcreate(*H5P_FILE_ACCESS)).expect("should always create FAPL"); + h5call!($set_driver(fapl)).expect("should always be able to set the driver"); + let id = h5call!(H5Pget_driver(fapl)).expect("should always be able to extract the driver"); + h5call!(H5Pclose(fapl)).expect("should always be able to close the FAPL"); + id + }}; +} + // Datatypes link_hid!(H5T_IEEE_F32BE, h5t::H5T_IEEE_F32BE); link_hid!(H5T_IEEE_F32LE, h5t::H5T_IEEE_F32LE); @@ -328,22 +340,37 @@ pub static H5R_DSET_REG_REF_BUF_SIZE: LazyLock = LazyLock::new(|| mem::size_of::() + 4); // File drivers -pub static H5FD_CORE: LazyLock = LazyLock::new(|| h5lock!(H5FD_core_init())); -pub static H5FD_SEC2: LazyLock = LazyLock::new(|| h5lock!(H5FD_sec2_init())); -pub static H5FD_STDIO: LazyLock = LazyLock::new(|| h5lock!(H5FD_stdio_init())); -pub static H5FD_FAMILY: LazyLock = LazyLock::new(|| h5lock!(H5FD_family_init())); -pub static H5FD_LOG: LazyLock = LazyLock::new(|| h5lock!(H5FD_log_init())); -pub static H5FD_MULTI: LazyLock = LazyLock::new(|| h5lock!(H5FD_multi_init())); +pub static H5FD_CORE: LazyLock = + LazyLock::new(|| h5lock!(get_driver!(|fapl| H5Pset_fapl_core(fapl, 0, 0)))); +pub static H5FD_SEC2: LazyLock = + LazyLock::new(|| h5lock!(get_driver!(|fapl| H5Pset_fapl_sec2(fapl)))); +pub static H5FD_STDIO: LazyLock = + LazyLock::new(|| h5lock!(get_driver!(|fapl| H5Pset_fapl_stdio(fapl)))); +pub static H5FD_FAMILY: LazyLock = + LazyLock::new(|| h5lock!(get_driver!(|fapl| H5Pset_fapl_family(fapl, 0, 0)))); +pub static H5FD_LOG: LazyLock = + LazyLock::new(|| h5lock!(get_driver!(|fapl| H5Pset_fapl_log(fapl, std::ptr::null(), 0, 0)))); +pub static H5FD_MULTI: LazyLock = LazyLock::new(|| { + h5lock!(get_driver!(|fapl| H5Pset_fapl_multi( + fapl, + std::ptr::null(), + std::ptr::null(), + std::ptr::null(), + std::ptr::null(), + 0 + ))) +}); // MPI-IO file driver #[cfg(feature = "have-parallel")] -pub static H5FD_MPIO: LazyLock = LazyLock::new(|| h5lock!(H5FD_mpio_init())); +pub static H5FD_MPIO: LazyLock = LazyLock::new(|| h5lock!(todo!())); #[cfg(not(feature = "have-parallel"))] pub static H5FD_MPIO: LazyLock = LazyLock::new(|| H5I_INVALID_HID); // Direct VFD #[cfg(feature = "have-direct")] -pub static H5FD_DIRECT: LazyLock = LazyLock::new(|| h5lock!(H5FD_direct_init())); +pub static H5FD_DIRECT: LazyLock = + LazyLock::new(|| h5lock!(get_driver!(|fapl| H5Pset_fapl_direct(fapl, 0, 0, 0)))); #[cfg(not(feature = "have-direct"))] pub static H5FD_DIRECT: LazyLock = LazyLock::new(|| H5I_INVALID_HID); diff --git a/hdf5/src/hl/group.rs b/hdf5/src/hl/group.rs index 32e57ec43..66c2e1bec 100644 --- a/hdf5/src/hl/group.rs +++ b/hdf5/src/hl/group.rs @@ -622,9 +622,14 @@ pub mod tests { file.new_dataset::().no_chunk().shape((10, 20)).create("a/foo").unwrap(); file.new_dataset::().no_chunk().shape((10, 20)).create("a/123").unwrap(); file.new_dataset::().no_chunk().shape((10, 20)).create("a/bar").unwrap(); - assert_eq!(group_a.member_names().unwrap(), vec!["123", "bar", "foo"]); + let group_a_names = group_a.member_names().unwrap(); + assert!(group_a_names.contains(&"123".to_string())); + assert!(group_a_names.contains(&"bar".to_string())); + assert!(group_a_names.contains(&"foo".to_string())); assert_eq!(group_b.member_names().unwrap().len(), 0); - assert_eq!(file.member_names().unwrap(), vec!["a", "b"]); + let file_names = file.member_names().unwrap(); + assert!(file_names.contains(&"a".to_string())); + assert!(file_names.contains(&"b".to_string())); }) } From 4d66fb5bb29e3705b44171650c86fa375c2a73bd Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Tue, 16 Dec 2025 20:42:16 +0100 Subject: [PATCH 104/141] cargo fmt --- hdf5-src/build.rs | 3 ++- hdf5-sys/build.rs | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/hdf5-src/build.rs b/hdf5-src/build.rs index 93f38b73c..e22638a9b 100644 --- a/hdf5-src/build.rs +++ b/hdf5-src/build.rs @@ -99,7 +99,8 @@ fn main() { if feature_enabled("HL") { cfg.define("HDF5_BUILD_HL_LIB", "ON"); - let hdf5_hl_lib = if cfg!(target_env = "msvc") { "libhdf5_hl" } else { "hdf5_hl" }.to_owned(); + let hdf5_hl_lib = + if cfg!(target_env = "msvc") { "libhdf5_hl" } else { "hdf5_hl" }.to_owned(); println!("cargo::metadata=hl_library={}", hdf5_hl_lib); } diff --git a/hdf5-sys/build.rs b/hdf5-sys/build.rs index 1772c0813..924416151 100644 --- a/hdf5-sys/build.rs +++ b/hdf5-sys/build.rs @@ -30,7 +30,8 @@ impl Version { } pub fn parse(s: &str) -> Option { - let re = Regex::new(r"^(1|2)\.(0|8|10|12|14)\.(\d\d?)(_|.\d+)?((-|.)(patch)?\d+)?$").ok()?; + let re = + Regex::new(r"^(1|2)\.(0|8|10|12|14)\.(\d\d?)(_|.\d+)?((-|.)(patch)?\d+)?$").ok()?; let captures = re.captures(s)?; Some(Self { major: captures.get(1).and_then(|c| c.as_str().parse::().ok())?, @@ -53,7 +54,7 @@ impl Debug for Version { fn known_hdf5_versions() -> Vec { // Keep up to date with known_hdf5_versions in hdf5 let mut vs = Vec::new(); - vs.push(Version::new(2,0,0)); // 2.0.0 + vs.push(Version::new(2, 0, 0)); // 2.0.0 vs.extend((5..=21).map(|v| Version::new(1, 8, v))); // 1.8.[5-23] vs.extend((0..=8).map(|v| Version::new(1, 10, v))); // 1.10.[0-10] vs.extend((0..=2).map(|v| Version::new(1, 12, v))); // 1.12.[0-2] From 08e50eb31104fb3ad8c371c634851a25ee76baa9 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Tue, 16 Dec 2025 20:42:48 +0100 Subject: [PATCH 105/141] Fix brew builds --- hdf5-sys/build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hdf5-sys/build.rs b/hdf5-sys/build.rs index 924416151..2e56f6446 100644 --- a/hdf5-sys/build.rs +++ b/hdf5-sys/build.rs @@ -345,7 +345,7 @@ mod macos { version.major == 1 && version.minor == 14, ) } else { - (false, false, false, false) + (false, false, false, false, false) }; println!( "Attempting to find HDF5 via Homebrew ({})...", From 64519383d6eb99ebf88a64ed6038ce0048797a8e Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Tue, 16 Dec 2025 20:44:20 +0100 Subject: [PATCH 106/141] Fix feature gate for mpio --- hdf5-sys/src/h5p.rs | 4 ++-- hdf5/src/globals.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hdf5-sys/src/h5p.rs b/hdf5-sys/src/h5p.rs index 5ceeefc91..13146a797 100644 --- a/hdf5-sys/src/h5p.rs +++ b/hdf5-sys/src/h5p.rs @@ -526,11 +526,11 @@ extern "C" { ) -> herr_t; // mpi-io - #[cfg(feature = "have-parallel")] + #[cfg(all(feature = "have-parallel", feature = "mpio"))] pub fn H5Pset_fapl_mpio( fapl_id: hid_t, comm: mpi_sys::MPI_Comm, info: mpi_sys::MPI_Info, ) -> herr_t; - #[cfg(feature = "have-parallel")] + #[cfg(all(feature = "have-parallel", feature = "mpio"))] pub fn H5Pget_fapl_mpio( fapl_id: hid_t, comm: *mut mpi_sys::MPI_Comm, info: *mut mpi_sys::MPI_Info, ) -> herr_t; diff --git a/hdf5/src/globals.rs b/hdf5/src/globals.rs index 6ddb064ea..3c1516409 100644 --- a/hdf5/src/globals.rs +++ b/hdf5/src/globals.rs @@ -5,7 +5,7 @@ use std::sync::LazyLock; #[cfg(feature = "have-direct")] use hdf5_sys::h5fp::H5Pset_fapl_direct; -#[cfg(feature = "have-parallel")] +#[cfg(all(feature = "have-parallel", feature = "mpio"))] use hdf5_sys::h5p::H5Pset_fapl_mpio; use hdf5_sys::h5p::{ H5Pclose, H5Pcreate, H5Pget_driver, H5Pset_fapl_core, H5Pset_fapl_family, H5Pset_fapl_log, From ef3a0d62314fa05a69a6e77f7f9545fc8ad58235 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Tue, 16 Dec 2025 21:51:45 +0100 Subject: [PATCH 107/141] Fix MPIO driver --- hdf5-sys/src/h5fd.rs | 23 +++++++++++++++++++++++ hdf5/build.rs | 1 + hdf5/src/globals.rs | 15 +++++++++------ hdf5/tests/test_plist.rs | 2 ++ 4 files changed, 35 insertions(+), 6 deletions(-) diff --git a/hdf5-sys/src/h5fd.rs b/hdf5-sys/src/h5fd.rs index 14ab38ac6..628359858 100644 --- a/hdf5-sys/src/h5fd.rs +++ b/hdf5-sys/src/h5fd.rs @@ -364,6 +364,11 @@ extern "C" { pub fn H5FDtruncate(file: *mut H5FD_t, dxpl_id: hid_t, closing: hbool_t) -> herr_t; } +#[cfg(all(not(feature = "2.0.0"), feature = "have-parallel"))] +extern "C" { + pub fn H5FD_mpio_init() -> hid_t; +} + #[cfg(feature = "1.10.0")] extern "C" { pub fn H5FDlock(file: *mut H5FD_t, rw: hbool_t) -> herr_t; @@ -467,3 +472,21 @@ extern "C" { pub fn H5FDis_driver_registered_by_value(driver_value: H5FD_class_value_t) -> htri_t; pub fn H5FDperform_init(p: H5FD_perform_init_func_t) -> hid_t; } + +#[cfg(all(feature = "2.0.0", not(all(target_env = "msvc", not(feature = "static")))))] +mod globals_2_0_0 { + pub use crate::h5i::hid_t as id_t; + #[cfg(feature = "have_parallel")] + extern_static!(H5FD_MPIO, H5FD_MPIO_id_g); +} + +#[cfg(all(feature = "2.0.0", all(target_env = "msvc", not(feature = "static"))))] +mod globals { + // dllimport hack + pub type id_t = usize; + #[cfg(feature = "have_parallel")] + extern_static!(H5FD_MPIO_id, __imp_H5FD_MPIO_id_g); +} + +#[cfg(feature = "2.0.0")] +use globals_2_0_0::*; diff --git a/hdf5/build.rs b/hdf5/build.rs index 52557f648..74deb6275 100644 --- a/hdf5/build.rs +++ b/hdf5/build.rs @@ -16,6 +16,7 @@ impl Version { fn known_hdf5_versions() -> Vec { // Keep up to date with known_hdf5_versions in hdf5-sys let mut vs = Vec::new(); + vs.push(Version::new(2, 0, 0)); // 2.0.0 vs.extend((5..=21).map(|v| Version::new(1, 8, v))); // 1.8.[5-23] vs.extend((0..=8).map(|v| Version::new(1, 10, v))); // 1.10.[0-10] vs.extend((0..=2).map(|v| Version::new(1, 12, v))); // 1.12.[0-2] diff --git a/hdf5/src/globals.rs b/hdf5/src/globals.rs index 3c1516409..729db9151 100644 --- a/hdf5/src/globals.rs +++ b/hdf5/src/globals.rs @@ -4,9 +4,7 @@ use std::mem; use std::sync::LazyLock; #[cfg(feature = "have-direct")] -use hdf5_sys::h5fp::H5Pset_fapl_direct; -#[cfg(all(feature = "have-parallel", feature = "mpio"))] -use hdf5_sys::h5p::H5Pset_fapl_mpio; +use hdf5_sys::h5p::H5Pset_fapl_direct; use hdf5_sys::h5p::{ H5Pclose, H5Pcreate, H5Pget_driver, H5Pset_fapl_core, H5Pset_fapl_family, H5Pset_fapl_log, H5Pset_fapl_multi, H5Pset_fapl_sec2, H5Pset_fapl_stdio, @@ -362,9 +360,14 @@ pub static H5FD_MULTI: LazyLock = LazyLock::new(|| { }); // MPI-IO file driver -#[cfg(feature = "have-parallel")] -pub static H5FD_MPIO: LazyLock = LazyLock::new(|| h5lock!(todo!())); -#[cfg(not(feature = "have-parallel"))] +#[cfg(all(feature = "2.0.0", all(feature = "have-parallel", feature = "mpio")))] +pub static H5FD_MPIO: LazyLock = LazyLock::new(|| *hdf5_sys::h5p::H5FD_MPIO_id); +#[cfg(all(feature = "2.0.0", not(all(feature = "have-parallel", feature = "mpio"))))] +pub static H5FD_MPIO: LazyLock = LazyLock::new(|| H5I_INVALID_HID); + +#[cfg(all(not(feature = "2.0.0"), all(feature = "have-parallel", feature = "mpio")))] +pub static H5FD_MPIO: LazyLock = LazyLock::new(|| h5lock!(hdf5_sys::h5fd::H5FD_mpio_init())); +#[cfg(all(not(feature = "2.0.0"), not(all(feature = "have-parallel", feature = "mpio"))))] pub static H5FD_MPIO: LazyLock = LazyLock::new(|| H5I_INVALID_HID); // Direct VFD diff --git a/hdf5/tests/test_plist.rs b/hdf5/tests/test_plist.rs index e5c960703..07db1f9d8 100644 --- a/hdf5/tests/test_plist.rs +++ b/hdf5/tests/test_plist.rs @@ -329,6 +329,8 @@ fn test_fapl_driver_mpio() -> hdf5::Result<()> { let mut b = FileAccess::build(); b.mpio(world_comm, None); + let driver = b.finish()?.get_driver()?; + println!("{:?}", driver); let d = check_matches!(b.finish()?.get_driver()?, d, FileDriver::Mpio(d)); let mut cmp = mem::MaybeUninit::uninit(); unsafe { MPI_Comm_compare(d.comm, world_comm, cmp.as_mut_ptr()) }; From 9730522563ab9590bbb72b363c791e0a21c7284c Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Wed, 17 Dec 2025 08:32:41 +0100 Subject: [PATCH 108/141] Add changelog --- CHANGELOG.md | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b60d0925e..12f8af4fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,14 +1,21 @@ # Changelog +## hdf5 unreleased +- Add support for hdf5 2.0.0 +- Added support for ZFP compression filters as an optional feature + ## hdf5-types unreleased +- Add support for hdf5 2.0.0 + ## hdf5-derive unreleased -## hdf5-sys unreleased -## hdf5-src unreleased +- Add support for hdf5 2.0.0 +## hdf5-sys unreleased +- Add support for hdf5 2.0.0 +- Remove `H5FD_*_init` functions, these are private to hdf5-c -## hdf5 unreleased -Released Nov 29, 2025 -- Added support for ZFP compression filters as an optional feature +## hdf5-src unreleased +- Use hdf5 2.0.0 ## hdf5 v0.11.0 Release date: Nov 23, 2025 From b84ce4280b9e50baced823343b2bbce9aafd96df Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Mon, 22 Dec 2025 19:42:27 +0100 Subject: [PATCH 109/141] Fix compatability with previous versions --- hdf5-sys/src/h5d.rs | 12 ++++++- hdf5-sys/src/h5e.rs | 8 +++++ hdf5-sys/src/h5f.rs | 10 +++++- hdf5-sys/src/h5fd.rs | 23 ++++++++++++-- hdf5-sys/src/h5i.rs | 10 +++++- hdf5-sys/src/h5p.rs | 6 ++++ hdf5-sys/src/h5t.rs | 74 +++++++++++++++++++++++++++++++++++++++++++- hdf5-sys/src/h5vl.rs | 30 +++++++++++++++++- 8 files changed, 165 insertions(+), 8 deletions(-) diff --git a/hdf5-sys/src/h5d.rs b/hdf5-sys/src/h5d.rs index 7b72b4b29..fef3cd4fd 100644 --- a/hdf5-sys/src/h5d.rs +++ b/hdf5-sys/src/h5d.rs @@ -240,14 +240,24 @@ pub use self::hdf5_1_10_0::*; #[cfg(feature = "1.10.3")] extern "C" { - pub fn H5Dread_chunk( + #[cfg_attr(not(feature = "2.0.0"), link_name = "H5Dread_chunk")] + pub fn H5Dread_chunk1( dset_id: hid_t, dxpl_id: hid_t, offset: *const hsize_t, filters: *mut u32, buf: *mut c_void, ) -> herr_t; + #[cfg(feature = "2.0.0")] + pub fn H5Dread_chunk2( + dset_id: hid_t, dxpl_id: hid_t, offset: *const hsize_t, filters: *mut u32, + buf: *mut c_void, buf_size: *mut size_t, + ) -> herr_t; pub fn H5Dwrite_chunk( dset_id: hid_t, dxpl_id: hid_t, filters: u32, offset: *const hsize_t, data_size: size_t, buf: *const c_void, ) -> herr_t; } +#[cfg(all(feature = "1.10.3", not(feature = "2.0.0")))] +pub use self::H5Dread_chunk1 as H5Dread_chunk; +#[cfg(feature = "2.0.0")] +pub use self::H5Dread_chunk2 as H5Dread_chunk; #[cfg(feature = "1.10.5")] extern "C" { diff --git a/hdf5-sys/src/h5e.rs b/hdf5-sys/src/h5e.rs index 8e9399d32..6d60ed033 100644 --- a/hdf5-sys/src/h5e.rs +++ b/hdf5-sys/src/h5e.rs @@ -323,6 +323,10 @@ mod globals { extern_static!(H5E_ID, H5E_ID_g); #[cfg(feature = "1.14.0")] extern_static!(H5E_UNMOUNT, H5E_UNMOUNT_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5E_RTREE, H5E_RTREE_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5E_THREADSAFE, H5E_THREADSAFE_g); } #[cfg(all(target_env = "msvc", not(feature = "static")))] @@ -505,4 +509,8 @@ mod globals { extern_static!(H5E_ID, __imp_H5E_ID_g); #[cfg(feature = "1.14.0")] extern_static!(H5E_UNMOUNT, __imp_H5E_UNMOUNT_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5E_RTREE, __imp_H5E_RTREE_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5E_THREADSAFE, __imp_H5E_THREADSAFE_g); } diff --git a/hdf5-sys/src/h5f.rs b/hdf5-sys/src/h5f.rs index a41fdf8ac..3a27780a5 100644 --- a/hdf5-sys/src/h5f.rs +++ b/hdf5-sys/src/h5f.rs @@ -125,11 +125,19 @@ pub enum H5F_libver_t { H5F_LIBVER_V112 = 3, #[cfg(feature = "1.14.0")] H5F_LIBVER_V114 = 4, + #[cfg(feature = "2.0.0")] + H5F_LIBVER_V200 = 5, H5F_LIBVER_NBOUNDS, } -#[cfg(feature = "1.10.2")] +#[cfg(all(feature = "1.10.2", not(feature = "1.12.0")))] pub const H5F_LIBVER_LATEST: H5F_libver_t = H5F_LIBVER_V110; +#[cfg(all(feature = "1.12.0", not(feature = "1.14.0")))] +pub const H5F_LIBVER_LATEST: H5F_libver_t = H5F_LIBVER_V112; +#[cfg(all(feature = "1.14.0", not(feature = "2.0.0")))] +pub const H5F_LIBVER_LATEST: H5F_libver_t = H5F_LIBVER_V114; +#[cfg(feature = "2.0.0")] +pub const H5F_LIBVER_LATEST: H5F_libver_t = H5F_LIBVER_V200; impl Default for H5F_libver_t { fn default() -> Self { diff --git a/hdf5-sys/src/h5fd.rs b/hdf5-sys/src/h5fd.rs index 628359858..64e450f98 100644 --- a/hdf5-sys/src/h5fd.rs +++ b/hdf5-sys/src/h5fd.rs @@ -470,12 +470,21 @@ extern "C" { pub fn H5FDdelete(name: *const c_char, fapl_id: hid_t) -> herr_t; pub fn H5FDis_driver_registered_by_name(driver_name: *const c_char) -> htri_t; pub fn H5FDis_driver_registered_by_value(driver_value: H5FD_class_value_t) -> htri_t; + #[cfg(not(feature = "2.0.0"))] pub fn H5FDperform_init(p: H5FD_perform_init_func_t) -> hid_t; } #[cfg(all(feature = "2.0.0", not(all(target_env = "msvc", not(feature = "static")))))] mod globals_2_0_0 { pub use crate::h5i::hid_t as id_t; + extern_static!(H5FD_CORE, H5FD_CORE_id_g); + extern_static!(H5FD_FAMILY, H5FD_FAMILY_id_g); + extern_static!(H5FD_LOG, H5FD_LOG_id_g); + extern_static!(H5FD_MULTI, H5FD_MULTI_id_g); + extern_static!(H5FD_ONION, H5FD_ONION_id_g); + extern_static!(H5FD_SEC2, H5FD_SEC2_id_g); + extern_static!(H5FD_SPLITTER, H5FD_SPLITTER_id_g); + extern_static!(H5FD_STDIO, H5FD_STDIO_id_g); #[cfg(feature = "have_parallel")] extern_static!(H5FD_MPIO, H5FD_MPIO_id_g); } @@ -484,9 +493,17 @@ mod globals_2_0_0 { mod globals { // dllimport hack pub type id_t = usize; - #[cfg(feature = "have_parallel")] - extern_static!(H5FD_MPIO_id, __imp_H5FD_MPIO_id_g); + extern_static!(H5FD_CORE, __imp_H5FD_CORE_id_g); + extern_static!(H5FD_FAMILY, __imp_H5FD_FAMILY_id_g); + extern_static!(H5FD_LOG, __imp_H5FD_LOG_id_g); + extern_static!(H5FD_MULTI, __imp_H5FD_MULTI_id_g); + extern_static!(H5FD_ONION, __imp_H5FD_ONION_id_g); + extern_static!(H5FD_SEC2, __imp_H5FD_SEC2_id_g); + extern_static!(H5FD_SPLITTER, __imp_H5FD_SPLITTER_id_g); + extern_static!(H5FD_STDIO, __imp_H5FD_STDIO_id_g); + #[cfg(feature = "have-parallel")] + extern_static!(H5FD_MPIO, __imp_H5FD_MPIO_id_g); } #[cfg(feature = "2.0.0")] -use globals_2_0_0::*; +pub use globals_2_0_0::*; diff --git a/hdf5-sys/src/h5i.rs b/hdf5-sys/src/h5i.rs index 9fb1578a8..6423cd3b8 100644 --- a/hdf5-sys/src/h5i.rs +++ b/hdf5-sys/src/h5i.rs @@ -62,9 +62,12 @@ extern "C" { pub fn H5Iinc_ref(id: hid_t) -> c_int; pub fn H5Idec_ref(id: hid_t) -> c_int; pub fn H5Iget_ref(id: hid_t) -> c_int; - pub fn H5Iregister_type( + #[cfg_attr(not(feature = "2.0.0"), link_name = "H5Iregister_type")] + pub fn H5Iregister_type1( hash_size: size_t, reserved: c_uint, free_func: H5I_free_t, ) -> H5I_type_t; + #[cfg(feature = "2.0.0")] + pub fn H5Iregister_type2(reserved: c_uint, free_func: H5I_free_t) -> H5I_type_t; pub fn H5Iclear_type(type_: H5I_type_t, force: hbool_t) -> herr_t; pub fn H5Idestroy_type(type_: H5I_type_t) -> herr_t; pub fn H5Iinc_type_ref(type_: H5I_type_t) -> c_int; @@ -78,6 +81,11 @@ extern "C" { pub fn H5Iis_valid(id: hid_t) -> htri_t; } +#[cfg(not(feature = "2.0.0"))] +pub use self::H5Iregister_type1 as H5Iregister_type; +#[cfg(feature = "2.0.0")] +pub use self::H5Iregister_type2 as H5Iregister_type; + #[cfg(feature = "1.14.0")] pub type H5I_future_realize_func_t = Option< unsafe extern "C" fn(future_object: *mut c_void, actual_object_id: *mut hid_t) -> herr_t, diff --git a/hdf5-sys/src/h5p.rs b/hdf5-sys/src/h5p.rs index 13146a797..90fa123b0 100644 --- a/hdf5-sys/src/h5p.rs +++ b/hdf5-sys/src/h5p.rs @@ -811,3 +811,9 @@ extern "C" { plist_id: hid_t, driver_value: H5FD_class_value_t, driver_config: *const c_char, ) -> herr_t; } + +#[cfg(feature = "2.0.0")] +extern "C" { + pub fn H5Pget_virtual_spatial_tree(dcpl_id: hid_t, use_tree: *mut hbool_t) -> herr_t; + pub fn H5Pset_virtual_spatial_tree(dapl_id: hid_t, use_tree: hbool_t) -> herr_t; +} diff --git a/hdf5-sys/src/h5t.rs b/hdf5-sys/src/h5t.rs index da2e6934f..d5221cb28 100644 --- a/hdf5-sys/src/h5t.rs +++ b/hdf5-sys/src/h5t.rs @@ -36,7 +36,14 @@ pub enum H5T_class_t { H5T_ENUM = 8, H5T_VLEN = 9, H5T_ARRAY = 10, + + #[cfg(not(feature = "2.0.0"))] H5T_NCLASSES = 11, + + #[cfg(feature = "2.0.0")] + H5T_COMPLEX = 11, + #[cfg(feature = "2.0.0")] + H5T_NCLASSES = 12, } #[cfg(feature = "1.8.6")] @@ -262,7 +269,10 @@ extern "C" { pub fn H5Tget_create_plist(type_id: hid_t) -> hid_t; pub fn H5Tcommitted(type_id: hid_t) -> htri_t; pub fn H5Tencode(obj_id: hid_t, buf: *mut c_void, nalloc: *mut size_t) -> herr_t; - pub fn H5Tdecode(buf: *const c_void) -> hid_t; + #[cfg_attr(not(feature = "2.0.0"), link_name = "H5Tdecode")] + pub fn H5Tdecode1(buf: *const c_void) -> hid_t; + #[cfg(feature = "2.0.0")] + pub fn H5Tdecode2(buf: *const c_void, buf_size: size_t) -> hid_t; pub fn H5Tinsert( parent_id: hid_t, name: *const c_char, offset: size_t, member_id: hid_t, ) -> herr_t; @@ -344,6 +354,11 @@ extern "C" { pub fn H5Tget_array_dims1(type_id: hid_t, dims: *mut hsize_t, perm: *mut c_int) -> c_int; } +#[cfg(not(feature = "2.0.0"))] +pub use self::H5Tdecode1 as H5Tdecode; +#[cfg(feature = "2.0.0")] +pub use self::H5Tdecode2 as H5Tdecode; + pub use self::globals::*; #[cfg(not(all(target_env = "msvc", not(feature = "static"))))] @@ -438,6 +453,32 @@ mod globals { extern_static!(H5T_NATIVE_UINT_FAST64, H5T_NATIVE_UINT_FAST64_g); #[cfg(feature = "1.12.0")] extern_static!(H5T_STD_REF, H5T_STD_REF_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_COMPLEX_IEEE_F16BE, H5T_COMPLEX_IEEE_F16BE_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_COMPLEX_IEEE_F16LE, H5T_COMPLEX_IEEE_F16LE_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_COMPLEX_IEEE_F32BE, H5T_COMPLEX_IEEE_F32BE_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_COMPLEX_IEEE_F32LE, H5T_COMPLEX_IEEE_F32LE_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_COMPLEX_IEEE_F64BE, H5T_COMPLEX_IEEE_F64BE_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_COMPLEX_IEEE_F64LE, H5T_COMPLEX_IEEE_F64LE_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_FLOAT_BFLOAT16BE, H5T_FLOAT_BFLOAT16BE_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_FLOAT_BFLOAT16LE, H5T_FLOAT_BFLOAT16LE_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_FLOAT_F8E4M3, H5T_FLOAT_F8E4M3_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_FLOAT_F8E5M2, H5T_FLOAT_F8E5M2_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_NATIVE_DOUBLE_COMPLEX, H5T_NATIVE_DOUBLE_COMPLEX_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_NATIVE_FLOAT_COMPLEX, H5T_NATIVE_FLOAT_COMPLEX_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_NATIVE_LDOUBLE_COMPLEX, H5T_NATIVE_LDOUBLE_COMPLEX_g); } #[cfg(all(target_env = "msvc", not(feature = "static")))] @@ -533,6 +574,32 @@ mod globals { extern_static!(H5T_NATIVE_UINT_FAST64, __imp_H5T_NATIVE_UINT_FAST64_g); #[cfg(feature = "1.12.0")] extern_static!(H5T_STD_REF, __imp_H5T_STD_REF_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_COMPLEX_IEEE_F16BE, __imp_H5T_COMPLEX_IEEE_F16BE_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_COMPLEX_IEEE_F16LE, __imp_H5T_COMPLEX_IEEE_F16LE_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_COMPLEX_IEEE_F32BE, __imp_H5T_COMPLEX_IEEE_F32BE_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_COMPLEX_IEEE_F32LE, __imp_H5T_COMPLEX_IEEE_F32LE_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_COMPLEX_IEEE_F64BE, __imp_H5T_COMPLEX_IEEE_F64BE_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_COMPLEX_IEEE_F64LE, __imp_H5T_COMPLEX_IEEE_F64LE_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_FLOAT_BFLOAT16BE, __imp_H5T_FLOAT_BFLOAT16BE_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_FLOAT_BFLOAT16LE, __imp_H5T_FLOAT_BFLOAT16LE_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_FLOAT_F8E4M3, __imp_H5T_FLOAT_F8E4M3_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_FLOAT_F8E5M2, __imp_H5T_FLOAT_F8E5M2_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_NATIVE_DOUBLE_COMPLEX, __imp_H5T_NATIVE_DOUBLE_COMPLEX_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_NATIVE_FLOAT_COMPLEX, __imp_H5T_NATIVE_FLOAT_COMPLEX_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_NATIVE_LDOUBLE_COMPLEX, __imp_H5T_NATIVE_LDOUBLE_COMPLEX_g); } #[cfg(feature = "1.10.0")] @@ -562,3 +629,8 @@ extern "C" { name: *const c_char, tapl_id: hid_t, es_id: hid_t, ) -> hid_t; } + +#[cfg(feature = "2.0.0")] +extern "C" { + pub fn H5Tcomplex_create(base_type_id: hid_t) -> hid_t; +} diff --git a/hdf5-sys/src/h5vl.rs b/hdf5-sys/src/h5vl.rs index 56446df7d..2f85aa90a 100644 --- a/hdf5-sys/src/h5vl.rs +++ b/hdf5-sys/src/h5vl.rs @@ -1923,14 +1923,42 @@ mod v1_14_0 { } extern "C" { + #[cfg(not(feature = "2.0.0"))] + pub fn H5VLstart_lib_state() -> herr_t; + #[cfg(not(feature = "2.0.0"))] pub fn H5VLfinish_lib_state() -> herr_t; pub fn H5VLintrospect_get_cap_flags( info: *const c_void, connector_id: hid_t, cap_flags: *mut c_uint, ) -> herr_t; - pub fn H5VLstart_lib_state() -> herr_t; } extern "C" { pub fn H5VLobject_is_native(obj_id: hid_t, is_native: *mut hbool_t) -> herr_t; } } + +#[cfg(feature = "2.0.0")] +extern "C" { + pub fn H5VLclose_lib_context(context: *mut c_void) -> herr_t; + pub fn H5VLopen_lib_context(context: *mut *mut c_void) -> herr_t; +} + +#[cfg(not(all(target_env = "msvc", not(feature = "static"))))] +#[cfg(feature = "2.0.0")] +mod globals { + pub use crate::h5i::hid_t as id_t; + extern_static!(H5VL_NATIVE, H5VL_NATIVE_g); + extern_static!(H5VL_PASSTHRU, H5VL_PASSTHRU_g); +} + +#[cfg(all(target_env = "msvc", not(feature = "static")))] +#[cfg(feature = "2.0.0")] +mod globals { + // dllimport hack + pub type id_t = usize; + extern_static!(H5VL_NATIVE, __imp_H5VL_NATIVE_g); + extern_static!(H5VL_PASSTHRU, __imp_H5VL_PASSTHRU_g); +} + +#[cfg(feature = "2.0.0")] +pub use globals::*; From 4f91a3684ce1ac4a3b90c1c590b38f910a8ee0eb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Jan 2026 20:26:09 +0000 Subject: [PATCH 110/141] Bump crate-ci/typos from 1.40.1 to 1.42.1 Bumps [crate-ci/typos](https://github.com/crate-ci/typos) from 1.40.1 to 1.42.1. - [Release notes](https://github.com/crate-ci/typos/releases) - [Changelog](https://github.com/crate-ci/typos/blob/master/CHANGELOG.md) - [Commits](https://github.com/crate-ci/typos/compare/1a319b54cc9e3b333fed6a5c88ba1a90324da514...65120634e79d8374d1aa2f27e54baa0c364fff5a) --- updated-dependencies: - dependency-name: crate-ci/typos dependency-version: 1.42.1 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 66ad64956..bedcfb2a5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,7 @@ jobs: - name: Checkout code uses: actions/checkout@v6 - name: Check spelling - uses: crate-ci/typos@1a319b54cc9e3b333fed6a5c88ba1a90324da514 # v1.40.1 + uses: crate-ci/typos@65120634e79d8374d1aa2f27e54baa0c364fff5a # v1.42.1 lint: name: lint From bd54c54f27846a2e38d3ca1be67353b164f41b6f Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Mon, 19 Jan 2026 21:57:20 +0100 Subject: [PATCH 111/141] Set versions for packages --- CHANGELOG.md | 19 +++++++++++++++---- Cargo.toml | 10 +++++----- hdf5-derive/Cargo.toml | 2 +- hdf5-src/Cargo.toml | 2 +- hdf5-sys/Cargo.toml | 2 +- hdf5-types/Cargo.toml | 2 +- hdf5/Cargo.toml | 2 +- 7 files changed, 25 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 12f8af4fc..1177c3bfd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,20 +1,31 @@ # Changelog ## hdf5 unreleased +## hdf5-types unreleased +## hdf5-derive unreleased +## hdf5-sys unreleased +## hdf5-src unreleased + +## hdf5 v0.12.0 +Release date: Jan 19, 2026 - Add support for hdf5 2.0.0 - Added support for ZFP compression filters as an optional feature -## hdf5-types unreleased +## hdf5-types v0.11.0 +Release date: Jan 19, 2026 - Add support for hdf5 2.0.0 -## hdf5-derive unreleased +## hdf5-derive v0.10.0 +Release date: Jan 19, 2026 - Add support for hdf5 2.0.0 -## hdf5-sys unreleased +## hdf5-sys v0.11.0 +Release date: Jan 19, 2026 - Add support for hdf5 2.0.0 - Remove `H5FD_*_init` functions, these are private to hdf5-c -## hdf5-src unreleased +## hdf5-src v0.10.0 +Release date: Jan 19, 2026 - Use hdf5 2.0.0 ## hdf5 v0.11.0 diff --git a/Cargo.toml b/Cargo.toml index f8839e1a1..b489a0242 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,11 +26,11 @@ num-complex = { version = "0.4", default-features = false } regex = "1.10" # internal -hdf5 = { package = "hdf5-metno", version = "0.9.3", path = "hdf5" } # !V -hdf5-derive = { package = "hdf5-metno-derive", version = "0.9.1", path = "hdf5-derive" } # !V -hdf5-src = { package = "hdf5-metno-src", version = "0.9.3", path = "hdf5-src" } # !V -hdf5-sys = { package = "hdf5-metno-sys", version = "0.10.1", path = "hdf5-sys" } # !V -hdf5-types = { package = "hdf5-metno-types", version = "0.10.0", path = "hdf5-types" } # !V +hdf5 = { package = "hdf5-metno", version = "0.12.0", path = "hdf5" } # !V +hdf5-derive = { package = "hdf5-metno-derive", version = "0.10.0", path = "hdf5-derive" } # !V +hdf5-types = { package = "hdf5-metno-types", version = "0.11.0", path = "hdf5-types" } # !V +hdf5-src = { package = "hdf5-metno-src", version = "0.10.0", path = "hdf5-src" } # !V +hdf5-sys = { package = "hdf5-metno-sys", version = "0.11.0", path = "hdf5-sys" } # !V [profile.dev] diff --git a/hdf5-derive/Cargo.toml b/hdf5-derive/Cargo.toml index af6574f40..95b9a1aff 100644 --- a/hdf5-derive/Cargo.toml +++ b/hdf5-derive/Cargo.toml @@ -3,7 +3,7 @@ name = "hdf5-metno-derive" description = "Derive macro for HDF5 structs and enums." categories = ["development-tools::procedural-macro-helpers"] readme = "README.md" -version = "0.9.3" # !V +version = "0.10.0" # !V rust-version.workspace = true authors.workspace = true keywords.workspace = true diff --git a/hdf5-src/Cargo.toml b/hdf5-src/Cargo.toml index 836ef627c..6aa3283ea 100644 --- a/hdf5-src/Cargo.toml +++ b/hdf5-src/Cargo.toml @@ -46,7 +46,7 @@ include = [ "!ext/hdf5/HDF5Examples/**", "!ext/hdf5/doxygen/**", ] -version = "0.9.5" # !V +version = "0.10.0" # !V rust-version.workspace = true authors.workspace = true keywords.workspace = true diff --git a/hdf5-sys/Cargo.toml b/hdf5-sys/Cargo.toml index 6813b0f6e..8a345d6d9 100644 --- a/hdf5-sys/Cargo.toml +++ b/hdf5-sys/Cargo.toml @@ -5,7 +5,7 @@ description = "Native bindings to the HDF5 library." links = "hdf5" readme = "README.md" categories = ["development-tools::ffi", "filesystem", "science"] -version = "0.10.1" # !V +version = "0.11.0" # !V rust-version.workspace = true authors.workspace = true keywords.workspace = true diff --git a/hdf5-types/Cargo.toml b/hdf5-types/Cargo.toml index b4818ceb4..861168b3d 100644 --- a/hdf5-types/Cargo.toml +++ b/hdf5-types/Cargo.toml @@ -4,7 +4,7 @@ description = "Native Rust equivalents of HDF5 types." readme = "README.md" build = "build.rs" categories = ["encoding"] -version = "0.10.2" # !V +version = "0.11.0" # !V rust-version.workspace = true authors.workspace = true keywords.workspace = true diff --git a/hdf5/Cargo.toml b/hdf5/Cargo.toml index d142ab1a1..3309b8687 100644 --- a/hdf5/Cargo.toml +++ b/hdf5/Cargo.toml @@ -4,7 +4,7 @@ readme = "../README.md" description = "Thread-safe Rust bindings for the HDF5 library." build = "build.rs" categories = ["science", "filesystem"] -version = "0.11.0" # !V +version = "0.12.0" # !V rust-version.workspace = true authors.workspace = true keywords.workspace = true From 930dd4210a73e8e06af1d40eae2a49cae034f7e4 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Mon, 19 Jan 2026 22:10:49 +0100 Subject: [PATCH 112/141] Set path to license file --- hdf5-src/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hdf5-src/Cargo.toml b/hdf5-src/Cargo.toml index 6aa3283ea..1d3350c75 100644 --- a/hdf5-src/Cargo.toml +++ b/hdf5-src/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "hdf5-metno-src" -license-file = "ext/hdf5/COPYING" +license-file = "ext/hdf5/LICENSE" build = "build.rs" description = "Build script for compiling HDF5 C library from source." links = "hdf5src" @@ -12,7 +12,7 @@ include = [ "README.md", "src/**/*.rs", "ext/hdf5/**/CMakeLists.txt", - "ext/hdf5/COPYING", + "ext/hdf5/LICENSE", "ext/hdf5/**/*.h", "ext/hdf5/**/*.c", "ext/hdf5/**/*.am", From 131222e03bd95bb6a800cfcab845207fa1fbaa3c Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Tue, 20 Jan 2026 21:12:03 +0100 Subject: [PATCH 113/141] Fix issue #140 --- CHANGELOG.md | 4 +++- hdf5-sys/src/h5fd.rs | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1177c3bfd..aecb3c00b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,13 @@ # Changelog -## hdf5 unreleased ## hdf5-types unreleased ## hdf5-derive unreleased ## hdf5-sys unreleased ## hdf5-src unreleased +## hdf5 unreleased +- Fixed an import for windows using prebuilt hdf5 2.0.0 + ## hdf5 v0.12.0 Release date: Jan 19, 2026 - Add support for hdf5 2.0.0 diff --git a/hdf5-sys/src/h5fd.rs b/hdf5-sys/src/h5fd.rs index 64e450f98..03da7113f 100644 --- a/hdf5-sys/src/h5fd.rs +++ b/hdf5-sys/src/h5fd.rs @@ -490,7 +490,7 @@ mod globals_2_0_0 { } #[cfg(all(feature = "2.0.0", all(target_env = "msvc", not(feature = "static"))))] -mod globals { +mod globals_2_0_0 { // dllimport hack pub type id_t = usize; extern_static!(H5FD_CORE, __imp_H5FD_CORE_id_g); From b28bbecfdc92cf7369b0895737ae6ee3886aebcb Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Tue, 20 Jan 2026 21:26:39 +0100 Subject: [PATCH 114/141] Release new version of hdf5 --- CHANGELOG.md | 3 ++- hdf5/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aecb3c00b..27613ad9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,8 @@ ## hdf5-sys unreleased ## hdf5-src unreleased -## hdf5 unreleased +## hdf5 v0.12.1 +Release date: Jan 20, 2026 - Fixed an import for windows using prebuilt hdf5 2.0.0 ## hdf5 v0.12.0 diff --git a/hdf5/Cargo.toml b/hdf5/Cargo.toml index 3309b8687..c0cf0c9b4 100644 --- a/hdf5/Cargo.toml +++ b/hdf5/Cargo.toml @@ -4,7 +4,7 @@ readme = "../README.md" description = "Thread-safe Rust bindings for the HDF5 library." build = "build.rs" categories = ["science", "filesystem"] -version = "0.12.0" # !V +version = "0.12.1" # !V rust-version.workspace = true authors.workspace = true keywords.workspace = true From bebb297f9b517b4272ba15a0aaf6740a9bf01fe5 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Tue, 20 Jan 2026 21:30:42 +0100 Subject: [PATCH 115/141] Release new version of hdf5-sys --- CHANGELOG.md | 7 ++++++- hdf5-sys/Cargo.toml | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 27613ad9b..6d9c52612 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,14 +1,19 @@ # Changelog +## hdf5 unreleased ## hdf5-types unreleased ## hdf5-derive unreleased ## hdf5-sys unreleased ## hdf5-src unreleased -## hdf5 v0.12.1 +## hdf5-sys v0.11.1 Release date: Jan 20, 2026 - Fixed an import for windows using prebuilt hdf5 2.0.0 +## hdf5 v0.12.1 +Release date: Jan 20, 2026 +- No changes + ## hdf5 v0.12.0 Release date: Jan 19, 2026 - Add support for hdf5 2.0.0 diff --git a/hdf5-sys/Cargo.toml b/hdf5-sys/Cargo.toml index 8a345d6d9..04c8246ff 100644 --- a/hdf5-sys/Cargo.toml +++ b/hdf5-sys/Cargo.toml @@ -5,7 +5,7 @@ description = "Native bindings to the HDF5 library." links = "hdf5" readme = "README.md" categories = ["development-tools::ffi", "filesystem", "science"] -version = "0.11.0" # !V +version = "0.11.1" # !V rust-version.workspace = true authors.workspace = true keywords.workspace = true From c5208995844a67649bfc3e8da621f1ae6472c4f2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Jan 2026 20:36:02 +0000 Subject: [PATCH 116/141] Bump crate-ci/typos from 1.42.1 to 1.42.2 Bumps [crate-ci/typos](https://github.com/crate-ci/typos) from 1.42.1 to 1.42.2. - [Release notes](https://github.com/crate-ci/typos/releases) - [Changelog](https://github.com/crate-ci/typos/blob/master/CHANGELOG.md) - [Commits](https://github.com/crate-ci/typos/compare/65120634e79d8374d1aa2f27e54baa0c364fff5a...a1d64977b4aa1709d6328d518aa753f4899352d8) --- updated-dependencies: - dependency-name: crate-ci/typos dependency-version: 1.42.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bedcfb2a5..0b9b705da 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,7 @@ jobs: - name: Checkout code uses: actions/checkout@v6 - name: Check spelling - uses: crate-ci/typos@65120634e79d8374d1aa2f27e54baa0c364fff5a # v1.42.1 + uses: crate-ci/typos@a1d64977b4aa1709d6328d518aa753f4899352d8 # v1.42.2 lint: name: lint From 241932c88839866b74189f8add9fbc2ac6d34d09 Mon Sep 17 00:00:00 2001 From: 42triangles <42triangles@tutanota.com> Date: Mon, 9 Feb 2026 09:06:58 +0100 Subject: [PATCH 117/141] Fix build failure when enabling both `hl` & `threadsafe` The option `HDF5_ALLOW_UNSUPPORTED` was renamed to include the `HDF5_` prefix here: https://github.com/HDFGroup/hdf5/commit/baa1e8e2922e520df740146141d24670db970a5e --- hdf5-src/build.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hdf5-src/build.rs b/hdf5-src/build.rs index e22638a9b..96c536531 100644 --- a/hdf5-src/build.rs +++ b/hdf5-src/build.rs @@ -61,7 +61,7 @@ fn main() { for option in &[ "HDF5_ENABLE_DEPRECATED_SYMBOLS", "HDF5_ENABLE_THREADSAFE", - "ALLOW_UNSUPPORTED", + "HDF5_ALLOW_UNSUPPORTED", "HDF5_BUILD_HL_LIB", "HDF5_ENABLE_NONSTANDARD_FEATURE_FLOAT16", "HDF5_ENABLE_SZIP_SUPPORT", @@ -91,7 +91,7 @@ fn main() { cfg.define("HDF5_ENABLE_THREADSAFE", "ON"); if feature_enabled("HL") { println!("cargo::warning=Unsupported HDF5 options: hl with threadsafe."); - cfg.define("ALLOW_UNSUPPORTED", "ON"); + cfg.define("HDF5_ALLOW_UNSUPPORTED", "ON"); } } From 1ecd172caf7525e936165a196873309f3d2e2572 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Mon, 9 Feb 2026 18:41:21 +0100 Subject: [PATCH 118/141] Update changelog --- CHANGELOG.md | 4 ++++ hdf5-src/Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d9c52612..7a04318df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ ## hdf5-sys unreleased ## hdf5-src unreleased +## hdf5-src v0.10.1 +Release date: Feb 09, 2026 +- Fixed name of cmake build option + ## hdf5-sys v0.11.1 Release date: Jan 20, 2026 - Fixed an import for windows using prebuilt hdf5 2.0.0 diff --git a/hdf5-src/Cargo.toml b/hdf5-src/Cargo.toml index 1d3350c75..7b016e47e 100644 --- a/hdf5-src/Cargo.toml +++ b/hdf5-src/Cargo.toml @@ -46,7 +46,7 @@ include = [ "!ext/hdf5/HDF5Examples/**", "!ext/hdf5/doxygen/**", ] -version = "0.10.0" # !V +version = "0.10.1" # !V rust-version.workspace = true authors.workspace = true keywords.workspace = true From fa2dfd465dc53078fccd17cd8d0a2ccf233be61c Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Mon, 9 Feb 2026 18:50:06 +0100 Subject: [PATCH 119/141] Bump brew hdf5 version --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0b9b705da..9ae72cb2b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,7 +70,7 @@ jobs: matrix: include: - {version: hdf5@1.10} - - {version: hdf5@1.14} + - {version: hdf5@2.0} - {version: hdf5-mpi, mpi: true} steps: - name: Checkout repository From 4c04c1d45c9d5075bbe5e2fd09af303e0fe0d1a0 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Mon, 9 Feb 2026 18:53:56 +0100 Subject: [PATCH 120/141] Fix MPIO driver in hdf5 2.0 --- CHANGELOG.md | 4 ++++ hdf5-sys/Cargo.toml | 2 +- hdf5/src/globals.rs | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a04318df..17df8fee6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ ## hdf5-sys unreleased ## hdf5-src unreleased +## hdf5-sys v0.11.2 +Release date: Feb 09, 2026 +- Fixed name of MPIO include + ## hdf5-src v0.10.1 Release date: Feb 09, 2026 - Fixed name of cmake build option diff --git a/hdf5-sys/Cargo.toml b/hdf5-sys/Cargo.toml index 04c8246ff..da16f97e9 100644 --- a/hdf5-sys/Cargo.toml +++ b/hdf5-sys/Cargo.toml @@ -5,7 +5,7 @@ description = "Native bindings to the HDF5 library." links = "hdf5" readme = "README.md" categories = ["development-tools::ffi", "filesystem", "science"] -version = "0.11.1" # !V +version = "0.11.2" # !V rust-version.workspace = true authors.workspace = true keywords.workspace = true diff --git a/hdf5/src/globals.rs b/hdf5/src/globals.rs index 729db9151..3ddadc65c 100644 --- a/hdf5/src/globals.rs +++ b/hdf5/src/globals.rs @@ -361,7 +361,7 @@ pub static H5FD_MULTI: LazyLock = LazyLock::new(|| { // MPI-IO file driver #[cfg(all(feature = "2.0.0", all(feature = "have-parallel", feature = "mpio")))] -pub static H5FD_MPIO: LazyLock = LazyLock::new(|| *hdf5_sys::h5p::H5FD_MPIO_id); +pub static H5FD_MPIO: LazyLock = LazyLock::new(|| *hdf5_sys::h5fd::H5FD_MPIO_id); #[cfg(all(feature = "2.0.0", not(all(feature = "have-parallel", feature = "mpio"))))] pub static H5FD_MPIO: LazyLock = LazyLock::new(|| H5I_INVALID_HID); From 010e24faff66ceb84c7e6dd1e2d0e438ddc3d750 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Mon, 9 Feb 2026 19:08:41 +0100 Subject: [PATCH 121/141] Find brew hdf5@2.0 --- hdf5-sys/build.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/hdf5-sys/build.rs b/hdf5-sys/build.rs index 2e56f6446..6c8857cad 100644 --- a/hdf5-sys/build.rs +++ b/hdf5-sys/build.rs @@ -363,6 +363,13 @@ mod macos { "any version" } ); + if !(v18 || v110 || v112 || v114) { + if let Some(out) = run_command("brew", &["--prefix", "hdf5@2.0"]) { + if is_root_dir(&out) { + config.inc_dir = Some(PathBuf::from(out).join("include")); + } + } + } if !(v18 || v110 || v112) { if let Some(out) = run_command("brew", &["--prefix", "hdf5@1.14"]) { if is_root_dir(&out) { From 293c67de285447d3f10ef84643edb8a9bbb58cf2 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Mon, 9 Feb 2026 19:11:28 +0100 Subject: [PATCH 122/141] Fix cfg typo --- hdf5-sys/src/h5fd.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hdf5-sys/src/h5fd.rs b/hdf5-sys/src/h5fd.rs index 03da7113f..d90b0c2d8 100644 --- a/hdf5-sys/src/h5fd.rs +++ b/hdf5-sys/src/h5fd.rs @@ -485,7 +485,7 @@ mod globals_2_0_0 { extern_static!(H5FD_SEC2, H5FD_SEC2_id_g); extern_static!(H5FD_SPLITTER, H5FD_SPLITTER_id_g); extern_static!(H5FD_STDIO, H5FD_STDIO_id_g); - #[cfg(feature = "have_parallel")] + #[cfg(feature = "have-parallel")] extern_static!(H5FD_MPIO, H5FD_MPIO_id_g); } From 4466da1746d41fb9abd17ac870e0d128756bdbdd Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Mon, 9 Feb 2026 19:12:33 +0100 Subject: [PATCH 123/141] Set new version of hdf5 crate --- CHANGELOG.md | 4 ++++ hdf5/Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 17df8fee6..c97bb1a44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ ## hdf5-sys unreleased ## hdf5-src unreleased +## hdf5 v0.12.2 +Release date: Feb 09, 2026 +- Fixed MPIO include + ## hdf5-sys v0.11.2 Release date: Feb 09, 2026 - Fixed name of MPIO include diff --git a/hdf5/Cargo.toml b/hdf5/Cargo.toml index c0cf0c9b4..e0eb03bbc 100644 --- a/hdf5/Cargo.toml +++ b/hdf5/Cargo.toml @@ -4,7 +4,7 @@ readme = "../README.md" description = "Thread-safe Rust bindings for the HDF5 library." build = "build.rs" categories = ["science", "filesystem"] -version = "0.12.1" # !V +version = "0.12.2" # !V rust-version.workspace = true authors.workspace = true keywords.workspace = true From 23948b2edeee2d42ce61e2a520d2c806c8e52b32 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Mon, 9 Feb 2026 19:17:30 +0100 Subject: [PATCH 124/141] Fix include name --- hdf5/src/globals.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hdf5/src/globals.rs b/hdf5/src/globals.rs index 3ddadc65c..3eeb5e9d8 100644 --- a/hdf5/src/globals.rs +++ b/hdf5/src/globals.rs @@ -361,7 +361,7 @@ pub static H5FD_MULTI: LazyLock = LazyLock::new(|| { // MPI-IO file driver #[cfg(all(feature = "2.0.0", all(feature = "have-parallel", feature = "mpio")))] -pub static H5FD_MPIO: LazyLock = LazyLock::new(|| *hdf5_sys::h5fd::H5FD_MPIO_id); +pub static H5FD_MPIO: LazyLock = LazyLock::new(|| *hdf5_sys::h5fd::H5FD_MPIO); #[cfg(all(feature = "2.0.0", not(all(feature = "have-parallel", feature = "mpio"))))] pub static H5FD_MPIO: LazyLock = LazyLock::new(|| H5I_INVALID_HID); From 8fa1712f03754ddc75ab2fed6ca18775138e9da6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Feb 2026 22:07:55 +0000 Subject: [PATCH 125/141] Bump crate-ci/typos from 1.42.2 to 1.43.4 Bumps [crate-ci/typos](https://github.com/crate-ci/typos) from 1.42.2 to 1.43.4. - [Release notes](https://github.com/crate-ci/typos/releases) - [Changelog](https://github.com/crate-ci/typos/blob/master/CHANGELOG.md) - [Commits](https://github.com/crate-ci/typos/compare/a1d64977b4aa1709d6328d518aa753f4899352d8...78bc6fb2c0d734235d57a2d6b9de923cc325ebdd) --- updated-dependencies: - dependency-name: crate-ci/typos dependency-version: 1.43.4 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9ae72cb2b..3e6a445fb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,7 @@ jobs: - name: Checkout code uses: actions/checkout@v6 - name: Check spelling - uses: crate-ci/typos@a1d64977b4aa1709d6328d518aa753f4899352d8 # v1.42.2 + uses: crate-ci/typos@78bc6fb2c0d734235d57a2d6b9de923cc325ebdd # v1.43.4 lint: name: lint From 5de87935bef9a14985fcce0942585cbc6424a70b Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Tue, 10 Feb 2026 08:30:32 +0100 Subject: [PATCH 126/141] Add some checks for non-compressible compressions --- hdf5/src/hl/filters/blosc.rs | 3 +++ hdf5/src/hl/filters/lzf.rs | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/hdf5/src/hl/filters/blosc.rs b/hdf5/src/hl/filters/blosc.rs index 720d9c9b1..70ac5ef40 100644 --- a/hdf5/src/hl/filters/blosc.rs +++ b/hdf5/src/hl/filters/blosc.rs @@ -145,6 +145,9 @@ impl Default for BloscConfig { } fn parse_blosc_cdata(cd_nelmts: size_t, cd_values: *const c_uint) -> Option { + if cd_values.is_null() || cd_nelmts < 1 { + return None; + } let cdata = unsafe { slice::from_raw_parts(cd_values, cd_nelmts as _) }; let mut cfg = BloscConfig { typesize: cdata[2] as _, diff --git a/hdf5/src/hl/filters/lzf.rs b/hdf5/src/hl/filters/lzf.rs index 37e288554..5aa095ca6 100644 --- a/hdf5/src/hl/filters/lzf.rs +++ b/hdf5/src/hl/filters/lzf.rs @@ -125,7 +125,11 @@ unsafe fn filter_lzf_decompress( cd_nelmts: size_t, cd_values: *const c_uint, nbytes: size_t, buf_size: *mut size_t, buf: *mut *mut c_void, ) -> size_t { - let cdata = slice::from_raw_parts(cd_values, cd_nelmts as _); + let cdata = if cd_values.is_null() || cd_nelmts < 1 { + &[] + } else { + slice::from_raw_parts(cd_values, cd_nelmts as _) + }; let mut outbuf_size = if cd_nelmts >= 3 && cdata[2] != 0 { cdata[2] as _ } else { *buf_size }; let mut outbuf: *mut c_void; let mut status: c_uint; From 820765e9bd483dc1df5c4afd17491a514e2ae428 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Tue, 10 Feb 2026 08:41:52 +0100 Subject: [PATCH 127/141] Check typesize for blosc --- hdf5/src/hl/filters/blosc.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hdf5/src/hl/filters/blosc.rs b/hdf5/src/hl/filters/blosc.rs index 70ac5ef40..75c1a9905 100644 --- a/hdf5/src/hl/filters/blosc.rs +++ b/hdf5/src/hl/filters/blosc.rs @@ -193,6 +193,9 @@ unsafe extern "C" fn filter_blosc( buf_size: *mut size_t, buf: *mut *mut c_void, ) -> size_t { let cfg = if let Some(cfg) = parse_blosc_cdata(cd_nelmts, cd_values) { + if cfg.typesize == 0 { + return 0; + } cfg } else { return 0; From aa52850234d9921de8c2290203d9e75ab8e51d38 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Tue, 10 Feb 2026 09:00:36 +0100 Subject: [PATCH 128/141] Move crashing example to tests --- hdf5/src/hl/filters.rs | 2 +- hdf5/tests/test_compressing_varlenunicode.rs | 140 +++++++++++++++++++ 2 files changed, 141 insertions(+), 1 deletion(-) create mode 100644 hdf5/tests/test_compressing_varlenunicode.rs diff --git a/hdf5/src/hl/filters.rs b/hdf5/src/hl/filters.rs index c36875c9e..cc48d1366 100644 --- a/hdf5/src/hl/filters.rs +++ b/hdf5/src/hl/filters.rs @@ -863,9 +863,9 @@ mod tests { { comp_filters.push(Filter::lzf()); } - assert_eq!(cfg!(feature = "blosc-all"), blosc_available()); #[cfg(feature = "blosc-all")] { + assert!(blosc_available()); use super::BloscShuffle; comp_filters.push(Filter::blosc_blosclz(1, false)); comp_filters.push(Filter::blosc_lz4(3, true)); diff --git a/hdf5/tests/test_compressing_varlenunicode.rs b/hdf5/tests/test_compressing_varlenunicode.rs new file mode 100644 index 000000000..df5ce99c7 --- /dev/null +++ b/hdf5/tests/test_compressing_varlenunicode.rs @@ -0,0 +1,140 @@ +use std::str::FromStr; + +use hdf5::types::VarLenUnicode; +use hdf5_metno as hdf5; + +mod common; + +use self::common::util::new_in_memory_file; + +#[test] +#[cfg(feature = "blosc")] +fn blosc_blosclz() -> Result<(), Box> { + let file = new_in_memory_file()?; + + let dset_name = "x"; + let n_samples = 10; + let input_data = vec![VarLenUnicode::from_str("test").unwrap(); n_samples]; + + let ds_builder = file + .new_dataset::() + .shape((n_samples,)) + .chunk((n_samples,)) + .blosc_blosclz(5, true); + + let ds = ds_builder.create(dset_name)?; + ds.write(&input_data)?; + + let read: ndarray::Array1 = + file.dataset(dset_name)?.read::()?; + assert_eq!(read.to_vec(), input_data, "read data must match written data"); + Ok(()) +} + +#[test] +#[cfg(feature = "blosc-lz4")] +fn blosc_lz4() -> Result<(), Box> { + let file = new_in_memory_file()?; + + let dset_name = "x"; + let n_samples = 10; + let input_data = vec![VarLenUnicode::from_str("test")?; n_samples]; + let ds_builder = file + .new_dataset::() + .shape((n_samples,)) + .chunk((n_samples,)) + .blosc_lz4(5, true); + + let ds = ds_builder.create(dset_name)?; + ds.write(&input_data)?; + + let read: ndarray::Array1 = + file.dataset(dset_name)?.read::()?; + assert_eq!(read.to_vec(), input_data, "read data must match written data"); + Ok(()) +} + +#[test] +#[cfg(feature = "blosc-zlib")] +fn blosc_zlib() -> Result<(), Box> { + let file = new_in_memory_file()?; + + let dset_name = "x"; + let n_samples = 10; + let input_data = vec![VarLenUnicode::from_str("test")?; n_samples]; + let ds_builder = file + .new_dataset::() + .shape((n_samples,)) + .chunk((n_samples,)) + .blosc_zlib(5, true); + + let ds = ds_builder.create(dset_name)?; + ds.write(&input_data)?; + + let read: ndarray::Array1 = + file.dataset(dset_name)?.read::()?; + assert_eq!(read.to_vec(), input_data, "read data must match written data"); + Ok(()) +} + +#[test] +#[cfg(feature = "lzf")] +fn lzf() -> Result<(), Box> { + let file = new_in_memory_file()?; + + let dset_name = "x"; + let n_samples = 10; + let input_data = vec![VarLenUnicode::from_str("test")?; n_samples]; + let ds_builder = + file.new_dataset::().shape((n_samples,)).chunk((n_samples,)).lzf(); + + let ds = ds_builder.create(dset_name)?; + ds.write(&input_data)?; + + let read: ndarray::Array1 = + file.dataset(dset_name)?.read::()?; + assert_eq!(read.to_vec(), input_data, "read data must match written data"); + Ok(()) +} + +#[test] +fn szip() -> Result<(), Box> { + let file = new_in_memory_file()?; + + let dset_name = "x"; + let n_samples = 10; + let input_data = vec![VarLenUnicode::from_str("test")?; n_samples]; + let ds_builder = file + .new_dataset::() + .shape((n_samples,)) + .chunk((n_samples,)) + .szip(hdf5::filters::SZip::Entropy, 8); + + let ds = ds_builder.create(dset_name)?; + ds.write(&input_data)?; + + let read: ndarray::Array1 = + file.dataset(dset_name)?.read::()?; + assert_eq!(read.to_vec(), input_data, "read data must match written data"); + Ok(()) +} + +#[test] +#[cfg(feature = "zlib")] +fn deflate() -> Result<(), Box> { + let file = new_in_memory_file()?; + + let dset_name = "x"; + let n_samples = 10; + let input_data = vec![VarLenUnicode::from_str("test")?; n_samples]; + let ds_builder = + file.new_dataset::().shape((n_samples,)).chunk((n_samples,)).deflate(5); + + let ds = ds_builder.create(dset_name)?; + ds.write(&input_data)?; + + let read: ndarray::Array1 = + file.dataset(dset_name)?.read::()?; + assert_eq!(read.to_vec(), input_data, "read data must match written data"); + Ok(()) +} From 6bc2bd9d18499ea5154600cb7b6a357985d1cf5a Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Tue, 10 Feb 2026 18:06:30 +0100 Subject: [PATCH 129/141] Quit early if szip not available --- hdf5/tests/test_compressing_varlenunicode.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hdf5/tests/test_compressing_varlenunicode.rs b/hdf5/tests/test_compressing_varlenunicode.rs index df5ce99c7..33d883d69 100644 --- a/hdf5/tests/test_compressing_varlenunicode.rs +++ b/hdf5/tests/test_compressing_varlenunicode.rs @@ -99,6 +99,9 @@ fn lzf() -> Result<(), Box> { #[test] fn szip() -> Result<(), Box> { + if !hdf5::filters::szip_available() { + return Ok(()); + } let file = new_in_memory_file()?; let dset_name = "x"; From 5992a439747f94397373487a6f1e03d89f8ded50 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Tue, 10 Feb 2026 18:07:58 +0100 Subject: [PATCH 130/141] Add to changelog --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c97bb1a44..061f35526 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,13 @@ # Changelog -## hdf5 unreleased ## hdf5-types unreleased ## hdf5-derive unreleased ## hdf5-sys unreleased ## hdf5-src unreleased +## hdf5 unreleased +- Fixed compression of VarLenUnicode resulting in errors + ## hdf5 v0.12.2 Release date: Feb 09, 2026 - Fixed MPIO include From 32b91a55381002b0035f6ea5e4157ff2d8b0e101 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Tue, 10 Feb 2026 18:26:38 +0100 Subject: [PATCH 131/141] Release new version of hdf5 --- CHANGELOG.md | 4 +++- hdf5/Cargo.toml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 061f35526..670984043 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,13 @@ # Changelog +## hdf5 unreleased ## hdf5-types unreleased ## hdf5-derive unreleased ## hdf5-sys unreleased ## hdf5-src unreleased -## hdf5 unreleased +## hdf5 v0.12.3 +Release date: Feb 10, 2026 - Fixed compression of VarLenUnicode resulting in errors ## hdf5 v0.12.2 diff --git a/hdf5/Cargo.toml b/hdf5/Cargo.toml index e0eb03bbc..eb77a6c3a 100644 --- a/hdf5/Cargo.toml +++ b/hdf5/Cargo.toml @@ -4,7 +4,7 @@ readme = "../README.md" description = "Thread-safe Rust bindings for the HDF5 library." build = "build.rs" categories = ["science", "filesystem"] -version = "0.12.2" # !V +version = "0.12.3" # !V rust-version.workspace = true authors.workspace = true keywords.workspace = true From 3971da713bb9651d2a86feeb01f2c3a24e5bf102 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Tue, 10 Feb 2026 22:48:18 +0100 Subject: [PATCH 132/141] Bump rand --- hdf5/Cargo.toml | 2 +- hdf5/tests/common/gen.rs | 2 +- hdf5/tests/test_dataset.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hdf5/Cargo.toml b/hdf5/Cargo.toml index eb77a6c3a..f055cb838 100644 --- a/hdf5/Cargo.toml +++ b/hdf5/Cargo.toml @@ -70,7 +70,7 @@ num-complex = { workspace = true } parking_lot = "0.12.3" paste = "1.0" pretty_assertions = "1.4" -rand = { version = "0.9", features = ["small_rng"] } +rand = { version = "0.10" } regex = { workspace = true } scopeguard = "1.2" tempfile = "3.9" diff --git a/hdf5/tests/common/gen.rs b/hdf5/tests/common/gen.rs index 296fdaf25..094deed8f 100644 --- a/hdf5/tests/common/gen.rs +++ b/hdf5/tests/common/gen.rs @@ -11,8 +11,8 @@ use ndarray::{ArrayD, SliceInfo, SliceInfoElem}; use num_complex::Complex; use rand::distr::StandardUniform; use rand::distr::{Alphanumeric, Uniform}; -use rand::prelude::Rng; use rand::prelude::{Distribution, IndexedRandom}; +use rand::prelude::{Rng, RngExt}; pub fn gen_shape(rng: &mut R, ndim: usize) -> Vec { iter::repeat(()).map(|_| rng.random_range(0..11)).take(ndim).collect() diff --git a/hdf5/tests/test_dataset.rs b/hdf5/tests/test_dataset.rs index 46938d232..bd4b4485a 100644 --- a/hdf5/tests/test_dataset.rs +++ b/hdf5/tests/test_dataset.rs @@ -3,7 +3,7 @@ use std::fmt; use std::io::{Read, Seek, SeekFrom}; use ndarray::{s, Array1, Array2, ArrayD, IxDyn, SliceInfo}; -use rand::prelude::{Rng, SeedableRng, SmallRng}; +use rand::prelude::{Rng, RngExt, SeedableRng, SmallRng}; use hdf5_metno as hdf5; use hdf5_types::TypeDescriptor; From 509643e17c68e0c7d12a1a757d93fbacc315482c Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Tue, 10 Feb 2026 22:53:44 +0100 Subject: [PATCH 133/141] Increase MSRV --- .github/workflows/ci.yml | 9 ++++----- CHANGELOG.md | 4 +++- Cargo.toml | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9ae72cb2b..268b0c716 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -289,14 +289,13 @@ jobs: with: {submodules: true} - name: Install Rust uses: dtolnay/rust-toolchain@stable - with: {toolchain: "1.80"} + with: {toolchain: "1.85.1"} - name: Create lock-file run: cargo update - - name: Override deps - run: | - cargo update half@2.7.1 --precise 2.4.1 - cargo update indexmap@2.13.0 --precise 2.11.4 + # - name: Override deps + # run: | + # cargo update half@2.7.1 --precise 2.4.1 - name: Build and test all crates run: cargo test --locked --workspace -vv --features=hdf5-sys/static,hdf5-sys/zlib --exclude=hdf5-metno-derive diff --git a/CHANGELOG.md b/CHANGELOG.md index 670984043..4f09939d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,13 @@ # Changelog -## hdf5 unreleased ## hdf5-types unreleased ## hdf5-derive unreleased ## hdf5-sys unreleased ## hdf5-src unreleased +## hdf5 unreleased +- MSRV has been increased to 1.85 + ## hdf5 v0.12.3 Release date: Feb 10, 2026 - Fixed compression of VarLenUnicode resulting in errors diff --git a/Cargo.toml b/Cargo.toml index b489a0242..a24f37c44 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ members = ["hdf5", "hdf5-types", "hdf5-derive", "hdf5-sys", "hdf5-src"] default-members = ["hdf5", "hdf5-types", "hdf5-derive", "hdf5-sys"] [workspace.package] -rust-version = "1.80.0" +rust-version = "1.85.1" authors = [ "Ivan Smirnov ", "Magnus Ulimoen ", From 4c09edff33908d3086f89b21497db1fe9fa1a94c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Mar 2026 20:44:50 +0000 Subject: [PATCH 134/141] Bump crate-ci/typos from 1.43.4 to 1.44.0 Bumps [crate-ci/typos](https://github.com/crate-ci/typos) from 1.43.4 to 1.44.0. - [Release notes](https://github.com/crate-ci/typos/releases) - [Changelog](https://github.com/crate-ci/typos/blob/master/CHANGELOG.md) - [Commits](https://github.com/crate-ci/typos/compare/78bc6fb2c0d734235d57a2d6b9de923cc325ebdd...631208b7aac2daa8b707f55e7331f9112b0e062d) --- updated-dependencies: - dependency-name: crate-ci/typos dependency-version: 1.44.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 68d01bcff..f67f95b1d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,7 @@ jobs: - name: Checkout code uses: actions/checkout@v6 - name: Check spelling - uses: crate-ci/typos@78bc6fb2c0d734235d57a2d6b9de923cc325ebdd # v1.43.4 + uses: crate-ci/typos@631208b7aac2daa8b707f55e7331f9112b0e062d # v1.44.0 lint: name: lint From 37f1d2010342b0923f04c4f33396addc91c5da42 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Mar 2026 19:22:30 +0000 Subject: [PATCH 135/141] deps: update winreg requirement from 0.55 to 0.56 Updates the requirements on [winreg](https://github.com/gentoo90/winreg-rs) to permit the latest version. - [Release notes](https://github.com/gentoo90/winreg-rs/releases) - [Changelog](https://github.com/gentoo90/winreg-rs/blob/master/CHANGELOG.md) - [Commits](https://github.com/gentoo90/winreg-rs/compare/v0.55.0...v0.56.0) --- updated-dependencies: - dependency-name: winreg dependency-version: 0.56.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- hdf5-sys/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hdf5-sys/Cargo.toml b/hdf5-sys/Cargo.toml index da16f97e9..791bb5c95 100644 --- a/hdf5-sys/Cargo.toml +++ b/hdf5-sys/Cargo.toml @@ -41,7 +41,7 @@ pkg-config = "0.3" [target.'cfg(windows)'.build-dependencies] serde = "1.0" serde_derive = "1.0" -winreg = { version = "0.55", features = ["serialization-serde"] } +winreg = { version = "0.56", features = ["serialization-serde"] } [package.metadata.docs.rs] features = ["static", "zlib"] From f1feb8945f9d4150d28672674cd9e7b0f89240e1 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Mon, 23 Mar 2026 20:56:31 +0100 Subject: [PATCH 136/141] Update package caches --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f67f95b1d..cb351c248 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -193,6 +193,7 @@ jobs: [ "${{matrix.mpi}}" == "mpich" ] && PACKAGES="libhdf5-mpich-dev mpich" [ "${{matrix.mpi}}" == "openmpi" ] && PACKAGES="libhdf5-openmpi-dev openmpi-bin" [ "${{matrix.mpi}}" == "serial" ] && PACKAGES="libhdf5-dev" + sudo apt-get update sudo apt-get install $PACKAGES - name: Build and test all crates run: | From 6ebfecc9741a52e4def446de4cf8cd0fee51789a Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Mon, 23 Mar 2026 20:49:20 +0100 Subject: [PATCH 137/141] Add support for hdf5 2.1.0 --- Cargo.toml | 2 +- hdf5-sys/Cargo.toml | 2 +- hdf5-sys/build.rs | 4 ++-- hdf5/Cargo.toml | 2 +- hdf5/build.rs | 1 + 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index a24f37c44..34b76e57c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,7 +30,7 @@ hdf5 = { package = "hdf5-metno", version = "0.12.0", path = "hdf5" } hdf5-derive = { package = "hdf5-metno-derive", version = "0.10.0", path = "hdf5-derive" } # !V hdf5-types = { package = "hdf5-metno-types", version = "0.11.0", path = "hdf5-types" } # !V hdf5-src = { package = "hdf5-metno-src", version = "0.10.0", path = "hdf5-src" } # !V -hdf5-sys = { package = "hdf5-metno-sys", version = "0.11.0", path = "hdf5-sys" } # !V +hdf5-sys = { package = "hdf5-metno-sys", version = "0.11.3", path = "hdf5-sys" } # !V [profile.dev] diff --git a/hdf5-sys/Cargo.toml b/hdf5-sys/Cargo.toml index 791bb5c95..1663481eb 100644 --- a/hdf5-sys/Cargo.toml +++ b/hdf5-sys/Cargo.toml @@ -5,7 +5,7 @@ description = "Native bindings to the HDF5 library." links = "hdf5" readme = "README.md" categories = ["development-tools::ffi", "filesystem", "science"] -version = "0.11.2" # !V +version = "0.11.3" # !V rust-version.workspace = true authors.workspace = true keywords.workspace = true diff --git a/hdf5-sys/build.rs b/hdf5-sys/build.rs index 6c8857cad..ddfebb6dd 100644 --- a/hdf5-sys/build.rs +++ b/hdf5-sys/build.rs @@ -31,7 +31,7 @@ impl Version { pub fn parse(s: &str) -> Option { let re = - Regex::new(r"^(1|2)\.(0|8|10|12|14)\.(\d\d?)(_|.\d+)?((-|.)(patch)?\d+)?$").ok()?; + Regex::new(r"^(1|2)\.(0|1|8|10|12|14)\.(\d\d?)(_|.\d+)?((-|.)(patch)?\d+)?$").ok()?; let captures = re.captures(s)?; Some(Self { major: captures.get(1).and_then(|c| c.as_str().parse::().ok())?, @@ -54,7 +54,7 @@ impl Debug for Version { fn known_hdf5_versions() -> Vec { // Keep up to date with known_hdf5_versions in hdf5 let mut vs = Vec::new(); - vs.push(Version::new(2, 0, 0)); // 2.0.0 + vs.extend((0..=1).map(|v| Version::new(2, v, 0))); vs.extend((5..=21).map(|v| Version::new(1, 8, v))); // 1.8.[5-23] vs.extend((0..=8).map(|v| Version::new(1, 10, v))); // 1.10.[0-10] vs.extend((0..=2).map(|v| Version::new(1, 12, v))); // 1.12.[0-2] diff --git a/hdf5/Cargo.toml b/hdf5/Cargo.toml index f055cb838..087ed2d5c 100644 --- a/hdf5/Cargo.toml +++ b/hdf5/Cargo.toml @@ -4,7 +4,7 @@ readme = "../README.md" description = "Thread-safe Rust bindings for the HDF5 library." build = "build.rs" categories = ["science", "filesystem"] -version = "0.12.3" # !V +version = "0.12.4" # !V rust-version.workspace = true authors.workspace = true keywords.workspace = true diff --git a/hdf5/build.rs b/hdf5/build.rs index 74deb6275..3f13d4865 100644 --- a/hdf5/build.rs +++ b/hdf5/build.rs @@ -17,6 +17,7 @@ fn known_hdf5_versions() -> Vec { // Keep up to date with known_hdf5_versions in hdf5-sys let mut vs = Vec::new(); vs.push(Version::new(2, 0, 0)); // 2.0.0 + vs.extend((0..=1).map(|v| Version::new(2, v, 0))); // 2.[0-1].0 vs.extend((5..=21).map(|v| Version::new(1, 8, v))); // 1.8.[5-23] vs.extend((0..=8).map(|v| Version::new(1, 10, v))); // 1.10.[0-10] vs.extend((0..=2).map(|v| Version::new(1, 12, v))); // 1.12.[0-2] From 0d9e9cb4581ef807526a09bbc4cc4435fac89924 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Mon, 23 Mar 2026 20:53:56 +0100 Subject: [PATCH 138/141] Use hdf5 2.1.0 --- hdf5-src/Cargo.toml | 2 +- hdf5-src/ext/hdf5 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hdf5-src/Cargo.toml b/hdf5-src/Cargo.toml index 7b016e47e..f4584ef91 100644 --- a/hdf5-src/Cargo.toml +++ b/hdf5-src/Cargo.toml @@ -46,7 +46,7 @@ include = [ "!ext/hdf5/HDF5Examples/**", "!ext/hdf5/doxygen/**", ] -version = "0.10.1" # !V +version = "0.10.2" # !V rust-version.workspace = true authors.workspace = true keywords.workspace = true diff --git a/hdf5-src/ext/hdf5 b/hdf5-src/ext/hdf5 index a6ff8aed2..24176582e 160000 --- a/hdf5-src/ext/hdf5 +++ b/hdf5-src/ext/hdf5 @@ -1 +1 @@ -Subproject commit a6ff8aed236ee1e1deff6415e88b16c42b22f17c +Subproject commit 24176582e5e4093f02399d726d04f25db1772fe3 From 71c0163d617c173e78507b17d3486e22c9e9d55b Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Mon, 23 Mar 2026 21:03:16 +0100 Subject: [PATCH 139/141] Add to changelog --- CHANGELOG.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f09939d3..b6a12a8d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,13 +1,24 @@ # Changelog +## hdf5 unreleased ## hdf5-types unreleased ## hdf5-derive unreleased ## hdf5-sys unreleased ## hdf5-src unreleased -## hdf5 unreleased +## hdf5 0.12.4 +Release date: Mar 23, 2026 +- Support for hdf5 2.1.0 - MSRV has been increased to 1.85 +## hdf5-sys 0.11.3 +Release date: Mar 23, 2026 +- Support for hdf5 2.1.0 + +## hdf5-src 0.10.2 +Release date: Mar 23, 2026 +- Update to hdf5 2.1.0 + ## hdf5 v0.12.3 Release date: Feb 10, 2026 - Fixed compression of VarLenUnicode resulting in errors From ee77af5debae06d438d5582bdbb1e9c2f609ec62 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Mon, 23 Mar 2026 21:11:58 +0100 Subject: [PATCH 140/141] Add homebrew version detection Using contributions from @JulianDicken --- .github/workflows/ci.yml | 1 + hdf5-sys/build.rs | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cb351c248..96fa5e2c9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,6 +71,7 @@ jobs: include: - {version: hdf5@1.10} - {version: hdf5@2.0} + - {version: hdf5} - {version: hdf5-mpi, mpi: true} steps: - name: Checkout repository diff --git a/hdf5-sys/build.rs b/hdf5-sys/build.rs index ddfebb6dd..94e8f3a01 100644 --- a/hdf5-sys/build.rs +++ b/hdf5-sys/build.rs @@ -336,8 +336,9 @@ mod macos { } // We have to explicitly support homebrew since the HDF5 bottle isn't // packaged with pkg-config metadata. - let (v20, v18, v110, v112, v114) = if let Some(version) = config.version { + let (v21, v20, v18, v110, v112, v114) = if let Some(version) = config.version { ( + version.major == 2 && version.minor == 1, version.major == 2 && version.minor == 0, version.major == 1 && version.minor == 8, version.major == 1 && version.minor == 10, @@ -345,11 +346,13 @@ mod macos { version.major == 1 && version.minor == 14, ) } else { - (false, false, false, false, false) + (false, false, false, false, false, false) }; println!( "Attempting to find HDF5 via Homebrew ({})...", - if v20 { + if v21 { + "2.1.*" + } else if v20 { "2.0.*" } else if v18 { "1.8.*" @@ -363,6 +366,13 @@ mod macos { "any version" } ); + if !(v18 || v110 || v112 || v114 || v20) { + if let Some(out) = run_command("brew", &["--prefix", "hdf5@2.1"]) { + if is_root_dir(&out) { + config.inc_dir = Some(PathBuf::from(out).join("include")); + } + } + } if !(v18 || v110 || v112 || v114) { if let Some(out) = run_command("brew", &["--prefix", "hdf5@2.0"]) { if is_root_dir(&out) { From 840f1a168e3bb3d14d7766d55b4be13697e88a37 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 May 2026 23:12:45 +0000 Subject: [PATCH 141/141] Bump crate-ci/typos from 1.44.0 to 1.46.0 Bumps [crate-ci/typos](https://github.com/crate-ci/typos) from 1.44.0 to 1.46.0. - [Release notes](https://github.com/crate-ci/typos/releases) - [Changelog](https://github.com/crate-ci/typos/blob/master/CHANGELOG.md) - [Commits](https://github.com/crate-ci/typos/compare/631208b7aac2daa8b707f55e7331f9112b0e062d...bbaefadf97b0ec5fdc942684b647f1a6ab250274) --- updated-dependencies: - dependency-name: crate-ci/typos dependency-version: 1.46.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 96fa5e2c9..beea4998d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,7 @@ jobs: - name: Checkout code uses: actions/checkout@v6 - name: Check spelling - uses: crate-ci/typos@631208b7aac2daa8b707f55e7331f9112b0e062d # v1.44.0 + uses: crate-ci/typos@bbaefadf97b0ec5fdc942684b647f1a6ab250274 # v1.46.0 lint: name: lint