diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f210a85e6..beea4998d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,8 +10,9 @@ on: schedule: - cron: '0 18 * * *' +# Ensure this workflow gets cancelled if a newer commit has been pushed on the same branch (unless it's the default branch) concurrency: - group: ${{ github.workflow }}-@{{ github.ref }} + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true env: @@ -23,9 +24,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Check spelling - uses: crate-ci/typos@7bc041cbb7ca9167c9e0e4ccbb26f48eb0f9d4e0 # v1.30.2 + uses: crate-ci/typos@bbaefadf97b0ec5fdc942684b647f1a6ab250274 # v1.46.0 lint: name: lint @@ -38,7 +39,7 @@ jobs: - {command: clippy, rust: stable} steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Install Rust (${{matrix.rust}}) uses: dtolnay/rust-toolchain@stable with: {toolchain: '${{matrix.rust}}', components: 'rustfmt, clippy'} @@ -51,15 +52,15 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: {submodules: true} - - name: Install Rust (${{matrix.rust}}) + - name: Install Rust uses: dtolnay/rust-toolchain@stable - with: {toolchain: nightly} + with: {toolchain: "nightly-2025-09-01"} - name: Document workspace env: RUSTDOCFLAGS: "--cfg docsrs" - run: cargo doc --features static,zlib,blosc-all,lzf,f16,complex + run: cargo doc --features static,zlib,blosc-all,lzf,f16,complex --no-deps brew: name: brew @@ -69,11 +70,12 @@ jobs: matrix: include: - {version: hdf5@1.10} - - {version: hdf5@1.14} + - {version: hdf5@2.0} + - {version: hdf5} - {version: hdf5-mpi, mpi: true} steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: {submodules: true} - name: Install Rust (${{matrix.rust}}) uses: dtolnay/rust-toolchain@stable @@ -94,7 +96,6 @@ jobs: include: - {os: ubuntu-latest, version: 1.8.16, channel: conda-forge, rust: stable} - {os: windows-latest, version: 1.8.17, channel: conda-forge, rust: stable} - - {os: macos-13, version: 1.8.18, channel: anaconda, rust: stable} - {os: ubuntu-latest, version: 1.8.20, channel: anaconda, rust: beta} - {os: ubuntu-latest, version: 1.10.1, channel: anaconda, rust: nightly} - {os: windows-latest, version: 1.10.2, channel: anaconda, rust: beta} @@ -102,14 +103,12 @@ jobs: - {os: windows-latest, version: 1.10.4, channel: anaconda, rust: nightly} - {os: ubuntu-latest, version: 1.10.4, mpi: openmpi, channel: conda-forge, rust: stable} - {os: ubuntu-latest, version: 1.10.5, channel: conda-forge, rust: beta} - - {os: macos-13, version: 1.10.5, mpi: openmpi, channel: conda-forge, rust: beta} - {os: ubuntu-latest, version: 1.10.6, channel: anaconda, rust: stable} - {os: ubuntu-latest, version: 1.10.6, mpi: mpich, channel: conda-forge, rust: nightly} - # - {os: ubuntu, version: 1.10.8, channel: conda-forge, rust: stable} - {os: ubuntu-latest, version: 1.12.0, mpi: openmpi, channel: conda-forge, rust: stable} - - {os: macos-latest, version: 1.12.0, channel: conda-forge, rust: stable} - {os: windows-latest, version: 1.12.0, channel: conda-forge, rust: stable} - {os: ubuntu-latest, version: 1.12.1, channel: conda-forge, rust: stable} + - {os: macos-14, version: 1.12.1, mpi: openmpi, channel: conda-forge, rust: beta} - {os: macos-latest, version: 1.14.0, channel: conda-forge, rust: stable} - {os: windows-latest, version: 1.14.0, channel: conda-forge, rust: stable} - {os: ubuntu-latest, version: 1.14.0, channel: conda-forge, rust: stable} @@ -118,7 +117,7 @@ jobs: shell: bash -l {0} steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: {submodules: true} - name: Install Rust (${{matrix.rust}}) uses: dtolnay/rust-toolchain@stable @@ -155,7 +154,7 @@ jobs: - {os: macos, rust: stable} steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: {submodules: true} - name: Install Rust (${{matrix.rust}}) uses: dtolnay/rust-toolchain@stable @@ -167,13 +166,16 @@ jobs: if: matrix.rust != 'stable-gnu' - name: Run examples run: | - cargo r --example simple --features hdf5-sys/static,hdf5-sys/zlib,lzf,blosc-all - cargo r --example chunking --features hdf5-sys/static,hdf5-sys/zlib,lzf,blosc-all + cargo r --example simple --features ${{ env.FEATURES }} + cargo r --example chunking --features ${{ env.FEATURES }} + cargo r --example continuously_rw_1d --features ${{ env.FEATURES }} if: matrix.rust != 'stable-gnu' + env: + FEATURES: hdf5-sys/static,hdf5-sys/zlib,lzf,blosc-all apt: name: apt - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: @@ -182,7 +184,7 @@ jobs: - {mpi: openmpi, rust: stable} steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: {submodules: true} - name: Install Rust (${{matrix.rust}}) uses: dtolnay/rust-toolchain@stable @@ -192,6 +194,7 @@ jobs: [ "${{matrix.mpi}}" == "mpich" ] && PACKAGES="libhdf5-mpich-dev mpich" [ "${{matrix.mpi}}" == "openmpi" ] && PACKAGES="libhdf5-openmpi-dev openmpi-bin" [ "${{matrix.mpi}}" == "serial" ] && PACKAGES="libhdf5-dev" + sudo apt-get update sudo apt-get install $PACKAGES - name: Build and test all crates run: | @@ -214,7 +217,7 @@ jobs: version: ["1.8", "1.10", "1.12", "1.14"] steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: {submodules: true} - name: Install Rust (${{matrix.rust}}) uses: dtolnay/rust-toolchain@stable @@ -260,7 +263,7 @@ jobs: # rust: [stable] # steps: # - name: Checkout repository - # uses: actions/checkout@v4 + # uses: actions/checkout@v6 # with: {submodules: true} # - name: Install Rust (${{matrix.rust}}) # uses: dtolnay/rust-toolchain@stable @@ -279,22 +282,22 @@ jobs: msrv: name: Minimal Supported Rust Version - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: fail-fast: false steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: {submodules: true} - name: Install Rust uses: dtolnay/rust-toolchain@stable - with: {toolchain: "1.80"} + with: {toolchain: "1.85.1"} - name: Create lock-file run: cargo update - - name: Override deps - run: - cargo update half@2.5.0 --precise 2.4.1 + # - name: Override deps + # run: | + # cargo update half@2.7.1 --precise 2.4.1 - name: Build and test all crates run: cargo test --locked --workspace -vv --features=hdf5-sys/static,hdf5-sys/zlib --exclude=hdf5-metno-derive @@ -304,7 +307,7 @@ jobs: # runs-on: ubuntu-latest # steps: # - name: Checkout repository - # uses: actions/checkout@v4 + # uses: actions/checkout@v6 # with: {submodules: true} # - name: Install Rust # uses: dtolnay/rust-toolchain@stable @@ -321,7 +324,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: {submodules: true} - name: Install Rust uses: dtolnay/rust-toolchain@stable diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5e800ea45..0e0b13633 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: submodules: true - name: Install libhdf5 diff --git a/.gitignore b/.gitignore index 2fb61a58a..36ac75c55 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,5 @@ target/ *.ipynb* .idea/ sweep.timestamp +.cargo/ +.cargo/config.toml \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 9cd5b8f8c..b6a12a8d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,88 @@ ## hdf5-sys unreleased ## hdf5-src unreleased +## hdf5 0.12.4 +Release date: Mar 23, 2026 +- Support for hdf5 2.1.0 +- MSRV has been increased to 1.85 + +## hdf5-sys 0.11.3 +Release date: Mar 23, 2026 +- Support for hdf5 2.1.0 + +## hdf5-src 0.10.2 +Release date: Mar 23, 2026 +- Update to hdf5 2.1.0 + +## hdf5 v0.12.3 +Release date: Feb 10, 2026 +- Fixed compression of VarLenUnicode resulting in errors + +## hdf5 v0.12.2 +Release date: Feb 09, 2026 +- Fixed MPIO include + +## hdf5-sys v0.11.2 +Release date: Feb 09, 2026 +- Fixed name of MPIO include + +## hdf5-src v0.10.1 +Release date: Feb 09, 2026 +- Fixed name of cmake build option + +## hdf5-sys v0.11.1 +Release date: Jan 20, 2026 +- Fixed an import for windows using prebuilt hdf5 2.0.0 + +## hdf5 v0.12.1 +Release date: Jan 20, 2026 +- No changes + +## hdf5 v0.12.0 +Release date: Jan 19, 2026 +- Add support for hdf5 2.0.0 +- Added support for ZFP compression filters as an optional feature + +## hdf5-types v0.11.0 +Release date: Jan 19, 2026 +- Add support for hdf5 2.0.0 + +## hdf5-derive v0.10.0 +Release date: Jan 19, 2026 +- Add support for hdf5 2.0.0 + +## hdf5-sys v0.11.0 +Release date: Jan 19, 2026 +- Add support for hdf5 2.0.0 +- Remove `H5FD_*_init` functions, these are private to hdf5-c + +## hdf5-src v0.10.0 +Release date: Jan 19, 2026 +- Use hdf5 2.0.0 + +## hdf5 v0.11.0 +Release date: Nov 23, 2025 +- Fixed incorrect retrieved name of attributes +- Added support for Single Writer Multiple Readers (SWMR) (breaking change, OpenMode has extra variant) +- Added support for object copying between Locations +- Improved debug printing for datatype descriptor + +## hdf5-types v0.10.2 +Release date: Nov 23, 2025 +- Added documentation on Typedescriptor::size + +## hdf5-derive v0.9.3 +Release date: Nov 23, 2025 +- Fixed derive macro when inside itself or hdf5_metno package + +## hdf5-src v0.9.5 +Release date: Nov 23, 2025 +- Improved support for static linking with msvc + +## hdf5 v0.10.2 (yanked) +Release date: Oct 16, 2025 +- Upgraded upper bound of ndarray + ## hdf5-types v0.10.1 Release date: Mar 19, 2025 - Fixed deref of null ptr in `VarLenAscii`/`VarLenUnicode` diff --git a/Cargo.toml b/Cargo.toml index ae8717633..34b76e57c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ members = ["hdf5", "hdf5-types", "hdf5-derive", "hdf5-sys", "hdf5-src"] default-members = ["hdf5", "hdf5-types", "hdf5-derive", "hdf5-sys"] [workspace.package] -rust-version = "1.80.0" +rust-version = "1.85.1" authors = [ "Ivan Smirnov ", "Magnus Ulimoen ", @@ -24,9 +24,28 @@ libz-sys = { version = "1.1", default-features = false } mpi-sys = "0.2" num-complex = { version = "0.4", default-features = false } regex = "1.10" + # internal -hdf5 = { package = "hdf5-metno", version = "0.9.3", path = "hdf5" } # !V -hdf5-derive = { package = "hdf5-metno-derive", version = "0.9.1", path = "hdf5-derive" } # !V -hdf5-src = { package = "hdf5-metno-src", version = "0.9.3", path = "hdf5-src" } # !V -hdf5-sys = { package = "hdf5-metno-sys", version = "0.10.1", path = "hdf5-sys" } # !V -hdf5-types = { package = "hdf5-metno-types", version = "0.10.0", path = "hdf5-types" } # !V +hdf5 = { package = "hdf5-metno", version = "0.12.0", path = "hdf5" } # !V +hdf5-derive = { package = "hdf5-metno-derive", version = "0.10.0", path = "hdf5-derive" } # !V +hdf5-types = { package = "hdf5-metno-types", version = "0.11.0", path = "hdf5-types" } # !V +hdf5-src = { package = "hdf5-metno-src", version = "0.10.0", path = "hdf5-src" } # !V +hdf5-sys = { package = "hdf5-metno-sys", version = "0.11.3", path = "hdf5-sys" } # !V + + +[profile.dev] +# Fast compile, reasonable runtime +opt-level = 0 # no optimizations = much faster compilation +debug = 1 # keep minimal debug info (0 is smaller, 1 is a nice compromise) +incremental = true # reuse previous compilation work +codegen-units = 16 # more parallel codegen units = faster compiles +lto = "off" # no link-time optimization +panic = "unwind" # default, keeps backtraces usable +overflow-checks = true # keep checks for safety in dev + +[profile.test] +# Inherit dev as a baseline, then tweak for tests +inherits = "dev" +opt-level = 0 # keep tests compiling fast +debug = 1 # enough info for backtraces +overflow-checks = true diff --git a/README.md b/README.md index cc828eda9..60342dd77 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,9 @@ fn main() -> Result<()> { } ``` +You can find this [example][readme-example] as well as other example projects in +the [example directory][examples]. + ## Compatibility ### Platforms @@ -206,3 +209,7 @@ Few things to note when building on Windows: `hdf5` crate is primarily distributed under the terms of both the MIT license and the Apache License (Version 2.0). See [LICENSE-APACHE](LICENSE-APACHE) and [LICENSE-MIT](LICENSE-MIT) for details. + + +[readme-example]: https://github.com/metno/hdf5-rust/blob/main/hdf5/examples/simple.rs +[examples]: https://github.com/metno/hdf5-rust/tree/main/hdf5/examples \ No newline at end of file diff --git a/hdf5-derive/Cargo.toml b/hdf5-derive/Cargo.toml index 48ea8860d..95b9a1aff 100644 --- a/hdf5-derive/Cargo.toml +++ b/hdf5-derive/Cargo.toml @@ -3,7 +3,7 @@ name = "hdf5-metno-derive" description = "Derive macro for HDF5 structs and enums." categories = ["development-tools::procedural-macro-helpers"] readme = "README.md" -version = "0.9.2" # !V +version = "0.10.0" # !V rust-version.workspace = true authors.workspace = true keywords.workspace = true diff --git a/hdf5-derive/src/lib.rs b/hdf5-derive/src/lib.rs index 41592f7ef..90a3b2c57 100644 --- a/hdf5-derive/src/lib.rs +++ b/hdf5-derive/src/lib.rs @@ -12,6 +12,7 @@ use syn::{ TypeGenerics, TypePath, }; +/// Derive macro generating an impl of the trait `H5Type`. #[proc_macro_derive(H5Type, attributes(hdf5))] #[proc_macro_error] pub fn derive(input: proc_macro::TokenStream) -> proc_macro::TokenStream { @@ -21,13 +22,27 @@ pub fn derive(input: proc_macro::TokenStream) -> proc_macro::TokenStream { let body = impl_trait(&name, &input.data, &input.attrs, &ty_generics); // Determine name of parent crate, even if renamed using "package" + // CARGO_CRATE_NAME is the name of the actual crate being compiled (e.g., "simple" for examples) + // If it matches the library name (hdf5_metno), we're compiling the library itself + let is_library = + std::env::var("CARGO_CRATE_NAME").map(|name| name == "hdf5_metno").unwrap_or(false); + let crate_name = match proc_macro_crate::crate_name("hdf5-metno").unwrap() { - proc_macro_crate::FoundCrate::Itself => quote!(::hdf5_metno), + proc_macro_crate::FoundCrate::Itself if is_library => { + // We're in the hdf5-metno library itself, use crate + quote!(crate) + } + proc_macro_crate::FoundCrate::Itself => { + // We're in an example/test of hdf5-metno, use the renamed path + quote!(::hdf5_metno) + } proc_macro_crate::FoundCrate::Name(name) => { + // We're in a different crate that depends on hdf5-metno let ident = Ident::new(&name, Span::call_site()); quote!( ::#ident ) } }; + let expanded = quote! { #[allow(dead_code, unused_variables, unused_attributes)] const _: () = { @@ -108,7 +123,7 @@ fn impl_enum(names: &[String], values: &[Expr], repr: &Ident) -> TokenStream { fn is_phantom_data(ty: &Type) -> bool { match *ty { Type::Path(TypePath { qself: None, ref path }) => { - path.segments.iter().last().is_some_and(|x| x.ident == "PhantomData") + path.segments.iter().next_back().is_some_and(|x| x.ident == "PhantomData") } _ => false, } diff --git a/hdf5-src/Cargo.toml b/hdf5-src/Cargo.toml index c863daef9..f4584ef91 100644 --- a/hdf5-src/Cargo.toml +++ b/hdf5-src/Cargo.toml @@ -1,28 +1,52 @@ [package] name = "hdf5-metno-src" -license-file = "ext/hdf5/COPYING" +license-file = "ext/hdf5/LICENSE" build = "build.rs" description = "Build script for compiling HDF5 C library from source." links = "hdf5src" readme = "README.md" categories = ["development-tools::ffi"] -exclude = [ - "ext/hdf5/bin/**", - "ext/hdf5/c++/**", - "ext/hdf5/examples/**", - "ext/hdf5/fortran/**", - "ext/hdf5/java/**", - "ext/hdf5/release_docs/**", - "ext/hdf5/test/**", - "ext/hdf5/testpar/**", - "ext/hdf5/tools/**", - "ext/hdf5/hl/test/**", - "ext/hdf5/hl/tools/**", - "ext/hdf5/hl/examples/**", - "ext/hdf5/HDF5Examples/**", - "ext/hdf5/doxygen/**", +include = [ + "Cargo.toml", + "build.rs", + "README.md", + "src/**/*.rs", + "ext/hdf5/**/CMakeLists.txt", + "ext/hdf5/LICENSE", + "ext/hdf5/**/*.h", + "ext/hdf5/**/*.c", + "ext/hdf5/**/*.am", + "ext/hdf5/**/*.cmake", + "ext/hdf5/**/*.in", + "ext/hdf5/config/*-warnings", + "ext/hdf5/config/*flags", + "ext/hdf5/config/apple", + "ext/hdf5/config/cygwin", + "ext/hdf5/config/freebsd", + "ext/hdf5/config/ibm-aix", + "ext/hdf5/config/linux-gnu", + "ext/hdf5/config/netbsd", + "ext/hdf5/config/solaris", + "ext/hdf5/release_docs/USING_HDF5_CMake.txt", + "ext/hdf5/release_docs/RELEASE.txt", + "!ext/hdf5/bin/**", + "!ext/hdf5/c++/**", + "!ext/hdf5/examples/**", + "!ext/hdf5/fortran/**", + "!ext/hdf5/java/**", + "!ext/hdf5/release_docs/**", + "!ext/hdf5/test/**", + "!ext/hdf5/testpar/**", + "!ext/hdf5/tools/**", + "!ext/hdf5/hl/test/**", + "!ext/hdf5/hl/tools/**", + "!ext/hdf5/hl/fortran/**", + "!ext/hdf5/hl/c++/**", + "!ext/hdf5/hl/examples/**", + "!ext/hdf5/HDF5Examples/**", + "!ext/hdf5/doxygen/**", ] -version = "0.9.4" # !V +version = "0.10.2" # !V rust-version.workspace = true authors.workspace = true keywords.workspace = true diff --git a/hdf5-src/build.rs b/hdf5-src/build.rs index 699adaaf9..96c536531 100644 --- a/hdf5-src/build.rs +++ b/hdf5-src/build.rs @@ -33,6 +33,13 @@ fn main() { println!("cargo::rerun-if-changed=build.rs"); let mut cfg = cmake::Config::new("ext/hdf5"); + if cfg!(target_env = "msvc") { + cfg.define("CMAKE_POLICY_DEFAULT_CMP0091", "NEW"); + if let Ok(var) = env::var("CMAKE_MSVC_RUNTIME_LIBRARY") { + cfg.define("CMAKE_MSVC_RUNTIME_LIBRARY", var); + } + } + // only build the static c library, disable everything else cfg.define("HDF5_NO_PACKAGES", "ON"); for option in &[ @@ -54,7 +61,7 @@ fn main() { for option in &[ "HDF5_ENABLE_DEPRECATED_SYMBOLS", "HDF5_ENABLE_THREADSAFE", - "ALLOW_UNSUPPORTED", + "HDF5_ALLOW_UNSUPPORTED", "HDF5_BUILD_HL_LIB", "HDF5_ENABLE_NONSTANDARD_FEATURE_FLOAT16", "HDF5_ENABLE_SZIP_SUPPORT", @@ -67,7 +74,7 @@ fn main() { let mut zlib_header = env::split_paths(&zlib_include_dir).next().unwrap(); zlib_header.push("zlib.h"); let zlib_lib = "z"; - cfg.define("HDF5_ENABLE_Z_LIB_SUPPORT", "ON") + cfg.define("HDF5_ENABLE_ZLIB_SUPPORT", "ON") .define("H5_ZLIB_HEADER", &zlib_header) .define("ZLIB_STATIC_LIBRARY", zlib_lib); println!("cargo::metadata=zlib_header={}", zlib_header.to_str().unwrap()); @@ -84,22 +91,16 @@ fn main() { cfg.define("HDF5_ENABLE_THREADSAFE", "ON"); if feature_enabled("HL") { println!("cargo::warning=Unsupported HDF5 options: hl with threadsafe."); - cfg.define("ALLOW_UNSUPPORTED", "ON"); + cfg.define("HDF5_ALLOW_UNSUPPORTED", "ON"); } } let targeting_windows = env::var("CARGO_CFG_TARGET_OS").unwrap() == "windows"; - let debug_postfix = if targeting_windows { "_D" } else { "_debug" }; if feature_enabled("HL") { cfg.define("HDF5_BUILD_HL_LIB", "ON"); - let mut hdf5_hl_lib = + let hdf5_hl_lib = if cfg!(target_env = "msvc") { "libhdf5_hl" } else { "hdf5_hl" }.to_owned(); - if let Ok(opt_level) = env::var("OPT_LEVEL") { - if opt_level == "0" { - hdf5_hl_lib.push_str(debug_postfix); - } - } println!("cargo::metadata=hl_library={}", hdf5_hl_lib); } @@ -117,12 +118,7 @@ fn main() { let hdf5_incdir = format!("{}/include", dst.display()); println!("cargo::metadata=include={}", hdf5_incdir); - let mut hdf5_lib = if cfg!(target_env = "msvc") { "libhdf5" } else { "hdf5" }.to_owned(); - if let Ok(opt_level) = env::var("OPT_LEVEL") { - if opt_level == "0" { - hdf5_lib.push_str(debug_postfix); - } - } + let hdf5_lib = if cfg!(target_env = "msvc") { "libhdf5" } else { "hdf5" }.to_owned(); println!("cargo::metadata=library={}", hdf5_lib); } diff --git a/hdf5-src/ext/hdf5 b/hdf5-src/ext/hdf5 index 0fe0459fc..24176582e 160000 --- a/hdf5-src/ext/hdf5 +++ b/hdf5-src/ext/hdf5 @@ -1 +1 @@ -Subproject commit 0fe0459fc24d71be13d5f266513c2832b525671b +Subproject commit 24176582e5e4093f02399d726d04f25db1772fe3 diff --git a/hdf5-sys/Cargo.toml b/hdf5-sys/Cargo.toml index 6813b0f6e..1663481eb 100644 --- a/hdf5-sys/Cargo.toml +++ b/hdf5-sys/Cargo.toml @@ -5,7 +5,7 @@ description = "Native bindings to the HDF5 library." links = "hdf5" readme = "README.md" categories = ["development-tools::ffi", "filesystem", "science"] -version = "0.10.1" # !V +version = "0.11.3" # !V rust-version.workspace = true authors.workspace = true keywords.workspace = true @@ -41,7 +41,7 @@ pkg-config = "0.3" [target.'cfg(windows)'.build-dependencies] serde = "1.0" serde_derive = "1.0" -winreg = { version = "0.55", features = ["serialization-serde"] } +winreg = { version = "0.56", features = ["serialization-serde"] } [package.metadata.docs.rs] features = ["static", "zlib"] diff --git a/hdf5-sys/README.md b/hdf5-sys/README.md index 228e12215..611504f3e 100644 --- a/hdf5-sys/README.md +++ b/hdf5-sys/README.md @@ -9,7 +9,7 @@ This crate supports linking to a static build of HDF5. The HDF5 C library is bui via the `hdf5-src` crate which is then linked in when the `static` feature is set. See below for a list of supported options for static builds. -As of the time of writing, the version of the HDF5 library that is built is 1.10.5, +As of the time of writing, the version of the HDF5 library that is built is 2.0.0, but it may be incremented later. ## Crate features diff --git a/hdf5-sys/build.rs b/hdf5-sys/build.rs index 142b1df9a..94e8f3a01 100644 --- a/hdf5-sys/build.rs +++ b/hdf5-sys/build.rs @@ -14,7 +14,7 @@ use std::process::Command; use regex::Regex; fn feature_enabled(feature: &str) -> bool { - env::var(format!("CARGO_FEATURE_{}", feature)).is_ok() + env::var(format!("CARGO_FEATURE_{feature}")).is_ok() } #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Default)] @@ -30,7 +30,8 @@ impl Version { } pub fn parse(s: &str) -> Option { - let re = Regex::new(r"^(1)\.(8|10|12|14)\.(\d\d?)(_|.\d+)?((-|.)(patch)?\d+)?$").ok()?; + let re = + Regex::new(r"^(1|2)\.(0|1|8|10|12|14)\.(\d\d?)(_|.\d+)?((-|.)(patch)?\d+)?$").ok()?; let captures = re.captures(s)?; Some(Self { major: captures.get(1).and_then(|c| c.as_str().parse::().ok())?, @@ -53,10 +54,11 @@ impl Debug for Version { fn known_hdf5_versions() -> Vec { // Keep up to date with known_hdf5_versions in hdf5 let mut vs = Vec::new(); + vs.extend((0..=1).map(|v| Version::new(2, v, 0))); vs.extend((5..=21).map(|v| Version::new(1, 8, v))); // 1.8.[5-23] vs.extend((0..=8).map(|v| Version::new(1, 10, v))); // 1.10.[0-10] vs.extend((0..=2).map(|v| Version::new(1, 12, v))); // 1.12.[0-2] - vs.extend((0..=5).map(|v| Version::new(1, 14, v))); // 1.14.[0-5] + vs.extend((0..=6).map(|v| Version::new(1, 14, v))); // 1.14.[0-6] vs } @@ -130,7 +132,7 @@ fn validate_runtime_version(config: &Config) { println!("Adding extra link paths (ld)..."); for caps in re.captures_iter(&ldv) { let path = &caps["path"]; - println!(" {}", path); + println!(" {path}"); link_paths.push(path.into()); } } else { @@ -144,10 +146,10 @@ fn validate_runtime_version(config: &Config) { if let Some(filename) = path.file_name() { let filename = filename.to_str().unwrap_or(""); if path.is_file() && libfiles.contains(&filename) { - println!("Attempting to load: {:?}", path); + println!("Attempting to load: {path:?}"); match get_runtime_version_single(&path) { Ok(version) => { - println!(" => runtime version = {:?}", version); + println!(" => runtime version = {version:?}"); if version == config.header.version { println!("HDF5 library runtime version matches headers."); return; @@ -158,7 +160,7 @@ fn validate_runtime_version(config: &Config) { ); } Err(err) => { - println!(" => {}", err); + println!(" => {err}"); } } } @@ -185,7 +187,7 @@ impl Header { let inc_dir = inc_dir.as_ref(); let header = get_conf_header(inc_dir); - println!("Parsing HDF5 config from:\n {:?}", header); + println!("Parsing HDF5 config from:\n {header:?}"); let contents = fs::read_to_string(header).unwrap(); let mut hdr = Self::default(); @@ -217,7 +219,7 @@ impl Header { if let Some(version) = Version::parse(value) { hdr.version = version; } else { - panic!("Invalid H5_VERSION: {:?}", value); + panic!("Invalid H5_VERSION: {value:?}"); } }; } @@ -274,11 +276,11 @@ mod pkgconf { println!("Found HDF5 pkg-config entry"); println!(" Include paths:"); for dir in &library.include_paths { - println!(" - {:?}", dir); + println!(" - {dir:?}"); } println!(" Link paths:"); for dir in &library.link_paths { - println!(" - {:?}", dir); + println!(" - {dir:?}"); } for dir in &library.include_paths { if is_inc_dir(dir) { @@ -289,7 +291,7 @@ mod pkgconf { } if let Some(ref inc_dir) = config.inc_dir { println!("Located HDF5 headers at:"); - println!(" {:?}", inc_dir); + println!(" {inc_dir:?}"); } else { println!("Unable to locate HDF5 headers from pkg-config info."); } @@ -314,8 +316,8 @@ mod unix { ("/usr/include", "/usr/lib64"), ] { if is_inc_dir(inc_dir) { - println!("Found HDF5 headers at:\n {:?}", inc_dir); - println!("Adding to link path:\n {:?}", lib_dir); + println!("Found HDF5 headers at:\n {inc_dir:?}"); + println!("Adding to link path:\n {lib_dir:?}"); config.inc_dir = Some(inc_dir.into()); config.link_paths.push(lib_dir.into()); break; @@ -334,19 +336,25 @@ mod macos { } // We have to explicitly support homebrew since the HDF5 bottle isn't // packaged with pkg-config metadata. - let (v18, v110, v112, v114) = if let Some(version) = config.version { + let (v21, v20, v18, v110, v112, v114) = if let Some(version) = config.version { ( + version.major == 2 && version.minor == 1, + version.major == 2 && version.minor == 0, version.major == 1 && version.minor == 8, version.major == 1 && version.minor == 10, version.major == 1 && version.minor == 12, version.major == 1 && version.minor == 14, ) } else { - (false, false, false, false) + (false, false, false, false, false, false) }; println!( "Attempting to find HDF5 via Homebrew ({})...", - if v18 { + if v21 { + "2.1.*" + } else if v20 { + "2.0.*" + } else if v18 { "1.8.*" } else if v110 { "1.10.*" @@ -358,6 +366,20 @@ mod macos { "any version" } ); + if !(v18 || v110 || v112 || v114 || v20) { + if let Some(out) = run_command("brew", &["--prefix", "hdf5@2.1"]) { + if is_root_dir(&out) { + config.inc_dir = Some(PathBuf::from(out).join("include")); + } + } + } + if !(v18 || v110 || v112 || v114) { + if let Some(out) = run_command("brew", &["--prefix", "hdf5@2.0"]) { + if is_root_dir(&out) { + config.inc_dir = Some(PathBuf::from(out).join("include")); + } + } + } if !(v18 || v110 || v112) { if let Some(out) = run_command("brew", &["--prefix", "hdf5@1.14"]) { if is_root_dir(&out) { @@ -395,7 +417,7 @@ mod macos { } if let Some(ref inc_dir) = config.inc_dir { println!("Found Homebrew HDF5 headers at:"); - println!(" {:?}", inc_dir); + println!(" {inc_dir:?}"); } } } @@ -462,7 +484,7 @@ mod windows { fn get_hdf5_app(version: Option) -> Option { if let Some(version) = version { - println!("Searching for installed HDF5 with version {:?}...", version); + println!("Searching for installed HDF5 with version {version:?}..."); } else { println!("Searching for installed HDF5 (any version)...") } @@ -484,7 +506,7 @@ mod windows { } if apps.len() > 1 { println!("Selecting the latest version ({:?}):", latest.version); - println!("- {:?}", latest); + println!("- {latest:?}"); } Some(latest.clone()) } @@ -510,7 +532,7 @@ mod windows { for path in env::split_paths(&var_path) { if let Ok(path) = path.canonicalize() { if path == bin_dir { - println!("Found in PATH: {:?}", path); + println!("Found in PATH: {path:?}"); return; } } @@ -525,7 +547,7 @@ impl LibrarySearcher { let mut config = Self::default(); if let Some(var) = env::var_os("HDF5_DIR") { println!("Setting HDF5 root from environment variable:"); - println!(" HDF5_DIR = {:?}", var); + println!(" HDF5_DIR = {var:?}"); let root = PathBuf::from(var); assert!(!root.is_relative(), "HDF5_DIR cannot be relative."); @@ -542,7 +564,7 @@ impl LibrarySearcher { let alt_inc_dir = root_dir.join("Library").join("include"); if !is_inc_dir(inc_dir) && is_inc_dir(&alt_inc_dir) { println!("Detected MSVC conda environment, changing headers dir to:"); - println!(" {:?}", alt_inc_dir); + println!(" {alt_inc_dir:?}"); config.inc_dir = Some(alt_inc_dir); } } @@ -550,11 +572,11 @@ impl LibrarySearcher { } if let Ok(var) = env::var("HDF5_VERSION") { println!("Setting HDF5 version from environment variable:"); - println!(" HDF5_VERSION = {:?}", var); + println!(" HDF5_VERSION = {var:?}"); if let Some(v) = Version::parse(&var) { config.version = Some(v); } else { - panic!("Invalid HDF5 version: {}", var); + panic!("Invalid HDF5 version: {var}"); } } config @@ -583,10 +605,10 @@ impl LibrarySearcher { if self.user_provided_dir { let lib_dir = format!("{}/lib", envdir.to_string_lossy()); println!("Custom HDF5_DIR provided; rpath can be set via:"); - println!(" RUSTFLAGS=\"-C link-args=-Wl,-rpath,{}\"", lib_dir); + println!(" RUSTFLAGS=\"-C link-args=-Wl,-rpath,{lib_dir}\""); if cfg!(target_os = "macos") { println!("On some OS X installations, you may also need to set:"); - println!(" DYLD_FALLBACK_LIBRARY_PATH=\"{}\"", lib_dir); + println!(" DYLD_FALLBACK_LIBRARY_PATH=\"{lib_dir}\""); } } } @@ -598,7 +620,7 @@ impl LibrarySearcher { pub fn finalize(self) -> Config { if let Some(ref inc_dir) = self.inc_dir { - assert!(is_inc_dir(inc_dir), "Invalid HDF5 headers directory: {:?}", inc_dir); + assert!(is_inc_dir(inc_dir), "Invalid HDF5 headers directory: {inc_dir:?}"); let mut link_paths = self.link_paths; if link_paths.is_empty() { if let Some(root_dir) = inc_dir.parent() { @@ -704,7 +726,7 @@ impl Config { for (flag, feature, native) in [ (!h.have_no_deprecated, "deprecated", "HDF5_ENABLE_DEPRECATED_SYMBOLS"), (h.have_threadsafe, "threadsafe", "HDF5_ENABLE_THREADSAFE"), - (h.have_filter_deflate, "zlib", "HDF5_ENABLE_Z_LIB_SUPPORT"), + (h.have_filter_deflate, "zlib", "HDF5_ENABLE_ZLIB_SUPPORT"), ] { if feature_enabled(&feature.to_ascii_uppercase()) { assert!( @@ -723,7 +745,7 @@ fn main() { let mut searcher = LibrarySearcher::new_from_env(); searcher.try_locate_hdf5_library(); let config = searcher.finalize(); - println!("{:#?}", config); + println!("{config:#?}"); config.emit_link_flags(); config.emit_cfg_flags(); write_hdf5_version(config.header.version); diff --git a/hdf5-sys/src/h5d.rs b/hdf5-sys/src/h5d.rs index 2a83e4149..fef3cd4fd 100644 --- a/hdf5-sys/src/h5d.rs +++ b/hdf5-sys/src/h5d.rs @@ -27,33 +27,22 @@ pub enum H5D_layout_t { H5D_NLAYOUTS = 3, } -impl Default for H5D_layout_t { - fn default() -> Self { - Self::H5D_CONTIGUOUS - } -} - pub type H5D_chunk_index_t = c_uint; pub const H5D_CHUNK_BTREE: H5D_chunk_index_t = 0; pub const H5D_CHUNK_IDX_BTREE: H5D_chunk_index_t = H5D_CHUNK_BTREE; #[repr(C)] -#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Debug)] +#[derive(Copy, Clone, Default, PartialEq, Eq, PartialOrd, Debug)] pub enum H5D_alloc_time_t { H5D_ALLOC_TIME_ERROR = -1, + #[default] H5D_ALLOC_TIME_DEFAULT = 0, H5D_ALLOC_TIME_EARLY = 1, H5D_ALLOC_TIME_LATE = 2, H5D_ALLOC_TIME_INCR = 3, } -impl Default for H5D_alloc_time_t { - fn default() -> Self { - Self::H5D_ALLOC_TIME_DEFAULT - } -} - #[repr(C)] #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Debug)] pub enum H5D_space_status_t { @@ -64,35 +53,25 @@ pub enum H5D_space_status_t { } #[repr(C)] -#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Debug)] +#[derive(Copy, Clone, Default, PartialEq, Eq, PartialOrd, Debug)] pub enum H5D_fill_time_t { H5D_FILL_TIME_ERROR = -1, H5D_FILL_TIME_ALLOC = 0, H5D_FILL_TIME_NEVER = 1, + #[default] H5D_FILL_TIME_IFSET = 2, } -impl Default for H5D_fill_time_t { - fn default() -> Self { - Self::H5D_FILL_TIME_IFSET - } -} - #[repr(C)] -#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Debug)] +#[derive(Copy, Clone, Default, PartialEq, Eq, PartialOrd, Debug)] pub enum H5D_fill_value_t { H5D_FILL_VALUE_ERROR = -1, H5D_FILL_VALUE_UNDEFINED = 0, + #[default] H5D_FILL_VALUE_DEFAULT = 1, H5D_FILL_VALUE_USER_DEFINED = 2, } -impl Default for H5D_fill_value_t { - fn default() -> Self { - Self::H5D_FILL_VALUE_DEFAULT - } -} - #[repr(C)] #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Debug)] pub enum H5D_mpio_actual_chunk_opt_mode_t { @@ -218,10 +197,11 @@ mod hdf5_1_10_0 { use super::*; #[repr(C)] - #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Debug)] + #[derive(Copy, Clone, Default, PartialEq, Eq, PartialOrd, Debug)] pub enum H5D_layout_t { H5D_LAYOUT_ERROR = -1, H5D_COMPACT = 0, + #[default] H5D_CONTIGUOUS = 1, H5D_CHUNKED = 2, H5D_VIRTUAL = 3, @@ -229,19 +209,14 @@ mod hdf5_1_10_0 { } #[repr(C)] - #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Debug)] + #[derive(Copy, Clone, Default, PartialEq, Eq, PartialOrd, Debug)] pub enum H5D_vds_view_t { H5D_VDS_ERROR = -1, H5D_VDS_FIRST_MISSING = 0, + #[default] H5D_VDS_LAST_AVAILABLE = 1, } - impl Default for H5D_vds_view_t { - fn default() -> Self { - Self::H5D_VDS_LAST_AVAILABLE - } - } - pub const H5D_CHUNK_DONT_FILTER_PARTIAL_CHUNKS: c_uint = 0x0002; pub type H5D_append_cb_t = Option< @@ -265,14 +240,24 @@ pub use self::hdf5_1_10_0::*; #[cfg(feature = "1.10.3")] extern "C" { - pub fn H5Dread_chunk( + #[cfg_attr(not(feature = "2.0.0"), link_name = "H5Dread_chunk")] + pub fn H5Dread_chunk1( dset_id: hid_t, dxpl_id: hid_t, offset: *const hsize_t, filters: *mut u32, buf: *mut c_void, ) -> herr_t; + #[cfg(feature = "2.0.0")] + pub fn H5Dread_chunk2( + dset_id: hid_t, dxpl_id: hid_t, offset: *const hsize_t, filters: *mut u32, + buf: *mut c_void, buf_size: *mut size_t, + ) -> herr_t; pub fn H5Dwrite_chunk( dset_id: hid_t, dxpl_id: hid_t, filters: u32, offset: *const hsize_t, data_size: size_t, buf: *const c_void, ) -> herr_t; } +#[cfg(all(feature = "1.10.3", not(feature = "2.0.0")))] +pub use self::H5Dread_chunk1 as H5Dread_chunk; +#[cfg(feature = "2.0.0")] +pub use self::H5Dread_chunk2 as H5Dread_chunk; #[cfg(feature = "1.10.5")] extern "C" { diff --git a/hdf5-sys/src/h5e.rs b/hdf5-sys/src/h5e.rs index 8e9399d32..6d60ed033 100644 --- a/hdf5-sys/src/h5e.rs +++ b/hdf5-sys/src/h5e.rs @@ -323,6 +323,10 @@ mod globals { extern_static!(H5E_ID, H5E_ID_g); #[cfg(feature = "1.14.0")] extern_static!(H5E_UNMOUNT, H5E_UNMOUNT_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5E_RTREE, H5E_RTREE_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5E_THREADSAFE, H5E_THREADSAFE_g); } #[cfg(all(target_env = "msvc", not(feature = "static")))] @@ -505,4 +509,8 @@ mod globals { extern_static!(H5E_ID, __imp_H5E_ID_g); #[cfg(feature = "1.14.0")] extern_static!(H5E_UNMOUNT, __imp_H5E_UNMOUNT_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5E_RTREE, __imp_H5E_RTREE_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5E_THREADSAFE, __imp_H5E_THREADSAFE_g); } diff --git a/hdf5-sys/src/h5f.rs b/hdf5-sys/src/h5f.rs index 092651d5c..3a27780a5 100644 --- a/hdf5-sys/src/h5f.rs +++ b/hdf5-sys/src/h5f.rs @@ -54,20 +54,15 @@ pub enum H5F_scope_t { } #[repr(C)] -#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Debug)] +#[derive(Copy, Clone, Default, PartialEq, Eq, PartialOrd, Debug)] pub enum H5F_close_degree_t { + #[default] H5F_CLOSE_DEFAULT = 0, H5F_CLOSE_WEAK = 1, H5F_CLOSE_SEMI = 2, H5F_CLOSE_STRONG = 3, } -impl Default for H5F_close_degree_t { - fn default() -> Self { - Self::H5F_CLOSE_DEFAULT - } -} - #[cfg_attr(feature = "1.10.0", deprecated(note = "deprecated in HDF5 1.10.0, use H5F_info2_t"))] #[repr(C)] #[derive(Debug, Copy, Clone)] @@ -130,11 +125,19 @@ pub enum H5F_libver_t { H5F_LIBVER_V112 = 3, #[cfg(feature = "1.14.0")] H5F_LIBVER_V114 = 4, + #[cfg(feature = "2.0.0")] + H5F_LIBVER_V200 = 5, H5F_LIBVER_NBOUNDS, } -#[cfg(feature = "1.10.2")] +#[cfg(all(feature = "1.10.2", not(feature = "1.12.0")))] pub const H5F_LIBVER_LATEST: H5F_libver_t = H5F_LIBVER_V110; +#[cfg(all(feature = "1.12.0", not(feature = "1.14.0")))] +pub const H5F_LIBVER_LATEST: H5F_libver_t = H5F_LIBVER_V112; +#[cfg(all(feature = "1.14.0", not(feature = "2.0.0")))] +pub const H5F_LIBVER_LATEST: H5F_libver_t = H5F_LIBVER_V114; +#[cfg(feature = "2.0.0")] +pub const H5F_LIBVER_LATEST: H5F_libver_t = H5F_LIBVER_V200; impl Default for H5F_libver_t { fn default() -> Self { diff --git a/hdf5-sys/src/h5fd.rs b/hdf5-sys/src/h5fd.rs index 157d247ae..d90b0c2d8 100644 --- a/hdf5-sys/src/h5fd.rs +++ b/hdf5-sys/src/h5fd.rs @@ -364,26 +364,11 @@ extern "C" { pub fn H5FDtruncate(file: *mut H5FD_t, dxpl_id: hid_t, closing: hbool_t) -> herr_t; } -// drivers -extern "C" { - pub fn H5FD_sec2_init() -> hid_t; - pub fn H5FD_core_init() -> hid_t; - pub fn H5FD_stdio_init() -> hid_t; - pub fn H5FD_family_init() -> hid_t; - pub fn H5FD_log_init() -> hid_t; - pub fn H5FD_multi_init() -> hid_t; -} - -#[cfg(feature = "have-parallel")] +#[cfg(all(not(feature = "2.0.0"), feature = "have-parallel"))] extern "C" { pub fn H5FD_mpio_init() -> hid_t; } -#[cfg(feature = "have-direct")] -extern "C" { - pub fn H5FD_direct_init() -> hid_t; -} - #[cfg(feature = "1.10.0")] extern "C" { pub fn H5FDlock(file: *mut H5FD_t, rw: hbool_t) -> herr_t; @@ -485,5 +470,40 @@ extern "C" { pub fn H5FDdelete(name: *const c_char, fapl_id: hid_t) -> herr_t; pub fn H5FDis_driver_registered_by_name(driver_name: *const c_char) -> htri_t; pub fn H5FDis_driver_registered_by_value(driver_value: H5FD_class_value_t) -> htri_t; + #[cfg(not(feature = "2.0.0"))] pub fn H5FDperform_init(p: H5FD_perform_init_func_t) -> hid_t; } + +#[cfg(all(feature = "2.0.0", not(all(target_env = "msvc", not(feature = "static")))))] +mod globals_2_0_0 { + pub use crate::h5i::hid_t as id_t; + extern_static!(H5FD_CORE, H5FD_CORE_id_g); + extern_static!(H5FD_FAMILY, H5FD_FAMILY_id_g); + extern_static!(H5FD_LOG, H5FD_LOG_id_g); + extern_static!(H5FD_MULTI, H5FD_MULTI_id_g); + extern_static!(H5FD_ONION, H5FD_ONION_id_g); + extern_static!(H5FD_SEC2, H5FD_SEC2_id_g); + extern_static!(H5FD_SPLITTER, H5FD_SPLITTER_id_g); + extern_static!(H5FD_STDIO, H5FD_STDIO_id_g); + #[cfg(feature = "have-parallel")] + extern_static!(H5FD_MPIO, H5FD_MPIO_id_g); +} + +#[cfg(all(feature = "2.0.0", all(target_env = "msvc", not(feature = "static"))))] +mod globals_2_0_0 { + // dllimport hack + pub type id_t = usize; + extern_static!(H5FD_CORE, __imp_H5FD_CORE_id_g); + extern_static!(H5FD_FAMILY, __imp_H5FD_FAMILY_id_g); + extern_static!(H5FD_LOG, __imp_H5FD_LOG_id_g); + extern_static!(H5FD_MULTI, __imp_H5FD_MULTI_id_g); + extern_static!(H5FD_ONION, __imp_H5FD_ONION_id_g); + extern_static!(H5FD_SEC2, __imp_H5FD_SEC2_id_g); + extern_static!(H5FD_SPLITTER, __imp_H5FD_SPLITTER_id_g); + extern_static!(H5FD_STDIO, __imp_H5FD_STDIO_id_g); + #[cfg(feature = "have-parallel")] + extern_static!(H5FD_MPIO, __imp_H5FD_MPIO_id_g); +} + +#[cfg(feature = "2.0.0")] +pub use globals_2_0_0::*; diff --git a/hdf5-sys/src/h5i.rs b/hdf5-sys/src/h5i.rs index 9fb1578a8..6423cd3b8 100644 --- a/hdf5-sys/src/h5i.rs +++ b/hdf5-sys/src/h5i.rs @@ -62,9 +62,12 @@ extern "C" { pub fn H5Iinc_ref(id: hid_t) -> c_int; pub fn H5Idec_ref(id: hid_t) -> c_int; pub fn H5Iget_ref(id: hid_t) -> c_int; - pub fn H5Iregister_type( + #[cfg_attr(not(feature = "2.0.0"), link_name = "H5Iregister_type")] + pub fn H5Iregister_type1( hash_size: size_t, reserved: c_uint, free_func: H5I_free_t, ) -> H5I_type_t; + #[cfg(feature = "2.0.0")] + pub fn H5Iregister_type2(reserved: c_uint, free_func: H5I_free_t) -> H5I_type_t; pub fn H5Iclear_type(type_: H5I_type_t, force: hbool_t) -> herr_t; pub fn H5Idestroy_type(type_: H5I_type_t) -> herr_t; pub fn H5Iinc_type_ref(type_: H5I_type_t) -> c_int; @@ -78,6 +81,11 @@ extern "C" { pub fn H5Iis_valid(id: hid_t) -> htri_t; } +#[cfg(not(feature = "2.0.0"))] +pub use self::H5Iregister_type1 as H5Iregister_type; +#[cfg(feature = "2.0.0")] +pub use self::H5Iregister_type2 as H5Iregister_type; + #[cfg(feature = "1.14.0")] pub type H5I_future_realize_func_t = Option< unsafe extern "C" fn(future_object: *mut c_void, actual_object_id: *mut hid_t) -> herr_t, diff --git a/hdf5-sys/src/h5p.rs b/hdf5-sys/src/h5p.rs index b6e46744e..90fa123b0 100644 --- a/hdf5-sys/src/h5p.rs +++ b/hdf5-sys/src/h5p.rs @@ -526,11 +526,11 @@ extern "C" { ) -> herr_t; // mpi-io - #[cfg(feature = "mpio")] + #[cfg(all(feature = "have-parallel", feature = "mpio"))] pub fn H5Pset_fapl_mpio( fapl_id: hid_t, comm: mpi_sys::MPI_Comm, info: mpi_sys::MPI_Info, ) -> herr_t; - #[cfg(feature = "mpio")] + #[cfg(all(feature = "have-parallel", feature = "mpio"))] pub fn H5Pget_fapl_mpio( fapl_id: hid_t, comm: *mut mpi_sys::MPI_Comm, info: *mut mpi_sys::MPI_Info, ) -> herr_t; @@ -811,3 +811,9 @@ extern "C" { plist_id: hid_t, driver_value: H5FD_class_value_t, driver_config: *const c_char, ) -> herr_t; } + +#[cfg(feature = "2.0.0")] +extern "C" { + pub fn H5Pget_virtual_spatial_tree(dcpl_id: hid_t, use_tree: *mut hbool_t) -> herr_t; + pub fn H5Pset_virtual_spatial_tree(dapl_id: hid_t, use_tree: hbool_t) -> herr_t; +} diff --git a/hdf5-sys/src/h5t.rs b/hdf5-sys/src/h5t.rs index eabad5b3b..d5221cb28 100644 --- a/hdf5-sys/src/h5t.rs +++ b/hdf5-sys/src/h5t.rs @@ -36,7 +36,14 @@ pub enum H5T_class_t { H5T_ENUM = 8, H5T_VLEN = 9, H5T_ARRAY = 10, + + #[cfg(not(feature = "2.0.0"))] H5T_NCLASSES = 11, + + #[cfg(feature = "2.0.0")] + H5T_COMPLEX = 11, + #[cfg(feature = "2.0.0")] + H5T_NCLASSES = 12, } #[cfg(feature = "1.8.6")] @@ -81,9 +88,10 @@ pub enum H5T_norm_t { } #[repr(C)] -#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Debug)] +#[derive(Copy, Clone, Default, PartialEq, Eq, PartialOrd, Debug)] pub enum H5T_cset_t { H5T_CSET_ERROR = -1, + #[default] H5T_CSET_ASCII = 0, H5T_CSET_UTF8 = 1, H5T_CSET_RESERVED_2 = 2, @@ -102,12 +110,6 @@ pub enum H5T_cset_t { H5T_CSET_RESERVED_15 = 15, } -impl Default for H5T_cset_t { - fn default() -> Self { - Self::H5T_CSET_ASCII - } -} - pub const H5T_NCSET: H5T_cset_t = H5T_CSET_RESERVED_2; #[repr(C)] @@ -267,7 +269,10 @@ extern "C" { pub fn H5Tget_create_plist(type_id: hid_t) -> hid_t; pub fn H5Tcommitted(type_id: hid_t) -> htri_t; pub fn H5Tencode(obj_id: hid_t, buf: *mut c_void, nalloc: *mut size_t) -> herr_t; - pub fn H5Tdecode(buf: *const c_void) -> hid_t; + #[cfg_attr(not(feature = "2.0.0"), link_name = "H5Tdecode")] + pub fn H5Tdecode1(buf: *const c_void) -> hid_t; + #[cfg(feature = "2.0.0")] + pub fn H5Tdecode2(buf: *const c_void, buf_size: size_t) -> hid_t; pub fn H5Tinsert( parent_id: hid_t, name: *const c_char, offset: size_t, member_id: hid_t, ) -> herr_t; @@ -349,6 +354,11 @@ extern "C" { pub fn H5Tget_array_dims1(type_id: hid_t, dims: *mut hsize_t, perm: *mut c_int) -> c_int; } +#[cfg(not(feature = "2.0.0"))] +pub use self::H5Tdecode1 as H5Tdecode; +#[cfg(feature = "2.0.0")] +pub use self::H5Tdecode2 as H5Tdecode; + pub use self::globals::*; #[cfg(not(all(target_env = "msvc", not(feature = "static"))))] @@ -443,6 +453,32 @@ mod globals { extern_static!(H5T_NATIVE_UINT_FAST64, H5T_NATIVE_UINT_FAST64_g); #[cfg(feature = "1.12.0")] extern_static!(H5T_STD_REF, H5T_STD_REF_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_COMPLEX_IEEE_F16BE, H5T_COMPLEX_IEEE_F16BE_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_COMPLEX_IEEE_F16LE, H5T_COMPLEX_IEEE_F16LE_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_COMPLEX_IEEE_F32BE, H5T_COMPLEX_IEEE_F32BE_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_COMPLEX_IEEE_F32LE, H5T_COMPLEX_IEEE_F32LE_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_COMPLEX_IEEE_F64BE, H5T_COMPLEX_IEEE_F64BE_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_COMPLEX_IEEE_F64LE, H5T_COMPLEX_IEEE_F64LE_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_FLOAT_BFLOAT16BE, H5T_FLOAT_BFLOAT16BE_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_FLOAT_BFLOAT16LE, H5T_FLOAT_BFLOAT16LE_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_FLOAT_F8E4M3, H5T_FLOAT_F8E4M3_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_FLOAT_F8E5M2, H5T_FLOAT_F8E5M2_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_NATIVE_DOUBLE_COMPLEX, H5T_NATIVE_DOUBLE_COMPLEX_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_NATIVE_FLOAT_COMPLEX, H5T_NATIVE_FLOAT_COMPLEX_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_NATIVE_LDOUBLE_COMPLEX, H5T_NATIVE_LDOUBLE_COMPLEX_g); } #[cfg(all(target_env = "msvc", not(feature = "static")))] @@ -538,6 +574,32 @@ mod globals { extern_static!(H5T_NATIVE_UINT_FAST64, __imp_H5T_NATIVE_UINT_FAST64_g); #[cfg(feature = "1.12.0")] extern_static!(H5T_STD_REF, __imp_H5T_STD_REF_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_COMPLEX_IEEE_F16BE, __imp_H5T_COMPLEX_IEEE_F16BE_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_COMPLEX_IEEE_F16LE, __imp_H5T_COMPLEX_IEEE_F16LE_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_COMPLEX_IEEE_F32BE, __imp_H5T_COMPLEX_IEEE_F32BE_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_COMPLEX_IEEE_F32LE, __imp_H5T_COMPLEX_IEEE_F32LE_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_COMPLEX_IEEE_F64BE, __imp_H5T_COMPLEX_IEEE_F64BE_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_COMPLEX_IEEE_F64LE, __imp_H5T_COMPLEX_IEEE_F64LE_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_FLOAT_BFLOAT16BE, __imp_H5T_FLOAT_BFLOAT16BE_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_FLOAT_BFLOAT16LE, __imp_H5T_FLOAT_BFLOAT16LE_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_FLOAT_F8E4M3, __imp_H5T_FLOAT_F8E4M3_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_FLOAT_F8E5M2, __imp_H5T_FLOAT_F8E5M2_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_NATIVE_DOUBLE_COMPLEX, __imp_H5T_NATIVE_DOUBLE_COMPLEX_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_NATIVE_FLOAT_COMPLEX, __imp_H5T_NATIVE_FLOAT_COMPLEX_g); + #[cfg(feature = "2.0.0")] + extern_static!(H5T_NATIVE_LDOUBLE_COMPLEX, __imp_H5T_NATIVE_LDOUBLE_COMPLEX_g); } #[cfg(feature = "1.10.0")] @@ -567,3 +629,8 @@ extern "C" { name: *const c_char, tapl_id: hid_t, es_id: hid_t, ) -> hid_t; } + +#[cfg(feature = "2.0.0")] +extern "C" { + pub fn H5Tcomplex_create(base_type_id: hid_t) -> hid_t; +} diff --git a/hdf5-sys/src/h5vl.rs b/hdf5-sys/src/h5vl.rs index 56446df7d..2f85aa90a 100644 --- a/hdf5-sys/src/h5vl.rs +++ b/hdf5-sys/src/h5vl.rs @@ -1923,14 +1923,42 @@ mod v1_14_0 { } extern "C" { + #[cfg(not(feature = "2.0.0"))] + pub fn H5VLstart_lib_state() -> herr_t; + #[cfg(not(feature = "2.0.0"))] pub fn H5VLfinish_lib_state() -> herr_t; pub fn H5VLintrospect_get_cap_flags( info: *const c_void, connector_id: hid_t, cap_flags: *mut c_uint, ) -> herr_t; - pub fn H5VLstart_lib_state() -> herr_t; } extern "C" { pub fn H5VLobject_is_native(obj_id: hid_t, is_native: *mut hbool_t) -> herr_t; } } + +#[cfg(feature = "2.0.0")] +extern "C" { + pub fn H5VLclose_lib_context(context: *mut c_void) -> herr_t; + pub fn H5VLopen_lib_context(context: *mut *mut c_void) -> herr_t; +} + +#[cfg(not(all(target_env = "msvc", not(feature = "static"))))] +#[cfg(feature = "2.0.0")] +mod globals { + pub use crate::h5i::hid_t as id_t; + extern_static!(H5VL_NATIVE, H5VL_NATIVE_g); + extern_static!(H5VL_PASSTHRU, H5VL_PASSTHRU_g); +} + +#[cfg(all(target_env = "msvc", not(feature = "static")))] +#[cfg(feature = "2.0.0")] +mod globals { + // dllimport hack + pub type id_t = usize; + extern_static!(H5VL_NATIVE, __imp_H5VL_NATIVE_g); + extern_static!(H5VL_PASSTHRU, __imp_H5VL_PASSTHRU_g); +} + +#[cfg(feature = "2.0.0")] +pub use globals::*; diff --git a/hdf5-types/Cargo.toml b/hdf5-types/Cargo.toml index 8d489835b..861168b3d 100644 --- a/hdf5-types/Cargo.toml +++ b/hdf5-types/Cargo.toml @@ -4,7 +4,7 @@ description = "Native Rust equivalents of HDF5 types." readme = "README.md" build = "build.rs" categories = ["encoding"] -version = "0.10.1" # !V +version = "0.11.0" # !V rust-version.workspace = true authors.workspace = true keywords.workspace = true diff --git a/hdf5-types/src/array.rs b/hdf5-types/src/array.rs index c6438e2b9..1345bd1b4 100644 --- a/hdf5-types/src/array.rs +++ b/hdf5-types/src/array.rs @@ -5,6 +5,7 @@ use std::ops::Deref; use std::ptr; use std::slice; +/// A variable-length array. #[repr(C)] pub struct VarLenArray { len: usize, @@ -13,6 +14,14 @@ pub struct VarLenArray { } impl VarLenArray { + /// Creates a `VarLenArray` by copying the first `len` elements stored at `p`. + /// + /// Returns an empty array if `p` is null. + /// + /// # Safety + /// + /// - `p` must be valid for reads of `len * size_of::()` bytes. + /// - `p` must point to `len` consecutive properly initialized and aligned values of type `T`. pub unsafe fn from_parts(p: *const T, len: usize) -> Self { let (len, ptr) = if !p.is_null() && len != 0 { let dst = crate::malloc(len * mem::size_of::()); @@ -24,26 +33,31 @@ impl VarLenArray { Self { len, ptr: ptr as *const _, tag: PhantomData } } + /// Creates a `VarLenArray` from a slice by copying its elements. #[inline] pub fn from_slice(arr: &[T]) -> Self { unsafe { Self::from_parts(arr.as_ptr(), arr.len()) } } + /// Returns a raw pointer to the array's buffer. #[inline] pub fn as_ptr(&self) -> *const T { self.ptr } + /// Returns the number of elements in the array. #[inline] pub fn len(&self) -> usize { self.len as _ } + /// Returns `true` if the array has a length of zero. #[inline] pub fn is_empty(&self) -> bool { self.len == 0 } + /// Returns a slice containing the entire array. #[inline] pub fn as_slice(&self) -> &[T] { self diff --git a/hdf5-types/src/dyn_value.rs b/hdf5-types/src/dyn_value.rs index 23a09357a..f7b39d90a 100644 --- a/hdf5-types/src/dyn_value.rs +++ b/hdf5-types/src/dyn_value.rs @@ -1,3 +1,5 @@ +//! Dynamically-typed values. + use std::fmt::{self, Debug, Display}; use std::mem; use std::ptr; @@ -26,6 +28,7 @@ unsafe trait DynClone { fn dyn_clone(&mut self, out: &mut [u8]); } +/// A dynamically-typed integer. #[derive(Copy, Clone, PartialEq, Eq)] pub enum DynInteger { Int8(i8), @@ -114,6 +117,7 @@ impl From for DynValue<'_> { } } +/// A dynamically-typed floating-point value. #[derive(Copy, Clone, PartialEq)] pub enum DynFloat { #[cfg(feature = "f16")] @@ -173,6 +177,7 @@ impl From for DynValue<'_> { } } +/// A dynamically-typed scalar value. #[derive(Copy, Clone, PartialEq)] pub enum DynScalar { Integer(DynInteger), @@ -212,6 +217,7 @@ impl From for DynValue<'static> { } } +/// A dynamically-typed enumeration value. #[derive(Copy, Clone)] pub struct DynEnum<'a> { tp: &'a EnumType, @@ -269,6 +275,7 @@ impl<'a> From> for DynValue<'a> { } } +/// A dynamically-typed compound value. pub struct DynCompound<'a> { tp: &'a CompoundType, buf: &'a [u8], @@ -279,7 +286,7 @@ impl<'a> DynCompound<'a> { Self { tp, buf } } - pub fn iter(&self) -> impl Iterator { + pub fn iter(&self) -> impl Iterator)> { self.tp.fields.iter().map(move |field| { ( field.name.as_ref(), @@ -354,6 +361,7 @@ impl<'a> From> for DynValue<'a> { } } +/// A dynamically-typed array. pub struct DynArray<'a> { tp: &'a TypeDescriptor, buf: &'a [u8], @@ -379,7 +387,7 @@ impl<'a> DynArray<'a> { } } - pub fn iter(&self) -> impl Iterator { + pub fn iter(&self) -> impl Iterator> { let ptr = self.get_ptr(); let len = self.get_len(); let size = self.tp.size(); @@ -470,6 +478,7 @@ impl<'a> From> for DynValue<'a> { } } +/// A fixed-length string with a dynamic encoding. pub struct DynFixedString<'a> { buf: &'a [u8], unicode: bool, @@ -529,6 +538,7 @@ impl<'a> From> for DynValue<'a> { } } +/// A variable-length string with a dynamic encoding. pub struct DynVarLenString<'a> { buf: &'a [u8], unicode: bool, @@ -633,6 +643,7 @@ impl<'a> From> for DynValue<'a> { } } +/// A dynamically-typed string. #[derive(PartialEq, Eq)] pub enum DynString<'a> { Fixed(DynFixedString<'a>), @@ -677,6 +688,7 @@ impl<'a> From> for DynValue<'a> { } } +/// A borrowed value with dynamic type. #[derive(PartialEq)] pub enum DynValue<'a> { Scalar(DynScalar), @@ -687,6 +699,7 @@ pub enum DynValue<'a> { } impl<'a> DynValue<'a> { + /// Constructs a new `DynValue` from a `TypeDescriptor` and a byte slice. pub fn new(tp: &'a TypeDescriptor, buf: &'a [u8]) -> Self { use TypeDescriptor::*; debug_assert_eq!(tp.size(), buf.len()); @@ -749,12 +762,14 @@ impl Display for DynValue<'_> { } } +/// An owned value with dynamic type. pub struct OwnedDynValue { tp: TypeDescriptor, buf: Box<[u8]>, } impl OwnedDynValue { + /// Constructs a new `OwnedDynValue` from the given value. pub fn new(value: T) -> Self { let ptr = (&value as *const T).cast::(); let len = mem::size_of_val(&value); @@ -763,10 +778,12 @@ impl OwnedDynValue { Self { tp: T::type_descriptor(), buf: buf.to_owned().into_boxed_slice() } } - pub fn get(&self) -> DynValue { + /// Returns a borrowed version of the contained value. + pub fn get(&self) -> DynValue<'_> { DynValue::new(&self.tp, &self.buf) } + /// Returns the value's type descriptor. pub fn type_descriptor(&self) -> &TypeDescriptor { &self.tp } @@ -781,9 +798,12 @@ impl OwnedDynValue { Self { tp, buf } } - /// Cast to the concrete type + /// Tries to downcast the value to a concrete type. + /// + /// # Errors /// - /// Will fail if the type-descriptors are not equal + /// If the type descriptors of `self` and `T` are not equal, this will fail and return a + /// `Result::Err` containing the original value. pub fn cast(mut self) -> Result { use mem::MaybeUninit; if self.tp != T::type_descriptor() { diff --git a/hdf5-types/src/h5type.rs b/hdf5-types/src/h5type.rs index ad18b7e42..fcaf65757 100644 --- a/hdf5-types/src/h5type.rs +++ b/hdf5-types/src/h5type.rs @@ -14,15 +14,21 @@ pub(crate) struct hvl_t { pub ptr: *mut c_void, } +/// A valid integer size. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] pub enum IntSize { + /// 1 byte. U1 = 1, + /// 2 bytes. U2 = 2, + /// 4 bytes. U4 = 4, + /// 8 bytes. U8 = 8, } impl IntSize { + /// Returns an `IntSize` of `size` bytes, or `None` if `size` is invalid. pub const fn from_int(size: usize) -> Option { if size == 1 { Some(Self::U1) @@ -38,15 +44,20 @@ impl IntSize { } } +/// A valid floating-point number size. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] pub enum FloatSize { + /// 2 bytes. #[cfg(feature = "f16")] U2 = 2, + /// 4 bytes. U4 = 4, + /// 8 bytes. U8 = 8, } impl FloatSize { + /// Returns a `FloatSize` of `size` bytes, or `None` if `size` is invalid. pub const fn from_int(size: usize) -> Option { #[cfg(feature = "f16")] { @@ -62,20 +73,28 @@ impl FloatSize { } } +/// A descriptor for an enumeration datatype member. #[derive(Clone, Debug, PartialEq, Eq)] pub struct EnumMember { + /// The name of the member. pub name: String, + /// The value of the member. pub value: u64, } +/// A descriptor for an enumeration datatype. #[derive(Clone, Debug, PartialEq, Eq)] pub struct EnumType { + /// The size of the underlying integer type. pub size: IntSize, + /// Whether to use a signed integer. pub signed: bool, + /// The enumeration datatype members. pub members: Vec, } impl EnumType { + /// Returns the type descriptor of the underlying integer datatype. #[inline] pub fn base_type(&self) -> TypeDescriptor { if self.signed { @@ -86,31 +105,42 @@ impl EnumType { } } +/// A descriptor for a compound datatype field. #[derive(Clone, Debug, PartialEq, Eq)] pub struct CompoundField { + /// The name of the field. pub name: String, + /// The type of the field. pub ty: TypeDescriptor, + /// The byte offset of the field. pub offset: usize, + /// The ordering of the field within the compound type. pub index: usize, } impl CompoundField { + /// Creates a new `CompoundField`. pub fn new(name: &str, ty: TypeDescriptor, offset: usize, index: usize) -> Self { Self { name: name.to_owned(), ty, offset, index } } + /// Creates a new `CompoundField` for a concrete type. pub fn typed(name: &str, offset: usize, index: usize) -> Self { Self::new(name, T::type_descriptor(), offset, index) } } +/// A descriptor for a compound datatype. #[derive(Clone, Debug, PartialEq, Eq)] pub struct CompoundType { + /// The fields of the datatype. pub fields: Vec, + /// The size in bytes of the datatype. pub size: usize, } impl CompoundType { + /// Converts `self` to a C struct representation. pub fn to_c_repr(&self) -> Self { let mut layout = self.clone(); layout.fields.sort_by_key(|f| f.index); @@ -133,6 +163,7 @@ impl CompoundType { layout } + /// Converts `self` to a packed representation. pub fn to_packed_repr(&self) -> Self { let mut layout = self.clone(); layout.fields.sort_by_key(|f| f.index); @@ -146,19 +177,32 @@ impl CompoundType { } } +/// A descriptor for an HDF5 datatype. #[derive(Clone, Debug, PartialEq, Eq)] pub enum TypeDescriptor { + /// A signed integer. Integer(IntSize), + /// An unsigned integer. Unsigned(IntSize), + /// A floating-point number. Float(FloatSize), + /// A boolean value. Boolean, + /// An enumeration datatype. Enum(EnumType), + /// A compound datatype. Compound(CompoundType), + /// A fixed-length array. FixedArray(Box, usize), + /// A fixed-length ASCII string. FixedAscii(usize), + /// A fixed-length UTF-8 string. FixedUnicode(usize), + /// A variable-length array. VarLenArray(Box), + /// A variable-length ASCII string. VarLenAscii, + /// A variable-length UTF-8 string. VarLenUnicode, Reference(Reference), } @@ -181,10 +225,10 @@ impl Display for TypeDescriptor { TypeDescriptor::Boolean => write!(f, "bool"), TypeDescriptor::Enum(ref tp) => write!(f, "enum ({})", tp.base_type()), TypeDescriptor::Compound(ref tp) => write!(f, "compound ({} fields)", tp.fields.len()), - TypeDescriptor::FixedArray(ref tp, n) => write!(f, "[{}; {}]", tp, n), - TypeDescriptor::FixedAscii(n) => write!(f, "string (len {})", n), - TypeDescriptor::FixedUnicode(n) => write!(f, "unicode (len {})", n), - TypeDescriptor::VarLenArray(ref tp) => write!(f, "[{}] (var len)", tp), + TypeDescriptor::FixedArray(ref tp, n) => write!(f, "[{tp}; {n}]"), + TypeDescriptor::FixedAscii(n) => write!(f, "string (len {n})"), + TypeDescriptor::FixedUnicode(n) => write!(f, "unicode (len {n})"), + TypeDescriptor::VarLenArray(ref tp) => write!(f, "[{tp}] (var len)"), TypeDescriptor::VarLenAscii => write!(f, "string (var len)"), TypeDescriptor::VarLenUnicode => write!(f, "unicode (var len)"), TypeDescriptor::Reference(Reference::Object) => write!(f, "reference (object)"), @@ -196,6 +240,7 @@ impl Display for TypeDescriptor { } impl TypeDescriptor { + /// Returns the size of the [`TypeDescriptor`] variant in bytes pub fn size(&self) -> usize { match *self { Self::Integer(size) | Self::Unsigned(size) => size as _, @@ -223,6 +268,7 @@ impl TypeDescriptor { } } + /// Converts `self` to a C-compatible representation. pub fn to_c_repr(&self) -> Self { match *self { Self::Compound(ref compound) => Self::Compound(compound.to_c_repr()), @@ -232,6 +278,7 @@ impl TypeDescriptor { } } + /// Converts `self` to a packed representation. pub fn to_packed_repr(&self) -> Self { match *self { Self::Compound(ref compound) => Self::Compound(compound.to_packed_repr()), @@ -242,7 +289,18 @@ impl TypeDescriptor { } } +/// A type that can be represented as an HDF5 datatype. +/// +/// # Derivable +/// This trait can be used with `#[derive]`. +/// +/// To `derive` on structs, they must be one of: `repr(C)`, `repr(packed)`, or `repr(transparent)`, +/// and have at least one field, all of which must implement `H5Type`. +/// +/// To `derive` on enums, they must have an explicit `repr` and at least one variant, all of which +/// must be unit variants with explicit discriminants. pub unsafe trait H5Type: 'static { + /// Returns a descriptor for an equivalent HDF5 datatype. fn type_descriptor() -> TypeDescriptor; } diff --git a/hdf5-types/src/lib.rs b/hdf5-types/src/lib.rs index a1daa8337..550c28ba7 100644 --- a/hdf5-types/src/lib.rs +++ b/hdf5-types/src/lib.rs @@ -8,10 +8,10 @@ //! //! Crate features: //! * `h5-alloc`: Use the `hdf5` allocator for varlen types and dynamic values. -//! This is necessary on platforms which uses different allocators -//! in different libraries (e.g. dynamic libraries on windows), -//! or if `hdf5-c` is compiled with the MEMCHECKER option. -//! This option is forced on in the case of using a `windows` DLL. +//! This is necessary on platforms which uses different allocators +//! in different libraries (e.g. dynamic libraries on windows), +//! or if `hdf5-c` is compiled with the MEMCHECKER option. +//! This option is forced on in the case of using a `windows` DLL. #[cfg(test)] #[macro_use] @@ -54,6 +54,7 @@ pub(crate) unsafe fn free(ptr: *mut core::ffi::c_void) { } } +/// Whether this crate is using the HDF5 library for allocations instead of `libc`. pub const USING_H5_ALLOCATOR: bool = { cfg_if::cfg_if! { if #[cfg(any(feature = "h5-alloc", windows_dll))] { diff --git a/hdf5-types/src/string.rs b/hdf5-types/src/string.rs index 0efc278f3..42377237f 100644 --- a/hdf5-types/src/string.rs +++ b/hdf5-types/src/string.rs @@ -11,6 +11,7 @@ use std::str::{self, FromStr}; use ascii::{AsAsciiStr, AsAsciiStrError, AsciiStr}; +/// Errors that can occur when attempting to interpret a byte sequence as a string. #[derive(Clone, Copy, PartialEq, Eq, Debug)] #[non_exhaustive] pub enum StringError { @@ -36,7 +37,7 @@ impl fmt::Display for StringError { StringError::InsufficientCapacity => { write!(f, "string error: insufficient capacity for fixed sized string") } - StringError::AsciiError(err) => write!(f, "string error: {}", err), + StringError::AsciiError(err) => write!(f, "string error: {err}"), } } } @@ -181,6 +182,7 @@ impl_string_traits!(VarLenUnicode); // ================================================================================ +/// A variable-length ASCII string. #[repr(C)] pub struct VarLenAscii { ptr: *mut u8, @@ -203,6 +205,7 @@ impl Clone for VarLenAscii { } impl VarLenAscii { + /// Creates a new empty `VarLenAscii` string. #[inline] pub fn new() -> Self { unsafe { @@ -220,6 +223,7 @@ impl VarLenAscii { Self { ptr } } + /// Returns the length of `self`. #[inline] pub fn len(&self) -> usize { if self.ptr.is_null() { @@ -230,16 +234,19 @@ impl VarLenAscii { unsafe { libc::strlen(self.ptr as *const _) } } + /// Returns `true` if `self` has a length of zero bytes. #[inline] pub fn is_empty(&self) -> bool { self.len() == 0 } + /// Returns a raw pointer to the string's buffer. #[inline] pub fn as_ptr(&self) -> *const u8 { self.ptr } + /// Returns the contents of the string as a byte slice. #[inline] pub fn as_bytes(&self) -> &[u8] { // Treat null as empty. This could be changed to option @@ -250,16 +257,28 @@ impl VarLenAscii { unsafe { slice::from_raw_parts(self.ptr as *const _, self.len()) } } + /// Returns the contents of the string as a string slice. #[inline] pub fn as_str(&self) -> &str { unsafe { str::from_utf8_unchecked(self.as_bytes()) } } + /// Converts a byte slice into a `VarLenAscii` without checking that the string contains only + /// non-zero ASCII bytes. + /// + /// # Safety + /// + /// The bytes must be valid ASCII and non-zero. #[inline] pub unsafe fn from_ascii_unchecked>(bytes: &B) -> Self { Self::from_bytes(bytes.as_ref()) } + /// Converts a byte slice into a `VarLenAscii`. + /// + /// # Errors + /// + /// Returns `Err` if the bytes are not valid ASCII, or if any byte is zero. pub fn from_ascii>(bytes: &B) -> Result { let bytes = bytes.as_ref(); if !bytes.iter().all(|&c| c != 0) { @@ -299,6 +318,7 @@ unsafe impl Sync for VarLenAscii {} // ================================================================================ +/// A variable-length UTF-8 string. #[repr(C)] pub struct VarLenUnicode { ptr: *mut u8, @@ -321,6 +341,7 @@ impl Clone for VarLenUnicode { } impl VarLenUnicode { + /// Creates a new empty `VarLenUnicode` string. #[inline] pub fn new() -> Self { unsafe { @@ -348,21 +369,25 @@ impl VarLenUnicode { libc::strlen(self.ptr as *const _) } + /// Returns the length of `self`. #[inline] pub fn len(&self) -> usize { self.as_str().len() } + /// Returns `true` if `self` has a length of zero bytes. #[inline] pub fn is_empty(&self) -> bool { unsafe { self.raw_len() == 0 } } + /// Returns a raw pointer to the string's buffer. #[inline] pub fn as_ptr(&self) -> *const u8 { self.ptr } + /// Returns the contents of the string as a byte slice. #[inline] pub fn as_bytes(&self) -> &[u8] { // Treat null as empty. This could be changed to option @@ -373,11 +398,18 @@ impl VarLenUnicode { unsafe { slice::from_raw_parts(self.ptr as *const _, self.raw_len()) } } + /// Returns the contents of the string as a string slice. #[inline] pub fn as_str(&self) -> &str { unsafe { str::from_utf8_unchecked(self.as_bytes()) } } + /// Converts a byte slice into a `VarLenUnicode` without checking that the string contains only + /// non-zero UTF-8 bytes. + /// + /// # Safety + /// + /// The bytes must be valid UTF-8 and non-zero. #[inline] pub unsafe fn from_str_unchecked>(s: S) -> Self { Self::from_bytes(s.borrow().as_bytes()) @@ -403,6 +435,7 @@ unsafe impl Sync for VarLenUnicode {} // ================================================================================ +/// A fixed-length ASCII string. #[repr(C)] #[derive(Copy, Clone)] pub struct FixedAscii { @@ -410,6 +443,7 @@ pub struct FixedAscii { } impl FixedAscii { + /// Creates a new empty `FixedAscii` string. #[inline] pub fn new() -> Self { unsafe { Self { buf: mem::zeroed() } } @@ -428,41 +462,59 @@ impl FixedAscii { unsafe { slice::from_raw_parts(self.buf.as_ptr(), N) } } + /// Returns the string's capacity in bytes. #[inline] pub const fn capacity() -> usize { N } + /// Returns the length of `self`. #[inline] pub fn len(&self) -> usize { self.as_raw_slice().iter().rev().skip_while(|&c| *c == 0).count() } + /// Returns `true` if `self` has a length of zero bytes. #[inline] pub fn is_empty(&self) -> bool { self.as_raw_slice().iter().all(|&c| c == 0) } + /// Returns a raw pointer to the string's buffer. #[inline] pub fn as_ptr(&self) -> *const u8 { self.buf.as_ptr() } + /// Returns the contents of the string as a byte slice. #[inline] pub fn as_bytes(&self) -> &[u8] { &self.as_raw_slice()[..self.len()] } + /// Returns the contents of the string as a string slice. #[inline] pub fn as_str(&self) -> &str { unsafe { str::from_utf8_unchecked(self.as_bytes()) } } + /// Converts a byte slice into a `FixedAscii` without checking that the string is valid ASCII, + /// and truncating at the type's capacity. + /// + /// # Safety + /// + /// The bytes must be valid ASCII. #[inline] pub unsafe fn from_ascii_unchecked>(bytes: &B) -> Self { Self::from_bytes(bytes.as_ref()) } + /// Converts a byte slice into a `FixedAscii`. + /// + /// # Errors + /// + /// Returns `Err` if the bytes are not valid ASCII or if the slice length is greater than the + /// type's capacity. pub fn from_ascii>(bytes: &B) -> Result { let bytes = bytes.as_ref(); if bytes.len() > N { @@ -497,6 +549,7 @@ impl AsAsciiStr for FixedAscii { // ================================================================================ +/// A fixed-length UTF-8 string. #[repr(C)] #[derive(Copy, Clone)] pub struct FixedUnicode { @@ -504,6 +557,7 @@ pub struct FixedUnicode { } impl FixedUnicode { + /// Creates a new empty `FixedUnicode` string. #[inline] pub fn new() -> Self { unsafe { Self { buf: mem::zeroed() } } @@ -527,36 +581,48 @@ impl FixedUnicode { self.as_raw_slice().iter().rev().skip_while(|&c| *c == 0).count() } + /// Returns the string's capacity in bytes. #[inline] pub const fn capacity() -> usize { N } + /// Returns the length of `self`. #[inline] pub fn len(&self) -> usize { self.as_str().len() } + /// Returns `true` if `self` has a length of zero bytes. #[inline] pub fn is_empty(&self) -> bool { self.raw_len() == 0 } + /// Returns a raw pointer to the string's buffer. #[inline] pub fn as_ptr(&self) -> *const u8 { self.buf.as_ptr() } + /// Returns the contents of the string as a byte slice. #[inline] pub fn as_bytes(&self) -> &[u8] { &self.as_raw_slice()[..self.raw_len()] } + /// Returns the contents of the string as a string slice. #[inline] pub fn as_str(&self) -> &str { unsafe { str::from_utf8_unchecked(self.as_bytes()) } } + /// Converts a byte slice into a `FixedUnicode` without checking that the string is valid UTF-8, + /// and truncating at the type's capacity. + /// + /// # Safety + /// + /// The bytes must be valid UTF-8. #[inline] pub unsafe fn from_str_unchecked>(s: S) -> Self { Self::from_bytes(s.borrow().as_bytes()) diff --git a/hdf5/Cargo.toml b/hdf5/Cargo.toml index 6733e9ff3..087ed2d5c 100644 --- a/hdf5/Cargo.toml +++ b/hdf5/Cargo.toml @@ -4,7 +4,7 @@ readme = "../README.md" description = "Thread-safe Rust bindings for the HDF5 library." build = "build.rs" categories = ["science", "filesystem"] -version = "0.10.1" # !V +version = "0.12.4" # !V rust-version.workspace = true authors.workspace = true keywords.workspace = true @@ -39,6 +39,8 @@ mpio = ["dep:mpi-sys", "hdf5-sys/mpio"] complex = ["hdf5-types/complex"] # Enable float16 type support. f16 = ["hdf5-types/f16"] +# Enable ZFP Support +zfp = ["dep:zfp-sys"] # The features with version numbers such as 1.10.3, 1.12.0 are metafeatures # and is only available when the HDF5 library is at least this version. @@ -51,12 +53,12 @@ bitflags = "2.4" blosc-sys = { version = "0.3", package = "blosc-src", optional = true } cfg-if = { workspace = true } errno = { version = "0.3", optional = true } -lazy_static = "1.4" libc = { workspace = true } lzf-sys = { version = "0.1", optional = true } mpi-sys = { workspace = true, optional = true } -ndarray = ">=0.15, <=0.16" +ndarray = ">=0.15, <=0.17" paste = "1.0" +zfp-sys = {version= "0.4.2", optional= true } # internal hdf5-derive = { workspace = true } hdf5-sys = { workspace = true } @@ -68,11 +70,10 @@ num-complex = { workspace = true } parking_lot = "0.12.3" paste = "1.0" pretty_assertions = "1.4" -rand = { version = "0.9", features = ["small_rng"] } +rand = { version = "0.10" } regex = { workspace = true } scopeguard = "1.2" tempfile = "3.9" [package.metadata.docs.rs] -features = ["static", "zlib", "blosc", "lzf", "f16", "complex"] -rustdoc-args = ["--cfg", "docsrs"] +features = ["static", "zlib", "blosc", "lzf", "f16", "complex","zfp"] diff --git a/hdf5/build.rs b/hdf5/build.rs index eea9ddc04..3f13d4865 100644 --- a/hdf5/build.rs +++ b/hdf5/build.rs @@ -16,10 +16,12 @@ impl Version { fn known_hdf5_versions() -> Vec { // Keep up to date with known_hdf5_versions in hdf5-sys let mut vs = Vec::new(); + vs.push(Version::new(2, 0, 0)); // 2.0.0 + vs.extend((0..=1).map(|v| Version::new(2, v, 0))); // 2.[0-1].0 vs.extend((5..=21).map(|v| Version::new(1, 8, v))); // 1.8.[5-23] vs.extend((0..=8).map(|v| Version::new(1, 10, v))); // 1.10.[0-10] vs.extend((0..=2).map(|v| Version::new(1, 12, v))); // 1.12.[0-2] - vs.extend((0..=5).map(|v| Version::new(1, 14, v))); // 1.14.[0-5] + vs.extend((0..=6).map(|v| Version::new(1, 14, v))); // 1.14.[0-6] vs } @@ -35,8 +37,8 @@ fn main() { } println!("cargo::rustc-check-cfg=cfg(msvc_dll_indirection)"); - let print_feature = |key: &str| println!("cargo::rustc-cfg=feature=\"{}\"", key); - let print_cfg = |key: &str| println!("cargo::rustc-cfg={}", key); + let print_feature = |key: &str| println!("cargo::rustc-cfg=feature=\"{key}\""); + let print_cfg = |key: &str| println!("cargo::rustc-cfg={key}"); for (key, _) in env::vars() { match key.as_str() { // public features diff --git a/hdf5/examples/continuously_rw_1d.rs b/hdf5/examples/continuously_rw_1d.rs new file mode 100644 index 000000000..8c68b8f50 --- /dev/null +++ b/hdf5/examples/continuously_rw_1d.rs @@ -0,0 +1,90 @@ +//! Create, and continuously write several chunks to a chunked 1D dataset +//! +//! Finally read it, get the chunking metadata, and use it to read slices aligned to the chunk size + +#[cfg(feature = "1.10.5")] +fn main() -> hdf5_metno::Result<()> { + example::write_hdf5()?; + example::read_hdf5()?; + Ok(()) +} + +#[cfg(not(feature = "1.10.5"))] +fn main() { + println!( + "needs version 1.10.5 or later for querying the number of chunks with H5Dget_num_chunks" + ); +} + +#[cfg(feature = "1.10.5")] +mod example { + use hdf5::{File, Result}; + use hdf5_metno::{self as hdf5, Extent}; + use ndarray::Array1; + + // Shared between read_hdf5() and write_hdf5(), otherwise they only rely on file metadata + const FILE_NAME: &str = "continuous_chunks.h5"; + const DATASET_NAME: &str = "count"; + + const CHUNK_SIZE: usize = 5; + const NUM_CHUNKS: usize = 3; // total chunks to write + + pub fn write_hdf5() -> Result<()> { + println!("Creating file '{FILE_NAME}' with 1D resizable dataset"); + let file = File::create(FILE_NAME)?; + let shape = Extent::resizable(0); // 1D resizable + let ds = + file.new_dataset::().chunk((CHUNK_SIZE,)).shape(shape).create(DATASET_NAME)?; + + // Simulate continuously accumulating data in a buffer + // and writing it to the dataset anytime there's enough to fill a chunk + let mut buf = Vec::with_capacity(CHUNK_SIZE); + for i in 0..NUM_CHUNKS * CHUNK_SIZE { + buf.push(i); + if buf.len() == CHUNK_SIZE { + let current_size = ds.size(); + println!("[{i}] writing new chunk, current size = {current_size}"); + ds.resize((current_size + CHUNK_SIZE,))?; + ds.write_slice(&buf, current_size..current_size + CHUNK_SIZE)?; + buf.clear(); + } + } + + Ok(()) + } + + pub fn read_hdf5() -> Result<()> { + println!("Reading file '{FILE_NAME}'"); + let file = File::open(FILE_NAME)?; + let ds = file.dataset(DATASET_NAME)?; + + // Check shape + let shape = ds.shape(); + println!("Dataset shape: {:?}", shape); + assert_eq!(shape, &[CHUNK_SIZE * NUM_CHUNKS]); + + // Get chunking metadata + let chunk_size = ds.chunk().unwrap()[0]; + println!("Chunk size: {chunk_size}"); + assert_eq!(chunk_size, CHUNK_SIZE); + + let num_chunks = ds.num_chunks().unwrap(); + println!("Number of chunks: {num_chunks}"); + assert_eq!(num_chunks, NUM_CHUNKS); + + // Read the dataset chunk for chunk + for chunk_idx in 0..num_chunks { + let chunk_start = chunk_idx * chunk_size; + let arr: Array1 = ds.read_slice(chunk_start..chunk_start + chunk_size)?; + println!("Dataset Chunk #{chunk_idx}: {arr:?}"); + assert_eq!(arr, Array1::from_iter(chunk_start..chunk_start + chunk_size)); + } + + // Read the full dataset in one go + let arr: Array1 = ds.read_1d()?; + println!("Full dataset: {arr:?}"); + assert_eq!(arr, Array1::from_iter(0..CHUNK_SIZE * NUM_CHUNKS)); + + Ok(()) + } +} diff --git a/hdf5/examples/swmr.rs b/hdf5/examples/swmr.rs new file mode 100644 index 000000000..b2912f6e7 --- /dev/null +++ b/hdf5/examples/swmr.rs @@ -0,0 +1,49 @@ +#[cfg(feature = "1.10.2")] +use hdf5_metno as hdf5; + +#[cfg(feature = "1.10.2")] +fn reader() { + let file = hdf5::File::open_as("swmr.h5", hdf5::OpenMode::ReadSWMR).unwrap(); + println!("Reader: Opened file"); + let var = file.dataset("foo").unwrap(); + + for _ in 0..5 { + var.refresh().unwrap(); + let shape = var.shape(); + println!("Reader: Got shape: {shape:?}"); + // If reading one should use the shape directly without + // using the convenience read_2d etc. functions which + // might get confused if the shape is changed during reading + std::thread::sleep(std::time::Duration::from_secs(5)); + } +} + +fn main() { + #[cfg(not(feature = "1.10.2"))] + println!("This examples requires hdf5 >= 1.10.2 to enable SWMR and set libver_bounds"); + + #[cfg(feature = "1.10.2")] + { + let file = hdf5::File::with_options() + .with_fapl(|fapl| fapl.libver_v110()) + .create("swmr.h5") + .unwrap(); + + let var = file.new_dataset::().shape((0.., 5)).create("foo").unwrap(); + + file.start_swmr().unwrap(); + println!("Writer: Wrote file"); + + let thread = std::thread::spawn(|| reader()); + + for i in 0..5 { + var.resize((i + 1, 5)).unwrap(); + var.write_slice(&[i, i, i, i, i], (i, 0..)).unwrap(); + var.flush().unwrap(); + println!("Writer: Wrote {i}"); + std::thread::sleep(std::time::Duration::from_secs(5)); + } + + thread.join().unwrap(); + } +} diff --git a/hdf5/src/class.rs b/hdf5/src/class.rs index 66a2c5314..2fcabdf07 100644 --- a/hdf5/src/class.rs +++ b/hdf5/src/class.rs @@ -81,6 +81,20 @@ pub trait ObjectClass: Sized { } } +/// Takes ownership of an object via its identifier. +/// +/// # Errors +/// +/// Returns an error if `id` does not refer to an object of type `T`. +/// +/// # Safety +/// +/// This should only be called with an identifier obtained from an object constructor in the HDF5 C +/// library. The reference count of a newly created object is 1, so this function creates a handle +/// for the object without incrementing its reference count. +/// +/// This function is unsafe because improper use may lead to the object being closed before all its +/// handles are dropped. pub unsafe fn from_id(id: hid_t) -> Result { T::from_id(id) } diff --git a/hdf5/src/error.rs b/hdf5/src/error.rs index e350a7294..3f100dd51 100644 --- a/hdf5/src/error.rs +++ b/hdf5/src/error.rs @@ -35,6 +35,7 @@ pub fn silence_errors(silence: bool) { h5lock!(silence_errors_no_sync(silence)); } +/// A stack of error records from an HDF5 library call. #[repr(transparent)] #[derive(Clone)] pub struct ErrorStack(Handle); @@ -107,6 +108,7 @@ impl ErrorStack { } } +/// An error record for an HDF5 library call. #[derive(Clone, Debug)] pub struct ErrorFrame { desc: String, @@ -127,19 +129,24 @@ impl ErrorFrame { } } + /// Returns the error description. pub fn desc(&self) -> &str { self.desc.as_ref() } + /// Returns a message with the error description and the relevant function name. pub fn description(&self) -> &str { self.description.as_ref() } + /// Returns a message with the error description and the relevant function name, file name, + /// and line number. pub fn detail(&self) -> Option { Some(format!("Error in {}(): {} [{}: {}]", self.func, self.desc, self.major, self.minor)) } } +/// A converted [`ErrorStack`] with methods to access [`ErrorFrame`] data. #[derive(Clone, Debug)] pub struct ExpandedErrorStack { frames: Vec, @@ -178,10 +185,12 @@ impl ExpandedErrorStack { } } + /// Returns the top [`ErrorFrame`] of the stack, or `None` if it is empty. pub fn top(&self) -> Option<&ErrorFrame> { self.first() } + /// Returns the description of the error on top of the stack. pub fn description(&self) -> &str { match self.description { None => "unknown library error", @@ -189,6 +198,7 @@ impl ExpandedErrorStack { } } + /// Returns a detailed message for the error on top of the stack, or `None` if it is empty. pub fn detail(&self) -> Option { self.top().and_then(ErrorFrame::detail) } diff --git a/hdf5/src/globals.rs b/hdf5/src/globals.rs index 07ac09a57..3eeb5e9d8 100644 --- a/hdf5/src/globals.rs +++ b/hdf5/src/globals.rs @@ -3,15 +3,11 @@ use std::mem; use std::sync::LazyLock; -use lazy_static::lazy_static; - #[cfg(feature = "have-direct")] -use hdf5_sys::h5fd::H5FD_direct_init; -#[cfg(feature = "have-parallel")] -use hdf5_sys::h5fd::H5FD_mpio_init; -use hdf5_sys::h5fd::{ - H5FD_core_init, H5FD_family_init, H5FD_log_init, H5FD_multi_init, H5FD_sec2_init, - H5FD_stdio_init, +use hdf5_sys::h5p::H5Pset_fapl_direct; +use hdf5_sys::h5p::{ + H5Pclose, H5Pcreate, H5Pget_driver, H5Pset_fapl_core, H5Pset_fapl_family, H5Pset_fapl_log, + H5Pset_fapl_multi, H5Pset_fapl_sec2, H5Pset_fapl_stdio, }; use hdf5_sys::{h5e, h5p, h5t}; @@ -46,6 +42,18 @@ macro_rules! link_hid { }; } +/// Fetches the driver ID using the workaround from https://github.com/HDFGroup/hdf5/issues/1809 +/// as the _init functions seem to be removed in HDF5 2.0.0 +macro_rules! get_driver { + ($set_driver:expr) => {{ + let fapl = h5call!(H5Pcreate(*H5P_FILE_ACCESS)).expect("should always create FAPL"); + h5call!($set_driver(fapl)).expect("should always be able to set the driver"); + let id = h5call!(H5Pget_driver(fapl)).expect("should always be able to extract the driver"); + h5call!(H5Pclose(fapl)).expect("should always be able to close the FAPL"); + id + }}; +} + // Datatypes link_hid!(H5T_IEEE_F32BE, h5t::H5T_IEEE_F32BE); link_hid!(H5T_IEEE_F32LE, h5t::H5T_IEEE_F32LE); @@ -325,45 +333,52 @@ link_hid!(H5E_CANTCONVERT, h5e::H5E_CANTCONVERT); link_hid!(H5E_BADSIZE, h5e::H5E_BADSIZE); // H5R constants -lazy_static! { - pub static ref H5R_OBJ_REF_BUF_SIZE: usize = mem::size_of::(); - pub static ref H5R_DSET_REG_REF_BUF_SIZE: usize = mem::size_of::() + 4; -} +pub static H5R_OBJ_REF_BUF_SIZE: LazyLock = LazyLock::new(|| mem::size_of::()); +pub static H5R_DSET_REG_REF_BUF_SIZE: LazyLock = + LazyLock::new(|| mem::size_of::() + 4); // File drivers -lazy_static! { - pub static ref H5FD_CORE: hid_t = h5lock!(H5FD_core_init()); - pub static ref H5FD_SEC2: hid_t = h5lock!(H5FD_sec2_init()); - pub static ref H5FD_STDIO: hid_t = h5lock!(H5FD_stdio_init()); - pub static ref H5FD_FAMILY: hid_t = h5lock!(H5FD_family_init()); - pub static ref H5FD_LOG: hid_t = h5lock!(H5FD_log_init()); - pub static ref H5FD_MULTI: hid_t = h5lock!(H5FD_multi_init()); -} +pub static H5FD_CORE: LazyLock = + LazyLock::new(|| h5lock!(get_driver!(|fapl| H5Pset_fapl_core(fapl, 0, 0)))); +pub static H5FD_SEC2: LazyLock = + LazyLock::new(|| h5lock!(get_driver!(|fapl| H5Pset_fapl_sec2(fapl)))); +pub static H5FD_STDIO: LazyLock = + LazyLock::new(|| h5lock!(get_driver!(|fapl| H5Pset_fapl_stdio(fapl)))); +pub static H5FD_FAMILY: LazyLock = + LazyLock::new(|| h5lock!(get_driver!(|fapl| H5Pset_fapl_family(fapl, 0, 0)))); +pub static H5FD_LOG: LazyLock = + LazyLock::new(|| h5lock!(get_driver!(|fapl| H5Pset_fapl_log(fapl, std::ptr::null(), 0, 0)))); +pub static H5FD_MULTI: LazyLock = LazyLock::new(|| { + h5lock!(get_driver!(|fapl| H5Pset_fapl_multi( + fapl, + std::ptr::null(), + std::ptr::null(), + std::ptr::null(), + std::ptr::null(), + 0 + ))) +}); // MPI-IO file driver -#[cfg(feature = "have-parallel")] -lazy_static! { - pub static ref H5FD_MPIO: hid_t = h5lock!(H5FD_mpio_init()); -} -#[cfg(not(feature = "have-parallel"))] -lazy_static! { - pub static ref H5FD_MPIO: hid_t = H5I_INVALID_HID; -} +#[cfg(all(feature = "2.0.0", all(feature = "have-parallel", feature = "mpio")))] +pub static H5FD_MPIO: LazyLock = LazyLock::new(|| *hdf5_sys::h5fd::H5FD_MPIO); +#[cfg(all(feature = "2.0.0", not(all(feature = "have-parallel", feature = "mpio"))))] +pub static H5FD_MPIO: LazyLock = LazyLock::new(|| H5I_INVALID_HID); + +#[cfg(all(not(feature = "2.0.0"), all(feature = "have-parallel", feature = "mpio")))] +pub static H5FD_MPIO: LazyLock = LazyLock::new(|| h5lock!(hdf5_sys::h5fd::H5FD_mpio_init())); +#[cfg(all(not(feature = "2.0.0"), not(all(feature = "have-parallel", feature = "mpio"))))] +pub static H5FD_MPIO: LazyLock = LazyLock::new(|| H5I_INVALID_HID); // Direct VFD #[cfg(feature = "have-direct")] -lazy_static! { - pub static ref H5FD_DIRECT: hid_t = h5lock!(H5FD_direct_init()); -} +pub static H5FD_DIRECT: LazyLock = + LazyLock::new(|| h5lock!(get_driver!(|fapl| H5Pset_fapl_direct(fapl, 0, 0, 0)))); #[cfg(not(feature = "have-direct"))] -lazy_static! { - pub static ref H5FD_DIRECT: hid_t = H5I_INVALID_HID; -} +pub static H5FD_DIRECT: LazyLock = LazyLock::new(|| H5I_INVALID_HID); #[cfg(target_os = "windows")] -lazy_static! { - pub static ref H5FD_WINDOWS: hid_t = *H5FD_SEC2; -} +pub static H5FD_WINDOWS: LazyLock = LazyLock::new(|| *H5FD_SEC2); #[cfg(test)] mod tests { diff --git a/hdf5/src/hl/attribute.rs b/hdf5/src/hl/attribute.rs index a7a457e97..bd076c0fd 100644 --- a/hdf5/src/hl/attribute.rs +++ b/hdf5/src/hl/attribute.rs @@ -2,6 +2,8 @@ use std::fmt::{self, Debug}; use std::ops::Deref; use std::ptr::addr_of_mut; +use hdf5_sys::h5a::H5Aget_name; +use hdf5_sys::h5p::H5Pcreate; use hdf5_sys::{ h5::{H5_index_t, H5_iter_order_t}, h5a::{H5A_info_t, H5A_operator2_t, H5Acreate2, H5Adelete, H5Aiterate2}, @@ -9,6 +11,7 @@ use hdf5_sys::{ use hdf5_types::TypeDescriptor; use ndarray::ArrayView; +use crate::globals::H5P_ATTRIBUTE_CREATE; use crate::internal_prelude::*; /// Represents the HDF5 attribute object. @@ -46,6 +49,14 @@ impl Deref for Attribute { } impl Attribute { + /// Returns the name of the attribute. + pub fn name(&self) -> String { + // Note: We must use H5Aget_name() here. H5Iget_name() (called by + // Location::name()) would return the name of the object this + // attribute is attached to, not the attribute's own name. + h5lock!(get_h5_str(|m, s| H5Aget_name(self.id(), s, m)).unwrap_or_else(|_| String::new())) + } + /// Returns names of all the members in the group, non-recursively. pub fn attr_names(obj: &Location) -> Result> { unsafe extern "C" fn attributes_callback( @@ -86,18 +97,22 @@ pub struct AttributeBuilder { } impl AttributeBuilder { + /// Creates a builder for a new attribute on a named object. pub fn new(parent: &Location) -> Self { Self { builder: AttributeBuilderInner::new(parent) } } + /// Sets the attribute's type without initializing its data. pub fn empty(self) -> AttributeBuilderEmpty { self.empty_as(&T::type_descriptor()) } + /// Sets the attribute's type from a type descriptor without initializing its data. pub fn empty_as(self, type_desc: &TypeDescriptor) -> AttributeBuilderEmpty { AttributeBuilderEmpty { builder: self.builder, type_desc: type_desc.clone() } } + /// Sets the data to store in the attribute. pub fn with_data<'d, A, T, D>(self, data: A) -> AttributeBuilderData<'d, T, D> where A: Into>, @@ -107,6 +122,7 @@ impl AttributeBuilder { self.with_data_as::(data, &T::type_descriptor()) } + /// Sets the data to store in the attribute and sets its element type with a type descriptor. pub fn with_data_as<'d, A, T, D>( self, data: A, type_desc: &TypeDescriptor, ) -> AttributeBuilderData<'d, T, D> @@ -139,6 +155,7 @@ pub struct AttributeBuilderEmpty { } impl AttributeBuilderEmpty { + /// Sets the shape of the attribute's data. pub fn shape>(self, extents: S) -> AttributeBuilderEmptyShape { AttributeBuilderEmptyShape { builder: self.builder, @@ -146,6 +163,8 @@ impl AttributeBuilderEmpty { extents: extents.into(), } } + + /// Creates the attribute. pub fn create<'n, T: Into<&'n str>>(self, name: T) -> Result { self.shape(()).create(name) } @@ -167,6 +186,7 @@ pub struct AttributeBuilderEmptyShape { } impl AttributeBuilderEmptyShape { + /// Creates the attribute. pub fn create<'n, T: Into<&'n str>>(&self, name: T) -> Result { h5lock!(self.builder.create(&self.type_desc, name.into(), &self.extents)) } @@ -205,6 +225,7 @@ where self } + /// Creates the attribute. pub fn create<'n, N: Into<&'n str>>(&self, name: N) -> Result { ensure!( self.data.is_standard_layout(), @@ -262,15 +283,22 @@ impl AttributeBuilderInner { let dataspace = Dataspace::try_new(extents)?; + let acpl = PropertyList::from_id(h5call!(H5Pcreate(*H5P_ATTRIBUTE_CREATE))?)?; + // Set UTF-8 encoding for the attribute name, as Rust strings are UTF-8. + h5call!(hdf5_sys::h5p::H5Pset_char_encoding( + acpl.id(), + hdf5_sys::h5t::H5T_cset_t::H5T_CSET_UTF8 + ))?; + let name = to_cstring(name)?; Attribute::from_id(h5try!(H5Acreate2( parent.id(), name.as_ptr(), datatype.id(), dataspace.id(), - // these args are currently unused as if HDF5 1.12 - // see details: https://portal.hdfgroup.org/display/HDF5/H5A_CREATE2 - H5P_DEFAULT, + acpl.id(), + // Unused as of v1.14 + // see more: https://hdfgroup.github.io/hdf5/v1_14/group___h5_a.html#ga4f4e5248c09f689633079ed8afc0b308 H5P_DEFAULT, ))) } @@ -298,12 +326,14 @@ pub mod attribute_tests { assert_eq!(d.size(), 6); assert_eq!(d.ndim(), 2); assert_eq!(d.is_scalar(), false); + assert_eq!(d.name(), "name1"); let d = file.new_attr::().shape(()).create("name2").unwrap(); assert_eq!(d.shape(), vec![]); assert_eq!(d.size(), 1); assert_eq!(d.ndim(), 0); assert_eq!(d.is_scalar(), true); + assert_eq!(d.name(), "name2"); }) } @@ -314,9 +344,7 @@ pub mod attribute_tests { let _ = file.new_attr::().shape(()).create("name2").unwrap(); let attr_names = file.attr_names().unwrap(); - assert_eq!(attr_names.len(), 2); - assert!(attr_names.contains(&"name1".to_string())); - assert!(attr_names.contains(&"name2".to_string())); + assert_eq!(attr_names, vec!["name1".to_owned(), "name2".to_owned()]); }) } @@ -329,9 +357,7 @@ pub mod attribute_tests { let _ = ds.new_attr::().shape(()).create("name2").unwrap(); let attr_names = ds.attr_names().unwrap(); - assert_eq!(attr_names.len(), 2); - assert!(attr_names.contains(&"name1".to_string())); - assert!(attr_names.contains(&"name2".to_string())); + assert_eq!(attr_names, vec!["name1".to_owned(), "name2".to_owned()]); }) } @@ -368,9 +394,7 @@ pub mod attribute_tests { let attr = file.new_attr::().shape((1, 2)).create("foo").unwrap(); assert!(attr.is_valid()); assert_eq!(attr.shape(), vec![1, 2]); - // FIXME - attr.name() returns "/" here, which is the name the attribute is connected to, - // not the name of the attribute. - //assert_eq!(attr.name(), "foo"); + assert_eq!(attr.name(), "foo"); assert_eq!(file.attr("foo").unwrap().shape(), vec![1, 2]); }) } @@ -383,9 +407,7 @@ pub mod attribute_tests { let attr = file.new_attr_builder().with_data(&arr).create("foo").unwrap(); assert!(attr.is_valid()); assert_eq!(attr.shape(), vec![2, 3]); - // FIXME - attr.name() returns "/" here, which is the name the attribute is connected to, - // not the name of the attribute. - //assert_eq!(attr.name(), "foo"); + assert_eq!(attr.name(), "foo"); assert_eq!(file.attr("foo").unwrap().shape(), vec![2, 3]); let read_attr = file.attr("foo").unwrap(); diff --git a/hdf5/src/hl/container.rs b/hdf5/src/hl/container.rs index ca59b40f8..9b68d55e6 100644 --- a/hdf5/src/hl/container.rs +++ b/hdf5/src/hl/container.rs @@ -12,6 +12,7 @@ use hdf5_sys::h5p::H5Pcreate; use crate::internal_prelude::*; +/// A type for reading data from a [`Container`]. #[derive(Debug)] pub struct Reader<'a> { obj: &'a Container, @@ -188,6 +189,7 @@ impl<'a> Reader<'a> { } } +/// A type for writing data into a [`Container`]. #[derive(Debug)] pub struct Writer<'a> { obj: &'a Container, @@ -347,6 +349,7 @@ impl<'a> Writer<'a> { } } +/// A reader for a 1-dimensional dataset of bytes. #[derive(Debug, Clone)] pub struct ByteReader { obj: Container, @@ -357,6 +360,14 @@ pub struct ByteReader { } impl ByteReader { + /// Creates a new `ByteReader` for the given [`Container`]. + /// + /// # Panics + /// Panics if `obj` is not 1-dimensional. + /// + /// # Errors + /// + /// Returns an error if `obj` does not contain bytes or if the underlying library calls fail. pub fn new(obj: &Container) -> Result { ensure!(!obj.is_attr(), "ByteReader cannot be used on attribute datasets"); @@ -382,6 +393,7 @@ impl ByteReader { self.dataset_len().saturating_sub(self.pos as usize) } + /// Returns `true` if the reader has no more bytes to read. pub fn is_empty(&self) -> bool { self.pos >= self.dataset_len() as u64 } @@ -479,13 +491,13 @@ impl Container { /// Creates a reader wrapper for this dataset/attribute, allowing to /// set custom type conversion options when reading. - pub fn as_reader(&self) -> Reader { + pub fn as_reader(&self) -> Reader<'_> { Reader::new(self) } /// Creates a writer wrapper for this dataset/attribute, allowing to /// set custom type conversion options when writing. - pub fn as_writer(&self) -> Writer { + pub fn as_writer(&self) -> Writer<'_> { Writer::new(self) } diff --git a/hdf5/src/hl/dataset.rs b/hdf5/src/hl/dataset.rs index 77807c530..8ceca1d96 100644 --- a/hdf5/src/hl/dataset.rs +++ b/hdf5/src/hl/dataset.rs @@ -1,18 +1,12 @@ +//! Interfaces for `Dataset` objects. + use std::fmt::{self, Debug}; use std::ops::Deref; use ndarray::{self, ArrayView}; -use hdf5_sys::h5::HADDR_UNDEF; -use hdf5_sys::h5d::{ - H5Dcreate2, H5Dcreate_anon, H5Dget_access_plist, H5Dget_create_plist, H5Dget_offset, - H5Dset_extent, -}; -use hdf5_sys::h5l::H5Ldelete; -use hdf5_sys::h5p::H5P_DEFAULT; -use hdf5_sys::h5z::H5Z_filter_t; -use hdf5_types::{OwnedDynValue, TypeDescriptor}; - +#[cfg(feature = "zfp")] +use crate::hl; #[cfg(feature = "blosc")] use crate::hl::filters::{Blosc, BloscShuffle}; use crate::hl::filters::{Filter, SZip, ScaleOffset}; @@ -26,6 +20,17 @@ use crate::hl::plist::dataset_create::{ }; use crate::hl::plist::link_create::{CharEncoding, LinkCreate, LinkCreateBuilder}; use crate::internal_prelude::*; +use hdf5_sys::h5::HADDR_UNDEF; +use hdf5_sys::h5d::{ + H5Dcreate2, H5Dcreate_anon, H5Dget_access_plist, H5Dget_create_plist, H5Dget_offset, + H5Dset_extent, +}; +#[cfg(feature = "1.10.0")] +use hdf5_sys::h5d::{H5Dflush, H5Drefresh}; +use hdf5_sys::h5l::H5Ldelete; +use hdf5_sys::h5p::H5P_DEFAULT; +use hdf5_sys::h5z::H5Z_filter_t; +use hdf5_types::{OwnedDynValue, TypeDescriptor}; /// Default chunk size when filters are enabled and the chunk size is not specified. pub const DEFAULT_CHUNK_SIZE_KB: usize = 64 * 1024; @@ -154,6 +159,22 @@ impl Dataset { pub fn filters(&self) -> Vec { self.dcpl().map_or(Vec::default(), |pl| pl.filters()) } + + /// Flush the dataset metadata from the metadata cache to the file + #[cfg(feature = "1.10.0")] + pub fn flush(&self) -> Result<()> { + let id = self.id(); + h5call!(H5Dflush(id))?; + Ok(()) + } + + /// Refresh metadata items assosicated with the dataset + #[cfg(feature = "1.10.0")] + pub fn refresh(&self) -> Result<()> { + let id = self.id(); + h5call!(H5Drefresh(id))?; + Ok(()) + } } pub struct Maybe(Option); @@ -227,6 +248,34 @@ impl DatasetBuilder { conv: Conversion::Soft, } } + // + // #[cfg(feature = "zfp")] + // pub fn zfp_rate(self, rate: f64) -> Self { + // let new_ds = self.with_dcpl(|p| p.set_filters(&vec![Filter::zfp_rate(rate)])); + // + // new_ds + // } + // + // #[cfg(feature = "zfp")] + // pub fn zfp_precision(self, precision: u8) -> Self { + // let new_ds = self.with_dcpl(|p| p.set_filters(&vec![Filter::zfp_precision(precision)])); + // + // new_ds + // } + // + // #[cfg(feature = "zfp")] + // pub fn zfp_accuracy(self, accuracy: f64) -> Self { + // let new_ds = self.with_dcpl(|p| p.set_filters(&vec![Filter::zfp_accuracy(accuracy)])); + // + // new_ds + // } + // + // #[cfg(feature = "zfp")] + // pub fn zfp_reversible(self) -> Self { + // let new_ds = self.with_dcpl(|p| p.set_filters(&vec![Filter::zfp_reversible()])); + // + // new_ds + // } } #[derive(Clone)] @@ -311,11 +360,15 @@ where } } +/// Options for how to chunk data. #[derive(Debug, Clone, PartialEq, Eq)] pub enum Chunk { - Exact(Vec), // exact chunk shape - MinKB(usize), // minimum chunk shape in KB - None, // leave it unchunked + /// Exact chunk shape. + Exact(Vec), + /// Minimum chunk shape in kilobytes. + MinKB(usize), + /// Leave the data unchunked. + None, } impl Default for Chunk { @@ -695,6 +748,36 @@ impl DatasetBuilderInner { self.with_dcpl(|pl| pl.blosc_zstd(clevel, shuffle)); } + #[cfg(feature = "zfp")] + pub fn zfp_rate(&mut self, rate: f64, chunk_dims: Vec, n_bytes: u8) { + hl::filters::zfp::register_zfp().expect("Failed to register ZFP filter"); + self.with_dcpl(|p| { + p.set_filters(&vec![Filter::zfp_rate(rate, chunk_dims.clone(), n_bytes)]) + }); + } + + #[cfg(feature = "zfp")] + pub fn zfp_precision(&mut self, precision: u8, chunk_dims: Vec, n_bytes: u8) { + hl::filters::zfp::register_zfp().expect("Failed to register ZFP filter"); + self.with_dcpl(|p| { + p.set_filters(&vec![Filter::zfp_precision(precision, chunk_dims.clone(), n_bytes)]) + }); + } + + #[cfg(feature = "zfp")] + pub fn zfp_accuracy(&mut self, accuracy: f64, chunk_dims: Vec, n_bytes: u8) { + hl::filters::zfp::register_zfp().expect("Failed to register ZFP filter"); + self.with_dcpl(|pl| pl.zfp_accuracy(accuracy, chunk_dims.clone(), n_bytes)); + } + + #[cfg(feature = "zfp")] + pub fn zfp_reversible(&mut self, chunk_dims: Vec, n_bytes: u8) { + hl::filters::zfp::register_zfp().expect("Failed to register ZFP filter"); + self.with_dcpl(|p| { + p.set_filters(&vec![Filter::zfp_reversible(chunk_dims.clone(), n_bytes)]) + }); + } + pub fn add_filter(&mut self, id: H5Z_filter_t, cdata: &[c_uint]) { self.with_dcpl(|pl| pl.add_filter(id, cdata)); } @@ -954,6 +1037,25 @@ macro_rules! impl_builder_methods { #[cfg(feature = "blosc-zstd")] DatasetCreate: blosc_zstd(clevel: u8, shuffle: impl Into) ); + + impl_builder!( + #[cfg(feature = "zfp")] + DatasetCreate: zfp_rate(rate: f64,chunk_dims: Vec,n_bytes: u8) + ); + impl_builder!( + #[cfg(feature = "zfp")] + DatasetCreate: zfp_accuracy(accuracy: f64,chunk_dims: Vec,n_bytes: u8) + ); + impl_builder!( + #[cfg(feature = "zfp")] + DatasetCreate: zfp_precision(rate: u8,chunk_dims: Vec,n_bytes: u8) + ); + impl_builder!( + #[cfg(feature = "zfp")] + DatasetCreate: zfp_reversible(chunk_dims: Vec,n_bytes: u8) + ); + + impl_builder!(DatasetCreate: add_filter(id: H5Z_filter_t, cdata: &[c_uint])); impl_builder!(DatasetCreate: clear_filters()); impl_builder!(DatasetCreate: alloc_time(alloc_time: Option)); diff --git a/hdf5/src/hl/dataspace.rs b/hdf5/src/hl/dataspace.rs index 58c9af2f6..d4d5f1f10 100644 --- a/hdf5/src/hl/dataspace.rs +++ b/hdf5/src/hl/dataspace.rs @@ -78,46 +78,62 @@ unsafe fn get_simple_extents(space_id: hid_t) -> Result { } impl Dataspace { + /// Tries to construct a `Dataspace` from the given extents. + /// + /// # Errors + /// + /// Returns an error if the extents are invalid. pub fn try_new>(extents: T) -> Result { Self::from_extents(&extents.into()) } + /// Creates a new `Dataspace` that is a copy of the current one. pub fn copy(&self) -> Self { Self::from_id(h5lock!(H5Scopy(self.id()))).unwrap_or_else(|_| Self::invalid()) } + /// Returns the number of dimensions in the space. pub fn ndim(&self) -> usize { h5call!(H5Sget_simple_extent_ndims(self.id())).unwrap_or(0) as _ } + /// Returns a vector containing the current size of each dimension. pub fn shape(&self) -> Vec { h5lock!(get_shape(self.id())).unwrap_or_default() } + /// Returns a vector containing the current maximum size (if set) of each dimension. pub fn maxdims(&self) -> Vec> { self.extents().unwrap_or(Extents::Null).maxdims() } + /// Returns `true` if any dimension is resizable. pub fn is_resizable(&self) -> bool { self.maxdims().iter().any(Option::is_none) } + /// Returns `true` if the extent type is null. pub fn is_null(&self) -> bool { h5lock!(H5Sget_simple_extent_type(self.id())) == H5S_class_t::H5S_NULL } + /// Returns `true` if the extent type is scalar. pub fn is_scalar(&self) -> bool { h5lock!(H5Sget_simple_extent_type(self.id())) == H5S_class_t::H5S_SCALAR } + /// Returns `true` if the extent type is simple. pub fn is_simple(&self) -> bool { h5lock!(H5Sget_simple_extent_type(self.id())) == H5S_class_t::H5S_SIMPLE } + /// Returns `true` if the selection for the space is within the dataspace extent if the current + /// offset is used. pub fn is_valid(&self) -> bool { h5lock!(H5Sselect_valid(self.id())) > 0 } + /// Returns the number of elements in the space. pub fn size(&self) -> usize { match h5lock!(H5Sget_simple_extent_type(self.id())) { H5S_class_t::H5S_SIMPLE => { @@ -128,6 +144,11 @@ impl Dataspace { } } + /// Encodes the dataspace description into a new byte vector. + /// + /// # Errors + /// + /// Returns an error if the underlying encoding call fails. #[allow(deprecated)] pub fn encode(&self) -> Result> { cfg_if::cfg_if! { @@ -152,6 +173,11 @@ impl Dataspace { } } + /// Tries to decode a dataspace from a byte buffer. + /// + /// # Errors + /// + /// Returns an error if the buffer does not decode to a valid dataspace. pub fn decode(buf: T) -> Result where T: AsRef<[u8]>, @@ -174,6 +200,11 @@ impl Dataspace { })) } + /// Returns the extents of the dataspace. + /// + /// # Errors + /// + /// Returns an error if the extents type is unsupported. #[allow(clippy::match_wildcard_for_single_variants)] pub fn extents(&self) -> Result { h5lock!(match H5Sget_simple_extent_type(self.id()) { @@ -184,6 +215,7 @@ impl Dataspace { }) } + /// Returns the number of elements in the dataspace selection. pub fn selection_size(&self) -> usize { h5call!(H5Sget_select_npoints(self.id())).ok().map_or(0, |x| x as _) } @@ -199,6 +231,7 @@ impl Dataspace { }) } + /// Selects part of the dataspace and returns a new dataspace selection object. pub fn select>(&self, selection: S) -> Result { let raw_sel = selection.into().into_raw(self.shape())?; self.select_raw(raw_sel) @@ -209,6 +242,7 @@ impl Dataspace { sync(|| unsafe { RawSelection::extract_from_dataspace(self.id()) }) } + /// Returns a description of the current dataspace selection. pub fn get_selection(&self) -> Result { let raw_sel = self.get_raw_selection()?; Selection::from_raw(raw_sel) diff --git a/hdf5/src/hl/datatype.rs b/hdf5/src/hl/datatype.rs index 6f4847313..ce4ebb29a 100644 --- a/hdf5/src/hl/datatype.rs +++ b/hdf5/src/hl/datatype.rs @@ -62,7 +62,26 @@ impl ObjectClass for Datatype { &self.0 } - // TODO: short_repr() + fn short_repr(&self) -> Option { + let repr = match self.to_descriptor() { + Ok(s) => s.to_string(), + Err(e) => { + // Fallback: show basic HDF5 info if descriptor conversion fails + h5lock!({ + let class = H5Tget_class(self.id()); + let size = H5Tget_size(self.id()); + format!("Invalid datatype: {e} )") + }) + } + }; + Some(repr) + } +} + +impl Display for Datatype { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", self.short_repr().expect("short_repr is always implemented")) + } } impl Debug for Datatype { @@ -85,10 +104,14 @@ impl PartialEq for Datatype { } } +/// A level of possible conversion between two types. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] pub enum Conversion { + /// No conversion. NoOp = 1, // TODO: rename to "None"? + /// Functions employing compiler casting. Hard, + /// Functions not employing compiler casting. Soft, } @@ -120,12 +143,18 @@ impl Default for Conversion { } } +/// The byte order of a datatype. #[derive(Copy, Debug, Clone, PartialEq, Eq)] pub enum ByteOrder { + /// Little endian. LittleEndian, + /// Big endian. BigEndian, + /// VAX mixed endian. Vax, + /// Compound type with mixed member orders. Mixed, + /// No particular order. None, } @@ -166,6 +195,7 @@ impl Datatype { h5lock!(H5Tget_order(self.id())).into() } + /// Returns the conversion function level from `self` to `dst`, if one exists. pub fn conv_path(&self, dst: D) -> Option where D: Borrow, @@ -187,33 +217,32 @@ impl Datatype { }) } + /// Returns the conversion function level from `self` to a concrete type, if one exists. pub fn conv_to(&self) -> Option { Self::from_type::().ok().and_then(|dtype| self.conv_path(dtype)) } + /// Returns the conversion function level from a concrete type to `self`, if one exists. pub fn conv_from(&self) -> Option { Self::from_type::().ok().and_then(|dtype| dtype.conv_path(self)) } + /// Returns `true` if `self` represents a concrete type. pub fn is(&self) -> bool { Self::from_type::().ok().map_or(false, |dtype| &dtype == self) } pub(crate) fn ensure_convertible(&self, dst: &Self, required: Conversion) -> Result<()> { - // TODO: more detailed error messages after Debug/Display are implemented for Datatype if let Some(conv) = self.conv_path(dst) { ensure!( - conv <= required, - "{} conversion path required; available: {} conversion", - required, - conv - ); + conv <= required, "Cannot convert from {self} to {dst}, required conversion {required}; available: {conv}",); Ok(()) } else { - fail!("no conversion paths found") + fail!("no conversion paths found from '{self:#?}' to '{dst:#?}'",) } } + /// Returns a type descriptor for the datatype. pub fn to_descriptor(&self) -> Result { use hdf5_types::TypeDescriptor as TD; @@ -307,10 +336,12 @@ impl Datatype { }) } + /// Creates a datatype from a concrete type. pub fn from_type() -> Result { Self::from_descriptor(&::type_descriptor()) } + /// Creates a datatype from a type descriptor. pub fn from_descriptor(desc: &TypeDescriptor) -> Result { use hdf5_types::TypeDescriptor as TD; @@ -427,3 +458,34 @@ impl Datatype { Self::from_id(datatype_id?) } } + +/// NOTE: tests of public functions are in hdf5/tests/test_datatype.rs +#[cfg(test)] +mod tests { + use super::*; + use hdf5_types::{FixedAscii, FixedUnicode}; + use pretty_assertions::assert_str_eq; + + #[test] + fn test_ensure_convertible_fail_err_msg() { + const SIZE: usize = 10; + let src = Datatype::from_type::>().unwrap(); + let dst = Datatype::from_type::>().unwrap(); + + let err_msg = src.ensure_convertible(&dst, Conversion::NoOp).unwrap_err().to_string(); + + assert_str_eq!(err_msg, "no conversion paths found from '' to ''"); + } + + #[test] + fn test_ensure_convertible_failed_required_conversion_hard_err_msg() { + let src = Datatype::from_type::().unwrap(); + let dst = Datatype::from_type::().unwrap(); + + let err_msg = src.ensure_convertible(&dst, Conversion::NoOp).unwrap_err().to_string(); + assert_str_eq!( + err_msg, + "Cannot convert from uint64 to int64, required conversion no-op; available: hard" + ); + } +} diff --git a/hdf5/src/hl/extents.rs b/hdf5/src/hl/extents.rs index 9eeba3d22..6b3128321 100644 --- a/hdf5/src/hl/extents.rs +++ b/hdf5/src/hl/extents.rs @@ -68,6 +68,7 @@ impl + Clone> From<&T> for Extent { } impl Extent { + /// Creates a new extent with a current size and (optional) maximum size. pub fn new(dim: Ix, max: Option) -> Self { Self { dim, max } } @@ -82,18 +83,22 @@ impl Extent { Self { dim, max: None } } + /// Returns `true` if the current dimension size is at least the maximum size, if set. pub fn is_fixed(&self) -> bool { self.max.map_or(false, |max| self.dim >= max) } + /// Returns `true` if the dimension is resizable. pub fn is_resizable(&self) -> bool { self.max.is_none() } + /// Returns `true` if the dimension has unlimited maximum size. pub fn is_unlimited(&self) -> bool { self.is_resizable() } + /// Returns `true` if the current dimension size is not greater than the maximum. pub fn is_valid(&self) -> bool { self.max.unwrap_or(self.dim) >= self.dim } @@ -113,10 +118,12 @@ pub struct SimpleExtents { } impl SimpleExtents { + /// Creates extents from a vector of `Extent` values. pub fn from_vec(extents: Vec) -> Self { Self { inner: extents } } + /// Creates extents from an iterable of extent-like values. pub fn new(extents: T) -> Self where T: IntoIterator, @@ -125,6 +132,8 @@ impl SimpleExtents { Self::from_vec(extents.into_iter().map(Into::into).collect()) } + /// Creates extents from an iterable of dimension sizes and sets their + /// maximum sizes equal to their current sizes. pub fn fixed(extents: T) -> Self where T: IntoIterator, @@ -142,38 +151,48 @@ impl SimpleExtents { Self::from_vec(extents.into_iter().map(|x| Extent::resizable(*x.borrow())).collect()) } + /// Returns the number of dimensions. pub fn ndim(&self) -> usize { self.inner.len() } + /// Returns a vector containing the current size of each dimension. pub fn dims(&self) -> Vec { self.inner.iter().map(|e| e.dim).collect() } + /// Returns a vector containing the maximum size (if set) of each dimension. pub fn maxdims(&self) -> Vec> { self.inner.iter().map(|e| e.max).collect() } + /// Returns the number of elements that the space can hold. pub fn size(&self) -> usize { self.inner.iter().fold(1, |acc, x| acc * x.dim) } + /// Returns `true` if there is at least one dimension and they all have fixed maximum size. pub fn is_fixed(&self) -> bool { !self.inner.is_empty() && self.inner.iter().map(Extent::is_fixed).all(identity) } + /// Returns `true` if there is at least one resizable dimension. pub fn is_resizable(&self) -> bool { !self.inner.is_empty() && self.inner.iter().map(Extent::is_unlimited).any(identity) } + /// Returns `true` if there is at least one dimension of unlimited size. pub fn is_unlimited(&self) -> bool { self.inner.iter().map(Extent::is_unlimited).any(identity) } + /// Returns `true` if the extents are valid and the dimensionality does not exceed + /// HDF5's max rank. pub fn is_valid(&self) -> bool { self.inner.iter().map(Extent::is_valid).all(identity) && self.ndim() <= H5S_MAX_RANK as _ } + /// Returns an iterator over the `Extent` values for each dimension. pub fn iter( &self, ) -> impl ExactSizeIterator + DoubleEndedIterator { @@ -290,6 +309,7 @@ impl_from!(RangeFrom); impl_from!(RangeInclusive); impl_from!(Extent); +/// The dimensionality of a dataspace. #[derive(Clone, Debug, PartialEq, Eq)] pub enum Extents { /// A null dataspace contains no data elements. @@ -324,6 +344,7 @@ pub enum Extents { } impl Extents { + /// Creates extents from a value. pub fn new>(extents: T) -> Self { extents.into() } @@ -389,18 +410,22 @@ impl Extents { } } + /// Returns `true` if the extents are valid. pub fn is_valid(&self) -> bool { self.as_simple().map_or(true, SimpleExtents::is_valid) } + /// Returns `true` if the extents have unlimited maximum size. pub fn is_unlimited(&self) -> bool { self.as_simple().map_or(true, SimpleExtents::is_unlimited) } + /// Returns `true` if there is at least one resizable dimension. pub fn is_resizable(&self) -> bool { self.as_simple().map_or(true, SimpleExtents::is_resizable) } + /// Returns a version of `self` that is resizable along all dimensions. pub fn resizable(self) -> Self { match self { Self::Simple(extents) => SimpleExtents::resizable(extents.dims()).into(), @@ -408,12 +433,15 @@ impl Extents { } } + /// Returns an iterator of `Extent` values, one for each dimension. pub fn iter( &self, ) -> impl ExactSizeIterator + DoubleEndedIterator { ExtentsIter { inner: self.as_simple().map(SimpleExtents::iter) } } + /// Returns a slice containing an `Extent` for each dimension, or `None` if the extents are + /// null or scalar. pub fn slice(&self) -> Option<&[Extent]> { if let Self::Simple(x) = self { Some(x) @@ -423,6 +451,7 @@ impl Extents { } } +/// An iterator over the dimensions of an `Extents`. pub struct ExtentsIter { inner: Option, } diff --git a/hdf5/src/hl/file.rs b/hdf5/src/hl/file.rs index 81ad839dc..3dd59f536 100644 --- a/hdf5/src/hl/file.rs +++ b/hdf5/src/hl/file.rs @@ -8,6 +8,8 @@ use hdf5_sys::h5f::{ H5Fget_freespace, H5Fget_intent, H5Fget_obj_count, H5Fget_obj_ids, H5Fopen, H5F_ACC_DEFAULT, H5F_ACC_EXCL, H5F_ACC_RDONLY, H5F_ACC_RDWR, H5F_ACC_TRUNC, H5F_SCOPE_LOCAL, }; +#[cfg(feature = "1.10.0")] +use hdf5_sys::h5f::{H5Fstart_swmr_write, H5F_ACC_SWMR_READ}; use crate::hl::plist::{ file_access::{FileAccess, FileAccessBuilder}, @@ -17,9 +19,13 @@ use crate::internal_prelude::*; /// File opening mode. #[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[cfg_attr(not(feature = "1.10.0"), non_exhaustive)] pub enum OpenMode { /// Open a file as read-only, file must exist. Read, + /// Open a file as read-only in SWMR mode, file must exist. + #[cfg(feature = "1.10.0")] + ReadSWMR, /// Open a file as read/write, file must exist. ReadWrite, /// Create a file, truncate if exists. @@ -178,6 +184,14 @@ impl File { pub fn fcpl(&self) -> Result { self.create_plist() } + + #[cfg(feature = "1.10.0")] + /// Mark this file as ready for opening as SWMR + pub fn start_swmr(&self) -> Result<()> { + let id = self.id(); + h5call!(H5Fstart_swmr_write(id))?; + Ok(()) + } } /// File builder allowing to customize file access/creation property lists. @@ -231,9 +245,13 @@ impl FileBuilder { )?; let flags = match mode { OpenMode::Read => H5F_ACC_RDONLY, + #[cfg(feature = "1.10.0")] + OpenMode::ReadSWMR => H5F_ACC_RDONLY | H5F_ACC_SWMR_READ, OpenMode::ReadWrite => H5F_ACC_RDWR, OpenMode::Create => H5F_ACC_TRUNC, OpenMode::CreateExcl | OpenMode::Append => H5F_ACC_EXCL, + #[cfg(not(feature = "1.10.0"))] + _ => unreachable!(), }; let fname_ptr = filename.as_ptr(); h5lock!({ @@ -242,6 +260,8 @@ impl FileBuilder { OpenMode::Read | OpenMode::ReadWrite => { File::from_id(h5try!(H5Fopen(fname_ptr, flags, fapl.id()))) } + #[cfg(feature = "1.10.0")] + OpenMode::ReadSWMR => File::from_id(h5try!(H5Fopen(fname_ptr, flags, fapl.id()))), _ => { let fcpl = self.fcpl.finish()?; File::from_id(h5try!(H5Fcreate(fname_ptr, flags, fcpl.id(), fapl.id()))) diff --git a/hdf5/src/hl/filters.rs b/hdf5/src/hl/filters.rs index a0056aa6e..cc48d1366 100644 --- a/hdf5/src/hl/filters.rs +++ b/hdf5/src/hl/filters.rs @@ -1,6 +1,8 @@ use std::collections::HashMap; use std::ptr::{self, addr_of_mut}; +#[cfg(feature = "zfp")] +use crate::globals::{H5E_CALLBACK, H5E_PLIST}; use hdf5_sys::h5p::{ H5Pget_filter2, H5Pget_nfilters, H5Pset_deflate, H5Pset_filter, H5Pset_fletcher32, H5Pset_nbit, H5Pset_scaleoffset, H5Pset_shuffle, H5Pset_szip, @@ -14,6 +16,7 @@ use hdf5_sys::h5z::{ H5_SZIP_NN_OPTION_MASK, }; +/// A filter identifier. pub use hdf5_sys::h5z::H5Z_filter_t; use crate::internal_prelude::*; @@ -22,21 +25,33 @@ use crate::internal_prelude::*; mod blosc; #[cfg(feature = "lzf")] mod lzf; +#[cfg(feature = "zfp")] +pub(crate) mod zfp; +#[cfg(feature = "zfp")] +use zfp_sys::{zfp_type_zfp_type_double, zfp_type_zfp_type_float}; + +/// Coding methods for Szip compression. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum SZip { + /// Entropy coding method. Entropy, + /// Nearest-neighbor coding method. NearestNeighbor, } +/// Scaling methods for scale-offset compression. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum ScaleOffset { + /// Integer scaling with some MinBits value. Integer(u16), + /// Floating-point D-scaling with some decimal scale factor. FloatDScale(u8), } #[cfg(feature = "blosc")] mod blosc_impl { + /// Available compressors for Blosc compression. #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[cfg(feature = "blosc")] #[non_exhaustive] @@ -103,25 +118,84 @@ mod blosc_impl { #[cfg(feature = "blosc")] pub use blosc_impl::*; +#[cfg(feature = "zfp")] +mod zfp_impl { + use crate::filters::ZfpMode::Reversible; + + #[derive(Clone, Copy, Debug)] + pub enum ZfpMode { + FixedRate(f64), + FixedPrecision(u8), + FixedAccuracy(f64), + Reversible, + } + + // Bitwise compare f64 so NaN and signed zero are deterministic + impl PartialEq for ZfpMode { + fn eq(&self, other: &Self) -> bool { + use ZfpMode::*; + match (self, other) { + (FixedRate(a), FixedRate(b)) => a.to_bits() == b.to_bits(), + (FixedPrecision(a), FixedPrecision(b)) => a == b, + (FixedAccuracy(a), FixedAccuracy(b)) => a.to_bits() == b.to_bits(), + (Reversible, Reversible) => true, + _ => false, + } + } + } + impl Eq for ZfpMode {} + + impl Default for ZfpMode { + fn default() -> Self { + ZfpMode::FixedRate(4.0) + } + } + + #[derive(Clone, Debug, Eq, PartialEq)] + pub struct FieldParam { + pub data_type_bytes: usize, + pub dims: Vec, + } +} + +#[cfg(feature = "zfp")] +pub use zfp_impl::*; + +/// An HDF5 filter configuration. #[derive(Clone, Debug, PartialEq, Eq)] pub enum Filter { + /// Gzip compression (deflation) with some compression level. Deflate(u8), + /// Shuffle algorithm. Shuffle, + /// Fletcher32 checksum. Fletcher32, + /// SZIP compression with some coding method and pixels per block. SZip(SZip, u8), + /// N-bit compression. NBit, + /// Scale-offset compression with some scaling mode. ScaleOffset(ScaleOffset), + /// LZF compression. #[cfg(feature = "lzf")] LZF, + /// Blosc compression with some compressor, compression level, and shuffle mode. #[cfg(feature = "blosc")] Blosc(Blosc, u8, BloscShuffle), + #[cfg(feature = "zfp")] + Zfp(ZfpMode, Vec, u8), + /// A user-defined filter with some parameters. User(H5Z_filter_t, Vec), } +/// Information about whether a filter is available and enabled for encoding/decoding. #[derive(Default, Clone, Copy, Debug, PartialEq, Eq)] pub struct FilterInfo { + /// Whether the filter is available. pub is_available: bool, + /// Whether the filter is configured to encode data. pub encode_enabled: bool, + /// Whether the filter is configured to decode data. pub decode_enabled: bool, } @@ -135,6 +209,10 @@ pub(crate) fn register_filters() { if let Err(e) = blosc::register_blosc() { eprintln!("Error while registering Blosc filter: {e}"); } + #[cfg(feature = "zfp")] + if let Err(e) = zfp::register_zfp() { + eprintln!("Error while registering ZFP filter: {e}"); + } } /// Returns `true` if deflate filter is available. @@ -164,7 +242,13 @@ pub fn blosc_available() -> bool { h5lock!(H5Zfilter_avail(32001) == 1) } +/// Returns `true` if ZFP filter is available. +pub fn zfp_available() -> bool { + h5lock!(H5Zfilter_avail(32013) == 1) +} + impl Filter { + /// Returns the filter's identifier. pub fn id(&self) -> H5Z_filter_t { match self { Self::Deflate(_) => H5Z_FILTER_DEFLATE, @@ -177,10 +261,13 @@ impl Filter { Self::LZF => lzf::LZF_FILTER_ID, #[cfg(feature = "blosc")] Self::Blosc(_, _, _) => blosc::BLOSC_FILTER_ID, + #[cfg(feature = "zfp")] + Self::Zfp(_, _, _) => zfp::ZFP_FILTER_ID, Self::User(id, _) => *id, } } + /// Returns metadata for the filter with the given identifier. pub fn get_info(filter_id: H5Z_filter_t) -> FilterInfo { if !h5call!(H5Zfilter_avail(filter_id)).map(|x| x > 0).unwrap_or_default() { return FilterInfo::default(); @@ -194,47 +281,59 @@ impl Filter { } } + /// Returns `true` if the filter is available. pub fn is_available(&self) -> bool { Self::get_info(self.id()).is_available } + /// Returns `true` if the filter is configured to encode data. pub fn encode_enabled(&self) -> bool { Self::get_info(self.id()).encode_enabled } + /// Returns `true` if the filter is configured to decode data. pub fn decode_enabled(&self) -> bool { Self::get_info(self.id()).decode_enabled } + /// Creates a deflation filter configuration with some compression level. pub fn deflate(level: u8) -> Self { Self::Deflate(level) } + /// Returns the shuffle algorithm filter configuration. pub fn shuffle() -> Self { Self::Shuffle } + /// Returns the Fletcher32 checksum filter configuration. pub fn fletcher32() -> Self { Self::Fletcher32 } + /// Creates an Szip filter configuration with some coding method and pixels per block. pub fn szip(coding: SZip, px_per_block: u8) -> Self { Self::SZip(coding, px_per_block) } + /// Returns the N-bit compression filter configuration. pub fn nbit() -> Self { Self::NBit } + /// Creates a scale-offset compression filter configuration with some scaling mode. pub fn scale_offset(mode: ScaleOffset) -> Self { Self::ScaleOffset(mode) } + /// Returns the LZF compression filter. #[cfg(feature = "lzf")] pub fn lzf() -> Self { Self::LZF } + /// Creates a Blosc compression filter configuration with some compressor, + /// compression level, and shuffle mode. #[cfg(feature = "blosc")] pub fn blosc(complib: Blosc, clevel: u8, shuffle: T) -> Self where @@ -243,6 +342,8 @@ impl Filter { Self::Blosc(complib, clevel, shuffle.into()) } + /// Creates a Blosc LZ compression filter configuration with some compression level and + /// shuffle mode. #[cfg(feature = "blosc")] pub fn blosc_blosclz(clevel: u8, shuffle: T) -> Self where @@ -251,6 +352,8 @@ impl Filter { Self::blosc(Blosc::BloscLZ, clevel, shuffle) } + /// Creates a Blosc LZ4 compression filter configuration with some compression level and + /// shuffle mode. #[cfg(feature = "blosc-lz4")] pub fn blosc_lz4(clevel: u8, shuffle: T) -> Self where @@ -259,6 +362,8 @@ impl Filter { Self::blosc(Blosc::LZ4, clevel, shuffle) } + /// Creates a Blosc LZ4HC compression filter configuration with some compression level and + /// shuffle mode. #[cfg(feature = "blosc-lz4")] pub fn blosc_lz4hc(clevel: u8, shuffle: T) -> Self where @@ -267,6 +372,8 @@ impl Filter { Self::blosc(Blosc::LZ4HC, clevel, shuffle) } + /// Creates a Blosc Snappy compression filter configuration with some compression level and + /// shuffle mode. #[cfg(feature = "blosc-snappy")] pub fn blosc_snappy(clevel: u8, shuffle: T) -> Self where @@ -275,6 +382,8 @@ impl Filter { Self::blosc(Blosc::Snappy, clevel, shuffle) } + /// Creates a Blosc Zlib compression filter configuration with some compression level and + /// shuffle mode. #[cfg(feature = "blosc-zlib")] pub fn blosc_zlib(clevel: u8, shuffle: T) -> Self where @@ -283,6 +392,8 @@ impl Filter { Self::blosc(Blosc::ZLib, clevel, shuffle) } + /// Creates a Blosc Zstd compression filter configuration with some compression level and + /// shuffle mode. #[cfg(feature = "blosc-zstd")] pub fn blosc_zstd(clevel: u8, shuffle: T) -> Self where @@ -291,6 +402,32 @@ impl Filter { Self::blosc(Blosc::ZStd, clevel, shuffle) } + #[cfg(feature = "zfp")] + pub fn zfp(mode: ZfpMode, chunk_dims: Vec, n_bytes: u8) -> Self { + Self::Zfp(mode, chunk_dims, n_bytes) + } + + #[cfg(feature = "zfp")] + pub fn zfp_rate(rate: f64, chunk_dims: Vec, n_bytes: u8) -> Self { + Self::zfp(ZfpMode::FixedRate(rate), chunk_dims, n_bytes) + } + + #[cfg(feature = "zfp")] + pub fn zfp_precision(precision: u8, chunk_dims: Vec, n_bytes: u8) -> Self { + Self::zfp(ZfpMode::FixedPrecision(precision), chunk_dims, n_bytes) + } + + #[cfg(feature = "zfp")] + pub fn zfp_accuracy(accuracy: f64, chunk_dims: Vec, n_bytes: u8) -> Self { + Self::zfp(ZfpMode::FixedAccuracy(accuracy), chunk_dims, n_bytes) + } + + #[cfg(feature = "zfp")] + pub fn zfp_reversible(chunk_dims: Vec, n_bytes: u8) -> Self { + Self::zfp(ZfpMode::Reversible, chunk_dims, n_bytes) + } + + /// Creates a user-defined filter configuration with some filter identifier and parameters. pub fn user(id: H5Z_filter_t, cdata: &[c_uint]) -> Self { Self::User(id, cdata.to_vec()) } @@ -397,6 +534,37 @@ impl Filter { Ok(Self::blosc(complib, clevel, shuffle)) } + #[cfg(feature = "zfp")] + fn parse_zfp(cdata: &[c_uint]) -> Result { + ensure!(cdata.len() >= 8, "expected at least length 8 cdata for zfp filter"); + let chunk_dims = cdata[4..6].iter().map(|&x| x as _).collect::>(); + let n_bytes = cdata[6] as u8; + let mode = if cdata.len() >= 8 { cdata[7] } else { 1 }; + let param1 = if cdata.len() >= 9 { cdata[8] } else { 0 }; + let param2 = if cdata.len() >= 10 { cdata[9] } else { 0 }; + let zfp_mode = match mode { + 1 => { + let rate = f64::from_bits(((param1 as u64) << 32) | (param2 as u64)); + + ZfpMode::FixedRate(rate) + } + 2 => ZfpMode::FixedPrecision(param1 as u8), + 3 => { + let accuracy = f64::from_bits(((param1 as u64) << 32) | (param2 as u64)); + ZfpMode::FixedAccuracy(accuracy) + } + 5 => ZfpMode::Reversible, + _ => fail!("invalid zfp mode: {}", mode), + }; + Ok(Self::zfp(zfp_mode, chunk_dims, n_bytes)) + } + + /// Tries to create a filter configuration from a filter identifier and parameters. + /// + /// # Errors + /// + /// Returns an error if the identifier is invalid or the parameters are invalid + /// for the specified filter. pub fn from_raw(filter_id: H5Z_filter_t, cdata: &[c_uint]) -> Result { ensure!(filter_id > 0, "invalid filter id: {}", filter_id); match filter_id { @@ -410,6 +578,8 @@ impl Filter { lzf::LZF_FILTER_ID => Self::parse_lzf(cdata), #[cfg(feature = "blosc")] blosc::BLOSC_FILTER_ID => Self::parse_blosc(cdata), + #[cfg(feature = "zfp")] + zfp::ZFP_FILTER_ID => Self::parse_zfp(cdata), _ => Ok(Self::user(filter_id, cdata)), } } @@ -478,6 +648,82 @@ impl Filter { Self::apply_user(plist_id, blosc::BLOSC_FILTER_ID, &cdata) } + #[cfg(feature = "zfp")] + /// Applies the ZFP filter to the given property list. + /// + /// This function configures the ZFP filter for compression on the specified dataset. + /// It determines the data type, removes singleton dimensions, and encodes the mode + /// information into the filter header. + /// + /// # Safety + /// This function is marked as unsafe because it interacts with raw pointers and + /// performs operations that require careful handling to avoid undefined behavior. + /// + /// # Parameters + /// - `plist_id`: The property list identifier to which the ZFP filter will be applied. + /// - `n_bytes`: The number of bytes per data element (4 for `float`, 8 for `double`). + /// - `chunk_dims`: A vector containing the dimensions of the data chunks. + /// - `mode`: The ZFP compression mode, which can be fixed rate, precision, accuracy, or reversible. + /// + /// # Returns + /// - `herr_t`: Returns 0 on success, or a negative value on failure. + unsafe fn apply_zfp( + plist_id: hid_t, n_bytes: u8, chunk_dims: Vec, mode: ZfpMode, + ) -> herr_t { + // get the chunk dimensiosn out of it. Could not reliably get the chunk_dims from plist_id + // during testing so opted to just pass it in during the build + + let ndims = chunk_dims.len(); + // Convert to `usize` and trim to used dims. + let chunk_dims_usize: Vec = + chunk_dims[..(ndims as usize)].iter().map(|&d| d as usize).collect(); + + // remove the singletons from the data + let mut dims_no_singleton: Vec = Vec::new(); + for &dim in chunk_dims_usize.iter() { + if dim != 1 { + dims_no_singleton.push(dim as u64); + } + } + let ndims_no_singleton = dims_no_singleton.len(); + + assert!(dims_no_singleton.len() <= zfp::MAX_NDIMS); + + // Get the type of the input data + let dtype_id = match n_bytes { + 4 => zfp_type_zfp_type_float, + 8 => zfp_type_zfp_type_double, + _ => { + h5err!( + "ZFP filter only supports 4 or 8 byte floating point data", + H5E_PLIST, + H5E_CALLBACK + ); + return -1; + } + }; + + // Build the Mode Information we need + let (mode_val, param1, param2) = match mode { + ZfpMode::FixedRate(rate) => { + let bits = rate.to_bits(); + (1, (bits >> 32) as c_uint, bits as c_uint) + } + ZfpMode::FixedPrecision(precision) => (2, precision as c_uint, 0), + ZfpMode::FixedAccuracy(accuracy) => { + let bits = accuracy.to_bits(); + (3, (bits >> 32) as c_uint, bits as c_uint) + } + ZfpMode::Reversible => (5, 0, 0), + }; + + // update values and encode into the header + let (hdr_cd_values, _) = + zfp::compute_hdr_cd_values(dtype_id, ndims_no_singleton, &dims_no_singleton, mode); + let hdf_cd_values_pass = hdr_cd_values.iter().map(|x| *x).collect::>(); + Self::apply_user(plist_id, zfp::ZFP_FILTER_ID, &hdf_cd_values_pass) + } + unsafe fn apply_user(plist_id: hid_t, filter_id: H5Z_filter_t, cdata: &[c_uint]) -> herr_t { // We're setting custom filters to optional, same way h5py does it, since // the only mention of H5Z_FLAG_MANDATORY in the HDF5 source itself is @@ -502,6 +748,9 @@ impl Filter { Self::Blosc(complib, clevel, shuffle) => { Self::apply_blosc(id, *complib, *clevel, *shuffle) } + #[cfg(feature = "zfp")] + Self::Zfp(mode, chunk_dims, n_bytes) => + Self::apply_zfp(id, *n_bytes, chunk_dims.clone(), *mode), Self::User(filter_id, ref cdata) => Self::apply_user(id, *filter_id, cdata), }); Ok(()) @@ -535,7 +784,8 @@ impl Filter { } } -const COMP_FILTER_IDS: &[H5Z_filter_t] = &[H5Z_FILTER_DEFLATE, H5Z_FILTER_SZIP, 32000, 32001]; +const COMP_FILTER_IDS: &[H5Z_filter_t] = + &[H5Z_FILTER_DEFLATE, H5Z_FILTER_SZIP, 32000, 32001, 32013]; pub(crate) fn validate_filters(filters: &[Filter], type_class: H5T_class_t) -> Result<()> { let mut map: HashMap = HashMap::new(); @@ -588,11 +838,14 @@ pub(crate) fn validate_filters(filters: &[Filter], type_class: H5T_class_t) -> R #[cfg(test)] mod tests { use hdf5_sys::h5t::H5T_class_t; + use ndarray::{Array2, Axis}; + use std::io::{Seek, SeekFrom}; use super::{ blosc_available, deflate_available, lzf_available, szip_available, validate_filters, Filter, FilterInfo, SZip, ScaleOffset, }; + use crate::hl::filters::zfp_available; use crate::test::with_tmp_file; use crate::{plist::DatasetCreate, Result}; @@ -610,9 +863,9 @@ mod tests { { comp_filters.push(Filter::lzf()); } - assert_eq!(cfg!(feature = "blosc-all"), blosc_available()); #[cfg(feature = "blosc-all")] { + assert!(blosc_available()); use super::BloscShuffle; comp_filters.push(Filter::blosc_blosclz(1, false)); comp_filters.push(Filter::blosc_lz4(3, true)); @@ -621,6 +874,16 @@ mod tests { comp_filters.push(Filter::blosc_zstd(9, BloscShuffle::Byte)); comp_filters.push(Filter::blosc_snappy(0, BloscShuffle::Bit)); } + + #[cfg(feature = "zfp")] + assert_eq!(cfg!(feature = "zfp"), zfp_available()); + #[cfg(feature = "zfp")] + { + comp_filters.push(Filter::zfp_rate(8.0, vec![10_000, 20], 4)); + comp_filters.push(Filter::zfp_precision(16, vec![10_000, 20], 4)); + comp_filters.push(Filter::zfp_accuracy(1e-3, vec![10_000, 20], 4)); + } + for c in &comp_filters { assert!(c.is_available()); assert!(c.encode_enabled()); @@ -672,4 +935,299 @@ mod tests { Ok(()) } + + #[test] + #[cfg(feature = "zfp")] + fn test_zfp_accuracy() -> Result<()> { + use super::zfp_available; + + if !zfp_available() { + println!("ZFP filter not available, skipping test"); + assert_eq!(1, 0); + return Ok(()); + } + + // test 1D Data + with_tmp_file(|file| { + let data = ndarray::Array1::::linspace(0.0, 1.0, 1000); + file.new_dataset_builder() + .with_data(&data) + .chunk((1000,)) + .zfp_accuracy(0.125, vec![1000], 4) + .create("zfp_precision_1d") + .unwrap(); + + let ds = file.dataset("zfp_precision_1d").unwrap(); + + let read_data: Vec = ds.read_raw().unwrap(); + + // ZFP is lossy, so we check approximate equality + assert_eq!(read_data.len(), data.len()); + + for (i, (original, compressed)) in data.iter().zip(read_data.iter()).enumerate() { + let diff = (original - compressed).abs(); + assert!( + diff < 0.1, + "Index {}: difference too large: {} vs {} (diff: {})", + i, + original, + compressed, + diff + ); + } + }); + + // Test 2D data + with_tmp_file(|file| { + let data = ndarray::Array1::::linspace(0.0, 1.0, 1000); + let data = data.to_shape((10, 100)).unwrap(); + file.new_dataset_builder() + .with_data(&data) + .chunk((5, 10)) + .zfp_accuracy(0.125, vec![5, 10], 4) + .create("zfp_precision_1d") + .unwrap(); + + let ds = file.dataset("zfp_precision_1d").unwrap(); + + let read_data: Vec = ds.read_raw().unwrap(); + + // ZFP is lossy, so we check approximate equality + assert_eq!(read_data.len(), data.len()); + + for (i, (original, compressed)) in data.iter().zip(read_data.iter()).enumerate() { + let diff = (original - compressed).abs(); + assert!( + diff < 0.125, + "Index {}: difference too large: {} vs {} (diff: {})", + i, + original, + compressed, + diff + ); + } + }); + + // Test 3D data + with_tmp_file(|file| { + let data = ndarray::Array1::::linspace(0.0, 1.0, 10000); + let data = data.to_shape((10, 10, 100)).unwrap(); + + file.new_dataset_builder() + .with_data(&data) + .chunk((2, 5, 25)) + .zfp_accuracy(0.125, vec![2, 5, 25], 4) + .create("zfp_precision_3d") + .unwrap(); + + let ds = file.dataset("zfp_precision_3d").unwrap(); + + let read_data: Vec = ds.read_raw().unwrap(); + let data_raw = data.as_slice().unwrap(); + + // ZFP is lossy, so we check approximate equality + assert_eq!(read_data.len(), data_raw.len()); + + for (i, (original, compressed)) in data_raw.iter().zip(read_data.iter()).enumerate() { + let diff = (original - compressed).abs(); + assert!( + diff < 0.125, + "Index {}: difference too large: {} vs {} (diff: {})", + i, + original, + compressed, + diff + ); + } + }); + + // Test 4D data + with_tmp_file(|file| { + let data = ndarray::Array1::::linspace(0.0, 1.0, 100000); + let data = data.to_shape((10, 10, 10, 100)).unwrap(); + file.new_dataset_builder() + .with_data(&data) + .chunk((2, 2, 5, 50)) + .zfp_accuracy(0.125, vec![2, 2, 5, 50], 4) + .create("zfp_precision_1d") + .unwrap(); + + let ds = file.dataset("zfp_precision_1d").unwrap(); + + let read_data: Vec = ds.read_raw().unwrap(); + let data_raw = data.as_slice().unwrap(); + + // ZFP is lossy, so we check approximate equality + assert_eq!(read_data.len(), data_raw.len()); + + for (i, (original, compressed)) in data_raw.iter().zip(read_data.iter()).enumerate() { + let diff = (original - compressed).abs(); + assert!( + diff < 0.125, + "Index {}: difference too large: {} vs {} (diff: {})", + i, + original, + compressed, + diff + ); + } + }); + Ok(()) + } + + #[test] + #[cfg(feature = "zfp")] + fn test_over_dim_data() -> Result<()> { + use super::zfp_available; + + if !zfp_available() { + println!("ZFP filter not available, skipping test"); + assert_eq!(1, 0); + return Ok(()); + } + + // Test 5D data with 3D chunks but should still fail + // test 1D Data + with_tmp_file(|file| { + let data = ndarray::Array1::::linspace(0.0, 1.0, 50_000); + let data = data.to_shape((2, 5, 10, 10, 50)).unwrap(); + + let bad_result = file + .new_dataset_builder() + .with_data(&data) + .chunk((2, 5, 5, 1, 1)) + .zfp_accuracy(0.125, vec![2, 5, 5, 1, 1], 4) + .create("zfp_precision_1d") + .unwrap_err(); + + assert_err!(bad_result, "ZFP filter supports up to 4D data only"); + }); + + Ok(()) + } + + #[test] + #[cfg(feature = "zfp")] + fn test_zfp_reversible() -> Result<()> { + use super::zfp_available; + + if !zfp_available() { + println!("ZFP filter not available, skipping test"); + assert_eq!(1, 0); + return Ok(()); + } + with_tmp_file(|file| { + let data = ndarray::Array1::::linspace(0.0, 1.0, 9600); + let data = data.insert_axis(Axis(0)); + let data = data.insert_axis(Axis(0)); + file.new_dataset_builder() + .chunk((1, 1, 960)) + .zfp_reversible(vec![1, 1, 960], 4) + .with_data(&data) + .create("zfp_reversible") + .unwrap(); + + let ds = file.dataset("zfp_reversible").unwrap(); + + let read_data: Vec = ds.read_raw().unwrap(); + let n_bytes = file.size(); + + // ZFP is lossy, so we check approximate equality + assert_eq!(read_data.len(), data.len()); + dbg!(&data.clone().into_raw_vec_and_offset().0[0..15]); + dbg!(&read_data[0..15]); + let target_bytes = (data.len() * 4) as u64; + assert!( + n_bytes <= target_bytes, + "Dataset size {} exceeds target {}", + n_bytes, + target_bytes + ); + assert_eq!(n_bytes, 29432); + // + for (i, (original, compressed)) in data.iter().zip(read_data.iter()).enumerate() { + let diff = (original - compressed).abs(); + dbg!(&diff); + assert!( + diff == 0.0, + "Index {}: difference too large: {} vs {} (diff: {})", + i, + original, + compressed, + diff + ); + } + }); + + Ok(()) + } + + #[test] + #[cfg(feature = "zfp")] + fn test_zfp_rate() -> Result<()> { + use super::zfp_available; + + if !zfp_available() { + println!("ZFP filter not available, skipping test"); + assert_eq!(1, 0); + return Ok(()); + } + with_tmp_file(|file| { + let data = ndarray::Array1::::linspace(0.0, 1.0, 1000); + file.new_dataset_builder() + .with_data(&data) + .chunk((1000,)) + .zfp_rate(2.0, vec![1000], 4) + .create("zfp_rate") + .unwrap(); + + let ds = file.dataset("zfp_rate").unwrap(); + + let read_data: Vec = ds.read_raw().unwrap(); + + // ZFP is lossy, so we check approximate equality + assert_eq!(read_data.len(), data.len()); + dbg!(&data.clone().into_raw_vec_and_offset().0[0..15]); + dbg!(&read_data[0..15]); + for (i, (original, compressed)) in data.iter().zip(read_data.iter()).enumerate() { + let diff = (original - compressed).abs(); + dbg!(&diff); + } + }); + + // test full rate compression. Should be "lossless" + + with_tmp_file(|file| { + let data = ndarray::Array1::::linspace(0.0, 1.0, 1000); + file.new_dataset_builder() + .with_data(&data) + .chunk((1000,)) + .zfp_rate(32.0, vec![1000], 4) + .create("zfp_rate") + .unwrap(); + + let ds = file.dataset("zfp_rate").unwrap(); + + let read_data: Vec = ds.read_raw().unwrap(); + + // ZFP is lossy, so we check approximate equality + assert_eq!(read_data.len(), data.len()); + dbg!(&data.clone().into_raw_vec_and_offset().0[0..15]); + dbg!(&read_data[0..15]); + for (i, (original, compressed)) in data.iter().zip(read_data.iter()).enumerate() { + let diff = (original - compressed).abs(); + dbg!(&diff); + assert!( + diff == 0.0, + "Index {}: difference too large: {} vs {} (diff: {})", + i, + original, + compressed, + diff + ); + } + }); + + Ok(()) + } } diff --git a/hdf5/src/hl/filters/blosc.rs b/hdf5/src/hl/filters/blosc.rs index af8021ffd..75c1a9905 100644 --- a/hdf5/src/hl/filters/blosc.rs +++ b/hdf5/src/hl/filters/blosc.rs @@ -1,7 +1,6 @@ use std::ptr::{self, addr_of_mut}; use std::slice; - -use lazy_static::lazy_static; +use std::sync::LazyLock; use hdf5_sys::h5p::{H5Pget_chunk, H5Pget_filter_by_id2, H5Pmodify_filter}; use hdf5_sys::h5t::{H5Tclose, H5Tget_class, H5Tget_size, H5Tget_super, H5T_ARRAY}; @@ -48,18 +47,16 @@ const BLOSC_FILTER_INFO: &H5Z_class2_t = &H5Z_class2_t { filter: Some(filter_blosc), }; -lazy_static! { - static ref BLOSC_INIT: Result<(), &'static str> = { - unsafe { - blosc_init(); - } - let ret = unsafe { H5Zregister((BLOSC_FILTER_INFO as *const H5Z_class2_t).cast()) }; - if H5ErrorCode::is_err_code(ret) { - return Err("Can't register Blosc filter"); - } - Ok(()) - }; -} +static BLOSC_INIT: LazyLock> = LazyLock::new(|| { + unsafe { + blosc_init(); + } + let ret = unsafe { H5Zregister((BLOSC_FILTER_INFO as *const H5Z_class2_t).cast()) }; + if H5ErrorCode::is_err_code(ret) { + return Err("Can't register Blosc filter"); + } + Ok(()) +}); pub fn register_blosc() -> Result<(), &'static str> { *BLOSC_INIT @@ -148,6 +145,9 @@ impl Default for BloscConfig { } fn parse_blosc_cdata(cd_nelmts: size_t, cd_values: *const c_uint) -> Option { + if cd_values.is_null() || cd_nelmts < 1 { + return None; + } let cdata = unsafe { slice::from_raw_parts(cd_values, cd_nelmts as _) }; let mut cfg = BloscConfig { typesize: cdata[2] as _, @@ -193,6 +193,9 @@ unsafe extern "C" fn filter_blosc( buf_size: *mut size_t, buf: *mut *mut c_void, ) -> size_t { let cfg = if let Some(cfg) = parse_blosc_cdata(cd_nelmts, cd_values) { + if cfg.typesize == 0 { + return 0; + } cfg } else { return 0; diff --git a/hdf5/src/hl/filters/lzf.rs b/hdf5/src/hl/filters/lzf.rs index 33f8e79e6..5aa095ca6 100644 --- a/hdf5/src/hl/filters/lzf.rs +++ b/hdf5/src/hl/filters/lzf.rs @@ -1,7 +1,7 @@ use std::ptr::{self, addr_of_mut}; use std::slice; +use std::sync::LazyLock; -use lazy_static::lazy_static; use lzf_sys::{lzf_compress, lzf_decompress, LZF_VERSION}; use hdf5_sys::h5p::{H5Pget_chunk, H5Pget_filter_by_id2, H5Pmodify_filter}; @@ -27,15 +27,13 @@ const LZF_FILTER_INFO: &H5Z_class2_t = &H5Z_class2_t { filter: Some(filter_lzf), }; -lazy_static! { - static ref LZF_INIT: Result<(), &'static str> = { - let ret = unsafe { H5Zregister((LZF_FILTER_INFO as *const H5Z_class2_t).cast()) }; - if H5ErrorCode::is_err_code(ret) { - return Err("Can't register LZF filter"); - } - Ok(()) - }; -} +static LZF_INIT: LazyLock> = LazyLock::new(|| { + let ret = unsafe { H5Zregister((LZF_FILTER_INFO as *const H5Z_class2_t).cast()) }; + if H5ErrorCode::is_err_code(ret) { + return Err("Can't register LZF filter"); + } + Ok(()) +}); pub fn register_lzf() -> Result<(), &'static str> { *LZF_INIT @@ -127,7 +125,11 @@ unsafe fn filter_lzf_decompress( cd_nelmts: size_t, cd_values: *const c_uint, nbytes: size_t, buf_size: *mut size_t, buf: *mut *mut c_void, ) -> size_t { - let cdata = slice::from_raw_parts(cd_values, cd_nelmts as _); + let cdata = if cd_values.is_null() || cd_nelmts < 1 { + &[] + } else { + slice::from_raw_parts(cd_values, cd_nelmts as _) + }; let mut outbuf_size = if cd_nelmts >= 3 && cdata[2] != 0 { cdata[2] as _ } else { *buf_size }; let mut outbuf: *mut c_void; let mut status: c_uint; diff --git a/hdf5/src/hl/filters/zfp.rs b/hdf5/src/hl/filters/zfp.rs new file mode 100644 index 000000000..c8411ddc6 --- /dev/null +++ b/hdf5/src/hl/filters/zfp.rs @@ -0,0 +1,718 @@ +use std::ptr::{self, addr_of_mut}; +use std::slice; +use std::sync::LazyLock; + +use hdf5_sys::h5p::{H5Pget_chunk, H5Pget_filter_by_id2, H5Pmodify_filter}; +use hdf5_sys::h5t::{H5Tclose, H5Tget_class, H5Tget_size, H5Tget_super, H5T_FLOAT}; +use hdf5_sys::h5z::{H5Z_class2_t, H5Z_filter_t, H5Zregister, H5Z_CLASS_T_VERS, H5Z_FLAG_REVERSE}; + +use crate::error::H5ErrorCode; +use crate::globals::{H5E_CALLBACK, H5E_PLIST}; +use crate::internal_prelude::*; + +use zfp_sys::zfp_stream; +pub use zfp_sys::{ + bitstream, stream_close, stream_open, zfp_codec_version, zfp_compress, zfp_decompress, + zfp_field, zfp_field_1d, zfp_field_2d, zfp_field_3d, zfp_field_4d, zfp_field_alloc, + zfp_field_dimensionality, zfp_field_free, zfp_field_metadata, zfp_field_size, zfp_field_type, + zfp_library_version, zfp_mode, zfp_mode_zfp_mode_fixed_accuracy, + zfp_mode_zfp_mode_fixed_precision, zfp_mode_zfp_mode_fixed_rate, zfp_read_header, + zfp_stream_accuracy, zfp_stream_close, zfp_stream_compression_mode, zfp_stream_flush, + zfp_stream_maximum_size, zfp_stream_open, zfp_stream_precision, zfp_stream_rate, + zfp_stream_rewind, zfp_stream_set_accuracy, zfp_stream_set_bit_stream, + zfp_stream_set_precision, zfp_stream_set_rate, zfp_stream_set_reversible, zfp_type, + zfp_type_zfp_type_double, zfp_type_zfp_type_float, zfp_write_header, ZFP_HEADER_FULL, + ZFP_HEADER_MAGIC, ZFP_HEADER_MAX_BITS, ZFP_HEADER_META, ZFP_HEADER_MODE, ZFP_VERSION_MAJOR, + ZFP_VERSION_MINOR, ZFP_VERSION_PATCH, ZFP_VERSION_TWEAK, +}; + +use crate::filters::ZfpMode; + +/// Major edits are needed to be in alignmeht with the H5Z-ZFP. What was previously implemented was +/// effectively a new implementation of H5Z_ZFP but was incompatible with any library built against +/// it. This results in bad c_data vectors being created and produces erratic behavior. + +pub(crate) const MAX_NDIMS: usize = 4; + +const ZFP_FILTER_NAME: &[u8] = b"zfp\0"; +pub const ZFP_FILTER_ID: H5Z_filter_t = 32013; +const ZFP_FILTER_VERSION: c_uint = 1; + +// ZFP mode constants +const ZFP_MODE_RATE: c_uint = 2; +const ZFP_MODE_PRECISION: c_uint = 3; +const ZFP_MODE_ACCURACY: c_uint = 4; +const ZFP_MODE_REVERSIBLE: c_uint = 5; +const ZFP_MODE_EXPERT: c_uint = 1; + +const ZFP_FILTER_INFO: &H5Z_class2_t = &H5Z_class2_t { + version: H5Z_CLASS_T_VERS as _, + id: ZFP_FILTER_ID, + encoder_present: 1, + decoder_present: 1, + name: ZFP_FILTER_NAME.as_ptr().cast(), + can_apply: Some(can_apply_zfp), + set_local: Some(set_local_zfp), + filter: Some(filter_zfp), +}; + +static ZFP_INIT: LazyLock> = LazyLock::new(|| { + let ret = unsafe { H5Zregister((ZFP_FILTER_INFO as *const H5Z_class2_t).cast()) }; + if H5ErrorCode::is_err_code(ret) { + return Err("Can't register ZFP filter"); + } + Ok(()) +}); + +pub fn register_zfp() -> Result<(), &'static str> { + *ZFP_INIT +} + +extern "C" fn can_apply_zfp(_dcpl_id: hid_t, type_id: hid_t, _space_id: hid_t) -> i32 { + let type_class = unsafe { H5Tget_class(type_id) }; + if type_class == H5T_FLOAT { + 1 + } else { + 0 + } +} + +/// Sets the local properties for the ZFP filter. +/// +/// This function is called during the creation of a dataset or attribute to set +/// the local properties of the ZFP filter. It retrieves the filter's configuration +/// data, validates the chunk dimensions, and updates the filter's parameters. +/// +/// # Parameters +/// - `dcpl_id`: The dataset creation property list identifier. +/// - `type_id`: The datatype identifier of the dataset or attribute. +/// - `_space_id`: The dataspace identifier (not used in this function). +/// +/// # Returns +/// - `herr_t`: Returns 1 on success, or -1 on failure. +extern "C" fn set_local_zfp(dcpl_id: hid_t, type_id: hid_t, _space_id: hid_t) -> herr_t { + const MAX_NDIMS: usize = 4; + let mut flags: c_uint = 0; + let mut nelmts: size_t = 4; + // start with a small buffer; H5Pget_filter_by_id2 will return the stored cdata (mode/params) + let mut values: Vec = vec![0; 4]; + let ret = unsafe { + H5Pget_filter_by_id2( + dcpl_id, + ZFP_FILTER_ID, + addr_of_mut!(flags), + addr_of_mut!(nelmts), + values.as_mut_ptr(), + 0, + ptr::null_mut(), + ptr::null_mut(), + ) + }; + if ret < 0 { + return -1; + } + // Preserve original small cdata (mode/params) returned by H5Pget_filter_by_id2. + let orig = values.clone(); + // ensure we have enough space for header + dims + parameters (we need at least indices up to 9) + nelmts = nelmts.max(10); + values.resize(nelmts as usize, 0); + // set version and header entries + values[0] = ZFP_FILTER_VERSION; + + let mut chunkdims: Vec = vec![0; MAX_NDIMS]; + let ndims: c_int = unsafe { H5Pget_chunk(dcpl_id, MAX_NDIMS as _, chunkdims.as_mut_ptr()) }; + if ndims < 0 { + return -1; + } + if ndims > MAX_NDIMS as _ { + h5err!("ZFP supports up to 4 dimensions", H5E_PLIST, H5E_CALLBACK); + return -1; + } + + let typesize: size_t = unsafe { H5Tget_size(type_id) }; + if typesize == 0 { + return -1; + } + + // fill header fields (ndims, typesize) and chunk dimensions + values[1] = ndims as c_uint; + values[2] = typesize as c_uint; + for i in 0..(ndims as usize).min(values.len().saturating_sub(3)) { + values[i + 3] = chunkdims[i] as c_uint; + } + // The Filter::apply_zfp() originally stored mode/param1/param2 at indices 0..2. + // parse_zfp expects these at indices 7..9 in the final cdata layout. Move/preserve them. + if values.len() >= 10 { + values[7] = orig.get(0).copied().unwrap_or(0); + values[8] = orig.get(1).copied().unwrap_or(0); + values[9] = orig.get(2).copied().unwrap_or(0); + } + // temp overrid and changed line 133 to orig instead of values + let nelmts = 4; + + let r = unsafe { H5Pmodify_filter(dcpl_id, ZFP_FILTER_ID, flags, nelmts, orig.as_ptr()) }; + if r < 0 { + -1 + } else { + 1 + } +} + +const H5Z_ZFP_CD_NELMTS_MAX: usize = 8; // whatever the header says; set correctly. + +/// Computes the header and configuration data values for the ZFP filter. +/// +/// This function generates the header and configuration data values (`cd_values`) +/// required for the ZFP filter. It creates a dummy ZFP field based on the provided +/// dimensions and data type, sets the compression mode, and writes the full header +/// into the `cd_values` buffer. +/// +/// # Parameters +/// - `zt`: The ZFP data type (e.g., `zfp_type_zfp_type_float` or `zfp_type_zfp_type_double`). +/// - `ndims_used`: The number of dimensions used in the data. +/// - `dims_used`: A slice containing the sizes of the dimensions. +/// - `mode`: The ZFP compression mode, which can be fixed rate, precision, accuracy, or reversible. +/// +/// # Returns +/// A tuple containing: +/// - `Vec`: The header and configuration data values. +/// - `usize`: The number of elements in the `cd_values` array. +/// +/// # Panics +/// This function will panic if the number of dimensions exceeds the supported range (1 to 4). +pub unsafe fn compute_hdr_cd_values( + zt: zfp_type, + ndims_used: usize, + dims_used: &[u64], + mode: ZfpMode, // your enum wrapping rate/precision/accuracy/reversible +) -> (Vec, usize) { + // 1. Build dummy_field like H5Z_zfp_set_local + let dummy_field: *mut zfp_field = match ndims_used { + 1 => zfp_field_1d(ptr::null_mut(), zt, dims_used[0].try_into().unwrap()), + 2 => zfp_field_2d( + ptr::null_mut(), + zt, + dims_used[1].try_into().unwrap(), + dims_used[0].try_into().unwrap(), + ), + 3 => zfp_field_3d( + ptr::null_mut(), + zt, + dims_used[2].try_into().unwrap(), + dims_used[1].try_into().unwrap(), + dims_used[0].try_into().unwrap(), + ), + 4 => zfp_field_4d( + ptr::null_mut(), + zt, + dims_used[3].try_into().unwrap(), + dims_used[2].try_into().unwrap(), + dims_used[1].try_into().unwrap(), + dims_used[0].try_into().unwrap(), + ), + _ => panic!("ZFP supports 1..4 non-unity dims"), + }; + assert!(!dummy_field.is_null()); + + // 2. Prepare the cd_values array like C code: u32 buffer + let mut hdr_cd_values = vec![0u32; H5Z_ZFP_CD_NELMTS_MAX]; + + // 3. Version word (use the macro layout: (ZFP_VERSION_NO<<16)|(ZFP_CODEC<<12)|H5Z_FILTER_ZFP_VERSION_NO) + hdr_cd_values[0] = make_version_word(); // see previous message + + // 4. Treat &hdr_cd_values[1] as bitstream buffer + let ptr_bytes = hdr_cd_values[1..].as_mut_ptr() as *mut c_void; + let bytes_len = (hdr_cd_values.len() - 1) * std::mem::size_of::(); + + let dummy_bstr: *mut bitstream = stream_open(ptr_bytes, bytes_len as usize); + + let dummy_zstr: *mut zfp_stream = zfp_stream_open(dummy_bstr); + // 5. Set mode the same way H5Z_zfp_set_local does + match mode { + ZfpMode::Reversible => { + zfp_stream_set_reversible(dummy_zstr); + } + ZfpMode::FixedAccuracy(acc) => { + zfp_stream_set_accuracy(dummy_zstr, acc); + } + + ZfpMode::FixedRate(rate) => { + zfp_stream_set_rate(dummy_zstr, rate, zt, ndims_used as u32, 0); + } + ZfpMode::FixedPrecision(precision) => { + zfp_stream_set_precision(dummy_zstr, precision as u32); + } + // handle Rate/Precision/Accuracy/Expert as needed + _ => unimplemented!(), + } + + // 6. Write FULL header (critical!) into the hdr_cd_values[1..] buffer + let hdr_bits = zfp_write_header(dummy_zstr, dummy_field, ZFP_HEADER_FULL as u32); + assert!(hdr_bits != 0); + + // 7. Flush and close (exactly like C) + zfp_stream_flush(dummy_zstr); + zfp_stream_close(dummy_zstr); + stream_close(dummy_bstr); + zfp_field_free(dummy_field); + + // 8. Compute hdr_bytes/hdr_cd_nelmts as in C + let hdr_bytes = 1 + ((hdr_bits - 1) / 8); + let mut hdr_cd_nelmts = 1 + ((hdr_bytes - 1) / std::mem::size_of::()); + hdr_cd_nelmts += 1; // for slot 0 + + (hdr_cd_values, hdr_cd_nelmts) +} + +/// Constructs a version word for the ZFP filter. +/// +/// This function generates a 32-bit version word that encodes the ZFP library version, +/// codec version, and filter version. The version word is structured as follows: +/// - High 24 bits: ZFP library version (major, minor, patch, tweak). +/// - Middle 8 bits: Codec version. +/// - Low 8 bits: Filter version. +/// +/// # Returns +/// A 32-bit unsigned integer representing the version word. +unsafe fn make_version_word() -> u32 { + // 0xM M P T: for 1.0.0.0 → 0x1000 + const ZFP_VERSION_NO: u32 = (ZFP_VERSION_MAJOR << 12) + | (ZFP_VERSION_MINOR << 8) + | (ZFP_VERSION_PATCH << 4) + | (ZFP_VERSION_TWEAK); + + const ZFP_CODEC: u32 = ZFP_VERSION_MINOR; // or 5 if you know you want codec 5 + + // Filter version: 1.1.0 → 0x0110 + const H5Z_FILTER_ZFP_VERSION_MAJOR: u32 = 1; + const H5Z_FILTER_ZFP_VERSION_MINOR: u32 = 1; + const H5Z_FILTER_ZFP_VERSION_PATCH: u32 = 0; + + const H5Z_FILTER_ZFP_VERSION_NO: u32 = (H5Z_FILTER_ZFP_VERSION_MAJOR << 8) + | (H5Z_FILTER_ZFP_VERSION_MINOR << 4) + | (H5Z_FILTER_ZFP_VERSION_PATCH); + + // One simple scheme: low 8 bits = codec, high 24 bits = lib version truncated. + (ZFP_VERSION_NO << 16) | (ZFP_CODEC << 12) | H5Z_FILTER_ZFP_VERSION_NO +} + +#[derive(Debug)] +struct ZfpConfig { + pub ndims: c_int, + pub typesize: size_t, + pub dims: [size_t; 4], + pub mode: c_uint, + pub rate: f64, + pub precision: u32, + pub accuracy: f64, +} + +/// Parses ZFP filter configuration data from the given input. +/// +/// This function extracts metadata and compression parameters from the +/// provided `cd_values` array, which represents the ZFP filter's configuration +/// data. It handles endian mismatches, validates the header, and retrieves +/// information such as dimensions, data type, and compression mode. +/// +/// # Safety +/// This function is marked as unsafe because it performs raw pointer +/// dereferencing and interacts with low-level C APIs, which require careful +/// handling to avoid undefined behavior. +/// +/// # Parameters +/// - `cd_nelmts`: The number of elements in the `cd_values` array. +/// - `cd_values`: A pointer to the array of configuration data values. +/// +/// # Returns +/// - `Option`: Returns a `ZfpConfig` struct containing the parsed +/// metadata and compression parameters if successful, or `None` if the +/// parsing fails. +pub unsafe fn parse_zfp_cdata(cd_nelmts: usize, cd_values: *const c_uint) -> Option { + if cd_nelmts < 2 || cd_values.is_null() { + return None; + } + + // Full cd array from HDF5: [version_word, header_words...] + let cdata: &[u32] = slice::from_raw_parts(cd_values, cd_nelmts); + + // ignore the version word, + let _version_word = cdata[0]; + + // ZFP header bitstream. + let header_words = &cdata[1..]; + if header_words.is_empty() { + return None; + } + + // Make a mutable copy so we can endian-swap in place if needed. + let mut header_copy: Vec = header_words.to_vec(); + let header_bytes = header_copy.len() * std::mem::size_of::(); + + // Open bitstream on the header buffer (like get_zfp_info_from_cd_values) + let bstr: *mut bitstream = stream_open(header_copy.as_mut_ptr() as *mut c_void, header_bytes); + if bstr.is_null() { + return None; + } + + // Open zfp_stream on that bitstream + let zstr: *mut zfp_stream = zfp_stream_open(bstr); + if zstr.is_null() { + stream_close(bstr); + return None; + } + + // Allocate a field for header metadata + let zfld: *mut zfp_field = zfp_field_alloc(); + if zfld.is_null() { + zfp_stream_close(zstr); + stream_close(bstr); + return None; + } + + //First read only MAGIC, to detect endian or codec mismatch + let mut bits = zfp_read_header(zstr, zfld, ZFP_HEADER_MAGIC); + if bits == 0 { + // Possible endian mismatch: byte-swap each u32 and retry. + for w in &mut header_copy { + *w = w.swap_bytes(); + } + + zfp_stream_rewind(zstr); + bits = zfp_read_header(zstr, zfld, ZFP_HEADER_MAGIC); + if bits == 0 { + zfp_field_free(zfld); + zfp_stream_close(zstr); + stream_close(bstr); + return None; + } + } + + //Rewind and read the full header. + zfp_stream_rewind(zstr); + if zfp_read_header(zstr, zfld, ZFP_HEADER_FULL) == 0 { + zfp_field_free(zfld); + zfp_stream_close(zstr); + stream_close(bstr); + return None; + } + + //Extract array metadata + let ndims = zfp_field_dimensionality(zfld) as i32; + + // zfp_field_size can fill per-dimension sizes; pass a buffer. + let mut size_per_dim: [usize; 4] = [0; 4]; + if ndims > 0 { + // zfp_field_size returns total number of elements and optionally fills size[i]. + // The C signature uses size_t*; just alias &mut [usize] here. + zfp_field_size(zfld, size_per_dim.as_mut_ptr() as *mut _); + } + + let mut dims: [usize; 4] = [0; 4]; + for i in 0..(ndims as usize).min(4) { + dims[i] = size_per_dim[i]; + } + + // Scalar type → element size in bytes. + let zt: zfp_type = zfp_field_type(zfld); + let typesize: usize = match zt { + // Adjust these to match the actual enum variants in your bindings + x if x == zfp_sys::zfp_type_zfp_type_int32 => std::mem::size_of::(), + x if x == zfp_sys::zfp_type_zfp_type_int64 => std::mem::size_of::(), + x if x == zfp_sys::zfp_type_zfp_type_float => std::mem::size_of::(), + x if x == zfp_sys::zfp_type_zfp_type_double => std::mem::size_of::(), + _ => { + zfp_field_free(zfld); + zfp_stream_close(zstr); + stream_close(bstr); + return None; + } + }; + // Extract compression mode and parameters from the stream itself. + let zmode_enum: zfp_mode = zfp_stream_compression_mode(zstr); + let mode = zmode_enum as u32; + + let mut rate: f64 = 0.0; + let mut precision: u32 = 0; + let mut accuracy: f64 = 0.0; + + // These getters are available on modern zfp (1.0+). + match zmode_enum { + m if m == zfp_sys::zfp_mode_zfp_mode_fixed_rate => { + rate = zfp_stream_rate(zstr, ndims as u32); + } + m if m == zfp_sys::zfp_mode_zfp_mode_fixed_precision => { + precision = zfp_stream_precision(zstr); + } + m if m == zfp_sys::zfp_mode_zfp_mode_fixed_accuracy => { + accuracy = zfp_stream_accuracy(zstr); + } + m if m == zfp_sys::zfp_mode_zfp_mode_reversible => { + // no params needed + } + + // Expert or reversible -> we don’t have a single scalar parameter to expose + _ => {} + } + + //Cleanup + zfp_field_free(zfld); + zfp_stream_close(zstr); + stream_close(bstr); + + Some(ZfpConfig { ndims, typesize, dims, mode, rate, precision, accuracy }) +} + +/// Applies the ZFP filter for compression or decompression. +/// +/// This function serves as the entry point for the ZFP filter, determining whether +/// to compress or decompress the data based on the provided flags. It parses the +/// filter configuration data, validates it, and then delegates the operation to +/// either the compression or decompression function. +/// +/// # Safety +/// This function is marked as unsafe because it interacts with raw pointers and +/// performs operations that require careful handling to avoid undefined behavior. +/// +/// # Parameters +/// - `flags`: A bitmask indicating the operation mode (e.g., compression or decompression). +/// - `cd_nelmts`: The number of elements in the `cd_values` array. +/// - `cd_values`: A pointer to the array of configuration data values. +/// - `nbytes`: The size of the input buffer in bytes. +/// - `buf_size`: A pointer to the size of the output buffer. +/// - `buf`: A pointer to the input/output buffer. +/// +/// # Returns +/// - `size_t`: The size of the processed data (compressed or decompressed) on success, +/// or 0 on failure. +unsafe extern "C" fn filter_zfp( + flags: c_uint, cd_nelmts: size_t, cd_values: *const c_uint, nbytes: size_t, + buf_size: *mut size_t, buf: *mut *mut c_void, +) -> size_t { + let cfg = if let Some(cfg) = parse_zfp_cdata(cd_nelmts, cd_values) { + cfg + } else { + return 0; + }; + + if flags & H5Z_FLAG_REVERSE == 0 { + unsafe { filter_zfp_compress(&cfg, buf_size, buf) } + } else { + unsafe { filter_zfp_decompress(&cfg, nbytes, buf_size, buf) } + } +} + +unsafe fn filter_zfp_compress( + cfg: &ZfpConfig, buf_size: *mut size_t, buf: *mut *mut c_void, +) -> size_t { + let zfp_stream = zfp_stream_open(ptr::null_mut()); + if zfp_stream.is_null() { + h5err!("Failed to open ZFP stream", H5E_PLIST, H5E_CALLBACK); + return 0; + } + + match cfg.mode { + ZFP_MODE_RATE => { + zfp_stream_set_rate(zfp_stream, cfg.rate, cfg.typesize as _, cfg.ndims as _, 0); + } + ZFP_MODE_PRECISION => { + zfp_stream_set_precision(zfp_stream, cfg.precision); + } + ZFP_MODE_ACCURACY => { + zfp_stream_set_accuracy(zfp_stream, cfg.accuracy); + } + ZFP_MODE_REVERSIBLE => zfp_stream_set_reversible(zfp_stream), + _ => { + zfp_stream_close(zfp_stream); + return 0; + } + } + + let field = if cfg.typesize == 4 { + match cfg.ndims { + 1 => zfp_field_1d((*buf).cast(), zfp_type_zfp_type_float, cfg.dims[0]), + 2 => zfp_field_2d((*buf).cast(), zfp_type_zfp_type_float, cfg.dims[0], cfg.dims[1]), + 3 => zfp_field_3d( + (*buf).cast(), + zfp_type_zfp_type_float, + cfg.dims[0], + cfg.dims[1], + cfg.dims[2], + ), + 4 => zfp_field_4d( + (*buf).cast(), + zfp_type_zfp_type_float, + cfg.dims[0], + cfg.dims[1], + cfg.dims[2], + cfg.dims[3], + ), + _ => ptr::null_mut(), + } + } else { + match cfg.ndims { + 1 => zfp_field_1d((*buf).cast(), zfp_type_zfp_type_double, cfg.dims[0]), + 2 => zfp_field_2d((*buf).cast(), zfp_type_zfp_type_double, cfg.dims[0], cfg.dims[1]), + 3 => zfp_field_3d( + (*buf).cast(), + zfp_type_zfp_type_double, + cfg.dims[0], + cfg.dims[1], + cfg.dims[2], + ), + 4 => zfp_field_4d( + (*buf).cast(), + zfp_type_zfp_type_double, + cfg.dims[0], + cfg.dims[1], + cfg.dims[2], + cfg.dims[3], + ), + _ => ptr::null_mut(), + } + }; + if field.is_null() { + zfp_stream_close(zfp_stream); + h5err!("Failed to create ZFP field", H5E_PLIST, H5E_CALLBACK); + return 0; + } + + let maxsize = zfp_stream_maximum_size(zfp_stream, field); + let outbuf = libc::malloc(maxsize); + if outbuf.is_null() { + zfp_field_free(field); + zfp_stream_close(zfp_stream); + h5err!("Can't allocate compression buffer", H5E_PLIST, H5E_CALLBACK); + return 0; + } + + let bitstream = stream_open(outbuf.cast(), maxsize); + zfp_stream_set_bit_stream(zfp_stream, bitstream); + zfp_stream_rewind(zfp_stream); + + let compressed_size = zfp_compress(zfp_stream, field); + stream_close(bitstream); + zfp_field_free(field); + zfp_stream_close(zfp_stream); + + if compressed_size == 0 { + libc::free(outbuf); + h5err!("ZFP compression failed", H5E_PLIST, H5E_CALLBACK); + return 0; + } + + libc::free(*buf); + *buf = outbuf; + *buf_size = compressed_size; + compressed_size +} + +unsafe fn filter_zfp_decompress( + cfg: &ZfpConfig, nbytes: size_t, buf_size: *mut size_t, buf: *mut *mut c_void, +) -> size_t { + let zfp_stream = zfp_stream_open(ptr::null_mut()); + if zfp_stream.is_null() { + h5err!("Failed to open ZFP stream", H5E_PLIST, H5E_CALLBACK); + return 0; + } + + match cfg.mode { + ZFP_MODE_RATE => { + zfp_stream_set_rate(zfp_stream, cfg.rate, cfg.typesize as _, cfg.ndims as _, 0); + } + ZFP_MODE_PRECISION => { + zfp_stream_set_precision(zfp_stream, cfg.precision); + } + ZFP_MODE_ACCURACY => { + zfp_stream_set_accuracy(zfp_stream, cfg.accuracy); + } + ZFP_MODE_REVERSIBLE => zfp_stream_set_reversible(zfp_stream), + _ => { + zfp_stream_close(zfp_stream); + return 0; + } + } + + let mut outbuf_size = cfg.typesize; + for i in 0..cfg.ndims as usize { + outbuf_size *= cfg.dims[i]; + } + + let outbuf = libc::malloc(outbuf_size); + if outbuf.is_null() { + zfp_stream_close(zfp_stream); + h5err!("Can't allocate decompression buffer", H5E_PLIST, H5E_CALLBACK); + return 0; + } + + let field = if cfg.typesize == 4 { + match cfg.ndims { + 1 => zfp_field_1d(outbuf.cast(), zfp_type_zfp_type_float, cfg.dims[0]), + 2 => zfp_field_2d(outbuf.cast(), zfp_type_zfp_type_float, cfg.dims[0], cfg.dims[1]), + 3 => zfp_field_3d( + outbuf.cast(), + zfp_type_zfp_type_float, + cfg.dims[0], + cfg.dims[1], + cfg.dims[2], + ), + 4 => zfp_field_4d( + outbuf.cast(), + zfp_type_zfp_type_float, + cfg.dims[0], + cfg.dims[1], + cfg.dims[2], + cfg.dims[3], + ), + _ => ptr::null_mut(), + } + } else { + match cfg.ndims { + 1 => zfp_field_1d(outbuf.cast(), zfp_type_zfp_type_double, cfg.dims[0]), + 2 => zfp_field_2d(outbuf.cast(), zfp_type_zfp_type_double, cfg.dims[0], cfg.dims[1]), + 3 => zfp_field_3d( + outbuf.cast(), + zfp_type_zfp_type_double, + cfg.dims[0], + cfg.dims[1], + cfg.dims[2], + ), + 4 => zfp_field_4d( + outbuf.cast(), + zfp_type_zfp_type_double, + cfg.dims[0], + cfg.dims[1], + cfg.dims[2], + cfg.dims[3], + ), + _ => ptr::null_mut(), + } + }; + + if field.is_null() { + libc::free(outbuf); + zfp_stream_close(zfp_stream); + h5err!("Failed to create ZFP field", H5E_PLIST, H5E_CALLBACK); + return 0; + } + + let bitstream = stream_open((*buf).cast(), nbytes); + zfp_stream_set_bit_stream(zfp_stream, bitstream); + zfp_stream_rewind(zfp_stream); + + let status = zfp_decompress(zfp_stream, field); + + stream_close(bitstream); + zfp_field_free(field); + zfp_stream_close(zfp_stream); + + if status == 0 { + libc::free(outbuf); + h5err!("ZFP decompression failed", H5E_PLIST, H5E_CALLBACK); + return 0; + } + + libc::free(*buf); + *buf = outbuf; + *buf_size = outbuf_size; + outbuf_size +} diff --git a/hdf5/src/hl/group.rs b/hdf5/src/hl/group.rs index ea85c361f..66c2e1bec 100644 --- a/hdf5/src/hl/group.rs +++ b/hdf5/src/hl/group.rs @@ -261,10 +261,14 @@ impl From for H5_iter_order_t { } } +/// The type of an object link. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum LinkType { + /// A hard link to an object within a single file. Hard, + /// A symbolic link to an object within a single file. Soft, + /// A symbolic link to an object in a different file. External, } @@ -278,6 +282,7 @@ impl From for LinkType { } } +/// Metadata describing an object link. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct LinkInfo { pub link_type: LinkType, @@ -617,9 +622,14 @@ pub mod tests { file.new_dataset::().no_chunk().shape((10, 20)).create("a/foo").unwrap(); file.new_dataset::().no_chunk().shape((10, 20)).create("a/123").unwrap(); file.new_dataset::().no_chunk().shape((10, 20)).create("a/bar").unwrap(); - assert_eq!(group_a.member_names().unwrap(), vec!["123", "bar", "foo"]); + let group_a_names = group_a.member_names().unwrap(); + assert!(group_a_names.contains(&"123".to_string())); + assert!(group_a_names.contains(&"bar".to_string())); + assert!(group_a_names.contains(&"foo".to_string())); assert_eq!(group_b.member_names().unwrap().len(), 0); - assert_eq!(file.member_names().unwrap(), vec!["a", "b"]); + let file_names = file.member_names().unwrap(); + assert!(file_names.contains(&"a".to_string())); + assert!(file_names.contains(&"b".to_string())); }) } diff --git a/hdf5/src/hl/location.rs b/hdf5/src/hl/location.rs index 2f68e378b..9f9261a52 100644 --- a/hdf5/src/hl/location.rs +++ b/hdf5/src/hl/location.rs @@ -3,6 +3,7 @@ use std::mem::MaybeUninit; use std::ops::Deref; use std::ptr; +use hdf5_sys::h5o::H5Ocopy; #[allow(deprecated)] use hdf5_sys::h5o::H5Oset_comment; #[cfg(feature = "1.12.0")] @@ -91,7 +92,7 @@ impl Location { comment.and_then(|c| if c.is_empty() { None } else { Some(c) }) } - /// Set or the comment attached to the named object. + /// Set the comment attached to the named object. #[deprecated(note = "attributes are preferred to comments")] pub fn set_comment(&self, comment: &str) -> Result<()> { // TODO: &mut self? @@ -108,19 +109,27 @@ impl Location { h5call!(H5Oset_comment(self.id(), ptr::null_mut())).and(Ok(())) } + /// Create a builder for a new attribute of known type. pub fn new_attr(&self) -> AttributeBuilderEmpty { AttributeBuilder::new(self).empty::() } + /// Create a builder for a new attribute. pub fn new_attr_builder(&self) -> AttributeBuilder { AttributeBuilder::new(self) } + /// Create a new named attribute on the object. pub fn attr(&self, name: &str) -> Result { let name = to_cstring(name)?; Attribute::from_id(h5try!(H5Aopen(self.id(), name.as_ptr(), H5P_DEFAULT))) } + /// Return the names of all attributes on the object. + /// + /// # Errors + /// + /// Returns an error if an underlying library call fails. pub fn attr_names(&self) -> Result> { Attribute::attr_names(self) } @@ -131,24 +140,37 @@ impl Location { Ok(()) } + /// Returns the object's metadata. pub fn loc_info(&self) -> Result { H5O_get_info(self.id(), true) } + /// Returns the object's type. pub fn loc_type(&self) -> Result { Ok(H5O_get_info(self.id(), false)?.loc_type) } + /// Returns the metadata of another object with name relative to `self`. + /// + /// # Errors + /// + /// Returns an error if the name is invalid. pub fn loc_info_by_name(&self, name: &str) -> Result { let name = to_cstring(name)?; H5O_get_info_by_name(self.id(), name.as_ptr(), true) } + /// Returns the type of another object with name relative to `self`. + /// + /// # Errors + /// + /// Returns an error if the name is invalid. pub fn loc_type_by_name(&self, name: &str) -> Result { let name = to_cstring(name)?; Ok(H5O_get_info_by_name(self.id(), name.as_ptr(), false)?.loc_type) } + /// Opens an object using its location token. pub fn open_by_token(&self, token: LocationToken) -> Result { H5O_open_by_token(self.id(), token) } @@ -166,14 +188,59 @@ impl Location { pub fn dereference(&self, reference: &R) -> Result { reference.dereference(self) } + + /// Copy this object to a destination location with default properties + pub fn copy_to(&self, dst_loc: &Location, dst_name: &str) -> Result<()> { + self.copy_to_with_props(dst_loc, dst_name, None, None) + } + + /// Copy this object to a destination location with custom property lists + /// + /// # Arguments + /// * `dst_loc` - Destination location (file or group) + /// * `dst_name` - Name for the copied object at the destination + /// * `ocpypl` - Optional object copy property list (controls copy behavior) + /// * `lcpl` - Optional link creation property list (controls link properties) + pub fn copy_to_with_props( + &self, dst_loc: &Location, dst_name: &str, ocpypl: Option<&PropertyList>, + lcpl: Option<&PropertyList>, + ) -> Result<()> { + // Validate property list classes if provided + if let Some(pl) = ocpypl { + if !pl.is_class(PropertyListClass::ObjectCopy) { + fail!("Property list must be of class ObjectCopy"); + } + } + if let Some(pl) = lcpl { + if !pl.is_class(PropertyListClass::LinkCreate) { + fail!("Property list must be of class LinkCreate"); + } + } + + let dst_name = to_cstring(dst_name)?; + let ocpypl_id = ocpypl.map_or(H5P_DEFAULT, |p| p.id()); + let lcpl_id = lcpl.map_or(H5P_DEFAULT, |p| p.id()); + + h5call!(H5Ocopy( + self.id(), // src_loc_id + b".\0".as_ptr() as *const c_char, // src_name (current object) + dst_loc.id(), // dst_loc_id + dst_name.as_ptr(), // dst_name + ocpypl_id, // ocpypl_id + lcpl_id // lcpl_id + ))?; + Ok(()) + } } +/// A token containing the address or identifier of a [`Location`]. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct LocationToken( #[cfg(not(feature = "1.12.0"))] haddr_t, #[cfg(feature = "1.12.0")] H5O_token_t, ); +/// The type of an object in a [`Location`]. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum LocationType { Group, @@ -320,7 +387,7 @@ fn H5O_open_by_token(loc_id: hid_t, token: LocationToken) -> Result { #[cfg(test)] pub mod tests { - use crate::internal_prelude::*; + use crate::{hl::plist::object_copy::ObjectCopy, internal_prelude::*, plist::LinkCreate}; #[test] pub fn test_filename() { @@ -437,4 +504,143 @@ pub mod tests { assert!(file.loc_info_by_name("gibberish").is_err()); }) } + + #[test] + pub fn test_copy_dataset_between_files() { + with_tmp_path(|src_path| { + with_tmp_path(|dst_path| { + // Create source file with a dataset + let src_file = File::create(&src_path).unwrap(); + let src_group = src_file.create_group("src_group").unwrap(); + + let src_dataset = + src_group.new_dataset::().shape([5]).create("src_group").unwrap(); + src_dataset.write(&[1, 2, 3, 4, 5]).unwrap(); + + let src_attr = src_dataset.new_attr::().create("src_attr").unwrap(); + src_attr.write_scalar(&42.0).unwrap(); + + let dst_file = File::create(&dst_path).unwrap(); + let dst_group = dst_file.create_group("dst_group").unwrap(); + + // Copy the dataset from source to destination and verify contents + src_dataset.copy_to(&dst_group, "copied_dataset").unwrap(); + + let copied_dataset = dst_group.dataset("copied_dataset").unwrap(); + let read_data: Vec = copied_dataset.read_1d().unwrap().to_vec(); + assert_eq!(read_data, &[1, 2, 3, 4, 5]); + + let copied_attr = copied_dataset.attr("src_attr").unwrap(); + let attr_value: f64 = copied_attr.read_scalar().unwrap(); + assert_eq!(attr_value, 42.0); + }) + }) + } + + #[test] + pub fn test_copy_group_with_nested_content() { + with_tmp_path(|src_path| { + with_tmp_path(|dst_path| { + // Create source file with nested structure + let src_file = File::create(&src_path).unwrap(); + let group1 = src_file.create_group("group1").unwrap(); + let subgroup = group1.create_group("subgroup").unwrap(); + + let ds1 = group1.new_dataset::().shape([3]).create("dataset1").unwrap(); + ds1.write(&[10, 20, 30]).unwrap(); + let ds2 = subgroup.new_dataset::().shape([2]).create("dataset2").unwrap(); + ds2.write(&[1.5, 2.5]).unwrap(); + + // Create destination file and copy entire group structure + let dst_file = File::create(&dst_path).unwrap(); + group1.copy_to(&dst_file, "copied_group1").unwrap(); + + // Verify the copied structure + let copied_group = dst_file.group("copied_group1").unwrap(); + + let copied_ds1 = copied_group.dataset("dataset1").unwrap(); + let data1: Vec = copied_ds1.read_1d().unwrap().to_vec(); + assert_eq!(data1, vec![10, 20, 30]); + + let copied_subgroup = copied_group.group("subgroup").unwrap(); + let copied_ds2 = copied_subgroup.dataset("dataset2").unwrap(); + let data2: Vec = copied_ds2.read_1d().unwrap().to_vec(); + assert_eq!(data2, vec![1.5, 2.5]); + }) + }) + } + + #[test] + pub fn test_copy_without_attributes() { + with_tmp_path(|src_path| { + with_tmp_path(|dst_path| { + let src_file = File::create(&src_path).unwrap(); + + let dataset = src_file.new_dataset::().shape([3]).create("data").unwrap(); + dataset.write(&[10, 20, 30]).unwrap(); + + let src_attr = dataset.new_attr::().create("foo_attr").unwrap(); + src_attr.write_scalar(&100).unwrap(); + + let dst_file = File::create(&dst_path).unwrap(); + + let ocpypl = ObjectCopy::build().copy_without_attr(true).finish().unwrap(); + + // Copy without attributes + dataset + .copy_to_with_props(&dst_file, "copied_no_attrs", Some(&ocpypl), None) + .unwrap(); + + src_file.close().unwrap(); + + let copied = dst_file.dataset("copied_no_attrs").unwrap(); + let copied_data: Vec = copied.read_1d().unwrap().to_vec(); + assert_eq!(copied_data, vec![10, 20, 30]); + + // Verify attributes were NOT copied + let attr_names = copied.attr_names().unwrap(); + assert!(attr_names.is_empty(), "Expected no attributes, but found: {attr_names:?}",); + }) + }) + } + + #[test] + pub fn test_copy_with_link_create_intermediate_groups() { + with_tmp_path(|src_path| { + with_tmp_path(|dst_path| { + let src_file = File::create(&src_path).unwrap(); + + let dataset = src_file.new_dataset::().shape([3]).create("data").unwrap(); + dataset.write(&[100, 200, 300]).unwrap(); + + let dst_file = File::create(&dst_path).unwrap(); + + // Fails without setting LinkCreate plist + assert!(dataset.copy_to(&dst_file, "level1/level2/level3/copied_data").is_err()); + + // Succeeds with LinkCreate + // Create link create property list that creates intermediate groups + dataset + .copy_to_with_props( + &dst_file, + "level1/level2/level3/copied_data", + None, + Some( + &LinkCreate::build().create_intermediate_group(true).finish().unwrap(), + ), + ) + .unwrap(); + + // Verify the intermediate groups were created + assert!(dst_file.group("level1").is_ok()); + assert!(dst_file.group("level1/level2").is_ok()); + assert!(dst_file.group("level1/level2/level3").is_ok()); + + // Verify the data + let copied = dst_file.dataset("level1/level2/level3/copied_data").unwrap(); + let data: Vec = copied.read_1d().unwrap().to_vec(); + assert_eq!(data, vec![100, 200, 300]); + }) + }) + } } diff --git a/hdf5/src/hl/object.rs b/hdf5/src/hl/object.rs index cabe0e970..f6d8724c5 100644 --- a/hdf5/src/hl/object.rs +++ b/hdf5/src/hl/object.rs @@ -29,6 +29,7 @@ impl Debug for Object { } impl Object { + /// Returns the object's identifier. pub fn id(&self) -> hid_t { self.0.id() } @@ -57,7 +58,7 @@ impl Object { macro_rules! impl_downcast { ($func:ident, $tp:ty) => { impl Object { - #[doc = "Downcast the object into $tp if possible."] + #[doc = concat!("Downcast the object into `", stringify!($tp), "` if possible.")] pub fn $func(&self) -> Result<$tp> { self.clone().cast() } diff --git a/hdf5/src/hl/plist.rs b/hdf5/src/hl/plist.rs index f80ea91c2..bdb4b013e 100644 --- a/hdf5/src/hl/plist.rs +++ b/hdf5/src/hl/plist.rs @@ -17,6 +17,7 @@ pub mod dataset_create; pub mod file_access; pub mod file_create; pub mod link_create; +pub mod object_copy; /// Represents the HDF5 property list. #[repr(transparent)] @@ -93,7 +94,7 @@ pub enum PropertyListClass { LinkCreate, /// Properties for object copying process. ObjectCopy, - /// Properties for object creatio. + /// Properties for object creation. ObjectCreate, /// Properties for character encoding. StringCreate, @@ -210,6 +211,7 @@ impl PropertyList { }) } + /// Returns `true` if the property list is a member of `class`. pub fn is_class(&self, class: PropertyListClass) -> bool { use crate::globals::*; h5lock!({ diff --git a/hdf5/src/hl/plist/dataset_access.rs b/hdf5/src/hl/plist/dataset_access.rs index 5a123aea2..4670c20d4 100644 --- a/hdf5/src/hl/plist/dataset_access.rs +++ b/hdf5/src/hl/plist/dataset_access.rs @@ -91,10 +91,13 @@ impl Clone for DatasetAccess { } } +/// Options for including or excluding missing mapped elements in a virtual dataset view. #[cfg(feature = "1.10.0")] #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum VirtualView { + /// Include all data before the first missing mapped data. FirstMissing, + /// Include all available mapped data, filling missing data with a fill value. LastAvailable, } @@ -165,29 +168,35 @@ impl DatasetAccessBuilder { Ok(builder) } + /// Sets the [`ChunkCache`] options. pub fn chunk_cache(&mut self, nslots: usize, nbytes: usize, w0: f64) -> &mut Self { self.chunk_cache = Some(ChunkCache { nslots, nbytes, w0 }); self } + /// Sets the external dataset storage file prefix. #[cfg(feature = "1.8.17")] pub fn efile_prefix(&mut self, prefix: &str) -> &mut Self { self.efile_prefix = Some(prefix.into()); self } + /// Sets the [`VirtualView`] options. #[cfg(feature = "1.10.0")] pub fn virtual_view(&mut self, view: VirtualView) -> &mut Self { self.virtual_view = Some(view); self } + /// Sets the maximum number of files/datasets allowed to be missing when determining the extent + /// of an unlimited virtual dataset with printf-style mappings. #[cfg(feature = "1.10.0")] pub fn virtual_printf_gap(&mut self, gap_size: usize) -> &mut Self { self.virtual_printf_gap = Some(gap_size); self } + /// Sets metadata I/O mode for read options to collective or independent. #[cfg(all(feature = "1.10.0", feature = "have-parallel"))] pub fn all_coll_metadata_ops(&mut self, is_collective: bool) -> &mut Self { self.all_coll_metadata_ops = Some(is_collective); @@ -223,10 +232,12 @@ impl DatasetAccessBuilder { Ok(()) } + /// Copies the builder settings into a dataset access property list. pub fn apply(&self, plist: &mut DatasetAccess) -> Result<()> { h5lock!(self.populate_plist(plist.id())) } + /// Constructs a new dataset access property list. pub fn finish(&self) -> Result { h5lock!({ let mut plist = DatasetAccess::try_new()?; @@ -237,14 +248,17 @@ impl DatasetAccessBuilder { /// Dataset access property list. impl DatasetAccess { + /// Creates a new dataset access property list. pub fn try_new() -> Result { Self::from_id(h5try!(H5Pcreate(*H5P_DATASET_ACCESS))) } + /// Creates a copy of the property list. pub fn copy(&self) -> Self { unsafe { self.deref().copy().cast_unchecked() } } + /// Creates a new dataset access property list builder. pub fn build() -> DatasetAccessBuilder { DatasetAccessBuilder::new() } @@ -260,6 +274,7 @@ impl DatasetAccess { ) } + /// Returns the raw data chunk cache parameters. pub fn chunk_cache(&self) -> ChunkCache { self.get_chunk_cache().unwrap_or_else(|_| ChunkCache::default()) } @@ -270,6 +285,7 @@ impl DatasetAccess { h5lock!(get_h5_str(|m, s| H5Pget_efile_prefix(self.id(), m, s))) } + /// Returns the external dataset storage file prefix. #[cfg(feature = "1.8.17")] pub fn efile_prefix(&self) -> String { self.get_efile_prefix().ok().unwrap_or_default() @@ -281,6 +297,7 @@ impl DatasetAccess { h5get!(H5Pget_virtual_view(self.id()): H5D_vds_view_t).map(Into::into) } + /// Returns the virtual dataset view options. #[cfg(feature = "1.10.0")] pub fn virtual_view(&self) -> VirtualView { self.get_virtual_view().ok().unwrap_or_default() @@ -292,6 +309,8 @@ impl DatasetAccess { h5get!(H5Pget_virtual_printf_gap(self.id()): hsize_t).map(|x| x as _) } + /// Returns the maximum number of files/datasets allowed to be missing when determining the + /// extent of an unlimited virtual dataset with printf-style mappings. #[cfg(feature = "1.10.0")] pub fn virtual_printf_gap(&self) -> usize { self.get_virtual_printf_gap().unwrap_or(0) @@ -303,6 +322,7 @@ impl DatasetAccess { h5get!(H5Pget_all_coll_metadata_ops(self.id()): hbool_t).map(|x| x > 0) } + /// Returns `true` if metadata I/O reads are set to collective, or `false` if independent. #[cfg(all(feature = "1.10.0", feature = "have-parallel"))] pub fn all_coll_metadata_ops(&self) -> bool { self.get_all_coll_metadata_ops().unwrap_or(false) diff --git a/hdf5/src/hl/plist/dataset_create.rs b/hdf5/src/hl/plist/dataset_create.rs index 0661c87f5..161723c16 100644 --- a/hdf5/src/hl/plist/dataset_create.rs +++ b/hdf5/src/hl/plist/dataset_create.rs @@ -30,6 +30,7 @@ use hdf5_sys::{ use hdf5_types::{OwnedDynValue, TypeDescriptor}; use crate::dim::Dimension; + use crate::globals::H5P_DATASET_CREATE; use crate::hl::datatype::Datatype; use crate::hl::filters::{validate_filters, Filter, SZip, ScaleOffset}; @@ -107,11 +108,16 @@ impl Clone for DatasetCreate { } } +/// Options for how to store raw data for a dataset. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Layout { + /// Raw data is stored in the file's object header. Compact, + /// Raw data is stored in a contiguous chunk in the file, outside the object header. Contiguous, + /// Raw data is stored in separate chunks in the file. Chunked, + /// Raw data is drawn from multiple datasets in different files. #[cfg(feature = "1.10.0")] Virtual, } @@ -148,8 +154,10 @@ impl From for H5D_layout_t { #[cfg(feature = "1.10.0")] bitflags! { + /// Edge chunk option flags. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct ChunkOpts: u32 { + /// Disable applying filters to partial edge chunks. const DONT_FILTER_PARTIAL_CHUNKS = H5D_CHUNK_DONT_FILTER_PARTIAL_CHUNKS; } } @@ -161,10 +169,14 @@ impl Default for ChunkOpts { } } +/// Options for when to allocate dataset storage space. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum AllocTime { + /// Allocate all space when the dataset is created. Early, + /// Allocate space incrementally as data is written to the dataset. Incr, + /// Allocate all space when data is first written to the dataset. Late, } @@ -188,10 +200,15 @@ impl From for H5D_alloc_time_t { } } +/// Options for when to write fill values to a dataset. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum FillTime { + /// Write fill values to the dataset when storage is allocated only if a user-defined fill + /// value is set. IfSet, + /// Write fill values to the dataset when storage is allocated. Alloc, + /// Never write fill values to the dataset. Never, } @@ -221,10 +238,14 @@ impl From for H5D_fill_time_t { } } +/// The status of a dataset creation property list's fill value. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum FillValue { + /// Fill value is undefined. Undefined, + /// Fill value is the library default. Default, + /// Fill value is defined by the application. UserDefined, } @@ -254,26 +275,38 @@ impl From for H5D_fill_value_t { } } +/// Properties of data stored in an external file. #[derive(Clone, Debug, PartialEq, Eq)] pub struct ExternalFile { + /// The name of the file. pub name: String, + /// The offset in bytes from the start of the file to the location where the data starts. pub offset: usize, + /// The number of bytes reserved in the file for data. pub size: usize, } +/// Properties of a mapping between virtual and source datasets. #[cfg(feature = "1.10.0")] #[derive(Clone, Debug, PartialEq, Eq)] pub struct VirtualMapping { + /// The name of the HDF5 file containing the source dataset. pub src_filename: String, + /// The path to the source dataset inside the file. pub src_dataset: String, + /// The dimensionality of the source dataset. pub src_extents: Extents, + /// The selection of the source dataset to be mapped. pub src_selection: Selection, + /// The dimensionality of the virtual dataset. pub vds_extents: Extents, + /// The selection of the virtual dataset to be mapped. pub vds_selection: Selection, } #[cfg(feature = "1.10.0")] impl VirtualMapping { + /// Constructs a `VirtualMapping` with the given parameters. pub fn new( src_filename: F, src_dataset: D, src_extents: E1, src_selection: S1, vds_extents: E2, vds_selection: S2, @@ -365,41 +398,49 @@ impl DatasetCreateBuilder { Ok(builder) } + /// Sets the dataset filters from a slice of filter specifiers. pub fn set_filters(&mut self, filters: &[Filter]) -> &mut Self { self.filters = filters.to_owned(); self } + /// Adds a deflation filter with some compression level to the dataset. pub fn deflate(&mut self, level: u8) -> &mut Self { self.filters.push(Filter::deflate(level)); self } + /// Adds a shuffle filter to the dataset. pub fn shuffle(&mut self) -> &mut Self { self.filters.push(Filter::shuffle()); self } + /// Adds a Fletcher32 checksum filter to the dataset. pub fn fletcher32(&mut self) -> &mut Self { self.filters.push(Filter::fletcher32()); self } + /// Adds an Szip compression filter with some coding method and pixels per block to the dataset. pub fn szip(&mut self, coding: SZip, px_per_block: u8) -> &mut Self { self.filters.push(Filter::szip(coding, px_per_block)); self } + /// Adds an N-bit compression filter to the dataset. pub fn nbit(&mut self) -> &mut Self { self.filters.push(Filter::nbit()); self } + /// Adds a scale-offset compression filter with some scaling mode to the dataset. pub fn scale_offset(&mut self, mode: ScaleOffset) -> &mut Self { self.filters.push(Filter::scale_offset(mode)); self } + /// Adds an LZF compression filter to the dataset. #[cfg(feature = "lzf")] pub fn lzf(&mut self) -> &mut Self { self.filters.push(Filter::lzf()); @@ -421,6 +462,7 @@ impl DatasetCreateBuilder { self } + /// Adds a Blosc filter with LZ compression to the dataset. #[cfg(feature = "blosc")] pub fn blosc_blosclz(&mut self, clevel: u8, shuffle: T) -> &mut Self where @@ -430,6 +472,7 @@ impl DatasetCreateBuilder { self } + /// Adds a Blosc filter with LZ4 compression to the dataset. #[cfg(feature = "blosc-lz4")] pub fn blosc_lz4(&mut self, clevel: u8, shuffle: T) -> &mut Self where @@ -439,6 +482,7 @@ impl DatasetCreateBuilder { self } + /// Adds a Blosc filter with LZ4HC compression to the dataset. #[cfg(feature = "blosc-lz4")] pub fn blosc_lz4hc(&mut self, clevel: u8, shuffle: T) -> &mut Self where @@ -448,6 +492,7 @@ impl DatasetCreateBuilder { self } + /// Adds a Blosc filter with Snappy compression to the dataset. #[cfg(feature = "blosc-snappy")] pub fn blosc_snappy(&mut self, clevel: u8, shuffle: T) -> &mut Self where @@ -457,6 +502,7 @@ impl DatasetCreateBuilder { self } + /// Adds a Blosc filter with Zlib compression to the dataset. #[cfg(feature = "blosc-zlib")] pub fn blosc_zlib(&mut self, clevel: u8, shuffle: T) -> &mut Self where @@ -466,6 +512,7 @@ impl DatasetCreateBuilder { self } + /// Adds a Blosc filter with Zstd compression to the dataset. #[cfg(feature = "blosc-zstd")] pub fn blosc_zstd(&mut self, clevel: u8, shuffle: T) -> &mut Self where @@ -475,21 +522,53 @@ impl DatasetCreateBuilder { self } + #[cfg(feature = "zfp")] + pub fn zfp_accuracy( + &mut self, accuracy: f64, chunk_dims: Vec, n_bytes: u8, + ) -> &mut Self { + self.filters.push(Filter::zfp_accuracy(accuracy, chunk_dims, n_bytes)); + self + } + + #[cfg(feature = "zfp")] + pub fn zfp_rate(&mut self, rate: f64, chunk_dims: Vec, n_bytes: u8) -> &mut Self { + self.filters.push(Filter::zfp_rate(rate, chunk_dims, n_bytes)); + self + } + + #[cfg(feature = "zfp")] + pub fn zfp_precision( + &mut self, precision: u8, chunk_dims: Vec, n_bytes: u8, + ) -> &mut Self { + self.filters.push(Filter::zfp_precision(precision, chunk_dims, n_bytes)); + self + } + + #[cfg(feature = "zfp")] + pub fn zfp_reversible(&mut self, chunk_dims: Vec, n_bytes: u8) -> &mut Self { + self.filters.push(Filter::zfp_reversible(chunk_dims, n_bytes)); + self + } + + /// Adds a user-defined filter with the given identifier and parameters to the dataset. pub fn add_filter(&mut self, id: H5Z_filter_t, cdata: &[c_uint]) -> &mut Self { self.filters.push(Filter::user(id, cdata)); self } + /// Removes all filters from the dataset. pub fn clear_filters(&mut self) -> &mut Self { self.filters.clear(); self } + /// Sets the dataset's storage space allocation timing. pub fn alloc_time(&mut self, alloc_time: Option) -> &mut Self { self.alloc_time = Some(alloc_time); self } + /// Sets the time when fill values should be written to the dataset. pub fn fill_time(&mut self, fill_time: FillTime) -> &mut Self { self.fill_time = Some(fill_time); self @@ -499,11 +578,13 @@ impl DatasetCreateBuilder { self.fill_time.is_some() } + /// Sets the dataset's fill value. pub fn fill_value>(&mut self, fill_value: T) -> &mut Self { self.fill_value = Some(fill_value.into()); self } + /// Clears the dataset's fill value. pub fn no_fill_value(&mut self) -> &mut Self { self.fill_value = None; self @@ -521,27 +602,32 @@ impl DatasetCreateBuilder { self } + /// Clears the dataset's chunking settings. pub fn no_chunk(&mut self) -> &mut Self { self.chunk = None; self } + /// Sets the dataset's raw data layout. pub fn layout(&mut self, layout: Layout) -> &mut Self { self.layout = Some(layout); self } + /// Sets the dataset's edge chunk options. #[cfg(feature = "1.10.0")] pub fn chunk_opts(&mut self, opts: ChunkOpts) -> &mut Self { self.chunk_opts = Some(opts); self } + /// Adds an external file to the dataset. pub fn external(&mut self, name: &str, offset: usize, size: usize) -> &mut Self { self.external.push(ExternalFile { name: name.to_owned(), offset, size }); self } + /// Adds a mapping between virtual and source datasets. #[cfg(feature = "1.10.0")] pub fn virtual_map( &mut self, src_filename: F, src_dataset: D, src_extents: E1, src_selection: S1, @@ -566,16 +652,19 @@ impl DatasetCreateBuilder { self } + /// Sets whether to record time data for the dataset. pub fn obj_track_times(&mut self, track_times: bool) -> &mut Self { self.obj_track_times = Some(track_times); self } + /// Sets the dataset's attribute storage phase change thresholds. pub fn attr_phase_change(&mut self, max_compact: u32, min_dense: u32) -> &mut Self { self.attr_phase_change = Some(AttrPhaseChange { max_compact, min_dense }); self } + /// Sets whether to track and/or index the dataset's attribute creation order. pub fn attr_creation_order(&mut self, attr_creation_order: AttrCreationOrder) -> &mut Self { self.attr_creation_order = Some(attr_creation_order); self @@ -650,10 +739,12 @@ impl DatasetCreateBuilder { !self.filters.is_empty() } + /// Copies the builder settings into a dataset creation property list. pub fn apply(&self, plist: &mut DatasetCreate) -> Result<()> { h5lock!(self.populate_plist(plist.id())) } + /// Constructs a new dataset creation property list. pub fn finish(&self) -> Result { h5lock!({ let mut plist = DatasetCreate::try_new()?; @@ -664,18 +755,22 @@ impl DatasetCreateBuilder { /// Dataset creation property list. impl DatasetCreate { + /// Constructs a new dataset creation property list. pub fn try_new() -> Result { Self::from_id(h5try!(H5Pcreate(*H5P_DATASET_CREATE))) } + /// Returns a copy of the dataset creation property list. pub fn copy(&self) -> Self { unsafe { self.deref().copy().cast_unchecked() } } + /// Returns a builder for configuring a dataset creation property list. pub fn build() -> DatasetCreateBuilder { DatasetCreateBuilder::new() } + /// Returns `true` if all required filters are available. pub fn all_filters_avail(&self) -> bool { h5lock!(H5Pall_filters_avail(self.id())) > 0 } @@ -685,10 +780,12 @@ impl DatasetCreate { Filter::extract_pipeline(self.id()) } + /// Returns a vector of the dataset's filter configurations. pub fn filters(&self) -> Vec { self.get_filters().unwrap_or_default() } + /// Returns `true` if there is at least one filter configured. pub fn has_filters(&self) -> bool { !self.filters().is_empty() } @@ -698,6 +795,7 @@ impl DatasetCreate { h5get!(H5Pget_alloc_time(self.id()): H5D_alloc_time_t).map(Into::into) } + /// Returns the storage allocation timing settings. pub fn alloc_time(&self) -> AllocTime { self.get_alloc_time().unwrap_or(AllocTime::Late) } @@ -707,6 +805,7 @@ impl DatasetCreate { h5get!(H5Pget_fill_time(self.id()): H5D_fill_time_t).map(Into::into) } + /// Returns the fill value timing settings. pub fn fill_time(&self) -> FillTime { self.get_fill_time().unwrap_or_default() } @@ -716,6 +815,7 @@ impl DatasetCreate { h5get!(H5Pfill_value_defined(self.id()): H5D_fill_value_t).map(Into::into) } + /// Returns the fill value status. pub fn fill_value_defined(&self) -> FillValue { self.get_fill_value_defined().unwrap_or(FillValue::Undefined) } @@ -736,6 +836,7 @@ impl DatasetCreate { } } + /// Returns the fill value converted to a dynamic type, or `None` if not set. pub fn fill_value(&self, tp: &TypeDescriptor) -> Option { self.get_fill_value(tp).unwrap_or_default() } @@ -752,6 +853,7 @@ impl DatasetCreate { .transpose() } + /// Returns the fill value converted to a concrete type, or `None` if not set. pub fn fill_value_as(&self) -> Option { self.get_fill_value_as::().unwrap_or_default() } @@ -768,6 +870,7 @@ impl DatasetCreate { } } + /// Returns a vector of chunk dimensions for the dataset, or `None` if it is not chunked. pub fn chunk(&self) -> Option> { self.get_chunk().unwrap_or_default() } @@ -779,6 +882,7 @@ impl DatasetCreate { Ok(layout.into()) } + /// Returns the layout setting for the dataset's raw data. pub fn layout(&self) -> Layout { self.get_layout().unwrap_or_default() } @@ -794,6 +898,7 @@ impl DatasetCreate { } } + /// Returns the edge chunk option setting, or `None` if the dataset is not chunked. #[cfg(feature = "1.10.0")] pub fn chunk_opts(&self) -> Option { self.get_chunk_opts().unwrap_or_default() @@ -828,6 +933,7 @@ impl DatasetCreate { }) } + /// Returns a vector of external file specifiers for the dataset. pub fn external(&self) -> Vec { self.get_external().unwrap_or_default() } @@ -868,6 +974,7 @@ impl DatasetCreate { }) } + /// Returns a vector of virtual mapping specifiers for the dataset. #[cfg(feature = "1.10.0")] pub fn virtual_map(&self) -> Vec { self.get_virtual_map().unwrap_or_default() @@ -878,6 +985,7 @@ impl DatasetCreate { h5get!(H5Pget_obj_track_times(self.id()): hbool_t).map(|x| x > 0) } + /// Returns `true` if object time tracking is enabled for the dataset. pub fn obj_track_times(&self) -> bool { self.get_obj_track_times().unwrap_or(true) } @@ -888,6 +996,7 @@ impl DatasetCreate { .map(|(mc, md)| AttrPhaseChange { max_compact: mc as _, min_dense: md as _ }) } + /// Returns the attribute storage phase change thresholds. pub fn attr_phase_change(&self) -> AttrPhaseChange { self.get_attr_phase_change().unwrap_or_default() } @@ -898,6 +1007,7 @@ impl DatasetCreate { .map(AttrCreationOrder::from_bits_truncate) } + /// Returns flags for whether attribute creation order will be tracked/indexed. pub fn attr_creation_order(&self) -> AttrCreationOrder { self.get_attr_creation_order().unwrap_or_default() } diff --git a/hdf5/src/hl/plist/file_access.rs b/hdf5/src/hl/plist/file_access.rs index 3d76f3dda..a943f2e9e 100644 --- a/hdf5/src/hl/plist/file_access.rs +++ b/hdf5/src/hl/plist/file_access.rs @@ -164,10 +164,14 @@ impl Clone for FileAccess { } } +/// Core file driver properties. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub struct CoreDriver { + /// Size, in bytes, of memory increments. pub increment: usize, + /// Whether to write the file contents to disk when the file is closed. pub filebacked: bool, + /// Size, in bytes, of write aggregation pages. Setting to 1 enables tracking with no paging. #[cfg(feature = "1.8.13")] pub write_tracking: usize, } @@ -183,8 +187,10 @@ impl Default for CoreDriver { } } +/// Family file driver properties. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct FamilyDriver { + /// Size in bytes of each file member. pub member_size: usize, } @@ -195,32 +201,63 @@ impl Default for FamilyDriver { } bitflags! { + /// Flags specifying types of logging activity for the logging virtual file driver. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct LogFlags: u64 { + /// Track truncate operations. const TRUNCATE = H5FD_LOG_TRUNCATE; + /// Track "meta" operations (e.g. truncate). const META_IO = H5FD_LOG_META_IO; + /// Track the location of every read. const LOC_READ = H5FD_LOG_LOC_READ; + /// Track the location of every write. const LOC_WRITE = H5FD_LOG_LOC_WRITE; + /// Track the location of every seek. const LOC_SEEK = H5FD_LOG_LOC_SEEK; + /// Track all I/O locations and lengths. + /// Equivalent to `LOC_READ | LOC_WRITE | LOC_SEEK`. const LOC_IO = H5FD_LOG_LOC_IO; + /// Track the number of times each byte is read. const FILE_READ = H5FD_LOG_FILE_READ; + /// Track the number of times each byte is written. const FILE_WRITE = H5FD_LOG_FILE_WRITE; + /// Track the number of all types of I/O operations. + /// Equivalent to `FILE_READ | FILE_WRITE`. const FILE_IO = H5FD_LOG_FILE_IO; + /// Track the type of information stored at each byte. const FLAVOR = H5FD_LOG_FLAVOR; + /// Track the total number of read operations. const NUM_READ = H5FD_LOG_NUM_READ; + /// Track the total number of write operations. const NUM_WRITE = H5FD_LOG_NUM_WRITE; + /// Track the total number of seek operations. const NUM_SEEK = H5FD_LOG_NUM_SEEK; + /// Track the total number of truncate operations. const NUM_TRUNCATE = H5FD_LOG_NUM_TRUNCATE; + /// Track the total number of all types of I/O operations. + /// Equivalent to `NUM_READ | NUM_WRITE | NUM_SEEK | NUM_TRUNCATE`. const NUM_IO = H5FD_LOG_NUM_IO; + /// Track the time spent in open operations. const TIME_OPEN = H5FD_LOG_TIME_OPEN; + /// Track the time spent in stat operations. const TIME_STAT = H5FD_LOG_TIME_STAT; + /// Track the time spent in read operations. const TIME_READ = H5FD_LOG_TIME_READ; + /// Track the time spent in write operations. const TIME_WRITE = H5FD_LOG_TIME_WRITE; + /// Track the time spent in seek operations. const TIME_SEEK = H5FD_LOG_TIME_SEEK; + /// Track the time spent in truncate operations. const TIME_TRUNCATE = H5FD_LOG_TIME_TRUNCATE; + /// Track the time spent in close operations. const TIME_CLOSE = H5FD_LOG_TIME_CLOSE; + /// Track the time spent in each I/O operation. + /// Equivalent to `TIME_OPEN | TIME_STAT | TIME_READ | TIME_WRITE | TIME_SEEK + /// | TIME_TRUNCATE | TIME_CLOSE`. const TIME_IO = H5FD_LOG_TIME_IO; + /// Track releases of space in the file. const FREE = H5FD_LOG_FREE; + /// Track everything. const ALL = H5FD_LOG_ALL; } } @@ -231,6 +268,7 @@ impl Default for LogFlags { } } +/// Logging virtual file driver properties. #[derive(Clone, Default, Debug, PartialEq, Eq)] pub struct LogOptions { logfile: Option, @@ -248,25 +286,36 @@ static FD_MEM_TYPES: &[H5F_mem_t] = &[ H5F_mem_t::H5FD_MEM_OHDR, ]; +/// Properties for a data storage used by the multi-file driver. #[derive(Clone, Debug, PartialEq, Eq)] pub struct MultiFile { + /// Name of the member file. pub name: String, + /// Offset within virtual address space where the storage begins. pub addr: u64, } impl MultiFile { + /// Creates a new `MultiFile`. pub fn new(name: &str, addr: u64) -> Self { Self { name: name.into(), addr } } } +/// A mapping of memory usage types to storage indices. #[derive(Clone, Debug, PartialEq, Eq)] pub struct MultiLayout { + /// Index of the superblock. pub mem_super: u8, + /// Index of the B-tree data. pub mem_btree: u8, + /// Index of the raw data. pub mem_draw: u8, + /// Index of the global heap data. pub mem_gheap: u8, + /// Index of the local heap data. pub mem_lheap: u8, + /// Index of the object headers. pub mem_object: u8, } @@ -302,10 +351,14 @@ impl MultiLayout { } } +/// Multi-file driver properties. #[derive(Clone, Debug, PartialEq, Eq)] pub struct MultiDriver { + /// The names and offsets of each type of data storage. pub files: Vec, + /// The mapping of memory usage types to file indices. pub layout: MultiLayout, + /// Whether to allow read-only access to incomplete file sets. pub relax: bool, } @@ -345,9 +398,12 @@ impl MultiDriver { } } +/// Split file driver properties. #[derive(Clone, Debug, PartialEq, Eq)] pub struct SplitDriver { + /// Metadata filename extension. pub meta_ext: String, + /// Raw data filename extension. pub raw_ext: String, } @@ -396,9 +452,12 @@ mod mpio { use super::{c_int, Result}; + /// MPI-I/O file driver properties. #[derive(Debug)] pub struct MpioDriver { + /// MPI-2 communicator. pub comm: MPI_Comm, + /// MPI-2 info object. pub info: MPI_Info, } @@ -453,11 +512,15 @@ mod mpio { #[cfg(feature = "mpio")] pub use self::mpio::*; +/// Direct I/O driver properties. #[cfg(feature = "have-direct")] #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct DirectDriver { + /// Required memory alignment boundary. pub alignment: usize, + /// File system block size. pub block_size: usize, + /// Size in bytes of the copy buffer. pub cbuf_size: usize, } @@ -468,26 +531,43 @@ impl Default for DirectDriver { } } +/// A low-level file driver configuration. #[derive(Clone, Debug)] pub enum FileDriver { + /// Uses POSIX filesystem functions to perform unbuffered access to a single file. Sec2, + /// Uses functions from the standard C `stdio.h` to perform buffered access to a single file. Stdio, + /// SEC2 with logging capabilities. Log, + /// Keeps file contents in memory until the file is closed, enabling faster access. Core(CoreDriver), + /// Partitions file address space into pieces and sends them to separate storage files. Family(FamilyDriver), + /// Allows data to be stored in multiple files according to the type of data. Multi(MultiDriver), + /// Special case of the Multi driver that stores metadata and raw data in separate files. Split(SplitDriver), + /// Uses the MPI standard for communication and I/O. #[cfg(feature = "mpio")] Mpio(MpioDriver), + /// SEC2 except data is accessed synchronously without being cached by the system. #[cfg(feature = "have-direct")] Direct(DirectDriver), } +/// Options for what to do when trying to close a file while there are open objects inside it. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum FileCloseDegree { + /// Let the driver choose the behavior. + /// + /// All drivers set this to `Weak`, except for MPI-I/O, which sets it to `Semi`. Default, + /// Terminate file identifier access, but delay closing until all objects are closed. Weak, + /// Return an error if the file has open objects. Semi, + /// Close all open objects, then close the file. Strong, } @@ -519,22 +599,33 @@ impl From for H5F_close_degree_t { } } +/// File alignment properties. +/// +/// Any file object with size of at least `threshold` bytes will be aligned on an address that is +/// a multiple of `alignment`. Addresses are relative to the end of the user block. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Alignment { + /// The byte size threshold. pub threshold: u64, + /// The alignment value. pub alignment: u64, } impl Default for Alignment { + /// Returns the default threshold and alignment of 1 (i.e. no alignment). fn default() -> Self { Self { threshold: 1, alignment: 1 } } } +/// Raw data chunk cache parameters. #[derive(Clone, Copy, Debug, PartialEq)] pub struct ChunkCache { + /// The number of objects in the cache. pub nslots: usize, + /// The total size of the cache in bytes. pub nbytes: usize, + /// The chunk preemption policy. pub w0: f64, } @@ -546,16 +637,25 @@ impl Default for ChunkCache { impl Eq for ChunkCache {} +/// Page buffer size properties. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub struct PageBufferSize { + /// Maximum size, in bytes, of the page buffer. pub buf_size: usize, + /// Minimum metadata percentage to keep in the buffer before allowing pages with metadata to be + /// evicted. pub min_meta_perc: u32, + /// Minimum raw data percentage to keep in the buffer before allowing pages with raw data to be + /// evicted. pub min_raw_perc: u32, } +/// Automatic cache size increase mode. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum CacheIncreaseMode { + /// Automatic increase is disabled. Off, + /// Automatic increase uses the hit rate threshold algorithm. Threshold, } @@ -577,9 +677,12 @@ impl From for H5C_cache_incr_mode { } } +/// Flash cache size increase mode. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum FlashIncreaseMode { + /// Flash cache size increase is disabled. Off, + /// Flash cache size increase uses the add space algorithm. AddSpace, } @@ -601,11 +704,16 @@ impl From for H5C_cache_flash_incr_mode { } } +/// Automatic cache size decrease mode. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum CacheDecreaseMode { + /// Automatic decrease is disabled. Off, + /// Automatic decrease uses the hit rate threshold algorithm. Threshold, + /// Automatic decrease uses the ageout algorithm. AgeOut, + /// Automatic decrease uses the ageout with hit rate threshold algorithm. AgeOutWithThreshold, } @@ -631,9 +739,13 @@ impl From for H5C_cache_decr_mode { } } +/// A strategy for writing metadata to disk. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum MetadataWriteStrategy { + /// Only process zero is allowed to write dirty metadata to disk. ProcessZeroOnly, + /// Process zero decides what entries to flush, but the flushes are distributed across + /// processes. Distributed, } @@ -661,36 +773,76 @@ impl From for c_int { } } +/// Metadata cache configuration. #[derive(Clone, Debug, PartialEq)] pub struct MetadataCacheConfig { + /// Whether the adaptive cache resize report function is enabled. pub rpt_fcn_enabled: bool, + /// Whether `trace_file_name` should be used to open a trace file for the cache. pub open_trace_file: bool, + /// Whether the current trace file (if any) should be closed. pub close_trace_file: bool, + /// Full path of the trace file to be opened if `open_trace_file` is `true`. pub trace_file_name: String, + /// Whether evictions from the metadata cache are enabled. pub evictions_enabled: bool, + /// Whether the cache should be created with a user-specified initial size. pub set_initial_size: bool, + /// Initial cache size in bytes if `set_initial_size` is `true`. pub initial_size: usize, + /// Minimum fraction of the cache that must be kept clean or empty. pub min_clean_fraction: f64, + /// Maximum number of bytes that adaptive cache resizing can select as the maximum cache size. pub max_size: usize, + /// Minimum number of bytes that adaptive cache resizing can select as the minimum cache size. pub min_size: usize, + /// Number of cache accesses between runs of the adaptive cache resize code. pub epoch_length: i64, + /// Automatic cache size increase mode. pub incr_mode: CacheIncreaseMode, + /// Hit rate threshold for the hit rate threshold cache size increment algorithm. pub lower_hr_threshold: f64, + /// Factor by which the hit rate threshold cache size increment algorithm multiplies the current + /// cache max size to obtain a tentative new size. pub increment: f64, + /// Whether to apply an upper limit to the size of cache size increases. pub apply_max_increment: bool, + /// Maximum number of bytes by which cache size can be increased in a single step, + /// if applicable. pub max_increment: usize, + /// Flash cache size increase mode. pub flash_incr_mode: FlashIncreaseMode, + /// Factor by which the size of the triggering entry / entry size increase is multiplied to + /// obtain the initial cache size increment. pub flash_multiple: f64, + /// Factor by which the current maximum cache size is multiplied to obtain the minimum size + /// entry / entry size increase which may trigger a flash cache size increase. pub flash_threshold: f64, + /// Automatic cache size decrease mode. pub decr_mode: CacheDecreaseMode, + /// Hit rate threshold for hit-rate-based cache size decrease algorithms. pub upper_hr_threshold: f64, + /// Factor by which the hit rate threshold cache size decrease algorithm multiplies the current + /// cache max size to obtain a tentative new size. pub decrement: f64, + /// Whether an upper limit should be applied to the size of cache size decreases. pub apply_max_decrement: bool, + /// Maximum number of bytes by which cache size can be decreased in a single step, + /// if applicable. pub max_decrement: usize, + /// Minimum number of epochs that an entry must remain unaccessed in cache before ageout-based + /// reduction algorithms try to evict it. pub epochs_before_eviction: i32, + /// Whether ageout-based decrement algorithms will maintain an empty reserve. pub apply_empty_reserve: bool, + /// Empty reserve as a fraction of maximum cache size. + /// Ageout-based algorithms will not decrease the maximum size unless the empty reserve can be + /// met. pub empty_reserve: f64, + /// Threshold number of bytes of dirty metadata that will trigger synchronization of + /// parallel metadata caches. pub dirty_bytes_threshold: usize, + /// Strategy for writing metadata to disk. pub metadata_write_strategy: MetadataWriteStrategy, } @@ -816,10 +968,15 @@ impl From for MetadataCacheConfig { mod cache_image_config { use super::*; + /// Metadata cache image configuration. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub struct CacheImageConfig { + /// Whether a cache image should be created on file close. pub generate_image: bool, + /// Whether the cache image should include the adaptive cache resize configuration and + /// status. pub save_resize_status: bool, + /// Maximum number of times a prefetched entry can appear in subsequent cache images. pub entry_ageout: i32, } @@ -858,11 +1015,15 @@ mod cache_image_config { #[cfg(feature = "1.10.1")] pub use self::cache_image_config::*; +/// Metadata cache logging options. #[cfg(feature = "1.10.0")] #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct CacheLogOptions { + /// Whether logging is enabled. pub is_enabled: bool, + /// File path of the log. (Must be ASCII on Windows) pub location: String, + /// Whether to begin logging as soon as the file is opened pub start_on_access: bool, } @@ -870,18 +1031,24 @@ pub struct CacheLogOptions { mod libver { use super::*; + /// Options for which library format version to use when storing objects. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] pub enum LibraryVersion { + /// Use the earliest possible format. Earliest = 0, + /// Use the latest v18 format. V18 = 1, + /// Use the latest v110 format. V110 = 2, } impl LibraryVersion { + /// Returns `true` if the version is set to `Earliest`. pub fn is_earliest(self) -> bool { self == Self::Earliest } + /// Returns the latest library version. pub const fn latest() -> Self { Self::V110 } @@ -907,9 +1074,12 @@ mod libver { } } + /// Library format version bounds for writing objects to a file. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct LibVerBounds { + /// The earliest version to use for writing objects. pub low: LibraryVersion, + /// The latest version to use for writing objects. pub high: LibraryVersion, } @@ -1037,27 +1207,32 @@ impl FileAccessBuilder { self } + /// Sets the file alignment parameters. pub fn alignment(&mut self, threshold: u64, alignment: u64) -> &mut Self { self.alignment = Some(Alignment { threshold, alignment }); self } + /// Sets the raw data chunk cache parameters. pub fn chunk_cache(&mut self, nslots: usize, nbytes: usize, w0: f64) -> &mut Self { self.chunk_cache = Some(ChunkCache { nslots, nbytes, w0 }); self } + /// Sets the number of files that can be held open in an external link open file cache. #[cfg(feature = "1.8.7")] pub fn elink_file_cache_size(&mut self, efc_size: u32) -> &mut Self { self.elink_file_cache_size = Some(efc_size); self } + /// Sets the minimum metadata block size in bytes. pub fn meta_block_size(&mut self, size: u64) -> &mut Self { self.meta_block_size = Some(size); self } + /// Sets the page buffer size properties. #[cfg(feature = "1.10.1")] pub fn page_buffer_size( &mut self, buf_size: usize, min_meta_perc: u32, min_raw_perc: u32, @@ -1066,28 +1241,34 @@ impl FileAccessBuilder { self } + /// Sets the maximum size of the data sieve buffer. pub fn sieve_buf_size(&mut self, size: usize) -> &mut Self { self.sieve_buf_size = Some(size); self } + /// Sets whether object metadata should be evicted from cache when an object is closed. #[cfg(feature = "1.10.1")] pub fn evict_on_close(&mut self, evict_on_close: bool) -> &mut Self { self.evict_on_close = Some(evict_on_close); self } + /// Sets the number of reads that the library will try when reading checksummed metadata in a + /// file opened with SWMR access. #[cfg(feature = "1.10.0")] pub fn metadata_read_attempts(&mut self, attempts: u32) -> &mut Self { self.metadata_read_attempts = Some(attempts); self } + /// Sets the metadata cache configuration. pub fn mdc_config(&mut self, config: &MetadataCacheConfig) -> &mut Self { self.mdc_config = Some(config.clone()); self } + /// Sets whether a cache image should be created on file close. #[cfg(feature = "1.10.1")] pub fn mdc_image_config(&mut self, generate_image: bool) -> &mut Self { self.mdc_image_config = Some(CacheImageConfig { @@ -1098,6 +1279,7 @@ impl FileAccessBuilder { self } + /// Sets metadata cache logging options. #[cfg(feature = "1.10.0")] pub fn mdc_log_options( &mut self, is_enabled: bool, location: &str, start_on_access: bool, @@ -1107,67 +1289,80 @@ impl FileAccessBuilder { self } + /// Sets whether metadata reads are collective. #[cfg(all(feature = "1.10.0", feature = "have-parallel"))] pub fn all_coll_metadata_ops(&mut self, is_collective: bool) -> &mut Self { self.all_coll_metadata_ops = Some(is_collective); self } + /// Sets whether metadata writes are collective. #[cfg(all(feature = "1.10.0", feature = "have-parallel"))] pub fn coll_metadata_write(&mut self, is_collective: bool) -> &mut Self { self.coll_metadata_write = Some(is_collective); self } + /// Sets whether reference garbage collection is enabled. pub fn gc_references(&mut self, gc_ref: bool) -> &mut Self { self.gc_references = Some(gc_ref); self } + /// Sets the maximum size in bytes of a contiguous block reserved for small data. pub fn small_data_block_size(&mut self, size: u64) -> &mut Self { self.small_data_block_size = Some(size); self } + /// Sets the range of library versions to use when writing objects. #[cfg(feature = "1.10.2")] pub fn libver_bounds(&mut self, low: LibraryVersion, high: LibraryVersion) -> &mut Self { self.libver_bounds = Some(LibVerBounds { low, high }); self } + /// Allows use of the earliest library version when writing objects. #[cfg(feature = "1.10.2")] pub fn libver_earliest(&mut self) -> &mut Self { self.libver_bounds(LibraryVersion::Earliest, LibraryVersion::latest()) } + /// Sets the earliest library version for writing objects to v18. #[cfg(feature = "1.10.2")] pub fn libver_v18(&mut self) -> &mut Self { self.libver_bounds(LibraryVersion::V18, LibraryVersion::latest()) } + /// Sets the earliest library version for writing objects to v110. #[cfg(feature = "1.10.2")] pub fn libver_v110(&mut self) -> &mut Self { self.libver_bounds(LibraryVersion::V110, LibraryVersion::latest()) } + /// Allows only the latest library version when writing objects. #[cfg(feature = "1.10.2")] pub fn libver_latest(&mut self) -> &mut Self { self.libver_bounds(LibraryVersion::latest(), LibraryVersion::latest()) } + /// Sets which file driver to use. pub fn driver(&mut self, file_driver: &FileDriver) -> &mut Self { self.file_driver = Some(file_driver.clone()); self } + /// Sets the file driver to SEC2 (POSIX). pub fn sec2(&mut self) -> &mut Self { self.driver(&FileDriver::Sec2) } + /// Sets the file driver to STDIO. pub fn stdio(&mut self) -> &mut Self { self.driver(&FileDriver::Stdio) } + /// Sets the file driver to SEC2 with logging and configures it. pub fn log_options( &mut self, logfile: Option<&str>, flags: LogFlags, buf_size: usize, ) -> &mut Self { @@ -1177,38 +1372,46 @@ impl FileAccessBuilder { self.driver(&FileDriver::Log) } + /// Sets the file driver to SEC2 with logging. pub fn log(&mut self) -> &mut Self { self.log_options(None, LogFlags::LOC_IO, 0) } + /// Sets the file driver to Core and configures it. pub fn core_options(&mut self, increment: usize, filebacked: bool) -> &mut Self { let drv = CoreDriver { increment, filebacked, ..CoreDriver::default() }; self.driver(&FileDriver::Core(drv)) } + /// Sets the file driver to Core and sets whether to write file contents to disk upon closing. pub fn core_filebacked(&mut self, filebacked: bool) -> &mut Self { let drv = CoreDriver { filebacked, ..CoreDriver::default() }; self.driver(&FileDriver::Core(drv)) } + /// Sets the file driver to Core. pub fn core(&mut self) -> &mut Self { self.driver(&FileDriver::Core(CoreDriver::default())) } + /// Sets the write tracking page size for the Core file driver. #[cfg(feature = "1.8.13")] pub fn write_tracking(&mut self, page_size: usize) -> &mut Self { self.write_tracking = Some(page_size); self } + /// Sets the file driver to Family. pub fn family(&mut self) -> &mut Self { self.driver(&FileDriver::Family(FamilyDriver::default())) } + /// Sets the file driver to Family and configures the file member size. pub fn family_options(&mut self, member_size: usize) -> &mut Self { self.driver(&FileDriver::Family(FamilyDriver { member_size })) } + /// Sets the file driver to Multi and configures it. pub fn multi_options( &mut self, files: &[MultiFile], layout: &MultiLayout, relax: bool, ) -> &mut Self { @@ -1219,10 +1422,12 @@ impl FileAccessBuilder { })) } + /// Sets the file driver to Multi. pub fn multi(&mut self) -> &mut Self { self.driver(&FileDriver::Multi(MultiDriver::default())) } + /// Sets the file driver to Split and configures it. pub fn split_options(&mut self, meta_ext: &str, raw_ext: &str) -> &mut Self { self.driver(&FileDriver::Split(SplitDriver { meta_ext: meta_ext.into(), @@ -1230,16 +1435,19 @@ impl FileAccessBuilder { })) } + /// Sets the file driver to Split. pub fn split(&mut self) -> &mut Self { self.driver(&FileDriver::Split(SplitDriver::default())) } + /// Sets the file driver to MPI-I/O and configures it. #[cfg(feature = "mpio")] pub fn mpio(&mut self, comm: mpi_sys::MPI_Comm, info: Option) -> &mut Self { // We use .unwrap() here since MPI will almost surely terminate the process anyway. self.driver(&FileDriver::Mpio(MpioDriver::try_new(comm, info).unwrap())) } + /// Sets the file driver to Direct and configures it. #[cfg(feature = "have-direct")] pub fn direct_options( &mut self, alignment: usize, block_size: usize, cbuf_size: usize, @@ -1247,6 +1455,7 @@ impl FileAccessBuilder { self.driver(&FileDriver::Direct(DirectDriver { alignment, block_size, cbuf_size })) } + /// Sets the file driver to Direct. #[cfg(feature = "have-direct")] pub fn direct(&mut self) -> &mut Self { self.driver(&FileDriver::Direct(DirectDriver::default())) @@ -1487,10 +1696,12 @@ impl FileAccessBuilder { Ok(()) } + /// Copies the builder settings into a file access property list. pub fn apply(&self, plist: &mut FileAccess) -> Result<()> { h5lock!(self.populate_plist(plist.id())) } + /// Constructs a new file access property list. pub fn finish(&self) -> Result { h5lock!({ let mut plist = FileAccess::try_new()?; @@ -1501,14 +1712,17 @@ impl FileAccessBuilder { /// File access property list. impl FileAccess { + /// Creates a new file access property list. pub fn try_new() -> Result { Self::from_id(h5try!(H5Pcreate(*H5P_FILE_ACCESS))) } + /// Creates a copy of the property list. pub fn copy(&self) -> Self { unsafe { self.deref().copy().cast_unchecked() } } + /// Creates a new file access property list builder. pub fn build() -> FileAccessBuilder { FileAccessBuilder::new() } @@ -1640,6 +1854,7 @@ impl FileAccess { } } + /// Returns the file driver properties. pub fn driver(&self) -> FileDriver { self.get_driver().unwrap_or(FileDriver::Sec2) } @@ -1649,6 +1864,7 @@ impl FileAccess { h5get!(H5Pget_fclose_degree(self.id()): H5F_close_degree_t).map(Into::into) } + /// Returns the file close degree. pub fn fclose_degree(&self) -> FileCloseDegree { self.get_fclose_degree().unwrap_or_else(|_| FileCloseDegree::default()) } @@ -1660,6 +1876,7 @@ impl FileAccess { }) } + /// Returns the file alignment properties. pub fn alignment(&self) -> Alignment { self.get_alignment().unwrap_or_else(|_| Alignment::default()) } @@ -1675,6 +1892,7 @@ impl FileAccess { ) } + /// Returns the raw data chunk cache properties. pub fn chunk_cache(&self) -> ChunkCache { self.get_chunk_cache().unwrap_or_else(|_| ChunkCache::default()) } @@ -1695,6 +1913,7 @@ impl FileAccess { h5get!(H5Pget_meta_block_size(self.id()): hsize_t).map(|x| x as _) } + /// Returns the metadata block size. pub fn meta_block_size(&self) -> u64 { self.get_meta_block_size().unwrap_or(2048) } @@ -1711,6 +1930,7 @@ impl FileAccess { ) } + /// Returns the page buffer size properties. #[cfg(feature = "1.10.1")] pub fn page_buffer_size(&self) -> PageBufferSize { self.get_page_buffer_size().unwrap_or_else(|_| PageBufferSize::default()) @@ -1721,6 +1941,7 @@ impl FileAccess { h5get!(H5Pget_sieve_buf_size(self.id()): size_t).map(|x| x as _) } + /// Returns the maximum data sieve buffer size. pub fn sieve_buf_size(&self) -> usize { self.get_sieve_buf_size().unwrap_or(64 * 1024) } @@ -1731,6 +1952,8 @@ impl FileAccess { h5get!(H5Pget_evict_on_close(self.id()): hbool_t).map(|x| x > 0) } + /// Returns `true` if an object will be evicted from the metadata cache when the object is + /// closed. #[cfg(feature = "1.10.1")] pub fn evict_on_close(&self) -> bool { self.get_evict_on_close().unwrap_or(false) @@ -1742,6 +1965,7 @@ impl FileAccess { h5get!(H5Pget_metadata_read_attempts(self.id()): c_uint).map(|x| x as _) } + /// Returns the number of read attempts for SWMR access. #[cfg(feature = "1.10.0")] pub fn metadata_read_attempts(&self) -> u32 { self.get_metadata_read_attempts().unwrap_or(1) @@ -1754,6 +1978,7 @@ impl FileAccess { h5call!(H5Pget_mdc_config(self.id(), &mut config)).map(|_| config.into()) } + /// Returns the metadata cache configuration. pub fn mdc_config(&self) -> MetadataCacheConfig { self.get_mdc_config().ok().unwrap_or_default() } @@ -1766,6 +1991,7 @@ impl FileAccess { h5call!(H5Pget_mdc_image_config(self.id(), &mut config)).map(|_| config.into()) } + /// Returns the metadata cache image configuration. #[cfg(feature = "1.10.1")] pub fn mdc_image_config(&self) -> CacheImageConfig { self.get_mdc_image_config().ok().unwrap_or_default() @@ -1801,6 +2027,7 @@ impl FileAccess { }) } + /// Returns the metadata cache logging options. #[cfg(feature = "1.10.0")] pub fn mdc_log_options(&self) -> CacheLogOptions { self.get_mdc_log_options().ok().unwrap_or_default() @@ -1812,6 +2039,7 @@ impl FileAccess { h5get!(H5Pget_all_coll_metadata_ops(self.id()): hbool_t).map(|x| x > 0) } + /// Returns `true` if metadata reads are collective. #[cfg(all(feature = "1.10.0", feature = "have-parallel"))] pub fn all_coll_metadata_ops(&self) -> bool { self.get_all_coll_metadata_ops().unwrap_or(false) @@ -1823,6 +2051,7 @@ impl FileAccess { h5get!(H5Pget_coll_metadata_write(self.id()): hbool_t).map(|x| x > 0) } + /// Returns `true` if metadata writes are collective. #[cfg(all(feature = "1.10.0", feature = "have-parallel"))] pub fn coll_metadata_write(&self) -> bool { self.get_coll_metadata_write().unwrap_or(false) @@ -1833,6 +2062,7 @@ impl FileAccess { h5get!(H5Pget_gc_references(self.id()): c_uint).map(|x| x > 0) } + /// Returns `true` if reference garbage collection is enabled. pub fn gc_references(&self) -> bool { self.get_gc_references().unwrap_or(false) } @@ -1842,6 +2072,7 @@ impl FileAccess { h5get!(H5Pget_small_data_block_size(self.id()): hsize_t).map(|x| x as _) } + /// Returns the size setting in bytes of the small data block. pub fn small_data_block_size(&self) -> u64 { self.get_small_data_block_size().unwrap_or(2048) } @@ -1853,11 +2084,13 @@ impl FileAccess { .map(|(low, high)| LibVerBounds { low: low.into(), high: high.into() }) } + /// Returns the library format version bounds for writing objects to a file. #[cfg(feature = "1.10.2")] pub fn libver_bounds(&self) -> LibVerBounds { self.get_libver_bounds().ok().unwrap_or_default() } + /// Returns the lower library format version bound for writing objects to a file. #[cfg(feature = "1.10.2")] pub fn libver(&self) -> LibraryVersion { self.get_libver_bounds().ok().unwrap_or_default().low diff --git a/hdf5/src/hl/plist/file_create.rs b/hdf5/src/hl/plist/file_create.rs index d4e109bde..bc2e9c994 100644 --- a/hdf5/src/hl/plist/file_create.rs +++ b/hdf5/src/hl/plist/file_create.rs @@ -424,10 +424,12 @@ impl FileCreateBuilder { Ok(()) } + /// Copies the builder settings into a file creation property list. pub fn apply(&self, plist: &mut FileCreate) -> Result<()> { h5lock!(self.populate_plist(plist.id())) } + /// Constructs a new file creation property list. pub fn finish(&self) -> Result { h5lock!({ let mut plist = FileCreate::try_new()?; @@ -438,14 +440,17 @@ impl FileCreateBuilder { /// File creation property list. impl FileCreate { + /// Creates a new file creation property list. pub fn try_new() -> Result { Self::from_id(h5try!(H5Pcreate(*H5P_FILE_CREATE))) } + /// Creates a copy of the file creation property list. pub fn copy(&self) -> Self { unsafe { self.deref().copy().cast_unchecked() } } + /// Returns a builder for configuring a file creation property list. pub fn build() -> FileCreateBuilder { FileCreateBuilder::new() } diff --git a/hdf5/src/hl/plist/link_create.rs b/hdf5/src/hl/plist/link_create.rs index 2bb6c7c74..9f507b050 100644 --- a/hdf5/src/hl/plist/link_create.rs +++ b/hdf5/src/hl/plist/link_create.rs @@ -69,9 +69,12 @@ impl Clone for LinkCreate { } } +/// The character encoding used to create a link or attribute name. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum CharEncoding { + /// US ASCII. Ascii, + /// UTF-8. Utf8, } @@ -96,11 +99,13 @@ impl LinkCreateBuilder { Ok(builder) } + /// Sets whether to create intermediate groups upon creation of an object. pub fn create_intermediate_group(&mut self, create: bool) -> &mut Self { self.create_intermediate_group = Some(create); self } + /// Sets the character encoding to use when creating links. pub fn char_encoding(&mut self, encoding: CharEncoding) -> &mut Self { self.char_encoding = Some(encoding); self @@ -120,10 +125,12 @@ impl LinkCreateBuilder { Ok(()) } + /// Copies the builder settings into a link creation property list. pub fn apply(&self, plist: &mut LinkCreate) -> Result<()> { h5lock!(self.populate_plist(plist.id())) } + /// Constructs a new link creation property list. pub fn finish(&self) -> Result { h5lock!({ let mut plist = LinkCreate::try_new()?; @@ -134,14 +141,17 @@ impl LinkCreateBuilder { /// Link create property list. impl LinkCreate { + /// Creates a new link creation property list. pub fn try_new() -> Result { Self::from_id(h5try!(H5Pcreate(*H5P_LINK_CREATE))) } + /// Creates a copy of the link creation property list. pub fn copy(&self) -> Self { unsafe { self.deref().copy().cast_unchecked() } } + /// Returns a builder for configuring a link creation property list. pub fn build() -> LinkCreateBuilder { LinkCreateBuilder::new() } @@ -151,6 +161,7 @@ impl LinkCreate { h5get!(H5Pget_create_intermediate_group(self.id()): c_uint).map(|x| x > 0) } + /// Returns `true` if intermediate groups will be created upon object creation. pub fn create_intermediate_group(&self) -> bool { self.get_create_intermediate_group().unwrap_or(false) } @@ -164,6 +175,7 @@ impl LinkCreate { }) } + /// Returns the character encoding used to create links. pub fn char_encoding(&self) -> CharEncoding { self.get_char_encoding().unwrap_or(CharEncoding::Ascii) } diff --git a/hdf5/src/hl/plist/object_copy.rs b/hdf5/src/hl/plist/object_copy.rs new file mode 100644 index 000000000..6d09184f1 --- /dev/null +++ b/hdf5/src/hl/plist/object_copy.rs @@ -0,0 +1,212 @@ +//! Object copy properties. +use std::fmt::{self, Debug}; +use std::ops::Deref; + +use hdf5_sys::h5o::{ + H5O_COPY_EXPAND_EXT_LINK_FLAG, H5O_COPY_EXPAND_SOFT_LINK_FLAG, H5O_COPY_SHALLOW_HIERARCHY_FLAG, + H5O_COPY_WITHOUT_ATTR_FLAG, +}; +use hdf5_sys::h5p::{H5Pcreate, H5Pget_copy_object, H5Pset_copy_object}; + +use crate::globals::H5P_OBJECT_COPY; +use crate::internal_prelude::*; + +/// Object copy properties. +#[repr(transparent)] +pub struct ObjectCopy(Handle); + +impl ObjectClass for ObjectCopy { + const NAME: &'static str = "object copy property list"; + const VALID_TYPES: &'static [H5I_type_t] = &[H5I_GENPROP_LST]; + + fn from_handle(handle: Handle) -> Self { + Self(handle) + } + + fn handle(&self) -> &Handle { + &self.0 + } + + fn validate(&self) -> Result<()> { + ensure!( + self.is_class(PropertyListClass::ObjectCopy), + "expected object copy property list, got {:?}", + self.class() + ); + Ok(()) + } +} + +impl Debug for ObjectCopy { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = f.debug_struct("ObjectCopy"); + formatter.field("copy_without_attr", &self.copy_without_attr()); + formatter.field("shallow_hierarchy", &self.shallow_hierarchy()); + formatter.field("expand_soft_links", &self.expand_soft_links()); + formatter.field("expand_ext_links", &self.expand_ext_links()); + formatter.finish() + } +} + +impl Deref for ObjectCopy { + type Target = PropertyList; + + fn deref(&self) -> &PropertyList { + unsafe { self.transmute() } + } +} + +impl PartialEq for ObjectCopy { + fn eq(&self, other: &Self) -> bool { + ::eq(self, other) + } +} + +impl Eq for ObjectCopy {} + +impl Clone for ObjectCopy { + fn clone(&self) -> Self { + unsafe { self.deref().clone().cast_unchecked() } + } +} + +/// Builder used to create object copy property list. +#[derive(Clone, Debug, Default)] +pub struct ObjectCopyBuilder { + copy_without_attr: Option, + shallow_hierarchy: Option, + expand_soft_links: Option, + expand_ext_links: Option, +} + +impl ObjectCopyBuilder { + /// Creates a new object copy property list builder. + pub fn new() -> Self { + Self::default() + } + + /// Creates a new builder from an existing property list. + pub fn from_plist(plist: &ObjectCopy) -> Result { + let mut builder = Self::default(); + let flags = plist.get_flags()?; + builder.copy_without_attr = Some(flags & H5O_COPY_WITHOUT_ATTR_FLAG != 0); + builder.shallow_hierarchy = Some(flags & H5O_COPY_SHALLOW_HIERARCHY_FLAG != 0); + builder.expand_soft_links = Some(flags & H5O_COPY_EXPAND_SOFT_LINK_FLAG != 0); + builder.expand_ext_links = Some(flags & H5O_COPY_EXPAND_EXT_LINK_FLAG != 0); + Ok(builder) + } + + /// Copy object without copying attributes. + pub fn copy_without_attr(&mut self, enable: bool) -> &mut Self { + self.copy_without_attr = Some(enable); + self + } + + /// Copy only immediate members of a group (shallow copy). + pub fn shallow_hierarchy(&mut self, enable: bool) -> &mut Self { + self.shallow_hierarchy = Some(enable); + self + } + + /// Expand soft links into new objects. + pub fn expand_soft_links(&mut self, enable: bool) -> &mut Self { + self.expand_soft_links = Some(enable); + self + } + + /// Expand external links into new objects. + pub fn expand_ext_links(&mut self, enable: bool) -> &mut Self { + self.expand_ext_links = Some(enable); + self + } + + fn populate_plist(&self, id: hid_t) -> Result<()> { + let mut flags = 0u32; + + if self.copy_without_attr.unwrap_or(false) { + flags |= H5O_COPY_WITHOUT_ATTR_FLAG; + } + if self.shallow_hierarchy.unwrap_or(false) { + flags |= H5O_COPY_SHALLOW_HIERARCHY_FLAG; + } + if self.expand_soft_links.unwrap_or(false) { + flags |= H5O_COPY_EXPAND_SOFT_LINK_FLAG; + } + if self.expand_ext_links.unwrap_or(false) { + flags |= H5O_COPY_EXPAND_EXT_LINK_FLAG; + } + + if flags != 0 { + h5try!(H5Pset_copy_object(id, flags)); + } + + Ok(()) + } + + pub fn apply(&self, plist: &mut ObjectCopy) -> Result<()> { + h5lock!(self.populate_plist(plist.id())) + } + + pub fn finish(&self) -> Result { + h5lock!({ + let mut plist = ObjectCopy::try_new()?; + self.apply(&mut plist).map(|()| plist) + }) + } +} + +/// Object copy property list. +impl ObjectCopy { + pub fn try_new() -> Result { + Self::from_id(h5try!(H5Pcreate(*H5P_OBJECT_COPY))) + } + + pub fn copy(&self) -> Self { + unsafe { self.deref().copy().cast_unchecked() } + } + + pub fn build() -> ObjectCopyBuilder { + ObjectCopyBuilder::new() + } + + #[doc(hidden)] + pub fn get_flags(&self) -> Result { + h5get!(H5Pget_copy_object(self.id()): c_uint) + } + + pub fn copy_without_attr(&self) -> bool { + self.get_flags().map(|f| f & H5O_COPY_WITHOUT_ATTR_FLAG != 0).unwrap_or(false) + } + + pub fn shallow_hierarchy(&self) -> bool { + self.get_flags().map(|f| f & H5O_COPY_SHALLOW_HIERARCHY_FLAG != 0).unwrap_or(false) + } + + pub fn expand_soft_links(&self) -> bool { + self.get_flags().map(|f| f & H5O_COPY_EXPAND_SOFT_LINK_FLAG != 0).unwrap_or(false) + } + + pub fn expand_ext_links(&self) -> bool { + self.get_flags().map(|f| f & H5O_COPY_EXPAND_EXT_LINK_FLAG != 0).unwrap_or(false) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + pub fn test_object_copy_builder_from_plist() { + let ocpypl = + ObjectCopy::build().copy_without_attr(true).expand_soft_links(true).finish().unwrap(); + + let builder = ObjectCopyBuilder::from_plist(&ocpypl).unwrap(); + let ocpypl2 = builder.finish().unwrap(); + + assert_eq!(ocpypl.copy_without_attr(), ocpypl2.copy_without_attr()); + assert_eq!(ocpypl.shallow_hierarchy(), ocpypl2.shallow_hierarchy()); + assert_eq!(ocpypl.expand_soft_links(), ocpypl2.expand_soft_links()); + assert_eq!(ocpypl.expand_ext_links(), ocpypl2.expand_ext_links()); + assert_eq!(ocpypl, ocpypl2); + } +} diff --git a/hdf5/src/hl/selection.rs b/hdf5/src/hl/selection.rs index 3d7db831a..8b987e626 100644 --- a/hdf5/src/hl/selection.rs +++ b/hdf5/src/hl/selection.rs @@ -772,8 +772,11 @@ impl Display for Hyperslab { #[derive(Clone, Debug, PartialEq, Eq)] /// A selection used for reading and writing to a [`Container`](Container). pub enum Selection { + /// The entire dataset. All, + /// A sequence of points. Points(Array2), + /// A hyperslab or compound hyperslab. Hyperslab(Hyperslab), } @@ -784,10 +787,16 @@ impl Default for Selection { } impl Selection { + /// Creates a new selection. pub fn new>(selection: T) -> Self { selection.into() } + /// Tries to create a new selection. + /// + /// # Errors + /// + /// Returns an error if the conversion method fails. pub fn try_new>(selection: T) -> Result { selection.try_into() } @@ -829,6 +838,8 @@ impl Selection { }) } + /// Returns the required number of dimensions for the input dataspace, + /// or `None` if selecting everything. pub fn in_ndim(&self) -> Option { match self { Self::All => None, @@ -843,6 +854,7 @@ impl Selection { } } + /// Returns the number of dimensions in the output dataspace, or `None` if selecting everything. pub fn out_ndim(&self) -> Option { match self { Self::All => None, @@ -853,6 +865,11 @@ impl Selection { } } + /// Returns the output shape that would result from applying the selection to some input shape. + /// + /// # Errors + /// + /// Returns an error if the input shape is incompatible with the selection. pub fn out_shape>(&self, in_shape: S) -> Result> { let in_shape = in_shape.as_ref(); match self { @@ -875,10 +892,12 @@ impl Selection { } } + /// Returns `true` if the selection is for the entire dataset. pub fn is_all(&self) -> bool { self == &Self::All } + /// Returns `true` if the selection is a sequence of points. pub fn is_points(&self) -> bool { if let Self::Points(ref points) = self { points.shape() != [0, 0] @@ -887,6 +906,7 @@ impl Selection { } } + /// Returns `true` if the selection is empty. pub fn is_none(&self) -> bool { if let Self::Points(points) = self { points.shape() == [0, 0] @@ -895,6 +915,7 @@ impl Selection { } } + /// Returns `true` if the selection is a hyperslab. pub fn is_hyperslab(&self) -> bool { matches!(self, Self::Hyperslab(_)) } diff --git a/hdf5/src/lib.rs b/hdf5/src/lib.rs index 34d283eb1..9f5c6faa6 100644 --- a/hdf5/src/lib.rs +++ b/hdf5/src/lib.rs @@ -76,10 +76,12 @@ mod export { pub use hdf5_derive::H5Type; pub use hdf5_types::H5Type; + /// Base types and interfaces for creating compound data types. pub mod types { pub use hdf5_types::*; } + /// Multi-dimensional datasets. pub mod dataset { #[cfg(feature = "1.10.5")] pub use crate::hl::chunks::ChunkInfo; @@ -90,40 +92,51 @@ mod export { pub use crate::hl::plist::dataset_create::*; } + /// Datatype objects for defining the layout of a data element. pub mod datatype { pub use crate::hl::datatype::{ByteOrder, Conversion, Datatype}; } + /// HDF5 file objects. pub mod file { pub use crate::hl::file::{File, FileBuilder, OpenMode}; pub use crate::hl::plist::file_access::*; pub use crate::hl::plist::file_create::*; } + /// Property list objects. pub mod plist { pub use crate::hl::plist::dataset_access::{DatasetAccess, DatasetAccessBuilder}; pub use crate::hl::plist::dataset_create::{DatasetCreate, DatasetCreateBuilder}; pub use crate::hl::plist::file_access::{FileAccess, FileAccessBuilder}; pub use crate::hl::plist::file_create::{FileCreate, FileCreateBuilder}; pub use crate::hl::plist::link_create::{LinkCreate, LinkCreateBuilder}; + pub use crate::hl::plist::object_copy::{ObjectCopy, ObjectCopyBuilder}; pub use crate::hl::plist::{PropertyList, PropertyListClass}; + /// Dataset access property lists. pub mod dataset_access { pub use crate::hl::plist::dataset_access::*; } + /// Dataset creation property lists. pub mod dataset_create { pub use crate::hl::plist::dataset_create::*; } + /// File access property lists. pub mod file_access { pub use crate::hl::plist::file_access::*; } + /// File creation property lists. pub mod file_create { pub use crate::hl::plist::file_create::*; } + /// Link creation property lists. pub mod link_create { pub use crate::hl::plist::link_create::*; } } + + /// Filters for data compression and validation during file I/O. pub mod filters { pub use crate::hl::filters::*; } diff --git a/hdf5/src/sync.rs b/hdf5/src/sync.rs index 7783325d1..a2142d240 100644 --- a/hdf5/src/sync.rs +++ b/hdf5/src/sync.rs @@ -43,14 +43,12 @@ where #[cfg(test)] mod tests { - use lazy_static::lazy_static; use parking_lot::ReentrantMutex; + use std::sync::LazyLock; #[test] pub fn test_reentrant_mutex() { - lazy_static! { - static ref LOCK: ReentrantMutex<()> = ReentrantMutex::new(()); - } + static LOCK: LazyLock> = LazyLock::new(|| ReentrantMutex::new(())); let g1 = LOCK.try_lock(); assert!(g1.is_some()); let g2 = LOCK.lock(); diff --git a/hdf5/tests/common/gen.rs b/hdf5/tests/common/gen.rs index 296fdaf25..094deed8f 100644 --- a/hdf5/tests/common/gen.rs +++ b/hdf5/tests/common/gen.rs @@ -11,8 +11,8 @@ use ndarray::{ArrayD, SliceInfo, SliceInfoElem}; use num_complex::Complex; use rand::distr::StandardUniform; use rand::distr::{Alphanumeric, Uniform}; -use rand::prelude::Rng; use rand::prelude::{Distribution, IndexedRandom}; +use rand::prelude::{Rng, RngExt}; pub fn gen_shape(rng: &mut R, ndim: usize) -> Vec { iter::repeat(()).map(|_| rng.random_range(0..11)).take(ndim).collect() diff --git a/hdf5/tests/test_compressing_varlenunicode.rs b/hdf5/tests/test_compressing_varlenunicode.rs new file mode 100644 index 000000000..33d883d69 --- /dev/null +++ b/hdf5/tests/test_compressing_varlenunicode.rs @@ -0,0 +1,143 @@ +use std::str::FromStr; + +use hdf5::types::VarLenUnicode; +use hdf5_metno as hdf5; + +mod common; + +use self::common::util::new_in_memory_file; + +#[test] +#[cfg(feature = "blosc")] +fn blosc_blosclz() -> Result<(), Box> { + let file = new_in_memory_file()?; + + let dset_name = "x"; + let n_samples = 10; + let input_data = vec![VarLenUnicode::from_str("test").unwrap(); n_samples]; + + let ds_builder = file + .new_dataset::() + .shape((n_samples,)) + .chunk((n_samples,)) + .blosc_blosclz(5, true); + + let ds = ds_builder.create(dset_name)?; + ds.write(&input_data)?; + + let read: ndarray::Array1 = + file.dataset(dset_name)?.read::()?; + assert_eq!(read.to_vec(), input_data, "read data must match written data"); + Ok(()) +} + +#[test] +#[cfg(feature = "blosc-lz4")] +fn blosc_lz4() -> Result<(), Box> { + let file = new_in_memory_file()?; + + let dset_name = "x"; + let n_samples = 10; + let input_data = vec![VarLenUnicode::from_str("test")?; n_samples]; + let ds_builder = file + .new_dataset::() + .shape((n_samples,)) + .chunk((n_samples,)) + .blosc_lz4(5, true); + + let ds = ds_builder.create(dset_name)?; + ds.write(&input_data)?; + + let read: ndarray::Array1 = + file.dataset(dset_name)?.read::()?; + assert_eq!(read.to_vec(), input_data, "read data must match written data"); + Ok(()) +} + +#[test] +#[cfg(feature = "blosc-zlib")] +fn blosc_zlib() -> Result<(), Box> { + let file = new_in_memory_file()?; + + let dset_name = "x"; + let n_samples = 10; + let input_data = vec![VarLenUnicode::from_str("test")?; n_samples]; + let ds_builder = file + .new_dataset::() + .shape((n_samples,)) + .chunk((n_samples,)) + .blosc_zlib(5, true); + + let ds = ds_builder.create(dset_name)?; + ds.write(&input_data)?; + + let read: ndarray::Array1 = + file.dataset(dset_name)?.read::()?; + assert_eq!(read.to_vec(), input_data, "read data must match written data"); + Ok(()) +} + +#[test] +#[cfg(feature = "lzf")] +fn lzf() -> Result<(), Box> { + let file = new_in_memory_file()?; + + let dset_name = "x"; + let n_samples = 10; + let input_data = vec![VarLenUnicode::from_str("test")?; n_samples]; + let ds_builder = + file.new_dataset::().shape((n_samples,)).chunk((n_samples,)).lzf(); + + let ds = ds_builder.create(dset_name)?; + ds.write(&input_data)?; + + let read: ndarray::Array1 = + file.dataset(dset_name)?.read::()?; + assert_eq!(read.to_vec(), input_data, "read data must match written data"); + Ok(()) +} + +#[test] +fn szip() -> Result<(), Box> { + if !hdf5::filters::szip_available() { + return Ok(()); + } + let file = new_in_memory_file()?; + + let dset_name = "x"; + let n_samples = 10; + let input_data = vec![VarLenUnicode::from_str("test")?; n_samples]; + let ds_builder = file + .new_dataset::() + .shape((n_samples,)) + .chunk((n_samples,)) + .szip(hdf5::filters::SZip::Entropy, 8); + + let ds = ds_builder.create(dset_name)?; + ds.write(&input_data)?; + + let read: ndarray::Array1 = + file.dataset(dset_name)?.read::()?; + assert_eq!(read.to_vec(), input_data, "read data must match written data"); + Ok(()) +} + +#[test] +#[cfg(feature = "zlib")] +fn deflate() -> Result<(), Box> { + let file = new_in_memory_file()?; + + let dset_name = "x"; + let n_samples = 10; + let input_data = vec![VarLenUnicode::from_str("test")?; n_samples]; + let ds_builder = + file.new_dataset::().shape((n_samples,)).chunk((n_samples,)).deflate(5); + + let ds = ds_builder.create(dset_name)?; + ds.write(&input_data)?; + + let read: ndarray::Array1 = + file.dataset(dset_name)?.read::()?; + assert_eq!(read.to_vec(), input_data, "read data must match written data"); + Ok(()) +} diff --git a/hdf5/tests/test_dataset.rs b/hdf5/tests/test_dataset.rs index 46938d232..bd4b4485a 100644 --- a/hdf5/tests/test_dataset.rs +++ b/hdf5/tests/test_dataset.rs @@ -3,7 +3,7 @@ use std::fmt; use std::io::{Read, Seek, SeekFrom}; use ndarray::{s, Array1, Array2, ArrayD, IxDyn, SliceInfo}; -use rand::prelude::{Rng, SeedableRng, SmallRng}; +use rand::prelude::{Rng, RngExt, SeedableRng, SmallRng}; use hdf5_metno as hdf5; use hdf5_types::TypeDescriptor; diff --git a/hdf5/tests/test_datatypes.rs b/hdf5/tests/test_datatypes.rs index 14d86558d..2035c87bd 100644 --- a/hdf5/tests/test_datatypes.rs +++ b/hdf5/tests/test_datatypes.rs @@ -4,8 +4,8 @@ mod common; use hdf5::types::{TypeDescriptor as TD, *}; use hdf5::{from_id, Datatype, H5Type}; use hdf5_metno as hdf5; - use hdf5_sys::h5i::H5I_INVALID_HID; +use pretty_assertions::{assert_eq, assert_str_eq}; macro_rules! check_roundtrip { ($ty:ty, $desc:expr) => {{ @@ -148,6 +148,74 @@ pub fn test_eq() { } #[test] -pub fn test_debug() { - assert_eq!(format!("{:?}", Datatype::from_type::().unwrap()), ""); +fn test_print_display_debug_datatype_bool() { + let dt = Datatype::from_type::().unwrap(); + + assert_str_eq!(format!("{dt}"), "bool"); + assert_str_eq!(format!("{dt:?}"), ""); + assert_str_eq!(format!("{dt:#?}"), ""); +} + +#[test] +fn test_print_display_debug_datatype_f64() { + let dt = Datatype::from_type::().unwrap(); + + assert_str_eq!(format!("{dt}"), "float64"); + assert_str_eq!(format!("{dt:?}"), ""); + assert_str_eq!(format!("{dt:#?}"), ""); +} + +#[test] +fn test_print_display_debug_datatype_color_enum() { + #[allow(dead_code)] // "we use the type, we just don't construct it" + #[derive(H5Type)] + #[repr(u8)] + enum Color { + R = 1, + G = 2, + B = 3, + } + let dt = Datatype::from_type::().unwrap(); + + assert_eq!( + dt.to_descriptor().unwrap(), + TD::Enum(EnumType { + size: IntSize::U1, + signed: false, + members: vec![ + EnumMember { name: "R".into(), value: 1 }, + EnumMember { name: "G".into(), value: 2 }, + EnumMember { name: "B".into(), value: 3 } + ] + }) + ); + + assert_str_eq!(format!("{dt}"), "enum (uint8)"); + assert_str_eq!(format!("{dt:?}"), ""); + assert_str_eq!(format!("{dt:#?}"), ""); +} + +#[test] +fn test_print_display_debug_datatype_var_len_unicode() { + let dt = Datatype::from_type::().unwrap(); + assert!(dt.is::()); + + assert_eq!(dt.to_descriptor().unwrap(), TD::VarLenUnicode); + + assert_str_eq!(format!("{dt}"), "unicode (var len)"); + assert_str_eq!(format!("{dt:?}"), ""); + assert_str_eq!(format!("{dt:#?}"), ""); +} + +#[test] +fn test_print_display_debug_datatype_fixed_len_unicode() { + const SIZE: usize = 10; + let dt = Datatype::from_type::>().unwrap(); + assert!(dt.is::>()); + + assert_eq!(dt.to_descriptor().unwrap(), TD::FixedUnicode(SIZE)); + + assert_str_eq!(format!("{dt}"), "unicode (len 10)"); + assert_str_eq!(format!("{dt:?}"), ""); + assert_str_eq!(format!("{dt:#?}"), ""); } diff --git a/hdf5/tests/test_plist.rs b/hdf5/tests/test_plist.rs index e5c960703..07db1f9d8 100644 --- a/hdf5/tests/test_plist.rs +++ b/hdf5/tests/test_plist.rs @@ -329,6 +329,8 @@ fn test_fapl_driver_mpio() -> hdf5::Result<()> { let mut b = FileAccess::build(); b.mpio(world_comm, None); + let driver = b.finish()?.get_driver()?; + println!("{:?}", driver); let d = check_matches!(b.finish()?.get_driver()?, d, FileDriver::Mpio(d)); let mut cmp = mem::MaybeUninit::uninit(); unsafe { MPI_Comm_compare(d.comm, world_comm, cmp.as_mut_ptr()) };