diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 0000000..b8d4e92 --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,6 @@ +# move IOError out to file +c0d7b4cb67e84fa055ddf57a88aacd556d741cda +# start object file +580581badbb5491a549218f0dd9dcdc9a1544b25 +# remove is useful in multiple test files +e93cd0e9ef3682e7ef848698ebc9c8a370efc547 diff --git a/Project.toml b/Project.toml index 5db1f04..dc3cc47 100644 --- a/Project.toml +++ b/Project.toml @@ -11,8 +11,10 @@ YAML = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6" julia = "1" [extras] +EllipsisNotation = "da5c29d0-fa7d-589e-88eb-ea29b0a81949" HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f" +StructArrays = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["HDF5", "Test"] +test = ["EllipsisNotation", "HDF5", "StructArrays", "Test"] diff --git a/src/Exdir.jl b/src/Exdir.jl index 1a5c5a5..af5227b 100644 --- a/src/Exdir.jl +++ b/src/Exdir.jl @@ -1,5 +1,6 @@ module Exdir +import NPZ import YAML export @@ -8,18 +9,21 @@ export create_dataset, create_group, create_raw, - delete!, + delete_object, exdiropen, - IOError, is_nonraw_object_directory, + require_dataset, require_group, require_raw, setattrs! +include("can_cast.jl") include("consistency.jl") include("constants.jl") +include("exceptions.jl") include("mode.jl") include("path.jl") +include("object.jl") abstract type AbstractObject end abstract type AbstractGroup <: AbstractObject end @@ -36,8 +40,7 @@ struct Object <: AbstractObject name::String function Object(; root_directory, parent_path, object_name, file) - relative_path = joinpath(parent_path, object_name) - relative_path = if relative_path == "." "" else relative_path end + relative_path = form_relative_path(parent_path, object_name) name = "/" * relative_path new( root_directory, @@ -127,7 +130,7 @@ function Base.:(==)(obj::AbstractObject, other) # if obj.file.io_mode == OpenMode::FILE_CLOSED # return false # end - if !isa(obj, AbstractObject) + if !isa(other, AbstractObject) false else obj.relative_path == other.relative_path && @@ -159,8 +162,7 @@ struct Raw <: AbstractObject name::String function Raw(; root_directory, parent_path, object_name, file=nothing) - relative_path = joinpath(parent_path, object_name) - relative_path = if relative_path == "." "" else relative_path end + relative_path = form_relative_path(parent_path, object_name) name = "/" * relative_path new( root_directory, @@ -182,8 +184,7 @@ struct Dataset <: AbstractObject name::String function Dataset(; root_directory, parent_path, object_name, file) - relative_path = joinpath(parent_path, object_name) - relative_path = if relative_path == "." "" else relative_path end + relative_path = form_relative_path(parent_path, object_name) name = "/" * relative_path new( root_directory, @@ -196,14 +197,102 @@ struct Dataset <: AbstractObject end end -function Base.iterate(dset::Dataset) - () +function Base.convert(::Type{Object}, dset::Dataset) + Object(; + root_directory = dset.root_directory, + parent_path = dset.parent_path, + object_name = dset.object_name, + file = dset.file, + ) end -Base.length(dset::Dataset) = prod(size(dset)) +_directory(root_directory, relative_path) = joinpath(root_directory, relative_path) +_directory(dset::Dataset) = _directory(dset.root_directory, dset.relative_path) +_dataset_filename(dset_directory::AbstractString) = joinpath(dset_directory, DSET_FILENAME) + +function Base.getproperty(dset::Dataset, sym::Symbol) + if sym == :dtype + return eltype(dset) + elseif sym == :data + return NPZ.npzread(dset.data_filename) + elseif sym == :data_filename + return _dataset_filename(_directory(dset)) + # TODO + elseif sym == :directory + return _directory(dset) + elseif sym == :attrs + return Attribute() + elseif sym == :meta + return Attribute() + elseif sym == :attributes_filename + return joinpath(dset.directory, ATTRIBUTES_FILENAME) + elseif sym == :meta_filename + return joinpath(dset.directory, META_FILENAME) + elseif sym == :parent + if length(splitpath(dset.parent_path)) < 1 + return nothing + end + (parent_parent_path, parent_name) = splitdir(dset.parent_path) + return Group( + root_directory = dset.root_directory, + parent_path = parent_parent_path, + dsetect_name = parent_name, + file = dset.file, + ) + else + return getfield(dset, sym) + end + # else + # return getproperty(convert(Object, dset), sym) + # end +end + +function Base.setproperty!(dset::Dataset, name::Symbol, value) + if name == :data + assert_file_open(dset.file) + (data, _, _) = _prepare_write(value, Dict(), Dict()) + NPZ.npzwrite(dset.data_filename, data) + else + error("Cannot set property $(name) on Datasets") + end +end -function Base.size(dset::Dataset) - () +Base.collect(dset::Dataset) = collect(dset.data) +Base.iterate(dset::Dataset) = iterate(dset.data) +Base.iterate(dset::Dataset, state) = iterate(dset.data, state) +Base.length(dset::Dataset) = prod(size(dset)) +Base.size(dset::Dataset) = size(dset.data) +Base.getindex(dset::Dataset, inds...) = getindex(dset.data, inds...) +# TODO +# Base.setindex!(dset::Dataset, val, inds...) = setindex!(dset.data, val, inds...) +function Base.setindex!(dset::Dataset, val, inds...) + tmp = dset.data + setindex!(tmp, val, inds...) + dset.data = tmp +end +Base.eltype(dset::Dataset) = eltype(dset.data) +Base.firstindex(dset::Dataset) = firstindex(dset.data) +Base.lastindex(dset::Dataset) = lastindex(dset.data) + +# TODO this may fail in a gross way if value is a group/file/other Exdir type. +# Can we restrict to scalars and arrays? +function Base.setindex!(grp::AbstractGroup, value, name::AbstractString) + assert_file_open(grp.file) + path = name_to_asserted_group_path(name) + parts = splitpath(path) + if length(parts) > 1 + grp[dirname(path)][basename(path)] = value + return nothing + end + if !in(name, grp) + create_dataset(grp, name; data=value) + return nothing + end + if !isa(grp[name], Dataset) + error("Unable to assign value, $(name) already exists") + end + grp[name].value = value + nothing end struct Group <: AbstractGroup @@ -215,8 +304,7 @@ struct Group <: AbstractGroup name::String function Group(; root_directory, parent_path, object_name, file=nothing) - relative_path = joinpath(parent_path, object_name) - relative_path = if relative_path == "." "" else relative_path end + relative_path = form_relative_path(parent_path, object_name) name = "/" * relative_path new( root_directory, @@ -244,9 +332,31 @@ function Base.in(name::AbstractString, grp::AbstractGroup) end end -# function Base.get(grp::AbstractGroup, name::AbstractString) -# nothing -# end +function Base.get(grp::AbstractGroup, name::AbstractString, default=nothing) + if name in grp + grp[name] + else + default + end +end + +function unsafe_dataset(grp::AbstractGroup, name) + Dataset( + root_directory = grp.root_directory, + parent_path = grp.relative_path, + object_name = name, + file = grp.file, + ) +end + +function unsafe_group(grp::AbstractGroup, name) + Group( + root_directory = grp.root_directory, + parent_path = grp.relative_path, + object_name = name, + file = grp.file + ) +end function Base.getindex(grp::AbstractGroup, name::AbstractString) assert_file_open(grp.file) @@ -283,32 +393,110 @@ function Base.getindex(grp::AbstractGroup, name::AbstractString) meta_data = YAML.load_file(meta_filename) typename = meta_data[EXDIR_METANAME][TYPE_METANAME] if typename == DATASET_TYPENAME - return _dataset(grp, name) + return unsafe_dataset(grp, name) elseif typename == GROUP_TYPENAME - return _group(grp, name) + return unsafe_group(grp, name) else error_string = "Object $name has data type $typename.\nWe cannot open objects of this type." throw(ArgumentError(error_string)) end end -function Base.iterate(grp::AbstractGroup) - () -end +struct GroupIteratorState + "Keep track of the base Group originally passed in" + base_grp + "Unused" + root + "Unused" + current_base + "The current object (group, dset) name we are looking at" + current_obj_name + "Result of collect(walkdir(grp.root_directory))" + itr + "Current index into itr" + index + "Fully-typed object" + obj +end + +# This is work on fully recursive iteration. + +# # Iterate over all the objects in the group. +# function Base.iterate(grp::AbstractGroup) +# itr = collect(walkdir(grp.root_directory)) +# # "This" directory (the passed-in group) will always be the first result +# # and we want to ignore it. +# if length(itr) < 2 +# return nothing +# end +# index = 2 +# (root, dirs, files) = itr[index] +# @assert startswith(root, grp.root_directory) +# @assert files == [META_FILENAME] +# len_prefix = length(grp.root_directory) +# current_obj_name = root[len_prefix + 1 : end] +# state = GroupIteratorState( +# grp, +# root, +# root, +# current_obj_name, +# itr, +# index + 1, +# getindex(grp, current_obj_name) +# ) +# item = state.obj.name +# (item, state) +# end -function Base.length(grp::AbstractGroup) - 0 -end +# # Iterate over all the objects in the group. +# function Base.iterate(grp::AbstractGroup, state) +# # assert_file_open(grp.file) +# if state.index <= length(state.itr) +# (root, dirs, files) = state.itr[state.index] +# @assert startswith(root, grp.root_directory) +# @assert files == [META_FILENAME] +# len_prefix = length(grp.root_directory) +# current_obj_name = root[len_prefix + 1 : end] +# new_state = GroupIteratorState( +# state.base_grp, +# state.root, +# state.root, +# current_obj_name, +# state.itr, +# state.index + 1, +# getindex(state.base_grp, current_obj_name) +# ) +# item = new_state.obj.name +# (item, new_state) +# else +# nothing +# end +# end -function delete!(grp::AbstractGroup, name::AbstractString) - nothing +function Base.iterate(grp::AbstractGroup, dirs=nothing) + if isnothing(dirs) + grp_root = joinpath(grp.root_directory, grp.relative_path) + itr = walkdir(grp_root) + (root, dirs, files) = first(itr) + @assert root == grp_root + @assert files == [META_FILENAME] + end + isempty(dirs) ? nothing : (dirs[1], dirs[2:end]) end -struct IOError <: Exception - msg::String -end +Base.length(grp::AbstractGroup) = length(first(walkdir(joinpath(grp.root_directory, grp.relative_path)))[2]) -Base.showerror(io::IO, e::IOError) = print(io, "IOError: $(e.msg)") +function Base.delete!(grp::AbstractGroup, name::AbstractString) + if !in(name, grp) + # throw(KeyError("No such object '$(name)' in path '$(grp.name)'")) + throw(KeyError(name)) + end + @assert !isabspath(name) + path = joinpath(grp.root_directory, grp.relative_path, name) + @assert isdir(path) + rm(path, recursive=true) +end +delete_object(grp::AbstractGroup, name::AbstractString) = delete!(grp, name) struct File <: AbstractGroup root_directory::String @@ -321,8 +509,7 @@ struct File <: AbstractGroup user_mode::String function File(; root_directory, parent_path, object_name, file, user_mode) - relative_path = joinpath(parent_path, object_name) - relative_path = if relative_path == "." "" else relative_path end + relative_path = form_relative_path(parent_path, object_name) name = "/" * relative_path new( root_directory, @@ -353,13 +540,8 @@ function Base.getindex(file::File, name::AbstractString) end end -# function Base.in(name::AbstractString, file::File) -# false -# end +Base.in(name::AbstractString, file::File) = in(remove_root(name), convert(Group, file)) -# MethodError: Cannot `convert` an object of type Exdir.File to an object of type Exdir.Group -# Closest candidates are: -# convert(::Type{T}, ::T) where T function Base.convert(::Type{Group}, file::File) Group(; root_directory = file.root_directory, @@ -369,23 +551,11 @@ function Base.convert(::Type{Group}, file::File) ) end -function Base.iterate(::File) - ("hello", "world") -end - -function Base.iterate(::File, ::String) - nothing -end - function Base.print(io::IO, file::File) msg = "" print(io, msg) end -function delete!(file::File, name::AbstractString) - nothing -end - const EXTENSION = ".exdir" """ @@ -483,10 +653,9 @@ function Base.close(file::File) nothing end -function Base.keys(grp::AbstractGroup) -end - +Base.keys(grp::AbstractGroup) = collect(grp) Base.haskey(grp::AbstractGroup, name::AbstractString) = in(name, grp) +Base.values(grp::AbstractGroup) = [getindex(grp, key) for key in keys(grp)] function Base.setindex!(attrs::Attribute, value, name::AbstractString) end @@ -496,7 +665,7 @@ function defaultmetadata(typename::String) end makemetadata(typename::String) = YAML.write(defaultmetadata(typename)) -makemetadata(_::Object) = error("makemetadata not implemented for Exdir.Object") +makemetadata(_::Object) = throw(NotImplementedError("makemetadata not implemented for Exdir.Object")) makemetadata(_::Dataset) = makemetadata(DATASET_TYPENAME) makemetadata(_::Group) = makemetadata(GROUP_TYPENAME) makemetadata(_::File) = makemetadata(FILE_TYPENAME) @@ -555,41 +724,17 @@ end """ -function create_group(file::File, name::AbstractString) - path = remove_root(name) - _create_group(file, name) -end +create_group(file::File, name::AbstractString) = _create_group(file, remove_root(name)) """ create_group(grp, name) """ -function create_group(grp::Group, name::AbstractString) - _create_group(grp, name) -end - -# """ -# require_group(file, name) - - -# """ -# function require_group(file::File, name::AbstractString) -# path = remove_root(name) -# Group( -# root_directory =, -# parent_path = , -# object_name = , -# file =, -# ) -# end - -""" - require_group(grp, name) +create_group(grp::Group, name::AbstractString) = _create_group(grp, name) - -""" -function require_group(grp::Group, name::AbstractString) +function _require_group(grp, name::AbstractString) + # assert_file_open path = name_to_asserted_group_path(name) if length(splitpath(path)) > 1 @@ -614,6 +759,21 @@ function require_group(grp::Group, name::AbstractString) create_group(grp, name) end +""" + require_group(file, name) + + +""" +require_group(file::File, name::AbstractString) = _require_group(file, remove_root(name)) + +""" + require_group(grp, name) + + +""" +require_group(grp::Group, name::AbstractString) = _require_group(grp, name) + + function Base.write(dset::Dataset, data) error("unimplemented") end @@ -632,7 +792,9 @@ end function create_dataset(grp::AbstractGroup, name::AbstractString; shape=nothing, dtype=nothing, - data=nothing) + exact::Bool=false, + data=nothing, + fillvalue=nothing) # https://github.com/CINPLA/exdir/blob/89c1d34a5ce65fefc09b6fe1c5e8fef68c494e75/exdir/core/group.py#L72 path = name_to_asserted_group_path(name) @@ -640,15 +802,17 @@ function create_dataset(grp::AbstractGroup, name::AbstractString; (parent, pname) = splitdir(path) subgroup = require_group(grp, parent) return create_dataset(subgroup, pname, - shape=shape, dtype=dtype, data=data) + shape=shape, dtype=dtype, data=data, fillvalue=fillvalue) end _assert_valid_name(name, grp) if isnothing(data) && isnothing(shape) - error("Cannot create dataset. Missing shape or data keyword.") + throw(ArgumentError("Cannot create dataset. Missing shape or data keyword.")) end + _assert_allowed_fillvalue(fillvalue) + (prepared_data, attrs, meta) = _prepare_write( data, Dict(), @@ -667,9 +831,10 @@ function create_dataset(grp::AbstractGroup, name::AbstractString; if isnothing(shape) prepared_data = nothing else - # TODO fillvalue as kwarg - fillvalue = 0.0 - prepared_data = fill(dtype(fillvalue), shape) + if isnothing(fillvalue) + fillvalue = 0.0 + end + prepared_data = fill(convert(dtype, fillvalue), shape) end end @@ -679,75 +844,87 @@ function create_dataset(grp::AbstractGroup, name::AbstractString; create_object_directory(joinpath(grp.directory, name), meta) - dataset = Dataset( + # TODO pass attrs, meta + dset = Dataset( root_directory = grp.root_directory, parent_path = grp.relative_path, object_name = name, file = grp.file ) - # dataset._reset_data(prepared_data, attrs, None) # meta already set above - dataset + dset.data = prepared_data + return dset end -# function create_dataset(grp::AbstractGroup, name::AbstractString; -# shape::Dims, -# dtype::DataType) -# create_dataset(grp, name; shape=shape, dtype=dtype, data=nothing) -# end +""" -# function create_dataset(grp::AbstractGroup, name::AbstractString; -# data) -# create_dataset(grp, name; shape=size(data), dtype=eltype(data), data=data) -# end +Open an existing dataset or create it if it does not exist. +""" +function require_dataset(grp::AbstractGroup, name::AbstractString; + shape=nothing, + dtype=nothing, + exact::Bool=false, + data=nothing, + fillvalue=nothing) + assert_file_open(grp.file) + if !in(name, grp) + return create_dataset(grp, name, + shape=shape, dtype=dtype, exact=exact, data=data, fillvalue=fillvalue) + end -# function create_dataset(grp::AbstractGroup, name::AbstractString; -# shape::Dims) -# create_dataset(grp, name; shape=shape, dtype=Float64, data=nothing) -# end + current_object = grp[name] -function root_directory(path::AbstractString) - # https://github.com/CINPLA/exdir/blob/89c1d34a5ce65fefc09b6fe1c5e8fef68c494e75/exdir/core/exdir_object.py#L128 - path = realpath(path) - found = false - while !found - (parent, pname) = splitdir(path) - if parent == path - return nothing - end - if !is_nonraw_object_directory(path) - path = parent - continue - end - meta_data = YAML.load_file(joinpath(path, META_FILENAME)) - if !haskey(meta_data, EXDIR_METANAME) - path = parent - continue - end - exdir_meta = meta_data[EXDIR_METANAME] - if !haskey(exdir_meta, TYPE_METANAME) - path = parent - continue - end - if FILE_TYPENAME != exdir_meta[TYPE_METANAME] - path = parent - continue - end - found = true + if !isa(current_object, Dataset) + throw( + TypeError( + :require_dataset, + "Incompatible object already exists", + Dataset, + typeof(current_object) + ) + ) end - path -end -function is_inside_exdir(path::AbstractString) - # https://github.com/CINPLA/exdir/blob/89c1d34a5ce65fefc09b6fe1c5e8fef68c494e75/exdir/core/exdir_object.py#L161 - path = realpath(path) - !isnothing(root_directory(path)) -end + (data, attrs, meta) = _prepare_write(data, Dict(), Dict()) + + # TODO verify proper attributes + + _assert_data_shape_dtype_match(data, shape, dtype) + (shape, dtype) = _data_to_shape_and_dtype(data, shape, dtype) + + shape_exist = size(current_object) + if shape != shape_exist + throw( + DimensionMismatch( + "Shapes do not match: existing $(shape_exist) vs. new $(shape)" + ) + ) + end -function assert_inside_exdir(path::AbstractString) - # https://github.com/CINPLA/exdir/blob/89c1d34a5ce65fefc09b6fe1c5e8fef68c494e75/exdir/core/exdir_object.py#L166 - if !is_inside_exdir(path) - error("Path " + path + " is not inside an Exdir repository.") + dtype_exist = eltype(current_object) + if dtype != dtype_exist + if exact + throw( + TypeError( + :require_dataset, + "Datatypes do not exactly match", + dtype_exist, + dtype + ) + ) + end + if !can_cast(dtype, dtype_exist) + throw( + TypeError( + :require_dataset, + "Cannot safely cast", + dtype_exist, + dtype + ) + ) + end end + + current_object end function open_object(directory::AbstractString) @@ -813,27 +990,4 @@ function _assert_valid_name(name::AbstractString, container) # container.file.name_validation(container.directory, name) end -function _dataset(grp::AbstractGroup, name::AbstractString) - Dataset( - root_directory = grp.root_directory, - parent_path = grp.relative_path, - object_name = name, - file = grp.file, - ) -end - -function _assert_data_shape_dtype_match(data, shape::Dims, dtype) - if !isnothing(data) - sz = size(data) - if prod(sz) != prod(shape) - error("Provided shape and size(data) do not match: $shape vs $sz") - end - et = eltype(data) - if et != dtype - error("Provided dtype and eltype(data) do not match: $dtype vs $et") - end - end - nothing -end - end diff --git a/src/can_cast.jl b/src/can_cast.jl new file mode 100644 index 0000000..cc78bca --- /dev/null +++ b/src/can_cast.jl @@ -0,0 +1,56 @@ +# Returns True if cast between data types can occur according to the +# casting rule. If from is a scalar or array scalar, also returns +# True if the scalar value can be cast without overflow or truncation +# to an integer. +# +# Parameters +# ---------- +# from_ : dtype, dtype specifier, scalar, or array +# Data type, scalar, or array to cast from. +# to : dtype or dtype specifier +# Data type to cast to. +# casting : {'no', 'equiv', 'safe', 'same_kind', 'unsafe'}, optional +# Controls what kind of data casting may occur. +# +# * 'no' means the data types should not be cast at all. +# * 'equiv' means only byte-order changes are allowed. +# * 'safe' means only casts which can preserve values are allowed. +# * 'same_kind' means only safe casts or casts within a kind, +# like float64 to float32, are allowed. +# * 'unsafe' means any data conversions may be done. +# +# Returns +# ------- +# out : bool +# True if cast can occur according to the casting rule. +# +# Notes +# ----- +# .. versionchanged:: 1.17.0 +# Casting between a simple data type and a structured one is possible only +# for "unsafe" casting. Casting to multiple fields is allowed, but +# casting from multiple fields is not. +# +# .. versionchanged:: 1.9.0 +# Casting from numeric to string types in 'safe' casting mode requires +# that the string dtype length is long enough to store the maximum +# integer/float value converted. +# +# See also +# -------- +# dtype, result_type +function can_cast(dtype_from::DataType, dtype_to::DataType, casting="safe") + if casting != "safe" + throw( + ArgumentError( + "can't handle anything other than safe casting for now" + ) + ) + end + if dtype_from == dtype_to + return true + elseif supertype(dtype_from) == supertype(dtype_to) + return sizeof(dtype_from) <= sizeof(dtype_to) + end + return false +end diff --git a/src/consistency.jl b/src/consistency.jl index 67577d4..413f728 100644 --- a/src/consistency.jl +++ b/src/consistency.jl @@ -7,6 +7,7 @@ function _data_to_shape_and_dtype(data, shape, dtype) if isnothing(dtype) dtype = eltype(data) end + _assert_data_shape_dtype_match(data, shape, dtype) return (shape, dtype) end if isnothing(dtype) @@ -15,14 +16,34 @@ function _data_to_shape_and_dtype(data, shape, dtype) (shape, dtype) end -function _assert_data_shape_dtype_match(data, shape, dtype) +function _assert_data_shape_dtype_match(data, shape::Union{Dims, Nothing}, dtype) # https://github.com/CINPLA/exdir/blob/89c1d34a5ce65fefc09b6fe1c5e8fef68c494e75/exdir/core/group.py#L39 if !isnothing(data) - if !isnothing(shape) && (prod(shape) != prod(size(data))) - error("Provided shape and size(data) do not match") + sz = size(data) + if !isnothing(shape) && (prod(sz) != prod(shape)) + throw(ArgumentError("Provided shape and size(data) do not match: $shape vs $sz")) end - if !isnothing(dtype) && (dtype != eltype(data)) - error("Provided dtype and eltype(data) do not match") + et = eltype(data) + if !isnothing(dtype) && (et != dtype) + throw(ArgumentError("Provided dtype and eltype(data) do not match: $dtype vs $et")) + end + end + nothing +end + +"""Only scalars and arrays of scalars are allowed as fill values.""" +function _assert_allowed_fillvalue(fillvalue) + if !isnothing(fillvalue) + # TODO + if isa(fillvalue, AbstractDict) + throw( + TypeError( + :allowed_fillvalue, + "fillvalue type is not supported", + AbstractArray, + typeof(fillvalue) + ) + ) end end end diff --git a/src/constants.jl b/src/constants.jl index eaec39c..2e2186b 100644 --- a/src/constants.jl +++ b/src/constants.jl @@ -7,6 +7,7 @@ const VERSION_METANAME = "version" const META_FILENAME = "exdir.yaml" const ATTRIBUTES_FILENAME = "attributes.yaml" const RAW_FOLDER_NAME = "__raw__" +const DSET_FILENAME = "data.npy" # typenames const DATASET_TYPENAME = "dataset" diff --git a/src/exceptions.jl b/src/exceptions.jl new file mode 100644 index 0000000..1c2b27a --- /dev/null +++ b/src/exceptions.jl @@ -0,0 +1,9 @@ +struct NotImplementedError <: Exception + msg::String +end + +struct IOError <: Exception + msg::String +end + +Base.showerror(io::IO, e::IOError) = print(io, "IOError: $(e.msg)") diff --git a/src/object.jl b/src/object.jl new file mode 100644 index 0000000..0f6952c --- /dev/null +++ b/src/object.jl @@ -0,0 +1,44 @@ +function root_directory(path::AbstractString) + # https://github.com/CINPLA/exdir/blob/89c1d34a5ce65fefc09b6fe1c5e8fef68c494e75/exdir/core/exdir_object.py#L128 + path = realpath(path) + found = false + while !found + (parent, pname) = splitdir(path) + if parent == path + return nothing + end + if !is_nonraw_object_directory(path) + path = parent + continue + end + meta_data = YAML.load_file(joinpath(path, META_FILENAME)) + if !haskey(meta_data, EXDIR_METANAME) + path = parent + continue + end + exdir_meta = meta_data[EXDIR_METANAME] + if !haskey(exdir_meta, TYPE_METANAME) + path = parent + continue + end + if FILE_TYPENAME != exdir_meta[TYPE_METANAME] + path = parent + continue + end + found = true + end + path +end + +function is_inside_exdir(path::AbstractString) + # https://github.com/CINPLA/exdir/blob/89c1d34a5ce65fefc09b6fe1c5e8fef68c494e75/exdir/core/exdir_object.py#L161 + path = realpath(path) + !isnothing(root_directory(path)) +end + +function assert_inside_exdir(path::AbstractString) + # https://github.com/CINPLA/exdir/blob/89c1d34a5ce65fefc09b6fe1c5e8fef68c494e75/exdir/core/exdir_object.py#L166 + if !is_inside_exdir(path) + error("Path " + path + " is not inside an Exdir repository.") + end +end diff --git a/src/path.jl b/src/path.jl index 1aa8e5b..68bae25 100644 --- a/src/path.jl +++ b/src/path.jl @@ -1,19 +1,29 @@ +function clean_path(path::AbstractString) + path = normpath(path) + if path[end] == '/' + path = path[1:end-1] + else + path + end +end + function name_to_asserted_group_path(name::AbstractString) - path = name + path = clean_path(name) if isabspath(path) - throw(ArgumentError("Absolute paths are currently not supported and unlikely to be implemented.")) + throw(NotImplementedError("Absolute paths are currently not supported and unlikely to be implemented.")) elseif splitpath(path) == [""] - throw(ArgumentError("Getting an item on a group with path '$name' is not supported and unlikely to be implemented.")) + throw(NotImplementedError("Getting an item on a group with path '$(name)' is not supported and unlikely to be implemented.")) end path end function remove_root(path::AbstractString) - components = splitpath(path) - rel = if components[1] == "/" - joinpath(components[2:length(components)]) - else - path + path = clean_path(path) + if isabspath(path) + path = relpath(path, "/") end - rel + path end + +form_relative_path(parent_path::AbstractString, object_name::AbstractString) = + joinpath(parent_path, object_name) |> clean_path diff --git a/test/attr.jl b/test/attr.jl index e69de29..035d51f 100644 --- a/test/attr.jl +++ b/test/attr.jl @@ -0,0 +1,3 @@ +@testset "attr" begin + +end diff --git a/test/can_cast.jl b/test/can_cast.jl new file mode 100644 index 0000000..994605b --- /dev/null +++ b/test/can_cast.jl @@ -0,0 +1,19 @@ +using Test + +import Exdir: + can_cast + +@testset "can_cast" begin + # safe + @test can_cast(Int32, Int32) + @test can_cast(Int32, Int64) + @test !can_cast(Int64, Int32) + + @test can_cast(Float32, Float32) + @test can_cast(Float32, Float64) + @test !can_cast(Float64, Float32) + + @test can_cast(UInt32, UInt32) + @test can_cast(UInt32, UInt64) + @test !can_cast(UInt64, UInt32) +end diff --git a/test/consistency.jl b/test/consistency.jl new file mode 100644 index 0000000..c4cf3a5 --- /dev/null +++ b/test/consistency.jl @@ -0,0 +1,23 @@ +using Test + +import Exdir: + _data_to_shape_and_dtype + +@testset "consistency" begin + @testset "data_to_shape_and_dtype" begin + default_dtype = Float64 + dim = (2, 3) + x = rand(default_dtype, dim...) + z = x * (1 + 1im) + + @test _data_to_shape_and_dtype(nothing, nothing, nothing) == (nothing, default_dtype) + @test _data_to_shape_and_dtype(nothing, nothing, Int32) == (nothing, Int32) + @test _data_to_shape_and_dtype(nothing, dim, nothing) == (dim, default_dtype) + @test _data_to_shape_and_dtype(nothing, dim, ComplexF16) == (dim, ComplexF16) + @test _data_to_shape_and_dtype(x, nothing, nothing) == (dim, default_dtype) + @test _data_to_shape_and_dtype(z, nothing, nothing) == (dim, ComplexF64) + + @test_throws ArgumentError _data_to_shape_and_dtype(x, (5, 9), nothing) + @test_throws ArgumentError _data_to_shape_and_dtype(x, nothing, Float16) + end +end diff --git a/test/dataset.jl b/test/dataset.jl index ed5c735..c90f879 100644 --- a/test/dataset.jl +++ b/test/dataset.jl @@ -1,70 +1,679 @@ +using EllipsisNotation using Exdir +using StructArrays using Test +import Exdir: NotImplementedError + +@testset "dataset" begin + # Create a scalar dataset. @testset "dataset_create_scalar" begin - # TODO fixture - f = exdiropen("dataset_create_scalar.exdir", "w") + (fx, f) = setup_teardown_file() + grp = create_group(f, "test") dset = create_dataset(grp, "foo"; shape=()) @test size(dset) == () + # TODO # @test collect(dset) == 0 + + cleanup_fixture(fx) end -# # Create a size-1 dataset. +# Create a size-1 dataset. @testset "dataset_create_simple" begin - f = exdiropen("dataset_create_simple.exdir", "w") + (fx, f) = setup_teardown_file() + grp = create_group(f, "test") dset = create_dataset(grp, "foo"; shape=(1,)) @test size(dset) == (1,) + # TODO + # @test collect(dset) + + cleanup_fixture(fx) +end + +# Create an extended dataset. +@testset "dataset_create_extended" begin + (fx, f) = setup_teardown_file() + + grp = create_group(f, "test") + + dset = create_dataset(grp, "foo"; shape=(63,)) + @test size(dset) == (63,) + @test length(dset) == 63 + + dset = create_dataset(grp, "bar"; shape=(6, 10)) + @test size(dset) == (6, 10) + @test length(dset) == 60 + + cleanup_fixture(fx) +end + +# Confirm that the default dtype is Float64. +@testset "dataset_default_dtype" begin + (fx, f) = setup_teardown_file() + + grp = create_group(f, "test") + + dset = create_dataset(grp, "foo"; shape=(63,)) + @test isa(collect(dset), AbstractArray{Float64}) + + cleanup_fixture(fx) +end + +# Missing shape raises TypeError in Python, ArgumentError in Julia. +@testset "dataset_missing_shape" begin + (fx, f) = setup_teardown_file() + + @test_throws ArgumentError create_dataset(f, "foo") + + cleanup_fixture(fx) +end + +# Confirm that an alternate dtype can be specified. +@testset "dataset_short_int" begin + (fx, f) = setup_teardown_file() + + dset = create_dataset(f, "foo"; shape=(63,), dtype=Int16) + @test isa(collect(dset), AbstractArray{Int16}) + + cleanup_fixture(fx) +end + +# Create a scalar dataset from existing array. +@testset "dataset_create_scalar_data" begin + (fx, f) = setup_teardown_file() + + grp = create_group(f, "test") + + data = ones() + dset = create_dataset(grp, "foo"; data=data) + @test size(dset) == size(data) + + cleanup_fixture(fx) +end + +# Create an extended dataset from existing data. +@testset "dataset_create_extended_data" begin + (fx, f) = setup_teardown_file() + + grp = create_group(f, "test") + + data = ones(63) + dset = create_dataset(grp, "foo"; data=data) + @test size(dset) == size(data) + + cleanup_fixture(fx) +end + +# Create dataset with missing intermediate groups. +@testset "dataset_intermediate_group" begin + (fx, f) = setup_teardown_file() + + # Trying to create intermediate groups that are absolute should fail just + # like when creating them on groups. + @test_throws NotImplementedError create_dataset(f, "/foo/bar/baz"; shape=(10, 10), dtype=Int32) + + ds = create_dataset(f, "foo/bar/baz"; shape=(10, 10), dtype=Int32) + @test isa(ds, Exdir.Dataset) + # Checking for an absolute path in a file should work, though. + @test "/foo/bar/baz" in f + + cleanup_fixture(fx) +end + +# Create from existing data, and make it fit a new shape. +@testset "dataset_reshape" begin + (fx, f) = setup_teardown_file() + + grp = create_group(f, "test") + + data = collect(Float64, 1:30) + dset = create_dataset(grp, "foo"; shape=(10, 3), data=data) + @test size(dset) == (10, 3) + @test dset.data == reshape(data, (10, 3)) + + cleanup_fixture(fx) +end + +# Feature: Datasets can be created only if they don't exist in the file +# Create new dataset with no conflicts. +@testset "dataset_create" begin + (fx, f) = setup_teardown_file() + + grp = create_group(f, "test") + + # def require_dataset(self, name, shape=None, dtype=None, exact=False, + # data=None, fillvalue=None): + dset = require_dataset(grp, "foo"; shape=(10, 3)) + @test isa(dset, Exdir.Dataset) + @test size(dset) == (10, 3) + + dset2 = require_dataset(grp, "bar"; data=(3, 10)) + dset3 = require_dataset(grp, "bar"; data=(4, 11)) + @test isa(dset2, Exdir.Dataset) + @test dset2[:] == [3, 10] + @test dset3[:] == [3, 10] + @test dset2 == dset3 + + cleanup_fixture(fx) +end + +# require_dataset yields existing dataset. +@testset "dataset_create_existing" begin + (fx, f) = setup_teardown_file() + + grp = create_group(f, "test") + + dset = require_dataset(grp, "foo"; shape=(10, 3), dtype=Float32) + dset2 = require_dataset(grp, "foo"; shape=(10, 3), dtype=Float32) + + @test dset == dset2 + + cleanup_fixture(fx) +end + +# require_dataset with shape conflict yields TypeError in Python. +@testset "dataset_shape_conflict" begin + (fx, f) = setup_teardown_file() + + grp = create_group(f, "test") + + create_dataset(grp, "foo"; shape=(10, 3)) + @test_throws DimensionMismatch require_dataset(grp, "foo"; shape=(10, 4)) + + cleanup_fixture(fx) +end + +# require_dataset with object type conflict yields TypeError. +@testset "dataset_type_conflict" begin + (fx, f) = setup_teardown_file() + + grp = create_group(f, "test") + + create_group(grp, "foo") + @test_throws TypeError require_dataset(grp, "foo"; shape=(10, 3)) + + cleanup_fixture(fx) +end + +# require_dataset with dtype conflict (strict mode) yields TypeError. +@testset "dataset_dtype_conflict" begin + (fx, f) = setup_teardown_file() + + grp = create_group(f, "test") + + create_dataset(grp, "foo"; shape=(10, 3), dtype=Float64) + @test_throws TypeError require_dataset(grp, "foo"; shape=(10, 3), dtype=UInt8) + + cleanup_fixture(fx) +end + +# # require_dataset with convertible type succeeds (non-strict mode)- +# @testset "dataset_dtype_close" begin +# (fx, f) = setup_teardown_file() + +# grp = create_group(f, "test") + +# dset = create_dataset(grp, "foo"; shape=(10, 3), dtype=Int32) +# dset2 = create_dataset(grp, "foo"; shape=(10, 3), dtype=Int16, exact=false) +# @test dset == dset2 +# @test eltype(dset2) == Int32 +# @test dset2.dtype == Int32 + +# cleanup_fixture(fx) +# end + +# Feature: Datasets can be created with fill value +# Fill value is reflected in dataset contents. +@testset "dataset_create_fillval" begin + (fx, f) = setup_teardown_file() + + grp = create_group(f, "test") + + dset = create_dataset(grp, "foo"; shape=(10,), fillvalue=4.0) + @test dset[1] == 4.0 + @test dset[8] == 4.0 + + cleanup_fixture(fx) +end + +# Fill value works with compound types. +@testset "dataset_compound_fill" begin + (fx, f) = setup_teardown_file() + + grp = create_group(f, "test") + + struct dt + a::Float32 + b::Int64 + end + v = StructArray{dt}((a = ones(1), b = ones(1)))[1] + dset = create_dataset(grp, "foo"; shape=(10,), dtype=dt, fillvalue=v) + + cleanup_fixture(fx) +end + +# Bogus fill value raises TypeError. +@testset "dataset_exc" begin + (fx, f) = setup_teardown_file() + + grp = create_group(f, "test") + + @test_throws TypeError create_dataset(grp, "foo"; shape=(10,), dtype=Float32, fillvalue=Dict("a" => 2)) + + cleanup_fixture(fx) +end + +# Assignment of fixed-length byte string produces a fixed-length ASCII dataset +@testset "dataset_string" begin + (fx, f) = setup_teardown_file() + + grp = create_group(f, "test") + + # TODO problem with underlying NPZ library + # dset = create_dataset(grp, "foo"; data="string") + # @test dset.data == "string" + @test_throws "unsupported type Char" create_dataset(grp, "foo"; data="string") + + cleanup_fixture(fx) +end + +# Feature: Dataset dtype is available as .dtype property in Python, eltype in Julia +# Retrieve dtype from dataset. +@testset "dataset_dtype" begin + (fx, f) = setup_teardown_file() + + grp = create_group(f, "test") + + dset = create_dataset(grp, "foo"; shape=(5,), dtype=UInt8) + @test eltype(dset) == UInt8 + + cleanup_fixture(fx) +end + +# Feature: Size of first axis is available via Python's len; +# For Julia, size(...) gives the full shape and length(...) gives the total number of elements. +@testset "dataset_len" begin + (fx, f) = setup_teardown_file() + + grp = create_group(f, "test") + + dset = create_dataset(grp, "foo"; shape=(312, 15)) + @test size(dset) == (312, 15) + @test length(dset) == 312 * 15 + + cleanup_fixture(fx) +end + +@testset "dataset_len_scalar" begin + (fx, f) = setup_teardown_file() + + grp = create_group(f, "test") + + dset = create_dataset(grp, "foo"; data=1) + @test size(dset) == () + @test length(dset) == 1 + + cleanup_fixture(fx) +end + +# Feature: Iterating over a dataset yields rows in Python, which is idiomatic +# for NumPy, but yields scalars in Julia. +@testset "dataset_iter" begin + (fx, f) = setup_teardown_file() + + grp = create_group(f, "test") + + dtype = Float64 + data = reshape(collect(dtype, 1:30), (10, 3)) + dset = create_dataset(grp, "foo"; data=data) + for (x, y) in zip(dset, data) + @test isa(x, dtype) + @test length(x) == 1 + @test size(x) == () + @test x == y + end + + cleanup_fixture(fx) end -# # Create an extended dataset. -# @testset "dataset_create_extended" begin -# f = exdiropen("dataset_create_extended.exdir", "w") +# # Iterating over scalar dataset raises TypeError. +# @testset "dataset_iter_scalar" begin +# (fx, f) = setup_teardown_file() + # grp = create_group(f, "test") -# dset = create_dataset(grp, "foo"; shape=(63,)) -# @test shape(dset) == (63,) -# @test length(dset) == 63 +# dset = create_dataset(grp, "foo"; shape=()) +# @test_throws TypeError [x for x in dset] -# dset = create_dataset(grp, "bar"; shape=(6, 10)) -# @test shape(dset) == (6, 10) -# @test length(dset) == 60 +# cleanup_fixture(fx) # end -# # Confirm that the default dtype is Float64. -# @testset "dataset_default_dtype" begin -# f = exdiropen("dataset_default_dtype.exdir", "w") +# Trailing slashes are unconditionally ignored. +@testset "dataset_trailing_slash" begin + (fx, f) = setup_teardown_file() + + f["dataset"] = 42 + @test "dataset/" in f + + cleanup_fixture(fx) +end + +# # Feature: Compound types correctly round-trip +# # Compound types are read back in correct order. +# @testset "dataset_compound" begin +# (fx, f) = setup_teardown_file() + # grp = create_group(f, "test") -# dset = create_dataset(grp, "foo"; shape=(63,)) -# @test isa(collect(dset), AbstractArray{Float64}) +# struct dt +# weight::Float64 +# cputime::Float64 +# walltime::Float64 +# parents_offset::UInt32 +# n_parents::UInt32 +# status::UInt8 +# endpoint_type::UInt8 +# end + + # TODO + # lo = 0 + # hi = 100 + # d = MappedDistribution(dt, + # Uniform(lo, hi), + # Uniform(lo, hi), + # Uniform(lo, hi), + # Uniform(lo, hi), + # Uniform(lo, hi), + # Uniform(lo, hi), + # Uniform(lo, hi) + # ) + + # dim = 16 + # testdata = StructArray{dt}(undef, dim) + # Random.rand!(testdata) + # testdata *= 100 + +# cleanup_fixture(fx) # end -# # Missing shape raises TypeError. -# @testset "dataset_missing_shape" begin - +# @testset "dataset_assign" begin +# (fx, f) = setup_teardown_file() + +# # TODO + +# cleanup_fixture(fx) # end -# # Confirm that an alternate dtype can be specified. -# @testset "dataset_short_int" begin -# f = exdiropen("dataset_short_int.exdir", "w") +# Set data works correctly. +@testset "dataset_set_data" begin + (fx, f) = setup_teardown_file() + + # TODO + # grp = create_group(f, "test") + + # testdata = ones(10, 2) + # grp["testdata"] = testdata + + cleanup_fixture(fx) +end + +@testset "dataset_eq_false" begin + (fx, f) = setup_teardown_file() + + grp = create_group(f, "test") + + dset = create_dataset(grp, "foo"; data=1) + dset2 = create_dataset(grp, "foobar"; shape=(2, 2)) + + @test dset != dset2 + @test dset != 2 + + cleanup_fixture(fx) +end + +@testset "dataset_eq" begin + (fx, f) = setup_teardown_file() + + grp = create_group(f, "test") -# dset = create_dataset(f, "foo"; shape=(63,), dtype=Int16) -# @test isa(collect(dset), AbstractArray{Int16}) + dset = create_dataset(grp, "foo"; data=ones(2, 2)) + + @test dset == dset + + cleanup_fixture(fx) +end + +# @testset "dataset_mmap" begin +# (fx, f) = setup_teardown_file() + +# # TODO + +# cleanup_fixture(fx) +# end + +# @testset "dataset_modify_view" begin +# (fx, f) = setup_teardown_file() + +# # TODO + +# cleanup_fixture(fx) # end -# @testset "dataset_create_scalar_data" begin +# @testset "dataset_single_index" begin +# (fx, f) = setup_teardown_file() + +# # TODO +# cleanup_fixture(fx) # end -# @testset "dataset_create_extended_data" begin +# @testset "dataset_single_null" begin +# (fx, f) = setup_teardown_file() +# # TODO + +# cleanup_fixture(fx) # end -# @testset "dataset_intermediate_group" begin +# @testset "dataset_scalar_index" begin +# (fx, f) = setup_teardown_file() + +# # TODO +# cleanup_fixture(fx) # end + +# @testset "dataset_scalar_null" begin +# (fx, f) = setup_teardown_file() + +# # TODO + +# cleanup_fixture(fx) +# end + +# @testset "dataset_compound_index" begin +# (fx, f) = setup_teardown_file() + +# # TODO + +# cleanup_fixture(fx) +# end + +# @testset "dataset_negative_stop" begin +# (fx, f) = setup_teardown_file() + +# # TODO + +# cleanup_fixture(fx) +# end + +# @testset "dataset_read" begin +# (fx, f) = setup_teardown_file() + +# # TODO + +# cleanup_fixture(fx) +# end + +# Array fill from constant is supported. +@testset "dataset_write_broadcast" begin + (fx, f) = setup_teardown_file() + + dt = Int8 + shape = (10,) + c = 42 + + dset = create_dataset(f, "x"; shape=shape, dtype=dt) + dset[..] .= c + + data = ones(dt, shape...) * c + + @test eltype(dset) == eltype(data) + @test isequal(dset.data, data) + + cleanup_fixture(fx) +end + +# Write a single element to the array. +@testset "dataset_write_element" begin + (fx, f) = setup_teardown_file() + + dt = Float16 + dset = create_dataset(f, "x"; shape=(10, 3), dtype=dt) + + data = dt.([1, 2, 3.0]) + # TODO Python is dset[5] + dset[5, :] = data + + out = dset[5, :] + @test eltype(out) == eltype(data) + @test isequal(out, data) + + cleanup_fixture(fx) +end + +# Write slices to array type. +@testset "dataset_write_slices" begin + (fx, f) = setup_teardown_file() + + dt = Int32 + data1 = ones(dt, 2, 3) + data2 = ones(dt, 4, 5, 3) + + dset = create_dataset(f, "x"; shape=(10, 9, 11), dtype=dt) + + dset[1, 1, 3:4] = data1 + @test eltype(dset[1, 1, 3:4]) == eltype(data1) + @test isequal(dset[1, 1, 3:4], data1) + + dset[4, 2:5, 7:11] = data2 + @test eltype(dset[4, 2:5, 7:11]) == eltype(data2) + @test isequal(dset[4, 2:5, 7:11], data2) + + cleanup_fixture(fx) +end + +# Read the contents of an array and write them back. +# +# The initialization is not the same as in Python, since NumPy allows for +# fancy dtypes where Julia could resort to structs without the array having a +# dtype of object. Use the third-party package StructArrays to efficiently +# emulate this. +@testset "dataset_roundtrip" begin + (fx, f) = setup_teardown_file() + + data = rand(10) + dset = create_dataset(f, "x"; data=data) + + out = dset[..] + @test out == data + dset[..] = out + @test dset[..] == out + @test dset[..] == data + + cleanup_fixture(fx) +end + +# Slice a dataset with a zero in its shape vector along the zero-length +# dimension. +@testset "dataset_slice_zero_length_dimension" begin + (fx, f) = setup_teardown_file() + + shapes = [(0,), (0, 3), (0, 2, 1)] + for (i, shape) in enumerate(shapes) + dset = create_dataset(f, "x$(i)"; shape=shape, dtype=Int32) + @test size(dset) == shape + out = dset[..] + # not AbstractArray, which Dataset obeys TODO + @test isa(out, Array) + @test size(out) == shape + out = dset[:] + @test isa(out, Array) + @test size(out) == shape + if length(shape) > 1 + out = dset[:, :2] + @test isa(out, Array) + @test size(out) == (0, 1) + end + end + + cleanup_fixture(fx) +end + +# Slice a dataset with a zero in its shape vector along a non-zero-length +# dimension. +@testset "dataset_slice_other_dimension" begin + (fx, f) = setup_teardown_file() + + shapes = [(3, 0), (1, 2, 0), (2, 0, 1)] + for (i, shape) in enumerate(shapes) + dset = create_dataset(f, "x$(i)"; shape=shape, dtype=Int32) + @test size(dset) == shape + out = dset[begin:2] + # not AbstractArray, which Dataset obeys TODO + @test isa(out, Array) + @test size(out) == (1, shape...) + end + + cleanup_fixture(fx) +end + +# Get a slice of length zero from a non-empty dataset. +@testset "dataset_slice_of_length_zero" begin + (fx, f) = setup_teardown_file() + + shapes = [(3,), (2, 2,), (2, 1, 5)] + for (i, shape) in enumerate(shapes) + dset = create_dataset(f, "x$(i)"; data=zeros(Int32, shape)) + @test size(dset) == shape + # Python + # rng = 2:2 + rng = 2:2:0 + out = dset[rng] + # not AbstractArray, which Dataset obeys TODO + @test isa(out, Array) + # TODO implications of this being different from Python? non-zero on all other dimensions? + # @test size(out) == (0, shape[2:end]) + @test size(out) == (0,) + end + + cleanup_fixture(fx) +end + +@testset "dataset_modify_all" begin + (fx, f) = setup_teardown_file() + + dset = create_dataset(f, "test"; data=1:10) + n = 4 + dset.data = ones(n) + @test dset.data == ones(n) + + cleanup_fixture(fx) +end + +end diff --git a/test/dummy.jl b/test/dummy.jl new file mode 100644 index 0000000..fcff6d7 --- /dev/null +++ b/test/dummy.jl @@ -0,0 +1,8 @@ +include("support.jl") +f = exdir_tmpfile() +g1 = create_group(f, "g1") +g2 = create_group(f, "g2") +g3 = create_group(f, "g3") +g4 = create_group(g2, "g4") +g5 = create_group(g2, "g5") +g6 = create_group(g3, "g6") diff --git a/test/file.jl b/test/file.jl index 1d6c781..137987f 100644 --- a/test/file.jl +++ b/test/file.jl @@ -7,19 +7,9 @@ import Exdir: create_object_directory, DATASET_TYPENAME, FILE_TYPENAME -include("support.jl") +# include("support.jl") -""" - remove(name) - -If name is a path or directory tree, recursively delete it. -Otherwise, do nothing. -""" -function remove(name) - if ispath(name) - rm(name, recursive=true) - end -end +@testset "file" begin @testset "form_location" begin @test form_location("/hello.exdir") == "/hello.exdir" @@ -478,3 +468,5 @@ end # assert isinstance(f, File) # assert not f + +end diff --git a/test/group.jl b/test/group.jl index 1c05e7a..df60067 100644 --- a/test/group.jl +++ b/test/group.jl @@ -1,12 +1,16 @@ using Exdir using Test -include("support.jl") +import Exdir: NotImplementedError + +# include("support.jl") + +@testset "group" begin @testset "group_init" begin fx = setup_teardown_folder() - grp = Group( + grp = Exdir.Group( root_directory = fx.testdir, parent_path = "", object_name = "test_object", @@ -23,6 +27,26 @@ include("support.jl") cleanup_fixture(fx) end +@testset "group_init_trailing" begin + fx = setup_teardown_folder() + + grp = Exdir.Group( + root_directory = fx.testdir, + parent_path = "", + object_name = "test_object2/", + file = nothing + ) + + @test grp.root_directory == fx.testdir + @test grp.object_name == "test_object2/" + @test grp.parent_path == "" + @test isnothing(grp.file) + @test grp.relative_path == "test_object2" + @test grp.name == "/test_object2" + + cleanup_fixture(fx) +end + @testset "group_create" begin (fx, f) = setup_teardown_file() @@ -45,7 +69,7 @@ end grp2 = create_group(grp, "a") - grp3 = create_group(grp2, "b") + grp3 = create_group(grp, "b") @test length(grp) == 2 @test length(grp2) == 0 @@ -71,12 +95,13 @@ end cleanup_fixture(fx) end +# Starting .create_group argument with /. @testset "group_create_absolute" begin (fx, f) = setup_teardown_file() grp = create_group(f, "/a") - @test_throws ArgumentError create_group(grp, "/b") + @test_throws NotImplementedError create_group(grp, "/b") cleanup_fixture(fx) end @@ -89,6 +114,7 @@ end end @testset "group_create_intermediate" begin + # intermediate groups can be created automatically. (fx, f) = setup_teardown_file() grp = create_group(f, "test") @@ -107,6 +133,7 @@ end cleanup_fixture(fx) end +# Name conflict causes group creation to fail with ArgumentError. @testset "group_create_exception" begin (fx, f) = setup_teardown_file() @@ -120,6 +147,8 @@ end cleanup_fixture(fx) end +# Feature: Groups can be auto-created, or opened via .require_group +# Existing group is opened and returned. @testset "group_open_existing" begin (fx, f) = setup_teardown_file() @@ -136,19 +165,47 @@ end cleanup_fixture(fx) end +# Group is created if it doesn't exist. @testset "group_create" begin (fx, f) = setup_teardown_file() + grp = create_group(f, "test") + + grp2 = require_group(grp, "foo") + @test isa(grp2, Exdir.Group) + @test grp2.name == "/test/foo" + cleanup_fixture(fx) end +# Opening conflicting object results in TODOError. @testset "group_require_exception" begin (fx, f) = setup_teardown_file() + grp = create_group(f, "test") + + # grp.create_dataset("foo", (1,)) + + # with pytest.raises(TypeError): + # grp.require_group("foo") + cleanup_fixture(fx) end -# set_item_intermediatex +# TODO +# @testset "group_set_item_intermediate" begin +# (_, f) = setup_teardown_file() + +# group1 = create_group(f, "group1") +# group2 = create_group(group1, "group2") +# group3 = create_group(group2, "group3") +# f["group1/group2/group3/dataset"] = [1, 2, 3] + +# @test_ isa(f["group1/group2/group3/dataset"], Exdir.Dataset) +# @test f["group1/group2/group3/dataset"].data == [1, 2, 3] + +# cleanup_fixture(fx) +# end @testset "group_delete" begin (fx, f) = setup_teardown_file() @@ -160,6 +217,8 @@ end delete!(grp, "foo") @test !in("foo", grp) + # alias delete_object as in HDF5.jl + create_group(grp, "bar") @test in("bar", grp) @@ -178,6 +237,8 @@ end delete!(f, "test") @test !in("test", f) + # alias delete_object as in HDF5.jl + create_group(f, "test2") @test in("test2", f) @@ -194,25 +255,28 @@ end create_raw(grp, "foo") @test in("foo", grp) - # Julia dicts delete!(grp, "foo") @test !in("foo", grp) + # alias delete_object as in HDF5.jl + create_raw(grp, "bar") @test in("bar", grp) - # HDF5.jl delete_object(grp, "bar") @test !in("bar", grp) cleanup_fixture(fx) end +# Deleting non-existent object raises TODOError @testset "group_nonexisting" begin (fx, f) = setup_teardown_file() - # TODO - match = "No such object: 'foo' in path *" + grp = create_group(f, "test") + + # @test_throws "KeyError: No such object: 'foo' in path *" delete!(grp, "foo") + @test_throws KeyError delete!(grp, "foo") cleanup_fixture(fx) end @@ -249,7 +313,7 @@ end @test grp2.name == grp4.name @test grp2 == grp4 - @test_throws ArgumentError grp["/test"] + @test_throws NotImplementedError grp["/test"] cleanup_fixture(fx) end @@ -286,9 +350,7 @@ end @test in("b", grp) @test !in("c", grp) - # TODO - # @test_throws ArgumentError in("/b", grp) - @test_throws ArgumentError "/b" in grp + @test_throws NotImplementedError "/b" in grp cleanup_fixture(fx) end @@ -329,7 +391,7 @@ end grp = create_group(f, "test") - @test_throws ArgumentError "/" in grp + @test_throws NotImplementedError "/" in grp cleanup_fixture(fx) end @@ -387,10 +449,13 @@ end grpd = create_group(grp, "d") grpc = create_group(grp, "c") - # TODO - # for i, (key, value) in enumerate(grp.items()): - # assert key == names[i] - # assert value == groups[i] + names = ["a", "b", "c", "d"] + groups = [grpa, grpb, grpc, grpd] + + for (i, (k, v)) in enumerate(pairs(grp)) + @test k == names[i] + @test v == groups[i] + end cleanup_fixture(fx) end @@ -448,3 +513,5 @@ end cleanup_fixture(fx) end + +end diff --git a/test/help_functions.jl b/test/help_functions.jl new file mode 100644 index 0000000..d26779c --- /dev/null +++ b/test/help_functions.jl @@ -0,0 +1,3 @@ +@testset "help_functions" begin + +end diff --git a/test/object.jl b/test/object.jl index 338e0b9..2530c43 100644 --- a/test/object.jl +++ b/test/object.jl @@ -1,9 +1,11 @@ using Exdir using Test -import Exdir: Object, open_object, ATTRIBUTES_FILENAME, META_FILENAME +import Exdir: IOError, Object, open_object, ATTRIBUTES_FILENAME, META_FILENAME -include("support.jl") +# include("support.jl") + +@testset "object" begin @testset "object_init" begin fx = setup_teardown_folder() @@ -89,3 +91,5 @@ end cleanup_fixture(fx) end + +end diff --git a/test/path.jl b/test/path.jl new file mode 100644 index 0000000..33a2c86 --- /dev/null +++ b/test/path.jl @@ -0,0 +1,42 @@ +using Test + +import Exdir: + clean_path, + name_to_asserted_group_path, + NotImplementedError, + remove_root, + form_relative_path + +@testset "path" begin + @testset "clean_path" begin + @test clean_path("/hello") == "/hello" + @test clean_path("/hello/") == "/hello" + @test clean_path("/hello///////") == "/hello" + @test clean_path("/hello////world///") == "/hello/world" + @test clean_path("./hello////world///") == "hello/world" + end + + @testset "name_to_asserted_group_path" begin + @test name_to_asserted_group_path("mything") == "mything" + @test name_to_asserted_group_path("mynested/thing") == "mynested/thing" + @test name_to_asserted_group_path("") == "." + @test_throws NotImplementedError name_to_asserted_group_path("/mynested/thing") + end + + @testset "remove_root" begin + @test remove_root("hello") == "hello" + @test remove_root("/hello") == "hello" + @test remove_root("///hello") == "hello" + end + + @testset "form_relative_path" begin + @test form_relative_path(".", "citrus") == "citrus" + @test form_relative_path("./citrus", "") == "citrus" + @test form_relative_path("citrus", "lemon") == "citrus/lemon" + @test form_relative_path("./citrus", "lemon") == "citrus/lemon" + @test form_relative_path("./citrus", "lemon/") == "citrus/lemon" + @test form_relative_path("./citrus", "lemon/meyer/") == "citrus/lemon/meyer" + @test form_relative_path(".", "") == "." + @test form_relative_path("./", "") == "." + end +end diff --git a/test/plugins.jl b/test/plugins.jl new file mode 100644 index 0000000..40d0382 --- /dev/null +++ b/test/plugins.jl @@ -0,0 +1,3 @@ +@testset "plugins" begin + +end diff --git a/test/prepare_write.jl b/test/prepare_write.jl new file mode 100644 index 0000000..3d5dedc --- /dev/null +++ b/test/prepare_write.jl @@ -0,0 +1,17 @@ +using Exdir +using Test + +import Exdir: _prepare_write + +@testset "prepare_write" begin + # Julia: _prepare_write(data, attrs::AbstractDict, meta::AbstractDict) + # Python: def _prepare_write(data, plugins, attrs, meta): + + ret = _prepare_write(42, Dict(), Dict()) + ref1 = collect(42) + @test ret[1] == ref1 + + ret = _prepare_write("string", Dict(), Dict()) + ref1 = collect("string") + @test ret[1] == ref1 +end diff --git a/test/quantities.jl b/test/quantities.jl new file mode 100644 index 0000000..98dc121 --- /dev/null +++ b/test/quantities.jl @@ -0,0 +1,3 @@ +@testset "quantities" begin + +end diff --git a/test/raw.jl b/test/raw.jl index cc1bee4..affc000 100644 --- a/test/raw.jl +++ b/test/raw.jl @@ -1,9 +1,11 @@ using Exdir using Test -import Exdir +import Exdir: IOError -include("support.jl") +# include("support.jl") + +@testset "raw" begin @testset "raw_init" begin fx = setup_teardown_folder() @@ -75,3 +77,5 @@ end @test ispath(joinpath(f.directory, "group", "dataset", "raw")) end + +end diff --git a/test/runtests.jl b/test/runtests.jl index c01e615..17ae937 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,12 +1,20 @@ using Test @testset "Exdir.jl" begin + include("support.jl") + include("can_cast.jl") # include("example_hdf5.jl") # include("example.jl") + include("path.jl") + include("consistency.jl") include("object.jl") + include("help_functions.jl") include("raw.jl") - # include("group.jl") + include("group.jl") # include("file.jl") - # include("dataset.jl") - # include("attr.jl") + include("prepare_write.jl") + include("dataset.jl") + include("attr.jl") + include("plugins.jl") + include("quantities.jl") end diff --git a/test/support.jl b/test/support.jl index ea556f2..e449cd7 100644 --- a/test/support.jl +++ b/test/support.jl @@ -60,3 +60,15 @@ function exdir_tmpfile() # close(f) # rm(tmpdir, recursive=true) end + +""" + remove(name) + +If name is a path or directory tree, recursively delete it. +Otherwise, do nothing. +""" +function remove(name) + if ispath(name) + rm(name, recursive=true) + end +end