diff --git a/Project.toml b/Project.toml index 0a5157e..208dab9 100644 --- a/Project.toml +++ b/Project.toml @@ -18,13 +18,21 @@ SparseArraysExt = "SparseArrays" StatisticsExt = "Statistics" [compat] +Aqua = "0.8.12" +Distributed = "<0.0.1, 1" ExplicitImports = "1.13.2" +LinearAlgebra = "<0.0.1, 1" Primes = "0.4, 0.5" +Random = "<0.0.1, 1" +Serialization = "<0.0.1, 1" SparseArrays = "<0.0.1, 1" +SpecialFunctions = "0.8, 1, 2" Statistics = "<0.0.1, 1" +Test = "<0.0.1, 1" julia = "1.10" [extras] +Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" ExplicitImports = "7d51a73a-1435-4ff3-83d9-f097790105c7" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" SpecialFunctions = "276daf66-3868-5448-9aa4-cd146d93841b" @@ -32,4 +40,4 @@ Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["ExplicitImports", "SparseArrays", "SpecialFunctions", "Statistics", "Test"] +test = ["Aqua", "ExplicitImports", "SparseArrays", "SpecialFunctions", "Statistics", "Test"] diff --git a/src/DistributedArrays.jl b/src/DistributedArrays.jl index 51f50ce..f9b0c4c 100644 --- a/src/DistributedArrays.jl +++ b/src/DistributedArrays.jl @@ -10,6 +10,8 @@ using Serialization: Serialization, AbstractSerializer, deserialize, serialize using Primes: factor +import SparseArrays + # DArray exports export DArray, SubDArray, SubOrDArray, @DArray export dzeros, dones, dfill, drand, drandn, distribute, localpart, localindices, ppeval diff --git a/src/broadcast.jl b/src/broadcast.jl index be6a4ac..a689faa 100644 --- a/src/broadcast.jl +++ b/src/broadcast.jl @@ -60,7 +60,7 @@ end # - Q: How do decide on the cuts # - then localise arguments on each node ## -@inline function Base.copyto!(dest::DDestArray, bc::Broadcasted) +@inline function Base.copyto!(dest::DDestArray, bc::Broadcasted{Nothing}) axes(dest) == axes(bc) || Broadcast.throwdm(axes(dest), axes(bc)) # Distribute Broadcasted diff --git a/src/darray.jl b/src/darray.jl index 9d4d8a4..e343c83 100644 --- a/src/darray.jl +++ b/src/darray.jl @@ -368,7 +368,9 @@ Can return a view into `localpart(A)` end # shortcut to set/get localparts of a distributed object -function Base.getindex(d::DArray, s::Symbol) +Base.getindex(d::DArray, s::Symbol) = _getindex(d, s) +Base.getindex(d::DistributedArrays.DArray{<:Any, 1}, s::Symbol) = _getindex(d, s) +function _getindex(d::DArray, s::Symbol) @assert s in [:L, :l, :LP, :lp] return localpart(d) end @@ -438,6 +440,14 @@ function Base.:(==)(d1::SubDArray, d2::SubDArray) return t end +# Fix method ambiguities +for T in (:DArray, :SubDArray) + @eval begin + Base.:(==)(d1::$T{<:Any,1}, d2::SparseArrays.ReadOnly) = d1 == parent(d2) + Base.:(==)(d1::SparseArrays.ReadOnly, d2::$T{<:Any,1}) = parent(d1) == d2 + end +end + """ locate(d::DArray, I::Int...) @@ -497,10 +507,28 @@ dfill(v, d1::Integer, drest::Integer...) = dfill(v, convert(Dims, tuple(d1, dres Construct a distributed uniform random array. Trailing arguments are the same as those accepted by `DArray`. """ -drand(r, dims::Dims, args...) = DArray(I -> rand(r, map(length,I)), dims, args...) -drand(r, d1::Integer, drest::Integer...) = drand(r, convert(Dims, tuple(d1, drest...))) -drand(d1::Integer, drest::Integer...) = drand(Float64, convert(Dims, tuple(d1, drest...))) -drand(d::Dims, args...) = drand(Float64, d, args...) +drand(::Type{T}, dims::Dims) where {T} = DArray(I -> rand(T, map(length, I)), dims) +drand(X, dims::Dims) = DArray(I -> rand(X, map(length, I)), dims) +drand(dims::Dims) = drand(Float64, dims) + +drand(::Type{T}, d1::Integer, drest::Integer...) where {T} = drand(T, Dims((d1, drest...))) +drand(X, d1::Integer, drest::Integer...) = drand(X, Dims((d1, drest...))) +drand(d1::Integer, drest::Integer...) = drand(Float64, Dims((d1, drest...))) + +# With optional process IDs and number of chunks +for N in (1, 2) + @eval begin + drand(::Type{T}, dims::Dims, args::Vararg{Any,$N}) where {T} = DArray(I -> rand(T, map(length, I)), dims, args...) + drand(X, dims::Dims, args::Vararg{Any,$N}) = DArray(I -> rand(X, map(length, I)), dims, args...) + drand(dims::Dims, args::Vararg{Any,$N}) = drand(Float64, dims, args...) + end +end + +# Fix method ambiguities +drand(dims::Dims, procs::Tuple{Vararg{Int}}) = drand(Float64, dims, procs) +drand(dims::Dims, procs::Tuple{Vararg{Int}}, dist) = drand(Float64, dims, procs, dist) +drand(X::Tuple{Vararg{Int}}, dim::Integer) = drand(X, Dims((dim,))) +drand(X::Tuple{Vararg{Int}}, d1::Integer, d2::Integer) = drand(X, Dims((d1, d2))) """ drandn(dims, ...) @@ -620,7 +648,7 @@ allowscalar(flag = true) = (_allowscalar[] = flag) _scalarindexingallowed() = _allowscalar[] || throw(ErrorException("scalar indexing disabled")) getlocalindex(d::DArray, idx...) = localpart(d)[idx...] -function getindex_tuple(d::DArray{T}, I::Tuple{Vararg{Int}}) where T +function getindex_tuple(d::DArray{T,N}, I::NTuple{N,Int}) where {T,N} chidx = locate(d, I...) idxs = d.indices[chidx...] localidx = ntuple(i -> (I[i] - first(idxs[i]) + 1), ndims(d)) @@ -632,11 +660,13 @@ function Base.getindex(d::DArray, i::Int) _scalarindexingallowed() return getindex_tuple(d, Tuple(CartesianIndices(d)[i])) end -function Base.getindex(d::DArray, i::Int...) +function Base.getindex(d::DArray{<:Any,N}, i::Vararg{Int,N}) where {N} _scalarindexingallowed() return getindex_tuple(d, i) end - +function Base.getindex(d::DArray{<:Any,N}, I::Vararg{Any,N}) where {N} + return view(d, I...) +end Base.getindex(d::DArray) = d[1] Base.getindex(d::SubDArray, I::Int...) = invoke(getindex, Tuple{SubArray{<:Any,N},Vararg{Int,N}} where N, d, I...) Base.getindex(d::SubOrDArray, I::Union{Int,UnitRange{Int},Colon,Vector{Int},StepRange{Int,Int}}...) = view(d, I...) @@ -667,6 +697,15 @@ function Base.copyto!(dest::SubOrDArray, src::AbstractArray) return dest end +# Fix method ambiguities +# TODO: Improve efficiency? +Base.copyto!(dest::SubOrDArray{<:Any,2}, src::SparseArrays.AbstractSparseMatrixCSC) = copyto!(dest, Matrix(src)) +@static if isdefined(SparseArrays, :CHOLMOD) + Base.copyto!(dest::SubOrDArray, src::SparseArrays.CHOLMOD.Dense) = copyto!(dest, Array(src)) + Base.copyto!(dest::SubOrDArray{T}, src::SparseArrays.CHOLMOD.Dense{T}) where {T<:Union{Float32,Float64,ComplexF32,ComplexF64}} = copyto!(dest, Array(src)) + Base.copyto!(dest::SubOrDArray{T,2}, src::SparseArrays.CHOLMOD.Dense{T}) where {T<:Union{Float32,Float64,ComplexF32,ComplexF64}} = copyto!(dest, Array(src)) +end + function Base.deepcopy(src::DArray) dest = similar(src) @sync for p in procs(src) diff --git a/src/mapreduce.jl b/src/mapreduce.jl index 3df1e10..ff4254b 100644 --- a/src/mapreduce.jl +++ b/src/mapreduce.jl @@ -11,7 +11,13 @@ function Base.map!(f::F, dest::DArray, src::DArray{<:Any,<:Any,A}) where {F,A} return dest end -function Base.reduce(f, d::DArray) +# Only defining `reduce(f, ::DArray)` causes method ambiguity issues with +# - `reduce(hcat, ::AbstractVector{<:AbstractVecOrMat})` +# - `reduce(vcat, ::AbstractVector{<:AbstractVecOrMat})` +Base.reduce(f, d::DArray) = _reduce(f, d) +Base.reduce(::typeof(hcat), d::DArray{<:AbstractVecOrMat, 1}) = _reduce(hcat, d) +Base.reduce(::typeof(vcat), d::DArray{<:AbstractVecOrMat, 1}) = _reduce(vcat, d) +function _reduce(f, d::DArray) results = asyncmap(procs(d)) do p remotecall_fetch(p) do return reduce(f, localpart(d)) diff --git a/test/aqua.jl b/test/aqua.jl new file mode 100644 index 0000000..ec7a70b --- /dev/null +++ b/test/aqua.jl @@ -0,0 +1,6 @@ +using DistributedArrays, Test +import Aqua + +@testset "Aqua" begin + Aqua.test_all(DistributedArrays) +end diff --git a/test/runtests.jl b/test/runtests.jl index dcdf380..2c1da1d 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -36,6 +36,7 @@ function check_leaks() end end +include("aqua.jl") include("explicit_imports.jl") include("darray.jl") include("spmd.jl")