handling corner case - when a column is of type Missing

Author: sl-solution

src/dataset/combine.jl

@@ -314,14 +314,14 @@ function _check_mutliple_rows_for_each_group(ds, ms)
         # byrow are not checked since they are not going to modify the number of rows
         if ms[i].first isa Tuple
             T = return_type(ms[i].second.first, ntuple(j-> ds[!, ms[i].first[j]].val, length(ms[i].first)))
-            if T <: AbstractVector
+            if T <: AbstractVector && T !== Union{}
                 return i
             end
         elseif !(ms[i].second.first isa Expr) &&
                  haskey(index(ds), ms[i].first) #&&
                     #!(ms[i].first ∈ map(x->x.second.second, view(ms, 1:(i-1)))) #TODO monitor this for any unseen problem
             T = return_type(ms[i].second.first, ds[!, ms[i].first].val)
-            if T <: AbstractVector
+            if T <: AbstractVector && T !== Union{}
                 return i
             end
         end
@@ -409,7 +409,7 @@ function _check_the_output_type(x, mssecond)
     # * AbstractVector{T} where T
     # * Vector{T}
     # * not a Vector
-    CT == Union{} && throw(ArgumentError("compiler cannot assess the return type of calling `$(mssecond)` on input, you may want to try using `byrow`"))
+    CT == Union{} && throw(ArgumentError("compiler cannot assess the return type of calling `$(mssecond)` on input, you may want to try using `byrow`."))
     if CT <: AbstractVector
         if hasproperty(CT, :var)
             T = Union{Missing, CT.var.ub}

src/dataset/transpose.jl

@@ -1,6 +1,6 @@
 # a helper function that checks if there is enough memory for the output data frame
 #  If type is not Number, probably something is wrong about setting the variables and it is better to be conservative. here 10^7 threshhold is arbitarary
-_check_allocation_limit(T, rows, cols) = T <: Number ? sizeof(T)*rows*cols / Base.Sys.total_memory() : rows*cols/10^7
+_check_allocation_limit(T, rows, cols) = T !== Union{} && T <: Number ? sizeof(T)*rows*cols / Base.Sys.total_memory() : rows*cols/10^7
 
 _default_renamecolid_function_withoutid(x, y) = "_c" * string(x)
 _default_renamecolid_function_withid(x, y) = identity(string(values(x)))

src/other/utils.jl

@@ -3,6 +3,7 @@ const FLOATS = Union{Float16, Float32, Float64}
 
 # modified return_type to suit for our purpose
 function return_type(f::Function, x)
+	eltype(x) == Missing && return Missing
     CT = nonmissingtype(eltype(x))
     if CT <: AbstractVector
         return return_type_tuple(f, x)

src/sort/groupby.jl

@@ -174,7 +174,6 @@ function combine(gds::Union{GroupBy, GatherBy}, @nospecialize(args...); dropgrou
 	# the transformation returning multiple rows must not be based on the previous columns in combine
 	# result (which seems reasonable ??)
 	_first_vector_res = _check_mutliple_rows_for_each_group(gds.parent, ms)
-
 	_is_groupingcols_modifed(gds, ms) && throw(ArgumentError("`combine` cannot modify the grouping or sorting columns, use a different name for the computed column"))
 
 	groupcols = gds.groupcols

test/grouping.jl

@@ -474,11 +474,13 @@ end
     @test combine(groupby(sds2, 1), :y=>sum) == Dataset(x=[1,2,3, missing],  sum_y=[missing, -7.0, 11.2,missing])
     @test combine(gatherby(sds2, 1, isgathered = true), :y=>sum) == Dataset(x = [1,3,2,missing,3,2], sum_y=[missing, 2.2, -1,missing, 9,-6])
 
-    ds = Dataset(x = [1,2,1,2,3], y1 = Union{Int8, Missing}[1,2,missing,4,missing], y2 = Union{Int32, Missing}[1,2,3,4,missing], y3=Union{Int16, Missing}[100,20,3000,4,missing], y4=Float16.(rand(5)), y5=rand(BigFloat, 5))
-    sds = view(ds, [1,2,3,4,5], [1,2,3,4,5,6])
+    ds = Dataset(x = [1,2,1,2,3], y1 = Union{Int8, Missing}[1,2,missing,4,missing], y2 = Union{Int32, Missing}[1,2,3,4,missing], y3=Union{Int16, Missing}[100,20,3000,4,missing], y4=Float16.(rand(5)), y5=rand(BigFloat, 5), y6=[missing, missing, missing, missing, missing])
+    sds = view(ds, [1,2,3,4,5], [1,2,3,4,5,6,7])
 
     @test combine(gatherby(sds, 1), 2:4 .=>Ref([sum, mean, maximum, minimum, IMD.n, IMD.nmissing])) == Dataset([Union{Missing, Int64}[1, 2, 3], Union{Missing, Int64}[1, 6, missing], Union{Missing, Float64}[1.0, 3.0, missing], Union{Missing, Int8}[1, 4, missing], Union{Missing, Int8}[1, 2, missing], Union{Missing, Int64}[1, 2, 0], Union{Missing, Int64}[1, 0, 1], Union{Missing, Int64}[4, 6, missing], Union{Missing, Float64}[2.0, 3.0, missing], Union{Missing, Int32}[3, 4, missing], Union{Missing, Int32}[1, 2, missing], Union{Missing, Int64}[2, 2, 0], Union{Missing, Int64}[0, 0, 1], Union{Missing, Int64}[3100, 24, missing], Union{Missing, Float64}[1550.0, 12.0, missing], Union{Missing, Int16}[3000, 20, missing], Union{Missing, Int16}[100, 4, missing], Union{Missing, Int64}[2, 2, 0], Union{Missing, Int64}[0, 0, 1]], ["x", "sum_y1", "mean_y1", "maximum_y1", "minimum_y1", "n_y1", "nmissing_y1", "sum_y2", "mean_y2", "maximum_y2", "minimum_y2", "n_y2", "nmissing_y2", "sum_y3", "mean_y3", "maximum_y3", "minimum_y3", "n_y3", "nmissing_y3"])
 
+    @test combine(gatherby(sds,1), :y6=>sum=>:mm) == combine(gatherby(sds,1), :y6=>minimum=>:mm) == combine(gatherby(sds,1), :y6=>var=>:mm) == combine(gatherby(sds,1), :y6=>(x->sum(x))=>:mm) == combine(gatherby(sds,1), :y6=>(x->var(x))=>:mm) == combine(gatherby(sds,1), :y6=>(x->minimum(x))=>:mm) == Dataset(x = [1,2,3] , mm = [missing, missing, missing])
+
     var1(x) = var(x)
     std1(x) = std(x)
     median1(x) = median(x)

test/transpose.jl

@@ -323,6 +323,13 @@ const ≅ = isequal
         dst = transpose(ds, :x, variable_name = nothing)
         dst_t = compare(ds, dst, on =[:g1=>:g1, :g2=>:g2, :x=>:_c1], mapformats = true)
         @test all(byrow(dst_t, all, :))
+
+        ds = Dataset(x = [1,2,1,2,3], y = [missing, missing, missing, missing, missing])
+        @test transpose(groupby(ds,1), :y) == Dataset(x = [1,2,3], _variables_=["y","y","y"], _c1=[missing, missing, missing], _c2=[missing, missing, missing])
+        @test transpose(groupby(ds,1), :y, default = 0) == Dataset(x = [1,2,3], _variables_=["y","y","y"], _c1=[missing, missing, missing], _c2=[missing, missing, 0])
+        @test transpose(gatherby(ds,1), :y, default = 0) == Dataset(x = [1,2,3], _variables_=["y","y","y"], _c1=[missing, missing, missing], _c2=[missing, missing, 0])
+
+
 end