fix #102 check consistency of grouped and sub datasets

Author: sl-solution

src/dataset/getindex.jl

@@ -61,6 +61,8 @@ function _check_consistency(ds::Dataset)
 end
 
 function _check_consistency(ds::AbstractDataset)
+    # FIXME We should check the created date of sub-data, however, it is not working in some situations, e.g. modify!(sds, ...) 
+    # TODO However, We should add this whenever it is possible :   getfield(ds, :created) == _get_lastmodified(_attributes(parent(ds)))
     if ds isa SubDataset
         @assert length(index(ds).remap) == length(index(parent(ds))) "The parent data set which this view is based on, has been modified. To fix the issue recreate the view"
     end

src/other/broadcasting.jl

@@ -226,7 +226,7 @@ function Base.Broadcast.broadcast_unalias(dest, src::AbstractDataset)
             if src isa SubDataset
                 if !wascopied
                     src = SubDataset(_our_copy(parent(src), copycols=false),
-                                       index(src), rows(src))
+                                       index(src), rows(src), _get_lastmodified(_attributes(parent(src))))
                 end
                 parentidx = parentcols(index(src), i)
                 parent(src)[!, parentidx] = Base.unaliascopy(_columns(parent(src))[parentidx])
@@ -254,7 +254,7 @@ function _broadcast_unalias_helper(dest::AbstractDataset, scol::AbstractVector,
             if src isa SubDataset
                 if !wascopied
                     src =SubDataset(_our_copy(parent(src), copycols=false),
-                                      index(src), rows(src))
+                                      index(src), rows(src), _get_lastmodified(_attributes(parent(src))))
                 end
                 parentidx = parentcols(index(src), col2)
                 parent(src)[!, parentidx] = Base.unaliascopy(_columns(parent(src))[parentidx])

src/sort/gatherby.jl

@@ -127,7 +127,10 @@ mutable struct GatherBy
     starts
 	created::DateTime
 end
-Base.copy(gds::GatherBy) = GatherBy(copy(gds.parent), copy(gds.groupcols), copy(gds.groups), gds.lastvalid, gds.mapformats, gds.perm === nothing ? nothing : copy(gds.perm), gds.starts === nothing ? nothing : copy(gds.starts), gds.created)
+function Base.copy(gds::GatherBy)
+	ds_cpy = copy(gds.parent)
+	GatherBy(copy(gds.parent), copy(gds.groupcols), copy(gds.groups), gds.lastvalid, gds.mapformats, gds.perm === nothing ? nothing : copy(gds.perm), gds.starts === nothing ? nothing : copy(gds.starts), _get_lastmodified(_attributes(ds_cpy)))
+end
 
 
 nrow(ds::GatherBy) = nrow(ds.parent)
@@ -149,6 +152,7 @@ Base.summary(gds::GatherBy) =
 function Base.show(io::IO, gds::GatherBy;
 
 	kwargs...)
+	_check_consistency(gds)
 	if length(_get_perms(gds)) > 200
 		_show(io, view(gds.parent, [first(gds.perm, 100);last(gds.perm, 100)], :); title = summary(gds), show_omitted_cell_summary=false, show_row_number  = false, kwargs...)
 	else

src/sort/groupby.jl

@@ -170,7 +170,10 @@ mutable struct GroupBy
 	created::DateTime
 end
 
-Base.copy(gds::GroupBy) = GroupBy(copy(gds.parent), copy(gds.groupcols), copy(gds.rev), copy(gds.perm), copy(gds.starts), gds.lastvalid, gds.mapformats, gds.created)
+function Base.copy(gds::GroupBy)
+	ds_cp = copy(gds.parent)
+	GroupBy(ds_cp, copy(gds.groupcols), copy(gds.rev), copy(gds.perm), copy(gds.starts), gds.lastvalid, gds.mapformats, _get_lastmodified(_attributes(ds_cp)))
+end
 
 nrow(ds::GroupBy) = nrow(ds.parent)
 ncol(ds::GroupBy) = ncol(ds.parent)
@@ -234,6 +237,7 @@ end
 
 modify(origninal_gds::Union{GroupBy, GatherBy}, @nospecialize(args...); threads::Bool = true) = modify!(copy(origninal_gds), args..., threads = threads)
 function modify!(gds::Union{GroupBy, GatherBy}, @nospecialize(args...); threads::Bool = true)
+	_check_consistency(gds)
 	if parent(gds) isa SubDataset
 		idx_cpy = copy(index(parent(gds)))
 	else
@@ -316,6 +320,7 @@ end
 
 
 function combine(gds::Union{GroupBy, GatherBy}, @nospecialize(args...); dropgroupcols = false, threads = true)
+	_check_consistency(gds)
 	idx_cpy::Index = Index(Dict{Symbol, Int}(), Symbol[], Dict{Int, Function}())
 	if !dropgroupcols
         for i in gds.groupcols
@@ -442,6 +447,7 @@ Base.summary(gds::GroupBy) =
 function Base.show(io::IO, gds::GroupBy;
 
 	kwargs...)
+	_check_consistency(gds)
 	#TODO pretty_table is very slow for large views, temporary workaround, later we should fix this
 	if length(gds.perm) > 200
 		_show(io, view(gds.parent, [first(gds.perm, 100);last(gds.perm, 100)], :); title = summary(gds), show_omitted_cell_summary=false, show_row_number  = false, kwargs...)

src/subdataset/subdataset.jl

@@ -1,5 +1,5 @@
 """
-    SubDataset{<:Dataset, <:AbstractIndex, <:AbstractVector{Int}} <: Dataset
+    SubDataset{<:Dataset, <:AbstractIndex, <:AbstractVector{Int}, DateTime} <: Dataset
 
 A view of a `Dataset`. It is returned by a call to the `view` function
 on an `Dataset` if a collections of rows and columns are specified.
@@ -90,6 +90,7 @@ struct SubDataset{D<:AbstractDataset, S<:AbstractIndex, T<:AbstractVector{Int}}
     parent::D
     colindex::S
     rows::T # maps from subds row indexes to parent row indexes
+    created::DateTime
 end
 
 _attributes(sds::SubDataset) = getfield(parent(sds), :attributes)
@@ -111,11 +112,25 @@ Base.@propagate_inbounds function SubDataset(parent::Dataset, rows::AbstractVect
     sindex = SubIndex(index(parent), cols)
     # SubDataset without columns should not have any row
     if all(==(0), sindex.remap)
-        SubDataset(parent, sindex, Int[])
+        SubDataset(parent, sindex, Int[], _get_lastmodified(_attributes(parent)))
     else
-        SubDataset(parent,sindex , rows)
+        SubDataset(parent,sindex , rows, _get_lastmodified(_attributes(parent)))
     end
 end
+
+Base.@propagate_inbounds function SubDataset(parent::Dataset, rows::AbstractVector{Int}, cols, created)
+    @boundscheck if !checkindex(Bool, axes(parent, 1), rows)
+        throw(BoundsError(parent, (rows, cols)))
+    end
+    sindex = SubIndex(index(parent), cols)
+    # SubDataset without columns should not have any row
+    if all(==(0), sindex.remap)
+        SubDataset(parent, sindex, Int[], created)
+    else
+        SubDataset(parent,sindex , rows, created)
+    end
+end
+
 Base.@propagate_inbounds SubDataset(parent::Dataset, ::Colon, cols) =
     SubDataset(parent, axes(parent, 1), cols)
 @inline SubDataset(parent::Dataset, row::Integer, cols) =
@@ -144,7 +159,7 @@ Base.@propagate_inbounds function SubDataset(parent::Dataset, rows::AbstractVect
 end
 
 Base.@propagate_inbounds SubDataset(sds::SubDataset, rowind, cols) =
-    SubDataset(parent(sds), rows(sds)[rowind], parentcols(index(sds), cols))
+    SubDataset(parent(sds), rows(sds)[rowind], parentcols(index(sds), cols), getfield(sds, :created))
 Base.@propagate_inbounds SubDataset(sds::SubDataset, rowind::Bool, cols) =
     throw(ArgumentError("invalid row index of type Bool"))
 
@@ -158,7 +173,7 @@ Base.@propagate_inbounds SubDataset(sds::SubDataset, rowind::Bool, cols) =
 Base.@propagate_inbounds SubDataset(sds::SubDataset, rowind::Bool, ::Colon) =
     throw(ArgumentError("invalid row index of type Bool"))
 Base.@propagate_inbounds SubDataset(sds::SubDataset, ::Colon, cols) =
-    SubDataset(parent(sds), rows(sds), parentcols(index(sds), cols))
+    SubDataset(parent(sds), rows(sds), parentcols(index(sds), cols), getfield(sds, :created))
 @inline SubDataset(sds::SubDataset, ::Colon, ::Colon) = sds
 
 # just for showing SubDataset
@@ -202,15 +217,15 @@ Base.@propagate_inbounds Base.view(ads::AbstractDataset, ::typeof(!), colind::Co
 
 @inline Base.view(ads::AbstractDataset, rowinds, colind::Bool) =
     throw(ArgumentError("invalid column index $colind of type `Bool`"))
-Base.@propagate_inbounds Base.view(ads::AbstractDataset, rowinds,
+Base.@propagate_inbounds Base.view(parent::AbstractDataset, rowinds,
                                    colinds::MultiColumnIndex) =
-    SubDataset(ads, rowinds, colinds)
-Base.@propagate_inbounds Base.view(ads::AbstractDataset, rowinds::typeof(!),
+    SubDataset(parent, rowinds, colinds)
+Base.@propagate_inbounds Base.view(parent::AbstractDataset, rowinds::typeof(!),
                                    colinds::MultiColumnIndex) =
-    SubDataset(ads, :, colinds)
-Base.@propagate_inbounds Base.view(ads::AbstractDataset, rowinds::Not,
+    SubDataset(parent, :, colinds)
+Base.@propagate_inbounds Base.view(parent::AbstractDataset, rowinds::Not,
                                    colinds::MultiColumnIndex) =
-    SubDataset(ads, axes(ads, 1)[rowinds], colinds)
+    SubDataset(parent, axes(parent, 1)[rowinds], colinds)
 
 ##############################################################################
 ##