From ddfa09e54eda94073f5e621280a66538062367e5 Mon Sep 17 00:00:00 2001 From: Johannes Terblanche Date: Fri, 3 Apr 2026 15:07:22 +0200 Subject: [PATCH] Refactor DFG single agent to multiple agents and rename blobstore->storelabel and :default -> :primary --- src/DataBlobs/entities/BlobEntry.jl | 19 ++- src/DataBlobs/entities/BlobStores.jl | 8 +- src/DataBlobs/services/BlobEntry.jl | 26 +++- src/DataBlobs/services/BlobStores.jl | 18 +-- src/DataBlobs/services/BlobWrappers.jl | 42 +++--- src/Deprecated.jl | 8 +- src/DistributedFactorGraphs.jl | 14 +- src/FileDFG/services/FileDFG.jl | 20 ++- src/GraphsDFG/GraphsDFG.jl | 2 +- src/GraphsDFG/entities/GraphsDFG.jl | 61 ++++----- src/GraphsDFG/services/GraphsDFG.jl | 51 ++++--- src/entities/Agent_and_Graph.jl | 5 + src/errors.jl | 14 ++ src/services/AbstractDFG.jl | 102 ++++++++++++-- src/services/Bloblet.jl | 40 ++++-- src/services/CompareUtils.jl | 2 +- src/services/CustomPrinting.jl | 4 +- src/services/Tags.jl | 16 +-- test/consol_DataEntryBlobTests.jl | 6 +- test/fileDFGTests.jl | 13 +- test/iifInterfaceTests.jl | 12 +- test/testBlobStoresAndWrappers.jl | 20 +-- test/testBlocks.jl | 176 ++++++++++++++----------- 23 files changed, 431 insertions(+), 248 deletions(-) diff --git a/src/DataBlobs/entities/BlobEntry.jl b/src/DataBlobs/entities/BlobEntry.jl index f89cff66..bcddb0dc 100644 --- a/src/DataBlobs/entities/BlobEntry.jl +++ b/src/DataBlobs/entities/BlobEntry.jl @@ -8,13 +8,13 @@ A `Blobentry` is a small about of structured data that holds reference informati can exist on different graph nodes spanning Agents and Factor Graphs which can all reference the same `Blob`. Notes: -- `blobid`s should be unique within a blobstore and are immutable. +- `blobid`s should be unique within a Blobstore and are immutable. """ StructUtils.@kwarg struct Blobentry """ Human friendly label of the `Blob` and also used as unique identifier per node on which a `Blobentry` is added. E.g. do "LEFTCAM_1", "LEFTCAM_2", ... of you need to repeat a label on the same variable. """ label::Symbol - """ The label of the `Blobstore` in which the `Blob` is stored. Default is `:default`.""" - blobstore::Symbol = :default + """ The label of the `Blobstore` in which the `Blob` is stored. Default is `:primary`.""" + storelabel::Symbol = :primary """ Machine friendly and unique within a `Blobstore` identifier of the 'Blob'.""" blobid::UUID = uuid4() # was blobId """ (Optional) crc32c hash value to ensure data consistency which must correspond to the stored hash upon retrieval.""" @@ -52,21 +52,21 @@ version(::Type{Blobentry}) = v"0.1.0" function Blobentry( label::Symbol, - blobstore = :default; + storelabel = :primary; metadata::Union{JSONText, AbstractDict, NamedTuple} = JSONText("{}"), kwargs..., ) if !(metadata isa JSONText) metadata = JSONText(JSON.json(metadata)) end - return Blobentry(; label, blobstore, metadata, kwargs...) + return Blobentry(; label, storelabel, metadata, kwargs...) end # construction helper from existing Blobentry for user overriding via kwargs function Blobentry( entry::Blobentry; blobid::UUID = entry.blobid, label::Symbol = entry.label, - blobstore::Symbol = entry.blobstore, + storelabel::Symbol = entry.storelabel, crchash = entry.crchash, shahash = entry.shahash, size::Int64 = entry.size, @@ -76,10 +76,15 @@ function Blobentry( metadata::JSONText = entry.metadata, timestamp::TimeDateZone = entry.timestamp, version = entry.version, + blobstore = nothing, # TODO note deprecated in v0.29 ) + !isnothing(blobstore) && Base.depwarn( + "The `blobstore` keyword argument has been renamed to `storelabel`", + :Blobentry, + ) return Blobentry(; label, - blobstore, + storelabel, blobid, crchash, shahash, diff --git a/src/DataBlobs/entities/BlobStores.jl b/src/DataBlobs/entities/BlobStores.jl index 5fce2617..f00de2bd 100644 --- a/src/DataBlobs/entities/BlobStores.jl +++ b/src/DataBlobs/entities/BlobStores.jl @@ -1,7 +1,7 @@ """ AbstractBlobstore{T} -Abstract supertype for all blobstore implementations. +Abstract supertype for all Blobstore implementations. # Usage @@ -16,9 +16,9 @@ The parameter `T` represents the type of blobs stored (e.g., `Vector{UInt8}` or See concrete implementations for details. Design Notes -- `blobid` is not considered unique across blobstores with different labels only within a single blobstore. -- We cannot guarantee that `blobid` is unique across different blobstores with the same label and this is up to the end user. -- Within a single blobstore `addBlob!` will fail if there is a UUID collision. +- `blobid` is not considered unique across Blobstores with different labels only within a single Blobstore. +- We cannot guarantee that `blobid` is unique across different Blobstores with the same label and this is up to the end user. +- Within a single Blobstore `addBlob!` will fail if there is a UUID collision. - TODO: We should consider using uuid7 for `blobid`s (requires jl v1.12). - `Blobstrores`are identified by a `label::Symbol`, which allows for multiple blobstores to coexist in the same system. diff --git a/src/DataBlobs/services/BlobEntry.jl b/src/DataBlobs/services/BlobEntry.jl index b2098c91..b1de9844 100644 --- a/src/DataBlobs/services/BlobEntry.jl +++ b/src/DataBlobs/services/BlobEntry.jl @@ -111,7 +111,7 @@ function Base.show(io::IO, ::MIME"text/plain", entry::Blobentry) println(io, "Blobentry {") println(io, " blobid: ", entry.blobid) println(io, " label: ", entry.label) - println(io, " blobstore: ", entry.blobstore) + println(io, " storelabel: ", entry.storelabel) println(io, " origin: ", entry.origin) println(io, " description: ", entry.description) println(io, " mimetype: ", entry.mimetype) @@ -255,8 +255,12 @@ function addFactorBlobentries!(dfg::AbstractDFG, fLbl::Symbol, entries::Vector{B return entries end -function addAgentBlobentries!(dfg::AbstractDFG, entries::Vector{Blobentry}) - addAgentBlobentry!.(dfg, entries) +function addAgentBlobentries!( + dfg::AbstractDFG, + agentlabel::Symbol, + entries::Vector{Blobentry}, +) + addAgentBlobentry!.(dfg, agentlabel, entries) return entries end @@ -277,8 +281,12 @@ function mergeFactorBlobentries!(dfg::AbstractDFG, fLbl::Symbol, entries::Vector mergeFactorBlobentry!.(dfg, fLbl, entries) return length(entries) end -function mergeAgentBlobentries!(dfg::AbstractDFG, entries::Vector{Blobentry}) - mergeAgentBlobentry!.(dfg, entries) +function mergeAgentBlobentries!( + dfg::AbstractDFG, + agentlabel::Symbol, + entries::Vector{Blobentry}, +) + mergeAgentBlobentry!.(dfg, agentlabel, entries) return length(entries) end function mergeGraphBlobentries!(dfg::AbstractDFG, entries::Vector{Blobentry}) @@ -308,9 +316,13 @@ function deleteFactorBlobentries!( return sum(cnts) end -function deleteAgentBlobentries!(dfg::AbstractDFG, labels::Vector{Symbol}) +function deleteAgentBlobentries!( + dfg::AbstractDFG, + agentlabel::Symbol, + labels::Vector{Symbol}, +) cnts = map(labels) do label - return deleteAgentBlobentry!(dfg, label) + return deleteAgentBlobentry!(dfg, agentlabel, label) end return sum(cnts) end diff --git a/src/DataBlobs/services/BlobStores.jl b/src/DataBlobs/services/BlobStores.jl index c7b2ce36..5dbc12bd 100644 --- a/src/DataBlobs/services/BlobStores.jl +++ b/src/DataBlobs/services/BlobStores.jl @@ -3,7 +3,7 @@ ##============================================================================== """ -Get the data blob for the specified blobstore or dfg. +Get the data blob for the specified Blobstore or DFG. Related [`getBlobentry`](@ref) @@ -15,7 +15,7 @@ $(METHODLIST) function getBlob end """ -Adds a blob to the blob store or dfg with the blobid. +Adds a blob to the Blobstore with the blobid. Related [`addBlobentry!`](@ref) @@ -53,9 +53,9 @@ function hasBlob end ##============================================================================== ## AbstractBlobstore derived CRUD for Blob ##============================================================================== -#TODO maybe we should generalize and move the cached blobstore to DFG. +#TODO maybe we should generalize and move the cached Blobstore to DFG. function getBlob(dfg::AbstractDFG, entry::Blobentry) - storeLabel = entry.blobstore + storeLabel = entry.storelabel store = getBlobstore(dfg, storeLabel) return getBlob(store, entry.blobid) end @@ -66,7 +66,7 @@ end #add function addBlob!(dfg::AbstractDFG, entry::Blobentry, data) - return addBlob!(getBlobstore(dfg, entry.blobstore), entry, data) + return addBlob!(getBlobstore(dfg, entry.storelabel), entry, data) end function addBlob!(store::AbstractBlobstore{T}, entry::Blobentry, data::T) where {T} @@ -78,7 +78,7 @@ addBlob!(store::AbstractBlobstore, data) = addBlob!(store, uuid4(), data) #delete function deleteBlob!(dfg::AbstractDFG, entry::Blobentry) - return deleteBlob!(getBlobstore(dfg, entry.blobstore), entry) + return deleteBlob!(getBlobstore(dfg, entry.storelabel), entry) end function deleteBlob!(store::AbstractBlobstore, entry::Blobentry) @@ -90,7 +90,7 @@ function hasBlob(store::AbstractBlobstore, entry::Blobentry) return hasBlob(store, entry.blobid) end function hasBlob(dfg::AbstractDFG, entry::Blobentry) - return hasBlob(getBlobstore(dfg, entry.blobstore), entry.blobid) + return hasBlob(getBlobstore(dfg, entry.storelabel), entry.blobid) end #TODO @@ -123,7 +123,7 @@ end FolderStore(label::Symbol, folder::String) = FolderStore{Vector{UInt8}}(label, folder) -function FolderStore(foldername::String; label::Symbol = :default, createfolder = true) +function FolderStore(foldername::String; label::Symbol = :primary, createfolder = true) storepath = expanduser(joinpath(foldername, string(label))) if createfolder && !isdir(storepath) @info "Folder '$storepath' doesn't exist - creating." @@ -212,7 +212,7 @@ end function InMemoryBlobstore{T}(storeKey::Symbol) where {T} return InMemoryBlobstore{T}(storeKey, Dict{UUID, T}()) end -function InMemoryBlobstore(storeKey::Symbol = :default_inmemory_store) +function InMemoryBlobstore(storeKey::Symbol = :primary) return InMemoryBlobstore{Vector{UInt8}}(storeKey) end diff --git a/src/DataBlobs/services/BlobWrappers.jl b/src/DataBlobs/services/BlobWrappers.jl index 63cf55fa..058d4017 100644 --- a/src/DataBlobs/services/BlobWrappers.jl +++ b/src/DataBlobs/services/BlobWrappers.jl @@ -113,10 +113,10 @@ function saveBlob_Variable!( variable_label::Symbol, blob::Vector{UInt8}, entry_label::Symbol, - blobstore::Symbol = :default; + storelabel::Symbol = :primary; blobentry_kwargs..., ) - entry = Blobentry(entry_label, blobstore; blobentry_kwargs...) + entry = Blobentry(entry_label, storelabel; blobentry_kwargs...) return saveBlob_Variable!(dfg, variable_label, blob, entry) end @@ -143,10 +143,10 @@ function saveBlob_Graph!( dfg::AbstractDFG, blob::Vector{UInt8}, entry_label::Symbol, - blobstore::Symbol = :default; + storelabel::Symbol = :primary; blobentry_kwargs..., ) - entry = Blobentry(entry_label, blobstore; blobentry_kwargs...) + entry = Blobentry(entry_label, storelabel; blobentry_kwargs...) return saveBlob_Graph!(dfg, blob, entry) end @@ -157,32 +157,38 @@ function deleteBlob_Graph!(dfg::AbstractDFG, entry_label::Symbol) return 2 end -function loadBlob_Agent(dfg::AbstractDFG, entry_label::Symbol;) - entry = getAgentBlobentry(dfg, entry_label) +function loadBlob_Agent(dfg::AbstractDFG, agentlabel::Symbol, entry_label::Symbol;) + entry = getAgentBlobentry(dfg, agentlabel, entry_label) blob = getBlob(dfg, entry) return entry, blob end -function saveBlob_Agent!(dfg::AbstractDFG, blob::Vector{UInt8}, entry::Blobentry) - addAgentBlobentry!(dfg, entry) +function saveBlob_Agent!( + dfg::AbstractDFG, + agentlabel::Symbol, + blob::Vector{UInt8}, + entry::Blobentry, +) + addAgentBlobentry!(dfg, agentlabel, entry) addBlob!(dfg, entry, blob) return entry end function saveBlob_Agent!( dfg::AbstractDFG, + agentlabel::Symbol, blob::Vector{UInt8}, entry_label::Symbol, - blobstore::Symbol = :default; + storelabel::Symbol = :primary; blobentry_kwargs..., ) - entry = Blobentry(entry_label, blobstore; blobentry_kwargs...) - return saveBlob_Agent!(dfg, blob, entry) + entry = Blobentry(entry_label, storelabel; blobentry_kwargs...) + return saveBlob_Agent!(dfg, agentlabel, blob, entry) end -function deleteBlob_Agent!(dfg::AbstractDFG, entry_label::Symbol) - entry = getAgentBlobentry(dfg, entry_label) - deleteAgentBlobentry!(dfg, entry_label) +function deleteBlob_Agent!(dfg::AbstractDFG, agentlabel::Symbol, entry_label::Symbol) + entry = getAgentBlobentry(dfg, agentlabel, entry_label) + deleteAgentBlobentry!(dfg, agentlabel, entry_label) deleteBlob!(dfg, entry) return 2 end @@ -209,10 +215,10 @@ function saveBlob_Factor!( factor_label::Symbol, blob::Vector{UInt8}, entry_label::Symbol, - blobstore::Symbol = :default; + storelabel::Symbol = :primary; blobentry_kwargs..., ) - entry = Blobentry(entry_label, blobstore; blobentry_kwargs...) + entry = Blobentry(entry_label, storelabel; blobentry_kwargs...) return saveBlob_Factor!(dfg, factor_label, blob, entry) end @@ -228,7 +234,7 @@ function saveImage_Variable!( variable_label::Symbol, img::AbstractMatrix, entry_label::Symbol, - blobstore::Symbol = :default; + storelabel::Symbol = :primary; entry_kwargs..., ) mimetype = get(entry_kwargs, :mimeType, MIME("image/png")) @@ -239,7 +245,7 @@ function saveImage_Variable!( entry = Blobentry( entry_label, - blobstore; + storelabel; blobid = uuid4(), entry_kwargs..., size = length(blob), diff --git a/src/Deprecated.jl b/src/Deprecated.jl index 05338515..80580ebf 100644 --- a/src/Deprecated.jl +++ b/src/Deprecated.jl @@ -117,7 +117,7 @@ function _getDuplicatedEmptyDFG( ) where {P <: AbstractDFGParams, V <: AbstractGraphVariable, F <: AbstractGraphFactor} Base.depwarn("_getDuplicatedEmptyDFG is deprecated.", :_getDuplicatedEmptyDFG) newDfg = GraphsDFG{P, V, F}(; - agentLabel = getAgentLabel(dfg), + agents = deepcopy(dfg.agents), graphLabel = getGraphLabel(dfg), solverParams = deepcopy(dfg.solverParams), ) @@ -184,7 +184,9 @@ end # Function to generate source string - agentLabel|graphLabel|varLabel # """ function buildSourceString(dfg::AbstractDFG, label::Symbol) - return "$(getAgentLabel(dfg))|$(getGraphLabel(dfg))|$label" + return error( + "buildSourceString is deprecated. Use agents with `listAgents(dfg)` and `getAgent(dfg, agentlabel)` instead.", + ) end getAgentMetadata(args...) = error("getAgentMetadata is obsolete, use Bloblets instead.") @@ -572,7 +574,7 @@ function deepcopyGraph( destDFG = T(; solverParams = getSolverParams(sourceDFG), graph = sourceDFG.graph, - agent = sourceDFG.agent, + agents = deepcopy(sourceDFG.agents), graphLabel, ) copyGraph!( diff --git a/src/DistributedFactorGraphs.jl b/src/DistributedFactorGraphs.jl index fa225ddb..4b2f535c 100644 --- a/src/DistributedFactorGraphs.jl +++ b/src/DistributedFactorGraphs.jl @@ -392,11 +392,16 @@ export loadDFG ## Other utility functions ##------------------------------------------------------------------------------ +## Agent CRUD (now in AbstractDFG services) +export addAgent! +export deleteAgent! +export listAgents +export getAgent +export hasAgent +# export mergeAgent! +# public refAgents + ## TODO maybe move to DFG from SDK -# addAgent! -# deleteAgent! -# listAgents -# getAgents # addGraph! # deleteGraph! # listGraphs @@ -445,7 +450,6 @@ const unstable_functions::Vector{Symbol} = [ :findVariablesNearTimestamp, :findShortestPathDijkstra, :findFactorsBetweenNaive, # TODO not really used - :getAgentLabel, #TODO check and mark as public :getGraphLabel, #TODO check and mark as public :getDescription, :getSolverParams, diff --git a/src/FileDFG/services/FileDFG.jl b/src/FileDFG/services/FileDFG.jl index 01cdbfeb..bbb738f8 100644 --- a/src/FileDFG/services/FileDFG.jl +++ b/src/FileDFG/services/FileDFG.jl @@ -53,7 +53,7 @@ function saveDFG(folder::AbstractString, dfg::AbstractDFG) p = Progress(4; desc = "Saving DFG Nodes") JSON.json("$savepath/graphroot.json", dfg.graph; style = DFGJSONStyle()) next!(p) - JSON.json("$savepath/agent.json", dfg.agent; style = DFGJSONStyle()) + JSON.json("$savepath/agents.json", dfg.agents; style = DFGJSONStyle()) next!(p) JSON.json("$savepath/solverparams.json", dfg.solverParams; style = DFGJSONStyle()) next!(p) @@ -168,12 +168,24 @@ function loadDFG(file::AbstractString) # only extract the json files needed to rebuild DFG object tar_gz = open(file) tar = CodecZlib.GzipDecompressorStream(tar_gz) - dfgnodenames = r"^(agent\.json|blobstores\.json|graphroot\.json|solverparams\.json)$" + dfgnodenames = r"^(agents?\.json|blobstores\.json|graphroot\.json|solverparams\.json)$" loaddir = Tar.extract(hdr -> contains(hdr.path, dfgnodenames), tar) close(tar) progess = Progress(4; desc = "Loading DFG Nodes") - agent = JSON.parsefile(joinpath(loaddir, "agent.json"), Agent; style = DFGJSONStyle()) + agents = if isfile(joinpath(loaddir, "agents.json")) + JSON.parsefile( + joinpath(loaddir, "agents.json"), + OrderedDict{Symbol, Agent}; + style = DFGJSONStyle(), + ) + elseif isfile(joinpath(loaddir, "agent.json")) + # backward compat: load old single-agent format + agent = JSON.parsefile(joinpath(loaddir, "agent.json"), Agent; style = DFGJSONStyle()) + OrderedDict{Symbol, Agent}(agent.label => agent) + else + OrderedDict{Symbol, Agent}() + end next!(progess) graph = JSON.parsefile( joinpath(loaddir, "graphroot.json"), @@ -194,7 +206,7 @@ function loadDFG(file::AbstractString) ) next!(progess) - dfg = GraphsDFG(; agent, graph, solverParams, blobstores) + dfg = GraphsDFG(; agents, graph, solverParams, blobstores) @debug "DFG.loadDFG is deleting a temp folder created during unzip, $loaddir" # cleanup temporary folder diff --git a/src/GraphsDFG/GraphsDFG.jl b/src/GraphsDFG/GraphsDFG.jl index ad5a67ac..2a76ee77 100644 --- a/src/GraphsDFG/GraphsDFG.jl +++ b/src/GraphsDFG/GraphsDFG.jl @@ -19,7 +19,7 @@ using ...DistributedFactorGraphs: filterDFG!, getSolvable, getStateKind, - getAgentLabel, + getAgent, getGraphLabel, isInitialized, Bloblets, diff --git a/src/GraphsDFG/entities/GraphsDFG.jl b/src/GraphsDFG/entities/GraphsDFG.jl index afa39f50..385e0819 100644 --- a/src/GraphsDFG/entities/GraphsDFG.jl +++ b/src/GraphsDFG/entities/GraphsDFG.jl @@ -12,14 +12,13 @@ mutable struct GraphsDFG{ F <: AbstractGraphFactor, } <: AbstractDFG{V, F} g::FactorGraph{Int, V, F} - # addHistory::Vector{Symbol} #TODO: Discuss more - is this an audit trail? - solverParams::T # Solver parameters - blobstores::Dict{Symbol, AbstractBlobstore} #TODO note v0.29 changed from camelCase blobStores + solverParams::T # Solver parameters #TODO resolve #1205 first + blobstores::OrderedDict{Symbol, AbstractBlobstore} #TODO note v0.29 changed from camelCase blobStores graph::Graphroot - agent::Agent + agents::OrderedDict{Symbol, Agent} #TODO note v0.29 added multiple agents, agent -> agents end -DFG.getAgent(dfg::GraphsDFG) = dfg.agent +DFG.refAgents(dfg::GraphsDFG) = dfg.agents DFG.getGraphLabel(dfg::GraphsDFG) = dfg.graph.label DFG.getDescription(dfg::GraphsDFG) = dfg.graph.description @@ -35,51 +34,45 @@ function GraphsDFG{T, V, F}( g::FactorGraph{Int, V, F} = FactorGraph{Int, V, F}(); # addHistory::Vector{Symbol} = Symbol[], solverParams::T = T(), - blobstores = Dict{Symbol, AbstractBlobstore}(), + blobstores = OrderedDict{Symbol, AbstractBlobstore}(), # graph - graphLabel::Symbol = Symbol("graph_", string(uuid4())[1:6]), + graphLabel::Symbol = :workspace, graphDescription::String = "", graphTags::Union{Set{Symbol}, Vector{Symbol}} = Set{Symbol}(), graphBloblets::Bloblets = Bloblets(), - graphBlobEntries = Blobentries(), + graphBlobentries = Blobentries(), graph::Graphroot = Graphroot( graphLabel, graphDescription, graphTags, graphBloblets, - graphBlobEntries, - ), - # agent - agentLabel::Symbol = :DefaultAgent, - agentDescription::String = "", - agentTags::Union{Set{Symbol}, Vector{Symbol}} = Set{Symbol}(), - agentBloblets::Bloblets = Bloblets(), - agentBlobEntries = Blobentries(), - agent::Agent = Agent( - agentLabel, - agentDescription, - agentTags, - agentBloblets, - agentBlobEntries, + graphBlobentries, ), + agents::OrderedDict{Symbol, Agent} = OrderedDict{Symbol, Agent}(), #TODO deprecated v0.29 graphMetadata = nothing, agentMetadata = nothing, + kwargs..., ) where {T <: AbstractDFGParams, V <: AbstractGraphVariable, F <: AbstractGraphFactor} - if !isnothing(graphMetadata) - @warn "The `graphMetadata` keyword argument is obsolete, use graphBloblets." - end - if !isnothing(agentMetadata) - @warn "The `agentMetadata` keyword argument is obsolete, use agentBloblets." - end + !isnothing(graphMetadata) && Base.depwarn( + "The `graphMetadata` keyword argument is obsolete, use graphBloblets.", + :GraphsDFG, + ) + !isnothing(agentMetadata) && Base.depwarn( + "The `agentMetadata` keyword argument is obsolete, use agentBloblets.", + :GraphsDFG, + ) + + !isempty(kwargs) && Base.depwarn( + "`agent...` keyword arguments are deprecated, use the `agents` kwargs or addAgent!", + :GraphsDFG, + ) - # Validate the graphLabel and agentLabel + #TODO move to Graphroot constructor. !DFG.isValidLabel(graphLabel) && throw(ArgumentError("'$graphLabel' is not a valid label")) - !DFG.isValidLabel(agentLabel) && - throw(ArgumentError("'$agentLabel' is not a valid label")) - return GraphsDFG{T, V, F}(g, solverParams, blobstores, graph, agent) + return GraphsDFG{T, V, F}(g, solverParams, blobstores, graph, agents) end # GraphsDFG{T}(; kwargs...) where T <: AbstractDFGParams = GraphsDFG{T,VariableDFG,FactorDFG}(;kwargs...) @@ -104,7 +97,7 @@ function GraphsDFG( solverParams = fg.solverParams, blobstores = fg.blobstores, graph = fg.graph, - agent = fg.agent, + agents = fg.agents, ) - return GraphsDFG(g, solverParams, blobstores, graph, agent) + return GraphsDFG(g, solverParams, blobstores, graph, agents) end diff --git a/src/GraphsDFG/services/GraphsDFG.jl b/src/GraphsDFG/services/GraphsDFG.jl index 90d885c1..03dde187 100644 --- a/src/GraphsDFG/services/GraphsDFG.jl +++ b/src/GraphsDFG/services/GraphsDFG.jl @@ -517,8 +517,8 @@ function listGraphBlobentries(fg::GraphsDFG; whereLabel::Union{Nothing, Function return labels end -function listAgentBlobentries(fg::GraphsDFG) - return collect(keys(fg.agent.blobentries)) +function listAgentBlobentries(fg::GraphsDFG, agentlabel::Symbol) + return collect(keys(fg.agents[agentlabel].blobentries)) end function addGraphBlobentry!(fg::GraphsDFG, entry::Blobentry) @@ -535,32 +535,37 @@ function addGraphBlobentries!(fg::GraphsDFG, entries::Vector{Blobentry}) end end -function DFG.addAgentBlobentry!(fg::GraphsDFG, entry::Blobentry) - if haskey(fg.agent.blobentries, entry.label) +function DFG.addAgentBlobentry!(fg::GraphsDFG, agentlabel::Symbol, entry::Blobentry) + if haskey(fg.agents[agentlabel].blobentries, entry.label) throw(LabelExistsError("Blobentry", entry.label)) end - push!(fg.agent.blobentries, entry.label => entry) + push!(fg.agents[agentlabel].blobentries, entry.label => entry) return entry end -function DFG.addAgentBlobentries!(fg::GraphsDFG, entries::Vector{Blobentry}) +function DFG.addAgentBlobentries!( + fg::GraphsDFG, + agentlabel::Symbol, + entries::Vector{Blobentry}, +) return map(entries) do entry - return addAgentBlobentry!(fg, entry) + return addAgentBlobentry!(fg, agentlabel, entry) end end -function DFG.getAgentBlobentry(fg::GraphsDFG, label::Symbol) - if !haskey(fg.agent.blobentries, label) +function DFG.getAgentBlobentry(fg::GraphsDFG, agentlabel::Symbol, label::Symbol) + if !haskey(fg.agents[agentlabel].blobentries, label) throw(LabelNotFoundError("Blobentry", label)) end - return fg.agent.blobentries[label] + return fg.agents[agentlabel].blobentries[label] end function DFG.getAgentBlobentries( - fg::GraphsDFG; + fg::GraphsDFG, + agentlabel::Symbol; whereLabel::Union{Nothing, Function} = nothing, ) - entries = collect(values(fg.agent.blobentries)) + entries = collect(values(fg.agents[agentlabel].blobentries)) filterDFG!(entries, whereLabel, getLabel) return entries end @@ -570,8 +575,8 @@ function DFG.mergeGraphBlobentry!(dfg::GraphsDFG, entry::Blobentry) return 1 end -function DFG.mergeAgentBlobentry!(dfg::GraphsDFG, entry::Blobentry) - DFG.refBlobentries(dfg.agent)[getLabel(entry)] = entry +function DFG.mergeAgentBlobentry!(dfg::GraphsDFG, agentlabel::Symbol, entry::Blobentry) + DFG.refBlobentries(dfg.agents[agentlabel])[getLabel(entry)] = entry return 1 end @@ -582,9 +587,13 @@ function DFG.mergeGraphBlobentries!(dfg::GraphsDFG, entries::Vector{Blobentry}) return sum(cnts) end -function DFG.mergeAgentBlobentries!(dfg::GraphsDFG, entries::Vector{Blobentry}) +function DFG.mergeAgentBlobentries!( + dfg::GraphsDFG, + agentlabel::Symbol, + entries::Vector{Blobentry}, +) cnts = map(entries) do entry - return mergeAgentBlobentry!(dfg, entry) + return mergeAgentBlobentry!(dfg, agentlabel, entry) end return sum(cnts) end @@ -631,9 +640,9 @@ function DFG.deleteGraphBlobentry!(dfg::GraphsDFG, label::Symbol) return 1 end -function DFG.deleteAgentBlobentry!(dfg::GraphsDFG, label::Symbol) - !haskey(dfg.agent.blobentries, label) && return 0 - delete!(dfg.agent.blobentries, label) +function DFG.deleteAgentBlobentry!(dfg::GraphsDFG, agentlabel::Symbol, label::Symbol) + !haskey(dfg.agents[agentlabel].blobentries, label) && return 0 + delete!(dfg.agents[agentlabel].blobentries, label) return 1 end @@ -641,6 +650,6 @@ function DFG.hasGraphBlobentry(dfg::GraphsDFG, label::Symbol) return haskey(dfg.graph.blobentries, label) end -function DFG.hasAgentBlobentry(dfg::GraphsDFG, label::Symbol) - return haskey(dfg.agent.blobentries, label) +function DFG.hasAgentBlobentry(dfg::GraphsDFG, agentlabel::Symbol, label::Symbol) + return haskey(dfg.agents[agentlabel].blobentries, label) end diff --git a/src/entities/Agent_and_Graph.jl b/src/entities/Agent_and_Graph.jl index e09ff59c..fa090c2e 100644 --- a/src/entities/Agent_and_Graph.jl +++ b/src/entities/Agent_and_Graph.jl @@ -15,6 +15,11 @@ end blobentries::Blobentries = Blobentries() end +#TODO +# Validate the graphLabel and agentLabel +# !DFG.isValidLabel(graphLabel) && +# throw(ArgumentError("'$graphLabel' is not a valid label")) + # Patching (for merge!) # standard patch description merging function patch(dest::String, src::String) diff --git a/src/errors.jl b/src/errors.jl index 7bae3e6b..fe36e754 100644 --- a/src/errors.jl +++ b/src/errors.jl @@ -107,3 +107,17 @@ end function Base.showerror(io::IO, ex::MergeConflictError) return print(io, "MergeConflictError: ", ex.msg) end + +""" + LinkConstraintError(msg) + +Error thrown when an operation violates a structural dependency, +e.g., deleting a graph that is still in use by a model. +""" +struct LinkConstraintError <: Exception + msg::String +end + +function Base.showerror(io::IO, ex::LinkConstraintError) + return print(io, "LinkConstraintError: ", ex.msg) +end diff --git a/src/services/AbstractDFG.jl b/src/services/AbstractDFG.jl index 194f46a1..c6ecb545 100644 --- a/src/services/AbstractDFG.jl +++ b/src/services/AbstractDFG.jl @@ -23,21 +23,11 @@ Base.Broadcast.broadcastable(dfg::AbstractDFG) = Ref(dfg) """ function getId end -""" - $(SIGNATURES) -""" -function getAgent end - """ $(SIGNATURES) """ function getGraph end -""" - $(SIGNATURES) -""" -getAgentLabel(dfg::AbstractDFG) = getLabel(getAgent(dfg)) - """ $(SIGNATURES) """ @@ -86,6 +76,88 @@ function setSolverParams!(dfg::AbstractDFG, solverParams::AbstractDFGParams) return dfg.solverParams = solverParams end +## ============================================================================= +## Agent in DFG CRUD +## ============================================================================= +""" + $(SIGNATURES) +""" +function refAgents end + +""" + $(SIGNATURES) +""" +function getAgent(dfg::AbstractDFG, label::Symbol) + agent = get(refAgents(dfg), label, nothing) + if isnothing(agent) + available = listAgents(dfg) + isempty(available) && + @info "No agents in graph: `$(getGraphLabel(dfg))`. Use `addAgent!(dfg, Agent(; label=:myAgent))` to add one" + throw(LabelNotFoundError("Agent", label, available)) + end + return agent +end + +function getAgents(dfg::AbstractDFG) + return map(listAgents(dfg)) do label + return getAgent(dfg, label) + end +end + +""" + $(SIGNATURES) +""" +function addAgent!(dfg::AbstractDFG, agent::Agent) + label = getLabel(agent) + haskey(refAgents(dfg), label) && throw(LabelExistsError("Agent", label)) + push!(refAgents(dfg), label => agent) + return agent +end + +""" + $(SIGNATURES) +""" +function deleteAgent!(dfg::AbstractDFG, label::Symbol) + !haskey(refAgents(dfg), label) && return 0 + pop!(refAgents(dfg), label) + return 1 +end + +""" + $(SIGNATURES) +""" +function listAgents(dfg::AbstractDFG) + return collect(keys(refAgents(dfg))) +end + +""" + $(SIGNATURES) +""" +function hasAgent(dfg::AbstractDFG, label::Symbol) + return haskey(refAgents(dfg), label) +end + +""" + $(SIGNATURES) +""" +function mergeAgent!(dfg::AbstractDFG, agent::Agent) + label = getLabel(agent) + if hasAgent(dfg, label) + mergeAgent!(refAgents(dfg)[label], agent) + else + addAgent!(dfg, agent) + end + return 1 +end + +function mergeAgents!(dfg::AbstractDFG, agents::Vector{Agent}) + count = 0 + for agent in agents + count += mergeAgent!(dfg, agent) + end + return count +end + ##============================================================================== ## AbstractBlobstore CRUD ##============================================================================== @@ -96,6 +168,8 @@ refBlobstores(dfg::AbstractDFG) = dfg.blobstores function getBlobstore(dfg::AbstractDFG, storeLabel::Symbol) store = get(refBlobstores(dfg), storeLabel, nothing) if isnothing(store) + isempty(refBlobstores(dfg)) && + @info "No blobstores in graph: `$(getGraphLabel(dfg))`. Use `addBlobstore!(dfg, FolderStore(\"path/to/store\"))` to add one." throw( LabelNotFoundError("Blobstore", storeLabel, collect(keys(refBlobstores(dfg)))), ) @@ -596,17 +670,17 @@ end $(SIGNATURES) Merge the source DFG into the destination DFG cascading down the hierarchy of DFG nodes. Merge rules: -- Variables, Factors, Agent, and Graphroot, with the same label are merged if they are equal. +- Variables, Factors, Agents, and Graphroot, with the same label are merged if they are equal. - On conflicts, a `MergeConflictError` is thrown. - Child nodes (eg. tags, Bloblets, Blobentries, States, etc.) are using `merge!`. -- The Blobstore links are merged provided they point to the same blobstore. +- The Blobstore links are merged provided they point to the same Blobstore. - On conflicts, a `MergeConflictError` is thrown. """ function mergeGraph!(destDFG::AbstractDFG, srcDFG::AbstractDFG) patch!(destDFG.graph, srcDFG.graph) mergeVariables!(destDFG, getVariables(srcDFG)) mergeFactors!(destDFG, getFactors(srcDFG)) - mergeAgent!(destDFG.agent, srcDFG.agent) + mergeAgents!(destDFG, getAgents(srcDFG)) mergeStorelinks!(destDFG, getBlobstores(srcDFG)) return destDFG end @@ -694,7 +768,7 @@ function getSummaryGraph(dfg::G) where {G <: AbstractDFG} #TODO fix deprecated constructor summaryDfg = GraphsDFG{NoSolverParams, VariableSummary, FactorSummary}(; graphDescription = "Summary of $(getDescription(dfg))", - agent = dfg.agent, + agents = deepcopy(dfg.agents), graphLabel = Symbol(getGraphLabel(dfg), "_summary_$(string(uuid4())[1:6])"), ) deepcopyGraph!(summaryDfg, dfg) diff --git a/src/services/Bloblet.jl b/src/services/Bloblet.jl index fe2a3413..e20e390c 100644 --- a/src/services/Bloblet.jl +++ b/src/services/Bloblet.jl @@ -155,49 +155,63 @@ end """ $(SIGNATURES) """ -getAgentBloblet(dfg::GraphsDFG, label::Symbol) = getBloblet(dfg.agent, label) +function getAgentBloblet(dfg::GraphsDFG, agentlabel::Symbol, label::Symbol) + return getBloblet(getAgent(dfg, agentlabel), label) +end """ $(SIGNATURES) """ -addAgentBloblet!(dfg::GraphsDFG, bloblet::Bloblet) = addBloblet!(dfg.agent, bloblet) +function addAgentBloblet!(dfg::GraphsDFG, agentlabel::Symbol, bloblet::Bloblet) + return addBloblet!(getAgent(dfg, agentlabel), bloblet) +end """ $(SIGNATURES) """ -mergeAgentBloblet!(dfg::GraphsDFG, bloblet::Bloblet) = mergeBloblet!(dfg.agent, bloblet) +function mergeAgentBloblet!(dfg::GraphsDFG, agentlabel::Symbol, bloblet::Bloblet) + return mergeBloblet!(getAgent(dfg, agentlabel), bloblet) +end """ $(SIGNATURES) """ -deleteAgentBloblet!(dfg::GraphsDFG, label::Symbol) = deleteBloblet!(dfg.agent, label) +function deleteAgentBloblet!(dfg::GraphsDFG, agentlabel::Symbol, label::Symbol) + return deleteBloblet!(getAgent(dfg, agentlabel), label) +end """ $(SIGNATURES) """ -getAgentBloblets(dfg::GraphsDFG) = getBloblets(dfg.agent) +function getAgentBloblets(dfg::GraphsDFG, agentlabel::Symbol) + return getBloblets(getAgent(dfg, agentlabel)) +end """ $(SIGNATURES) """ -function addAgentBloblets!(dfg::GraphsDFG, bloblets::Vector{Bloblet}) - return addBloblets!(dfg.agent, bloblets) +function addAgentBloblets!(dfg::GraphsDFG, agentlabel::Symbol, bloblets::Vector{Bloblet}) + return addBloblets!(getAgent(dfg, agentlabel), bloblets) end """ $(SIGNATURES) """ -function mergeAgentBloblets!(dfg::GraphsDFG, bloblets::Vector{Bloblet}) - return mergeBloblets!(dfg.agent, bloblets) +function mergeAgentBloblets!(dfg::GraphsDFG, agentlabel::Symbol, bloblets::Vector{Bloblet}) + return mergeBloblets!(getAgent(dfg, agentlabel), bloblets) end """ $(SIGNATURES) """ -function deleteAgentBloblets!(dfg::GraphsDFG, labels::Vector{Symbol}) - return deleteBloblets!(dfg.agent, labels) +function deleteAgentBloblets!(dfg::GraphsDFG, agentlabel::Symbol, labels::Vector{Symbol}) + return deleteBloblets!(getAgent(dfg, agentlabel), labels) end """ $(SIGNATURES) """ -listAgentBloblets(dfg::GraphsDFG) = listBloblets(dfg.agent) +function listAgentBloblets(dfg::GraphsDFG, agentlabel::Symbol) + return listBloblets(getAgent(dfg, agentlabel)) +end """ $(SIGNATURES) """ -hasAgentBloblet(dfg::GraphsDFG, label::Symbol) = hasBloblet(dfg.agent, label) +function hasAgentBloblet(dfg::GraphsDFG, agentlabel::Symbol, label::Symbol) + return hasBloblet(getAgent(dfg, agentlabel), label) +end ##============================================================================== ## Graph Bloblets diff --git a/src/services/CompareUtils.jl b/src/services/CompareUtils.jl index b0ccb3b0..f6b45531 100644 --- a/src/services/CompareUtils.jl +++ b/src/services/CompareUtils.jl @@ -539,7 +539,7 @@ function compareFactorGraphs( :fifo, :solverParams, :factorOperationalMemoryType, - :agent, + :agents, :graph, ] skiplist = union(skiplist, skip) diff --git a/src/services/CustomPrinting.jl b/src/services/CustomPrinting.jl index f1203654..cf675271 100644 --- a/src/services/CustomPrinting.jl +++ b/src/services/CustomPrinting.jl @@ -186,12 +186,12 @@ end function Base.show(io::IO, ::MIME"text/plain", dfg::AbstractDFG) summary(io, dfg) println(io) - println(io, " AgentLabel: ", getAgentLabel(dfg)) println(io, " GraphLabel: ", getGraphLabel(dfg)) println(io, " Description: ", getDescription(dfg)) println(io, " Nr variables: ", length(ls(dfg))) println(io, " Nr factors: ", length(lsf(dfg))) - println(io, " Agent Bloblets: ", listAgentBloblets(dfg)) println(io, " Graph Bloblets: ", listGraphBloblets(dfg)) + println(io, " Agents: ", listAgents(dfg)) + println(io, " Blobstores: ", listBlobstores(dfg)) return end diff --git a/src/services/Tags.jl b/src/services/Tags.jl index 7623f908..5c376740 100644 --- a/src/services/Tags.jl +++ b/src/services/Tags.jl @@ -54,8 +54,8 @@ function listGraphTags(dfg::InMemoryDFGTypes) return listTags(dfg.graph) end -function listAgentTags(dfg::InMemoryDFGTypes) - return listTags(dfg.agent) +function listAgentTags(dfg::InMemoryDFGTypes, agentlabel::Symbol) + return listTags(getAgent(dfg, agentlabel)) end # function mergeVariableTags!(dfg::AbstractDFG, sym::Symbol, tags) @@ -85,8 +85,8 @@ function mergeGraphTags!(dfg::InMemoryDFGTypes, tags) return length(tags) end -function mergeAgentTags!(dfg::InMemoryDFGTypes, tags) - mergeTags!(dfg.agent, tags) +function mergeAgentTags!(dfg::InMemoryDFGTypes, agentlabel::Symbol, tags) + mergeTags!(getAgent(dfg, agentlabel), tags) return length(tags) end @@ -105,8 +105,8 @@ function deleteGraphTags!(dfg::InMemoryDFGTypes, tags) return length(tags) end -function deleteAgentTags!(dfg::InMemoryDFGTypes, tags) - deleteTags!(dfg.agent, tags) +function deleteAgentTags!(dfg::InMemoryDFGTypes, agentlabel::Symbol, tags) + deleteTags!(getAgent(dfg, agentlabel), tags) return length(tags) end @@ -124,8 +124,8 @@ function hasGraphTags(dfg::AbstractDFG, tags::Vector{Symbol}) return tags ⊆ listGraphTags(dfg) end -function hasAgentTags(dfg::AbstractDFG, tags::Vector{Symbol}) - return tags ⊆ listAgentTags(dfg) +function hasAgentTags(dfg::AbstractDFG, agentlabel::Symbol, tags::Vector{Symbol}) + return tags ⊆ listAgentTags(dfg, agentlabel) end ## diff --git a/test/consol_DataEntryBlobTests.jl b/test/consol_DataEntryBlobTests.jl index 748b02f6..a2333021 100644 --- a/test/consol_DataEntryBlobTests.jl +++ b/test/consol_DataEntryBlobTests.jl @@ -111,7 +111,7 @@ DFG.deleteBlob_Variable!(dfg, :x2, :random) #test default folder store dfs = FolderStore("/tmp/defaultfolderstore") @test dfs.folder == "/tmp/defaultfolderstore" -@test getLabel(dfs) == :default +@test getLabel(dfs) == :primary @test dfs isa FolderStore{Vector{UInt8}} ##============================================================================== @@ -122,14 +122,14 @@ dfs = FolderStore("/tmp/defaultfolderstore") ds = InMemoryBlobstore() addBlobstore!(dfg, ds) -ade = DFG.saveBlob_Variable!(dfg, :x1, dataset1, :random, :default_inmemory_store) +ade = DFG.saveBlob_Variable!(dfg, :x1, dataset1, :random, :primary) gde, gdb = DFG.loadBlob_Variable(dfg, :x1, :random) @test DFG.deleteBlob_Variable!(dfg, :x1, :random) == 2 @test ade == gde @test dataset1 == gdb -ade2 = DFG.saveBlob_Variable!(dfg, :x2, dataset1, :random, :default_inmemory_store) +ade2 = DFG.saveBlob_Variable!(dfg, :x2, dataset1, :random, :primary) # ade3,adb3 = updateBlob!(dfg, :x2, deepcopy(ade), dataset1) @test hasBlob(dfg, ade2) diff --git a/test/fileDFGTests.jl b/test/fileDFGTests.jl index fe402d59..ffe565cd 100644 --- a/test/fileDFGTests.jl +++ b/test/fileDFGTests.jl @@ -28,7 +28,7 @@ using UUIDs Blobentry(; blobid = uuid4(), label = :testing, - blobstore = :store, + storelabel = :store, # timestamp = now(localzone()), timestamp = DFG.TimeDateZone(now(localzone())), ), @@ -41,7 +41,7 @@ using UUIDs Blobentry(; blobid = uuid4(), label = :testing2, - blobstore = :store, + storelabel = :store, # timestamp = ZonedDateTime(2014, 5, 30, 21, tz"UTC-4"), timestamp = DFG.TimeDateZone(ZonedDateTime(2014, 5, 30, 21, tz"UTC-4")), ), @@ -79,18 +79,21 @@ using UUIDs be = Blobentry(; blobid = uuid4(), label = :testing2, - blobstore = :store, + storelabel = :store, # timestamp = ZonedDateTime(2023, 2, 3, 20, tz"UTC+1"), timestamp = DFG.TimeDateZone(ZonedDateTime(2023, 2, 3, 20, tz"UTC+1")), ) addGraphBlobentry!(dfg, be) + agentlabel = :testFileAgent + !DFG.hasAgent(dfg, agentlabel) && + DFG.addAgent!(dfg, DFG.Agent(; label = agentlabel)) agentBloblets = [Bloblet(:a, "43"), Bloblet(:b, "small_robot")] graphBloblets = [Bloblet(:a, "44"), Bloblet(:b, "small_session")] - DFG.addAgentBloblets!(dfg, agentBloblets) - DFG.addGraphBloblets!(dfg, graphBloblets) + DFG.mergeAgentBloblets!(dfg, agentlabel, agentBloblets) + DFG.mergeGraphBloblets!(dfg, graphBloblets) # Save and load the graph to test. saveDFG(filename, dfg) diff --git a/test/iifInterfaceTests.jl b/test/iifInterfaceTests.jl index ab94a7e8..391b4200 100644 --- a/test/iifInterfaceTests.jl +++ b/test/iifInterfaceTests.jl @@ -199,23 +199,25 @@ end @test_throws LabelNotFoundError isInitialized(v2, :second) # Graph and Agent small data tests + agentlabel = :testAgent + DFG.addAgent!(dfg, DFG.Agent(; label = agentlabel)) agentBloblets = [Bloblet(:a, "43"), Bloblet(:b, "Hello")] graphBloblets = [Bloblet(:c, "44"), Bloblet(:d, "Hello")] - DFG.addAgentBloblets!(dfg, agentBloblets) + DFG.addAgentBloblets!(dfg, agentlabel, agentBloblets) DFG.addGraphBloblets!(dfg, graphBloblets) - @test DFG.listAgentBloblets(dfg) == [:a, :b] + @test DFG.listAgentBloblets(dfg, agentlabel) == [:a, :b] @test DFG.listGraphBloblets(dfg) == [:c, :d] end @testset "Data Entries" begin - de1 = Blobentry(; blobid = uuid4(), label = :key1, blobstore = :test) + de1 = Blobentry(; blobid = uuid4(), label = :key1, storelabel = :test) - de2 = Blobentry(; blobid = uuid4(), label = :key2, blobstore = :test) + de2 = Blobentry(; blobid = uuid4(), label = :key2, storelabel = :test) de2_update = Blobentry(; blobid = uuid4(), label = :key2, - blobstore = :test, + storelabel = :test, mimetype = MIME("image/jpg"), ) diff --git a/test/testBlobStoresAndWrappers.jl b/test/testBlobStoresAndWrappers.jl index a7d60fc7..0668b745 100644 --- a/test/testBlobStoresAndWrappers.jl +++ b/test/testBlobStoresAndWrappers.jl @@ -161,13 +161,13 @@ end ##============================================================================== @testset "loadBlob/saveBlob/deleteBlob Factor" begin dfg, _, _ = connectivityTestGraph(testDFGAPI, VariableDFG, FactorDFG) - ds = InMemoryBlobstore(:default) + ds = InMemoryBlobstore(:primary) addBlobstore!(dfg, ds) dataset = rand(UInt8, 200) # saveBlob_Factor! with entry_label - entry = DFG.saveBlob_Factor!(dfg, :x1x2f1, dataset, :factor_data, :default) + entry = DFG.saveBlob_Factor!(dfg, :x1x2f1, dataset, :factor_data, :primary) @test entry isa Blobentry @test entry.label == :factor_data @@ -177,7 +177,7 @@ end @test loaded_blob == dataset # saveBlob_Factor! with explicit Blobentry - entry2 = Blobentry(:factor_data_2, :default) + entry2 = Blobentry(:factor_data_2, :primary) DFG.saveBlob_Factor!(dfg, :x1x2f1, dataset, entry2) loaded_entry2, loaded_blob2 = DFG.loadBlob_Factor(dfg, :x1x2f1, :factor_data_2) @test loaded_entry2.label == :factor_data_2 @@ -188,7 +188,7 @@ end @test !hasFactorBlobentry(dfg, :x1x2f1, :factor_data) # Multiple factors can have blobs - DFG.saveBlob_Factor!(dfg, :x2x3f1, dataset, :another_blob, :default) + DFG.saveBlob_Factor!(dfg, :x2x3f1, dataset, :another_blob, :primary) e, b = DFG.loadBlob_Factor(dfg, :x2x3f1, :another_blob) @test b == dataset @test DFG.deleteBlob_Factor!(dfg, :x2x3f1, :another_blob) == 2 @@ -202,7 +202,7 @@ end ##============================================================================== @testset "loadBlob/saveBlob/deleteBlob Variable (expanded)" begin dfg, _, _ = connectivityTestGraph(testDFGAPI, VariableDFG, FactorDFG) - ds = InMemoryBlobstore(:default) + ds = InMemoryBlobstore(:primary) addBlobstore!(dfg, ds) dataset = rand(UInt8, 300) @@ -213,7 +213,7 @@ end :x1, dataset, :var_blob, - :default; + :primary; description = "test blob", ) @test entry.label == :var_blob @@ -225,7 +225,7 @@ end @test loaded_blob == dataset # saveBlob_Variable! with explicit Blobentry - entry2 = Blobentry(:var_blob_2, :default) + entry2 = Blobentry(:var_blob_2, :primary) DFG.saveBlob_Variable!(dfg, :x1, dataset, entry2) _, blob2 = DFG.loadBlob_Variable(dfg, :x1, :var_blob_2) @test blob2 == dataset @@ -240,7 +240,7 @@ end ##============================================================================== @testset "saveImage_Variable! / loadImage_Variable" begin dfg, _, _ = connectivityTestGraph(testDFGAPI, VariableDFG, FactorDFG) - ds = InMemoryBlobstore(:default) + ds = InMemoryBlobstore(:primary) addBlobstore!(dfg, ds) # Create a small test "image" (matrix of floats, like a grayscale image) @@ -249,7 +249,7 @@ end # We test that the interface works by checking that it calls through correctly # For a real image test we'd need ImageIO/PNGFiles, so test the error path img = rand(Float64, 4, 4) - entry = DFG.saveImage_Variable!(dfg, :x1, img, :test_img, :default) + entry = DFG.saveImage_Variable!(dfg, :x1, img, :test_img, :primary) @test entry.label == :test_img @test entry.mimetype == MIME("image/png") @@ -257,7 +257,7 @@ end # (tests the dispatch path through unpackBlob(entry, blob)) json_str = """{"px":[1,2,3]}""" blob, _ = DFG.packBlob(format"JSON", json_str) - entry = Blobentry(:json_as_img, :default; mimetype = MIME("application/json")) + entry = Blobentry(:json_as_img, :primary; mimetype = MIME("application/json")) DFG.saveBlob_Variable!(dfg, :x1, blob, entry) loaded_entry, loaded_data = DFG.loadImage_Variable(dfg, :x1, :json_as_img) @test loaded_entry.label == :json_as_img diff --git a/test/testBlocks.jl b/test/testBlocks.jl index 5d4c546b..c4a7bf18 100644 --- a/test/testBlocks.jl +++ b/test/testBlocks.jl @@ -77,8 +77,8 @@ function DFGStructureAndAccessors( fg = T(; solverParams = solparams) #TODO test something better @test isa(fg, T) - @test getAgentLabel(fg) == :DefaultAgent - @test string(getGraphLabel(fg))[1:6] == "graph_" + @test isempty(listAgents(fg)) + @test getGraphLabel(fg) == :workspace # Test the validation of the robot, session, and user IDs. notAllowedList = [ @@ -94,7 +94,6 @@ function DFGStructureAndAccessors( for s in notAllowedList @test_throws ArgumentError T(solverParams = solparams, graphLabel = s) - @test_throws ArgumentError T(solverParams = solparams, agentLabel = s) end des = "description for runtest" @@ -104,23 +103,21 @@ function DFGStructureAndAccessors( sd = DFG.Bloblets(:sd => DFG.Bloblet(:sd, "sdEntry")) fg = T(; graphDescription = des, - agentLabel = rId, graphLabel = sId, - agentBloblets = rd, graphBloblets = sd, solverParams = solparams, ) + addAgent!(fg, Agent(; label = rId, bloblets = rd)) # accesssors # get @test getDescription(fg) == des - @test getAgentLabel(fg) == rId @test getGraphLabel(fg) == sId @test getSolverParams(fg) == NoSolverParams() #FIXME test bloblets - @test DFG.getAgentBloblet(fg, :rd) == rd[:rd] + @test DFG.getAgentBloblet(fg, rId, :rd) == rd[:rd] @test DFG.getGraphBloblet(fg, :sd) == sd[:sd] # NOTE see note in AbstractDFG.jl setSolverParams! @@ -141,17 +138,23 @@ end # User, Robot, Session Data function GraphAgentBloblets!(fg::AbstractDFG) + # First add an agent to test with + agentlabel = :testAgent + addAgent!(fg, Agent(; label = agentlabel)) + # Agent-level bloblets agent_blob = Bloblet(:agent_blob, "ready") - @test addAgentBloblet!(fg, agent_blob) == agent_blob - @test getAgentBloblet(fg, agent_blob.label) == agent_blob + @test addAgentBloblet!(fg, agentlabel, agent_blob) == agent_blob + @test getAgentBloblet(fg, agentlabel, agent_blob.label) == agent_blob updated_agent_blob = Bloblet(agent_blob.label, "updated") - @test mergeAgentBloblet!(fg, updated_agent_blob) == 1 - @test getAgentBloblet(fg, agent_blob.label) == updated_agent_blob - @test agent_blob.label in listAgentBloblets(fg) - @test deleteAgentBloblet!(fg, agent_blob.label) == 1 - @test_throws DFG.LabelNotFoundError getAgentBloblet(fg, agent_blob.label) - @test deleteAgentBloblet!(fg, agent_blob.label) == 0 + @test mergeAgentBloblet!(fg, agentlabel, updated_agent_blob) == 1 + @test getAgentBloblet(fg, agentlabel, agent_blob.label) == updated_agent_blob + @test agent_blob.label in listAgentBloblets(fg, agentlabel) + @test deleteAgentBloblet!(fg, agentlabel, agent_blob.label) == 1 + @test_throws DFG.LabelNotFoundError getAgentBloblet(fg, agentlabel, agent_blob.label) + @test deleteAgentBloblet!(fg, agentlabel, agent_blob.label) == 0 + + deleteAgent!(fg, agentlabel) # Graph-level bloblets graph_blob = Bloblet(:graph_blob, "running") @@ -168,26 +171,30 @@ end # User, Robot, Session Data Blob Entries function GraphAgentBlobentries!(fg::AbstractDFG) - be = Blobentry(; label = :key1, blobstore = :b) + be = Blobentry(; label = :key1, storelabel = :b) + + # First add an agent to test with + agentlabel = :testBlobentryAgent + addAgent!(fg, Agent(; label = agentlabel)) # Agent Blob Entries - ae = addAgentBlobentry!(fg, be) + ae = addAgentBlobentry!(fg, agentlabel, be) @test ae == be - @test_throws DFG.LabelExistsError addAgentBlobentry!(fg, be) - ge = getAgentBlobentry(fg, :key1) + @test_throws DFG.LabelExistsError addAgentBlobentry!(fg, agentlabel, be) + ge = getAgentBlobentry(fg, agentlabel, :key1) @test ge == be - @test hasAgentBlobentry(fg, :key1) - me = mergeAgentBlobentry!(fg, be) + @test hasAgentBlobentry(fg, agentlabel, :key1) + me = mergeAgentBlobentry!(fg, agentlabel, be) @test me == 1 - de = deleteAgentBlobentry!(fg, :key1) + de = deleteAgentBlobentry!(fg, agentlabel, :key1) @test de == 1 - @test hasAgentBlobentry(fg, :key1) == false - @test_throws DFG.LabelNotFoundError getAgentBlobentry(fg, :key1) - @test deleteAgentBlobentry!(fg, :key1) == 0 - @test addAgentBlobentries!(fg, [be]) == [be] - @test deleteAgentBlobentries!(fg, [:key1]) == 1 - @test mergeAgentBlobentries!(fg, [be]) == 1 - @test deleteAgentBlobentries!(fg, [:key1]) == 1 + @test hasAgentBlobentry(fg, agentlabel, :key1) == false + @test_throws DFG.LabelNotFoundError getAgentBlobentry(fg, agentlabel, :key1) + @test deleteAgentBlobentry!(fg, agentlabel, :key1) == 0 + @test addAgentBlobentries!(fg, agentlabel, [be]) == [be] + @test deleteAgentBlobentries!(fg, agentlabel, [:key1]) == 1 + @test mergeAgentBlobentries!(fg, agentlabel, [be]) == 1 + @test deleteAgentBlobentries!(fg, agentlabel, [:key1]) == 1 # Graph Blob Entries ae = addGraphBlobentry!(fg, be) @@ -208,21 +215,21 @@ function GraphAgentBlobentries!(fg::AbstractDFG) @test mergeGraphBlobentries!(fg, [be]) == 1 @test deleteGraphBlobentries!(fg, [:key1]) == 1 - be2 = Blobentry(; blobid = uuid4(), label = :key2, blobstore = :b) + be2 = Blobentry(; blobid = uuid4(), label = :key2, storelabel = :b) bes = [be, be2] - ae = addAgentBlobentries!(fg, bes) + ae = addAgentBlobentries!(fg, agentlabel, bes) @test length(ae) == 2 - @test_throws DFG.LabelExistsError addAgentBlobentries!(fg, bes) - besr = getAgentBlobentries(fg) + @test_throws DFG.LabelExistsError addAgentBlobentries!(fg, agentlabel, bes) + besr = getAgentBlobentries(fg, agentlabel) @test length(besr) == 2 - me = mergeAgentBlobentries!(fg, bes) + me = mergeAgentBlobentries!(fg, agentlabel, bes) @test me == 2 - de = deleteAgentBlobentries!(fg, [:key1, :key2]) + de = deleteAgentBlobentries!(fg, agentlabel, [:key1, :key2]) @test de == 2 - @test_throws DFG.LabelNotFoundError getAgentBlobentry(fg, :key1) - @test_throws DFG.LabelNotFoundError getAgentBlobentry(fg, :key2) + @test_throws DFG.LabelNotFoundError getAgentBlobentry(fg, agentlabel, :key1) + @test_throws DFG.LabelNotFoundError getAgentBlobentry(fg, agentlabel, :key2) ae = addGraphBlobentries!(fg, bes) @test length(ae) == 2 @@ -580,16 +587,20 @@ function tagsTestBlock!(fg, v1, v1_tags) @test deleteGraphTags!(fg, [:GRAPH_TAG]) == 1 @test !hasGraphTags(fg, [:GRAPH_TAG]) - @test mergeAgentTags!(fg, [:AGENT_TAG]) == 1 - @test :AGENT_TAG ∈ listAgentTags(fg) - @test hasAgentTags(fg, [:AGENT_TAG]) - @test deleteAgentTags!(fg, [:AGENT_TAG]) == 1 - @test !hasAgentTags(fg, [:AGENT_TAG]) + agentlabel = :testTagAgent + addAgent!(fg, Agent(; label = agentlabel)) + @test mergeAgentTags!(fg, agentlabel, [:AGENT_TAG]) == 1 + @test :AGENT_TAG ∈ listAgentTags(fg, agentlabel) + @test hasAgentTags(fg, agentlabel, [:AGENT_TAG]) + @test deleteAgentTags!(fg, agentlabel, [:AGENT_TAG]) == 1 + @test !hasAgentTags(fg, agentlabel, [:AGENT_TAG]) @test listVariableTags(fg, :a) isa Vector{Symbol} @test listFactorTags(fg, :abf1) isa Vector{Symbol} @test listGraphTags(fg) isa Vector{Symbol} - @test listAgentTags(fg) isa Vector{Symbol} + @test listAgentTags(fg, agentlabel) isa Vector{Symbol} + deleteAgent!(fg, agentlabel) + return nothing end function VSDTestBlock!(fg, v1) @@ -754,11 +765,14 @@ end function hasBlobletTestBlock!(fg) # Agent bloblets has - addAgentBloblet!(fg, Bloblet(:agent_has_test, "data")) - @test hasAgentBloblet(fg, :agent_has_test) - @test !hasAgentBloblet(fg, :nonexistent) - deleteAgentBloblet!(fg, :agent_has_test) - @test !hasAgentBloblet(fg, :agent_has_test) + agentlabel = :testHasBlobletAgent + addAgent!(fg, Agent(; label = agentlabel)) + addAgentBloblet!(fg, agentlabel, Bloblet(:agent_has_test, "data")) + @test hasAgentBloblet(fg, agentlabel, :agent_has_test) + @test !hasAgentBloblet(fg, agentlabel, :nonexistent) + deleteAgentBloblet!(fg, agentlabel, :agent_has_test) + @test !hasAgentBloblet(fg, agentlabel, :agent_has_test) + deleteAgent!(fg, agentlabel) # Graph bloblets has addGraphBloblet!(fg, Bloblet(:graph_has_test, "data")) @@ -854,22 +868,22 @@ function DataEntriesTestBlock!(fg, v2) # listBlobentries # emptyDataEntries # mergeDataEntries - storeEntry = Blobentry(; blobid = uuid4(), label = :a, blobstore = :b) + storeEntry = Blobentry(; blobid = uuid4(), label = :a, storelabel = :b) @test getLabel(storeEntry) == storeEntry.label @test getTimestamp(storeEntry) == storeEntry.timestamp # oid = zeros(UInt8,12); oid[12] = 0x01 # de1 = MongodbDataEntry(:key1, uuid4(), NTuple{12,UInt8}(oid), "", now(localzone())) - de1 = Blobentry(; blobid = uuid4(), label = :key1, blobstore = :b) + de1 = Blobentry(; blobid = uuid4(), label = :key1, storelabel = :b) # oid = zeros(UInt8,12); oid[12] = 0x02 # de2 = MongodbDataEntry(:key2, uuid4(), NTuple{12,UInt8}(oid), "", now(localzone())) - de2 = Blobentry(; blobid = uuid4(), label = :key2, blobstore = :b) + de2 = Blobentry(; blobid = uuid4(), label = :key2, storelabel = :b) # oid = zeros(UInt8,12); oid[12] = 0x03 # de2_update = MongodbDataEntry(:key2, uuid4(), NTuple{12,UInt8}(oid), "", now(localzone())) de2_update = - Blobentry(; blobid = uuid4(), label = :key2, blobstore = :b, description = "Yay") + Blobentry(; blobid = uuid4(), label = :key2, storelabel = :b, description = "Yay") #add v1 = getVariable(fg, :a) @@ -956,26 +970,31 @@ function DataEntriesTestBlock!(fg, v2) @test listGraphBlobentries(fg) == Symbol[] # agent blobentries - @test addAgentBlobentry!(fg, de1) == de1 - @test_throws LabelExistsError addAgentBlobentry!(fg, de1) - @test de1 == getAgentBlobentry(fg, getLabel(de1)) - @test_throws LabelNotFoundError getAgentBlobentry(fg, :nope) - @test mergeAgentBlobentry!(fg, de2_update) == 1 - @test listAgentBlobentries(fg) == [getLabel(de1), getLabel(de2_update)] - @test deleteAgentBlobentry!(fg, getLabel(de2_update)) == 1 - @test deleteAgentBlobentry!(fg, getLabel(de2_update)) == 0 - @test getAgentBlobentries(fg) == [de1] - @test addAgentBlobentries!(fg, [de2]) == [de2] - @test mergeAgentBlobentries!(fg, [de1, de2_update]) == 2 - @test deleteAgentBlobentries!(fg, [getLabel(de1), getLabel(de2_update)]) == 2 - @test listAgentBlobentries(fg) == Symbol[] + agentlabel = :testBEAgent + addAgent!(fg, Agent(; label = agentlabel)) + @test addAgentBlobentry!(fg, agentlabel, de1) == de1 + @test_throws LabelExistsError addAgentBlobentry!(fg, agentlabel, de1) + @test de1 == getAgentBlobentry(fg, agentlabel, getLabel(de1)) + @test_throws LabelNotFoundError getAgentBlobentry(fg, agentlabel, :nope) + @test mergeAgentBlobentry!(fg, agentlabel, de2_update) == 1 + @test listAgentBlobentries(fg, agentlabel) == [getLabel(de1), getLabel(de2_update)] + @test deleteAgentBlobentry!(fg, agentlabel, getLabel(de2_update)) == 1 + @test deleteAgentBlobentry!(fg, agentlabel, getLabel(de2_update)) == 0 + @test getAgentBlobentries(fg, agentlabel) == [de1] + @test addAgentBlobentries!(fg, agentlabel, [de2]) == [de2] + @test mergeAgentBlobentries!(fg, agentlabel, [de1, de2_update]) == 2 + @test deleteAgentBlobentries!(fg, agentlabel, [getLabel(de1), getLabel(de2_update)]) == + 2 + @test listAgentBlobentries(fg, agentlabel) == Symbol[] + deleteAgent!(fg, agentlabel) + return nothing end function blobsStoresTestBlock!(fg) de1 = Blobentry(; blobid = uuid4(), label = :label1, - blobstore = :store1, + storelabel = :store1, crchash = 0xAAAA, origin = "origin1", description = "description1", @@ -984,7 +1003,7 @@ function blobsStoresTestBlock!(fg) de2 = Blobentry(; blobid = uuid4(), label = :label2, - blobstore = :store2, + storelabel = :store2, crchash = 0xFFFF, origin = "origin2", description = "description2", @@ -994,7 +1013,7 @@ function blobsStoresTestBlock!(fg) de2_update = Blobentry(; blobid = uuid4(), label = :label2, - blobstore = :store2, + storelabel = :store2, crchash = 0x0123, origin = "origin2", description = "description2", @@ -1101,14 +1120,23 @@ function blobsStoresTestBlock!(fg) @test_throws DFG.LabelNotFoundError DFG.loadBlob_Graph(fg, :testing) # on Agent - newentry = DFG.saveBlob_Agent!(fg, testData, :testing, fs.label) - @test_throws DFG.LabelExistsError DFG.saveBlob_Agent!(fg, testData, :testing, fs.label) - @test :testing in listAgentBlobentries(fg) - be, blob = DFG.loadBlob_Agent(fg, :testing) + agentlabel = :testBlobWrapperAgent + addAgent!(fg, Agent(; label = agentlabel)) + newentry = DFG.saveBlob_Agent!(fg, agentlabel, testData, :testing, fs.label) + @test_throws DFG.LabelExistsError DFG.saveBlob_Agent!( + fg, + agentlabel, + testData, + :testing, + fs.label, + ) + @test :testing in listAgentBlobentries(fg, agentlabel) + be, blob = DFG.loadBlob_Agent(fg, agentlabel, :testing) @test newentry == be @test blob == testData - @test DFG.deleteBlob_Agent!(fg, :testing) == 2 - @test_throws DFG.LabelNotFoundError DFG.loadBlob_Agent(fg, :testing) + @test DFG.deleteBlob_Agent!(fg, agentlabel, :testing) == 2 + @test_throws DFG.LabelNotFoundError DFG.loadBlob_Agent(fg, agentlabel, :testing) + deleteAgent!(fg, agentlabel) return nothing end