Merge pull request #207 from alyst/vars_api

Cleanup  Variables API

Author: Maximilian-Stefan-Ernst

src/StructuralEquationModels.jl

@@ -30,6 +30,7 @@ include("additional_functions/commutation_matrix.jl")
 # fitted objects
 include("frontend/fit/SemFit.jl")
 # specification of models
+include("frontend/common.jl")
 include("frontend/specification/checks.jl")
 include("frontend/specification/ParameterTable.jl")
 include("frontend/specification/RAMMatrices.jl")
@@ -39,7 +40,7 @@ include("frontend/fit/summary.jl")
 # pretty printing
 include("frontend/pretty_printing.jl")
 # observed
-include("observed/get_colnames.jl")
+include("observed/abstract.jl")
 include("observed/covariance.jl")
 include("observed/data.jl")
 include("observed/missing.jl")
@@ -48,6 +49,7 @@ include("observed/EM.jl")
 include("frontend/specification/Sem.jl")
 include("frontend/specification/documentation.jl")
 # imply
+include("imply/abstract.jl")
 include("imply/RAM/symbolic.jl")
 include("imply/RAM/generic.jl")
 include("imply/empty.jl")
@@ -80,10 +82,8 @@ include("frontend/fit/fitmeasures/BIC.jl")
 include("frontend/fit/fitmeasures/chi2.jl")
 include("frontend/fit/fitmeasures/df.jl")
 include("frontend/fit/fitmeasures/minus2ll.jl")
-include("frontend/fit/fitmeasures/n_obs.jl")
 include("frontend/fit/fitmeasures/p.jl")
 include("frontend/fit/fitmeasures/RMSEA.jl")
-include("frontend/fit/fitmeasures/n_man.jl")
 include("frontend/fit/fitmeasures/fit_measures.jl")
 # standard errors
 include("frontend/fit/standard_errors/hessian.jl")
@@ -126,6 +126,10 @@ export AbstractSem,
     SemObservedCovariance,
     SemObservedMissing,
     observed,
+    obs_cov,
+    obs_mean,
+    nsamples,
+    samples,
     sem_fit,
     SemFit,
     minimum,
@@ -138,6 +142,8 @@ export AbstractSem,
     objective_hessian!,
     gradient_hessian!,
     objective_gradient_hessian!,
+    SemSpecification,
+    RAMMatrices,
     ParameterTable,
     EnsembleParameterTable,
     update_partable!,
@@ -150,9 +156,14 @@ export AbstractSem,
     start,
     Label,
     label,
+    nvars,
+    vars,
+    nlatent_vars,
+    latent_vars,
+    nobserved_vars,
+    observed_vars,
     sort_vars!,
     sort_vars,
-    RAMMatrices,
     params,
     nparams,
     param_indices,
@@ -163,10 +174,8 @@ export AbstractSem,
     df,
     fit_measures,
     minus2ll,
-    n_obs,
     p_value,
     RMSEA,
-    n_man,
     EmMVNModel,
     se_hessian,
     se_bootstrap,

src/additional_functions/start_val/start_fabin3.jl

@@ -31,26 +31,26 @@ function start_fabin3(observed::SemObservedMissing, imply, optimizer, args...; k
 end
 
 function start_fabin3(ram_matrices::RAMMatrices, Σ, μ)
-    A_ind, S_ind, F_ind, M_ind, params = ram_matrices.A_ind,
+    A_ind, S_ind, F_ind, M_ind, n_par = ram_matrices.A_ind,
     ram_matrices.S_ind,
     ram_matrices.F_ind,
     ram_matrices.M_ind,
-    ram_matrices.params
+    nparams(ram_matrices)
 
-    n_par = length(params)
     start_val = zeros(n_par)
-    n_var, n_nod = ram_matrices.size_F
-    n_latent = n_nod - n_var
+    n_obs = nobserved_vars(ram_matrices)
+    n_var = nvars(ram_matrices)
+    n_latent = nlatent_vars(ram_matrices)
 
-    C_indices = CartesianIndices((n_nod, n_nod))
+    C_indices = CartesianIndices((n_var, n_var))
 
     # check in which matrix each parameter appears
 
     indices = Vector{CartesianIndex{2}}(undef, n_par)
 
     #=     in_S = length.(S_ind) .!= 0
         in_A = length.(A_ind) .!= 0
-        A_ind_c = [linear2cartesian(ind, (n_nod, n_nod)) for ind in A_ind]
+        A_ind_c = [linear2cartesian(ind, (n_var, n_var)) for ind in A_ind]
         in_Λ = [any(ind[2] .∈ F_ind) for ind in A_ind_c]
 
         if !isnothing(M)
@@ -77,7 +77,7 @@ function start_fabin3(ram_matrices::RAMMatrices, Σ, μ)
 
     # set loadings
     constants = ram_matrices.constants
-    A_ind_c = [linear2cartesian(ind, (n_nod, n_nod)) for ind in A_ind]
+    A_ind_c = [linear2cartesian(ind, (n_var, n_var)) for ind in A_ind]
     # ind_Λ = findall([is_in_Λ(ind_vec, F_ind) for ind_vec in A_ind_c])
 
     function calculate_lambda(
@@ -99,7 +99,7 @@ function start_fabin3(ram_matrices::RAMMatrices, Σ, μ)
         end
     end
 
-    for i in setdiff(1:n_nod, F_ind)
+    for i in setdiff(1:n_var, F_ind)
         reference = Int64[]
         indicators = Int64[]
         indicator2parampos = Dict{Int, Int}()

src/additional_functions/start_val/start_simple.jl

@@ -62,17 +62,17 @@ function start_simple(
     start_means = 0.0,
     kwargs...,
 )
-    A_ind, S_ind, F_ind, M_ind, params = ram_matrices.A_ind,
+    A_ind, S_ind, F_ind, M_ind, n_par = ram_matrices.A_ind,
     ram_matrices.S_ind,
     ram_matrices.F_ind,
     ram_matrices.M_ind,
-    ram_matrices.params
+    nparams(ram_matrices)
 
-    n_par = length(params)
     start_val = zeros(n_par)
-    n_var, n_nod = ram_matrices.size_F
+    n_obs = nobserved_vars(ram_matrices)
+    n_var = nvars(ram_matrices)
 
-    C_indices = CartesianIndices((n_nod, n_nod))
+    C_indices = CartesianIndices((n_var, n_var))
 
     for i in 1:n_par
         if length(S_ind[i]) != 0

src/frontend/common.jl

@@ -0,0 +1,57 @@
+# API methods supported by multiple SEM.jl types
+
+"""
+    nparams(semobj)
+
+Return the number of parameters in a SEM model associated with `semobj`.
+
+See also [`params`](@ref).
+"""
+nparams(semobj) = length(params(semobj))
+
+"""
+    nvars(semobj)
+
+Return the number of variables in a SEM model associated with `semobj`.
+
+See also [`vars`](@ref).
+"""
+nvars(semobj) = length(vars(semobj))
+
+"""
+    nobserved_vars(semobj)
+
+Return the number of observed variables in a SEM model associated with `semobj`.
+"""
+nobserved_vars(semobj) = length(observed_vars(semobj))
+
+"""
+    nlatent_vars(semobj)
+
+Return the number of latent variables in a SEM model associated with `semobj`.
+"""
+nlatent_vars(semobj) = length(latent_vars(semobj))
+
+"""
+    param_indices(semobj)
+
+Returns a dict of parameter names and their indices in `semobj`.
+
+# Examples
+```julia
+parind = param_indices(my_fitted_sem)
+parind[:param_name]
+```
+
+See also [`params`](@ref).
+"""
+param_indices(semobj) = Dict(par => i for (i, par) in enumerate(params(semobj)))
+
+"""
+    nsamples(semobj)
+
+Return the number of samples (observed data points).
+
+For ensemble models, return the sum over all submodels.
+"""
+function nsamples end

src/frontend/fit/SemFit.jl

@@ -48,6 +48,7 @@ end
 
 params(fit::SemFit) = params(fit.model)
 nparams(fit::SemFit) = nparams(fit.model)
+nsamples(fit::SemFit) = nsamples(fit.model)
 
 # access fields
 minimum(sem_fit::SemFit) = sem_fit.minimum

src/frontend/fit/fitmeasures/BIC.jl

@@ -3,4 +3,4 @@
 
 Return the bayesian information criterion.
 """
-BIC(sem_fit::SemFit) = minus2ll(sem_fit) + log(n_obs(sem_fit)) * nparams(sem_fit)
+BIC(sem_fit::SemFit) = minus2ll(sem_fit) + log(nsamples(sem_fit)) * nparams(sem_fit)

src/frontend/fit/fitmeasures/RMSEA.jl

@@ -6,13 +6,13 @@ Return the RMSEA.
 function RMSEA end
 
 RMSEA(sem_fit::SemFit{Mi, So, St, Mo, O} where {Mi, So, St, Mo <: AbstractSemSingle, O}) =
-    RMSEA(df(sem_fit), χ²(sem_fit), n_obs(sem_fit))
+    RMSEA(df(sem_fit), χ²(sem_fit), nsamples(sem_fit))
 
 RMSEA(sem_fit::SemFit{Mi, So, St, Mo, O} where {Mi, So, St, Mo <: SemEnsemble, O}) =
-    sqrt(length(sem_fit.model.sems)) * RMSEA(df(sem_fit), χ²(sem_fit), n_obs(sem_fit))
+    sqrt(length(sem_fit.model.sems)) * RMSEA(df(sem_fit), χ²(sem_fit), nsamples(sem_fit))
 
-function RMSEA(df, chi2, n_obs)
-    rmsea = (chi2 - df) / (n_obs * df)
+function RMSEA(df, chi2, nsamples)
+    rmsea = (chi2 - df) / (nsamples * df)
     rmsea > 0 ? nothing : rmsea = 0
     return sqrt(rmsea)
 end

src/frontend/fit/fitmeasures/chi2.jl

@@ -20,11 +20,12 @@ function χ² end
 
 # RAM + SemML
 χ²(sem_fit::SemFit, observed, imp::Union{RAM, RAMSymbolic}, optimizer, loss_ml::SemML) =
-    (n_obs(sem_fit) - 1) * (sem_fit.minimum - logdet(observed.obs_cov) - observed.n_man)
+    (nsamples(sem_fit) - 1) *
+    (sem_fit.minimum - logdet(observed.obs_cov) - nobserved_vars(observed))
 
 # bollen, p. 115, only correct for GLS weight matrix
 χ²(sem_fit::SemFit, observed, imp::Union{RAM, RAMSymbolic}, optimizer, loss_ml::SemWLS) =
-    (n_obs(sem_fit) - 1) * sem_fit.minimum
+    (nsamples(sem_fit) - 1) * sem_fit.minimum
 
 # FIML
 function χ²(sem_fit::SemFit, observed::SemObservedMissing, imp, optimizer, loss_ml::SemFIML)
@@ -45,18 +46,18 @@ end
 function χ²(sem_fit::SemFit, model::SemEnsemble, lossfun::L) where {L <: SemWLS}
     check_ensemble_length(model)
     check_lossfun_types(model, L)
-    return (sum(n_obs.(model.sems)) - 1) * sem_fit.minimum
+    return (nsamples(model) - 1) * sem_fit.minimum
 end
 
 function χ²(sem_fit::SemFit, model::SemEnsemble, lossfun::L) where {L <: SemML}
     check_ensemble_length(model)
     check_lossfun_types(model, L)
     F_G = sem_fit.minimum
     F_G -= sum([
-        w * (logdet(m.observed.obs_cov) + m.observed.n_man) for
+        w * (logdet(m.observed.obs_cov) + nobserved_vars(m.observed)) for
         (w, m) in zip(model.weights, model.sems)
     ])
-    return (sum(n_obs.(model.sems)) - 1) * F_G
+    return (nsamples(model) - 1) * F_G
 end
 
 function χ²(sem_fit::SemFit, model::SemEnsemble, lossfun::L) where {L <: SemFIML}

src/frontend/fit/fitmeasures/df.jl

@@ -11,10 +11,10 @@ df(sem_fit::SemFit) = df(sem_fit.model)
 df(model::AbstractSem) = n_dp(model) - nparams(model)
 
 function n_dp(model::AbstractSemSingle)
-    nman = n_man(model)
-    ndp = 0.5(nman^2 + nman)
+    nvars = nobserved_vars(model)
+    ndp = 0.5(nvars^2 + nvars)
     if !isnothing(model.imply.μ)
-        ndp += n_man(model)
+        ndp += nvars
     end
     return ndp
 end

src/frontend/fit/fitmeasures/minus2ll.jl

@@ -25,7 +25,7 @@ minus2ll(sem_fit::SemFit, obs, imp, optimizer, args...) =
 
 # SemML ------------------------------------------------------------------------------------
 minus2ll(minimum::Number, obs, imp::Union{RAM, RAMSymbolic}, optimizer, loss_ml::SemML) =
-    n_obs(obs) * (minimum + log(2π) * n_man(obs))
+    nsamples(obs) * (minimum + log(2π) * nobserved_vars(obs))
 
 # WLS --------------------------------------------------------------------------------------
 minus2ll(minimum::Number, obs, imp::Union{RAM, RAMSymbolic}, optimizer, loss_ml::SemWLS) =
@@ -41,8 +41,8 @@ function minus2ll(
     loss_ml::SemFIML,
 )
     F = minimum
-    F *= n_obs(observed)
-    F += sum(log(2π) * observed.pattern_n_obs .* observed.pattern_nvar_obs)
+    F *= nsamples(observed)
+    F += sum(log(2π) * observed.pattern_nsamples .* observed.pattern_nobs_vars)
     return F
 end
 
@@ -53,26 +53,26 @@ function minus2ll(observed::SemObservedMissing)
         minus2ll(
             observed.em_model.μ,
             observed.em_model.Σ,
-            observed.n_obs,
-            observed.rows,
+            nsamples(observed),
+            pattern_rows(observed),
             observed.patterns,
             observed.obs_mean,
             observed.obs_cov,
-            observed.pattern_n_obs,
-            observed.pattern_nvar_obs,
+            observed.pattern_nsamples,
+            observed.pattern_nobs_vars,
         )
     else
         em_mvn(observed)
         minus2ll(
             observed.em_model.μ,
             observed.em_model.Σ,
-            observed.n_obs,
-            observed.rows,
+            nsamples(observed),
+            pattern_rows(observed),
             observed.patterns,
             observed.obs_mean,
             observed.obs_cov,
-            observed.pattern_n_obs,
-            observed.pattern_nvar_obs,
+            observed.pattern_nsamples,
+            observed.pattern_nobs_vars,
         )
     end
 end
@@ -85,13 +85,13 @@ function minus2ll(
     patterns,
     obs_mean,
     obs_cov,
-    pattern_n_obs,
-    pattern_nvar_obs,
+    pattern_nsamples,
+    pattern_nobs_vars,
 )
     F = 0.0
 
     for i in 1:length(rows)
-        nᵢ = pattern_n_obs[i]
+        nᵢ = pattern_nsamples[i]
         # missing pattern
         pattern = patterns[i]
         # observed data
@@ -106,7 +106,7 @@ function minus2ll(
         F += F_one_pattern(meandiffᵢ, Σᵢ⁻¹, Sᵢ, ld, nᵢ)
     end
 
-    F += sum(log(2π) * pattern_n_obs .* pattern_nvar_obs)
+    F += sum(log(2π) * pattern_nsamples .* pattern_nobs_vars)
     #F *= N
 
     return F