Merge pull request #20 from ParallelGSReg/development

Add options

Author: adanmauri

Project.toml

@@ -1,7 +1,7 @@
 name = "ModelSelection"
 uuid = "51b78214-913b-40cf-9250-c64eb33811f7"
 authors = ["Demian Panigo <panigo@gmail.com>", "Adán Mauri Ungaro <adan.mauri@gmail.com>", "Nicolás Monzón <nicomzn4@gmail.com>", "Valentin Mari <valentinmari@hotmail.com>"]
-version = "1.3.0"
+version = "1.3.1"
 
 [deps]
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"

TODO.md

@@ -5,6 +5,7 @@
 
 # TBD
  - Validation in every module
- - Documentation in AllSubsetRegression
+ - Pending documentation in AllSubsetRegression
  - Major refactoring
  - General documentation
+ - Notify progress in modules based in the time that takes in average

src/AllSubsetRegression/core.jl

@@ -59,6 +59,7 @@ function all_subset_regression(
     modelavg::Bool = MODELAVG_DEFAULT,
     residualtest::Bool = RESIDUALTEST_DEFAULT,
     orderresults::Bool = ORDERRESULTS_DEFAULT,
+    notify = nothing
 )
     validate_test(ttest = ttest, ztest = ztest)
 
@@ -71,6 +72,7 @@ function all_subset_regression(
             modelavg = modelavg,
             residualtest = residualtest,
             orderresults = orderresults,
+            notify = notify ,
         )
     elseif estimator == :logit
         AllSubsetRegression.logit!(
@@ -81,6 +83,7 @@ function all_subset_regression(
             modelavg = modelavg,
             residualtest = residualtest,
             orderresults = orderresults,
+            notify = notify ,
         )
     else
         throw(ArgumentError(INVALID_ESTIMATOR))

src/AllSubsetRegression/estimators/logit.jl

@@ -49,6 +49,7 @@ function logit(
     modelavg::Bool = MODELAVG_DEFAULT,
     residualtest::Bool = RESIDUALTEST_DEFAULT,
     orderresults::Bool = ORDERRESULTS_DEFAULT,
+    notify = nothing,
 )
     return logit!(
         ModelSelection.copy_modelselectiondata(data),
@@ -58,6 +59,7 @@ function logit(
         modelavg = modelavg,
         residualtest = residualtest,
         orderresults = orderresults,
+        notify = notify ,
     )
 end
 
@@ -110,7 +112,9 @@ function logit!(
     modelavg::Bool = MODELAVG_DEFAULT,
     residualtest::Bool = RESIDUALTEST_DEFAULT,
     orderresults::Bool = ORDERRESULTS_DEFAULT,
+    notify = nothing,
 )
+    ModelSelection.notification(notify, "Performing All Subset Regression", Dict(:estimator => :logit, :progress => 0))
     validate_criteria(criteria, AVAILABLE_LOGIT_CRITERIA)
     result = create_result(
         data,
@@ -121,7 +125,7 @@ function logit!(
         orderresults,
         ztest = ztest,
     )
-    logit_execute!(data, result)
+    logit_execute!(data, result, notify = notify )
     ModelSelection.addresult!(data, result)
     data = addextras!(data, result)
     return data
@@ -153,7 +157,9 @@ explanatory variables in the given `ModelSelectionData`. This function mutates t
 logit_execute!(model_selection_data, all_subset_regression_result)
 ```
 """
-function logit_execute!(data::ModelSelectionData, result::AllSubsetRegressionResult)
+function logit_execute!(data::ModelSelectionData, result::AllSubsetRegressionResult; notify = nothing)
+    ModelSelection.notification(notify, "Performing All Subset Regression", Dict(:estimator => :logit, :progress => 5))
+
     if !data.removemissings
         data = ModelSelection.filter_data_by_empty_values!(data)
     end
@@ -189,6 +195,7 @@ function logit_execute!(data::ModelSelectionData, result::AllSubsetRegressionRes
         fullexpvars_without_outsample_subset =
             hcat(expvars_without_outsample_subset, fixedvariables_without_outsample_subset)
     end
+    ModelSelection.notification(notify, "Performing All Subset Regression", Dict(:estimator => :logit, :progress => 15))
 
     gum_model = GLM.fit(
         GeneralizedLinearModel,
@@ -200,6 +207,7 @@ function logit_execute!(data::ModelSelectionData, result::AllSubsetRegressionRes
     )
     start_coef = coeftable(gum_model).cols[1]
 
+    ModelSelection.notification(notify, "Performing All Subset Regression", Dict(:estimator => :logit, :progress => 25))
     if nprocs() == nworkers()
         for order = 1:num_operations
             # TODO: Split in multiple lines
@@ -287,6 +295,7 @@ function logit_execute!(data::ModelSelectionData, result::AllSubsetRegressionRes
             end
         end
     end
+    ModelSelection.notification(notify, "Performing All Subset Regression", Dict(:estimator => :logit, :progress => 75))
 
     result.data = Array(result_data)
 
@@ -351,6 +360,7 @@ function logit_execute!(data::ModelSelectionData, result::AllSubsetRegressionRes
     end
 
     result.nobs = result.bestresult_data[datanames_index[:nobs]]
+    ModelSelection.notification(notify, "Performing All Subset Regression", Dict(:estimator => :logit, :progress => 100))
 
     return result
 end

src/AllSubsetRegression/estimators/ols.jl

@@ -49,6 +49,7 @@ function ols(
     modelavg::Bool = MODELAVG_DEFAULT,
     residualtest::Bool = RESIDUALTEST_DEFAULT,
     orderresults::Bool = ORDERRESULTS_DEFAULT,
+    notify = nothing,
 )
     return ols!(
         ModelSelection.copy_modelselectiondata(data),
@@ -58,6 +59,7 @@ function ols(
         modelavg = modelavg,
         residualtest = residualtest,
         orderresults = orderresults,
+        notify = notify ,
     )
 end
 
@@ -112,7 +114,9 @@ function ols!(
     modelavg::Bool = MODELAVG_DEFAULT,
     residualtest::Bool = RESIDUALTEST_DEFAULT,
     orderresults::Bool = ORDERRESULTS_DEFAULT,
+    notify = nothing
 )
+    ModelSelection.notification(notify, "Performing All Subset Regression", Dict(:estimator => :ols, :progress => 0))
     validate_criteria(criteria, AVAILABLE_OLS_CRITERIA)
     result = create_result(
         data,
@@ -123,7 +127,7 @@ function ols!(
         orderresults,
         ttest = ttest,
     )
-    ols_execute!(data, result)
+    ols_execute!(data, result, notify=notify)
     ModelSelection.addresult!(data, result)
     data = addextras!(data, result)
     return data
@@ -155,7 +159,9 @@ explanatory variables in the given `ModelSelectionData`. This function mutates t
 ols_execute!(model_selection_data, all_subset_regression_result)
 ```
 """
-function ols_execute!(data::ModelSelectionData, result::AllSubsetRegressionResult)
+function ols_execute!(data::ModelSelectionData, result::AllSubsetRegressionResult; notify = nothing)
+    ModelSelection.notification(notify, "Performing All Subset Regression", Dict(:estimator => :ols, :progress => 5))
+
     if !data.removemissings
         data = ModelSelection.filter_data_by_empty_values!(data)
     end
@@ -174,6 +180,7 @@ function ols_execute!(data::ModelSelectionData, result::AllSubsetRegressionResul
     result_data =
         fill!(SharedArray{data.datatype}(num_operations, size(result.datanames, 1)), NaN)
     datanames_index = ModelSelection.create_datanames_index(result.datanames)
+    ModelSelection.notification(notify, "Performing All Subset Regression", Dict(:estimator => :ols, :progress => 25))
     if nprocs() == nworkers()
         for order = 1:num_operations
             ols_execute_row!(
@@ -289,7 +296,7 @@ function ols_execute!(data::ModelSelectionData, result::AllSubsetRegressionResul
                 ) ./ std(result.data[:, datanames_index[criteria]])
             )
     end
-
+    ModelSelection.notification(notify, "Performing All Subset Regression", Dict(:estimator => :ols, :progress => 75))
     if result.modelavg
         delta =
             maximum(result.data[:, datanames_index[:order]]) .-
@@ -352,6 +359,7 @@ function ols_execute!(data::ModelSelectionData, result::AllSubsetRegressionResul
     result.bestresult_data[datanames_index[:nobs]] =
         Int64(round(result.bestresult_data[datanames_index[:nobs]]))
     result.nobs = result.bestresult_data[datanames_index[:nobs]]
+    ModelSelection.notification(notify, "Performing All Subset Regression", Dict(:estimator => :ols, :progress => 100))
     return result
 end
 

src/CrossValidation/core.jl

@@ -32,19 +32,21 @@ function kfoldcrossvalidation!(
     previousresult::ModelSelection.ModelSelectionData,
     data::ModelSelection.ModelSelectionData,
     k::Int,
-    s::Float64,
+    s::Float64;
+    notify = nothing,
 )
-    kfoldcrossvalidation(previousresult, data, k, s)
+    kfoldcrossvalidation(previousresult, data, k, s; notify = notify )
 end
 
 
 function kfoldcrossvalidation(
     previousresult::ModelSelection.ModelSelectionData,
     data::ModelSelection.ModelSelectionData,
     k::Int,
-    s::Float64,
+    s::Float64;
+    notify = nothing,
 )
-
+    ModelSelection.notification(notify, "Performing Cross validation", Dict(:progress => 0))
     #db = randperm(data.nobs)
     db = collect(1:data.nobs)
     folds = split_database(db, k)
@@ -64,6 +66,10 @@ function kfoldcrossvalidation(
 
     bestmodels = []
 
+    progress = 0
+    step = floor(Int64, 50 / k)
+    ModelSelection.notification(notify, "Performing Cross validation", Dict(:progress => progress))
+
     for obs in LOOCV(k)
         dataset = collect(Iterators.flatten(folds[obs]))
         testset = setdiff(1:data.nobs, dataset)
@@ -107,8 +113,12 @@ function kfoldcrossvalidation(
                 :datanames => backup.results[1].datanames,
             ),
         )
+        progress = progress + step
+        ModelSelection.notification(notify, "Performing Cross validation", Dict(:progress => progress))
     end
 
+    ModelSelection.notification(notify, "Performing Cross validation", Dict(:progress => 50))
+
     datanames = unique(Iterators.flatten(model[:datanames] for model in bestmodels))
 
     data = Array{Any,2}(zeros(size(bestmodels, 1), size(datanames, 1)))
@@ -145,6 +155,7 @@ function kfoldcrossvalidation(
     previousresult = ModelSelection.addresult!(previousresult, result)
 
     addextras(previousresult, result)
+    ModelSelection.notification(notify, "Performing Cross validation", Dict(:progress => 100))
 
     return previousresult
 end

src/FeatureExtraction/core.jl

@@ -6,6 +6,7 @@ function featureextraction(
     fe_lag::Union{Array{Pair{Symbol,Int64}},Array{Pair{String,Int64}},Nothing} = nothing,
     interaction::Union{Nothing,Array,Dict} = nothing,
     removemissings::Bool = REMOVEMISSINGS_DEFAULT,
+    notify = nothing,
 )
     return featureextraction!(
         ModelSelection.copy_modelselectiondata(data),
@@ -15,6 +16,7 @@ function featureextraction(
         fe_lag = fe_lag,
         interaction = interaction,
         removemissings = removemissings,
+        notify = notify ,
     )
 end
 
@@ -26,6 +28,7 @@ function featureextraction!(
     fe_lag::Union{Array{Pair{Symbol,Int64}},Array{Pair{String,Int64}},Nothing} = nothing,
     interaction::Union{Nothing,Array,Dict} = nothing,
     removemissings::Bool = REMOVEMISSINGS_DEFAULT,
+    notify = nothing,
 )
     data = execute!(
         data,
@@ -35,6 +38,7 @@ function featureextraction!(
         fe_lag = fe_lag,
         interaction = interaction,
         removemissings = removemissings,
+        notify = notify ,
     )
 
     data = addextras(data, fe_sqr, fe_log, fe_inv, fe_lag, interaction, removemissings)
@@ -50,7 +54,9 @@ function execute!(
     fe_lag::Union{Array{Pair{Symbol,Int64}},Array{Pair{String,Int64}},Nothing} = nothing,
     interaction::Union{Nothing,Array,Dict} = nothing,
     removemissings::Bool = REMOVEMISSINGS_DEFAULT,
+    notify = nothing,
 )
+    ModelSelection.notification(notify, "Performing Feature extraction", Dict(:progress => 0))
     if data.intercept
         ModelSelection.remove_intercept!(data)
     end
@@ -87,6 +93,6 @@ function execute!(
     end
 
     data = ModelSelection.convert_data!(data)
-
+    ModelSelection.notification(notify, "Performing Feature extraction", Dict(:progress => 100))
     return data
 end

src/ModelSelection/core/gsr.jl

@@ -34,7 +34,7 @@ function gsr(
     kfoldcrossvalidation::Bool = CrossValidation.KFOLDCROSSVALIDATION_DEFAULT,
     numfolds::Int = CrossValidation.NUMFOLDS_DEFAULT,
     testsetshare::Union{Float32,Float64} = CrossValidation.TESTSETSHARE_DEFAULT,
-    notify = NOTIFY_DEFAULT,
+    notify = nothing,
 )
     gsr(
         estimator,
@@ -104,12 +104,10 @@ function gsr(
     kfoldcrossvalidation::Bool = CrossValidation.KFOLDCROSSVALIDATION_DEFAULT,
     numfolds::Int = CrossValidation.NUMFOLDS_DEFAULT,
     testsetshare::Union{Float32,Float64} = CrossValidation.TESTSETSHARE_DEFAULT,
-    notify = NOTIFY_DEFAULT,
+    notify = nothing,
 )
     removemissings = fe_lag === nothing
 
-    # TODO: Move notification to every module
-    notification(notify, "Processing parameters")
     data = Preprocessing.input(
         equation,
         data = data,
@@ -122,11 +120,10 @@ function gsr(
         seasonaladjustment = seasonaladjustment,
         removeoutliers = removeoutliers,
         removemissings = removemissings,
+        notify = notify,
     )
 
     if featureextraction_enabled(fe_sqr, fe_log, fe_inv, fe_lag, interaction)
-        # TODO: Move notification to every module
-        notification(notify, "Performing feature extraction")
         data = FeatureExtraction.featureextraction!(
             data,
             fe_sqr = fe_sqr,
@@ -135,15 +132,14 @@ function gsr(
             fe_inv = fe_inv,
             interaction = interaction,
             removemissings = true,
+            notify = notify ,
         )
     end
 
     original_data = copy_modelselectiondata(data)
 
     if preliminaryselection_enabled(preliminaryselection)
-        # TODO: Move notification to every module
-        notification(notify, "Performing preliminary selection")
-        data = PreliminarySelection.preliminary_selection!(preliminaryselection, data)
+        data = PreliminarySelection.preliminary_selection!(preliminaryselection, data, notify = notify )
         original_data.extras = data.extras
     end
 
@@ -157,17 +153,43 @@ function gsr(
         modelavg = modelavg,
         residualtest = residualtest,
         orderresults = orderresults,
+        notify = notify ,
     )
 
     original_data.extras = data.extras
 
     if crossvalidation_enabled(kfoldcrossvalidation)
-        # TODO: Move notification to every module
-        notification(notify, "Performing cross validation")
-        CrossValidation.kfoldcrossvalidation!(data, original_data, numfolds, testsetshare)
+        CrossValidation.kfoldcrossvalidation!(data, original_data, numfolds, testsetshare, notify = notify)
     end
 
     data.original_data = original_data
 
+    data.options[:estimator] = estimator
+    data.options[:equation] = equation
+    data.options[:datanames] = datanames
+    data.options[:method] = method
+    data.options[:intercept] = intercept
+    data.options[:panel] = panel
+    data.options[:time] = time
+    data.options[:seasonaladjustment] = seasonaladjustment
+    data.options[:removeoutliers] = removeoutliers
+    data.options[:fe_sqr] = fe_sqr
+    data.options[:fe_log] = fe_log
+    data.options[:fe_inv] = fe_inv
+    data.options[:fe_lag] = fe_lag
+    data.options[:interaction] = interaction
+    data.options[:preliminaryselection] = preliminaryselection
+    data.options[:fixedvariables] = fixedvariables
+    data.options[:outsample] = outsample
+    data.options[:criteria] = criteria
+    data.options[:ttest] = ttest
+    data.options[:ztest] = ztest
+    data.options[:modelavg] = modelavg
+    data.options[:residualtest] = residualtest
+    data.options[:orderresults] = orderresults
+    data.options[:kfoldcrossvalidation] = kfoldcrossvalidation
+    data.options[:numfolds] = numfolds
+    data.options[:testsetshare] = testsetshare
+
     return data
 end