ProcessBatchOutputs.ipy

# %%
%config IPCompleter.use_jedi = False
%pdb off
# %load_ext autoreload
# %autoreload 3
import os
import sys
import platform
from copy import deepcopy
from typing import List, Dict, Optional, Union, Callable
from pathlib import Path
import pathlib
import numpy as np
import pandas as pd
import tables as tb
from copy import deepcopy
from datetime import datetime, timedelta
from attrs import define, field, Factory

# # required to enable non-blocking interaction:
# %gui qt5 ## TODO 2024-01-18 - this causes kernel to crash when running notebook remotely via VSCode's ssh remote

## Pho's Custom Libraries:
from pyphocorehelpers.Filesystem.path_helpers import find_first_extant_path

# Jupyter interactivity:
from IPython.display import display

# pyPhoPlaceCellAnalysis:
# NeuroPy (Diba Lab Python Repo) Loading
from neuropy.core.session.Formats.BaseDataSessionFormats import DataSessionFormatRegistryHolder
from neuropy.core.session.Formats.Specific.BapunDataSessionFormat import BapunDataSessionFormatRegisteredClass
from neuropy.core.session.Formats.Specific.KDibaOldDataSessionFormat import KDibaOldDataSessionFormatRegisteredClass
from neuropy.core.session.Formats.Specific.RachelDataSessionFormat import RachelDataSessionFormat
from neuropy.core.session.Formats.Specific.HiroDataSessionFormat import HiroDataSessionFormatRegisteredClass

## For computation parameters:
from neuropy.utils.result_context import IdentifyingContext
from neuropy.core.session.Formats.BaseDataSessionFormats import find_local_session_paths
from neuropy.core import Epoch

from pyphocorehelpers.programming_helpers import copy_to_clipboard

from pyphoplacecellanalysis.General.Pipeline.Stages.Loading import saveData, loadData
import pyphoplacecellanalysis.General.Batch.runBatch
from pyphoplacecellanalysis.General.Batch.runBatch import BatchRun, BatchResultDataframeAccessor, run_diba_batch, BatchComputationProcessOptions, BatchSessionCompletionHandler, SavingOptions
from pyphoplacecellanalysis.General.Pipeline.NeuropyPipeline import PipelineSavingScheme

from neuropy.core.user_annotations import UserAnnotationsManager
from pyphoplacecellanalysis.General.Batch.runBatch import SessionBatchProgress
from pyphoplacecellanalysis.SpecificResults.AcrossSessionResults import AcrossSessionsResults, AcrossSessionTables, AcrossSessionsVisualizations

from pyphoplacecellanalysis.General.Batch.pythonScriptTemplating import build_vscode_workspace, build_windows_powershell_run_script

from pyphoplacecellanalysis.General.Batch.runBatch import PipelineCompletionResult, BatchSessionCompletionHandler

from pyphocorehelpers.Filesystem.metadata_helpers import FilesystemMetadata, get_file_metadata
from pyphocorehelpers.Filesystem.path_helpers import discover_data_files, generate_copydict, copy_movedict, copy_file, save_copydict_to_text_file, read_copydict_from_text_file, invert_filedict
from pyphoplacecellanalysis.General.Batch.runBatch import get_file_str_if_file_exists
from pyphoplacecellanalysis.SpecificResults.AcrossSessionResults import check_output_h5_files, copy_files_in_filelist_to_dest
from pyphoplacecellanalysis.General.Batch.runBatch import ConcreteSessionFolder, BackupMethods

# from pyphocorehelpers.gui.Jupyter.simple_widgets import fullwidth_path_widget, filesystem_path_folder_contents_widget

import inspect
from jinja2 import Template
from pyphoplacecellanalysis.General.Batch.BatchJobCompletion.UserCompletionHelpers.batch_user_completion_helpers import MAIN_get_template_string, write_test_script
from pyphoplacecellanalysis.General.Batch.BatchJobCompletion.UserCompletionHelpers.batch_user_completion_helpers import export_session_h5_file_completion_function, curr_runtime_context_header_template, export_rank_order_results_completion_function, figures_rank_order_results_completion_function, determine_session_t_delta_completion_function, perform_sweep_decoding_time_bin_sizes_marginals_dfs_completion_function, compute_and_export_decoders_epochs_decoding_and_evaluation_dfs_completion_function, compute_and_export_session_wcorr_shuffles_completion_function, compute_and_export_session_instantaneous_spike_rates_completion_function, compute_and_export_session_extended_placefield_peak_information_completion_function, compute_and_export_session_alternative_replay_wcorr_shuffles_completion_function, backup_previous_session_files_completion_function, compute_and_export_session_trial_by_trial_performance_completion_function, save_custom_session_files_completion_function, compute_and_export_cell_first_spikes_characteristics_completion_function, figures_plot_cell_first_spikes_characteristics_completion_function
from pyphoplacecellanalysis.General.Batch.BatchJobCompletion.UserCompletionHelpers.batch_user_completion_helpers import kdiba_session_post_fixup_completion_function, generalized_decode_epochs_dict_and_export_results_completion_function, export_decoded_context_uncertainty_by_position_completion_function, figures_plot_generalized_decode_epochs_dict_and_export_results_completion_function
from pyphoplacecellanalysis.General.Batch.pythonScriptTemplating import ProcessingScriptPhases

from pyphoplacecellanalysis.General.Batch.pythonScriptTemplating import BatchScriptsCollection, generate_batch_single_session_scripts


def execute_code_block(code_block: str) -> None:
    """Executes a block of code in the terminal."""
    import subprocess
    try:
        result = subprocess.run(code_block, shell=True, check=True, text=True)
        print(result.stdout)  # Output from the command
    except subprocess.CalledProcessError as e:
        print(f"Error executing command: {e}")


# enable_auto_code_block_execution: bool = True
enable_auto_code_block_execution: bool = False


BATCH_DAY_DATE: str = '2026-04-09'
# BATCH_DATE_TO_USE = f'{BATCH_DAY_DATE}_Apogee' # used for filenames throught the notebook
BATCH_DATE_TO_USE = f'{BATCH_DAY_DATE}_GL' # used for filenames throught the notebook
# BATCH_DATE_TO_USE = f'{BATCH_DAY_DATE}_Lab' # used for filenames throught the notebook
# BATCH_DATE_TO_USE = f'{BATCH_DAY_DATE}_rMBP' # used for filenames throught the notebook

# scripts_output_path = Path('/home/halechr/cloud/turbo/Data/Output/gen_scripts/').resolve() # Greatlakes
# scripts_output_path = Path('output/gen_scripts/').resolve() # Apogee
# # scripts_output_path = Path('/home/halechr/FastData/gen_scripts/').resolve() # Lab
# assert scripts_output_path.exists()
known_scripts_output_paths = [Path(v).resolve() for v in ['/Users/pho/repo/Pho Secondary Workspace/Spike3DEnv/cloud/turbo/Data/Output/gen_scripts', '/home/halechr/cloud/turbo/Data/Output/gen_scripts/', '/Users/pho/University of Michigan Dropbox/MED-DibaLabDropbox/Data/Pho/Outputs/output/collected_outputs', '/home/halechr/FastData/gen_scripts/', 'output/gen_scripts/']]
scripts_output_path = find_first_extant_path(known_scripts_output_paths)
assert scripts_output_path.exists(), f"scripts_output_path: {scripts_output_path} does not exist! Is the right computer's config commented out above?"
# fullwidth_path_widget(scripts_output_path, file_name_label='Scripts Output Path:')
print(f'scripts_output_path: {scripts_output_path}')

collected_outputs_path = scripts_output_path.joinpath('../collected_outputs').resolve()
collected_outputs_path.mkdir(exist_ok=True)
assert collected_outputs_path.exists()
print(f'collected_outputs_path: {collected_outputs_path}')

# %% [markdown]
# ## Build Processing Scripts:

# %%
# Get the generated code for user-contributed functions:
# phase_any_run_custom_user_completion_functions_dict = None
phase_any_run_custom_user_completion_functions_dict = {
#     # 'backup_previous_session_files_completion_function': backup_previous_session_files_completion_function,
#     # "export_rank_order_results_completion_function": export_rank_order_results_completion_function, # ran 2024-09-26 3pm
# # # # #     # "figures_rank_order_results_completion_function": figures_rank_order_results_completion_function,
#     # "determine_session_t_delta_completion_function": determine_session_t_delta_completion_function,  # ran 2024-05-28 6am
#     # 'perform_sweep_decoding_time_bin_sizes_marginals_dfs_completion_function': perform_sweep_decoding_time_bin_sizes_marginals_dfs_completion_function, # ran 2024-09-26 3pm
#     # 'compute_and_export_decoders_epochs_decoding_and_evaluation_dfs_completion_function': compute_and_export_decoders_epochs_decoding_and_evaluation_dfs_completion_function, # ran 2024-09-26 3pm
#     # 'kdiba_session_post_fixup_completion_function': kdiba_session_post_fixup_completion_function,
#     # # 'compute_and_export_session_wcorr_shuffles_completion_function': compute_and_export_session_wcorr_shuffles_completion_function,
#     # # 'compute_and_export_session_alternative_replay_wcorr_shuffles_completion_function': compute_and_export_session_alternative_replay_wcorr_shuffles_completion_function,
#     # 'compute_and_export_session_instantaneous_spike_rates_completion_function': compute_and_export_session_instantaneous_spike_rates_completion_function,
#     # 'compute_and_export_session_extended_placefield_peak_information_completion_function': compute_and_export_session_extended_placefield_peak_information_completion_function,
#     # 'compute_and_export_session_trial_by_trial_performance_completion_function': compute_and_export_session_trial_by_trial_performance_completion_function, 
#     'export_session_h5_file_completion_function': export_session_h5_file_completion_function, # ran 2024-09-26 3pm
#     # 'save_custom_session_files_completion_function': save_custom_session_files_completion_function,
#     # 'compute_and_export_cell_first_spikes_characteristics_completion_function': compute_and_export_cell_first_spikes_characteristics_completion_function,
#     # 'figures_plot_cell_first_spikes_characteristics_completion_function': figures_plot_cell_first_spikes_characteristics_completion_function,
	'generalized_decode_epochs_dict_and_export_results_completion_function': generalized_decode_epochs_dict_and_export_results_completion_function,
    'export_decoded_context_uncertainty_by_position_completion_function': export_decoded_context_uncertainty_by_position_completion_function,
    'figures_plot_generalized_decode_epochs_dict_and_export_results_completion_function': figures_plot_generalized_decode_epochs_dict_and_export_results_completion_function,
}

# %%


# Hardcoded included_session_contexts:
included_session_contexts = [
    IdentifyingContext(format_name='kdiba',animal='gor01',exper_name='one',session_name='2006-6-07_11-26-53'), # OK
    IdentifyingContext(format_name='kdiba',animal='gor01',exper_name='one',session_name='2006-6-08_14-26-15'), # 2025-04-03 18:59 EXCLUDE
    # IdentifyingContext(format_name='kdiba',animal='gor01',exper_name='one',session_name='2006-6-09_1-22-43'), # 2025-04-03 18:59 EXCLUDE
    # IdentifyingContext(format_name='kdiba',animal='gor01',exper_name='one',session_name='2006-6-12_15-55-31'), # 
    IdentifyingContext(format_name='kdiba',animal='gor01',exper_name='two',session_name='2006-6-07_16-40-19'), 
    IdentifyingContext(format_name='kdiba',animal='gor01',exper_name='two',session_name='2006-6-08_21-16-25'),
    IdentifyingContext(format_name='kdiba',animal='gor01',exper_name='two',session_name='2006-6-09_22-24-40'),
    IdentifyingContext(format_name='kdiba',animal='gor01',exper_name='two',session_name='2006-6-12_16-53-46'),
    IdentifyingContext(format_name='kdiba',animal='vvp01',exper_name='one',session_name='2006-4-09_17-29-30'),
    IdentifyingContext(format_name='kdiba',animal='vvp01',exper_name='one',session_name='2006-4-10_12-25-50'),
    IdentifyingContext(format_name='kdiba',animal='vvp01',exper_name='two',session_name='2006-4-09_16-40-54'),
    IdentifyingContext(format_name='kdiba',animal='vvp01',exper_name='two',session_name='2006-4-10_12-58-3'),
    IdentifyingContext(format_name='kdiba',animal='pin01',exper_name='one',session_name='11-02_17-46-44'), #
    IdentifyingContext(format_name='kdiba',animal='pin01',exper_name='one',session_name='11-02_19-28-0'),
    IdentifyingContext(format_name='kdiba',animal='pin01',exper_name='one',session_name='11-03_12-3-25'),
    IdentifyingContext(format_name='kdiba',animal='pin01',exper_name='one',session_name='fet11-01_12-58-54'), # OK
]


## Setup Functions:
force_recompute_override_computations_includelist = []
# force_recompute_override_computations_includelist = ['non_PBE_epochs_results', 'generalized_specific_epochs_decoding']
# force_recompute_override_computations_includelist = ['non_PBE_epochs_results',]
# force_recompute_override_computations_includelist = ['split_to_directional_laps','merged_directional_placefields','directional_decoders_evaluate_epochs','directional_decoders_epoch_heuristic_scoring']
# force_recompute_override_computations_includelist = ['directional_decoders_evaluate_epochs','directional_decoders_epoch_heuristic_scoring']
# force_recompute_override_computations_includelist = ['lap_direction_determination', 'split_to_directional_laps', 'pf_computation', 'pfdt_computation','firing_rate_trends','merged_directional_placefields','rank_order_shuffle_analysis','directional_decoders_evaluate_epochs']
# force_recompute_override_computations_includelist = ['firing_rate_trends', 'extended_stats', 'long_short_decoding_analyses', 'jonathan_firing_rate_analysis', 'long_short_fr_indicies_analyses', 'short_long_pf_overlap_analyses', 'long_short_post_decoding', ]
# force_recompute_override_computations_includelist = ['rank_order_shuffle_analysis','wcorr_shuffle_analysis','trial_by_trial_metrics']

extra_extended_computations_include_includelist = [
    # 'wcorr_shuffle_analysis'
    # 'non_PBE_epochs_results',
    # 'non_PBE_epochs_results', 'generalized_specific_epochs_decoding',
]

custom_phase_extended_computations_include_includelist = None
# custom_phase_extended_computations_include_includelist=['lap_direction_determination', 'pf_computation', 'pfdt_computation', 'firing_rate_trends',
# 	'pf_dt_sequential_surprise',
# 	'extended_stats',
# 	'long_short_decoding_analyses', 'jonathan_firing_rate_analysis', 'long_short_fr_indicies_analyses', 'short_long_pf_overlap_analyses', 'long_short_post_decoding', 
# 	'ratemap_peaks_prominence2d',
# 	'long_short_inst_spike_rate_groups',
# 	'long_short_endcap_analysis',
# 	# 'spike_burst_detection',
# 	'split_to_directional_laps',
# 	'merged_directional_placefields',
# 	'rank_order_shuffle_analysis',
# 	# 'directional_train_test_split',
# 	'directional_decoders_decode_continuous',
# 	'directional_decoders_evaluate_epochs',
# 	'directional_decoders_epoch_heuristic_scoring',
#     'wcorr_shuffle_analysis',
#     'trial_by_trial_metrics',
#     'extended_pf_peak_information',
# ]

active_global_batch_result_filename=f'global_batch_result_{BATCH_DATE_TO_USE}.pkl'

debug_print = True
known_global_data_root_parent_paths = [Path('/Users/pho/data'), Path(r'/nfs/turbo/umms-kdiba/Data'), Path(r'/media/halechr/BETAMAX/Data'), Path(r'/media/halechr/MAX/Data'), Path(r'W:/Data'), Path(r'/home/halechr/FastData'), Path(r'/Volumes/MoverNew/data')] # Path(r'/home/halechr/cloud/turbo/Data'), , Path(r'/nfs/turbo/umms-kdiba/Data'), Path(r'/home/halechr/turbo/Data'), 
global_data_root_parent_path = find_first_extant_path(known_global_data_root_parent_paths)
assert global_data_root_parent_path.exists(), f"global_data_root_parent_path: {global_data_root_parent_path} does not exist! Is the right computer's config commented out above?"
good_session_concrete_folders = ConcreteSessionFolder.build_concrete_session_folders(global_data_root_parent_path, included_session_contexts)

## Different run configurations:
# ==================================================================================================================== #
# ACTIVE PHASE                                                                                                         #
# ==================================================================================================================== #

# active_phase = ProcessingScriptPhases.clean_run
active_phase = ProcessingScriptPhases.continued_run
# active_phase = ProcessingScriptPhases.final_run
# active_phase = ProcessingScriptPhases.figure_run
# active_phase = ProcessingScriptPhases.figure_run


# END ________________________________________________________________________________________________________________ #
# custom_user_completion_functions_dict = active_phase.get_custom_user_completion_functions_dict(extra_run_functions=phase_any_run_custom_user_completion_functions_dict)
custom_user_completion_functions_dict = phase_any_run_custom_user_completion_functions_dict ## directly override the outputs:

# print(f'custom_user_completion_functions_dict: {custom_user_completion_functions_dict}')
custom_user_completion_function_template_code, custom_user_completion_functions_dict = MAIN_get_template_string(BATCH_DATE_TO_USE=BATCH_DATE_TO_USE, collected_outputs_path=collected_outputs_path, override_custom_user_completion_functions_dict=custom_user_completion_functions_dict)
# print(f'custom_user_completion_function_template_code: {custom_user_completion_function_template_code}')
active_phase_dict = active_phase.get_run_configuration(custom_user_completion_function_template_code=custom_user_completion_function_template_code, extra_extended_computations_include_includelist=extra_extended_computations_include_includelist)

## Completely replace with custom functions:
if custom_phase_extended_computations_include_includelist is not None:
    print(f'WARNING: custom_phase_extended_computations_include_includelist: {custom_phase_extended_computations_include_includelist} is provided so only these extended_computation_fns will be used (overwritting the phase defaults!).')
    active_phase_dict['extended_computations_include_includelist'] = custom_phase_extended_computations_include_includelist

# active_phase_dict['extended_computations_include_includelist'].remove('wcorr_shuffle_analysis')
active_phase_dict['should_freeze_pipeline_updates'] = False
# active_phase_dict['should_freeze_pipeline_updates'] = True # `should_freeze_pipeline_updates`
# active_phase_dict['should_create_vscode_workspace'] = True

# ## Default non-overriden for custom suffix:
# active_phase_dict['override_custom_pickle_suffix'] = '' # default, None
# active_phase_dict['force_recompute_override_computation_kwargs_dict'] = {}

# ==================================================================================================================== #
# Custom Suffix and overrides                                                                                          #
# ==================================================================================================================== #
# # '_withNormalComputedReplays-frateThresh_5.0-qclu_[1, 2]' ___________________________________________________________ #
# active_phase_dict['override_custom_pickle_suffix'] = '_withNormalComputedReplays-qclu_[1, 2]-frateThresh_5.0'
# included_qclu_values = [1,2]
# minimum_inclusion_fr_Hz = 5.0
# job_suffix = f"_withNormalComputedReplays-qclu_12-frateThresh_5.0" ## ends up being something like # job_suffix = f"_withNormalComputedReplays-qclu_12-frateThresh_5.0_tbin_25ms"


# # # "_withNormalComputedReplays-frateThresh_5.0-qclu_[1, 2, 4, 6, 7, 9]" _______________________________________________ #
# active_phase_dict['override_custom_pickle_suffix'] = "_withNormalComputedReplays-qclu_[1, 2, 4, 6, 7, 9]-frateThresh_5.0" ##
# included_qclu_values = [1, 2, 4, 6, 7, 9]
# minimum_inclusion_fr_Hz = 5.0
# job_suffix = f"_withNormalComputedReplays-qclu_124679-frateThresh_5.0"

# # "_withNormalComputedReplays-frateThresh_5.0-qclu_[1, 2]" _______________________________________________ #
# active_phase_dict['override_custom_pickle_suffix'] = "_withNormalComputedReplays-qclu_[1, 2]-frateThresh_5.0" 
# included_qclu_values = [1, 2]
# minimum_inclusion_fr_Hz = 5.0
# job_suffix = f"_withNormalComputedReplays-qclu_12-frateThresh_5.0"

# # "_withNormalComputedReplays-frateThresh_5.0-qclu_[1, 2, 4, 6, 7, 8, 9]" _______________________________________________ #
active_phase_dict['override_custom_pickle_suffix'] = "_withNormalComputedReplays-qclu_[1, 2, 4, 6, 7, 8, 9]-frateThresh_5.0"
included_qclu_values = [1, 2, 4, 6, 7, 8, 9]
minimum_inclusion_fr_Hz = 5.0
job_suffix = f"_withNormalComputedReplays-qclu_1246789-frateThresh_5.0"

# ==================================================================================================================== #
# Common:                                                                                                              #
# ==================================================================================================================== #
num_shuffles = 1024
drop_previous_result_and_compute_fresh = True

# laps_decoding_time_bin_size = None
# laps_decoding_time_bin_size = 0.025
# laps_decoding_time_bin_size = 0.025
laps_decoding_time_bin_size = 0.050

# ripple_decoding_time_bin_size = 0.058
# job_suffix='tbin_58ms'
# ripple_decoding_time_bin_size = 0.025
ripple_decoding_time_bin_size = 0.050
# job_suffix='tbin_50ms'
# ripple_decoding_time_bin_size = 0.025
# job_suffix='tbin_25ms'


# job_suffix = f"{active_phase_dict['override_custom_pickle_suffix']}_tbin_25ms"
# job_suffix = f"{job_suffix}_tbin_25ms"
job_suffix = f"{job_suffix}_tbin_50ms"



# END ________________________________________________________________________________________________________________ #

active_phase_dict['force_recompute_override_computation_kwargs_dict'] = {
    'merged_directional_placefields': {'laps_decoding_time_bin_size': laps_decoding_time_bin_size, 'ripple_decoding_time_bin_size': ripple_decoding_time_bin_size},
    'directional_decoders_evaluate_epochs': {'should_skip_radon_transform': False},
    'rank_order_shuffle_analysis': {'num_shuffles': num_shuffles, 'skip_laps': False, 'minimum_inclusion_fr_Hz':minimum_inclusion_fr_Hz, 'included_qclu_values':included_qclu_values},
    'perform_wcorr_shuffle_analysis': dict(num_shuffles=num_shuffles, drop_previous_result_and_compute_fresh=True),
    'non_PBE_epochs_results': dict(epochs_decoding_time_bin_size=ripple_decoding_time_bin_size, frame_divide_bin_size = 10.0, training_data_portion = 5.0/6.0,
                                compute_1D=True, compute_2D=False,
                                drop_previous_result_and_compute_fresh=drop_previous_result_and_compute_fresh, skip_training_test_split=True),
    'generalized_specific_epochs_decoding': dict(epochs_decoding_time_bin_size=ripple_decoding_time_bin_size, drop_previous_result_and_compute_fresh=drop_previous_result_and_compute_fresh),
}
# active_phase_dict['force_recompute_override_computations_includelist'] = list(active_phase_dict.get('force_recompute_override_computation_kwargs_dict', {}).keys())

# active_phase_dict['force_recompute_override_computations_includelist'] = [] # empty list so that everything isn't forced to recompute each time. NEEDS TO BE CHANGED WHEN qclus/fr_Hz change
# active_phase_dict['force_recompute_override_computations_includelist'] = ['directional_decoders_evaluate_epochs', 'directional_decoders_epoch_heuristic_scoring']
active_phase_dict['force_recompute_override_computations_includelist'] = list(active_phase_dict.get('force_recompute_override_computation_kwargs_dict', {}).keys())

active_phase_dict['override_parameters_flat_keypaths_dict'] = {'rank_order_shuffle_analysis.included_qclu_values': included_qclu_values, 'rank_order_shuffle_analysis.minimum_inclusion_fr_Hz': minimum_inclusion_fr_Hz,
} # need to provide `override_parameters_flat_keypaths_dict`

active_phase_dict['custom_user_completion_function_override_kwargs_dict'] = {
    'compute_and_export_session_alternative_replay_wcorr_shuffles_completion_function': dict(included_qclu_values=included_qclu_values, minimum_inclusion_fr_Hz=minimum_inclusion_fr_Hz, ripple_decoding_time_bin_size=ripple_decoding_time_bin_size, num_wcorr_shuffles=num_shuffles, drop_previous_result_and_compute_fresh=drop_previous_result_and_compute_fresh),
    # 'backup_previous_session_files_completion_function': dict(desired_suffix='Pre2024-11-21'),
    # 'compute_and_export_session_instantaneous_spike_rates_completion_function': dict(instantaneous_time_bin_size_seconds_list=[1000.0], epoch_handling_mode='UseAllEpochsMode', save_hdf=True, save_pickle=True, save_across_session_hdf=False),
    # 'compute_and_export_session_wcorr_shuffles_completion_function': dict(should_skip_previous_saved_shuffles=False, with_data_name=None),
    # 'perform_sweep_decoding_time_bin_sizes_marginals_dfs_completion_function': dict(save_hdf=True, desired_shared_decoding_time_bin_sizes=None, custom_all_param_sweep_options=None, additional_session_context=None),
    # 'compute_and_export_session_trial_by_trial_performance_completion_function': dict(active_laps_decoding_time_bin_size=0.25),
    'compute_and_export_decoders_epochs_decoding_and_evaluation_dfs_completion_function': dict(ripple_decoding_time_bin_size_override=ripple_decoding_time_bin_size, laps_decoding_time_bin_size_override=laps_decoding_time_bin_size,
        needs_recompute_heuristics=True, force_recompute_all_decoding=True,
        save_hdf=False, allow_append_to_session_h5_file=False,
        max_ignore_bins=2, same_thresh_cm=10.7, max_jump_distance_cm=60.0, # Heuristic settings
        ),
    'generalized_decode_epochs_dict_and_export_results_completion_function': dict(epochs_decoding_time_bin_size=ripple_decoding_time_bin_size, force_recompute=drop_previous_result_and_compute_fresh, debug_print=True),
    'export_decoded_context_uncertainty_by_position_completion_function': dict(time_bin_size=laps_decoding_time_bin_size, show_pos_by_ctxt_joint_figure=True),
    'figures_plot_generalized_decode_epochs_dict_and_export_results_completion_function': dict(),
}

# ==================================================================================================================== #
# Last-Chance Overrides                                                                                                #
# ==================================================================================================================== #
# Interactive Debugging in Notebooks:
active_phase_dict.update(dict(should_use_neptune_logging=False, should_create_vscode_workspace=True, 
	should_freeze_pipeline_updates=True,
))

# ## No custom suffix!
# active_phase_dict.update(dict(job_suffix=job_suffix, should_freeze_pipeline_updates=False, should_use_neptune_logging=False, should_generate_run_notebooks=True, should_create_vscode_workspace=True, create_slurm_scripts=True,
# ))


# Custom Job Suffix that will not interfere with running jobs ________________________________________________________ #
phase_job_suffix: str = active_phase_dict.pop('phase_job_suffix', None)
if phase_job_suffix is not None:
    job_suffix = f"{job_suffix}_{phase_job_suffix}" ## append phase to end of job_suffix
active_phase_dict.update(dict(job_suffix=job_suffix, should_freeze_pipeline_updates=True, should_use_neptune_logging=False, create_slurm_scripts=True, should_generate_run_scripts=True, should_generate_run_notebooks=True, should_create_vscode_workspace=True,
))
print(f'job_suffix: "{job_suffix}"')

# #TODO 2024-11-20 07:12: - [ ] Run with this, I hardcoded the parameter change for now
# {'ripple_decoding_time_bin_size': 0.058}

## Non-interactive batch computation in scripts
# active_phase_dict.update(dict(should_use_neptune_logging=True, should_generate_run_notebooks=False))
# active_phase_dict.update(dict(should_use_neptune_logging=True, should_generate_run_notebooks=False))

# END ________________________________________________________________________________________________________________ #


# ==================================================================================================================== #
# BEGIN MAIN SCRIPT BODY                                                                                               #
# ==================================================================================================================== #

## Build Slurm Scripts:
session_basedirs_dict: Dict[IdentifyingContext, Path] = {a_session_folder.context:a_session_folder.path for a_session_folder in good_session_concrete_folders}
batch_scripts_collection: BatchScriptsCollection = generate_batch_single_session_scripts(global_data_root_parent_path, session_batch_basedirs=session_basedirs_dict, included_session_contexts=included_session_contexts,
                                                                                        output_directory=scripts_output_path, use_separate_run_directories=True,
                                                                                        # should_freeze_pipeline_updates=False, 
                                                                                        # create_slurm_scripts=True, should_create_vscode_workspace=True,
                                                                                        # extended_computations_include_includelist=extended_computations_include_includelist,
                                                                                        # force_recompute_override_computations_includelist=force_recompute_override_computations_includelist, # ['split_to_directional_laps', 'rank_order_shuffle_analysis'],
                                                                                        should_use_file_redirected_output_logging=True,
                                                                                        create_non_slurm_bash_scripts=True,
                                                                                        # should_perform_figure_generation_to_file=False,
                                                                                        # batch_session_completion_handler_kwargs=dict(enable_hdf5_output=False),
                                                                                        # custom_user_completion_functions=custom_user_completion_functions,
                                                                                        # custom_user_completion_function_template_code=custom_user_completion_function_template_code,
                                                                                        # should_force_reload_all=True,
                                                                                        # should_force_reload_all=False,
                                                                                        **active_phase_dict
                                                                                    )


# batch_scripts_collection.included_session_contexts, output_python_scripts, output_slurm_scripts

output_python_scripts = batch_scripts_collection.output_python_scripts
output_jupyter_notebooks = batch_scripts_collection.output_jupyter_notebooks
output_slurm_scripts = batch_scripts_collection.output_slurm_scripts
vscode_workspace_path = batch_scripts_collection.vscode_workspace_path

if vscode_workspace_path is not None:
    print(f"vscode_workspace_path:: {vscode_workspace_path}")

print(f"extended_computations_include_includelist: {active_phase_dict['extended_computations_include_includelist']}")
computation_script_paths = [x[0] for x in output_python_scripts]
generate_figures_script_paths = [x[1] for x in output_python_scripts]
print(f'generate_figures_script_paths: {generate_figures_script_paths}')

# %%
## INPUTS: active_phase, output_slurm_scripts, computation_script_paths, generate_figures_script_paths
also_show_figure_script_outputs: bool = False

# Maximum number of parallel script executions
max_parallel_executions = 1
# List of your script paths
# if active_phase.value == ProcessingScriptPhases.figure_run:
if active_phase.is_figure_phase:
    print(f'fig mode!')
    script_paths = generate_figures_script_paths
else:
    script_paths = computation_script_paths


if (platform.system() == 'Windows'):
    powershell_script_path = build_windows_powershell_run_script(script_paths, max_concurrent_jobs=max_parallel_executions, script_name='run_scripts')
    # print(f'powershell_script_path: {powershell_script_path}')
    # display(fullwidth_path_widget(powershell_script_path, file_name_label='powershell_script_path:'))
    print(f"powershell_script_path: '{powershell_script_path}'\n")

    if also_show_figure_script_outputs or active_phase_dict['should_perform_figure_generation_to_file']:
        powershell_figures_script_path = build_windows_powershell_run_script(generate_figures_script_paths, max_concurrent_jobs=1, script_name='run_figure_scripts')
        # display(fullwidth_path_widget(powershell_figures_script_path, file_name_label='powershell_figures_script_path:'))
        print(f"powershell_figures_script_path: '{powershell_figures_script_path}'\n")

    print(f'\n\n')

# # Function to perform the copy to clipboard action
# def perform_copy_to_clipboard(code_to_copy: str):
# 	payload = f"navigator.clipboard.writeText(`{code_to_copy}`)"
# 	js_command = f"eval({payload})"
# 	display(widgets.HTML(value=f'<img src onerror="{js_command}">'))

# Function to execute code in terminal
# def perform_execute(b):
# 	# Use IPython's system command
# 	display(Javascript(f'IPython.notebook.kernel.execute("!{sbatch_run_start_slurm_scripts}")'))
     
if (platform.system() != 'Windows'):
    ## Linux Only: Slurm Scripts
    # sbatch_start_slurm_scripts: str = "\n".join([f"sbatch '{slurm_script}'" for slurm_script in output_slurm_scripts])
    sbatch_run_start_slurm_scripts: str = "\n".join([f"sbatch '{slurm_script}'" for slurm_script in output_slurm_scripts['run']])
    sbatch_figs_start_slurm_scripts: str = "\n".join([f"sbatch '{slurm_script}'" for slurm_script in output_slurm_scripts['figs']])

    if not active_phase.is_figure_phase:
        print("RunMain:")
        print(sbatch_run_start_slurm_scripts)
        print(f'\n\n')
        copy_to_clipboard(sbatch_run_start_slurm_scripts, message_print=True)

        if enable_auto_code_block_execution:
            print(f'executing code block!')
            execute_code_block(sbatch_run_start_slurm_scripts)

    if (active_phase.is_figure_phase or also_show_figure_script_outputs and (len(sbatch_figs_start_slurm_scripts)>0)):
        print("Figures:")
        print(sbatch_figs_start_slurm_scripts)
        print(f'\n\n')
        if (active_phase.is_figure_phase):
            copy_to_clipboard(sbatch_figs_start_slurm_scripts, message_print=True)
            if enable_auto_code_block_execution:
                print(f'executing code block!')
                execute_code_block(sbatch_figs_start_slurm_scripts)



# Batch 1
# sbatch '/nfs/turbo/umms-kdiba/Data/Output/gen_scripts/run_kdiba_pin01_one_11-02_17-46-44/run_kdiba_pin01_one_11-02_17-46-44.sh'
# sbatch '/nfs/turbo/umms-kdiba/Data/Output/gen_scripts/run_kdiba_pin01_one_11-02_19-28-0/run_kdiba_pin01_one_11-02_19-28-0.sh'
# sbatch '/nfs/turbo/umms-kdiba/Data/Output/gen_scripts/run_kdiba_pin01_one_11-03_12-3-25/run_kdiba_pin01_one_11-03_12-3-25.sh'
# sbatch '/nfs/turbo/umms-kdiba/Data/Output/gen_scripts/run_kdiba_pin01_one_fet11-01_12-58-54/run_kdiba_pin01_one_fet11-01_12-58-54.sh'

# Batch 2:
# sbatch '/nfs/turbo/umms-kdiba/Data/Output/gen_scripts/run_kdiba_vvp01_one_2006-4-09_17-29-30/run_kdiba_vvp01_one_2006-4-09_17-29-30.sh'
# sbatch '/nfs/turbo/umms-kdiba/Data/Output/gen_scripts/run_kdiba_vvp01_one_2006-4-10_12-25-50/run_kdiba_vvp01_one_2006-4-10_12-25-50.sh'
# sbatch '/nfs/turbo/umms-kdiba/Data/Output/gen_scripts/run_kdiba_vvp01_two_2006-4-09_16-40-54/run_kdiba_vvp01_two_2006-4-09_16-40-54.sh'
# sbatch '/nfs/turbo/umms-kdiba/Data/Output/gen_scripts/run_kdiba_vvp01_two_2006-4-10_12-58-3/run_kdiba_vvp01_two_2006-4-10_12-58-3.sh'

# Batch 3
# sbatch '/nfs/turbo/umms-kdiba/Data/Output/gen_scripts/run_kdiba_gor01_one_2006-6-08_14-26-15/run_kdiba_gor01_one_2006-6-08_14-26-15.sh'
# sbatch '/nfs/turbo/umms-kdiba/Data/Output/gen_scripts/run_kdiba_gor01_one_2006-6-09_1-22-43/run_kdiba_gor01_one_2006-6-09_1-22-43.sh'
# sbatch '/nfs/turbo/umms-kdiba/Data/Output/gen_scripts/run_kdiba_gor01_two_2006-6-07_16-40-19/run_kdiba_gor01_two_2006-6-07_16-40-19.sh'
# sbatch '/nfs/turbo/umms-kdiba/Data/Output/gen_scripts/run_kdiba_gor01_two_2006-6-08_21-16-25/run_kdiba_gor01_two_2006-6-08_21-16-25.sh'
# sbatch '/nfs/turbo/umms-kdiba/Data/Output/gen_scripts/run_kdiba_gor01_two_2006-6-09_22-24-40/run_kdiba_gor01_two_2006-6-09_22-24-40.sh'
# sbatch '/nfs/turbo/umms-kdiba/Data/Output/gen_scripts/run_kdiba_gor01_two_2006-6-12_16-53-46/run_kdiba_gor01_two_2006-6-12_16-53-46.sh'