Merge pull request #10 from EvoTestOps/directory-path-improvements

Directory path improvements

Author: saimouu

.gitignore

@@ -1,5 +1,8 @@
 test_data
-log_data
+
+log_data/*
+!log_data/LO2/
+
 output
 __pycache__/
 .env

dash_app/callbacks/callback_functions.py

@@ -81,7 +81,8 @@ def run_anomaly_detection(
     test_data,
     detectors,
     enhancement,
-    include_items,
+    include_test_items,
+    include_train_items,
     mask_type,
     vectorizer_type,
     analysis_name=None,
@@ -94,7 +95,8 @@ def run_anomaly_detection(
         log_format,
         detectors,
         enhancement,
-        include_items,
+        include_test_items,
+        include_train_items,
         mask_type,
         vectorizer_type,
         analysis_name,
@@ -215,8 +217,9 @@ def fetch_project_name(project_id: int) -> str:
     return response.json().get("name", "404")
 
 
-def get_log_data_directory_options():
-    labels, values = get_all_root_log_directories()
+def get_log_data_directory_options(project_id: int):
+    base_path = _fetch_project_base_path(project_id).get("base_path")
+    labels, values = get_all_root_log_directories(base_path)
     return [{"label": lab, "value": val} for (lab, val) in zip(labels, values)]
 
 
@@ -238,6 +241,15 @@ def fetch_project_settings(project_id: int) -> dict:
     return response.json()
 
 
+def _fetch_project_base_path(project_id: int) -> dict:
+    response, error = make_api_call({}, f"projects/{project_id}/base_path", "GET")
+    if error or response is None:
+        raise ValueError(
+            f"Was not able to retrieve base_path for project id {project_id}: {error}"
+        )
+    return response.json()
+
+
 def _fetch_analysis_results(analysis_id: int) -> bytes:
     response, error = make_api_call({}, f"analyses/{analysis_id}", "GET")
     if error or response is None:
@@ -264,7 +276,8 @@ def _build_test_train_payload(
     log_format,
     detectors,
     enhancement,
-    include_items,
+    include_test_items,
+    include_train_items,
     mask_type,
     vectorizer_type,
     analysis_name,
@@ -282,17 +295,20 @@ def _build_test_train_payload(
     }
 
     if level == "directory":
-        payload["runs_to_include"] = include_items
+        payload["runs_to_include"] = include_test_items
+        payload["runs_to_include_train"] = include_train_items
         payload["run_level"] = True
     elif level == "file":
-        payload["files_to_include"] = include_items
+        payload["files_to_include"] = include_test_items
+        payload["files_to_include_train"] = include_train_items
         payload["file_level"] = True
     elif level == "line":
-        payload["runs_to_include"] = include_items
+        payload["runs_to_include"] = include_test_items
+        payload["runs_to_include_train"] = include_train_items
         payload["file_level"] = False
         payload["run_level"] = False
     else:
-        raise ValueError("Level must be either 'file' or 'directory'")
+        raise ValueError("Level must be 'line', 'file' or 'directory'")
 
     return payload
 

dash_app/components/form_inputs.py

@@ -93,7 +93,7 @@ def delete_button(id, label):
     )
 
 
-def runs_filter_input(id, manual=False):
+def runs_filter_input(id, label=None, manual=False):
     if manual:
         return dbc.Col(
             [
@@ -111,7 +111,11 @@ def runs_filter_input(id, manual=False):
         )
     return dbc.Col(
         [
-            dbc.Label("Runs to include in test data", html_for=id, width="auto"),
+            dbc.Label(
+                label if label else "Directories to include in test data",
+                html_for=id,
+                width="auto",
+            ),
             dcc.Dropdown(
                 multi=True,
                 id=id,
@@ -124,12 +128,12 @@ def runs_filter_input(id, manual=False):
     )
 
 
-def files_filter_input(id, manual=False):
+def files_filter_input(id, label=None, manual=False):
     if manual:
         return dbc.Col(
             [
                 dbc.Label(
-                    "Files to include in test data (manual, comma-separated)",
+                    f"{label if label else 'Files to include in test data'} (manual, comma-separated)",
                     html_for=id,
                     width="auto",
                 ),
@@ -142,7 +146,11 @@ def files_filter_input(id, manual=False):
         )
     return dbc.Col(
         [
-            dbc.Label("Files to include in test data", html_for=id, width="auto"),
+            dbc.Label(
+                label if label else "Files to include in test data",
+                html_for=id,
+                width="auto",
+            ),
             dcc.Dropdown(
                 multi=True,
                 id=id,
@@ -405,3 +413,28 @@ def analysis_name_input(id):
             dbc.Input(id=id, placeholder="Optional", type="text"),
         ],
     )
+
+
+def base_path_input(id):
+    label_id = f"{id}-label"
+    return dbc.Col(
+        [
+            dbc.Label(
+                "Base path",
+                id=label_id,
+                html_for=id,
+                width="auto",
+                style={"textDecoration": "underline", "cursor": "pointer"},
+            ),
+            dbc.Tooltip(
+                "Optional. Must be directory inside log_data/. Leave empty or unchanged to use the default. Directory must already exist.",
+                target=label_id,
+                placement="bottom",
+            ),
+            dbc.Input(
+                id=id,
+                type="text",
+                value="./log_data",
+            ),
+        ],
+    )

dash_app/components/forms.py

@@ -22,6 +22,7 @@
     files_to_include_input,
     columns_to_include_input,
     analysis_name_input,
+    base_path_input,
 )
 
 
@@ -32,6 +33,7 @@ def test_train_form(
     detectors_id,
     enhancement_id,
     runs_filter_id,
+    runs_filter_train_id,
     mask_input_id,
     vectorizer_id,
     results_redirect_id,
@@ -61,15 +63,24 @@ def test_train_form(
             ),
             dbc.Row(
                 [
-                    detectors_unsupervised_input(detectors_id),
+                    runs_filter_input(
+                        id=runs_filter_train_id,
+                        label="Directories to include in train data",
+                    ),
                     runs_filter_input(runs_filter_id),
                 ],
                 class_name="mb-3",
             ),
             dbc.Row(
                 [
-                    enhancement_input(enhancement_id),
+                    detectors_unsupervised_input(detectors_id),
                     vectorizer_input(vectorizer_id),
+                ],
+                class_name="mb-3",
+            ),
+            dbc.Row(
+                [
+                    enhancement_input(enhancement_id),
                     mask_input(mask_input_id),
                 ],
                 class_name="mb-3",
@@ -90,6 +101,7 @@ def test_train_file_level_form(
     detectors_id,
     enhancement_id,
     runs_filter_id,
+    runs_filter_train_id,
     mask_input_id,
     vectorizer_id,
     results_redirect_id,
@@ -122,6 +134,20 @@ def test_train_file_level_form(
                 ],
                 class_name="mb-3",
             ),
+            dbc.Row(
+                dcc.Loading(
+                    type="circle",
+                    overlay_style={"visibility": "visible", "filter": "blur(2px)"},
+                    children=[
+                        files_filter_input(
+                            runs_filter_train_id,
+                            label="Files to include in train data",
+                            manual=manual_filenames,
+                        ),
+                    ],
+                ),
+                class_name="mb-3",
+            ),
             dbc.Row(
                 dcc.Loading(
                     type="circle",
@@ -340,13 +366,14 @@ def distance_file_level_form(
     return form
 
 
-def project_form(submit_id, name_id):
+def project_form(submit_id, name_id, base_path_id):
     submit_btn = submit_button(submit_id, "Create")
     form = dbc.Form(
         [
             dbc.Row(
                 [
-                    name_input((name_id)),
+                    dbc.Col(name_input((name_id))),
+                    dbc.Col(base_path_input((base_path_id))),
                     dbc.Col(
                         submit_btn,
                         class_name="d-flex justify-content-end align-middle align-items-end",

dash_app/components/layouts.py

@@ -377,7 +377,7 @@ def create_project_layout(
 
     task_error_modal = dbc.Row(dbc.Modal(id=task_error_modal_id, is_open=False))
     task_logs_modal = dbc.Row(
-        dbc.Modal(id=task_logs_modal_id, size="lg", is_open=False)
+        dbc.Modal(id=task_logs_modal_id, size="xl", is_open=False)
     )
 
     analysis_edit_name_modal = dbc.Row(
@@ -454,7 +454,7 @@ def create_high_level_viz_result_layout(
 
     layout = [
         dbc.Container([table_row, error_toast_row, success_toast_row]),
-        dbc.Container(plot_row, fluid=True),
+        dbc.Container(plot_row, fluid=True, style={"paddingBottom": "200px"}),
     ]
 
     return layout

dash_app/page_templates/new_analysis_page_base.py

@@ -103,20 +103,20 @@ def get_project_id(search):
 
         @callback(
             Output(form_ids["directory_id"], "options"),
-            Input(base_ids["url_id"], "search"),
+            Input(base_ids["project_store_id"], "data"),
         )
-        def get_log_data_directories(_):
-            return get_log_data_directory_options()
+        def get_log_data_directories(project_id):
+            return get_log_data_directory_options(project_id)
 
     else:
 
         @callback(
             Output(form_ids["train_data_id"], "options"),
             Output(form_ids["test_data_id"], "options"),
-            Input(base_ids["url_id"], "search"),
+            Input(base_ids["project_store_id"], "data"),
         )
-        def get_log_data_directories(_):
-            options = get_log_data_directory_options()
+        def get_log_data_directories(project_id):
+            options = get_log_data_directory_options(project_id)
             return options, options
 
     if (
@@ -144,15 +144,24 @@ def get_comparison_and_target_options(directory_path, manual_filenames):
 
         @callback(
             Output(form_ids["runs_filter_id"], "options"),
+            Output(form_ids["runs_filter_train_id"], "options"),
             Input(form_ids["test_data_id"], "value"),
+            Input(form_ids["train_data_id"], "value"),
             State(base_ids["manual_filenames_id"], "data"),
         )
-        def get_comparison_options(directory_path, manual_filenames):
+        def get_comparison_options(dir_path_test, dir_path_train, manual_filenames):
             if manual_filenames:
                 return [], []
             runs_or_files = "files" if config["level"] == "file" else "runs"
-            options = get_filter_options(directory_path, runs_or_files=runs_or_files)
-            return options
+
+            options_test = get_filter_options(
+                dir_path_test, runs_or_files=runs_or_files
+            )
+            options_train = get_filter_options(
+                dir_path_train, runs_or_files=runs_or_files
+            )
+
+            return options_test, options_train
 
     @callback(
         Output(base_ids["error_toast_id"], "children", allow_duplicate=True),

dash_app/pages/create_analysis_pages/new_ano_directory_level.py

@@ -28,6 +28,7 @@
         "detectors_id": "detectors-ano-dir-new",
         "enhancement_id": "enhancement-ano-dir-new",
         "runs_filter_id": "filter-ano-dir-new",
+        "runs_filter_train_id": "filter-train-ano-dir-new",
         "mask_input_id": "mask-ano-dir-new",
         "vectorizer_id": "vectorizer-ano-dir-new",
         "results_redirect_id": "results-redirect-ano-dir-new",
@@ -39,6 +40,7 @@
         "detectors_id",
         "enhancement_id",
         "runs_filter_id",
+        "runs_filter_train_id",
         "mask_input_id",
         "vectorizer_id",
         "results_redirect_id",

dash_app/pages/create_analysis_pages/new_ano_file_level.py

@@ -31,6 +31,7 @@
         "detectors_id": "detectors-ano-file-new",
         "enhancement_id": "enhancement-ano-file-new",
         "runs_filter_id": "filter-ano-file-new",
+        "runs_filter_train_id": "filter-train-ano-file-new",
         "mask_input_id": "mask-ano-file-new",
         "vectorizer_id": "vectorizer-ano-file-new",
         "results_redirect_id": "results-redirect-ano-file-new",
@@ -42,6 +43,7 @@
         "detectors_id",
         "enhancement_id",
         "runs_filter_id",
+        "runs_filter_train_id",
         "mask_input_id",
         "vectorizer_id",
         "results_redirect_id",

dash_app/pages/create_analysis_pages/new_ano_line_level.py

@@ -28,6 +28,7 @@
         "detectors_id": "detectors-ano-line-new",
         "enhancement_id": "enhancement-ano-line-new",
         "runs_filter_id": "filter-ano-line-new",
+        "runs_filter_train_id": "filter-train-ano-line-new",
         "mask_input_id": "mask-ano-line-new",
         "vectorizer_id": "vectorizer-ano-line-new",
         "results_redirect_id": "results-redirect-ano-line-new",
@@ -39,6 +40,7 @@
         "detectors_id",
         "enhancement_id",
         "runs_filter_id",
+        "runs_filter_train_id",
         "mask_input_id",
         "vectorizer_id",
         "results_redirect_id",