add time before job 2 submission

Author: Himani Anil Deshpande

tests/integration-tests/tests/common/scaling_common.py

@@ -414,8 +414,11 @@ def setup_create_fleet_override_to_emulate_ice(
     """
     ice_instance_types = [f"ICE-{it}" for it in instance_types]
     overrides = _build_create_fleet_override(cluster_name, queue, compute_resource, ice_instance_types, subnet_id)
-    logging.info("Writing create_fleet_overrides.json with invalid InstanceTypes to emulate ICE "
-                 "for queue=%s, cr=%s", queue, compute_resource)
+    logging.info(
+        "Writing create_fleet_overrides.json with invalid InstanceTypes to emulate ICE " "for queue=%s, cr=%s",
+        queue,
+        compute_resource,
+    )
     _write_json_override(remote_command_executor, CREATE_FLEET_OVERRIDES_PATH, overrides)
 
 
@@ -430,6 +433,10 @@ def recover_create_fleet_override_from_ice(
     call succeeds.
     """
     overrides = _build_create_fleet_override(cluster_name, queue, compute_resource, real_instance_types, subnet_id)
-    logging.info("Recovering from ICE: writing real InstanceTypes=%s in create_fleet_overrides.json "
-                 "for queue=%s, cr=%s", real_instance_types, queue, compute_resource)
+    logging.info(
+        "Recovering from ICE: writing real InstanceTypes=%s in create_fleet_overrides.json " "for queue=%s, cr=%s",
+        real_instance_types,
+        queue,
+        compute_resource,
+    )
     _write_json_override(remote_command_executor, CREATE_FLEET_OVERRIDES_PATH, overrides)

tests/integration-tests/tests/schedulers/test_slurm.py

@@ -659,6 +659,7 @@ def _submit_jobs_and_simulate_ice(common_cluster_details, jobs):
                 "other_options": f"{requeue_opt}--exclusive",
             }
         )
+        time.sleep(10)
         logging.info("Submitted %s (%s) ID: %s", job["label"], job_type, jid)
         job_ids.append(jid)
 
@@ -770,26 +771,26 @@ def test_expedited_requeue(
     }
 
     # Submit 3 jobs in a single ICE cycle:
-    # job1 (normal), job2 (expedited), job3 (expedited)
+    # job1 (expedited), job2 (normal), job3 (expedited)
+    # TODO: Improve the test by making Job 2 as expedited and Job 1 as normal
+    #  so that its clear that Job 2 goes at the top of the queue
     jobs = [
-        {"label": "job1", "expedited": False},
-        {"label": "job2", "expedited": True},
+        {"label": "job1", "expedited": True},
+        {"label": "job2", "expedited": False},
         # {"label": "job3", "expedited": True},
     ]
     job_ids = _submit_jobs_and_simulate_ice(common_cluster_details, jobs)
     _recover_from_ice_and_wait_for_jobs(common_cluster_details, job_ids)
     start_epochs = _collect_start_epochs(remote_command_executor, job_ids)
 
     # Expected ordering after ICE recovery:
-    # Expedited jobs (job2, job3) run first in submission order, then normal job (job1) last
+    # Expedited jobs (job1, job3) run first in submission order, then normal job (job2) last
     logging.info("Start epochs: %s", dict(zip([j["label"] for j in jobs], start_epochs)))
 
-
-    assert_that(start_epochs[2]).is_less_than_or_equal_to(start_epochs[1])  # job2 (expedited) before job1 (normal)
-    # assert_that(start_epochs[3]).is_less_than_or_equal_to(start_epochs[1])  # job3 (expedited) before job1 (normal)
-    # assert_that(start_epochs[2]).is_less_than_or_equal_to(start_epochs[3])  # job2 (expedited) before job3 (expedited)
-    logging.info("Verified: expedited jobs (job2, job3) ran before normal job (job1)")
-
+    assert_that(start_epochs[0]).is_less_than_or_equal_to(start_epochs[1])  # job1 (expedited) before job2 (normal)
+    # assert_that(start_epochs[2]).is_less_than_or_equal_to(start_epochs[1])  # job3 (expedited) before job2 (normal)
+    # assert_that(start_epochs[0]).is_less_than_or_equal_to(start_epochs[2])  # job1 (expedited) before job3 (expedited)
+    logging.info("Verified: expedited jobs (job1) ran before normal job (job1)")
 
 
 @pytest.mark.usefixtures("region", "os", "instance", "scheduler")