integration test

Author: iliana

nexus/test-utils/src/background.rs

@@ -32,6 +32,44 @@ fn most_recent_activate_time(
     }
 }
 
+/// Given the name of a background task, wait for it to complete if it's
+/// running, then return the last polled `BackgroundTask` object. Panics if the
+/// task has never been activated.
+pub async fn wait_background_task(
+    internal_client: &ClientTestContext,
+    task_name: &str,
+) -> BackgroundTask {
+    // Wait for the task to finish
+    let last_task_poll = wait_for_condition(
+        || async {
+            let task = NexusRequest::object_get(
+                internal_client,
+                &format!("/bgtasks/view/{task_name}"),
+            )
+            .execute_and_parse_unwrap::<BackgroundTask>()
+            .await;
+
+            // Wait until the task has actually run and then is idle
+            if matches!(&task.current, CurrentStatus::Idle) {
+                match &task.last {
+                    LastResult::Completed(_) => Ok(task),
+                    LastResult::NeverCompleted => {
+                        panic!("task never activated")
+                    }
+                }
+            } else {
+                Err(CondCheckError::<()>::NotYet)
+            }
+        },
+        &Duration::from_millis(500),
+        &Duration::from_secs(60),
+    )
+    .await
+    .unwrap();
+
+    last_task_poll
+}
+
 /// Given the name of a background task, activate it, then wait for it to
 /// complete. Return the last polled `BackgroundTask` object.
 pub async fn activate_background_task(
@@ -337,3 +375,68 @@ pub async fn run_replacement_tasks_to_completion(
     .await
     .unwrap();
 }
+
+pub async fn wait_tuf_artifact_replication_step(
+    internal_client: &ClientTestContext,
+) -> TufArtifactReplicationStatus {
+    let last_background_task =
+        wait_background_task(&internal_client, "tuf_artifact_replication")
+            .await;
+
+    let LastResult::Completed(last_result_completed) =
+        last_background_task.last
+    else {
+        panic!(
+            "unexpected {:?} returned from tuf_artifact_replication task",
+            last_background_task.last,
+        );
+    };
+
+    let status = serde_json::from_value::<TufArtifactReplicationStatus>(
+        last_result_completed.details,
+    )
+    .unwrap();
+    assert_eq!(status.requests_err, 0);
+    status
+}
+
+pub async fn run_tuf_artifact_replication_step(
+    internal_client: &ClientTestContext,
+) -> TufArtifactReplicationStatus {
+    let last_background_task =
+        activate_background_task(&internal_client, "tuf_artifact_replication")
+            .await;
+
+    let LastResult::Completed(last_result_completed) =
+        last_background_task.last
+    else {
+        panic!(
+            "unexpected {:?} returned from tuf_artifact_replication task",
+            last_background_task.last,
+        );
+    };
+
+    let status = serde_json::from_value::<TufArtifactReplicationStatus>(
+        last_result_completed.details,
+    )
+    .unwrap();
+    assert_eq!(status.requests_err, 0);
+    status
+}
+
+/// Run the `tuf_artifact_replication` task until the status indicates the task
+/// has stabilized: no outstanding requests, and no local repositories. Panics
+/// if the status does not change between runs.
+pub async fn run_tuf_artifact_replication_to_completion(
+    internal_client: &ClientTestContext,
+) {
+    let mut status = run_tuf_artifact_replication_step(internal_client).await;
+    while status.requests_outstanding > 0 || status.local_repos > 0 {
+        let new_status =
+            run_tuf_artifact_replication_step(internal_client).await;
+        if new_status == status {
+            panic!("TUF artifact replication stalled: {new_status:?}");
+        }
+        status = new_status;
+    }
+}

nexus/tests/config.test.toml

@@ -151,7 +151,9 @@ region_snapshot_replacement_start.period_secs = 60
 region_snapshot_replacement_garbage_collection.period_secs = 60
 region_snapshot_replacement_step.period_secs = 60
 region_snapshot_replacement_finish.period_secs = 60
-tuf_artifact_replication.period_secs = 60
+# The default activation period for this task is 60s, but we want to activate it
+# manually to test the result of each activation.
+tuf_artifact_replication.period_secs = 600
 
 [default_region_allocation_strategy]
 # we only have one sled in the test environment, so we need to use the