From 9e836f5f31916a2ef535c22c26465468fbdc2dfb Mon Sep 17 00:00:00 2001 From: Delan Azabani Date: Mon, 8 Dec 2025 14:59:57 +0800 Subject: [PATCH 1/4] monitor: register runners with unique labels --- Cargo.lock | 2 ++ monitor/Cargo.toml | 1 + monitor/src/github.rs | 12 ++++++++++-- monitor/src/image.rs | 31 ++++++++++++++++++++++--------- monitor/src/image/macos13.rs | 8 ++------ monitor/src/image/ubuntu2204.rs | 4 ++-- monitor/src/image/windows10.rs | 4 ++-- monitor/src/policy.rs | 5 ++++- monitor/src/runner.rs | 2 ++ 9 files changed, 47 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 292b9896..c661d34e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1246,6 +1246,7 @@ dependencies = [ "toml", "tracing", "tracing-subscriber 0.3.18", + "uuid", "web", ] @@ -2539,6 +2540,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d" dependencies = [ "getrandom 0.2.15", + "serde", ] [[package]] diff --git a/monitor/Cargo.toml b/monitor/Cargo.toml index b25c817e..ce4db6b8 100644 --- a/monitor/Cargo.toml +++ b/monitor/Cargo.toml @@ -31,6 +31,7 @@ tracing = { workspace = true } tracing-subscriber = { workspace = true } web = { workspace = true } rand = "0.9.1" +uuid = { version = "1.4.1", features = ["serde", "v4"] } [dev-dependencies] settings = { workspace = true, features = ["test"] } diff --git a/monitor/src/github.rs b/monitor/src/github.rs index 0ef0bbe1..ff6efc42 100644 --- a/monitor/src/github.rs +++ b/monitor/src/github.rs @@ -124,13 +124,21 @@ pub fn list_registered_runners_for_host() -> eyre::Result> { Ok(result.collect()) } -pub fn register_runner(runner_name: &str, label: &str, work_folder: &str) -> eyre::Result { +pub fn register_runner( + runner_name: &str, + work_folder: &str, + labels: &[String], +) -> eyre::Result { let github_api_suffix = &TOML.github_api_suffix; let github_api_scope = &TOML.github_api_scope; + let label_options = labels + .into_iter() + .flat_map(|label| ["-f".to_owned(), format!("labels[]={label}")]) + .collect::>(); let result = run_fun!(gh api --method POST -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "$github_api_scope/actions/runners/generate-jitconfig" -f "name=$runner_name@$github_api_suffix" -F "runner_group_id=1" -f "work_folder=$work_folder" - -f "labels[]=self-hosted" -f "labels[]=X64" -f "labels[]=$label")?; + -f "labels[]=self-hosted" $[label_options])?; Ok(result) } diff --git a/monitor/src/image.rs b/monitor/src/image.rs index b6fb5646..82595ea6 100644 --- a/monitor/src/image.rs +++ b/monitor/src/image.rs @@ -24,6 +24,7 @@ use settings::{ TOML, }; use tracing::{debug, error, info, warn}; +use uuid::Uuid; use crate::{ libvirt::{list_rebuild_guests, list_template_guests}, @@ -298,16 +299,28 @@ pub fn delete_template(profile: &Profile, snapshot_name: &str) -> eyre::Result<( } } -pub fn register_runner(profile: &Profile, runner_guest_name: &str) -> eyre::Result { +pub fn register_runner( + profile: &Profile, + runner_guest_name: &str, + runner_uuid: Uuid, +) -> eyre::Result { + let labels = vec![ + format!("self-hosted-profile:{}", profile.profile_name), + format!( + "self-hosted-runner:{}@{}", + runner_guest_name, TOML.github_api_suffix + ), + format!("self-hosted-uuid:{}", runner_uuid), + ]; match &*profile.profile_name { - "servo-macos13" => macos13::register_runner(profile, runner_guest_name), - "servo-macos14" => macos13::register_runner(profile, runner_guest_name), - "servo-macos15" => macos13::register_runner(profile, runner_guest_name), - "servo-ubuntu2204" => ubuntu2204::register_runner(profile, runner_guest_name), - "servo-ubuntu2204-bench" => ubuntu2204::register_runner(profile, runner_guest_name), - "base-ubuntu2204" => ubuntu2204::register_runner(profile, runner_guest_name), - "servo-ubuntu2204-wpt" => ubuntu2204::register_runner(profile, runner_guest_name), - "servo-windows10" => windows10::register_runner(profile, runner_guest_name), + "servo-macos13" => macos13::register_runner(runner_guest_name, &labels), + "servo-macos14" => macos13::register_runner(runner_guest_name, &labels), + "servo-macos15" => macos13::register_runner(runner_guest_name, &labels), + "servo-ubuntu2204" => ubuntu2204::register_runner(runner_guest_name, &labels), + "servo-ubuntu2204-bench" => ubuntu2204::register_runner(runner_guest_name, &labels), + "base-ubuntu2204" => ubuntu2204::register_runner(runner_guest_name, &labels), + "servo-ubuntu2204-wpt" => ubuntu2204::register_runner(runner_guest_name, &labels), + "servo-windows10" => windows10::register_runner(runner_guest_name, &labels), other => todo!("Runner registration not yet implemented: {other}"), } } diff --git a/monitor/src/image/macos13.rs b/monitor/src/image/macos13.rs index e4885f53..50e8bf0b 100644 --- a/monitor/src/image/macos13.rs +++ b/monitor/src/image/macos13.rs @@ -92,12 +92,8 @@ pub(super) fn delete_template(profile: &Profile, snapshot_name: &str) -> eyre::R Ok(()) } -pub fn register_runner(profile: &Profile, runner_guest_name: &str) -> eyre::Result { - monitor::github::register_runner( - runner_guest_name, - &profile.github_runner_label, - "/Users/servo/a", - ) +pub fn register_runner(runner_guest_name: &str, labels: &[String]) -> eyre::Result { + monitor::github::register_runner(runner_guest_name, "/Users/servo/a", labels) } pub fn create_runner( diff --git a/monitor/src/image/ubuntu2204.rs b/monitor/src/image/ubuntu2204.rs index 96f4c99f..8055c4ba 100644 --- a/monitor/src/image/ubuntu2204.rs +++ b/monitor/src/image/ubuntu2204.rs @@ -100,8 +100,8 @@ pub(super) fn delete_template(profile: &Profile, snapshot_name: &str) -> eyre::R Ok(()) } -pub fn register_runner(profile: &Profile, runner_guest_name: &str) -> eyre::Result { - monitor::github::register_runner(runner_guest_name, &profile.github_runner_label, "/a") +pub fn register_runner(runner_guest_name: &str, labels: &[String]) -> eyre::Result { + monitor::github::register_runner(runner_guest_name, "/a", labels) } pub fn create_runner( diff --git a/monitor/src/image/windows10.rs b/monitor/src/image/windows10.rs index 07b818de..f807d671 100644 --- a/monitor/src/image/windows10.rs +++ b/monitor/src/image/windows10.rs @@ -106,8 +106,8 @@ pub(super) fn delete_template(profile: &Profile, snapshot_name: &str) -> eyre::R Ok(()) } -pub fn register_runner(profile: &Profile, runner_guest_name: &str) -> eyre::Result { - monitor::github::register_runner(runner_guest_name, &profile.github_runner_label, r"C:\a") +pub fn register_runner(runner_guest_name: &str, labels: &[String]) -> eyre::Result { + monitor::github::register_runner(runner_guest_name, r"C:\a", labels) } pub fn create_runner( diff --git a/monitor/src/policy.rs b/monitor/src/policy.rs index eff1534b..c251eea8 100644 --- a/monitor/src/policy.rs +++ b/monitor/src/policy.rs @@ -21,6 +21,7 @@ use settings::{ TOML, }; use tracing::{debug, info, info_span, warn}; +use uuid::Uuid; use crate::{ data::{get_profile_configuration_path, get_profile_data_path, get_runner_data_path}, @@ -283,16 +284,18 @@ impl Policy { match profile.image_type { ImageType::Rust => { create_dir(get_runner_data_path(id, None)?)?; + let runner_uuid = Uuid::new_v4(); let mut runner_toml = File::create_new(get_runner_data_path(id, Path::new("runner.toml"))?)?; writeln!(runner_toml, r#"image_type = "Rust""#)?; + writeln!(runner_toml, r#"runner_uuid = "{}""#, runner_uuid)?; symlink( get_profile_configuration_path(&profile, Path::new("boot-script"))?, get_runner_data_path(id, Path::new("boot-script"))?, )?; if !TOML.dont_register_runners() { let github_api_registration = - register_runner(&profile, &runner_guest_name)?; + register_runner(&profile, &runner_guest_name, runner_uuid)?; let mut github_api_registration_file = File::create_new( get_runner_data_path(id, Path::new("github-api-registration"))?, )?; diff --git a/monitor/src/runner.rs b/monitor/src/runner.rs index 3c34906c..15dbd667 100644 --- a/monitor/src/runner.rs +++ b/monitor/src/runner.rs @@ -16,6 +16,7 @@ use monitor::github::{reserve_runner, ApiRunner}; use serde::{Deserialize, Serialize}; use settings::{profile::ImageType, TOML}; use tracing::{error, info, trace, warn}; +use uuid::Uuid; use crate::{ data::get_runner_data_path, @@ -43,6 +44,7 @@ pub struct Runner { #[derive(Debug, Default, Deserialize, Serialize, Clone)] pub struct RunnerDetails { image_type: ImageType, + runner_uuid: Uuid, } #[derive(Clone, Debug, PartialEq)] From 48b201a6e23564586ff3c6040da8688bfd470f0a Mon Sep 17 00:00:00 2001 From: Delan Azabani Date: Mon, 8 Dec 2025 15:51:31 +0800 Subject: [PATCH 2/4] monitor: reserve runners using local file --- monitor/src/github.rs | 19 +---------------- monitor/src/runner.rs | 49 +++++++++++++++++++++++++++++++++++++++---- 2 files changed, 46 insertions(+), 22 deletions(-) diff --git a/monitor/src/github.rs b/monitor/src/github.rs index ff6efc42..91c535e1 100644 --- a/monitor/src/github.rs +++ b/monitor/src/github.rs @@ -1,6 +1,6 @@ use std::{ fmt::Debug, - time::{Duration, Instant, SystemTime, UNIX_EPOCH}, + time::{Duration, Instant}, }; use chrono::{DateTime, FixedOffset}; @@ -151,23 +151,6 @@ pub fn unregister_runner(id: usize) -> eyre::Result<()> { Ok(()) } -pub fn reserve_runner( - id: usize, - unique_id: &str, - reserved_since: SystemTime, - reserved_by: &str, -) -> eyre::Result<()> { - let github_api_scope = &TOML.github_api_scope; - let reserved_since = reserved_since.duration_since(UNIX_EPOCH)?.as_secs(); - run_cmd!(gh api --method POST -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" - "$github_api_scope/actions/runners/$id/labels" - -f "labels[]=reserved-for:$unique_id" - -f "labels[]=reserved-since:$reserved_since" - -f "labels[]=reserved-by:$reserved_by")?; - - Ok(()) -} - pub fn list_workflow_run_artifacts( qualified_repo: &str, run_id: &str, diff --git a/monitor/src/runner.rs b/monitor/src/runner.rs index 15dbd667..122a99ff 100644 --- a/monitor/src/runner.rs +++ b/monitor/src/runner.rs @@ -2,7 +2,7 @@ use std::{ collections::{BTreeMap, BTreeSet}, fmt::{Debug, Display}, fs::File, - io::Read, + io::{Read, Write}, net::Ipv4Addr, path::Path, time::{Duration, SystemTime, UNIX_EPOCH}, @@ -12,7 +12,7 @@ use cfg_if::cfg_if; use itertools::Itertools; use jane_eyre::eyre::{self, bail}; use mktemp::Temp; -use monitor::github::{reserve_runner, ApiRunner}; +use monitor::github::ApiRunner; use serde::{Deserialize, Serialize}; use settings::{profile::ImageType, TOML}; use tracing::{error, info, trace, warn}; @@ -38,6 +38,7 @@ pub struct Runner { ipv4_address: Option, #[serde(skip)] github_jitconfig: Option, + reservation: Option, details: RunnerDetails, } @@ -57,6 +58,13 @@ pub enum Status { DoneOrUnregistered, } +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct Reservation { + reserved_since: u64, + unique_id: String, + run_url: String, +} + impl Runners { pub fn new(registrations: Vec, guest_names: Vec) -> Self { // Gather all known runner ids with live resources. @@ -140,8 +148,19 @@ impl Runners { bail!("Tried to reserve an unregistered runner"); }; info!(runner_id = id, registration.id, "Reserving runner"); - let reserved_by = format!("{qualified_repo}/actions/runs/{run_id}"); - reserve_runner(registration.id, unique_id, SystemTime::now(), &reserved_by) + let mut reservation = + File::create_new(get_runner_data_path(id, Path::new("reservation.toml"))?)?; + writeln!( + reservation, + r#"reserved_since = {}"#, + SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs() + )?; + writeln!(reservation, r#"unique_id = "{unique_id}""#)?; + writeln!( + reservation, + r#"run_url = "{qualified_repo}/actions/runs/{run_id}""# + )?; + Ok(()) } pub fn screenshot_runner(&self, id: usize) -> eyre::Result { @@ -240,6 +259,13 @@ impl Runner { None } }; + let reservation = match read_reservation(id) { + Ok(result) => result, + Err(error) => { + warn!(?error, "Failed to read `reserved-by`"); + None + } + }; let details = runner_details(id)?; @@ -250,6 +276,7 @@ impl Runner { guest_name: None, ipv4_address: None, github_jitconfig: github_jitconfig, + reservation, details, }) } @@ -366,6 +393,16 @@ cfg_if! { Ok(result.encoded_jit_config) } + fn read_reservation(id: usize) -> eyre::Result> { + let path = get_runner_data_path(id, Path::new("reservation.toml"))?; + let result = match std::fs::read_to_string(path) { + Ok(result) => Ok(result), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => Err(error), + }?; + Ok(Some(toml::from_str(&result)?)) + } + fn runner_created_time(id: usize) -> eyre::Result { let created_time_path = get_runner_data_path(id, Path::new("created-time"))?; let runner_toml_path = get_runner_data_path(id, Path::new("runner.toml"))?; @@ -404,6 +441,10 @@ cfg_if! { Ok("".to_owned()) } + fn read_reservation(_id: usize) -> eyre::Result> { + Ok(None) + } + fn runner_created_time(id: usize) -> eyre::Result { RUNNER_CREATED_TIMES.with_borrow(|created_times| { created_times.get(&id).copied().ok_or_eyre("Failed to check runner created time (fake)") From 554898902d129948d8aff7f100542ebafe16d2f8 Mon Sep 17 00:00:00 2001 From: Delan Azabani Date: Mon, 8 Dec 2025 16:25:45 +0800 Subject: [PATCH 3/4] monitor: fix tests --- monitor/src/policy.rs | 21 ++++++++----------- monitor/src/runner.rs | 49 ++++++++++++++++++++++++++++++------------- 2 files changed, 43 insertions(+), 27 deletions(-) diff --git a/monitor/src/policy.rs b/monitor/src/policy.rs index c251eea8..f2890a51 100644 --- a/monitor/src/policy.rs +++ b/monitor/src/policy.rs @@ -892,12 +892,15 @@ mod test { use chrono::{SecondsFormat, Utc}; use jane_eyre::eyre; - use monitor::github::{ApiRunner, ApiRunnerLabel}; + use monitor::github::ApiRunner; use settings::{profile::Profile, TOML}; use crate::{ policy::{Override, RunnerChanges}, - runner::{set_runner_created_time_for_test, Runners, Status}, + runner::{ + clear_runner_reserved_since_for_test, set_runner_created_time_for_test, + set_runner_reserved_since_for_test, Runners, Status, + }, }; use super::Policy; @@ -984,6 +987,7 @@ mod test { let mut registrations = vec![]; let mut guest_names = vec![]; + clear_runner_reserved_since_for_test(); for fake in fake_runners { let (runner_id, guest_name) = make_runner_id_and_guest_name(fake.profile_key); set_runner_created_time_for_test(runner_id, fake.created_time); @@ -997,18 +1001,11 @@ mod test { guest_names.push(guest_name); } Status::Reserved => { - let mut api_runner = make_registration(&guest_name); - api_runner.labels.push(ApiRunnerLabel { - name: "reserved-for:".to_owned(), // any value - }); + registrations.push(make_registration(&guest_name)); + guest_names.push(guest_name); if let Some(reserved_since) = fake.reserved_since { - let reserved_since = reserved_since.as_secs(); - api_runner.labels.push(ApiRunnerLabel { - name: format!("reserved-since:{reserved_since}"), - }); + set_runner_reserved_since_for_test(runner_id, reserved_since.as_secs()); } - registrations.push(api_runner); - guest_names.push(guest_name); } Status::Idle => { let mut api_runner = make_registration(&guest_name); diff --git a/monitor/src/runner.rs b/monitor/src/runner.rs index 122a99ff..72aa6f5c 100644 --- a/monitor/src/runner.rs +++ b/monitor/src/runner.rs @@ -310,12 +310,16 @@ impl Runner { registration.labels().join(","), ); } - if let Some(workflow_run) = registration.label_with_key("reserved-by") { - info!( - "[{}] - workflow run page: https://github.com/{}", - self.id, workflow_run - ); - } + } + if let Some(reservation) = self.reservation.as_ref() { + info!( + "[{}] - reserved for unique id: {}", + self.id, reservation.unique_id + ); + info!( + "[{}] - run url: https://github.com/{}", + self.id, reservation.run_url + ); } } @@ -324,12 +328,9 @@ impl Runner { } pub fn reserved_since(&self) -> eyre::Result> { - if let Some(registration) = &self.registration { - if let Some(reserved_since) = registration.label_with_key("reserved-since") { - let reserved_since = reserved_since.parse::()?; - let reserved_since = UNIX_EPOCH + Duration::from_secs(reserved_since); - return Ok(Some(reserved_since.elapsed()?)); - } + if let Some(reservation) = self.reservation.as_ref() { + let reserved_since = UNIX_EPOCH + Duration::from_secs(reservation.reserved_since); + return Ok(Some(reserved_since.elapsed()?)); } Ok(None) @@ -345,7 +346,7 @@ impl Runner { if registration.busy { return Status::Busy; } - if registration.label_with_key("reserved-for").is_some() { + if self.reservation.is_some() { return Status::Reserved; } if registration.status == "online" { @@ -434,6 +435,7 @@ cfg_if! { use jane_eyre::eyre::OptionExt; thread_local! { + static RUNNER_RESERVED_SINCE: RefCell> = RefCell::new(BTreeMap::new()); static RUNNER_CREATED_TIMES: RefCell> = RefCell::new(BTreeMap::new()); } @@ -441,8 +443,11 @@ cfg_if! { Ok("".to_owned()) } - fn read_reservation(_id: usize) -> eyre::Result> { - Ok(None) + fn read_reservation(id: usize) -> eyre::Result> { + let Some(reserved_since) = RUNNER_RESERVED_SINCE.with_borrow(|reserved_since_times| { + reserved_since_times.get(&id).copied() + }) else { return Ok(None) }; + Ok(Some(Reservation { reserved_since, unique_id: "".to_owned(), run_url: "".to_owned() })) } fn runner_created_time(id: usize) -> eyre::Result { @@ -451,6 +456,20 @@ cfg_if! { }) } + pub(crate) fn clear_runner_reserved_since_for_test() { + RUNNER_RESERVED_SINCE.with_borrow_mut(|reserved_since_map| reserved_since_map.clear()); + } + + pub(crate) fn set_runner_reserved_since_for_test(id: usize, reserved_since: u64) { + RUNNER_RESERVED_SINCE.with_borrow_mut(|reserved_since_map| { + if let Some(reserved_since) = reserved_since.into() { + reserved_since_map.insert(id, reserved_since); + } else { + reserved_since_map.remove(&id); + } + }); + } + pub(crate) fn set_runner_created_time_for_test(id: usize, created_time: impl Into>) { RUNNER_CREATED_TIMES.with_borrow_mut(|created_times| { if let Some(created_time) = created_time.into() { From 690ff44fcdad57e44569d075b6595829f908371b Mon Sep 17 00:00:00 2001 From: Delan Azabani Date: Mon, 8 Dec 2025 16:29:52 +0800 Subject: [PATCH 4/4] actions: use runner uuid in runner-select --- actions/runner-select/action.yml | 14 +++++++------- monitor/src/lib.rs | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/actions/runner-select/action.yml b/actions/runner-select/action.yml index 4221572c..5f45c3d6 100644 --- a/actions/runner-select/action.yml +++ b/actions/runner-select/action.yml @@ -6,7 +6,7 @@ inputs: github-hosted-runner-label: required: true type: string - self-hosted-image-name: + self-hosted-profile: required: true type: string force-github-hosted-runner: @@ -37,7 +37,7 @@ runs: shell: bash run: | github_hosted_runner_label='${{ inputs.github-hosted-runner-label }}' - self_hosted_image_name='${{ inputs.self-hosted-image-name }}' + self_hosted_profile='${{ inputs.self-hosted-profile }}' set -euo pipefail @@ -71,7 +71,7 @@ runs: echo "artifact_path=$artifact_path" | tee -a $GITHUB_OUTPUT echo "unique_id=$unique_id" | tee -a "$artifact_path" - echo "self_hosted_image_name=$self_hosted_image_name" | tee -a "$artifact_path" + echo "self_hosted_profile=$self_hosted_profile" | tee -a "$artifact_path" echo "qualified_repo=${{ github.repository }}" | tee -a "$artifact_path" echo "run_id=${{ github.run_id }}" | tee -a "$artifact_path" @@ -89,7 +89,7 @@ runs: shell: bash run: | github_hosted_runner_label='${{ inputs.github-hosted-runner-label }}' - self_hosted_image_name='${{ inputs.self-hosted-image-name }}' + self_hosted_profile='${{ inputs.self-hosted-profile }}' disabled='${{ steps.init.outputs.disabled }}' unique_id='${{ steps.init.outputs.unique_id }}' @@ -131,10 +131,10 @@ runs: # Retry for up to 3600 seconds, nominally once per second for up to 3600 times, # but actually respecting Retry-After for up to 3600 times. if curl -sS --fail-with-body --retry-max-time 3600 --retry 3600 --retry-delay 1 -X POST "$take_url_with_token" > $result \ - && jq -e . $result > /dev/null; then + && jq -er '.[0]' $result > /dev/null; then echo - echo "selected_runner_label=reserved-for:$unique_id" | tee -a $GITHUB_OUTPUT - echo "runner_type_label=self-hosted-image:$self_hosted_image_name" | tee -a $GITHUB_OUTPUT + echo "selected_runner_label=self-hosted-uuid:$(jq -er '.[0].runner.details.runner_uuid' $result)" | tee -a $GITHUB_OUTPUT + echo "runner_type_label=self-hosted-profile:$self_hosted_profile" | tee -a $GITHUB_OUTPUT echo 'is_self_hosted=true' | tee -a $GITHUB_OUTPUT exit 0 fi diff --git a/monitor/src/lib.rs b/monitor/src/lib.rs index 496f91f4..27a79b94 100644 --- a/monitor/src/lib.rs +++ b/monitor/src/lib.rs @@ -57,7 +57,7 @@ pub fn validate_tokenless_select( "Wrong run_id in artifact" )))?; } - let Some(profile_key) = args.remove("self_hosted_image_name") else { + let Some(profile_key) = args.remove("self_hosted_profile") else { Err(EyreReport::InternalServerError(eyre!( "Wrong run_id in artifact" )))?