Allow codspeed runner to run outside of a git directory

src/local_logger.rs

@@ -12,7 +12,7 @@ use log::Log;
 use simplelog::{CombinedLogger, SharedLogger};
 use std::io::Write;
 
-use crate::logger::{get_group_event, GroupEvent};
+use crate::logger::{get_group_event, get_json_event, GroupEvent, JsonEvent};
 
 pub const CODSPEED_U8_COLOR_CODE: u8 = 208; // #FF8700
 
@@ -66,7 +66,7 @@ impl Log for LocalLogger {
         if let Some(group_event) = get_group_event(record) {
             match group_event {
                 GroupEvent::Start(name) | GroupEvent::StartOpened(name) => {
-                    println!(
+                    eprintln!(
                         "\n{}",
                         style(format!("►►► {} ", name))
                             .bold()
@@ -89,7 +89,7 @@ impl Log for LocalLogger {
                         spinner.enable_steady_tick(Duration::from_millis(100));
                         SPINNER.lock().unwrap().replace(spinner);
                     } else {
-                        println!("{}...", name);
+                        eprintln!("{}...", name);
                     }
                 }
                 GroupEvent::End => {
@@ -105,6 +105,11 @@ impl Log for LocalLogger {
             return;
         }
 
+        if let Some(JsonEvent(json_string)) = get_json_event(record) {
+            println!("{json_string}");
+            return;
+        }
+
         suspend_progress_bar(|| print_record(record));
     }
 
@@ -124,12 +129,12 @@ fn print_record(record: &log::Record) {
     match record.level() {
         log::Level::Error => eprintln!("{}", error_style.apply_to(record.args())),
         log::Level::Warn => eprintln!("{}", warn_style.apply_to(record.args())),
-        log::Level::Info => println!("{}", info_style.apply_to(record.args())),
-        log::Level::Debug => println!(
+        log::Level::Info => eprintln!("{}", info_style.apply_to(record.args())),
+        log::Level::Debug => eprintln!(
             "{}",
             debug_style.apply_to(format!("[DEBUG::{}] {}", record.target(), record.args())),
         ),
-        log::Level::Trace => println!(
+        log::Level::Trace => eprintln!(
             "{}",
             trace_style.apply_to(format!("[TRACE::{}] {}", record.target(), record.args()))
         ),

src/logger.rs

@@ -71,3 +71,23 @@ pub(super) fn get_group_event(record: &log::Record) -> Option<GroupEvent> {
         _ => None,
     }
 }
+
+#[macro_export]
+/// Log a structured JSON output
+macro_rules! log_json {
+    ($value:expr) => {
+        log::log!(target: $crate::logger::JSON_TARGET, log::Level::Info, "{}", $value);
+    };
+}
+
+pub struct JsonEvent(pub String);
+
+pub const JSON_TARGET: &str = "codspeed::json";
+
+pub(super) fn get_json_event(record: &log::Record) -> Option<JsonEvent> {
+    if record.target() != JSON_TARGET {
+        return None;
+    }
+
+    Some(JsonEvent(record.args().to_string()))
+}

src/prelude.rs

@@ -1,4 +1,4 @@
-pub use crate::{end_group, start_group, start_opened_group};
+pub use crate::{end_group, log_json, start_group, start_opened_group};
 #[allow(unused_imports)]
 pub use anyhow::{anyhow, bail, ensure, Context, Error, Result};
 pub use itertools::Itertools;

src/run/config.rs

@@ -2,6 +2,7 @@ use crate::prelude::*;
 use crate::run::instruments::Instruments;
 use url::Url;
 
+use crate::run::run_environment::RepositoryProvider;
 use crate::run::RunArgs;
 
 use super::RunnerMode;
@@ -10,6 +11,7 @@ use super::RunnerMode;
 pub struct Config {
     pub upload_url: Url,
     pub token: Option<String>,
+    pub repository_override: Option<RepositoryOverride>,
     pub working_directory: Option<String>,
     pub command: String,
 
@@ -20,6 +22,13 @@ pub struct Config {
     pub skip_setup: bool,
 }
 
+#[derive(Debug, PartialEq, Clone)]
+pub struct RepositoryOverride {
+    pub owner: String,
+    pub repository: String,
+    pub repository_provider: RepositoryProvider,
+}
+
 impl Config {
     pub fn set_token(&mut self, token: Option<String>) {
         self.token = token;
@@ -33,6 +42,7 @@ impl Config {
         Self {
             upload_url: Url::parse(DEFAULT_UPLOAD_URL).unwrap(),
             token: None,
+            repository_override: None,
             working_directory: None,
             command: "".into(),
             mode: RunnerMode::Instrumentation,
@@ -52,9 +62,22 @@ impl TryFrom<RunArgs> for Config {
         let raw_upload_url = args.upload_url.unwrap_or_else(|| DEFAULT_UPLOAD_URL.into());
         let upload_url = Url::parse(&raw_upload_url)
             .map_err(|e| anyhow!("Invalid upload URL: {}, {}", raw_upload_url, e))?;
+
         Ok(Self {
             upload_url,
             token: args.token,
+            repository_override: args
+                .repository
+                .map(|respository_and_owner| -> Result<RepositoryOverride> {
+                    let (owner, repository) =
+                        extract_owner_and_repository_from_arg(&respository_and_owner)?;
+                    Ok(RepositoryOverride {
+                        owner,
+                        repository,
+                        repository_provider: args.provider.unwrap_or_default(),
+                    })
+                })
+                .transpose()?,
             working_directory: args.working_directory,
             mode: args.mode,
             instruments,
@@ -65,6 +88,13 @@ impl TryFrom<RunArgs> for Config {
     }
 }
 
+fn extract_owner_and_repository_from_arg(owner_and_repository: &str) -> Result<(String, String)> {
+    let (owner, repository) = owner_and_repository
+        .split_once('/')
+        .context("Invalid owner/repository format")?;
+    Ok((owner.to_string(), repository.to_string()))
+}
+
 #[cfg(test)]
 mod tests {
     use crate::run::instruments::MongoDBConfig;
@@ -76,17 +106,21 @@ mod tests {
         let config = Config::try_from(RunArgs {
             upload_url: None,
             token: None,
+            repository: None,
+            provider: None,
             working_directory: None,
             mode: RunnerMode::Instrumentation,
             instruments: vec![],
             mongo_uri_env_name: None,
+            message_format: None,
             skip_upload: false,
             skip_setup: false,
             command: vec!["cargo".into(), "codspeed".into(), "bench".into()],
         })
         .unwrap();
         assert_eq!(config.upload_url, Url::parse(DEFAULT_UPLOAD_URL).unwrap());
         assert_eq!(config.token, None);
+        assert_eq!(config.repository_override, None);
         assert_eq!(config.working_directory, None);
         assert_eq!(config.instruments, Instruments { mongodb: None });
         assert!(!config.skip_upload);
@@ -99,10 +133,13 @@ mod tests {
         let config = Config::try_from(RunArgs {
             upload_url: Some("https://example.com/upload".into()),
             token: Some("token".into()),
+            repository: Some("owner/repo".into()),
+            provider: Some(RepositoryProvider::GitLab),
             working_directory: Some("/tmp".into()),
             mode: RunnerMode::Instrumentation,
             instruments: vec!["mongodb".into()],
             mongo_uri_env_name: Some("MONGODB_URI".into()),
+            message_format: Some(crate::run::MessageFormat::Json),
             skip_upload: true,
             skip_setup: true,
             command: vec!["cargo".into(), "codspeed".into(), "bench".into()],
@@ -114,6 +151,14 @@ mod tests {
             Url::parse("https://example.com/upload").unwrap()
         );
         assert_eq!(config.token, Some("token".into()));
+        assert_eq!(
+            config.repository_override,
+            Some(RepositoryOverride {
+                owner: "owner".into(),
+                repository: "repo".into(),
+                repository_provider: RepositoryProvider::GitLab,
+            })
+        );
         assert_eq!(config.working_directory, Some("/tmp".into()));
         assert_eq!(
             config.instruments,
@@ -127,4 +172,18 @@ mod tests {
         assert!(config.skip_setup);
         assert_eq!(config.command, "cargo codspeed bench");
     }
+
+    #[test]
+    fn test_extract_owner_and_repository_from_arg() {
+        let owner_and_repository = "CodSpeedHQ/runner";
+        let (owner, repository) =
+            extract_owner_and_repository_from_arg(owner_and_repository).unwrap();
+        assert_eq!(owner, "CodSpeedHQ");
+        assert_eq!(repository, "runner");
+
+        let owner_and_repository = "CodSpeedHQ_runner";
+
+        let result = extract_owner_and_repository_from_arg(owner_and_repository);
+        assert!(result.is_err());
+    }
 }

src/run/mod.rs

@@ -6,7 +6,7 @@ use crate::VERSION;
 use check_system::SystemInfo;
 use clap::{Args, ValueEnum};
 use instruments::mongo_tracer::{install_mongodb_tracer, MongoTracer};
-use run_environment::interfaces::RunEnvironment;
+use run_environment::interfaces::{RepositoryProvider, RunEnvironment};
 use runner::get_run_data;
 use serde::Serialize;
 
@@ -37,14 +37,6 @@ fn show_banner() {
     debug!("codspeed v{}", VERSION);
 }
 
-#[derive(ValueEnum, Clone, Default, Debug, Serialize)]
-#[serde(rename_all = "lowercase")]
-pub enum RunnerMode {
-    #[default]
-    Instrumentation,
-    Walltime,
-}
-
 #[derive(Args, Debug)]
 pub struct RunArgs {
     /// The upload URL to use for uploading the results, useful for on-premises installations
@@ -55,6 +47,19 @@ pub struct RunArgs {
     #[arg(long, env = "CODSPEED_TOKEN")]
     pub token: Option<String>,
 
+    /// The repository the benchmark is associated with, under the format `owner/repo`.
+    #[arg(short, long, env = "CODSPEED_REPOSITORY")]
+    pub repository: Option<String>,
+
+    /// The repository provider to use in case --repository is used. Defaults to github
+    #[arg(
+        long,
+        env = "CODSPEED_PROVIDER",
+        requires = "repository",
+        ignore_case = true
+    )]
+    pub provider: Option<RepositoryProvider>,
+
     /// The directory where the command will be executed.
     #[arg(long)]
     pub working_directory: Option<String>,
@@ -74,6 +79,9 @@ pub struct RunArgs {
     #[arg(long)]
     pub mongo_uri_env_name: Option<String>,
 
+    #[arg(long, hide = true)]
+    pub message_format: Option<MessageFormat>,
+
     /// Only for debugging purposes, skips the upload of the results
     #[arg(
         long,
@@ -91,17 +99,33 @@ pub struct RunArgs {
     pub command: Vec<String>,
 }
 
+#[derive(ValueEnum, Clone, Default, Debug, Serialize)]
+#[serde(rename_all = "lowercase")]
+pub enum RunnerMode {
+    #[default]
+    Instrumentation,
+    Walltime,
+}
+
+#[derive(ValueEnum, Clone, Debug, PartialEq)]
+pub enum MessageFormat {
+    Json,
+}
+
 #[cfg(test)]
 impl RunArgs {
     /// Constructs a new `RunArgs` with default values for testing purposes
     pub fn test() -> Self {
         Self {
             upload_url: None,
             token: None,
+            repository: None,
+            provider: None,
             working_directory: None,
             mode: RunnerMode::Instrumentation,
             instruments: vec![],
             mongo_uri_env_name: None,
+            message_format: None,
             skip_upload: false,
             skip_setup: false,
             command: vec![],
@@ -110,6 +134,7 @@ impl RunArgs {
 }
 
 pub async fn run(args: RunArgs, api_client: &CodSpeedAPIClient) -> Result<()> {
+    let output_json = args.message_format == Some(MessageFormat::Json);
     let mut config = Config::try_from(args)?;
     let provider = run_environment::get_provider(&config)?;
     let codspeed_config = CodSpeedConfig::load()?;
@@ -179,10 +204,33 @@ pub async fn run(args: RunArgs, api_client: &CodSpeedAPIClient) -> Result<()> {
 
         if provider.get_run_environment() == RunEnvironment::Local {
             start_group!("Fetching the results");
+            let run_id = upload_result.run_id.clone();
             poll_results::poll_results(api_client, &provider, upload_result.run_id).await?;
+            if output_json {
+                // TODO: Refactor `log_json` to avoid having to format the json manually
+                // We could make use of structured logging for this https://docs.rs/log/latest/log/#structured-logging
+                log_json!(format!(
+                    "{{\"event\": \"run_finished\", \"run_id\": \"{}\"}}",
+                    run_id
+                ));
+            }
             end_group!();
         }
     }
 
     Ok(())
 }
+
+// We have to implement this manually, because deriving the trait makes the CLI values `git-hub`
+// and `git-lab`
+impl clap::ValueEnum for RepositoryProvider {
+    fn value_variants<'a>() -> &'a [Self] {
+        &[Self::GitLab, Self::GitHub]
+    }
+    fn to_possible_value<'a>(&self) -> ::std::option::Option<clap::builder::PossibleValue> {
+        match self {
+            Self::GitLab => Some(clap::builder::PossibleValue::new("gitlab").aliases(["gl"])),
+            Self::GitHub => Some(clap::builder::PossibleValue::new("github").aliases(["gh"])),
+        }
+    }
+}

src/run/run_environment/buildkite/logger.rs

@@ -1,5 +1,5 @@
 use crate::{
-    logger::{get_group_event, GroupEvent},
+    logger::{get_group_event, get_json_event, GroupEvent},
     run::run_environment::logger::should_provider_logger_handle_record,
 };
 use log::*;
@@ -49,6 +49,10 @@ impl Log for BuildkiteLogger {
             return;
         }
 
+        if get_json_event(record).is_some() {
+            return;
+        }
+
         if level > self.log_level {
             return;
         }

src/run/run_environment/buildkite/provider.rs

@@ -61,6 +61,10 @@ impl TryFrom<&Config> for BuildkiteProvider {
             bail!("Token authentication is required for Buildkite");
         }
 
+        if config.repository_override.is_some() {
+            bail!("Specifying owner and repository from CLI is not supported for Buildkite");
+        }
+
         let is_pr = get_pr_number()?.is_some();
         let repository_url = get_env_variable("BUILDKITE_REPO")?;
         let GitRemote {