Skip to content

Commit c999e1d

Browse files
fargitofargito
committed
feat: add run_part to upload metadata
1 parent 8004646 commit c999e1d

17 files changed

Lines changed: 494 additions & 13 deletions

‎src/run/run_environment/buildkite/provider.rs‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ use simplelog::SharedLogger;
44

55
use crate::prelude::*;
66
use crate::run::helpers::{parse_git_remote, GitRemote};
7-
use crate::run::run_environment::interfaces::RunEnvironment;
7+
use crate::run::run_environment::{RunEnvironment, RunPart};
88
use crate::run::{
99
config::Config,
1010
helpers::{find_repository_root, get_env_variable},
@@ -146,6 +146,11 @@ impl RunEnvironmentProvider for BuildkiteProvider {
146146
sender: None,
147147
})
148148
}
149+
150+
/// For Buildkite, we don't support multipart uploads
151+
fn get_platform_run_part(&self) -> Option<RunPart> {
152+
None
153+
}
149154
}
150155

151156
#[cfg(test)]

‎src/run/run_environment/github_actions/provider.rs‎

Lines changed: 312 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,11 @@ use lazy_static::lazy_static;
22
use regex::Regex;
33
use serde_json::Value;
44
use simplelog::SharedLogger;
5+
use std::collections::BTreeMap;
56
use std::{env, fs};
67

78
use crate::prelude::*;
8-
use crate::run::run_environment::interfaces::RunEnvironment;
9+
use crate::run::run_environment::{RunEnvironment, RunPart};
910
use crate::run::{
1011
config::Config,
1112
helpers::{find_repository_root, get_env_variable},
@@ -147,6 +148,68 @@ impl RunEnvironmentProvider for GitHubActionsProvider {
147148
repository_root_path: self.repository_root_path.clone(),
148149
})
149150
}
151+
152+
/// For Github, the platform run part is the most complicated
153+
/// since we support matrix jobs.
154+
///
155+
/// Computing the `run_part_id`:
156+
/// - not in a matrix:
157+
/// - simply take the job name
158+
/// - in a matrix:
159+
/// - take the job name
160+
/// - concatenate it with key-values from `matrix` and `strategy`
161+
///
162+
/// `GH_MATRIX` and `GH_STRATEGY` are environment variables computed by
163+
/// https://github.com/CodSpeedHQ/action:
164+
/// - `GH_MATRIX`: ${{ toJson(matrix) }}
165+
/// - `GH_STRATEGY`: ${{ toJson(strategy) }}
166+
///
167+
/// A note on parsing:
168+
///
169+
/// The issue is these variables from Github Actions are multiline.
170+
/// As we need to use them compute an identifier, we need them as a single line.
171+
/// Plus we are interested in the content of these objects,
172+
/// so it makes sense to parse and re-serialize them.
173+
fn get_platform_run_part(&self) -> Option<RunPart> {
174+
let job_name = self.gh_data.job.clone();
175+
176+
let mut metadata = BTreeMap::new();
177+
178+
let gh_matrix = get_env_variable("GH_MATRIX")
179+
.ok()
180+
.and_then(|v| serde_json::from_str::<Value>(&v).ok());
181+
182+
let gh_strategy = get_env_variable("GH_STRATEGY")
183+
.ok()
184+
.and_then(|v| serde_json::from_str::<Value>(&v).ok());
185+
186+
let run_part_id = if let (Some(Value::Object(matrix)), Some(Value::Object(mut strategy))) =
187+
(gh_matrix, gh_strategy)
188+
{
189+
// remove useless values from the strategy
190+
strategy.remove("fail-fast");
191+
strategy.remove("max-parallel");
192+
193+
// The re-serialization is on purpose here. We want to serialize it as a single line.
194+
let matrix_str = serde_json::to_string(&matrix).expect("Unable to re-serialize matrix");
195+
let strategy_str =
196+
serde_json::to_string(&strategy).expect("Unable to re-serialize strategy");
197+
198+
metadata.extend(matrix);
199+
metadata.extend(strategy);
200+
201+
format!("{job_name}-{matrix_str}-{strategy_str}")
202+
} else {
203+
job_name
204+
};
205+
206+
Some(RunPart {
207+
run_id: self.gh_data.run_id.clone(),
208+
run_part_id,
209+
job_name: self.gh_data.job.clone(),
210+
metadata,
211+
})
212+
}
150213
}
151214

152215
#[cfg(test)]
@@ -248,13 +311,15 @@ mod tests {
248311
let run_environment_metadata = github_actions_provider
249312
.get_run_environment_metadata()
250313
.unwrap();
314+
let run_part = github_actions_provider.get_platform_run_part().unwrap();
251315

252316
assert_json_snapshot!(run_environment_metadata, {
253317
".runner.version" => insta::dynamic_redaction(|value,_path| {
254318
assert_eq!(value.as_str().unwrap(), VERSION.to_string());
255319
"[version]"
256320
}),
257321
});
322+
assert_json_snapshot!(run_part);
258323
},
259324
);
260325
}
@@ -284,6 +349,7 @@ mod tests {
284349
("GITHUB_REPOSITORY", Some("my-org/adrien-python-test")),
285350
("GITHUB_RUN_ID", Some("6957110437")),
286351
("VERSION", Some("0.1.0")),
352+
("GH_MATRIX", Some("null")),
287353
],
288354
|| {
289355
let config = Config {
@@ -294,6 +360,7 @@ mod tests {
294360
let run_environment_metadata = github_actions_provider
295361
.get_run_environment_metadata()
296362
.unwrap();
363+
let run_part = github_actions_provider.get_platform_run_part().unwrap();
297364

298365
assert_eq!(run_environment_metadata.owner, "my-org");
299366
assert_eq!(run_environment_metadata.repository, "adrien-python-test");
@@ -302,13 +369,257 @@ mod tests {
302369
run_environment_metadata.head_ref,
303370
Some("fork-owner:feat/codspeed-runner".into())
304371
);
372+
373+
assert_json_snapshot!(run_environment_metadata, {
374+
".runner.version" => insta::dynamic_redaction(|value,_path| {
375+
assert_eq!(value.as_str().unwrap(), VERSION.to_string());
376+
"[version]"
377+
}),
378+
});
379+
assert_json_snapshot!(run_part);
380+
},
381+
);
382+
}
383+
384+
#[test]
385+
fn test_matrix_job_run_environment_metadata() {
386+
with_vars(
387+
[
388+
("GITHUB_ACTIONS", Some("true")),
389+
("GITHUB_ACTOR_ID", Some("19605940")),
390+
("GITHUB_ACTOR", Some("adriencaccia")),
391+
("GITHUB_BASE_REF", Some("main")),
392+
("GITHUB_EVENT_NAME", Some("pull_request")),
393+
(
394+
"GITHUB_EVENT_PATH",
395+
Some(
396+
format!(
397+
"{}/src/run/run_environment/github_actions/samples/pr-event.json",
398+
env!("CARGO_MANIFEST_DIR")
399+
)
400+
.as_str(),
401+
),
402+
),
403+
("GITHUB_HEAD_REF", Some("feat/codspeed-runner")),
404+
("GITHUB_JOB", Some("log-env")),
405+
("GITHUB_REF", Some("refs/pull/22/merge")),
406+
("GITHUB_REPOSITORY", Some("my-org/adrien-python-test")),
407+
("GITHUB_RUN_ID", Some("6957110437")),
408+
("VERSION", Some("0.1.0")),
409+
(
410+
"GH_MATRIX",
411+
Some(
412+
r#"{
413+
"runner-version":"3.2.1",
414+
"numeric-value":123456789
415+
}"#,
416+
),
417+
),
418+
(
419+
"GH_STRATEGY",
420+
Some(
421+
r#"{
422+
"fail-fast":true,
423+
"job-index":1,
424+
"job-total":2,
425+
"max-parallel":2
426+
}"#,
427+
),
428+
),
429+
],
430+
|| {
431+
let config = Config {
432+
token: Some("token".into()),
433+
..Config::test()
434+
};
435+
let github_actions_provider = GitHubActionsProvider::try_from(&config).unwrap();
436+
let run_environment_metadata = github_actions_provider
437+
.get_run_environment_metadata()
438+
.unwrap();
439+
let run_part = github_actions_provider.get_platform_run_part().unwrap();
440+
305441
assert_json_snapshot!(run_environment_metadata, {
306442
".runner.version" => insta::dynamic_redaction(|value,_path| {
307443
assert_eq!(value.as_str().unwrap(), VERSION.to_string());
308444
"[version]"
309445
}),
310446
});
447+
assert_json_snapshot!(run_part);
311448
},
312449
);
313450
}
451+
452+
#[test]
453+
fn test_get_run_part_no_matrix() {
454+
with_vars([("GITHUB_ACTIONS", Some("true"))], || {
455+
let github_actions_provider = GitHubActionsProvider {
456+
owner: "owner".into(),
457+
repository: "repository".into(),
458+
ref_: "refs/head/my-branch".into(),
459+
head_ref: Some("my-branch".into()),
460+
base_ref: None,
461+
sender: None,
462+
gh_data: GhData {
463+
job: "my_job".into(),
464+
run_id: "123789".into(),
465+
},
466+
event: RunEvent::Push,
467+
repository_root_path: "/home/work/my-repo".into(),
468+
};
469+
470+
let run_part = github_actions_provider.get_platform_run_part().unwrap();
471+
472+
assert_eq!(run_part.run_id, "123789");
473+
assert_eq!(run_part.job_name, "my_job");
474+
assert_eq!(run_part.run_part_id, "my_job");
475+
assert_json_snapshot!(run_part.metadata, @"{}");
476+
})
477+
}
478+
479+
#[test]
480+
fn test_get_run_part_null_matrix() {
481+
with_vars(
482+
[
483+
("GH_MATRIX", Some("null")),
484+
(
485+
"GH_STRATEGY",
486+
Some(
487+
r#"{
488+
"fail-fast":true,
489+
"job-index":0,
490+
"job-total":1,
491+
"max-parallel":1
492+
}"#,
493+
),
494+
),
495+
],
496+
|| {
497+
let github_actions_provider = GitHubActionsProvider {
498+
owner: "owner".into(),
499+
repository: "repository".into(),
500+
ref_: "refs/head/my-branch".into(),
501+
head_ref: Some("my-branch".into()),
502+
base_ref: None,
503+
sender: None,
504+
gh_data: GhData {
505+
job: "my_job".into(),
506+
run_id: "123789".into(),
507+
},
508+
event: RunEvent::Push,
509+
repository_root_path: "/home/work/my-repo".into(),
510+
};
511+
512+
let run_part = github_actions_provider.get_platform_run_part().unwrap();
513+
514+
assert_eq!(run_part.run_id, "123789");
515+
assert_eq!(run_part.job_name, "my_job");
516+
assert_eq!(run_part.run_part_id, "my_job");
517+
assert_json_snapshot!(run_part.metadata, @"{}");
518+
},
519+
)
520+
}
521+
522+
#[test]
523+
fn test_get_matrix_run_part() {
524+
with_vars(
525+
[
526+
(
527+
"GH_MATRIX",
528+
Some(
529+
r#"{
530+
"runner-version":"3.2.1",
531+
"numeric-value":123456789
532+
}"#,
533+
),
534+
),
535+
(
536+
"GH_STRATEGY",
537+
Some(
538+
r#"{
539+
"fail-fast":true,
540+
"job-index":1,
541+
"job-total":2,
542+
"max-parallel":2
543+
}"#,
544+
),
545+
),
546+
],
547+
|| {
548+
let github_actions_provider = GitHubActionsProvider {
549+
owner: "owner".into(),
550+
repository: "repository".into(),
551+
ref_: "refs/head/my-branch".into(),
552+
head_ref: Some("my-branch".into()),
553+
base_ref: None,
554+
sender: None,
555+
gh_data: GhData {
556+
job: "my_job".into(),
557+
run_id: "123789".into(),
558+
},
559+
event: RunEvent::Push,
560+
repository_root_path: "/home/work/my-repo".into(),
561+
};
562+
563+
let run_part = github_actions_provider.get_platform_run_part().unwrap();
564+
565+
assert_eq!(run_part.run_id, "123789");
566+
assert_eq!(run_part.job_name, "my_job");
567+
assert_eq!(run_part.run_part_id, "my_job-{\"runner-version\":\"3.2.1\",\"numeric-value\":123456789}-{\"job-total\":2,\"job-index\":1}");
568+
assert_json_snapshot!(run_part.metadata, @r#"
569+
{
570+
"job-index": 1,
571+
"job-total": 2,
572+
"numeric-value": 123456789,
573+
"runner-version": "3.2.1"
574+
}
575+
"#);
576+
},
577+
)
578+
}
579+
580+
#[test]
581+
fn test_get_inline_matrix_run_part() {
582+
with_vars(
583+
[
584+
(
585+
"GH_MATRIX",
586+
Some("{\"runner-version\":\"3.2.1\",\"numeric-value\":123456789}"),
587+
),
588+
(
589+
"GH_STRATEGY",
590+
Some("{\"fail-fast\":true,\"job-index\":1,\"job-total\":2,\"max-parallel\":2}"),
591+
),
592+
],
593+
|| {
594+
let github_actions_provider = GitHubActionsProvider {
595+
owner: "owner".into(),
596+
repository: "repository".into(),
597+
ref_: "refs/head/my-branch".into(),
598+
head_ref: Some("my-branch".into()),
599+
base_ref: None,
600+
sender: None,
601+
gh_data: GhData {
602+
job: "my_job".into(),
603+
run_id: "123789".into(),
604+
},
605+
event: RunEvent::Push,
606+
repository_root_path: "/home/work/my-repo".into(),
607+
};
608+
609+
let run_part = github_actions_provider.get_platform_run_part().unwrap();
610+
611+
assert_eq!(run_part.run_id, "123789");
612+
assert_eq!(run_part.job_name, "my_job");
613+
assert_eq!(run_part.run_part_id, "my_job-{\"runner-version\":\"3.2.1\",\"numeric-value\":123456789}-{\"job-total\":2,\"job-index\":1}");
614+
assert_json_snapshot!(run_part.metadata, @r#"
615+
{
616+
"job-index": 1,
617+
"job-total": 2,
618+
"numeric-value": 123456789,
619+
"runner-version": "3.2.1"
620+
}
621+
"#);
622+
},
623+
)
624+
}
314625
}

0 commit comments

Comments
 (0)