Skip to content

Commit f5f1005

Browse files
fargitofargito
committed
feat: add run_part to upload metadata
1 parent 00f68a6 commit f5f1005

16 files changed

Lines changed: 489 additions & 11 deletions

‎src/run/ci_provider/buildkite/provider.rs‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ use std::env;
33
use simplelog::SharedLogger;
44

55
use crate::prelude::*;
6-
use crate::run::ci_provider::interfaces::Platform;
6+
use crate::run::ci_provider::interfaces::{Platform, RunPart};
77
use crate::run::helpers::{parse_git_remote, GitRemote};
88
use crate::run::{
99
ci_provider::{
@@ -132,6 +132,11 @@ impl CIProvider for BuildkiteProvider {
132132
Platform::Buildkite
133133
}
134134

135+
/// For Buildkite, we don't support multipart uploads
136+
fn get_platform_run_part(&self) -> Option<RunPart> {
137+
None
138+
}
139+
135140
fn get_ci_provider_metadata(&self) -> Result<CIProviderMetadata> {
136141
Ok(CIProviderMetadata {
137142
base_ref: self.base_ref.clone(),

‎src/run/ci_provider/github_actions/provider.rs‎

Lines changed: 310 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,11 @@ use lazy_static::lazy_static;
22
use regex::Regex;
33
use serde_json::Value;
44
use simplelog::SharedLogger;
5+
use std::collections::BTreeMap;
56
use std::{env, fs};
67

78
use crate::prelude::*;
8-
use crate::run::ci_provider::interfaces::Platform;
9+
use crate::run::ci_provider::interfaces::{Platform, RunPart};
910
use crate::run::{
1011
ci_provider::{
1112
interfaces::{CIProviderMetadata, GhData, RepositoryProvider, RunEvent, Sender},
@@ -133,6 +134,68 @@ impl CIProvider for GitHubActionsProvider {
133134
Platform::GithubActions
134135
}
135136

137+
/// For Github, the platform run part is the most complicated
138+
/// since we support matrix jobs.
139+
///
140+
/// Computing the `run_part_id`:
141+
/// - not in a matrix:
142+
/// - simply take the job name
143+
/// - in a matrix:
144+
/// - take the job name
145+
/// - concatenate it with key-values from `matrix` and `strategy`
146+
///
147+
/// `GH_MATRIX` and `GH_STRATEGY` are environment variables computed by
148+
/// https://github.com/CodSpeedHQ/action:
149+
/// - `GH_MATRIX`: ${{ toJson(matrix) }}
150+
/// - `GH_STRATEGY`: ${{ toJson(strategy) }}
151+
///
152+
/// A note on parsing:
153+
///
154+
/// The issue is these variables from Github Actions are multiline.
155+
/// As we need to use them compute an identifier, we need them as a single line.
156+
/// Plus we are interested in the content of these objects,
157+
/// so it makes sense to parse and re-serialize them.
158+
fn get_platform_run_part(&self) -> Option<RunPart> {
159+
let job_name = self.gh_data.job.clone();
160+
161+
let mut metadata = BTreeMap::new();
162+
163+
let gh_matrix = get_env_variable("GH_MATRIX")
164+
.ok()
165+
.and_then(|v| serde_json::from_str::<Value>(&v).ok());
166+
167+
let gh_strategy = get_env_variable("GH_STRATEGY")
168+
.ok()
169+
.and_then(|v| serde_json::from_str::<Value>(&v).ok());
170+
171+
let run_part_id = if let (Some(Value::Object(matrix)), Some(Value::Object(mut strategy))) =
172+
(gh_matrix, gh_strategy)
173+
{
174+
// remove useless values from the strategy
175+
strategy.remove("fail-fast");
176+
strategy.remove("max-parallel");
177+
178+
// The re-serialization is on purpose here. We want to serialize it as a single line.
179+
let matrix_str = serde_json::to_string(&matrix).expect("Unable to re-serialize matrix");
180+
let strategy_str =
181+
serde_json::to_string(&strategy).expect("Unable to re-serialize strategy");
182+
183+
metadata.extend(matrix);
184+
metadata.extend(strategy);
185+
186+
format!("{job_name}-{matrix_str}-{strategy_str}")
187+
} else {
188+
job_name
189+
};
190+
191+
Some(RunPart {
192+
run_id: self.gh_data.run_id.clone(),
193+
run_part_id,
194+
job_name: self.gh_data.job.clone(),
195+
metadata,
196+
})
197+
}
198+
136199
fn get_ci_provider_metadata(&self) -> Result<CIProviderMetadata> {
137200
Ok(CIProviderMetadata {
138201
base_ref: self.base_ref.clone(),
@@ -246,13 +309,15 @@ mod tests {
246309
};
247310
let github_actions_provider = GitHubActionsProvider::try_from(&config).unwrap();
248311
let provider_metadata = github_actions_provider.get_ci_provider_metadata().unwrap();
312+
let run_part = github_actions_provider.get_platform_run_part().unwrap();
249313

250314
assert_json_snapshot!(provider_metadata, {
251315
".runner.version" => insta::dynamic_redaction(|value,_path| {
252316
assert_eq!(value.as_str().unwrap(), VERSION.to_string());
253317
"[version]"
254318
}),
255319
});
320+
assert_json_snapshot!(run_part);
256321
},
257322
);
258323
}
@@ -282,6 +347,7 @@ mod tests {
282347
("GITHUB_REPOSITORY", Some("my-org/adrien-python-test")),
283348
("GITHUB_RUN_ID", Some("6957110437")),
284349
("VERSION", Some("0.1.0")),
350+
("GH_MATRIX", Some("null")),
285351
],
286352
|| {
287353
let config = Config {
@@ -290,6 +356,7 @@ mod tests {
290356
};
291357
let github_actions_provider = GitHubActionsProvider::try_from(&config).unwrap();
292358
let provider_metadata = github_actions_provider.get_ci_provider_metadata().unwrap();
359+
let run_part = github_actions_provider.get_platform_run_part().unwrap();
293360

294361
assert_eq!(provider_metadata.owner, "my-org");
295362
assert_eq!(provider_metadata.repository, "adrien-python-test");
@@ -298,13 +365,255 @@ mod tests {
298365
provider_metadata.head_ref,
299366
Some("fork-owner:feat/codspeed-runner".into())
300367
);
368+
369+
assert_json_snapshot!(provider_metadata, {
370+
".runner.version" => insta::dynamic_redaction(|value,_path| {
371+
assert_eq!(value.as_str().unwrap(), VERSION.to_string());
372+
"[version]"
373+
}),
374+
});
375+
assert_json_snapshot!(run_part);
376+
},
377+
);
378+
}
379+
380+
#[test]
381+
fn test_matrix_job_provider_metadata() {
382+
with_vars(
383+
[
384+
("GITHUB_ACTIONS", Some("true")),
385+
("GITHUB_ACTOR_ID", Some("19605940")),
386+
("GITHUB_ACTOR", Some("adriencaccia")),
387+
("GITHUB_BASE_REF", Some("main")),
388+
("GITHUB_EVENT_NAME", Some("pull_request")),
389+
(
390+
"GITHUB_EVENT_PATH",
391+
Some(
392+
format!(
393+
"{}/src/run/ci_provider/github_actions/samples/pr-event.json",
394+
env!("CARGO_MANIFEST_DIR")
395+
)
396+
.as_str(),
397+
),
398+
),
399+
("GITHUB_HEAD_REF", Some("feat/codspeed-runner")),
400+
("GITHUB_JOB", Some("log-env")),
401+
("GITHUB_REF", Some("refs/pull/22/merge")),
402+
("GITHUB_REPOSITORY", Some("my-org/adrien-python-test")),
403+
("GITHUB_RUN_ID", Some("6957110437")),
404+
("VERSION", Some("0.1.0")),
405+
(
406+
"GH_MATRIX",
407+
Some(
408+
r#"{
409+
"runner-version":"3.2.1",
410+
"numeric-value":123456789
411+
}"#,
412+
),
413+
),
414+
(
415+
"GH_STRATEGY",
416+
Some(
417+
r#"{
418+
"fail-fast":true,
419+
"job-index":1,
420+
"job-total":2,
421+
"max-parallel":2
422+
}"#,
423+
),
424+
),
425+
],
426+
|| {
427+
let config = Config {
428+
token: Some("token".into()),
429+
..Config::test()
430+
};
431+
let github_actions_provider = GitHubActionsProvider::try_from(&config).unwrap();
432+
let provider_metadata = github_actions_provider.get_ci_provider_metadata().unwrap();
433+
let run_part = github_actions_provider.get_platform_run_part().unwrap();
434+
301435
assert_json_snapshot!(provider_metadata, {
302436
".runner.version" => insta::dynamic_redaction(|value,_path| {
303437
assert_eq!(value.as_str().unwrap(), VERSION.to_string());
304438
"[version]"
305439
}),
306440
});
441+
assert_json_snapshot!(run_part);
307442
},
308443
);
309444
}
445+
446+
#[test]
447+
fn test_get_run_part_no_matrix() {
448+
with_vars([("GITHUB_ACTIONS", Some("true"))], || {
449+
let github_actions_provider = GitHubActionsProvider {
450+
owner: "owner".into(),
451+
repository: "repository".into(),
452+
ref_: "refs/head/my-branch".into(),
453+
head_ref: Some("my-branch".into()),
454+
base_ref: None,
455+
sender: None,
456+
gh_data: GhData {
457+
job: "my_job".into(),
458+
run_id: "123789".into(),
459+
},
460+
event: RunEvent::Push,
461+
repository_root_path: "/home/work/my-repo".into(),
462+
};
463+
464+
let run_part = github_actions_provider.get_platform_run_part().unwrap();
465+
466+
assert_eq!(run_part.run_id, "123789");
467+
assert_eq!(run_part.job_name, "my_job");
468+
assert_eq!(run_part.run_part_id, "my_job");
469+
assert_json_snapshot!(run_part.metadata, @"{}");
470+
})
471+
}
472+
473+
#[test]
474+
fn test_get_run_part_null_matrix() {
475+
with_vars(
476+
[
477+
("GH_MATRIX", Some("null")),
478+
(
479+
"GH_STRATEGY",
480+
Some(
481+
r#"{
482+
"fail-fast":true,
483+
"job-index":0,
484+
"job-total":1,
485+
"max-parallel":1
486+
}"#,
487+
),
488+
),
489+
],
490+
|| {
491+
let github_actions_provider = GitHubActionsProvider {
492+
owner: "owner".into(),
493+
repository: "repository".into(),
494+
ref_: "refs/head/my-branch".into(),
495+
head_ref: Some("my-branch".into()),
496+
base_ref: None,
497+
sender: None,
498+
gh_data: GhData {
499+
job: "my_job".into(),
500+
run_id: "123789".into(),
501+
},
502+
event: RunEvent::Push,
503+
repository_root_path: "/home/work/my-repo".into(),
504+
};
505+
506+
let run_part = github_actions_provider.get_platform_run_part().unwrap();
507+
508+
assert_eq!(run_part.run_id, "123789");
509+
assert_eq!(run_part.job_name, "my_job");
510+
assert_eq!(run_part.run_part_id, "my_job");
511+
assert_json_snapshot!(run_part.metadata, @"{}");
512+
},
513+
)
514+
}
515+
516+
#[test]
517+
fn test_get_matrix_run_part() {
518+
with_vars(
519+
[
520+
(
521+
"GH_MATRIX",
522+
Some(
523+
r#"{
524+
"runner-version":"3.2.1",
525+
"numeric-value":123456789
526+
}"#,
527+
),
528+
),
529+
(
530+
"GH_STRATEGY",
531+
Some(
532+
r#"{
533+
"fail-fast":true,
534+
"job-index":1,
535+
"job-total":2,
536+
"max-parallel":2
537+
}"#,
538+
),
539+
),
540+
],
541+
|| {
542+
let github_actions_provider = GitHubActionsProvider {
543+
owner: "owner".into(),
544+
repository: "repository".into(),
545+
ref_: "refs/head/my-branch".into(),
546+
head_ref: Some("my-branch".into()),
547+
base_ref: None,
548+
sender: None,
549+
gh_data: GhData {
550+
job: "my_job".into(),
551+
run_id: "123789".into(),
552+
},
553+
event: RunEvent::Push,
554+
repository_root_path: "/home/work/my-repo".into(),
555+
};
556+
557+
let run_part = github_actions_provider.get_platform_run_part().unwrap();
558+
559+
assert_eq!(run_part.run_id, "123789");
560+
assert_eq!(run_part.job_name, "my_job");
561+
assert_eq!(run_part.run_part_id, "my_job-{\"runner-version\":\"3.2.1\",\"numeric-value\":123456789}-{\"job-total\":2,\"job-index\":1}");
562+
assert_json_snapshot!(run_part.metadata, @r#"
563+
{
564+
"job-index": 1,
565+
"job-total": 2,
566+
"numeric-value": 123456789,
567+
"runner-version": "3.2.1"
568+
}
569+
"#);
570+
},
571+
)
572+
}
573+
574+
#[test]
575+
fn test_get_inline_matrix_run_part() {
576+
with_vars(
577+
[
578+
(
579+
"GH_MATRIX",
580+
Some("{\"runner-version\":\"3.2.1\",\"numeric-value\":123456789}"),
581+
),
582+
(
583+
"GH_STRATEGY",
584+
Some("{\"fail-fast\":true,\"job-index\":1,\"job-total\":2,\"max-parallel\":2}"),
585+
),
586+
],
587+
|| {
588+
let github_actions_provider = GitHubActionsProvider {
589+
owner: "owner".into(),
590+
repository: "repository".into(),
591+
ref_: "refs/head/my-branch".into(),
592+
head_ref: Some("my-branch".into()),
593+
base_ref: None,
594+
sender: None,
595+
gh_data: GhData {
596+
job: "my_job".into(),
597+
run_id: "123789".into(),
598+
},
599+
event: RunEvent::Push,
600+
repository_root_path: "/home/work/my-repo".into(),
601+
};
602+
603+
let run_part = github_actions_provider.get_platform_run_part().unwrap();
604+
605+
assert_eq!(run_part.run_id, "123789");
606+
assert_eq!(run_part.job_name, "my_job");
607+
assert_eq!(run_part.run_part_id, "my_job-{\"runner-version\":\"3.2.1\",\"numeric-value\":123456789}-{\"job-total\":2,\"job-index\":1}");
608+
assert_json_snapshot!(run_part.metadata, @r#"
609+
{
610+
"job-index": 1,
611+
"job-total": 2,
612+
"numeric-value": 123456789,
613+
"runner-version": "3.2.1"
614+
}
615+
"#);
616+
},
617+
)
618+
}
310619
}

0 commit comments

Comments
 (0)