Skip to content

Commit eefc308

Browse files
GuillaumeLagrangeGuillaumeLagrange
committed
feat: factorize polling code and add retry logic on timeout
In dev environments, the request often times out because of cold start and sst building the handler. Adding some retry on this makes the runner more resilient.
1 parent 4dc171b commit eefc308

4 files changed

Lines changed: 133 additions & 108 deletions

File tree

‎src/exec/poll_results.rs‎

Lines changed: 11 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,13 @@
1-
use std::time::Duration;
2-
31
use console::style;
4-
use tabled::settings::Style;
5-
use tabled::{Table, Tabled};
62
use tokio::time::{Instant, sleep};
73

84
use crate::api_client::{
95
CodSpeedAPIClient, FetchLocalExecReportResponse, FetchLocalExecReportVars, RunStatus,
106
};
117
use crate::prelude::*;
12-
use crate::run::helpers;
13-
14-
const RUN_PROCESSING_MAX_DURATION: Duration = Duration::from_secs(60 * 5); // 5 minutes
15-
const POLLING_INTERVAL: Duration = Duration::from_secs(1);
16-
17-
#[derive(Tabled)]
18-
struct BenchmarkRow {
19-
#[tabled(rename = "Benchmark")]
20-
name: String,
21-
#[tabled(rename = "Time")]
22-
time: String,
23-
}
24-
25-
fn build_benchmark_table(results: &[crate::api_client::FetchLocalRunBenchmarkResult]) -> String {
26-
let table_rows: Vec<BenchmarkRow> = results
27-
.iter()
28-
.map(|result| BenchmarkRow {
29-
name: result.benchmark.name.clone(),
30-
time: helpers::format_duration(result.time, Some(2)),
31-
})
32-
.collect();
33-
34-
Table::new(&table_rows).with(Style::modern()).to_string()
35-
}
8+
use crate::run::helpers::poll_results::{
9+
build_benchmark_table, retry_on_timeout, POLLING_INTERVAL, RUN_PROCESSING_MAX_DURATION,
10+
};
3611

3712
#[allow(clippy::borrowed_box)]
3813
pub async fn poll_results(api_client: &CodSpeedAPIClient, run_id: String) -> Result<()> {
@@ -50,10 +25,14 @@ pub async fn poll_results(api_client: &CodSpeedAPIClient, run_id: String) -> Res
5025
bail!("Polling results timed out");
5126
}
5227

53-
match api_client
54-
.fetch_local_exec_report(fetch_local_exec_report_vars.clone())
55-
.await?
56-
{
28+
let fetch_result = retry_on_timeout(|| async {
29+
api_client
30+
.fetch_local_exec_report(fetch_local_exec_report_vars.clone())
31+
.await
32+
})
33+
.await?;
34+
35+
match fetch_result {
5736
FetchLocalExecReportResponse { run, .. }
5837
if run.status == RunStatus::Pending || run.status == RunStatus::Processing =>
5938
{

‎src/run/helpers/mod.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ mod find_repository_root;
33
mod format_duration;
44
mod get_env_var;
55
mod parse_git_remote;
6+
pub(crate) mod poll_results;
67

78
pub(crate) use download_file::download_file;
89
pub(crate) use find_repository_root::find_repository_root;

‎src/run/helpers/poll_results.rs‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
use std::future::Future;
2+
use std::time::Duration;
3+
4+
use tabled::settings::Style;
5+
use tabled::{Table, Tabled};
6+
use tokio::time::sleep;
7+
8+
use crate::prelude::*;
9+
use crate::run::helpers;
10+
11+
pub const RUN_PROCESSING_MAX_DURATION: Duration = Duration::from_secs(60 * 5); // 5 minutes
12+
pub const POLLING_INTERVAL: Duration = Duration::from_secs(1);
13+
pub const MAX_FETCH_RETRIES: u32 = 3;
14+
pub const FETCH_RETRY_DELAY: Duration = Duration::from_secs(5);
15+
16+
#[derive(Tabled)]
17+
struct BenchmarkRow {
18+
#[tabled(rename = "Benchmark")]
19+
name: String,
20+
#[tabled(rename = "Time")]
21+
time: String,
22+
}
23+
24+
pub fn build_benchmark_table(
25+
results: &[crate::api_client::FetchLocalRunBenchmarkResult],
26+
) -> String {
27+
let table_rows: Vec<BenchmarkRow> = results
28+
.iter()
29+
.map(|result| BenchmarkRow {
30+
name: result.benchmark.name.clone(),
31+
time: helpers::format_duration(result.time, Some(2)),
32+
})
33+
.collect();
34+
35+
Table::new(&table_rows).with(Style::modern()).to_string()
36+
}
37+
38+
/// Retry logic for API calls that may timeout due to cold start in dev environments
39+
pub async fn retry_on_timeout<F, Fut, T>(fetch_fn: F) -> Result<T>
40+
where
41+
F: Fn() -> Fut,
42+
Fut: Future<Output = Result<T>>,
43+
{
44+
let mut fetch_attempt = 0;
45+
loop {
46+
fetch_attempt += 1;
47+
match fetch_fn().await {
48+
Ok(result) => return Ok(result),
49+
Err(err) => {
50+
let error_message = err.to_string();
51+
let is_timeout =
52+
error_message.contains("timed out") || error_message.contains("timeout");
53+
54+
if is_timeout && fetch_attempt < MAX_FETCH_RETRIES {
55+
debug!(
56+
"Fetch request timed out (attempt {fetch_attempt}/{MAX_FETCH_RETRIES}), retrying in {FETCH_RETRY_DELAY:?}..."
57+
);
58+
sleep(FETCH_RETRY_DELAY).await;
59+
continue;
60+
}
61+
62+
return Err(err);
63+
}
64+
}
65+
}
66+
}
67+
68+
#[cfg(test)]
69+
mod tests {
70+
use super::*;
71+
use crate::api_client::{FetchLocalRunBenchmark, FetchLocalRunBenchmarkResult};
72+
73+
#[test]
74+
fn test_benchmark_table_formatting() {
75+
let results = vec![
76+
FetchLocalRunBenchmarkResult {
77+
benchmark: FetchLocalRunBenchmark {
78+
name: "benchmark_fast".to_string(),
79+
},
80+
time: 0.001234, // 1.23 ms
81+
},
82+
FetchLocalRunBenchmarkResult {
83+
benchmark: FetchLocalRunBenchmark {
84+
name: "benchmark_slow".to_string(),
85+
},
86+
time: 1.5678, // 1.57 s
87+
},
88+
FetchLocalRunBenchmarkResult {
89+
benchmark: FetchLocalRunBenchmark {
90+
name: "benchmark_medium".to_string(),
91+
},
92+
time: 0.000567, // 567 µs
93+
},
94+
];
95+
96+
let table = build_benchmark_table(&results);
97+
98+
insta::assert_snapshot!(table, @r###"
99+
┌──────────────────┬───────────┐
100+
│ Benchmark │ Time │
101+
├──────────────────┼───────────┤
102+
│ benchmark_fast │ 1.23 ms │
103+
├──────────────────┼───────────┤
104+
│ benchmark_slow │ 1.57 s │
105+
├──────────────────┼───────────┤
106+
│ benchmark_medium │ 567.00 µs │
107+
└──────────────────┴───────────┘
108+
"###);
109+
}
110+
}

‎src/run/poll_results.rs‎

Lines changed: 11 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -1,40 +1,15 @@
1-
use std::time::Duration;
2-
31
use console::style;
4-
use tabled::settings::Style;
5-
use tabled::{Table, Tabled};
62
use tokio::time::{Instant, sleep};
73

84
use crate::api_client::{
95
CodSpeedAPIClient, FetchLocalRunReportResponse, FetchLocalRunReportVars, RunStatus,
106
};
117
use crate::prelude::*;
12-
use crate::run::helpers;
8+
use crate::run::helpers::poll_results::{
9+
POLLING_INTERVAL, RUN_PROCESSING_MAX_DURATION, build_benchmark_table, retry_on_timeout,
10+
};
1311
use crate::run_environment::RunEnvironmentProvider;
1412

15-
const RUN_PROCESSING_MAX_DURATION: Duration = Duration::from_secs(60 * 5); // 5 minutes
16-
const POLLING_INTERVAL: Duration = Duration::from_secs(1);
17-
18-
#[derive(Tabled)]
19-
struct BenchmarkRow {
20-
#[tabled(rename = "Benchmark")]
21-
name: String,
22-
#[tabled(rename = "Time")]
23-
time: String,
24-
}
25-
26-
fn build_benchmark_table(results: &[crate::api_client::FetchLocalRunBenchmarkResult]) -> String {
27-
let table_rows: Vec<BenchmarkRow> = results
28-
.iter()
29-
.map(|result| BenchmarkRow {
30-
name: result.benchmark.name.clone(),
31-
time: helpers::format_duration(result.time, Some(2)),
32-
})
33-
.collect();
34-
35-
Table::new(&table_rows).with(Style::modern()).to_string()
36-
}
37-
3813
#[allow(clippy::borrowed_box)]
3914
pub async fn poll_results(
4015
api_client: &CodSpeedAPIClient,
@@ -59,10 +34,14 @@ pub async fn poll_results(
5934
bail!("Polling results timed out");
6035
}
6136

62-
match api_client
63-
.fetch_local_run_report(fetch_local_run_report_vars.clone())
64-
.await?
65-
{
37+
let fetch_result = retry_on_timeout(|| async {
38+
api_client
39+
.fetch_local_run_report(fetch_local_run_report_vars.clone())
40+
.await
41+
})
42+
.await?;
43+
44+
match fetch_result {
6645
FetchLocalRunReportResponse { run, .. }
6746
if run.status == RunStatus::Pending || run.status == RunStatus::Processing =>
6847
{
@@ -139,47 +118,3 @@ pub async fn poll_results(
139118

140119
Ok(())
141120
}
142-
143-
#[cfg(test)]
144-
mod tests {
145-
use super::*;
146-
use crate::api_client::{FetchLocalRunBenchmark, FetchLocalRunBenchmarkResult};
147-
148-
#[test]
149-
fn test_benchmark_table_formatting() {
150-
let results = vec![
151-
FetchLocalRunBenchmarkResult {
152-
benchmark: FetchLocalRunBenchmark {
153-
name: "benchmark_fast".to_string(),
154-
},
155-
time: 0.001234, // 1.23 ms
156-
},
157-
FetchLocalRunBenchmarkResult {
158-
benchmark: FetchLocalRunBenchmark {
159-
name: "benchmark_slow".to_string(),
160-
},
161-
time: 1.5678, // 1.57 s
162-
},
163-
FetchLocalRunBenchmarkResult {
164-
benchmark: FetchLocalRunBenchmark {
165-
name: "benchmark_medium".to_string(),
166-
},
167-
time: 0.000567, // 567 µs
168-
},
169-
];
170-
171-
let table = build_benchmark_table(&results);
172-
173-
insta::assert_snapshot!(table, @r###"
174-
┌──────────────────┬───────────┐
175-
│ Benchmark │ Time │
176-
├──────────────────┼───────────┤
177-
│ benchmark_fast │ 1.23 ms │
178-
├──────────────────┼───────────┤
179-
│ benchmark_slow │ 1.57 s │
180-
├──────────────────┼───────────┤
181-
│ benchmark_medium │ 567.00 µs │
182-
└──────────────────┴───────────┘
183-
"###);
184-
}
185-
}

0 commit comments

Comments
 (0)