--wip-- [skip ci]

Author: GuillaumeLagrange

src/api_client.rs

@@ -225,7 +225,6 @@ nest! {
     pub struct CompareRunsHeadRun {
         pub id: String,
         pub status: RunStatus,
-        pub url: String,
     }
 }
 
@@ -234,11 +233,9 @@ nest! {
     #[serde(rename_all = "camelCase")]*
     struct CompareRunsData {
         repository: struct CompareRunsRepository {
-            settings: struct CompareRunsSettings {
-                allowed_regression: f64,
-            },
             paginated_compare_runs: pub struct CompareRunsComparison {
                 pub impact: Option<f64>,
+                pub url: String,
                 pub head_run: CompareRunsHeadRun,
                 pub result_comparisons: Vec<CompareRunsBenchmarkResult>,
             },
@@ -247,10 +244,15 @@ nest! {
 }
 
 pub struct CompareRunsResponse {
-    pub allowed_regression: f64,
     pub comparison: CompareRunsComparison,
 }
 
+pub enum CompareRunsOutcome {
+    Success(CompareRunsResponse),
+    BaseRunNotFound,
+    ExecutorMismatch,
+}
+
 #[derive(Serialize, Clone)]
 #[serde(rename_all = "camelCase")]
 pub struct GetOrCreateProjectRepositoryVars {
@@ -321,8 +323,7 @@ impl CodSpeedAPIClient {
         }
     }
 
-    /// Returns `None` if the base run was not found.
-    pub async fn compare_runs(&self, vars: CompareRunsVars) -> Result<Option<CompareRunsResponse>> {
+    pub async fn compare_runs(&self, vars: CompareRunsVars) -> Result<CompareRunsOutcome> {
         let response = self
             .gql_client
             .query_with_vars_unwrap::<CompareRunsData, CompareRunsVars>(
@@ -331,15 +332,19 @@ impl CodSpeedAPIClient {
             )
             .await;
         match response {
-            Ok(response) => Ok(Some(CompareRunsResponse {
-                allowed_regression: response.repository.settings.allowed_regression,
+            Ok(response) => Ok(CompareRunsOutcome::Success(CompareRunsResponse {
                 comparison: response.repository.paginated_compare_runs,
             })),
             Err(err) if err.contains_error_code("UNAUTHENTICATED") => {
                 bail!("Your session has expired, please login again using `codspeed auth login`")
             }
-            Err(err) if err.contains_error_code("RUN_NOT_FOUND") => Ok(None),
-            Err(err) => bail!("Failed to compare runs: {err}"),
+            Err(err) if err.contains_error_code("RUN_NOT_FOUND") => {
+                Ok(CompareRunsOutcome::BaseRunNotFound)
+            }
+            Err(err) if err.contains_error_code("NOT_FOUND") => {
+                Ok(CompareRunsOutcome::ExecutorMismatch)
+            }
+            Err(err) => bail!("Failed to compare runs: {err:?}"),
         }
     }
 

src/queries/CompareRuns.gql

@@ -1,14 +1,11 @@
 query CompareRuns($owner: String!, $name: String!, $baseRunId: String!, $headRunId: String!) {
   repository(owner: $owner, name: $name) {
-    settings {
-      allowedRegression
-    }
     paginatedCompareRuns(baseRunId: $baseRunId, headRunId: $headRunId) {
       impact
+      url
       headRun {
         id
         status
-        url
       }
       resultComparisons {
         benchmark {

src/upload/benchmark_display.rs

@@ -11,6 +11,9 @@ use tabled::settings::style::HorizontalLine;
 use tabled::settings::{Alignment, Color, Modify, Padding, Style};
 use tabled::{Table, Tabled};
 
+/// Changes below this threshold are displayed as "~0%" to avoid noise.
+const CHANGE_DISPLAY_EPSILON: f64 = 0.005;
+
 fn format_with_thousands_sep(n: u64) -> String {
     let s = n.to_string();
     let mut result = String::new();
@@ -366,15 +369,17 @@ pub fn build_comparison_table(results: &[CompareRunsBenchmarkResult]) -> String
                     };
 
                     let change_str = match result.change {
+                        Some(c) if c.abs() < CHANGE_DISPLAY_EPSILON => {
+                            format!("{}", style("~0%").dim())
+                        }
                         Some(c) if c > 0.0 => {
                             let pct = (c * 100.0).round();
                             format!("{}", style(format!("+{pct}%")).red().bold())
                         }
-                        Some(c) if c < 0.0 => {
+                        Some(c) => {
                             let pct = (c * 100.0).round();
                             format!("{}", style(format!("{pct}%")).green().bold())
                         }
-                        Some(_) => format!("{}", style("0%").dim()),
                         None => "-".to_string(),
                     };
 

src/upload/poll_results.rs

@@ -8,8 +8,8 @@ use super::benchmark_display::{
     build_benchmark_table, build_comparison_table, build_detailed_summary,
 };
 use crate::api_client::{
-    CodSpeedAPIClient, CompareRunsResponse, CompareRunsVars, FetchLocalRunResponse,
-    FetchLocalRunVars, RunStatus,
+    CodSpeedAPIClient, CompareRunsOutcome, CompareRunsResponse, CompareRunsVars,
+    FetchLocalRunResponse, FetchLocalRunVars, RunStatus,
 };
 use crate::local_logger::{start_spinner, stop_spinner};
 use crate::prelude::*;
@@ -48,12 +48,18 @@ pub async fn poll_results(
         stop_spinner();
 
         match compare_result? {
-            Some(response) => {
+            CompareRunsOutcome::Success(response) => {
                 return display_comparison_results(upload_result, options, response).await;
             }
-            None => {
+            // Fall back to single run display when comparison is not possible
+            CompareRunsOutcome::BaseRunNotFound => {
                 warn!(
-                    "Base run ID \"{base_run_id}\" was not found, falling back to single-run results"
+                    "Base run ID \"{base_run_id}\" was not found, we cannot compare results against it."
+                );
+            }
+            CompareRunsOutcome::ExecutorMismatch => {
+                warn!(
+                    "Base run ID \"{base_run_id}\" uses a different executor, we cannot compare results against it."
                 );
             }
         }
@@ -118,18 +124,34 @@ async fn poll_compare_runs(
     api_client: &CodSpeedAPIClient,
     upload_result: &UploadResult,
     base_run_id: &str,
-) -> Result<Option<CompareRunsResponse>> {
+) -> Result<CompareRunsOutcome> {
     let vars = CompareRunsVars {
         owner: upload_result.owner.clone(),
         name: upload_result.repository.clone(),
         base_run_id: base_run_id.to_string(),
         head_run_id: upload_result.run_id.clone(),
     };
-    poll_until_processed(
-        || api_client.compare_runs(vars.clone()),
-        |r: &CompareRunsResponse| &r.comparison.head_run.status,
-    )
-    .await
+
+    let start = Instant::now();
+    debug!("Waiting for results to be processed...");
+
+    loop {
+        if start.elapsed() > RUN_PROCESSING_MAX_DURATION {
+            bail!("Polling results timed out after 5 minutes. Please try again later.");
+        }
+
+        match api_client.compare_runs(vars.clone()).await? {
+            outcome @ (CompareRunsOutcome::BaseRunNotFound
+            | CompareRunsOutcome::ExecutorMismatch) => return Ok(outcome),
+            CompareRunsOutcome::Success(response) => match &response.comparison.head_run.status {
+                RunStatus::Pending | RunStatus::Processing => sleep(POLLING_INTERVAL).await,
+                RunStatus::Failure => {
+                    bail!("Run failed to be processed, try again in a few minutes")
+                }
+                _ => return Ok(CompareRunsOutcome::Success(response)),
+            },
+        }
+    }
 }
 
 async fn display_single_run_results(
@@ -175,7 +197,7 @@ async fn display_single_run_results(
             style("View full report:").dim(),
             style(&response.run.url).blue().bold().underlined(),
             style("To compare future runs against this one, use:").dim(),
-            style(format!("codspeed run --base {run_id} <command>")).cyan(),
+            style(format!("--base {run_id}")).cyan(),
         );
     }
 
@@ -189,28 +211,6 @@ async fn display_comparison_results(
 ) -> Result<()> {
     let comparison = &response.comparison;
 
-    if let Some(impact) = comparison.impact {
-        let rounded_impact = (impact * 100.0).round();
-        let (arrow, impact_text) = if impact > 0.0 {
-            (
-                style("\u{f062}").green(),
-                style(format!("+{rounded_impact}%")).green().bold(),
-            )
-        } else if impact < 0.0 {
-            (
-                style("\u{f063}").red(),
-                style(format!("{rounded_impact}%")).red().bold(),
-            )
-        } else {
-            (
-                style("\u{25CF}").dim(),
-                style(format!("{rounded_impact}%")).dim().bold(),
-            )
-        };
-        let allowed = (response.allowed_regression * 100.0).round();
-        info!("{arrow} Impact: {impact_text} (allowed regression: -{allowed}%)");
-    }
-
     if options.output_json {
         log_json!(format!(
             "{{\"event\": \"run_finished\", \"run_id\": \"{}\"}}",
@@ -226,6 +226,27 @@ async fn display_comparison_results(
         end_group!();
         start_opened_group!("Benchmark results");
 
+        if let Some(impact) = comparison.impact {
+            let rounded_impact = (impact * 100.0).round();
+            let (arrow, impact_text) = if impact > 0.0 {
+                (
+                    style("\u{f062}").green(),
+                    style(format!("+{rounded_impact}%")).green().bold(),
+                )
+            } else if impact < 0.0 {
+                (
+                    style("\u{f063}").red(),
+                    style(format!("{rounded_impact}%")).red().bold(),
+                )
+            } else {
+                (
+                    style("\u{25CF}").dim(),
+                    style(format!("{rounded_impact}%")).dim().bold(),
+                )
+            };
+            info!("{arrow} Impact: {impact_text}");
+        }
+
         let table = build_comparison_table(&comparison.result_comparisons);
         info!("{table}\n");
 
@@ -243,7 +264,7 @@ async fn display_comparison_results(
         info!(
             "\n{} {}",
             style("View comparison report:").dim(),
-            style(&comparison.head_run.url).blue().bold().underlined()
+            style(&comparison.url).blue().bold().underlined()
         );
     }