From b8bacec8fcce71d171e927bb9a72b585d4f83cf4 Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Mon, 18 May 2026 17:07:16 -0400 Subject: [PATCH 1/2] test(e2e): close Podman driver test coverage gaps Add podman_gateway_resume test following the VM pattern (no container-state assertions since Podman keeps containers running across gateway restarts). Widen websocket_conformance feature gate from e2e-docker to e2e-host-gateway so it runs on both Docker and Podman. Fix e2e-podman.sh to default to the e2e-podman feature so Podman- specific and host-gateway tests are actually included in Podman CI runs. --- e2e/rust/Cargo.toml | 7 +- e2e/rust/e2e-podman.sh | 2 +- e2e/rust/tests/podman_gateway_resume.rs | 178 ++++++++++++++++++++++++ e2e/rust/tests/websocket_conformance.rs | 6 +- 4 files changed, 188 insertions(+), 5 deletions(-) create mode 100644 e2e/rust/tests/podman_gateway_resume.rs diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index 03ad930b4..7d7f1411c 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -46,6 +46,11 @@ name = "gateway_resume" path = "tests/gateway_resume.rs" required-features = ["e2e-docker"] +[[test]] +name = "podman_gateway_resume" +path = "tests/podman_gateway_resume.rs" +required-features = ["e2e-podman"] + [[test]] name = "vm_gateway_resume" path = "tests/vm_gateway_resume.rs" @@ -54,7 +59,7 @@ required-features = ["e2e-vm"] [[test]] name = "websocket_conformance" path = "tests/websocket_conformance.rs" -required-features = ["e2e-docker"] +required-features = ["e2e-host-gateway"] [[test]] name = "user_namespaces" diff --git a/e2e/rust/e2e-podman.sh b/e2e/rust/e2e-podman.sh index c82891338..5d4cc99e6 100755 --- a/e2e/rust/e2e-podman.sh +++ b/e2e/rust/e2e-podman.sh @@ -10,7 +10,7 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" E2E_TEST="${OPENSHELL_E2E_PODMAN_TEST:-}" -E2E_FEATURES="${OPENSHELL_E2E_PODMAN_FEATURES:-e2e}" +E2E_FEATURES="${OPENSHELL_E2E_PODMAN_FEATURES:-e2e-podman}" cargo build -p openshell-cli --features openshell-core/dev-settings diff --git a/e2e/rust/tests/podman_gateway_resume.rs b/e2e/rust/tests/podman_gateway_resume.rs new file mode 100644 index 000000000..0abc7a837 --- /dev/null +++ b/e2e/rust/tests/podman_gateway_resume.rs @@ -0,0 +1,178 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#![cfg(feature = "e2e-podman")] + +//! Podman-specific E2E coverage for resuming sandboxes after a standalone +//! gateway restart. +//! +//! Unlike the Docker driver, Podman does not stop sandbox containers when the +//! gateway process exits — the containers keep running and the restarted +//! gateway re-adopts them. This test follows the `vm_gateway_resume.rs` +//! pattern: verify sandbox survival at the application level without asserting +//! intermediate container-state transitions. + +use std::process::Stdio; +use std::time::{Duration, Instant}; + +use openshell_e2e::harness::binary::openshell_cmd; +use openshell_e2e::harness::gateway::ManagedGateway; +use openshell_e2e::harness::output::strip_ansi; +use openshell_e2e::harness::sandbox::SandboxGuard; +use tokio::time::sleep; + +const READY_MARKER: &str = "podman-gateway-resume-ready"; +const RESUME_FILE: &str = "/sandbox/podman-gateway-resume-state"; + +async fn run_cli(args: &[&str]) -> (String, i32) { + let mut cmd = openshell_cmd(); + cmd.args(args).stdout(Stdio::piped()).stderr(Stdio::piped()); + + let output = cmd.output().await.expect("spawn openshell"); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + let combined = format!("{stdout}{stderr}"); + let code = output.status.code().unwrap_or(-1); + (combined, code) +} + +async fn wait_for_healthy(timeout: Duration) -> Result<(), String> { + let start = Instant::now(); + let mut last_output: String; + + loop { + let (output, code) = run_cli(&["status"]).await; + let clean = strip_ansi(&output); + let lower = clean.to_lowercase(); + if code == 0 + && (lower.contains("healthy") + || lower.contains("running") + || lower.contains("connected")) + { + return Ok(()); + } + last_output = clean; + + if start.elapsed() > timeout { + return Err(format!( + "gateway did not become healthy within {}s. Last output:\n{last_output}", + timeout.as_secs() + )); + } + sleep(Duration::from_secs(2)).await; + } +} + +async fn sandbox_names() -> Result, String> { + let (output, code) = run_cli(&["sandbox", "list", "--names"]).await; + let clean = strip_ansi(&output); + if code != 0 { + return Err(format!("sandbox list failed (exit {code}):\n{clean}")); + } + + Ok(clean + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + .map(ToOwned::to_owned) + .collect()) +} + +async fn wait_for_sandbox_exec_contains( + sandbox_name: &str, + command: &[&str], + expected: &str, + timeout: Duration, +) -> Result<(), String> { + let start = Instant::now(); + let mut last_output: String; + + loop { + let mut cmd = openshell_cmd(); + cmd.args(["sandbox", "exec", "--name", sandbox_name, "--no-tty", "--"]) + .args(command) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + match cmd.output().await { + Ok(output) => { + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + last_output = strip_ansi(&format!("{stdout}{stderr}")); + if output.status.success() && last_output.contains(expected) { + return Ok(()); + } + } + Err(err) => { + last_output = format!("failed to spawn openshell sandbox exec: {err}"); + } + } + + if start.elapsed() > timeout { + return Err(format!( + "sandbox '{sandbox_name}' exec did not produce '{expected}' within {}s. Last output:\n{last_output}", + timeout.as_secs() + )); + } + sleep(Duration::from_secs(2)).await; + } +} + +#[tokio::test] +async fn podman_gateway_restart_resumes_running_sandbox() { + if std::env::var("OPENSHELL_E2E_DRIVER").as_deref() != Ok("podman") { + eprintln!("Skipping Podman gateway resume test: e2e driver is not podman"); + return; + } + let Some(gateway) = ManagedGateway::from_env().expect("load managed e2e gateway metadata") + else { + eprintln!( + "Skipping Podman gateway resume test: e2e gateway is not managed by this test run" + ); + return; + }; + + wait_for_healthy(Duration::from_secs(30)) + .await + .expect("gateway should start healthy"); + + let script = format!( + "echo before-restart > {RESUME_FILE}; echo {READY_MARKER}; while true; do sleep 1; done" + ); + let mut sandbox = SandboxGuard::create_keep(&["sh", "-lc", &script], READY_MARKER) + .await + .expect("create long-running Podman sandbox"); + + let before_restart = sandbox + .exec(&["cat", RESUME_FILE]) + .await + .expect("read Podman sandbox state before restart"); + assert!( + before_restart.contains("before-restart"), + "sandbox state was not written before restart:\n{before_restart}" + ); + + gateway.stop().expect("stop e2e gateway"); + gateway.start().expect("restart e2e gateway"); + wait_for_healthy(Duration::from_secs(120)) + .await + .expect("gateway should become healthy after restart"); + + let names = sandbox_names().await.expect("list sandboxes after restart"); + assert!( + names.contains(&sandbox.name), + "sandbox '{}' should still be listed after gateway restart. Names: {names:?}", + sandbox.name + ); + + wait_for_sandbox_exec_contains( + &sandbox.name, + &["cat", RESUME_FILE], + "before-restart", + Duration::from_secs(240), + ) + .await + .expect("Podman sandbox should become ready again with its state preserved"); + + sandbox.cleanup().await; +} diff --git a/e2e/rust/tests/websocket_conformance.rs b/e2e/rust/tests/websocket_conformance.rs index d87c9b9dd..65ba19aa1 100644 --- a/e2e/rust/tests/websocket_conformance.rs +++ b/e2e/rust/tests/websocket_conformance.rs @@ -3,8 +3,8 @@ #![cfg(feature = "e2e")] -//! E2E regression: WebSocket credential placeholders are resolved on the real -//! Docker-backed sandbox path after an RFC 6455 upgrade. +//! E2E regression: WebSocket credential placeholders are resolved on the +//! sandbox path after an RFC 6455 upgrade. //! //! The sandbox process sends its provider-managed placeholder in a masked text //! frame. The local upstream only reports whether it saw the real secret and @@ -425,7 +425,7 @@ with connect_with_retry(HOST, PORT) as sock: } #[tokio::test] -async fn websocket_text_placeholder_is_rewritten_in_docker_sandbox() { +async fn websocket_text_placeholder_is_rewritten_in_sandbox() { let _provider_lock = PROVIDER_LOCK .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); From cb9c2ccff724064094575811056bd53a2dd0782a Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Mon, 18 May 2026 18:44:57 -0400 Subject: [PATCH 2/2] refactor(e2e): extract shared CLI helpers from gateway_resume tests Move run_cli, wait_for_healthy, sandbox_names, and wait_for_sandbox_exec_contains into a new harness::cli module. All three gateway_resume variants (Docker, Podman, VM) now import these shared helpers instead of defining identical copies. Docker-specific container introspection functions remain local to gateway_resume.rs. --- e2e/rust/src/harness/cli.rs | 107 ++++++++++++++++++++++ e2e/rust/src/harness/mod.rs | 1 + e2e/rust/tests/gateway_resume.rs | 112 ++---------------------- e2e/rust/tests/podman_gateway_resume.rs | 101 +-------------------- e2e/rust/tests/vm_gateway_resume.rs | 110 ++--------------------- 5 files changed, 123 insertions(+), 308 deletions(-) create mode 100644 e2e/rust/src/harness/cli.rs diff --git a/e2e/rust/src/harness/cli.rs b/e2e/rust/src/harness/cli.rs new file mode 100644 index 000000000..53392d752 --- /dev/null +++ b/e2e/rust/src/harness/cli.rs @@ -0,0 +1,107 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Shared CLI helpers for e2e tests that need to invoke `openshell` commands +//! and poll for readiness. + +use std::process::Stdio; +use std::time::{Duration, Instant}; + +use tokio::time::sleep; + +use super::binary::openshell_cmd; +use super::output::strip_ansi; + +pub async fn run_cli(args: &[&str]) -> (String, i32) { + let mut cmd = openshell_cmd(); + cmd.args(args).stdout(Stdio::piped()).stderr(Stdio::piped()); + + let output = cmd.output().await.expect("spawn openshell"); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + let combined = format!("{stdout}{stderr}"); + let code = output.status.code().unwrap_or(-1); + (combined, code) +} + +pub async fn wait_for_healthy(timeout: Duration) -> Result<(), String> { + let start = Instant::now(); + let mut last_output: String; + + loop { + let (output, code) = run_cli(&["status"]).await; + let clean = strip_ansi(&output); + let lower = clean.to_lowercase(); + if code == 0 + && (lower.contains("healthy") + || lower.contains("running") + || lower.contains("connected")) + { + return Ok(()); + } + last_output = clean; + + if start.elapsed() > timeout { + return Err(format!( + "gateway did not become healthy within {}s. Last output:\n{last_output}", + timeout.as_secs() + )); + } + sleep(Duration::from_secs(2)).await; + } +} + +pub async fn sandbox_names() -> Result, String> { + let (output, code) = run_cli(&["sandbox", "list", "--names"]).await; + let clean = strip_ansi(&output); + if code != 0 { + return Err(format!("sandbox list failed (exit {code}):\n{clean}")); + } + + Ok(clean + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + .map(ToOwned::to_owned) + .collect()) +} + +pub async fn wait_for_sandbox_exec_contains( + sandbox_name: &str, + command: &[&str], + expected: &str, + timeout: Duration, +) -> Result<(), String> { + let start = Instant::now(); + let mut last_output: String; + + loop { + let mut cmd = openshell_cmd(); + cmd.args(["sandbox", "exec", "--name", sandbox_name, "--no-tty", "--"]) + .args(command) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + match cmd.output().await { + Ok(output) => { + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + last_output = strip_ansi(&format!("{stdout}{stderr}")); + if output.status.success() && last_output.contains(expected) { + return Ok(()); + } + } + Err(err) => { + last_output = format!("failed to spawn openshell sandbox exec: {err}"); + } + } + + if start.elapsed() > timeout { + return Err(format!( + "sandbox '{sandbox_name}' exec did not produce '{expected}' within {}s. Last output:\n{last_output}", + timeout.as_secs() + )); + } + sleep(Duration::from_secs(2)).await; + } +} diff --git a/e2e/rust/src/harness/mod.rs b/e2e/rust/src/harness/mod.rs index 5feb21c70..f2dfd5ec9 100644 --- a/e2e/rust/src/harness/mod.rs +++ b/e2e/rust/src/harness/mod.rs @@ -4,6 +4,7 @@ //! Shared test harness modules for CLI e2e tests. pub mod binary; +pub mod cli; pub mod container; pub mod gateway; pub mod output; diff --git a/e2e/rust/tests/gateway_resume.rs b/e2e/rust/tests/gateway_resume.rs index e3a2e6664..8f850e485 100644 --- a/e2e/rust/tests/gateway_resume.rs +++ b/e2e/rust/tests/gateway_resume.rs @@ -10,11 +10,12 @@ //! gateway process, so they skip this restart-only coverage. use std::process::{Command, Stdio}; -use std::time::{Duration, Instant}; +use std::time::Duration; -use openshell_e2e::harness::binary::openshell_cmd; +use openshell_e2e::harness::cli::{ + sandbox_names, wait_for_healthy, wait_for_sandbox_exec_contains, +}; use openshell_e2e::harness::gateway::ManagedGateway; -use openshell_e2e::harness::output::strip_ansi; use openshell_e2e::harness::sandbox::SandboxGuard; use tokio::time::sleep; @@ -24,100 +25,6 @@ const RESUME_FILE: &str = "/sandbox/gateway-resume-state"; const SANDBOX_NAMESPACE_LABEL: &str = "openshell.ai/sandbox-namespace"; const SANDBOX_NAME_LABEL: &str = "openshell.ai/sandbox-name"; -async fn run_cli(args: &[&str]) -> (String, i32) { - let mut cmd = openshell_cmd(); - cmd.args(args).stdout(Stdio::piped()).stderr(Stdio::piped()); - - let output = cmd.output().await.expect("spawn openshell"); - let stdout = String::from_utf8_lossy(&output.stdout); - let stderr = String::from_utf8_lossy(&output.stderr); - let combined = format!("{stdout}{stderr}"); - let code = output.status.code().unwrap_or(-1); - (combined, code) -} - -async fn wait_for_healthy(timeout: Duration) -> Result<(), String> { - let start = Instant::now(); - let mut last_output: String; - - loop { - let (output, code) = run_cli(&["status"]).await; - let clean = strip_ansi(&output); - let lower = clean.to_lowercase(); - if code == 0 - && (lower.contains("healthy") - || lower.contains("running") - || lower.contains("connected")) - { - return Ok(()); - } - last_output = clean; - - if start.elapsed() > timeout { - return Err(format!( - "gateway did not become healthy within {}s. Last output:\n{last_output}", - timeout.as_secs() - )); - } - sleep(Duration::from_secs(2)).await; - } -} - -async fn sandbox_names() -> Result, String> { - let (output, code) = run_cli(&["sandbox", "list", "--names"]).await; - let clean = strip_ansi(&output); - if code != 0 { - return Err(format!("sandbox list failed (exit {code}):\n{clean}")); - } - - Ok(clean - .lines() - .map(str::trim) - .filter(|line| !line.is_empty()) - .map(ToOwned::to_owned) - .collect()) -} - -async fn wait_for_sandbox_exec_contains( - sandbox_name: &str, - command: &[&str], - expected: &str, - timeout: Duration, -) -> Result<(), String> { - let start = Instant::now(); - let mut last_output: String; - - loop { - let mut cmd = openshell_cmd(); - cmd.args(["sandbox", "exec", "--name", sandbox_name, "--no-tty", "--"]) - .args(command) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - - match cmd.output().await { - Ok(output) => { - let stdout = String::from_utf8_lossy(&output.stdout); - let stderr = String::from_utf8_lossy(&output.stderr); - last_output = strip_ansi(&format!("{stdout}{stderr}")); - if output.status.success() && last_output.contains(expected) { - return Ok(()); - } - } - Err(err) => { - last_output = format!("failed to spawn openshell sandbox exec: {err}"); - } - } - - if start.elapsed() > timeout { - return Err(format!( - "sandbox '{sandbox_name}' exec did not produce '{expected}' within {}s. Last output:\n{last_output}", - timeout.as_secs() - )); - } - sleep(Duration::from_secs(2)).await; - } -} - fn sandbox_container_id(namespace: &str, sandbox_name: &str) -> Result { let namespace_filter = format!("label={SANDBOX_NAMESPACE_LABEL}={namespace}"); let sandbox_name_filter = format!("label={SANDBOX_NAME_LABEL}={sandbox_name}"); @@ -189,7 +96,7 @@ async fn wait_for_container_running( expected: bool, timeout: Duration, ) -> Result<(), String> { - let start = Instant::now(); + let start = std::time::Instant::now(); let mut last_state: String; loop { @@ -231,12 +138,9 @@ async fn docker_gateway_restart_resumes_running_sandbox() { let script = format!( "echo before-restart > {RESUME_FILE}; echo {READY_MARKER}; while true; do sleep 1; done" ); - let mut sandbox = SandboxGuard::create_keep( - &["sh", "-lc", &script], - READY_MARKER, - ) - .await - .expect("create long-running sandbox"); + let mut sandbox = SandboxGuard::create_keep(&["sh", "-lc", &script], READY_MARKER) + .await + .expect("create long-running sandbox"); let before_restart = sandbox .exec(&["cat", RESUME_FILE]) diff --git a/e2e/rust/tests/podman_gateway_resume.rs b/e2e/rust/tests/podman_gateway_resume.rs index 0abc7a837..fea2fab3e 100644 --- a/e2e/rust/tests/podman_gateway_resume.rs +++ b/e2e/rust/tests/podman_gateway_resume.rs @@ -12,112 +12,15 @@ //! pattern: verify sandbox survival at the application level without asserting //! intermediate container-state transitions. -use std::process::Stdio; -use std::time::{Duration, Instant}; +use std::time::Duration; -use openshell_e2e::harness::binary::openshell_cmd; +use openshell_e2e::harness::cli::{sandbox_names, wait_for_healthy, wait_for_sandbox_exec_contains}; use openshell_e2e::harness::gateway::ManagedGateway; -use openshell_e2e::harness::output::strip_ansi; use openshell_e2e::harness::sandbox::SandboxGuard; -use tokio::time::sleep; const READY_MARKER: &str = "podman-gateway-resume-ready"; const RESUME_FILE: &str = "/sandbox/podman-gateway-resume-state"; -async fn run_cli(args: &[&str]) -> (String, i32) { - let mut cmd = openshell_cmd(); - cmd.args(args).stdout(Stdio::piped()).stderr(Stdio::piped()); - - let output = cmd.output().await.expect("spawn openshell"); - let stdout = String::from_utf8_lossy(&output.stdout); - let stderr = String::from_utf8_lossy(&output.stderr); - let combined = format!("{stdout}{stderr}"); - let code = output.status.code().unwrap_or(-1); - (combined, code) -} - -async fn wait_for_healthy(timeout: Duration) -> Result<(), String> { - let start = Instant::now(); - let mut last_output: String; - - loop { - let (output, code) = run_cli(&["status"]).await; - let clean = strip_ansi(&output); - let lower = clean.to_lowercase(); - if code == 0 - && (lower.contains("healthy") - || lower.contains("running") - || lower.contains("connected")) - { - return Ok(()); - } - last_output = clean; - - if start.elapsed() > timeout { - return Err(format!( - "gateway did not become healthy within {}s. Last output:\n{last_output}", - timeout.as_secs() - )); - } - sleep(Duration::from_secs(2)).await; - } -} - -async fn sandbox_names() -> Result, String> { - let (output, code) = run_cli(&["sandbox", "list", "--names"]).await; - let clean = strip_ansi(&output); - if code != 0 { - return Err(format!("sandbox list failed (exit {code}):\n{clean}")); - } - - Ok(clean - .lines() - .map(str::trim) - .filter(|line| !line.is_empty()) - .map(ToOwned::to_owned) - .collect()) -} - -async fn wait_for_sandbox_exec_contains( - sandbox_name: &str, - command: &[&str], - expected: &str, - timeout: Duration, -) -> Result<(), String> { - let start = Instant::now(); - let mut last_output: String; - - loop { - let mut cmd = openshell_cmd(); - cmd.args(["sandbox", "exec", "--name", sandbox_name, "--no-tty", "--"]) - .args(command) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - - match cmd.output().await { - Ok(output) => { - let stdout = String::from_utf8_lossy(&output.stdout); - let stderr = String::from_utf8_lossy(&output.stderr); - last_output = strip_ansi(&format!("{stdout}{stderr}")); - if output.status.success() && last_output.contains(expected) { - return Ok(()); - } - } - Err(err) => { - last_output = format!("failed to spawn openshell sandbox exec: {err}"); - } - } - - if start.elapsed() > timeout { - return Err(format!( - "sandbox '{sandbox_name}' exec did not produce '{expected}' within {}s. Last output:\n{last_output}", - timeout.as_secs() - )); - } - sleep(Duration::from_secs(2)).await; - } -} - #[tokio::test] async fn podman_gateway_restart_resumes_running_sandbox() { if std::env::var("OPENSHELL_E2E_DRIVER").as_deref() != Ok("podman") { diff --git a/e2e/rust/tests/vm_gateway_resume.rs b/e2e/rust/tests/vm_gateway_resume.rs index 488be681a..3bff91df7 100644 --- a/e2e/rust/tests/vm_gateway_resume.rs +++ b/e2e/rust/tests/vm_gateway_resume.rs @@ -9,112 +9,15 @@ //! This test is gated behind the `e2e-vm` feature because it requires the VM //! driver runtime prepared by `e2e/rust/e2e-vm.sh`. -use std::process::Stdio; -use std::time::{Duration, Instant}; +use std::time::Duration; -use openshell_e2e::harness::binary::openshell_cmd; +use openshell_e2e::harness::cli::{sandbox_names, wait_for_healthy, wait_for_sandbox_exec_contains}; use openshell_e2e::harness::gateway::ManagedGateway; -use openshell_e2e::harness::output::strip_ansi; use openshell_e2e::harness::sandbox::SandboxGuard; -use tokio::time::sleep; const READY_MARKER: &str = "vm-gateway-resume-ready"; const RESUME_FILE: &str = "/sandbox/vm-gateway-resume-state"; -async fn run_cli(args: &[&str]) -> (String, i32) { - let mut cmd = openshell_cmd(); - cmd.args(args).stdout(Stdio::piped()).stderr(Stdio::piped()); - - let output = cmd.output().await.expect("spawn openshell"); - let stdout = String::from_utf8_lossy(&output.stdout); - let stderr = String::from_utf8_lossy(&output.stderr); - let combined = format!("{stdout}{stderr}"); - let code = output.status.code().unwrap_or(-1); - (combined, code) -} - -async fn wait_for_healthy(timeout: Duration) -> Result<(), String> { - let start = Instant::now(); - let mut last_output: String; - - loop { - let (output, code) = run_cli(&["status"]).await; - let clean = strip_ansi(&output); - let lower = clean.to_lowercase(); - if code == 0 - && (lower.contains("healthy") - || lower.contains("running") - || lower.contains("connected")) - { - return Ok(()); - } - last_output = clean; - - if start.elapsed() > timeout { - return Err(format!( - "gateway did not become healthy within {}s. Last output:\n{last_output}", - timeout.as_secs() - )); - } - sleep(Duration::from_secs(2)).await; - } -} - -async fn sandbox_names() -> Result, String> { - let (output, code) = run_cli(&["sandbox", "list", "--names"]).await; - let clean = strip_ansi(&output); - if code != 0 { - return Err(format!("sandbox list failed (exit {code}):\n{clean}")); - } - - Ok(clean - .lines() - .map(str::trim) - .filter(|line| !line.is_empty()) - .map(ToOwned::to_owned) - .collect()) -} - -async fn wait_for_sandbox_exec_contains( - sandbox_name: &str, - command: &[&str], - expected: &str, - timeout: Duration, -) -> Result<(), String> { - let start = Instant::now(); - let mut last_output: String; - - loop { - let mut cmd = openshell_cmd(); - cmd.args(["sandbox", "exec", "--name", sandbox_name, "--no-tty", "--"]) - .args(command) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - - match cmd.output().await { - Ok(output) => { - let stdout = String::from_utf8_lossy(&output.stdout); - let stderr = String::from_utf8_lossy(&output.stderr); - last_output = strip_ansi(&format!("{stdout}{stderr}")); - if output.status.success() && last_output.contains(expected) { - return Ok(()); - } - } - Err(err) => { - last_output = format!("failed to spawn openshell sandbox exec: {err}"); - } - } - - if start.elapsed() > timeout { - return Err(format!( - "sandbox '{sandbox_name}' exec did not produce '{expected}' within {}s. Last output:\n{last_output}", - timeout.as_secs() - )); - } - sleep(Duration::from_secs(2)).await; - } -} - #[tokio::test] async fn vm_gateway_restart_resumes_running_sandbox() { if std::env::var("OPENSHELL_E2E_DRIVER").as_deref() != Ok("vm") { @@ -134,12 +37,9 @@ async fn vm_gateway_restart_resumes_running_sandbox() { let script = format!( "echo before-restart > {RESUME_FILE}; echo {READY_MARKER}; while true; do sleep 1; done" ); - let mut sandbox = SandboxGuard::create_keep( - &["sh", "-lc", &script], - READY_MARKER, - ) - .await - .expect("create long-running VM sandbox"); + let mut sandbox = SandboxGuard::create_keep(&["sh", "-lc", &script], READY_MARKER) + .await + .expect("create long-running VM sandbox"); let before_restart = sandbox .exec(&["cat", RESUME_FILE])