Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion e2e/rust/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,11 @@ name = "gateway_resume"
path = "tests/gateway_resume.rs"
required-features = ["e2e-docker"]

[[test]]
name = "podman_gateway_resume"
path = "tests/podman_gateway_resume.rs"
required-features = ["e2e-podman"]

[[test]]
name = "vm_gateway_resume"
path = "tests/vm_gateway_resume.rs"
Expand All @@ -54,7 +59,7 @@ required-features = ["e2e-vm"]
[[test]]
name = "websocket_conformance"
path = "tests/websocket_conformance.rs"
required-features = ["e2e-docker"]
required-features = ["e2e-host-gateway"]

[[test]]
name = "user_namespaces"
Expand Down
2 changes: 1 addition & 1 deletion e2e/rust/e2e-podman.sh
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ set -euo pipefail

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
E2E_TEST="${OPENSHELL_E2E_PODMAN_TEST:-}"
E2E_FEATURES="${OPENSHELL_E2E_PODMAN_FEATURES:-e2e}"
E2E_FEATURES="${OPENSHELL_E2E_PODMAN_FEATURES:-e2e-podman}"

cargo build -p openshell-cli --features openshell-core/dev-settings

Expand Down
107 changes: 107 additions & 0 deletions e2e/rust/src/harness/cli.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

//! Shared CLI helpers for e2e tests that need to invoke `openshell` commands
//! and poll for readiness.

use std::process::Stdio;
use std::time::{Duration, Instant};

use tokio::time::sleep;

use super::binary::openshell_cmd;
use super::output::strip_ansi;

pub async fn run_cli(args: &[&str]) -> (String, i32) {
let mut cmd = openshell_cmd();
cmd.args(args).stdout(Stdio::piped()).stderr(Stdio::piped());

let output = cmd.output().await.expect("spawn openshell");
let stdout = String::from_utf8_lossy(&output.stdout);
let stderr = String::from_utf8_lossy(&output.stderr);
let combined = format!("{stdout}{stderr}");
let code = output.status.code().unwrap_or(-1);
(combined, code)
}

pub async fn wait_for_healthy(timeout: Duration) -> Result<(), String> {
let start = Instant::now();
let mut last_output: String;

loop {
let (output, code) = run_cli(&["status"]).await;
let clean = strip_ansi(&output);
let lower = clean.to_lowercase();
if code == 0
&& (lower.contains("healthy")
|| lower.contains("running")
|| lower.contains("connected"))
{
return Ok(());
}
last_output = clean;

if start.elapsed() > timeout {
return Err(format!(
"gateway did not become healthy within {}s. Last output:\n{last_output}",
timeout.as_secs()
));
}
sleep(Duration::from_secs(2)).await;
}
}

pub async fn sandbox_names() -> Result<Vec<String>, String> {
let (output, code) = run_cli(&["sandbox", "list", "--names"]).await;
let clean = strip_ansi(&output);
if code != 0 {
return Err(format!("sandbox list failed (exit {code}):\n{clean}"));
}

Ok(clean
.lines()
.map(str::trim)
.filter(|line| !line.is_empty())
.map(ToOwned::to_owned)
.collect())
}

pub async fn wait_for_sandbox_exec_contains(
sandbox_name: &str,
command: &[&str],
expected: &str,
timeout: Duration,
) -> Result<(), String> {
let start = Instant::now();
let mut last_output: String;

loop {
let mut cmd = openshell_cmd();
cmd.args(["sandbox", "exec", "--name", sandbox_name, "--no-tty", "--"])
.args(command)
.stdout(Stdio::piped())
.stderr(Stdio::piped());

match cmd.output().await {
Ok(output) => {
let stdout = String::from_utf8_lossy(&output.stdout);
let stderr = String::from_utf8_lossy(&output.stderr);
last_output = strip_ansi(&format!("{stdout}{stderr}"));
if output.status.success() && last_output.contains(expected) {
return Ok(());
}
}
Err(err) => {
last_output = format!("failed to spawn openshell sandbox exec: {err}");
}
}

if start.elapsed() > timeout {
return Err(format!(
"sandbox '{sandbox_name}' exec did not produce '{expected}' within {}s. Last output:\n{last_output}",
timeout.as_secs()
));
}
sleep(Duration::from_secs(2)).await;
}
}
1 change: 1 addition & 0 deletions e2e/rust/src/harness/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
//! Shared test harness modules for CLI e2e tests.

pub mod binary;
pub mod cli;
pub mod container;
pub mod gateway;
pub mod output;
Expand Down
112 changes: 8 additions & 104 deletions e2e/rust/tests/gateway_resume.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,12 @@
//! gateway process, so they skip this restart-only coverage.

use std::process::{Command, Stdio};
use std::time::{Duration, Instant};
use std::time::Duration;

use openshell_e2e::harness::binary::openshell_cmd;
use openshell_e2e::harness::cli::{
sandbox_names, wait_for_healthy, wait_for_sandbox_exec_contains,
};
use openshell_e2e::harness::gateway::ManagedGateway;
use openshell_e2e::harness::output::strip_ansi;
use openshell_e2e::harness::sandbox::SandboxGuard;
use tokio::time::sleep;

Expand All @@ -24,100 +25,6 @@ const RESUME_FILE: &str = "/sandbox/gateway-resume-state";
const SANDBOX_NAMESPACE_LABEL: &str = "openshell.ai/sandbox-namespace";
const SANDBOX_NAME_LABEL: &str = "openshell.ai/sandbox-name";

async fn run_cli(args: &[&str]) -> (String, i32) {
let mut cmd = openshell_cmd();
cmd.args(args).stdout(Stdio::piped()).stderr(Stdio::piped());

let output = cmd.output().await.expect("spawn openshell");
let stdout = String::from_utf8_lossy(&output.stdout);
let stderr = String::from_utf8_lossy(&output.stderr);
let combined = format!("{stdout}{stderr}");
let code = output.status.code().unwrap_or(-1);
(combined, code)
}

async fn wait_for_healthy(timeout: Duration) -> Result<(), String> {
let start = Instant::now();
let mut last_output: String;

loop {
let (output, code) = run_cli(&["status"]).await;
let clean = strip_ansi(&output);
let lower = clean.to_lowercase();
if code == 0
&& (lower.contains("healthy")
|| lower.contains("running")
|| lower.contains("connected"))
{
return Ok(());
}
last_output = clean;

if start.elapsed() > timeout {
return Err(format!(
"gateway did not become healthy within {}s. Last output:\n{last_output}",
timeout.as_secs()
));
}
sleep(Duration::from_secs(2)).await;
}
}

async fn sandbox_names() -> Result<Vec<String>, String> {
let (output, code) = run_cli(&["sandbox", "list", "--names"]).await;
let clean = strip_ansi(&output);
if code != 0 {
return Err(format!("sandbox list failed (exit {code}):\n{clean}"));
}

Ok(clean
.lines()
.map(str::trim)
.filter(|line| !line.is_empty())
.map(ToOwned::to_owned)
.collect())
}

async fn wait_for_sandbox_exec_contains(
sandbox_name: &str,
command: &[&str],
expected: &str,
timeout: Duration,
) -> Result<(), String> {
let start = Instant::now();
let mut last_output: String;

loop {
let mut cmd = openshell_cmd();
cmd.args(["sandbox", "exec", "--name", sandbox_name, "--no-tty", "--"])
.args(command)
.stdout(Stdio::piped())
.stderr(Stdio::piped());

match cmd.output().await {
Ok(output) => {
let stdout = String::from_utf8_lossy(&output.stdout);
let stderr = String::from_utf8_lossy(&output.stderr);
last_output = strip_ansi(&format!("{stdout}{stderr}"));
if output.status.success() && last_output.contains(expected) {
return Ok(());
}
}
Err(err) => {
last_output = format!("failed to spawn openshell sandbox exec: {err}");
}
}

if start.elapsed() > timeout {
return Err(format!(
"sandbox '{sandbox_name}' exec did not produce '{expected}' within {}s. Last output:\n{last_output}",
timeout.as_secs()
));
}
sleep(Duration::from_secs(2)).await;
}
}

fn sandbox_container_id(namespace: &str, sandbox_name: &str) -> Result<String, String> {
let namespace_filter = format!("label={SANDBOX_NAMESPACE_LABEL}={namespace}");
let sandbox_name_filter = format!("label={SANDBOX_NAME_LABEL}={sandbox_name}");
Expand Down Expand Up @@ -189,7 +96,7 @@ async fn wait_for_container_running(
expected: bool,
timeout: Duration,
) -> Result<(), String> {
let start = Instant::now();
let start = std::time::Instant::now();
let mut last_state: String;

loop {
Expand Down Expand Up @@ -231,12 +138,9 @@ async fn docker_gateway_restart_resumes_running_sandbox() {
let script = format!(
"echo before-restart > {RESUME_FILE}; echo {READY_MARKER}; while true; do sleep 1; done"
);
let mut sandbox = SandboxGuard::create_keep(
&["sh", "-lc", &script],
READY_MARKER,
)
.await
.expect("create long-running sandbox");
let mut sandbox = SandboxGuard::create_keep(&["sh", "-lc", &script], READY_MARKER)
.await
.expect("create long-running sandbox");

let before_restart = sandbox
.exec(&["cat", RESUME_FILE])
Expand Down
81 changes: 81 additions & 0 deletions e2e/rust/tests/podman_gateway_resume.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

#![cfg(feature = "e2e-podman")]

//! Podman-specific E2E coverage for resuming sandboxes after a standalone
//! gateway restart.
//!
//! Unlike the Docker driver, Podman does not stop sandbox containers when the
//! gateway process exits — the containers keep running and the restarted
//! gateway re-adopts them. This test follows the `vm_gateway_resume.rs`
Comment thread
TaylorMutch marked this conversation as resolved.
//! pattern: verify sandbox survival at the application level without asserting
//! intermediate container-state transitions.

use std::time::Duration;

use openshell_e2e::harness::cli::{sandbox_names, wait_for_healthy, wait_for_sandbox_exec_contains};
use openshell_e2e::harness::gateway::ManagedGateway;
use openshell_e2e::harness::sandbox::SandboxGuard;

const READY_MARKER: &str = "podman-gateway-resume-ready";
const RESUME_FILE: &str = "/sandbox/podman-gateway-resume-state";

#[tokio::test]
async fn podman_gateway_restart_resumes_running_sandbox() {
if std::env::var("OPENSHELL_E2E_DRIVER").as_deref() != Ok("podman") {
eprintln!("Skipping Podman gateway resume test: e2e driver is not podman");
return;
}
let Some(gateway) = ManagedGateway::from_env().expect("load managed e2e gateway metadata")
else {
eprintln!(
"Skipping Podman gateway resume test: e2e gateway is not managed by this test run"
);
return;
};

wait_for_healthy(Duration::from_secs(30))
.await
.expect("gateway should start healthy");

let script = format!(
"echo before-restart > {RESUME_FILE}; echo {READY_MARKER}; while true; do sleep 1; done"
);
let mut sandbox = SandboxGuard::create_keep(&["sh", "-lc", &script], READY_MARKER)
.await
.expect("create long-running Podman sandbox");

let before_restart = sandbox
.exec(&["cat", RESUME_FILE])
.await
.expect("read Podman sandbox state before restart");
assert!(
before_restart.contains("before-restart"),
"sandbox state was not written before restart:\n{before_restart}"
);

gateway.stop().expect("stop e2e gateway");
gateway.start().expect("restart e2e gateway");
wait_for_healthy(Duration::from_secs(120))
.await
.expect("gateway should become healthy after restart");

let names = sandbox_names().await.expect("list sandboxes after restart");
assert!(
names.contains(&sandbox.name),
"sandbox '{}' should still be listed after gateway restart. Names: {names:?}",
sandbox.name
);

wait_for_sandbox_exec_contains(
&sandbox.name,
&["cat", RESUME_FILE],
"before-restart",
Duration::from_secs(240),
)
.await
.expect("Podman sandbox should become ready again with its state preserved");

sandbox.cleanup().await;
}
Loading
Loading