Skip to content

Commit e4d6f92

Browse files
drewdrew
authored
feat(vm): add standalone libkrun compute driver (#858)
1 parent 4e8dbcf commit e4d6f92

27 files changed

Lines changed: 4921 additions & 28 deletions

File tree

‎.github/workflows/release-vm-dev.yml‎

Lines changed: 279 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -416,12 +416,253 @@ jobs:
416416
path: artifacts/*.tar.gz
417417
retention-days: 5
418418

419+
# ---------------------------------------------------------------------------
420+
# Build openshell-driver-vm binary (Linux — native on each arch)
421+
# ---------------------------------------------------------------------------
422+
build-driver-vm-linux:
423+
name: Build Driver VM (Linux ${{ matrix.arch }})
424+
needs: [compute-versions, download-kernel-runtime, build-rootfs]
425+
strategy:
426+
matrix:
427+
include:
428+
- arch: arm64
429+
runner: build-arm64
430+
target: aarch64-unknown-linux-gnu
431+
platform: linux-aarch64
432+
guest_arch: aarch64
433+
- arch: amd64
434+
runner: build-amd64
435+
target: x86_64-unknown-linux-gnu
436+
platform: linux-x86_64
437+
guest_arch: x86_64
438+
runs-on: ${{ matrix.runner }}
439+
timeout-minutes: 30
440+
container:
441+
image: ghcr.io/nvidia/openshell/ci:latest
442+
credentials:
443+
username: ${{ github.actor }}
444+
password: ${{ secrets.GITHUB_TOKEN }}
445+
options: --privileged
446+
env:
447+
MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
448+
SCCACHE_MEMCACHED_ENDPOINT: ${{ vars.SCCACHE_MEMCACHED_ENDPOINT }}
449+
OPENSHELL_IMAGE_TAG: dev
450+
steps:
451+
- uses: actions/checkout@v4
452+
with:
453+
fetch-depth: 0
454+
455+
- name: Mark workspace safe for git
456+
run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
457+
458+
- name: Fetch tags
459+
run: git fetch --tags --force
460+
461+
- name: Install tools
462+
run: mise install
463+
464+
- name: Cache Rust target and registry
465+
uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
466+
with:
467+
shared-key: driver-vm-linux-${{ matrix.arch }}
468+
cache-directories: .cache/sccache
469+
cache-targets: "true"
470+
471+
- name: Install zstd
472+
run: apt-get update && apt-get install -y --no-install-recommends zstd && rm -rf /var/lib/apt/lists/*
473+
474+
- name: Download kernel runtime tarball
475+
uses: actions/download-artifact@v4
476+
with:
477+
name: kernel-runtime-tarballs
478+
path: runtime-download/
479+
480+
- name: Download rootfs tarball
481+
uses: actions/download-artifact@v4
482+
with:
483+
name: rootfs-${{ matrix.arch }}
484+
path: rootfs-download/
485+
486+
- name: Stage compressed runtime for embedding
487+
run: |
488+
set -euo pipefail
489+
COMPRESSED_DIR="${PWD}/target/vm-runtime-compressed"
490+
mkdir -p "$COMPRESSED_DIR"
491+
492+
# Extract kernel runtime tarball and re-compress individual files
493+
EXTRACT_DIR=$(mktemp -d)
494+
zstd -d "runtime-download/vm-runtime-${{ matrix.platform }}.tar.zst" --stdout \
495+
| tar -xf - -C "$EXTRACT_DIR"
496+
497+
echo "Extracted runtime files:"
498+
ls -lah "$EXTRACT_DIR"
499+
500+
for file in "$EXTRACT_DIR"/*; do
501+
[ -f "$file" ] || continue
502+
name=$(basename "$file")
503+
[ "$name" = "provenance.json" ] && continue
504+
zstd -19 -f -q -T0 -o "${COMPRESSED_DIR}/${name}.zst" "$file"
505+
done
506+
507+
# Copy rootfs tarball (already zstd-compressed)
508+
cp rootfs-download/rootfs.tar.zst "${COMPRESSED_DIR}/rootfs.tar.zst"
509+
510+
echo "Staged compressed artifacts:"
511+
ls -lah "$COMPRESSED_DIR"
512+
513+
- name: Scope workspace to driver-vm crates
514+
run: |
515+
set -euo pipefail
516+
sed -i 's|members = \["crates/\*"\]|members = ["crates/openshell-driver-vm", "crates/openshell-core"]|' Cargo.toml
517+
518+
- name: Patch workspace version
519+
if: needs.compute-versions.outputs.cargo_version != ''
520+
run: |
521+
set -euo pipefail
522+
sed -i -E '/^\[workspace\.package\]/,/^\[/{s/^version[[:space:]]*=[[:space:]]*".*"/version = "'"${{ needs.compute-versions.outputs.cargo_version }}"'"/}' Cargo.toml
523+
524+
- name: Build openshell-driver-vm
525+
run: |
526+
set -euo pipefail
527+
OPENSHELL_VM_RUNTIME_COMPRESSED_DIR="${PWD}/target/vm-runtime-compressed" \
528+
mise x -- cargo build --release -p openshell-driver-vm
529+
530+
- name: sccache stats
531+
if: always()
532+
run: mise x -- sccache --show-stats
533+
534+
- name: Package binary
535+
run: |
536+
set -euo pipefail
537+
mkdir -p artifacts
538+
tar -czf "artifacts/openshell-driver-vm-${{ matrix.target }}.tar.gz" \
539+
-C target/release openshell-driver-vm
540+
ls -lh artifacts/
541+
542+
- name: Upload artifact
543+
uses: actions/upload-artifact@v4
544+
with:
545+
name: driver-vm-linux-${{ matrix.arch }}
546+
path: artifacts/*.tar.gz
547+
retention-days: 5
548+
549+
# ---------------------------------------------------------------------------
550+
# Build openshell-driver-vm binary (macOS ARM64 via osxcross)
551+
# ---------------------------------------------------------------------------
552+
build-driver-vm-macos:
553+
name: Build Driver VM (macOS)
554+
needs: [compute-versions, download-kernel-runtime, build-rootfs]
555+
runs-on: build-amd64
556+
timeout-minutes: 60
557+
container:
558+
image: ghcr.io/nvidia/openshell/ci:latest
559+
credentials:
560+
username: ${{ github.actor }}
561+
password: ${{ secrets.GITHUB_TOKEN }}
562+
options: --privileged
563+
volumes:
564+
- /var/run/docker.sock:/var/run/docker.sock
565+
env:
566+
MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
567+
SCCACHE_MEMCACHED_ENDPOINT: ${{ vars.SCCACHE_MEMCACHED_ENDPOINT }}
568+
steps:
569+
- uses: actions/checkout@v4
570+
with:
571+
fetch-depth: 0
572+
573+
- name: Mark workspace safe for git
574+
run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
575+
576+
- name: Fetch tags
577+
run: git fetch --tags --force
578+
579+
- name: Log in to GHCR
580+
run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin
581+
582+
- name: Set up Docker Buildx
583+
uses: ./.github/actions/setup-buildx
584+
585+
- name: Install zstd
586+
run: apt-get update && apt-get install -y --no-install-recommends zstd && rm -rf /var/lib/apt/lists/*
587+
588+
- name: Download kernel runtime tarball
589+
uses: actions/download-artifact@v4
590+
with:
591+
name: kernel-runtime-tarballs
592+
path: runtime-download/
593+
594+
- name: Download rootfs tarball (arm64)
595+
uses: actions/download-artifact@v4
596+
with:
597+
name: rootfs-arm64
598+
path: rootfs-download/
599+
600+
- name: Prepare compressed runtime directory
601+
run: |
602+
set -euo pipefail
603+
COMPRESSED_DIR="${PWD}/target/vm-runtime-compressed-macos"
604+
mkdir -p "$COMPRESSED_DIR"
605+
606+
# Extract the darwin runtime tarball and re-compress for embedding.
607+
# The macOS embedded.rs expects: libkrun.dylib.zst, libkrunfw.5.dylib.zst, gvproxy.zst
608+
EXTRACT_DIR=$(mktemp -d)
609+
zstd -d "runtime-download/vm-runtime-darwin-aarch64.tar.zst" --stdout \
610+
| tar -xf - -C "$EXTRACT_DIR"
611+
612+
echo "Extracted darwin runtime files:"
613+
ls -lah "$EXTRACT_DIR"
614+
615+
for file in "$EXTRACT_DIR"/*; do
616+
[ -f "$file" ] || continue
617+
name=$(basename "$file")
618+
[ "$name" = "provenance.json" ] && continue
619+
zstd -19 -f -q -T0 -o "${COMPRESSED_DIR}/${name}.zst" "$file"
620+
done
621+
622+
# The macOS VM guest is always Linux ARM64, so use the arm64 rootfs
623+
cp rootfs-download/rootfs.tar.zst "${COMPRESSED_DIR}/rootfs.tar.zst"
624+
625+
echo "Staged macOS compressed artifacts:"
626+
ls -lah "$COMPRESSED_DIR"
627+
628+
- name: Build macOS binary via Docker (osxcross)
629+
run: |
630+
set -euo pipefail
631+
docker buildx build \
632+
--file deploy/docker/Dockerfile.driver-vm-macos \
633+
--build-arg OPENSHELL_CARGO_VERSION="${{ needs.compute-versions.outputs.cargo_version }}" \
634+
--build-arg OPENSHELL_IMAGE_TAG=dev \
635+
--build-arg CARGO_TARGET_CACHE_SCOPE="${{ github.sha }}" \
636+
--build-context vm-runtime-compressed="${PWD}/target/vm-runtime-compressed-macos" \
637+
--target binary \
638+
--output type=local,dest=out/ \
639+
.
640+
641+
- name: Package binary
642+
run: |
643+
set -euo pipefail
644+
mkdir -p artifacts
645+
tar -czf artifacts/openshell-driver-vm-aarch64-apple-darwin.tar.gz \
646+
-C out openshell-driver-vm
647+
ls -lh artifacts/
648+
649+
- name: Upload artifact
650+
uses: actions/upload-artifact@v4
651+
with:
652+
name: driver-vm-macos
653+
path: artifacts/*.tar.gz
654+
retention-days: 5
655+
419656
# ---------------------------------------------------------------------------
420657
# Upload all VM binaries to the vm-dev rolling release
421658
# ---------------------------------------------------------------------------
422659
release-vm-dev:
423660
name: Release VM Dev
424-
needs: [build-vm-linux, build-vm-macos]
661+
needs:
662+
- build-vm-linux
663+
- build-vm-macos
664+
- build-driver-vm-linux
665+
- build-driver-vm-macos
425666
runs-on: build-amd64
426667
timeout-minutes: 10
427668
steps:
@@ -430,29 +671,38 @@ jobs:
430671
- name: Download all VM binary artifacts
431672
uses: actions/download-artifact@v4
432673
with:
433-
pattern: vm-*
674+
pattern: "{vm-*,driver-vm-*}"
434675
path: release/
435676
merge-multiple: true
436677

437678
- name: Filter to only binary tarballs
438679
run: |
439680
set -euo pipefail
440681
mkdir -p release-final
441-
# Only include the openshell-vm binary tarballs, not kernel runtime
442-
cp release/openshell-vm-*.tar.gz release-final/
443-
count=$(ls release-final/openshell-vm-*.tar.gz 2>/dev/null | wc -l)
444-
if [ "$count" -eq 0 ]; then
445-
echo "ERROR: No VM binary tarballs found in release/" >&2
682+
# Include both openshell-vm and openshell-driver-vm binary tarballs.
683+
# Exclude kernel runtime tarballs (they come from release-vm-kernel.yml).
684+
for pattern in 'openshell-vm-*.tar.gz' 'openshell-driver-vm-*.tar.gz'; do
685+
for file in release/${pattern}; do
686+
[ -f "$file" ] || continue
687+
cp "$file" release-final/
688+
done
689+
done
690+
vm_count=$(ls release-final/openshell-vm-*.tar.gz 2>/dev/null | wc -l)
691+
driver_count=$(ls release-final/openshell-driver-vm-*.tar.gz 2>/dev/null | wc -l)
692+
if [ "$vm_count" -eq 0 ] || [ "$driver_count" -eq 0 ]; then
693+
echo "ERROR: Missing binary tarballs (openshell-vm=${vm_count}, openshell-driver-vm=${driver_count})" >&2
694+
ls -la release/ || true
446695
exit 1
447696
fi
448-
echo "Release artifacts (${count} binaries):"
697+
echo "Release artifacts (openshell-vm=${vm_count}, openshell-driver-vm=${driver_count}):"
449698
ls -lh release-final/
450699
451700
- name: Generate checksums
452701
run: |
453702
set -euo pipefail
454703
cd release-final
455-
sha256sum openshell-vm-*.tar.gz > vm-binary-checksums-sha256.txt
704+
sha256sum openshell-vm-*.tar.gz openshell-driver-vm-*.tar.gz \
705+
> vm-binary-checksums-sha256.txt
456706
cat vm-binary-checksums-sha256.txt
457707
458708
- name: Ensure vm-dev tag exists
@@ -479,7 +729,11 @@ jobs:
479729
}
480730
// Delete old VM binary assets (keep kernel runtime assets)
481731
for (const asset of release.data.assets) {
482-
if (asset.name.startsWith('openshell-vm-') || asset.name === 'vm-binary-checksums-sha256.txt') {
732+
if (
733+
asset.name.startsWith('openshell-vm-') ||
734+
asset.name.startsWith('openshell-driver-vm-') ||
735+
asset.name === 'vm-binary-checksums-sha256.txt'
736+
) {
483737
core.info(`Deleting stale asset: ${asset.name}`);
484738
await github.rest.repos.deleteReleaseAsset({ owner, repo, asset_id: asset.id });
485739
}
@@ -520,6 +774,18 @@ jobs:
520774
| Linux x86_64 | `openshell-vm-x86_64-unknown-linux-gnu.tar.gz` |
521775
| macOS ARM64 | `openshell-vm-aarch64-apple-darwin.tar.gz` |
522776
777+
### VM Compute Driver Binaries
778+
779+
`openshell-driver-vm` binaries with embedded kernel runtime and sandbox rootfs.
780+
Launched by the gateway when `--drivers=vm` is configured. Rebuilt on every
781+
push to main alongside the openshell-vm binaries.
782+
783+
| Platform | Artifact |
784+
|----------|----------|
785+
| Linux ARM64 | `openshell-driver-vm-aarch64-unknown-linux-gnu.tar.gz` |
786+
| Linux x86_64 | `openshell-driver-vm-x86_64-unknown-linux-gnu.tar.gz` |
787+
| macOS ARM64 | `openshell-driver-vm-aarch64-apple-darwin.tar.gz` |
788+
523789
### Quick install
524790
525791
```
@@ -532,4 +798,7 @@ jobs:
532798
release-final/openshell-vm-aarch64-unknown-linux-gnu.tar.gz
533799
release-final/openshell-vm-x86_64-unknown-linux-gnu.tar.gz
534800
release-final/openshell-vm-aarch64-apple-darwin.tar.gz
801+
release-final/openshell-driver-vm-aarch64-unknown-linux-gnu.tar.gz
802+
release-final/openshell-driver-vm-x86_64-unknown-linux-gnu.tar.gz
803+
release-final/openshell-driver-vm-aarch64-apple-darwin.tar.gz
535804
release-final/vm-binary-checksums-sha256.txt

‎AGENTS.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@ These pipelines connect skills into end-to-end workflows. Individual skill files
4040
| `crates/openshell-providers/` | Provider management | Credential provider backends |
4141
| `crates/openshell-tui/` | Terminal UI | Ratatui-based dashboard for monitoring |
4242
| `crates/openshell-vm/` | MicroVM runtime | Experimental, work-in-progress libkrun-based VM execution |
43+
| `crates/openshell-driver-kubernetes/` | Kubernetes compute driver | In-process `ComputeDriver` backend for K8s sandbox pods |
44+
| `crates/openshell-driver-vm/` | VM compute driver | Standalone libkrun-backed `ComputeDriver` subprocess (embeds its own rootfs + runtime) |
4345
| `python/openshell/` | Python SDK | Python bindings and CLI packaging |
4446
| `proto/` | Protobuf definitions | gRPC service contracts |
4547
| `deploy/` | Docker, Helm, K8s | Dockerfiles, Helm chart, manifests |

‎Cargo.lock‎

Lines changed: 23 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)