Skip to content

Commit b2208ff

Browse files
authored
feat(resume-build): add -shell flag for envd-backed PTY access (#2528)
Adds a -shell flag to resume-build that opens an interactive PTY session against envd inside the resumed sandbox, instead of relying on sshd in the template image. The host terminal goes into raw mode and is proxied through to /bin/bash -l; Ctrl+D exits and tears the sandbox down. TERM and locale vars are forwarded explicitly so curses apps (htop, tmux, vim) initialise correctly - envd only inherits PATH/HOME/USER/LOGNAME by default. -shell is interactive-only - combining it with -cmd, any -pause variant, or -iterations is rejected.
1 parent 26f0fbf commit b2208ff

3 files changed

Lines changed: 315 additions & 6 deletions

File tree

packages/orchestrator/cmd/resume-build/main.go

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@ func main() {
7171
cmdPause := flag.String("cmd-pause", "", "execute command in sandbox, then pause on success")
7272
cmdSignalPause := flag.String("cmd-signal-pause", "", "execute command in sandbox, then wait for SIGUSR1 before pausing")
7373
optimize := flag.Bool("optimize", false, "collect fresh prefetch mapping after pause (resumes snapshot to record page faults)")
74+
shell := flag.Bool("shell", false, "attach an interactive PTY shell via envd (no sshd required in the sandbox)")
7475

7576
flag.Parse()
7677

@@ -125,6 +126,10 @@ func main() {
125126
log.Fatal("-optimize is incompatible with -iterations (benchmarking doesn't upload)")
126127
}
127128

129+
if *shell && (isCmdMode || isPauseMode || *iterations > 0) {
130+
log.Fatal("-shell can only be used in interactive mode (no -cmd, no pause flags, no -iterations)")
131+
}
132+
128133
// Generate new build ID if not specified and pause mode is enabled
129134
outputBuildID := *toBuild
130135
if isPauseMode && outputBuildID == "" {
@@ -159,7 +164,7 @@ func main() {
159164
iterations: *iterations,
160165
}
161166

162-
err := run(ctx, *fromBuild, *iterations, *coldStart, *noPrefetch, *noEgress, *verbose, pauseOpts, runOpts)
167+
err := run(ctx, *fromBuild, *iterations, *coldStart, *noPrefetch, *noEgress, *verbose, *shell, pauseOpts, runOpts)
163168
cancel()
164169

165170
if err != nil {
@@ -274,6 +279,7 @@ type runner struct {
274279
cache *template.Cache
275280
coldStart bool
276281
noPrefetch bool
282+
shell bool
277283
config cfg.BuilderConfig
278284
storage storage.StorageProvider
279285
}
@@ -314,11 +320,23 @@ func (r *runner) interactive(ctx context.Context) error {
314320

315321
fmt.Printf("✅ Running (resumed in %s)\n", time.Since(t0))
316322
fmt.Printf(" sudo nsenter --net=/var/run/netns/%s ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@169.254.0.21\n", sbx.Slot.NamespaceID())
317-
fmt.Println("Ctrl+C to stop")
318323

324+
defer func() {
325+
fmt.Println("🧹 Cleanup...")
326+
sbx.Close(context.WithoutCancel(ctx))
327+
}()
328+
329+
if r.shell {
330+
err := attachShell(ctx, sbx)
331+
if err != nil && !isShellExited(err) {
332+
return err
333+
}
334+
335+
return nil
336+
}
337+
338+
fmt.Println("Ctrl+C to stop")
319339
<-ctx.Done()
320-
fmt.Println("🧹 Cleanup...")
321-
sbx.Close(context.WithoutCancel(ctx))
322340

323341
return nil
324342
}
@@ -959,7 +977,7 @@ func (r *runner) benchmark(ctx context.Context, n int) error {
959977
return lastErr
960978
}
961979

962-
func run(ctx context.Context, buildID string, iterations int, coldStart, noPrefetch, noEgress, verbose bool, pauseOpts pauseOptions, runOpts runOptions) error {
980+
func run(ctx context.Context, buildID string, iterations int, coldStart, noPrefetch, noEgress, verbose, shell bool, pauseOpts pauseOptions, runOpts runOptions) error {
963981
// Silence other loggers unless verbose mode
964982
var l logger.Logger
965983
if !verbose {
@@ -1122,6 +1140,7 @@ func run(ctx context.Context, buildID string, iterations int, coldStart, noPrefe
11221140
cache: cache,
11231141
coldStart: coldStart,
11241142
noPrefetch: noPrefetch,
1143+
shell: shell,
11251144
config: config.BuilderConfig,
11261145
storage: persistence,
11271146
sbxConfig: sbxCfg,
Lines changed: 290 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,290 @@
1+
package main
2+
3+
import (
4+
"context"
5+
"errors"
6+
"fmt"
7+
"net/http"
8+
"os"
9+
"os/signal"
10+
"sync"
11+
"syscall"
12+
"time"
13+
14+
"connectrpc.com/connect"
15+
"golang.org/x/term"
16+
17+
"github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox"
18+
"github.com/e2b-dev/infra/packages/shared/pkg/consts"
19+
"github.com/e2b-dev/infra/packages/shared/pkg/grpc"
20+
"github.com/e2b-dev/infra/packages/shared/pkg/grpc/envd/process"
21+
"github.com/e2b-dev/infra/packages/shared/pkg/grpc/envd/process/processconnect"
22+
)
23+
24+
// shellExitedError is returned when the in-guest shell exits cleanly
25+
// (e.g. user pressed Ctrl+D). Callers use it to distinguish a normal
26+
// session end from a real transport/setup error.
27+
type shellExitedError struct{ exitCode int32 }
28+
29+
func (e *shellExitedError) Error() string {
30+
return fmt.Sprintf("shell exited with code %d", e.exitCode)
31+
}
32+
33+
func isShellExited(err error) bool {
34+
var s *shellExitedError
35+
36+
return errors.As(err, &s)
37+
}
38+
39+
// shellEnv builds the environment passed into the in-guest PTY shell.
40+
// envd intentionally only inherits PATH/HOME/USER/LOGNAME plus its own
41+
// configured globals, so we must propagate TERM (and a few common locale
42+
// vars) explicitly — otherwise curses apps like htop, tmux, vim and less
43+
// fail to initialise.
44+
func shellEnv() map[string]string {
45+
envs := map[string]string{}
46+
47+
if t := os.Getenv("TERM"); t != "" {
48+
envs["TERM"] = t
49+
} else {
50+
envs["TERM"] = "xterm-256color"
51+
}
52+
53+
for _, k := range []string{"LANG", "LC_ALL", "LC_CTYPE", "COLORTERM"} {
54+
if v := os.Getenv(k); v != "" {
55+
envs[k] = v
56+
}
57+
}
58+
59+
return envs
60+
}
61+
62+
// attachShell opens an interactive PTY shell against envd inside sbx,
63+
// proxying the host terminal through. The shell is /bin/bash -l; if
64+
// bash is missing in the guest the wrapper's stderr ("nice: '/bin/bash':
65+
// No such file or directory") will surface in the user's terminal.
66+
//
67+
// Returns when the in-guest shell exits (Ctrl+D), or when ctx is cancelled.
68+
func attachShell(ctx context.Context, sbx *sandbox.Sandbox) error {
69+
if !term.IsTerminal(int(os.Stdin.Fd())) {
70+
return errors.New("-shell requires an interactive terminal on stdin")
71+
}
72+
73+
envdURL := fmt.Sprintf("http://%s:%d", sbx.Slot.HostIPString(), consts.DefaultEnvdServerPort)
74+
hc := http.Client{
75+
// No request timeout — interactive sessions can be long-lived.
76+
Transport: sandbox.SandboxHttpTransport,
77+
}
78+
processC := processconnect.NewProcessClient(&hc, envdURL)
79+
80+
cols, rows, err := term.GetSize(int(os.Stdout.Fd()))
81+
if err != nil || cols == 0 || rows == 0 {
82+
cols, rows = 80, 24
83+
}
84+
85+
startReq := connect.NewRequest(&process.StartRequest{
86+
Process: &process.ProcessConfig{
87+
Cmd: "/bin/bash",
88+
Args: []string{"-l"},
89+
Envs: shellEnv(),
90+
},
91+
Pty: &process.PTY{
92+
Size: &process.PTY_Size{Cols: uint32(cols), Rows: uint32(rows)},
93+
},
94+
})
95+
grpc.SetUserHeader(startReq.Header(), "root")
96+
if sbx.Config.Envd.AccessToken != nil {
97+
startReq.Header().Set("X-Access-Token", *sbx.Config.Envd.AccessToken)
98+
}
99+
100+
stream, err := processC.Start(ctx, startReq)
101+
if err != nil {
102+
return fmt.Errorf("start shell: %w", err)
103+
}
104+
closeStream := sync.OnceFunc(func() { _ = stream.Close() })
105+
defer closeStream()
106+
107+
// Wait for the StartEvent so we have a pid to address input/resize at.
108+
var pid uint32
109+
for stream.Receive() {
110+
event := stream.Msg().GetEvent().GetEvent()
111+
switch e := event.(type) {
112+
case *process.ProcessEvent_Start:
113+
pid = e.Start.GetPid()
114+
case *process.ProcessEvent_Data:
115+
// Push any data that arrived before we exited the bootstrap loop.
116+
if pty := e.Data.GetPty(); pty != nil {
117+
_, _ = os.Stdout.Write(pty)
118+
}
119+
case *process.ProcessEvent_End:
120+
return endToError(e.End)
121+
}
122+
if pid != 0 {
123+
break
124+
}
125+
}
126+
if pid == 0 {
127+
if err := stream.Err(); err != nil {
128+
return fmt.Errorf("stream closed before start: %w", err)
129+
}
130+
131+
return errors.New("no start event received")
132+
}
133+
134+
fmt.Println("📟 Attaching shell via envd (Ctrl+D to exit)")
135+
136+
oldState, err := term.MakeRaw(int(os.Stdin.Fd()))
137+
if err != nil {
138+
return fmt.Errorf("raw mode: %w", err)
139+
}
140+
141+
sessionCtx, cancel := context.WithCancel(ctx)
142+
defer cancel()
143+
144+
streamDone := make(chan struct{})
145+
endCh := make(chan *process.ProcessEvent_EndEvent, 1)
146+
go func() {
147+
defer close(streamDone)
148+
defer cancel()
149+
for stream.Receive() {
150+
event := stream.Msg().GetEvent().GetEvent()
151+
switch e := event.(type) {
152+
case *process.ProcessEvent_Data:
153+
if pty := e.Data.GetPty(); pty != nil {
154+
_, _ = os.Stdout.Write(pty)
155+
}
156+
case *process.ProcessEvent_End:
157+
endCh <- e.End
158+
159+
return
160+
}
161+
}
162+
}()
163+
164+
// Input pump: stdin → StreamInput as PTY bytes.
165+
go pumpInput(sessionCtx, processC, sbx, pid)
166+
167+
// Resize: forward SIGWINCH to envd via Update.
168+
go pumpResize(sessionCtx, processC, sbx, pid)
169+
170+
<-sessionCtx.Done()
171+
172+
// Drain the output goroutine before restoring the terminal — otherwise
173+
// late PTY bytes land on a cooked-mode terminal and render stairstepped.
174+
closeStream()
175+
<-streamDone
176+
177+
_ = term.Restore(int(os.Stdin.Fd()), oldState)
178+
fmt.Println()
179+
180+
select {
181+
case end := <-endCh:
182+
return endToError(end)
183+
default:
184+
}
185+
186+
if err := stream.Err(); err != nil && !errors.Is(err, context.Canceled) {
187+
return fmt.Errorf("shell stream: %w", err)
188+
}
189+
190+
return nil
191+
}
192+
193+
func pumpInput(
194+
ctx context.Context,
195+
processC processconnect.ProcessClient,
196+
sbx *sandbox.Sandbox,
197+
pid uint32,
198+
) {
199+
in := processC.StreamInput(ctx)
200+
grpc.SetUserHeader(in.RequestHeader(), "root")
201+
if sbx.Config.Envd.AccessToken != nil {
202+
in.RequestHeader().Set("X-Access-Token", *sbx.Config.Envd.AccessToken)
203+
}
204+
205+
if err := in.Send(&process.StreamInputRequest{
206+
Event: &process.StreamInputRequest_Start{
207+
Start: &process.StreamInputRequest_StartEvent{
208+
Process: &process.ProcessSelector{
209+
Selector: &process.ProcessSelector_Pid{Pid: pid},
210+
},
211+
},
212+
},
213+
}); err != nil {
214+
return
215+
}
216+
217+
buf := make([]byte, 4096)
218+
for {
219+
// In raw mode, Read blocks until a byte arrives. We can't easily
220+
// interrupt it on ctx.Done, but the parent process will exit soon
221+
// after the stream closes, which is acceptable for a CLI.
222+
n, err := os.Stdin.Read(buf)
223+
if n > 0 {
224+
data := append([]byte(nil), buf[:n]...)
225+
if sendErr := in.Send(&process.StreamInputRequest{
226+
Event: &process.StreamInputRequest_Data{
227+
Data: &process.StreamInputRequest_DataEvent{
228+
Input: &process.ProcessInput{
229+
Input: &process.ProcessInput_Pty{Pty: data},
230+
},
231+
},
232+
},
233+
}); sendErr != nil {
234+
return
235+
}
236+
}
237+
if err != nil {
238+
return
239+
}
240+
if ctx.Err() != nil {
241+
return
242+
}
243+
}
244+
}
245+
246+
func pumpResize(
247+
ctx context.Context,
248+
processC processconnect.ProcessClient,
249+
sbx *sandbox.Sandbox,
250+
pid uint32,
251+
) {
252+
winch := make(chan os.Signal, 1)
253+
signal.Notify(winch, syscall.SIGWINCH)
254+
defer signal.Stop(winch)
255+
256+
for {
257+
select {
258+
case <-ctx.Done():
259+
return
260+
case <-winch:
261+
cols, rows, err := term.GetSize(int(os.Stdout.Fd()))
262+
if err != nil || cols == 0 || rows == 0 {
263+
continue
264+
}
265+
req := connect.NewRequest(&process.UpdateRequest{
266+
Process: &process.ProcessSelector{
267+
Selector: &process.ProcessSelector_Pid{Pid: pid},
268+
},
269+
Pty: &process.PTY{
270+
Size: &process.PTY_Size{Cols: uint32(cols), Rows: uint32(rows)},
271+
},
272+
})
273+
grpc.SetUserHeader(req.Header(), "root")
274+
if sbx.Config.Envd.AccessToken != nil {
275+
req.Header().Set("X-Access-Token", *sbx.Config.Envd.AccessToken)
276+
}
277+
updateCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
278+
_, _ = processC.Update(updateCtx, req)
279+
cancel()
280+
}
281+
}
282+
}
283+
284+
func endToError(end *process.ProcessEvent_EndEvent) error {
285+
if end == nil {
286+
return nil
287+
}
288+
289+
return &shellExitedError{exitCode: end.GetExitCode()}
290+
}

packages/orchestrator/go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ require (
7272
go.uber.org/zap v1.27.1
7373
golang.org/x/sync v0.20.0
7474
golang.org/x/sys v0.43.0
75+
golang.org/x/term v0.42.0
7576
google.golang.org/api v0.267.0
7677
google.golang.org/grpc v1.80.0
7778
google.golang.org/protobuf v1.36.11
@@ -319,7 +320,6 @@ require (
319320
golang.org/x/mod v0.35.0 // indirect
320321
golang.org/x/net v0.53.0 // indirect
321322
golang.org/x/oauth2 v0.36.0 // indirect
322-
golang.org/x/term v0.42.0 // indirect
323323
golang.org/x/text v0.36.0 // indirect
324324
golang.org/x/time v0.14.0 // indirect
325325
golang.org/x/tools v0.44.0 // indirect

0 commit comments

Comments
 (0)