Skip to content

Commit b5f2631

Browse files
mishushakovclaude
andauthored
feat: add gzip content encoding option for file operations (#1252)
## Summary - Adds optional `gzip` parameter to sandbox file read/write operations across JS and Python SDKs - Uploads are gzip-compressed via `CompressionStream` (JS) / `gzip.compress` (Python) when enabled, downloads request `Accept-Encoding: gzip` - Only applies to the octet-stream upload path (envd >= 0.5.7), so older envd versions are unaffected - Includes tests for both SDKs covering write+read with gzip, write gzip + read plain, multi-file writes, and byte format reads ## Test plan - [ ] Run JS SDK content encoding tests (`contentEncoding.test.ts`) - [ ] Run Python async/sync content encoding tests (`test_content_encoding.py`) - [ ] Integration test with envd backend supporting `Content-Encoding: gzip` 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 58ecd78 commit b5f2631

10 files changed

Lines changed: 378 additions & 47 deletions

File tree

.changeset/gzip-file-upload.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
---
2+
'@e2b/python-sdk': patch
3+
'e2b': patch
4+
---
5+
6+
added gzip support for sandbox file upload/download

packages/js-sdk/src/index.ts

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,13 @@ export type { Logger } from './logs'
2525
export { getSignature } from './sandbox/signature'
2626

2727
export { FileType } from './sandbox/filesystem'
28-
export type { WriteInfo, EntryInfo, Filesystem } from './sandbox/filesystem'
28+
export type {
29+
WriteInfo,
30+
EntryInfo,
31+
Filesystem,
32+
FilesystemWriteOpts,
33+
FilesystemReadOpts,
34+
} from './sandbox/filesystem'
2935
export { FilesystemEventType } from './sandbox/filesystem/watchHandle'
3036
export type {
3137
FilesystemEvent,

packages/js-sdk/src/sandbox/filesystem/index.ts

Lines changed: 52 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ import {
3737
InvalidArgumentError,
3838
TemplateError,
3939
} from '../../errors'
40-
import { toBlob } from '../../utils'
40+
import { toBlob, toUploadBody } from '../../utils'
4141

4242
const FILESYSTEM_HTTP_ERROR_MAP: Record<number, (message: string) => Error> = {
4343
404: (message: string) => new FileNotFoundError(message),
@@ -164,6 +164,26 @@ export interface FilesystemRequestOpts
164164
user?: Username
165165
}
166166

167+
/**
168+
* Options for writing files to the sandbox filesystem.
169+
*/
170+
export interface FilesystemWriteOpts extends FilesystemRequestOpts {
171+
/**
172+
* When true, the upload will be gzip-compressed.
173+
*/
174+
gzip?: boolean
175+
}
176+
177+
/**
178+
* Options for reading files from the sandbox filesystem.
179+
*/
180+
export interface FilesystemReadOpts extends FilesystemRequestOpts {
181+
/**
182+
* When true, the download will request gzip-encoded responses.
183+
*/
184+
gzip?: boolean
185+
}
186+
167187
export interface FilesystemListOpts extends FilesystemRequestOpts {
168188
/**
169189
* Depth of the directory to list.
@@ -222,7 +242,7 @@ export class Filesystem {
222242
*/
223243
async read(
224244
path: string,
225-
opts?: FilesystemRequestOpts & { format?: 'text' }
245+
opts?: FilesystemReadOpts & { format?: 'text' }
226246
): Promise<string>
227247
/**
228248
* Read file content as a `Uint8Array`.
@@ -237,7 +257,7 @@ export class Filesystem {
237257
*/
238258
async read(
239259
path: string,
240-
opts?: FilesystemRequestOpts & { format: 'bytes' }
260+
opts?: FilesystemReadOpts & { format: 'bytes' }
241261
): Promise<Uint8Array>
242262
/**
243263
* Read file content as a `Blob`.
@@ -252,7 +272,7 @@ export class Filesystem {
252272
*/
253273
async read(
254274
path: string,
255-
opts?: FilesystemRequestOpts & { format: 'blob' }
275+
opts?: FilesystemReadOpts & { format: 'blob' }
256276
): Promise<Blob>
257277
/**
258278
* Read file content as a `ReadableStream`.
@@ -267,12 +287,12 @@ export class Filesystem {
267287
*/
268288
async read(
269289
path: string,
270-
opts?: FilesystemRequestOpts & { format: 'stream' }
290+
opts?: FilesystemReadOpts & { format: 'stream' }
271291
): Promise<ReadableStream<Uint8Array>>
272292
async read(
273293
path: string,
274-
opts?: FilesystemRequestOpts & {
275-
format?: 'text' | 'stream' | 'bytes' | 'blob'
294+
opts?: FilesystemReadOpts & {
295+
format?: 'text' | 'bytes' | 'blob' | 'stream'
276296
}
277297
): Promise<unknown> {
278298
const format = opts?.format ?? 'text'
@@ -285,6 +305,11 @@ export class Filesystem {
285305
user = defaultUsername
286306
}
287307

308+
const headers: Record<string, string> = {}
309+
if (opts?.gzip) {
310+
headers['Accept-Encoding'] = 'gzip'
311+
}
312+
288313
const res = await this.envdApi.api.GET('/files', {
289314
params: {
290315
query: {
@@ -294,6 +319,7 @@ export class Filesystem {
294319
},
295320
parseAs: format === 'bytes' ? 'arrayBuffer' : format,
296321
signal: this.connectionConfig.getSignal(opts?.requestTimeoutMs),
322+
headers,
297323
})
298324

299325
const err = await handleFilesystemEnvdApiError(res)
@@ -332,11 +358,11 @@ export class Filesystem {
332358
async write(
333359
path: string,
334360
data: string | ArrayBuffer | Blob | ReadableStream,
335-
opts?: FilesystemRequestOpts
361+
opts?: FilesystemWriteOpts
336362
): Promise<WriteInfo>
337363
async write(
338364
files: WriteEntry[],
339-
opts?: FilesystemRequestOpts
365+
opts?: FilesystemWriteOpts
340366
): Promise<WriteInfo[]>
341367
async write(
342368
pathOrFiles: string | WriteEntry[],
@@ -345,8 +371,8 @@ export class Filesystem {
345371
| ArrayBuffer
346372
| Blob
347373
| ReadableStream
348-
| FilesystemRequestOpts,
349-
opts?: FilesystemRequestOpts
374+
| FilesystemWriteOpts,
375+
opts?: FilesystemWriteOpts
350376
): Promise<WriteInfo | WriteInfo[]> {
351377
if (typeof pathOrFiles !== 'string' && !Array.isArray(pathOrFiles)) {
352378
throw new Error('Path or files are required')
@@ -362,7 +388,7 @@ export class Filesystem {
362388
typeof pathOrFiles === 'string'
363389
? {
364390
path: pathOrFiles,
365-
writeOpts: opts as FilesystemRequestOpts,
391+
writeOpts: opts as FilesystemWriteOpts,
366392
writeFiles: [
367393
{
368394
data: dataOrOpts as
@@ -375,7 +401,7 @@ export class Filesystem {
375401
}
376402
: {
377403
path: undefined,
378-
writeOpts: dataOrOpts as FilesystemRequestOpts,
404+
writeOpts: dataOrOpts as FilesystemWriteOpts,
379405
writeFiles: pathOrFiles as WriteEntry[],
380406
}
381407

@@ -394,11 +420,20 @@ export class Filesystem {
394420

395421
const results: WriteInfo[] = []
396422

423+
const useGzip = writeOpts?.gzip === true
424+
397425
if (useOctetStream) {
426+
const headers: Record<string, string> = {
427+
'Content-Type': 'application/octet-stream',
428+
}
429+
if (useGzip) {
430+
headers['Content-Encoding'] = 'gzip'
431+
}
432+
398433
const uploadResults = await Promise.all(
399434
writeFiles.map(async (file) => {
400435
const filePath = path ?? (file as WriteEntry).path
401-
const blob = await toBlob(file.data)
436+
const body = await toUploadBody(file.data, useGzip)
402437

403438
const res = await this.envdApi.api.POST('/files', {
404439
params: {
@@ -407,10 +442,8 @@ export class Filesystem {
407442
username: user,
408443
},
409444
},
410-
bodySerializer: () => blob,
411-
headers: {
412-
'Content-Type': 'application/octet-stream',
413-
},
445+
bodySerializer: () => body,
446+
headers,
414447
signal: this.connectionConfig.getSignal(
415448
writeOpts?.requestTimeoutMs
416449
),
@@ -491,7 +524,7 @@ export class Filesystem {
491524
*/
492525
async writeFiles(
493526
files: WriteEntry[],
494-
opts?: FilesystemRequestOpts
527+
opts?: FilesystemWriteOpts
495528
): Promise<WriteInfo[]> {
496529
return this.write(files, opts) as Promise<WriteInfo[]>
497530
}

packages/js-sdk/src/utils.ts

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,3 +124,25 @@ export function toBlob(
124124
// ReadableStream - must consume to get Blob
125125
return new Response(data).blob()
126126
}
127+
128+
/**
129+
* Prepare data for upload as a BodyInit, optionally gzip-compressed.
130+
* When gzip is enabled, compresses the data and returns a Blob.
131+
*/
132+
export async function toUploadBody(
133+
data: string | ArrayBuffer | Blob | ReadableStream,
134+
gzip?: boolean
135+
): Promise<BodyInit> {
136+
if (gzip) {
137+
const stream =
138+
data instanceof ReadableStream
139+
? data
140+
: data instanceof Blob
141+
? data.stream()
142+
: new Blob([data]).stream()
143+
const compressed = stream.pipeThrough(new CompressionStream('gzip'))
144+
return new Response(compressed).blob()
145+
}
146+
147+
return toBlob(data)
148+
}
Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
import { assert } from 'vitest'
2+
3+
import { WriteEntry } from '../../../src/sandbox/filesystem'
4+
import { isDebug, sandboxTest } from '../../setup.js'
5+
6+
sandboxTest(
7+
'write and read file with gzip content encoding',
8+
async ({ sandbox }) => {
9+
const filename = 'test_gzip_write.txt'
10+
const content = 'This is a test file with gzip encoding.'
11+
12+
const info = await sandbox.files.write(filename, content, {
13+
gzip: true,
14+
})
15+
assert.equal(info.name, filename)
16+
assert.equal(info.type, 'file')
17+
assert.equal(info.path, `/home/user/${filename}`)
18+
19+
const readContent = await sandbox.files.read(filename, {
20+
gzip: true,
21+
})
22+
assert.equal(readContent, content)
23+
24+
if (isDebug) {
25+
await sandbox.files.remove(filename)
26+
}
27+
}
28+
)
29+
30+
sandboxTest(
31+
'write with gzip and read without encoding',
32+
async ({ sandbox }) => {
33+
const filename = 'test_gzip_write_plain_read.txt'
34+
const content = 'Written with gzip, read without.'
35+
36+
await sandbox.files.write(filename, content, {
37+
gzip: true,
38+
})
39+
40+
const readContent = await sandbox.files.read(filename)
41+
assert.equal(readContent, content)
42+
43+
if (isDebug) {
44+
await sandbox.files.remove(filename)
45+
}
46+
}
47+
)
48+
49+
sandboxTest('writeFiles with gzip content encoding', async ({ sandbox }) => {
50+
const files: WriteEntry[] = [
51+
{ path: 'gzip_multi_1.txt', data: 'File 1 content' },
52+
{ path: 'gzip_multi_2.txt', data: 'File 2 content' },
53+
{ path: 'gzip_multi_3.txt', data: 'File 3 content' },
54+
]
55+
56+
const infos = await sandbox.files.writeFiles(files, {
57+
gzip: true,
58+
})
59+
60+
assert.equal(infos.length, files.length)
61+
62+
for (let i = 0; i < files.length; i++) {
63+
const readContent = await sandbox.files.read(files[i].path)
64+
assert.equal(readContent, files[i].data)
65+
}
66+
67+
if (isDebug) {
68+
for (const file of files) {
69+
await sandbox.files.remove(file.path)
70+
}
71+
}
72+
})
73+
74+
sandboxTest(
75+
'read file as bytes with gzip content encoding',
76+
async ({ sandbox }) => {
77+
const filename = 'test_gzip_bytes.txt'
78+
const content = 'Binary content with gzip.'
79+
80+
await sandbox.files.write(filename, content)
81+
82+
const readBytes = await sandbox.files.read(filename, {
83+
format: 'bytes',
84+
gzip: true,
85+
})
86+
assert.instanceOf(readBytes, Uint8Array)
87+
const decoded = new TextDecoder().decode(readBytes)
88+
assert.equal(decoded, content)
89+
90+
if (isDebug) {
91+
await sandbox.files.remove(filename)
92+
}
93+
}
94+
)

packages/python-sdk/e2b/sandbox/filesystem/filesystem.py

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,12 @@
1+
import gzip
12
from dataclasses import dataclass
23
from datetime import datetime
34
from enum import Enum
5+
from io import IOBase, TextIOBase
46
from typing import IO, Optional, Union, TypedDict
57

68
from e2b.envd.filesystem import filesystem_pb2
9+
from e2b.exceptions import InvalidArgumentException
710

811

912
class FileType(Enum):
@@ -92,3 +95,22 @@ class WriteEntry(TypedDict):
9295

9396
path: str
9497
data: Union[str, bytes, IO]
98+
99+
100+
def to_upload_body(
101+
data: Union[str, bytes, IO],
102+
use_gzip: bool = False,
103+
) -> bytes:
104+
"""Prepare file data for upload, optionally gzip-compressed."""
105+
if isinstance(data, str):
106+
raw = data.encode("utf-8")
107+
elif isinstance(data, bytes):
108+
raw = data
109+
elif isinstance(data, TextIOBase):
110+
raw = data.read().encode("utf-8")
111+
elif isinstance(data, IOBase):
112+
raw = data.read()
113+
else:
114+
raise InvalidArgumentException(f"Unsupported data type: {type(data)}")
115+
116+
return gzip.compress(raw) if use_gzip else raw

0 commit comments

Comments
 (0)