Merge 1323a27 into b97fd4d

Author: mishushakov

.changeset/fix-js-sdk-upload-content-length.md

@@ -0,0 +1,5 @@
+---
+"e2b": patch
+---
+
+fix(js-sdk): buffer template tar archive before upload so `fetch` sets `Content-Length` instead of falling back to `Transfer-Encoding: chunked`. S3 presigned PUT URLs reject chunked requests with `501 NotImplemented`, breaking template uploads in self-hosted deployments backed by S3-compatible storage. Aligns the JS SDK with the Python SDK, which already buffers via `io.BytesIO`.

packages/js-sdk/src/template/buildApi.ts

@@ -1,5 +1,5 @@
 import { ApiClient, handleApiError, paths, components } from '../api'
-import { stripAnsi } from '../utils'
+import { dynamicImport, stripAnsi } from '../utils'
 import { BuildError, FileUploadError, TemplateError } from '../errors'
 import { LogEntry } from './logger'
 import { getBuildStepIndex, tarFileStreamUpload } from './utils'
@@ -119,12 +119,25 @@ export async function uploadFile(
       resolveSymlinks
     )
 
-    // The compiler assumes this is Web fetch API, but it's actually Node.js fetch API
+    // Buffer the archive before uploading so fetch sets Content-Length.
+    // S3 presigned PUT URLs reject Transfer-Encoding: chunked with 501
+    // NotImplemented, which is what Node's fetch falls back to when the
+    // body is a Readable without a known length. See e2b-dev/e2b#1243.
+    // The Python SDK takes the same approach (build_api.py:upload_file).
+    // Dynamically import so the browser bundle doesn't pull in node:stream.
+    // tar's Pack extends Minipass and is iterable as AsyncIterable<Buffer> at
+    // runtime, but the cli's tsconfig (preserveSymlinks) doesn't surface that
+    // through the type chain — cast via unknown.
+    const { buffer } = await dynamicImport<
+      typeof import('node:stream/consumers')
+    >('node:stream/consumers')
+    const uploadBody = await buffer(
+      uploadStream as unknown as AsyncIterable<Buffer>
+    )
+
     const res = await fetch(url, {
       method: 'PUT',
-      // @ts-expect-error
-      body: uploadStream,
-      duplex: 'half',
+      body: uploadBody,
     })
 
     if (!res.ok) {

packages/js-sdk/tests/template/uploadFile.test.ts

@@ -0,0 +1,68 @@
+import { describe, test, expect, beforeAll, afterAll } from 'vitest'
+import { writeFile, mkdtemp, rm } from 'fs/promises'
+import { join } from 'path'
+import { tmpdir } from 'os'
+import { createServer, type IncomingMessage, type Server } from 'http'
+import { AddressInfo } from 'net'
+import { uploadFile } from '../../src/template/buildApi'
+
+// Regression test for e2b-dev/e2b#1243 — uploadFile used to pass a Node
+// Readable directly to fetch, which made undici fall back to
+// Transfer-Encoding: chunked. S3 presigned PUT URLs reject that with 501
+// NotImplemented. The fix buffers the archive first so Content-Length is set.
+describe('uploadFile transfer encoding', () => {
+  let testDir: string
+  let server: Server
+  let baseUrl: string
+  let capturedHeaders: IncomingMessage['headers'] = {}
+  let capturedBodyLength = 0
+
+  beforeAll(async () => {
+    testDir = await mkdtemp(join(tmpdir(), 'uploadFile-test-'))
+    await writeFile(join(testDir, 'hello.txt'), 'hello world')
+
+    server = createServer((req, res) => {
+      capturedHeaders = req.headers
+      let bytes = 0
+      req.on('data', (chunk: Buffer) => {
+        bytes += chunk.length
+      })
+      req.on('end', () => {
+        capturedBodyLength = bytes
+        res.writeHead(200)
+        res.end()
+      })
+    })
+    await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve))
+    const { port } = server.address() as AddressInfo
+    baseUrl = `http://127.0.0.1:${port}/upload`
+  })
+
+  afterAll(async () => {
+    await new Promise<void>((resolve) => server.close(() => resolve()))
+    await rm(testDir, { recursive: true, force: true })
+  })
+
+  test('sets Content-Length and does not use chunked transfer encoding', async () => {
+    await uploadFile(
+      {
+        fileName: '*.txt',
+        fileContextPath: testDir,
+        url: baseUrl,
+        ignorePatterns: [],
+        resolveSymlinks: false,
+      },
+      undefined
+    )
+
+    expect(capturedHeaders['content-length']).toBeDefined()
+    const contentLength = Number(capturedHeaders['content-length'])
+    expect(contentLength).toBeGreaterThan(0)
+    expect(contentLength).toBe(capturedBodyLength)
+
+    const transferEncoding = capturedHeaders['transfer-encoding']
+    if (transferEncoding !== undefined) {
+      expect(transferEncoding.toLowerCase()).not.toContain('chunked')
+    }
+  })
+})

packages/python-sdk/tests/async/template_async/test_upload_file.py

@@ -0,0 +1,71 @@
+import threading
+from http.server import BaseHTTPRequestHandler, HTTPServer
+
+from e2b.api.client.client import AuthenticatedClient
+from e2b.template_async.build_api import upload_file
+
+
+# Regression test for e2b-dev/e2b#1243 — upload_file must set Content-Length
+# and must not fall back to Transfer-Encoding: chunked. S3 presigned PUT URLs
+# reject chunked encoding with 501 NotImplemented. httpx sets Content-Length
+# automatically when we pass bytes (tar_buffer.getvalue()); this test guards
+# against someone swapping the bytes for a generator/stream later.
+#
+# The mock server runs in a daemon thread and doesn't need to be async — the
+# httpx.AsyncClient connects to it via asyncio sockets without blocking the
+# event loop.
+
+
+def _make_server():
+    state = {"headers": None, "body_length": 0}
+
+    class Handler(BaseHTTPRequestHandler):
+        def do_PUT(self):
+            state["headers"] = dict(self.headers)
+            length = int(self.headers.get("Content-Length", 0))
+            body = self.rfile.read(length) if length else b""
+            state["body_length"] = len(body)
+            self.send_response(200)
+            self.end_headers()
+
+        def log_message(self, *args, **kwargs):
+            return
+
+    server = HTTPServer(("127.0.0.1", 0), Handler)
+    thread = threading.Thread(target=server.serve_forever, daemon=True)
+    thread.start()
+    return server, thread, state
+
+
+async def test_upload_file_sets_content_length_and_no_chunked_encoding(tmp_path):
+    (tmp_path / "hello.txt").write_text("hello world")
+
+    server, thread, state = _make_server()
+    host, port = server.server_address
+    url = f"http://{host}:{port}/upload"
+
+    try:
+        client = AuthenticatedClient(base_url="http://test", token="test")
+        await upload_file(
+            api_client=client,
+            file_name="*.txt",
+            context_path=str(tmp_path),
+            url=url,
+            ignore_patterns=[],
+            resolve_symlinks=False,
+            stack_trace=None,
+        )
+    finally:
+        server.shutdown()
+        server.server_close()
+        thread.join(timeout=5)
+
+    assert state["headers"] is not None
+    content_length = state["headers"].get("Content-Length")
+    assert content_length is not None
+    assert int(content_length) > 0
+    assert int(content_length) == state["body_length"]
+
+    transfer_encoding = state["headers"].get("Transfer-Encoding")
+    if transfer_encoding is not None:
+        assert "chunked" not in transfer_encoding.lower()

packages/python-sdk/tests/sync/template_sync/test_upload_file.py

@@ -0,0 +1,67 @@
+import threading
+from http.server import BaseHTTPRequestHandler, HTTPServer
+
+from e2b.api.client.client import AuthenticatedClient
+from e2b.template_sync.build_api import upload_file
+
+
+# Regression test for e2b-dev/e2b#1243 — upload_file must set Content-Length
+# and must not fall back to Transfer-Encoding: chunked. S3 presigned PUT URLs
+# reject chunked encoding with 501 NotImplemented. httpx sets Content-Length
+# automatically when we pass bytes (tar_buffer.getvalue()); this test guards
+# against someone swapping the bytes for a generator/stream later.
+
+
+def _make_server():
+    state = {"headers": None, "body_length": 0}
+
+    class Handler(BaseHTTPRequestHandler):
+        def do_PUT(self):
+            state["headers"] = dict(self.headers)
+            length = int(self.headers.get("Content-Length", 0))
+            body = self.rfile.read(length) if length else b""
+            state["body_length"] = len(body)
+            self.send_response(200)
+            self.end_headers()
+
+        def log_message(self, *args, **kwargs):
+            return
+
+    server = HTTPServer(("127.0.0.1", 0), Handler)
+    thread = threading.Thread(target=server.serve_forever, daemon=True)
+    thread.start()
+    return server, thread, state
+
+
+def test_upload_file_sets_content_length_and_no_chunked_encoding(tmp_path):
+    (tmp_path / "hello.txt").write_text("hello world")
+
+    server, thread, state = _make_server()
+    host, port = server.server_address
+    url = f"http://{host}:{port}/upload"
+
+    try:
+        client = AuthenticatedClient(base_url="http://test", token="test")
+        upload_file(
+            api_client=client,
+            file_name="*.txt",
+            context_path=str(tmp_path),
+            url=url,
+            ignore_patterns=[],
+            resolve_symlinks=False,
+            stack_trace=None,
+        )
+    finally:
+        server.shutdown()
+        server.server_close()
+        thread.join(timeout=5)
+
+    assert state["headers"] is not None
+    content_length = state["headers"].get("Content-Length")
+    assert content_length is not None
+    assert int(content_length) > 0
+    assert int(content_length) == state["body_length"]
+
+    transfer_encoding = state["headers"].get("Transfer-Encoding")
+    if transfer_encoding is not None:
+        assert "chunked" not in transfer_encoding.lower()