fix #4343: integrity check for binary download

Author: evanw

CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## Unreleased
+
+* Add integrity checks to fallback download path ([#4343](https://github.com/evanw/esbuild/issues/4343))
+
+    Installing esbuild via npm is somewhat complicated with several different edge cases (see [esbuild's documentation](https://esbuild.github.io/getting-started/#additional-npm-flags) for details). If the regular installation of esbuild's platform-specific package fails, esbuild's install script attempts to download the platform-specific package itself (first with the `npm` command, and then with a HTTP request to `registry.npmjs.org` as a last resort).
+
+    This last resort path previously didn't have any integrity checks. With this release, esbuild will now verify that the hash of the downloaded binary matches the expected hash for the current release. This means the hashes for all of esbuild's platform-specific binary packages will now be embedded in the top-level `esbuild` package. Hopefully this should work without any problems. But just in case, this change is being done as a breaking change release.
+
 ## 0.27.7
 
 * Fix lowering of define semantics for TypeScript parameter properties ([#4421](https://github.com/evanw/esbuild/issues/4421))

Makefile

@@ -317,13 +317,20 @@ version-go:
 	node scripts/esbuild.js --update-version-go
 
 platform-all: \
-	platform-aix-ppc64 \
 	platform-android-arm \
-	platform-android-arm64 \
 	platform-android-x64 \
+	platform-deno \
+	platform-neutral \
+	platform-openharmony-arm64 \
+	platform-wasi-preview1 \
+	platform-wasm
+
+# Note: The dependency list here must match `generateBinaryHashes` in `./scripts/esbuild.js`
+platform-neutral: \
+	platform-aix-ppc64 \
+	platform-android-arm64 \
 	platform-darwin-arm64 \
 	platform-darwin-x64 \
-	platform-deno \
 	platform-freebsd-arm64 \
 	platform-freebsd-x64 \
 	platform-linux-arm \
@@ -337,17 +344,19 @@ platform-all: \
 	platform-linux-x64 \
 	platform-netbsd-arm64 \
 	platform-netbsd-x64 \
-	platform-neutral \
 	platform-openbsd-arm64 \
 	platform-openbsd-x64 \
-	platform-openharmony-arm64 \
 	platform-sunos-x64 \
-	platform-wasi-preview1 \
-	platform-wasm \
 	platform-win32-arm64 \
 	platform-win32-ia32 \
 	platform-win32-x64
 
+	# This must happen last because it hashes everything from the steps above
+	@echo
+	@echo "# Build: npm/esbuild"
+	node scripts/esbuild.js npm/esbuild/package.json --version
+	node scripts/esbuild.js ./esbuild --neutral
+
 platform-internal:
 	@test -n "$(GOOS)" || (echo "The environment variable GOOS must be provided" && false)
 	@test -n "$(GOARCH)" || (echo "The environment variable GOARCH must be provided" && false)
@@ -453,12 +462,6 @@ platform-wasm: esbuild go-compiler
 	$(GO_COMPILER) "$(NODE)" scripts/esbuild.js ./esbuild --wasm
 	@shasum -a 256 npm/esbuild-wasm/esbuild.wasm
 
-platform-neutral: esbuild
-	@echo
-	@echo "# Build: npm/esbuild"
-	node scripts/esbuild.js npm/esbuild/package.json --version
-	node scripts/esbuild.js ./esbuild --neutral
-
 platform-deno: platform-wasm go-compiler
 	@echo
 	@echo "# Build: deno"

lib/npm/node-install.ts

@@ -5,9 +5,15 @@ import os = require('os')
 import path = require('path')
 import zlib = require('zlib')
 import https = require('https')
+import crypto = require('crypto')
 import child_process = require('child_process')
 
-const versionFromPackageJSON: string = require(path.join(__dirname, 'package.json')).version
+interface PackageJSON {
+  version: string
+  'esbuild.binaryHashes': Record<string, string>
+}
+
+const packageJSON: PackageJSON = require(path.join(__dirname, 'package.json')) as typeof import('../../npm/esbuild/package.json')
 const toPath = path.join(__dirname, 'bin', 'esbuild')
 let isToPathJS = true
 
@@ -48,8 +54,8 @@ which means the "esbuild" binary executable can't be run. You can either:
     }
     throw err
   }
-  if (stdout !== versionFromPackageJSON) {
-    throw new Error(`Expected ${JSON.stringify(versionFromPackageJSON)} but got ${JSON.stringify(stdout)}`)
+  if (stdout !== packageJSON.version) {
+    throw new Error(`Expected ${JSON.stringify(packageJSON.version)} but got ${JSON.stringify(stdout)}`)
   }
 }
 
@@ -115,13 +121,14 @@ function installUsingNPM(pkg: string, subpath: string, binPath: string): void {
     // command instead of a HTTP request so that it hopefully works in situations
     // where HTTP requests are blocked but the "npm" command still works due to,
     // for example, a custom configured npm registry and special firewall rules.
-    child_process.execSync(`npm install --loglevel=error --prefer-offline --no-audit --progress=false ${pkg}@${versionFromPackageJSON}`,
+    child_process.execSync(`npm install --loglevel=error --prefer-offline --no-audit --progress=false ${pkg}@${packageJSON.version}`,
       { cwd: installDir, stdio: 'pipe', env })
 
     // Move the downloaded binary executable into place. The destination path
     // is the same one that the JavaScript API code uses so it will be able to
     // find the binary executable here later.
     const installedBinPath = path.join(installDir, 'node_modules', pkg, subpath)
+    binaryIntegrityCheck(pkg, subpath, fs.readFileSync(installedBinPath))
     fs.renameSync(installedBinPath, binPath)
   } finally {
     // Try to clean up afterward so we don't unnecessarily waste file system
@@ -168,7 +175,7 @@ function applyManualBinaryPathOverride(overridePath: string): void {
   fs.writeFileSync(libMain, `var ESBUILD_BINARY_PATH = ${pathString};\n${code}`)
 }
 
-function maybeOptimizePackage(binPath: string): void {
+function maybeOptimizePackage(binPath: string, isWASM: boolean): void {
   // This package contains a "bin/esbuild" JavaScript file that finds and runs
   // the appropriate binary executable. However, this means that running the
   // "esbuild" command runs another instance of "node" which is way slower than
@@ -190,7 +197,6 @@ function maybeOptimizePackage(binPath: string): void {
   //
   // This optimization also doesn't apply when npm's "--ignore-scripts" flag is
   // used since in that case this install script will not be run.
-  const { isWASM } = pkgAndSubpathForCurrentPlatform()
   if (os.platform() !== 'win32' && !isYarn() && !isWASM) {
     const tempPath = path.join(__dirname, 'bin-esbuild')
     try {
@@ -219,13 +225,23 @@ function maybeOptimizePackage(binPath: string): void {
   }
 }
 
+function binaryIntegrityCheck(pkg: string, subpath: string, bytes: Uint8Array): void {
+  const hash = crypto.createHash('sha256').update(bytes).digest('hex')
+  const key = `${pkg}/${subpath}`
+  const expected = packageJSON['esbuild.binaryHashes'][key]
+  if (!expected) throw new Error(`Missing hash for "${key}"`)
+  if (hash !== expected) throw new Error(`"${hash.slice(0, 8)}..." doesn't match "${expected.slice(0, 8)}..."`)
+}
+
 async function downloadDirectlyFromNPM(pkg: string, subpath: string, binPath: string): Promise<void> {
   // If that fails, the user could have npm configured incorrectly or could not
   // have npm installed. Try downloading directly from npm as a last resort.
-  const url = `https://registry.npmjs.org/${pkg}/-/${pkg.replace('@esbuild/', '')}-${versionFromPackageJSON}.tgz`
+  const url = `https://registry.npmjs.org/${pkg}/-/${pkg.replace('@esbuild/', '')}-${packageJSON.version}.tgz`
   console.error(`[esbuild] Trying to download ${JSON.stringify(url)}`)
   try {
-    fs.writeFileSync(binPath, extractFileFromTarGzip(await fetch(url), subpath))
+    const bytes = extractFileFromTarGzip(await fetch(url), subpath)
+    binaryIntegrityCheck(pkg, subpath, bytes)
+    fs.writeFileSync(binPath, bytes)
     fs.chmodSync(binPath, 0o755)
   } catch (e: any) {
     console.error(`[esbuild] Failed to download ${JSON.stringify(url)}: ${e && e.message || e}`)
@@ -246,7 +262,7 @@ async function checkAndPreparePackage(): Promise<void> {
     }
   }
 
-  const { pkg, subpath } = pkgAndSubpathForCurrentPlatform()
+  const { pkg, subpath, isWASM } = pkgAndSubpathForCurrentPlatform()
 
   let binPath: string
   try {
@@ -262,6 +278,12 @@ for your current platform. This install script will now attempt to work around
 this. If that fails, you need to remove the "--no-optional" flag to use esbuild.
 `)
 
+    // The "binary" in the WebAssembly package is not actually a binary, and is
+    // not self-contained. It's a JavaScript file that references another
+    // binary "esbuild.wasm" file. The fallback code below assumes that the
+    // binary is self-contained, so fail now if this is a WebAssembly fallback.
+    if (isWASM) throw new Error(`Failed to install package "${pkg}"`)
+
     // If that didn't work, then someone probably installed esbuild with the
     // "--no-optional" flag. Attempt to compensate for this by downloading the
     // package using a nested call to "npm" instead.
@@ -289,7 +311,7 @@ this. If that fails, you need to remove the "--no-optional" flag to use esbuild.
     }
   }
 
-  maybeOptimizePackage(binPath)
+  maybeOptimizePackage(binPath, isWASM)
 }
 
 checkAndPreparePackage().then(() => {

lib/tsconfig-deno.json

@@ -3,6 +3,7 @@
     "module": "esnext", // Allow the "import.meta" and top-level await syntax
     "target": "es2017", // Allow calling APIs such as "Object.entries"
     "strict": true,
+    "resolveJsonModule": true,
     "skipLibCheck": true,
     "types": [
       "node"

lib/tsconfig-nolib.json

@@ -3,6 +3,7 @@
     "module": "CommonJS", // Allow the "import assignment" syntax
     "target": "es2017", // Allow calling APIs such as "Object.entries"
     "strict": true,
+    "resolveJsonModule": true,
     "lib": [
       // Omit "dom" to test what happens with the "WebAssembly" type, which is defined in "dom"
       "ES5",

lib/tsconfig.json

@@ -3,6 +3,7 @@
     "module": "CommonJS", // Allow the "import assignment" syntax
     "target": "es2017", // Allow calling APIs such as "Object.entries"
     "strict": true,
+    "resolveJsonModule": true,
     "types": [
       "node"
     ],

npm/esbuild/package.json

@@ -45,5 +45,30 @@
     "@esbuild/win32-ia32": "0.27.7",
     "@esbuild/win32-x64": "0.27.7"
   },
-  "license": "MIT"
+  "license": "MIT",
+  "esbuild.binaryHashes": {
+    "@esbuild/aix-ppc64/bin/esbuild": "e189c30ba15cbdf0ff36985af2b31d3c45921e5c4f7d7a20d0779ced42c9a461",
+    "@esbuild/android-arm64/bin/esbuild": "f98558df51a9c5bc7611f327e740a49b444eea45549418d5043cef623add2a6c",
+    "@esbuild/darwin-arm64/bin/esbuild": "b82c6243eb58d520d0bc55cddcf6993dd0f5572f3ab8dc2ba6d77058be043e62",
+    "@esbuild/darwin-x64/bin/esbuild": "616ac773dfd3ab720d8a000244cc8ef460245d7a2292e94a82a9d17c0919a826",
+    "@esbuild/freebsd-arm64/bin/esbuild": "55423b64f0d643ed537ec346b45c012e9e38b143331a57b18bc116bfb3010714",
+    "@esbuild/freebsd-x64/bin/esbuild": "7df169572f3b6f9ef598c83b52e9f132b68c0e20d67d6fc47f96ed4da885f755",
+    "@esbuild/linux-arm/bin/esbuild": "97d76dfa7aa0e7716eadf5f14a3bce2c010b7fd22b1940b11b89eae04d3135ac",
+    "@esbuild/linux-arm64/bin/esbuild": "57fa0b46e197f4ff8d67763cc2b1bdf28090578fc5e6bbe71b18473c296fcfcf",
+    "@esbuild/linux-ia32/bin/esbuild": "b61ad8f30aa458b7dfd79a4b6a744aa101971b85f4f9a508970f9ccf919e8386",
+    "@esbuild/linux-loong64/bin/esbuild": "d69c5174b36d5ce9c5d9e0455e5e9e97b6b0e007709d16d9761fe0d1c8646740",
+    "@esbuild/linux-mips64el/bin/esbuild": "ff85f4b896b656eeab590f9389f796792f8afa607200396a1955cf85624d095e",
+    "@esbuild/linux-ppc64/bin/esbuild": "6b6bdb97ba7523f77c015246579dfd13a7d0a321908221b84fc2f3788f3299f7",
+    "@esbuild/linux-riscv64/bin/esbuild": "8cb98fa08a9f13b43723387fdd47a842d41e481e4ebacaed8aa516890b5f77f4",
+    "@esbuild/linux-s390x/bin/esbuild": "03c3adfb52f099b1d61e126cf34738446730bd6160a50e5d52913c95984e8f2e",
+    "@esbuild/linux-x64/bin/esbuild": "c638273fcf95573ca74af586677800b4dd874c55f8f945adb54316d6902ba14b",
+    "@esbuild/netbsd-arm64/bin/esbuild": "2fe6fa1e6bd28ac94b0068d0608a26cef41fe264d0c69b4b941a56737b640474",
+    "@esbuild/netbsd-x64/bin/esbuild": "b7b6b75dad26113b545e5d33ed94c06f7b273f8ebcd0aa501ed6130dae0543c1",
+    "@esbuild/openbsd-arm64/bin/esbuild": "5f3e6a235bd9d1377011f29a9dca9daf95b8734904c51db2dfdb493529bff052",
+    "@esbuild/openbsd-x64/bin/esbuild": "79ffa254e3067803e76da4d71a6611cf18620de749fd271318e28abadeabce08",
+    "@esbuild/sunos-x64/bin/esbuild": "6406fd8a315d5a31c31170d8cf4dc9a97beeed1d24f4499588259edde7c1501f",
+    "@esbuild/win32-arm64/esbuild.exe": "3ed4a42ee89b22193e33e700b7eb834b1d70e5490f26f0437d65c94888f7d4b1",
+    "@esbuild/win32-ia32/esbuild.exe": "70d1ad7f1bc85e1fdc0334cf718d8b326bba3c123bf90aed55a6e8a1ea4b4b07",
+    "@esbuild/win32-x64/esbuild.exe": "44ce6728d54c891b1c5a6d7dbfb1a0f13419884cca0b090f1fbcf0dcd8bee0e9"
+  }
 }

scripts/esbuild.js

@@ -1,4 +1,5 @@
 const childProcess = require('child_process')
+const crypto = require('crypto')
 const path = require('path')
 const fs = require('fs')
 const os = require('os')
@@ -81,9 +82,49 @@ const buildNeutralLib = (esbuildPath) => {
 
   // Update "npm/esbuild/package.json"
   const pjPath = path.join(npmDir, 'package.json')
-  const package_json = JSON.parse(fs.readFileSync(pjPath, 'utf8'))
-  package_json.optionalDependencies = optionalDependencies
-  fs.writeFileSync(pjPath, JSON.stringify(package_json, null, 2) + '\n')
+  const packageJSON = JSON.parse(fs.readFileSync(pjPath, 'utf8'))
+  packageJSON.optionalDependencies = optionalDependencies
+  packageJSON['esbuild.binaryHashes'] = generateBinaryHashes()
+  fs.writeFileSync(pjPath, JSON.stringify(packageJSON, null, 2) + '\n')
+}
+
+function generateBinaryHashes() {
+  // Note: The dependency list here must match `platform-neutral` in `../Makefile`
+  const toHash = [
+    // Unix-like
+    '@esbuild/aix-ppc64/bin/esbuild',
+    '@esbuild/android-arm64/bin/esbuild',
+    '@esbuild/darwin-arm64/bin/esbuild',
+    '@esbuild/darwin-x64/bin/esbuild',
+    '@esbuild/freebsd-arm64/bin/esbuild',
+    '@esbuild/freebsd-x64/bin/esbuild',
+    '@esbuild/linux-arm/bin/esbuild',
+    '@esbuild/linux-arm64/bin/esbuild',
+    '@esbuild/linux-ia32/bin/esbuild',
+    '@esbuild/linux-loong64/bin/esbuild',
+    '@esbuild/linux-mips64el/bin/esbuild',
+    '@esbuild/linux-ppc64/bin/esbuild',
+    '@esbuild/linux-riscv64/bin/esbuild',
+    '@esbuild/linux-s390x/bin/esbuild',
+    '@esbuild/linux-x64/bin/esbuild',
+    '@esbuild/netbsd-arm64/bin/esbuild',
+    '@esbuild/netbsd-x64/bin/esbuild',
+    '@esbuild/openbsd-arm64/bin/esbuild',
+    '@esbuild/openbsd-x64/bin/esbuild',
+    '@esbuild/sunos-x64/bin/esbuild',
+
+    // Windows
+    '@esbuild/win32-arm64/esbuild.exe',
+    '@esbuild/win32-ia32/esbuild.exe',
+    '@esbuild/win32-x64/esbuild.exe',
+  ]
+
+  const hashes = {}
+  for (const key of toHash) {
+    const bytes = fs.readFileSync(path.join(repoDir, 'npm', key))
+    hashes[key] = crypto.createHash('sha256').update(bytes).digest('hex')
+  }
+  return hashes
 }
 
 async function generateWorkerCode({ esbuildPath, wasm_exec_js, minify, target }) {