🧪 Redesign testing to use vscode-test CLI (#235)

VSCode-required E2E tests and Mocha tests are run separately now based on whether they are in a thing.tests.ts or thing.vscode.tests.ts file.

Author: JustinGrote

.mocharc.json

@@ -0,0 +1,9 @@
+require: 'esbuild-register'
+spec:
+  - 'src/**/*.test.ts'
+  - 'test/**/*.test.ts'
+ignore: ['**/*.vscode.test.ts']
+exit: true
+failZero: true
+color: true
+enable-source-maps: true

.mocharc.yml

@@ -1,2 +1,3 @@
+# Dont forget to update this in package.json script esbuild-base too!
 use-node-version=18.15.0
 engine-strict=true

.npmrc

@@ -0,0 +1,11 @@
+// .vscode-test.js
+const { defineConfig } = require('@vscode/test-cli')
+
+module.exports = defineConfig({
+	files: 'dist/test/**/*.vscode.test.js',
+	launchArgs: ['--profile=vscode-pester-test'],
+	mocha: {
+		ui: 'bdd',
+		timeout: 600000 // 10 minutes to allow for debugging
+	}
+})

.vscode-test.js

@@ -5,6 +5,7 @@
 		"github.vscode-github-actions",
 		"dbaeumer.vscode-eslint",
 		"connor4312.esbuild-problem-matchers",
-		"hbenl.vscode-mocha-test-adapter"
+		"hbenl.vscode-mocha-test-adapter",
+		"ms-vscode.extension-test-runner"
 	]
 }

.vscode/extensions.json

@@ -52,27 +52,6 @@
 			"showAsyncStacks": true,
 			"smartStep": true,
 			"preLaunchTask": "build-watch"
-		},
-		{
-			"name": "Test Extension",
-			"type": "node",
-			"request": "launch",
-			"program": "${workspaceFolder}/test/runTests.ts",
-			"runtimeArgs": ["-r", "@swc-node/register"],
-			"sourceMaps": true,
-			"resolveSourceMapLocations": ["${workspaceFolder}/**", "!**/node_modules/**", "!**/.vscode-test/**"],
-			"smartStep": true,
-			"attachSimplePort": 59229, // The default is 9229 but we want to avoid conflicts because we will have two vscode instances running.
-			"env": {
-				"__TEST_DEBUG_INSPECT_PORT": "59229" // Needs to match attachSimplePort
-			},
-			"presentation": {
-				"hidden": false,
-				"group": "_test"
-			},
-			"internalConsoleOptions": "neverOpen",
-			"console": "integratedTerminal",
-			"autoAttachChildProcesses": false
 		}
 	]
 }

.vscode/launch.json

@@ -6,9 +6,11 @@
 	"[typescript]": {
 		"editor.defaultFormatter": "vscode.typescript-language-features"
 	},
-	"mochaExplorer.configFile": ".mocharc.json",
 	"mochaExplorer.logpanel": true,
 	"testExplorer.useNativeTesting": true,
+	"extension-test-runner.debugOptions": {
+		"resolveSourceMapLocations": ["${workspaceFolder}/**", "!**/node_modules/**"]
+	},
 	"mochaExplorer.env": {
 		"VSCODE_VERSION": "insiders",
 		"ELECTRON_RUN_AS_NODE": null

.vscode/settings.json

@@ -10,50 +10,39 @@
 			"type": "npm",
 			"script": "build-watch",
 			"group": "build",
-			"problemMatcher": {
-				"owner": "typescript",
-				"source": "rollup",
-				"pattern": {
-					"regexp": "^(.*):(\\d+):(\\d+) - (error|warning) TS(\\d+): (.*)$",
-					"file": 1,
-					"line": 2,
-					"column": 3,
-					"severity": 4,
-					"code": 5,
-					"message": 6
-				},
-				"severity": "info",
-				"fileLocation": ["relative", "${workspaceFolder}"],
-				"background": {
-					"activeOnStart": true,
-					"beginsPattern": "^rollup v\\d+\\.\\d+\\.\\d+$",
-					"endsPattern": {
-						"regexp": "^\\[[:-\\s\\d]+\\] waiting for changes\\.\\.\\.$"
-					}
-				}
+			"problemMatcher": "$esbuild-watch",
+			"isBackground": true
+		},
+		{
+			"label": "build-test-vscode-watch",
+			"icon": {
+				"color": "terminal.ansiBlue",
+				"id": "beaker"
 			},
+			"type": "npm",
+			"script": "build-test-vscode-watch",
+			"group": "build",
+			"problemMatcher": "$esbuild-watch",
 			"isBackground": true
 		},
 		{
 			"label": "test-watch",
 			"icon": {
-				"color": "terminal.ansiCyan",
-				"id": "sync"
+				"color": "terminal.ansiBlue",
+				"id": "beaker"
 			},
 			"type": "npm",
 			"script": "test-watch",
 			"group": "test",
+			"isBackground": true,
 			"problemMatcher": {
 				"owner": "typescript",
 				"source": "mocha",
 				"pattern": {
-					"regexp": "^(.*):(\\d+):(\\d+) - (error|warning) TS(\\d+): (.*)$",
-					"file": 1,
-					"line": 2,
-					"column": 3,
-					"severity": 4,
-					"code": 5,
-					"message": 6
+					"regexp": "\\w+?(Error): (.+)\\w+?\\((.+)\\)",
+					"severity": 1,
+					"message": 2,
+					"file": 3
 				},
 				"severity": "info",
 				"fileLocation": ["relative", "${workspaceFolder}"],
@@ -64,8 +53,7 @@
 						"regexp": "^ℹ \\[mocha\\] waiting for changes...$"
 					}
 				}
-			},
-			"isBackground": true
+			}
 		}
 	]
 }

.vscode/tasks.json

@@ -86,26 +86,24 @@
 	],
 	"main": "./dist/extension.js",
 	"devDependencies": {
-		"@rollup/plugin-commonjs": "^25.0.7",
-		"@rollup/plugin-node-resolve": "^15.2.3",
-		"@rollup/plugin-swc": "^0.3.0",
-		"@rollup/plugin-terser": "^0.4.4",
-		"@swc-node/register": "^1.6.8",
 		"@types/chai": "^4.3.5",
 		"@types/debounce-promise": "^3.1.9",
 		"@types/glob": "^8.1.0",
 		"@types/json-parse-safe": "^2.0.3",
 		"@types/mocha": "^10.0.6",
 		"@types/node": "^20.10.6",
 		"@types/readline-transform": "^1.0.4",
+		"@types/vscode": "^1.77.0",
 		"@typescript-eslint/eslint-plugin": "^6.16.0",
 		"@typescript-eslint/parser": "^6.16.0",
 		"@vscode/dts": "^0.4.0",
+		"@vscode/test-cli": "^0.0.4",
 		"@vscode/test-electron": "^2.3.8",
 		"@vscode/vsce": "^2.22.0",
 		"chai": "^5.0.0",
 		"debounce-promise": "^3.1.2",
 		"esbuild": "^0.19.11",
+		"esbuild-register": "^3.5.0",
 		"eslint": "^8.56.0",
 		"eslint-config-prettier": "^9.1.0",
 		"eslint-config-standard-with-typescript": "^43.0.0",
@@ -116,13 +114,11 @@
 		"json-parse-safe": "^2.0.0",
 		"lookpath": "^1.2.2",
 		"mocha": "^10.2.0",
-		"mocha-multi-reporters": "^1.5.1",
 		"path": "^0.12.7",
 		"pkgroll": "^2.0.1",
 		"prettier": "^3.1.1",
 		"prettier-eslint": "^16.2.0",
 		"readline-transform": "^1.0.0",
-		"rollup": "^4.9.2",
 		"sinon": "^17.0.1",
 		"tslog": "^4.9.2",
 		"typescript": "^5.3.3",
@@ -145,25 +141,32 @@
 		}
 	],
 	"scripts": {
+		"esbuild-base": "pnpm esbuild --minify --bundle --platform=node --target=node18.15 --format=cjs --sourcemap --sources-content=false --external:mocha --external:vscode",
 		"package": "vsce package --no-update-package-json --no-git-tag-version --no-dependencies",
 		"publish": "pnpm run publishStable --pre-release --no-git-tag-version",
 		"publishStable": "vsce publish --no-update-package-json --no-dependencies",
-		"build": "rollup --config",
+		"build": "pnpm run esbuild-base src/extension.ts --outdir=dist",
 		"build-watch": "pnpm run build --watch",
+		"build-test-vscode": "pnpm run esbuild-base src/**/*.vscode.test.ts test/**/*.vscode.test.ts --outdir=dist/test",
+		"build-test-vscode-watch": "pnpm run build-test-vscode --watch",
+		"test-mocha": "mocha",
+		"test-mocha-vscode": "pnpm vscode-test",
+		"test": "pnpm test-mocha && pnpm test-mocha-vscode",
+		"test-watch": "pnpm test-mocha --watch",
 		"lint-eslint": "eslint . --ext .ts",
 		"lint-prettier": "prettier --check .",
 		"lint-tsc": "tsc --noemit",
 		"lint": "pnpm lint-eslint || pnpm lint-prettier || pnpm lint-tsc",
-		"test-mocha": "mocha",
-		"test-mocha-vscode": "node -r @swc-node/register test/runTests.ts",
-		"test": "pnpm test-mocha && pnpm test-mocha-vscode",
-		"test-watch": "pnpm test-mocha --watch",
 		"download-api": "vscode-dts main",
 		"postdownload-api": "vscode-dts main",
 		"vscode:prepublish": "pnpm run build"
 	},
 	"dependencies": {
-		"@ctiterm/strip-ansi": "^1.0.0",
-		"@swc/core": "^1.3.102"
+		"@ctiterm/strip-ansi": "^1.0.0"
+	},
+	"pnpm": {
+		"patchedDependencies": {
+			"@vscode/test-cli@0.0.4": "patches/@vscode__test-cli@0.0.4.patch"
+		}
 	}
 }

package.json

@@ -0,0 +1,14 @@
+diff --git a/out/bin.mjs b/out/bin.mjs
+index 5282c55bd62d0b77c17f77a173ff848286c0c2f8..857e6d9ca1b6275185c11ab7ebadb1646b5a5afe 100644
+--- a/out/bin.mjs
++++ b/out/bin.mjs
+@@ -11,6 +11,9 @@ import { dirname, isAbsolute, join, resolve } from 'path';
+ import supportsColor from 'supports-color';
+ import { fileURLToPath, pathToFileURL } from 'url';
+ import yargs from 'yargs';
++import { createRequire } from 'node:module';
++
++const require = createRequire(import.meta.url);
+ const rulesAndBehavior = 'Mocha: Rules & Behavior';
+ const reportingAndOutput = 'Mocha: Reporting & Output';
+ const fileHandling = 'Mocha: File Handling';