Add more structured output for tests

The diff and the errors were always available, but they were not being
sent to the output.

Additionally, make sure to send output to both the test explorer log and
the codeql test log.

Author: aeisenberg

extensions/ql-vscode/CHANGELOG.md

@@ -14,6 +14,7 @@
 - Clear the problems view of all CodeQL query results when a database is removed.
 - Add a `View DIL` command on query history items. This opens a text editor containing the Datalog Intermediary Language representation of the compiled query.
 - Alter structure of the _Test Explorer_ tree. It now follows the structure of the filesystem instead of the qlpacks.
+- Ensure output of CodeQL test runs includes compilation error messages and test failure messages.
 
 ## 1.3.3 - 16 September 2020
 

extensions/ql-vscode/src/cli.ts

@@ -14,6 +14,7 @@ import { DistributionProvider } from './distribution';
 import { assertNever } from './helpers-pure';
 import { QueryMetadata, SortDirection } from './interface-types';
 import { Logger, ProgressReporter } from './logging';
+import { CompilationMessage } from './messages';
 
 /**
  * The version of the SARIF format that we are using.
@@ -87,10 +88,11 @@ export interface TestRunOptions {
 export interface TestCompleted {
   test: string;
   pass: boolean;
-  messages: string[];
+  messages: CompilationMessage[];
   compilationMs: number;
   evaluationMs: number;
   expected: string;
+  diff: string[] | undefined;
 }
 
 /**

extensions/ql-vscode/src/test-adapter.ts

@@ -221,10 +221,26 @@ export class QLTestAdapter extends DisposableObject implements TestAdapter {
       cancellationToken: cancellationToken,
       logger: testLogger
     })) {
+      const state = event.pass
+        ? 'passed'
+        : event.messages?.length
+        ? 'errored'
+        : 'failed';
+      let message: string | undefined;
+      if (event.diff?.length) {
+        message = ['', `${state}: ${event.test}`, ...event.diff, ''].join('\n');
+        testLogger.log(message);
+      }
+      (event.diff || []).join('\n');
       this._testStates.fire({
         type: 'test',
-        state: event.pass ? 'passed' : 'failed',
-        test: event.test
+        state,
+        test: event.test,
+        message,
+        decorations: event.messages?.map(msg => ({
+          line: msg.position.line,
+          message: msg.message
+        }))
       });
     }
   }

extensions/ql-vscode/src/vscode-tests/no-workspace/databases.test.ts

@@ -24,8 +24,9 @@ describe('databases', () => {
     databaseManager = new DatabaseManager(
       {
         workspaceState: {
-          update: updateSpy
-        }
+          update: updateSpy,
+          get: sinon.stub()
+        },
       } as unknown as ExtensionContext,
       {} as QueryServerConfig,
       {} as Logger,

extensions/ql-vscode/src/vscode-tests/no-workspace/test-adapter.test.ts

@@ -0,0 +1,106 @@
+import 'vscode-test';
+import 'mocha';
+import * as sinon from 'sinon';
+import { Uri, WorkspaceFolder } from 'vscode';
+import { expect } from 'chai';
+
+import { QLTestAdapter } from '../../test-adapter';
+import { CodeQLCliServer } from '../../cli';
+
+describe('test-adapter', () => {
+  let adapter: QLTestAdapter;
+  let runTestsSpy: sinon.SinonStub;
+  let resolveTestsSpy: sinon.SinonStub;
+  let resolveQlpacksSpy: sinon.SinonStub;
+  let sandox: sinon.SinonSandbox;
+
+  beforeEach(() => {
+    sandox = sinon.createSandbox();
+    mockRunTests();
+    resolveQlpacksSpy = sandox.stub().resolves({});
+    resolveTestsSpy = sandox.stub().resolves([]);
+    adapter = new QLTestAdapter({
+      name: 'ABC',
+      uri: Uri.parse('file:/ab/c')
+    } as WorkspaceFolder, {
+      runTests: runTestsSpy,
+      resolveQlpacks: resolveQlpacksSpy,
+      resolveTests: resolveTestsSpy
+    } as unknown as CodeQLCliServer);
+  });
+
+  afterEach(() => {
+    sandox.restore();
+  });
+
+  it('should run some tests', async () => {
+
+    const listenerSpy = sandox.spy();
+    adapter.testStates(listenerSpy);
+    const testsPath = Uri.parse('file:/ab/c').fsPath;
+    const dPath = Uri.parse('file:/ab/c/d.ql').fsPath;
+    const gPath = Uri.parse('file:/ab/c/e/f/g.ql').fsPath;
+    const hPath = Uri.parse('file:/ab/c/e/f/h.ql').fsPath;
+
+    await adapter.run([testsPath]);
+
+    expect(listenerSpy.getCall(0).args).to.deep.eq([
+      { type: 'started', tests: [testsPath] }
+    ]);
+    expect(listenerSpy.getCall(1).args).to.deep.eq([{
+      type: 'test',
+      state: 'passed',
+      test: dPath,
+      message: undefined,
+      decorations: []
+    }]);
+    expect(listenerSpy.getCall(2).args).to.deep.eq([{
+      type: 'test',
+      state: 'errored',
+      test: gPath,
+      message: `\nerrored: ${gPath}\npqr\nxyz\n`,
+      decorations: [
+        { line: 1, message: 'abc' }
+      ]
+    }]);
+    expect(listenerSpy.getCall(3).args).to.deep.eq([{
+      type: 'test',
+      state: 'failed',
+      test: hPath,
+      message: `\nfailed: ${hPath}\njkh\ntuv\n`,
+      decorations: []
+    }]);
+    expect(listenerSpy.getCall(4).args).to.deep.eq([{ type: 'finished' }]);
+    expect(listenerSpy).to.have.callCount(5);
+  });
+
+  function mockRunTests() {
+    // runTests is an async generator function. This is not directly supported in sinon
+    // However, we can pretend the same thing by just returning an async array.
+    runTestsSpy = sandox.stub();
+    runTestsSpy.returns(
+      (async function*() {
+        yield Promise.resolve({
+          test: Uri.parse('file:/ab/c/d.ql').fsPath,
+          pass: true,
+          messages: []
+        });
+        yield Promise.resolve({
+          test: Uri.parse('file:/ab/c/e/f/g.ql').fsPath,
+          pass: false,
+          diff: ['pqr', 'xyz'],
+          // a compile error
+          messages: [
+            { position: { line: 1 }, message: 'abc' }
+          ]
+        });
+        yield Promise.resolve({
+          test: Uri.parse('file:/ab/c/e/f/h.ql').fsPath,
+          pass: false,
+          diff: ['jkh', 'tuv'],
+          messages: []
+        });
+      })()
+    );
+  }
+});