Fix large sarif handling

extensions/ql-vscode/CHANGELOG.md

@@ -3,6 +3,8 @@
 ## [UNRELEASED]
 
 - Fix the _CodeQL: Open Referenced File_ command for Windows systems. [#979](https://github.com/github/vscode-codeql/pull/979)
+- Fix the _CodeQL: Open Referenced File_ command for Windows systems. [#979](https://github.com/github/vscode-codeql/pull/979) 
+- Fix a bug that causes VSCode to crash when handling large SARIF files (>4GB) [#1004](https://github.com/github/vscode-codeql/pull/1004)
 - Fix a bug that shows 'Set current database' when hovering over the currently selected database in the databases view. [#976](https://github.com/github/vscode-codeql/pull/976)
 - Fix a bug with importing large databases. Databases over 4GB can now be imported directly from LGTM or from a zip file. This functionality is only available when using CodeQL CLI version 2.6.0 or later. [#971](https://github.com/github/vscode-codeql/pull/971)
 - Replace certain control codes (`U+0000` - `U+001F`) with their corresponding control labels (`U+2400` - `U+241F`) in the results view. [#963](https://github.com/github/vscode-codeql/pull/963)

extensions/ql-vscode/package.json

@@ -969,6 +969,8 @@
     "react": "^16.8.6",
     "react-dom": "^16.8.6",
     "semver": "~7.3.2",
+    "stream-chain": "~2.2.4",
+    "stream-json": "~1.7.3",
     "tmp": "^0.1.0",
     "tmp-promise": "~3.0.2",
     "tree-kill": "~1.2.2",
@@ -1003,6 +1005,8 @@
     "@types/semver": "~7.2.0",
     "@types/sinon": "~7.5.2",
     "@types/sinon-chai": "~3.2.3",
+    "@types/stream-chain": "~2.0.1",
+    "@types/stream-json": "~1.7.1",
     "@types/through2": "^2.0.36",
     "@types/tmp": "^0.1.0",
     "@types/unzipper": "~0.10.1",

extensions/ql-vscode/src/cli.ts

@@ -2,6 +2,10 @@ import * as cpp from 'child-process-promise';
 import * as child_process from 'child_process';
 import * as fs from 'fs-extra';
 import * as path from 'path';
+import { parser } from 'stream-json';
+import { pick } from 'stream-json/filters/Pick';
+import Assembler = require('stream-json/Assembler');
+import { chain } from 'stream-chain';
 import * as sarif from 'sarif';
 import { SemVer } from 'semver';
 import { Readable } from 'stream';
@@ -34,6 +38,8 @@ const CSV_FORMAT = 'csv';
  */
 const LOGGING_FLAGS = ['-v', '--log-to-stderr'];
 
+const DUMMY_TOOL : sarif.Tool = {driver: {name: ''}};
+
 /**
  * The expected output of `codeql resolve library-path`.
  */
@@ -576,6 +582,46 @@ export class CodeQLCliServer implements Disposable {
     }
   }
 
+  static async parseSarif(interpretedResultsPath: string) : Promise<sarif.Log> {
+    try {
+      // Parse the SARIF file into token streams, filtering out only the results array.
+      const p = parser();
+      const pipeline = chain([
+        fs.createReadStream(interpretedResultsPath),
+        p,
+        pick({filter: 'runs.0.results'})
+      ]);
+
+      // Creates JavaScript objects from the token stream
+      const asm = Assembler.connectTo(pipeline);
+
+      // Returns a constructed Log object with the results or an empty array if no results were found.
+      // If the parser fails for any reason, it will reject the promise.
+      return await new Promise((resolve, reject) => {
+        pipeline.on('error', (error) => {
+          reject(error);
+        });
+
+        asm.on('done', (asm) => {
+
+          const log : sarif.Log = {
+            version: '2.1.0', 
+            runs: [
+              { 
+                tool: DUMMY_TOOL, 
+                results: asm.current ?? []
+              }
+            ]
+          };
+
+          resolve(log);
+        });
+      });
+    } catch (err) {
+      throw new Error(`Parsing output of interpretation failed: ${err.stderr || err}`);
+    }
+  }
+
   /**
    * Gets the metadata for a query.
    * @param queryPath The path to the query.
@@ -682,22 +728,7 @@ export class CodeQLCliServer implements Disposable {
 
   async interpretBqrs(metadata: QueryMetadata, resultsPath: string, interpretedResultsPath: string, sourceInfo?: SourceInfo): Promise<sarif.Log> {
     await this.runInterpretCommand(SARIF_FORMAT, metadata, resultsPath, interpretedResultsPath, sourceInfo);
-
-    let output: string;
-    try {
-      output = await fs.readFile(interpretedResultsPath, 'utf8');
-    } catch (e) {
-      const rawMessage = e.stderr || e.message;
-      const errorMessage = rawMessage.startsWith('Cannot create a string')
-        ? `SARIF too large. ${rawMessage}`
-        : rawMessage;
-      throw new Error(`Reading output of interpretation failed: ${errorMessage}`);
-    }
-    try {
-      return JSON.parse(output) as sarif.Log;
-    } catch (err) {
-      throw new Error(`Parsing output of interpretation failed: ${err.stderr || err}`);
-    }
+    return await CodeQLCliServer.parseSarif(interpretedResultsPath);
   }
 
   async generateResultsCsv(metadata: QueryMetadata, resultsPath: string, csvPath: string, sourceInfo?: SourceInfo): Promise<void> {
@@ -1143,7 +1174,7 @@ export class CliVersionConstraint {
 
   /**
    * CLI version where database registration was introduced
-   */
+  */
   public static CLI_VERSION_WITH_DB_REGISTRATION = new SemVer('2.4.1');
 
   /**

extensions/ql-vscode/src/vscode-tests/no-workspace/cli.test.ts

@@ -0,0 +1,23 @@
+import * as chai from 'chai';
+import * as chaiAsPromised from 'chai-as-promised';
+
+import { CodeQLCliServer } from '../../cli';
+
+chai.use(chaiAsPromised);
+const expect = chai.expect;
+
+describe.only('cliServerTests', function() {
+
+  it('should parse a valid SARIF file', async () => {
+    const result = await CodeQLCliServer.parseSarif(__dirname + '/data/sarif/validSarif.sarif');
+    expect(result.version).to.exist;
+    expect(result.runs).to.exist;
+    expect(result.runs[0].tool).to.exist;
+    expect(result.runs[0].tool.driver).to.exist;
+  });
+
+  it('should return an empty array if there are no results', async () => {
+    const result = await CodeQLCliServer.parseSarif(__dirname + '/data/sarif/emptyResultsSarif.sarif');
+    expect(result.runs[0].results).to.be.empty;
+  });
+});

extensions/ql-vscode/src/vscode-tests/no-workspace/data/sarif/emptyResultsSarif.sarif

@@ -0,0 +1,34 @@
+{
+  "version": "2.1.0",
+  "$schema": "http://json.schemastore.org/sarif-2.1.0-rtm.4",
+  "runs": [
+    {
+      "tool": {
+        "driver": {
+          "name": "ESLint",
+          "informationUri": "https://eslint.org",
+          "rules": [
+            {
+              "id": "no-unused-vars",
+              "shortDescription": {
+                "text": "disallow unused variables"
+              },
+              "helpUri": "https://eslint.org/docs/rules/no-unused-vars",
+              "properties": {
+                "category": "Variables"
+              }
+            }
+          ]
+        }
+      },
+      "artifacts": [
+        {
+          "location": {
+            "uri": "file:///C:/dev/sarif/sarif-tutorials/samples/Introduction/simple-example.js"
+          }
+        }
+      ],
+      "results": []
+    }
+  ]
+}

extensions/ql-vscode/src/vscode-tests/no-workspace/data/sarif/invalidSarif.sarif

@@ -0,0 +1,33 @@
+{
+  "version": "2.1.0",
+  "$schema": "http://json.schemastore.org/sarif-2.1.0-rtm.4",
+  "runs": [
+    {
+      "tool": {
+        "driver": {
+          "name": "ESLint",
+          "informationUri": "https://eslint.org",
+          "rules": [
+            {
+              "id": "no-unused-vars",
+              "shortDescription": {
+                "text": "disallow unused variables"
+              },
+              "helpUri": "https://eslint.org/docs/rules/no-unused-vars",
+              "properties": {
+                "category": "Variables"
+              }
+            }
+          ]
+        }
+      },
+      "artifacts": [
+        {
+          "location": {
+            "uri": "file:///C:/dev/sarif/sarif-tutorials/samples/Introduction/simple-example.js"
+          }
+        }
+      ]
+    }
+  ]
+}