Merge branch 'main' into case-insensitive-slugs

Author: mgsium

extensions/ql-vscode/CHANGELOG.md

@@ -2,14 +2,18 @@
 
 ## [UNRELEASED]
 
+- Fix a bug that shows 'Set current database' when hovering over the currently selected database in the databases view. [#976](https://github.com/github/vscode-codeql/pull/976)
+- Fix a bug with importing large databases. Databases over 4GB can now be imported directly from LGTM or from a zip file. This functionality is only available when using CodeQL CLI version 2.6.0 or later. [#971](https://github.com/github/vscode-codeql/pull/971)
+- Replace certain control codes (`U+0000` - `U+001F`) with their corresponding control labels (`U+2400` - `U+241F`)  in the results view. [#963](https://github.com/github/vscode-codeql/pull/963)
+- Make project slug for GitHub repositories case-insensitive when adding a CodeQL database from LGTM. [#978](https://github.com/github/vscode-codeql/pull/961)
+
+## 1.5.6 - 07 October 2021
+
 - Add progress messages to LGTM download option. This makes the two-step process (selecting a project, then selecting a language) more clear. [#960](https://github.com/github/vscode-codeql/pull/960)
 - Remove line about selecting a language from the dropdown when downloading database from LGTM. This makes the download progress visible when the popup is not expanded. [#957](https://github.com/github/vscode-codeql/pull/957)
-- Fixed a bug where copying the version information fails when a CodeQL CLI cannot be found. [#958](https://github.com/github/vscode-codeql/pull/958)
 - Fix a bug where copying the version information fails when a CodeQL CLI cannot be found. [#958](https://github.com/github/vscode-codeql/pull/958)
 - Avoid a race condition when deleting databases that can cause occasional errors. [#959](https://github.com/github/vscode-codeql/pull/959)
 - Update CodeQL logos. [#965](https://github.com/github/vscode-codeql/pull/965)
-- Fixed a bug where copying the version information fails when a CodeQL CLI cannot be found. [#958](https://github.com/github/vscode-codeql/pull/958)
-- Make project slug for GitHub repositories case-insensitive when adding a CodeQL database from LGTM. [#961](https://github.com/github/vscode-codeql/pull/961)
 
 ## 1.5.5 - 08 September 2021
 

extensions/ql-vscode/media/VS-marketplace-CodeQL-icon.png

@@ -4,7 +4,7 @@
   "description": "CodeQL for Visual Studio Code",
   "author": "GitHub",
   "private": true,
-  "version": "1.5.6",
+  "version": "1.5.7",
   "publisher": "GitHub",
   "license": "MIT",
   "icon": "media/VS-marketplace-CodeQL-icon.png",
@@ -604,7 +604,7 @@
         {
           "command": "codeQLDatabases.setCurrentDatabase",
           "group": "inline",
-          "when": "view == codeQLDatabases"
+          "when": "view == codeQLDatabases && viewItem != currentDatabase"
         },
         {
           "command": "codeQLDatabases.removeDatabase",

extensions/ql-vscode/media/logo.svg

@@ -600,6 +600,15 @@ export class CodeQLCliServer implements Disposable {
     return await this.runJsonCodeQlCliCommand<BQRSInfo>(['bqrs', 'info'], subcommandArgs, 'Reading bqrs header');
   }
 
+  async databaseUnbundle(archivePath: string, target: string, name?: string): Promise<string> {
+    const subcommandArgs = [];
+    if (target) subcommandArgs.push('--target', target);
+    if (name) subcommandArgs.push('--name', name);
+    subcommandArgs.push(archivePath);
+ 
+    return await this.runCodeQlCliCommand(['database', 'unbundle'], subcommandArgs, `Extracting ${archivePath} to directory ${target}`);
+  }
+
   /**
   * Gets the results from a bqrs.
   * @param bqrsPath The path to the bqrs.
@@ -1074,6 +1083,11 @@ export class CliVersionConstraint {
    */
   public static CLI_VERSION_WITH_ALLOW_LIBRARY_PACKS_IN_RESOLVE_QUERIES = new SemVer('2.6.1');
 
+  /**
+   * CLI version where the `database unbundle` subcommand was introduced.
+   */
+  public static CLI_VERSION_WITH_DATABASE_UNBUNDLE = new SemVer('2.6.0');
+
   constructor(private readonly cli: CodeQLCliServer) {
     /**/
   }
@@ -1105,4 +1119,9 @@ export class CliVersionConstraint {
   async supportsDatabaseRegistration() {
     return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_DB_REGISTRATION);
   }
+
+  async supportsDatabaseUnbundle() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_DATABASE_UNBUNDLE);
+  }
+
 }

extensions/ql-vscode/package-lock.json

@@ -1,12 +1,13 @@
 import fetch, { Response } from 'node-fetch';
-import * as unzipper from 'unzipper';
 import { zip } from 'zip-a-folder';
+import * as unzipper from 'unzipper';
 import {
   Uri,
   CancellationToken,
   commands,
   window,
 } from 'vscode';
+import { CodeQLCliServer } from './cli';
 import * as fs from 'fs-extra';
 import * as path from 'path';
 
@@ -32,6 +33,7 @@ export async function promptImportInternetDatabase(
   storagePath: string,
   progress: ProgressCallback,
   token: CancellationToken,
+  cli?: CodeQLCliServer
 ): Promise<DatabaseItem | undefined> {
   const databaseUrl = await window.showInputBox({
     prompt: 'Enter URL of zipfile of database to download',
@@ -47,7 +49,8 @@ export async function promptImportInternetDatabase(
     databaseManager,
     storagePath,
     progress,
-    token
+    token,
+    cli
   );
 
   if (item) {
@@ -70,7 +73,8 @@ export async function promptImportLgtmDatabase(
   databaseManager: DatabaseManager,
   storagePath: string,
   progress: ProgressCallback,
-  token: CancellationToken
+  token: CancellationToken,
+  cli?: CodeQLCliServer
 ): Promise<DatabaseItem | undefined> {
   progress({
     message: 'Choose project',
@@ -93,7 +97,8 @@ export async function promptImportLgtmDatabase(
         databaseManager,
         storagePath,
         progress,
-        token
+        token,
+        cli
       );
       if (item) {
         await commands.executeCommand('codeQLDatabases.focus');
@@ -130,14 +135,16 @@ export async function importArchiveDatabase(
   storagePath: string,
   progress: ProgressCallback,
   token: CancellationToken,
+  cli?: CodeQLCliServer,
 ): Promise<DatabaseItem | undefined> {
   try {
     const item = await databaseArchiveFetcher(
       databaseUrl,
       databaseManager,
       storagePath,
       progress,
-      token
+      token,
+      cli
     );
     if (item) {
       await commands.executeCommand('codeQLDatabases.focus');
@@ -169,7 +176,8 @@ async function databaseArchiveFetcher(
   databaseManager: DatabaseManager,
   storagePath: string,
   progress: ProgressCallback,
-  token: CancellationToken
+  token: CancellationToken,
+  cli?: CodeQLCliServer,
 ): Promise<DatabaseItem> {
   progress({
     message: 'Getting database',
@@ -183,9 +191,9 @@ async function databaseArchiveFetcher(
   const unzipPath = await getStorageFolder(storagePath, databaseUrl);
 
   if (isFile(databaseUrl)) {
-    await readAndUnzip(databaseUrl, unzipPath, progress);
+    await readAndUnzip(databaseUrl, unzipPath, cli, progress);
   } else {
-    await fetchAndUnzip(databaseUrl, unzipPath, progress);
+    await fetchAndUnzip(databaseUrl, unzipPath, cli, progress);
   }
 
   progress({
@@ -259,6 +267,7 @@ function validateHttpsUrl(databaseUrl: string) {
 async function readAndUnzip(
   zipUrl: string,
   unzipPath: string,
+  cli?: CodeQLCliServer,
   progress?: ProgressCallback
 ) {
   // TODO: Providing progress as the file is unzipped is currently blocked
@@ -269,16 +278,22 @@ async function readAndUnzip(
     step: 9,
     message: `Unzipping into ${path.basename(unzipPath)}`
   });
-  // Must get the zip central directory since streaming the
-  // zip contents may not have correct local file headers.
-  // Instead, we can only rely on the central directory.
-  const directory = await unzipper.Open.file(zipFile);
-  await directory.extract({ path: unzipPath });
+  if (cli && await cli.cliConstraints.supportsDatabaseUnbundle()) {
+    // Use the `database unbundle` command if the installed cli version supports it
+    await cli.databaseUnbundle(zipFile, unzipPath);
+  } else {
+    // Must get the zip central directory since streaming the
+    // zip contents may not have correct local file headers.
+    // Instead, we can only rely on the central directory.
+    const directory = await unzipper.Open.file(zipFile);
+    await directory.extract({ path: unzipPath });
+  }
 }
 
 async function fetchAndUnzip(
   databaseUrl: string,
   unzipPath: string,
+  cli?: CodeQLCliServer,
   progress?: ProgressCallback
 ) {
   // Although it is possible to download and stream directly to an unzipped directory,
@@ -308,7 +323,8 @@ async function fetchAndUnzip(
       .on('error', reject)
   );
 
-  await readAndUnzip(Uri.file(archivePath).toString(true), unzipPath, progress);
+  await readAndUnzip(Uri.file(archivePath).toString(true), unzipPath, cli, progress);
+
 
   // remove archivePath eagerly since these archives can be large.
   await fs.remove(archivePath);

extensions/ql-vscode/package.json

@@ -135,6 +135,7 @@ class DatabaseTreeDataProvider extends DisposableObject
         this.extensionPath,
         SELECTED_DATABASE_ICON
       );
+      item.contextValue = 'currentDatabase';
     } else if (element.error !== undefined) {
       item.iconPath = joinThemableIconPath(
         this.extensionPath,
@@ -451,14 +452,13 @@ export class DatabaseUI extends DisposableObject {
   handleChooseDatabaseInternet = async (
     progress: ProgressCallback,
     token: CancellationToken
-  ): Promise<
-    DatabaseItem | undefined
-  > => {
+  ): Promise<DatabaseItem | undefined> => {
     return await promptImportInternetDatabase(
       this.databaseManager,
       this.storagePath,
       progress,
-      token
+      token,
+      this.queryServer?.cliServer
     );
   };
 
@@ -470,7 +470,8 @@ export class DatabaseUI extends DisposableObject {
       this.databaseManager,
       this.storagePath,
       progress,
-      token
+      token,
+      this.queryServer?.cliServer
     );
   };
 
@@ -580,7 +581,8 @@ export class DatabaseUI extends DisposableObject {
           this.databaseManager,
           this.storagePath,
           progress,
-          token
+          token,
+          this.queryServer?.cliServer
         );
       } else {
         await this.setCurrentDatabase(progress, token, uri);
@@ -696,7 +698,6 @@ export class DatabaseUI extends DisposableObject {
     token: CancellationToken,
   ): Promise<DatabaseItem | undefined> {
     const uri = await chooseDatabaseDir(byFolder);
-
     if (!uri) {
       return undefined;
     }
@@ -713,7 +714,8 @@ export class DatabaseUI extends DisposableObject {
         this.databaseManager,
         this.storagePath,
         progress,
-        token
+        token,
+        this.queryServer?.cliServer
       );
     }
   }

extensions/ql-vscode/src/cli.ts

@@ -731,6 +731,8 @@ export class DatabaseManager extends DisposableObject {
       this._currentDatabaseItem = item;
       this.updatePersistedCurrentDatabaseItem();
 
+      await vscode.commands.executeCommand('setContext', 'codeQL.currentDatabaseItem', item?.name);
+      
       this._onDidChangeCurrentDatabaseItem.fire({
         item,
         kind: DatabaseEventKind.Change
@@ -811,6 +813,9 @@ export class DatabaseManager extends DisposableObject {
       vscode.workspace.updateWorkspaceFolders(folderIndex, 1);
     }
 
+    // Remove this database item from the allow-list
+    await this.deregisterDatabase(progress, token, item);
+
     // Delete folder from file system only if it is controlled by the extension
     if (this.isExtensionControlledLocation(item.databaseUri)) {
       void logger.log('Deleting database from filesystem.');
@@ -819,9 +824,6 @@ export class DatabaseManager extends DisposableObject {
         e => void logger.log(`Failed to delete '${item.databaseUri.fsPath}'. Reason: ${e.message}`));
     }
 
-    // Remove this database item from the allow-list
-    await this.deregisterDatabase(progress, token, item);
-
     // note that we use undefined as the item in order to reset the entire tree
     this._onDidChangeDatabaseItem.fire({
       item: undefined,

extensions/ql-vscode/src/databaseFetcher.ts

@@ -9,14 +9,14 @@ interface Props {
 }
 
 export default function RawTableValue(props: Props): JSX.Element {
-  const v = props.value;
+  const rawValue = props.value;
   if (
-    typeof v === 'string'
-    || typeof v === 'number'
-    || typeof v === 'boolean'
+    typeof rawValue === 'string'
+    || typeof rawValue === 'number'
+    || typeof rawValue === 'boolean'
   ) {
-    return <span>{v.toString()}</span>;
+    return <span>{renderLocation(undefined, rawValue.toString())}</span>;
   }
 
-  return renderLocation(v.url, v.label, props.databaseUri);
+  return renderLocation(rawValue.url, rawValue.label, props.databaseUri);
 }