Remove canary requirement for GitHub database download

extensions/ql-vscode/CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## [UNRELEASED]
 
+- Add ability for users to download databases directly from GitHub. [#1485](https://github.com/github/vscode-codeql/pull/1485)
+
 ## 1.6.11 - 25 August 2022
 
 No user facing changes.

extensions/ql-vscode/package.json

@@ -664,7 +664,7 @@
         },
         {
           "command": "codeQLDatabases.chooseDatabaseGithub",
-          "when": "config.codeQL.canary && view == codeQLDatabases",
+          "when": "view == codeQLDatabases",
           "group": "navigation"
         },
         {
@@ -926,10 +926,6 @@
           "command": "codeQL.viewCfg",
           "when": "resourceScheme == codeql-zip-archive && config.codeQL.canary"
         },
-        {
-          "command": "codeQL.chooseDatabaseGithub",
-          "when": "config.codeQL.canary"
-        },
         {
           "command": "codeQLDatabases.setCurrentDatabase",
           "when": "false"
@@ -1175,7 +1171,7 @@
       },
       {
         "view": "codeQLDatabases",
-        "contents": "Add a CodeQL database:\n[From a folder](command:codeQLDatabases.chooseDatabaseFolder)\n[From an archive](command:codeQLDatabases.chooseDatabaseArchive)\n[From a URL (as a zip file)](command:codeQLDatabases.chooseDatabaseInternet)\n[From LGTM](command:codeQLDatabases.chooseDatabaseLgtm)"
+        "contents": "Add a CodeQL database:\n[From a folder](command:codeQLDatabases.chooseDatabaseFolder)\n[From an archive](command:codeQLDatabases.chooseDatabaseArchive)\n[From a URL (as a zip file)](command:codeQLDatabases.chooseDatabaseInternet)\n[From GitHub](command:codeQLDatabases.chooseDatabaseGithub)\n[From LGTM](command:codeQLDatabases.chooseDatabaseLgtm)"
       },
       {
         "view": "codeQLEvalLogViewer",

extensions/ql-vscode/src/authentication.ts

@@ -76,16 +76,27 @@ export class Credentials {
     }));
   }
 
-  async getOctokit(): Promise<Octokit.Octokit> {
+  /**
+   * Creates or returns an instance of Octokit.
+   *
+   * @param requireAuthentication Whether the Octokit instance needs to be authenticated as user.
+   * @returns An instance of Octokit.
+   */
+  async getOctokit(requireAuthentication = true): Promise<Octokit.Octokit> {
     if (this.octokit) {
       return this.octokit;
     }
 
-    this.octokit = await this.createOctokit(true);
-    // octokit shouldn't be undefined, since we've set "createIfNone: true".
-    // The following block is mainly here to prevent a compiler error.
+    this.octokit = await this.createOctokit(requireAuthentication);
+
     if (!this.octokit) {
-      throw new Error('Did not initialize Octokit.');
+      if (requireAuthentication) {
+        throw new Error('Did not initialize Octokit.');
+      }
+
+      // We don't want to set this in this.octokit because that would prevent
+      // authenticating when requireCredentials is true.
+      return new Octokit.Octokit({ retry });
     }
     return this.octokit;
   }

extensions/ql-vscode/src/databaseFetcher.ts

@@ -10,6 +10,8 @@ import {
 import { CodeQLCliServer } from './cli';
 import * as fs from 'fs-extra';
 import * as path from 'path';
+import * as Octokit from '@octokit/rest';
+import { retry } from '@octokit/plugin-retry';
 
 import { DatabaseManager, DatabaseItem } from './databases';
 import {
@@ -76,7 +78,7 @@ export async function promptImportInternetDatabase(
 export async function promptImportGithubDatabase(
   databaseManager: DatabaseManager,
   storagePath: string,
-  credentials: Credentials,
+  credentials: Credentials | undefined,
   progress: ProgressCallback,
   token: CancellationToken,
   cli?: CodeQLCliServer
@@ -99,14 +101,15 @@ export async function promptImportGithubDatabase(
     throw new Error(`Invalid GitHub repository: ${githubRepo}`);
   }
 
-  const result = await convertGithubNwoToDatabaseUrl(githubRepo, credentials, progress);
+  const octokit = credentials ? await credentials.getOctokit(true) : new Octokit.Octokit({ retry });
+
+  const result = await convertGithubNwoToDatabaseUrl(githubRepo, octokit, progress);
   if (!result) {
     return;
   }
 
   const { databaseUrl, name, owner } = result;
 
-  const octokit = await credentials.getOctokit();
   /**
    * The 'token' property of the token object returned by `octokit.auth()`.
    * The object is undocumented, but looks something like this:
@@ -118,14 +121,9 @@ export async function promptImportGithubDatabase(
    * We only need the actual token string.
    */
   const octokitToken = (await octokit.auth() as { token: string })?.token;
-  if (!octokitToken) {
-    // Just print a generic error message for now. Ideally we could show more debugging info, like the
-    // octokit object, but that would expose a user token.
-    throw new Error('Unable to get GitHub token.');
-  }
   const item = await databaseArchiveFetcher(
     databaseUrl,
-    { 'Accept': 'application/zip', 'Authorization': `Bearer ${octokitToken}` },
+    { 'Accept': 'application/zip', 'Authorization': octokitToken ? `Bearer ${octokitToken}` : '' },
     databaseManager,
     storagePath,
     `${owner}/${name}`,
@@ -523,7 +521,7 @@ function convertGitHubUrlToNwo(githubUrl: string): string | undefined {
 
 export async function convertGithubNwoToDatabaseUrl(
   githubRepo: string,
-  credentials: Credentials,
+  octokit: Octokit.Octokit,
   progress: ProgressCallback): Promise<{
     databaseUrl: string,
     owner: string,
@@ -533,7 +531,6 @@ export async function convertGithubNwoToDatabaseUrl(
     const nwo = convertGitHubUrlToNwo(githubRepo) || githubRepo;
     const [owner, repo] = nwo.split('/');
 
-    const octokit = await credentials.getOctokit();
     const response = await octokit.request('GET /repos/:owner/:repo/code-scanning/codeql/databases', { owner, repo });
 
     const languages = response.data.map((db: any) => db.language);

extensions/ql-vscode/src/databases-ui.ts

@@ -40,6 +40,7 @@ import {
 import { CancellationToken } from 'vscode';
 import { asyncFilter, getErrorMessage } from './pure/helpers-pure';
 import { Credentials } from './authentication';
+import { isCanary } from './config';
 
 type ThemableIconPath = { light: string; dark: string } | string;
 
@@ -301,7 +302,7 @@ export class DatabaseUI extends DisposableObject {
           progress: ProgressCallback,
           token: CancellationToken
         ) => {
-          const credentials = await this.getCredentials();
+          const credentials = isCanary() ? await this.getCredentials() : undefined;
           await this.handleChooseDatabaseGithub(credentials, progress, token);
         },
         {
@@ -480,7 +481,7 @@ export class DatabaseUI extends DisposableObject {
   };
 
   handleChooseDatabaseGithub = async (
-    credentials: Credentials,
+    credentials: Credentials | undefined,
     progress: ProgressCallback,
     token: CancellationToken
   ): Promise<DatabaseItem | undefined> => {

extensions/ql-vscode/src/extension.ts

@@ -970,7 +970,7 @@ async function activateWithInstalledDistribution(
       progress: ProgressCallback,
       token: CancellationToken
     ) => {
-      const credentials = await Credentials.initialize(ctx);
+      const credentials = isCanary() ? await Credentials.initialize(ctx) : undefined;
       await databaseUI.handleChooseDatabaseGithub(credentials, progress, token);
     },
       {
@@ -1018,19 +1018,16 @@ async function activateWithInstalledDistribution(
     }
   };
 
-  // The "authenticateToGitHub" command is internal-only.
   ctx.subscriptions.push(
     commandRunner('codeQL.authenticateToGitHub', async () => {
-      if (isCanary()) {
-        /**
-         * Credentials for authenticating to GitHub.
-         * These are used when making API calls.
-         */
-        const credentials = await Credentials.initialize(ctx);
-        const octokit = await credentials.getOctokit();
-        const userInfo = await octokit.users.getAuthenticated();
-        void showAndLogInformationMessage(`Authenticated to GitHub as user: ${userInfo.data.login}`);
-      }
+      /**
+       * Credentials for authenticating to GitHub.
+       * These are used when making API calls.
+       */
+      const credentials = await Credentials.initialize(ctx);
+      const octokit = await credentials.getOctokit();
+      const userInfo = await octokit.users.getAuthenticated();
+      void showAndLogInformationMessage(`Authenticated to GitHub as user: ${userInfo.data.login}`);
     }));
 
   ctx.subscriptions.push(

extensions/ql-vscode/src/vscode-tests/no-workspace/databaseFetcher.test.ts

@@ -6,15 +6,14 @@ import { expect } from 'chai';
 import { window } from 'vscode';
 
 import {
+  convertGithubNwoToDatabaseUrl,
   convertLgtmUrlToDatabaseUrl,
   looksLikeLgtmUrl,
   findDirWithFile,
   looksLikeGithubRepo,
 } from '../../databaseFetcher';
 import { ProgressCallback } from '../../commandRunner';
-import * as pq from 'proxyquire';
-
-const proxyquire = pq.noPreserveCache();
+import * as Octokit from '@octokit/rest';
 
 describe('databaseFetcher', function() {
   // These tests make API calls and may need extra time to complete.
@@ -25,20 +24,16 @@ describe('databaseFetcher', function() {
     let quickPickSpy: sinon.SinonStub;
     let progressSpy: ProgressCallback;
     let mockRequest: sinon.SinonStub;
-    let mod: any;
-
-    const credentials = getMockCredentials(0);
+    let octokit: Octokit.Octokit;
 
     beforeEach(() => {
       sandbox = sinon.createSandbox();
       quickPickSpy = sandbox.stub(window, 'showQuickPick');
       progressSpy = sandbox.spy();
       mockRequest = sandbox.stub();
-      mod = proxyquire('../../databaseFetcher', {
-        './authentication': {
-          Credentials: credentials,
-        },
-      });
+      octokit = ({
+        request: mockRequest,
+      }) as unknown as Octokit.Octokit;
     });
 
     afterEach(() => {
@@ -90,11 +85,17 @@ describe('databaseFetcher', function() {
       mockRequest.resolves(mockApiResponse);
       quickPickSpy.resolves('javascript');
       const githubRepo = 'github/codeql';
-      const { databaseUrl, name, owner } = await mod.convertGithubNwoToDatabaseUrl(
+      const result = await convertGithubNwoToDatabaseUrl(
         githubRepo,
-        credentials,
+        octokit,
         progressSpy
       );
+      expect(result).not.to.be.undefined;
+      if (result === undefined) {
+        return;
+      }
+
+      const { databaseUrl, name, owner } = result;
 
       expect(databaseUrl).to.equal(
         'https://api.github.com/repos/github/codeql/code-scanning/codeql/databases/javascript'
@@ -119,7 +120,7 @@ describe('databaseFetcher', function() {
       mockRequest.resolves(mockApiResponse);
       const githubRepo = 'foo/bar-not-real';
       await expect(
-        mod.convertGithubNwoToDatabaseUrl(githubRepo, credentials, progressSpy)
+        convertGithubNwoToDatabaseUrl(githubRepo, octokit, progressSpy)
       ).to.be.rejectedWith(/Unable to get database/);
       expect(progressSpy).to.have.callCount(0);
     });
@@ -133,19 +134,10 @@ describe('databaseFetcher', function() {
       mockRequest.resolves(mockApiResponse);
       const githubRepo = 'foo/bar-with-no-dbs';
       await expect(
-        mod.convertGithubNwoToDatabaseUrl(githubRepo, credentials, progressSpy)
+        convertGithubNwoToDatabaseUrl(githubRepo, octokit, progressSpy)
       ).to.be.rejectedWith(/Unable to get database/);
       expect(progressSpy).to.have.been.calledOnce;
     });
-
-    function getMockCredentials(response: any) {
-      mockRequest = sinon.stub().resolves(response);
-      return {
-        getOctokit: () => ({
-          request: mockRequest,
-        }),
-      };
-    }
   });
 
   describe('convertLgtmUrlToDatabaseUrl', () => {