From 6da1f936ddd0ac8fbea0d52cfef1f211aa14200c Mon Sep 17 00:00:00 2001 From: Nora Date: Fri, 2 Jun 2023 12:26:45 +0000 Subject: [PATCH 1/6] Add no-language option --- .../ql-vscode/src/databases/ui/db-panel.ts | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/extensions/ql-vscode/src/databases/ui/db-panel.ts b/extensions/ql-vscode/src/databases/ui/db-panel.ts index 9172b28220b..24019f5f674 100644 --- a/extensions/ql-vscode/src/databases/ui/db-panel.ts +++ b/extensions/ql-vscode/src/databases/ui/db-panel.ts @@ -351,13 +351,19 @@ export class DbPanel extends DisposableObject { const listName = treeViewItem.dbItem.listName; - const languageQuickPickItems: CodeSearchQuickPickItem[] = Object.values( - QueryLanguage, - ).map((language) => ({ - label: language.toString(), - alwaysShow: true, - language: language.toString(), - })); + const languageQuickPickItems: CodeSearchQuickPickItem[] = [ + { + label: "No specific language", + alwaysShow: true, + language: "", + }, + ].concat( + Object.values(QueryLanguage).map((language) => ({ + label: language.toString(), + alwaysShow: true, + language: language.toString(), + })), + ); const codeSearchLanguage = await window.showQuickPick( @@ -372,6 +378,10 @@ export class DbPanel extends DisposableObject { return; } + const languagePrompt = codeSearchLanguage.language + ? `language:${codeSearchLanguage.language}` + : ""; + const codeSearchQuery = await window.showInputBox({ title: "GitHub Code Search", prompt: @@ -393,7 +403,7 @@ export class DbPanel extends DisposableObject { const repositories = await getCodeSearchRepositories( this.app.credentials, - `${codeSearchQuery} language:${codeSearchLanguage.language}`, + `${codeSearchQuery} ${languagePrompt}`, progress, token, ); From 5467c50395b12c293aa8d73ca9dfe0233e908527 Mon Sep 17 00:00:00 2001 From: Nora Date: Fri, 2 Jun 2023 12:27:26 +0000 Subject: [PATCH 2/6] Implement throttling --- .../variant-analysis/gh-api/gh-api-client.ts | 38 +++++++++++++++++-- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts b/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts index 0e8d68d1f7a..0e04cab6851 100644 --- a/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts +++ b/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts @@ -9,6 +9,8 @@ import { import { Repository } from "./repository"; import { Progress } from "vscode"; import { CancellationToken } from "vscode-jsonrpc"; +import { throttling } from "@octokit/plugin-throttling"; +import { Octokit } from "@octokit/rest"; export async function getCodeSearchRepositories( credentials: Credentials, @@ -20,18 +22,46 @@ export async function getCodeSearchRepositories( token: CancellationToken, ): Promise { let nwos: string[] = []; - const octokit = await credentials.getOctokit(); + const MyOctokit = Octokit.plugin(throttling); + const auth = await credentials.getAccessToken(); + + const octokit = new MyOctokit({ + auth, + throttle: { + onRateLimit: ( + retryAfter: number, + options: any, + octokit: Octokit, + ): boolean => { + octokit.log.warn( + `Request quota exhausted for request ${options.method} ${options.url}. Retrying after ${retryAfter} seconds!`, + ); + + return true; + }, + onSecondaryRateLimit: ( + _retryAfter: number, + options: any, + octokit: Octokit, + ): void => { + octokit.log.warn( + `SecondaryRateLimit detected for request ${options.method} ${options.url}`, + ); + }, + }, + }); + for await (const response of octokit.paginate.iterator( - octokit.rest.search.repos, + octokit.rest.search.code, { q: query, per_page: 100, }, )) { - nwos.push(...response.data.map((item) => item.full_name)); + nwos.push(...response.data.map((item) => item.repository.full_name)); // calculate progress bar: 80% of the progress bar is used for the code search const totalNumberOfRequests = Math.ceil(response.data.total_count / 100); - // Since we have a maximum 10 of requests, we use a fixed increment whenever the totalNumberOfRequests is greater than 10 + // Since we have a maximum of 1000 responses of the api, we can use a fixed increment whenever the totalNumberOfRequests would be greater than 10 const increment = totalNumberOfRequests < 10 ? 80 / totalNumberOfRequests : 8; progress.report({ increment }); From fef28806b157889ebbf2a74cf4b7fbea8a416b57 Mon Sep 17 00:00:00 2001 From: Nora Date: Fri, 2 Jun 2023 12:59:37 +0000 Subject: [PATCH 3/6] Use showAndLogWarning to surface rateLimit --- .../variant-analysis/gh-api/gh-api-client.ts | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts b/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts index 0e04cab6851..876e7715c64 100644 --- a/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts +++ b/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts @@ -11,6 +11,7 @@ import { Progress } from "vscode"; import { CancellationToken } from "vscode-jsonrpc"; import { throttling } from "@octokit/plugin-throttling"; import { Octokit } from "@octokit/rest"; +import { showAndLogWarningMessage } from "../../helpers"; export async function getCodeSearchRepositories( credentials: Credentials, @@ -28,23 +29,15 @@ export async function getCodeSearchRepositories( const octokit = new MyOctokit({ auth, throttle: { - onRateLimit: ( - retryAfter: number, - options: any, - octokit: Octokit, - ): boolean => { - octokit.log.warn( + onRateLimit: (retryAfter: number, options: any): boolean => { + void showAndLogWarningMessage( `Request quota exhausted for request ${options.method} ${options.url}. Retrying after ${retryAfter} seconds!`, ); return true; }, - onSecondaryRateLimit: ( - _retryAfter: number, - options: any, - octokit: Octokit, - ): void => { - octokit.log.warn( + onSecondaryRateLimit: (_retryAfter: number, options: any): void => { + void showAndLogWarningMessage( `SecondaryRateLimit detected for request ${options.method} ${options.url}`, ); }, From b47006129d72880b52fa9656dd8986eb370f1110 Mon Sep 17 00:00:00 2001 From: Nora Date: Fri, 2 Jun 2023 15:34:29 +0000 Subject: [PATCH 4/6] Move octokit initialization to db panel to send log messages to the user --- .../ql-vscode/src/databases/ui/db-panel.ts | 32 ++++++++++++++++++- .../variant-analysis/gh-api/gh-api-client.ts | 24 +------------- 2 files changed, 32 insertions(+), 24 deletions(-) diff --git a/extensions/ql-vscode/src/databases/ui/db-panel.ts b/extensions/ql-vscode/src/databases/ui/db-panel.ts index 24019f5f674..9acc8f0817a 100644 --- a/extensions/ql-vscode/src/databases/ui/db-panel.ts +++ b/extensions/ql-vscode/src/databases/ui/db-panel.ts @@ -17,6 +17,7 @@ import { import { showAndLogErrorMessage, showAndLogInformationMessage, + showAndLogWarningMessage, } from "../../helpers"; import { DisposableObject } from "../../pure/disposable-object"; import { @@ -38,6 +39,9 @@ import { DatabasePanelCommands } from "../../common/commands"; import { App } from "../../common/app"; import { getCodeSearchRepositories } from "../../variant-analysis/gh-api/gh-api-client"; import { QueryLanguage } from "../../common/query-language"; +import { retry } from "@octokit/plugin-retry"; +import { throttling } from "@octokit/plugin-throttling"; +import { Octokit } from "@octokit/rest"; export interface RemoteDatabaseQuickPickItem extends QuickPickItem { remoteDatabaseKind: string; @@ -402,10 +406,10 @@ export class DbPanel extends DisposableObject { progress.report({ increment: 10 }); const repositories = await getCodeSearchRepositories( - this.app.credentials, `${codeSearchQuery} ${languagePrompt}`, progress, token, + await this.getOctokitForSearch(), ); token.onCancellationRequested(() => { @@ -499,4 +503,30 @@ export class DbPanel extends DisposableObject { ); } } + + // since we don't have access to the extensions log methods we initialize octokit here + private async getOctokitForSearch(): Promise { + const MyOctokit = Octokit.plugin(throttling); + const auth = await this.app.credentials.getAccessToken(); + + const octokit = new MyOctokit({ + auth, + retry, + throttle: { + onRateLimit: (retryAfter: number, options: any): boolean => { + void showAndLogWarningMessage( + `Request quota exhausted for request ${options.method} ${options.url}. Retrying after ${retryAfter} seconds!`, + ); + + return true; + }, + onSecondaryRateLimit: (_retryAfter: number, options: any): void => { + void showAndLogWarningMessage( + `SecondaryRateLimit detected for request ${options.method} ${options.url}`, + ); + }, + }, + }); + return octokit; + } } diff --git a/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts b/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts index 876e7715c64..01568bcc37b 100644 --- a/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts +++ b/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts @@ -9,40 +9,18 @@ import { import { Repository } from "./repository"; import { Progress } from "vscode"; import { CancellationToken } from "vscode-jsonrpc"; -import { throttling } from "@octokit/plugin-throttling"; import { Octokit } from "@octokit/rest"; -import { showAndLogWarningMessage } from "../../helpers"; export async function getCodeSearchRepositories( - credentials: Credentials, query: string, progress: Progress<{ message?: string | undefined; increment?: number | undefined; }>, token: CancellationToken, + octokit: Octokit, ): Promise { let nwos: string[] = []; - const MyOctokit = Octokit.plugin(throttling); - const auth = await credentials.getAccessToken(); - - const octokit = new MyOctokit({ - auth, - throttle: { - onRateLimit: (retryAfter: number, options: any): boolean => { - void showAndLogWarningMessage( - `Request quota exhausted for request ${options.method} ${options.url}. Retrying after ${retryAfter} seconds!`, - ); - - return true; - }, - onSecondaryRateLimit: (_retryAfter: number, options: any): void => { - void showAndLogWarningMessage( - `SecondaryRateLimit detected for request ${options.method} ${options.url}`, - ); - }, - }, - }); for await (const response of octokit.paginate.iterator( octokit.rest.search.code, From 876b92aa9846ca2eca74942e4e6ebe280127f956 Mon Sep 17 00:00:00 2001 From: Nora Date: Mon, 5 Jun 2023 09:27:10 +0000 Subject: [PATCH 5/6] Move code search api call to its own file --- .../src/databases/code-search-api.ts | 70 +++++++++++++++++++ .../ql-vscode/src/databases/ui/db-panel.ts | 34 +-------- .../variant-analysis/gh-api/gh-api-client.ts | 38 ---------- 3 files changed, 72 insertions(+), 70 deletions(-) create mode 100644 extensions/ql-vscode/src/databases/code-search-api.ts diff --git a/extensions/ql-vscode/src/databases/code-search-api.ts b/extensions/ql-vscode/src/databases/code-search-api.ts new file mode 100644 index 00000000000..53ea0bd7245 --- /dev/null +++ b/extensions/ql-vscode/src/databases/code-search-api.ts @@ -0,0 +1,70 @@ +import { retry } from "@octokit/plugin-retry"; +import { throttling } from "@octokit/plugin-throttling"; +import { Octokit } from "@octokit/rest"; +import { Progress, CancellationToken } from "vscode"; +import { showAndLogWarningMessage } from "../helpers"; +import { Credentials } from "../common/authentication"; + +export async function getCodeSearchRepositories( + query: string, + progress: Progress<{ + message?: string | undefined; + increment?: number | undefined; + }>, + token: CancellationToken, + credentials: Credentials, +): Promise { + let nwos: string[] = []; + const octokit = await provideOctokitWithThrottling(credentials); + + for await (const response of octokit.paginate.iterator( + octokit.rest.search.code, + { + q: query, + per_page: 100, + }, + )) { + nwos.push(...response.data.map((item) => item.repository.full_name)); + // calculate progress bar: 80% of the progress bar is used for the code search + const totalNumberOfRequests = Math.ceil(response.data.total_count / 100); + // Since we have a maximum of 1000 responses of the api, we can use a fixed increment whenever the totalNumberOfRequests would be greater than 10 + const increment = + totalNumberOfRequests < 10 ? 80 / totalNumberOfRequests : 8; + progress.report({ increment }); + + if (token.isCancellationRequested) { + nwos = []; + break; + } + } + + return [...new Set(nwos)]; +} + +async function provideOctokitWithThrottling( + credentials: Credentials, +): Promise { + const MyOctokit = Octokit.plugin(throttling); + const auth = await credentials.getAccessToken(); + + const octokit = new MyOctokit({ + auth, + retry, + throttle: { + onRateLimit: (retryAfter: number, options: any): boolean => { + void showAndLogWarningMessage( + `Request quota exhausted for request ${options.method} ${options.url}. Retrying after ${retryAfter} seconds!`, + ); + + return true; + }, + onSecondaryRateLimit: (_retryAfter: number, options: any): void => { + void showAndLogWarningMessage( + `SecondaryRateLimit detected for request ${options.method} ${options.url}`, + ); + }, + }, + }); + + return octokit; +} diff --git a/extensions/ql-vscode/src/databases/ui/db-panel.ts b/extensions/ql-vscode/src/databases/ui/db-panel.ts index 9acc8f0817a..7e64fb4f2a2 100644 --- a/extensions/ql-vscode/src/databases/ui/db-panel.ts +++ b/extensions/ql-vscode/src/databases/ui/db-panel.ts @@ -17,7 +17,6 @@ import { import { showAndLogErrorMessage, showAndLogInformationMessage, - showAndLogWarningMessage, } from "../../helpers"; import { DisposableObject } from "../../pure/disposable-object"; import { @@ -37,11 +36,8 @@ import { getControllerRepo } from "../../variant-analysis/run-remote-query"; import { getErrorMessage } from "../../pure/helpers-pure"; import { DatabasePanelCommands } from "../../common/commands"; import { App } from "../../common/app"; -import { getCodeSearchRepositories } from "../../variant-analysis/gh-api/gh-api-client"; import { QueryLanguage } from "../../common/query-language"; -import { retry } from "@octokit/plugin-retry"; -import { throttling } from "@octokit/plugin-throttling"; -import { Octokit } from "@octokit/rest"; +import { getCodeSearchRepositories } from "../code-search-api"; export interface RemoteDatabaseQuickPickItem extends QuickPickItem { remoteDatabaseKind: string; @@ -409,7 +405,7 @@ export class DbPanel extends DisposableObject { `${codeSearchQuery} ${languagePrompt}`, progress, token, - await this.getOctokitForSearch(), + this.app.credentials, ); token.onCancellationRequested(() => { @@ -503,30 +499,4 @@ export class DbPanel extends DisposableObject { ); } } - - // since we don't have access to the extensions log methods we initialize octokit here - private async getOctokitForSearch(): Promise { - const MyOctokit = Octokit.plugin(throttling); - const auth = await this.app.credentials.getAccessToken(); - - const octokit = new MyOctokit({ - auth, - retry, - throttle: { - onRateLimit: (retryAfter: number, options: any): boolean => { - void showAndLogWarningMessage( - `Request quota exhausted for request ${options.method} ${options.url}. Retrying after ${retryAfter} seconds!`, - ); - - return true; - }, - onSecondaryRateLimit: (_retryAfter: number, options: any): void => { - void showAndLogWarningMessage( - `SecondaryRateLimit detected for request ${options.method} ${options.url}`, - ); - }, - }, - }); - return octokit; - } } diff --git a/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts b/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts index 01568bcc37b..00bbafde926 100644 --- a/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts +++ b/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts @@ -7,44 +7,6 @@ import { VariantAnalysisSubmissionRequest, } from "./variant-analysis"; import { Repository } from "./repository"; -import { Progress } from "vscode"; -import { CancellationToken } from "vscode-jsonrpc"; -import { Octokit } from "@octokit/rest"; - -export async function getCodeSearchRepositories( - query: string, - progress: Progress<{ - message?: string | undefined; - increment?: number | undefined; - }>, - token: CancellationToken, - octokit: Octokit, -): Promise { - let nwos: string[] = []; - - for await (const response of octokit.paginate.iterator( - octokit.rest.search.code, - { - q: query, - per_page: 100, - }, - )) { - nwos.push(...response.data.map((item) => item.repository.full_name)); - // calculate progress bar: 80% of the progress bar is used for the code search - const totalNumberOfRequests = Math.ceil(response.data.total_count / 100); - // Since we have a maximum of 1000 responses of the api, we can use a fixed increment whenever the totalNumberOfRequests would be greater than 10 - const increment = - totalNumberOfRequests < 10 ? 80 / totalNumberOfRequests : 8; - progress.report({ increment }); - - if (token.isCancellationRequested) { - nwos = []; - break; - } - } - - return [...new Set(nwos)]; -} export async function submitVariantAnalysis( credentials: Credentials, From 945594d47a4131b3a01be5eb5561719dd6309a8d Mon Sep 17 00:00:00 2001 From: Nora Date: Mon, 5 Jun 2023 15:42:00 +0000 Subject: [PATCH 6/6] Adjust error response wording --- extensions/ql-vscode/src/databases/code-search-api.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions/ql-vscode/src/databases/code-search-api.ts b/extensions/ql-vscode/src/databases/code-search-api.ts index 53ea0bd7245..6478470f1f0 100644 --- a/extensions/ql-vscode/src/databases/code-search-api.ts +++ b/extensions/ql-vscode/src/databases/code-search-api.ts @@ -53,14 +53,14 @@ async function provideOctokitWithThrottling( throttle: { onRateLimit: (retryAfter: number, options: any): boolean => { void showAndLogWarningMessage( - `Request quota exhausted for request ${options.method} ${options.url}. Retrying after ${retryAfter} seconds!`, + `Rate Limit detected for request ${options.method} ${options.url}. Retrying after ${retryAfter} seconds!`, ); return true; }, onSecondaryRateLimit: (_retryAfter: number, options: any): void => { void showAndLogWarningMessage( - `SecondaryRateLimit detected for request ${options.method} ${options.url}`, + `Secondary Rate Limit detected for request ${options.method} ${options.url}`, ); }, },