diff --git a/extensions/ql-vscode/src/databases/code-search-api.ts b/extensions/ql-vscode/src/databases/code-search-api.ts new file mode 100644 index 00000000000..6478470f1f0 --- /dev/null +++ b/extensions/ql-vscode/src/databases/code-search-api.ts @@ -0,0 +1,70 @@ +import { retry } from "@octokit/plugin-retry"; +import { throttling } from "@octokit/plugin-throttling"; +import { Octokit } from "@octokit/rest"; +import { Progress, CancellationToken } from "vscode"; +import { showAndLogWarningMessage } from "../helpers"; +import { Credentials } from "../common/authentication"; + +export async function getCodeSearchRepositories( + query: string, + progress: Progress<{ + message?: string | undefined; + increment?: number | undefined; + }>, + token: CancellationToken, + credentials: Credentials, +): Promise { + let nwos: string[] = []; + const octokit = await provideOctokitWithThrottling(credentials); + + for await (const response of octokit.paginate.iterator( + octokit.rest.search.code, + { + q: query, + per_page: 100, + }, + )) { + nwos.push(...response.data.map((item) => item.repository.full_name)); + // calculate progress bar: 80% of the progress bar is used for the code search + const totalNumberOfRequests = Math.ceil(response.data.total_count / 100); + // Since we have a maximum of 1000 responses of the api, we can use a fixed increment whenever the totalNumberOfRequests would be greater than 10 + const increment = + totalNumberOfRequests < 10 ? 80 / totalNumberOfRequests : 8; + progress.report({ increment }); + + if (token.isCancellationRequested) { + nwos = []; + break; + } + } + + return [...new Set(nwos)]; +} + +async function provideOctokitWithThrottling( + credentials: Credentials, +): Promise { + const MyOctokit = Octokit.plugin(throttling); + const auth = await credentials.getAccessToken(); + + const octokit = new MyOctokit({ + auth, + retry, + throttle: { + onRateLimit: (retryAfter: number, options: any): boolean => { + void showAndLogWarningMessage( + `Rate Limit detected for request ${options.method} ${options.url}. Retrying after ${retryAfter} seconds!`, + ); + + return true; + }, + onSecondaryRateLimit: (_retryAfter: number, options: any): void => { + void showAndLogWarningMessage( + `Secondary Rate Limit detected for request ${options.method} ${options.url}`, + ); + }, + }, + }); + + return octokit; +} diff --git a/extensions/ql-vscode/src/databases/ui/db-panel.ts b/extensions/ql-vscode/src/databases/ui/db-panel.ts index 9172b28220b..7e64fb4f2a2 100644 --- a/extensions/ql-vscode/src/databases/ui/db-panel.ts +++ b/extensions/ql-vscode/src/databases/ui/db-panel.ts @@ -36,8 +36,8 @@ import { getControllerRepo } from "../../variant-analysis/run-remote-query"; import { getErrorMessage } from "../../pure/helpers-pure"; import { DatabasePanelCommands } from "../../common/commands"; import { App } from "../../common/app"; -import { getCodeSearchRepositories } from "../../variant-analysis/gh-api/gh-api-client"; import { QueryLanguage } from "../../common/query-language"; +import { getCodeSearchRepositories } from "../code-search-api"; export interface RemoteDatabaseQuickPickItem extends QuickPickItem { remoteDatabaseKind: string; @@ -351,13 +351,19 @@ export class DbPanel extends DisposableObject { const listName = treeViewItem.dbItem.listName; - const languageQuickPickItems: CodeSearchQuickPickItem[] = Object.values( - QueryLanguage, - ).map((language) => ({ - label: language.toString(), - alwaysShow: true, - language: language.toString(), - })); + const languageQuickPickItems: CodeSearchQuickPickItem[] = [ + { + label: "No specific language", + alwaysShow: true, + language: "", + }, + ].concat( + Object.values(QueryLanguage).map((language) => ({ + label: language.toString(), + alwaysShow: true, + language: language.toString(), + })), + ); const codeSearchLanguage = await window.showQuickPick( @@ -372,6 +378,10 @@ export class DbPanel extends DisposableObject { return; } + const languagePrompt = codeSearchLanguage.language + ? `language:${codeSearchLanguage.language}` + : ""; + const codeSearchQuery = await window.showInputBox({ title: "GitHub Code Search", prompt: @@ -392,10 +402,10 @@ export class DbPanel extends DisposableObject { progress.report({ increment: 10 }); const repositories = await getCodeSearchRepositories( - this.app.credentials, - `${codeSearchQuery} language:${codeSearchLanguage.language}`, + `${codeSearchQuery} ${languagePrompt}`, progress, token, + this.app.credentials, ); token.onCancellationRequested(() => { diff --git a/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts b/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts index 0e8d68d1f7a..00bbafde926 100644 --- a/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts +++ b/extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts @@ -7,43 +7,6 @@ import { VariantAnalysisSubmissionRequest, } from "./variant-analysis"; import { Repository } from "./repository"; -import { Progress } from "vscode"; -import { CancellationToken } from "vscode-jsonrpc"; - -export async function getCodeSearchRepositories( - credentials: Credentials, - query: string, - progress: Progress<{ - message?: string | undefined; - increment?: number | undefined; - }>, - token: CancellationToken, -): Promise { - let nwos: string[] = []; - const octokit = await credentials.getOctokit(); - for await (const response of octokit.paginate.iterator( - octokit.rest.search.repos, - { - q: query, - per_page: 100, - }, - )) { - nwos.push(...response.data.map((item) => item.full_name)); - // calculate progress bar: 80% of the progress bar is used for the code search - const totalNumberOfRequests = Math.ceil(response.data.total_count / 100); - // Since we have a maximum 10 of requests, we use a fixed increment whenever the totalNumberOfRequests is greater than 10 - const increment = - totalNumberOfRequests < 10 ? 80 / totalNumberOfRequests : 8; - progress.report({ increment }); - - if (token.isCancellationRequested) { - nwos = []; - break; - } - } - - return [...new Set(nwos)]; -} export async function submitVariantAnalysis( credentials: Credentials,