From 37ab1b7a492199337553324e1ca63115cd8374b6 Mon Sep 17 00:00:00 2001 From: Andrew Eisenberg Date: Wed, 8 Sep 2021 09:25:19 -0700 Subject: [PATCH 1/4] Fix AST Viewer The previous synthetic query suite was not finding the ast query because the `qlpack` directive in a query suite only matches queries from the default suite, which `printAST.ql` is not part of. This changes to using `from` and `queries` directives. Also, adds an integration test to ensure we find the queries using different CLIs. However, this only tests using the latest `main` from the codeql repository. I wonder if we should start testing using different versions of the repo. --- .../ql-vscode/src/contextual/queryResolver.ts | 24 +++++++++---------- .../cli-integration/run-cli.test.ts | 14 +++++++++++ 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/extensions/ql-vscode/src/contextual/queryResolver.ts b/extensions/ql-vscode/src/contextual/queryResolver.ts index 99c4c42050b..17f21b0a89d 100644 --- a/extensions/ql-vscode/src/contextual/queryResolver.ts +++ b/extensions/ql-vscode/src/contextual/queryResolver.ts @@ -31,26 +31,24 @@ export async function qlpackOfDatabase(cli: CodeQLCliServer, db: DatabaseItem): * @returns The found queries from the first pack in which any matching queries were found. */ async function resolveQueriesFromPacks(cli: CodeQLCliServer, qlpacks: string[], keyType: KeyType): Promise { + const suiteFile = (await tmp.file({ + postfix: '.qls' + })).path; + const suiteYaml = []; for (const qlpack of qlpacks) { - const suiteFile = (await tmp.file({ - postfix: '.qls' - })).path; - const suiteYaml = { - qlpack, + suiteYaml.push({ + from: qlpack, + queries: '.', include: { kind: kindOfKeyType(keyType), 'tags contain': tagOfKeyType(keyType) } - }; - await fs.writeFile(suiteFile, yaml.safeDump(suiteYaml), 'utf8'); - - const queries = await cli.resolveQueriesInSuite(suiteFile, helpers.getOnDiskWorkspaceFolders()); - if (queries.length > 0) { - return queries; - } + }); } + await fs.writeFile(suiteFile, yaml.safeDump(suiteYaml), 'utf8'); - return []; + const queries = await cli.resolveQueriesInSuite(suiteFile, helpers.getOnDiskWorkspaceFolders()); + return queries; } export async function resolveQueries(cli: CodeQLCliServer, qlpacks: QlPacksForLanguage, keyType: KeyType): Promise { diff --git a/extensions/ql-vscode/src/vscode-tests/cli-integration/run-cli.test.ts b/extensions/ql-vscode/src/vscode-tests/cli-integration/run-cli.test.ts index 7429cd21a03..0ac6fc817b4 100644 --- a/extensions/ql-vscode/src/vscode-tests/cli-integration/run-cli.test.ts +++ b/extensions/ql-vscode/src/vscode-tests/cli-integration/run-cli.test.ts @@ -7,6 +7,8 @@ import { CodeQLCliServer, QueryInfoByLanguage } from '../../cli'; import { CodeQLExtensionInterface } from '../../extension'; import { skipIfNoCodeQL } from '../ensureCli'; import { getOnDiskWorkspaceFolders } from '../../helpers'; +import { resolveQueries } from '../../contextual/queryResolver'; +import { KeyType } from '../../contextual/keyType'; /** * Perform proper integration tests by running the CLI @@ -68,4 +70,16 @@ describe('Use cli', function() { const queryInfo: QueryInfoByLanguage = await cli.resolveQueryByLanguage(getOnDiskWorkspaceFolders(), Uri.file(queryPath)); expect((Object.keys(queryInfo.byLanguage))[0]).to.eql('javascript'); }); + + it.only('should resolve printAST queries', async function() { + skipIfNoCodeQL(this); + + const result = await resolveQueries(cli, { + dbschemePack: 'codeql/javascript-all', + dbschemePackIsLibraryPack: true, + queryPack: 'codeql/javascript-queries' + }, KeyType.PrintAstQuery); + expect(result.length).to.eq(1); + expect(result[0].endsWith('javascript/ql/src/printAst.ql')).to.be.true; + }); }); From c4a8b9b6c156e5075d25f5a7b04448c55082193c Mon Sep 17 00:00:00 2001 From: Andrew Eisenberg Date: Wed, 8 Sep 2021 11:17:33 -0700 Subject: [PATCH 2/4] Expand qlpack resolution integration test to all languages Go is not yet supported since we do not include the go submodule in the integration tests. --- extensions/ql-vscode/src/helpers.ts | 6 +++ .../cli-integration/run-cli.test.ts | 37 +++++++++++++------ .../contextual/queryResolver.test.ts | 8 ++-- 3 files changed, 35 insertions(+), 16 deletions(-) diff --git a/extensions/ql-vscode/src/helpers.ts b/extensions/ql-vscode/src/helpers.ts index 65eb3b29ffa..9caf7991b67 100644 --- a/extensions/ql-vscode/src/helpers.ts +++ b/extensions/ql-vscode/src/helpers.ts @@ -423,6 +423,12 @@ const dbSchemeToLanguage = { 'go.dbscheme': 'go' }; +export const languageToDbScheme = Object.entries(dbSchemeToLanguage).reduce((acc, [k, v]) => { + acc[v] = k; + return acc; +}, {} as { [k: string]: string }); + + /** * Returns the initial contents for an empty query, based on the language of the selected * databse. diff --git a/extensions/ql-vscode/src/vscode-tests/cli-integration/run-cli.test.ts b/extensions/ql-vscode/src/vscode-tests/cli-integration/run-cli.test.ts index 0ac6fc817b4..3c04b3ae6c3 100644 --- a/extensions/ql-vscode/src/vscode-tests/cli-integration/run-cli.test.ts +++ b/extensions/ql-vscode/src/vscode-tests/cli-integration/run-cli.test.ts @@ -6,7 +6,7 @@ import { SemVer } from 'semver'; import { CodeQLCliServer, QueryInfoByLanguage } from '../../cli'; import { CodeQLExtensionInterface } from '../../extension'; import { skipIfNoCodeQL } from '../ensureCli'; -import { getOnDiskWorkspaceFolders } from '../../helpers'; +import { getOnDiskWorkspaceFolders, getQlPackForDbscheme, languageToDbScheme } from '../../helpers'; import { resolveQueries } from '../../contextual/queryResolver'; import { KeyType } from '../../contextual/keyType'; @@ -14,6 +14,8 @@ import { KeyType } from '../../contextual/keyType'; * Perform proper integration tests by running the CLI */ describe('Use cli', function() { + const supportedLanguages = ['cpp', 'csharp', 'go', 'java', 'javascript', 'python']; + this.timeout(60000); let cli: CodeQLCliServer; @@ -51,7 +53,7 @@ describe('Use cli', function() { // Depending on the version of the CLI, the qlpacks may have different names // (e.g. "codeql/javascript-all" vs "codeql-javascript"), // so we just check that the expected languages are included. - for (const expectedLanguage of ['cpp', 'csharp', 'go', 'java', 'javascript', 'python']) { + for (const expectedLanguage of supportedLanguages) { expect((Object.keys(qlpacks)).includes(expectedLanguage)); } }); @@ -59,7 +61,7 @@ describe('Use cli', function() { it('should resolve languages', async function() { skipIfNoCodeQL(this); const languages = await cli.resolveLanguages(); - for (const expectedLanguage of ['cpp', 'csharp', 'go', 'java', 'javascript', 'python']) { + for (const expectedLanguage of supportedLanguages) { expect(languages).to.have.property(expectedLanguage).that.is.not.undefined; } }); @@ -71,15 +73,26 @@ describe('Use cli', function() { expect((Object.keys(queryInfo.byLanguage))[0]).to.eql('javascript'); }); - it.only('should resolve printAST queries', async function() { - skipIfNoCodeQL(this); - const result = await resolveQueries(cli, { - dbschemePack: 'codeql/javascript-all', - dbschemePackIsLibraryPack: true, - queryPack: 'codeql/javascript-queries' - }, KeyType.PrintAstQuery); - expect(result.length).to.eq(1); - expect(result[0].endsWith('javascript/ql/src/printAst.ql')).to.be.true; + supportedLanguages.forEach(lang => { + if (lang === 'go') { + // The codeql-go submodule is not available in the integration tests. + return; + } + it(`should resolve printAST queries for ${lang}`, async function() { + skipIfNoCodeQL(this); + + const pack = await getQlPackForDbscheme(cli, languageToDbScheme[lang]); + expect(pack.dbschemePack).to.contain(lang); + if (pack.dbschemePackIsLibraryPack) { + expect(pack.queryPack).to.contain(lang); + } + + const result = await resolveQueries(cli, pack, KeyType.PrintAstQuery); + + // It doesn't matter what the name or path of the query is, only + // that we have found exactly one query. + expect(result.length).to.eq(1); + }); }); }); diff --git a/extensions/ql-vscode/src/vscode-tests/no-workspace/contextual/queryResolver.test.ts b/extensions/ql-vscode/src/vscode-tests/no-workspace/contextual/queryResolver.test.ts index 09ac5cad78f..a97a211883a 100644 --- a/extensions/ql-vscode/src/vscode-tests/no-workspace/contextual/queryResolver.test.ts +++ b/extensions/ql-vscode/src/vscode-tests/no-workspace/contextual/queryResolver.test.ts @@ -30,19 +30,19 @@ describe('queryResolver', () => { }); describe('resolveQueries', () => { - it('should resolve a query', async () => { mockCli.resolveQueriesInSuite.returns(['a', 'b']); const result = await module.resolveQueries(mockCli, { dbschemePack: 'my-qlpack' }, KeyType.DefinitionQuery); expect(result).to.deep.equal(['a', 'b']); expect(writeFileSpy.getCall(0).args[0]).to.match(/.qls$/); - expect(yaml.safeLoad(writeFileSpy.getCall(0).args[1])).to.deep.equal({ - qlpack: 'my-qlpack', + expect(yaml.safeLoad(writeFileSpy.getCall(0).args[1])).to.deep.equal([{ + from: 'my-qlpack', + queries: '.', include: { kind: 'definitions', 'tags contain': 'ide-contextual-queries/local-definitions' } - }); + }]); }); it('should resolve a query from the queries pack if this is an old CLI', async () => { From 024d3e6b969bb03c5085a88494dba8a4be0387c5 Mon Sep 17 00:00:00 2001 From: Andrew Eisenberg Date: Wed, 8 Sep 2021 11:18:01 -0700 Subject: [PATCH 3/4] Remove support for 2.2.6 CLI This is old enough that we don't need to support it. --- .github/workflows/main.yml | 2 +- .../no-workspace/contextual/queryResolver.test.ts | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 863deb963a1..3a89e5a9e2f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -151,7 +151,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest] - version: ['v2.2.6', 'v2.3.3', 'v2.4.6', 'v2.5.9', 'v2.6.1', 'nightly'] + version: ['v2.3.3', 'v2.4.6', 'v2.5.9', 'v2.6.1', 'nightly'] env: CLI_VERSION: ${{ matrix.version }} NIGHTLY_URL: ${{ needs.find-nightly.outputs.url }} diff --git a/extensions/ql-vscode/src/vscode-tests/no-workspace/contextual/queryResolver.test.ts b/extensions/ql-vscode/src/vscode-tests/no-workspace/contextual/queryResolver.test.ts index a97a211883a..71a463785be 100644 --- a/extensions/ql-vscode/src/vscode-tests/no-workspace/contextual/queryResolver.test.ts +++ b/extensions/ql-vscode/src/vscode-tests/no-workspace/contextual/queryResolver.test.ts @@ -52,13 +52,14 @@ describe('queryResolver', () => { const result = await module.resolveQueries(mockCli, { dbschemePackIsLibraryPack: true, dbschemePack: 'my-qlpack', queryPack: 'my-qlpack2' }, KeyType.DefinitionQuery); expect(result).to.deep.equal(['a', 'b']); expect(writeFileSpy.getCall(0).args[0]).to.match(/.qls$/); - expect(yaml.safeLoad(writeFileSpy.getCall(0).args[1])).to.deep.equal({ - qlpack: 'my-qlpack2', + expect(yaml.safeLoad(writeFileSpy.getCall(0).args[1])).to.deep.equal([{ + from: 'my-qlpack2', + queries: '.', include: { kind: 'definitions', 'tags contain': 'ide-contextual-queries/local-definitions' } - }); + }]); }); it('should throw an error when there are no queries found', async () => { From 5b4479f1c8c805bdc69e22978bde904c8aa9b584 Mon Sep 17 00:00:00 2001 From: Andrew Eisenberg Date: Wed, 8 Sep 2021 11:54:40 -0700 Subject: [PATCH 4/4] Update changelog --- extensions/ql-vscode/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/extensions/ql-vscode/CHANGELOG.md b/extensions/ql-vscode/CHANGELOG.md index e82b018883b..53dfffb72f2 100644 --- a/extensions/ql-vscode/CHANGELOG.md +++ b/extensions/ql-vscode/CHANGELOG.md @@ -3,6 +3,7 @@ ## [UNRELEASED] - Fix bug where a query is sometimes run before the file is saved. [#947](https://github.com/github/vscode-codeql/pull/947) +- Fix broken contextual queries, including _View AST_. [#949](https://github.com/github/vscode-codeql/pull/949) ## 1.5.4 - 02 September 2021