From 694ead6290d7c9c4c38d6f80b2b865571e7a9959 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Fri, 17 Apr 2026 11:03:02 -0400 Subject: [PATCH] Add nonclustered indexes for query/procedure/query store lookups MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 3 OUTER APPLY hydration of compressed query_text/plan_text was forcing an Eager Index Spool over the full collect.query_stats table (and similar for procedure_stats / query_store_data), which took 104 seconds on a 742K-row table in #835. Changes: - Remove CONVERT(binary(8), nvarchar-hash, 1) anti-pattern from OUTER APPLY WHERE clauses by keeping query_hash as native binary(8) in temp tables. query_hash is only converted to nvarchar(20) in the final output projection. - Add three nonclustered indexes (install script and upgrade script): IX_query_stats_hash_lookup (query_hash, database_name, collection_time DESC) IX_procedure_stats_name_lookup (database_name, schema_name, object_name, collection_time DESC) IX_query_store_data_id_lookup (database_name, query_id, collection_time DESC) - Indexes use SORT_IN_TEMPDB = ON and DATA_COMPRESSION = PAGE. - ONLINE = ON is applied conditionally via dynamic SQL based on SERVERPROPERTY('EngineEdition') — Enterprise/Developer/Azure only, since Standard/Web/Express don't support online index operations. Tested against CADelete's 742K-row table: Phase 3 went from 104s to well under 1s (5s total for the full three-phase query). Fixes #835 Co-Authored-By: Claude Opus 4.7 (1M context) --- .../DatabaseService.QueryPerformance.cs | 14 +-- install/02_create_tables.sql | 78 +++++++++++++++ .../01_add_query_lookup_indexes.sql | 98 +++++++++++++++++++ upgrades/2.7.0-to-2.8.0/upgrade.txt | 1 + 4 files changed, 184 insertions(+), 7 deletions(-) create mode 100644 upgrades/2.7.0-to-2.8.0/01_add_query_lookup_indexes.sql create mode 100644 upgrades/2.7.0-to-2.8.0/upgrade.txt diff --git a/Dashboard/Services/DatabaseService.QueryPerformance.cs b/Dashboard/Services/DatabaseService.QueryPerformance.cs index 7f5f0e27..9ca0fec5 100644 --- a/Dashboard/Services/DatabaseService.QueryPerformance.cs +++ b/Dashboard/Services/DatabaseService.QueryPerformance.cs @@ -1123,7 +1123,7 @@ USE HINT('ENABLE_PARALLEL_PLAN_PREFERENCE') SELECT TOP (500) database_name = pl.database_name, - query_hash = CONVERT(nvarchar(20), pl.query_hash, 1), + query_hash = pl.query_hash, object_type = MAX(pl.object_type), object_name = CASE MAX(pl.object_type) @@ -1180,7 +1180,7 @@ HASH GROUP /*Phase 3: hydrate text and plan XML for the TOP 500 winners only*/ SELECT tr.database_name, - tr.query_hash, + query_hash = CONVERT(nvarchar(20), tr.query_hash, 1), tr.object_type, tr.object_name, tr.first_execution_time, @@ -1224,7 +1224,7 @@ OUTER APPLY SELECT TOP (1) query_text = CAST(DECOMPRESS(qs2.query_text) AS nvarchar(max)) FROM collect.query_stats AS qs2 - WHERE qs2.query_hash = CONVERT(binary(8), tr.query_hash, 1) + WHERE qs2.query_hash = tr.query_hash AND qs2.database_name = tr.database_name ORDER BY qs2.collection_time DESC ) AS qt @@ -1233,7 +1233,7 @@ OUTER APPLY SELECT TOP (1) query_plan_xml = CAST(DECOMPRESS(qs3.query_plan_text) AS nvarchar(max)) FROM collect.query_stats AS qs3 - WHERE qs3.query_hash = CONVERT(binary(8), tr.query_hash, 1) + WHERE qs3.query_hash = tr.query_hash AND qs3.database_name = tr.database_name AND qs3.query_plan_text IS NOT NULL ORDER BY qs3.collection_time DESC @@ -4379,7 +4379,7 @@ USE HINT('ENABLE_PARALLEL_PLAN_PREFERENCE') SELECT TOP (@top + 5) database_name = pl.database_name, - query_hash = CONVERT(nvarchar(20), pl.query_hash, 1), + query_hash = pl.query_hash, object_type = MAX(pl.object_type), object_name = CASE MAX(pl.object_type) @@ -4438,7 +4438,7 @@ HASH GROUP /*Phase 3: hydrate text for winners only, apply WAITFOR filter*/ SELECT TOP (@top) tr.database_name, - tr.query_hash, + query_hash = CONVERT(nvarchar(20), tr.query_hash, 1), tr.object_type, tr.object_name, tr.first_execution_time, @@ -4481,7 +4481,7 @@ OUTER APPLY SELECT TOP (1) query_text = CAST(DECOMPRESS(qs2.query_text) AS nvarchar(max)) FROM collect.query_stats AS qs2 - WHERE qs2.query_hash = CONVERT(binary(8), tr.query_hash, 1) + WHERE qs2.query_hash = tr.query_hash AND qs2.database_name = tr.database_name ORDER BY qs2.collection_time DESC ) AS qt diff --git a/install/02_create_tables.sql b/install/02_create_tables.sql index 2a94183b..9cd74877 100644 --- a/install/02_create_tables.sql +++ b/install/02_create_tables.sql @@ -1495,5 +1495,83 @@ BEGIN PRINT 'Created collect.server_properties table'; END; +/* +Nonclustered indexes to support OUTER APPLY lookups in the Dashboard's +query/procedure/query store grid queries. Without these, the optimizer builds +an Eager Index Spool over the entire table to service the lookups, which can +take minutes on large installations (see #835). + +ONLINE = ON is only supported on Enterprise/Developer/Azure editions. The +options string is built dynamically based on SERVERPROPERTY('EngineEdition'). +*/ +DECLARE @online_option nvarchar(20) = + CASE + WHEN CAST(SERVERPROPERTY(N'EngineEdition') AS integer) IN (3, 5, 8) + THEN N', ONLINE = ON' + ELSE N'' + END; + +DECLARE @index_sql nvarchar(max); + +IF NOT EXISTS +( + SELECT + 1/0 + FROM sys.indexes + WHERE object_id = OBJECT_ID(N'collect.query_stats') + AND name = N'IX_query_stats_hash_lookup' +) +BEGIN + SET @index_sql = N' + CREATE NONCLUSTERED INDEX + IX_query_stats_hash_lookup + ON collect.query_stats + (query_hash, database_name, collection_time DESC) + WITH + (SORT_IN_TEMPDB = ON, DATA_COMPRESSION = PAGE' + @online_option + N');'; + EXEC sys.sp_executesql @index_sql; + PRINT 'Created collect.query_stats.IX_query_stats_hash_lookup index'; +END; + +IF NOT EXISTS +( + SELECT + 1/0 + FROM sys.indexes + WHERE object_id = OBJECT_ID(N'collect.procedure_stats') + AND name = N'IX_procedure_stats_name_lookup' +) +BEGIN + SET @index_sql = N' + CREATE NONCLUSTERED INDEX + IX_procedure_stats_name_lookup + ON collect.procedure_stats + (database_name, schema_name, object_name, collection_time DESC) + WITH + (SORT_IN_TEMPDB = ON, DATA_COMPRESSION = PAGE' + @online_option + N');'; + EXEC sys.sp_executesql @index_sql; + PRINT 'Created collect.procedure_stats.IX_procedure_stats_name_lookup index'; +END; + +IF NOT EXISTS +( + SELECT + 1/0 + FROM sys.indexes + WHERE object_id = OBJECT_ID(N'collect.query_store_data') + AND name = N'IX_query_store_data_id_lookup' +) +BEGIN + SET @index_sql = N' + CREATE NONCLUSTERED INDEX + IX_query_store_data_id_lookup + ON collect.query_store_data + (database_name, query_id, collection_time DESC) + WITH + (SORT_IN_TEMPDB = ON, DATA_COMPRESSION = PAGE' + @online_option + N');'; + EXEC sys.sp_executesql @index_sql; + PRINT 'Created collect.query_store_data.IX_query_store_data_id_lookup index'; +END; + PRINT 'All collection tables created successfully'; GO diff --git a/upgrades/2.7.0-to-2.8.0/01_add_query_lookup_indexes.sql b/upgrades/2.7.0-to-2.8.0/01_add_query_lookup_indexes.sql new file mode 100644 index 00000000..698aff92 --- /dev/null +++ b/upgrades/2.7.0-to-2.8.0/01_add_query_lookup_indexes.sql @@ -0,0 +1,98 @@ +/* +Copyright 2026 Darling Data, LLC +https://www.erikdarling.com/ + +Upgrade from 2.7.0 to 2.8.0 + +Add nonclustered indexes on collect.query_stats, collect.procedure_stats, and +collect.query_store_data to support OUTER APPLY lookups in the Dashboard's +grid queries. Without these, the optimizer builds an Eager Index Spool over +the entire table to service the lookups, which can take minutes on large +installations (see #835). + +ONLINE = ON is only supported on Enterprise/Developer/Azure editions. The +options string is built dynamically based on SERVERPROPERTY('EngineEdition'). +*/ + +SET ANSI_NULLS ON; +SET ANSI_PADDING ON; +SET ANSI_WARNINGS ON; +SET ARITHABORT ON; +SET CONCAT_NULL_YIELDS_NULL ON; +SET QUOTED_IDENTIFIER ON; +SET NUMERIC_ROUNDABORT OFF; +SET IMPLICIT_TRANSACTIONS OFF; +SET STATISTICS TIME, IO OFF; +GO + +USE PerformanceMonitor; +GO + +DECLARE @online_option nvarchar(20) = + CASE + WHEN CAST(SERVERPROPERTY(N'EngineEdition') AS integer) IN (3, 5, 8) + THEN N', ONLINE = ON' + ELSE N'' + END; + +DECLARE @index_sql nvarchar(max); + +IF NOT EXISTS +( + SELECT + 1/0 + FROM sys.indexes + WHERE object_id = OBJECT_ID(N'collect.query_stats') + AND name = N'IX_query_stats_hash_lookup' +) +BEGIN + SET @index_sql = N' + CREATE NONCLUSTERED INDEX + IX_query_stats_hash_lookup + ON collect.query_stats + (query_hash, database_name, collection_time DESC) + WITH + (SORT_IN_TEMPDB = ON, DATA_COMPRESSION = PAGE' + @online_option + N');'; + EXEC sys.sp_executesql @index_sql; + PRINT 'Created collect.query_stats.IX_query_stats_hash_lookup index'; +END; + +IF NOT EXISTS +( + SELECT + 1/0 + FROM sys.indexes + WHERE object_id = OBJECT_ID(N'collect.procedure_stats') + AND name = N'IX_procedure_stats_name_lookup' +) +BEGIN + SET @index_sql = N' + CREATE NONCLUSTERED INDEX + IX_procedure_stats_name_lookup + ON collect.procedure_stats + (database_name, schema_name, object_name, collection_time DESC) + WITH + (SORT_IN_TEMPDB = ON, DATA_COMPRESSION = PAGE' + @online_option + N');'; + EXEC sys.sp_executesql @index_sql; + PRINT 'Created collect.procedure_stats.IX_procedure_stats_name_lookup index'; +END; + +IF NOT EXISTS +( + SELECT + 1/0 + FROM sys.indexes + WHERE object_id = OBJECT_ID(N'collect.query_store_data') + AND name = N'IX_query_store_data_id_lookup' +) +BEGIN + SET @index_sql = N' + CREATE NONCLUSTERED INDEX + IX_query_store_data_id_lookup + ON collect.query_store_data + (database_name, query_id, collection_time DESC) + WITH + (SORT_IN_TEMPDB = ON, DATA_COMPRESSION = PAGE' + @online_option + N');'; + EXEC sys.sp_executesql @index_sql; + PRINT 'Created collect.query_store_data.IX_query_store_data_id_lookup index'; +END; diff --git a/upgrades/2.7.0-to-2.8.0/upgrade.txt b/upgrades/2.7.0-to-2.8.0/upgrade.txt new file mode 100644 index 00000000..8c221a3a --- /dev/null +++ b/upgrades/2.7.0-to-2.8.0/upgrade.txt @@ -0,0 +1 @@ +01_add_query_lookup_indexes.sql