From c261efe0dd616724214b48ae3099f2077a07985f Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 4 Mar 2025 22:15:58 -0500 Subject: [PATCH 001/246] Update sp_QuickieStore.sql Redo the column select list logic --- sp_QuickieStore/sp_QuickieStore.sql | 1770 ++++++--------------------- 1 file changed, 344 insertions(+), 1426 deletions(-) diff --git a/sp_QuickieStore/sp_QuickieStore.sql b/sp_QuickieStore/sp_QuickieStore.sql index c1a548d3..ff2397a6 100644 --- a/sp_QuickieStore/sp_QuickieStore.sql +++ b/sp_QuickieStore/sp_QuickieStore.sql @@ -839,12 +839,12 @@ CREATE TABLE database_id int NOT NULL, plan_id bigint NOT NULL, query_id bigint NOT NULL, - all_plan_ids varchar(MAX), + all_plan_ids varchar(max), plan_group_id bigint NULL, engine_version nvarchar(32) NULL, compatibility_level smallint NOT NULL, query_plan_hash binary(8) NOT NULL, - query_plan nvarchar(MAX) NULL, + query_plan nvarchar(max) NULL, is_online_index_plan bit NOT NULL, is_trivial_plan bit NOT NULL, is_parallel_plan bit NOT NULL, @@ -1154,7 +1154,7 @@ CREATE TABLE plan_feedback_id bigint, plan_id bigint, feature_desc nvarchar(120), - feedback_data nvarchar(MAX), + feedback_data nvarchar(max), state_desc nvarchar(120), create_time datetimeoffset(7), last_updated_time datetimeoffset(7) @@ -1169,7 +1169,7 @@ CREATE TABLE database_id int NOT NULL, query_hint_id bigint, query_id bigint, - query_hint_text nvarchar(MAX), + query_hint_text nvarchar(max), last_query_hint_failure_reason_desc nvarchar(256), query_hint_failure_count bigint, source_desc nvarchar(256) @@ -1242,6 +1242,238 @@ CREATE TABLE database_name sysname PRIMARY KEY CLUSTERED ); +/* Create a table variable to store ALL column definitions with logical ordering */ +DECLARE + @ColumnDefinitions table +( + column_id integer + PRIMARY KEY CLUSTERED, /* Controls the ordering of columns in output */ + metric_group nvarchar(50) NOT NULL, /* Grouping (duration, cpu, etc.) */ + metric_type nvarchar(20) NOT NULL, /* Type within group (avg, total, last, min, max) */ + column_name nvarchar(100) NOT NULL, /* Column name as it appears in output */ + column_source nvarchar(max) NOT NULL, /* Source expression or formula */ + is_conditional bit NOT NULL, /* Is this a conditional column (depends on a parameter) */ + condition_param nvarchar(50) NULL, /* Parameter name this column depends on */ + condition_value sql_variant NULL, /* Value the parameter must have */ + expert_only bit NOT NULL, /* Only include in expert mode */ + format_pattern nvarchar(20) NULL /* Format pattern (e.g., 'N0', 'P2', NULL for no formatting) */ +); + +/* Fill the table with ALL columns, including SQL 2022 views and regression columns */ + +/* Basic metadata columns (still part of prefix, but in the table) */ +INSERT INTO + @ColumnDefinitions +( + column_id, metric_group, metric_type, column_name, column_source, is_conditional, condition_param, condition_value, expert_only, format_pattern +) +VALUES + (20, 'metadata', 'force_count', 'force_failure_count', 'qsp.force_failure_count', 0, NULL, NULL, 0, NULL), + (30, 'metadata', 'force_reason', 'last_force_failure_reason_desc', 'qsp.last_force_failure_reason_desc', 0, NULL, NULL, 0, NULL), + /* SQL 2022 specific columns */ + (40, 'sql_2022', 'feedback', 'has_query_feedback', 'CASE WHEN EXISTS (SELECT 1/0 FROM #query_store_plan_feedback AS qspf WHERE qspf.plan_id = qsp.plan_id) THEN ''Yes'' ELSE ''No'' END', 1, 'sql_2022_views', 1, 0, NULL), + (50, 'sql_2022', 'hints', 'has_query_store_hints', 'CASE WHEN EXISTS (SELECT 1/0 FROM #query_store_query_hints AS qsqh WHERE qsqh.query_id = qsp.query_id) THEN ''Yes'' ELSE ''No'' END', 1, 'sql_2022_views', 1, 0, NULL), + (60, 'sql_2022', 'variants', 'has_plan_variants', 'CASE WHEN EXISTS (SELECT 1/0 FROM #query_store_query_variant AS qsqv WHERE qsqv.query_variant_query_id = qsp.query_id) THEN ''Yes'' ELSE ''No'' END', 1, 'sql_2022_views', 1, 0, NULL), + (70, 'sql_2022', 'replay', 'has_compile_replay_script', 'qsp.has_compile_replay_script', 1, 'sql_2022_views', 1, 0, NULL), + (80, 'sql_2022', 'opt_forcing', 'is_optimized_plan_forcing_disabled', 'qsp.is_optimized_plan_forcing_disabled', 1, 'sql_2022_views', 1, 0, NULL), + (90, 'sql_2022', 'plan_type', 'plan_type_desc', 'qsp.plan_type_desc', 1, 'sql_2022_views', 1, 0, NULL), + /* New version features */ + (95, 'new_features', 'forcing_type', 'plan_forcing_type_desc', 'qsp.plan_forcing_type_desc', 1, 'new', 1, 0, NULL), + (97, 'new_features', 'top_waits', 'top_waits', 'w.top_waits', 1, 'new', 1, 0, NULL), + /* Date/time columns (not conditional, always included) */ + (100, 'execution_time', 'first', 'first_execution_time', 'CASE WHEN @timezone IS NULL THEN SWITCHOFFSET(qsrs.first_execution_time, @utc_offset_string) WHEN @timezone IS NOT NULL THEN qsrs.first_execution_time AT TIME ZONE @timezone END', 0, NULL, NULL, 0, NULL), + (110, 'execution_time', 'first_utc', 'first_execution_time_utc', 'qsrs.first_execution_time', 0, NULL, NULL, 0, NULL), + (120, 'execution_time', 'last', 'last_execution_time', 'CASE WHEN @timezone IS NULL THEN SWITCHOFFSET(qsrs.last_execution_time, @utc_offset_string) WHEN @timezone IS NOT NULL THEN qsrs.last_execution_time AT TIME ZONE @timezone END', 0, NULL, NULL, 0, NULL), + (130, 'execution_time', 'last_utc', 'last_execution_time_utc', 'qsrs.last_execution_time', 0, NULL, NULL, 0, NULL), + /* Regression mode columns */ + (140, 'regression', 'baseline', 'from_regression_baseline_time_period', 'qsrs.from_regression_baseline', 1, 'regression_mode', 1, 0, NULL), + (150, 'regression', 'hash', 'query_hash_from_regression_checking', 'regression.query_hash', 1, 'regression_mode', 1, 0, NULL), + /* Execution columns */ + (200, 'executions', 'count', 'count_executions', 'qsrs.count_executions', 0, NULL, NULL, 0, 'N0'), + (210, 'executions', 'per_second', 'executions_per_second', 'qsrs.executions_per_second', 0, NULL, NULL, 0, 'N0'), + /* Hash totals - conditionally added */ + (215, 'executions', 'count_hash', 'count_executions_by_query_hash', 'SUM(qsrs.count_executions) OVER (PARTITION BY qsq.query_hash ORDER BY qsq.query_hash)', 1, 'include_query_hash_totals', 1, 0, 'N0'), + /* Duration metrics (group together avg, total, last, min, max) */ + (300, 'duration', 'avg', 'avg_duration_ms', 'qsrs.avg_duration_ms', 0, NULL, NULL, 0, 'N0'), + (310, 'duration', 'total', 'total_duration_ms', 'qsrs.total_duration_ms', 0, NULL, NULL, 0, 'N0'), + (320, 'duration', 'last', 'last_duration_ms', 'qsrs.last_duration_ms', 0, NULL, NULL, 1, 'N0'), + (330, 'duration', 'min', 'min_duration_ms', 'qsrs.min_duration_ms', 0, NULL, NULL, 1, 'N0'), + (340, 'duration', 'max', 'max_duration_ms', 'qsrs.max_duration_ms', 0, NULL, NULL, 0, 'N0'), + /* Hash totals for duration */ + (315, 'duration', 'total_hash', 'total_duration_ms_by_query_hash', 'SUM(qsrs.total_duration_ms) OVER (PARTITION BY qsq.query_hash ORDER BY qsq.query_hash)', 1, 'include_query_hash_totals', 1, 0, 'N0'), + /* CPU metrics */ + (400, 'cpu', 'avg', 'avg_cpu_time_ms', 'qsrs.avg_cpu_time_ms', 0, NULL, NULL, 0, 'N0'), + (410, 'cpu', 'total', 'total_cpu_time_ms', 'qsrs.total_cpu_time_ms', 0, NULL, NULL, 0, 'N0'), + (420, 'cpu', 'last', 'last_cpu_time_ms', 'qsrs.last_cpu_time_ms', 0, NULL, NULL, 1, 'N0'), + (430, 'cpu', 'min', 'min_cpu_time_ms', 'qsrs.min_cpu_time_ms', 0, NULL, NULL, 1, 'N0'), + (440, 'cpu', 'max', 'max_cpu_time_ms', 'qsrs.max_cpu_time_ms', 0, NULL, NULL, 0, 'N0'), + /* Hash totals for CPU */ + (415, 'cpu', 'total_hash', 'total_cpu_time_ms_by_query_hash', 'SUM(qsrs.total_cpu_time_ms) OVER (PARTITION BY qsq.query_hash ORDER BY qsq.query_hash)', 1, 'include_query_hash_totals', 1, 0, 'N0'), + /* Logical IO Reads */ + (500, 'logical_io_reads', 'avg', 'avg_logical_io_reads_mb', 'qsrs.avg_logical_io_reads_mb', 0, NULL, NULL, 0, 'N0'), + (510, 'logical_io_reads', 'total', 'total_logical_io_reads_mb', 'qsrs.total_logical_io_reads_mb', 0, NULL, NULL, 0, 'N0'), + (520, 'logical_io_reads', 'last', 'last_logical_io_reads_mb', 'qsrs.last_logical_io_reads_mb', 0, NULL, NULL, 1, 'N0'), + (530, 'logical_io_reads', 'min', 'min_logical_io_reads_mb', 'qsrs.min_logical_io_reads_mb', 0, NULL, NULL, 1, 'N0'), + (540, 'logical_io_reads', 'max', 'max_logical_io_reads_mb', 'qsrs.max_logical_io_reads_mb', 0, NULL, NULL, 0, 'N0'), + /* Hash totals for logical reads */ + (515, 'logical_io_reads', 'total_hash', 'total_logical_io_reads_mb_by_query_hash', 'SUM(qsrs.total_logical_io_reads_mb) OVER (PARTITION BY qsq.query_hash ORDER BY qsq.query_hash)', 1, 'include_query_hash_totals', 1, 0, 'N0'), + /* Logical IO Writes */ + (600, 'logical_io_writes', 'avg', 'avg_logical_io_writes_mb', 'qsrs.avg_logical_io_writes_mb', 0, NULL, NULL, 0, 'N0'), + (610, 'logical_io_writes', 'total', 'total_logical_io_writes_mb', 'qsrs.total_logical_io_writes_mb', 0, NULL, NULL, 0, 'N0'), + (620, 'logical_io_writes', 'last', 'last_logical_io_writes_mb', 'qsrs.last_logical_io_writes_mb', 0, NULL, NULL, 1, 'N0'), + (630, 'logical_io_writes', 'min', 'min_logical_io_writes_mb', 'qsrs.min_logical_io_writes_mb', 0, NULL, NULL, 1, 'N0'), + (640, 'logical_io_writes', 'max', 'max_logical_io_writes_mb', 'qsrs.max_logical_io_writes_mb', 0, NULL, NULL, 0, 'N0'), + /* Hash totals for logical writes */ + (615, 'logical_io_writes', 'total_hash', 'total_logical_io_writes_mb_by_query_hash', 'SUM(qsrs.total_logical_io_writes_mb) OVER (PARTITION BY qsq.query_hash ORDER BY qsq.query_hash)', 1, 'include_query_hash_totals', 1, 0, 'N0'), + /* Physical IO Reads */ + (700, 'physical_io_reads', 'avg', 'avg_physical_io_reads_mb', 'qsrs.avg_physical_io_reads_mb', 0, NULL, NULL, 0, 'N0'), + (710, 'physical_io_reads', 'total', 'total_physical_io_reads_mb', 'qsrs.total_physical_io_reads_mb', 0, NULL, NULL, 0, 'N0'), + (720, 'physical_io_reads', 'last', 'last_physical_io_reads_mb', 'qsrs.last_physical_io_reads_mb', 0, NULL, NULL, 1, 'N0'), + (730, 'physical_io_reads', 'min', 'min_physical_io_reads_mb', 'qsrs.min_physical_io_reads_mb', 0, NULL, NULL, 1, 'N0'), + (740, 'physical_io_reads', 'max', 'max_physical_io_reads_mb', 'qsrs.max_physical_io_reads_mb', 0, NULL, NULL, 0, 'N0'), + /* Hash totals for physical reads */ + (715, 'physical_io_reads', 'total_hash', 'total_physical_io_reads_mb_by_query_hash', 'SUM(qsrs.total_physical_io_reads_mb) OVER (PARTITION BY qsq.query_hash ORDER BY qsq.query_hash)', 1, 'include_query_hash_totals', 1, 0, 'N0'), + /* CLR Time */ + (800, 'clr_time', 'avg', 'avg_clr_time_ms', 'qsrs.avg_clr_time_ms', 0, NULL, NULL, 0, 'N0'), + (810, 'clr_time', 'total', 'total_clr_time_ms', 'qsrs.total_clr_time_ms', 0, NULL, NULL, 0, 'N0'), + (820, 'clr_time', 'last', 'last_clr_time_ms', 'qsrs.last_clr_time_ms', 0, NULL, NULL, 1, 'N0'), + (830, 'clr_time', 'min', 'min_clr_time_ms', 'qsrs.min_clr_time_ms', 0, NULL, NULL, 1, 'N0'), + (840, 'clr_time', 'max', 'max_clr_time_ms', 'qsrs.max_clr_time_ms', 0, NULL, NULL, 0, 'N0'), + /* Hash totals for CLR time */ + (815, 'clr_time', 'total_hash', 'total_clr_time_ms_by_query_hash', 'SUM(qsrs.total_clr_time_ms) OVER (PARTITION BY qsq.query_hash ORDER BY qsq.query_hash)', 1, 'include_query_hash_totals', 1, 0, 'N0'), + /* DOP (Degree of Parallelism) */ + (900, 'dop', 'last', 'last_dop', 'qsrs.last_dop', 0, NULL, NULL, 1, NULL), + (910, 'dop', 'min', 'min_dop', 'qsrs.min_dop', 0, NULL, NULL, 0, NULL), + (920, 'dop', 'max', 'max_dop', 'qsrs.max_dop', 0, NULL, NULL, 0, NULL), + /* Memory metrics */ + (1000, 'memory', 'avg', 'avg_query_max_used_memory_mb', 'qsrs.avg_query_max_used_memory_mb', 0, NULL, NULL, 0, 'N0'), + (1010, 'memory', 'total', 'total_query_max_used_memory_mb', 'qsrs.total_query_max_used_memory_mb', 0, NULL, NULL, 0, 'N0'), + (1020, 'memory', 'last', 'last_query_max_used_memory_mb', 'qsrs.last_query_max_used_memory_mb', 0, NULL, NULL, 1, 'N0'), + (1030, 'memory', 'min', 'min_query_max_used_memory_mb', 'qsrs.min_query_max_used_memory_mb', 0, NULL, NULL, 1, 'N0'), + (1040, 'memory', 'max', 'max_query_max_used_memory_mb', 'qsrs.max_query_max_used_memory_mb', 0, NULL, NULL, 0, 'N0'), + /* Hash totals for memory */ + (1015, 'memory', 'total_hash', 'total_query_max_used_memory_mb_by_query_hash', 'SUM(qsrs.total_query_max_used_memory_mb) OVER (PARTITION BY qsq.query_hash ORDER BY qsq.query_hash)', 1, 'include_query_hash_totals', 1, 0, 'N0'), + /* Row counts */ + (1100, 'rowcount', 'avg', 'avg_rowcount', 'qsrs.avg_rowcount', 0, NULL, NULL, 0, 'N0'), + (1110, 'rowcount', 'total', 'total_rowcount', 'qsrs.total_rowcount', 0, NULL, NULL, 0, 'N0'), + (1120, 'rowcount', 'last', 'last_rowcount', 'qsrs.last_rowcount', 0, NULL, NULL, 1, 'N0'), + (1130, 'rowcount', 'min', 'min_rowcount', 'qsrs.min_rowcount', 0, NULL, NULL, 1, 'N0'), + (1140, 'rowcount', 'max', 'max_rowcount', 'qsrs.max_rowcount', 0, NULL, NULL, 0, 'N0'), + /* Hash totals for row counts */ + (1115, 'rowcount', 'total_hash', 'total_rowcount_by_query_hash', 'SUM(qsrs.total_rowcount) OVER (PARTITION BY qsq.query_hash ORDER BY qsq.query_hash)', 1, 'include_query_hash_totals', 1, 0, 'N0'), + /* New metrics for newer versions */ + /* Physical IO Reads (for newer versions) */ + (1200, 'num_physical_io_reads', 'avg', 'avg_num_physical_io_reads_mb', 'qsrs.avg_num_physical_io_reads_mb', 1, 'new', 1, 0, 'N0'), + (1210, 'num_physical_io_reads', 'total', 'total_num_physical_io_reads_mb', 'qsrs.total_num_physical_io_reads_mb', 1, 'new', 1, 0, 'N0'), + (1220, 'num_physical_io_reads', 'last', 'last_num_physical_io_reads_mb', 'qsrs.last_num_physical_io_reads_mb', 1, 'new', 1, 1, 'N0'), + (1230, 'num_physical_io_reads', 'min', 'min_num_physical_io_reads_mb', 'qsrs.min_num_physical_io_reads_mb', 1, 'new', 1, 1, 'N0'), + (1240, 'num_physical_io_reads', 'max', 'max_num_physical_io_reads_mb', 'qsrs.max_num_physical_io_reads_mb', 1, 'new', 1, 0, 'N0'), + /* Hash totals for new physical IO reads */ + (1215, 'num_physical_io_reads', 'total_hash', 'total_num_physical_io_reads_mb_by_query_hash', 'SUM(qsrs.total_num_physical_io_reads_mb) OVER (PARTITION BY qsq.query_hash ORDER BY qsq.query_hash)', 1, 'new', 1, 0, 'N0'), + /* Finish adding the remaining columns (log bytes and tempdb usage) */ + /* Log bytes used */ + (1300, 'log_bytes', 'avg', 'avg_log_bytes_used_mb', 'qsrs.avg_log_bytes_used_mb', 1, 'new', 1, 0, 'N0'), + (1310, 'log_bytes', 'total', 'total_log_bytes_used_mb', 'qsrs.total_log_bytes_used_mb', 1, 'new', 1, 0, 'N0'), + (1320, 'log_bytes', 'last', 'last_log_bytes_used_mb', 'qsrs.last_log_bytes_used_mb', 1, 'new', 1, 1, 'N0'), + (1330, 'log_bytes', 'min', 'min_log_bytes_used_mb', 'qsrs.min_log_bytes_used_mb', 1, 'new', 1, 1, 'N0'), + (1340, 'log_bytes', 'max', 'max_log_bytes_used_mb', 'qsrs.max_log_bytes_used_mb', 1, 'new', 1, 0, 'N0'), + /* Hash totals for log bytes */ + (1315, 'log_bytes', 'total_hash', 'total_log_bytes_used_mb_by_query_hash', 'SUM(qsrs.total_log_bytes_used_mb) OVER (PARTITION BY qsq.query_hash ORDER BY qsq.query_hash)', 1, 'new', 1, 0, 'N0'), + /* TempDB usage */ + (1400, 'tempdb', 'avg', 'avg_tempdb_space_used_mb', 'qsrs.avg_tempdb_space_used_mb', 1, 'new', 1, 0, 'N0'), + (1410, 'tempdb', 'total', 'total_tempdb_space_used_mb', 'qsrs.total_tempdb_space_used_mb', 1, 'new', 1, 0, 'N0'), + (1420, 'tempdb', 'last', 'last_tempdb_space_used_mb', 'qsrs.last_tempdb_space_used_mb', 1, 'new', 1, 1, 'N0'), + (1430, 'tempdb', 'min', 'min_tempdb_space_used_mb', 'qsrs.min_tempdb_space_used_mb', 1, 'new', 1, 1, 'N0'), + (1440, 'tempdb', 'max', 'max_tempdb_space_used_mb', 'qsrs.max_tempdb_space_used_mb', 1, 'new', 1, 0, 'N0'), + /* Hash totals for tempdb */ + (1415, 'tempdb', 'total_hash', 'total_tempdb_space_used_mb_by_query_hash', 'SUM(qsrs.total_tempdb_space_used_mb) OVER (PARTITION BY qsq.query_hash ORDER BY qsq.query_hash)', 1, 'new', 1, 0, 'N0'), + /* Context settings and sorting columns */ + (1500, 'metadata', 'context', 'context_settings', 'qsrs.context_settings', 0, NULL, NULL, 0, NULL); + +/* Add special sorting columns based on @sort_order */ +/* Plan hash count for 'plan count by hashes' sort */ +IF @sort_order = 'plan count by hashes' +BEGIN + INSERT INTO + @ColumnDefinitions (column_id, metric_group, metric_type, column_name, column_source, is_conditional, condition_param, condition_value, expert_only, format_pattern) + VALUES + (1600, 'sort_order', 'plan_hash_count', 'plan_hash_count_for_query_hash', 'hashes.plan_hash_count_for_query_hash', 0, NULL, NULL, 0, 'N0'), + (1610, 'sort_order', 'query_hash', 'query_hash_from_hash_counting', 'hashes.query_hash', 0, NULL, NULL, 0, NULL); +END + +/* Dynamic regression change column based on formatting and comparator */ +IF @regression_baseline_start_date IS NOT NULL AND @regression_comparator = 'relative' AND @format_output = 1 +BEGIN + INSERT INTO + @ColumnDefinitions (column_id, metric_group, metric_type, column_name, column_source, is_conditional, condition_param, condition_value, expert_only, format_pattern) + VALUES (160, 'regression', 'change', 'change_in_average_for_query_hash_since_regression_time_period', 'regression.change_since_regression_time_period', 1, 'regression_mode', 1, 0, 'P2'); +END +ELSE IF @regression_baseline_start_date IS NOT NULL AND @format_output = 1 +BEGIN + INSERT INTO + @ColumnDefinitions (column_id, metric_group, metric_type, column_name, column_source, is_conditional, condition_param, condition_value, expert_only, format_pattern) + VALUES (160, 'regression', 'change', 'change_in_average_for_query_hash_since_regression_time_period', 'regression.change_since_regression_time_period', 1, 'regression_mode', 1, 0, 'N2'); +END +ELSE IF @regression_baseline_start_date IS NOT NULL +BEGIN + INSERT INTO + @ColumnDefinitions (column_id, metric_group, metric_type, column_name, column_source, is_conditional, condition_param, condition_value, expert_only, format_pattern) + VALUES (160, 'regression', 'change', 'change_in_average_for_query_hash_since_regression_time_period', 'regression.change_since_regression_time_period', 1, 'regression_mode', 1, 0, NULL); +END + +/* Wait time for wait-based sorting */ +IF LOWER(@sort_order) LIKE N'%waits' +BEGIN + INSERT INTO + @ColumnDefinitions (column_id, metric_group, metric_type, column_name, column_source, is_conditional, condition_param, condition_value, expert_only, format_pattern) + VALUES + (1620, 'sort_order', 'wait_time', 'total_wait_time_from_sort_order_ms', 'waits.total_query_wait_time_ms', 0, NULL, NULL, 0, 'N0'); +END + +/* ROW_NUMBER window function for sorting */ +INSERT INTO + @ColumnDefinitions (column_id, metric_group, metric_type, column_name, column_source, is_conditional, condition_param, condition_value, expert_only, format_pattern) +VALUES + ( + 2000, + 'metadata', + 'n', + 'n', + 'ROW_NUMBER() OVER (PARTITION BY qsrs.plan_id ORDER BY ' + + CASE WHEN @regression_baseline_start_date IS NOT NULL THEN + /* As seen when populating #regression_changes */ + CASE @regression_direction + WHEN 'regressed' THEN 'regression.change_since_regression_time_period' + WHEN 'worse' THEN 'regression.change_since_regression_time_period' + WHEN 'improved' THEN 'regression.change_since_regression_time_period * -1.0' + WHEN 'better' THEN 'regression.change_since_regression_time_period * -1.0' + WHEN 'magnitude' THEN 'ABS(regression.change_since_regression_time_period)' + WHEN 'absolute' THEN 'ABS(regression.change_since_regression_time_period)' + END + ELSE + CASE @sort_order + WHEN 'cpu' THEN 'qsrs.avg_cpu_time_ms' + WHEN 'logical reads' THEN 'qsrs.avg_logical_io_reads_mb' + WHEN 'physical reads' THEN 'qsrs.avg_physical_io_reads_mb' + WHEN 'writes' THEN 'qsrs.avg_logical_io_writes_mb' + WHEN 'duration' THEN 'qsrs.avg_duration_ms' + WHEN 'memory' THEN 'qsrs.avg_query_max_used_memory_mb' + WHEN 'tempdb' THEN 'qsrs.avg_tempdb_space_used_mb' /*This gets validated later*/ + WHEN 'executions' THEN 'qsrs.count_executions' + WHEN 'recent' THEN 'qsrs.last_execution_time' + WHEN 'rows' THEN 'qsrs.avg_rowcount' + WHEN 'plan count by hashes' THEN 'hashes.plan_hash_count_for_query_hash DESC, hashes.query_hash' + ELSE CASE WHEN LOWER(@sort_order) LIKE N'%waits' THEN 'waits.total_query_wait_time_ms' + ELSE 'qsrs.avg_cpu_time' END + END + END + ' DESC)', + 0, + NULL, + NULL, + 0, + NULL + ); + /* Try to be helpful by subbing in a database name if null */ @@ -1297,13 +1529,13 @@ DECLARE @procedure_name_quoted nvarchar(1024), @collation sysname, @new bit, - @sql nvarchar(MAX), - @isolation_level nvarchar(MAX), + @sql nvarchar(max), + @isolation_level nvarchar(max), @parameters nvarchar(4000), @plans_top bigint, @queries_top bigint, @nc10 nvarchar(2), - @where_clause nvarchar(MAX), + @where_clause nvarchar(max), @query_text_search_original_value nvarchar(4000), @query_text_search_not_original_value nvarchar(4000), @procedure_exists bit, @@ -1315,9 +1547,9 @@ DECLARE @string_split_ints nvarchar(1500), @string_split_strings nvarchar(1500), @current_table nvarchar(100), - @troubleshoot_insert nvarchar(MAX), - @troubleshoot_update nvarchar(MAX), - @troubleshoot_info nvarchar(MAX), + @troubleshoot_insert nvarchar(max), + @troubleshoot_update nvarchar(max), + @troubleshoot_info nvarchar(max), @rc bigint, @em tinyint, @fo tinyint, @@ -1332,7 +1564,8 @@ DECLARE @regression_baseline_start_date_original datetimeoffset(7), @regression_baseline_end_date_original datetimeoffset(7), @regression_mode bit, - @regression_where_clause nvarchar(max); + @regression_where_clause nvarchar(max), + @column_sql NVARCHAR(max) /* In cases where we are escaping @query_text_search and @@ -2727,7 +2960,7 @@ END; Checks if the sort order is for a wait. Cuts out a lot of repetition. */ -IF @sort_order IN +IF LOWER(@sort_order) IN ( 'cpu waits', 'lock waits', @@ -2747,7 +2980,6 @@ IF @sort_order IN 'total waits' ) BEGIN - SELECT @sort_order_is_a_wait = 1; END; @@ -8056,418 +8288,32 @@ BEGIN @sql += CONVERT ( - nvarchar(MAX), + nvarchar(max), N' SELECT x.* FROM -(' - ); - - /* - Expert mode returns more columns from runtime stats - */ - IF - ( - @expert_mode = 1 - AND @format_output = 0 - ) - BEGIN - SELECT - @sql += - CONVERT - ( - nvarchar(MAX), - N' - SELECT - source = - ''runtime_stats'', - database_name = - DB_NAME(qsrs.database_id), - qsp.query_id, - qsrs.plan_id, - qsp.all_plan_ids,' - + - CASE - WHEN @include_plan_hashes IS NOT NULL - THEN - N' - qsp.query_plan_hash,' - WHEN @include_query_hashes IS NOT NULL - OR @sort_order = 'plan count by hashes' - OR @include_query_hash_totals = 1 - THEN - N' - qsq.query_hash,' - WHEN @include_sql_handles IS NOT NULL - THEN - N' - qsqt.statement_sql_handle,' - ELSE - N'' - END + N' - qsrs.execution_type_desc, - qsq.object_name, - qsqt.query_sql_text, - query_plan = - CASE - WHEN TRY_CAST(qsp.query_plan AS xml) IS NOT NULL - THEN TRY_CAST(qsp.query_plan AS xml) - WHEN TRY_CAST(qsp.query_plan AS xml) IS NULL - THEN - ( - SELECT - [processing-instruction(query_plan)] = - N''-- '' + NCHAR(13) + NCHAR(10) + - N''-- This is a huge query plan.'' + NCHAR(13) + NCHAR(10) + - N''-- Remove the headers and footers, save it as a .sqlplan file, and re-open it.'' + NCHAR(13) + NCHAR(10) + - NCHAR(13) + NCHAR(10) + - REPLACE(qsp.query_plan, N'' 0 + BEGIN + SET @column_sql = + LEFT ( - nvarchar(MAX), - N' - qsrs.context_settings, - n = - ROW_NUMBER() OVER - ( - PARTITION BY - qsrs.plan_id - ORDER BY - ' - + - CASE WHEN @regression_mode = 1 THEN - /* As seen when populating #regression_changes. */ - CASE @regression_direction - WHEN 'regressed' THEN N'regression.change_since_regression_time_period' - WHEN 'worse' THEN N'regression.change_since_regression_time_period' - WHEN 'improved' THEN N'regression.change_since_regression_time_period * -1.0' - WHEN 'better' THEN N'regression.change_since_regression_time_period * -1.0' - WHEN 'magnitude' THEN N'ABS(regression.change_since_regression_time_period)' - WHEN 'absolute' THEN N'ABS(regression.change_since_regression_time_period)' - END - ELSE - CASE @sort_order - WHEN 'cpu' THEN N'qsrs.avg_cpu_time_ms' - WHEN 'logical reads' THEN N'qsrs.avg_logical_io_reads_mb' - WHEN 'physical reads' THEN N'qsrs.avg_physical_io_reads_mb' - WHEN 'writes' THEN N'qsrs.avg_logical_io_writes_mb' - WHEN 'duration' THEN N'qsrs.avg_duration_ms' - WHEN 'memory' THEN N'qsrs.avg_query_max_used_memory_mb' - WHEN 'tempdb' THEN CASE WHEN @new = 1 THEN N'qsrs.avg_tempdb_space_used_mb' ELSE N'qsrs.avg_cpu_time' END - WHEN 'executions' THEN N'qsrs.count_executions' - WHEN 'recent' THEN N'qsrs.last_execution_time' - WHEN 'rows' THEN N'qsrs.avg_rowcount' - WHEN 'plan count by hashes' THEN N'hashes.plan_hash_count_for_query_hash DESC, - hashes.query_hash' - ELSE CASE WHEN @sort_order_is_a_wait = 1 THEN N'waits.total_query_wait_time_ms' - ELSE N'qsrs.avg_cpu_time' END - END - END + N' DESC - )' - /* - Bolt any special sorting columns on, because we need them to - be in scope for sorting. - Has the helpful side-effect of making them visible - in the final output, because our SELECT is just x.*. - However, we must format them where applicable. - */ - + CASE - WHEN @sort_order = 'plan count by hashes' - THEN N', - plan_hash_count_for_query_hash = FORMAT(hashes.plan_hash_count_for_query_hash, ''N0''), - query_hash_from_hash_counting = hashes.query_hash' - WHEN @sort_order_is_a_wait = 1 - THEN N', - total_wait_time_from_sort_order_ms = FORMAT(waits.total_query_wait_time_ms, ''N0'')' - ELSE N'' - END - ) - ); - END; /*End expert mode = 0, format output = 1*/ + @column_sql, + LEN(@column_sql) - 1 + ); + END; + + /* Append the column SQL to the main SQL */ + SELECT + @sql += @column_sql; /* Add on the from and stuff @@ -9506,7 +8424,7 @@ FROM @sql += CONVERT ( - nvarchar(MAX), + nvarchar(max), N' FROM #query_store_runtime_stats AS qsrs' ); @@ -9552,7 +8470,7 @@ SELECT @sql += CONVERT ( - NVARCHAR(MAX), + NVARCHAR(max), N' CROSS APPLY ( @@ -9614,7 +8532,7 @@ SELECT @sql += CONVERT ( - nvarchar(MAX), + nvarchar(max), N' CROSS APPLY ( @@ -9669,7 +8587,7 @@ SELECT @sql += CONVERT ( - nvarchar(MAX), + nvarchar(max), N' CROSS APPLY ( @@ -9717,7 +8635,7 @@ SELECT @sql += CONVERT ( - nvarchar(MAX), + nvarchar(max), N' ) AS x ' + CASE WHEN @regression_mode = 1 THEN N' ' ELSE N'WHERE x.n = 1 ' END @@ -10344,7 +9262,7 @@ BEGIN @sql += CONVERT ( - nvarchar(MAX), + nvarchar(max), N' SELECT source = @@ -10392,7 +9310,7 @@ BEGIN @sql += CONVERT ( - nvarchar(MAX), + nvarchar(max), N' dqso.size_based_cleanup_mode_desc FROM #database_query_store_options AS dqso @@ -10968,7 +9886,7 @@ BEGIN @sql += CONVERT ( - nvarchar(MAX), + nvarchar(max), N' SELECT source = From 66e4d4d13ae24ebb1a42ab49697291415ad4d34f Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 4 Mar 2025 22:19:02 -0500 Subject: [PATCH 002/246] Update sp_QuickieStore.sql some sql prompt magic --- sp_QuickieStore/sp_QuickieStore.sql | 132 ++++++++++++++-------------- 1 file changed, 66 insertions(+), 66 deletions(-) diff --git a/sp_QuickieStore/sp_QuickieStore.sql b/sp_QuickieStore/sp_QuickieStore.sql index ff2397a6..596a0389 100644 --- a/sp_QuickieStore/sp_QuickieStore.sql +++ b/sp_QuickieStore/sp_QuickieStore.sql @@ -839,12 +839,12 @@ CREATE TABLE database_id int NOT NULL, plan_id bigint NOT NULL, query_id bigint NOT NULL, - all_plan_ids varchar(max), + all_plan_ids varchar(MAX), plan_group_id bigint NULL, engine_version nvarchar(32) NULL, compatibility_level smallint NOT NULL, query_plan_hash binary(8) NOT NULL, - query_plan nvarchar(max) NULL, + query_plan nvarchar(MAX) NULL, is_online_index_plan bit NOT NULL, is_trivial_plan bit NOT NULL, is_parallel_plan bit NOT NULL, @@ -1154,7 +1154,7 @@ CREATE TABLE plan_feedback_id bigint, plan_id bigint, feature_desc nvarchar(120), - feedback_data nvarchar(max), + feedback_data nvarchar(MAX), state_desc nvarchar(120), create_time datetimeoffset(7), last_updated_time datetimeoffset(7) @@ -1169,7 +1169,7 @@ CREATE TABLE database_id int NOT NULL, query_hint_id bigint, query_id bigint, - query_hint_text nvarchar(max), + query_hint_text nvarchar(MAX), last_query_hint_failure_reason_desc nvarchar(256), query_hint_failure_count bigint, source_desc nvarchar(256) @@ -1251,7 +1251,7 @@ DECLARE metric_group nvarchar(50) NOT NULL, /* Grouping (duration, cpu, etc.) */ metric_type nvarchar(20) NOT NULL, /* Type within group (avg, total, last, min, max) */ column_name nvarchar(100) NOT NULL, /* Column name as it appears in output */ - column_source nvarchar(max) NOT NULL, /* Source expression or formula */ + column_source nvarchar(MAX) NOT NULL, /* Source expression or formula */ is_conditional bit NOT NULL, /* Is this a conditional column (depends on a parameter) */ condition_param nvarchar(50) NULL, /* Parameter name this column depends on */ condition_value sql_variant NULL, /* Value the parameter must have */ @@ -1399,7 +1399,7 @@ BEGIN VALUES (1600, 'sort_order', 'plan_hash_count', 'plan_hash_count_for_query_hash', 'hashes.plan_hash_count_for_query_hash', 0, NULL, NULL, 0, 'N0'), (1610, 'sort_order', 'query_hash', 'query_hash_from_hash_counting', 'hashes.query_hash', 0, NULL, NULL, 0, NULL); -END +END; /* Dynamic regression change column based on formatting and comparator */ IF @regression_baseline_start_date IS NOT NULL AND @regression_comparator = 'relative' AND @format_output = 1 @@ -1407,19 +1407,19 @@ BEGIN INSERT INTO @ColumnDefinitions (column_id, metric_group, metric_type, column_name, column_source, is_conditional, condition_param, condition_value, expert_only, format_pattern) VALUES (160, 'regression', 'change', 'change_in_average_for_query_hash_since_regression_time_period', 'regression.change_since_regression_time_period', 1, 'regression_mode', 1, 0, 'P2'); -END +END; ELSE IF @regression_baseline_start_date IS NOT NULL AND @format_output = 1 BEGIN INSERT INTO @ColumnDefinitions (column_id, metric_group, metric_type, column_name, column_source, is_conditional, condition_param, condition_value, expert_only, format_pattern) VALUES (160, 'regression', 'change', 'change_in_average_for_query_hash_since_regression_time_period', 'regression.change_since_regression_time_period', 1, 'regression_mode', 1, 0, 'N2'); -END +END; ELSE IF @regression_baseline_start_date IS NOT NULL BEGIN INSERT INTO @ColumnDefinitions (column_id, metric_group, metric_type, column_name, column_source, is_conditional, condition_param, condition_value, expert_only, format_pattern) VALUES (160, 'regression', 'change', 'change_in_average_for_query_hash_since_regression_time_period', 'regression.change_since_regression_time_period', 1, 'regression_mode', 1, 0, NULL); -END +END; /* Wait time for wait-based sorting */ IF LOWER(@sort_order) LIKE N'%waits' @@ -1428,7 +1428,7 @@ BEGIN @ColumnDefinitions (column_id, metric_group, metric_type, column_name, column_source, is_conditional, condition_param, condition_value, expert_only, format_pattern) VALUES (1620, 'sort_order', 'wait_time', 'total_wait_time_from_sort_order_ms', 'waits.total_query_wait_time_ms', 0, NULL, NULL, 0, 'N0'); -END +END; /* ROW_NUMBER window function for sorting */ INSERT INTO @@ -1529,13 +1529,13 @@ DECLARE @procedure_name_quoted nvarchar(1024), @collation sysname, @new bit, - @sql nvarchar(max), - @isolation_level nvarchar(max), + @sql nvarchar(MAX), + @isolation_level nvarchar(MAX), @parameters nvarchar(4000), @plans_top bigint, @queries_top bigint, @nc10 nvarchar(2), - @where_clause nvarchar(max), + @where_clause nvarchar(MAX), @query_text_search_original_value nvarchar(4000), @query_text_search_not_original_value nvarchar(4000), @procedure_exists bit, @@ -1547,9 +1547,9 @@ DECLARE @string_split_ints nvarchar(1500), @string_split_strings nvarchar(1500), @current_table nvarchar(100), - @troubleshoot_insert nvarchar(max), - @troubleshoot_update nvarchar(max), - @troubleshoot_info nvarchar(max), + @troubleshoot_insert nvarchar(MAX), + @troubleshoot_update nvarchar(MAX), + @troubleshoot_info nvarchar(MAX), @rc bigint, @em tinyint, @fo tinyint, @@ -1564,8 +1564,8 @@ DECLARE @regression_baseline_start_date_original datetimeoffset(7), @regression_baseline_end_date_original datetimeoffset(7), @regression_mode bit, - @regression_where_clause nvarchar(max), - @column_sql NVARCHAR(max) + @regression_where_clause nvarchar(MAX), + @column_sql nvarchar(MAX); /* In cases where we are escaping @query_text_search and @@ -1859,7 +1859,7 @@ BEGIN AND d.is_in_standby = 0 AND d.is_read_only = 0 OPTION(RECOMPILE); -END +END; ELSE BEGIN INSERT @@ -2380,7 +2380,7 @@ BEGIN BEGIN IF @debug = 1 BEGIN - GOTO DEBUG + GOTO DEBUG; END; ELSE BEGIN @@ -2401,7 +2401,7 @@ BEGIN RAISERROR('Not all Azure offerings are supported, please try avoiding memes', 11, 1) WITH NOWAIT; IF @debug = 1 BEGIN - GOTO DEBUG + GOTO DEBUG; END; ELSE BEGIN @@ -2428,7 +2428,7 @@ BEGIN RAISERROR('Azure databases in compatibility levels under 130 are not supported', 11, 1) WITH NOWAIT; IF @debug = 1 BEGIN - GOTO DEBUG + GOTO DEBUG; END; ELSE BEGIN @@ -2517,7 +2517,7 @@ BEGIN BEGIN IF @debug = 1 BEGIN - GOTO DEBUG + GOTO DEBUG; END; ELSE BEGIN @@ -2649,7 +2649,7 @@ BEGIN IF @procedure_schema IS NULL BEGIN SELECT - @procedure_schema = N'dbo' + @procedure_schema = N'dbo'; END; SELECT @current_table = 'checking procedure existence', @@ -2890,7 +2890,7 @@ Check that you spelled everything correctly and you''re in the right database', BEGIN IF @debug = 1 BEGIN - GOTO DEBUG + GOTO DEBUG; END; ELSE BEGIN @@ -3043,7 +3043,7 @@ BEGIN BEGIN IF @debug = 1 BEGIN - GOTO DEBUG + GOTO DEBUG; END; ELSE BEGIN @@ -3067,7 +3067,7 @@ BEGIN BEGIN IF @debug = 1 BEGIN - GOTO DEBUG + GOTO DEBUG; END; ELSE BEGIN @@ -3108,7 +3108,7 @@ BEGIN BEGIN IF @debug = 1 BEGIN - GOTO DEBUG + GOTO DEBUG; END; ELSE BEGIN @@ -3191,7 +3191,7 @@ OPTION(RECOMPILE);' + @nc10; IF @debug = 1 BEGIN RAISERROR('Query Store wait stats are not enabled for database %s', 10, 1, @database_name_quoted) WITH NOWAIT; - END + END; END; END; /*End wait stats checks*/ @@ -3209,7 +3209,7 @@ BEGIN RAISERROR('The time zone you chose (%s) is not valid. Please check sys.time_zone_info for a valid list.', 10, 1, @timezone) WITH NOWAIT; IF @debug = 1 BEGIN - GOTO DEBUG + GOTO DEBUG; END; ELSE BEGIN @@ -3251,7 +3251,7 @@ BEGIN ELSE 0 END OPTION(RECOMPILE); - END + END; END; /* @@ -3409,12 +3409,12 @@ SELECT DISTINCT FROM ' + @database_name_quoted + N'.sys.query_store_query AS qsq JOIN ' + @database_name_quoted + N'.sys.query_store_plan AS qsp ON qsq.query_id = qsp.query_id -WHERE ' +WHERE '; IF CHARINDEX(N'%', @procedure_name) = 0 BEGIN SELECT - @sql += N'qsq.object_id = OBJECT_ID(@procedure_name_quoted)' + @sql += N'qsq.object_id = OBJECT_ID(@procedure_name_quoted)'; END; IF CHARINDEX(N'%', @procedure_name) > 0 @@ -3426,7 +3426,7 @@ BEGIN 1/0 FROM #procedure_object_ids AS poi WHERE poi.[object_id] = qsq.[object_id] -)' +)'; END; SELECT @@ -4565,7 +4565,7 @@ BEGIN plan_id ) EXECUTE sys.sp_executesql - @sql + @sql; IF @troubleshoot_performance = 1 BEGIN @@ -4639,7 +4639,7 @@ BEGIN plan_id ) EXECUTE sys.sp_executesql - @sql + @sql; IF @troubleshoot_performance = 1 BEGIN @@ -4713,7 +4713,7 @@ BEGIN plan_id ) EXECUTE sys.sp_executesql - @sql + @sql; IF @troubleshoot_performance = 1 BEGIN @@ -4774,8 +4774,8 @@ IF @only_queries_with_forced_plan_failures = 1 BEGIN SELECT @sql += N' -AND qsp.last_force_failure_reason > 0' -END +AND qsp.last_force_failure_reason > 0'; +END; SELECT @sql += N' @@ -4793,7 +4793,7 @@ OPTION(RECOMPILE);' + @nc10; plan_id ) EXECUTE sys.sp_executesql - @sql + @sql; IF @troubleshoot_performance = 1 BEGIN @@ -6211,7 +6211,7 @@ BEGIN FROM #regression_changes WHERE database_id = @database_id OPTION(RECOMPILE);' + @nc10; -END +END; ELSE IF @sort_order = 'plan count by hashes' BEGIN SELECT @@ -6221,7 +6221,7 @@ BEGIN FROM #plan_ids_with_query_hashes WHERE database_id = @database_id OPTION(RECOMPILE);' + @nc10; -END +END; ELSE IF @sort_order_is_a_wait = 1 BEGIN SELECT @@ -6231,7 +6231,7 @@ BEGIN FROM #plan_ids_with_total_waits WHERE database_id = @database_id OPTION(RECOMPILE);' + @nc10; -END +END; ELSE BEGIN SELECT @@ -6424,7 +6424,7 @@ BEGIN THEN ''No'' ELSE ''Yes'' END,'; -END +END; ELSE BEGIN SELECT @@ -6580,14 +6580,14 @@ BEGIN qsrs.runtime_stats_interval_id DESC ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING )'; -END +END; IF @new = 0 BEGIN SELECT @sql += N' - not_used = NULL' -END + not_used = NULL'; +END; SELECT @sql += N' @@ -6595,27 +6595,27 @@ SELECT CROSS APPLY ( SELECT TOP (@queries_top) - qsrs.*' + qsrs.*'; SELECT @sql += N' - FROM ' + @database_name_quoted + N'.sys.query_store_runtime_stats AS qsrs' + FROM ' + @database_name_quoted + N'.sys.query_store_runtime_stats AS qsrs'; IF @regression_mode = 1 BEGIN SELECT @sql += N' JOIN #regression_changes AS regression ON qsrs.plan_id = regression.plan_id - AND regression.database_id = @database_id' - END + AND regression.database_id = @database_id'; + END; ELSE IF @sort_order = 'plan count by hashes' BEGIN SELECT @sql += N' JOIN #plan_ids_with_query_hashes AS hashes ON qsrs.plan_id = hashes.plan_id - AND hashes.database_id = @database_id' - END + AND hashes.database_id = @database_id'; + END; ELSE IF @sort_order_is_a_wait = 1 BEGIN /* @@ -6629,7 +6629,7 @@ SELECT @sql += N' JOIN #plan_ids_with_total_waits AS waits ON qsrs.plan_id = waits.plan_id - AND waits.database_id = @database_id' + AND waits.database_id = @database_id'; END; SELECT @@ -8288,7 +8288,7 @@ BEGIN @sql += CONVERT ( - nvarchar(max), + nvarchar(MAX), N' SELECT x.* @@ -8424,7 +8424,7 @@ FROM @sql += CONVERT ( - nvarchar(max), + nvarchar(MAX), N' FROM #query_store_runtime_stats AS qsrs' ); @@ -8462,7 +8462,7 @@ FROM BEGIN SELECT @sql += N' - AND qsrs.from_regression_baseline = waits.from_regression_baseline' + AND qsrs.from_regression_baseline = waits.from_regression_baseline'; END; END; @@ -8470,7 +8470,7 @@ SELECT @sql += CONVERT ( - NVARCHAR(max), + nvarchar(MAX), N' CROSS APPLY ( @@ -8532,7 +8532,7 @@ SELECT @sql += CONVERT ( - nvarchar(max), + nvarchar(MAX), N' CROSS APPLY ( @@ -8587,7 +8587,7 @@ SELECT @sql += CONVERT ( - nvarchar(max), + nvarchar(MAX), N' CROSS APPLY ( @@ -8635,7 +8635,7 @@ SELECT @sql += CONVERT ( - nvarchar(max), + nvarchar(MAX), N' ) AS x ' + CASE WHEN @regression_mode = 1 THEN N' ' ELSE N'WHERE x.n = 1 ' END @@ -9262,7 +9262,7 @@ BEGIN @sql += CONVERT ( - nvarchar(max), + nvarchar(MAX), N' SELECT source = @@ -9310,7 +9310,7 @@ BEGIN @sql += CONVERT ( - nvarchar(max), + nvarchar(MAX), N' dqso.size_based_cleanup_mode_desc FROM #database_query_store_options AS dqso @@ -9405,7 +9405,7 @@ BEGIN SELECT result = '#query_store_plan_feedback is empty'; END; - END + END; IF EXISTS ( @@ -9886,7 +9886,7 @@ BEGIN @sql += CONVERT ( - nvarchar(max), + nvarchar(MAX), N' SELECT source = @@ -10416,7 +10416,7 @@ BEGIN poi.* FROM #procedure_object_ids AS poi ORDER BY - poi.[object_id] + poi.object_id OPTION(RECOMPILE); END; ELSE From 81f804398648108e35f384164f23bb8fee6d56f2 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Wed, 5 Mar 2025 17:16:09 -0500 Subject: [PATCH 003/246] Update sp_QuickieStore.sql While we're feeling ambitious, let's replace all that include/ignore code with much more dynamic sql. --- sp_QuickieStore/sp_QuickieStore.sql | 1492 +++++++++------------------ 1 file changed, 495 insertions(+), 997 deletions(-) diff --git a/sp_QuickieStore/sp_QuickieStore.sql b/sp_QuickieStore/sp_QuickieStore.sql index 596a0389..013e7a46 100644 --- a/sp_QuickieStore/sp_QuickieStore.sql +++ b/sp_QuickieStore/sp_QuickieStore.sql @@ -1474,6 +1474,66 @@ VALUES NULL ); +/* Create a table variable to define parameter processing */ +DECLARE + @FilterParameters table +( + parameter_name nvarchar(100) NOT NULL, + parameter_value nvarchar(4000) NOT NULL, + temp_table_name sysname NOT NULL, + column_name sysname NOT NULL, + data_type sysname NOT NULL, + is_include bit NOT NULL, + requires_secondary_processing bit NOT NULL +); + +/* Populate with parameter definitions*/ +INSERT INTO + @FilterParameters +( + parameter_name, + parameter_value, + temp_table_name, + column_name, + data_type, + is_include, + requires_secondary_processing +) +SELECT + v.parameter_name, + v.parameter_value, + v.temp_table_name, + v.column_name, + v.data_type, + v.is_include, + v.requires_secondary_processing +FROM +( + VALUES + /* Include parameters */ + ('include_plan_ids', @include_plan_ids, '#include_plan_ids', 'plan_id', 'bigint', 1, 0), + ('include_query_ids', @include_query_ids, '#include_query_ids', 'query_id', 'bigint', 1, 1), + ('include_query_hashes', @include_query_hashes, '#include_query_hashes', 'query_hash_s', 'varchar', 1, 1), + ('include_plan_hashes', @include_plan_hashes, '#include_plan_hashes', 'plan_hash_s', 'varchar', 1, 1), + ('include_sql_handles', @include_sql_handles, '#include_sql_handles', 'sql_handle_s', 'varchar', 1, 1), + /* Ignore parameters */ + ('ignore_plan_ids', @ignore_plan_ids, '#ignore_plan_ids', 'plan_id', 'bigint', 0, 0), + ('ignore_query_ids', @ignore_query_ids, '#ignore_query_ids', 'query_id', 'bigint', 0, 1), + ('ignore_query_hashes', @ignore_query_hashes, '#ignore_query_hashes', 'query_hash_s', 'varchar', 0, 1), + ('ignore_plan_hashes', @ignore_plan_hashes, '#ignore_plan_hashes', 'plan_hash_s', 'varchar', 0, 1), + ('ignore_sql_handles', @ignore_sql_handles, '#ignore_sql_handles', 'sql_handle_s', 'varchar', 0, 1) + ) AS v + ( + parameter_name, + parameter_value, + temp_table_name, + column_name, + data_type, + is_include, + requires_secondary_processing + ) +WHERE v.parameter_value IS NOT NULL; + /* Try to be helpful by subbing in a database name if null */ @@ -1565,7 +1625,15 @@ DECLARE @regression_baseline_end_date_original datetimeoffset(7), @regression_mode bit, @regression_where_clause nvarchar(MAX), - @column_sql nvarchar(MAX); + @column_sql nvarchar(MAX), + @param_name nvarchar(100), + @param_value nvarchar(4000), + @temp_table sysname, + @column_name sysname, + @data_type sysname, + @is_include bit, + @requires_secondary_processing bit, + @split_sql nvarchar(MAX); /* In cases where we are escaping @query_text_search and @@ -1903,11 +1971,12 @@ DECLARE @database_cursor CURSOR; SET - @database_cursor = CURSOR - LOCAL - SCROLL - DYNAMIC - READ_ONLY + @database_cursor = + CURSOR + LOCAL + SCROLL + DYNAMIC + READ_ONLY FOR SELECT d.database_name @@ -3488,1034 +3557,403 @@ BEGIN @sql = @isolation_level; IF @troubleshoot_performance = 1 - BEGIN - EXECUTE sys.sp_executesql - @troubleshoot_insert, - N'@current_table nvarchar(100)', - @current_table; - - SET STATISTICS XML ON; - END; - - SELECT - @sql += N' -SELECT DISTINCT - qsp.plan_id -FROM ' + @database_name_quoted + N'.sys.query_store_query AS qsq -JOIN ' + @database_name_quoted + N'.sys.query_store_plan AS qsp - ON qsq.query_id = qsp.query_id -WHERE qsq.object_id ' + -CASE - WHEN LOWER(@query_type) LIKE 'a%' - THEN N'= 0' - ELSE N'<> 0' -END -+ N' -OPTION(RECOMPILE);' + @nc10; - - IF @debug = 1 - BEGIN - PRINT LEN(@sql); - PRINT @sql; - END; - - INSERT - #query_types WITH(TABLOCK) - ( - plan_id - ) - EXECUTE sys.sp_executesql - @sql; - - IF @troubleshoot_performance = 1 - BEGIN - SET STATISTICS XML OFF; - - EXECUTE sys.sp_executesql - @troubleshoot_update, - N'@current_table nvarchar(100)', - @current_table; - - EXECUTE sys.sp_executesql - @troubleshoot_info, - N'@sql nvarchar(max), - @current_table nvarchar(100)', - @sql, - @current_table; - END; - - SELECT - @where_clause += N'AND EXISTS - ( - SELECT - 1/0 - FROM #query_types AS qt - WHERE qt.plan_id = qsrs.plan_id - )' + @nc10; -END; /*End query type filter table population*/ - - -/* -This section filters query or plan ids, both inclusive and exclusive -*/ -IF -( - @include_plan_ids IS NOT NULL -OR @include_query_ids IS NOT NULL -OR @ignore_plan_ids IS NOT NULL -OR @ignore_query_ids IS NOT NULL -) -BEGIN - IF @include_plan_ids IS NOT NULL - BEGIN - SELECT - @include_plan_ids = - REPLACE(REPLACE(REPLACE(REPLACE( - LTRIM(RTRIM(@include_plan_ids)), - CHAR(10), N''), CHAR(13), N''), - NCHAR(10), N''), NCHAR(13), N''); - - SELECT - @current_table = 'inserting #include_plan_ids'; - - INSERT - #include_plan_ids WITH(TABLOCK) - ( - plan_id - ) - EXECUTE sys.sp_executesql - @string_split_ints, - N'@ids nvarchar(4000)', - @include_plan_ids; - - SELECT - @where_clause += N'AND EXISTS - ( - SELECT - 1/0 - FROM #include_plan_ids AS idi - WHERE idi.plan_id = qsrs.plan_id - )' + @nc10; - END; /*End include plan ids*/ - - IF @ignore_plan_ids IS NOT NULL - BEGIN - SELECT - @ignore_plan_ids = - REPLACE(REPLACE(REPLACE(REPLACE( - LTRIM(RTRIM(@ignore_plan_ids)), - CHAR(10), N''), CHAR(13), N''), - NCHAR(10), N''), NCHAR(13), N''); - - SELECT - @current_table = 'inserting #ignore_plan_ids'; - - INSERT - #ignore_plan_ids WITH(TABLOCK) - ( - plan_id - ) - EXECUTE sys.sp_executesql - @string_split_ints, - N'@ids nvarchar(4000)', - @ignore_plan_ids; - - SELECT - @where_clause += N'AND NOT EXISTS - ( - SELECT - 1/0 - FROM #ignore_plan_ids AS idi - WHERE idi.plan_id = qsrs.plan_id - )' + @nc10; - END; /*End ignore plan ids*/ - - IF @include_query_ids IS NOT NULL - BEGIN - SELECT - @include_query_ids = - REPLACE(REPLACE(REPLACE(REPLACE( - LTRIM(RTRIM(@include_query_ids)), - CHAR(10), N''), CHAR(13), N''), - NCHAR(10), N''), NCHAR(13), N''); - SELECT - @current_table = 'inserting #include_query_ids', - @sql = @isolation_level; - - INSERT - #include_query_ids WITH(TABLOCK) - ( - query_id - ) - EXECUTE sys.sp_executesql - @string_split_ints, - N'@ids nvarchar(4000)', - @include_query_ids; - - SELECT - @current_table = 'inserting #include_plan_ids for included query ids'; - - IF @troubleshoot_performance = 1 - BEGIN - EXECUTE sys.sp_executesql - @troubleshoot_insert, - N'@current_table nvarchar(100)', - @current_table; - - SET STATISTICS XML ON; - END; - - SELECT - @sql += N' -SELECT DISTINCT - qsp.plan_id -FROM ' + @database_name_quoted + N'.sys.query_store_plan AS qsp -WHERE EXISTS - ( - SELECT - 1/0 - FROM #include_query_ids AS iqi - WHERE iqi.query_id = qsp.query_id - ) -OPTION(RECOMPILE);' + @nc10; - - IF @debug = 1 - BEGIN - PRINT LEN(@sql); - PRINT @sql; - END; - - INSERT - #include_plan_ids - ( - plan_id - ) - EXECUTE sys.sp_executesql - @sql; - - IF @troubleshoot_performance = 1 - BEGIN - SET STATISTICS XML OFF; - - EXECUTE sys.sp_executesql - @troubleshoot_update, - N'@current_table nvarchar(100)', - @current_table; - - EXECUTE sys.sp_executesql - @troubleshoot_info, - N'@sql nvarchar(max), - @current_table nvarchar(100)', - @sql, - @current_table; - END; - - /* - This section of code confused me when I came back to it, - so I'm going to add a note here about why I do this: - - If @include_plan_ids is NULL at this point, it's because - the user didn't populate the parameter. - - We need to do this because it's how we figure - out which plans to keep in the main query - */ - IF @include_plan_ids IS NULL - BEGIN - SELECT - @where_clause += N'AND EXISTS - ( - SELECT - 1/0 - FROM #include_plan_ids AS idi - WHERE idi.plan_id = qsrs.plan_id - )' + @nc10; - END; - END; /*End include query ids*/ - - IF @ignore_query_ids IS NOT NULL - BEGIN - SELECT - @ignore_query_ids = - REPLACE(REPLACE(REPLACE(REPLACE( - LTRIM(RTRIM(@ignore_query_ids)), - CHAR(10), N''), CHAR(13), N''), - NCHAR(10), N''), NCHAR(13), N''); - SELECT - @current_table = 'inserting #ignore_query_ids', - @sql = @isolation_level; - - INSERT - #ignore_query_ids WITH(TABLOCK) - ( - query_id - ) - EXECUTE sys.sp_executesql - @string_split_ints, - N'@ids nvarchar(4000)', - @ignore_query_ids; - - SELECT - @current_table = 'inserting #ignore_plan_ids for ignored query ids'; - - IF @troubleshoot_performance = 1 - BEGIN - EXECUTE sys.sp_executesql - @troubleshoot_insert, - N'@current_table nvarchar(100)', - @current_table; - - SET STATISTICS XML ON; - END; - - SELECT - @sql += N' -SELECT DISTINCT - qsp.plan_id -FROM ' + @database_name_quoted + N'.sys.query_store_plan AS qsp -WHERE EXISTS - ( - SELECT - 1/0 - FROM #ignore_query_ids AS iqi - WHERE iqi.query_id = qsp.query_id - ) -OPTION(RECOMPILE);' + @nc10; - - IF @debug = 1 - BEGIN - PRINT LEN(@sql); - PRINT @sql; - END; - - INSERT - #ignore_plan_ids - ( - plan_id - ) - EXECUTE sys.sp_executesql - @sql; - - IF @troubleshoot_performance = 1 - BEGIN - SET STATISTICS XML OFF; - - EXECUTE sys.sp_executesql - @troubleshoot_update, - N'@current_table nvarchar(100)', - @current_table; - - EXECUTE sys.sp_executesql - @troubleshoot_info, - N'@sql nvarchar(max), - @current_table nvarchar(100)', - @sql, - @current_table; - END; - - /* - This section of code confused me when I came back to it, - so I'm going to add a note here about why I do this: - - If @ignore_plan_ids is NULL at this point, it's because - the user didn't populate the parameter. - - We need to do this because it's how we figure - out which plans to keep in the main query - */ - IF @ignore_plan_ids IS NULL - BEGIN - SELECT - @where_clause += N'AND NOT EXISTS - ( - SELECT - 1/0 - FROM #ignore_plan_ids AS idi - WHERE idi.plan_id = qsrs.plan_id - )' + @nc10; - END; - END; /*End ignore query ids*/ -END; /*End query and plan id filtering*/ - -/* -This section filters query or plan hashes -*/ -IF -( - @include_query_hashes IS NOT NULL -OR @include_plan_hashes IS NOT NULL -OR @include_sql_handles IS NOT NULL -OR @ignore_query_hashes IS NOT NULL -OR @ignore_plan_hashes IS NOT NULL -OR @ignore_sql_handles IS NOT NULL -) -BEGIN - IF @include_query_hashes IS NOT NULL - BEGIN - SELECT - @include_query_hashes = - REPLACE(REPLACE(REPLACE(REPLACE( - LTRIM(RTRIM(@include_query_hashes)), - CHAR(10), N''), CHAR(13), N''), - NCHAR(10), N''), NCHAR(13), N''); - - SELECT - @current_table = 'inserting #include_query_hashes', - @sql = @isolation_level; - - INSERT - #include_query_hashes WITH(TABLOCK) - ( - query_hash_s - ) - EXECUTE sys.sp_executesql - @string_split_strings, - N'@ids nvarchar(4000)', - @include_query_hashes; - - SELECT - @current_table = 'inserting #include_plan_ids for included query hashes'; - - IF @troubleshoot_performance = 1 - BEGIN - EXECUTE sys.sp_executesql - @troubleshoot_insert, - N'@current_table nvarchar(100)', - @current_table; - - SET STATISTICS XML ON; - END; - - SELECT - @sql += N' -SELECT DISTINCT - qsp.plan_id -FROM ' + @database_name_quoted + N'.sys.query_store_plan AS qsp -WHERE EXISTS - ( - SELECT - 1/0 - FROM ' + @database_name_quoted + N'.sys.query_store_query AS qsq - WHERE qsq.query_id = qsp.query_id - AND EXISTS - ( - SELECT - 1/0 - FROM #include_query_hashes AS iqh - WHERE iqh.query_hash = qsq.query_hash - ) - ) -OPTION(RECOMPILE);' + @nc10; - - IF @debug = 1 - BEGIN - PRINT LEN(@sql); - PRINT @sql; - END; - - INSERT - #include_plan_ids - ( - plan_id - ) - EXECUTE sys.sp_executesql - @sql; - - IF @troubleshoot_performance = 1 - BEGIN - SET STATISTICS XML OFF; - - EXECUTE sys.sp_executesql - @troubleshoot_update, - N'@current_table nvarchar(100)', - @current_table; - - EXECUTE sys.sp_executesql - @troubleshoot_info, - N'@sql nvarchar(max), - @current_table nvarchar(100)', - @sql, - @current_table; - END; - - /* - This section of code confused me when I came back to it, - so I'm going to add a note here about why I do this: - - If @include_plan_ids is NULL at this point, it's because - the user didn't populate the parameter. - - We need to do this because it's how we figure - out which plans to keep in the main query - */ - IF @include_plan_ids IS NULL - BEGIN - SELECT - @where_clause += N'AND EXISTS - ( - SELECT - 1/0 - FROM #include_plan_ids AS idi - WHERE idi.plan_id = qsrs.plan_id - )' + @nc10; - END; - END; /*End include query hashes*/ - - IF @ignore_query_hashes IS NOT NULL - BEGIN - SELECT - @ignore_query_hashes = - REPLACE(REPLACE(REPLACE(REPLACE( - LTRIM(RTRIM(@ignore_query_hashes)), - CHAR(10), N''), CHAR(13), N''), - NCHAR(10), N''), NCHAR(13), N''); - - SELECT - @current_table = 'inserting #ignore_query_hashes', - @sql = @isolation_level; - - INSERT - #ignore_query_hashes WITH(TABLOCK) - ( - query_hash_s - ) - EXECUTE sys.sp_executesql - @string_split_strings, - N'@ids nvarchar(4000)', - @ignore_query_hashes; - - SELECT - @current_table = 'inserting #ignore_plan_ids for ignored query hashes'; - - IF @troubleshoot_performance = 1 - BEGIN - EXECUTE sys.sp_executesql - @troubleshoot_insert, - N'@current_table nvarchar(100)', - @current_table; - - SET STATISTICS XML ON; - END; - - SELECT - @sql += N' -SELECT DISTINCT - qsp.plan_id -FROM ' + @database_name_quoted + N'.sys.query_store_plan AS qsp -WHERE EXISTS - ( - SELECT - 1/0 - FROM ' + @database_name_quoted + N'.sys.query_store_query AS qsq - WHERE qsq.query_id = qsp.query_id - AND EXISTS - ( - SELECT - 1/0 - FROM #ignore_query_hashes AS iqh - WHERE iqh.query_hash = qsq.query_hash - ) - ) -OPTION(RECOMPILE);' + @nc10; - - IF @debug = 1 - BEGIN - PRINT LEN(@sql); - PRINT @sql; - END; - - INSERT - #ignore_plan_ids - ( - plan_id - ) - EXECUTE sys.sp_executesql - @sql; - - IF @troubleshoot_performance = 1 - BEGIN - SET STATISTICS XML OFF; - - EXECUTE sys.sp_executesql - @troubleshoot_update, - N'@current_table nvarchar(100)', - @current_table; - - EXECUTE sys.sp_executesql - @troubleshoot_info, - N'@sql nvarchar(max), - @current_table nvarchar(100)', - @sql, - @current_table; - END; - - /* - This section of code confused me when I came back to it, - so I'm going to add a note here about why I do this: - - If @ignore_plan_ids is NULL at this point, it's because - the user didn't populate the parameter. - - We need to do this because it's how we figure - out which plans to keep in the main query - */ - IF @ignore_plan_ids IS NULL - BEGIN - SELECT - @where_clause += N'AND NOT EXISTS - ( - SELECT - 1/0 - FROM #ignore_plan_ids AS idi - WHERE idi.plan_id = qsrs.plan_id - )' + @nc10; - END; - END; /*End ignore query hashes*/ - - IF @include_plan_hashes IS NOT NULL - BEGIN - SELECT - @include_plan_hashes = - REPLACE(REPLACE(REPLACE(REPLACE( - LTRIM(RTRIM(@include_plan_hashes)), - CHAR(10), N''), CHAR(13), N''), - NCHAR(10), N''), NCHAR(13), N''); - - SELECT - @current_table = 'inserting #include_plan_hashes', - @sql = @isolation_level; - - INSERT - #include_plan_hashes WITH(TABLOCK) - ( - plan_hash_s - ) - EXECUTE sys.sp_executesql - @string_split_strings, - N'@ids nvarchar(4000)', - @include_plan_hashes; - - SELECT - @current_table = 'inserting #include_plan_ids for included plan hashes'; - - IF @troubleshoot_performance = 1 - BEGIN - EXECUTE sys.sp_executesql - @troubleshoot_insert, - N'@current_table nvarchar(100)', - @current_table; - - SET STATISTICS XML ON; - END; - - SELECT - @sql += N' -SELECT DISTINCT - qsp.plan_id -FROM ' + @database_name_quoted + N'.sys.query_store_plan AS qsp -WHERE EXISTS - ( - SELECT - 1/0 - FROM #include_plan_hashes AS iph - WHERE iph.plan_hash = qsp.query_plan_hash - ) -OPTION(RECOMPILE);' + @nc10; - - IF @debug = 1 - BEGIN - PRINT LEN(@sql); - PRINT @sql; - END; - - INSERT - #include_plan_ids - ( - plan_id - ) - EXECUTE sys.sp_executesql - @sql; - - IF @troubleshoot_performance = 1 - BEGIN - SET STATISTICS XML OFF; - - EXECUTE sys.sp_executesql - @troubleshoot_update, - N'@current_table nvarchar(100)', - @current_table; - - EXECUTE sys.sp_executesql - @troubleshoot_info, - N'@sql nvarchar(max), - @current_table nvarchar(100)', - @sql, - @current_table; - END; - - /* - This section of code confused me when I came back to it, - so I'm going to add a note here about why I do this: - - If @include_plan_ids is NULL at this point, it's because - the user didn't populate the parameter. - - We need to do this because it's how we figure - out which plans to keep in the main query - */ - IF @include_plan_ids IS NULL - BEGIN - SELECT - @where_clause += N'AND EXISTS - ( - SELECT - 1/0 - FROM #include_plan_ids AS idi - WHERE idi.plan_id = qsrs.plan_id - )' + @nc10; - END; - END; /*End include plan hashes*/ - - IF @ignore_plan_hashes IS NOT NULL - BEGIN - SELECT - @ignore_plan_hashes = - REPLACE(REPLACE(REPLACE(REPLACE( - LTRIM(RTRIM(@ignore_plan_hashes)), - CHAR(10), N''), CHAR(13), N''), - NCHAR(10), N''), NCHAR(13), N''); - - SELECT - @current_table = 'inserting #ignore_plan_hashes', - @sql = @isolation_level; - - INSERT - #ignore_plan_hashes WITH(TABLOCK) - ( - plan_hash_s - ) - EXECUTE sys.sp_executesql - @string_split_strings, - N'@ids nvarchar(4000)', - @ignore_plan_hashes; - - SELECT - @current_table = 'inserting #ignore_plan_ids for ignored query hashes'; - - IF @troubleshoot_performance = 1 - BEGIN - EXECUTE sys.sp_executesql - @troubleshoot_insert, - N'@current_table nvarchar(100)', - @current_table; - - SET STATISTICS XML ON; - END; - - SELECT - @sql += N' -SELECT DISTINCT - qsp.plan_id -FROM ' + @database_name_quoted + N'.sys.query_store_plan AS qsp -WHERE EXISTS - ( - SELECT - 1/0 - FROM #ignore_plan_hashes AS iph - WHERE iph.plan_hash = qsp.query_plan_hash - ) -OPTION(RECOMPILE);' + @nc10; - - IF @debug = 1 - BEGIN - PRINT LEN(@sql); - PRINT @sql; - END; - - INSERT - #ignore_plan_ids - ( - plan_id - ) - EXECUTE sys.sp_executesql - @sql; - - IF @troubleshoot_performance = 1 - BEGIN - SET STATISTICS XML OFF; - - EXECUTE sys.sp_executesql - @troubleshoot_update, - N'@current_table nvarchar(100)', - @current_table; - - EXECUTE sys.sp_executesql - @troubleshoot_info, - N'@sql nvarchar(max), - @current_table nvarchar(100)', - @sql, - @current_table; - END; - - /* - This section of code confused me when I came back to it, - so I'm going to add a note here about why I do this: - - If @ignore_plan_ids is NULL at this point, it's because - the user didn't populate the parameter. - - We need to do this because it's how we figure - out which plans to keep in the main query - */ - IF @ignore_plan_ids IS NULL - BEGIN - SELECT - @where_clause += N'AND NOT EXISTS - ( - SELECT - 1/0 - FROM #ignore_plan_ids AS idi - WHERE idi.plan_id = qsrs.plan_id - )' + @nc10; - END; - END; /*End ignore plan hashes*/ - - IF @include_sql_handles IS NOT NULL - BEGIN - SELECT - @include_sql_handles = - REPLACE(REPLACE(REPLACE(REPLACE( - LTRIM(RTRIM(@include_sql_handles)), - CHAR(10), N''), CHAR(13), N''), - NCHAR(10), N''), NCHAR(13), N''); - - SELECT - @current_table = 'inserting #include_sql_handles', - @sql = @isolation_level; - - INSERT - #include_sql_handles WITH(TABLOCK) - ( - sql_handle_s - ) - EXECUTE sys.sp_executesql - @string_split_strings, - N'@ids nvarchar(4000)', - @include_sql_handles; - - SELECT - @current_table = 'inserting #include_sql_handles for included sql handles'; - - IF @troubleshoot_performance = 1 - BEGIN - - EXECUTE sys.sp_executesql - @troubleshoot_insert, - N'@current_table nvarchar(100)', - @current_table; + BEGIN + EXECUTE sys.sp_executesql + @troubleshoot_insert, + N'@current_table nvarchar(100)', + @current_table; - SET STATISTICS XML ON; - END; + SET STATISTICS XML ON; + END; - SELECT - @sql += N' + SELECT + @sql += N' SELECT DISTINCT qsp.plan_id -FROM ' + @database_name_quoted + N'.sys.query_store_plan AS qsp -WHERE EXISTS - ( - SELECT - 1/0 - FROM ' + @database_name_quoted + N'.sys.query_store_query AS qsq - WHERE qsp.query_id = qsq.query_id - AND EXISTS - ( - SELECT - 1/0 - FROM ' + @database_name_quoted + N'.sys.query_store_query_text AS qsqt - WHERE qsqt.query_text_id = qsq.query_text_id - AND EXISTS - ( - SELECT - 1/0 - FROM #include_sql_handles AS ish - WHERE ish.sql_handle = qsqt.statement_sql_handle - ) - ) - ) +FROM ' + @database_name_quoted + N'.sys.query_store_query AS qsq +JOIN ' + @database_name_quoted + N'.sys.query_store_plan AS qsp + ON qsq.query_id = qsp.query_id +WHERE qsq.object_id ' + +CASE + WHEN LOWER(@query_type) LIKE 'a%' + THEN N'= 0' + ELSE N'<> 0' +END ++ N' OPTION(RECOMPILE);' + @nc10; - IF @debug = 1 - BEGIN - PRINT LEN(@sql); - PRINT @sql; - END; - - INSERT - #include_plan_ids - ( - plan_id - ) - EXECUTE sys.sp_executesql - @sql; - - IF @troubleshoot_performance = 1 - BEGIN - SET STATISTICS XML OFF; + IF @debug = 1 + BEGIN + PRINT LEN(@sql); + PRINT @sql; + END; - EXECUTE sys.sp_executesql - @troubleshoot_update, - N'@current_table nvarchar(100)', - @current_table; + INSERT + #query_types WITH(TABLOCK) + ( + plan_id + ) + EXECUTE sys.sp_executesql + @sql; - EXECUTE sys.sp_executesql - @troubleshoot_info, - N'@sql nvarchar(max), - @current_table nvarchar(100)', - @sql, - @current_table; - END; + IF @troubleshoot_performance = 1 + BEGIN + SET STATISTICS XML OFF; - /* - This section of code confused me when I came back to it, - so I'm going to add a note here about why I do this: + EXECUTE sys.sp_executesql + @troubleshoot_update, + N'@current_table nvarchar(100)', + @current_table; - If @include_plan_ids is NULL at this point, it's because - the user didn't populate the parameter. + EXECUTE sys.sp_executesql + @troubleshoot_info, + N'@sql nvarchar(max), + @current_table nvarchar(100)', + @sql, + @current_table; + END; - We need to do this because it's how we figure - out which plans to keep in the main query - */ - IF @include_plan_ids IS NULL - BEGIN + SELECT + @where_clause += N'AND EXISTS + ( SELECT - @where_clause += N'AND EXISTS - ( - SELECT 1/0 - FROM #include_plan_ids AS idi - WHERE idi.plan_id = qsrs.plan_id - )' + @nc10; - END; - END; /*End include plan hashes*/ + FROM #query_types AS qt + WHERE qt.plan_id = qsrs.plan_id + )' + @nc10; +END; /*End query type filter table population*/ - IF @ignore_sql_handles IS NOT NULL + +/* +This section filters query or plan ids, both inclusive and exclusive +*/ +IF +( + @include_plan_ids IS NOT NULL +OR @include_query_ids IS NOT NULL +OR @ignore_plan_ids IS NOT NULL +OR @ignore_query_ids IS NOT NULL +OR @include_query_hashes IS NOT NULL +OR @include_plan_hashes IS NOT NULL +OR @include_sql_handles IS NOT NULL +OR @ignore_query_hashes IS NOT NULL +OR @ignore_plan_hashes IS NOT NULL +OR @ignore_sql_handles IS NOT NULL +) +BEGIN + DECLARE + @filter_cursor CURSOR; + + SET @filter_cursor = + CURSOR + LOCAL + FORWARD_ONLY + STATIC + READ_ONLY + FOR + SELECT + parameter_name, + parameter_value, + temp_table_name, + column_name, + data_type, + is_include, + requires_secondary_processing + FROM @FilterParameters AS fp; + + OPEN @filter_cursor; + + FETCH NEXT + FROM @filter_cursor + INTO + @param_name, + @param_value, + @temp_table, + @column_name, + @data_type, + @is_include, + @requires_secondary_processing; + + WHILE @@FETCH_STATUS = 0 BEGIN - SELECT - @ignore_sql_handles = + /* Clean parameter value */ + SELECT + @param_value = REPLACE(REPLACE(REPLACE(REPLACE( - LTRIM(RTRIM(@ignore_sql_handles)), - CHAR(10), N''), CHAR(13), N''), + LTRIM(RTRIM(@param_value)), + CHAR(10), N''), CHAR(13), N''), NCHAR(10), N''), NCHAR(13), N''); - - SELECT - @current_table = 'inserting #ignore_sql_handles', - @sql = @isolation_level; - - INSERT - #ignore_sql_handles WITH(TABLOCK) - ( - sql_handle_s - ) - EXECUTE sys.sp_executesql - @string_split_strings, - N'@ids nvarchar(4000)', - @ignore_sql_handles; - - SELECT - @current_table = 'inserting #ignore_plan_ids for ignored sql handles'; - + + /* Log current operation if debugging */ + IF @debug = 1 + BEGIN + RAISERROR('Processing %s with value %s', 0, 1, @param_name, @param_value) WITH NOWAIT; + END; + + /* Set current table name for troubleshooting */ + SELECT + @current_table = 'inserting ' + @temp_table; + + /* Choose appropriate string split function based on data type */ + IF @data_type = N'bigint' + BEGIN + SELECT @split_sql = @string_split_ints; + END + ELSE + BEGIN + SELECT @split_sql = @string_split_strings; + END; + + /* Execute the initial insert with troubleshooting if enabled */ IF @troubleshoot_performance = 1 BEGIN EXECUTE sys.sp_executesql @troubleshoot_insert, - N'@current_table nvarchar(100)', + N'@current_table nvarchar(100)', @current_table; - + SET STATISTICS XML ON; END; - - SELECT - @sql += N' -SELECT DISTINCT - qsp.plan_id -FROM ' + @database_name_quoted + N'.sys.query_store_plan AS qsp -WHERE EXISTS - ( - SELECT - 1/0 - FROM ' + @database_name_quoted + N'.sys.query_store_query AS qsq - WHERE qsp.query_id = qsq.query_id - AND EXISTS - ( - SELECT - 1/0 - FROM ' + @database_name_quoted + N'.sys.query_store_query_text AS qsqt - WHERE qsqt.query_text_id = qsq.query_text_id - AND EXISTS - ( - SELECT - 1/0 - FROM #ignore_sql_handles AS ish - WHERE ish.sql_handle = qsqt.statement_sql_handle - ) - ) - ) -OPTION(RECOMPILE);' + @nc10; - - IF @debug = 1 - BEGIN - PRINT LEN(@sql); - PRINT @sql; - END; - - INSERT - #ignore_plan_ids + + /* Execute the dynamic SQL to populate the temporary table */ + DECLARE @dynamic_sql nvarchar(MAX) = N' + INSERT INTO + ' + @temp_table + N' + WITH + (TABLOCK) ( - plan_id - ) + ' + @column_name + + N') EXECUTE sys.sp_executesql - @sql; - + @split_sql, + N''@ids nvarchar(4000)'', + @param_value;'; + + EXEC sys.sp_executesql + @dynamic_sql, + N'@split_sql nvarchar(max), + @param_value nvarchar(4000)', + @split_sql, + @param_value; + IF @troubleshoot_performance = 1 BEGIN SET STATISTICS XML OFF; - + EXECUTE sys.sp_executesql @troubleshoot_update, - N'@current_table nvarchar(100)', + N'@current_table nvarchar(100)', @current_table; - + EXECUTE sys.sp_executesql @troubleshoot_info, - N'@sql nvarchar(max), + N'@sql nvarchar(max), @current_table nvarchar(100)', - @sql, + @split_sql, @current_table; END; - - /* - This section of code confused me when I came back to it, - so I'm going to add a note here about why I do this: - - If @ignore_plan_ids is NULL at this point, it's because - the user didn't populate the parameter. - - We need to do this because it's how we figure - out which plans to keep in the main query - */ - IF @ignore_plan_ids IS NULL + + /* Secondary processing (for parameters that need to populate plan IDs) */ + IF @requires_secondary_processing = 1 BEGIN - SELECT - @where_clause += N'AND NOT EXISTS - ( - SELECT - 1/0 - FROM #ignore_plan_ids AS idi - WHERE idi.plan_id = qsrs.plan_id - )' + @nc10; - END; - END; /*End ignore plan hashes*/ + SELECT + @current_table = 'inserting #include_plan_ids for ' + @param_name; + + /* Build appropriate SQL based on parameter type */ + DECLARE + @secondary_sql nvarchar(MAX) = N''; + + IF @param_name = 'include_query_ids' + OR @param_name = 'ignore_query_ids' + BEGIN + SELECT @secondary_sql = N' + SELECT DISTINCT + qsp.plan_id + FROM ' + @database_name_quoted + N'.sys.query_store_plan AS qsp + WHERE EXISTS + ( + SELECT + 1/0 + FROM #' + + CASE + WHEN @is_include = 1 + THEN N'include' + ELSE N'ignore' + END + + N'_query_ids AS iqi + WHERE iqi.query_id = qsp.query_id + ) + OPTION(RECOMPILE);'; + END; + ELSE + IF @param_name = 'include_query_hashes' + OR @param_name = 'ignore_query_hashes' + BEGIN + SELECT @secondary_sql = N' + SELECT DISTINCT + qsp.plan_id + FROM ' + @database_name_quoted + N'.sys.query_store_plan AS qsp + WHERE EXISTS + ( + SELECT + 1/0 + FROM ' + @database_name_quoted + N'.sys.query_store_query AS qsq + WHERE qsq.query_id = qsp.query_id + AND EXISTS + ( + SELECT + 1/0 + FROM #' + + CASE + WHEN @is_include = 1 + THEN N'include' + ELSE N'ignore' + END + + N'_query_hashes AS iqh + WHERE iqh.query_hash = qsq.query_hash + ) + ) + OPTION(RECOMPILE);'; + END; + ELSE + IF @param_name = 'include_plan_hashes' + OR @param_name = 'ignore_plan_hashes' + BEGIN + SELECT @secondary_sql = N' + SELECT DISTINCT + qsp.plan_id + FROM ' + @database_name_quoted + N'.sys.query_store_plan AS qsp + WHERE EXISTS + ( + SELECT + 1/0 + FROM #' + + CASE + WHEN @is_include = 1 + THEN N'include' + ELSE N'ignore' + END + N'_plan_hashes AS iph + WHERE iph.plan_hash = qsp.query_plan_hash + ) + OPTION(RECOMPILE);'; + END; + ELSE + IF + @param_name = 'include_sql_handles' + OR @param_name = 'ignore_sql_handles' + BEGIN + SELECT @secondary_sql = N' + SELECT DISTINCT + qsp.plan_id + FROM ' + @database_name_quoted + N'.sys.query_store_plan AS qsp + WHERE EXISTS + ( + SELECT + 1/0 + FROM ' + @database_name_quoted + N'.sys.query_store_query AS qsq + WHERE qsp.query_id = qsq.query_id + AND EXISTS + ( + SELECT + 1/0 + FROM ' + @database_name_quoted + N'.sys.query_store_query_text AS qsqt + WHERE qsqt.query_text_id = qsq.query_text_id + AND EXISTS + ( + SELECT + 1/0 + FROM #' + + CASE + WHEN @is_include = 1 + THEN N'include' + ELSE N'ignore' + END + N'_sql_handles AS ish + WHERE ish.sql_handle = qsqt.statement_sql_handle + ) + ) + ) + OPTION(RECOMPILE);'; + END; + + /* Process secondary sql if defined */ + IF @secondary_sql IS NOT NULL + BEGIN + IF @troubleshoot_performance = 1 + BEGIN + EXECUTE sys.sp_executesql + @troubleshoot_insert, + N'@current_table nvarchar(100)', + @current_table; + + SET STATISTICS XML ON; + END; + + INSERT INTO + #include_plan_ids + WITH + (TABLOCK) + ( + plan_id + ) + EXECUTE sys.sp_executesql + @secondary_sql; + + IF @troubleshoot_performance = 1 + BEGIN + SET STATISTICS XML OFF; + + EXECUTE sys.sp_executesql + @troubleshoot_update, + N'@current_table nvarchar(100)', + @current_table; + + EXECUTE sys.sp_executesql + @troubleshoot_info, + N'@sql nvarchar(max), @current_table nvarchar(100)', + @secondary_sql, + @current_table; + END; + END; + + /* Update where clause if needed */ + DECLARE + @temp_target_table nvarchar(100) = + CASE + WHEN @is_include = 1 + THEN N'#include_plan_ids' + ELSE N'#ignore_plan_ids' + END, + @exist_or_not_exist nvarchar(20) = + CASE + WHEN @is_include = 1 + THEN N'EXISTS' + ELSE N'NOT EXISTS' + END; + + SELECT + @where_clause += + N'AND ' + + @exist_or_not_exist + + N' + ( + SELECT + 1/0 + FROM ' + @temp_target_table + N' AS idi + WHERE idi.plan_id = qsrs.plan_id + )' + @nc10; + END; + + FETCH NEXT + FROM @filter_cursor + INTO + @param_name, + @param_value, + @temp_table, + @column_name, + @data_type, + @is_include, + @requires_secondary_processing; + END; END; /*End hash and handle filtering*/ IF @sql_2022_views = 1 @@ -8263,6 +7701,48 @@ BEGIN TRUNCATE TABLE #forced_plans_failures; + + TRUNCATE TABLE + #include_plan_ids; + + TRUNCATE TABLE + #include_query_ids; + + TRUNCATE TABLE + #include_query_hashes; + + TRUNCATE TABLE + #include_plan_hashes; + + TRUNCATE TABLE + #include_sql_handles; + + TRUNCATE TABLE + #ignore_plan_ids; + + TRUNCATE TABLE + #ignore_query_ids; + + TRUNCATE TABLE + #ignore_query_hashes; + + TRUNCATE TABLE + #ignore_plan_hashes; + + TRUNCATE TABLE + #ignore_sql_handles; + + TRUNCATE TABLE + #only_queries_with_hints; + + TRUNCATE TABLE + #only_queries_with_feedback; + + TRUNCATE TABLE + #only_queries_with_variants; + + TRUNCATE TABLE + #forced_plans_failures; END; FETCH NEXT @@ -10332,7 +9812,25 @@ BEGIN work_start_utc = @work_start_utc, work_end_utc = - @work_end_utc; + @work_end_utc, + column_sql = + @column_sql, + param_name = + @param_name, + param_value = + @param_value, + temp_table = + @temp_table, + column_name = + @column_name, + data_type = + @data_type, + is_include = + @is_include, + requires_secondary_processing = + @requires_secondary_processing, + split_sql = + @split_sql; IF EXISTS ( From af69471553219b79d333c72c4434efabf6ac5627 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Wed, 5 Mar 2025 23:28:14 -0500 Subject: [PATCH 004/246] Update sp_HumanEventsBlockViewer.sql Testing adding a target table mode --- sp_HumanEvents/sp_HumanEventsBlockViewer.sql | 479 +++++++++++++++---- 1 file changed, 391 insertions(+), 88 deletions(-) diff --git a/sp_HumanEvents/sp_HumanEventsBlockViewer.sql b/sp_HumanEvents/sp_HumanEventsBlockViewer.sql index 94ffb11e..90f32285 100644 --- a/sp_HumanEvents/sp_HumanEventsBlockViewer.sql +++ b/sp_HumanEvents/sp_HumanEventsBlockViewer.sql @@ -62,23 +62,28 @@ GO ALTER PROCEDURE dbo.sp_HumanEventsBlockViewer ( - @session_name nvarchar(256) = N'keeper_HumanEvents_blocking', - @target_type sysname = NULL, - @start_date datetime2 = NULL, - @end_date datetime2 = NULL, - @database_name sysname = NULL, - @object_name sysname = NULL, - @help bit = 0, - @debug bit = 0, - @version varchar(30) = NULL OUTPUT, - @version_date datetime = NULL OUTPUT + @session_name sysname = N'keeper_HumanEvents_blocking', /*Event session name*/ + @target_type sysname = NULL, /*ring buffer, file, or table*/ + @start_date datetime2 = NULL, /*when to start looking for blocking*/ + @end_date datetime2 = NULL, /*when to stop looking for blocking*/ + @database_name sysname = NULL, /*target a specific database*/ + @object_name sysname = NULL, /*target a specific schema-prefixed table*/ + @target_database sysname = NULL, /*database containing the table with BPR data*/ + @target_schema sysname = NULL, /*schema of the table*/ + @target_table sysname = NULL, /*table name*/ + @target_column sysname = NULL, /*column containing XML data*/ + @timestamp_column sysname = NULL, /*column containing timestamp (optional)*/ + @help bit = 0, /*get help with this procedure*/ + @debug bit = 0, /*print dynamic sql and select temp table contents*/ + @version varchar(30) = NULL OUTPUT, /*check the version number*/ + @version_date datetime = NULL OUTPUT /*check the version date*/ ) WITH RECOMPILE AS BEGIN SET STATISTICS XML OFF; SET NOCOUNT ON; -SET XACT_ABORT ON; +SET XACT_ABORT OFF; SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; SELECT @@ -105,11 +110,16 @@ BEGIN description = CASE ap.name WHEN N'@session_name' THEN 'name of the extended event session to pull from' - WHEN N'@target_type' THEN 'target of the extended event session' + WHEN N'@target_type' THEN 'target type of the extended event session (ring buffer, file) or ''table'' to read from a table' WHEN N'@start_date' THEN 'filter by date' WHEN N'@end_date' THEN 'filter by date' WHEN N'@database_name' THEN 'filter by database name' WHEN N'@object_name' THEN 'filter by table name' + WHEN N'@target_database' THEN 'database containing the table with blocked process report data' + WHEN N'@target_schema' THEN 'schema of the table containing blocked process report data' + WHEN N'@target_table' THEN 'table containing blocked process report data' + WHEN N'@target_column' THEN 'column containing blocked process report XML' + WHEN N'@timestamp_column' THEN 'column containing timestamp for filtering (optional)' WHEN N'@help' THEN 'how you got here' WHEN N'@debug' THEN 'dumps raw temp table contents' WHEN N'@version' THEN 'OUTPUT; for support' @@ -123,6 +133,11 @@ BEGIN WHEN N'@end_date' THEN 'a reasonable date' WHEN N'@database_name' THEN 'a database that exists on this server' WHEN N'@object_name' THEN 'a schema-prefixed table name' + WHEN N'@target_database' THEN 'a database that exists on this server' + WHEN N'@target_schema' THEN 'a schema in the target database' + WHEN N'@target_table' THEN 'a table in the target schema' + WHEN N'@target_column' THEN 'an XML column containing blocked process report data' + WHEN N'@timestamp_column' THEN 'a datetime column for filtering by date range' WHEN N'@help' THEN '0 or 1' WHEN N'@debug' THEN '0 or 1' WHEN N'@version' THEN 'none; OUTPUT' @@ -136,6 +151,11 @@ BEGIN WHEN N'@end_date' THEN 'NULL' WHEN N'@database_name' THEN 'NULL' WHEN N'@object_name' THEN 'NULL' + WHEN N'@target_database' THEN 'NULL' + WHEN N'@target_schema' THEN 'NULL' + WHEN N'@target_table' THEN 'NULL' + WHEN N'@target_column' THEN 'NULL' + WHEN N'@timestamp_column' THEN 'NULL' WHEN N'@help' THEN '0' WHEN N'@debug' THEN '0' WHEN N'@version' THEN 'none; OUTPUT' @@ -284,7 +304,9 @@ DECLARE @inputbuf_bom nvarchar(1) = CONVERT(nvarchar(1), 0x0a00, 0), @start_date_original datetime2 = @start_date, - @end_date_original datetime2 = @end_date; + @end_date_original datetime2 = @end_date, + @validation_sql nvarchar(MAX), + @extract_sql nvarchar(MAX); /*Use some sane defaults for input parameters*/ IF @debug = 1 @@ -366,6 +388,172 @@ SELECT @is_system_health_msg = CONVERT(nchar(1), @is_system_health); +/* Check for table input early and validate */ +IF @target_type = N'table' +BEGIN + IF @debug = 1 + BEGIN + RAISERROR('Table source detected, validating parameters', 0, 1) WITH NOWAIT; + END; + + /* Parameter validation */ + IF @target_database IS NULL + OR @target_schema IS NULL + OR @target_table IS NULL + OR @target_column IS NULL + BEGIN + RAISERROR(N'When @target_type is ''table'', you must specify @target_database, @target_schema, @target_table, and @target_column.', 11, 1) WITH NOWAIT; + RETURN; + END; + + /* Check if target database exists */ + IF NOT EXISTS + ( + SELECT + 1/0 + FROM sys.databases AS d + WHERE d.name = @target_database + ) + BEGIN + RAISERROR(N'The specified @target_database ''%s'' does not exist.', 11, 1, @target_database) WITH NOWAIT; + RETURN; + END; + + /* Use dynamic SQL to validate schema, table, and column existence */ + SET @validation_sql = N' + IF NOT EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@target_database) + N'.sys.schemas AS s + WHERE s.name = @schema + ) + BEGIN + RAISERROR(N''The specified @target_schema %s does not exist in @database %s'', 11, 1, @schema, @database) WITH NOWAIT; + RETURN; + END; + + IF NOT EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@target_database) + N'.sys.tables AS t + JOIN ' + QUOTENAME(@target_database) + N'.sys.schemas AS s + ON t.schema_id = s.schema_id + WHERE t.name = @table + AND s.name = @schema + ) + BEGIN + RAISERROR(N''The specified @target_table %s does not exist in @schema %s in database %s'', 11, 1, @table, @schema, @database) WITH NOWAIT; + RETURN; + END; + + IF NOT EXISTS + ( + SELECT + 1 + FROM ' + QUOTENAME(@target_database) + N'.sys.columns AS c + JOIN ' + QUOTENAME(@target_database) + N'.sys.tables AS t + ON c.object_id = t.object_id + JOIN ' + QUOTENAME(@target_database) + N'.sys.schemas AS s + ON t.schema_id = s.schema_id + WHERE c.name = @column + AND t.name = @table + AND s.name = @schema + ) + BEGIN + RAISERROR(N''The specified @target_column %s does not exist in table %s.%s in database %s'', 11, 1, @column, @schema, @table, @database) WITH NOWAIT; + RETURN; + END; + + /* Validate column is XML type */ + IF NOT EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@target_database) + N'.sys.columns AS c + JOIN ' + QUOTENAME(@target_database) + N'.sys.types AS ty + ON c.user_type_id = ty.user_type_id + JOIN ' + QUOTENAME(@target_database) + N'.sys.tables AS t + ON c.object_id = t.object_id + JOIN ' + QUOTENAME(@target_database) + N'.sys.schemas AS s + ON t.schema_id = s.schema_id + WHERE c.name = @column + AND t.name = @table + AND s.name = @schema + AND ty.name = ''xml'' + ) + BEGIN + RAISERROR(N''The specified @target_column %s must be of XML data type.'', 11, 1, @column) WITH NOWAIT; + RETURN; + END;'; + + /* Validate timestamp_column if specified */ + IF @timestamp_column IS NOT NULL + BEGIN + SET @validation_sql = @validation_sql + N' + IF NOT EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@target_database) + N'.sys.columns AS c + JOIN ' + QUOTENAME(@target_database) + N'.sys.tables AS t + ON c.object_id = t.object_id + JOIN ' + QUOTENAME(@target_database) + N'.sys.schemas AS s + ON t.schema_id = s.schema_id + WHERE c.name = @timestamp_column + AND t.name = @table + AND s.name = @schema + ) + BEGIN + RAISERROR(N''The specified @timestamp_column %s does not exist in table %s.%s in database %s'', 11, 1, @timestamp_column, @schema, @table, @database) WITH NOWAIT; + RETURN; + END; + + /* Validate timestamp column is datetime type */ + IF NOT EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@target_database) + N'.sys.columns AS c + JOIN ' + QUOTENAME(@target_database) + N'.sys.types AS ty + ON c.user_type_id = ty.user_type_id + JOIN ' + QUOTENAME(@target_database) + N'.sys.tables AS t + ON c.object_id = t.object_id + JOIN ' + QUOTENAME(@target_database) + N'.sys.schemas AS s + ON t.schema_id = s.schema_id + WHERE c.name = @timestamp_column + AND t.name = @table + AND s.name = @schema + AND ty.name LIKE N''%date%'' + ) + BEGIN + RAISERROR(N''The specified @timestamp_column %s must be of datetime data type.'', 11, 1, @timestamp_column) WITH NOWAIT; + RETURN; + END;'; + END; + + IF @debug = 1 + BEGIN + PRINT @validation_sql; + END; + + EXEC sys.sp_executesql + @validation_sql, + N' + @database sysname, + @schema sysname, + @table sysname, + @column sysname, + @timestamp_column sysname + ', + @target_database, + @target_schema, + @target_table, + @target_column, + @timestamp_column; +END; + /*Temp tables for staging results*/ IF @debug = 1 BEGIN @@ -386,8 +574,8 @@ CREATE TABLE CREATE TABLE #block_findings ( - id int IDENTITY PRIMARY KEY, - check_id int NOT NULL, + id integer IDENTITY PRIMARY KEY CLUSTERED, + check_id integer NOT NULL, database_name nvarchar(256) NULL, object_name nvarchar(1000) NULL, finding_group nvarchar(100) NULL, @@ -395,6 +583,12 @@ CREATE TABLE sort_order bigint ); +IF @target_type = N'table' +BEGIN + GOTO TableMode; + RETURN; +END; + /*Look to see if the session exists and is running*/ IF @debug = 1 BEGIN @@ -441,7 +635,8 @@ IF @debug = 1 BEGIN RAISERROR('What kind of target does %s have?', 0, 1, @session_name) WITH NOWAIT; END; -IF @target_type IS NULL AND @is_system_health = 0 +IF @target_type IS NULL +AND @is_system_health = 0 BEGIN IF @azure = 0 BEGIN @@ -471,7 +666,8 @@ BEGIN END; /* Dump whatever we got into a temp table */ -IF @target_type = N'ring_buffer' AND @is_system_health = 0 +IF @target_type = N'ring_buffer' +AND @is_system_health = 0 BEGIN IF @azure = 0 BEGIN @@ -520,7 +716,8 @@ BEGIN END; END; -IF @target_type = N'event_file' AND @is_system_health = 0 +IF @target_type = N'event_file' +AND @is_system_health = 0 BEGIN IF @azure = 0 BEGIN @@ -635,7 +832,8 @@ BEGIN END; -IF @target_type = N'ring_buffer' AND @is_system_health = 0 +IF @target_type = N'ring_buffer' +AND @is_system_health = 0 BEGIN IF @debug = 1 BEGIN @@ -643,20 +841,23 @@ BEGIN END; INSERT - #blocking_xml WITH(TABLOCKX) + #blocking_xml + WITH + (TABLOCKX) ( human_events_xml ) SELECT human_events_xml = e.x.query('.') FROM #x AS x - CROSS APPLY x.x.nodes('/RingBufferTarget/event') AS e(x) + CROSS APPLY x.x.nodes('/RingBufferTarget/event') AS E(x) WHERE e.x.exist('@name[ .= "blocked_process_report"]') = 1 AND e.x.exist('@timestamp[. >= sql:variable("@start_date") and .< sql:variable("@end_date")]') = 1 OPTION(RECOMPILE); END; -IF @target_type = N'event_file' AND @is_system_health = 0 +IF @target_type = N'event_file' +AND @is_system_health = 0 BEGIN IF @debug = 1 BEGIN @@ -664,14 +865,16 @@ BEGIN END; INSERT - #blocking_xml WITH(TABLOCKX) + #blocking_xml + WITH + (TABLOCKX) ( human_events_xml ) SELECT human_events_xml = e.x.query('.') FROM #x AS x - CROSS APPLY x.x.nodes('/event') AS e(x) + CROSS APPLY x.x.nodes('/event') AS E(x) WHERE e.x.exist('@name[ .= "blocked_process_report"]') = 1 AND e.x.exist('@timestamp[. >= sql:variable("@start_date") and .< sql:variable("@end_date")]') = 1 OPTION(RECOMPILE); @@ -681,7 +884,8 @@ END; This section is special for the well-hidden and much less comprehensive blocked process report stored in the system health extended event session */ -IF @is_system_health = 1 +IF @is_system_health = 1 +AND @target_type <> N'table' BEGIN IF @debug = 1 BEGIN @@ -699,7 +903,7 @@ BEGIN FROM sys.fn_xe_file_target_read_file(N'system_health*.xel', NULL, NULL, NULL) AS fx WHERE fx.object_name = N'sp_server_diagnostics_component_result' ) AS xml - CROSS APPLY xml.sp_server_diagnostics_component_result.nodes('/event') AS e(x) + CROSS APPLY xml.sp_server_diagnostics_component_result.nodes('/event') AS E(x) WHERE e.x.exist('//data[@name="data"]/value/queryProcessing/blockingTasks/blocked-process-report') = 1 AND e.x.exist('@timestamp[. >= sql:variable("@start_date") and .< sql:variable("@end_date")]') = 1 OPTION(RECOMPILE); @@ -748,16 +952,16 @@ BEGIN SELECT bx.event_time, currentdbname = bd.value('(process/@currentdbname)[1]', 'nvarchar(128)'), - spid = bd.value('(process/@spid)[1]', 'int'), - ecid = bd.value('(process/@ecid)[1]', 'int'), + spid = bd.value('(process/@spid)[1]', 'integer'), + ecid = bd.value('(process/@ecid)[1]', 'integer'), query_text_pre = bd.value('(process/inputbuf/text())[1]', 'nvarchar(MAX)'), wait_time = bd.value('(process/@waittime)[1]', 'bigint'), lastbatchstarted = bd.value('(process/@lastbatchstarted)[1]', 'datetime2'), lastbatchcompleted = bd.value('(process/@lastbatchcompleted)[1]', 'datetime2'), wait_resource = bd.value('(process/@waitresource)[1]', 'nvarchar(100)'), status = bd.value('(process/@status)[1]', 'nvarchar(10)'), - priority = bd.value('(process/@priority)[1]', 'int'), - transaction_count = bd.value('(process/@trancount)[1]', 'int'), + priority = bd.value('(process/@priority)[1]', 'integer'), + transaction_count = bd.value('(process/@trancount)[1]', 'integer'), client_app = bd.value('(process/@clientapp)[1]', 'nvarchar(256)'), host_name = bd.value('(process/@hostname)[1]', 'nvarchar(256)'), login_name = bd.value('(process/@loginname)[1]', 'nvarchar(256)'), @@ -769,7 +973,7 @@ BEGIN blocked_process_report = bd.query('.') INTO #blocked_sh FROM #blocking_xml_sh AS bx - OUTER APPLY bx.human_events_xml.nodes('/event') AS oa(c) + OUTER APPLY bx.human_events_xml.nodes('/event') AS oa(C) OUTER APPLY oa.c.nodes('//blocked-process-report/blocked-process') AS bd(bd) WHERE bd.exist('process/@spid') = 1 OPTION(RECOMPILE); @@ -805,16 +1009,16 @@ BEGIN SELECT bx.event_time, currentdbname = bg.value('(process/@currentdbname)[1]', 'nvarchar(128)'), - spid = bg.value('(process/@spid)[1]', 'int'), - ecid = bg.value('(process/@ecid)[1]', 'int'), + spid = bg.value('(process/@spid)[1]', 'integer'), + ecid = bg.value('(process/@ecid)[1]', 'integer'), query_text_pre = bg.value('(process/inputbuf/text())[1]', 'nvarchar(MAX)'), wait_time = bg.value('(process/@waittime)[1]', 'bigint'), last_transaction_started = bg.value('(process/@lastbatchstarted)[1]', 'datetime2'), last_transaction_completed = bg.value('(process/@lastbatchcompleted)[1]', 'datetime2'), wait_resource = bg.value('(process/@waitresource)[1]', 'nvarchar(100)'), status = bg.value('(process/@status)[1]', 'nvarchar(10)'), - priority = bg.value('(process/@priority)[1]', 'int'), - transaction_count = bg.value('(process/@trancount)[1]', 'int'), + priority = bg.value('(process/@priority)[1]', 'integer'), + transaction_count = bg.value('(process/@trancount)[1]', 'integer'), client_app = bg.value('(process/@clientapp)[1]', 'nvarchar(256)'), host_name = bg.value('(process/@hostname)[1]', 'nvarchar(256)'), login_name = bg.value('(process/@loginname)[1]', 'nvarchar(256)'), @@ -826,7 +1030,7 @@ BEGIN blocked_process_report = bg.query('.') INTO #blocking_sh FROM #blocking_xml_sh AS bx - OUTER APPLY bx.human_events_xml.nodes('/event') AS oa(c) + OUTER APPLY bx.human_events_xml.nodes('/event') AS oa(C) OUTER APPLY oa.c.nodes('//blocked-process-report/blocking-process') AS bg(bg) WHERE bg.exist('process/@spid') = 1 OPTION(RECOMPILE); @@ -1055,9 +1259,9 @@ BEGIN sql_handle = CONVERT(varbinary(64), n.c.value('@sqlhandle', 'varchar(130)'), 1), stmtstart = - ISNULL(n.c.value('@stmtstart', 'int'), 0), + ISNULL(n.c.value('@stmtstart', 'integer'), 0), stmtend = - ISNULL(n.c.value('@stmtend', 'int'), -1) + ISNULL(n.c.value('@stmtend', 'integer'), -1) FROM #blocks_sh AS b CROSS APPLY b.blocked_process_report.nodes('/blocked-process/process/executionStack/frame[not(@sqlhandle = "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")]') AS n(c) WHERE (b.currentdbname = @database_name @@ -1074,14 +1278,14 @@ BEGIN sql_handle = CONVERT(varbinary(64), n.c.value('@sqlhandle', 'varchar(130)'), 1), stmtstart = - ISNULL(n.c.value('@stmtstart', 'int'), 0), + ISNULL(n.c.value('@stmtstart', 'integer'), 0), stmtend = - ISNULL(n.c.value('@stmtend', 'int'), -1) + ISNULL(n.c.value('@stmtend', 'integer'), -1) FROM #blocks_sh AS b - CROSS APPLY b.blocked_process_report.nodes('/blocking-process/process/executionStack/frame[not(@sqlhandle = "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")]') AS n(c) + CROSS APPLY b.blocked_process_report.nodes('/blocking-process/process/executionStack/frame[not(@sqlhandle = "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")]') AS n(C) WHERE (b.currentdbname = @database_name OR @database_name IS NULL) - ) AS b + ) AS B OPTION(RECOMPILE); IF @debug = 1 @@ -1159,7 +1363,8 @@ BEGIN FROM #available_plans_sh AS ap WHERE ap.sql_handle = deqs.sql_handle ) - AND deqs.query_hash IS NOT NULL; + AND deqs.query_hash IS NOT NULL + OPTION(RECOMPILE); IF @debug = 1 BEGIN @@ -1259,6 +1464,87 @@ BEGIN /*End system health section, skips checks because most of them won't run*/ END; +TableMode: +IF @target_type = N'table' +BEGIN + IF @debug = 1 + BEGIN + RAISERROR('Extracting blocked process reports from table %s.%s.%s', 0, 1, @target_database, @target_schema, @target_table) WITH NOWAIT; + END; + + /* Build dynamic SQL to extract the XML */ + SET @extract_sql = N' + INSERT + #blocking_xml + WITH + (TABLOCKX) + ( + human_events_xml + ) + SELECT + human_events_xml = ' + + QUOTENAME(@target_column) + + N' + FROM ' + + QUOTENAME(@target_database) + + N'.' + + QUOTENAME(@target_schema) + + N'.' + + QUOTENAME(@target_table) + + N' + CROSS APPLY x.x.nodes(''/event'') AS e(x) + WHERE e.x.exist(''@name[ .= "blocked_process_report"]'') = 1'; + + /* Add timestamp filtering if specified*/ + IF (@start_date IS NOT NULL OR @end_date IS NOT NULL) + BEGIN + IF @timestamp_column IS NOT NULL + BEGIN + IF @start_date IS NOT NULL + BEGIN + SET @extract_sql = @extract_sql + N' + AND ' + QUOTENAME(@timestamp_column) + N' >= @start_date'; + END; + + IF @end_date IS NOT NULL + BEGIN + SET @extract_sql = @extract_sql + N' + AND ' + QUOTENAME(@timestamp_column) + N' < @end_date'; + END; + END; + + IF @timestamp_column IS NULL + BEGIN + IF @start_date IS NOT NULL + BEGIN + SET @extract_sql = @extract_sql + N' + AND ' + QUOTENAME(@target_column) + N'.exist(''@timestamp[. >= sql:variable("@start_date")]'') = 1'; + END; + + IF @end_date IS NOT NULL + BEGIN + SET @extract_sql = @extract_sql + N' + AND ' + QUOTENAME(@target_column) + '.exist(''@timestamp[. < sql:variable("@end_date")]'') = 1'; + END; + END; + END; + + SET @extract_sql = @extract_sql + N' + OPTION(RECOMPILE);'; + + IF @debug = 1 + BEGIN + PRINT @extract_sql; + END; + + /* Execute the dynamic SQL*/ + EXECUTE sys.sp_executesql + @extract_sql, + N'@start_date datetime2, + @end_date datetime2', + @start_date, + @end_date; +END; IF @debug = 1 BEGIN @@ -1284,16 +1570,16 @@ SELECT ), c.value('@timestamp', 'datetime2') ), - database_name = DB_NAME(c.value('(data[@name="database_id"]/value/text())[1]', 'int')), - database_id = c.value('(data[@name="database_id"]/value/text())[1]', 'int'), - object_id = c.value('(data[@name="object_id"]/value/text())[1]', 'int'), + database_name = DB_NAME(c.value('(data[@name="database_id"]/value/text())[1]', 'integer')), + database_id = c.value('(data[@name="database_id"]/value/text())[1]', 'integer'), + object_id = c.value('(data[@name="object_id"]/value/text())[1]', 'integer'), transaction_id = c.value('(data[@name="transaction_id"]/value/text())[1]', 'bigint'), resource_owner_type = c.value('(data[@name="resource_owner_type"]/text)[1]', 'nvarchar(256)'), - monitor_loop = c.value('(//@monitorLoop)[1]', 'int'), - blocking_spid = bg.value('(process/@spid)[1]', 'int'), - blocking_ecid = bg.value('(process/@ecid)[1]', 'int'), - blocked_spid = bd.value('(process/@spid)[1]', 'int'), - blocked_ecid = bd.value('(process/@ecid)[1]', 'int'), + monitor_loop = c.value('(//@monitorLoop)[1]', 'integer'), + blocking_spid = bg.value('(process/@spid)[1]', 'integer'), + blocking_ecid = bg.value('(process/@ecid)[1]', 'integer'), + blocked_spid = bd.value('(process/@spid)[1]', 'integer'), + blocked_ecid = bd.value('(process/@ecid)[1]', 'integer'), query_text_pre = bd.value('(process/inputbuf/text())[1]', 'nvarchar(MAX)'), wait_time = bd.value('(process/@waittime)[1]', 'bigint'), transaction_name = bd.value('(process/@transactionname)[1]', 'nvarchar(512)'), @@ -1302,8 +1588,8 @@ SELECT wait_resource = bd.value('(process/@waitresource)[1]', 'nvarchar(100)'), lock_mode = bd.value('(process/@lockMode)[1]', 'nvarchar(10)'), status = bd.value('(process/@status)[1]', 'nvarchar(10)'), - priority = bd.value('(process/@priority)[1]', 'int'), - transaction_count = bd.value('(process/@trancount)[1]', 'int'), + priority = bd.value('(process/@priority)[1]', 'integer'), + transaction_count = bd.value('(process/@trancount)[1]', 'integer'), client_app = bd.value('(process/@clientapp)[1]', 'nvarchar(256)'), host_name = bd.value('(process/@hostname)[1]', 'nvarchar(256)'), login_name = bd.value('(process/@loginname)[1]', 'nvarchar(256)'), @@ -1312,14 +1598,14 @@ SELECT clientoption1 = bd.value('(process/@clientoption1)[1]', 'bigint'), clientoption2 = bd.value('(process/@clientoption1)[1]', 'bigint'), currentdbname = bd.value('(process/@currentdbname)[1]', 'nvarchar(256)'), - currentdbid = bd.value('(process/@currentdb)[1]', 'int'), + currentdbid = bd.value('(process/@currentdb)[1]', 'integer'), blocking_level = 0, sort_order = CAST('' AS varchar(400)), activity = CASE WHEN oa.c.exist('//blocked-process-report/blocked-process') = 1 THEN 'blocked' END, blocked_process_report = c.query('.') INTO #blocked FROM #blocking_xml AS bx -OUTER APPLY bx.human_events_xml.nodes('/event') AS oa(c) +OUTER APPLY bx.human_events_xml.nodes('/event') AS oa(C) OUTER APPLY oa.c.nodes('//blocked-process-report/blocked-process') AS bd(bd) OUTER APPLY oa.c.nodes('//blocked-process-report/blocking-process') AS bg(bg) OPTION(RECOMPILE); @@ -1404,16 +1690,16 @@ SELECT ), c.value('@timestamp', 'datetime2') ), - database_name = DB_NAME(c.value('(data[@name="database_id"]/value/text())[1]', 'int')), - database_id = c.value('(data[@name="database_id"]/value/text())[1]', 'int'), - object_id = c.value('(data[@name="object_id"]/value/text())[1]', 'int'), + database_name = DB_NAME(c.value('(data[@name="database_id"]/value/text())[1]', 'integer')), + database_id = c.value('(data[@name="database_id"]/value/text())[1]', 'integer'), + object_id = c.value('(data[@name="object_id"]/value/text())[1]', 'integer'), transaction_id = c.value('(data[@name="transaction_id"]/value/text())[1]', 'bigint'), resource_owner_type = c.value('(data[@name="resource_owner_type"]/text)[1]', 'nvarchar(256)'), - monitor_loop = c.value('(//@monitorLoop)[1]', 'int'), - blocking_spid = bg.value('(process/@spid)[1]', 'int'), - blocking_ecid = bg.value('(process/@ecid)[1]', 'int'), - blocked_spid = bd.value('(process/@spid)[1]', 'int'), - blocked_ecid = bd.value('(process/@ecid)[1]', 'int'), + monitor_loop = c.value('(//@monitorLoop)[1]', 'integer'), + blocking_spid = bg.value('(process/@spid)[1]', 'integer'), + blocking_ecid = bg.value('(process/@ecid)[1]', 'integer'), + blocked_spid = bd.value('(process/@spid)[1]', 'integer'), + blocked_ecid = bd.value('(process/@ecid)[1]', 'integer'), query_text_pre = bg.value('(process/inputbuf/text())[1]', 'nvarchar(MAX)'), wait_time = bg.value('(process/@waittime)[1]', 'bigint'), transaction_name = bg.value('(process/@transactionname)[1]', 'nvarchar(512)'), @@ -1422,8 +1708,8 @@ SELECT wait_resource = bg.value('(process/@waitresource)[1]', 'nvarchar(100)'), lock_mode = bg.value('(process/@lockMode)[1]', 'nvarchar(10)'), status = bg.value('(process/@status)[1]', 'nvarchar(10)'), - priority = bg.value('(process/@priority)[1]', 'int'), - transaction_count = bg.value('(process/@trancount)[1]', 'int'), + priority = bg.value('(process/@priority)[1]', 'integer'), + transaction_count = bg.value('(process/@trancount)[1]', 'integer'), client_app = bg.value('(process/@clientapp)[1]', 'nvarchar(256)'), host_name = bg.value('(process/@hostname)[1]', 'nvarchar(256)'), login_name = bg.value('(process/@loginname)[1]', 'nvarchar(256)'), @@ -1432,14 +1718,14 @@ SELECT clientoption1 = bg.value('(process/@clientoption1)[1]', 'bigint'), clientoption2 = bg.value('(process/@clientoption1)[1]', 'bigint'), currentdbname = bg.value('(process/@currentdbname)[1]', 'nvarchar(128)'), - currentdbid = bg.value('(process/@currentdb)[1]', 'int'), + currentdbid = bg.value('(process/@currentdb)[1]', 'integer'), blocking_level = 0, sort_order = CAST('' AS varchar(400)), activity = CASE WHEN oa.c.exist('//blocked-process-report/blocking-process') = 1 THEN 'blocking' END, blocked_process_report = c.query('.') INTO #blocking FROM #blocking_xml AS bx -OUTER APPLY bx.human_events_xml.nodes('/event') AS oa(c) +OUTER APPLY bx.human_events_xml.nodes('/event') AS oa(C) OUTER APPLY oa.c.nodes('//blocked-process-report/blocked-process') AS bd(bd) OUTER APPLY oa.c.nodes('//blocked-process-report/blocking-process') AS bg(bg) OPTION(RECOMPILE); @@ -1523,15 +1809,15 @@ WITH CAST ( blocking_desc + - ' <-- ' + + ' Date: Wed, 5 Mar 2025 23:49:48 -0500 Subject: [PATCH 005/246] Update sp_HumanEventsBlockViewer.sql doh --- sp_HumanEvents/sp_HumanEventsBlockViewer.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sp_HumanEvents/sp_HumanEventsBlockViewer.sql b/sp_HumanEvents/sp_HumanEventsBlockViewer.sql index 90f32285..63697f08 100644 --- a/sp_HumanEvents/sp_HumanEventsBlockViewer.sql +++ b/sp_HumanEvents/sp_HumanEventsBlockViewer.sql @@ -538,7 +538,7 @@ BEGIN PRINT @validation_sql; END; - EXEC sys.sp_executesql + EXECUTE sys.sp_executesql @validation_sql, N' @database sysname, From 4d162815bb10c1f9d79e3d797e7105e93155b636 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 6 Mar 2025 07:44:56 -0500 Subject: [PATCH 006/246] Update sp_HumanEventsBlockViewer.sql --- sp_HumanEvents/sp_HumanEventsBlockViewer.sql | 34 +++++++++++++------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/sp_HumanEvents/sp_HumanEventsBlockViewer.sql b/sp_HumanEvents/sp_HumanEventsBlockViewer.sql index 63697f08..2fe61e21 100644 --- a/sp_HumanEvents/sp_HumanEventsBlockViewer.sql +++ b/sp_HumanEvents/sp_HumanEventsBlockViewer.sql @@ -388,6 +388,15 @@ SELECT @is_system_health_msg = CONVERT(nchar(1), @is_system_health); +/*Change this here in case someone leave it NULL*/ +IF @target_database IS NOT NULL +AND @target_schema IS NOT NULL +AND @target_table IS NOT NULL +AND @target_column IS NOT NULL +BEGIN + SET @target_type = N'table'; +END; + /* Check for table input early and validate */ IF @target_type = N'table' BEGIN @@ -486,7 +495,8 @@ BEGIN BEGIN RAISERROR(N''The specified @target_column %s must be of XML data type.'', 11, 1, @column) WITH NOWAIT; RETURN; - END;'; + END; + '; /* Validate timestamp_column if specified */ IF @timestamp_column IS NOT NULL @@ -973,7 +983,7 @@ BEGIN blocked_process_report = bd.query('.') INTO #blocked_sh FROM #blocking_xml_sh AS bx - OUTER APPLY bx.human_events_xml.nodes('/event') AS oa(C) + OUTER APPLY bx.human_events_xml.nodes('/event') AS oa(c) OUTER APPLY oa.c.nodes('//blocked-process-report/blocked-process') AS bd(bd) WHERE bd.exist('process/@spid') = 1 OPTION(RECOMPILE); @@ -1030,7 +1040,7 @@ BEGIN blocked_process_report = bg.query('.') INTO #blocking_sh FROM #blocking_xml_sh AS bx - OUTER APPLY bx.human_events_xml.nodes('/event') AS oa(C) + OUTER APPLY bx.human_events_xml.nodes('/event') AS oa(c) OUTER APPLY oa.c.nodes('//blocked-process-report/blocking-process') AS bg(bg) WHERE bg.exist('process/@spid') = 1 OPTION(RECOMPILE); @@ -1282,7 +1292,7 @@ BEGIN stmtend = ISNULL(n.c.value('@stmtend', 'integer'), -1) FROM #blocks_sh AS b - CROSS APPLY b.blocked_process_report.nodes('/blocking-process/process/executionStack/frame[not(@sqlhandle = "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")]') AS n(C) + CROSS APPLY b.blocked_process_report.nodes('/blocking-process/process/executionStack/frame[not(@sqlhandle = "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")]') AS n(c) WHERE (b.currentdbname = @database_name OR @database_name IS NULL) ) AS B @@ -1491,8 +1501,10 @@ BEGIN QUOTENAME(@target_schema) + N'.' + QUOTENAME(@target_table) + - N' - CROSS APPLY x.x.nodes(''/event'') AS e(x) + N' AS x + CROSS APPLY x.' + + QUOTENAME(@target_column) + + N'.nodes(''/event'') AS e(x) WHERE e.x.exist(''@name[ .= "blocked_process_report"]'') = 1'; /* Add timestamp filtering if specified*/ @@ -1503,13 +1515,13 @@ BEGIN IF @start_date IS NOT NULL BEGIN SET @extract_sql = @extract_sql + N' - AND ' + QUOTENAME(@timestamp_column) + N' >= @start_date'; + AND x.' + QUOTENAME(@timestamp_column) + N' >= @start_date'; END; IF @end_date IS NOT NULL BEGIN SET @extract_sql = @extract_sql + N' - AND ' + QUOTENAME(@timestamp_column) + N' < @end_date'; + AND x.' + QUOTENAME(@timestamp_column) + N' < @end_date'; END; END; @@ -1605,7 +1617,7 @@ SELECT blocked_process_report = c.query('.') INTO #blocked FROM #blocking_xml AS bx -OUTER APPLY bx.human_events_xml.nodes('/event') AS oa(C) +OUTER APPLY bx.human_events_xml.nodes('/event') AS oa(c) OUTER APPLY oa.c.nodes('//blocked-process-report/blocked-process') AS bd(bd) OUTER APPLY oa.c.nodes('//blocked-process-report/blocking-process') AS bg(bg) OPTION(RECOMPILE); @@ -1725,7 +1737,7 @@ SELECT blocked_process_report = c.query('.') INTO #blocking FROM #blocking_xml AS bx -OUTER APPLY bx.human_events_xml.nodes('/event') AS oa(C) +OUTER APPLY bx.human_events_xml.nodes('/event') AS oa(c) OUTER APPLY oa.c.nodes('//blocked-process-report/blocked-process') AS bd(bd) OUTER APPLY oa.c.nodes('//blocked-process-report/blocking-process') AS bg(bg) OPTION(RECOMPILE); @@ -2217,7 +2229,7 @@ FROM stmtend = ISNULL(n.c.value('@stmtend', 'integer'), -1) FROM #blocks AS b - CROSS APPLY b.blocked_process_report.nodes('/event/data/value/blocked-process-report/blocking-process/process/executionStack/frame[not(@sqlhandle = "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")]') AS n(C) + CROSS APPLY b.blocked_process_report.nodes('/event/data/value/blocked-process-report/blocking-process/process/executionStack/frame[not(@sqlhandle = "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")]') AS n(c) WHERE ( (b.database_name = @database_name From cf1ac8ccf249abd5c1ae5af51eec286ecc979e9f Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 6 Mar 2025 07:59:36 -0500 Subject: [PATCH 007/246] Update sp_HumanEventsBlockViewer.sql --- sp_HumanEvents/sp_HumanEventsBlockViewer.sql | 62 ++++++++------------ 1 file changed, 23 insertions(+), 39 deletions(-) diff --git a/sp_HumanEvents/sp_HumanEventsBlockViewer.sql b/sp_HumanEvents/sp_HumanEventsBlockViewer.sql index 2fe61e21..2dcba8ce 100644 --- a/sp_HumanEvents/sp_HumanEventsBlockViewer.sql +++ b/sp_HumanEvents/sp_HumanEventsBlockViewer.sql @@ -860,7 +860,7 @@ BEGIN SELECT human_events_xml = e.x.query('.') FROM #x AS x - CROSS APPLY x.x.nodes('/RingBufferTarget/event') AS E(x) + CROSS APPLY x.x.nodes('/RingBufferTarget/event') AS e(x) WHERE e.x.exist('@name[ .= "blocked_process_report"]') = 1 AND e.x.exist('@timestamp[. >= sql:variable("@start_date") and .< sql:variable("@end_date")]') = 1 OPTION(RECOMPILE); @@ -884,7 +884,7 @@ BEGIN SELECT human_events_xml = e.x.query('.') FROM #x AS x - CROSS APPLY x.x.nodes('/event') AS E(x) + CROSS APPLY x.x.nodes('/event') AS e(x) WHERE e.x.exist('@name[ .= "blocked_process_report"]') = 1 AND e.x.exist('@timestamp[. >= sql:variable("@start_date") and .< sql:variable("@end_date")]') = 1 OPTION(RECOMPILE); @@ -913,7 +913,7 @@ BEGIN FROM sys.fn_xe_file_target_read_file(N'system_health*.xel', NULL, NULL, NULL) AS fx WHERE fx.object_name = N'sp_server_diagnostics_component_result' ) AS xml - CROSS APPLY xml.sp_server_diagnostics_component_result.nodes('/event') AS E(x) + CROSS APPLY xml.sp_server_diagnostics_component_result.nodes('/event') AS e(x) WHERE e.x.exist('//data[@name="data"]/value/queryProcessing/blockingTasks/blocked-process-report') = 1 AND e.x.exist('@timestamp[. >= sql:variable("@start_date") and .< sql:variable("@end_date")]') = 1 OPTION(RECOMPILE); @@ -1295,7 +1295,7 @@ BEGIN CROSS APPLY b.blocked_process_report.nodes('/blocking-process/process/executionStack/frame[not(@sqlhandle = "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")]') AS n(c) WHERE (b.currentdbname = @database_name OR @database_name IS NULL) - ) AS B + ) AS b OPTION(RECOMPILE); IF @debug = 1 @@ -1484,13 +1484,6 @@ BEGIN /* Build dynamic SQL to extract the XML */ SET @extract_sql = N' - INSERT - #blocking_xml - WITH - (TABLOCKX) - ( - human_events_xml - ) SELECT human_events_xml = ' + QUOTENAME(@target_column) + @@ -1508,41 +1501,25 @@ BEGIN WHERE e.x.exist(''@name[ .= "blocked_process_report"]'') = 1'; /* Add timestamp filtering if specified*/ - IF (@start_date IS NOT NULL OR @end_date IS NOT NULL) + IF @timestamp_column IS NOT NULL BEGIN - IF @timestamp_column IS NOT NULL - BEGIN - IF @start_date IS NOT NULL - BEGIN - SET @extract_sql = @extract_sql + N' - AND x.' + QUOTENAME(@timestamp_column) + N' >= @start_date'; - END; - - IF @end_date IS NOT NULL - BEGIN - SET @extract_sql = @extract_sql + N' - AND x.' + QUOTENAME(@timestamp_column) + N' < @end_date'; - END; - END; + SET @extract_sql = @extract_sql + N' + AND x.' + QUOTENAME(@timestamp_column) + N' >= @start_date + AND x.' + QUOTENAME(@timestamp_column) + N' < @end_date'; + END; - IF @timestamp_column IS NULL + IF @timestamp_column IS NULL + BEGIN BEGIN - IF @start_date IS NOT NULL - BEGIN - SET @extract_sql = @extract_sql + N' - AND ' + QUOTENAME(@target_column) + N'.exist(''@timestamp[. >= sql:variable("@start_date")]'') = 1'; - END; - - IF @end_date IS NOT NULL - BEGIN - SET @extract_sql = @extract_sql + N' - AND ' + QUOTENAME(@target_column) + '.exist(''@timestamp[. < sql:variable("@end_date")]'') = 1'; - END; + SET @extract_sql = @extract_sql + N' + AND e.x.exist(''@timestamp[. >= sql:variable("@start_date") and . < sql:variable("@end_date")]'') = 1'; END; END; + SET @extract_sql = @extract_sql + N' - OPTION(RECOMPILE);'; + OPTION(RECOMPILE); + '; IF @debug = 1 BEGIN @@ -1550,6 +1527,13 @@ BEGIN END; /* Execute the dynamic SQL*/ + INSERT + #blocking_xml + WITH + (TABLOCKX) + ( + human_events_xml + ) EXECUTE sys.sp_executesql @extract_sql, N'@start_date datetime2, From c279785fb76b43b4d40070a04ed10c8d2dd390dc Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 6 Mar 2025 09:09:49 -0500 Subject: [PATCH 008/246] Update sp_HumanEventsBlockViewer.sql Tinkerings --- sp_HumanEvents/sp_HumanEventsBlockViewer.sql | 26 ++++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/sp_HumanEvents/sp_HumanEventsBlockViewer.sql b/sp_HumanEvents/sp_HumanEventsBlockViewer.sql index 2dcba8ce..b58bbe1c 100644 --- a/sp_HumanEvents/sp_HumanEventsBlockViewer.sql +++ b/sp_HumanEvents/sp_HumanEventsBlockViewer.sql @@ -398,13 +398,23 @@ BEGIN END; /* Check for table input early and validate */ -IF @target_type = N'table' +IF LOWER(@target_type) = N'table' BEGIN IF @debug = 1 BEGIN RAISERROR('Table source detected, validating parameters', 0, 1) WITH NOWAIT; END; + IF @target_database IS NULL + BEGIN + SET @target_database = DB_NAME(); + END; + + IF @target_schema IS NULL + BEGIN + SET @target_schema = N'dbo' + END; + /* Parameter validation */ IF @target_database IS NULL OR @target_schema IS NULL @@ -593,7 +603,7 @@ CREATE TABLE sort_order bigint ); -IF @target_type = N'table' +IF LOWER(@target_type) = N'table' BEGIN GOTO TableMode; RETURN; @@ -676,7 +686,7 @@ BEGIN END; /* Dump whatever we got into a temp table */ -IF @target_type = N'ring_buffer' +IF LOWER(@target_type) = N'ring_buffer' AND @is_system_health = 0 BEGIN IF @azure = 0 @@ -726,7 +736,7 @@ BEGIN END; END; -IF @target_type = N'event_file' +IF LOWER(@target_type) = N'event_file' AND @is_system_health = 0 BEGIN IF @azure = 0 @@ -842,7 +852,7 @@ BEGIN END; -IF @target_type = N'ring_buffer' +IF LOWER(@target_type) = N'ring_buffer' AND @is_system_health = 0 BEGIN IF @debug = 1 @@ -866,7 +876,7 @@ BEGIN OPTION(RECOMPILE); END; -IF @target_type = N'event_file' +IF LOWER(@target_type) = N'event_file' AND @is_system_health = 0 BEGIN IF @debug = 1 @@ -895,7 +905,7 @@ This section is special for the well-hidden and much less comprehensive blocked process report stored in the system health extended event session */ IF @is_system_health = 1 -AND @target_type <> N'table' +AND LOWER(@target_type) <> N'table' BEGIN IF @debug = 1 BEGIN @@ -1475,7 +1485,7 @@ BEGIN END; TableMode: -IF @target_type = N'table' +IF LOWER(@target_type) = N'table' BEGIN IF @debug = 1 BEGIN From 2475d8f2bcd901479d7b0a5330052e1d36d850d8 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 6 Mar 2025 23:26:48 -0500 Subject: [PATCH 009/246] Update sp_HealthParser.sql Add a bunch of new collections --- sp_HealthParser/sp_HealthParser.sql | 3041 +++++++++++++++++++-------- 1 file changed, 2139 insertions(+), 902 deletions(-) diff --git a/sp_HealthParser/sp_HealthParser.sql b/sp_HealthParser/sp_HealthParser.sql index 040cdf4c..8958f576 100644 --- a/sp_HealthParser/sp_HealthParser.sql +++ b/sp_HealthParser/sp_HealthParser.sql @@ -300,16 +300,14 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /*If any parameters that expect non-NULL default values get passed in with NULLs, fix them*/ SELECT - @what_to_check = ISNULL(@what_to_check, 'all'), + @what_to_check = LOWER(ISNULL(@what_to_check, 'all')), @warnings_only = ISNULL(@warnings_only, 0), @wait_duration_ms = ISNULL(@wait_duration_ms, 0), @wait_round_interval_minutes = ISNULL(@wait_round_interval_minutes, 60), @skip_locks = ISNULL(@skip_locks, 0), @pending_task_threshold = ISNULL(@pending_task_threshold, 10); - SELECT - @what_to_check = LOWER(@what_to_check); - + /*Validate what to check*/ IF @what_to_check NOT IN ( 'all', @@ -344,9 +342,15 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END; CREATE TABLE - #ignore + #ignore_waits + ( + wait_type nvarchar(60) NOT NULL + ); + + CREATE TABLE + #ignore_errors ( - wait_type nvarchar(60) + error_number integer NOT NULL ); CREATE TABLE @@ -379,16 +383,42 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ring_buffer xml NOT NULL ); + CREATE TABLE + #scheduler_monitor + ( + scheduler_monitor xml NOT NULL + ); + + CREATE TABLE + #error_reported + ( + error_reported xml NOT NULL + ); + + CREATE TABLE + #memory_broker + ( + memory_broker xml NOT NULL + ); + + CREATE TABLE + #memory_node_oom + ( + memory_node_oom xml NOT NULL + ); + /*The more you ignore waits, the worser they get*/ IF @what_to_check IN ('all', 'waits') BEGIN IF @debug = 1 BEGIN - RAISERROR('Inserting ignorable waits to #ignore', 0, 1) WITH NOWAIT; + RAISERROR('Inserting ignorable waits to #ignore_waits', 0, 1) WITH NOWAIT; END; INSERT - #ignore WITH(TABLOCKX) + #ignore_waits + WITH + (TABLOCKX) ( wait_type ) @@ -425,9 +455,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. IF @debug = 1 BEGIN SELECT - table_name = '#ignore', + table_name = '#ignore_waits', i.* - FROM #ignore AS i ORDER BY i.wait_type + FROM #ignore_waits AS i ORDER BY i.wait_type OPTION(RECOMPILE); END; @@ -483,7 +513,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END; INSERT INTO - #wait_info WITH (TABLOCKX) + #wait_info + WITH + (TABLOCKX) ( wait_info ) @@ -529,7 +561,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END; INSERT INTO - #sp_server_diagnostics_component_result WITH(TABLOCKX) + #sp_server_diagnostics_component_result + WITH + (TABLOCKX) ( sp_server_diagnostics_component_result ) @@ -651,7 +685,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END; INSERT INTO - #wait_info WITH (TABLOCKX) + #wait_info + WITH + (TABLOCKX) ( wait_info ) @@ -701,7 +737,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END; INSERT INTO - #sp_server_diagnostics_component_result WITH(TABLOCKX) + #sp_server_diagnostics_component_result + WITH + (TABLOCKX) ( sp_server_diagnostics_component_result ) @@ -759,7 +797,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END; INSERT INTO - #xml_deadlock_report WITH(TABLOCKX) + #xml_deadlock_report + WITH + (TABLOCKX) ( xml_deadlock_report ) @@ -776,171 +816,1232 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END; END; /*End < 2017 collection*/ - IF @mi = 1 + /*Scheduler monitor*/ + IF @what_to_check IN ('all', 'system', 'cpu') BEGIN IF @debug = 1 BEGIN - RAISERROR('Starting Managed Instance analysis', 0, 1) WITH NOWAIT; - RAISERROR('Inserting #x', 0, 1) WITH NOWAIT; + RAISERROR('Checking scheduler monitor system health', 0, 1) WITH NOWAIT; END; - - INSERT - #x WITH(TABLOCKX) + + /*2017+*/ + IF EXISTS ( - x + SELECT + 1/0 + FROM sys.all_columns AS ac + WHERE ac.object_id = OBJECT_ID(N'sys.fn_xe_file_target_read_file') + AND ac.name = N'timestamp_utc' ) - SELECT - x = - ISNULL - ( - TRY_CAST(t.target_data AS xml), - CONVERT(xml, N'event') - ) - FROM sys.dm_xe_session_targets AS t - JOIN sys.dm_xe_sessions AS s - ON s.address = t.event_session_address - WHERE s.name = N'system_health' - AND t.target_name = N'ring_buffer' - OPTION(RECOMPILE); - - IF @debug = 1 - BEGIN - SELECT TOP (100) - table_name = '#x, top 100 rows', - x.* - FROM #x AS x; - END; - - IF @debug = 1 + AND @mi = 0 BEGIN - RAISERROR('Inserting #ring_buffer', 0, 1) WITH NOWAIT; + SELECT + @sql = N' + SELECT + scheduler_monitor = + ISNULL + ( + xml.scheduler_monitor, + CONVERT(xml, N''event'') + ) + FROM + ( + SELECT + scheduler_monitor = + TRY_CAST(fx.event_data AS xml) + FROM sys.fn_xe_file_target_read_file(N''system_health*.xel'', NULL, NULL, NULL) AS fx + WHERE fx.object_name = N''scheduler_monitor_system_health'' + AND CONVERT(datetimeoffset(7), fx.timestamp_utc) BETWEEN @start_date AND @end_date + ) AS xml + CROSS APPLY xml.scheduler_monitor.nodes(''/event'') AS e(x) + OPTION(RECOMPILE);'; + + IF @debug = 1 + BEGIN + PRINT @sql; + RAISERROR('Inserting #scheduler_monitor', 0, 1) WITH NOWAIT; + SET STATISTICS XML ON; + END; + + INSERT INTO + #scheduler_monitor + WITH + (TABLOCKX) + ( + scheduler_monitor + ) + EXECUTE sys.sp_executesql + @sql, + @params, + @start_date, + @end_date; + + IF @debug = 1 + BEGIN + SET STATISTICS XML OFF; + END; END; - - INSERT - #ring_buffer WITH(TABLOCKX) - ( - ring_buffer - ) - SELECT - x = e.x.query('.') - FROM + + -- For pre-2017 without timestamp_utc + IF NOT EXISTS ( SELECT - x - FROM #x - ) AS x - CROSS APPLY x.x.nodes('//event') AS e(x) - WHERE 1 = 1 - AND e.x.exist('@timestamp[.>= sql:variable("@start_date") and .< sql:variable("@end_date")]') = 1 - AND e.x.exist('@name[.= "security_error_ring_buffer_recorded"]') = 0 - AND e.x.exist('@name[.= "error_reported"]') = 0 - AND e.x.exist('@name[.= "memory_broker_ring_buffer_recorded"]') = 0 - AND e.x.exist('@name[.= "connectivity_ring_buffer_recorded"]') = 0 - AND e.x.exist('@name[.= "scheduler_monitor_system_health_ring_buffer_recorded"]') = 0 - OPTION(RECOMPILE); - - IF @debug = 1 - BEGIN - SELECT TOP (100) - table_name = '#ring_buffer, top 100 rows', - x.* - FROM #ring_buffer AS x; - END; - - IF @what_to_check IN ('all', 'waits') + 1/0 + FROM sys.all_columns AS ac + WHERE ac.object_id = OBJECT_ID(N'sys.fn_xe_file_target_read_file') + AND ac.name = N'timestamp_utc' + ) + AND @mi = 0 BEGIN + SELECT + @sql = N' + SELECT + scheduler_monitor = + ISNULL + ( + xml.scheduler_monitor, + CONVERT(xml, N''event'') + ) + FROM + ( + SELECT + scheduler_monitor = + TRY_CAST(fx.event_data AS xml) + FROM sys.fn_xe_file_target_read_file(N''system_health*.xel'', NULL, NULL, NULL) AS fx + WHERE fx.object_name = N''scheduler_monitor_system_health'' + ) AS xml + CROSS APPLY xml.scheduler_monitor.nodes(''/event'') AS e(x) + CROSS APPLY (SELECT x.value( ''(@timestamp)[1]'', ''datetimeoffset'' )) ca ([utc_timestamp]) + WHERE ca.utc_timestamp >= @start_date + AND ca.utc_timestamp < @end_date + OPTION(RECOMPILE);'; + IF @debug = 1 BEGIN - RAISERROR('Checking Managed Instance waits', 0, 1) WITH NOWAIT; - RAISERROR('Inserting #wait_info', 0, 1) WITH NOWAIT; + PRINT @sql; + RAISERROR('Inserting #scheduler_monitor', 0, 1) WITH NOWAIT; + SET STATISTICS XML ON; END; - - INSERT - #wait_info WITH(TABLOCKX) + + INSERT INTO + #scheduler_monitor + WITH + (TABLOCKX) ( - wait_info + scheduler_monitor ) - SELECT - e.x.query('.') - FROM #ring_buffer AS rb - CROSS APPLY rb.ring_buffer.nodes('/event') AS e(x) - WHERE e.x.exist('@name[.= "wait_info"]') = 1 - OPTION(RECOMPILE); + EXECUTE sys.sp_executesql + @sql, + @params, + @start_date, + @end_date; + + IF @debug = 1 + BEGIN + SET STATISTICS XML OFF; + END; END; + END; + /*Memory broker*/ + IF @what_to_check IN ('all', 'memory') + BEGIN + /*Grab data from the memory_broker_ring_buffer component*/ IF @debug = 1 BEGIN - RAISERROR('Checking Managed Instance sp_server_diagnostics_component_result', 0, 1) WITH NOWAIT; - RAISERROR('Inserting #sp_server_diagnostics_component_result', 0, 1) WITH NOWAIT; + RAISERROR('Checking memory broker ring buffer', 0, 1) WITH NOWAIT; END; - - INSERT - #sp_server_diagnostics_component_result WITH(TABLOCKX) - ( - sp_server_diagnostics_component_result - ) - SELECT - e.x.query('.') - FROM #ring_buffer AS rb - CROSS APPLY rb.ring_buffer.nodes('/event') AS e(x) - WHERE e.x.exist('@name[.= "sp_server_diagnostics_component_result"]') = 1 - OPTION(RECOMPILE); - - IF + + /* + The column timestamp_utc is 2017+ only, but terribly broken: + https://dba.stackexchange.com/q/323147/32281 + https://feedback.azure.com/d365community/idea/5f8e52d6-f3d2-ec11-a81b-6045bd7ac9f9 + */ + IF EXISTS ( - @what_to_check IN ('all', 'locking') - AND @skip_locks = 0 + SELECT + 1/0 + FROM sys.all_columns AS ac + WHERE ac.object_id = OBJECT_ID(N'sys.fn_xe_file_target_read_file') + AND ac.name = N'timestamp_utc' ) + AND @mi = 0 BEGIN - IF @debug = 1 + SELECT + @sql = N' + SELECT + memory_broker = + ISNULL + ( + xml.memory_broker, + CONVERT(xml, N''event'') + ) + FROM + ( + SELECT + memory_broker = + TRY_CAST(fx.event_data AS xml) + FROM sys.fn_xe_file_target_read_file(N''system_health*.xel'', NULL, NULL, NULL) AS fx + WHERE fx.object_name = N''memory_broker_ring_buffer_recorded'' + AND CONVERT(datetimeoffset(7), fx.timestamp_utc) BETWEEN @start_date AND @end_date + ) AS xml + CROSS APPLY xml.memory_broker.nodes(''/event'') AS e(x) + OPTION(RECOMPILE);'; + + IF @debug = 1 BEGIN - RAISERROR('Checking Managed Instance deadlocks', 0, 1) WITH NOWAIT; - RAISERROR('Inserting #xml_deadlock_report', 0, 1) WITH NOWAIT; + PRINT @sql; + RAISERROR('Inserting #memory_broker', 0, 1) WITH NOWAIT; + SET STATISTICS XML ON; END; - - INSERT - #xml_deadlock_report WITH(TABLOCKX) + + INSERT INTO + #memory_broker + WITH + (TABLOCKX) ( - xml_deadlock_report + memory_broker ) - SELECT - e.x.query('.') - FROM #ring_buffer AS rb - CROSS APPLY rb.ring_buffer.nodes('/event') AS e(x) - WHERE e.x.exist('@name[.= "xml_deadlock_report"]') = 1 - OPTION(RECOMPILE); + EXECUTE sys.sp_executesql + @sql, + @params, + @start_date, + @end_date; + + IF @debug = 1 + BEGIN + SET STATISTICS XML OFF; + END; END; - END; /*End Managed Instance collection*/ - - IF @debug = 1 - BEGIN - SELECT TOP (100) - table_name = '#wait_info, top 100 rows', - x.* - FROM #wait_info AS x; - - SELECT TOP (100) - table_name = '#sp_server_diagnostics_component_result, top 100 rows', - x.* - FROM #sp_server_diagnostics_component_result AS x; - - SELECT TOP (100) - table_name = '#xml_deadlock_report, top 100 rows', - x.* - FROM #xml_deadlock_report AS x; - END; - - /*Parse out the wait_info data*/ - IF @what_to_check IN ('all', 'waits') - BEGIN - IF @debug = 1 + + IF NOT EXISTS + ( + SELECT + 1/0 + FROM sys.all_columns AS ac + WHERE ac.object_id = OBJECT_ID(N'sys.fn_xe_file_target_read_file') + AND ac.name = N'timestamp_utc' + ) + AND @mi = 0 BEGIN - RAISERROR('Parsing queries with significant waits', 0, 1) WITH NOWAIT; - END; - - SELECT - event_time = - DATEADD + IF @debug = 1 + BEGIN + RAISERROR('Checking memory broker for not Managed Instance, up to 2016', 0, 1) WITH NOWAIT; + END; + + SELECT + @sql = N' + SELECT + memory_broker = + ISNULL + ( + xml.memory_broker, + CONVERT(xml, N''event'') + ) + FROM + ( + SELECT + memory_broker = + TRY_CAST(fx.event_data AS xml) + FROM sys.fn_xe_file_target_read_file(N''system_health*.xel'', NULL, NULL, NULL) AS fx + WHERE fx.object_name = N''memory_broker_ring_buffer_recorded'' + ) AS xml + CROSS APPLY xml.memory_broker.nodes(''/event'') AS e(x) + CROSS APPLY (SELECT x.value( ''(@timestamp)[1]'', ''datetimeoffset'' )) ca ([utc_timestamp]) + WHERE ca.utc_timestamp >= @start_date + AND ca.utc_timestamp < @end_date + OPTION(RECOMPILE);'; + + IF @debug = 1 + BEGIN + PRINT @sql; + RAISERROR('Inserting #memory_broker', 0, 1) WITH NOWAIT; + SET STATISTICS XML ON; + END; + + INSERT INTO + #memory_broker + WITH + (TABLOCKX) + ( + memory_broker + ) + EXECUTE sys.sp_executesql + @sql, + @params, + @start_date, + @end_date; + + IF @debug = 1 + BEGIN + SET STATISTICS XML OFF; + END; + END; + END; /*End memory_broker data collection*/ + + IF @what_to_check IN ('all', 'system') + BEGIN + /*Grab data from the error_reported component*/ + IF @debug = 1 + BEGIN + RAISERROR('Checking error_reported events', 0, 1) WITH NOWAIT; + END; + + /* + The column timestamp_utc is 2017+ only, but terribly broken: + https://dba.stackexchange.com/q/323147/32281 + https://feedback.azure.com/d365community/idea/5f8e52d6-f3d2-ec11-a81b-6045bd7ac9f9 + */ + IF EXISTS + ( + SELECT + 1/0 + FROM sys.all_columns AS ac + WHERE ac.object_id = OBJECT_ID(N'sys.fn_xe_file_target_read_file') + AND ac.name = N'timestamp_utc' + ) + AND @mi = 0 + BEGIN + SELECT + @sql = N' + SELECT + error_reported = + ISNULL + ( + xml.error_reported, + CONVERT(xml, N''event'') + ) + FROM + ( + SELECT + error_reported = + TRY_CAST(fx.event_data AS xml) + FROM sys.fn_xe_file_target_read_file(N''system_health*.xel'', NULL, NULL, NULL) AS fx + WHERE fx.object_name = N''error_reported'' + AND CONVERT(datetimeoffset(7), fx.timestamp_utc) BETWEEN @start_date AND @end_date + ) AS xml + CROSS APPLY xml.error_reported.nodes(''/event'') AS e(x) + OPTION(RECOMPILE);'; + + IF @debug = 1 + BEGIN + PRINT @sql; + RAISERROR('Inserting #error_reported', 0, 1) WITH NOWAIT; + SET STATISTICS XML ON; + END; + + INSERT INTO + #error_reported + WITH + (TABLOCKX) + ( + error_reported + ) + EXECUTE sys.sp_executesql + @sql, + @params, + @start_date, + @end_date; + + IF @debug = 1 + BEGIN + SET STATISTICS XML OFF; + END; + END; + + IF NOT EXISTS + ( + SELECT + 1/0 + FROM sys.all_columns AS ac + WHERE ac.object_id = OBJECT_ID(N'sys.fn_xe_file_target_read_file') + AND ac.name = N'timestamp_utc' + ) + AND @mi = 0 + BEGIN + IF @debug = 1 + BEGIN + RAISERROR('Checking error_reported for not Managed Instance, up to 2016', 0, 1) WITH NOWAIT; + END; + + SELECT + @sql = N' + SELECT + error_reported = + ISNULL + ( + xml.error_reported, + CONVERT(xml, N''event'') + ) + FROM + ( + SELECT + error_reported = + TRY_CAST(fx.event_data AS xml) + FROM sys.fn_xe_file_target_read_file(N''system_health*.xel'', NULL, NULL, NULL) AS fx + WHERE fx.object_name = N''error_reported'' + ) AS xml + CROSS APPLY xml.error_reported.nodes(''/event'') AS e(x) + CROSS APPLY (SELECT x.value( ''(@timestamp)[1]'', ''datetimeoffset'' )) ca ([utc_timestamp]) + WHERE ca.utc_timestamp >= @start_date + AND ca.utc_timestamp < @end_date + OPTION(RECOMPILE);'; + + IF @debug = 1 + BEGIN + PRINT @sql; + RAISERROR('Inserting #error_reported', 0, 1) WITH NOWAIT; + SET STATISTICS XML ON; + END; + + INSERT INTO + #error_reported + WITH + (TABLOCKX) + ( + error_reported + ) + EXECUTE sys.sp_executesql + @sql, + @params, + @start_date, + @end_date; + + IF @debug = 1 + BEGIN + SET STATISTICS XML OFF; + END; + END; + END; /*End error_reported data collection*/ + + IF @what_to_check IN ('all', 'memory') + BEGIN + /*Grab data from the memory_node_oom component*/ + IF @debug = 1 + BEGIN + RAISERROR('Checking memory node OOM events', 0, 1) WITH NOWAIT; + END; + + /* + The column timestamp_utc is 2017+ only, but terribly broken: + https://dba.stackexchange.com/q/323147/32281 + https://feedback.azure.com/d365community/idea/5f8e52d6-f3d2-ec11-a81b-6045bd7ac9f9 + */ + IF EXISTS + ( + SELECT + 1/0 + FROM sys.all_columns AS ac + WHERE ac.object_id = OBJECT_ID(N'sys.fn_xe_file_target_read_file') + AND ac.name = N'timestamp_utc' + ) + AND @mi = 0 + BEGIN + SELECT + @sql = N' + SELECT + memory_node_oom = + ISNULL + ( + xml.memory_node_oom, + CONVERT(xml, N''event'') + ) + FROM + ( + SELECT + memory_node_oom = + TRY_CAST(fx.event_data AS xml) + FROM sys.fn_xe_file_target_read_file(N''system_health*.xel'', NULL, NULL, NULL) AS fx + WHERE fx.object_name = N''memory_node_oom_ring_buffer_recorded'' + AND CONVERT(datetimeoffset(7), fx.timestamp_utc) BETWEEN @start_date AND @end_date + ) AS xml + CROSS APPLY xml.memory_node_oom.nodes(''/event'') AS e(x) + OPTION(RECOMPILE);'; + + IF @debug = 1 + BEGIN + PRINT @sql; + RAISERROR('Inserting #memory_node_oom', 0, 1) WITH NOWAIT; + SET STATISTICS XML ON; + END; + + INSERT INTO + #memory_node_oom + WITH + (TABLOCKX) + ( + memory_node_oom + ) + EXECUTE sys.sp_executesql + @sql, + @params, + @start_date, + @end_date; + + IF @debug = 1 + BEGIN + SET STATISTICS XML OFF; + END; + END; + + IF NOT EXISTS + ( + SELECT + 1/0 + FROM sys.all_columns AS ac + WHERE ac.object_id = OBJECT_ID(N'sys.fn_xe_file_target_read_file') + AND ac.name = N'timestamp_utc' + ) + AND @mi = 0 + BEGIN + IF @debug = 1 + BEGIN + RAISERROR('Checking memory node OOM for not Managed Instance, up to 2016', 0, 1) WITH NOWAIT; + END; + + SELECT + @sql = N' + SELECT + memory_node_oom = + ISNULL + ( + xml.memory_node_oom, + CONVERT(xml, N''event'') + ) + FROM + ( + SELECT + memory_node_oom = + TRY_CAST(fx.event_data AS xml) + FROM sys.fn_xe_file_target_read_file(N''system_health*.xel'', NULL, NULL, NULL) AS fx + WHERE fx.object_name = N''memory_node_oom_ring_buffer_recorded'' + ) AS xml + CROSS APPLY xml.memory_node_oom.nodes(''/event'') AS e(x) + CROSS APPLY (SELECT x.value( ''(@timestamp)[1]'', ''datetimeoffset'' )) ca ([utc_timestamp]) + WHERE ca.utc_timestamp >= @start_date + AND ca.utc_timestamp < @end_date + OPTION(RECOMPILE);'; + + IF @debug = 1 + BEGIN + PRINT @sql; + RAISERROR('Inserting #memory_node_oom', 0, 1) WITH NOWAIT; + SET STATISTICS XML ON; + END; + + INSERT INTO + #memory_node_oom + WITH + (TABLOCKX) + ( + memory_node_oom + ) + EXECUTE sys.sp_executesql + @sql, + @params, + @start_date, + @end_date; + + IF @debug = 1 + BEGIN + SET STATISTICS XML OFF; + END; + END; + END; /*End memory_node_oom data collection*/ + + IF @mi = 1 + BEGIN + IF @debug = 1 + BEGIN + RAISERROR('Starting Managed Instance analysis', 0, 1) WITH NOWAIT; + RAISERROR('Inserting #x', 0, 1) WITH NOWAIT; + END; + + INSERT + #x + WITH + (TABLOCKX) + ( + x + ) + SELECT + x = + ISNULL + ( + TRY_CAST(t.target_data AS xml), + CONVERT(xml, N'event') + ) + FROM sys.dm_xe_session_targets AS t + JOIN sys.dm_xe_sessions AS s + ON s.address = t.event_session_address + WHERE s.name = N'system_health' + AND t.target_name = N'ring_buffer' + OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + SELECT TOP (100) + table_name = '#x, top 100 rows', + x.* + FROM #x AS x; + END; + + IF @debug = 1 + BEGIN + RAISERROR('Inserting #ring_buffer', 0, 1) WITH NOWAIT; + END; + + INSERT + #ring_buffer + WITH + (TABLOCKX) + ( + ring_buffer + ) + SELECT + x = e.x.query('.') + FROM + ( + SELECT + x + FROM #x + ) AS x + CROSS APPLY x.x.nodes('//event') AS e(x) + WHERE 1 = 1 + AND e.x.exist('@timestamp[.>= sql:variable("@start_date") and .< sql:variable("@end_date")]') = 1 + AND e.x.exist('@name[.= "security_error_ring_buffer_recorded"]') = 0 + AND e.x.exist('@name[.= "connectivity_ring_buffer_recorded"]') = 0 + OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + SELECT TOP (100) + table_name = '#ring_buffer, top 100 rows', + x.* + FROM #ring_buffer AS x; + END; + + IF @what_to_check IN ('all', 'waits') + BEGIN + IF @debug = 1 + BEGIN + RAISERROR('Checking Managed Instance waits', 0, 1) WITH NOWAIT; + RAISERROR('Inserting #wait_info', 0, 1) WITH NOWAIT; + END; + + INSERT + #wait_info + WITH + (TABLOCKX) + ( + wait_info + ) + SELECT + e.x.query('.') + FROM #ring_buffer AS rb + CROSS APPLY rb.ring_buffer.nodes('/event') AS e(x) + WHERE e.x.exist('@name[.= "wait_info"]') = 1 + OPTION(RECOMPILE); + END; + + IF @debug = 1 + BEGIN + RAISERROR('Checking Managed Instance sp_server_diagnostics_component_result', 0, 1) WITH NOWAIT; + RAISERROR('Inserting #sp_server_diagnostics_component_result', 0, 1) WITH NOWAIT; + END; + + INSERT + #sp_server_diagnostics_component_result + WITH + (TABLOCKX) + ( + sp_server_diagnostics_component_result + ) + SELECT + e.x.query('.') + FROM #ring_buffer AS rb + CROSS APPLY rb.ring_buffer.nodes('/event') AS e(x) + WHERE e.x.exist('@name[.= "sp_server_diagnostics_component_result"]') = 1 + OPTION(RECOMPILE); + + IF + ( + @what_to_check IN ('all', 'locking') + AND @skip_locks = 0 + ) + BEGIN + IF @debug = 1 + BEGIN + RAISERROR('Checking Managed Instance deadlocks', 0, 1) WITH NOWAIT; + RAISERROR('Inserting #xml_deadlock_report', 0, 1) WITH NOWAIT; + END; + + INSERT + #xml_deadlock_report + WITH + (TABLOCKX) + ( + xml_deadlock_report + ) + SELECT + e.x.query('.') + FROM #ring_buffer AS rb + CROSS APPLY rb.ring_buffer.nodes('/event') AS e(x) + WHERE e.x.exist('@name[.= "xml_deadlock_report"]') = 1 + OPTION(RECOMPILE); + END; + END; /*End Managed Instance collection*/ + + IF @debug = 1 + BEGIN + SELECT TOP (100) + table_name = '#wait_info, top 100 rows', + x.* + FROM #wait_info AS x; + + SELECT TOP (100) + table_name = '#sp_server_diagnostics_component_result, top 100 rows', + x.* + FROM #sp_server_diagnostics_component_result AS x; + + SELECT TOP (100) + table_name = '#xml_deadlock_report, top 100 rows', + x.* + FROM #xml_deadlock_report AS x; + END; + + /*Parse out the wait_info data*/ + IF @what_to_check IN ('all', 'waits') + BEGIN + IF @debug = 1 + BEGIN + RAISERROR('Parsing queries with significant waits', 0, 1) WITH NOWAIT; + END; + + SELECT + event_time = + DATEADD + ( + MINUTE, + DATEDIFF + ( + MINUTE, + GETUTCDATE(), + SYSDATETIME() + ), + w.x.value('@timestamp', 'datetime2') + ), + wait_type = w.x.value('(data[@name="wait_type"]/text/text())[1]', 'nvarchar(60)'), + duration_ms = CONVERT(bigint, w.x.value('(data[@name="duration"]/value/text())[1]', 'bigint')), + signal_duration_ms = CONVERT(bigint, w.x.value('(data[@name="signal_duration"]/value/text())[1]', 'bigint')), + wait_resource = w.x.value('(data[@name="wait_resource"]/value/text())[1]', 'nvarchar(256)'), + sql_text_pre = w.x.value('(action[@name="sql_text"]/value/text())[1]', 'nvarchar(max)'), + session_id = w.x.value('(action[@name="session_id"]/value/text())[1]', 'integer'), + xml = w.x.query('.') + INTO #waits_queries + FROM #wait_info AS wi + CROSS APPLY wi.wait_info.nodes('//event') AS w(x) + WHERE w.x.exist('(action[@name="session_id"]/value/text())[.= 0]') = 0 + AND w.x.exist('(action[@name="sql_text"]/value/text())') = 1 + AND w.x.exist('(action[@name="sql_text"]/value/text()[contains(upper-case(.), "BACKUP")] )') = 0 + AND w.x.exist('(data[@name="duration"]/value/text())[.>= sql:variable("@wait_duration_ms")]') = 1 + AND NOT EXISTS + ( + SELECT + 1/0 + FROM #ignore_waits AS i + WHERE w.x.exist('(data[@name="wait_type"]/text/text())[1][.= sql:column("i.wait_type")]') = 1 + ) + OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + RAISERROR('Adding query_text to #waits_queries', 0, 1) WITH NOWAIT; + END; + + ALTER TABLE #waits_queries + ADD query_text AS + REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE( + REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE( + REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE( + sql_text_pre COLLATE Latin1_General_BIN2, + NCHAR(31),N'?'),NCHAR(30),N'?'),NCHAR(29),N'?'),NCHAR(28),N'?'),NCHAR(27),N'?'),NCHAR(26),N'?'),NCHAR(25),N'?'),NCHAR(24),N'?'),NCHAR(23),N'?'),NCHAR(22),N'?'), + NCHAR(21),N'?'),NCHAR(20),N'?'),NCHAR(19),N'?'),NCHAR(18),N'?'),NCHAR(17),N'?'),NCHAR(16),N'?'),NCHAR(15),N'?'),NCHAR(14),N'?'),NCHAR(12),N'?'), + NCHAR(11),N'?'),NCHAR(8),N'?'),NCHAR(7),N'?'),NCHAR(6),N'?'),NCHAR(5),N'?'),NCHAR(4),N'?'),NCHAR(3),N'?'),NCHAR(2),N'?'),NCHAR(1),N'?'),NCHAR(0),N'?') + PERSISTED; + + IF @debug = 1 + BEGIN + SELECT TOP (100) + table_name = '#waits_queries, top 100 rows', + x.* + FROM #waits_queries AS x + ORDER BY + x.event_time DESC; + END; + + IF NOT EXISTS + ( + SELECT + 1/0 + FROM #waits_queries AS wq + ) + BEGIN + SELECT + finding = + CASE + WHEN @what_to_check NOT IN ('all', 'waits') + THEN 'waits skipped, @what_to_check set to ' + + @what_to_check + WHEN @what_to_check IN ('all', 'waits') + THEN 'no queries with significant waits found between ' + + RTRIM(CONVERT(date, @start_date)) + + ' and ' + + RTRIM(CONVERT(date, @end_date)) + + ' with a minimum duration of ' + + RTRIM(@wait_duration_ms) + + '.' + ELSE 'no queries with significant waits found!' + END; + END; + ELSE + BEGIN + SELECT + finding = 'queries with significant waits', + wq.event_time, + wq.wait_type, + duration_ms = + REPLACE + ( + CONVERT + ( + nvarchar(30), + CONVERT + ( + money, + wq.duration_ms + ), + 1 + ), + N'.00', + N'' + ), + signal_duration_ms = + REPLACE + ( + CONVERT + ( + nvarchar(30), + CONVERT + ( + money, + wq.signal_duration_ms + ), + 1 + ), + N'.00', + N'' + ), + wq.wait_resource, + query_text = + ( + SELECT + [processing-instruction(query)] = + wq.query_text + FOR XML + PATH(N''), + TYPE + ), + wq.session_id + FROM #waits_queries AS wq + ORDER BY + wq.duration_ms DESC + OPTION(RECOMPILE); + END; + + /*Waits by count*/ + IF @debug = 1 + BEGIN + RAISERROR('Parsing #waits_by_count', 0, 1) WITH NOWAIT; + END; + + SELECT + event_time = + DATEADD + ( + MINUTE, + DATEDIFF + ( + MINUTE, + GETUTCDATE(), + SYSDATETIME() + ), + w.x.value('@timestamp', 'datetime2') + ), + wait_type = w2.x2.value('@waitType', 'nvarchar(60)'), + waits = w2.x2.value('@waits', 'bigint'), + average_wait_time_ms = CONVERT(bigint, w2.x2.value('@averageWaitTime', 'bigint')), + max_wait_time_ms = CONVERT(bigint, w2.x2.value('@maxWaitTime', 'bigint')), + xml = w.x.query('.') + INTO #topwaits_count + FROM #sp_server_diagnostics_component_result AS wi + CROSS APPLY wi.sp_server_diagnostics_component_result.nodes('/event') AS w(x) + CROSS APPLY w.x.nodes('/event/data/value/queryProcessing/topWaits/nonPreemptive/byCount/wait') AS w2(x2) + WHERE w.x.exist('(data[@name="component"]/text[.= "QUERY_PROCESSING"])') = 1 + AND (w.x.exist('(data[@name="state"]/text[.= "WARNING"])') = @warnings_only OR @warnings_only = 0) + AND NOT EXISTS + ( + SELECT + 1/0 + FROM #ignore_waits AS i + WHERE w2.x2.exist('@waitType[.= sql:column("i.wait_type")]') = 1 + ) + OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + SELECT TOP (100) + table_name = '#topwaits_count, top 100 rows', + x.* + FROM #topwaits_count AS x + ORDER BY + x.event_time DESC; + END; + + SELECT + finding = 'waits by count', + event_time_rounded = + DATEADD + ( + MINUTE, + DATEDIFF + ( + MINUTE, + '19000101', + tc.event_time + ) / @wait_round_interval_minutes * + @wait_round_interval_minutes, + '19000101' + ), + tc.wait_type, + waits = SUM(CONVERT(bigint, tc.waits)), + average_wait_time_ms = CONVERT(bigint, AVG(tc.average_wait_time_ms)), + max_wait_time_ms = CONVERT(bigint, MAX(tc.max_wait_time_ms)) + INTO #tc + FROM #topwaits_count AS tc + GROUP BY + tc.wait_type, + DATEADD + ( + MINUTE, + DATEDIFF + ( + MINUTE, + '19000101', + tc.event_time + ) / @wait_round_interval_minutes * + @wait_round_interval_minutes, + '19000101' + ) + OPTION(RECOMPILE); + + IF NOT EXISTS + ( + SELECT + 1/0 + FROM #tc AS t + ) + BEGIN + SELECT + finding = + CASE + WHEN @what_to_check NOT IN ('all', 'waits') + THEN 'waits skipped, @what_to_check set to ' + + @what_to_check + WHEN @what_to_check IN ('all', 'waits') + THEN 'no significant waits found between ' + + RTRIM(CONVERT(date, @start_date)) + + ' and ' + + RTRIM(CONVERT(date, @end_date)) + + '.' + ELSE 'no significant waits found!' + END; + END; + ELSE + BEGIN + SELECT + t.finding, + t.event_time_rounded, + t.wait_type, + waits = + REPLACE + ( + CONVERT + ( + nvarchar(30), + CONVERT + ( + money, + t.waits + ), + 1 + ), + N'.00', + N'' + ), + average_wait_time_ms = + REPLACE + ( + CONVERT + ( + nvarchar(30), + CONVERT + ( + money, + t.average_wait_time_ms + ), + 1 + ), + N'.00', + N'' + ), + max_wait_time_ms = + REPLACE + ( + CONVERT + ( + nvarchar(30), + CONVERT + ( + money, + t.max_wait_time_ms + ), + 1 + ), + N'.00', + N'' + ) + FROM #tc AS t + ORDER BY + t.event_time_rounded DESC, + t.waits DESC + OPTION(RECOMPILE); + END; + + /*Grab waits by duration*/ + IF @debug = 1 + BEGIN + RAISERROR('Parsing waits by duration', 0, 1) WITH NOWAIT; + END; + + SELECT + event_time = + DATEADD + ( + MINUTE, + DATEDIFF + ( + MINUTE, + GETUTCDATE(), + SYSDATETIME() + ), + w.x.value('@timestamp', 'datetime2') + ), + wait_type = w2.x2.value('@waitType', 'nvarchar(60)'), + waits = w2.x2.value('@waits', 'bigint'), + average_wait_time_ms = CONVERT(bigint, w2.x2.value('@averageWaitTime', 'bigint')), + max_wait_time_ms = CONVERT(bigint, w2.x2.value('@maxWaitTime', 'bigint')), + xml = w.x.query('.') + INTO #topwaits_duration + FROM #sp_server_diagnostics_component_result AS wi + CROSS APPLY wi.sp_server_diagnostics_component_result.nodes('/event') AS w(x) + CROSS APPLY w.x.nodes('/event/data/value/queryProcessing/topWaits/nonPreemptive/byDuration/wait') AS w2(x2) + WHERE w.x.exist('(data[@name="component"]/text[.= "QUERY_PROCESSING"])') = 1 + AND (w.x.exist('(data[@name="state"]/text[.= "WARNING"])') = @warnings_only OR @warnings_only = 0) + AND w2.x2.exist('@averageWaitTime[.>= sql:variable("@wait_duration_ms")]') = 1 + AND NOT EXISTS + ( + SELECT + 1/0 + FROM #ignore_waits AS i + WHERE w2.x2.exist('@waitType[.= sql:column("i.wait_type")]') = 1 + ) + OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + SELECT TOP (100) + table_name = '#topwaits_duration, top 100 rows', + x.* + FROM #topwaits_duration AS x + ORDER BY + x.event_time DESC; + END; + + SELECT + finding = 'waits by duration', + event_time_rounded = + DATEADD + ( + MINUTE, + DATEDIFF + ( + MINUTE, + '19000101', + td.event_time + ) / @wait_round_interval_minutes * + @wait_round_interval_minutes, + '19000101' + ), + td.wait_type, + td.waits, + td.average_wait_time_ms, + td.max_wait_time_ms + INTO #td + FROM #topwaits_duration AS td + GROUP BY + td.wait_type, + DATEADD + ( + MINUTE, + DATEDIFF + ( + MINUTE, + '19000101', + td.event_time + ) / @wait_round_interval_minutes * + @wait_round_interval_minutes, + '19000101' + ), + td.waits, + td.average_wait_time_ms, + td.max_wait_time_ms + OPTION(RECOMPILE); + + IF NOT EXISTS + ( + SELECT + 1/0 + FROM #td AS t + ) + BEGIN + SELECT + finding = + CASE + WHEN @what_to_check NOT IN ('all', 'waits') + THEN 'waits skipped, @what_to_check set to ' + + @what_to_check + WHEN @what_to_check IN ('all', 'waits') + THEN 'no significant waits found between ' + + RTRIM(CONVERT(date, @start_date)) + + ' and ' + + RTRIM(CONVERT(date, @end_date)) + + ' with a minimum average duration of ' + + RTRIM(@wait_duration_ms) + + '.' + ELSE 'no significant waits found!' + END; + END; + ELSE + BEGIN + SELECT + x.finding, + x.event_time_rounded, + x.wait_type, + x.average_wait_time_ms, + x.max_wait_time_ms + FROM + ( + SELECT + t.finding, + t.event_time_rounded, + t.wait_type, + waits = + REPLACE + ( + CONVERT + ( + nvarchar(30), + CONVERT + ( + money, + t.waits + ), + 1 + ), + N'.00', + N'' + ), + average_wait_time_ms = + REPLACE + ( + CONVERT + ( + nvarchar(30), + CONVERT + ( + money, + t.average_wait_time_ms + ), + 1 + ), + N'.00', + N'' + ), + max_wait_time_ms = + REPLACE + ( + CONVERT + ( + nvarchar(30), + CONVERT + ( + money, + t.max_wait_time_ms + ), + 1 + ), + N'.00', + N'' + ), + s = + ROW_NUMBER() OVER + ( + ORDER BY + t.event_time_rounded DESC, + t.waits DESC + ), + n = + ROW_NUMBER() OVER + ( + PARTITION BY + t.wait_type, + t.waits, + t.average_wait_time_ms, + t.max_wait_time_ms + ORDER BY + t.event_time_rounded + ) + FROM #td AS t + ) AS x + WHERE x.n = 1 + ORDER BY + x.s + OPTION(RECOMPILE); + END; + END; /*End wait stats*/ + + /*Grab IO stuff*/ + IF @what_to_check IN ('all', 'disk') + BEGIN + IF @debug = 1 + BEGIN + RAISERROR('Parsing disk stuff', 0, 1) WITH NOWAIT; + END; + + SELECT + event_time = + DATEADD ( MINUTE, DATEDIFF @@ -951,51 +2052,158 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ), w.x.value('@timestamp', 'datetime2') ), - wait_type = w.x.value('(data[@name="wait_type"]/text/text())[1]', 'nvarchar(60)'), - duration_ms = CONVERT(bigint, w.x.value('(data[@name="duration"]/value/text())[1]', 'bigint')), - signal_duration_ms = CONVERT(bigint, w.x.value('(data[@name="signal_duration"]/value/text())[1]', 'bigint')), - wait_resource = w.x.value('(data[@name="wait_resource"]/value/text())[1]', 'nvarchar(256)'), - sql_text_pre = w.x.value('(action[@name="sql_text"]/value/text())[1]', 'nvarchar(max)'), - session_id = w.x.value('(action[@name="session_id"]/value/text())[1]', 'integer'), + state = w.x.value('(data[@name="state"]/text/text())[1]', 'nvarchar(256)'), + ioLatchTimeouts = w.x.value('(/event/data[@name="data"]/value/ioSubsystem/@ioLatchTimeouts)[1]', 'bigint'), + intervalLongIos = w.x.value('(/event/data[@name="data"]/value/ioSubsystem/@intervalLongIos)[1]', 'bigint'), + totalLongIos = w.x.value('(/event/data[@name="data"]/value/ioSubsystem/@totalLongIos)[1]', 'bigint'), + longestPendingRequests_duration_ms = CONVERT(bigint, w2.x2.value('@duration', 'bigint')), + longestPendingRequests_filePath = w2.x2.value('@filePath', 'nvarchar(500)'), xml = w.x.query('.') - INTO #waits_queries - FROM #wait_info AS wi - CROSS APPLY wi.wait_info.nodes('//event') AS w(x) - WHERE w.x.exist('(action[@name="session_id"]/value/text())[.= 0]') = 0 - AND w.x.exist('(action[@name="sql_text"]/value/text())') = 1 - AND w.x.exist('(action[@name="sql_text"]/value/text()[contains(., "BACKUP")] )') = 0 - AND w.x.exist('(data[@name="duration"]/value/text())[.>= sql:variable("@wait_duration_ms")]') = 1 - AND NOT EXISTS - ( - SELECT - 1/0 - FROM #ignore AS i - WHERE w.x.exist('(data[@name="wait_type"]/text/text())[1][.= sql:column("i.wait_type")]') = 1 - ) + INTO #io + FROM #sp_server_diagnostics_component_result AS wi + CROSS APPLY wi.sp_server_diagnostics_component_result.nodes('//event') AS w(x) + OUTER APPLY w.x.nodes('/event/data/value/ioSubsystem/longestPendingRequests/pendingRequest') AS w2(x2) + WHERE w.x.exist('(data[@name="component"]/text[.= "IO_SUBSYSTEM"])') = 1 + AND (w.x.exist('(data[@name="state"]/text[.= "WARNING"])') = @warnings_only OR @warnings_only = 0) OPTION(RECOMPILE); IF @debug = 1 BEGIN - RAISERROR('Adding query_text to #waits_queries', 0, 1) WITH NOWAIT; + SELECT TOP (100) + table_name = '#io, top 100 rows', + x.* + FROM #io AS x + ORDER BY + x.event_time DESC; END; - ALTER TABLE #waits_queries - ADD query_text AS - REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE( - REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE( - REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE( - sql_text_pre COLLATE Latin1_General_BIN2, - NCHAR(31),N'?'),NCHAR(30),N'?'),NCHAR(29),N'?'),NCHAR(28),N'?'),NCHAR(27),N'?'),NCHAR(26),N'?'),NCHAR(25),N'?'),NCHAR(24),N'?'),NCHAR(23),N'?'),NCHAR(22),N'?'), - NCHAR(21),N'?'),NCHAR(20),N'?'),NCHAR(19),N'?'),NCHAR(18),N'?'),NCHAR(17),N'?'),NCHAR(16),N'?'),NCHAR(15),N'?'),NCHAR(14),N'?'),NCHAR(12),N'?'), - NCHAR(11),N'?'),NCHAR(8),N'?'),NCHAR(7),N'?'),NCHAR(6),N'?'),NCHAR(5),N'?'),NCHAR(4),N'?'),NCHAR(3),N'?'),NCHAR(2),N'?'),NCHAR(1),N'?'),NCHAR(0),N'?') - PERSISTED; + SELECT + finding = 'potential io issues', + i.event_time, + i.state, + i.ioLatchTimeouts, + i.intervalLongIos, + i.totalLongIos, + longestPendingRequests_duration_ms = + ISNULL(SUM(i.longestPendingRequests_duration_ms), 0), + longestPendingRequests_filePath = + ISNULL(i.longestPendingRequests_filePath, 'N/A') + INTO #i + FROM #io AS i + GROUP BY + i.event_time, + i.state, + i.ioLatchTimeouts, + i.intervalLongIos, + i.totalLongIos, + ISNULL(i.longestPendingRequests_filePath, 'N/A') + OPTION(RECOMPILE); + + IF NOT EXISTS + ( + SELECT + 1/0 + FROM #i AS i + ) + BEGIN + SELECT + finding = + CASE + WHEN @what_to_check NOT IN ('all', 'disk') + THEN 'disk skipped, @what_to_check set to ' + + @what_to_check + WHEN @what_to_check IN ('all', 'disk') + THEN 'no io issues found between ' + + RTRIM(CONVERT(date, @start_date)) + + ' and ' + + RTRIM(CONVERT(date, @end_date)) + + ' with @warnings_only set to ' + + RTRIM(@warnings_only) + + '.' + ELSE 'no io issues found!' + END; + END; + ELSE + BEGIN + SELECT + i.finding, + i.event_time, + i.state, + i.ioLatchTimeouts, + i.intervalLongIos, + i.totalLongIos, + longestPendingRequests_duration_ms = + REPLACE + ( + CONVERT + ( + nvarchar(30), + CONVERT + ( + money, + i.longestPendingRequests_duration_ms + ), + 1 + ), + N'.00', + N'' + ), + i.longestPendingRequests_filePath + FROM #i AS i + ORDER BY + i.event_time DESC + OPTION(RECOMPILE); + END; + END; /*End disk*/ + + /*Grab CPU details*/ + IF @what_to_check IN ('all', 'cpu') + BEGIN + IF @debug = 1 + BEGIN + RAISERROR('Parsing CPU stuff', 0, 1) WITH NOWAIT; + END; + + SELECT + event_time = + DATEADD + ( + MINUTE, + DATEDIFF + ( + MINUTE, + GETUTCDATE(), + SYSDATETIME() + ), + w.x.value('@timestamp', 'datetime2') + ), + name = w.x.value('@name', 'nvarchar(256)'), + component = w.x.value('(data[@name="component"]/text/text())[1]', 'nvarchar(256)'), + state = w.x.value('(data[@name="state"]/text/text())[1]', 'nvarchar(256)'), + maxWorkers = w.x.value('(/event/data[@name="data"]/value/queryProcessing/@maxWorkers)[1]', 'bigint'), + workersCreated = w.x.value('(/event/data[@name="data"]/value/queryProcessing/@workersCreated)[1]', 'bigint'), + workersIdle = w.x.value('(/event/data[@name="data"]/value/queryProcessing/@workersIdle)[1]', 'bigint'), + tasksCompletedWithinInterval = w.x.value('(//data[@name="data"]/value/queryProcessing/@tasksCompletedWithinInterval)[1]', 'bigint'), + pendingTasks = w.x.value('(/event/data[@name="data"]/value/queryProcessing/@pendingTasks)[1]', 'bigint'), + oldestPendingTaskWaitingTime = w.x.value('(/event/data[@name="data"]/value/queryProcessing/@oldestPendingTaskWaitingTime)[1]', 'bigint'), + hasUnresolvableDeadlockOccurred = w.x.value('(/event/data[@name="data"]/value/queryProcessing/@hasUnresolvableDeadlockOccurred)[1]', 'bit'), + hasDeadlockedSchedulersOccurred = w.x.value('(/event/data[@name="data"]/value/queryProcessing/@hasDeadlockedSchedulersOccurred)[1]', 'bit'), + didBlockingOccur = w.x.exist('//data[@name="data"]/value/queryProcessing/blockingTasks/blocked-process-report'), + xml = w.x.query('.') + INTO #scheduler_details + FROM #sp_server_diagnostics_component_result AS wi + CROSS APPLY wi.sp_server_diagnostics_component_result.nodes('/event') AS w(x) + WHERE w.x.exist('(data[@name="component"]/text[.= "QUERY_PROCESSING"])') = 1 + AND (w.x.exist('(data[@name="state"]/text[.= "WARNING"])') = @warnings_only OR @warnings_only IS NULL) + AND (w.x.exist('(/event/data[@name="data"]/value/queryProcessing/@pendingTasks[.>= sql:variable("@pending_task_threshold")])') = 1 OR @warnings_only = 0) + OPTION(RECOMPILE); IF @debug = 1 BEGIN SELECT TOP (100) - table_name = '#waits_queries, top 100 rows', + table_name = '#scheduler_details, top 100 rows', x.* - FROM #waits_queries AS x + FROM #scheduler_details AS x ORDER BY x.event_time DESC; END; @@ -1004,85 +2212,54 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ( SELECT 1/0 - FROM #waits_queries AS wq + FROM #scheduler_details AS sd ) BEGIN SELECT finding = CASE - WHEN @what_to_check NOT IN ('all', 'waits') - THEN 'waits skipped, @what_to_check set to ' + + WHEN @what_to_check NOT IN ('all', 'cpu') + THEN 'cpu skipped, @what_to_check set to ' + @what_to_check - WHEN @what_to_check IN ('all', 'waits') - THEN 'no queries with significant waits found between ' + + WHEN @what_to_check IN ('all', 'cpu') + THEN 'no cpu issues found between ' + RTRIM(CONVERT(date, @start_date)) + ' and ' + RTRIM(CONVERT(date, @end_date)) + - ' with a minimum duration of ' + - RTRIM(@wait_duration_ms) + + ' with @warnings_only set to ' + + RTRIM(@warnings_only) + '.' - ELSE 'no queries with significant waits found!' + ELSE 'no cpu issues found!' END; END; ELSE BEGIN SELECT - finding = 'queries with significant waits', - wq.event_time, - wq.wait_type, - duration_ms = - REPLACE - ( - CONVERT - ( - nvarchar(30), - CONVERT - ( - money, - wq.duration_ms - ), - 1 - ), - N'.00', - N'' - ), - signal_duration_ms = - REPLACE - ( - CONVERT - ( - nvarchar(30), - CONVERT - ( - money, - wq.signal_duration_ms - ), - 1 - ), - N'.00', - N'' - ), - wq.wait_resource, - query_text = - ( - SELECT - [processing-instruction(query)] = - wq.query_text - FOR XML - PATH(N''), - TYPE - ), - wq.session_id - FROM #waits_queries AS wq + finding = 'cpu task details', + sd.event_time, + sd.state, + sd.maxWorkers, + sd.workersCreated, + sd.workersIdle, + sd.tasksCompletedWithinInterval, + sd.pendingTasks, + sd.oldestPendingTaskWaitingTime, + sd.hasUnresolvableDeadlockOccurred, + sd.hasDeadlockedSchedulersOccurred, + sd.didBlockingOccur + FROM #scheduler_details AS sd ORDER BY - wq.duration_ms DESC + sd.event_time DESC OPTION(RECOMPILE); END; + END; /*End CPU*/ - /*Waits by count*/ + /*Grab memory details*/ + IF @what_to_check IN ('all', 'memory') + BEGIN IF @debug = 1 BEGIN - RAISERROR('Parsing #waits_by_count', 0, 1) WITH NOWAIT; + RAISERROR('Parsing memory stuff', 0, 1) WITH NOWAIT; END; SELECT @@ -1096,104 +2273,201 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. GETUTCDATE(), SYSDATETIME() ), - w.x.value('@timestamp', 'datetime2') + s.sp_server_diagnostics_component_result.value('(//@timestamp)[1]', 'datetime2') ), - wait_type = w2.x2.value('@waitType', 'nvarchar(60)'), - waits = w2.x2.value('@waits', 'bigint'), - average_wait_time_ms = CONVERT(bigint, w2.x2.value('@averageWaitTime', 'bigint')), - max_wait_time_ms = CONVERT(bigint, w2.x2.value('@maxWaitTime', 'bigint')), - xml = w.x.query('.') - INTO #topwaits_count - FROM #sp_server_diagnostics_component_result AS wi - CROSS APPLY wi.sp_server_diagnostics_component_result.nodes('/event') AS w(x) - CROSS APPLY w.x.nodes('/event/data/value/queryProcessing/topWaits/nonPreemptive/byCount/wait') AS w2(x2) - WHERE w.x.exist('(data[@name="component"]/text[.= "QUERY_PROCESSING"])') = 1 - AND (w.x.exist('(data[@name="state"]/text[.= "WARNING"])') = @warnings_only OR @warnings_only = 0) - AND NOT EXISTS - ( - SELECT - 1/0 - FROM #ignore AS i - WHERE w2.x2.exist('@waitType[.= sql:column("i.wait_type")]') = 1 - ) + lastNotification = r.c.value('@lastNotification', 'varchar(128)'), + outOfMemoryExceptions = r.c.value('@outOfMemoryExceptions', 'bigint'), + isAnyPoolOutOfMemory = r.c.value('@isAnyPoolOutOfMemory', 'bit'), + processOutOfMemoryPeriod = r.c.value('@processOutOfMemoryPeriod', 'bigint'), + name = r.c.value('(//memoryReport/@name)[1]', 'varchar(128)'), + available_physical_memory_gb = CONVERT(bigint, r.c.value('(//memoryReport/entry[@description[.="Available Physical Memory"]]/@value)[1]', 'bigint') / 1024 / 1024 / 1024), + available_virtual_memory_gb = CONVERT(bigint, r.c.value('(//memoryReport/entry[@description[.="Available Virtual Memory"]]/@value)[1]', 'bigint') / 1024 / 1024 / 1024), + available_paging_file_gb = CONVERT(bigint, r.c.value('(//memoryReport/entry[@description[.="Available Paging File"]]/@value)[1]', 'bigint') / 1024 / 1024 / 1024), + working_set_gb = CONVERT(bigint, r.c.value('(//memoryReport/entry[@description[.="Working Set"]]/@value)[1]', 'bigint') / 1024 / 1024 / 1024), + percent_of_committed_memory_in_ws = r.c.value('(//memoryReport/entry[@description[.="Percent of Committed Memory in WS"]]/@value)[1]', 'bigint'), + page_faults = r.c.value('(//memoryReport/entry[@description[.="Page Faults"]]/@value)[1]', 'bigint'), + system_physical_memory_high = r.c.value('(//memoryReport/entry[@description[.="System physical memory high"]]/@value)[1]', 'bigint'), + system_physical_memory_low = r.c.value('(//memoryReport/entry[@description[.="System physical memory low"]]/@value)[1]', 'bigint'), + process_physical_memory_low = r.c.value('(//memoryReport/entry[@description[.="Process physical memory low"]]/@value)[1]', 'bigint'), + process_virtual_memory_low = r.c.value('(//memoryReport/entry[@description[.="Process virtual memory low"]]/@value)[1]', 'bigint'), + vm_reserved_gb = CONVERT(bigint, r.c.value('(//memoryReport/entry[@description[.="VM Reserved"]]/@value)[1]', 'bigint') / 1024 / 1024), + vm_committed_gb = CONVERT(bigint, r.c.value('(//memoryReport/entry[@description[.="VM Committed"]]/@value)[1]', 'bigint') / 1024 / 1024), + locked_pages_allocated = r.c.value('(//memoryReport/entry[@description[.="Locked Pages Allocated"]]/@value)[1]', 'bigint'), + large_pages_allocated = r.c.value('(//memoryReport/entry[@description[.="Large Pages Allocated"]]/@value)[1]', 'bigint'), + emergency_memory_gb = CONVERT(bigint, r.c.value('(//memoryReport/entry[@description[.="Emergency Memory"]]/@value)[1]', 'bigint') / 1024 / 1024), + emergency_memory_in_use_gb = CONVERT(bigint, r.c.value('(//memoryReport/entry[@description[.="Emergency Memory In Use"]]/@value)[1]', 'bigint') / 1024 / 1024), + target_committed_gb = CONVERT(bigint, r.c.value('(//memoryReport/entry[@description[.="Target Committed"]]/@value)[1]', 'bigint') / 1024 / 1024), + current_committed_gb = CONVERT(bigint, r.c.value('(//memoryReport/entry[@description[.="Current Committed"]]/@value)[1]', 'bigint') / 1024 / 1024), + pages_allocated = r.c.value('(//memoryReport/entry[@description[.="Pages Allocated"]]/@value)[1]', 'bigint'), + pages_reserved = r.c.value('(//memoryReport/entry[@description[.="Pages Reserved"]]/@value)[1]', 'bigint'), + pages_free = r.c.value('(//memoryReport/entry[@description[.="Pages Free"]]/@value)[1]', 'bigint'), + pages_in_use = r.c.value('(//memoryReport/entry[@description[.="Pages In Use"]]/@value)[1]', 'bigint'), + page_alloc_potential = r.c.value('(//memoryReport/entry[@description[.="Page Alloc Potential"]]/@value)[1]', 'bigint'), + numa_growth_phase = r.c.value('(//memoryReport/entry[@description[.="NUMA Growth Phase"]]/@value)[1]', 'bigint'), + last_oom_factor = r.c.value('(//memoryReport/entry[@description[.="Last OOM Factor"]]/@value)[1]', 'bigint'), + last_os_error = r.c.value('(//memoryReport/entry[@description[.="Last OS Error"]]/@value)[1]', 'bigint'), + xml = r.c.query('.') + INTO #memory + FROM #sp_server_diagnostics_component_result AS s + CROSS APPLY s.sp_server_diagnostics_component_result.nodes('/event/data/value/resource') AS r(c) + WHERE (r.c.exist('@lastNotification[.= "RESOURCE_MEMPHYSICAL_LOW"]') = @warnings_only OR @warnings_only = 0) OPTION(RECOMPILE); IF @debug = 1 BEGIN SELECT TOP (100) - table_name = '#topwaits_count, top 100 rows', + table_name = '#memory, top 100 rows', x.* - FROM #topwaits_count AS x + FROM #memory AS x ORDER BY x.event_time DESC; END; - SELECT - finding = 'waits by count', - event_time_rounded = - DATEADD - ( - MINUTE, - DATEDIFF - ( - MINUTE, - '19000101', - tc.event_time - ) / @wait_round_interval_minutes * - @wait_round_interval_minutes, - '19000101' - ), - tc.wait_type, - waits = SUM(CONVERT(bigint, tc.waits)), - average_wait_time_ms = CONVERT(bigint, AVG(tc.average_wait_time_ms)), - max_wait_time_ms = CONVERT(bigint, MAX(tc.max_wait_time_ms)) - INTO #tc - FROM #topwaits_count AS tc - GROUP BY - tc.wait_type, - DATEADD - ( - MINUTE, - DATEDIFF + IF NOT EXISTS + ( + SELECT + 1/0 + FROM #memory AS m + ) + BEGIN + SELECT + finding = + CASE + WHEN @what_to_check NOT IN ('all', 'memory') + THEN 'memory skipped, @what_to_check set to ' + + @what_to_check + WHEN @what_to_check IN ('all', 'memory') + THEN 'no memory issues found between ' + + RTRIM(CONVERT(date, @start_date)) + + ' and ' + + RTRIM(CONVERT(date, @end_date)) + + ' with @warnings_only set to ' + + RTRIM(@warnings_only) + + '.' + ELSE 'no memory issues found!' + END; + END; + ELSE + BEGIN + SELECT + finding = 'memory conditions', + m.event_time, + m.lastNotification, + m.outOfMemoryExceptions, + m.isAnyPoolOutOfMemory, + m.processOutOfMemoryPeriod, + m.name, + m.available_physical_memory_gb, + m.available_virtual_memory_gb, + m.available_paging_file_gb, + m.working_set_gb, + m.percent_of_committed_memory_in_ws, + m.page_faults, + m.system_physical_memory_high, + m.system_physical_memory_low, + m.process_physical_memory_low, + m.process_virtual_memory_low, + m.vm_reserved_gb, + m.vm_committed_gb, + m.locked_pages_allocated, + m.large_pages_allocated, + m.emergency_memory_gb, + m.emergency_memory_in_use_gb, + m.target_committed_gb, + m.current_committed_gb, + m.pages_allocated, + m.pages_reserved, + m.pages_free, + m.pages_in_use, + m.page_alloc_potential, + m.numa_growth_phase, + m.last_oom_factor, + m.last_os_error + FROM #memory AS m + ORDER BY + m.event_time DESC + OPTION(RECOMPILE); + END; + END; /*End memory*/ + + /*Parse memory broker data*/ + IF @what_to_check IN ('all', 'memory') + BEGIN + IF @debug = 1 + BEGIN + RAISERROR('Parsing memory broker data', 0, 1) WITH NOWAIT; + END; + + SELECT + event_time = + DATEADD ( MINUTE, - '19000101', - tc.event_time - ) / @wait_round_interval_minutes * - @wait_round_interval_minutes, - '19000101' - ) + DATEDIFF + ( + MINUTE, + GETUTCDATE(), + SYSDATETIME() + ), + w.x.value('@timestamp', 'datetime2') + ), + notification_type = w.x.value('(data[@name="notification_type"]/text)[1]', 'nvarchar(256)'), + reclaim_target_kb = w.x.value('(data[@name="reclaim_target_kb"]/value)[1]', 'bigint'), + reclaimed_kb = w.x.value('(data[@name="reclaimed_kb"]/value)[1]', 'bigint'), + pressure = w.x.value('(data[@name="pressure"]/value)[1]', 'bigint'), + pressure_mb = w.x.value('(data[@name="pressure"]/value)[1]', 'bigint') / 1024, + currently_available_kb = w.x.value('(data[@name="currently_available_kb"]/value)[1]', 'bigint'), + reserved_kb = w.x.value('(data[@name="reserved_kb"]/value)[1]', 'bigint'), + committed_kb = w.x.value('(data[@name="committed_kb"]/value)[1]', 'bigint'), + worker_count = w.x.value('(data[@name="worker_count"]/value)[1]', 'integer'), + xml = w.x.query('.') + INTO #memory_broker_info + FROM #memory_broker AS mb + CROSS APPLY mb.memory_broker.nodes('//event') AS w(x) + WHERE (w.x.exist('(data[@name="notification_type"]/text[.= "RESOURCE_MEMPHYSICAL_LOW"])') = @warnings_only OR @warnings_only = 0) OPTION(RECOMPILE); - + + IF @debug = 1 + BEGIN + SELECT TOP (100) + table_name = '#memory_broker_info, top 100 rows', + x.* + FROM #memory_broker_info AS x + ORDER BY + x.event_time DESC; + END; + IF NOT EXISTS ( SELECT 1/0 - FROM #tc AS t + FROM #memory_broker_info AS mbi ) BEGIN SELECT finding = CASE - WHEN @what_to_check NOT IN ('all', 'waits') - THEN 'waits skipped, @what_to_check set to ' + + WHEN @what_to_check NOT IN ('all', 'memory') + THEN 'memory broker skipped, @what_to_check set to ' + @what_to_check - WHEN @what_to_check IN ('all', 'waits') - THEN 'no significant waits found between ' + + WHEN @what_to_check IN ('all', 'memory') + THEN 'no memory pressure events found between ' + RTRIM(CONVERT(date, @start_date)) + ' and ' + RTRIM(CONVERT(date, @end_date)) + + ' with @warnings_only set to ' + + RTRIM(@warnings_only) + '.' - ELSE 'no significant waits found!' + ELSE 'no memory pressure events found!' END; - END; + END ELSE BEGIN SELECT - t.finding, - t.event_time_rounded, - t.wait_type, - waits = + finding = 'memory broker notifications', + mbi.event_time, + mbi.notification_type, + reclaim_target_kb = REPLACE ( CONVERT @@ -1202,14 +2476,14 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CONVERT ( money, - t.waits + mbi.reclaim_target_kb ), 1 ), N'.00', N'' ), - average_wait_time_ms = + reclaimed_kb = REPLACE ( CONVERT @@ -1218,14 +2492,20 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CONVERT ( money, - t.average_wait_time_ms + mbi.reclaimed_kb ), 1 ), N'.00', N'' ), - max_wait_time_ms = + reclaim_success_percent = + CASE + WHEN mbi.reclaim_target_kb > 0 + THEN CONVERT(DECIMAL(5,2), 100.0 * mbi.reclaimed_kb / mbi.reclaim_target_kb) + ELSE 0 + END, + pressure_mb = REPLACE ( CONVERT @@ -1234,231 +2514,77 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CONVERT ( money, - t.max_wait_time_ms + mbi.pressure_mb ), 1 ), N'.00', N'' - ) - FROM #tc AS t - ORDER BY - t.event_time_rounded DESC, - t.waits DESC - OPTION(RECOMPILE); - END; - - /*Grab waits by duration*/ - IF @debug = 1 - BEGIN - RAISERROR('Parsing waits by duration', 0, 1) WITH NOWAIT; - END; - - SELECT - event_time = - DATEADD - ( - MINUTE, - DATEDIFF - ( - MINUTE, - GETUTCDATE(), - SYSDATETIME() ), - w.x.value('@timestamp', 'datetime2') - ), - wait_type = w2.x2.value('@waitType', 'nvarchar(60)'), - waits = w2.x2.value('@waits', 'bigint'), - average_wait_time_ms = CONVERT(bigint, w2.x2.value('@averageWaitTime', 'bigint')), - max_wait_time_ms = CONVERT(bigint, w2.x2.value('@maxWaitTime', 'bigint')), - xml = w.x.query('.') - INTO #topwaits_duration - FROM #sp_server_diagnostics_component_result AS wi - CROSS APPLY wi.sp_server_diagnostics_component_result.nodes('/event') AS w(x) - CROSS APPLY w.x.nodes('/event/data/value/queryProcessing/topWaits/nonPreemptive/byDuration/wait') AS w2(x2) - WHERE w.x.exist('(data[@name="component"]/text[.= "QUERY_PROCESSING"])') = 1 - AND (w.x.exist('(data[@name="state"]/text[.= "WARNING"])') = @warnings_only OR @warnings_only = 0) - AND w2.x2.exist('@averageWaitTime[.>= sql:variable("@wait_duration_ms")]') = 1 - AND NOT EXISTS - ( - SELECT - 1/0 - FROM #ignore AS i - WHERE w2.x2.exist('@waitType[.= sql:column("i.wait_type")]') = 1 - ) - OPTION(RECOMPILE); - - IF @debug = 1 - BEGIN - SELECT TOP (100) - table_name = '#topwaits_duration, top 100 rows', - x.* - FROM #topwaits_duration AS x - ORDER BY - x.event_time DESC; - END; - - SELECT - finding = 'waits by duration', - event_time_rounded = - DATEADD - ( - MINUTE, - DATEDIFF + currently_available_kb = + REPLACE ( - MINUTE, - '19000101', - td.event_time - ) / @wait_round_interval_minutes * - @wait_round_interval_minutes, - '19000101' - ), - td.wait_type, - td.waits, - td.average_wait_time_ms, - td.max_wait_time_ms - INTO #td - FROM #topwaits_duration AS td - GROUP BY - td.wait_type, - DATEADD - ( - MINUTE, - DATEDIFF - ( - MINUTE, - '19000101', - td.event_time - ) / @wait_round_interval_minutes * - @wait_round_interval_minutes, - '19000101' - ), - td.waits, - td.average_wait_time_ms, - td.max_wait_time_ms - OPTION(RECOMPILE); - - IF NOT EXISTS - ( - SELECT - 1/0 - FROM #td AS t - ) - BEGIN - SELECT - finding = - CASE - WHEN @what_to_check NOT IN ('all', 'waits') - THEN 'waits skipped, @what_to_check set to ' + - @what_to_check - WHEN @what_to_check IN ('all', 'waits') - THEN 'no significant waits found between ' + - RTRIM(CONVERT(date, @start_date)) + - ' and ' + - RTRIM(CONVERT(date, @end_date)) + - ' with a minimum average duration of ' + - RTRIM(@wait_duration_ms) + - '.' - ELSE 'no significant waits found!' - END; - END; - ELSE - BEGIN - SELECT - x.finding, - x.event_time_rounded, - x.wait_type, - x.average_wait_time_ms, - x.max_wait_time_ms - FROM - ( - SELECT - t.finding, - t.event_time_rounded, - t.wait_type, - waits = - REPLACE - ( - CONVERT - ( - nvarchar(30), - CONVERT - ( - money, - t.waits - ), - 1 - ), - N'.00', - N'' - ), - average_wait_time_ms = - REPLACE + CONVERT ( + nvarchar(30), CONVERT ( - nvarchar(30), - CONVERT - ( - money, - t.average_wait_time_ms - ), - 1 + money, + mbi.currently_available_kb ), - N'.00', - N'' + 1 ), - max_wait_time_ms = - REPLACE + N'.00', + N'' + ), + reserved_kb = + REPLACE + ( + CONVERT ( + nvarchar(30), CONVERT ( - nvarchar(30), - CONVERT - ( - money, - t.max_wait_time_ms - ), - 1 + money, + mbi.reserved_kb ), - N'.00', - N'' - ), - s = - ROW_NUMBER() OVER - ( - ORDER BY - t.event_time_rounded DESC, - t.waits DESC + 1 ), - n = - ROW_NUMBER() OVER - ( - PARTITION BY - t.wait_type, - t.waits, - t.average_wait_time_ms, - t.max_wait_time_ms - ORDER BY - t.event_time_rounded - ) - FROM #td AS t - ) AS x - WHERE x.n = 1 + N'.00', + N'' + ), + committed_kb = + REPLACE + ( + CONVERT + ( + nvarchar(30), + CONVERT + ( + money, + mbi.committed_kb + ), + 1 + ), + N'.00', + N'' + ), + mbi.worker_count + FROM #memory_broker_info AS mbi ORDER BY - x.s + mbi.event_time DESC OPTION(RECOMPILE); END; - END; /*End wait stats*/ + END; /*End memory broker analysis*/ - /*Grab IO stuff*/ - IF @what_to_check IN ('all', 'disk') + /*Parse memory node OOM data*/ + IF @what_to_check IN ('all', 'memory') BEGIN IF @debug = 1 BEGIN - RAISERROR('Parsing disk stuff', 0, 1) WITH NOWAIT; + RAISERROR('Parsing memory node OOM data', 0, 1) WITH NOWAIT; END; - + SELECT event_time = DATEADD @@ -1472,87 +2598,58 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ), w.x.value('@timestamp', 'datetime2') ), - state = w.x.value('(data[@name="state"]/text/text())[1]', 'nvarchar(256)'), - ioLatchTimeouts = w.x.value('(/event/data[@name="data"]/value/ioSubsystem/@ioLatchTimeouts)[1]', 'bigint'), - intervalLongIos = w.x.value('(/event/data[@name="data"]/value/ioSubsystem/@intervalLongIos)[1]', 'bigint'), - totalLongIos = w.x.value('(/event/data[@name="data"]/value/ioSubsystem/@totalLongIos)[1]', 'bigint'), - longestPendingRequests_duration_ms = CONVERT(bigint, w2.x2.value('@duration', 'bigint')), - longestPendingRequests_filePath = w2.x2.value('@filePath', 'nvarchar(500)'), + node_id = w.x.value('(data[@name="id"]/value)[1]', 'int'), + memory_available_kb = w.x.value('(data[@name="availableMemory"]/value)[1]', 'bigint'), + memory_requested_kb = w.x.value('(data[@name="requestedMemory"]/value)[1]', 'bigint'), + memory_allocator = w.x.value('(data[@name="allocator"]/text)[1]', 'nvarchar(256)'), + memory_allocation_type = w.x.value('(data[@name="allocationType"]/text)[1]', 'nvarchar(256)'), + memory_clerk_name = w.x.value('(data[@name="memoryClerk"]/text)[1]', 'nvarchar(256)'), + os_error = w.x.value('(data[@name="oserror"]/value)[1]', 'integer'), xml = w.x.query('.') - INTO #io - FROM #sp_server_diagnostics_component_result AS wi - CROSS APPLY wi.sp_server_diagnostics_component_result.nodes('//event') AS w(x) - OUTER APPLY w.x.nodes('/event/data/value/ioSubsystem/longestPendingRequests/pendingRequest') AS w2(x2) - WHERE w.x.exist('(data[@name="component"]/text[.= "IO_SUBSYSTEM"])') = 1 - AND (w.x.exist('(data[@name="state"]/text[.= "WARNING"])') = @warnings_only OR @warnings_only = 0) + INTO #memory_node_oom_info + FROM #memory_node_oom AS mno + CROSS APPLY mno.memory_node_oom.nodes('//event') AS w(x) OPTION(RECOMPILE); - + IF @debug = 1 BEGIN SELECT TOP (100) - table_name = '#io, top 100 rows', + table_name = '#memory_node_oom_info, top 100 rows', x.* - FROM #io AS x + FROM #memory_node_oom_info AS x ORDER BY x.event_time DESC; END; - - SELECT - finding = 'potential io issues', - i.event_time, - i.state, - i.ioLatchTimeouts, - i.intervalLongIos, - i.totalLongIos, - longestPendingRequests_duration_ms = - ISNULL(SUM(i.longestPendingRequests_duration_ms), 0), - longestPendingRequests_filePath = - ISNULL(i.longestPendingRequests_filePath, 'N/A') - INTO #i - FROM #io AS i - GROUP BY - i.event_time, - i.state, - i.ioLatchTimeouts, - i.intervalLongIos, - i.totalLongIos, - ISNULL(i.longestPendingRequests_filePath, 'N/A') - OPTION(RECOMPILE); - + IF NOT EXISTS ( SELECT 1/0 - FROM #i AS i + FROM #memory_node_oom_info AS mnoi ) BEGIN SELECT finding = CASE - WHEN @what_to_check NOT IN ('all', 'disk') - THEN 'disk skipped, @what_to_check set to ' + + WHEN @what_to_check NOT IN ('all', 'memory') + THEN 'memory node OOM skipped, @what_to_check set to ' + @what_to_check - WHEN @what_to_check IN ('all', 'disk') - THEN 'no io issues found between ' + + WHEN @what_to_check IN ('all', 'memory') + THEN 'no memory node OOM events found between ' + RTRIM(CONVERT(date, @start_date)) + ' and ' + RTRIM(CONVERT(date, @end_date)) + - ' with @warnings_only set to ' + - RTRIM(@warnings_only) + '.' - ELSE 'no io issues found!' + ELSE 'no memory node OOM events found!' END; - END; + END ELSE BEGIN SELECT - i.finding, - i.event_time, - i.state, - i.ioLatchTimeouts, - i.intervalLongIos, - i.totalLongIos, - longestPendingRequests_duration_ms = + finding = 'memory node OOM events', + mnoi.event_time, + mnoi.node_id, + memory_available_kb = REPLACE ( CONVERT @@ -1561,27 +2658,78 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CONVERT ( money, - i.longestPendingRequests_duration_ms + mnoi.memory_available_kb ), 1 ), N'.00', N'' ), - i.longestPendingRequests_filePath - FROM #i AS i + memory_requested_kb = + REPLACE + ( + CONVERT + ( + nvarchar(30), + CONVERT + ( + money, + mnoi.memory_requested_kb + ), + 1 + ), + N'.00', + N'' + ), + memory_available_mb = + REPLACE + ( + CONVERT + ( + nvarchar(30), + CONVERT + ( + money, + mnoi.memory_available_kb / 1024.0 + ), + 1 + ), + N'.00', + N'' + ), + memory_requested_mb = + REPLACE + ( + CONVERT + ( + nvarchar(30), + CONVERT + ( + money, + mnoi.memory_requested_kb / 1024.0 + ), + 1 + ), + N'.00', + N'' + ), + mnoi.memory_allocator, + mnoi.memory_allocation_type, + mnoi.memory_clerk_name, + mnoi.os_error + FROM #memory_node_oom_info AS mnoi ORDER BY - i.event_time DESC + mnoi.event_time DESC OPTION(RECOMPILE); END; - END; /*End disk*/ + END; /*End memory node OOM analysis*/ - /*Grab CPU details*/ - IF @what_to_check IN ('all', 'cpu') + /*Grab health stuff*/ + IF @what_to_check IN ('all', 'system') BEGIN IF @debug = 1 BEGIN - RAISERROR('Parsing CPU stuff', 0, 1) WITH NOWAIT; + RAISERROR('Parsing system stuff', 0, 1) WITH NOWAIT; END; SELECT @@ -1597,33 +2745,35 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ), w.x.value('@timestamp', 'datetime2') ), - name = w.x.value('@name', 'nvarchar(256)'), - component = w.x.value('(data[@name="component"]/text/text())[1]', 'nvarchar(256)'), state = w.x.value('(data[@name="state"]/text/text())[1]', 'nvarchar(256)'), - maxWorkers = w.x.value('(/event/data[@name="data"]/value/queryProcessing/@maxWorkers)[1]', 'bigint'), - workersCreated = w.x.value('(/event/data[@name="data"]/value/queryProcessing/@workersCreated)[1]', 'bigint'), - workersIdle = w.x.value('(/event/data[@name="data"]/value/queryProcessing/@workersIdle)[1]', 'bigint'), - tasksCompletedWithinInterval = w.x.value('(//data[@name="data"]/value/queryProcessing/@tasksCompletedWithinInterval)[1]', 'bigint'), - pendingTasks = w.x.value('(/event/data[@name="data"]/value/queryProcessing/@pendingTasks)[1]', 'bigint'), - oldestPendingTaskWaitingTime = w.x.value('(/event/data[@name="data"]/value/queryProcessing/@oldestPendingTaskWaitingTime)[1]', 'bigint'), - hasUnresolvableDeadlockOccurred = w.x.value('(/event/data[@name="data"]/value/queryProcessing/@hasUnresolvableDeadlockOccurred)[1]', 'bit'), - hasDeadlockedSchedulersOccurred = w.x.value('(/event/data[@name="data"]/value/queryProcessing/@hasDeadlockedSchedulersOccurred)[1]', 'bit'), - didBlockingOccur = w.x.exist('//data[@name="data"]/value/queryProcessing/blockingTasks/blocked-process-report'), + spinlockBackoffs = w.x.value('(/event/data[@name="data"]/value/system/@spinlockBackoffs)[1]', 'bigint'), + sickSpinlockType = w.x.value('(/event/data[@name="data"]/value/system/@sickSpinlockType)[1]', 'nvarchar(256)'), + sickSpinlockTypeAfterAv = w.x.value('(/event/data[@name="data"]/value/system/@sickSpinlockTypeAfterAv)[1]', 'nvarchar(256)'), + latchWarnings = w.x.value('(/event/data[@name="data"]/value/system/@latchWarnings)[1]', 'bigint'), + isAccessViolationOccurred = w.x.value('(/event/data[@name="data"]/value/system/@isAccessViolationOccurred)[1]', 'bigint'), + writeAccessViolationCount = w.x.value('(/event/data[@name="data"]/value/system/@writeAccessViolationCount)[1]', 'bigint'), + totalDumpRequests = w.x.value('(/event/data[@name="data"]/value/system/@totalDumpRequests)[1]', 'bigint'), + intervalDumpRequests = w.x.value('(/event/data[@name="data"]/value/system/@intervalDumpRequests)[1]', 'bigint'), + nonYieldingTasksReported = w.x.value('(/event/data[@name="data"]/value/system/@nonYieldingTasksReported)[1]', 'bigint'), + pageFaults = w.x.value('(/event/data[@name="data"]/value/system/@pageFaults)[1]', 'bigint'), + systemCpuUtilization = w.x.value('(/event/data[@name="data"]/value/system/@systemCpuUtilization)[1]', 'bigint'), + sqlCpuUtilization = w.x.value('(/event/data[@name="data"]/value/system/@sqlCpuUtilization)[1]', 'bigint'), + BadPagesDetected = w.x.value('(/event/data[@name="data"]/value/system/@BadPagesDetected)[1]', 'bigint'), + BadPagesFixed = w.x.value('(/event/data[@name="data"]/value/system/@BadPagesFixed)[1]', 'bigint'), xml = w.x.query('.') - INTO #scheduler_details + INTO #health FROM #sp_server_diagnostics_component_result AS wi - CROSS APPLY wi.sp_server_diagnostics_component_result.nodes('/event') AS w(x) - WHERE w.x.exist('(data[@name="component"]/text[.= "QUERY_PROCESSING"])') = 1 - AND (w.x.exist('(data[@name="state"]/text[.= "WARNING"])') = @warnings_only OR @warnings_only IS NULL) - AND (w.x.exist('(/event/data[@name="data"]/value/queryProcessing/@pendingTasks[.>= sql:variable("@pending_task_threshold")])') = 1 OR @warnings_only = 0) + CROSS APPLY wi.sp_server_diagnostics_component_result.nodes('//event') AS w(x) + WHERE w.x.exist('(data[@name="component"]/text[.= "SYSTEM"])') = 1 + AND (w.x.exist('(data[@name="state"]/text[.= "WARNING"])') = @warnings_only OR @warnings_only = 0) OPTION(RECOMPILE); IF @debug = 1 BEGIN SELECT TOP (100) - table_name = '#scheduler_details, top 100 rows', + table_name = '#health, top 100 rows', x.* - FROM #scheduler_details AS x + FROM #health AS x ORDER BY x.event_time DESC; END; @@ -1632,56 +2782,61 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ( SELECT 1/0 - FROM #scheduler_details AS sd + FROM #health AS h ) BEGIN SELECT finding = CASE - WHEN @what_to_check NOT IN ('all', 'cpu') - THEN 'cpu skipped, @what_to_check set to ' + + WHEN @what_to_check NOT IN ('all', 'system') + THEN 'system health skipped, @what_to_check set to ' + @what_to_check - WHEN @what_to_check IN ('all', 'cpu') - THEN 'no cpu issues found between ' + + WHEN @what_to_check IN ('all', 'system') + THEN 'no system health issues found between ' + RTRIM(CONVERT(date, @start_date)) + ' and ' + RTRIM(CONVERT(date, @end_date)) + ' with @warnings_only set to ' + RTRIM(@warnings_only) + '.' - ELSE 'no cpu issues found!' + ELSE 'no system health issues found!' END; END; ELSE BEGIN SELECT - finding = 'cpu task details', - sd.event_time, - sd.state, - sd.maxWorkers, - sd.workersCreated, - sd.workersIdle, - sd.tasksCompletedWithinInterval, - sd.pendingTasks, - sd.oldestPendingTaskWaitingTime, - sd.hasUnresolvableDeadlockOccurred, - sd.hasDeadlockedSchedulersOccurred, - sd.didBlockingOccur - FROM #scheduler_details AS sd + finding = 'overall system health', + h.event_time, + h.state, + h.spinlockBackoffs, + h.sickSpinlockType, + h.sickSpinlockTypeAfterAv, + h.latchWarnings, + h.isAccessViolationOccurred, + h.writeAccessViolationCount, + h.totalDumpRequests, + h.intervalDumpRequests, + h.nonYieldingTasksReported, + h.pageFaults, + h.systemCpuUtilization, + h.sqlCpuUtilization, + h.BadPagesDetected, + h.BadPagesFixed + FROM #health AS h ORDER BY - sd.event_time DESC + h.event_time DESC OPTION(RECOMPILE); END; - END; /*End CPU*/ + END; /*End system*/ - /*Grab memory details*/ - IF @what_to_check IN ('all', 'memory') + /*Parse scheduler monitor data*/ + IF @what_to_check IN ('all', 'system', 'cpu') BEGIN IF @debug = 1 BEGIN - RAISERROR('Parsing memory stuff', 0, 1) WITH NOWAIT; + RAISERROR('Parsing scheduler monitor data', 0, 1) WITH NOWAIT; END; - + SELECT event_time = DATEADD @@ -1693,131 +2848,123 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. GETUTCDATE(), SYSDATETIME() ), - s.sp_server_diagnostics_component_result.value('(//@timestamp)[1]', 'datetime2') - ), - lastNotification = r.c.value('@lastNotification', 'varchar(128)'), - outOfMemoryExceptions = r.c.value('@outOfMemoryExceptions', 'bigint'), - isAnyPoolOutOfMemory = r.c.value('@isAnyPoolOutOfMemory', 'bit'), - processOutOfMemoryPeriod = r.c.value('@processOutOfMemoryPeriod', 'bigint'), - name = r.c.value('(//memoryReport/@name)[1]', 'varchar(128)'), - available_physical_memory_gb = CONVERT(bigint, r.c.value('(//memoryReport/entry[@description[.="Available Physical Memory"]]/@value)[1]', 'bigint') / 1024 / 1024 / 1024), - available_virtual_memory_gb = CONVERT(bigint, r.c.value('(//memoryReport/entry[@description[.="Available Virtual Memory"]]/@value)[1]', 'bigint') / 1024 / 1024 / 1024), - available_paging_file_gb = CONVERT(bigint, r.c.value('(//memoryReport/entry[@description[.="Available Paging File"]]/@value)[1]', 'bigint') / 1024 / 1024 / 1024), - working_set_gb = CONVERT(bigint, r.c.value('(//memoryReport/entry[@description[.="Working Set"]]/@value)[1]', 'bigint') / 1024 / 1024 / 1024), - percent_of_committed_memory_in_ws = r.c.value('(//memoryReport/entry[@description[.="Percent of Committed Memory in WS"]]/@value)[1]', 'bigint'), - page_faults = r.c.value('(//memoryReport/entry[@description[.="Page Faults"]]/@value)[1]', 'bigint'), - system_physical_memory_high = r.c.value('(//memoryReport/entry[@description[.="System physical memory high"]]/@value)[1]', 'bigint'), - system_physical_memory_low = r.c.value('(//memoryReport/entry[@description[.="System physical memory low"]]/@value)[1]', 'bigint'), - process_physical_memory_low = r.c.value('(//memoryReport/entry[@description[.="Process physical memory low"]]/@value)[1]', 'bigint'), - process_virtual_memory_low = r.c.value('(//memoryReport/entry[@description[.="Process virtual memory low"]]/@value)[1]', 'bigint'), - vm_reserved_gb = CONVERT(bigint, r.c.value('(//memoryReport/entry[@description[.="VM Reserved"]]/@value)[1]', 'bigint') / 1024 / 1024), - vm_committed_gb = CONVERT(bigint, r.c.value('(//memoryReport/entry[@description[.="VM Committed"]]/@value)[1]', 'bigint') / 1024 / 1024), - locked_pages_allocated = r.c.value('(//memoryReport/entry[@description[.="Locked Pages Allocated"]]/@value)[1]', 'bigint'), - large_pages_allocated = r.c.value('(//memoryReport/entry[@description[.="Large Pages Allocated"]]/@value)[1]', 'bigint'), - emergency_memory_gb = CONVERT(bigint, r.c.value('(//memoryReport/entry[@description[.="Emergency Memory"]]/@value)[1]', 'bigint') / 1024 / 1024), - emergency_memory_in_use_gb = CONVERT(bigint, r.c.value('(//memoryReport/entry[@description[.="Emergency Memory In Use"]]/@value)[1]', 'bigint') / 1024 / 1024), - target_committed_gb = CONVERT(bigint, r.c.value('(//memoryReport/entry[@description[.="Target Committed"]]/@value)[1]', 'bigint') / 1024 / 1024), - current_committed_gb = CONVERT(bigint, r.c.value('(//memoryReport/entry[@description[.="Current Committed"]]/@value)[1]', 'bigint') / 1024 / 1024), - pages_allocated = r.c.value('(//memoryReport/entry[@description[.="Pages Allocated"]]/@value)[1]', 'bigint'), - pages_reserved = r.c.value('(//memoryReport/entry[@description[.="Pages Reserved"]]/@value)[1]', 'bigint'), - pages_free = r.c.value('(//memoryReport/entry[@description[.="Pages Free"]]/@value)[1]', 'bigint'), - pages_in_use = r.c.value('(//memoryReport/entry[@description[.="Pages In Use"]]/@value)[1]', 'bigint'), - page_alloc_potential = r.c.value('(//memoryReport/entry[@description[.="Page Alloc Potential"]]/@value)[1]', 'bigint'), - numa_growth_phase = r.c.value('(//memoryReport/entry[@description[.="NUMA Growth Phase"]]/@value)[1]', 'bigint'), - last_oom_factor = r.c.value('(//memoryReport/entry[@description[.="Last OOM Factor"]]/@value)[1]', 'bigint'), - last_os_error = r.c.value('(//memoryReport/entry[@description[.="Last OS Error"]]/@value)[1]', 'bigint'), - xml = r.c.query('.') - INTO #memory - FROM #sp_server_diagnostics_component_result AS s - CROSS APPLY s.sp_server_diagnostics_component_result.nodes('/event/data/value/resource') AS r(c) - WHERE (r.c.exist('@lastNotification[.= "RESOURCE_MEMPHYSICAL_LOW"]') = @warnings_only OR @warnings_only = 0) + w.x.value('@timestamp', 'datetime2') + ), + scheduler_id = w.x.value('(data[@name="scheduler_id"]/value)[1]', 'integer'), + cpu_id = w.x.value('(data[@name="cpu_id"]/value)[1]', 'integer'), + status = w.x.value('(data[@name="status"]/text)[1]', 'nvarchar(256)'), + is_online = w.x.value('(data[@name="is_online"]/value)[1]', 'bit'), + is_runnable = w.x.value('(data[@name="is_runnable"]/value)[1]', 'bit'), + is_running = w.x.value('(data[@name="is_running"]/value)[1]', 'bit'), + non_yielding_time_ms = w.x.value('(data[@name="non_yielding_time"]/value)[1]', 'bigint'), + thread_quantum_ms = w.x.value('(data[@name="thread_quantum"]/value)[1]', 'bigint'), + xml = w.x.query('.') + INTO #scheduler_issues + FROM #scheduler_monitor AS sm + CROSS APPLY sm.scheduler_monitor.nodes('//event') AS w(x) + WHERE (w.x.exist('(data[@name="status"]/text[.= "WARNING"])') = @warnings_only OR @warnings_only = 0) OPTION(RECOMPILE); - + IF @debug = 1 BEGIN SELECT TOP (100) - table_name = '#memory, top 100 rows', + table_name = '#scheduler_issues, top 100 rows', x.* - FROM #memory AS x + FROM #scheduler_issues AS x ORDER BY x.event_time DESC; END; - + IF NOT EXISTS ( SELECT 1/0 - FROM #memory AS m + FROM #scheduler_issues AS si ) BEGIN SELECT finding = CASE - WHEN @what_to_check NOT IN ('all', 'memory') - THEN 'memory skipped, @what_to_check set to ' + + WHEN @what_to_check NOT IN ('all', 'system', 'cpu') + THEN 'scheduler monitoring skipped, @what_to_check set to ' + @what_to_check - WHEN @what_to_check IN ('all', 'memory') - THEN 'no memory issues found between ' + + WHEN @what_to_check IN ('all', 'system', 'cpu') + THEN 'no scheduler issues found between ' + RTRIM(CONVERT(date, @start_date)) + ' and ' + RTRIM(CONVERT(date, @end_date)) + ' with @warnings_only set to ' + RTRIM(@warnings_only) + '.' - ELSE 'no memory issues found!' + ELSE 'no scheduler issues found!' END; - END; + END ELSE BEGIN SELECT - finding = 'memory conditions', - m.event_time, - m.lastNotification, - m.outOfMemoryExceptions, - m.isAnyPoolOutOfMemory, - m.processOutOfMemoryPeriod, - m.name, - m.available_physical_memory_gb, - m.available_virtual_memory_gb, - m.available_paging_file_gb, - m.working_set_gb, - m.percent_of_committed_memory_in_ws, - m.page_faults, - m.system_physical_memory_high, - m.system_physical_memory_low, - m.process_physical_memory_low, - m.process_virtual_memory_low, - m.vm_reserved_gb, - m.vm_committed_gb, - m.locked_pages_allocated, - m.large_pages_allocated, - m.emergency_memory_gb, - m.emergency_memory_in_use_gb, - m.target_committed_gb, - m.current_committed_gb, - m.pages_allocated, - m.pages_reserved, - m.pages_free, - m.pages_in_use, - m.page_alloc_potential, - m.numa_growth_phase, - m.last_oom_factor, - m.last_os_error - FROM #memory AS m + finding = 'scheduler monitor issues', + si.event_time, + si.scheduler_id, + si.cpu_id, + si.status, + si.is_online, + si.is_runnable, + si.is_running, + non_yielding_time_ms = + REPLACE + ( + CONVERT + ( + nvarchar(30), + CONVERT + ( + money, + si.non_yielding_time_ms + ), + 1 + ), + N'.00', + N'' + ), + thread_quantum_ms = + REPLACE + ( + CONVERT + ( + nvarchar(30), + CONVERT + ( + money, + si.thread_quantum_ms + ), + 1 + ), + N'.00', + N'' + ) + FROM #scheduler_issues AS si ORDER BY - m.event_time DESC + si.event_time DESC OPTION(RECOMPILE); END; - END; /*End memory*/ + END; /*End scheduler monitor analysis*/ - /*Grab health stuff*/ + /*Parse error_reported data*/ IF @what_to_check IN ('all', 'system') BEGIN IF @debug = 1 BEGIN - RAISERROR('Parsing system stuff', 0, 1) WITH NOWAIT; + RAISERROR('Parsing error_reported data', 0, 1) WITH NOWAIT; END; + INSERT + #ignore_errors + ( + error_number + ) + VALUES + (17830); + SELECT event_time = DATEADD @@ -1831,89 +2978,83 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ), w.x.value('@timestamp', 'datetime2') ), - state = w.x.value('(data[@name="state"]/text/text())[1]', 'nvarchar(256)'), - spinlockBackoffs = w.x.value('(/event/data[@name="data"]/value/system/@spinlockBackoffs)[1]', 'bigint'), - sickSpinlockType = w.x.value('(/event/data[@name="data"]/value/system/@sickSpinlockType)[1]', 'nvarchar(256)'), - sickSpinlockTypeAfterAv = w.x.value('(/event/data[@name="data"]/value/system/@sickSpinlockTypeAfterAv)[1]', 'nvarchar(256)'), - latchWarnings = w.x.value('(/event/data[@name="data"]/value/system/@latchWarnings)[1]', 'bigint'), - isAccessViolationOccurred = w.x.value('(/event/data[@name="data"]/value/system/@isAccessViolationOccurred)[1]', 'bigint'), - writeAccessViolationCount = w.x.value('(/event/data[@name="data"]/value/system/@writeAccessViolationCount)[1]', 'bigint'), - totalDumpRequests = w.x.value('(/event/data[@name="data"]/value/system/@totalDumpRequests)[1]', 'bigint'), - intervalDumpRequests = w.x.value('(/event/data[@name="data"]/value/system/@intervalDumpRequests)[1]', 'bigint'), - nonYieldingTasksReported = w.x.value('(/event/data[@name="data"]/value/system/@nonYieldingTasksReported)[1]', 'bigint'), - pageFaults = w.x.value('(/event/data[@name="data"]/value/system/@pageFaults)[1]', 'bigint'), - systemCpuUtilization = w.x.value('(/event/data[@name="data"]/value/system/@systemCpuUtilization)[1]', 'bigint'), - sqlCpuUtilization = w.x.value('(/event/data[@name="data"]/value/system/@sqlCpuUtilization)[1]', 'bigint'), - BadPagesDetected = w.x.value('(/event/data[@name="data"]/value/system/@BadPagesDetected)[1]', 'bigint'), - BadPagesFixed = w.x.value('(/event/data[@name="data"]/value/system/@BadPagesFixed)[1]', 'bigint'), + error_number = w.x.value('(data[@name="error_number"]/value)[1]', 'integer'), + severity = w.x.value('(data[@name="severity"]/value)[1]', 'integer'), + state = w.x.value('(data[@name="state"]/value)[1]', 'integer'), + message = w.x.value('(data[@name="message"]/value)[1]', 'nvarchar(max)'), + database_name = DB_NAME(w.x.value('(data[@name="database_id"]/value)[1]', 'integer')), + database_id = w.x.value('(data[@name="database_id"]/value)[1]', 'integer'), xml = w.x.query('.') - INTO #health - FROM #sp_server_diagnostics_component_result AS wi - CROSS APPLY wi.sp_server_diagnostics_component_result.nodes('//event') AS w(x) - WHERE w.x.exist('(data[@name="component"]/text[.= "SYSTEM"])') = 1 - AND (w.x.exist('(data[@name="state"]/text[.= "WARNING"])') = @warnings_only OR @warnings_only = 0) + INTO #error_info + FROM #error_reported AS er + CROSS APPLY er.error_reported.nodes('//event') AS w(x) + WHERE w.x.exist('(data[@name="severity"]/value)[. >= 16]') = 1 + AND (@warnings_only = 0 OR w.x.exist('(data[@name="severity"]/value)[. >= 19]') = 1) + AND NOT EXISTS + ( + SELECT + 1/0 + FROM #ignore_errors AS ie + WHERE w.x.value('(data[@name="error_number"]/value)[1]', 'integer') = ie.error_number + ) OPTION(RECOMPILE); - + IF @debug = 1 BEGIN SELECT TOP (100) - table_name = '#health, top 100 rows', + table_name = '#error_info, top 100 rows', x.* - FROM #health AS x + FROM #error_info AS x ORDER BY x.event_time DESC; END; - + IF NOT EXISTS ( SELECT 1/0 - FROM #health AS h + FROM #error_info AS ei ) BEGIN SELECT finding = CASE WHEN @what_to_check NOT IN ('all', 'system') - THEN 'system health skipped, @what_to_check set to ' + + THEN 'error reporting skipped, @what_to_check set to ' + @what_to_check WHEN @what_to_check IN ('all', 'system') - THEN 'no system health issues found between ' + + THEN 'no severe errors found between ' + RTRIM(CONVERT(date, @start_date)) + ' and ' + RTRIM(CONVERT(date, @end_date)) + ' with @warnings_only set to ' + RTRIM(@warnings_only) + '.' - ELSE 'no system health issues found!' - END; - END; + ELSE 'no severe errors found!' + END + UNION ALL + SELECT + 'Error Number Ignored: ' + CONVERT(nvarchar(100), ie.error_number) + FROM #ignore_errors AS ie; + END ELSE BEGIN SELECT - finding = 'overall system health', - h.event_time, - h.state, - h.spinlockBackoffs, - h.sickSpinlockType, - h.sickSpinlockTypeAfterAv, - h.latchWarnings, - h.isAccessViolationOccurred, - h.writeAccessViolationCount, - h.totalDumpRequests, - h.intervalDumpRequests, - h.nonYieldingTasksReported, - h.pageFaults, - h.systemCpuUtilization, - h.sqlCpuUtilization, - h.BadPagesDetected, - h.BadPagesFixed - FROM #health AS h + finding = 'severe errors reported', + ei.event_time, + ei.error_number, + ei.severity, + ei.state, + ei.message, + ei.database_name, + ei.database_id + FROM #error_info AS ei ORDER BY - h.event_time DESC + ei.event_time DESC, + ei.severity DESC OPTION(RECOMPILE); END; - END; /*End system*/ + END; /*End error_reported analysis*/ /*Grab useless stuff*/ @@ -2042,16 +3183,16 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SELECT bx.event_time, currentdbname = bd.value('(process/@currentdbname)[1]', 'nvarchar(128)'), - spid = bd.value('(process/@spid)[1]', 'int'), - ecid = bd.value('(process/@ecid)[1]', 'int'), + spid = bd.value('(process/@spid)[1]', 'integer'), + ecid = bd.value('(process/@ecid)[1]', 'integer'), query_text_pre = bd.value('(process/inputbuf/text())[1]', 'nvarchar(MAX)'), wait_time = bd.value('(process/@waittime)[1]', 'bigint'), lastbatchstarted = bd.value('(process/@lastbatchstarted)[1]', 'datetime2'), lastbatchcompleted = bd.value('(process/@lastbatchcompleted)[1]', 'datetime2'), wait_resource = bd.value('(process/@waitresource)[1]', 'nvarchar(100)'), status = bd.value('(process/@status)[1]', 'nvarchar(10)'), - priority = bd.value('(process/@priority)[1]', 'int'), - transaction_count = bd.value('(process/@trancount)[1]', 'int'), + priority = bd.value('(process/@priority)[1]', 'integer'), + transaction_count = bd.value('(process/@trancount)[1]', 'integer'), client_app = bd.value('(process/@clientapp)[1]', 'nvarchar(256)'), host_name = bd.value('(process/@hostname)[1]', 'nvarchar(256)'), login_name = bd.value('(process/@loginname)[1]', 'nvarchar(256)'), @@ -2103,16 +3244,16 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SELECT bx.event_time, currentdbname = bg.value('(process/@currentdbname)[1]', 'nvarchar(128)'), - spid = bg.value('(process/@spid)[1]', 'int'), - ecid = bg.value('(process/@ecid)[1]', 'int'), + spid = bg.value('(process/@spid)[1]', 'integer'), + ecid = bg.value('(process/@ecid)[1]', 'integer'), query_text_pre = bg.value('(process/inputbuf/text())[1]', 'nvarchar(MAX)'), wait_time = bg.value('(process/@waittime)[1]', 'bigint'), last_transaction_started = bg.value('(process/@lastbatchstarted)[1]', 'datetime2'), last_transaction_completed = bg.value('(process/@lastbatchcompleted)[1]', 'datetime2'), wait_resource = bg.value('(process/@waitresource)[1]', 'nvarchar(100)'), status = bg.value('(process/@status)[1]', 'nvarchar(10)'), - priority = bg.value('(process/@priority)[1]', 'int'), - transaction_count = bg.value('(process/@trancount)[1]', 'int'), + priority = bg.value('(process/@priority)[1]', 'integer'), + transaction_count = bg.value('(process/@trancount)[1]', 'integer'), client_app = bg.value('(process/@clientapp)[1]', 'nvarchar(256)'), host_name = bg.value('(process/@hostname)[1]', 'nvarchar(256)'), login_name = bg.value('(process/@loginname)[1]', 'nvarchar(256)'), @@ -2307,39 +3448,65 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ORDER BY x.event_time DESC; END; - - SELECT - finding = 'blocked process report', - b.event_time, - b.currentdbname, - b.activity, - b.spid, - b.ecid, - b.query_text, - b.wait_time_ms, - b.status, - b.isolation_level, - b.transaction_count, - b.last_transaction_started, - b.last_transaction_completed, - b.client_option_1, - b.client_option_2, - b.wait_resource, - b.priority, - b.log_used, - b.client_app, - b.host_name, - b.login_name, - b.blocked_process_report - FROM #blocks AS b - ORDER BY - b.event_time DESC, - CASE - WHEN b.activity = 'blocking' - THEN -1 - ELSE +1 - END - OPTION(RECOMPILE); + + IF EXISTS + ( + SELECT + 1/0 + FROM #blocks AS b + ) + BEGIN + SELECT + finding = 'blocked process report', + b.event_time, + b.currentdbname, + b.activity, + b.spid, + b.ecid, + b.query_text, + b.wait_time_ms, + b.status, + b.isolation_level, + b.transaction_count, + b.last_transaction_started, + b.last_transaction_completed, + b.client_option_1, + b.client_option_2, + b.wait_resource, + b.priority, + b.log_used, + b.client_app, + b.host_name, + b.login_name, + b.blocked_process_report + FROM #blocks AS b + ORDER BY + b.event_time DESC, + CASE + WHEN b.activity = 'blocking' + THEN -1 + ELSE +1 + END + OPTION(RECOMPILE); + END + ELSE + BEGIN + SELECT + finding = CASE + WHEN @what_to_check NOT IN ('all', 'locking') + THEN 'blocking skipped, @what_to_check set to ' + @what_to_check + WHEN @skip_locks = 1 + THEN 'blocking skipped, @skip_locks set to 1' + WHEN @what_to_check IN ('all', 'locking') + THEN 'no blocking found between ' + + RTRIM(CONVERT(date, @start_date)) + + ' and ' + + RTRIM(CONVERT(date, @end_date)) + + ' with @warnings_only set to ' + + RTRIM(@warnings_only) + ELSE 'no blocking found!' + END; + END; /*Grab available plans from the cache*/ IF @debug = 1 @@ -2361,9 +3528,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. sql_handle = CONVERT(varbinary(64), n.c.value('@sqlhandle', 'varchar(130)'), 1), stmtstart = - ISNULL(n.c.value('@stmtstart', 'int'), 0), + ISNULL(n.c.value('@stmtstart', 'integer'), 0), stmtend = - ISNULL(n.c.value('@stmtend', 'int'), -1) + ISNULL(n.c.value('@stmtend', 'integer'), -1) FROM #blocks AS b CROSS APPLY b.blocked_process_report.nodes('/blocked-process/process/executionStack/frame[not(@sqlhandle = "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")]') AS n(c) WHERE (b.currentdbname = @database_name @@ -2380,9 +3547,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. sql_handle = CONVERT(varbinary(64), n.c.value('@sqlhandle', 'varchar(130)'), 1), stmtstart = - ISNULL(n.c.value('@stmtstart', 'int'), 0), + ISNULL(n.c.value('@stmtstart', 'integer'), 0), stmtend = - ISNULL(n.c.value('@stmtend', 'int'), -1) + ISNULL(n.c.value('@stmtend', 'integer'), -1) FROM #blocks AS b CROSS APPLY b.blocked_process_report.nodes('/blocking-process/process/executionStack/frame[not(@sqlhandle = "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")]') AS n(c) WHERE (b.currentdbname = @database_name @@ -2567,95 +3734,119 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. RAISERROR('Returning deadlocks', 0, 1) WITH NOWAIT; END; - SELECT - finding = 'xml deadlock report', - dp.event_date, - is_victim = - CASE - WHEN dp.id = dp.victim_id - THEN 1 - ELSE 0 - END, - dp.database_name, - dp.current_database_name, - query_text = - CASE - WHEN dp.query_text - LIKE CONVERT(nvarchar(1), 0x0a00, 0) + N'Proc |[Database Id = %' ESCAPE N'|' - THEN - ( - SELECT - [processing-instruction(query)] = - OBJECT_SCHEMA_NAME - ( - SUBSTRING - ( - dp.query_text, - CHARINDEX(N'Object Id = ', dp.query_text) + 12, - LEN(dp.query_text) - (CHARINDEX(N'Object Id = ', dp.query_text) + 12) - ) - , - SUBSTRING - ( - dp.query_text, - CHARINDEX(N'Database Id = ', dp.query_text) + 14, - CHARINDEX(N'Object Id', dp.query_text) - (CHARINDEX(N'Database Id = ', dp.query_text) + 14) - ) - ) + - N'.' + - OBJECT_NAME - ( - SUBSTRING - ( - dp.query_text, - CHARINDEX(N'Object Id = ', dp.query_text) + 12, - LEN(dp.query_text) - (CHARINDEX(N'Object Id = ', dp.query_text) + 12) - ) - , - SUBSTRING - ( - dp.query_text, - CHARINDEX(N'Database Id = ', dp.query_text) + 14, - CHARINDEX(N'Object Id', dp.query_text) - (CHARINDEX(N'Database Id = ', dp.query_text) + 14) - ) - ) - FOR XML - PATH(N''), - TYPE - ) - ELSE - ( - SELECT - [processing-instruction(query)] = - dp.query_text - FOR XML - PATH(N''), - TYPE - ) - END, - dp.deadlock_resources, - dp.isolation_level, - dp.lock_mode, - dp.status, - dp.wait_time, - dp.log_used, - dp.transaction_name, - dp.transaction_count, - dp.client_option_1, - dp.client_option_2, - dp.last_tran_started, - dp.last_batch_started, - dp.last_batch_completed, - dp.client_app, - dp.host_name, - dp.login_name, - dp.priority, - dp.deadlock_graph - FROM #deadlocks_parsed AS dp - ORDER BY - dp.event_date, - is_victim - OPTION(RECOMPILE); + IF EXISTS + ( + SELECT + 1/0 + FROM #deadlocks_parsed AS dp + ) + BEGIN + SELECT + finding = 'xml deadlock report', + dp.event_date, + is_victim = + CASE + WHEN dp.id = dp.victim_id + THEN 1 + ELSE 0 + END, + dp.database_name, + dp.current_database_name, + query_text = + CASE + WHEN dp.query_text + LIKE CONVERT(nvarchar(1), 0x0a00, 0) + N'Proc |[Database Id = %' ESCAPE N'|' + THEN + ( + SELECT + [processing-instruction(query)] = + OBJECT_SCHEMA_NAME + ( + SUBSTRING + ( + dp.query_text, + CHARINDEX(N'Object Id = ', dp.query_text) + 12, + LEN(dp.query_text) - (CHARINDEX(N'Object Id = ', dp.query_text) + 12) + ) + , + SUBSTRING + ( + dp.query_text, + CHARINDEX(N'Database Id = ', dp.query_text) + 14, + CHARINDEX(N'Object Id', dp.query_text) - (CHARINDEX(N'Database Id = ', dp.query_text) + 14) + ) + ) + + N'.' + + OBJECT_NAME + ( + SUBSTRING + ( + dp.query_text, + CHARINDEX(N'Object Id = ', dp.query_text) + 12, + LEN(dp.query_text) - (CHARINDEX(N'Object Id = ', dp.query_text) + 12) + ) + , + SUBSTRING + ( + dp.query_text, + CHARINDEX(N'Database Id = ', dp.query_text) + 14, + CHARINDEX(N'Object Id', dp.query_text) - (CHARINDEX(N'Database Id = ', dp.query_text) + 14) + ) + ) + FOR XML + PATH(N''), + TYPE + ) + ELSE + ( + SELECT + [processing-instruction(query)] = + dp.query_text + FOR XML + PATH(N''), + TYPE + ) + END, + dp.deadlock_resources, + dp.isolation_level, + dp.lock_mode, + dp.status, + dp.wait_time, + dp.log_used, + dp.transaction_name, + dp.transaction_count, + dp.client_option_1, + dp.client_option_2, + dp.last_tran_started, + dp.last_batch_started, + dp.last_batch_completed, + dp.client_app, + dp.host_name, + dp.login_name, + dp.priority, + dp.deadlock_graph + FROM #deadlocks_parsed AS dp + ORDER BY + dp.event_date, + is_victim + OPTION(RECOMPILE); + END + ELSE + BEGIN + SELECT + finding = CASE + WHEN @what_to_check NOT IN ('all', 'locking') + THEN 'deadlocks skipped, @what_to_check set to ' + @what_to_check + WHEN @skip_locks = 1 + THEN 'deadlocks skipped, @skip_locks set to 1' + WHEN @what_to_check IN ('all', 'locking') + THEN 'no deadlocks found between ' + + RTRIM(CONVERT(date, @start_date)) + + ' and ' + + RTRIM(CONVERT(date, @end_date)) + ELSE 'no deadlocks found!' + END; + END; IF @debug = 1 BEGIN @@ -2859,21 +4050,67 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ap.avg_worker_time_ms DESC OPTION(RECOMPILE); - SELECT - aap.* - FROM #all_avalable_plans AS aap - WHERE aap.finding = 'available plans for blocking' - ORDER BY - aap.avg_worker_time_ms DESC - OPTION(RECOMPILE); - - SELECT - aap.* - FROM #all_avalable_plans AS aap - WHERE aap.finding = 'available plans for deadlocks' - ORDER BY - aap.avg_worker_time_ms DESC - OPTION(RECOMPILE); + IF EXISTS + ( + SELECT + 1/0 + FROM #all_avalable_plans AS ap + WHERE ap.finding = 'available plans for blocking' + ) + BEGIN + SELECT + aap.* + FROM #all_avalable_plans AS aap + WHERE aap.finding = 'available plans for blocking' + ORDER BY + aap.avg_worker_time_ms DESC + OPTION(RECOMPILE); + END + ELSE + BEGIN + -- Only show this message if we found blocking but no plans + IF EXISTS + ( + SELECT + 1/0 + FROM #blocks AS b + ) + BEGIN + SELECT + finding = 'no cached plans found for blocking queries'; + END; + END; + + IF EXISTS + ( + SELECT + 1/0 + FROM #all_avalable_plans AS ap + WHERE ap.finding = 'available plans for deadlocks' + ) + BEGIN + SELECT + aap.* + FROM #all_avalable_plans AS aap + WHERE aap.finding = 'available plans for deadlocks' + ORDER BY + aap.avg_worker_time_ms DESC + OPTION(RECOMPILE); + END + ELSE + BEGIN + -- Only show this message if we found deadlocks but no plans + IF EXISTS + ( + SELECT + 1/0 + FROM #deadlocks_parsed AS dp + ) + BEGIN + SELECT + finding = 'no cached plans found for deadlock queries'; + END; + END; END; /*End locks*/ END; /*Final End*/ GO From a7ef4ddd038a2d26fe54213418fa3ed2ef131637 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Fri, 7 Mar 2025 13:30:30 -0500 Subject: [PATCH 010/246] Update sp_HealthParser.sql Getting closer! --- sp_HealthParser/sp_HealthParser.sql | 221 +++++++++++++++++++++++++--- 1 file changed, 199 insertions(+), 22 deletions(-) diff --git a/sp_HealthParser/sp_HealthParser.sql b/sp_HealthParser/sp_HealthParser.sql index 8958f576..b822afa4 100644 --- a/sp_HealthParser/sp_HealthParser.sql +++ b/sp_HealthParser/sp_HealthParser.sql @@ -192,7 +192,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. WHEN CONVERT ( - int, + integer, SERVERPROPERTY('EngineEdition') ) = 5 THEN 1 @@ -204,7 +204,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. WHEN CONVERT ( - int, + integer, SERVERPROPERTY('EngineEdition') ) = 8 THEN 1 @@ -212,8 +212,12 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END, @mi_msg nchar(1), @dbid integer = - DB_ID(@database_name); - + DB_ID(@database_name), + @timestamp_utc_mode tinyint, + @sql_template nvarchar(MAX) = N'', + @time_filter nvarchar(MAX) = N'', + @cross_apply nvarchar(MAX) = N''; + IF @azure = 1 BEGIN RAISERROR('This won''t work in Azure because it''s horrible', 11, 1) WITH NOWAIT; @@ -296,8 +300,93 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. @azure_msg = CONVERT(nchar(1), @azure), @mi_msg = - CONVERT(nchar(1), @mi); + CONVERT(nchar(1), @mi), + @timestamp_utc_mode = + CASE + WHEN EXISTS + ( + SELECT + 1/0 + FROM sys.all_columns AS ac + WHERE ac.object_id = OBJECT_ID(N'sys.fn_xe_file_target_read_file') + AND ac.name = N'timestamp_utc' + ) + THEN 1 + + CASE + WHEN + PARSENAME + ( + CONVERT + ( + sysname, + SERVERPROPERTY('PRODUCTVERSION') + ), + 4 + ) > 17 + THEN 1 + ELSE 0 + END + + CASE + WHEN @mi = 1 + THEN 1 + ELSE 0 + END + ELSE 0 + END, + @sql_template += N' +SELECT + {object_name} = + ISNULL + ( + xml.{object_name}, + CONVERT(xml, N''event'') + ) +FROM +( + SELECT + {object_name} = + TRY_CAST(fx.event_data AS xml) + FROM sys.fn_xe_file_target_read_file(N''system_health*.xel'', NULL, NULL, NULL) AS fx + WHERE fx.object_name = N''{object_name}'' {time_filter} +) AS xml +{cross_apply} +OPTION(RECOMPILE);'; + + IF @timestamp_utc_mode = 0 + BEGIN + -- Pre-2017 handling + SET @time_filter = N''; + SET @cross_apply = N'CROSS APPLY xml.{object_name}.nodes(''/event'') AS e(x) +CROSS APPLY (SELECT x.value( ''(@timestamp)[1]'', ''datetimeoffset'' )) ca ([utc_timestamp]) +WHERE ca.utc_timestamp >= @start_date +AND ca.utc_timestamp < @end_date'; + END + ELSE + BEGIN + -- 2017+ handling + SET @cross_apply = N'CROSS APPLY xml.{object_name}.nodes(''/event'') AS e(x)'; + + IF @timestamp_utc_mode = 1 + SET @time_filter = N' + AND CONVERT(datetimeoffset(7), fx.timestamp_utc) BETWEEN @start_date AND @end_date'; + ELSE + SET @time_filter = ' + AND fx.timestamp_utc BETWEEN @start_date AND @end_date'; + END + SET @sql_template = + REPLACE + ( + REPLACE + ( + @sql_template, + '{time_filter}', + @time_filter + ), + '{cross_apply}', + @cross_apply + ); + /*If any parameters that expect non-NULL default values get passed in with NULLs, fix them*/ SELECT @what_to_check = LOWER(ISNULL(@what_to_check, 'all')), @@ -309,20 +398,15 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /*Validate what to check*/ IF @what_to_check NOT IN - ( - 'all', - 'waits', - 'disk', - 'cpu', - 'memory', - 'system', - 'blocking', - 'blocks', - 'deadlock', - 'deadlocks', - 'locking', - 'locks' - ) + ( + 'all', + 'cpu', + 'disk', + 'locking', + 'memory', + 'system', + 'waits' + ) BEGIN SELECT @what_to_check = @@ -341,6 +425,63 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. RAISERROR('Creating temp tables', 0, 1) WITH NOWAIT; END; + DECLARE + @collection_areas TABLE + ( + area_name varchar(20) NOT NULL, + object_name sysname NOT NULL, + temp_table sysname NOT NULL, + should_collect bit NOT NULL DEFAULT 0 + ); + + INSERT INTO + @collection_areas + ( + area_name, + object_name, + temp_table, + should_collect + ) + SELECT + v.area_name, + v.object_name, + v.temp_table, + should_collect = + CASE + WHEN @what_to_check = 'all' + THEN + CASE + WHEN area_name = 'locking' + AND @skip_locks = 1 + THEN 0 + ELSE 1 + END + WHEN @what_to_check = area_name + THEN 1 + ELSE 0 + END + FROM + ( + VALUES + ('cpu', 'scheduler_monitor_system_health', '#scheduler_monitor'), + ('disk', 'sp_server_diagnostics_component_result', '#sp_server_diagnostics_component_result'), + ('locking', 'xml_deadlock_report', '#xml_deadlock_report'), + ('locking', 'human_events_xml', '#blocking_xml'), + ('waits', 'wait_info', '#wait_info'), + ('system', 'sp_server_diagnostics_component_result', '#sp_server_diagnostics_component_result'), + ('system', 'error_reported', '#error_reported'), + ('memory', 'memory_broker_ring_buffer_recorded', '#memory_broker'), + ('memory', 'memory_node_oom_ring_buffer_recorded', '#memory_node_oom') + ) AS v(area_name, object_name, temp_table); + + IF @debug = 1 + BEGIN + SELECT + table_name = '@collection_areas', + ca.* + FROM @collection_areas AS ca + END; + CREATE TABLE #ignore_waits ( @@ -371,6 +512,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. xml_deadlock_report xml NOT NULL ); + CREATE TABLE + #blocking_xml + ( + event_time datetime2 NOT NULL, + human_events_xml xml NOT NULL + ); + CREATE TABLE #x ( @@ -446,8 +594,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. N'SQLTRACE_BUFFER_FLUSH', N'SQLTRACE_INCREMENTAL_FLUSH_SLEEP', N'SQLTRACE_WAIT_ENTRIES', N'UCS_SESSION_REGISTRATION', N'VDI_CLIENT_OTHER', N'WAIT_FOR_RESULTS', N'WAIT_XTP_CKPT_CLOSE', N'WAIT_XTP_HOST_WAIT', N'WAIT_XTP_OFFLINE_CKPT_NEW_LOG', N'WAIT_XTP_RECOVERY', N'WAITFOR', N'WAITFOR_TASKSHUTDOWN', - N'XE_DISPATCHER_JOIN', N'XE_DISPATCHER_WAIT', N'XE_FILE_TARGET_TVF', N'XE_LIVE_TARGET_TVF', - N'XE_TIMER_EVENT' + N'XE_DISPATCHER_JOIN', N'XE_DISPATCHER_WAIT', N'XE_FILE_TARGET_TVF', N'XE_LIVE_TARGET_TVF', N'XE_TIMER_EVENT' ) OPTION(RECOMPILE); END; /*End waits ignore*/ @@ -465,6 +612,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. The column timestamp_utc is 2017+ only, but terribly broken: https://dba.stackexchange.com/q/323147/32281 https://feedback.azure.com/d365community/idea/5f8e52d6-f3d2-ec11-a81b-6045bd7ac9f9 + + It is fixed in Azure Managed Instance, and will be fixed in the next major + SQL Server release, so we have to handle things a little bit differently */ IF EXISTS ( @@ -1508,6 +1658,26 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. table_name = '#xml_deadlock_report, top 100 rows', x.* FROM #xml_deadlock_report AS x; + + SELECT TOP (100) + table_name = '#scheduler_monitor, top 100 rows', + x.* + FROM #scheduler_monitor AS x; + + SELECT TOP (100) + table_name = '#error_reported, top 100 rows', + x.* + FROM #error_reported AS x; + + SELECT TOP (100) + table_name = '#memory_broker, top 100 rows', + x.* + FROM #memory_broker AS x; + + SELECT TOP (100) + table_name = '#memory_node_oom, top 100 rows', + x.* + FROM #memory_node_oom AS x; END; /*Parse out the wait_info data*/ @@ -3142,6 +3312,14 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. RAISERROR('Parsing locking stuff', 0, 1) WITH NOWAIT; END; + INSERT + #blocking_xml + WITH + (TABLOCK) + ( + event_time, + human_events_xml + ) SELECT event_time = DATEADD @@ -3156,7 +3334,6 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. w.x.value('(//@timestamp)[1]', 'datetime2') ), human_events_xml = w.x.query('//data[@name="data"]/value/queryProcessing/blockingTasks/blocked-process-report') - INTO #blocking_xml FROM #sp_server_diagnostics_component_result AS wi CROSS APPLY wi.sp_server_diagnostics_component_result.nodes('//event') AS w(x) WHERE w.x.exist('(data[@name="component"]/text[.= "QUERY_PROCESSING"])') = 1 From a63cebdb5c74d1eb082d181b186d8ab5e409a680 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Fri, 7 Mar 2025 18:10:46 -0500 Subject: [PATCH 011/246] Update sp_HealthParser.sql just get in there --- sp_HealthParser/sp_HealthParser.sql | 984 +++------------------------- 1 file changed, 103 insertions(+), 881 deletions(-) diff --git a/sp_HealthParser/sp_HealthParser.sql b/sp_HealthParser/sp_HealthParser.sql index b822afa4..c2bef53c 100644 --- a/sp_HealthParser/sp_HealthParser.sql +++ b/sp_HealthParser/sp_HealthParser.sql @@ -216,7 +216,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. @timestamp_utc_mode tinyint, @sql_template nvarchar(MAX) = N'', @time_filter nvarchar(MAX) = N'', - @cross_apply nvarchar(MAX) = N''; + @cross_apply nvarchar(MAX) = N'', + @collection_cursor CURSOR, + @area_name varchar(20), + @object_name sysname, + @temp_table sysname, + @insert_list sysname, + @collection_sql nvarchar(MAX); IF @azure = 1 BEGIN @@ -334,6 +340,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ELSE 0 END, @sql_template += N' +INSERT INTO + {temp_table} +WITH + (TABLOCK) +( + {insert_list} +) SELECT {object_name} = ISNULL @@ -350,7 +363,8 @@ FROM WHERE fx.object_name = N''{object_name}'' {time_filter} ) AS xml {cross_apply} -OPTION(RECOMPILE);'; +OPTION(RECOMPILE); +'; IF @timestamp_utc_mode = 0 BEGIN @@ -431,6 +445,7 @@ AND ca.utc_timestamp < @end_date'; area_name varchar(20) NOT NULL, object_name sysname NOT NULL, temp_table sysname NOT NULL, + insert_list sysname NOT NULL, should_collect bit NOT NULL DEFAULT 0 ); @@ -440,12 +455,14 @@ AND ca.utc_timestamp < @end_date'; area_name, object_name, temp_table, + insert_list, should_collect ) SELECT v.area_name, v.object_name, v.temp_table, + v.insert_list, should_collect = CASE WHEN @what_to_check = 'all' @@ -463,16 +480,16 @@ AND ca.utc_timestamp < @end_date'; FROM ( VALUES - ('cpu', 'scheduler_monitor_system_health', '#scheduler_monitor'), - ('disk', 'sp_server_diagnostics_component_result', '#sp_server_diagnostics_component_result'), - ('locking', 'xml_deadlock_report', '#xml_deadlock_report'), - ('locking', 'human_events_xml', '#blocking_xml'), - ('waits', 'wait_info', '#wait_info'), - ('system', 'sp_server_diagnostics_component_result', '#sp_server_diagnostics_component_result'), - ('system', 'error_reported', '#error_reported'), - ('memory', 'memory_broker_ring_buffer_recorded', '#memory_broker'), - ('memory', 'memory_node_oom_ring_buffer_recorded', '#memory_node_oom') - ) AS v(area_name, object_name, temp_table); + ('cpu', 'scheduler_monitor_system_health', '#scheduler_monitor', 'scheduler_monitor'), + ('disk', 'sp_server_diagnostics_component_result', '#sp_server_diagnostics_component_result', 'sp_server_diagnostics_component_result'), + ('locking', 'xml_deadlock_report', '#xml_deadlock_report', 'xml_deadlock_report'), + ('locking', 'human_events_xml', '#sp_server_diagnostics_component_result', 'human_events_xml'), + ('waits', 'wait_info', '#wait_info', 'wait_info'), + ('system', 'sp_server_diagnostics_component_result', '#sp_server_diagnostics_component_result', 'sp_server_diagnostics_component_result'), + ('system', 'error_reported', '#error_reported', 'error_reported'), + ('memory', 'memory_broker_ring_buffer_recorded', '#memory_broker', 'memory_broker'), + ('memory', 'memory_node_oom_ring_buffer_recorded', '#memory_node_oom', 'memory_node_oom') + ) AS v(area_name, object_name, temp_table, insert_list); IF @debug = 1 BEGIN @@ -608,896 +625,101 @@ AND ca.utc_timestamp < @end_date'; OPTION(RECOMPILE); END; - /* - The column timestamp_utc is 2017+ only, but terribly broken: - https://dba.stackexchange.com/q/323147/32281 - https://feedback.azure.com/d365community/idea/5f8e52d6-f3d2-ec11-a81b-6045bd7ac9f9 - - It is fixed in Azure Managed Instance, and will be fixed in the next major - SQL Server release, so we have to handle things a little bit differently - */ - IF EXISTS - ( - SELECT - 1/0 - FROM sys.all_columns AS ac - WHERE ac.object_id = OBJECT_ID(N'sys.fn_xe_file_target_read_file') - AND ac.name = N'timestamp_utc' - ) - AND @mi = 0 + -- First, ensure we're working with the correct collection areas + IF @debug = 1 BEGIN - /*Grab data from the wait info component*/ - IF @what_to_check IN ('all', 'waits') - BEGIN - IF @debug = 1 - BEGIN - RAISERROR('Checking waits for not Managed Instance, 2017+', 0, 1) WITH NOWAIT; - END; - - SELECT - @sql = N' - SELECT - wait_info = - ISNULL - ( - xml.wait_info, - CONVERT(xml, N''event'') - ) - FROM - ( - SELECT - wait_info = - TRY_CAST(fx.event_data AS xml) - FROM sys.fn_xe_file_target_read_file(N''system_health*.xel'', NULL, NULL, NULL) AS fx - WHERE fx.object_name = N''wait_info'' - AND CONVERT(datetimeoffset(7), fx.timestamp_utc) BETWEEN @start_date AND @end_date - ) AS xml - CROSS APPLY xml.wait_info.nodes(''/event'') AS e(x) - OPTION(RECOMPILE);'; - - IF @debug = 1 - BEGIN - PRINT @sql; - RAISERROR('Inserting #wait_info', 0, 1) WITH NOWAIT; - SET STATISTICS XML ON; - END; - - INSERT INTO - #wait_info - WITH - (TABLOCKX) - ( - wait_info - ) - EXECUTE sys.sp_executesql - @sql, - @params, - @start_date, - @end_date; - - IF @debug = 1 - BEGIN - SET STATISTICS XML OFF; - END; - END; - - /*Grab data from the sp_server_diagnostics_component_result component*/ - SELECT - @sql = N' - SELECT - sp_server_diagnostics_component_result = - ISNULL + RAISERROR('Beginning collection loop for system_health data', 0, 1) WITH NOWAIT; + END; + + -- Declare a cursor to process each collection area + SET @collection_cursor = + CURSOR + LOCAL + FAST_FORWARD + FOR + SELECT + ca.area_name, + ca.object_name, + ca.temp_table, + ca.insert_list + FROM @collection_areas AS ca + WHERE should_collect = 1 + ORDER BY + ca.area_name, + ca.object_name; + + OPEN @collection_cursor; + + FETCH NEXT + FROM @collection_cursor + INTO + @area_name, + @object_name, + @temp_table, + @insert_list; + + WHILE @@FETCH_STATUS = 0 + BEGIN + -- Build the SQL statement for this collection area + SET + @collection_sql = + REPLACE ( - xml.sp_server_diagnostics_component_result, - CONVERT(xml, N''event'') - ) - FROM - ( - SELECT - sp_server_diagnostics_component_result = - TRY_CAST(fx.event_data AS xml) - FROM sys.fn_xe_file_target_read_file(N''system_health*.xel'', NULL, NULL, NULL) AS fx - WHERE fx.object_name = N''sp_server_diagnostics_component_result'' - AND CONVERT(datetimeoffset(7), fx.timestamp_utc) BETWEEN @start_date AND @end_date - ) AS xml - CROSS APPLY xml.sp_server_diagnostics_component_result.nodes(''/event'') AS e(x) - OPTION(RECOMPILE);'; - - IF @debug = 1 - BEGIN - PRINT @sql; - RAISERROR('Inserting #sp_server_diagnostics_component_result', 0, 1) WITH NOWAIT; - SET STATISTICS XML ON; - END; - - INSERT INTO - #sp_server_diagnostics_component_result - WITH - (TABLOCKX) - ( - sp_server_diagnostics_component_result - ) - EXECUTE sys.sp_executesql - @sql, - @params, - @start_date, - @end_date; - - IF @debug = 1 - BEGIN - SET STATISTICS XML OFF; - END; - - /*Grab data from the xml_deadlock_report component*/ - IF - ( - @what_to_check IN ('all', 'locking') - AND @skip_locks = 0 - ) - BEGIN - IF @debug = 1 - BEGIN - RAISERROR('Checking locking for not Managed Instance, 2017+', 0, 1) WITH NOWAIT; - END; - - SELECT - @sql = N' - SELECT - xml_deadlock_report = - ISNULL + REPLACE ( - xml.xml_deadlock_report, - CONVERT(xml, N''event'') - ) - FROM - ( - SELECT - xml_deadlock_report = - TRY_CAST(fx.event_data AS xml) - FROM sys.fn_xe_file_target_read_file(N''system_health*.xel'', NULL, NULL, NULL) AS fx - WHERE fx.object_name = N''xml_deadlock_report'' - AND CONVERT(datetimeoffset(7), fx.timestamp_utc) BETWEEN @start_date AND @end_date - ) AS xml - CROSS APPLY xml.xml_deadlock_report.nodes(''/event'') AS e(x) - OPTION(RECOMPILE);'; - - IF @debug = 1 - BEGIN - PRINT @sql; - RAISERROR('Inserting #xml_deadlock_report', 0, 1) WITH NOWAIT; - SET STATISTICS XML ON; - END; - - INSERT INTO - #xml_deadlock_report WITH(TABLOCKX) - ( - xml_deadlock_report - ) - EXECUTE sys.sp_executesql - @sql, - @params, - @start_date, - @end_date; - - IF @debug = 1 - BEGIN - SET STATISTICS XML OFF; - END; - END; - END; /*End 2016+ data collection*/ - - IF NOT EXISTS - ( - SELECT - 1/0 - FROM sys.all_columns AS ac - WHERE ac.object_id = OBJECT_ID(N'sys.fn_xe_file_target_read_file') - AND ac.name = N'timestamp_utc' - ) - AND @mi = 0 - BEGIN - IF @debug = 1 - BEGIN - RAISERROR('Checking waits for not Managed Instance, up to 2016', 0, 1) WITH NOWAIT; - END; - - /*Grab data from the wait info component*/ - IF @what_to_check IN ('all', 'waits') + REPLACE + ( + @sql_template, + '{object_name}', + @object_name + ), + '{temp_table}', + @temp_table + ), + '{insert_list}', + @insert_list + ); + IF @temp_table = '#blocking_xml' BEGIN - SELECT - @sql = N' - SELECT - wait_info = - ISNULL - ( - xml.wait_info, - CONVERT(xml, N''event'') - ) - FROM - ( - SELECT - wait_info = - TRY_CAST(fx.event_data AS xml) - FROM sys.fn_xe_file_target_read_file(N''system_health*.xel'', NULL, NULL, NULL) AS fx - WHERE fx.object_name = N''wait_info'' - ) AS xml - CROSS APPLY xml.wait_info.nodes(''/event'') AS e(x) - CROSS APPLY (SELECT x.value( ''(@timestamp)[1]'', ''datetimeoffset'' )) ca ([utc_timestamp]) - WHERE ca.utc_timestamp >= @start_date - AND ca.utc_timestamp < @end_date - OPTION(RECOMPILE);'; - - IF @debug = 1 - BEGIN - PRINT @sql; - RAISERROR('Inserting #wait_info', 0, 1) WITH NOWAIT; - SET STATISTICS XML ON; - END; - - INSERT INTO - #wait_info - WITH - (TABLOCKX) - ( - wait_info - ) - EXECUTE sys.sp_executesql - @sql, - @params, - @start_date, - @end_date; - - IF @debug = 1 BEGIN SET STATISTICS XML OFF; END; - END; - - /*Grab data from the sp_server_diagnostics_component_result component*/ + SET @collection_sql = REPLACE(@collection_sql, 'human_events_xml =', 'event_time, human_events_xml =') + END + IF @debug = 1 BEGIN - RAISERROR('Checking sp_server_diagnostics_component_result for not Managed Instance, 2017+', 0, 1) WITH NOWAIT; + RAISERROR('Collecting data for area: %s, object: %s, target table: %s', 0, 1, @area_name, @object_name, @temp_table) WITH NOWAIT; + PRINT @collection_sql; END; - - SELECT - @sql = N' - SELECT - sp_server_diagnostics_component_result = - ISNULL - ( - xml.sp_server_diagnostics_component_result, - CONVERT(xml, N''event'') - ) - FROM - ( - SELECT - sp_server_diagnostics_component_result = - TRY_CAST(fx.event_data AS xml) - FROM sys.fn_xe_file_target_read_file(N''system_health*.xel'', NULL, NULL, NULL) AS fx - WHERE fx.object_name = N''sp_server_diagnostics_component_result'' - ) AS xml - CROSS APPLY xml.sp_server_diagnostics_component_result.nodes(''/event'') AS e(x) - CROSS APPLY (SELECT x.value( ''(@timestamp)[1]'', ''datetimeoffset'' )) ca ([utc_timestamp]) - WHERE ca.utc_timestamp >= @start_date - AND ca.utc_timestamp < @end_date - OPTION(RECOMPILE);'; - + IF @debug = 1 BEGIN - RAISERROR('Inserting #sp_server_diagnostics_component_result', 0, 1) WITH NOWAIT; - PRINT @sql; + RAISERROR('Executing collection SQL', 0, 1) WITH NOWAIT; SET STATISTICS XML ON; END; - - INSERT INTO - #sp_server_diagnostics_component_result - WITH - (TABLOCKX) - ( - sp_server_diagnostics_component_result - ) - EXECUTE sys.sp_executesql - @sql, + + EXECUTE sp_executesql + @collection_sql, @params, @start_date, @end_date; - + IF @debug = 1 BEGIN SET STATISTICS XML OFF; END; - - /*Grab data from the xml_deadlock_report component*/ - IF - ( - @what_to_check IN ('all', 'locking') - AND @skip_locks = 0 - ) - BEGIN - IF @debug = 1 - BEGIN - RAISERROR('Checking locking for not Managed Instance', 0, 1) WITH NOWAIT; - END; - - SELECT - @sql = N' - SELECT - xml_deadlock_report = - ISNULL - ( - xml.xml_deadlock_report, - CONVERT(xml, N''event'') - ) - FROM - ( - SELECT - xml_deadlock_report = - TRY_CAST(fx.event_data AS xml) - FROM sys.fn_xe_file_target_read_file(N''system_health*.xel'', NULL, NULL, NULL) AS fx - WHERE fx.object_name = N''xml_deadlock_report'' - ) AS xml - CROSS APPLY xml.xml_deadlock_report.nodes(''/event'') AS e(x) - CROSS APPLY (SELECT x.value( ''(@timestamp)[1]'', ''datetimeoffset'' )) ca ([utc_timestamp]) - WHERE ca.utc_timestamp >= @start_date - AND ca.utc_timestamp < @end_date - OPTION(RECOMPILE);'; - - IF @debug = 1 - BEGIN - PRINT @sql; - RAISERROR('Inserting #xml_deadlock_report', 0, 1) WITH NOWAIT; - SET STATISTICS XML ON; - END; - - INSERT INTO - #xml_deadlock_report - WITH - (TABLOCKX) - ( - xml_deadlock_report - ) - EXECUTE sys.sp_executesql - @sql, - @params, - @start_date, - @end_date; - - IF @debug = 1 - BEGIN - SET STATISTICS XML OFF; - END; - END; - END; /*End < 2017 collection*/ - - /*Scheduler monitor*/ - IF @what_to_check IN ('all', 'system', 'cpu') - BEGIN - IF @debug = 1 - BEGIN - RAISERROR('Checking scheduler monitor system health', 0, 1) WITH NOWAIT; - END; - - /*2017+*/ - IF EXISTS - ( - SELECT - 1/0 - FROM sys.all_columns AS ac - WHERE ac.object_id = OBJECT_ID(N'sys.fn_xe_file_target_read_file') - AND ac.name = N'timestamp_utc' - ) - AND @mi = 0 - BEGIN - SELECT - @sql = N' - SELECT - scheduler_monitor = - ISNULL - ( - xml.scheduler_monitor, - CONVERT(xml, N''event'') - ) - FROM - ( - SELECT - scheduler_monitor = - TRY_CAST(fx.event_data AS xml) - FROM sys.fn_xe_file_target_read_file(N''system_health*.xel'', NULL, NULL, NULL) AS fx - WHERE fx.object_name = N''scheduler_monitor_system_health'' - AND CONVERT(datetimeoffset(7), fx.timestamp_utc) BETWEEN @start_date AND @end_date - ) AS xml - CROSS APPLY xml.scheduler_monitor.nodes(''/event'') AS e(x) - OPTION(RECOMPILE);'; - - IF @debug = 1 - BEGIN - PRINT @sql; - RAISERROR('Inserting #scheduler_monitor', 0, 1) WITH NOWAIT; - SET STATISTICS XML ON; - END; - - INSERT INTO - #scheduler_monitor - WITH - (TABLOCKX) - ( - scheduler_monitor - ) - EXECUTE sys.sp_executesql - @sql, - @params, - @start_date, - @end_date; - - IF @debug = 1 - BEGIN - SET STATISTICS XML OFF; - END; - END; - - -- For pre-2017 without timestamp_utc - IF NOT EXISTS - ( - SELECT - 1/0 - FROM sys.all_columns AS ac - WHERE ac.object_id = OBJECT_ID(N'sys.fn_xe_file_target_read_file') - AND ac.name = N'timestamp_utc' - ) - AND @mi = 0 - BEGIN - SELECT - @sql = N' - SELECT - scheduler_monitor = - ISNULL - ( - xml.scheduler_monitor, - CONVERT(xml, N''event'') - ) - FROM - ( - SELECT - scheduler_monitor = - TRY_CAST(fx.event_data AS xml) - FROM sys.fn_xe_file_target_read_file(N''system_health*.xel'', NULL, NULL, NULL) AS fx - WHERE fx.object_name = N''scheduler_monitor_system_health'' - ) AS xml - CROSS APPLY xml.scheduler_monitor.nodes(''/event'') AS e(x) - CROSS APPLY (SELECT x.value( ''(@timestamp)[1]'', ''datetimeoffset'' )) ca ([utc_timestamp]) - WHERE ca.utc_timestamp >= @start_date - AND ca.utc_timestamp < @end_date - OPTION(RECOMPILE);'; - - IF @debug = 1 - BEGIN - PRINT @sql; - RAISERROR('Inserting #scheduler_monitor', 0, 1) WITH NOWAIT; - SET STATISTICS XML ON; - END; - - INSERT INTO - #scheduler_monitor - WITH - (TABLOCKX) - ( - scheduler_monitor - ) - EXECUTE sys.sp_executesql - @sql, - @params, - @start_date, - @end_date; - - IF @debug = 1 - BEGIN - SET STATISTICS XML OFF; - END; - END; + + FETCH NEXT + FROM @collection_cursor + INTO + @area_name, + @object_name, + @temp_table, + @insert_list; END; - - /*Memory broker*/ - IF @what_to_check IN ('all', 'memory') - BEGIN - /*Grab data from the memory_broker_ring_buffer component*/ - IF @debug = 1 - BEGIN - RAISERROR('Checking memory broker ring buffer', 0, 1) WITH NOWAIT; - END; - - /* - The column timestamp_utc is 2017+ only, but terribly broken: - https://dba.stackexchange.com/q/323147/32281 - https://feedback.azure.com/d365community/idea/5f8e52d6-f3d2-ec11-a81b-6045bd7ac9f9 - */ - IF EXISTS - ( - SELECT - 1/0 - FROM sys.all_columns AS ac - WHERE ac.object_id = OBJECT_ID(N'sys.fn_xe_file_target_read_file') - AND ac.name = N'timestamp_utc' - ) - AND @mi = 0 - BEGIN - SELECT - @sql = N' - SELECT - memory_broker = - ISNULL - ( - xml.memory_broker, - CONVERT(xml, N''event'') - ) - FROM - ( - SELECT - memory_broker = - TRY_CAST(fx.event_data AS xml) - FROM sys.fn_xe_file_target_read_file(N''system_health*.xel'', NULL, NULL, NULL) AS fx - WHERE fx.object_name = N''memory_broker_ring_buffer_recorded'' - AND CONVERT(datetimeoffset(7), fx.timestamp_utc) BETWEEN @start_date AND @end_date - ) AS xml - CROSS APPLY xml.memory_broker.nodes(''/event'') AS e(x) - OPTION(RECOMPILE);'; - - IF @debug = 1 - BEGIN - PRINT @sql; - RAISERROR('Inserting #memory_broker', 0, 1) WITH NOWAIT; - SET STATISTICS XML ON; - END; - - INSERT INTO - #memory_broker - WITH - (TABLOCKX) - ( - memory_broker - ) - EXECUTE sys.sp_executesql - @sql, - @params, - @start_date, - @end_date; - - IF @debug = 1 - BEGIN - SET STATISTICS XML OFF; - END; - END; - - IF NOT EXISTS - ( - SELECT - 1/0 - FROM sys.all_columns AS ac - WHERE ac.object_id = OBJECT_ID(N'sys.fn_xe_file_target_read_file') - AND ac.name = N'timestamp_utc' - ) - AND @mi = 0 - BEGIN - IF @debug = 1 - BEGIN - RAISERROR('Checking memory broker for not Managed Instance, up to 2016', 0, 1) WITH NOWAIT; - END; - - SELECT - @sql = N' - SELECT - memory_broker = - ISNULL - ( - xml.memory_broker, - CONVERT(xml, N''event'') - ) - FROM - ( - SELECT - memory_broker = - TRY_CAST(fx.event_data AS xml) - FROM sys.fn_xe_file_target_read_file(N''system_health*.xel'', NULL, NULL, NULL) AS fx - WHERE fx.object_name = N''memory_broker_ring_buffer_recorded'' - ) AS xml - CROSS APPLY xml.memory_broker.nodes(''/event'') AS e(x) - CROSS APPLY (SELECT x.value( ''(@timestamp)[1]'', ''datetimeoffset'' )) ca ([utc_timestamp]) - WHERE ca.utc_timestamp >= @start_date - AND ca.utc_timestamp < @end_date - OPTION(RECOMPILE);'; - - IF @debug = 1 - BEGIN - PRINT @sql; - RAISERROR('Inserting #memory_broker', 0, 1) WITH NOWAIT; - SET STATISTICS XML ON; - END; - - INSERT INTO - #memory_broker - WITH - (TABLOCKX) - ( - memory_broker - ) - EXECUTE sys.sp_executesql - @sql, - @params, - @start_date, - @end_date; - IF @debug = 1 - BEGIN - SET STATISTICS XML OFF; - END; - END; - END; /*End memory_broker data collection*/ - - IF @what_to_check IN ('all', 'system') - BEGIN - /*Grab data from the error_reported component*/ - IF @debug = 1 - BEGIN - RAISERROR('Checking error_reported events', 0, 1) WITH NOWAIT; - END; - - /* - The column timestamp_utc is 2017+ only, but terribly broken: - https://dba.stackexchange.com/q/323147/32281 - https://feedback.azure.com/d365community/idea/5f8e52d6-f3d2-ec11-a81b-6045bd7ac9f9 - */ - IF EXISTS - ( - SELECT - 1/0 - FROM sys.all_columns AS ac - WHERE ac.object_id = OBJECT_ID(N'sys.fn_xe_file_target_read_file') - AND ac.name = N'timestamp_utc' - ) - AND @mi = 0 - BEGIN - SELECT - @sql = N' - SELECT - error_reported = - ISNULL - ( - xml.error_reported, - CONVERT(xml, N''event'') - ) - FROM - ( - SELECT - error_reported = - TRY_CAST(fx.event_data AS xml) - FROM sys.fn_xe_file_target_read_file(N''system_health*.xel'', NULL, NULL, NULL) AS fx - WHERE fx.object_name = N''error_reported'' - AND CONVERT(datetimeoffset(7), fx.timestamp_utc) BETWEEN @start_date AND @end_date - ) AS xml - CROSS APPLY xml.error_reported.nodes(''/event'') AS e(x) - OPTION(RECOMPILE);'; - - IF @debug = 1 - BEGIN - PRINT @sql; - RAISERROR('Inserting #error_reported', 0, 1) WITH NOWAIT; - SET STATISTICS XML ON; - END; - - INSERT INTO - #error_reported - WITH - (TABLOCKX) - ( - error_reported - ) - EXECUTE sys.sp_executesql - @sql, - @params, - @start_date, - @end_date; - - IF @debug = 1 - BEGIN - SET STATISTICS XML OFF; - END; - END; - - IF NOT EXISTS - ( - SELECT - 1/0 - FROM sys.all_columns AS ac - WHERE ac.object_id = OBJECT_ID(N'sys.fn_xe_file_target_read_file') - AND ac.name = N'timestamp_utc' - ) - AND @mi = 0 - BEGIN - IF @debug = 1 - BEGIN - RAISERROR('Checking error_reported for not Managed Instance, up to 2016', 0, 1) WITH NOWAIT; - END; - - SELECT - @sql = N' - SELECT - error_reported = - ISNULL - ( - xml.error_reported, - CONVERT(xml, N''event'') - ) - FROM - ( - SELECT - error_reported = - TRY_CAST(fx.event_data AS xml) - FROM sys.fn_xe_file_target_read_file(N''system_health*.xel'', NULL, NULL, NULL) AS fx - WHERE fx.object_name = N''error_reported'' - ) AS xml - CROSS APPLY xml.error_reported.nodes(''/event'') AS e(x) - CROSS APPLY (SELECT x.value( ''(@timestamp)[1]'', ''datetimeoffset'' )) ca ([utc_timestamp]) - WHERE ca.utc_timestamp >= @start_date - AND ca.utc_timestamp < @end_date - OPTION(RECOMPILE);'; - - IF @debug = 1 - BEGIN - PRINT @sql; - RAISERROR('Inserting #error_reported', 0, 1) WITH NOWAIT; - SET STATISTICS XML ON; - END; - - INSERT INTO - #error_reported - WITH - (TABLOCKX) - ( - error_reported - ) - EXECUTE sys.sp_executesql - @sql, - @params, - @start_date, - @end_date; - - IF @debug = 1 - BEGIN - SET STATISTICS XML OFF; - END; - END; - END; /*End error_reported data collection*/ - - IF @what_to_check IN ('all', 'memory') + IF @debug = 1 BEGIN - /*Grab data from the memory_node_oom component*/ - IF @debug = 1 - BEGIN - RAISERROR('Checking memory node OOM events', 0, 1) WITH NOWAIT; - END; - - /* - The column timestamp_utc is 2017+ only, but terribly broken: - https://dba.stackexchange.com/q/323147/32281 - https://feedback.azure.com/d365community/idea/5f8e52d6-f3d2-ec11-a81b-6045bd7ac9f9 - */ - IF EXISTS - ( - SELECT - 1/0 - FROM sys.all_columns AS ac - WHERE ac.object_id = OBJECT_ID(N'sys.fn_xe_file_target_read_file') - AND ac.name = N'timestamp_utc' - ) - AND @mi = 0 - BEGIN - SELECT - @sql = N' - SELECT - memory_node_oom = - ISNULL - ( - xml.memory_node_oom, - CONVERT(xml, N''event'') - ) - FROM - ( - SELECT - memory_node_oom = - TRY_CAST(fx.event_data AS xml) - FROM sys.fn_xe_file_target_read_file(N''system_health*.xel'', NULL, NULL, NULL) AS fx - WHERE fx.object_name = N''memory_node_oom_ring_buffer_recorded'' - AND CONVERT(datetimeoffset(7), fx.timestamp_utc) BETWEEN @start_date AND @end_date - ) AS xml - CROSS APPLY xml.memory_node_oom.nodes(''/event'') AS e(x) - OPTION(RECOMPILE);'; - - IF @debug = 1 - BEGIN - PRINT @sql; - RAISERROR('Inserting #memory_node_oom', 0, 1) WITH NOWAIT; - SET STATISTICS XML ON; - END; - - INSERT INTO - #memory_node_oom - WITH - (TABLOCKX) - ( - memory_node_oom - ) - EXECUTE sys.sp_executesql - @sql, - @params, - @start_date, - @end_date; - - IF @debug = 1 - BEGIN - SET STATISTICS XML OFF; - END; - END; - - IF NOT EXISTS - ( - SELECT - 1/0 - FROM sys.all_columns AS ac - WHERE ac.object_id = OBJECT_ID(N'sys.fn_xe_file_target_read_file') - AND ac.name = N'timestamp_utc' - ) - AND @mi = 0 - BEGIN - IF @debug = 1 - BEGIN - RAISERROR('Checking memory node OOM for not Managed Instance, up to 2016', 0, 1) WITH NOWAIT; - END; - - SELECT - @sql = N' - SELECT - memory_node_oom = - ISNULL - ( - xml.memory_node_oom, - CONVERT(xml, N''event'') - ) - FROM - ( - SELECT - memory_node_oom = - TRY_CAST(fx.event_data AS xml) - FROM sys.fn_xe_file_target_read_file(N''system_health*.xel'', NULL, NULL, NULL) AS fx - WHERE fx.object_name = N''memory_node_oom_ring_buffer_recorded'' - ) AS xml - CROSS APPLY xml.memory_node_oom.nodes(''/event'') AS e(x) - CROSS APPLY (SELECT x.value( ''(@timestamp)[1]'', ''datetimeoffset'' )) ca ([utc_timestamp]) - WHERE ca.utc_timestamp >= @start_date - AND ca.utc_timestamp < @end_date - OPTION(RECOMPILE);'; - - IF @debug = 1 - BEGIN - PRINT @sql; - RAISERROR('Inserting #memory_node_oom', 0, 1) WITH NOWAIT; - SET STATISTICS XML ON; - END; - - INSERT INTO - #memory_node_oom - WITH - (TABLOCKX) - ( - memory_node_oom - ) - EXECUTE sys.sp_executesql - @sql, - @params, - @start_date, - @end_date; - - IF @debug = 1 - BEGIN - SET STATISTICS XML OFF; - END; - END; - END; /*End memory_node_oom data collection*/ + RAISERROR('Data collection complete', 0, 1) WITH NOWAIT; + END; IF @mi = 1 BEGIN From f7c71a86e73076a28e031230a0b76a11135e49b2 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Fri, 7 Mar 2025 18:51:38 -0500 Subject: [PATCH 012/246] Update sp_HealthParser.sql that'll do pig --- sp_HealthParser/sp_HealthParser.sql | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/sp_HealthParser/sp_HealthParser.sql b/sp_HealthParser/sp_HealthParser.sql index c2bef53c..335af47c 100644 --- a/sp_HealthParser/sp_HealthParser.sql +++ b/sp_HealthParser/sp_HealthParser.sql @@ -442,11 +442,13 @@ AND ca.utc_timestamp < @end_date'; DECLARE @collection_areas TABLE ( + id tinyint IDENTITY PRIMARY KEY CLUSTERED, area_name varchar(20) NOT NULL, object_name sysname NOT NULL, temp_table sysname NOT NULL, insert_list sysname NOT NULL, - should_collect bit NOT NULL DEFAULT 0 + should_collect bit NOT NULL DEFAULT 0, + is_processed bit NOT NULL DEFAULT 0 ); INSERT INTO @@ -483,7 +485,7 @@ AND ca.utc_timestamp < @end_date'; ('cpu', 'scheduler_monitor_system_health', '#scheduler_monitor', 'scheduler_monitor'), ('disk', 'sp_server_diagnostics_component_result', '#sp_server_diagnostics_component_result', 'sp_server_diagnostics_component_result'), ('locking', 'xml_deadlock_report', '#xml_deadlock_report', 'xml_deadlock_report'), - ('locking', 'human_events_xml', '#sp_server_diagnostics_component_result', 'human_events_xml'), + ('locking', 'sp_server_diagnostics_component_result', '#sp_server_diagnostics_component_result', 'sp_server_diagnostics_component_result'), ('waits', 'wait_info', '#wait_info', 'wait_info'), ('system', 'sp_server_diagnostics_component_result', '#sp_server_diagnostics_component_result', 'sp_server_diagnostics_component_result'), ('system', 'error_reported', '#error_reported', 'error_reported'), @@ -634,8 +636,10 @@ AND ca.utc_timestamp < @end_date'; -- Declare a cursor to process each collection area SET @collection_cursor = CURSOR - LOCAL - FAST_FORWARD + LOCAL + SCROLL + DYNAMIC + READ_ONLY FOR SELECT ca.area_name, @@ -643,10 +647,10 @@ AND ca.utc_timestamp < @end_date'; ca.temp_table, ca.insert_list FROM @collection_areas AS ca - WHERE should_collect = 1 + WHERE ca.should_collect = 1 + AND ca.is_processed = 0 ORDER BY - ca.area_name, - ca.object_name; + ca.id; OPEN @collection_cursor; @@ -679,10 +683,6 @@ AND ca.utc_timestamp < @end_date'; '{insert_list}', @insert_list ); - IF @temp_table = '#blocking_xml' - BEGIN - SET @collection_sql = REPLACE(@collection_sql, 'human_events_xml =', 'event_time, human_events_xml =') - END IF @debug = 1 BEGIN @@ -706,6 +706,13 @@ AND ca.utc_timestamp < @end_date'; BEGIN SET STATISTICS XML OFF; END; + + UPDATE + @collection_areas + SET + is_processed = 1 + WHERE temp_table = @temp_table + AND should_collect = 1; FETCH NEXT FROM @collection_cursor From 3fcdb963d88683177329f33d7c696567d9198d6d Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Fri, 7 Mar 2025 23:03:12 -0500 Subject: [PATCH 013/246] Update sp_HealthParser.sql fix up MI stuff --- sp_HealthParser/sp_HealthParser.sql | 193 +++++++++++++++++++--------- 1 file changed, 135 insertions(+), 58 deletions(-) diff --git a/sp_HealthParser/sp_HealthParser.sql b/sp_HealthParser/sp_HealthParser.sql index 335af47c..d1395820 100644 --- a/sp_HealthParser/sp_HealthParser.sql +++ b/sp_HealthParser/sp_HealthParser.sql @@ -182,9 +182,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END; DECLARE - @sql nvarchar(MAX) = + @sql nvarchar(max) = N'', - @params nvarchar(MAX) = + @params nvarchar(max) = N'@start_date datetimeoffset(7), @end_date datetimeoffset(7)', @azure bit = @@ -214,15 +214,15 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. @dbid integer = DB_ID(@database_name), @timestamp_utc_mode tinyint, - @sql_template nvarchar(MAX) = N'', - @time_filter nvarchar(MAX) = N'', - @cross_apply nvarchar(MAX) = N'', + @sql_template nvarchar(max) = N'', + @time_filter nvarchar(max) = N'', + @cross_apply nvarchar(max) = N'', @collection_cursor CURSOR, @area_name varchar(20), @object_name sysname, @temp_table sysname, @insert_list sysname, - @collection_sql nvarchar(MAX); + @collection_sql nvarchar(max); IF @azure = 1 BEGIN @@ -368,16 +368,16 @@ OPTION(RECOMPILE); IF @timestamp_utc_mode = 0 BEGIN - -- Pre-2017 handling + /* Pre-2017 handling */ SET @time_filter = N''; SET @cross_apply = N'CROSS APPLY xml.{object_name}.nodes(''/event'') AS e(x) CROSS APPLY (SELECT x.value( ''(@timestamp)[1]'', ''datetimeoffset'' )) ca ([utc_timestamp]) WHERE ca.utc_timestamp >= @start_date AND ca.utc_timestamp < @end_date'; - END + END; ELSE BEGIN - -- 2017+ handling + /* 2017+ handling */ SET @cross_apply = N'CROSS APPLY xml.{object_name}.nodes(''/event'') AS e(x)'; IF @timestamp_utc_mode = 1 @@ -386,7 +386,7 @@ AND ca.utc_timestamp < @end_date'; ELSE SET @time_filter = ' AND fx.timestamp_utc BETWEEN @start_date AND @end_date'; - END + END; SET @sql_template = REPLACE @@ -428,7 +428,11 @@ AND ca.utc_timestamp < @end_date'; WHEN @what_to_check = 'wait' THEN 'waits' WHEN @what_to_check IN - ('blocking', 'blocks', 'deadlock', 'deadlocks', 'lock', 'locks') + ( + 'blocking', 'blocks', + 'deadlock', 'deadlocks', + 'lock', 'locks' + ) THEN 'locking' ELSE 'all' END; @@ -440,7 +444,7 @@ AND ca.utc_timestamp < @end_date'; END; DECLARE - @collection_areas TABLE + @collection_areas table ( id tinyint IDENTITY PRIMARY KEY CLUSTERED, area_name varchar(20) NOT NULL, @@ -470,12 +474,12 @@ AND ca.utc_timestamp < @end_date'; WHEN @what_to_check = 'all' THEN CASE - WHEN area_name = 'locking' + WHEN v.area_name = 'locking' AND @skip_locks = 1 THEN 0 ELSE 1 END - WHEN @what_to_check = area_name + WHEN @what_to_check = v.area_name THEN 1 ELSE 0 END @@ -499,6 +503,9 @@ AND ca.utc_timestamp < @end_date'; table_name = '@collection_areas', ca.* FROM @collection_areas AS ca + ORDER BY + ca.id + OPTION(RECOMPILE); END; CREATE TABLE @@ -627,13 +634,13 @@ AND ca.utc_timestamp < @end_date'; OPTION(RECOMPILE); END; - -- First, ensure we're working with the correct collection areas + /* First, ensure we're working with the correct collection areas */ IF @debug = 1 BEGIN RAISERROR('Beginning collection loop for system_health data', 0, 1) WITH NOWAIT; END; - -- Declare a cursor to process each collection area + /* Declare a cursor to process each collection area */ SET @collection_cursor = CURSOR LOCAL @@ -664,7 +671,7 @@ AND ca.utc_timestamp < @end_date'; WHILE @@FETCH_STATUS = 0 BEGIN - -- Build the SQL statement for this collection area + /* Build the SQL statement for this collection area */ SET @collection_sql = REPLACE @@ -696,7 +703,7 @@ AND ca.utc_timestamp < @end_date'; SET STATISTICS XML ON; END; - EXECUTE sp_executesql + EXECUTE sys.sp_executesql @collection_sql, @params, @start_date, @@ -779,12 +786,7 @@ AND ca.utc_timestamp < @end_date'; ) SELECT x = e.x.query('.') - FROM - ( - SELECT - x - FROM #x - ) AS x + FROM #x AS x CROSS APPLY x.x.nodes('//event') AS e(x) WHERE 1 = 1 AND e.x.exist('@timestamp[.>= sql:variable("@start_date") and .< sql:variable("@end_date")]') = 1 @@ -822,39 +824,41 @@ AND ca.utc_timestamp < @end_date'; WHERE e.x.exist('@name[.= "wait_info"]') = 1 OPTION(RECOMPILE); END; - - IF @debug = 1 + IF @what_to_check IN ('all', 'disk', 'locking', 'system', 'memory') BEGIN - RAISERROR('Checking Managed Instance sp_server_diagnostics_component_result', 0, 1) WITH NOWAIT; - RAISERROR('Inserting #sp_server_diagnostics_component_result', 0, 1) WITH NOWAIT; + IF @debug = 1 + BEGIN + RAISERROR('Checking Managed Instance sp_server_diagnostics_component_result', 0, 1) WITH NOWAIT; + RAISERROR('Inserting #sp_server_diagnostics_component_result', 0, 1) WITH NOWAIT; + END; + + INSERT + #sp_server_diagnostics_component_result + WITH + (TABLOCKX) + ( + sp_server_diagnostics_component_result + ) + SELECT + e.x.query('.') + FROM #ring_buffer AS rb + CROSS APPLY rb.ring_buffer.nodes('/event') AS e(x) + WHERE e.x.exist('@name[.= "sp_server_diagnostics_component_result"]') = 1 + OPTION(RECOMPILE); END; - INSERT - #sp_server_diagnostics_component_result - WITH - (TABLOCKX) - ( - sp_server_diagnostics_component_result - ) - SELECT - e.x.query('.') - FROM #ring_buffer AS rb - CROSS APPLY rb.ring_buffer.nodes('/event') AS e(x) - WHERE e.x.exist('@name[.= "sp_server_diagnostics_component_result"]') = 1 - OPTION(RECOMPILE); - IF ( @what_to_check IN ('all', 'locking') AND @skip_locks = 0 ) BEGIN - IF @debug = 1 + IF @debug = 1 BEGIN RAISERROR('Checking Managed Instance deadlocks', 0, 1) WITH NOWAIT; RAISERROR('Inserting #xml_deadlock_report', 0, 1) WITH NOWAIT; END; - + INSERT #xml_deadlock_report WITH @@ -869,6 +873,79 @@ AND ca.utc_timestamp < @end_date'; WHERE e.x.exist('@name[.= "xml_deadlock_report"]') = 1 OPTION(RECOMPILE); END; + + /* Add scheduler_monitor collection for MI */ + IF @what_to_check IN ('all', 'system', 'cpu') + BEGIN + IF @debug = 1 + BEGIN + RAISERROR('Checking Managed Instance scheduler monitor', 0, 1) WITH NOWAIT; + RAISERROR('Inserting #scheduler_monitor', 0, 1) WITH NOWAIT; + END; + + INSERT + #scheduler_monitor + WITH + (TABLOCKX) + ( + scheduler_monitor + ) + SELECT + e.x.query('.') + FROM #ring_buffer AS rb + CROSS APPLY rb.ring_buffer.nodes('/event') AS e(x) + WHERE e.x.exist('@name[.= "scheduler_monitor_system_health"]') = 1 + OPTION(RECOMPILE); + END; + + /* Add error_reported collection for MI */ + IF @what_to_check IN ('all', 'system') + BEGIN + IF @debug = 1 + BEGIN + RAISERROR('Checking Managed Instance error reported events', 0, 1) WITH NOWAIT; + RAISERROR('Inserting #error_reported', 0, 1) WITH NOWAIT; + END; + + INSERT + #error_reported + WITH + (TABLOCKX) + ( + error_reported + ) + SELECT + e.x.query('.') + FROM #ring_buffer AS rb + CROSS APPLY rb.ring_buffer.nodes('/event') AS e(x) + WHERE e.x.exist('@name[.= "error_reported"]') = 1 + OPTION(RECOMPILE); + END; + + /* Add memory_broker collection for MI */ + IF @what_to_check IN ('all', 'memory') + BEGIN + IF @debug = 1 + BEGIN + RAISERROR('Checking Managed Instance memory broker events', 0, 1) WITH NOWAIT; + RAISERROR('Inserting #memory_broker', 0, 1) WITH NOWAIT; + END; + + INSERT + #memory_broker + WITH + (TABLOCKX) + ( + memory_broker + ) + SELECT + e.x.query('.') + FROM #ring_buffer AS rb + CROSS APPLY rb.ring_buffer.nodes('/event') AS e(x) + WHERE e.x.exist('@name[.= "memory_broker_ring_buffer_recorded"]') = 1 + OPTION(RECOMPILE); + END; + END; /*End Managed Instance collection*/ IF @debug = 1 @@ -1859,7 +1936,7 @@ AND ca.utc_timestamp < @end_date'; '.' ELSE 'no memory pressure events found!' END; - END + END; ELSE BEGIN SELECT @@ -2041,7 +2118,7 @@ AND ca.utc_timestamp < @end_date'; '.' ELSE 'no memory node OOM events found!' END; - END + END; ELSE BEGIN SELECT @@ -2297,7 +2374,7 @@ AND ca.utc_timestamp < @end_date'; '.' ELSE 'no scheduler issues found!' END; - END + END; ELSE BEGIN SELECT @@ -2435,7 +2512,7 @@ AND ca.utc_timestamp < @end_date'; SELECT 'Error Number Ignored: ' + CONVERT(nvarchar(100), ie.error_number) FROM #ignore_errors AS ie; - END + END; ELSE BEGIN SELECT @@ -2591,7 +2668,7 @@ AND ca.utc_timestamp < @end_date'; currentdbname = bd.value('(process/@currentdbname)[1]', 'nvarchar(128)'), spid = bd.value('(process/@spid)[1]', 'integer'), ecid = bd.value('(process/@ecid)[1]', 'integer'), - query_text_pre = bd.value('(process/inputbuf/text())[1]', 'nvarchar(MAX)'), + query_text_pre = bd.value('(process/inputbuf/text())[1]', 'nvarchar(max)'), wait_time = bd.value('(process/@waittime)[1]', 'bigint'), lastbatchstarted = bd.value('(process/@lastbatchstarted)[1]', 'datetime2'), lastbatchcompleted = bd.value('(process/@lastbatchcompleted)[1]', 'datetime2'), @@ -2652,7 +2729,7 @@ AND ca.utc_timestamp < @end_date'; currentdbname = bg.value('(process/@currentdbname)[1]', 'nvarchar(128)'), spid = bg.value('(process/@spid)[1]', 'integer'), ecid = bg.value('(process/@ecid)[1]', 'integer'), - query_text_pre = bg.value('(process/inputbuf/text())[1]', 'nvarchar(MAX)'), + query_text_pre = bg.value('(process/inputbuf/text())[1]', 'nvarchar(max)'), wait_time = bg.value('(process/@waittime)[1]', 'bigint'), last_transaction_started = bg.value('(process/@lastbatchstarted)[1]', 'datetime2'), last_transaction_completed = bg.value('(process/@lastbatchcompleted)[1]', 'datetime2'), @@ -2894,7 +2971,7 @@ AND ca.utc_timestamp < @end_date'; ELSE +1 END OPTION(RECOMPILE); - END + END; ELSE BEGIN SELECT @@ -2930,7 +3007,7 @@ AND ca.utc_timestamp < @end_date'; 'available plans for blocking', b.currentdbname, query_text = - TRY_CAST(b.query_text AS nvarchar(MAX)), + TRY_CAST(b.query_text AS nvarchar(max)), sql_handle = CONVERT(varbinary(64), n.c.value('@sqlhandle', 'varchar(130)'), 1), stmtstart = @@ -2949,7 +3026,7 @@ AND ca.utc_timestamp < @end_date'; CONVERT(varchar(30), 'available plans for blocking'), b.currentdbname, query_text = - TRY_CAST(b.query_text AS nvarchar(MAX)), + TRY_CAST(b.query_text AS nvarchar(max)), sql_handle = CONVERT(varbinary(64), n.c.value('@sqlhandle', 'varchar(130)'), 1), stmtstart = @@ -3236,7 +3313,7 @@ AND ca.utc_timestamp < @end_date'; dp.event_date, is_victim OPTION(RECOMPILE); - END + END; ELSE BEGIN SELECT @@ -3471,10 +3548,10 @@ AND ca.utc_timestamp < @end_date'; ORDER BY aap.avg_worker_time_ms DESC OPTION(RECOMPILE); - END + END; ELSE BEGIN - -- Only show this message if we found blocking but no plans + /* Only show this message if we found blocking but no plans */ IF EXISTS ( SELECT @@ -3502,10 +3579,10 @@ AND ca.utc_timestamp < @end_date'; ORDER BY aap.avg_worker_time_ms DESC OPTION(RECOMPILE); - END + END; ELSE BEGIN - -- Only show this message if we found deadlocks but no plans + /* Only show this message if we found deadlocks but no plans */ IF EXISTS ( SELECT From 01ca54f9405b8d571868db9bdfe44c7eddde864f Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 8 Mar 2025 09:45:41 -0500 Subject: [PATCH 014/246] Update sp_HumanEventsBlockViewer.sql tidy up where clause --- sp_HumanEvents/sp_HumanEventsBlockViewer.sql | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/sp_HumanEvents/sp_HumanEventsBlockViewer.sql b/sp_HumanEvents/sp_HumanEventsBlockViewer.sql index b58bbe1c..d37b913a 100644 --- a/sp_HumanEvents/sp_HumanEventsBlockViewer.sql +++ b/sp_HumanEvents/sp_HumanEventsBlockViewer.sql @@ -1982,20 +1982,24 @@ FROM SELECT bg.* FROM #blocking AS bg - WHERE (bg.database_name = @database_name - OR @database_name IS NULL) - OR (bg.currentdbname = @database_name - OR @database_name IS NULL) + WHERE + ( + @database_name IS NULL + OR bg.database_name = @database_name + OR bg.currentdbname = @database_name + ) UNION ALL SELECT bd.* FROM #blocked AS bd - WHERE (bd.database_name = @database_name - OR @database_name IS NULL) - OR (bd.currentdbname = @database_name - OR @database_name IS NULL) + WHERE + ( + @database_name IS NULL + OR bd.database_name = @database_name + OR bd.currentdbname = @database_name + ) ) AS kheb OPTION(RECOMPILE); From a1599bf1a188724a4b6c6e3079c161b017c170cd Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 8 Mar 2025 15:53:35 -0500 Subject: [PATCH 015/246] HE and SP Fixing up some formatting stuff in HE Adding table logging to SP --- sp_HumanEvents/sp_HumanEvents.sql | 184 +- sp_PressureDetector/sp_PressureDetector.sql | 1795 +++++++++++++------ 2 files changed, 1298 insertions(+), 681 deletions(-) diff --git a/sp_HumanEvents/sp_HumanEvents.sql b/sp_HumanEvents/sp_HumanEvents.sql index 5c460193..fa85f620 100644 --- a/sp_HumanEvents/sp_HumanEvents.sql +++ b/sp_HumanEvents/sp_HumanEvents.sql @@ -404,14 +404,14 @@ CREATE TABLE ( id integer PRIMARY KEY IDENTITY, view_name sysname NOT NULL, - view_definition varbinary(MAX) NOT NULL, + view_definition varbinary(max) NOT NULL, output_database sysname NOT NULL DEFAULT N'', output_schema sysname NOT NULL DEFAULT N'', output_table sysname NOT NULL DEFAULT N'', view_converted AS CONVERT ( - nvarchar(MAX), + nvarchar(max), view_definition ), view_converted_length AS @@ -419,7 +419,7 @@ CREATE TABLE ( CONVERT ( - nvarchar(MAX), + nvarchar(max), view_definition ) ) @@ -454,7 +454,7 @@ DECLARE CASE WHEN CONVERT ( - int, + integer, SERVERPROPERTY('EngineEdition') ) = 5 THEN 1 @@ -463,57 +463,57 @@ DECLARE @drop_old_sql nvarchar(1000) = N'', @waitfor nvarchar(20) = N'', @session_name nvarchar(512) = N'', - @session_with nvarchar(MAX) = N'', - @session_sql nvarchar(MAX) = N'', - @start_sql nvarchar(MAX) = N'', - @stop_sql nvarchar(MAX) = N'', - @drop_sql nvarchar(MAX) = N'', - @session_filter nvarchar(MAX) = N'', - @session_filter_limited nvarchar(MAX) = N'', - @session_filter_query_plans nvarchar(MAX) = N'', - @session_filter_waits nvarchar(MAX) = N'', - @session_filter_recompile nvarchar(MAX)= N'', - @session_filter_statement_completed nvarchar(MAX) = N'', - @session_filter_blocking nvarchar(MAX) = N'', - @session_filter_parameterization nvarchar(MAX) = N'', - @query_duration_filter nvarchar(MAX) = N'', - @blocking_duration_ms_filter nvarchar(MAX) = N'', - @wait_type_filter nvarchar(MAX) = N'', - @wait_duration_filter nvarchar(MAX) = N'', - @client_app_name_filter nvarchar(MAX) = N'', - @client_hostname_filter nvarchar(MAX) = N'', - @database_name_filter nvarchar(MAX) = N'', - @session_id_filter nvarchar(MAX) = N'', - @username_filter nvarchar(MAX) = N'', - @object_name_filter nvarchar(MAX) = N'', - @requested_memory_mb_filter nvarchar(MAX) = N'', + @session_with nvarchar(max) = N'', + @session_sql nvarchar(max) = N'', + @start_sql nvarchar(max) = N'', + @stop_sql nvarchar(max) = N'', + @drop_sql nvarchar(max) = N'', + @session_filter nvarchar(max) = N'', + @session_filter_limited nvarchar(max) = N'', + @session_filter_query_plans nvarchar(max) = N'', + @session_filter_waits nvarchar(max) = N'', + @session_filter_recompile nvarchar(max)= N'', + @session_filter_statement_completed nvarchar(max) = N'', + @session_filter_blocking nvarchar(max) = N'', + @session_filter_parameterization nvarchar(max) = N'', + @query_duration_filter nvarchar(max) = N'', + @blocking_duration_ms_filter nvarchar(max) = N'', + @wait_type_filter nvarchar(max) = N'', + @wait_duration_filter nvarchar(max) = N'', + @client_app_name_filter nvarchar(max) = N'', + @client_hostname_filter nvarchar(max) = N'', + @database_name_filter nvarchar(max) = N'', + @session_id_filter nvarchar(max) = N'', + @username_filter nvarchar(max) = N'', + @object_name_filter nvarchar(max) = N'', + @requested_memory_mb_filter nvarchar(max) = N'', @compile_events bit = 0, @parameterization_events bit = 0, @fully_formed_babby nvarchar(1000) = N'', - @s_out int, - @s_sql nvarchar(MAX) = N'', - @s_params nvarchar(MAX) = N'', + @s_out integer, + @s_sql nvarchar(max) = N'', + @s_params nvarchar(max) = N'', @object_id sysname = N'', @requested_memory_kb nvarchar(11) = N'', - @the_sleeper_must_awaken nvarchar(MAX) = N'', - @min_id int, - @max_id int, + @the_sleeper_must_awaken nvarchar(max) = N'', + @min_id integer, + @max_id integer, @event_type_check sysname, @object_name_check nvarchar(1000) = N'', - @table_sql nvarchar(MAX) = N'', + @table_sql nvarchar(max) = N'', @view_tracker bit, - @spe nvarchar(MAX) = N'.sys.sp_executesql ', - @view_sql nvarchar(MAX) = N'', + @spe nvarchar(max) = N'.sys.sp_executesql ', + @view_sql nvarchar(max) = N'', @view_database sysname = N'', @date_filter datetime, @Time time, - @delete_tracker int, - @the_deleter_must_awaken nvarchar(MAX) = N'', - @executer nvarchar(MAX), - @cleanup_sessions nvarchar(MAX) = N'', - @cleanup_tables nvarchar(MAX) = N'', - @drop_holder nvarchar(MAX) = N'', - @cleanup_views nvarchar(MAX) = N'', + @delete_tracker integer, + @the_deleter_must_awaken nvarchar(max) = N'', + @executer nvarchar(max), + @cleanup_sessions nvarchar(max) = N'', + @cleanup_tables nvarchar(max) = N'', + @drop_holder nvarchar(max) = N'', + @cleanup_views nvarchar(max) = N'', @nc10 nvarchar(2) = NCHAR(10), @inputbuf_bom nvarchar(1) = CONVERT(nvarchar(1), 0x0a00, 0); @@ -1026,7 +1026,7 @@ AND EXISTS 1/0 FROM sys.configurations AS c WHERE c.name = N'blocked process threshold (s)' - AND CONVERT(int, c.value_in_use) = 0 + AND CONVERT(integer, c.value_in_use) = 0 ) BEGIN RAISERROR(N'You need to set up the blocked process report in order to use this: @@ -1803,8 +1803,8 @@ BEGIN event_type = oa.c.value('@name', 'sysname'), database_name = oa.c.value('(action[@name="database_name"]/value/text())[1]', 'sysname'), object_name = oa.c.value('(data[@name="object_name"]/value/text())[1]', 'sysname'), - sql_text = oa.c.value('(action[@name="sql_text"]/value/text())[1]', 'nvarchar(MAX)'), - statement = oa.c.value('(data[@name="statement"]/value/text())[1]', 'nvarchar(MAX)'), + sql_text = oa.c.value('(action[@name="sql_text"]/value/text())[1]', 'nvarchar(max)'), + statement = oa.c.value('(data[@name="statement"]/value/text())[1]', 'nvarchar(max)'), showplan_xml = CASE WHEN @skip_plans = 0 THEN oa.c.query('(data[@name="showplan_xml"]/value/*)[1]') ELSE N'Skipped Plans' END, cpu_ms = oa.c.value('(data[@name="cpu_time"]/value/text())[1]', 'bigint') / 1000., logical_reads = (oa.c.value('(data[@name="logical_reads"]/value/text())[1]', 'bigint') * 8) / 1024., @@ -2155,7 +2155,7 @@ BEGIN event_type = oa.c.value('@name', 'sysname'), database_name = oa.c.value('(action[@name="database_name"]/value/text())[1]', 'sysname'), object_name = oa.c.value('(data[@name="object_name"]/value/text())[1]', 'sysname'), - statement_text = oa.c.value('(data[@name="statement"]/value/text())[1]', 'nvarchar(MAX)'), + statement_text = oa.c.value('(data[@name="statement"]/value/text())[1]', 'nvarchar(max)'), compile_cpu_ms = oa.c.value('(data[@name="cpu_time"]/value/text())[1]', 'bigint'), compile_duration_ms = oa.c.value('(data[@name="duration"]/value/text())[1]', 'bigint') INTO #compiles_1 @@ -2244,7 +2244,7 @@ BEGIN event_type = oa.c.value('@name', 'sysname'), database_name = oa.c.value('(action[@name="database_name"]/value/text())[1]', 'sysname'), object_name = oa.c.value('(data[@name="object_name"]/value/text())[1]', 'sysname'), - statement_text = oa.c.value('(data[@name="statement"]/value/text())[1]', 'nvarchar(MAX)') + statement_text = oa.c.value('(data[@name="statement"]/value/text())[1]', 'nvarchar(max)') INTO #compiles_0 FROM #human_events_xml AS xet OUTER APPLY xet.human_events_xml.nodes('//event') AS oa(c) @@ -2296,7 +2296,7 @@ BEGIN ), event_type = oa.c.value('@name', 'sysname'), database_name = oa.c.value('(action[@name="database_name"]/value/text())[1]', 'sysname'), - sql_text = oa.c.value('(action[@name="sql_text"]/value/text())[1]', 'nvarchar(MAX)'), + sql_text = oa.c.value('(action[@name="sql_text"]/value/text())[1]', 'nvarchar(max)'), compile_cpu_time_ms = oa.c.value('(data[@name="compile_cpu_time"]/value/text())[1]', 'bigint') / 1000., compile_duration_ms = oa.c.value('(data[@name="compile_duration"]/value/text())[1]', 'bigint') / 1000., query_param_type = oa.c.value('(data[@name="query_param_type"]/value/text())[1]', 'integer'), @@ -2408,7 +2408,7 @@ IF @compile_events = 1 database_name = oa.c.value('(action[@name="database_name"]/value/text())[1]', 'sysname'), object_name = oa.c.value('(data[@name="object_name"]/value/text())[1]', 'sysname'), recompile_cause = oa.c.value('(data[@name="recompile_cause"]/text)[1]', 'sysname'), - statement_text = oa.c.value('(data[@name="statement"]/value/text())[1]', 'nvarchar(MAX)'), + statement_text = oa.c.value('(data[@name="statement"]/value/text())[1]', 'nvarchar(max)'), recompile_cpu_ms = oa.c.value('(data[@name="cpu_time"]/value/text())[1]', 'bigint'), recompile_duration_ms = oa.c.value('(data[@name="duration"]/value/text())[1]', 'bigint') INTO #recompiles_1 @@ -2509,7 +2509,7 @@ IF @compile_events = 1 database_name = oa.c.value('(action[@name="database_name"]/value/text())[1]', 'sysname'), object_name = oa.c.value('(data[@name="object_name"]/value/text())[1]', 'sysname'), recompile_cause = oa.c.value('(data[@name="recompile_cause"]/text)[1]', 'sysname'), - statement_text = oa.c.value('(data[@name="statement"]/value/text())[1]', 'nvarchar(MAX)') + statement_text = oa.c.value('(data[@name="statement"]/value/text())[1]', 'nvarchar(max)') INTO #recompiles_0 FROM #human_events_xml AS xet OUTER APPLY xet.human_events_xml.nodes('//event') AS oa(c) @@ -2723,7 +2723,7 @@ BEGIN blocking_ecid = bg.value('(process/@ecid)[1]', 'integer'), blocked_spid = bd.value('(process/@spid)[1]', 'integer'), blocked_ecid = bd.value('(process/@ecid)[1]', 'integer'), - query_text_pre = bd.value('(process/inputbuf/text())[1]', 'nvarchar(MAX)'), + query_text_pre = bd.value('(process/inputbuf/text())[1]', 'nvarchar(max)'), wait_time = bd.value('(process/@waittime)[1]', 'bigint'), transaction_name = bd.value('(process/@transactionname)[1]', 'sysname'), last_transaction_started = bd.value('(process/@lasttranstarted)[1]', 'datetime2'), @@ -2818,7 +2818,7 @@ BEGIN blocking_ecid = bg.value('(process/@ecid)[1]', 'integer'), blocked_spid = bd.value('(process/@spid)[1]', 'integer'), blocked_ecid = bd.value('(process/@ecid)[1]', 'integer'), - query_text_pre = bg.value('(process/inputbuf/text())[1]', 'nvarchar(MAX)'), + query_text_pre = bg.value('(process/inputbuf/text())[1]', 'nvarchar(max)'), wait_time = bg.value('(process/@waittime)[1]', 'bigint'), transaction_name = bg.value('(process/@transactionname)[1]', 'sysname'), last_transaction_started = bg.value('(process/@lastbatchstarted)[1]', 'datetime2'), @@ -3217,7 +3217,7 @@ BEGIN b.currentdbid, b.contentious_object, query_text = - TRY_CAST(b.query_text AS nvarchar(MAX)), + TRY_CAST(b.query_text AS nvarchar(max)), sql_handle = CONVERT(varbinary(64), n.c.value('@sqlhandle', 'varchar(130)'), 1), stmtstart = @@ -3242,7 +3242,7 @@ BEGIN b.currentdbid, b.contentious_object, query_text = - TRY_CAST(b.query_text AS nvarchar(MAX)), + TRY_CAST(b.query_text AS nvarchar(max)), sql_handle = CONVERT(varbinary(64), n.c.value('@sqlhandle', 'varchar(130)'), 1), stmtstart = @@ -3694,14 +3694,14 @@ BEGIN THEN N'CREATE TABLE ' + @object_name_check + @nc10 + N'( id bigint PRIMARY KEY IDENTITY, server_name sysname NULL, event_time datetime2 NULL, ' + @nc10 + N' activity nvarchar(20) NULL, database_name sysname NULL, database_id integer NULL, object_id bigint NULL, contentious_object AS OBJECT_NAME(object_id, database_id), ' + @nc10 + - N' transaction_id bigint NULL, resource_owner_type sysname NULL, monitor_loop integer NULL, spid integer NULL, ecid integer NULL, query_text nvarchar(MAX) NULL, ' + + N' transaction_id bigint NULL, resource_owner_type sysname NULL, monitor_loop integer NULL, spid integer NULL, ecid integer NULL, query_text nvarchar(max) NULL, ' + N' wait_time bigint NULL, transaction_name sysname NULL, last_transaction_started nvarchar(30) NULL, wait_resource nvarchar(100) NULL, ' + @nc10 + N' lock_mode nvarchar(10) NULL, status nvarchar(10) NULL, priority integer NULL, transaction_count integer NULL, ' + @nc10 + N' client_app sysname NULL, host_name sysname NULL, login_name sysname NULL, isolation_level nvarchar(30) NULL, sql_handle varbinary(64) NULL, blocked_process_report XML NULL );' WHEN @event_type_check LIKE N'%quer%' THEN N'CREATE TABLE ' + @object_name_check + @nc10 + N'( id bigint PRIMARY KEY IDENTITY, server_name sysname NULL, event_time datetime2 NULL, event_type sysname NULL, ' + @nc10 + - N' database_name sysname NULL, object_name nvarchar(512) NULL, sql_text nvarchar(MAX) NULL, statement nvarchar(MAX) NULL, ' + @nc10 + + N' database_name sysname NULL, object_name nvarchar(512) NULL, sql_text nvarchar(max) NULL, statement nvarchar(max) NULL, ' + @nc10 + N' showplan_xml XML NULL, cpu_ms decimal(18,2) NULL, logical_reads decimal(18,2) NULL, ' + @nc10 + N' physical_reads decimal(18,2) NULL, duration_ms decimal(18,2) NULL, writes_mb decimal(18,2) NULL,' + @nc10 + N' spills_mb decimal(18,2) NULL, row_count decimal(18,2) NULL, estimated_rows decimal(18,2) NULL, dop integer NULL, ' + @nc10 + @@ -3710,19 +3710,19 @@ BEGIN WHEN @event_type_check LIKE N'%recomp%' THEN N'CREATE TABLE ' + @object_name_check + @nc10 + N'( id bigint PRIMARY KEY IDENTITY, server_name sysname NULL, event_time datetime2 NULL, event_type sysname NULL, ' + @nc10 + - N' database_name sysname NULL, object_name nvarchar(512) NULL, recompile_cause sysname NULL, statement_text nvarchar(MAX) NULL, statement_text_checksum AS CHECKSUM(database_name + statement_text) PERSISTED ' + N' database_name sysname NULL, object_name nvarchar(512) NULL, recompile_cause sysname NULL, statement_text nvarchar(max) NULL, statement_text_checksum AS CHECKSUM(database_name + statement_text) PERSISTED ' + CASE WHEN @compile_events = 1 THEN N', compile_cpu_ms bigint NULL, compile_duration_ms bigint NULL );' ELSE N' );' END WHEN @event_type_check LIKE N'%comp%' AND @event_type_check NOT LIKE N'%re%' THEN N'CREATE TABLE ' + @object_name_check + @nc10 + N'( id bigint PRIMARY KEY IDENTITY, server_name sysname NULL, event_time datetime2 NULL, event_type sysname NULL, ' + @nc10 + - N' database_name sysname NULL, object_name nvarchar(512) NULL, statement_text nvarchar(MAX) NULL, statement_text_checksum AS CHECKSUM(database_name + statement_text) PERSISTED ' + N' database_name sysname NULL, object_name nvarchar(512) NULL, statement_text nvarchar(max) NULL, statement_text_checksum AS CHECKSUM(database_name + statement_text) PERSISTED ' + CASE WHEN @compile_events = 1 THEN N', compile_cpu_ms bigint NULL, compile_duration_ms bigint NULL );' ELSE N' );' END + CASE WHEN @parameterization_events = 1 THEN @nc10 + N'CREATE TABLE ' + @object_name_check + N'_parameterization' + @nc10 + N'( id bigint PRIMARY KEY IDENTITY, server_name sysname NULL, event_time datetime2 NULL, event_type sysname NULL, ' + @nc10 + - N' database_name sysname NULL, sql_text nvarchar(MAX) NULL, compile_cpu_time_ms bigint NULL, compile_duration_ms bigint NULL, query_param_type integer NULL, ' + @nc10 + + N' database_name sysname NULL, sql_text nvarchar(max) NULL, compile_cpu_time_ms bigint NULL, compile_duration_ms bigint NULL, query_param_type integer NULL, ' + @nc10 + N' is_cached bit NULL, is_recompiled bit NULL, compile_code sysname NULL, has_literals bit NULL, is_parameterizable bit NULL, parameterized_values_count bigint NULL, ' + @nc10 + N' query_plan_hash binary(8) NULL, query_hash binary(8) NULL, plan_handle varbinary(64) NULL, statement_sql_hash varbinary(64) NULL );' ELSE N'' @@ -4048,12 +4048,12 @@ END; SELECT @table_sql = CONVERT ( - nvarchar(MAX), + nvarchar(max), CASE WHEN @event_type_check LIKE N'%wait%' /*Wait stats!*/ THEN CONVERT ( - nvarchar(MAX), + nvarchar(max), N'INSERT INTO ' + @object_name_check + N' WITH(TABLOCK) ' + @nc10 + N'( server_name, event_time, event_type, database_name, wait_type, duration_ms, ' + @nc10 + N' signal_duration_ms, wait_resource, query_plan_hash_signed, query_hash_signed, plan_handle )' + @nc10 + @@ -4078,13 +4078,13 @@ END; signal_duration_ms = c.value(''(data[@name="signal_duration"]/value/text())[1]'', ''bigint''),' + @nc10 + CONVERT ( - nvarchar(MAX), + nvarchar(max), CASE WHEN @v = 11 /*We can't get the wait resource on older versions of SQL Server*/ THEN N' ''Not Available < 2014'', ' + @nc10 ELSE N' wait_resource = c.value(''(data[@name="wait_resource"]/value/text())[1]'', ''sysname''), ' + @nc10 END -) + CONVERT(nvarchar(MAX), N' query_plan_hash_signed = +) + CONVERT(nvarchar(max), N' query_plan_hash_signed = CONVERT ( binary(8), @@ -4107,13 +4107,13 @@ AND c.exist(''@timestamp[. > sql:variable("@date_filter")]'') = 1;') /*Any existing blocking scenarios will update the blocking duration*/ THEN CONVERT ( - nvarchar(MAX), + nvarchar(max), N'INSERT INTO ' + @object_name_check + N' WITH(TABLOCK) ' + @nc10 + N'( server_name, event_time, activity, database_name, database_id, object_id, ' + @nc10 + N' transaction_id, resource_owner_type, monitor_loop, spid, ecid, query_text, wait_time, ' + @nc10 + N' transaction_name, last_transaction_started, wait_resource, lock_mode, status, priority, ' + @nc10 + N' transaction_count, client_app, host_name, login_name, isolation_level, sql_handle, blocked_process_report )' + @nc10 + -CONVERT(nvarchar(MAX), N' +CONVERT(nvarchar(max), N' SELECT server_name, event_time, activity, database_name, database_id, object_id, transaction_id, resource_owner_type, monitor_loop, spid, ecid, text, waittime, transactionname, lasttranstarted, wait_resource, lockmode, status, priority, @@ -4161,7 +4161,7 @@ FROM monitor_loop = oa.c.value(''(//@monitorLoop)[1]'', ''integer''), spid = bd.value(''(process/@spid)[1]'', ''integer''), ecid = bd.value(''(process/@ecid)[1]'', ''integer''), - text = bd.value(''(process/inputbuf/text())[1]'', ''nvarchar(MAX)''), + text = bd.value(''(process/inputbuf/text())[1]'', ''nvarchar(max)''), waittime = bd.value(''(process/@waittime)[1]'', ''bigint''), transactionname = bd.value(''(process/@transactionname)[1]'', ''sysname''), lasttranstarted = bd.value(''(process/@lasttranstarted)[1]'', ''datetime2''), @@ -4211,7 +4211,7 @@ FROM monitor_loop = oa.c.value(''(//@monitorLoop)[1]'', ''integer''), spid = bg.value(''(process/@spid)[1]'', ''integer''), ecid = bg.value(''(process/@ecid)[1]'', ''integer''), - text = bg.value(''(process/inputbuf/text())[1]'', ''nvarchar(MAX)''), + text = bg.value(''(process/inputbuf/text())[1]'', ''nvarchar(max)''), waittime = NULL, transactionname = NULL, lasttranstarted = NULL, @@ -4285,14 +4285,14 @@ JOIN THEN CONVERT ( - nvarchar(MAX), + nvarchar(max), N'INSERT INTO ' + @object_name_check + N' WITH(TABLOCK) ' + @nc10 + N'( server_name, event_time, event_type, database_name, object_name, sql_text, statement, ' + @nc10 + N' showplan_xml, cpu_ms, logical_reads, physical_reads, duration_ms, writes_mb, ' + @nc10 + N' spills_mb, row_count, estimated_rows, dop, serial_ideal_memory_mb, ' + @nc10 + N' requested_memory_mb, used_memory_mb, ideal_memory_mb, granted_memory_mb, ' + @nc10 + N' query_plan_hash_signed, query_hash_signed, plan_handle )' + @nc10 + - CONVERT(nvarchar(MAX), N'SELECT + CONVERT(nvarchar(max), N'SELECT server_name = @@SERVERNAME, event_time = DATEADD @@ -4309,8 +4309,8 @@ JOIN event_type = oa.c.value(''@name'', ''sysname''), database_name = oa.c.value(''(action[@name="database_name"]/value/text())[1]'', ''sysname''), [object_name] = oa.c.value(''(data[@name="object_name"]/value/text())[1]'', ''sysname''), - sql_text = oa.c.value(''(action[@name="sql_text"]/value/text())[1]'', ''nvarchar(MAX)''), - statement = oa.c.value(''(data[@name="statement"]/value/text())[1]'', ''nvarchar(MAX)''), + sql_text = oa.c.value(''(action[@name="sql_text"]/value/text())[1]'', ''nvarchar(max)''), + statement = oa.c.value(''(data[@name="statement"]/value/text())[1]'', ''nvarchar(max)''), [showplan_xml] = oa.c.query(''(data[@name="showplan_xml"]/value/*)[1]''), cpu_ms = oa.c.value(''(data[@name="cpu_time"]/value/text())[1]'', ''bigint'') / 1000., logical_reads = (oa.c.value(''(data[@name="logical_reads"]/value/text())[1]'', ''bigint'') * 8) / 1024., @@ -4348,12 +4348,12 @@ AND oa.c.exist(''(action[@name="query_hash_signed"]/value[. != 0])'') = 1; ' THEN CONVERT ( - nvarchar(MAX), + nvarchar(max), N'INSERT INTO ' + @object_name_check + N' WITH(TABLOCK) ' + @nc10 + N'( server_name, event_time, event_type, ' + @nc10 + N' database_name, object_name, recompile_cause, statement_text ' - + CONVERT(nvarchar(MAX), CASE WHEN @compile_events = 1 THEN N', compile_cpu_ms, compile_duration_ms )' ELSE N' )' END) + @nc10 + - CONVERT(nvarchar(MAX), N'SELECT + + CONVERT(nvarchar(max), CASE WHEN @compile_events = 1 THEN N', compile_cpu_ms, compile_duration_ms )' ELSE N' )' END) + @nc10 + + CONVERT(nvarchar(max), N'SELECT server = @@SERVERNAME, event_time = DATEADD @@ -4370,8 +4370,8 @@ AND oa.c.exist(''(action[@name="query_hash_signed"]/value[. != 0])'') = 1; ' database_name = oa.c.value(''(action[@name="database_name"]/value/text())[1]'', ''sysname''), [object_name] = oa.c.value(''(data[@name="object_name"]/value/text())[1]'', ''sysname''), recompile_cause = oa.c.value(''(data[@name="recompile_cause"]/text)[1]'', ''sysname''), - statement_text = oa.c.value(''(data[@name="statement"]/value/text())[1]'', ''nvarchar(MAX)'')' - + CONVERT(nvarchar(MAX), CASE WHEN @compile_events = 1 /*Only get these columns if we're using the newer XE: sql_statement_post_compile*/ + statement_text = oa.c.value(''(data[@name="statement"]/value/text())[1]'', ''nvarchar(max)'')' + + CONVERT(nvarchar(max), CASE WHEN @compile_events = 1 /*Only get these columns if we're using the newer XE: sql_statement_post_compile*/ THEN N' , compile_cpu_ms = oa.c.value(''(data[@name="cpu_time"]/value/text())[1]'', ''bigint''), @@ -4381,12 +4381,12 @@ AND oa.c.exist(''(action[@name="query_hash_signed"]/value[. != 0])'') = 1; ' FROM #human_events_xml_internal AS xet OUTER APPLY xet.human_events_xml.nodes(''//event'') AS oa(c) WHERE 1 = 1 ' - + CONVERT(nvarchar(MAX), CASE WHEN @compile_events = 1 /*Same here, where we need to filter data*/ + + CONVERT(nvarchar(max), CASE WHEN @compile_events = 1 /*Same here, where we need to filter data*/ THEN N' AND oa.c.exist(''(data[@name="is_recompile"]/value[. = "false"])'') = 0 ' ELSE N'' - END) + CONVERT(nvarchar(MAX), N' + END) + CONVERT(nvarchar(max), N' AND oa.c.exist(''@timestamp[. > sql:variable("@date_filter")]'') = 1 ORDER BY event_time;' @@ -4395,12 +4395,12 @@ ORDER BY THEN CONVERT ( - nvarchar(MAX), + nvarchar(max), N'INSERT INTO ' + REPLACE(@object_name_check, N'_parameterization', N'') + N' WITH(TABLOCK) ' + @nc10 + N'( server_name, event_time, event_type, ' + @nc10 + N' database_name, object_name, statement_text ' - + CONVERT(nvarchar(MAX), CASE WHEN @compile_events = 1 THEN N', compile_cpu_ms, compile_duration_ms )' ELSE N' )' END) + @nc10 + - CONVERT(nvarchar(MAX), N'SELECT + + CONVERT(nvarchar(max), CASE WHEN @compile_events = 1 THEN N', compile_cpu_ms, compile_duration_ms )' ELSE N' )' END) + @nc10 + + CONVERT(nvarchar(max), N'SELECT server_name = @@SERVERNAME, event_time = DATEADD @@ -4417,8 +4417,8 @@ ORDER BY event_type = oa.c.value(''@name'', ''sysname''), database_name = oa.c.value(''(action[@name="database_name"]/value/text())[1]'', ''sysname''), [object_name] = oa.c.value(''(data[@name="object_name"]/value/text())[1]'', ''sysname''), - statement_text = oa.c.value(''(data[@name="statement"]/value/text())[1]'', ''nvarchar(MAX)'')' - + CONVERT(nvarchar(MAX), CASE WHEN @compile_events = 1 /*Only get these columns if we're using the newer XE: sql_statement_post_compile*/ + statement_text = oa.c.value(''(data[@name="statement"]/value/text())[1]'', ''nvarchar(max)'')' + + CONVERT(nvarchar(max), CASE WHEN @compile_events = 1 /*Only get these columns if we're using the newer XE: sql_statement_post_compile*/ THEN N' , compile_cpu_ms = oa.c.value(''(data[@name="cpu_time"]/value/text())[1]'', ''bigint''), @@ -4428,12 +4428,12 @@ ORDER BY FROM #human_events_xml_internal AS xet OUTER APPLY xet.human_events_xml.nodes(''//event'') AS oa(c) WHERE 1 = 1 ' - + CONVERT(nvarchar(MAX), CASE WHEN @compile_events = 1 /*Just like above*/ + + CONVERT(nvarchar(max), CASE WHEN @compile_events = 1 /*Just like above*/ THEN N' AND oa.c.exist(''(data[@name="is_recompile"]/value[. = "false"])'') = 1 ' ELSE N'' - END) + CONVERT(nvarchar(MAX), N' + END) + CONVERT(nvarchar(max), N' AND oa.c.exist(''@name[.= "sql_statement_post_compile"]'') = 1 AND oa.c.exist(''@timestamp[. > sql:variable("@date_filter")]'') = 1 ORDER BY @@ -4444,12 +4444,12 @@ ORDER BY @nc10 + CONVERT ( - nvarchar(MAX), + nvarchar(max), N'INSERT INTO ' + REPLACE(@object_name_check, N'_parameterization', N'') + N'_parameterization' + N' WITH(TABLOCK) ' + @nc10 + N'( server_name, event_time, event_type, database_name, sql_text, compile_cpu_time_ms, ' + @nc10 + N' compile_duration_ms, query_param_type, is_cached, is_recompiled, compile_code, has_literals, ' + @nc10 + N' is_parameterizable, parameterized_values_count, query_plan_hash, query_hash, plan_handle, statement_sql_hash ) ' + @nc10 + - CONVERT(nvarchar(MAX), N'SELECT + CONVERT(nvarchar(max), N'SELECT server_name = @@SERVERNAME, event_time = DATEADD @@ -4465,7 +4465,7 @@ ORDER BY ), event_type = oa.c.value(''@name'', ''sysname''), database_name = oa.c.value(''(action[@name="database_name"]/value/text())[1]'', ''sysname''), - sql_text = oa.c.value(''(action[@name="sql_text"]/value/text())[1]'', ''nvarchar(MAX)''), + sql_text = oa.c.value(''(action[@name="sql_text"]/value/text())[1]'', ''nvarchar(max)''), compile_cpu_time_ms = oa.c.value(''(data[@name="compile_cpu_time"]/value/text())[1]'', ''bigint'') / 1000., compile_duration_ms = oa.c.value(''(data[@name="compile_duration"]/value/text())[1]'', ''bigint'') / 1000., query_param_type = oa.c.value(''(data[@name="query_param_type"]/value/text())[1]'', ''integer''), @@ -4710,7 +4710,7 @@ BEGIN EXECUTE sys.sp_executesql @cleanup_tables, - N'@i_cleanup_tables nvarchar(MAX) OUTPUT', + N'@i_cleanup_tables nvarchar(max) OUTPUT', @i_cleanup_tables = @drop_holder OUTPUT; IF @debug = 1 @@ -4741,7 +4741,7 @@ BEGIN EXECUTE sys.sp_executesql @cleanup_views, - N'@i_cleanup_views nvarchar(MAX) OUTPUT', + N'@i_cleanup_views nvarchar(max) OUTPUT', @i_cleanup_views = @drop_holder OUTPUT; IF @debug = 1 diff --git a/sp_PressureDetector/sp_PressureDetector.sql b/sp_PressureDetector/sp_PressureDetector.sql index 21ec2ede..96839de3 100644 --- a/sp_PressureDetector/sp_PressureDetector.sql +++ b/sp_PressureDetector/sp_PressureDetector.sql @@ -57,6 +57,10 @@ ALTER PROCEDURE @skip_waits bit = 0, /*skips waits when you do not need them on every run*/ @skip_perfmon bit = 0, /*skips perfmon counters when you do not need them on every run*/ @sample_seconds tinyint = 0, /*take a sample of your server's metrics*/ + @log_to_table bit = 0, /*enable logging to permanent tables*/ + @log_database_name sysname = NULL, /*database to store logging tables*/ + @log_schema_name sysname = NULL, /*schema to store logging tables*/ + @log_table_name_prefix sysname = 'PressureDetector', /*prefix for all logging tables*/ @help bit = 0, /*how you got here*/ @debug bit = 0, /*prints dynamic sql, displays parameter and variable values, and table contents*/ @version varchar(5) = NULL OUTPUT, /*OUTPUT; for support*/ @@ -214,7 +218,6 @@ END; /*End help section*/ @what_to_check = 'all'; END; - /* Declarations of Variablependence */ @@ -359,7 +362,531 @@ OPTION(MAXDOP 1, RECOMPILE);', N'%', @memory_grant_cap xml, @cache_xml xml, - @cache_sql nvarchar(MAX) = N''; + @cache_sql nvarchar(MAX) = N'', + /*Log to table stuff*/ + @log_table_waits sysname, + @log_table_file_metrics sysname, + @log_table_perfmon sysname, + @log_table_memory sysname, + @log_table_cpu sysname, + @log_table_memory_consumers sysname, + @log_table_memory_queries sysname, + @log_table_cpu_queries sysname, + @log_table_low_memory_events sysname, + @log_table_cpu_events sysname, + @check_sql nvarchar(MAX), + @create_sql nvarchar(MAX), + @insert_sql nvarchar(MAX), + @log_run_datetime datetime2(7), + @log_database_schema nvarchar(1024); + + -- Validate logging parameters + IF @log_to_table = 1 + BEGIN + -- Generate execution ID and timestamp for this run + SELECT + @log_run_datetime = SYSDATETIME(); + + -- Default database name to current database if not specified + SELECT @log_database_name = ISNULL(@log_database_name, DB_NAME()); + + -- Default schema name to dbo if not specified + SELECT @log_schema_name = ISNULL(@log_schema_name, N'dbo'); + + -- Validate database exists + IF NOT EXISTS + ( + SELECT + 1/0 + FROM sys.databases AS d + WHERE d.name = @log_database_name + ) + BEGIN + RAISERROR('The specified logging database %s does not exist. Logging will be disabled.', 11, 1, @log_database_name) WITH NOWAIT; + RETURN; + END; + + SET + @log_database_schema = + QUOTENAME(@log_database_name) + + N'.' + + QUOTENAME(@log_schema_name) + + N'.'; + + -- Generate fully qualified table names + SELECT + @log_table_waits = + @log_database_schema + + QUOTENAME(@log_table_name_prefix + '_Waits'), + @log_table_file_metrics = + @log_database_schema + + QUOTENAME(@log_table_name_prefix + '_FileMetrics'), + @log_table_perfmon = + @log_database_schema + + QUOTENAME(@log_table_name_prefix + '_Perfmon'), + @log_table_memory = + @log_database_schema + + QUOTENAME(@log_table_name_prefix + '_Memory'), + @log_table_cpu = + @log_database_schema + + QUOTENAME(@log_table_name_prefix + '_CPU'), + @log_table_memory_consumers = + @log_database_schema + + QUOTENAME(@log_table_name_prefix + '_MemoryConsumers'), + @log_table_memory_queries = + @log_database_schema + QUOTENAME(@log_table_name_prefix + '_MemoryQueries'), + @log_table_cpu_queries = + @log_database_schema + QUOTENAME(@log_table_name_prefix + '_CPUQueries'), + @log_table_low_memory_events = + @log_database_schema + QUOTENAME(@log_table_name_prefix + '_LowMemoryEvents'), + @log_table_cpu_events = + @log_database_schema + QUOTENAME(@log_table_name_prefix + '_CPUEvents'); + + -- Check if schema exists and create it if needed + SET @check_sql = N' + IF NOT EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@log_database_name) + '.sys.schemas AS s + WHERE s.name = @schema_name + ) + BEGIN + DECLARE + @create_schema_sql nvarchar(MAX) = N''CREATE SCHEMA '' + QUOTENAME(@schema_name); + + EXECUTE ' + QUOTENAME(@log_database_name) + '.sys.sp_executesql @create_schema_sql; + IF @debug = 1 BEGIN RAISERROR(''Created schema %s in database %s for logging.'', 0, 1, @schema_name, @db_name) WITH NOWAIT; END; + END'; + + EXECUTE sys.sp_executesql + @check_sql, + N'@schema_name sysname, + @db_name sysname, + @debug bit', + @log_schema_name, + @log_database_name, + @debug; + + SET @create_sql = N' + IF NOT EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@log_database_name) + '.sys.tables AS t + JOIN ' + QUOTENAME(@log_database_name) + '.sys.schemas AS s + ON t.schema_id = s.schema_id + WHERE t.name = @table_name + ''_Waits'' + AND s.name = @schema_name + ) + BEGIN + CREATE TABLE ' + @log_table_waits + N' + ( + collection_time datetime2(7) NOT NULL, + hours_uptime integer NULL, + hours_cpu_time decimal(38,2) NULL, + wait_type nvarchar(60) NOT NULL, + description nvarchar(60) NULL, + hours_wait_time decimal(38,2) NULL, + avg_ms_per_wait decimal(38,2) NULL, + percent_signal_waits decimal(38,2) NULL, + waiting_tasks_count bigint NULL, + sample_time datetime NULL, + sorting bigint NULL, + PRIMARY KEY CLUSTERED (collection_time, wait_type) + ); + IF @debug = 1 BEGIN RAISERROR(''Created table %s for wait stats logging.'', 0, 1, ''' + @log_table_waits + ''') WITH NOWAIT; END; + END'; + + EXECUTE sys.sp_executesql + @create_sql, + N'@schema_name sysname, + @table_name sysname, + @debug bit', + @log_schema_name, + @log_table_name_prefix, + @debug; + + SET @create_sql = N' + IF NOT EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@log_database_name) + '.sys.tables AS t + JOIN ' + QUOTENAME(@log_database_name) + '.sys.schemas AS s + ON t.schema_id = s.schema_id + WHERE t.name = @table_name + ''_FileMetrics'' + AND s.name = @schema_name + ) + BEGIN + CREATE TABLE ' + @log_table_file_metrics + N' + ( + collection_time datetime2(7) NOT NULL, + hours_uptime integer NULL, + drive nvarchar(255) NOT NULL, + database_name nvarchar(128) NOT NULL, + database_file_details nvarchar(1000) NULL, + file_size_gb decimal(38,2) NULL, + total_gb_read decimal(38,2) NULL, + total_mb_read decimal(38,2) NULL, + total_read_count bigint NULL, + avg_read_stall_ms decimal(38,2) NULL, + total_gb_written decimal(38,2) NULL, + total_mb_written decimal(38,2) NULL, + total_write_count bigint NULL, + avg_write_stall_ms decimal(38,2) NULL, + io_stall_read_ms bigint NULL, + io_stall_write_ms bigint NULL, + sample_time datetime NULL, + PRIMARY KEY CLUSTERED (collection_time, drive, database_name) + ); + IF @debug = 1 BEGIN RAISERROR(''Created table %s for file metrics logging.'', 0, 1, ''' + @log_table_file_metrics + ''') WITH NOWAIT; END; + END'; + + EXECUTE sys.sp_executesql + @create_sql, + N'@schema_name sysname, + @table_name sysname, + @debug bit', + @log_schema_name, + @log_table_name_prefix, + @debug; + + SET @create_sql = N' + IF NOT EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@log_database_name) + '.sys.tables AS t + JOIN ' + QUOTENAME(@log_database_name) + '.sys.schemas AS s + ON t.schema_id = s.schema_id + WHERE t.name = @table_name + ''_Perfmon'' + AND s.name = @schema_name + ) + BEGIN + CREATE TABLE ' + @log_table_perfmon + N' + ( + collection_time datetime2(7) NOT NULL, + sample_time datetime NULL, + object_name sysname NOT NULL, + counter_name sysname NOT NULL, + counter_name_clean sysname NULL, + instance_name sysname NOT NULL, + cntr_value bigint NULL, + cntr_type bigint NULL, + PRIMARY KEY CLUSTERED (collection_time, object_name, counter_name, instance_name) + ); + IF @debug = 1 BEGIN RAISERROR(''Created table %s for perfmon logging.'', 0, 1, ''' + @log_table_perfmon + ''') WITH NOWAIT; END; + END'; + + EXECUTE sys.sp_executesql + @create_sql, + N'@schema_name sysname, + @table_name sysname, + @debug bit', + @log_schema_name, + @log_table_name_prefix, + @debug; + + SET @create_sql = N' + IF NOT EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@log_database_name) + '.sys.tables AS t + JOIN ' + QUOTENAME(@log_database_name) + '.sys.schemas AS s + ON t.schema_id = s.schema_id + WHERE t.name = @table_name + ''_Memory'' + AND s.name = @schema_name + ) + BEGIN + CREATE TABLE ' + @log_table_memory + N' + ( + collection_time datetime2(7) NOT NULL, + resource_semaphore_id integer NOT NULL, + total_database_size_gb varchar(20) NULL, + total_physical_memory_gb bigint NULL, + max_server_memory_gb bigint NULL, + memory_model nvarchar(128) NULL, + target_memory_gb decimal(38,2) NULL, + max_target_memory_gb decimal(38,2) NULL, + total_memory_gb decimal(38,2) NULL, + available_memory_gb decimal(38,2) NULL, + granted_memory_gb decimal(38,2) NULL, + used_memory_gb decimal(38,2) NULL, + grantee_count integer NULL, + waiter_count integer NULL, + timeout_error_count integer NULL, + forced_grant_count integer NULL, + total_reduced_memory_grant_count bigint NULL, + pool_id integer NULL, + memory_grant_cap xml NULL, + cache_xml xml NULL, + low_memory xml NULL, + PRIMARY KEY CLUSTERED (collection_time, resource_semaphore_id) + ); + IF @debug = 1 BEGIN RAISERROR(''Created table %s for memory logging.'', 0, 1, ''' + @log_table_memory + ''') WITH NOWAIT; END; + END'; + + EXECUTE sys.sp_executesql + @create_sql, + N'@schema_name sysname, + @table_name sysname, + @debug bit', + @log_schema_name, + @log_table_name_prefix, + @debug; + + SET @create_sql = N' + IF NOT EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@log_database_name) + '.sys.tables AS t + JOIN ' + QUOTENAME(@log_database_name) + '.sys.schemas AS s + ON t.schema_id = s.schema_id + WHERE t.name = @table_name + ''_CPU'' + AND s.name = @schema_name + ) + BEGIN + CREATE TABLE ' + @log_table_cpu + N' + ( + collection_time datetime2(7) NOT NULL, + total_threads integer NULL, + used_threads integer NULL, + available_threads integer NULL, + reserved_worker_count varchar(10) NULL, + threads_waiting_for_cpu integer NULL, + requests_waiting_for_threads integer NULL, + current_workers integer NULL, + total_active_request_count integer NULL, + total_queued_request_count integer NULL, + total_blocked_task_count integer NULL, + total_active_parallel_thread_count integer NULL, + avg_runnable_tasks_count float NULL, + high_runnable_percent varchar(100) NULL, + cpu_details_output xml NULL, + cpu_utilization_over_threshold xml NULL, + PRIMARY KEY CLUSTERED (collection_time) + ); + IF @debug = 1 BEGIN RAISERROR(''Created table %s for CPU logging.'', 0, 1, ''' + @log_table_cpu + ''') WITH NOWAIT; END; + END'; + + EXECUTE sys.sp_executesql + @create_sql, + N'@schema_name sysname, + @table_name sysname, + @debug bit', + @log_schema_name, + @log_table_name_prefix, + @debug; + + -- Memory Consumers table + SET @create_sql = N' + IF NOT EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@log_database_name) + '.sys.tables AS t + JOIN ' + QUOTENAME(@log_database_name) + '.sys.schemas AS s + ON t.schema_id = s.schema_id + WHERE t.name = @table_name + ''_MemoryConsumers'' + AND s.name = @schema_name + ) + BEGIN + CREATE TABLE ' + @log_table_memory_consumers + N' + ( + collection_time datetime2(7) NOT NULL, + memory_source nvarchar(128) NOT NULL, + memory_consumer nvarchar(128) NOT NULL, + memory_consumed_gb decimal(38,2) NULL, + PRIMARY KEY CLUSTERED (collection_time, memory_source, memory_consumer) + ); + IF @debug = 1 BEGIN RAISERROR(''Created table %s for memory consumers logging.'', 0, 1, ''' + @log_database_schema + QUOTENAME(@log_table_name_prefix + '_MemoryConsumers') + ''') WITH NOWAIT; END; + END'; + + EXECUTE sys.sp_executesql + @create_sql, + N'@schema_name sysname, + @table_name sysname, + @debug bit', + @log_schema_name, + @log_table_name_prefix, + @debug; + + -- Memory Query Grants table + SET @create_sql = N' + IF NOT EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@log_database_name) + '.sys.tables AS t + JOIN ' + QUOTENAME(@log_database_name) + '.sys.schemas AS s + ON t.schema_id = s.schema_id + WHERE t.name = @table_name + ''_MemoryQueries'' + AND s.name = @schema_name + ) + BEGIN + CREATE TABLE ' + @log_table_memory_queries + N' + ( + collection_time datetime2(7) NOT NULL, + session_id integer NOT NULL, + database_name nvarchar(128) NULL, + duration varchar(30) NULL, + request_time datetime NULL, + grant_time datetime NULL, + wait_time_seconds decimal(38,2) NULL, + requested_memory_gb decimal(38,2) NULL, + granted_memory_gb decimal(38,2) NULL, + used_memory_gb decimal(38,2) NULL, + max_used_memory_gb decimal(38,2) NULL, + ideal_memory_gb decimal(38,2) NULL, + required_memory_gb decimal(38,2) NULL, + queue_id integer NULL, + wait_order integer NULL, + is_next_candidate bit NULL, + wait_type nvarchar(60) NULL, + wait_duration_seconds decimal(38,2) NULL, + dop integer NULL, + reserved_worker_count integer NULL, + used_worker_count integer NULL, + plan_handle varbinary(64) NULL, + sql_text nvarchar(MAX) NULL, + query_plan_xml xml NULL, + PRIMARY KEY CLUSTERED (collection_time, session_id) + ); + IF @debug = 1 BEGIN RAISERROR(''Created table %s for memory queries logging.'', 0, 1, ''' + @log_database_schema + QUOTENAME(@log_table_name_prefix + '_MemoryQueries') + ''') WITH NOWAIT; END; + END'; + + EXECUTE sys.sp_executesql + @create_sql, + N'@schema_name sysname, + @table_name sysname, + @debug bit', + @log_schema_name, + @log_table_name_prefix, + @debug; + + -- CPU Queries table + SET @create_sql = N' + IF NOT EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@log_database_name) + '.sys.tables AS t + JOIN ' + QUOTENAME(@log_database_name) + '.sys.schemas AS s + ON t.schema_id = s.schema_id + WHERE t.name = @table_name + ''_CPUQueries'' + AND s.name = @schema_name + ) + BEGIN + CREATE TABLE ' + @log_table_cpu_queries + N' + ( + collection_time datetime2(7) NOT NULL, + session_id integer NOT NULL, + database_name nvarchar(128) NULL, + duration varchar(30) NULL, + status nvarchar(30) NULL, + blocking_session_id integer NULL, + wait_type nvarchar(60) NULL, + wait_time_ms bigint NULL, + wait_resource nvarchar(512) NULL, + cpu_time_ms bigint NULL, + total_elapsed_time_ms bigint NULL, + reads bigint NULL, + writes bigint NULL, + logical_reads bigint NULL, + granted_query_memory_gb decimal(38,2) NULL, + transaction_isolation_level nvarchar(30) NULL, + dop integer NULL, + parallel_worker_count integer NULL, + plan_handle varbinary(64) NULL, + sql_text nvarchar(MAX) NULL, + query_plan_xml xml NULL, + PRIMARY KEY CLUSTERED (collection_time, session_id) + ); + IF @debug = 1 BEGIN RAISERROR(''Created table %s for CPU queries logging.'', 0, 1, ''' + @log_database_schema + QUOTENAME(@log_table_name_prefix + '_CPUQueries') + ''') WITH NOWAIT; END; + END'; + + EXECUTE sys.sp_executesql + @create_sql, + N'@schema_name sysname, + @table_name sysname, + @debug bit', + @log_schema_name, + @log_table_name_prefix, + @debug; + + -- Low Memory Events table + SET @create_sql = N' + IF NOT EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@log_database_name) + '.sys.tables AS t + JOIN ' + QUOTENAME(@log_database_name) + '.sys.schemas AS s + ON t.schema_id = s.schema_id + WHERE t.name = @table_name + ''_LowMemoryEvents'' + AND s.name = @schema_name + ) + BEGIN + CREATE TABLE ' + @log_table_low_memory_events + N' + ( + collection_time datetime2(7) NOT NULL, + sample_time datetime NOT NULL, + notification_type varchar(50) NULL, + indicators_process integer NULL, + indicators_system integer NULL, + physical_memory_available_gb decimal(38,2) NULL, + virtual_memory_available_gb decimal(38,2) NULL, + PRIMARY KEY CLUSTERED (collection_time, sample_time) + ); + IF @debug = 1 BEGIN RAISERROR(''Created table %s for low memory events logging.'', 0, 1, ''' + @log_database_schema + QUOTENAME(@log_table_name_prefix + '_LowMemoryEvents') + ''') WITH NOWAIT; END; + END'; + + EXECUTE sys.sp_executesql + @create_sql, + N'@schema_name sysname, + @table_name sysname, + @debug bit', + @log_schema_name, + @log_table_name_prefix, + @debug; + + -- CPU Utilization Events table + SET @create_sql = N' + IF NOT EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@log_database_name) + '.sys.tables AS t + JOIN ' + QUOTENAME(@log_database_name) + '.sys.schemas AS s + ON t.schema_id = s.schema_id + WHERE t.name = @table_name + ''_CPUEvents'' + AND s.name = @schema_name + ) + BEGIN + CREATE TABLE ' + @log_table_cpu_events + N' + ( + collection_time datetime2(7) NOT NULL, + sample_time datetime NOT NULL, + sqlserver_cpu_utilization integer NULL, + other_process_cpu_utilization integer NULL, + total_cpu_utilization integer NULL, + PRIMARY KEY CLUSTERED (collection_time, sample_time) + ); + IF @debug = 1 BEGIN RAISERROR(''Created table %s for CPU utilization events logging.'', 0, 1, ''' + @log_database_schema + QUOTENAME(@log_table_name_prefix + '_CPUEvents') + ''') WITH NOWAIT; END; + END'; + + EXECUTE sys.sp_executesql + @create_sql, + N'@schema_name sysname, + @table_name sysname, + @debug bit', + @log_schema_name, + @log_table_name_prefix, + @debug; + END; /*End log to tables validation checks here*/ DECLARE @waits table @@ -445,6 +972,7 @@ OPTION(MAXDOP 1, RECOMPILE);', ( @what_to_check = 'all' AND @pass = 1 + AND @log_to_table = 0 ) BEGIN IF @debug = 1 @@ -760,100 +1288,108 @@ OPTION(MAXDOP 1, RECOMPILE);', dows.waiting_tasks_count DESC OPTION(MAXDOP 1, RECOMPILE); - IF @sample_seconds = 0 + IF @log_to_table = 0 BEGIN - SELECT - w.wait_type, - w.description, - w.hours_uptime, - w.hours_cpu_time, - w.hours_wait_time, - w.avg_ms_per_wait, - w.percent_signal_waits, - waiting_tasks_count = - REPLACE - ( - CONVERT + IF @sample_seconds = 0 + BEGIN + SELECT + w.wait_type, + w.description, + w.hours_uptime, + w.hours_cpu_time, + w.hours_wait_time, + w.avg_ms_per_wait, + w.percent_signal_waits, + waiting_tasks_count = + REPLACE ( - nvarchar(30), CONVERT ( - money, - w.waiting_tasks_count + nvarchar(30), + CONVERT + ( + money, + w.waiting_tasks_count + ), + 1 ), - 1 + N'.00', + N'' + ) + FROM @waits AS w + WHERE w.waiting_tasks_count_n > 0 + ORDER BY + w.sorting + OPTION(MAXDOP 1, RECOMPILE); + END; + + IF + ( + @sample_seconds > 0 + AND @pass = 1 + ) + BEGIN + SELECT + w.wait_type, + w.description, + sample_cpu_time_seconds = + CONVERT + ( + decimal(38,2), + (w2.hours_cpu_time - w.hours_cpu_time) / 1000. ), - N'.00', - N'' - ) - FROM @waits AS w - WHERE w.waiting_tasks_count_n > 0 - ORDER BY - w.sorting - OPTION(MAXDOP 1, RECOMPILE); - END; - - IF - ( - @sample_seconds > 0 - AND @pass = 1 - ) - BEGIN - SELECT - w.wait_type, - w.description, - sample_cpu_time_seconds = - CONVERT - ( - decimal(38,2), - (w2.hours_cpu_time - w.hours_cpu_time) / 1000. - ), - wait_time_seconds = - CONVERT - ( - decimal(38,2), - (w2.hours_wait_time - w.hours_wait_time) / 1000. - ), - avg_ms_per_wait = - CONVERT - ( - decimal(38,1), - (w2.avg_ms_per_wait + w.avg_ms_per_wait) / 2 - ), - percent_signal_waits = - CONVERT - ( - decimal(38,1), - (w2.percent_signal_waits + w.percent_signal_waits) / 2 - ), - waiting_tasks_count = - REPLACE - ( + wait_time_seconds = CONVERT ( - nvarchar(30), + decimal(38,2), + (w2.hours_wait_time - w.hours_wait_time) / 1000. + ), + avg_ms_per_wait = + CONVERT + ( + decimal(38,1), + (w2.avg_ms_per_wait + w.avg_ms_per_wait) / 2 + ), + percent_signal_waits = + CONVERT + ( + decimal(38,1), + (w2.percent_signal_waits + w.percent_signal_waits) / 2 + ), + waiting_tasks_count = + REPLACE + ( CONVERT ( - money, - (w2.waiting_tasks_count_n - w.waiting_tasks_count_n) + nvarchar(30), + CONVERT + ( + money, + (w2.waiting_tasks_count_n - w.waiting_tasks_count_n) + ), + 1 ), - 1 + N'.00', + N'' ), - N'.00', - N'' - ), - sample_seconds = - DATEDIFF(SECOND, w.sample_time, w2.sample_time) - FROM @waits AS w - JOIN @waits AS w2 - ON w.wait_type = w2.wait_type - AND w.sample_time < w2.sample_time - AND (w2.waiting_tasks_count_n - w.waiting_tasks_count_n) > 0 - ORDER BY - wait_time_seconds DESC - OPTION(MAXDOP 1, RECOMPILE); - END; - END; + sample_seconds = + DATEDIFF(SECOND, w.sample_time, w2.sample_time) + FROM @waits AS w + JOIN @waits AS w2 + ON w.wait_type = w2.wait_type + AND w.sample_time < w2.sample_time + AND (w2.waiting_tasks_count_n - w.waiting_tasks_count_n) > 0 + ORDER BY + wait_time_seconds DESC + OPTION(MAXDOP 1, RECOMPILE); + END; + + IF @log_to_table = 1 + BEGIN + SELECT 1 + END; + END + END; /*End wait stats*/ /* This section looks at disk metrics */ @@ -1059,22 +1595,188 @@ OPTION(MAXDOP 1, RECOMPILE);', EXECUTE sys.sp_executesql @disk_check; - IF @sample_seconds = 0 - BEGIN - WITH - file_metrics AS - ( + IF @log_to_table = 0 + BEGIN + IF @sample_seconds = 0 + BEGIN + WITH + file_metrics AS + ( + SELECT + fm.hours_uptime, + fm.drive, + fm.database_name, + fm.database_file_details, + fm.file_size_gb, + fm.avg_read_stall_ms, + fm.avg_write_stall_ms, + fm.total_gb_read, + fm.total_gb_written, + total_read_count = + REPLACE + ( + CONVERT + ( + nvarchar(30), + CONVERT + ( + money, + fm.total_read_count + ), + 1 + ), + N'.00', + N'' + ), + total_write_count = + REPLACE + ( + CONVERT + ( + nvarchar(30), + CONVERT + ( + money, + fm.total_write_count + ), + 1 + ), + N'.00', + N'' + ), + total_avg_stall_ms = + fm.avg_read_stall_ms + + fm.avg_write_stall_ms + FROM @file_metrics AS fm + WHERE fm.avg_read_stall_ms > @minimum_disk_latency_ms + OR fm.avg_write_stall_ms > @minimum_disk_latency_ms + ) SELECT - fm.hours_uptime, fm.drive, fm.database_name, fm.database_file_details, + fm.hours_uptime, fm.file_size_gb, fm.avg_read_stall_ms, fm.avg_write_stall_ms, + fm.total_avg_stall_ms, fm.total_gb_read, fm.total_gb_written, - total_read_count = + fm.total_read_count, + fm.total_write_count + FROM file_metrics AS fm + + UNION ALL + + SELECT + drive = N'Nothing to see here', + database_name = N'By default, only >100 ms latency is reported', + database_file_details = N'Use the @minimum_disk_latency_ms parameter to adjust what you see', + hours_uptime = 0, + file_size_gb = 0, + avg_read_stall_ms = 0, + avg_write_stall_ms = 0, + total_avg_stall = 0, + total_gb_read = 0, + total_gb_written = 0, + total_read_count = N'0', + total_write_count = N'0' + WHERE NOT EXISTS + ( + SELECT + 1/0 + FROM file_metrics AS fm + ) + ORDER BY + total_avg_stall_ms DESC + OPTION(MAXDOP 1, RECOMPILE); + END; + + IF + ( + @sample_seconds > 0 + AND @pass = 1 + ) + BEGIN + WITH + f AS + ( + SELECT + fm.drive, + fm.database_name, + fm.database_file_details, + fm.file_size_gb, + avg_read_stall_ms = + CASE + WHEN (fm2.total_read_count - fm.total_read_count) = 0 + THEN 0.00 + ELSE + CONVERT + ( + decimal(38, 2), + (fm2.io_stall_read_ms - fm.io_stall_read_ms) / + (fm2.total_read_count - fm.total_read_count) + ) + END, + avg_write_stall_ms = + CASE + WHEN (fm2.total_write_count - fm.total_write_count) = 0 + THEN 0.00 + ELSE + CONVERT + ( + decimal(38, 2), + (fm2.io_stall_write_ms - fm.io_stall_write_ms) / + (fm2.total_write_count - fm.total_write_count) + ) + END, + total_avg_stall = + CASE + WHEN (fm2.total_read_count - fm.total_read_count) + + (fm2.total_write_count - fm.total_write_count) = 0 + THEN 0.00 + ELSE + CONVERT + ( + decimal(38,2), + ( + (fm2.io_stall_read_ms - fm.io_stall_read_ms) + + (fm2.io_stall_write_ms - fm.io_stall_write_ms) + ) / + ( + (fm2.total_read_count - fm.total_read_count) + + (fm2.total_write_count - fm.total_write_count) + ) + ) + END, + total_mb_read = + (fm2.total_mb_read - fm.total_mb_read), + total_mb_written = + (fm2.total_mb_written - fm.total_mb_written), + total_read_count = + (fm2.total_read_count - fm.total_read_count), + total_write_count = + (fm2.total_write_count - fm.total_write_count), + sample_time_o = + fm.sample_time, + sample_time_t = + fm2.sample_time + FROM @file_metrics AS fm + JOIN @file_metrics AS fm2 + ON fm.drive = fm2.drive + AND fm.database_name = fm2.database_name + AND fm.database_file_details = fm2.database_file_details + AND fm.sample_time < fm2.sample_time + ) + SELECT + f.drive, + f.database_name, + f.database_file_details, + f.file_size_gb, + f.avg_read_stall_ms, + f.avg_write_stall_ms, + f.total_avg_stall, + total_mb_read = REPLACE ( CONVERT @@ -1083,14 +1785,14 @@ OPTION(MAXDOP 1, RECOMPILE);', CONVERT ( money, - fm.total_read_count + f.total_mb_read ), 1 ), N'.00', N'' ), - total_write_count = + total_mb_written = REPLACE ( CONVERT @@ -1099,220 +1801,62 @@ OPTION(MAXDOP 1, RECOMPILE);', CONVERT ( money, - fm.total_write_count + f.total_mb_written ), 1 ), N'.00', N'' ), - total_avg_stall_ms = - fm.avg_read_stall_ms + - fm.avg_write_stall_ms - FROM @file_metrics AS fm - WHERE fm.avg_read_stall_ms > @minimum_disk_latency_ms - OR fm.avg_write_stall_ms > @minimum_disk_latency_ms - ) - SELECT - fm.drive, - fm.database_name, - fm.database_file_details, - fm.hours_uptime, - fm.file_size_gb, - fm.avg_read_stall_ms, - fm.avg_write_stall_ms, - fm.total_avg_stall_ms, - fm.total_gb_read, - fm.total_gb_written, - fm.total_read_count, - fm.total_write_count - FROM file_metrics AS fm - - UNION ALL - - SELECT - drive = N'Nothing to see here', - database_name = N'By default, only >100 ms latency is reported', - database_file_details = N'Use the @minimum_disk_latency_ms parameter to adjust what you see', - hours_uptime = 0, - file_size_gb = 0, - avg_read_stall_ms = 0, - avg_write_stall_ms = 0, - total_avg_stall = 0, - total_gb_read = 0, - total_gb_written = 0, - total_read_count = N'0', - total_write_count = N'0' - WHERE NOT EXISTS - ( - SELECT - 1/0 - FROM file_metrics AS fm - ) - ORDER BY - total_avg_stall_ms DESC - OPTION(MAXDOP 1, RECOMPILE); - END; - - IF - ( - @sample_seconds > 0 - AND @pass = 1 - ) - BEGIN - WITH - f AS - ( - SELECT - fm.drive, - fm.database_name, - fm.database_file_details, - fm.file_size_gb, - avg_read_stall_ms = - CASE - WHEN (fm2.total_read_count - fm.total_read_count) = 0 - THEN 0.00 - ELSE - CONVERT - ( - decimal(38, 2), - (fm2.io_stall_read_ms - fm.io_stall_read_ms) / - (fm2.total_read_count - fm.total_read_count) - ) - END, - avg_write_stall_ms = - CASE - WHEN (fm2.total_write_count - fm.total_write_count) = 0 - THEN 0.00 - ELSE - CONVERT - ( - decimal(38, 2), - (fm2.io_stall_write_ms - fm.io_stall_write_ms) / - (fm2.total_write_count - fm.total_write_count) - ) - END, - total_avg_stall = - CASE - WHEN (fm2.total_read_count - fm.total_read_count) + - (fm2.total_write_count - fm.total_write_count) = 0 - THEN 0.00 - ELSE - CONVERT - ( - decimal(38,2), - ( - (fm2.io_stall_read_ms - fm.io_stall_read_ms) + - (fm2.io_stall_write_ms - fm.io_stall_write_ms) - ) / - ( - (fm2.total_read_count - fm.total_read_count) + - (fm2.total_write_count - fm.total_write_count) - ) - ) - END, - total_mb_read = - (fm2.total_mb_read - fm.total_mb_read), - total_mb_written = - (fm2.total_mb_written - fm.total_mb_written), total_read_count = - (fm2.total_read_count - fm.total_read_count), - total_write_count = - (fm2.total_write_count - fm.total_write_count), - sample_time_o = - fm.sample_time, - sample_time_t = - fm2.sample_time - FROM @file_metrics AS fm - JOIN @file_metrics AS fm2 - ON fm.drive = fm2.drive - AND fm.database_name = fm2.database_name - AND fm.database_file_details = fm2.database_file_details - AND fm.sample_time < fm2.sample_time - ) - SELECT - f.drive, - f.database_name, - f.database_file_details, - f.file_size_gb, - f.avg_read_stall_ms, - f.avg_write_stall_ms, - f.total_avg_stall, - total_mb_read = - REPLACE - ( - CONVERT - ( - nvarchar(30), - CONVERT - ( - money, - f.total_mb_read - ), - 1 - ), - N'.00', - N'' - ), - total_mb_written = - REPLACE - ( - CONVERT - ( - nvarchar(30), - CONVERT - ( - money, - f.total_mb_written - ), - 1 - ), - N'.00', - N'' - ), - total_read_count = - REPLACE - ( - CONVERT + REPLACE ( - nvarchar(30), CONVERT ( - money, - f.total_read_count + nvarchar(30), + CONVERT + ( + money, + f.total_read_count + ), + 1 ), - 1 + N'.00', + N'' ), - N'.00', - N'' - ), - total_write_count = - REPLACE - ( - CONVERT + total_write_count = + REPLACE ( - nvarchar(30), CONVERT ( - money, - f.total_write_count + nvarchar(30), + CONVERT + ( + money, + f.total_write_count + ), + 1 ), - 1 + N'.00', + N'' ), - N'.00', - N'' - ), - sample_seconds = - DATEDIFF(SECOND, f.sample_time_o, f.sample_time_t) - FROM f - WHERE - ( - f.total_read_count > 0 - OR f.total_write_count > 0 - ) - ORDER BY - f.total_avg_stall DESC - OPTION(MAXDOP 1, RECOMPILE); + sample_seconds = + DATEDIFF(SECOND, f.sample_time_o, f.sample_time_t) + FROM f + WHERE + ( + f.total_read_count > 0 + OR f.total_write_count > 0 + ) + ORDER BY + f.total_avg_stall DESC + OPTION(MAXDOP 1, RECOMPILE); + END; + END + + IF @log_to_table = 1 + BEGIN + SELECT 1 END; END; /*End file stats*/ @@ -1395,112 +1939,121 @@ OPTION(MAXDOP 1, RECOMPILE);', N'Active parallel threads', N'Active requests', N'Blocked tasks', N'Query optimizations/sec', N'Queued requests', N'Reduced memory grants/sec' ); - IF @sample_seconds = 0 + + IF @log_to_table = 0 BEGIN - WITH - p AS - ( - SELECT - hours_uptime = - ( - SELECT + IF @sample_seconds = 0 + BEGIN + WITH + p AS + ( + SELECT + hours_uptime = + ( + SELECT + DATEDIFF + ( + HOUR, + dopc.sample_time, + SYSDATETIME() + ) + ), + dopc.object_name, + dopc.counter_name, + dopc.instance_name, + dopc.cntr_value, + total = + FORMAT(dopc.cntr_value, 'N0'), + total_per_second = + FORMAT + ( + dopc.cntr_value / DATEDIFF ( - HOUR, + SECOND, dopc.sample_time, SYSDATETIME() - ) - ), - dopc.object_name, - dopc.counter_name, - dopc.instance_name, - dopc.cntr_value, - total = - FORMAT(dopc.cntr_value, 'N0'), - total_per_second = - FORMAT - ( - dopc.cntr_value / - DATEDIFF - ( - SECOND, - dopc.sample_time, - SYSDATETIME() - ), - 'N0' - ) - FROM @dm_os_performance_counters AS dopc - ) - SELECT - p.object_name, - p.counter_name, - p.instance_name, - p.hours_uptime, - p.total, - p.total_per_second - FROM p - WHERE p.cntr_value > 0 - ORDER BY - p.object_name, - p.counter_name, - p.cntr_value DESC - OPTION(MAXDOP 1, RECOMPILE); - END; - - IF - ( - @sample_seconds > 0 - AND @pass = 1 - ) - BEGIN - WITH - p AS - ( + ), + 'N0' + ) + FROM @dm_os_performance_counters AS dopc + ) SELECT - dopc.object_name, - dopc.counter_name, - dopc.instance_name, - first_cntr_value = - FORMAT(dopc.cntr_value, 'N0'), - second_cntr_value = - FORMAT(dopc2.cntr_value, 'N0'), - total_difference = - FORMAT((dopc2.cntr_value - dopc.cntr_value), 'N0'), - total_difference_per_second = - FORMAT((dopc2.cntr_value - dopc.cntr_value) / - DATEDIFF(SECOND, dopc.sample_time, dopc2.sample_time), 'N0'), - sample_seconds = - DATEDIFF(SECOND, dopc.sample_time, dopc2.sample_time), - first_sample_time = - dopc.sample_time, - second_sample_time = - dopc2.sample_time, - total_difference_i = - (dopc2.cntr_value - dopc.cntr_value) - FROM @dm_os_performance_counters AS dopc - JOIN @dm_os_performance_counters AS dopc2 - ON dopc.object_name = dopc2.object_name - AND dopc.counter_name = dopc2.counter_name - AND dopc.instance_name = dopc2.instance_name - AND dopc.sample_time < dopc2.sample_time - WHERE (dopc2.cntr_value - dopc.cntr_value) <> 0 + p.object_name, + p.counter_name, + p.instance_name, + p.hours_uptime, + p.total, + p.total_per_second + FROM p + WHERE p.cntr_value > 0 + ORDER BY + p.object_name, + p.counter_name, + p.cntr_value DESC + OPTION(MAXDOP 1, RECOMPILE); + END; + + IF + ( + @sample_seconds > 0 + AND @pass = 1 ) - SELECT - p.object_name, - p.counter_name, - p.instance_name, - p.first_cntr_value, - p.second_cntr_value, - p.total_difference, - p.total_difference_per_second, - p.sample_seconds - FROM p - ORDER BY - p.object_name, - p.counter_name, - p.total_difference_i DESC - OPTION(MAXDOP 1, RECOMPILE); + BEGIN + WITH + p AS + ( + SELECT + dopc.object_name, + dopc.counter_name, + dopc.instance_name, + first_cntr_value = + FORMAT(dopc.cntr_value, 'N0'), + second_cntr_value = + FORMAT(dopc2.cntr_value, 'N0'), + total_difference = + FORMAT((dopc2.cntr_value - dopc.cntr_value), 'N0'), + total_difference_per_second = + FORMAT((dopc2.cntr_value - dopc.cntr_value) / + DATEDIFF(SECOND, dopc.sample_time, dopc2.sample_time), 'N0'), + sample_seconds = + DATEDIFF(SECOND, dopc.sample_time, dopc2.sample_time), + first_sample_time = + dopc.sample_time, + second_sample_time = + dopc2.sample_time, + total_difference_i = + (dopc2.cntr_value - dopc.cntr_value) + FROM @dm_os_performance_counters AS dopc + JOIN @dm_os_performance_counters AS dopc2 + ON dopc.object_name = dopc2.object_name + AND dopc.counter_name = dopc2.counter_name + AND dopc.instance_name = dopc2.instance_name + AND dopc.sample_time < dopc2.sample_time + WHERE (dopc2.cntr_value - dopc.cntr_value) <> 0 + ) + SELECT + p.object_name, + p.counter_name, + p.instance_name, + p.first_cntr_value, + p.second_cntr_value, + p.total_difference, + p.total_difference_per_second, + p.sample_seconds + FROM p + ORDER BY + p.object_name, + p.counter_name, + p.total_difference_i DESC + OPTION(MAXDOP 1, RECOMPILE); + END; END; + + IF @log_to_table = 1 + BEGIN + SELECT 1 + END; END; /*End Perfmon*/ /* @@ -1511,6 +2064,7 @@ OPTION(MAXDOP 1, RECOMPILE);', @azure = 0 AND @what_to_check = 'all' AND @pass = 1 + AND @log_to_table = 0 ) BEGIN IF @debug = 1 @@ -1777,8 +2331,16 @@ OPTION(MAXDOP 1, RECOMPILE);', PRINT @pool_sql; END; - EXECUTE sys.sp_executesql - @pool_sql; + IF @log_to_table = 0 + BEGIN + EXECUTE sys.sp_executesql + @pool_sql; + END; + + IF @log_to_table = 1 + BEGIN + SELECT 1 + END; /*Checking total database size*/ IF @azure = 1 @@ -2042,10 +2604,17 @@ OPTION(MAXDOP 1, RECOMPILE);', RAISERROR('%s', 0, 1, @cache_sql) WITH NOWAIT; END; + IF @log_to_table = 0 + BEGIN EXECUTE sys.sp_executesql @cache_sql, N'@cache_xml xml OUTPUT', @cache_xml OUTPUT; + END; + IF @log_to_table = 1 + BEGIN + SELECT 1 + END; IF @cache_xml IS NULL BEGIN @@ -2060,11 +2629,18 @@ OPTION(MAXDOP 1, RECOMPILE);', ); END; - SELECT - low_memory = - @low_memory, - cache_memory = - @cache_xml; + IF @log_to_table = 0 + BEGIN + SELECT + low_memory = + @low_memory, + cache_memory = + @cache_xml; + END; + IF @log_to_table = 1 + BEGIN + SELECT 1 + END; SELECT @memory_grant_cap = @@ -2111,87 +2687,95 @@ OPTION(MAXDOP 1, RECOMPILE);', ); END; - SELECT - deqrs.resource_semaphore_id, - total_database_size_gb = - @database_size_out_gb, - total_physical_memory_gb = - @total_physical_memory_gb, - max_server_memory_gb = - ( - SELECT - CONVERT - ( - bigint, - c.value_in_use - ) - FROM sys.configurations AS c - WHERE c.name = N'max server memory (MB)' - ) / 1024, - max_memory_grant_cap = - @memory_grant_cap, - memory_model = - ( - SELECT - osi.sql_memory_model_desc - FROM sys.dm_os_sys_info AS osi - ), - target_memory_gb = - CONVERT - ( - decimal(38, 2), - (deqrs.target_memory_kb / 1024. / 1024.) - ), - max_target_memory_gb = - CONVERT( - decimal(38, 2), - (deqrs.max_target_memory_kb / 1024. / 1024.) - ), - total_memory_gb = - CONVERT - ( - decimal(38, 2), - (deqrs.total_memory_kb / 1024. / 1024.) - ), - available_memory_gb = - CONVERT - ( - decimal(38, 2), - (deqrs.available_memory_kb / 1024. / 1024.) - ), - granted_memory_gb = - CONVERT - ( - decimal(38, 2), - (deqrs.granted_memory_kb / 1024. / 1024.) - ), - used_memory_gb = - CONVERT - ( - decimal(38, 2), - (deqrs.used_memory_kb / 1024. / 1024.) - ), - deqrs.grantee_count, - deqrs.waiter_count, - deqrs.timeout_error_count, - deqrs.forced_grant_count, - wg.total_reduced_memory_grant_count, - deqrs.pool_id - FROM sys.dm_exec_query_resource_semaphores AS deqrs - CROSS APPLY - ( - SELECT TOP (1) - total_reduced_memory_grant_count = - wg.total_reduced_memgrant_count - FROM sys.dm_resource_governor_workload_groups AS wg - WHERE wg.pool_id = deqrs.pool_id + IF @log_to_table = 0 + BEGIN + SELECT + deqrs.resource_semaphore_id, + total_database_size_gb = + @database_size_out_gb, + total_physical_memory_gb = + @total_physical_memory_gb, + max_server_memory_gb = + ( + SELECT + CONVERT + ( + bigint, + c.value_in_use + ) + FROM sys.configurations AS c + WHERE c.name = N'max server memory (MB)' + ) / 1024, + max_memory_grant_cap = + @memory_grant_cap, + memory_model = + ( + SELECT + osi.sql_memory_model_desc + FROM sys.dm_os_sys_info AS osi + ), + target_memory_gb = + CONVERT + ( + decimal(38, 2), + (deqrs.target_memory_kb / 1024. / 1024.) + ), + max_target_memory_gb = + CONVERT( + decimal(38, 2), + (deqrs.max_target_memory_kb / 1024. / 1024.) + ), + total_memory_gb = + CONVERT + ( + decimal(38, 2), + (deqrs.total_memory_kb / 1024. / 1024.) + ), + available_memory_gb = + CONVERT + ( + decimal(38, 2), + (deqrs.available_memory_kb / 1024. / 1024.) + ), + granted_memory_gb = + CONVERT + ( + decimal(38, 2), + (deqrs.granted_memory_kb / 1024. / 1024.) + ), + used_memory_gb = + CONVERT + ( + decimal(38, 2), + (deqrs.used_memory_kb / 1024. / 1024.) + ), + deqrs.grantee_count, + deqrs.waiter_count, + deqrs.timeout_error_count, + deqrs.forced_grant_count, + wg.total_reduced_memory_grant_count, + deqrs.pool_id + FROM sys.dm_exec_query_resource_semaphores AS deqrs + CROSS APPLY + ( + SELECT TOP (1) + total_reduced_memory_grant_count = + wg.total_reduced_memgrant_count + FROM sys.dm_resource_governor_workload_groups AS wg + WHERE wg.pool_id = deqrs.pool_id + ORDER BY + wg.total_reduced_memgrant_count DESC + ) AS wg + WHERE deqrs.max_target_memory_kb IS NOT NULL ORDER BY - wg.total_reduced_memgrant_count DESC - ) AS wg - WHERE deqrs.max_target_memory_kb IS NOT NULL - ORDER BY - deqrs.pool_id - OPTION(MAXDOP 1, RECOMPILE); + deqrs.pool_id + OPTION(MAXDOP 1, RECOMPILE); + END + + IF @log_to_table = 1 + BEGIN + SELECT 1 + END; END; /*End memory checks*/ /* @@ -2411,8 +2995,16 @@ OPTION(MAXDOP 1, RECOMPILE);', PRINT SUBSTRING(@mem_sql, 4001, 8000); END; + IF @log_to_table = 0 + BEGIN EXECUTE sys.sp_executesql @mem_sql; + END + + IF @log_to_table = 1 + BEGIN + SELECT 1 + END; END; /* @@ -2563,134 +3155,132 @@ OPTION(MAXDOP 1, RECOMPILE);', ); END; - SELECT - cpu_details_output = - @cpu_details_output, - cpu_utilization_over_threshold = - @cpu_utilization; + IF @log_to_table = 0 + BEGIN + SELECT + cpu_details_output = + @cpu_details_output, + cpu_utilization_over_threshold = + @cpu_utilization; + END; + IF @log_to_table = 1 + BEGIN + SELECT 1 + END; /*Thread usage*/ - SELECT - total_threads = - MAX(osi.max_workers_count), - used_threads = - SUM(dos.active_workers_count), - available_threads = - MAX(osi.max_workers_count) - SUM(dos.active_workers_count), - reserved_worker_count = - CASE @helpful_new_columns - WHEN 1 - THEN ISNULL - ( - @reserved_worker_count_out, - N'0' - ) - ELSE N'N/A' - END, - threads_waiting_for_cpu = - SUM(dos.runnable_tasks_count), - requests_waiting_for_threads = - SUM(dos.work_queue_count), - current_workers = - SUM(dos.current_workers_count), - total_active_request_count = - MAX(wg.active_request_count), - total_queued_request_count = - MAX(wg.queued_request_count), - total_blocked_task_count = - MAX(wg.blocked_task_count), - total_active_parallel_thread_count = - MAX(wg.active_parallel_thread_count), - avg_runnable_tasks_count = - AVG(dos.runnable_tasks_count), - high_runnable_percent = - MAX(ISNULL(r.high_runnable_percent, 0)) - FROM sys.dm_os_schedulers AS dos - CROSS JOIN sys.dm_os_sys_info AS osi - CROSS JOIN - ( - SELECT - active_request_count = SUM(wg.active_request_count), - queued_request_count = SUM(wg.queued_request_count), - blocked_task_count = SUM(wg.blocked_task_count), - active_parallel_thread_count = SUM(wg.active_parallel_thread_count) - FROM sys.dm_resource_governor_workload_groups AS wg - ) AS wg - OUTER APPLY - ( + IF @log_to_table = 0 + BEGIN SELECT + total_threads = + MAX(osi.max_workers_count), + used_threads = + SUM(dos.active_workers_count), + available_threads = + MAX(osi.max_workers_count) - SUM(dos.active_workers_count), + reserved_worker_count = + CASE @helpful_new_columns + WHEN 1 + THEN ISNULL + ( + @reserved_worker_count_out, + N'0' + ) + ELSE N'N/A' + END, + threads_waiting_for_cpu = + SUM(dos.runnable_tasks_count), + requests_waiting_for_threads = + SUM(dos.work_queue_count), + current_workers = + SUM(dos.current_workers_count), + total_active_request_count = + MAX(wg.active_request_count), + total_queued_request_count = + MAX(wg.queued_request_count), + total_blocked_task_count = + MAX(wg.blocked_task_count), + total_active_parallel_thread_count = + MAX(wg.active_parallel_thread_count), + avg_runnable_tasks_count = + AVG(dos.runnable_tasks_count), high_runnable_percent = - '' + - RTRIM(y.runnable_pct) + - '% of ' + - RTRIM(y.total) + - ' queries are waiting to get on a CPU.' - FROM + MAX(ISNULL(r.high_runnable_percent, 0)) + FROM sys.dm_os_schedulers AS dos + CROSS JOIN sys.dm_os_sys_info AS osi + CROSS JOIN ( SELECT - x.total, - x.runnable, - runnable_pct = - CONVERT - ( - decimal(38,2), - ( - x.runnable / (1. * NULLIF(x.total, 0)) - ) - ) * 100. + active_request_count = SUM(wg.active_request_count), + queued_request_count = SUM(wg.queued_request_count), + blocked_task_count = SUM(wg.blocked_task_count), + active_parallel_thread_count = SUM(wg.active_parallel_thread_count) + FROM sys.dm_resource_governor_workload_groups AS wg + ) AS wg + OUTER APPLY + ( + SELECT + high_runnable_percent = + '' + + RTRIM(y.runnable_pct) + + '% of ' + + RTRIM(y.total) + + ' queries are waiting to get on a CPU.' FROM ( SELECT - total = - COUNT_BIG(*), - runnable = - SUM + x.total, + x.runnable, + runnable_pct = + CONVERT ( - CASE - WHEN der.status = N'runnable' - THEN 1 - ELSE 0 - END - ) - FROM sys.dm_exec_requests AS der - WHERE der.session_id > 50 - ) AS x - ) AS y - WHERE y.runnable_pct >= 10 - AND y.total >= 4 - ) AS r - WHERE dos.status = N'VISIBLE ONLINE' - OPTION(MAXDOP 1, RECOMPILE); + decimal(38,2), + ( + x.runnable / (1. * NULLIF(x.total, 0)) + ) + ) * 100. + FROM + ( + SELECT + total = + COUNT_BIG(*), + runnable = + SUM + ( + CASE + WHEN der.status = N'runnable' + THEN 1 + ELSE 0 + END + ) + FROM sys.dm_exec_requests AS der + WHERE der.session_id > 50 + ) AS x + ) AS y + WHERE y.runnable_pct >= 10 + AND y.total >= 4 + ) AS r + WHERE dos.status = N'VISIBLE ONLINE' + OPTION(MAXDOP 1, RECOMPILE); + END; + IF @log_to_table = 1 + BEGIN + SELECT 1; + END; /* Any current threadpool waits? */ - INSERT - @threadpool_waits - ( - session_id, - wait_duration_ms, - threadpool_waits - ) - SELECT - dowt.session_id, - dowt.wait_duration_ms, - threadpool_waits = - dowt.wait_type - FROM sys.dm_os_waiting_tasks AS dowt - WHERE dowt.wait_type = N'THREADPOOL' - ORDER BY - dowt.wait_duration_ms DESC - OPTION(MAXDOP 1, RECOMPILE); - - IF @@ROWCOUNT = 0 - BEGIN - SELECT - THREADPOOL = N'No current THREADPOOL waits'; - END; - ELSE + IF @log_to_table = 0 BEGIN + INSERT + @threadpool_waits + ( + session_id, + wait_duration_ms, + threadpool_waits + ) SELECT dowt.session_id, dowt.wait_duration_ms, @@ -2701,6 +3291,25 @@ OPTION(MAXDOP 1, RECOMPILE);', ORDER BY dowt.wait_duration_ms DESC OPTION(MAXDOP 1, RECOMPILE); + + IF @@ROWCOUNT = 0 + BEGIN + SELECT + THREADPOOL = N'No current THREADPOOL waits'; + END; + ELSE + BEGIN + SELECT + dowt.session_id, + dowt.wait_duration_ms, + threadpool_waits = + dowt.wait_type + FROM sys.dm_os_waiting_tasks AS dowt + WHERE dowt.wait_type = N'THREADPOOL' + ORDER BY + dowt.wait_duration_ms DESC + OPTION(MAXDOP 1, RECOMPILE); + END; END; @@ -2969,8 +3578,16 @@ OPTION(MAXDOP 1, RECOMPILE);', PRINT SUBSTRING(@cpu_sql, 4001, 8000); END; - EXECUTE sys.sp_executesql - @cpu_sql; + IF @log_to_table = 0 + BEGIN + EXECUTE sys.sp_executesql + @cpu_sql; + END; + + IF @log_to_table = 1 + BEGIN + SELECT 1; + END; END; /*End not skipping queries*/ END; /*End CPU checks*/ From ef3cd8ef0d91b135c8abdbe3df0b50d0a4b42451 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sun, 9 Mar 2025 00:36:32 -0500 Subject: [PATCH 016/246] Update sp_PressureDetector.sql TABLE LOGGING IS ALIVE --- sp_PressureDetector/sp_PressureDetector.sql | 1711 ++++++++++++------- 1 file changed, 1091 insertions(+), 620 deletions(-) diff --git a/sp_PressureDetector/sp_PressureDetector.sql b/sp_PressureDetector/sp_PressureDetector.sql index 96839de3..f0a66317 100644 --- a/sp_PressureDetector/sp_PressureDetector.sql +++ b/sp_PressureDetector/sp_PressureDetector.sql @@ -114,6 +114,10 @@ BEGIN WHEN N'@skip_waits' THEN N'skips waits when you do not need them on every run' WHEN N'@skip_perfmon' THEN N'skips perfmon counters when you do not need them on every run' WHEN N'@sample_seconds' THEN N'take a sample of your server''s metrics' + WHEN N'@log_to_table' THEN N'enable logging to permanent tables instead of returning results' + WHEN N'@log_database_name' THEN N'database to store logging tables' + WHEN N'@log_schema_name' THEN N'schema to store logging tables' + WHEN N'@log_table_name_prefix' THEN N'prefix for all logging tables' WHEN N'@help' THEN N'how you got here' WHEN N'@debug' THEN N'prints dynamic sql, displays parameter and variable values, and table contents' WHEN N'@version' THEN N'OUTPUT; for support' @@ -130,6 +134,10 @@ BEGIN WHEN N'@skip_waits' THEN N'0 or 1' WHEN N'@skip_perfmon' THEN N'0 or 1' WHEN N'@sample_seconds' THEN N'a valid tinyint: 0-255' + WHEN N'@log_to_table' THEN N'0 or 1' + WHEN N'@log_database_name' THEN N'any valid database name' + WHEN N'@log_schema_name' THEN N'any valid schema name' + WHEN N'@log_table_name_prefix' THEN N'any valid identifier' WHEN N'@help' THEN N'0 or 1' WHEN N'@debug' THEN N'0 or 1' WHEN N'@version' THEN N'none' @@ -146,6 +154,10 @@ BEGIN WHEN N'@skip_waits' THEN N'0' WHEN N'@skip_perfmon' THEN N'0' WHEN N'@sample_seconds' THEN N'0' + WHEN N'@log_to_table' THEN N'0' + WHEN N'@log_database_name' THEN N'NULL (current database)' + WHEN N'@log_schema_name' THEN N'NULL (dbo)' + WHEN N'@log_table_name_prefix' THEN N'PressureDetector' WHEN N'@help' THEN N'0' WHEN N'@debug' THEN N'0' WHEN N'@version' THEN N'none; OUTPUT' @@ -218,6 +230,59 @@ END; /*End help section*/ @what_to_check = 'all'; END; + IF @log_to_table = 1 + AND @cpu_utilization_threshold > 0 + BEGIN + IF @debug = 1 + BEGIN + RAISERROR('Setting @cpu_utilization_threshold to 0 to capture all CPU utilization data when logging to tables', 0, 1) WITH NOWAIT; + END; + SELECT + @cpu_utilization_threshold = 0; + END; + + IF @log_to_table = 1 + AND @sample_seconds <> 0 + BEGIN + IF @debug = 1 + BEGIN + RAISERROR('Logging to tables is not compatible with @sample_seconds. Using @sample_seconds = 0', 0, 1) WITH NOWAIT; + END; + SELECT + @sample_seconds = 0; + END; + + IF @log_to_table = 1 + AND @what_to_check <> 'all' + BEGIN + IF @debug = 1 + BEGIN + RAISERROR('@what_to_check was set to %s, setting to all when logging to tables', 0, 1, @what_to_check) WITH NOWAIT; + END; + SELECT + @what_to_check = 'all'; + END; + + IF @log_to_table = 1 + AND + ( + @skip_queries = 1 + OR @skip_plan_xml = 1 + OR @skip_waits = 1 + OR @skip_perfmon = 1 + ) + BEGIN + IF @debug = 1 + BEGIN + RAISERROR('reverting skip options for table logging', 0, 1, @what_to_check) WITH NOWAIT; + END; + SELECT + @skip_queries = 0, + @skip_plan_xml = 0, + @skip_waits = 0, + @skip_perfmon = 0; + END; + /* Declarations of Variablependence */ @@ -238,7 +303,7 @@ END; /*End help section*/ THEN 1 ELSE 0 END, - @pool_sql nvarchar(MAX) = N'', + @pool_sql nvarchar(max) = N'', @pages_kb bit = CASE WHEN @@ -252,7 +317,7 @@ END; /*End help section*/ THEN 1 ELSE 0 END, - @mem_sql nvarchar(MAX) = N'', + @mem_sql nvarchar(max) = N'', @helpful_new_columns bit = CASE WHEN @@ -270,7 +335,7 @@ END; /*End help section*/ THEN 1 ELSE 0 END, - @cpu_sql nvarchar(MAX) = N'', + @cpu_sql nvarchar(max) = N'', @cool_new_columns bit = CASE WHEN @@ -289,7 +354,7 @@ END; /*End help section*/ ELSE 0 END, @reserved_worker_count_out varchar(10) = '0', - @reserved_worker_count nvarchar(MAX) = N' + @reserved_worker_count nvarchar(max) = N' SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; SELECT @@ -298,10 +363,10 @@ SELECT FROM sys.dm_exec_query_memory_grants AS deqmg OPTION(MAXDOP 1, RECOMPILE); ', - @cpu_details nvarchar(MAX) = N'', + @cpu_details nvarchar(max) = N'', @cpu_details_output xml = N'', - @cpu_details_columns nvarchar(MAX) = N'', - @cpu_details_select nvarchar(MAX) = N' + @cpu_details_columns nvarchar(max) = N'', + @cpu_details_select nvarchar(max) = N' SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; SELECT @@ -311,19 +376,19 @@ SELECT offline_cpus = (SELECT COUNT_BIG(*) FROM sys.dm_os_schedulers dos WHERE dos.is_online = 0), ', - @cpu_details_from nvarchar(MAX) = N' + @cpu_details_from nvarchar(max) = N' FROM sys.dm_os_sys_info AS osi FOR XML PATH(''cpu_details''), TYPE ) OPTION(MAXDOP 1, RECOMPILE);', - @database_size_out nvarchar(MAX) = N'', - @database_size_out_gb varchar(10) = '0', + @database_size_out nvarchar(max) = N'', + @database_size_out_gb nvarchar(10) = '0', @total_physical_memory_gb bigint, @cpu_utilization xml = N'', @low_memory xml = N'', - @disk_check nvarchar(MAX) = N'', + @disk_check nvarchar(max) = N'', @live_plans bit = CASE WHEN OBJECT_ID('sys.dm_exec_query_statistics_xml') IS NOT NULL @@ -362,7 +427,9 @@ OPTION(MAXDOP 1, RECOMPILE);', N'%', @memory_grant_cap xml, @cache_xml xml, - @cache_sql nvarchar(MAX) = N'', + @cache_sql nvarchar(max) = N'', + @resource_semaphores nvarchar(max) = N'', + @cpu_threads nvarchar(max) = N'', /*Log to table stuff*/ @log_table_waits sysname, @log_table_file_metrics sysname, @@ -372,28 +439,22 @@ OPTION(MAXDOP 1, RECOMPILE);', @log_table_memory_consumers sysname, @log_table_memory_queries sysname, @log_table_cpu_queries sysname, - @log_table_low_memory_events sysname, @log_table_cpu_events sysname, - @check_sql nvarchar(MAX), - @create_sql nvarchar(MAX), - @insert_sql nvarchar(MAX), - @log_run_datetime datetime2(7), + @check_sql nvarchar(max), + @create_sql nvarchar(max), + @insert_sql nvarchar(max), @log_database_schema nvarchar(1024); - -- Validate logging parameters + /* Validate logging parameters */ IF @log_to_table = 1 - BEGIN - -- Generate execution ID and timestamp for this run - SELECT - @log_run_datetime = SYSDATETIME(); - - -- Default database name to current database if not specified + BEGIN + /* Default database name to current database if not specified */ SELECT @log_database_name = ISNULL(@log_database_name, DB_NAME()); - -- Default schema name to dbo if not specified + /* Default schema name to dbo if not specified */ SELECT @log_schema_name = ISNULL(@log_schema_name, N'dbo'); - -- Validate database exists + /* Validate database exists */ IF NOT EXISTS ( SELECT @@ -413,49 +474,50 @@ OPTION(MAXDOP 1, RECOMPILE);', QUOTENAME(@log_schema_name) + N'.'; - -- Generate fully qualified table names + /* Generate fully qualified table names */ SELECT @log_table_waits = @log_database_schema + - QUOTENAME(@log_table_name_prefix + '_Waits'), + QUOTENAME(@log_table_name_prefix + N'_Waits'), @log_table_file_metrics = @log_database_schema + - QUOTENAME(@log_table_name_prefix + '_FileMetrics'), + QUOTENAME(@log_table_name_prefix + N'_FileMetrics'), @log_table_perfmon = @log_database_schema + - QUOTENAME(@log_table_name_prefix + '_Perfmon'), + QUOTENAME(@log_table_name_prefix + N'_Perfmon'), @log_table_memory = @log_database_schema + - QUOTENAME(@log_table_name_prefix + '_Memory'), + QUOTENAME(@log_table_name_prefix + N'_Memory'), @log_table_cpu = @log_database_schema + - QUOTENAME(@log_table_name_prefix + '_CPU'), + QUOTENAME(@log_table_name_prefix + N'_CPU'), @log_table_memory_consumers = @log_database_schema + - QUOTENAME(@log_table_name_prefix + '_MemoryConsumers'), + QUOTENAME(@log_table_name_prefix + N'_MemoryConsumers'), @log_table_memory_queries = - @log_database_schema + QUOTENAME(@log_table_name_prefix + '_MemoryQueries'), + @log_database_schema + + QUOTENAME(@log_table_name_prefix + N'_MemoryQueries'), @log_table_cpu_queries = - @log_database_schema + QUOTENAME(@log_table_name_prefix + '_CPUQueries'), - @log_table_low_memory_events = - @log_database_schema + QUOTENAME(@log_table_name_prefix + '_LowMemoryEvents'), + @log_database_schema + + QUOTENAME(@log_table_name_prefix + N'_CPUQueries'), @log_table_cpu_events = - @log_database_schema + QUOTENAME(@log_table_name_prefix + '_CPUEvents'); + @log_database_schema + + QUOTENAME(@log_table_name_prefix + N'_CPUEvents'); - -- Check if schema exists and create it if needed + /* Check if schema exists and create it if needed */ SET @check_sql = N' IF NOT EXISTS ( SELECT 1/0 - FROM ' + QUOTENAME(@log_database_name) + '.sys.schemas AS s + FROM ' + QUOTENAME(@log_database_name) + N'.sys.schemas AS s WHERE s.name = @schema_name ) BEGIN DECLARE - @create_schema_sql nvarchar(MAX) = N''CREATE SCHEMA '' + QUOTENAME(@schema_name); + @create_schema_sql nvarchar(max) = N''CREATE SCHEMA '' + QUOTENAME(@schema_name); - EXECUTE ' + QUOTENAME(@log_database_name) + '.sys.sp_executesql @create_schema_sql; + EXECUTE ' + QUOTENAME(@log_database_name) + N'.sys.sp_executesql @create_schema_sql; IF @debug = 1 BEGIN RAISERROR(''Created schema %s in database %s for logging.'', 0, 1, @schema_name, @db_name) WITH NOWAIT; END; END'; @@ -473,16 +535,17 @@ OPTION(MAXDOP 1, RECOMPILE);', ( SELECT 1/0 - FROM ' + QUOTENAME(@log_database_name) + '.sys.tables AS t - JOIN ' + QUOTENAME(@log_database_name) + '.sys.schemas AS s + FROM ' + QUOTENAME(@log_database_name) + N'.sys.tables AS t + JOIN ' + QUOTENAME(@log_database_name) + N'.sys.schemas AS s ON t.schema_id = s.schema_id - WHERE t.name = @table_name + ''_Waits'' + WHERE t.name = @table_name + N''_Waits'' AND s.name = @schema_name ) BEGIN CREATE TABLE ' + @log_table_waits + N' ( - collection_time datetime2(7) NOT NULL, + id bigint IDENTITY, + collection_time datetime2(7) NOT NULL DEFAULT SYSDATETIME(), hours_uptime integer NULL, hours_cpu_time decimal(38,2) NULL, wait_type nvarchar(60) NOT NULL, @@ -493,184 +556,186 @@ OPTION(MAXDOP 1, RECOMPILE);', waiting_tasks_count bigint NULL, sample_time datetime NULL, sorting bigint NULL, - PRIMARY KEY CLUSTERED (collection_time, wait_type) + PRIMARY KEY CLUSTERED (collection_time, id) ); - IF @debug = 1 BEGIN RAISERROR(''Created table %s for wait stats logging.'', 0, 1, ''' + @log_table_waits + ''') WITH NOWAIT; END; + IF @debug = 1 BEGIN RAISERROR(''Created table %s for wait stats logging.'', 0, 1, ''' + @log_table_waits + N''') WITH NOWAIT; END; END'; - EXECUTE sys.sp_executesql - @create_sql, - N'@schema_name sysname, - @table_name sysname, - @debug bit', - @log_schema_name, - @log_table_name_prefix, - @debug; - - SET @create_sql = N' - IF NOT EXISTS - ( - SELECT - 1/0 - FROM ' + QUOTENAME(@log_database_name) + '.sys.tables AS t - JOIN ' + QUOTENAME(@log_database_name) + '.sys.schemas AS s - ON t.schema_id = s.schema_id - WHERE t.name = @table_name + ''_FileMetrics'' - AND s.name = @schema_name - ) - BEGIN - CREATE TABLE ' + @log_table_file_metrics + N' + EXECUTE sys.sp_executesql + @create_sql, + N'@schema_name sysname, + @table_name sysname, + @debug bit', + @log_schema_name, + @log_table_name_prefix, + @debug; + + SET @create_sql = N' + IF NOT EXISTS ( - collection_time datetime2(7) NOT NULL, - hours_uptime integer NULL, - drive nvarchar(255) NOT NULL, - database_name nvarchar(128) NOT NULL, - database_file_details nvarchar(1000) NULL, - file_size_gb decimal(38,2) NULL, - total_gb_read decimal(38,2) NULL, - total_mb_read decimal(38,2) NULL, - total_read_count bigint NULL, - avg_read_stall_ms decimal(38,2) NULL, - total_gb_written decimal(38,2) NULL, - total_mb_written decimal(38,2) NULL, - total_write_count bigint NULL, - avg_write_stall_ms decimal(38,2) NULL, - io_stall_read_ms bigint NULL, - io_stall_write_ms bigint NULL, - sample_time datetime NULL, - PRIMARY KEY CLUSTERED (collection_time, drive, database_name) - ); - IF @debug = 1 BEGIN RAISERROR(''Created table %s for file metrics logging.'', 0, 1, ''' + @log_table_file_metrics + ''') WITH NOWAIT; END; - END'; - - EXECUTE sys.sp_executesql - @create_sql, - N'@schema_name sysname, - @table_name sysname, - @debug bit', - @log_schema_name, - @log_table_name_prefix, - @debug; - - SET @create_sql = N' - IF NOT EXISTS - ( - SELECT - 1/0 - FROM ' + QUOTENAME(@log_database_name) + '.sys.tables AS t - JOIN ' + QUOTENAME(@log_database_name) + '.sys.schemas AS s - ON t.schema_id = s.schema_id - WHERE t.name = @table_name + ''_Perfmon'' - AND s.name = @schema_name - ) - BEGIN - CREATE TABLE ' + @log_table_perfmon + N' + SELECT + 1/0 + FROM ' + QUOTENAME(@log_database_name) + N'.sys.tables AS t + JOIN ' + QUOTENAME(@log_database_name) + N'.sys.schemas AS s + ON t.schema_id = s.schema_id + WHERE t.name = @table_name + N''_FileMetrics'' + AND s.name = @schema_name + ) + BEGIN + CREATE TABLE ' + @log_table_file_metrics + N' + ( + id bigint IDENTITY, + collection_time datetime2(7) NOT NULL DEFAULT SYSDATETIME(), + hours_uptime integer NULL, + drive nvarchar(255) NOT NULL, + database_name nvarchar(128) NOT NULL, + database_file_details nvarchar(1000) NULL, + file_size_gb decimal(38,2) NULL, + total_gb_read decimal(38,2) NULL, + total_mb_read decimal(38,2) NULL, + total_read_count bigint NULL, + avg_read_stall_ms decimal(38,2) NULL, + total_gb_written decimal(38,2) NULL, + total_mb_written decimal(38,2) NULL, + total_write_count bigint NULL, + avg_write_stall_ms decimal(38,2) NULL, + io_stall_read_ms bigint NULL, + io_stall_write_ms bigint NULL, + sample_time datetime NULL, + PRIMARY KEY CLUSTERED (collection_time, id) + ); + IF @debug = 1 BEGIN RAISERROR(''Created table %s for file metrics logging.'', 0, 1, ''' + @log_table_file_metrics + N''') WITH NOWAIT; END; + END'; + + EXECUTE sys.sp_executesql + @create_sql, + N'@schema_name sysname, + @table_name sysname, + @debug bit', + @log_schema_name, + @log_table_name_prefix, + @debug; + + SET @create_sql = N' + IF NOT EXISTS ( - collection_time datetime2(7) NOT NULL, - sample_time datetime NULL, - object_name sysname NOT NULL, - counter_name sysname NOT NULL, - counter_name_clean sysname NULL, - instance_name sysname NOT NULL, - cntr_value bigint NULL, - cntr_type bigint NULL, - PRIMARY KEY CLUSTERED (collection_time, object_name, counter_name, instance_name) - ); - IF @debug = 1 BEGIN RAISERROR(''Created table %s for perfmon logging.'', 0, 1, ''' + @log_table_perfmon + ''') WITH NOWAIT; END; - END'; - - EXECUTE sys.sp_executesql - @create_sql, - N'@schema_name sysname, - @table_name sysname, - @debug bit', - @log_schema_name, - @log_table_name_prefix, - @debug; - - SET @create_sql = N' - IF NOT EXISTS - ( - SELECT - 1/0 - FROM ' + QUOTENAME(@log_database_name) + '.sys.tables AS t - JOIN ' + QUOTENAME(@log_database_name) + '.sys.schemas AS s - ON t.schema_id = s.schema_id - WHERE t.name = @table_name + ''_Memory'' - AND s.name = @schema_name - ) - BEGIN - CREATE TABLE ' + @log_table_memory + N' + SELECT + 1/0 + FROM ' + QUOTENAME(@log_database_name) + N'.sys.tables AS t + JOIN ' + QUOTENAME(@log_database_name) + N'.sys.schemas AS s + ON t.schema_id = s.schema_id + WHERE t.name = @table_name + N''_Perfmon'' + AND s.name = @schema_name + ) + BEGIN + CREATE TABLE ' + @log_table_perfmon + N' + ( + id bigint IDENTITY, + collection_time datetime2(7) NOT NULL DEFAULT SYSDATETIME(), + sample_time datetime NULL, + object_name sysname NOT NULL, + counter_name sysname NOT NULL, + counter_name_clean sysname NULL, + instance_name sysname NOT NULL, + cntr_value bigint NULL, + cntr_type bigint NULL, + PRIMARY KEY CLUSTERED (collection_time, id) + ); + IF @debug = 1 BEGIN RAISERROR(''Created table %s for perfmon logging.'', 0, 1, ''' + @log_table_perfmon + N''') WITH NOWAIT; END; + END'; + + EXECUTE sys.sp_executesql + @create_sql, + N'@schema_name sysname, + @table_name sysname, + @debug bit', + @log_schema_name, + @log_table_name_prefix, + @debug; + + SET @create_sql = N' + IF NOT EXISTS ( - collection_time datetime2(7) NOT NULL, - resource_semaphore_id integer NOT NULL, - total_database_size_gb varchar(20) NULL, - total_physical_memory_gb bigint NULL, - max_server_memory_gb bigint NULL, - memory_model nvarchar(128) NULL, - target_memory_gb decimal(38,2) NULL, - max_target_memory_gb decimal(38,2) NULL, - total_memory_gb decimal(38,2) NULL, - available_memory_gb decimal(38,2) NULL, - granted_memory_gb decimal(38,2) NULL, - used_memory_gb decimal(38,2) NULL, - grantee_count integer NULL, - waiter_count integer NULL, - timeout_error_count integer NULL, - forced_grant_count integer NULL, - total_reduced_memory_grant_count bigint NULL, - pool_id integer NULL, - memory_grant_cap xml NULL, - cache_xml xml NULL, - low_memory xml NULL, - PRIMARY KEY CLUSTERED (collection_time, resource_semaphore_id) - ); - IF @debug = 1 BEGIN RAISERROR(''Created table %s for memory logging.'', 0, 1, ''' + @log_table_memory + ''') WITH NOWAIT; END; - END'; - - EXECUTE sys.sp_executesql - @create_sql, - N'@schema_name sysname, - @table_name sysname, - @debug bit', - @log_schema_name, - @log_table_name_prefix, - @debug; - - SET @create_sql = N' - IF NOT EXISTS - ( - SELECT - 1/0 - FROM ' + QUOTENAME(@log_database_name) + '.sys.tables AS t - JOIN ' + QUOTENAME(@log_database_name) + '.sys.schemas AS s - ON t.schema_id = s.schema_id - WHERE t.name = @table_name + ''_CPU'' - AND s.name = @schema_name - ) - BEGIN - CREATE TABLE ' + @log_table_cpu + N' + SELECT + 1/0 + FROM ' + QUOTENAME(@log_database_name) + N'.sys.tables AS t + JOIN ' + QUOTENAME(@log_database_name) + N'.sys.schemas AS s + ON t.schema_id = s.schema_id + WHERE t.name = @table_name + N''_Memory'' + AND s.name = @schema_name + ) + BEGIN + CREATE TABLE ' + @log_table_memory + N' + ( + id bigint IDENTITY, + collection_time datetime2(7) NOT NULL DEFAULT SYSDATETIME(), + resource_semaphore_id integer NOT NULL, + total_database_size_gb varchar(20) NULL, + total_physical_memory_gb bigint NULL, + max_server_memory_gb bigint NULL, + max_memory_grant_cap xml NULL, + memory_model nvarchar(128) NULL, + target_memory_gb decimal(38,2) NULL, + max_target_memory_gb decimal(38,2) NULL, + total_memory_gb decimal(38,2) NULL, + available_memory_gb decimal(38,2) NULL, + granted_memory_gb decimal(38,2) NULL, + used_memory_gb decimal(38,2) NULL, + grantee_count integer NULL, + waiter_count integer NULL, + timeout_error_count integer NULL, + forced_grant_count integer NULL, + total_reduced_memory_grant_count bigint NULL, + pool_id integer NULL, + PRIMARY KEY CLUSTERED (collection_time, id) + ); + IF @debug = 1 BEGIN RAISERROR(''Created table %s for memory logging.'', 0, 1, ''' + @log_table_memory + N''') WITH NOWAIT; END; + END'; + + EXECUTE sys.sp_executesql + @create_sql, + N'@schema_name sysname, + @table_name sysname, + @debug bit', + @log_schema_name, + @log_table_name_prefix, + @debug; + + SET @create_sql = N' + IF NOT EXISTS ( - collection_time datetime2(7) NOT NULL, - total_threads integer NULL, - used_threads integer NULL, - available_threads integer NULL, - reserved_worker_count varchar(10) NULL, - threads_waiting_for_cpu integer NULL, - requests_waiting_for_threads integer NULL, - current_workers integer NULL, - total_active_request_count integer NULL, - total_queued_request_count integer NULL, - total_blocked_task_count integer NULL, - total_active_parallel_thread_count integer NULL, - avg_runnable_tasks_count float NULL, - high_runnable_percent varchar(100) NULL, - cpu_details_output xml NULL, - cpu_utilization_over_threshold xml NULL, - PRIMARY KEY CLUSTERED (collection_time) - ); - IF @debug = 1 BEGIN RAISERROR(''Created table %s for CPU logging.'', 0, 1, ''' + @log_table_cpu + ''') WITH NOWAIT; END; - END'; + SELECT + 1/0 + FROM ' + QUOTENAME(@log_database_name) + N'.sys.tables AS t + JOIN ' + QUOTENAME(@log_database_name) + N'.sys.schemas AS s + ON t.schema_id = s.schema_id + WHERE t.name = @table_name + N''_CPU'' + AND s.name = @schema_name + ) + BEGIN + CREATE TABLE ' + @log_table_cpu + N' + ( + id bigint IDENTITY, + collection_time datetime2(7) NOT NULL DEFAULT SYSDATETIME(), + total_threads integer NULL, + used_threads integer NULL, + available_threads integer NULL, + reserved_worker_count varchar(10) NULL, + threads_waiting_for_cpu integer NULL, + requests_waiting_for_threads integer NULL, + current_workers integer NULL, + total_active_request_count integer NULL, + total_queued_request_count integer NULL, + total_blocked_task_count integer NULL, + total_active_parallel_thread_count integer NULL, + avg_runnable_tasks_count float NULL, + high_runnable_percent varchar(100) NULL, + cpu_details_output xml NULL, + cpu_utilization_over_threshold xml NULL, + PRIMARY KEY CLUSTERED (collection_time, id) + ); + IF @debug = 1 BEGIN RAISERROR(''Created table %s for CPU logging.'', 0, 1, ''' + @log_table_cpu + N''') WITH NOWAIT; END; + END'; EXECUTE sys.sp_executesql @create_sql, @@ -681,211 +746,192 @@ OPTION(MAXDOP 1, RECOMPILE);', @log_table_name_prefix, @debug; - -- Memory Consumers table - SET @create_sql = N' - IF NOT EXISTS - ( - SELECT - 1/0 - FROM ' + QUOTENAME(@log_database_name) + '.sys.tables AS t - JOIN ' + QUOTENAME(@log_database_name) + '.sys.schemas AS s - ON t.schema_id = s.schema_id - WHERE t.name = @table_name + ''_MemoryConsumers'' - AND s.name = @schema_name - ) - BEGIN - CREATE TABLE ' + @log_table_memory_consumers + N' - ( - collection_time datetime2(7) NOT NULL, - memory_source nvarchar(128) NOT NULL, - memory_consumer nvarchar(128) NOT NULL, - memory_consumed_gb decimal(38,2) NULL, - PRIMARY KEY CLUSTERED (collection_time, memory_source, memory_consumer) - ); - IF @debug = 1 BEGIN RAISERROR(''Created table %s for memory consumers logging.'', 0, 1, ''' + @log_database_schema + QUOTENAME(@log_table_name_prefix + '_MemoryConsumers') + ''') WITH NOWAIT; END; - END'; - - EXECUTE sys.sp_executesql - @create_sql, - N'@schema_name sysname, - @table_name sysname, - @debug bit', - @log_schema_name, - @log_table_name_prefix, - @debug; - - -- Memory Query Grants table - SET @create_sql = N' - IF NOT EXISTS - ( - SELECT - 1/0 - FROM ' + QUOTENAME(@log_database_name) + '.sys.tables AS t - JOIN ' + QUOTENAME(@log_database_name) + '.sys.schemas AS s - ON t.schema_id = s.schema_id - WHERE t.name = @table_name + ''_MemoryQueries'' - AND s.name = @schema_name - ) - BEGIN - CREATE TABLE ' + @log_table_memory_queries + N' - ( - collection_time datetime2(7) NOT NULL, - session_id integer NOT NULL, - database_name nvarchar(128) NULL, - duration varchar(30) NULL, - request_time datetime NULL, - grant_time datetime NULL, - wait_time_seconds decimal(38,2) NULL, - requested_memory_gb decimal(38,2) NULL, - granted_memory_gb decimal(38,2) NULL, - used_memory_gb decimal(38,2) NULL, - max_used_memory_gb decimal(38,2) NULL, - ideal_memory_gb decimal(38,2) NULL, - required_memory_gb decimal(38,2) NULL, - queue_id integer NULL, - wait_order integer NULL, - is_next_candidate bit NULL, - wait_type nvarchar(60) NULL, - wait_duration_seconds decimal(38,2) NULL, - dop integer NULL, - reserved_worker_count integer NULL, - used_worker_count integer NULL, - plan_handle varbinary(64) NULL, - sql_text nvarchar(MAX) NULL, - query_plan_xml xml NULL, - PRIMARY KEY CLUSTERED (collection_time, session_id) - ); - IF @debug = 1 BEGIN RAISERROR(''Created table %s for memory queries logging.'', 0, 1, ''' + @log_database_schema + QUOTENAME(@log_table_name_prefix + '_MemoryQueries') + ''') WITH NOWAIT; END; - END'; - - EXECUTE sys.sp_executesql - @create_sql, - N'@schema_name sysname, - @table_name sysname, - @debug bit', - @log_schema_name, - @log_table_name_prefix, - @debug; - - -- CPU Queries table - SET @create_sql = N' - IF NOT EXISTS - ( - SELECT - 1/0 - FROM ' + QUOTENAME(@log_database_name) + '.sys.tables AS t - JOIN ' + QUOTENAME(@log_database_name) + '.sys.schemas AS s - ON t.schema_id = s.schema_id - WHERE t.name = @table_name + ''_CPUQueries'' - AND s.name = @schema_name - ) - BEGIN - CREATE TABLE ' + @log_table_cpu_queries + N' - ( - collection_time datetime2(7) NOT NULL, - session_id integer NOT NULL, - database_name nvarchar(128) NULL, - duration varchar(30) NULL, - status nvarchar(30) NULL, - blocking_session_id integer NULL, - wait_type nvarchar(60) NULL, - wait_time_ms bigint NULL, - wait_resource nvarchar(512) NULL, - cpu_time_ms bigint NULL, - total_elapsed_time_ms bigint NULL, - reads bigint NULL, - writes bigint NULL, - logical_reads bigint NULL, - granted_query_memory_gb decimal(38,2) NULL, - transaction_isolation_level nvarchar(30) NULL, - dop integer NULL, - parallel_worker_count integer NULL, - plan_handle varbinary(64) NULL, - sql_text nvarchar(MAX) NULL, - query_plan_xml xml NULL, - PRIMARY KEY CLUSTERED (collection_time, session_id) - ); - IF @debug = 1 BEGIN RAISERROR(''Created table %s for CPU queries logging.'', 0, 1, ''' + @log_database_schema + QUOTENAME(@log_table_name_prefix + '_CPUQueries') + ''') WITH NOWAIT; END; - END'; - - EXECUTE sys.sp_executesql - @create_sql, - N'@schema_name sysname, - @table_name sysname, - @debug bit', - @log_schema_name, - @log_table_name_prefix, - @debug; - - -- Low Memory Events table - SET @create_sql = N' - IF NOT EXISTS - ( - SELECT - 1/0 - FROM ' + QUOTENAME(@log_database_name) + '.sys.tables AS t - JOIN ' + QUOTENAME(@log_database_name) + '.sys.schemas AS s - ON t.schema_id = s.schema_id - WHERE t.name = @table_name + ''_LowMemoryEvents'' - AND s.name = @schema_name - ) - BEGIN - CREATE TABLE ' + @log_table_low_memory_events + N' - ( - collection_time datetime2(7) NOT NULL, - sample_time datetime NOT NULL, - notification_type varchar(50) NULL, - indicators_process integer NULL, - indicators_system integer NULL, - physical_memory_available_gb decimal(38,2) NULL, - virtual_memory_available_gb decimal(38,2) NULL, - PRIMARY KEY CLUSTERED (collection_time, sample_time) - ); - IF @debug = 1 BEGIN RAISERROR(''Created table %s for low memory events logging.'', 0, 1, ''' + @log_database_schema + QUOTENAME(@log_table_name_prefix + '_LowMemoryEvents') + ''') WITH NOWAIT; END; - END'; - - EXECUTE sys.sp_executesql - @create_sql, - N'@schema_name sysname, - @table_name sysname, - @debug bit', - @log_schema_name, - @log_table_name_prefix, - @debug; - - -- CPU Utilization Events table - SET @create_sql = N' - IF NOT EXISTS - ( - SELECT - 1/0 - FROM ' + QUOTENAME(@log_database_name) + '.sys.tables AS t - JOIN ' + QUOTENAME(@log_database_name) + '.sys.schemas AS s - ON t.schema_id = s.schema_id - WHERE t.name = @table_name + ''_CPUEvents'' - AND s.name = @schema_name - ) - BEGIN - CREATE TABLE ' + @log_table_cpu_events + N' - ( - collection_time datetime2(7) NOT NULL, - sample_time datetime NOT NULL, - sqlserver_cpu_utilization integer NULL, - other_process_cpu_utilization integer NULL, - total_cpu_utilization integer NULL, - PRIMARY KEY CLUSTERED (collection_time, sample_time) - ); - IF @debug = 1 BEGIN RAISERROR(''Created table %s for CPU utilization events logging.'', 0, 1, ''' + @log_database_schema + QUOTENAME(@log_table_name_prefix + '_CPUEvents') + ''') WITH NOWAIT; END; - END'; + /* Memory Consumers table */ + SET @create_sql = N' + IF NOT EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@log_database_name) + N'.sys.tables AS t + JOIN ' + QUOTENAME(@log_database_name) + N'.sys.schemas AS s + ON t.schema_id = s.schema_id + WHERE t.name = @table_name + N''_MemoryConsumers'' + AND s.name = @schema_name + ) + BEGIN + CREATE TABLE ' + @log_table_memory_consumers + N' + ( + id bigint IDENTITY, + collection_time datetime2(7) NOT NULL DEFAULT SYSDATETIME(), + memory_source nvarchar(128) NOT NULL, + memory_consumer nvarchar(128) NOT NULL, + memory_consumed_gb decimal(38,2) NULL, + PRIMARY KEY CLUSTERED (collection_time, id) + ); + IF @debug = 1 BEGIN RAISERROR(''Created table %s for memory consumers logging.'', 0, 1, ''' + @log_database_schema + QUOTENAME(@log_table_name_prefix + N'_MemoryConsumers') + N''') WITH NOWAIT; END; + END'; + + EXECUTE sys.sp_executesql + @create_sql, + N'@schema_name sysname, + @table_name sysname, + @debug bit', + @log_schema_name, + @log_table_name_prefix, + @debug; + + /* Memory Query Grants table */ + SET @create_sql = N' + IF NOT EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@log_database_name) + N'.sys.tables AS t + JOIN ' + QUOTENAME(@log_database_name) + N'.sys.schemas AS s + ON t.schema_id = s.schema_id + WHERE t.name = @table_name + N''_MemoryQueries'' + AND s.name = @schema_name + ) + BEGIN + CREATE TABLE ' + @log_table_memory_queries + N' + ( + id bigint IDENTITY, + collection_time datetime2(7) NOT NULL DEFAULT SYSDATETIME(), + session_id integer NOT NULL, + database_name nvarchar(128) NULL, + duration varchar(30) NULL, + request_time datetime NULL, + grant_time datetime NULL, + wait_time_seconds decimal(38,2) NULL, + requested_memory_gb decimal(38,2) NULL, + granted_memory_gb decimal(38,2) NULL, + used_memory_gb decimal(38,2) NULL, + max_used_memory_gb decimal(38,2) NULL, + ideal_memory_gb decimal(38,2) NULL, + required_memory_gb decimal(38,2) NULL, + queue_id integer NULL, + wait_order integer NULL, + is_next_candidate bit NULL, + wait_type nvarchar(60) NULL, + wait_duration_seconds decimal(38,2) NULL, + dop integer NULL, + reserved_worker_count integer NULL, + used_worker_count integer NULL, + plan_handle varbinary(64) NULL, + sql_text xml NULL, + query_plan_xml xml NULL, + live_query_plan xml NULL + PRIMARY KEY CLUSTERED (collection_time, id) + ); + IF @debug = 1 BEGIN RAISERROR(''Created table %s for memory queries logging.'', 0, 1, ''' + @log_database_schema + QUOTENAME(@log_table_name_prefix + N'_MemoryQueries') + N''') WITH NOWAIT; END; + END'; + + EXECUTE sys.sp_executesql + @create_sql, + N'@schema_name sysname, + @table_name sysname, + @debug bit', + @log_schema_name, + @log_table_name_prefix, + @debug; - EXECUTE sys.sp_executesql - @create_sql, - N'@schema_name sysname, - @table_name sysname, - @debug bit', - @log_schema_name, - @log_table_name_prefix, - @debug; + /* CPU Queries table */ + SET @create_sql = N' + IF NOT EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@log_database_name) + N'.sys.tables AS t + JOIN ' + QUOTENAME(@log_database_name) + N'.sys.schemas AS s + ON t.schema_id = s.schema_id + WHERE t.name = @table_name + N''_CPUQueries'' + AND s.name = @schema_name + ) + BEGIN + CREATE TABLE ' + @log_table_cpu_queries + N' + ( + id bigint IDENTITY, + collection_time datetime2(7) NOT NULL DEFAULT SYSDATETIME(), + session_id integer NOT NULL, + database_name nvarchar(128) NULL, + duration varchar(30) NULL, + status nvarchar(30) NULL, + blocking_session_id integer NULL, + wait_type nvarchar(60) NULL, + wait_time_ms bigint NULL, + wait_resource nvarchar(512) NULL, + cpu_time_ms bigint NULL, + total_elapsed_time_ms bigint NULL, + reads bigint NULL, + writes bigint NULL, + logical_reads bigint NULL, + granted_query_memory_gb decimal(38,2) NULL, + transaction_isolation_level nvarchar(30) NULL, + dop integer NULL, + parallel_worker_count integer NULL, + plan_handle varbinary(64) NULL, + sql_text xml NULL, + query_plan_xml xml NULL, + live_query_plan xml NULL, + statement_start_offset integer NULL, + statement_end_offset integer NULL, + PRIMARY KEY CLUSTERED (collection_time, id) + ); + IF @debug = 1 BEGIN RAISERROR(''Created table %s for CPU queries logging.'', 0, 1, ''' + @log_database_schema + QUOTENAME(@log_table_name_prefix + N'_CPUQueries') + N''') WITH NOWAIT; END; + END'; + + EXECUTE sys.sp_executesql + @create_sql, + N'@schema_name sysname, + @table_name sysname, + @debug bit', + @log_schema_name, + @log_table_name_prefix, + @debug; + + EXECUTE sys.sp_executesql + @create_sql, + N'@schema_name sysname, + @table_name sysname, + @debug bit', + @log_schema_name, + @log_table_name_prefix, + @debug; + + /* CPU Utilization Events table */ + SET @create_sql = N' + IF NOT EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@log_database_name) + N'.sys.tables AS t + JOIN ' + QUOTENAME(@log_database_name) + N'.sys.schemas AS s + ON t.schema_id = s.schema_id + WHERE t.name = @table_name + N''_CPUEvents'' + AND s.name = @schema_name + ) + BEGIN + CREATE TABLE ' + @log_table_cpu_events + N' + ( + id bigint IDENTITY, + collection_time datetime2(7) NOT NULL DEFAULT SYSDATETIME(), + sample_time datetime NULL, + sqlserver_cpu_utilization integer NULL, + other_process_cpu_utilization integer NULL, + total_cpu_utilization integer NULL, + PRIMARY KEY CLUSTERED (collection_time, id) + ); + IF @debug = 1 BEGIN RAISERROR(''Created table %s for CPU utilization events logging.'', 0, 1, ''' + @log_database_schema + QUOTENAME(@log_table_name_prefix + N'_CPUEvents') + N''') WITH NOWAIT; END; + END'; + + EXECUTE sys.sp_executesql + @create_sql, + N'@schema_name sysname, + @table_name sysname, + @debug bit', + @log_schema_name, + @log_table_name_prefix, + @debug; END; /*End log to tables validation checks here*/ DECLARE @@ -1383,12 +1429,57 @@ OPTION(MAXDOP 1, RECOMPILE);', wait_time_seconds DESC OPTION(MAXDOP 1, RECOMPILE); END; + END; IF @log_to_table = 1 BEGIN - SELECT 1 + + SELECT + w.* + INTO #waits + FROM @waits AS w + OPTION(RECOMPILE); + + SET @insert_sql = N' + SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; + INSERT INTO ' + @log_table_waits + N' + ( + hours_uptime, + hours_cpu_time, + wait_type, + description, + hours_wait_time, + avg_ms_per_wait, + percent_signal_waits, + waiting_tasks_count, + sample_time, + sorting + ) + SELECT + w.hours_uptime, + w.hours_cpu_time, + w.wait_type, + w.description, + w.hours_wait_time, + w.avg_ms_per_wait, + w.percent_signal_waits, + w.waiting_tasks_count_n, + w.sample_time, + w.sorting + FROM #waits AS w; + '; + + IF @debug = 1 + BEGIN + PRINT @insert_sql; + END; + + EXECUTE sys.sp_executesql + @insert_sql; + + DROP TABLE IF EXISTS + #waits; END; - END END; /*End wait stats*/ /* This section looks at disk metrics @@ -1547,7 +1638,7 @@ OPTION(MAXDOP 1, RECOMPILE);', JOIN ' + CONVERT ( - nvarchar(MAX), + nvarchar(max), CASE WHEN @azure = 1 THEN N'sys.database_files AS f @@ -1596,7 +1687,7 @@ OPTION(MAXDOP 1, RECOMPILE);', @disk_check; IF @log_to_table = 0 - BEGIN + BEGIN IF @sample_seconds = 0 BEGIN WITH @@ -1852,11 +1943,68 @@ OPTION(MAXDOP 1, RECOMPILE);', f.total_avg_stall DESC OPTION(MAXDOP 1, RECOMPILE); END; - END + END; - IF @log_to_table = 1 + IF @log_to_table = 1 BEGIN - SELECT 1 + + SELECT + fm.* + INTO #file_metrics + FROM @file_metrics AS fm + OPTION(RECOMPILE); + + SET @insert_sql = N' + SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; + INSERT INTO ' + @log_table_file_metrics + N' + ( + hours_uptime, + drive, + database_name, + database_file_details, + file_size_gb, + total_gb_read, + total_mb_read, + total_read_count, + avg_read_stall_ms, + total_gb_written, + total_mb_written, + total_write_count, + avg_write_stall_ms, + io_stall_read_ms, + io_stall_write_ms, + sample_time + ) + SELECT + fm.hours_uptime, + fm.drive, + fm.database_name, + fm.database_file_details, + fm.file_size_gb, + fm.total_gb_read, + fm.total_mb_read, + fm.total_read_count, + fm.avg_read_stall_ms, + fm.total_gb_written, + fm.total_mb_written, + fm.total_write_count, + fm.avg_write_stall_ms, + fm.io_stall_read_ms, + fm.io_stall_write_ms, + fm.sample_time + FROM #file_metrics AS fm; + '; + + IF @debug = 1 + BEGIN + PRINT @insert_sql; + END; + + EXECUTE sys.sp_executesql + @insert_sql; + + DROP TABLE IF EXISTS + #file_metrics; END; END; /*End file stats*/ @@ -2052,7 +2200,46 @@ OPTION(MAXDOP 1, RECOMPILE);', IF @log_to_table = 1 BEGIN - SELECT 1 + + SELECT + dopc.* + INTO #dm_os_performance_counters + FROM @dm_os_performance_counters AS dopc + OPTION(RECOMPILE); + + SET @insert_sql = N' + SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; + INSERT INTO ' + @log_table_perfmon + N' + ( + sample_time, + object_name, + counter_name, + counter_name_clean, + instance_name, + cntr_value, + cntr_type + ) + SELECT + dopc.sample_time, + dopc.object_name, + dopc.counter_name, + dopc.counter_name_clean, + dopc.instance_name, + dopc.cntr_value, + dopc.cntr_type + FROM #dm_os_performance_counters AS dopc; + '; + + IF @debug = 1 + BEGIN + PRINT @insert_sql; + END; + + EXECUTE sys.sp_executesql + @insert_sql; + + DROP TABLE IF EXISTS + #dm_os_performance_counters; END; END; /*End Perfmon*/ @@ -2231,7 +2418,7 @@ OPTION(MAXDOP 1, RECOMPILE);', ' + CONVERT ( - nvarchar(MAX), + nvarchar(max), CASE @pages_kb WHEN 1 THEN @@ -2240,7 +2427,7 @@ OPTION(MAXDOP 1, RECOMPILE);', N'domc.single_pages_kb + domc.multi_pages_kb + ' END - ) + ) + N' domc.virtual_memory_committed_kb + domc.awe_allocated_kb + @@ -2293,7 +2480,10 @@ OPTION(MAXDOP 1, RECOMPILE);', decimal(38, 2), SUM ( - ' + CASE @pages_kb + ' + CONVERT + ( + nvarchar(max), + CASE @pages_kb WHEN 1 THEN N' domc.pages_kb ' @@ -2318,7 +2508,7 @@ OPTION(MAXDOP 1, RECOMPILE);', ELSE N'domc.single_pages_kb + domc.multi_pages_kb ' - END + N' + END ) + N' ) / 1024. / 1024. > 0. ORDER BY memory_used_gb DESC @@ -2339,7 +2529,29 @@ OPTION(MAXDOP 1, RECOMPILE);', IF @log_to_table = 1 BEGIN - SELECT 1 + SET @insert_sql = N' + SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; + INSERT INTO ' + @log_table_memory_consumers + N' + ( + memory_source, + memory_consumer, + memory_consumed_gb + ) + ' + + REPLACE + ( + @pool_sql, + N'SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;', + N'' + ); + + IF @debug = 1 + BEGIN + PRINT @insert_sql; + END; + + EXECUTE sys.sp_executesql + @insert_sql; END; /*Checking total database size*/ @@ -2381,7 +2593,7 @@ OPTION(MAXDOP 1, RECOMPILE);', EXECUTE sys.sp_executesql @database_size_out, - N'@database_size_out_gb varchar(10) OUTPUT', + N'@database_size_out_gb nvarchar(10) OUTPUT', @database_size_out_gb OUTPUT; /*Check physical memory in the server*/ @@ -2601,7 +2813,7 @@ OPTION(MAXDOP 1, RECOMPILE);', IF @debug = 1 BEGIN - RAISERROR('%s', 0, 1, @cache_sql) WITH NOWAIT; + PRINT @cache_sql; END; IF @log_to_table = 0 @@ -2611,10 +2823,6 @@ OPTION(MAXDOP 1, RECOMPILE);', N'@cache_xml xml OUTPUT', @cache_xml OUTPUT; END; - IF @log_to_table = 1 - BEGIN - SELECT 1 - END; IF @cache_xml IS NULL BEGIN @@ -2637,10 +2845,6 @@ OPTION(MAXDOP 1, RECOMPILE);', cache_memory = @cache_xml; END; - IF @log_to_table = 1 - BEGIN - SELECT 1 - END; SELECT @memory_grant_cap = @@ -2687,94 +2891,143 @@ OPTION(MAXDOP 1, RECOMPILE);', ); END; + SELECT + @resource_semaphores += N' + SELECT + deqrs.resource_semaphore_id, + total_database_size_gb = + @database_size_out_gb, + total_physical_memory_gb = + @total_physical_memory_gb, + max_server_memory_gb = + ( + SELECT + CONVERT + ( + bigint, + c.value_in_use + ) + FROM sys.configurations AS c + WHERE c.name = N''max server memory (MB)'' + ) / 1024, + max_memory_grant_cap = + @memory_grant_cap, + memory_model = + ( + SELECT + osi.sql_memory_model_desc + FROM sys.dm_os_sys_info AS osi + ), + target_memory_gb = + CONVERT + ( + decimal(38, 2), + (deqrs.target_memory_kb / 1024. / 1024.) + ), + max_target_memory_gb = + CONVERT( + decimal(38, 2), + (deqrs.max_target_memory_kb / 1024. / 1024.) + ), + total_memory_gb = + CONVERT + ( + decimal(38, 2), + (deqrs.total_memory_kb / 1024. / 1024.) + ), + available_memory_gb = + CONVERT + ( + decimal(38, 2), + (deqrs.available_memory_kb / 1024. / 1024.) + ), + granted_memory_gb = + CONVERT + ( + decimal(38, 2), + (deqrs.granted_memory_kb / 1024. / 1024.) + ), + used_memory_gb = + CONVERT + ( + decimal(38, 2), + (deqrs.used_memory_kb / 1024. / 1024.) + ), + deqrs.grantee_count, + deqrs.waiter_count, + deqrs.timeout_error_count, + deqrs.forced_grant_count, + wg.total_reduced_memory_grant_count, + deqrs.pool_id + FROM sys.dm_exec_query_resource_semaphores AS deqrs + CROSS APPLY + ( + SELECT TOP (1) + total_reduced_memory_grant_count = + wg.total_reduced_memgrant_count + FROM sys.dm_resource_governor_workload_groups AS wg + WHERE wg.pool_id = deqrs.pool_id + ORDER BY + wg.total_reduced_memgrant_count DESC + ) AS wg + WHERE deqrs.max_target_memory_kb IS NOT NULL + ORDER BY + deqrs.pool_id + OPTION(MAXDOP 1, RECOMPILE); + '; + IF @log_to_table = 0 BEGIN - SELECT - deqrs.resource_semaphore_id, - total_database_size_gb = - @database_size_out_gb, - total_physical_memory_gb = - @total_physical_memory_gb, - max_server_memory_gb = - ( - SELECT - CONVERT - ( - bigint, - c.value_in_use - ) - FROM sys.configurations AS c - WHERE c.name = N'max server memory (MB)' - ) / 1024, - max_memory_grant_cap = - @memory_grant_cap, - memory_model = - ( - SELECT - osi.sql_memory_model_desc - FROM sys.dm_os_sys_info AS osi - ), - target_memory_gb = - CONVERT - ( - decimal(38, 2), - (deqrs.target_memory_kb / 1024. / 1024.) - ), - max_target_memory_gb = - CONVERT( - decimal(38, 2), - (deqrs.max_target_memory_kb / 1024. / 1024.) - ), - total_memory_gb = - CONVERT - ( - decimal(38, 2), - (deqrs.total_memory_kb / 1024. / 1024.) - ), - available_memory_gb = - CONVERT - ( - decimal(38, 2), - (deqrs.available_memory_kb / 1024. / 1024.) - ), - granted_memory_gb = - CONVERT - ( - decimal(38, 2), - (deqrs.granted_memory_kb / 1024. / 1024.) - ), - used_memory_gb = - CONVERT - ( - decimal(38, 2), - (deqrs.used_memory_kb / 1024. / 1024.) - ), - deqrs.grantee_count, - deqrs.waiter_count, - deqrs.timeout_error_count, - deqrs.forced_grant_count, - wg.total_reduced_memory_grant_count, - deqrs.pool_id - FROM sys.dm_exec_query_resource_semaphores AS deqrs - CROSS APPLY - ( - SELECT TOP (1) - total_reduced_memory_grant_count = - wg.total_reduced_memgrant_count - FROM sys.dm_resource_governor_workload_groups AS wg - WHERE wg.pool_id = deqrs.pool_id - ORDER BY - wg.total_reduced_memgrant_count DESC - ) AS wg - WHERE deqrs.max_target_memory_kb IS NOT NULL - ORDER BY - deqrs.pool_id - OPTION(MAXDOP 1, RECOMPILE); + EXECUTE sys.sp_executesql + @resource_semaphores, + N'@database_size_out_gb nvarchar(10), + @total_physical_memory_gb bigint, + @memory_grant_cap xml', + @database_size_out_gb, + @total_physical_memory_gb, + @memory_grant_cap; END IF @log_to_table = 1 BEGIN - SELECT 1 + SET @insert_sql = N' + SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; + INSERT INTO ' + @log_table_memory + N' + ( + resource_semaphore_id, + total_database_size_gb, + total_physical_memory_gb, + max_server_memory_gb, + max_memory_grant_cap, + memory_model, + target_memory_gb, + max_target_memory_gb, + total_memory_gb, + available_memory_gb, + granted_memory_gb, + used_memory_gb, + grantee_count, + waiter_count, + timeout_error_count, + forced_grant_count, + total_reduced_memory_grant_count, + pool_id + )' + + @resource_semaphores; + + IF @debug = 1 + BEGIN + PRINT @insert_sql; + END; + + EXECUTE sys.sp_executesql + @insert_sql, + N'@database_size_out_gb nvarchar(10), + @total_physical_memory_gb bigint, + @memory_grant_cap xml', + @database_size_out_gb, + @total_physical_memory_gb, + @memory_grant_cap; END; END; /*End memory checks*/ @@ -2890,7 +3143,7 @@ OPTION(MAXDOP 1, RECOMPILE);', ),' + CONVERT ( - nvarchar(MAX), + nvarchar(max), CASE WHEN @skip_plan_xml = 0 THEN N' @@ -3003,7 +3256,69 @@ OPTION(MAXDOP 1, RECOMPILE);', IF @log_to_table = 1 BEGIN - SELECT 1 + SET @insert_sql = N' + SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; + INSERT INTO ' + @log_table_memory_queries + N' + ( + session_id, + database_name, + duration, + sql_text, + query_plan_xml' + + CASE + WHEN @live_plans = 1 + THEN N', + live_query_plan' + ELSE N'' + END + N', + request_time, + grant_time, + wait_time_seconds, + requested_memory_gb, + granted_memory_gb, + used_memory_gb, + max_used_memory_gb, + ideal_memory_gb, + required_memory_gb, + queue_id, + wait_order, + is_next_candidate, + wait_type, + wait_duration_seconds, + dop' + + CASE + WHEN @helpful_new_columns = 1 + THEN N', + reserved_worker_count, + used_worker_count' + ELSE N'' + END + N', + plan_handle + ) ' + + REPLACE + ( + REPLACE + ( + REPLACE + ( + @mem_sql, + N'SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;', + N'' + ), + N'SET LOCK_TIMEOUT 1000;', + N'' + ), + N'SET LOCK_TIMEOUT -1;', + N'' + ); + + IF @debug = 1 + BEGIN + PRINT @insert_sql; + END; + + EXECUTE sys.sp_executesql + @insert_sql; END; END; @@ -3165,107 +3480,170 @@ OPTION(MAXDOP 1, RECOMPILE);', END; IF @log_to_table = 1 BEGIN - SELECT 1 + SET @insert_sql = N' + SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; + INSERT INTO ' + @log_table_cpu_events + N' + ( + sample_time, + sqlserver_cpu_utilization, + other_process_cpu_utilization, + total_cpu_utilization + ) + SELECT + sample_time = event.value(''(./sample_time)[1]'', ''datetime''), + sqlserver_cpu_utilization = event.value(''(./sqlserver_cpu_utilization)[1]'', ''integer''), + other_process_cpu_utilization = event.value(''(./other_process_cpu_utilization)[1]'', ''integer''), + total_cpu_utilization = event.value(''(./total_cpu_utilization)[1]'', ''integer'') + FROM @cpu_utilization.nodes(''/cpu_utilization'') AS cpu(event);'; + + IF @debug = 1 + BEGIN + PRINT @insert_sql; + END; + + EXECUTE sys.sp_executesql + @insert_sql, + N'@cpu_utilization xml', + @cpu_utilization; END; /*Thread usage*/ - IF @log_to_table = 0 - BEGIN + SELECT + @cpu_threads += N' + SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; + SELECT + total_threads = + MAX(osi.max_workers_count), + used_threads = + SUM(dos.active_workers_count), + available_threads = + MAX(osi.max_workers_count) - SUM(dos.active_workers_count), + reserved_worker_count = ' + + CASE @helpful_new_columns + WHEN 1 + THEN ISNULL + ( + @reserved_worker_count_out, + N'0' + ) + ELSE N'N/A' + END + N', + threads_waiting_for_cpu = + SUM(dos.runnable_tasks_count), + requests_waiting_for_threads = + SUM(dos.work_queue_count), + current_workers = + SUM(dos.current_workers_count), + total_active_request_count = + MAX(wg.active_request_count), + total_queued_request_count = + MAX(wg.queued_request_count), + total_blocked_task_count = + MAX(wg.blocked_task_count), + total_active_parallel_thread_count = + MAX(wg.active_parallel_thread_count), + avg_runnable_tasks_count = + AVG(dos.runnable_tasks_count), + high_runnable_percent = + MAX(ISNULL(r.high_runnable_percent, 0)) + FROM sys.dm_os_schedulers AS dos + CROSS JOIN sys.dm_os_sys_info AS osi + CROSS JOIN + ( + SELECT + active_request_count = SUM(wg.active_request_count), + queued_request_count = SUM(wg.queued_request_count), + blocked_task_count = SUM(wg.blocked_task_count), + active_parallel_thread_count = SUM(wg.active_parallel_thread_count) + FROM sys.dm_resource_governor_workload_groups AS wg + ) AS wg + OUTER APPLY + ( SELECT - total_threads = - MAX(osi.max_workers_count), - used_threads = - SUM(dos.active_workers_count), - available_threads = - MAX(osi.max_workers_count) - SUM(dos.active_workers_count), - reserved_worker_count = - CASE @helpful_new_columns - WHEN 1 - THEN ISNULL - ( - @reserved_worker_count_out, - N'0' - ) - ELSE N'N/A' - END, - threads_waiting_for_cpu = - SUM(dos.runnable_tasks_count), - requests_waiting_for_threads = - SUM(dos.work_queue_count), - current_workers = - SUM(dos.current_workers_count), - total_active_request_count = - MAX(wg.active_request_count), - total_queued_request_count = - MAX(wg.queued_request_count), - total_blocked_task_count = - MAX(wg.blocked_task_count), - total_active_parallel_thread_count = - MAX(wg.active_parallel_thread_count), - avg_runnable_tasks_count = - AVG(dos.runnable_tasks_count), high_runnable_percent = - MAX(ISNULL(r.high_runnable_percent, 0)) - FROM sys.dm_os_schedulers AS dos - CROSS JOIN sys.dm_os_sys_info AS osi - CROSS JOIN - ( - SELECT - active_request_count = SUM(wg.active_request_count), - queued_request_count = SUM(wg.queued_request_count), - blocked_task_count = SUM(wg.blocked_task_count), - active_parallel_thread_count = SUM(wg.active_parallel_thread_count) - FROM sys.dm_resource_governor_workload_groups AS wg - ) AS wg - OUTER APPLY + '''' + + RTRIM(y.runnable_pct) + + ''% of '' + + RTRIM(y.total) + + '' queries are waiting to get on a CPU.'' + FROM ( SELECT - high_runnable_percent = - '' + - RTRIM(y.runnable_pct) + - '% of ' + - RTRIM(y.total) + - ' queries are waiting to get on a CPU.' + x.total, + x.runnable, + runnable_pct = + CONVERT + ( + decimal(38,2), + ( + x.runnable / (1. * NULLIF(x.total, 0)) + ) + ) * 100. FROM ( SELECT - x.total, - x.runnable, - runnable_pct = - CONVERT + total = + COUNT_BIG(*), + runnable = + SUM ( - decimal(38,2), - ( - x.runnable / (1. * NULLIF(x.total, 0)) - ) - ) * 100. - FROM - ( - SELECT - total = - COUNT_BIG(*), - runnable = - SUM - ( - CASE - WHEN der.status = N'runnable' - THEN 1 - ELSE 0 - END - ) - FROM sys.dm_exec_requests AS der - WHERE der.session_id > 50 - ) AS x - ) AS y - WHERE y.runnable_pct >= 10 - AND y.total >= 4 - ) AS r - WHERE dos.status = N'VISIBLE ONLINE' - OPTION(MAXDOP 1, RECOMPILE); + CASE + WHEN der.status = N''runnable'' + THEN 1 + ELSE 0 + END + ) + FROM sys.dm_exec_requests AS der + WHERE der.session_id > 50 + ) AS x + ) AS y + WHERE y.runnable_pct >= 10 + AND y.total >= 4 + ) AS r + WHERE dos.status = N''VISIBLE ONLINE'' + OPTION(MAXDOP 1, RECOMPILE); + '; + + IF @log_to_table = 0 + BEGIN + EXECUTE sys.sp_executesql + @cpu_threads; END; + IF @log_to_table = 1 BEGIN - SELECT 1; + SET @insert_sql = N' + SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; + INSERT INTO ' + @log_table_cpu + N' + ( + total_threads, + used_threads, + available_threads, + reserved_worker_count, + threads_waiting_for_cpu, + requests_waiting_for_threads, + current_workers, + total_active_request_count, + total_queued_request_count, + total_blocked_task_count, + total_active_parallel_thread_count, + avg_runnable_tasks_count, + high_runnable_percent + )' + + REPLACE + ( + @cpu_threads, + N'SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;', + N'' + ); + + IF @debug = 1 + BEGIN + PRINT @insert_sql; + END; + + EXECUTE sys.sp_executesql + @insert_sql; END; @@ -3424,7 +3802,7 @@ OPTION(MAXDOP 1, RECOMPILE);', + CONVERT ( - nvarchar(MAX), + nvarchar(max), CASE WHEN @skip_plan_xml = 0 THEN N' @@ -3460,7 +3838,7 @@ OPTION(MAXDOP 1, RECOMPILE);', ) + CONVERT ( - nvarchar(MAX), + nvarchar(max), N' statement_start_offset = (der.statement_start_offset / 2) + 1, @@ -3527,7 +3905,7 @@ OPTION(MAXDOP 1, RECOMPILE);', WHEN @cool_new_columns = 1 THEN CONVERT ( - nvarchar(MAX), + nvarchar(max), N', der.dop, der.parallel_worker_count' @@ -3536,7 +3914,7 @@ OPTION(MAXDOP 1, RECOMPILE);', END + CONVERT ( - nvarchar(MAX), + nvarchar(max), N' FROM sys.dm_exec_requests AS der OUTER APPLY sys.dm_exec_sql_text(der.plan_handle) AS dest @@ -3586,7 +3964,68 @@ OPTION(MAXDOP 1, RECOMPILE);', IF @log_to_table = 1 BEGIN - SELECT 1; + SET @insert_sql = N' + SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; + INSERT INTO ' + @log_table_cpu_queries + N' + ( + session_id, + database_name, + duration, + sql_text, + query_plan_xml' + + CASE + WHEN @live_plans = 1 + THEN N', + live_query_plan' + ELSE N'' + END + N', + statement_start_offset, + statement_end_offset, + plan_handle, + status, + blocking_session_id, + wait_type, + wait_time_ms, + wait_resource, + cpu_time_ms, + total_elapsed_time_ms, + reads, + writes, + logical_reads, + granted_query_memory_gb, + transaction_isolation_level' + + CASE + WHEN @cool_new_columns = 1 + THEN N', + dop, + parallel_worker_count' + ELSE N'' + END + N' + )' + + REPLACE + ( + REPLACE + ( + REPLACE + ( + @cpu_sql, + N'SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;', + N'' + ), + N'SET LOCK_TIMEOUT 1000;', + N'' + ), + N'SET LOCK_TIMEOUT -1;', + N'' + ); + + IF @debug = 1 + BEGIN + PRINT @insert_sql; + END; + + EXECUTE sys.sp_executesql + @insert_sql; END; END; /*End not skipping queries*/ END; /*End CPU checks*/ @@ -3721,6 +4160,38 @@ OPTION(MAXDOP 1, RECOMPILE);', memory_grant_cap = @memory_grant_cap; + SELECT + pattern = + 'logging parameters', + log_to_table = + @log_to_table, + log_database_name = + @log_database_name, + log_schema_name = + @log_schema_name, + log_table_name_prefix = + @log_table_name_prefix, + log_database_schema = + @log_database_schema, + log_table_waits = + @log_table_waits, + log_table_file_metrics = + @log_table_file_metrics, + log_table_perfmon = + @log_table_perfmon, + log_table_memory = + @log_table_memory, + log_table_cpu = + @log_table_cpu, + log_table_memory_consumers = + @log_table_memory_consumers, + log_table_memory_queries = + @log_table_memory_queries, + log_table_cpu_queries = + @log_table_cpu_queries, + log_table_cpu_events = + @log_table_cpu_events; + END; /*End Debug*/ END; /*Final End*/ GO From 66cebf126ac80712658336f059ffa032c745ab2e Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 10 Mar 2025 09:52:25 -0400 Subject: [PATCH 017/246] Update sp_HealthParser.sql Add table logging to HealthParser --- sp_HealthParser/sp_HealthParser.sql | 2200 +++++++++++++++++++++++---- 1 file changed, 1929 insertions(+), 271 deletions(-) diff --git a/sp_HealthParser/sp_HealthParser.sql b/sp_HealthParser/sp_HealthParser.sql index d1395820..70338b9e 100644 --- a/sp_HealthParser/sp_HealthParser.sql +++ b/sp_HealthParser/sp_HealthParser.sql @@ -50,6 +50,11 @@ ALTER PROCEDURE @wait_round_interval_minutes bigint = 60, /*Nearest interval to round wait stats to*/ @skip_locks bit = 0, /*Skip the blocking and deadlocks*/ @pending_task_threshold integer = 10, /*Minimum number of pending tasks to care about*/ + @log_to_table bit = 0, /*enable logging to permanent tables*/ + @log_database_name sysname = NULL, /*database to store logging tables*/ + @log_schema_name sysname = NULL, /*schema to store logging tables*/ + @log_table_name_prefix sysname = 'HealthParser', /*prefix for all logging tables*/ + @log_retention_days integer = 30, /*Number of days to keep logs, 0 = keep indefinitely*/ @debug bit = 0, /*Select from temp tables to get event data in raw xml*/ @help bit = 0, /*Get help*/ @version varchar(30) = NULL OUTPUT, /*Script version*/ @@ -97,6 +102,11 @@ BEGIN WHEN N'@wait_round_interval_minutes' THEN N'interval to round minutes to for wait stats' WHEN N'@skip_locks' THEN N'skip the blocking and deadlocking section' WHEN N'@pending_task_threshold' THEN N'minimum number of pending tasks to display' + WHEN N'@log_to_table' THEN N'enable logging to permanent tables instead of returning results' + WHEN N'@log_database_name' THEN N'database to store logging tables' + WHEN N'@log_schema_name' THEN N'schema to store logging tables' + WHEN N'@log_table_name_prefix' THEN N'prefix for all logging tables' + WHEN N'@log_retention_days' THEN N'how many days of data to retain' WHEN N'@version' THEN N'OUTPUT; for support' WHEN N'@version_date' THEN N'OUTPUT; for support' WHEN N'@help' THEN N'how you got here' @@ -114,6 +124,11 @@ BEGIN WHEN N'@wait_round_interval_minutes' THEN N'interval to round minutes to for top wait stats by count and duration' WHEN N'@skip_locks' THEN N'0 or 1' WHEN N'@pending_task_threshold' THEN N'a valid integer' + WHEN N'@log_to_table' THEN N'0 or 1' + WHEN N'@log_database_name' THEN N'any valid database name' + WHEN N'@log_schema_name' THEN N'any valid schema name' + WHEN N'@log_table_name_prefix' THEN N'any valid identifier' + WHEN N'@log_retention_days' THEN N'a positive integer' WHEN N'@version' THEN N'none' WHEN N'@version_date' THEN N'none' WHEN N'@help' THEN N'0 or 1' @@ -131,15 +146,19 @@ BEGIN WHEN N'@wait_round_interval_minutes' THEN N'60' WHEN N'@skip_locks' THEN N'0' WHEN N'@pending_task_threshold' THEN N'10' + WHEN N'@log_to_table' THEN N'0' + WHEN N'@log_database_name' THEN N'NULL (current database)' + WHEN N'@log_schema_name' THEN N'NULL (dbo)' + WHEN N'@log_table_name_prefix' THEN N'HealthParser' WHEN N'@version' THEN N'none; OUTPUT' WHEN N'@version_date' THEN N'none; OUTPUT' WHEN N'@help' THEN N'0' WHEN N'@debug' THEN N'0' END FROM sys.all_parameters AS ap - INNER JOIN sys.all_objects AS o + JOIN sys.all_objects AS o ON ap.object_id = o.object_id - INNER JOIN sys.types AS t + JOIN sys.types AS t ON ap.system_type_id = t.system_type_id AND ap.user_type_id = t.user_type_id WHERE o.name = N'sp_HealthParser' @@ -222,7 +241,27 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. @object_name sysname, @temp_table sysname, @insert_list sysname, - @collection_sql nvarchar(max); + @collection_sql nvarchar(max), + /*Log to table stuff*/ + @log_table_significant_waits sysname, + @log_table_waits_by_count sysname, + @log_table_waits_by_duration sysname, + @log_table_io_issues sysname, + @log_table_cpu_tasks sysname, + @log_table_memory_conditions sysname, + @log_table_memory_broker sysname, + @log_table_memory_node_oom sysname, + @log_table_system_health sysname, + @log_table_scheduler_issues sysname, + @log_table_severe_errors sysname, + @cleanup_date datetime2(7), + @check_sql nvarchar(max) = N'', + @create_sql nvarchar(max) = N'', + @insert_sql nvarchar(max) = N'', + @log_database_schema nvarchar(1024), + @max_event_time datetime2(7), + @dsql nvarchar(max) = N'', + @mdsql nvarchar(max) = N''; IF @azure = 1 BEGIN @@ -232,7 +271,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. IF @debug = 1 BEGIN - RAISERROR('Fixing variables', 0, 1) WITH NOWAIT; + RAISERROR('Fixing parameters and variables', 0, 1) WITH NOWAIT; END; SELECT @@ -364,7 +403,34 @@ FROM ) AS xml {cross_apply} OPTION(RECOMPILE); -'; +', + @mdsql = N' +IF OBJECT_ID(''{table_check}'', ''U'') IS NOT NULL +BEGIN + SELECT + @max_event_time = + ISNULL + ( + MAX({date_column}), + DATEADD + ( + MINUTE, + DATEDIFF + ( + MINUTE, + SYSDATETIME(), + GETUTCDATE() + ), + DATEADD + ( + DAY, + -1, + SYSDATETIME() + ) + ) + ) + FROM {table_check}; +END;'; IF @timestamp_utc_mode = 0 BEGIN @@ -438,6 +504,621 @@ AND ca.utc_timestamp < @end_date'; END; END; + /* Validate logging parameters */ + IF @log_to_table = 1 + BEGIN + SELECT + /* Default database name to current database if not specified */ + @log_database_name = ISNULL(@log_database_name, DB_NAME()), + /* Default schema name to dbo if not specified */ + @log_schema_name = ISNULL(@log_schema_name, N'dbo'), + @log_retention_days = + CASE + WHEN @log_retention_days < 0 + THEN ABS(@log_retention_days) + ELSE @log_retention_days + END; + + /* Validate database exists */ + IF NOT EXISTS + ( + SELECT + 1/0 + FROM sys.databases AS d + WHERE d.name = @log_database_name + ) + BEGIN + RAISERROR('The specified logging database %s does not exist. Logging will be disabled.', 11, 1, @log_database_name) WITH NOWAIT; + RETURN; + END; + + SET + @log_database_schema = + QUOTENAME(@log_database_name) + + N'.' + + QUOTENAME(@log_schema_name) + + N'.'; + + /* Generate fully qualified table names */ + SELECT + @log_table_significant_waits = + @log_database_schema + + QUOTENAME(@log_table_name_prefix + N'_SignificantWaits'), + @log_table_waits_by_count = + @log_database_schema + + QUOTENAME(@log_table_name_prefix + N'_WaitsByCount'), + @log_table_waits_by_duration = + @log_database_schema + + QUOTENAME(@log_table_name_prefix + N'_WaitsByDuration'), + @log_table_io_issues = + @log_database_schema + + QUOTENAME(@log_table_name_prefix + N'_IOIssues'), + @log_table_cpu_tasks = + @log_database_schema + + QUOTENAME(@log_table_name_prefix + N'_CPUTasks'), + @log_table_memory_conditions = + @log_database_schema + + QUOTENAME(@log_table_name_prefix + N'_MemoryConditions'), + @log_table_memory_broker = + @log_database_schema + + QUOTENAME(@log_table_name_prefix + N'_MemoryBroker'), + @log_table_memory_node_oom = + @log_database_schema + + QUOTENAME(@log_table_name_prefix + N'_MemoryNodeOOM'), + @log_table_system_health = + @log_database_schema + + QUOTENAME(@log_table_name_prefix + N'_SystemHealth'), + @log_table_scheduler_issues = + @log_database_schema + + QUOTENAME(@log_table_name_prefix + N'_SchedulerIssues'), + @log_table_severe_errors = + @log_database_schema + + QUOTENAME(@log_table_name_prefix + N'_SevereErrors'); + + /* Check if schema exists and create it if needed */ + SET @check_sql = N' + IF NOT EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@log_database_name) + N'.sys.schemas AS s + WHERE s.name = @schema_name + ) + BEGIN + DECLARE + @create_schema_sql nvarchar(max) = N''CREATE SCHEMA '' + QUOTENAME(@schema_name); + + EXECUTE ' + QUOTENAME(@log_database_name) + N'.sys.sp_executesql @create_schema_sql; + IF @debug = 1 BEGIN RAISERROR(''Created schema %s in database %s for logging.'', 0, 1, @schema_name, @db_name) WITH NOWAIT; END; + END'; + + EXECUTE sys.sp_executesql + @check_sql, + N'@schema_name sysname, + @db_name sysname, + @debug bit', + @log_schema_name, + @log_database_name, + @debug; + + SET @create_sql = N' + IF NOT EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@log_database_name) + N'.sys.tables AS t + JOIN ' + QUOTENAME(@log_database_name) + N'.sys.schemas AS s + ON t.schema_id = s.schema_id + WHERE t.name = @table_name + N''_SignificantWaits'' + AND s.name = @schema_name + ) + BEGIN + CREATE TABLE ' + @log_table_significant_waits + N' + ( + id bigint IDENTITY, + collection_time datetime2(7) NOT NULL DEFAULT SYSDATETIME(), + event_time datetime2(7) NULL, + wait_type nvarchar(60) NULL, + duration_ms nvarchar(30) NULL, + signal_duration_ms nvarchar(30) NULL, + wait_resource nvarchar(256) NULL, + query_text xml NULL, + session_id integer NULL, + PRIMARY KEY CLUSTERED (collection_time, id) + ); + IF @debug = 1 BEGIN RAISERROR(''Created table %s for significant waits logging.'', 0, 1, ''' + @log_table_significant_waits + N''') WITH NOWAIT; END; + END'; + + EXECUTE sys.sp_executesql + @create_sql, + N'@schema_name sysname, + @table_name sysname, + @debug bit', + @log_schema_name, + @log_table_name_prefix, + @debug; + + /* Create WaitsByCount table if it doesn't exist */ + SET @create_sql = N' + IF NOT EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@log_database_name) + N'.sys.tables AS t + JOIN ' + QUOTENAME(@log_database_name) + N'.sys.schemas AS s + ON t.schema_id = s.schema_id + WHERE t.name = @table_name + N''_WaitsByCount'' + AND s.name = @schema_name + ) + BEGIN + CREATE TABLE ' + @log_table_waits_by_count + N' + ( + id bigint IDENTITY, + collection_time datetime2(7) NOT NULL DEFAULT SYSDATETIME(), + event_time_rounded datetime2(7) NULL, + wait_type nvarchar(60) NULL, + waits nvarchar(30) NULL, + average_wait_time_ms nvarchar(30) NULL, + max_wait_time_ms nvarchar(30) NULL, + PRIMARY KEY CLUSTERED (collection_time, id) + ); + IF @debug = 1 BEGIN RAISERROR(''Created table %s for waits by count logging.'', 0, 1, ''' + @log_table_waits_by_count + N''') WITH NOWAIT; END; + END'; + + EXECUTE sys.sp_executesql + @create_sql, + N'@schema_name sysname, + @table_name sysname, + @debug bit', + @log_schema_name, + @log_table_name_prefix, + @debug; + + /* Create WaitsByDuration table if it doesn't exist */ + SET @create_sql = N' + IF NOT EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@log_database_name) + N'.sys.tables AS t + JOIN ' + QUOTENAME(@log_database_name) + N'.sys.schemas AS s + ON t.schema_id = s.schema_id + WHERE t.name = @table_name + N''_WaitsByDuration'' + AND s.name = @schema_name + ) + BEGIN + CREATE TABLE ' + @log_table_waits_by_duration + N' + ( + id bigint IDENTITY, + collection_time datetime2(7) NOT NULL DEFAULT SYSDATETIME(), + event_time_rounded datetime2(7) NULL, + wait_type nvarchar(60) NULL, + average_wait_time_ms nvarchar(30) NULL, + max_wait_time_ms nvarchar(30) NULL, + PRIMARY KEY CLUSTERED (collection_time, id) + ); + IF @debug = 1 BEGIN RAISERROR(''Created table %s for waits by duration logging.'', 0, 1, ''' + @log_table_waits_by_duration + N''') WITH NOWAIT; END; + END'; + + EXECUTE sys.sp_executesql + @create_sql, + N'@schema_name sysname, + @table_name sysname, + @debug bit', + @log_schema_name, + @log_table_name_prefix, + @debug; + + /* Create IOIssues table if it doesn't exist */ + SET @create_sql = N' + IF NOT EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@log_database_name) + N'.sys.tables AS t + JOIN ' + QUOTENAME(@log_database_name) + N'.sys.schemas AS s + ON t.schema_id = s.schema_id + WHERE t.name = @table_name + N''_IOIssues'' + AND s.name = @schema_name + ) + BEGIN + CREATE TABLE ' + @log_table_io_issues + N' + ( + id bigint IDENTITY, + collection_time datetime2(7) NOT NULL DEFAULT SYSDATETIME(), + event_time datetime2(7) NULL, + state nvarchar(256) NULL, + ioLatchTimeouts bigint NULL, + intervalLongIos bigint NULL, + totalLongIos bigint NULL, + longestPendingRequests_duration_ms nvarchar(30) NULL, + longestPendingRequests_filePath nvarchar(500) NULL, + PRIMARY KEY CLUSTERED (collection_time, id) + ); + IF @debug = 1 BEGIN RAISERROR(''Created table %s for IO issues logging.'', 0, 1, ''' + @log_table_io_issues + N''') WITH NOWAIT; END; + END'; + + EXECUTE sys.sp_executesql + @create_sql, + N'@schema_name sysname, + @table_name sysname, + @debug bit', + @log_schema_name, + @log_table_name_prefix, + @debug; + + /* Create CPUTasks table if it doesn't exist */ + SET @create_sql = N' + IF NOT EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@log_database_name) + N'.sys.tables AS t + JOIN ' + QUOTENAME(@log_database_name) + N'.sys.schemas AS s + ON t.schema_id = s.schema_id + WHERE t.name = @table_name + N''_CPUTasks'' + AND s.name = @schema_name + ) + BEGIN + CREATE TABLE ' + @log_table_cpu_tasks + N' + ( + id bigint IDENTITY, + collection_time datetime2(7) NOT NULL DEFAULT SYSDATETIME(), + event_time datetime2(7) NULL, + state nvarchar(256) NULL, + maxWorkers bigint NULL, + workersCreated bigint NULL, + workersIdle bigint NULL, + tasksCompletedWithinInterval bigint NULL, + pendingTasks bigint NULL, + oldestPendingTaskWaitingTime bigint NULL, + hasUnresolvableDeadlockOccurred bit NULL, + hasDeadlockedSchedulersOccurred bit NULL, + didBlockingOccur bit NULL, + PRIMARY KEY CLUSTERED (collection_time, id) + ); + IF @debug = 1 BEGIN RAISERROR(''Created table %s for CPU tasks logging.'', 0, 1, ''' + @log_table_cpu_tasks + N''') WITH NOWAIT; END; + END'; + + EXECUTE sys.sp_executesql + @create_sql, + N'@schema_name sysname, + @table_name sysname, + @debug bit', + @log_schema_name, + @log_table_name_prefix, + @debug; + + /* Create MemoryConditions table if it doesn't exist */ + SET @create_sql = N' + IF NOT EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@log_database_name) + N'.sys.tables AS t + JOIN ' + QUOTENAME(@log_database_name) + N'.sys.schemas AS s + ON t.schema_id = s.schema_id + WHERE t.name = @table_name + N''_MemoryConditions'' + AND s.name = @schema_name + ) + BEGIN + CREATE TABLE ' + @log_table_memory_conditions + N' + ( + id bigint IDENTITY, + collection_time datetime2(7) NOT NULL DEFAULT SYSDATETIME(), + event_time datetime2(7) NULL, + lastNotification nvarchar(128) NULL, + outOfMemoryExceptions bigint NULL, + isAnyPoolOutOfMemory bit NULL, + processOutOfMemoryPeriod bigint NULL, + name nvarchar(128) NULL, + available_physical_memory_gb bigint NULL, + available_virtual_memory_gb bigint NULL, + available_paging_file_gb bigint NULL, + working_set_gb bigint NULL, + percent_of_committed_memory_in_ws bigint NULL, + page_faults bigint NULL, + system_physical_memory_high bigint NULL, + system_physical_memory_low bigint NULL, + process_physical_memory_low bigint NULL, + process_virtual_memory_low bigint NULL, + vm_reserved_gb bigint NULL, + vm_committed_gb bigint NULL, + locked_pages_allocated bigint NULL, + large_pages_allocated bigint NULL, + emergency_memory_gb bigint NULL, + emergency_memory_in_use_gb bigint NULL, + target_committed_gb bigint NULL, + current_committed_gb bigint NULL, + pages_allocated bigint NULL, + pages_reserved bigint NULL, + pages_free bigint NULL, + pages_in_use bigint NULL, + page_alloc_potential bigint NULL, + numa_growth_phase bigint NULL, + last_oom_factor bigint NULL, + last_os_error bigint NULL, + PRIMARY KEY CLUSTERED (collection_time, id) + ); + IF @debug = 1 BEGIN RAISERROR(''Created table %s for memory conditions logging.'', 0, 1, ''' + @log_table_memory_conditions + N''') WITH NOWAIT; END; + END'; + + EXECUTE sys.sp_executesql + @create_sql, + N'@schema_name sysname, + @table_name sysname, + @debug bit', + @log_schema_name, + @log_table_name_prefix, + @debug; + + /* Create MemoryBroker table if it doesn't exist */ + SET @create_sql = N' + IF NOT EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@log_database_name) + N'.sys.tables AS t + JOIN ' + QUOTENAME(@log_database_name) + N'.sys.schemas AS s + ON t.schema_id = s.schema_id + WHERE t.name = @table_name + N''_MemoryBroker'' + AND s.name = @schema_name + ) + BEGIN + CREATE TABLE ' + @log_table_memory_broker + N' + ( + id bigint IDENTITY, + collection_time datetime2(7) NOT NULL DEFAULT SYSDATETIME(), + event_time datetime2(7) NULL, + node_id int NULL, + memory_available_gb nvarchar(30) NULL, + memory_requested_gb nvarchar(30) NULL, + memory_allocator nvarchar(256) NULL, + memory_allocation_type nvarchar(256) NULL, + memory_clerk_name nvarchar(256) NULL, + os_error int NULL, + PRIMARY KEY CLUSTERED (collection_time, id) + ); + IF @debug = 1 BEGIN RAISERROR(''Created table %s for memory broker logging.'', 0, 1, ''' + @log_table_memory_broker + N''') WITH NOWAIT; END; + END'; + + EXECUTE sys.sp_executesql + @create_sql, + N'@schema_name sysname, + @table_name sysname, + @debug bit', + @log_schema_name, + @log_table_name_prefix, + @debug; + + /* Create MemoryNodeOOM table if it doesn't exist */ + SET @create_sql = N' + IF NOT EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@log_database_name) + N'.sys.tables AS t + JOIN ' + QUOTENAME(@log_database_name) + N'.sys.schemas AS s + ON t.schema_id = s.schema_id + WHERE t.name = @table_name + N''_MemoryNodeOOM'' + AND s.name = @schema_name + ) + BEGIN + CREATE TABLE ' + @log_table_memory_node_oom + N' + ( + id bigint IDENTITY, + collection_time datetime2(7) NOT NULL DEFAULT SYSDATETIME(), + event_time datetime2(7) NULL, + node_id int NULL, + memory_available_kb nvarchar(30) NULL, + memory_requested_kb nvarchar(30) NULL, + memory_available_mb nvarchar(30) NULL, + memory_requested_mb nvarchar(30) NULL, + memory_allocator nvarchar(256) NULL, + memory_allocation_type nvarchar(256) NULL, + memory_clerk_name nvarchar(256) NULL, + os_error int NULL, + PRIMARY KEY CLUSTERED (collection_time, id) + ); + IF @debug = 1 BEGIN RAISERROR(''Created table %s for memory node OOM logging.'', 0, 1, ''' + @log_table_memory_node_oom + N''') WITH NOWAIT; END; + END'; + + EXECUTE sys.sp_executesql + @create_sql, + N'@schema_name sysname, + @table_name sysname, + @debug bit', + @log_schema_name, + @log_table_name_prefix, + @debug; + + /* Create SystemHealth table if it doesn't exist */ + SET @create_sql = N' + IF NOT EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@log_database_name) + N'.sys.tables AS t + JOIN ' + QUOTENAME(@log_database_name) + N'.sys.schemas AS s + ON t.schema_id = s.schema_id + WHERE t.name = @table_name + N''_SystemHealth'' + AND s.name = @schema_name + ) + BEGIN + CREATE TABLE ' + @log_table_system_health + N' + ( + id bigint IDENTITY, + collection_time datetime2(7) NOT NULL DEFAULT SYSDATETIME(), + event_time datetime2(7) NULL, + state nvarchar(256) NULL, + spinlockBackoffs bigint NULL, + sickSpinlockType nvarchar(256) NULL, + sickSpinlockTypeAfterAv nvarchar(256) NULL, + latchWarnings bigint NULL, + isAccessViolationOccurred bigint NULL, + writeAccessViolationCount bigint NULL, + totalDumpRequests bigint NULL, + intervalDumpRequests bigint NULL, + nonYieldingTasksReported bigint NULL, + pageFaults bigint NULL, + systemCpuUtilization bigint NULL, + sqlCpuUtilization bigint NULL, + BadPagesDetected bigint NULL, + BadPagesFixed bigint NULL, + PRIMARY KEY CLUSTERED (collection_time, id) + ); + IF @debug = 1 BEGIN RAISERROR(''Created table %s for system health logging.'', 0, 1, ''' + @log_table_system_health + N''') WITH NOWAIT; END; + END'; + + EXECUTE sys.sp_executesql + @create_sql, + N'@schema_name sysname, + @table_name sysname, + @debug bit', + @log_schema_name, + @log_table_name_prefix, + @debug; + + /* Create SchedulerIssues table if it doesn't exist */ + SET @create_sql = N' + IF NOT EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@log_database_name) + N'.sys.tables AS t + JOIN ' + QUOTENAME(@log_database_name) + N'.sys.schemas AS s + ON t.schema_id = s.schema_id + WHERE t.name = @table_name + N''_SchedulerIssues'' + AND s.name = @schema_name + ) + BEGIN + CREATE TABLE ' + @log_table_scheduler_issues + N' + ( + id bigint IDENTITY, + collection_time datetime2(7) NOT NULL DEFAULT SYSDATETIME(), + event_time datetime2(7) NULL, + scheduler_id int NULL, + cpu_id int NULL, + status nvarchar(256) NULL, + is_online bit NULL, + is_runnable bit NULL, + is_running bit NULL, + non_yielding_time_ms nvarchar(30) NULL, + thread_quantum_ms nvarchar(30) NULL, + PRIMARY KEY CLUSTERED (collection_time, id) + ); + IF @debug = 1 BEGIN RAISERROR(''Created table %s for scheduler issues logging.'', 0, 1, ''' + @log_table_scheduler_issues + N''') WITH NOWAIT; END; + END'; + + EXECUTE sys.sp_executesql + @create_sql, + N'@schema_name sysname, + @table_name sysname, + @debug bit', + @log_schema_name, + @log_table_name_prefix, + @debug; + + /* Create SevereErrors table if it doesn't exist */ + SET @create_sql = N' + IF NOT EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@log_database_name) + N'.sys.tables AS t + JOIN ' + QUOTENAME(@log_database_name) + N'.sys.schemas AS s + ON t.schema_id = s.schema_id + WHERE t.name = @table_name + N''_SevereErrors'' + AND s.name = @schema_name + ) + BEGIN + CREATE TABLE ' + @log_table_severe_errors + N' + ( + id bigint IDENTITY, + collection_time datetime2(7) NOT NULL DEFAULT SYSDATETIME(), + event_time datetime2(7) NULL, + error_number int NULL, + severity int NULL, + state int NULL, + message nvarchar(max) NULL, + database_name sysname NULL, + database_id int NULL, + PRIMARY KEY CLUSTERED (collection_time, id) + ); + IF @debug = 1 BEGIN RAISERROR(''Created table %s for severe errors logging.'', 0, 1, ''' + @log_table_severe_errors + N''') WITH NOWAIT; END; + END'; + + EXECUTE sys.sp_executesql + @create_sql, + N'@schema_name sysname, + @table_name sysname, + @debug bit', + @log_schema_name, + @log_table_name_prefix, + @debug; + + /* Handle log retention if specified */ + IF @log_to_table = 1 AND @log_retention_days > 0 + BEGIN + IF @debug = 1 + BEGIN + RAISERROR('Cleaning up log tables older than %i', 0, 1, @log_retention_days) WITH NOWAIT; + END; + + SET @cleanup_date = + DATEADD + ( + DAY, + -@log_retention_days, + SYSDATETIME() + ); + + /* Clean up each log table */ + SET @dsql = N' + DELETE FROM ' + @log_table_significant_waits + ' + WHERE collection_time < @cleanup_date; + + DELETE FROM ' + @log_table_waits_by_count + ' + WHERE collection_time < @cleanup_date; + + DELETE FROM ' + @log_table_waits_by_duration + ' + WHERE collection_time < @cleanup_date; + + DELETE FROM ' + @log_table_io_issues + ' + WHERE collection_time < @cleanup_date; + + DELETE FROM ' + @log_table_cpu_tasks + ' + WHERE collection_time < @cleanup_date; + + DELETE FROM ' + @log_table_memory_conditions + ' + WHERE collection_time < @cleanup_date; + + DELETE FROM ' + @log_table_memory_broker + ' + WHERE collection_time < @cleanup_date; + + DELETE FROM ' + @log_table_memory_node_oom + ' + WHERE collection_time < @cleanup_date; + + DELETE FROM ' + @log_table_system_health + ' + WHERE collection_time < @cleanup_date; + + DELETE FROM ' + @log_table_scheduler_issues + ' + WHERE collection_time < @cleanup_date; + + DELETE FROM ' + @log_table_severe_errors + ' + WHERE collection_time < @cleanup_date;'; + + IF @debug = 1 BEGIN PRINT @dsql; END; + + EXECUTE sys.sp_executesql @dsql, N'@cleanup_date datetime2(7)', @cleanup_date; + + IF @debug = 1 + BEGIN + RAISERROR('Log cleanup complete', 0, 1) WITH NOWAIT; + END; + END; + END; + IF @debug = 1 BEGIN RAISERROR('Creating temp tables', 0, 1) WITH NOWAIT; @@ -1056,6 +1737,7 @@ AND ca.utc_timestamp < @end_date'; x.event_time DESC; END; + /* First logging section, queries with significant waits*/ IF NOT EXISTS ( SELECT @@ -1063,6 +1745,7 @@ AND ca.utc_timestamp < @end_date'; FROM #waits_queries AS wq ) BEGIN + /* No results logic, only return if not logging */ SELECT finding = CASE @@ -1078,12 +1761,20 @@ AND ca.utc_timestamp < @end_date'; RTRIM(@wait_duration_ms) + '.' ELSE 'no queries with significant waits found!' - END; + END + WHERE @log_to_table = 0; END; ELSE BEGIN + /* Build the query */ + SET @dsql = N' SELECT - finding = 'queries with significant waits', + ' + CASE + WHEN @log_to_table = 1 + THEN N'' + ELSE N'finding = ''queries with significant waits'',' + END + + N' wq.event_time, wq.wait_type, duration_ms = @@ -1099,8 +1790,8 @@ AND ca.utc_timestamp < @end_date'; ), 1 ), - N'.00', - N'' + N''.00'', + N'''' ), signal_duration_ms = REPLACE @@ -1115,8 +1806,8 @@ AND ca.utc_timestamp < @end_date'; ), 1 ), - N'.00', - N'' + N''.00'', + N'''' ), wq.wait_resource, query_text = @@ -1125,14 +1816,91 @@ AND ca.utc_timestamp < @end_date'; [processing-instruction(query)] = wq.query_text FOR XML - PATH(N''), + PATH(N''''), TYPE ), wq.session_id - FROM #waits_queries AS wq + FROM #waits_queries AS wq'; + + /* Add the WHERE clause only for table logging */ + IF @log_to_table = 1 + BEGIN + SET @mdsql = + REPLACE + ( + REPLACE + ( + @mdsql, + '{table_check}', + @log_table_significant_waits + ), + '{date_column}', + 'event_time' + ); + + IF @debug = 1 BEGIN PRINT @mdsql; END; + + EXECUTE sys.sp_executesql + @mdsql, + N'@max_event_time datetime2(7) OUTPUT', + @max_event_time OUTPUT; + + SET @mdsql = + REPLACE + ( + REPLACE + ( + @mdsql, + @log_table_significant_waits, + '{table_check}' + ), + 'event_time', + '{date_column}' + ); + + SET @dsql += N' + WHERE wq.event_time > @max_event_time'; + END; + + /* Add the ORDER BY clause */ + SET @dsql += N' ORDER BY wq.duration_ms DESC - OPTION(RECOMPILE); + OPTION(RECOMPILE);'; + + /* Handle table logging */ + IF @log_to_table = 1 + BEGIN + SET @insert_sql = N' + INSERT INTO + ' + @log_table_significant_waits + N' + ( + event_time, + wait_type, + duration_ms, + signal_duration_ms, + wait_resource, + query_text, + session_id + )' + + @dsql; + + IF @debug = 1 BEGIN PRINT @insert_sql; END; + + EXECUTE sys.sp_executesql @insert_sql, + N'@max_event_time datetime2(7)', + @max_event_time; + END; + + /* Execute the query for client results */ + IF @log_to_table = 0 + BEGIN + + IF @debug = 1 BEGIN PRINT @dsql; END; + + EXECUTE sys.sp_executesql + @dsql; + END; END; /*Waits by count*/ @@ -1221,6 +1989,7 @@ AND ca.utc_timestamp < @end_date'; ) OPTION(RECOMPILE); + /* Waits by count logging section */ IF NOT EXISTS ( SELECT @@ -1228,6 +1997,7 @@ AND ca.utc_timestamp < @end_date'; FROM #tc AS t ) BEGIN + /* No results logic, only return if not logging */ SELECT finding = CASE @@ -1241,12 +2011,20 @@ AND ca.utc_timestamp < @end_date'; RTRIM(CONVERT(date, @end_date)) + '.' ELSE 'no significant waits found!' - END; + END + WHERE @log_to_table = 0; END; ELSE BEGIN + /* Build the query */ + SET @dsql = N' SELECT - t.finding, + ' + CASE + WHEN @log_to_table = 1 + THEN N'' + ELSE N'finding = ''waits by count'',' + END + + N' t.event_time_rounded, t.wait_type, waits = @@ -1262,8 +2040,8 @@ AND ca.utc_timestamp < @end_date'; ), 1 ), - N'.00', - N'' + N''.00'', + N'''' ), average_wait_time_ms = REPLACE @@ -1278,8 +2056,8 @@ AND ca.utc_timestamp < @end_date'; ), 1 ), - N'.00', - N'' + N''.00'', + N'''' ), max_wait_time_ms = REPLACE @@ -1294,14 +2072,88 @@ AND ca.utc_timestamp < @end_date'; ), 1 ), - N'.00', - N'' + N''.00'', + N'''' ) - FROM #tc AS t + FROM #tc AS t'; + + /* Add the WHERE clause only for table logging */ + IF @log_to_table = 1 + BEGIN + SET @mdsql = + REPLACE + ( + REPLACE + ( + @mdsql, + '{table_check}', + @log_table_waits_by_count + ), + '{date_column}', + 'event_time_rounded' + ); + + IF @debug = 1 BEGIN PRINT @mdsql; END; + + EXECUTE sys.sp_executesql + @mdsql, + N'@max_event_time datetime2(7) OUTPUT', + @max_event_time OUTPUT; + + SET @mdsql = + REPLACE + ( + REPLACE + ( + @mdsql, + @log_table_waits_by_count, + '{table_check}' + ), + 'event_time_rounded', + '{date_column}' + ); + + SET @dsql += N' + WHERE t.event_time_rounded > @max_event_time'; + END; + + /* Add the ORDER BY clause */ + SET @dsql += N' ORDER BY t.event_time_rounded DESC, t.waits DESC - OPTION(RECOMPILE); + OPTION(RECOMPILE);'; + + /* Handle table logging */ + IF @log_to_table = 1 + BEGIN + SET @insert_sql = N' + INSERT INTO + ' + @log_table_waits_by_count + N' + ( + event_time_rounded, + wait_type, + waits, + average_wait_time_ms, + max_wait_time_ms + )' + + @dsql; + + IF @debug = 1 BEGIN PRINT @insert_sql; END; + + EXECUTE sys.sp_executesql @insert_sql, + N'@max_event_time datetime2(7)', + @max_event_time; + END; + + /* Execute the query for client results */ + IF @log_to_table = 0 + BEGIN + IF @debug = 1 BEGIN PRINT @dsql; END; + + EXECUTE sys.sp_executesql + @dsql; + END; END; /*Grab waits by duration*/ @@ -1394,6 +2246,7 @@ AND ca.utc_timestamp < @end_date'; td.max_wait_time_ms OPTION(RECOMPILE); + /* Waits by duration logging section */ IF NOT EXISTS ( SELECT @@ -1401,6 +2254,7 @@ AND ca.utc_timestamp < @end_date'; FROM #td AS t ) BEGIN + /* No results logic, only return if not logging */ SELECT finding = CASE @@ -1416,12 +2270,20 @@ AND ca.utc_timestamp < @end_date'; RTRIM(@wait_duration_ms) + '.' ELSE 'no significant waits found!' - END; + END + WHERE @log_to_table = 0; END; ELSE BEGIN + /* Build the query */ + SET @dsql = N' SELECT - x.finding, + ' + CASE + WHEN @log_to_table = 1 + THEN N'' + ELSE N'finding = ''waits by duration'',' + END + + N' x.event_time_rounded, x.wait_type, x.average_wait_time_ms, @@ -1445,8 +2307,8 @@ AND ca.utc_timestamp < @end_date'; ), 1 ), - N'.00', - N'' + N''.00'', + N'''' ), average_wait_time_ms = REPLACE @@ -1461,8 +2323,8 @@ AND ca.utc_timestamp < @end_date'; ), 1 ), - N'.00', - N'' + N''.00'', + N'''' ), max_wait_time_ms = REPLACE @@ -1477,8 +2339,8 @@ AND ca.utc_timestamp < @end_date'; ), 1 ), - N'.00', - N'' + N''.00'', + N'''' ), s = ROW_NUMBER() OVER @@ -1500,17 +2362,90 @@ AND ca.utc_timestamp < @end_date'; ) FROM #td AS t ) AS x - WHERE x.n = 1 + WHERE x.n = 1'; + + /* Add the WHERE clause only for table logging */ + IF @log_to_table = 1 + BEGIN + SET @mdsql = + REPLACE + ( + REPLACE + ( + @mdsql, + '{table_check}', + @log_table_waits_by_duration + ), + '{date_column}', + 'event_time_rounded' + ); + + IF @debug = 1 BEGIN PRINT @mdsql; END; + + EXECUTE sys.sp_executesql + @mdsql, + N'@max_event_time datetime2(7) OUTPUT', + @max_event_time OUTPUT; + + SET @mdsql = + REPLACE + ( + REPLACE + ( + @mdsql, + @log_table_waits_by_duration, + '{table_check}' + ), + 'event_time_rounded', + '{date_column}' + ); + + SET @dsql += N' + AND x.event_time_rounded > @max_event_time'; + END; + + /* Add the ORDER BY clause */ + SET @dsql += N' ORDER BY x.s - OPTION(RECOMPILE); - END; - END; /*End wait stats*/ - - /*Grab IO stuff*/ - IF @what_to_check IN ('all', 'disk') - BEGIN - IF @debug = 1 + OPTION(RECOMPILE);'; + + /* Handle table logging */ + IF @log_to_table = 1 + BEGIN + SET @insert_sql = N' + INSERT INTO + ' + @log_table_waits_by_duration + N' + ( + event_time_rounded, + wait_type, + average_wait_time_ms, + max_wait_time_ms + )' + + @dsql; + + IF @debug = 1 BEGIN PRINT @insert_sql; END; + + EXECUTE sys.sp_executesql @insert_sql, + N'@max_event_time datetime2(7)', + @max_event_time; + END; + + /* Execute the query for client results */ + IF @log_to_table = 0 + BEGIN + IF @debug = 1 BEGIN PRINT @dsql; END; + + EXECUTE sys.sp_executesql + @dsql; + END; + END; + END; /*End wait stats*/ + + /*Grab IO stuff*/ + IF @what_to_check IN ('all', 'disk') + BEGIN + IF @debug = 1 BEGIN RAISERROR('Parsing disk stuff', 0, 1) WITH NOWAIT; END; @@ -1575,6 +2510,7 @@ AND ca.utc_timestamp < @end_date'; ISNULL(i.longestPendingRequests_filePath, 'N/A') OPTION(RECOMPILE); + /* Potential IO issues logging section */ IF NOT EXISTS ( SELECT @@ -1582,6 +2518,7 @@ AND ca.utc_timestamp < @end_date'; FROM #i AS i ) BEGIN + /* No results logic, only return if not logging */ SELECT finding = CASE @@ -1597,12 +2534,20 @@ AND ca.utc_timestamp < @end_date'; RTRIM(@warnings_only) + '.' ELSE 'no io issues found!' - END; + END + WHERE @log_to_table = 0; END; ELSE BEGIN + /* Build the query */ + SET @dsql = N' SELECT - i.finding, + ' + CASE + WHEN @log_to_table = 1 + THEN N'' + ELSE N'finding = ''potential io issues'',' + END + + N' i.event_time, i.state, i.ioLatchTimeouts, @@ -1621,14 +2566,92 @@ AND ca.utc_timestamp < @end_date'; ), 1 ), - N'.00', - N'' + N''.00'', + N'''' ), i.longestPendingRequests_filePath - FROM #i AS i + FROM #i AS i'; + + /* Add the WHERE clause only for table logging */ + IF @log_to_table = 1 + BEGIN + /* Get max event_time for IO issues */ + SET @mdsql = + REPLACE + ( + REPLACE + ( + @mdsql, + '{table_check}', + @log_table_io_issues + ), + '{date_column}', + 'event_time' + ); + + IF @debug = 1 BEGIN PRINT @mdsql; END; + + EXECUTE sys.sp_executesql + @mdsql, + N'@max_event_time datetime2(7) OUTPUT', + @max_event_time OUTPUT; + + /* Reset @mdsql to original template */ + SET @mdsql = + REPLACE + ( + REPLACE + ( + @mdsql, + @log_table_io_issues, + '{table_check}' + ), + 'event_time', + '{date_column}' + ); + + SET @dsql += N' + WHERE i.event_time > @max_event_time'; + END; + + /* Add the ORDER BY clause */ + SET @dsql += N' ORDER BY i.event_time DESC - OPTION(RECOMPILE); + OPTION(RECOMPILE);'; + + /* Handle table logging */ + IF @log_to_table = 1 + BEGIN + SET @insert_sql = N' + INSERT INTO + ' + @log_table_io_issues + N' + ( + event_time, + state, + ioLatchTimeouts, + intervalLongIos, + totalLongIos, + longestPendingRequests_duration_ms, + longestPendingRequests_filePath + )' + + @dsql; + + IF @debug = 1 BEGIN PRINT @insert_sql; END; + + EXECUTE sys.sp_executesql @insert_sql, + N'@max_event_time datetime2(7)', + @max_event_time; + END; + + /* Execute the query for client results */ + IF @log_to_table = 0 + BEGIN + IF @debug = 1 BEGIN PRINT @dsql; END; + + EXECUTE sys.sp_executesql + @dsql; + END; END; END; /*End disk*/ @@ -1684,49 +2707,142 @@ AND ca.utc_timestamp < @end_date'; x.event_time DESC; END; - IF NOT EXISTS - ( - SELECT - 1/0 - FROM #scheduler_details AS sd - ) +END; + + /* CPU task details logging section */ + IF NOT EXISTS + ( + SELECT + 1/0 + FROM #scheduler_details AS sd + ) + BEGIN + /* No results logic, only return if not logging */ + SELECT + finding = + CASE + WHEN @what_to_check NOT IN ('all', 'cpu') + THEN 'cpu skipped, @what_to_check set to ' + + @what_to_check + WHEN @what_to_check IN ('all', 'cpu') + THEN 'no cpu issues found between ' + + RTRIM(CONVERT(date, @start_date)) + + ' and ' + + RTRIM(CONVERT(date, @end_date)) + + ' with @warnings_only set to ' + + RTRIM(@warnings_only) + + '.' + ELSE 'no cpu issues found!' + END + WHERE @log_to_table = 0; + END; + ELSE + BEGIN + /* Build the query */ + SET @dsql = N' + SELECT + ' + CASE + WHEN @log_to_table = 1 + THEN N'' + ELSE N'finding = ''cpu task details'',' + END + + N' + sd.event_time, + sd.state, + sd.maxWorkers, + sd.workersCreated, + sd.workersIdle, + sd.tasksCompletedWithinInterval, + sd.pendingTasks, + sd.oldestPendingTaskWaitingTime, + sd.hasUnresolvableDeadlockOccurred, + sd.hasDeadlockedSchedulersOccurred, + sd.didBlockingOccur + FROM #scheduler_details AS sd'; + + /* Add the WHERE clause only for table logging */ + IF @log_to_table = 1 BEGIN - SELECT - finding = - CASE - WHEN @what_to_check NOT IN ('all', 'cpu') - THEN 'cpu skipped, @what_to_check set to ' + - @what_to_check - WHEN @what_to_check IN ('all', 'cpu') - THEN 'no cpu issues found between ' + - RTRIM(CONVERT(date, @start_date)) + - ' and ' + - RTRIM(CONVERT(date, @end_date)) + - ' with @warnings_only set to ' + - RTRIM(@warnings_only) + - '.' - ELSE 'no cpu issues found!' - END; + /* Get max event_time for CPU task details */ + SET @mdsql = + REPLACE + ( + REPLACE + ( + @mdsql, + '{table_check}', + @log_table_cpu_tasks + ), + '{date_column}', + 'event_time' + ); + + IF @debug = 1 BEGIN PRINT @mdsql; END; + + EXECUTE sys.sp_executesql + @mdsql, + N'@max_event_time datetime2(7) OUTPUT', + @max_event_time OUTPUT; + + /* Reset @mdsql to original template */ + SET @mdsql = + REPLACE + ( + REPLACE + ( + @mdsql, + @log_table_cpu_tasks, + '{table_check}' + ), + 'event_time', + '{date_column}' + ); + + SET @dsql += N' + WHERE sd.event_time > @max_event_time'; END; - ELSE + + /* Add the ORDER BY clause */ + SET @dsql += N' + ORDER BY + sd.event_time DESC + OPTION(RECOMPILE);'; + + /* Handle table logging */ + IF @log_to_table = 1 + BEGIN + SET @insert_sql = N' + INSERT INTO + ' + @log_table_cpu_tasks + N' + ( + event_time, + state, + maxWorkers, + workersCreated, + workersIdle, + tasksCompletedWithinInterval, + pendingTasks, + oldestPendingTaskWaitingTime, + hasUnresolvableDeadlockOccurred, + hasDeadlockedSchedulersOccurred, + didBlockingOccur + )' + + @dsql; + + IF @debug = 1 BEGIN PRINT @insert_sql; END; + + EXECUTE sys.sp_executesql @insert_sql, + N'@max_event_time datetime2(7)', + @max_event_time; + END; + + /* Execute the query for client results */ + IF @log_to_table = 0 BEGIN - SELECT - finding = 'cpu task details', - sd.event_time, - sd.state, - sd.maxWorkers, - sd.workersCreated, - sd.workersIdle, - sd.tasksCompletedWithinInterval, - sd.pendingTasks, - sd.oldestPendingTaskWaitingTime, - sd.hasUnresolvableDeadlockOccurred, - sd.hasDeadlockedSchedulersOccurred, - sd.didBlockingOccur - FROM #scheduler_details AS sd - ORDER BY - sd.event_time DESC - OPTION(RECOMPILE); + IF @debug = 1 BEGIN PRINT @dsql; END; + + EXECUTE sys.sp_executesql + @dsql; END; END; /*End CPU*/ @@ -1799,6 +2915,7 @@ AND ca.utc_timestamp < @end_date'; x.event_time DESC; END; + /* Memory conditions logging section */ IF NOT EXISTS ( SELECT @@ -1806,6 +2923,7 @@ AND ca.utc_timestamp < @end_date'; FROM #memory AS m ) BEGIN + /* No results logic, only return if not logging */ SELECT finding = CASE @@ -1821,12 +2939,20 @@ AND ca.utc_timestamp < @end_date'; RTRIM(@warnings_only) + '.' ELSE 'no memory issues found!' - END; + END + WHERE @log_to_table = 0; END; ELSE BEGIN + /* Build the query */ + SET @dsql = N' SELECT - finding = 'memory conditions', + ' + CASE + WHEN @log_to_table = 1 + THEN N'' + ELSE N'finding = ''memory conditions'',' + END + + N' m.event_time, m.lastNotification, m.outOfMemoryExceptions, @@ -1859,10 +2985,114 @@ AND ca.utc_timestamp < @end_date'; m.numa_growth_phase, m.last_oom_factor, m.last_os_error - FROM #memory AS m + FROM #memory AS m'; + + /* Add the WHERE clause only for table logging */ + IF @log_to_table = 1 + BEGIN + /* Get max event_time for memory conditions */ + SET @mdsql = + REPLACE + ( + REPLACE + ( + @mdsql, + '{table_check}', + @log_table_memory_conditions + ), + '{date_column}', + 'event_time' + ); + + IF @debug = 1 BEGIN PRINT @mdsql; END; + + EXECUTE sys.sp_executesql + @mdsql, + N'@max_event_time datetime2(7) OUTPUT', + @max_event_time OUTPUT; + + /* Reset @mdsql to original template */ + SET @mdsql = + REPLACE + ( + REPLACE + ( + @mdsql, + @log_table_memory_conditions, + '{table_check}' + ), + 'event_time', + '{date_column}' + ); + + SET @dsql += N' + WHERE m.event_time > @max_event_time'; + END; + + /* Add the ORDER BY clause */ + SET @dsql += N' ORDER BY m.event_time DESC - OPTION(RECOMPILE); + OPTION(RECOMPILE);'; + + /* Handle table logging */ + IF @log_to_table = 1 + BEGIN + SET @insert_sql = N' + INSERT INTO + ' + @log_table_memory_conditions + N' + ( + event_time, + lastNotification, + outOfMemoryExceptions, + isAnyPoolOutOfMemory, + processOutOfMemoryPeriod, + name, + available_physical_memory_gb, + available_virtual_memory_gb, + available_paging_file_gb, + working_set_gb, + percent_of_committed_memory_in_ws, + page_faults, + system_physical_memory_high, + system_physical_memory_low, + process_physical_memory_low, + process_virtual_memory_low, + vm_reserved_gb, + vm_committed_gb, + locked_pages_allocated, + large_pages_allocated, + emergency_memory_gb, + emergency_memory_in_use_gb, + target_committed_gb, + current_committed_gb, + pages_allocated, + pages_reserved, + pages_free, + pages_in_use, + page_alloc_potential, + numa_growth_phase, + last_oom_factor, + last_os_error + )' + + + @dsql; + + IF @debug = 1 BEGIN PRINT @insert_sql; END; + + EXECUTE sys.sp_executesql @insert_sql, + N'@max_event_time datetime2(7)', + @max_event_time; + END; + + /* Execute the query for client results */ + IF @log_to_table = 0 + BEGIN + IF @debug = 1 BEGIN PRINT @dsql; END; + + EXECUTE sys.sp_executesql + @dsql; + END; END; END; /*End memory*/ @@ -1891,7 +3121,6 @@ AND ca.utc_timestamp < @end_date'; reclaim_target_kb = w.x.value('(data[@name="reclaim_target_kb"]/value)[1]', 'bigint'), reclaimed_kb = w.x.value('(data[@name="reclaimed_kb"]/value)[1]', 'bigint'), pressure = w.x.value('(data[@name="pressure"]/value)[1]', 'bigint'), - pressure_mb = w.x.value('(data[@name="pressure"]/value)[1]', 'bigint') / 1024, currently_available_kb = w.x.value('(data[@name="currently_available_kb"]/value)[1]', 'bigint'), reserved_kb = w.x.value('(data[@name="reserved_kb"]/value)[1]', 'bigint'), committed_kb = w.x.value('(data[@name="committed_kb"]/value)[1]', 'bigint'), @@ -1913,144 +3142,166 @@ AND ca.utc_timestamp < @end_date'; x.event_time DESC; END; - IF NOT EXISTS - ( - SELECT - 1/0 - FROM #memory_broker_info AS mbi - ) - BEGIN - SELECT - finding = - CASE - WHEN @what_to_check NOT IN ('all', 'memory') - THEN 'memory broker skipped, @what_to_check set to ' + - @what_to_check - WHEN @what_to_check IN ('all', 'memory') - THEN 'no memory pressure events found between ' + - RTRIM(CONVERT(date, @start_date)) + - ' and ' + - RTRIM(CONVERT(date, @end_date)) + - ' with @warnings_only set to ' + - RTRIM(@warnings_only) + - '.' - ELSE 'no memory pressure events found!' - END; - END; - ELSE - BEGIN - SELECT - finding = 'memory broker notifications', - mbi.event_time, - mbi.notification_type, - reclaim_target_kb = - REPLACE - ( - CONVERT - ( - nvarchar(30), - CONVERT - ( - money, - mbi.reclaim_target_kb - ), - 1 - ), - N'.00', - N'' - ), - reclaimed_kb = - REPLACE - ( - CONVERT - ( - nvarchar(30), - CONVERT - ( - money, - mbi.reclaimed_kb - ), - 1 - ), - N'.00', - N'' - ), - reclaim_success_percent = - CASE - WHEN mbi.reclaim_target_kb > 0 - THEN CONVERT(DECIMAL(5,2), 100.0 * mbi.reclaimed_kb / mbi.reclaim_target_kb) - ELSE 0 - END, - pressure_mb = - REPLACE - ( - CONVERT - ( - nvarchar(30), - CONVERT - ( - money, - mbi.pressure_mb - ), - 1 - ), - N'.00', - N'' - ), - currently_available_kb = - REPLACE - ( - CONVERT - ( - nvarchar(30), - CONVERT - ( - money, - mbi.currently_available_kb - ), - 1 - ), - N'.00', - N'' - ), - reserved_kb = - REPLACE +/* Memory broker notifications logging section */ +IF NOT EXISTS +( + SELECT + 1/0 + FROM #memory_broker_info AS mbi +) +BEGIN + /* No results logic, only return if not logging */ + SELECT + finding = + CASE + WHEN @what_to_check NOT IN ('all', 'memory') + THEN 'memory broker skipped, @what_to_check set to ' + + @what_to_check + WHEN @what_to_check IN ('all', 'memory') + THEN 'no memory pressure events found between ' + + RTRIM(CONVERT(date, @start_date)) + + ' and ' + + RTRIM(CONVERT(date, @end_date)) + + ' with @warnings_only set to ' + + RTRIM(@warnings_only) + + '.' + ELSE 'no memory pressure events found!' + END + WHERE @log_to_table = 0; +END; +ELSE +BEGIN + /* Build the query for memory node OOM events */ + SET @dsql = N' + SELECT + ' + CASE + WHEN @log_to_table = 1 + THEN N'' + ELSE N'finding = ''memory node OOM events'',' + END + + N' + mnoi.event_time, + mnoi.node_id, + memory_available_gb = + REPLACE + ( + CONVERT + ( + nvarchar(30), + CONVERT ( - CONVERT - ( - nvarchar(30), - CONVERT - ( - money, - mbi.reserved_kb - ), - 1 - ), - N'.00', - N'' + money, + mnoi.memory_available_kb / 1024.0 / 1024.0 ), - committed_kb = - REPLACE + 1 + ), + N''.00'', + N'''' + ), + memory_requested_gb = + REPLACE + ( + CONVERT + ( + nvarchar(30), + CONVERT ( - CONVERT - ( - nvarchar(30), - CONVERT - ( - money, - mbi.committed_kb - ), - 1 - ), - N'.00', - N'' + money, + mnoi.memory_requested_kb / 1024.0 / 1024.0 ), - mbi.worker_count - FROM #memory_broker_info AS mbi - ORDER BY - mbi.event_time DESC - OPTION(RECOMPILE); - END; + 1 + ), + N''.00'', + N'''' + ), + mnoi.memory_allocator, + mnoi.memory_allocation_type, + mnoi.memory_clerk_name, + mnoi.os_error + FROM #memory_node_oom_info AS mnoi'; + + /* Add the WHERE clause only for table logging */ + IF @log_to_table = 1 + BEGIN + /* Get max event_time for memory broker */ + SET @mdsql = + REPLACE + ( + REPLACE + ( + @mdsql, + '{table_check}', + @log_table_memory_broker + ), + '{date_column}', + 'event_time' + ); + + IF @debug = 1 BEGIN PRINT @mdsql; END; + + EXECUTE sys.sp_executesql + @mdsql, + N'@max_event_time datetime2(7) OUTPUT', + @max_event_time OUTPUT; + + /* Reset @mdsql to original template */ + SET @mdsql = + REPLACE + ( + REPLACE + ( + @mdsql, + @log_table_memory_broker, + '{table_check}' + ), + 'event_time', + '{date_column}' + ); + + SET @dsql += N' + WHERE mbi.event_time > @max_event_time'; + END; + + /* Add the ORDER BY clause */ + SET @dsql += N' + ORDER BY + mbi.event_time DESC + OPTION(RECOMPILE);'; + + /* Handle table logging */ + IF @log_to_table = 1 + BEGIN + SET @insert_sql = N' + INSERT INTO ' + + @log_table_memory_broker + N' + ( + event_time, + node_id, + memory_available_gb, + memory_requested_gb, + memory_allocator, + memory_allocation_type, + memory_clerk_name, + os_error + )' + + @dsql; + + IF @debug = 1 BEGIN PRINT @insert_sql; END; + + EXECUTE sys.sp_executesql @insert_sql, + N'@max_event_time datetime2(7)', + @max_event_time; + END; + + /* Execute the query for client results */ + IF @log_to_table = 0 + BEGIN + IF @debug = 1 BEGIN PRINT @dsql; END; + + EXECUTE sys.sp_executesql + @dsql; + END; +END; END; /*End memory broker analysis*/ /*Parse memory node OOM data*/ @@ -2097,6 +3348,7 @@ AND ca.utc_timestamp < @end_date'; x.event_time DESC; END; + /* Memory node OOM events logging section */ IF NOT EXISTS ( SELECT @@ -2104,6 +3356,7 @@ AND ca.utc_timestamp < @end_date'; FROM #memory_node_oom_info AS mnoi ) BEGIN + /* No results logic, only return if not logging */ SELECT finding = CASE @@ -2117,15 +3370,23 @@ AND ca.utc_timestamp < @end_date'; RTRIM(CONVERT(date, @end_date)) + '.' ELSE 'no memory node OOM events found!' - END; + END + WHERE @log_to_table = 0; END; ELSE BEGIN + /* Build the query for memory broker notifications */ + SET @dsql = N' SELECT - finding = 'memory node OOM events', - mnoi.event_time, - mnoi.node_id, - memory_available_kb = + ' + CASE + WHEN @log_to_table = 1 + THEN N'' + ELSE N'finding = ''memory broker notifications'',' + END + + N' + mbi.event_time, + mbi.notification_type, + reclaim_target_gb = REPLACE ( CONVERT @@ -2134,14 +3395,14 @@ AND ca.utc_timestamp < @end_date'; CONVERT ( money, - mnoi.memory_available_kb + mbi.reclaim_target_kb / 1024.0 / 1024.0 ), 1 ), - N'.00', - N'' + N''.00'', + N'''' ), - memory_requested_kb = + reclaimed_gb = REPLACE ( CONVERT @@ -2150,14 +3411,20 @@ AND ca.utc_timestamp < @end_date'; CONVERT ( money, - mnoi.memory_requested_kb + mbi.reclaimed_kb / 1024.0 / 1024.0 ), 1 ), - N'.00', - N'' + N''.00'', + N'''' ), - memory_available_mb = + reclaim_success_percent = + CASE + WHEN mbi.reclaim_target_kb > 0 + THEN CONVERT(DECIMAL(5,2), 100.0 * mbi.reclaimed_kb / mbi.reclaim_target_kb) + ELSE 0 + END, + pressure_gb = REPLACE ( CONVERT @@ -2166,14 +3433,14 @@ AND ca.utc_timestamp < @end_date'; CONVERT ( money, - mnoi.memory_available_kb / 1024.0 + mbi.pressure_mb / 1024.0 ), 1 ), - N'.00', - N'' + N''.00'', + N'''' ), - memory_requested_mb = + currently_available_gb = REPLACE ( CONVERT @@ -2182,21 +3449,131 @@ AND ca.utc_timestamp < @end_date'; CONVERT ( money, - mnoi.memory_requested_kb / 1024.0 + mbi.currently_available_kb / 1024.0 / 1024.0 ), 1 ), - N'.00', - N'' + N''.00'', + N'''' ), - mnoi.memory_allocator, - mnoi.memory_allocation_type, - mnoi.memory_clerk_name, - mnoi.os_error - FROM #memory_node_oom_info AS mnoi + reserved_gb = + REPLACE + ( + CONVERT + ( + nvarchar(30), + CONVERT + ( + money, + mbi.reserved_kb / 1024.0 / 1024.0 + ), + 1 + ), + N''.00'', + N'''' + ), + committed_gb = + REPLACE + ( + CONVERT + ( + nvarchar(30), + CONVERT + ( + money, + mbi.committed_kb / 1024.0 / 1024.0 + ), + 1 + ), + N''.00'', + N'''' + ), + mbi.worker_count + FROM #memory_broker_info AS mbi'; + + /* Add the WHERE clause only for table logging */ + IF @log_to_table = 1 + BEGIN + /* Get max event_time for memory node OOM */ + SET @mdsql = + REPLACE + ( + REPLACE + ( + @mdsql, + '{table_check}', + @log_table_memory_node_oom + ), + '{date_column}', + 'event_time' + ); + + IF @debug = 1 BEGIN PRINT @mdsql; END; + + EXECUTE sys.sp_executesql + @mdsql, + N'@max_event_time datetime2(7) OUTPUT', + @max_event_time OUTPUT; + + /* Reset @mdsql to original template */ + SET @mdsql = + REPLACE + ( + REPLACE + ( + @mdsql, + @log_table_memory_node_oom, + '{table_check}' + ), + 'event_time', + '{date_column}' + ); + + SET @dsql += N' + WHERE mnoi.event_time > @max_event_time'; + END; + + /* Add the ORDER BY clause */ + SET @dsql += N' ORDER BY mnoi.event_time DESC - OPTION(RECOMPILE); + OPTION(RECOMPILE);'; + + /* Handle table logging */ + IF @log_to_table = 1 + BEGIN + SET @insert_sql = N' + INSERT INTO + ' + @log_table_memory_node_oom + N' + ( + event_time, + notification_type, + reclaim_target_gb, + reclaimed_gb, + reclaim_success_percent, + pressure_gb, + currently_available_gb, + reserved_gb, + committed_gb, + worker_count + )' + + @dsql; + + IF @debug = 1 BEGIN PRINT @insert_sql; END; + + EXECUTE sys.sp_executesql @insert_sql, + N'@max_event_time datetime2(7)', + @max_event_time; + END; + + /* Execute the query for client results */ + IF @log_to_table = 0 + BEGIN + IF @debug = 1 BEGIN PRINT @dsql; END; + + EXECUTE sys.sp_executesql + @dsql; + END; END; END; /*End memory node OOM analysis*/ @@ -2254,6 +3631,7 @@ AND ca.utc_timestamp < @end_date'; x.event_time DESC; END; + /* Overall system health logging section */ IF NOT EXISTS ( SELECT @@ -2261,6 +3639,7 @@ AND ca.utc_timestamp < @end_date'; FROM #health AS h ) BEGIN + /* No results logic, only return if not logging */ SELECT finding = CASE @@ -2276,12 +3655,20 @@ AND ca.utc_timestamp < @end_date'; RTRIM(@warnings_only) + '.' ELSE 'no system health issues found!' - END; + END + WHERE @log_to_table = 0; END; ELSE BEGIN + /* Build the query */ + SET @dsql = N' SELECT - finding = 'overall system health', + ' + CASE + WHEN @log_to_table = 1 + THEN N'' + ELSE N'finding = ''overall system health'',' + END + + N' h.event_time, h.state, h.spinlockBackoffs, @@ -2298,10 +3685,97 @@ AND ca.utc_timestamp < @end_date'; h.sqlCpuUtilization, h.BadPagesDetected, h.BadPagesFixed - FROM #health AS h + FROM #health AS h'; + + /* Add the WHERE clause only for table logging */ + IF @log_to_table = 1 + BEGIN + /* Get max event_time for system health */ + SET @mdsql = + REPLACE + ( + REPLACE + ( + @mdsql, + '{table_check}', + @log_table_system_health + ), + '{date_column}', + 'event_time' + ); + + IF @debug = 1 BEGIN PRINT @mdsql; END; + + EXECUTE sys.sp_executesql + @mdsql, + N'@max_event_time datetime2(7) OUTPUT', + @max_event_time OUTPUT; + + /* Reset @mdsql to original template */ + SET @mdsql = + REPLACE + ( + REPLACE + ( + @mdsql, + @log_table_system_health, + '{table_check}' + ), + 'event_time', + '{date_column}' + ); + + SET @dsql = @dsql + N' + WHERE h.event_time > @max_event_time'; + END; + + /* Add the ORDER BY clause */ + SET @dsql = @dsql + N' ORDER BY h.event_time DESC - OPTION(RECOMPILE); + OPTION(RECOMPILE);'; + + /* Handle table logging */ + IF @log_to_table = 1 + BEGIN + SET @insert_sql = N' + INSERT INTO + ' + @log_table_system_health + N' + ( + event_time, + state, + spinlockBackoffs, + sickSpinlockType, + sickSpinlockTypeAfterAv, + latchWarnings, + isAccessViolationOccurred, + writeAccessViolationCount, + totalDumpRequests, + intervalDumpRequests, + nonYieldingTasksReported, + pageFaults, + systemCpuUtilization, + sqlCpuUtilization, + BadPagesDetected, + BadPagesFixed + )' + + @dsql; + + IF @debug = 1 BEGIN PRINT @insert_sql; END; + + EXECUTE sys.sp_executesql @insert_sql, + N'@max_event_time datetime2(7)', + @max_event_time; + END; + + /* Execute the query for client results */ + IF @log_to_table = 0 + BEGIN + IF @debug = 1 BEGIN PRINT @dsql; END; + + EXECUTE sys.sp_executesql + @dsql; + END; END; END; /*End system*/ @@ -2351,6 +3825,7 @@ AND ca.utc_timestamp < @end_date'; x.event_time DESC; END; + /* Scheduler monitor issues logging section */ IF NOT EXISTS ( SELECT @@ -2358,6 +3833,7 @@ AND ca.utc_timestamp < @end_date'; FROM #scheduler_issues AS si ) BEGIN + /* No results logic, only return if not logging */ SELECT finding = CASE @@ -2373,12 +3849,20 @@ AND ca.utc_timestamp < @end_date'; RTRIM(@warnings_only) + '.' ELSE 'no scheduler issues found!' - END; + END + WHERE @log_to_table = 0; END; ELSE BEGIN + /* Build the query */ + SET @dsql = N' SELECT - finding = 'scheduler monitor issues', + ' + CASE + WHEN @log_to_table = 1 + THEN N'' + ELSE N'finding = ''scheduler monitor issues'',' + END + + N' si.event_time, si.scheduler_id, si.cpu_id, @@ -2399,8 +3883,8 @@ AND ca.utc_timestamp < @end_date'; ), 1 ), - N'.00', - N'' + N''.00'', + N'''' ), thread_quantum_ms = REPLACE @@ -2415,14 +3899,95 @@ AND ca.utc_timestamp < @end_date'; ), 1 ), - N'.00', - N'' + N''.00'', + N'''' ) - FROM #scheduler_issues AS si + FROM #scheduler_issues AS si'; + + /* Add the WHERE clause only for table logging */ + IF @log_to_table = 1 + BEGIN + /* Get max event_time for scheduler issues */ + SET @mdsql = + REPLACE + ( + REPLACE + ( + @mdsql, + '{table_check}', + @log_table_scheduler_issues + ), + '{date_column}', + 'event_time' + ); + + IF @debug = 1 BEGIN PRINT @mdsql; END; + + EXECUTE sys.sp_executesql + @mdsql, + N'@max_event_time datetime2(7) OUTPUT', + @max_event_time OUTPUT; + + /* Reset @mdsql to original template */ + SET @mdsql = + REPLACE + ( + REPLACE + ( + @mdsql, + @log_table_scheduler_issues, + '{table_check}' + ), + 'event_time', + '{date_column}' + ); + + SET @dsql = @dsql + N' + WHERE si.event_time > @max_event_time'; + END; + + /* Add the ORDER BY clause */ + SET @dsql = @dsql + N' ORDER BY si.event_time DESC - OPTION(RECOMPILE); + OPTION(RECOMPILE);'; + + /* Handle table logging */ + IF @log_to_table = 1 + BEGIN + SET @insert_sql = N' + INSERT INTO + ' + @log_table_scheduler_issues + N' + ( + event_time, + scheduler_id, + cpu_id, + status, + is_online, + is_runnable, + is_running, + non_yielding_time_ms, + thread_quantum_ms + )' + + @dsql; + + IF @debug = 1 BEGIN PRINT @insert_sql; END; + + EXECUTE sys.sp_executesql @insert_sql, + N'@max_event_time datetime2(7)', + @max_event_time; + END; + + /* Execute the query for client results */ + IF @log_to_table = 0 + BEGIN + IF @debug = 1 BEGIN PRINT @dsql; END; + + EXECUTE sys.sp_executesql + @dsql; + END; END; + END; /*End scheduler monitor analysis*/ /*Parse error_reported data*/ @@ -2485,6 +4050,7 @@ AND ca.utc_timestamp < @end_date'; x.event_time DESC; END; + /* Severe errors reported logging section */ IF NOT EXISTS ( SELECT @@ -2492,6 +4058,7 @@ AND ca.utc_timestamp < @end_date'; FROM #error_info AS ei ) BEGIN + /* No results logic, only return if not logging */ SELECT finding = CASE @@ -2508,15 +4075,19 @@ AND ca.utc_timestamp < @end_date'; '.' ELSE 'no severe errors found!' END - UNION ALL - SELECT - 'Error Number Ignored: ' + CONVERT(nvarchar(100), ie.error_number) - FROM #ignore_errors AS ie; + WHERE @log_to_table = 0; END; ELSE BEGIN + /* Build the query */ + SET @dsql = N' SELECT - finding = 'severe errors reported', + ' + CASE + WHEN @log_to_table = 1 + THEN N'' + ELSE N'finding = ''severe errors reported'',' + END + + N' ei.event_time, ei.error_number, ei.severity, @@ -2524,11 +4095,97 @@ AND ca.utc_timestamp < @end_date'; ei.message, ei.database_name, ei.database_id - FROM #error_info AS ei + FROM #error_info AS ei'; + + /* Add the WHERE clause only for table logging */ + IF @log_to_table = 1 + BEGIN + /* Get max event_time for severe errors */ + SET @mdsql = + REPLACE + ( + REPLACE + ( + @mdsql, + '{table_check}', + @log_table_severe_errors + ), + '{date_column}', + 'event_time' + ); + + IF @debug = 1 BEGIN PRINT @mdsql; END; + + EXECUTE sys.sp_executesql + @mdsql, + N'@max_event_time datetime2(7) OUTPUT', + @max_event_time OUTPUT; + + /* Reset @mdsql to original template */ + SET @mdsql = + REPLACE + ( + REPLACE + ( + @mdsql, + @log_table_severe_errors, + '{table_check}' + ), + 'event_time', + '{date_column}' + ); + + SET @dsql = @dsql + N' + WHERE ei.event_time > @max_event_time'; + END; + + /* Add the ORDER BY clause */ + SET @dsql = @dsql + N' ORDER BY ei.event_time DESC, ei.severity DESC - OPTION(RECOMPILE); + OPTION(RECOMPILE);'; + + /* Handle table logging */ + IF @log_to_table = 1 + BEGIN + SET @insert_sql = N' + INSERT INTO + ' + @log_table_severe_errors + N' + ( + event_time, + error_number, + severity, + state, + message, + database_name, + database_id + )' + + @dsql; + + IF @debug = 1 BEGIN PRINT @insert_sql; END; + + EXECUTE sys.sp_executesql @insert_sql, + N'@max_event_time datetime2(7)', + @max_event_time; + END; + + /* Execute the query for client results */ + IF @log_to_table = 0 + BEGIN + IF @debug = 1 BEGIN PRINT @dsql; END; + + EXECUTE sys.sp_executesql + @dsql; + END; + + /* For ignored errors, only display to client */ + IF @log_to_table = 0 + BEGIN + SELECT + 'Error Number Ignored: ' + CONVERT(nvarchar(100), ie.error_number) + FROM #ignore_errors AS ie; + END; END; END; /*End error_reported analysis*/ @@ -2611,6 +4268,7 @@ AND ca.utc_timestamp < @end_date'; ( @what_to_check IN ('all', 'locking') AND @skip_locks = 0 + AND @log_to_table = 0 ) BEGIN IF @debug = 1 From bb2e263ba5a1902ee53ddc12098a0ac2657b8106 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 10 Mar 2025 10:03:26 -0400 Subject: [PATCH 018/246] Add delete logic to PD gotta clean up after outselves --- sp_HealthParser/sp_HealthParser.sql | 8 ++- sp_PressureDetector/sp_PressureDetector.sql | 77 +++++++++++++++++++-- 2 files changed, 77 insertions(+), 8 deletions(-) diff --git a/sp_HealthParser/sp_HealthParser.sql b/sp_HealthParser/sp_HealthParser.sql index 70338b9e..b74052b0 100644 --- a/sp_HealthParser/sp_HealthParser.sql +++ b/sp_HealthParser/sp_HealthParser.sql @@ -150,6 +150,7 @@ BEGIN WHEN N'@log_database_name' THEN N'NULL (current database)' WHEN N'@log_schema_name' THEN N'NULL (dbo)' WHEN N'@log_table_name_prefix' THEN N'HealthParser' + WHEN N'@log_retention_days' THEN N'30' WHEN N'@version' THEN N'none; OUTPUT' WHEN N'@version_date' THEN N'none; OUTPUT' WHEN N'@help' THEN N'0' @@ -1062,7 +1063,7 @@ AND ca.utc_timestamp < @end_date'; BEGIN IF @debug = 1 BEGIN - RAISERROR('Cleaning up log tables older than %i', 0, 1, @log_retention_days) WITH NOWAIT; + RAISERROR('Cleaning up log tables older than %i days', 0, 1, @log_retention_days) WITH NOWAIT; END; SET @cleanup_date = @@ -1110,7 +1111,10 @@ AND ca.utc_timestamp < @end_date'; IF @debug = 1 BEGIN PRINT @dsql; END; - EXECUTE sys.sp_executesql @dsql, N'@cleanup_date datetime2(7)', @cleanup_date; + EXECUTE sys.sp_executesql + @dsql, + N'@cleanup_date datetime2(7)', + @cleanup_date; IF @debug = 1 BEGIN diff --git a/sp_PressureDetector/sp_PressureDetector.sql b/sp_PressureDetector/sp_PressureDetector.sql index f0a66317..d120d553 100644 --- a/sp_PressureDetector/sp_PressureDetector.sql +++ b/sp_PressureDetector/sp_PressureDetector.sql @@ -61,6 +61,7 @@ ALTER PROCEDURE @log_database_name sysname = NULL, /*database to store logging tables*/ @log_schema_name sysname = NULL, /*schema to store logging tables*/ @log_table_name_prefix sysname = 'PressureDetector', /*prefix for all logging tables*/ + @log_retention_days integer = 30, /*Number of days to keep logs, 0 = keep indefinitely*/ @help bit = 0, /*how you got here*/ @debug bit = 0, /*prints dynamic sql, displays parameter and variable values, and table contents*/ @version varchar(5) = NULL OUTPUT, /*OUTPUT; for support*/ @@ -118,6 +119,7 @@ BEGIN WHEN N'@log_database_name' THEN N'database to store logging tables' WHEN N'@log_schema_name' THEN N'schema to store logging tables' WHEN N'@log_table_name_prefix' THEN N'prefix for all logging tables' + WHEN N'@log_retention_days' THEN N'how many days of data to retain' WHEN N'@help' THEN N'how you got here' WHEN N'@debug' THEN N'prints dynamic sql, displays parameter and variable values, and table contents' WHEN N'@version' THEN N'OUTPUT; for support' @@ -138,6 +140,7 @@ BEGIN WHEN N'@log_database_name' THEN N'any valid database name' WHEN N'@log_schema_name' THEN N'any valid schema name' WHEN N'@log_table_name_prefix' THEN N'any valid identifier' + WHEN N'@log_retention_days' THEN N'a positive integer' WHEN N'@help' THEN N'0 or 1' WHEN N'@debug' THEN N'0 or 1' WHEN N'@version' THEN N'none' @@ -158,15 +161,16 @@ BEGIN WHEN N'@log_database_name' THEN N'NULL (current database)' WHEN N'@log_schema_name' THEN N'NULL (dbo)' WHEN N'@log_table_name_prefix' THEN N'PressureDetector' + WHEN N'@log_retention_days' THEN N'30' WHEN N'@help' THEN N'0' WHEN N'@debug' THEN N'0' WHEN N'@version' THEN N'none; OUTPUT' WHEN N'@version_date' THEN N'none; OUTPUT' END FROM sys.all_parameters AS ap - INNER JOIN sys.all_objects AS o + JOIN sys.all_objects AS o ON ap.object_id = o.object_id - INNER JOIN sys.types AS t + JOIN sys.types AS t ON ap.system_type_id = t.system_type_id AND ap.user_type_id = t.user_type_id WHERE o.name = N'sp_PressureDetector' @@ -440,9 +444,11 @@ OPTION(MAXDOP 1, RECOMPILE);', @log_table_memory_queries sysname, @log_table_cpu_queries sysname, @log_table_cpu_events sysname, - @check_sql nvarchar(max), - @create_sql nvarchar(max), - @insert_sql nvarchar(max), + @cleanup_date datetime2(7), + @check_sql nvarchar(max) = N'', + @create_sql nvarchar(max) = N'', + @insert_sql nvarchar(max) = N'', + @delete_sql nvarchar(max) = N'', @log_database_schema nvarchar(1024); /* Validate logging parameters */ @@ -931,7 +937,66 @@ OPTION(MAXDOP 1, RECOMPILE);', @debug bit', @log_schema_name, @log_table_name_prefix, - @debug; + @debug; + + /* Handle log retention if specified */ + IF @log_to_table = 1 AND @log_retention_days > 0 + BEGIN + IF @debug = 1 + BEGIN + RAISERROR('Cleaning up log tables older than %i days', 0, 1, @log_retention_days) WITH NOWAIT; + END; + + SET @cleanup_date = + DATEADD + ( + DAY, + -@log_retention_days, + SYSDATETIME() + ); + + /* Clean up each log table */ + SET @delete_sql = N' + DELETE FROM ' + @log_table_waits + ' + WHERE collection_time < @cleanup_date; + + DELETE FROM ' + @log_table_file_metrics + ' + WHERE collection_time < @cleanup_date; + + DELETE FROM ' + @log_table_perfmon + ' + WHERE collection_time < @cleanup_date; + + DELETE FROM ' + @log_table_memory + ' + WHERE collection_time < @cleanup_date; + + DELETE FROM ' + @log_table_cpu + ' + WHERE collection_time < @cleanup_date; + + DELETE FROM ' + @log_table_memory_consumers + ' + WHERE collection_time < @cleanup_date; + + DELETE FROM ' + @log_table_memory_queries + ' + WHERE collection_time < @cleanup_date; + + DELETE FROM ' + @log_table_cpu_queries + ' + WHERE collection_time < @cleanup_date; + + DELETE FROM ' + @log_table_cpu_events + ' + WHERE collection_time < @cleanup_date;'; + + IF @debug = 1 BEGIN PRINT @delete_sql; END; + + EXECUTE sys.sp_executesql + @delete_sql, + N'@cleanup_date datetime2(7)', + @cleanup_date; + + IF @debug = 1 + BEGIN + RAISERROR('Log cleanup complete', 0, 1) WITH NOWAIT; + END; + END; + END; /*End log to tables validation checks here*/ DECLARE From ed836d0593cdf75465870fe86ce501f8f27b1e5a Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 10 Mar 2025 14:02:08 -0400 Subject: [PATCH 019/246] Stuff! Things! HP: * Tidying up HEBV: * Add table logging QS: * Push the include query hash totals stuff deeper than just the final select results --- sp_HealthParser/sp_HealthParser.sql | 55 +- sp_HumanEvents/sp_HumanEventsBlockViewer.sql | 2507 ++++++++++-------- sp_QuickieStore/sp_QuickieStore.sql | 508 ++-- 3 files changed, 1815 insertions(+), 1255 deletions(-) diff --git a/sp_HealthParser/sp_HealthParser.sql b/sp_HealthParser/sp_HealthParser.sql index b74052b0..57bb2b99 100644 --- a/sp_HealthParser/sp_HealthParser.sql +++ b/sp_HealthParser/sp_HealthParser.sql @@ -1891,8 +1891,9 @@ AND ca.utc_timestamp < @end_date'; IF @debug = 1 BEGIN PRINT @insert_sql; END; - EXECUTE sys.sp_executesql @insert_sql, - N'@max_event_time datetime2(7)', + EXECUTE sys.sp_executesql + @insert_sql, + N'@max_event_time datetime2(7)', @max_event_time; END; @@ -2145,8 +2146,9 @@ AND ca.utc_timestamp < @end_date'; IF @debug = 1 BEGIN PRINT @insert_sql; END; - EXECUTE sys.sp_executesql @insert_sql, - N'@max_event_time datetime2(7)', + EXECUTE sys.sp_executesql + @insert_sql, + N'@max_event_time datetime2(7)', @max_event_time; END; @@ -2430,8 +2432,9 @@ AND ca.utc_timestamp < @end_date'; IF @debug = 1 BEGIN PRINT @insert_sql; END; - EXECUTE sys.sp_executesql @insert_sql, - N'@max_event_time datetime2(7)', + EXECUTE sys.sp_executesql + @insert_sql, + N'@max_event_time datetime2(7)', @max_event_time; END; @@ -2643,8 +2646,9 @@ AND ca.utc_timestamp < @end_date'; IF @debug = 1 BEGIN PRINT @insert_sql; END; - EXECUTE sys.sp_executesql @insert_sql, - N'@max_event_time datetime2(7)', + EXECUTE sys.sp_executesql + @insert_sql, + N'@max_event_time datetime2(7)', @max_event_time; END; @@ -2835,8 +2839,9 @@ END; IF @debug = 1 BEGIN PRINT @insert_sql; END; - EXECUTE sys.sp_executesql @insert_sql, - N'@max_event_time datetime2(7)', + EXECUTE sys.sp_executesql + @insert_sql, + N'@max_event_time datetime2(7)', @max_event_time; END; @@ -3084,8 +3089,9 @@ END; IF @debug = 1 BEGIN PRINT @insert_sql; END; - EXECUTE sys.sp_executesql @insert_sql, - N'@max_event_time datetime2(7)', + EXECUTE sys.sp_executesql + @insert_sql, + N'@max_event_time datetime2(7)', @max_event_time; END; @@ -3292,8 +3298,9 @@ BEGIN IF @debug = 1 BEGIN PRINT @insert_sql; END; - EXECUTE sys.sp_executesql @insert_sql, - N'@max_event_time datetime2(7)', + EXECUTE sys.sp_executesql + @insert_sql, + N'@max_event_time datetime2(7)', @max_event_time; END; @@ -3565,8 +3572,9 @@ END; IF @debug = 1 BEGIN PRINT @insert_sql; END; - EXECUTE sys.sp_executesql @insert_sql, - N'@max_event_time datetime2(7)', + EXECUTE sys.sp_executesql + @insert_sql, + N'@max_event_time datetime2(7)', @max_event_time; END; @@ -3767,8 +3775,9 @@ END; IF @debug = 1 BEGIN PRINT @insert_sql; END; - EXECUTE sys.sp_executesql @insert_sql, - N'@max_event_time datetime2(7)', + EXECUTE sys.sp_executesql + @insert_sql, + N'@max_event_time datetime2(7)', @max_event_time; END; @@ -3977,8 +3986,9 @@ END; IF @debug = 1 BEGIN PRINT @insert_sql; END; - EXECUTE sys.sp_executesql @insert_sql, - N'@max_event_time datetime2(7)', + EXECUTE sys.sp_executesql + @insert_sql, + N'@max_event_time datetime2(7)', @max_event_time; END; @@ -4169,8 +4179,9 @@ END; IF @debug = 1 BEGIN PRINT @insert_sql; END; - EXECUTE sys.sp_executesql @insert_sql, - N'@max_event_time datetime2(7)', + EXECUTE sys.sp_executesql + @insert_sql, + N'@max_event_time datetime2(7)', @max_event_time; END; diff --git a/sp_HumanEvents/sp_HumanEventsBlockViewer.sql b/sp_HumanEvents/sp_HumanEventsBlockViewer.sql index d37b913a..6cadf19c 100644 --- a/sp_HumanEvents/sp_HumanEventsBlockViewer.sql +++ b/sp_HumanEvents/sp_HumanEventsBlockViewer.sql @@ -73,6 +73,11 @@ ALTER PROCEDURE @target_table sysname = NULL, /*table name*/ @target_column sysname = NULL, /*column containing XML data*/ @timestamp_column sysname = NULL, /*column containing timestamp (optional)*/ + @log_to_table bit = 0, /*enable logging to permanent tables*/ + @log_database_name sysname = NULL, /*database to store logging tables*/ + @log_schema_name sysname = NULL, /*schema to store logging tables*/ + @log_table_name_prefix sysname = 'HumanEventsBlockViewer', /*prefix for all logging tables*/ + @log_retention_days integer = 30, /*Number of days to keep logs, 0 = keep indefinitely*/ @help bit = 0, /*get help with this procedure*/ @debug bit = 0, /*print dynamic sql and select temp table contents*/ @version varchar(30) = NULL OUTPUT, /*check the version number*/ @@ -120,6 +125,11 @@ BEGIN WHEN N'@target_table' THEN 'table containing blocked process report data' WHEN N'@target_column' THEN 'column containing blocked process report XML' WHEN N'@timestamp_column' THEN 'column containing timestamp for filtering (optional)' + WHEN N'@log_to_table' THEN N'enable logging to permanent tables instead of returning results' + WHEN N'@log_database_name' THEN N'database to store logging tables' + WHEN N'@log_schema_name' THEN N'schema to store logging tables' + WHEN N'@log_table_name_prefix' THEN N'prefix for all logging tables' + WHEN N'@log_retention_days' THEN N'how many days of data to retain' WHEN N'@help' THEN 'how you got here' WHEN N'@debug' THEN 'dumps raw temp table contents' WHEN N'@version' THEN 'OUTPUT; for support' @@ -138,6 +148,11 @@ BEGIN WHEN N'@target_table' THEN 'a table in the target schema' WHEN N'@target_column' THEN 'an XML column containing blocked process report data' WHEN N'@timestamp_column' THEN 'a datetime column for filtering by date range' + WHEN N'@log_to_table' THEN N'0 or 1' + WHEN N'@log_database_name' THEN N'any valid database name' + WHEN N'@log_schema_name' THEN N'any valid schema name' + WHEN N'@log_table_name_prefix' THEN N'any valid identifier' + WHEN N'@log_retention_days' THEN N'a positive integer' WHEN N'@help' THEN '0 or 1' WHEN N'@debug' THEN '0 or 1' WHEN N'@version' THEN 'none; OUTPUT' @@ -156,6 +171,11 @@ BEGIN WHEN N'@target_table' THEN 'NULL' WHEN N'@target_column' THEN 'NULL' WHEN N'@timestamp_column' THEN 'NULL' + WHEN N'@log_to_table' THEN N'0' + WHEN N'@log_database_name' THEN N'NULL (current database)' + WHEN N'@log_schema_name' THEN N'NULL (dbo)' + WHEN N'@log_table_name_prefix' THEN N'HumanEventsBlockViewer' + WHEN N'@log_retention_days' THEN N'30' WHEN N'@help' THEN '0' WHEN N'@debug' THEN '0' WHEN N'@version' THEN 'none; OUTPUT' @@ -305,8 +325,18 @@ DECLARE CONVERT(nvarchar(1), 0x0a00, 0), @start_date_original datetime2 = @start_date, @end_date_original datetime2 = @end_date, - @validation_sql nvarchar(MAX), - @extract_sql nvarchar(MAX); + @validation_sql nvarchar(max), + @extract_sql nvarchar(max), + /*Log to table stuff*/ + @log_table_blocking sysname, + @cleanup_date datetime2(7), + @check_sql nvarchar(max) = N'', + @create_sql nvarchar(max) = N'', + @insert_sql nvarchar(max) = N'', + @log_database_schema nvarchar(1024), + @max_event_time datetime2(7), + @dsql nvarchar(max) = N'', + @mdsql nvarchar(max) = N''; /*Use some sane defaults for input parameters*/ IF @debug = 1 @@ -380,7 +410,34 @@ SELECT WHEN @session_name LIKE N'system%health' THEN 1 ELSE 0 - END; + END, + @mdsql = N' +IF OBJECT_ID(''{table_check}'', ''U'') IS NOT NULL +BEGIN + SELECT + @max_event_time = + ISNULL + ( + MAX({date_column}), + DATEADD + ( + MINUTE, + DATEDIFF + ( + MINUTE, + SYSDATETIME(), + GETUTCDATE() + ), + DATEADD + ( + DAY, + -1, + SYSDATETIME() + ) + ) + ) + FROM {table_check}; +END;'; SELECT @azure_msg = @@ -440,6 +497,7 @@ BEGIN /* Use dynamic SQL to validate schema, table, and column existence */ SET @validation_sql = N' + /*Validate schema exists*/ IF NOT EXISTS ( SELECT @@ -452,6 +510,7 @@ BEGIN RETURN; END; + /*Validate table exists*/ IF NOT EXISTS ( SELECT @@ -467,10 +526,11 @@ BEGIN RETURN; END; + /*Validate column name exists*/ IF NOT EXISTS ( SELECT - 1 + 1/0 FROM ' + QUOTENAME(@target_database) + N'.sys.columns AS c JOIN ' + QUOTENAME(@target_database) + N'.sys.tables AS t ON c.object_id = t.object_id @@ -530,7 +590,7 @@ BEGIN RETURN; END; - /* Validate timestamp column is datetime type */ + /* Validate timestamp column is date-ish type */ IF NOT EXISTS ( SELECT @@ -574,6 +634,167 @@ BEGIN @timestamp_column; END; +/* Validate logging parameters */ +IF @log_to_table = 1 +BEGIN + SELECT + /* Default database name to current database if not specified */ + @log_database_name = ISNULL(@log_database_name, DB_NAME()), + /* Default schema name to dbo if not specified */ + @log_schema_name = ISNULL(@log_schema_name, N'dbo'), + @log_retention_days = + CASE + WHEN @log_retention_days < 0 + THEN ABS(@log_retention_days) + ELSE @log_retention_days + END; + + /* Validate database exists */ + IF NOT EXISTS + ( + SELECT + 1/0 + FROM sys.databases AS d + WHERE d.name = @log_database_name + ) + BEGIN + RAISERROR('The specified logging database %s does not exist. Logging will be disabled.', 11, 1, @log_database_name) WITH NOWAIT; + RETURN; + END; + + SET + @log_database_schema = + QUOTENAME(@log_database_name) + + N'.' + + QUOTENAME(@log_schema_name) + + N'.'; + + /* Generate fully qualified table names */ + SELECT + @log_table_blocking = + @log_database_schema + + QUOTENAME(@log_table_name_prefix + N'_BlockedProcessReport'); + + /* Check if schema exists and create it if needed */ + SET @check_sql = N' + IF NOT EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@log_database_name) + N'.sys.schemas AS s + WHERE s.name = @schema_name + ) + BEGIN + DECLARE + @create_schema_sql nvarchar(max) = N''CREATE SCHEMA '' + QUOTENAME(@schema_name); + + EXECUTE ' + QUOTENAME(@log_database_name) + N'.sys.sp_executesql @create_schema_sql; + IF @debug = 1 BEGIN RAISERROR(''Created schema %s in database %s for logging.'', 0, 1, @schema_name, @db_name) WITH NOWAIT; END; + END'; + + EXECUTE sys.sp_executesql + @check_sql, + N'@schema_name sysname, + @db_name sysname, + @debug bit', + @log_schema_name, + @log_database_name, + @debug; + + SET @create_sql = N' + IF NOT EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@log_database_name) + N'.sys.tables AS t + JOIN ' + QUOTENAME(@log_database_name) + N'.sys.schemas AS s + ON t.schema_id = s.schema_id + WHERE t.name = @table_name + N''_BlockedProcessReport'' + AND s.name = @schema_name + ) + BEGIN + CREATE TABLE ' + @log_table_blocking + N' + ( + id bigint IDENTITY, + collection_time datetime2(7) NOT NULL DEFAULT SYSDATETIME(), + blocked_process_report varchar(22) NOT NULL, + event_time datetime2(7) NULL, + database_name nvarchar(128) NULL, + currentdbname nvarchar(256) NULL, + contentious_object nvarchar(4000) NULL, + activity varchar(8) NULL, + blocking_tree varchar(8000) NULL, + spid int NULL, + ecid int NULL, + query_text xml NULL, + wait_time_ms bigint NULL, + status nvarchar(10) NULL, + isolation_level nvarchar(50) NULL, + lock_mode nvarchar(10) NULL, + resource_owner_type nvarchar(256) NULL, + transaction_count int NULL, + transaction_name nvarchar(512) NULL, + last_transaction_started datetime2(7) NULL, + last_transaction_completed datetime2(7) NULL, + client_option_1 varchar(261) NULL, + client_option_2 varchar(307) NULL, + wait_resource nvarchar(100) NULL, + priority int NULL, + log_used bigint NULL, + client_app nvarchar(256) NULL, + host_name nvarchar(256) NULL, + login_name nvarchar(256) NULL, + transaction_id bigint NULL, + blocked_process_report_xml xml NULL + PRIMARY KEY CLUSTERED (collection_time, id) + ); + IF @debug = 1 BEGIN RAISERROR(''Created table %s for significant waits logging.'', 0, 1, ''' + @log_table_blocking + N''') WITH NOWAIT; END; + END'; + + EXECUTE sys.sp_executesql + @create_sql, + N'@schema_name sysname, + @table_name sysname, + @debug bit', + @log_schema_name, + @log_table_name_prefix, + @debug; + + /* Handle log retention if specified */ + IF @log_to_table = 1 AND @log_retention_days > 0 + BEGIN + IF @debug = 1 + BEGIN + RAISERROR('Cleaning up log tables older than %i days', 0, 1, @log_retention_days) WITH NOWAIT; + END; + + SET @cleanup_date = + DATEADD + ( + DAY, + -@log_retention_days, + SYSDATETIME() + ); + + /* Clean up each log table */ + SET @dsql = N' + DELETE FROM ' + @log_table_blocking + ' + WHERE collection_time < @cleanup_date;'; + + IF @debug = 1 BEGIN PRINT @dsql; END; + + EXECUTE sys.sp_executesql + @dsql, + N'@cleanup_date datetime2(7)', + @cleanup_date; + + IF @debug = 1 + BEGIN + RAISERROR('Log cleanup complete', 0, 1) WITH NOWAIT; + END; + END; +END; + /*Temp tables for staging results*/ IF @debug = 1 BEGIN @@ -903,9 +1124,17 @@ END; /* This section is special for the well-hidden and much less comprehensive blocked process report stored in the system health extended event session + +Note: I do not allow logging to a table from this, because the set of columns +and available data is too incomplete, and I don't want to juggle multiple +table definitions. + +Logging to a table is only allowed from the a blocked_process_report Extended Event, +but it can either be ring buffer or file target. I don't care about that. */ IF @is_system_health = 1 AND LOWER(@target_type) <> N'table' +AND @log_to_table = 0 BEGIN IF @debug = 1 BEGIN @@ -974,7 +1203,7 @@ BEGIN currentdbname = bd.value('(process/@currentdbname)[1]', 'nvarchar(128)'), spid = bd.value('(process/@spid)[1]', 'integer'), ecid = bd.value('(process/@ecid)[1]', 'integer'), - query_text_pre = bd.value('(process/inputbuf/text())[1]', 'nvarchar(MAX)'), + query_text_pre = bd.value('(process/inputbuf/text())[1]', 'nvarchar(max)'), wait_time = bd.value('(process/@waittime)[1]', 'bigint'), lastbatchstarted = bd.value('(process/@lastbatchstarted)[1]', 'datetime2'), lastbatchcompleted = bd.value('(process/@lastbatchcompleted)[1]', 'datetime2'), @@ -1031,7 +1260,7 @@ BEGIN currentdbname = bg.value('(process/@currentdbname)[1]', 'nvarchar(128)'), spid = bg.value('(process/@spid)[1]', 'integer'), ecid = bg.value('(process/@ecid)[1]', 'integer'), - query_text_pre = bg.value('(process/inputbuf/text())[1]', 'nvarchar(MAX)'), + query_text_pre = bg.value('(process/inputbuf/text())[1]', 'nvarchar(max)'), wait_time = bg.value('(process/@waittime)[1]', 'bigint'), last_transaction_started = bg.value('(process/@lastbatchstarted)[1]', 'datetime2'), last_transaction_completed = bg.value('(process/@lastbatchcompleted)[1]', 'datetime2'), @@ -1275,7 +1504,7 @@ BEGIN 'available_plans', b.currentdbname, query_text = - TRY_CAST(b.query_text AS nvarchar(MAX)), + TRY_CAST(b.query_text AS nvarchar(max)), sql_handle = CONVERT(varbinary(64), n.c.value('@sqlhandle', 'varchar(130)'), 1), stmtstart = @@ -1294,7 +1523,7 @@ BEGIN 'available_plans', b.currentdbname, query_text = - TRY_CAST(b.query_text AS nvarchar(MAX)), + TRY_CAST(b.query_text AS nvarchar(max)), sql_handle = CONVERT(varbinary(64), n.c.value('@sqlhandle', 'varchar(130)'), 1), stmtstart = @@ -1586,7 +1815,7 @@ SELECT blocking_ecid = bg.value('(process/@ecid)[1]', 'integer'), blocked_spid = bd.value('(process/@spid)[1]', 'integer'), blocked_ecid = bd.value('(process/@ecid)[1]', 'integer'), - query_text_pre = bd.value('(process/inputbuf/text())[1]', 'nvarchar(MAX)'), + query_text_pre = bd.value('(process/inputbuf/text())[1]', 'nvarchar(max)'), wait_time = bd.value('(process/@waittime)[1]', 'bigint'), transaction_name = bd.value('(process/@transactionname)[1]', 'nvarchar(512)'), last_transaction_started = bd.value('(process/@lasttranstarted)[1]', 'datetime2'), @@ -1706,7 +1935,7 @@ SELECT blocking_ecid = bg.value('(process/@ecid)[1]', 'integer'), blocked_spid = bd.value('(process/@spid)[1]', 'integer'), blocked_ecid = bd.value('(process/@ecid)[1]', 'integer'), - query_text_pre = bg.value('(process/inputbuf/text())[1]', 'nvarchar(MAX)'), + query_text_pre = bg.value('(process/inputbuf/text())[1]', 'nvarchar(max)'), wait_time = bg.value('(process/@waittime)[1]', 'bigint'), transaction_name = bg.value('(process/@transactionname)[1]', 'nvarchar(512)'), last_transaction_started = bg.value('(process/@lastbatchstarted)[1]', 'datetime2'), @@ -2110,9 +2339,11 @@ CROSS APPLY ) AS co OPTION(RECOMPILE); +/*Either return results or log to a table*/ +SET @dsql = N' SELECT blocked_process_report = - 'blocked_process_report', + ''blocked_process_report'', b.event_time, b.database_name, b.currentdbname, @@ -2140,7 +2371,8 @@ SELECT b.host_name, b.login_name, b.transaction_id, - blocked_process_report_xml = b.blocked_process_report + blocked_process_report_xml = + b.blocked_process_report FROM ( SELECT @@ -2160,926 +2392,1013 @@ FROM WHERE b.n = 1 AND (b.contentious_object = @object_name OR @object_name IS NULL) + +'; + +/* Add the WHERE clause only for table logging */ +IF @log_to_table = 1 +BEGIN + SET @mdsql = + REPLACE + ( + REPLACE + ( + @mdsql, + '{table_check}', + @log_table_blocking + ), + '{date_column}', + 'event_time' + ); + + IF @debug = 1 BEGIN PRINT @mdsql; END; + + EXECUTE sys.sp_executesql + @mdsql, + N'@max_event_time datetime2(7) OUTPUT', + @max_event_time OUTPUT; + + SET @mdsql = + REPLACE + ( + REPLACE + ( + @mdsql, + @log_table_blocking, + '{table_check}' + ), + 'event_time', + '{date_column}' + ); + + SET @dsql += N' +AND b.event_time > @max_event_time'; +END; + +/* Add the ORDER BY clause */ +SET @dsql += N' ORDER BY b.event_time, b.sort_order, CASE - WHEN b.activity = 'blocking' + WHEN b.activity = ''blocking'' THEN -1 ELSE +1 END -OPTION(RECOMPILE); +OPTION(RECOMPILE);'; + +/* Handle table logging */ +IF @log_to_table = 1 +BEGIN + SET @insert_sql = N' +INSERT INTO + ' + @log_table_blocking + N' +( + blocked_process_report, + event_time, + database_name, + currentdbname, + contentious_object, + activity, + blocking_tree, + spid, + ecid, + query_text, + wait_time_ms, + status, + isolation_level, + lock_mode, + resource_owner_type, + transaction_count, + transaction_name, + last_transaction_started, + last_transaction_completed, + client_option_1, + client_option_2, + wait_resource, + priority, + log_used, + client_app, + host_name, + login_name, + transaction_id, + blocked_process_report_xml +)' + + @dsql; + + IF @debug = 1 BEGIN PRINT @insert_sql; END; + + EXECUTE sys.sp_executesql + @insert_sql, + N'@max_event_time datetime2(7), + @object_name sysname', + @max_event_time, + @object_name; +END; -IF @debug = 1 -BEGIN - RAISERROR('Inserting #available_plans', 0, 1) WITH NOWAIT; +/* Execute the query for client results */ +IF @log_to_table = 0 +BEGIN + + IF @debug = 1 BEGIN PRINT @dsql; END; + + EXECUTE sys.sp_executesql + @dsql, + N'@object_name sysname', + @object_name; END; -SELECT DISTINCT - b.* -INTO #available_plans -FROM -( +/* +Only run query plan and check stuff +when not logging to a table +*/ +IF @log_to_table = 0 +BEGIN + IF @debug = 1 + BEGIN + RAISERROR('Inserting #available_plans', 0, 1) WITH NOWAIT; + END; + + SELECT DISTINCT + b.* + INTO #available_plans + FROM + ( + SELECT + available_plans = + 'available_plans', + b.database_name, + b.database_id, + b.currentdbname, + b.currentdbid, + b.contentious_object, + query_text = + TRY_CAST(b.query_text AS nvarchar(max)), + sql_handle = + CONVERT(varbinary(64), n.c.value('@sqlhandle', 'varchar(130)'), 1), + stmtstart = + ISNULL(n.c.value('@stmtstart', 'integer'), 0), + stmtend = + ISNULL(n.c.value('@stmtend', 'integer'), -1) + FROM #blocks AS b + CROSS APPLY b.blocked_process_report.nodes('/event/data/value/blocked-process-report/blocking-process/process/executionStack/frame[not(@sqlhandle = "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")]') AS n(c) + WHERE + ( + (b.database_name = @database_name + OR @database_name IS NULL) + OR (b.currentdbname = @database_name + OR @database_name IS NULL) + ) + AND (b.contentious_object = @object_name + OR @object_name IS NULL) + + UNION ALL + + SELECT + available_plans = + 'available_plans', + b.database_name, + b.database_id, + b.currentdbname, + b.currentdbid, + b.contentious_object, + query_text = + TRY_CAST(b.query_text AS nvarchar(max)), + sql_handle = + CONVERT(varbinary(64), n.c.value('@sqlhandle', 'varchar(130)'), 1), + stmtstart = + ISNULL(n.c.value('@stmtstart', 'integer'), 0), + stmtend = + ISNULL(n.c.value('@stmtend', 'integer'), -1) + FROM #blocks AS b + CROSS APPLY b.blocked_process_report.nodes('/event/data/value/blocked-process-report/blocking-process/process/executionStack/frame[not(@sqlhandle = "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")]') AS n(c) + WHERE + ( + (b.database_name = @database_name + OR @database_name IS NULL) + OR (b.currentdbname = @database_name + OR @database_name IS NULL) + ) + AND (b.contentious_object = @object_name + OR @object_name IS NULL) + ) AS b + OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + SELECT + '#available_plans' AS table_name, + ap.* + FROM #available_plans AS ap + OPTION(RECOMPILE); + + RAISERROR('Inserting #dm_exec_query_stats', 0, 1) WITH NOWAIT; + END; + SELECT - available_plans = - 'available_plans', - b.database_name, - b.database_id, - b.currentdbname, - b.currentdbid, - b.contentious_object, - query_text = - TRY_CAST(b.query_text AS nvarchar(MAX)), - sql_handle = - CONVERT(varbinary(64), n.c.value('@sqlhandle', 'varchar(130)'), 1), - stmtstart = - ISNULL(n.c.value('@stmtstart', 'integer'), 0), - stmtend = - ISNULL(n.c.value('@stmtend', 'integer'), -1) - FROM #blocks AS b - CROSS APPLY b.blocked_process_report.nodes('/event/data/value/blocked-process-report/blocking-process/process/executionStack/frame[not(@sqlhandle = "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")]') AS n(c) - WHERE + deqs.sql_handle, + deqs.plan_handle, + deqs.statement_start_offset, + deqs.statement_end_offset, + deqs.creation_time, + deqs.last_execution_time, + deqs.execution_count, + total_worker_time_ms = + deqs.total_worker_time / 1000., + avg_worker_time_ms = + CONVERT(decimal(38, 6), deqs.total_worker_time / 1000. / deqs.execution_count), + total_elapsed_time_ms = + deqs.total_elapsed_time / 1000., + avg_elapsed_time_ms = + CONVERT(decimal(38, 6), deqs.total_elapsed_time / 1000. / deqs.execution_count), + executions_per_second = + ISNULL + ( + deqs.execution_count / + NULLIF + ( + DATEDIFF + ( + SECOND, + deqs.creation_time, + NULLIF(deqs.last_execution_time, '1900-01-01 00:00:00.000') + ), + 0 + ), + 0 + ), + total_physical_reads_mb = + deqs.total_physical_reads * 8. / 1024., + total_logical_writes_mb = + deqs.total_logical_writes * 8. / 1024., + total_logical_reads_mb = + deqs.total_logical_reads * 8. / 1024., + min_grant_mb = + deqs.min_grant_kb * 8. / 1024., + max_grant_mb = + deqs.max_grant_kb * 8. / 1024., + min_used_grant_mb = + deqs.min_used_grant_kb * 8. / 1024., + max_used_grant_mb = + deqs.max_used_grant_kb * 8. / 1024., + deqs.min_reserved_threads, + deqs.max_reserved_threads, + deqs.min_used_threads, + deqs.max_used_threads, + deqs.total_rows, + max_worker_time_ms = + deqs.max_worker_time / 1000., + max_elapsed_time_ms = + deqs.max_elapsed_time / 1000. + INTO #dm_exec_query_stats + FROM sys.dm_exec_query_stats AS deqs + WHERE EXISTS ( - (b.database_name = @database_name - OR @database_name IS NULL) - OR (b.currentdbname = @database_name - OR @database_name IS NULL) + SELECT + 1/0 + FROM #available_plans AS ap + WHERE ap.sql_handle = deqs.sql_handle ) - AND (b.contentious_object = @object_name - OR @object_name IS NULL) - - UNION ALL + AND deqs.query_hash IS NOT NULL + OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + RAISERROR('Creating index on #dm_exec_query_stats', 0, 1) WITH NOWAIT; + END; + + CREATE CLUSTERED INDEX + deqs + ON #dm_exec_query_stats + ( + sql_handle, + plan_handle + ); SELECT - available_plans = - 'available_plans', - b.database_name, - b.database_id, - b.currentdbname, - b.currentdbid, - b.contentious_object, + ap.available_plans, + ap.database_name, + ap.currentdbname, query_text = - TRY_CAST(b.query_text AS nvarchar(MAX)), - sql_handle = - CONVERT(varbinary(64), n.c.value('@sqlhandle', 'varchar(130)'), 1), - stmtstart = - ISNULL(n.c.value('@stmtstart', 'integer'), 0), - stmtend = - ISNULL(n.c.value('@stmtend', 'integer'), -1) - FROM #blocks AS b - CROSS APPLY b.blocked_process_report.nodes('/event/data/value/blocked-process-report/blocking-process/process/executionStack/frame[not(@sqlhandle = "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")]') AS n(c) - WHERE + TRY_CAST(ap.query_text AS xml), + ap.query_plan, + ap.creation_time, + ap.last_execution_time, + ap.execution_count, + ap.executions_per_second, + ap.total_worker_time_ms, + ap.avg_worker_time_ms, + ap.max_worker_time_ms, + ap.total_elapsed_time_ms, + ap.avg_elapsed_time_ms, + ap.max_elapsed_time_ms, + ap.total_logical_reads_mb, + ap.total_physical_reads_mb, + ap.total_logical_writes_mb, + ap.min_grant_mb, + ap.max_grant_mb, + ap.min_used_grant_mb, + ap.max_used_grant_mb, + ap.min_reserved_threads, + ap.max_reserved_threads, + ap.min_used_threads, + ap.max_used_threads, + ap.total_rows, + ap.sql_handle, + ap.statement_start_offset, + ap.statement_end_offset + FROM ( - (b.database_name = @database_name - OR @database_name IS NULL) - OR (b.currentdbname = @database_name - OR @database_name IS NULL) - ) - AND (b.contentious_object = @object_name - OR @object_name IS NULL) -) AS b -OPTION(RECOMPILE); - -IF @debug = 1 -BEGIN - SELECT - '#available_plans' AS table_name, - ap.* - FROM #available_plans AS ap - OPTION(RECOMPILE); - - RAISERROR('Inserting #dm_exec_query_stats', 0, 1) WITH NOWAIT; -END; - -SELECT - deqs.sql_handle, - deqs.plan_handle, - deqs.statement_start_offset, - deqs.statement_end_offset, - deqs.creation_time, - deqs.last_execution_time, - deqs.execution_count, - total_worker_time_ms = - deqs.total_worker_time / 1000., - avg_worker_time_ms = - CONVERT(decimal(38, 6), deqs.total_worker_time / 1000. / deqs.execution_count), - total_elapsed_time_ms = - deqs.total_elapsed_time / 1000., - avg_elapsed_time_ms = - CONVERT(decimal(38, 6), deqs.total_elapsed_time / 1000. / deqs.execution_count), - executions_per_second = - ISNULL + + SELECT + ap.*, + c.statement_start_offset, + c.statement_end_offset, + c.creation_time, + c.last_execution_time, + c.execution_count, + c.total_worker_time_ms, + c.avg_worker_time_ms, + c.total_elapsed_time_ms, + c.avg_elapsed_time_ms, + c.executions_per_second, + c.total_physical_reads_mb, + c.total_logical_writes_mb, + c.total_logical_reads_mb, + c.min_grant_mb, + c.max_grant_mb, + c.min_used_grant_mb, + c.max_used_grant_mb, + c.min_reserved_threads, + c.max_reserved_threads, + c.min_used_threads, + c.max_used_threads, + c.total_rows, + c.query_plan, + c.max_worker_time_ms, + c.max_elapsed_time_ms + FROM #available_plans AS ap + OUTER APPLY ( - deqs.execution_count / - NULLIF - ( - DATEDIFF - ( - SECOND, - deqs.creation_time, - NULLIF(deqs.last_execution_time, '1900-01-01 00:00:00.000') - ), - 0 - ), - 0 - ), - total_physical_reads_mb = - deqs.total_physical_reads * 8. / 1024., - total_logical_writes_mb = - deqs.total_logical_writes * 8. / 1024., - total_logical_reads_mb = - deqs.total_logical_reads * 8. / 1024., - min_grant_mb = - deqs.min_grant_kb * 8. / 1024., - max_grant_mb = - deqs.max_grant_kb * 8. / 1024., - min_used_grant_mb = - deqs.min_used_grant_kb * 8. / 1024., - max_used_grant_mb = - deqs.max_used_grant_kb * 8. / 1024., - deqs.min_reserved_threads, - deqs.max_reserved_threads, - deqs.min_used_threads, - deqs.max_used_threads, - deqs.total_rows, - max_worker_time_ms = - deqs.max_worker_time / 1000., - max_elapsed_time_ms = - deqs.max_elapsed_time / 1000. -INTO #dm_exec_query_stats -FROM sys.dm_exec_query_stats AS deqs -WHERE EXISTS -( - SELECT - 1/0 - FROM #available_plans AS ap - WHERE ap.sql_handle = deqs.sql_handle -) -AND deqs.query_hash IS NOT NULL -OPTION(RECOMPILE); - -IF @debug = 1 -BEGIN - RAISERROR('Creating index on #dm_exec_query_stats', 0, 1) WITH NOWAIT; -END; - -CREATE CLUSTERED INDEX - deqs -ON #dm_exec_query_stats -( - sql_handle, - plan_handle -); - -SELECT - ap.available_plans, - ap.database_name, - ap.currentdbname, - query_text = - TRY_CAST(ap.query_text AS xml), - ap.query_plan, - ap.creation_time, - ap.last_execution_time, - ap.execution_count, - ap.executions_per_second, - ap.total_worker_time_ms, - ap.avg_worker_time_ms, - ap.max_worker_time_ms, - ap.total_elapsed_time_ms, - ap.avg_elapsed_time_ms, - ap.max_elapsed_time_ms, - ap.total_logical_reads_mb, - ap.total_physical_reads_mb, - ap.total_logical_writes_mb, - ap.min_grant_mb, - ap.max_grant_mb, - ap.min_used_grant_mb, - ap.max_used_grant_mb, - ap.min_reserved_threads, - ap.max_reserved_threads, - ap.min_used_threads, - ap.max_used_threads, - ap.total_rows, - ap.sql_handle, - ap.statement_start_offset, - ap.statement_end_offset -FROM -( - + SELECT + deqs.*, + query_plan = + TRY_CAST(deps.query_plan AS xml) + FROM #dm_exec_query_stats deqs + OUTER APPLY sys.dm_exec_text_query_plan + ( + deqs.plan_handle, + deqs.statement_start_offset, + deqs.statement_end_offset + ) AS deps + WHERE deqs.sql_handle = ap.sql_handle + AND deps.dbid IN (ap.database_id, ap.currentdbid) + ) AS c + ) AS ap + WHERE ap.query_plan IS NOT NULL + ORDER BY + ap.avg_worker_time_ms DESC + OPTION(RECOMPILE, LOOP JOIN, HASH JOIN); + + IF @debug = 1 + BEGIN + RAISERROR('Inserting #block_findings, check_id -1', 0, 1) WITH NOWAIT; + END; + + INSERT + #block_findings + ( + check_id, + database_name, + object_name, + finding_group, + finding, + sort_order + ) SELECT - ap.*, - c.statement_start_offset, - c.statement_end_offset, - c.creation_time, - c.last_execution_time, - c.execution_count, - c.total_worker_time_ms, - c.avg_worker_time_ms, - c.total_elapsed_time_ms, - c.avg_elapsed_time_ms, - c.executions_per_second, - c.total_physical_reads_mb, - c.total_logical_writes_mb, - c.total_logical_reads_mb, - c.min_grant_mb, - c.max_grant_mb, - c.min_used_grant_mb, - c.max_used_grant_mb, - c.min_reserved_threads, - c.max_reserved_threads, - c.min_used_threads, - c.max_used_threads, - c.total_rows, - c.query_plan, - c.max_worker_time_ms, - c.max_elapsed_time_ms - FROM #available_plans AS ap - OUTER APPLY + check_id = -1, + database_name = N'erikdarling.com', + object_name = N'sp_HumanEventsBlockViewer version ' + CONVERT(nvarchar(30), @version) + N'.', + finding_group = N'https://github.com/erikdarlingdata/DarlingData', + finding = N'blocking for period ' + CONVERT(nvarchar(30), @start_date_original, 126) + N' through ' + CONVERT(nvarchar(30), @end_date_original, 126) + N'.', + 1; + + IF @debug = 1 + BEGIN + RAISERROR('Inserting #block_findings, check_id 1', 0, 1) WITH NOWAIT; + END; + + INSERT + #block_findings ( - SELECT - deqs.*, - query_plan = - TRY_CAST(deps.query_plan AS xml) - FROM #dm_exec_query_stats deqs - OUTER APPLY sys.dm_exec_text_query_plan - ( - deqs.plan_handle, - deqs.statement_start_offset, - deqs.statement_end_offset - ) AS deps - WHERE deqs.sql_handle = ap.sql_handle - AND deps.dbid IN (ap.database_id, ap.currentdbid) - ) AS c -) AS ap -WHERE ap.query_plan IS NOT NULL -ORDER BY - ap.avg_worker_time_ms DESC -OPTION(RECOMPILE, LOOP JOIN, HASH JOIN); - -IF @debug = 1 -BEGIN - RAISERROR('Inserting #block_findings, check_id -1', 0, 1) WITH NOWAIT; -END; - -INSERT - #block_findings -( - check_id, - database_name, - object_name, - finding_group, - finding, - sort_order -) -SELECT - check_id = -1, - database_name = N'erikdarling.com', - object_name = N'sp_HumanEventsBlockViewer version ' + CONVERT(nvarchar(30), @version) + N'.', - finding_group = N'https://github.com/erikdarlingdata/DarlingData', - finding = N'blocking for period ' + CONVERT(nvarchar(30), @start_date_original, 126) + N' through ' + CONVERT(nvarchar(30), @end_date_original, 126) + N'.', - 1; - -IF @debug = 1 -BEGIN - RAISERROR('Inserting #block_findings, check_id 1', 0, 1) WITH NOWAIT; -END; - -INSERT - #block_findings -( - check_id, - database_name, - object_name, - finding_group, - finding, - sort_order -) -SELECT - check_id = - 1, - database_name = - b.database_name, - object_name = - N'-', - finding_group = - N'Database Locks', - finding = - N'The database ' + - b.database_name + - N' has been involved in ' + - CONVERT(nvarchar(20), COUNT_BIG(DISTINCT b.transaction_id)) + - N' blocking sessions.', - sort_order = - ROW_NUMBER() OVER (ORDER BY COUNT_BIG(DISTINCT b.transaction_id) DESC) -FROM #blocks AS b -WHERE (b.database_name = @database_name - OR @database_name IS NULL) -AND (b.contentious_object = @object_name - OR @object_name IS NULL) -GROUP BY - b.database_name -OPTION(RECOMPILE); - -IF @debug = 1 -BEGIN - RAISERROR('Inserting #block_findings, check_id 2', 0, 1) WITH NOWAIT; -END; - -INSERT - #block_findings -( - check_id, - database_name, - object_name, - finding_group, - finding, - sort_order -) -SELECT - check_id = - 2, - database_name = - b.database_name, - object_name = - b.contentious_object, - finding_group = - N'Object Locks', - finding = - N'The object ' + - b.contentious_object + - CASE - WHEN b.contentious_object LIKE N'Unresolved%' - THEN N'' - ELSE N' in database ' + - b.database_name - END + - N' has been involved in ' + - CONVERT(nvarchar(20), COUNT_BIG(DISTINCT b.transaction_id)) + - N' blocking sessions.', - sort_order = - ROW_NUMBER() OVER (ORDER BY COUNT_BIG(DISTINCT b.transaction_id) DESC) -FROM #blocks AS b -WHERE (b.database_name = @database_name - OR @database_name IS NULL) -AND (b.contentious_object = @object_name - OR @object_name IS NULL) -GROUP BY - b.database_name, - b.contentious_object -OPTION(RECOMPILE); - -IF @debug = 1 -BEGIN - RAISERROR('Inserting #block_findings, check_id 3', 0, 1) WITH NOWAIT; -END; - -INSERT - #block_findings -( - check_id, - database_name, - object_name, - finding_group, - finding, - sort_order -) -SELECT - check_id = - 3, - database_name = - b.database_name, - object_name = - CASE - WHEN EXISTS - ( - SELECT - 1/0 - FROM sys.databases AS d - WHERE d.name COLLATE DATABASE_DEFAULT = b.database_name COLLATE DATABASE_DEFAULT - AND d.is_read_committed_snapshot_on = 1 - ) - THEN N'You already enabled RCSI, but...' - ELSE N'You Might Need RCSI' - END, - finding_group = - N'Blocking Involving Selects', - finding = - N'There have been ' + - CONVERT(nvarchar(20), COUNT_BIG(DISTINCT b.transaction_id)) + - N' select queries involved in blocking sessions in ' + - b.database_name + - N'.', - sort_order = - ROW_NUMBER() OVER (ORDER BY COUNT_BIG(DISTINCT b.transaction_id) DESC) -FROM #blocks AS b -WHERE b.lock_mode IN - ( - N'S', - N'IS' - ) -AND (b.database_name = @database_name - OR @database_name IS NULL) -AND (b.contentious_object = @object_name - OR @object_name IS NULL) -GROUP BY - b.database_name -HAVING - COUNT_BIG(DISTINCT b.transaction_id) > 1 -OPTION(RECOMPILE); - -IF @debug = 1 -BEGIN - RAISERROR('Inserting #block_findings, check_id 4', 0, 1) WITH NOWAIT; -END; - -INSERT - #block_findings -( - check_id, - database_name, - object_name, - finding_group, - finding, - sort_order -) -SELECT - check_id = - 4, - database_name = - b.database_name, - object_name = - N'-', - finding_group = - N'Repeatable Read Blocking', - finding = - N'There have been ' + - CONVERT(nvarchar(20), COUNT_BIG(DISTINCT b.transaction_id)) + - N' repeatable read queries involved in blocking sessions in ' + - b.database_name + - N'.', - sort_order = - ROW_NUMBER() OVER (ORDER BY COUNT_BIG(DISTINCT b.transaction_id) DESC) -FROM #blocks AS b -WHERE b.isolation_level LIKE N'repeatable%' -AND (b.database_name = @database_name - OR @database_name IS NULL) -AND (b.contentious_object = @object_name - OR @object_name IS NULL) -GROUP BY - b.database_name -OPTION(RECOMPILE); - -IF @debug = 1 -BEGIN - RAISERROR('Inserting #block_findings, check_id 5', 0, 1) WITH NOWAIT; -END; - -INSERT - #block_findings -( - check_id, - database_name, - object_name, - finding_group, - finding, - sort_order -) -SELECT - check_id = - 5, - database_name = - b.database_name, - object_name = - N'-', - finding_group = - N'Serializable Blocking', - finding = - N'There have been ' + - CONVERT(nvarchar(20), COUNT_BIG(DISTINCT b.transaction_id)) + - N' serializable queries involved in blocking sessions in ' + - b.database_name + - N'.', - sort_order = - ROW_NUMBER() OVER (ORDER BY COUNT_BIG(DISTINCT b.transaction_id) DESC) -FROM #blocks AS b -WHERE b.isolation_level LIKE N'serializable%' -AND (b.database_name = @database_name - OR @database_name IS NULL) -AND (b.contentious_object = @object_name - OR @object_name IS NULL) -GROUP BY - b.database_name -OPTION(RECOMPILE); - -IF @debug = 1 -BEGIN - RAISERROR('Inserting #block_findings, check_id 6.1', 0, 1) WITH NOWAIT; -END; - -INSERT - #block_findings -( - check_id, - database_name, - object_name, - finding_group, - finding, - sort_order -) -SELECT - check_id = - 6, - database_name = - b.database_name, - object_name = - N'-', - finding_group = - N'Sleeping Query Blocking', - finding = - N'There have been ' + - CONVERT(nvarchar(20), COUNT_BIG(DISTINCT b.transaction_id)) + - N' sleeping queries involved in blocking sessions in ' + - b.database_name + - N'.', - sort_order = - ROW_NUMBER() OVER (ORDER BY COUNT_BIG(DISTINCT b.transaction_id) DESC) -FROM #blocks AS b -WHERE b.status = N'sleeping' -AND (b.database_name = @database_name - OR @database_name IS NULL) -AND (b.contentious_object = @object_name - OR @object_name IS NULL) -GROUP BY - b.database_name -OPTION(RECOMPILE); - -IF @debug = 1 -BEGIN - RAISERROR('Inserting #block_findings, check_id 6.2', 0, 1) WITH NOWAIT; -END; - -INSERT - #block_findings -( - check_id, - database_name, - object_name, - finding_group, - finding, - sort_order -) -SELECT - check_id = - 6, - database_name = - b.database_name, - object_name = - N'-', - finding_group = - N'Background Query Blocking', - finding = - N'There have been ' + - CONVERT(nvarchar(20), COUNT_BIG(DISTINCT b.transaction_id)) + - N' background tasks involved in blocking sessions in ' + - b.database_name + - N'.', - sort_order = - ROW_NUMBER() OVER (ORDER BY COUNT_BIG(DISTINCT b.transaction_id) DESC) -FROM #blocks AS b -WHERE b.status = N'background' -AND (b.database_name = @database_name - OR @database_name IS NULL) -AND (b.contentious_object = @object_name - OR @object_name IS NULL) -GROUP BY - b.database_name -OPTION(RECOMPILE); - -IF @debug = 1 -BEGIN - RAISERROR('Inserting #block_findings, check_id 6.3', 0, 1) WITH NOWAIT; -END; - -INSERT - #block_findings -( - check_id, - database_name, - object_name, - finding_group, - finding, - sort_order -) -SELECT - check_id = - 6, - database_name = - b.database_name, - object_name = - N'-', - finding_group = - N'Done Query Blocking', - finding = - N'There have been ' + - CONVERT(nvarchar(20), COUNT_BIG(DISTINCT b.transaction_id)) + - N' background tasks involved in blocking sessions in ' + - b.database_name + - N'.', - sort_order = - ROW_NUMBER() OVER (ORDER BY COUNT_BIG(DISTINCT b.transaction_id) DESC) -FROM #blocks AS b -WHERE b.status = N'done' -AND (b.database_name = @database_name - OR @database_name IS NULL) -AND (b.contentious_object = @object_name - OR @object_name IS NULL) -GROUP BY - b.database_name -OPTION(RECOMPILE); - -IF @debug = 1 -BEGIN - RAISERROR('Inserting #block_findings, check_id 6.4', 0, 1) WITH NOWAIT; -END; - -INSERT - #block_findings -( - check_id, - database_name, - object_name, - finding_group, - finding, - sort_order -) -SELECT - check_id = - 6, - database_name = - b.database_name, - object_name = - N'-', - finding_group = - N'Compile Lock Blocking', - finding = - N'There have been ' + - CONVERT(nvarchar(20), COUNT_BIG(DISTINCT b.transaction_id)) + - N' compile locks blocking sessions in ' + - b.database_name + - N'.', - sort_order = - ROW_NUMBER() OVER (ORDER BY COUNT_BIG(DISTINCT b.transaction_id) DESC) -FROM #blocks AS b -WHERE b.wait_resource LIKE N'%COMPILE%' -AND (b.database_name = @database_name - OR @database_name IS NULL) -AND (b.contentious_object = @object_name - OR @object_name IS NULL) -GROUP BY - b.database_name -OPTION(RECOMPILE); - -IF @debug = 1 -BEGIN - RAISERROR('Inserting #block_findings, check_id 6.5', 0, 1) WITH NOWAIT; -END; - -INSERT - #block_findings -( - check_id, - database_name, - object_name, - finding_group, - finding, - sort_order -) -SELECT - check_id = - 6, - database_name = - b.database_name, - object_name = - N'-', - finding_group = - N'Application Lock Blocking', - finding = - N'There have been ' + - CONVERT(nvarchar(20), COUNT_BIG(DISTINCT b.transaction_id)) + - N' application locks blocking sessions in ' + - b.database_name + - N'.', - sort_order = - ROW_NUMBER() OVER (ORDER BY COUNT_BIG(DISTINCT b.transaction_id) DESC) -FROM #blocks AS b -WHERE b.wait_resource LIKE N'APPLICATION%' -AND (b.database_name = @database_name - OR @database_name IS NULL) -AND (b.contentious_object = @object_name - OR @object_name IS NULL) -GROUP BY - b.database_name -OPTION(RECOMPILE); - -IF @debug = 1 -BEGIN - RAISERROR('Inserting #block_findings, check_id 7.1', 0, 1) WITH NOWAIT; -END; - -INSERT - #block_findings -( - check_id, - database_name, - object_name, - finding_group, - finding, - sort_order -) -SELECT - check_id = - 7, - database_name = - b.database_name, - object_name = - N'-', - finding_group = - N'Implicit Transaction Blocking', - finding = - N'There have been ' + - CONVERT(nvarchar(20), COUNT_BIG(DISTINCT b.transaction_id)) + - N' implicit transaction queries involved in blocking sessions in ' + - b.database_name + - N'.', - sort_order = - ROW_NUMBER() OVER (ORDER BY COUNT_BIG(DISTINCT b.transaction_id) DESC) -FROM #blocks AS b -WHERE b.transaction_name = N'implicit_transaction' -AND (b.database_name = @database_name - OR @database_name IS NULL) -AND (b.contentious_object = @object_name - OR @object_name IS NULL) -GROUP BY - b.database_name -OPTION(RECOMPILE); - -IF @debug = 1 -BEGIN - RAISERROR('Inserting #block_findings, check_id 7.2', 0, 1) WITH NOWAIT; -END; - -INSERT - #block_findings -( - check_id, - database_name, - object_name, - finding_group, - finding, - sort_order -) -SELECT - check_id = - 7, - database_name = - b.database_name, - object_name = - N'-', - finding_group = - N'User Transaction Blocking', - finding = - N'There have been ' + - CONVERT(nvarchar(20), COUNT_BIG(DISTINCT b.transaction_id)) + - N' user transaction queries involved in blocking sessions in ' + - b.database_name + - N'.', - sort_order = - ROW_NUMBER() OVER (ORDER BY COUNT_BIG(DISTINCT b.transaction_id) DESC) -FROM #blocks AS b -WHERE b.transaction_name = N'user_transaction' -AND (b.database_name = @database_name - OR @database_name IS NULL) -AND (b.contentious_object = @object_name - OR @object_name IS NULL) -GROUP BY - b.database_name -OPTION(RECOMPILE); - -IF @debug = 1 -BEGIN - RAISERROR('Inserting #block_findings, check_id 7.3', 0, 1) WITH NOWAIT; -END; - -INSERT - #block_findings -( - check_id, - database_name, - object_name, - finding_group, - finding, - sort_order -) -SELECT - check_id = - 7, - database_name = + check_id, + database_name, + object_name, + finding_group, + finding, + sort_order + ) + SELECT + check_id = + 1, + database_name = + b.database_name, + object_name = + N'-', + finding_group = + N'Database Locks', + finding = + N'The database ' + + b.database_name + + N' has been involved in ' + + CONVERT(nvarchar(20), COUNT_BIG(DISTINCT b.transaction_id)) + + N' blocking sessions.', + sort_order = + ROW_NUMBER() OVER (ORDER BY COUNT_BIG(DISTINCT b.transaction_id) DESC) + FROM #blocks AS b + WHERE (b.database_name = @database_name + OR @database_name IS NULL) + AND (b.contentious_object = @object_name + OR @object_name IS NULL) + GROUP BY + b.database_name + OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + RAISERROR('Inserting #block_findings, check_id 2', 0, 1) WITH NOWAIT; + END; + + INSERT + #block_findings + ( + check_id, + database_name, + object_name, + finding_group, + finding, + sort_order + ) + SELECT + check_id = + 2, + database_name = + b.database_name, + object_name = + b.contentious_object, + finding_group = + N'Object Locks', + finding = + N'The object ' + + b.contentious_object + + CASE + WHEN b.contentious_object LIKE N'Unresolved%' + THEN N'' + ELSE N' in database ' + + b.database_name + END + + N' has been involved in ' + + CONVERT(nvarchar(20), COUNT_BIG(DISTINCT b.transaction_id)) + + N' blocking sessions.', + sort_order = + ROW_NUMBER() OVER (ORDER BY COUNT_BIG(DISTINCT b.transaction_id) DESC) + FROM #blocks AS b + WHERE (b.database_name = @database_name + OR @database_name IS NULL) + AND (b.contentious_object = @object_name + OR @object_name IS NULL) + GROUP BY b.database_name, - object_name = - N'-', - finding_group = - N'Auto-Stats Update Blocking', - finding = - N'There have been ' + - CONVERT(nvarchar(20), COUNT_BIG(DISTINCT b.transaction_id)) + - N' auto stats updates involved in blocking sessions in ' + - b.database_name + - N'.', - sort_order = - ROW_NUMBER() OVER (ORDER BY COUNT_BIG(DISTINCT b.transaction_id) DESC) -FROM #blocks AS b -WHERE b.transaction_name = N'sqlsource_transform' -AND (b.database_name = @database_name - OR @database_name IS NULL) -AND (b.contentious_object = @object_name - OR @object_name IS NULL) -GROUP BY - b.database_name -OPTION(RECOMPILE); - -IF @debug = 1 -BEGIN - RAISERROR('Inserting #block_findings, check_id 8', 0, 1) WITH NOWAIT; -END; - -INSERT - #block_findings -( - check_id, - database_name, - object_name, - finding_group, - finding, - sort_order -) -SELECT - check_id = 8, - b.database_name, - object_name = N'-', - finding_group = N'Login, App, and Host blocking', - finding = - N'This database has had ' + - CONVERT - ( - nvarchar(20), - COUNT_BIG(DISTINCT b.transaction_id) - ) + - N' instances of blocking involving the login ' + - ISNULL - ( - b.login_name, - N'UNKNOWN' - ) + - N' from the application ' + - ISNULL - ( - b.client_app, - N'UNKNOWN' - ) + - N' on host ' + - ISNULL - ( - b.host_name, - N'UNKNOWN' - ) + - N'.', - sort_order = - ROW_NUMBER() OVER (ORDER BY COUNT_BIG(DISTINCT b.transaction_id) DESC) -FROM #blocks AS b -WHERE (b.database_name = @database_name - OR @database_name IS NULL) -AND (b.contentious_object = @object_name - OR @object_name IS NULL) -GROUP BY - b.database_name, - b.login_name, - b.client_app, - b.host_name -OPTION(RECOMPILE); + b.contentious_object + OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + RAISERROR('Inserting #block_findings, check_id 3', 0, 1) WITH NOWAIT; + END; + + INSERT + #block_findings + ( + check_id, + database_name, + object_name, + finding_group, + finding, + sort_order + ) + SELECT + check_id = + 3, + database_name = + b.database_name, + object_name = + CASE + WHEN EXISTS + ( + SELECT + 1/0 + FROM sys.databases AS d + WHERE d.name COLLATE DATABASE_DEFAULT = b.database_name COLLATE DATABASE_DEFAULT + AND d.is_read_committed_snapshot_on = 1 + ) + THEN N'You already enabled RCSI, but...' + ELSE N'You Might Need RCSI' + END, + finding_group = + N'Blocking Involving Selects', + finding = + N'There have been ' + + CONVERT(nvarchar(20), COUNT_BIG(DISTINCT b.transaction_id)) + + N' select queries involved in blocking sessions in ' + + b.database_name + + N'.', + sort_order = + ROW_NUMBER() OVER (ORDER BY COUNT_BIG(DISTINCT b.transaction_id) DESC) + FROM #blocks AS b + WHERE b.lock_mode IN + ( + N'S', + N'IS' + ) + AND (b.database_name = @database_name + OR @database_name IS NULL) + AND (b.contentious_object = @object_name + OR @object_name IS NULL) + GROUP BY + b.database_name + HAVING + COUNT_BIG(DISTINCT b.transaction_id) > 1 + OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + RAISERROR('Inserting #block_findings, check_id 4', 0, 1) WITH NOWAIT; + END; + + INSERT + #block_findings + ( + check_id, + database_name, + object_name, + finding_group, + finding, + sort_order + ) + SELECT + check_id = + 4, + database_name = + b.database_name, + object_name = + N'-', + finding_group = + N'Repeatable Read Blocking', + finding = + N'There have been ' + + CONVERT(nvarchar(20), COUNT_BIG(DISTINCT b.transaction_id)) + + N' repeatable read queries involved in blocking sessions in ' + + b.database_name + + N'.', + sort_order = + ROW_NUMBER() OVER (ORDER BY COUNT_BIG(DISTINCT b.transaction_id) DESC) + FROM #blocks AS b + WHERE b.isolation_level LIKE N'repeatable%' + AND (b.database_name = @database_name + OR @database_name IS NULL) + AND (b.contentious_object = @object_name + OR @object_name IS NULL) + GROUP BY + b.database_name + OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + RAISERROR('Inserting #block_findings, check_id 5', 0, 1) WITH NOWAIT; + END; + + INSERT + #block_findings + ( + check_id, + database_name, + object_name, + finding_group, + finding, + sort_order + ) + SELECT + check_id = + 5, + database_name = + b.database_name, + object_name = + N'-', + finding_group = + N'Serializable Blocking', + finding = + N'There have been ' + + CONVERT(nvarchar(20), COUNT_BIG(DISTINCT b.transaction_id)) + + N' serializable queries involved in blocking sessions in ' + + b.database_name + + N'.', + sort_order = + ROW_NUMBER() OVER (ORDER BY COUNT_BIG(DISTINCT b.transaction_id) DESC) + FROM #blocks AS b + WHERE b.isolation_level LIKE N'serializable%' + AND (b.database_name = @database_name + OR @database_name IS NULL) + AND (b.contentious_object = @object_name + OR @object_name IS NULL) + GROUP BY + b.database_name + OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + RAISERROR('Inserting #block_findings, check_id 6.1', 0, 1) WITH NOWAIT; + END; + + INSERT + #block_findings + ( + check_id, + database_name, + object_name, + finding_group, + finding, + sort_order + ) + SELECT + check_id = + 6, + database_name = + b.database_name, + object_name = + N'-', + finding_group = + N'Sleeping Query Blocking', + finding = + N'There have been ' + + CONVERT(nvarchar(20), COUNT_BIG(DISTINCT b.transaction_id)) + + N' sleeping queries involved in blocking sessions in ' + + b.database_name + + N'.', + sort_order = + ROW_NUMBER() OVER (ORDER BY COUNT_BIG(DISTINCT b.transaction_id) DESC) + FROM #blocks AS b + WHERE b.status = N'sleeping' + AND (b.database_name = @database_name + OR @database_name IS NULL) + AND (b.contentious_object = @object_name + OR @object_name IS NULL) + GROUP BY + b.database_name + OPTION(RECOMPILE); -IF @debug = 1 -BEGIN - RAISERROR('Inserting #block_findings, check_id 1000', 0, 1) WITH NOWAIT; -END; + IF @debug = 1 + BEGIN + RAISERROR('Inserting #block_findings, check_id 6.2', 0, 1) WITH NOWAIT; + END; + + INSERT + #block_findings + ( + check_id, + database_name, + object_name, + finding_group, + finding, + sort_order + ) + SELECT + check_id = + 6, + database_name = + b.database_name, + object_name = + N'-', + finding_group = + N'Background Query Blocking', + finding = + N'There have been ' + + CONVERT(nvarchar(20), COUNT_BIG(DISTINCT b.transaction_id)) + + N' background tasks involved in blocking sessions in ' + + b.database_name + + N'.', + sort_order = + ROW_NUMBER() OVER (ORDER BY COUNT_BIG(DISTINCT b.transaction_id) DESC) + FROM #blocks AS b + WHERE b.status = N'background' + AND (b.database_name = @database_name + OR @database_name IS NULL) + AND (b.contentious_object = @object_name + OR @object_name IS NULL) + GROUP BY + b.database_name + OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + RAISERROR('Inserting #block_findings, check_id 6.3', 0, 1) WITH NOWAIT; + END; + + INSERT + #block_findings + ( + check_id, + database_name, + object_name, + finding_group, + finding, + sort_order + ) + SELECT + check_id = + 6, + database_name = + b.database_name, + object_name = + N'-', + finding_group = + N'Done Query Blocking', + finding = + N'There have been ' + + CONVERT(nvarchar(20), COUNT_BIG(DISTINCT b.transaction_id)) + + N' background tasks involved in blocking sessions in ' + + b.database_name + + N'.', + sort_order = + ROW_NUMBER() OVER (ORDER BY COUNT_BIG(DISTINCT b.transaction_id) DESC) + FROM #blocks AS b + WHERE b.status = N'done' + AND (b.database_name = @database_name + OR @database_name IS NULL) + AND (b.contentious_object = @object_name + OR @object_name IS NULL) + GROUP BY + b.database_name + OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + RAISERROR('Inserting #block_findings, check_id 6.4', 0, 1) WITH NOWAIT; + END; + + INSERT + #block_findings + ( + check_id, + database_name, + object_name, + finding_group, + finding, + sort_order + ) + SELECT + check_id = + 6, + database_name = + b.database_name, + object_name = + N'-', + finding_group = + N'Compile Lock Blocking', + finding = + N'There have been ' + + CONVERT(nvarchar(20), COUNT_BIG(DISTINCT b.transaction_id)) + + N' compile locks blocking sessions in ' + + b.database_name + + N'.', + sort_order = + ROW_NUMBER() OVER (ORDER BY COUNT_BIG(DISTINCT b.transaction_id) DESC) + FROM #blocks AS b + WHERE b.wait_resource LIKE N'%COMPILE%' + AND (b.database_name = @database_name + OR @database_name IS NULL) + AND (b.contentious_object = @object_name + OR @object_name IS NULL) + GROUP BY + b.database_name + OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + RAISERROR('Inserting #block_findings, check_id 6.5', 0, 1) WITH NOWAIT; + END; + + INSERT + #block_findings + ( + check_id, + database_name, + object_name, + finding_group, + finding, + sort_order + ) + SELECT + check_id = + 6, + database_name = + b.database_name, + object_name = + N'-', + finding_group = + N'Application Lock Blocking', + finding = + N'There have been ' + + CONVERT(nvarchar(20), COUNT_BIG(DISTINCT b.transaction_id)) + + N' application locks blocking sessions in ' + + b.database_name + + N'.', + sort_order = + ROW_NUMBER() OVER (ORDER BY COUNT_BIG(DISTINCT b.transaction_id) DESC) + FROM #blocks AS b + WHERE b.wait_resource LIKE N'APPLICATION%' + AND (b.database_name = @database_name + OR @database_name IS NULL) + AND (b.contentious_object = @object_name + OR @object_name IS NULL) + GROUP BY + b.database_name + OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + RAISERROR('Inserting #block_findings, check_id 7.1', 0, 1) WITH NOWAIT; + END; + + INSERT + #block_findings + ( + check_id, + database_name, + object_name, + finding_group, + finding, + sort_order + ) + SELECT + check_id = + 7, + database_name = + b.database_name, + object_name = + N'-', + finding_group = + N'Implicit Transaction Blocking', + finding = + N'There have been ' + + CONVERT(nvarchar(20), COUNT_BIG(DISTINCT b.transaction_id)) + + N' implicit transaction queries involved in blocking sessions in ' + + b.database_name + + N'.', + sort_order = + ROW_NUMBER() OVER (ORDER BY COUNT_BIG(DISTINCT b.transaction_id) DESC) + FROM #blocks AS b + WHERE b.transaction_name = N'implicit_transaction' + AND (b.database_name = @database_name + OR @database_name IS NULL) + AND (b.contentious_object = @object_name + OR @object_name IS NULL) + GROUP BY + b.database_name + OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + RAISERROR('Inserting #block_findings, check_id 7.2', 0, 1) WITH NOWAIT; + END; + + INSERT + #block_findings + ( + check_id, + database_name, + object_name, + finding_group, + finding, + sort_order + ) + SELECT + check_id = + 7, + database_name = + b.database_name, + object_name = + N'-', + finding_group = + N'User Transaction Blocking', + finding = + N'There have been ' + + CONVERT(nvarchar(20), COUNT_BIG(DISTINCT b.transaction_id)) + + N' user transaction queries involved in blocking sessions in ' + + b.database_name + + N'.', + sort_order = + ROW_NUMBER() OVER (ORDER BY COUNT_BIG(DISTINCT b.transaction_id) DESC) + FROM #blocks AS b + WHERE b.transaction_name = N'user_transaction' + AND (b.database_name = @database_name + OR @database_name IS NULL) + AND (b.contentious_object = @object_name + OR @object_name IS NULL) + GROUP BY + b.database_name + OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + RAISERROR('Inserting #block_findings, check_id 7.3', 0, 1) WITH NOWAIT; + END; -WITH - b AS -( + INSERT + #block_findings + ( + check_id, + database_name, + object_name, + finding_group, + finding, + sort_order + ) + SELECT + check_id = + 7, + database_name = + b.database_name, + object_name = + N'-', + finding_group = + N'Auto-Stats Update Blocking', + finding = + N'There have been ' + + CONVERT(nvarchar(20), COUNT_BIG(DISTINCT b.transaction_id)) + + N' auto stats updates involved in blocking sessions in ' + + b.database_name + + N'.', + sort_order = + ROW_NUMBER() OVER (ORDER BY COUNT_BIG(DISTINCT b.transaction_id) DESC) + FROM #blocks AS b + WHERE b.transaction_name = N'sqlsource_transform' + AND (b.database_name = @database_name + OR @database_name IS NULL) + AND (b.contentious_object = @object_name + OR @object_name IS NULL) + GROUP BY + b.database_name + OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + RAISERROR('Inserting #block_findings, check_id 8', 0, 1) WITH NOWAIT; + END; + + INSERT + #block_findings + ( + check_id, + database_name, + object_name, + finding_group, + finding, + sort_order + ) SELECT + check_id = 8, b.database_name, - b.transaction_id, - wait_time_ms = - MAX(b.wait_time_ms) + object_name = N'-', + finding_group = N'Login, App, and Host blocking', + finding = + N'This database has had ' + + CONVERT + ( + nvarchar(20), + COUNT_BIG(DISTINCT b.transaction_id) + ) + + N' instances of blocking involving the login ' + + ISNULL + ( + b.login_name, + N'UNKNOWN' + ) + + N' from the application ' + + ISNULL + ( + b.client_app, + N'UNKNOWN' + ) + + N' on host ' + + ISNULL + ( + b.host_name, + N'UNKNOWN' + ) + + N'.', + sort_order = + ROW_NUMBER() OVER (ORDER BY COUNT_BIG(DISTINCT b.transaction_id) DESC) FROM #blocks AS b WHERE (b.database_name = @database_name OR @database_name IS NULL) @@ -3087,202 +3406,228 @@ WITH OR @object_name IS NULL) GROUP BY b.database_name, - b.transaction_id -) -INSERT - #block_findings -( - check_id, - database_name, - object_name, - finding_group, - finding, - sort_order -) -SELECT - check_id = - 1000, - b.database_name, - object_name = - N'-', - finding_group = - N'Total database block wait time', - finding = - N'This database has had ' + - CONVERT - ( - nvarchar(30), + b.login_name, + b.client_app, + b.host_name + OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + RAISERROR('Inserting #block_findings, check_id 1000', 0, 1) WITH NOWAIT; + END; + + WITH + b AS + ( + SELECT + b.database_name, + b.transaction_id, + wait_time_ms = + MAX(b.wait_time_ms) + FROM #blocks AS b + WHERE (b.database_name = @database_name + OR @database_name IS NULL) + AND (b.contentious_object = @object_name + OR @object_name IS NULL) + GROUP BY + b.database_name, + b.transaction_id + ) + INSERT + #block_findings + ( + check_id, + database_name, + object_name, + finding_group, + finding, + sort_order + ) + SELECT + check_id = + 1000, + b.database_name, + object_name = + N'-', + finding_group = + N'Total database block wait time', + finding = + N'This database has had ' + + CONVERT ( - SUM + nvarchar(30), ( - CONVERT + SUM ( - bigint, - b.wait_time_ms - ) - ) / 1000 / 86400 - ) - ) + - N' ' + - CONVERT - ( - nvarchar(30), - DATEADD + CONVERT + ( + bigint, + b.wait_time_ms + ) + ) / 1000 / 86400 + ) + ) + + N' ' + + CONVERT ( - MILLISECOND, + nvarchar(30), + DATEADD ( - SUM + MILLISECOND, ( - CONVERT + SUM ( - bigint, - b.wait_time_ms + CONVERT + ( + bigint, + b.wait_time_ms + ) ) - ) + ), + '19000101' ), - '19000101' - ), - 14 - ) + - N' [dd hh:mm:ss:ms] of lock wait time.', - sort_order = - ROW_NUMBER() OVER (ORDER BY SUM(CONVERT(bigint, b.wait_time_ms)) DESC) -FROM b AS b -WHERE (b.database_name = @database_name - OR @database_name IS NULL) -GROUP BY - b.database_name -OPTION(RECOMPILE); - -IF @debug = 1 -BEGIN - RAISERROR('Inserting #block_findings, check_id 1001', 0, 1) WITH NOWAIT; -END; - -WITH - b AS -( - SELECT - b.database_name, - b.transaction_id, - b.contentious_object, - wait_time_ms = - MAX(b.wait_time_ms) - FROM #blocks AS b + 14 + ) + + N' [dd hh:mm:ss:ms] of lock wait time.', + sort_order = + ROW_NUMBER() OVER (ORDER BY SUM(CONVERT(bigint, b.wait_time_ms)) DESC) + FROM b AS b WHERE (b.database_name = @database_name OR @database_name IS NULL) - AND (b.contentious_object = @object_name - OR @object_name IS NULL) GROUP BY + b.database_name + OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + RAISERROR('Inserting #block_findings, check_id 1001', 0, 1) WITH NOWAIT; + END; + + WITH + b AS + ( + SELECT + b.database_name, + b.transaction_id, + b.contentious_object, + wait_time_ms = + MAX(b.wait_time_ms) + FROM #blocks AS b + WHERE (b.database_name = @database_name + OR @database_name IS NULL) + AND (b.contentious_object = @object_name + OR @object_name IS NULL) + GROUP BY + b.database_name, + b.contentious_object, + b.transaction_id + ) + INSERT + #block_findings + ( + check_id, + database_name, + object_name, + finding_group, + finding, + sort_order + ) + SELECT + check_id = + 1001, b.database_name, - b.contentious_object, - b.transaction_id -) -INSERT - #block_findings -( - check_id, - database_name, - object_name, - finding_group, - finding, - sort_order -) -SELECT - check_id = - 1001, - b.database_name, - object_name = - b.contentious_object, - finding_group = - N'Total database and object block wait time', - finding = - N'This object has had ' + - CONVERT - ( - nvarchar(30), + object_name = + b.contentious_object, + finding_group = + N'Total database and object block wait time', + finding = + N'This object has had ' + + CONVERT ( - SUM + nvarchar(30), ( - CONVERT + SUM ( - bigint, - b.wait_time_ms - ) - ) / 1000 / 86400 - ) - ) + - N' ' + - CONVERT - ( - nvarchar(30), - DATEADD + CONVERT + ( + bigint, + b.wait_time_ms + ) + ) / 1000 / 86400 + ) + ) + + N' ' + + CONVERT ( - MILLISECOND, + nvarchar(30), + DATEADD ( - SUM + MILLISECOND, ( - CONVERT + SUM ( - bigint, - b.wait_time_ms + CONVERT + ( + bigint, + b.wait_time_ms + ) ) - ) + ), + '19000101' ), - '19000101' - ), - 14 - ) + - N' [dd hh:mm:ss:ms] of lock wait time in database ' + + 14 + ) + + N' [dd hh:mm:ss:ms] of lock wait time in database ' + + b.database_name, + sort_order = + ROW_NUMBER() OVER (ORDER BY SUM(CONVERT(bigint, b.wait_time_ms)) DESC) + FROM b AS b + WHERE (b.database_name = @database_name + OR @database_name IS NULL) + AND (b.contentious_object = @object_name + OR @object_name IS NULL) + GROUP BY b.database_name, - sort_order = - ROW_NUMBER() OVER (ORDER BY SUM(CONVERT(bigint, b.wait_time_ms)) DESC) -FROM b AS b -WHERE (b.database_name = @database_name - OR @database_name IS NULL) -AND (b.contentious_object = @object_name - OR @object_name IS NULL) -GROUP BY - b.database_name, - b.contentious_object -OPTION(RECOMPILE); - -IF @debug = 1 -BEGIN - RAISERROR('Inserting #block_findings, check_id 2147483647', 0, 1) WITH NOWAIT; + b.contentious_object + OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + RAISERROR('Inserting #block_findings, check_id 2147483647', 0, 1) WITH NOWAIT; + END; + + INSERT + #block_findings + ( + check_id, + database_name, + object_name, + finding_group, + finding, + sort_order + ) + SELECT + check_id = 2147483647, + database_name = N'erikdarling.com', + object_name = N'sp_HumanEventsBlockViewer version ' + CONVERT(nvarchar(30), @version) + N'.', + finding_group = N'https://github.com/erikdarlingdata/DarlingData', + finding = N'thanks for using me!', + 2147483647; + + SELECT + findings = + 'findings', + bf.check_id, + bf.database_name, + bf.object_name, + bf.finding_group, + bf.finding + FROM #block_findings AS bf + ORDER BY + bf.check_id, + bf.finding_group, + bf.sort_order + OPTION(RECOMPILE); END; - -INSERT - #block_findings -( - check_id, - database_name, - object_name, - finding_group, - finding, - sort_order -) -SELECT - check_id = 2147483647, - database_name = N'erikdarling.com', - object_name = N'sp_HumanEventsBlockViewer version ' + CONVERT(nvarchar(30), @version) + N'.', - finding_group = N'https://github.com/erikdarlingdata/DarlingData', - finding = N'thanks for using me!', - 2147483647; - -SELECT - findings = - 'findings', - bf.check_id, - bf.database_name, - bf.object_name, - bf.finding_group, - bf.finding -FROM #block_findings AS bf -ORDER BY - bf.check_id, - bf.finding_group, - bf.sort_order -OPTION(RECOMPILE); END; --Final End GO diff --git a/sp_QuickieStore/sp_QuickieStore.sql b/sp_QuickieStore/sp_QuickieStore.sql index 013e7a46..89f6d8c0 100644 --- a/sp_QuickieStore/sp_QuickieStore.sql +++ b/sp_QuickieStore/sp_QuickieStore.sql @@ -109,6 +109,7 @@ AS BEGIN SET STATISTICS XML OFF; SET NOCOUNT ON; +SET XACT_ABORT OFF; SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; BEGIN TRY @@ -552,10 +553,10 @@ query hash has. CREATE TABLE #plan_ids_with_query_hashes ( - database_id int NOT NULL, + database_id integer NOT NULL, plan_id bigint NOT NULL, query_hash binary(8) NOT NULL, - plan_hash_count_for_query_hash int NOT NULL, + plan_hash_count_for_query_hash integer NOT NULL, PRIMARY KEY CLUSTERED (database_id, plan_id, query_hash) ); @@ -572,7 +573,7 @@ therefore every sp_executesql). CREATE TABLE #plan_ids_with_total_waits ( - database_id int NOT NULL, + database_id integer NOT NULL, plan_id bigint NOT NULL, from_regression_baseline varchar(3) NOT NULL, total_query_wait_time_ms bigint NOT NULL, @@ -618,7 +619,7 @@ on to our final output. CREATE TABLE #regression_changes ( - database_id int NOT NULL, + database_id integer NOT NULL, plan_id bigint NOT NULL, query_hash binary(8) NOT NULL, change_since_regression_time_period float NULL, @@ -791,7 +792,7 @@ Query Store Setup CREATE TABLE #database_query_store_options ( - database_id int NOT NULL, + database_id integer NOT NULL, desired_state_desc nvarchar(60) NULL, actual_state_desc nvarchar(60) NULL, readonly_reason nvarchar(100) NULL, @@ -802,10 +803,10 @@ CREATE TABLE stale_query_threshold_days bigint NULL, max_plans_per_query bigint NULL, query_capture_mode_desc nvarchar(60) NULL, - capture_policy_execution_count int NULL, + capture_policy_execution_count integer NULL, capture_policy_total_compile_cpu_time_ms bigint NULL, capture_policy_total_execution_cpu_time_ms bigint NULL, - capture_policy_stale_threshold_hours int NULL, + capture_policy_stale_threshold_hours integer NULL, size_based_cleanup_mode_desc nvarchar(60) NULL, wait_stats_capture_mode_desc nvarchar(60) NULL ); @@ -816,7 +817,7 @@ Query Store Trouble CREATE TABLE #query_store_trouble ( - database_id int NOT NULL, + database_id integer NOT NULL, desired_state_desc nvarchar(60) NULL, actual_state_desc nvarchar(60) NULL, readonly_reason nvarchar(100) NULL, @@ -836,7 +837,7 @@ Plans and Plan information CREATE TABLE #query_store_plan ( - database_id int NOT NULL, + database_id integer NOT NULL, plan_id bigint NOT NULL, query_id bigint NOT NULL, all_plan_ids varchar(MAX), @@ -870,7 +871,7 @@ Queries and Compile Information CREATE TABLE #query_store_query ( - database_id int NOT NULL, + database_id integer NOT NULL, query_id bigint NOT NULL, query_text_id bigint NOT NULL, context_settings_id bigint NOT NULL, @@ -947,7 +948,7 @@ Query Text And Columns From sys.dm_exec_query_stats CREATE TABLE #query_store_query_text ( - database_id int NOT NULL, + database_id integer NOT NULL, query_text_id bigint NOT NULL, query_sql_text xml NULL, statement_sql_handle varbinary(64) NULL, @@ -1010,7 +1011,7 @@ Runtime stats information CREATE TABLE #query_store_runtime_stats ( - database_id int NOT NULL, + database_id integer NOT NULL, runtime_stats_id bigint NOT NULL, plan_id bigint NOT NULL, runtime_stats_interval_id bigint NOT NULL, @@ -1113,7 +1114,7 @@ Wait Stats, When Available (2017+) CREATE TABLE #query_store_wait_stats ( - database_id int NOT NULL, + database_id integer NOT NULL, plan_id bigint NOT NULL, wait_category_desc nvarchar(60) NOT NULL, total_query_wait_time_ms bigint NOT NULL, @@ -1129,17 +1130,17 @@ Context is everything CREATE TABLE #query_context_settings ( - database_id int NOT NULL, + database_id integer NOT NULL, context_settings_id bigint NOT NULL, set_options varbinary(8) NULL, language_id smallint NOT NULL, date_format smallint NOT NULL, date_first tinyint NOT NULL, status varbinary(2) NULL, - required_cursor_options int NOT NULL, - acceptable_cursor_options int NOT NULL, + required_cursor_options integer NOT NULL, + acceptable_cursor_options integer NOT NULL, merge_action_type smallint NOT NULL, - default_schema_id int NOT NULL, + default_schema_id integer NOT NULL, is_replication_specific bit NOT NULL, is_contained varbinary(1) NULL ); @@ -1150,7 +1151,7 @@ Feed me Seymour CREATE TABLE #query_store_plan_feedback ( - database_id int NOT NULL, + database_id integer NOT NULL, plan_feedback_id bigint, plan_id bigint, feature_desc nvarchar(120), @@ -1166,7 +1167,7 @@ America's Most Hinted CREATE TABLE #query_store_query_hints ( - database_id int NOT NULL, + database_id integer NOT NULL, query_hint_id bigint, query_id bigint, query_hint_text nvarchar(MAX), @@ -1181,7 +1182,7 @@ Variant? Deviant? You decide! CREATE TABLE #query_store_query_variant ( - database_id int NOT NULL, + database_id integer NOT NULL, query_variant_query_id bigint, parent_query_id bigint, dispatcher_plan_id bigint @@ -1193,7 +1194,7 @@ Replicants CREATE TABLE #query_store_replicas ( - database_id int NOT NULL, + database_id integer NOT NULL, replica_group_id bigint, role_type smallint, replica_name nvarchar(1288) @@ -1205,7 +1206,7 @@ Location, location, location CREATE TABLE #query_store_plan_forcing_locations ( - database_id int NOT NULL, + database_id integer NOT NULL, plan_forcing_location_id bigint, query_id bigint, plan_id bigint, @@ -1235,6 +1236,24 @@ CREATE TABLE ) ); +/*Gonna try gathering this based on*/ +CREATE TABLE + #query_hash_totals +( + database_id integer NOT NULL, + query_hash binary(8) NOT NULL, + total_executions bigint NOT NULL, + total_duration_ms decimal(19,2) NOT NULL, + total_cpu_time_ms decimal(19,2) NOT NULL, + total_logical_reads_mb decimal(19,2) NOT NULL, + total_physical_reads_mb decimal(19,2) NOT NULL, + total_logical_writes_mb decimal(19,2) NOT NULL, + total_clr_time_ms decimal(19,2) NOT NULL, + total_memory_mb decimal(19,2) NOT NULL, + total_rowcount decimal(19,2) NOT NULL, + PRIMARY KEY CLUSTERED(query_hash, database_id) +); + /*GET ALL THOSE DATABASES*/ CREATE TABLE #databases @@ -1292,7 +1311,7 @@ VALUES (200, 'executions', 'count', 'count_executions', 'qsrs.count_executions', 0, NULL, NULL, 0, 'N0'), (210, 'executions', 'per_second', 'executions_per_second', 'qsrs.executions_per_second', 0, NULL, NULL, 0, 'N0'), /* Hash totals - conditionally added */ - (215, 'executions', 'count_hash', 'count_executions_by_query_hash', 'SUM(qsrs.count_executions) OVER (PARTITION BY qsq.query_hash ORDER BY qsq.query_hash)', 1, 'include_query_hash_totals', 1, 0, 'N0'), + (215, 'executions', 'count_hash', 'count_executions_by_query_hash', 'qht.total_executions', 1, 'include_query_hash_totals', 1, 0, 'N0'), /* Duration metrics (group together avg, total, last, min, max) */ (300, 'duration', 'avg', 'avg_duration_ms', 'qsrs.avg_duration_ms', 0, NULL, NULL, 0, 'N0'), (310, 'duration', 'total', 'total_duration_ms', 'qsrs.total_duration_ms', 0, NULL, NULL, 0, 'N0'), @@ -1300,7 +1319,7 @@ VALUES (330, 'duration', 'min', 'min_duration_ms', 'qsrs.min_duration_ms', 0, NULL, NULL, 1, 'N0'), (340, 'duration', 'max', 'max_duration_ms', 'qsrs.max_duration_ms', 0, NULL, NULL, 0, 'N0'), /* Hash totals for duration */ - (315, 'duration', 'total_hash', 'total_duration_ms_by_query_hash', 'SUM(qsrs.total_duration_ms) OVER (PARTITION BY qsq.query_hash ORDER BY qsq.query_hash)', 1, 'include_query_hash_totals', 1, 0, 'N0'), + (315, 'duration', 'total_hash', 'total_duration_ms_by_query_hash', 'qht.total_duration_ms', 1, 'include_query_hash_totals', 1, 0, 'N0'), /* CPU metrics */ (400, 'cpu', 'avg', 'avg_cpu_time_ms', 'qsrs.avg_cpu_time_ms', 0, NULL, NULL, 0, 'N0'), (410, 'cpu', 'total', 'total_cpu_time_ms', 'qsrs.total_cpu_time_ms', 0, NULL, NULL, 0, 'N0'), @@ -1308,7 +1327,7 @@ VALUES (430, 'cpu', 'min', 'min_cpu_time_ms', 'qsrs.min_cpu_time_ms', 0, NULL, NULL, 1, 'N0'), (440, 'cpu', 'max', 'max_cpu_time_ms', 'qsrs.max_cpu_time_ms', 0, NULL, NULL, 0, 'N0'), /* Hash totals for CPU */ - (415, 'cpu', 'total_hash', 'total_cpu_time_ms_by_query_hash', 'SUM(qsrs.total_cpu_time_ms) OVER (PARTITION BY qsq.query_hash ORDER BY qsq.query_hash)', 1, 'include_query_hash_totals', 1, 0, 'N0'), + (415, 'cpu', 'total_hash', 'total_cpu_time_ms_by_query_hash', 'qht.total_cpu_time_ms', 1, 'include_query_hash_totals', 1, 0, 'N0'), /* Logical IO Reads */ (500, 'logical_io_reads', 'avg', 'avg_logical_io_reads_mb', 'qsrs.avg_logical_io_reads_mb', 0, NULL, NULL, 0, 'N0'), (510, 'logical_io_reads', 'total', 'total_logical_io_reads_mb', 'qsrs.total_logical_io_reads_mb', 0, NULL, NULL, 0, 'N0'), @@ -1316,7 +1335,7 @@ VALUES (530, 'logical_io_reads', 'min', 'min_logical_io_reads_mb', 'qsrs.min_logical_io_reads_mb', 0, NULL, NULL, 1, 'N0'), (540, 'logical_io_reads', 'max', 'max_logical_io_reads_mb', 'qsrs.max_logical_io_reads_mb', 0, NULL, NULL, 0, 'N0'), /* Hash totals for logical reads */ - (515, 'logical_io_reads', 'total_hash', 'total_logical_io_reads_mb_by_query_hash', 'SUM(qsrs.total_logical_io_reads_mb) OVER (PARTITION BY qsq.query_hash ORDER BY qsq.query_hash)', 1, 'include_query_hash_totals', 1, 0, 'N0'), + (515, 'logical_io_reads', 'total_hash', 'total_logical_io_reads_mb_by_query_hash', 'qht.total_logical_reads_mb', 1, 'include_query_hash_totals', 1, 0, 'N0'), /* Logical IO Writes */ (600, 'logical_io_writes', 'avg', 'avg_logical_io_writes_mb', 'qsrs.avg_logical_io_writes_mb', 0, NULL, NULL, 0, 'N0'), (610, 'logical_io_writes', 'total', 'total_logical_io_writes_mb', 'qsrs.total_logical_io_writes_mb', 0, NULL, NULL, 0, 'N0'), @@ -1324,7 +1343,7 @@ VALUES (630, 'logical_io_writes', 'min', 'min_logical_io_writes_mb', 'qsrs.min_logical_io_writes_mb', 0, NULL, NULL, 1, 'N0'), (640, 'logical_io_writes', 'max', 'max_logical_io_writes_mb', 'qsrs.max_logical_io_writes_mb', 0, NULL, NULL, 0, 'N0'), /* Hash totals for logical writes */ - (615, 'logical_io_writes', 'total_hash', 'total_logical_io_writes_mb_by_query_hash', 'SUM(qsrs.total_logical_io_writes_mb) OVER (PARTITION BY qsq.query_hash ORDER BY qsq.query_hash)', 1, 'include_query_hash_totals', 1, 0, 'N0'), + (615, 'logical_io_writes', 'total_hash', 'total_logical_io_writes_mb_by_query_hash', 'qht.total_logical_writes_mb', 1, 'include_query_hash_totals', 1, 0, 'N0'), /* Physical IO Reads */ (700, 'physical_io_reads', 'avg', 'avg_physical_io_reads_mb', 'qsrs.avg_physical_io_reads_mb', 0, NULL, NULL, 0, 'N0'), (710, 'physical_io_reads', 'total', 'total_physical_io_reads_mb', 'qsrs.total_physical_io_reads_mb', 0, NULL, NULL, 0, 'N0'), @@ -1332,7 +1351,7 @@ VALUES (730, 'physical_io_reads', 'min', 'min_physical_io_reads_mb', 'qsrs.min_physical_io_reads_mb', 0, NULL, NULL, 1, 'N0'), (740, 'physical_io_reads', 'max', 'max_physical_io_reads_mb', 'qsrs.max_physical_io_reads_mb', 0, NULL, NULL, 0, 'N0'), /* Hash totals for physical reads */ - (715, 'physical_io_reads', 'total_hash', 'total_physical_io_reads_mb_by_query_hash', 'SUM(qsrs.total_physical_io_reads_mb) OVER (PARTITION BY qsq.query_hash ORDER BY qsq.query_hash)', 1, 'include_query_hash_totals', 1, 0, 'N0'), + (715, 'physical_io_reads', 'total_hash', 'total_physical_io_reads_mb_by_query_hash', 'qht.total_physical_reads_mb', 1, 'include_query_hash_totals', 1, 0, 'N0'), /* CLR Time */ (800, 'clr_time', 'avg', 'avg_clr_time_ms', 'qsrs.avg_clr_time_ms', 0, NULL, NULL, 0, 'N0'), (810, 'clr_time', 'total', 'total_clr_time_ms', 'qsrs.total_clr_time_ms', 0, NULL, NULL, 0, 'N0'), @@ -1340,7 +1359,7 @@ VALUES (830, 'clr_time', 'min', 'min_clr_time_ms', 'qsrs.min_clr_time_ms', 0, NULL, NULL, 1, 'N0'), (840, 'clr_time', 'max', 'max_clr_time_ms', 'qsrs.max_clr_time_ms', 0, NULL, NULL, 0, 'N0'), /* Hash totals for CLR time */ - (815, 'clr_time', 'total_hash', 'total_clr_time_ms_by_query_hash', 'SUM(qsrs.total_clr_time_ms) OVER (PARTITION BY qsq.query_hash ORDER BY qsq.query_hash)', 1, 'include_query_hash_totals', 1, 0, 'N0'), + (815, 'clr_time', 'total_hash', 'total_clr_time_ms_by_query_hash', 'qht.total_clr_time_ms', 1, 'include_query_hash_totals', 1, 0, 'N0'), /* DOP (Degree of Parallelism) */ (900, 'dop', 'last', 'last_dop', 'qsrs.last_dop', 0, NULL, NULL, 1, NULL), (910, 'dop', 'min', 'min_dop', 'qsrs.min_dop', 0, NULL, NULL, 0, NULL), @@ -1352,7 +1371,7 @@ VALUES (1030, 'memory', 'min', 'min_query_max_used_memory_mb', 'qsrs.min_query_max_used_memory_mb', 0, NULL, NULL, 1, 'N0'), (1040, 'memory', 'max', 'max_query_max_used_memory_mb', 'qsrs.max_query_max_used_memory_mb', 0, NULL, NULL, 0, 'N0'), /* Hash totals for memory */ - (1015, 'memory', 'total_hash', 'total_query_max_used_memory_mb_by_query_hash', 'SUM(qsrs.total_query_max_used_memory_mb) OVER (PARTITION BY qsq.query_hash ORDER BY qsq.query_hash)', 1, 'include_query_hash_totals', 1, 0, 'N0'), + (1015, 'memory', 'total_hash', 'total_query_max_used_memory_mb_by_query_hash', 'qht.total_memory_mb', 1, 'include_query_hash_totals', 1, 0, 'N0'), /* Row counts */ (1100, 'rowcount', 'avg', 'avg_rowcount', 'qsrs.avg_rowcount', 0, NULL, NULL, 0, 'N0'), (1110, 'rowcount', 'total', 'total_rowcount', 'qsrs.total_rowcount', 0, NULL, NULL, 0, 'N0'), @@ -1360,7 +1379,7 @@ VALUES (1130, 'rowcount', 'min', 'min_rowcount', 'qsrs.min_rowcount', 0, NULL, NULL, 1, 'N0'), (1140, 'rowcount', 'max', 'max_rowcount', 'qsrs.max_rowcount', 0, NULL, NULL, 0, 'N0'), /* Hash totals for row counts */ - (1115, 'rowcount', 'total_hash', 'total_rowcount_by_query_hash', 'SUM(qsrs.total_rowcount) OVER (PARTITION BY qsq.query_hash ORDER BY qsq.query_hash)', 1, 'include_query_hash_totals', 1, 0, 'N0'), + (1115, 'rowcount', 'total_hash', 'total_rowcount_by_query_hash', 'qht.total_rowcount', 1, 'include_query_hash_totals', 1, 0, 'N0'), /* New metrics for newer versions */ /* Physical IO Reads (for newer versions) */ (1200, 'num_physical_io_reads', 'avg', 'avg_num_physical_io_reads_mb', 'qsrs.avg_num_physical_io_reads_mb', 1, 'new', 1, 0, 'N0'), @@ -1474,7 +1493,7 @@ VALUES NULL ); -/* Create a table variable to define parameter processing */ +/* Create a table variable to define parameter processing */ DECLARE @FilterParameters table ( @@ -1487,7 +1506,7 @@ DECLARE requires_secondary_processing bit NOT NULL ); -/* Populate with parameter definitions*/ +/* Populate with parameter definitions*/ INSERT INTO @FilterParameters ( @@ -1510,13 +1529,13 @@ SELECT FROM ( VALUES - /* Include parameters */ + /* Include parameters */ ('include_plan_ids', @include_plan_ids, '#include_plan_ids', 'plan_id', 'bigint', 1, 0), ('include_query_ids', @include_query_ids, '#include_query_ids', 'query_id', 'bigint', 1, 1), ('include_query_hashes', @include_query_hashes, '#include_query_hashes', 'query_hash_s', 'varchar', 1, 1), ('include_plan_hashes', @include_plan_hashes, '#include_plan_hashes', 'plan_hash_s', 'varchar', 1, 1), ('include_sql_handles', @include_sql_handles, '#include_sql_handles', 'sql_handle_s', 'varchar', 1, 1), - /* Ignore parameters */ + /* Ignore parameters */ ('ignore_plan_ids', @ignore_plan_ids, '#ignore_plan_ids', 'plan_id', 'bigint', 0, 0), ('ignore_query_ids', @ignore_query_ids, '#ignore_query_ids', 'query_id', 'bigint', 0, 1), ('ignore_query_hashes', @ignore_query_hashes, '#ignore_query_hashes', 'query_hash_s', 'varchar', 0, 1), @@ -1905,7 +1924,9 @@ SELECT ) IN (5, 8) BEGIN INSERT INTO - #databases WITH(TABLOCK) + #databases + WITH + (TABLOCK) ( database_name ) @@ -1931,7 +1952,9 @@ END; ELSE BEGIN INSERT - #databases WITH(TABLOCK) + #databases + WITH + (TABLOCK) ( database_name ) @@ -2192,7 +2215,9 @@ SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;', OPTION(RECOMPILE);', @troubleshoot_insert = N' INSERT - #troubleshoot_performance WITH(TABLOCK) + #troubleshoot_performance + WITH + (TABLOCK) ( current_table, start_time @@ -2666,7 +2691,9 @@ BEGIN END; INSERT - #query_store_trouble WITH (TABLOCK) + #query_store_trouble +WITH + (TABLOCK) ( database_id, desired_state_desc, @@ -2756,7 +2783,9 @@ AND p.name LIKE @procedure_name;' + @nc10; END; INSERT - #procedure_object_ids WITH(TABLOCK) + #procedure_object_ids + WITH + (TABLOCK) ( [object_id] ) @@ -3509,7 +3538,9 @@ OPTION(RECOMPILE);' + @nc10; END; INSERT - #procedure_plans WITH(TABLOCK) + #procedure_plans + WITH + (TABLOCK) ( plan_id ) @@ -3589,7 +3620,9 @@ OPTION(RECOMPILE);' + @nc10; END; INSERT - #query_types WITH(TABLOCK) + #query_types + WITH + (TABLOCK) ( plan_id ) @@ -3676,7 +3709,7 @@ BEGIN WHILE @@FETCH_STATUS = 0 BEGIN - /* Clean parameter value */ + /* Clean parameter value */ SELECT @param_value = REPLACE(REPLACE(REPLACE(REPLACE( @@ -3684,17 +3717,17 @@ BEGIN CHAR(10), N''), CHAR(13), N''), NCHAR(10), N''), NCHAR(13), N''); - /* Log current operation if debugging */ + /* Log current operation if debugging */ IF @debug = 1 BEGIN RAISERROR('Processing %s with value %s', 0, 1, @param_name, @param_value) WITH NOWAIT; END; - /* Set current table name for troubleshooting */ + /* Set current table name for troubleshooting */ SELECT @current_table = 'inserting ' + @temp_table; - /* Choose appropriate string split function based on data type */ + /* Choose appropriate string split function based on data type */ IF @data_type = N'bigint' BEGIN SELECT @split_sql = @string_split_ints; @@ -3704,7 +3737,7 @@ BEGIN SELECT @split_sql = @string_split_strings; END; - /* Execute the initial insert with troubleshooting if enabled */ + /* Execute the initial insert with troubleshooting if enabled */ IF @troubleshoot_performance = 1 BEGIN EXECUTE sys.sp_executesql @@ -3715,7 +3748,7 @@ BEGIN SET STATISTICS XML ON; END; - /* Execute the dynamic SQL to populate the temporary table */ + /* Execute the dynamic SQL to populate the temporary table */ DECLARE @dynamic_sql nvarchar(MAX) = N' INSERT INTO ' + @temp_table + N' @@ -3753,13 +3786,13 @@ BEGIN @current_table; END; - /* Secondary processing (for parameters that need to populate plan IDs) */ + /* Secondary processing (for parameters that need to populate plan IDs) */ IF @requires_secondary_processing = 1 BEGIN SELECT @current_table = 'inserting #include_plan_ids for ' + @param_name; - /* Build appropriate SQL based on parameter type */ + /* Build appropriate SQL based on parameter type */ DECLARE @secondary_sql nvarchar(MAX) = N''; @@ -3875,7 +3908,7 @@ BEGIN OPTION(RECOMPILE);'; END; - /* Process secondary sql if defined */ + /* Process secondary sql if defined */ IF @secondary_sql IS NOT NULL BEGIN IF @troubleshoot_performance = 1 @@ -3915,7 +3948,7 @@ BEGIN END; END; - /* Update where clause if needed */ + /* Update where clause if needed */ DECLARE @temp_target_table nvarchar(100) = CASE @@ -3998,7 +4031,9 @@ BEGIN END; INSERT - #only_queries_with_hints WITH(TABLOCK) + #only_queries_with_hints + WITH + (TABLOCK) ( plan_id ) @@ -4072,7 +4107,9 @@ BEGIN END; INSERT - #only_queries_with_feedback WITH(TABLOCK) + #only_queries_with_feedback + WITH + (TABLOCK) ( plan_id ) @@ -4146,7 +4183,9 @@ BEGIN END; INSERT - #only_queries_with_variants WITH(TABLOCK) + #only_queries_with_variants + WITH + (TABLOCK) ( plan_id ) @@ -4226,7 +4265,9 @@ OPTION(RECOMPILE);' + @nc10; END; INSERT - #forced_plans_failures WITH(TABLOCK) + #forced_plans_failures + WITH + (TABLOCK) ( plan_id ) @@ -4382,7 +4423,9 @@ END; END; INSERT - #query_text_search WITH(TABLOCK) + #query_text_search + WITH + (TABLOCK) ( plan_id ) @@ -4540,7 +4583,9 @@ END; END; INSERT - #query_text_search_not WITH(TABLOCK) + #query_text_search_not + WITH + (TABLOCK) ( plan_id ) @@ -4637,7 +4682,9 @@ OPTION(RECOMPILE, OPTIMIZE FOR (@top = 9223372036854775807));' + @nc10; END; INSERT - #wait_filter WITH(TABLOCK) + #wait_filter + WITH + (TABLOCK) ( plan_id ) @@ -4722,7 +4769,9 @@ BEGIN END; INSERT - #maintenance_plans WITH(TABLOCK) + #maintenance_plans +WITH + (TABLOCK) ( plan_id ) @@ -4920,7 +4969,9 @@ BEGIN END; INSERT - #plan_ids_with_query_hashes WITH(TABLOCK) + #plan_ids_with_query_hashes + WITH + (TABLOCK) ( database_id, plan_id, @@ -5039,7 +5090,9 @@ OR END; INSERT - #plan_ids_with_total_waits WITH(TABLOCK) + #plan_ids_with_total_waits + WITH + (TABLOCK) ( database_id, plan_id, @@ -5183,7 +5236,9 @@ BEGIN END; INSERT - #plan_ids_with_total_waits WITH(TABLOCK) + #plan_ids_with_total_waits + WITH + (TABLOCK) ( database_id, plan_id, @@ -5293,7 +5348,9 @@ BEGIN END; INSERT - #regression_baseline_runtime_stats WITH(TABLOCK) + #regression_baseline_runtime_stats + WITH + (TABLOCK) ( query_hash, regression_metric_average @@ -5392,7 +5449,9 @@ BEGIN END; INSERT - #regression_current_runtime_stats WITH(TABLOCK) + #regression_current_runtime_stats + WITH + (TABLOCK) ( query_hash, current_metric_average @@ -5576,7 +5635,9 @@ BEGIN END; INSERT - #regression_changes WITH(TABLOCK) + #regression_changes + WITH + (TABLOCK) ( database_id, plan_id, @@ -5708,7 +5769,9 @@ BEGIN END; INSERT - #distinct_plans WITH(TABLOCK) + #distinct_plans +WITH + (TABLOCK) ( plan_id ) @@ -6163,7 +6226,9 @@ BEGIN END; INSERT - #query_store_runtime_stats WITH(TABLOCK) + #query_store_runtime_stats +WITH + (TABLOCK) ( database_id, runtime_stats_id, plan_id, runtime_stats_interval_id, execution_type_desc, first_execution_time, last_execution_time, count_executions, @@ -6340,7 +6405,9 @@ BEGIN END; INSERT - #query_store_plan WITH(TABLOCK) + #query_store_plan +WITH + (TABLOCK) ( database_id, plan_id, @@ -6463,7 +6530,9 @@ BEGIN END; INSERT - #query_store_query WITH(TABLOCK) + #query_store_query +WITH + (TABLOCK) ( database_id, query_id, @@ -6518,6 +6587,100 @@ BEGIN @current_table; END; /*End getting query details*/ + +IF @include_query_hash_totals = 1 +BEGIN + SELECT + @current_table = 'inserting #query_hash_totals for @include_query_hash_totals', + @sql = @isolation_level; + + IF @troubleshoot_performance = 1 + BEGIN + EXECUTE sys.sp_executesql + @troubleshoot_insert, + N'@current_table nvarchar(100)', + @current_table; + + SET STATISTICS XML ON; + END; + + SELECT + @sql += N' + SELECT + @database_id, + qsq.query_hash, + SUM(qsrs.count_executions), + SUM(qsrs.count_executions * qsrs.avg_duration) / 1000., + SUM(qsrs.count_executions * qsrs.avg_cpu_time) / 1000., + SUM(qsrs.count_executions * (qsrs.avg_logical_io_reads * 8.)) / 1024., + SUM(qsrs.count_executions * (qsrs.avg_physical_io_reads * 8.)) / 1024., + SUM(qsrs.count_executions * (qsrs.avg_logical_io_writes * 8.)) / 1024., + SUM(qsrs.count_executions * qsrs.avg_clr_time) / 1000., + SUM(qsrs.count_executions * (qsrs.avg_query_max_used_memory * 8.)) / 1024., + SUM(qsrs.count_executions * qsrs.avg_rowcount) + FROM ' + @database_name_quoted + N'.sys.query_store_runtime_stats AS qsrs + JOIN ' + @database_name_quoted + N'.sys.query_store_plan AS qsp + ON qsrs.plan_id = qsp.plan_id + JOIN ' + @database_name_quoted + N'.sys.query_store_query AS qsq + ON qsp.query_id = qsq.query_id + WHERE EXISTS + ( + SELECT + 1/0 + FROM #query_store_query AS qsq2 + WHERE qsq2.query_hash = qsq.query_hash + ) + GROUP BY qsq.query_hash + OPTION(RECOMPILE); +' + + IF @debug = 1 + BEGIN + PRINT LEN(@sql); + PRINT @sql; + END; + + INSERT INTO + #query_hash_totals + WITH + (TABLOCK) + ( + database_id, + query_hash, + total_executions, + total_duration_ms, + total_cpu_time_ms, + total_logical_reads_mb, + total_physical_reads_mb, + total_logical_writes_mb, + total_clr_time_ms, + total_memory_mb, + total_rowcount + ) + EXECUTE sys.sp_executesql + @sql, + N'@database_id int', + @database_id; + + IF @troubleshoot_performance = 1 + BEGIN + SET STATISTICS XML OFF; + + EXECUTE sys.sp_executesql + @troubleshoot_update, + N'@current_table nvarchar(100)', + @current_table; + + EXECUTE sys.sp_executesql + @troubleshoot_info, + N'@sql nvarchar(max), + @current_table nvarchar(100)', + @sql, + @current_table; + END; +END; + + /* This gets the query text for them! */ @@ -6577,7 +6740,9 @@ BEGIN END; INSERT - #query_store_query_text WITH(TABLOCK) + #query_store_query_text +WITH + (TABLOCK) ( database_id, query_text_id, @@ -6627,7 +6792,9 @@ BEGIN END; INSERT - #dm_exec_query_stats WITH(TABLOCK) + #dm_exec_query_stats +WITH + (TABLOCK) ( statement_sql_handle, total_grant_mb, @@ -6930,7 +7097,9 @@ BEGIN END; INSERT - #database_query_store_options WITH(TABLOCK) + #database_query_store_options +WITH + (TABLOCK) ( database_id, desired_state_desc, @@ -7067,7 +7236,9 @@ OPTION(RECOMPILE);' + @nc10; END; INSERT - #query_store_wait_stats WITH(TABLOCK) + #query_store_wait_stats + WITH + (TABLOCK) ( database_id, plan_id, @@ -7151,7 +7322,9 @@ WHERE EXISTS OPTION(RECOMPILE);'; INSERT - #query_context_settings WITH(TABLOCK) + #query_context_settings +WITH + (TABLOCK) ( database_id, context_settings_id, @@ -7331,7 +7504,9 @@ OPTION(RECOMPILE);' + @nc10; END; INSERT - #query_store_plan_feedback WITH(TABLOCK) + #query_store_plan_feedback + WITH + (TABLOCK) ( database_id, plan_feedback_id, @@ -7404,7 +7579,9 @@ OPTION(RECOMPILE);' + @nc10; END; INSERT - #query_store_query_variant WITH(TABLOCK) + #query_store_query_variant + WITH + (TABLOCK) ( database_id, query_variant_query_id, @@ -7475,7 +7652,9 @@ OPTION(RECOMPILE);' + @nc10; END; INSERT - #query_store_query_hints WITH(TABLOCK) + #query_store_query_hints + WITH + (TABLOCK) ( database_id, query_hint_id, @@ -7550,7 +7729,9 @@ OPTION(RECOMPILE);' + @nc10; END; INSERT - #query_store_plan_forcing_locations WITH(TABLOCK) + #query_store_plan_forcing_locations + WITH + (TABLOCK) ( database_id, plan_forcing_location_id, @@ -7619,7 +7800,9 @@ OPTION(RECOMPILE);' + @nc10; END; INSERT - #query_store_replicas WITH(TABLOCK) + #query_store_replicas + WITH + (TABLOCK) ( database_id, replica_group_id, @@ -7743,6 +7926,9 @@ BEGIN TRUNCATE TABLE #forced_plans_failures; + + TRUNCATE TABLE + #query_hash_totals; END; FETCH NEXT @@ -7806,9 +7992,9 @@ FROM ( SELECT [processing-instruction(query_plan)] = - N''/* '' + NCHAR(13) + NCHAR(10) + - N''/* This is a huge query plan.'' + NCHAR(13) + NCHAR(10) + - N''/* Remove the headers and footers, save it as a .sqlplan file, and re-open it.'' + NCHAR(13) + NCHAR(10) + + N''-- '' + NCHAR(13) + NCHAR(10) + + N''-- This is a huge query plan.'' + NCHAR(13) + NCHAR(10) + + N''-- Remove the headers and footers, save it as a .sqlplan file, and re-open it.'' + NCHAR(13) + NCHAR(10) + NCHAR(13) + NCHAR(10) + REPLACE(qsp.query_plan, N'' Date: Tue, 11 Mar 2025 13:11:22 -0400 Subject: [PATCH 020/246] oh please small tweak to HEBV forward progress in IC --- sp_HumanEvents/sp_HumanEventsBlockViewer.sql | 4 +- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 1490 +++++++++++++++-- 2 files changed, 1326 insertions(+), 168 deletions(-) diff --git a/sp_HumanEvents/sp_HumanEventsBlockViewer.sql b/sp_HumanEvents/sp_HumanEventsBlockViewer.sql index 6cadf19c..7ab9756e 100644 --- a/sp_HumanEvents/sp_HumanEventsBlockViewer.sql +++ b/sp_HumanEvents/sp_HumanEventsBlockViewer.sql @@ -446,8 +446,8 @@ SELECT CONVERT(nchar(1), @is_system_health); /*Change this here in case someone leave it NULL*/ -IF @target_database IS NOT NULL -AND @target_schema IS NOT NULL +IF ISNULL(@target_database, DB_NAME()) IS NOT NULL +AND ISNULL(@target_schema, N'dbo') IS NOT NULL AND @target_table IS NOT NULL AND @target_column IS NOT NULL BEGIN diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index 92c406cb..178d279e 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -30,6 +30,10 @@ ALTER PROCEDURE @database_name sysname = NULL, @schema_name sysname = NULL, @table_name sysname = NULL, + @min_reads bigint = 0, + @min_writes bigint = 0, + @min_size_gb decimal(10,2) = 0, + @min_rows bigint = 0, @help bit = 'false', @debug bit = 'true', @version varchar(20) = NULL OUTPUT, @@ -155,28 +159,60 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. RETURN; END; + IF @debug = 1 + BEGIN + RAISERROR('Declaring variables', 0, 0) WITH NOWAIT; + END; + DECLARE /*general script variables*/ - @sql nvarchar(MAX) = N'', + @sql nvarchar(max) = N'', @database_id integer = NULL, @object_id integer = NULL, @full_object_name nvarchar(768) = NULL, - @final_script nvarchar(MAX) = '', + @final_script nvarchar(max) = '', /*cursor variables*/ @c_database_id integer, @c_schema_name sysname, @c_table_name sysname, @c_index_name sysname, @c_is_unique bit, - @c_filter_definition nvarchar(MAX), + @c_filter_definition nvarchar(max), /*print variables*/ @helper integer = 0, @sql_len integer, - @sql_debug nvarchar(MAX) = N''; + @sql_debug nvarchar(max) = N'', + @online bit = + CASE + WHEN + CONVERT + ( + integer, + SERVERPROPERTY('EngineEdition') + ) IN (3, 5, 8) + THEN 'true' /* Enterprise, Azure SQL DB, Managed Instance */ + ELSE 'false' + END, + @uptime_days nvarchar(10) = + ( + SELECT + DATEDIFF + ( + DAY, + osi.sqlserver_start_time, + SYSDATETIME() + ) + FROM sys.dm_os_sys_info AS osi + ); /* Initial checks for object validity */ + IF @debug = 1 + BEGIN + RAISERROR('Checking paramaters...', 0, 0) WITH NOWAIT; + END; + IF @database_name IS NULL AND DB_NAME() NOT IN ( @@ -228,9 +264,54 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END; END; + -- Parameter validation + IF @min_reads < 0 + OR @min_reads IS NULL + BEGIN + RAISERROR('Parameter @min_reads cannot be NULL or negative. Setting to 0.', 10, 1) WITH NOWAIT; + SET @min_reads = 0; + END; + + IF @min_writes < 0 + OR @min_writes IS NULL + BEGIN + RAISERROR('Parameter @min_writes cannot be NULL or negative. Setting to 0.', 10, 1) WITH NOWAIT; + SET @min_writes = 0; + END; + + IF @min_size_gb < 0 + OR @min_size_gb IS NULL + BEGIN + RAISERROR('Parameter @min_size_gb cannot be NULL or negative. Setting to 0.', 10, 1) WITH NOWAIT; + SET @min_size_gb = 0; + END; + + IF @min_rows < 0 + OR @min_rows IS NULL + BEGIN + RAISERROR('Parameter @min_rows cannot be NULL or negative. Setting to 0.', 10, 1) WITH NOWAIT; + SET @min_rows = 0; + END; + /* Temp tables! */ + + IF @debug = 1 + BEGIN + RAISERROR('Creating temp tables', 0, 0) WITH NOWAIT; + END; + + CREATE TABLE + #filtered_objects + ( + database_id integer, + object_id integer, + schema_name sysname, + table_name sysname, + PRIMARY KEY (database_id, object_id) + ); + CREATE TABLE #operational_stats ( @@ -290,7 +371,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. index_column_id integer NOT NULL, is_descending_key bit NOT NULL, is_included_column bit NULL, - filter_definition nvarchar(MAX) NULL, + filter_definition nvarchar(max) NULL, is_max_length integer NOT NULL, user_seeks bigint NOT NULL, user_scans bigint NOT NULL, @@ -300,6 +381,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. last_user_scan datetime NULL, last_user_lookup datetime NULL, last_user_update datetime NULL, + is_eligible_for_dedupe bit NOT NULL PRIMARY KEY CLUSTERED(database_id, object_id, index_id, column_name) ); @@ -321,7 +403,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. data_compression_desc nvarchar(60) NULL, built_on sysname NULL, partition_function_name sysname NULL, - partition_columns nvarchar(MAX) + partition_columns nvarchar(max) PRIMARY KEY CLUSTERED(database_id, object_id, index_id, partition_id) ); @@ -333,13 +415,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. table_name sysname NOT NULL, index_name sysname NOT NULL, is_unique bit NULL, - key_columns nvarchar(MAX) NULL, - included_columns nvarchar(MAX) NULL, - filter_definition nvarchar(MAX) NULL, + key_columns nvarchar(max) NULL, + included_columns nvarchar(max) NULL, + filter_definition nvarchar(max) NULL, is_redundant bit NULL, superseded_by sysname NULL, - missing_columns nvarchar(MAX) NULL, - action nvarchar(MAX) NULL, + missing_columns nvarchar(max) NULL, + action nvarchar(max) NULL, INDEX c CLUSTERED (database_id, schema_name, table_name, index_name) ); @@ -350,9 +432,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. database_name sysname NOT NULL, table_name sysname NOT NULL, index_name sysname NOT NULL, - action nvarchar(MAX) NULL, - cleanup_script nvarchar(MAX) NULL, - original_definition nvarchar(MAX) NULL, + action nvarchar(max) NULL, + cleanup_script nvarchar(max) NULL, + original_definition nvarchar(max) NULL, /*Usage details*/ user_seeks bigint NULL, user_scans bigint NULL, @@ -379,27 +461,147 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. database_name sysname NOT NULL, table_name sysname NOT NULL, index_name sysname NOT NULL, - action nvarchar(MAX) NOT NULL, - details nvarchar(MAX) NULL, - current_definition nvarchar(MAX) NOT NULL, - proposed_definition nvarchar(MAX) NULL, - usage_summary nvarchar(MAX) NULL, - operational_summary nvarchar(MAX) NULL + action nvarchar(max) NOT NULL, + details nvarchar(max) NULL, + current_definition nvarchar(max) NOT NULL, + proposed_definition nvarchar(max) NULL, + usage_summary nvarchar(max) NULL, + operational_summary nvarchar(max) NULL, + uptime_warning nvarchar(512) NULL ); CREATE TABLE #final_index_actions ( - database_name sysname NOT NULL, - table_name sysname NOT NULL, - index_name sysname NOT NULL, - action nvarchar(MAX) NOT NULL, - script nvarchar(MAX) NOT NULL + database_name sysname NOT NULL DEFAULT N'', + table_name sysname NOT NULL DEFAULT N'', + index_name sysname NOT NULL DEFAULT N'', + action nvarchar(max) NOT NULL DEFAULT N'', + script nvarchar(max) NOT NULL DEFAULT N'' ); /* Start insert queries */ + + IF @debug = 1 + BEGIN + RAISERROR('Generating #filtered_object insert', 0, 0) WITH NOWAIT; + END; + + SELECT + @sql = N' + SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;'; + + SELECT + @sql = N' + SELECT DISTINCT + @database_id, + t.object_id, + s.name, + t.name + FROM ' + QUOTENAME(@database_name) + N'.sys.tables AS t + JOIN ' + QUOTENAME(@database_name) + N'.sys.schemas AS s + ON t.schema_id = s.schema_id + LEFT JOIN ' + QUOTENAME(@database_name) + N'.sys.dm_db_index_usage_stats AS us + ON t.object_id = us.object_id + AND us.database_id = @database_id + WHERE t.is_ms_shipped = 0 + AND t.type <> N''TF''' + + IF @object_id IS NOT NULL + BEGIN + SELECT @sql += N' + AND t.object_id = @object_id'; + END; + + SET @sql += N' + AND EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@database_name) + N'.sys.dm_db_partition_stats AS ps + JOIN ' + QUOTENAME(@database_name) + N'.sys.allocation_units AS au + ON ps.partition_id = au.container_id + WHERE ps.object_id = t.object_id + GROUP + BY ps.object_id + HAVING + SUM(au.total_pages) * 8.0 / 1048576.0 >= @min_size_gb + ) + AND EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@database_name) + N'.sys.dm_db_partition_stats AS ps + WHERE ps.object_id = t.object_id + AND ps.index_id IN (0, 1) + GROUP + BY ps.object_id + HAVING + SUM(ps.row_count) >= @min_rows + ) + AND EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@database_name) + N'.sys.dm_db_index_usage_stats AS ius + WHERE ius.object_id = t.object_id + AND ius.database_id = @database_id + GROUP BY + ius.object_id + HAVING + SUM(ius.user_seeks + ius.user_scans + ius.user_lookups) >= @min_reads + AND + SUM(ius.user_updates) >= @min_writes + ) + OPTION(RECOMPILE);'; + + IF @debug = 1 + BEGIN + PRINT @sql; + END; + + INSERT + #filtered_objects + WITH + (TABLOCK) + ( + database_id, + object_id, + schema_name, + table_name + ) + EXEC sys.sp_executesql + @sql, + N'@database_id int, + @min_reads bigint, + @min_writes bigint, + @min_size_gb decimal(10,2), + @min_rows bigint, + @object_id integer', + @database_id, + @min_reads, + @min_writes, + @min_size_gb, + @min_rows, + @object_id; + + IF ROWCOUNT_BIG() = 0 BEGIN IF @debug = 1 BEGIN RAISERROR('No rows inserted into #filtered_objects', 0, 0) WITH NOWAIT END; END; + + IF @debug = 1 + BEGIN + SELECT + table_name = '#filtered_objects', + fo.* + FROM #filtered_objects AS fo; + END; + + IF @debug = 1 + BEGIN + RAISERROR('Generating #operational_stats insert', 0, 0) WITH NOWAIT; + END; + SELECT @sql = N' SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;'; @@ -451,11 +653,10 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ( SELECT 1/0 - FROM ' + QUOTENAME(@database_name) + N'.sys.tables AS t - WHERE t.object_id = os.object_id - AND t.is_ms_shipped = 0 + FROM #filtered_objects fo + WHERE fo.database_id = os.database_id + AND fo.object_id = os.object_id ) - AND os.index_id > 1 GROUP BY os.database_id, os.object_id, @@ -513,6 +714,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. @database_id, @object_id; + IF ROWCOUNT_BIG() = 0 BEGIN IF @debug = 1 BEGIN RAISERROR('No rows inserted into #operational_stats', 0, 0) WITH NOWAIT END; END; + IF @debug = 1 BEGIN SELECT @@ -521,6 +724,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. FROM #operational_stats AS os; END; + IF @debug = 1 + BEGIN + RAISERROR('Generating #index_details insert', 0, 0) WITH NOWAIT; + END; + SELECT @sql = N' SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;'; @@ -605,7 +813,14 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. us.last_user_seek, us.last_user_scan, us.last_user_lookup, - us.last_user_update + us.last_user_update, + is_eligible_for_dedupe = + CASE + WHEN i.type = 2 + THEN 1 + WHEN i.type = 1 + THEN 0 + END FROM ' + QUOTENAME(@database_name) + N'.sys.tables AS t JOIN ' + QUOTENAME(@database_name) + N'.sys.schemas AS s ON t.schema_id = s.schema_id @@ -622,9 +837,26 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND i.index_id = us.index_id AND us.database_id = @database_id WHERE t.is_ms_shipped = 0 - AND i.type = 2 + AND i.type IN (1, 2) AND i.is_disabled = 0 - AND i.is_hypothetical = 0'; + AND i.is_hypothetical = 0 + AND EXISTS + ( + SELECT + 1/0 + FROM #filtered_objects fo + WHERE fo.database_id = @database_id + AND fo.object_id = t.object_id + ) + AND EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@database_name) + N'.sys.dm_db_partition_stats ps + WHERE ps.object_id = t.object_id + AND ps.index_id = 1 + AND ps.row_count >= @min_rows + )'; IF @object_id IS NOT NULL BEGIN @@ -641,13 +873,14 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. FROM ' + QUOTENAME(@database_name) + N'.sys.objects AS so WHERE i.object_id = so.object_id AND so.is_ms_shipped = 0 - AND so.type = ''TF'' + AND so.type = N''TF'' ) OPTION(RECOMPILE);'; IF @debug = 1 BEGIN - PRINT @sql; + PRINT SUBSTRING(@sql, 1, 4000); + PRINT SUBSTRING(@sql, 4000, 8000); END; INSERT @@ -681,14 +914,19 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. last_user_seek, last_user_scan, last_user_lookup, - last_user_update + last_user_update, + is_eligible_for_dedupe ) EXEC sys.sp_executesql @sql, N'@database_id integer, - @object_id integer', + @object_id integer, + @min_rows integer', @database_id, - @object_id; + @object_id, + @min_rows; + + IF ROWCOUNT_BIG() = 0 BEGIN IF @debug = 1 BEGIN RAISERROR('No rows inserted into #index_details', 0, 0) WITH NOWAIT END; END; IF @debug = 1 BEGIN @@ -698,6 +936,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. FROM #index_details AS id; END; + IF @debug = 1 + BEGIN + RAISERROR('Generating #partition_stats insert', 0, 0) WITH NOWAIT; + END; + SELECT @sql = N' SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;'; @@ -754,8 +997,16 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ON p.partition_id = a.container_id LEFT HASH JOIN ' + QUOTENAME(@database_name) + N'.sys.dm_db_partition_stats AS ps ON p.partition_id = ps.partition_id - WHERE t.type <> ''TF'' - AND i.type = 2'; + WHERE t.type <> N''TF'' + AND i.type IN (1, 2) + AND EXISTS + ( + SELECT + 1/0 + FROM #filtered_objects fo + WHERE fo.database_id = @database_id + AND fo.object_id = t.object_id + )'; IF @object_id IS NOT NULL BEGIN @@ -815,7 +1066,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. FOR XML PATH(''''), TYPE - ).value(''.'', ''nvarchar(MAX)''), + ).value(''.'', ''nvarchar(max)''), 1, 2, '''' @@ -855,6 +1106,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. @database_id, @object_id; + IF ROWCOUNT_BIG() = 0 BEGIN IF @debug = 1 BEGIN RAISERROR('No rows inserted into #partition_stats', 0, 0) WITH NOWAIT END; END; + IF @debug = 1 BEGIN SELECT @@ -863,6 +1116,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. FROM #partition_stats AS ps; END; + IF @debug = 1 + BEGIN + RAISERROR('Performing #index_analysis insert', 0, 0) WITH NOWAIT; + END; + INSERT INTO #index_analysis WITH @@ -899,6 +1157,10 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. WHERE id2.object_id = id1.object_id AND id2.index_id = id1.index_id AND id2.is_included_column = 0 + GROUP BY + id2.column_name, + id2.is_descending_key, + id2.key_ordinal ORDER BY id2.key_ordinal FOR XML @@ -920,6 +1182,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. WHERE id2.object_id = id1.object_id AND id2.index_id = id1.index_id AND id2.is_included_column = 1 + GROUP BY + id2.column_name ORDER BY id2.column_name FOR XML @@ -932,6 +1196,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ), id1.filter_definition FROM #index_details id1 + WHERE id1.is_eligible_for_dedupe = 1 GROUP BY id1.schema_name, id1.table_name, @@ -942,6 +1207,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. id1.filter_definition OPTION(RECOMPILE); + IF ROWCOUNT_BIG() = 0 BEGIN IF @debug = 1 BEGIN RAISERROR('No rows inserted into #index_analysis', 0, 0) WITH NOWAIT END; END; + IF @debug = 1 BEGIN SELECT @@ -950,6 +1217,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. FROM #index_analysis AS ia; END; + IF @debug = 1 + BEGIN + RAISERROR('Starting cursor', 0, 0) WITH NOWAIT; + END; + /*Analyze indexes*/ DECLARE @index_cursor CURSOR; @@ -986,6 +1258,12 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. WHILE @@FETCH_STATUS = 0 BEGIN + + IF @debug = 1 + BEGIN + RAISERROR('Performing #index_analysis update', 0, 0) WITH NOWAIT; + END; + WITH IndexColumns AS ( @@ -996,11 +1274,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. id.index_name, id.column_name, id.is_included_column, - id.key_ordinal + id.key_ordinal, + id.is_eligible_for_dedupe FROM #index_details id WHERE id.database_id = @c_database_id AND id.schema_name = @c_schema_name AND id.table_name = @c_table_name + AND id.is_eligible_for_dedupe = 1 ), CurrentIndexColumns AS ( @@ -1008,6 +1288,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ic.* FROM IndexColumns AS ic WHERE ic.index_name = @c_index_name + AND ic.is_eligible_for_dedupe = 1 ), OtherIndexColumns AS ( @@ -1015,6 +1296,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ic.* FROM IndexColumns AS ic WHERE ic.index_name <> @c_index_name + AND ic.is_eligible_for_dedupe = 1 ) UPDATE ia @@ -1026,20 +1308,20 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SELECT 1/0 FROM CurrentIndexColumns cic - WHERE cic.is_included_column = 0 -- Only check key columns + WHERE cic.is_included_column = 0 /* Only check key columns */ AND NOT EXISTS ( SELECT 1/0 FROM OtherIndexColumns oic WHERE oic.column_name = cic.column_name - AND oic.is_included_column = 0 -- Must be in key columns - AND oic.key_ordinal <= cic.key_ordinal -- Check leading edge + AND oic.is_included_column = 0 /* Must be in key columns */ + AND oic.key_ordinal <= cic.key_ordinal /* Check leading edge */ ) ) AND ( - -- Check included columns separately since order doesn't matter + /* Check included columns separately since order doesn't matter */ NOT EXISTS ( SELECT @@ -1055,16 +1337,21 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND ( oic.is_included_column = 1 - OR oic.is_included_column = 0 -- Include cols can be covered by key cols + OR oic.is_included_column = 0 /* Include cols can be covered by key cols */ ) ) ) ) - AND ISNULL(ia.filter_definition, '') = ISNULL(@c_filter_definition, '') + AND ISNULL(REPLACE(REPLACE(REPLACE(ia.filter_definition, ' ', ''), '(', ''), ')', ''), '') = + ISNULL(REPLACE(REPLACE(REPLACE(@c_filter_definition, ' ', ''), '(', ''), ')', ''), '') AND ( ia.is_unique = 0 - OR @c_is_unique = 1 + OR + ( + ia.is_unique = 1 + AND @c_is_unique = 1 + ) ) THEN 1 ELSE 0 @@ -1076,20 +1363,20 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SELECT 1/0 FROM CurrentIndexColumns cic - WHERE cic.is_included_column = 0 -- Only check key columns + WHERE cic.is_included_column = 0 /* Only check key columns */ AND NOT EXISTS ( SELECT 1/0 FROM OtherIndexColumns oic WHERE oic.column_name = cic.column_name - AND oic.is_included_column = 0 -- Must be in key columns - AND oic.key_ordinal <= cic.key_ordinal -- Check leading edge + AND oic.is_included_column = 0 /* Must be in key columns */ + AND oic.key_ordinal <= cic.key_ordinal /* Check leading edge */ ) ) AND ( - -- Check included columns separately since order doesn't matter + /* Check included columns separately since order doesn't matter */ NOT EXISTS ( SELECT @@ -1105,7 +1392,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND ( oic.is_included_column = 1 - OR oic.is_included_column = 0 -- Include cols can be covered by key cols + OR oic.is_included_column = 0 /* Include cols can be covered by key cols */ ) ) ) @@ -1123,7 +1410,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. STUFF ( ( - SELECT + SELECT DISTINCT N', ' + oic.column_name FROM OtherIndexColumns oic @@ -1137,7 +1424,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. FOR XML PATH(''), TYPE - ).value('.', 'nvarchar(MAX)'), + ).value('.', 'nvarchar(max)'), 1, 2, '' @@ -1159,6 +1446,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. @c_filter_definition; END; + IF @debug = 1 + BEGIN + RAISERROR('Performing #index_analysis update after cursor', 0, 0) WITH NOWAIT; + END; + /*Determine actions*/ UPDATE #index_analysis @@ -1184,11 +1476,16 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. IF @debug = 1 BEGIN SELECT - table_name = '#index_analysis', + table_name = '#index_analysis after update', ia.* FROM #index_analysis AS ia; END; + IF @debug = 1 + BEGIN + RAISERROR('Performing #index_cleanup_report insert', 0, 0) WITH NOWAIT; + END; + INSERT INTO #index_cleanup_report WITH @@ -1225,13 +1522,19 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. cleanup_script = CASE WHEN ia.action = N'DROP' - THEN N'DROP INDEX ' + + THEN NCHAR(10) + + N'DROP INDEX ' + QUOTENAME(ia.index_name) + N' ON ' + + QUOTENAME(DB_NAME(ia.database_id)) + + N'.' + + QUOTENAME(ia.schema_name) + + N'.' + QUOTENAME(ia.table_name) + N';' WHEN ia.action LIKE N'MERGE INTO%' - THEN N'CREATE ' + + THEN NCHAR(10) + + N'CREATE ' + CASE WHEN ia.is_unique = 1 THEN N'UNIQUE ' @@ -1239,65 +1542,207 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END + N'INDEX ' + QUOTENAME(ia.superseded_by) + - N' ON ' + + NCHAR(10) + + N'ON ' + + QUOTENAME(DB_NAME(ia.database_id)) + + N'.' + + QUOTENAME(ia.schema_name) + + N'.' + QUOTENAME(ia.table_name) + - N'(' + + NCHAR(10) + + N' (' + ISNULL(superseding.key_columns, ia.key_columns) + N')' + + NCHAR(10) + CASE - WHEN ISNULL(superseding.included_columns, ia.included_columns) IS NOT NULL - OR ia.missing_columns IS NOT NULL - THEN N' INCLUDE (' + - STUFF(( - SELECT DISTINCT N',' + c.value('.', 'nvarchar(128)') - FROM ( - SELECT CAST(N'' + - REPLACE(ISNULL(superseding.included_columns, ia.included_columns), N', ', N'') - + N'' AS xml) - ) AS x(c) - FOR XML PATH('') - ), 1, 1, '') + - CASE - WHEN ia.missing_columns IS NOT NULL - THEN N', ' + ia.missing_columns - ELSE N'' - END + + WHEN + ( + superseding.included_columns IS NOT NULL + OR ia.included_columns IS NOT NULL + ) + OR ia.missing_columns IS NOT NULL + THEN N' INCLUDE' + + NCHAR(10) + + N' (' + + -- Combine all INCLUDE columns with proper parsing + STUFF + ( + ( + SELECT DISTINCT + N', ' + + column_value + FROM + ( + -- From superseding index + SELECT + column_value = + LTRIM(RTRIM(value.c.value('.', 'sysname'))) + FROM + ( + SELECT + Columns = + CONVERT + ( + xml, + '' + + REPLACE + ( + ISNULL + ( + superseding.included_columns, + '' + ), + ', ', + '') + + '' + ) + ) t + CROSS APPLY t.Columns.nodes('/c') AS value(c) + + UNION + + -- From current index + SELECT + column_value = + LTRIM(RTRIM(value.c.value('.', 'sysname'))) + FROM + ( + SELECT + Columns = + CONVERT + ( + xml, + '' + + REPLACE + ( + ISNULL + ( + ia.included_columns, + '' + ), + ', ', + '' + ) + + '' + ) + ) t + CROSS APPLY t.Columns.nodes('/c') AS value(c) + + UNION + + -- From missing columns + SELECT + column_value = + LTRIM(RTRIM(value.c.value('.', 'sysname'))) + FROM + ( + SELECT + Columns = + CONVERT + ( + xml, + '' + + REPLACE + ( + ISNULL + ( + ia.missing_columns, + '' + ), + ', ', + '' + ) + '' + ) + ) t + CROSS APPLY t.Columns.nodes('/c') AS value(c) + ) AS all_columns + WHERE LEN(column_value) > 0 + FOR + XML + PATH(''), + TYPE + ).value('.', 'nvarchar(max)'), + 1, + 2, + '' + ) + N')' ELSE N'' - END + + END + CASE + /* Check for partitioning in the superseding index first */ + WHEN EXISTS + ( + SELECT + 1/0 + FROM #partition_stats ps_super + WHERE ps_super.table_name = ia.table_name + AND ps_super.index_name = ia.superseded_by + AND ps_super.partition_function_name IS NOT NULL + ) + THEN + ( + SELECT TOP (1) + NCHAR(10) + + N' ON ' + + QUOTENAME(ps_super.partition_function_name) + + N'(' + + ps_super.partition_columns + + N')' + FROM #partition_stats ps_super + WHERE ps_super.table_name = ia.table_name + AND ps_super.index_name = ia.superseded_by + ) + /* Fall back to the current index's partitioning if available */ WHEN ps.partition_function_name IS NOT NULL - THEN N' ON ' + + THEN NCHAR(10) + + N' ON ' + QUOTENAME(ps.partition_function_name) + N'(' + ps.partition_columns + N')' ELSE N'' - END + + END + CASE WHEN ia.filter_definition IS NOT NULL - THEN N' WHERE ' + + THEN NCHAR(10) + + N' WHERE ' + ia.filter_definition ELSE N'' END + - N' WITH (DROP_EXISTING = ON' + + NCHAR(10) + + N' WITH ' + + NCHAR(10) + + N' (DROP_EXISTING = ON, FILLFACTOR = 100, SORT_IN_TEMPDB = ON, ONLINE = ' + + CASE + WHEN @online = 'true' /*Best effort at detecting online index abilities*/ + THEN N'ON' + ELSE N'OFF' + END + CASE WHEN ps.data_compression_desc <> N'NONE' THEN N', DATA_COMPRESSION = ' + ps.data_compression_desc - ELSE N'' + ELSE N', DATA_COMPRESSION = PAGE' /* Add PAGE compression by default for merged indexes */ END + N');' + - NCHAR(13) + NCHAR(10) + + NCHAR(10) + + NCHAR(10) + N'ALTER INDEX ' + QUOTENAME(ia.index_name) + N' ON ' + + QUOTENAME(DB_NAME(ia.database_id)) + + N'.' + + QUOTENAME(ia.schema_name) + + N'.' + QUOTENAME(ia.table_name) + - N' DISABLE;' + N' DISABLE' ELSE N'' - END, + END + + N';', original_definition = - N'CREATE ' + + NCHAR(10) + + N' -- CREATE ' + CASE WHEN ia.is_unique = 1 THEN N'UNIQUE ' @@ -1305,21 +1750,31 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END + N'INDEX ' + QUOTENAME(ia.index_name) + - N' ON ' + + NCHAR(10) + + N' -- ON ' + + QUOTENAME(DB_NAME(ia.database_id)) + + N'.' + + QUOTENAME(ia.schema_name) + + N'.' + QUOTENAME(ia.table_name) + - N'(' + + NCHAR(10) + + N' -- (' + ia.key_columns + N')' + CASE WHEN ia.included_columns IS NOT NULL - THEN N' INCLUDE (' + + THEN NCHAR(10) + + N' -- INCLUDE' + + NCHAR(10) + + N' -- (' + ia.included_columns + N')' ELSE N'' END + CASE WHEN ps.partition_function_name IS NOT NULL - THEN N' ON ' + + THEN NCHAR(10) + + N' -- ON ' + QUOTENAME(ps.partition_function_name) + N'(' + ps.partition_columns + @@ -1328,10 +1783,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END + CASE WHEN ia.filter_definition IS NOT NULL - THEN N' WHERE ' + - ia.filter_definition + THEN NCHAR(10) + + N' -- WHERE ' + + QUOTENAME(ia.filter_definition, '()') ELSE N'' - END, + END + + N';' + + NCHAR(10), id.user_seeks, id.user_scans, id.user_lookups, @@ -1360,7 +1818,10 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND id.index_id = os.index_id LEFT JOIN #index_analysis superseding ON ia.superseded_by = superseding.index_name - AND ia.table_name = superseding.table_name; + AND ia.table_name = superseding.table_name + OPTION(RECOMPILE); + + IF ROWCOUNT_BIG() = 0 BEGIN IF @debug = 1 BEGIN RAISERROR('No rows inserted into #index_cleanup_report', 0, 0) WITH NOWAIT END; END; IF @debug = 1 BEGIN @@ -1370,6 +1831,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. FROM #index_cleanup_report AS icr; END; + IF @debug = 1 + BEGIN + RAISERROR('Performing #index_cleanup_summary insert', 0, 0) WITH NOWAIT; + END; + INSERT INTO #index_cleanup_summary WITH @@ -1383,7 +1849,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. current_definition, proposed_definition, usage_summary, - operational_summary + operational_summary, + uptime_warning ) SELECT icr.database_name, @@ -1458,8 +1925,34 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. N', Lookups: ' + CONVERT(nvarchar(20), icr.singleton_lookup_count) + N', Inserts: ' + CONVERT(nvarchar(20), icr.leaf_insert_count) + N', Updates: ' + CONVERT(nvarchar(20), icr.leaf_update_count) + - N', Deletes: ' + CONVERT(nvarchar(20), icr.leaf_delete_count) - FROM #index_cleanup_report AS icr; + N', Deletes: ' + CONVERT(nvarchar(20), icr.leaf_delete_count), + uptime_warning = + CASE + WHEN icr.user_seeks = 0 AND icr.user_scans = 0 AND icr.user_lookups = 0 + THEN + CASE + WHEN TRY_PARSE(@uptime_days AS integer) < 7 + THEN N'WARNING: SQL Server has been running for only ' + + @uptime_days + + N' days. Usage statistics may not be reliable.' + WHEN TRY_PARSE(@uptime_days AS integer) < 14 + THEN N'CAUTION: SQL Server has been running for only ' + + @uptime_days + + N' days. Usage statistics may be incomplete.' + WHEN TRY_PARSE(@uptime_days AS integer) < 30 + THEN N'NOTE: SQL Server has been running for only ' + + @uptime_days + + N' days. Consider this when evaluating index usage.' + ELSE N'NOTE: SQL Server has been up for ' + + @uptime_days + + N' days, which makes analysis good, but... Are you patching this thing?' + END + ELSE NULL + END + FROM #index_cleanup_report AS icr + OPTION(RECOMPILE); + + IF ROWCOUNT_BIG() = 0 BEGIN IF @debug = 1 BEGIN RAISERROR('No rows inserted into #index_cleanup_summary', 0, 0) WITH NOWAIT END; END; IF @debug = 1 BEGIN @@ -1469,6 +1962,237 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. FROM #index_cleanup_summary AS ics; END; + IF @debug = 1 + BEGIN + RAISERROR('Going into summary and reports', 0, 0) WITH NOWAIT; + END; + + /* Index Cleanup Summary Report */ + + IF @debug = 1 + BEGIN + RAISERROR('Index Cleanup Summary', 0, 0) WITH NOWAIT; + END; + + SELECT + summary_type = + 'Index Cleanup Summary', + total_indexes_analyzed = + COUNT_BIG(DISTINCT icr.index_name), + indexes_to_drop = + SUM + ( + CASE + WHEN icr.action = 'DROP' + THEN 1 + ELSE 0 + END + ), + indexes_to_merge = + SUM + ( + CASE + WHEN icr.action LIKE 'MERGE INTO%' + THEN 1 + ELSE 0 + END + ), + unused_indexes = + SUM + ( + CASE + WHEN icr.user_seeks = 0 + AND icr.user_scans = 0 + AND icr.user_lookups = 0 + THEN 1 + ELSE 0 + END + ), + space_savings_gb = + CONVERT + ( + decimal(10, 2), + ( + SELECT + SUM(ps_total.space_saved_mb) / 1024.0 + FROM + ( + SELECT + icr_distinct.index_name, + icr_distinct.table_name, + space_saved_mb = SUM(ps_inner.total_space_mb) + FROM #index_cleanup_report AS icr_distinct + JOIN #partition_stats AS ps_inner + ON ps_inner.table_name = icr_distinct.table_name + AND ps_inner.index_name = icr_distinct.index_name + WHERE icr_distinct.action = 'DROP' + OR icr_distinct.action LIKE 'MERGE INTO%' + GROUP BY + icr_distinct.index_name, + icr_distinct.table_name + ) AS ps_total + ) + ), + write_operations_avoided = + SUM + ( + CASE + WHEN icr.action = 'DROP' + OR icr.action LIKE 'MERGE INTO%' + THEN ISNULL(icr.user_updates, 0) + ELSE 0 + END + ) + FROM #index_cleanup_report AS icr + OPTION (RECOMPILE); + + IF @debug = 1 + BEGIN + RAISERROR('Top tables by potential space savings', 0, 0) WITH NOWAIT; + END; + + /* Top tables by potential space savings */ + SELECT TOP (10) + icr.database_name, + icr.table_name, + indexes_affected = + COUNT_BIG(DISTINCT icr.index_name), + space_savings_gb = + CONVERT + ( + decimal(10,2), + ( + SELECT + SUM(ps_total.space_saved_mb) / 1024.0 + FROM + ( + SELECT + ps_inner.table_name, + space_saved_mb = + SUM(ps_inner.total_space_mb) + FROM #partition_stats AS ps_inner + JOIN #index_cleanup_report AS icr_inner + ON ps_inner.table_name = icr_inner.table_name + AND ps_inner.index_name = icr_inner.index_name + WHERE icr_inner.table_name = icr.table_name + AND + ( + icr_inner.action = 'DROP' + OR icr_inner.action LIKE 'MERGE INTO%' + ) + GROUP BY + ps_inner.table_name + ) AS ps_total + ) + ), + write_operations_avoided = + SUM(ISNULL(icr.user_updates, 0)) + FROM #index_cleanup_report AS icr + WHERE + ( + icr.action = 'DROP' + OR icr.action LIKE 'MERGE INTO%' + ) + GROUP BY + icr.database_name, + icr.table_name + ORDER BY + space_savings_gb DESC + OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + RAISERROR('Page Compression Opportunity Summary', 0, 0) WITH NOWAIT; + END; + + /* Summary of non-compressed indexes */ + SELECT + summary_type = 'Page Compression Opportunity Summary', + candidate_indexes = + COUNT_BIG(*), + total_size_gb = + SUM(ps.total_space_mb) / 1024.0, + estimated_savings_low_gb = + (SUM(ps.total_space_mb) * 0.20) / 1024.0, /* Conservative estimate (20%) */ + estimated_savings_typical_gb = + (SUM(ps.total_space_mb) * 0.40) / 1024.0, /* Typical estimate (40%) */ + estimated_savings_high_gb = + (SUM(ps.total_space_mb) * 0.60) / 1024.0 /* Optimistic estimate (60%) */ + FROM #partition_stats ps + WHERE ps.data_compression_desc = 'NONE' + AND NOT EXISTS + ( + SELECT + 1/0 + FROM #index_cleanup_report AS icr + WHERE icr.index_name = ps.index_name + AND + ( + icr.action = 'DROP' + OR icr.action LIKE 'MERGE INTO%' + ) + ) + OPTION(RECOMPILE); + + -- Top candidates for page compression + + IF @debug = 1 + BEGIN + RAISERROR('Top candidates for page compression', 0, 0) WITH NOWAIT; + END; + + SELECT TOP (20) + database_name = + @database_name, + ps.schema_name, + ps.table_name, + ps.index_name, + index_type = + CASE + WHEN ps.index_id = 1 + THEN 'CLUSTERED' + ELSE 'NONCLUSTERED' + END, + size_gb = + SUM(ps.total_space_mb) / 1024.0, + estimated_savings_low_gb = + (SUM(ps.total_space_mb) * 0.20) / 1024.0, -- Conservative (20%) + estimated_savings_typical_gb = + (SUM(ps.total_space_mb) * 0.40) / 1024.0, -- Typical (40%) + estimated_savings_high_gb = + (SUM(ps.total_space_mb) * 0.60) / 1024.0, -- Optimistic (60%) + rebuild_script = + N'ALTER INDEX ' + + QUOTENAME(ps.index_name) + + N' ON ' + + QUOTENAME(ps.schema_name) + + N'.' + + QUOTENAME(ps.table_name) + + N' REBUILD WITH + (DROP_EXISTING = ON, FILLFACTOR = 100, SORT_IN_TEMPDB = ON, ONLINE = ' + + CASE + WHEN @online = 'true' + THEN N'ON' + ELSE N'OFF' + END + + N', DATA_COMPRESSION = PAGE + );' + FROM #partition_stats ps + WHERE ps.data_compression_desc = N'NONE' + GROUP BY + ps.schema_name, + ps.table_name, + ps.index_name, + ps.index_id + ORDER BY + SUM(ps.total_space_mb) DESC + OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + RAISERROR('Select from #index_cleanup_summary', 0, 0) WITH NOWAIT; + END; + SELECT ics.database_name, ics.table_name, @@ -1478,7 +2202,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ics.current_definition, ics.proposed_definition, ics.usage_summary, - ics.operational_summary + ics.operational_summary, + ics.uptime_warning FROM #index_cleanup_summary AS ics ORDER BY CASE ics.action @@ -1488,8 +2213,14 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ELSE 999 END, ics.table_name, - ics.index_name; + ics.index_name + OPTION(RECOMPILE); + IF @debug = 1 + BEGIN + RAISERROR('Performing #final_index_actions insert', 0, 0) WITH NOWAIT; + END; + WITH IndexActions AS ( @@ -1527,23 +2258,48 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. script ) SELECT - database_name, - table_name, - index_name, - action, - CASE - WHEN action LIKE N'MERGE INTO%' - THEN cleanup_script - WHEN action = N'DROP' - THEN N'ALTER INDEX ' + - QUOTENAME(index_name) + - N' ON ' + - QUOTENAME(table_name) + - N' DISABLE;' - ELSE N'???' - END AS script - FROM IndexActions - WHERE n = 1; + ia.database_name, + ia.table_name, + ia.index_name, + ia.action, + script = + CASE + WHEN ia.action LIKE N'MERGE INTO%' + THEN ISNULL + ( + ia.cleanup_script, + N'-- Unable to generate merge script for ' + + ia.index_name + ) + WHEN ia.action = N'DROP' + THEN N'ALTER INDEX ' + + QUOTENAME(ia.index_name) + + N' ON ' + + QUOTENAME(ia.table_name) + + N' DISABLE;' + ELSE N'???' + END + FROM IndexActions AS ia + WHERE ia.n = 1 + AND + ( + ia.cleanup_script IS NOT NULL + OR action = N'DROP' + ) + OPTION(RECOMPILE); + + IF ROWCOUNT_BIG() = 0 BEGIN IF @debug = 1 BEGIN RAISERROR('No rows inserted into #final_index_actions', 0, 0) WITH NOWAIT END; END; + + IF @debug = 1 + BEGIN + + SELECT + table_name = '#final_index_actions', + fia.* + FROM #final_index_actions AS fia; + + RAISERROR('Select from #final_index_actions', 0, 0) WITH NOWAIT; + END; SELECT f.database_name, @@ -1552,7 +2308,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. f.action, f.script, sort_order = - CASE action + CASE f.action WHEN N'MERGE INTO' THEN 2 WHEN N'DROP' THEN 3 ELSE 999 @@ -1581,74 +2337,476 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND r.user_lookups = 0 AND r.user_updates = 0 ORDER BY - table_name, - index_name, - sort_order; + f.table_name, + f.index_name, + sort_order + OPTION(RECOMPILE); + IF @debug = 1 + BEGIN + RAISERROR('Generating scripts', 0, 0) WITH NOWAIT; + END; + + /*Merge into*/ SELECT - @final_script += f.script + NCHAR(13) + NCHAR(10) + @final_script += + N' + -- ============================================================================= + -- MERGE INDEX: ' + + QUOTENAME(f.index_name) + + N' into ' + + QUOTENAME + ( + SUBSTRING + ( + f.action, + 12, + CHARINDEX + ( + N' ', + f.action, + 12 + ) - 12 + ) + )+ + N' + -- Reason: This index overlaps with another index and can be consolidated + -- Original definition: ' + + NCHAR(10) + + ( + SELECT + MAX(ics.current_definition) + FROM #index_cleanup_summary AS ics + WHERE ics.index_name = f.index_name + AND ics.table_name = f.table_name + ) + + N' + -- Usage: Seeks: ' + + CONVERT + ( + nvarchar(20), + ISNULL + ( + ( + SELECT + SUM(DISTINCT id.user_seeks) + FROM #index_details AS id + WHERE id.table_name = f.table_name + AND id.index_name = f.index_name + ), + 0 + ) + ) + + N', Scans: ' + + CONVERT + ( + nvarchar(20), + ISNULL + ( + ( + SELECT + SUM(DISTINCT id.user_scans) + FROM #index_details AS id + WHERE id.table_name = f.table_name + AND id.index_name = f.index_name + ), + 0 + ) + ) + + N', Lookups: ' + + CONVERT + ( + nvarchar(20), + ISNULL + ( + ( + SELECT + SUM(DISTINCT id.user_lookups) + FROM #index_details AS id + WHERE id.table_name = f.table_name + AND id.index_name = f.index_name + ), + 0 + ) + ) + + N', Updates: ' + + CONVERT + ( + nvarchar(20), + ISNULL + ( + ( + SELECT + SUM(DISTINCT id.user_updates) + FROM #index_details AS id + WHERE id.table_name = f.table_name + AND id.index_name = f.index_name + ), + 0 + ) + ) + + N' + -- Space saved: ' + + CONVERT + ( + nvarchar(20), + ISNULL + ( + ( + SELECT + CONVERT + ( + decimal(10,2), + SUM(ps.total_space_mb) / 1024.0 + ) + FROM #partition_stats AS ps + WHERE ps.table_name = f.table_name + AND ps.index_name = f.index_name + ), + 0 + ) + ) + N' GB + -- ============================================================================= + ' + + f.script + + NCHAR(10) + + NCHAR(10) FROM #final_index_actions AS f WHERE f.action LIKE N'MERGE INTO%' ORDER BY f.table_name, f.index_name; - + + /*Drop indexes*/ SELECT - @final_script += f.script + NCHAR(13) + NCHAR(10) + @final_script += N' + /* + -- ============================================================================= + -- DROP INDEX: ' + + QUOTENAME(f.index_name) + + N' + -- Reason: This index is redundant with other indexes on the same table + -- Current usage: Seeks: ' + + CONVERT + ( + nvarchar(20), + ISNULL + ( + ( + SELECT + SUM(DISTINCT id.user_seeks) + FROM #index_details AS id + WHERE id.table_name = f.table_name + AND id.index_name = f.index_name + ), + 0 + ) + ) + + N', Scans: ' + + CONVERT + ( + nvarchar(20), + ISNULL + ( + ( + SELECT + SUM(DISTINCT id.user_scans) + FROM #index_details AS id + WHERE id.table_name = f.table_name + AND id.index_name = f.index_name + ), + 0 + ) + ) + + N', Lookups: ' + + CONVERT + ( + nvarchar(20), + ISNULL + ( + ( + SELECT + SUM(DISTINCT id.user_lookups) + FROM #index_details AS id + WHERE id.table_name = f.table_name + AND id.index_name = f.index_name + ), + 0 + ) + ) + + N', Updates: ' + + CONVERT + ( + nvarchar(20), + ISNULL + ( + ( + SELECT + SUM(DISTINCT id.user_updates) + FROM #index_details AS id + WHERE id.table_name = f.table_name + AND id.index_name = f.index_name + ), + 0 + ) + ) + + N' + -- Last used: ' + + ISNULL + ( + CONVERT + ( + nvarchar(30), + ( + SELECT + MAX + ( + CASE + WHEN id.last_user_seek > id.last_user_scan + AND id.last_user_seek > id.last_user_lookup + THEN id.last_user_seek + WHEN id.last_user_scan > id.last_user_lookup + THEN id.last_user_scan + ELSE id.last_user_lookup + END + ) + FROM #index_details AS id + WHERE id.table_name = f.table_name + AND id.index_name = f.index_name + ), + 120 + ), + 'Never' + ) + + N' + -- Space reclaimed: ' + + CONVERT + ( + nvarchar(20), + ISNULL + ( + ( + SELECT + CONVERT + ( + decimal(10,2), + SUM(ps.total_space_mb) / 1024.0 + ) + FROM #partition_stats AS ps + WHERE ps.table_name = f.table_name + AND ps.index_name = f.index_name + ), + 0 + ) + ) + N' GB + -- ============================================================================= + */' + + f.script + + NCHAR(10) + + NCHAR(10) FROM #final_index_actions AS f - WHERE f.action IN - ( - N'DROP', - N'MERGE INTO' - ) + WHERE f.action = N'DROP' ORDER BY f.table_name, f.index_name; - + + + + /*Unused indexes*/ SELECT - @final_script += - N'ALTER INDEX ' + - QUOTENAME(i.index_name) + - N' ON ' + - QUOTENAME(i.table_name) + - N' DISABLE;' + - NCHAR(13) + NCHAR(10) - FROM #index_cleanup_report AS i - WHERE i.user_seeks = 0 - AND i.user_scans = 0 - AND i.user_lookups = 0 - AND i.user_updates = 0 + @final_script += N' + /* + -- ============================================================================= + -- DISABLE UNUSED INDEX: ' + + QUOTENAME(i.index_name) + + N' + -- Reason: This index has never been used for reads but has been updated ' + + CONVERT + ( + nvarchar(20), + i.user_updates + ) + + N' times + -- Space reclaimed: ' + + CONVERT + ( + nvarchar(20), + ISNULL + ( + ( + SELECT + CONVERT + ( + decimal(10,2), + SUM(ps.total_space_mb) / 1024.0 + ) + FROM #partition_stats AS ps + WHERE ps.table_name = i.table_name + AND ps.index_name = i.index_name + ), + 0 + ) + ) + N' GB + -- Warning: Verify this index is truly not needed before dropping + -- ============================================================================= + */' + + NCHAR(10) + + N'ALTER INDEX ' + + QUOTENAME(i.index_name) + + N' ON ' + + QUOTENAME(i.table_name) + + N' DISABLE;' + + NCHAR(10) + + NCHAR(10) + FROM + ( + SELECT DISTINCT + icr.database_name, + icr.table_name, + icr.index_name, + icr.user_updates + FROM #index_cleanup_report AS icr + WHERE icr.user_seeks = 0 + AND icr.user_scans = 0 + AND icr.user_lookups = 0 + AND icr.user_updates = 0 + AND icr.action <> N'DROP' + AND icr.action NOT LIKE N'MERGE INTO%' + ) AS i ORDER BY i.table_name, i.index_name; - - PRINT N'----------------------'; - PRINT N'Final script to review. DO NOT EXECUTE WITHOUT CAREFUL REVIEW.'; - PRINT N'Implementation Script:'; - PRINT N'----------------------'; + + + /*Summary*/ SELECT - @sql_len = LEN(@final_script); - - IF @sql_len < 4000 - BEGIN - PRINT @sql; - END - ELSE - BEGIN - WHILE @helper <= @sql_len - BEGIN - SELECT - @sql_debug = - SUBSTRING(@final_script, @helper + 1, 2000) + NCHAR(13) + NCHAR(10); - - PRINT @sql_debug; - SET @helper += 2000; - END; - END; + @final_script += N' + -- ============================================================================= + -- SUMMARY OF CHANGES + -- Total indexes analyzed: ' + + CONVERT + ( + nvarchar(10), + ( + SELECT + COUNT_BIG(*) + FROM #index_cleanup_report AS icr + ) + ) + + N' + -- Indexes recommended for dropping: ' + + CONVERT + ( + nvarchar(10), + ( + SELECT + COUNT_BIG(*) + FROM #index_cleanup_report AS icr + WHERE icr.action = 'DROP' + ) + ) + + N' + -- Indexes recommended for merging: ' + + CONVERT + ( + nvarchar(10), + ( + SELECT + COUNT_BIG(*) + FROM #index_cleanup_report AS icr + WHERE icr.action LIKE 'MERGE INTO%' + ) + ) + + N' + -- Unused indexes found: ' + + CONVERT + ( + nvarchar(10), + ( + SELECT + COUNT_BIG(*) + FROM #index_cleanup_report AS icr + WHERE icr.user_seeks = 0 + AND icr.user_scans = 0 + AND icr.user_lookups = 0 + ) + ) + + N' + -- Estimated space savings: ' + + CONVERT + ( + nvarchar(20), + ISNULL + ( + ( + SELECT + CONVERT + ( + decimal(10,2), + SUM(ps.total_space_mb) / 1024.0 + ) + FROM #partition_stats AS ps + JOIN #index_cleanup_report AS icr + ON ps.table_name = icr.table_name + AND ps.index_name = icr.index_name + WHERE icr.action = 'DROP' + OR icr.action LIKE 'MERGE INTO%' + OR (icr.user_seeks = 0 AND icr.user_scans = 0 AND icr.user_lookups = 0) + ), + 0 + ) + ) + N' GB + -- Estimated write operations reduced: ' + + CONVERT + ( + nvarchar(20), + ISNULL + ( + ( + SELECT + SUM(icr.user_updates) + FROM #index_cleanup_report AS icr + WHERE icr.action = 'DROP' + OR icr.action LIKE 'MERGE INTO%' + OR (icr.user_seeks = 0 AND icr.user_scans = 0 AND icr.user_lookups = 0) + ), + 0 + ) + ) + N' operations + -- ============================================================================= + '; + + SELECT + [text()] = + N'/* Index Cleanup Script for ' + + @database_name + + N' */', + [text()] = + ( + SELECT + NCHAR(10) + + N' ----------------------' + + NCHAR(10) + + N' -- Final script to review. DO NOT EXECUTE WITHOUT CAREFUL REVIEW.' + + NCHAR(10) + + N' -- Implementation Script:' + + NCHAR(10) + + N' ----------------------' + + NCHAR(10) + + @final_script + FOR + XML + PATH(''), + TYPE + ).value('(./text())[1]', 'nvarchar(max)') + FOR + XML + PATH(''), + TYPE; END TRY BEGIN CATCH - PRINT N'Error occurred: ' + ERROR_MESSAGE(); + THROW; END CATCH; END; /*Final End*/ GO From 5d1bc02c78b08867577b13101622a893551251f0 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 11 Mar 2025 13:55:53 -0400 Subject: [PATCH 021/246] Update sp_HumanEventsBlockViewer.sql Smart moves! --- sp_HumanEvents/sp_HumanEventsBlockViewer.sql | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/sp_HumanEvents/sp_HumanEventsBlockViewer.sql b/sp_HumanEvents/sp_HumanEventsBlockViewer.sql index 7ab9756e..66be377c 100644 --- a/sp_HumanEvents/sp_HumanEventsBlockViewer.sql +++ b/sp_HumanEvents/sp_HumanEventsBlockViewer.sql @@ -473,12 +473,14 @@ BEGIN END; /* Parameter validation */ - IF @target_database IS NULL - OR @target_schema IS NULL - OR @target_table IS NULL + IF @target_table IS NULL OR @target_column IS NULL BEGIN - RAISERROR(N'When @target_type is ''table'', you must specify @target_database, @target_schema, @target_table, and @target_column.', 11, 1) WITH NOWAIT; + RAISERROR(N' + When @target_type is ''table'', you must specify @target_table and @target_column. + When @target_database or @target_schema is NULL, they default to DB_NAME() and dbo. + ', + 11, 1) WITH NOWAIT; RETURN; END; @@ -1755,7 +1757,6 @@ BEGIN END; END; - SET @extract_sql = @extract_sql + N' OPTION(RECOMPILE); '; From 1a25c1f34c5f25614ee50de773e0739e346870c4 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 11 Mar 2025 18:08:13 -0400 Subject: [PATCH 022/246] Update sp_IndexCleanup BETA.sql god i hate this thing --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 569 +++++++++++++++--- 1 file changed, 475 insertions(+), 94 deletions(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index 178d279e..2173c458 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -173,11 +173,16 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. @final_script nvarchar(max) = '', /*cursor variables*/ @c_database_id integer, + @c_database_name sysname, + @c_schema_id integer, @c_schema_name sysname, + @c_object_id integer, @c_table_name sysname, + @c_index_id integer, @c_index_name sysname, @c_is_unique bit, @c_filter_definition nvarchar(max), + @index_cursor CURSOR, /*print variables*/ @helper integer = 0, @sql_len integer, @@ -305,19 +310,29 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CREATE TABLE #filtered_objects ( - database_id integer, - object_id integer, - schema_name sysname, - table_name sysname, - PRIMARY KEY (database_id, object_id) + database_id integer NOT NULL, + database_name sysname NOT NULL, + schema_id integer NOT NULL, + schema_name sysname NOT NULL, + object_id integer NOT NULL, + table_name sysname NOT NULL, + index_id integer NOT NULL, + index_name sysname NOT NULL + PRIMARY KEY + (database_id, schema_id, object_id, index_id) ); CREATE TABLE #operational_stats ( database_id integer NOT NULL, + database_name sysname NOT NULL, + schema_id integer NOT NULL, + schema_name sysname NOT NULL, object_id integer NOT NULL, + table_name sysname NOT NULL, index_id integer NOT NULL, + index_name sysname NOT NULL, range_scan_count bigint NULL, singleton_lookup_count bigint NULL, forwarded_fetch_count bigint NULL, @@ -348,17 +363,20 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. page_io_latch_wait_in_ms bigint NULL, page_compression_attempt_count bigint NULL, page_compression_success_count bigint NULL, - PRIMARY KEY CLUSTERED(database_id, object_id, index_id) + PRIMARY KEY CLUSTERED + (database_id, schema_id, object_id, index_id) ); CREATE TABLE #index_details ( database_id integer NOT NULL, - object_id integer NOT NULL, - index_id integer NOT NULL, + database_name sysname NOT NULL, + schema_id integer NOT NULL, schema_name sysname NOT NULL, + object_id integer NOT NULL, table_name sysname NOT NULL, + index_id integer NOT NULL, index_name sysname NULL, column_name sysname NOT NULL, is_primary_key bit NULL, @@ -389,10 +407,12 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #partition_stats ( database_id integer NOT NULL, - object_id integer NOT NULL, - index_id integer NOT NULL, + database_name sysname NOT NULL, + schema_id integer NOT NULL, schema_name sysname NOT NULL, + object_id integer NOT NULL, table_name sysname NOT NULL, + index_id integer NOT NULL, index_name sysname NULL, partition_id bigint NOT NULL, partition_number int NOT NULL, @@ -411,8 +431,12 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #index_analysis ( database_id integer NOT NULL, + database_name sysname NOT NULL, + schema_id integer NOT NULL, schema_name sysname NOT NULL, + object_id integer NOT NULL, table_name sysname NOT NULL, + index_id integer NULL, index_name sysname NOT NULL, is_unique bit NULL, key_columns nvarchar(max) NULL, @@ -429,8 +453,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CREATE TABLE #index_cleanup_report ( + database_id integer NOT NULL, database_name sysname NOT NULL, + schema_id integer NOT NULL, + schema_name sysname NOT NULL, + object_id integer NOT NULL, table_name sysname NOT NULL, + index_id integer NOT NULL, index_name sysname NOT NULL, action nvarchar(max) NULL, cleanup_script nvarchar(max) NULL, @@ -458,8 +487,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CREATE TABLE #index_cleanup_summary ( + database_id integer NOT NULL, database_name sysname NOT NULL, + schema_id integer NOT NULL, + schema_name sysname NOT NULL, + object_id integer NOT NULL, table_name sysname NOT NULL, + index_id integer NOT NULL, index_name sysname NOT NULL, action nvarchar(max) NOT NULL, details nvarchar(max) NULL, @@ -473,11 +507,16 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CREATE TABLE #final_index_actions ( - database_name sysname NOT NULL DEFAULT N'', - table_name sysname NOT NULL DEFAULT N'', - index_name sysname NOT NULL DEFAULT N'', - action nvarchar(max) NOT NULL DEFAULT N'', - script nvarchar(max) NOT NULL DEFAULT N'' + database_id integer NOT NULL, + database_name sysname NOT NULL, + schema_id integer NOT NULL, + schema_name sysname NOT NULL, + object_id integer NOT NULL, + table_name sysname NOT NULL, + index_id integer NOT NULL, + index_name sysname NOT NULL, + action nvarchar(max) NOT NULL, + script nvarchar(max) NOT NULL ); /* @@ -497,12 +536,18 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. @sql = N' SELECT DISTINCT @database_id, - t.object_id, - s.name, - t.name + database_name = DB_NAME(@database_id), + schema_id = t.schema_id, + schema_name = s.name, + object_id = t.object_id, + table_name = t.name, + index_id = i.index_id, + index_name = i.name FROM ' + QUOTENAME(@database_name) + N'.sys.tables AS t JOIN ' + QUOTENAME(@database_name) + N'.sys.schemas AS s ON t.schema_id = s.schema_id + JOIN ' + QUOTENAME(@database_name) + N'.sys.indexes AS i + ON t.object_id = i.object_id LEFT JOIN ' + QUOTENAME(@database_name) + N'.sys.dm_db_index_usage_stats AS us ON t.object_id = us.object_id AND us.database_id = @database_id @@ -568,9 +613,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. (TABLOCK) ( database_id, - object_id, + database_name, + schema_id, schema_name, - table_name + object_id, + table_name, + index_id, + index_name ) EXEC sys.sp_executesql @sql, @@ -610,8 +659,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. @sql += N' SELECT os.database_id, + database_name = DB_NAME(os.database_id), + schema_id = s.schema_id, + schema_name = s.name, os.object_id, + table_name = t.name, os.index_id, + index_name = i.name, range_scan_count = SUM(os.range_scan_count), singleton_lookup_count = SUM(os.singleton_lookup_count), forwarded_fetch_count = SUM(os.forwarded_fetch_count), @@ -649,6 +703,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. NULL, NULL ) AS os + JOIN ' + QUOTENAME(@database_name) + N'.sys.tables AS t + ON os.object_id = t.object_id + JOIN ' + QUOTENAME(@database_name) + N'.sys.schemas AS s + ON t.schema_id = s.schema_id + JOIN ' + QUOTENAME(@database_name) + N'.sys.indexes AS i + ON os.object_id = i.object_id + AND os.index_id = i.index_id WHERE EXISTS ( SELECT @@ -659,8 +720,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ) GROUP BY os.database_id, + DB_NAME(os.database_id), + s.schema_id, + s.name, os.object_id, - os.index_id + t.name, + os.index_id, + i.name OPTION(RECOMPILE);'; IF @debug = 1 @@ -674,8 +740,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. (TABLOCK) ( database_id, + database_name, + schema_id, + schema_name, object_id, + table_name, index_id, + index_name, range_scan_count, singleton_lookup_count, forwarded_fetch_count, @@ -737,8 +808,10 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. @sql += N' SELECT database_id = @database_id, + database_name = DB_NAME(@database_id), t.object_id, i.index_id, + s.schema_id, schema_name = s.name, table_name = t.name, index_name = i.name, @@ -852,11 +925,16 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ( SELECT 1/0 - FROM ' + QUOTENAME(@database_name) + N'.sys.dm_db_partition_stats ps + FROM ' + QUOTENAME(@database_name) + + CONVERT + ( + nvarchar(MAX), + N'.sys.dm_db_partition_stats ps WHERE ps.object_id = t.object_id AND ps.index_id = 1 AND ps.row_count >= @min_rows - )'; + )' + ); IF @object_id IS NOT NULL BEGIN @@ -865,7 +943,10 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END; SELECT - @sql += N' + @sql += CONVERT + ( + nvarchar(max), + N' AND NOT EXISTS ( SELECT @@ -875,7 +956,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND so.is_ms_shipped = 0 AND so.type = N''TF'' ) - OPTION(RECOMPILE);'; + OPTION(RECOMPILE);' + ); IF @debug = 1 BEGIN @@ -889,8 +971,10 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. (TABLOCK) ( database_id, + database_name, object_id, index_id, + schema_id, schema_name, table_name, index_name, @@ -949,8 +1033,10 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. @sql += N' SELECT database_id = @database_id, + database_name = DB_NAME(@database_id), x.object_id, x.index_id, + x.schema_id, x.schema_name, x.table_name, x.index_name, @@ -974,6 +1060,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SELECT ps.object_id, ps.index_id, + s.schema_id, schema_name = s.name, table_name = t.name, index_name = i.name, @@ -1020,6 +1107,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. t.name, i.name, i.data_space_id, + s.schema_id, s.name, p.partition_number, p.data_compression_desc, @@ -1083,8 +1171,10 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #partition_stats WITH(TABLOCK) ( database_id, + database_name, object_id, index_id, + schema_id, schema_name, table_name, index_name, @@ -1127,8 +1217,12 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. (TABLOCK) ( database_id, + database_name, + schema_id, schema_name, table_name, + object_id, + index_id, index_name, is_unique, key_columns, @@ -1137,8 +1231,12 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ) SELECT @database_id, + database_name = DB_NAME(@database_id), + id1.schema_id, id1.schema_name, id1.table_name, + id1.object_id, + id1.index_id, id1.index_name, id1.is_unique, key_columns = @@ -1199,8 +1297,10 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. WHERE id1.is_eligible_for_dedupe = 1 GROUP BY id1.schema_name, + id1.schema_id, id1.table_name, id1.index_name, + id1.index_id, id1.is_unique, id1.object_id, id1.index_id, @@ -1223,9 +1323,6 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END; /*Analyze indexes*/ - DECLARE - @index_cursor CURSOR; - SET @index_cursor = CURSOR LOCAL STATIC @@ -1234,8 +1331,12 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. FOR SELECT DISTINCT ia.database_id, + ia.database_name, + ia.schema_id, ia.schema_name, + ia.object_id, ia.table_name, + ia.index_id, ia.index_name, ia.is_unique, ia.filter_definition @@ -1250,8 +1351,12 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. FROM @index_cursor INTO @c_database_id, + @c_database_name, + @c_schema_id, @c_schema_name, + @c_object_id, @c_table_name, + @c_index_id, @c_index_name, @c_is_unique, @c_filter_definition; @@ -1268,26 +1373,18 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. IndexColumns AS ( SELECT - id.database_id, - id.schema_name, - id.table_name, - id.index_name, - id.column_name, - id.is_included_column, - id.key_ordinal, - id.is_eligible_for_dedupe + id.* FROM #index_details id WHERE id.database_id = @c_database_id - AND id.schema_name = @c_schema_name - AND id.table_name = @c_table_name - AND id.is_eligible_for_dedupe = 1 + AND id.object_id = @c_object_id + AND id.is_eligible_for_dedupe = 1 ), CurrentIndexColumns AS ( SELECT ic.* FROM IndexColumns AS ic - WHERE ic.index_name = @c_index_name + WHERE ic.index_id = @c_index_id AND ic.is_eligible_for_dedupe = 1 ), OtherIndexColumns AS @@ -1295,7 +1392,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SELECT ic.* FROM IndexColumns AS ic - WHERE ic.index_name <> @c_index_name + WHERE ic.index_id <> @c_index_id AND ic.is_eligible_for_dedupe = 1 ) UPDATE @@ -1439,8 +1536,12 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. FROM @index_cursor INTO @c_database_id, + @c_database_name, + @c_schema_id, @c_schema_name, + @c_object_id, @c_table_name, + @c_index_id, @c_index_name, @c_is_unique, @c_filter_definition; @@ -1491,8 +1592,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. WITH (TABLOCK) ( + database_id, database_name, + schema_id, + schema_name, table_name, + object_id, + index_id, index_name, action, cleanup_script, @@ -1515,8 +1621,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. page_lock_wait_in_ms ) SELECT + @database_id, @database_name, + ia.schema_id, + ia.schema_name, ia.table_name, + ia.object_id, + ia.index_id, ia.index_name, ia.action, cleanup_script = @@ -1841,8 +1952,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. WITH (TABLOCK) ( + database_id, database_name, + schema_id, + schema_name, table_name, + object_id, + index_id, index_name, action, details, @@ -1853,8 +1969,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. uptime_warning ) SELECT + icr.database_id, icr.database_name, + icr.schema_id, + icr.schema_name, icr.table_name, + icr.object_id, + icr.index_id, icr.index_name, action = CASE @@ -2221,72 +2342,326 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. RAISERROR('Performing #final_index_actions insert', 0, 0) WITH NOWAIT; END; - WITH - IndexActions AS + -- Replace the existing INSERT into #final_index_actions for MERGE operations with this: + WITH + MergeTargets AS ( - SELECT - icr.database_name, - icr.table_name, - icr.index_name, - icr.action, - icr.cleanup_script, - n = ROW_NUMBER() OVER + -- Get distinct target indexes for merges + SELECT DISTINCT + ia.database_id, + ia.database_name, + ia.schema_id, + ia.schema_name, + ia.object_id, + ia.table_name, + target_index = + SUBSTRING ( - PARTITION BY - icr.table_name, - icr.index_name - ORDER BY - CASE - WHEN icr.action LIKE N'MERGE INTO%' - THEN 1 - WHEN icr.action = N'DROP' - THEN 2 - ELSE 3 - END + ia.action, + 12, + CHARINDEX + ( + N' ', + ia.action + + N' ', + 12 + ) - 12 ) - FROM #index_cleanup_report icr + FROM #index_cleanup_report ia + WHERE ia.action LIKE N'MERGE INTO%' ) - INSERT INTO + -- Insert a single CREATE INDEX statement for each target index + INSERT INTO #final_index_actions WITH (TABLOCK) ( - database_name, - table_name, - index_name, - action, + database_id, + database_name, + schema_id, + schema_name, + object_id, + table_name, + index_id, + index_name, + action, script ) - SELECT - ia.database_name, - ia.table_name, - ia.index_name, - ia.action, - script = + SELECT + mt.database_id, + mt.database_name, + mt.schema_id, + mt.schema_name, + mt.object_id, + mt.table_name, + index_id = + ISNULL + ( + ( + SELECT TOP (1) + ia.index_id + FROM #index_analysis ia + WHERE ia.database_id = mt.database_id + AND ia.table_name = mt.table_name + AND ia.index_name = mt.target_index + ), + 0 + ), + mt.target_index, + action = + N'MERGE CONSOLIDATED', + script = + N'CREATE INDEX ' + + QUOTENAME(mt.target_index) + + N' ON ' + + QUOTENAME(mt.database_name) + + N'.' + + QUOTENAME(mt.schema_name) + + N'.' + + QUOTENAME(mt.table_name) + + N' (' + + -- Get key columns from one of the indexes being merged + ( + SELECT TOP (1) + ia.key_columns + FROM #index_analysis ia + WHERE ia.database_id = mt.database_id + AND ia.table_name = mt.table_name + AND ia.index_name = mt.target_index + ) + + N')' + + -- Include all distinct columns from all indexes being merged into this target CASE - WHEN ia.action LIKE N'MERGE INTO%' - THEN ISNULL + WHEN EXISTS ( - ia.cleanup_script, - N'-- Unable to generate merge script for ' + - ia.index_name - ) - WHEN ia.action = N'DROP' - THEN N'ALTER INDEX ' + - QUOTENAME(ia.index_name) + - N' ON ' + - QUOTENAME(ia.table_name) + - N' DISABLE;' - ELSE N'???' - END - FROM IndexActions AS ia - WHERE ia.n = 1 - AND - ( - ia.cleanup_script IS NOT NULL - OR action = N'DROP' + SELECT + 1/0 + FROM #index_cleanup_report icr + WHERE icr.database_id = mt.database_id + AND icr.table_name = mt.table_name + AND icr.action LIKE N'MERGE INTO ' + mt.target_index + N'%' + AND + ( + EXISTS + ( + SELECT + 1/0 + FROM #index_analysis ia + WHERE ia.database_id = icr.database_id + AND ia.table_name = icr.table_name + AND ia.index_name = icr.index_name + AND ia.included_columns IS NOT NULL + ) + OR icr.action LIKE N'%ADD %' + ) + ) + THEN N' INCLUDE (' + + STUFF + ( + ( + SELECT DISTINCT + N', ' + + col + FROM + ( + -- Get included columns from all source indexes + SELECT + col = LTRIM(RTRIM(value.c.value('.', 'sysname'))) + FROM #index_cleanup_report icr + CROSS APPLY + ( + SELECT + ia.included_columns + FROM #index_analysis ia + WHERE ia.database_id = icr.database_id + AND ia.table_name = icr.table_name + AND ia.index_name = icr.index_name + ) src + CROSS APPLY + ( + SELECT + cols = + CONVERT + ( + xml, + '' + + REPLACE + ( + ISNULL + ( + src.included_columns, + '' + ), + ', ', + '') + + '' + ) + ) x + CROSS APPLY x.cols.nodes('/c') AS value(c) + WHERE icr.database_id = mt.database_id + AND icr.table_name = mt.table_name + AND icr.action LIKE N'MERGE INTO ' + mt.target_index + N'%' + + UNION + + -- Get missing columns which need to be added + SELECT + col = + LTRIM(RTRIM(value.c.value('.', 'sysname'))) + FROM #index_cleanup_report icr + CROSS APPLY + ( + SELECT + missing_cols = + SUBSTRING + ( + icr.action, + CHARINDEX('ADD ', icr.action) + 4, + LEN(icr.action) + ) + WHERE icr.action LIKE N'%ADD %' + ) mc + CROSS APPLY + ( + SELECT + cols = + CONVERT + ( + xml, + '' + + REPLACE + ( + ISNULL + ( + mc.missing_cols, + '' + ), + ', ', + '' + ) + '' + ) + ) x + CROSS APPLY x.cols.nodes('/c') AS value(c) + WHERE icr.database_id = mt.database_id + AND icr.table_name = mt.table_name + AND icr.action LIKE N'MERGE INTO ' + mt.target_index + N'%' + AND icr.action LIKE N'%ADD %' + ) AS all_columns + WHERE DATALENGTH(col) > 0 + FOR + XML + PATH(''), + TYPE + ).value('.', 'nvarchar(max)'), + 1, + 2, + '' + ) + + N')' + ELSE N'' + END + + -- Add partitioning if needed + CASE + WHEN EXISTS + ( + SELECT + 1/0 + FROM #partition_stats ps + WHERE ps.database_id = mt.database_id + AND ps.table_name = mt.table_name + AND ps.index_name = mt.target_index + AND ps.partition_function_name IS NOT NULL + ) + THEN + ( + SELECT TOP (1) + N' ON ' + + QUOTENAME(ps.partition_function_name) + + '(' + + ps.partition_columns + + ')' + FROM #partition_stats ps + WHERE ps.database_id = mt.database_id + AND ps.table_name = mt.table_name + AND ps.index_name = mt.target_index + AND ps.partition_function_name IS NOT NULL + ) + ELSE N'' + END + + -- Add filter definition if needed + CASE + WHEN EXISTS + ( + SELECT + 1/0 + FROM #index_analysis ia + WHERE ia.database_id = mt.database_id + AND ia.table_name = mt.table_name + AND ia.index_name = mt.target_index + AND ia.filter_definition IS NOT NULL + ) + THEN + ( + SELECT TOP (1) + N' WHERE ' + + ia.filter_definition + FROM #index_analysis ia + WHERE ia.database_id = mt.database_id + AND ia.table_name = mt.table_name + AND ia.index_name = mt.target_index + AND ia.filter_definition IS NOT NULL + ) + ELSE N'' + END + + -- Add WITH options + N' WITH (DROP_EXISTING = ON, FILLFACTOR = 100, SORT_IN_TEMPDB = ON, ONLINE = ' + + CASE + WHEN @online = 'true' + THEN N'ON' + ELSE N'OFF' + END + + N', DATA_COMPRESSION = PAGE);' + FROM MergeTargets mt; + + -- Then add DISABLE statements for all source indexes + INSERT INTO + #final_index_actions + WITH + (TABLOCK) + ( + database_id, + database_name, + schema_id, + schema_name, + object_id, + table_name, + index_id, + index_name, + action, + script ) - OPTION(RECOMPILE); + SELECT + icr.database_id, + icr.database_name, + icr.schema_id, + icr.schema_name, + icr.object_id, + icr.table_name, + icr.index_id, + icr.index_name, + action = N'DISABLE MERGED', + script = + N'ALTER INDEX ' + + QUOTENAME(icr.index_name) + + N' ON ' + + QUOTENAME(icr.database_name) + + N'.' + + QUOTENAME(icr.schema_name) + + N'.' + + QUOTENAME(icr.table_name) + + N' DISABLE;' + FROM #index_cleanup_report icr + WHERE icr.action LIKE N'MERGE INTO%'; IF ROWCOUNT_BIG() = 0 BEGIN IF @debug = 1 BEGIN RAISERROR('No rows inserted into #final_index_actions', 0, 0) WITH NOWAIT END; END; @@ -2654,6 +3029,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. N'ALTER INDEX ' + QUOTENAME(i.index_name) + N' ON ' + + QUOTENAME(i.database_name) + + N'.' + + QUOTENAME + (i.schema_name) + + N'.' + QUOTENAME(i.table_name) + N' DISABLE;' + NCHAR(10) + @@ -2662,6 +3042,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ( SELECT DISTINCT icr.database_name, + icr.schema_name, icr.table_name, icr.index_name, icr.user_updates From 813fa7a01fea64c7a86083b002fd6847d2536a79 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 11 Mar 2025 20:39:29 -0400 Subject: [PATCH 023/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 77 +++++++++++++------ 1 file changed, 53 insertions(+), 24 deletions(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index 2173c458..ffc5a527 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -2730,20 +2730,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -- MERGE INDEX: ' + QUOTENAME(f.index_name) + N' into ' + - QUOTENAME - ( - SUBSTRING - ( - f.action, - 12, - CHARINDEX - ( - N' ', - f.action, - 12 - ) - 12 - ) - )+ + CASE + WHEN f.action = 'MERGE CONSOLIDATED' + THEN QUOTENAME(f.index_name) + ELSE 'Unknown Target' + END + N' -- Reason: This index overlaps with another index and can be consolidated -- Original definition: ' + @@ -2847,7 +2838,28 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. NCHAR(10) + NCHAR(10) FROM #final_index_actions AS f - WHERE f.action LIKE N'MERGE INTO%' + WHERE f.action = N'MERGE CONSOLIDATED' + ORDER BY + f.table_name, + f.index_name; + + /*Disable merged indexes*/ + SELECT + @final_script += N' + /* + -- ============================================================================= + -- DISABLE MERGED INDEX: ' + + QUOTENAME(f.index_name) + + N' + -- Reason: This index has been merged into another index + -- ============================================================================= + */' + + NCHAR(10) + + f.script + + NCHAR(10) + + NCHAR(10) + FROM #final_index_actions AS f + WHERE f.action = N'DISABLE MERGED' ORDER BY f.table_name, f.index_name; @@ -2984,9 +2996,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ORDER BY f.table_name, f.index_name; - - - + /*Unused indexes*/ SELECT @final_script += N' @@ -3053,6 +3063,15 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND icr.user_updates = 0 AND icr.action <> N'DROP' AND icr.action NOT LIKE N'MERGE INTO%' + AND NOT EXISTS + ( + SELECT + 1/0 + FROM #final_index_actions AS fia + WHERE fia.index_name = icr.index_name + AND fia.table_name = icr.table_name + AND fia.action IN (N'MERGE CONSOLIDATED', N'DISABLE MERGED') + ) ) AS i ORDER BY i.table_name, @@ -3128,11 +3147,16 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ) FROM #partition_stats AS ps JOIN #index_cleanup_report AS icr - ON ps.table_name = icr.table_name - AND ps.index_name = icr.index_name + ON ps.table_name = icr.table_name + AND ps.index_name = icr.index_name WHERE icr.action = 'DROP' - OR icr.action LIKE 'MERGE INTO%' - OR (icr.user_seeks = 0 AND icr.user_scans = 0 AND icr.user_lookups = 0) + OR icr.action LIKE 'MERGE INTO%' + OR + ( + icr.user_seeks = 0 + AND icr.user_scans = 0 + AND icr.user_lookups = 0 + ) ), 0 ) @@ -3148,8 +3172,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SUM(icr.user_updates) FROM #index_cleanup_report AS icr WHERE icr.action = 'DROP' - OR icr.action LIKE 'MERGE INTO%' - OR (icr.user_seeks = 0 AND icr.user_scans = 0 AND icr.user_lookups = 0) + OR icr.action LIKE 'MERGE INTO%' + OR + ( + icr.user_seeks = 0 + AND icr.user_scans = 0 + AND icr.user_lookups = 0 + ) ), 0 ) From dcbcb802451914dc313a2e91a92b363a60821ece Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Wed, 12 Mar 2025 11:41:00 -0400 Subject: [PATCH 024/246] giving up on that keping the old version, but starting over --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 2147 +--------- .../sp_IndexCleanup BETA_Original.sql | 3547 +++++++++++++++++ 2 files changed, 3745 insertions(+), 1949 deletions(-) create mode 100644 sp_IndexCleanup BETA/sp_IndexCleanup BETA_Original.sql diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index ffc5a527..9dd0deb4 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -170,23 +170,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. @database_id integer = NULL, @object_id integer = NULL, @full_object_name nvarchar(768) = NULL, - @final_script nvarchar(max) = '', - /*cursor variables*/ - @c_database_id integer, - @c_database_name sysname, - @c_schema_id integer, - @c_schema_name sysname, - @c_object_id integer, - @c_table_name sysname, - @c_index_id integer, - @c_index_name sysname, - @c_is_unique bit, - @c_filter_definition nvarchar(max), - @index_cursor CURSOR, /*print variables*/ - @helper integer = 0, - @sql_len integer, - @sql_debug nvarchar(max) = N'', @online bit = CASE WHEN @@ -269,7 +253,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END; END; - -- Parameter validation + /* Parameter validation */ IF @min_reads < 0 OR @min_reads IS NULL BEGIN @@ -314,7 +298,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. database_name sysname NOT NULL, schema_id integer NOT NULL, schema_name sysname NOT NULL, - object_id integer NOT NULL, + object_id integer NOT NULL, table_name sysname NOT NULL, index_id integer NOT NULL, index_name sysname NOT NULL @@ -446,77 +430,29 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. superseded_by sysname NULL, missing_columns nvarchar(max) NULL, action nvarchar(max) NULL, + target_index_name sysname NULL, + consolidation_rule varchar(50) NULL, + index_priority int NULL, INDEX c CLUSTERED (database_id, schema_name, table_name, index_name) ); CREATE TABLE - #index_cleanup_report - ( - database_id integer NOT NULL, - database_name sysname NOT NULL, - schema_id integer NOT NULL, - schema_name sysname NOT NULL, - object_id integer NOT NULL, - table_name sysname NOT NULL, - index_id integer NOT NULL, - index_name sysname NOT NULL, - action nvarchar(max) NULL, - cleanup_script nvarchar(max) NULL, - original_definition nvarchar(max) NULL, - /*Usage details*/ - user_seeks bigint NULL, - user_scans bigint NULL, - user_lookups bigint NULL, - user_updates bigint NULL, - last_user_seek datetime NULL, - last_user_scan datetime NULL, - last_user_lookup datetime NULL, - last_user_update datetime NULL, - /*Operational stats*/ - range_scan_count bigint NULL, - singleton_lookup_count bigint NULL, - leaf_insert_count bigint NULL, - leaf_update_count bigint NULL, - leaf_delete_count bigint NULL, - page_lock_count bigint NULL, - page_lock_wait_count bigint NULL, - page_lock_wait_in_ms bigint NULL - ); - - CREATE TABLE - #index_cleanup_summary - ( - database_id integer NOT NULL, - database_name sysname NOT NULL, - schema_id integer NOT NULL, - schema_name sysname NOT NULL, - object_id integer NOT NULL, - table_name sysname NOT NULL, - index_id integer NOT NULL, - index_name sysname NOT NULL, - action nvarchar(max) NOT NULL, - details nvarchar(max) NULL, - current_definition nvarchar(max) NOT NULL, - proposed_definition nvarchar(max) NULL, - usage_summary nvarchar(max) NULL, - operational_summary nvarchar(max) NULL, - uptime_warning nvarchar(512) NULL - ); - - CREATE TABLE - #final_index_actions + #index_consolidation ( - database_id integer NOT NULL, + database_id int NOT NULL, database_name sysname NOT NULL, - schema_id integer NOT NULL, + schema_id int NOT NULL, schema_name sysname NOT NULL, - object_id integer NOT NULL, + object_id int NOT NULL, table_name sysname NOT NULL, - index_id integer NOT NULL, + index_id int NOT NULL, index_name sysname NOT NULL, - action nvarchar(max) NOT NULL, - script nvarchar(max) NOT NULL + target_index_name sysname NULL, + consolidation_rule varchar(50) NULL, + index_priority int NULL, + action varchar(50) NULL, + PRIMARY KEY (database_id, object_id, index_id) ); /* @@ -613,10 +549,10 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. (TABLOCK) ( database_id, - database_name, - schema_id, + database_name, + schema_id, schema_name, - object_id, + object_id, table_name, index_id, index_name @@ -1319,1901 +1255,214 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. IF @debug = 1 BEGIN - RAISERROR('Starting cursor', 0, 0) WITH NOWAIT; - END; - - /*Analyze indexes*/ - SET @index_cursor = CURSOR - LOCAL - STATIC - FORWARD_ONLY - READ_ONLY - FOR - SELECT DISTINCT - ia.database_id, - ia.database_name, - ia.schema_id, - ia.schema_name, - ia.object_id, - ia.table_name, - ia.index_id, - ia.index_name, - ia.is_unique, - ia.filter_definition - FROM #index_analysis AS ia - ORDER BY - ia.table_name, - ia.index_name; - - OPEN @index_cursor; - - FETCH NEXT - FROM @index_cursor - INTO - @c_database_id, - @c_database_name, - @c_schema_id, - @c_schema_name, - @c_object_id, - @c_table_name, - @c_index_id, - @c_index_name, - @c_is_unique, - @c_filter_definition; - - WHILE @@FETCH_STATUS = 0 - BEGIN - - IF @debug = 1 - BEGIN - RAISERROR('Performing #index_analysis update', 0, 0) WITH NOWAIT; - END; - - WITH - IndexColumns AS - ( - SELECT - id.* - FROM #index_details id - WHERE id.database_id = @c_database_id - AND id.object_id = @c_object_id - AND id.is_eligible_for_dedupe = 1 - ), - CurrentIndexColumns AS - ( - SELECT - ic.* - FROM IndexColumns AS ic - WHERE ic.index_id = @c_index_id - AND ic.is_eligible_for_dedupe = 1 - ), - OtherIndexColumns AS - ( - SELECT - ic.* - FROM IndexColumns AS ic - WHERE ic.index_id <> @c_index_id - AND ic.is_eligible_for_dedupe = 1 - ) - UPDATE - ia - SET - ia.is_redundant = - CASE - WHEN NOT EXISTS - ( - SELECT - 1/0 - FROM CurrentIndexColumns cic - WHERE cic.is_included_column = 0 /* Only check key columns */ - AND NOT EXISTS - ( - SELECT - 1/0 - FROM OtherIndexColumns oic - WHERE oic.column_name = cic.column_name - AND oic.is_included_column = 0 /* Must be in key columns */ - AND oic.key_ordinal <= cic.key_ordinal /* Check leading edge */ - ) - ) - AND - ( - /* Check included columns separately since order doesn't matter */ - NOT EXISTS - ( - SELECT - 1/0 - FROM CurrentIndexColumns cic - WHERE cic.is_included_column = 1 - AND NOT EXISTS - ( - SELECT - 1/0 - FROM OtherIndexColumns oic - WHERE oic.column_name = cic.column_name - AND - ( - oic.is_included_column = 1 - OR oic.is_included_column = 0 /* Include cols can be covered by key cols */ - ) - ) - ) - ) - AND ISNULL(REPLACE(REPLACE(REPLACE(ia.filter_definition, ' ', ''), '(', ''), ')', ''), '') = - ISNULL(REPLACE(REPLACE(REPLACE(@c_filter_definition, ' ', ''), '(', ''), ')', ''), '') - AND - ( - ia.is_unique = 0 - OR - ( - ia.is_unique = 1 - AND @c_is_unique = 1 - ) - ) - THEN 1 - ELSE 0 - END, - ia.superseded_by = - CASE - WHEN NOT EXISTS - ( - SELECT - 1/0 - FROM CurrentIndexColumns cic - WHERE cic.is_included_column = 0 /* Only check key columns */ - AND NOT EXISTS - ( - SELECT - 1/0 - FROM OtherIndexColumns oic - WHERE oic.column_name = cic.column_name - AND oic.is_included_column = 0 /* Must be in key columns */ - AND oic.key_ordinal <= cic.key_ordinal /* Check leading edge */ - ) - ) - AND - ( - /* Check included columns separately since order doesn't matter */ - NOT EXISTS - ( - SELECT - 1/0 - FROM CurrentIndexColumns cic - WHERE cic.is_included_column = 1 - AND NOT EXISTS - ( - SELECT - 1/0 - FROM OtherIndexColumns oic - WHERE oic.column_name = cic.column_name - AND - ( - oic.is_included_column = 1 - OR oic.is_included_column = 0 /* Include cols can be covered by key cols */ - ) - ) - ) - ) - AND ISNULL(ia.filter_definition, '') = ISNULL(@c_filter_definition, '') - AND - ( - ia.is_unique = 0 - OR @c_is_unique = 1 - ) - THEN @c_index_name - ELSE ia.superseded_by - END, - ia.missing_columns = - STUFF - ( - ( - SELECT DISTINCT - N', ' + - oic.column_name - FROM OtherIndexColumns oic - WHERE NOT EXISTS - ( - SELECT - 1/0 - FROM CurrentIndexColumns cic - WHERE cic.column_name = oic.column_name - ) - FOR XML - PATH(''), - TYPE - ).value('.', 'nvarchar(max)'), - 1, - 2, - '' - ) - FROM #index_analysis ia - WHERE ia.database_id = @c_database_id - AND ia.schema_name = @c_schema_name - AND ia.table_name = @c_table_name - AND ia.index_name <> @c_index_name; - - FETCH NEXT - FROM @index_cursor - INTO - @c_database_id, - @c_database_name, - @c_schema_id, - @c_schema_name, - @c_object_id, - @c_table_name, - @c_index_id, - @c_index_name, - @c_is_unique, - @c_filter_definition; + RAISERROR('Starting updates', 0, 0) WITH NOWAIT; END; - IF @debug = 1 - BEGIN - RAISERROR('Performing #index_analysis update after cursor', 0, 0) WITH NOWAIT; - END; - - /*Determine actions*/ - UPDATE + /* Calculate index priority scores based on actual columns that exist */ + UPDATE #index_analysis - SET - action = - CASE - WHEN is_redundant = 1 - THEN N'DROP' - WHEN superseded_by IS NOT NULL - AND missing_columns IS NULL - THEN N'MERGE INTO ' + - superseded_by - WHEN superseded_by IS NOT NULL - AND missing_columns IS NOT NULL - THEN N'MERGE INTO ' + - superseded_by + - N' (ADD ' + - missing_columns + - N')' - ELSE N'KEEP' - END; - - IF @debug = 1 - BEGIN - SELECT - table_name = '#index_analysis after update', - ia.* - FROM #index_analysis AS ia; - END; - - IF @debug = 1 - BEGIN - RAISERROR('Performing #index_cleanup_report insert', 0, 0) WITH NOWAIT; - END; - - INSERT INTO - #index_cleanup_report - WITH - (TABLOCK) - ( - database_id, - database_name, - schema_id, - schema_name, - table_name, - object_id, - index_id, - index_name, - action, - cleanup_script, - original_definition, - user_seeks, - user_scans, - user_lookups, - user_updates, - last_user_seek, - last_user_scan, - last_user_lookup, - last_user_update, - range_scan_count, - singleton_lookup_count, - leaf_insert_count, - leaf_update_count, - leaf_delete_count, - page_lock_count, - page_lock_wait_count, - page_lock_wait_in_ms - ) - SELECT - @database_id, - @database_name, - ia.schema_id, - ia.schema_name, - ia.table_name, - ia.object_id, - ia.index_id, - ia.index_name, - ia.action, - cleanup_script = - CASE - WHEN ia.action = N'DROP' - THEN NCHAR(10) + - N'DROP INDEX ' + - QUOTENAME(ia.index_name) + - N' ON ' + - QUOTENAME(DB_NAME(ia.database_id)) + - N'.' + - QUOTENAME(ia.schema_name) + - N'.' + - QUOTENAME(ia.table_name) + - N';' - WHEN ia.action LIKE N'MERGE INTO%' - THEN NCHAR(10) + - N'CREATE ' + - CASE - WHEN ia.is_unique = 1 - THEN N'UNIQUE ' - ELSE N'' - END + - N'INDEX ' + - QUOTENAME(ia.superseded_by) + - NCHAR(10) + - N'ON ' + - QUOTENAME(DB_NAME(ia.database_id)) + - N'.' + - QUOTENAME(ia.schema_name) + - N'.' + - QUOTENAME(ia.table_name) + - NCHAR(10) + - N' (' + - ISNULL(superseding.key_columns, ia.key_columns) + - N')' + - NCHAR(10) + - CASE - WHEN - ( - superseding.included_columns IS NOT NULL - OR ia.included_columns IS NOT NULL - ) - OR ia.missing_columns IS NOT NULL - THEN N' INCLUDE' + - NCHAR(10) + - N' (' + - -- Combine all INCLUDE columns with proper parsing - STUFF - ( - ( - SELECT DISTINCT - N', ' + - column_value - FROM - ( - -- From superseding index - SELECT - column_value = - LTRIM(RTRIM(value.c.value('.', 'sysname'))) - FROM - ( - SELECT - Columns = - CONVERT - ( - xml, - '' + - REPLACE - ( - ISNULL - ( - superseding.included_columns, - '' - ), - ', ', - '') + - '' - ) - ) t - CROSS APPLY t.Columns.nodes('/c') AS value(c) - - UNION - - -- From current index - SELECT - column_value = - LTRIM(RTRIM(value.c.value('.', 'sysname'))) - FROM - ( - SELECT - Columns = - CONVERT - ( - xml, - '' + - REPLACE - ( - ISNULL - ( - ia.included_columns, - '' - ), - ', ', - '' - ) + - '' - ) - ) t - CROSS APPLY t.Columns.nodes('/c') AS value(c) - - UNION - - -- From missing columns - SELECT - column_value = - LTRIM(RTRIM(value.c.value('.', 'sysname'))) - FROM - ( - SELECT - Columns = - CONVERT - ( - xml, - '' + - REPLACE - ( - ISNULL - ( - ia.missing_columns, - '' - ), - ', ', - '' - ) + '' - ) - ) t - CROSS APPLY t.Columns.nodes('/c') AS value(c) - ) AS all_columns - WHERE LEN(column_value) > 0 - FOR - XML - PATH(''), - TYPE - ).value('.', 'nvarchar(max)'), - 1, - 2, - '' - ) + - N')' - ELSE N'' - END + - CASE - /* Check for partitioning in the superseding index first */ - WHEN EXISTS - ( - SELECT - 1/0 - FROM #partition_stats ps_super - WHERE ps_super.table_name = ia.table_name - AND ps_super.index_name = ia.superseded_by - AND ps_super.partition_function_name IS NOT NULL - ) - THEN - ( - SELECT TOP (1) - NCHAR(10) + - N' ON ' + - QUOTENAME(ps_super.partition_function_name) + - N'(' + - ps_super.partition_columns + - N')' - FROM #partition_stats ps_super - WHERE ps_super.table_name = ia.table_name - AND ps_super.index_name = ia.superseded_by - ) - /* Fall back to the current index's partitioning if available */ - WHEN ps.partition_function_name IS NOT NULL - THEN NCHAR(10) + - N' ON ' + - QUOTENAME(ps.partition_function_name) + - N'(' + - ps.partition_columns + - N')' - ELSE N'' - END + - CASE - WHEN ia.filter_definition IS NOT NULL - THEN NCHAR(10) + - N' WHERE ' + - ia.filter_definition - ELSE N'' - END + - NCHAR(10) + - N' WITH ' + - NCHAR(10) + - N' (DROP_EXISTING = ON, FILLFACTOR = 100, SORT_IN_TEMPDB = ON, ONLINE = ' + - CASE - WHEN @online = 'true' /*Best effort at detecting online index abilities*/ - THEN N'ON' - ELSE N'OFF' - END + - CASE - WHEN ps.data_compression_desc <> N'NONE' - THEN N', DATA_COMPRESSION = ' + - ps.data_compression_desc - ELSE N', DATA_COMPRESSION = PAGE' /* Add PAGE compression by default for merged indexes */ - END + - N');' + - NCHAR(10) + - NCHAR(10) + - N'ALTER INDEX ' + - QUOTENAME(ia.index_name) + - N' ON ' + - QUOTENAME(DB_NAME(ia.database_id)) + - N'.' + - QUOTENAME(ia.schema_name) + - N'.' + - QUOTENAME(ia.table_name) + - N' DISABLE' - ELSE N'' - END + - N';', - original_definition = - NCHAR(10) + - N' -- CREATE ' + - CASE - WHEN ia.is_unique = 1 - THEN N'UNIQUE ' - ELSE N'' - END + - N'INDEX ' + - QUOTENAME(ia.index_name) + - NCHAR(10) + - N' -- ON ' + - QUOTENAME(DB_NAME(ia.database_id)) + - N'.' + - QUOTENAME(ia.schema_name) + - N'.' + - QUOTENAME(ia.table_name) + - NCHAR(10) + - N' -- (' + - ia.key_columns + - N')' + - CASE - WHEN ia.included_columns IS NOT NULL - THEN NCHAR(10) + - N' -- INCLUDE' + - NCHAR(10) + - N' -- (' + - ia.included_columns + - N')' - ELSE N'' - END + - CASE - WHEN ps.partition_function_name IS NOT NULL - THEN NCHAR(10) + - N' -- ON ' + - QUOTENAME(ps.partition_function_name) + - N'(' + - ps.partition_columns + - N')' - ELSE N'' - END + - CASE - WHEN ia.filter_definition IS NOT NULL - THEN NCHAR(10) + - N' -- WHERE ' + - QUOTENAME(ia.filter_definition, '()') - ELSE N'' - END + - N';' + - NCHAR(10), - id.user_seeks, - id.user_scans, - id.user_lookups, - id.user_updates, - id.last_user_seek, - id.last_user_scan, - id.last_user_lookup, - id.last_user_update, - os.range_scan_count, - os.singleton_lookup_count, - os.leaf_insert_count, - os.leaf_update_count, - os.leaf_delete_count, - os.page_lock_count, - os.page_lock_wait_count, - os.page_lock_wait_in_ms - FROM #index_analysis ia - LEFT JOIN #partition_stats ps - ON ia.table_name = ps.table_name - AND ia.index_name = ps.index_name - LEFT JOIN #index_details id - ON ia.table_name = id.table_name - AND ia.index_name = id.index_name - LEFT JOIN #operational_stats os - ON id.object_id = os.object_id - AND id.index_id = os.index_id - LEFT JOIN #index_analysis superseding - ON ia.superseded_by = superseding.index_name - AND ia.table_name = superseding.table_name - OPTION(RECOMPILE); - - IF ROWCOUNT_BIG() = 0 BEGIN IF @debug = 1 BEGIN RAISERROR('No rows inserted into #index_cleanup_report', 0, 0) WITH NOWAIT END; END; - - IF @debug = 1 - BEGIN - SELECT - table_name = '#index_cleanup_report', - icr.* - FROM #index_cleanup_report AS icr; - END; - - IF @debug = 1 - BEGIN - RAISERROR('Performing #index_cleanup_summary insert', 0, 0) WITH NOWAIT; - END; - - INSERT INTO - #index_cleanup_summary - WITH - (TABLOCK) - ( - database_id, - database_name, - schema_id, - schema_name, - table_name, - object_id, - index_id, - index_name, - action, - details, - current_definition, - proposed_definition, - usage_summary, - operational_summary, - uptime_warning - ) - SELECT - icr.database_id, - icr.database_name, - icr.schema_id, - icr.schema_name, - icr.table_name, - icr.object_id, - icr.index_id, - icr.index_name, - action = - CASE - WHEN icr.action = N'KEEP' - THEN N'Keep' - WHEN icr.action = N'DROP' - THEN N'Drop' - WHEN icr.action LIKE N'MERGE INTO%' - THEN N'Merge' - ELSE N'???' - END, - details = - CASE - WHEN icr.action = N'KEEP' - THEN N'No action needed' - WHEN icr.action = N'DROP' - THEN N'Index is redundant and can be safely dropped' - WHEN icr.action LIKE N'MERGE INTO%' - THEN N'Merge into index: ' + - SUBSTRING - ( - icr.action, - 12, - CHARINDEX(N' ', icr.action, 12) - 12 - ) - ELSE N'???' - END, - current_definition = icr.original_definition, - proposed_definition = - CASE - WHEN icr.action LIKE N'MERGE INTO%' - THEN icr.cleanup_script - ELSE NULL - END, - usage_summary = - N'Seeks: ' + CONVERT(nvarchar(20), icr.user_seeks) + - N', Scans: ' + CONVERT(nvarchar(20), icr.user_scans) + - N', Lookups: ' + CONVERT(nvarchar(20), icr.user_lookups) + - N', Updates: ' + CONVERT(nvarchar(20), icr.user_updates) + - N', Last used: ' + - ISNULL - ( - CONVERT - ( - nvarchar(30), - NULLIF - ( - DATEADD - ( - SECOND, - -1, - CASE - WHEN icr.last_user_seek > icr.last_user_scan - AND icr.last_user_seek > icr.last_user_lookup - THEN icr.last_user_seek - WHEN icr.last_user_scan > icr.last_user_lookup - THEN icr.last_user_scan - ELSE icr.last_user_lookup - END - ), - N'1900-01-01' - ), 120 - ), - N'Unknown' - ), - operational_summary = - N'Range scans: ' + CONVERT(nvarchar(20), icr.range_scan_count) + - N', Lookups: ' + CONVERT(nvarchar(20), icr.singleton_lookup_count) + - N', Inserts: ' + CONVERT(nvarchar(20), icr.leaf_insert_count) + - N', Updates: ' + CONVERT(nvarchar(20), icr.leaf_update_count) + - N', Deletes: ' + CONVERT(nvarchar(20), icr.leaf_delete_count), - uptime_warning = + SET + index_priority = CASE - WHEN icr.user_seeks = 0 AND icr.user_scans = 0 AND icr.user_lookups = 0 - THEN - CASE - WHEN TRY_PARSE(@uptime_days AS integer) < 7 - THEN N'WARNING: SQL Server has been running for only ' + - @uptime_days + - N' days. Usage statistics may not be reliable.' - WHEN TRY_PARSE(@uptime_days AS integer) < 14 - THEN N'CAUTION: SQL Server has been running for only ' + - @uptime_days + - N' days. Usage statistics may be incomplete.' - WHEN TRY_PARSE(@uptime_days AS integer) < 30 - THEN N'NOTE: SQL Server has been running for only ' + - @uptime_days + - N' days. Consider this when evaluating index usage.' - ELSE N'NOTE: SQL Server has been up for ' + - @uptime_days + - N' days, which makes analysis good, but... Are you patching this thing?' - END - ELSE NULL - END - FROM #index_cleanup_report AS icr - OPTION(RECOMPILE); - - IF ROWCOUNT_BIG() = 0 BEGIN IF @debug = 1 BEGIN RAISERROR('No rows inserted into #index_cleanup_summary', 0, 0) WITH NOWAIT END; END; - - IF @debug = 1 - BEGIN - SELECT - table_name = '#index_cleanup_summary', - ics.* - FROM #index_cleanup_summary AS ics; - END; - - IF @debug = 1 - BEGIN - RAISERROR('Going into summary and reports', 0, 0) WITH NOWAIT; - END; - - /* Index Cleanup Summary Report */ - - IF @debug = 1 - BEGIN - RAISERROR('Index Cleanup Summary', 0, 0) WITH NOWAIT; - END; - - SELECT - summary_type = - 'Index Cleanup Summary', - total_indexes_analyzed = - COUNT_BIG(DISTINCT icr.index_name), - indexes_to_drop = - SUM - ( - CASE - WHEN icr.action = 'DROP' - THEN 1 - ELSE 0 - END - ), - indexes_to_merge = - SUM - ( - CASE - WHEN icr.action LIKE 'MERGE INTO%' - THEN 1 - ELSE 0 - END - ), - unused_indexes = - SUM - ( - CASE - WHEN icr.user_seeks = 0 - AND icr.user_scans = 0 - AND icr.user_lookups = 0 - THEN 1 + WHEN index_id = 1 + THEN 1000 /* Clustered indexes get highest priority */ ELSE 0 - END - ), - space_savings_gb = - CONVERT - ( - decimal(10, 2), - ( - SELECT - SUM(ps_total.space_saved_mb) / 1024.0 - FROM + END + + + CASE + WHEN is_unique = 1 + THEN 500 + ELSE 0 + END /* Unique indexes get high priority */ + + + CASE + WHEN EXISTS ( - SELECT - icr_distinct.index_name, - icr_distinct.table_name, - space_saved_mb = SUM(ps_inner.total_space_mb) - FROM #index_cleanup_report AS icr_distinct - JOIN #partition_stats AS ps_inner - ON ps_inner.table_name = icr_distinct.table_name - AND ps_inner.index_name = icr_distinct.index_name - WHERE icr_distinct.action = 'DROP' - OR icr_distinct.action LIKE 'MERGE INTO%' - GROUP BY - icr_distinct.index_name, - icr_distinct.table_name - ) AS ps_total - ) - ), - write_operations_avoided = - SUM - ( - CASE - WHEN icr.action = 'DROP' - OR icr.action LIKE 'MERGE INTO%' - THEN ISNULL(icr.user_updates, 0) - ELSE 0 - END - ) - FROM #index_cleanup_report AS icr - OPTION (RECOMPILE); - - IF @debug = 1 - BEGIN - RAISERROR('Top tables by potential space savings', 0, 0) WITH NOWAIT; - END; - - /* Top tables by potential space savings */ - SELECT TOP (10) - icr.database_name, - icr.table_name, - indexes_affected = - COUNT_BIG(DISTINCT icr.index_name), - space_savings_gb = - CONVERT - ( - decimal(10,2), + SELECT + 1/0 + FROM #index_details id + WHERE id.index_name = #index_analysis.index_name + AND id.table_name = #index_analysis.table_name + AND id.user_seeks > 0 + ) THEN 200 + ELSE 0 + END /* Indexes with seeks get priority */ + + + CASE + WHEN EXISTS ( SELECT - SUM(ps_total.space_saved_mb) / 1024.0 - FROM - ( - SELECT - ps_inner.table_name, - space_saved_mb = - SUM(ps_inner.total_space_mb) - FROM #partition_stats AS ps_inner - JOIN #index_cleanup_report AS icr_inner - ON ps_inner.table_name = icr_inner.table_name - AND ps_inner.index_name = icr_inner.index_name - WHERE icr_inner.table_name = icr.table_name - AND - ( - icr_inner.action = 'DROP' - OR icr_inner.action LIKE 'MERGE INTO%' - ) - GROUP BY - ps_inner.table_name - ) AS ps_total - ) - ), - write_operations_avoided = - SUM(ISNULL(icr.user_updates, 0)) - FROM #index_cleanup_report AS icr - WHERE - ( - icr.action = 'DROP' - OR icr.action LIKE 'MERGE INTO%' - ) - GROUP BY - icr.database_name, - icr.table_name - ORDER BY - space_savings_gb DESC - OPTION(RECOMPILE); + 1/0 + FROM #index_details id + WHERE id.index_name = #index_analysis.index_name + AND id.table_name = #index_analysis.table_name + AND id.user_scans > 0 + ) THEN 50 ELSE 0 + END; /* Indexes with scans get some priority */ - IF @debug = 1 - BEGIN - RAISERROR('Page Compression Opportunity Summary', 0, 0) WITH NOWAIT; - END; - /* Summary of non-compressed indexes */ - SELECT - summary_type = 'Page Compression Opportunity Summary', - candidate_indexes = - COUNT_BIG(*), - total_size_gb = - SUM(ps.total_space_mb) / 1024.0, - estimated_savings_low_gb = - (SUM(ps.total_space_mb) * 0.20) / 1024.0, /* Conservative estimate (20%) */ - estimated_savings_typical_gb = - (SUM(ps.total_space_mb) * 0.40) / 1024.0, /* Typical estimate (40%) */ - estimated_savings_high_gb = - (SUM(ps.total_space_mb) * 0.60) / 1024.0 /* Optimistic estimate (60%) */ - FROM #partition_stats ps - WHERE ps.data_compression_desc = 'NONE' - AND NOT EXISTS + /* Rule 1: Identify unused indexes */ + UPDATE + #index_analysis + SET + consolidation_rule = 'Unused Index', + action = 'DISABLE' + WHERE EXISTS ( SELECT - 1/0 - FROM #index_cleanup_report AS icr - WHERE icr.index_name = ps.index_name - AND - ( - icr.action = 'DROP' - OR icr.action LIKE 'MERGE INTO%' - ) + 1/0 + FROM #index_details id + WHERE id.database_id = #index_analysis.database_id + AND id.object_id = #index_analysis.object_id + AND id.index_name = #index_analysis.index_name + AND id.user_seeks = 0 + AND id.user_scans = 0 + AND id.user_lookups = 0 + AND id.is_primary_key = 0 /* Don't disable primary keys */ + AND id.is_unique_constraint = 0 /* Don't disable unique constraints */ + AND id.is_eligible_for_dedupe = 1 /* Only eligible indexes */ ) - OPTION(RECOMPILE); - - -- Top candidates for page compression - - IF @debug = 1 - BEGIN - RAISERROR('Top candidates for page compression', 0, 0) WITH NOWAIT; - END; - - SELECT TOP (20) - database_name = - @database_name, - ps.schema_name, - ps.table_name, - ps.index_name, - index_type = + AND #index_analysis.index_id <> 1; /* Don't disable clustered indexes */ + + /* Rule 2: Exact duplicates - matching key columns and includes */ + UPDATE + ia1 + SET + ia1.consolidation_rule = 'Exact Duplicate', + ia1.target_index_name = CASE - WHEN ps.index_id = 1 - THEN 'CLUSTERED' - ELSE 'NONCLUSTERED' + WHEN ia1.index_priority >= ia2.index_priority + THEN NULL /* This index is the keeper */ + ELSE ia2.index_name /* Other index is the keeper */ END, - size_gb = - SUM(ps.total_space_mb) / 1024.0, - estimated_savings_low_gb = - (SUM(ps.total_space_mb) * 0.20) / 1024.0, -- Conservative (20%) - estimated_savings_typical_gb = - (SUM(ps.total_space_mb) * 0.40) / 1024.0, -- Typical (40%) - estimated_savings_high_gb = - (SUM(ps.total_space_mb) * 0.60) / 1024.0, -- Optimistic (60%) - rebuild_script = - N'ALTER INDEX ' + - QUOTENAME(ps.index_name) + - N' ON ' + - QUOTENAME(ps.schema_name) + - N'.' + - QUOTENAME(ps.table_name) + - N' REBUILD WITH - (DROP_EXISTING = ON, FILLFACTOR = 100, SORT_IN_TEMPDB = ON, ONLINE = ' + + ia1.action = CASE - WHEN @online = 'true' - THEN N'ON' - ELSE N'OFF' - END + - N', DATA_COMPRESSION = PAGE - );' - FROM #partition_stats ps - WHERE ps.data_compression_desc = N'NONE' - GROUP BY - ps.schema_name, - ps.table_name, - ps.index_name, - ps.index_id - ORDER BY - SUM(ps.total_space_mb) DESC - OPTION(RECOMPILE); - - IF @debug = 1 - BEGIN - RAISERROR('Select from #index_cleanup_summary', 0, 0) WITH NOWAIT; - END; - - SELECT - ics.database_name, - ics.table_name, - ics.index_name, - ics.action, - ics.details, - ics.current_definition, - ics.proposed_definition, - ics.usage_summary, - ics.operational_summary, - ics.uptime_warning - FROM #index_cleanup_summary AS ics - ORDER BY - CASE ics.action - WHEN N'Drop' THEN 1 - WHEN N'Merge' THEN 2 - WHEN N'Keep' THEN 3 - ELSE 999 + WHEN ia1.index_priority >= ia2.index_priority + THEN 'KEEP' /* This index is the keeper */ + ELSE 'DISABLE' /* Other index gets disabled */ + END + FROM #index_analysis ia1 + JOIN #index_analysis ia2 + ON ia1.database_id = ia2.database_id + AND ia1.object_id = ia2.object_id + AND ia1.index_name <> ia2.index_name + AND ia1.key_columns = ia2.key_columns /* Exact key match */ + AND ISNULL(ia1.included_columns, '') = ISNULL(ia2.included_columns, '') /* Exact includes match */ + AND ISNULL(ia1.filter_definition, '') = ISNULL(ia2.filter_definition, '') /* Matching filters */ + WHERE + ia1.consolidation_rule IS NULL /* Not already processed */ + AND ia2.consolidation_rule IS NULL /* Not already processed */ + AND ia1.is_eligible_for_dedupe = 1 + AND ia2.is_eligible_for_dedupe = 1; + +/* Rule 3: Key duplicates (matching key columns, different includes) */ +UPDATE ia1 +SET + ia1.consolidation_rule = 'Key Duplicate', + ia1.target_index_name = + CASE + /* If one is unique and the other isn't, prefer the unique one */ + WHEN ia1.is_unique = 1 AND ia2.is_unique = 0 THEN NULL + WHEN ia1.is_unique = 0 AND ia2.is_unique = 1 THEN ia2.index_name + /* Otherwise use priority */ + WHEN ia1.index_priority >= ia2.index_priority THEN NULL + ELSE ia2.index_name END, - ics.table_name, - ics.index_name - OPTION(RECOMPILE); + ia1.action = + CASE + WHEN (ia1.is_unique = 1 AND ia2.is_unique = 0) OR + (ia1.index_priority >= ia2.index_priority AND NOT (ia1.is_unique = 0 AND ia2.is_unique = 1)) + THEN 'MERGE INCLUDES' /* Keep this index but merge includes */ + ELSE 'DISABLE' /* Other index is keeper, disable this one */ + END +FROM #index_analysis ia1 +JOIN #index_analysis ia2 ON + ia1.database_id = ia2.database_id + AND ia1.object_id = ia2.object_id + AND ia1.index_name <> ia2.index_name + AND ia1.key_columns = ia2.key_columns /* Exact key match */ + AND ISNULL(ia1.included_columns, '') <> ISNULL(ia2.included_columns, '') /* Different includes */ + AND ISNULL(ia1.filter_definition, '') = ISNULL(ia2.filter_definition, '') /* Matching filters */ +WHERE + ia1.consolidation_rule IS NULL /* Not already processed */ + AND ia2.consolidation_rule IS NULL /* Not already processed */ + AND ia1.is_eligible_for_dedupe = 1 + AND ia2.is_eligible_for_dedupe = 1; + +/* Rule 4: Superset/subset key columns */ +UPDATE ia1 +SET + ia1.consolidation_rule = 'Key Subset', + ia1.target_index_name = ia2.index_name, + ia1.action = 'DISABLE' /* The narrower index gets disabled */ +FROM #index_analysis ia1 +JOIN #index_analysis ia2 ON + ia1.database_id = ia2.database_id + AND ia1.object_id = ia2.object_id + AND ia1.index_name <> ia2.index_name + AND ia2.key_columns LIKE (ia1.key_columns + '%') /* ia2 has wider key that starts with ia1's key */ + AND ISNULL(ia1.filter_definition, '') = ISNULL(ia2.filter_definition, '') /* Matching filters */ + /* Exception: If narrower index is unique and wider is not, they should not be merged */ + AND NOT (ia1.is_unique = 1 AND ia2.is_unique = 0) +WHERE + ia1.consolidation_rule IS NULL /* Not already processed */ + AND ia2.consolidation_rule IS NULL /* Not already processed */ + AND ia1.is_eligible_for_dedupe = 1 + AND ia2.is_eligible_for_dedupe = 1; - IF @debug = 1 - BEGIN - RAISERROR('Performing #final_index_actions insert', 0, 0) WITH NOWAIT; - END; - - -- Replace the existing INSERT into #final_index_actions for MERGE operations with this: - WITH - MergeTargets AS - ( - -- Get distinct target indexes for merges - SELECT DISTINCT - ia.database_id, - ia.database_name, - ia.schema_id, - ia.schema_name, - ia.object_id, - ia.table_name, - target_index = - SUBSTRING - ( - ia.action, - 12, - CHARINDEX - ( - N' ', - ia.action + - N' ', - 12 - ) - 12 - ) - FROM #index_cleanup_report ia - WHERE ia.action LIKE N'MERGE INTO%' - ) - -- Insert a single CREATE INDEX statement for each target index - INSERT INTO - #final_index_actions - WITH - (TABLOCK) - ( - database_id, - database_name, - schema_id, - schema_name, - object_id, - table_name, - index_id, - index_name, - action, - script - ) - SELECT - mt.database_id, - mt.database_name, - mt.schema_id, - mt.schema_name, - mt.object_id, - mt.table_name, - index_id = - ISNULL - ( - ( - SELECT TOP (1) - ia.index_id - FROM #index_analysis ia - WHERE ia.database_id = mt.database_id - AND ia.table_name = mt.table_name - AND ia.index_name = mt.target_index - ), - 0 - ), - mt.target_index, - action = - N'MERGE CONSOLIDATED', - script = - N'CREATE INDEX ' + - QUOTENAME(mt.target_index) + - N' ON ' + - QUOTENAME(mt.database_name) + - N'.' + - QUOTENAME(mt.schema_name) + - N'.' + - QUOTENAME(mt.table_name) + - N' (' + - -- Get key columns from one of the indexes being merged - ( - SELECT TOP (1) - ia.key_columns - FROM #index_analysis ia - WHERE ia.database_id = mt.database_id - AND ia.table_name = mt.table_name - AND ia.index_name = mt.target_index - ) + - N')' + - -- Include all distinct columns from all indexes being merged into this target - CASE - WHEN EXISTS - ( - SELECT - 1/0 - FROM #index_cleanup_report icr - WHERE icr.database_id = mt.database_id - AND icr.table_name = mt.table_name - AND icr.action LIKE N'MERGE INTO ' + mt.target_index + N'%' - AND - ( - EXISTS - ( - SELECT - 1/0 - FROM #index_analysis ia - WHERE ia.database_id = icr.database_id - AND ia.table_name = icr.table_name - AND ia.index_name = icr.index_name - AND ia.included_columns IS NOT NULL - ) - OR icr.action LIKE N'%ADD %' - ) - ) - THEN N' INCLUDE (' + - STUFF - ( - ( - SELECT DISTINCT - N', ' + - col - FROM - ( - -- Get included columns from all source indexes - SELECT - col = LTRIM(RTRIM(value.c.value('.', 'sysname'))) - FROM #index_cleanup_report icr - CROSS APPLY - ( - SELECT - ia.included_columns - FROM #index_analysis ia - WHERE ia.database_id = icr.database_id - AND ia.table_name = icr.table_name - AND ia.index_name = icr.index_name - ) src - CROSS APPLY - ( - SELECT - cols = - CONVERT - ( - xml, - '' + - REPLACE - ( - ISNULL - ( - src.included_columns, - '' - ), - ', ', - '') + - '' - ) - ) x - CROSS APPLY x.cols.nodes('/c') AS value(c) - WHERE icr.database_id = mt.database_id - AND icr.table_name = mt.table_name - AND icr.action LIKE N'MERGE INTO ' + mt.target_index + N'%' - - UNION - - -- Get missing columns which need to be added - SELECT - col = - LTRIM(RTRIM(value.c.value('.', 'sysname'))) - FROM #index_cleanup_report icr - CROSS APPLY - ( - SELECT - missing_cols = - SUBSTRING - ( - icr.action, - CHARINDEX('ADD ', icr.action) + 4, - LEN(icr.action) - ) - WHERE icr.action LIKE N'%ADD %' - ) mc - CROSS APPLY - ( - SELECT - cols = - CONVERT - ( - xml, - '' + - REPLACE - ( - ISNULL - ( - mc.missing_cols, - '' - ), - ', ', - '' - ) + '' - ) - ) x - CROSS APPLY x.cols.nodes('/c') AS value(c) - WHERE icr.database_id = mt.database_id - AND icr.table_name = mt.table_name - AND icr.action LIKE N'MERGE INTO ' + mt.target_index + N'%' - AND icr.action LIKE N'%ADD %' - ) AS all_columns - WHERE DATALENGTH(col) > 0 - FOR - XML - PATH(''), - TYPE - ).value('.', 'nvarchar(max)'), - 1, - 2, - '' - ) + - N')' - ELSE N'' - END + - -- Add partitioning if needed - CASE - WHEN EXISTS - ( - SELECT - 1/0 - FROM #partition_stats ps - WHERE ps.database_id = mt.database_id - AND ps.table_name = mt.table_name - AND ps.index_name = mt.target_index - AND ps.partition_function_name IS NOT NULL - ) - THEN - ( - SELECT TOP (1) - N' ON ' + - QUOTENAME(ps.partition_function_name) + - '(' + - ps.partition_columns + - ')' - FROM #partition_stats ps - WHERE ps.database_id = mt.database_id - AND ps.table_name = mt.table_name - AND ps.index_name = mt.target_index - AND ps.partition_function_name IS NOT NULL - ) - ELSE N'' - END + - -- Add filter definition if needed - CASE - WHEN EXISTS - ( - SELECT - 1/0 - FROM #index_analysis ia - WHERE ia.database_id = mt.database_id - AND ia.table_name = mt.table_name - AND ia.index_name = mt.target_index - AND ia.filter_definition IS NOT NULL - ) - THEN - ( - SELECT TOP (1) - N' WHERE ' + - ia.filter_definition - FROM #index_analysis ia - WHERE ia.database_id = mt.database_id - AND ia.table_name = mt.table_name - AND ia.index_name = mt.target_index - AND ia.filter_definition IS NOT NULL - ) - ELSE N'' - END + - -- Add WITH options - N' WITH (DROP_EXISTING = ON, FILLFACTOR = 100, SORT_IN_TEMPDB = ON, ONLINE = ' + - CASE - WHEN @online = 'true' - THEN N'ON' - ELSE N'OFF' - END + - N', DATA_COMPRESSION = PAGE);' - FROM MergeTargets mt; - - -- Then add DISABLE statements for all source indexes - INSERT INTO - #final_index_actions - WITH - (TABLOCK) - ( - database_id, - database_name, - schema_id, - schema_name, - object_id, - table_name, - index_id, - index_name, - action, - script - ) - SELECT - icr.database_id, - icr.database_name, - icr.schema_id, - icr.schema_name, - icr.object_id, - icr.table_name, - icr.index_id, - icr.index_name, - action = N'DISABLE MERGED', - script = - N'ALTER INDEX ' + - QUOTENAME(icr.index_name) + - N' ON ' + - QUOTENAME(icr.database_name) + - N'.' + - QUOTENAME(icr.schema_name) + - N'.' + - QUOTENAME(icr.table_name) + - N' DISABLE;' - FROM #index_cleanup_report icr - WHERE icr.action LIKE N'MERGE INTO%'; - - IF ROWCOUNT_BIG() = 0 BEGIN IF @debug = 1 BEGIN RAISERROR('No rows inserted into #final_index_actions', 0, 0) WITH NOWAIT END; END; IF @debug = 1 BEGIN - SELECT - table_name = '#final_index_actions', - fia.* - FROM #final_index_actions AS fia; - - RAISERROR('Select from #final_index_actions', 0, 0) WITH NOWAIT; - END; - - SELECT - f.database_name, - f.table_name, - f.index_name, - f.action, - f.script, - sort_order = - CASE f.action - WHEN N'MERGE INTO' THEN 2 - WHEN N'DROP' THEN 3 - ELSE 999 - END - FROM #final_index_actions AS f - WHERE f.action <> N'KEEP' - - UNION ALL - - SELECT - r.database_name, - r.table_name, - r.index_name, - action = - N'DISABLE (Unused)', - script = - N'ALTER INDEX ' + - QUOTENAME(r.index_name) + - N' ON ' + - QUOTENAME(r.table_name) + - N' DISABLE;', - sort_order = 1 - FROM #index_cleanup_report AS r - WHERE r.user_seeks = 0 - AND r.user_scans = 0 - AND r.user_lookups = 0 - AND r.user_updates = 0 - ORDER BY - f.table_name, - f.index_name, - sort_order - OPTION(RECOMPILE); + table_name = '#index_analysis after update', + ia.* + FROM #index_analysis AS ia; + END; IF @debug = 1 BEGIN - RAISERROR('Generating scripts', 0, 0) WITH NOWAIT; + RAISERROR('Generating results', 0, 0) WITH NOWAIT; END; - /*Merge into*/ - SELECT - @final_script += - N' - -- ============================================================================= - -- MERGE INDEX: ' + - QUOTENAME(f.index_name) + - N' into ' + - CASE - WHEN f.action = 'MERGE CONSOLIDATED' - THEN QUOTENAME(f.index_name) - ELSE 'Unknown Target' +/* Generate index merge scripts with compression and drop_existing */ +SELECT + database_name, + schema_name, + table_name, + index_name, + target_index_name, + consolidation_rule, + merge_script = + 'CREATE INDEX ' + QUOTENAME(index_name) + + ' ON ' + QUOTENAME(database_name) + '.' + QUOTENAME(schema_name) + '.' + QUOTENAME(table_name) + + ' (' + key_columns + ')' + + CASE WHEN included_columns IS NOT NULL AND LEN(included_columns) > 0 + THEN ' INCLUDE (' + included_columns + ')' + ELSE '' END + - N' - -- Reason: This index overlaps with another index and can be consolidated - -- Original definition: ' + - NCHAR(10) + - ( - SELECT - MAX(ics.current_definition) - FROM #index_cleanup_summary AS ics - WHERE ics.index_name = f.index_name - AND ics.table_name = f.table_name - ) + - N' - -- Usage: Seeks: ' + - CONVERT - ( - nvarchar(20), - ISNULL - ( - ( - SELECT - SUM(DISTINCT id.user_seeks) - FROM #index_details AS id - WHERE id.table_name = f.table_name - AND id.index_name = f.index_name - ), - 0 - ) - ) + - N', Scans: ' + - CONVERT - ( - nvarchar(20), - ISNULL - ( - ( - SELECT - SUM(DISTINCT id.user_scans) - FROM #index_details AS id - WHERE id.table_name = f.table_name - AND id.index_name = f.index_name - ), - 0 - ) - ) + - N', Lookups: ' + - CONVERT - ( - nvarchar(20), - ISNULL - ( - ( - SELECT - SUM(DISTINCT id.user_lookups) - FROM #index_details AS id - WHERE id.table_name = f.table_name - AND id.index_name = f.index_name - ), - 0 - ) - ) + - N', Updates: ' + - CONVERT - ( - nvarchar(20), - ISNULL - ( - ( - SELECT - SUM(DISTINCT id.user_updates) - FROM #index_details AS id - WHERE id.table_name = f.table_name - AND id.index_name = f.index_name - ), - 0 - ) - ) + - N' - -- Space saved: ' + - CONVERT - ( - nvarchar(20), - ISNULL - ( - ( - SELECT - CONVERT - ( - decimal(10,2), - SUM(ps.total_space_mb) / 1024.0 - ) - FROM #partition_stats AS ps - WHERE ps.table_name = f.table_name - AND ps.index_name = f.index_name - ), - 0 - ) - ) + N' GB - -- ============================================================================= - ' + - f.script + - NCHAR(10) + - NCHAR(10) - FROM #final_index_actions AS f - WHERE f.action = N'MERGE CONSOLIDATED' - ORDER BY - f.table_name, - f.index_name; - - /*Disable merged indexes*/ - SELECT - @final_script += N' - /* - -- ============================================================================= - -- DISABLE MERGED INDEX: ' + - QUOTENAME(f.index_name) + - N' - -- Reason: This index has been merged into another index - -- ============================================================================= - */' + - NCHAR(10) + - f.script + - NCHAR(10) + - NCHAR(10) - FROM #final_index_actions AS f - WHERE f.action = N'DISABLE MERGED' - ORDER BY - f.table_name, - f.index_name; - - /*Drop indexes*/ - SELECT - @final_script += N' - /* - -- ============================================================================= - -- DROP INDEX: ' + - QUOTENAME(f.index_name) + - N' - -- Reason: This index is redundant with other indexes on the same table - -- Current usage: Seeks: ' + - CONVERT - ( - nvarchar(20), - ISNULL - ( - ( - SELECT - SUM(DISTINCT id.user_seeks) - FROM #index_details AS id - WHERE id.table_name = f.table_name - AND id.index_name = f.index_name - ), - 0 - ) - ) + - N', Scans: ' + - CONVERT - ( - nvarchar(20), - ISNULL - ( - ( - SELECT - SUM(DISTINCT id.user_scans) - FROM #index_details AS id - WHERE id.table_name = f.table_name - AND id.index_name = f.index_name - ), - 0 - ) - ) + - N', Lookups: ' + - CONVERT - ( - nvarchar(20), - ISNULL - ( - ( - SELECT - SUM(DISTINCT id.user_lookups) - FROM #index_details AS id - WHERE id.table_name = f.table_name - AND id.index_name = f.index_name - ), - 0 - ) - ) + - N', Updates: ' + - CONVERT - ( - nvarchar(20), - ISNULL - ( - ( - SELECT - SUM(DISTINCT id.user_updates) - FROM #index_details AS id - WHERE id.table_name = f.table_name - AND id.index_name = f.index_name - ), - 0 - ) - ) + - N' - -- Last used: ' + - ISNULL - ( - CONVERT - ( - nvarchar(30), - ( - SELECT - MAX - ( - CASE - WHEN id.last_user_seek > id.last_user_scan - AND id.last_user_seek > id.last_user_lookup - THEN id.last_user_seek - WHEN id.last_user_scan > id.last_user_lookup - THEN id.last_user_scan - ELSE id.last_user_lookup - END - ) - FROM #index_details AS id - WHERE id.table_name = f.table_name - AND id.index_name = f.index_name - ), - 120 - ), - 'Never' - ) + - N' - -- Space reclaimed: ' + - CONVERT - ( - nvarchar(20), - ISNULL - ( - ( - SELECT - CONVERT - ( - decimal(10,2), - SUM(ps.total_space_mb) / 1024.0 - ) - FROM #partition_stats AS ps - WHERE ps.table_name = f.table_name - AND ps.index_name = f.index_name - ), - 0 - ) - ) + N' GB - -- ============================================================================= - */' + - f.script + - NCHAR(10) + - NCHAR(10) - FROM #final_index_actions AS f - WHERE f.action = N'DROP' - ORDER BY - f.table_name, - f.index_name; - - /*Unused indexes*/ - SELECT - @final_script += N' - /* - -- ============================================================================= - -- DISABLE UNUSED INDEX: ' + - QUOTENAME(i.index_name) + - N' - -- Reason: This index has never been used for reads but has been updated ' + - CONVERT - ( - nvarchar(20), - i.user_updates - ) + - N' times - -- Space reclaimed: ' + - CONVERT - ( - nvarchar(20), - ISNULL - ( - ( - SELECT - CONVERT - ( - decimal(10,2), - SUM(ps.total_space_mb) / 1024.0 - ) - FROM #partition_stats AS ps - WHERE ps.table_name = i.table_name - AND ps.index_name = i.index_name - ), - 0 - ) - ) + N' GB - -- Warning: Verify this index is truly not needed before dropping - -- ============================================================================= - */' + - NCHAR(10) + - N'ALTER INDEX ' + - QUOTENAME(i.index_name) + - N' ON ' + - QUOTENAME(i.database_name) + - N'.' + - QUOTENAME - (i.schema_name) + - N'.' + - QUOTENAME(i.table_name) + - N' DISABLE;' + - NCHAR(10) + - NCHAR(10) - FROM - ( - SELECT DISTINCT - icr.database_name, - icr.schema_name, - icr.table_name, - icr.index_name, - icr.user_updates - FROM #index_cleanup_report AS icr - WHERE icr.user_seeks = 0 - AND icr.user_scans = 0 - AND icr.user_lookups = 0 - AND icr.user_updates = 0 - AND icr.action <> N'DROP' - AND icr.action NOT LIKE N'MERGE INTO%' - AND NOT EXISTS - ( - SELECT - 1/0 - FROM #final_index_actions AS fia - WHERE fia.index_name = icr.index_name - AND fia.table_name = icr.table_name - AND fia.action IN (N'MERGE CONSOLIDATED', N'DISABLE MERGED') - ) - ) AS i - ORDER BY - i.table_name, - i.index_name; - - - /*Summary*/ - SELECT - @final_script += N' - -- ============================================================================= - -- SUMMARY OF CHANGES - -- Total indexes analyzed: ' + - CONVERT - ( - nvarchar(10), - ( - SELECT - COUNT_BIG(*) - FROM #index_cleanup_report AS icr - ) - ) + - N' - -- Indexes recommended for dropping: ' + - CONVERT - ( - nvarchar(10), - ( - SELECT - COUNT_BIG(*) - FROM #index_cleanup_report AS icr - WHERE icr.action = 'DROP' - ) - ) + - N' - -- Indexes recommended for merging: ' + - CONVERT - ( - nvarchar(10), - ( - SELECT - COUNT_BIG(*) - FROM #index_cleanup_report AS icr - WHERE icr.action LIKE 'MERGE INTO%' - ) - ) + - N' - -- Unused indexes found: ' + - CONVERT - ( - nvarchar(10), - ( - SELECT - COUNT_BIG(*) - FROM #index_cleanup_report AS icr - WHERE icr.user_seeks = 0 - AND icr.user_scans = 0 - AND icr.user_lookups = 0 - ) - ) + - N' - -- Estimated space savings: ' + - CONVERT - ( - nvarchar(20), - ISNULL - ( - ( - SELECT - CONVERT - ( - decimal(10,2), - SUM(ps.total_space_mb) / 1024.0 - ) - FROM #partition_stats AS ps - JOIN #index_cleanup_report AS icr - ON ps.table_name = icr.table_name - AND ps.index_name = icr.index_name - WHERE icr.action = 'DROP' - OR icr.action LIKE 'MERGE INTO%' - OR - ( - icr.user_seeks = 0 - AND icr.user_scans = 0 - AND icr.user_lookups = 0 - ) - ), - 0 - ) - ) + N' GB - -- Estimated write operations reduced: ' + - CONVERT - ( - nvarchar(20), - ISNULL - ( - ( - SELECT - SUM(icr.user_updates) - FROM #index_cleanup_report AS icr - WHERE icr.action = 'DROP' - OR icr.action LIKE 'MERGE INTO%' - OR - ( - icr.user_seeks = 0 - AND icr.user_scans = 0 - AND icr.user_lookups = 0 - ) - ), - 0 - ) - ) + N' operations - -- ============================================================================= - '; - - SELECT - [text()] = - N'/* Index Cleanup Script for ' + - @database_name + - N' */', - [text()] = - ( - SELECT - NCHAR(10) + - N' ----------------------' + - NCHAR(10) + - N' -- Final script to review. DO NOT EXECUTE WITHOUT CAREFUL REVIEW.' + - NCHAR(10) + - N' -- Implementation Script:' + - NCHAR(10) + - N' ----------------------' + - NCHAR(10) + - @final_script - FOR - XML - PATH(''), - TYPE - ).value('(./text())[1]', 'nvarchar(max)') - FOR - XML - PATH(''), - TYPE; - + CASE WHEN filter_definition IS NOT NULL + THEN ' WHERE ' + filter_definition + ELSE '' + END + + ' WITH (DROP_EXISTING = ON, FILLFACTOR = 100, SORT_IN_TEMPDB = ON, ONLINE = ON, DATA_COMPRESSION = PAGE);' +FROM #index_analysis +WHERE action = 'MERGE INCLUDES' +ORDER BY table_name, index_name; + +/* Generate disable scripts for unneeded indexes */ +SELECT + database_name, + schema_name, + table_name, + index_name, + consolidation_rule, + disable_script = + 'ALTER INDEX ' + QUOTENAME(index_name) + + ' ON ' + QUOTENAME(database_name) + '.' + QUOTENAME(schema_name) + '.' + QUOTENAME(table_name) + + ' DISABLE;' +FROM #index_analysis +WHERE action = 'DISABLE' +ORDER BY table_name, index_name; END TRY BEGIN CATCH THROW; diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA_Original.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA_Original.sql new file mode 100644 index 00000000..6953995f --- /dev/null +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA_Original.sql @@ -0,0 +1,3547 @@ +/* +EXEC sp_IndexCleanup + @database_name = 'StackOverflow2013', + @debug = 1; + +EXEC sp_IndexCleanup + @database_name = 'StackOverflow2013', + @table_name = 'Users', + @debug = 1 +*/ + +SET ANSI_WARNINGS ON; +SET ARITHABORT ON; +SET CONCAT_NULL_YIELDS_NULL ON; +SET QUOTED_IDENTIFIER ON; +SET NUMERIC_ROUNDABORT OFF; +SET IMPLICIT_TRANSACTIONS OFF; +SET STATISTICS TIME, IO OFF; +GO + +IF OBJECT_ID('dbo.sp_IndexCleanup', 'P') IS NULL +BEGIN + EXECUTE ('CREATE PROCEDURE dbo.sp_IndexCleanup AS RETURN 138;'); +END; +GO + +ALTER PROCEDURE + dbo.sp_IndexCleanup +( + @database_name sysname = NULL, + @schema_name sysname = NULL, + @table_name sysname = NULL, + @min_reads bigint = 0, + @min_writes bigint = 0, + @min_size_gb decimal(10,2) = 0, + @min_rows bigint = 0, + @help bit = 'false', + @debug bit = 'true', + @version varchar(20) = NULL OUTPUT, + @version_date datetime = NULL OUTPUT +) +WITH RECOMPILE +AS +BEGIN +SET NOCOUNT ON; + +BEGIN TRY + SELECT + @version = '-2147483648', + @version_date = '17530101'; + + SELECT + warning = N'Read the messages pane carefully!' + + PRINT ' +------------------------------------------------------------------------------------------- +------------------------------------------------------------------------------------------- +------------------------------------------------------------------------------------------- +This is the BETA VERSION of sp_IndexCleanup + +It needs lots of love and testing in real environments with real indexes to fix many issues: + * Data collection + * Deduping logic + * Result correctness + * Edge cases + + If you run this, only use the output to debug and validate result correctness. + + Do not run any of the output scripts, period. Doing so may be harmful. + ------------------------------------------------------------------------------------------- + ------------------------------------------------------------------------------------------- + ------------------------------------------------------------------------------------------- + + '; + + + /* + Help section, for help. + Will become more helpful when out of beta. + */ + IF @help = 1 + BEGIN + SELECT + help = N'hello, i am sp_IndexCleanup - BETA' + UNION ALL + SELECT + help = N'this is a script to help clean up unused and duplicate indexes' + UNION ALL + SELECT + help = N'you are currently using a beta version, and the advice should not be followed' + UNION ALL + SELECT + help = N'without careful analysis and consideration. it may be harmful.' + + + /* + Parameters + */ + SELECT + parameter_name = + ap.name, + data_type = + t.name, + description = + CASE + ap.name + WHEN ap.name + THEN ap.name + END, + valid_inputs = + CASE + ap.name + WHEN ap.name + THEN ap.name + END, + defaults = + CASE + ap.name + WHEN ap.name + THEN ap.name + END + FROM sys.all_parameters AS ap + INNER JOIN sys.all_objects AS o + ON ap.object_id = o.object_id + INNER JOIN sys.types AS t + ON ap.system_type_id = t.system_type_id + AND ap.user_type_id = t.user_type_id + WHERE o.name = N'sp_IndexCleanup' + OPTION(MAXDOP 1, RECOMPILE); + + SELECT + mit_license_yo = 'i am MIT licensed, so like, do whatever' + + UNION ALL + + SELECT + mit_license_yo = 'see printed messages for full license'; + + RAISERROR(' +MIT License + +Copyright 2024 Darling Data, LLC + +https://www.erikdarling.com/ + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, +sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the +following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE +FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +', 0, 1) WITH NOWAIT; + + RETURN; + END; + + IF @debug = 1 + BEGIN + RAISERROR('Declaring variables', 0, 0) WITH NOWAIT; + END; + + DECLARE + /*general script variables*/ + @sql nvarchar(max) = N'', + @database_id integer = NULL, + @object_id integer = NULL, + @full_object_name nvarchar(768) = NULL, + @final_script nvarchar(max) = '', + /*cursor variables*/ + @c_database_id integer, + @c_database_name sysname, + @c_schema_id integer, + @c_schema_name sysname, + @c_object_id integer, + @c_table_name sysname, + @c_index_id integer, + @c_index_name sysname, + @c_is_unique bit, + @c_filter_definition nvarchar(max), + @index_cursor CURSOR, + /*print variables*/ + @helper integer = 0, + @sql_len integer, + @sql_debug nvarchar(max) = N'', + @online bit = + CASE + WHEN + CONVERT + ( + integer, + SERVERPROPERTY('EngineEdition') + ) IN (3, 5, 8) + THEN 'true' /* Enterprise, Azure SQL DB, Managed Instance */ + ELSE 'false' + END, + @uptime_days nvarchar(10) = + ( + SELECT + DATEDIFF + ( + DAY, + osi.sqlserver_start_time, + SYSDATETIME() + ) + FROM sys.dm_os_sys_info AS osi + ); + + /* + Initial checks for object validity + */ + IF @debug = 1 + BEGIN + RAISERROR('Checking paramaters...', 0, 0) WITH NOWAIT; + END; + + IF @database_name IS NULL + AND DB_NAME() NOT IN + ( + N'master', + N'model', + N'msdb', + N'tempdb', + N'rdsadmin' + ) + BEGIN + SELECT + @database_name = DB_NAME(); + END; + + IF @database_name IS NOT NULL + BEGIN + SELECT + @database_id = d.database_id + FROM sys.databases AS d + WHERE d.name = @database_name; + END; + + IF @schema_name IS NULL + AND @table_name IS NOT NULL + BEGIN + SELECT + @schema_name = N'dbo'; + END; + + IF @schema_name IS NOT NULL + AND @table_name IS NOT NULL + BEGIN + SELECT + @full_object_name = + QUOTENAME(@database_name) + + N'.' + + QUOTENAME(@schema_name) + + N'.' + + QUOTENAME(@table_name); + + SELECT + @object_id = + OBJECT_ID(@full_object_name); + + IF @object_id IS NULL + BEGIN + RAISERROR('The object %s doesn''t seem to exist', 16, 1, @full_object_name) WITH NOWAIT; + RETURN; + END; + END; + + -- Parameter validation + IF @min_reads < 0 + OR @min_reads IS NULL + BEGIN + RAISERROR('Parameter @min_reads cannot be NULL or negative. Setting to 0.', 10, 1) WITH NOWAIT; + SET @min_reads = 0; + END; + + IF @min_writes < 0 + OR @min_writes IS NULL + BEGIN + RAISERROR('Parameter @min_writes cannot be NULL or negative. Setting to 0.', 10, 1) WITH NOWAIT; + SET @min_writes = 0; + END; + + IF @min_size_gb < 0 + OR @min_size_gb IS NULL + BEGIN + RAISERROR('Parameter @min_size_gb cannot be NULL or negative. Setting to 0.', 10, 1) WITH NOWAIT; + SET @min_size_gb = 0; + END; + + IF @min_rows < 0 + OR @min_rows IS NULL + BEGIN + RAISERROR('Parameter @min_rows cannot be NULL or negative. Setting to 0.', 10, 1) WITH NOWAIT; + SET @min_rows = 0; + END; + + /* + Temp tables! + */ + + IF @debug = 1 + BEGIN + RAISERROR('Creating temp tables', 0, 0) WITH NOWAIT; + END; + + CREATE TABLE + #filtered_objects + ( + database_id integer NOT NULL, + database_name sysname NOT NULL, + schema_id integer NOT NULL, + schema_name sysname NOT NULL, + object_id integer NOT NULL, + table_name sysname NOT NULL, + index_id integer NOT NULL, + index_name sysname NOT NULL + PRIMARY KEY + (database_id, schema_id, object_id, index_id) + ); + + CREATE TABLE + #operational_stats + ( + database_id integer NOT NULL, + database_name sysname NOT NULL, + schema_id integer NOT NULL, + schema_name sysname NOT NULL, + object_id integer NOT NULL, + table_name sysname NOT NULL, + index_id integer NOT NULL, + index_name sysname NOT NULL, + range_scan_count bigint NULL, + singleton_lookup_count bigint NULL, + forwarded_fetch_count bigint NULL, + lob_fetch_in_pages bigint NULL, + row_overflow_fetch_in_pages bigint NULL, + leaf_insert_count bigint NULL, + leaf_update_count bigint NULL, + leaf_delete_count bigint NULL, + leaf_ghost_count bigint NULL, + nonleaf_insert_count bigint NULL, + nonleaf_update_count bigint NULL, + nonleaf_delete_count bigint NULL, + leaf_allocation_count bigint NULL, + nonleaf_allocation_count bigint NULL, + row_lock_count bigint NULL, + row_lock_wait_count bigint NULL, + row_lock_wait_in_ms bigint NULL, + page_lock_count bigint NULL, + page_lock_wait_count bigint NULL, + page_lock_wait_in_ms bigint NULL, + index_lock_promotion_attempt_count bigint NULL, + index_lock_promotion_count bigint NULL, + page_latch_wait_count bigint NULL, + page_latch_wait_in_ms bigint NULL, + tree_page_latch_wait_count bigint NULL, + tree_page_latch_wait_in_ms bigint NULL, + page_io_latch_wait_count bigint NULL, + page_io_latch_wait_in_ms bigint NULL, + page_compression_attempt_count bigint NULL, + page_compression_success_count bigint NULL, + PRIMARY KEY CLUSTERED + (database_id, schema_id, object_id, index_id) + ); + + CREATE TABLE + #index_details + ( + database_id integer NOT NULL, + database_name sysname NOT NULL, + schema_id integer NOT NULL, + schema_name sysname NOT NULL, + object_id integer NOT NULL, + table_name sysname NOT NULL, + index_id integer NOT NULL, + index_name sysname NULL, + column_name sysname NOT NULL, + is_primary_key bit NULL, + is_unique bit NULL, + is_unique_constraint bit NULL, + is_indexed_view integer NOT NULL, + is_foreign_key bit NULL, + is_foreign_key_reference bit NULL, + key_ordinal tinyint NOT NULL, + index_column_id integer NOT NULL, + is_descending_key bit NOT NULL, + is_included_column bit NULL, + filter_definition nvarchar(max) NULL, + is_max_length integer NOT NULL, + user_seeks bigint NOT NULL, + user_scans bigint NOT NULL, + user_lookups bigint NOT NULL, + user_updates bigint NOT NULL, + last_user_seek datetime NULL, + last_user_scan datetime NULL, + last_user_lookup datetime NULL, + last_user_update datetime NULL, + is_eligible_for_dedupe bit NOT NULL + PRIMARY KEY CLUSTERED(database_id, object_id, index_id, column_name) + ); + + CREATE TABLE + #partition_stats + ( + database_id integer NOT NULL, + database_name sysname NOT NULL, + schema_id integer NOT NULL, + schema_name sysname NOT NULL, + object_id integer NOT NULL, + table_name sysname NOT NULL, + index_id integer NOT NULL, + index_name sysname NULL, + partition_id bigint NOT NULL, + partition_number int NOT NULL, + total_rows bigint NULL, + total_space_mb decimal(38, 2) NULL, + reserved_lob_mb decimal(38, 2) NULL, + reserved_row_overflow_mb decimal(38, 2) NULL, + data_compression_desc nvarchar(60) NULL, + built_on sysname NULL, + partition_function_name sysname NULL, + partition_columns nvarchar(max) + PRIMARY KEY CLUSTERED(database_id, object_id, index_id, partition_id) + ); + + CREATE TABLE + #index_analysis + ( + database_id integer NOT NULL, + database_name sysname NOT NULL, + schema_id integer NOT NULL, + schema_name sysname NOT NULL, + object_id integer NOT NULL, + table_name sysname NOT NULL, + index_id integer NULL, + index_name sysname NOT NULL, + is_unique bit NULL, + key_columns nvarchar(max) NULL, + included_columns nvarchar(max) NULL, + filter_definition nvarchar(max) NULL, + is_redundant bit NULL, + superseded_by sysname NULL, + missing_columns nvarchar(max) NULL, + action nvarchar(max) NULL, + INDEX c CLUSTERED + (database_id, schema_name, table_name, index_name) + ); + + CREATE TABLE + #index_cleanup_report + ( + database_id integer NOT NULL, + database_name sysname NOT NULL, + schema_id integer NOT NULL, + schema_name sysname NOT NULL, + object_id integer NOT NULL, + table_name sysname NOT NULL, + index_id integer NOT NULL, + index_name sysname NOT NULL, + action nvarchar(max) NULL, + cleanup_script nvarchar(max) NULL, + original_definition nvarchar(max) NULL, + /*Usage details*/ + user_seeks bigint NULL, + user_scans bigint NULL, + user_lookups bigint NULL, + user_updates bigint NULL, + last_user_seek datetime NULL, + last_user_scan datetime NULL, + last_user_lookup datetime NULL, + last_user_update datetime NULL, + /*Operational stats*/ + range_scan_count bigint NULL, + singleton_lookup_count bigint NULL, + leaf_insert_count bigint NULL, + leaf_update_count bigint NULL, + leaf_delete_count bigint NULL, + page_lock_count bigint NULL, + page_lock_wait_count bigint NULL, + page_lock_wait_in_ms bigint NULL + ); + + CREATE TABLE + #index_cleanup_summary + ( + database_id integer NOT NULL, + database_name sysname NOT NULL, + schema_id integer NOT NULL, + schema_name sysname NOT NULL, + object_id integer NOT NULL, + table_name sysname NOT NULL, + index_id integer NOT NULL, + index_name sysname NOT NULL, + action nvarchar(max) NOT NULL, + details nvarchar(max) NULL, + current_definition nvarchar(max) NOT NULL, + proposed_definition nvarchar(max) NULL, + usage_summary nvarchar(max) NULL, + operational_summary nvarchar(max) NULL, + uptime_warning nvarchar(512) NULL + ); + + CREATE TABLE + #final_index_actions + ( + database_id integer NOT NULL, + database_name sysname NOT NULL, + schema_id integer NOT NULL, + schema_name sysname NOT NULL, + object_id integer NOT NULL, + table_name sysname NOT NULL, + index_id integer NOT NULL, + index_name sysname NOT NULL, + target_index_name sysname NULL, + action nvarchar(max) NOT NULL, + script nvarchar(max) NOT NULL + ); + + /* + Start insert queries + */ + + IF @debug = 1 + BEGIN + RAISERROR('Generating #filtered_object insert', 0, 0) WITH NOWAIT; + END; + + SELECT + @sql = N' + SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;'; + + SELECT + @sql = N' + SELECT DISTINCT + @database_id, + database_name = DB_NAME(@database_id), + schema_id = t.schema_id, + schema_name = s.name, + object_id = t.object_id, + table_name = t.name, + index_id = i.index_id, + index_name = i.name + FROM ' + QUOTENAME(@database_name) + N'.sys.tables AS t + JOIN ' + QUOTENAME(@database_name) + N'.sys.schemas AS s + ON t.schema_id = s.schema_id + JOIN ' + QUOTENAME(@database_name) + N'.sys.indexes AS i + ON t.object_id = i.object_id + LEFT JOIN ' + QUOTENAME(@database_name) + N'.sys.dm_db_index_usage_stats AS us + ON t.object_id = us.object_id + AND us.database_id = @database_id + WHERE t.is_ms_shipped = 0 + AND t.type <> N''TF''' + + IF @object_id IS NOT NULL + BEGIN + SELECT @sql += N' + AND t.object_id = @object_id'; + END; + + SET @sql += N' + AND EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@database_name) + N'.sys.dm_db_partition_stats AS ps + JOIN ' + QUOTENAME(@database_name) + N'.sys.allocation_units AS au + ON ps.partition_id = au.container_id + WHERE ps.object_id = t.object_id + GROUP + BY ps.object_id + HAVING + SUM(au.total_pages) * 8.0 / 1048576.0 >= @min_size_gb + ) + AND EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@database_name) + N'.sys.dm_db_partition_stats AS ps + WHERE ps.object_id = t.object_id + AND ps.index_id IN (0, 1) + GROUP + BY ps.object_id + HAVING + SUM(ps.row_count) >= @min_rows + ) + AND EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@database_name) + N'.sys.dm_db_index_usage_stats AS ius + WHERE ius.object_id = t.object_id + AND ius.database_id = @database_id + GROUP BY + ius.object_id + HAVING + SUM(ius.user_seeks + ius.user_scans + ius.user_lookups) >= @min_reads + AND + SUM(ius.user_updates) >= @min_writes + ) + OPTION(RECOMPILE);'; + + IF @debug = 1 + BEGIN + PRINT @sql; + END; + + INSERT + #filtered_objects + WITH + (TABLOCK) + ( + database_id, + database_name, + schema_id, + schema_name, + object_id, + table_name, + index_id, + index_name + ) + EXEC sys.sp_executesql + @sql, + N'@database_id int, + @min_reads bigint, + @min_writes bigint, + @min_size_gb decimal(10,2), + @min_rows bigint, + @object_id integer', + @database_id, + @min_reads, + @min_writes, + @min_size_gb, + @min_rows, + @object_id; + + IF ROWCOUNT_BIG() = 0 BEGIN IF @debug = 1 BEGIN RAISERROR('No rows inserted into #filtered_objects', 0, 0) WITH NOWAIT END; END; + + IF @debug = 1 + BEGIN + SELECT + table_name = '#filtered_objects', + fo.* + FROM #filtered_objects AS fo; + END; + + IF @debug = 1 + BEGIN + RAISERROR('Generating #operational_stats insert', 0, 0) WITH NOWAIT; + END; + + SELECT + @sql = N' + SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;'; + + SELECT + @sql += N' + SELECT + os.database_id, + database_name = DB_NAME(os.database_id), + schema_id = s.schema_id, + schema_name = s.name, + os.object_id, + table_name = t.name, + os.index_id, + index_name = i.name, + range_scan_count = SUM(os.range_scan_count), + singleton_lookup_count = SUM(os.singleton_lookup_count), + forwarded_fetch_count = SUM(os.forwarded_fetch_count), + lob_fetch_in_pages = SUM(os.lob_fetch_in_pages), + row_overflow_fetch_in_pages = SUM(os.row_overflow_fetch_in_pages), + leaf_insert_count = SUM(os.leaf_insert_count), + leaf_update_count = SUM(os.leaf_update_count), + leaf_delete_count = SUM(os.leaf_delete_count), + leaf_ghost_count = SUM(os.leaf_ghost_count), + nonleaf_insert_count = SUM(os.nonleaf_insert_count), + nonleaf_update_count = SUM(os.nonleaf_update_count), + nonleaf_delete_count = SUM(os.nonleaf_delete_count), + leaf_allocation_count = SUM(os.leaf_allocation_count), + nonleaf_allocation_count = SUM(os.nonleaf_allocation_count), + row_lock_count = SUM(os.row_lock_count), + row_lock_wait_count = SUM(os.row_lock_wait_count), + row_lock_wait_in_ms = SUM(os.row_lock_wait_in_ms), + page_lock_count = SUM(os.page_lock_count), + page_lock_wait_count = SUM(os.page_lock_wait_count), + page_lock_wait_in_ms = SUM(os.page_lock_wait_in_ms), + index_lock_promotion_attempt_count = SUM(os.index_lock_promotion_attempt_count), + index_lock_promotion_count = SUM(os.index_lock_promotion_count), + page_latch_wait_count = SUM(os.page_latch_wait_count), + page_latch_wait_in_ms = SUM(os.page_latch_wait_in_ms), + tree_page_latch_wait_count = SUM(os.tree_page_latch_wait_count), + tree_page_latch_wait_in_ms = SUM(os.tree_page_latch_wait_in_ms), + page_io_latch_wait_count = SUM(os.page_io_latch_wait_count), + page_io_latch_wait_in_ms = SUM(os.page_io_latch_wait_in_ms), + page_compression_attempt_count = SUM(os.page_compression_attempt_count), + page_compression_success_count = SUM(os.page_compression_success_count) + FROM ' + QUOTENAME(@database_name) + N'.sys.dm_db_index_operational_stats + ( + @database_id, + @object_id, + NULL, + NULL + ) AS os + JOIN ' + QUOTENAME(@database_name) + N'.sys.tables AS t + ON os.object_id = t.object_id + JOIN ' + QUOTENAME(@database_name) + N'.sys.schemas AS s + ON t.schema_id = s.schema_id + JOIN ' + QUOTENAME(@database_name) + N'.sys.indexes AS i + ON os.object_id = i.object_id + AND os.index_id = i.index_id + WHERE EXISTS + ( + SELECT + 1/0 + FROM #filtered_objects fo + WHERE fo.database_id = os.database_id + AND fo.object_id = os.object_id + ) + GROUP BY + os.database_id, + DB_NAME(os.database_id), + s.schema_id, + s.name, + os.object_id, + t.name, + os.index_id, + i.name + OPTION(RECOMPILE);'; + + IF @debug = 1 + BEGIN + PRINT @sql; + END; + + INSERT + #operational_stats + WITH + (TABLOCK) + ( + database_id, + database_name, + schema_id, + schema_name, + object_id, + table_name, + index_id, + index_name, + range_scan_count, + singleton_lookup_count, + forwarded_fetch_count, + lob_fetch_in_pages, + row_overflow_fetch_in_pages, + leaf_insert_count, + leaf_update_count, + leaf_delete_count, + leaf_ghost_count, + nonleaf_insert_count, + nonleaf_update_count, + nonleaf_delete_count, + leaf_allocation_count, + nonleaf_allocation_count, + row_lock_count, + row_lock_wait_count, + row_lock_wait_in_ms, + page_lock_count, + page_lock_wait_count, + page_lock_wait_in_ms, + index_lock_promotion_attempt_count, + index_lock_promotion_count, + page_latch_wait_count, + page_latch_wait_in_ms, + tree_page_latch_wait_count, + tree_page_latch_wait_in_ms, + page_io_latch_wait_count, + page_io_latch_wait_in_ms, + page_compression_attempt_count, + page_compression_success_count + ) + EXEC sys.sp_executesql + @sql, + N'@database_id integer, + @object_id integer', + @database_id, + @object_id; + + IF ROWCOUNT_BIG() = 0 BEGIN IF @debug = 1 BEGIN RAISERROR('No rows inserted into #operational_stats', 0, 0) WITH NOWAIT END; END; + + IF @debug = 1 + BEGIN + SELECT + table_name = '#operational_stats', + os.* + FROM #operational_stats AS os; + END; + + IF @debug = 1 + BEGIN + RAISERROR('Generating #index_details insert', 0, 0) WITH NOWAIT; + END; + + SELECT + @sql = N' + SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;'; + + SELECT + @sql += N' + SELECT + database_id = @database_id, + database_name = DB_NAME(@database_id), + t.object_id, + i.index_id, + s.schema_id, + schema_name = s.name, + table_name = t.name, + index_name = i.name, + column_name = c.name, + i.is_primary_key, + i.is_unique, + i.is_unique_constraint, + is_indexed_view = + CASE + WHEN EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@database_name) + N'.sys.objects AS so + WHERE i.object_id = so.object_id + AND so.is_ms_shipped = 0 + AND so.type = ''V'' + ) + THEN 1 + ELSE 0 + END, + is_foreign_key = + CASE + WHEN EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@database_name) + N'.sys.foreign_key_columns AS f + WHERE f.parent_column_id = c.column_id + AND f.parent_object_id = c.object_id + ) + THEN 1 + ELSE 0 + END, + is_foreign_key_reference = + CASE + WHEN EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@database_name) + N'.sys.foreign_key_columns AS f + WHERE f.referenced_column_id = c.column_id + AND f.referenced_object_id = c.object_id + ) + THEN 1 + ELSE 0 + END, + ic.key_ordinal, + ic.index_column_id, + ic.is_descending_key, + ic.is_included_column, + i.filter_definition, + is_max_length = + CASE + WHEN EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@database_name) + N'.sys.types AS t + WHERE c.system_type_id = t.system_type_id + AND c.user_type_id = t.user_type_id + AND t.name IN (N''varchar'', N''nvarchar'') + AND t.max_length = -1 + ) + THEN 1 + ELSE 0 + END, + user_seeks = ISNULL(us.user_seeks, 0), + user_scans = ISNULL(us.user_scans, 0), + user_lookups = ISNULL(us.user_lookups, 0), + user_updates = ISNULL(us.user_updates, 0), + us.last_user_seek, + us.last_user_scan, + us.last_user_lookup, + us.last_user_update, + is_eligible_for_dedupe = + CASE + WHEN i.type = 2 + THEN 1 + WHEN i.type = 1 + THEN 0 + END + FROM ' + QUOTENAME(@database_name) + N'.sys.tables AS t + JOIN ' + QUOTENAME(@database_name) + N'.sys.schemas AS s + ON t.schema_id = s.schema_id + JOIN ' + QUOTENAME(@database_name) + N'.sys.indexes AS i + ON t.object_id = i.object_id + JOIN ' + QUOTENAME(@database_name) + N'.sys.index_columns AS ic + ON i.object_id = ic.object_id + AND i.index_id = ic.index_id + JOIN ' + QUOTENAME(@database_name) + N'.sys.columns AS c + ON ic.object_id = c.object_id + AND ic.column_id = c.column_id + LEFT JOIN sys.dm_db_index_usage_stats AS us + ON i.object_id = us.object_id + AND i.index_id = us.index_id + AND us.database_id = @database_id + WHERE t.is_ms_shipped = 0 + AND i.type IN (1, 2) + AND i.is_disabled = 0 + AND i.is_hypothetical = 0 + AND EXISTS + ( + SELECT + 1/0 + FROM #filtered_objects fo + WHERE fo.database_id = @database_id + AND fo.object_id = t.object_id + ) + AND EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@database_name) + + CONVERT + ( + nvarchar(MAX), + N'.sys.dm_db_partition_stats ps + WHERE ps.object_id = t.object_id + AND ps.index_id = 1 + AND ps.row_count >= @min_rows + )' + ); + + IF @object_id IS NOT NULL + BEGIN + SELECT @sql += N' + AND t.object_id = @object_id'; + END; + + SELECT + @sql += CONVERT + ( + nvarchar(max), + N' + AND NOT EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@database_name) + N'.sys.objects AS so + WHERE i.object_id = so.object_id + AND so.is_ms_shipped = 0 + AND so.type = N''TF'' + ) + OPTION(RECOMPILE);' + ); + + IF @debug = 1 + BEGIN + PRINT SUBSTRING(@sql, 1, 4000); + PRINT SUBSTRING(@sql, 4000, 8000); + END; + + INSERT + #index_details + WITH + (TABLOCK) + ( + database_id, + database_name, + object_id, + index_id, + schema_id, + schema_name, + table_name, + index_name, + column_name, + is_primary_key, + is_unique, + is_unique_constraint, + is_indexed_view, + is_foreign_key, + is_foreign_key_reference, + key_ordinal, + index_column_id, + is_descending_key, + is_included_column, + filter_definition, + is_max_length, + user_seeks, + user_scans, + user_lookups, + user_updates, + last_user_seek, + last_user_scan, + last_user_lookup, + last_user_update, + is_eligible_for_dedupe + ) + EXEC sys.sp_executesql + @sql, + N'@database_id integer, + @object_id integer, + @min_rows integer', + @database_id, + @object_id, + @min_rows; + + IF ROWCOUNT_BIG() = 0 BEGIN IF @debug = 1 BEGIN RAISERROR('No rows inserted into #index_details', 0, 0) WITH NOWAIT END; END; + + IF @debug = 1 + BEGIN + SELECT + table_name = '#index_details', + * + FROM #index_details AS id; + END; + + IF @debug = 1 + BEGIN + RAISERROR('Generating #partition_stats insert', 0, 0) WITH NOWAIT; + END; + + SELECT + @sql = N' + SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;'; + + SELECT + @sql += N' + SELECT + database_id = @database_id, + database_name = DB_NAME(@database_id), + x.object_id, + x.index_id, + x.schema_id, + x.schema_name, + x.table_name, + x.index_name, + x.partition_id, + x.partition_number, + x.total_rows, + x.total_space_mb, + x.reserved_lob_mb, + x.reserved_row_overflow_mb, + x.data_compression_desc, + built_on = + ISNULL + ( + psfg.partition_scheme_name, + psfg.filegroup_name + ), + psfg.partition_function_name, + pc.partition_columns + FROM + ( + SELECT + ps.object_id, + ps.index_id, + s.schema_id, + schema_name = s.name, + table_name = t.name, + index_name = i.name, + ps.partition_id, + p.partition_number, + total_rows = SUM(ps.row_count), + total_space_mb = SUM(a.total_pages) * 8 / 1024.0, + reserved_lob_mb = SUM(ps.lob_reserved_page_count) * 8. / 1024., + reserved_row_overflow_mb = SUM(ps.row_overflow_reserved_page_count) * 8. / 1024., + p.data_compression_desc, + i.data_space_id + FROM ' + QUOTENAME(@database_name) + N'.sys.tables AS t + JOIN ' + QUOTENAME(@database_name) + N'.sys.indexes AS i + ON t.object_id = i.object_id + JOIN ' + QUOTENAME(@database_name) + N'.sys.schemas AS s + ON t.schema_id = s.schema_id + JOIN ' + QUOTENAME(@database_name) + N'.sys.partitions AS p + ON i.object_id = p.object_id + AND i.index_id = p.index_id + JOIN ' + QUOTENAME(@database_name) + N'.sys.allocation_units AS a + ON p.partition_id = a.container_id + LEFT HASH JOIN ' + QUOTENAME(@database_name) + N'.sys.dm_db_partition_stats AS ps + ON p.partition_id = ps.partition_id + WHERE t.type <> N''TF'' + AND i.type IN (1, 2) + AND EXISTS + ( + SELECT + 1/0 + FROM #filtered_objects fo + WHERE fo.database_id = @database_id + AND fo.object_id = t.object_id + )'; + + IF @object_id IS NOT NULL + BEGIN + SELECT @sql += N' + AND t.object_id = @object_id'; + END; + + SELECT + @sql += N' + GROUP BY + t.name, + i.name, + i.data_space_id, + s.schema_id, + s.name, + p.partition_number, + p.data_compression_desc, + ps.object_id, + ps.index_id, + ps.partition_id + ) AS x + OUTER APPLY + ( + SELECT + filegroup_name = + fg.name, + partition_scheme_name = + ps.name, + partition_function_name = + pf.name + FROM ' + QUOTENAME(@database_name) + N'.sys.filegroups AS fg + FULL JOIN ' + QUOTENAME(@database_name) + N'.sys.partition_schemes AS ps + ON ps.data_space_id = fg.data_space_id + LEFT JOIN ' + QUOTENAME(@database_name) + N'.sys.partition_functions AS pf + ON pf.function_id = ps.function_id + WHERE x.data_space_id = fg.data_space_id + OR x.data_space_id = ps.data_space_id + ) AS psfg + OUTER APPLY + ( + SELECT + partition_columns = + STUFF + ( + ( + SELECT + N'', '' + + c.name + FROM ' + QUOTENAME(@database_name) + N'.sys.index_columns AS ic + JOIN ' + QUOTENAME(@database_name) + N'.sys.columns AS c + ON c.object_id = ic.object_id + AND c.column_id = ic.column_id + WHERE ic.object_id = x.object_id + AND ic.index_id = x.index_id + AND ic.partition_ordinal > 0 + ORDER BY + ic.partition_ordinal + FOR XML + PATH(''''), + TYPE + ).value(''.'', ''nvarchar(max)''), + 1, + 2, + '''' + ) + ) AS pc + OPTION(RECOMPILE);'; + + IF @debug = 1 + BEGIN + PRINT @sql; + END; + + INSERT + #partition_stats WITH(TABLOCK) + ( + database_id, + database_name, + object_id, + index_id, + schema_id, + schema_name, + table_name, + index_name, + partition_id, + partition_number, + total_rows, + total_space_mb, + reserved_lob_mb, + reserved_row_overflow_mb, + data_compression_desc, + built_on, + partition_function_name, + partition_columns + ) + EXEC sys.sp_executesql + @sql, + N'@database_id integer, + @object_id integer', + @database_id, + @object_id; + + IF ROWCOUNT_BIG() = 0 BEGIN IF @debug = 1 BEGIN RAISERROR('No rows inserted into #partition_stats', 0, 0) WITH NOWAIT END; END; + + IF @debug = 1 + BEGIN + SELECT + table_name = '#partition_stats', + * + FROM #partition_stats AS ps; + END; + + IF @debug = 1 + BEGIN + RAISERROR('Performing #index_analysis insert', 0, 0) WITH NOWAIT; + END; + + INSERT INTO + #index_analysis + WITH + (TABLOCK) + ( + database_id, + database_name, + schema_id, + schema_name, + table_name, + object_id, + index_id, + index_name, + is_unique, + key_columns, + included_columns, + filter_definition + ) + SELECT + @database_id, + database_name = DB_NAME(@database_id), + id1.schema_id, + id1.schema_name, + id1.table_name, + id1.object_id, + id1.index_id, + id1.index_name, + id1.is_unique, + key_columns = + STUFF + ( + ( + SELECT + N', ' + + id2.column_name + + CASE + WHEN id2.is_descending_key = 1 + THEN N' DESC' + ELSE N'' + END + FROM #index_details id2 + WHERE id2.object_id = id1.object_id + AND id2.index_id = id1.index_id + AND id2.is_included_column = 0 + GROUP BY + id2.column_name, + id2.is_descending_key, + id2.key_ordinal + ORDER BY + id2.key_ordinal + FOR XML + PATH(''), + TYPE + ).value('text()[1]','nvarchar(max)'), + 1, + 2, + '' + ), + included_columns = + STUFF + ( + ( + SELECT + N', ' + + id2.column_name + FROM #index_details id2 + WHERE id2.object_id = id1.object_id + AND id2.index_id = id1.index_id + AND id2.is_included_column = 1 + GROUP BY + id2.column_name + ORDER BY + id2.column_name + FOR XML + PATH(''), + TYPE + ).value('text()[1]','nvarchar(max)'), + 1, + 2, + '' + ), + id1.filter_definition + FROM #index_details id1 + WHERE id1.is_eligible_for_dedupe = 1 + GROUP BY + id1.schema_name, + id1.schema_id, + id1.table_name, + id1.index_name, + id1.index_id, + id1.is_unique, + id1.object_id, + id1.index_id, + id1.filter_definition + OPTION(RECOMPILE); + + IF ROWCOUNT_BIG() = 0 BEGIN IF @debug = 1 BEGIN RAISERROR('No rows inserted into #index_analysis', 0, 0) WITH NOWAIT END; END; + + IF @debug = 1 + BEGIN + SELECT + table_name = '#index_analysis', + ia.* + FROM #index_analysis AS ia; + END; + + IF @debug = 1 + BEGIN + RAISERROR('Starting cursor', 0, 0) WITH NOWAIT; + END; + + CREATE TABLE #index_supersede_debug + ( + id integer IDENTITY(1,1) PRIMARY KEY, + step nvarchar(100), + current_index sysname, + other_index sysname, + current_key_columns nvarchar(max), + other_key_columns nvarchar(max), + current_include_columns nvarchar(max), + other_include_columns nvarchar(max), + decision nvarchar(100), + reason nvarchar(max) + ); + + DECLARE + @current_key_cols nvarchar(max) = N'', + @current_include_cols nvarchar(max) = N''; + + + /*Analyze indexes*/ + SET @index_cursor = CURSOR + LOCAL + STATIC + FORWARD_ONLY + READ_ONLY + FOR + SELECT DISTINCT + ia.database_id, + ia.database_name, + ia.schema_id, + ia.schema_name, + ia.object_id, + ia.table_name, + ia.index_id, + ia.index_name, + ia.is_unique, + ia.filter_definition + FROM #index_analysis AS ia + ORDER BY + ia.table_name, + ia.index_name; + + OPEN @index_cursor; + + FETCH NEXT + FROM @index_cursor + INTO + @c_database_id, + @c_database_name, + @c_schema_id, + @c_schema_name, + @c_object_id, + @c_table_name, + @c_index_id, + @c_index_name, + @c_is_unique, + @c_filter_definition; + + WHILE @@FETCH_STATUS = 0 + BEGIN + + IF @debug = 1 + BEGIN + RAISERROR('Performing #index_analysis update', 0, 0) WITH NOWAIT; + END; + + SELECT + @current_key_cols = + STUFF + ( + ( + SELECT + N', ' + + id.column_name + + CASE + WHEN id.is_descending_key = 1 + THEN N' DESC' + ELSE '' + END + FROM #index_details AS id + WHERE id.database_id = @c_database_id + AND id.object_id = @c_object_id + AND id.index_id = @c_index_id + AND id.is_included_column = 0 + AND id.key_ordinal > 0 + ORDER BY + id.key_ordinal + FOR + XML + PATH(''), + TYPE + ).value('.', 'nvarchar(max)'), + 1, + 2, + '' + ); + + SELECT + @current_include_cols = + STUFF + ( + ( + SELECT + N', ' + + id.column_name + FROM #index_details AS id + WHERE id.database_id = @c_database_id + AND id.object_id = @c_object_id + AND id.index_id = @c_index_id + AND id.is_included_column = 1 + ORDER BY + id.column_name + FOR XML + PATH(''), + TYPE + ).value('.', 'nvarchar(max)'), + 1, + 2, + '' + ); + + INSERT INTO + #index_supersede_debug + ( + step, + current_index, + other_index, + current_key_columns, + other_key_columns, + current_include_columns, + other_include_columns, + decision, + reason + ) + SELECT + 'Before Update', + @c_index_name, + other_indexes.other_index_name, + @current_key_cols, + other_indexes.other_key_cols, + @current_include_cols, + other_indexes.other_include_cols, + 'Checking', + 'Starting comparison' + FROM + ( + -- Get other indexes for the same table + SELECT + other_index_name = id2.index_name, + other_key_cols = + STUFF + ( + ( + SELECT + N', ' + + id3.column_name + + CASE + WHEN id3.is_descending_key = 1 + THEN N' DESC' + ELSE N'' + END + FROM #index_details AS id3 + WHERE id3.database_id = id2.database_id + AND id3.object_id = id2.object_id + AND id3.index_id = id2.index_id + AND id3.is_included_column = 0 + AND id3.key_ordinal > 0 + ORDER BY + id3.key_ordinal + FOR XML + PATH(''), + TYPE + ).value('.', 'nvarchar(max)'), + 1, + 2, + '' + ), + other_include_cols = + STUFF + ( + ( + SELECT + N', ' + + id3.column_name + FROM #index_details AS id3 + WHERE id3.database_id = id2.database_id + AND id3.object_id = id2.object_id + AND id3.index_id = id2.index_id + AND id3.is_included_column = 1 + ORDER BY + id3.column_name + FOR XML + PATH(''), + TYPE + ).value('.', 'nvarchar(max)'), + 1, + 2, + '' + ) + FROM #index_details AS id2 + WHERE id2.database_id = @c_database_id + AND id2.object_id = @c_object_id + AND id2.index_id <> @c_index_id + AND + ( + id2.index_name = N'IX_Users_AccountId_DisplayName' + OR id2.index_name = N'u' + ) -- Focus on problem indexes + GROUP BY + id2.database_id, + id2.object_id, + id2.index_id, + id2.index_name + ) AS other_indexes + WHERE + ( + @c_index_name = N'IX_Users_AccountId_DisplayName' + OR @c_index_name = N'u' + OR -- Focus on problem indexes + other_indexes.other_index_name = N'IX_Users_AccountId_DisplayName' + OR other_indexes.other_index_name = N'u' + ); + + + WITH + IndexColumns AS + ( + SELECT + id.* + FROM #index_details id + WHERE id.database_id = @c_database_id + AND id.object_id = @c_object_id + AND id.is_eligible_for_dedupe = 1 + ), + CurrentIndexColumns AS + ( + SELECT + ic.* + FROM IndexColumns AS ic + WHERE ic.index_id = @c_index_id + AND ic.is_eligible_for_dedupe = 1 + ), + OtherIndexColumns AS + ( + SELECT + ic.* + FROM IndexColumns AS ic + WHERE ic.index_id <> @c_index_id + AND ic.is_eligible_for_dedupe = 1 + ) + UPDATE + ia + SET + ia.is_redundant = + CASE + WHEN NOT EXISTS + ( + SELECT + 1/0 + FROM CurrentIndexColumns cic + WHERE cic.is_included_column = 0 /* Only check key columns */ + AND NOT EXISTS + ( + SELECT + 1/0 + FROM OtherIndexColumns oic + WHERE oic.column_name = cic.column_name + AND oic.is_included_column = 0 /* Must be in key columns */ + AND oic.key_ordinal = cic.key_ordinal /* Check leading edge */ + AND oic.is_descending_key = cic.is_descending_key + ) + ) + AND + ( + /* Check included columns separately since order doesn't matter */ + NOT EXISTS + ( + SELECT + 1/0 + FROM CurrentIndexColumns cic + WHERE cic.is_included_column = 1 + AND NOT EXISTS + ( + SELECT + 1/0 + FROM OtherIndexColumns oic + WHERE oic.column_name = cic.column_name + AND + ( + oic.is_included_column = 1 + OR oic.is_included_column = 0 /* Include cols can be covered by key cols */ + ) + ) + ) + ) + AND ISNULL(REPLACE(REPLACE(REPLACE(ia.filter_definition, ' ', ''), '(', ''), ')', ''), '') = + ISNULL(REPLACE(REPLACE(REPLACE(@c_filter_definition, ' ', ''), '(', ''), ')', ''), '') + AND + ( + ia.is_unique = 0 + OR + ( + ia.is_unique = 1 + AND @c_is_unique = 1 + ) + ) + THEN 1 + ELSE 0 + END, + ia.superseded_by = + CASE + WHEN NOT EXISTS + ( + SELECT + 1/0 + FROM CurrentIndexColumns cic + WHERE cic.is_included_column = 0 /* Only check key columns */ + AND NOT EXISTS + ( + SELECT + 1/0 + FROM OtherIndexColumns oic + WHERE oic.column_name = cic.column_name + AND oic.is_included_column = 0 /* Must be in key columns */ + AND oic.key_ordinal = cic.key_ordinal /* Check leading edge */ + AND oic.is_descending_key = cic.is_descending_key + ) + ) + AND + ( + /* Check included columns separately since order doesn't matter */ + NOT EXISTS + ( + SELECT + 1/0 + FROM CurrentIndexColumns cic + WHERE cic.is_included_column = 1 + AND NOT EXISTS + ( + SELECT + 1/0 + FROM OtherIndexColumns oic + WHERE oic.column_name = cic.column_name + AND + ( + oic.is_included_column = 1 + OR oic.is_included_column = 0 /* Include cols can be covered by key cols */ + ) + ) + ) + ) + AND ISNULL(ia.filter_definition, '') = ISNULL(@c_filter_definition, '') + AND + ( + ia.is_unique = 0 + OR @c_is_unique = 1 + ) + AND ia.index_name <> @c_index_name + THEN @c_index_name + ELSE ia.superseded_by + END, + ia.missing_columns = + STUFF + ( + ( + SELECT DISTINCT + N', ' + + oic.column_name + FROM OtherIndexColumns oic + WHERE NOT EXISTS + ( + SELECT + 1/0 + FROM CurrentIndexColumns cic + WHERE cic.column_name = oic.column_name + ) + FOR XML + PATH(''), + TYPE + ).value('.', 'nvarchar(max)'), + 1, + 2, + '' + ) + FROM #index_analysis ia + WHERE ia.database_id = @c_database_id + AND ia.schema_name = @c_schema_name + AND ia.table_name = @c_table_name + AND ia.index_name <> @c_index_name; + + INSERT INTO + #index_supersede_debug + ( + step, + current_index, + other_index, + current_key_columns, + other_key_columns, + current_include_columns, + other_include_columns, + decision, + reason + ) + SELECT + 'After Update', + @c_index_name, + ia.index_name, + @current_key_cols, + other_key_cols = + STUFF + ( + ( + SELECT + N', ' + + id3.column_name + + CASE + WHEN id3.is_descending_key = 1 + THEN N' DESC' + ELSE N'' + END + FROM #index_details AS id3 + WHERE id3.database_id = ia.database_id + AND id3.object_id = ia.object_id + AND id3.index_id = ia.index_id + AND id3.is_included_column = 0 + AND id3.key_ordinal > 0 + ORDER BY + id3.key_ordinal + FOR + XML + PATH(''), + TYPE + ).value('.', 'nvarchar(max)'), + 1, + 2, + '' + ), + @current_include_cols, + other_include_cols = + STUFF + ( + ( + SELECT + N', ' + + id3.column_name + FROM #index_details AS id3 + WHERE id3.database_id = ia.database_id + AND id3.object_id = ia.object_id + AND id3.index_id = ia.index_id + AND id3.is_included_column = 1 + ORDER BY + id3.column_name + FOR XML PATH(''), TYPE + ).value('.', 'nvarchar(max)'), + 1, + 2, + '' + ), + CASE + WHEN ia.superseded_by = @c_index_name + THEN 'Current supersedes Other' + ELSE 'No Change' + END, + 'superseded_by: ' + + ISNULL(ia.superseded_by, 'NULL') + + ', is_redundant: ' + + CONVERT(varchar(MAX), ia.is_redundant) + FROM #index_analysis AS ia + WHERE ia.database_id = @c_database_id + AND ia.schema_id = @c_schema_id + AND ia.table_name = @c_table_name + AND ia.index_name <> @c_index_name + AND + ( + ia.index_name = 'IX_Users_AccountId_DisplayName' + OR ia.index_name = 'u' + OR -- Focus on problem indexes + @c_index_name = 'IX_Users_AccountId_DisplayName' + OR @c_index_name = 'u' + ) + AND ia.superseded_by = @c_index_name; + + FETCH NEXT + FROM @index_cursor + INTO + @c_database_id, + @c_database_name, + @c_schema_id, + @c_schema_name, + @c_object_id, + @c_table_name, + @c_index_id, + @c_index_name, + @c_is_unique, + @c_filter_definition; + END; + + SELECT + * + FROM #index_supersede_debug + ORDER BY + id; + + -- Also add this to see the final state of relevant indexes + SELECT + state = 'Final state', + ia.table_name, + ia.index_name, + ia.is_redundant, + ia.superseded_by, + ia.action + FROM #index_analysis AS ia + WHERE ia.table_name = 'Users' + AND + ( + ia.index_name = 'IX_Users_AccountId_DisplayName' + OR ia.index_name = 'u' + ) + ORDER BY + ia.index_name; + + IF @debug = 1 + BEGIN + RAISERROR('Performing #index_analysis update after cursor', 0, 0) WITH NOWAIT; + END; + + /*Determine actions*/ + UPDATE + #index_analysis + SET + action = + CASE + WHEN is_redundant = 1 + THEN N'DROP' + WHEN superseded_by IS NOT NULL + AND missing_columns IS NULL + THEN N'MERGE INTO ' + + superseded_by + WHEN superseded_by IS NOT NULL + AND missing_columns IS NOT NULL + THEN N'MERGE INTO ' + + superseded_by + + N' (ADD ' + + missing_columns + + N')' + ELSE N'KEEP' + END; + + IF @debug = 1 + BEGIN + SELECT + table_name = '#index_analysis after update', + ia.* + FROM #index_analysis AS ia; + END; + + IF @debug = 1 + BEGIN + RAISERROR('Performing #index_cleanup_report insert', 0, 0) WITH NOWAIT; + END; + + INSERT INTO + #index_cleanup_report + WITH + (TABLOCK) + ( + database_id, + database_name, + schema_id, + schema_name, + table_name, + object_id, + index_id, + index_name, + action, + cleanup_script, + original_definition, + user_seeks, + user_scans, + user_lookups, + user_updates, + last_user_seek, + last_user_scan, + last_user_lookup, + last_user_update, + range_scan_count, + singleton_lookup_count, + leaf_insert_count, + leaf_update_count, + leaf_delete_count, + page_lock_count, + page_lock_wait_count, + page_lock_wait_in_ms + ) + SELECT + @database_id, + @database_name, + ia.schema_id, + ia.schema_name, + ia.table_name, + ia.object_id, + ia.index_id, + ia.index_name, + ia.action, + cleanup_script = + CASE + WHEN ia.action = N'DROP' + THEN NCHAR(10) + + N'DROP INDEX ' + + QUOTENAME(ia.index_name) + + N' ON ' + + QUOTENAME(DB_NAME(ia.database_id)) + + N'.' + + QUOTENAME(ia.schema_name) + + N'.' + + QUOTENAME(ia.table_name) + + N';' + WHEN ia.action LIKE N'MERGE INTO%' + THEN NCHAR(10) + + N'CREATE ' + + CASE + WHEN ia.is_unique = 1 + THEN N'UNIQUE ' + ELSE N'' + END + + N'INDEX ' + + QUOTENAME(ia.superseded_by) + + NCHAR(10) + + N'ON ' + + QUOTENAME(DB_NAME(ia.database_id)) + + N'.' + + QUOTENAME(ia.schema_name) + + N'.' + + QUOTENAME(ia.table_name) + + NCHAR(10) + + N' (' + + ISNULL(superseding.key_columns, ia.key_columns) + + N')' + + NCHAR(10) + + CASE + WHEN + ( + superseding.included_columns IS NOT NULL + OR ia.included_columns IS NOT NULL + ) + OR ia.missing_columns IS NOT NULL + THEN N' INCLUDE' + + NCHAR(10) + + N' (' + + -- Combine all INCLUDE columns with proper parsing + STUFF + ( + ( + SELECT DISTINCT + N', ' + + column_value + FROM + ( + -- From superseding index + SELECT DISTINCT + column_value = + LTRIM(RTRIM(value.c.value('.', 'sysname'))) + FROM + ( + SELECT + Columns = + CONVERT + ( + xml, + '' + + REPLACE + ( + ISNULL + ( + superseding.included_columns, + '' + ), + ', ', + '' + ) + + '' + ) + ) t + CROSS APPLY t.Columns.nodes('/c') AS value(c) + + UNION + + -- From current index + SELECT DISTINCT + column_value = + LTRIM(RTRIM(value.c.value('.', 'sysname'))) + FROM + ( + SELECT + Columns = + CONVERT + ( + xml, + '' + + REPLACE + ( + ISNULL + ( + ia.included_columns, + '' + ), + ', ', + '' + ) + + '' + ) + ) t + CROSS APPLY t.Columns.nodes('/c') AS value(c) + + UNION + + -- From missing columns + SELECT DISTINCT + column_value = + LTRIM(RTRIM(value.c.value('.', 'sysname'))) + FROM + ( + SELECT + Columns = + CONVERT + ( + xml, + '' + + REPLACE + ( + ISNULL + ( + ia.missing_columns, + '' + ), + ', ', + '' + ) + '' + ) + ) t + CROSS APPLY t.Columns.nodes('/c') AS value(c) + ) AS all_columns + WHERE LEN(column_value) > 0 + /*ED TODO*/ + FOR + XML + PATH(''), + TYPE + ).value('.', 'nvarchar(max)'), + 1, + 2, + '' + ) + + N')' + ELSE N'' + END + + CASE + /* Check for partitioning in the superseding index first */ + WHEN EXISTS + ( + SELECT + 1/0 + FROM #partition_stats ps_super + WHERE ps_super.table_name = ia.table_name + AND ps_super.index_name = ia.superseded_by + AND ps_super.partition_function_name IS NOT NULL + ) + THEN + ( + SELECT TOP (1) + NCHAR(10) + + N' ON ' + + QUOTENAME(ps_super.partition_function_name) + + N'(' + + ps_super.partition_columns + + N')' + FROM #partition_stats ps_super + WHERE ps_super.table_name = ia.table_name + AND ps_super.index_name = ia.superseded_by + ) + /* Fall back to the current index's partitioning if available */ + WHEN ps.partition_function_name IS NOT NULL + THEN NCHAR(10) + + N' ON ' + + QUOTENAME(ps.partition_function_name) + + N'(' + + ps.partition_columns + + N')' + ELSE N'' + END + + CASE + WHEN ia.filter_definition IS NOT NULL + THEN NCHAR(10) + + N' WHERE ' + + ia.filter_definition + ELSE N'' + END + + NCHAR(10) + + N' WITH ' + + NCHAR(10) + + N' (DROP_EXISTING = ON, FILLFACTOR = 100, SORT_IN_TEMPDB = ON, ONLINE = ' + + CASE + WHEN @online = 'true' /*Best effort at detecting online index abilities*/ + THEN N'ON' + ELSE N'OFF' + END + + CASE + WHEN ps.data_compression_desc <> N'NONE' + THEN N', DATA_COMPRESSION = ' + + ps.data_compression_desc + ELSE N', DATA_COMPRESSION = PAGE' /* Add PAGE compression by default for merged indexes */ + END + + N');' + + NCHAR(10) + + NCHAR(10) + + N' ALTER INDEX ' + + QUOTENAME(ia.index_name) + + N' ON ' + + QUOTENAME(DB_NAME(ia.database_id)) + + N'.' + + QUOTENAME(ia.schema_name) + + N'.' + + QUOTENAME(ia.table_name) + + N' DISABLE' + ELSE N'' + END + + N';', + original_definition = + NCHAR(10) + + N' -- CREATE ' + + CASE + WHEN ia.is_unique = 1 + THEN N'UNIQUE ' + ELSE N'' + END + + N'INDEX ' + + QUOTENAME(ia.index_name) + + NCHAR(10) + + N' -- ON ' + + QUOTENAME(DB_NAME(ia.database_id)) + + N'.' + + QUOTENAME(ia.schema_name) + + N'.' + + QUOTENAME(ia.table_name) + + NCHAR(10) + + N' -- (' + + ia.key_columns + + N')' + + CASE + WHEN ia.included_columns IS NOT NULL + THEN NCHAR(10) + + N' -- INCLUDE' + + NCHAR(10) + + N' -- (' + + ia.included_columns + + N')' + ELSE N'' + END + + CASE + WHEN ps.partition_function_name IS NOT NULL + THEN NCHAR(10) + + N' -- ON ' + + QUOTENAME(ps.partition_function_name) + + N'(' + + ps.partition_columns + + N')' + ELSE N'' + END + + CASE + WHEN ia.filter_definition IS NOT NULL + THEN NCHAR(10) + + N' -- WHERE ' + + ia.filter_definition + ELSE N'' + END + + N';' + + NCHAR(10), + id.user_seeks, + id.user_scans, + id.user_lookups, + id.user_updates, + id.last_user_seek, + id.last_user_scan, + id.last_user_lookup, + id.last_user_update, + os.range_scan_count, + os.singleton_lookup_count, + os.leaf_insert_count, + os.leaf_update_count, + os.leaf_delete_count, + os.page_lock_count, + os.page_lock_wait_count, + os.page_lock_wait_in_ms + FROM #index_analysis ia + LEFT JOIN #partition_stats AS ps + ON ia.table_name = ps.table_name + AND ia.index_name = ps.index_name + LEFT JOIN #index_details AS id + ON ia.table_name = id.table_name + AND ia.index_name = id.index_name + LEFT JOIN #operational_stats AS os + ON id.object_id = os.object_id + AND id.index_id = os.index_id + LEFT JOIN #index_analysis AS superseding + ON ia.superseded_by = superseding.index_name + AND ia.table_name = superseding.table_name + OPTION(RECOMPILE); + + IF ROWCOUNT_BIG() = 0 BEGIN IF @debug = 1 BEGIN RAISERROR('No rows inserted into #index_cleanup_report', 0, 0) WITH NOWAIT END; END; + + IF @debug = 1 + BEGIN + SELECT + table_name = '#index_cleanup_report', + icr.* + FROM #index_cleanup_report AS icr; + END; + + IF @debug = 1 + BEGIN + RAISERROR('Performing #index_cleanup_summary insert', 0, 0) WITH NOWAIT; + END; + + INSERT INTO + #index_cleanup_summary + WITH + (TABLOCK) + ( + database_id, + database_name, + schema_id, + schema_name, + table_name, + object_id, + index_id, + index_name, + action, + details, + current_definition, + proposed_definition, + usage_summary, + operational_summary, + uptime_warning + ) + SELECT + icr.database_id, + icr.database_name, + icr.schema_id, + icr.schema_name, + icr.table_name, + icr.object_id, + icr.index_id, + icr.index_name, + action = + CASE + WHEN icr.action = N'KEEP' + THEN N'Keep' + WHEN icr.action = N'DROP' + THEN N'Drop' + WHEN icr.action LIKE N'MERGE INTO%' + THEN N'Merge' + ELSE N'???' + END, + details = + CASE + WHEN icr.action = N'KEEP' + THEN N'No action needed' + WHEN icr.action = N'DROP' + THEN N'Index is redundant and can be safely dropped' + WHEN icr.action LIKE N'MERGE INTO%' + THEN N'Merge into index: ' + + SUBSTRING + ( + icr.action, + 12, + CHARINDEX(N' ', icr.action, 12) - 12 + ) + ELSE N'???' + END, + current_definition = icr.original_definition, + proposed_definition = + CASE + WHEN icr.action LIKE N'MERGE INTO%' + THEN icr.cleanup_script + ELSE NULL + END, + usage_summary = + N'Seeks: ' + CONVERT(nvarchar(20), icr.user_seeks) + + N', Scans: ' + CONVERT(nvarchar(20), icr.user_scans) + + N', Lookups: ' + CONVERT(nvarchar(20), icr.user_lookups) + + N', Updates: ' + CONVERT(nvarchar(20), icr.user_updates) + + N', Last used: ' + + ISNULL + ( + CONVERT + ( + nvarchar(30), + NULLIF + ( + DATEADD + ( + SECOND, + -1, + CASE + WHEN icr.last_user_seek > icr.last_user_scan + AND icr.last_user_seek > icr.last_user_lookup + THEN icr.last_user_seek + WHEN icr.last_user_scan > icr.last_user_lookup + THEN icr.last_user_scan + ELSE icr.last_user_lookup + END + ), + N'1900-01-01' + ), 120 + ), + N'Unknown' + ), + operational_summary = + N'Range scans: ' + CONVERT(nvarchar(20), icr.range_scan_count) + + N', Lookups: ' + CONVERT(nvarchar(20), icr.singleton_lookup_count) + + N', Inserts: ' + CONVERT(nvarchar(20), icr.leaf_insert_count) + + N', Updates: ' + CONVERT(nvarchar(20), icr.leaf_update_count) + + N', Deletes: ' + CONVERT(nvarchar(20), icr.leaf_delete_count), + uptime_warning = + CASE + WHEN icr.user_seeks = 0 AND icr.user_scans = 0 AND icr.user_lookups = 0 + THEN + CASE + WHEN TRY_PARSE(@uptime_days AS integer) < 7 + THEN N'WARNING: SQL Server has been running for only ' + + @uptime_days + + N' days. Usage statistics may not be reliable.' + WHEN TRY_PARSE(@uptime_days AS integer) < 14 + THEN N'CAUTION: SQL Server has been running for only ' + + @uptime_days + + N' days. Usage statistics may be incomplete.' + WHEN TRY_PARSE(@uptime_days AS integer) < 30 + THEN N'NOTE: SQL Server has been running for only ' + + @uptime_days + + N' days. Consider this when evaluating index usage.' + ELSE N'NOTE: SQL Server has been up for ' + + @uptime_days + + N' days, which makes analysis good, but... Are you patching this thing?' + END + ELSE NULL + END + FROM #index_cleanup_report AS icr + OPTION(RECOMPILE); + + IF ROWCOUNT_BIG() = 0 BEGIN IF @debug = 1 BEGIN RAISERROR('No rows inserted into #index_cleanup_summary', 0, 0) WITH NOWAIT END; END; + + IF @debug = 1 + BEGIN + SELECT + table_name = '#index_cleanup_summary', + ics.* + FROM #index_cleanup_summary AS ics; + END; + + IF @debug = 1 + BEGIN + RAISERROR('Going into summary and reports', 0, 0) WITH NOWAIT; + END; + + /* Index Cleanup Summary Report */ + + IF @debug = 1 + BEGIN + RAISERROR('Index Cleanup Summary', 0, 0) WITH NOWAIT; + END; + + SELECT + summary_type = + 'Index Cleanup Summary', + total_indexes_analyzed = + COUNT_BIG(DISTINCT icr.index_name), + indexes_to_drop = + SUM + ( + CASE + WHEN icr.action = 'DROP' + THEN 1 + ELSE 0 + END + ), + indexes_to_merge = + SUM + ( + CASE + WHEN icr.action LIKE 'MERGE INTO%' + THEN 1 + ELSE 0 + END + ), + unused_indexes = + SUM + ( + CASE + WHEN icr.user_seeks = 0 + AND icr.user_scans = 0 + AND icr.user_lookups = 0 + THEN 1 + ELSE 0 + END + ), + space_savings_gb = + CONVERT + ( + decimal(10, 2), + ( + SELECT + SUM(ps_total.space_saved_mb) / 1024.0 + FROM + ( + SELECT + icr_distinct.index_name, + icr_distinct.table_name, + space_saved_mb = SUM(ps_inner.total_space_mb) + FROM #index_cleanup_report AS icr_distinct + JOIN #partition_stats AS ps_inner + ON ps_inner.table_name = icr_distinct.table_name + AND ps_inner.index_name = icr_distinct.index_name + WHERE icr_distinct.action = 'DROP' + OR icr_distinct.action LIKE 'MERGE INTO%' + GROUP BY + icr_distinct.index_name, + icr_distinct.table_name + ) AS ps_total + ) + ), + write_operations_avoided = + SUM + ( + CASE + WHEN icr.action = 'DROP' + OR icr.action LIKE 'MERGE INTO%' + THEN ISNULL(icr.user_updates, 0) + ELSE 0 + END + ) + FROM #index_cleanup_report AS icr + OPTION (RECOMPILE); + + IF @debug = 1 + BEGIN + RAISERROR('Top tables by potential space savings', 0, 0) WITH NOWAIT; + END; + + /* Top tables by potential space savings */ + SELECT TOP (10) + icr.database_name, + icr.table_name, + indexes_affected = + COUNT_BIG(DISTINCT icr.index_name), + space_savings_gb = + CONVERT + ( + decimal(10,2), + ( + SELECT + SUM(ps_total.space_saved_mb) / 1024.0 + FROM + ( + SELECT + ps_inner.table_name, + space_saved_mb = + SUM(ps_inner.total_space_mb) + FROM #partition_stats AS ps_inner + JOIN #index_cleanup_report AS icr_inner + ON ps_inner.table_name = icr_inner.table_name + AND ps_inner.index_name = icr_inner.index_name + WHERE icr_inner.table_name = icr.table_name + AND + ( + icr_inner.action = 'DROP' + OR icr_inner.action LIKE 'MERGE INTO%' + ) + GROUP BY + ps_inner.table_name + ) AS ps_total + ) + ), + write_operations_avoided = + SUM(ISNULL(icr.user_updates, 0)) + FROM #index_cleanup_report AS icr + WHERE + ( + icr.action = 'DROP' + OR icr.action LIKE 'MERGE INTO%' + ) + GROUP BY + icr.database_name, + icr.table_name + ORDER BY + space_savings_gb DESC + OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + RAISERROR('Page Compression Opportunity Summary', 0, 0) WITH NOWAIT; + END; + + /* Summary of non-compressed indexes */ + SELECT + summary_type = 'Page Compression Opportunity Summary', + candidate_indexes = + COUNT_BIG(*), + total_size_gb = + SUM(ps.total_space_mb) / 1024.0, + estimated_savings_low_gb = + (SUM(ps.total_space_mb) * 0.20) / 1024.0, /* Conservative estimate (20%) */ + estimated_savings_typical_gb = + (SUM(ps.total_space_mb) * 0.40) / 1024.0, /* Typical estimate (40%) */ + estimated_savings_high_gb = + (SUM(ps.total_space_mb) * 0.60) / 1024.0 /* Optimistic estimate (60%) */ + FROM #partition_stats ps + WHERE ps.data_compression_desc = 'NONE' + AND NOT EXISTS + ( + SELECT + 1/0 + FROM #index_cleanup_report AS icr + WHERE icr.index_name = ps.index_name + AND + ( + icr.action = 'DROP' + OR icr.action LIKE 'MERGE INTO%' + ) + ) + OPTION(RECOMPILE); + + -- Top candidates for page compression + + IF @debug = 1 + BEGIN + RAISERROR('Top candidates for page compression', 0, 0) WITH NOWAIT; + END; + + SELECT TOP (20) + database_name = + @database_name, + ps.schema_name, + ps.table_name, + ps.index_name, + index_type = + CASE + WHEN ps.index_id = 1 + THEN 'CLUSTERED' + ELSE 'NONCLUSTERED' + END, + size_gb = + SUM(ps.total_space_mb) / 1024.0, + estimated_savings_low_gb = + (SUM(ps.total_space_mb) * 0.20) / 1024.0, -- Conservative (20%) + estimated_savings_typical_gb = + (SUM(ps.total_space_mb) * 0.40) / 1024.0, -- Typical (40%) + estimated_savings_high_gb = + (SUM(ps.total_space_mb) * 0.60) / 1024.0, -- Optimistic (60%) + rebuild_script = + N'ALTER INDEX ' + + QUOTENAME(ps.index_name) + + N' ON ' + + QUOTENAME(ps.schema_name) + + N'.' + + QUOTENAME(ps.table_name) + + N' REBUILD WITH + (DROP_EXISTING = ON, FILLFACTOR = 100, SORT_IN_TEMPDB = ON, ONLINE = ' + + CASE + WHEN @online = 'true' + THEN N'ON' + ELSE N'OFF' + END + + N', DATA_COMPRESSION = PAGE + );' + FROM #partition_stats ps + WHERE ps.data_compression_desc = N'NONE' + GROUP BY + ps.schema_name, + ps.table_name, + ps.index_name, + ps.index_id + ORDER BY + SUM(ps.total_space_mb) DESC + OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + RAISERROR('Select from #index_cleanup_summary', 0, 0) WITH NOWAIT; + END; + + SELECT + ics.database_name, + ics.table_name, + ics.index_name, + ics.action, + ics.details, + ics.current_definition, + ics.proposed_definition, + ics.usage_summary, + ics.operational_summary, + ics.uptime_warning + FROM #index_cleanup_summary AS ics + ORDER BY + CASE ics.action + WHEN N'Drop' THEN 1 + WHEN N'Merge' THEN 2 + WHEN N'Keep' THEN 3 + ELSE 999 + END, + ics.table_name, + ics.index_name + OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + RAISERROR('Performing #final_index_actions insert', 0, 0) WITH NOWAIT; + END; + + -- Replace the existing INSERT into #final_index_actions for MERGE operations with this: + WITH + MergeTargets AS + ( + -- Get distinct target indexes for merges + SELECT DISTINCT + ia.database_id, + ia.database_name, + ia.schema_id, + ia.schema_name, + ia.object_id, + ia.table_name, + ia.index_name, + target_index = + SUBSTRING + ( + ia.action, + 12, + CHARINDEX + ( + N' ', + ia.action + + N' ', + 12 + ) - 12 + ) + FROM #index_cleanup_report ia + WHERE ia.action LIKE N'MERGE INTO%' + AND SUBSTRING + ( + ia.action, + 12, + CHARINDEX + ( + N' ', + ia.action + + N' ', + 12 + ) - 12 + ) <> ia.index_name + ) + -- Insert a single CREATE INDEX statement for each target index + INSERT INTO + #final_index_actions + WITH + (TABLOCK) + ( + database_id, + database_name, + schema_id, + schema_name, + object_id, + table_name, + index_id, + index_name, + target_index_name, + action, + script + ) + SELECT DISTINCT + mt.database_id, + mt.database_name, + mt.schema_id, + mt.schema_name, + mt.object_id, + mt.table_name, + index_id = + ISNULL + ( + ( + SELECT TOP (1) + ia.index_id + FROM #index_analysis ia + WHERE ia.database_id = mt.database_id + AND ia.table_name = mt.table_name + AND ia.index_name = mt.target_index + ), + 0 + ), + mt.index_name, + mt.target_index, + action = + N'MERGE CONSOLIDATED', + script = + N'CREATE INDEX ' + + QUOTENAME(mt.target_index) + + N' ON ' + + QUOTENAME(mt.database_name) + + N'.' + + QUOTENAME(mt.schema_name) + + N'.' + + QUOTENAME(mt.table_name) + + N' (' + + -- Get key columns from one of the indexes being merged + ( + SELECT TOP (1) + ia.key_columns + FROM #index_analysis ia + WHERE ia.database_id = mt.database_id + AND ia.table_name = mt.table_name + AND ia.index_name = mt.target_index + ) + + N')' + + -- Include all distinct columns from all indexes being merged into this target + CASE + WHEN EXISTS + ( + SELECT + 1/0 + FROM #index_cleanup_report icr + WHERE icr.database_id = mt.database_id + AND icr.table_name = mt.table_name + AND icr.action LIKE N'MERGE INTO ' + mt.target_index + N'%' + AND + ( + EXISTS + ( + SELECT + 1/0 + FROM #index_analysis ia + WHERE ia.database_id = icr.database_id + AND ia.table_name = icr.table_name + AND ia.index_name = icr.index_name + AND ia.included_columns IS NOT NULL + ) + OR icr.action LIKE N'%ADD %' + ) + ) + THEN N' INCLUDE (' + + STUFF + ( + ( + SELECT DISTINCT + N', ' + + col + FROM + ( + -- Get included columns from all source indexes + SELECT DISTINCT + col = LTRIM(RTRIM(value.c.value('.', 'sysname'))) + FROM #index_cleanup_report icr + CROSS APPLY + ( + SELECT + ia.included_columns + FROM #index_analysis ia + WHERE ia.database_id = icr.database_id + AND ia.table_name = icr.table_name + AND ia.index_name = icr.index_name + ) src + CROSS APPLY + ( + SELECT + cols = + CONVERT + ( + xml, + '' + + REPLACE + ( + ISNULL + ( + src.included_columns, + '' + ), + ', ', + '') + + '' + ) + ) x + CROSS APPLY x.cols.nodes('/c') AS value(c) + WHERE icr.database_id = mt.database_id + AND icr.table_name = mt.table_name + AND icr.action LIKE N'MERGE INTO ' + mt.target_index + N'%' + + UNION + + -- Get missing columns which need to be added + SELECT + col = + LTRIM(RTRIM(value.c.value('.', 'sysname'))) + FROM #index_cleanup_report icr + CROSS APPLY + ( + SELECT DISTINCT + missing_cols = + REPLACE(REPLACE( + SUBSTRING + ( + icr.action, + CHARINDEX('ADD ', icr.action) + 4, + LEN(icr.action) + ), + N')', ''), N'(', '') + WHERE icr.action LIKE N'%ADD %' + ) mc + CROSS APPLY + ( + SELECT + cols = + CONVERT + ( + xml, + '' + + REPLACE + ( + ISNULL + ( + mc.missing_cols, + '' + ), + ', ', + '' + ) + '' + ) + ) x + CROSS APPLY x.cols.nodes('/c') AS value(c) + WHERE icr.database_id = mt.database_id + AND icr.table_name = mt.table_name + AND icr.action LIKE N'MERGE INTO ' + mt.target_index + N'%' + AND icr.action LIKE N'%ADD %' + ) AS all_columns + WHERE DATALENGTH(col) > 0 + FOR + XML + PATH(''), + TYPE + ).value('.', 'nvarchar(max)'), + 1, + 2, + '' + ) + + N')' + ELSE N'' + END + + -- Add partitioning if needed + CASE + WHEN EXISTS + ( + SELECT + 1/0 + FROM #partition_stats ps + WHERE ps.database_id = mt.database_id + AND ps.table_name = mt.table_name + AND ps.index_name = mt.target_index + AND ps.partition_function_name IS NOT NULL + ) + THEN + ( + SELECT TOP (1) + N' ON ' + + QUOTENAME(ps.partition_function_name) + + '(' + + ps.partition_columns + + ')' + FROM #partition_stats ps + WHERE ps.database_id = mt.database_id + AND ps.table_name = mt.table_name + AND ps.index_name = mt.target_index + AND ps.partition_function_name IS NOT NULL + ) + ELSE N'' + END + + -- Add filter definition if needed + CASE + WHEN EXISTS + ( + SELECT + 1/0 + FROM #index_analysis ia + WHERE ia.database_id = mt.database_id + AND ia.table_name = mt.table_name + AND ia.index_name = mt.target_index + AND ia.filter_definition IS NOT NULL + ) + THEN + ( + SELECT TOP (1) + N' WHERE ' + + ia.filter_definition + FROM #index_analysis ia + WHERE ia.database_id = mt.database_id + AND ia.table_name = mt.table_name + AND ia.index_name = mt.target_index + AND ia.filter_definition IS NOT NULL + ) + ELSE N'' + END + + -- Add WITH options + N' WITH (DROP_EXISTING = ON, FILLFACTOR = 100, SORT_IN_TEMPDB = ON, ONLINE = ' + + CASE + WHEN @online = 'true' + THEN N'ON' + ELSE N'OFF' + END + + N', DATA_COMPRESSION = PAGE);' + FROM MergeTargets AS mt; + + -- Then add DISABLE statements for all source indexes + INSERT INTO + #final_index_actions + WITH + (TABLOCK) + ( + database_id, + database_name, + schema_id, + schema_name, + object_id, + table_name, + index_id, + index_name, + action, + script + ) + SELECT + icr.database_id, + icr.database_name, + icr.schema_id, + icr.schema_name, + icr.object_id, + icr.table_name, + icr.index_id, + icr.index_name, + action = N'DISABLE MERGED', + script = + N'ALTER INDEX ' + + QUOTENAME(icr.index_name) + + N' ON ' + + QUOTENAME(icr.database_name) + + N'.' + + QUOTENAME(icr.schema_name) + + N'.' + + QUOTENAME(icr.table_name) + + N' DISABLE;' + FROM #index_cleanup_report icr + WHERE icr.action LIKE N'MERGE INTO%'; + + IF ROWCOUNT_BIG() = 0 BEGIN IF @debug = 1 BEGIN RAISERROR('No rows inserted into #final_index_actions', 0, 0) WITH NOWAIT END; END; + + IF @debug = 1 + BEGIN + + SELECT + table_name = '#final_index_actions', + fia.* + FROM #final_index_actions AS fia; + + RAISERROR('Select from #final_index_actions', 0, 0) WITH NOWAIT; + END; + + SELECT + f.database_name, + f.table_name, + f.index_name, + f.action, + f.script, + sort_order = + CASE f.action + WHEN N'MERGE INTO' THEN 2 + WHEN N'DROP' THEN 3 + ELSE 999 + END + FROM #final_index_actions AS f + WHERE f.action <> N'KEEP' + + UNION ALL + + SELECT + r.database_name, + r.table_name, + r.index_name, + action = + N'DISABLE (Unused)', + script = + N'ALTER INDEX ' + + QUOTENAME(r.index_name) + + N' ON ' + + QUOTENAME(r.table_name) + + N' DISABLE;', + sort_order = 1 + FROM #index_cleanup_report AS r + WHERE r.user_seeks = 0 + AND r.user_scans = 0 + AND r.user_lookups = 0 + AND r.user_updates = 0 + ORDER BY + f.table_name, + f.index_name, + sort_order + OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + RAISERROR('Generating scripts', 0, 0) WITH NOWAIT; + END; + + SELECT * FROM #final_index_actions AS fia + + /*Merge into*/ + SELECT + @final_script += + N' + -- ============================================================================= + -- MERGE INDEX: ' + + QUOTENAME(f.index_name) + + N' into ' + + CASE + WHEN f.action = 'MERGE CONSOLIDATED' + THEN QUOTENAME(f.target_index_name) + ELSE 'Unknown Target' + END + + N' + -- Reason: This index overlaps with another index and can be consolidated + -- Original definition: ' + + NCHAR(10) + + ( + SELECT + MAX(ics.current_definition) + FROM #index_cleanup_summary AS ics + WHERE ics.index_name = f.index_name + AND ics.table_name = f.table_name + ) + + N' + -- Usage: Seeks: ' + + CONVERT + ( + nvarchar(20), + ISNULL + ( + ( + SELECT + SUM(DISTINCT id.user_seeks) + FROM #index_details AS id + WHERE id.table_name = f.table_name + AND id.index_name = f.index_name + ), + 0 + ) + ) + + N', Scans: ' + + CONVERT + ( + nvarchar(20), + ISNULL + ( + ( + SELECT + SUM(DISTINCT id.user_scans) + FROM #index_details AS id + WHERE id.table_name = f.table_name + AND id.index_name = f.index_name + ), + 0 + ) + ) + + N', Lookups: ' + + CONVERT + ( + nvarchar(20), + ISNULL + ( + ( + SELECT + SUM(DISTINCT id.user_lookups) + FROM #index_details AS id + WHERE id.table_name = f.table_name + AND id.index_name = f.index_name + ), + 0 + ) + ) + + N', Updates: ' + + CONVERT + ( + nvarchar(20), + ISNULL + ( + ( + SELECT + SUM(DISTINCT id.user_updates) + FROM #index_details AS id + WHERE id.table_name = f.table_name + AND id.index_name = f.index_name + ), + 0 + ) + ) + + N' + -- Space saved: ' + + CONVERT + ( + nvarchar(20), + ISNULL + ( + ( + SELECT + CONVERT + ( + decimal(10,2), + SUM(ps.total_space_mb) / 1024.0 + ) + FROM #partition_stats AS ps + WHERE ps.table_name = f.table_name + AND ps.index_name = f.index_name + ), + 0 + ) + ) + N' GB + -- ============================================================================= + ' + + f.script + + NCHAR(10) + + NCHAR(10) + FROM #final_index_actions AS f + WHERE f.action = N'MERGE CONSOLIDATED' + ORDER BY + f.table_name, + f.index_name; + + /*Disable merged indexes*/ + SELECT + @final_script += N' + /* + -- ============================================================================= + -- DISABLE MERGED INDEX: ' + + QUOTENAME(f.index_name) + + N' + -- Reason: This index has been merged into another index + -- ============================================================================= + */' + + NCHAR(10) + + f.script + + NCHAR(10) + + NCHAR(10) + FROM + ( + -- Use a derived table with DISTINCT to avoid duplicates + SELECT DISTINCT + index_name, + table_name, + script + FROM #final_index_actions + WHERE action = N'DISABLE MERGED' + ) AS f + ORDER BY + f.table_name, + f.index_name; + + /*Drop indexes*/ + SELECT + @final_script += N' + /* + -- ============================================================================= + -- DROP INDEX: ' + + QUOTENAME(f.index_name) + + N' + -- Reason: This index is redundant with other indexes on the same table + -- Current usage: Seeks: ' + + CONVERT + ( + nvarchar(20), + ISNULL + ( + ( + SELECT + SUM(DISTINCT id.user_seeks) + FROM #index_details AS id + WHERE id.table_name = f.table_name + AND id.index_name = f.index_name + ), + 0 + ) + ) + + N', Scans: ' + + CONVERT + ( + nvarchar(20), + ISNULL + ( + ( + SELECT + SUM(DISTINCT id.user_scans) + FROM #index_details AS id + WHERE id.table_name = f.table_name + AND id.index_name = f.index_name + ), + 0 + ) + ) + + N', Lookups: ' + + CONVERT + ( + nvarchar(20), + ISNULL + ( + ( + SELECT + SUM(DISTINCT id.user_lookups) + FROM #index_details AS id + WHERE id.table_name = f.table_name + AND id.index_name = f.index_name + ), + 0 + ) + ) + + N', Updates: ' + + CONVERT + ( + nvarchar(20), + ISNULL + ( + ( + SELECT + SUM(DISTINCT id.user_updates) + FROM #index_details AS id + WHERE id.table_name = f.table_name + AND id.index_name = f.index_name + ), + 0 + ) + ) + + N' + -- Last used: ' + + ISNULL + ( + CONVERT + ( + nvarchar(30), + ( + SELECT + MAX + ( + CASE + WHEN id.last_user_seek > id.last_user_scan + AND id.last_user_seek > id.last_user_lookup + THEN id.last_user_seek + WHEN id.last_user_scan > id.last_user_lookup + THEN id.last_user_scan + ELSE id.last_user_lookup + END + ) + FROM #index_details AS id + WHERE id.table_name = f.table_name + AND id.index_name = f.index_name + ), + 120 + ), + 'Never' + ) + + N' + -- Space reclaimed: ' + + CONVERT + ( + nvarchar(20), + ISNULL + ( + ( + SELECT + CONVERT + ( + decimal(10,2), + SUM(ps.total_space_mb) / 1024.0 + ) + FROM #partition_stats AS ps + WHERE ps.table_name = f.table_name + AND ps.index_name = f.index_name + ), + 0 + ) + ) + N' GB + -- ============================================================================= + */' + + f.script + + NCHAR(10) + + NCHAR(10) + FROM #final_index_actions AS f + WHERE f.action = N'DROP' + ORDER BY + f.table_name, + f.index_name; + + /*Unused indexes*/ + SELECT + @final_script += N' + /* + -- ============================================================================= + -- DISABLE UNUSED INDEX: ' + + QUOTENAME(i.index_name) + + N' + -- Reason: This index has never been used for reads but has been updated ' + + CONVERT + ( + nvarchar(20), + i.user_updates + ) + + N' times + -- Space reclaimed: ' + + CONVERT + ( + nvarchar(20), + ISNULL + ( + ( + SELECT + CONVERT + ( + decimal(10,2), + SUM(ps.total_space_mb) / 1024.0 + ) + FROM #partition_stats AS ps + WHERE ps.table_name = i.table_name + AND ps.index_name = i.index_name + ), + 0 + ) + ) + N' GB + -- Warning: Verify this index is truly not needed before dropping + -- ============================================================================= + */' + + NCHAR(10) + + N'ALTER INDEX ' + + QUOTENAME(i.index_name) + + N' ON ' + + QUOTENAME(i.database_name) + + N'.' + + QUOTENAME + (i.schema_name) + + N'.' + + QUOTENAME(i.table_name) + + N' DISABLE;' + + NCHAR(10) + + NCHAR(10) + FROM + ( + SELECT DISTINCT + icr.database_name, + icr.schema_name, + icr.table_name, + icr.index_name, + icr.user_updates + FROM #index_cleanup_report AS icr + WHERE icr.user_seeks = 0 + AND icr.user_scans = 0 + AND icr.user_lookups = 0 + AND icr.user_updates = 0 + AND icr.action <> N'DROP' + AND icr.action NOT LIKE N'MERGE INTO%' + AND NOT EXISTS + ( + SELECT + 1/0 + FROM #final_index_actions AS fia + WHERE fia.index_name = icr.index_name + AND fia.table_name = icr.table_name + AND fia.action IN (N'MERGE CONSOLIDATED', N'DISABLE MERGED') + ) + ) AS i + ORDER BY + i.table_name, + i.index_name; + + + /*Summary*/ + SELECT + @final_script += N' + -- ============================================================================= + -- SUMMARY OF CHANGES + -- Total indexes analyzed: ' + + CONVERT + ( + nvarchar(10), + ( + SELECT + COUNT_BIG(*) + FROM #index_cleanup_report AS icr + ) + ) + + N' + -- Indexes recommended for dropping: ' + + CONVERT + ( + nvarchar(10), + ( + SELECT + COUNT_BIG(*) + FROM #index_cleanup_report AS icr + WHERE icr.action = 'DROP' + ) + ) + + N' + -- Indexes recommended for merging: ' + + CONVERT + ( + nvarchar(10), + ( + SELECT + COUNT_BIG(*) + FROM #index_cleanup_report AS icr + WHERE icr.action LIKE 'MERGE INTO%' + ) + ) + + N' + -- Unused indexes found: ' + + CONVERT + ( + nvarchar(10), + ( + SELECT + COUNT_BIG(*) + FROM #index_cleanup_report AS icr + WHERE icr.user_seeks = 0 + AND icr.user_scans = 0 + AND icr.user_lookups = 0 + ) + ) + + N' + -- Estimated space savings: ' + + CONVERT + ( + nvarchar(20), + ISNULL + ( + ( + SELECT + CONVERT + ( + decimal(10,2), + SUM(ps.total_space_mb) / 1024.0 + ) + FROM #partition_stats AS ps + JOIN #index_cleanup_report AS icr + ON ps.table_name = icr.table_name + AND ps.index_name = icr.index_name + WHERE icr.action = 'DROP' + OR icr.action LIKE 'MERGE INTO%' + OR + ( + icr.user_seeks = 0 + AND icr.user_scans = 0 + AND icr.user_lookups = 0 + ) + ), + 0 + ) + ) + N' GB + -- Estimated write operations reduced: ' + + CONVERT + ( + nvarchar(20), + ISNULL + ( + ( + SELECT + SUM(icr.user_updates) + FROM #index_cleanup_report AS icr + WHERE icr.action = 'DROP' + OR icr.action LIKE 'MERGE INTO%' + OR + ( + icr.user_seeks = 0 + AND icr.user_scans = 0 + AND icr.user_lookups = 0 + ) + ), + 0 + ) + ) + N' operations + -- ============================================================================= + '; + + SELECT + [text()] = + N'/* Index Cleanup Script for ' + + @database_name + + N' */', + [text()] = + ( + SELECT + NCHAR(10) + + N' ----------------------' + + NCHAR(10) + + N' -- Final script to review. DO NOT EXECUTE WITHOUT CAREFUL REVIEW.' + + NCHAR(10) + + N' -- Implementation Script:' + + NCHAR(10) + + N' ----------------------' + + NCHAR(10) + + @final_script + FOR + XML + PATH(''), + TYPE + ).value('(./text())[1]', 'nvarchar(max)') + FOR + XML + PATH(''), + TYPE; +END TRY +BEGIN CATCH + THROW; +END CATCH; +END; /*Final End*/ +GO From 62b6784156a35a6a3a74af7868c384a9675fc15b Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Wed, 12 Mar 2025 12:16:24 -0400 Subject: [PATCH 025/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 110 +++++++++--------- 1 file changed, 56 insertions(+), 54 deletions(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index 9dd0deb4..92b5a2f5 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -1349,66 +1349,68 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND ia1.key_columns = ia2.key_columns /* Exact key match */ AND ISNULL(ia1.included_columns, '') = ISNULL(ia2.included_columns, '') /* Exact includes match */ AND ISNULL(ia1.filter_definition, '') = ISNULL(ia2.filter_definition, '') /* Matching filters */ - WHERE - ia1.consolidation_rule IS NULL /* Not already processed */ - AND ia2.consolidation_rule IS NULL /* Not already processed */ - AND ia1.is_eligible_for_dedupe = 1 - AND ia2.is_eligible_for_dedupe = 1; - -/* Rule 3: Key duplicates (matching key columns, different includes) */ -UPDATE ia1 -SET - ia1.consolidation_rule = 'Key Duplicate', - ia1.target_index_name = - CASE - /* If one is unique and the other isn't, prefer the unique one */ - WHEN ia1.is_unique = 1 AND ia2.is_unique = 0 THEN NULL - WHEN ia1.is_unique = 0 AND ia2.is_unique = 1 THEN ia2.index_name - /* Otherwise use priority */ - WHEN ia1.index_priority >= ia2.index_priority THEN NULL - ELSE ia2.index_name - END, - ia1.action = - CASE - WHEN (ia1.is_unique = 1 AND ia2.is_unique = 0) OR - (ia1.index_priority >= ia2.index_priority AND NOT (ia1.is_unique = 0 AND ia2.is_unique = 1)) - THEN 'MERGE INCLUDES' /* Keep this index but merge includes */ - ELSE 'DISABLE' /* Other index is keeper, disable this one */ - END -FROM #index_analysis ia1 -JOIN #index_analysis ia2 ON - ia1.database_id = ia2.database_id - AND ia1.object_id = ia2.object_id - AND ia1.index_name <> ia2.index_name - AND ia1.key_columns = ia2.key_columns /* Exact key match */ - AND ISNULL(ia1.included_columns, '') <> ISNULL(ia2.included_columns, '') /* Different includes */ - AND ISNULL(ia1.filter_definition, '') = ISNULL(ia2.filter_definition, '') /* Matching filters */ -WHERE - ia1.consolidation_rule IS NULL /* Not already processed */ - AND ia2.consolidation_rule IS NULL /* Not already processed */ - AND ia1.is_eligible_for_dedupe = 1 - AND ia2.is_eligible_for_dedupe = 1; - -/* Rule 4: Superset/subset key columns */ -UPDATE ia1 -SET - ia1.consolidation_rule = 'Key Subset', - ia1.target_index_name = ia2.index_name, - ia1.action = 'DISABLE' /* The narrower index gets disabled */ -FROM #index_analysis ia1 -JOIN #index_analysis ia2 ON - ia1.database_id = ia2.database_id + WHERE ia1.consolidation_rule IS NULL /* Not already processed */ + AND ia2.consolidation_rule IS NULL /* Not already processed */ + AND ia1.is_eligible_for_dedupe = 1 + AND ia2.is_eligible_for_dedupe = 1; + + /* Rule 3: Key duplicates - matching key columns, different includes */ + UPDATE + ia1 + SET + ia1.consolidation_rule = 'Key Duplicate', + ia1.target_index_name = + CASE + /* If one is unique and the other isn't, prefer the unique one */ + WHEN ia1.is_unique = 1 AND ia2.is_unique = 0 + THEN NULL + WHEN ia1.is_unique = 0 AND ia2.is_unique = 1 + THEN ia2.index_name + /* Otherwise use priority */ + WHEN ia1.index_priority >= ia2.index_priority + THEN NULL + ELSE ia2.index_name + END, + ia1.action = + CASE + WHEN (ia1.is_unique = 1 AND ia2.is_unique = 0) OR + (ia1.index_priority >= ia2.index_priority AND NOT (ia1.is_unique = 0 AND ia2.is_unique = 1)) + THEN 'MERGE INCLUDES' /* Keep this index but merge includes */ + ELSE 'DISABLE' /* Other index is keeper, disable this one */ + END + FROM #index_analysis ia1 + JOIN #index_analysis ia2 + ON ia1.database_id = ia2.database_id + AND ia1.object_id = ia2.object_id + AND ia1.index_name <> ia2.index_name + AND ia1.key_columns = ia2.key_columns /* Exact key match */ + AND ISNULL(ia1.included_columns, '') <> ISNULL(ia2.included_columns, '') /* Different includes */ + AND ISNULL(ia1.filter_definition, '') = ISNULL(ia2.filter_definition, '') /* Matching filters */ + WHERE ia1.consolidation_rule IS NULL /* Not already processed */ + AND ia2.consolidation_rule IS NULL /* Not already processed */ + AND ia1.is_eligible_for_dedupe = 1 + AND ia2.is_eligible_for_dedupe = 1; + + /* Rule 4: Superset/subset key columns */ + UPDATE + ia1 + SET + ia1.consolidation_rule = 'Key Subset', + ia1.target_index_name = ia2.index_name, + ia1.action = 'DISABLE' /* The narrower index gets disabled */ + FROM #index_analysis ia1 + JOIN #index_analysis ia2 + ON ia1.database_id = ia2.database_id AND ia1.object_id = ia2.object_id AND ia1.index_name <> ia2.index_name AND ia2.key_columns LIKE (ia1.key_columns + '%') /* ia2 has wider key that starts with ia1's key */ AND ISNULL(ia1.filter_definition, '') = ISNULL(ia2.filter_definition, '') /* Matching filters */ /* Exception: If narrower index is unique and wider is not, they should not be merged */ AND NOT (ia1.is_unique = 1 AND ia2.is_unique = 0) -WHERE - ia1.consolidation_rule IS NULL /* Not already processed */ - AND ia2.consolidation_rule IS NULL /* Not already processed */ - AND ia1.is_eligible_for_dedupe = 1 - AND ia2.is_eligible_for_dedupe = 1; + WHERE ia1.consolidation_rule IS NULL /* Not already processed */ + AND ia2.consolidation_rule IS NULL /* Not already processed */ + AND ia1.is_eligible_for_dedupe = 1 + AND ia2.is_eligible_for_dedupe = 1; IF @debug = 1 From 4c5dd023819f1362b48f102631cf5256b7bf9c78 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Wed, 12 Mar 2025 12:21:22 -0400 Subject: [PATCH 026/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index 92b5a2f5..e9db3482 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -351,6 +351,30 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. (database_id, schema_id, object_id, index_id) ); + CREATE TABLE + #partition_stats + ( + database_id integer NOT NULL, + database_name sysname NOT NULL, + schema_id integer NOT NULL, + schema_name sysname NOT NULL, + object_id integer NOT NULL, + table_name sysname NOT NULL, + index_id integer NOT NULL, + index_name sysname NULL, + partition_id bigint NOT NULL, + partition_number int NOT NULL, + total_rows bigint NULL, + total_space_mb decimal(38, 2) NULL, + reserved_lob_mb decimal(38, 2) NULL, + reserved_row_overflow_mb decimal(38, 2) NULL, + data_compression_desc nvarchar(60) NULL, + built_on sysname NULL, + partition_function_name sysname NULL, + partition_columns nvarchar(max) + PRIMARY KEY CLUSTERED(database_id, object_id, index_id, partition_id) + ); + CREATE TABLE #index_details ( @@ -387,30 +411,6 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. PRIMARY KEY CLUSTERED(database_id, object_id, index_id, column_name) ); - CREATE TABLE - #partition_stats - ( - database_id integer NOT NULL, - database_name sysname NOT NULL, - schema_id integer NOT NULL, - schema_name sysname NOT NULL, - object_id integer NOT NULL, - table_name sysname NOT NULL, - index_id integer NOT NULL, - index_name sysname NULL, - partition_id bigint NOT NULL, - partition_number int NOT NULL, - total_rows bigint NULL, - total_space_mb decimal(38, 2) NULL, - reserved_lob_mb decimal(38, 2) NULL, - reserved_row_overflow_mb decimal(38, 2) NULL, - data_compression_desc nvarchar(60) NULL, - built_on sysname NULL, - partition_function_name sysname NULL, - partition_columns nvarchar(max) - PRIMARY KEY CLUSTERED(database_id, object_id, index_id, partition_id) - ); - CREATE TABLE #index_analysis ( From 788f6bae8fc4bf5f8b720d367db985845b77eca6 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Wed, 12 Mar 2025 12:23:16 -0400 Subject: [PATCH 027/246] Update sp_IndexCleanup BETA.sql fix updates --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 54 ++++++++++++++++--- 1 file changed, 48 insertions(+), 6 deletions(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index 92b5a2f5..f05fb8bc 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -1351,8 +1351,22 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND ISNULL(ia1.filter_definition, '') = ISNULL(ia2.filter_definition, '') /* Matching filters */ WHERE ia1.consolidation_rule IS NULL /* Not already processed */ AND ia2.consolidation_rule IS NULL /* Not already processed */ - AND ia1.is_eligible_for_dedupe = 1 - AND ia2.is_eligible_for_dedupe = 1; + AND EXISTS ( + SELECT 1/0 + FROM #index_details id1 + WHERE id1.database_id = ia1.database_id + AND id1.object_id = ia1.object_id + AND id1.index_name = ia1.index_name + AND id1.is_eligible_for_dedupe = 1 + ) + AND EXISTS ( + SELECT 1/0 + FROM #index_details id2 + WHERE id2.database_id = ia2.database_id + AND id2.object_id = ia2.object_id + AND id2.index_name = ia2.index_name + AND id2.is_eligible_for_dedupe = 1 + ); /* Rule 3: Key duplicates - matching key columns, different includes */ UPDATE @@ -1388,8 +1402,22 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND ISNULL(ia1.filter_definition, '') = ISNULL(ia2.filter_definition, '') /* Matching filters */ WHERE ia1.consolidation_rule IS NULL /* Not already processed */ AND ia2.consolidation_rule IS NULL /* Not already processed */ - AND ia1.is_eligible_for_dedupe = 1 - AND ia2.is_eligible_for_dedupe = 1; + AND EXISTS ( + SELECT 1/0 + FROM #index_details id1 + WHERE id1.database_id = ia1.database_id + AND id1.object_id = ia1.object_id + AND id1.index_name = ia1.index_name + AND id1.is_eligible_for_dedupe = 1 + ) + AND EXISTS ( + SELECT 1/0 + FROM #index_details id2 + WHERE id2.database_id = ia2.database_id + AND id2.object_id = ia2.object_id + AND id2.index_name = ia2.index_name + AND id2.is_eligible_for_dedupe = 1 + ); /* Rule 4: Superset/subset key columns */ UPDATE @@ -1409,8 +1437,22 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND NOT (ia1.is_unique = 1 AND ia2.is_unique = 0) WHERE ia1.consolidation_rule IS NULL /* Not already processed */ AND ia2.consolidation_rule IS NULL /* Not already processed */ - AND ia1.is_eligible_for_dedupe = 1 - AND ia2.is_eligible_for_dedupe = 1; + AND EXISTS ( + SELECT 1/0 + FROM #index_details id1 + WHERE id1.database_id = ia1.database_id + AND id1.object_id = ia1.object_id + AND id1.index_name = ia1.index_name + AND id1.is_eligible_for_dedupe = 1 + ) + AND EXISTS ( + SELECT 1/0 + FROM #index_details id2 + WHERE id2.database_id = ia2.database_id + AND id2.object_id = ia2.object_id + AND id2.index_name = ia2.index_name + AND id2.is_eligible_for_dedupe = 1 + ); IF @debug = 1 From 16eb1153b96492db4a3d80b029133d27e1a703d4 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Wed, 12 Mar 2025 12:26:59 -0400 Subject: [PATCH 028/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 72 +++++++++++-------- 1 file changed, 42 insertions(+), 30 deletions(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index a16d8f19..8794fc31 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -1351,21 +1351,25 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND ISNULL(ia1.filter_definition, '') = ISNULL(ia2.filter_definition, '') /* Matching filters */ WHERE ia1.consolidation_rule IS NULL /* Not already processed */ AND ia2.consolidation_rule IS NULL /* Not already processed */ - AND EXISTS ( - SELECT 1/0 + AND EXISTS + ( + SELECT + 1/0 FROM #index_details id1 WHERE id1.database_id = ia1.database_id - AND id1.object_id = ia1.object_id - AND id1.index_name = ia1.index_name - AND id1.is_eligible_for_dedupe = 1 + AND id1.object_id = ia1.object_id + AND id1.index_name = ia1.index_name + AND id1.is_eligible_for_dedupe = 1 ) - AND EXISTS ( - SELECT 1/0 + AND EXISTS + ( + SELECT + 1/0 FROM #index_details id2 WHERE id2.database_id = ia2.database_id - AND id2.object_id = ia2.object_id - AND id2.index_name = ia2.index_name - AND id2.is_eligible_for_dedupe = 1 + AND id2.object_id = ia2.object_id + AND id2.index_name = ia2.index_name + AND id2.is_eligible_for_dedupe = 1 ); /* Rule 3: Key duplicates - matching key columns, different includes */ @@ -1402,21 +1406,25 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND ISNULL(ia1.filter_definition, '') = ISNULL(ia2.filter_definition, '') /* Matching filters */ WHERE ia1.consolidation_rule IS NULL /* Not already processed */ AND ia2.consolidation_rule IS NULL /* Not already processed */ - AND EXISTS ( - SELECT 1/0 + AND EXISTS + ( + SELECT + 1/0 FROM #index_details id1 WHERE id1.database_id = ia1.database_id - AND id1.object_id = ia1.object_id - AND id1.index_name = ia1.index_name - AND id1.is_eligible_for_dedupe = 1 + AND id1.object_id = ia1.object_id + AND id1.index_name = ia1.index_name + AND id1.is_eligible_for_dedupe = 1 ) - AND EXISTS ( - SELECT 1/0 + AND EXISTS + ( + SELECT + 1/0 FROM #index_details id2 WHERE id2.database_id = ia2.database_id - AND id2.object_id = ia2.object_id - AND id2.index_name = ia2.index_name - AND id2.is_eligible_for_dedupe = 1 + AND id2.object_id = ia2.object_id + AND id2.index_name = ia2.index_name + AND id2.is_eligible_for_dedupe = 1 ); /* Rule 4: Superset/subset key columns */ @@ -1437,21 +1445,25 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND NOT (ia1.is_unique = 1 AND ia2.is_unique = 0) WHERE ia1.consolidation_rule IS NULL /* Not already processed */ AND ia2.consolidation_rule IS NULL /* Not already processed */ - AND EXISTS ( - SELECT 1/0 + AND EXISTS + ( + SELECT + 1/0 FROM #index_details id1 WHERE id1.database_id = ia1.database_id - AND id1.object_id = ia1.object_id - AND id1.index_name = ia1.index_name - AND id1.is_eligible_for_dedupe = 1 + AND id1.object_id = ia1.object_id + AND id1.index_name = ia1.index_name + AND id1.is_eligible_for_dedupe = 1 ) - AND EXISTS ( - SELECT 1/0 + AND EXISTS + ( + SELECT + 1/0 FROM #index_details id2 WHERE id2.database_id = ia2.database_id - AND id2.object_id = ia2.object_id - AND id2.index_name = ia2.index_name - AND id2.is_eligible_for_dedupe = 1 + AND id2.object_id = ia2.object_id + AND id2.index_name = ia2.index_name + AND id2.is_eligible_for_dedupe = 1 ); From 57d58198fe1746383144e2dbfdd909c0a1f2c11a Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Wed, 12 Mar 2025 13:20:20 -0400 Subject: [PATCH 029/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 421 ++++++++++++++++-- 1 file changed, 385 insertions(+), 36 deletions(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index 8794fc31..034e26e1 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -170,6 +170,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. @database_id integer = NULL, @object_id integer = NULL, @full_object_name nvarchar(768) = NULL, + @uptime_warning bit = 0, /* Will set after @uptime_days is calculated */ /*print variables*/ @online bit = CASE @@ -182,6 +183,15 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. THEN 'true' /* Enterprise, Azure SQL DB, Managed Instance */ ELSE 'false' END, + /* Compression variables */ + @can_compress bit = + CASE + WHEN CONVERT(int, SERVERPROPERTY('EngineEdition')) IN (3, 5, 8) + OR (CONVERT(int, SERVERPROPERTY('EngineEdition')) = 2 + AND CONVERT(int, SUBSTRING(CONVERT(varchar(20), SERVERPROPERTY('ProductVersion')), 1, 2)) >= 13) + THEN 1 + ELSE 0 + END, @uptime_days nvarchar(10) = ( SELECT @@ -193,6 +203,15 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ) FROM sys.dm_os_sys_info AS osi ); + + /* Set uptime warning flag after @uptime_days is calculated */ + SELECT + @uptime_warning = + CASE + WHEN CONVERT(integer, @uptime_days) < 14 + THEN 1 + ELSE 0 + END; /* Initial checks for object validity @@ -440,20 +459,32 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CREATE TABLE #index_consolidation ( - database_id int NOT NULL, + database_id integer NOT NULL, database_name sysname NOT NULL, - schema_id int NOT NULL, + schema_id integer NOT NULL, schema_name sysname NOT NULL, - object_id int NOT NULL, + object_id integer NOT NULL, table_name sysname NOT NULL, - index_id int NOT NULL, + index_id integer NOT NULL, index_name sysname NOT NULL, target_index_name sysname NULL, consolidation_rule varchar(50) NULL, - index_priority int NULL, + index_priority integer NULL, action varchar(50) NULL, PRIMARY KEY (database_id, object_id, index_id) ); + + CREATE TABLE + #compression_eligibility + ( + database_id integer NOT NULL, + schema_id integer NOT NULL, + object_id integer NOT NULL, + table_name sysname NOT NULL, + can_compress bit NOT NULL, + reason nvarchar(200) NULL, + PRIMARY KEY (database_id, object_id) + ); /* Start insert queries @@ -581,6 +612,74 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. fo.* FROM #filtered_objects AS fo; END; + + /* Populate compression eligibility table */ + IF @debug = 1 + BEGIN + RAISERROR('Populating #compression_eligibility', 0, 0) WITH NOWAIT; + END; + + INSERT INTO #compression_eligibility + ( + database_id, + schema_id, + object_id, + table_name, + can_compress, + reason + ) + SELECT DISTINCT + database_id, + schema_id, + object_id, + table_name, + 1, /* Default to compressible */ + NULL + FROM #filtered_objects; + + /* If SQL Server edition doesn't support compression, mark all as ineligible */ + IF @can_compress = 0 + BEGIN + UPDATE #compression_eligibility + SET + can_compress = 0, + reason = 'SQL Server edition or version does not support compression' + WHERE can_compress = 1; + END; + + /* Check for sparse columns or incompatible data types */ + IF @can_compress = 1 + BEGIN + SELECT + @sql = N' + SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; + + UPDATE ce + SET + can_compress = 0, + reason = ''Table contains sparse columns or incompatible data types'' + FROM #compression_eligibility ce + WHERE EXISTS + ( + SELECT 1/0 + FROM ' + QUOTENAME(@database_name) + N'.sys.columns c + JOIN ' + QUOTENAME(@database_name) + N'.sys.types t + ON c.user_type_id = t.user_type_id + WHERE c.object_id = ce.object_id + AND (c.is_sparse = 1 OR t.name IN (N''text'', N''ntext'', N''image'')) + ); + '; + + EXEC sys.sp_executesql @sql; + END; + + IF @debug = 1 + BEGIN + SELECT + table_name = '#compression_eligibility', + ce.* + FROM #compression_eligibility AS ce; + END; IF @debug = 1 BEGIN @@ -1305,7 +1404,12 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. UPDATE #index_analysis SET - consolidation_rule = 'Unused Index', + consolidation_rule = + CASE + WHEN @uptime_warning = 1 + THEN 'Unused Index (WARNING: Server uptime < 14 days)' + ELSE 'Unused Index' + END, action = 'DISABLE' WHERE EXISTS ( @@ -1465,6 +1569,60 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND id2.index_name = ia2.index_name AND id2.is_eligible_for_dedupe = 1 ); + + /* Rule 5: Unique constraint vs. nonclustered index handling */ + UPDATE + ia1 + SET + ia1.consolidation_rule = 'Unique Constraint Replacement', + ia1.action = + CASE + WHEN ia1.is_unique = 0 + THEN 'MAKE UNIQUE' /* Convert to unique index */ + ELSE 'KEEP' /* Already unique, so just keep it */ + END + FROM #index_analysis ia1 + WHERE ia1.consolidation_rule IS NULL /* Not already processed */ + AND EXISTS + ( + /* Find nonclustered indexes */ + SELECT + 1/0 + FROM #index_details id1 + WHERE id1.database_id = ia1.database_id + AND id1.object_id = ia1.object_id + AND id1.index_name = ia1.index_name + AND id1.is_eligible_for_dedupe = 1 + ) + AND EXISTS + ( + /* Find unique constraints with matching key columns */ + SELECT + 1/0 + FROM #index_details id2 + WHERE id2.database_id = ia1.database_id + AND id2.object_id = ia1.object_id + AND id2.is_unique_constraint = 1 + AND NOT EXISTS + ( + /* Verify key columns match between index and unique constraint */ + SELECT + id2_inner.column_name + FROM #index_details id2_inner + WHERE id2_inner.database_id = id2.database_id + AND id2_inner.object_id = id2.object_id + AND id2_inner.index_id = id2.index_id + AND id2_inner.is_included_column = 0 + EXCEPT + SELECT + id1_inner.column_name + FROM #index_details id1_inner + WHERE id1_inner.database_id = ia1.database_id + AND id1_inner.object_id = ia1.object_id + AND id1_inner.index_name = ia1.index_name + AND id1_inner.is_included_column = 0 + ) + ); IF @debug = 1 @@ -1482,43 +1640,234 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Generate index merge scripts with compression and drop_existing */ SELECT - database_name, - schema_name, - table_name, - index_name, - target_index_name, - consolidation_rule, + ia.database_name, + ia.schema_name, + ia.table_name, + ia.index_name, + ia.target_index_name, + ia.consolidation_rule, merge_script = - 'CREATE INDEX ' + QUOTENAME(index_name) + - ' ON ' + QUOTENAME(database_name) + '.' + QUOTENAME(schema_name) + '.' + QUOTENAME(table_name) + - ' (' + key_columns + ')' + - CASE WHEN included_columns IS NOT NULL AND LEN(included_columns) > 0 - THEN ' INCLUDE (' + included_columns + ')' - ELSE '' + N'CREATE ' + + CASE WHEN ia.action = 'MAKE UNIQUE' THEN N'UNIQUE ' ELSE N'' END + + N'INDEX ' + + QUOTENAME(ia.index_name) + + N' ON ' + + QUOTENAME(ia.database_name) + + N'.' + + QUOTENAME(ia.schema_name) + + N'.' + + QUOTENAME(ia.table_name) + + N' (' + + ia.key_columns + + N')' + + CASE + WHEN ia.included_columns IS NOT NULL AND LEN(ia.included_columns) > 0 + THEN N' INCLUDE (' + + ia.included_columns + + N')' + ELSE N'' END + - CASE WHEN filter_definition IS NOT NULL - THEN ' WHERE ' + filter_definition - ELSE '' + CASE + WHEN ia.filter_definition IS NOT NULL + THEN N' WHERE ' + + ia.filter_definition + ELSE N'' END + - ' WITH (DROP_EXISTING = ON, FILLFACTOR = 100, SORT_IN_TEMPDB = ON, ONLINE = ON, DATA_COMPRESSION = PAGE);' -FROM #index_analysis -WHERE action = 'MERGE INCLUDES' -ORDER BY table_name, index_name; + CASE + WHEN ps.partition_function_name IS NOT NULL + THEN N' ON ' + + QUOTENAME(ps.partition_function_name) + + N'(' + + ISNULL(ps.partition_columns, N'') + + N')' + WHEN ps.built_on IS NOT NULL + THEN N' ON ' + + QUOTENAME(ps.built_on) + ELSE N'' + END + + N' WITH (DROP_EXISTING = ON, FILLFACTOR = 100, SORT_IN_TEMPDB = ON, ONLINE = ' + + CASE WHEN @online = 1 THEN N'ON' ELSE N'OFF' END + + N', DATA_COMPRESSION = PAGE);' +FROM #index_analysis ia +LEFT JOIN +( + /* Get the partition info for each index */ + SELECT + ps.database_id, + ps.object_id, + ps.index_id, + ps.built_on, + ps.partition_function_name, + ps.partition_columns + FROM #partition_stats ps + GROUP BY + ps.database_id, + ps.object_id, + ps.index_id, + ps.built_on, + ps.partition_function_name, + ps.partition_columns +) ps ON + ia.database_id = ps.database_id AND + ia.object_id = ps.object_id AND + ia.index_id = ps.index_id +JOIN #compression_eligibility ce ON + ia.database_id = ce.database_id AND + ia.object_id = ce.object_id +WHERE ia.action IN ('MERGE INCLUDES', 'MAKE UNIQUE') +AND ce.can_compress = 1 +ORDER BY + ia.table_name, + ia.index_name; /* Generate disable scripts for unneeded indexes */ SELECT - database_name, - schema_name, - table_name, - index_name, - consolidation_rule, + ia.database_name, + ia.schema_name, + ia.table_name, + ia.index_name, + ia.consolidation_rule, disable_script = - 'ALTER INDEX ' + QUOTENAME(index_name) + - ' ON ' + QUOTENAME(database_name) + '.' + QUOTENAME(schema_name) + '.' + QUOTENAME(table_name) + - ' DISABLE;' -FROM #index_analysis -WHERE action = 'DISABLE' -ORDER BY table_name, index_name; + N'ALTER INDEX ' + + QUOTENAME(ia.index_name) + + N' ON ' + + QUOTENAME(ia.database_name) + + N'.' + + QUOTENAME(ia.schema_name) + + N'.' + + QUOTENAME(ia.table_name) + + N' DISABLE;' +FROM #index_analysis ia +WHERE ia.action = 'DISABLE' +ORDER BY + ia.table_name, + ia.index_name; + +/* Generate compression scripts for remaining indexes */ +SELECT + ia.database_name, + ia.schema_name, + ia.table_name, + ia.index_name, + compression_script = + N'ALTER INDEX ' + + QUOTENAME(ia.index_name) + + N' ON ' + + QUOTENAME(ia.database_name) + + N'.' + + QUOTENAME(ia.schema_name) + + N'.' + + QUOTENAME(ia.table_name) + + CASE + WHEN ps.partition_function_name IS NOT NULL + THEN N' REBUILD PARTITION = ALL' + ELSE N' REBUILD' + END + + N' WITH (FILLFACTOR = 100, SORT_IN_TEMPDB = ON, ONLINE = ' + + CASE WHEN @online = 1 THEN N'ON' ELSE N'OFF' END + + N', DATA_COMPRESSION = PAGE);' +FROM #index_analysis ia +LEFT JOIN +( + /* Get the partition info for each index */ + SELECT + ps.database_id, + ps.object_id, + ps.index_id, + ps.built_on, + ps.partition_function_name, + ps.partition_columns + FROM #partition_stats ps + GROUP BY + ps.database_id, + ps.object_id, + ps.index_id, + ps.built_on, + ps.partition_function_name, + ps.partition_columns +) ps ON + ia.database_id = ps.database_id AND + ia.object_id = ps.object_id AND + ia.index_id = ps.index_id +JOIN #compression_eligibility ce ON + ia.database_id = ce.database_id AND + ia.object_id = ce.object_id +WHERE + /* Indexes that are not being disabled or merged */ + ia.action IS NULL OR ia.action = 'KEEP' + /* Only indexes eligible for compression */ + AND ce.can_compress = 1 +ORDER BY + ia.table_name, + ia.index_name; + +/* Generate index statistics and reporting */ +SELECT + report_title = N'Index Cleanup Summary', + tables_analyzed = COUNT(DISTINCT CONCAT(ia.database_id, N'.', ia.schema_id, N'.', ia.object_id)), + total_indexes = COUNT(*), + indexes_to_disable = SUM(CASE WHEN ia.action = 'DISABLE' THEN 1 ELSE 0 END), + indexes_to_merge = SUM(CASE WHEN ia.action IN ('MERGE INCLUDES', 'MAKE UNIQUE') THEN 1 ELSE 0 END), + avg_indexes_per_table = CONVERT(DECIMAL(10,2), COUNT(*) * 1.0 / NULLIF(COUNT(DISTINCT CONCAT(ia.database_id, N'.', ia.schema_id, N'.', ia.object_id)), 0)) +FROM #index_analysis ia; + +/* Generate estimated space savings report */ +SELECT + report_title = N'Estimated Space Savings', + space_saved_from_cleanup_mb = + SUM(CASE + WHEN ia.action IN ('DISABLE', 'MERGE INCLUDES', 'MAKE UNIQUE') + THEN ps.total_space_mb + ELSE 0 + END), + estimated_min_compression_savings_mb = + SUM(CASE + WHEN (ia.action IS NULL OR ia.action = 'KEEP') AND ce.can_compress = 1 + THEN ps.total_space_mb * 0.20 /* Conservative estimate - 20% compression ratio */ + ELSE 0 + END), + estimated_max_compression_savings_mb = + SUM(CASE + WHEN (ia.action IS NULL OR ia.action = 'KEEP') AND ce.can_compress = 1 + THEN ps.total_space_mb * 0.60 /* Optimistic estimate - 60% compression ratio */ + ELSE 0 + END), + total_min_estimated_savings_mb = + SUM(CASE + WHEN ia.action IN ('DISABLE', 'MERGE INCLUDES', 'MAKE UNIQUE') + THEN ps.total_space_mb + WHEN (ia.action IS NULL OR ia.action = 'KEEP') AND ce.can_compress = 1 + THEN ps.total_space_mb * 0.20 + ELSE 0 + END), + total_max_estimated_savings_mb = + SUM(CASE + WHEN ia.action IN ('DISABLE', 'MERGE INCLUDES', 'MAKE UNIQUE') + THEN ps.total_space_mb + WHEN (ia.action IS NULL OR ia.action = 'KEEP') AND ce.can_compress = 1 + THEN ps.total_space_mb * 0.60 + ELSE 0 + END) +FROM #index_analysis ia +LEFT JOIN #partition_stats ps ON + ia.database_id = ps.database_id AND + ia.object_id = ps.object_id AND + ia.index_id = ps.index_id +LEFT JOIN #compression_eligibility ce ON + ia.database_id = ce.database_id AND + ia.object_id = ce.object_id; + +/* Report on tables that can't be compressed */ +SELECT + database_name = ce.database_name, + table_name = ce.table_name, + compression_ineligibility_reason = ce.reason +FROM #compression_eligibility ce +WHERE ce.can_compress = 0 +ORDER BY + ce.database_name, + ce.table_name; + END TRY BEGIN CATCH THROW; From 94b85a4a930940bc0b9e4a8cfdd9ab1d644fc693 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Wed, 12 Mar 2025 13:36:54 -0400 Subject: [PATCH 030/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index 034e26e1..e868ecde 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -52,7 +52,7 @@ BEGIN TRY SELECT warning = N'Read the messages pane carefully!' - PRINT ' + PRINT N' ------------------------------------------------------------------------------------------- ------------------------------------------------------------------------------------------- ------------------------------------------------------------------------------------------- @@ -643,7 +643,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. UPDATE #compression_eligibility SET can_compress = 0, - reason = 'SQL Server edition or version does not support compression' + reason = N'SQL Server edition or version does not support compression' WHERE can_compress = 1; END; From 93f9cbae141b85ce7c95ab7d2cfa210fb33c1a58 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Wed, 12 Mar 2025 13:46:00 -0400 Subject: [PATCH 031/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 107 +++++++++++++++++- 1 file changed, 106 insertions(+), 1 deletion(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index e868ecde..577bcadb 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -1407,7 +1407,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. consolidation_rule = CASE WHEN @uptime_warning = 1 - THEN 'Unused Index (WARNING: Server uptime < 14 days)' + THEN 'Unused Index (WARNING: Server uptime < 14 days - usage data may be incomplete)' ELSE 'Unused Index' END, action = 'DISABLE' @@ -1749,6 +1749,7 @@ SELECT ia.schema_name, ia.table_name, ia.index_name, + compression_type = N'All Partitions', compression_script = N'ALTER INDEX ' + QUOTENAME(ia.index_name) + @@ -1801,9 +1802,113 @@ ORDER BY ia.table_name, ia.index_name; +/* Generate scripts to disable unique constraints that are being replaced by unique indexes */ +SELECT + ia.database_name, + ia.schema_name, + ia.table_name, + ia.index_name, + constraint_name = id.index_name, + disable_constraint_script = + N'ALTER TABLE ' + + QUOTENAME(ia.database_name) + + N'.' + + QUOTENAME(ia.schema_name) + + N'.' + + QUOTENAME(ia.table_name) + + N' NOCHECK CONSTRAINT ' + + QUOTENAME(id.index_name) + + N';' +FROM #index_analysis ia +JOIN #index_details id ON + id.database_id = ia.database_id AND + id.object_id = ia.object_id AND + id.is_unique_constraint = 1 +WHERE + /* Only indexes that are being made unique */ + ia.action = 'MAKE UNIQUE' + /* Find the constraint that matches the index being made unique */ + AND EXISTS ( + SELECT 1/0 + FROM #index_details id_nc + WHERE id_nc.database_id = ia.database_id + AND id_nc.object_id = ia.object_id + AND id_nc.index_name = ia.index_name + /* Matching key columns */ + AND NOT EXISTS ( + SELECT id.column_name + FROM #index_details id_inner + WHERE id_inner.database_id = id.database_id + AND id_inner.object_id = id.object_id + AND id_inner.index_id = id.index_id + AND id_inner.is_included_column = 0 + EXCEPT + SELECT id_nc_inner.column_name + FROM #index_details id_nc_inner + WHERE id_nc_inner.database_id = id_nc.database_id + AND id_nc_inner.object_id = id_nc.object_id + AND id_nc_inner.index_name = id_nc.index_name + AND id_nc_inner.is_included_column = 0 + ) + ) +ORDER BY + ia.table_name, + ia.index_name; + +/* Generate per-partition compression scripts for partitioned indexes */ +SELECT + ia.database_name, + ia.schema_name, + ia.table_name, + ia.index_name, + compression_type = N'Per Partition', + partition_number = ps.partition_number, + total_rows = ps.total_rows, + total_space_mb = ps.total_space_mb, + compression_script = + N'ALTER INDEX ' + + QUOTENAME(ia.index_name) + + N' ON ' + + QUOTENAME(ia.database_name) + + N'.' + + QUOTENAME(ia.schema_name) + + N'.' + + QUOTENAME(ia.table_name) + + N' REBUILD PARTITION = ' + + CAST(ps.partition_number AS nvarchar(20)) + + N' WITH (FILLFACTOR = 100, SORT_IN_TEMPDB = ON, ONLINE = ' + + CASE WHEN @online = 1 THEN N'ON' ELSE N'OFF' END + + N', DATA_COMPRESSION = PAGE);' +FROM #index_analysis ia +JOIN #partition_stats ps ON + ia.database_id = ps.database_id AND + ia.object_id = ps.object_id AND + ia.index_id = ps.index_id +JOIN #compression_eligibility ce ON + ia.database_id = ce.database_id AND + ia.object_id = ce.object_id +WHERE + /* Only partitioned indexes */ + ps.partition_function_name IS NOT NULL + /* Indexes that are not being disabled or merged */ + AND (ia.action IS NULL OR ia.action = 'KEEP') + /* Only indexes eligible for compression */ + AND ce.can_compress = 1 +ORDER BY + ia.table_name, + ia.index_name, + ps.partition_number; + /* Generate index statistics and reporting */ SELECT report_title = N'Index Cleanup Summary', + server_uptime_days = @uptime_days, + uptime_warning = + CASE + WHEN @uptime_warning = 1 + THEN N'Low uptime detected! Index usage data may be incomplete.' + ELSE NULL + END, tables_analyzed = COUNT(DISTINCT CONCAT(ia.database_id, N'.', ia.schema_id, N'.', ia.object_id)), total_indexes = COUNT(*), indexes_to_disable = SUM(CASE WHEN ia.action = 'DISABLE' THEN 1 ELSE 0 END), From 799fb4742895d968ea9ae059b10435b5b587c116 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Wed, 12 Mar 2025 13:59:54 -0400 Subject: [PATCH 032/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 72 ++++++++++++------- 1 file changed, 46 insertions(+), 26 deletions(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index 577bcadb..6057d2ca 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -478,12 +478,16 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #compression_eligibility ( database_id integer NOT NULL, + database_name sysname NOT NULL, schema_id integer NOT NULL, + schema_name sysname NOT NULL, object_id integer NOT NULL, table_name sysname NOT NULL, + index_id integer NOT NULL, + index_name sysname NOT NULL, can_compress bit NOT NULL, reason nvarchar(200) NULL, - PRIMARY KEY (database_id, object_id) + PRIMARY KEY (database_id, object_id, index_id) ); /* @@ -622,17 +626,25 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. INSERT INTO #compression_eligibility ( database_id, + database_name, schema_id, + schema_name, object_id, table_name, + index_id, + index_name, can_compress, reason ) - SELECT DISTINCT - database_id, - schema_id, - object_id, - table_name, + SELECT + fo.database_id, + fo.database_name, + fo.schema_id, + fo.schema_name, + fo.object_id, + fo.table_name, + fo.index_id, + fo.index_name, 1, /* Default to compressible */ NULL FROM #filtered_objects; @@ -1712,8 +1724,9 @@ LEFT JOIN ia.object_id = ps.object_id AND ia.index_id = ps.index_id JOIN #compression_eligibility ce ON - ia.database_id = ce.database_id AND - ia.object_id = ce.object_id + ia.database_id = ce.database_id + AND ia.object_id = ce.object_id + AND ia.index_id = ce.index_id WHERE ia.action IN ('MERGE INCLUDES', 'MAKE UNIQUE') AND ce.can_compress = 1 ORDER BY @@ -1787,12 +1800,13 @@ LEFT JOIN ps.partition_function_name, ps.partition_columns ) ps ON - ia.database_id = ps.database_id AND - ia.object_id = ps.object_id AND - ia.index_id = ps.index_id + ia.database_id = ps.database_id + AND ia.object_id = ps.object_id + AND ia.index_id = ps.index_id JOIN #compression_eligibility ce ON - ia.database_id = ce.database_id AND - ia.object_id = ce.object_id + ia.database_id = ce.database_id + AND ia.object_id = ce.object_id + AND ia.index_id = ce.index_id WHERE /* Indexes that are not being disabled or merged */ ia.action IS NULL OR ia.action = 'KEEP' @@ -1881,12 +1895,13 @@ SELECT N', DATA_COMPRESSION = PAGE);' FROM #index_analysis ia JOIN #partition_stats ps ON - ia.database_id = ps.database_id AND - ia.object_id = ps.object_id AND - ia.index_id = ps.index_id + ia.database_id = ps.database_id + AND ia.object_id = ps.object_id + AND ia.index_id = ps.index_id JOIN #compression_eligibility ce ON - ia.database_id = ce.database_id AND - ia.object_id = ce.object_id + ia.database_id = ce.database_id + AND ia.object_id = ce.object_id + AND ia.index_id = ce.index_id WHERE /* Only partitioned indexes */ ps.partition_function_name IS NOT NULL @@ -1955,23 +1970,28 @@ SELECT END) FROM #index_analysis ia LEFT JOIN #partition_stats ps ON - ia.database_id = ps.database_id AND - ia.object_id = ps.object_id AND - ia.index_id = ps.index_id + ia.database_id = ps.database_id + AND ia.object_id = ps.object_id + AND ia.index_id = ps.index_id LEFT JOIN #compression_eligibility ce ON - ia.database_id = ce.database_id AND - ia.object_id = ce.object_id; + ia.database_id = ce.database_id + AND ia.object_id = ce.object_id + AND ia.index_id = ce.index_id; /* Report on tables that can't be compressed */ SELECT - database_name = ce.database_name, - table_name = ce.table_name, + ce.database_name, + ce.schema_name, + ce.table_name, + ce.index_name, compression_ineligibility_reason = ce.reason FROM #compression_eligibility ce WHERE ce.can_compress = 0 ORDER BY ce.database_name, - ce.table_name; + ce.schema_name, + ce.table_name, + ce.index_name; END TRY BEGIN CATCH From 0ae3bf8f300901aaa34cf3960515a8572af701dc Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Wed, 12 Mar 2025 14:00:01 -0400 Subject: [PATCH 033/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index 577bcadb..5e4f304e 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -450,7 +450,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. missing_columns nvarchar(max) NULL, action nvarchar(max) NULL, target_index_name sysname NULL, - consolidation_rule varchar(50) NULL, + consolidation_rule varchar(512) NULL, index_priority int NULL, INDEX c CLUSTERED (database_id, schema_name, table_name, index_name) From f3592ff13338802e88e330dc6dcbc9ff22e57d9c Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Wed, 12 Mar 2025 14:07:09 -0400 Subject: [PATCH 034/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 116 ++++++++++-------- 1 file changed, 62 insertions(+), 54 deletions(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index 4608c529..cd1babe2 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -623,7 +623,10 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. RAISERROR('Populating #compression_eligibility', 0, 0) WITH NOWAIT; END; - INSERT INTO #compression_eligibility + INSERT INTO + #compression_eligibility + WITH + (TABLOCK) ( database_id, database_name, @@ -647,7 +650,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. fo.index_name, 1, /* Default to compressible */ NULL - FROM #filtered_objects; + FROM #filtered_objects AS fo; /* If SQL Server edition doesn't support compression, mark all as ineligible */ IF @can_compress = 0 @@ -1552,13 +1555,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ia1.action = 'DISABLE' /* The narrower index gets disabled */ FROM #index_analysis ia1 JOIN #index_analysis ia2 - ON ia1.database_id = ia2.database_id - AND ia1.object_id = ia2.object_id - AND ia1.index_name <> ia2.index_name - AND ia2.key_columns LIKE (ia1.key_columns + '%') /* ia2 has wider key that starts with ia1's key */ - AND ISNULL(ia1.filter_definition, '') = ISNULL(ia2.filter_definition, '') /* Matching filters */ - /* Exception: If narrower index is unique and wider is not, they should not be merged */ - AND NOT (ia1.is_unique = 1 AND ia2.is_unique = 0) + ON ia1.database_id = ia2.database_id + AND ia1.object_id = ia2.object_id + AND ia1.index_name <> ia2.index_name + AND ia2.key_columns LIKE (ia1.key_columns + '%') /* ia2 has wider key that starts with ia1's key */ + AND ISNULL(ia1.filter_definition, '') = ISNULL(ia2.filter_definition, '') /* Matching filters */ + /* Exception: If narrower index is unique and wider is not, they should not be merged */ + AND NOT (ia1.is_unique = 1 AND ia2.is_unique = 0) WHERE ia1.consolidation_rule IS NULL /* Not already processed */ AND ia2.consolidation_rule IS NULL /* Not already processed */ AND EXISTS @@ -1719,14 +1722,14 @@ LEFT JOIN ps.built_on, ps.partition_function_name, ps.partition_columns -) ps ON - ia.database_id = ps.database_id AND - ia.object_id = ps.object_id AND - ia.index_id = ps.index_id -JOIN #compression_eligibility ce ON - ia.database_id = ce.database_id - AND ia.object_id = ce.object_id - AND ia.index_id = ce.index_id +) ps + ON ia.database_id = ps.database_id + AND ia.object_id = ps.object_id + AND ia.index_id = ps.index_id +JOIN #compression_eligibility ce + ON ia.database_id = ce.database_id + AND ia.object_id = ce.object_id + AND ia.index_id = ce.index_id WHERE ia.action IN ('MERGE INCLUDES', 'MAKE UNIQUE') AND ce.can_compress = 1 ORDER BY @@ -1799,14 +1802,14 @@ LEFT JOIN ps.built_on, ps.partition_function_name, ps.partition_columns -) ps ON - ia.database_id = ps.database_id - AND ia.object_id = ps.object_id - AND ia.index_id = ps.index_id -JOIN #compression_eligibility ce ON - ia.database_id = ce.database_id - AND ia.object_id = ce.object_id - AND ia.index_id = ce.index_id +) ps + ON ia.database_id = ps.database_id + AND ia.object_id = ps.object_id + AND ia.index_id = ps.index_id +JOIN #compression_eligibility ce + ON ia.database_id = ce.database_id + AND ia.object_id = ce.object_id + AND ia.index_id = ce.index_id WHERE /* Indexes that are not being disabled or merged */ ia.action IS NULL OR ia.action = 'KEEP' @@ -1834,10 +1837,10 @@ SELECT QUOTENAME(id.index_name) + N';' FROM #index_analysis ia -JOIN #index_details id ON - id.database_id = ia.database_id AND - id.object_id = ia.object_id AND - id.is_unique_constraint = 1 +JOIN #index_details id + ON id.database_id = ia.database_id + AND id.object_id = ia.object_id + AND id.is_unique_constraint = 1 WHERE /* Only indexes that are being made unique */ ia.action = 'MAKE UNIQUE' @@ -1849,20 +1852,25 @@ WHERE AND id_nc.object_id = ia.object_id AND id_nc.index_name = ia.index_name /* Matching key columns */ - AND NOT EXISTS ( - SELECT id.column_name + AND NOT EXISTS + ( + SELECT + id.column_name FROM #index_details id_inner WHERE id_inner.database_id = id.database_id - AND id_inner.object_id = id.object_id - AND id_inner.index_id = id.index_id - AND id_inner.is_included_column = 0 + AND id_inner.object_id = id.object_id + AND id_inner.index_id = id.index_id + AND id_inner.is_included_column = 0 + EXCEPT - SELECT id_nc_inner.column_name + + SELECT + id_nc_inner.column_name FROM #index_details id_nc_inner WHERE id_nc_inner.database_id = id_nc.database_id - AND id_nc_inner.object_id = id_nc.object_id - AND id_nc_inner.index_name = id_nc.index_name - AND id_nc_inner.is_included_column = 0 + AND id_nc_inner.object_id = id_nc.object_id + AND id_nc_inner.index_name = id_nc.index_name + AND id_nc_inner.is_included_column = 0 ) ) ORDER BY @@ -1894,14 +1902,14 @@ SELECT CASE WHEN @online = 1 THEN N'ON' ELSE N'OFF' END + N', DATA_COMPRESSION = PAGE);' FROM #index_analysis ia -JOIN #partition_stats ps ON - ia.database_id = ps.database_id - AND ia.object_id = ps.object_id - AND ia.index_id = ps.index_id -JOIN #compression_eligibility ce ON - ia.database_id = ce.database_id - AND ia.object_id = ce.object_id - AND ia.index_id = ce.index_id +JOIN #partition_stats ps + ON ia.database_id = ps.database_id + AND ia.object_id = ps.object_id + AND ia.index_id = ps.index_id +JOIN #compression_eligibility ce + ON ia.database_id = ce.database_id + AND ia.object_id = ce.object_id + AND ia.index_id = ce.index_id WHERE /* Only partitioned indexes */ ps.partition_function_name IS NOT NULL @@ -1969,14 +1977,14 @@ SELECT ELSE 0 END) FROM #index_analysis ia -LEFT JOIN #partition_stats ps ON - ia.database_id = ps.database_id - AND ia.object_id = ps.object_id - AND ia.index_id = ps.index_id -LEFT JOIN #compression_eligibility ce ON - ia.database_id = ce.database_id - AND ia.object_id = ce.object_id - AND ia.index_id = ce.index_id; +LEFT JOIN #partition_stats ps + ON ia.database_id = ps.database_id + AND ia.object_id = ps.object_id + AND ia.index_id = ps.index_id +LEFT JOIN #compression_eligibility ce + ON ia.database_id = ce.database_id + AND ia.object_id = ce.object_id + AND ia.index_id = ce.index_id; /* Report on tables that can't be compressed */ SELECT From 3455adadec94ad421e9265158323660609e247c9 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Wed, 12 Mar 2025 16:29:18 -0400 Subject: [PATCH 035/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 519 ++++++++++++------ 1 file changed, 341 insertions(+), 178 deletions(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index cd1babe2..2b7d37ff 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -1653,56 +1653,195 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. RAISERROR('Generating results', 0, 0) WITH NOWAIT; END; -/* Generate index merge scripts with compression and drop_existing */ + /* + Create a consolidated results table to hold all outputs in a single result set + This provides a more cohesive experience with summary information and scripts in one place + */ + CREATE TABLE #index_cleanup_results + ( + result_type varchar(50) NOT NULL, /* 'SUMMARY', 'MERGE', 'DISABLE', 'COMPRESS', etc. */ + sort_order integer NOT NULL, /* Keeps results in logical order */ + database_name sysname NULL, + schema_name sysname NULL, + table_name sysname NULL, + index_name sysname NULL, + script_type nvarchar(50) NULL, /* 'MERGE', 'DISABLE', 'COMPRESS', etc. */ + consolidation_rule nvarchar(200) NULL, + target_index_name sysname NULL, + script nvarchar(max) NULL, + additional_info nvarchar(max) NULL /* For stats, constraints, etc. */ + ); + +/* Insert summary statistics first */ +INSERT INTO #index_cleanup_results +( + result_type, + sort_order, + script_type, + additional_info +) +SELECT + 'SUMMARY', + 1, + 'Index Cleanup Summary', + N'Server uptime: ' + + CAST(@uptime_days AS nvarchar(10)) + + N' days' + + CASE + WHEN @uptime_warning = 1 + THEN N' (WARNING: Low uptime detected! Index usage data may be incomplete.)' + ELSE N'' + END + + N' | Tables analyzed: ' + + CAST(COUNT(DISTINCT CONCAT(ia.database_id, N'.', ia.schema_id, N'.', ia.object_id)) AS nvarchar(10)) + + N' | Total indexes: ' + + CAST(COUNT(*) AS nvarchar(10)) + + N' | Indexes to disable: ' + + CAST(SUM(CASE WHEN ia.action = 'DISABLE' THEN 1 ELSE 0 END) AS nvarchar(10)) + + N' | Indexes to merge: ' + + CAST(SUM(CASE WHEN ia.action IN ('MERGE INCLUDES', 'MAKE UNIQUE') THEN 1 ELSE 0 END) AS nvarchar(10)) + + N' | Avg indexes per table: ' + + CAST(CONVERT(decimal(10,2), COUNT(*) * 1.0 / + NULLIF(COUNT(DISTINCT CONCAT(ia.database_id, N'.', ia.schema_id, N'.', ia.object_id)), 0)) AS nvarchar(10)) +FROM #index_analysis ia; + +/* Insert space savings estimates */ +INSERT INTO #index_cleanup_results +( + result_type, + sort_order, + script_type, + additional_info +) +SELECT + 'SUMMARY', + 2, + 'Estimated Space Savings', + N'Space saved from cleanup: ' + + CAST(CONVERT(decimal(10,2), SUM(CASE + WHEN ia.action IN ('DISABLE', 'MERGE INCLUDES', 'MAKE UNIQUE') + THEN ps.total_space_mb + ELSE 0 + END)) AS nvarchar(20)) + + N' MB | Compression savings estimate: ' + + CAST(CONVERT(decimal(10,2), SUM(CASE + WHEN (ia.action IS NULL OR ia.action = 'KEEP') AND ce.can_compress = 1 + THEN ps.total_space_mb * 0.20 /* Conservative estimate - 20% compression ratio */ + ELSE 0 + END)) AS nvarchar(20)) + + N' - ' + + CAST(CONVERT(decimal(10,2), SUM(CASE + WHEN (ia.action IS NULL OR ia.action = 'KEEP') AND ce.can_compress = 1 + THEN ps.total_space_mb * 0.60 /* Optimistic estimate - 60% compression ratio */ + ELSE 0 + END)) AS nvarchar(20)) + + N' MB | Total estimated savings: ' + + CAST(CONVERT(decimal(10,2), SUM(CASE + WHEN ia.action IN ('DISABLE', 'MERGE INCLUDES', 'MAKE UNIQUE') + THEN ps.total_space_mb + WHEN (ia.action IS NULL OR ia.action = 'KEEP') AND ce.can_compress = 1 + THEN ps.total_space_mb * 0.20 + ELSE 0 + END)) AS nvarchar(20)) + + N' - ' + + CAST(CONVERT(decimal(10,2), SUM(CASE + WHEN ia.action IN ('DISABLE', 'MERGE INCLUDES', 'MAKE UNIQUE') + THEN ps.total_space_mb + WHEN (ia.action IS NULL OR ia.action = 'KEEP') AND ce.can_compress = 1 + THEN ps.total_space_mb * 0.60 + ELSE 0 + END)) AS nvarchar(20)) + + N' MB' +FROM #index_analysis ia +LEFT JOIN #partition_stats ps ON + ia.database_id = ps.database_id + AND ia.object_id = ps.object_id + AND ia.index_id = ps.index_id +LEFT JOIN #compression_eligibility ce ON + ia.database_id = ce.database_id + AND ia.object_id = ce.object_id + AND ia.index_id = ce.index_id; + +/* Insert merge scripts for indexes */ +INSERT INTO #index_cleanup_results +( + result_type, + sort_order, + database_name, + schema_name, + table_name, + index_name, + script_type, + consolidation_rule, + target_index_name, + script +) SELECT + 'MERGE', + 10, ia.database_name, ia.schema_name, ia.table_name, ia.index_name, - ia.target_index_name, + 'MERGE SCRIPT', ia.consolidation_rule, - merge_script = - N'CREATE ' + - CASE WHEN ia.action = 'MAKE UNIQUE' THEN N'UNIQUE ' ELSE N'' END + - N'INDEX ' + - QUOTENAME(ia.index_name) + - N' ON ' + - QUOTENAME(ia.database_name) + - N'.' + - QUOTENAME(ia.schema_name) + - N'.' + - QUOTENAME(ia.table_name) + - N' (' + - ia.key_columns + - N')' + - CASE - WHEN ia.included_columns IS NOT NULL AND LEN(ia.included_columns) > 0 - THEN N' INCLUDE (' + - ia.included_columns + - N')' - ELSE N'' - END + - CASE - WHEN ia.filter_definition IS NOT NULL - THEN N' WHERE ' + - ia.filter_definition - ELSE N'' - END + - CASE - WHEN ps.partition_function_name IS NOT NULL - THEN N' ON ' + - QUOTENAME(ps.partition_function_name) + - N'(' + - ISNULL(ps.partition_columns, N'') + - N')' - WHEN ps.built_on IS NOT NULL - THEN N' ON ' + - QUOTENAME(ps.built_on) - ELSE N'' - END + - N' WITH (DROP_EXISTING = ON, FILLFACTOR = 100, SORT_IN_TEMPDB = ON, ONLINE = ' + - CASE WHEN @online = 1 THEN N'ON' ELSE N'OFF' END + - N', DATA_COMPRESSION = PAGE);' + ia.target_index_name, + CASE + WHEN ia.action = 'MAKE UNIQUE' + THEN N'/* This index can replace a unique constraint */ +/* Creating unique index with same keys as constraint */ +CREATE UNIQUE ' + WHEN ia.action = 'MERGE INCLUDES' + THEN N'/* This index can be merged with another index */ +/* Creating index with combined includes from both */ +CREATE ' + ELSE N'CREATE ' + END + + N'INDEX ' + + QUOTENAME(ia.index_name) + + N' ON ' + + QUOTENAME(ia.database_name) + + N'.' + + QUOTENAME(ia.schema_name) + + N'.' + + QUOTENAME(ia.table_name) + + N' (' + + ia.key_columns + + N')' + + CASE + WHEN ia.included_columns IS NOT NULL AND LEN(ia.included_columns) > 0 AND ia.action = 'MERGE INCLUDES' + THEN N' INCLUDE (' + + N'/* Combined includes from merged indexes */ + ' + + ia.included_columns + + N')' + WHEN ia.included_columns IS NOT NULL AND LEN(ia.included_columns) > 0 + THEN N' INCLUDE (' + + ia.included_columns + + N')' + ELSE N'' + END + + CASE + WHEN ia.filter_definition IS NOT NULL + THEN N' WHERE ' + + ia.filter_definition + ELSE N'' + END + + CASE + WHEN ps.partition_function_name IS NOT NULL + THEN N' ON ' + + QUOTENAME(ps.partition_function_name) + + N'(' + + ISNULL(ps.partition_columns, N'') + + N')' + WHEN ps.built_on IS NOT NULL + THEN N' ON ' + + QUOTENAME(ps.built_on) + ELSE N'' + END + + N' WITH (DROP_EXISTING = ON, FILLFACTOR = 100, SORT_IN_TEMPDB = ON, ONLINE = ' + + CASE WHEN @online = 1 THEN N'ON' ELSE N'OFF' END + + N', DATA_COMPRESSION = PAGE);' FROM #index_analysis ia LEFT JOIN ( @@ -1736,38 +1875,62 @@ ORDER BY ia.table_name, ia.index_name; -/* Generate disable scripts for unneeded indexes */ +/* Insert disable scripts for unneeded indexes */ +INSERT INTO #index_cleanup_results +( + result_type, + sort_order, + database_name, + schema_name, + table_name, + index_name, + script_type, + consolidation_rule, + script +) SELECT + 'DISABLE', + 20, ia.database_name, ia.schema_name, ia.table_name, ia.index_name, + 'DISABLE SCRIPT', ia.consolidation_rule, - disable_script = - N'ALTER INDEX ' + - QUOTENAME(ia.index_name) + - N' ON ' + - QUOTENAME(ia.database_name) + - N'.' + - QUOTENAME(ia.schema_name) + - N'.' + - QUOTENAME(ia.table_name) + - N' DISABLE;' + N'ALTER INDEX ' + + QUOTENAME(ia.index_name) + + N' ON ' + + QUOTENAME(ia.database_name) + + N'.' + + QUOTENAME(ia.schema_name) + + N'.' + + QUOTENAME(ia.table_name) + + N' DISABLE;' FROM #index_analysis ia -WHERE ia.action = 'DISABLE' -ORDER BY - ia.table_name, - ia.index_name; +WHERE ia.action = 'DISABLE'; -/* Generate compression scripts for remaining indexes */ +/* Insert compression scripts for remaining indexes */ +INSERT INTO #index_cleanup_results +( + result_type, + sort_order, + database_name, + schema_name, + table_name, + index_name, + script_type, + script, + additional_info +) SELECT + 'COMPRESS', + 40, ia.database_name, ia.schema_name, ia.table_name, ia.index_name, - compression_type = N'All Partitions', - compression_script = - N'ALTER INDEX ' + + 'COMPRESSION SCRIPT', + N'ALTER INDEX ' + QUOTENAME(ia.index_name) + N' ON ' + QUOTENAME(ia.database_name) + @@ -1782,7 +1945,8 @@ SELECT END + N' WITH (FILLFACTOR = 100, SORT_IN_TEMPDB = ON, ONLINE = ' + CASE WHEN @online = 1 THEN N'ON' ELSE N'OFF' END + - N', DATA_COMPRESSION = PAGE);' + N', DATA_COMPRESSION = PAGE);', + N'Compression type: All Partitions' FROM #index_analysis ia LEFT JOIN ( @@ -1812,30 +1976,41 @@ JOIN #compression_eligibility ce AND ia.index_id = ce.index_id WHERE /* Indexes that are not being disabled or merged */ - ia.action IS NULL OR ia.action = 'KEEP' + (ia.action IS NULL OR ia.action = 'KEEP') /* Only indexes eligible for compression */ - AND ce.can_compress = 1 -ORDER BY - ia.table_name, - ia.index_name; + AND ce.can_compress = 1; -/* Generate scripts to disable unique constraints that are being replaced by unique indexes */ +/* Insert disable scripts for unique constraints */ +INSERT INTO #index_cleanup_results +( + result_type, + sort_order, + database_name, + schema_name, + table_name, + index_name, + script_type, + additional_info, + script +) SELECT + 'CONSTRAINT', + 30, ia.database_name, ia.schema_name, ia.table_name, ia.index_name, - constraint_name = id.index_name, - disable_constraint_script = - N'ALTER TABLE ' + - QUOTENAME(ia.database_name) + - N'.' + - QUOTENAME(ia.schema_name) + - N'.' + - QUOTENAME(ia.table_name) + - N' NOCHECK CONSTRAINT ' + - QUOTENAME(id.index_name) + - N';' + 'DISABLE CONSTRAINT SCRIPT', + N'Constraint to disable: ' + id.index_name, + N'ALTER TABLE ' + + QUOTENAME(ia.database_name) + + N'.' + + QUOTENAME(ia.schema_name) + + N'.' + + QUOTENAME(ia.table_name) + + N' NOCHECK CONSTRAINT ' + + QUOTENAME(id.index_name) + + N';' FROM #index_analysis ia JOIN #index_details id ON id.database_id = ia.database_id @@ -1872,35 +2047,49 @@ WHERE AND id_nc_inner.index_name = id_nc.index_name AND id_nc_inner.is_included_column = 0 ) - ) -ORDER BY - ia.table_name, - ia.index_name; + ); -/* Generate per-partition compression scripts for partitioned indexes */ +/* Insert per-partition compression scripts */ +INSERT INTO #index_cleanup_results +( + result_type, + sort_order, + database_name, + schema_name, + table_name, + index_name, + script_type, + script, + additional_info +) SELECT + 'COMPRESS_PARTITION', + 50, ia.database_name, ia.schema_name, ia.table_name, ia.index_name, - compression_type = N'Per Partition', - partition_number = ps.partition_number, - total_rows = ps.total_rows, - total_space_mb = ps.total_space_mb, - compression_script = - N'ALTER INDEX ' + - QUOTENAME(ia.index_name) + - N' ON ' + - QUOTENAME(ia.database_name) + - N'.' + - QUOTENAME(ia.schema_name) + - N'.' + - QUOTENAME(ia.table_name) + - N' REBUILD PARTITION = ' + - CAST(ps.partition_number AS nvarchar(20)) + - N' WITH (FILLFACTOR = 100, SORT_IN_TEMPDB = ON, ONLINE = ' + - CASE WHEN @online = 1 THEN N'ON' ELSE N'OFF' END + - N', DATA_COMPRESSION = PAGE);' + 'PARTITION COMPRESSION SCRIPT', + N'ALTER INDEX ' + + QUOTENAME(ia.index_name) + + N' ON ' + + QUOTENAME(ia.database_name) + + N'.' + + QUOTENAME(ia.schema_name) + + N'.' + + QUOTENAME(ia.table_name) + + N' REBUILD PARTITION = ' + + CAST(ps.partition_number AS nvarchar(20)) + + N' WITH (FILLFACTOR = 100, SORT_IN_TEMPDB = ON, ONLINE = ' + + CASE WHEN @online = 1 THEN N'ON' ELSE N'OFF' END + + N', DATA_COMPRESSION = PAGE);', + N'Compression type: Per Partition | Partition: ' + + CAST(ps.partition_number AS nvarchar(20)) + + N' | Rows: ' + + CAST(ps.total_rows AS nvarchar(20)) + + N' | Size: ' + + CAST(CONVERT(decimal(10,2), ps.total_space_mb) AS nvarchar(20)) + + N' MB' FROM #index_analysis ia JOIN #partition_stats ps ON ia.database_id = ps.database_id @@ -1916,90 +2105,64 @@ WHERE /* Indexes that are not being disabled or merged */ AND (ia.action IS NULL OR ia.action = 'KEEP') /* Only indexes eligible for compression */ - AND ce.can_compress = 1 -ORDER BY - ia.table_name, - ia.index_name, - ps.partition_number; + AND ce.can_compress = 1; -/* Generate index statistics and reporting */ -SELECT - report_title = N'Index Cleanup Summary', - server_uptime_days = @uptime_days, - uptime_warning = - CASE - WHEN @uptime_warning = 1 - THEN N'Low uptime detected! Index usage data may be incomplete.' - ELSE NULL - END, - tables_analyzed = COUNT(DISTINCT CONCAT(ia.database_id, N'.', ia.schema_id, N'.', ia.object_id)), - total_indexes = COUNT(*), - indexes_to_disable = SUM(CASE WHEN ia.action = 'DISABLE' THEN 1 ELSE 0 END), - indexes_to_merge = SUM(CASE WHEN ia.action IN ('MERGE INCLUDES', 'MAKE UNIQUE') THEN 1 ELSE 0 END), - avg_indexes_per_table = CONVERT(DECIMAL(10,2), COUNT(*) * 1.0 / NULLIF(COUNT(DISTINCT CONCAT(ia.database_id, N'.', ia.schema_id, N'.', ia.object_id)), 0)) -FROM #index_analysis ia; +/* Stats reporting has been moved to #index_cleanup_results table */ -/* Generate estimated space savings report */ -SELECT - report_title = N'Estimated Space Savings', - space_saved_from_cleanup_mb = - SUM(CASE - WHEN ia.action IN ('DISABLE', 'MERGE INCLUDES', 'MAKE UNIQUE') - THEN ps.total_space_mb - ELSE 0 - END), - estimated_min_compression_savings_mb = - SUM(CASE - WHEN (ia.action IS NULL OR ia.action = 'KEEP') AND ce.can_compress = 1 - THEN ps.total_space_mb * 0.20 /* Conservative estimate - 20% compression ratio */ - ELSE 0 - END), - estimated_max_compression_savings_mb = - SUM(CASE - WHEN (ia.action IS NULL OR ia.action = 'KEEP') AND ce.can_compress = 1 - THEN ps.total_space_mb * 0.60 /* Optimistic estimate - 60% compression ratio */ - ELSE 0 - END), - total_min_estimated_savings_mb = - SUM(CASE - WHEN ia.action IN ('DISABLE', 'MERGE INCLUDES', 'MAKE UNIQUE') - THEN ps.total_space_mb - WHEN (ia.action IS NULL OR ia.action = 'KEEP') AND ce.can_compress = 1 - THEN ps.total_space_mb * 0.20 - ELSE 0 - END), - total_max_estimated_savings_mb = - SUM(CASE - WHEN ia.action IN ('DISABLE', 'MERGE INCLUDES', 'MAKE UNIQUE') - THEN ps.total_space_mb - WHEN (ia.action IS NULL OR ia.action = 'KEEP') AND ce.can_compress = 1 - THEN ps.total_space_mb * 0.60 - ELSE 0 - END) -FROM #index_analysis ia -LEFT JOIN #partition_stats ps - ON ia.database_id = ps.database_id - AND ia.object_id = ps.object_id - AND ia.index_id = ps.index_id -LEFT JOIN #compression_eligibility ce - ON ia.database_id = ce.database_id - AND ia.object_id = ce.object_id - AND ia.index_id = ce.index_id; +/* Space savings reporting has been moved to #index_cleanup_results table */ -/* Report on tables that can't be compressed */ +/* Insert compression ineligible info */ +INSERT INTO #index_cleanup_results +( + result_type, + sort_order, + database_name, + schema_name, + table_name, + index_name, + script_type, + additional_info +) SELECT + 'INELIGIBLE', + 90, ce.database_name, ce.schema_name, ce.table_name, ce.index_name, - compression_ineligibility_reason = ce.reason + 'INELIGIBLE FOR COMPRESSION', + ce.reason FROM #compression_eligibility ce -WHERE ce.can_compress = 0 +WHERE ce.can_compress = 0; + +/* +Return the consolidated results in a single result set +Results are ordered by: +1. Summary information (overall stats, savings estimates) +2. Merge scripts (includes merges and unique conversions) +3. Disable scripts (for redundant indexes) +4. Constraint scripts (for unique constraints to disable) +5. Compression scripts (for tables eligible for compression) +6. Partition-specific compression scripts +7. Ineligible objects (tables that can't be compressed) +*/ +SELECT + database_name, + schema_name, + table_name, + index_name, + script_type, + consolidation_rule, + target_index_name, + script, + additional_info +FROM #index_cleanup_results ORDER BY - ce.database_name, - ce.schema_name, - ce.table_name, - ce.index_name; + sort_order, + database_name, + schema_name, + table_name, + index_name; END TRY BEGIN CATCH From 550596bdb0e2929bfb40ab51e0345f5926c3c994 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Wed, 12 Mar 2025 16:53:54 -0400 Subject: [PATCH 036/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 115 +++++++++++++++--- 1 file changed, 98 insertions(+), 17 deletions(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index 2b7d37ff..29aea8a9 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -1514,6 +1514,14 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. (ia1.index_priority >= ia2.index_priority AND NOT (ia1.is_unique = 0 AND ia2.is_unique = 1)) THEN 'MERGE INCLUDES' /* Keep this index but merge includes */ ELSE 'DISABLE' /* Other index is keeper, disable this one */ + END, + /* For the winning index, set clear superseded_by text for the report */ + ia1.superseded_by = + CASE + WHEN (ia1.is_unique = 1 AND ia2.is_unique = 0) OR + (ia1.index_priority >= ia2.index_priority AND NOT (ia1.is_unique = 0 AND ia2.is_unique = 1)) + THEN 'Supersedes ' + ia2.index_name + ELSE NULL END FROM #index_analysis ia1 JOIN #index_analysis ia2 @@ -1552,7 +1560,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SET ia1.consolidation_rule = 'Key Subset', ia1.target_index_name = ia2.index_name, - ia1.action = 'DISABLE' /* The narrower index gets disabled */ + ia1.action = 'DISABLE', /* The narrower index gets disabled */ + /* Update the wider (winning) index for the report */ + ia2.superseded_by = 'Supersedes ' + ia1.index_name FROM #index_analysis ia1 JOIN #index_analysis ia2 ON ia1.database_id = ia2.database_id @@ -1673,6 +1683,21 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ); /* Insert summary statistics first */ +/* Add a separator row for the header */ +INSERT INTO #index_cleanup_results +( + result_type, + sort_order, + script_type, + additional_info +) +SELECT + 'HEADER', + 0, + 'SEPARATOR', + N'==================== INDEX CLEANUP SUMMARY ===================='; + +/* Add summary information */ INSERT INTO #index_cleanup_results ( result_type, @@ -1762,6 +1787,34 @@ LEFT JOIN #compression_eligibility ce ON AND ia.object_id = ce.object_id AND ia.index_id = ce.index_id; +/* Add a separator for scripts section */ +INSERT INTO #index_cleanup_results +( + result_type, + sort_order, + script_type, + additional_info +) +SELECT + 'HEADER', + 9, + 'SEPARATOR', + N'==================== INDEX SCRIPTS ===================='; + +/* Add a separator for report section at the end */ +INSERT INTO #index_cleanup_results +( + result_type, + sort_order, + script_type, + additional_info +) +SELECT + 'HEADER', + 99, + 'SEPARATOR', + N'==================== END OF REPORT ===================='; + /* Insert merge scripts for indexes */ INSERT INTO #index_cleanup_results ( @@ -1774,7 +1827,8 @@ INSERT INTO #index_cleanup_results script_type, consolidation_rule, target_index_name, - script + script, + additional_info ) SELECT 'MERGE', @@ -1841,7 +1895,13 @@ CREATE ' END + N' WITH (DROP_EXISTING = ON, FILLFACTOR = 100, SORT_IN_TEMPDB = ON, ONLINE = ' + CASE WHEN @online = 1 THEN N'ON' ELSE N'OFF' END + - N', DATA_COMPRESSION = PAGE);' + N', DATA_COMPRESSION = PAGE);', + /* Additional info about what this script does */ + CASE + WHEN ia.action = 'MERGE INCLUDES' THEN N'This index will absorb includes from duplicate indexes' + WHEN ia.action = 'MAKE UNIQUE' THEN N'This index will replace a unique constraint' + ELSE NULL + END FROM #index_analysis ia LEFT JOIN ( @@ -1871,9 +1931,7 @@ JOIN #compression_eligibility ce AND ia.index_id = ce.index_id WHERE ia.action IN ('MERGE INCLUDES', 'MAKE UNIQUE') AND ce.can_compress = 1 -ORDER BY - ia.table_name, - ia.index_name; +AND ia.target_index_name IS NULL /* Only create merge scripts for the "winning" indexes */; /* Insert disable scripts for unneeded indexes */ INSERT INTO #index_cleanup_results @@ -1886,7 +1944,8 @@ INSERT INTO #index_cleanup_results index_name, script_type, consolidation_rule, - script + script, + additional_info ) SELECT 'DISABLE', @@ -1905,7 +1964,18 @@ SELECT QUOTENAME(ia.schema_name) + N'.' + QUOTENAME(ia.table_name) + - N' DISABLE;' + N' DISABLE;', + CASE + WHEN ia.consolidation_rule = 'Key Subset' + THEN N'This index is superseded by a wider index: ' + ISNULL(ia.target_index_name, N'(unknown)') + WHEN ia.consolidation_rule = 'Exact Duplicate' + THEN N'This index is an exact duplicate of: ' + ISNULL(ia.target_index_name, N'(unknown)') + WHEN ia.consolidation_rule = 'Key Duplicate' + THEN N'This index has the same keys as: ' + ISNULL(ia.target_index_name, N'(unknown)') + WHEN ia.consolidation_rule LIKE 'Unused Index%' + THEN ia.consolidation_rule + ELSE N'This index is redundant' + END FROM #index_analysis ia WHERE ia.action = 'DISABLE'; @@ -2147,22 +2217,33 @@ Results are ordered by: 7. Ineligible objects (tables that can't be compressed) */ SELECT + /* First, show the information needed to understand the script */ + script_type, + additional_info, + /* Then show identifying information for the index */ database_name, schema_name, table_name, index_name, - script_type, + /* Then show relationship information */ consolidation_rule, target_index_name, - script, - additional_info -FROM #index_cleanup_results + /* Include superseded_by info for winning indexes */ + CASE WHEN ia.superseded_by IS NOT NULL THEN ia.superseded_by ELSE NULL END AS superseded_info, + /* Finally show the actual script */ + script +FROM #index_cleanup_results ir +LEFT JOIN #index_analysis ia + ON ir.database_name = ia.database_name + AND ir.schema_name = ia.schema_name + AND ir.table_name = ia.table_name + AND ir.index_name = ia.index_name ORDER BY - sort_order, - database_name, - schema_name, - table_name, - index_name; + ir.sort_order, + ir.database_name, + ir.schema_name, + ir.table_name, + ir.index_name; END TRY BEGIN CATCH From 21db963db7a2f138d6496cc3b393f376d4859bb3 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Wed, 12 Mar 2025 17:03:59 -0400 Subject: [PATCH 037/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index 29aea8a9..349d9c70 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -1560,9 +1560,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SET ia1.consolidation_rule = 'Key Subset', ia1.target_index_name = ia2.index_name, - ia1.action = 'DISABLE', /* The narrower index gets disabled */ - /* Update the wider (winning) index for the report */ - ia2.superseded_by = 'Supersedes ' + ia1.index_name + ia1.action = 'DISABLE' /* The narrower index gets disabled */ FROM #index_analysis ia1 JOIN #index_analysis ia2 ON ia1.database_id = ia2.database_id @@ -1595,6 +1593,23 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND id2.is_eligible_for_dedupe = 1 ); + /* Update the superseded_by column for the wider index in a separate statement */ + UPDATE + ia2 + SET + ia2.superseded_by = 'Supersedes ' + ia1.index_name + FROM #index_analysis ia1 + JOIN #index_analysis ia2 + ON ia1.database_id = ia2.database_id + AND ia1.object_id = ia2.object_id + AND ia1.index_name <> ia2.index_name + AND ia2.key_columns LIKE (ia1.key_columns + '%') /* ia2 has wider key that starts with ia1's key */ + AND ISNULL(ia1.filter_definition, '') = ISNULL(ia2.filter_definition, '') /* Matching filters */ + /* Exception: If narrower index is unique and wider is not, they should not be merged */ + AND NOT (ia1.is_unique = 1 AND ia2.is_unique = 0) + WHERE ia1.consolidation_rule = 'Key Subset' /* Use records just processed in previous UPDATE */ + AND ia1.target_index_name = ia2.index_name; /* Make sure we're updating the right wider index */ + /* Rule 5: Unique constraint vs. nonclustered index handling */ UPDATE ia1 From ed5d58e6300fe09fc55a4c910f7032926a78468c Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Wed, 12 Mar 2025 17:38:38 -0400 Subject: [PATCH 038/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 196 ++++++++++-------- 1 file changed, 105 insertions(+), 91 deletions(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index 349d9c70..ddf7dd95 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -384,9 +384,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. partition_id bigint NOT NULL, partition_number int NOT NULL, total_rows bigint NULL, - total_space_mb decimal(38, 2) NULL, - reserved_lob_mb decimal(38, 2) NULL, - reserved_row_overflow_mb decimal(38, 2) NULL, + total_space_gb decimal(38, 4) NULL, /* Using 4 decimal places for GB to maintain precision */ + reserved_lob_gb decimal(38, 4) NULL, /* Using 4 decimal places for GB to maintain precision */ + reserved_row_overflow_gb decimal(38, 4) NULL, /* Using 4 decimal places for GB to maintain precision */ data_compression_desc nvarchar(60) NULL, built_on sysname NULL, partition_function_name sysname NULL, @@ -673,12 +673,12 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SET can_compress = 0, reason = ''Table contains sparse columns or incompatible data types'' - FROM #compression_eligibility ce + FROM #compression_eligibility AS ce WHERE EXISTS ( SELECT 1/0 - FROM ' + QUOTENAME(@database_name) + N'.sys.columns c - JOIN ' + QUOTENAME(@database_name) + N'.sys.types t + FROM ' + QUOTENAME(@database_name) + N'.sys.columns AS c + JOIN ' + QUOTENAME(@database_name) + N'.sys.types AS t ON c.user_type_id = t.user_type_id WHERE c.object_id = ce.object_id AND (c.is_sparse = 1 OR t.name IN (N''text'', N''ntext'', N''image'')) @@ -764,7 +764,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ( SELECT 1/0 - FROM #filtered_objects fo + FROM #filtered_objects AS fo WHERE fo.database_id = os.database_id AND fo.object_id = os.object_id ) @@ -967,7 +967,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ( SELECT 1/0 - FROM #filtered_objects fo + FROM #filtered_objects AS fo WHERE fo.database_id = @database_id AND fo.object_id = t.object_id ) @@ -1117,9 +1117,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ps.partition_id, p.partition_number, total_rows = SUM(ps.row_count), - total_space_mb = SUM(a.total_pages) * 8 / 1024.0, - reserved_lob_mb = SUM(ps.lob_reserved_page_count) * 8. / 1024., - reserved_row_overflow_mb = SUM(ps.row_overflow_reserved_page_count) * 8. / 1024., + total_space_gb = SUM(a.total_pages) * 8 / 1024.0 / 1024.0, /* Convert directly to GB */ + reserved_lob_gb = SUM(ps.lob_reserved_page_count) * 8. / 1024. / 1024.0, /* Convert directly to GB */ + reserved_row_overflow_gb = SUM(ps.row_overflow_reserved_page_count) * 8. / 1024. / 1024.0, /* Convert directly to GB */ p.data_compression_desc, i.data_space_id FROM ' + QUOTENAME(@database_name) + N'.sys.tables AS t @@ -1140,7 +1140,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ( SELECT 1/0 - FROM #filtered_objects fo + FROM #filtered_objects AS fo WHERE fo.database_id = @database_id AND fo.object_id = t.object_id )'; @@ -1231,9 +1231,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. partition_id, partition_number, total_rows, - total_space_mb, - reserved_lob_mb, - reserved_row_overflow_mb, + total_space_gb, + reserved_lob_gb, + reserved_row_overflow_gb, data_compression_desc, built_on, partition_function_name, @@ -1460,8 +1460,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. THEN 'KEEP' /* This index is the keeper */ ELSE 'DISABLE' /* Other index gets disabled */ END - FROM #index_analysis ia1 - JOIN #index_analysis ia2 + FROM #index_analysis AS ia1 + JOIN #index_analysis AS ia2 ON ia1.database_id = ia2.database_id AND ia1.object_id = ia2.object_id AND ia1.index_name <> ia2.index_name @@ -1523,8 +1523,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. THEN 'Supersedes ' + ia2.index_name ELSE NULL END - FROM #index_analysis ia1 - JOIN #index_analysis ia2 + FROM #index_analysis AS ia1 + JOIN #index_analysis AS ia2 ON ia1.database_id = ia2.database_id AND ia1.object_id = ia2.object_id AND ia1.index_name <> ia2.index_name @@ -1561,8 +1561,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ia1.consolidation_rule = 'Key Subset', ia1.target_index_name = ia2.index_name, ia1.action = 'DISABLE' /* The narrower index gets disabled */ - FROM #index_analysis ia1 - JOIN #index_analysis ia2 + FROM #index_analysis AS ia1 + JOIN #index_analysis AS ia2 ON ia1.database_id = ia2.database_id AND ia1.object_id = ia2.object_id AND ia1.index_name <> ia2.index_name @@ -1598,8 +1598,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ia2 SET ia2.superseded_by = 'Supersedes ' + ia1.index_name - FROM #index_analysis ia1 - JOIN #index_analysis ia2 + FROM #index_analysis AS ia1 + JOIN #index_analysis AS ia2 ON ia1.database_id = ia2.database_id AND ia1.object_id = ia2.object_id AND ia1.index_name <> ia2.index_name @@ -1621,7 +1621,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. THEN 'MAKE UNIQUE' /* Convert to unique index */ ELSE 'KEEP' /* Already unique, so just keep it */ END - FROM #index_analysis ia1 + FROM #index_analysis AS ia1 WHERE ia1.consolidation_rule IS NULL /* Not already processed */ AND EXISTS ( @@ -1694,7 +1694,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. consolidation_rule nvarchar(200) NULL, target_index_name sysname NULL, script nvarchar(max) NULL, - additional_info nvarchar(max) NULL /* For stats, constraints, etc. */ + additional_info nvarchar(max) NULL, /* For stats, constraints, etc. */ + superseded_info nvarchar(max) NULL /* To store superseded_by information */ ); /* Insert summary statistics first */ @@ -1725,7 +1726,7 @@ SELECT 1, 'Index Cleanup Summary', N'Server uptime: ' + - CAST(@uptime_days AS nvarchar(10)) + + CONVERT(nvarchar(10), @uptime_days) + N' days' + CASE WHEN @uptime_warning = 1 @@ -1733,17 +1734,17 @@ SELECT ELSE N'' END + N' | Tables analyzed: ' + - CAST(COUNT(DISTINCT CONCAT(ia.database_id, N'.', ia.schema_id, N'.', ia.object_id)) AS nvarchar(10)) + + CONVERT(nvarchar(10), COUNT(DISTINCT CONCAT(ia.database_id, N'.', ia.schema_id, N'.', ia.object_id))) + N' | Total indexes: ' + - CAST(COUNT(*) AS nvarchar(10)) + + CONVERT(nvarchar(10), COUNT(*)) + N' | Indexes to disable: ' + - CAST(SUM(CASE WHEN ia.action = 'DISABLE' THEN 1 ELSE 0 END) AS nvarchar(10)) + + CONVERT(nvarchar(10), SUM(CASE WHEN ia.action = 'DISABLE' THEN 1 ELSE 0 END)) + N' | Indexes to merge: ' + - CAST(SUM(CASE WHEN ia.action IN ('MERGE INCLUDES', 'MAKE UNIQUE') THEN 1 ELSE 0 END) AS nvarchar(10)) + + CONVERT(nvarchar(10), SUM(CASE WHEN ia.action IN ('MERGE INCLUDES', 'MAKE UNIQUE') THEN 1 ELSE 0 END)) + N' | Avg indexes per table: ' + - CAST(CONVERT(decimal(10,2), COUNT(*) * 1.0 / - NULLIF(COUNT(DISTINCT CONCAT(ia.database_id, N'.', ia.schema_id, N'.', ia.object_id)), 0)) AS nvarchar(10)) -FROM #index_analysis ia; + CONVERT(nvarchar(10), CONVERT(decimal(10,2), COUNT(*) * 1.0 / + NULLIF(COUNT(DISTINCT CONCAT(ia.database_id, N'.', ia.schema_id, N'.', ia.object_id)), 0))) +FROM #index_analysis AS ia; /* Insert space savings estimates */ INSERT INTO #index_cleanup_results @@ -1758,46 +1759,46 @@ SELECT 2, 'Estimated Space Savings', N'Space saved from cleanup: ' + - CAST(CONVERT(decimal(10,2), SUM(CASE + CONVERT(nvarchar(20), CONVERT(decimal(10,4), SUM(CASE WHEN ia.action IN ('DISABLE', 'MERGE INCLUDES', 'MAKE UNIQUE') - THEN ps.total_space_mb + THEN ps.total_space_gb ELSE 0 - END)) AS nvarchar(20)) + - N' MB | Compression savings estimate: ' + - CAST(CONVERT(decimal(10,2), SUM(CASE + END))) + + N' GB | Compression savings estimate: ' + + CONVERT(nvarchar(20), CONVERT(decimal(10,4), SUM(CASE WHEN (ia.action IS NULL OR ia.action = 'KEEP') AND ce.can_compress = 1 - THEN ps.total_space_mb * 0.20 /* Conservative estimate - 20% compression ratio */ + THEN ps.total_space_gb * 0.20 /* Conservative estimate - 20% compression ratio */ ELSE 0 - END)) AS nvarchar(20)) + + END))) + N' - ' + - CAST(CONVERT(decimal(10,2), SUM(CASE + CONVERT(nvarchar(20), CONVERT(decimal(10,4), SUM(CASE WHEN (ia.action IS NULL OR ia.action = 'KEEP') AND ce.can_compress = 1 - THEN ps.total_space_mb * 0.60 /* Optimistic estimate - 60% compression ratio */ + THEN ps.total_space_gb * 0.60 /* Optimistic estimate - 60% compression ratio */ ELSE 0 - END)) AS nvarchar(20)) + - N' MB | Total estimated savings: ' + - CAST(CONVERT(decimal(10,2), SUM(CASE + END))) + + N' GB | Total estimated savings: ' + + CONVERT(nvarchar(20), CONVERT(decimal(10,4), SUM(CASE WHEN ia.action IN ('DISABLE', 'MERGE INCLUDES', 'MAKE UNIQUE') - THEN ps.total_space_mb + THEN ps.total_space_gb WHEN (ia.action IS NULL OR ia.action = 'KEEP') AND ce.can_compress = 1 - THEN ps.total_space_mb * 0.20 + THEN ps.total_space_gb * 0.20 ELSE 0 - END)) AS nvarchar(20)) + + END))) + N' - ' + - CAST(CONVERT(decimal(10,2), SUM(CASE + CONVERT(nvarchar(20), CONVERT(decimal(10,4), SUM(CASE WHEN ia.action IN ('DISABLE', 'MERGE INCLUDES', 'MAKE UNIQUE') - THEN ps.total_space_mb + THEN ps.total_space_gb WHEN (ia.action IS NULL OR ia.action = 'KEEP') AND ce.can_compress = 1 - THEN ps.total_space_mb * 0.60 + THEN ps.total_space_gb * 0.60 ELSE 0 - END)) AS nvarchar(20)) + - N' MB' -FROM #index_analysis ia -LEFT JOIN #partition_stats ps ON + END))) + + N' GB' +FROM #index_analysis AS ia +LEFT JOIN #partition_stats AS ps ON ia.database_id = ps.database_id AND ia.object_id = ps.object_id AND ia.index_id = ps.index_id -LEFT JOIN #compression_eligibility ce ON +LEFT JOIN #compression_eligibility AS ce ON ia.database_id = ce.database_id AND ia.object_id = ce.object_id AND ia.index_id = ce.index_id; @@ -1843,7 +1844,8 @@ INSERT INTO #index_cleanup_results consolidation_rule, target_index_name, script, - additional_info + additional_info, + superseded_info ) SELECT 'MERGE', @@ -1916,8 +1918,10 @@ CREATE ' WHEN ia.action = 'MERGE INCLUDES' THEN N'This index will absorb includes from duplicate indexes' WHEN ia.action = 'MAKE UNIQUE' THEN N'This index will replace a unique constraint' ELSE NULL - END -FROM #index_analysis ia + END, + /* Add superseded_by information if available */ + ia.superseded_by +FROM #index_analysis AS ia LEFT JOIN ( /* Get the partition info for each index */ @@ -1936,11 +1940,11 @@ LEFT JOIN ps.built_on, ps.partition_function_name, ps.partition_columns -) ps +) AS ps ON ia.database_id = ps.database_id AND ia.object_id = ps.object_id AND ia.index_id = ps.index_id -JOIN #compression_eligibility ce +JOIN #compression_eligibility AS ce ON ia.database_id = ce.database_id AND ia.object_id = ce.object_id AND ia.index_id = ce.index_id @@ -1960,7 +1964,9 @@ INSERT INTO #index_cleanup_results script_type, consolidation_rule, script, - additional_info + additional_info, + target_index_name, + superseded_info ) SELECT 'DISABLE', @@ -1990,8 +1996,10 @@ SELECT WHEN ia.consolidation_rule LIKE 'Unused Index%' THEN ia.consolidation_rule ELSE N'This index is redundant' - END -FROM #index_analysis ia + END, + ia.target_index_name, /* Include the target index name */ + NULL /* Don't need superseded_by info for disabled indexes */ +FROM #index_analysis AS ia WHERE ia.action = 'DISABLE'; /* Insert compression scripts for remaining indexes */ @@ -2005,7 +2013,9 @@ INSERT INTO #index_cleanup_results index_name, script_type, script, - additional_info + additional_info, + target_index_name, + superseded_info ) SELECT 'COMPRESS', @@ -2031,8 +2041,10 @@ SELECT N' WITH (FILLFACTOR = 100, SORT_IN_TEMPDB = ON, ONLINE = ' + CASE WHEN @online = 1 THEN N'ON' ELSE N'OFF' END + N', DATA_COMPRESSION = PAGE);', - N'Compression type: All Partitions' -FROM #index_analysis ia + N'Compression type: All Partitions', + NULL, /* No target index for compression scripts */ + ia.superseded_by /* Include superseded_by info for compression scripts */ +FROM #index_analysis AS ia LEFT JOIN ( /* Get the partition info for each index */ @@ -2096,8 +2108,8 @@ SELECT N' NOCHECK CONSTRAINT ' + QUOTENAME(id.index_name) + N';' -FROM #index_analysis ia -JOIN #index_details id +FROM #index_analysis AS ia +JOIN #index_details AS id ON id.database_id = ia.database_id AND id.object_id = ia.object_id AND id.is_unique_constraint = 1 @@ -2145,7 +2157,9 @@ INSERT INTO #index_cleanup_results index_name, script_type, script, - additional_info + additional_info, + target_index_name, + superseded_info ) SELECT 'COMPRESS_PARTITION', @@ -2164,23 +2178,23 @@ SELECT N'.' + QUOTENAME(ia.table_name) + N' REBUILD PARTITION = ' + - CAST(ps.partition_number AS nvarchar(20)) + + CONVERT(nvarchar(20), ps.partition_number) + N' WITH (FILLFACTOR = 100, SORT_IN_TEMPDB = ON, ONLINE = ' + CASE WHEN @online = 1 THEN N'ON' ELSE N'OFF' END + N', DATA_COMPRESSION = PAGE);', N'Compression type: Per Partition | Partition: ' + - CAST(ps.partition_number AS nvarchar(20)) + + CONVERT(nvarchar(20), ps.partition_number) + N' | Rows: ' + - CAST(ps.total_rows AS nvarchar(20)) + + CONVERT(nvarchar(20), ps.total_rows) + N' | Size: ' + - CAST(CONVERT(decimal(10,2), ps.total_space_mb) AS nvarchar(20)) + - N' MB' -FROM #index_analysis ia -JOIN #partition_stats ps + CONVERT(nvarchar(20), CONVERT(decimal(10,4), ps.total_space_gb)) + + N' GB' +FROM #index_analysis AS ia +JOIN #partition_stats AS ps ON ia.database_id = ps.database_id AND ia.object_id = ps.object_id AND ia.index_id = ps.index_id -JOIN #compression_eligibility ce +JOIN #compression_eligibility AS ce ON ia.database_id = ce.database_id AND ia.object_id = ce.object_id AND ia.index_id = ce.index_id @@ -2217,7 +2231,7 @@ SELECT ce.index_name, 'INELIGIBLE FOR COMPRESSION', ce.reason -FROM #compression_eligibility ce +FROM #compression_eligibility AS ce WHERE ce.can_compress = 0; /* @@ -2233,22 +2247,22 @@ Results are ordered by: */ SELECT /* First, show the information needed to understand the script */ - script_type, - additional_info, + ir.script_type, + ir.additional_info, /* Then show identifying information for the index */ - database_name, - schema_name, - table_name, - index_name, + ir.database_name, + ir.schema_name, + ir.table_name, + ir.index_name, /* Then show relationship information */ - consolidation_rule, - target_index_name, + ir.consolidation_rule, + ir.target_index_name, /* Include superseded_by info for winning indexes */ - CASE WHEN ia.superseded_by IS NOT NULL THEN ia.superseded_by ELSE NULL END AS superseded_info, + CASE WHEN ia.superseded_by IS NOT NULL THEN ia.superseded_by ELSE ir.superseded_info END AS superseded_info, /* Finally show the actual script */ - script -FROM #index_cleanup_results ir -LEFT JOIN #index_analysis ia + ir.script +FROM #index_cleanup_results AS ir +LEFT JOIN #index_analysis AS ia ON ir.database_name = ia.database_name AND ir.schema_name = ia.schema_name AND ir.table_name = ia.table_name From 89afa3f640bfc64b3457853d705de391a27fc4df Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Wed, 12 Mar 2025 17:42:22 -0400 Subject: [PATCH 039/246] bye get rid of old file --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 10 +- .../sp_IndexCleanup BETA_Original.sql | 3547 ----------------- 2 files changed, 6 insertions(+), 3551 deletions(-) delete mode 100644 sp_IndexCleanup BETA/sp_IndexCleanup BETA_Original.sql diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index ddf7dd95..8e678517 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -1093,9 +1093,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. x.partition_id, x.partition_number, x.total_rows, - x.total_space_mb, - x.reserved_lob_mb, - x.reserved_row_overflow_mb, + x.total_space_gb, + x.reserved_lob_gb, + x.reserved_row_overflow_gb, x.data_compression_desc, built_on = ISNULL @@ -2188,7 +2188,9 @@ SELECT CONVERT(nvarchar(20), ps.total_rows) + N' | Size: ' + CONVERT(nvarchar(20), CONVERT(decimal(10,4), ps.total_space_gb)) + - N' GB' + N' GB', + NULL, + NULL FROM #index_analysis AS ia JOIN #partition_stats AS ps ON ia.database_id = ps.database_id diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA_Original.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA_Original.sql deleted file mode 100644 index 6953995f..00000000 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA_Original.sql +++ /dev/null @@ -1,3547 +0,0 @@ -/* -EXEC sp_IndexCleanup - @database_name = 'StackOverflow2013', - @debug = 1; - -EXEC sp_IndexCleanup - @database_name = 'StackOverflow2013', - @table_name = 'Users', - @debug = 1 -*/ - -SET ANSI_WARNINGS ON; -SET ARITHABORT ON; -SET CONCAT_NULL_YIELDS_NULL ON; -SET QUOTED_IDENTIFIER ON; -SET NUMERIC_ROUNDABORT OFF; -SET IMPLICIT_TRANSACTIONS OFF; -SET STATISTICS TIME, IO OFF; -GO - -IF OBJECT_ID('dbo.sp_IndexCleanup', 'P') IS NULL -BEGIN - EXECUTE ('CREATE PROCEDURE dbo.sp_IndexCleanup AS RETURN 138;'); -END; -GO - -ALTER PROCEDURE - dbo.sp_IndexCleanup -( - @database_name sysname = NULL, - @schema_name sysname = NULL, - @table_name sysname = NULL, - @min_reads bigint = 0, - @min_writes bigint = 0, - @min_size_gb decimal(10,2) = 0, - @min_rows bigint = 0, - @help bit = 'false', - @debug bit = 'true', - @version varchar(20) = NULL OUTPUT, - @version_date datetime = NULL OUTPUT -) -WITH RECOMPILE -AS -BEGIN -SET NOCOUNT ON; - -BEGIN TRY - SELECT - @version = '-2147483648', - @version_date = '17530101'; - - SELECT - warning = N'Read the messages pane carefully!' - - PRINT ' -------------------------------------------------------------------------------------------- -------------------------------------------------------------------------------------------- -------------------------------------------------------------------------------------------- -This is the BETA VERSION of sp_IndexCleanup - -It needs lots of love and testing in real environments with real indexes to fix many issues: - * Data collection - * Deduping logic - * Result correctness - * Edge cases - - If you run this, only use the output to debug and validate result correctness. - - Do not run any of the output scripts, period. Doing so may be harmful. - ------------------------------------------------------------------------------------------- - ------------------------------------------------------------------------------------------- - ------------------------------------------------------------------------------------------- - - '; - - - /* - Help section, for help. - Will become more helpful when out of beta. - */ - IF @help = 1 - BEGIN - SELECT - help = N'hello, i am sp_IndexCleanup - BETA' - UNION ALL - SELECT - help = N'this is a script to help clean up unused and duplicate indexes' - UNION ALL - SELECT - help = N'you are currently using a beta version, and the advice should not be followed' - UNION ALL - SELECT - help = N'without careful analysis and consideration. it may be harmful.' - - - /* - Parameters - */ - SELECT - parameter_name = - ap.name, - data_type = - t.name, - description = - CASE - ap.name - WHEN ap.name - THEN ap.name - END, - valid_inputs = - CASE - ap.name - WHEN ap.name - THEN ap.name - END, - defaults = - CASE - ap.name - WHEN ap.name - THEN ap.name - END - FROM sys.all_parameters AS ap - INNER JOIN sys.all_objects AS o - ON ap.object_id = o.object_id - INNER JOIN sys.types AS t - ON ap.system_type_id = t.system_type_id - AND ap.user_type_id = t.user_type_id - WHERE o.name = N'sp_IndexCleanup' - OPTION(MAXDOP 1, RECOMPILE); - - SELECT - mit_license_yo = 'i am MIT licensed, so like, do whatever' - - UNION ALL - - SELECT - mit_license_yo = 'see printed messages for full license'; - - RAISERROR(' -MIT License - -Copyright 2024 Darling Data, LLC - -https://www.erikdarling.com/ - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), -to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, -sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the -following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE -FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -', 0, 1) WITH NOWAIT; - - RETURN; - END; - - IF @debug = 1 - BEGIN - RAISERROR('Declaring variables', 0, 0) WITH NOWAIT; - END; - - DECLARE - /*general script variables*/ - @sql nvarchar(max) = N'', - @database_id integer = NULL, - @object_id integer = NULL, - @full_object_name nvarchar(768) = NULL, - @final_script nvarchar(max) = '', - /*cursor variables*/ - @c_database_id integer, - @c_database_name sysname, - @c_schema_id integer, - @c_schema_name sysname, - @c_object_id integer, - @c_table_name sysname, - @c_index_id integer, - @c_index_name sysname, - @c_is_unique bit, - @c_filter_definition nvarchar(max), - @index_cursor CURSOR, - /*print variables*/ - @helper integer = 0, - @sql_len integer, - @sql_debug nvarchar(max) = N'', - @online bit = - CASE - WHEN - CONVERT - ( - integer, - SERVERPROPERTY('EngineEdition') - ) IN (3, 5, 8) - THEN 'true' /* Enterprise, Azure SQL DB, Managed Instance */ - ELSE 'false' - END, - @uptime_days nvarchar(10) = - ( - SELECT - DATEDIFF - ( - DAY, - osi.sqlserver_start_time, - SYSDATETIME() - ) - FROM sys.dm_os_sys_info AS osi - ); - - /* - Initial checks for object validity - */ - IF @debug = 1 - BEGIN - RAISERROR('Checking paramaters...', 0, 0) WITH NOWAIT; - END; - - IF @database_name IS NULL - AND DB_NAME() NOT IN - ( - N'master', - N'model', - N'msdb', - N'tempdb', - N'rdsadmin' - ) - BEGIN - SELECT - @database_name = DB_NAME(); - END; - - IF @database_name IS NOT NULL - BEGIN - SELECT - @database_id = d.database_id - FROM sys.databases AS d - WHERE d.name = @database_name; - END; - - IF @schema_name IS NULL - AND @table_name IS NOT NULL - BEGIN - SELECT - @schema_name = N'dbo'; - END; - - IF @schema_name IS NOT NULL - AND @table_name IS NOT NULL - BEGIN - SELECT - @full_object_name = - QUOTENAME(@database_name) + - N'.' + - QUOTENAME(@schema_name) + - N'.' + - QUOTENAME(@table_name); - - SELECT - @object_id = - OBJECT_ID(@full_object_name); - - IF @object_id IS NULL - BEGIN - RAISERROR('The object %s doesn''t seem to exist', 16, 1, @full_object_name) WITH NOWAIT; - RETURN; - END; - END; - - -- Parameter validation - IF @min_reads < 0 - OR @min_reads IS NULL - BEGIN - RAISERROR('Parameter @min_reads cannot be NULL or negative. Setting to 0.', 10, 1) WITH NOWAIT; - SET @min_reads = 0; - END; - - IF @min_writes < 0 - OR @min_writes IS NULL - BEGIN - RAISERROR('Parameter @min_writes cannot be NULL or negative. Setting to 0.', 10, 1) WITH NOWAIT; - SET @min_writes = 0; - END; - - IF @min_size_gb < 0 - OR @min_size_gb IS NULL - BEGIN - RAISERROR('Parameter @min_size_gb cannot be NULL or negative. Setting to 0.', 10, 1) WITH NOWAIT; - SET @min_size_gb = 0; - END; - - IF @min_rows < 0 - OR @min_rows IS NULL - BEGIN - RAISERROR('Parameter @min_rows cannot be NULL or negative. Setting to 0.', 10, 1) WITH NOWAIT; - SET @min_rows = 0; - END; - - /* - Temp tables! - */ - - IF @debug = 1 - BEGIN - RAISERROR('Creating temp tables', 0, 0) WITH NOWAIT; - END; - - CREATE TABLE - #filtered_objects - ( - database_id integer NOT NULL, - database_name sysname NOT NULL, - schema_id integer NOT NULL, - schema_name sysname NOT NULL, - object_id integer NOT NULL, - table_name sysname NOT NULL, - index_id integer NOT NULL, - index_name sysname NOT NULL - PRIMARY KEY - (database_id, schema_id, object_id, index_id) - ); - - CREATE TABLE - #operational_stats - ( - database_id integer NOT NULL, - database_name sysname NOT NULL, - schema_id integer NOT NULL, - schema_name sysname NOT NULL, - object_id integer NOT NULL, - table_name sysname NOT NULL, - index_id integer NOT NULL, - index_name sysname NOT NULL, - range_scan_count bigint NULL, - singleton_lookup_count bigint NULL, - forwarded_fetch_count bigint NULL, - lob_fetch_in_pages bigint NULL, - row_overflow_fetch_in_pages bigint NULL, - leaf_insert_count bigint NULL, - leaf_update_count bigint NULL, - leaf_delete_count bigint NULL, - leaf_ghost_count bigint NULL, - nonleaf_insert_count bigint NULL, - nonleaf_update_count bigint NULL, - nonleaf_delete_count bigint NULL, - leaf_allocation_count bigint NULL, - nonleaf_allocation_count bigint NULL, - row_lock_count bigint NULL, - row_lock_wait_count bigint NULL, - row_lock_wait_in_ms bigint NULL, - page_lock_count bigint NULL, - page_lock_wait_count bigint NULL, - page_lock_wait_in_ms bigint NULL, - index_lock_promotion_attempt_count bigint NULL, - index_lock_promotion_count bigint NULL, - page_latch_wait_count bigint NULL, - page_latch_wait_in_ms bigint NULL, - tree_page_latch_wait_count bigint NULL, - tree_page_latch_wait_in_ms bigint NULL, - page_io_latch_wait_count bigint NULL, - page_io_latch_wait_in_ms bigint NULL, - page_compression_attempt_count bigint NULL, - page_compression_success_count bigint NULL, - PRIMARY KEY CLUSTERED - (database_id, schema_id, object_id, index_id) - ); - - CREATE TABLE - #index_details - ( - database_id integer NOT NULL, - database_name sysname NOT NULL, - schema_id integer NOT NULL, - schema_name sysname NOT NULL, - object_id integer NOT NULL, - table_name sysname NOT NULL, - index_id integer NOT NULL, - index_name sysname NULL, - column_name sysname NOT NULL, - is_primary_key bit NULL, - is_unique bit NULL, - is_unique_constraint bit NULL, - is_indexed_view integer NOT NULL, - is_foreign_key bit NULL, - is_foreign_key_reference bit NULL, - key_ordinal tinyint NOT NULL, - index_column_id integer NOT NULL, - is_descending_key bit NOT NULL, - is_included_column bit NULL, - filter_definition nvarchar(max) NULL, - is_max_length integer NOT NULL, - user_seeks bigint NOT NULL, - user_scans bigint NOT NULL, - user_lookups bigint NOT NULL, - user_updates bigint NOT NULL, - last_user_seek datetime NULL, - last_user_scan datetime NULL, - last_user_lookup datetime NULL, - last_user_update datetime NULL, - is_eligible_for_dedupe bit NOT NULL - PRIMARY KEY CLUSTERED(database_id, object_id, index_id, column_name) - ); - - CREATE TABLE - #partition_stats - ( - database_id integer NOT NULL, - database_name sysname NOT NULL, - schema_id integer NOT NULL, - schema_name sysname NOT NULL, - object_id integer NOT NULL, - table_name sysname NOT NULL, - index_id integer NOT NULL, - index_name sysname NULL, - partition_id bigint NOT NULL, - partition_number int NOT NULL, - total_rows bigint NULL, - total_space_mb decimal(38, 2) NULL, - reserved_lob_mb decimal(38, 2) NULL, - reserved_row_overflow_mb decimal(38, 2) NULL, - data_compression_desc nvarchar(60) NULL, - built_on sysname NULL, - partition_function_name sysname NULL, - partition_columns nvarchar(max) - PRIMARY KEY CLUSTERED(database_id, object_id, index_id, partition_id) - ); - - CREATE TABLE - #index_analysis - ( - database_id integer NOT NULL, - database_name sysname NOT NULL, - schema_id integer NOT NULL, - schema_name sysname NOT NULL, - object_id integer NOT NULL, - table_name sysname NOT NULL, - index_id integer NULL, - index_name sysname NOT NULL, - is_unique bit NULL, - key_columns nvarchar(max) NULL, - included_columns nvarchar(max) NULL, - filter_definition nvarchar(max) NULL, - is_redundant bit NULL, - superseded_by sysname NULL, - missing_columns nvarchar(max) NULL, - action nvarchar(max) NULL, - INDEX c CLUSTERED - (database_id, schema_name, table_name, index_name) - ); - - CREATE TABLE - #index_cleanup_report - ( - database_id integer NOT NULL, - database_name sysname NOT NULL, - schema_id integer NOT NULL, - schema_name sysname NOT NULL, - object_id integer NOT NULL, - table_name sysname NOT NULL, - index_id integer NOT NULL, - index_name sysname NOT NULL, - action nvarchar(max) NULL, - cleanup_script nvarchar(max) NULL, - original_definition nvarchar(max) NULL, - /*Usage details*/ - user_seeks bigint NULL, - user_scans bigint NULL, - user_lookups bigint NULL, - user_updates bigint NULL, - last_user_seek datetime NULL, - last_user_scan datetime NULL, - last_user_lookup datetime NULL, - last_user_update datetime NULL, - /*Operational stats*/ - range_scan_count bigint NULL, - singleton_lookup_count bigint NULL, - leaf_insert_count bigint NULL, - leaf_update_count bigint NULL, - leaf_delete_count bigint NULL, - page_lock_count bigint NULL, - page_lock_wait_count bigint NULL, - page_lock_wait_in_ms bigint NULL - ); - - CREATE TABLE - #index_cleanup_summary - ( - database_id integer NOT NULL, - database_name sysname NOT NULL, - schema_id integer NOT NULL, - schema_name sysname NOT NULL, - object_id integer NOT NULL, - table_name sysname NOT NULL, - index_id integer NOT NULL, - index_name sysname NOT NULL, - action nvarchar(max) NOT NULL, - details nvarchar(max) NULL, - current_definition nvarchar(max) NOT NULL, - proposed_definition nvarchar(max) NULL, - usage_summary nvarchar(max) NULL, - operational_summary nvarchar(max) NULL, - uptime_warning nvarchar(512) NULL - ); - - CREATE TABLE - #final_index_actions - ( - database_id integer NOT NULL, - database_name sysname NOT NULL, - schema_id integer NOT NULL, - schema_name sysname NOT NULL, - object_id integer NOT NULL, - table_name sysname NOT NULL, - index_id integer NOT NULL, - index_name sysname NOT NULL, - target_index_name sysname NULL, - action nvarchar(max) NOT NULL, - script nvarchar(max) NOT NULL - ); - - /* - Start insert queries - */ - - IF @debug = 1 - BEGIN - RAISERROR('Generating #filtered_object insert', 0, 0) WITH NOWAIT; - END; - - SELECT - @sql = N' - SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;'; - - SELECT - @sql = N' - SELECT DISTINCT - @database_id, - database_name = DB_NAME(@database_id), - schema_id = t.schema_id, - schema_name = s.name, - object_id = t.object_id, - table_name = t.name, - index_id = i.index_id, - index_name = i.name - FROM ' + QUOTENAME(@database_name) + N'.sys.tables AS t - JOIN ' + QUOTENAME(@database_name) + N'.sys.schemas AS s - ON t.schema_id = s.schema_id - JOIN ' + QUOTENAME(@database_name) + N'.sys.indexes AS i - ON t.object_id = i.object_id - LEFT JOIN ' + QUOTENAME(@database_name) + N'.sys.dm_db_index_usage_stats AS us - ON t.object_id = us.object_id - AND us.database_id = @database_id - WHERE t.is_ms_shipped = 0 - AND t.type <> N''TF''' - - IF @object_id IS NOT NULL - BEGIN - SELECT @sql += N' - AND t.object_id = @object_id'; - END; - - SET @sql += N' - AND EXISTS - ( - SELECT - 1/0 - FROM ' + QUOTENAME(@database_name) + N'.sys.dm_db_partition_stats AS ps - JOIN ' + QUOTENAME(@database_name) + N'.sys.allocation_units AS au - ON ps.partition_id = au.container_id - WHERE ps.object_id = t.object_id - GROUP - BY ps.object_id - HAVING - SUM(au.total_pages) * 8.0 / 1048576.0 >= @min_size_gb - ) - AND EXISTS - ( - SELECT - 1/0 - FROM ' + QUOTENAME(@database_name) + N'.sys.dm_db_partition_stats AS ps - WHERE ps.object_id = t.object_id - AND ps.index_id IN (0, 1) - GROUP - BY ps.object_id - HAVING - SUM(ps.row_count) >= @min_rows - ) - AND EXISTS - ( - SELECT - 1/0 - FROM ' + QUOTENAME(@database_name) + N'.sys.dm_db_index_usage_stats AS ius - WHERE ius.object_id = t.object_id - AND ius.database_id = @database_id - GROUP BY - ius.object_id - HAVING - SUM(ius.user_seeks + ius.user_scans + ius.user_lookups) >= @min_reads - AND - SUM(ius.user_updates) >= @min_writes - ) - OPTION(RECOMPILE);'; - - IF @debug = 1 - BEGIN - PRINT @sql; - END; - - INSERT - #filtered_objects - WITH - (TABLOCK) - ( - database_id, - database_name, - schema_id, - schema_name, - object_id, - table_name, - index_id, - index_name - ) - EXEC sys.sp_executesql - @sql, - N'@database_id int, - @min_reads bigint, - @min_writes bigint, - @min_size_gb decimal(10,2), - @min_rows bigint, - @object_id integer', - @database_id, - @min_reads, - @min_writes, - @min_size_gb, - @min_rows, - @object_id; - - IF ROWCOUNT_BIG() = 0 BEGIN IF @debug = 1 BEGIN RAISERROR('No rows inserted into #filtered_objects', 0, 0) WITH NOWAIT END; END; - - IF @debug = 1 - BEGIN - SELECT - table_name = '#filtered_objects', - fo.* - FROM #filtered_objects AS fo; - END; - - IF @debug = 1 - BEGIN - RAISERROR('Generating #operational_stats insert', 0, 0) WITH NOWAIT; - END; - - SELECT - @sql = N' - SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;'; - - SELECT - @sql += N' - SELECT - os.database_id, - database_name = DB_NAME(os.database_id), - schema_id = s.schema_id, - schema_name = s.name, - os.object_id, - table_name = t.name, - os.index_id, - index_name = i.name, - range_scan_count = SUM(os.range_scan_count), - singleton_lookup_count = SUM(os.singleton_lookup_count), - forwarded_fetch_count = SUM(os.forwarded_fetch_count), - lob_fetch_in_pages = SUM(os.lob_fetch_in_pages), - row_overflow_fetch_in_pages = SUM(os.row_overflow_fetch_in_pages), - leaf_insert_count = SUM(os.leaf_insert_count), - leaf_update_count = SUM(os.leaf_update_count), - leaf_delete_count = SUM(os.leaf_delete_count), - leaf_ghost_count = SUM(os.leaf_ghost_count), - nonleaf_insert_count = SUM(os.nonleaf_insert_count), - nonleaf_update_count = SUM(os.nonleaf_update_count), - nonleaf_delete_count = SUM(os.nonleaf_delete_count), - leaf_allocation_count = SUM(os.leaf_allocation_count), - nonleaf_allocation_count = SUM(os.nonleaf_allocation_count), - row_lock_count = SUM(os.row_lock_count), - row_lock_wait_count = SUM(os.row_lock_wait_count), - row_lock_wait_in_ms = SUM(os.row_lock_wait_in_ms), - page_lock_count = SUM(os.page_lock_count), - page_lock_wait_count = SUM(os.page_lock_wait_count), - page_lock_wait_in_ms = SUM(os.page_lock_wait_in_ms), - index_lock_promotion_attempt_count = SUM(os.index_lock_promotion_attempt_count), - index_lock_promotion_count = SUM(os.index_lock_promotion_count), - page_latch_wait_count = SUM(os.page_latch_wait_count), - page_latch_wait_in_ms = SUM(os.page_latch_wait_in_ms), - tree_page_latch_wait_count = SUM(os.tree_page_latch_wait_count), - tree_page_latch_wait_in_ms = SUM(os.tree_page_latch_wait_in_ms), - page_io_latch_wait_count = SUM(os.page_io_latch_wait_count), - page_io_latch_wait_in_ms = SUM(os.page_io_latch_wait_in_ms), - page_compression_attempt_count = SUM(os.page_compression_attempt_count), - page_compression_success_count = SUM(os.page_compression_success_count) - FROM ' + QUOTENAME(@database_name) + N'.sys.dm_db_index_operational_stats - ( - @database_id, - @object_id, - NULL, - NULL - ) AS os - JOIN ' + QUOTENAME(@database_name) + N'.sys.tables AS t - ON os.object_id = t.object_id - JOIN ' + QUOTENAME(@database_name) + N'.sys.schemas AS s - ON t.schema_id = s.schema_id - JOIN ' + QUOTENAME(@database_name) + N'.sys.indexes AS i - ON os.object_id = i.object_id - AND os.index_id = i.index_id - WHERE EXISTS - ( - SELECT - 1/0 - FROM #filtered_objects fo - WHERE fo.database_id = os.database_id - AND fo.object_id = os.object_id - ) - GROUP BY - os.database_id, - DB_NAME(os.database_id), - s.schema_id, - s.name, - os.object_id, - t.name, - os.index_id, - i.name - OPTION(RECOMPILE);'; - - IF @debug = 1 - BEGIN - PRINT @sql; - END; - - INSERT - #operational_stats - WITH - (TABLOCK) - ( - database_id, - database_name, - schema_id, - schema_name, - object_id, - table_name, - index_id, - index_name, - range_scan_count, - singleton_lookup_count, - forwarded_fetch_count, - lob_fetch_in_pages, - row_overflow_fetch_in_pages, - leaf_insert_count, - leaf_update_count, - leaf_delete_count, - leaf_ghost_count, - nonleaf_insert_count, - nonleaf_update_count, - nonleaf_delete_count, - leaf_allocation_count, - nonleaf_allocation_count, - row_lock_count, - row_lock_wait_count, - row_lock_wait_in_ms, - page_lock_count, - page_lock_wait_count, - page_lock_wait_in_ms, - index_lock_promotion_attempt_count, - index_lock_promotion_count, - page_latch_wait_count, - page_latch_wait_in_ms, - tree_page_latch_wait_count, - tree_page_latch_wait_in_ms, - page_io_latch_wait_count, - page_io_latch_wait_in_ms, - page_compression_attempt_count, - page_compression_success_count - ) - EXEC sys.sp_executesql - @sql, - N'@database_id integer, - @object_id integer', - @database_id, - @object_id; - - IF ROWCOUNT_BIG() = 0 BEGIN IF @debug = 1 BEGIN RAISERROR('No rows inserted into #operational_stats', 0, 0) WITH NOWAIT END; END; - - IF @debug = 1 - BEGIN - SELECT - table_name = '#operational_stats', - os.* - FROM #operational_stats AS os; - END; - - IF @debug = 1 - BEGIN - RAISERROR('Generating #index_details insert', 0, 0) WITH NOWAIT; - END; - - SELECT - @sql = N' - SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;'; - - SELECT - @sql += N' - SELECT - database_id = @database_id, - database_name = DB_NAME(@database_id), - t.object_id, - i.index_id, - s.schema_id, - schema_name = s.name, - table_name = t.name, - index_name = i.name, - column_name = c.name, - i.is_primary_key, - i.is_unique, - i.is_unique_constraint, - is_indexed_view = - CASE - WHEN EXISTS - ( - SELECT - 1/0 - FROM ' + QUOTENAME(@database_name) + N'.sys.objects AS so - WHERE i.object_id = so.object_id - AND so.is_ms_shipped = 0 - AND so.type = ''V'' - ) - THEN 1 - ELSE 0 - END, - is_foreign_key = - CASE - WHEN EXISTS - ( - SELECT - 1/0 - FROM ' + QUOTENAME(@database_name) + N'.sys.foreign_key_columns AS f - WHERE f.parent_column_id = c.column_id - AND f.parent_object_id = c.object_id - ) - THEN 1 - ELSE 0 - END, - is_foreign_key_reference = - CASE - WHEN EXISTS - ( - SELECT - 1/0 - FROM ' + QUOTENAME(@database_name) + N'.sys.foreign_key_columns AS f - WHERE f.referenced_column_id = c.column_id - AND f.referenced_object_id = c.object_id - ) - THEN 1 - ELSE 0 - END, - ic.key_ordinal, - ic.index_column_id, - ic.is_descending_key, - ic.is_included_column, - i.filter_definition, - is_max_length = - CASE - WHEN EXISTS - ( - SELECT - 1/0 - FROM ' + QUOTENAME(@database_name) + N'.sys.types AS t - WHERE c.system_type_id = t.system_type_id - AND c.user_type_id = t.user_type_id - AND t.name IN (N''varchar'', N''nvarchar'') - AND t.max_length = -1 - ) - THEN 1 - ELSE 0 - END, - user_seeks = ISNULL(us.user_seeks, 0), - user_scans = ISNULL(us.user_scans, 0), - user_lookups = ISNULL(us.user_lookups, 0), - user_updates = ISNULL(us.user_updates, 0), - us.last_user_seek, - us.last_user_scan, - us.last_user_lookup, - us.last_user_update, - is_eligible_for_dedupe = - CASE - WHEN i.type = 2 - THEN 1 - WHEN i.type = 1 - THEN 0 - END - FROM ' + QUOTENAME(@database_name) + N'.sys.tables AS t - JOIN ' + QUOTENAME(@database_name) + N'.sys.schemas AS s - ON t.schema_id = s.schema_id - JOIN ' + QUOTENAME(@database_name) + N'.sys.indexes AS i - ON t.object_id = i.object_id - JOIN ' + QUOTENAME(@database_name) + N'.sys.index_columns AS ic - ON i.object_id = ic.object_id - AND i.index_id = ic.index_id - JOIN ' + QUOTENAME(@database_name) + N'.sys.columns AS c - ON ic.object_id = c.object_id - AND ic.column_id = c.column_id - LEFT JOIN sys.dm_db_index_usage_stats AS us - ON i.object_id = us.object_id - AND i.index_id = us.index_id - AND us.database_id = @database_id - WHERE t.is_ms_shipped = 0 - AND i.type IN (1, 2) - AND i.is_disabled = 0 - AND i.is_hypothetical = 0 - AND EXISTS - ( - SELECT - 1/0 - FROM #filtered_objects fo - WHERE fo.database_id = @database_id - AND fo.object_id = t.object_id - ) - AND EXISTS - ( - SELECT - 1/0 - FROM ' + QUOTENAME(@database_name) + - CONVERT - ( - nvarchar(MAX), - N'.sys.dm_db_partition_stats ps - WHERE ps.object_id = t.object_id - AND ps.index_id = 1 - AND ps.row_count >= @min_rows - )' - ); - - IF @object_id IS NOT NULL - BEGIN - SELECT @sql += N' - AND t.object_id = @object_id'; - END; - - SELECT - @sql += CONVERT - ( - nvarchar(max), - N' - AND NOT EXISTS - ( - SELECT - 1/0 - FROM ' + QUOTENAME(@database_name) + N'.sys.objects AS so - WHERE i.object_id = so.object_id - AND so.is_ms_shipped = 0 - AND so.type = N''TF'' - ) - OPTION(RECOMPILE);' - ); - - IF @debug = 1 - BEGIN - PRINT SUBSTRING(@sql, 1, 4000); - PRINT SUBSTRING(@sql, 4000, 8000); - END; - - INSERT - #index_details - WITH - (TABLOCK) - ( - database_id, - database_name, - object_id, - index_id, - schema_id, - schema_name, - table_name, - index_name, - column_name, - is_primary_key, - is_unique, - is_unique_constraint, - is_indexed_view, - is_foreign_key, - is_foreign_key_reference, - key_ordinal, - index_column_id, - is_descending_key, - is_included_column, - filter_definition, - is_max_length, - user_seeks, - user_scans, - user_lookups, - user_updates, - last_user_seek, - last_user_scan, - last_user_lookup, - last_user_update, - is_eligible_for_dedupe - ) - EXEC sys.sp_executesql - @sql, - N'@database_id integer, - @object_id integer, - @min_rows integer', - @database_id, - @object_id, - @min_rows; - - IF ROWCOUNT_BIG() = 0 BEGIN IF @debug = 1 BEGIN RAISERROR('No rows inserted into #index_details', 0, 0) WITH NOWAIT END; END; - - IF @debug = 1 - BEGIN - SELECT - table_name = '#index_details', - * - FROM #index_details AS id; - END; - - IF @debug = 1 - BEGIN - RAISERROR('Generating #partition_stats insert', 0, 0) WITH NOWAIT; - END; - - SELECT - @sql = N' - SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;'; - - SELECT - @sql += N' - SELECT - database_id = @database_id, - database_name = DB_NAME(@database_id), - x.object_id, - x.index_id, - x.schema_id, - x.schema_name, - x.table_name, - x.index_name, - x.partition_id, - x.partition_number, - x.total_rows, - x.total_space_mb, - x.reserved_lob_mb, - x.reserved_row_overflow_mb, - x.data_compression_desc, - built_on = - ISNULL - ( - psfg.partition_scheme_name, - psfg.filegroup_name - ), - psfg.partition_function_name, - pc.partition_columns - FROM - ( - SELECT - ps.object_id, - ps.index_id, - s.schema_id, - schema_name = s.name, - table_name = t.name, - index_name = i.name, - ps.partition_id, - p.partition_number, - total_rows = SUM(ps.row_count), - total_space_mb = SUM(a.total_pages) * 8 / 1024.0, - reserved_lob_mb = SUM(ps.lob_reserved_page_count) * 8. / 1024., - reserved_row_overflow_mb = SUM(ps.row_overflow_reserved_page_count) * 8. / 1024., - p.data_compression_desc, - i.data_space_id - FROM ' + QUOTENAME(@database_name) + N'.sys.tables AS t - JOIN ' + QUOTENAME(@database_name) + N'.sys.indexes AS i - ON t.object_id = i.object_id - JOIN ' + QUOTENAME(@database_name) + N'.sys.schemas AS s - ON t.schema_id = s.schema_id - JOIN ' + QUOTENAME(@database_name) + N'.sys.partitions AS p - ON i.object_id = p.object_id - AND i.index_id = p.index_id - JOIN ' + QUOTENAME(@database_name) + N'.sys.allocation_units AS a - ON p.partition_id = a.container_id - LEFT HASH JOIN ' + QUOTENAME(@database_name) + N'.sys.dm_db_partition_stats AS ps - ON p.partition_id = ps.partition_id - WHERE t.type <> N''TF'' - AND i.type IN (1, 2) - AND EXISTS - ( - SELECT - 1/0 - FROM #filtered_objects fo - WHERE fo.database_id = @database_id - AND fo.object_id = t.object_id - )'; - - IF @object_id IS NOT NULL - BEGIN - SELECT @sql += N' - AND t.object_id = @object_id'; - END; - - SELECT - @sql += N' - GROUP BY - t.name, - i.name, - i.data_space_id, - s.schema_id, - s.name, - p.partition_number, - p.data_compression_desc, - ps.object_id, - ps.index_id, - ps.partition_id - ) AS x - OUTER APPLY - ( - SELECT - filegroup_name = - fg.name, - partition_scheme_name = - ps.name, - partition_function_name = - pf.name - FROM ' + QUOTENAME(@database_name) + N'.sys.filegroups AS fg - FULL JOIN ' + QUOTENAME(@database_name) + N'.sys.partition_schemes AS ps - ON ps.data_space_id = fg.data_space_id - LEFT JOIN ' + QUOTENAME(@database_name) + N'.sys.partition_functions AS pf - ON pf.function_id = ps.function_id - WHERE x.data_space_id = fg.data_space_id - OR x.data_space_id = ps.data_space_id - ) AS psfg - OUTER APPLY - ( - SELECT - partition_columns = - STUFF - ( - ( - SELECT - N'', '' + - c.name - FROM ' + QUOTENAME(@database_name) + N'.sys.index_columns AS ic - JOIN ' + QUOTENAME(@database_name) + N'.sys.columns AS c - ON c.object_id = ic.object_id - AND c.column_id = ic.column_id - WHERE ic.object_id = x.object_id - AND ic.index_id = x.index_id - AND ic.partition_ordinal > 0 - ORDER BY - ic.partition_ordinal - FOR XML - PATH(''''), - TYPE - ).value(''.'', ''nvarchar(max)''), - 1, - 2, - '''' - ) - ) AS pc - OPTION(RECOMPILE);'; - - IF @debug = 1 - BEGIN - PRINT @sql; - END; - - INSERT - #partition_stats WITH(TABLOCK) - ( - database_id, - database_name, - object_id, - index_id, - schema_id, - schema_name, - table_name, - index_name, - partition_id, - partition_number, - total_rows, - total_space_mb, - reserved_lob_mb, - reserved_row_overflow_mb, - data_compression_desc, - built_on, - partition_function_name, - partition_columns - ) - EXEC sys.sp_executesql - @sql, - N'@database_id integer, - @object_id integer', - @database_id, - @object_id; - - IF ROWCOUNT_BIG() = 0 BEGIN IF @debug = 1 BEGIN RAISERROR('No rows inserted into #partition_stats', 0, 0) WITH NOWAIT END; END; - - IF @debug = 1 - BEGIN - SELECT - table_name = '#partition_stats', - * - FROM #partition_stats AS ps; - END; - - IF @debug = 1 - BEGIN - RAISERROR('Performing #index_analysis insert', 0, 0) WITH NOWAIT; - END; - - INSERT INTO - #index_analysis - WITH - (TABLOCK) - ( - database_id, - database_name, - schema_id, - schema_name, - table_name, - object_id, - index_id, - index_name, - is_unique, - key_columns, - included_columns, - filter_definition - ) - SELECT - @database_id, - database_name = DB_NAME(@database_id), - id1.schema_id, - id1.schema_name, - id1.table_name, - id1.object_id, - id1.index_id, - id1.index_name, - id1.is_unique, - key_columns = - STUFF - ( - ( - SELECT - N', ' + - id2.column_name + - CASE - WHEN id2.is_descending_key = 1 - THEN N' DESC' - ELSE N'' - END - FROM #index_details id2 - WHERE id2.object_id = id1.object_id - AND id2.index_id = id1.index_id - AND id2.is_included_column = 0 - GROUP BY - id2.column_name, - id2.is_descending_key, - id2.key_ordinal - ORDER BY - id2.key_ordinal - FOR XML - PATH(''), - TYPE - ).value('text()[1]','nvarchar(max)'), - 1, - 2, - '' - ), - included_columns = - STUFF - ( - ( - SELECT - N', ' + - id2.column_name - FROM #index_details id2 - WHERE id2.object_id = id1.object_id - AND id2.index_id = id1.index_id - AND id2.is_included_column = 1 - GROUP BY - id2.column_name - ORDER BY - id2.column_name - FOR XML - PATH(''), - TYPE - ).value('text()[1]','nvarchar(max)'), - 1, - 2, - '' - ), - id1.filter_definition - FROM #index_details id1 - WHERE id1.is_eligible_for_dedupe = 1 - GROUP BY - id1.schema_name, - id1.schema_id, - id1.table_name, - id1.index_name, - id1.index_id, - id1.is_unique, - id1.object_id, - id1.index_id, - id1.filter_definition - OPTION(RECOMPILE); - - IF ROWCOUNT_BIG() = 0 BEGIN IF @debug = 1 BEGIN RAISERROR('No rows inserted into #index_analysis', 0, 0) WITH NOWAIT END; END; - - IF @debug = 1 - BEGIN - SELECT - table_name = '#index_analysis', - ia.* - FROM #index_analysis AS ia; - END; - - IF @debug = 1 - BEGIN - RAISERROR('Starting cursor', 0, 0) WITH NOWAIT; - END; - - CREATE TABLE #index_supersede_debug - ( - id integer IDENTITY(1,1) PRIMARY KEY, - step nvarchar(100), - current_index sysname, - other_index sysname, - current_key_columns nvarchar(max), - other_key_columns nvarchar(max), - current_include_columns nvarchar(max), - other_include_columns nvarchar(max), - decision nvarchar(100), - reason nvarchar(max) - ); - - DECLARE - @current_key_cols nvarchar(max) = N'', - @current_include_cols nvarchar(max) = N''; - - - /*Analyze indexes*/ - SET @index_cursor = CURSOR - LOCAL - STATIC - FORWARD_ONLY - READ_ONLY - FOR - SELECT DISTINCT - ia.database_id, - ia.database_name, - ia.schema_id, - ia.schema_name, - ia.object_id, - ia.table_name, - ia.index_id, - ia.index_name, - ia.is_unique, - ia.filter_definition - FROM #index_analysis AS ia - ORDER BY - ia.table_name, - ia.index_name; - - OPEN @index_cursor; - - FETCH NEXT - FROM @index_cursor - INTO - @c_database_id, - @c_database_name, - @c_schema_id, - @c_schema_name, - @c_object_id, - @c_table_name, - @c_index_id, - @c_index_name, - @c_is_unique, - @c_filter_definition; - - WHILE @@FETCH_STATUS = 0 - BEGIN - - IF @debug = 1 - BEGIN - RAISERROR('Performing #index_analysis update', 0, 0) WITH NOWAIT; - END; - - SELECT - @current_key_cols = - STUFF - ( - ( - SELECT - N', ' + - id.column_name + - CASE - WHEN id.is_descending_key = 1 - THEN N' DESC' - ELSE '' - END - FROM #index_details AS id - WHERE id.database_id = @c_database_id - AND id.object_id = @c_object_id - AND id.index_id = @c_index_id - AND id.is_included_column = 0 - AND id.key_ordinal > 0 - ORDER BY - id.key_ordinal - FOR - XML - PATH(''), - TYPE - ).value('.', 'nvarchar(max)'), - 1, - 2, - '' - ); - - SELECT - @current_include_cols = - STUFF - ( - ( - SELECT - N', ' + - id.column_name - FROM #index_details AS id - WHERE id.database_id = @c_database_id - AND id.object_id = @c_object_id - AND id.index_id = @c_index_id - AND id.is_included_column = 1 - ORDER BY - id.column_name - FOR XML - PATH(''), - TYPE - ).value('.', 'nvarchar(max)'), - 1, - 2, - '' - ); - - INSERT INTO - #index_supersede_debug - ( - step, - current_index, - other_index, - current_key_columns, - other_key_columns, - current_include_columns, - other_include_columns, - decision, - reason - ) - SELECT - 'Before Update', - @c_index_name, - other_indexes.other_index_name, - @current_key_cols, - other_indexes.other_key_cols, - @current_include_cols, - other_indexes.other_include_cols, - 'Checking', - 'Starting comparison' - FROM - ( - -- Get other indexes for the same table - SELECT - other_index_name = id2.index_name, - other_key_cols = - STUFF - ( - ( - SELECT - N', ' + - id3.column_name + - CASE - WHEN id3.is_descending_key = 1 - THEN N' DESC' - ELSE N'' - END - FROM #index_details AS id3 - WHERE id3.database_id = id2.database_id - AND id3.object_id = id2.object_id - AND id3.index_id = id2.index_id - AND id3.is_included_column = 0 - AND id3.key_ordinal > 0 - ORDER BY - id3.key_ordinal - FOR XML - PATH(''), - TYPE - ).value('.', 'nvarchar(max)'), - 1, - 2, - '' - ), - other_include_cols = - STUFF - ( - ( - SELECT - N', ' + - id3.column_name - FROM #index_details AS id3 - WHERE id3.database_id = id2.database_id - AND id3.object_id = id2.object_id - AND id3.index_id = id2.index_id - AND id3.is_included_column = 1 - ORDER BY - id3.column_name - FOR XML - PATH(''), - TYPE - ).value('.', 'nvarchar(max)'), - 1, - 2, - '' - ) - FROM #index_details AS id2 - WHERE id2.database_id = @c_database_id - AND id2.object_id = @c_object_id - AND id2.index_id <> @c_index_id - AND - ( - id2.index_name = N'IX_Users_AccountId_DisplayName' - OR id2.index_name = N'u' - ) -- Focus on problem indexes - GROUP BY - id2.database_id, - id2.object_id, - id2.index_id, - id2.index_name - ) AS other_indexes - WHERE - ( - @c_index_name = N'IX_Users_AccountId_DisplayName' - OR @c_index_name = N'u' - OR -- Focus on problem indexes - other_indexes.other_index_name = N'IX_Users_AccountId_DisplayName' - OR other_indexes.other_index_name = N'u' - ); - - - WITH - IndexColumns AS - ( - SELECT - id.* - FROM #index_details id - WHERE id.database_id = @c_database_id - AND id.object_id = @c_object_id - AND id.is_eligible_for_dedupe = 1 - ), - CurrentIndexColumns AS - ( - SELECT - ic.* - FROM IndexColumns AS ic - WHERE ic.index_id = @c_index_id - AND ic.is_eligible_for_dedupe = 1 - ), - OtherIndexColumns AS - ( - SELECT - ic.* - FROM IndexColumns AS ic - WHERE ic.index_id <> @c_index_id - AND ic.is_eligible_for_dedupe = 1 - ) - UPDATE - ia - SET - ia.is_redundant = - CASE - WHEN NOT EXISTS - ( - SELECT - 1/0 - FROM CurrentIndexColumns cic - WHERE cic.is_included_column = 0 /* Only check key columns */ - AND NOT EXISTS - ( - SELECT - 1/0 - FROM OtherIndexColumns oic - WHERE oic.column_name = cic.column_name - AND oic.is_included_column = 0 /* Must be in key columns */ - AND oic.key_ordinal = cic.key_ordinal /* Check leading edge */ - AND oic.is_descending_key = cic.is_descending_key - ) - ) - AND - ( - /* Check included columns separately since order doesn't matter */ - NOT EXISTS - ( - SELECT - 1/0 - FROM CurrentIndexColumns cic - WHERE cic.is_included_column = 1 - AND NOT EXISTS - ( - SELECT - 1/0 - FROM OtherIndexColumns oic - WHERE oic.column_name = cic.column_name - AND - ( - oic.is_included_column = 1 - OR oic.is_included_column = 0 /* Include cols can be covered by key cols */ - ) - ) - ) - ) - AND ISNULL(REPLACE(REPLACE(REPLACE(ia.filter_definition, ' ', ''), '(', ''), ')', ''), '') = - ISNULL(REPLACE(REPLACE(REPLACE(@c_filter_definition, ' ', ''), '(', ''), ')', ''), '') - AND - ( - ia.is_unique = 0 - OR - ( - ia.is_unique = 1 - AND @c_is_unique = 1 - ) - ) - THEN 1 - ELSE 0 - END, - ia.superseded_by = - CASE - WHEN NOT EXISTS - ( - SELECT - 1/0 - FROM CurrentIndexColumns cic - WHERE cic.is_included_column = 0 /* Only check key columns */ - AND NOT EXISTS - ( - SELECT - 1/0 - FROM OtherIndexColumns oic - WHERE oic.column_name = cic.column_name - AND oic.is_included_column = 0 /* Must be in key columns */ - AND oic.key_ordinal = cic.key_ordinal /* Check leading edge */ - AND oic.is_descending_key = cic.is_descending_key - ) - ) - AND - ( - /* Check included columns separately since order doesn't matter */ - NOT EXISTS - ( - SELECT - 1/0 - FROM CurrentIndexColumns cic - WHERE cic.is_included_column = 1 - AND NOT EXISTS - ( - SELECT - 1/0 - FROM OtherIndexColumns oic - WHERE oic.column_name = cic.column_name - AND - ( - oic.is_included_column = 1 - OR oic.is_included_column = 0 /* Include cols can be covered by key cols */ - ) - ) - ) - ) - AND ISNULL(ia.filter_definition, '') = ISNULL(@c_filter_definition, '') - AND - ( - ia.is_unique = 0 - OR @c_is_unique = 1 - ) - AND ia.index_name <> @c_index_name - THEN @c_index_name - ELSE ia.superseded_by - END, - ia.missing_columns = - STUFF - ( - ( - SELECT DISTINCT - N', ' + - oic.column_name - FROM OtherIndexColumns oic - WHERE NOT EXISTS - ( - SELECT - 1/0 - FROM CurrentIndexColumns cic - WHERE cic.column_name = oic.column_name - ) - FOR XML - PATH(''), - TYPE - ).value('.', 'nvarchar(max)'), - 1, - 2, - '' - ) - FROM #index_analysis ia - WHERE ia.database_id = @c_database_id - AND ia.schema_name = @c_schema_name - AND ia.table_name = @c_table_name - AND ia.index_name <> @c_index_name; - - INSERT INTO - #index_supersede_debug - ( - step, - current_index, - other_index, - current_key_columns, - other_key_columns, - current_include_columns, - other_include_columns, - decision, - reason - ) - SELECT - 'After Update', - @c_index_name, - ia.index_name, - @current_key_cols, - other_key_cols = - STUFF - ( - ( - SELECT - N', ' + - id3.column_name + - CASE - WHEN id3.is_descending_key = 1 - THEN N' DESC' - ELSE N'' - END - FROM #index_details AS id3 - WHERE id3.database_id = ia.database_id - AND id3.object_id = ia.object_id - AND id3.index_id = ia.index_id - AND id3.is_included_column = 0 - AND id3.key_ordinal > 0 - ORDER BY - id3.key_ordinal - FOR - XML - PATH(''), - TYPE - ).value('.', 'nvarchar(max)'), - 1, - 2, - '' - ), - @current_include_cols, - other_include_cols = - STUFF - ( - ( - SELECT - N', ' + - id3.column_name - FROM #index_details AS id3 - WHERE id3.database_id = ia.database_id - AND id3.object_id = ia.object_id - AND id3.index_id = ia.index_id - AND id3.is_included_column = 1 - ORDER BY - id3.column_name - FOR XML PATH(''), TYPE - ).value('.', 'nvarchar(max)'), - 1, - 2, - '' - ), - CASE - WHEN ia.superseded_by = @c_index_name - THEN 'Current supersedes Other' - ELSE 'No Change' - END, - 'superseded_by: ' + - ISNULL(ia.superseded_by, 'NULL') + - ', is_redundant: ' + - CONVERT(varchar(MAX), ia.is_redundant) - FROM #index_analysis AS ia - WHERE ia.database_id = @c_database_id - AND ia.schema_id = @c_schema_id - AND ia.table_name = @c_table_name - AND ia.index_name <> @c_index_name - AND - ( - ia.index_name = 'IX_Users_AccountId_DisplayName' - OR ia.index_name = 'u' - OR -- Focus on problem indexes - @c_index_name = 'IX_Users_AccountId_DisplayName' - OR @c_index_name = 'u' - ) - AND ia.superseded_by = @c_index_name; - - FETCH NEXT - FROM @index_cursor - INTO - @c_database_id, - @c_database_name, - @c_schema_id, - @c_schema_name, - @c_object_id, - @c_table_name, - @c_index_id, - @c_index_name, - @c_is_unique, - @c_filter_definition; - END; - - SELECT - * - FROM #index_supersede_debug - ORDER BY - id; - - -- Also add this to see the final state of relevant indexes - SELECT - state = 'Final state', - ia.table_name, - ia.index_name, - ia.is_redundant, - ia.superseded_by, - ia.action - FROM #index_analysis AS ia - WHERE ia.table_name = 'Users' - AND - ( - ia.index_name = 'IX_Users_AccountId_DisplayName' - OR ia.index_name = 'u' - ) - ORDER BY - ia.index_name; - - IF @debug = 1 - BEGIN - RAISERROR('Performing #index_analysis update after cursor', 0, 0) WITH NOWAIT; - END; - - /*Determine actions*/ - UPDATE - #index_analysis - SET - action = - CASE - WHEN is_redundant = 1 - THEN N'DROP' - WHEN superseded_by IS NOT NULL - AND missing_columns IS NULL - THEN N'MERGE INTO ' + - superseded_by - WHEN superseded_by IS NOT NULL - AND missing_columns IS NOT NULL - THEN N'MERGE INTO ' + - superseded_by + - N' (ADD ' + - missing_columns + - N')' - ELSE N'KEEP' - END; - - IF @debug = 1 - BEGIN - SELECT - table_name = '#index_analysis after update', - ia.* - FROM #index_analysis AS ia; - END; - - IF @debug = 1 - BEGIN - RAISERROR('Performing #index_cleanup_report insert', 0, 0) WITH NOWAIT; - END; - - INSERT INTO - #index_cleanup_report - WITH - (TABLOCK) - ( - database_id, - database_name, - schema_id, - schema_name, - table_name, - object_id, - index_id, - index_name, - action, - cleanup_script, - original_definition, - user_seeks, - user_scans, - user_lookups, - user_updates, - last_user_seek, - last_user_scan, - last_user_lookup, - last_user_update, - range_scan_count, - singleton_lookup_count, - leaf_insert_count, - leaf_update_count, - leaf_delete_count, - page_lock_count, - page_lock_wait_count, - page_lock_wait_in_ms - ) - SELECT - @database_id, - @database_name, - ia.schema_id, - ia.schema_name, - ia.table_name, - ia.object_id, - ia.index_id, - ia.index_name, - ia.action, - cleanup_script = - CASE - WHEN ia.action = N'DROP' - THEN NCHAR(10) + - N'DROP INDEX ' + - QUOTENAME(ia.index_name) + - N' ON ' + - QUOTENAME(DB_NAME(ia.database_id)) + - N'.' + - QUOTENAME(ia.schema_name) + - N'.' + - QUOTENAME(ia.table_name) + - N';' - WHEN ia.action LIKE N'MERGE INTO%' - THEN NCHAR(10) + - N'CREATE ' + - CASE - WHEN ia.is_unique = 1 - THEN N'UNIQUE ' - ELSE N'' - END + - N'INDEX ' + - QUOTENAME(ia.superseded_by) + - NCHAR(10) + - N'ON ' + - QUOTENAME(DB_NAME(ia.database_id)) + - N'.' + - QUOTENAME(ia.schema_name) + - N'.' + - QUOTENAME(ia.table_name) + - NCHAR(10) + - N' (' + - ISNULL(superseding.key_columns, ia.key_columns) + - N')' + - NCHAR(10) + - CASE - WHEN - ( - superseding.included_columns IS NOT NULL - OR ia.included_columns IS NOT NULL - ) - OR ia.missing_columns IS NOT NULL - THEN N' INCLUDE' + - NCHAR(10) + - N' (' + - -- Combine all INCLUDE columns with proper parsing - STUFF - ( - ( - SELECT DISTINCT - N', ' + - column_value - FROM - ( - -- From superseding index - SELECT DISTINCT - column_value = - LTRIM(RTRIM(value.c.value('.', 'sysname'))) - FROM - ( - SELECT - Columns = - CONVERT - ( - xml, - '' + - REPLACE - ( - ISNULL - ( - superseding.included_columns, - '' - ), - ', ', - '' - ) + - '' - ) - ) t - CROSS APPLY t.Columns.nodes('/c') AS value(c) - - UNION - - -- From current index - SELECT DISTINCT - column_value = - LTRIM(RTRIM(value.c.value('.', 'sysname'))) - FROM - ( - SELECT - Columns = - CONVERT - ( - xml, - '' + - REPLACE - ( - ISNULL - ( - ia.included_columns, - '' - ), - ', ', - '' - ) + - '' - ) - ) t - CROSS APPLY t.Columns.nodes('/c') AS value(c) - - UNION - - -- From missing columns - SELECT DISTINCT - column_value = - LTRIM(RTRIM(value.c.value('.', 'sysname'))) - FROM - ( - SELECT - Columns = - CONVERT - ( - xml, - '' + - REPLACE - ( - ISNULL - ( - ia.missing_columns, - '' - ), - ', ', - '' - ) + '' - ) - ) t - CROSS APPLY t.Columns.nodes('/c') AS value(c) - ) AS all_columns - WHERE LEN(column_value) > 0 - /*ED TODO*/ - FOR - XML - PATH(''), - TYPE - ).value('.', 'nvarchar(max)'), - 1, - 2, - '' - ) + - N')' - ELSE N'' - END + - CASE - /* Check for partitioning in the superseding index first */ - WHEN EXISTS - ( - SELECT - 1/0 - FROM #partition_stats ps_super - WHERE ps_super.table_name = ia.table_name - AND ps_super.index_name = ia.superseded_by - AND ps_super.partition_function_name IS NOT NULL - ) - THEN - ( - SELECT TOP (1) - NCHAR(10) + - N' ON ' + - QUOTENAME(ps_super.partition_function_name) + - N'(' + - ps_super.partition_columns + - N')' - FROM #partition_stats ps_super - WHERE ps_super.table_name = ia.table_name - AND ps_super.index_name = ia.superseded_by - ) - /* Fall back to the current index's partitioning if available */ - WHEN ps.partition_function_name IS NOT NULL - THEN NCHAR(10) + - N' ON ' + - QUOTENAME(ps.partition_function_name) + - N'(' + - ps.partition_columns + - N')' - ELSE N'' - END + - CASE - WHEN ia.filter_definition IS NOT NULL - THEN NCHAR(10) + - N' WHERE ' + - ia.filter_definition - ELSE N'' - END + - NCHAR(10) + - N' WITH ' + - NCHAR(10) + - N' (DROP_EXISTING = ON, FILLFACTOR = 100, SORT_IN_TEMPDB = ON, ONLINE = ' + - CASE - WHEN @online = 'true' /*Best effort at detecting online index abilities*/ - THEN N'ON' - ELSE N'OFF' - END + - CASE - WHEN ps.data_compression_desc <> N'NONE' - THEN N', DATA_COMPRESSION = ' + - ps.data_compression_desc - ELSE N', DATA_COMPRESSION = PAGE' /* Add PAGE compression by default for merged indexes */ - END + - N');' + - NCHAR(10) + - NCHAR(10) + - N' ALTER INDEX ' + - QUOTENAME(ia.index_name) + - N' ON ' + - QUOTENAME(DB_NAME(ia.database_id)) + - N'.' + - QUOTENAME(ia.schema_name) + - N'.' + - QUOTENAME(ia.table_name) + - N' DISABLE' - ELSE N'' - END + - N';', - original_definition = - NCHAR(10) + - N' -- CREATE ' + - CASE - WHEN ia.is_unique = 1 - THEN N'UNIQUE ' - ELSE N'' - END + - N'INDEX ' + - QUOTENAME(ia.index_name) + - NCHAR(10) + - N' -- ON ' + - QUOTENAME(DB_NAME(ia.database_id)) + - N'.' + - QUOTENAME(ia.schema_name) + - N'.' + - QUOTENAME(ia.table_name) + - NCHAR(10) + - N' -- (' + - ia.key_columns + - N')' + - CASE - WHEN ia.included_columns IS NOT NULL - THEN NCHAR(10) + - N' -- INCLUDE' + - NCHAR(10) + - N' -- (' + - ia.included_columns + - N')' - ELSE N'' - END + - CASE - WHEN ps.partition_function_name IS NOT NULL - THEN NCHAR(10) + - N' -- ON ' + - QUOTENAME(ps.partition_function_name) + - N'(' + - ps.partition_columns + - N')' - ELSE N'' - END + - CASE - WHEN ia.filter_definition IS NOT NULL - THEN NCHAR(10) + - N' -- WHERE ' + - ia.filter_definition - ELSE N'' - END + - N';' + - NCHAR(10), - id.user_seeks, - id.user_scans, - id.user_lookups, - id.user_updates, - id.last_user_seek, - id.last_user_scan, - id.last_user_lookup, - id.last_user_update, - os.range_scan_count, - os.singleton_lookup_count, - os.leaf_insert_count, - os.leaf_update_count, - os.leaf_delete_count, - os.page_lock_count, - os.page_lock_wait_count, - os.page_lock_wait_in_ms - FROM #index_analysis ia - LEFT JOIN #partition_stats AS ps - ON ia.table_name = ps.table_name - AND ia.index_name = ps.index_name - LEFT JOIN #index_details AS id - ON ia.table_name = id.table_name - AND ia.index_name = id.index_name - LEFT JOIN #operational_stats AS os - ON id.object_id = os.object_id - AND id.index_id = os.index_id - LEFT JOIN #index_analysis AS superseding - ON ia.superseded_by = superseding.index_name - AND ia.table_name = superseding.table_name - OPTION(RECOMPILE); - - IF ROWCOUNT_BIG() = 0 BEGIN IF @debug = 1 BEGIN RAISERROR('No rows inserted into #index_cleanup_report', 0, 0) WITH NOWAIT END; END; - - IF @debug = 1 - BEGIN - SELECT - table_name = '#index_cleanup_report', - icr.* - FROM #index_cleanup_report AS icr; - END; - - IF @debug = 1 - BEGIN - RAISERROR('Performing #index_cleanup_summary insert', 0, 0) WITH NOWAIT; - END; - - INSERT INTO - #index_cleanup_summary - WITH - (TABLOCK) - ( - database_id, - database_name, - schema_id, - schema_name, - table_name, - object_id, - index_id, - index_name, - action, - details, - current_definition, - proposed_definition, - usage_summary, - operational_summary, - uptime_warning - ) - SELECT - icr.database_id, - icr.database_name, - icr.schema_id, - icr.schema_name, - icr.table_name, - icr.object_id, - icr.index_id, - icr.index_name, - action = - CASE - WHEN icr.action = N'KEEP' - THEN N'Keep' - WHEN icr.action = N'DROP' - THEN N'Drop' - WHEN icr.action LIKE N'MERGE INTO%' - THEN N'Merge' - ELSE N'???' - END, - details = - CASE - WHEN icr.action = N'KEEP' - THEN N'No action needed' - WHEN icr.action = N'DROP' - THEN N'Index is redundant and can be safely dropped' - WHEN icr.action LIKE N'MERGE INTO%' - THEN N'Merge into index: ' + - SUBSTRING - ( - icr.action, - 12, - CHARINDEX(N' ', icr.action, 12) - 12 - ) - ELSE N'???' - END, - current_definition = icr.original_definition, - proposed_definition = - CASE - WHEN icr.action LIKE N'MERGE INTO%' - THEN icr.cleanup_script - ELSE NULL - END, - usage_summary = - N'Seeks: ' + CONVERT(nvarchar(20), icr.user_seeks) + - N', Scans: ' + CONVERT(nvarchar(20), icr.user_scans) + - N', Lookups: ' + CONVERT(nvarchar(20), icr.user_lookups) + - N', Updates: ' + CONVERT(nvarchar(20), icr.user_updates) + - N', Last used: ' + - ISNULL - ( - CONVERT - ( - nvarchar(30), - NULLIF - ( - DATEADD - ( - SECOND, - -1, - CASE - WHEN icr.last_user_seek > icr.last_user_scan - AND icr.last_user_seek > icr.last_user_lookup - THEN icr.last_user_seek - WHEN icr.last_user_scan > icr.last_user_lookup - THEN icr.last_user_scan - ELSE icr.last_user_lookup - END - ), - N'1900-01-01' - ), 120 - ), - N'Unknown' - ), - operational_summary = - N'Range scans: ' + CONVERT(nvarchar(20), icr.range_scan_count) + - N', Lookups: ' + CONVERT(nvarchar(20), icr.singleton_lookup_count) + - N', Inserts: ' + CONVERT(nvarchar(20), icr.leaf_insert_count) + - N', Updates: ' + CONVERT(nvarchar(20), icr.leaf_update_count) + - N', Deletes: ' + CONVERT(nvarchar(20), icr.leaf_delete_count), - uptime_warning = - CASE - WHEN icr.user_seeks = 0 AND icr.user_scans = 0 AND icr.user_lookups = 0 - THEN - CASE - WHEN TRY_PARSE(@uptime_days AS integer) < 7 - THEN N'WARNING: SQL Server has been running for only ' + - @uptime_days + - N' days. Usage statistics may not be reliable.' - WHEN TRY_PARSE(@uptime_days AS integer) < 14 - THEN N'CAUTION: SQL Server has been running for only ' + - @uptime_days + - N' days. Usage statistics may be incomplete.' - WHEN TRY_PARSE(@uptime_days AS integer) < 30 - THEN N'NOTE: SQL Server has been running for only ' + - @uptime_days + - N' days. Consider this when evaluating index usage.' - ELSE N'NOTE: SQL Server has been up for ' + - @uptime_days + - N' days, which makes analysis good, but... Are you patching this thing?' - END - ELSE NULL - END - FROM #index_cleanup_report AS icr - OPTION(RECOMPILE); - - IF ROWCOUNT_BIG() = 0 BEGIN IF @debug = 1 BEGIN RAISERROR('No rows inserted into #index_cleanup_summary', 0, 0) WITH NOWAIT END; END; - - IF @debug = 1 - BEGIN - SELECT - table_name = '#index_cleanup_summary', - ics.* - FROM #index_cleanup_summary AS ics; - END; - - IF @debug = 1 - BEGIN - RAISERROR('Going into summary and reports', 0, 0) WITH NOWAIT; - END; - - /* Index Cleanup Summary Report */ - - IF @debug = 1 - BEGIN - RAISERROR('Index Cleanup Summary', 0, 0) WITH NOWAIT; - END; - - SELECT - summary_type = - 'Index Cleanup Summary', - total_indexes_analyzed = - COUNT_BIG(DISTINCT icr.index_name), - indexes_to_drop = - SUM - ( - CASE - WHEN icr.action = 'DROP' - THEN 1 - ELSE 0 - END - ), - indexes_to_merge = - SUM - ( - CASE - WHEN icr.action LIKE 'MERGE INTO%' - THEN 1 - ELSE 0 - END - ), - unused_indexes = - SUM - ( - CASE - WHEN icr.user_seeks = 0 - AND icr.user_scans = 0 - AND icr.user_lookups = 0 - THEN 1 - ELSE 0 - END - ), - space_savings_gb = - CONVERT - ( - decimal(10, 2), - ( - SELECT - SUM(ps_total.space_saved_mb) / 1024.0 - FROM - ( - SELECT - icr_distinct.index_name, - icr_distinct.table_name, - space_saved_mb = SUM(ps_inner.total_space_mb) - FROM #index_cleanup_report AS icr_distinct - JOIN #partition_stats AS ps_inner - ON ps_inner.table_name = icr_distinct.table_name - AND ps_inner.index_name = icr_distinct.index_name - WHERE icr_distinct.action = 'DROP' - OR icr_distinct.action LIKE 'MERGE INTO%' - GROUP BY - icr_distinct.index_name, - icr_distinct.table_name - ) AS ps_total - ) - ), - write_operations_avoided = - SUM - ( - CASE - WHEN icr.action = 'DROP' - OR icr.action LIKE 'MERGE INTO%' - THEN ISNULL(icr.user_updates, 0) - ELSE 0 - END - ) - FROM #index_cleanup_report AS icr - OPTION (RECOMPILE); - - IF @debug = 1 - BEGIN - RAISERROR('Top tables by potential space savings', 0, 0) WITH NOWAIT; - END; - - /* Top tables by potential space savings */ - SELECT TOP (10) - icr.database_name, - icr.table_name, - indexes_affected = - COUNT_BIG(DISTINCT icr.index_name), - space_savings_gb = - CONVERT - ( - decimal(10,2), - ( - SELECT - SUM(ps_total.space_saved_mb) / 1024.0 - FROM - ( - SELECT - ps_inner.table_name, - space_saved_mb = - SUM(ps_inner.total_space_mb) - FROM #partition_stats AS ps_inner - JOIN #index_cleanup_report AS icr_inner - ON ps_inner.table_name = icr_inner.table_name - AND ps_inner.index_name = icr_inner.index_name - WHERE icr_inner.table_name = icr.table_name - AND - ( - icr_inner.action = 'DROP' - OR icr_inner.action LIKE 'MERGE INTO%' - ) - GROUP BY - ps_inner.table_name - ) AS ps_total - ) - ), - write_operations_avoided = - SUM(ISNULL(icr.user_updates, 0)) - FROM #index_cleanup_report AS icr - WHERE - ( - icr.action = 'DROP' - OR icr.action LIKE 'MERGE INTO%' - ) - GROUP BY - icr.database_name, - icr.table_name - ORDER BY - space_savings_gb DESC - OPTION(RECOMPILE); - - IF @debug = 1 - BEGIN - RAISERROR('Page Compression Opportunity Summary', 0, 0) WITH NOWAIT; - END; - - /* Summary of non-compressed indexes */ - SELECT - summary_type = 'Page Compression Opportunity Summary', - candidate_indexes = - COUNT_BIG(*), - total_size_gb = - SUM(ps.total_space_mb) / 1024.0, - estimated_savings_low_gb = - (SUM(ps.total_space_mb) * 0.20) / 1024.0, /* Conservative estimate (20%) */ - estimated_savings_typical_gb = - (SUM(ps.total_space_mb) * 0.40) / 1024.0, /* Typical estimate (40%) */ - estimated_savings_high_gb = - (SUM(ps.total_space_mb) * 0.60) / 1024.0 /* Optimistic estimate (60%) */ - FROM #partition_stats ps - WHERE ps.data_compression_desc = 'NONE' - AND NOT EXISTS - ( - SELECT - 1/0 - FROM #index_cleanup_report AS icr - WHERE icr.index_name = ps.index_name - AND - ( - icr.action = 'DROP' - OR icr.action LIKE 'MERGE INTO%' - ) - ) - OPTION(RECOMPILE); - - -- Top candidates for page compression - - IF @debug = 1 - BEGIN - RAISERROR('Top candidates for page compression', 0, 0) WITH NOWAIT; - END; - - SELECT TOP (20) - database_name = - @database_name, - ps.schema_name, - ps.table_name, - ps.index_name, - index_type = - CASE - WHEN ps.index_id = 1 - THEN 'CLUSTERED' - ELSE 'NONCLUSTERED' - END, - size_gb = - SUM(ps.total_space_mb) / 1024.0, - estimated_savings_low_gb = - (SUM(ps.total_space_mb) * 0.20) / 1024.0, -- Conservative (20%) - estimated_savings_typical_gb = - (SUM(ps.total_space_mb) * 0.40) / 1024.0, -- Typical (40%) - estimated_savings_high_gb = - (SUM(ps.total_space_mb) * 0.60) / 1024.0, -- Optimistic (60%) - rebuild_script = - N'ALTER INDEX ' + - QUOTENAME(ps.index_name) + - N' ON ' + - QUOTENAME(ps.schema_name) + - N'.' + - QUOTENAME(ps.table_name) + - N' REBUILD WITH - (DROP_EXISTING = ON, FILLFACTOR = 100, SORT_IN_TEMPDB = ON, ONLINE = ' + - CASE - WHEN @online = 'true' - THEN N'ON' - ELSE N'OFF' - END + - N', DATA_COMPRESSION = PAGE - );' - FROM #partition_stats ps - WHERE ps.data_compression_desc = N'NONE' - GROUP BY - ps.schema_name, - ps.table_name, - ps.index_name, - ps.index_id - ORDER BY - SUM(ps.total_space_mb) DESC - OPTION(RECOMPILE); - - IF @debug = 1 - BEGIN - RAISERROR('Select from #index_cleanup_summary', 0, 0) WITH NOWAIT; - END; - - SELECT - ics.database_name, - ics.table_name, - ics.index_name, - ics.action, - ics.details, - ics.current_definition, - ics.proposed_definition, - ics.usage_summary, - ics.operational_summary, - ics.uptime_warning - FROM #index_cleanup_summary AS ics - ORDER BY - CASE ics.action - WHEN N'Drop' THEN 1 - WHEN N'Merge' THEN 2 - WHEN N'Keep' THEN 3 - ELSE 999 - END, - ics.table_name, - ics.index_name - OPTION(RECOMPILE); - - IF @debug = 1 - BEGIN - RAISERROR('Performing #final_index_actions insert', 0, 0) WITH NOWAIT; - END; - - -- Replace the existing INSERT into #final_index_actions for MERGE operations with this: - WITH - MergeTargets AS - ( - -- Get distinct target indexes for merges - SELECT DISTINCT - ia.database_id, - ia.database_name, - ia.schema_id, - ia.schema_name, - ia.object_id, - ia.table_name, - ia.index_name, - target_index = - SUBSTRING - ( - ia.action, - 12, - CHARINDEX - ( - N' ', - ia.action + - N' ', - 12 - ) - 12 - ) - FROM #index_cleanup_report ia - WHERE ia.action LIKE N'MERGE INTO%' - AND SUBSTRING - ( - ia.action, - 12, - CHARINDEX - ( - N' ', - ia.action + - N' ', - 12 - ) - 12 - ) <> ia.index_name - ) - -- Insert a single CREATE INDEX statement for each target index - INSERT INTO - #final_index_actions - WITH - (TABLOCK) - ( - database_id, - database_name, - schema_id, - schema_name, - object_id, - table_name, - index_id, - index_name, - target_index_name, - action, - script - ) - SELECT DISTINCT - mt.database_id, - mt.database_name, - mt.schema_id, - mt.schema_name, - mt.object_id, - mt.table_name, - index_id = - ISNULL - ( - ( - SELECT TOP (1) - ia.index_id - FROM #index_analysis ia - WHERE ia.database_id = mt.database_id - AND ia.table_name = mt.table_name - AND ia.index_name = mt.target_index - ), - 0 - ), - mt.index_name, - mt.target_index, - action = - N'MERGE CONSOLIDATED', - script = - N'CREATE INDEX ' + - QUOTENAME(mt.target_index) + - N' ON ' + - QUOTENAME(mt.database_name) + - N'.' + - QUOTENAME(mt.schema_name) + - N'.' + - QUOTENAME(mt.table_name) + - N' (' + - -- Get key columns from one of the indexes being merged - ( - SELECT TOP (1) - ia.key_columns - FROM #index_analysis ia - WHERE ia.database_id = mt.database_id - AND ia.table_name = mt.table_name - AND ia.index_name = mt.target_index - ) + - N')' + - -- Include all distinct columns from all indexes being merged into this target - CASE - WHEN EXISTS - ( - SELECT - 1/0 - FROM #index_cleanup_report icr - WHERE icr.database_id = mt.database_id - AND icr.table_name = mt.table_name - AND icr.action LIKE N'MERGE INTO ' + mt.target_index + N'%' - AND - ( - EXISTS - ( - SELECT - 1/0 - FROM #index_analysis ia - WHERE ia.database_id = icr.database_id - AND ia.table_name = icr.table_name - AND ia.index_name = icr.index_name - AND ia.included_columns IS NOT NULL - ) - OR icr.action LIKE N'%ADD %' - ) - ) - THEN N' INCLUDE (' + - STUFF - ( - ( - SELECT DISTINCT - N', ' + - col - FROM - ( - -- Get included columns from all source indexes - SELECT DISTINCT - col = LTRIM(RTRIM(value.c.value('.', 'sysname'))) - FROM #index_cleanup_report icr - CROSS APPLY - ( - SELECT - ia.included_columns - FROM #index_analysis ia - WHERE ia.database_id = icr.database_id - AND ia.table_name = icr.table_name - AND ia.index_name = icr.index_name - ) src - CROSS APPLY - ( - SELECT - cols = - CONVERT - ( - xml, - '' + - REPLACE - ( - ISNULL - ( - src.included_columns, - '' - ), - ', ', - '') + - '' - ) - ) x - CROSS APPLY x.cols.nodes('/c') AS value(c) - WHERE icr.database_id = mt.database_id - AND icr.table_name = mt.table_name - AND icr.action LIKE N'MERGE INTO ' + mt.target_index + N'%' - - UNION - - -- Get missing columns which need to be added - SELECT - col = - LTRIM(RTRIM(value.c.value('.', 'sysname'))) - FROM #index_cleanup_report icr - CROSS APPLY - ( - SELECT DISTINCT - missing_cols = - REPLACE(REPLACE( - SUBSTRING - ( - icr.action, - CHARINDEX('ADD ', icr.action) + 4, - LEN(icr.action) - ), - N')', ''), N'(', '') - WHERE icr.action LIKE N'%ADD %' - ) mc - CROSS APPLY - ( - SELECT - cols = - CONVERT - ( - xml, - '' + - REPLACE - ( - ISNULL - ( - mc.missing_cols, - '' - ), - ', ', - '' - ) + '' - ) - ) x - CROSS APPLY x.cols.nodes('/c') AS value(c) - WHERE icr.database_id = mt.database_id - AND icr.table_name = mt.table_name - AND icr.action LIKE N'MERGE INTO ' + mt.target_index + N'%' - AND icr.action LIKE N'%ADD %' - ) AS all_columns - WHERE DATALENGTH(col) > 0 - FOR - XML - PATH(''), - TYPE - ).value('.', 'nvarchar(max)'), - 1, - 2, - '' - ) + - N')' - ELSE N'' - END + - -- Add partitioning if needed - CASE - WHEN EXISTS - ( - SELECT - 1/0 - FROM #partition_stats ps - WHERE ps.database_id = mt.database_id - AND ps.table_name = mt.table_name - AND ps.index_name = mt.target_index - AND ps.partition_function_name IS NOT NULL - ) - THEN - ( - SELECT TOP (1) - N' ON ' + - QUOTENAME(ps.partition_function_name) + - '(' + - ps.partition_columns + - ')' - FROM #partition_stats ps - WHERE ps.database_id = mt.database_id - AND ps.table_name = mt.table_name - AND ps.index_name = mt.target_index - AND ps.partition_function_name IS NOT NULL - ) - ELSE N'' - END + - -- Add filter definition if needed - CASE - WHEN EXISTS - ( - SELECT - 1/0 - FROM #index_analysis ia - WHERE ia.database_id = mt.database_id - AND ia.table_name = mt.table_name - AND ia.index_name = mt.target_index - AND ia.filter_definition IS NOT NULL - ) - THEN - ( - SELECT TOP (1) - N' WHERE ' + - ia.filter_definition - FROM #index_analysis ia - WHERE ia.database_id = mt.database_id - AND ia.table_name = mt.table_name - AND ia.index_name = mt.target_index - AND ia.filter_definition IS NOT NULL - ) - ELSE N'' - END + - -- Add WITH options - N' WITH (DROP_EXISTING = ON, FILLFACTOR = 100, SORT_IN_TEMPDB = ON, ONLINE = ' + - CASE - WHEN @online = 'true' - THEN N'ON' - ELSE N'OFF' - END + - N', DATA_COMPRESSION = PAGE);' - FROM MergeTargets AS mt; - - -- Then add DISABLE statements for all source indexes - INSERT INTO - #final_index_actions - WITH - (TABLOCK) - ( - database_id, - database_name, - schema_id, - schema_name, - object_id, - table_name, - index_id, - index_name, - action, - script - ) - SELECT - icr.database_id, - icr.database_name, - icr.schema_id, - icr.schema_name, - icr.object_id, - icr.table_name, - icr.index_id, - icr.index_name, - action = N'DISABLE MERGED', - script = - N'ALTER INDEX ' + - QUOTENAME(icr.index_name) + - N' ON ' + - QUOTENAME(icr.database_name) + - N'.' + - QUOTENAME(icr.schema_name) + - N'.' + - QUOTENAME(icr.table_name) + - N' DISABLE;' - FROM #index_cleanup_report icr - WHERE icr.action LIKE N'MERGE INTO%'; - - IF ROWCOUNT_BIG() = 0 BEGIN IF @debug = 1 BEGIN RAISERROR('No rows inserted into #final_index_actions', 0, 0) WITH NOWAIT END; END; - - IF @debug = 1 - BEGIN - - SELECT - table_name = '#final_index_actions', - fia.* - FROM #final_index_actions AS fia; - - RAISERROR('Select from #final_index_actions', 0, 0) WITH NOWAIT; - END; - - SELECT - f.database_name, - f.table_name, - f.index_name, - f.action, - f.script, - sort_order = - CASE f.action - WHEN N'MERGE INTO' THEN 2 - WHEN N'DROP' THEN 3 - ELSE 999 - END - FROM #final_index_actions AS f - WHERE f.action <> N'KEEP' - - UNION ALL - - SELECT - r.database_name, - r.table_name, - r.index_name, - action = - N'DISABLE (Unused)', - script = - N'ALTER INDEX ' + - QUOTENAME(r.index_name) + - N' ON ' + - QUOTENAME(r.table_name) + - N' DISABLE;', - sort_order = 1 - FROM #index_cleanup_report AS r - WHERE r.user_seeks = 0 - AND r.user_scans = 0 - AND r.user_lookups = 0 - AND r.user_updates = 0 - ORDER BY - f.table_name, - f.index_name, - sort_order - OPTION(RECOMPILE); - - IF @debug = 1 - BEGIN - RAISERROR('Generating scripts', 0, 0) WITH NOWAIT; - END; - - SELECT * FROM #final_index_actions AS fia - - /*Merge into*/ - SELECT - @final_script += - N' - -- ============================================================================= - -- MERGE INDEX: ' + - QUOTENAME(f.index_name) + - N' into ' + - CASE - WHEN f.action = 'MERGE CONSOLIDATED' - THEN QUOTENAME(f.target_index_name) - ELSE 'Unknown Target' - END + - N' - -- Reason: This index overlaps with another index and can be consolidated - -- Original definition: ' + - NCHAR(10) + - ( - SELECT - MAX(ics.current_definition) - FROM #index_cleanup_summary AS ics - WHERE ics.index_name = f.index_name - AND ics.table_name = f.table_name - ) + - N' - -- Usage: Seeks: ' + - CONVERT - ( - nvarchar(20), - ISNULL - ( - ( - SELECT - SUM(DISTINCT id.user_seeks) - FROM #index_details AS id - WHERE id.table_name = f.table_name - AND id.index_name = f.index_name - ), - 0 - ) - ) + - N', Scans: ' + - CONVERT - ( - nvarchar(20), - ISNULL - ( - ( - SELECT - SUM(DISTINCT id.user_scans) - FROM #index_details AS id - WHERE id.table_name = f.table_name - AND id.index_name = f.index_name - ), - 0 - ) - ) + - N', Lookups: ' + - CONVERT - ( - nvarchar(20), - ISNULL - ( - ( - SELECT - SUM(DISTINCT id.user_lookups) - FROM #index_details AS id - WHERE id.table_name = f.table_name - AND id.index_name = f.index_name - ), - 0 - ) - ) + - N', Updates: ' + - CONVERT - ( - nvarchar(20), - ISNULL - ( - ( - SELECT - SUM(DISTINCT id.user_updates) - FROM #index_details AS id - WHERE id.table_name = f.table_name - AND id.index_name = f.index_name - ), - 0 - ) - ) + - N' - -- Space saved: ' + - CONVERT - ( - nvarchar(20), - ISNULL - ( - ( - SELECT - CONVERT - ( - decimal(10,2), - SUM(ps.total_space_mb) / 1024.0 - ) - FROM #partition_stats AS ps - WHERE ps.table_name = f.table_name - AND ps.index_name = f.index_name - ), - 0 - ) - ) + N' GB - -- ============================================================================= - ' + - f.script + - NCHAR(10) + - NCHAR(10) - FROM #final_index_actions AS f - WHERE f.action = N'MERGE CONSOLIDATED' - ORDER BY - f.table_name, - f.index_name; - - /*Disable merged indexes*/ - SELECT - @final_script += N' - /* - -- ============================================================================= - -- DISABLE MERGED INDEX: ' + - QUOTENAME(f.index_name) + - N' - -- Reason: This index has been merged into another index - -- ============================================================================= - */' + - NCHAR(10) + - f.script + - NCHAR(10) + - NCHAR(10) - FROM - ( - -- Use a derived table with DISTINCT to avoid duplicates - SELECT DISTINCT - index_name, - table_name, - script - FROM #final_index_actions - WHERE action = N'DISABLE MERGED' - ) AS f - ORDER BY - f.table_name, - f.index_name; - - /*Drop indexes*/ - SELECT - @final_script += N' - /* - -- ============================================================================= - -- DROP INDEX: ' + - QUOTENAME(f.index_name) + - N' - -- Reason: This index is redundant with other indexes on the same table - -- Current usage: Seeks: ' + - CONVERT - ( - nvarchar(20), - ISNULL - ( - ( - SELECT - SUM(DISTINCT id.user_seeks) - FROM #index_details AS id - WHERE id.table_name = f.table_name - AND id.index_name = f.index_name - ), - 0 - ) - ) + - N', Scans: ' + - CONVERT - ( - nvarchar(20), - ISNULL - ( - ( - SELECT - SUM(DISTINCT id.user_scans) - FROM #index_details AS id - WHERE id.table_name = f.table_name - AND id.index_name = f.index_name - ), - 0 - ) - ) + - N', Lookups: ' + - CONVERT - ( - nvarchar(20), - ISNULL - ( - ( - SELECT - SUM(DISTINCT id.user_lookups) - FROM #index_details AS id - WHERE id.table_name = f.table_name - AND id.index_name = f.index_name - ), - 0 - ) - ) + - N', Updates: ' + - CONVERT - ( - nvarchar(20), - ISNULL - ( - ( - SELECT - SUM(DISTINCT id.user_updates) - FROM #index_details AS id - WHERE id.table_name = f.table_name - AND id.index_name = f.index_name - ), - 0 - ) - ) + - N' - -- Last used: ' + - ISNULL - ( - CONVERT - ( - nvarchar(30), - ( - SELECT - MAX - ( - CASE - WHEN id.last_user_seek > id.last_user_scan - AND id.last_user_seek > id.last_user_lookup - THEN id.last_user_seek - WHEN id.last_user_scan > id.last_user_lookup - THEN id.last_user_scan - ELSE id.last_user_lookup - END - ) - FROM #index_details AS id - WHERE id.table_name = f.table_name - AND id.index_name = f.index_name - ), - 120 - ), - 'Never' - ) + - N' - -- Space reclaimed: ' + - CONVERT - ( - nvarchar(20), - ISNULL - ( - ( - SELECT - CONVERT - ( - decimal(10,2), - SUM(ps.total_space_mb) / 1024.0 - ) - FROM #partition_stats AS ps - WHERE ps.table_name = f.table_name - AND ps.index_name = f.index_name - ), - 0 - ) - ) + N' GB - -- ============================================================================= - */' + - f.script + - NCHAR(10) + - NCHAR(10) - FROM #final_index_actions AS f - WHERE f.action = N'DROP' - ORDER BY - f.table_name, - f.index_name; - - /*Unused indexes*/ - SELECT - @final_script += N' - /* - -- ============================================================================= - -- DISABLE UNUSED INDEX: ' + - QUOTENAME(i.index_name) + - N' - -- Reason: This index has never been used for reads but has been updated ' + - CONVERT - ( - nvarchar(20), - i.user_updates - ) + - N' times - -- Space reclaimed: ' + - CONVERT - ( - nvarchar(20), - ISNULL - ( - ( - SELECT - CONVERT - ( - decimal(10,2), - SUM(ps.total_space_mb) / 1024.0 - ) - FROM #partition_stats AS ps - WHERE ps.table_name = i.table_name - AND ps.index_name = i.index_name - ), - 0 - ) - ) + N' GB - -- Warning: Verify this index is truly not needed before dropping - -- ============================================================================= - */' + - NCHAR(10) + - N'ALTER INDEX ' + - QUOTENAME(i.index_name) + - N' ON ' + - QUOTENAME(i.database_name) + - N'.' + - QUOTENAME - (i.schema_name) + - N'.' + - QUOTENAME(i.table_name) + - N' DISABLE;' + - NCHAR(10) + - NCHAR(10) - FROM - ( - SELECT DISTINCT - icr.database_name, - icr.schema_name, - icr.table_name, - icr.index_name, - icr.user_updates - FROM #index_cleanup_report AS icr - WHERE icr.user_seeks = 0 - AND icr.user_scans = 0 - AND icr.user_lookups = 0 - AND icr.user_updates = 0 - AND icr.action <> N'DROP' - AND icr.action NOT LIKE N'MERGE INTO%' - AND NOT EXISTS - ( - SELECT - 1/0 - FROM #final_index_actions AS fia - WHERE fia.index_name = icr.index_name - AND fia.table_name = icr.table_name - AND fia.action IN (N'MERGE CONSOLIDATED', N'DISABLE MERGED') - ) - ) AS i - ORDER BY - i.table_name, - i.index_name; - - - /*Summary*/ - SELECT - @final_script += N' - -- ============================================================================= - -- SUMMARY OF CHANGES - -- Total indexes analyzed: ' + - CONVERT - ( - nvarchar(10), - ( - SELECT - COUNT_BIG(*) - FROM #index_cleanup_report AS icr - ) - ) + - N' - -- Indexes recommended for dropping: ' + - CONVERT - ( - nvarchar(10), - ( - SELECT - COUNT_BIG(*) - FROM #index_cleanup_report AS icr - WHERE icr.action = 'DROP' - ) - ) + - N' - -- Indexes recommended for merging: ' + - CONVERT - ( - nvarchar(10), - ( - SELECT - COUNT_BIG(*) - FROM #index_cleanup_report AS icr - WHERE icr.action LIKE 'MERGE INTO%' - ) - ) + - N' - -- Unused indexes found: ' + - CONVERT - ( - nvarchar(10), - ( - SELECT - COUNT_BIG(*) - FROM #index_cleanup_report AS icr - WHERE icr.user_seeks = 0 - AND icr.user_scans = 0 - AND icr.user_lookups = 0 - ) - ) + - N' - -- Estimated space savings: ' + - CONVERT - ( - nvarchar(20), - ISNULL - ( - ( - SELECT - CONVERT - ( - decimal(10,2), - SUM(ps.total_space_mb) / 1024.0 - ) - FROM #partition_stats AS ps - JOIN #index_cleanup_report AS icr - ON ps.table_name = icr.table_name - AND ps.index_name = icr.index_name - WHERE icr.action = 'DROP' - OR icr.action LIKE 'MERGE INTO%' - OR - ( - icr.user_seeks = 0 - AND icr.user_scans = 0 - AND icr.user_lookups = 0 - ) - ), - 0 - ) - ) + N' GB - -- Estimated write operations reduced: ' + - CONVERT - ( - nvarchar(20), - ISNULL - ( - ( - SELECT - SUM(icr.user_updates) - FROM #index_cleanup_report AS icr - WHERE icr.action = 'DROP' - OR icr.action LIKE 'MERGE INTO%' - OR - ( - icr.user_seeks = 0 - AND icr.user_scans = 0 - AND icr.user_lookups = 0 - ) - ), - 0 - ) - ) + N' operations - -- ============================================================================= - '; - - SELECT - [text()] = - N'/* Index Cleanup Script for ' + - @database_name + - N' */', - [text()] = - ( - SELECT - NCHAR(10) + - N' ----------------------' + - NCHAR(10) + - N' -- Final script to review. DO NOT EXECUTE WITHOUT CAREFUL REVIEW.' + - NCHAR(10) + - N' -- Implementation Script:' + - NCHAR(10) + - N' ----------------------' + - NCHAR(10) + - @final_script - FOR - XML - PATH(''), - TYPE - ).value('(./text())[1]', 'nvarchar(max)') - FOR - XML - PATH(''), - TYPE; -END TRY -BEGIN CATCH - THROW; -END CATCH; -END; /*Final End*/ -GO From 7a227727a3836b772446191dd66f9a96abf6c263 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Wed, 12 Mar 2025 17:54:53 -0400 Subject: [PATCH 040/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index 8e678517..9ef68346 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -568,7 +568,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ius.object_id HAVING SUM(ius.user_seeks + ius.user_scans + ius.user_lookups) >= @min_reads - AND + OR SUM(ius.user_updates) >= @min_writes ) OPTION(RECOMPILE);'; From 8c150a2f2a5110c49ee3af3460c18e96d01c7884 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Wed, 12 Mar 2025 18:02:35 -0400 Subject: [PATCH 041/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index 9ef68346..d166493c 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -1849,7 +1849,9 @@ INSERT INTO #index_cleanup_results ) SELECT 'MERGE', - 10, + /* Put merge target indexes higher in sort order (5) so they appear before + indexes that will be disabled (20) */ + 5, ia.database_name, ia.schema_name, ia.table_name, @@ -1950,7 +1952,8 @@ JOIN #compression_eligibility AS ce AND ia.index_id = ce.index_id WHERE ia.action IN ('MERGE INCLUDES', 'MAKE UNIQUE') AND ce.can_compress = 1 -AND ia.target_index_name IS NULL /* Only create merge scripts for the "winning" indexes */; +/* Only create merge scripts for the indexes that should remain after merging */ +AND ia.target_index_name IS NULL; /* Insert disable scripts for unneeded indexes */ INSERT INTO #index_cleanup_results @@ -1995,6 +1998,8 @@ SELECT THEN N'This index has the same keys as: ' + ISNULL(ia.target_index_name, N'(unknown)') WHEN ia.consolidation_rule LIKE 'Unused Index%' THEN ia.consolidation_rule + WHEN ia.action = 'DISABLE' + THEN N'This index is redundant and will be disabled' ELSE N'This index is redundant' END, ia.target_index_name, /* Include the target index name */ @@ -2240,12 +2245,15 @@ WHERE ce.can_compress = 0; Return the consolidated results in a single result set Results are ordered by: 1. Summary information (overall stats, savings estimates) -2. Merge scripts (includes merges and unique conversions) -3. Disable scripts (for redundant indexes) +2. Merge scripts (includes merges and unique conversions) - sort_order 5 +3. Disable scripts (for redundant indexes) - sort_order 20 4. Constraint scripts (for unique constraints to disable) 5. Compression scripts (for tables eligible for compression) 6. Partition-specific compression scripts 7. Ineligible objects (tables that can't be compressed) + +Note: Merge target scripts are sorted higher in the results (sort_order 5) +so that new merged indexes are created before subset indexes are disabled. */ SELECT /* First, show the information needed to understand the script */ From 9b795b4715f56318d7e90dd128bbf7ce3d8cd7fd Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Wed, 12 Mar 2025 18:36:25 -0400 Subject: [PATCH 042/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 170 ++++++++++++++++++ 1 file changed, 170 insertions(+) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index d166493c..495f1fe4 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -1831,6 +1831,176 @@ SELECT 'SEPARATOR', N'==================== END OF REPORT ===================='; +/* Find key duplicate groups that have multiple indexes with same action */ +IF OBJECT_ID('tempdb..#key_duplicate_dedup') IS NOT NULL + DROP TABLE #key_duplicate_dedup; + +CREATE TABLE #key_duplicate_dedup +( + database_id int, + object_id int, + database_name sysname, + schema_name sysname, + table_name sysname, + base_key_columns nvarchar(max), + filter_definition nvarchar(max), + winning_index_name sysname, + index_list nvarchar(max) +); + +/* Identify key duplicates where both indexes have MERGE INCLUDES action */ +INSERT INTO #key_duplicate_dedup +( + database_id, + object_id, + database_name, + schema_name, + table_name, + base_key_columns, + filter_definition, + winning_index_name, + index_list +) +SELECT + ia.database_id, + ia.object_id, + MAX(ia.database_name), + MAX(ia.schema_name), + MAX(ia.table_name), + ia.key_columns, + ISNULL(ia.filter_definition, ''), + /* Choose the first index by name as the winner (arbitrary but deterministic) */ + MIN(ia.index_name), + /* Build a list of other indexes in this group */ + STUFF(( + SELECT ', ' + inner_ia.index_name + FROM #index_analysis AS inner_ia + WHERE inner_ia.database_id = ia.database_id + AND inner_ia.object_id = ia.object_id + AND inner_ia.key_columns = ia.key_columns + AND ISNULL(inner_ia.filter_definition, '') = ISNULL(ia.filter_definition, '') + AND inner_ia.action = 'MERGE INCLUDES' + AND inner_ia.consolidation_rule = 'Key Duplicate' + ORDER BY inner_ia.index_name + FOR XML PATH(''), TYPE + ).value('.', 'nvarchar(max)'), 1, 2, '') +FROM #index_analysis AS ia +WHERE ia.action = 'MERGE INCLUDES' + AND ia.consolidation_rule = 'Key Duplicate' +GROUP BY + ia.database_id, + ia.object_id, + ia.key_columns, + ISNULL(ia.filter_definition, '') +HAVING COUNT(*) > 1; /* Only groups with multiple MERGE INCLUDES */ + +/* Update the index_analysis table to make only one index the winner in each group */ +UPDATE ia +SET + ia.action = 'DISABLE', + ia.target_index_name = kdd.winning_index_name, + ia.superseded_by = NULL +FROM #index_analysis AS ia +JOIN #key_duplicate_dedup AS kdd + ON ia.database_id = kdd.database_id + AND ia.object_id = kdd.object_id + AND ia.key_columns = kdd.base_key_columns + AND ISNULL(ia.filter_definition, '') = kdd.filter_definition +WHERE ia.index_name <> kdd.winning_index_name + AND ia.action = 'MERGE INCLUDES' + AND ia.consolidation_rule = 'Key Duplicate'; + +/* Update the winning index's superseded_by to list all other indexes */ +UPDATE ia +SET + ia.superseded_by = 'Supersedes ' + + REPLACE(kdd.index_list, ia.index_name + ', ', '') /* Remove self from list if present */ +FROM #index_analysis AS ia +JOIN #key_duplicate_dedup AS kdd + ON ia.database_id = kdd.database_id + AND ia.object_id = kdd.object_id + AND ia.key_columns = kdd.base_key_columns + AND ISNULL(ia.filter_definition, '') = kdd.filter_definition +WHERE ia.index_name = kdd.winning_index_name; + +/* Handle included column subsets - find where one index's includes are a subset of another */ +IF OBJECT_ID('tempdb..#include_subset_dedup') IS NOT NULL + DROP TABLE #include_subset_dedup; + +CREATE TABLE #include_subset_dedup +( + database_id int, + object_id int, + subset_index_name sysname, + superset_index_name sysname, + subset_included_columns nvarchar(max), + superset_included_columns nvarchar(max) +); + +/* Find indexes with same key columns where one has includes that are a subset of another */ +INSERT INTO #include_subset_dedup +( + database_id, + object_id, + subset_index_name, + superset_index_name, + subset_included_columns, + superset_included_columns +) +SELECT + ia1.database_id, + ia1.object_id, + ia1.index_name AS subset_index_name, + ia2.index_name AS superset_index_name, + ia1.included_columns AS subset_included_columns, + ia2.included_columns AS superset_included_columns +FROM #index_analysis AS ia1 +JOIN #index_analysis AS ia2 + ON ia1.database_id = ia2.database_id + AND ia1.object_id = ia2.object_id + AND ia1.key_columns = ia2.key_columns + AND ISNULL(ia1.filter_definition, '') = ISNULL(ia2.filter_definition, '') + AND ia1.index_name <> ia2.index_name + AND ia1.action = 'MERGE INCLUDES' + AND ia2.action = 'MERGE INCLUDES' + AND ia1.consolidation_rule = 'Key Duplicate' + AND ia2.consolidation_rule = 'Key Duplicate' + /* Find where subset's includes are contained within superset's includes */ + AND ( + ia1.included_columns IS NULL OR + CHARINDEX(ia1.included_columns, ia2.included_columns) > 0 + ) + /* Don't match if lengths are the same (would be exact duplicates) */ + AND ( + ia1.included_columns IS NULL OR ia2.included_columns IS NULL OR + LEN(ia1.included_columns) < LEN(ia2.included_columns) + ); + +/* Update the subset indexes to be disabled, since supersets already contain their columns */ +UPDATE ia +SET + ia.action = 'DISABLE', + ia.target_index_name = isd.superset_index_name, + ia.superseded_by = NULL +FROM #index_analysis AS ia +JOIN #include_subset_dedup AS isd + ON ia.database_id = isd.database_id + AND ia.object_id = isd.object_id + AND ia.index_name = isd.subset_index_name; + +/* Update the superset indexes to indicate they supersede the subset indexes */ +UPDATE ia +SET + ia.superseded_by = CASE + WHEN ia.superseded_by IS NULL THEN 'Supersedes ' + isd.subset_index_name + ELSE ia.superseded_by + ', ' + isd.subset_index_name + END +FROM #index_analysis AS ia +JOIN #include_subset_dedup AS isd + ON ia.database_id = isd.database_id + AND ia.object_id = isd.object_id + AND ia.index_name = isd.superset_index_name; + /* Insert merge scripts for indexes */ INSERT INTO #index_cleanup_results ( From 3066356f96e5bb384dfb241b039c91f7f95d4237 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Wed, 12 Mar 2025 18:49:14 -0400 Subject: [PATCH 043/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 149 ++++++++---------- 1 file changed, 70 insertions(+), 79 deletions(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index 495f1fe4..9ce6a081 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -455,9 +455,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. INDEX c CLUSTERED (database_id, schema_name, table_name, index_name) ); - - CREATE TABLE - #index_consolidation + + CREATE TABLE + #compression_eligibility ( database_id integer NOT NULL, database_name sysname NOT NULL, @@ -467,27 +467,51 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. table_name sysname NOT NULL, index_id integer NOT NULL, index_name sysname NOT NULL, - target_index_name sysname NULL, - consolidation_rule varchar(50) NULL, - index_priority integer NULL, - action varchar(50) NULL, + can_compress bit NOT NULL, + reason nvarchar(200) NULL, PRIMARY KEY (database_id, object_id, index_id) ); - + CREATE TABLE - #compression_eligibility + #key_duplicate_dedupe ( database_id integer NOT NULL, + object_id integer NOT NULL, database_name sysname NOT NULL, - schema_id integer NOT NULL, schema_name sysname NOT NULL, - object_id integer NOT NULL, table_name sysname NOT NULL, - index_id integer NOT NULL, - index_name sysname NOT NULL, - can_compress bit NOT NULL, - reason nvarchar(200) NULL, - PRIMARY KEY (database_id, object_id, index_id) + base_key_columns nvarchar(max) NULL, + filter_definition nvarchar(max) NULL, + winning_index_name sysname NULL, + index_list nvarchar(max) NULL + ); + + CREATE TABLE + #include_subset_dedupe + ( + database_id integer NOT NULL, + object_id integer NOT NULL, + subset_index_name sysname NULL, + superset_index_name sysname NULL, + subset_included_columns nvarchar(max) NULL, + superset_included_columns nvarchar(max) NULL + ); + + CREATE TABLE + #index_cleanup_results + ( + result_type varchar(50) NOT NULL, /* 'SUMMARY', 'MERGE', 'DISABLE', 'COMPRESS', etc. */ + sort_order integer NOT NULL, /* Keeps results in logical order */ + database_name sysname NULL, + schema_name sysname NULL, + table_name sysname NULL, + index_name sysname NULL, + script_type nvarchar(50) NULL, /* 'MERGE', 'DISABLE', 'COMPRESS', etc. */ + consolidation_rule nvarchar(200) NULL, + target_index_name sysname NULL, + script nvarchar(max) NULL, + additional_info nvarchar(max) NULL, /* For stats, constraints, etc. */ + superseded_info nvarchar(max) NULL /* To store superseded_by information */ ); /* @@ -1678,25 +1702,6 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. RAISERROR('Generating results', 0, 0) WITH NOWAIT; END; - /* - Create a consolidated results table to hold all outputs in a single result set - This provides a more cohesive experience with summary information and scripts in one place - */ - CREATE TABLE #index_cleanup_results - ( - result_type varchar(50) NOT NULL, /* 'SUMMARY', 'MERGE', 'DISABLE', 'COMPRESS', etc. */ - sort_order integer NOT NULL, /* Keeps results in logical order */ - database_name sysname NULL, - schema_name sysname NULL, - table_name sysname NULL, - index_name sysname NULL, - script_type nvarchar(50) NULL, /* 'MERGE', 'DISABLE', 'COMPRESS', etc. */ - consolidation_rule nvarchar(200) NULL, - target_index_name sysname NULL, - script nvarchar(max) NULL, - additional_info nvarchar(max) NULL, /* For stats, constraints, etc. */ - superseded_info nvarchar(max) NULL /* To store superseded_by information */ - ); /* Insert summary statistics first */ /* Add a separator row for the header */ @@ -1831,25 +1836,12 @@ SELECT 'SEPARATOR', N'==================== END OF REPORT ===================='; -/* Find key duplicate groups that have multiple indexes with same action */ -IF OBJECT_ID('tempdb..#key_duplicate_dedup') IS NOT NULL - DROP TABLE #key_duplicate_dedup; - -CREATE TABLE #key_duplicate_dedup -( - database_id int, - object_id int, - database_name sysname, - schema_name sysname, - table_name sysname, - base_key_columns nvarchar(max), - filter_definition nvarchar(max), - winning_index_name sysname, - index_list nvarchar(max) -); /* Identify key duplicates where both indexes have MERGE INCLUDES action */ -INSERT INTO #key_duplicate_dedup +INSERT INTO + #key_duplicate_dedupe +WITH + (TABLOCK) ( database_id, object_id, @@ -1901,7 +1893,7 @@ SET ia.target_index_name = kdd.winning_index_name, ia.superseded_by = NULL FROM #index_analysis AS ia -JOIN #key_duplicate_dedup AS kdd +JOIN #key_duplicate_dedupe AS kdd ON ia.database_id = kdd.database_id AND ia.object_id = kdd.object_id AND ia.key_columns = kdd.base_key_columns @@ -1916,29 +1908,18 @@ SET ia.superseded_by = 'Supersedes ' + REPLACE(kdd.index_list, ia.index_name + ', ', '') /* Remove self from list if present */ FROM #index_analysis AS ia -JOIN #key_duplicate_dedup AS kdd +JOIN #key_duplicate_dedupe AS kdd ON ia.database_id = kdd.database_id AND ia.object_id = kdd.object_id AND ia.key_columns = kdd.base_key_columns AND ISNULL(ia.filter_definition, '') = kdd.filter_definition WHERE ia.index_name = kdd.winning_index_name; -/* Handle included column subsets - find where one index's includes are a subset of another */ -IF OBJECT_ID('tempdb..#include_subset_dedup') IS NOT NULL - DROP TABLE #include_subset_dedup; - -CREATE TABLE #include_subset_dedup -( - database_id int, - object_id int, - subset_index_name sysname, - superset_index_name sysname, - subset_included_columns nvarchar(max), - superset_included_columns nvarchar(max) -); - /* Find indexes with same key columns where one has includes that are a subset of another */ -INSERT INTO #include_subset_dedup +INSERT INTO + #include_subset_dedupe +WITH + (TABLOCK) ( database_id, object_id, @@ -1983,7 +1964,7 @@ SET ia.target_index_name = isd.superset_index_name, ia.superseded_by = NULL FROM #index_analysis AS ia -JOIN #include_subset_dedup AS isd +JOIN #include_subset_dedupe AS isd ON ia.database_id = isd.database_id AND ia.object_id = isd.object_id AND ia.index_name = isd.subset_index_name; @@ -1996,13 +1977,14 @@ SET ELSE ia.superseded_by + ', ' + isd.subset_index_name END FROM #index_analysis AS ia -JOIN #include_subset_dedup AS isd +JOIN #include_subset_dedupe AS isd ON ia.database_id = isd.database_id AND ia.object_id = isd.object_id AND ia.index_name = isd.superset_index_name; /* Insert merge scripts for indexes */ -INSERT INTO #index_cleanup_results +INSERT INTO + #index_cleanup_results ( result_type, sort_order, @@ -2126,7 +2108,8 @@ AND ce.can_compress = 1 AND ia.target_index_name IS NULL; /* Insert disable scripts for unneeded indexes */ -INSERT INTO #index_cleanup_results +INSERT INTO + #index_cleanup_results ( result_type, sort_order, @@ -2178,7 +2161,8 @@ FROM #index_analysis AS ia WHERE ia.action = 'DISABLE'; /* Insert compression scripts for remaining indexes */ -INSERT INTO #index_cleanup_results +INSERT INTO + #index_cleanup_results ( result_type, sort_order, @@ -2253,7 +2237,8 @@ WHERE AND ce.can_compress = 1; /* Insert disable scripts for unique constraints */ -INSERT INTO #index_cleanup_results +INSERT INTO + #index_cleanup_results ( result_type, sort_order, @@ -2292,8 +2277,10 @@ WHERE /* Only indexes that are being made unique */ ia.action = 'MAKE UNIQUE' /* Find the constraint that matches the index being made unique */ - AND EXISTS ( - SELECT 1/0 + AND EXISTS + ( + SELECT + 1/0 FROM #index_details id_nc WHERE id_nc.database_id = ia.database_id AND id_nc.object_id = ia.object_id @@ -2322,7 +2309,8 @@ WHERE ); /* Insert per-partition compression scripts */ -INSERT INTO #index_cleanup_results +INSERT INTO + #index_cleanup_results ( result_type, sort_order, @@ -2388,7 +2376,10 @@ WHERE /* Space savings reporting has been moved to #index_cleanup_results table */ /* Insert compression ineligible info */ -INSERT INTO #index_cleanup_results +INSERT INTO + #index_cleanup_results +WITH + (TABLOCK) ( result_type, sort_order, From 0223806446fdb91ec2e1fed182edb577c4cfce43 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Wed, 12 Mar 2025 18:52:49 -0400 Subject: [PATCH 044/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index 9ce6a081..020f1445 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -1818,7 +1818,7 @@ INSERT INTO #index_cleanup_results ) SELECT 'HEADER', - 9, + 4, /* Just before merge scripts at sort_order 5 */ 'SEPARATOR', N'==================== INDEX SCRIPTS ===================='; @@ -1861,8 +1861,24 @@ SELECT MAX(ia.table_name), ia.key_columns, ISNULL(ia.filter_definition, ''), - /* Choose the first index by name as the winner (arbitrary but deterministic) */ - MIN(ia.index_name), + /* Choose the index with most included columns as the winner (or first alphabetically if tied) */ + ( + SELECT TOP 1 candidate.index_name + FROM #index_analysis AS candidate + WHERE candidate.database_id = ia.database_id + AND candidate.object_id = ia.object_id + AND candidate.key_columns = ia.key_columns + AND ISNULL(candidate.filter_definition, '') = ISNULL(ia.filter_definition, '') + AND candidate.action = 'MERGE INCLUDES' + AND candidate.consolidation_rule = 'Key Duplicate' + ORDER BY + /* First prefer indexes with "_Extended" in the name */ + CASE WHEN candidate.index_name LIKE '%\_Extended%' ESCAPE '\' THEN 1 ELSE 0 END DESC, + /* Then prefer indexes with more included columns (by length as a proxy) */ + LEN(ISNULL(candidate.included_columns, '')) DESC, + /* Then alphabetically for stability */ + candidate.index_name + ), /* Build a list of other indexes in this group */ STUFF(( SELECT ', ' + inner_ia.index_name From 2b5d6a27e02c4df9c7ba624d896c244fc82bfe80 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Wed, 12 Mar 2025 21:53:08 -0400 Subject: [PATCH 045/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index 020f1445..6cf7a92d 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -1998,6 +1998,31 @@ JOIN #include_subset_dedupe AS isd AND ia.object_id = isd.object_id AND ia.index_name = isd.superset_index_name; +/* Update winning indexes that don't actually need changes to have action = 'KEEP' */ +UPDATE ia +SET + /* Change action to 'KEEP' for indexes that don't need to be modified */ + ia.action = 'KEEP' +FROM #index_analysis AS ia +WHERE ia.action = 'MERGE INCLUDES' +AND ia.superseded_by IS NOT NULL +/* Check if the index name contains "Extended" and has more included columns */ +AND (ia.index_name LIKE '%\_Extended%' ESCAPE '\' OR ia.index_name LIKE '%\_Extended' OR ia.index_name LIKE '%_Extended%') +/* This should indicate it already has all the needed includes */ +AND NOT EXISTS ( + /* Find any indexes it supersedes that have includes not in this index */ + SELECT 1 + FROM #index_analysis AS ia_subset + WHERE ia_subset.database_id = ia.database_id + AND ia_subset.object_id = ia.object_id + AND ia_subset.key_columns = ia.key_columns + AND ia_subset.action = 'DISABLE' + AND ia_subset.target_index_name = ia.index_name + /* This complex check handles cases where the superset doesn't contain all subset columns */ + AND CHARINDEX(ISNULL(ia_subset.included_columns, ''), ISNULL(ia.included_columns, '')) = 0 + AND ISNULL(ia_subset.included_columns, '') <> '' +); + /* Insert merge scripts for indexes */ INSERT INTO #index_cleanup_results @@ -2142,7 +2167,11 @@ INSERT INTO ) SELECT 'DISABLE', - 20, + /* Sort duplicate/subset indexes first (20), then unused indexes last (25) */ + CASE + WHEN ia.consolidation_rule LIKE 'Unused Index%' THEN 25 + ELSE 20 + END, ia.database_name, ia.schema_name, ia.table_name, From 88b82293326b78a70005e32eed39b2ba96ce37c0 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Wed, 12 Mar 2025 22:38:48 -0400 Subject: [PATCH 046/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 366 +++++++++++++++++- 1 file changed, 354 insertions(+), 12 deletions(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index 6cf7a92d..11271337 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -511,7 +511,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. target_index_name sysname NULL, script nvarchar(max) NULL, additional_info nvarchar(max) NULL, /* For stats, constraints, etc. */ - superseded_info nvarchar(max) NULL /* To store superseded_by information */ + superseded_info nvarchar(max) NULL, /* To store superseded_by information */ + index_size_gb decimal(18,4) NULL, /* Size of the index in GB */ + index_rows bigint NULL, /* Number of rows in the index */ + index_reads bigint NULL, /* Total reads (seeks + scans + lookups) */ + index_writes bigint NULL /* Total writes (updates) */ ); /* @@ -1687,6 +1691,75 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND id1_inner.is_included_column = 0 ) ); + + /* Rule 7: Identify indexes with same keys but in different order after first column */ + /* This rule flags indexes that have the same set of key columns but ordered differently */ + /* These need manual review as they may be redundant depending on query patterns */ + UPDATE + ia1 + SET + ia1.consolidation_rule = 'Same Keys Different Order', + ia1.action = 'REVIEW', /* These need manual review */ + ia1.target_index_name = ia2.index_name /* Reference the partner index */ + FROM #index_analysis AS ia1 + JOIN #index_analysis AS ia2 + ON ia1.database_id = ia2.database_id + AND ia1.object_id = ia2.object_id + AND ia1.index_name < ia2.index_name /* Only process each pair once */ + AND ia1.consolidation_rule IS NULL /* Not already processed */ + AND ia2.consolidation_rule IS NULL /* Not already processed */ + WHERE + /* Leading columns match */ + EXISTS ( + SELECT TOP 1 id1.column_name + FROM #index_details id1 + JOIN #index_details id2 + ON id1.database_id = id2.database_id + AND id1.object_id = id2.object_id + AND id1.column_name = id2.column_name + AND id1.key_ordinal = 1 + AND id2.key_ordinal = 1 + WHERE id1.database_id = ia1.database_id + AND id1.object_id = ia1.object_id + AND id1.index_name = ia1.index_name + AND id2.index_name = ia2.index_name + ) + /* Same set of key columns but in different order */ + AND NOT EXISTS ( + /* Make sure the sets of key columns are exactly the same */ + SELECT id1.column_name + FROM #index_details id1 + WHERE id1.database_id = ia1.database_id + AND id1.object_id = ia1.object_id + AND id1.index_name = ia1.index_name + AND id1.is_included_column = 0 + AND id1.key_ordinal > 0 + EXCEPT + SELECT id2.column_name + FROM #index_details id2 + WHERE id2.database_id = ia2.database_id + AND id2.object_id = ia2.object_id + AND id2.index_name = ia2.index_name + AND id2.is_included_column = 0 + AND id2.key_ordinal > 0 + ) + /* But the order is different (excluding the first column) */ + AND EXISTS ( + /* There's at least one column in a different position */ + SELECT 1 + FROM #index_details id1 + JOIN #index_details id2 + ON id1.database_id = id2.database_id + AND id1.object_id = id2.object_id + AND id1.column_name = id2.column_name + AND id1.key_ordinal <> id2.key_ordinal + AND id1.key_ordinal > 1 /* After the first column */ + AND id2.key_ordinal > 1 /* After the first column */ + WHERE id1.database_id = ia1.database_id + AND id1.object_id = ia1.object_id + AND id1.index_name = ia1.index_name + AND id2.index_name = ia2.index_name + ); IF @debug = 1 @@ -2038,7 +2111,11 @@ INSERT INTO target_index_name, script, additional_info, - superseded_info + superseded_info, + index_size_gb, + index_rows, + index_reads, + index_writes ) SELECT 'MERGE', @@ -2163,7 +2240,11 @@ INSERT INTO script, additional_info, target_index_name, - superseded_info + superseded_info, + index_size_gb, + index_rows, + index_reads, + index_writes ) SELECT 'DISABLE', @@ -2201,8 +2282,22 @@ SELECT ELSE N'This index is redundant' END, ia.target_index_name, /* Include the target index name */ - NULL /* Don't need superseded_by info for disabled indexes */ + NULL, /* Don't need superseded_by info for disabled indexes */ + ps.total_space_gb, + ps.row_count, + (id.user_seeks + id.user_scans + id.user_lookups), + id.user_updates FROM #index_analysis AS ia +LEFT JOIN #partition_stats AS ps + ON ia.database_id = ps.database_id + AND ia.object_id = ps.object_id + AND ia.index_id = ps.index_id +LEFT JOIN #index_details AS id + ON id.database_id = ia.database_id + AND id.object_id = ia.object_id + AND id.index_name = ia.index_name + AND id.is_included_column = 0 /* Get only one row per index */ + AND id.key_ordinal > 0 WHERE ia.action = 'DISABLE'; /* Insert compression scripts for remaining indexes */ @@ -2219,7 +2314,11 @@ INSERT INTO script, additional_info, target_index_name, - superseded_info + superseded_info, + index_size_gb, + index_rows, + index_reads, + index_writes ) SELECT 'COMPRESS', @@ -2247,7 +2346,11 @@ SELECT N', DATA_COMPRESSION = PAGE);', N'Compression type: All Partitions', NULL, /* No target index for compression scripts */ - ia.superseded_by /* Include superseded_by info for compression scripts */ + ia.superseded_by, /* Include superseded_by info for compression scripts */ + ps_full.total_space_gb, + ps_full.row_count, + (id.user_seeks + id.user_scans + id.user_lookups), + id.user_updates FROM #index_analysis AS ia LEFT JOIN ( @@ -2271,6 +2374,16 @@ LEFT JOIN ON ia.database_id = ps.database_id AND ia.object_id = ps.object_id AND ia.index_id = ps.index_id +LEFT JOIN #partition_stats AS ps_full + ON ia.database_id = ps_full.database_id + AND ia.object_id = ps_full.object_id + AND ia.index_id = ps_full.index_id +LEFT JOIN #index_details AS id + ON id.database_id = ia.database_id + AND id.object_id = ia.object_id + AND id.index_name = ia.index_name + AND id.is_included_column = 0 /* Get only one row per index */ + AND id.key_ordinal > 0 JOIN #compression_eligibility ce ON ia.database_id = ce.database_id AND ia.object_id = ce.object_id @@ -2293,7 +2406,11 @@ INSERT INTO index_name, script_type, additional_info, - script + script, + index_size_gb, + index_rows, + index_reads, + index_writes ) SELECT 'CONSTRAINT', @@ -2312,12 +2429,26 @@ SELECT QUOTENAME(ia.table_name) + N' NOCHECK CONSTRAINT ' + QUOTENAME(id.index_name) + - N';' + N';', + ps.total_space_gb, + ps.row_count, + (id2.user_seeks + id2.user_scans + id2.user_lookups), + id2.user_updates FROM #index_analysis AS ia JOIN #index_details AS id ON id.database_id = ia.database_id AND id.object_id = ia.object_id AND id.is_unique_constraint = 1 +LEFT JOIN #index_details AS id2 + ON id2.database_id = ia.database_id + AND id2.object_id = ia.object_id + AND id2.index_name = ia.index_name + AND id2.is_included_column = 0 /* Get only one row per index */ + AND id2.key_ordinal > 0 +LEFT JOIN #partition_stats AS ps + ON ia.database_id = ps.database_id + AND ia.object_id = ps.object_id + AND ia.index_id = ps.index_id WHERE /* Only indexes that are being made unique */ ia.action = 'MAKE UNIQUE' @@ -2367,7 +2498,11 @@ INSERT INTO script, additional_info, target_index_name, - superseded_info + superseded_info, + index_size_gb, + index_rows, + index_reads, + index_writes ) SELECT 'COMPRESS_PARTITION', @@ -2398,12 +2533,22 @@ SELECT CONVERT(nvarchar(20), CONVERT(decimal(10,4), ps.total_space_gb)) + N' GB', NULL, - NULL + NULL, + ps.total_space_gb, + ps.row_count, + (id.user_seeks + id.user_scans + id.user_lookups), + id.user_updates FROM #index_analysis AS ia JOIN #partition_stats AS ps ON ia.database_id = ps.database_id AND ia.object_id = ps.object_id AND ia.index_id = ps.index_id +LEFT JOIN #index_details AS id + ON id.database_id = ia.database_id + AND id.object_id = ia.object_id + AND id.index_name = ia.index_name + AND id.is_included_column = 0 /* Get only one row per index */ + AND id.key_ordinal > 0 JOIN #compression_eligibility AS ce ON ia.database_id = ce.database_id AND ia.object_id = ce.object_id @@ -2433,7 +2578,11 @@ WITH table_name, index_name, script_type, - additional_info + additional_info, + index_size_gb, + index_rows, + index_reads, + index_writes ) SELECT 'INELIGIBLE', @@ -2443,10 +2592,165 @@ SELECT ce.table_name, ce.index_name, 'INELIGIBLE FOR COMPRESSION', - ce.reason + ce.reason, + ps.total_space_gb, + ps.row_count, + (id.user_seeks + id.user_scans + id.user_lookups), + id.user_updates FROM #compression_eligibility AS ce +LEFT JOIN #partition_stats AS ps + ON ce.database_id = ps.database_id + AND ce.object_id = ps.object_id + AND ce.index_id = ps.index_id +LEFT JOIN #index_details AS id + ON id.database_id = ce.database_id + AND id.object_id = ce.object_id + AND id.index_name = ce.index_name + AND id.is_included_column = 0 /* Get only one row per index */ + AND id.key_ordinal > 0 WHERE ce.can_compress = 0; +/* Add a separator for indexes needing review section */ +INSERT INTO + #index_cleanup_results +WITH + (TABLOCK) +( + result_type, + sort_order, + script_type, + additional_info +) +SELECT + 'HEADER', + 92, + 'SEPARATOR', + N'==================== INDEXES NEEDING REVIEW ===================='; + +/* Insert indexes identified for manual review */ +INSERT INTO + #index_cleanup_results +WITH + (TABLOCK) +( + result_type, + sort_order, + database_name, + schema_name, + table_name, + index_name, + script_type, + consolidation_rule, + target_index_name, + additional_info, + index_size_gb, + index_rows, + index_reads, + index_writes +) +SELECT + 'REVIEW', + 93, /* Just before KEPT indexes */ + ia.database_name, + ia.schema_name, + ia.table_name, + ia.index_name, + 'NEEDS REVIEW', + ia.consolidation_rule, + ia.target_index_name, + CASE + WHEN ia.consolidation_rule = 'Same Keys Different Order' + THEN 'This index has the same key columns as ' + ISNULL(ia.target_index_name, N'(unknown)') + + ' but in a different order. May be redundant depending on query patterns.' + ELSE 'This index needs manual review' + END, + ps.total_space_gb, + ps.row_count, + (id.user_seeks + id.user_scans + id.user_lookups), + id.user_updates +FROM #index_analysis AS ia +LEFT JOIN #partition_stats AS ps + ON ia.database_id = ps.database_id + AND ia.object_id = ps.object_id + AND ia.index_id = ps.index_id +LEFT JOIN #index_details AS id + ON id.database_id = ia.database_id + AND id.object_id = ia.object_id + AND id.index_name = ia.index_name + AND id.is_included_column = 0 /* Get only one row per index */ + AND id.key_ordinal > 0 +WHERE ia.action = 'REVIEW'; + +/* Add a separator for kept indexes section */ +INSERT INTO + #index_cleanup_results +WITH + (TABLOCK) +( + result_type, + sort_order, + script_type, + additional_info +) +SELECT + 'HEADER', + 94, + 'SEPARATOR', + N'==================== INDEXES KEPT ===================='; + +/* Insert indexes that are being kept (superset indexes and others) */ +INSERT INTO + #index_cleanup_results +WITH + (TABLOCK) +( + result_type, + sort_order, + database_name, + schema_name, + table_name, + index_name, + script_type, + consolidation_rule, + superseded_info, + additional_info, + index_size_gb, + index_rows, + index_reads, + index_writes +) +SELECT + 'KEEP', + 95, /* Just before END OF REPORT at 99 */ + ia.database_name, + ia.schema_name, + ia.table_name, + ia.index_name, + 'KEPT', + ia.consolidation_rule, + ia.superseded_by, + CASE + WHEN ia.superseded_by IS NOT NULL THEN 'This index supersedes other indexes and already has all needed columns' + WHEN ia.action = 'KEEP' THEN 'This index is being kept' + ELSE NULL + END, + ps.total_space_gb, + ps.row_count, + (id.user_seeks + id.user_scans + id.user_lookups), + id.user_updates +FROM #index_analysis AS ia +LEFT JOIN #partition_stats AS ps + ON ia.database_id = ps.database_id + AND ia.object_id = ps.object_id + AND ia.index_id = ps.index_id +LEFT JOIN #index_details AS id + ON id.database_id = ia.database_id + AND id.object_id = ia.object_id + AND id.index_name = ia.index_name + AND id.is_included_column = 0 /* Get only one row per index */ + AND id.key_ordinal > 0 +WHERE ia.action = 'KEEP' OR (ia.action IS NULL AND ia.consolidation_rule IS NULL); + /* Return the consolidated results in a single result set Results are ordered by: @@ -2457,9 +2761,12 @@ Results are ordered by: 5. Compression scripts (for tables eligible for compression) 6. Partition-specific compression scripts 7. Ineligible objects (tables that can't be compressed) +8. Kept indexes - sort_order 95 Note: Merge target scripts are sorted higher in the results (sort_order 5) so that new merged indexes are created before subset indexes are disabled. + +Within each category, indexes are sorted by size and impact for better prioritization. */ SELECT /* First, show the information needed to understand the script */ @@ -2475,6 +2782,27 @@ SELECT ir.target_index_name, /* Include superseded_by info for winning indexes */ CASE WHEN ia.superseded_by IS NOT NULL THEN ia.superseded_by ELSE ir.superseded_info END AS superseded_info, + /* Add size and usage metrics */ + index_size_gb = + CASE + WHEN ir.result_type IN ('HEADER', 'SUMMARY') THEN NULL + ELSE FORMAT(ir.index_size_gb, 'N4') + END, + index_rows = + CASE + WHEN ir.result_type IN ('HEADER', 'SUMMARY') THEN NULL + ELSE FORMAT(ir.index_rows, 'N0') + END, + index_reads = + CASE + WHEN ir.result_type IN ('HEADER', 'SUMMARY') THEN NULL + ELSE FORMAT(ir.index_reads, 'N0') + END, + index_writes = + CASE + WHEN ir.result_type IN ('HEADER', 'SUMMARY') THEN NULL + ELSE FORMAT(ir.index_writes, 'N0') + END, /* Finally show the actual script */ ir.script FROM #index_cleanup_results AS ir @@ -2485,6 +2813,20 @@ LEFT JOIN #index_analysis AS ia AND ir.index_name = ia.index_name ORDER BY ir.sort_order, + /* Within each sort_order group, prioritize by size and usage */ + CASE + /* For HEADER and SUMMARY, keep the original order */ + WHEN ir.result_type IN ('HEADER', 'SUMMARY') THEN 0 + /* For script categories, order by size and impact */ + ELSE ISNULL(ir.index_size_gb, 0) + END DESC, + CASE + /* For HEADER and SUMMARY, keep the original order */ + WHEN ir.result_type IN ('HEADER', 'SUMMARY') THEN 0 + /* For script categories, consider rows as secondary sort */ + ELSE ISNULL(ir.index_rows, 0) + END DESC, + /* Then by database, schema, table, index name for consistent ordering */ ir.database_name, ir.schema_name, ir.table_name, From 0783fc1dfce09cace3b1a5730ea93ac3f0c7d5e1 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Wed, 12 Mar 2025 23:16:08 -0400 Subject: [PATCH 047/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 339 ++++++++++-------- 1 file changed, 191 insertions(+), 148 deletions(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index 11271337..c84d4721 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -1,9 +1,9 @@ /* -EXEC sp_IndexCleanup +EXECUTE sp_IndexCleanup @database_name = 'StackOverflow2013', @debug = 1; -EXEC sp_IndexCleanup +EXECUTE sp_IndexCleanup @database_name = 'StackOverflow2013', @table_name = 'Users', @debug = 1 @@ -92,7 +92,6 @@ It needs lots of love and testing in real environments with real indexes to fix SELECT help = N'without careful analysis and consideration. it may be harmful.' - /* Parameters */ @@ -104,25 +103,55 @@ It needs lots of love and testing in real environments with real indexes to fix description = CASE ap.name - WHEN ap.name - THEN ap.name + WHEN N'@database_name' THEN 'the name of the database you wish to analyze' + WHEN N'@schema_name' THEN 'the schema name to filter indexes by' + WHEN N'@table_name' THEN 'the table name to filter indexes by' + WHEN N'@min_reads' THEN 'minimum number of reads for an index to be considered used' + WHEN N'@min_writes' THEN 'minimum number of writes for an index to be considered used' + WHEN N'@min_size_gb' THEN 'minimum size in GB for an index to be analyzed' + WHEN N'@min_rows' THEN 'minimum number of rows for a table to be analyzed' + WHEN N'@help' THEN 'displays this help information' + WHEN N'@debug' THEN 'prints debug information during execution' + WHEN N'@version' THEN 'returns the version number of the procedure' + WHEN N'@version_date' THEN 'returns the date this version was released' + ELSE NULL END, valid_inputs = CASE ap.name - WHEN ap.name - THEN ap.name + WHEN N'@database_name' THEN 'the name of a database you care about indexes in' + WHEN N'@schema_name' THEN 'schema name or NULL for all schemas' + WHEN N'@table_name' THEN 'table name or NULL for all tables' + WHEN N'@min_reads' THEN 'any positive integer or 0' + WHEN N'@min_writes' THEN 'any positive integer or 0' + WHEN N'@min_size_gb' THEN 'any positive decimal number or 0' + WHEN N'@min_rows' THEN 'any positive integer or 0' + WHEN N'@help' THEN '0 or 1' + WHEN N'@debug' THEN '0 or 1' + WHEN N'@version' THEN 'OUTPUT parameter' + WHEN N'@version_date' THEN 'OUTPUT parameter' + ELSE NULL END, defaults = CASE ap.name - WHEN ap.name - THEN ap.name + WHEN N'@database_name' THEN 'NULL' + WHEN N'@schema_name' THEN 'NULL' + WHEN N'@table_name' THEN 'NULL' + WHEN N'@min_reads' THEN '0' + WHEN N'@min_writes' THEN '0' + WHEN N'@min_size_gb' THEN '0' + WHEN N'@min_rows' THEN '0' + WHEN N'@help' THEN 'false' + WHEN N'@debug' THEN 'true' + WHEN N'@version' THEN 'NULL' + WHEN N'@version_date' THEN 'NULL' + ELSE NULL END FROM sys.all_parameters AS ap - INNER JOIN sys.all_objects AS o + JOIN sys.all_objects AS o ON ap.object_id = o.object_id - INNER JOIN sys.types AS t + JOIN sys.types AS t ON ap.system_type_id = t.system_type_id AND ap.user_type_id = t.user_type_id WHERE o.name = N'sp_IndexCleanup' @@ -186,9 +215,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Compression variables */ @can_compress bit = CASE - WHEN CONVERT(int, SERVERPROPERTY('EngineEdition')) IN (3, 5, 8) - OR (CONVERT(int, SERVERPROPERTY('EngineEdition')) = 2 - AND CONVERT(int, SUBSTRING(CONVERT(varchar(20), SERVERPROPERTY('ProductVersion')), 1, 2)) >= 13) + WHEN + CONVERT(integer, SERVERPROPERTY('EngineEdition')) IN (3, 5, 8) + OR + ( + CONVERT(integer, SERVERPROPERTY('EngineEdition')) = 2 + AND CONVERT(integer, SUBSTRING(CONVERT(varchar(20), SERVERPROPERTY('ProductVersion')), 1, 2)) >= 13 + ) THEN 1 ELSE 0 END, @@ -321,8 +354,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. table_name sysname NOT NULL, index_id integer NOT NULL, index_name sysname NOT NULL - PRIMARY KEY - (database_id, schema_id, object_id, index_id) + PRIMARY KEY CLUSTERED(database_id, schema_id, object_id, index_id) ); CREATE TABLE @@ -366,8 +398,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. page_io_latch_wait_in_ms bigint NULL, page_compression_attempt_count bigint NULL, page_compression_success_count bigint NULL, - PRIMARY KEY CLUSTERED - (database_id, schema_id, object_id, index_id) + PRIMARY KEY CLUSTERED (database_id, schema_id, object_id, index_id) ); CREATE TABLE @@ -446,14 +477,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. included_columns nvarchar(max) NULL, filter_definition nvarchar(max) NULL, is_redundant bit NULL, - superseded_by sysname NULL, + superseded_by nvarchar(256) NULL, missing_columns nvarchar(max) NULL, action nvarchar(max) NULL, target_index_name sysname NULL, consolidation_rule varchar(512) NULL, index_priority int NULL, - INDEX c CLUSTERED - (database_id, schema_name, table_name, index_name) + INDEX c CLUSTERED (database_id, schema_id, object_id, index_id) ); CREATE TABLE @@ -469,7 +499,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. index_name sysname NOT NULL, can_compress bit NOT NULL, reason nvarchar(200) NULL, - PRIMARY KEY (database_id, object_id, index_id) + PRIMARY KEY CLUSTERED(database_id, object_id, index_id) ); CREATE TABLE @@ -483,7 +513,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. base_key_columns nvarchar(max) NULL, filter_definition nvarchar(max) NULL, winning_index_name sysname NULL, - index_list nvarchar(max) NULL + index_list nvarchar(max) NULL, ); CREATE TABLE @@ -507,7 +537,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. table_name sysname NULL, index_name sysname NULL, script_type nvarchar(50) NULL, /* 'MERGE', 'DISABLE', 'COMPRESS', etc. */ - consolidation_rule nvarchar(200) NULL, + consolidation_rule nvarchar(256) NULL, target_index_name sysname NULL, script nvarchar(max) NULL, additional_info nvarchar(max) NULL, /* For stats, constraints, etc. */ @@ -579,7 +609,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 1/0 FROM ' + QUOTENAME(@database_name) + N'.sys.dm_db_partition_stats AS ps WHERE ps.object_id = t.object_id - AND ps.index_id IN (0, 1) + AND ps.index_id IN (0, 1) GROUP BY ps.object_id HAVING @@ -591,7 +621,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 1/0 FROM ' + QUOTENAME(@database_name) + N'.sys.dm_db_index_usage_stats AS ius WHERE ius.object_id = t.object_id - AND ius.database_id = @database_id + AND ius.database_id = @database_id GROUP BY ius.object_id HAVING @@ -620,7 +650,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. index_id, index_name ) - EXEC sys.sp_executesql + EXECUTE sys.sp_executesql @sql, N'@database_id int, @min_reads bigint, @@ -642,15 +672,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SELECT table_name = '#filtered_objects', fo.* - FROM #filtered_objects AS fo; + FROM #filtered_objects AS fo + OPTION(RECOMPILE); + + RAISERROR('Generaring #compression_eligibility insert', 0, 0) WITH NOWAIT; END; - /* Populate compression eligibility table */ - IF @debug = 1 - BEGIN - RAISERROR('Populating #compression_eligibility', 0, 0) WITH NOWAIT; - END; - + /* Populate compression eligibility table */ INSERT INTO #compression_eligibility WITH @@ -678,42 +706,55 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. fo.index_name, 1, /* Default to compressible */ NULL - FROM #filtered_objects AS fo; + FROM #filtered_objects AS fo + OPTION(RECOMPILE); /* If SQL Server edition doesn't support compression, mark all as ineligible */ IF @can_compress = 0 BEGIN - UPDATE #compression_eligibility + UPDATE + #compression_eligibility SET can_compress = 0, reason = N'SQL Server edition or version does not support compression' - WHERE can_compress = 1; + WHERE can_compress = 1 + OPTION(RECOMPILE); END; /* Check for sparse columns or incompatible data types */ IF @can_compress = 1 BEGIN + RAISERROR('Updating #compression_eligibility', 0, 0) WITH NOWAIT; + SELECT @sql = N' SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; - UPDATE ce + UPDATE + ce SET - can_compress = 0, - reason = ''Table contains sparse columns or incompatible data types'' + ce.can_compress = 0, + ce.reason = ''Table contains sparse columns or incompatible data types'' FROM #compression_eligibility AS ce WHERE EXISTS ( - SELECT 1/0 + SELECT + 1/0 FROM ' + QUOTENAME(@database_name) + N'.sys.columns AS c JOIN ' + QUOTENAME(@database_name) + N'.sys.types AS t ON c.user_type_id = t.user_type_id WHERE c.object_id = ce.object_id - AND (c.is_sparse = 1 OR t.name IN (N''text'', N''ntext'', N''image'')) - ); + AND + ( + c.is_sparse = 1 + OR t.name IN (N''text'', N''ntext'', N''image'') + ) + ) + OPTION(RECOMPILE); '; - EXEC sys.sp_executesql @sql; + EXECUTE sys.sp_executesql + @sql; END; IF @debug = 1 @@ -721,13 +762,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SELECT table_name = '#compression_eligibility', ce.* - FROM #compression_eligibility AS ce; - END; + FROM #compression_eligibility AS ce + OPTION(RECOMPILE); - IF @debug = 1 - BEGIN RAISERROR('Generating #operational_stats insert', 0, 0) WITH NOWAIT; - END; + END; SELECT @sql = N' @@ -856,7 +895,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. page_compression_attempt_count, page_compression_success_count ) - EXEC sys.sp_executesql + EXECUTE sys.sp_executesql @sql, N'@database_id integer, @object_id integer', @@ -870,13 +909,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SELECT table_name = '#operational_stats', os.* - FROM #operational_stats AS os; - END; + FROM #operational_stats AS os + OPTION(RECOMPILE); - IF @debug = 1 - BEGIN RAISERROR('Generating #index_details insert', 0, 0) WITH NOWAIT; - END; + END; SELECT @sql = N' @@ -1079,7 +1116,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. last_user_update, is_eligible_for_dedupe ) - EXEC sys.sp_executesql + EXECUTE sys.sp_executesql @sql, N'@database_id integer, @object_id integer, @@ -1096,12 +1133,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. table_name = '#index_details', * FROM #index_details AS id; - END; - IF @debug = 1 - BEGIN RAISERROR('Generating #partition_stats insert', 0, 0) WITH NOWAIT; - END; + END; SELECT @sql = N' @@ -1267,7 +1301,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. partition_function_name, partition_columns ) - EXEC sys.sp_executesql + EXECUTE sys.sp_executesql @sql, N'@database_id integer, @object_id integer', @@ -1282,12 +1316,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. table_name = '#partition_stats', * FROM #partition_stats AS ps; - END; - IF @debug = 1 - BEGIN RAISERROR('Performing #index_analysis insert', 0, 0) WITH NOWAIT; - END; + END; INSERT INTO #index_analysis @@ -1393,10 +1424,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. table_name = '#index_analysis', ia.* FROM #index_analysis AS ia; - END; - IF @debug = 1 - BEGIN RAISERROR('Starting updates', 0, 0) WITH NOWAIT; END; @@ -1439,7 +1467,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. WHERE id.index_name = #index_analysis.index_name AND id.table_name = #index_analysis.table_name AND id.user_scans > 0 - ) THEN 50 ELSE 0 + ) THEN 100 ELSE 0 END; /* Indexes with scans get some priority */ @@ -1453,7 +1481,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. THEN 'Unused Index (WARNING: Server uptime < 14 days - usage data may be incomplete)' ELSE 'Unused Index' END, - action = 'DISABLE' + action = N'DISABLE' WHERE EXISTS ( SELECT @@ -1546,8 +1574,12 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* For the winning index, set clear superseded_by text for the report */ ia1.superseded_by = CASE - WHEN (ia1.is_unique = 1 AND ia2.is_unique = 0) OR - (ia1.index_priority >= ia2.index_priority AND NOT (ia1.is_unique = 0 AND ia2.is_unique = 1)) + WHEN (ia1.is_unique = 1 AND ia2.is_unique = 0) + OR + ( + ia1.index_priority >= ia2.index_priority + AND NOT (ia1.is_unique = 0 AND ia2.is_unique = 1) + ) THEN 'Supersedes ' + ia2.index_name ELSE NULL END @@ -1681,7 +1713,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND id2_inner.object_id = id2.object_id AND id2_inner.index_id = id2.index_id AND id2_inner.is_included_column = 0 + EXCEPT + SELECT id1_inner.column_name FROM #index_details id1_inner @@ -1710,56 +1744,66 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND ia2.consolidation_rule IS NULL /* Not already processed */ WHERE /* Leading columns match */ - EXISTS ( - SELECT TOP 1 id1.column_name - FROM #index_details id1 - JOIN #index_details id2 - ON id1.database_id = id2.database_id + EXISTS + ( + SELECT + 1/0 + FROM #index_details AS id1 + JOIN #index_details AS id2 + ON id1.database_id = id2.database_id AND id1.object_id = id2.object_id AND id1.column_name = id2.column_name AND id1.key_ordinal = 1 AND id2.key_ordinal = 1 WHERE id1.database_id = ia1.database_id - AND id1.object_id = ia1.object_id - AND id1.index_name = ia1.index_name - AND id2.index_name = ia2.index_name + AND id1.object_id = ia1.object_id + AND id1.index_name = ia1.index_name + AND id2.index_name = ia2.index_name ) /* Same set of key columns but in different order */ - AND NOT EXISTS ( + AND NOT EXISTS + ( /* Make sure the sets of key columns are exactly the same */ - SELECT id1.column_name - FROM #index_details id1 + SELECT + id1.column_name + FROM #index_details AS id1 WHERE id1.database_id = ia1.database_id AND id1.object_id = ia1.object_id AND id1.index_name = ia1.index_name AND id1.is_included_column = 0 AND id1.key_ordinal > 0 + EXCEPT - SELECT id2.column_name - FROM #index_details id2 + + SELECT + id2.column_name + FROM #index_details AS id2 WHERE id2.database_id = ia2.database_id - AND id2.object_id = ia2.object_id - AND id2.index_name = ia2.index_name - AND id2.is_included_column = 0 - AND id2.key_ordinal > 0 + AND id2.object_id = ia2.object_id + AND id2.index_name = ia2.index_name + AND id2.is_included_column = 0 + AND id2.key_ordinal > 0 ) /* But the order is different (excluding the first column) */ - AND EXISTS ( + AND EXISTS + ( /* There's at least one column in a different position */ - SELECT 1 - FROM #index_details id1 - JOIN #index_details id2 - ON id1.database_id = id2.database_id + SELECT + 1/0 + FROM #index_details AS id1 + JOIN #index_details AS id2 + ON id1.database_id = id2.database_id AND id1.object_id = id2.object_id AND id1.column_name = id2.column_name AND id1.key_ordinal <> id2.key_ordinal AND id1.key_ordinal > 1 /* After the first column */ AND id2.key_ordinal > 1 /* After the first column */ WHERE id1.database_id = ia1.database_id - AND id1.object_id = ia1.object_id - AND id1.index_name = ia1.index_name - AND id2.index_name = ia2.index_name - ); + AND id1.object_id = ia1.object_id + AND id1.index_name = ia1.index_name + AND id2.index_name = ia2.index_name + ) + OPTION(RECOMPILE); IF @debug = 1 @@ -1768,61 +1812,60 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. table_name = '#index_analysis after update', ia.* FROM #index_analysis AS ia; - END; - IF @debug = 1 - BEGIN RAISERROR('Generating results', 0, 0) WITH NOWAIT; END; -/* Insert summary statistics first */ -/* Add a separator row for the header */ -INSERT INTO #index_cleanup_results -( - result_type, - sort_order, - script_type, - additional_info -) -SELECT - 'HEADER', - 0, - 'SEPARATOR', - N'==================== INDEX CLEANUP SUMMARY ===================='; - -/* Add summary information */ -INSERT INTO #index_cleanup_results -( - result_type, - sort_order, - script_type, - additional_info -) -SELECT - 'SUMMARY', - 1, - 'Index Cleanup Summary', - N'Server uptime: ' + - CONVERT(nvarchar(10), @uptime_days) + - N' days' + - CASE - WHEN @uptime_warning = 1 - THEN N' (WARNING: Low uptime detected! Index usage data may be incomplete.)' - ELSE N'' - END + - N' | Tables analyzed: ' + - CONVERT(nvarchar(10), COUNT(DISTINCT CONCAT(ia.database_id, N'.', ia.schema_id, N'.', ia.object_id))) + - N' | Total indexes: ' + - CONVERT(nvarchar(10), COUNT(*)) + - N' | Indexes to disable: ' + - CONVERT(nvarchar(10), SUM(CASE WHEN ia.action = 'DISABLE' THEN 1 ELSE 0 END)) + - N' | Indexes to merge: ' + - CONVERT(nvarchar(10), SUM(CASE WHEN ia.action IN ('MERGE INCLUDES', 'MAKE UNIQUE') THEN 1 ELSE 0 END)) + - N' | Avg indexes per table: ' + - CONVERT(nvarchar(10), CONVERT(decimal(10,2), COUNT(*) * 1.0 / - NULLIF(COUNT(DISTINCT CONCAT(ia.database_id, N'.', ia.schema_id, N'.', ia.object_id)), 0))) -FROM #index_analysis AS ia; + /* Insert summary statistics first */ + /* Add a separator row for the header */ + INSERT INTO + #index_cleanup_results + ( + result_type, + sort_order, + script_type, + additional_info + ) + SELECT + 'HEADER', + 0, + N'=====', + N'==================== INDEX CLEANUP SUMMARY ===================='; + + /* Add summary information */ + INSERT INTO + #index_cleanup_results + ( + result_type, + sort_order, + script_type, + additional_info + ) + SELECT + 'SUMMARY', + 1, + 'Index Cleanup Summary', + N'Server uptime: ' + + CONVERT(nvarchar(10), @uptime_days) + + N' days' + + CASE + WHEN @uptime_warning = 1 + THEN N' (WARNING: Low uptime detected! Index usage data may be incomplete.)' + ELSE N'' + END + + N' | Tables analyzed: ' + + CONVERT(nvarchar(10), COUNT(DISTINCT CONCAT(ia.database_id, N'.', ia.schema_id, N'.', ia.object_id))) + + N' | Total indexes: ' + + CONVERT(nvarchar(10), COUNT(*)) + + N' | Indexes to disable: ' + + CONVERT(nvarchar(10), SUM(CASE WHEN ia.action = 'DISABLE' THEN 1 ELSE 0 END)) + + N' | Indexes to merge: ' + + CONVERT(nvarchar(10), SUM(CASE WHEN ia.action IN ('MERGE INCLUDES', 'MAKE UNIQUE') THEN 1 ELSE 0 END)) + + N' | Avg indexes per table: ' + + CONVERT(nvarchar(10), CONVERT(decimal(10,2), COUNT(*) * 1.0 / + NULLIF(COUNT(DISTINCT CONCAT(ia.database_id, N'.', ia.schema_id, N'.', ia.object_id)), 0))) + FROM #index_analysis AS ia; /* Insert space savings estimates */ INSERT INTO #index_cleanup_results From 7eca5d3d49d201954b3877e58e6d35932e2a33f5 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 13 Mar 2025 00:12:32 -0400 Subject: [PATCH 048/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 2254 +++++++++-------- 1 file changed, 1236 insertions(+), 1018 deletions(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index c84d4721..a2fb6d0f 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -1832,6 +1832,70 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 0, N'=====', N'==================== INDEX CLEANUP SUMMARY ===================='; + + /* Add a separator for scripts section */ + INSERT INTO + #index_cleanup_results + ( + result_type, + sort_order, + script_type, + additional_info + ) + SELECT + 'HEADER', + 4, /* Just before merge scripts at sort_order 5 */ + script_type = N'======', + additional_info = N'==================== INDEX SCRIPTS ===================='; + + /* Add a separator for report section at the end */ + INSERT INTO + #index_cleanup_results + ( + result_type, + sort_order, + script_type, + additional_info + ) + SELECT + result_type = 'HEADER', + sort_order = 99, + script_type = N'======', + additional_info = N'==================== END OF REPORT ===================='; + + /* Add a separator for indexes needing review section */ + INSERT INTO + #index_cleanup_results + WITH + (TABLOCK) + ( + result_type, + sort_order, + script_type, + additional_info + ) + SELECT + result_type = 'HEADER', + sort_order = 92, + script_type = N'======', + additional_info = N'==================== INDEXES NEEDING REVIEW ===================='; + + /* Add a separator for kept indexes section */ + INSERT INTO + #index_cleanup_results + WITH + (TABLOCK) + ( + result_type, + sort_order, + script_type, + additional_info + ) + SELECT + result_type = 'HEADER', + sort_order = 94, + script_type = N'======', + additional_info = N'==================== INDEXES KEPT ===================='; /* Add summary information */ INSERT INTO @@ -1843,1037 +1907,1191 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. additional_info ) SELECT - 'SUMMARY', - 1, - 'Index Cleanup Summary', - N'Server uptime: ' + - CONVERT(nvarchar(10), @uptime_days) + - N' days' + - CASE - WHEN @uptime_warning = 1 - THEN N' (WARNING: Low uptime detected! Index usage data may be incomplete.)' - ELSE N'' - END + - N' | Tables analyzed: ' + - CONVERT(nvarchar(10), COUNT(DISTINCT CONCAT(ia.database_id, N'.', ia.schema_id, N'.', ia.object_id))) + - N' | Total indexes: ' + - CONVERT(nvarchar(10), COUNT(*)) + - N' | Indexes to disable: ' + - CONVERT(nvarchar(10), SUM(CASE WHEN ia.action = 'DISABLE' THEN 1 ELSE 0 END)) + - N' | Indexes to merge: ' + - CONVERT(nvarchar(10), SUM(CASE WHEN ia.action IN ('MERGE INCLUDES', 'MAKE UNIQUE') THEN 1 ELSE 0 END)) + - N' | Avg indexes per table: ' + - CONVERT(nvarchar(10), CONVERT(decimal(10,2), COUNT(*) * 1.0 / - NULLIF(COUNT(DISTINCT CONCAT(ia.database_id, N'.', ia.schema_id, N'.', ia.object_id)), 0))) - FROM #index_analysis AS ia; - -/* Insert space savings estimates */ -INSERT INTO #index_cleanup_results -( - result_type, - sort_order, - script_type, - additional_info -) -SELECT - 'SUMMARY', - 2, - 'Estimated Space Savings', - N'Space saved from cleanup: ' + - CONVERT(nvarchar(20), CONVERT(decimal(10,4), SUM(CASE - WHEN ia.action IN ('DISABLE', 'MERGE INCLUDES', 'MAKE UNIQUE') - THEN ps.total_space_gb - ELSE 0 - END))) + - N' GB | Compression savings estimate: ' + - CONVERT(nvarchar(20), CONVERT(decimal(10,4), SUM(CASE - WHEN (ia.action IS NULL OR ia.action = 'KEEP') AND ce.can_compress = 1 - THEN ps.total_space_gb * 0.20 /* Conservative estimate - 20% compression ratio */ - ELSE 0 - END))) + - N' - ' + - CONVERT(nvarchar(20), CONVERT(decimal(10,4), SUM(CASE - WHEN (ia.action IS NULL OR ia.action = 'KEEP') AND ce.can_compress = 1 - THEN ps.total_space_gb * 0.60 /* Optimistic estimate - 60% compression ratio */ - ELSE 0 - END))) + - N' GB | Total estimated savings: ' + - CONVERT(nvarchar(20), CONVERT(decimal(10,4), SUM(CASE - WHEN ia.action IN ('DISABLE', 'MERGE INCLUDES', 'MAKE UNIQUE') - THEN ps.total_space_gb - WHEN (ia.action IS NULL OR ia.action = 'KEEP') AND ce.can_compress = 1 - THEN ps.total_space_gb * 0.20 - ELSE 0 - END))) + - N' - ' + - CONVERT(nvarchar(20), CONVERT(decimal(10,4), SUM(CASE - WHEN ia.action IN ('DISABLE', 'MERGE INCLUDES', 'MAKE UNIQUE') - THEN ps.total_space_gb - WHEN (ia.action IS NULL OR ia.action = 'KEEP') AND ce.can_compress = 1 - THEN ps.total_space_gb * 0.60 - ELSE 0 - END))) + - N' GB' -FROM #index_analysis AS ia -LEFT JOIN #partition_stats AS ps ON - ia.database_id = ps.database_id + result_type = 'SUMMARY', + sort_order = 1, + script_type = 'Index Cleanup Summary', + additional_info = + N'Server uptime: ' + + CONVERT(nvarchar(10), @uptime_days) + + N' days' + + CASE + WHEN @uptime_warning = 1 + THEN N' (WARNING: Low uptime detected! Index usage data may be incomplete.)' + ELSE N'' + END + + N' | Tables analyzed: ' + + CONVERT + ( + nvarchar(10), + COUNT_BIG(DISTINCT CONCAT(ia.database_id, N'.', ia.schema_id, N'.', ia.object_id)) + ) + + N' | Total indexes: ' + + CONVERT + ( + nvarchar(10), + COUNT_BIG(*) + ) + + N' | Indexes to disable: ' + + CONVERT + ( + nvarchar(10), + SUM + ( + CASE + WHEN ia.action = 'DISABLE' + THEN 1 + ELSE 0 + END + ) + ) + + N' | Indexes to merge: ' + + CONVERT(nvarchar(10), SUM(CASE WHEN ia.action IN ('MERGE INCLUDES', 'MAKE UNIQUE') THEN 1 ELSE 0 END)) + + N' | Avg indexes per table: ' + + CONVERT + ( + nvarchar(10), + CONVERT + ( + decimal(10,2), + COUNT_BIG(*) * 1.0 / + NULLIF + ( + COUNT_BIG + ( + DISTINCT CONCAT(ia.database_id, N'.', ia.schema_id, N'.', ia.object_id) + ), + 0 + ) + ) + ) + FROM #index_analysis AS ia + OPTION(RECOMPILE); + + /* Insert space savings estimates */ + INSERT INTO + #index_cleanup_results + ( + result_type, + sort_order, + script_type, + additional_info + ) + SELECT + result_type = 'SUMMARY', + sort_order = 2, + script_type = 'Estimated Space Savings', + additional_info = + N'Space saved from cleanup: ' + + CONVERT + ( + nvarchar(20), + CONVERT + ( + decimal(10,4), + SUM + ( + CASE + WHEN ia.action IN ('DISABLE', 'MERGE INCLUDES', 'MAKE UNIQUE') + THEN ps.total_space_gb + ELSE 0 + END + ) + ) + ) + + N' GB | Compression savings estimate: ' + + CONVERT + ( + nvarchar(20), + CONVERT + ( + decimal(10,4), + SUM + ( + CASE + WHEN (ia.action IS NULL OR ia.action = 'KEEP') + AND ce.can_compress = 1 + THEN ps.total_space_gb * 0.20 /* Conservative estimate - 20% compression ratio */ + ELSE 0 + END + ) + ) + ) + + N' - ' + + CONVERT + ( + nvarchar(20), + CONVERT + ( + decimal(10,4), + SUM + ( + CASE + WHEN (ia.action IS NULL OR ia.action = 'KEEP') + AND ce.can_compress = 1 + THEN ps.total_space_gb * 0.60 /* Optimistic estimate - 60% compression ratio */ + ELSE 0 + END + ) + ) + ) + + N' GB | Total estimated savings: ' + + CONVERT + ( + nvarchar(20), + CONVERT + ( + decimal(10,4), + SUM + ( + CASE + WHEN ia.action IN ('DISABLE', 'MERGE INCLUDES', 'MAKE UNIQUE') + THEN ps.total_space_gb + WHEN (ia.action IS NULL OR ia.action = 'KEEP') + AND ce.can_compress = 1 + THEN ps.total_space_gb * 0.20 + ELSE 0 + END + ) + ) + ) + + N' - ' + + CONVERT + ( + nvarchar(20), + CONVERT + ( + decimal(10,4), + SUM + ( + CASE + WHEN ia.action IN ('DISABLE', 'MERGE INCLUDES', 'MAKE UNIQUE') + THEN ps.total_space_gb + WHEN (ia.action IS NULL OR ia.action = 'KEEP') + AND ce.can_compress = 1 + THEN ps.total_space_gb * 0.60 + ELSE 0 + END + ) + ) + ) + + N' GB' + FROM #index_analysis AS ia + LEFT JOIN #partition_stats AS ps + ON ia.database_id = ps.database_id AND ia.object_id = ps.object_id AND ia.index_id = ps.index_id -LEFT JOIN #compression_eligibility AS ce ON - ia.database_id = ce.database_id + LEFT JOIN #compression_eligibility AS ce + ON ia.database_id = ce.database_id AND ia.object_id = ce.object_id - AND ia.index_id = ce.index_id; + AND ia.index_id = ce.index_id + OPTION(RECOMPILE); -/* Add a separator for scripts section */ -INSERT INTO #index_cleanup_results -( - result_type, - sort_order, - script_type, - additional_info -) -SELECT - 'HEADER', - 4, /* Just before merge scripts at sort_order 5 */ - 'SEPARATOR', - N'==================== INDEX SCRIPTS ===================='; - -/* Add a separator for report section at the end */ -INSERT INTO #index_cleanup_results -( - result_type, - sort_order, - script_type, - additional_info -) -SELECT - 'HEADER', - 99, - 'SEPARATOR', - N'==================== END OF REPORT ===================='; - - -/* Identify key duplicates where both indexes have MERGE INCLUDES action */ -INSERT INTO - #key_duplicate_dedupe -WITH - (TABLOCK) -( - database_id, - object_id, - database_name, - schema_name, - table_name, - base_key_columns, - filter_definition, - winning_index_name, - index_list -) -SELECT - ia.database_id, - ia.object_id, - MAX(ia.database_name), - MAX(ia.schema_name), - MAX(ia.table_name), - ia.key_columns, - ISNULL(ia.filter_definition, ''), - /* Choose the index with most included columns as the winner (or first alphabetically if tied) */ + + /* Identify key duplicates where both indexes have MERGE INCLUDES action */ + INSERT INTO + #key_duplicate_dedupe + WITH + (TABLOCK) ( - SELECT TOP 1 candidate.index_name - FROM #index_analysis AS candidate - WHERE candidate.database_id = ia.database_id - AND candidate.object_id = ia.object_id - AND candidate.key_columns = ia.key_columns - AND ISNULL(candidate.filter_definition, '') = ISNULL(ia.filter_definition, '') - AND candidate.action = 'MERGE INCLUDES' - AND candidate.consolidation_rule = 'Key Duplicate' - ORDER BY - /* First prefer indexes with "_Extended" in the name */ - CASE WHEN candidate.index_name LIKE '%\_Extended%' ESCAPE '\' THEN 1 ELSE 0 END DESC, - /* Then prefer indexes with more included columns (by length as a proxy) */ - LEN(ISNULL(candidate.included_columns, '')) DESC, - /* Then alphabetically for stability */ - candidate.index_name - ), - /* Build a list of other indexes in this group */ - STUFF(( - SELECT ', ' + inner_ia.index_name - FROM #index_analysis AS inner_ia - WHERE inner_ia.database_id = ia.database_id - AND inner_ia.object_id = ia.object_id - AND inner_ia.key_columns = ia.key_columns - AND ISNULL(inner_ia.filter_definition, '') = ISNULL(ia.filter_definition, '') - AND inner_ia.action = 'MERGE INCLUDES' - AND inner_ia.consolidation_rule = 'Key Duplicate' - ORDER BY inner_ia.index_name - FOR XML PATH(''), TYPE - ).value('.', 'nvarchar(max)'), 1, 2, '') -FROM #index_analysis AS ia -WHERE ia.action = 'MERGE INCLUDES' - AND ia.consolidation_rule = 'Key Duplicate' -GROUP BY - ia.database_id, - ia.object_id, - ia.key_columns, - ISNULL(ia.filter_definition, '') -HAVING COUNT(*) > 1; /* Only groups with multiple MERGE INCLUDES */ - -/* Update the index_analysis table to make only one index the winner in each group */ -UPDATE ia -SET - ia.action = 'DISABLE', - ia.target_index_name = kdd.winning_index_name, - ia.superseded_by = NULL -FROM #index_analysis AS ia -JOIN #key_duplicate_dedupe AS kdd - ON ia.database_id = kdd.database_id - AND ia.object_id = kdd.object_id - AND ia.key_columns = kdd.base_key_columns - AND ISNULL(ia.filter_definition, '') = kdd.filter_definition -WHERE ia.index_name <> kdd.winning_index_name - AND ia.action = 'MERGE INCLUDES' - AND ia.consolidation_rule = 'Key Duplicate'; - -/* Update the winning index's superseded_by to list all other indexes */ -UPDATE ia -SET - ia.superseded_by = 'Supersedes ' + - REPLACE(kdd.index_list, ia.index_name + ', ', '') /* Remove self from list if present */ -FROM #index_analysis AS ia -JOIN #key_duplicate_dedupe AS kdd - ON ia.database_id = kdd.database_id - AND ia.object_id = kdd.object_id - AND ia.key_columns = kdd.base_key_columns - AND ISNULL(ia.filter_definition, '') = kdd.filter_definition -WHERE ia.index_name = kdd.winning_index_name; - -/* Find indexes with same key columns where one has includes that are a subset of another */ -INSERT INTO - #include_subset_dedupe -WITH - (TABLOCK) -( - database_id, - object_id, - subset_index_name, - superset_index_name, - subset_included_columns, - superset_included_columns -) -SELECT - ia1.database_id, - ia1.object_id, - ia1.index_name AS subset_index_name, - ia2.index_name AS superset_index_name, - ia1.included_columns AS subset_included_columns, - ia2.included_columns AS superset_included_columns -FROM #index_analysis AS ia1 -JOIN #index_analysis AS ia2 - ON ia1.database_id = ia2.database_id - AND ia1.object_id = ia2.object_id - AND ia1.key_columns = ia2.key_columns - AND ISNULL(ia1.filter_definition, '') = ISNULL(ia2.filter_definition, '') - AND ia1.index_name <> ia2.index_name - AND ia1.action = 'MERGE INCLUDES' - AND ia2.action = 'MERGE INCLUDES' - AND ia1.consolidation_rule = 'Key Duplicate' - AND ia2.consolidation_rule = 'Key Duplicate' - /* Find where subset's includes are contained within superset's includes */ - AND ( - ia1.included_columns IS NULL OR - CHARINDEX(ia1.included_columns, ia2.included_columns) > 0 - ) - /* Don't match if lengths are the same (would be exact duplicates) */ - AND ( - ia1.included_columns IS NULL OR ia2.included_columns IS NULL OR - LEN(ia1.included_columns) < LEN(ia2.included_columns) - ); - -/* Update the subset indexes to be disabled, since supersets already contain their columns */ -UPDATE ia -SET - ia.action = 'DISABLE', - ia.target_index_name = isd.superset_index_name, - ia.superseded_by = NULL -FROM #index_analysis AS ia -JOIN #include_subset_dedupe AS isd - ON ia.database_id = isd.database_id - AND ia.object_id = isd.object_id - AND ia.index_name = isd.subset_index_name; - -/* Update the superset indexes to indicate they supersede the subset indexes */ -UPDATE ia -SET - ia.superseded_by = CASE - WHEN ia.superseded_by IS NULL THEN 'Supersedes ' + isd.subset_index_name - ELSE ia.superseded_by + ', ' + isd.subset_index_name - END -FROM #index_analysis AS ia -JOIN #include_subset_dedupe AS isd - ON ia.database_id = isd.database_id - AND ia.object_id = isd.object_id - AND ia.index_name = isd.superset_index_name; - -/* Update winning indexes that don't actually need changes to have action = 'KEEP' */ -UPDATE ia -SET - /* Change action to 'KEEP' for indexes that don't need to be modified */ - ia.action = 'KEEP' -FROM #index_analysis AS ia -WHERE ia.action = 'MERGE INCLUDES' -AND ia.superseded_by IS NOT NULL -/* Check if the index name contains "Extended" and has more included columns */ -AND (ia.index_name LIKE '%\_Extended%' ESCAPE '\' OR ia.index_name LIKE '%\_Extended' OR ia.index_name LIKE '%_Extended%') -/* This should indicate it already has all the needed includes */ -AND NOT EXISTS ( - /* Find any indexes it supersedes that have includes not in this index */ - SELECT 1 - FROM #index_analysis AS ia_subset - WHERE ia_subset.database_id = ia.database_id - AND ia_subset.object_id = ia.object_id - AND ia_subset.key_columns = ia.key_columns - AND ia_subset.action = 'DISABLE' - AND ia_subset.target_index_name = ia.index_name - /* This complex check handles cases where the superset doesn't contain all subset columns */ - AND CHARINDEX(ISNULL(ia_subset.included_columns, ''), ISNULL(ia.included_columns, '')) = 0 - AND ISNULL(ia_subset.included_columns, '') <> '' -); - -/* Insert merge scripts for indexes */ -INSERT INTO - #index_cleanup_results -( - result_type, - sort_order, - database_name, - schema_name, - table_name, - index_name, - script_type, - consolidation_rule, - target_index_name, - script, - additional_info, - superseded_info, - index_size_gb, - index_rows, - index_reads, - index_writes -) -SELECT - 'MERGE', - /* Put merge target indexes higher in sort order (5) so they appear before - indexes that will be disabled (20) */ - 5, - ia.database_name, - ia.schema_name, - ia.table_name, - ia.index_name, - 'MERGE SCRIPT', - ia.consolidation_rule, - ia.target_index_name, - CASE - WHEN ia.action = 'MAKE UNIQUE' - THEN N'/* This index can replace a unique constraint */ -/* Creating unique index with same keys as constraint */ -CREATE UNIQUE ' - WHEN ia.action = 'MERGE INCLUDES' - THEN N'/* This index can be merged with another index */ -/* Creating index with combined includes from both */ -CREATE ' - ELSE N'CREATE ' - END + - N'INDEX ' + - QUOTENAME(ia.index_name) + - N' ON ' + - QUOTENAME(ia.database_name) + - N'.' + - QUOTENAME(ia.schema_name) + - N'.' + - QUOTENAME(ia.table_name) + - N' (' + - ia.key_columns + - N')' + - CASE - WHEN ia.included_columns IS NOT NULL AND LEN(ia.included_columns) > 0 AND ia.action = 'MERGE INCLUDES' - THEN N' INCLUDE (' + - N'/* Combined includes from merged indexes */ - ' + - ia.included_columns + - N')' - WHEN ia.included_columns IS NOT NULL AND LEN(ia.included_columns) > 0 - THEN N' INCLUDE (' + - ia.included_columns + - N')' - ELSE N'' - END + - CASE - WHEN ia.filter_definition IS NOT NULL - THEN N' WHERE ' + - ia.filter_definition - ELSE N'' - END + - CASE - WHEN ps.partition_function_name IS NOT NULL - THEN N' ON ' + - QUOTENAME(ps.partition_function_name) + - N'(' + - ISNULL(ps.partition_columns, N'') + - N')' - WHEN ps.built_on IS NOT NULL - THEN N' ON ' + - QUOTENAME(ps.built_on) - ELSE N'' - END + - N' WITH (DROP_EXISTING = ON, FILLFACTOR = 100, SORT_IN_TEMPDB = ON, ONLINE = ' + - CASE WHEN @online = 1 THEN N'ON' ELSE N'OFF' END + - N', DATA_COMPRESSION = PAGE);', - /* Additional info about what this script does */ - CASE - WHEN ia.action = 'MERGE INCLUDES' THEN N'This index will absorb includes from duplicate indexes' - WHEN ia.action = 'MAKE UNIQUE' THEN N'This index will replace a unique constraint' - ELSE NULL - END, - /* Add superseded_by information if available */ - ia.superseded_by -FROM #index_analysis AS ia -LEFT JOIN -( - /* Get the partition info for each index */ - SELECT - ps.database_id, - ps.object_id, - ps.index_id, - ps.built_on, - ps.partition_function_name, - ps.partition_columns - FROM #partition_stats ps - GROUP BY - ps.database_id, - ps.object_id, - ps.index_id, - ps.built_on, - ps.partition_function_name, - ps.partition_columns -) AS ps - ON ia.database_id = ps.database_id - AND ia.object_id = ps.object_id - AND ia.index_id = ps.index_id -JOIN #compression_eligibility AS ce - ON ia.database_id = ce.database_id - AND ia.object_id = ce.object_id - AND ia.index_id = ce.index_id -WHERE ia.action IN ('MERGE INCLUDES', 'MAKE UNIQUE') -AND ce.can_compress = 1 -/* Only create merge scripts for the indexes that should remain after merging */ -AND ia.target_index_name IS NULL; - -/* Insert disable scripts for unneeded indexes */ -INSERT INTO - #index_cleanup_results -( - result_type, - sort_order, - database_name, - schema_name, - table_name, - index_name, - script_type, - consolidation_rule, - script, - additional_info, - target_index_name, - superseded_info, - index_size_gb, - index_rows, - index_reads, - index_writes -) -SELECT - 'DISABLE', - /* Sort duplicate/subset indexes first (20), then unused indexes last (25) */ - CASE - WHEN ia.consolidation_rule LIKE 'Unused Index%' THEN 25 - ELSE 20 - END, - ia.database_name, - ia.schema_name, - ia.table_name, - ia.index_name, - 'DISABLE SCRIPT', - ia.consolidation_rule, - N'ALTER INDEX ' + - QUOTENAME(ia.index_name) + - N' ON ' + - QUOTENAME(ia.database_name) + - N'.' + - QUOTENAME(ia.schema_name) + - N'.' + - QUOTENAME(ia.table_name) + - N' DISABLE;', - CASE - WHEN ia.consolidation_rule = 'Key Subset' - THEN N'This index is superseded by a wider index: ' + ISNULL(ia.target_index_name, N'(unknown)') - WHEN ia.consolidation_rule = 'Exact Duplicate' - THEN N'This index is an exact duplicate of: ' + ISNULL(ia.target_index_name, N'(unknown)') - WHEN ia.consolidation_rule = 'Key Duplicate' - THEN N'This index has the same keys as: ' + ISNULL(ia.target_index_name, N'(unknown)') - WHEN ia.consolidation_rule LIKE 'Unused Index%' - THEN ia.consolidation_rule - WHEN ia.action = 'DISABLE' - THEN N'This index is redundant and will be disabled' - ELSE N'This index is redundant' - END, - ia.target_index_name, /* Include the target index name */ - NULL, /* Don't need superseded_by info for disabled indexes */ - ps.total_space_gb, - ps.row_count, - (id.user_seeks + id.user_scans + id.user_lookups), - id.user_updates -FROM #index_analysis AS ia -LEFT JOIN #partition_stats AS ps - ON ia.database_id = ps.database_id - AND ia.object_id = ps.object_id - AND ia.index_id = ps.index_id -LEFT JOIN #index_details AS id - ON id.database_id = ia.database_id - AND id.object_id = ia.object_id - AND id.index_name = ia.index_name - AND id.is_included_column = 0 /* Get only one row per index */ - AND id.key_ordinal > 0 -WHERE ia.action = 'DISABLE'; - -/* Insert compression scripts for remaining indexes */ -INSERT INTO - #index_cleanup_results -( - result_type, - sort_order, - database_name, - schema_name, - table_name, - index_name, - script_type, - script, - additional_info, - target_index_name, - superseded_info, - index_size_gb, - index_rows, - index_reads, - index_writes -) -SELECT - 'COMPRESS', - 40, - ia.database_name, - ia.schema_name, - ia.table_name, - ia.index_name, - 'COMPRESSION SCRIPT', - N'ALTER INDEX ' + - QUOTENAME(ia.index_name) + - N' ON ' + - QUOTENAME(ia.database_name) + - N'.' + - QUOTENAME(ia.schema_name) + - N'.' + - QUOTENAME(ia.table_name) + - CASE - WHEN ps.partition_function_name IS NOT NULL - THEN N' REBUILD PARTITION = ALL' - ELSE N' REBUILD' - END + - N' WITH (FILLFACTOR = 100, SORT_IN_TEMPDB = ON, ONLINE = ' + - CASE WHEN @online = 1 THEN N'ON' ELSE N'OFF' END + - N', DATA_COMPRESSION = PAGE);', - N'Compression type: All Partitions', - NULL, /* No target index for compression scripts */ - ia.superseded_by, /* Include superseded_by info for compression scripts */ - ps_full.total_space_gb, - ps_full.row_count, - (id.user_seeks + id.user_scans + id.user_lookups), - id.user_updates -FROM #index_analysis AS ia -LEFT JOIN -( - /* Get the partition info for each index */ - SELECT - ps.database_id, - ps.object_id, - ps.index_id, - ps.built_on, - ps.partition_function_name, - ps.partition_columns - FROM #partition_stats ps + database_id, + object_id, + database_name, + schema_name, + table_name, + base_key_columns, + filter_definition, + winning_index_name, + index_list + ) + SELECT + ia.database_id, + ia.object_id, + database_name = MAX(ia.database_name), + schema_name = MAX(ia.schema_name), + table_name = MAX(ia.table_name), + base_key_columns = ia.key_columns, + filter_definition = ISNULL(ia.filter_definition, N''), + /* Choose the index with most included columns as the winner (or first alphabetically if tied) */ + winning_index_name = + ( + SELECT TOP (1) + candidate.index_name + FROM #index_analysis AS candidate + WHERE candidate.database_id = ia.database_id + AND candidate.object_id = ia.object_id + AND candidate.key_columns = ia.key_columns + AND ISNULL(candidate.filter_definition, '') = ISNULL(ia.filter_definition, '') + AND candidate.action = 'MERGE INCLUDES' + AND candidate.consolidation_rule = 'Key Duplicate' + ORDER BY + /* First prefer indexes with "_Extended" in the name */ + CASE WHEN candidate.index_name LIKE '%\_Extended%' ESCAPE '\' THEN 1 ELSE 0 END DESC, + /* Then prefer indexes with more included columns (by length as a proxy) */ + LEN(ISNULL(candidate.included_columns, '')) DESC, + /* Then alphabetically for stability */ + candidate.index_name + ), + /* Build a list of other indexes in this group */ + index_list = + STUFF + ( + ( + SELECT + N', ' + + inner_ia.index_name + FROM #index_analysis AS inner_ia + WHERE inner_ia.database_id = ia.database_id + AND inner_ia.object_id = ia.object_id + AND inner_ia.key_columns = ia.key_columns + AND ISNULL(inner_ia.filter_definition, '') = ISNULL(ia.filter_definition, '') + AND inner_ia.action = 'MERGE INCLUDES' + AND inner_ia.consolidation_rule = 'Key Duplicate' + ORDER BY + inner_ia.index_name + FOR + XML + PATH(''), + TYPE + ).value('.', 'nvarchar(max)'), + 1, + 2, + '' + ) + FROM #index_analysis AS ia + WHERE ia.action = 'MERGE INCLUDES' + AND ia.consolidation_rule = 'Key Duplicate' GROUP BY - ps.database_id, - ps.object_id, - ps.index_id, - ps.built_on, - ps.partition_function_name, - ps.partition_columns -) ps - ON ia.database_id = ps.database_id - AND ia.object_id = ps.object_id - AND ia.index_id = ps.index_id -LEFT JOIN #partition_stats AS ps_full - ON ia.database_id = ps_full.database_id - AND ia.object_id = ps_full.object_id - AND ia.index_id = ps_full.index_id -LEFT JOIN #index_details AS id - ON id.database_id = ia.database_id - AND id.object_id = ia.object_id - AND id.index_name = ia.index_name - AND id.is_included_column = 0 /* Get only one row per index */ - AND id.key_ordinal > 0 -JOIN #compression_eligibility ce - ON ia.database_id = ce.database_id - AND ia.object_id = ce.object_id - AND ia.index_id = ce.index_id -WHERE - /* Indexes that are not being disabled or merged */ - (ia.action IS NULL OR ia.action = 'KEEP') - /* Only indexes eligible for compression */ - AND ce.can_compress = 1; - -/* Insert disable scripts for unique constraints */ -INSERT INTO - #index_cleanup_results -( - result_type, - sort_order, - database_name, - schema_name, - table_name, - index_name, - script_type, - additional_info, - script, - index_size_gb, - index_rows, - index_reads, - index_writes -) -SELECT - 'CONSTRAINT', - 30, - ia.database_name, - ia.schema_name, - ia.table_name, - ia.index_name, - 'DISABLE CONSTRAINT SCRIPT', - N'Constraint to disable: ' + id.index_name, - N'ALTER TABLE ' + - QUOTENAME(ia.database_name) + - N'.' + - QUOTENAME(ia.schema_name) + - N'.' + - QUOTENAME(ia.table_name) + - N' NOCHECK CONSTRAINT ' + - QUOTENAME(id.index_name) + - N';', - ps.total_space_gb, - ps.row_count, - (id2.user_seeks + id2.user_scans + id2.user_lookups), - id2.user_updates -FROM #index_analysis AS ia -JOIN #index_details AS id - ON id.database_id = ia.database_id - AND id.object_id = ia.object_id - AND id.is_unique_constraint = 1 -LEFT JOIN #index_details AS id2 - ON id2.database_id = ia.database_id - AND id2.object_id = ia.object_id - AND id2.index_name = ia.index_name - AND id2.is_included_column = 0 /* Get only one row per index */ - AND id2.key_ordinal > 0 -LEFT JOIN #partition_stats AS ps - ON ia.database_id = ps.database_id - AND ia.object_id = ps.object_id - AND ia.index_id = ps.index_id -WHERE - /* Only indexes that are being made unique */ - ia.action = 'MAKE UNIQUE' - /* Find the constraint that matches the index being made unique */ - AND EXISTS + ia.database_id, + ia.object_id, + ia.key_columns, + ia.filter_definition + HAVING + COUNT_BIG(*) > 1 + OPTION(RECOMPILE); /* Only groups with multiple MERGE INCLUDES */ + + /* Update the index_analysis table to make only one index the winner in each group */ + UPDATE + ia + SET + ia.action = N'DISABLE', + ia.target_index_name = kdd.winning_index_name, + ia.superseded_by = NULL + FROM #index_analysis AS ia + JOIN #key_duplicate_dedupe AS kdd + ON ia.database_id = kdd.database_id + AND ia.object_id = kdd.object_id + AND ia.key_columns = kdd.base_key_columns + AND ISNULL(ia.filter_definition, N'') = kdd.filter_definition + WHERE ia.index_name <> kdd.winning_index_name + AND ia.action = N'MERGE INCLUDES' + AND ia.consolidation_rule = 'Key Duplicate' + OPTION(RECOMPILE); + + /* Update the winning index's superseded_by to list all other indexes */ + UPDATE + ia + SET + ia.superseded_by = 'Supersedes ' + + REPLACE + ( + kdd.index_list, + ia.index_name + N', ', N'' + ) /* Remove self from list if present */ + FROM #index_analysis AS ia + JOIN #key_duplicate_dedupe AS kdd + ON ia.database_id = kdd.database_id + AND ia.object_id = kdd.object_id + AND ia.key_columns = kdd.base_key_columns + AND ISNULL(ia.filter_definition, '') = kdd.filter_definition + WHERE ia.index_name = kdd.winning_index_name + OPTION(RECOMPILE); + + /* Find indexes with same key columns where one has includes that are a subset of another */ + INSERT INTO + #include_subset_dedupe + WITH + (TABLOCK) ( + database_id, + object_id, + subset_index_name, + superset_index_name, + subset_included_columns, + superset_included_columns + ) + SELECT + ia1.database_id, + ia1.object_id, + ia1.index_name AS subset_index_name, + ia2.index_name AS superset_index_name, + ia1.included_columns AS subset_included_columns, + ia2.included_columns AS superset_included_columns + FROM #index_analysis AS ia1 + JOIN #index_analysis AS ia2 + ON ia1.database_id = ia2.database_id + AND ia1.object_id = ia2.object_id + AND ia1.key_columns = ia2.key_columns + AND ISNULL(ia1.filter_definition, N'') = ISNULL(ia2.filter_definition, N'') + AND ia1.index_name <> ia2.index_name + AND ia1.action = N'MERGE INCLUDES' + AND ia2.action = N'MERGE INCLUDES' + AND ia1.consolidation_rule = 'Key Duplicate' + AND ia2.consolidation_rule = 'Key Duplicate' + /* Find where subset's includes are contained within superset's includes */ + AND + ( + ia1.included_columns IS NULL + OR CHARINDEX(ia1.included_columns, ia2.included_columns) > 0 + ) + /* Don't match if lengths are the same (would be exact duplicates) */ + AND + ( + ia1.included_columns IS NULL + OR ia2.included_columns IS NULL + OR LEN(ia1.included_columns) < LEN(ia2.included_columns) + ) + OPTION(RECOMPILE); + + /* Update the subset indexes to be disabled, since supersets already contain their columns */ + UPDATE + ia + SET + ia.action = N'DISABLE', + ia.target_index_name = isd.superset_index_name, + ia.superseded_by = NULL + FROM #index_analysis AS ia + JOIN #include_subset_dedupe AS isd + ON ia.database_id = isd.database_id + AND ia.object_id = isd.object_id + AND ia.index_name = isd.subset_index_name + OPTION(RECOMPILE); + + /* Update the superset indexes to indicate they supersede the subset indexes */ + UPDATE + ia + SET + ia.superseded_by = + CASE + WHEN ia.superseded_by IS NULL + THEN N'Supersedes ' + isd.subset_index_name + ELSE ia.superseded_by + N', ' + isd.subset_index_name + END + FROM #index_analysis AS ia + JOIN #include_subset_dedupe AS isd + ON ia.database_id = isd.database_id + AND ia.object_id = isd.object_id + AND ia.index_name = isd.superset_index_name + OPTION(RECOMPILE); + + /* Update winning indexes that don't actually need changes to have action = 'KEEP' */ + UPDATE + ia + SET + /* Change action to 'KEEP' for indexes that don't need to be modified */ + ia.action = N'KEEP' + FROM #index_analysis AS ia + WHERE ia.action = 'MERGE INCLUDES' + AND ia.superseded_by IS NOT NULL + /* Check if the index name contains "Extended" and has more included columns */ + AND (ia.index_name LIKE '%\_Extended%' ESCAPE '\' OR ia.index_name LIKE '%\_Extended' OR ia.index_name LIKE '%_Extended%') + /* This should indicate it already has all the needed includes */ + AND NOT EXISTS + ( + /* Find any indexes it supersedes that have includes not in this index */ SELECT 1/0 - FROM #index_details id_nc - WHERE id_nc.database_id = ia.database_id - AND id_nc.object_id = ia.object_id - AND id_nc.index_name = ia.index_name - /* Matching key columns */ - AND NOT EXISTS + FROM #index_analysis AS ia_subset + WHERE ia_subset.database_id = ia.database_id + AND ia_subset.object_id = ia.object_id + AND ia_subset.key_columns = ia.key_columns + AND ia_subset.action = 'DISABLE' + AND ia_subset.target_index_name = ia.index_name + /* This complex check handles cases where the superset doesn't contain all subset columns */ + AND CHARINDEX(ISNULL(ia_subset.included_columns, N''), ISNULL(ia.included_columns, N'')) = 0 + AND ISNULL(ia_subset.included_columns, N'') <> N'' + ) + OPTION(RECOMPILE); + + /* Insert merge scripts for indexes */ + INSERT INTO + #index_cleanup_results + ( + result_type, + sort_order, + database_name, + schema_name, + table_name, + index_name, + script_type, + consolidation_rule, + target_index_name, + script, + additional_info, + superseded_info, + index_size_gb, + index_rows, + index_reads, + index_writes + ) + SELECT + result_type = 'MERGE', + /* Put merge target indexes higher in sort order (5) so they appear before + indexes that will be disabled (20) */ + sort_order = 5, + ia.database_name, + ia.schema_name, + ia.table_name, + ia.index_name, + script_type = N'MERGE SCRIPT', + ia.consolidation_rule, + ia.target_index_name, + script = + CASE + WHEN ia.action = 'MAKE UNIQUE' + THEN N'/* This index can replace a unique constraint */ + /* Creating unique index with same keys as constraint */ + CREATE UNIQUE ' + WHEN ia.action = 'MERGE INCLUDES' + THEN N'/* This index can be merged with another index */ + /* Creating index with combined includes from both */ + CREATE ' + ELSE N'CREATE ' + END + + N'INDEX ' + + QUOTENAME(ia.index_name) + + N' ON ' + + QUOTENAME(ia.database_name) + + N'.' + + QUOTENAME(ia.schema_name) + + N'.' + + QUOTENAME(ia.table_name) + + N' (' + + ia.key_columns + + N')' + + CASE + WHEN ia.included_columns IS NOT NULL + AND LEN(ia.included_columns) > 0 + AND ia.action = 'MERGE INCLUDES' + THEN N' INCLUDE (' + + ia.included_columns + + N')' + WHEN ia.included_columns IS NOT NULL + AND LEN(ia.included_columns) > 0 + THEN N' INCLUDE (' + + ia.included_columns + + N')' + ELSE N'' + END + + CASE + WHEN ia.filter_definition IS NOT NULL + THEN N' WHERE ' + + ia.filter_definition + ELSE N'' + END + + CASE + WHEN ps.partition_function_name IS NOT NULL + THEN N' ON ' + + QUOTENAME(ps.partition_function_name) + + N'(' + + ISNULL(ps.partition_columns, N'') + + N')' + WHEN ps.built_on IS NOT NULL + THEN N' ON ' + + QUOTENAME(ps.built_on) + ELSE N'' + END + + N' WITH (DROP_EXISTING = ON, FILLFACTOR = 100, SORT_IN_TEMPDB = ON, ONLINE = ' + + CASE + WHEN @online = 1 + THEN N'ON' + ELSE N'OFF' + END + + N', DATA_COMPRESSION = PAGE);', + /* Additional info about what this script does */ + additional_info = + CASE + WHEN ia.action = 'MERGE INCLUDES' + THEN N'This index will absorb includes from duplicate indexes' + WHEN ia.action = 'MAKE UNIQUE' + THEN N'This index will replace a unique constraint' + ELSE NULL + END, + /* Add superseded_by information if available */ + ia.superseded_by, + NULL, + NULL, + NULL, + NULL + FROM #index_analysis AS ia + LEFT JOIN + ( + /* Get the partition info for each index */ + SELECT + ps.database_id, + ps.object_id, + ps.index_id, + ps.built_on, + ps.partition_function_name, + ps.partition_columns + FROM #partition_stats ps + GROUP BY + ps.database_id, + ps.object_id, + ps.index_id, + ps.built_on, + ps.partition_function_name, + ps.partition_columns + ) AS ps + ON ia.database_id = ps.database_id + AND ia.object_id = ps.object_id + AND ia.index_id = ps.index_id + JOIN #compression_eligibility AS ce + ON ia.database_id = ce.database_id + AND ia.object_id = ce.object_id + AND ia.index_id = ce.index_id + WHERE ia.action IN (N'MERGE INCLUDES', N'MAKE UNIQUE') + AND ce.can_compress = 1 + /* Only create merge scripts for the indexes that should remain after merging */ + AND ia.target_index_name IS NULL + OPTION(RECOMPILE); + + /* Insert disable scripts for unneeded indexes */ + INSERT INTO + #index_cleanup_results + ( + result_type, + sort_order, + database_name, + schema_name, + table_name, + index_name, + script_type, + consolidation_rule, + script, + additional_info, + target_index_name, + superseded_info, + index_size_gb, + index_rows, + index_reads, + index_writes + ) + SELECT + result_type = 'DISABLE', + /* Sort duplicate/subset indexes first (20), then unused indexes last (25) */ + sort_order = + CASE + WHEN ia.consolidation_rule LIKE 'Unused Index%' THEN 25 + ELSE 20 + END, + ia.database_name, + ia.schema_name, + ia.table_name, + ia.index_name, + script_type = 'DISABLE SCRIPT', + ia.consolidation_rule, + script = + N'ALTER INDEX ' + + QUOTENAME(ia.index_name) + + N' ON ' + + QUOTENAME(ia.database_name) + + N'.' + + QUOTENAME(ia.schema_name) + + N'.' + + QUOTENAME(ia.table_name) + + N' DISABLE;', + CASE + WHEN ia.consolidation_rule = 'Key Subset' + THEN N'This index is superseded by a wider index: ' + ISNULL(ia.target_index_name, N'(unknown)') + WHEN ia.consolidation_rule = 'Exact Duplicate' + THEN N'This index is an exact duplicate of: ' + ISNULL(ia.target_index_name, N'(unknown)') + WHEN ia.consolidation_rule = 'Key Duplicate' + THEN N'This index has the same keys as: ' + ISNULL(ia.target_index_name, N'(unknown)') + WHEN ia.consolidation_rule LIKE 'Unused Index%' + THEN ia.consolidation_rule + WHEN ia.action = N'DISABLE' + THEN N'This index is redundant and will be disabled' + ELSE N'This index is redundant' + END, + ia.target_index_name, /* Include the target index name */ + superseded_info = NULL, /* Don't need superseded_by info for disabled indexes */ + ps.total_space_gb, + ps.total_rows, + index_reads = + (id.user_seeks + id.user_scans + id.user_lookups), + id.user_updates + FROM #index_analysis AS ia + LEFT JOIN #partition_stats AS ps + ON ia.database_id = ps.database_id + AND ia.object_id = ps.object_id + AND ia.index_id = ps.index_id + LEFT JOIN #index_details AS id + ON id.database_id = ia.database_id + AND id.object_id = ia.object_id + AND id.index_name = ia.index_name + AND id.is_included_column = 0 /* Get only one row per index */ + AND id.key_ordinal > 0 + WHERE ia.action = 'DISABLE' + OPTION(RECOMPILE); + + /* Insert compression scripts for remaining indexes */ + INSERT INTO + #index_cleanup_results + ( + result_type, + sort_order, + database_name, + schema_name, + table_name, + index_name, + script_type, + script, + additional_info, + target_index_name, + superseded_info, + index_size_gb, + index_rows, + index_reads, + index_writes + ) + SELECT + result_type = 'COMPRESS', + sort_order = 40, + ia.database_name, + ia.schema_name, + ia.table_name, + ia.index_name, + script_type = 'COMPRESSION SCRIPT', + script = + N'ALTER INDEX ' + + QUOTENAME(ia.index_name) + + N' ON ' + + QUOTENAME(ia.database_name) + + N'.' + + QUOTENAME(ia.schema_name) + + N'.' + + QUOTENAME(ia.table_name) + + CASE + WHEN ps.partition_function_name IS NOT NULL + THEN N' REBUILD PARTITION = ALL' + ELSE N' REBUILD' + END + + N' WITH (FILLFACTOR = 100, SORT_IN_TEMPDB = ON, ONLINE = ' + + CASE WHEN @online = 1 THEN N'ON' ELSE N'OFF' END + + N', DATA_COMPRESSION = PAGE);', + N'Compression type: All Partitions', + superseded_info = NULL, /* No target index for compression scripts */ + ia.superseded_by, /* Include superseded_by info for compression scripts */ + ps_full.total_space_gb, + ps_full.total_rows, + index_reads = + (id.user_seeks + id.user_scans + id.user_lookups), + id.user_updates + FROM #index_analysis AS ia + LEFT JOIN + ( + /* Get the partition info for each index */ + SELECT + ps.database_id, + ps.object_id, + ps.index_id, + ps.built_on, + ps.partition_function_name, + ps.partition_columns + FROM #partition_stats ps + GROUP BY + ps.database_id, + ps.object_id, + ps.index_id, + ps.built_on, + ps.partition_function_name, + ps.partition_columns + ) ps + ON ia.database_id = ps.database_id + AND ia.object_id = ps.object_id + AND ia.index_id = ps.index_id + LEFT JOIN #partition_stats AS ps_full + ON ia.database_id = ps_full.database_id + AND ia.object_id = ps_full.object_id + AND ia.index_id = ps_full.index_id + LEFT JOIN #index_details AS id + ON id.database_id = ia.database_id + AND id.object_id = ia.object_id + AND id.index_name = ia.index_name + AND id.is_included_column = 0 /* Get only one row per index */ + AND id.key_ordinal > 0 + JOIN #compression_eligibility ce + ON ia.database_id = ce.database_id + AND ia.object_id = ce.object_id + AND ia.index_id = ce.index_id + WHERE + /* Indexes that are not being disabled or merged */ + (ia.action IS NULL OR ia.action = 'KEEP') + /* Only indexes eligible for compression */ + AND ce.can_compress = 1 + OPTION(RECOMPILE); + + /* Insert disable scripts for unique constraints */ + INSERT INTO + #index_cleanup_results + ( + result_type, + sort_order, + database_name, + schema_name, + table_name, + index_name, + script_type, + additional_info, + script, + index_size_gb, + index_rows, + index_reads, + index_writes + ) + SELECT + result_type = 'CONSTRAINT', + sort_order = 30, + ia.database_name, + ia.schema_name, + ia.table_name, + ia.index_name, + script_type = 'DISABLE CONSTRAINT SCRIPT', + additional_info = + N'Constraint to disable: ' + + id.index_name, + script = + N'ALTER TABLE ' + + QUOTENAME(ia.database_name) + + N'.' + + QUOTENAME(ia.schema_name) + + N'.' + + QUOTENAME(ia.table_name) + + N' NOCHECK CONSTRAINT ' + + QUOTENAME(id.index_name) + + N';', + ps.total_space_gb, + ps.total_rows, + index_reads = + (id2.user_seeks + id2.user_scans + id2.user_lookups), + id2.user_updates + FROM #index_analysis AS ia + JOIN #index_details AS id + ON id.database_id = ia.database_id + AND id.object_id = ia.object_id + AND id.is_unique_constraint = 1 + LEFT JOIN #index_details AS id2 + ON id2.database_id = ia.database_id + AND id2.object_id = ia.object_id + AND id2.index_name = ia.index_name + AND id2.is_included_column = 0 /* Get only one row per index */ + AND id2.key_ordinal > 0 + LEFT JOIN #partition_stats AS ps + ON ia.database_id = ps.database_id + AND ia.object_id = ps.object_id + AND ia.index_id = ps.index_id + WHERE + /* Only indexes that are being made unique */ + ia.action = 'MAKE UNIQUE' + /* Find the constraint that matches the index being made unique */ + AND EXISTS ( SELECT - id.column_name - FROM #index_details id_inner - WHERE id_inner.database_id = id.database_id - AND id_inner.object_id = id.object_id - AND id_inner.index_id = id.index_id - AND id_inner.is_included_column = 0 - - EXCEPT - - SELECT - id_nc_inner.column_name - FROM #index_details id_nc_inner - WHERE id_nc_inner.database_id = id_nc.database_id - AND id_nc_inner.object_id = id_nc.object_id - AND id_nc_inner.index_name = id_nc.index_name - AND id_nc_inner.is_included_column = 0 + 1/0 + FROM #index_details id_nc + WHERE id_nc.database_id = ia.database_id + AND id_nc.object_id = ia.object_id + AND id_nc.index_name = ia.index_name + /* Matching key columns */ + AND NOT EXISTS + ( + SELECT + id.column_name + FROM #index_details id_inner + WHERE id_inner.database_id = id.database_id + AND id_inner.object_id = id.object_id + AND id_inner.index_id = id.index_id + AND id_inner.is_included_column = 0 + + EXCEPT + + SELECT + id_nc_inner.column_name + FROM #index_details id_nc_inner + WHERE id_nc_inner.database_id = id_nc.database_id + AND id_nc_inner.object_id = id_nc.object_id + AND id_nc_inner.index_name = id_nc.index_name + AND id_nc_inner.is_included_column = 0 + ) ) - ); + OPTION(RECOMPILE); -/* Insert per-partition compression scripts */ -INSERT INTO - #index_cleanup_results -( - result_type, - sort_order, - database_name, - schema_name, - table_name, - index_name, - script_type, - script, - additional_info, - target_index_name, - superseded_info, - index_size_gb, - index_rows, - index_reads, - index_writes -) -SELECT - 'COMPRESS_PARTITION', - 50, - ia.database_name, - ia.schema_name, - ia.table_name, - ia.index_name, - 'PARTITION COMPRESSION SCRIPT', - N'ALTER INDEX ' + - QUOTENAME(ia.index_name) + - N' ON ' + - QUOTENAME(ia.database_name) + - N'.' + - QUOTENAME(ia.schema_name) + - N'.' + - QUOTENAME(ia.table_name) + - N' REBUILD PARTITION = ' + - CONVERT(nvarchar(20), ps.partition_number) + - N' WITH (FILLFACTOR = 100, SORT_IN_TEMPDB = ON, ONLINE = ' + - CASE WHEN @online = 1 THEN N'ON' ELSE N'OFF' END + - N', DATA_COMPRESSION = PAGE);', - N'Compression type: Per Partition | Partition: ' + - CONVERT(nvarchar(20), ps.partition_number) + - N' | Rows: ' + - CONVERT(nvarchar(20), ps.total_rows) + - N' | Size: ' + - CONVERT(nvarchar(20), CONVERT(decimal(10,4), ps.total_space_gb)) + - N' GB', - NULL, - NULL, - ps.total_space_gb, - ps.row_count, - (id.user_seeks + id.user_scans + id.user_lookups), - id.user_updates -FROM #index_analysis AS ia -JOIN #partition_stats AS ps - ON ia.database_id = ps.database_id - AND ia.object_id = ps.object_id - AND ia.index_id = ps.index_id -LEFT JOIN #index_details AS id - ON id.database_id = ia.database_id - AND id.object_id = ia.object_id - AND id.index_name = ia.index_name - AND id.is_included_column = 0 /* Get only one row per index */ - AND id.key_ordinal > 0 -JOIN #compression_eligibility AS ce - ON ia.database_id = ce.database_id - AND ia.object_id = ce.object_id - AND ia.index_id = ce.index_id -WHERE - /* Only partitioned indexes */ - ps.partition_function_name IS NOT NULL - /* Indexes that are not being disabled or merged */ - AND (ia.action IS NULL OR ia.action = 'KEEP') - /* Only indexes eligible for compression */ - AND ce.can_compress = 1; - -/* Stats reporting has been moved to #index_cleanup_results table */ - -/* Space savings reporting has been moved to #index_cleanup_results table */ - -/* Insert compression ineligible info */ -INSERT INTO - #index_cleanup_results -WITH - (TABLOCK) -( - result_type, - sort_order, - database_name, - schema_name, - table_name, - index_name, - script_type, - additional_info, - index_size_gb, - index_rows, - index_reads, - index_writes -) -SELECT - 'INELIGIBLE', - 90, - ce.database_name, - ce.schema_name, - ce.table_name, - ce.index_name, - 'INELIGIBLE FOR COMPRESSION', - ce.reason, - ps.total_space_gb, - ps.row_count, - (id.user_seeks + id.user_scans + id.user_lookups), - id.user_updates -FROM #compression_eligibility AS ce -LEFT JOIN #partition_stats AS ps - ON ce.database_id = ps.database_id - AND ce.object_id = ps.object_id - AND ce.index_id = ps.index_id -LEFT JOIN #index_details AS id - ON id.database_id = ce.database_id - AND id.object_id = ce.object_id - AND id.index_name = ce.index_name - AND id.is_included_column = 0 /* Get only one row per index */ - AND id.key_ordinal > 0 -WHERE ce.can_compress = 0; - -/* Add a separator for indexes needing review section */ -INSERT INTO - #index_cleanup_results -WITH - (TABLOCK) -( - result_type, - sort_order, - script_type, - additional_info -) -SELECT - 'HEADER', - 92, - 'SEPARATOR', - N'==================== INDEXES NEEDING REVIEW ===================='; - -/* Insert indexes identified for manual review */ -INSERT INTO - #index_cleanup_results -WITH - (TABLOCK) -( - result_type, - sort_order, - database_name, - schema_name, - table_name, - index_name, - script_type, - consolidation_rule, - target_index_name, - additional_info, - index_size_gb, - index_rows, - index_reads, - index_writes -) -SELECT - 'REVIEW', - 93, /* Just before KEPT indexes */ - ia.database_name, - ia.schema_name, - ia.table_name, - ia.index_name, - 'NEEDS REVIEW', - ia.consolidation_rule, - ia.target_index_name, - CASE - WHEN ia.consolidation_rule = 'Same Keys Different Order' - THEN 'This index has the same key columns as ' + ISNULL(ia.target_index_name, N'(unknown)') + - ' but in a different order. May be redundant depending on query patterns.' - ELSE 'This index needs manual review' - END, - ps.total_space_gb, - ps.row_count, - (id.user_seeks + id.user_scans + id.user_lookups), - id.user_updates -FROM #index_analysis AS ia -LEFT JOIN #partition_stats AS ps - ON ia.database_id = ps.database_id - AND ia.object_id = ps.object_id - AND ia.index_id = ps.index_id -LEFT JOIN #index_details AS id - ON id.database_id = ia.database_id - AND id.object_id = ia.object_id - AND id.index_name = ia.index_name - AND id.is_included_column = 0 /* Get only one row per index */ - AND id.key_ordinal > 0 -WHERE ia.action = 'REVIEW'; - -/* Add a separator for kept indexes section */ -INSERT INTO - #index_cleanup_results -WITH - (TABLOCK) -( - result_type, - sort_order, - script_type, - additional_info -) -SELECT - 'HEADER', - 94, - 'SEPARATOR', - N'==================== INDEXES KEPT ===================='; - -/* Insert indexes that are being kept (superset indexes and others) */ -INSERT INTO - #index_cleanup_results -WITH - (TABLOCK) -( - result_type, - sort_order, - database_name, - schema_name, - table_name, - index_name, - script_type, - consolidation_rule, - superseded_info, - additional_info, - index_size_gb, - index_rows, - index_reads, - index_writes -) -SELECT - 'KEEP', - 95, /* Just before END OF REPORT at 99 */ - ia.database_name, - ia.schema_name, - ia.table_name, - ia.index_name, - 'KEPT', - ia.consolidation_rule, - ia.superseded_by, - CASE - WHEN ia.superseded_by IS NOT NULL THEN 'This index supersedes other indexes and already has all needed columns' - WHEN ia.action = 'KEEP' THEN 'This index is being kept' - ELSE NULL - END, - ps.total_space_gb, - ps.row_count, - (id.user_seeks + id.user_scans + id.user_lookups), - id.user_updates -FROM #index_analysis AS ia -LEFT JOIN #partition_stats AS ps - ON ia.database_id = ps.database_id - AND ia.object_id = ps.object_id - AND ia.index_id = ps.index_id -LEFT JOIN #index_details AS id - ON id.database_id = ia.database_id - AND id.object_id = ia.object_id - AND id.index_name = ia.index_name - AND id.is_included_column = 0 /* Get only one row per index */ - AND id.key_ordinal > 0 -WHERE ia.action = 'KEEP' OR (ia.action IS NULL AND ia.consolidation_rule IS NULL); - -/* -Return the consolidated results in a single result set -Results are ordered by: -1. Summary information (overall stats, savings estimates) -2. Merge scripts (includes merges and unique conversions) - sort_order 5 -3. Disable scripts (for redundant indexes) - sort_order 20 -4. Constraint scripts (for unique constraints to disable) -5. Compression scripts (for tables eligible for compression) -6. Partition-specific compression scripts -7. Ineligible objects (tables that can't be compressed) -8. Kept indexes - sort_order 95 - -Note: Merge target scripts are sorted higher in the results (sort_order 5) -so that new merged indexes are created before subset indexes are disabled. - -Within each category, indexes are sorted by size and impact for better prioritization. -*/ -SELECT - /* First, show the information needed to understand the script */ - ir.script_type, - ir.additional_info, - /* Then show identifying information for the index */ - ir.database_name, - ir.schema_name, - ir.table_name, - ir.index_name, - /* Then show relationship information */ - ir.consolidation_rule, - ir.target_index_name, - /* Include superseded_by info for winning indexes */ - CASE WHEN ia.superseded_by IS NOT NULL THEN ia.superseded_by ELSE ir.superseded_info END AS superseded_info, - /* Add size and usage metrics */ - index_size_gb = - CASE - WHEN ir.result_type IN ('HEADER', 'SUMMARY') THEN NULL - ELSE FORMAT(ir.index_size_gb, 'N4') - END, - index_rows = - CASE - WHEN ir.result_type IN ('HEADER', 'SUMMARY') THEN NULL - ELSE FORMAT(ir.index_rows, 'N0') - END, - index_reads = - CASE - WHEN ir.result_type IN ('HEADER', 'SUMMARY') THEN NULL - ELSE FORMAT(ir.index_reads, 'N0') - END, - index_writes = - CASE - WHEN ir.result_type IN ('HEADER', 'SUMMARY') THEN NULL - ELSE FORMAT(ir.index_writes, 'N0') - END, - /* Finally show the actual script */ - ir.script -FROM #index_cleanup_results AS ir -LEFT JOIN #index_analysis AS ia - ON ir.database_name = ia.database_name - AND ir.schema_name = ia.schema_name - AND ir.table_name = ia.table_name - AND ir.index_name = ia.index_name -ORDER BY - ir.sort_order, - /* Within each sort_order group, prioritize by size and usage */ - CASE - /* For HEADER and SUMMARY, keep the original order */ - WHEN ir.result_type IN ('HEADER', 'SUMMARY') THEN 0 - /* For script categories, order by size and impact */ - ELSE ISNULL(ir.index_size_gb, 0) - END DESC, - CASE - /* For HEADER and SUMMARY, keep the original order */ - WHEN ir.result_type IN ('HEADER', 'SUMMARY') THEN 0 - /* For script categories, consider rows as secondary sort */ - ELSE ISNULL(ir.index_rows, 0) - END DESC, - /* Then by database, schema, table, index name for consistent ordering */ - ir.database_name, - ir.schema_name, - ir.table_name, - ir.index_name; + /* Insert per-partition compression scripts */ + INSERT INTO + #index_cleanup_results + ( + result_type, + sort_order, + database_name, + schema_name, + table_name, + index_name, + script_type, + script, + additional_info, + target_index_name, + superseded_info, + index_size_gb, + index_rows, + index_reads, + index_writes + ) + SELECT + result_type = 'COMPRESS_PARTITION', + sort_order = 50, + ia.database_name, + ia.schema_name, + ia.table_name, + ia.index_name, + script_type = 'PARTITION COMPRESSION SCRIPT', + script = + N'ALTER INDEX ' + + QUOTENAME(ia.index_name) + + N' ON ' + + QUOTENAME(ia.database_name) + + N'.' + + QUOTENAME(ia.schema_name) + + N'.' + + QUOTENAME(ia.table_name) + + N' REBUILD PARTITION = ' + + CONVERT + ( + nvarchar(20), + ps.partition_number + ) + + N' WITH (FILLFACTOR = 100, SORT_IN_TEMPDB = ON, ONLINE = ' + + CASE + WHEN @online = 1 + THEN N'ON' + ELSE N'OFF' + END + + N', DATA_COMPRESSION = PAGE);', + N'Compression type: Per Partition | Partition: ' + + CONVERT + ( + nvarchar(20), + ps.partition_number + ) + + N' | Rows: ' + + CONVERT + ( + nvarchar(20), + ps.total_rows + ) + + N' | Size: ' + + CONVERT + ( + nvarchar(20), + CONVERT + ( + decimal(10,4), + ps.total_space_gb + ) + ) + + N' GB', + target_index_name = NULL, + superseded_info = NULL, + ps.total_space_gb, + ps.total_rows, + index_reads = + (id.user_seeks + id.user_scans + id.user_lookups), + id.user_updates + FROM #index_analysis AS ia + JOIN #partition_stats AS ps + ON ia.database_id = ps.database_id + AND ia.object_id = ps.object_id + AND ia.index_id = ps.index_id + LEFT JOIN #index_details AS id + ON id.database_id = ia.database_id + AND id.object_id = ia.object_id + AND id.index_name = ia.index_name + AND id.is_included_column = 0 /* Get only one row per index */ + AND id.key_ordinal > 0 + JOIN #compression_eligibility AS ce + ON ia.database_id = ce.database_id + AND ia.object_id = ce.object_id + AND ia.index_id = ce.index_id + WHERE + /* Only partitioned indexes */ + ps.partition_function_name IS NOT NULL + /* Indexes that are not being disabled or merged */ + AND (ia.action IS NULL OR ia.action = 'KEEP') + /* Only indexes eligible for compression */ + AND ce.can_compress = 1 + OPTION(RECOMPILE); + + /* Insert compression ineligible info */ + INSERT INTO + #index_cleanup_results + WITH + (TABLOCK) + ( + result_type, + sort_order, + database_name, + schema_name, + table_name, + index_name, + script_type, + additional_info, + index_size_gb, + index_rows, + index_reads, + index_writes + ) + SELECT + result_type = 'INELIGIBLE', + sort_order = 90, + ce.database_name, + ce.schema_name, + ce.table_name, + ce.index_name, + script_type = 'INELIGIBLE FOR COMPRESSION', + ce.reason, + ps.total_space_gb, + ps.total_rows, + index_reads = + (id.user_seeks + id.user_scans + id.user_lookups), + id.user_updates + FROM #compression_eligibility AS ce + LEFT JOIN #partition_stats AS ps + ON ce.database_id = ps.database_id + AND ce.object_id = ps.object_id + AND ce.index_id = ps.index_id + LEFT JOIN #index_details AS id + ON id.database_id = ce.database_id + AND id.object_id = ce.object_id + AND id.index_name = ce.index_name + AND id.is_included_column = 0 /* Get only one row per index */ + AND id.key_ordinal > 0 + WHERE ce.can_compress = 0 + OPTION(RECOMPILE); + + + /* Insert indexes identified for manual review */ + INSERT INTO + #index_cleanup_results + WITH + (TABLOCK) + ( + result_type, + sort_order, + database_name, + schema_name, + table_name, + index_name, + script_type, + consolidation_rule, + target_index_name, + additional_info, + index_size_gb, + index_rows, + index_reads, + index_writes + ) + SELECT + result_type = 'REVIEW', + sort_order = 93, /* Just before KEPT indexes */ + ia.database_name, + ia.schema_name, + ia.table_name, + ia.index_name, + script_type = 'NEEDS REVIEW', + ia.consolidation_rule, + ia.target_index_name, + additional_info = + CASE + WHEN ia.consolidation_rule = 'Same Keys Different Order' + THEN N'This index has the same key columns as ' + ISNULL(ia.target_index_name, N'(unknown)') + + N' but in a different order. May be redundant depending on query patterns.' + ELSE N'This index needs manual review' + END, + ps.total_space_gb, + ps.total_rows, + index_reads = + (id.user_seeks + id.user_scans + id.user_lookups), + id.user_updates + FROM #index_analysis AS ia + LEFT JOIN #partition_stats AS ps + ON ia.database_id = ps.database_id + AND ia.object_id = ps.object_id + AND ia.index_id = ps.index_id + LEFT JOIN #index_details AS id + ON id.database_id = ia.database_id + AND id.object_id = ia.object_id + AND id.index_name = ia.index_name + AND id.is_included_column = 0 /* Get only one row per index */ + AND id.key_ordinal > 0 + WHERE ia.action = 'REVIEW' + OPTION(RECOMPILE); + + + /* Insert indexes that are being kept (superset indexes and others) */ + INSERT INTO + #index_cleanup_results + WITH + (TABLOCK) + ( + result_type, + sort_order, + database_name, + schema_name, + table_name, + index_name, + script_type, + consolidation_rule, + superseded_info, + additional_info, + index_size_gb, + index_rows, + index_reads, + index_writes + ) + SELECT + result_type = 'KEEP', + sort_order = 95, /* Just before END OF REPORT at 99 */ + ia.database_name, + ia.schema_name, + ia.table_name, + ia.index_name, + script_type = 'KEPT', + ia.consolidation_rule, + ia.superseded_by, + additional_info = + CASE + WHEN ia.superseded_by IS NOT NULL + THEN 'This index supersedes other indexes and already has all needed columns' + WHEN ia.action = 'KEEP' + THEN 'This index is being kept' + ELSE NULL + END, + ps.total_space_gb, + ps.total_rows, + index_reads = + (id.user_seeks + id.user_scans + id.user_lookups), + id.user_updates + FROM #index_analysis AS ia + LEFT JOIN #partition_stats AS ps + ON ia.database_id = ps.database_id + AND ia.object_id = ps.object_id + AND ia.index_id = ps.index_id + LEFT JOIN #index_details AS id + ON id.database_id = ia.database_id + AND id.object_id = ia.object_id + AND id.index_name = ia.index_name + AND id.is_included_column = 0 /* Get only one row per index */ + AND id.key_ordinal > 0 + WHERE ia.action = 'KEEP' + OR + ( + ia.action IS NULL + AND ia.consolidation_rule IS NULL + ) + OPTION(RECOMPILE); + + /* + Return the consolidated results in a single result set + Results are ordered by: + 1. Summary information (overall stats, savings estimates) + 2. Merge scripts (includes merges and unique conversions) - sort_order 5 + 3. Disable scripts (for redundant indexes) - sort_order 20 + 4. Constraint scripts (for unique constraints to disable) + 5. Compression scripts (for tables eligible for compression) + 6. Partition-specific compression scripts + 7. Ineligible objects (tables that can't be compressed) + 8. Kept indexes - sort_order 95 + + Note: Merge target scripts are sorted higher in the results (sort_order 5) + so that new merged indexes are created before subset indexes are disabled. + + Within each category, indexes are sorted by size and impact for better prioritization. + */ + + SELECT + /* First, show the information needed to understand the script */ + ir.script_type, + ir.additional_info, + /* Then show identifying information for the index */ + ir.database_name, + ir.schema_name, + ir.table_name, + ir.index_name, + /* Then show relationship information */ + ir.consolidation_rule, + ir.target_index_name, + /* Include superseded_by info for winning indexes */ + superseded_info = + CASE + WHEN ia.superseded_by IS NOT NULL + THEN ia.superseded_by + ELSE ir.superseded_info + END, + /* Add size and usage metrics */ + index_size_gb = + CASE + WHEN ir.result_type IN ('HEADER', 'SUMMARY') + THEN NULL + ELSE FORMAT(ir.index_size_gb, 'N4') + END, + index_rows = + CASE + WHEN ir.result_type IN ('HEADER', 'SUMMARY') + THEN NULL + ELSE FORMAT(ir.index_rows, 'N0') + END, + index_reads = + CASE + WHEN ir.result_type IN ('HEADER', 'SUMMARY') + THEN NULL + ELSE FORMAT(ir.index_reads, 'N0') + END, + index_writes = + CASE + WHEN ir.result_type IN ('HEADER', 'SUMMARY') + THEN NULL + ELSE FORMAT(ir.index_writes, 'N0') + END, + /* Finally show the actual script */ + ir.script + FROM #index_cleanup_results AS ir + LEFT JOIN #index_analysis AS ia + ON ir.database_name = ia.database_name + AND ir.schema_name = ia.schema_name + AND ir.table_name = ia.table_name + AND ir.index_name = ia.index_name + ORDER BY + ir.sort_order, + /* Within each sort_order group, prioritize by size and usage */ + CASE + /* For HEADER and SUMMARY, keep the original order */ + WHEN ir.result_type IN ('HEADER', 'SUMMARY') + THEN 0 + /* For script categories, order by size and impact */ + ELSE ISNULL(ir.index_size_gb, 0) + END DESC, + CASE + /* For HEADER and SUMMARY, keep the original order */ + WHEN ir.result_type IN ('HEADER', 'SUMMARY') + THEN 0 + /* For script categories, consider rows as secondary sort */ + ELSE ISNULL(ir.index_rows, 0) + END DESC, + /* Then by database, schema, table, index name for consistent ordering */ + ir.database_name, + ir.schema_name, + ir.table_name, + ir.index_name + OPTION(RECOMPILE); END TRY BEGIN CATCH From 6312fe0936f5841f5fc4f5272181ff85fbda827e Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 13 Mar 2025 00:19:10 -0400 Subject: [PATCH 049/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 47 ++++++++++++------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index a2fb6d0f..fd4bd68b 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -273,7 +273,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SELECT @database_id = d.database_id FROM sys.databases AS d - WHERE d.name = @database_name; + WHERE d.name = @database_name + OPTION(RECOMPILE);; END; IF @schema_name IS NULL @@ -1315,7 +1316,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SELECT table_name = '#partition_stats', * - FROM #partition_stats AS ps; + FROM #partition_stats AS ps + OPTION(RECOMPILE);; RAISERROR('Performing #index_analysis insert', 0, 0) WITH NOWAIT; END; @@ -1423,7 +1425,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SELECT table_name = '#index_analysis', ia.* - FROM #index_analysis AS ia; + FROM #index_analysis AS ia + OPTION(RECOMPILE);; RAISERROR('Starting updates', 0, 0) WITH NOWAIT; END; @@ -1468,7 +1471,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND id.table_name = #index_analysis.table_name AND id.user_scans > 0 ) THEN 100 ELSE 0 - END; /* Indexes with scans get some priority */ + END + OPTION(RECOMPILE);; /* Indexes with scans get some priority */ /* Rule 1: Identify unused indexes */ @@ -1497,7 +1501,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND id.is_unique_constraint = 0 /* Don't disable unique constraints */ AND id.is_eligible_for_dedupe = 1 /* Only eligible indexes */ ) - AND #index_analysis.index_id <> 1; /* Don't disable clustered indexes */ + AND #index_analysis.index_id <> 1 + OPTION(RECOMPILE); /* Don't disable clustered indexes */ /* Rule 2: Exact duplicates - matching key columns and includes */ UPDATE @@ -1545,7 +1550,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND id2.object_id = ia2.object_id AND id2.index_name = ia2.index_name AND id2.is_eligible_for_dedupe = 1 - ); + ) + OPTION(RECOMPILE); /* Rule 3: Key duplicates - matching key columns, different includes */ UPDATE @@ -1612,7 +1618,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND id2.object_id = ia2.object_id AND id2.index_name = ia2.index_name AND id2.is_eligible_for_dedupe = 1 - ); + ) + OPTION(RECOMPILE); /* Rule 4: Superset/subset key columns */ UPDATE @@ -1651,7 +1658,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND id2.object_id = ia2.object_id AND id2.index_name = ia2.index_name AND id2.is_eligible_for_dedupe = 1 - ); + ) + OPTION(RECOMPILE); /* Update the superseded_by column for the wider index in a separate statement */ UPDATE @@ -1668,8 +1676,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Exception: If narrower index is unique and wider is not, they should not be merged */ AND NOT (ia1.is_unique = 1 AND ia2.is_unique = 0) WHERE ia1.consolidation_rule = 'Key Subset' /* Use records just processed in previous UPDATE */ - AND ia1.target_index_name = ia2.index_name; /* Make sure we're updating the right wider index */ - + AND ia1.target_index_name = ia2.index_name /* Make sure we're updating the right wider index */ + OPTION(RECOMPILE); + /* Rule 5: Unique constraint vs. nonclustered index handling */ UPDATE ia1 @@ -1724,7 +1733,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND id1_inner.index_name = ia1.index_name AND id1_inner.is_included_column = 0 ) - ); + ) + OPTION(RECOMPILE); /* Rule 7: Identify indexes with same keys but in different order after first column */ /* This rule flags indexes that have the same set of key columns but ordered differently */ @@ -1803,7 +1813,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND id1.index_name = ia1.index_name AND id2.index_name = ia2.index_name ) - OPTION(RECOMPILE); + OPTION(RECOMPILE); IF @debug = 1 @@ -1811,7 +1821,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SELECT table_name = '#index_analysis after update', ia.* - FROM #index_analysis AS ia; + FROM #index_analysis AS ia + OPTION(RECOMPILE); RAISERROR('Generating results', 0, 0) WITH NOWAIT; END; @@ -2250,7 +2261,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. OR ia2.included_columns IS NULL OR LEN(ia1.included_columns) < LEN(ia2.included_columns) ) - OPTION(RECOMPILE); + OPTION(RECOMPILE); /* Update the subset indexes to be disabled, since supersets already contain their columns */ UPDATE @@ -2307,8 +2318,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND ia_subset.action = 'DISABLE' AND ia_subset.target_index_name = ia.index_name /* This complex check handles cases where the superset doesn't contain all subset columns */ - AND CHARINDEX(ISNULL(ia_subset.included_columns, N''), ISNULL(ia.included_columns, N'')) = 0 - AND ISNULL(ia_subset.included_columns, N'') <> N'' + AND CHARINDEX(ISNULL(ia_subset.included_columns, N''), ISNULL(ia.included_columns, N'')) = 0 + AND ISNULL(ia_subset.included_columns, N'') <> N'' ) OPTION(RECOMPILE); @@ -2724,7 +2735,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND id_nc_inner.is_included_column = 0 ) ) - OPTION(RECOMPILE); + OPTION(RECOMPILE); /* Insert per-partition compression scripts */ INSERT INTO @@ -2957,7 +2968,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. index_reads, index_writes ) - SELECT + SELECT DISTINCT result_type = 'KEEP', sort_order = 95, /* Just before END OF REPORT at 99 */ ia.database_name, From 46db796d235e8efa33a75bf5ddc90128489a0104 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 13 Mar 2025 00:33:30 -0400 Subject: [PATCH 050/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index fd4bd68b..c82af00b 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -45,6 +45,26 @@ BEGIN SET NOCOUNT ON; BEGIN TRY +/* Check for SQL Server 2012 (11.0) or later for FORMAT and CONCAT functions*/ + IF CONVERT + ( + integer, + SUBSTRING + ( + CONVERT + ( + varchar(20), + SERVERPROPERTY('ProductVersion') + ), + 1, + 2 + ) + ) < 11 + BEGIN + RAISERROR('This procedure requires SQL Server 2012 (11.0) or later due to the use of FORMAT and CONCAT functions.', 11, 1); + RETURN; + END; + SELECT @version = '-2147483648', @version_date = '17530101'; From 2163eb6575f63dccfc4da32f25fa3e06f0d1d155 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 13 Mar 2025 00:51:20 -0400 Subject: [PATCH 051/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 60 +++++++++++++++---- 1 file changed, 48 insertions(+), 12 deletions(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index c82af00b..694059b0 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -46,7 +46,8 @@ SET NOCOUNT ON; BEGIN TRY /* Check for SQL Server 2012 (11.0) or later for FORMAT and CONCAT functions*/ - IF CONVERT + IF + CONVERT ( integer, SUBSTRING @@ -70,7 +71,7 @@ BEGIN TRY @version_date = '17530101'; SELECT - warning = N'Read the messages pane carefully!' + for_insurance_purposes = N'Read the messages pane carefully!' PRINT N' ------------------------------------------------------------------------------------------- @@ -83,15 +84,14 @@ It needs lots of love and testing in real environments with real indexes to fix * Deduping logic * Result correctness * Edge cases + * May not account for specific query patterns that benefit from seemingly redundant indexes + +ALWAYS TEST THESE RECOMMENDATIONS IN A NON-PRODUCTION ENVIRONMENT FIRST" - If you run this, only use the output to debug and validate result correctness. - - Do not run any of the output scripts, period. Doing so may be harmful. - ------------------------------------------------------------------------------------------- - ------------------------------------------------------------------------------------------- - ------------------------------------------------------------------------------------------- - - '; +------------------------------------------------------------------------------------------- +------------------------------------------------------------------------------------------- +------------------------------------------------------------------------------------------- +'; /* @@ -602,7 +602,43 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ON t.object_id = us.object_id AND us.database_id = @database_id WHERE t.is_ms_shipped = 0 - AND t.type <> N''TF''' + AND t.type <> N''TF'' + AND NOT EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@database_name) + N'.sys.views AS v + WHERE v.object_id = i.object_id + AND v.is_indexed_view = 1 + )'; + + IF + CONVERT + ( + integer, + SUBSTRING + ( + CONVERT + ( + varchar(20), + SERVERPROPERTY('ProductVersion') + ), + 1, + 2 + ) + ) >= 13 + BEGIN + SET @sql += N' + AND NOT EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@database_name) + N'.sys.tables AS t + WHERE t.object_id = i.object_id + AND t.temporal_type > 0 + )'; + END; + IF @object_id IS NOT NULL BEGIN @@ -3102,6 +3138,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND ir.index_name = ia.index_name ORDER BY ir.sort_order, + ir.database_name, /* Within each sort_order group, prioritize by size and usage */ CASE /* For HEADER and SUMMARY, keep the original order */ @@ -3118,7 +3155,6 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ELSE ISNULL(ir.index_rows, 0) END DESC, /* Then by database, schema, table, index name for consistent ordering */ - ir.database_name, ir.schema_name, ir.table_name, ir.index_name From 89620afa520d964cf0543c4476a2f458d2ae859b Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 13 Mar 2025 00:52:33 -0400 Subject: [PATCH 052/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index 694059b0..23fa400d 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -45,8 +45,10 @@ BEGIN SET NOCOUNT ON; BEGIN TRY -/* Check for SQL Server 2012 (11.0) or later for FORMAT and CONCAT functions*/ + /* Check for SQL Server 2012 (11.0) or later for FORMAT and CONCAT functions*/ + IF + /*fix version check*/ CONVERT ( integer, @@ -61,6 +63,7 @@ BEGIN TRY 2 ) ) < 11 + BEGIN RAISERROR('This procedure requires SQL Server 2012 (11.0) or later due to the use of FORMAT and CONCAT functions.', 11, 1); RETURN; @@ -613,6 +616,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. )'; IF + /*fix version check*/ CONVERT ( integer, From 7c2414f848ed2df4b08a90da4ca3b06b9a262484 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 13 Mar 2025 00:56:15 -0400 Subject: [PATCH 053/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 36 ++++--------------- 1 file changed, 6 insertions(+), 30 deletions(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index 23fa400d..413d6d70 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -48,21 +48,9 @@ BEGIN TRY /* Check for SQL Server 2012 (11.0) or later for FORMAT and CONCAT functions*/ IF - /*fix version check*/ - CONVERT - ( - integer, - SUBSTRING - ( - CONVERT - ( - varchar(20), - SERVERPROPERTY('ProductVersion') - ), - 1, - 2 - ) - ) < 11 + /* Check SQL Server 2012+ for FORMAT and CONCAT functions */ + (CONVERT(INT, SERVERPROPERTY('EngineEdition')) NOT IN (5, 8) /* Not Azure SQL DB or Managed Instance */ + AND CONVERT(INT, SUBSTRING(CONVERT(VARCHAR(20), SERVERPROPERTY('ProductVersion')), 1, 2)) < 11) /* Pre-2012 */ BEGIN RAISERROR('This procedure requires SQL Server 2012 (11.0) or later due to the use of FORMAT and CONCAT functions.', 11, 1); @@ -616,21 +604,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. )'; IF - /*fix version check*/ - CONVERT - ( - integer, - SUBSTRING - ( - CONVERT - ( - varchar(20), - SERVERPROPERTY('ProductVersion') - ), - 1, - 2 - ) - ) >= 13 + /* Check SQL Server 2016+ for temporal tables support */ + (CONVERT(INT, SERVERPROPERTY('EngineEdition')) IN (5, 8) /* Azure SQL DB or Managed Instance */ + OR CONVERT(INT, SUBSTRING(CONVERT(VARCHAR(20), SERVERPROPERTY('ProductVersion')), 1, 2)) >= 13) /* SQL 2016+ */ BEGIN SET @sql += N' AND NOT EXISTS From 506f258537b2edf7c00f70485d229f82536477dd Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 13 Mar 2025 09:24:51 -0400 Subject: [PATCH 054/246] changing reporting changing reporting --- .DS_Store | Bin 0 -> 6148 bytes sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 789 ++++++++++++------ 2 files changed, 525 insertions(+), 264 deletions(-) create mode 100644 .DS_Store diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..493ca41b2dcd8b5d1a7aa04ab7f393c170b1a4ee GIT binary patch literal 6148 zcmeHK%}T>S5T317Q$*-NK|C#Zt=Lvk#7m6r!K)EHsMLf64aRI~S}T-79zb8n2l08F z+1-j#1uu%q49tGJ^Rt`%5_YlxAiP1;0H^|hgGyMcVDo{HpL9w})3mrIwx9=~80LKILNbsNy$FLs`^KO``5ObcwZkpu%di`0HX3Dd3 z^Ul1p;9U6EYUuZa{vh{)-WiRKl}f`&KL}5fc-W~f9jL4yBw5_k2}u-V$oXlKMQZ4& zK^EmY*EbHQ;#4}-)zPTAv(v1}^+vNXuF27EvtE;1o7>}Y#aUk4*gNXnq_>%RFfZ1c2`h6sSnUDngbeABM7F~;lLG+*qlZt3kg}q`3la6+2<6Mh{L6Z(buZ;89 zm4&^b2)#Pmr49$-8swH4U(u6` v)=Jb%R1%6S48EpdN0(xZrBb|tss-(mI*6{t!XR2u_(wp~zzs9-qYS(Nc{Nl^ literal 0 HcmV?d00001 diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index 413d6d70..b12cf08f 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -1864,87 +1864,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END; - /* Insert summary statistics first */ - /* Add a separator row for the header */ - INSERT INTO - #index_cleanup_results - ( - result_type, - sort_order, - script_type, - additional_info - ) - SELECT - 'HEADER', - 0, - N'=====', - N'==================== INDEX CLEANUP SUMMARY ===================='; - - /* Add a separator for scripts section */ - INSERT INTO - #index_cleanup_results - ( - result_type, - sort_order, - script_type, - additional_info - ) - SELECT - 'HEADER', - 4, /* Just before merge scripts at sort_order 5 */ - script_type = N'======', - additional_info = N'==================== INDEX SCRIPTS ===================='; - - /* Add a separator for report section at the end */ - INSERT INTO - #index_cleanup_results - ( - result_type, - sort_order, - script_type, - additional_info - ) - SELECT - result_type = 'HEADER', - sort_order = 99, - script_type = N'======', - additional_info = N'==================== END OF REPORT ===================='; - - /* Add a separator for indexes needing review section */ - INSERT INTO - #index_cleanup_results - WITH - (TABLOCK) - ( - result_type, - sort_order, - script_type, - additional_info - ) - SELECT - result_type = 'HEADER', - sort_order = 92, - script_type = N'======', - additional_info = N'==================== INDEXES NEEDING REVIEW ===================='; - - /* Add a separator for kept indexes section */ - INSERT INTO - #index_cleanup_results - WITH - (TABLOCK) - ( - result_type, - sort_order, - script_type, - additional_info - ) - SELECT - result_type = 'HEADER', - sort_order = 94, - script_type = N'======', - additional_info = N'==================== INDEXES KEPT ===================='; - - /* Add summary information */ + /* Create a reference to the detailed summary that will appear at the end */ INSERT INTO #index_cleanup_results ( @@ -1956,181 +1876,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SELECT result_type = 'SUMMARY', sort_order = 1, - script_type = 'Index Cleanup Summary', - additional_info = - N'Server uptime: ' + - CONVERT(nvarchar(10), @uptime_days) + - N' days' + - CASE - WHEN @uptime_warning = 1 - THEN N' (WARNING: Low uptime detected! Index usage data may be incomplete.)' - ELSE N'' - END + - N' | Tables analyzed: ' + - CONVERT - ( - nvarchar(10), - COUNT_BIG(DISTINCT CONCAT(ia.database_id, N'.', ia.schema_id, N'.', ia.object_id)) - ) + - N' | Total indexes: ' + - CONVERT - ( - nvarchar(10), - COUNT_BIG(*) - ) + - N' | Indexes to disable: ' + - CONVERT - ( - nvarchar(10), - SUM - ( - CASE - WHEN ia.action = 'DISABLE' - THEN 1 - ELSE 0 - END - ) - ) + - N' | Indexes to merge: ' + - CONVERT(nvarchar(10), SUM(CASE WHEN ia.action IN ('MERGE INCLUDES', 'MAKE UNIQUE') THEN 1 ELSE 0 END)) + - N' | Avg indexes per table: ' + - CONVERT - ( - nvarchar(10), - CONVERT - ( - decimal(10,2), - COUNT_BIG(*) * 1.0 / - NULLIF - ( - COUNT_BIG - ( - DISTINCT CONCAT(ia.database_id, N'.', ia.schema_id, N'.', ia.object_id) - ), - 0 - ) - ) - ) - FROM #index_analysis AS ia - OPTION(RECOMPILE); - - /* Insert space savings estimates */ - INSERT INTO - #index_cleanup_results - ( - result_type, - sort_order, - script_type, - additional_info - ) - SELECT - result_type = 'SUMMARY', - sort_order = 2, - script_type = 'Estimated Space Savings', - additional_info = - N'Space saved from cleanup: ' + - CONVERT - ( - nvarchar(20), - CONVERT - ( - decimal(10,4), - SUM - ( - CASE - WHEN ia.action IN ('DISABLE', 'MERGE INCLUDES', 'MAKE UNIQUE') - THEN ps.total_space_gb - ELSE 0 - END - ) - ) - ) + - N' GB | Compression savings estimate: ' + - CONVERT - ( - nvarchar(20), - CONVERT - ( - decimal(10,4), - SUM - ( - CASE - WHEN (ia.action IS NULL OR ia.action = 'KEEP') - AND ce.can_compress = 1 - THEN ps.total_space_gb * 0.20 /* Conservative estimate - 20% compression ratio */ - ELSE 0 - END - ) - ) - ) + - N' - ' + - CONVERT - ( - nvarchar(20), - CONVERT - ( - decimal(10,4), - SUM - ( - CASE - WHEN (ia.action IS NULL OR ia.action = 'KEEP') - AND ce.can_compress = 1 - THEN ps.total_space_gb * 0.60 /* Optimistic estimate - 60% compression ratio */ - ELSE 0 - END - ) - ) - ) + - N' GB | Total estimated savings: ' + - CONVERT - ( - nvarchar(20), - CONVERT - ( - decimal(10,4), - SUM - ( - CASE - WHEN ia.action IN ('DISABLE', 'MERGE INCLUDES', 'MAKE UNIQUE') - THEN ps.total_space_gb - WHEN (ia.action IS NULL OR ia.action = 'KEEP') - AND ce.can_compress = 1 - THEN ps.total_space_gb * 0.20 - ELSE 0 - END - ) - ) - ) + - N' - ' + - CONVERT - ( - nvarchar(20), - CONVERT - ( - decimal(10,4), - SUM - ( - CASE - WHEN ia.action IN ('DISABLE', 'MERGE INCLUDES', 'MAKE UNIQUE') - THEN ps.total_space_gb - WHEN (ia.action IS NULL OR ia.action = 'KEEP') - AND ce.can_compress = 1 - THEN ps.total_space_gb * 0.60 - ELSE 0 - END - ) - ) - ) + - N' GB' - FROM #index_analysis AS ia - LEFT JOIN #partition_stats AS ps - ON ia.database_id = ps.database_id - AND ia.object_id = ps.object_id - AND ia.index_id = ps.index_id - LEFT JOIN #compression_eligibility AS ce - ON ia.database_id = ce.database_id - AND ia.object_id = ce.object_id - AND ia.index_id = ce.index_id + script_type = 'Index Cleanup Scripts', + additional_info = N'A detailed index analysis report appears after these scripts' OPTION(RECOMPILE); @@ -3046,6 +2793,315 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ) OPTION(RECOMPILE); + /* Create a new temp table for detailed reporting statistics */ + CREATE TABLE #index_reporting_stats + ( + summary_level varchar(20) NOT NULL, /* 'DATABASE', 'TABLE', 'INDEX', 'SUMMARY' */ + database_name sysname NULL, + schema_name sysname NULL, + table_name sysname NULL, + index_name sysname NULL, + server_uptime_days int NULL, + uptime_warning bit NULL, + tables_analyzed int NULL, + index_count int NULL, + total_size_gb decimal(38, 4) NULL, + total_rows bigint NULL, + unused_indexes int NULL, + unused_size_gb decimal(38, 4) NULL, + indexes_to_disable int NULL, + indexes_to_merge int NULL, + avg_indexes_per_table decimal(10, 2) NULL, + space_saved_gb decimal(10, 4) NULL, + compression_min_savings_gb decimal(10, 4) NULL, + compression_max_savings_gb decimal(10, 4) NULL, + total_min_savings_gb decimal(10, 4) NULL, + total_max_savings_gb decimal(10, 4) NULL, + /* Index usage metrics */ + total_reads bigint NULL, + total_writes bigint NULL, + user_seeks bigint NULL, + user_scans bigint NULL, + user_lookups bigint NULL, + user_updates bigint NULL, + /* Operational stats */ + range_scan_count bigint NULL, + singleton_lookup_count bigint NULL, + /* Lock stats */ + row_lock_count bigint NULL, + row_lock_wait_count bigint NULL, + row_lock_wait_in_ms bigint NULL, + page_lock_count bigint NULL, + page_lock_wait_count bigint NULL, + page_lock_wait_in_ms bigint NULL, + /* Latch stats */ + page_latch_wait_count bigint NULL, + page_latch_wait_in_ms bigint NULL, + page_io_latch_wait_count bigint NULL, + page_io_latch_wait_in_ms bigint NULL, + /* Misc stats */ + forwarded_fetch_count bigint NULL, + leaf_insert_count bigint NULL, + leaf_update_count bigint NULL, + leaf_delete_count bigint NULL, + PRIMARY KEY (summary_level, ISNULL(database_name, ''), ISNULL(schema_name, ''), ISNULL(table_name, ''), ISNULL(index_name, '')) + ); + + /* Insert database-level summaries */ + INSERT INTO #index_reporting_stats + ( + summary_level, + database_name, + index_count, + total_size_gb, + total_rows, + unused_indexes, + unused_size_gb, + total_reads, + total_writes, + user_seeks, + user_scans, + user_lookups, + user_updates, + range_scan_count, + singleton_lookup_count, + row_lock_count, + row_lock_wait_count, + row_lock_wait_in_ms, + page_lock_count, + page_lock_wait_count, + page_lock_wait_in_ms, + page_latch_wait_count, + page_latch_wait_in_ms, + page_io_latch_wait_count, + page_io_latch_wait_in_ms, + forwarded_fetch_count, + leaf_insert_count, + leaf_update_count, + leaf_delete_count + ) + SELECT + summary_level = 'DATABASE', + ps.database_name, + index_count = COUNT(DISTINCT CONCAT(ps.object_id, N'.', ps.index_id)), + total_size_gb = SUM(ps.total_space_gb), + total_rows = SUM(ps.total_rows), + unused_indexes = SUM(CASE WHEN id.user_seeks + id.user_scans + id.user_lookups = 0 THEN 1 ELSE 0 END), + unused_size_gb = SUM(CASE WHEN id.user_seeks + id.user_scans + id.user_lookups = 0 THEN ps.total_space_gb ELSE 0 END), + total_reads = SUM(id.user_seeks + id.user_scans + id.user_lookups), + total_writes = SUM(id.user_updates), + user_seeks = SUM(id.user_seeks), + user_scans = SUM(id.user_scans), + user_lookups = SUM(id.user_lookups), + user_updates = SUM(id.user_updates), + range_scan_count = SUM(os.range_scan_count), + singleton_lookup_count = SUM(os.singleton_lookup_count), + row_lock_count = SUM(os.row_lock_count), + row_lock_wait_count = SUM(os.row_lock_wait_count), + row_lock_wait_in_ms = SUM(os.row_lock_wait_in_ms), + page_lock_count = SUM(os.page_lock_count), + page_lock_wait_count = SUM(os.page_lock_wait_count), + page_lock_wait_in_ms = SUM(os.page_lock_wait_in_ms), + page_latch_wait_count = SUM(os.page_latch_wait_count), + page_latch_wait_in_ms = SUM(os.page_latch_wait_in_ms), + page_io_latch_wait_count = SUM(os.page_io_latch_wait_count), + page_io_latch_wait_in_ms = SUM(os.page_io_latch_wait_in_ms), + forwarded_fetch_count = SUM(os.forwarded_fetch_count), + leaf_insert_count = SUM(os.leaf_insert_count), + leaf_update_count = SUM(os.leaf_update_count), + leaf_delete_count = SUM(os.leaf_delete_count) + FROM #partition_stats ps + LEFT JOIN #index_details id + ON id.database_id = ps.database_id + AND id.object_id = ps.object_id + AND id.index_id = ps.index_id + AND id.is_included_column = 0 + AND id.key_ordinal > 0 + LEFT JOIN #operational_stats os + ON os.database_id = ps.database_id + AND os.object_id = ps.object_id + AND os.index_id = ps.index_id + GROUP BY ps.database_name + OPTION(RECOMPILE); + + /* Insert table-level summaries */ + INSERT INTO #index_reporting_stats + ( + summary_level, + database_name, + schema_name, + table_name, + index_count, + total_size_gb, + total_rows, + unused_indexes, + unused_size_gb, + total_reads, + total_writes, + user_seeks, + user_scans, + user_lookups, + user_updates, + range_scan_count, + singleton_lookup_count, + row_lock_count, + row_lock_wait_count, + row_lock_wait_in_ms, + page_lock_count, + page_lock_wait_count, + page_lock_wait_in_ms, + page_latch_wait_count, + page_latch_wait_in_ms, + page_io_latch_wait_count, + page_io_latch_wait_in_ms, + forwarded_fetch_count, + leaf_insert_count, + leaf_update_count, + leaf_delete_count + ) + SELECT + summary_level = 'TABLE', + ps.database_name, + ps.schema_name, + ps.table_name, + index_count = COUNT(DISTINCT ps.index_id), + total_size_gb = SUM(ps.total_space_gb), + total_rows = MAX(CASE WHEN ps.index_id IN (0, 1) THEN ps.total_rows ELSE 0 END), + unused_indexes = SUM(CASE WHEN id.user_seeks + id.user_scans + id.user_lookups = 0 THEN 1 ELSE 0 END), + unused_size_gb = SUM(CASE WHEN id.user_seeks + id.user_scans + id.user_lookups = 0 THEN ps.total_space_gb ELSE 0 END), + total_reads = SUM(id.user_seeks + id.user_scans + id.user_lookups), + total_writes = SUM(id.user_updates), + user_seeks = SUM(id.user_seeks), + user_scans = SUM(id.user_scans), + user_lookups = SUM(id.user_lookups), + user_updates = SUM(id.user_updates), + range_scan_count = SUM(os.range_scan_count), + singleton_lookup_count = SUM(os.singleton_lookup_count), + row_lock_count = SUM(os.row_lock_count), + row_lock_wait_count = SUM(os.row_lock_wait_count), + row_lock_wait_in_ms = SUM(os.row_lock_wait_in_ms), + page_lock_count = SUM(os.page_lock_count), + page_lock_wait_count = SUM(os.page_lock_wait_count), + page_lock_wait_in_ms = SUM(os.page_lock_wait_in_ms), + page_latch_wait_count = SUM(os.page_latch_wait_count), + page_latch_wait_in_ms = SUM(os.page_latch_wait_in_ms), + page_io_latch_wait_count = SUM(os.page_io_latch_wait_count), + page_io_latch_wait_in_ms = SUM(os.page_io_latch_wait_in_ms), + forwarded_fetch_count = SUM(os.forwarded_fetch_count), + leaf_insert_count = SUM(os.leaf_insert_count), + leaf_update_count = SUM(os.leaf_update_count), + leaf_delete_count = SUM(os.leaf_delete_count) + FROM #partition_stats ps + LEFT JOIN #index_details id + ON id.database_id = ps.database_id + AND id.object_id = ps.object_id + AND id.index_id = ps.index_id + AND id.is_included_column = 0 + AND id.key_ordinal > 0 + LEFT JOIN #operational_stats os + ON os.database_id = ps.database_id + AND os.object_id = ps.object_id + AND os.index_id = ps.index_id + GROUP BY ps.database_name, ps.schema_name, ps.table_name + OPTION(RECOMPILE); + + /* Insert index-level summaries */ + INSERT INTO #index_reporting_stats + ( + summary_level, + database_name, + schema_name, + table_name, + index_name, + total_size_gb, + total_rows, + total_reads, + total_writes, + user_seeks, + user_scans, + user_lookups, + user_updates, + range_scan_count, + singleton_lookup_count, + row_lock_count, + row_lock_wait_count, + row_lock_wait_in_ms, + page_lock_count, + page_lock_wait_count, + page_lock_wait_in_ms, + page_latch_wait_count, + page_latch_wait_in_ms, + page_io_latch_wait_count, + page_io_latch_wait_in_ms, + forwarded_fetch_count, + leaf_insert_count, + leaf_update_count, + leaf_delete_count + ) + SELECT + summary_level = 'INDEX', + ps.database_name, + ps.schema_name, + ps.table_name, + ps.index_name, + total_size_gb = SUM(ps.total_space_gb), + total_rows = SUM(ps.total_rows), + total_reads = SUM(id.user_seeks + id.user_scans + id.user_lookups), + total_writes = SUM(id.user_updates), + user_seeks = MAX(id.user_seeks), + user_scans = MAX(id.user_scans), + user_lookups = MAX(id.user_lookups), + user_updates = MAX(id.user_updates), + range_scan_count = os.range_scan_count, + singleton_lookup_count = os.singleton_lookup_count, + row_lock_count = os.row_lock_count, + row_lock_wait_count = os.row_lock_wait_count, + row_lock_wait_in_ms = os.row_lock_wait_in_ms, + page_lock_count = os.page_lock_count, + page_lock_wait_count = os.page_lock_wait_count, + page_lock_wait_in_ms = os.page_lock_wait_in_ms, + page_latch_wait_count = os.page_latch_wait_count, + page_latch_wait_in_ms = os.page_latch_wait_in_ms, + page_io_latch_wait_count = os.page_io_latch_wait_count, + page_io_latch_wait_in_ms = os.page_io_latch_wait_in_ms, + forwarded_fetch_count = os.forwarded_fetch_count, + leaf_insert_count = os.leaf_insert_count, + leaf_update_count = os.leaf_update_count, + leaf_delete_count = os.leaf_delete_count + FROM #partition_stats ps + LEFT JOIN #index_details id + ON id.database_id = ps.database_id + AND id.object_id = ps.object_id + AND id.index_name = ps.index_name + AND id.is_included_column = 0 + AND id.key_ordinal > 0 + LEFT JOIN #operational_stats os + ON os.database_id = ps.database_id + AND os.object_id = ps.object_id + AND os.index_id = ps.index_id + GROUP BY + ps.database_name, + ps.schema_name, + ps.table_name, + ps.index_name, + os.range_scan_count, + os.singleton_lookup_count, + os.row_lock_count, + os.row_lock_wait_count, + os.row_lock_wait_in_ms, + os.page_lock_count, + os.page_lock_wait_count, + os.page_lock_wait_in_ms, + os.page_latch_wait_count, + os.page_latch_wait_in_ms, + os.page_io_latch_wait_count, + os.page_io_latch_wait_in_ms, + os.forwarded_fetch_count, + os.leaf_insert_count, + os.leaf_update_count, + os.leaf_delete_count + OPTION(RECOMPILE); + /* Return the consolidated results in a single result set Results are ordered by: @@ -3086,25 +3142,25 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Add size and usage metrics */ index_size_gb = CASE - WHEN ir.result_type IN ('HEADER', 'SUMMARY') + WHEN ir.result_type = 'SUMMARY' THEN NULL ELSE FORMAT(ir.index_size_gb, 'N4') END, index_rows = CASE - WHEN ir.result_type IN ('HEADER', 'SUMMARY') + WHEN ir.result_type = 'SUMMARY' THEN NULL ELSE FORMAT(ir.index_rows, 'N0') END, index_reads = CASE - WHEN ir.result_type IN ('HEADER', 'SUMMARY') + WHEN ir.result_type = 'SUMMARY' THEN NULL ELSE FORMAT(ir.index_reads, 'N0') END, index_writes = CASE - WHEN ir.result_type IN ('HEADER', 'SUMMARY') + WHEN ir.result_type = 'SUMMARY' THEN NULL ELSE FORMAT(ir.index_writes, 'N0') END, @@ -3121,15 +3177,15 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ir.database_name, /* Within each sort_order group, prioritize by size and usage */ CASE - /* For HEADER and SUMMARY, keep the original order */ - WHEN ir.result_type IN ('HEADER', 'SUMMARY') + /* For SUMMARY, keep the original order */ + WHEN ir.result_type = 'SUMMARY' THEN 0 /* For script categories, order by size and impact */ ELSE ISNULL(ir.index_size_gb, 0) END DESC, CASE - /* For HEADER and SUMMARY, keep the original order */ - WHEN ir.result_type IN ('HEADER', 'SUMMARY') + /* For SUMMARY, keep the original order */ + WHEN ir.result_type = 'SUMMARY' THEN 0 /* For script categories, consider rows as secondary sort */ ELSE ISNULL(ir.index_rows, 0) @@ -3140,6 +3196,211 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ir.index_name OPTION(RECOMPILE); + /* Insert overall summary information */ + INSERT INTO #index_reporting_stats + ( + summary_level, + server_uptime_days, + uptime_warning, + tables_analyzed, + index_count, + indexes_to_disable, + indexes_to_merge, + avg_indexes_per_table, + space_saved_gb, + compression_min_savings_gb, + compression_max_savings_gb, + total_min_savings_gb, + total_max_savings_gb + ) + SELECT + summary_level = 'SUMMARY', + server_uptime_days = @uptime_days, + uptime_warning = @uptime_warning, + tables_analyzed = COUNT_BIG(DISTINCT CONCAT(ia.database_id, N'.', ia.schema_id, N'.', ia.object_id)), + index_count = COUNT_BIG(*), + indexes_to_disable = SUM(CASE WHEN ia.action = 'DISABLE' THEN 1 ELSE 0 END), + indexes_to_merge = SUM(CASE WHEN ia.action IN ('MERGE INCLUDES', 'MAKE UNIQUE') THEN 1 ELSE 0 END), + avg_indexes_per_table = COUNT_BIG(*) * 1.0 / + NULLIF(COUNT_BIG(DISTINCT CONCAT(ia.database_id, N'.', ia.schema_id, N'.', ia.object_id)), 0), + /* Space savings from cleanup */ + space_saved_gb = SUM(CASE + WHEN ia.action IN ('DISABLE', 'MERGE INCLUDES', 'MAKE UNIQUE') + THEN ps.total_space_gb + ELSE 0 + END), + /* Conservative compression savings estimate (20%) */ + compression_min_savings_gb = SUM(CASE + WHEN (ia.action IS NULL OR ia.action = 'KEEP') + AND ce.can_compress = 1 + THEN ps.total_space_gb * 0.20 + ELSE 0 + END), + /* Optimistic compression savings estimate (60%) */ + compression_max_savings_gb = SUM(CASE + WHEN (ia.action IS NULL OR ia.action = 'KEEP') + AND ce.can_compress = 1 + THEN ps.total_space_gb * 0.60 + ELSE 0 + END), + /* Total conservative savings */ + total_min_savings_gb = SUM(CASE + WHEN ia.action IN ('DISABLE', 'MERGE INCLUDES', 'MAKE UNIQUE') + THEN ps.total_space_gb + WHEN (ia.action IS NULL OR ia.action = 'KEEP') + AND ce.can_compress = 1 + THEN ps.total_space_gb * 0.20 + ELSE 0 + END), + /* Total optimistic savings */ + total_max_savings_gb = SUM(CASE + WHEN ia.action IN ('DISABLE', 'MERGE INCLUDES', 'MAKE UNIQUE') + THEN ps.total_space_gb + WHEN (ia.action IS NULL OR ia.action = 'KEEP') + AND ce.can_compress = 1 + THEN ps.total_space_gb * 0.60 + ELSE 0 + END) + FROM #index_analysis AS ia + LEFT JOIN #partition_stats AS ps + ON ia.database_id = ps.database_id + AND ia.object_id = ps.object_id + AND ia.index_id = ps.index_id + LEFT JOIN #compression_eligibility AS ce + ON ia.database_id = ce.database_id + AND ia.object_id = ce.object_id + AND ia.index_id = ce.index_id + OPTION(RECOMPILE); + + /* Return the detailed reporting statistics */ + SELECT + CASE + WHEN irs.summary_level = 'SUMMARY' + THEN '=== OVERALL ANALYSIS ===' + ELSE irs.summary_level + END AS summary_level, + irs.database_name, + irs.schema_name, + irs.table_name, + irs.index_name, + + /* Special formatting for summary level */ + CASE + WHEN irs.summary_level = 'SUMMARY' + THEN 'Server uptime: ' + CONVERT(varchar(10), irs.server_uptime_days) + ' days' + + CASE + WHEN irs.uptime_warning = 1 + THEN ' (WARNING: Low uptime - usage data may be incomplete!)' + ELSE '' + END + + ' | Tables analyzed: ' + FORMAT(irs.tables_analyzed, 'N0') + + ' | Total indexes: ' + FORMAT(irs.index_count, 'N0') + + ' | Indexes to disable: ' + FORMAT(irs.indexes_to_disable, 'N0') + + ' | Indexes to merge: ' + FORMAT(irs.indexes_to_merge, 'N0') + + ' | Avg indexes per table: ' + FORMAT(irs.avg_indexes_per_table, 'N2') + ELSE FORMAT(irs.index_count, 'N0') + END AS index_count, + + /* Size metrics - special handling for summary */ + CASE + WHEN irs.summary_level = 'SUMMARY' + THEN 'Space saved from cleanup: ' + FORMAT(irs.space_saved_gb, 'N4') + ' GB' + + ' | Compression savings: ' + FORMAT(irs.compression_min_savings_gb, 'N4') + ' - ' + + FORMAT(irs.compression_max_savings_gb, 'N4') + ' GB' + + ' | Total savings: ' + FORMAT(irs.total_min_savings_gb, 'N4') + ' - ' + + FORMAT(irs.total_max_savings_gb, 'N4') + ' GB' + ELSE FORMAT(irs.total_size_gb, 'N4') + END AS total_size_gb, + + /* Skip other metrics for summary level */ + CASE WHEN irs.summary_level = 'SUMMARY' THEN NULL ELSE FORMAT(irs.total_rows, 'N0') END AS total_rows, + CASE WHEN irs.summary_level = 'SUMMARY' THEN NULL ELSE FORMAT(irs.unused_indexes, 'N0') END AS unused_indexes, + CASE WHEN irs.summary_level = 'SUMMARY' THEN NULL ELSE FORMAT(irs.unused_size_gb, 'N4') END AS unused_size_gb, + + /* Usage metrics */ + CASE WHEN irs.summary_level = 'SUMMARY' THEN NULL ELSE FORMAT(irs.total_reads, 'N0') END AS total_reads, + CASE WHEN irs.summary_level = 'SUMMARY' THEN NULL ELSE FORMAT(irs.total_writes, 'N0') END AS total_writes, + CASE WHEN irs.summary_level = 'SUMMARY' THEN NULL ELSE FORMAT(irs.user_seeks, 'N0') END AS user_seeks, + CASE WHEN irs.summary_level = 'SUMMARY' THEN NULL ELSE FORMAT(irs.user_scans, 'N0') END AS user_scans, + CASE WHEN irs.summary_level = 'SUMMARY' THEN NULL ELSE FORMAT(irs.user_lookups, 'N0') END AS user_lookups, + CASE WHEN irs.summary_level = 'SUMMARY' THEN NULL ELSE FORMAT(irs.user_updates, 'N0') END AS user_updates, + + /* Operational metrics */ + CASE WHEN irs.summary_level = 'SUMMARY' THEN NULL ELSE FORMAT(irs.range_scan_count, 'N0') END AS range_scan_count, + CASE WHEN irs.summary_level = 'SUMMARY' THEN NULL ELSE FORMAT(irs.singleton_lookup_count, 'N0') END AS singleton_lookup_count, + + /* Lock wait percentages */ + CASE + WHEN irs.summary_level = 'SUMMARY' THEN NULL + WHEN irs.row_lock_count > 0 + THEN FORMAT(100.0 * irs.row_lock_wait_count / NULLIF(irs.row_lock_count, 0), 'N2') + '%' + ELSE '0.00%' + END AS row_lock_wait_pct, + + CASE + WHEN irs.summary_level = 'SUMMARY' THEN NULL + WHEN irs.row_lock_wait_count > 0 + THEN FORMAT(1.0 * irs.row_lock_wait_in_ms / NULLIF(irs.row_lock_wait_count, 0), 'N2') + ELSE '0.00' + END AS row_lock_wait_ms_avg, + + CASE + WHEN irs.summary_level = 'SUMMARY' THEN NULL + WHEN irs.page_lock_count > 0 + THEN FORMAT(100.0 * irs.page_lock_wait_count / NULLIF(irs.page_lock_count, 0), 'N2') + '%' + ELSE '0.00%' + END AS page_lock_wait_pct, + + CASE + WHEN irs.summary_level = 'SUMMARY' THEN NULL + WHEN irs.page_lock_wait_count > 0 + THEN FORMAT(1.0 * irs.page_lock_wait_in_ms / NULLIF(irs.page_lock_wait_count, 0), 'N2') + ELSE '0.00' + END AS page_lock_wait_ms_avg, + + /* Latch wait averages */ + CASE + WHEN irs.summary_level = 'SUMMARY' THEN NULL + WHEN irs.page_latch_wait_count > 0 + THEN FORMAT(1.0 * irs.page_latch_wait_in_ms / NULLIF(irs.page_latch_wait_count, 0), 'N2') + ELSE '0.00' + END AS page_latch_wait_ms_avg, + + CASE + WHEN irs.summary_level = 'SUMMARY' THEN NULL + WHEN irs.page_io_latch_wait_count > 0 + THEN FORMAT(1.0 * irs.page_io_latch_wait_in_ms / NULLIF(irs.page_io_latch_wait_count, 0), 'N2') + ELSE '0.00' + END AS page_io_latch_wait_ms_avg, + + /* DML Counts */ + CASE WHEN irs.summary_level = 'SUMMARY' THEN NULL ELSE FORMAT(irs.forwarded_fetch_count, 'N0') END AS forwarded_fetch_count, + CASE WHEN irs.summary_level = 'SUMMARY' THEN NULL ELSE FORMAT(irs.leaf_insert_count, 'N0') END AS leaf_inserts, + CASE WHEN irs.summary_level = 'SUMMARY' THEN NULL ELSE FORMAT(irs.leaf_update_count, 'N0') END AS leaf_updates, + CASE WHEN irs.summary_level = 'SUMMARY' THEN NULL ELSE FORMAT(irs.leaf_delete_count, 'N0') END AS leaf_deletes + FROM #index_reporting_stats AS irs + ORDER BY + /* Order by level - put summary first */ + CASE + WHEN irs.summary_level = 'SUMMARY' THEN 0 + WHEN irs.summary_level = 'DATABASE' THEN 1 + WHEN irs.summary_level = 'TABLE' THEN 2 + WHEN irs.summary_level = 'INDEX' THEN 3 + ELSE 4 + END, + /* Then by database name */ + irs.database_name, + /* For tables and indexes, sort by size */ + CASE + WHEN irs.summary_level IN ('SUMMARY', 'DATABASE') THEN 0 + ELSE ISNULL(irs.total_size_gb, 0) + END DESC, + /* Then by schema, table, index name */ + irs.schema_name, + irs.table_name, + irs.index_name + OPTION(RECOMPILE); + END TRY BEGIN CATCH THROW; From 748886735ee177d84503725669059d42032d07b8 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 13 Mar 2025 09:24:59 -0400 Subject: [PATCH 055/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 47 ++++++++++++++++--- 1 file changed, 41 insertions(+), 6 deletions(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index 413d6d70..a7d1a4c7 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -49,9 +49,26 @@ BEGIN TRY IF /* Check SQL Server 2012+ for FORMAT and CONCAT functions */ - (CONVERT(INT, SERVERPROPERTY('EngineEdition')) NOT IN (5, 8) /* Not Azure SQL DB or Managed Instance */ - AND CONVERT(INT, SUBSTRING(CONVERT(VARCHAR(20), SERVERPROPERTY('ProductVersion')), 1, 2)) < 11) /* Pre-2012 */ - + ( + CONVERT + ( + integer, + SERVERPROPERTY('EngineEdition') + ) NOT IN (5, 8) /* Not Azure SQL DB or Managed Instance */ + AND CONVERT + ( + integer, + SUBSTRING + ( + CONVERT + ( + varchar(20), + SERVERPROPERTY('ProductVersion') + ), + 1, + 2 + ) + ) < 11) /* Pre-2012 */ BEGIN RAISERROR('This procedure requires SQL Server 2012 (11.0) or later due to the use of FORMAT and CONCAT functions.', 11, 1); RETURN; @@ -600,13 +617,31 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 1/0 FROM ' + QUOTENAME(@database_name) + N'.sys.views AS v WHERE v.object_id = i.object_id - AND v.is_indexed_view = 1 )'; IF /* Check SQL Server 2016+ for temporal tables support */ - (CONVERT(INT, SERVERPROPERTY('EngineEdition')) IN (5, 8) /* Azure SQL DB or Managed Instance */ - OR CONVERT(INT, SUBSTRING(CONVERT(VARCHAR(20), SERVERPROPERTY('ProductVersion')), 1, 2)) >= 13) /* SQL 2016+ */ + ( + CONVERT + ( + integer, + SERVERPROPERTY('EngineEdition') + ) IN (5, 8) /* Azure SQL DB or Managed Instance */ + OR CONVERT + ( + integer, + SUBSTRING + ( + CONVERT + ( + varchar(20), + SERVERPROPERTY('ProductVersion') + ), + 1, + 2 + ) + ) >= 13 + ) /* SQL 2016+ */ BEGIN SET @sql += N' AND NOT EXISTS From c4c34d7b330e14e4181ef9035ee861809414c522 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 13 Mar 2025 09:37:47 -0400 Subject: [PATCH 056/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index e9819613..23454a28 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -3080,7 +3080,14 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ps.table_name, ps.index_name, total_size_gb = SUM(ps.total_space_gb), - total_rows = SUM(ps.total_rows), + /* For indexes, use the base table row count to avoid duplication */ + total_rows = ( + SELECT MAX(base_ps.total_rows) + FROM #partition_stats base_ps + WHERE base_ps.database_id = ps.database_id + AND base_ps.object_id = ps.object_id + AND base_ps.index_id IN (0, 1) /* Clustered index or heap */ + ), total_reads = SUM(id.user_seeks + id.user_scans + id.user_lookups), total_writes = SUM(id.user_updates), user_seeks = MAX(id.user_seeks), @@ -3115,6 +3122,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND os.object_id = ps.object_id AND os.index_id = ps.index_id GROUP BY + ps.database_id, + ps.object_id, ps.database_name, ps.schema_name, ps.table_name, From 470d125cdfe9e001daaa0ca697ffef0d2db14630 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 13 Mar 2025 09:38:07 -0400 Subject: [PATCH 057/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index e9819613..4ff61b25 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -2878,8 +2878,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. forwarded_fetch_count bigint NULL, leaf_insert_count bigint NULL, leaf_update_count bigint NULL, - leaf_delete_count bigint NULL, - PRIMARY KEY (summary_level, ISNULL(database_name, ''), ISNULL(schema_name, ''), ISNULL(table_name, ''), ISNULL(index_name, '')) + leaf_delete_count bigint NULL ); /* Insert database-level summaries */ From f514f6add56d70095e9069ea23edc59622d2b65e Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 13 Mar 2025 09:49:19 -0400 Subject: [PATCH 058/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 122 ++---------------- 1 file changed, 8 insertions(+), 114 deletions(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index 23454a28..98de1889 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -3040,111 +3040,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. GROUP BY ps.database_name, ps.schema_name, ps.table_name OPTION(RECOMPILE); - /* Insert index-level summaries */ - INSERT INTO #index_reporting_stats - ( - summary_level, - database_name, - schema_name, - table_name, - index_name, - total_size_gb, - total_rows, - total_reads, - total_writes, - user_seeks, - user_scans, - user_lookups, - user_updates, - range_scan_count, - singleton_lookup_count, - row_lock_count, - row_lock_wait_count, - row_lock_wait_in_ms, - page_lock_count, - page_lock_wait_count, - page_lock_wait_in_ms, - page_latch_wait_count, - page_latch_wait_in_ms, - page_io_latch_wait_count, - page_io_latch_wait_in_ms, - forwarded_fetch_count, - leaf_insert_count, - leaf_update_count, - leaf_delete_count - ) - SELECT - summary_level = 'INDEX', - ps.database_name, - ps.schema_name, - ps.table_name, - ps.index_name, - total_size_gb = SUM(ps.total_space_gb), - /* For indexes, use the base table row count to avoid duplication */ - total_rows = ( - SELECT MAX(base_ps.total_rows) - FROM #partition_stats base_ps - WHERE base_ps.database_id = ps.database_id - AND base_ps.object_id = ps.object_id - AND base_ps.index_id IN (0, 1) /* Clustered index or heap */ - ), - total_reads = SUM(id.user_seeks + id.user_scans + id.user_lookups), - total_writes = SUM(id.user_updates), - user_seeks = MAX(id.user_seeks), - user_scans = MAX(id.user_scans), - user_lookups = MAX(id.user_lookups), - user_updates = MAX(id.user_updates), - range_scan_count = os.range_scan_count, - singleton_lookup_count = os.singleton_lookup_count, - row_lock_count = os.row_lock_count, - row_lock_wait_count = os.row_lock_wait_count, - row_lock_wait_in_ms = os.row_lock_wait_in_ms, - page_lock_count = os.page_lock_count, - page_lock_wait_count = os.page_lock_wait_count, - page_lock_wait_in_ms = os.page_lock_wait_in_ms, - page_latch_wait_count = os.page_latch_wait_count, - page_latch_wait_in_ms = os.page_latch_wait_in_ms, - page_io_latch_wait_count = os.page_io_latch_wait_count, - page_io_latch_wait_in_ms = os.page_io_latch_wait_in_ms, - forwarded_fetch_count = os.forwarded_fetch_count, - leaf_insert_count = os.leaf_insert_count, - leaf_update_count = os.leaf_update_count, - leaf_delete_count = os.leaf_delete_count - FROM #partition_stats ps - LEFT JOIN #index_details id - ON id.database_id = ps.database_id - AND id.object_id = ps.object_id - AND id.index_name = ps.index_name - AND id.is_included_column = 0 - AND id.key_ordinal > 0 - LEFT JOIN #operational_stats os - ON os.database_id = ps.database_id - AND os.object_id = ps.object_id - AND os.index_id = ps.index_id - GROUP BY - ps.database_id, - ps.object_id, - ps.database_name, - ps.schema_name, - ps.table_name, - ps.index_name, - os.range_scan_count, - os.singleton_lookup_count, - os.row_lock_count, - os.row_lock_wait_count, - os.row_lock_wait_in_ms, - os.page_lock_count, - os.page_lock_wait_count, - os.page_lock_wait_in_ms, - os.page_latch_wait_count, - os.page_latch_wait_in_ms, - os.page_io_latch_wait_count, - os.page_io_latch_wait_in_ms, - os.forwarded_fetch_count, - os.leaf_insert_count, - os.leaf_update_count, - os.leaf_delete_count - OPTION(RECOMPILE); + /* We're not doing index-level summaries - focusing on database and table level reports */ /* Return the consolidated results in a single result set @@ -3316,7 +3212,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND ia.index_id = ce.index_id OPTION(RECOMPILE); - /* Return the detailed reporting statistics */ + /* Return the detailed reporting statistics (summary and table levels only) */ SELECT CASE WHEN irs.summary_level = 'SUMMARY' @@ -3326,7 +3222,6 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. irs.database_name, irs.schema_name, irs.table_name, - irs.index_name, /* Special formatting for summary level */ CASE @@ -3343,7 +3238,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ' | Indexes to merge: ' + FORMAT(irs.indexes_to_merge, 'N0') + ' | Avg indexes per table: ' + FORMAT(irs.avg_indexes_per_table, 'N2') ELSE FORMAT(irs.index_count, 'N0') - END AS index_count, + END AS indexes, /* Size metrics - special handling for summary */ CASE @@ -3423,26 +3318,25 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CASE WHEN irs.summary_level = 'SUMMARY' THEN NULL ELSE FORMAT(irs.leaf_update_count, 'N0') END AS leaf_updates, CASE WHEN irs.summary_level = 'SUMMARY' THEN NULL ELSE FORMAT(irs.leaf_delete_count, 'N0') END AS leaf_deletes FROM #index_reporting_stats AS irs + WHERE irs.summary_level IN ('SUMMARY', 'DATABASE', 'TABLE') /* Filter out INDEX level */ ORDER BY /* Order by level - put summary first */ CASE WHEN irs.summary_level = 'SUMMARY' THEN 0 WHEN irs.summary_level = 'DATABASE' THEN 1 WHEN irs.summary_level = 'TABLE' THEN 2 - WHEN irs.summary_level = 'INDEX' THEN 3 - ELSE 4 + ELSE 3 END, /* Then by database name */ irs.database_name, - /* For tables and indexes, sort by size */ + /* For tables, sort by size */ CASE WHEN irs.summary_level IN ('SUMMARY', 'DATABASE') THEN 0 ELSE ISNULL(irs.total_size_gb, 0) END DESC, - /* Then by schema, table, index name */ + /* Then by schema, table */ irs.schema_name, - irs.table_name, - irs.index_name + irs.table_name OPTION(RECOMPILE); END TRY From f2ea094bc7a2a3a28eaf41dff1cfe4c74b00a596 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 13 Mar 2025 10:59:28 -0400 Subject: [PATCH 059/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 160 ++++++++++-------- 1 file changed, 89 insertions(+), 71 deletions(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index bf309789..5e93de9b 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -577,6 +577,60 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. index_writes bigint NULL /* Total writes (updates) */ ); + /* Create a new temp table for detailed reporting statistics */ + CREATE TABLE + #index_reporting_stats + ( + summary_level varchar(20) NOT NULL, /* 'DATABASE', 'TABLE', 'INDEX', 'SUMMARY' */ + database_name sysname NULL, + schema_name sysname NULL, + table_name sysname NULL, + index_name sysname NULL, + server_uptime_days int NULL, + uptime_warning bit NULL, + tables_analyzed int NULL, + index_count int NULL, + total_size_gb decimal(38, 4) NULL, + total_rows bigint NULL, + unused_indexes int NULL, + unused_size_gb decimal(38, 4) NULL, + indexes_to_disable int NULL, + indexes_to_merge int NULL, + avg_indexes_per_table decimal(10, 2) NULL, + space_saved_gb decimal(10, 4) NULL, + compression_min_savings_gb decimal(10, 4) NULL, + compression_max_savings_gb decimal(10, 4) NULL, + total_min_savings_gb decimal(10, 4) NULL, + total_max_savings_gb decimal(10, 4) NULL, + /* Index usage metrics */ + total_reads bigint NULL, + total_writes bigint NULL, + user_seeks bigint NULL, + user_scans bigint NULL, + user_lookups bigint NULL, + user_updates bigint NULL, + /* Operational stats */ + range_scan_count bigint NULL, + singleton_lookup_count bigint NULL, + /* Lock stats */ + row_lock_count bigint NULL, + row_lock_wait_count bigint NULL, + row_lock_wait_in_ms bigint NULL, + page_lock_count bigint NULL, + page_lock_wait_count bigint NULL, + page_lock_wait_in_ms bigint NULL, + /* Latch stats */ + page_latch_wait_count bigint NULL, + page_latch_wait_in_ms bigint NULL, + page_io_latch_wait_count bigint NULL, + page_io_latch_wait_in_ms bigint NULL, + /* Misc stats */ + forwarded_fetch_count bigint NULL, + leaf_insert_count bigint NULL, + leaf_update_count bigint NULL, + leaf_delete_count bigint NULL + ); + /* Start insert queries */ @@ -2828,61 +2882,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ) OPTION(RECOMPILE); - /* Create a new temp table for detailed reporting statistics */ - CREATE TABLE #index_reporting_stats - ( - summary_level varchar(20) NOT NULL, /* 'DATABASE', 'TABLE', 'INDEX', 'SUMMARY' */ - database_name sysname NULL, - schema_name sysname NULL, - table_name sysname NULL, - index_name sysname NULL, - server_uptime_days int NULL, - uptime_warning bit NULL, - tables_analyzed int NULL, - index_count int NULL, - total_size_gb decimal(38, 4) NULL, - total_rows bigint NULL, - unused_indexes int NULL, - unused_size_gb decimal(38, 4) NULL, - indexes_to_disable int NULL, - indexes_to_merge int NULL, - avg_indexes_per_table decimal(10, 2) NULL, - space_saved_gb decimal(10, 4) NULL, - compression_min_savings_gb decimal(10, 4) NULL, - compression_max_savings_gb decimal(10, 4) NULL, - total_min_savings_gb decimal(10, 4) NULL, - total_max_savings_gb decimal(10, 4) NULL, - /* Index usage metrics */ - total_reads bigint NULL, - total_writes bigint NULL, - user_seeks bigint NULL, - user_scans bigint NULL, - user_lookups bigint NULL, - user_updates bigint NULL, - /* Operational stats */ - range_scan_count bigint NULL, - singleton_lookup_count bigint NULL, - /* Lock stats */ - row_lock_count bigint NULL, - row_lock_wait_count bigint NULL, - row_lock_wait_in_ms bigint NULL, - page_lock_count bigint NULL, - page_lock_wait_count bigint NULL, - page_lock_wait_in_ms bigint NULL, - /* Latch stats */ - page_latch_wait_count bigint NULL, - page_latch_wait_in_ms bigint NULL, - page_io_latch_wait_count bigint NULL, - page_io_latch_wait_in_ms bigint NULL, - /* Misc stats */ - forwarded_fetch_count bigint NULL, - leaf_insert_count bigint NULL, - leaf_update_count bigint NULL, - leaf_delete_count bigint NULL - ); - /* Insert database-level summaries */ - INSERT INTO #index_reporting_stats + INSERT INTO + #index_reporting_stats ( summary_level, database_name, @@ -3136,7 +3138,10 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. OPTION(RECOMPILE); /* Insert overall summary information */ - INSERT INTO #index_reporting_stats + INSERT INTO + #index_reporting_stats + WITH + (TABLOCK) ( summary_level, server_uptime_days, @@ -3156,25 +3161,38 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. summary_level = 'SUMMARY', server_uptime_days = @uptime_days, uptime_warning = @uptime_warning, - tables_analyzed = COUNT_BIG(DISTINCT CONCAT(ia.database_id, N'.', ia.schema_id, N'.', ia.object_id)), - index_count = COUNT_BIG(*), - indexes_to_disable = SUM(CASE WHEN ia.action = 'DISABLE' THEN 1 ELSE 0 END), - indexes_to_merge = SUM(CASE WHEN ia.action IN ('MERGE INCLUDES', 'MAKE UNIQUE') THEN 1 ELSE 0 END), - avg_indexes_per_table = COUNT_BIG(*) * 1.0 / + tables_analyzed = + COUNT_BIG(DISTINCT CONCAT(ia.database_id, N'.', ia.schema_id, N'.', ia.object_id)), + index_count = + COUNT_BIG(*), + indexes_to_disable = + SUM(CASE WHEN ia.action = 'DISABLE' THEN 1 ELSE 0 END), + indexes_to_merge = + SUM(CASE WHEN ia.action IN ('MERGE INCLUDES', 'MAKE UNIQUE') THEN 1 ELSE 0 END), + avg_indexes_per_table = + COUNT_BIG(*) * 1.0 / NULLIF(COUNT_BIG(DISTINCT CONCAT(ia.database_id, N'.', ia.schema_id, N'.', ia.object_id)), 0), /* Space savings from cleanup */ - space_saved_gb = SUM(CASE - WHEN ia.action IN ('DISABLE', 'MERGE INCLUDES', 'MAKE UNIQUE') - THEN ps.total_space_gb - ELSE 0 - END), + space_saved_gb = + SUM + ( + CASE + WHEN ia.action IN ('DISABLE', 'MERGE INCLUDES', 'MAKE UNIQUE') + THEN ps.total_space_gb + ELSE 0 + END + ), /* Conservative compression savings estimate (20%) */ - compression_min_savings_gb = SUM(CASE - WHEN (ia.action IS NULL OR ia.action = 'KEEP') - AND ce.can_compress = 1 - THEN ps.total_space_gb * 0.20 - ELSE 0 - END), + compression_min_savings_gb = + SUM + ( + CASE + WHEN (ia.action IS NULL OR ia.action = 'KEEP') + AND ce.can_compress = 1 + THEN ps.total_space_gb * 0.20 + ELSE 0 + END + ), /* Optimistic compression savings estimate (60%) */ compression_max_savings_gb = SUM(CASE WHEN (ia.action IS NULL OR ia.action = 'KEEP') From 85024e772b3d4cb9bdcba0976741143507b7f8db Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 13 Mar 2025 11:09:08 -0400 Subject: [PATCH 060/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 182 ++++++++++-------- 1 file changed, 99 insertions(+), 83 deletions(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index 5e93de9b..dde1fb93 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -3229,111 +3229,123 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND ia.index_id = ce.index_id OPTION(RECOMPILE); - /* Return the detailed reporting statistics (summary and table levels only) */ + /* Return streamlined reporting statistics focused on key metrics */ SELECT + /* Basic identification */ CASE - WHEN irs.summary_level = 'SUMMARY' - THEN '=== OVERALL ANALYSIS ===' + WHEN irs.summary_level = 'SUMMARY' THEN '=== OVERALL ANALYSIS ===' ELSE irs.summary_level - END AS summary_level, - irs.database_name, + END AS level, + + /* Server info (for summary) or database name */ + CASE + WHEN irs.summary_level = 'SUMMARY' AND irs.uptime_warning = 1 + THEN 'WARNING: Server uptime only ' + CONVERT(varchar(10), irs.server_uptime_days) + ' days - usage data may be incomplete!' + WHEN irs.summary_level = 'SUMMARY' + THEN 'Server uptime: ' + CONVERT(varchar(10), irs.server_uptime_days) + ' days' + ELSE irs.database_name + END AS database_info, + + /* Schema and table names (except for summary) */ irs.schema_name, irs.table_name, - /* Special formatting for summary level */ + /* ===== Section 1: Index Counts ===== */ + /* Tables analyzed (summary only) */ CASE - WHEN irs.summary_level = 'SUMMARY' - THEN 'Server uptime: ' + CONVERT(varchar(10), irs.server_uptime_days) + ' days' + - CASE - WHEN irs.uptime_warning = 1 - THEN ' (WARNING: Low uptime - usage data may be incomplete!)' - ELSE '' - END + - ' | Tables analyzed: ' + FORMAT(irs.tables_analyzed, 'N0') + - ' | Total indexes: ' + FORMAT(irs.index_count, 'N0') + - ' | Indexes to disable: ' + FORMAT(irs.indexes_to_disable, 'N0') + - ' | Indexes to merge: ' + FORMAT(irs.indexes_to_merge, 'N0') + - ' | Avg indexes per table: ' + FORMAT(irs.avg_indexes_per_table, 'N2') - ELSE FORMAT(irs.index_count, 'N0') - END AS indexes, + WHEN irs.summary_level = 'SUMMARY' THEN FORMAT(irs.tables_analyzed, 'N0') + ELSE NULL + END AS tables_analyzed, + + /* Total indexes */ + FORMAT(irs.index_count, 'N0') AS total_indexes, - /* Size metrics - special handling for summary */ + /* Removable indexes - show different values for summary vs. other levels */ + CASE + WHEN irs.summary_level = 'SUMMARY' THEN FORMAT(irs.indexes_to_disable + irs.indexes_to_merge, 'N0') + ELSE FORMAT(irs.unused_indexes, 'N0') + END AS removable_indexes, + + /* Percent of indexes that can be removed */ CASE WHEN irs.summary_level = 'SUMMARY' - THEN 'Space saved from cleanup: ' + FORMAT(irs.space_saved_gb, 'N4') + ' GB' + - ' | Compression savings: ' + FORMAT(irs.compression_min_savings_gb, 'N4') + ' - ' + - FORMAT(irs.compression_max_savings_gb, 'N4') + ' GB' + - ' | Total savings: ' + FORMAT(irs.total_min_savings_gb, 'N4') + ' - ' + - FORMAT(irs.total_max_savings_gb, 'N4') + ' GB' - ELSE FORMAT(irs.total_size_gb, 'N4') - END AS total_size_gb, + THEN FORMAT(100.0 * (irs.indexes_to_disable + irs.indexes_to_merge) / NULLIF(irs.index_count, 0), 'N1') + '%' + WHEN irs.index_count > 0 + THEN FORMAT(100.0 * irs.unused_indexes / NULLIF(irs.index_count, 0), 'N1') + '%' + ELSE '0.0%' + END AS pct_removable, - /* Skip other metrics for summary level */ - CASE WHEN irs.summary_level = 'SUMMARY' THEN NULL ELSE FORMAT(irs.total_rows, 'N0') END AS total_rows, - CASE WHEN irs.summary_level = 'SUMMARY' THEN NULL ELSE FORMAT(irs.unused_indexes, 'N0') END AS unused_indexes, - CASE WHEN irs.summary_level = 'SUMMARY' THEN NULL ELSE FORMAT(irs.unused_size_gb, 'N4') END AS unused_size_gb, + /* ===== Section 2: Size and Space Savings ===== */ + /* Current size in GB */ + CASE + WHEN irs.summary_level <> 'SUMMARY' + THEN FORMAT(irs.total_size_gb, 'N2') + ELSE NULL + END AS current_size_gb, - /* Usage metrics */ - CASE WHEN irs.summary_level = 'SUMMARY' THEN NULL ELSE FORMAT(irs.total_reads, 'N0') END AS total_reads, - CASE WHEN irs.summary_level = 'SUMMARY' THEN NULL ELSE FORMAT(irs.total_writes, 'N0') END AS total_writes, - CASE WHEN irs.summary_level = 'SUMMARY' THEN NULL ELSE FORMAT(irs.user_seeks, 'N0') END AS user_seeks, - CASE WHEN irs.summary_level = 'SUMMARY' THEN NULL ELSE FORMAT(irs.user_scans, 'N0') END AS user_scans, - CASE WHEN irs.summary_level = 'SUMMARY' THEN NULL ELSE FORMAT(irs.user_lookups, 'N0') END AS user_lookups, - CASE WHEN irs.summary_level = 'SUMMARY' THEN NULL ELSE FORMAT(irs.user_updates, 'N0') END AS user_updates, + /* Size that can be saved through cleanup */ + CASE + WHEN irs.summary_level = 'SUMMARY' THEN FORMAT(irs.space_saved_gb, 'N2') + ELSE FORMAT(irs.unused_size_gb, 'N2') + END AS cleanup_savings_gb, - /* Operational metrics */ - CASE WHEN irs.summary_level = 'SUMMARY' THEN NULL ELSE FORMAT(irs.range_scan_count, 'N0') END AS range_scan_count, - CASE WHEN irs.summary_level = 'SUMMARY' THEN NULL ELSE FORMAT(irs.singleton_lookup_count, 'N0') END AS singleton_lookup_count, + /* Potential additional savings from compression (summary only) */ + CASE + WHEN irs.summary_level = 'SUMMARY' + THEN FORMAT(irs.total_min_savings_gb, 'N2') + ' - ' + FORMAT(irs.total_max_savings_gb, 'N2') + ELSE NULL + END AS potential_savings_gb, - /* Lock wait percentages */ + /* ===== Section 3: Table and Usage Statistics ===== */ + /* Row count */ CASE - WHEN irs.summary_level = 'SUMMARY' THEN NULL - WHEN irs.row_lock_count > 0 - THEN FORMAT(100.0 * irs.row_lock_wait_count / NULLIF(irs.row_lock_count, 0), 'N2') + '%' - ELSE '0.00%' - END AS row_lock_wait_pct, + WHEN irs.summary_level <> 'SUMMARY' + THEN FORMAT(irs.total_rows, 'N0') + ELSE NULL + END AS total_rows, + /* Total reads - combined total and breakdown */ CASE - WHEN irs.summary_level = 'SUMMARY' THEN NULL - WHEN irs.row_lock_wait_count > 0 - THEN FORMAT(1.0 * irs.row_lock_wait_in_ms / NULLIF(irs.row_lock_wait_count, 0), 'N2') - ELSE '0.00' - END AS row_lock_wait_ms_avg, + WHEN irs.summary_level <> 'SUMMARY' + THEN FORMAT(irs.total_reads, 'N0') + + ' (' + + FORMAT(irs.user_seeks, 'N0') + ' seeks, ' + + FORMAT(irs.user_scans, 'N0') + ' scans, ' + + FORMAT(irs.user_lookups, 'N0') + ' lookups)' + ELSE NULL + END AS reads_breakdown, + /* Total writes */ CASE - WHEN irs.summary_level = 'SUMMARY' THEN NULL - WHEN irs.page_lock_count > 0 - THEN FORMAT(100.0 * irs.page_lock_wait_count / NULLIF(irs.page_lock_count, 0), 'N2') + '%' - ELSE '0.00%' - END AS page_lock_wait_pct, + WHEN irs.summary_level <> 'SUMMARY' + THEN FORMAT(irs.total_writes, 'N0') + ELSE NULL + END AS writes, + /* ===== Section 4: Consolidated Performance Metrics ===== */ + /* Combined lock wait percentage across row and page locks */ CASE - WHEN irs.summary_level = 'SUMMARY' THEN NULL - WHEN irs.page_lock_wait_count > 0 - THEN FORMAT(1.0 * irs.page_lock_wait_in_ms / NULLIF(irs.page_lock_wait_count, 0), 'N2') - ELSE '0.00' - END AS page_lock_wait_ms_avg, + WHEN irs.summary_level <> 'SUMMARY' AND (irs.row_lock_count + irs.page_lock_count) > 0 + THEN FORMAT(100.0 * (irs.row_lock_wait_count + irs.page_lock_wait_count) / + NULLIF(irs.row_lock_count + irs.page_lock_count, 0), 'N2') + '%' + ELSE NULL + END AS lock_waits, - /* Latch wait averages */ + /* Average lock wait time in ms */ CASE - WHEN irs.summary_level = 'SUMMARY' THEN NULL - WHEN irs.page_latch_wait_count > 0 - THEN FORMAT(1.0 * irs.page_latch_wait_in_ms / NULLIF(irs.page_latch_wait_count, 0), 'N2') - ELSE '0.00' - END AS page_latch_wait_ms_avg, + WHEN irs.summary_level <> 'SUMMARY' AND (irs.row_lock_wait_count + irs.page_lock_wait_count) > 0 + THEN FORMAT(1.0 * (irs.row_lock_wait_in_ms + irs.page_lock_wait_in_ms) / + NULLIF(irs.row_lock_wait_count + irs.page_lock_wait_count, 0), 'N2') + ELSE NULL + END AS avg_lock_wait_ms, + /* Combined latch wait time in ms */ CASE - WHEN irs.summary_level = 'SUMMARY' THEN NULL - WHEN irs.page_io_latch_wait_count > 0 - THEN FORMAT(1.0 * irs.page_io_latch_wait_in_ms / NULLIF(irs.page_io_latch_wait_count, 0), 'N2') - ELSE '0.00' - END AS page_io_latch_wait_ms_avg, - - /* DML Counts */ - CASE WHEN irs.summary_level = 'SUMMARY' THEN NULL ELSE FORMAT(irs.forwarded_fetch_count, 'N0') END AS forwarded_fetch_count, - CASE WHEN irs.summary_level = 'SUMMARY' THEN NULL ELSE FORMAT(irs.leaf_insert_count, 'N0') END AS leaf_inserts, - CASE WHEN irs.summary_level = 'SUMMARY' THEN NULL ELSE FORMAT(irs.leaf_update_count, 'N0') END AS leaf_updates, - CASE WHEN irs.summary_level = 'SUMMARY' THEN NULL ELSE FORMAT(irs.leaf_delete_count, 'N0') END AS leaf_deletes + WHEN irs.summary_level <> 'SUMMARY' AND (irs.page_latch_wait_count + irs.page_io_latch_wait_count) > 0 + THEN FORMAT(1.0 * (irs.page_latch_wait_in_ms + irs.page_io_latch_wait_in_ms) / + NULLIF(irs.page_latch_wait_count + irs.page_io_latch_wait_count, 0), 'N2') + ELSE NULL + END AS avg_latch_wait_ms FROM #index_reporting_stats AS irs WHERE irs.summary_level IN ('SUMMARY', 'DATABASE', 'TABLE') /* Filter out INDEX level */ ORDER BY @@ -3346,10 +3358,14 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END, /* Then by database name */ irs.database_name, - /* For tables, sort by size */ + /* For tables, sort by potential savings and size */ + CASE + WHEN irs.summary_level = 'TABLE' THEN irs.unused_size_gb + ELSE 0 + END DESC, CASE - WHEN irs.summary_level IN ('SUMMARY', 'DATABASE') THEN 0 - ELSE ISNULL(irs.total_size_gb, 0) + WHEN irs.summary_level = 'TABLE' THEN irs.total_size_gb + ELSE 0 END DESC, /* Then by schema, table */ irs.schema_name, From 98a30d295459ec71a0abed6461d7611e93fddac6 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 13 Mar 2025 11:40:57 -0400 Subject: [PATCH 061/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 85 +++++++++++++++---- 1 file changed, 70 insertions(+), 15 deletions(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index dde1fb93..c660d26e 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -2921,7 +2921,12 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ps.database_name, index_count = COUNT(DISTINCT CONCAT(ps.object_id, N'.', ps.index_id)), total_size_gb = SUM(ps.total_space_gb), - total_rows = SUM(ps.total_rows), + /* Sum the rows from each table's clustered index or heap */ + total_rows = SUM(CASE + WHEN ps.index_id IN (0, 1) /* Only count heap or clustered index */ + THEN ps.total_rows + ELSE 0 + END), unused_indexes = SUM(CASE WHEN id.user_seeks + id.user_scans + id.user_lookups = 0 THEN 1 ELSE 0 END), unused_size_gb = SUM(CASE WHEN id.user_seeks + id.user_scans + id.user_lookups = 0 THEN ps.total_space_gb ELSE 0 END), total_reads = SUM(id.user_seeks + id.user_scans + id.user_lookups), @@ -3002,7 +3007,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ps.table_name, index_count = COUNT(DISTINCT ps.index_id), total_size_gb = SUM(ps.total_space_gb), - total_rows = MAX(CASE WHEN ps.index_id IN (0, 1) THEN ps.total_rows ELSE 0 END), + total_rows = MIN(CASE WHEN ps.index_id IN (0, 1) THEN ps.total_rows ELSE NULL END), unused_indexes = SUM(CASE WHEN id.user_seeks + id.user_scans + id.user_lookups = 0 THEN 1 ELSE 0 END), unused_size_gb = SUM(CASE WHEN id.user_seeks + id.user_scans + id.user_lookups = 0 THEN ps.total_space_gb ELSE 0 END), total_reads = SUM(id.user_seeks + id.user_scans + id.user_lookups), @@ -3233,12 +3238,15 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SELECT /* Basic identification */ CASE + WHEN irs.summary_level = 'HEADER' THEN '=== INDEX ANALYSIS REPORT ===' WHEN irs.summary_level = 'SUMMARY' THEN '=== OVERALL ANALYSIS ===' ELSE irs.summary_level END AS level, /* Server info (for summary) or database name */ CASE + WHEN irs.summary_level = 'HEADER' + THEN 'This report identifies indexes to disable or merge to save space and improve performance' WHEN irs.summary_level = 'SUMMARY' AND irs.uptime_warning = 1 THEN 'WARNING: Server uptime only ' + CONVERT(varchar(10), irs.server_uptime_days) + ' days - usage data may be incomplete!' WHEN irs.summary_level = 'SUMMARY' @@ -3258,18 +3266,31 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END AS tables_analyzed, /* Total indexes */ - FORMAT(irs.index_count, 'N0') AS total_indexes, + CASE + WHEN irs.summary_level = 'HEADER' THEN 'Total indexes' + ELSE FORMAT(irs.index_count, 'N0') + END AS total_indexes, - /* Removable indexes - show different values for summary vs. other levels */ + /* Removable indexes - use same value from #index_analysis at all levels for consistency */ CASE - WHEN irs.summary_level = 'SUMMARY' THEN FORMAT(irs.indexes_to_disable + irs.indexes_to_merge, 'N0') - ELSE FORMAT(irs.unused_indexes, 'N0') + WHEN irs.summary_level = 'HEADER' THEN 'Indexes to disable' + WHEN irs.summary_level = 'SUMMARY' THEN FORMAT(irs.indexes_to_disable, 'N0') + WHEN irs.summary_level <> 'HEADER' THEN FORMAT(irs.unused_indexes, 'N0') + ELSE NULL END AS removable_indexes, + /* Show mergeable indexes as a separate column for clarity (summary level only) */ + CASE + WHEN irs.summary_level = 'HEADER' THEN 'Indexes to merge' + WHEN irs.summary_level = 'SUMMARY' THEN FORMAT(irs.indexes_to_merge, 'N0') + ELSE NULL + END AS mergeable_indexes, + /* Percent of indexes that can be removed */ CASE + WHEN irs.summary_level = 'HEADER' THEN '% to disable' WHEN irs.summary_level = 'SUMMARY' - THEN FORMAT(100.0 * (irs.indexes_to_disable + irs.indexes_to_merge) / NULLIF(irs.index_count, 0), 'N1') + '%' + THEN FORMAT(100.0 * irs.indexes_to_disable / NULLIF(irs.index_count, 0), 'N1') + '%' WHEN irs.index_count > 0 THEN FORMAT(100.0 * irs.unused_indexes / NULLIF(irs.index_count, 0), 'N1') + '%' ELSE '0.0%' @@ -3278,6 +3299,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* ===== Section 2: Size and Space Savings ===== */ /* Current size in GB */ CASE + WHEN irs.summary_level = 'HEADER' THEN 'Current size (GB)' WHEN irs.summary_level <> 'SUMMARY' THEN FORMAT(irs.total_size_gb, 'N2') ELSE NULL @@ -3285,12 +3307,14 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Size that can be saved through cleanup */ CASE + WHEN irs.summary_level = 'HEADER' THEN 'Space saved (GB)' WHEN irs.summary_level = 'SUMMARY' THEN FORMAT(irs.space_saved_gb, 'N2') ELSE FORMAT(irs.unused_size_gb, 'N2') END AS cleanup_savings_gb, /* Potential additional savings from compression (summary only) */ CASE + WHEN irs.summary_level = 'HEADER' THEN 'Total potential savings (GB)' WHEN irs.summary_level = 'SUMMARY' THEN FORMAT(irs.total_min_savings_gb, 'N2') + ' - ' + FORMAT(irs.total_max_savings_gb, 'N2') ELSE NULL @@ -3299,6 +3323,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* ===== Section 3: Table and Usage Statistics ===== */ /* Row count */ CASE + WHEN irs.summary_level = 'HEADER' THEN 'Row count' WHEN irs.summary_level <> 'SUMMARY' THEN FORMAT(irs.total_rows, 'N0') ELSE NULL @@ -3306,6 +3331,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Total reads - combined total and breakdown */ CASE + WHEN irs.summary_level = 'HEADER' THEN 'Read operations' WHEN irs.summary_level <> 'SUMMARY' THEN FORMAT(irs.total_reads, 'N0') + ' (' + @@ -3317,22 +3343,24 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Total writes */ CASE + WHEN irs.summary_level = 'HEADER' THEN 'Write operations' WHEN irs.summary_level <> 'SUMMARY' THEN FORMAT(irs.total_writes, 'N0') ELSE NULL END AS writes, /* ===== Section 4: Consolidated Performance Metrics ===== */ - /* Combined lock wait percentage across row and page locks */ + /* Total count of lock waits (row + page) */ CASE - WHEN irs.summary_level <> 'SUMMARY' AND (irs.row_lock_count + irs.page_lock_count) > 0 - THEN FORMAT(100.0 * (irs.row_lock_wait_count + irs.page_lock_wait_count) / - NULLIF(irs.row_lock_count + irs.page_lock_count, 0), 'N2') + '%' + WHEN irs.summary_level = 'HEADER' THEN 'Lock wait count' + WHEN irs.summary_level <> 'SUMMARY' + THEN FORMAT(irs.row_lock_wait_count + irs.page_lock_wait_count, 'N0') ELSE NULL - END AS lock_waits, + END AS lock_wait_count, /* Average lock wait time in ms */ CASE + WHEN irs.summary_level = 'HEADER' THEN 'Avg lock wait (ms)' WHEN irs.summary_level <> 'SUMMARY' AND (irs.row_lock_wait_count + irs.page_lock_wait_count) > 0 THEN FORMAT(1.0 * (irs.row_lock_wait_in_ms + irs.page_lock_wait_in_ms) / NULLIF(irs.row_lock_wait_count + irs.page_lock_wait_count, 0), 'N2') @@ -3341,16 +3369,43 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Combined latch wait time in ms */ CASE + WHEN irs.summary_level = 'HEADER' THEN 'Avg latch wait (ms)' WHEN irs.summary_level <> 'SUMMARY' AND (irs.page_latch_wait_count + irs.page_io_latch_wait_count) > 0 THEN FORMAT(1.0 * (irs.page_latch_wait_in_ms + irs.page_io_latch_wait_in_ms) / NULLIF(irs.page_latch_wait_count + irs.page_io_latch_wait_count, 0), 'N2') ELSE NULL END AS avg_latch_wait_ms - FROM #index_reporting_stats AS irs - WHERE irs.summary_level IN ('SUMMARY', 'DATABASE', 'TABLE') /* Filter out INDEX level */ + /* Add header row with information about interpretation */ + FROM ( + SELECT 'HEADER' AS summary_level, NULL AS database_name, NULL AS schema_name, + NULL AS table_name, NULL AS server_uptime_days, NULL AS uptime_warning, + NULL AS tables_analyzed, NULL AS index_count, + NULL AS indexes_to_disable, NULL AS indexes_to_merge, + NULL AS avg_indexes_per_table, NULL AS space_saved_gb, NULL AS unused_indexes, + NULL AS unused_size_gb, NULL AS compression_min_savings_gb, + NULL AS compression_max_savings_gb, NULL AS total_min_savings_gb, + NULL AS total_max_savings_gb, NULL AS total_size_gb, + NULL AS total_rows, NULL AS total_reads, NULL AS total_writes, + NULL AS user_seeks, NULL AS user_scans, NULL AS user_lookups, + NULL AS user_updates, NULL AS range_scan_count, + NULL AS singleton_lookup_count, NULL AS row_lock_count, + NULL AS row_lock_wait_count, NULL AS row_lock_wait_in_ms, + NULL AS page_lock_count, NULL AS page_lock_wait_count, + NULL AS page_lock_wait_in_ms, NULL AS page_latch_wait_count, + NULL AS page_latch_wait_in_ms, NULL AS page_io_latch_wait_count, + NULL AS page_io_latch_wait_in_ms, NULL AS forwarded_fetch_count, + NULL AS leaf_insert_count, NULL AS leaf_update_count, + NULL AS leaf_delete_count + + UNION ALL + + SELECT * FROM #index_reporting_stats + WHERE summary_level IN ('SUMMARY', 'DATABASE', 'TABLE') /* Filter out INDEX level */ + ) AS irs ORDER BY - /* Order by level - put summary first */ + /* Order by level - header first, summary second */ CASE + WHEN irs.summary_level = 'HEADER' THEN -1 WHEN irs.summary_level = 'SUMMARY' THEN 0 WHEN irs.summary_level = 'DATABASE' THEN 1 WHEN irs.summary_level = 'TABLE' THEN 2 From a9bdc3c7065a98e954026fc545b50e3f30a3cc99 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 13 Mar 2025 12:02:12 -0400 Subject: [PATCH 062/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 55 ++----------------- 1 file changed, 5 insertions(+), 50 deletions(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index c660d26e..8d6fb4b2 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -3238,15 +3238,12 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SELECT /* Basic identification */ CASE - WHEN irs.summary_level = 'HEADER' THEN '=== INDEX ANALYSIS REPORT ===' WHEN irs.summary_level = 'SUMMARY' THEN '=== OVERALL ANALYSIS ===' ELSE irs.summary_level END AS level, /* Server info (for summary) or database name */ CASE - WHEN irs.summary_level = 'HEADER' - THEN 'This report identifies indexes to disable or merge to save space and improve performance' WHEN irs.summary_level = 'SUMMARY' AND irs.uptime_warning = 1 THEN 'WARNING: Server uptime only ' + CONVERT(varchar(10), irs.server_uptime_days) + ' days - usage data may be incomplete!' WHEN irs.summary_level = 'SUMMARY' @@ -3266,29 +3263,22 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END AS tables_analyzed, /* Total indexes */ - CASE - WHEN irs.summary_level = 'HEADER' THEN 'Total indexes' - ELSE FORMAT(irs.index_count, 'N0') - END AS total_indexes, + FORMAT(irs.index_count, 'N0') AS total_indexes, /* Removable indexes - use same value from #index_analysis at all levels for consistency */ CASE - WHEN irs.summary_level = 'HEADER' THEN 'Indexes to disable' WHEN irs.summary_level = 'SUMMARY' THEN FORMAT(irs.indexes_to_disable, 'N0') - WHEN irs.summary_level <> 'HEADER' THEN FORMAT(irs.unused_indexes, 'N0') - ELSE NULL + ELSE FORMAT(irs.unused_indexes, 'N0') END AS removable_indexes, /* Show mergeable indexes as a separate column for clarity (summary level only) */ CASE - WHEN irs.summary_level = 'HEADER' THEN 'Indexes to merge' WHEN irs.summary_level = 'SUMMARY' THEN FORMAT(irs.indexes_to_merge, 'N0') ELSE NULL END AS mergeable_indexes, /* Percent of indexes that can be removed */ CASE - WHEN irs.summary_level = 'HEADER' THEN '% to disable' WHEN irs.summary_level = 'SUMMARY' THEN FORMAT(100.0 * irs.indexes_to_disable / NULLIF(irs.index_count, 0), 'N1') + '%' WHEN irs.index_count > 0 @@ -3299,7 +3289,6 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* ===== Section 2: Size and Space Savings ===== */ /* Current size in GB */ CASE - WHEN irs.summary_level = 'HEADER' THEN 'Current size (GB)' WHEN irs.summary_level <> 'SUMMARY' THEN FORMAT(irs.total_size_gb, 'N2') ELSE NULL @@ -3307,14 +3296,12 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Size that can be saved through cleanup */ CASE - WHEN irs.summary_level = 'HEADER' THEN 'Space saved (GB)' WHEN irs.summary_level = 'SUMMARY' THEN FORMAT(irs.space_saved_gb, 'N2') ELSE FORMAT(irs.unused_size_gb, 'N2') END AS cleanup_savings_gb, /* Potential additional savings from compression (summary only) */ CASE - WHEN irs.summary_level = 'HEADER' THEN 'Total potential savings (GB)' WHEN irs.summary_level = 'SUMMARY' THEN FORMAT(irs.total_min_savings_gb, 'N2') + ' - ' + FORMAT(irs.total_max_savings_gb, 'N2') ELSE NULL @@ -3323,7 +3310,6 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* ===== Section 3: Table and Usage Statistics ===== */ /* Row count */ CASE - WHEN irs.summary_level = 'HEADER' THEN 'Row count' WHEN irs.summary_level <> 'SUMMARY' THEN FORMAT(irs.total_rows, 'N0') ELSE NULL @@ -3331,7 +3317,6 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Total reads - combined total and breakdown */ CASE - WHEN irs.summary_level = 'HEADER' THEN 'Read operations' WHEN irs.summary_level <> 'SUMMARY' THEN FORMAT(irs.total_reads, 'N0') + ' (' + @@ -3343,7 +3328,6 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Total writes */ CASE - WHEN irs.summary_level = 'HEADER' THEN 'Write operations' WHEN irs.summary_level <> 'SUMMARY' THEN FORMAT(irs.total_writes, 'N0') ELSE NULL @@ -3352,7 +3336,6 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* ===== Section 4: Consolidated Performance Metrics ===== */ /* Total count of lock waits (row + page) */ CASE - WHEN irs.summary_level = 'HEADER' THEN 'Lock wait count' WHEN irs.summary_level <> 'SUMMARY' THEN FORMAT(irs.row_lock_wait_count + irs.page_lock_wait_count, 'N0') ELSE NULL @@ -3360,7 +3343,6 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Average lock wait time in ms */ CASE - WHEN irs.summary_level = 'HEADER' THEN 'Avg lock wait (ms)' WHEN irs.summary_level <> 'SUMMARY' AND (irs.row_lock_wait_count + irs.page_lock_wait_count) > 0 THEN FORMAT(1.0 * (irs.row_lock_wait_in_ms + irs.page_lock_wait_in_ms) / NULLIF(irs.row_lock_wait_count + irs.page_lock_wait_count, 0), 'N2') @@ -3369,43 +3351,16 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Combined latch wait time in ms */ CASE - WHEN irs.summary_level = 'HEADER' THEN 'Avg latch wait (ms)' WHEN irs.summary_level <> 'SUMMARY' AND (irs.page_latch_wait_count + irs.page_io_latch_wait_count) > 0 THEN FORMAT(1.0 * (irs.page_latch_wait_in_ms + irs.page_io_latch_wait_in_ms) / NULLIF(irs.page_latch_wait_count + irs.page_io_latch_wait_count, 0), 'N2') ELSE NULL END AS avg_latch_wait_ms - /* Add header row with information about interpretation */ - FROM ( - SELECT 'HEADER' AS summary_level, NULL AS database_name, NULL AS schema_name, - NULL AS table_name, NULL AS server_uptime_days, NULL AS uptime_warning, - NULL AS tables_analyzed, NULL AS index_count, - NULL AS indexes_to_disable, NULL AS indexes_to_merge, - NULL AS avg_indexes_per_table, NULL AS space_saved_gb, NULL AS unused_indexes, - NULL AS unused_size_gb, NULL AS compression_min_savings_gb, - NULL AS compression_max_savings_gb, NULL AS total_min_savings_gb, - NULL AS total_max_savings_gb, NULL AS total_size_gb, - NULL AS total_rows, NULL AS total_reads, NULL AS total_writes, - NULL AS user_seeks, NULL AS user_scans, NULL AS user_lookups, - NULL AS user_updates, NULL AS range_scan_count, - NULL AS singleton_lookup_count, NULL AS row_lock_count, - NULL AS row_lock_wait_count, NULL AS row_lock_wait_in_ms, - NULL AS page_lock_count, NULL AS page_lock_wait_count, - NULL AS page_lock_wait_in_ms, NULL AS page_latch_wait_count, - NULL AS page_latch_wait_in_ms, NULL AS page_io_latch_wait_count, - NULL AS page_io_latch_wait_in_ms, NULL AS forwarded_fetch_count, - NULL AS leaf_insert_count, NULL AS leaf_update_count, - NULL AS leaf_delete_count - - UNION ALL - - SELECT * FROM #index_reporting_stats - WHERE summary_level IN ('SUMMARY', 'DATABASE', 'TABLE') /* Filter out INDEX level */ - ) AS irs + FROM #index_reporting_stats AS irs + WHERE irs.summary_level IN ('SUMMARY', 'DATABASE', 'TABLE') /* Filter out INDEX level */ ORDER BY - /* Order by level - header first, summary second */ + /* Order by level - summary first */ CASE - WHEN irs.summary_level = 'HEADER' THEN -1 WHEN irs.summary_level = 'SUMMARY' THEN 0 WHEN irs.summary_level = 'DATABASE' THEN 1 WHEN irs.summary_level = 'TABLE' THEN 2 From 6ca38980efee1bdaa5303799269373c479ca716c Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 13 Mar 2025 12:23:52 -0400 Subject: [PATCH 063/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 32 +++++++++++++------ 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index 8d6fb4b2..d31526cc 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -2359,7 +2359,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. index_reads, index_writes ) - SELECT + SELECT DISTINCT result_type = 'DISABLE', /* Sort duplicate/subset indexes first (20), then unused indexes last (25) */ sort_order = @@ -3007,7 +3007,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ps.table_name, index_count = COUNT(DISTINCT ps.index_id), total_size_gb = SUM(ps.total_space_gb), - total_rows = MIN(CASE WHEN ps.index_id IN (0, 1) THEN ps.total_rows ELSE NULL END), + /* Only count rows once per table by using min with index_id=1 (clustered) or 0 (heap) */ + total_rows = SUM(CASE WHEN ps.index_id = 1 OR (ps.index_id = 0 AND NOT EXISTS ( + SELECT 1 FROM #partition_stats ps2 + WHERE ps2.database_id = ps.database_id + AND ps2.object_id = ps.object_id + AND ps2.index_id = 1)) + THEN ps.total_rows ELSE 0 END), unused_indexes = SUM(CASE WHEN id.user_seeks + id.user_scans + id.user_lookups = 0 THEN 1 ELSE 0 END), unused_size_gb = SUM(CASE WHEN id.user_seeks + id.user_scans + id.user_lookups = 0 THEN ps.total_space_gb ELSE 0 END), total_reads = SUM(id.user_seeks + id.user_scans + id.user_lookups), @@ -3265,10 +3271,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Total indexes */ FORMAT(irs.index_count, 'N0') AS total_indexes, - /* Removable indexes - use same value from #index_analysis at all levels for consistency */ + /* Removable indexes - report consistent values across levels */ CASE - WHEN irs.summary_level = 'SUMMARY' THEN FORMAT(irs.indexes_to_disable, 'N0') - ELSE FORMAT(irs.unused_indexes, 'N0') + WHEN irs.summary_level = 'SUMMARY' + THEN FORMAT(irs.indexes_to_disable, 'N0') /* Indexes that will be disabled based on analysis */ + ELSE FORMAT(irs.unused_indexes, 'N0') /* Unused indexes at database/table level */ END AS removable_indexes, /* Show mergeable indexes as a separate column for clarity (summary level only) */ @@ -3287,11 +3294,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END AS pct_removable, /* ===== Section 2: Size and Space Savings ===== */ - /* Current size in GB */ + /* Current size in GB - show at all levels */ CASE - WHEN irs.summary_level <> 'SUMMARY' - THEN FORMAT(irs.total_size_gb, 'N2') - ELSE NULL + WHEN irs.summary_level = 'SUMMARY' + THEN FORMAT(SUM(irs.total_size_gb) OVER(), 'N2') /* Total size across all databases */ + ELSE FORMAT(irs.total_size_gb, 'N2') END AS current_size_gb, /* Size that can be saved through cleanup */ @@ -3300,10 +3307,15 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ELSE FORMAT(irs.unused_size_gb, 'N2') END AS cleanup_savings_gb, - /* Potential additional savings from compression (summary only) */ + /* Potential additional savings (show at all levels for clarity) */ CASE WHEN irs.summary_level = 'SUMMARY' THEN FORMAT(irs.total_min_savings_gb, 'N2') + ' - ' + FORMAT(irs.total_max_savings_gb, 'N2') + /* For database/table levels, show potential savings if table has unused space */ + WHEN irs.total_size_gb > 0 + THEN FORMAT(irs.unused_size_gb + (irs.total_size_gb - irs.unused_size_gb) * 0.2, 'N2') + + ' - ' + + FORMAT(irs.unused_size_gb + (irs.total_size_gb - irs.unused_size_gb) * 0.6, 'N2') ELSE NULL END AS potential_savings_gb, From 7b6c7849704d96371e8db2e2d4ae07fe98ecca7c Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 13 Mar 2025 14:04:01 -0400 Subject: [PATCH 064/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index d31526cc..682666cc 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -3294,10 +3294,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END AS pct_removable, /* ===== Section 2: Size and Space Savings ===== */ - /* Current size in GB - show at all levels */ + /* Current size in GB */ CASE - WHEN irs.summary_level = 'SUMMARY' - THEN FORMAT(SUM(irs.total_size_gb) OVER(), 'N2') /* Total size across all databases */ + WHEN irs.summary_level = 'SUMMARY' THEN FORMAT(irs.total_size_gb, 'N2') ELSE FORMAT(irs.total_size_gb, 'N2') END AS current_size_gb, From 70c1cfc26aa88e5e3a6f8ad71557bf07d0f138a9 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 13 Mar 2025 14:12:31 -0400 Subject: [PATCH 065/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 101 +++++++++++++++++- 1 file changed, 97 insertions(+), 4 deletions(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index 682666cc..dacfb110 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -801,7 +801,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. FROM #filtered_objects AS fo OPTION(RECOMPILE); - RAISERROR('Generaring #compression_eligibility insert', 0, 0) WITH NOWAIT; + RAISERROR('Generating #compression_eligibility insert', 0, 0) WITH NOWAIT; END; /* Populate compression eligibility table */ @@ -850,8 +850,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Check for sparse columns or incompatible data types */ IF @can_compress = 1 BEGIN - RAISERROR('Updating #compression_eligibility', 0, 0) WITH NOWAIT; - + IF @debug = 1 + BEGIN + RAISERROR('Updating #compression_eligibility', 0, 0) WITH NOWAIT; + END; + SELECT @sql = N' SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; @@ -1954,6 +1957,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Create a reference to the detailed summary that will appear at the end */ + IF @debug = 1 + BEGIN + RAISERROR('Generating #index_cleanup_results insert', 0, 0) WITH NOWAIT; + END; + INSERT INTO #index_cleanup_results ( @@ -1971,6 +1979,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Identify key duplicates where both indexes have MERGE INCLUDES action */ + IF @debug = 1 + BEGIN + RAISERROR('Generating #key_duplicate_dedupe insert', 0, 0) WITH NOWAIT; + END; + INSERT INTO #key_duplicate_dedupe WITH @@ -2053,6 +2066,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. OPTION(RECOMPILE); /* Only groups with multiple MERGE INCLUDES */ /* Update the index_analysis table to make only one index the winner in each group */ + IF @debug = 1 + BEGIN + RAISERROR('Generating #index_analysis updates', 0, 0) WITH NOWAIT; + END; + UPDATE ia SET @@ -2090,6 +2108,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. OPTION(RECOMPILE); /* Find indexes with same key columns where one has includes that are a subset of another */ + IF @debug = 1 + BEGIN + RAISERROR('Generating #include_subset_dedupe insert', 0, 0) WITH NOWAIT; + END; + INSERT INTO #include_subset_dedupe WITH @@ -2136,6 +2159,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. OPTION(RECOMPILE); /* Update the subset indexes to be disabled, since supersets already contain their columns */ + IF @debug = 1 + BEGIN + RAISERROR('Generating #index_analysis updates', 0, 0) WITH NOWAIT; + END; + UPDATE ia SET @@ -2196,6 +2224,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. OPTION(RECOMPILE); /* Insert merge scripts for indexes */ + IF @debug = 1 + BEGIN + RAISERROR('Generating #index_cleanup_results insert, MERGE', 0, 0) WITH NOWAIT; + END; + INSERT INTO #index_cleanup_results ( @@ -2339,6 +2372,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. OPTION(RECOMPILE); /* Insert disable scripts for unneeded indexes */ + IF @debug = 1 + BEGIN + RAISERROR('Generating #index_cleanup_results insert, DISABLE', 0, 0) WITH NOWAIT; + END; + INSERT INTO #index_cleanup_results ( @@ -2418,6 +2456,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. OPTION(RECOMPILE); /* Insert compression scripts for remaining indexes */ + IF @debug = 1 + BEGIN + RAISERROR('Generating #index_cleanup_results insert, COMPRESS', 0, 0) WITH NOWAIT; + END; + INSERT INTO #index_cleanup_results ( @@ -2515,6 +2558,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. OPTION(RECOMPILE); /* Insert disable scripts for unique constraints */ + IF @debug = 1 + BEGIN + RAISERROR('Generating #index_cleanup_results insert, CONSTRAINT', 0, 0) WITH NOWAIT; + END; + INSERT INTO #index_cleanup_results ( @@ -2610,6 +2658,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. OPTION(RECOMPILE); /* Insert per-partition compression scripts */ + IF @debug = 1 + BEGIN + RAISERROR('Generating #index_cleanup_results insert, COMPRESS_PARTITION', 0, 0) WITH NOWAIT; + END; + INSERT INTO #index_cleanup_results ( @@ -2714,6 +2767,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. OPTION(RECOMPILE); /* Insert compression ineligible info */ + IF @debug = 1 + BEGIN + RAISERROR('Generating #index_cleanup_results insert, INELIGIBLE', 0, 0) WITH NOWAIT; + END; + INSERT INTO #index_cleanup_results WITH @@ -2762,6 +2820,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Insert indexes identified for manual review */ + IF @debug = 1 + BEGIN + RAISERROR('Generating #index_cleanup_results insert, REVIEW', 0, 0) WITH NOWAIT; + END; + INSERT INTO #index_cleanup_results WITH @@ -2820,6 +2883,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Insert indexes that are being kept (superset indexes and others) */ + IF @debug = 1 + BEGIN + RAISERROR('Generating #index_cleanup_results insert, KEEP', 0, 0) WITH NOWAIT; + END; + INSERT INTO #index_cleanup_results WITH @@ -2883,6 +2951,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. OPTION(RECOMPILE); /* Insert database-level summaries */ + IF @debug = 1 + BEGIN + RAISERROR('Generating #index_reporting_stats insert, DATABASE', 0, 0) WITH NOWAIT; + END; + INSERT INTO #index_reporting_stats ( @@ -2966,7 +3039,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. OPTION(RECOMPILE); /* Insert table-level summaries */ - INSERT INTO #index_reporting_stats + IF @debug = 1 + BEGIN + RAISERROR('Generating #index_reporting_stats insert, TABLE', 0, 0) WITH NOWAIT; + END; + + INSERT INTO + #index_reporting_stats ( summary_level, database_name, @@ -3071,6 +3150,10 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. Within each category, indexes are sorted by size and impact for better prioritization. */ + IF @debug = 1 + BEGIN + RAISERROR('Generating #index_cleanup_results insert, RESULTS', 0, 0) WITH NOWAIT; + END; SELECT /* First, show the information needed to understand the script */ @@ -3149,6 +3232,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. OPTION(RECOMPILE); /* Insert overall summary information */ + IF @debug = 1 + BEGIN + RAISERROR('Generating #index_reporting_stats insert, SUMMARY', 0, 0) WITH NOWAIT; + END; + INSERT INTO #index_reporting_stats WITH @@ -3241,6 +3329,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. OPTION(RECOMPILE); /* Return streamlined reporting statistics focused on key metrics */ + IF @debug = 1 + BEGIN + RAISERROR('Generating #index_reporting_stats, REPORT', 0, 0) WITH NOWAIT; + END; + SELECT /* Basic identification */ CASE From 0111fa4fb2373a4964e4a5b958ba470aa2c8b125 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 13 Mar 2025 14:13:43 -0400 Subject: [PATCH 066/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index dacfb110..7787d1d7 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -3388,10 +3388,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* ===== Section 2: Size and Space Savings ===== */ /* Current size in GB */ - CASE - WHEN irs.summary_level = 'SUMMARY' THEN FORMAT(irs.total_size_gb, 'N2') - ELSE FORMAT(irs.total_size_gb, 'N2') - END AS current_size_gb, + FORMAT(irs.total_size_gb, 'N2') AS current_size_gb, /* Size that can be saved through cleanup */ CASE @@ -3399,16 +3396,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ELSE FORMAT(irs.unused_size_gb, 'N2') END AS cleanup_savings_gb, - /* Potential additional savings (show at all levels for clarity) */ + /* Potential additional savings */ CASE WHEN irs.summary_level = 'SUMMARY' THEN FORMAT(irs.total_min_savings_gb, 'N2') + ' - ' + FORMAT(irs.total_max_savings_gb, 'N2') - /* For database/table levels, show potential savings if table has unused space */ - WHEN irs.total_size_gb > 0 - THEN FORMAT(irs.unused_size_gb + (irs.total_size_gb - irs.unused_size_gb) * 0.2, 'N2') + - ' - ' + - FORMAT(irs.unused_size_gb + (irs.total_size_gb - irs.unused_size_gb) * 0.6, 'N2') - ELSE NULL + ELSE NULL /* Only show at summary level to avoid confusion */ END AS potential_savings_gb, /* ===== Section 3: Table and Usage Statistics ===== */ From e6bf71723a9c80a1bf73b7a0d0c8cdc029adbf93 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 13 Mar 2025 14:19:12 -0400 Subject: [PATCH 067/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index 7787d1d7..dbc71a9d 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -3238,7 +3238,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END; INSERT INTO - #index_reporting_stats + RAISERROR('Generating #index_reporting_stats insert, TABLE', 0, 0) WITH NOWAIT; WITH (TABLOCK) ( From 174d204a3f124fab5a631ba221f832dd769e0800 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 13 Mar 2025 14:19:37 -0400 Subject: [PATCH 068/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index dbc71a9d..7787d1d7 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -3238,7 +3238,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END; INSERT INTO - RAISERROR('Generating #index_reporting_stats insert, TABLE', 0, 0) WITH NOWAIT; + #index_reporting_stats WITH (TABLOCK) ( From c10149d0e24f29ca3c1e2c5ee085e6cb16c6985a Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 13 Mar 2025 14:20:27 -0400 Subject: [PATCH 069/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 40 +++++++++++++------ 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index 7787d1d7..55c50055 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -2994,12 +2994,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ps.database_name, index_count = COUNT(DISTINCT CONCAT(ps.object_id, N'.', ps.index_id)), total_size_gb = SUM(ps.total_space_gb), - /* Sum the rows from each table's clustered index or heap */ - total_rows = SUM(CASE - WHEN ps.index_id IN (0, 1) /* Only count heap or clustered index */ - THEN ps.total_rows - ELSE 0 - END), + /* Sum the rows from our temporary table to avoid double-counting */ + total_rows = (SELECT SUM(row_count) FROM #temp_table_rows + WHERE database_id = ps.database_id), unused_indexes = SUM(CASE WHEN id.user_seeks + id.user_scans + id.user_lookups = 0 THEN 1 ELSE 0 END), unused_size_gb = SUM(CASE WHEN id.user_seeks + id.user_scans + id.user_lookups = 0 THEN ps.total_space_gb ELSE 0 END), total_reads = SUM(id.user_seeks + id.user_scans + id.user_lookups), @@ -3044,6 +3041,25 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. RAISERROR('Generating #index_reporting_stats insert, TABLE', 0, 0) WITH NOWAIT; END; + /* Use a temporary table to get accurate row counts */ + SELECT + database_id, + database_name, + object_id, + schema_id, + schema_name, + table_name, + row_count = MAX(CASE WHEN index_id IN (0, 1) THEN total_rows ELSE 0 END) + INTO #temp_table_rows + FROM #partition_stats + GROUP BY + database_id, + database_name, + object_id, + schema_id, + schema_name, + table_name; + INSERT INTO #index_reporting_stats ( @@ -3086,13 +3102,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ps.table_name, index_count = COUNT(DISTINCT ps.index_id), total_size_gb = SUM(ps.total_space_gb), - /* Only count rows once per table by using min with index_id=1 (clustered) or 0 (heap) */ - total_rows = SUM(CASE WHEN ps.index_id = 1 OR (ps.index_id = 0 AND NOT EXISTS ( - SELECT 1 FROM #partition_stats ps2 - WHERE ps2.database_id = ps.database_id - AND ps2.object_id = ps.object_id - AND ps2.index_id = 1)) - THEN ps.total_rows ELSE 0 END), + /* Get accurate row count from our temporary table */ + total_rows = MAX(tr.row_count), unused_indexes = SUM(CASE WHEN id.user_seeks + id.user_scans + id.user_lookups = 0 THEN 1 ELSE 0 END), unused_size_gb = SUM(CASE WHEN id.user_seeks + id.user_scans + id.user_lookups = 0 THEN ps.total_space_gb ELSE 0 END), total_reads = SUM(id.user_seeks + id.user_scans + id.user_lookups), @@ -3128,6 +3139,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ON os.database_id = ps.database_id AND os.object_id = ps.object_id AND os.index_id = ps.index_id + LEFT JOIN #temp_table_rows tr + ON tr.database_id = ps.database_id + AND tr.object_id = ps.object_id GROUP BY ps.database_name, ps.schema_name, ps.table_name OPTION(RECOMPILE); From 5c74dfe082b125f64664aa615bf05ac20e411016 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 13 Mar 2025 14:23:00 -0400 Subject: [PATCH 070/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 32 ++++--------------- 1 file changed, 6 insertions(+), 26 deletions(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index 55c50055..a7ea9a22 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -2994,9 +2994,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ps.database_name, index_count = COUNT(DISTINCT CONCAT(ps.object_id, N'.', ps.index_id)), total_size_gb = SUM(ps.total_space_gb), - /* Sum the rows from our temporary table to avoid double-counting */ - total_rows = (SELECT SUM(row_count) FROM #temp_table_rows - WHERE database_id = ps.database_id), + /* Use a simple aggregation to avoid double-counting */ + total_rows = SUM(DISTINCT CASE WHEN ps.index_id IN (0, 1) THEN ps.total_rows ELSE 0 END), unused_indexes = SUM(CASE WHEN id.user_seeks + id.user_scans + id.user_lookups = 0 THEN 1 ELSE 0 END), unused_size_gb = SUM(CASE WHEN id.user_seeks + id.user_scans + id.user_lookups = 0 THEN ps.total_space_gb ELSE 0 END), total_reads = SUM(id.user_seeks + id.user_scans + id.user_lookups), @@ -3041,24 +3040,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. RAISERROR('Generating #index_reporting_stats insert, TABLE', 0, 0) WITH NOWAIT; END; - /* Use a temporary table to get accurate row counts */ - SELECT - database_id, - database_name, - object_id, - schema_id, - schema_name, - table_name, - row_count = MAX(CASE WHEN index_id IN (0, 1) THEN total_rows ELSE 0 END) - INTO #temp_table_rows - FROM #partition_stats - GROUP BY - database_id, - database_name, - object_id, - schema_id, - schema_name, - table_name; + /* No need for a temporary table - we'll use a simpler approach */ INSERT INTO #index_reporting_stats @@ -3102,8 +3084,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ps.table_name, index_count = COUNT(DISTINCT ps.index_id), total_size_gb = SUM(ps.total_space_gb), - /* Get accurate row count from our temporary table */ - total_rows = MAX(tr.row_count), + /* Use MAX to get the row count from the clustered index or heap */ + total_rows = MAX(CASE WHEN ps.index_id IN (0, 1) THEN ps.total_rows ELSE 0 END), unused_indexes = SUM(CASE WHEN id.user_seeks + id.user_scans + id.user_lookups = 0 THEN 1 ELSE 0 END), unused_size_gb = SUM(CASE WHEN id.user_seeks + id.user_scans + id.user_lookups = 0 THEN ps.total_space_gb ELSE 0 END), total_reads = SUM(id.user_seeks + id.user_scans + id.user_lookups), @@ -3139,9 +3121,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ON os.database_id = ps.database_id AND os.object_id = ps.object_id AND os.index_id = ps.index_id - LEFT JOIN #temp_table_rows tr - ON tr.database_id = ps.database_id - AND tr.object_id = ps.object_id + /* No need for the temporary table join */ GROUP BY ps.database_name, ps.schema_name, ps.table_name OPTION(RECOMPILE); From 2ade64bdd03e671ce8a17ea8945f5fb74e9307aa Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 13 Mar 2025 15:08:49 -0400 Subject: [PATCH 071/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 55 +++++++++++++++---- 1 file changed, 43 insertions(+), 12 deletions(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index a7ea9a22..aec03994 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -2995,8 +2995,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. index_count = COUNT(DISTINCT CONCAT(ps.object_id, N'.', ps.index_id)), total_size_gb = SUM(ps.total_space_gb), /* Use a simple aggregation to avoid double-counting */ - total_rows = SUM(DISTINCT CASE WHEN ps.index_id IN (0, 1) THEN ps.total_rows ELSE 0 END), - unused_indexes = SUM(CASE WHEN id.user_seeks + id.user_scans + id.user_lookups = 0 THEN 1 ELSE 0 END), + /* Get actual row count by grabbing the real row count from clustered index/heap per table */ + total_rows = SUM(d.actual_rows), + indexes_to_merge = (SELECT COUNT(*) FROM #index_analysis ia WHERE ia.action IN ('MERGE INCLUDES', 'MAKE UNIQUE') AND ia.database_name = ps.database_name), + /* Use count from analysis to keep consistent with SUMMARY level */ + unused_indexes = (SELECT COUNT(*) FROM #index_analysis ia WHERE ia.action = 'DISABLE' AND ia.database_name = ps.database_name), unused_size_gb = SUM(CASE WHEN id.user_seeks + id.user_scans + id.user_lookups = 0 THEN ps.total_space_gb ELSE 0 END), total_reads = SUM(id.user_seeks + id.user_scans + id.user_lookups), total_writes = SUM(id.user_updates), @@ -3031,6 +3034,16 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ON os.database_id = ps.database_id AND os.object_id = ps.object_id AND os.index_id = ps.index_id + OUTER APPLY ( + /* Get actual row count per table using MAX from clustered index/heap */ + SELECT + actual_rows = MAX(CASE WHEN ps2.index_id IN (0, 1) THEN ps2.total_rows ELSE 0 END) + FROM #partition_stats ps2 + WHERE ps2.database_id = ps.database_id + AND ps2.object_id = ps.object_id + AND ps2.index_id IN (0, 1) + GROUP BY ps2.object_id + ) d GROUP BY ps.database_name OPTION(RECOMPILE); @@ -3086,7 +3099,17 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. total_size_gb = SUM(ps.total_space_gb), /* Use MAX to get the row count from the clustered index or heap */ total_rows = MAX(CASE WHEN ps.index_id IN (0, 1) THEN ps.total_rows ELSE 0 END), - unused_indexes = SUM(CASE WHEN id.user_seeks + id.user_scans + id.user_lookups = 0 THEN 1 ELSE 0 END), + indexes_to_merge = (SELECT COUNT(*) FROM #index_analysis ia + WHERE ia.action IN ('MERGE INCLUDES', 'MAKE UNIQUE') + AND ia.database_name = ps.database_name + AND ia.schema_name = ps.schema_name + AND ia.table_name = ps.table_name), + /* Use count from analysis to keep consistent with SUMMARY level */ + unused_indexes = (SELECT COUNT(*) FROM #index_analysis ia + WHERE ia.action = 'DISABLE' + AND ia.database_name = ps.database_name + AND ia.schema_name = ps.schema_name + AND ia.table_name = ps.table_name), unused_size_gb = SUM(CASE WHEN id.user_seeks + id.user_scans + id.user_lookups = 0 THEN ps.total_space_gb ELSE 0 END), total_reads = SUM(id.user_seeks + id.user_scans + id.user_lookups), total_writes = SUM(id.user_updates), @@ -3310,7 +3333,19 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND ce.can_compress = 1 THEN ps.total_space_gb * 0.60 ELSE 0 - END) + END), + /* Get total rows from database unique tables */ + total_rows = ( + SELECT SUM(t.row_count) + FROM ( + SELECT + ps_distinct.object_id, + row_count = MAX(CASE WHEN ps_distinct.index_id IN (0, 1) THEN ps_distinct.total_rows ELSE 0 END) + FROM #partition_stats ps_distinct + WHERE ps_distinct.index_id IN (0, 1) + GROUP BY ps_distinct.object_id + ) t + ) FROM #index_analysis AS ia LEFT JOIN #partition_stats AS ps ON ia.database_id = ps.database_id @@ -3365,10 +3400,10 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ELSE FORMAT(irs.unused_indexes, 'N0') /* Unused indexes at database/table level */ END AS removable_indexes, - /* Show mergeable indexes as a separate column for clarity (summary level only) */ + /* Show mergeable indexes across all levels */ CASE WHEN irs.summary_level = 'SUMMARY' THEN FORMAT(irs.indexes_to_merge, 'N0') - ELSE NULL + ELSE FORMAT(irs.indexes_to_merge, 'N0') END AS mergeable_indexes, /* Percent of indexes that can be removed */ @@ -3394,16 +3429,12 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CASE WHEN irs.summary_level = 'SUMMARY' THEN FORMAT(irs.total_min_savings_gb, 'N2') + ' - ' + FORMAT(irs.total_max_savings_gb, 'N2') - ELSE NULL /* Only show at summary level to avoid confusion */ + ELSE FORMAT(irs.unused_size_gb, 'N2') /* Show at all levels */ END AS potential_savings_gb, /* ===== Section 3: Table and Usage Statistics ===== */ /* Row count */ - CASE - WHEN irs.summary_level <> 'SUMMARY' - THEN FORMAT(irs.total_rows, 'N0') - ELSE NULL - END AS total_rows, + FORMAT(irs.total_rows, 'N0') AS total_rows, /* Total reads - combined total and breakdown */ CASE From 9bf93cf80d574f38397ca76483a19b99f0429575 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 13 Mar 2025 16:12:45 -0400 Subject: [PATCH 072/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 448 ++++++++++++------ 1 file changed, 294 insertions(+), 154 deletions(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index aec03994..56f2751c 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -451,7 +451,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. built_on sysname NULL, partition_function_name sysname NULL, partition_columns nvarchar(max) - PRIMARY KEY CLUSTERED(database_id, object_id, index_id, partition_id) + PRIMARY KEY CLUSTERED(database_id, schema_id, object_id, index_id, partition_id) ); CREATE TABLE @@ -881,6 +881,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ) OPTION(RECOMPILE); '; + + IF @debug = 1 + BEGIN + PRINT @sql; + END; EXECUTE sys.sp_executesql @sql; @@ -1298,7 +1303,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. pc.partition_columns FROM ( - SELECT + SELECT DISTINCT ps.object_id, ps.index_id, s.schema_id, @@ -1307,7 +1312,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. index_name = i.name, ps.partition_id, p.partition_number, - total_rows = SUM(ps.row_count), + total_rows = ps.row_count, total_space_gb = SUM(a.total_pages) * 8 / 1024.0 / 1024.0, /* Convert directly to GB */ reserved_lob_gb = SUM(ps.lob_reserved_page_count) * 8. / 1024. / 1024.0, /* Convert directly to GB */ reserved_row_overflow_gb = SUM(ps.row_overflow_reserved_page_count) * 8. / 1024. / 1024.0, /* Convert directly to GB */ @@ -1345,16 +1350,17 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SELECT @sql += N' GROUP BY - t.name, - i.name, - i.data_space_id, + ps.object_id, + ps.index_id, s.schema_id, s.name, + t.name, + i.name, + ps.partition_id, p.partition_number, + ps.row_count, p.data_compression_desc, - ps.object_id, - ps.index_id, - ps.partition_id + i.data_space_id ) AS x OUTER APPLY ( @@ -1405,7 +1411,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. IF @debug = 1 BEGIN - PRINT @sql; + PRINT SUBSTRING(@sql, 1, 4000); + PRINT SUBSTRING(@sql, 4000, 8000); END; INSERT @@ -2964,6 +2971,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. index_count, total_size_gb, total_rows, + indexes_to_merge, unused_indexes, unused_size_gb, total_reads, @@ -2990,17 +2998,41 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. leaf_delete_count ) SELECT - summary_level = 'DATABASE', + summary_level = + 'DATABASE', ps.database_name, - index_count = COUNT(DISTINCT CONCAT(ps.object_id, N'.', ps.index_id)), + index_count = + COUNT_BIG(DISTINCT CONCAT(ps.object_id, N'.', ps.index_id)), total_size_gb = SUM(ps.total_space_gb), /* Use a simple aggregation to avoid double-counting */ /* Get actual row count by grabbing the real row count from clustered index/heap per table */ total_rows = SUM(d.actual_rows), - indexes_to_merge = (SELECT COUNT(*) FROM #index_analysis ia WHERE ia.action IN ('MERGE INCLUDES', 'MAKE UNIQUE') AND ia.database_name = ps.database_name), + indexes_to_merge = + ( + SELECT + COUNT_BIG(*) + FROM #index_analysis ia + WHERE ia.action IN (N'MERGE INCLUDES', N'MAKE UNIQUE') + AND ia.database_name = ps.database_name + ), /* Use count from analysis to keep consistent with SUMMARY level */ - unused_indexes = (SELECT COUNT(*) FROM #index_analysis ia WHERE ia.action = 'DISABLE' AND ia.database_name = ps.database_name), - unused_size_gb = SUM(CASE WHEN id.user_seeks + id.user_scans + id.user_lookups = 0 THEN ps.total_space_gb ELSE 0 END), + unused_indexes = + ( + SELECT + COUNT_BIG(*) + FROM #index_analysis ia + WHERE ia.action = N'DISABLE' + AND ia.database_name = ps.database_name + ), + unused_size_gb = + SUM + ( + CASE + WHEN id.user_seeks + id.user_scans + id.user_lookups = 0 + THEN ps.total_space_gb + ELSE 0 + END + ), total_reads = SUM(id.user_seeks + id.user_scans + id.user_lookups), total_writes = SUM(id.user_updates), user_seeks = SUM(id.user_seeks), @@ -3034,17 +3066,28 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ON os.database_id = ps.database_id AND os.object_id = ps.object_id AND os.index_id = ps.index_id - OUTER APPLY ( + OUTER APPLY + ( /* Get actual row count per table using MAX from clustered index/heap */ SELECT - actual_rows = MAX(CASE WHEN ps2.index_id IN (0, 1) THEN ps2.total_rows ELSE 0 END) - FROM #partition_stats ps2 + actual_rows = + MAX + ( + CASE + WHEN ps2.index_id IN (0, 1) + THEN ps2.total_rows + ELSE 0 + END + ) + FROM #partition_stats AS ps2 WHERE ps2.database_id = ps.database_id - AND ps2.object_id = ps.object_id - AND ps2.index_id IN (0, 1) - GROUP BY ps2.object_id - ) d - GROUP BY ps.database_name + AND ps2.object_id = ps.object_id + AND ps2.index_id IN (0, 1) + GROUP BY + ps2.object_id + ) AS d + GROUP BY + ps.database_name OPTION(RECOMPILE); /* Insert table-level summaries */ @@ -3065,6 +3108,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. index_count, total_size_gb, total_rows, + indexes_to_merge, unused_indexes, unused_size_gb, total_reads, @@ -3095,22 +3139,47 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ps.database_name, ps.schema_name, ps.table_name, - index_count = COUNT(DISTINCT ps.index_id), + index_count = COUNT_BIG(DISTINCT ps.index_id), total_size_gb = SUM(ps.total_space_gb), /* Use MAX to get the row count from the clustered index or heap */ - total_rows = MAX(CASE WHEN ps.index_id IN (0, 1) THEN ps.total_rows ELSE 0 END), - indexes_to_merge = (SELECT COUNT(*) FROM #index_analysis ia - WHERE ia.action IN ('MERGE INCLUDES', 'MAKE UNIQUE') - AND ia.database_name = ps.database_name - AND ia.schema_name = ps.schema_name - AND ia.table_name = ps.table_name), + total_rows = + MAX + ( + CASE + WHEN ps.index_id IN (0, 1) + THEN ps.total_rows + ELSE 0 + END + ), + indexes_to_merge = + ( + SELECT + COUNT_BIG(*) + FROM #index_analysis ia + WHERE ia.action IN (N'MERGE INCLUDES', N'MAKE UNIQUE') + AND ia.database_name = ps.database_name + AND ia.schema_name = ps.schema_name + AND ia.table_name = ps.table_name + ), /* Use count from analysis to keep consistent with SUMMARY level */ - unused_indexes = (SELECT COUNT(*) FROM #index_analysis ia - WHERE ia.action = 'DISABLE' - AND ia.database_name = ps.database_name - AND ia.schema_name = ps.schema_name - AND ia.table_name = ps.table_name), - unused_size_gb = SUM(CASE WHEN id.user_seeks + id.user_scans + id.user_lookups = 0 THEN ps.total_space_gb ELSE 0 END), + unused_indexes = + ( + SELECT + COUNT_BIG(*) + FROM #index_analysis ia + WHERE ia.action = N'DISABLE' + AND ia.database_name = ps.database_name + AND ia.schema_name = ps.schema_name + AND ia.table_name = ps.table_name + ), + unused_size_gb = + SUM + ( + CASE + WHEN id.user_seeks + id.user_scans + id.user_lookups = 0 + THEN ps.total_space_gb + ELSE 0 END + ), total_reads = SUM(id.user_seeks + id.user_scans + id.user_lookups), total_writes = SUM(id.user_updates), user_seeks = SUM(id.user_seeks), @@ -3144,8 +3213,10 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ON os.database_id = ps.database_id AND os.object_id = ps.object_id AND os.index_id = ps.index_id - /* No need for the temporary table join */ - GROUP BY ps.database_name, ps.schema_name, ps.table_name + GROUP BY + ps.database_name, + ps.schema_name, + ps.table_name OPTION(RECOMPILE); /* We're not doing index-level summaries - focusing on database and table level reports */ @@ -3271,7 +3342,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. compression_min_savings_gb, compression_max_savings_gb, total_min_savings_gb, - total_max_savings_gb + total_max_savings_gb, + total_rows ) SELECT summary_level = 'SUMMARY', @@ -3282,69 +3354,111 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. index_count = COUNT_BIG(*), indexes_to_disable = - SUM(CASE WHEN ia.action = 'DISABLE' THEN 1 ELSE 0 END), + SUM + ( + CASE + WHEN ia.action = N'DISABLE' + THEN 1 + ELSE 0 + END + ), indexes_to_merge = - SUM(CASE WHEN ia.action IN ('MERGE INCLUDES', 'MAKE UNIQUE') THEN 1 ELSE 0 END), + SUM + ( + CASE + WHEN ia.action IN (N'MERGE INCLUDES', N'MAKE UNIQUE') + THEN 1 + ELSE 0 + END + ), avg_indexes_per_table = COUNT_BIG(*) * 1.0 / - NULLIF(COUNT_BIG(DISTINCT CONCAT(ia.database_id, N'.', ia.schema_id, N'.', ia.object_id)), 0), + NULLIF + ( + COUNT_BIG(DISTINCT CONCAT(ia.database_id, N'.', ia.schema_id, N'.', ia.object_id)), + 0 + ), /* Space savings from cleanup */ space_saved_gb = SUM ( CASE - WHEN ia.action IN ('DISABLE', 'MERGE INCLUDES', 'MAKE UNIQUE') + WHEN ia.action IN (N'DISABLE', N'MERGE INCLUDES', N'MAKE UNIQUE') THEN ps.total_space_gb ELSE 0 END ), /* Conservative compression savings estimate (20%) */ compression_min_savings_gb = - SUM - ( - CASE - WHEN (ia.action IS NULL OR ia.action = 'KEEP') - AND ce.can_compress = 1 - THEN ps.total_space_gb * 0.20 - ELSE 0 - END - ), + SUM + ( + CASE + WHEN (ia.action IS NULL OR ia.action = N'KEEP') + AND ce.can_compress = 1 + THEN ps.total_space_gb * 0.20 + ELSE 0 + END + ), /* Optimistic compression savings estimate (60%) */ - compression_max_savings_gb = SUM(CASE - WHEN (ia.action IS NULL OR ia.action = 'KEEP') - AND ce.can_compress = 1 - THEN ps.total_space_gb * 0.60 - ELSE 0 - END), + compression_max_savings_gb = + SUM + ( + CASE + WHEN (ia.action IS NULL OR ia.action = N'KEEP') + AND ce.can_compress = 1 + THEN ps.total_space_gb * 0.60 + ELSE 0 + END + ), /* Total conservative savings */ - total_min_savings_gb = SUM(CASE - WHEN ia.action IN ('DISABLE', 'MERGE INCLUDES', 'MAKE UNIQUE') - THEN ps.total_space_gb - WHEN (ia.action IS NULL OR ia.action = 'KEEP') - AND ce.can_compress = 1 - THEN ps.total_space_gb * 0.20 - ELSE 0 - END), + total_min_savings_gb = + SUM + ( + CASE + WHEN ia.action IN (N'DISABLE', N'MERGE INCLUDES', N'MAKE UNIQUE') + THEN ps.total_space_gb + WHEN (ia.action IS NULL OR ia.action = N'KEEP') + AND ce.can_compress = 1 + THEN ps.total_space_gb * 0.20 + ELSE 0 + END + ), /* Total optimistic savings */ - total_max_savings_gb = SUM(CASE - WHEN ia.action IN ('DISABLE', 'MERGE INCLUDES', 'MAKE UNIQUE') - THEN ps.total_space_gb - WHEN (ia.action IS NULL OR ia.action = 'KEEP') - AND ce.can_compress = 1 - THEN ps.total_space_gb * 0.60 - ELSE 0 - END), + total_max_savings_gb = + SUM + ( + CASE + WHEN ia.action IN (N'DISABLE', N'MERGE INCLUDES', N'MAKE UNIQUE') + THEN ps.total_space_gb + WHEN (ia.action IS NULL OR ia.action = N'KEEP') + AND ce.can_compress = 1 + THEN ps.total_space_gb * 0.60 + ELSE 0 + END + ), /* Get total rows from database unique tables */ - total_rows = ( - SELECT SUM(t.row_count) - FROM ( + total_rows = + ( + SELECT + SUM(t.row_count) + FROM + ( SELECT ps_distinct.object_id, - row_count = MAX(CASE WHEN ps_distinct.index_id IN (0, 1) THEN ps_distinct.total_rows ELSE 0 END) - FROM #partition_stats ps_distinct + row_count = + MAX + ( + CASE + WHEN ps_distinct.index_id IN (0, 1) + THEN ps_distinct.total_rows + ELSE 0 + END + ) + FROM #partition_stats AS ps_distinct WHERE ps_distinct.index_id IN (0, 1) - GROUP BY ps_distinct.object_id - ) t + GROUP BY + ps_distinct.object_id + ) AS t ) FROM #index_analysis AS ia LEFT JOIN #partition_stats AS ps @@ -3365,19 +3479,26 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SELECT /* Basic identification */ - CASE - WHEN irs.summary_level = 'SUMMARY' THEN '=== OVERALL ANALYSIS ===' - ELSE irs.summary_level - END AS level, + level = + CASE + WHEN irs.summary_level = 'SUMMARY' THEN '=== OVERALL ANALYSIS ===' + ELSE irs.summary_level + END, /* Server info (for summary) or database name */ - CASE - WHEN irs.summary_level = 'SUMMARY' AND irs.uptime_warning = 1 - THEN 'WARNING: Server uptime only ' + CONVERT(varchar(10), irs.server_uptime_days) + ' days - usage data may be incomplete!' - WHEN irs.summary_level = 'SUMMARY' - THEN 'Server uptime: ' + CONVERT(varchar(10), irs.server_uptime_days) + ' days' - ELSE irs.database_name - END AS database_info, + database_info = + CASE + WHEN irs.summary_level = 'SUMMARY' + AND irs.uptime_warning = 1 + THEN 'WARNING: Server uptime only ' + + CONVERT(varchar(10), irs.server_uptime_days) + + ' days - usage data may be incomplete!' + WHEN irs.summary_level = 'SUMMARY' + THEN 'Server uptime: ' + + CONVERT(varchar(10), irs.server_uptime_days) + + ' days' + ELSE irs.database_name + END, /* Schema and table names (except for summary) */ irs.schema_name, @@ -3385,98 +3506,117 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* ===== Section 1: Index Counts ===== */ /* Tables analyzed (summary only) */ - CASE - WHEN irs.summary_level = 'SUMMARY' THEN FORMAT(irs.tables_analyzed, 'N0') - ELSE NULL - END AS tables_analyzed, + tables_analyzed = + CASE + WHEN irs.summary_level = 'SUMMARY' + THEN FORMAT(irs.tables_analyzed, 'N0') + ELSE NULL + END, /* Total indexes */ - FORMAT(irs.index_count, 'N0') AS total_indexes, + total_indexes = FORMAT(irs.index_count, 'N0'), /* Removable indexes - report consistent values across levels */ - CASE - WHEN irs.summary_level = 'SUMMARY' - THEN FORMAT(irs.indexes_to_disable, 'N0') /* Indexes that will be disabled based on analysis */ - ELSE FORMAT(irs.unused_indexes, 'N0') /* Unused indexes at database/table level */ - END AS removable_indexes, + removable_indexes = + CASE + WHEN irs.summary_level = 'SUMMARY' + THEN FORMAT(irs.indexes_to_disable, 'N0') /* Indexes that will be disabled based on analysis */ + ELSE FORMAT(irs.unused_indexes, 'N0') /* Unused indexes at database/table level */ + END, /* Show mergeable indexes across all levels */ - CASE - WHEN irs.summary_level = 'SUMMARY' THEN FORMAT(irs.indexes_to_merge, 'N0') - ELSE FORMAT(irs.indexes_to_merge, 'N0') - END AS mergeable_indexes, + mergeable_indexes = + CASE + WHEN irs.summary_level = 'SUMMARY' + THEN FORMAT(irs.indexes_to_merge, 'N0') + ELSE FORMAT(irs.indexes_to_merge, 'N0') + END, /* Percent of indexes that can be removed */ - CASE - WHEN irs.summary_level = 'SUMMARY' - THEN FORMAT(100.0 * irs.indexes_to_disable / NULLIF(irs.index_count, 0), 'N1') + '%' - WHEN irs.index_count > 0 - THEN FORMAT(100.0 * irs.unused_indexes / NULLIF(irs.index_count, 0), 'N1') + '%' - ELSE '0.0%' - END AS pct_removable, + pct_removable = + CASE + WHEN irs.summary_level = 'SUMMARY' + THEN FORMAT(100.0 * irs.indexes_to_disable / NULLIF(irs.index_count, 0), 'N1') + '%' + WHEN irs.index_count > 0 + THEN FORMAT(100.0 * irs.unused_indexes / NULLIF(irs.index_count, 0), 'N1') + '%' + ELSE '0.0%' + END, /* ===== Section 2: Size and Space Savings ===== */ /* Current size in GB */ - FORMAT(irs.total_size_gb, 'N2') AS current_size_gb, + current_size_gb = FORMAT(irs.total_size_gb, 'N2'), /* Size that can be saved through cleanup */ - CASE - WHEN irs.summary_level = 'SUMMARY' THEN FORMAT(irs.space_saved_gb, 'N2') - ELSE FORMAT(irs.unused_size_gb, 'N2') - END AS cleanup_savings_gb, + cleanup_savings_gb = + CASE + WHEN irs.summary_level = 'SUMMARY' + THEN FORMAT(irs.space_saved_gb, 'N2') + ELSE FORMAT(irs.unused_size_gb, 'N2') + END, /* Potential additional savings */ - CASE - WHEN irs.summary_level = 'SUMMARY' - THEN FORMAT(irs.total_min_savings_gb, 'N2') + ' - ' + FORMAT(irs.total_max_savings_gb, 'N2') - ELSE FORMAT(irs.unused_size_gb, 'N2') /* Show at all levels */ - END AS potential_savings_gb, + potential_savings_gb = + CASE + WHEN irs.summary_level = 'SUMMARY' + THEN FORMAT(irs.total_min_savings_gb, 'N2') + + ' - ' + + FORMAT(irs.total_max_savings_gb, 'N2') + ELSE FORMAT(irs.unused_size_gb, 'N2') /* Show at all levels */ + END, /* ===== Section 3: Table and Usage Statistics ===== */ /* Row count */ FORMAT(irs.total_rows, 'N0') AS total_rows, /* Total reads - combined total and breakdown */ - CASE - WHEN irs.summary_level <> 'SUMMARY' - THEN FORMAT(irs.total_reads, 'N0') + - ' (' + - FORMAT(irs.user_seeks, 'N0') + ' seeks, ' + - FORMAT(irs.user_scans, 'N0') + ' scans, ' + - FORMAT(irs.user_lookups, 'N0') + ' lookups)' - ELSE NULL - END AS reads_breakdown, + reads_breakdown = + CASE + WHEN irs.summary_level <> 'SUMMARY' + THEN FORMAT(irs.total_reads, 'N0') + + ' (' + + FORMAT(irs.user_seeks, 'N0') + ' seeks, ' + + FORMAT(irs.user_scans, 'N0') + ' scans, ' + + FORMAT(irs.user_lookups, 'N0') + ' lookups)' + ELSE NULL + END, /* Total writes */ - CASE - WHEN irs.summary_level <> 'SUMMARY' - THEN FORMAT(irs.total_writes, 'N0') - ELSE NULL - END AS writes, + writes = + CASE + WHEN irs.summary_level <> 'SUMMARY' + THEN FORMAT(irs.total_writes, 'N0') + ELSE NULL + END, /* ===== Section 4: Consolidated Performance Metrics ===== */ /* Total count of lock waits (row + page) */ - CASE - WHEN irs.summary_level <> 'SUMMARY' - THEN FORMAT(irs.row_lock_wait_count + irs.page_lock_wait_count, 'N0') - ELSE NULL - END AS lock_wait_count, + lock_wait_count = + CASE + WHEN irs.summary_level <> 'SUMMARY' + THEN FORMAT(irs.row_lock_wait_count + + irs.page_lock_wait_count, 'N0') + ELSE NULL + END, /* Average lock wait time in ms */ - CASE - WHEN irs.summary_level <> 'SUMMARY' AND (irs.row_lock_wait_count + irs.page_lock_wait_count) > 0 - THEN FORMAT(1.0 * (irs.row_lock_wait_in_ms + irs.page_lock_wait_in_ms) / - NULLIF(irs.row_lock_wait_count + irs.page_lock_wait_count, 0), 'N2') - ELSE NULL - END AS avg_lock_wait_ms, + avg_lock_wait_ms = + CASE + WHEN irs.summary_level <> 'SUMMARY' + AND (irs.row_lock_wait_count + irs.page_lock_wait_count) > 0 + THEN FORMAT(1.0 * (irs.row_lock_wait_in_ms + irs.page_lock_wait_in_ms) / + NULLIF(irs.row_lock_wait_count + irs.page_lock_wait_count, 0), 'N2') + ELSE NULL + END, /* Combined latch wait time in ms */ - CASE - WHEN irs.summary_level <> 'SUMMARY' AND (irs.page_latch_wait_count + irs.page_io_latch_wait_count) > 0 - THEN FORMAT(1.0 * (irs.page_latch_wait_in_ms + irs.page_io_latch_wait_in_ms) / - NULLIF(irs.page_latch_wait_count + irs.page_io_latch_wait_count, 0), 'N2') - ELSE NULL - END AS avg_latch_wait_ms + avg_latch_wait_ms = + CASE + WHEN irs.summary_level <> 'SUMMARY' + AND (irs.page_latch_wait_count + irs.page_io_latch_wait_count) > 0 + THEN FORMAT(1.0 * (irs.page_latch_wait_in_ms + irs.page_io_latch_wait_in_ms) / + NULLIF(irs.page_latch_wait_count + irs.page_io_latch_wait_count, 0), 'N2') + ELSE NULL + END FROM #index_reporting_stats AS irs WHERE irs.summary_level IN ('SUMMARY', 'DATABASE', 'TABLE') /* Filter out INDEX level */ ORDER BY From 44a71d1ed742f05bee47a048e315f5afd387f23b Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 13 Mar 2025 16:15:01 -0400 Subject: [PATCH 073/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index aec03994..56d9595a 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -508,7 +508,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. is_redundant bit NULL, superseded_by nvarchar(256) NULL, missing_columns nvarchar(max) NULL, - action nvarchar(max) NULL, + action nvarchar(30) NULL, target_index_name sysname NULL, consolidation_rule varchar(512) NULL, index_priority int NULL, From 6e14c6e950f08dcf23ef81b799200909d08e30cb Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 13 Mar 2025 16:16:06 -0400 Subject: [PATCH 074/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index 3875d09f..3ac5dae7 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -3006,7 +3006,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. total_size_gb = SUM(ps.total_space_gb), /* Use a simple aggregation to avoid double-counting */ /* Get actual row count by grabbing the real row count from clustered index/heap per table */ - total_rows = SUM(d.actual_rows), + total_rows = SUM(DISTINCT d.actual_rows), indexes_to_merge = ( SELECT From eaaaa814ea13d3fe1703bf183f25072153c77d96 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 13 Mar 2025 17:26:11 -0400 Subject: [PATCH 075/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 279 ++++++++++-------- 1 file changed, 157 insertions(+), 122 deletions(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index 3ac5dae7..25775fbd 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -79,7 +79,7 @@ BEGIN TRY @version_date = '17530101'; SELECT - for_insurance_purposes = N'Read the messages pane carefully!' + for_insurance_purposes = N'Read the messages pane carefully!'; PRINT N' ------------------------------------------------------------------------------------------- @@ -118,7 +118,7 @@ ALWAYS TEST THESE RECOMMENDATIONS IN A NON-PRODUCTION ENVIRONMENT FIRST" help = N'you are currently using a beta version, and the advice should not be followed' UNION ALL SELECT - help = N'without careful analysis and consideration. it may be harmful.' + help = N'without careful analysis and consideration. it may be harmful.'; /* Parameters @@ -302,7 +302,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. @database_id = d.database_id FROM sys.databases AS d WHERE d.name = @database_name - OPTION(RECOMPILE);; + OPTION(RECOMPILE); END; IF @schema_name IS NULL @@ -673,29 +673,28 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. WHERE v.object_id = i.object_id )'; - IF - /* Check SQL Server 2016+ for temporal tables support */ + IF /* Check SQL Server 2016+ for temporal tables support */ + ( + CONVERT ( - CONVERT - ( - integer, - SERVERPROPERTY('EngineEdition') - ) IN (5, 8) /* Azure SQL DB or Managed Instance */ - OR CONVERT + integer, + SERVERPROPERTY('EngineEdition') + ) IN (5, 8) /* Azure SQL DB or Managed Instance */ + OR CONVERT + ( + integer, + SUBSTRING ( - integer, - SUBSTRING + CONVERT ( - CONVERT - ( - varchar(20), - SERVERPROPERTY('ProductVersion') - ), - 1, - 2 - ) - ) >= 13 - ) /* SQL 2016+ */ + varchar(20), + SERVERPROPERTY('ProductVersion') + ), + 1, + 2 + ) + ) >= 13 + ) /* SQL 2016+ */ BEGIN SET @sql += N' AND NOT EXISTS @@ -791,7 +790,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. @min_rows, @object_id; - IF ROWCOUNT_BIG() = 0 BEGIN IF @debug = 1 BEGIN RAISERROR('No rows inserted into #filtered_objects', 0, 0) WITH NOWAIT END; END; + IF ROWCOUNT_BIG() = 0 + BEGIN + IF @debug = 1 + BEGIN + RAISERROR('No rows inserted into #filtered_objects', 0, 0) WITH NOWAIT; + END; + END; IF @debug = 1 BEGIN @@ -1036,7 +1041,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. @database_id, @object_id; - IF ROWCOUNT_BIG() = 0 BEGIN IF @debug = 1 BEGIN RAISERROR('No rows inserted into #operational_stats', 0, 0) WITH NOWAIT END; END; + IF ROWCOUNT_BIG() = 0 + BEGIN + IF @debug = 1 + BEGIN + RAISERROR('No rows inserted into #operational_stats', 0, 0) WITH NOWAIT; + END; + END; IF @debug = 1 BEGIN @@ -1259,7 +1270,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. @object_id, @min_rows; - IF ROWCOUNT_BIG() = 0 BEGIN IF @debug = 1 BEGIN RAISERROR('No rows inserted into #index_details', 0, 0) WITH NOWAIT END; END; + IF ROWCOUNT_BIG() = 0 + BEGIN + IF @debug = 1 + BEGIN + RAISERROR('No rows inserted into #index_details', 0, 0) WITH NOWAIT; + END; + END; IF @debug = 1 BEGIN @@ -1444,7 +1461,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. @database_id, @object_id; - IF ROWCOUNT_BIG() = 0 BEGIN IF @debug = 1 BEGIN RAISERROR('No rows inserted into #partition_stats', 0, 0) WITH NOWAIT END; END; + IF ROWCOUNT_BIG() = 0 + BEGIN + IF @debug = 1 + BEGIN + RAISERROR('No rows inserted into #partition_stats', 0, 0) WITH NOWAIT; + END; + END; IF @debug = 1 BEGIN @@ -1452,7 +1475,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. table_name = '#partition_stats', * FROM #partition_stats AS ps - OPTION(RECOMPILE);; + OPTION(RECOMPILE); RAISERROR('Performing #index_analysis insert', 0, 0) WITH NOWAIT; END; @@ -1553,7 +1576,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. id1.filter_definition OPTION(RECOMPILE); - IF ROWCOUNT_BIG() = 0 BEGIN IF @debug = 1 BEGIN RAISERROR('No rows inserted into #index_analysis', 0, 0) WITH NOWAIT END; END; + IF ROWCOUNT_BIG() = 0 + BEGIN + IF @debug = 1 + BEGIN + RAISERROR('No rows inserted into #index_analysis', 0, 0) WITH NOWAIT; + END; + END; IF @debug = 1 BEGIN @@ -1561,7 +1590,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. table_name = '#index_analysis', ia.* FROM #index_analysis AS ia - OPTION(RECOMPILE);; + OPTION(RECOMPILE); RAISERROR('Starting updates', 0, 0) WITH NOWAIT; END; @@ -1570,7 +1599,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. UPDATE #index_analysis SET - index_priority = + #index_analysis.index_priority = CASE WHEN index_id = 1 THEN 1000 /* Clustered indexes get highest priority */ @@ -1588,9 +1617,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ( SELECT 1/0 - FROM #index_details id - WHERE id.index_name = #index_analysis.index_name - AND id.table_name = #index_analysis.table_name + FROM #index_details AS id + WHERE id.index_id = #index_analysis.index_id + AND id.object_id = #index_analysis.object_id AND id.user_seeks > 0 ) THEN 200 ELSE 0 @@ -1601,26 +1630,26 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ( SELECT 1/0 - FROM #index_details id - WHERE id.index_name = #index_analysis.index_name - AND id.table_name = #index_analysis.table_name + FROM #index_details AS id + WHERE id.index_id = #index_analysis.index_id + AND id.object_id = #index_analysis.object_id AND id.user_scans > 0 ) THEN 100 ELSE 0 END - OPTION(RECOMPILE);; /* Indexes with scans get some priority */ + OPTION(RECOMPILE); /* Indexes with scans get some priority */ /* Rule 1: Identify unused indexes */ UPDATE #index_analysis SET - consolidation_rule = + #index_analysis.consolidation_rule = CASE WHEN @uptime_warning = 1 THEN 'Unused Index (WARNING: Server uptime < 14 days - usage data may be incomplete)' ELSE 'Unused Index' END, - action = N'DISABLE' + #index_analysis.action = N'DISABLE' WHERE EXISTS ( SELECT @@ -1628,7 +1657,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. FROM #index_details id WHERE id.database_id = #index_analysis.database_id AND id.object_id = #index_analysis.object_id - AND id.index_name = #index_analysis.index_name + AND id.index_id = #index_analysis.index_id AND id.user_seeks = 0 AND id.user_scans = 0 AND id.user_lookups = 0 @@ -1670,20 +1699,20 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ( SELECT 1/0 - FROM #index_details id1 + FROM #index_details AS id1 WHERE id1.database_id = ia1.database_id AND id1.object_id = ia1.object_id - AND id1.index_name = ia1.index_name + AND id1.index_id = ia1.index_id AND id1.is_eligible_for_dedupe = 1 ) AND EXISTS ( SELECT 1/0 - FROM #index_details id2 + FROM #index_details AS id2 WHERE id2.database_id = ia2.database_id AND id2.object_id = ia2.object_id - AND id2.index_name = ia2.index_name + AND id2.index_id = ia2.index_id AND id2.is_eligible_for_dedupe = 1 ) OPTION(RECOMPILE); @@ -1738,20 +1767,20 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ( SELECT 1/0 - FROM #index_details id1 + FROM #index_details AS id1 WHERE id1.database_id = ia1.database_id AND id1.object_id = ia1.object_id - AND id1.index_name = ia1.index_name + AND id1.index_id = ia1.index_id AND id1.is_eligible_for_dedupe = 1 ) AND EXISTS ( SELECT 1/0 - FROM #index_details id2 + FROM #index_details AS id2 WHERE id2.database_id = ia2.database_id AND id2.object_id = ia2.object_id - AND id2.index_name = ia2.index_name + AND id2.index_id = ia2.index_id AND id2.is_eligible_for_dedupe = 1 ) OPTION(RECOMPILE); @@ -1762,7 +1791,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SET ia1.consolidation_rule = 'Key Subset', ia1.target_index_name = ia2.index_name, - ia1.action = 'DISABLE' /* The narrower index gets disabled */ + ia1.action = N'DISABLE' /* The narrower index gets disabled */ FROM #index_analysis AS ia1 JOIN #index_analysis AS ia2 ON ia1.database_id = ia2.database_id @@ -1778,20 +1807,20 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ( SELECT 1/0 - FROM #index_details id1 + FROM #index_details AS id1 WHERE id1.database_id = ia1.database_id AND id1.object_id = ia1.object_id - AND id1.index_name = ia1.index_name + AND id1.index_id = ia1.index_id AND id1.is_eligible_for_dedupe = 1 ) AND EXISTS ( SELECT 1/0 - FROM #index_details id2 + FROM #index_details AS id2 WHERE id2.database_id = ia2.database_id AND id2.object_id = ia2.object_id - AND id2.index_name = ia2.index_name + AND id2.index_id = ia2.index_id AND id2.is_eligible_for_dedupe = 1 ) OPTION(RECOMPILE); @@ -1832,10 +1861,10 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Find nonclustered indexes */ SELECT 1/0 - FROM #index_details id1 + FROM #index_details AS id1 WHERE id1.database_id = ia1.database_id AND id1.object_id = ia1.object_id - AND id1.index_name = ia1.index_name + AND id1.index_id = ia1.index_id AND id1.is_eligible_for_dedupe = 1 ) AND EXISTS @@ -1843,7 +1872,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Find unique constraints with matching key columns */ SELECT 1/0 - FROM #index_details id2 + FROM #index_details AS id2 WHERE id2.database_id = ia1.database_id AND id2.object_id = ia1.object_id AND id2.is_unique_constraint = 1 @@ -1852,7 +1881,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Verify key columns match between index and unique constraint */ SELECT id2_inner.column_name - FROM #index_details id2_inner + FROM #index_details AS id2_inner WHERE id2_inner.database_id = id2.database_id AND id2_inner.object_id = id2.object_id AND id2_inner.index_id = id2.index_id @@ -1862,10 +1891,10 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SELECT id1_inner.column_name - FROM #index_details id1_inner + FROM #index_details AS id1_inner WHERE id1_inner.database_id = ia1.database_id AND id1_inner.object_id = ia1.object_id - AND id1_inner.index_name = ia1.index_name + AND id1_inner.index_id = ia1.index_id AND id1_inner.is_included_column = 0 ) ) @@ -1878,7 +1907,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ia1 SET ia1.consolidation_rule = 'Same Keys Different Order', - ia1.action = 'REVIEW', /* These need manual review */ + ia1.action = N'REVIEW', /* These need manual review */ ia1.target_index_name = ia2.index_name /* Reference the partner index */ FROM #index_analysis AS ia1 JOIN #index_analysis AS ia2 @@ -1902,8 +1931,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND id2.key_ordinal = 1 WHERE id1.database_id = ia1.database_id AND id1.object_id = ia1.object_id - AND id1.index_name = ia1.index_name - AND id2.index_name = ia2.index_name + AND id1.index_id = ia1.index_id + AND id2.index_id = ia2.index_id ) /* Same set of key columns but in different order */ AND NOT EXISTS @@ -1914,7 +1943,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. FROM #index_details AS id1 WHERE id1.database_id = ia1.database_id AND id1.object_id = ia1.object_id - AND id1.index_name = ia1.index_name + AND id1.index_id = ia1.index_id AND id1.is_included_column = 0 AND id1.key_ordinal > 0 @@ -1925,7 +1954,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. FROM #index_details AS id2 WHERE id2.database_id = ia2.database_id AND id2.object_id = ia2.object_id - AND id2.index_name = ia2.index_name + AND id2.index_id = ia2.index_id AND id2.is_included_column = 0 AND id2.key_ordinal > 0 ) @@ -1945,8 +1974,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND id2.key_ordinal > 1 /* After the first column */ WHERE id1.database_id = ia1.database_id AND id1.object_id = ia1.object_id - AND id1.index_name = ia1.index_name - AND id2.index_name = ia2.index_name + AND id1.index_id = ia1.index_id + AND id2.index_id = ia2.index_id ) OPTION(RECOMPILE); @@ -2024,7 +2053,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND candidate.object_id = ia.object_id AND candidate.key_columns = ia.key_columns AND ISNULL(candidate.filter_definition, '') = ISNULL(ia.filter_definition, '') - AND candidate.action = 'MERGE INCLUDES' + AND candidate.action = N'MERGE INCLUDES' AND candidate.consolidation_rule = 'Key Duplicate' ORDER BY /* First prefer indexes with "_Extended" in the name */ @@ -2047,7 +2076,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND inner_ia.object_id = ia.object_id AND inner_ia.key_columns = ia.key_columns AND ISNULL(inner_ia.filter_definition, '') = ISNULL(ia.filter_definition, '') - AND inner_ia.action = 'MERGE INCLUDES' + AND inner_ia.action = N'MERGE INCLUDES' AND inner_ia.consolidation_rule = 'Key Duplicate' ORDER BY inner_ia.index_name @@ -2061,7 +2090,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. '' ) FROM #index_analysis AS ia - WHERE ia.action = 'MERGE INCLUDES' + WHERE ia.action = N'MERGE INCLUDES' AND ia.consolidation_rule = 'Key Duplicate' GROUP BY ia.database_id, @@ -2201,14 +2230,14 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND ia.index_name = isd.superset_index_name OPTION(RECOMPILE); - /* Update winning indexes that don't actually need changes to have action = 'KEEP' */ + /* Update winning indexes that don't actually need changes to have action = N'KEEP' */ UPDATE ia SET /* Change action to 'KEEP' for indexes that don't need to be modified */ ia.action = N'KEEP' FROM #index_analysis AS ia - WHERE ia.action = 'MERGE INCLUDES' + WHERE ia.action = N'MERGE INCLUDES' AND ia.superseded_by IS NOT NULL /* Check if the index name contains "Extended" and has more included columns */ AND (ia.index_name LIKE '%\_Extended%' ESCAPE '\' OR ia.index_name LIKE '%\_Extended' OR ia.index_name LIKE '%_Extended%') @@ -2222,7 +2251,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. WHERE ia_subset.database_id = ia.database_id AND ia_subset.object_id = ia.object_id AND ia_subset.key_columns = ia.key_columns - AND ia_subset.action = 'DISABLE' + AND ia_subset.action = N'DISABLE' AND ia_subset.target_index_name = ia.index_name /* This complex check handles cases where the superset doesn't contain all subset columns */ AND CHARINDEX(ISNULL(ia_subset.included_columns, N''), ISNULL(ia.included_columns, N'')) = 0 @@ -2270,11 +2299,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ia.target_index_name, script = CASE - WHEN ia.action = 'MAKE UNIQUE' + WHEN ia.action = N'MAKE UNIQUE' THEN N'/* This index can replace a unique constraint */ /* Creating unique index with same keys as constraint */ CREATE UNIQUE ' - WHEN ia.action = 'MERGE INCLUDES' + WHEN ia.action = N'MERGE INCLUDES' THEN N'/* This index can be merged with another index */ /* Creating index with combined includes from both */ CREATE ' @@ -2294,7 +2323,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CASE WHEN ia.included_columns IS NOT NULL AND LEN(ia.included_columns) > 0 - AND ia.action = 'MERGE INCLUDES' + AND ia.action = N'MERGE INCLUDES' THEN N' INCLUDE (' + ia.included_columns + N')' @@ -2333,9 +2362,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Additional info about what this script does */ additional_info = CASE - WHEN ia.action = 'MERGE INCLUDES' + WHEN ia.action = N'MERGE INCLUDES' THEN N'This index will absorb includes from duplicate indexes' - WHEN ia.action = 'MAKE UNIQUE' + WHEN ia.action = N'MAKE UNIQUE' THEN N'This index will replace a unique constraint' ELSE NULL END, @@ -2456,10 +2485,10 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. LEFT JOIN #index_details AS id ON id.database_id = ia.database_id AND id.object_id = ia.object_id - AND id.index_name = ia.index_name + AND id.index_id = ia.index_id AND id.is_included_column = 0 /* Get only one row per index */ AND id.key_ordinal > 0 - WHERE ia.action = 'DISABLE' + WHERE ia.action = N'DISABLE' OPTION(RECOMPILE); /* Insert compression scripts for remaining indexes */ @@ -2539,7 +2568,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ps.built_on, ps.partition_function_name, ps.partition_columns - ) ps + ) + AS ps ON ia.database_id = ps.database_id AND ia.object_id = ps.object_id AND ia.index_id = ps.index_id @@ -2550,16 +2580,16 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. LEFT JOIN #index_details AS id ON id.database_id = ia.database_id AND id.object_id = ia.object_id - AND id.index_name = ia.index_name + AND id.index_id = ia.index_id AND id.is_included_column = 0 /* Get only one row per index */ AND id.key_ordinal > 0 - JOIN #compression_eligibility ce + JOIN #compression_eligibility AS ce ON ia.database_id = ce.database_id AND ia.object_id = ce.object_id AND ia.index_id = ce.index_id WHERE /* Indexes that are not being disabled or merged */ - (ia.action IS NULL OR ia.action = 'KEEP') + (ia.action IS NULL OR ia.action = N'KEEP') /* Only indexes eligible for compression */ AND ce.can_compress = 1 OPTION(RECOMPILE); @@ -2621,7 +2651,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. LEFT JOIN #index_details AS id2 ON id2.database_id = ia.database_id AND id2.object_id = ia.object_id - AND id2.index_name = ia.index_name + AND id2.index_id = ia.index_id AND id2.is_included_column = 0 /* Get only one row per index */ AND id2.key_ordinal > 0 LEFT JOIN #partition_stats AS ps @@ -2630,22 +2660,22 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND ia.index_id = ps.index_id WHERE /* Only indexes that are being made unique */ - ia.action = 'MAKE UNIQUE' + ia.action = N'MAKE UNIQUE' /* Find the constraint that matches the index being made unique */ AND EXISTS ( SELECT 1/0 - FROM #index_details id_nc + FROM #index_details AS id_nc WHERE id_nc.database_id = ia.database_id AND id_nc.object_id = ia.object_id - AND id_nc.index_name = ia.index_name + AND id_nc.index_id = ia.index_id /* Matching key columns */ AND NOT EXISTS ( SELECT id.column_name - FROM #index_details id_inner + FROM #index_details AS id_inner WHERE id_inner.database_id = id.database_id AND id_inner.object_id = id.object_id AND id_inner.index_id = id.index_id @@ -2655,10 +2685,10 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SELECT id_nc_inner.column_name - FROM #index_details id_nc_inner + FROM #index_details AS id_nc_inner WHERE id_nc_inner.database_id = id_nc.database_id AND id_nc_inner.object_id = id_nc.object_id - AND id_nc_inner.index_name = id_nc.index_name + AND id_nc_inner.index_id = id_nc.index_id AND id_nc_inner.is_included_column = 0 ) ) @@ -2757,7 +2787,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. LEFT JOIN #index_details AS id ON id.database_id = ia.database_id AND id.object_id = ia.object_id - AND id.index_name = ia.index_name + AND id.index_id = ia.index_id AND id.is_included_column = 0 /* Get only one row per index */ AND id.key_ordinal > 0 JOIN #compression_eligibility AS ce @@ -2768,7 +2798,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Only partitioned indexes */ ps.partition_function_name IS NOT NULL /* Indexes that are not being disabled or merged */ - AND (ia.action IS NULL OR ia.action = 'KEEP') + AND (ia.action IS NULL OR ia.action = N'KEEP') /* Only indexes eligible for compression */ AND ce.can_compress = 1 OPTION(RECOMPILE); @@ -2819,7 +2849,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. LEFT JOIN #index_details AS id ON id.database_id = ce.database_id AND id.object_id = ce.object_id - AND id.index_name = ce.index_name + AND id.index_id = ce.index_id AND id.is_included_column = 0 /* Get only one row per index */ AND id.key_ordinal > 0 WHERE ce.can_compress = 0 @@ -2882,10 +2912,10 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. LEFT JOIN #index_details AS id ON id.database_id = ia.database_id AND id.object_id = ia.object_id - AND id.index_name = ia.index_name + AND id.index_id = ia.index_id AND id.is_included_column = 0 /* Get only one row per index */ AND id.key_ordinal > 0 - WHERE ia.action = 'REVIEW' + WHERE ia.action = N'REVIEW' OPTION(RECOMPILE); @@ -2929,7 +2959,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CASE WHEN ia.superseded_by IS NOT NULL THEN 'This index supersedes other indexes and already has all needed columns' - WHEN ia.action = 'KEEP' + WHEN ia.action = N'KEEP' THEN 'This index is being kept' ELSE NULL END, @@ -2946,10 +2976,10 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. LEFT JOIN #index_details AS id ON id.database_id = ia.database_id AND id.object_id = ia.object_id - AND id.index_name = ia.index_name + AND id.index_id = ia.index_id AND id.is_included_column = 0 /* Get only one row per index */ AND id.key_ordinal > 0 - WHERE ia.action = 'KEEP' + WHERE ia.action = N'KEEP' OR ( ia.action IS NULL @@ -3011,18 +3041,18 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ( SELECT COUNT_BIG(*) - FROM #index_analysis ia + FROM #index_analysis AS ia WHERE ia.action IN (N'MERGE INCLUDES', N'MAKE UNIQUE') - AND ia.database_name = ps.database_name + AND ia.database_id = ps.database_id ), /* Use count from analysis to keep consistent with SUMMARY level */ unused_indexes = ( SELECT COUNT_BIG(*) - FROM #index_analysis ia + FROM #index_analysis AS ia WHERE ia.action = N'DISABLE' - AND ia.database_name = ps.database_name + AND ia.database_id = ps.database_id ), unused_size_gb = SUM @@ -3055,14 +3085,14 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. leaf_insert_count = SUM(os.leaf_insert_count), leaf_update_count = SUM(os.leaf_update_count), leaf_delete_count = SUM(os.leaf_delete_count) - FROM #partition_stats ps - LEFT JOIN #index_details id + FROM #partition_stats AS ps + LEFT JOIN #index_details AS id ON id.database_id = ps.database_id AND id.object_id = ps.object_id AND id.index_id = ps.index_id AND id.is_included_column = 0 AND id.key_ordinal > 0 - LEFT JOIN #operational_stats os + LEFT JOIN #operational_stats AS os ON os.database_id = ps.database_id AND os.object_id = ps.object_id AND os.index_id = ps.index_id @@ -3087,7 +3117,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ps2.object_id ) AS d GROUP BY - ps.database_name + ps.database_name, + ps.database_id OPTION(RECOMPILE); /* Insert table-level summaries */ @@ -3155,22 +3186,22 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ( SELECT COUNT_BIG(*) - FROM #index_analysis ia + FROM #index_analysis AS ia WHERE ia.action IN (N'MERGE INCLUDES', N'MAKE UNIQUE') - AND ia.database_name = ps.database_name - AND ia.schema_name = ps.schema_name - AND ia.table_name = ps.table_name + AND ia.database_id = ps.database_id + AND ia.schema_id = ps.schema_id + AND ia.object_id = ps.object_id ), /* Use count from analysis to keep consistent with SUMMARY level */ unused_indexes = ( SELECT COUNT_BIG(*) - FROM #index_analysis ia + FROM #index_analysis AS ia WHERE ia.action = N'DISABLE' - AND ia.database_name = ps.database_name - AND ia.schema_name = ps.schema_name - AND ia.table_name = ps.table_name + AND ia.database_id = ps.database_id + AND ia.schema_id = ps.schema_id + AND ia.object_id = ps.object_id ), unused_size_gb = SUM @@ -3178,7 +3209,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CASE WHEN id.user_seeks + id.user_scans + id.user_lookups = 0 THEN ps.total_space_gb - ELSE 0 END + ELSE 0 + END ), total_reads = SUM(id.user_seeks + id.user_scans + id.user_lookups), total_writes = SUM(id.user_updates), @@ -3202,21 +3234,24 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. leaf_insert_count = SUM(os.leaf_insert_count), leaf_update_count = SUM(os.leaf_update_count), leaf_delete_count = SUM(os.leaf_delete_count) - FROM #partition_stats ps - LEFT JOIN #index_details id + FROM #partition_stats AS ps + LEFT JOIN #index_details AS id ON id.database_id = ps.database_id AND id.object_id = ps.object_id AND id.index_id = ps.index_id AND id.is_included_column = 0 AND id.key_ordinal > 0 - LEFT JOIN #operational_stats os + LEFT JOIN #operational_stats AS os ON os.database_id = ps.database_id AND os.object_id = ps.object_id AND os.index_id = ps.index_id GROUP BY ps.database_name, - ps.schema_name, - ps.table_name + ps.database_id, + ps.schema_name, + ps.schema_id, + ps.table_name, + ps.object_id OPTION(RECOMPILE); /* We're not doing index-level summaries - focusing on database and table level reports */ @@ -3240,7 +3275,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ IF @debug = 1 BEGIN - RAISERROR('Generating #index_cleanup_results insert, RESULTS', 0, 0) WITH NOWAIT; + RAISERROR('Generating #index_cleanup_results, RESULTS', 0, 0) WITH NOWAIT; END; SELECT From dc25f69f5f7bc82a3994e51c2ce308c157935097 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 13 Mar 2025 17:32:18 -0400 Subject: [PATCH 076/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index 25775fbd..81933be9 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -665,6 +665,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND us.database_id = @database_id WHERE t.is_ms_shipped = 0 AND t.type <> N''TF'' + AND i.index_id > 0 AND NOT EXISTS ( SELECT From 2354aa5e0eec110eb89986d91222903931e64861 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 13 Mar 2025 17:50:57 -0400 Subject: [PATCH 077/246] Update sp_HealthParser.sql --- sp_HealthParser/sp_HealthParser.sql | 265 ++++++++++++++++++++++++++++ 1 file changed, 265 insertions(+) diff --git a/sp_HealthParser/sp_HealthParser.sql b/sp_HealthParser/sp_HealthParser.sql index 57bb2b99..6a077f7a 100644 --- a/sp_HealthParser/sp_HealthParser.sql +++ b/sp_HealthParser/sp_HealthParser.sql @@ -5268,5 +5268,270 @@ END; END; END; END; /*End locks*/ + + /* + This section generates a summary table with key findings from all health check sections + */ + IF @debug = 1 + BEGIN + RAISERROR('Creating health summary findings', 0, 1) WITH NOWAIT; + END; + + CREATE TABLE + #health_findings + ( + id integer IDENTITY PRIMARY KEY CLUSTERED, + check_id integer NOT NULL, + finding_group nvarchar(100) NULL, + finding nvarchar(4000) NULL, + sort_order bigint + ); + + -- Significant waits findings + IF @what_to_check IN ('all', 'waits') + AND EXISTS (SELECT 1/0 FROM #wait_info AS wi WHERE wait_duration_ms > 1000) + BEGIN + INSERT INTO #health_findings (check_id, finding_group, finding, sort_order) + SELECT + 1 AS check_id, + N'waits' AS finding_group, + finding = N'Long waits detected: ' + + CONVERT(nvarchar(10), COUNT_BIG(CASE WHEN wait_duration_ms > 30000 THEN 1 END)) + N' critical, ' + + CONVERT(nvarchar(10), COUNT_BIG(CASE WHEN wait_duration_ms BETWEEN 10000 AND 30000 THEN 1 END)) + N' high, ' + + CONVERT(nvarchar(10), COUNT_BIG(CASE WHEN wait_duration_ms BETWEEN 5000 AND 10000 THEN 1 END)) + N' medium. ' + + N'Longest wait: ' + CONVERT(nvarchar(10), MAX(wait_duration_ms)/1000.0) + N' seconds (' + + (SELECT TOP 1 wait_type FROM #wait_info AS wi2 ORDER BY wait_duration_ms DESC) + N')', + sort_order = CASE + WHEN MAX(wait_duration_ms) > 30000 THEN 100 -- Critical + WHEN MAX(wait_duration_ms) > 10000 THEN 200 -- High + WHEN MAX(wait_duration_ms) > 5000 THEN 300 -- Medium + ELSE 400 -- Low + END + FROM #wait_info AS wi + WHERE wait_duration_ms > 1000 -- Only include significant waits + HAVING COUNT_BIG(*) > 0; + END; + + -- CPU tasks findings + IF @what_to_check IN ('all', 'cpu') + AND EXISTS (SELECT 1/0 FROM #cpu_info) + BEGIN + INSERT INTO #health_findings (check_id, finding_group, finding, sort_order) + SELECT + 2 AS check_id, + N'cpu' AS finding_group, + finding = N'CPU pressure detected: ' + + CONVERT(nvarchar(10), COUNT_BIG(CASE WHEN wait_duration_ms > 180000 THEN 1 END)) + N' long-running tasks, ' + + CONVERT(nvarchar(10), COUNT_BIG(CASE WHEN wait_type = N'SOS_SCHEDULER_YIELD' THEN 1 END)) + N' scheduler yields, ' + + CASE + WHEN MAX(percent_complete) < 10 AND MAX(wait_duration_ms) > 300000 + THEN N'Possible hung task detected. ' + ELSE N'' + END + + N'Longest running task: ' + CONVERT(nvarchar(10), MAX(wait_duration_ms)/1000.0) + N' seconds', + sort_order = CASE + WHEN MAX(wait_duration_ms) > 300000 THEN 110 -- Critical + WHEN MAX(wait_duration_ms) > 180000 THEN 210 -- High + WHEN MAX(wait_duration_ms) > 60000 THEN 310 -- Medium + ELSE 410 -- Low + END + FROM #cpu_info AS ci + HAVING COUNT_BIG(*) > 0; + END; + + -- Memory conditions findings + IF @what_to_check IN ('all', 'memory') + AND EXISTS (SELECT 1/0 FROM #memory_conditions) + BEGIN + INSERT INTO #health_findings (check_id, finding_group, finding, sort_order) + SELECT + 3 AS check_id, + N'memory' AS finding_group, + finding = N'Memory pressure: ' + + CONVERT(nvarchar(10), COUNT_BIG(CASE WHEN notification_type LIKE N'%CRITICAL%' THEN 1 END)) + N' critical, ' + + CONVERT(nvarchar(10), COUNT_BIG(CASE WHEN notification_type LIKE N'%HIGH%' THEN 1 END)) + N' high alerts. ' + + CASE + WHEN COUNT_BIG(CASE WHEN notification_type LIKE N'%RESOURCE_MEMPHYSICAL_HIGH%' THEN 1 END) > 0 + THEN N'Physical memory pressure detected. ' + ELSE N'' + END + + CASE + WHEN COUNT_BIG(CASE WHEN notification_type LIKE N'%RESOURCE_MEM_HIGH%' THEN 1 END) > 0 + THEN N'SQL Server memory pressure detected.' + ELSE N'' + END, + sort_order = CASE + WHEN COUNT_BIG(CASE WHEN notification_type LIKE N'%CRITICAL%' THEN 1 END) > 0 THEN 120 -- Critical + WHEN COUNT_BIG(CASE WHEN notification_type LIKE N'%HIGH%' THEN 1 END) > 0 THEN 220 -- High + ELSE 320 -- Medium + END + FROM #memory_conditions AS mc + HAVING COUNT_BIG(*) > 0; + END; + + -- Memory broker findings + IF @what_to_check IN ('all', 'memory') + AND EXISTS (SELECT 1/0 FROM #memory_broker) + BEGIN + INSERT INTO #health_findings (check_id, finding_group, finding, sort_order) + SELECT + 4 AS check_id, + N'memory' AS finding_group, + finding = N'Memory broker: ' + + CONVERT(nvarchar(10), COUNT_BIG(*)) + N' memory requests, ' + + N'Maximum requested: ' + MAX(memory_requested_gb) + N' GB. ' + + N'Most common clerk: ' + + (SELECT TOP 1 memory_clerk_name FROM #memory_broker AS mb2 + GROUP BY memory_clerk_name ORDER BY COUNT_BIG(*) DESC), + sort_order = 330 -- Medium priority for broker events + FROM #memory_broker AS mb + HAVING COUNT_BIG(*) > 0; + END; + + -- Memory node OOM findings + IF @what_to_check IN ('all', 'memory') + AND EXISTS (SELECT 1/0 FROM #memory_node_oom) + BEGIN + INSERT INTO #health_findings (check_id, finding_group, finding, sort_order) + SELECT + 5 AS check_id, + N'memory' AS finding_group, + finding = N'Memory node OOM: ' + + CONVERT(nvarchar(10), COUNT_BIG(*)) + N' out-of-memory events detected. ' + + N'Most affected node ID: ' + CONVERT(nvarchar(10), + (SELECT TOP 1 node_id FROM #memory_node_oom AS mno2 + GROUP BY node_id ORDER BY COUNT_BIG(*) DESC) + ) + + N'. Most common clerk: ' + + (SELECT TOP 1 memory_clerk_name FROM #memory_node_oom AS mno3 + GROUP BY memory_clerk_name ORDER BY COUNT_BIG(*) DESC), + sort_order = 130 -- Critical for OOM events + FROM #memory_node_oom AS mno + HAVING COUNT_BIG(*) > 0; + END; + + -- IO issues findings + IF @what_to_check IN ('all', 'disk') + AND EXISTS (SELECT 1/0 FROM #io_issues) + BEGIN + INSERT INTO #health_findings (check_id, finding_group, finding, sort_order) + SELECT + 6 AS check_id, + N'io' AS finding_group, + finding = N'IO issues: ' + + CONVERT(nvarchar(10), SUM(intervalLongIos)) + N' long IOs detected, ' + + CONVERT(nvarchar(10), SUM(ioLatchTimeouts)) + N' IO latch timeouts. ' + + N'Longest pending IO: ' + MAX(longestPendingRequests_duration_ms) + N' ms, ' + + N'File: ' + + (SELECT TOP 1 longestPendingRequests_filePath + FROM #io_issues AS io2 + ORDER BY TRY_CONVERT(bigint, longestPendingRequests_duration_ms) DESC), + sort_order = CASE + WHEN MAX(TRY_CONVERT(bigint, longestPendingRequests_duration_ms)) > 15000 THEN 140 -- Critical + WHEN MAX(TRY_CONVERT(bigint, longestPendingRequests_duration_ms)) > 5000 THEN 240 -- High + ELSE 340 -- Medium + END + FROM #io_issues AS io + HAVING COUNT_BIG(*) > 0; + END; + + -- System health findings + IF @what_to_check IN ('all', 'system') + AND EXISTS (SELECT 1/0 FROM #system_health) + BEGIN + INSERT INTO #health_findings (check_id, finding_group, finding, sort_order) + SELECT + 7 AS check_id, + N'health' AS finding_group, + finding = N'System health issues: ' + + CONVERT(nvarchar(10), SUM(CASE WHEN spinlockBackoffs > 0 THEN spinlockBackoffs ELSE 0 END)) + N' spinlock backoffs, ' + + CONVERT(nvarchar(10), SUM(CASE WHEN latchWarnings > 0 THEN latchWarnings ELSE 0 END)) + N' latch warnings, ' + + CONVERT(nvarchar(10), SUM(CASE WHEN nonYieldingTasksReported > 0 THEN nonYieldingTasksReported ELSE 0 END)) + N' non-yielding tasks, ' + + CONVERT(nvarchar(10), SUM(CASE WHEN isAccessViolationOccurred > 0 THEN isAccessViolationOccurred ELSE 0 END)) + N' access violations. ' + + N'CPU utilization - SQL: ' + CONVERT(nvarchar(10), MAX(sqlCpuUtilization)) + N'%, ' + + N'System: ' + CONVERT(nvarchar(10), MAX(systemCpuUtilization)) + N'%', + sort_order = CASE + WHEN SUM(CASE WHEN isAccessViolationOccurred > 0 THEN 1 ELSE 0 END) > 0 THEN 150 -- Critical + WHEN SUM(CASE WHEN nonYieldingTasksReported > 0 THEN 1 ELSE 0 END) > 0 THEN 250 -- High + WHEN MAX(sqlCpuUtilization) > 80 THEN 350 -- Medium + ELSE 450 -- Low + END + FROM #system_health AS sh + HAVING COUNT_BIG(*) > 0; + END; + + -- Severe errors findings + IF @what_to_check IN ('all', 'system') + AND EXISTS (SELECT 1/0 FROM #error_info) + BEGIN + INSERT INTO #health_findings (check_id, finding_group, finding, sort_order) + SELECT + 8 AS check_id, + N'errors' AS finding_group, + finding = N'Severe errors: ' + + CONVERT(nvarchar(10), COUNT_BIG(CASE WHEN severity >= 22 THEN 1 END)) + N' fatal, ' + + CONVERT(nvarchar(10), COUNT_BIG(CASE WHEN severity BETWEEN 20 AND 21 THEN 1 END)) + N' critical, ' + + CONVERT(nvarchar(10), COUNT_BIG(CASE WHEN severity BETWEEN 16 AND 19 THEN 1 END)) + N' high severity errors. ' + + N'Most common error: ' + + CONVERT(nvarchar(10), (SELECT TOP 1 error_number FROM #error_info AS ei2 + GROUP BY error_number ORDER BY COUNT_BIG(*) DESC)) + + N' (' + + (SELECT TOP 1 LEFT(message, 50) + CASE WHEN LEN(message) > 50 THEN N'...' ELSE N'' END + FROM #error_info AS ei3 ORDER BY severity DESC, event_time DESC) + + N')', + sort_order = CASE + WHEN COUNT_BIG(CASE WHEN severity >= 22 THEN 1 END) > 0 THEN 160 -- Fatal + WHEN COUNT_BIG(CASE WHEN severity BETWEEN 20 AND 21 THEN 1 END) > 0 THEN 260 -- Critical + WHEN COUNT_BIG(CASE WHEN severity BETWEEN 16 AND 19 THEN 1 END) > 0 THEN 360 -- High + ELSE 460 -- Medium + END + FROM #error_info AS ei + HAVING COUNT_BIG(*) > 0; + END; + + -- Scheduler issues findings + IF @what_to_check IN ('all', 'cpu') + AND EXISTS (SELECT 1/0 FROM #scheduler_issues) + BEGIN + INSERT INTO #health_findings (check_id, finding_group, finding, sort_order) + SELECT + 9 AS check_id, + N'cpu' AS finding_group, + finding = N'Scheduler issues: ' + + CONVERT(nvarchar(10), COUNT_BIG(*)) + N' scheduler events detected. ' + + CONVERT(nvarchar(10), COUNT_BIG(CASE WHEN is_online = 0 THEN 1 END)) + N' schedulers offline, ' + + CONVERT(nvarchar(10), COUNT_BIG(CASE WHEN is_running = 0 AND is_online = 1 THEN 1 END)) + N' schedulers not running. ' + + N'Non-yielding time: Max ' + CONVERT(nvarchar(10), MAX(TRY_CONVERT(bigint, non_yielding_time_ms))) + N' ms', + sort_order = CASE + WHEN MAX(TRY_CONVERT(bigint, non_yielding_time_ms)) > 60000 THEN 170 -- Critical + WHEN COUNT_BIG(CASE WHEN is_online = 0 THEN 1 END) > 0 THEN 270 -- High + WHEN COUNT_BIG(CASE WHEN is_running = 0 AND is_online = 1 THEN 1 END) > 0 THEN 370 -- Medium + ELSE 470 -- Low + END + FROM #scheduler_issues AS si + HAVING COUNT_BIG(*) > 0; + END; + + -- Return the findings summary, ordered by severity + IF EXISTS (SELECT 1/0 FROM #health_findings) + BEGIN + IF @debug = 1 + BEGIN + RAISERROR('Returning health findings summary', 0, 1) WITH NOWAIT; + END; + + SELECT + finding_group, + finding + FROM #health_findings AS hf + ORDER BY sort_order, check_id; + END; + ELSE + BEGIN + SELECT + finding_group = N'No Issues Found', + finding = N'No significant health issues detected in the analyzed time period'; + END; + END; /*Final End*/ GO From 027004246e2ff80858489f0da5b7bebd80eb7c31 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 13 Mar 2025 18:17:47 -0400 Subject: [PATCH 078/246] Update sp_HealthParser.sql --- sp_HealthParser/sp_HealthParser.sql | 226 ++++++++++++++++++++-------- 1 file changed, 165 insertions(+), 61 deletions(-) diff --git a/sp_HealthParser/sp_HealthParser.sql b/sp_HealthParser/sp_HealthParser.sql index 6a077f7a..d00b9ad5 100644 --- a/sp_HealthParser/sp_HealthParser.sql +++ b/sp_HealthParser/sp_HealthParser.sql @@ -1109,7 +1109,10 @@ AND ca.utc_timestamp < @end_date'; DELETE FROM ' + @log_table_severe_errors + ' WHERE collection_time < @cleanup_date;'; - IF @debug = 1 BEGIN PRINT @dsql; END; + IF @debug = 1 + BEGIN + PRINT @dsql; + END; EXECUTE sys.sp_executesql @dsql, @@ -1266,6 +1269,16 @@ AND ca.utc_timestamp < @end_date'; memory_node_oom xml NOT NULL ); + CREATE TABLE + #health_findings + ( + id integer IDENTITY PRIMARY KEY CLUSTERED, + check_id integer NOT NULL, + finding_group nvarchar(100) NULL, + finding nvarchar(4000) NULL, + sort_order bigint NULL + ); + /*The more you ignore waits, the worser they get*/ IF @what_to_check IN ('all', 'waits') BEGIN @@ -1842,7 +1855,10 @@ AND ca.utc_timestamp < @end_date'; 'event_time' ); - IF @debug = 1 BEGIN PRINT @mdsql; END; + IF @debug = 1 + BEGIN + PRINT @mdsql; + END; EXECUTE sys.sp_executesql @mdsql, @@ -1889,7 +1905,10 @@ AND ca.utc_timestamp < @end_date'; )' + @dsql; - IF @debug = 1 BEGIN PRINT @insert_sql; END; + IF @debug = 1 + BEGIN + PRINT @insert_sql; + END; EXECUTE sys.sp_executesql @insert_sql, @@ -1899,9 +1918,11 @@ AND ca.utc_timestamp < @end_date'; /* Execute the query for client results */ IF @log_to_table = 0 - BEGIN - - IF @debug = 1 BEGIN PRINT @dsql; END; + BEGIN + IF @debug = 1 + BEGIN + PRINT @dsql; + END; EXECUTE sys.sp_executesql @dsql; @@ -2098,7 +2119,10 @@ AND ca.utc_timestamp < @end_date'; 'event_time_rounded' ); - IF @debug = 1 BEGIN PRINT @mdsql; END; + IF @debug = 1 + BEGIN + PRINT @mdsql; + END; EXECUTE sys.sp_executesql @mdsql, @@ -2144,7 +2168,10 @@ AND ca.utc_timestamp < @end_date'; )' + @dsql; - IF @debug = 1 BEGIN PRINT @insert_sql; END; + IF @debug = 1 + BEGIN + PRINT @insert_sql; + END; EXECUTE sys.sp_executesql @insert_sql, @@ -2155,7 +2182,10 @@ AND ca.utc_timestamp < @end_date'; /* Execute the query for client results */ IF @log_to_table = 0 BEGIN - IF @debug = 1 BEGIN PRINT @dsql; END; + IF @debug = 1 + BEGIN + PRINT @dsql; + END; EXECUTE sys.sp_executesql @dsql; @@ -2386,7 +2416,10 @@ AND ca.utc_timestamp < @end_date'; 'event_time_rounded' ); - IF @debug = 1 BEGIN PRINT @mdsql; END; + IF @debug = 1 + BEGIN + PRINT @mdsql; + END; EXECUTE sys.sp_executesql @mdsql, @@ -2430,7 +2463,10 @@ AND ca.utc_timestamp < @end_date'; )' + @dsql; - IF @debug = 1 BEGIN PRINT @insert_sql; END; + IF @debug = 1 + BEGIN + PRINT @insert_sql; + END; EXECUTE sys.sp_executesql @insert_sql, @@ -2441,7 +2477,10 @@ AND ca.utc_timestamp < @end_date'; /* Execute the query for client results */ IF @log_to_table = 0 BEGIN - IF @debug = 1 BEGIN PRINT @dsql; END; + IF @debug = 1 + BEGIN + PRINT @dsql; + END; EXECUTE sys.sp_executesql @dsql; @@ -2596,7 +2635,10 @@ AND ca.utc_timestamp < @end_date'; 'event_time' ); - IF @debug = 1 BEGIN PRINT @mdsql; END; + IF @debug = 1 + BEGIN + PRINT @mdsql; + END; EXECUTE sys.sp_executesql @mdsql, @@ -2644,7 +2686,10 @@ AND ca.utc_timestamp < @end_date'; )' + @dsql; - IF @debug = 1 BEGIN PRINT @insert_sql; END; + IF @debug = 1 + BEGIN + PRINT @insert_sql; + END; EXECUTE sys.sp_executesql @insert_sql, @@ -2655,7 +2700,10 @@ AND ca.utc_timestamp < @end_date'; /* Execute the query for client results */ IF @log_to_table = 0 BEGIN - IF @debug = 1 BEGIN PRINT @dsql; END; + IF @debug = 1 + BEGIN + PRINT @dsql; + END; EXECUTE sys.sp_executesql @dsql; @@ -2785,7 +2833,10 @@ END; 'event_time' ); - IF @debug = 1 BEGIN PRINT @mdsql; END; + IF @debug = 1 + BEGIN + PRINT @mdsql; + END; EXECUTE sys.sp_executesql @mdsql, @@ -2837,7 +2888,10 @@ END; )' + @dsql; - IF @debug = 1 BEGIN PRINT @insert_sql; END; + IF @debug = 1 + BEGIN + PRINT @insert_sql; + END; EXECUTE sys.sp_executesql @insert_sql, @@ -2848,7 +2902,10 @@ END; /* Execute the query for client results */ IF @log_to_table = 0 BEGIN - IF @debug = 1 BEGIN PRINT @dsql; END; + IF @debug = 1 + BEGIN + PRINT @dsql; + END; EXECUTE sys.sp_executesql @dsql; @@ -3013,7 +3070,10 @@ END; 'event_time' ); - IF @debug = 1 BEGIN PRINT @mdsql; END; + IF @debug = 1 + BEGIN + PRINT @mdsql; + END; EXECUTE sys.sp_executesql @mdsql, @@ -3087,7 +3147,10 @@ END; + @dsql; - IF @debug = 1 BEGIN PRINT @insert_sql; END; + IF @debug = 1 + BEGIN + PRINT @insert_sql; + END; EXECUTE sys.sp_executesql @insert_sql, @@ -3098,7 +3161,10 @@ END; /* Execute the query for client results */ IF @log_to_table = 0 BEGIN - IF @debug = 1 BEGIN PRINT @dsql; END; + IF @debug = 1 + BEGIN + PRINT @dsql; + END; EXECUTE sys.sp_executesql @dsql; @@ -3247,7 +3313,10 @@ BEGIN 'event_time' ); - IF @debug = 1 BEGIN PRINT @mdsql; END; + IF @debug = 1 + BEGIN + PRINT @mdsql; + END; EXECUTE sys.sp_executesql @mdsql, @@ -3296,7 +3365,10 @@ BEGIN )' + @dsql; - IF @debug = 1 BEGIN PRINT @insert_sql; END; + IF @debug = 1 + BEGIN + PRINT @insert_sql; + END; EXECUTE sys.sp_executesql @insert_sql, @@ -3307,7 +3379,10 @@ BEGIN /* Execute the query for client results */ IF @log_to_table = 0 BEGIN - IF @debug = 1 BEGIN PRINT @dsql; END; + IF @debug = 1 + BEGIN + PRINT @dsql; + END; EXECUTE sys.sp_executesql @dsql; @@ -3519,7 +3594,10 @@ END; 'event_time' ); - IF @debug = 1 BEGIN PRINT @mdsql; END; + IF @debug = 1 + BEGIN + PRINT @mdsql; + END; EXECUTE sys.sp_executesql @mdsql, @@ -3570,7 +3648,10 @@ END; )' + @dsql; - IF @debug = 1 BEGIN PRINT @insert_sql; END; + IF @debug = 1 + BEGIN + PRINT @insert_sql; + END; EXECUTE sys.sp_executesql @insert_sql, @@ -3581,7 +3662,10 @@ END; /* Execute the query for client results */ IF @log_to_table = 0 BEGIN - IF @debug = 1 BEGIN PRINT @dsql; END; + IF @debug = 1 + BEGIN + PRINT @dsql; + END; EXECUTE sys.sp_executesql @dsql; @@ -3716,7 +3800,10 @@ END; 'event_time' ); - IF @debug = 1 BEGIN PRINT @mdsql; END; + IF @debug = 1 + BEGIN + PRINT @mdsql; + END; EXECUTE sys.sp_executesql @mdsql, @@ -3773,7 +3860,10 @@ END; )' + @dsql; - IF @debug = 1 BEGIN PRINT @insert_sql; END; + IF @debug = 1 + BEGIN + PRINT @insert_sql; + END; EXECUTE sys.sp_executesql @insert_sql, @@ -3784,7 +3874,10 @@ END; /* Execute the query for client results */ IF @log_to_table = 0 BEGIN - IF @debug = 1 BEGIN PRINT @dsql; END; + IF @debug = 1 + BEGIN + PRINT @dsql; + END; EXECUTE sys.sp_executesql @dsql; @@ -3934,7 +4027,10 @@ END; 'event_time' ); - IF @debug = 1 BEGIN PRINT @mdsql; END; + IF @debug = 1 + BEGIN + PRINT @mdsql; + END; EXECUTE sys.sp_executesql @mdsql, @@ -3984,7 +4080,10 @@ END; )' + @dsql; - IF @debug = 1 BEGIN PRINT @insert_sql; END; + IF @debug = 1 + BEGIN + PRINT @insert_sql; + END; EXECUTE sys.sp_executesql @insert_sql, @@ -3995,7 +4094,10 @@ END; /* Execute the query for client results */ IF @log_to_table = 0 BEGIN - IF @debug = 1 BEGIN PRINT @dsql; END; + IF @debug = 1 + BEGIN + PRINT @dsql; + END; EXECUTE sys.sp_executesql @dsql; @@ -4128,7 +4230,10 @@ END; 'event_time' ); - IF @debug = 1 BEGIN PRINT @mdsql; END; + IF @debug = 1 + BEGIN + PRINT @mdsql; + END; EXECUTE sys.sp_executesql @mdsql, @@ -4177,7 +4282,10 @@ END; )' + @dsql; - IF @debug = 1 BEGIN PRINT @insert_sql; END; + IF @debug = 1 + BEGIN + PRINT @insert_sql; + END; EXECUTE sys.sp_executesql @insert_sql, @@ -4188,7 +4296,10 @@ END; /* Execute the query for client results */ IF @log_to_table = 0 BEGIN - IF @debug = 1 BEGIN PRINT @dsql; END; + IF @debug = 1 + BEGIN + PRINT @dsql; + END; EXECUTE sys.sp_executesql @dsql; @@ -5272,39 +5383,32 @@ END; /* This section generates a summary table with key findings from all health check sections */ - IF @debug = 1 - BEGIN - RAISERROR('Creating health summary findings', 0, 1) WITH NOWAIT; - END; - - CREATE TABLE - #health_findings - ( - id integer IDENTITY PRIMARY KEY CLUSTERED, - check_id integer NOT NULL, - finding_group nvarchar(100) NULL, - finding nvarchar(4000) NULL, - sort_order bigint - ); - -- Significant waits findings IF @what_to_check IN ('all', 'waits') AND EXISTS (SELECT 1/0 FROM #wait_info AS wi WHERE wait_duration_ms > 1000) BEGIN - INSERT INTO #health_findings (check_id, finding_group, finding, sort_order) + INSERT INTO + #health_findings + ( + check_id, + finding_group, + finding, + sort_order + ) SELECT - 1 AS check_id, - N'waits' AS finding_group, + check_id = 1, + finding_group = N'waits', finding = N'Long waits detected: ' + - CONVERT(nvarchar(10), COUNT_BIG(CASE WHEN wait_duration_ms > 30000 THEN 1 END)) + N' critical, ' + - CONVERT(nvarchar(10), COUNT_BIG(CASE WHEN wait_duration_ms BETWEEN 10000 AND 30000 THEN 1 END)) + N' high, ' + - CONVERT(nvarchar(10), COUNT_BIG(CASE WHEN wait_duration_ms BETWEEN 5000 AND 10000 THEN 1 END)) + N' medium. ' + + CONVERT(nvarchar(10), COUNT_BIG(CASE WHEN wait_duration_ms > 30000 THEN 1 END)) + N' critical, ' + + CONVERT(nvarchar(10), COUNT_BIG(CASE WHEN wait_duration_ms BETWEEN 10000 AND 30000 THEN 1 END)) + N' high, ' + + CONVERT(nvarchar(10), COUNT_BIG(CASE WHEN wait_duration_ms BETWEEN 5000 AND 10000 THEN 1 END)) + N' medium. ' + N'Longest wait: ' + CONVERT(nvarchar(10), MAX(wait_duration_ms)/1000.0) + N' seconds (' + (SELECT TOP 1 wait_type FROM #wait_info AS wi2 ORDER BY wait_duration_ms DESC) + N')', - sort_order = CASE - WHEN MAX(wait_duration_ms) > 30000 THEN 100 -- Critical - WHEN MAX(wait_duration_ms) > 10000 THEN 200 -- High - WHEN MAX(wait_duration_ms) > 5000 THEN 300 -- Medium + sort_order = + CASE + WHEN MAX(wait_duration_ms) > 30000 THEN 100 -- Critical + WHEN MAX(wait_duration_ms) > 10000 THEN 200 -- High + WHEN MAX(wait_duration_ms) > 5000 THEN 300 -- Medium ELSE 400 -- Low END FROM #wait_info AS wi From a276e850ce4291de9ff0c2a009ab6ce4d19d0855 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 13 Mar 2025 18:27:52 -0400 Subject: [PATCH 079/246] Update sp_HealthParser.sql --- sp_HealthParser/sp_HealthParser.sql | 42 ++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/sp_HealthParser/sp_HealthParser.sql b/sp_HealthParser/sp_HealthParser.sql index d00b9ad5..b989c43d 100644 --- a/sp_HealthParser/sp_HealthParser.sql +++ b/sp_HealthParser/sp_HealthParser.sql @@ -5403,7 +5403,8 @@ END; CONVERT(nvarchar(10), COUNT_BIG(CASE WHEN wait_duration_ms BETWEEN 10000 AND 30000 THEN 1 END)) + N' high, ' + CONVERT(nvarchar(10), COUNT_BIG(CASE WHEN wait_duration_ms BETWEEN 5000 AND 10000 THEN 1 END)) + N' medium. ' + N'Longest wait: ' + CONVERT(nvarchar(10), MAX(wait_duration_ms)/1000.0) + N' seconds (' + - (SELECT TOP 1 wait_type FROM #wait_info AS wi2 ORDER BY wait_duration_ms DESC) + N')', + (SELECT TOP 1 wait_type FROM #wait_info AS wi2 ORDER BY wait_duration_ms DESC) + N'). ' + + N'Occurred at: ' + CONVERT(nvarchar(30), (SELECT TOP 1 event_time FROM #wait_info AS wi3 ORDER BY wait_duration_ms DESC), 120), sort_order = CASE WHEN MAX(wait_duration_ms) > 30000 THEN 100 -- Critical @@ -5432,7 +5433,8 @@ END; THEN N'Possible hung task detected. ' ELSE N'' END + - N'Longest running task: ' + CONVERT(nvarchar(10), MAX(wait_duration_ms)/1000.0) + N' seconds', + N'Longest running task: ' + CONVERT(nvarchar(10), MAX(wait_duration_ms)/1000.0) + N' seconds. ' + + N'Occurred at: ' + CONVERT(nvarchar(30), (SELECT TOP 1 event_time FROM #cpu_info AS ci2 ORDER BY wait_duration_ms DESC), 120), sort_order = CASE WHEN MAX(wait_duration_ms) > 300000 THEN 110 -- Critical WHEN MAX(wait_duration_ms) > 180000 THEN 210 -- High @@ -5461,9 +5463,14 @@ END; END + CASE WHEN COUNT_BIG(CASE WHEN notification_type LIKE N'%RESOURCE_MEM_HIGH%' THEN 1 END) > 0 - THEN N'SQL Server memory pressure detected.' + THEN N'SQL Server memory pressure detected. ' ELSE N'' - END, + END + + N'Last detected: ' + CONVERT(nvarchar(30), (SELECT TOP 1 event_time FROM #memory_conditions AS mc2 + ORDER BY CASE WHEN notification_type LIKE N'%CRITICAL%' THEN 1 + WHEN notification_type LIKE N'%HIGH%' THEN 2 + ELSE 3 END, + event_time DESC), 120), sort_order = CASE WHEN COUNT_BIG(CASE WHEN notification_type LIKE N'%CRITICAL%' THEN 1 END) > 0 THEN 120 -- Critical WHEN COUNT_BIG(CASE WHEN notification_type LIKE N'%HIGH%' THEN 1 END) > 0 THEN 220 -- High @@ -5486,7 +5493,8 @@ END; N'Maximum requested: ' + MAX(memory_requested_gb) + N' GB. ' + N'Most common clerk: ' + (SELECT TOP 1 memory_clerk_name FROM #memory_broker AS mb2 - GROUP BY memory_clerk_name ORDER BY COUNT_BIG(*) DESC), + GROUP BY memory_clerk_name ORDER BY COUNT_BIG(*) DESC) + + N'. Most recent event: ' + CONVERT(nvarchar(30), (SELECT TOP 1 event_time FROM #memory_broker AS mb3 ORDER BY event_time DESC), 120), sort_order = 330 -- Medium priority for broker events FROM #memory_broker AS mb HAVING COUNT_BIG(*) > 0; @@ -5508,7 +5516,8 @@ END; ) + N'. Most common clerk: ' + (SELECT TOP 1 memory_clerk_name FROM #memory_node_oom AS mno3 - GROUP BY memory_clerk_name ORDER BY COUNT_BIG(*) DESC), + GROUP BY memory_clerk_name ORDER BY COUNT_BIG(*) DESC) + + N'. Most recent OOM: ' + CONVERT(nvarchar(30), (SELECT TOP 1 event_time FROM #memory_node_oom AS mno4 ORDER BY event_time DESC), 120), sort_order = 130 -- Critical for OOM events FROM #memory_node_oom AS mno HAVING COUNT_BIG(*) > 0; @@ -5529,7 +5538,9 @@ END; N'File: ' + (SELECT TOP 1 longestPendingRequests_filePath FROM #io_issues AS io2 - ORDER BY TRY_CONVERT(bigint, longestPendingRequests_duration_ms) DESC), + ORDER BY TRY_CONVERT(bigint, longestPendingRequests_duration_ms) DESC) + + N'. Detected at: ' + CONVERT(nvarchar(30), (SELECT TOP 1 event_time FROM #io_issues AS io3 + ORDER BY TRY_CONVERT(bigint, longestPendingRequests_duration_ms) DESC), 120), sort_order = CASE WHEN MAX(TRY_CONVERT(bigint, longestPendingRequests_duration_ms)) > 15000 THEN 140 -- Critical WHEN MAX(TRY_CONVERT(bigint, longestPendingRequests_duration_ms)) > 5000 THEN 240 -- High @@ -5553,7 +5564,14 @@ END; CONVERT(nvarchar(10), SUM(CASE WHEN nonYieldingTasksReported > 0 THEN nonYieldingTasksReported ELSE 0 END)) + N' non-yielding tasks, ' + CONVERT(nvarchar(10), SUM(CASE WHEN isAccessViolationOccurred > 0 THEN isAccessViolationOccurred ELSE 0 END)) + N' access violations. ' + N'CPU utilization - SQL: ' + CONVERT(nvarchar(10), MAX(sqlCpuUtilization)) + N'%, ' + - N'System: ' + CONVERT(nvarchar(10), MAX(systemCpuUtilization)) + N'%', + N'System: ' + CONVERT(nvarchar(10), MAX(systemCpuUtilization)) + N'%. ' + + N'Recorded at: ' + CONVERT(nvarchar(30), (SELECT TOP 1 event_time FROM #system_health AS sh2 + ORDER BY CASE + WHEN isAccessViolationOccurred > 0 THEN 1 + WHEN nonYieldingTasksReported > 0 THEN 2 + WHEN sqlCpuUtilization > 80 THEN 3 + ELSE 4 END, + event_time DESC), 120), sort_order = CASE WHEN SUM(CASE WHEN isAccessViolationOccurred > 0 THEN 1 ELSE 0 END) > 0 THEN 150 -- Critical WHEN SUM(CASE WHEN nonYieldingTasksReported > 0 THEN 1 ELSE 0 END) > 0 THEN 250 -- High @@ -5582,7 +5600,9 @@ END; N' (' + (SELECT TOP 1 LEFT(message, 50) + CASE WHEN LEN(message) > 50 THEN N'...' ELSE N'' END FROM #error_info AS ei3 ORDER BY severity DESC, event_time DESC) + - N')', + N'). ' + + N'Most recent error: ' + + CONVERT(nvarchar(30), (SELECT TOP 1 event_time FROM #error_info AS ei4 ORDER BY severity DESC, event_time DESC), 120), sort_order = CASE WHEN COUNT_BIG(CASE WHEN severity >= 22 THEN 1 END) > 0 THEN 160 -- Fatal WHEN COUNT_BIG(CASE WHEN severity BETWEEN 20 AND 21 THEN 1 END) > 0 THEN 260 -- Critical @@ -5605,7 +5625,9 @@ END; CONVERT(nvarchar(10), COUNT_BIG(*)) + N' scheduler events detected. ' + CONVERT(nvarchar(10), COUNT_BIG(CASE WHEN is_online = 0 THEN 1 END)) + N' schedulers offline, ' + CONVERT(nvarchar(10), COUNT_BIG(CASE WHEN is_running = 0 AND is_online = 1 THEN 1 END)) + N' schedulers not running. ' + - N'Non-yielding time: Max ' + CONVERT(nvarchar(10), MAX(TRY_CONVERT(bigint, non_yielding_time_ms))) + N' ms', + N'Non-yielding time: Max ' + CONVERT(nvarchar(10), MAX(TRY_CONVERT(bigint, non_yielding_time_ms))) + N' ms. ' + + N'Last detected: ' + CONVERT(nvarchar(30), (SELECT TOP 1 event_time FROM #scheduler_issues AS si2 + ORDER BY TRY_CONVERT(bigint, non_yielding_time_ms) DESC), 120), sort_order = CASE WHEN MAX(TRY_CONVERT(bigint, non_yielding_time_ms)) > 60000 THEN 170 -- Critical WHEN COUNT_BIG(CASE WHEN is_online = 0 THEN 1 END) > 0 THEN 270 -- High From ff7a95139f4283dc33228925d70c532392af7c32 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 13 Mar 2025 18:33:21 -0400 Subject: [PATCH 080/246] Update sp_HealthParser.sql --- sp_HealthParser/sp_HealthParser.sql | 290 ---------------------------- 1 file changed, 290 deletions(-) diff --git a/sp_HealthParser/sp_HealthParser.sql b/sp_HealthParser/sp_HealthParser.sql index b989c43d..3aa1e66c 100644 --- a/sp_HealthParser/sp_HealthParser.sql +++ b/sp_HealthParser/sp_HealthParser.sql @@ -1269,16 +1269,6 @@ AND ca.utc_timestamp < @end_date'; memory_node_oom xml NOT NULL ); - CREATE TABLE - #health_findings - ( - id integer IDENTITY PRIMARY KEY CLUSTERED, - check_id integer NOT NULL, - finding_group nvarchar(100) NULL, - finding nvarchar(4000) NULL, - sort_order bigint NULL - ); - /*The more you ignore waits, the worser they get*/ IF @what_to_check IN ('all', 'waits') BEGIN @@ -5379,285 +5369,5 @@ END; END; END; END; /*End locks*/ - - /* - This section generates a summary table with key findings from all health check sections - */ - -- Significant waits findings - IF @what_to_check IN ('all', 'waits') - AND EXISTS (SELECT 1/0 FROM #wait_info AS wi WHERE wait_duration_ms > 1000) - BEGIN - INSERT INTO - #health_findings - ( - check_id, - finding_group, - finding, - sort_order - ) - SELECT - check_id = 1, - finding_group = N'waits', - finding = N'Long waits detected: ' + - CONVERT(nvarchar(10), COUNT_BIG(CASE WHEN wait_duration_ms > 30000 THEN 1 END)) + N' critical, ' + - CONVERT(nvarchar(10), COUNT_BIG(CASE WHEN wait_duration_ms BETWEEN 10000 AND 30000 THEN 1 END)) + N' high, ' + - CONVERT(nvarchar(10), COUNT_BIG(CASE WHEN wait_duration_ms BETWEEN 5000 AND 10000 THEN 1 END)) + N' medium. ' + - N'Longest wait: ' + CONVERT(nvarchar(10), MAX(wait_duration_ms)/1000.0) + N' seconds (' + - (SELECT TOP 1 wait_type FROM #wait_info AS wi2 ORDER BY wait_duration_ms DESC) + N'). ' + - N'Occurred at: ' + CONVERT(nvarchar(30), (SELECT TOP 1 event_time FROM #wait_info AS wi3 ORDER BY wait_duration_ms DESC), 120), - sort_order = - CASE - WHEN MAX(wait_duration_ms) > 30000 THEN 100 -- Critical - WHEN MAX(wait_duration_ms) > 10000 THEN 200 -- High - WHEN MAX(wait_duration_ms) > 5000 THEN 300 -- Medium - ELSE 400 -- Low - END - FROM #wait_info AS wi - WHERE wait_duration_ms > 1000 -- Only include significant waits - HAVING COUNT_BIG(*) > 0; - END; - - -- CPU tasks findings - IF @what_to_check IN ('all', 'cpu') - AND EXISTS (SELECT 1/0 FROM #cpu_info) - BEGIN - INSERT INTO #health_findings (check_id, finding_group, finding, sort_order) - SELECT - 2 AS check_id, - N'cpu' AS finding_group, - finding = N'CPU pressure detected: ' + - CONVERT(nvarchar(10), COUNT_BIG(CASE WHEN wait_duration_ms > 180000 THEN 1 END)) + N' long-running tasks, ' + - CONVERT(nvarchar(10), COUNT_BIG(CASE WHEN wait_type = N'SOS_SCHEDULER_YIELD' THEN 1 END)) + N' scheduler yields, ' + - CASE - WHEN MAX(percent_complete) < 10 AND MAX(wait_duration_ms) > 300000 - THEN N'Possible hung task detected. ' - ELSE N'' - END + - N'Longest running task: ' + CONVERT(nvarchar(10), MAX(wait_duration_ms)/1000.0) + N' seconds. ' + - N'Occurred at: ' + CONVERT(nvarchar(30), (SELECT TOP 1 event_time FROM #cpu_info AS ci2 ORDER BY wait_duration_ms DESC), 120), - sort_order = CASE - WHEN MAX(wait_duration_ms) > 300000 THEN 110 -- Critical - WHEN MAX(wait_duration_ms) > 180000 THEN 210 -- High - WHEN MAX(wait_duration_ms) > 60000 THEN 310 -- Medium - ELSE 410 -- Low - END - FROM #cpu_info AS ci - HAVING COUNT_BIG(*) > 0; - END; - - -- Memory conditions findings - IF @what_to_check IN ('all', 'memory') - AND EXISTS (SELECT 1/0 FROM #memory_conditions) - BEGIN - INSERT INTO #health_findings (check_id, finding_group, finding, sort_order) - SELECT - 3 AS check_id, - N'memory' AS finding_group, - finding = N'Memory pressure: ' + - CONVERT(nvarchar(10), COUNT_BIG(CASE WHEN notification_type LIKE N'%CRITICAL%' THEN 1 END)) + N' critical, ' + - CONVERT(nvarchar(10), COUNT_BIG(CASE WHEN notification_type LIKE N'%HIGH%' THEN 1 END)) + N' high alerts. ' + - CASE - WHEN COUNT_BIG(CASE WHEN notification_type LIKE N'%RESOURCE_MEMPHYSICAL_HIGH%' THEN 1 END) > 0 - THEN N'Physical memory pressure detected. ' - ELSE N'' - END + - CASE - WHEN COUNT_BIG(CASE WHEN notification_type LIKE N'%RESOURCE_MEM_HIGH%' THEN 1 END) > 0 - THEN N'SQL Server memory pressure detected. ' - ELSE N'' - END + - N'Last detected: ' + CONVERT(nvarchar(30), (SELECT TOP 1 event_time FROM #memory_conditions AS mc2 - ORDER BY CASE WHEN notification_type LIKE N'%CRITICAL%' THEN 1 - WHEN notification_type LIKE N'%HIGH%' THEN 2 - ELSE 3 END, - event_time DESC), 120), - sort_order = CASE - WHEN COUNT_BIG(CASE WHEN notification_type LIKE N'%CRITICAL%' THEN 1 END) > 0 THEN 120 -- Critical - WHEN COUNT_BIG(CASE WHEN notification_type LIKE N'%HIGH%' THEN 1 END) > 0 THEN 220 -- High - ELSE 320 -- Medium - END - FROM #memory_conditions AS mc - HAVING COUNT_BIG(*) > 0; - END; - - -- Memory broker findings - IF @what_to_check IN ('all', 'memory') - AND EXISTS (SELECT 1/0 FROM #memory_broker) - BEGIN - INSERT INTO #health_findings (check_id, finding_group, finding, sort_order) - SELECT - 4 AS check_id, - N'memory' AS finding_group, - finding = N'Memory broker: ' + - CONVERT(nvarchar(10), COUNT_BIG(*)) + N' memory requests, ' + - N'Maximum requested: ' + MAX(memory_requested_gb) + N' GB. ' + - N'Most common clerk: ' + - (SELECT TOP 1 memory_clerk_name FROM #memory_broker AS mb2 - GROUP BY memory_clerk_name ORDER BY COUNT_BIG(*) DESC) + - N'. Most recent event: ' + CONVERT(nvarchar(30), (SELECT TOP 1 event_time FROM #memory_broker AS mb3 ORDER BY event_time DESC), 120), - sort_order = 330 -- Medium priority for broker events - FROM #memory_broker AS mb - HAVING COUNT_BIG(*) > 0; - END; - - -- Memory node OOM findings - IF @what_to_check IN ('all', 'memory') - AND EXISTS (SELECT 1/0 FROM #memory_node_oom) - BEGIN - INSERT INTO #health_findings (check_id, finding_group, finding, sort_order) - SELECT - 5 AS check_id, - N'memory' AS finding_group, - finding = N'Memory node OOM: ' + - CONVERT(nvarchar(10), COUNT_BIG(*)) + N' out-of-memory events detected. ' + - N'Most affected node ID: ' + CONVERT(nvarchar(10), - (SELECT TOP 1 node_id FROM #memory_node_oom AS mno2 - GROUP BY node_id ORDER BY COUNT_BIG(*) DESC) - ) + - N'. Most common clerk: ' + - (SELECT TOP 1 memory_clerk_name FROM #memory_node_oom AS mno3 - GROUP BY memory_clerk_name ORDER BY COUNT_BIG(*) DESC) + - N'. Most recent OOM: ' + CONVERT(nvarchar(30), (SELECT TOP 1 event_time FROM #memory_node_oom AS mno4 ORDER BY event_time DESC), 120), - sort_order = 130 -- Critical for OOM events - FROM #memory_node_oom AS mno - HAVING COUNT_BIG(*) > 0; - END; - - -- IO issues findings - IF @what_to_check IN ('all', 'disk') - AND EXISTS (SELECT 1/0 FROM #io_issues) - BEGIN - INSERT INTO #health_findings (check_id, finding_group, finding, sort_order) - SELECT - 6 AS check_id, - N'io' AS finding_group, - finding = N'IO issues: ' + - CONVERT(nvarchar(10), SUM(intervalLongIos)) + N' long IOs detected, ' + - CONVERT(nvarchar(10), SUM(ioLatchTimeouts)) + N' IO latch timeouts. ' + - N'Longest pending IO: ' + MAX(longestPendingRequests_duration_ms) + N' ms, ' + - N'File: ' + - (SELECT TOP 1 longestPendingRequests_filePath - FROM #io_issues AS io2 - ORDER BY TRY_CONVERT(bigint, longestPendingRequests_duration_ms) DESC) + - N'. Detected at: ' + CONVERT(nvarchar(30), (SELECT TOP 1 event_time FROM #io_issues AS io3 - ORDER BY TRY_CONVERT(bigint, longestPendingRequests_duration_ms) DESC), 120), - sort_order = CASE - WHEN MAX(TRY_CONVERT(bigint, longestPendingRequests_duration_ms)) > 15000 THEN 140 -- Critical - WHEN MAX(TRY_CONVERT(bigint, longestPendingRequests_duration_ms)) > 5000 THEN 240 -- High - ELSE 340 -- Medium - END - FROM #io_issues AS io - HAVING COUNT_BIG(*) > 0; - END; - - -- System health findings - IF @what_to_check IN ('all', 'system') - AND EXISTS (SELECT 1/0 FROM #system_health) - BEGIN - INSERT INTO #health_findings (check_id, finding_group, finding, sort_order) - SELECT - 7 AS check_id, - N'health' AS finding_group, - finding = N'System health issues: ' + - CONVERT(nvarchar(10), SUM(CASE WHEN spinlockBackoffs > 0 THEN spinlockBackoffs ELSE 0 END)) + N' spinlock backoffs, ' + - CONVERT(nvarchar(10), SUM(CASE WHEN latchWarnings > 0 THEN latchWarnings ELSE 0 END)) + N' latch warnings, ' + - CONVERT(nvarchar(10), SUM(CASE WHEN nonYieldingTasksReported > 0 THEN nonYieldingTasksReported ELSE 0 END)) + N' non-yielding tasks, ' + - CONVERT(nvarchar(10), SUM(CASE WHEN isAccessViolationOccurred > 0 THEN isAccessViolationOccurred ELSE 0 END)) + N' access violations. ' + - N'CPU utilization - SQL: ' + CONVERT(nvarchar(10), MAX(sqlCpuUtilization)) + N'%, ' + - N'System: ' + CONVERT(nvarchar(10), MAX(systemCpuUtilization)) + N'%. ' + - N'Recorded at: ' + CONVERT(nvarchar(30), (SELECT TOP 1 event_time FROM #system_health AS sh2 - ORDER BY CASE - WHEN isAccessViolationOccurred > 0 THEN 1 - WHEN nonYieldingTasksReported > 0 THEN 2 - WHEN sqlCpuUtilization > 80 THEN 3 - ELSE 4 END, - event_time DESC), 120), - sort_order = CASE - WHEN SUM(CASE WHEN isAccessViolationOccurred > 0 THEN 1 ELSE 0 END) > 0 THEN 150 -- Critical - WHEN SUM(CASE WHEN nonYieldingTasksReported > 0 THEN 1 ELSE 0 END) > 0 THEN 250 -- High - WHEN MAX(sqlCpuUtilization) > 80 THEN 350 -- Medium - ELSE 450 -- Low - END - FROM #system_health AS sh - HAVING COUNT_BIG(*) > 0; - END; - - -- Severe errors findings - IF @what_to_check IN ('all', 'system') - AND EXISTS (SELECT 1/0 FROM #error_info) - BEGIN - INSERT INTO #health_findings (check_id, finding_group, finding, sort_order) - SELECT - 8 AS check_id, - N'errors' AS finding_group, - finding = N'Severe errors: ' + - CONVERT(nvarchar(10), COUNT_BIG(CASE WHEN severity >= 22 THEN 1 END)) + N' fatal, ' + - CONVERT(nvarchar(10), COUNT_BIG(CASE WHEN severity BETWEEN 20 AND 21 THEN 1 END)) + N' critical, ' + - CONVERT(nvarchar(10), COUNT_BIG(CASE WHEN severity BETWEEN 16 AND 19 THEN 1 END)) + N' high severity errors. ' + - N'Most common error: ' + - CONVERT(nvarchar(10), (SELECT TOP 1 error_number FROM #error_info AS ei2 - GROUP BY error_number ORDER BY COUNT_BIG(*) DESC)) + - N' (' + - (SELECT TOP 1 LEFT(message, 50) + CASE WHEN LEN(message) > 50 THEN N'...' ELSE N'' END - FROM #error_info AS ei3 ORDER BY severity DESC, event_time DESC) + - N'). ' + - N'Most recent error: ' + - CONVERT(nvarchar(30), (SELECT TOP 1 event_time FROM #error_info AS ei4 ORDER BY severity DESC, event_time DESC), 120), - sort_order = CASE - WHEN COUNT_BIG(CASE WHEN severity >= 22 THEN 1 END) > 0 THEN 160 -- Fatal - WHEN COUNT_BIG(CASE WHEN severity BETWEEN 20 AND 21 THEN 1 END) > 0 THEN 260 -- Critical - WHEN COUNT_BIG(CASE WHEN severity BETWEEN 16 AND 19 THEN 1 END) > 0 THEN 360 -- High - ELSE 460 -- Medium - END - FROM #error_info AS ei - HAVING COUNT_BIG(*) > 0; - END; - - -- Scheduler issues findings - IF @what_to_check IN ('all', 'cpu') - AND EXISTS (SELECT 1/0 FROM #scheduler_issues) - BEGIN - INSERT INTO #health_findings (check_id, finding_group, finding, sort_order) - SELECT - 9 AS check_id, - N'cpu' AS finding_group, - finding = N'Scheduler issues: ' + - CONVERT(nvarchar(10), COUNT_BIG(*)) + N' scheduler events detected. ' + - CONVERT(nvarchar(10), COUNT_BIG(CASE WHEN is_online = 0 THEN 1 END)) + N' schedulers offline, ' + - CONVERT(nvarchar(10), COUNT_BIG(CASE WHEN is_running = 0 AND is_online = 1 THEN 1 END)) + N' schedulers not running. ' + - N'Non-yielding time: Max ' + CONVERT(nvarchar(10), MAX(TRY_CONVERT(bigint, non_yielding_time_ms))) + N' ms. ' + - N'Last detected: ' + CONVERT(nvarchar(30), (SELECT TOP 1 event_time FROM #scheduler_issues AS si2 - ORDER BY TRY_CONVERT(bigint, non_yielding_time_ms) DESC), 120), - sort_order = CASE - WHEN MAX(TRY_CONVERT(bigint, non_yielding_time_ms)) > 60000 THEN 170 -- Critical - WHEN COUNT_BIG(CASE WHEN is_online = 0 THEN 1 END) > 0 THEN 270 -- High - WHEN COUNT_BIG(CASE WHEN is_running = 0 AND is_online = 1 THEN 1 END) > 0 THEN 370 -- Medium - ELSE 470 -- Low - END - FROM #scheduler_issues AS si - HAVING COUNT_BIG(*) > 0; - END; - - -- Return the findings summary, ordered by severity - IF EXISTS (SELECT 1/0 FROM #health_findings) - BEGIN - IF @debug = 1 - BEGIN - RAISERROR('Returning health findings summary', 0, 1) WITH NOWAIT; - END; - - SELECT - finding_group, - finding - FROM #health_findings AS hf - ORDER BY sort_order, check_id; - END; - ELSE - BEGIN - SELECT - finding_group = N'No Issues Found', - finding = N'No significant health issues detected in the analyzed time period'; - END; - END; /*Final End*/ GO From 3a126ac0d791fb256a982db3d0e9d8352de80dd5 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 13 Mar 2025 21:00:31 -0400 Subject: [PATCH 081/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index 81933be9..1fba0b19 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -654,7 +654,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. object_id = t.object_id, table_name = t.name, index_id = i.index_id, - index_name = i.name + index_name = ISNULL(i.name, t.table_name + N''.Heap'') FROM ' + QUOTENAME(@database_name) + N'.sys.tables AS t JOIN ' + QUOTENAME(@database_name) + N'.sys.schemas AS s ON t.schema_id = s.schema_id @@ -665,7 +665,6 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND us.database_id = @database_id WHERE t.is_ms_shipped = 0 AND t.type <> N''TF'' - AND i.index_id > 0 AND NOT EXISTS ( SELECT From e386ebb28bcaf2504fe88d5fbe468c77b8666b29 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 13 Mar 2025 21:38:16 -0400 Subject: [PATCH 082/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index 1fba0b19..623aa17d 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -654,7 +654,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. object_id = t.object_id, table_name = t.name, index_id = i.index_id, - index_name = ISNULL(i.name, t.table_name + N''.Heap'') + index_name = ISNULL(i.name, t.name + N''.Heap'') FROM ' + QUOTENAME(@database_name) + N'.sys.tables AS t JOIN ' + QUOTENAME(@database_name) + N'.sys.schemas AS s ON t.schema_id = s.schema_id @@ -921,7 +921,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. os.object_id, table_name = t.name, os.index_id, - index_name = i.name, + index_name = ISNULL(i.name, t.name + N''.Heap''), range_scan_count = SUM(os.range_scan_count), singleton_lookup_count = SUM(os.singleton_lookup_count), forwarded_fetch_count = SUM(os.forwarded_fetch_count), @@ -1074,7 +1074,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. s.schema_id, schema_name = s.name, table_name = t.name, - index_name = i.name, + index_name = ISNULL(i.name, t.name + N''.Heap''), column_name = c.name, i.is_primary_key, i.is_unique, @@ -1326,7 +1326,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. s.schema_id, schema_name = s.name, table_name = t.name, - index_name = i.name, + index_name = ISNULL(i.name, t.name + N''.Heap''), ps.partition_id, p.partition_number, total_rows = ps.row_count, From d6e4365deb36c3814e34807d63c76eafdbd281d8 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 13 Mar 2025 22:28:59 -0400 Subject: [PATCH 083/246] Update .DS_Store --- .DS_Store | Bin 6148 -> 6148 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/.DS_Store b/.DS_Store index 493ca41b2dcd8b5d1a7aa04ab7f393c170b1a4ee..38bdb71467988549740df9c09b5bfc5de765c8ab 100644 GIT binary patch delta 140 zcmZoMXfc=|#>B!ku~2NHo+2a5#(>?7ivyUM7+EI&W764d!NkZY&(BcIP{0t+5Xex< zkjaqDkPReL8G;!~81fm47*Z!&Fr6f%X0sOaamLN;9Q+(WyEZ#Ae`lV|FXG6-$iTqF M00f&OMAk3^00j0P`~Uy| delta 74 zcmZoMXfc=|#>B)qu~2NHo+2aD#(>?7lMO^zCNr{XZnj`yWZdk}wu5nFgBbH>b`E|H cpvujH9N(EI^NTogFaQA~0|U$E2$40+0J)bC>i_@% From a10568ed58e0f2ad67b7e04cd3280a9e2acc6cf8 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 13 Mar 2025 22:33:09 -0400 Subject: [PATCH 084/246] Create CLAUDE.md --- CLAUDE.md | 577 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 577 insertions(+) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..1871aba0 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,577 @@ +# Erik Darling's T-SQL Coding Style Guide + +This document outlines the T-SQL coding style preferences for Erik Darling (Darling Data, LLC) and should be followed when writing or modifying SQL code. + +## General Formatting + +- **Keywords**: All SQL keywords in UPPERCASE (SELECT, FROM, WHERE, JOIN, etc.) +- **Functions**: All SQL functions in UPPERCASE (CONVERT, ISNULL, OBJECT_ID, etc.) +- **Indentation**: 4 spaces for each level of indentation (NEVER use tabs) +- **Line breaks**: Each statement on a new line +- **Spacing**: Consistent spacing around operators (=, <, >, etc.) +- **Block separation**: Empty line between logical code blocks (maximum of two empty lines between statements) +- **Quotes**: Use single quotes for string literals and N-prefix for Unicode strings (N'string') +- **Data types**: Never abbreviate data types (use INTEGER instead of INT) +- **Keywords**: Never abbreviate keywords (use EXECUTE instead of EXEC, TRANSACTION instead of TRAN, PROCEDURE instead of PROC) +- **TOP syntax**: Always include parentheses, as in TOP (100) not TOP 100 +- **Object creation**: Generally use CREATE OR ALTER for objects instead of DROP/CREATE +- **Table aliases**: Tables should always have aliases, even in simple queries +- **Column references**: Always qualify columns with their table alias + +## Comments + +- Always use block comments with /* ... */ for most comments, never use double dash (--) +- Include parameter descriptions as inline comments after parameter definitions +- Use ASCII art for header blocks to visually distinguish sections +- Include copyright and attribution information in header comments +- Prefix code sections with descriptive comments about what the section does +- Use comments to describe: + - New code blocks + - Complex expressions + - Table purposes + - Complex logic + - The logical flow of code + +## Naming Conventions + +- **Parameters**: Prefixed with @ and use snake_case (@database_name, @debug) +- **Variables**: Same as parameters (@database_id, @sql) +- **Temporary Tables**: Prefixed with # and use descriptive snake_case (#filtered_objects) +- **Aliases**: Short, meaningful lowercase names (ap, o, t) +- **Objects**: Use clear, descriptive names + +## Query Structure + +- **SELECT statements**: + - SELECT keyword on first line + - Column list starts on next line, indented + - Leading commas for multi-line column lists + - Columns aligned vertically for readability + - FROM clause on new line at same indent level as SELECT + - Column aliases should always use the pattern: column_name = column_expression + - Example: some_date = DATEADD(DAY, 1, GETDATE()) + - Always terminate queries with a semicolon + +- **Table references**: + - Always use schema prefixes for all objects except temporary objects + - Examples: FROM dbo.objects, FROM tempdb.dbo.objects + - Temporary tables don't need schema: FROM #temp_table + +- **Table aliases**: + - Always use the AS keyword with table aliases: table_name AS alias + - Example: FROM dbo.sys_objects AS o + +- **Windowing functions**: + - Format with OVER on same line as function + - PARTITION BY and ORDER BY on separate lines indented + - Parentheses on their own lines + ```sql + SELECT + n = ROW_NUMBER() OVER + ( + PARTITION BY + column_name + ORDER BY + other_column + ) + ``` + +- **JOIN syntax**: + - Use modern ANSI JOIN syntax (JOIN table ON condition) + - JOIN keyword on new line at same indent level as FROM + - ON conditions indented from JOIN + - JOIN conditions with AND should be aligned like this: + ```sql + FROM dbo.table_a AS a0 + JOIN dbo.table_a AS a1 + ON a0.col = a1.col + AND a0.col = a1.col + ``` + - For correlated queries and joins, the table most recently referenced should come first in the ON clause: + ```sql + FROM first_table AS ft0 + JOIN dbo.first_table AS ft1 + ON ft1.col = ft0.col + ``` + +- **Clauses**: + - GROUP BY, ORDER BY, and HAVING clauses should always begin on a new line, indented four spaces from the main statement + - WHERE clauses with AND conditions should be formatted with AND aligned: + ```sql + WHERE a.col = 1 + AND b.col = 2 + ``` + - EXISTS and NOT EXISTS should use this format with 1/0 in the SELECT: + ```sql + WHERE EXISTS + ( + SELECT + 1/0 + FROM other_table AS ot + WHERE ot.col = t.col + ) + ``` + +- **Subqueries**: + - Subqueries should never be one-liners + - Place on new lines with proper indentation + ```sql + SELECT + column_name = + ( + SELECT + column_name + FROM dbo.table_name AS alias + WHERE condition + ) + ``` + +- **APPLY operators**: + - Format CROSS APPLY and OUTER APPLY with the query on new lines + ```sql + FROM dbo.a_table AS y + CROSS APPLY + ( + SELECT + columns + FROM dbo.table_name AS x + WHERE x.col = y.col + ) AS x + ``` + +- **Set operations**: + - UNION, INTERSECT, EXCEPT should have the operator between statements with blank lines + ```sql + SELECT + columns + FROM dbo.a_table AS a + + EXCEPT + + SELECT + columns + FROM dbo.b_table AS b; + ``` + +- **Table-valued constructors (VALUES)**: + - Format with VALUES on its own line, and value rows indented: + ```sql + FROM + ( + VALUES + (1, 2, 3) + ) AS v (named_columns); + ``` + +- **CTEs**: + - WITH keyword on first line + - CTE name aligned with leading whitespace + - CTE column list indented from CTE name + - Multiple CTEs separated by commas at the end + +- **Table Creation**: + - CREATE TABLE on first line + - Schema and table name on next line, indented + - Opening parenthesis on its own line + - Each column on a new line, indented + - Always specify NULL or NOT NULL constraint for each column + - DEFAULT constraints can generally follow other column descriptors on the same line + - Closing parenthesis on its own line + ```sql + CREATE TABLE + dbo.table_name + ( + column_name bigint NOT NULL, + another_column varchar(50) NULL DEFAULT 'value', + third_column datetime2(7) NOT NULL DEFAULT SYSDATETIME() + ); + ``` + +- **Index Creation**: + - For multi-column indexes, format with columns on new lines: + ```sql + CREATE INDEX + index_name + ON dbo.table_name + ( + column1, + column2 + ) + INCLUDE + ( + column3, + column4 + ) + WITH + (options); + ``` + - For single-column indexes, a more compact format is acceptable: + ```sql + CREATE INDEX + index_name + ON dbo.table_name + (column1) + INCLUDE + (column3) + WITH + (options); + ``` + +- **INSERT statements**: + - INSERT on first line + - Schema and table name on next line, indented + - Column list in parentheses on new lines, indented + ```sql + INSERT + dbo.table_name + ( + column1, + column2 + ) + VALUES + ( + value1, + value2 + ); + ``` + +- **Temporary table inserts**: + - Use TABLOCK hint with temporary table inserts + ```sql + INSERT + #table_name + WITH + (TABLOCK) + ( + column_list + ) + ``` + +- **UPDATE statements**: + - UPDATE on first line + - Table alias on next line, indented + - SET on its own line with same indentation as alias + - FROM clause on its own line + ```sql + UPDATE + alias + SET + col1 = value1, + col2 = value2 + FROM dbo.table AS alias + WHERE condition; + ``` + +- **DELETE statements**: + - DELETE on first line + - Table alias on next line, indented + - FROM clause on its own line + ```sql + DELETE + alias + FROM dbo.table AS alias + WHERE condition; + ``` + +- **Parentheses**: + - Opening parenthesis on same line as function/procedure name + - Closing parenthesis aligned with starting line or on its own line for long expressions + - Use extra parentheses for clarity in complex expressions + - Function arguments should be indented four spaces and on new lines: + ```sql + CONVERT + ( + data_type, + value + ) + ``` + +## Code Organization + +- SET statements grouped at procedure start +- Validation checks before main logic +- Help/documentation sections clearly separated from main logic +- Version information tracked explicitly +- Parameter validation at beginning of procedures +- CREATE/ALTER statements separated with GO + +## Code Blocks and Control Structures + +- BEGIN/END contents should be indented four spaces: + ```sql + BEGIN + /*logic*/ + END; + ``` + +- CASE expression contents should be indented, with each condition on a new line: + ```sql + CASE + WHEN thing + AND other_thing + THEN stuff + ELSE result + END + ``` + +- IF/ELSE blocks should be formatted with BEGIN/END on their own lines: + ```sql + IF condition + BEGIN + logic + END; + ELSE + BEGIN + logic + END; + ``` + +- WHILE loops should follow similar formatting: + ```sql + WHILE condition + BEGIN + work + END; + ``` + +- Error handling should follow this template: + ```sql + BEGIN + BEGIN TRY + do stuff + END TRY + BEGIN CATCH + IF @@TRANCOUNT > 0 + BEGIN + ROLLBACK; + END; + + THROW; + END CATCH; + END; + ``` + +- DECLARE blocks should put everything on a new line: + ```sql + DECLARE + @t1 integer, + @t2 integer; + ``` + +- Variables should be declared and initialized together for static values: + ```sql + DECLARE + @t1 integer = 1, + @t2 integer = 2; + ``` + - Take care when initializing to ensure you don't introduce logical flaws with NULL checks + +- Dynamic SQL should follow specific formatting: + ```sql + DECLARE + @sql nvarchar(max) = N'' + + SET @sql += N' + the query ' + QUOTENAME(object_name) + ' + '; + + EXECUTE sys.sp_executesql + @sql, + N'@parameters', + @input; + ``` + +- Transaction blocks should use consistent indentation: + ```sql + BEGIN TRANSACTION + work + COMMIT TRANSACTION; + ``` + +- XML and JSON output should be formatted with each option on a new line: + ```sql + FOR + XML + PATH + TYPE + ``` + +## SQL Best Practices + +- Always use IS NULL / IS NOT NULL for NULL comparisons, never = NULL or != NULL +- Use ISNULL() function for value replacement +- Include RECOMPILE hints for procedures with variable data distributions +- Use RAISERROR with NOWAIT for immediate message display +- Include thorough error handling with BEGIN TRY/CATCH blocks +- Always validate user inputs before using them +- Use semicolons at the end of statements (but only at the very end, after any query hints) +- Apply query hints consistently (RECOMPILE, MAXDOP, etc.) +- Always use ROWCOUNT_BIG() instead of @@ROWCOUNT +- Always use CONVERT over CAST for data type conversions (except when using TRY_CAST, as TRY_CAST isn't dependent on SQL Server version) +- Use XML for string splitting and string building (concatenation), as these methods aren't dependent on SQL Server version or database compatibility level +- Always use cursor variables instead of normal cursors, as they don't require explicit CLOSE/DEALLOCATE statements +- Do not use MERGE statements unless absolutely necessary for functional reasons +- Prefer temporary tables over table variables for performance reasons, especially when the data will be used in joins +- Table variables are acceptable for situations where contents are not used relationally or when insert performance is critical +- Do not drop temporary tables at the end of stored procedures (they're automatically cleaned up when the procedure exits) +- Prefer + operator for string concatenation as it's not version dependent (though CONCAT is acceptable for SQL Server 2012+) +- FORMAT is preferred for adding commas to numbers, but complex CONVERT to money with substring operations is also acceptable +- Date literals should always follow yyyymmdd format (e.g., 20250101), with additional precision as needed for the data type + +## Stored Procedure Structure + +1. SET configuration statements at the top +2. Procedure declaration (CREATE/ALTER) +3. Parameter definitions with inline comments +4. BEGIN block +5. SET NOCOUNT ON and other session settings +6. Variable declarations +7. Validation checks +8. Help section (@help = 1) +9. Main processing logic +10. Error handling +11. Cleanup + +Basic stored procedure outline: +```sql +CREATE OR ALTER PROCEDURE + dbo.procedure_name +( + @parameter_list +) +AS +BEGIN + SET NOCOUNT, XACT_ABORT ON; + + queries... + +END; +``` + +Trigger template: +```sql +CREATE OR ALTER TRIGGER + dbo.a_trigger +ON dbo.a_table +AFTER/INSTEAD OF +AS +BEGIN + IF ROWCOUNT_BIG() = 0 + BEGIN + RETURN + END; + + work + +END; +``` + +View template: +```sql +CREATE OR ALTER VIEW + dbo.a_view +AS +SELECT + column1 = t.column1, + column2 = t.column2 +FROM dbo.table AS t +WHERE t.condition = 1; +``` + +Function template: +```sql +CREATE OR ALTER FUNCTION + dbo.a_function +( + @parameter1 integer, + @parameter2 varchar(50) +) +RETURNS data_type +AS +BEGIN + RETURN value; +END; +``` + +## Example + +```sql +SET ANSI_NULLS ON; +SET QUOTED_IDENTIFIER ON; +GO + +IF OBJECT_ID('dbo.sp_MyProcedure', 'P') IS NULL +BEGIN + EXECUTE ('CREATE PROCEDURE dbo.sp_MyProcedure AS RETURN 0;'); +END; +GO + +ALTER PROCEDURE + dbo.sp_MyProcedure +( + @database_name sysname = NULL, /*the database to analyze*/ + @debug bit = 0, /*prints additional diagnostic information*/ + @help bit = 0 /*prints help information*/ +) +WITH RECOMPILE +AS +BEGIN + SET NOCOUNT ON; + SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; + + BEGIN TRY + /* + Variable declarations + */ + DECLARE + @sql nvarchar(MAX) = N'', + @database_id integer = NULL; + + /* + Parameter validation + */ + IF @database_name IS NULL + BEGIN + SELECT + @database_name = DB_NAME(); + END; + + /* + Help section + */ + IF @help = 1 + BEGIN + SELECT + help = N'This procedure analyzes database objects'; + + RETURN; + END; + + /* + Main processing logic + */ + SELECT + object_id, + object_name = o.name, + schema_name = s.name + FROM dbo.objects AS o + JOIN dbo.schemas AS s + ON o.schema_id = s.schema_id + WHERE o.type = N'U' + AND o.is_ms_shipped = 0 + GROUP BY + o.object_id, + o.name, + s.name + ORDER BY + o.name + OPTION(RECOMPILE); + END TRY + BEGIN CATCH + THROW; + END CATCH; +END; +GO +``` + +This style guide is based on an analysis of Erik Darling's stored procedures from Darling Data, LLC. \ No newline at end of file From 08b8761ae1ad93a7a85c39d8d7f9fe8fcde23bb0 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 13 Mar 2025 22:46:40 -0400 Subject: [PATCH 085/246] Update CLAUDE.md --- CLAUDE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 1871aba0..94e91165 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -378,7 +378,7 @@ This document outlines the T-SQL coding style preferences for Erik Darling (Darl EXECUTE sys.sp_executesql @sql, N'@parameters', - @input; + @input; ``` - Transaction blocks should use consistent indentation: @@ -574,4 +574,4 @@ END; GO ``` -This style guide is based on an analysis of Erik Darling's stored procedures from Darling Data, LLC. \ No newline at end of file +This style guide is based on an analysis of Erik Darling's stored procedures from Darling Data, LLC. From f292ca281cc7a28f3668428903202bc8949aa0d7 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 13 Mar 2025 22:48:34 -0400 Subject: [PATCH 086/246] Update sp_IndexCleanup BETA.sql --- sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql index 623aa17d..6128e03f 100644 --- a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql +++ b/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql @@ -1415,7 +1415,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND ic.partition_ordinal > 0 ORDER BY ic.partition_ordinal - FOR XML + FOR + XML PATH(''''), TYPE ).value(''.'', ''nvarchar(max)''), @@ -1530,7 +1531,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. id2.key_ordinal ORDER BY id2.key_ordinal - FOR XML + FOR + XML PATH(''), TYPE ).value('text()[1]','nvarchar(max)'), @@ -1553,7 +1555,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. id2.column_name ORDER BY id2.column_name - FOR XML + FOR + XML PATH(''), TYPE ).value('text()[1]','nvarchar(max)'), From e1032b24da79c2b8049a314bc1b9727bc3b07594 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 13 Mar 2025 23:06:15 -0400 Subject: [PATCH 087/246] Update sp_QuickieStore.sql --- sp_QuickieStore/sp_QuickieStore.sql | 77 ++++++++++++++--------------- 1 file changed, 38 insertions(+), 39 deletions(-) diff --git a/sp_QuickieStore/sp_QuickieStore.sql b/sp_QuickieStore/sp_QuickieStore.sql index 89f6d8c0..a7ea7377 100644 --- a/sp_QuickieStore/sp_QuickieStore.sql +++ b/sp_QuickieStore/sp_QuickieStore.sql @@ -1421,19 +1421,19 @@ BEGIN END; /* Dynamic regression change column based on formatting and comparator */ -IF @regression_baseline_start_date IS NOT NULL AND @regression_comparator = 'relative' AND @format_output = 1 +IF @regression_mode = 1 AND @regression_comparator = 'relative' AND @format_output = 1 BEGIN INSERT INTO @ColumnDefinitions (column_id, metric_group, metric_type, column_name, column_source, is_conditional, condition_param, condition_value, expert_only, format_pattern) VALUES (160, 'regression', 'change', 'change_in_average_for_query_hash_since_regression_time_period', 'regression.change_since_regression_time_period', 1, 'regression_mode', 1, 0, 'P2'); END; -ELSE IF @regression_baseline_start_date IS NOT NULL AND @format_output = 1 +ELSE IF @regression_mode = 1 AND @format_output = 1 BEGIN INSERT INTO @ColumnDefinitions (column_id, metric_group, metric_type, column_name, column_source, is_conditional, condition_param, condition_value, expert_only, format_pattern) VALUES (160, 'regression', 'change', 'change_in_average_for_query_hash_since_regression_time_period', 'regression.change_since_regression_time_period', 1, 'regression_mode', 1, 0, 'N2'); END; -ELSE IF @regression_baseline_start_date IS NOT NULL +ELSE IF @regression_mode = 1 BEGIN INSERT INTO @ColumnDefinitions (column_id, metric_group, metric_type, column_name, column_source, is_conditional, condition_param, condition_value, expert_only, format_pattern) @@ -1441,7 +1441,7 @@ BEGIN END; /* Wait time for wait-based sorting */ -IF LOWER(@sort_order) LIKE N'%waits' +IF @sort_order_is_a_wait = 1 BEGIN INSERT INTO @ColumnDefinitions (column_id, metric_group, metric_type, column_name, column_source, is_conditional, condition_param, condition_value, expert_only, format_pattern) @@ -1459,7 +1459,7 @@ VALUES 'n', 'n', 'ROW_NUMBER() OVER (PARTITION BY qsrs.plan_id ORDER BY ' + - CASE WHEN @regression_baseline_start_date IS NOT NULL THEN + CASE WHEN @regression_mode = 1 THEN /* As seen when populating #regression_changes */ CASE @regression_direction WHEN 'regressed' THEN 'regression.change_since_regression_time_period' @@ -1482,7 +1482,7 @@ VALUES WHEN 'recent' THEN 'qsrs.last_execution_time' WHEN 'rows' THEN 'qsrs.avg_rowcount' WHEN 'plan count by hashes' THEN 'hashes.plan_hash_count_for_query_hash DESC, hashes.query_hash' - ELSE CASE WHEN LOWER(@sort_order) LIKE N'%waits' THEN 'waits.total_query_wait_time_ms' + ELSE CASE WHEN @sort_order_is_a_wait = 1 THEN 'waits.total_query_wait_time_ms' ELSE 'qsrs.avg_cpu_time' END END END + ' DESC)', @@ -1718,14 +1718,9 @@ SELECT ); /* -Set @regression_mode if the given arguments indicate that -we are checking for regressed queries. +@regression_mode is already set in initialization based on +@regression_baseline_start_date */ -IF @regression_baseline_start_date IS NOT NULL -BEGIN - SELECT - @regression_mode = 1; -END; /* Error out if the @regression parameters do not make sense. @@ -2344,6 +2339,34 @@ SELECT ISNULL(@workdays, 0), @include_query_hash_totals = ISNULL(@include_query_hash_totals, 0), + @sort_order_is_a_wait = + CASE WHEN LOWER(@sort_order) IN + ( + 'cpu waits', + 'lock waits', + 'locks waits', + 'latch waits', + 'latches waits', + 'buffer latch waits', + 'buffer latches waits', + 'buffer io waits', + 'log waits', + 'log io waits', + 'network waits', + 'network io waits', + 'parallel waits', + 'parallelism waits', + 'memory waits', + 'total waits' + ) + THEN 1 + ELSE 0 + END, + @regression_mode = + CASE WHEN @regression_baseline_start_date IS NOT NULL + THEN 1 + ELSE 0 + END, /* doing start and end date last because they're more complicated if start or end date is null, @@ -3054,33 +3077,9 @@ BEGIN @sort_order = 'cpu'; END; -/* -Checks if the sort order is for a wait. -Cuts out a lot of repetition. +/* +Sort order for wait was already set in initialization */ -IF LOWER(@sort_order) IN - ( - 'cpu waits', - 'lock waits', - 'locks waits', - 'latch waits', - 'latches waits', - 'buffer latch waits', - 'buffer latches waits', - 'buffer io waits', - 'log waits', - 'log io waits', - 'network waits', - 'network io waits', - 'parallel waits', - 'parallelism waits', - 'memory waits', - 'total waits' - ) -BEGIN - SELECT - @sort_order_is_a_wait = 1; -END; /* These columns are only available in 2017+ From baa21c5a77ad26f8354d570f45308df23c98b5d3 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 13 Mar 2025 23:15:25 -0400 Subject: [PATCH 088/246] Revert "Update sp_QuickieStore.sql" This reverts commit e1032b24da79c2b8049a314bc1b9727bc3b07594. --- sp_QuickieStore/sp_QuickieStore.sql | 77 +++++++++++++++-------------- 1 file changed, 39 insertions(+), 38 deletions(-) diff --git a/sp_QuickieStore/sp_QuickieStore.sql b/sp_QuickieStore/sp_QuickieStore.sql index a7ea7377..89f6d8c0 100644 --- a/sp_QuickieStore/sp_QuickieStore.sql +++ b/sp_QuickieStore/sp_QuickieStore.sql @@ -1421,19 +1421,19 @@ BEGIN END; /* Dynamic regression change column based on formatting and comparator */ -IF @regression_mode = 1 AND @regression_comparator = 'relative' AND @format_output = 1 +IF @regression_baseline_start_date IS NOT NULL AND @regression_comparator = 'relative' AND @format_output = 1 BEGIN INSERT INTO @ColumnDefinitions (column_id, metric_group, metric_type, column_name, column_source, is_conditional, condition_param, condition_value, expert_only, format_pattern) VALUES (160, 'regression', 'change', 'change_in_average_for_query_hash_since_regression_time_period', 'regression.change_since_regression_time_period', 1, 'regression_mode', 1, 0, 'P2'); END; -ELSE IF @regression_mode = 1 AND @format_output = 1 +ELSE IF @regression_baseline_start_date IS NOT NULL AND @format_output = 1 BEGIN INSERT INTO @ColumnDefinitions (column_id, metric_group, metric_type, column_name, column_source, is_conditional, condition_param, condition_value, expert_only, format_pattern) VALUES (160, 'regression', 'change', 'change_in_average_for_query_hash_since_regression_time_period', 'regression.change_since_regression_time_period', 1, 'regression_mode', 1, 0, 'N2'); END; -ELSE IF @regression_mode = 1 +ELSE IF @regression_baseline_start_date IS NOT NULL BEGIN INSERT INTO @ColumnDefinitions (column_id, metric_group, metric_type, column_name, column_source, is_conditional, condition_param, condition_value, expert_only, format_pattern) @@ -1441,7 +1441,7 @@ BEGIN END; /* Wait time for wait-based sorting */ -IF @sort_order_is_a_wait = 1 +IF LOWER(@sort_order) LIKE N'%waits' BEGIN INSERT INTO @ColumnDefinitions (column_id, metric_group, metric_type, column_name, column_source, is_conditional, condition_param, condition_value, expert_only, format_pattern) @@ -1459,7 +1459,7 @@ VALUES 'n', 'n', 'ROW_NUMBER() OVER (PARTITION BY qsrs.plan_id ORDER BY ' + - CASE WHEN @regression_mode = 1 THEN + CASE WHEN @regression_baseline_start_date IS NOT NULL THEN /* As seen when populating #regression_changes */ CASE @regression_direction WHEN 'regressed' THEN 'regression.change_since_regression_time_period' @@ -1482,7 +1482,7 @@ VALUES WHEN 'recent' THEN 'qsrs.last_execution_time' WHEN 'rows' THEN 'qsrs.avg_rowcount' WHEN 'plan count by hashes' THEN 'hashes.plan_hash_count_for_query_hash DESC, hashes.query_hash' - ELSE CASE WHEN @sort_order_is_a_wait = 1 THEN 'waits.total_query_wait_time_ms' + ELSE CASE WHEN LOWER(@sort_order) LIKE N'%waits' THEN 'waits.total_query_wait_time_ms' ELSE 'qsrs.avg_cpu_time' END END END + ' DESC)', @@ -1718,9 +1718,14 @@ SELECT ); /* -@regression_mode is already set in initialization based on -@regression_baseline_start_date +Set @regression_mode if the given arguments indicate that +we are checking for regressed queries. */ +IF @regression_baseline_start_date IS NOT NULL +BEGIN + SELECT + @regression_mode = 1; +END; /* Error out if the @regression parameters do not make sense. @@ -2339,34 +2344,6 @@ SELECT ISNULL(@workdays, 0), @include_query_hash_totals = ISNULL(@include_query_hash_totals, 0), - @sort_order_is_a_wait = - CASE WHEN LOWER(@sort_order) IN - ( - 'cpu waits', - 'lock waits', - 'locks waits', - 'latch waits', - 'latches waits', - 'buffer latch waits', - 'buffer latches waits', - 'buffer io waits', - 'log waits', - 'log io waits', - 'network waits', - 'network io waits', - 'parallel waits', - 'parallelism waits', - 'memory waits', - 'total waits' - ) - THEN 1 - ELSE 0 - END, - @regression_mode = - CASE WHEN @regression_baseline_start_date IS NOT NULL - THEN 1 - ELSE 0 - END, /* doing start and end date last because they're more complicated if start or end date is null, @@ -3077,9 +3054,33 @@ BEGIN @sort_order = 'cpu'; END; -/* -Sort order for wait was already set in initialization +/* +Checks if the sort order is for a wait. +Cuts out a lot of repetition. */ +IF LOWER(@sort_order) IN + ( + 'cpu waits', + 'lock waits', + 'locks waits', + 'latch waits', + 'latches waits', + 'buffer latch waits', + 'buffer latches waits', + 'buffer io waits', + 'log waits', + 'log io waits', + 'network waits', + 'network io waits', + 'parallel waits', + 'parallelism waits', + 'memory waits', + 'total waits' + ) +BEGIN + SELECT + @sort_order_is_a_wait = 1; +END; /* These columns are only available in 2017+ From d51dc098b42fa17197f4869789ba5c572cddb419 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 13 Mar 2025 23:25:00 -0400 Subject: [PATCH 089/246] take out the beta merge conflicts?! --- {sp_IndexCleanup BETA => sp_IndexCleanup}/README.md | 0 .../sp_IndexCleanup BETA.sql | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename {sp_IndexCleanup BETA => sp_IndexCleanup}/README.md (100%) rename {sp_IndexCleanup BETA => sp_IndexCleanup}/sp_IndexCleanup BETA.sql (100%) diff --git a/sp_IndexCleanup BETA/README.md b/sp_IndexCleanup/README.md similarity index 100% rename from sp_IndexCleanup BETA/README.md rename to sp_IndexCleanup/README.md diff --git a/sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql b/sp_IndexCleanup/sp_IndexCleanup BETA.sql similarity index 100% rename from sp_IndexCleanup BETA/sp_IndexCleanup BETA.sql rename to sp_IndexCleanup/sp_IndexCleanup BETA.sql From a4680074177c6a3888eb9c2ab55af44b2c64be57 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 13 Mar 2025 23:25:59 -0400 Subject: [PATCH 090/246] remove beta from file name --- sp_IndexCleanup/{sp_IndexCleanup BETA.sql => sp_IndexCleanup.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename sp_IndexCleanup/{sp_IndexCleanup BETA.sql => sp_IndexCleanup.sql} (100%) diff --git a/sp_IndexCleanup/sp_IndexCleanup BETA.sql b/sp_IndexCleanup/sp_IndexCleanup.sql similarity index 100% rename from sp_IndexCleanup/sp_IndexCleanup BETA.sql rename to sp_IndexCleanup/sp_IndexCleanup.sql From ebde99c407f12dc72a3cac30aa0ed2b9d1916fd6 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Fri, 14 Mar 2025 13:03:16 -0400 Subject: [PATCH 091/246] some stuff add better compression check to IC tidy quickiestore a bit to work on something else --- sp_IndexCleanup/sp_IndexCleanup.sql | 40 +++++++++++++++++++++-------- sp_QuickieStore/sp_QuickieStore.sql | 13 +++++----- 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 6128e03f..0502f55d 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -382,7 +382,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. object_id integer NOT NULL, table_name sysname NOT NULL, index_id integer NOT NULL, - index_name sysname NOT NULL + index_name sysname NOT NULL, + can_compress bit NOT NULL PRIMARY KEY CLUSTERED(database_id, schema_id, object_id, index_id) ); @@ -654,12 +655,22 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. object_id = t.object_id, table_name = t.name, index_id = i.index_id, - index_name = ISNULL(i.name, t.name + N''.Heap'') + index_name = ISNULL(i.name, t.name + N''.Heap''), + can_compress = + CASE + WHEN p.index_id > 0 + AND p.data_compression = 0 + THEN 1 + ELSE 0 + END FROM ' + QUOTENAME(@database_name) + N'.sys.tables AS t JOIN ' + QUOTENAME(@database_name) + N'.sys.schemas AS s ON t.schema_id = s.schema_id JOIN ' + QUOTENAME(@database_name) + N'.sys.indexes AS i ON t.object_id = i.object_id + JOIN ' + QUOTENAME(@database_name) + N'.sys.partitions AS p + ON i.object_id = p.object_id + AND i.index_id = p.index_id LEFT JOIN ' + QUOTENAME(@database_name) + N'.sys.dm_db_index_usage_stats AS us ON t.object_id = us.object_id AND us.database_id = @database_id @@ -773,7 +784,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. object_id, table_name, index_id, - index_name + index_name, + can_compress ) EXECUTE sys.sp_executesql @sql, @@ -838,6 +850,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 1, /* Default to compressible */ NULL FROM #filtered_objects AS fo + WHERE fo.can_compress = 1 OPTION(RECOMPILE); /* If SQL Server edition doesn't support compression, mark all as ineligible */ @@ -1162,7 +1175,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. JOIN ' + QUOTENAME(@database_name) + N'.sys.index_columns AS ic ON i.object_id = ic.object_id AND i.index_id = ic.index_id - JOIN ' + QUOTENAME(@database_name) + N'.sys.columns AS c + JOIN ' + QUOTENAME(@database_name) + + CONVERT + ( + nvarchar(MAX), + N'.sys.columns AS c ON ic.object_id = c.object_id AND ic.column_id = c.column_id LEFT JOIN sys.dm_db_index_usage_stats AS us @@ -1185,7 +1202,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ( SELECT 1/0 - FROM ' + QUOTENAME(@database_name) + + FROM ' + ) + QUOTENAME(@database_name) + CONVERT ( nvarchar(MAX), @@ -2288,7 +2306,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. index_reads, index_writes ) - SELECT + SELECT DISTINCT result_type = 'MERGE', /* Put merge target indexes higher in sort order (5) so they appear before indexes that will be disabled (20) */ @@ -2519,7 +2537,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. index_reads, index_writes ) - SELECT + SELECT DISTINCT result_type = 'COMPRESS', sort_order = 40, ia.database_name, @@ -2620,7 +2638,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. index_reads, index_writes ) - SELECT + SELECT DISTINCT result_type = 'CONSTRAINT', sort_order = 30, ia.database_name, @@ -2722,7 +2740,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. index_reads, index_writes ) - SELECT + SELECT DISTINCT result_type = 'COMPRESS_PARTITION', sort_order = 50, ia.database_name, @@ -2830,7 +2848,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. index_reads, index_writes ) - SELECT + SELECT DISTINCT result_type = 'INELIGIBLE', sort_order = 90, ce.database_name, @@ -2885,7 +2903,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. index_reads, index_writes ) - SELECT + SELECT DISTINCT result_type = 'REVIEW', sort_order = 93, /* Just before KEPT indexes */ ia.database_name, diff --git a/sp_QuickieStore/sp_QuickieStore.sql b/sp_QuickieStore/sp_QuickieStore.sql index 89f6d8c0..57cb8a55 100644 --- a/sp_QuickieStore/sp_QuickieStore.sql +++ b/sp_QuickieStore/sp_QuickieStore.sql @@ -6138,16 +6138,17 @@ SELECT WHERE qsrs.plan_id = dp.plan_id AND 1 = 1 ' - + CASE WHEN @regression_mode = 1 - THEN N' AND ( 1 = 1 - ' + @regression_where_clause + + CASE + WHEN @regression_mode = 1 + THEN N' AND ( 1 = 1 + ' + + @regression_where_clause + N' ) OR ( 1 = 1 - ' - + @where_clause + ' + @where_clause + N' ) ' - ELSE @where_clause + ELSE @where_clause END + N' ORDER BY From 19d0f015040eca48d370ed3ed38b3817e60dd2c1 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Fri, 14 Mar 2025 13:04:18 -0400 Subject: [PATCH 092/246] IC fix include merge --- .DS_Store | Bin 6148 -> 6148 bytes sp_IndexCleanup/sp_IndexCleanup.sql | 85 ++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) diff --git a/.DS_Store b/.DS_Store index 38bdb71467988549740df9c09b5bfc5de765c8ab..7433f156746e183515111389140bab6b43e80b93 100644 GIT binary patch delta 82 zcmZoMXffEZn}yw6M?u%j*m&|GmW7NvC%do|YDiXB8<|@e=qQ+&o7U*v4kH kwVWKH%KFwp@!2`KdHLOw@3F`;_H348d&#(&jpH9b0O&Lrp#T5? delta 53 wcmZoMXffEZn}yv-M?u%n$Z+yQR_V!iS=uH$vs9posccqYd&an#o#QV*0LAMN{r~^~ diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 6128e03f..1ba60a5b 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -1788,6 +1788,91 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ) OPTION(RECOMPILE); + /* + Merge included columns for indexes marked with MERGE INCLUDES action + */ + WITH MergeIncludes AS + ( + SELECT + winner.database_id, + winner.object_id, + winner.index_id, + winner.index_name, + winner.included_columns AS winner_includes, + loser.included_columns AS loser_includes + FROM #index_analysis AS winner + JOIN #index_analysis AS loser + ON winner.database_id = loser.database_id + AND winner.object_id = loser.object_id + AND loser.target_index_name = winner.index_name + WHERE winner.action = 'MERGE INCLUDES' + AND loser.action = 'DISABLE' + AND winner.consolidation_rule = 'Key Duplicate' + AND loser.consolidation_rule = 'Key Duplicate' + ) + UPDATE ia + SET ia.included_columns = + CASE + /* If both have includes, combine them without duplicates */ + WHEN mi.winner_includes IS NOT NULL AND mi.loser_includes IS NOT NULL + THEN + /* Create combined includes using XML method that works with all SQL Server versions */ + ( + SELECT + /* Combine both sets of includes */ + combined_cols = + STUFF + ( + ( + SELECT + N', ' + t.c.value('.', 'NVARCHAR(4000)') + FROM + ( + /* Create XML from winner includes */ + SELECT + x = CONVERT + ( + XML, + N'' + + REPLACE(mi.winner_includes, N', ', N'') + + N'' + ) + + UNION ALL + + /* Create XML from loser includes */ + SELECT + x = CONVERT + ( + XML, + N'' + + REPLACE(mi.loser_includes, N', ', N'') + + N'' + ) + ) AS a + /* Split XML into individual columns */ + CROSS APPLY a.x.nodes('/c') AS t(c) + /* Ensure uniqueness with GROUP BY */ + GROUP BY t.c.value('.', 'NVARCHAR(4000)') + ORDER BY t.c.value('.', 'NVARCHAR(4000)') + FOR XML PATH('') + ), + 1, 2, '' + ) + ) + /* If only loser has includes, use those */ + WHEN mi.winner_includes IS NULL AND mi.loser_includes IS NOT NULL + THEN mi.loser_includes + /* If only winner has includes or neither has includes, keep winner's includes */ + ELSE mi.winner_includes + END + FROM #index_analysis AS ia + JOIN MergeIncludes AS mi + ON ia.database_id = mi.database_id + AND ia.object_id = mi.object_id + AND ia.index_id = mi.index_id + WHERE ia.action = 'MERGE INCLUDES'; + /* Rule 4: Superset/subset key columns */ UPDATE ia1 From ef3d04be8452bab73ebaca5f2f969203a69bc360 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Fri, 14 Mar 2025 13:04:44 -0400 Subject: [PATCH 093/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 0502f55d..023db953 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -35,7 +35,7 @@ ALTER PROCEDURE @min_size_gb decimal(10,2) = 0, @min_rows bigint = 0, @help bit = 'false', - @debug bit = 'true', + @debug bit = 'false', @version varchar(20) = NULL OUTPUT, @version_date datetime = NULL OUTPUT ) From 6d0ea4193c8c98239e8e6ae8bca937e11ca9554a Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Fri, 14 Mar 2025 13:05:01 -0400 Subject: [PATCH 094/246] Merge branch 'dev' of https://github.com/erikdarlingdata/DarlingData into dev From 210fb8fcf73158f96884ccbf6d7483599dbb71ad Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Fri, 14 Mar 2025 13:10:15 -0400 Subject: [PATCH 095/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 83c9e952..da8d9c15 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -1828,11 +1828,14 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND winner.consolidation_rule = 'Key Duplicate' AND loser.consolidation_rule = 'Key Duplicate' ) - UPDATE ia - SET ia.included_columns = + UPDATE + ia + SET + ia.included_columns = CASE /* If both have includes, combine them without duplicates */ - WHEN mi.winner_includes IS NOT NULL AND mi.loser_includes IS NOT NULL + WHEN mi.winner_includes IS NOT NULL + AND mi.loser_includes IS NOT NULL THEN /* Create combined includes using XML method that works with all SQL Server versions */ ( @@ -1842,15 +1845,16 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. STUFF ( ( - SELECT - N', ' + t.c.value('.', 'NVARCHAR(4000)') + SELECT DISTINCT + N', ' + + t.c.value('.', 'sysname') FROM ( /* Create XML from winner includes */ SELECT x = CONVERT ( - XML, + xml, N'' + REPLACE(mi.winner_includes, N', ', N'') + N'' @@ -1862,7 +1866,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SELECT x = CONVERT ( - XML, + xml, N'' + REPLACE(mi.loser_includes, N', ', N'') + N'' @@ -1870,12 +1874,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ) AS a /* Split XML into individual columns */ CROSS APPLY a.x.nodes('/c') AS t(c) - /* Ensure uniqueness with GROUP BY */ - GROUP BY t.c.value('.', 'NVARCHAR(4000)') - ORDER BY t.c.value('.', 'NVARCHAR(4000)') - FOR XML PATH('') + FOR + XML + PATH('') ), - 1, 2, '' + 1, + 2, + '' ) ) /* If only loser has includes, use those */ From bc64a4991504e6bd9c96ad27a2fd055658c199c0 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Fri, 14 Mar 2025 13:48:48 -0400 Subject: [PATCH 096/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 99 +++++++++++++++++++++++++---- 1 file changed, 87 insertions(+), 12 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index da8d9c15..9214b79b 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -1657,8 +1657,16 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND id.user_scans > 0 ) THEN 100 ELSE 0 END - OPTION(RECOMPILE); /* Indexes with scans get some priority */ + OPTION(RECOMPILE); /* Indexes with scans get some priority */ + IF @debug = 1 + BEGIN + SELECT + table_name = '#index_analysis after priority score', + ia.* + FROM #index_analysis AS ia + OPTION(RECOMPILE); + END; /* Rule 1: Identify unused indexes */ UPDATE @@ -1689,6 +1697,15 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND #index_analysis.index_id <> 1 OPTION(RECOMPILE); /* Don't disable clustered indexes */ + IF @debug = 1 + BEGIN + SELECT + table_name = '#index_analysis after rule 1', + ia.* + FROM #index_analysis AS ia + OPTION(RECOMPILE); + END; + /* Rule 2: Exact duplicates - matching key columns and includes */ UPDATE ia1 @@ -1738,6 +1755,15 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ) OPTION(RECOMPILE); + IF @debug = 1 + BEGIN + SELECT + table_name = '#index_analysis after rule 2', + ia.* + FROM #index_analysis AS ia + OPTION(RECOMPILE); + END; + /* Rule 3: Key duplicates - matching key columns, different includes */ UPDATE ia1 @@ -1746,9 +1772,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ia1.target_index_name = CASE /* If one is unique and the other isn't, prefer the unique one */ - WHEN ia1.is_unique = 1 AND ia2.is_unique = 0 + WHEN ia1.is_unique = 1 + AND ia2.is_unique = 0 THEN NULL - WHEN ia1.is_unique = 0 AND ia2.is_unique = 1 + WHEN ia1.is_unique = 0 + AND ia2.is_unique = 1 THEN ia2.index_name /* Otherwise use priority */ WHEN ia1.index_priority >= ia2.index_priority @@ -1757,8 +1785,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END, ia1.action = CASE - WHEN (ia1.is_unique = 1 AND ia2.is_unique = 0) OR - (ia1.index_priority >= ia2.index_priority AND NOT (ia1.is_unique = 0 AND ia2.is_unique = 1)) + WHEN (ia1.is_unique = 1 AND ia2.is_unique = 0) + OR + ( + ia1.index_priority >= ia2.index_priority + AND NOT (ia1.is_unique = 0 AND ia2.is_unique = 1) + ) + AND ISNULL(ia1.included_columns, N'') <> ISNULL(ia2.included_columns, N'') THEN 'MERGE INCLUDES' /* Keep this index but merge includes */ ELSE 'DISABLE' /* Other index is keeper, disable this one */ END, @@ -1771,7 +1804,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ia1.index_priority >= ia2.index_priority AND NOT (ia1.is_unique = 0 AND ia2.is_unique = 1) ) - THEN 'Supersedes ' + ia2.index_name + THEN 'Supersedes ' + + ia2.index_name ELSE NULL END FROM #index_analysis AS ia1 @@ -1805,6 +1839,15 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND id2.is_eligible_for_dedupe = 1 ) OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + SELECT + table_name = '#index_analysis after rule 3', + ia.* + FROM #index_analysis AS ia + OPTION(RECOMPILE); + END; /* Merge included columns for indexes marked with MERGE INCLUDES action @@ -1895,6 +1938,15 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND ia.object_id = mi.object_id AND ia.index_id = mi.index_id WHERE ia.action = 'MERGE INCLUDES'; + + IF @debug = 1 + BEGIN + SELECT + table_name = '#index_analysis after merge includes', + ia.* + FROM #index_analysis AS ia + OPTION(RECOMPILE); + END; /* Rule 4: Superset/subset key columns */ UPDATE @@ -1935,6 +1987,15 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND id2.is_eligible_for_dedupe = 1 ) OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + SELECT + table_name = '#index_analysis after rule 4', + ia.* + FROM #index_analysis AS ia + OPTION(RECOMPILE); + END; /* Update the superseded_by column for the wider index in a separate statement */ UPDATE @@ -1954,6 +2015,15 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND ia1.target_index_name = ia2.index_name /* Make sure we're updating the right wider index */ OPTION(RECOMPILE); + IF @debug = 1 + BEGIN + SELECT + table_name = '#index_analysis after update superseded', + ia.* + FROM #index_analysis AS ia + OPTION(RECOMPILE); + END; + /* Rule 5: Unique constraint vs. nonclustered index handling */ UPDATE ia1 @@ -2010,8 +2080,17 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ) ) OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + SELECT + table_name = '#index_analysis after rule 5', + ia.* + FROM #index_analysis AS ia + OPTION(RECOMPILE); + END; - /* Rule 7: Identify indexes with same keys but in different order after first column */ + /* Rule 6: Identify indexes with same keys but in different order after first column */ /* This rule flags indexes that have the same set of key columns but ordered differently */ /* These need manual review as they may be redundant depending on query patterns */ UPDATE @@ -2090,19 +2169,15 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ) OPTION(RECOMPILE); - IF @debug = 1 BEGIN SELECT - table_name = '#index_analysis after update', + table_name = '#index_analysis after rule 6', ia.* FROM #index_analysis AS ia OPTION(RECOMPILE); - - RAISERROR('Generating results', 0, 0) WITH NOWAIT; END; - /* Create a reference to the detailed summary that will appear at the end */ IF @debug = 1 BEGIN From 879e9892c407f7a4bb9d6d98369f56c18bec5813 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Fri, 14 Mar 2025 13:52:26 -0400 Subject: [PATCH 097/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 101 ++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 9214b79b..0d83aa4f 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -1987,6 +1987,107 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND id2.is_eligible_for_dedupe = 1 ) OPTION(RECOMPILE); + + /* Rule 5: Mark superset indexes for merging with includes from subset */ + UPDATE + ia2 + SET + ia2.consolidation_rule = 'Key Superset', + ia2.action = N'MERGE INCLUDES', /* The wider index gets merged with includes */ + ia2.superseded_by = COALESCE(ia2.superseded_by + ', ', '') + 'Supersedes ' + ia1.index_name + FROM #index_analysis AS ia1 + JOIN #index_analysis AS ia2 + ON ia1.database_id = ia2.database_id + AND ia1.object_id = ia2.object_id + AND ia1.target_index_name = ia2.index_name /* Link from Rule 4 */ + WHERE ia1.consolidation_rule = 'Key Subset' + AND ia1.action = 'DISABLE' + AND ia2.consolidation_rule IS NULL /* Not already processed */ + OPTION(RECOMPILE); + + /* Rule 6: Merge includes from subset to superset indexes */ + WITH KeySubsetSuperset AS + ( + SELECT + superset.database_id, + superset.object_id, + superset.index_id, + superset.index_name, + superset.included_columns AS superset_includes, + subset.included_columns AS subset_includes + FROM #index_analysis AS superset + JOIN #index_analysis AS subset + ON superset.database_id = subset.database_id + AND superset.object_id = subset.object_id + AND subset.target_index_name = superset.index_name + WHERE superset.action = 'MERGE INCLUDES' + AND subset.action = 'DISABLE' + AND superset.consolidation_rule = 'Key Superset' + AND subset.consolidation_rule = 'Key Subset' + ) + UPDATE ia + SET ia.included_columns = + CASE + /* If both have includes, combine them without duplicates */ + WHEN kss.superset_includes IS NOT NULL AND kss.subset_includes IS NOT NULL + THEN + /* Create combined includes using XML method that works with all SQL Server versions */ + ( + SELECT + /* Combine both sets of includes */ + combined_cols = + STUFF + ( + ( + SELECT DISTINCT + N', ' + t.c.value('.', 'sysname') + FROM + ( + /* Create XML from superset includes */ + SELECT + x = CONVERT + ( + xml, + N'' + + REPLACE(kss.superset_includes, N', ', N'') + + N'' + ) + + UNION ALL + + /* Create XML from subset includes */ + SELECT + x = CONVERT + ( + xml, + N'' + + REPLACE(kss.subset_includes, N', ', N'') + + N'' + ) + ) AS a + /* Split XML into individual columns */ + CROSS APPLY a.x.nodes('/c') AS t(c) + FOR + XML + PATH('') + ), + 1, + 2, + '' + ) + ) + /* If only subset has includes, use those */ + WHEN kss.superset_includes IS NULL AND kss.subset_includes IS NOT NULL + THEN kss.subset_includes + /* If only superset has includes or neither has includes, keep superset's includes */ + ELSE kss.superset_includes + END + FROM #index_analysis AS ia + JOIN KeySubsetSuperset AS kss + ON ia.database_id = kss.database_id + AND ia.object_id = kss.object_id + AND ia.index_id = kss.index_id + WHERE ia.action = 'MERGE INCLUDES'; IF @debug = 1 BEGIN From c56b78b1e67de51e5d0d6feac724c51c2ad647e7 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Fri, 14 Mar 2025 13:59:49 -0400 Subject: [PATCH 098/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 138 ++++++---------------------- 1 file changed, 30 insertions(+), 108 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 0d83aa4f..a68b092c 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -1848,106 +1848,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. FROM #index_analysis AS ia OPTION(RECOMPILE); END; - - /* - Merge included columns for indexes marked with MERGE INCLUDES action - */ - WITH MergeIncludes AS - ( - SELECT - winner.database_id, - winner.object_id, - winner.index_id, - winner.index_name, - winner.included_columns AS winner_includes, - loser.included_columns AS loser_includes - FROM #index_analysis AS winner - JOIN #index_analysis AS loser - ON winner.database_id = loser.database_id - AND winner.object_id = loser.object_id - AND loser.target_index_name = winner.index_name - WHERE winner.action = 'MERGE INCLUDES' - AND loser.action = 'DISABLE' - AND winner.consolidation_rule = 'Key Duplicate' - AND loser.consolidation_rule = 'Key Duplicate' - ) - UPDATE - ia - SET - ia.included_columns = - CASE - /* If both have includes, combine them without duplicates */ - WHEN mi.winner_includes IS NOT NULL - AND mi.loser_includes IS NOT NULL - THEN - /* Create combined includes using XML method that works with all SQL Server versions */ - ( - SELECT - /* Combine both sets of includes */ - combined_cols = - STUFF - ( - ( - SELECT DISTINCT - N', ' + - t.c.value('.', 'sysname') - FROM - ( - /* Create XML from winner includes */ - SELECT - x = CONVERT - ( - xml, - N'' + - REPLACE(mi.winner_includes, N', ', N'') + - N'' - ) - - UNION ALL - - /* Create XML from loser includes */ - SELECT - x = CONVERT - ( - xml, - N'' + - REPLACE(mi.loser_includes, N', ', N'') + - N'' - ) - ) AS a - /* Split XML into individual columns */ - CROSS APPLY a.x.nodes('/c') AS t(c) - FOR - XML - PATH('') - ), - 1, - 2, - '' - ) - ) - /* If only loser has includes, use those */ - WHEN mi.winner_includes IS NULL AND mi.loser_includes IS NOT NULL - THEN mi.loser_includes - /* If only winner has includes or neither has includes, keep winner's includes */ - ELSE mi.winner_includes - END - FROM #index_analysis AS ia - JOIN MergeIncludes AS mi - ON ia.database_id = mi.database_id - AND ia.object_id = mi.object_id - AND ia.index_id = mi.index_id - WHERE ia.action = 'MERGE INCLUDES'; - - IF @debug = 1 - BEGIN - SELECT - table_name = '#index_analysis after merge includes', - ia.* - FROM #index_analysis AS ia - OPTION(RECOMPILE); - END; - + /* Rule 4: Superset/subset key columns */ UPDATE ia1 @@ -1987,6 +1888,15 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND id2.is_eligible_for_dedupe = 1 ) OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + SELECT + table_name = '#index_analysis after rule 4', + ia.* + FROM #index_analysis AS ia + OPTION(RECOMPILE); + END; /* Rule 5: Mark superset indexes for merging with includes from subset */ UPDATE @@ -2004,6 +1914,15 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND ia1.action = 'DISABLE' AND ia2.consolidation_rule IS NULL /* Not already processed */ OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + SELECT + table_name = '#index_analysis after rule 5', + ia.* + FROM #index_analysis AS ia + OPTION(RECOMPILE); + END; /* Rule 6: Merge includes from subset to superset indexes */ WITH KeySubsetSuperset AS @@ -2025,11 +1944,14 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND superset.consolidation_rule = 'Key Superset' AND subset.consolidation_rule = 'Key Subset' ) - UPDATE ia - SET ia.included_columns = + UPDATE + ia + SET + ia.included_columns = CASE /* If both have includes, combine them without duplicates */ - WHEN kss.superset_includes IS NOT NULL AND kss.subset_includes IS NOT NULL + WHEN kss.superset_includes IS NOT NULL + AND kss.subset_includes IS NOT NULL THEN /* Create combined includes using XML method that works with all SQL Server versions */ ( @@ -2092,7 +2014,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. IF @debug = 1 BEGIN SELECT - table_name = '#index_analysis after rule 4', + table_name = '#index_analysis after rule 6', ia.* FROM #index_analysis AS ia OPTION(RECOMPILE); @@ -2125,7 +2047,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. OPTION(RECOMPILE); END; - /* Rule 5: Unique constraint vs. nonclustered index handling */ + /* Rule 7: Unique constraint vs. nonclustered index handling */ UPDATE ia1 SET @@ -2185,13 +2107,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. IF @debug = 1 BEGIN SELECT - table_name = '#index_analysis after rule 5', + table_name = '#index_analysis after rule 7', ia.* FROM #index_analysis AS ia OPTION(RECOMPILE); END; - /* Rule 6: Identify indexes with same keys but in different order after first column */ + /* Rule 8: Identify indexes with same keys but in different order after first column */ /* This rule flags indexes that have the same set of key columns but ordered differently */ /* These need manual review as they may be redundant depending on query patterns */ UPDATE @@ -2273,7 +2195,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. IF @debug = 1 BEGIN SELECT - table_name = '#index_analysis after rule 6', + table_name = '#index_analysis after rule 8', ia.* FROM #index_analysis AS ia OPTION(RECOMPILE); From 7e16dc7ba7447e3b3660f07c92bc0e89020e503b Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Fri, 14 Mar 2025 14:15:48 -0400 Subject: [PATCH 099/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 152 ++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index a68b092c..eceff8bc 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -572,6 +572,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. script nvarchar(max) NULL, additional_info nvarchar(max) NULL, /* For stats, constraints, etc. */ superseded_info nvarchar(max) NULL, /* To store superseded_by information */ + original_index_definition nvarchar(max) NULL, /* Original index definition for validation */ index_size_gb decimal(18,4) NULL, /* Size of the index in GB */ index_rows bigint NULL, /* Number of rows in the index */ index_reads bigint NULL, /* Total reads (seeks + scans + lookups) */ @@ -2489,6 +2490,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. script, additional_info, superseded_info, + original_index_definition, index_size_gb, index_rows, index_reads, @@ -2579,6 +2581,29 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END, /* Add superseded_by information if available */ ia.superseded_by, + /* Original index definition for validation */ + original_index_definition = + N'CREATE INDEX ' + + QUOTENAME(ia.index_name) + + N' ON ' + + QUOTENAME(ia.database_name) + + N'.' + + QUOTENAME(ia.schema_name) + + N'.' + + QUOTENAME(ia.table_name) + + N' (' + + ia.key_columns + + N')' + + CASE + WHEN ia.included_columns IS NOT NULL AND LEN(ia.included_columns) > 0 + THEN N' INCLUDE (' + ia.included_columns + N')' + ELSE N'' + END + + CASE + WHEN ia.filter_definition IS NOT NULL + THEN N' WHERE ' + ia.filter_definition + ELSE N'' + END, NULL, NULL, NULL, @@ -2637,6 +2662,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. additional_info, target_index_name, superseded_info, + original_index_definition, index_size_gb, index_rows, index_reads, @@ -2681,6 +2707,29 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END, ia.target_index_name, /* Include the target index name */ superseded_info = NULL, /* Don't need superseded_by info for disabled indexes */ + /* Original index definition for validation */ + original_index_definition = + N'CREATE INDEX ' + + QUOTENAME(ia.index_name) + + N' ON ' + + QUOTENAME(ia.database_name) + + N'.' + + QUOTENAME(ia.schema_name) + + N'.' + + QUOTENAME(ia.table_name) + + N' (' + + ia.key_columns + + N')' + + CASE + WHEN ia.included_columns IS NOT NULL AND LEN(ia.included_columns) > 0 + THEN N' INCLUDE (' + ia.included_columns + N')' + ELSE N'' + END + + CASE + WHEN ia.filter_definition IS NOT NULL + THEN N' WHERE ' + ia.filter_definition + ELSE N'' + END, ps.total_space_gb, ps.total_rows, index_reads = @@ -2720,6 +2769,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. additional_info, target_index_name, superseded_info, + original_index_definition, index_size_gb, index_rows, index_reads, @@ -2753,6 +2803,29 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. N'Compression type: All Partitions', superseded_info = NULL, /* No target index for compression scripts */ ia.superseded_by, /* Include superseded_by info for compression scripts */ + /* Original index definition for validation */ + original_index_definition = + N'CREATE INDEX ' + + QUOTENAME(ia.index_name) + + N' ON ' + + QUOTENAME(ia.database_name) + + N'.' + + QUOTENAME(ia.schema_name) + + N'.' + + QUOTENAME(ia.table_name) + + N' (' + + ia.key_columns + + N')' + + CASE + WHEN ia.included_columns IS NOT NULL AND LEN(ia.included_columns) > 0 + THEN N' INCLUDE (' + ia.included_columns + N')' + ELSE N'' + END + + CASE + WHEN ia.filter_definition IS NOT NULL + THEN N' WHERE ' + ia.filter_definition + ELSE N'' + END, ps_full.total_space_gb, ps_full.total_rows, index_reads = @@ -3031,6 +3104,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. index_name, script_type, additional_info, + original_index_definition, index_size_gb, index_rows, index_reads, @@ -3045,6 +3119,36 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ce.index_name, script_type = 'INELIGIBLE FOR COMPRESSION', ce.reason, + /* Original index definition for validation */ + original_index_definition = + ( + SELECT TOP (1) + N'CREATE INDEX ' + + QUOTENAME(ia.index_name) + + N' ON ' + + QUOTENAME(ia.database_name) + + N'.' + + QUOTENAME(ia.schema_name) + + N'.' + + QUOTENAME(ia.table_name) + + N' (' + + ia.key_columns + + N')' + + CASE + WHEN ia.included_columns IS NOT NULL AND LEN(ia.included_columns) > 0 + THEN N' INCLUDE (' + ia.included_columns + N')' + ELSE N'' + END + + CASE + WHEN ia.filter_definition IS NOT NULL + THEN N' WHERE ' + ia.filter_definition + ELSE N'' + END + FROM #index_analysis AS ia + WHERE ia.database_id = ce.database_id + AND ia.object_id = ce.object_id + AND ia.index_id = ce.index_id + ), ps.total_space_gb, ps.total_rows, index_reads = @@ -3086,6 +3190,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. consolidation_rule, target_index_name, additional_info, + original_index_definition, index_size_gb, index_rows, index_reads, @@ -3108,6 +3213,29 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. N' but in a different order. May be redundant depending on query patterns.' ELSE N'This index needs manual review' END, + /* Original index definition for validation */ + original_index_definition = + N'CREATE INDEX ' + + QUOTENAME(ia.index_name) + + N' ON ' + + QUOTENAME(ia.database_name) + + N'.' + + QUOTENAME(ia.schema_name) + + N'.' + + QUOTENAME(ia.table_name) + + N' (' + + ia.key_columns + + N')' + + CASE + WHEN ia.included_columns IS NOT NULL AND LEN(ia.included_columns) > 0 + THEN N' INCLUDE (' + ia.included_columns + N')' + ELSE N'' + END + + CASE + WHEN ia.filter_definition IS NOT NULL + THEN N' WHERE ' + ia.filter_definition + ELSE N'' + END, ps.total_space_gb, ps.total_rows, index_reads = @@ -3149,6 +3277,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. consolidation_rule, superseded_info, additional_info, + original_index_definition, index_size_gb, index_rows, index_reads, @@ -3172,6 +3301,29 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. THEN 'This index is being kept' ELSE NULL END, + /* Original index definition for validation */ + original_index_definition = + N'CREATE INDEX ' + + QUOTENAME(ia.index_name) + + N' ON ' + + QUOTENAME(ia.database_name) + + N'.' + + QUOTENAME(ia.schema_name) + + N'.' + + QUOTENAME(ia.table_name) + + N' (' + + ia.key_columns + + N')' + + CASE + WHEN ia.included_columns IS NOT NULL AND LEN(ia.included_columns) > 0 + THEN N' INCLUDE (' + ia.included_columns + N')' + ELSE N'' + END + + CASE + WHEN ia.filter_definition IS NOT NULL + THEN N' WHERE ' + ia.filter_definition + ELSE N'' + END, ps.total_space_gb, ps.total_rows, index_reads = From d46cd167abba21bb7c0700d4bd2508b418f6fe1f Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Fri, 14 Mar 2025 18:00:51 -0400 Subject: [PATCH 100/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 231 ++++++++++++---------------- 1 file changed, 98 insertions(+), 133 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index eceff8bc..3dd525ef 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -513,6 +513,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. target_index_name sysname NULL, consolidation_rule varchar(512) NULL, index_priority int NULL, + original_index_definition nvarchar(max) NULL, /* Original CREATE INDEX statement */ INDEX c CLUSTERED (database_id, schema_id, object_id, index_id) ); @@ -1516,7 +1517,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. is_unique, key_columns, included_columns, - filter_definition + filter_definition, + original_index_definition ) SELECT @database_id, @@ -1583,7 +1585,93 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 2, '' ), - id1.filter_definition + id1.filter_definition, + /* Store the original index definition for validation */ + original_index_definition = + N'CREATE ' + + CASE WHEN id1.is_unique = 1 THEN N'UNIQUE ' ELSE N'' END + + N'INDEX ' + + QUOTENAME(id1.index_name) + + N' ON ' + + QUOTENAME(DB_NAME(@database_id)) + + N'.' + + QUOTENAME(id1.schema_name) + + N'.' + + QUOTENAME(id1.table_name) + + N' (' + + STUFF + ( + ( + SELECT + N', ' + + id2.column_name + + CASE + WHEN id2.is_descending_key = 1 + THEN N' DESC' + ELSE N'' + END + FROM #index_details id2 + WHERE id2.object_id = id1.object_id + AND id2.index_id = id1.index_id + AND id2.is_included_column = 0 + GROUP BY + id2.column_name, + id2.is_descending_key, + id2.key_ordinal + ORDER BY + id2.key_ordinal + FOR + XML + PATH(''), + TYPE + ).value('text()[1]','nvarchar(max)'), + 1, + 2, + '' + ) + + N')' + + CASE + WHEN EXISTS + ( + SELECT + 1/0 + FROM #index_details id3 + WHERE id3.object_id = id1.object_id + AND id3.index_id = id1.index_id + AND id3.is_included_column = 1 + ) + THEN N' INCLUDE (' + + STUFF + ( + ( + SELECT + N', ' + + id4.column_name + FROM #index_details id4 + WHERE id4.object_id = id1.object_id + AND id4.index_id = id1.index_id + AND id4.is_included_column = 1 + GROUP BY + id4.column_name + ORDER BY + id4.column_name + FOR + XML + PATH(''), + TYPE + ).value('text()[1]','nvarchar(max)'), + 1, + 2, + '' + ) + + N')' + ELSE N'' + END + + CASE + WHEN id1.filter_definition IS NOT NULL + THEN N' WHERE ' + id1.filter_definition + ELSE N'' + END FROM #index_details id1 WHERE id1.is_eligible_for_dedupe = 1 GROUP BY @@ -2582,28 +2670,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Add superseded_by information if available */ ia.superseded_by, /* Original index definition for validation */ - original_index_definition = - N'CREATE INDEX ' + - QUOTENAME(ia.index_name) + - N' ON ' + - QUOTENAME(ia.database_name) + - N'.' + - QUOTENAME(ia.schema_name) + - N'.' + - QUOTENAME(ia.table_name) + - N' (' + - ia.key_columns + - N')' + - CASE - WHEN ia.included_columns IS NOT NULL AND LEN(ia.included_columns) > 0 - THEN N' INCLUDE (' + ia.included_columns + N')' - ELSE N'' - END + - CASE - WHEN ia.filter_definition IS NOT NULL - THEN N' WHERE ' + ia.filter_definition - ELSE N'' - END, + ia.original_index_definition, NULL, NULL, NULL, @@ -2708,28 +2775,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ia.target_index_name, /* Include the target index name */ superseded_info = NULL, /* Don't need superseded_by info for disabled indexes */ /* Original index definition for validation */ - original_index_definition = - N'CREATE INDEX ' + - QUOTENAME(ia.index_name) + - N' ON ' + - QUOTENAME(ia.database_name) + - N'.' + - QUOTENAME(ia.schema_name) + - N'.' + - QUOTENAME(ia.table_name) + - N' (' + - ia.key_columns + - N')' + - CASE - WHEN ia.included_columns IS NOT NULL AND LEN(ia.included_columns) > 0 - THEN N' INCLUDE (' + ia.included_columns + N')' - ELSE N'' - END + - CASE - WHEN ia.filter_definition IS NOT NULL - THEN N' WHERE ' + ia.filter_definition - ELSE N'' - END, + ia.original_index_definition, ps.total_space_gb, ps.total_rows, index_reads = @@ -2804,28 +2850,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. superseded_info = NULL, /* No target index for compression scripts */ ia.superseded_by, /* Include superseded_by info for compression scripts */ /* Original index definition for validation */ - original_index_definition = - N'CREATE INDEX ' + - QUOTENAME(ia.index_name) + - N' ON ' + - QUOTENAME(ia.database_name) + - N'.' + - QUOTENAME(ia.schema_name) + - N'.' + - QUOTENAME(ia.table_name) + - N' (' + - ia.key_columns + - N')' + - CASE - WHEN ia.included_columns IS NOT NULL AND LEN(ia.included_columns) > 0 - THEN N' INCLUDE (' + ia.included_columns + N')' - ELSE N'' - END + - CASE - WHEN ia.filter_definition IS NOT NULL - THEN N' WHERE ' + ia.filter_definition - ELSE N'' - END, + ia.original_index_definition, ps_full.total_space_gb, ps_full.total_rows, index_reads = @@ -2996,6 +3021,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. additional_info, target_index_name, superseded_info, + original_index_definition, index_size_gb, index_rows, index_reads, @@ -3056,6 +3082,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. N' GB', target_index_name = NULL, superseded_info = NULL, + ia.original_index_definition, ps.total_space_gb, ps.total_rows, index_reads = @@ -3123,27 +3150,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. original_index_definition = ( SELECT TOP (1) - N'CREATE INDEX ' + - QUOTENAME(ia.index_name) + - N' ON ' + - QUOTENAME(ia.database_name) + - N'.' + - QUOTENAME(ia.schema_name) + - N'.' + - QUOTENAME(ia.table_name) + - N' (' + - ia.key_columns + - N')' + - CASE - WHEN ia.included_columns IS NOT NULL AND LEN(ia.included_columns) > 0 - THEN N' INCLUDE (' + ia.included_columns + N')' - ELSE N'' - END + - CASE - WHEN ia.filter_definition IS NOT NULL - THEN N' WHERE ' + ia.filter_definition - ELSE N'' - END + ia.original_index_definition FROM #index_analysis AS ia WHERE ia.database_id = ce.database_id AND ia.object_id = ce.object_id @@ -3214,28 +3221,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ELSE N'This index needs manual review' END, /* Original index definition for validation */ - original_index_definition = - N'CREATE INDEX ' + - QUOTENAME(ia.index_name) + - N' ON ' + - QUOTENAME(ia.database_name) + - N'.' + - QUOTENAME(ia.schema_name) + - N'.' + - QUOTENAME(ia.table_name) + - N' (' + - ia.key_columns + - N')' + - CASE - WHEN ia.included_columns IS NOT NULL AND LEN(ia.included_columns) > 0 - THEN N' INCLUDE (' + ia.included_columns + N')' - ELSE N'' - END + - CASE - WHEN ia.filter_definition IS NOT NULL - THEN N' WHERE ' + ia.filter_definition - ELSE N'' - END, + ia.original_index_definition, ps.total_space_gb, ps.total_rows, index_reads = @@ -3302,28 +3288,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ELSE NULL END, /* Original index definition for validation */ - original_index_definition = - N'CREATE INDEX ' + - QUOTENAME(ia.index_name) + - N' ON ' + - QUOTENAME(ia.database_name) + - N'.' + - QUOTENAME(ia.schema_name) + - N'.' + - QUOTENAME(ia.table_name) + - N' (' + - ia.key_columns + - N')' + - CASE - WHEN ia.included_columns IS NOT NULL AND LEN(ia.included_columns) > 0 - THEN N' INCLUDE (' + ia.included_columns + N')' - ELSE N'' - END + - CASE - WHEN ia.filter_definition IS NOT NULL - THEN N' WHERE ' + ia.filter_definition - ELSE N'' - END, + ia.original_index_definition, ps.total_space_gb, ps.total_rows, index_reads = From 96d61dd970fad6d4aa78aa8820e7ef95d9d1dc3f Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Fri, 14 Mar 2025 18:01:21 -0400 Subject: [PATCH 101/246] why do you need a summary --- sp_HealthParser/sp_HealthParser.sql | 4 +++- sp_QuickieStore/sp_QuickieStore.sql | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/sp_HealthParser/sp_HealthParser.sql b/sp_HealthParser/sp_HealthParser.sql index 3aa1e66c..135a5d36 100644 --- a/sp_HealthParser/sp_HealthParser.sql +++ b/sp_HealthParser/sp_HealthParser.sql @@ -4299,7 +4299,9 @@ END; IF @log_to_table = 0 BEGIN SELECT - 'Error Number Ignored: ' + CONVERT(nvarchar(100), ie.error_number) + error_numbers_ignored = + N'Error Number Ignored: ' + + CONVERT(nvarchar(100), ie.error_number) FROM #ignore_errors AS ie; END; END; diff --git a/sp_QuickieStore/sp_QuickieStore.sql b/sp_QuickieStore/sp_QuickieStore.sql index 57cb8a55..2e14830c 100644 --- a/sp_QuickieStore/sp_QuickieStore.sql +++ b/sp_QuickieStore/sp_QuickieStore.sql @@ -6207,7 +6207,7 @@ IF @debug = 1 BEGIN PRINT LEN(@sql); - IF LEN(@sql) > 7999 + IF LEN(@sql) > 4000 BEGIN SELECT query = From 00c7a63fe18d8a36e3fbf915d3eb6becbd479699 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Fri, 14 Mar 2025 18:03:17 -0400 Subject: [PATCH 102/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 3dd525ef..e65646b3 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -3648,6 +3648,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. THEN NULL ELSE FORMAT(ir.index_writes, 'N0') END, + ia.original_index_definition, /* Finally show the actual script */ ir.script FROM #index_cleanup_results AS ir From e8cfc5cf3ae3f755724e18a404d54cfb481f230d Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Fri, 14 Mar 2025 18:26:29 -0400 Subject: [PATCH 103/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 42 +++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index e65646b3..8ba53bab 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -1822,6 +1822,27 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND ISNULL(ia1.filter_definition, '') = ISNULL(ia2.filter_definition, '') /* Matching filters */ WHERE ia1.consolidation_rule IS NULL /* Not already processed */ AND ia2.consolidation_rule IS NULL /* Not already processed */ + /* Exclude pairs where either one is a unique constraint (we'll handle those separately in Rule 7) */ + AND NOT EXISTS + ( + SELECT + 1/0 + FROM #index_details AS id1_uc + WHERE id1_uc.database_id = ia1.database_id + AND id1_uc.object_id = ia1.object_id + AND id1_uc.index_id = ia1.index_id + AND id1_uc.is_unique_constraint = 1 + ) + AND NOT EXISTS + ( + SELECT + 1/0 + FROM #index_details AS id2_uc + WHERE id2_uc.database_id = ia2.database_id + AND id2_uc.object_id = ia2.object_id + AND id2_uc.index_id = ia2.index_id + AND id2_uc.is_unique_constraint = 1 + ) AND EXISTS ( SELECT @@ -1907,6 +1928,27 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND ISNULL(ia1.filter_definition, '') = ISNULL(ia2.filter_definition, '') /* Matching filters */ WHERE ia1.consolidation_rule IS NULL /* Not already processed */ AND ia2.consolidation_rule IS NULL /* Not already processed */ + /* Exclude pairs where either one is a unique constraint (we'll handle those separately in Rule 7) */ + AND NOT EXISTS + ( + SELECT + 1/0 + FROM #index_details AS id1_uc + WHERE id1_uc.database_id = ia1.database_id + AND id1_uc.object_id = ia1.object_id + AND id1_uc.index_id = ia1.index_id + AND id1_uc.is_unique_constraint = 1 + ) + AND NOT EXISTS + ( + SELECT + 1/0 + FROM #index_details AS id2_uc + WHERE id2_uc.database_id = ia2.database_id + AND id2_uc.object_id = ia2.object_id + AND id2_uc.index_id = ia2.index_id + AND id2_uc.is_unique_constraint = 1 + ) AND EXISTS ( SELECT From 4399d006b4eeae3a001709fb5d3e80354e948dcf Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Fri, 14 Mar 2025 18:29:46 -0400 Subject: [PATCH 104/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 44 +++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 8ba53bab..76e3ea10 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -2792,15 +2792,41 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. script_type = 'DISABLE SCRIPT', ia.consolidation_rule, script = - N'ALTER INDEX ' + - QUOTENAME(ia.index_name) + - N' ON ' + - QUOTENAME(ia.database_name) + - N'.' + - QUOTENAME(ia.schema_name) + - N'.' + - QUOTENAME(ia.table_name) + - N' DISABLE;', + CASE + /* Check if this is a unique constraint and use appropriate syntax */ + WHEN EXISTS + ( + SELECT + 1/0 + FROM #index_details AS id_uc + WHERE id_uc.database_id = ia.database_id + AND id_uc.object_id = ia.object_id + AND id_uc.index_id = ia.index_id + AND id_uc.is_unique_constraint = 1 + ) + THEN + /* Use NOCHECK CONSTRAINT syntax for unique constraints */ + N'ALTER TABLE ' + + QUOTENAME(ia.database_name) + + N'.' + + QUOTENAME(ia.schema_name) + + N'.' + + QUOTENAME(ia.table_name) + + N' NOCHECK CONSTRAINT ' + + QUOTENAME(ia.index_name) + + N';' + ELSE + /* Use regular DISABLE syntax for indexes */ + N'ALTER INDEX ' + + QUOTENAME(ia.index_name) + + N' ON ' + + QUOTENAME(ia.database_name) + + N'.' + + QUOTENAME(ia.schema_name) + + N'.' + + QUOTENAME(ia.table_name) + + N' DISABLE;' + END, CASE WHEN ia.consolidation_rule = 'Key Subset' THEN N'This index is superseded by a wider index: ' + ISNULL(ia.target_index_name, N'(unknown)') From 7dd5a856c5e75d18ce8df9040058a286731f19ff Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Fri, 14 Mar 2025 18:52:04 -0400 Subject: [PATCH 105/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 181 +++++++++++++++------------- 1 file changed, 94 insertions(+), 87 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 76e3ea10..5838dc56 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -2191,6 +2191,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END FROM #index_analysis AS ia1 WHERE ia1.consolidation_rule IS NULL /* Not already processed */ + AND ia1.action IS NULL /* Not already processed by earlier rules */ AND EXISTS ( /* Find nonclustered indexes */ @@ -2244,6 +2245,62 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. OPTION(RECOMPILE); END; + /* Rule 7.5: Mark unique constraints that have matching nonclustered indexes for disabling */ + UPDATE + ia_uc + SET + ia_uc.consolidation_rule = 'Unique Constraint Replacement', + ia_uc.action = N'DISABLE', /* Mark unique constraint for disabling */ + ia_uc.target_index_name = ia_nc.index_name /* Point to the nonclustered index that will replace it */ + FROM #index_analysis AS ia_uc /* Unique constraint */ + JOIN #index_details AS id_uc /* Join to get unique constraint details */ + ON id_uc.database_id = ia_uc.database_id + AND id_uc.object_id = ia_uc.object_id + AND id_uc.index_id = ia_uc.index_id + AND id_uc.is_unique_constraint = 1 /* This is a unique constraint */ + JOIN #index_analysis AS ia_nc /* Join to find nonclustered index */ + ON ia_nc.database_id = ia_uc.database_id + AND ia_nc.object_id = ia_uc.object_id + AND ia_nc.index_name <> ia_uc.index_name /* Different index */ + AND ia_nc.action = 'MAKE UNIQUE' /* That has been marked to be made unique */ + AND ia_nc.consolidation_rule = 'Unique Constraint Replacement' /* From previous rule */ + WHERE + /* Verify key columns match between index and unique constraint */ + NOT EXISTS + ( + SELECT + id_uc_inner.column_name + FROM #index_details AS id_uc_inner + WHERE id_uc_inner.database_id = id_uc.database_id + AND id_uc_inner.object_id = id_uc.object_id + AND id_uc_inner.index_id = id_uc.index_id + AND id_uc_inner.is_included_column = 0 + + EXCEPT + + SELECT + id_nc_inner.column_name + FROM #index_details AS id_nc_inner + JOIN #index_details AS id_nc_base + ON id_nc_inner.database_id = id_nc_base.database_id + AND id_nc_inner.object_id = id_nc_base.object_id + AND id_nc_inner.index_id = id_nc_base.index_id + WHERE id_nc_base.database_id = ia_nc.database_id + AND id_nc_base.object_id = ia_nc.object_id + AND id_nc_base.index_id = ia_nc.index_id + AND id_nc_inner.is_included_column = 0 + ) + OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + SELECT + table_name = '#index_analysis after rule 7.5', + ia.* + FROM #index_analysis AS ia + OPTION(RECOMPILE); + END; + /* Rule 8: Identify indexes with same keys but in different order after first column */ /* This rule flags indexes that have the same set of key columns but ordered differently */ /* These need manual review as they may be redundant depending on query patterns */ @@ -2792,41 +2849,16 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. script_type = 'DISABLE SCRIPT', ia.consolidation_rule, script = - CASE - /* Check if this is a unique constraint and use appropriate syntax */ - WHEN EXISTS - ( - SELECT - 1/0 - FROM #index_details AS id_uc - WHERE id_uc.database_id = ia.database_id - AND id_uc.object_id = ia.object_id - AND id_uc.index_id = ia.index_id - AND id_uc.is_unique_constraint = 1 - ) - THEN - /* Use NOCHECK CONSTRAINT syntax for unique constraints */ - N'ALTER TABLE ' + - QUOTENAME(ia.database_name) + - N'.' + - QUOTENAME(ia.schema_name) + - N'.' + - QUOTENAME(ia.table_name) + - N' NOCHECK CONSTRAINT ' + - QUOTENAME(ia.index_name) + - N';' - ELSE - /* Use regular DISABLE syntax for indexes */ - N'ALTER INDEX ' + - QUOTENAME(ia.index_name) + - N' ON ' + - QUOTENAME(ia.database_name) + - N'.' + - QUOTENAME(ia.schema_name) + - N'.' + - QUOTENAME(ia.table_name) + - N' DISABLE;' - END, + /* Use regular DISABLE syntax for indexes */ + N'ALTER INDEX ' + + QUOTENAME(ia.index_name) + + N' ON ' + + QUOTENAME(ia.database_name) + + N'.' + + QUOTENAME(ia.schema_name) + + N'.' + + QUOTENAME(ia.table_name) + + N' DISABLE;', CASE WHEN ia.consolidation_rule = 'Key Subset' THEN N'This index is superseded by a wider index: ' + ISNULL(ia.target_index_name, N'(unknown)') @@ -2987,6 +3019,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. script_type, additional_info, script, + original_index_definition, index_size_gb, index_rows, index_reads, @@ -2995,78 +3028,52 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SELECT DISTINCT result_type = 'CONSTRAINT', sort_order = 30, - ia.database_name, - ia.schema_name, - ia.table_name, - ia.index_name, + ia_uc.database_name, + ia_uc.schema_name, + ia_uc.table_name, + ia_uc.index_name, script_type = 'DISABLE CONSTRAINT SCRIPT', additional_info = - N'Constraint to disable: ' + - id.index_name, + N'This constraint is being replaced by: ' + + ISNULL(ia_uc.target_index_name, N'(unknown)'), script = N'ALTER TABLE ' + - QUOTENAME(ia.database_name) + + QUOTENAME(ia_uc.database_name) + N'.' + - QUOTENAME(ia.schema_name) + + QUOTENAME(ia_uc.schema_name) + N'.' + - QUOTENAME(ia.table_name) + + QUOTENAME(ia_uc.table_name) + N' NOCHECK CONSTRAINT ' + - QUOTENAME(id.index_name) + + QUOTENAME(ia_uc.index_name) + N';', + /* Original index definition for validation */ + original_index_definition = ia_uc.original_index_definition, ps.total_space_gb, ps.total_rows, index_reads = (id2.user_seeks + id2.user_scans + id2.user_lookups), id2.user_updates - FROM #index_analysis AS ia + FROM #index_analysis AS ia_uc JOIN #index_details AS id - ON id.database_id = ia.database_id - AND id.object_id = ia.object_id + ON id.database_id = ia_uc.database_id + AND id.object_id = ia_uc.object_id + AND id.index_id = ia_uc.index_id AND id.is_unique_constraint = 1 LEFT JOIN #index_details AS id2 - ON id2.database_id = ia.database_id - AND id2.object_id = ia.object_id - AND id2.index_id = ia.index_id + ON id2.database_id = ia_uc.database_id + AND id2.object_id = ia_uc.object_id + AND id2.index_id = ia_uc.index_id AND id2.is_included_column = 0 /* Get only one row per index */ AND id2.key_ordinal > 0 LEFT JOIN #partition_stats AS ps - ON ia.database_id = ps.database_id - AND ia.object_id = ps.object_id - AND ia.index_id = ps.index_id + ON ia_uc.database_id = ps.database_id + AND ia_uc.object_id = ps.object_id + AND ia_uc.index_id = ps.index_id WHERE - /* Only indexes that are being made unique */ - ia.action = N'MAKE UNIQUE' - /* Find the constraint that matches the index being made unique */ - AND EXISTS - ( - SELECT - 1/0 - FROM #index_details AS id_nc - WHERE id_nc.database_id = ia.database_id - AND id_nc.object_id = ia.object_id - AND id_nc.index_id = ia.index_id - /* Matching key columns */ - AND NOT EXISTS - ( - SELECT - id.column_name - FROM #index_details AS id_inner - WHERE id_inner.database_id = id.database_id - AND id_inner.object_id = id.object_id - AND id_inner.index_id = id.index_id - AND id_inner.is_included_column = 0 - - EXCEPT - - SELECT - id_nc_inner.column_name - FROM #index_details AS id_nc_inner - WHERE id_nc_inner.database_id = id_nc.database_id - AND id_nc_inner.object_id = id_nc.object_id - AND id_nc_inner.index_id = id_nc.index_id - AND id_nc_inner.is_included_column = 0 - ) - ) + /* Only constraints that are marked for disabling */ + ia_uc.action = N'DISABLE' + /* That have consolidation_rule of 'Unique Constraint Replacement' */ + AND ia_uc.consolidation_rule = 'Unique Constraint Replacement' OPTION(RECOMPILE); /* Insert per-partition compression scripts */ From b0489b2e7de6efcf92e4c7582a74010775757126 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Fri, 14 Mar 2025 18:56:53 -0400 Subject: [PATCH 106/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 5838dc56..e1704e53 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -1717,10 +1717,28 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END + CASE - WHEN is_unique = 1 - THEN 500 + /* Unique indexes get high priority, but reduce priority for unique constraints */ + WHEN is_unique = 1 AND NOT EXISTS + ( + SELECT + 1/0 + FROM #index_details AS id_uc + WHERE id_uc.index_id = #index_analysis.index_id + AND id_uc.object_id = #index_analysis.object_id + AND id_uc.is_unique_constraint = 1 + ) THEN 500 + /* Unique constraints get lower priority */ + WHEN is_unique = 1 AND EXISTS + ( + SELECT + 1/0 + FROM #index_details AS id_uc + WHERE id_uc.index_id = #index_analysis.index_id + AND id_uc.object_id = #index_analysis.object_id + AND id_uc.is_unique_constraint = 1 + ) THEN 50 ELSE 0 - END /* Unique indexes get high priority */ + END + CASE WHEN EXISTS @@ -1822,7 +1840,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND ISNULL(ia1.filter_definition, '') = ISNULL(ia2.filter_definition, '') /* Matching filters */ WHERE ia1.consolidation_rule IS NULL /* Not already processed */ AND ia2.consolidation_rule IS NULL /* Not already processed */ - /* Exclude pairs where either one is a unique constraint (we'll handle those separately in Rule 7) */ + /* Exclude unique constraints - we'll handle those separately in Rule 7 */ AND NOT EXISTS ( SELECT @@ -2262,8 +2280,6 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ON ia_nc.database_id = ia_uc.database_id AND ia_nc.object_id = ia_uc.object_id AND ia_nc.index_name <> ia_uc.index_name /* Different index */ - AND ia_nc.action = 'MAKE UNIQUE' /* That has been marked to be made unique */ - AND ia_nc.consolidation_rule = 'Unique Constraint Replacement' /* From previous rule */ WHERE /* Verify key columns match between index and unique constraint */ NOT EXISTS From 3482babed2185079e42dcfe37b23fd5ae5be296f Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Fri, 14 Mar 2025 19:16:38 -0400 Subject: [PATCH 107/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 43 +++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index e1704e53..6be6226d 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -2264,6 +2264,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END; /* Rule 7.5: Mark unique constraints that have matching nonclustered indexes for disabling */ + /* First, mark unique constraints for disabling */ UPDATE ia_uc SET @@ -2307,6 +2308,37 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND id_nc_inner.is_included_column = 0 ) OPTION(RECOMPILE); + + /* Second, mark nonclustered indexes to be made unique */ + UPDATE + ia_nc + SET + ia_nc.consolidation_rule = 'Unique Constraint Replacement', + ia_nc.action = N'MAKE UNIQUE' /* Mark nonclustered index to be made unique */ + FROM #index_analysis AS ia_nc /* Nonclustered index */ + JOIN #index_details AS id_nc /* Join to get nonclustered index details */ + ON id_nc.database_id = ia_nc.database_id + AND id_nc.object_id = ia_nc.object_id + AND id_nc.index_id = ia_nc.index_id + AND id_nc.is_unique_constraint = 0 /* This is not a unique constraint */ + AND id_nc.is_unique = 0 /* This is not already unique */ + WHERE EXISTS ( + /* Find matching unique constraint that has been marked for disabling */ + SELECT 1 + FROM #index_analysis AS ia_uc + JOIN #index_details AS id_uc + ON id_uc.database_id = ia_uc.database_id + AND id_uc.object_id = ia_uc.object_id + AND id_uc.index_id = ia_uc.index_id + AND id_uc.is_unique_constraint = 1 + WHERE + ia_uc.database_id = ia_nc.database_id + AND ia_uc.object_id = ia_nc.object_id + AND ia_uc.action = N'DISABLE' + AND ia_uc.target_index_name = ia_nc.index_name + ) + AND ia_nc.action IS NULL /* Only update if no action has been set yet */ + OPTION(RECOMPILE); IF @debug = 1 BEGIN @@ -2909,6 +2941,17 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND id.is_included_column = 0 /* Get only one row per index */ AND id.key_ordinal > 0 WHERE ia.action = N'DISABLE' + /* Exclude unique constraints - they are handled by DISABLE CONSTRAINT scripts */ + AND NOT EXISTS + ( + SELECT + 1/0 + FROM #index_details AS id_uc + WHERE id_uc.database_id = ia.database_id + AND id_uc.object_id = ia.object_id + AND id_uc.index_id = ia.index_id + AND id_uc.is_unique_constraint = 1 + ) OPTION(RECOMPILE); /* Insert compression scripts for remaining indexes */ From 2d0ad6e3c1f4a3fb2e876250fc6dcfd8feb16b5d Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Fri, 14 Mar 2025 23:12:05 -0400 Subject: [PATCH 108/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 6be6226d..c12aa18f 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -2337,7 +2337,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND ia_uc.action = N'DISABLE' AND ia_uc.target_index_name = ia_nc.index_name ) - AND ia_nc.action IS NULL /* Only update if no action has been set yet */ + /* Allow overriding existing actions - special constraint handling takes priority */ + /* Explicitly apply to indexes named uq_i_a for debugging - REMOVE THIS CONDITION LATER */ + OR ia_nc.index_name = 'uq_i_a' OPTION(RECOMPILE); IF @debug = 1 @@ -2347,6 +2349,19 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ia.* FROM #index_analysis AS ia OPTION(RECOMPILE); + + /* Special debug for uq_a and uq_i_a */ + RAISERROR('Special debug for uq_a and uq_i_a after rule 7.5:', 0, 0) WITH NOWAIT; + SELECT + index_name, + action, + consolidation_rule, + target_index_name, + included_columns + FROM #index_analysis + WHERE index_name IN ('uq_a', 'uq_i_a') + ORDER BY index_name + OPTION(RECOMPILE); END; /* Rule 8: Identify indexes with same keys but in different order after first column */ From 7b4d9955085168cd56e9ec8d9aec74b23da33b24 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Fri, 14 Mar 2025 23:19:02 -0400 Subject: [PATCH 109/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 58 +++++++++++++++++++---------- 1 file changed, 38 insertions(+), 20 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index c12aa18f..bcb2f10c 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -2314,32 +2314,37 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ia_nc SET ia_nc.consolidation_rule = 'Unique Constraint Replacement', - ia_nc.action = N'MAKE UNIQUE' /* Mark nonclustered index to be made unique */ + ia_nc.action = N'MAKE UNIQUE', /* Mark nonclustered index to be made unique */ + /* IMPORTANT: Clear the target_index_name to ensure it gets a MERGE script */ + ia_nc.target_index_name = NULL FROM #index_analysis AS ia_nc /* Nonclustered index */ JOIN #index_details AS id_nc /* Join to get nonclustered index details */ ON id_nc.database_id = ia_nc.database_id AND id_nc.object_id = ia_nc.object_id AND id_nc.index_id = ia_nc.index_id AND id_nc.is_unique_constraint = 0 /* This is not a unique constraint */ - AND id_nc.is_unique = 0 /* This is not already unique */ - WHERE EXISTS ( - /* Find matching unique constraint that has been marked for disabling */ - SELECT 1 - FROM #index_analysis AS ia_uc - JOIN #index_details AS id_uc - ON id_uc.database_id = ia_uc.database_id - AND id_uc.object_id = ia_uc.object_id - AND id_uc.index_id = ia_uc.index_id - AND id_uc.is_unique_constraint = 1 - WHERE - ia_uc.database_id = ia_nc.database_id - AND ia_uc.object_id = ia_nc.object_id - AND ia_uc.action = N'DISABLE' - AND ia_uc.target_index_name = ia_nc.index_name - ) - /* Allow overriding existing actions - special constraint handling takes priority */ - /* Explicitly apply to indexes named uq_i_a for debugging - REMOVE THIS CONDITION LATER */ - OR ia_nc.index_name = 'uq_i_a' + WHERE + /* Two conditions for matching: + 1. Index key columns exactly match a unique constraint's key columns + 2. A unique constraint is already marked for DISABLE and has this index as target */ + (EXISTS ( + /* Find unique constraint with matching keys that should be disabled */ + SELECT 1 + FROM #index_analysis AS ia_uc + JOIN #index_details AS id_uc + ON id_uc.database_id = ia_uc.database_id + AND id_uc.object_id = ia_uc.object_id + AND id_uc.index_id = ia_uc.index_id + AND id_uc.is_unique_constraint = 1 + WHERE + ia_uc.database_id = ia_nc.database_id + AND ia_uc.object_id = ia_nc.object_id + /* Verify the key columns match */ + AND (ia_uc.key_columns = ia_nc.key_columns + OR ia_uc.target_index_name = ia_nc.index_name) /* Or it's already targeted */ + ) + /* Special case for debugging - can be removed later */ + OR ia_nc.index_name = 'uq_i_a') OPTION(RECOMPILE); IF @debug = 1 @@ -2967,6 +2972,19 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND id_uc.index_id = ia.index_id AND id_uc.is_unique_constraint = 1 ) + /* Also exclude any index that is also going to be made unique in rule 7.5 */ + AND NOT EXISTS + ( + SELECT + 1/0 + FROM #index_analysis AS ia_unique + WHERE ia_unique.database_id = ia.database_id + AND ia_unique.object_id = ia.object_id + AND ia_unique.index_name = ia.index_name + AND ia_unique.action = N'MAKE UNIQUE' + ) + /* Explicit exclusion for our test index */ + AND ia.index_name <> 'uq_i_a' OPTION(RECOMPILE); /* Insert compression scripts for remaining indexes */ From ce6a93c716ad6fcf25cd7062a273ca4adc09b4f4 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Fri, 14 Mar 2025 23:32:49 -0400 Subject: [PATCH 110/246] stuff stuff --- .DS_Store | Bin 6148 -> 6148 bytes sp_IndexCleanup/sp_IndexCleanup.sql | 83 +++++++++++++++++++++++++++- 2 files changed, 80 insertions(+), 3 deletions(-) diff --git a/.DS_Store b/.DS_Store index 7433f156746e183515111389140bab6b43e80b93..01762da1ba08e3fe8df02781df8d49caf96b5a20 100644 GIT binary patch delta 38 ucmZoMXffE3%sSbbrHsSEz(hwu*V1J216G;IcUe?6E3iFd+|17LmmdJ;jNOy(vB)#_Y?fnt$+($~;~zf& DWDySf diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index bcb2f10c..53827781 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -2315,7 +2315,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SET ia_nc.consolidation_rule = 'Unique Constraint Replacement', ia_nc.action = N'MAKE UNIQUE', /* Mark nonclustered index to be made unique */ - /* IMPORTANT: Clear the target_index_name to ensure it gets a MERGE script */ + /* CRITICAL: Set target_index_name to NULL to ensure it gets a MERGE script */ ia_nc.target_index_name = NULL FROM #index_analysis AS ia_nc /* Nonclustered index */ JOIN #index_details AS id_nc /* Join to get nonclustered index details */ @@ -2347,6 +2347,29 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. OR ia_nc.index_name = 'uq_i_a') OPTION(RECOMPILE); + /* Run this update again to fix any target_index_name cross-references */ + /* This is necessary because constraints might point to indexes and indexes might point to constraints */ + UPDATE #index_analysis + SET target_index_name = NULL + WHERE action = N'MAKE UNIQUE'; + + /* Make sure the nonclustered index has the superseded_by field set correctly */ + UPDATE ia_nc + SET + ia_nc.superseded_by = + CASE + WHEN ia_nc.superseded_by IS NULL THEN N'Will replace constraint ' + ia_uc.index_name + ELSE ia_nc.superseded_by + N', will replace constraint ' + ia_uc.index_name + END + FROM #index_analysis AS ia_nc + JOIN #index_analysis AS ia_uc + ON ia_uc.database_id = ia_nc.database_id + AND ia_uc.object_id = ia_nc.object_id + AND ia_uc.action = N'DISABLE' + AND ia_uc.target_index_name = ia_nc.index_name + WHERE ia_nc.action = N'MAKE UNIQUE' + OPTION(RECOMPILE); + IF @debug = 1 BEGIN SELECT @@ -2362,11 +2385,42 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. action, consolidation_rule, target_index_name, - included_columns + superseded_by, + included_columns, + index_priority FROM #index_analysis WHERE index_name IN ('uq_a', 'uq_i_a') ORDER BY index_name OPTION(RECOMPILE); + + /* Check the merge script eligibility */ + RAISERROR('Checking MERGE script eligibility for uq_i_a:', 0, 0) WITH NOWAIT; + SELECT + 'uq_i_a eligibility check', + ia.index_name, + ia.action, + ia.target_index_name, + ce.can_compress, + /* Show which conditions are being met for script generation */ + condition1 = CASE WHEN ia.action IN (N'MERGE INCLUDES', N'MAKE UNIQUE') THEN 'YES' ELSE 'NO' END, + condition2 = CASE WHEN ce.can_compress = 1 THEN 'YES' ELSE 'NO' END, + condition3 = CASE WHEN ia.target_index_name IS NULL THEN 'YES' ELSE 'NO' END, + /* Will this index get a MERGE script? */ + will_get_merge_script = + CASE + WHEN ia.action IN (N'MERGE INCLUDES', N'MAKE UNIQUE') + AND ce.can_compress = 1 + AND ia.target_index_name IS NULL + THEN 'YES' + ELSE 'NO' + END + FROM #index_analysis AS ia + JOIN #compression_eligibility AS ce + ON ia.database_id = ce.database_id + AND ia.object_id = ce.object_id + AND ia.index_id = ce.index_id + WHERE ia.index_name = 'uq_i_a' + OPTION(RECOMPILE); END; /* Rule 8: Identify indexes with same keys but in different order after first column */ @@ -2872,9 +2926,32 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. WHERE ia.action IN (N'MERGE INCLUDES', N'MAKE UNIQUE') AND ce.can_compress = 1 /* Only create merge scripts for the indexes that should remain after merging */ - AND ia.target_index_name IS NULL + AND (ia.target_index_name IS NULL OR ia.index_name = 'uq_i_a') OPTION(RECOMPILE); + /* Debug which indexes are getting MERGE scripts */ + IF @debug = 1 + BEGIN + RAISERROR('Indexes getting MERGE scripts:', 0, 0) WITH NOWAIT; + SELECT + index_name, + action, + consolidation_rule, + target_index_name, + script_type = 'WILL GET MERGE SCRIPT', + included_columns + FROM #index_analysis AS ia + JOIN #compression_eligibility AS ce + ON ia.database_id = ce.database_id + AND ia.object_id = ce.object_id + AND ia.index_id = ce.index_id + WHERE ia.action IN (N'MERGE INCLUDES', N'MAKE UNIQUE') + AND ce.can_compress = 1 + AND (ia.target_index_name IS NULL OR ia.index_name = 'uq_i_a') + ORDER BY index_name + OPTION(RECOMPILE); + END; + /* Insert disable scripts for unneeded indexes */ IF @debug = 1 BEGIN From 513a64108b5e96c2bf96549fc4660ca2973d6aef Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Fri, 14 Mar 2025 23:34:13 -0400 Subject: [PATCH 111/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 53827781..6d48561e 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -2926,7 +2926,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. WHERE ia.action IN (N'MERGE INCLUDES', N'MAKE UNIQUE') AND ce.can_compress = 1 /* Only create merge scripts for the indexes that should remain after merging */ - AND (ia.target_index_name IS NULL OR ia.index_name = 'uq_i_a') + /* Special handling for indexes that need to be made unique (Rule 7.5) */ + AND (ia.target_index_name IS NULL OR ia.action = N'MAKE UNIQUE') OPTION(RECOMPILE); /* Debug which indexes are getting MERGE scripts */ @@ -2934,12 +2935,12 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. BEGIN RAISERROR('Indexes getting MERGE scripts:', 0, 0) WITH NOWAIT; SELECT - index_name, - action, - consolidation_rule, - target_index_name, + ia.index_name, + ia.action, + ia.consolidation_rule, + ia.target_index_name, script_type = 'WILL GET MERGE SCRIPT', - included_columns + ia.included_columns FROM #index_analysis AS ia JOIN #compression_eligibility AS ce ON ia.database_id = ce.database_id @@ -2947,8 +2948,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND ia.index_id = ce.index_id WHERE ia.action IN (N'MERGE INCLUDES', N'MAKE UNIQUE') AND ce.can_compress = 1 - AND (ia.target_index_name IS NULL OR ia.index_name = 'uq_i_a') - ORDER BY index_name + AND (ia.target_index_name IS NULL OR ia.action = N'MAKE UNIQUE') + ORDER BY ia.index_name OPTION(RECOMPILE); END; @@ -3060,8 +3061,6 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND ia_unique.index_name = ia.index_name AND ia_unique.action = N'MAKE UNIQUE' ) - /* Explicit exclusion for our test index */ - AND ia.index_name <> 'uq_i_a' OPTION(RECOMPILE); /* Insert compression scripts for remaining indexes */ From 2d87fa715168b58e53da73d3d757d4dfaed65ec3 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Fri, 14 Mar 2025 23:42:58 -0400 Subject: [PATCH 112/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 54 +++++++++++++++++++---------- 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 6d48561e..576833d1 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -1588,17 +1588,33 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. id1.filter_definition, /* Store the original index definition for validation */ original_index_definition = - N'CREATE ' + - CASE WHEN id1.is_unique = 1 THEN N'UNIQUE ' ELSE N'' END + - N'INDEX ' + - QUOTENAME(id1.index_name) + - N' ON ' + - QUOTENAME(DB_NAME(@database_id)) + - N'.' + - QUOTENAME(id1.schema_name) + - N'.' + - QUOTENAME(id1.table_name) + - N' (' + + CASE + /* For unique constraints, use ALTER TABLE ADD CONSTRAINT syntax */ + WHEN id1.is_unique_constraint = 1 + THEN + N'ALTER TABLE ' + + QUOTENAME(DB_NAME(@database_id)) + + N'.' + + QUOTENAME(id1.schema_name) + + N'.' + + QUOTENAME(id1.table_name) + + N' ADD CONSTRAINT ' + + QUOTENAME(id1.index_name) + + N' UNIQUE (' + /* For regular indexes, use CREATE INDEX syntax */ + ELSE + N'CREATE ' + + CASE WHEN id1.is_unique = 1 THEN N'UNIQUE ' ELSE N'' END + + N'INDEX ' + + QUOTENAME(id1.index_name) + + N' ON ' + + QUOTENAME(DB_NAME(@database_id)) + + N'.' + + QUOTENAME(id1.schema_name) + + N'.' + + QUOTENAME(id1.table_name) + + N' (' + END + STUFF ( ( @@ -2860,6 +2876,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ia.filter_definition ELSE N'' END + + N' WITH (DROP_EXISTING = ON, FILLFACTOR = 100, SORT_IN_TEMPDB = ON, ONLINE = ' + + CASE + WHEN @online = 1 + THEN N'ON' + ELSE N'OFF' + END + + N', DATA_COMPRESSION = PAGE)' + CASE WHEN ps.partition_function_name IS NOT NULL THEN N' ON ' + @@ -2871,14 +2894,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. THEN N' ON ' + QUOTENAME(ps.built_on) ELSE N'' - END + - N' WITH (DROP_EXISTING = ON, FILLFACTOR = 100, SORT_IN_TEMPDB = ON, ONLINE = ' + - CASE - WHEN @online = 1 - THEN N'ON' - ELSE N'OFF' - END + - N', DATA_COMPRESSION = PAGE);', + END + N';', /* Additional info about what this script does */ additional_info = CASE From 5229ee391ecb3552a1667199c72588c980b67330 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Fri, 14 Mar 2025 23:49:48 -0400 Subject: [PATCH 113/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 576833d1..3ec58a43 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -1699,7 +1699,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. id1.is_unique, id1.object_id, id1.index_id, - id1.filter_definition + id1.filter_definition, + id1.is_unique_constraint OPTION(RECOMPILE); IF ROWCOUNT_BIG() = 0 @@ -2836,13 +2837,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. script = CASE WHEN ia.action = N'MAKE UNIQUE' - THEN N'/* This index can replace a unique constraint */ - /* Creating unique index with same keys as constraint */ - CREATE UNIQUE ' + THEN N'CREATE UNIQUE ' WHEN ia.action = N'MERGE INCLUDES' - THEN N'/* This index can be merged with another index */ - /* Creating index with combined includes from both */ - CREATE ' + THEN N'CREATE ' ELSE N'CREATE ' END + N'INDEX ' + From cb92a441b017a7444d1859b78943709a0a0eb35f Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 15 Mar 2025 00:05:31 -0400 Subject: [PATCH 114/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 36 ++++------------------------- 1 file changed, 5 insertions(+), 31 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 3ec58a43..b14a622d 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -2299,31 +2299,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND ia_nc.object_id = ia_uc.object_id AND ia_nc.index_name <> ia_uc.index_name /* Different index */ WHERE - /* Verify key columns match between index and unique constraint */ - NOT EXISTS - ( - SELECT - id_uc_inner.column_name - FROM #index_details AS id_uc_inner - WHERE id_uc_inner.database_id = id_uc.database_id - AND id_uc_inner.object_id = id_uc.object_id - AND id_uc_inner.index_id = id_uc.index_id - AND id_uc_inner.is_included_column = 0 - - EXCEPT - - SELECT - id_nc_inner.column_name - FROM #index_details AS id_nc_inner - JOIN #index_details AS id_nc_base - ON id_nc_inner.database_id = id_nc_base.database_id - AND id_nc_inner.object_id = id_nc_base.object_id - AND id_nc_inner.index_id = id_nc_base.index_id - WHERE id_nc_base.database_id = ia_nc.database_id - AND id_nc_base.object_id = ia_nc.object_id - AND id_nc_base.index_id = ia_nc.index_id - AND id_nc_inner.is_included_column = 0 - ) + /* Verify key columns EXACT match between index and unique constraint */ + ia_uc.key_columns = ia_nc.key_columns OPTION(RECOMPILE); /* Second, mark nonclustered indexes to be made unique */ @@ -2356,12 +2333,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. WHERE ia_uc.database_id = ia_nc.database_id AND ia_uc.object_id = ia_nc.object_id - /* Verify the key columns match */ - AND (ia_uc.key_columns = ia_nc.key_columns - OR ia_uc.target_index_name = ia_nc.index_name) /* Or it's already targeted */ - ) - /* Special case for debugging - can be removed later */ - OR ia_nc.index_name = 'uq_i_a') + /* Check that both indexes have EXACTLY the same key columns */ + AND ia_uc.key_columns = ia_nc.key_columns + )) OPTION(RECOMPILE); /* Run this update again to fix any target_index_name cross-references */ From ef4613b9972770f8cb2fe31ca9dfd150e6495320 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 15 Mar 2025 00:16:20 -0400 Subject: [PATCH 115/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index b14a622d..a9096dda 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -2338,11 +2338,24 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. )) OPTION(RECOMPILE); - /* Run this update again to fix any target_index_name cross-references */ - /* This is necessary because constraints might point to indexes and indexes might point to constraints */ - UPDATE #index_analysis - SET target_index_name = NULL - WHERE action = N'MAKE UNIQUE'; + /* CRITICAL: Ensure that only the unique constraints that exactly match get this treatment */ + /* And remove any incorrect MAKE UNIQUE actions */ + UPDATE ia + SET action = NULL, + consolidation_rule = NULL, + target_index_name = NULL + FROM #index_analysis AS ia + WHERE ia.action = N'MAKE UNIQUE' + AND NOT EXISTS ( + /* Check if there's a unique constraint with matching keys that points to this index */ + SELECT 1 + FROM #index_analysis AS ia_uc + WHERE ia_uc.database_id = ia.database_id + AND ia_uc.object_id = ia.object_id + AND ia_uc.key_columns = ia.key_columns + AND ia_uc.action = N'DISABLE' + AND ia_uc.target_index_name = ia.index_name + ); /* Make sure the nonclustered index has the superseded_by field set correctly */ UPDATE ia_nc @@ -2913,8 +2926,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. WHERE ia.action IN (N'MERGE INCLUDES', N'MAKE UNIQUE') AND ce.can_compress = 1 /* Only create merge scripts for the indexes that should remain after merging */ - /* Special handling for indexes that need to be made unique (Rule 7.5) */ - AND (ia.target_index_name IS NULL OR ia.action = N'MAKE UNIQUE') + AND ia.target_index_name IS NULL OPTION(RECOMPILE); /* Debug which indexes are getting MERGE scripts */ @@ -2935,7 +2947,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND ia.index_id = ce.index_id WHERE ia.action IN (N'MERGE INCLUDES', N'MAKE UNIQUE') AND ce.can_compress = 1 - AND (ia.target_index_name IS NULL OR ia.action = N'MAKE UNIQUE') + AND ia.target_index_name IS NULL ORDER BY ia.index_name OPTION(RECOMPILE); END; From 756ef8478d429c259931cb81c4047fd6bf67ebfa Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 15 Mar 2025 00:40:25 -0400 Subject: [PATCH 116/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 154 +++++++++++++++++++++++++++- 1 file changed, 152 insertions(+), 2 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index a9096dda..f8cab5ab 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -1837,14 +1837,18 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ia1.consolidation_rule = 'Exact Duplicate', ia1.target_index_name = CASE - WHEN ia1.index_priority >= ia2.index_priority + WHEN ia1.index_priority > ia2.index_priority THEN NULL /* This index is the keeper */ + WHEN ia1.index_priority = ia2.index_priority AND ia1.index_name < ia2.index_name + THEN NULL /* When tied, use alphabetical ordering for consistency */ ELSE ia2.index_name /* Other index is the keeper */ END, ia1.action = CASE - WHEN ia1.index_priority >= ia2.index_priority + WHEN ia1.index_priority > ia2.index_priority THEN 'KEEP' /* This index is the keeper */ + WHEN ia1.index_priority = ia2.index_priority AND ia1.index_name < ia2.index_name + THEN 'KEEP' /* When tied, use alphabetical ordering for consistency */ ELSE 'DISABLE' /* Other index gets disabled */ END FROM #index_analysis AS ia1 @@ -1907,6 +1911,34 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ia.* FROM #index_analysis AS ia OPTION(RECOMPILE); + + /* Special debug for exact duplicates */ + RAISERROR('Special debug for exact duplicates after rule 2:', 0, 0) WITH NOWAIT; + SELECT + ia1.index_name AS index1_name, + ia1.action AS index1_action, + ia1.consolidation_rule AS index1_rule, + ia1.index_priority AS index1_priority, + ia1.target_index_name AS index1_target, + ia1.filter_definition AS index1_filter, + ia2.index_name AS index2_name, + ia2.action AS index2_action, + ia2.consolidation_rule AS index2_rule, + ia2.index_priority AS index2_priority, + ia2.target_index_name AS index2_target, + ia2.filter_definition AS index2_filter + FROM #index_analysis AS ia1 + JOIN #index_analysis AS ia2 + ON ia1.database_id = ia2.database_id + AND ia1.object_id = ia2.object_id + AND ia1.index_name <> ia2.index_name + AND ia1.key_columns = ia2.key_columns /* Exact key match */ + AND ISNULL(ia1.included_columns, '') = ISNULL(ia2.included_columns, '') /* Exact includes match */ + AND ISNULL(ia1.filter_definition, '') = ISNULL(ia2.filter_definition, '') /* Matching filters */ + WHERE ia1.consolidation_rule = 'Exact Duplicate' + OR ia2.consolidation_rule = 'Exact Duplicate' + ORDER BY ia1.index_name + OPTION(RECOMPILE); END; /* Rule 3: Key duplicates - matching key columns, different includes */ @@ -2956,6 +2988,57 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. IF @debug = 1 BEGIN RAISERROR('Generating #index_cleanup_results insert, DISABLE', 0, 0) WITH NOWAIT; + + /* Debug for indexes that should get DISABLE scripts */ + RAISERROR('Indexes that should get DISABLE scripts:', 0, 0) WITH NOWAIT; + SELECT + ia.index_name, + ia.consolidation_rule, + ia.action, + ia.target_index_name, + ia.is_unique, + ia.index_priority, + is_unique_constraint = + CASE WHEN EXISTS ( + SELECT 1 + FROM #index_details AS id + WHERE id.database_id = ia.database_id + AND id.object_id = ia.object_id + AND id.index_id = ia.index_id + AND id.is_unique_constraint = 1 + ) THEN 'YES' ELSE 'NO' END, + make_unique_target = + CASE WHEN EXISTS ( + SELECT 1 + FROM #index_analysis AS ia_make + WHERE ia_make.database_id = ia.database_id + AND ia_make.object_id = ia.object_id + AND ia_make.action = 'MAKE UNIQUE' + AND ia_make.target_index_name = ia.index_name + ) THEN 'YES' ELSE 'NO' END, + will_get_script = + CASE WHEN ia.action = 'DISABLE' AND NOT EXISTS ( + SELECT 1 + FROM #index_details AS id_uc + WHERE id_uc.database_id = ia.database_id + AND id_uc.object_id = ia.object_id + AND id_uc.index_id = ia.index_id + AND id_uc.is_unique_constraint = 1 + ) THEN 'YES' ELSE 'NO' END + FROM #index_analysis AS ia + WHERE ia.index_name LIKE 'ix_filtered_%' OR ia.index_name LIKE 'ix_desc_%' + ORDER BY ia.index_name; + + /* Debug for all indexes marked with action = DISABLE */ + RAISERROR('All indexes with action = DISABLE:', 0, 0) WITH NOWAIT; + SELECT + ia.index_name, + ia.consolidation_rule, + ia.action, + ia.target_index_name + FROM #index_analysis AS ia + WHERE ia.action = 'DISABLE' + ORDER BY ia.index_name; END; INSERT INTO @@ -3173,6 +3256,73 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. BEGIN RAISERROR('Generating #index_cleanup_results insert, CONSTRAINT', 0, 0) WITH NOWAIT; END; + + /* Add code to insert KEPT indexes into the results - THESE WERE MISSING! */ + IF @debug = 1 + BEGIN + RAISERROR('Generating #index_cleanup_results insert, KEPT', 0, 0) WITH NOWAIT; + END; + + /* Insert KEPT indexes into results */ + INSERT INTO + #index_cleanup_results + ( + result_type, + sort_order, + database_name, + schema_name, + table_name, + index_name, + script_type, + consolidation_rule, + additional_info, + script, + original_index_definition, + index_size_gb, + index_rows, + index_reads, + index_writes + ) + SELECT DISTINCT + result_type = 'KEPT', + sort_order = 95, /* Put kept indexes at the end */ + ia.database_name, + ia.schema_name, + ia.table_name, + ia.index_name, + script_type = NULL, + ia.consolidation_rule, + additional_info = + CASE + WHEN ia.consolidation_rule IS NOT NULL + THEN 'This index is being kept' + ELSE NULL + END, + script = NULL, /* No script for kept indexes */ + /* Original index definition for validation */ + ia.original_index_definition, + ps.total_space_gb, + ps.total_rows, + index_reads = + (id.user_seeks + id.user_scans + id.user_lookups), + id.user_updates + FROM #index_analysis AS ia + LEFT JOIN #partition_stats AS ps + ON ia.database_id = ps.database_id + AND ia.object_id = ps.object_id + AND ia.index_id = ps.index_id + LEFT JOIN #index_details AS id + ON id.database_id = ia.database_id + AND id.object_id = ia.object_id + AND id.index_id = ia.index_id + AND id.is_included_column = 0 /* Get only one row per index */ + AND id.key_ordinal > 0 + WHERE + /* Include indexes marked KEEP */ + (ia.action = 'KEEP') + /* And all indexes we haven't determined an action for (not disable, merge, etc.) */ + OR (ia.action IS NULL AND ia.index_id > 0) + OPTION(RECOMPILE); INSERT INTO #index_cleanup_results From 614dfbee29de78cdd929df5f78a59606f6e03efa Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 15 Mar 2025 00:43:01 -0400 Subject: [PATCH 117/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index f8cab5ab..f0635132 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -3317,11 +3317,21 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND id.index_id = ia.index_id AND id.is_included_column = 0 /* Get only one row per index */ AND id.key_ordinal > 0 - WHERE + /* Check that this index is not already in the results */ + WHERE NOT EXISTS ( + SELECT 1 FROM #index_cleanup_results AS ir + WHERE ir.database_name = ia.database_name + AND ir.schema_name = ia.schema_name + AND ir.table_name = ia.table_name + AND ir.index_name = ia.index_name + ) + /* And include only indexes that should be kept */ + AND ( /* Include indexes marked KEEP */ (ia.action = 'KEEP') /* And all indexes we haven't determined an action for (not disable, merge, etc.) */ OR (ia.action IS NULL AND ia.index_id > 0) + ) OPTION(RECOMPILE); INSERT INTO @@ -3996,9 +4006,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. RAISERROR('Generating #index_cleanup_results, RESULTS', 0, 0) WITH NOWAIT; END; - SELECT + SELECT DISTINCT /* First, show the information needed to understand the script */ - ir.script_type, + script_type = CASE WHEN ir.result_type = 'KEPT' AND ir.script_type IS NULL THEN 'KEPT' ELSE ir.script_type END, ir.additional_info, /* Then show identifying information for the index */ ir.database_name, From 419c26d94f1076fb0f72bfe7870d240cff9bc71f Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 15 Mar 2025 00:46:20 -0400 Subject: [PATCH 118/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index f0635132..6cc34c44 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -4006,7 +4006,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. RAISERROR('Generating #index_cleanup_results, RESULTS', 0, 0) WITH NOWAIT; END; - SELECT DISTINCT + SELECT /* First, show the information needed to understand the script */ script_type = CASE WHEN ir.result_type = 'KEPT' AND ir.script_type IS NULL THEN 'KEPT' ELSE ir.script_type END, ir.additional_info, @@ -4053,12 +4053,22 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ia.original_index_definition, /* Finally show the actual script */ ir.script - FROM #index_cleanup_results AS ir + FROM + ( + /* Use a subquery with ROW_NUMBER to ensure we only get one row per index */ + SELECT *, + ROW_NUMBER() OVER( + PARTITION BY database_name, schema_name, table_name, index_name + ORDER BY result_type DESC /* Prefer non-NULL result types */ + ) AS rn + FROM #index_cleanup_results + ) AS ir LEFT JOIN #index_analysis AS ia ON ir.database_name = ia.database_name AND ir.schema_name = ia.schema_name AND ir.table_name = ia.table_name AND ir.index_name = ia.index_name + WHERE ir.rn = 1 /* Take only the first row for each index */ ORDER BY ir.sort_order, ir.database_name, From 36f40ae779a90e6d8cff5d015bcfd5df878843e3 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 15 Mar 2025 09:38:03 -0400 Subject: [PATCH 119/246] Update sp_HealthParser.sql fix some indenting/spacing issues with dynamic sql cleaned up redundant replace code on the max event dynamic sql template added some debug raiserror output for no results found --- sp_HealthParser/sp_HealthParser.sql | 924 ++++++++++++---------------- 1 file changed, 404 insertions(+), 520 deletions(-) diff --git a/sp_HealthParser/sp_HealthParser.sql b/sp_HealthParser/sp_HealthParser.sql index 135a5d36..4be9b955 100644 --- a/sp_HealthParser/sp_HealthParser.sql +++ b/sp_HealthParser/sp_HealthParser.sql @@ -191,14 +191,14 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -', 0, 1) WITH NOWAIT; +', 0, 0) WITH NOWAIT; RETURN; END; /*End help section*/ IF @debug = 1 BEGIN - RAISERROR('Declaring variables', 0, 1) WITH NOWAIT; + RAISERROR('Declaring variables', 0, 0) WITH NOWAIT; END; DECLARE @@ -262,7 +262,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. @log_database_schema nvarchar(1024), @max_event_time datetime2(7), @dsql nvarchar(max) = N'', - @mdsql nvarchar(max) = N''; + @mdsql_template nvarchar(max) = N'', + @mdsql_execute nvarchar(MAX) = N''; IF @azure = 1 BEGIN @@ -272,7 +273,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. IF @debug = 1 BEGIN - RAISERROR('Fixing parameters and variables', 0, 1) WITH NOWAIT; + RAISERROR('Fixing parameters and variables', 0, 0) WITH NOWAIT; END; SELECT @@ -380,58 +381,59 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ELSE 0 END, @sql_template += N' -INSERT INTO - {temp_table} -WITH - (TABLOCK) -( - {insert_list} -) -SELECT - {object_name} = - ISNULL - ( - xml.{object_name}, - CONVERT(xml, N''event'') - ) -FROM -( + INSERT INTO + {temp_table} + WITH + (TABLOCK) + ( + {insert_list} + ) SELECT {object_name} = - TRY_CAST(fx.event_data AS xml) - FROM sys.fn_xe_file_target_read_file(N''system_health*.xel'', NULL, NULL, NULL) AS fx - WHERE fx.object_name = N''{object_name}'' {time_filter} -) AS xml -{cross_apply} -OPTION(RECOMPILE); -', - @mdsql = N' -IF OBJECT_ID(''{table_check}'', ''U'') IS NOT NULL -BEGIN - SELECT - @max_event_time = ISNULL ( - MAX({date_column}), - DATEADD - ( - MINUTE, - DATEDIFF - ( - MINUTE, - SYSDATETIME(), - GETUTCDATE() - ), - DATEADD - ( - DAY, - -1, - SYSDATETIME() - ) - ) + xml.{object_name}, + CONVERT(xml, N''event'') ) - FROM {table_check}; -END;'; + FROM + ( + SELECT + {object_name} = + TRY_CAST(fx.event_data AS xml) + FROM sys.fn_xe_file_target_read_file(N''system_health*.xel'', NULL, NULL, NULL) AS fx + WHERE fx.object_name = N''{object_name}'' {time_filter} + ) AS xml + {cross_apply} + OPTION(RECOMPILE); +', + @mdsql_template = N' + IF OBJECT_ID(''{table_check}'', ''U'') IS NOT NULL + BEGIN + SELECT + @max_event_time = + ISNULL + ( + MAX({date_column}), + DATEADD + ( + MINUTE, + DATEDIFF + ( + MINUTE, + SYSDATETIME(), + GETUTCDATE() + ), + DATEADD + ( + DAY, + -1, + SYSDATETIME() + ) + ) + ) + FROM {table_check}; + END; + '; IF @timestamp_utc_mode = 0 BEGIN @@ -1076,38 +1078,39 @@ AND ca.utc_timestamp < @end_date'; /* Clean up each log table */ SET @dsql = N' - DELETE FROM ' + @log_table_significant_waits + ' - WHERE collection_time < @cleanup_date; - - DELETE FROM ' + @log_table_waits_by_count + ' - WHERE collection_time < @cleanup_date; - - DELETE FROM ' + @log_table_waits_by_duration + ' - WHERE collection_time < @cleanup_date; - - DELETE FROM ' + @log_table_io_issues + ' - WHERE collection_time < @cleanup_date; - - DELETE FROM ' + @log_table_cpu_tasks + ' - WHERE collection_time < @cleanup_date; - - DELETE FROM ' + @log_table_memory_conditions + ' - WHERE collection_time < @cleanup_date; - - DELETE FROM ' + @log_table_memory_broker + ' - WHERE collection_time < @cleanup_date; - - DELETE FROM ' + @log_table_memory_node_oom + ' - WHERE collection_time < @cleanup_date; - - DELETE FROM ' + @log_table_system_health + ' - WHERE collection_time < @cleanup_date; - - DELETE FROM ' + @log_table_scheduler_issues + ' - WHERE collection_time < @cleanup_date; - - DELETE FROM ' + @log_table_severe_errors + ' - WHERE collection_time < @cleanup_date;'; + DELETE FROM ' + @log_table_significant_waits + ' + WHERE collection_time < @cleanup_date; + + DELETE FROM ' + @log_table_waits_by_count + ' + WHERE collection_time < @cleanup_date; + + DELETE FROM ' + @log_table_waits_by_duration + ' + WHERE collection_time < @cleanup_date; + + DELETE FROM ' + @log_table_io_issues + ' + WHERE collection_time < @cleanup_date; + + DELETE FROM ' + @log_table_cpu_tasks + ' + WHERE collection_time < @cleanup_date; + + DELETE FROM ' + @log_table_memory_conditions + ' + WHERE collection_time < @cleanup_date; + + DELETE FROM ' + @log_table_memory_broker + ' + WHERE collection_time < @cleanup_date; + + DELETE FROM ' + @log_table_memory_node_oom + ' + WHERE collection_time < @cleanup_date; + + DELETE FROM ' + @log_table_system_health + ' + WHERE collection_time < @cleanup_date; + + DELETE FROM ' + @log_table_scheduler_issues + ' + WHERE collection_time < @cleanup_date; + + DELETE FROM ' + @log_table_severe_errors + ' + WHERE collection_time < @cleanup_date; + '; IF @debug = 1 BEGIN @@ -1121,14 +1124,14 @@ AND ca.utc_timestamp < @end_date'; IF @debug = 1 BEGIN - RAISERROR('Log cleanup complete', 0, 1) WITH NOWAIT; + RAISERROR('Log cleanup complete', 0, 0) WITH NOWAIT; END; END; END; IF @debug = 1 BEGIN - RAISERROR('Creating temp tables', 0, 1) WITH NOWAIT; + RAISERROR('Creating temp tables', 0, 0) WITH NOWAIT; END; DECLARE @@ -1274,7 +1277,7 @@ AND ca.utc_timestamp < @end_date'; BEGIN IF @debug = 1 BEGIN - RAISERROR('Inserting ignorable waits to #ignore_waits', 0, 1) WITH NOWAIT; + RAISERROR('Inserting ignorable waits to #ignore_waits', 0, 0) WITH NOWAIT; END; INSERT @@ -1325,7 +1328,7 @@ AND ca.utc_timestamp < @end_date'; /* First, ensure we're working with the correct collection areas */ IF @debug = 1 BEGIN - RAISERROR('Beginning collection loop for system_health data', 0, 1) WITH NOWAIT; + RAISERROR('Beginning collection loop for system_health data', 0, 0) WITH NOWAIT; END; /* Declare a cursor to process each collection area */ @@ -1387,7 +1390,7 @@ AND ca.utc_timestamp < @end_date'; IF @debug = 1 BEGIN - RAISERROR('Executing collection SQL', 0, 1) WITH NOWAIT; + RAISERROR('Executing collection SQL', 0, 0) WITH NOWAIT; SET STATISTICS XML ON; END; @@ -1420,15 +1423,15 @@ AND ca.utc_timestamp < @end_date'; IF @debug = 1 BEGIN - RAISERROR('Data collection complete', 0, 1) WITH NOWAIT; + RAISERROR('Data collection complete', 0, 0) WITH NOWAIT; END; IF @mi = 1 BEGIN IF @debug = 1 BEGIN - RAISERROR('Starting Managed Instance analysis', 0, 1) WITH NOWAIT; - RAISERROR('Inserting #x', 0, 1) WITH NOWAIT; + RAISERROR('Starting Managed Instance analysis', 0, 0) WITH NOWAIT; + RAISERROR('Inserting #x', 0, 0) WITH NOWAIT; END; INSERT @@ -1462,7 +1465,7 @@ AND ca.utc_timestamp < @end_date'; IF @debug = 1 BEGIN - RAISERROR('Inserting #ring_buffer', 0, 1) WITH NOWAIT; + RAISERROR('Inserting #ring_buffer', 0, 0) WITH NOWAIT; END; INSERT @@ -1494,8 +1497,8 @@ AND ca.utc_timestamp < @end_date'; BEGIN IF @debug = 1 BEGIN - RAISERROR('Checking Managed Instance waits', 0, 1) WITH NOWAIT; - RAISERROR('Inserting #wait_info', 0, 1) WITH NOWAIT; + RAISERROR('Checking Managed Instance waits', 0, 0) WITH NOWAIT; + RAISERROR('Inserting #wait_info', 0, 0) WITH NOWAIT; END; INSERT @@ -1516,8 +1519,8 @@ AND ca.utc_timestamp < @end_date'; BEGIN IF @debug = 1 BEGIN - RAISERROR('Checking Managed Instance sp_server_diagnostics_component_result', 0, 1) WITH NOWAIT; - RAISERROR('Inserting #sp_server_diagnostics_component_result', 0, 1) WITH NOWAIT; + RAISERROR('Checking Managed Instance sp_server_diagnostics_component_result', 0, 0) WITH NOWAIT; + RAISERROR('Inserting #sp_server_diagnostics_component_result', 0, 0) WITH NOWAIT; END; INSERT @@ -1543,8 +1546,8 @@ AND ca.utc_timestamp < @end_date'; BEGIN IF @debug = 1 BEGIN - RAISERROR('Checking Managed Instance deadlocks', 0, 1) WITH NOWAIT; - RAISERROR('Inserting #xml_deadlock_report', 0, 1) WITH NOWAIT; + RAISERROR('Checking Managed Instance deadlocks', 0, 0) WITH NOWAIT; + RAISERROR('Inserting #xml_deadlock_report', 0, 0) WITH NOWAIT; END; INSERT @@ -1567,8 +1570,8 @@ AND ca.utc_timestamp < @end_date'; BEGIN IF @debug = 1 BEGIN - RAISERROR('Checking Managed Instance scheduler monitor', 0, 1) WITH NOWAIT; - RAISERROR('Inserting #scheduler_monitor', 0, 1) WITH NOWAIT; + RAISERROR('Checking Managed Instance scheduler monitor', 0, 0) WITH NOWAIT; + RAISERROR('Inserting #scheduler_monitor', 0, 0) WITH NOWAIT; END; INSERT @@ -1591,8 +1594,8 @@ AND ca.utc_timestamp < @end_date'; BEGIN IF @debug = 1 BEGIN - RAISERROR('Checking Managed Instance error reported events', 0, 1) WITH NOWAIT; - RAISERROR('Inserting #error_reported', 0, 1) WITH NOWAIT; + RAISERROR('Checking Managed Instance error reported events', 0, 0) WITH NOWAIT; + RAISERROR('Inserting #error_reported', 0, 0) WITH NOWAIT; END; INSERT @@ -1615,8 +1618,8 @@ AND ca.utc_timestamp < @end_date'; BEGIN IF @debug = 1 BEGIN - RAISERROR('Checking Managed Instance memory broker events', 0, 1) WITH NOWAIT; - RAISERROR('Inserting #memory_broker', 0, 1) WITH NOWAIT; + RAISERROR('Checking Managed Instance memory broker events', 0, 0) WITH NOWAIT; + RAISERROR('Inserting #memory_broker', 0, 0) WITH NOWAIT; END; INSERT @@ -1679,7 +1682,7 @@ AND ca.utc_timestamp < @end_date'; BEGIN IF @debug = 1 BEGIN - RAISERROR('Parsing queries with significant waits', 0, 1) WITH NOWAIT; + RAISERROR('Parsing queries with significant waits', 0, 0) WITH NOWAIT; END; SELECT @@ -1720,7 +1723,7 @@ AND ca.utc_timestamp < @end_date'; IF @debug = 1 BEGIN - RAISERROR('Adding query_text to #waits_queries', 0, 1) WITH NOWAIT; + RAISERROR('Adding query_text to #waits_queries', 0, 0) WITH NOWAIT; END; ALTER TABLE #waits_queries @@ -1770,6 +1773,8 @@ AND ca.utc_timestamp < @end_date'; ELSE 'no queries with significant waits found!' END WHERE @log_to_table = 0; + + RAISERROR('No queries with significant waits found', 0, 0) WITH NOWAIT; END; ELSE BEGIN @@ -1832,12 +1837,12 @@ AND ca.utc_timestamp < @end_date'; /* Add the WHERE clause only for table logging */ IF @log_to_table = 1 BEGIN - SET @mdsql = + SET @mdsql_execute = REPLACE ( REPLACE ( - @mdsql, + @mdsql_template, '{table_check}', @log_table_significant_waits ), @@ -1847,26 +1852,13 @@ AND ca.utc_timestamp < @end_date'; IF @debug = 1 BEGIN - PRINT @mdsql; + PRINT @mdsql_execute; END; EXECUTE sys.sp_executesql - @mdsql, + @mdsql_execute, N'@max_event_time datetime2(7) OUTPUT', @max_event_time OUTPUT; - - SET @mdsql = - REPLACE - ( - REPLACE - ( - @mdsql, - @log_table_significant_waits, - '{table_check}' - ), - 'event_time', - '{date_column}' - ); SET @dsql += N' WHERE wq.event_time > @max_event_time'; @@ -1876,7 +1868,8 @@ AND ca.utc_timestamp < @end_date'; SET @dsql += N' ORDER BY wq.duration_ms DESC - OPTION(RECOMPILE);'; + OPTION(RECOMPILE); + '; /* Handle table logging */ IF @log_to_table = 1 @@ -1922,7 +1915,7 @@ AND ca.utc_timestamp < @end_date'; /*Waits by count*/ IF @debug = 1 BEGIN - RAISERROR('Parsing #waits_by_count', 0, 1) WITH NOWAIT; + RAISERROR('Parsing #waits_by_count', 0, 0) WITH NOWAIT; END; SELECT @@ -2029,6 +2022,8 @@ AND ca.utc_timestamp < @end_date'; ELSE 'no significant waits found!' END WHERE @log_to_table = 0; + + RAISERROR('No waits by count found', 0, 0) WITH NOWAIT; END; ELSE BEGIN @@ -2096,12 +2091,12 @@ AND ca.utc_timestamp < @end_date'; /* Add the WHERE clause only for table logging */ IF @log_to_table = 1 BEGIN - SET @mdsql = + SET @mdsql_execute = REPLACE ( REPLACE ( - @mdsql, + @mdsql_template, '{table_check}', @log_table_waits_by_count ), @@ -2111,26 +2106,13 @@ AND ca.utc_timestamp < @end_date'; IF @debug = 1 BEGIN - PRINT @mdsql; + PRINT @mdsql_execute; END; EXECUTE sys.sp_executesql - @mdsql, + @mdsql_execute, N'@max_event_time datetime2(7) OUTPUT', @max_event_time OUTPUT; - - SET @mdsql = - REPLACE - ( - REPLACE - ( - @mdsql, - @log_table_waits_by_count, - '{table_check}' - ), - 'event_time_rounded', - '{date_column}' - ); SET @dsql += N' WHERE t.event_time_rounded > @max_event_time'; @@ -2141,7 +2123,8 @@ AND ca.utc_timestamp < @end_date'; ORDER BY t.event_time_rounded DESC, t.waits DESC - OPTION(RECOMPILE);'; + OPTION(RECOMPILE); + '; /* Handle table logging */ IF @log_to_table = 1 @@ -2185,7 +2168,7 @@ AND ca.utc_timestamp < @end_date'; /*Grab waits by duration*/ IF @debug = 1 BEGIN - RAISERROR('Parsing waits by duration', 0, 1) WITH NOWAIT; + RAISERROR('Parsing waits by duration', 0, 0) WITH NOWAIT; END; SELECT @@ -2298,6 +2281,8 @@ AND ca.utc_timestamp < @end_date'; ELSE 'no significant waits found!' END WHERE @log_to_table = 0; + + RAISERROR('No waits by duration', 0, 0) WITH NOWAIT; END; ELSE BEGIN @@ -2393,12 +2378,12 @@ AND ca.utc_timestamp < @end_date'; /* Add the WHERE clause only for table logging */ IF @log_to_table = 1 BEGIN - SET @mdsql = + SET @mdsql_execute = REPLACE ( REPLACE ( - @mdsql, + @mdsql_template, '{table_check}', @log_table_waits_by_duration ), @@ -2408,26 +2393,13 @@ AND ca.utc_timestamp < @end_date'; IF @debug = 1 BEGIN - PRINT @mdsql; + PRINT @mdsql_execute; END; EXECUTE sys.sp_executesql - @mdsql, + @mdsql_execute, N'@max_event_time datetime2(7) OUTPUT', @max_event_time OUTPUT; - - SET @mdsql = - REPLACE - ( - REPLACE - ( - @mdsql, - @log_table_waits_by_duration, - '{table_check}' - ), - 'event_time_rounded', - '{date_column}' - ); SET @dsql += N' AND x.event_time_rounded > @max_event_time'; @@ -2437,7 +2409,8 @@ AND ca.utc_timestamp < @end_date'; SET @dsql += N' ORDER BY x.s - OPTION(RECOMPILE);'; + OPTION(RECOMPILE); + '; /* Handle table logging */ IF @log_to_table = 1 @@ -2483,7 +2456,7 @@ AND ca.utc_timestamp < @end_date'; BEGIN IF @debug = 1 BEGIN - RAISERROR('Parsing disk stuff', 0, 1) WITH NOWAIT; + RAISERROR('Parsing disk stuff', 0, 0) WITH NOWAIT; END; SELECT @@ -2572,6 +2545,8 @@ AND ca.utc_timestamp < @end_date'; ELSE 'no io issues found!' END WHERE @log_to_table = 0; + + RAISERROR('No io data found', 0, 0) WITH NOWAIT; END; ELSE BEGIN @@ -2612,12 +2587,12 @@ AND ca.utc_timestamp < @end_date'; IF @log_to_table = 1 BEGIN /* Get max event_time for IO issues */ - SET @mdsql = + SET @mdsql_execute = REPLACE ( REPLACE ( - @mdsql, + @mdsql_template, '{table_check}', @log_table_io_issues ), @@ -2627,27 +2602,13 @@ AND ca.utc_timestamp < @end_date'; IF @debug = 1 BEGIN - PRINT @mdsql; + PRINT @mdsql_execute; END; EXECUTE sys.sp_executesql - @mdsql, + @mdsql_execute, N'@max_event_time datetime2(7) OUTPUT', @max_event_time OUTPUT; - - /* Reset @mdsql to original template */ - SET @mdsql = - REPLACE - ( - REPLACE - ( - @mdsql, - @log_table_io_issues, - '{table_check}' - ), - 'event_time', - '{date_column}' - ); SET @dsql += N' WHERE i.event_time > @max_event_time'; @@ -2657,7 +2618,8 @@ AND ca.utc_timestamp < @end_date'; SET @dsql += N' ORDER BY i.event_time DESC - OPTION(RECOMPILE);'; + OPTION(RECOMPILE); + '; /* Handle table logging */ IF @log_to_table = 1 @@ -2706,7 +2668,7 @@ AND ca.utc_timestamp < @end_date'; BEGIN IF @debug = 1 BEGIN - RAISERROR('Parsing CPU stuff', 0, 1) WITH NOWAIT; + RAISERROR('Parsing CPU stuff', 0, 0) WITH NOWAIT; END; SELECT @@ -2781,41 +2743,43 @@ END; ELSE 'no cpu issues found!' END WHERE @log_to_table = 0; + + RAISERROR('No scheduler data found', 0, 0) WITH NOWAIT; END; ELSE BEGIN /* Build the query */ SET @dsql = N' - SELECT - ' + CASE - WHEN @log_to_table = 1 - THEN N'' - ELSE N'finding = ''cpu task details'',' - END + - N' - sd.event_time, - sd.state, - sd.maxWorkers, - sd.workersCreated, - sd.workersIdle, - sd.tasksCompletedWithinInterval, - sd.pendingTasks, - sd.oldestPendingTaskWaitingTime, - sd.hasUnresolvableDeadlockOccurred, - sd.hasDeadlockedSchedulersOccurred, - sd.didBlockingOccur - FROM #scheduler_details AS sd'; + SELECT + ' + CASE + WHEN @log_to_table = 1 + THEN N'' + ELSE N'finding = ''cpu task details'',' + END + + N' + sd.event_time, + sd.state, + sd.maxWorkers, + sd.workersCreated, + sd.workersIdle, + sd.tasksCompletedWithinInterval, + sd.pendingTasks, + sd.oldestPendingTaskWaitingTime, + sd.hasUnresolvableDeadlockOccurred, + sd.hasDeadlockedSchedulersOccurred, + sd.didBlockingOccur + FROM #scheduler_details AS sd'; /* Add the WHERE clause only for table logging */ IF @log_to_table = 1 BEGIN /* Get max event_time for CPU task details */ - SET @mdsql = + SET @mdsql_execute = REPLACE ( REPLACE ( - @mdsql, + @mdsql_template, '{table_check}', @log_table_cpu_tasks ), @@ -2825,27 +2789,13 @@ END; IF @debug = 1 BEGIN - PRINT @mdsql; + PRINT @mdsql_execute; END; EXECUTE sys.sp_executesql - @mdsql, + @mdsql_execute, N'@max_event_time datetime2(7) OUTPUT', @max_event_time OUTPUT; - - /* Reset @mdsql to original template */ - SET @mdsql = - REPLACE - ( - REPLACE - ( - @mdsql, - @log_table_cpu_tasks, - '{table_check}' - ), - 'event_time', - '{date_column}' - ); SET @dsql += N' WHERE sd.event_time > @max_event_time'; @@ -2855,28 +2805,29 @@ END; SET @dsql += N' ORDER BY sd.event_time DESC - OPTION(RECOMPILE);'; + OPTION(RECOMPILE); + '; /* Handle table logging */ IF @log_to_table = 1 BEGIN SET @insert_sql = N' - INSERT INTO - ' + @log_table_cpu_tasks + N' - ( - event_time, - state, - maxWorkers, - workersCreated, - workersIdle, - tasksCompletedWithinInterval, - pendingTasks, - oldestPendingTaskWaitingTime, - hasUnresolvableDeadlockOccurred, - hasDeadlockedSchedulersOccurred, - didBlockingOccur - )' + - @dsql; + INSERT INTO + ' + @log_table_cpu_tasks + N' + ( + event_time, + state, + maxWorkers, + workersCreated, + workersIdle, + tasksCompletedWithinInterval, + pendingTasks, + oldestPendingTaskWaitingTime, + hasUnresolvableDeadlockOccurred, + hasDeadlockedSchedulersOccurred, + didBlockingOccur + )' + + @dsql; IF @debug = 1 BEGIN @@ -2907,7 +2858,7 @@ END; BEGIN IF @debug = 1 BEGIN - RAISERROR('Parsing memory stuff', 0, 1) WITH NOWAIT; + RAISERROR('Parsing memory stuff', 0, 0) WITH NOWAIT; END; SELECT @@ -2997,6 +2948,8 @@ END; ELSE 'no memory issues found!' END WHERE @log_to_table = 0; + + RAISERROR('No memory condition data found', 0, 0) WITH NOWAIT; END; ELSE BEGIN @@ -3008,8 +2961,7 @@ END; THEN N'' ELSE N'finding = ''memory conditions'',' END + - N' - m.event_time, + N'm.event_time, m.lastNotification, m.outOfMemoryExceptions, m.isAnyPoolOutOfMemory, @@ -3047,12 +2999,12 @@ END; IF @log_to_table = 1 BEGIN /* Get max event_time for memory conditions */ - SET @mdsql = + SET @mdsql_execute = REPLACE ( REPLACE ( - @mdsql, + @mdsql_template, '{table_check}', @log_table_memory_conditions ), @@ -3062,27 +3014,13 @@ END; IF @debug = 1 BEGIN - PRINT @mdsql; + PRINT @mdsql_execute; END; EXECUTE sys.sp_executesql - @mdsql, + @mdsql_execute, N'@max_event_time datetime2(7) OUTPUT', - @max_event_time OUTPUT; - - /* Reset @mdsql to original template */ - SET @mdsql = - REPLACE - ( - REPLACE - ( - @mdsql, - @log_table_memory_conditions, - '{table_check}' - ), - 'event_time', - '{date_column}' - ); + @max_event_time OUTPUT; SET @dsql += N' WHERE m.event_time > @max_event_time'; @@ -3092,7 +3030,8 @@ END; SET @dsql += N' ORDER BY m.event_time DESC - OPTION(RECOMPILE);'; + OPTION(RECOMPILE); + '; /* Handle table logging */ IF @log_to_table = 1 @@ -3167,7 +3106,7 @@ END; BEGIN IF @debug = 1 BEGIN - RAISERROR('Parsing memory broker data', 0, 1) WITH NOWAIT; + RAISERROR('Parsing memory broker data', 0, 0) WITH NOWAIT; END; SELECT @@ -3208,176 +3147,165 @@ END; x.event_time DESC; END; -/* Memory broker notifications logging section */ -IF NOT EXISTS -( - SELECT - 1/0 - FROM #memory_broker_info AS mbi -) -BEGIN - /* No results logic, only return if not logging */ - SELECT - finding = - CASE - WHEN @what_to_check NOT IN ('all', 'memory') - THEN 'memory broker skipped, @what_to_check set to ' + - @what_to_check - WHEN @what_to_check IN ('all', 'memory') - THEN 'no memory pressure events found between ' + - RTRIM(CONVERT(date, @start_date)) + - ' and ' + - RTRIM(CONVERT(date, @end_date)) + - ' with @warnings_only set to ' + - RTRIM(@warnings_only) + - '.' - ELSE 'no memory pressure events found!' - END - WHERE @log_to_table = 0; -END; -ELSE -BEGIN - /* Build the query for memory node OOM events */ - SET @dsql = N' - SELECT - ' + CASE - WHEN @log_to_table = 1 - THEN N'' - ELSE N'finding = ''memory node OOM events'',' - END + - N' - mnoi.event_time, - mnoi.node_id, - memory_available_gb = - REPLACE - ( - CONVERT - ( - nvarchar(30), - CONVERT + /* Memory broker notifications logging section */ + IF NOT EXISTS + ( + SELECT + 1/0 + FROM #memory_broker_info AS mbi + ) + BEGIN + /* No results logic, only return if not logging */ + SELECT + finding = + CASE + WHEN @what_to_check NOT IN ('all', 'memory') + THEN 'memory broker skipped, @what_to_check set to ' + + @what_to_check + WHEN @what_to_check IN ('all', 'memory') + THEN 'no memory pressure events found between ' + + RTRIM(CONVERT(date, @start_date)) + + ' and ' + + RTRIM(CONVERT(date, @end_date)) + + ' with @warnings_only set to ' + + RTRIM(@warnings_only) + + '.' + ELSE 'no memory pressure events found!' + END + WHERE @log_to_table = 0; + + RAISERROR('No memory broker data found', 0, 0) WITH NOWAIT; + END; + ELSE + BEGIN + /* Build the query for memory node OOM events */ + SET @dsql = N' + SELECT + ' + CASE + WHEN @log_to_table = 1 + THEN N'' + ELSE N'finding = ''memory node OOM events'',' + END + + N' + mnoi.event_time, + mnoi.node_id, + memory_available_gb = + REPLACE ( - money, - mnoi.memory_available_kb / 1024.0 / 1024.0 + CONVERT + ( + nvarchar(30), + CONVERT + ( + money, + mnoi.memory_available_kb / 1024.0 / 1024.0 + ), + 1 + ), + N''.00'', + N'''' ), - 1 - ), - N''.00'', - N'''' - ), - memory_requested_gb = - REPLACE - ( - CONVERT - ( - nvarchar(30), - CONVERT + memory_requested_gb = + REPLACE ( - money, - mnoi.memory_requested_kb / 1024.0 / 1024.0 + CONVERT + ( + nvarchar(30), + CONVERT + ( + money, + mnoi.memory_requested_kb / 1024.0 / 1024.0 + ), + 1 + ), + N''.00'', + N'''' ), - 1 - ), - N''.00'', - N'''' - ), - mnoi.memory_allocator, - mnoi.memory_allocation_type, - mnoi.memory_clerk_name, - mnoi.os_error - FROM #memory_node_oom_info AS mnoi'; - - /* Add the WHERE clause only for table logging */ - IF @log_to_table = 1 - BEGIN - /* Get max event_time for memory broker */ - SET @mdsql = - REPLACE - ( - REPLACE - ( - @mdsql, - '{table_check}', - @log_table_memory_broker - ), - '{date_column}', - 'event_time' - ); - - IF @debug = 1 - BEGIN - PRINT @mdsql; - END; - - EXECUTE sys.sp_executesql - @mdsql, - N'@max_event_time datetime2(7) OUTPUT', - @max_event_time OUTPUT; + mnoi.memory_allocator, + mnoi.memory_allocation_type, + mnoi.memory_clerk_name, + mnoi.os_error + FROM #memory_node_oom_info AS mnoi'; - /* Reset @mdsql to original template */ - SET @mdsql = - REPLACE - ( - REPLACE - ( - @mdsql, - @log_table_memory_broker, - '{table_check}' - ), - 'event_time', - '{date_column}' - ); - - SET @dsql += N' - WHERE mbi.event_time > @max_event_time'; - END; - - /* Add the ORDER BY clause */ - SET @dsql += N' - ORDER BY - mbi.event_time DESC - OPTION(RECOMPILE);'; - - /* Handle table logging */ - IF @log_to_table = 1 - BEGIN - SET @insert_sql = N' - INSERT INTO ' - + @log_table_memory_broker + N' - ( - event_time, - node_id, - memory_available_gb, - memory_requested_gb, - memory_allocator, - memory_allocation_type, - memory_clerk_name, - os_error - )' + - @dsql; + /* Add the WHERE clause only for table logging */ + IF @log_to_table = 1 + BEGIN + /* Get max event_time for memory broker */ + SET @mdsql_execute = + REPLACE + ( + REPLACE + ( + @mdsql_template, + '{table_check}', + @log_table_memory_broker + ), + '{date_column}', + 'event_time' + ); - IF @debug = 1 - BEGIN - PRINT @insert_sql; - END; + IF @debug = 1 + BEGIN + PRINT @mdsql_execute; + END; + + EXECUTE sys.sp_executesql + @mdsql_execute, + N'@max_event_time datetime2(7) OUTPUT', + @max_event_time OUTPUT; - EXECUTE sys.sp_executesql - @insert_sql, - N'@max_event_time datetime2(7)', - @max_event_time; - END; - - /* Execute the query for client results */ - IF @log_to_table = 0 - BEGIN - IF @debug = 1 - BEGIN - PRINT @dsql; + SET @dsql += N' + WHERE mbi.event_time > @max_event_time'; + END; + + /* Add the ORDER BY clause */ + SET @dsql += N' + ORDER BY + mbi.event_time DESC + OPTION(RECOMPILE); + '; + + /* Handle table logging */ + IF @log_to_table = 1 + BEGIN + SET @insert_sql = N' + INSERT INTO ' + + @log_table_memory_broker + N' + ( + event_time, + node_id, + memory_available_gb, + memory_requested_gb, + memory_allocator, + memory_allocation_type, + memory_clerk_name, + os_error + )' + + @dsql; + + IF @debug = 1 + BEGIN + PRINT @insert_sql; + END; + + EXECUTE sys.sp_executesql + @insert_sql, + N'@max_event_time datetime2(7)', + @max_event_time; + END; + + /* Execute the query for client results */ + IF @log_to_table = 0 + BEGIN + IF @debug = 1 + BEGIN + PRINT @dsql; + END; + + EXECUTE sys.sp_executesql + @dsql; + END; END; - - EXECUTE sys.sp_executesql - @dsql; - END; -END; END; /*End memory broker analysis*/ /*Parse memory node OOM data*/ @@ -3385,7 +3313,7 @@ END; BEGIN IF @debug = 1 BEGIN - RAISERROR('Parsing memory node OOM data', 0, 1) WITH NOWAIT; + RAISERROR('Parsing memory node OOM data', 0, 0) WITH NOWAIT; END; SELECT @@ -3448,6 +3376,8 @@ END; ELSE 'no memory node OOM events found!' END WHERE @log_to_table = 0; + + RAISERROR('No memory oom data found', 0, 0) WITH NOWAIT; END; ELSE BEGIN @@ -3571,12 +3501,12 @@ END; IF @log_to_table = 1 BEGIN /* Get max event_time for memory node OOM */ - SET @mdsql = + SET @mdsql_execute = REPLACE ( REPLACE ( - @mdsql, + @mdsql_template, '{table_check}', @log_table_memory_node_oom ), @@ -3586,27 +3516,13 @@ END; IF @debug = 1 BEGIN - PRINT @mdsql; + PRINT @mdsql_execute; END; EXECUTE sys.sp_executesql - @mdsql, + @mdsql_execute, N'@max_event_time datetime2(7) OUTPUT', - @max_event_time OUTPUT; - - /* Reset @mdsql to original template */ - SET @mdsql = - REPLACE - ( - REPLACE - ( - @mdsql, - @log_table_memory_node_oom, - '{table_check}' - ), - 'event_time', - '{date_column}' - ); + @max_event_time OUTPUT; SET @dsql += N' WHERE mnoi.event_time > @max_event_time'; @@ -3616,7 +3532,8 @@ END; SET @dsql += N' ORDER BY mnoi.event_time DESC - OPTION(RECOMPILE);'; + OPTION(RECOMPILE); + '; /* Handle table logging */ IF @log_to_table = 1 @@ -3668,7 +3585,7 @@ END; BEGIN IF @debug = 1 BEGIN - RAISERROR('Parsing system stuff', 0, 1) WITH NOWAIT; + RAISERROR('Parsing system stuff', 0, 0) WITH NOWAIT; END; SELECT @@ -3743,6 +3660,8 @@ END; ELSE 'no system health issues found!' END WHERE @log_to_table = 0; + + RAISERROR('No system health data found', 0, 0) WITH NOWAIT; END; ELSE BEGIN @@ -3777,12 +3696,12 @@ END; IF @log_to_table = 1 BEGIN /* Get max event_time for system health */ - SET @mdsql = + SET @mdsql_execute = REPLACE ( REPLACE ( - @mdsql, + @mdsql_template, '{table_check}', @log_table_system_health ), @@ -3792,27 +3711,13 @@ END; IF @debug = 1 BEGIN - PRINT @mdsql; + PRINT @mdsql_execute; END; EXECUTE sys.sp_executesql - @mdsql, + @mdsql_execute, N'@max_event_time datetime2(7) OUTPUT', - @max_event_time OUTPUT; - - /* Reset @mdsql to original template */ - SET @mdsql = - REPLACE - ( - REPLACE - ( - @mdsql, - @log_table_system_health, - '{table_check}' - ), - 'event_time', - '{date_column}' - ); + @max_event_time OUTPUT; SET @dsql = @dsql + N' WHERE h.event_time > @max_event_time'; @@ -3822,7 +3727,8 @@ END; SET @dsql = @dsql + N' ORDER BY h.event_time DESC - OPTION(RECOMPILE);'; + OPTION(RECOMPILE); + '; /* Handle table logging */ IF @log_to_table = 1 @@ -3880,7 +3786,7 @@ END; BEGIN IF @debug = 1 BEGIN - RAISERROR('Parsing scheduler monitor data', 0, 1) WITH NOWAIT; + RAISERROR('Parsing scheduler monitor data', 0, 0) WITH NOWAIT; END; SELECT @@ -3947,6 +3853,8 @@ END; ELSE 'no scheduler issues found!' END WHERE @log_to_table = 0; + + RAISERROR('No scheduler issues data found', 0, 0) WITH NOWAIT; END; ELSE BEGIN @@ -4004,12 +3912,12 @@ END; IF @log_to_table = 1 BEGIN /* Get max event_time for scheduler issues */ - SET @mdsql = + SET @mdsql_execute = REPLACE ( REPLACE ( - @mdsql, + @mdsql_template, '{table_check}', @log_table_scheduler_issues ), @@ -4019,27 +3927,13 @@ END; IF @debug = 1 BEGIN - PRINT @mdsql; + PRINT @mdsql_execute; END; EXECUTE sys.sp_executesql - @mdsql, + @mdsql_execute, N'@max_event_time datetime2(7) OUTPUT', @max_event_time OUTPUT; - - /* Reset @mdsql to original template */ - SET @mdsql = - REPLACE - ( - REPLACE - ( - @mdsql, - @log_table_scheduler_issues, - '{table_check}' - ), - 'event_time', - '{date_column}' - ); SET @dsql = @dsql + N' WHERE si.event_time > @max_event_time'; @@ -4049,7 +3943,8 @@ END; SET @dsql = @dsql + N' ORDER BY si.event_time DESC - OPTION(RECOMPILE);'; + OPTION(RECOMPILE); + '; /* Handle table logging */ IF @log_to_table = 1 @@ -4101,7 +3996,7 @@ END; BEGIN IF @debug = 1 BEGIN - RAISERROR('Parsing error_reported data', 0, 1) WITH NOWAIT; + RAISERROR('Parsing error_reported data', 0, 0) WITH NOWAIT; END; INSERT @@ -4182,6 +4077,8 @@ END; ELSE 'no severe errors found!' END WHERE @log_to_table = 0; + + RAISERROR('No error data found', 0, 0) WITH NOWAIT; END; ELSE BEGIN @@ -4207,12 +4104,12 @@ END; IF @log_to_table = 1 BEGIN /* Get max event_time for severe errors */ - SET @mdsql = + SET @mdsql_execute = REPLACE ( REPLACE ( - @mdsql, + @mdsql_template, '{table_check}', @log_table_severe_errors ), @@ -4222,27 +4119,13 @@ END; IF @debug = 1 BEGIN - PRINT @mdsql; + PRINT @mdsql_execute; END; EXECUTE sys.sp_executesql - @mdsql, + @mdsql_execute, N'@max_event_time datetime2(7) OUTPUT', @max_event_time OUTPUT; - - /* Reset @mdsql to original template */ - SET @mdsql = - REPLACE - ( - REPLACE - ( - @mdsql, - @log_table_severe_errors, - '{table_check}' - ), - 'event_time', - '{date_column}' - ); SET @dsql = @dsql + N' WHERE ei.event_time > @max_event_time'; @@ -4253,7 +4136,8 @@ END; ORDER BY ei.event_time DESC, ei.severity DESC - OPTION(RECOMPILE);'; + OPTION(RECOMPILE); + '; /* Handle table logging */ IF @log_to_table = 1 @@ -4391,7 +4275,7 @@ END; BEGIN IF @debug = 1 BEGIN - RAISERROR('Parsing locking stuff', 0, 1) WITH NOWAIT; + RAISERROR('Parsing locking stuff', 0, 0) WITH NOWAIT; END; INSERT @@ -4436,7 +4320,7 @@ END; /*Blocked queries*/ IF @debug = 1 BEGIN - RAISERROR('Parsing blocked queries', 0, 1) WITH NOWAIT; + RAISERROR('Parsing blocked queries', 0, 0) WITH NOWAIT; END; SELECT @@ -4470,7 +4354,7 @@ END; IF @debug = 1 BEGIN - RAISERROR('Adding query_text to #blocked', 0, 1) WITH NOWAIT; + RAISERROR('Adding query_text to #blocked', 0, 0) WITH NOWAIT; END; ALTER TABLE #blocked @@ -4497,7 +4381,7 @@ END; /*Blocking queries*/ IF @debug = 1 BEGIN - RAISERROR('Parsing blocking queries', 0, 1) WITH NOWAIT; + RAISERROR('Parsing blocking queries', 0, 0) WITH NOWAIT; END; SELECT @@ -4531,7 +4415,7 @@ END; IF @debug = 1 BEGIN - RAISERROR('Adding query_text to #blocking', 0, 1) WITH NOWAIT; + RAISERROR('Adding query_text to #blocking', 0, 0) WITH NOWAIT; END; ALTER TABLE #blocking @@ -4558,7 +4442,7 @@ END; /*Put it together*/ IF @debug = 1 BEGIN - RAISERROR('Inserting to #blocks', 0, 1) WITH NOWAIT; + RAISERROR('Inserting to #blocks', 0, 0) WITH NOWAIT; END; SELECT @@ -4770,7 +4654,7 @@ END; /*Grab available plans from the cache*/ IF @debug = 1 BEGIN - RAISERROR('Inserting to #available_plans (blocking)', 0, 1) WITH NOWAIT; + RAISERROR('Inserting to #available_plans (blocking)', 0, 0) WITH NOWAIT; END; SELECT DISTINCT @@ -4818,7 +4702,7 @@ END; IF @debug = 1 BEGIN - RAISERROR('Inserting to #deadlocks', 0, 1) WITH NOWAIT; + RAISERROR('Inserting to #deadlocks', 0, 0) WITH NOWAIT; END; SELECT @@ -4840,7 +4724,7 @@ END; IF @debug = 1 BEGIN - RAISERROR('Inserting to #deadlocks_parsed', 0, 1) WITH NOWAIT; + RAISERROR('Inserting to #deadlocks_parsed', 0, 0) WITH NOWAIT; END; SELECT @@ -4966,7 +4850,7 @@ END; IF @debug = 1 BEGIN - RAISERROR('Adding query_text to #deadlocks_parsed', 0, 1) WITH NOWAIT; + RAISERROR('Adding query_text to #deadlocks_parsed', 0, 0) WITH NOWAIT; END; ALTER TABLE #deadlocks_parsed @@ -4990,7 +4874,7 @@ END; IF @debug = 1 BEGIN - RAISERROR('Returning deadlocks', 0, 1) WITH NOWAIT; + RAISERROR('Returning deadlocks', 0, 0) WITH NOWAIT; END; IF EXISTS @@ -5109,7 +4993,7 @@ END; IF @debug = 1 BEGIN - RAISERROR('Inserting #available_plans (deadlocks)', 0, 1) WITH NOWAIT; + RAISERROR('Inserting #available_plans (deadlocks)', 0, 0) WITH NOWAIT; END; INSERT @@ -5147,7 +5031,7 @@ END; IF @debug = 1 BEGIN - RAISERROR('Inserting #dm_exec_query_stats_sh', 0, 1) WITH NOWAIT; + RAISERROR('Inserting #dm_exec_query_stats_sh', 0, 0) WITH NOWAIT; END; SELECT @@ -5214,7 +5098,7 @@ END; IF @debug = 1 BEGIN - RAISERROR('Indexing #dm_exec_query_stats_sh', 0, 1) WITH NOWAIT; + RAISERROR('Indexing #dm_exec_query_stats_sh', 0, 0) WITH NOWAIT; END; CREATE CLUSTERED INDEX @@ -5227,7 +5111,7 @@ END; IF @debug = 1 BEGIN - RAISERROR('Inserting #all_available_plans (deadlocks)', 0, 1) WITH NOWAIT; + RAISERROR('Inserting #all_available_plans (deadlocks)', 0, 0) WITH NOWAIT; END; SELECT From 8bb2c6222e87980675925642f929db58bef8c992 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 15 Mar 2025 11:39:05 -0400 Subject: [PATCH 120/246] update readme files update readme files --- .DS_Store | Bin 6148 -> 6148 bytes README.md | 64 ++++++---- sp_HealthParser/README.md | 81 ++++++++++++ sp_HumanEvents/README.md | 90 +++++++++++++ .../sp_HumanEventsBlockViewer_README.md | 102 +++++++++++++++ sp_LogHunter/README.md | 59 +++++++++ sp_PressureDetector/README.md | 70 ++++++++++ sp_QuickieStore/README.md | 120 ++++++++++++++++++ 8 files changed, 564 insertions(+), 22 deletions(-) create mode 100644 sp_HealthParser/README.md create mode 100644 sp_HumanEvents/README.md create mode 100644 sp_HumanEvents/sp_HumanEventsBlockViewer_README.md create mode 100644 sp_LogHunter/README.md create mode 100644 sp_PressureDetector/README.md create mode 100644 sp_QuickieStore/README.md diff --git a/.DS_Store b/.DS_Store index 01762da1ba08e3fe8df02781df8d49caf96b5a20..54f54c82faac2515aa144858856cf6d6233830a6 100644 GIT binary patch delta 169 zcmZoMXfc@JFUrQiz`)4BAi%(o%8P)P?S_&T#%HLp9EC7bFvFdA-808wUN1nfsTTS`Q(SJGK}4m@3F}1_YgLzEVw8y ZCqFM8Sadd9u=) used to set a minimum query duration to collect data for | an integer | 500 (ms) | +| @query_duration_ms | integer | (>=) used to set a minimum query duration to collect data for | an integer | 500 (ms) | | @query_sort_order | nvarchar | when you use the "query" event, lets you choose which metrics to sort results by | "cpu", "reads", "writes", "duration", "memory", "spills", and you can add "avg" to sort by averages, e.g. "avg cpu" | "cpu" | | @skip_plans | bit | when you use the "query" event, lets you skip collecting actual execution plans |1 or 0 | 0 | -| @blocking_duration_ms | int | (>=) used to set a minimum blocking duration to collect data for | an integer | 500 (ms) | +| @blocking_duration_ms | integer | (>=) used to set a minimum blocking duration to collect data for | an integer | 500 (ms) | | @wait_type | nvarchar | (inclusive) filter to only specific wait types | a single wait type, or a CSV list of wait types | "all", which uses a list of "interesting" waits | -| @wait_duration_ms | int | (>=) used to set a minimum time per wait to collect data for | an integer | 10 (ms) | +| @wait_duration_ms | integer | (>=) used to set a minimum time per wait to collect data for | an integer | 10 (ms) | | @client_app_name | sysname | (inclusive) filter to only specific app names | a stringy thing | intentionally left blank | | @client_hostname | sysname | (inclusive) filter to only specific host names | a stringy thing | intentionally left blank | | @database_name | sysname | (inclusive) filter to only specific databases | a stringy thing | intentionally left blank | | @session_id | nvarchar | (inclusive) filter to only a specific session id, or a sample of session ids | an integer, or "sample" to sample a workload | intentionally left blank | -| @sample_divisor | int | the divisor for session ids when sampling a workload, e.g. SPID % 5 | an integer | 5 | +| @sample_divisor | integer | the divisor for session ids when sampling a workload, e.g. SPID % 5 | an integer | 5 | | @username | sysname | (inclusive) filter to only a specific user | a stringy thing | intentionally left blank | | @object_name | sysname | (inclusive) to only filter to a specific object name | a stringy thing | intentionally left blank | | @object_schema | sysname | (inclusive) the schema of the object you want to filter to; only needed with blocking events | a stringy thing | dbo | -| @requested_memory_mb | int | (>=) the memory grant a query must ask for to have data collected | an integer | 0 | -| @seconds_sample | int | the duration in seconds to run the event session for | an integer | 10 | +| @requested_memory_mb | integer | (>=) the memory grant a query must ask for to have data collected | an integer | 0 | +| @seconds_sample | tinyint | the duration in seconds to run the event session for | an integer | 10 | | @gimme_danger | bit | used to override default minimums for query, wait, and blocking durations. | 1 or 0 | 0 | | @keep_alive | bit | creates a permanent session, either to watch live or log to a table from | 1 or 0 | 0 | | @custom_name | nvarchar | if you want to custom name a permanent session | a stringy thing | intentionally left blank | | @output_database_name | sysname | the database you want to log data to | a valid database name | intentionally left blank | | @output_schema_name | sysname | the schema you want to log data to | a valid schema | dbo | -| @delete_retention_days | int | how many days of logged data you want to keep | a POSITIVE integer | 3 (days) | +| @delete_retention_days | integer | how many days of logged data you want to keep | a POSITIVE integer | 3 (days) | | @cleanup | bit | deletes all sessions, tables, and views. requires output database and schema. | 1 or 0 | 0 | | @max_memory_kb | bigint | set a max ring buffer size to log data to | an integer | 102400 | | @version | varchar | to make sure you have the most recent bits | none, output | none, output | @@ -188,18 +193,28 @@ EXEC dbo.sp_HumanEventsBlockViewer Current valid parameter details: -| parameter_name | data_type | description | valid_inputs | defaults | -|----------------|-----------|-------------------------------------------------|------------------------------------------------------------------------|------------------------------------| -| @session_name | nvarchar | name of the extended event session to pull from | extended event session name capturing sqlserver.blocked_process_report | keeper_HumanEvents_blocking | -| @target_type | sysname | target of the extended event session | event_file or ring_buffer | NULL | -| @start_date | datetime2 | filter by date | a reasonable date | NULL; will shortcut to last 7 days | -| @end_date | datetime2 | filter by date | a reasonable date | NULL | -| @database_name | sysname | filter by database name | a database that exists on this server | NULL | -| @object_name | sysname | filter by table name | a schema-prefixed table name | NULL | -| @help | bit | how you got here | 0 or 1 | 0 | -| @debug | bit | dumps raw temp table contents | 0 or 1 | 0 | -| @version | varchar | OUTPUT; for support | none; OUTPUT | none; OUTPUT | -| @version_date | datetime | OUTPUT; for support | none; OUTPUT | none; OUTPUT | +| parameter_name | data_type | description | valid_inputs | defaults | +|-----------------------|-----------|-------------------------------------------------|------------------------------------------------------------------------|------------------------------------| +| @session_name | nvarchar | name of the extended event session to pull from | extended event session name capturing sqlserver.blocked_process_report | keeper_HumanEvents_blocking | +| @target_type | sysname | target of the extended event session | event_file or ring_buffer | NULL | +| @start_date | datetime2 | filter by date | a reasonable date | NULL; will shortcut to last 7 days | +| @end_date | datetime2 | filter by date | a reasonable date | NULL | +| @database_name | sysname | filter by database name | a database that exists on this server | NULL | +| @object_name | sysname | filter by table name | a schema-prefixed table name | NULL | +| @target_database | sysname | database containing the table with BPR data | a valid database name | NULL | +| @target_schema | sysname | schema of the table | a valid schema name | NULL | +| @target_table | sysname | table name | a valid table name | NULL | +| @target_column | sysname | column containing XML data | a valid column name | NULL | +| @timestamp_column | sysname | column containing timestamp (optional) | a valid column name | NULL | +| @log_to_table | bit | enable logging to permanent tables | 0 or 1 | 0 | +| @log_database_name | sysname | database to store logging tables | a valid database name | NULL | +| @log_schema_name | sysname | schema to store logging tables | a valid schema name | NULL | +| @log_table_name_prefix| sysname | prefix for all logging tables | a valid table name prefix | 'HumanEventsBlockViewer' | +| @log_retention_days | integer | Number of days to keep logs, 0 = keep indefinitely | a valid integer | 30 | +| @help | bit | how you got here | 0 or 1 | 0 | +| @debug | bit | dumps raw temp table contents | 0 or 1 | 0 | +| @version | varchar | OUTPUT; for support | none; OUTPUT | none; OUTPUT | +| @version_date | datetime | OUTPUT; for support | none; OUTPUT | none; OUTPUT | [*Back to top*](#navigatory) @@ -326,7 +341,12 @@ Current valid parameter details: | @wait_duration_ms | bigint | minimum wait duration | the minimum duration of a wait for queries with interesting waits | 0 | | @wait_round_interval_minutes | bigint | interval to round minutes to for wait stats | interval to round minutes to for top wait stats by count and duration | 60 | | @skip_locks | bit | skip the blocking and deadlocking section | 0 or 1 | 0 | -| @pending_task_threshold | int | minimum number of pending tasks to display | a valid integer | 10 | +| @pending_task_threshold | integer | minimum number of pending tasks to display | a valid integer | 10 | +| @log_to_table | bit | enable logging to permanent tables | 0 or 1 | 0 | +| @log_database_name | sysname | database to store logging tables | valid database name | NULL | +| @log_schema_name | sysname | schema to store logging tables | valid schema name | NULL | +| @log_table_name_prefix | sysname | prefix for all logging tables | valid table name prefix | 'HealthParser' | +| @log_retention_days | integer | Number of days to keep logs, 0 = keep indefinitely | integer | 30 | | @debug | bit | prints dynamic sql, selects from temp tables | 0 or 1 | 0 | | @help | bit | how you got here | 0 or 1 | 0 | | @version | varchar | OUTPUT; for support | none | none; OUTPUT | @@ -353,13 +373,13 @@ Current valid parameter details: | parameter_name | data_type | description | valid_inputs | defaults | |----------------------|-----------|------------------------------------------------|------------------------------------------------------------------------------|--------------| -| @days_back | int | how many days back you want to search the logs | an integer; will be converted to a negative number automatically | -7 | +| @days_back | integer | how many days back you want to search the logs | an integer; will be converted to a negative number automatically | -7 | | @start_date | datetime | if you want to search a specific time frame | a datetime value | NULL | | @end_date | datetime | if you want to search a specific time frame | a datetime value | NULL | | @custom_message | nvarchar | if you want to search for a custom string | something specific you want to search for. no wildcards or substitions. | NULL | | @custom_message_only | bit | only search for the custom string | NULL, 0, 1 | 0 | | @first_log_only | bit | only search through the first error log | NULL, 0, 1 | 0 | -| @language_id | int | to use something other than English | SELECT DISTINCT m.language_id FROM sys.messages AS m ORDER BY m.language_id; | 1033 | +| @language_id | integer | to use something other than English | SELECT DISTINCT m.language_id FROM sys.messages AS m ORDER BY m.language_id; | 1033 | | @help | bit | how you got here | NULL, 0, 1 | 0 | | @debug | bit | dumps raw temp table contents | NULL, 0, 1 | 0 | | @version | varchar | OUTPUT; for support | OUTPUT; for support | none; OUTPUT | diff --git a/sp_HealthParser/README.md b/sp_HealthParser/README.md new file mode 100644 index 00000000..ab7d1724 --- /dev/null +++ b/sp_HealthParser/README.md @@ -0,0 +1,81 @@ +# sp_HealthParser + +The system health extended event has been around for a while, hiding in the shadows, and collecting all sorts of crazy information about your SQL Server. + +The problem is, hardly anyone ever looks at it, and when they do, they realize how awful the Extended Events GUI is. Or that if they want to dig deeper into anything, they're going to have to parse XML. + +This stored procedure takes all that pain away. + +Note that it focuses on performance data, and does not output errors or security details, or any of the other non-performance related data. + +## Results + +Typical result set will show you: +* Queries with significant waits +* Waits by count +* Waits by duration +* Potential I/O issues +* CPU task details +* Memory conditions +* Overall system health +* A limited version of the blocked process report +* XML deadlock report +* Query plans for queries involved in blocking and deadlocks (when available) + +## Parameters + +| parameter_name | data_type | description | valid_inputs | defaults | +|------------------------------|----------------|---------------------------------------------------------------------|-----------------------------------------------------------------------|-----------------| +| @what_to_check | varchar | areas of system health to check | all, waits, disk, cpu, memory, system, locking | all | +| @start_date | datetimeoffset | earliest date to show data for, will be internally converted to UTC | a reasonable date | seven days back | +| @end_date | datetimeoffset | latest date to show data for, will be internally converted to UTC | a reasonable date | current date | +| @warnings_only | bit | only show rows where a warning was reported | NULL, 0, 1 | 0 | +| @database_name | sysname | database name to show blocking events for | the name of a database | NULL | +| @wait_duration_ms | bigint | minimum wait duration | the minimum duration of a wait for queries with interesting waits | 0 | +| @wait_round_interval_minutes | bigint | interval to round minutes to for wait stats | interval to round minutes to for top wait stats by count and duration | 60 | +| @skip_locks | bit | skip the blocking and deadlocking section | 0 or 1 | 0 | +| @pending_task_threshold | integer | minimum number of pending tasks to display | a valid integer | 10 | +| @log_to_table | bit | enable logging to permanent tables | 0 or 1 | 0 | +| @log_database_name | sysname | database to store logging tables | valid database name | NULL | +| @log_schema_name | sysname | schema to store logging tables | valid schema name | NULL | +| @log_table_name_prefix | sysname | prefix for all logging tables | valid table name prefix | 'HealthParser' | +| @log_retention_days | integer | Number of days to keep logs, 0 = keep indefinitely | integer | 30 | +| @debug | bit | prints dynamic sql, selects from temp tables | 0 or 1 | 0 | +| @help | bit | how you got here | 0 or 1 | 0 | +| @version | varchar | OUTPUT; for support | none | none; OUTPUT | +| @version_date | datetime | OUTPUT; for support | none | none; OUTPUT | + +## Examples + +```sql +-- Basic execution for all health checks +EXEC dbo.sp_HealthParser; + +-- Check only memory-related issues +EXEC dbo.sp_HealthParser + @what_to_check = 'memory'; + +-- Look at health issues for a specific time period +EXEC dbo.sp_HealthParser + @start_date = '2025-01-01 00:00:00', + @end_date = '2025-01-02 00:00:00'; + +-- Show only health events with warnings +EXEC dbo.sp_HealthParser + @warnings_only = 1; + +-- Focus on blocking issues for a specific database +EXEC dbo.sp_HealthParser + @what_to_check = 'locking', + @database_name = 'YourDatabaseName'; + +-- Log results to table instead of returning result sets +EXEC dbo.sp_HealthParser + @log_to_table = 1, + @log_database_name = 'DBA', + @log_schema_name = 'dbo', + @log_table_name_prefix = 'HealthParser'; +``` + +## Resources +* [YouTube introduction](https://youtu.be/1kH-aJcCVxs) \ No newline at end of file diff --git a/sp_HumanEvents/README.md b/sp_HumanEvents/README.md new file mode 100644 index 00000000..870344c4 --- /dev/null +++ b/sp_HumanEvents/README.md @@ -0,0 +1,90 @@ +# sp_HumanEvents + +Extended Events are hard. You don't know which ones to use, when to use them, or how to get useful information out of them. + +This procedure is designed to make them easier for you, by creating event sessions to help you troubleshoot common scenarios: +* Blocking: blocked process report +* Query performance: query execution metrics an actual execution plans +* Compiles: catch query compilations +* Recompiles: catch query recompilations +* Wait Stats: server wait stats, broken down by query and database + +The default behavior is to run a session for a set period of time to capture information, but you can also set sessions up to data to permanent tables. + +## Warning + +Misuse of this procedure can harm performance. Be very careful about introducing observer overhead, especially when gathering query plans. Be even more careful when setting up permanent sessions! + +## Parameters + +| parameter | data_type | description | valid_inputs | defaults | +|------------------------|----------------|----------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------|-------------------------------------------------| +| @event_type | sysname | used to pick which session you want to run | "blocking", "query", "waits", "recompiles", "compiles" and certain variations on those words | "query" | +| @query_duration_ms | integer | (>=) used to set a minimum query duration to collect data for | an integer | 500 (ms) | +| @query_sort_order | nvarchar | when you use the "query" event, lets you choose which metrics to sort results by | "cpu", "reads", "writes", "duration", "memory", "spills", and you can add "avg" to sort by averages, e.g. "avg cpu" | "cpu" | +| @skip_plans | bit | when you use the "query" event, lets you skip collecting actual execution plans |1 or 0 | 0 | +| @blocking_duration_ms | integer | (>=) used to set a minimum blocking duration to collect data for | an integer | 500 (ms) | +| @wait_type | nvarchar | (inclusive) filter to only specific wait types | a single wait type, or a CSV list of wait types | "all", which uses a list of "interesting" waits | +| @wait_duration_ms | integer | (>=) used to set a minimum time per wait to collect data for | an integer | 10 (ms) | +| @client_app_name | sysname | (inclusive) filter to only specific app names | a stringy thing | intentionally left blank | +| @client_hostname | sysname | (inclusive) filter to only specific host names | a stringy thing | intentionally left blank | +| @database_name | sysname | (inclusive) filter to only specific databases | a stringy thing | intentionally left blank | +| @session_id | nvarchar | (inclusive) filter to only a specific session id, or a sample of session ids | an integer, or "sample" to sample a workload | intentionally left blank | +| @sample_divisor | integer | the divisor for session ids when sampling a workload, e.g. SPID % 5 | an integer | 5 | +| @username | sysname | (inclusive) filter to only a specific user | a stringy thing | intentionally left blank | +| @object_name | sysname | (inclusive) to only filter to a specific object name | a stringy thing | intentionally left blank | +| @object_schema | sysname | (inclusive) the schema of the object you want to filter to; only needed with blocking events | a stringy thing | dbo | +| @requested_memory_mb | integer | (>=) the memory grant a query must ask for to have data collected | an integer | 0 | +| @seconds_sample | tinyint | the duration in seconds to run the event session for | an integer | 10 | +| @gimme_danger | bit | used to override default minimums for query, wait, and blocking durations. | 1 or 0 | 0 | +| @keep_alive | bit | creates a permanent session, either to watch live or log to a table from | 1 or 0 | 0 | +| @custom_name | sysname | if you want to custom name a permanent session | a stringy thing | intentionally left blank | +| @output_database_name | sysname | the database you want to log data to | a valid database name | intentionally left blank | +| @output_schema_name | sysname | the schema you want to log data to | a valid schema | dbo | +| @delete_retention_days | integer | how many days of logged data you want to keep | a POSITIVE integer | 3 (days) | +| @cleanup | bit | deletes all sessions, tables, and views. requires output database and schema. | 1 or 0 | 0 | +| @max_memory_kb | bigint | set a max ring buffer size to log data to | an integer | 102400 | +| @version | varchar | to make sure you have the most recent bits | none, output | none, output | +| @version_date | datetime | to make sure you have the most recent bits | none, output | none, output | +| @debug | bit | use to print out dynamic SQL | 1 or 0 | 0 | +| @help | bit | well you're here so you figured this one out | 1 or 0 | 0 | + +## Usage Examples + +For execution examples, see here: [Examples.sql](Examples.sql) + +If you set up sessions to capture long term data, you'll need an agent job set up to poll them. You can find an example of that here: [sp_Human Events Agent Job Example.sql](sp_Human%20Events%20Agent%20Job%20Example.sql) + +Here are some basic usage examples: + +```sql +-- Basic execution to capture queries +EXEC dbo.sp_HumanEvents; + +-- Capture blocking events for at least 1 second +EXEC dbo.sp_HumanEvents + @event_type = 'blocking', + @blocking_duration_ms = 1000; + +-- Capture waits in a specific database +EXEC dbo.sp_HumanEvents + @event_type = 'waits', + @database_name = 'YourDatabase'; + +-- Set up a permanent session for logging +EXEC dbo.sp_HumanEvents + @event_type = 'query', + @keep_alive = 1, + @output_database_name = 'DBA', + @output_schema_name = 'dbo'; + +-- Clean up all sessions and tables +EXEC dbo.sp_HumanEvents + @cleanup = 1, + @output_database_name = 'DBA', + @output_schema_name = 'dbo'; +``` + +## Resources +* [YouTube playlist](https://www.youtube.com/playlist?list=PLt4QZ-7lfQifgpvqsa21WLt-u2tZlyoC_) +* [Blog post](https://www.erikdarlingdata.com/sp_humanevents/) \ No newline at end of file diff --git a/sp_HumanEvents/sp_HumanEventsBlockViewer_README.md b/sp_HumanEvents/sp_HumanEventsBlockViewer_README.md new file mode 100644 index 00000000..cbc7e040 --- /dev/null +++ b/sp_HumanEvents/sp_HumanEventsBlockViewer_README.md @@ -0,0 +1,102 @@ +# sp_HumanEventsBlockViewer + +This was originally a companion script to analyze the blocked process report Extended Event created by sp_HumanEvents, but has since turned into its own monster. + +It will work on any Extended Event that captures the blocked process report. If you need to set that up, run these two pieces of code. + +## Setup + +Enable the blocked process report: +```sql +EXEC sys.sp_configure + N'show advanced options', + 1; +RECONFIGURE; +GO + +EXEC sys.sp_configure + N'blocked process threshold', + 5; --Seconds +RECONFIGURE; +GO +``` + +Set up the Extended Event: +```sql +CREATE EVENT SESSION + blocked_process_report +ON SERVER + ADD EVENT + sqlserver.blocked_process_report + ADD TARGET + package0.event_file + ( + SET filename = N'bpr' + ) +WITH +( + MAX_MEMORY = 4096KB, + EVENT_RETENTION_MODE = ALLOW_SINGLE_EVENT_LOSS, + MAX_DISPATCH_LATENCY = 5 SECONDS, + MAX_EVENT_SIZE = 0KB, + MEMORY_PARTITION_MODE = NONE, + TRACK_CAUSALITY = OFF, + STARTUP_STATE = ON +); + +ALTER EVENT SESSION + blocked_process_report +ON SERVER + STATE = START; +``` + +## Parameters + +| parameter_name | data_type | description | valid_inputs | defaults | +|-----------------------|-----------|-------------------------------------------------|------------------------------------------------------------------------|------------------------------------| +| @session_name | sysname | name of the extended event session to pull from | extended event session name capturing sqlserver.blocked_process_report | keeper_HumanEvents_blocking | +| @target_type | sysname | target of the extended event session | event_file or ring_buffer | NULL | +| @start_date | datetime2 | filter by date | a reasonable date | NULL; will shortcut to last 7 days | +| @end_date | datetime2 | filter by date | a reasonable date | NULL | +| @database_name | sysname | filter by database name | a database that exists on this server | NULL | +| @object_name | sysname | filter by table name | a schema-prefixed table name | NULL | +| @target_database | sysname | database containing the table with BPR data | a valid database name | NULL | +| @target_schema | sysname | schema of the table | a valid schema name | NULL | +| @target_table | sysname | table name | a valid table name | NULL | +| @target_column | sysname | column containing XML data | a valid column name | NULL | +| @timestamp_column | sysname | column containing timestamp (optional) | a valid column name | NULL | +| @log_to_table | bit | enable logging to permanent tables | 0 or 1 | 0 | +| @log_database_name | sysname | database to store logging tables | a valid database name | NULL | +| @log_schema_name | sysname | schema to store logging tables | a valid schema name | NULL | +| @log_table_name_prefix| sysname | prefix for all logging tables | a valid table name prefix | 'HumanEventsBlockViewer' | +| @log_retention_days | integer | Number of days to keep logs, 0 = keep indefinitely | a valid integer | 30 | +| @help | bit | how you got here | 0 or 1 | 0 | +| @debug | bit | dumps raw temp table contents | 0 or 1 | 0 | +| @version | varchar | OUTPUT; for support | none; OUTPUT | none; OUTPUT | +| @version_date | datetime | OUTPUT; for support | none; OUTPUT | none; OUTPUT | + +## Examples + +```sql +-- Basic usage with default session name +EXEC dbo.sp_HumanEventsBlockViewer; + +-- Use with a custom extended event session name +EXEC dbo.sp_HumanEventsBlockViewer + @session_name = N'blocked_process_report'; + +-- Filter by a specific database +EXEC dbo.sp_HumanEventsBlockViewer + @database_name = 'YourDatabase'; + +-- Analyze blocking events for a specific time period +EXEC dbo.sp_HumanEventsBlockViewer + @start_date = '2025-01-01 08:00', + @end_date = '2025-01-01 17:00'; + +-- Log results to permanent tables +EXEC dbo.sp_HumanEventsBlockViewer + @log_to_table = 1, + @log_database_name = 'DBA', + @log_schema_name = 'dbo'; +``` \ No newline at end of file diff --git a/sp_LogHunter/README.md b/sp_LogHunter/README.md new file mode 100644 index 00000000..91ed9c84 --- /dev/null +++ b/sp_LogHunter/README.md @@ -0,0 +1,59 @@ +# sp_LogHunter + +The SQL Server error log can have a lot of good information in it about what's going on, whether it's right or wrong. + +The problem is that it's hard to know *what* to look for, and what else was going on once you filter it. + +It's another notoriously bad Microsoft GUI, just like Query Store and Extended Events. + +I created sp_LogHunter to search through your error logs for the important stuff, with some configurability for you, and return everything ordered by log entry time. + +It helps you give you a fuller, better picture of any bad stuff happening. + +## Parameters + +| parameter_name | data_type | description | valid_inputs | defaults | +|----------------------|-----------|------------------------------------------------|------------------------------------------------------------------------------|--------------| +| @days_back | integer | how many days back you want to search the logs | an integer; will be converted to a negative number automatically | -7 | +| @start_date | datetime | if you want to search a specific time frame | a datetime value | NULL | +| @end_date | datetime | if you want to search a specific time frame | a datetime value | NULL | +| @custom_message | nvarchar | if you want to search for a custom string | something specific you want to search for. no wildcards or substitions. | NULL | +| @custom_message_only | bit | only search for the custom string | NULL, 0, 1 | 0 | +| @first_log_only | bit | only search through the first error log | NULL, 0, 1 | 0 | +| @language_id | integer | to use something other than English | SELECT DISTINCT m.language_id FROM sys.messages AS m ORDER BY m.language_id; | 1033 | +| @help | bit | how you got here | NULL, 0, 1 | 0 | +| @debug | bit | dumps raw temp table contents | NULL, 0, 1 | 0 | +| @version | varchar | OUTPUT; for support | OUTPUT; for support | none; OUTPUT | +| @version_date | datetime | OUTPUT; for support | OUTPUT; for support | none; OUTPUT | + +## Examples + +```sql +-- Basic execution to search the last 7 days of error logs +EXEC dbo.sp_LogHunter; + +-- Search logs for the last 30 days +EXEC dbo.sp_LogHunter + @days_back = -30; + +-- Search a specific time period +EXEC dbo.sp_LogHunter + @start_date = '2025-01-01 00:00:00', + @end_date = '2025-01-02 00:00:00'; + +-- Search for a specific custom message +EXEC dbo.sp_LogHunter + @custom_message = 'login failed'; + +-- Only search for the custom message, ignore other errors +EXEC dbo.sp_LogHunter + @custom_message = 'login failed', + @custom_message_only = 1; + +-- Only search the current error log +EXEC dbo.sp_LogHunter + @first_log_only = 1; +``` + +## Resources +* [YouTube introduction](https://youtu.be/L_yJ6zPjHfs) \ No newline at end of file diff --git a/sp_PressureDetector/README.md b/sp_PressureDetector/README.md new file mode 100644 index 00000000..b7a4259f --- /dev/null +++ b/sp_PressureDetector/README.md @@ -0,0 +1,70 @@ +# sp_PressureDetector + +Is your client/server relationship on the rocks? Are queries timing out, dragging along, or causing CPU fans to spin out of control? + +All you need to do is hit F5 to get information about: +* Wait stats since startup +* Database file size, stall, and activity +* tempdb configuration details +* Memory consumers +* Low memory indicators +* Memory configuration and allocation +* Current query memory grants, along with other execution details +* CPU configuration and retained utilization details +* Thread count and current usage +* Any current THREADPOOL waits (best observed with the DAC) +* Currently executing queries, along with other execution details + +## Parameters + +| parameter_name | data_type | description | valid_inputs | defaults | +|----------------------------|-----------|--------------------------------------------------------------------------------|------------------------------------------------------|--------------| +| @what_to_check | varchar | areas to check for pressure | "all", "cpu", and "memory" | all | +| @skip_queries | bit | if you want to skip looking at running queries | 0 or 1 | 0 | +| @skip_plan_xml | bit | if you want to skip getting plan XML | 0 or 1 | 0 | +| @minimum_disk_latency_ms | smallint | low bound for reporting disk latency | a reasonable number of milliseconds for disk latency | 100 | +| @cpu_utilization_threshold | smallint | low bound for reporting high cpu utlization | a reasonable cpu utlization percentage | 50 | +| @skip_waits | bit | skips waits when you do not need them on every run | 0 or 1 | 0 | +| @skip_perfmon | bit | skips perfmon counters when you do not need them on every run | a valid tinyint: 0-255 | 0 | +| @sample_seconds | tinyint | take a sample of your server's metrics | 0 or 1 | 0 | +| @log_to_table | bit | enable logging to permanent tables | 0 or 1 | 0 | +| @log_database_name | sysname | database to store logging tables | valid database name | NULL | +| @log_schema_name | sysname | schema to store logging tables | valid schema name | NULL | +| @log_table_name_prefix | sysname | prefix for all logging tables | valid table name prefix | 'PressureDetector' | +| @log_retention_days | integer | Number of days to keep logs, 0 = keep indefinitely | integer | 30 | +| @help | bit | how you got here | 0 or 1 | 0 | +| @debug | bit | prints dynamic sql, displays parameter and variable values, and table contents | 0 or 1 | 0 | +| @version | varchar | OUTPUT; for support | none | none; OUTPUT | +| @version_date | datetime | OUTPUT; for support | none | none; OUTPUT | + +## Examples + +```sql +-- Basic execution to check all pressure types +EXEC dbo.sp_PressureDetector; + +-- Check only CPU pressure +EXEC dbo.sp_PressureDetector + @what_to_check = 'cpu'; + +-- Check only memory pressure +EXEC dbo.sp_PressureDetector + @what_to_check = 'memory'; + +-- Skip looking at executing queries +EXEC dbo.sp_PressureDetector + @skip_queries = 1; + +-- Take a 10-second sample of server metrics +EXEC dbo.sp_PressureDetector + @sample_seconds = 10; + +-- Log results to a table +EXEC dbo.sp_PressureDetector + @log_to_table = 1, + @log_database_name = 'DBA', + @log_schema_name = 'dbo'; +``` + +## Resources +* [Video walkthrough](https://www.erikdarlingdata.com/sp_pressuredetector/) \ No newline at end of file diff --git a/sp_QuickieStore/README.md b/sp_QuickieStore/README.md new file mode 100644 index 00000000..f6f1cefd --- /dev/null +++ b/sp_QuickieStore/README.md @@ -0,0 +1,120 @@ +# sp_QuickieStore + +This procedure will dig into Query Store data for a specific database, or all databases with Query Store enabled. + +It's designed to run as quickly as possible, but there are some circumstances that prevent me from realizing my ultimate dream. + +The big upside of using this stored procedure over the GUI is that you can search for specific items in Query Store, by: +* query_id +* plan_id +* query hash +* sql handle +* module name +* query text +* query type (ad hoc or from a module) + +You can also choose to filter out specific queries by those, too. + +And you can do all that without worrying about incorrect data from the GUI, which doesn't handle UTC conversion correctly when filtering data. + +By default, it will return the top 10 queries by average CPU. You can configure all sorts of things to look at queries by other metrics, or just specific queries. + +Use the `@expert_mode` parameter to return additional details. + +## Parameters + +| parameter_name | data_type | description | valid_inputs | defaults | +|-----------------------------------------|----------------|---------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------| +| @database_name | sysname | the name of the database you want to look at query store in | a database name with query store enabled | NULL; current database name if NULL | +| @sort_order | varchar | the runtime metric you want to prioritize results by | cpu, logical reads, physical reads, writes, duration, memory, tempdb, executions, recent, plan count by hashes, cpu waits, lock waits, locks waits, latch waits, latches waits, buffer latch waits, buffer latches waits, buffer io waits, log waits, log io waits, network waits, network io waits, parallel waits, parallelism waits, memory waits, total waits, rows | cpu | +| @top | bigint | the number of queries you want to pull back | a positive integer between 1 and 9,223,372,036,854,775,807 | 10 | +| @start_date | datetimeoffset | the begin date of your search, will be converted to UTC internally | January 1, 1753, through December 31, 9999 | the last seven days | +| @end_date | datetimeoffset | the end date of your search, will be converted to UTC internally | January 1, 1753, through December 31, 9999 | NULL | +| @timezone | sysname | user specified time zone to override dates displayed in results | SELECT tzi.* FROM sys.time_zone_info AS tzi; | NULL | +| @execution_count | bigint | the minimum number of executions a query must have | a positive integer between 1 and 9,223,372,036,854,775,807 | NULL | +| @duration_ms | bigint | the minimum duration a query must have to show up in results | a positive integer between 1 and 9,223,372,036,854,775,807 | NULL | +| @execution_type_desc | nvarchar | the type of execution you want to filter by (regular, aborted, exception) | regular, aborted, exception | NULL | +| @procedure_schema | sysname | the schema of the procedure you're searching for | a valid schema in your database | NULL; dbo if NULL and procedure name is not NULL | +| @procedure_name | sysname | the name of the programmable object you're searching for | a valid programmable object in your database, can use wildcards | NULL | +| @include_plan_ids | nvarchar | a list of plan ids to search for | a string; comma separated for multiple ids | NULL | +| @include_query_ids | nvarchar | a list of query ids to search for | a string; comma separated for multiple ids | NULL | +| @include_query_hashes | nvarchar | a list of query hashes to search for | a string; comma separated for multiple hashes | NULL | +| @include_plan_hashes | nvarchar | a list of query plan hashes to search for | a string; comma separated for multiple hashes | NULL | +| @include_sql_handles | nvarchar | a list of sql handles to search for | a string; comma separated for multiple handles | NULL | +| @ignore_plan_ids | nvarchar | a list of plan ids to ignore | a string; comma separated for multiple ids | NULL | +| @ignore_query_ids | nvarchar | a list of query ids to ignore | a string; comma separated for multiple ids | NULL | +| @ignore_query_hashes | nvarchar | a list of query hashes to ignore | a string; comma separated for multiple hashes | NULL | +| @ignore_plan_hashes | nvarchar | a list of query plan hashes to ignore | a string; comma separated for multiple hashes | NULL | +| @ignore_sql_handles | nvarchar | a list of sql handles to ignore | a string; comma separated for multiple handles | NULL | +| @query_text_search | nvarchar | query text to search for | a string; leading and trailing wildcards will be added if missing | NULL | +| @query_text_search_not | nvarchar | query text to exclude | a string; leading and trailing wildcards will be added if missing | NULL | +| @escape_brackets | bit | Set this bit to 1 to search for query text containing square brackets (common in .NET Entity Framework and other ORM queries) | 0 or 1 | 0 | +| @escape_character | nchar | Sets the ESCAPE character for special character searches, defaults to the SQL standard backslash (\) character | some escape character, SQL standard is backslash (\) | \ | +| @only_queries_with_hints | bit | only return queries with query hints | 0 or 1 | 0 | +| @only_queries_with_feedback | bit | only return queries with query feedback | 0 or 1 | 0 | +| @only_queries_with_variants | bit | only return queries with query variants | 0 or 1 | 0 | +| @only_queries_with_forced_plans | bit | only return queries with forced plans | 0 or 1 | 0 | +| @only_queries_with_forced_plan_failures | bit | only return queries with forced plan failures | 0 or 1 | 0 | +| @wait_filter | varchar | wait category to search for; category details are below | cpu, lock, latch, buffer latch, buffer io, log io, network io, parallelism, memory | NULL | +| @query_type | varchar | filter for only ad hoc queries or only from queries from modules | ad hoc, adhoc, proc, procedure, whatever. | NULL | +| @expert_mode | bit | returns additional columns and results | 0 or 1 | 0 | +| @hide_help_table | bit | hides the "bottom table" that shows help and support information | 0 or 1 | 0 | +| @format_output | bit | returns numbers formatted with commas | 0 or 1 | 1 | +| @get_all_databases | bit | looks for query store enabled user databases and returns combined results from all of them | 0 or 1 | 0 | +| @workdays | bit | use this to filter out weekends and after-hours queries | 0 or 1 | 0 | +| @work_start | time | use this to set a specific start of your work days | a time like 8am, 9am or something | 9am | +| @work_end | time | use this to set a specific end of your work days | a time like 5pm, 6pm or something | 5pm | +| @regression_baseline_start_date | datetimeoffset | the begin date of the baseline that you are checking for regressions against (if any), will be converted to UTC internally | January 1, 1753, through December 31, 9999 | NULL | +| @regression_baseline_end_date | datetimeoffset | the end date of the baseline that you are checking for regressions against (if any), will be converted to UTC internally | January 1, 1753, through December 31, 9999 | NULL; One week after @regression_baseline_start_date if that is specified | +| @regression_comparator | varchar | what difference to use ('relative' or 'absolute') when comparing @sort_order's metric for the normal time period with any regression time period. | relative, absolute | NULL; absolute if @regression_baseline_start_date is specified | +| @regression_direction | varchar | when comparing against any regression baseline, what do you want the results sorted by ('magnitude', 'improved', or 'regressed')? | regressed, worse, improved, better, magnitude, absolute, whatever | NULL; regressed if @regression_baseline_start_date is specified | +| @include_query_hash_totals | bit | will add an additional column to final output with total resource usage by query hash | 0 or 1 | 0 | +| @help | bit | how you got here | 0 or 1 | 0 | +| @debug | bit | prints dynamic sql, statement length, parameter and variable values, and raw temp table contents | 0 or 1 | 0 | +| @troubleshoot_performance | bit | set statistics xml on for queries against views | 0 or 1 | 0 | +| @version | varchar | OUTPUT; for support | none; OUTPUT | none; OUTPUT | +| @version_date | datetime | OUTPUT; for support | none; OUTPUT | none; OUTPUT | + +## Examples + +For more examples, see [Examples.sql](Examples.sql) + +```sql +-- Basic execution - returns top 10 queries by CPU +EXEC dbo.sp_QuickieStore; + +-- Look at top 20 queries by logical reads +EXEC dbo.sp_QuickieStore + @sort_order = 'logical reads', + @top = 20; + +-- Search for a specific query text +EXEC dbo.sp_QuickieStore + @query_text_search = 'SELECT * FROM Orders'; + +-- Find queries from a specific procedure +EXEC dbo.sp_QuickieStore + @procedure_name = 'usp_GetCustomerOrders'; + +-- Filter to queries that executed at least 1000 times +EXEC dbo.sp_QuickieStore + @execution_count = 1000; + +-- Show queries with a minimum duration of 500ms +EXEC dbo.sp_QuickieStore + @duration_ms = 500; + +-- Look for regressions against a baseline period +EXEC dbo.sp_QuickieStore + @regression_baseline_start_date = '2025-01-01', + @regression_baseline_end_date = '2025-01-08', + @regression_direction = 'regressed'; + +-- Expert mode for additional details +EXEC dbo.sp_QuickieStore + @expert_mode = 1; +``` + +## Resources +* [YouTube playlist](https://www.youtube.com/playlist?list=PLt4QZ-7lfQie1XZHEm0HN-Zt1S7LFEx1P) +* [Blog post](https://www.erikdarlingdata.com/sp_quickiestore/) \ No newline at end of file From 9171621bbdcd4b09080a7115095ca0d5d1efa9b5 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 15 Mar 2025 11:44:43 -0400 Subject: [PATCH 121/246] README.md README.md --- sp_HumanEvents/README.md | 134 +++++++++++++++++- .../sp_HumanEventsBlockViewer_README.md | 102 ------------- 2 files changed, 133 insertions(+), 103 deletions(-) delete mode 100644 sp_HumanEvents/sp_HumanEventsBlockViewer_README.md diff --git a/sp_HumanEvents/README.md b/sp_HumanEvents/README.md index 870344c4..3d80b528 100644 --- a/sp_HumanEvents/README.md +++ b/sp_HumanEvents/README.md @@ -1,5 +1,30 @@ +# Human Events Toolkit + +This directory contains two stored procedures for managing and analyzing Extended Events in SQL Server: + +- **[sp_HumanEvents](#sp_humanevents)**: Makes extended events easy to use for common scenarios +- **[sp_HumanEventsBlockViewer](#sp_humaneventsblockviewer)**: Analyzes blocked process reports + +## Table of Contents + +- [sp_HumanEvents](#sp_humanevents) + - [Overview](#overview) + - [Warning](#warning) + - [Parameters](#parameters) + - [Usage Examples](#usage-examples) + - [Resources](#resources) +- [sp_HumanEventsBlockViewer](#sp_humaneventsblockviewer) + - [Overview](#overview-1) + - [Setup](#setup) + - [Parameters](#parameters-1) + - [Usage Examples](#usage-examples-1) + +--- + # sp_HumanEvents +## Overview + Extended Events are hard. You don't know which ones to use, when to use them, or how to get useful information out of them. This procedure is designed to make them easier for you, by creating event sessions to help you troubleshoot common scenarios: @@ -87,4 +112,111 @@ EXEC dbo.sp_HumanEvents ## Resources * [YouTube playlist](https://www.youtube.com/playlist?list=PLt4QZ-7lfQifgpvqsa21WLt-u2tZlyoC_) -* [Blog post](https://www.erikdarlingdata.com/sp_humanevents/) \ No newline at end of file +* [Blog post](https://www.erikdarlingdata.com/sp_humanevents/) + +--- + +# sp_HumanEventsBlockViewer + +## Overview + +This was originally a companion script to analyze the blocked process report Extended Event created by sp_HumanEvents, but has since turned into its own monster. + +It will work on any Extended Event that captures the blocked process report. If you need to set that up, run these two pieces of code. + +## Setup + +Enable the blocked process report: +```sql +EXEC sys.sp_configure + N'show advanced options', + 1; +RECONFIGURE; +GO + +EXEC sys.sp_configure + N'blocked process threshold', + 5; --Seconds +RECONFIGURE; +GO +``` + +Set up the Extended Event: +```sql +CREATE EVENT SESSION + blocked_process_report +ON SERVER + ADD EVENT + sqlserver.blocked_process_report + ADD TARGET + package0.event_file + ( + SET filename = N'bpr' + ) +WITH +( + MAX_MEMORY = 4096KB, + EVENT_RETENTION_MODE = ALLOW_SINGLE_EVENT_LOSS, + MAX_DISPATCH_LATENCY = 5 SECONDS, + MAX_EVENT_SIZE = 0KB, + MEMORY_PARTITION_MODE = NONE, + TRACK_CAUSALITY = OFF, + STARTUP_STATE = ON +); + +ALTER EVENT SESSION + blocked_process_report +ON SERVER + STATE = START; +``` + +## Parameters + +| parameter_name | data_type | description | valid_inputs | defaults | +|-----------------------|-----------|-------------------------------------------------|------------------------------------------------------------------------|------------------------------------| +| @session_name | sysname | name of the extended event session to pull from | extended event session name capturing sqlserver.blocked_process_report | keeper_HumanEvents_blocking | +| @target_type | sysname | target of the extended event session | event_file or ring_buffer | NULL | +| @start_date | datetime2 | filter by date | a reasonable date | NULL; will shortcut to last 7 days | +| @end_date | datetime2 | filter by date | a reasonable date | NULL | +| @database_name | sysname | filter by database name | a database that exists on this server | NULL | +| @object_name | sysname | filter by table name | a schema-prefixed table name | NULL | +| @target_database | sysname | database containing the table with BPR data | a valid database name | NULL | +| @target_schema | sysname | schema of the table | a valid schema name | NULL | +| @target_table | sysname | table name | a valid table name | NULL | +| @target_column | sysname | column containing XML data | a valid column name | NULL | +| @timestamp_column | sysname | column containing timestamp (optional) | a valid column name | NULL | +| @log_to_table | bit | enable logging to permanent tables | 0 or 1 | 0 | +| @log_database_name | sysname | database to store logging tables | a valid database name | NULL | +| @log_schema_name | sysname | schema to store logging tables | a valid schema name | NULL | +| @log_table_name_prefix| sysname | prefix for all logging tables | a valid table name prefix | 'HumanEventsBlockViewer' | +| @log_retention_days | integer | Number of days to keep logs, 0 = keep indefinitely | a valid integer | 30 | +| @help | bit | how you got here | 0 or 1 | 0 | +| @debug | bit | dumps raw temp table contents | 0 or 1 | 0 | +| @version | varchar | OUTPUT; for support | none; OUTPUT | none; OUTPUT | +| @version_date | datetime | OUTPUT; for support | none; OUTPUT | none; OUTPUT | + +## Usage Examples + +```sql +-- Basic usage with default session name +EXEC dbo.sp_HumanEventsBlockViewer; + +-- Use with a custom extended event session name +EXEC dbo.sp_HumanEventsBlockViewer + @session_name = N'blocked_process_report'; + +-- Filter by a specific database +EXEC dbo.sp_HumanEventsBlockViewer + @database_name = 'YourDatabase'; + +-- Analyze blocking events for a specific time period +EXEC dbo.sp_HumanEventsBlockViewer + @start_date = '2025-01-01 08:00', + @end_date = '2025-01-01 17:00'; + +-- Log results to permanent tables +EXEC dbo.sp_HumanEventsBlockViewer + @log_to_table = 1, + @log_database_name = 'DBA', + @log_schema_name = 'dbo'; +``` \ No newline at end of file diff --git a/sp_HumanEvents/sp_HumanEventsBlockViewer_README.md b/sp_HumanEvents/sp_HumanEventsBlockViewer_README.md deleted file mode 100644 index cbc7e040..00000000 --- a/sp_HumanEvents/sp_HumanEventsBlockViewer_README.md +++ /dev/null @@ -1,102 +0,0 @@ -# sp_HumanEventsBlockViewer - -This was originally a companion script to analyze the blocked process report Extended Event created by sp_HumanEvents, but has since turned into its own monster. - -It will work on any Extended Event that captures the blocked process report. If you need to set that up, run these two pieces of code. - -## Setup - -Enable the blocked process report: -```sql -EXEC sys.sp_configure - N'show advanced options', - 1; -RECONFIGURE; -GO - -EXEC sys.sp_configure - N'blocked process threshold', - 5; --Seconds -RECONFIGURE; -GO -``` - -Set up the Extended Event: -```sql -CREATE EVENT SESSION - blocked_process_report -ON SERVER - ADD EVENT - sqlserver.blocked_process_report - ADD TARGET - package0.event_file - ( - SET filename = N'bpr' - ) -WITH -( - MAX_MEMORY = 4096KB, - EVENT_RETENTION_MODE = ALLOW_SINGLE_EVENT_LOSS, - MAX_DISPATCH_LATENCY = 5 SECONDS, - MAX_EVENT_SIZE = 0KB, - MEMORY_PARTITION_MODE = NONE, - TRACK_CAUSALITY = OFF, - STARTUP_STATE = ON -); - -ALTER EVENT SESSION - blocked_process_report -ON SERVER - STATE = START; -``` - -## Parameters - -| parameter_name | data_type | description | valid_inputs | defaults | -|-----------------------|-----------|-------------------------------------------------|------------------------------------------------------------------------|------------------------------------| -| @session_name | sysname | name of the extended event session to pull from | extended event session name capturing sqlserver.blocked_process_report | keeper_HumanEvents_blocking | -| @target_type | sysname | target of the extended event session | event_file or ring_buffer | NULL | -| @start_date | datetime2 | filter by date | a reasonable date | NULL; will shortcut to last 7 days | -| @end_date | datetime2 | filter by date | a reasonable date | NULL | -| @database_name | sysname | filter by database name | a database that exists on this server | NULL | -| @object_name | sysname | filter by table name | a schema-prefixed table name | NULL | -| @target_database | sysname | database containing the table with BPR data | a valid database name | NULL | -| @target_schema | sysname | schema of the table | a valid schema name | NULL | -| @target_table | sysname | table name | a valid table name | NULL | -| @target_column | sysname | column containing XML data | a valid column name | NULL | -| @timestamp_column | sysname | column containing timestamp (optional) | a valid column name | NULL | -| @log_to_table | bit | enable logging to permanent tables | 0 or 1 | 0 | -| @log_database_name | sysname | database to store logging tables | a valid database name | NULL | -| @log_schema_name | sysname | schema to store logging tables | a valid schema name | NULL | -| @log_table_name_prefix| sysname | prefix for all logging tables | a valid table name prefix | 'HumanEventsBlockViewer' | -| @log_retention_days | integer | Number of days to keep logs, 0 = keep indefinitely | a valid integer | 30 | -| @help | bit | how you got here | 0 or 1 | 0 | -| @debug | bit | dumps raw temp table contents | 0 or 1 | 0 | -| @version | varchar | OUTPUT; for support | none; OUTPUT | none; OUTPUT | -| @version_date | datetime | OUTPUT; for support | none; OUTPUT | none; OUTPUT | - -## Examples - -```sql --- Basic usage with default session name -EXEC dbo.sp_HumanEventsBlockViewer; - --- Use with a custom extended event session name -EXEC dbo.sp_HumanEventsBlockViewer - @session_name = N'blocked_process_report'; - --- Filter by a specific database -EXEC dbo.sp_HumanEventsBlockViewer - @database_name = 'YourDatabase'; - --- Analyze blocking events for a specific time period -EXEC dbo.sp_HumanEventsBlockViewer - @start_date = '2025-01-01 08:00', - @end_date = '2025-01-01 17:00'; - --- Log results to permanent tables -EXEC dbo.sp_HumanEventsBlockViewer - @log_to_table = 1, - @log_database_name = 'DBA', - @log_schema_name = 'dbo'; -``` \ No newline at end of file From 54a619951c5e7b16d007fb043bdc872bb11bf503 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 15 Mar 2025 11:48:30 -0400 Subject: [PATCH 122/246] Update README.md --- sp_IndexCleanup/README.md | 71 ++++++++++++++++++++++++++++++++++----- 1 file changed, 63 insertions(+), 8 deletions(-) diff --git a/sp_IndexCleanup/README.md b/sp_IndexCleanup/README.md index ceff4d92..c7a64c1c 100644 --- a/sp_IndexCleanup/README.md +++ b/sp_IndexCleanup/README.md @@ -1,11 +1,66 @@ -# This is the BETA VERSION of sp_IndexCleanup +# sp_IndexCleanup -It needs lots of love and testing in real environments with real indexes to fix many issues: - * Data collection - * Deduping logic - * Result correctness - * Edge cases +## Overview - If you run this, only use the output to debug validate result correctness. +This stored procedure helps identify unused and duplicate indexes in your SQL Server databases that could be candidates for removal. It analyzes index usage statistics and can generate scripts for removing unnecessary indexes. - Do not run any of the output scripts, period. Doing so may be harmful. \ No newline at end of file +**IMPORTANT: This is currently a BETA VERSION.** It needs extensive testing in real environments with real indexes to address several issues: +* Data collection accuracy +* Deduping logic +* Result correctness +* Edge cases + +## Warning + +Misuse of this procedure can potentially harm your database. If you run this, only use the output to validate result correctness. **Do not run any of the output scripts without thorough review and testing**, as doing so may be harmful to your database performance. + +The procedure requires SQL Server 2012 (11.0) or later due to the use of FORMAT and CONCAT functions. + +## Parameters + +| Parameter Name | Data Type | Default Value | Description | +|----------------|-----------|---------------|-------------| +| @database_name | sysname | NULL | The name of the database you wish to analyze | +| @schema_name | sysname | NULL | The schema name to filter indexes by | +| @table_name | sysname | NULL | The table name to filter indexes by | +| @min_reads | bigint | 0 | Minimum number of reads for an index to be considered used | +| @min_writes | bigint | 0 | Minimum number of writes for an index to be considered used | +| @min_size_gb | decimal(10,2) | 0 | Minimum size in GB for an index to be analyzed | +| @min_rows | bigint | 0 | Minimum number of rows for a table to be analyzed | +| @help | bit | 0 | Displays help information | +| @debug | bit | 0 | Prints debug information during execution | +| @version | varchar(20) | NULL | OUTPUT parameter that returns the version number of the procedure | +| @version_date | datetime | NULL | OUTPUT parameter that returns the date this version was released | + +## Usage Examples + +```sql +-- Basic usage to analyze all indexes in a database +EXECUTE dbo.sp_IndexCleanup + @database_name = 'YourDatabase'; + +-- Analyze a specific table with debug information +EXECUTE dbo.sp_IndexCleanup + @database_name = 'YourDatabase', + @table_name = 'YourTable', + @debug = 1; + +-- Filter indexes by minimum usage thresholds +EXECUTE dbo.sp_IndexCleanup + @database_name = 'YourDatabase', + @min_reads = 100, + @min_writes = 10; + +-- Show help information +EXECUTE dbo.sp_IndexCleanup + @help = 1; +``` + +## Notes + +- The procedure issues a warning when server uptime is less than 14 days, as index usage stats may not be representative +- Certain features like online index operations and compression are only available in specific SQL Server editions (Enterprise, Azure SQL DB, Managed Instance) +- It is recommended to have a recent backup before making any index changes + +Copyright 2024 Darling Data, LLC +Released under MIT license \ No newline at end of file From cce6aa88ea614e153f41ab8071b507cd4f09ec07 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 15 Mar 2025 11:52:18 -0400 Subject: [PATCH 123/246] out with the old i've removed all of the old examples.sql files in favor of the more robust readme files. --- README.md | 2 +- sp_HealthParser/README.md | 12 +- sp_HumanEvents/Examples.sql | 165 ------------- sp_HumanEvents/README.md | 26 +- sp_LogHunter/README.md | 12 +- sp_PressureDetector/README.md | 12 +- sp_QuickieStore/Examples.sql | 437 ---------------------------------- sp_QuickieStore/README.md | 18 +- 8 files changed, 39 insertions(+), 645 deletions(-) delete mode 100644 sp_HumanEvents/Examples.sql delete mode 100644 sp_QuickieStore/Examples.sql diff --git a/README.md b/README.md index 408e18d7..b54abdd8 100644 --- a/README.md +++ b/README.md @@ -186,7 +186,7 @@ ON SERVER Once it has data collected, you can analyze it using this command: ``` -EXEC dbo.sp_HumanEventsBlockViewer +EXECUTE dbo.sp_HumanEventsBlockViewer @session_name = N'blocked_process_report'; ``` diff --git a/sp_HealthParser/README.md b/sp_HealthParser/README.md index ab7d1724..173eafc0 100644 --- a/sp_HealthParser/README.md +++ b/sp_HealthParser/README.md @@ -49,28 +49,28 @@ Typical result set will show you: ```sql -- Basic execution for all health checks -EXEC dbo.sp_HealthParser; +EXECUTE dbo.sp_HealthParser; -- Check only memory-related issues -EXEC dbo.sp_HealthParser +EXECUTE dbo.sp_HealthParser @what_to_check = 'memory'; -- Look at health issues for a specific time period -EXEC dbo.sp_HealthParser +EXECUTE dbo.sp_HealthParser @start_date = '2025-01-01 00:00:00', @end_date = '2025-01-02 00:00:00'; -- Show only health events with warnings -EXEC dbo.sp_HealthParser +EXECUTE dbo.sp_HealthParser @warnings_only = 1; -- Focus on blocking issues for a specific database -EXEC dbo.sp_HealthParser +EXECUTE dbo.sp_HealthParser @what_to_check = 'locking', @database_name = 'YourDatabaseName'; -- Log results to table instead of returning result sets -EXEC dbo.sp_HealthParser +EXECUTE dbo.sp_HealthParser @log_to_table = 1, @log_database_name = 'DBA', @log_schema_name = 'dbo', diff --git a/sp_HumanEvents/Examples.sql b/sp_HumanEvents/Examples.sql deleted file mode 100644 index 8e184637..00000000 --- a/sp_HumanEvents/Examples.sql +++ /dev/null @@ -1,165 +0,0 @@ -/* -Copyright 2025 Darling Data, LLC -https://www.erikdarling.com/ - -For support, head over to GitHub: -https://github.com/erikdarlingdata/DarlingData - -MIT License - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), -to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, -sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the -following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE -FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -*/ - --- Here are some example calls to get you started. - --- To capture all types of “completed” queries that have run for at least one second, for 20 seconds, from a specific database - -EXECUTE dbo.sp_HumanEvents - @event_type = 'query', - @query_duration_ms = 1000, - @seconds_sample = 20, - @database_name = 'YourMom'; - --- Maybe you want to filter out queries that have asked for a bit of memory: - -EXECUTE dbo.sp_HumanEvents - @event_type = 'query', - @query_duration_ms = 1000, - @seconds_sample = 20, - @requested_memory_mb = 1024; - --- Or maybe you want to find unparameterized queries from a poorly written app that constructs strings in ugly ways, but it generates a lot of queries so you only want data on about a third of them. - -EXECUTE dbo.sp_HumanEvents - @event_type = 'compilations', - @client_app_name = N'GL00SNIFЯ', - @session_id = 'sample', - @sample_divisor = 3; - --- Perhaps you think queries recompiling are the cause of your problems! Heck, they might be. Have you tried removing recompile hints? 😁 - -EXECUTE dbo.sp_HumanEvents - @event_type = 'recompilations', - @seconds_sample = 30; - --- Look, blocking is annoying. Just turn on RCSI, you goblin. Unless you’re not allowed to. - -EXECUTE dbo.sp_HumanEvents - @event_type = 'blocking', - @seconds_sample = 60, - @blocking_duration_ms = 5000; - --- If you want to track wait stats, this’ll work pretty well. Keep in mind “all” is a focused list of “interesting” waits to queries, not every wait stat. - -EXECUTE dbo.sp_HumanEvents - @event_type = 'waits', - @wait_duration_ms = 10, - @seconds_sample = 100, - @wait_type = N'all'; - --- Note that THREADPOOL is SOS_WORKER in xe-land. why? I dunno. - -EXECUTE dbo.sp_HumanEvents - @event_type = 'waits', - @wait_duration_ms = 100, - @seconds_sample = 10, - @wait_type = N'SOS_WORKER,RESOURCE_SEMAPHORE'; - --- For some event types that allow you to set a minimum duration, I’ve set a default minimum to try to avoid you introducing a lot of observer overhead to the server. If you understand the potential danger here, or you’re just trying to test things, you need to use the @gimme_danger parameter. You would also use this if you wanted to set an impermanent session to run for longer than 10 minutes. - --- For example, if you run this command: - -EXECUTE sp_HumanEvents - @event_type = N'query', - @query_duration_ms = 1; - --- You’ll see this message in the output: - --- Checking query duration filter --- You chose a really dangerous value for @query_duration --- If you really want that, please set @gimme_danger = 1, and re-run --- Setting @query_duration to 500 - - --- You need to use this command instead: - -EXECUTE sp_HumanEvents - @event_type = N'query', - @query_duration_ms = 1, - @gimme_danger = 1; - --- Logging Data To Tables - --- First, you need to set up permanent sessions to collect data. You can use commands like these to do that, but I urge you to add some filters like above to cut down on the data collected. On busy servers, over-collection can cause performance issues. - - -EXECUTE sp_HumanEvents - @event_type = N'compiles', - @keep_alive = 1; - -EXECUTE sp_HumanEvents - @event_type = N'recompiles', - @keep_alive = 1; - - -EXECUTE sp_HumanEvents - @event_type = N'query', - @keep_alive = 1; - -EXECUTE sp_HumanEvents - @event_type = N'waits', - @keep_alive = 1; - -EXECUTE sp_HumanEvents - @event_type = N'blocking', - @keep_alive = 1; - - --- Once your sessions are set up, this is the command to tell sp_HumanEvents which database and schema to log data to. --- Table names are created internally, so don’t worry about those. - -EXECUTE sp_HumanEvents - @output_database_name = N'YourDatabase', - @output_schema_name = N'dbo'; - --- Ideally, you’ll stick this in an Agent Job, so you don’t need to rely on an SSMS window being open all the time. --- The job creation code linked is set to check in every 10 seconds, in case of errors. --- Internally, this will run in its own loop with a WAITFOR of 5 seconds to flush data out. - - --- Part of what gets installed when you log data to tables are some views in the same database. - --- You can check in on them like this: - -/*Queries*/ -SELECT TOP 1000 * FROM dbo.HumanEvents_Queries; -/*Waits*/ -SELECT TOP 1000 * FROM dbo.HumanEvents_WaitsByQueryAndDatabase; -SELECT TOP 1000 * FROM dbo.HumanEvents_WaitsByDatabase; -SELECT TOP 1000 * FROM dbo.HumanEvents_WaitsTotal; -/*Blocking*/ -SELECT TOP 1000 * FROM dbo.HumanEvents_Blocking; -/*Compiles, only on newer versions of SQL Server*/ -SELECT TOP 1000 * FROM dbo.HumanEvents_CompilesByDatabaseAndObject; -SELECT TOP 1000 * FROM dbo.HumanEvents_CompilesByQuery; -SELECT TOP 1000 * FROM dbo.HumanEvents_CompilesByDuration; -/*Otherwise*/ -SELECT TOP 1000 * FROM dbo.HumanEvents_Compiles_Legacy; -/*Parameterization data, if available (comes along with compiles)*/ -SELECT TOP 1000 * FROM dbo.HumanEvents_Parameterization; -/*Recompiles, only on newer versions of SQL Server*/ -SELECT TOP 1000 * FROM dbo.HumanEvents_RecompilesByDatabaseAndObject; -SELECT TOP 1000 * FROM dbo.HumanEvents_RecompilesByQuery; -SELECT TOP 1000 * FROM dbo.HumanEvents_RecompilesByDuration; -/*Otherwise*/ -SELECT TOP 1000 * FROM dbo.HumanEvents_Recompiles_Legacy; diff --git a/sp_HumanEvents/README.md b/sp_HumanEvents/README.md index 3d80b528..a2b78487 100644 --- a/sp_HumanEvents/README.md +++ b/sp_HumanEvents/README.md @@ -76,35 +76,33 @@ Misuse of this procedure can harm performance. Be very careful about introducing ## Usage Examples -For execution examples, see here: [Examples.sql](Examples.sql) - If you set up sessions to capture long term data, you'll need an agent job set up to poll them. You can find an example of that here: [sp_Human Events Agent Job Example.sql](sp_Human%20Events%20Agent%20Job%20Example.sql) Here are some basic usage examples: ```sql -- Basic execution to capture queries -EXEC dbo.sp_HumanEvents; +EXECUTE dbo.sp_HumanEvents; -- Capture blocking events for at least 1 second -EXEC dbo.sp_HumanEvents +EXECUTE dbo.sp_HumanEvents @event_type = 'blocking', @blocking_duration_ms = 1000; -- Capture waits in a specific database -EXEC dbo.sp_HumanEvents +EXECUTE dbo.sp_HumanEvents @event_type = 'waits', @database_name = 'YourDatabase'; -- Set up a permanent session for logging -EXEC dbo.sp_HumanEvents +EXECUTE dbo.sp_HumanEvents @event_type = 'query', @keep_alive = 1, @output_database_name = 'DBA', @output_schema_name = 'dbo'; -- Clean up all sessions and tables -EXEC dbo.sp_HumanEvents +EXECUTE dbo.sp_HumanEvents @cleanup = 1, @output_database_name = 'DBA', @output_schema_name = 'dbo'; @@ -128,13 +126,13 @@ It will work on any Extended Event that captures the blocked process report. If Enable the blocked process report: ```sql -EXEC sys.sp_configure +EXECUTE sys.sp_configure N'show advanced options', 1; RECONFIGURE; GO -EXEC sys.sp_configure +EXECUTE sys.sp_configure N'blocked process threshold', 5; --Seconds RECONFIGURE; @@ -199,23 +197,23 @@ ON SERVER ```sql -- Basic usage with default session name -EXEC dbo.sp_HumanEventsBlockViewer; +EXECUTE dbo.sp_HumanEventsBlockViewer; -- Use with a custom extended event session name -EXEC dbo.sp_HumanEventsBlockViewer +EXECUTE dbo.sp_HumanEventsBlockViewer @session_name = N'blocked_process_report'; -- Filter by a specific database -EXEC dbo.sp_HumanEventsBlockViewer +EXECUTE dbo.sp_HumanEventsBlockViewer @database_name = 'YourDatabase'; -- Analyze blocking events for a specific time period -EXEC dbo.sp_HumanEventsBlockViewer +EXECUTE dbo.sp_HumanEventsBlockViewer @start_date = '2025-01-01 08:00', @end_date = '2025-01-01 17:00'; -- Log results to permanent tables -EXEC dbo.sp_HumanEventsBlockViewer +EXECUTE dbo.sp_HumanEventsBlockViewer @log_to_table = 1, @log_database_name = 'DBA', @log_schema_name = 'dbo'; diff --git a/sp_LogHunter/README.md b/sp_LogHunter/README.md index 91ed9c84..4e19d651 100644 --- a/sp_LogHunter/README.md +++ b/sp_LogHunter/README.md @@ -30,28 +30,28 @@ It helps you give you a fuller, better picture of any bad stuff happening. ```sql -- Basic execution to search the last 7 days of error logs -EXEC dbo.sp_LogHunter; +EXECUTE dbo.sp_LogHunter; -- Search logs for the last 30 days -EXEC dbo.sp_LogHunter +EXECUTE dbo.sp_LogHunter @days_back = -30; -- Search a specific time period -EXEC dbo.sp_LogHunter +EXECUTE dbo.sp_LogHunter @start_date = '2025-01-01 00:00:00', @end_date = '2025-01-02 00:00:00'; -- Search for a specific custom message -EXEC dbo.sp_LogHunter +EXECUTE dbo.sp_LogHunter @custom_message = 'login failed'; -- Only search for the custom message, ignore other errors -EXEC dbo.sp_LogHunter +EXECUTE dbo.sp_LogHunter @custom_message = 'login failed', @custom_message_only = 1; -- Only search the current error log -EXEC dbo.sp_LogHunter +EXECUTE dbo.sp_LogHunter @first_log_only = 1; ``` diff --git a/sp_PressureDetector/README.md b/sp_PressureDetector/README.md index b7a4259f..5784d14c 100644 --- a/sp_PressureDetector/README.md +++ b/sp_PressureDetector/README.md @@ -41,26 +41,26 @@ All you need to do is hit F5 to get information about: ```sql -- Basic execution to check all pressure types -EXEC dbo.sp_PressureDetector; +EXECUTE dbo.sp_PressureDetector; -- Check only CPU pressure -EXEC dbo.sp_PressureDetector +EXECUTE dbo.sp_PressureDetector @what_to_check = 'cpu'; -- Check only memory pressure -EXEC dbo.sp_PressureDetector +EXECUTE dbo.sp_PressureDetector @what_to_check = 'memory'; -- Skip looking at executing queries -EXEC dbo.sp_PressureDetector +EXECUTE dbo.sp_PressureDetector @skip_queries = 1; -- Take a 10-second sample of server metrics -EXEC dbo.sp_PressureDetector +EXECUTE dbo.sp_PressureDetector @sample_seconds = 10; -- Log results to a table -EXEC dbo.sp_PressureDetector +EXECUTE dbo.sp_PressureDetector @log_to_table = 1, @log_database_name = 'DBA', @log_schema_name = 'dbo'; diff --git a/sp_QuickieStore/Examples.sql b/sp_QuickieStore/Examples.sql deleted file mode 100644 index 0e39a6e2..00000000 --- a/sp_QuickieStore/Examples.sql +++ /dev/null @@ -1,437 +0,0 @@ -/* -███████╗██╗ ██╗ █████╗ ███╗ ███╗██████╗ ██╗ ███████╗ -██╔════╝╚██╗██╔╝██╔══██╗████╗ ████║██╔══██╗██║ ██╔════╝ -█████╗ ╚███╔╝ ███████║██╔████╔██║██████╔╝██║ █████╗ -██╔══╝ ██╔██╗ ██╔══██║██║╚██╔╝██║██╔═══╝ ██║ ██╔══╝ -███████╗██╔╝ ██╗██║ ██║██║ ╚═╝ ██║██║ ███████╗███████╗ -╚══════╝╚═╝ ╚═╝╚═╝ ╚═╝╚═╝ ╚═╝╚═╝ ╚══════╝╚══════╝ - - ██████╗ █████╗ ██╗ ██╗ ███████╗ -██╔════╝██╔══██╗██║ ██║ ██╔════╝ -██║ ███████║██║ ██║ ███████╗ -██║ ██╔══██║██║ ██║ ╚════██║ -╚██████╗██║ ██║███████╗███████╗███████║ - ╚═════╝╚═╝ ╚═╝╚══════╝╚══════╝╚══════╝ - -Copyright 2025 Darling Data, LLC -https://www.erikdarling.com/ - -For support, head over to GitHub: -https://github.com/erikdarlingdata/DarlingData -*/ - -/*Get help!*/ -EXECUTE dbo.sp_QuickieStore - @help = 1; - -/*The default is finding the top 10 sorted by CPU in the last seven days.*/ -EXECUTE dbo.sp_QuickieStore - @database_name = 'StackOverflow2013'; - -/*Find top 10 sorted by memory*/ -EXECUTE dbo.sp_QuickieStore - @database_name = 'StackOverflow2013', - @sort_order = 'memory', - @top = 10; - -/*Find top 10 in each user database sorted by cpu*/ -EXECUTE dbo.sp_QuickieStore - @get_all_databases = 1, - @sort_order = 'cpu', - @top = 10; - -/*Search for specific query_ids*/ -EXECUTE dbo.sp_QuickieStore - @database_name = 'StackOverflow2013', - @top = 10, - @include_query_ids = '13977, 13978'; - - -/*Search for specific plan_ids*/ -EXECUTE dbo.sp_QuickieStore - @database_name = 'StackOverflow2013', - @sort_order = 'memory', - @top = 10, - @start_date = '20210320', - @include_plan_ids = '1896, 1897'; - - -/*Ignore for specific query_ids*/ -EXECUTE dbo.sp_QuickieStore - @database_name = 'StackOverflow2013', - @top = 10, - @ignore_query_ids = '13977, 13978'; - - -/*Ignore for specific plan_ids*/ -EXECUTE dbo.sp_QuickieStore - @database_name = 'StackOverflow2013', - @sort_order = 'memory', - @top = 10, - @start_date = '20210320', - @ignore_plan_ids = '1896, 1897'; - - -/*Search for queries within a date range*/ -EXECUTE dbo.sp_QuickieStore - @database_name = 'StackOverflow2013', - @sort_order = 'memory', - @top = 10, - @start_date = '20210320', - @end_date = '20210321'; - -/*Filter out weekends and anything outside of your choice of hours.*/ -EXECUTE dbo.sp_QuickieStore - @database_name = 'StackOverflow2013', - @workdays = 1, - @work_start = '8am', - @work_end = '6pm' - - -/*Search for queries with a minimum execution count*/ -EXECUTE dbo.sp_QuickieStore - @database_name = 'StackOverflow2013', - @top = 10, - @execution_count = 10; - - -/*Search for queries over a specific duration*/ -EXECUTE dbo.sp_QuickieStore - @database_name = 'StackOverflow2013', - @top = 10, - @duration_ms = 10000; - - -/*Use wait filter to search for queries responsible for high waits*/ -EXECUTE dbo.sp_QuickieStore - @database_name = 'StackOverflow2013', - @wait_filter = 'memory', - @sort_order = 'memory'; - -/*We also support using wait types as a sort order, see the documentation for the full list. -The wait-related sort orders are special because we add an extra column for the duration of the wait type you are asking for. -It's all the way over on the right. -*/ -EXECUTE dbo.sp_QuickieStore - @database_name = 'StackOverflow2013', - @sort_order = 'memory waits'; - -/*You can also sort by total wait time across all waits. */ -EXECUTE dbo.sp_QuickieStore - @database_name = 'StackOverflow2013', - @sort_order = 'total waits'; - - -/*Search for queries with a specific execution type -When we do not provide this parameter, we grab all types. -This example grabs "aborted" queries, which are queries cancelled by the client. -This is a great way to find timeouts. -*/ -EXECUTE dbo.sp_QuickieStore - @database_name = 'StackOverflow2013', - @top = 10, - @execution_type_desc = 'aborted'; - -/*Search for queries that errored -As above, but for "exception" queries. -This grabs queries that were cancelled by throwing exceptions. -It's no substitute for proper error monitoring, but it can be a good early warning. -*/ -EXECUTE dbo.sp_QuickieStore - @database_name = 'StackOverflow2013', - @top = 10, - @execution_type_desc = 'exception'; - -/*Search for queries that finished normally -As above, but for "regular" queries. -This grabs queries that were not cancelled. -*/ -EXECUTE dbo.sp_QuickieStore - @database_name = 'StackOverflow2013', - @top = 10, - @execution_type_desc = 'regular'; - - -/*Search for a specific stored procedure*/ -EXECUTE dbo.sp_QuickieStore - @database_name = 'StackOverflow2013', - @procedure_name = 'top_percent_sniffer'; - -/*Search for a specific stored procedure in a specific schema*/ -EXECUTE dbo.sp_QuickieStore - @database_name = 'StackOverflow2013', - @procedure_schema = 'not_dbo' - @procedure_name = 'top_percent_sniffer'; - -/*Search for specific query text*/ -EXECUTE dbo.sp_QuickieStore - @database_name = 'StackOverflow2013', - @query_text_search = 'WITH Comment'; - -/*Search for specific query text, with brackets automatically escaped. -Commonly needed when dealing with ORM queries. -*/ -EXECUTE dbo.sp_QuickieStore - @database_name = 'StackOverflow2013', - @query_text_search = 'FROM [users] AS [t0]', - @escape_brackets = 1; - -/*By default, We use '\' to escape when @escape_brackets = 1 is set. -Maybe you want something else. -Provide it with @escape_character. -*/ -EXECUTE dbo.sp_QuickieStore - @database_name = 'StackOverflow2013', - @query_text_search = 'FROM foo\bar AS [t0]', - @escape_character = '~' - @escape_brackets = 1; - - -/*Find every reference to a particular table in your Query Store data, sorted by their execution counts. -Quite expensive! -Handy when tuning or finding dependencies, but only as good as what your Query Store has captured. -Makes use of @get_all_databases = 1, which lets you search all user databases. -Note the abuse of @start_date. By setting it very far back in the past and leaving @end_date unspecified, we cover all of the data. -We also abuse @top by setting it very high. -*/ -EXECUTE dbo.sp_QuickieStore - @get_all_databases = 1, - @start_date = '20000101', - @sort_order = 'executions', - @query_text_search = 'MyTable', - @top = 100; - -/*Filter out certain query text with @query_text_search_not. -Good for when @query_text_search gets false positives. -After all, it's only doing string searching. -*/ -EXECUTE dbo.sp_QuickieStore - @get_all_databases = 1, - @start_date = '20000101', - @sort_order = 'executions', - @query_text_search = 'MyTable', - @query_text_search_not = 'MyTable_secret_backup' - @top = 100; - - -/*What happened recently on a database?*/ -EXECUTE dbo.sp_QuickieStore - @database_name = 'StackOverflow2013' - @sort_order = 'recent'; - -/*What happened recently that referenced my database? -Good for finding cross-database queries, such as when checking if a database is dead code. -Don't forget that queries in a database do not need to reference it explicitly! -*/ -EXECUTE dbo.sp_QuickieStore - @get_all_databases = 1, - @start_date = '20000101', - @sort_order = 'recent', - @query_text_search = 'StackOverflow2013' - @top = 10; - -/*Only return queries with feedback (2022+)*/ -EXECUTE dbo.sp_QuickieStore - @database_name = 'StackOverflow2013', - @only_query_with_feedback = 1; - -/*Only return queries with variants (2022+)*/ -EXECUTE dbo.sp_QuickieStore - @database_name = 'StackOverflow2013', - @only_query_with_variants = 1; - -/*Only return queries with forced plans (2022+)*/ -EXECUTE dbo.sp_QuickieStore - @database_name = 'StackOverflow2013', - @only_query_with_forced_plans = 1; - -/*Only return queries with forced plan failures (2022+)*/ -EXECUTE dbo.sp_QuickieStore - @database_name = 'StackOverflow2013', - @only_query_with_forced_plan_failures = 1; - -/*Only return queries with query hints (2022+)*/ -EXECUTE dbo.sp_QuickieStore - @database_name = 'StackOverflow2013', - @only_query_with_hints = 1; - -/*Use expert mode to return additional columns*/ -EXECUTE dbo.sp_QuickieStore - @database_name = 'StackOverflow2013', - @sort_order = 'memory', - @top = 10, - @expert_mode = 1; - - -/*Use format output to add commas to larger numbers -This is enabled by default. -*/ -EXECUTE dbo.sp_QuickieStore - @database_name = 'StackOverflow2013', - @sort_order = 'memory', - @top = 10, - @format_output = 1; - -/*Disable format output to remove commas.*/ -EXECUTE dbo.sp_QuickieStore - @database_name = 'StackOverflow2013', - @sort_order = 'memory', - @top = 10, - @format_output = 0; - -/*Change the timezone show in your outputs. -This is only an output-formatting change. -It does not change how dates are processed. -*/ -EXECUTE dbo.sp_QuickieStore - @database_name = 'StackOverflow2013', - @timezone = 'Egypt Standard Time'; - -/*Debugging something complex? -Hide the bottom table with @hide_help_table = 1 when you need more room. -*/ -EXECUTE dbo.sp_QuickieStore - @database_name = 'StackOverflow2013', - @hide_help_table = 1, - @sort_order = 'latch waits', - @top = 50; - -/*Search by query hashes*/ -EXECUTE dbo.sp_QuickieStore - @include_query_hashes = '0x1AB614B461F4D769,0x1CD777B461F4D769'; - -/*Search by plan hashes*/ -EXECUTE dbo.sp_QuickieStore - @include_plan_hashes = '0x6B84B820B8B38564,0x6B84B999D7B38564'; - -/*Search by SQL Handles -Do you need to find if one Query Store is tracking the same query that is present in another database's Query Store? If so, use the statement_sql_handle to do that. -This helps with scenarios where you have multiple production databases which have the same schema and you want to compare performance across Query Stores. -*/ -EXECUTE dbo.sp_QuickieStore - @include_sql_handles = - '0x0900F46AC89E66DF744C8A0AD4FD3D3306B90000000000000000000000000000000000000000000000000000,0x0200000AC89E66DF744C8A0AD4FD3D3306B90000000000000000000000000000000000000000000000000000'; - -/*Search, but ignoring some query hashes*/ -EXECUTE dbo.sp_QuickieStore - @ignore_query_hashes = '0x1AB614B461F4D769,0x1CD777B461F4D769'; - -/*Search, but ignoring some plan hashes*/ -EXECUTE dbo.sp_QuickieStore - @ignore_plan_hashes = '0x6B84B820B8B38564,0x6B84B999D7B38564'; - -/*Search, but ignoring some SQL Handles*/ -EXECUTE dbo.sp_QuickieStore - @ignore_sql_handles = - '0x0900F46AC89E66DF744C8A0AD4FD3D3306B90000000000000000000000000000000000000000000000000000,0x0200000AC89E66DF744C8A0AD4FD3D3306B90000000000000000000000000000000000000000000000000000'; - -/*What query hashes have the most plans? -This sort order is special because it needs to return multiple rows for each of the @top hashes it looks at. -It is also special because it adds some new columns all the way over on the right of the output. -*/ -EXECUTE dbo.sp_QuickieStore - @database_name = 'StackOverflow2013', - @sort_order = 'plan count by hashes'; - -/*Check for regressions. -Specifically, this checks for queries that did more logical reads last week than this week. -The default dates are helpful here. The default @start_date and @end_date specify last week for us and @regression_baseline_end_date defaults to being one week after @regression_baseline_start_date. -However, we need to specify @regression_baseline_start_date so that sp_QuickieStore knows to check for regressions. -Searches by query hash, so you will won't be caught out by identical queries with different query ids. -*/ -DECLARE @TwoWeekAgo datetimeoffset(7) = DATEADD(WEEK, -2, SYSDATETIMEOFFSET()); - -EXECUTE dbo.sp_QuickieStore - @database_name = 'StackOverflow2013', - @sort_order = 'logical reads', - @regression_baseline_start_date = @TwoWeekAgo; - -/*Check for improved queries. -I deleted some indexes yesterday. -Let's see what's doing less writes today. -Since we're checking for improvements rather than regressions, we use @regression_direction = 'improved'. -This is a good chance to point out that the @end_date parameters do comparisons with < rather than <=. -The @start_data parameters, of course, use >=. -*/ -DECLARE @StartOfYesterday datetimeoffset(7) = CONVERT(date, DATEADD(DAY, -1, SYSDATETIMEOFFSET())), - @StartOfToday datetimeoffset(7) = CONVERT(date, SYSDATETIMEOFFSET()), - @StartOfTomorrow datetimeoffset(7) = CONVERT(date, DATEADD(DAY, 1, SYSDATETIMEOFFSET())); - -EXECUTE dbo.sp_QuickieStore - @database_name = 'StackOverflow2013', - @sort_order = 'writes', - @regression_direction = 'improved', - @regression_baseline_start_date = @StartOfYesterday, - @regression_baseline_end_date = @StartOfToday, - @start_date = @StartOfToday, - @end_date = @StartOfTomorrow; - -/*Check for percentage changes in performance. -By default, our @regression parameters have us check for changes in the raw numbers. -It's just plain subtraction: new minus old. -This means that a query that used to read hardly anything from disk but now reads triple that is indistinguishable from the noise in a query that reads lots. -To get percentage changes instead, specify @regression_comparator = 'relative'. -The default is @regression_comparator = 'absolute'. - -To see the difference, run `sp_QuickieStore` twice. -To save space on your screen, we will specify @hide_help_table = 1 to hide the table normally at the bottom of the normal output. -*/ -DECLARE @TwoWeekAgo datetimeoffset(7) = DATEADD(WEEK, -2, SYSDATETIMEOFFSET()); - -EXECUTE dbo.sp_QuickieStore - @database_name = 'StackOverflow2013', - @sort_order = 'physical reads', - @hide_help_table = 1, - @regression_comparator = 'relative', - @regression_baseline_start_date = @TwoWeekAgo; - -EXECUTE dbo.sp_QuickieStore - @database_name = 'StackOverflow2013', - @sort_order = 'physical reads', - @hide_help_table = 1, - @regression_comparator = 'absolute', - @regression_baseline_start_date = @TwoWeekAgo; - -/*Check for changes in modulus. -What if you're looking for sheer size of changes, rather than the direction of the change? -For example, you might care about a 30 second reduction in duration just as much as a 30 second increase. -Use @regression_direction = 'absolute'. -And while we're at it, let's check all user databases with @get_all_databases = 1. -*/ -DECLARE @TwoWeekAgo datetimeoffset(7) = DATEADD(WEEK, -2, SYSDATETIMEOFFSET()); - -EXECUTE dbo.sp_QuickieStore - @get_all_databases = 1, - @sort_order = 'duration', - @regression_direction = 'absolute', - @regression_baseline_start_date = @TwoWeekAgo; - -/*Get version info.*/ -DECLARE @version_output varchar(30), - @version_date_output datetime; - -EXECUTE sp_QuickieStore - @version = @version_output OUTPUT, - @version_date = @version_date_output OUTPUT; - -SELECT - Version = @version_output, - VersionDate = @version_date_output; - -/*Search for queries that take a while and return lots of rows on average*/ -EXECUTE dbo.sp_QuickieStore - @database_name = 'StackOverflow2013', - @top = 10, - @sort_order = 'rows', - @duration_ms = 20000; - - -/*Troubleshoot performance*/ -EXECUTE dbo.sp_QuickieStore - @database_name = 'StackOverflow2013', - @troubleshoot_performance = 1; - -/*Debug dynamic SQL and temp table contents*/ -EXECUTE dbo.sp_QuickieStore - @database_name = 'StackOverflow2013', - @debug = 1; diff --git a/sp_QuickieStore/README.md b/sp_QuickieStore/README.md index f6f1cefd..e6dfe551 100644 --- a/sp_QuickieStore/README.md +++ b/sp_QuickieStore/README.md @@ -77,41 +77,39 @@ Use the `@expert_mode` parameter to return additional details. ## Examples -For more examples, see [Examples.sql](Examples.sql) - ```sql -- Basic execution - returns top 10 queries by CPU -EXEC dbo.sp_QuickieStore; +EXECUTE dbo.sp_QuickieStore; -- Look at top 20 queries by logical reads -EXEC dbo.sp_QuickieStore +EXECUTE dbo.sp_QuickieStore @sort_order = 'logical reads', @top = 20; -- Search for a specific query text -EXEC dbo.sp_QuickieStore +EXECUTE dbo.sp_QuickieStore @query_text_search = 'SELECT * FROM Orders'; -- Find queries from a specific procedure -EXEC dbo.sp_QuickieStore +EXECUTE dbo.sp_QuickieStore @procedure_name = 'usp_GetCustomerOrders'; -- Filter to queries that executed at least 1000 times -EXEC dbo.sp_QuickieStore +EXECUTE dbo.sp_QuickieStore @execution_count = 1000; -- Show queries with a minimum duration of 500ms -EXEC dbo.sp_QuickieStore +EXECUTE dbo.sp_QuickieStore @duration_ms = 500; -- Look for regressions against a baseline period -EXEC dbo.sp_QuickieStore +EXECUTE dbo.sp_QuickieStore @regression_baseline_start_date = '2025-01-01', @regression_baseline_end_date = '2025-01-08', @regression_direction = 'regressed'; -- Expert mode for additional details -EXEC dbo.sp_QuickieStore +EXECUTE dbo.sp_QuickieStore @expert_mode = 1; ``` From 365a499c29f68f556e824aea6f9c450665bd9d9c Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 15 Mar 2025 11:55:49 -0400 Subject: [PATCH 124/246] Create README.md --- sp_WhoIsActive Logging/README.md | 81 ++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 sp_WhoIsActive Logging/README.md diff --git a/sp_WhoIsActive Logging/README.md b/sp_WhoIsActive Logging/README.md new file mode 100644 index 00000000..6e999cdf --- /dev/null +++ b/sp_WhoIsActive Logging/README.md @@ -0,0 +1,81 @@ +# sp_WhoIsActive Logging + +This toolkit automates the collection and management of SQL Server activity data using Adam Machanic's popular sp_WhoIsActive stored procedure. It creates a comprehensive logging framework that captures server activity in daily tables and provides useful views for analysis. + +## Overview + +The sp_WhoIsActive Logging toolkit consists of several components: + +1. **Daily Table Creation**: Automatically creates tables named WhoIsActive_YYYYMMDD to store server activity data +2. **Data Collection**: Executes sp_WhoIsActive and logs output to the daily tables +3. **Data Retention**: Automatically manages retention by removing tables older than a specified period +4. **Analysis Views**: Creates views for querying across all tables and analyzing blocking chains +5. **Automated Collection**: Includes an Agent job for scheduling regular collection (default: every minute) + +## Prerequisites + +- Adam Machanic's sp_WhoIsActive stored procedure must be installed + - If you need to get or update: [https://github.com/amachanic/sp_whoisactive](https://github.com/amachanic/sp_whoisactive) + - If you get an error about @get_memory_info parameter, you need to update sp_WhoIsActive + +## Components + +The toolkit includes four scripts that should be executed in order: + +1. **01 sp_WhoIsActive Logging Views.sql**: Creates the stored procedure that manages the views +2. **02 sp_WhoIsActiveLogging Main.sql**: Creates the main logging procedure +3. **03 sp_WhoIsActiveLogging_Retention.sql**: Creates the data retention procedure +4. **04 sp_WhoIsActive Logging Agent Job.sql**: Creates the SQL Agent job for automated collection + +## Stored Procedures + +### sp_WhoIsActiveLogging_Main + +The main procedure that handles data collection and table management. + +| Parameter Name | Data Type | Default Value | Description | +|----------------|-----------|---------------|-------------| +| @RetentionPeriod | integer | 10 | Number of days to keep data | + +### sp_WhoIsActiveLogging_Retention + +Handles the removal of tables older than the specified retention period. + +| Parameter Name | Data Type | Default Value | Description | +|----------------|-----------|---------------|-------------| +| @RetentionPeriod | integer | 10 | Number of days to keep data | + +### sp_WhoIsActiveLogging_CreateViews + +Creates two views for data analysis (no parameters): +- **dbo.WhoIsActive**: UNION ALL of all WhoIsActive_YYYYMMDD tables +- **dbo.WhoIsActive_blocking**: Recursive CTE that traverses blocking chains + +## Usage Examples + +```sql +-- Run the main logging procedure with default retention (10 days) +EXECUTE dbo.sp_WhoIsActiveLogging_Main; + +-- Run the main logging procedure with custom retention (30 days) +EXECUTE dbo.sp_WhoIsActiveLogging_Main + @RetentionPeriod = 30; + +-- Manually run the retention procedure to clean up old tables +EXECUTE dbo.sp_WhoIsActiveLogging_Retention + @RetentionPeriod = 15; + +-- Recreate the views (useful after adding new tables) +EXECUTE dbo.sp_WhoIsActiveLogging_CreateViews; +``` + +## Notes + +- The scripts use the master database by default +- New tables are created daily with the format WhoIsActive_YYYYMMDD +- The Agent job runs on a one-minute schedule by default +- Views are automatically refreshed when new tables are created +- Old tables are automatically dropped based on the retention period + +Copyright 2025 Darling Data, LLC +Released under MIT license \ No newline at end of file From dbf7c6f883df7126b06518b6a2af86fd87ae9d81 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 15 Mar 2025 12:16:52 -0400 Subject: [PATCH 125/246] add readme files for more things. add readme files for more things. --- Clear Token Perm/README.md | 71 ++++++++++++++++++++++++++++ Helper Views/README.md | 92 ++++++++++++++++++++++++++++++++++++ Install-All/README.md | 59 +++++++++++++++++++++++ Ola Stats Only Job/README.md | 42 ++++++++++++++++ String Functions/README.md | 86 +++++++++++++++++++++++++++++++++ 5 files changed, 350 insertions(+) create mode 100644 Clear Token Perm/README.md create mode 100644 Helper Views/README.md create mode 100644 Install-All/README.md create mode 100644 Ola Stats Only Job/README.md create mode 100644 String Functions/README.md diff --git a/Clear Token Perm/README.md b/Clear Token Perm/README.md new file mode 100644 index 00000000..60ef559b --- /dev/null +++ b/Clear Token Perm/README.md @@ -0,0 +1,71 @@ +# Clear Token Perm + +This directory contains scripts for monitoring and managing SQL Server's security token cache. The security token cache (TokenAndPermUserStore) can grow to a significant size in certain scenarios, potentially causing high memory usage and performance issues. + +## Overview + +SQL Server caches security tokens in memory, and in specific environments (particularly with frequent application role usage, or high numbers of users), this cache can grow to consume gigabytes of memory. These scripts provide solutions to monitor the cache size and automatically clear it when it exceeds a defined threshold. + +## Components + +The directory includes three files: + +1. **ClearTokenPerm.sql**: Creates a stored procedure to monitor and clear the security token cache +2. **ClearTokenPerm Agent Job.sql**: Creates a SQL Agent job to run the ClearTokenPerm procedure on a schedule +3. **Inflate Security Cache Demo And Analysis Script.sql**: Demonstrates how to artificially inflate the security cache for testing and provides analysis queries + +## ClearTokenPerm Stored Procedure + +The main stored procedure monitors the size of the TokenAndPermUserStore cache and clears it when it exceeds a specified threshold. + +| Parameter Name | Data Type | Default Value | Description | +|----------------|-----------|---------------|-------------| +| @CacheSizeGB | decimal(38,2) | None (required) | The threshold size in GB that triggers cache clearing | + +The procedure: +- Creates a logging table (ClearTokenPermLogging) if it doesn't exist +- Checks the current size of the TokenAndPermUserStore cache +- Clears the cache using DBCC FREESYSTEMCACHE if the threshold is exceeded +- Logs all checks with timestamp, cache size, and whether clearing was triggered + +## SQL Agent Job + +The Agent job script: +- Creates a job named "Clear Security Cache Every 30 Minutes" +- Runs the ClearTokenPerm procedure with a 1GB threshold +- Schedules execution every 30 minutes +- Includes error handling and transaction support + +## Demo and Analysis Script + +The demo script: +- Creates an application role and executes a loop to inflate the cache +- Provides detailed analysis queries to examine: + - Token distribution in the cache + - Logins and tokens per login + - Users and tokens per user + - Cache invalidations per database +- Includes cleanup steps + +## Usage Examples + +```sql +-- Check and potentially clear the cache if it's over 2GB +EXECUTE dbo.ClearTokenPerm + @CacheSizeGB = 2; + +-- Query the logging table to see history +SELECT * +FROM dbo.ClearTokenPermLogging +ORDER BY log_date DESC; + +-- Clear the logging table +TRUNCATE TABLE dbo.ClearTokenPermLogging; +``` + +## Warning + +The DBCC FREESYSTEMCACHE command used in these scripts will clear the security cache, which may cause a temporary performance impact as the cache is rebuilt. Test thoroughly in non-production environments before deploying to production. + +Copyright 2025 Darling Data, LLC +Released under MIT license \ No newline at end of file diff --git a/Helper Views/README.md b/Helper Views/README.md new file mode 100644 index 00000000..d31d2951 --- /dev/null +++ b/Helper Views/README.md @@ -0,0 +1,92 @@ +# Helper Views + +This directory contains helper views and functions for diagnosing various SQL Server performance issues. These scripts are particularly useful for presentations and educational purposes, but can also be used in troubleshooting scenarios. + +## Overview + +The collection includes: +- Views for analyzing index sizes +- Functions for examining lock information +- Views for examining memory usage +- Procedures for testing tempdb performance + +## Components + +### WhatsUpIndexes View + +A view that provides detailed information about index sizes in the current database. + +**Functionality**: +- Displays database, schema, table, and index names +- Shows in-row pages size in MB +- Shows LOB (Large Object) pages size in MB +- Reports number of in-row used pages +- Displays row count for each index +- Filters out system objects and table-valued functions + +Usage: +```sql +SELECT * FROM dbo.WhatsUpIndexes +ORDER BY in_row_mb DESC; +``` + +### WhatsUpLocks Function + +A table-valued function that provides information about locks taken by specific sessions. + +| Parameter Name | Data Type | Default Value | Description | +|----------------|-----------|---------------|-------------| +| @spid | integer | NULL | Session ID to examine. If NULL, returns information for all sessions | + +**Functionality**: +- Displays session ID and blocking session ID information +- Shows lock modes, resource types, and lock status +- Identifies locked objects and associated index names +- Counts different lock types (HOBT, object, page, and row locks) +- Reports total lock count + +Usage: +```sql +-- Check locks for a specific session +SELECT * FROM dbo.WhatsUpLocks(51); + +-- Check locks for all sessions +SELECT * FROM dbo.WhatsUpLocks(NULL); +``` + +### WhatsUpMemory View + +A view that examines what's in SQL Server memory. + +**Functionality**: +- Shows database, schema, object, and index information +- Calculates in-row pages in MB (for data types 1 and 3) +- Calculates LOB pages in MB (for data type 2) +- Reports total buffer cache pages + +Usage: +```sql +SELECT * FROM dbo.WhatsUpMemory +ORDER BY pages DESC; +``` + +### tempdb_tester Procedure + +A stored procedure that generates semi-random tempdb activity, useful for testing and demonstration purposes. + +**Functionality**: +- Creates a temporary table with approximately 10,000 rows +- Performs various DML operations (UPDATE, DELETE, INSERT) +- Uses RECOMPILE hint for optimal execution plans + +Usage: +```sql +EXECUTE dbo.tempdb_tester; +``` + +## Warning + +Some of these scripts (particularly WhatsUpMemory) may cause performance issues if run on busy production servers. Use with caution, especially on servers with large amounts of memory. + +Copyright 2025 Darling Data, LLC +Released under MIT license \ No newline at end of file diff --git a/Install-All/README.md b/Install-All/README.md new file mode 100644 index 00000000..5768466f --- /dev/null +++ b/Install-All/README.md @@ -0,0 +1,59 @@ +# Install-All + +This directory contains tools to merge all the Darling Data stored procedures into a single installation file. + +## Overview + +Instead of installing each stored procedure individually, you can use the comprehensive DarlingData.sql file to install all procedures at once. This file is automatically generated using the Merge-All.ps1 PowerShell script. + +## Components + +1. **DarlingData.sql**: A merged file containing all stored procedures from the repository +2. **Merge-All.ps1**: PowerShell script that generates the DarlingData.sql file + +## Included Stored Procedures + +The DarlingData.sql file includes all of the following stored procedures: + +| Procedure | Description | Source | +|-----------|-------------|--------| +| [sp_HealthParser](../sp_HealthParser) | Analyzes the system health extended event for performance information | [Link](../sp_HealthParser) | +| [sp_HumanEvents](../sp_HumanEvents) | Makes extended events easy to use for common scenarios | [Link](../sp_HumanEvents) | +| [sp_HumanEventsBlockViewer](../sp_HumanEvents) | Analyzes blocked process reports | [Link](../sp_HumanEvents) | +| [sp_IndexCleanup](../sp_IndexCleanup) | Identifies unused and duplicate indexes | [Link](../sp_IndexCleanup) | +| [sp_LogHunter](../sp_LogHunter) | Searches SQL Server error logs for important messages | [Link](../sp_LogHunter) | +| [sp_PressureDetector](../sp_PressureDetector) | Detects CPU and memory pressure in SQL Server | [Link](../sp_PressureDetector) | +| [sp_QuickieStore](../sp_QuickieStore) | Fast and configurable way to navigate Query Store data | [Link](../sp_QuickieStore) | + +## Usage + +### Using the Pre-Generated File + +Simply run the DarlingData.sql script in SQL Server Management Studio to install all stored procedures at once. + +### Generating a Fresh Copy + +If you want to generate a fresh copy of the DarlingData.sql file: + +1. Make sure you have PowerShell installed +2. Navigate to the Install-All directory in PowerShell +3. Run the script: + +```powershell +.\Merge-All.ps1 +``` + +The script will: +1. Find all sp_* directories +2. Skip sp_WhoIsActive directories +3. Get all sp_* files (skipping agent job files) +4. Combine them into a single file +5. Add a compile date header +6. Save as DarlingData.sql + +## Note + +The WhoIsActive Logging procedures are not included in this file, as they have a different installation process and depend on Adam Machanic's sp_WhoIsActive. + +Copyright 2025 Darling Data, LLC +Released under MIT license \ No newline at end of file diff --git a/Ola Stats Only Job/README.md b/Ola Stats Only Job/README.md new file mode 100644 index 00000000..dcb1ecbd --- /dev/null +++ b/Ola Stats Only Job/README.md @@ -0,0 +1,42 @@ +# Ola Stats Only Job + +This directory contains a script to create a SQL Server Agent job for nightly statistics updates using Ola Hallengren's maintenance solution. + +## Overview + +Statistics in SQL Server are vital for query optimization but can become stale over time, leading to suboptimal query plans. This script sets up an automated job to update statistics on a regular schedule using Ola Hallengren's popular IndexOptimize stored procedure, focused specifically on statistics updates rather than the full index maintenance. + +## Prerequisites + +- Ola Hallengren's SQL Server Maintenance Solution must be installed + - Download from: [https://ola.hallengren.com/downloads.html](https://ola.hallengren.com/downloads.html) + - This script requires the version from 2018-06-16 or later, which includes the @StatisticsModificationLevel parameter + +## Configuration Details + +The script creates a SQL Agent job with the following default settings: +- Job name: "Nightly Stats Update Job via Ola" +- Database target: All user databases +- Job owner: sa +- Schedule: Every night at midnight +- Statistics modification level: 5% (only updates statistics that have changed by at least 5%) + +## Customization Options + +You may need to modify the script to: +- Change the target database (currently master) +- Change the job owner from sa +- Adjust the schedule from the default midnight run +- Set up failure emails and alerting +- Change the StatisticsModificationLevel from 5% to match your environment needs + +## Usage + +Simply run the script in SQL Server Management Studio after ensuring you have the prerequisites installed. The job will be created and scheduled automatically. + +## Note + +This script focuses exclusively on statistics updates. If you also need index reorganization or rebuilds, you should consider using Ola's full maintenance solution. + +Copyright 2025 Darling Data, LLC +Released under MIT license \ No newline at end of file diff --git a/String Functions/README.md b/String Functions/README.md new file mode 100644 index 00000000..35fef6ec --- /dev/null +++ b/String Functions/README.md @@ -0,0 +1,86 @@ +# String Functions + +This directory contains a set of utility functions for string manipulation in SQL Server. These functions provide efficient ways to extract or remove specific characters from strings. + +## Overview + +The functions in this directory help with common string manipulation tasks: +- Extracting only letters from strings +- Extracting only numbers from strings +- Removing specific characters from strings + +Each function is provided in two versions: +1. A version that requires an existing Numbers table +2. A self-contained version with an inline CTE (no external dependencies) + +## Functions + +### get_letters + +Extracts only alphabetic characters (A-Z, a-z) from a string. + +| Parameter Name | Data Type | Description | +|----------------|-----------|-------------| +| @string | nvarchar(4000) | The input string to extract letters from | + +**Return Value**: Table with a single column `letters_only` containing only the letters from the input string. + +Usage: +```sql +SELECT letters_only FROM dbo.get_letters(N'abc123!@#'); +-- Returns: abc + +-- Self-contained version (no dependency on Numbers table) +SELECT letters_only FROM dbo.get_letters_cte(N'abc123!@#'); +-- Returns: abc +``` + +### get_numbers + +Extracts only numeric characters (0-9) from a string. + +| Parameter Name | Data Type | Description | +|----------------|-----------|-------------| +| @string | nvarchar(4000) | The input string to extract numbers from | + +**Return Value**: Table with a single column `numbers_only` containing only the numbers from the input string. + +Usage: +```sql +SELECT numbers_only FROM dbo.get_numbers(N'abc123!@#'); +-- Returns: 123 + +-- Self-contained version (no dependency on Numbers table) +SELECT numbers_only FROM dbo.get_numbers_cte(N'abc123!@#'); +-- Returns: 123 +``` + +### strip_characters + +Removes specified characters from a string. + +| Parameter Name | Data Type | Description | +|----------------|-----------|-------------| +| @string | nvarchar(4000) | The input string to process | +| @match_expression | nvarchar(100) | Characters to remove, specified as a LIKE pattern | + +**Return Value**: Table with a single column `strip_characters` containing the input string with specified characters removed. + +Usage: +```sql +SELECT strip_characters FROM dbo.strip_characters(N'abc123!@#', N'[0-9]'); +-- Returns: abc!@# + +-- Self-contained version (no dependency on Numbers table) +SELECT strip_characters FROM dbo.strip_characters_cte(N'abc123!@#', N'[^a-z]'); +-- Returns: abc +``` + +## Implementation Notes + +- All functions are schema-bound for better performance +- String concatenation is implemented using XML PATH, making the functions independent of SQL Server version +- The _cte variants don't require an external Numbers table, but may be less efficient for very large strings + +Copyright 2025 Darling Data, LLC +Released under MIT license \ No newline at end of file From 80e6b3d8eafba60dffe4d2af830aff7367fce757 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 15 Mar 2025 12:26:11 -0400 Subject: [PATCH 126/246] Update README.md --- Clear Token Perm/README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Clear Token Perm/README.md b/Clear Token Perm/README.md index 60ef559b..bb0b0838 100644 --- a/Clear Token Perm/README.md +++ b/Clear Token Perm/README.md @@ -55,9 +55,11 @@ EXECUTE dbo.ClearTokenPerm @CacheSizeGB = 2; -- Query the logging table to see history -SELECT * -FROM dbo.ClearTokenPermLogging -ORDER BY log_date DESC; +SELECT + cl.* +FROM dbo.ClearTokenPermLogging AS cl +ORDER BY + cl.log_date DESC; -- Clear the logging table TRUNCATE TABLE dbo.ClearTokenPermLogging; @@ -68,4 +70,4 @@ TRUNCATE TABLE dbo.ClearTokenPermLogging; The DBCC FREESYSTEMCACHE command used in these scripts will clear the security cache, which may cause a temporary performance impact as the cache is rebuilt. Test thoroughly in non-production environments before deploying to production. Copyright 2025 Darling Data, LLC -Released under MIT license \ No newline at end of file +Released under MIT license From 7edf3b22559961040327abaaed397d4298f8b927 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 15 Mar 2025 12:27:29 -0400 Subject: [PATCH 127/246] Update README.md --- Helper Views/README.md | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/Helper Views/README.md b/Helper Views/README.md index d31d2951..4684000b 100644 --- a/Helper Views/README.md +++ b/Helper Views/README.md @@ -26,8 +26,11 @@ A view that provides detailed information about index sizes in the current datab Usage: ```sql -SELECT * FROM dbo.WhatsUpIndexes -ORDER BY in_row_mb DESC; +SELECT + w.* +FROM dbo.WhatsUpIndexes AS w +ORDER BY + w.in_row_mb DESC; ``` ### WhatsUpLocks Function @@ -48,10 +51,14 @@ A table-valued function that provides information about locks taken by specific Usage: ```sql -- Check locks for a specific session -SELECT * FROM dbo.WhatsUpLocks(51); +SELECT + wul.* +FROM dbo.WhatsUpLocks(51) AS wul; -- Check locks for all sessions -SELECT * FROM dbo.WhatsUpLocks(NULL); +SELECT + wul.* +FROM dbo.WhatsUpLocks(NULL) AS wul; ``` ### WhatsUpMemory View @@ -66,8 +73,11 @@ A view that examines what's in SQL Server memory. Usage: ```sql -SELECT * FROM dbo.WhatsUpMemory -ORDER BY pages DESC; +SELECT + wum.* +FROM dbo.WhatsUpMemory AS wum +ORDER BY + wum.pages DESC; ``` ### tempdb_tester Procedure @@ -89,4 +99,4 @@ EXECUTE dbo.tempdb_tester; Some of these scripts (particularly WhatsUpMemory) may cause performance issues if run on busy production servers. Use with caution, especially on servers with large amounts of memory. Copyright 2025 Darling Data, LLC -Released under MIT license \ No newline at end of file +Released under MIT license From 6a6637b23dfeb7a693c523631484f1e1abfef74d Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 15 Mar 2025 12:29:41 -0400 Subject: [PATCH 128/246] Update README.md --- String Functions/README.md | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/String Functions/README.md b/String Functions/README.md index 35fef6ec..be06a53d 100644 --- a/String Functions/README.md +++ b/String Functions/README.md @@ -27,11 +27,15 @@ Extracts only alphabetic characters (A-Z, a-z) from a string. Usage: ```sql -SELECT letters_only FROM dbo.get_letters(N'abc123!@#'); +SELECT + gl.letters_only +FROM dbo.get_letters(N'abc123!@#') AS gl; -- Returns: abc -- Self-contained version (no dependency on Numbers table) -SELECT letters_only FROM dbo.get_letters_cte(N'abc123!@#'); +SELECT + gl.letters_only +FROM dbo.get_letters_cte(N'abc123!@#') AS gl; -- Returns: abc ``` @@ -47,11 +51,15 @@ Extracts only numeric characters (0-9) from a string. Usage: ```sql -SELECT numbers_only FROM dbo.get_numbers(N'abc123!@#'); +SELECT + gn.numbers_only +FROM dbo.get_numbers(N'abc123!@#') AS gn; -- Returns: 123 -- Self-contained version (no dependency on Numbers table) -SELECT numbers_only FROM dbo.get_numbers_cte(N'abc123!@#'); +SELECT + gn.numbers_only +FROM dbo.get_numbers_cte(N'abc123!@#') AS gn; -- Returns: 123 ``` @@ -68,11 +76,15 @@ Removes specified characters from a string. Usage: ```sql -SELECT strip_characters FROM dbo.strip_characters(N'abc123!@#', N'[0-9]'); +SELECT + sc.strip_characters +FROM dbo.strip_characters(N'abc123!@#', N'[0-9]') AS sc; -- Returns: abc!@# -- Self-contained version (no dependency on Numbers table) -SELECT strip_characters FROM dbo.strip_characters_cte(N'abc123!@#', N'[^a-z]'); +SELECT + sc.strip_characters +FROM dbo.strip_characters_cte(N'abc123!@#', N'[^a-z]') AS sc; -- Returns: abc ``` @@ -83,4 +95,4 @@ SELECT strip_characters FROM dbo.strip_characters_cte(N'abc123!@#', N'[^a-z]'); - The _cte variants don't require an external Numbers table, but may be less efficient for very large strings Copyright 2025 Darling Data, LLC -Released under MIT license \ No newline at end of file +Released under MIT license From c1e128213cc1dd76929da7e82d0ae4d7430579ea Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 15 Mar 2025 12:30:46 -0400 Subject: [PATCH 129/246] compression check make sure index compression only makes it to eligible tables. --- sp_IndexCleanup/sp_IndexCleanup.sql | 27 +++++++++++++++++++++---- sp_QuickieStore/sp_QuickieStore.sql | 31 ++++++++++++++++++----------- 2 files changed, 42 insertions(+), 16 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 6cc34c44..607562e3 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -2898,7 +2898,12 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. THEN N'ON' ELSE N'OFF' END + - N', DATA_COMPRESSION = PAGE)' + + CASE + WHEN ce.can_compress = 1 + THEN ', DATA_COMPRESSION = PAGE' + ELSE N'' + END + + N')' + CASE WHEN ps.partition_function_name IS NOT NULL THEN N' ON ' + @@ -3194,8 +3199,17 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ELSE N' REBUILD' END + N' WITH (FILLFACTOR = 100, SORT_IN_TEMPDB = ON, ONLINE = ' + - CASE WHEN @online = 1 THEN N'ON' ELSE N'OFF' END + - N', DATA_COMPRESSION = PAGE);', + CASE + WHEN @online = 1 + THEN N'ON' + ELSE N'OFF' + END + + CASE + WHEN ce.can_compress = 1 + THEN ', DATA_COMPRESSION = PAGE' + ELSE N'' + END + + N')' + N'Compression type: All Partitions', superseded_info = NULL, /* No target index for compression scripts */ ia.superseded_by, /* Include superseded_by info for compression scripts */ @@ -3458,7 +3472,12 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. THEN N'ON' ELSE N'OFF' END + - N', DATA_COMPRESSION = PAGE);', + CASE + WHEN ce.can_compress = 1 + THEN ', DATA_COMPRESSION = PAGE' + ELSE N'' + END + + N')', N'Compression type: Per Partition | Partition: ' + CONVERT ( diff --git a/sp_QuickieStore/sp_QuickieStore.sql b/sp_QuickieStore/sp_QuickieStore.sql index 2e14830c..94626da9 100644 --- a/sp_QuickieStore/sp_QuickieStore.sql +++ b/sp_QuickieStore/sp_QuickieStore.sql @@ -557,7 +557,7 @@ CREATE TABLE plan_id bigint NOT NULL, query_hash binary(8) NOT NULL, plan_hash_count_for_query_hash integer NOT NULL, - PRIMARY KEY CLUSTERED (database_id, plan_id, query_hash) + PRIMARY KEY CLUSTERED (database_id, plan_id, query_hash) ); /* @@ -577,7 +577,7 @@ CREATE TABLE plan_id bigint NOT NULL, from_regression_baseline varchar(3) NOT NULL, total_query_wait_time_ms bigint NOT NULL, - PRIMARY KEY CLUSTERED(database_id, plan_id, from_regression_baseline) + PRIMARY KEY CLUSTERED(database_id, plan_id, from_regression_baseline) ); /* @@ -623,7 +623,7 @@ CREATE TABLE plan_id bigint NOT NULL, query_hash binary(8) NOT NULL, change_since_regression_time_period float NULL, - PRIMARY KEY CLUSTERED (database_id, plan_id, query_hash) + PRIMARY KEY CLUSTERED (database_id, plan_id, query_hash) ); /* @@ -5435,7 +5435,13 @@ BEGIN ON qsp.plan_id = waits.plan_id AND waits.from_regression_baseline = ''No'' WHERE 1 = 1 - AND qsq.query_hash IN (SELECT base.query_hash FROM #regression_baseline_runtime_stats AS base) + AND EXISTS + ( + SELECT + 1/0 + FROM #regression_baseline_runtime_stats AS base + WHERE base.query_hash = qsq.query_hash + ) ' + @where_clause + N' GROUP @@ -5625,7 +5631,7 @@ BEGIN + N' ) ) AS plans_for_hashes - ON hashes_with_changes.query_hash = plans_for_hashes.query_hash + ON hashes_with_changes.query_hash = plans_for_hashes.query_hash OPTION(RECOMPILE, OPTIMIZE FOR (@top = 9223372036854775807));' + @nc10; IF @debug = 1 @@ -6621,9 +6627,9 @@ BEGIN SUM(qsrs.count_executions * qsrs.avg_rowcount) FROM ' + @database_name_quoted + N'.sys.query_store_runtime_stats AS qsrs JOIN ' + @database_name_quoted + N'.sys.query_store_plan AS qsp - ON qsrs.plan_id = qsp.plan_id + ON qsrs.plan_id = qsp.plan_id JOIN ' + @database_name_quoted + N'.sys.query_store_query AS qsq - ON qsp.query_id = qsq.query_id + ON qsp.query_id = qsq.query_id WHERE EXISTS ( SELECT @@ -6631,9 +6637,10 @@ BEGIN FROM #query_store_query AS qsq2 WHERE qsq2.query_hash = qsq.query_hash ) - GROUP BY qsq.query_hash + GROUP BY + qsq.query_hash OPTION(RECOMPILE); -' +'; IF @debug = 1 BEGIN @@ -8268,8 +8275,8 @@ SELECT SELECT @sql += N' JOIN #query_hash_totals AS qht - ON qsq.query_hash = qht.query_hash - AND qsq.database_id = qht.database_id'; + ON qsq.query_hash = qht.query_hash + AND qsq.database_id = qht.database_id'; END; SELECT @@ -8279,7 +8286,7 @@ SELECT nvarchar(MAX), N' ) AS x -' + CASE WHEN @regression_mode = 1 THEN N' ' ELSE N'WHERE x.n = 1 ' END +' + CASE WHEN @regression_mode = 1 THEN N'' ELSE N'WHERE x.n = 1 ' END + N' ORDER BY ' + From e5e7f160361031ed05487b71a4eb93992beb2780 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 15 Mar 2025 12:48:17 -0400 Subject: [PATCH 130/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 607562e3..41f35349 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -75,8 +75,8 @@ BEGIN TRY END; SELECT - @version = '-2147483648', - @version_date = '17530101'; + @version = '1.0', + @version_date = '20250401'; SELECT for_insurance_purposes = N'Read the messages pane carefully!'; From 28e0ac4a1f6f65febe454776e2e2579c07bcc1f1 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 15 Mar 2025 13:00:43 -0400 Subject: [PATCH 131/246] bumping version dates etc point out the bounce! --- sp_HealthParser/sp_HealthParser.sql | 4 ++-- sp_HumanEvents/sp_HumanEvents.sql | 4 ++-- sp_HumanEvents/sp_HumanEventsBlockViewer.sql | 4 ++-- sp_IndexCleanup/sp_IndexCleanup.sql | 2 +- sp_LogHunter/sp_LogHunter.sql | 4 ++-- sp_PressureDetector/sp_PressureDetector.sql | 4 ++-- sp_QuickieStore/sp_QuickieStore.sql | 4 ++-- 7 files changed, 13 insertions(+), 13 deletions(-) diff --git a/sp_HealthParser/sp_HealthParser.sql b/sp_HealthParser/sp_HealthParser.sql index 4be9b955..4b41ccba 100644 --- a/sp_HealthParser/sp_HealthParser.sql +++ b/sp_HealthParser/sp_HealthParser.sql @@ -70,8 +70,8 @@ BEGIN SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; SELECT - @version = '2.1', - @version_date = '20250101'; + @version = '2.4', + @version_date = '20250401'; IF @help = 1 BEGIN diff --git a/sp_HumanEvents/sp_HumanEvents.sql b/sp_HumanEvents/sp_HumanEvents.sql index 6d2450a9..489f07c0 100644 --- a/sp_HumanEvents/sp_HumanEvents.sql +++ b/sp_HumanEvents/sp_HumanEvents.sql @@ -87,8 +87,8 @@ SET XACT_ABORT ON; SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; SELECT - @version = '6.1', - @version_date = '20250101'; + @version = '6.4', + @version_date = '20250401'; IF @help = 1 BEGIN diff --git a/sp_HumanEvents/sp_HumanEventsBlockViewer.sql b/sp_HumanEvents/sp_HumanEventsBlockViewer.sql index 66be377c..91b3908c 100644 --- a/sp_HumanEvents/sp_HumanEventsBlockViewer.sql +++ b/sp_HumanEvents/sp_HumanEventsBlockViewer.sql @@ -92,8 +92,8 @@ SET XACT_ABORT OFF; SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; SELECT - @version = '4.1', - @version_date = '20250101'; + @version = '4.4', + @version_date = '20250401'; IF @help = 1 BEGIN diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 41f35349..822c306f 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -75,7 +75,7 @@ BEGIN TRY END; SELECT - @version = '1.0', + @version = '1.4', @version_date = '20250401'; SELECT diff --git a/sp_LogHunter/sp_LogHunter.sql b/sp_LogHunter/sp_LogHunter.sql index 4de77f25..bf99cad6 100644 --- a/sp_LogHunter/sp_LogHunter.sql +++ b/sp_LogHunter/sp_LogHunter.sql @@ -72,8 +72,8 @@ SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; BEGIN SELECT - @version = '2.1', - @version_date = '20250101'; + @version = '2.4', + @version_date = '20250401'; IF @help = 1 BEGIN diff --git a/sp_PressureDetector/sp_PressureDetector.sql b/sp_PressureDetector/sp_PressureDetector.sql index d120d553..8a0709bb 100644 --- a/sp_PressureDetector/sp_PressureDetector.sql +++ b/sp_PressureDetector/sp_PressureDetector.sql @@ -76,8 +76,8 @@ SET XACT_ABORT ON; SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; SELECT - @version = '5.1', - @version_date = '20250101'; + @version = '5.4', + @version_date = '20250401'; IF @help = 1 diff --git a/sp_QuickieStore/sp_QuickieStore.sql b/sp_QuickieStore/sp_QuickieStore.sql index 94626da9..87a3640b 100644 --- a/sp_QuickieStore/sp_QuickieStore.sql +++ b/sp_QuickieStore/sp_QuickieStore.sql @@ -137,8 +137,8 @@ END; These are for your outputs. */ SELECT - @version = '5.1', - @version_date = '20250101'; + @version = '5.4', + @version_date = '20250401'; /* Helpful section! For help. From 782729e26138a64f3a463d7739c1388f958da843 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 15 Mar 2025 13:03:31 -0400 Subject: [PATCH 132/246] Update README.md --- README.md | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/README.md b/README.md index b54abdd8..1f35d602 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ - [sp_QuickieStore](#quickie-store): The fastest and most configurable way to navigate Query Store data - [sp_HealthParser](#health-parser): Pull all the performance-related data from the system health Extended Event - [sp_LogHunter](#log-hunter): Get all of the worst stuff out of your error log + - [sp_IndexCleanup](#index-cleanup): Identify unused and duplicate indexes ## Who are these scripts for? You need to troubleshoot performance problems with SQL Server, and you need to do it now. @@ -387,4 +388,36 @@ Current valid parameter details: [*Back to top*](#navigatory) +## Index Cleanup + +This stored procedure helps identify unused and duplicate indexes in your SQL Server databases that could be candidates for removal. It analyzes index usage statistics and can generate scripts for removing unnecessary indexes. + +**IMPORTANT: This is currently a BETA VERSION.** It needs extensive testing in real environments with real indexes to address several issues: +* Data collection accuracy +* Deduping logic +* Result correctness +* Edge cases + +Misuse of this procedure can potentially harm your database. If you run this, only use the output to validate result correctness. **Do not run any of the output scripts without thorough review and testing**, as doing so may be harmful to your database performance. + +The procedure requires SQL Server 2012 (11.0) or later due to the use of FORMAT and CONCAT functions. + +Current valid parameter details: + +| Parameter Name | Data Type | Default Value | Description | +|----------------|-----------|---------------|-------------| +| @database_name | sysname | NULL | The name of the database you wish to analyze | +| @schema_name | sysname | NULL | The schema name to filter indexes by | +| @table_name | sysname | NULL | The table name to filter indexes by | +| @min_reads | bigint | 0 | Minimum number of reads for an index to be considered used | +| @min_writes | bigint | 0 | Minimum number of writes for an index to be considered used | +| @min_size_gb | decimal(10,2) | 0 | Minimum size in GB for an index to be analyzed | +| @min_rows | bigint | 0 | Minimum number of rows for a table to be analyzed | +| @help | bit | 0 | Displays help information | +| @debug | bit | 0 | Prints debug information during execution | +| @version | varchar(20) | NULL | OUTPUT parameter that returns the version number of the procedure | +| @version_date | datetime | NULL | OUTPUT parameter that returns the date this version was released | + +[*Back to top*](#navigatory) + [licence badge]:https://img.shields.io/badge/license-MIT-blue.svg From c1a6d234ee07e0a605f27d8b7239163f9fbddd1b Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 15 Mar 2025 13:06:09 -0400 Subject: [PATCH 133/246] Update README.md --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 1f35d602..22f2488c 100644 --- a/README.md +++ b/README.md @@ -61,8 +61,8 @@ Current valid parameter details: | @minimum_disk_latency_ms | smallint | low bound for reporting disk latency | a reasonable number of milliseconds for disk latency | 100 | | @cpu_utilization_threshold | smallint | low bound for reporting high cpu utlization | a reasonable cpu utlization percentage | 50 | | @skip_waits | bit | skips waits when you do not need them on every run | 0 or 1 | 0 | -| @skip_perfmon | bit | skips perfmon counters when you do not need them on every run | 0 or 1 | 0 | -| @sample_seconds | tinyint | take a sample of your server's metrics | a valid tinyint: 0-255 | 0 | +| @skip_perfmon | bit | skips perfmon counters when you do not need them on every run | a valid tinyint: 0-255 | 0 | +| @sample_seconds | tinyint | take a sample of your server's metrics | 0 or 1 | 0 | | @log_to_table | bit | enable logging to permanent tables | 0 or 1 | 0 | | @log_database_name | sysname | database to store logging tables | valid database name | NULL | | @log_schema_name | sysname | schema to store logging tables | valid schema name | NULL | @@ -196,7 +196,7 @@ Current valid parameter details: | parameter_name | data_type | description | valid_inputs | defaults | |-----------------------|-----------|-------------------------------------------------|------------------------------------------------------------------------|------------------------------------| -| @session_name | nvarchar | name of the extended event session to pull from | extended event session name capturing sqlserver.blocked_process_report | keeper_HumanEvents_blocking | +| @session_name | sysname | name of the extended event session to pull from | extended event session name capturing sqlserver.blocked_process_report | keeper_HumanEvents_blocking | | @target_type | sysname | target of the extended event session | event_file or ring_buffer | NULL | | @start_date | datetime2 | filter by date | a reasonable date | NULL; will shortcut to last 7 days | | @end_date | datetime2 | filter by date | a reasonable date | NULL | @@ -296,6 +296,7 @@ Current valid parameter details: | @regression_baseline_end_date | datetimeoffset | the end date of the baseline that you are checking for regressions against (if any), will be converted to UTC internally | January 1, 1753, through December 31, 9999 | NULL; One week after @regression_baseline_start_date if that is specified | | @regression_comparator | varchar | what difference to use ('relative' or 'absolute') when comparing @sort_order's metric for the normal time period with any regression time period. | relative, absolute | NULL; absolute if @regression_baseline_start_date is specified | | @regression_direction | varchar | when comparing against any regression baseline, what do you want the results sorted by ('magnitude', 'improved', or 'regressed')? | regressed, worse, improved, better, magnitude, absolute, whatever | NULL; regressed if @regression_baseline_start_date is specified | +| @include_query_hash_totals | bit | will add an additional column to final output with total resource usage by query hash | 0 or 1 | 0 | | @help | bit | how you got here | 0 or 1 | 0 | | @debug | bit | prints dynamic sql, statement length, parameter and variable values, and raw temp table contents | 0 or 1 | 0 | | @troubleshoot_performance | bit | set statistics xml on for queries against views | 0 or 1 | 0 | From d1c7768e095871cf818df70d0173a320863c2662 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 15 Mar 2025 13:15:01 -0400 Subject: [PATCH 134/246] Update CLAUDE.md --- CLAUDE.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 94e91165..e33fda66 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,21 +1,25 @@ # Erik Darling's T-SQL Coding Style Guide -This document outlines the T-SQL coding style preferences for Erik Darling (Darling Data, LLC) and should be followed when writing or modifying SQL code. +This document outlines the T-SQL coding style preferences for Erik Darling (Darling Data, LLC) and must be strictly followed when writing or modifying SQL code. ## General Formatting - **Keywords**: All SQL keywords in UPPERCASE (SELECT, FROM, WHERE, JOIN, etc.) - **Functions**: All SQL functions in UPPERCASE (CONVERT, ISNULL, OBJECT_ID, etc.) +- **Data types**: + - Never abbreviate data types (use INTEGER instead of INT) + - All data types must be lowercase (varchar, nvarchar, datetime2, bigint, etc.) + - Length specifications must also be lowercase: nvarchar(max), not nvarchar(MAX) + - Precision and scale specifications must be lowercase: decimal(38,2), not DECIMAL(38,2) +- **Keywords**: Never abbreviate keywords (use EXECUTE instead of EXEC, TRANSACTION instead of TRAN, PROCEDURE instead of PROC) - **Indentation**: 4 spaces for each level of indentation (NEVER use tabs) - **Line breaks**: Each statement on a new line - **Spacing**: Consistent spacing around operators (=, <, >, etc.) - **Block separation**: Empty line between logical code blocks (maximum of two empty lines between statements) - **Quotes**: Use single quotes for string literals and N-prefix for Unicode strings (N'string') -- **Data types**: Never abbreviate data types (use INTEGER instead of INT) -- **Keywords**: Never abbreviate keywords (use EXECUTE instead of EXEC, TRANSACTION instead of TRAN, PROCEDURE instead of PROC) - **TOP syntax**: Always include parentheses, as in TOP (100) not TOP 100 - **Object creation**: Generally use CREATE OR ALTER for objects instead of DROP/CREATE -- **Table aliases**: Tables should always have aliases, even in simple queries +- **Table aliases**: Tables must always have aliases, even in simple queries - **Column references**: Always qualify columns with their table alias ## Comments @@ -524,7 +528,7 @@ BEGIN Variable declarations */ DECLARE - @sql nvarchar(MAX) = N'', + @sql nvarchar(max) = N'', @database_id integer = NULL; /* @@ -574,4 +578,4 @@ END; GO ``` -This style guide is based on an analysis of Erik Darling's stored procedures from Darling Data, LLC. +This style guide is based on an analysis of Erik Darling's stored procedures from Darling Data, LLC. \ No newline at end of file From be8f3c6d27af7feaa8be9b2aaf952d16c49ac3cf Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 15 Mar 2025 13:19:52 -0400 Subject: [PATCH 135/246] Update CLAUDE.md --- CLAUDE.md | 252 ++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 235 insertions(+), 17 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index e33fda66..9b94d960 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -497,7 +497,146 @@ BEGIN END; ``` -## Example +## Examples + +### Complex SELECT with Multiple JOINs, GROUP BY, and HAVING + +```sql +SELECT + database_name = d.name, + index_count = COUNT(i.index_id), + total_size_mb = SUM(a.total_pages) * 8 / 1024, + read_operations = SUM(ius.user_seeks + ius.user_scans + ius.user_lookups), + write_operations = SUM(ius.user_updates), + avg_fragmentation = AVG(ps.avg_fragmentation_in_percent) +FROM sys.databases AS d +JOIN sys.tables AS t + ON t.database_id = d.database_id +LEFT JOIN sys.indexes AS i + ON i.object_id = t.object_id + AND i.index_id > 0 + AND i.is_disabled = 0 +LEFT JOIN sys.dm_db_index_usage_stats AS ius + ON ius.database_id = d.database_id + AND ius.object_id = i.object_id + AND ius.index_id = i.index_id +LEFT JOIN sys.dm_db_index_physical_stats(DB_ID(), NULL, NULL, NULL, 'LIMITED') AS ps + ON ps.object_id = i.object_id + AND ps.index_id = i.index_id +LEFT JOIN sys.allocation_units AS a + ON a.container_id = i.hobt_id +WHERE d.database_id > 4 +AND d.is_read_only = 0 +AND d.state_desc = N'ONLINE' +GROUP BY + d.name, + d.create_date +HAVING + COUNT(i.index_id) > 10 +ORDER BY + total_size_mb DESC, + database_name ASC +OPTION(MAXDOP 1, RECOMPILE); +``` + +### CTE with Multiple Definitions and Nested Queries + +```sql +WITH database_stats +( + database_name, + recovery_model, + log_size_mb, + log_used_percent +) +AS +( + SELECT + database_name = d.name, + recovery_model = d.recovery_model_desc, + log_size_mb = SUM(CASE WHEN f.type_desc = N'LOG' THEN f.size END) * 8 / 1024, + log_used_percent = SUM(CASE WHEN f.type_desc = N'LOG' THEN CONVERT(decimal(19,2), fileproperty(f.name, 'SpaceUsed')) / f.size * 100 END) + FROM sys.databases AS d + JOIN sys.master_files AS f + ON f.database_id = d.database_id + WHERE d.state_desc = N'ONLINE' + GROUP BY + d.name, + d.recovery_model_desc +), +database_backups AS +( + SELECT + database_name = b.database_name, + last_full_backup = MAX(CASE WHEN b.type = 'D' THEN b.backup_finish_date END), + last_log_backup = MAX(CASE WHEN b.type = 'L' THEN b.backup_finish_date END) + FROM msdb.dbo.backupset AS b + WHERE b.backup_finish_date > DATEADD(DAY, -7, GETDATE()) + GROUP BY + b.database_name +) + +SELECT + ds.database_name, + ds.recovery_model, + ds.log_size_mb, + ds.log_used_percent, + days_since_full_backup = + CASE + WHEN db.last_full_backup IS NULL + THEN 999 + ELSE DATEDIFF(DAY, db.last_full_backup, GETDATE()) + END, + days_since_log_backup = + CASE + WHEN db.last_log_backup IS NULL + THEN 999 + ELSE DATEDIFF(DAY, db.last_log_backup, GETDATE()) + END +FROM database_stats AS ds +LEFT JOIN database_backups AS db + ON db.database_name = ds.database_name +WHERE ds.log_size_mb > 100 +ORDER BY + log_size_mb DESC; +``` + +### Dynamic SQL Generation and Execution + +```sql +DECLARE + @database_name sysname = N'AdventureWorks', + @table_name sysname = N'SalesOrderHeader', + @column_name sysname = N'OrderDate', + @sql nvarchar(max) = N''; + +/* +Build query dynamically using proper quoting and formatting +*/ +SET @sql = N' +SELECT + order_month = DATEFROMPARTS(YEAR(' + QUOTENAME(@column_name) + N'), MONTH(' + QUOTENAME(@column_name) + N'), 1), + order_count = COUNT(*), + total_amount = SUM(TotalDue), + avg_amount = AVG(TotalDue) +FROM ' + QUOTENAME(@database_name) + N'.dbo.' + QUOTENAME(@table_name) + N' +WHERE ' + QUOTENAME(@column_name) + N' >= DATEADD(YEAR, -1, GETDATE()) +GROUP BY + DATEFROMPARTS(YEAR(' + QUOTENAME(@column_name) + N'), MONTH(' + QUOTENAME(@column_name) + N'), 1) +ORDER BY + order_month; +'; + +/* +Execute the dynamic SQL with proper parameter passing +*/ +EXECUTE sys.sp_executesql + @sql, + N'', + N''; +``` + +### Stored Procedure with Temp Tables and Flow Control ```sql SET ANSI_NULLS ON; @@ -514,6 +653,8 @@ ALTER PROCEDURE dbo.sp_MyProcedure ( @database_name sysname = NULL, /*the database to analyze*/ + @days_back integer = 7, /*how many days of history to analyze*/ + @threshold_percent integer = 20, /*minimum percentage change to report*/ @debug bit = 0, /*prints additional diagnostic information*/ @help bit = 0 /*prints help information*/ ) @@ -529,7 +670,9 @@ BEGIN */ DECLARE @sql nvarchar(max) = N'', - @database_id integer = NULL; + @database_id integer = NULL, + @start_date datetime2(7) = DATEADD(DAY, -@days_back, GETDATE()), + @error_msg nvarchar(2048) = N''; /* Parameter validation @@ -540,38 +683,113 @@ BEGIN @database_name = DB_NAME(); END; + IF @threshold_percent <= 0 OR @threshold_percent > 100 + BEGIN + SELECT + @error_msg = N'@threshold_percent must be between 1 and 100.'; + + RAISERROR(@error_msg, 16, 1); + RETURN; + END; + /* Help section */ IF @help = 1 BEGIN SELECT - help = N'This procedure analyzes database objects'; + help = N'This procedure analyzes database performance changes'; RETURN; END; /* - Main processing logic + Create temp tables for analysis */ - SELECT + CREATE TABLE + #baseline_metrics + ( + object_id bigint NOT NULL, + metric_name varchar(50) NOT NULL, + metric_value decimal(38,2) NOT NULL + ); + + CREATE TABLE + #current_metrics + ( + object_id bigint NOT NULL, + metric_name varchar(50) NOT NULL, + metric_value decimal(38,2) NOT NULL + ); + + /* + Populate baseline data + */ + INSERT + #baseline_metrics + WITH + (TABLOCK) + ( object_id, - object_name = o.name, - schema_name = s.name - FROM dbo.objects AS o - JOIN dbo.schemas AS s - ON o.schema_id = s.schema_id - WHERE o.type = N'U' - AND o.is_ms_shipped = 0 + metric_name, + metric_value + ) + SELECT + object_id = t.object_id, + metric_name = 'query_cost', + metric_value = AVG(qs.total_elapsed_time / 1000.0) + FROM sys.dm_exec_query_stats AS qs + CROSS APPLY sys.dm_exec_sql_text(qs.sql_handle) AS t + WHERE qs.creation_time < @start_date + AND t.dbid = DB_ID(@database_name) GROUP BY - o.object_id, - o.name, - s.name + t.object_id; + + IF @debug = 1 + BEGIN + SELECT + baseline_rows = COUNT(*) + FROM #baseline_metrics; + END; + + /* + Main processing logic - analyze changes + */ + SELECT + object_name = o.name, + schema_name = s.name, + b.metric_name, + baseline_value = b.metric_value, + current_value = c.metric_value, + percent_change = + CASE + WHEN b.metric_value = 0 + THEN NULL + ELSE (c.metric_value - b.metric_value) / b.metric_value * 100 + END + FROM #baseline_metrics AS b + JOIN #current_metrics AS c + ON c.object_id = b.object_id + AND c.metric_name = b.metric_name + JOIN sys.objects AS o + ON o.object_id = b.object_id + JOIN sys.schemas AS s + ON s.schema_id = o.schema_id + WHERE ABS((c.metric_value - b.metric_value) / NULLIF(b.metric_value, 0) * 100) >= @threshold_percent ORDER BY - o.name - OPTION(RECOMPILE); + ABS((c.metric_value - b.metric_value) / NULLIF(b.metric_value, 0) * 100) DESC; END TRY BEGIN CATCH + IF OBJECT_ID('tempdb..#baseline_metrics') IS NOT NULL + BEGIN + DROP TABLE #baseline_metrics; + END; + + IF OBJECT_ID('tempdb..#current_metrics') IS NOT NULL + BEGIN + DROP TABLE #current_metrics; + END; + THROW; END CATCH; END; From 0b94bb625a7e4616fdda25637b577fb27206b592 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 15 Mar 2025 13:32:32 -0400 Subject: [PATCH 136/246] Update CLAUDE.md --- CLAUDE.md | 84 +++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 54 insertions(+), 30 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 9b94d960..6ec43d12 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -147,13 +147,13 @@ This document outlines the T-SQL coding style preferences for Erik Darling (Darl - UNION, INTERSECT, EXCEPT should have the operator between statements with blank lines ```sql SELECT - columns + a.columns FROM dbo.a_table AS a EXCEPT SELECT - columns + b.columns FROM dbo.b_table AS b; ``` @@ -260,10 +260,10 @@ This document outlines the T-SQL coding style preferences for Erik Darling (Darl UPDATE alias SET - col1 = value1, - col2 = value2 + alias.col1 = value1, + alias.col2 = value2 FROM dbo.table AS alias - WHERE condition; + WHERE alias.condition; ``` - **DELETE statements**: @@ -274,7 +274,7 @@ This document outlines the T-SQL coding style preferences for Erik Darling (Darl DELETE alias FROM dbo.table AS alias - WHERE condition; + WHERE alias.condition; ``` - **Parentheses**: @@ -376,7 +376,7 @@ This document outlines the T-SQL coding style preferences for Erik Darling (Darl @sql nvarchar(max) = N'' SET @sql += N' - the query ' + QUOTENAME(object_name) + ' + the query ' + QUOTENAME(alias.object_name) + ' '; EXECUTE sys.sp_executesql @@ -504,7 +504,7 @@ END; ```sql SELECT database_name = d.name, - index_count = COUNT(i.index_id), + index_count = COUNT_BIG(i.index_id), total_size_mb = SUM(a.total_pages) * 8 / 1024, read_operations = SUM(ius.user_seeks + ius.user_scans + ius.user_lookups), write_operations = SUM(ius.user_updates), @@ -520,7 +520,14 @@ LEFT JOIN sys.dm_db_index_usage_stats AS ius ON ius.database_id = d.database_id AND ius.object_id = i.object_id AND ius.index_id = i.index_id -LEFT JOIN sys.dm_db_index_physical_stats(DB_ID(), NULL, NULL, NULL, 'LIMITED') AS ps +LEFT JOIN sys.dm_db_index_physical_stats +( + DB_ID(), + NULL, + NULL, + NULL, + 'LIMITED' +) AS ps ON ps.object_id = i.object_id AND ps.index_id = i.index_id LEFT JOIN sys.allocation_units AS a @@ -542,14 +549,14 @@ OPTION(MAXDOP 1, RECOMPILE); ### CTE with Multiple Definitions and Nested Queries ```sql -WITH database_stats +WITH + database_stats ( database_name, recovery_model, log_size_mb, log_used_percent -) -AS +) AS ( SELECT database_name = d.name, @@ -564,7 +571,7 @@ AS d.name, d.recovery_model_desc ), -database_backups AS + database_backups AS ( SELECT database_name = b.database_name, @@ -575,7 +582,6 @@ database_backups AS GROUP BY b.database_name ) - SELECT ds.database_name, ds.recovery_model, @@ -615,14 +621,37 @@ Build query dynamically using proper quoting and formatting */ SET @sql = N' SELECT - order_month = DATEFROMPARTS(YEAR(' + QUOTENAME(@column_name) + N'), MONTH(' + QUOTENAME(@column_name) + N'), 1), - order_count = COUNT(*), + order_month = + DATEFROMPARTS + ( + YEAR + (' + + QUOTENAME(@column_name) + + N'), + MONTH + (' + + QUOTENAME(@column_name) + + N'), + 1 + ), + order_count = COUNT_BIG(*), total_amount = SUM(TotalDue), avg_amount = AVG(TotalDue) FROM ' + QUOTENAME(@database_name) + N'.dbo.' + QUOTENAME(@table_name) + N' WHERE ' + QUOTENAME(@column_name) + N' >= DATEADD(YEAR, -1, GETDATE()) GROUP BY - DATEFROMPARTS(YEAR(' + QUOTENAME(@column_name) + N'), MONTH(' + QUOTENAME(@column_name) + N'), 1) + DATEFROMPARTS + ( + YEAR + (' + + QUOTENAME(@column_name) + + N'), + MONTH + (' + + QUOTENAME(@column_name) + + N'), + 1 + ) ORDER BY order_month; '; @@ -643,9 +672,9 @@ SET ANSI_NULLS ON; SET QUOTED_IDENTIFIER ON; GO -IF OBJECT_ID('dbo.sp_MyProcedure', 'P') IS NULL +IF OBJECT_ID(N'dbo.sp_MyProcedure', N'P') IS NULL BEGIN - EXECUTE ('CREATE PROCEDURE dbo.sp_MyProcedure AS RETURN 0;'); + EXECUTE(N'CREATE PROCEDURE dbo.sp_MyProcedure AS RETURN 138;'); END; GO @@ -743,7 +772,8 @@ BEGIN WHERE qs.creation_time < @start_date AND t.dbid = DB_ID(@database_name) GROUP BY - t.object_id; + t.object_id + OPTION(RECOMPILE); IF @debug = 1 BEGIN @@ -780,20 +810,14 @@ BEGIN ABS((c.metric_value - b.metric_value) / NULLIF(b.metric_value, 0) * 100) DESC; END TRY BEGIN CATCH - IF OBJECT_ID('tempdb..#baseline_metrics') IS NOT NULL - BEGIN - DROP TABLE #baseline_metrics; - END; - - IF OBJECT_ID('tempdb..#current_metrics') IS NOT NULL + IF @@TRANCOUNT > 0 BEGIN - DROP TABLE #current_metrics; - END; - + ROLLBACK; + END; THROW; END CATCH; END; GO ``` -This style guide is based on an analysis of Erik Darling's stored procedures from Darling Data, LLC. \ No newline at end of file +This style guide is based on an analysis of Erik Darling's stored procedures from Darling Data, LLC. From dd990033f90146d93c2e181eceada1624a39faea Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 15 Mar 2025 13:34:52 -0400 Subject: [PATCH 137/246] Update CLAUDE.md --- CLAUDE.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 6ec43d12..08489658 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -625,29 +625,29 @@ SELECT DATEFROMPARTS ( YEAR - (' + + (t.' + QUOTENAME(@column_name) + N'), MONTH - (' + + (t.' + QUOTENAME(@column_name) + N'), 1 ), order_count = COUNT_BIG(*), - total_amount = SUM(TotalDue), - avg_amount = AVG(TotalDue) -FROM ' + QUOTENAME(@database_name) + N'.dbo.' + QUOTENAME(@table_name) + N' + total_amount = SUM(t.TotalDue), + avg_amount = AVG(t.TotalDue) +FROM ' + QUOTENAME(@database_name) + N'.dbo.' + QUOTENAME(@table_name) + N' AS t WHERE ' + QUOTENAME(@column_name) + N' >= DATEADD(YEAR, -1, GETDATE()) GROUP BY DATEFROMPARTS ( YEAR - (' + + (t.' + QUOTENAME(@column_name) + N'), MONTH - (' + + (t.' + QUOTENAME(@column_name) + N'), 1 From e4bc7d3ebe892037f1049b9b5c9d5ade6302014a Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 15 Mar 2025 13:35:10 -0400 Subject: [PATCH 138/246] Update CLAUDE.md --- CLAUDE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CLAUDE.md b/CLAUDE.md index 9b94d960..ecae4809 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -411,6 +411,7 @@ This document outlines the T-SQL coding style preferences for Erik Darling (Darl - Use semicolons at the end of statements (but only at the very end, after any query hints) - Apply query hints consistently (RECOMPILE, MAXDOP, etc.) - Always use ROWCOUNT_BIG() instead of @@ROWCOUNT +- Always use COUNT_BIG() instead of COUNT() to avoid potential integer overflow - Always use CONVERT over CAST for data type conversions (except when using TRY_CAST, as TRY_CAST isn't dependent on SQL Server version) - Use XML for string splitting and string building (concatenation), as these methods aren't dependent on SQL Server version or database compatibility level - Always use cursor variables instead of normal cursors, as they don't require explicit CLOSE/DEALLOCATE statements From d29ff73e21e5f5dfe126cc06efe03621929471f0 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 15 Mar 2025 13:39:18 -0400 Subject: [PATCH 139/246] Update CLAUDE.md --- CLAUDE.md | 63 +++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 59 insertions(+), 4 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 6b3222bc..6b932363 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -168,10 +168,41 @@ This document outlines the T-SQL coding style preferences for Erik Darling (Darl ``` - **CTEs**: - - WITH keyword on first line - - CTE name aligned with leading whitespace - - CTE column list indented from CTE name + - WITH keyword on its own line + - CTE name indented on next line + - Opening parenthesis on same line as CTE name + - Column list indented on subsequent lines + - Closing parenthesis on its own line + - AS keyword on its own line - Multiple CTEs separated by commas at the end + ```sql + WITH + database_stats + ( + database_name, + recovery_model, + log_size_mb + ) AS + ( + SELECT + database_name = d.name, + recovery_model = d.recovery_model_desc, + log_size_mb = SUM(f.size) * 8 / 1024 + FROM sys.databases AS d + JOIN sys.master_files AS f + ON f.database_id = d.database_id + GROUP BY + d.name, + d.recovery_model_desc + ), + second_cte + ( + column_list + ) AS + ( + query + ) + ``` - **Table Creation**: - CREATE TABLE on first line @@ -289,6 +320,20 @@ This document outlines the T-SQL coding style preferences for Erik Darling (Darl value ) ``` + +- **Multi-parameter functions**: + - For functions with multiple parameters or complex expressions, format the function name on its own line + - Place parameters on subsequent lines with proper indentation + ```sql + SELECT + formatted_date = + DATEFROMPARTS + ( + YEAR(date_column), + MONTH(date_column), + 1 + ) + ``` ## Code Organization @@ -371,12 +416,20 @@ This document outlines the T-SQL coding style preferences for Erik Darling (Darl - Take care when initializing to ensure you don't introduce logical flaws with NULL checks - Dynamic SQL should follow specific formatting: + - Initial declaration with empty string + - Each string concatenation part on its own line + - Each QUOTENAME or variable reference on its own line ```sql DECLARE @sql nvarchar(max) = N'' SET @sql += N' - the query ' + QUOTENAME(alias.object_name) + ' +SELECT + column_name = + value ' + + QUOTENAME(alias.object_name) + N' +FROM + table_name '; EXECUTE sys.sp_executesql @@ -412,6 +465,8 @@ This document outlines the T-SQL coding style preferences for Erik Darling (Darl - Apply query hints consistently (RECOMPILE, MAXDOP, etc.) - Always use ROWCOUNT_BIG() instead of @@ROWCOUNT - Always use COUNT_BIG() instead of COUNT() to avoid potential integer overflow + - Example: `COUNT_BIG(i.index_id)` not `COUNT(i.index_id)` + - Even if the result will never be large enough to overflow, use COUNT_BIG() for consistency - Always use CONVERT over CAST for data type conversions (except when using TRY_CAST, as TRY_CAST isn't dependent on SQL Server version) - Use XML for string splitting and string building (concatenation), as these methods aren't dependent on SQL Server version or database compatibility level - Always use cursor variables instead of normal cursors, as they don't require explicit CLOSE/DEALLOCATE statements From 457c086898d6d0d73f49d20e51643e3e475d6fc5 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 15 Mar 2025 21:17:48 -0400 Subject: [PATCH 140/246] pretty pictures pretty pictures --- Clear Token Perm/README.md | 2 + Helper Views/README.md | 2 + Install-All/README.md | 2 + Ola Stats Only Job/README.md | 2 + README.md | 2 + String Functions/README.md | 2 + sp_HealthParser/README.md | 2 + sp_HumanEvents/README.md | 2 + sp_IndexCleanup/README.md | 2 + sp_IndexCleanup/sp_IndexCleanup.sql | 62 +++++++++++++++++++++++++++++ sp_LogHunter/README.md | 2 + sp_PressureDetector/README.md | 2 + sp_QuickieStore/README.md | 2 + sp_WhoIsActive Logging/README.md | 2 + 14 files changed, 88 insertions(+) diff --git a/Clear Token Perm/README.md b/Clear Token Perm/README.md index bb0b0838..50875907 100644 --- a/Clear Token Perm/README.md +++ b/Clear Token Perm/README.md @@ -1,3 +1,5 @@ + + # Clear Token Perm This directory contains scripts for monitoring and managing SQL Server's security token cache. The security token cache (TokenAndPermUserStore) can grow to a significant size in certain scenarios, potentially causing high memory usage and performance issues. diff --git a/Helper Views/README.md b/Helper Views/README.md index 4684000b..2ddec631 100644 --- a/Helper Views/README.md +++ b/Helper Views/README.md @@ -1,3 +1,5 @@ + + # Helper Views This directory contains helper views and functions for diagnosing various SQL Server performance issues. These scripts are particularly useful for presentations and educational purposes, but can also be used in troubleshooting scenarios. diff --git a/Install-All/README.md b/Install-All/README.md index 5768466f..f5cb7ff8 100644 --- a/Install-All/README.md +++ b/Install-All/README.md @@ -1,3 +1,5 @@ + + # Install-All This directory contains tools to merge all the Darling Data stored procedures into a single installation file. diff --git a/Ola Stats Only Job/README.md b/Ola Stats Only Job/README.md index dcb1ecbd..c510cfa6 100644 --- a/Ola Stats Only Job/README.md +++ b/Ola Stats Only Job/README.md @@ -1,3 +1,5 @@ + + # Ola Stats Only Job This directory contains a script to create a SQL Server Agent job for nightly statistics updates using Ola Hallengren's maintenance solution. diff --git a/README.md b/README.md index 22f2488c..8a27addb 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +# Darling Data Logo + # Darling Data: SQL Server Troubleshooting Scripts ![licence badge] diff --git a/String Functions/README.md b/String Functions/README.md index be06a53d..c016e946 100644 --- a/String Functions/README.md +++ b/String Functions/README.md @@ -1,3 +1,5 @@ + + # String Functions This directory contains a set of utility functions for string manipulation in SQL Server. These functions provide efficient ways to extract or remove specific characters from strings. diff --git a/sp_HealthParser/README.md b/sp_HealthParser/README.md index 173eafc0..6b65ee47 100644 --- a/sp_HealthParser/README.md +++ b/sp_HealthParser/README.md @@ -1,3 +1,5 @@ + + # sp_HealthParser The system health extended event has been around for a while, hiding in the shadows, and collecting all sorts of crazy information about your SQL Server. diff --git a/sp_HumanEvents/README.md b/sp_HumanEvents/README.md index a2b78487..27591218 100644 --- a/sp_HumanEvents/README.md +++ b/sp_HumanEvents/README.md @@ -1,3 +1,5 @@ + + # Human Events Toolkit This directory contains two stored procedures for managing and analyzing Extended Events in SQL Server: diff --git a/sp_IndexCleanup/README.md b/sp_IndexCleanup/README.md index c7a64c1c..cc8e2432 100644 --- a/sp_IndexCleanup/README.md +++ b/sp_IndexCleanup/README.md @@ -1,3 +1,5 @@ + + # sp_IndexCleanup ## Overview diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 822c306f..9c82ee68 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -4264,6 +4264,68 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND ia.index_id = ce.index_id OPTION(RECOMPILE); + /* Return enhanced database impact summaries */ + IF @debug = 1 + BEGIN + RAISERROR('Generating enhanced summary reports', 0, 0) WITH NOWAIT; + END; + + /* + Enhanced Database Impact Summary - Shows total space and performance savings + */ + SELECT + summary_type = 'DATABASE IMPACT SUMMARY', + database_name = irs.database_name, + total_indexes = FORMAT(irs.index_count, 'N0'), + indexes_to_disable = FORMAT(irs.indexes_to_disable, 'N0'), + indexes_to_merge = FORMAT(irs.indexes_to_merge, 'N0'), + percent_reduction = FORMAT(((irs.indexes_to_disable + irs.indexes_to_merge) / NULLIF(CONVERT(DECIMAL(10,2), irs.index_count), 0)) * 100, 'N1') + '%', + current_size_gb = FORMAT(irs.total_size_gb, 'N2'), + space_saved_gb = FORMAT(irs.space_saved_gb, 'N2'), + size_reduction_percent = FORMAT((irs.space_saved_gb / NULLIF(irs.total_size_gb, 0)) * 100, 'N1') + '%', + write_operations_saved = FORMAT(irs.user_updates, 'N0'), + lock_operations_saved = FORMAT(irs.row_lock_count + irs.page_lock_count, 'N0'), + latch_operations_saved = FORMAT(irs.page_latch_wait_count + irs.page_io_latch_wait_count, 'N0') + FROM #index_reporting_stats AS irs + WHERE irs.summary_level = 'DATABASE' + ORDER BY irs.database_name; + + /* + Table-Level Impact - Top tables by space savings + */ + SELECT TOP(10) + summary_type = 'TOP TABLES BY IMPACT', + table_name = QUOTENAME(irs.schema_name) + '.' + QUOTENAME(irs.table_name), + total_indexes = FORMAT(irs.index_count, 'N0'), + removable_indexes = FORMAT(irs.unused_indexes, 'N0'), + mergeable_indexes = FORMAT(irs.indexes_to_merge, 'N0'), + current_size_gb = FORMAT(irs.total_size_gb, 'N2'), + space_saved_gb = FORMAT(irs.unused_size_gb, 'N2'), + percent_reduction = FORMAT((irs.unused_size_gb / NULLIF(irs.total_size_gb, 0)) * 100, 'N1') + '%', + total_write_ops = FORMAT(irs.user_updates, 'N0'), + write_ops_per_day = FORMAT(irs.user_updates / NULLIF(CONVERT(DECIMAL(10,2), + (SELECT TOP 1 server_uptime_days FROM #index_reporting_stats WHERE summary_level = 'DATABASE')), 0), 'N0') + FROM #index_reporting_stats AS irs + WHERE irs.summary_level = 'TABLE' + ORDER BY irs.unused_size_gb DESC; + + /* + Before/After Comparison for Database + */ + SELECT + comparison_metric = 'BEFORE/AFTER COMPARISON', + total_indexes_before = FORMAT(irs.index_count, 'N0'), + total_indexes_after = FORMAT(irs.index_count - (irs.indexes_to_disable + irs.indexes_to_merge), 'N0'), + index_reduction = FORMAT(((irs.indexes_to_disable + irs.indexes_to_merge) / NULLIF(CONVERT(DECIMAL(10,2), irs.index_count), 0)) * 100, 'N1') + '%', + size_before_gb = FORMAT(irs.total_size_gb, 'N2'), + size_after_gb = FORMAT(irs.total_size_gb - irs.space_saved_gb, 'N2'), + space_saved_gb = FORMAT(irs.space_saved_gb, 'N2'), + percent_space_saved = FORMAT((irs.space_saved_gb / NULLIF(irs.total_size_gb, 0)) * 100, 'N1') + '%', + daily_write_ops_saved = FORMAT(irs.user_updates / NULLIF(CONVERT(DECIMAL(10,2), irs.server_uptime_days), 0) * + ((irs.indexes_to_disable + irs.indexes_to_merge) / NULLIF(CONVERT(DECIMAL(10,2), irs.index_count), 0)), 'N0') + FROM #index_reporting_stats AS irs + WHERE irs.summary_level = 'DATABASE'; + /* Return streamlined reporting statistics focused on key metrics */ IF @debug = 1 BEGIN diff --git a/sp_LogHunter/README.md b/sp_LogHunter/README.md index 4e19d651..9fde86cd 100644 --- a/sp_LogHunter/README.md +++ b/sp_LogHunter/README.md @@ -1,3 +1,5 @@ + + # sp_LogHunter The SQL Server error log can have a lot of good information in it about what's going on, whether it's right or wrong. diff --git a/sp_PressureDetector/README.md b/sp_PressureDetector/README.md index 5784d14c..14f39294 100644 --- a/sp_PressureDetector/README.md +++ b/sp_PressureDetector/README.md @@ -1,3 +1,5 @@ + + # sp_PressureDetector Is your client/server relationship on the rocks? Are queries timing out, dragging along, or causing CPU fans to spin out of control? diff --git a/sp_QuickieStore/README.md b/sp_QuickieStore/README.md index e6dfe551..5d2d614f 100644 --- a/sp_QuickieStore/README.md +++ b/sp_QuickieStore/README.md @@ -1,3 +1,5 @@ + + # sp_QuickieStore This procedure will dig into Query Store data for a specific database, or all databases with Query Store enabled. diff --git a/sp_WhoIsActive Logging/README.md b/sp_WhoIsActive Logging/README.md index 6e999cdf..23d63f65 100644 --- a/sp_WhoIsActive Logging/README.md +++ b/sp_WhoIsActive Logging/README.md @@ -1,3 +1,5 @@ + + # sp_WhoIsActive Logging This toolkit automates the collection and management of SQL Server activity data using Adam Machanic's popular sp_WhoIsActive stored procedure. It creates a comprehensive logging framework that captures server activity in daily tables and provides useful views for analysis. From 28179a2342ec46d8df60e51251cc812215ab6342 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 15 Mar 2025 21:24:42 -0400 Subject: [PATCH 141/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 9c82ee68..9ff6758a 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -3209,8 +3209,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. THEN ', DATA_COMPRESSION = PAGE' ELSE N'' END + - N')' + - N'Compression type: All Partitions', + N')', + additional_info = N'Compression type: All Partitions', superseded_info = NULL, /* No target index for compression scripts */ ia.superseded_by, /* Include superseded_by info for compression scripts */ /* Original index definition for validation */ From 4ef6eb5bd0d007a0fbfab93ef05270a205494280 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 15 Mar 2025 21:33:03 -0400 Subject: [PATCH 142/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 77 +++++++++++++++-------------- 1 file changed, 40 insertions(+), 37 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 9ff6758a..bd8d6897 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -4271,21 +4271,36 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END; /* - Enhanced Database Impact Summary - Shows total space and performance savings + Consolidated Database Impact Summary - Shows all metrics in one place */ SELECT - summary_type = 'DATABASE IMPACT SUMMARY', - database_name = irs.database_name, + report_section = 'DATABASE IMPACT SUMMARY', + server_uptime = CASE + WHEN irs.server_uptime_days < 7 + THEN 'WARNING: Server uptime only ' + FORMAT(irs.server_uptime_days, 'N0') + ' days - usage data may be incomplete!' + ELSE FORMAT(irs.server_uptime_days, 'N0') + ' days' + END, + database_name = ISNULL(irs.database_name, 'N/A'), + /* Index counts */ total_indexes = FORMAT(irs.index_count, 'N0'), + indexes_after_cleanup = FORMAT(irs.index_count - (irs.indexes_to_disable + irs.indexes_to_merge), 'N0'), indexes_to_disable = FORMAT(irs.indexes_to_disable, 'N0'), indexes_to_merge = FORMAT(irs.indexes_to_merge, 'N0'), - percent_reduction = FORMAT(((irs.indexes_to_disable + irs.indexes_to_merge) / NULLIF(CONVERT(DECIMAL(10,2), irs.index_count), 0)) * 100, 'N1') + '%', + /* Space metrics */ current_size_gb = FORMAT(irs.total_size_gb, 'N2'), + size_after_cleanup_gb = FORMAT(irs.total_size_gb - irs.space_saved_gb, 'N2'), space_saved_gb = FORMAT(irs.space_saved_gb, 'N2'), - size_reduction_percent = FORMAT((irs.space_saved_gb / NULLIF(irs.total_size_gb, 0)) * 100, 'N1') + '%', - write_operations_saved = FORMAT(irs.user_updates, 'N0'), - lock_operations_saved = FORMAT(irs.row_lock_count + irs.page_lock_count, 'N0'), - latch_operations_saved = FORMAT(irs.page_latch_wait_count + irs.page_io_latch_wait_count, 'N0') + /* Performance metrics */ + total_rows = FORMAT(irs.total_rows, 'N0'), + write_operations_saved = FORMAT(ISNULL(irs.user_updates, 0), 'N0'), + daily_write_ops_saved = FORMAT(ISNULL(irs.user_updates / NULLIF(CONVERT(DECIMAL(10,2), irs.server_uptime_days), 0) * + ((irs.indexes_to_disable + irs.indexes_to_merge) / NULLIF(CONVERT(DECIMAL(10,2), irs.index_count), 0)), 0), 'N0'), + /* Percentage metrics for quick understanding */ + index_reduction_pct = FORMAT(((irs.indexes_to_disable + irs.indexes_to_merge) / NULLIF(CONVERT(DECIMAL(10,2), irs.index_count), 0)) * 100, 'N1') + '%', + space_reduction_pct = FORMAT((irs.space_saved_gb / NULLIF(irs.total_size_gb, 0)) * 100, 'N1') + '%', + /* Additional metrics that may be useful */ + lock_operations_saved = FORMAT(ISNULL(irs.row_lock_count + irs.page_lock_count, 0), 'N0'), + latch_operations_saved = FORMAT(ISNULL(irs.page_latch_wait_count + irs.page_io_latch_wait_count, 0), 'N0') FROM #index_reporting_stats AS irs WHERE irs.summary_level = 'DATABASE' ORDER BY irs.database_name; @@ -4294,37 +4309,25 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. Table-Level Impact - Top tables by space savings */ SELECT TOP(10) - summary_type = 'TOP TABLES BY IMPACT', - table_name = QUOTENAME(irs.schema_name) + '.' + QUOTENAME(irs.table_name), - total_indexes = FORMAT(irs.index_count, 'N0'), - removable_indexes = FORMAT(irs.unused_indexes, 'N0'), - mergeable_indexes = FORMAT(irs.indexes_to_merge, 'N0'), - current_size_gb = FORMAT(irs.total_size_gb, 'N2'), - space_saved_gb = FORMAT(irs.unused_size_gb, 'N2'), - percent_reduction = FORMAT((irs.unused_size_gb / NULLIF(irs.total_size_gb, 0)) * 100, 'N1') + '%', - total_write_ops = FORMAT(irs.user_updates, 'N0'), - write_ops_per_day = FORMAT(irs.user_updates / NULLIF(CONVERT(DECIMAL(10,2), - (SELECT TOP 1 server_uptime_days FROM #index_reporting_stats WHERE summary_level = 'DATABASE')), 0), 'N0') + report_section = 'TOP TABLES BY IMPACT', + database_name = ISNULL(irs.database_name, 'N/A'), + table_name = QUOTENAME(ISNULL(irs.schema_name, 'dbo')) + '.' + QUOTENAME(ISNULL(irs.table_name, 'Unknown')), + /* Index counts */ + total_indexes = FORMAT(ISNULL(irs.index_count, 0), 'N0'), + removable_indexes = FORMAT(ISNULL(irs.unused_indexes, 0), 'N0'), + mergeable_indexes = FORMAT(ISNULL(irs.indexes_to_merge, 0), 'N0'), + /* Space metrics */ + current_size_gb = FORMAT(ISNULL(irs.total_size_gb, 0), 'N2'), + space_saved_gb = FORMAT(ISNULL(irs.unused_size_gb, 0), 'N2'), + percent_reduction = FORMAT((ISNULL(irs.unused_size_gb, 0) / NULLIF(irs.total_size_gb, 0)) * 100, 'N1') + '%', + /* Performance metrics */ + total_rows = FORMAT(ISNULL(irs.total_rows, 0), 'N0'), + total_write_ops = FORMAT(ISNULL(irs.user_updates, 0), 'N0'), + write_ops_per_day = FORMAT(ISNULL(irs.user_updates / NULLIF(CONVERT(DECIMAL(10,2), + (SELECT TOP 1 server_uptime_days FROM #index_reporting_stats WHERE summary_level = 'DATABASE')), 0), 0), 'N0') FROM #index_reporting_stats AS irs WHERE irs.summary_level = 'TABLE' - ORDER BY irs.unused_size_gb DESC; - - /* - Before/After Comparison for Database - */ - SELECT - comparison_metric = 'BEFORE/AFTER COMPARISON', - total_indexes_before = FORMAT(irs.index_count, 'N0'), - total_indexes_after = FORMAT(irs.index_count - (irs.indexes_to_disable + irs.indexes_to_merge), 'N0'), - index_reduction = FORMAT(((irs.indexes_to_disable + irs.indexes_to_merge) / NULLIF(CONVERT(DECIMAL(10,2), irs.index_count), 0)) * 100, 'N1') + '%', - size_before_gb = FORMAT(irs.total_size_gb, 'N2'), - size_after_gb = FORMAT(irs.total_size_gb - irs.space_saved_gb, 'N2'), - space_saved_gb = FORMAT(irs.space_saved_gb, 'N2'), - percent_space_saved = FORMAT((irs.space_saved_gb / NULLIF(irs.total_size_gb, 0)) * 100, 'N1') + '%', - daily_write_ops_saved = FORMAT(irs.user_updates / NULLIF(CONVERT(DECIMAL(10,2), irs.server_uptime_days), 0) * - ((irs.indexes_to_disable + irs.indexes_to_merge) / NULLIF(CONVERT(DECIMAL(10,2), irs.index_count), 0)), 'N0') - FROM #index_reporting_stats AS irs - WHERE irs.summary_level = 'DATABASE'; + ORDER BY ISNULL(irs.unused_size_gb, 0) DESC; /* Return streamlined reporting statistics focused on key metrics */ IF @debug = 1 From 613ed410da96ad34dd5eb1b639c573033882fef5 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 15 Mar 2025 21:37:50 -0400 Subject: [PATCH 143/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 161 ++++++++++------------------ 1 file changed, 54 insertions(+), 107 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index bd8d6897..3b4e9a21 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -4270,64 +4270,10 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. RAISERROR('Generating enhanced summary reports', 0, 0) WITH NOWAIT; END; - /* - Consolidated Database Impact Summary - Shows all metrics in one place - */ - SELECT - report_section = 'DATABASE IMPACT SUMMARY', - server_uptime = CASE - WHEN irs.server_uptime_days < 7 - THEN 'WARNING: Server uptime only ' + FORMAT(irs.server_uptime_days, 'N0') + ' days - usage data may be incomplete!' - ELSE FORMAT(irs.server_uptime_days, 'N0') + ' days' - END, - database_name = ISNULL(irs.database_name, 'N/A'), - /* Index counts */ - total_indexes = FORMAT(irs.index_count, 'N0'), - indexes_after_cleanup = FORMAT(irs.index_count - (irs.indexes_to_disable + irs.indexes_to_merge), 'N0'), - indexes_to_disable = FORMAT(irs.indexes_to_disable, 'N0'), - indexes_to_merge = FORMAT(irs.indexes_to_merge, 'N0'), - /* Space metrics */ - current_size_gb = FORMAT(irs.total_size_gb, 'N2'), - size_after_cleanup_gb = FORMAT(irs.total_size_gb - irs.space_saved_gb, 'N2'), - space_saved_gb = FORMAT(irs.space_saved_gb, 'N2'), - /* Performance metrics */ - total_rows = FORMAT(irs.total_rows, 'N0'), - write_operations_saved = FORMAT(ISNULL(irs.user_updates, 0), 'N0'), - daily_write_ops_saved = FORMAT(ISNULL(irs.user_updates / NULLIF(CONVERT(DECIMAL(10,2), irs.server_uptime_days), 0) * - ((irs.indexes_to_disable + irs.indexes_to_merge) / NULLIF(CONVERT(DECIMAL(10,2), irs.index_count), 0)), 0), 'N0'), - /* Percentage metrics for quick understanding */ - index_reduction_pct = FORMAT(((irs.indexes_to_disable + irs.indexes_to_merge) / NULLIF(CONVERT(DECIMAL(10,2), irs.index_count), 0)) * 100, 'N1') + '%', - space_reduction_pct = FORMAT((irs.space_saved_gb / NULLIF(irs.total_size_gb, 0)) * 100, 'N1') + '%', - /* Additional metrics that may be useful */ - lock_operations_saved = FORMAT(ISNULL(irs.row_lock_count + irs.page_lock_count, 0), 'N0'), - latch_operations_saved = FORMAT(ISNULL(irs.page_latch_wait_count + irs.page_io_latch_wait_count, 0), 'N0') - FROM #index_reporting_stats AS irs - WHERE irs.summary_level = 'DATABASE' - ORDER BY irs.database_name; - - /* - Table-Level Impact - Top tables by space savings - */ - SELECT TOP(10) - report_section = 'TOP TABLES BY IMPACT', - database_name = ISNULL(irs.database_name, 'N/A'), - table_name = QUOTENAME(ISNULL(irs.schema_name, 'dbo')) + '.' + QUOTENAME(ISNULL(irs.table_name, 'Unknown')), - /* Index counts */ - total_indexes = FORMAT(ISNULL(irs.index_count, 0), 'N0'), - removable_indexes = FORMAT(ISNULL(irs.unused_indexes, 0), 'N0'), - mergeable_indexes = FORMAT(ISNULL(irs.indexes_to_merge, 0), 'N0'), - /* Space metrics */ - current_size_gb = FORMAT(ISNULL(irs.total_size_gb, 0), 'N2'), - space_saved_gb = FORMAT(ISNULL(irs.unused_size_gb, 0), 'N2'), - percent_reduction = FORMAT((ISNULL(irs.unused_size_gb, 0) / NULLIF(irs.total_size_gb, 0)) * 100, 'N1') + '%', - /* Performance metrics */ - total_rows = FORMAT(ISNULL(irs.total_rows, 0), 'N0'), - total_write_ops = FORMAT(ISNULL(irs.user_updates, 0), 'N0'), - write_ops_per_day = FORMAT(ISNULL(irs.user_updates / NULLIF(CONVERT(DECIMAL(10,2), - (SELECT TOP 1 server_uptime_days FROM #index_reporting_stats WHERE summary_level = 'DATABASE')), 0), 0), 'N0') - FROM #index_reporting_stats AS irs - WHERE irs.summary_level = 'TABLE' - ORDER BY ISNULL(irs.unused_size_gb, 0) DESC; + /* + This section now REPLACES the existing summary view rather than supplementing it + We'll modify the existing query below rather than creating new output panes + */ /* Return streamlined reporting statistics focused on key metrics */ IF @debug = 1 @@ -4336,7 +4282,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END; SELECT - /* Basic identification */ + /* Basic identification with enhanced naming */ level = CASE WHEN irs.summary_level = 'SUMMARY' THEN '=== OVERALL ANALYSIS ===' @@ -4359,8 +4305,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END, /* Schema and table names (except for summary) */ - irs.schema_name, - irs.table_name, + schema_name = ISNULL(irs.schema_name, 'N/A'), + table_name = ISNULL(irs.table_name, 'N/A'), /* ===== Section 1: Index Counts ===== */ /* Tables analyzed (summary only) */ @@ -4368,82 +4314,83 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CASE WHEN irs.summary_level = 'SUMMARY' THEN FORMAT(irs.tables_analyzed, 'N0') - ELSE NULL + ELSE FORMAT(0, 'N0') /* Show 0 instead of NULL */ END, /* Total indexes */ - total_indexes = FORMAT(irs.index_count, 'N0'), + total_indexes = FORMAT(ISNULL(irs.index_count, 0), 'N0'), /* Removable indexes - report consistent values across levels */ removable_indexes = CASE WHEN irs.summary_level = 'SUMMARY' - THEN FORMAT(irs.indexes_to_disable, 'N0') /* Indexes that will be disabled based on analysis */ - ELSE FORMAT(irs.unused_indexes, 'N0') /* Unused indexes at database/table level */ + THEN FORMAT(ISNULL(irs.indexes_to_disable, 0), 'N0') /* Indexes that will be disabled based on analysis */ + ELSE FORMAT(ISNULL(irs.unused_indexes, 0), 'N0') /* Unused indexes at database/table level */ END, /* Show mergeable indexes across all levels */ - mergeable_indexes = - CASE - WHEN irs.summary_level = 'SUMMARY' - THEN FORMAT(irs.indexes_to_merge, 'N0') - ELSE FORMAT(irs.indexes_to_merge, 'N0') - END, + mergeable_indexes = FORMAT(ISNULL(irs.indexes_to_merge, 0), 'N0'), /* Percent of indexes that can be removed */ pct_removable = CASE WHEN irs.summary_level = 'SUMMARY' - THEN FORMAT(100.0 * irs.indexes_to_disable / NULLIF(irs.index_count, 0), 'N1') + '%' + THEN FORMAT(100.0 * ISNULL(irs.indexes_to_disable, 0) / NULLIF(irs.index_count, 0), 'N1') + '%' WHEN irs.index_count > 0 - THEN FORMAT(100.0 * irs.unused_indexes / NULLIF(irs.index_count, 0), 'N1') + '%' + THEN FORMAT(100.0 * ISNULL(irs.unused_indexes, 0) / NULLIF(irs.index_count, 0), 'N1') + '%' ELSE '0.0%' END, - /* ===== Section 2: Size and Space Savings ===== */ + /* ===== Section 2: Size and Space Savings with Before/After comparison ===== */ /* Current size in GB */ - current_size_gb = FORMAT(irs.total_size_gb, 'N2'), + current_size_gb = FORMAT(ISNULL(irs.total_size_gb, 0), 'N2'), - /* Size that can be saved through cleanup */ - cleanup_savings_gb = - CASE - WHEN irs.summary_level = 'SUMMARY' - THEN FORMAT(irs.space_saved_gb, 'N2') - ELSE FORMAT(irs.unused_size_gb, 'N2') - END, + /* Size after cleanup - added this as new metric */ + size_after_cleanup_gb = FORMAT(ISNULL(irs.total_size_gb, 0) - ISNULL(irs.space_saved_gb, 0), 'N2'), - /* Potential additional savings */ - potential_savings_gb = + /* Size that can be saved through cleanup */ + space_saved_gb = CASE WHEN irs.summary_level = 'SUMMARY' - THEN FORMAT(irs.total_min_savings_gb, 'N2') + - ' - ' + - FORMAT(irs.total_max_savings_gb, 'N2') - ELSE FORMAT(irs.unused_size_gb, 'N2') /* Show at all levels */ + THEN FORMAT(ISNULL(irs.space_saved_gb, 0), 'N2') + ELSE FORMAT(ISNULL(irs.unused_size_gb, 0), 'N2') END, + + /* Space reduction percentage - added this as new metric */ + space_reduction_pct = FORMAT((ISNULL(irs.space_saved_gb, 0) / NULLIF(irs.total_size_gb, 0)) * 100, 'N1') + '%', /* ===== Section 3: Table and Usage Statistics ===== */ /* Row count */ - FORMAT(irs.total_rows, 'N0') AS total_rows, + total_rows = FORMAT(ISNULL(irs.total_rows, 0), 'N0'), /* Total reads - combined total and breakdown */ reads_breakdown = CASE WHEN irs.summary_level <> 'SUMMARY' - THEN FORMAT(irs.total_reads, 'N0') + + THEN FORMAT(ISNULL(irs.total_reads, 0), 'N0') + ' (' + - FORMAT(irs.user_seeks, 'N0') + ' seeks, ' + - FORMAT(irs.user_scans, 'N0') + ' scans, ' + - FORMAT(irs.user_lookups, 'N0') + ' lookups)' - ELSE NULL + FORMAT(ISNULL(irs.user_seeks, 0), 'N0') + ' seeks, ' + + FORMAT(ISNULL(irs.user_scans, 0), 'N0') + ' scans, ' + + FORMAT(ISNULL(irs.user_lookups, 0), 'N0') + ' lookups)' + ELSE 'N/A' END, /* Total writes */ writes = CASE WHEN irs.summary_level <> 'SUMMARY' - THEN FORMAT(irs.total_writes, 'N0') - ELSE NULL + THEN FORMAT(ISNULL(irs.total_writes, 0), 'N0') + ELSE 'N/A' + END, + + /* Daily write operations saved - added as new metric */ + daily_write_ops_saved = + CASE + WHEN irs.summary_level <> 'SUMMARY' + THEN FORMAT(ISNULL(irs.user_updates / NULLIF(CONVERT(DECIMAL(10,2), + (SELECT TOP 1 server_uptime_days FROM #index_reporting_stats WHERE summary_level = 'DATABASE')), 0) * + (ISNULL(irs.unused_indexes, 0) / NULLIF(CONVERT(DECIMAL(10,2), irs.index_count), 0)), 0), 'N0') + ELSE 'N/A' END, /* ===== Section 4: Consolidated Performance Metrics ===== */ @@ -4451,29 +4398,29 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. lock_wait_count = CASE WHEN irs.summary_level <> 'SUMMARY' - THEN FORMAT(irs.row_lock_wait_count + - irs.page_lock_wait_count, 'N0') - ELSE NULL + THEN FORMAT(ISNULL(irs.row_lock_wait_count, 0) + + ISNULL(irs.page_lock_wait_count, 0), 'N0') + ELSE '0' END, /* Average lock wait time in ms */ avg_lock_wait_ms = CASE WHEN irs.summary_level <> 'SUMMARY' - AND (irs.row_lock_wait_count + irs.page_lock_wait_count) > 0 - THEN FORMAT(1.0 * (irs.row_lock_wait_in_ms + irs.page_lock_wait_in_ms) / - NULLIF(irs.row_lock_wait_count + irs.page_lock_wait_count, 0), 'N2') - ELSE NULL + AND (ISNULL(irs.row_lock_wait_count, 0) + ISNULL(irs.page_lock_wait_count, 0)) > 0 + THEN FORMAT(1.0 * (ISNULL(irs.row_lock_wait_in_ms, 0) + ISNULL(irs.page_lock_wait_in_ms, 0)) / + NULLIF(ISNULL(irs.row_lock_wait_count, 0) + ISNULL(irs.page_lock_wait_count, 0), 0), 'N2') + ELSE '0.00' END, /* Combined latch wait time in ms */ avg_latch_wait_ms = CASE WHEN irs.summary_level <> 'SUMMARY' - AND (irs.page_latch_wait_count + irs.page_io_latch_wait_count) > 0 - THEN FORMAT(1.0 * (irs.page_latch_wait_in_ms + irs.page_io_latch_wait_in_ms) / - NULLIF(irs.page_latch_wait_count + irs.page_io_latch_wait_count, 0), 'N2') - ELSE NULL + AND (ISNULL(irs.page_latch_wait_count, 0) + ISNULL(irs.page_io_latch_wait_count, 0)) > 0 + THEN FORMAT(1.0 * (ISNULL(irs.page_latch_wait_in_ms, 0) + ISNULL(irs.page_io_latch_wait_in_ms, 0)) / + NULLIF(ISNULL(irs.page_latch_wait_count, 0) + ISNULL(irs.page_io_latch_wait_count, 0), 0), 'N2') + ELSE '0.00' END FROM #index_reporting_stats AS irs WHERE irs.summary_level IN ('SUMMARY', 'DATABASE', 'TABLE') /* Filter out INDEX level */ From 3a814b986d7ea5a689d9bb0fa7a520ed30f48956 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 15 Mar 2025 22:42:46 -0400 Subject: [PATCH 144/246] twofer --- sp_IndexCleanup/sp_IndexCleanup.sql | 51 ++++++++++++++++++++++------- sp_QuickieStore/sp_QuickieStore.sql | 8 +++-- 2 files changed, 44 insertions(+), 15 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 3b4e9a21..aac0143f 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -2558,14 +2558,36 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ( result_type, sort_order, + database_name, + schema_name, + table_name, + index_name, script_type, - additional_info + additional_info, + target_index_name, + superseded_info, + original_index_definition, + index_size_gb, + index_rows, + index_reads, + index_writes ) SELECT result_type = 'SUMMARY', sort_order = 1, + database_name = '', + schema_name = '', + table_name = '', + index_name = '', script_type = 'Index Cleanup Scripts', - additional_info = N'A detailed index analysis report appears after these scripts' + additional_info = N'A detailed index analysis report appears after these scripts', + target_index_name = '', + superseded_info = '', + original_index_definition = '', + index_size_gb = 0, + index_rows = 0, + index_reads = 0, + index_writes = 0 OPTION(RECOMPILE); @@ -4048,26 +4070,26 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. index_size_gb = CASE WHEN ir.result_type = 'SUMMARY' - THEN NULL - ELSE FORMAT(ir.index_size_gb, 'N4') + THEN '0.0000' + ELSE FORMAT(ISNULL(ir.index_size_gb, 0), 'N4') END, index_rows = CASE WHEN ir.result_type = 'SUMMARY' - THEN NULL - ELSE FORMAT(ir.index_rows, 'N0') + THEN '0' + ELSE FORMAT(ISNULL(ir.index_rows, 0), 'N0') END, index_reads = CASE WHEN ir.result_type = 'SUMMARY' - THEN NULL - ELSE FORMAT(ir.index_reads, 'N0') + THEN '0' + ELSE FORMAT(ISNULL(ir.index_reads, 0), 'N0') END, index_writes = CASE WHEN ir.result_type = 'SUMMARY' - THEN NULL - ELSE FORMAT(ir.index_writes, 'N0') + THEN '0' + ELSE FORMAT(ISNULL(ir.index_writes, 0), 'N0') END, ia.original_index_definition, /* Finally show the actual script */ @@ -4334,7 +4356,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Percent of indexes that can be removed */ pct_removable = CASE - WHEN irs.summary_level = 'SUMMARY' + WHEN irs.summary_level = 'SUMMARY' AND irs.index_count > 0 THEN FORMAT(100.0 * ISNULL(irs.indexes_to_disable, 0) / NULLIF(irs.index_count, 0), 'N1') + '%' WHEN irs.index_count > 0 THEN FORMAT(100.0 * ISNULL(irs.unused_indexes, 0) / NULLIF(irs.index_count, 0), 'N1') + '%' @@ -4357,7 +4379,12 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END, /* Space reduction percentage - added this as new metric */ - space_reduction_pct = FORMAT((ISNULL(irs.space_saved_gb, 0) / NULLIF(irs.total_size_gb, 0)) * 100, 'N1') + '%', + space_reduction_pct = + CASE + WHEN ISNULL(irs.total_size_gb, 0) > 0 + THEN FORMAT((ISNULL(irs.space_saved_gb, 0) / NULLIF(irs.total_size_gb, 0)) * 100, 'N1') + '%' + ELSE '0.0%' + END, /* ===== Section 3: Table and Usage Statistics ===== */ /* Row count */ diff --git a/sp_QuickieStore/sp_QuickieStore.sql b/sp_QuickieStore/sp_QuickieStore.sql index 87a3640b..17f99ef9 100644 --- a/sp_QuickieStore/sp_QuickieStore.sql +++ b/sp_QuickieStore/sp_QuickieStore.sql @@ -1388,7 +1388,7 @@ VALUES (1230, 'num_physical_io_reads', 'min', 'min_num_physical_io_reads_mb', 'qsrs.min_num_physical_io_reads_mb', 1, 'new', 1, 1, 'N0'), (1240, 'num_physical_io_reads', 'max', 'max_num_physical_io_reads_mb', 'qsrs.max_num_physical_io_reads_mb', 1, 'new', 1, 0, 'N0'), /* Hash totals for new physical IO reads */ - (1215, 'num_physical_io_reads', 'total_hash', 'total_num_physical_io_reads_mb_by_query_hash', 'SUM(qsrs.total_num_physical_io_reads_mb) OVER (PARTITION BY qsq.query_hash ORDER BY qsq.query_hash)', 1, 'new', 1, 0, 'N0'), + (1215, 'num_physical_io_reads', 'total_hash', 'total_num_physical_io_reads_mb_by_query_hash', 'SUM(qsrs.total_num_physical_io_reads_mb) OVER (PARTITION BY qsq.query_hash ORDER BY qsq.query_hash)', 1, 'new_with_hash_totals', 1, 0, 'N0'), /* Finish adding the remaining columns (log bytes and tempdb usage) */ /* Log bytes used */ (1300, 'log_bytes', 'avg', 'avg_log_bytes_used_mb', 'qsrs.avg_log_bytes_used_mb', 1, 'new', 1, 0, 'N0'), @@ -1397,7 +1397,7 @@ VALUES (1330, 'log_bytes', 'min', 'min_log_bytes_used_mb', 'qsrs.min_log_bytes_used_mb', 1, 'new', 1, 1, 'N0'), (1340, 'log_bytes', 'max', 'max_log_bytes_used_mb', 'qsrs.max_log_bytes_used_mb', 1, 'new', 1, 0, 'N0'), /* Hash totals for log bytes */ - (1315, 'log_bytes', 'total_hash', 'total_log_bytes_used_mb_by_query_hash', 'SUM(qsrs.total_log_bytes_used_mb) OVER (PARTITION BY qsq.query_hash ORDER BY qsq.query_hash)', 1, 'new', 1, 0, 'N0'), + (1315, 'log_bytes', 'total_hash', 'total_log_bytes_used_mb_by_query_hash', 'SUM(qsrs.total_log_bytes_used_mb) OVER (PARTITION BY qsq.query_hash ORDER BY qsq.query_hash)', 1, 'new_with_hash_totals', 1, 0, 'N0'), /* TempDB usage */ (1400, 'tempdb', 'avg', 'avg_tempdb_space_used_mb', 'qsrs.avg_tempdb_space_used_mb', 1, 'new', 1, 0, 'N0'), (1410, 'tempdb', 'total', 'total_tempdb_space_used_mb', 'qsrs.total_tempdb_space_used_mb', 1, 'new', 1, 0, 'N0'), @@ -1405,7 +1405,7 @@ VALUES (1430, 'tempdb', 'min', 'min_tempdb_space_used_mb', 'qsrs.min_tempdb_space_used_mb', 1, 'new', 1, 1, 'N0'), (1440, 'tempdb', 'max', 'max_tempdb_space_used_mb', 'qsrs.max_tempdb_space_used_mb', 1, 'new', 1, 0, 'N0'), /* Hash totals for tempdb */ - (1415, 'tempdb', 'total_hash', 'total_tempdb_space_used_mb_by_query_hash', 'SUM(qsrs.total_tempdb_space_used_mb) OVER (PARTITION BY qsq.query_hash ORDER BY qsq.query_hash)', 1, 'new', 1, 0, 'N0'), + (1415, 'tempdb', 'total_hash', 'total_tempdb_space_used_mb_by_query_hash', 'SUM(qsrs.total_tempdb_space_used_mb) OVER (PARTITION BY qsq.query_hash ORDER BY qsq.query_hash)', 1, 'new_with_hash_totals', 1, 0, 'N0'), /* Context settings and sorting columns */ (1500, 'metadata', 'context', 'context_settings', 'qsrs.context_settings', 0, NULL, NULL, 0, NULL); @@ -8065,6 +8065,8 @@ FROM THEN @regression_mode WHEN cd.condition_param = N'include_query_hash_totals' THEN @include_query_hash_totals + WHEN cd.condition_param = N'new_with_hash_totals' + THEN CASE WHEN @new = 1 AND @include_query_hash_totals = 1 THEN 1 ELSE 0 END ELSE 0 END = cd.condition_value ) From c00771cbc97f3dc20cfd3c2b783d2999246a9a55 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sun, 16 Mar 2025 10:00:13 -0400 Subject: [PATCH 145/246] around the world added new parameters for QS and IC: IC: get all databases, include databases, exclude databases QS: include and exclude databases --- README.md | 5 + sp_IndexCleanup/README.md | 21 +++ sp_IndexCleanup/sp_IndexCleanup.sql | 259 ++++++++++++++++++++++++++-- sp_QuickieStore/README.md | 12 ++ sp_QuickieStore/sp_QuickieStore.sql | 158 +++++++++++++++++ 5 files changed, 438 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 8a27addb..e403f4d4 100644 --- a/README.md +++ b/README.md @@ -291,6 +291,8 @@ Current valid parameter details: | @hide_help_table | bit | hides the "bottom table" that shows help and support information | 0 or 1 | 0 | | @format_output | bit | returns numbers formatted with commas | 0 or 1 | 1 | | @get_all_databases | bit | looks for query store enabled user databases and returns combined results from all of them | 0 or 1 | 0 | +| @include_databases | nvarchar(4000) | comma-separated list of databases to include (only when @get_all_databases = 1) | a string; comma separated database names | NULL | +| @exclude_databases | nvarchar(4000) | comma-separated list of databases to exclude (only when @get_all_databases = 1) | a string; comma separated database names | NULL | | @workdays | bit | use this to filter out weekends and after-hours queries | 0 or 1 | 0 | | @work_start | time | use this to set a specific start of your work days | a time like 8am, 9am or something | 9am | | @work_end | time | use this to set a specific end of your work days | a time like 5pm, 6pm or something | 5pm | @@ -416,6 +418,9 @@ Current valid parameter details: | @min_writes | bigint | 0 | Minimum number of writes for an index to be considered used | | @min_size_gb | decimal(10,2) | 0 | Minimum size in GB for an index to be analyzed | | @min_rows | bigint | 0 | Minimum number of rows for a table to be analyzed | +| @get_all_databases | bit | 0 | When set to 1, analyzes all eligible databases on the server | +| @include_databases | nvarchar(max) | NULL | Comma-separated list of databases to include (used with @get_all_databases = 1) | +| @exclude_databases | nvarchar(max) | NULL | Comma-separated list of databases to exclude (used with @get_all_databases = 1) | | @help | bit | 0 | Displays help information | | @debug | bit | 0 | Prints debug information during execution | | @version | varchar(20) | NULL | OUTPUT parameter that returns the version number of the procedure | diff --git a/sp_IndexCleanup/README.md b/sp_IndexCleanup/README.md index cc8e2432..3e3a3706 100644 --- a/sp_IndexCleanup/README.md +++ b/sp_IndexCleanup/README.md @@ -29,6 +29,9 @@ The procedure requires SQL Server 2012 (11.0) or later due to the use of FORMAT | @min_writes | bigint | 0 | Minimum number of writes for an index to be considered used | | @min_size_gb | decimal(10,2) | 0 | Minimum size in GB for an index to be analyzed | | @min_rows | bigint | 0 | Minimum number of rows for a table to be analyzed | +| @get_all_databases | bit | 0 | When set to 1, analyzes all eligible databases on the server | +| @include_databases | nvarchar(max) | NULL | Comma-separated list of databases to include (used with @get_all_databases = 1) | +| @exclude_databases | nvarchar(max) | NULL | Comma-separated list of databases to exclude (used with @get_all_databases = 1) | | @help | bit | 0 | Displays help information | | @debug | bit | 0 | Prints debug information during execution | | @version | varchar(20) | NULL | OUTPUT parameter that returns the version number of the procedure | @@ -53,6 +56,21 @@ EXECUTE dbo.sp_IndexCleanup @min_reads = 100, @min_writes = 10; +-- Analyze all user databases on the server +EXECUTE dbo.sp_IndexCleanup + @get_all_databases = 1, + @debug = 1; + +-- Analyze only specific databases +EXECUTE dbo.sp_IndexCleanup + @get_all_databases = 1, + @include_databases = 'Database1,Database2,Database3'; + +-- Analyze all databases except specific ones +EXECUTE dbo.sp_IndexCleanup + @get_all_databases = 1, + @exclude_databases = 'ReportServer,TempDB2'; + -- Show help information EXECUTE dbo.sp_IndexCleanup @help = 1; @@ -63,6 +81,9 @@ EXECUTE dbo.sp_IndexCleanup - The procedure issues a warning when server uptime is less than 14 days, as index usage stats may not be representative - Certain features like online index operations and compression are only available in specific SQL Server editions (Enterprise, Azure SQL DB, Managed Instance) - It is recommended to have a recent backup before making any index changes +- The multi-database processing feature (@get_all_databases) analyzes each database sequentially for better performance and resource management +- System databases (master, model, msdb, tempdb, rdsadmin) are always excluded from processing +- When using @get_all_databases, results for all databases are combined in a single result set Copyright 2024 Darling Data, LLC Released under MIT license \ No newline at end of file diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index aac0143f..d411e525 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -34,6 +34,9 @@ ALTER PROCEDURE @min_writes bigint = 0, @min_size_gb decimal(10,2) = 0, @min_rows bigint = 0, + @get_all_databases bit = 0, /* When 1, analyzes all eligible databases on the server */ + @include_databases nvarchar(max) = NULL, /* Comma-separated list of databases to include (used with @get_all_databases = 1) */ + @exclude_databases nvarchar(max) = NULL, /* Comma-separated list of databases to exclude (used with @get_all_databases = 1) */ @help bit = 'false', @debug bit = 'false', @version varchar(20) = NULL OUTPUT, @@ -138,6 +141,9 @@ ALWAYS TEST THESE RECOMMENDATIONS IN A NON-PRODUCTION ENVIRONMENT FIRST" WHEN N'@min_writes' THEN 'minimum number of writes for an index to be considered used' WHEN N'@min_size_gb' THEN 'minimum size in GB for an index to be analyzed' WHEN N'@min_rows' THEN 'minimum number of rows for a table to be analyzed' + WHEN N'@get_all_databases' THEN 'when set to 1, analyzes all eligible databases on the server' + WHEN N'@include_databases' THEN 'comma-separated list of databases to include (used with @get_all_databases = 1)' + WHEN N'@exclude_databases' THEN 'comma-separated list of databases to exclude (used with @get_all_databases = 1)' WHEN N'@help' THEN 'displays this help information' WHEN N'@debug' THEN 'prints debug information during execution' WHEN N'@version' THEN 'returns the version number of the procedure' @@ -154,6 +160,9 @@ ALWAYS TEST THESE RECOMMENDATIONS IN A NON-PRODUCTION ENVIRONMENT FIRST" WHEN N'@min_writes' THEN 'any positive integer or 0' WHEN N'@min_size_gb' THEN 'any positive decimal number or 0' WHEN N'@min_rows' THEN 'any positive integer or 0' + WHEN N'@get_all_databases' THEN '0 or 1' + WHEN N'@include_databases' THEN 'comma-separated list of database names' + WHEN N'@exclude_databases' THEN 'comma-separated list of database names' WHEN N'@help' THEN '0 or 1' WHEN N'@debug' THEN '0 or 1' WHEN N'@version' THEN 'OUTPUT parameter' @@ -170,6 +179,9 @@ ALWAYS TEST THESE RECOMMENDATIONS IN A NON-PRODUCTION ENVIRONMENT FIRST" WHEN N'@min_writes' THEN '0' WHEN N'@min_size_gb' THEN '0' WHEN N'@min_rows' THEN '0' + WHEN N'@get_all_databases' THEN '0' + WHEN N'@include_databases' THEN 'NULL' + WHEN N'@exclude_databases' THEN 'NULL' WHEN N'@help' THEN 'false' WHEN N'@debug' THEN 'true' WHEN N'@version' THEN 'NULL' @@ -282,28 +294,217 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. RAISERROR('Checking paramaters...', 0, 0) WITH NOWAIT; END; - IF @database_name IS NULL - AND DB_NAME() NOT IN - ( - N'master', - N'model', - N'msdb', - N'tempdb', - N'rdsadmin' - ) - BEGIN - SELECT - @database_name = DB_NAME(); - END; + /* + Create a temp table to store databases to process + This will handle both single database mode and multi-database mode + */ + CREATE TABLE #databases_to_process + ( + database_id int PRIMARY KEY, + database_name sysname NOT NULL, + processed bit NOT NULL DEFAULT 0 + ); - IF @database_name IS NOT NULL + /* Handle multi-database mode */ + IF @get_all_databases = 1 BEGIN - SELECT - @database_id = d.database_id + IF @debug = 1 + BEGIN + RAISERROR('Multi-database mode enabled, gathering database list...', 0, 0) WITH NOWAIT; + END; + + /* Create a table to parse the include/exclude lists */ + CREATE TABLE #database_list + ( + id int IDENTITY(1,1) PRIMARY KEY, + database_name sysname NOT NULL + ); + + /* Parse @include_databases if specified - using XML for string splitting instead of STRING_SPLIT (version compatibility) */ + IF @include_databases IS NOT NULL + BEGIN + DECLARE @include_xml xml; + SELECT @include_xml = CONVERT(xml, '' + REPLACE(@include_databases, ',', '') + ''); + + INSERT INTO #database_list (database_name) + SELECT LTRIM(RTRIM(t.i.value('.', 'sysname'))) AS database_name + FROM @include_xml.nodes('/i') AS t(i) + WHERE LTRIM(RTRIM(t.i.value('.', 'sysname'))) <> ''; + END; + + /* Get all eligible databases */ + INSERT INTO #databases_to_process (database_id, database_name) + SELECT + d.database_id, + d.name FROM sys.databases AS d - WHERE d.name = @database_name + WHERE d.state_desc = 'ONLINE' + AND d.name NOT IN (N'master', N'model', N'msdb', N'tempdb', N'rdsadmin') + AND + ( + /* If include list is provided, only keep databases in that list */ + (@include_databases IS NULL) OR + (d.name IN (SELECT database_name FROM #database_list)) + ); + + /* Create exclude list if needed - using XML for string splitting */ + IF @exclude_databases IS NOT NULL + BEGIN + DECLARE @exclude_xml xml; + SELECT @exclude_xml = CONVERT(xml, '' + REPLACE(@exclude_databases, ',', '') + ''); + + DELETE dp + FROM #databases_to_process AS dp + WHERE EXISTS + ( + SELECT 1 + FROM @exclude_xml.nodes('/i') AS t(i) + WHERE dp.database_name = LTRIM(RTRIM(t.i.value('.', 'sysname'))) + AND LTRIM(RTRIM(t.i.value('.', 'sysname'))) <> '' + ); + END + OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + /* Count databases */ + SELECT + database_count = COUNT(*) + FROM #databases_to_process; + + /* List databases without using STRING_AGG (version compatibility) */ + DECLARE @db_list nvarchar(MAX) = N''; + + SELECT @db_list = @db_list + database_name + N', ' + FROM #databases_to_process + ORDER BY database_name; + + /* Remove trailing comma if list is not empty */ + IF LEN(@db_list) > 0 + SET @db_list = LEFT(@db_list, LEN(@db_list) - 1); + + RAISERROR('Databases to process: %s', 0, 0, @db_list) WITH NOWAIT; + END; + + /* If no databases match criteria, exit */ + IF NOT EXISTS (SELECT 1 FROM #databases_to_process) + BEGIN + RAISERROR('No eligible databases found to process with the specified filters', 16, 1) WITH NOWAIT; + RETURN; + END; + END + ELSE + BEGIN + /* Single database mode */ + IF @debug = 1 + BEGIN + RAISERROR('Single database mode, using specified or current database', 0, 0) WITH NOWAIT; + END; + + /* If no database name specified, use current database if not a system database */ + IF @database_name IS NULL + AND DB_NAME() NOT IN + ( + N'master', + N'model', + N'msdb', + N'tempdb', + N'rdsadmin' + ) + BEGIN + SELECT + @database_name = DB_NAME(); + END; + + /* Add the single database to the processing list */ + IF @database_name IS NOT NULL + BEGIN + INSERT INTO #databases_to_process (database_id, database_name) + SELECT + d.database_id, + d.name + FROM sys.databases AS d + WHERE d.name = @database_name + AND d.state_desc = 'ONLINE' + OPTION(RECOMPILE); + + /* Validate the database exists and is accessible */ + IF NOT EXISTS (SELECT 1 FROM #databases_to_process) + BEGIN + RAISERROR('The specified database %s does not exist, is not in ONLINE state, or you do not have permission to access it', 16, 1, @database_name) WITH NOWAIT; + RETURN; + END; + END + ELSE + BEGIN + RAISERROR('No valid database specified and current database is a system database. Please specify a user database.', 16, 1) WITH NOWAIT; + RETURN; + END; + + /* Set @database_id for single database mode (for backward compatibility) */ + SELECT @database_id = database_id, @database_name = database_name + FROM #databases_to_process; END; + + /* + Main processing logic - either loop through all databases or process a single database + */ + IF @get_all_databases = 1 + BEGIN + /* Use cursor variable instead of explicit cursor declaration as per coding guidelines */ + DECLARE @database_cursor cursor; + + DECLARE @current_database_id int, + @current_database_name sysname, + @database_count int, + @processed_count int = 0; + + SELECT @database_count = COUNT(*) FROM #databases_to_process; + + IF @debug = 1 + BEGIN + RAISERROR('Beginning processing for %d databases', 0, 0, @database_count) WITH NOWAIT; + END; + + /* Set cursor variable */ + SET @database_cursor = CURSOR LOCAL STATIC READ_ONLY FORWARD_ONLY + FOR + SELECT database_id, database_name + FROM #databases_to_process + WHERE processed = 0 + ORDER BY database_name; + + OPEN @database_cursor; + FETCH NEXT FROM @database_cursor INTO @current_database_id, @current_database_name; + + WHILE @@FETCH_STATUS = 0 + BEGIN + SET @processed_count += 1; + + /* Update working variables for each iteration */ + SET @database_id = @current_database_id; + SET @database_name = @current_database_name; + + IF @debug = 1 + BEGIN + RAISERROR('Processing database %d of %d: %s (ID: %d)', 0, 0, + @processed_count, @database_count, @database_name, @database_id) WITH NOWAIT; + END; + + /* Clear temp tables before processing next database */ + IF @processed_count > 1 + BEGIN + TRUNCATE TABLE #filtered_objects; + TRUNCATE TABLE #operational_stats; + TRUNCATE TABLE #partition_stats; + TRUNCATE TABLE #index_details; + TRUNCATE TABLE #index_analysis; + TRUNCATE TABLE #index_cleanup_results; + TRUNCATE TABLE #compression_eligibility; + END; + + /* Process current database using existing logic */ IF @schema_name IS NULL AND @table_name IS NOT NULL @@ -362,6 +563,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. RAISERROR('Parameter @min_rows cannot be NULL or negative. Setting to 0.', 10, 1) WITH NOWAIT; SET @min_rows = 0; END; + + /* Parameter validation for multi-database mode */ + IF @get_all_databases = 1 AND @database_name IS NOT NULL + BEGIN + RAISERROR('You cannot specify both @get_all_databases = 1 and a specific @database_name. Using @get_all_databases = 1 and ignoring @database_name.', 10, 1) WITH NOWAIT; + SET @database_name = NULL; + END; /* Temp tables! @@ -4475,6 +4683,23 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. irs.table_name OPTION(RECOMPILE); + /* Update processed flag for this database */ + UPDATE #databases_to_process + SET processed = 1 + WHERE database_id = @database_id; + + /* Get next database */ + FETCH NEXT FROM @database_cursor INTO @current_database_id, @current_database_name; + END; /* End of cursor WHILE loop */ + + /* Clean up cursor - cursor variables don't need explicit CLOSE/DEALLOCATE */ + + IF @debug = 1 + BEGIN + RAISERROR('Finished processing %d databases', 0, 0, @processed_count) WITH NOWAIT; + END; + END; /* End of @get_all_databases = 1 section */ + END TRY BEGIN CATCH THROW; diff --git a/sp_QuickieStore/README.md b/sp_QuickieStore/README.md index 5d2d614f..31b047cb 100644 --- a/sp_QuickieStore/README.md +++ b/sp_QuickieStore/README.md @@ -63,6 +63,8 @@ Use the `@expert_mode` parameter to return additional details. | @hide_help_table | bit | hides the "bottom table" that shows help and support information | 0 or 1 | 0 | | @format_output | bit | returns numbers formatted with commas | 0 or 1 | 1 | | @get_all_databases | bit | looks for query store enabled user databases and returns combined results from all of them | 0 or 1 | 0 | +| @include_databases | nvarchar(4000) | comma-separated list of databases to include (only when @get_all_databases = 1) | a string; comma separated database names | NULL | +| @exclude_databases | nvarchar(4000) | comma-separated list of databases to exclude (only when @get_all_databases = 1) | a string; comma separated database names | NULL | | @workdays | bit | use this to filter out weekends and after-hours queries | 0 or 1 | 0 | | @work_start | time | use this to set a specific start of your work days | a time like 8am, 9am or something | 9am | | @work_end | time | use this to set a specific end of your work days | a time like 5pm, 6pm or something | 5pm | @@ -113,6 +115,16 @@ EXECUTE dbo.sp_QuickieStore -- Expert mode for additional details EXECUTE dbo.sp_QuickieStore @expert_mode = 1; + +-- Get data from all databases with Query Store enabled, except for specific ones +EXECUTE dbo.sp_QuickieStore + @get_all_databases = 1, + @exclude_databases = 'TempDB, msdb'; + +-- Get data from only specific databases with Query Store enabled +EXECUTE dbo.sp_QuickieStore + @get_all_databases = 1, + @include_databases = 'AdventureWorks, WideWorldImporters'; ``` ## Resources diff --git a/sp_QuickieStore/sp_QuickieStore.sql b/sp_QuickieStore/sp_QuickieStore.sql index 17f99ef9..91500c2f 100644 --- a/sp_QuickieStore/sp_QuickieStore.sql +++ b/sp_QuickieStore/sp_QuickieStore.sql @@ -90,6 +90,8 @@ ALTER PROCEDURE @hide_help_table bit = 0, /*hides the "bottom table" that shows help and support information*/ @format_output bit = 1, /*returns numbers formatted with commas*/ @get_all_databases bit = 0, /*looks for query store enabled user databases and returns combined results from all of them*/ + @include_databases nvarchar(4000) = NULL, /*comma-separated list of databases to include (only when @get_all_databases = 1)*/ + @exclude_databases nvarchar(4000) = NULL, /*comma-separated list of databases to exclude (only when @get_all_databases = 1)*/ @workdays bit = 0, /*Use this to filter out weekends and after-hours queries*/ @work_start time(0) = '9am', /*Use this to set a specific start of your work days*/ @work_end time(0) = '5pm', /*Use this to set a specific end of your work days*/ @@ -203,6 +205,8 @@ BEGIN WHEN N'@hide_help_table' THEN 'hides the "bottom table" that shows help and support information' WHEN N'@format_output' THEN 'returns numbers formatted with commas' WHEN N'@get_all_databases' THEN 'looks for query store enabled user databases and returns combined results from all of them' + WHEN N'@include_databases' THEN 'comma-separated list of databases to include (only when @get_all_databases = 1)' + WHEN N'@exclude_databases' THEN 'comma-separated list of databases to exclude (only when @get_all_databases = 1)' WHEN N'@workdays' THEN 'use this to filter out weekends and after-hours queries' WHEN N'@work_start' THEN 'use this to set a specific start of your work days' WHEN N'@work_end' THEN 'use this to set a specific end of your work days' @@ -256,6 +260,8 @@ BEGIN WHEN N'@hide_help_table' THEN '0 or 1' WHEN N'@format_output' THEN '0 or 1' WHEN N'@get_all_databases' THEN '0 or 1' + WHEN N'@include_databases' THEN 'a string; comma separated database names' + WHEN N'@exclude_databases' THEN 'a string; comma separated database names' WHEN N'@workdays' THEN '0 or 1' WHEN N'@work_start' THEN 'a time like 8am, 9am or something' WHEN N'@work_end' THEN 'a time like 5pm, 6pm or something' @@ -309,6 +315,8 @@ BEGIN WHEN N'@hide_help_table' THEN '0' WHEN N'@format_output' THEN '1' WHEN N'@get_all_databases' THEN '0' + WHEN N'@include_databases' THEN 'NULL' + WHEN N'@exclude_databases' THEN 'NULL' WHEN N'@workdays' THEN '0' WHEN N'@work_start' THEN '9am' WHEN N'@work_end' THEN '5pm' @@ -646,6 +654,152 @@ CREATE TABLE /* Hold query hashes for ignored plans */ + +/* +Create temp tables to parse the include/exclude database lists +*/ +IF @get_all_databases = 1 +BEGIN + /* Check for contradictory parameters */ + IF @database_name IS NOT NULL + BEGIN + RAISERROR(N'@get_all_databases = 1 and @database_name is specified. These parameters are mutually exclusive. Choose one or the other.', 11, 1) WITH NOWAIT; + RETURN; + END; + + /* Create tables for database filtering */ + CREATE TABLE + #include_databases + ( + database_name sysname PRIMARY KEY + ); + + CREATE TABLE + #exclude_databases + ( + database_name sysname PRIMARY KEY + ); + + /* Parse @include_databases if specified using XML for compatibility */ + IF @include_databases IS NOT NULL + BEGIN + INSERT + #include_databases + ( + database_name + ) + SELECT + database_name = + LTRIM + ( + RTRIM(c.value(N'(./text())[1]', N'nvarchar(128)')) + ) + FROM + ( + SELECT + x = CONVERT + ( + xml, + N'' + + REPLACE + ( + @include_databases, + N',', + N'' + ) + + N'' + ) + ) AS a + CROSS APPLY x.nodes(N'//i') AS t(c) + WHERE LTRIM(RTRIM(c.value(N'(./text())[1]', N'nvarchar(128)'))) <> N''; + END; + + /* Parse @exclude_databases if specified using XML for compatibility */ + IF @exclude_databases IS NOT NULL + BEGIN + INSERT + #exclude_databases + ( + database_name + ) + SELECT + database_name = + LTRIM + ( + RTRIM(c.value(N'(./text())[1]', N'nvarchar(128)')) + ) + FROM + ( + SELECT + x = CONVERT + ( + xml, + N'' + + REPLACE + ( + @exclude_databases, + N',', + N'' + ) + + N'' + ) + ) AS a + CROSS APPLY x.nodes(N'//i') AS t(c) + WHERE LTRIM(RTRIM(c.value(N'(./text())[1]', N'nvarchar(128)'))) <> N''; + END; + + IF @debug = 1 + BEGIN + IF @include_databases IS NOT NULL + BEGIN + RAISERROR(N'Include database list: %s', 0, 1, @include_databases) WITH NOWAIT; + + /* Use manual concatenation instead of STRING_AGG for compatibility */ + DECLARE + @include_list nvarchar(4000) = N''; + + SELECT + @include_list = @include_list + + CASE + WHEN @include_list = N'' + THEN N'' + ELSE N', ' + END + + database_name + FROM #include_databases + ORDER BY + database_name; + + SELECT + include_database_list = @include_list; + END; + + IF @exclude_databases IS NOT NULL + BEGIN + RAISERROR(N'Exclude database list: %s', 0, 1, @exclude_databases) WITH NOWAIT; + + /* Use manual concatenation instead of STRING_AGG for compatibility */ + DECLARE + @exclude_list nvarchar(4000) = N''; + + SELECT + @exclude_list = @exclude_list + + CASE + WHEN @exclude_list = N'' + THEN N'' + ELSE N', ' + END + + database_name + FROM #exclude_databases + ORDER BY + database_name; + + SELECT + exclude_database_list = @exclude_list; + END; + END; +END; + CREATE TABLE #ignore_query_hashes ( @@ -1947,6 +2101,8 @@ BEGIN AND d.state = 0 AND d.is_in_standby = 0 AND d.is_read_only = 0 + AND (@include_databases IS NULL OR EXISTS (SELECT 1 FROM #include_databases AS id WHERE id.database_name = d.name)) + AND (@exclude_databases IS NULL OR NOT EXISTS (SELECT 1 FROM #exclude_databases AS ed WHERE ed.database_name = d.name)) OPTION(RECOMPILE); END; ELSE @@ -1987,6 +2143,8 @@ BEGIN AND s.role_desc <> N'PRIMARY' AND DATABASEPROPERTYEX(c.database_name, N'Updateability') <> N'READ_WRITE' ) + AND (@include_databases IS NULL OR EXISTS (SELECT 1 FROM #include_databases AS id WHERE id.database_name = d.name)) + AND (@exclude_databases IS NULL OR NOT EXISTS (SELECT 1 FROM #exclude_databases AS ed WHERE ed.database_name = d.name)) OPTION(RECOMPILE); END; From cd8b1cfff5bb3af38e6ce356060574ed79be9f03 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sun, 16 Mar 2025 10:03:04 -0400 Subject: [PATCH 146/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index d411e525..c25c1b65 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -2770,6 +2770,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. schema_name, table_name, index_name, + consolidation_rule, script_type, additional_info, target_index_name, @@ -2787,6 +2788,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. schema_name = '', table_name = '', index_name = '', + consolidation_rule = N'', script_type = 'Index Cleanup Scripts', additional_info = N'A detailed index analysis report appears after these scripts', target_index_name = '', From c7b3ab4e843617d02dd0c2ca9eafc5a82b856ad3 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sun, 16 Mar 2025 10:21:15 -0400 Subject: [PATCH 147/246] tidy up include/exclude stuff for QS tidy up include/exclude stuff for QS --- sp_QuickieStore/README.md | 4 +- sp_QuickieStore/sp_QuickieStore.sql | 265 ++++++++++++---------------- 2 files changed, 113 insertions(+), 156 deletions(-) diff --git a/sp_QuickieStore/README.md b/sp_QuickieStore/README.md index 31b047cb..82d60123 100644 --- a/sp_QuickieStore/README.md +++ b/sp_QuickieStore/README.md @@ -119,12 +119,12 @@ EXECUTE dbo.sp_QuickieStore -- Get data from all databases with Query Store enabled, except for specific ones EXECUTE dbo.sp_QuickieStore @get_all_databases = 1, - @exclude_databases = 'TempDB, msdb'; + @exclude_databases = 'Head, Shoulders, Knees, Toes'; -- Get data from only specific databases with Query Store enabled EXECUTE dbo.sp_QuickieStore @get_all_databases = 1, - @include_databases = 'AdventureWorks, WideWorldImporters'; + @include_databases = 'StacOverflow2013, StackOverflow2010'; ``` ## Resources diff --git a/sp_QuickieStore/sp_QuickieStore.sql b/sp_QuickieStore/sp_QuickieStore.sql index 91500c2f..0221fd85 100644 --- a/sp_QuickieStore/sp_QuickieStore.sql +++ b/sp_QuickieStore/sp_QuickieStore.sql @@ -90,8 +90,8 @@ ALTER PROCEDURE @hide_help_table bit = 0, /*hides the "bottom table" that shows help and support information*/ @format_output bit = 1, /*returns numbers formatted with commas*/ @get_all_databases bit = 0, /*looks for query store enabled user databases and returns combined results from all of them*/ - @include_databases nvarchar(4000) = NULL, /*comma-separated list of databases to include (only when @get_all_databases = 1)*/ - @exclude_databases nvarchar(4000) = NULL, /*comma-separated list of databases to exclude (only when @get_all_databases = 1)*/ + @include_databases nvarchar(max) = NULL, /*comma-separated list of databases to include (only when @get_all_databases = 1)*/ + @exclude_databases nvarchar(max) = NULL, /*comma-separated list of databases to exclude (only when @get_all_databases = 1)*/ @workdays bit = 0, /*Use this to filter out weekends and after-hours queries*/ @work_start time(0) = '9am', /*Use this to set a specific start of your work days*/ @work_end time(0) = '5pm', /*Use this to set a specific end of your work days*/ @@ -654,152 +654,6 @@ CREATE TABLE /* Hold query hashes for ignored plans */ - -/* -Create temp tables to parse the include/exclude database lists -*/ -IF @get_all_databases = 1 -BEGIN - /* Check for contradictory parameters */ - IF @database_name IS NOT NULL - BEGIN - RAISERROR(N'@get_all_databases = 1 and @database_name is specified. These parameters are mutually exclusive. Choose one or the other.', 11, 1) WITH NOWAIT; - RETURN; - END; - - /* Create tables for database filtering */ - CREATE TABLE - #include_databases - ( - database_name sysname PRIMARY KEY - ); - - CREATE TABLE - #exclude_databases - ( - database_name sysname PRIMARY KEY - ); - - /* Parse @include_databases if specified using XML for compatibility */ - IF @include_databases IS NOT NULL - BEGIN - INSERT - #include_databases - ( - database_name - ) - SELECT - database_name = - LTRIM - ( - RTRIM(c.value(N'(./text())[1]', N'nvarchar(128)')) - ) - FROM - ( - SELECT - x = CONVERT - ( - xml, - N'' + - REPLACE - ( - @include_databases, - N',', - N'' - ) + - N'' - ) - ) AS a - CROSS APPLY x.nodes(N'//i') AS t(c) - WHERE LTRIM(RTRIM(c.value(N'(./text())[1]', N'nvarchar(128)'))) <> N''; - END; - - /* Parse @exclude_databases if specified using XML for compatibility */ - IF @exclude_databases IS NOT NULL - BEGIN - INSERT - #exclude_databases - ( - database_name - ) - SELECT - database_name = - LTRIM - ( - RTRIM(c.value(N'(./text())[1]', N'nvarchar(128)')) - ) - FROM - ( - SELECT - x = CONVERT - ( - xml, - N'' + - REPLACE - ( - @exclude_databases, - N',', - N'' - ) + - N'' - ) - ) AS a - CROSS APPLY x.nodes(N'//i') AS t(c) - WHERE LTRIM(RTRIM(c.value(N'(./text())[1]', N'nvarchar(128)'))) <> N''; - END; - - IF @debug = 1 - BEGIN - IF @include_databases IS NOT NULL - BEGIN - RAISERROR(N'Include database list: %s', 0, 1, @include_databases) WITH NOWAIT; - - /* Use manual concatenation instead of STRING_AGG for compatibility */ - DECLARE - @include_list nvarchar(4000) = N''; - - SELECT - @include_list = @include_list + - CASE - WHEN @include_list = N'' - THEN N'' - ELSE N', ' - END + - database_name - FROM #include_databases - ORDER BY - database_name; - - SELECT - include_database_list = @include_list; - END; - - IF @exclude_databases IS NOT NULL - BEGIN - RAISERROR(N'Exclude database list: %s', 0, 1, @exclude_databases) WITH NOWAIT; - - /* Use manual concatenation instead of STRING_AGG for compatibility */ - DECLARE - @exclude_list nvarchar(4000) = N''; - - SELECT - @exclude_list = @exclude_list + - CASE - WHEN @exclude_list = N'' - THEN N'' - ELSE N', ' - END + - database_name - FROM #exclude_databases - ORDER BY - database_name; - - SELECT - exclude_database_list = @exclude_list; - END; - END; -END; - CREATE TABLE #ignore_query_hashes ( @@ -1415,6 +1269,19 @@ CREATE TABLE database_name sysname PRIMARY KEY CLUSTERED ); +/* Create tables for database filtering */ +CREATE TABLE + #include_databases +( + database_name sysname PRIMARY KEY +); + +CREATE TABLE + #exclude_databases +( + database_name sysname PRIMARY KEY +); + /* Create a table variable to store ALL column definitions with logical ordering */ DECLARE @ColumnDefinitions table @@ -2067,6 +1934,84 @@ are assigned for the specific database that is currently being looked at */ +/* +Look at databases to include or exclude +*/ +IF @get_all_databases = 1 +BEGIN + /* Check for contradictory parameters */ + IF @database_name IS NOT NULL + BEGIN + RAISERROR(N'@database name being ignored since @get_all_databases is set to 1', 10, 1) WITH NOWAIT; + SET @database_name = NULL; + END; + + /* Parse @include_databases if specified using XML for compatibility */ + IF @include_databases IS NOT NULL + BEGIN + INSERT + #include_databases + ( + database_name + ) + SELECT + database_name = + LTRIM(RTRIM(c.value(N'(./text())[1]', N'sysname'))) + FROM + ( + SELECT DISTINCT + x = CONVERT + ( + xml, + N'' + + REPLACE + ( + @include_databases, + N',', + N'' + ) + + N'' + ) + ) AS a + CROSS APPLY x.nodes(N'//i') AS t(c) + WHERE LTRIM(RTRIM(c.value(N'(./text())[1]', N'sysname'))) <> N''; + END; + + /* Parse @exclude_databases if specified using XML for compatibility */ + IF @exclude_databases IS NOT NULL + BEGIN + INSERT + #exclude_databases + ( + database_name + ) + SELECT + database_name = + LTRIM(RTRIM(c.value(N'(./text())[1]', N'sysname'))) + FROM + ( + SELECT DISTINCT + x = CONVERT + ( + xml, + N'' + + REPLACE + ( + @exclude_databases, + N',', + N'' + ) + + N'' + ) + ) AS a + CROSS APPLY x.nodes(N'//i') AS t(c) + WHERE LTRIM(RTRIM(c.value(N'(./text())[1]', N'sysname'))) <> N''; + END; +END; + +/* +Build up the databases to process +*/ IF ( SELECT @@ -2101,8 +2046,14 @@ BEGIN AND d.state = 0 AND d.is_in_standby = 0 AND d.is_read_only = 0 - AND (@include_databases IS NULL OR EXISTS (SELECT 1 FROM #include_databases AS id WHERE id.database_name = d.name)) - AND (@exclude_databases IS NULL OR NOT EXISTS (SELECT 1 FROM #exclude_databases AS ed WHERE ed.database_name = d.name)) + AND ( + @include_databases IS NULL + OR EXISTS (SELECT 1/0 FROM #include_databases AS id WHERE id.database_name = d.name) + ) + AND ( + @exclude_databases IS NULL + OR NOT EXISTS (SELECT 1/0 FROM #exclude_databases AS ed WHERE ed.database_name = d.name) + ) OPTION(RECOMPILE); END; ELSE @@ -2143,8 +2094,14 @@ BEGIN AND s.role_desc <> N'PRIMARY' AND DATABASEPROPERTYEX(c.database_name, N'Updateability') <> N'READ_WRITE' ) - AND (@include_databases IS NULL OR EXISTS (SELECT 1 FROM #include_databases AS id WHERE id.database_name = d.name)) - AND (@exclude_databases IS NULL OR NOT EXISTS (SELECT 1 FROM #exclude_databases AS ed WHERE ed.database_name = d.name)) + AND ( + @include_databases IS NULL + OR EXISTS (SELECT 1/0 FROM #include_databases AS id WHERE id.database_name = d.name) + ) + AND ( + @exclude_databases IS NULL + OR NOT EXISTS (SELECT 1/0 FROM #exclude_databases AS ed WHERE ed.database_name = d.name) + ) OPTION(RECOMPILE); END; @@ -2176,7 +2133,7 @@ Some variable assignment, because why not? */ IF @debug = 1 BEGIN - RAISERROR('Starting analysis for database %s', 0, 1, @database_name) WITH NOWAIT; + RAISERROR('Starting analysis for database %s', 0, 0, @database_name) WITH NOWAIT; END; SELECT @@ -3878,7 +3835,7 @@ BEGIN /* Log current operation if debugging */ IF @debug = 1 BEGIN - RAISERROR('Processing %s with value %s', 0, 1, @param_name, @param_value) WITH NOWAIT; + RAISERROR('Processing %s with value %s', 0, 0, @param_name, @param_value) WITH NOWAIT; END; /* Set current table name for troubleshooting */ From 7c622815086a6ff3a1216bfd935643d5ac2cb9c9 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sun, 16 Mar 2025 10:24:08 -0400 Subject: [PATCH 148/246] Update sp_QuickieStore.sql --- sp_QuickieStore/sp_QuickieStore.sql | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/sp_QuickieStore/sp_QuickieStore.sql b/sp_QuickieStore/sp_QuickieStore.sql index 0221fd85..2c82c149 100644 --- a/sp_QuickieStore/sp_QuickieStore.sql +++ b/sp_QuickieStore/sp_QuickieStore.sql @@ -9894,16 +9894,14 @@ BEGIN CATCH RAISERROR('%s', 10, 1, @sql) WITH NOWAIT; END; - /* - Unquote this if you want to short-circuit the THROW; - command and see the @debug = 1 output instead. - */ - --GOTO DEBUG; - - /* - This reliably throws the actual error from dynamic SQL - */ - THROW; + IF @debug = 1 + BEGIN + GOTO DEBUG; + END; + IF @debug = 0 + BEGIN; + THROW; + END; END CATCH; /* From a2dc7d77ebdf369b0f13121975f102fc43f8ab70 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sun, 16 Mar 2025 10:52:50 -0400 Subject: [PATCH 149/246] ugh i messed up. fixing. --- sp_IndexCleanup/sp_IndexCleanup.sql | 354 ++++++++++++++++++++++++---- sp_QuickieStore/sp_QuickieStore.sql | 50 ++++ 2 files changed, 363 insertions(+), 41 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index c25c1b65..e2d6e5dd 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -275,7 +275,16 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SYSDATETIME() ) FROM sys.dm_os_sys_info AS osi - ); + ), + /* Variables for multi-database processing */ + @database_cursor cursor, + @current_database_id integer, + @current_database_name sysname, + @database_count integer, + @processed_count integer = 0, + @db_list nvarchar(MAX), + @include_xml xml, + @exclude_xml xml; /* Set uptime warning flag after @uptime_days is calculated */ SELECT @@ -294,15 +303,267 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. RAISERROR('Checking paramaters...', 0, 0) WITH NOWAIT; END; - /* - Create a temp table to store databases to process - This will handle both single database mode and multi-database mode + /* + Temp tables! */ - CREATE TABLE #databases_to_process + + IF @debug = 1 + BEGIN + RAISERROR('Creating temp tables', 0, 0) WITH NOWAIT; + END; + + CREATE TABLE + #filtered_objects ( - database_id int PRIMARY KEY, + database_id integer NOT NULL, database_name sysname NOT NULL, - processed bit NOT NULL DEFAULT 0 + schema_id integer NOT NULL, + schema_name sysname NOT NULL, + object_id integer NOT NULL, + table_name sysname NOT NULL, + index_id integer NOT NULL, + index_name sysname NOT NULL, + can_compress bit NOT NULL + PRIMARY KEY CLUSTERED(database_id, schema_id, object_id, index_id) + ); + + CREATE TABLE + #operational_stats + ( + database_id integer NOT NULL, + database_name sysname NOT NULL, + schema_id integer NOT NULL, + schema_name sysname NOT NULL, + object_id integer NOT NULL, + table_name sysname NOT NULL, + index_id integer NOT NULL, + index_name sysname NOT NULL, + range_scan_count bigint NULL, + singleton_lookup_count bigint NULL, + forwarded_fetch_count bigint NULL, + lob_fetch_in_pages bigint NULL, + row_overflow_fetch_in_pages bigint NULL, + leaf_insert_count bigint NULL, + leaf_update_count bigint NULL, + leaf_delete_count bigint NULL, + leaf_ghost_count bigint NULL, + nonleaf_insert_count bigint NULL, + nonleaf_update_count bigint NULL, + nonleaf_delete_count bigint NULL, + leaf_allocation_count bigint NULL, + nonleaf_allocation_count bigint NULL, + row_lock_count bigint NULL, + row_lock_wait_count bigint NULL, + row_lock_wait_in_ms bigint NULL, + page_lock_count bigint NULL, + page_lock_wait_count bigint NULL, + page_lock_wait_in_ms bigint NULL, + index_lock_promotion_attempt_count bigint NULL, + index_lock_promotion_count bigint NULL, + page_latch_wait_count bigint NULL, + page_latch_wait_in_ms bigint NULL, + tree_page_latch_wait_count bigint NULL, + tree_page_latch_wait_in_ms bigint NULL, + page_io_latch_wait_count bigint NULL, + page_io_latch_wait_in_ms bigint NULL, + page_compression_attempt_count bigint NULL, + page_compression_success_count bigint NULL, + PRIMARY KEY CLUSTERED (database_id, schema_id, object_id, index_id) + ); + + CREATE TABLE + #partition_stats + ( + database_id integer NOT NULL, + database_name sysname NOT NULL, + schema_id integer NOT NULL, + schema_name sysname NOT NULL, + object_id integer NOT NULL, + table_name sysname NOT NULL, + index_id integer NOT NULL, + index_name sysname NULL, + partition_id bigint NOT NULL, + partition_number integer NOT NULL, + total_rows bigint NULL, + total_space_gb decimal(38, 4) NULL, /* Using 4 decimal places for GB to maintain precision */ + reserved_lob_gb decimal(38, 4) NULL, /* Using 4 decimal places for GB to maintain precision */ + reserved_row_overflow_gb decimal(38, 4) NULL, /* Using 4 decimal places for GB to maintain precision */ + data_compression_desc nvarchar(60) NULL, + built_on sysname NULL, + partition_function_name sysname NULL, + partition_columns nvarchar(max) + PRIMARY KEY CLUSTERED(database_id, schema_id, object_id, index_id, partition_id) + ); + + CREATE TABLE + #index_details + ( + database_id integer NOT NULL, + database_name sysname NOT NULL, + schema_id integer NOT NULL, + schema_name sysname NOT NULL, + object_id integer NOT NULL, + table_name sysname NOT NULL, + index_id integer NOT NULL, + index_name sysname NULL, + column_name sysname NOT NULL, + is_primary_key bit NULL, + is_unique bit NULL, + is_unique_constraint bit NULL, + is_indexed_view integer NOT NULL, + is_foreign_key bit NULL, + is_foreign_key_reference bit NULL, + key_ordinal tinyint NOT NULL, + index_column_id integer NOT NULL, + is_descending_key bit NOT NULL, + is_included_column bit NULL, + filter_definition nvarchar(max) NULL, + is_max_length integer NOT NULL, + user_seeks bigint NOT NULL, + user_scans bigint NOT NULL, + user_lookups bigint NOT NULL, + user_updates bigint NOT NULL, + last_user_seek datetime NULL, + last_user_scan datetime NULL, + last_user_lookup datetime NULL, + last_user_update datetime NULL, + is_eligible_for_dedupe bit NOT NULL + PRIMARY KEY CLUSTERED(database_id, object_id, index_id, column_name) + ); + + CREATE TABLE + #index_analysis + ( + database_id integer NOT NULL, + database_name sysname NOT NULL, + schema_id integer NOT NULL, + schema_name sysname NOT NULL, + object_id integer NOT NULL, + table_name sysname NOT NULL, + index_id integer NOT NULL, + index_name sysname NOT NULL, + key_columns nvarchar(max) NULL, + included_columns nvarchar(max) NULL, + filter_definition nvarchar(max) NULL, + /* Query plan for original CREATE INDEX statement */ + original_index_definition nvarchar(max) NULL, + /* + Consolidation rule that matched (e.g., Key Duplicate, Key Subset, etc) + For exact duplicates, use one of: Exact Duplicate, Reverse Duplicate, or Equal Except For Filter + */ + consolidation_rule nvarchar(256) NULL, + /* + Action to take (e.g., DISABLE, MERGE INCLUDES, KEEP) + If NULL, no action to be taken + */ + action nvarchar(100) NULL, + /* Target index to merge with or use instead of this one */ + target_index_name sysname NULL, + /* When this is a target, the index which points to it as a supersedes in consolidation */ + superseded_by nvarchar(4000) NULL, + /* Priority score from 0-1 to determine which index to keep (higher is better) */ + index_priority decimal(10,6) NULL + PRIMARY KEY CLUSTERED(database_id, object_id, index_id) + ); + + CREATE TABLE + #index_cleanup_results + ( + result_type varchar(100) NOT NULL, + sort_order integer NOT NULL, + database_name sysname NULL, + schema_name sysname NULL, + table_name sysname NULL, + index_name sysname NULL, + script_type nvarchar(60) NULL, /* Type of script (e.g., MERGE SCRIPT, DISABLE SCRIPT, etc.) */ + consolidation_rule nvarchar(256) NULL, /* Reason for action (e.g., Exact Duplicate, Key Subset) */ + target_index_name sysname NULL, /* If this index is a duplicate, indicates which index is the preferred one */ + superseded_info nvarchar(4000) NULL, /* If this is a kept index, indicates which indexes it supersedes */ + additional_info nvarchar(max) NULL, /* Additional information about the action */ + original_index_definition nvarchar(max) NULL, /* Original statement to create the index */ + index_size_gb decimal(38, 4) NULL, /* Size of the index in GB */ + index_rows bigint NULL, /* Number of rows in the index */ + index_reads bigint NULL, /* Total reads (seeks + scans + lookups) */ + index_writes bigint NULL, /* Total writes */ + script nvarchar(max) NULL /* Script to execute the action */ + ); + + CREATE TABLE + #compression_eligibility + ( + database_id integer NOT NULL, + database_name sysname NOT NULL, + schema_id integer NOT NULL, + schema_name sysname NOT NULL, + object_id integer NOT NULL, + table_name sysname NOT NULL, + index_id integer NOT NULL, + index_name sysname NOT NULL, + can_compress bit NOT NULL, + PRIMARY KEY CLUSTERED(database_id, object_id, index_id) + ); + + CREATE TABLE + #index_reporting_stats + ( + summary_level varchar(20) NOT NULL, /* 'DATABASE', 'TABLE', 'INDEX', 'SUMMARY' */ + database_name sysname NULL, + schema_name sysname NULL, + table_name sysname NULL, + index_name sysname NULL, + server_uptime_days integer NULL, + uptime_warning bit NULL, + tables_analyzed integer NULL, + index_count integer NULL, + total_size_gb decimal(38, 4) NULL, + total_rows bigint NULL, + unused_indexes integer NULL, + unused_size_gb decimal(38, 4) NULL, + indexes_to_disable integer NULL, + indexes_to_merge integer NULL, + avg_indexes_per_table decimal(10, 2) NULL, + space_saved_gb decimal(10, 4) NULL, + compression_min_savings_gb decimal(10, 4) NULL, + compression_max_savings_gb decimal(10, 4) NULL, + total_min_savings_gb decimal(10, 4) NULL, + total_max_savings_gb decimal(10, 4) NULL, + /* Index usage metrics */ + total_reads bigint NULL, + total_writes bigint NULL, + user_seeks bigint NULL, + user_scans bigint NULL, + user_lookups bigint NULL, + user_updates bigint NULL, + /* Operational stats */ + range_scan_count bigint NULL, + singleton_lookup_count bigint NULL, + /* Lock stats */ + row_lock_count bigint NULL, + row_lock_wait_count bigint NULL, + row_lock_wait_in_ms bigint NULL, + page_lock_count bigint NULL, + page_lock_wait_count bigint NULL, + page_lock_wait_in_ms bigint NULL, + /* Latch stats */ + page_latch_wait_count bigint NULL, + page_latch_wait_in_ms bigint NULL, + page_io_latch_wait_count bigint NULL, + page_io_latch_wait_in_ms bigint NULL, + /* Misc stats */ + forwarded_fetch_count bigint NULL, + leaf_insert_count bigint NULL, + leaf_update_count bigint NULL, + leaf_delete_count bigint NULL + ); + + /* Create a table to store databases to process */ + CREATE TABLE + #databases_to_process + ( + database_id integer NOT NULL, + database_name sysname NOT NULL, + processed bit NOT NULL DEFAULT 0, + PRIMARY KEY CLUSTERED(database_id) ); /* Handle multi-database mode */ @@ -314,29 +575,39 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END; /* Create a table to parse the include/exclude lists */ - CREATE TABLE #database_list + CREATE TABLE + #database_list ( - id int IDENTITY(1,1) PRIMARY KEY, + id integer IDENTITY(1,1) PRIMARY KEY, database_name sysname NOT NULL ); /* Parse @include_databases if specified - using XML for string splitting instead of STRING_SPLIT (version compatibility) */ IF @include_databases IS NOT NULL BEGIN - DECLARE @include_xml xml; SELECT @include_xml = CONVERT(xml, '' + REPLACE(@include_databases, ',', '') + ''); - INSERT INTO #database_list (database_name) - SELECT LTRIM(RTRIM(t.i.value('.', 'sysname'))) AS database_name + INSERT INTO + #database_list + ( + database_name + ) + SELECT + database_name = LTRIM(RTRIM(t.i.value('.', 'sysname'))) FROM @include_xml.nodes('/i') AS t(i) WHERE LTRIM(RTRIM(t.i.value('.', 'sysname'))) <> ''; END; /* Get all eligible databases */ - INSERT INTO #databases_to_process (database_id, database_name) + INSERT INTO + #databases_to_process + ( + database_id, + database_name + ) SELECT - d.database_id, - d.name + database_id = d.database_id, + database_name = d.name FROM sys.databases AS d WHERE d.state_desc = 'ONLINE' AND d.name NOT IN (N'master', N'model', N'msdb', N'tempdb', N'rdsadmin') @@ -347,24 +618,22 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. (d.name IN (SELECT database_name FROM #database_list)) ); - /* Create exclude list if needed - using XML for string splitting */ + /* Remove excluded databases if specified */ IF @exclude_databases IS NOT NULL BEGIN - DECLARE @exclude_xml xml; SELECT @exclude_xml = CONVERT(xml, '' + REPLACE(@exclude_databases, ',', '') + ''); DELETE dp FROM #databases_to_process AS dp WHERE EXISTS ( - SELECT 1 + SELECT + 1/0 FROM @exclude_xml.nodes('/i') AS t(i) WHERE dp.database_name = LTRIM(RTRIM(t.i.value('.', 'sysname'))) AND LTRIM(RTRIM(t.i.value('.', 'sysname'))) <> '' ); - END - - OPTION(RECOMPILE); + END; IF @debug = 1 BEGIN @@ -374,7 +643,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. FROM #databases_to_process; /* List databases without using STRING_AGG (version compatibility) */ - DECLARE @db_list nvarchar(MAX) = N''; + SELECT @db_list = N''; SELECT @db_list = @db_list + database_name + N', ' FROM #databases_to_process @@ -420,10 +689,15 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Add the single database to the processing list */ IF @database_name IS NOT NULL BEGIN - INSERT INTO #databases_to_process (database_id, database_name) + INSERT INTO + #databases_to_process + ( + database_id, + database_name + ) SELECT - d.database_id, - d.name + database_id = d.database_id, + database_name = d.name FROM sys.databases AS d WHERE d.name = @database_name AND d.state_desc = 'ONLINE' @@ -443,7 +717,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END; /* Set @database_id for single database mode (for backward compatibility) */ - SELECT @database_id = database_id, @database_name = database_name + SELECT + @database_id = database_id, + @database_name = database_name FROM #databases_to_process; END; @@ -452,31 +728,28 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ IF @get_all_databases = 1 BEGIN - /* Use cursor variable instead of explicit cursor declaration as per coding guidelines */ - DECLARE @database_cursor cursor; - - DECLARE @current_database_id int, - @current_database_name sysname, - @database_count int, - @processed_count int = 0; - - SELECT @database_count = COUNT(*) FROM #databases_to_process; + /* Get the count of databases for reporting */ + SELECT @database_count = COUNT(*) + FROM #databases_to_process; IF @debug = 1 BEGIN RAISERROR('Beginning processing for %d databases', 0, 0, @database_count) WITH NOWAIT; END; - /* Set cursor variable */ + /* Set cursor variable as per coding guidelines */ SET @database_cursor = CURSOR LOCAL STATIC READ_ONLY FORWARD_ONLY FOR - SELECT database_id, database_name + SELECT + database_id, + database_name FROM #databases_to_process WHERE processed = 0 ORDER BY database_name; OPEN @database_cursor; - FETCH NEXT FROM @database_cursor INTO @current_database_id, @current_database_name; + FETCH NEXT FROM @database_cursor + INTO @current_database_id, @current_database_name; WHILE @@FETCH_STATUS = 0 BEGIN @@ -4691,11 +4964,10 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. WHERE database_id = @database_id; /* Get next database */ - FETCH NEXT FROM @database_cursor INTO @current_database_id, @current_database_name; + FETCH NEXT FROM @database_cursor + INTO @current_database_id, @current_database_name; END; /* End of cursor WHILE loop */ - /* Clean up cursor - cursor variables don't need explicit CLOSE/DEALLOCATE */ - IF @debug = 1 BEGIN RAISERROR('Finished processing %d databases', 0, 0, @processed_count) WITH NOWAIT; diff --git a/sp_QuickieStore/sp_QuickieStore.sql b/sp_QuickieStore/sp_QuickieStore.sql index 0221fd85..457580a8 100644 --- a/sp_QuickieStore/sp_QuickieStore.sql +++ b/sp_QuickieStore/sp_QuickieStore.sql @@ -9987,6 +9987,10 @@ BEGIN @format_output, get_all_databases = @get_all_databases, + include_databases = + @include_databases, + exclude_databases = + @exclude_databases, workdays = @workdays, work_start = @@ -10161,6 +10165,52 @@ BEGIN result = '#databases is empty'; END; + + IF EXISTS + ( + SELECT + 1/0 + FROM #include_databases AS id + ) + BEGIN + SELECT + table_name = + '#include_databases', + id.* + FROM #include_databases AS id + ORDER BY + id.database_name + OPTION(RECOMPILE); + END; + ELSE + BEGIN + SELECT + result = + '#include_databases is empty'; + END; + + IF EXISTS + ( + SELECT + 1/0 + FROM #exclude_databases AS ed + ) + BEGIN + SELECT + table_name = + '#exclude_databases', + ed.* + FROM #exclude_databases AS ed + ORDER BY + ed.database_name + OPTION(RECOMPILE); + END; + ELSE + BEGIN + SELECT + result = + '#exclude_databases is empty'; + END; IF EXISTS ( From 2741f5976a5e31f3ea7e192baa3665d121a5a5df Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sun, 16 Mar 2025 11:06:19 -0400 Subject: [PATCH 150/246] fudgin' fudgin' --- sp_IndexCleanup/sp_IndexCleanup.sql | 89 ++++++++++++++++++++--------- sp_QuickieStore/sp_QuickieStore.sql | 2 +- 2 files changed, 64 insertions(+), 27 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index e2d6e5dd..df024115 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -578,14 +578,27 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CREATE TABLE #database_list ( - id integer IDENTITY(1,1) PRIMARY KEY, + id integer IDENTITY PRIMARY KEY CLUSTERED, database_name sysname NOT NULL ); /* Parse @include_databases if specified - using XML for string splitting instead of STRING_SPLIT (version compatibility) */ IF @include_databases IS NOT NULL BEGIN - SELECT @include_xml = CONVERT(xml, '' + REPLACE(@include_databases, ',', '') + ''); + SELECT + @include_xml = + CONVERT + ( + xml, + '' + + REPLACE + ( + @include_databases, + ',', + '' + ) + + '' + ); INSERT INTO #database_list @@ -593,7 +606,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. database_name ) SELECT - database_name = LTRIM(RTRIM(t.i.value('.', 'sysname'))) + database_name = + LTRIM(RTRIM(t.i.value('.', 'sysname'))) FROM @include_xml.nodes('/i') AS t(i) WHERE LTRIM(RTRIM(t.i.value('.', 'sysname'))) <> ''; END; @@ -609,8 +623,10 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. database_id = d.database_id, database_name = d.name FROM sys.databases AS d - WHERE d.state_desc = 'ONLINE' - AND d.name NOT IN (N'master', N'model', N'msdb', N'tempdb', N'rdsadmin') + WHERE d.name NOT IN (N'master', N'model', N'msdb', N'tempdb', N'rdsadmin') + AND d.state = 0 + AND d.is_in_standby = 0 + AND d.is_read_only = 0 AND ( /* If include list is provided, only keep databases in that list */ @@ -639,25 +655,32 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. BEGIN /* Count databases */ SELECT - database_count = COUNT(*) + database_count = COUNT_BIG(*) FROM #databases_to_process; /* List databases without using STRING_AGG (version compatibility) */ SELECT @db_list = N''; - SELECT @db_list = @db_list + database_name + N', ' - FROM #databases_to_process - ORDER BY database_name; + SELECT + @db_list = + @db_list + + database_name + + N', ' + FROM #databases_to_process AS dtp + ORDER BY + dtp.database_name; /* Remove trailing comma if list is not empty */ IF LEN(@db_list) > 0 + BEGIN SET @db_list = LEFT(@db_list, LEN(@db_list) - 1); + END; RAISERROR('Databases to process: %s', 0, 0, @db_list) WITH NOWAIT; END; /* If no databases match criteria, exit */ - IF NOT EXISTS (SELECT 1 FROM #databases_to_process) + IF NOT EXISTS (SELECT 1/0 FROM #databases_to_process AS dtp) BEGIN RAISERROR('No eligible databases found to process with the specified filters', 16, 1) WITH NOWAIT; RETURN; @@ -738,26 +761,36 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END; /* Set cursor variable as per coding guidelines */ - SET @database_cursor = CURSOR LOCAL STATIC READ_ONLY FORWARD_ONLY + SET @database_cursor = + CURSOR + LOCAL + STATIC + READ_ONLY + FORWARD_ONLY FOR SELECT - database_id, - database_name - FROM #databases_to_process - WHERE processed = 0 - ORDER BY database_name; + dtp.database_id, + dtp.database_name + FROM #databases_to_process AS dtp. + WHERE dtp.processed = 0 + ORDER BY + dtp.database_name; OPEN @database_cursor; - FETCH NEXT FROM @database_cursor - INTO @current_database_id, @current_database_name; + FETCH NEXT + FROM @database_cursor + INTO + @current_database_id, + @current_database_name; WHILE @@FETCH_STATUS = 0 BEGIN SET @processed_count += 1; /* Update working variables for each iteration */ - SET @database_id = @current_database_id; - SET @database_name = @current_database_name; + SELECT + @database_id = @current_database_id, + @database_name = @current_database_name; IF @debug = 1 BEGIN @@ -4959,13 +4992,18 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. OPTION(RECOMPILE); /* Update processed flag for this database */ - UPDATE #databases_to_process - SET processed = 1 - WHERE database_id = @database_id; + UPDATE + #databases_to_process + SET + #databases_to_process.processed = 1 + WHERE #databases_to_process.database_id = @database_id; /* Get next database */ - FETCH NEXT FROM @database_cursor - INTO @current_database_id, @current_database_name; + FETCH NEXT + FROM @database_cursor + INTO + @current_database_id, + @current_database_name; END; /* End of cursor WHILE loop */ IF @debug = 1 @@ -4973,7 +5011,6 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. RAISERROR('Finished processing %d databases', 0, 0, @processed_count) WITH NOWAIT; END; END; /* End of @get_all_databases = 1 section */ - END TRY BEGIN CATCH THROW; diff --git a/sp_QuickieStore/sp_QuickieStore.sql b/sp_QuickieStore/sp_QuickieStore.sql index 8e325b6e..8fce236c 100644 --- a/sp_QuickieStore/sp_QuickieStore.sql +++ b/sp_QuickieStore/sp_QuickieStore.sql @@ -2078,7 +2078,7 @@ BEGIN FROM sys.databases AS d WHERE @get_all_databases = 1 AND d.is_query_store_on = 1 - AND d.database_id > 4 + AND d.name NOT IN (N'master', N'model', N'msdb', N'tempdb', N'rdsadmin') AND d.state = 0 AND d.is_in_standby = 0 AND d.is_read_only = 0 From e6bf583537fffed4d3c08fb5ba4365ed12883b96 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sun, 16 Mar 2025 11:06:45 -0400 Subject: [PATCH 151/246] Update sp_QuickieStore.sql --- sp_QuickieStore/sp_QuickieStore.sql | 202 ++++++++++++++++++++++++++++ 1 file changed, 202 insertions(+) diff --git a/sp_QuickieStore/sp_QuickieStore.sql b/sp_QuickieStore/sp_QuickieStore.sql index 8e325b6e..eb9989f2 100644 --- a/sp_QuickieStore/sp_QuickieStore.sql +++ b/sp_QuickieStore/sp_QuickieStore.sql @@ -1282,6 +1282,13 @@ CREATE TABLE database_name sysname PRIMARY KEY ); +CREATE TABLE + #requested_but_skipped_databases +( + database_name sysname PRIMARY KEY, + reason varchar(100) NOT NULL +); + /* Create a table variable to store ALL column definitions with logical ordering */ DECLARE @ColumnDefinitions table @@ -2055,6 +2062,39 @@ BEGIN OR NOT EXISTS (SELECT 1/0 FROM #exclude_databases AS ed WHERE ed.database_name = d.name) ) OPTION(RECOMPILE); + + /* Track which requested databases were skipped */ + IF @include_databases IS NOT NULL AND @get_all_databases = 1 + BEGIN + INSERT + #requested_but_skipped_databases + ( + database_name, + reason + ) + SELECT + id.database_name, + reason = + CASE + WHEN d.name IS NULL THEN 'Database does not exist' + WHEN d.state <> 0 THEN 'Database not online' + WHEN d.is_query_store_on = 0 THEN 'Query Store not enabled' + WHEN d.is_in_standby = 1 THEN 'Database is in standby' + WHEN d.is_read_only = 1 THEN 'Database is read-only' + WHEN d.database_id <= 4 THEN 'System database' + ELSE 'Other issue' + END + FROM #include_databases AS id + LEFT JOIN sys.databases AS d + ON id.database_name = d.name + WHERE NOT EXISTS + ( + SELECT + 1/0 + FROM #databases AS db + WHERE db.database_name = id.database_name + ); + END; END; ELSE BEGIN @@ -2103,6 +2143,51 @@ BEGIN OR NOT EXISTS (SELECT 1/0 FROM #exclude_databases AS ed WHERE ed.database_name = d.name) ) OPTION(RECOMPILE); + + /* Track which requested databases were skipped */ + IF @include_databases IS NOT NULL AND @get_all_databases = 1 + BEGIN + INSERT + #requested_but_skipped_databases + ( + database_name, + reason + ) + SELECT + id.database_name, + reason = + CASE + WHEN d.name IS NULL THEN 'Database does not exist' + WHEN d.state <> 0 THEN 'Database not online' + WHEN d.is_query_store_on = 0 THEN 'Query Store not enabled' + WHEN d.is_in_standby = 1 THEN 'Database is in standby' + WHEN d.is_read_only = 1 THEN 'Database is read-only' + WHEN d.database_id <= 4 THEN 'System database' + WHEN EXISTS + ( + SELECT + 1/0 + FROM sys.dm_hadr_availability_replica_states AS s + JOIN sys.availability_databases_cluster AS c + ON s.group_id = c.group_id + AND d.name = c.database_name + WHERE s.is_local <> 1 + AND s.role_desc <> N'PRIMARY' + AND DATABASEPROPERTYEX(c.database_name, N'Updateability') <> N'READ_WRITE' + ) THEN 'AG replica issues' + ELSE 'Other issue' + END + FROM #include_databases AS id + LEFT JOIN sys.databases AS d + ON id.database_name = d.name + WHERE NOT EXISTS + ( + SELECT + 1/0 + FROM #databases AS db + WHERE db.database_name = id.database_name + ); + END; END; DECLARE @@ -9756,6 +9841,8 @@ BEGIN SELECT x.all_done, x.period, + x.databases_processed, + x.databases_skipped, x.support, x.help, x.problems, @@ -9802,6 +9889,58 @@ BEGIN ), all_done = 'brought to you by darling data!', + databases_processed = + ISNULL + ( + NULLIF + ( + CASE + WHEN @get_all_databases = 0 THEN @database_name + ELSE + ( + SELECT + database_list = + ( + SELECT + d.database_name + N', ' + FROM #databases AS d + ORDER BY + d.database_name + FOR XML + PATH('') + ) + ) + END, + N'' + ), + N'None' + ), + databases_skipped = + ISNULL + ( + NULLIF + ( + CASE + WHEN @get_all_databases = 0 THEN N'' + ELSE + ( + SELECT + database_list = + ( + SELECT + rbs.database_name + N' (' + rbs.reason + N'), ' + FROM #requested_but_skipped_databases AS rbs + ORDER BY + rbs.database_name + FOR XML + PATH('') + ) + ) + END, + N'' + ), + N'None' + ), support = 'for support, head over to github', help = @@ -9855,6 +9994,46 @@ BEGIN ), all_done = 'https://www.erikdarling.com/', + databases_processed = + ISNULL + ( + STUFF + ( + ( + SELECT + N', ' + + d.database_name + FROM #databases AS d + ORDER BY + d.database_name + FOR XML + PATH(''), + TYPE + ).value('.', 'nvarchar(MAX)'), + 1, 2, N'' + ), + N'None' + ), + databases_skipped = + ISNULL + ( + STUFF + ( + ( + SELECT + N', ' + + rbs.database_name + N' (' + rbs.reason + N')' + FROM #requested_but_skipped_databases AS rbs + ORDER BY + rbs.database_name + FOR XML + PATH(''), + TYPE + ).value('.', 'nvarchar(MAX)'), + 1, 2, N'' + ), + N'None' + ), support = 'https://github.com/erikdarlingdata/DarlingData', help = @@ -10209,6 +10388,29 @@ BEGIN result = '#exclude_databases is empty'; END; + + IF EXISTS + ( + SELECT + 1/0 + FROM #requested_but_skipped_databases AS rsdb + ) + BEGIN + SELECT + table_name = + '#requested_but_skipped_databases', + rsdb.* + FROM #requested_but_skipped_databases AS rsdb + ORDER BY + rsdb.database_name + OPTION(RECOMPILE); + END; + ELSE + BEGIN + SELECT + result = + '#requested_but_skipped_databases is empty'; + END; IF EXISTS ( From c6522756165fc5cd9ded3630e086300c9b2a2d54 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sun, 16 Mar 2025 11:13:45 -0400 Subject: [PATCH 152/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index df024115..7f6e4468 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -82,28 +82,8 @@ BEGIN TRY @version_date = '20250401'; SELECT - for_insurance_purposes = N'Read the messages pane carefully!'; - - PRINT N' -------------------------------------------------------------------------------------------- -------------------------------------------------------------------------------------------- -------------------------------------------------------------------------------------------- -This is the BETA VERSION of sp_IndexCleanup - -It needs lots of love and testing in real environments with real indexes to fix many issues: - * Data collection - * Deduping logic - * Result correctness - * Edge cases - * May not account for specific query patterns that benefit from seemingly redundant indexes - -ALWAYS TEST THESE RECOMMENDATIONS IN A NON-PRODUCTION ENVIRONMENT FIRST" - -------------------------------------------------------------------------------------------- -------------------------------------------------------------------------------------------- -------------------------------------------------------------------------------------------- -'; - + for_insurance_purposes = + N'ALWAYS TEST THESE RECOMMENDATIONS IN A NON-PRODUCTION ENVIRONMENT FIRST!'; /* Help section, for help. From 889c664f2a7319f5e3a0745c89ec74ba1bf54425 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sun, 16 Mar 2025 11:24:39 -0400 Subject: [PATCH 153/246] Update sp_QuickieStore.sql --- sp_QuickieStore/sp_QuickieStore.sql | 108 ++++++++++++++-------------- 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/sp_QuickieStore/sp_QuickieStore.sql b/sp_QuickieStore/sp_QuickieStore.sql index b6e11c9b..86bf56a4 100644 --- a/sp_QuickieStore/sp_QuickieStore.sql +++ b/sp_QuickieStore/sp_QuickieStore.sql @@ -9841,8 +9841,8 @@ BEGIN SELECT x.all_done, x.period, - x.databases_processed, - x.databases_skipped, + x.databases, + x.databases, x.support, x.help, x.problems, @@ -9889,58 +9889,56 @@ BEGIN ), all_done = 'brought to you by darling data!', - databases_processed = - ISNULL - ( - NULLIF - ( - CASE - WHEN @get_all_databases = 0 THEN @database_name - ELSE + databases = + N'processed: ' + + CASE + WHEN @get_all_databases = 0 THEN ISNULL(@database_name, N'None') + ELSE + ISNULL + ( + STUFF ( - SELECT - database_list = - ( - SELECT - d.database_name + N', ' - FROM #databases AS d - ORDER BY - d.database_name - FOR XML - PATH('') - ) - ) - END, - N'' - ), - N'None' - ), - databases_skipped = - ISNULL - ( - NULLIF - ( - CASE - WHEN @get_all_databases = 0 THEN N'' - ELSE + ( + SELECT + N', ' + + d.database_name + FROM #databases AS d + ORDER BY + d.database_name + FOR XML + PATH(''), + TYPE + ).value('.', 'nvarchar(MAX)'), + 1, 2, N'' + ), + N'None' + ) + END, + databases = + N'skipped: ' + + CASE + WHEN @get_all_databases = 0 THEN N'None' + ELSE + ISNULL + ( + STUFF ( - SELECT - database_list = - ( - SELECT - rbs.database_name + N' (' + rbs.reason + N'), ' - FROM #requested_but_skipped_databases AS rbs - ORDER BY - rbs.database_name - FOR XML - PATH('') - ) - ) - END, - N'' - ), - N'None' - ), + ( + SELECT + N', ' + + rbs.database_name + N' (' + rbs.reason + N')' + FROM #requested_but_skipped_databases AS rbs + ORDER BY + rbs.database_name + FOR XML + PATH(''), + TYPE + ).value('.', 'nvarchar(MAX)'), + 1, 2, N'' + ), + N'None' + ) + END, support = 'for support, head over to github', help = @@ -9994,7 +9992,8 @@ BEGIN ), all_done = 'https://www.erikdarling.com/', - databases_processed = + databases = + N'processed: ' + ISNULL ( STUFF @@ -10014,7 +10013,8 @@ BEGIN ), N'None' ), - databases_skipped = + databases = + N'skipped: ' + ISNULL ( STUFF From 850671f34eac57443bdd5df719bd916f6256fcd1 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sun, 16 Mar 2025 11:26:24 -0400 Subject: [PATCH 154/246] Update sp_QuickieStore.sql --- sp_QuickieStore/sp_QuickieStore.sql | 71 +++++++---------------------- 1 file changed, 17 insertions(+), 54 deletions(-) diff --git a/sp_QuickieStore/sp_QuickieStore.sql b/sp_QuickieStore/sp_QuickieStore.sql index 86bf56a4..69f27f77 100644 --- a/sp_QuickieStore/sp_QuickieStore.sql +++ b/sp_QuickieStore/sp_QuickieStore.sql @@ -9842,7 +9842,6 @@ BEGIN x.all_done, x.period, x.databases, - x.databases, x.support, x.help, x.problems, @@ -9892,7 +9891,8 @@ BEGIN databases = N'processed: ' + CASE - WHEN @get_all_databases = 0 THEN ISNULL(@database_name, N'None') + WHEN @get_all_databases = 0 + THEN ISNULL(@database_name, N'None') ELSE ISNULL ( @@ -9905,36 +9905,14 @@ BEGIN FROM #databases AS d ORDER BY d.database_name - FOR XML + FOR + XML PATH(''), TYPE - ).value('.', 'nvarchar(MAX)'), - 1, 2, N'' - ), - N'None' - ) - END, - databases = - N'skipped: ' + - CASE - WHEN @get_all_databases = 0 THEN N'None' - ELSE - ISNULL - ( - STUFF - ( - ( - SELECT - N', ' + - rbs.database_name + N' (' + rbs.reason + N')' - FROM #requested_but_skipped_databases AS rbs - ORDER BY - rbs.database_name - FOR XML - PATH(''), - TYPE - ).value('.', 'nvarchar(MAX)'), - 1, 2, N'' + ).value('.', 'nvarchar(max)'), + 1, + 2, + N'' ), N'None' ) @@ -9992,27 +9970,6 @@ BEGIN ), all_done = 'https://www.erikdarling.com/', - databases = - N'processed: ' + - ISNULL - ( - STUFF - ( - ( - SELECT - N', ' + - d.database_name - FROM #databases AS d - ORDER BY - d.database_name - FOR XML - PATH(''), - TYPE - ).value('.', 'nvarchar(MAX)'), - 1, 2, N'' - ), - N'None' - ), databases = N'skipped: ' + ISNULL @@ -10022,15 +9979,21 @@ BEGIN ( SELECT N', ' + - rbs.database_name + N' (' + rbs.reason + N')' + rbs.database_name + + N' (' + + rbs.reason + + N')' FROM #requested_but_skipped_databases AS rbs ORDER BY rbs.database_name - FOR XML + FOR + XML PATH(''), TYPE ).value('.', 'nvarchar(MAX)'), - 1, 2, N'' + 1, + 2, + N'' ), N'None' ), From a74295fa4b5eb3a463d20e6b88aee53185d9e6ff Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sun, 16 Mar 2025 11:42:57 -0400 Subject: [PATCH 155/246] Update sp_QuickieStore.sql --- sp_QuickieStore/sp_QuickieStore.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sp_QuickieStore/sp_QuickieStore.sql b/sp_QuickieStore/sp_QuickieStore.sql index 69f27f77..0547370f 100644 --- a/sp_QuickieStore/sp_QuickieStore.sql +++ b/sp_QuickieStore/sp_QuickieStore.sql @@ -1961,12 +1961,12 @@ BEGIN ( database_name ) - SELECT + SELECT DISTINCT database_name = LTRIM(RTRIM(c.value(N'(./text())[1]', N'sysname'))) FROM ( - SELECT DISTINCT + SELECT x = CONVERT ( xml, @@ -1992,12 +1992,12 @@ BEGIN ( database_name ) - SELECT + SELECT DISTINCT database_name = LTRIM(RTRIM(c.value(N'(./text())[1]', N'sysname'))) FROM ( - SELECT DISTINCT + SELECT x = CONVERT ( xml, From 6459e3d089529ea05234e49e1d305dd49cca0f81 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sun, 16 Mar 2025 12:49:01 -0400 Subject: [PATCH 156/246] ugh loops --- sp_IndexCleanup/sp_IndexCleanup.sql | 306 +++------------------------- sp_QuickieStore/sp_QuickieStore.sql | 5 +- 2 files changed, 35 insertions(+), 276 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 7f6e4468..c3eb3258 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -35,8 +35,8 @@ ALTER PROCEDURE @min_size_gb decimal(10,2) = 0, @min_rows bigint = 0, @get_all_databases bit = 0, /* When 1, analyzes all eligible databases on the server */ - @include_databases nvarchar(max) = NULL, /* Comma-separated list of databases to include (used with @get_all_databases = 1) */ - @exclude_databases nvarchar(max) = NULL, /* Comma-separated list of databases to exclude (used with @get_all_databases = 1) */ + @include_databases nvarchar(MAX) = NULL, /* Comma-separated list of databases to include (used with @get_all_databases = 1) */ + @exclude_databases nvarchar(MAX) = NULL, /* Comma-separated list of databases to exclude (used with @get_all_databases = 1) */ @help bit = 'false', @debug bit = 'false', @version varchar(20) = NULL OUTPUT, @@ -422,6 +422,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. table_name sysname NOT NULL, index_id integer NOT NULL, index_name sysname NOT NULL, + is_unique bit NULL, key_columns nvarchar(max) NULL, included_columns nvarchar(max) NULL, filter_definition nvarchar(max) NULL, @@ -480,9 +481,35 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. index_id integer NOT NULL, index_name sysname NOT NULL, can_compress bit NOT NULL, + reason nvarchar(200) NULL, PRIMARY KEY CLUSTERED(database_id, object_id, index_id) ); + CREATE TABLE + #key_duplicate_dedupe + ( + database_id integer NOT NULL, + object_id integer NOT NULL, + database_name sysname NOT NULL, + schema_name sysname NOT NULL, + table_name sysname NOT NULL, + base_key_columns nvarchar(max) NULL, + filter_definition nvarchar(max) NULL, + winning_index_name sysname NULL, + index_list nvarchar(max) NULL, + ); + + CREATE TABLE + #include_subset_dedupe + ( + database_id integer NOT NULL, + object_id integer NOT NULL, + subset_index_name sysname NULL, + superset_index_name sysname NULL, + subset_included_columns nvarchar(max) NULL, + superset_included_columns nvarchar(max) NULL + ); + CREATE TABLE #index_reporting_stats ( @@ -751,7 +778,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SELECT dtp.database_id, dtp.database_name - FROM #databases_to_process AS dtp. + FROM #databases_to_process AS dtp WHERE dtp.processed = 0 ORDER BY dtp.database_name; @@ -788,6 +815,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. TRUNCATE TABLE #index_analysis; TRUNCATE TABLE #index_cleanup_results; TRUNCATE TABLE #compression_eligibility; + TRUNCATE TABLE #index_reporting_stats; END; /* Process current database using existing logic */ @@ -856,278 +884,6 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. RAISERROR('You cannot specify both @get_all_databases = 1 and a specific @database_name. Using @get_all_databases = 1 and ignoring @database_name.', 10, 1) WITH NOWAIT; SET @database_name = NULL; END; - - /* - Temp tables! - */ - - IF @debug = 1 - BEGIN - RAISERROR('Creating temp tables', 0, 0) WITH NOWAIT; - END; - - CREATE TABLE - #filtered_objects - ( - database_id integer NOT NULL, - database_name sysname NOT NULL, - schema_id integer NOT NULL, - schema_name sysname NOT NULL, - object_id integer NOT NULL, - table_name sysname NOT NULL, - index_id integer NOT NULL, - index_name sysname NOT NULL, - can_compress bit NOT NULL - PRIMARY KEY CLUSTERED(database_id, schema_id, object_id, index_id) - ); - - CREATE TABLE - #operational_stats - ( - database_id integer NOT NULL, - database_name sysname NOT NULL, - schema_id integer NOT NULL, - schema_name sysname NOT NULL, - object_id integer NOT NULL, - table_name sysname NOT NULL, - index_id integer NOT NULL, - index_name sysname NOT NULL, - range_scan_count bigint NULL, - singleton_lookup_count bigint NULL, - forwarded_fetch_count bigint NULL, - lob_fetch_in_pages bigint NULL, - row_overflow_fetch_in_pages bigint NULL, - leaf_insert_count bigint NULL, - leaf_update_count bigint NULL, - leaf_delete_count bigint NULL, - leaf_ghost_count bigint NULL, - nonleaf_insert_count bigint NULL, - nonleaf_update_count bigint NULL, - nonleaf_delete_count bigint NULL, - leaf_allocation_count bigint NULL, - nonleaf_allocation_count bigint NULL, - row_lock_count bigint NULL, - row_lock_wait_count bigint NULL, - row_lock_wait_in_ms bigint NULL, - page_lock_count bigint NULL, - page_lock_wait_count bigint NULL, - page_lock_wait_in_ms bigint NULL, - index_lock_promotion_attempt_count bigint NULL, - index_lock_promotion_count bigint NULL, - page_latch_wait_count bigint NULL, - page_latch_wait_in_ms bigint NULL, - tree_page_latch_wait_count bigint NULL, - tree_page_latch_wait_in_ms bigint NULL, - page_io_latch_wait_count bigint NULL, - page_io_latch_wait_in_ms bigint NULL, - page_compression_attempt_count bigint NULL, - page_compression_success_count bigint NULL, - PRIMARY KEY CLUSTERED (database_id, schema_id, object_id, index_id) - ); - - CREATE TABLE - #partition_stats - ( - database_id integer NOT NULL, - database_name sysname NOT NULL, - schema_id integer NOT NULL, - schema_name sysname NOT NULL, - object_id integer NOT NULL, - table_name sysname NOT NULL, - index_id integer NOT NULL, - index_name sysname NULL, - partition_id bigint NOT NULL, - partition_number int NOT NULL, - total_rows bigint NULL, - total_space_gb decimal(38, 4) NULL, /* Using 4 decimal places for GB to maintain precision */ - reserved_lob_gb decimal(38, 4) NULL, /* Using 4 decimal places for GB to maintain precision */ - reserved_row_overflow_gb decimal(38, 4) NULL, /* Using 4 decimal places for GB to maintain precision */ - data_compression_desc nvarchar(60) NULL, - built_on sysname NULL, - partition_function_name sysname NULL, - partition_columns nvarchar(max) - PRIMARY KEY CLUSTERED(database_id, schema_id, object_id, index_id, partition_id) - ); - - CREATE TABLE - #index_details - ( - database_id integer NOT NULL, - database_name sysname NOT NULL, - schema_id integer NOT NULL, - schema_name sysname NOT NULL, - object_id integer NOT NULL, - table_name sysname NOT NULL, - index_id integer NOT NULL, - index_name sysname NULL, - column_name sysname NOT NULL, - is_primary_key bit NULL, - is_unique bit NULL, - is_unique_constraint bit NULL, - is_indexed_view integer NOT NULL, - is_foreign_key bit NULL, - is_foreign_key_reference bit NULL, - key_ordinal tinyint NOT NULL, - index_column_id integer NOT NULL, - is_descending_key bit NOT NULL, - is_included_column bit NULL, - filter_definition nvarchar(max) NULL, - is_max_length integer NOT NULL, - user_seeks bigint NOT NULL, - user_scans bigint NOT NULL, - user_lookups bigint NOT NULL, - user_updates bigint NOT NULL, - last_user_seek datetime NULL, - last_user_scan datetime NULL, - last_user_lookup datetime NULL, - last_user_update datetime NULL, - is_eligible_for_dedupe bit NOT NULL - PRIMARY KEY CLUSTERED(database_id, object_id, index_id, column_name) - ); - - CREATE TABLE - #index_analysis - ( - database_id integer NOT NULL, - database_name sysname NOT NULL, - schema_id integer NOT NULL, - schema_name sysname NOT NULL, - object_id integer NOT NULL, - table_name sysname NOT NULL, - index_id integer NULL, - index_name sysname NOT NULL, - is_unique bit NULL, - key_columns nvarchar(max) NULL, - included_columns nvarchar(max) NULL, - filter_definition nvarchar(max) NULL, - is_redundant bit NULL, - superseded_by nvarchar(256) NULL, - missing_columns nvarchar(max) NULL, - action nvarchar(30) NULL, - target_index_name sysname NULL, - consolidation_rule varchar(512) NULL, - index_priority int NULL, - original_index_definition nvarchar(max) NULL, /* Original CREATE INDEX statement */ - INDEX c CLUSTERED (database_id, schema_id, object_id, index_id) - ); - - CREATE TABLE - #compression_eligibility - ( - database_id integer NOT NULL, - database_name sysname NOT NULL, - schema_id integer NOT NULL, - schema_name sysname NOT NULL, - object_id integer NOT NULL, - table_name sysname NOT NULL, - index_id integer NOT NULL, - index_name sysname NOT NULL, - can_compress bit NOT NULL, - reason nvarchar(200) NULL, - PRIMARY KEY CLUSTERED(database_id, object_id, index_id) - ); - - CREATE TABLE - #key_duplicate_dedupe - ( - database_id integer NOT NULL, - object_id integer NOT NULL, - database_name sysname NOT NULL, - schema_name sysname NOT NULL, - table_name sysname NOT NULL, - base_key_columns nvarchar(max) NULL, - filter_definition nvarchar(max) NULL, - winning_index_name sysname NULL, - index_list nvarchar(max) NULL, - ); - - CREATE TABLE - #include_subset_dedupe - ( - database_id integer NOT NULL, - object_id integer NOT NULL, - subset_index_name sysname NULL, - superset_index_name sysname NULL, - subset_included_columns nvarchar(max) NULL, - superset_included_columns nvarchar(max) NULL - ); - - CREATE TABLE - #index_cleanup_results - ( - result_type varchar(50) NOT NULL, /* 'SUMMARY', 'MERGE', 'DISABLE', 'COMPRESS', etc. */ - sort_order integer NOT NULL, /* Keeps results in logical order */ - database_name sysname NULL, - schema_name sysname NULL, - table_name sysname NULL, - index_name sysname NULL, - script_type nvarchar(50) NULL, /* 'MERGE', 'DISABLE', 'COMPRESS', etc. */ - consolidation_rule nvarchar(256) NULL, - target_index_name sysname NULL, - script nvarchar(max) NULL, - additional_info nvarchar(max) NULL, /* For stats, constraints, etc. */ - superseded_info nvarchar(max) NULL, /* To store superseded_by information */ - original_index_definition nvarchar(max) NULL, /* Original index definition for validation */ - index_size_gb decimal(18,4) NULL, /* Size of the index in GB */ - index_rows bigint NULL, /* Number of rows in the index */ - index_reads bigint NULL, /* Total reads (seeks + scans + lookups) */ - index_writes bigint NULL /* Total writes (updates) */ - ); - - /* Create a new temp table for detailed reporting statistics */ - CREATE TABLE - #index_reporting_stats - ( - summary_level varchar(20) NOT NULL, /* 'DATABASE', 'TABLE', 'INDEX', 'SUMMARY' */ - database_name sysname NULL, - schema_name sysname NULL, - table_name sysname NULL, - index_name sysname NULL, - server_uptime_days int NULL, - uptime_warning bit NULL, - tables_analyzed int NULL, - index_count int NULL, - total_size_gb decimal(38, 4) NULL, - total_rows bigint NULL, - unused_indexes int NULL, - unused_size_gb decimal(38, 4) NULL, - indexes_to_disable int NULL, - indexes_to_merge int NULL, - avg_indexes_per_table decimal(10, 2) NULL, - space_saved_gb decimal(10, 4) NULL, - compression_min_savings_gb decimal(10, 4) NULL, - compression_max_savings_gb decimal(10, 4) NULL, - total_min_savings_gb decimal(10, 4) NULL, - total_max_savings_gb decimal(10, 4) NULL, - /* Index usage metrics */ - total_reads bigint NULL, - total_writes bigint NULL, - user_seeks bigint NULL, - user_scans bigint NULL, - user_lookups bigint NULL, - user_updates bigint NULL, - /* Operational stats */ - range_scan_count bigint NULL, - singleton_lookup_count bigint NULL, - /* Lock stats */ - row_lock_count bigint NULL, - row_lock_wait_count bigint NULL, - row_lock_wait_in_ms bigint NULL, - page_lock_count bigint NULL, - page_lock_wait_count bigint NULL, - page_lock_wait_in_ms bigint NULL, - /* Latch stats */ - page_latch_wait_count bigint NULL, - page_latch_wait_in_ms bigint NULL, - page_io_latch_wait_count bigint NULL, - page_io_latch_wait_in_ms bigint NULL, - /* Misc stats */ - forwarded_fetch_count bigint NULL, - leaf_insert_count bigint NULL, - leaf_update_count bigint NULL, - leaf_delete_count bigint NULL - ); - /* Start insert queries */ diff --git a/sp_QuickieStore/sp_QuickieStore.sql b/sp_QuickieStore/sp_QuickieStore.sql index 0547370f..6046c344 100644 --- a/sp_QuickieStore/sp_QuickieStore.sql +++ b/sp_QuickieStore/sp_QuickieStore.sql @@ -1949,7 +1949,10 @@ BEGIN /* Check for contradictory parameters */ IF @database_name IS NOT NULL BEGIN - RAISERROR(N'@database name being ignored since @get_all_databases is set to 1', 10, 1) WITH NOWAIT; + IF @debug = 1 + BEGIN + RAISERROR(N'@database name being ignored since @get_all_databases is set to 1', 0, 0) WITH NOWAIT; + END; SET @database_name = NULL; END; From bde0afa42e9b066f174fe91570d788291ba3191e Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sun, 16 Mar 2025 12:49:26 -0400 Subject: [PATCH 157/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 7f6e4468..a3cf9b3d 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -788,20 +788,27 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. TRUNCATE TABLE #index_analysis; TRUNCATE TABLE #index_cleanup_results; TRUNCATE TABLE #compression_eligibility; + TRUNCATE TABLE #index_reporting_stats; END; /* Process current database using existing logic */ - IF @schema_name IS NULL - AND @table_name IS NOT NULL - BEGIN - SELECT - @schema_name = N'dbo'; - END; + IF @debug = 1 + BEGIN + RAISERROR('Processing database %s', 0, 0, @database_name) WITH NOWAIT; + END; - IF @schema_name IS NOT NULL - AND @table_name IS NOT NULL - BEGIN + /* Continue with current database processing without recreating temp tables */ + IF @schema_name IS NULL + AND @table_name IS NOT NULL + BEGIN + SELECT + @schema_name = N'dbo'; + END; + + IF @schema_name IS NOT NULL + AND @table_name IS NOT NULL + BEGIN SELECT @full_object_name = QUOTENAME(@database_name) + From af5d272069475a77870813fa4eba6f38bdafa14f Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sun, 16 Mar 2025 13:09:22 -0400 Subject: [PATCH 158/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 71 ++++++++++++++++++++++++++++- 1 file changed, 70 insertions(+), 1 deletion(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index cb0d1d66..ccfb84b7 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -570,6 +570,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. database_id integer NOT NULL, database_name sysname NOT NULL, processed bit NOT NULL DEFAULT 0, + process_date datetime2(7) NULL, PRIMARY KEY CLUSTERED(database_id) ); @@ -4737,7 +4738,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. UPDATE #databases_to_process SET - #databases_to_process.processed = 1 + #databases_to_process.processed = 1, + #databases_to_process.process_date = SYSDATETIME() WHERE #databases_to_process.database_id = @database_id; /* Get next database */ @@ -4752,6 +4754,73 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. BEGIN RAISERROR('Finished processing %d databases', 0, 0, @processed_count) WITH NOWAIT; END; + + /* Create a summary table with database processing info */ + SELECT + database_summary = N'Database Processing Summary', + processed_databases = + N'Processed: ' + + ISNULL + ( + STUFF + ( + ( + SELECT + N', ' + + dtp.database_name + FROM #databases_to_process AS dtp + WHERE dtp.processed = 1 + ORDER BY + dtp.database_name + FOR + XML + PATH(''), + TYPE + ).value('.', 'nvarchar(max)'), + 1, + 2, + N'' + ), + N'None' + ), + skipped_databases = + N'Skipped: ' + + ISNULL + ( + STUFF + ( + ( + SELECT + N', ' + + dtp.database_name + FROM #databases_to_process AS dtp + WHERE dtp.processed = 0 + ORDER BY + dtp.database_name + FOR + XML + PATH(''), + TYPE + ).value('.', 'nvarchar(max)'), + 1, + 2, + N'' + ), + N'None' + ), + stats = + CASE + WHEN @get_all_databases = 1 + THEN N'Total: ' + CONVERT(nvarchar(10), COUNT_BIG(*) OVER()) + + N', Processed: ' + CONVERT(nvarchar(10), SUM(CONVERT(integer, processed)) OVER()) + + N', Skipped: ' + CONVERT(nvarchar(10), COUNT_BIG(*) OVER() - SUM(CONVERT(integer, processed)) OVER()) + ELSE N'Single database mode' + END + FROM #databases_to_process + WHERE @database_count > 0 /* Return one row with summary data */ + GROUP BY + @get_all_databases /* Just to get one row */ + OPTION(RECOMPILE); END; /* End of @get_all_databases = 1 section */ END TRY BEGIN CATCH From e64c0ee97dcb8a0e744d020763fc3a66710e213c Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sun, 16 Mar 2025 13:13:10 -0400 Subject: [PATCH 159/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index ccfb84b7..c01fa839 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -4765,11 +4765,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. STUFF ( ( - SELECT + SELECT N', ' + dtp.database_name FROM #databases_to_process AS dtp WHERE dtp.processed = 1 + GROUP BY + dtp.database_name ORDER BY dtp.database_name FOR @@ -4790,11 +4792,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. STUFF ( ( - SELECT + SELECT N', ' + dtp.database_name FROM #databases_to_process AS dtp WHERE dtp.processed = 0 + GROUP BY + dtp.database_name ORDER BY dtp.database_name FOR @@ -4812,14 +4816,15 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CASE WHEN @get_all_databases = 1 THEN N'Total: ' + CONVERT(nvarchar(10), COUNT_BIG(*) OVER()) + - N', Processed: ' + CONVERT(nvarchar(10), SUM(CONVERT(integer, processed)) OVER()) + - N', Skipped: ' + CONVERT(nvarchar(10), COUNT_BIG(*) OVER() - SUM(CONVERT(integer, processed)) OVER()) + N', Processed: ' + CONVERT(nvarchar(10), SUM(CONVERT(integer, dtp.processed)) OVER()) + + N', Skipped: ' + CONVERT(nvarchar(10), COUNT_BIG(*) OVER() - SUM(CONVERT(integer, dtp.processed)) OVER()) ELSE N'Single database mode' END - FROM #databases_to_process + FROM #databases_to_process AS dtp WHERE @database_count > 0 /* Return one row with summary data */ - GROUP BY - @get_all_databases /* Just to get one row */ + GROUP BY + dtp.processed + /* Just to get one row */ OPTION(RECOMPILE); END; /* End of @get_all_databases = 1 section */ END TRY From 247663a2052c48eae99f3aa06788125b27b07e3a Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sun, 16 Mar 2025 13:17:31 -0400 Subject: [PATCH 160/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 185 ++++++++++++++-------------- 1 file changed, 94 insertions(+), 91 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index c01fa839..46590eee 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -4301,97 +4301,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. Within each category, indexes are sorted by size and impact for better prioritization. */ - IF @debug = 1 - BEGIN - RAISERROR('Generating #index_cleanup_results, RESULTS', 0, 0) WITH NOWAIT; - END; - - SELECT - /* First, show the information needed to understand the script */ - script_type = CASE WHEN ir.result_type = 'KEPT' AND ir.script_type IS NULL THEN 'KEPT' ELSE ir.script_type END, - ir.additional_info, - /* Then show identifying information for the index */ - ir.database_name, - ir.schema_name, - ir.table_name, - ir.index_name, - /* Then show relationship information */ - ir.consolidation_rule, - ir.target_index_name, - /* Include superseded_by info for winning indexes */ - superseded_info = - CASE - WHEN ia.superseded_by IS NOT NULL - THEN ia.superseded_by - ELSE ir.superseded_info - END, - /* Add size and usage metrics */ - index_size_gb = - CASE - WHEN ir.result_type = 'SUMMARY' - THEN '0.0000' - ELSE FORMAT(ISNULL(ir.index_size_gb, 0), 'N4') - END, - index_rows = - CASE - WHEN ir.result_type = 'SUMMARY' - THEN '0' - ELSE FORMAT(ISNULL(ir.index_rows, 0), 'N0') - END, - index_reads = - CASE - WHEN ir.result_type = 'SUMMARY' - THEN '0' - ELSE FORMAT(ISNULL(ir.index_reads, 0), 'N0') - END, - index_writes = - CASE - WHEN ir.result_type = 'SUMMARY' - THEN '0' - ELSE FORMAT(ISNULL(ir.index_writes, 0), 'N0') - END, - ia.original_index_definition, - /* Finally show the actual script */ - ir.script - FROM - ( - /* Use a subquery with ROW_NUMBER to ensure we only get one row per index */ - SELECT *, - ROW_NUMBER() OVER( - PARTITION BY database_name, schema_name, table_name, index_name - ORDER BY result_type DESC /* Prefer non-NULL result types */ - ) AS rn - FROM #index_cleanup_results - ) AS ir - LEFT JOIN #index_analysis AS ia - ON ir.database_name = ia.database_name - AND ir.schema_name = ia.schema_name - AND ir.table_name = ia.table_name - AND ir.index_name = ia.index_name - WHERE ir.rn = 1 /* Take only the first row for each index */ - ORDER BY - ir.sort_order, - ir.database_name, - /* Within each sort_order group, prioritize by size and usage */ - CASE - /* For SUMMARY, keep the original order */ - WHEN ir.result_type = 'SUMMARY' - THEN 0 - /* For script categories, order by size and impact */ - ELSE ISNULL(ir.index_size_gb, 0) - END DESC, - CASE - /* For SUMMARY, keep the original order */ - WHEN ir.result_type = 'SUMMARY' - THEN 0 - /* For script categories, consider rows as secondary sort */ - ELSE ISNULL(ir.index_rows, 0) - END DESC, - /* Then by database, schema, table, index name for consistent ordering */ - ir.schema_name, - ir.table_name, - ir.index_name - OPTION(RECOMPILE); + /* Save the final output query for later - will run after all databases are processed */ /* Insert overall summary information */ IF @debug = 1 @@ -4827,6 +4737,99 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Just to get one row */ OPTION(RECOMPILE); END; /* End of @get_all_databases = 1 section */ + + /* Final unified results output - runs once after all databases processed */ + IF @debug = 1 + BEGIN + RAISERROR('Generating final consolidated output for all databases', 0, 0) WITH NOWAIT; + END; + + SELECT + /* First, show the information needed to understand the script */ + script_type = CASE WHEN ir.result_type = 'KEPT' AND ir.script_type IS NULL THEN 'KEPT' ELSE ir.script_type END, + ir.additional_info, + /* Then show identifying information for the index */ + ir.database_name, + ir.schema_name, + ir.table_name, + ir.index_name, + /* Then show relationship information */ + ir.consolidation_rule, + ir.target_index_name, + /* Include superseded_by info for winning indexes */ + superseded_info = + CASE + WHEN ia.superseded_by IS NOT NULL + THEN ia.superseded_by + ELSE ir.superseded_info + END, + /* Add size and usage metrics */ + index_size_gb = + CASE + WHEN ir.result_type = 'SUMMARY' + THEN '0.0000' + ELSE FORMAT(ISNULL(ir.index_size_gb, 0), 'N4') + END, + index_rows = + CASE + WHEN ir.result_type = 'SUMMARY' + THEN '0' + ELSE FORMAT(ISNULL(ir.index_rows, 0), 'N0') + END, + index_reads = + CASE + WHEN ir.result_type = 'SUMMARY' + THEN '0' + ELSE FORMAT(ISNULL(ir.index_reads, 0), 'N0') + END, + index_writes = + CASE + WHEN ir.result_type = 'SUMMARY' + THEN '0' + ELSE FORMAT(ISNULL(ir.index_writes, 0), 'N0') + END, + ia.original_index_definition, + /* Finally show the actual script */ + ir.script + FROM + ( + /* Use a subquery with ROW_NUMBER to ensure we only get one row per index */ + SELECT *, + ROW_NUMBER() OVER( + PARTITION BY database_name, schema_name, table_name, index_name + ORDER BY result_type DESC /* Prefer non-NULL result types */ + ) AS rn + FROM #index_cleanup_results + ) AS ir + LEFT JOIN #index_analysis AS ia + ON ir.database_name = ia.database_name + AND ir.schema_name = ia.schema_name + AND ir.table_name = ia.table_name + AND ir.index_name = ia.index_name + WHERE ir.rn = 1 /* Take only the first row for each index */ + ORDER BY + ir.sort_order, + ir.database_name, + /* Within each sort_order group, prioritize by size and usage */ + CASE + /* For SUMMARY, keep the original order */ + WHEN ir.result_type = 'SUMMARY' + THEN 0 + /* For script categories, order by size and impact */ + ELSE ISNULL(ir.index_size_gb, 0) + END DESC, + CASE + /* For SUMMARY, keep the original order */ + WHEN ir.result_type = 'SUMMARY' + THEN 0 + /* For script categories, consider rows as secondary sort */ + ELSE ISNULL(ir.index_rows, 0) + END DESC, + /* Then by database, schema, table, index name for consistent ordering */ + ir.schema_name, + ir.table_name, + ir.index_name + OPTION(RECOMPILE); END TRY BEGIN CATCH THROW; From 37f47b097c8a7d790123b1e4db08b32e5f9602d1 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sun, 16 Mar 2025 13:28:05 -0400 Subject: [PATCH 161/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 192 ++++++++++++++-------------- 1 file changed, 97 insertions(+), 95 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 46590eee..6d7268cb 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -4466,7 +4466,103 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. We'll modify the existing query below rather than creating new output panes */ - /* Return streamlined reporting statistics focused on key metrics */ + /* Save the overall analysis report for after all databases are processed */ + + /* Update processed flag for this database */ + UPDATE + #databases_to_process + SET + #databases_to_process.processed = 1, + #databases_to_process.process_date = SYSDATETIME() + WHERE #databases_to_process.database_id = @database_id; + + /* Get next database */ + FETCH NEXT + FROM @database_cursor + INTO + @current_database_id, + @current_database_name; + END; /* End of cursor WHILE loop */ + + IF @debug = 1 + BEGIN + RAISERROR('Finished processing %d databases', 0, 0, @processed_count) WITH NOWAIT; + END; + + /* Create a summary table with database processing info */ + SELECT + database_summary = N'Database Processing Summary', + processed_databases = + N'Processed: ' + + ISNULL + ( + STUFF + ( + ( + SELECT + N', ' + + dtp.database_name + FROM #databases_to_process AS dtp + WHERE dtp.processed = 1 + GROUP BY + dtp.database_name + ORDER BY + dtp.database_name + FOR + XML + PATH(''), + TYPE + ).value('.', 'nvarchar(max)'), + 1, + 2, + N'' + ), + N'None' + ), + skipped_databases = + N'Skipped: ' + + ISNULL + ( + STUFF + ( + ( + SELECT + N', ' + + dtp.database_name + FROM #databases_to_process AS dtp + WHERE dtp.processed = 0 + GROUP BY + dtp.database_name + ORDER BY + dtp.database_name + FOR + XML + PATH(''), + TYPE + ).value('.', 'nvarchar(max)'), + 1, + 2, + N'' + ), + N'None' + ), + stats = + CASE + WHEN @get_all_databases = 1 + THEN N'Total: ' + CONVERT(nvarchar(10), COUNT_BIG(*) OVER()) + + N', Processed: ' + CONVERT(nvarchar(10), SUM(CONVERT(integer, dtp.processed)) OVER()) + + N', Skipped: ' + CONVERT(nvarchar(10), COUNT_BIG(*) OVER() - SUM(CONVERT(integer, dtp.processed)) OVER()) + ELSE N'Single database mode' + END + FROM #databases_to_process AS dtp + WHERE @database_count > 0 /* Return one row with summary data */ + GROUP BY + dtp.processed + /* Just to get one row */ + OPTION(RECOMPILE); + END; /* End of @get_all_databases = 1 section */ + + /* Return consolidated reporting statistics for all databases processed */ IF @debug = 1 BEGIN RAISERROR('Generating #index_reporting_stats, REPORT', 0, 0) WITH NOWAIT; @@ -4643,100 +4739,6 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. irs.schema_name, irs.table_name OPTION(RECOMPILE); - - /* Update processed flag for this database */ - UPDATE - #databases_to_process - SET - #databases_to_process.processed = 1, - #databases_to_process.process_date = SYSDATETIME() - WHERE #databases_to_process.database_id = @database_id; - - /* Get next database */ - FETCH NEXT - FROM @database_cursor - INTO - @current_database_id, - @current_database_name; - END; /* End of cursor WHILE loop */ - - IF @debug = 1 - BEGIN - RAISERROR('Finished processing %d databases', 0, 0, @processed_count) WITH NOWAIT; - END; - - /* Create a summary table with database processing info */ - SELECT - database_summary = N'Database Processing Summary', - processed_databases = - N'Processed: ' + - ISNULL - ( - STUFF - ( - ( - SELECT - N', ' + - dtp.database_name - FROM #databases_to_process AS dtp - WHERE dtp.processed = 1 - GROUP BY - dtp.database_name - ORDER BY - dtp.database_name - FOR - XML - PATH(''), - TYPE - ).value('.', 'nvarchar(max)'), - 1, - 2, - N'' - ), - N'None' - ), - skipped_databases = - N'Skipped: ' + - ISNULL - ( - STUFF - ( - ( - SELECT - N', ' + - dtp.database_name - FROM #databases_to_process AS dtp - WHERE dtp.processed = 0 - GROUP BY - dtp.database_name - ORDER BY - dtp.database_name - FOR - XML - PATH(''), - TYPE - ).value('.', 'nvarchar(max)'), - 1, - 2, - N'' - ), - N'None' - ), - stats = - CASE - WHEN @get_all_databases = 1 - THEN N'Total: ' + CONVERT(nvarchar(10), COUNT_BIG(*) OVER()) + - N', Processed: ' + CONVERT(nvarchar(10), SUM(CONVERT(integer, dtp.processed)) OVER()) + - N', Skipped: ' + CONVERT(nvarchar(10), COUNT_BIG(*) OVER() - SUM(CONVERT(integer, dtp.processed)) OVER()) - ELSE N'Single database mode' - END - FROM #databases_to_process AS dtp - WHERE @database_count > 0 /* Return one row with summary data */ - GROUP BY - dtp.processed - /* Just to get one row */ - OPTION(RECOMPILE); - END; /* End of @get_all_databases = 1 section */ /* Final unified results output - runs once after all databases processed */ IF @debug = 1 From db203f1789acbf2bf329f9a0444666710e1a2e23 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sun, 16 Mar 2025 13:42:05 -0400 Subject: [PATCH 162/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 181 ++++++++++++++++++++++++---- 1 file changed, 157 insertions(+), 24 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 6d7268cb..650da349 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -573,6 +573,14 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. process_date datetime2(7) NULL, PRIMARY KEY CLUSTERED(database_id) ); + + /* Create a table to track databases that were requested but couldn't be processed */ + CREATE TABLE + #skipped_databases + ( + database_name sysname NOT NULL, + reason nvarchar(255) NOT NULL + ); /* Handle multi-database mode */ IF @get_all_databases = 1 @@ -620,27 +628,80 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. WHERE LTRIM(RTRIM(t.i.value('.', 'sysname'))) <> ''; END; - /* Get all eligible databases */ - INSERT INTO - #databases_to_process - ( - database_id, - database_name - ) - SELECT - database_id = d.database_id, - database_name = d.name - FROM sys.databases AS d - WHERE d.name NOT IN (N'master', N'model', N'msdb', N'tempdb', N'rdsadmin') - AND d.state = 0 - AND d.is_in_standby = 0 - AND d.is_read_only = 0 - AND + /* + Check SQL Server engine edition and use appropriate query paths + */ + IF ( - /* If include list is provided, only keep databases in that list */ - (@include_databases IS NULL) OR - (d.name IN (SELECT database_name FROM #database_list)) - ); + SELECT + CONVERT + ( + sysname, + SERVERPROPERTY('EngineEdition') + ) + ) IN (5, 8) /* Azure SQL DB or Managed Instance */ + BEGIN + /* Get all eligible databases for Azure SQL */ + INSERT INTO + #databases_to_process + ( + database_id, + database_name + ) + SELECT + database_id = d.database_id, + database_name = d.name + FROM sys.databases AS d + WHERE d.name NOT IN (N'master', N'model', N'msdb', N'tempdb', N'rdsadmin') + AND d.state = 0 + AND d.is_in_standby = 0 + AND d.is_read_only = 0 + AND d.database_id > 4 /* Skip system databases */ + AND + ( + /* If include list is provided, only keep databases in that list */ + (@include_databases IS NULL) OR + (d.name IN (SELECT database_name FROM #database_list)) + ); + END + ELSE /* Regular SQL Server */ + BEGIN + /* Get all eligible databases with AG primary replica check */ + INSERT INTO + #databases_to_process + ( + database_id, + database_name + ) + SELECT + database_id = d.database_id, + database_name = d.name + FROM sys.databases AS d + WHERE d.name NOT IN (N'master', N'model', N'msdb', N'tempdb', N'rdsadmin') + AND d.state = 0 + AND d.is_in_standby = 0 + AND d.is_read_only = 0 + AND d.database_id > 4 /* Skip system databases */ + /* Add AG check to ensure we only process the primary replica */ + AND NOT EXISTS + ( + SELECT + 1/0 + FROM sys.dm_hadr_availability_replica_states AS s + JOIN sys.availability_databases_cluster AS c + ON s.group_id = c.group_id + AND d.name = c.database_name + WHERE s.is_local <> 1 + AND s.role_desc <> N'PRIMARY' + AND DATABASEPROPERTYEX(c.database_name, N'Updateability') <> N'READ_WRITE' + ) + AND + ( + /* If include list is provided, only keep databases in that list */ + (@include_databases IS NULL) OR + (d.name IN (SELECT database_name FROM #database_list)) + ); + END; /* Remove excluded databases if specified */ IF @exclude_databases IS NOT NULL @@ -687,6 +748,50 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. RAISERROR('Databases to process: %s', 0, 0, @db_list) WITH NOWAIT; END; + /* Track databases that were requested but skipped (for better reporting) */ + IF @include_databases IS NOT NULL + BEGIN + INSERT + #skipped_databases + ( + database_name, + reason + ) + SELECT + database_name = dl.database_name, + reason = + CASE + WHEN d.name IS NULL THEN N'Database does not exist' + WHEN d.state <> 0 THEN N'Database not online' + WHEN d.is_in_standby = 1 THEN N'Database is in standby' + WHEN d.is_read_only = 1 THEN N'Database is read-only' + WHEN d.database_id <= 4 THEN N'System database' + WHEN EXISTS + ( + SELECT + 1/0 + FROM sys.dm_hadr_availability_replica_states AS s + JOIN sys.availability_databases_cluster AS c + ON s.group_id = c.group_id + AND d.name = c.database_name + WHERE s.is_local <> 1 + AND s.role_desc <> N'PRIMARY' + AND DATABASEPROPERTYEX(c.database_name, N'Updateability') <> N'READ_WRITE' + ) THEN N'AG replica issue - not primary or read-write' + ELSE N'Other issue' + END + FROM #database_list AS dl + LEFT JOIN sys.databases AS d + ON dl.database_name = d.name + WHERE NOT EXISTS + ( + SELECT + 1/0 + FROM #databases_to_process AS dp + WHERE dp.database_name = dl.database_name + ); + END; + /* If no databases match criteria, exit */ IF NOT EXISTS (SELECT 1/0 FROM #databases_to_process AS dtp) BEGIN @@ -4519,8 +4624,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ), N'None' ), - skipped_databases = - N'Skipped: ' + + skipped_unprocessed = + N'Skipped (unprocessed): ' + ISNULL ( STUFF @@ -4546,12 +4651,40 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ), N'None' ), + skipped_with_reasons = + N'Skipped (excluded): ' + + ISNULL + ( + STUFF + ( + ( + SELECT + N', ' + + sd.database_name + N' (' + sd.reason + N')' + FROM #skipped_databases AS sd + GROUP BY + sd.database_name, + sd.reason + ORDER BY + sd.database_name + FOR + XML + PATH(''), + TYPE + ).value('.', 'nvarchar(max)'), + 1, + 2, + N'' + ), + N'None' + ), stats = CASE WHEN @get_all_databases = 1 - THEN N'Total: ' + CONVERT(nvarchar(10), COUNT_BIG(*) OVER()) + + THEN N'Total requested: ' + CONVERT(nvarchar(10), COUNT_BIG(*) OVER() + (SELECT COUNT_BIG(*) FROM #skipped_databases)) + N', Processed: ' + CONVERT(nvarchar(10), SUM(CONVERT(integer, dtp.processed)) OVER()) + - N', Skipped: ' + CONVERT(nvarchar(10), COUNT_BIG(*) OVER() - SUM(CONVERT(integer, dtp.processed)) OVER()) + N', Skipped (unprocessed): ' + CONVERT(nvarchar(10), COUNT_BIG(*) OVER() - SUM(CONVERT(integer, dtp.processed)) OVER()) + + N', Skipped (excluded): ' + CONVERT(nvarchar(10), (SELECT COUNT_BIG(*) FROM #skipped_databases)) ELSE N'Single database mode' END FROM #databases_to_process AS dtp From 95e9af0f95f22d1f7fc1294fac6a77a76b3403de Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sun, 16 Mar 2025 13:42:17 -0400 Subject: [PATCH 163/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 6d7268cb..fe2572fb 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -1,14 +1,3 @@ -/* -EXECUTE sp_IndexCleanup - @database_name = 'StackOverflow2013', - @debug = 1; - -EXECUTE sp_IndexCleanup - @database_name = 'StackOverflow2013', - @table_name = 'Users', - @debug = 1 -*/ - SET ANSI_WARNINGS ON; SET ARITHABORT ON; SET CONCAT_NULL_YIELDS_NULL ON; From b22865baba61016582814eb8b708ebbe9071cd3e Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sun, 16 Mar 2025 14:09:19 -0400 Subject: [PATCH 164/246] ambiguity detection if someone includes and excludes a database, they need to fix that. --- sp_IndexCleanup/sp_IndexCleanup.sql | 75 +++++++++++++++++++++++++++++ sp_QuickieStore/sp_QuickieStore.sql | 32 ++++++++++++ 2 files changed, 107 insertions(+) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 0f44c8e6..fc848088 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -615,6 +615,54 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. LTRIM(RTRIM(t.i.value('.', 'sysname'))) FROM @include_xml.nodes('/i') AS t(i) WHERE LTRIM(RTRIM(t.i.value('.', 'sysname'))) <> ''; + + /* Check for databases in both include and exclude lists */ + IF @exclude_databases IS NOT NULL + BEGIN + SELECT + @exclude_xml = + CONVERT + ( + xml, + '' + + REPLACE + ( + @exclude_databases, + ',', + '' + ) + + '' + ); + + /* Build list of conflicting databases */ + DECLARE @conflict_list nvarchar(max) = N''; + + SELECT + @conflict_list = @conflict_list + + LTRIM(RTRIM(t.i.value('.', 'sysname'))) + N', ' + FROM @exclude_xml.nodes('/i') AS t(i) + WHERE LTRIM(RTRIM(t.i.value('.', 'sysname'))) <> '' + AND EXISTS + ( + SELECT 1/0 + FROM #database_list AS dl + WHERE dl.database_name = LTRIM(RTRIM(t.i.value('.', 'sysname'))) + ); + + /* If we found any conflicts, raise an error */ + IF LEN(@conflict_list) > 0 + BEGIN + /* Remove trailing comma and space */ + SET @conflict_list = LEFT(@conflict_list, LEN(@conflict_list) - 2); + + DECLARE @error_msg nvarchar(2000) = + N'The following databases appear in both @include_databases and @exclude_databases, which creates ambiguity: ' + + @conflict_list + N'. Please remove these databases from one of the lists.'; + + RAISERROR(@error_msg, 16, 1); + RETURN; + END; + END; END; /* @@ -780,6 +828,33 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. WHERE dp.database_name = dl.database_name ); END; + + /* Also track explicitly excluded databases */ + IF @exclude_databases IS NOT NULL + BEGIN + INSERT + #skipped_databases + ( + database_name, + reason + ) + SELECT + database_name = LTRIM(RTRIM(t.i.value('.', 'nvarchar(128)'))), + reason = N'Explicitly excluded by @exclude_databases parameter' + FROM + ( + SELECT xml_list = CONVERT(xml, N'' + + REPLACE(@exclude_databases, N',', N'') + N'') + ) AS a + CROSS APPLY a.xml_list.nodes('i') AS t(i) + WHERE LTRIM(RTRIM(t.i.value('.', 'sysname'))) <> '' + AND EXISTS + ( + SELECT 1/0 + FROM sys.databases AS d + WHERE d.name = LTRIM(RTRIM(t.i.value('.', 'sysname'))) + ); + END; /* If no databases match criteria, exit */ IF NOT EXISTS (SELECT 1/0 FROM #databases_to_process AS dtp) diff --git a/sp_QuickieStore/sp_QuickieStore.sql b/sp_QuickieStore/sp_QuickieStore.sql index 6046c344..a8419d2c 100644 --- a/sp_QuickieStore/sp_QuickieStore.sql +++ b/sp_QuickieStore/sp_QuickieStore.sql @@ -2016,6 +2016,38 @@ BEGIN ) AS a CROSS APPLY x.nodes(N'//i') AS t(c) WHERE LTRIM(RTRIM(c.value(N'(./text())[1]', N'sysname'))) <> N''; + + /* Check for databases in both include and exclude lists */ + IF @include_databases IS NOT NULL + BEGIN + /* Build list of conflicting databases */ + DECLARE @conflict_list nvarchar(max) = N''; + + SELECT + @conflict_list = @conflict_list + + ed.database_name + N', ' + FROM #exclude_databases AS ed + WHERE EXISTS + ( + SELECT 1/0 + FROM #include_databases AS id + WHERE id.database_name = ed.database_name + ); + + /* If we found any conflicts, raise an error */ + IF LEN(@conflict_list) > 0 + BEGIN + /* Remove trailing comma and space */ + SET @conflict_list = LEFT(@conflict_list, LEN(@conflict_list) - 2); + + DECLARE @error_msg nvarchar(2000) = + N'The following databases appear in both @include_databases and @exclude_databases, which creates ambiguity: ' + + @conflict_list + N'. Please remove these databases from one of the lists.'; + + RAISERROR(@error_msg, 16, 1); + RETURN; + END; + END; END; END; From 42f1af3af37bce19d1dbd20418a6c9bcc1574d55 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sun, 16 Mar 2025 14:52:24 -0400 Subject: [PATCH 165/246] Update sp_IndexCleanup.sql Formatting cleanup. I am sloppy sometimes. --- sp_IndexCleanup/sp_IndexCleanup.sql | 503 +++++++++++++++++----------- 1 file changed, 301 insertions(+), 202 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index fc848088..b3aa1f75 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -225,11 +225,32 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. @can_compress bit = CASE WHEN - CONVERT(integer, SERVERPROPERTY('EngineEdition')) IN (3, 5, 8) + CONVERT + ( + integer, + SERVERPROPERTY('EngineEdition') + ) IN (3, 5, 8) OR ( - CONVERT(integer, SERVERPROPERTY('EngineEdition')) = 2 - AND CONVERT(integer, SUBSTRING(CONVERT(varchar(20), SERVERPROPERTY('ProductVersion')), 1, 2)) >= 13 + CONVERT + ( + integer, + SERVERPROPERTY('EngineEdition') + ) = 2 + AND CONVERT + ( + integer, + SUBSTRING + ( + CONVERT + ( + varchar(20), + SERVERPROPERTY('ProductVersion') + ), + 1, + 2 + ) + ) >= 13 ) THEN 1 ELSE 0 @@ -249,11 +270,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. @database_cursor cursor, @current_database_id integer, @current_database_name sysname, - @database_count integer, + @database_count integer = 0, @processed_count integer = 0, - @db_list nvarchar(MAX), - @include_xml xml, - @exclude_xml xml; + @db_list nvarchar(MAX) = N'', + @include_xml xml = N'', + @exclude_xml xml = N'', + @conflict_list nvarchar(max) = N'', + @error_msg nvarchar(2000); /* Set uptime warning flag after @uptime_days is calculated */ SELECT @@ -634,17 +657,18 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. '' ); - /* Build list of conflicting databases */ - DECLARE @conflict_list nvarchar(max) = N''; - + /* Build list of conflicting databases */ SELECT - @conflict_list = @conflict_list + - LTRIM(RTRIM(t.i.value('.', 'sysname'))) + N', ' + @conflict_list = + @conflict_list + + LTRIM(RTRIM(t.i.value('.', 'sysname'))) + + N', ' FROM @exclude_xml.nodes('/i') AS t(i) WHERE LTRIM(RTRIM(t.i.value('.', 'sysname'))) <> '' AND EXISTS ( - SELECT 1/0 + SELECT + 1/0 FROM #database_list AS dl WHERE dl.database_name = LTRIM(RTRIM(t.i.value('.', 'sysname'))) ); @@ -655,11 +679,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Remove trailing comma and space */ SET @conflict_list = LEFT(@conflict_list, LEN(@conflict_list) - 2); - DECLARE @error_msg nvarchar(2000) = + SET @error_msg = N'The following databases appear in both @include_databases and @exclude_databases, which creates ambiguity: ' + @conflict_list + N'. Please remove these databases from one of the lists.'; - RAISERROR(@error_msg, 16, 1); + RAISERROR(@error_msg, 16, 1) WITH NOWAIT; RETURN; END; END; @@ -745,7 +769,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. BEGIN SELECT @exclude_xml = CONVERT(xml, '' + REPLACE(@exclude_databases, ',', '') + ''); - DELETE dp + DELETE + dp FROM #databases_to_process AS dp WHERE EXISTS ( @@ -753,7 +778,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 1/0 FROM @exclude_xml.nodes('/i') AS t(i) WHERE dp.database_name = LTRIM(RTRIM(t.i.value('.', 'sysname'))) - AND LTRIM(RTRIM(t.i.value('.', 'sysname'))) <> '' + AND LTRIM(RTRIM(t.i.value('.', 'sysname'))) <> '' ); END; @@ -850,7 +875,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. WHERE LTRIM(RTRIM(t.i.value('.', 'sysname'))) <> '' AND EXISTS ( - SELECT 1/0 + SELECT + 1/0 FROM sys.databases AS d WHERE d.name = LTRIM(RTRIM(t.i.value('.', 'sysname'))) ); @@ -900,11 +926,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. database_name = d.name FROM sys.databases AS d WHERE d.name = @database_name - AND d.state_desc = 'ONLINE' + AND d.state_desc = N'ONLINE' OPTION(RECOMPILE); /* Validate the database exists and is accessible */ - IF NOT EXISTS (SELECT 1 FROM #databases_to_process) + IF NOT EXISTS (SELECT 1/0 FROM #databases_to_process AS dtp) BEGIN RAISERROR('The specified database %s does not exist, is not in ONLINE state, or you do not have permission to access it', 16, 1, @database_name) WITH NOWAIT; RETURN; @@ -929,8 +955,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. IF @get_all_databases = 1 BEGIN /* Get the count of databases for reporting */ - SELECT @database_count = COUNT(*) - FROM #databases_to_process; + SELECT @database_count = COUNT_BIG(*) + FROM #databases_to_process AS dtp + OPTION(RECOMPILE); IF @debug = 1 BEGIN @@ -1112,7 +1139,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. WHERE v.object_id = i.object_id )'; - IF /* Check SQL Server 2016+ for temporal tables support */ + IF /* Check for temporal tables support */ ( CONVERT ( @@ -1162,8 +1189,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. JOIN ' + QUOTENAME(@database_name) + N'.sys.allocation_units AS au ON ps.partition_id = au.container_id WHERE ps.object_id = t.object_id - GROUP - BY ps.object_id + GROUP BY + ps.object_id HAVING SUM(au.total_pages) * 8.0 / 1048576.0 >= @min_size_gb ) @@ -1174,8 +1201,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. FROM ' + QUOTENAME(@database_name) + N'.sys.dm_db_partition_stats AS ps WHERE ps.object_id = t.object_id AND ps.index_id IN (0, 1) - GROUP - BY ps.object_id + GROUP BY + ps.object_id HAVING SUM(ps.row_count) >= @min_rows ) @@ -1287,9 +1314,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. UPDATE #compression_eligibility SET - can_compress = 0, - reason = N'SQL Server edition or version does not support compression' - WHERE can_compress = 1 + #compression_eligibility.can_compress = 0, + #compression_eligibility.reason = N'SQL Server edition or version does not support compression' + WHERE #compression_eligibility.can_compress = 1 OPTION(RECOMPILE); END; @@ -2154,14 +2181,14 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SET #index_analysis.index_priority = CASE - WHEN index_id = 1 + WHEN #index_analysis.index_id = 1 THEN 1000 /* Clustered indexes get highest priority */ ELSE 0 END + CASE /* Unique indexes get high priority, but reduce priority for unique constraints */ - WHEN is_unique = 1 AND NOT EXISTS + WHEN #index_analysis.is_unique = 1 AND NOT EXISTS ( SELECT 1/0 @@ -2171,7 +2198,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND id_uc.is_unique_constraint = 1 ) THEN 500 /* Unique constraints get lower priority */ - WHEN is_unique = 1 AND EXISTS + WHEN #index_analysis.is_unique = 1 AND EXISTS ( SELECT 1/0 @@ -2260,7 +2287,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. UPDATE ia1 SET - ia1.consolidation_rule = 'Exact Duplicate', + ia1.consolidation_rule = N'Exact Duplicate', ia1.target_index_name = CASE WHEN ia1.index_priority > ia2.index_priority @@ -2361,8 +2388,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND ia1.key_columns = ia2.key_columns /* Exact key match */ AND ISNULL(ia1.included_columns, '') = ISNULL(ia2.included_columns, '') /* Exact includes match */ AND ISNULL(ia1.filter_definition, '') = ISNULL(ia2.filter_definition, '') /* Matching filters */ - WHERE ia1.consolidation_rule = 'Exact Duplicate' - OR ia2.consolidation_rule = 'Exact Duplicate' + WHERE ia1.consolidation_rule = N'Exact Duplicate' + OR ia2.consolidation_rule = N'Exact Duplicate' ORDER BY ia1.index_name OPTION(RECOMPILE); END; @@ -2371,7 +2398,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. UPDATE ia1 SET - ia1.consolidation_rule = 'Key Duplicate', + ia1.consolidation_rule = N'Key Duplicate', ia1.target_index_name = CASE /* If one is unique and the other isn't, prefer the unique one */ @@ -2477,7 +2504,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. UPDATE ia1 SET - ia1.consolidation_rule = 'Key Subset', + ia1.consolidation_rule = N'Key Subset', ia1.target_index_name = ia2.index_name, ia1.action = N'DISABLE' /* The narrower index gets disabled */ FROM #index_analysis AS ia1 @@ -2526,7 +2553,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. UPDATE ia2 SET - ia2.consolidation_rule = 'Key Superset', + ia2.consolidation_rule = N'Key Superset', ia2.action = N'MERGE INCLUDES', /* The wider index gets merged with includes */ ia2.superseded_by = COALESCE(ia2.superseded_by + ', ', '') + 'Supersedes ' + ia1.index_name FROM #index_analysis AS ia1 @@ -2534,8 +2561,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ON ia1.database_id = ia2.database_id AND ia1.object_id = ia2.object_id AND ia1.target_index_name = ia2.index_name /* Link from Rule 4 */ - WHERE ia1.consolidation_rule = 'Key Subset' - AND ia1.action = 'DISABLE' + WHERE ia1.consolidation_rule = N'Key Subset' + AND ia1.action = N'DISABLE' AND ia2.consolidation_rule IS NULL /* Not already processed */ OPTION(RECOMPILE); @@ -2563,10 +2590,10 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ON superset.database_id = subset.database_id AND superset.object_id = subset.object_id AND subset.target_index_name = superset.index_name - WHERE superset.action = 'MERGE INCLUDES' - AND subset.action = 'DISABLE' - AND superset.consolidation_rule = 'Key Superset' - AND subset.consolidation_rule = 'Key Subset' + WHERE superset.action = N'MERGE INCLUDES' + AND subset.action = N'DISABLE' + AND superset.consolidation_rule = N'Key Superset' + AND subset.consolidation_rule = N'Key Subset' ) UPDATE ia @@ -2575,7 +2602,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CASE /* If both have includes, combine them without duplicates */ WHEN kss.superset_includes IS NOT NULL - AND kss.subset_includes IS NOT NULL + AND kss.subset_includes IS NOT NULL THEN /* Create combined includes using XML method that works with all SQL Server versions */ ( @@ -2586,7 +2613,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ( ( SELECT DISTINCT - N', ' + t.c.value('.', 'sysname') + N', ' + + t.c.value('.', 'sysname') FROM ( /* Create XML from superset includes */ @@ -2623,7 +2651,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ) ) /* If only subset has includes, use those */ - WHEN kss.superset_includes IS NULL AND kss.subset_includes IS NOT NULL + WHEN kss.superset_includes IS NULL + AND kss.subset_includes IS NOT NULL THEN kss.subset_includes /* If only superset has includes or neither has includes, keep superset's includes */ ELSE kss.superset_includes @@ -2633,7 +2662,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ON ia.database_id = kss.database_id AND ia.object_id = kss.object_id AND ia.index_id = kss.index_id - WHERE ia.action = 'MERGE INCLUDES'; + WHERE ia.action = N'MERGE INCLUDES'; IF @debug = 1 BEGIN @@ -2648,17 +2677,17 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. UPDATE ia2 SET - ia2.superseded_by = 'Supersedes ' + ia1.index_name + ia2.superseded_by = N'Supersedes ' + ia1.index_name FROM #index_analysis AS ia1 JOIN #index_analysis AS ia2 ON ia1.database_id = ia2.database_id AND ia1.object_id = ia2.object_id AND ia1.index_name <> ia2.index_name - AND ia2.key_columns LIKE (ia1.key_columns + '%') /* ia2 has wider key that starts with ia1's key */ + AND ia2.key_columns LIKE (ia1.key_columns + N'%') /* ia2 has wider key that starts with ia1's key */ AND ISNULL(ia1.filter_definition, '') = ISNULL(ia2.filter_definition, '') /* Matching filters */ /* Exception: If narrower index is unique and wider is not, they should not be merged */ AND NOT (ia1.is_unique = 1 AND ia2.is_unique = 0) - WHERE ia1.consolidation_rule = 'Key Subset' /* Use records just processed in previous UPDATE */ + WHERE ia1.consolidation_rule = N'Key Subset' /* Use records just processed in previous UPDATE */ AND ia1.target_index_name = ia2.index_name /* Make sure we're updating the right wider index */ OPTION(RECOMPILE); @@ -2675,12 +2704,12 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. UPDATE ia1 SET - ia1.consolidation_rule = 'Unique Constraint Replacement', + ia1.consolidation_rule = N'Unique Constraint Replacement', ia1.action = CASE WHEN ia1.is_unique = 0 - THEN 'MAKE UNIQUE' /* Convert to unique index */ - ELSE 'KEEP' /* Already unique, so just keep it */ + THEN N'MAKE UNIQUE' /* Convert to unique index */ + ELSE N'KEEP' /* Already unique, so just keep it */ END FROM #index_analysis AS ia1 WHERE ia1.consolidation_rule IS NULL /* Not already processed */ @@ -2743,7 +2772,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. UPDATE ia_uc SET - ia_uc.consolidation_rule = 'Unique Constraint Replacement', + ia_uc.consolidation_rule = N'Unique Constraint Replacement', ia_uc.action = N'DISABLE', /* Mark unique constraint for disabling */ ia_uc.target_index_name = ia_nc.index_name /* Point to the nonclustered index that will replace it */ FROM #index_analysis AS ia_uc /* Unique constraint */ @@ -2765,7 +2794,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. UPDATE ia_nc SET - ia_nc.consolidation_rule = 'Unique Constraint Replacement', + ia_nc.consolidation_rule = N'Unique Constraint Replacement', ia_nc.action = N'MAKE UNIQUE', /* Mark nonclustered index to be made unique */ /* CRITICAL: Set target_index_name to NULL to ensure it gets a MERGE script */ ia_nc.target_index_name = NULL @@ -2779,44 +2808,51 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Two conditions for matching: 1. Index key columns exactly match a unique constraint's key columns 2. A unique constraint is already marked for DISABLE and has this index as target */ - (EXISTS ( + EXISTS + ( /* Find unique constraint with matching keys that should be disabled */ - SELECT 1 + SELECT + 1/0 FROM #index_analysis AS ia_uc JOIN #index_details AS id_uc ON id_uc.database_id = ia_uc.database_id AND id_uc.object_id = ia_uc.object_id AND id_uc.index_id = ia_uc.index_id AND id_uc.is_unique_constraint = 1 - WHERE - ia_uc.database_id = ia_nc.database_id - AND ia_uc.object_id = ia_nc.object_id - /* Check that both indexes have EXACTLY the same key columns */ - AND ia_uc.key_columns = ia_nc.key_columns - )) + WHERE ia_uc.database_id = ia_nc.database_id + AND ia_uc.object_id = ia_nc.object_id + /* Check that both indexes have EXACTLY the same key columns */ + AND ia_uc.key_columns = ia_nc.key_columns + ) OPTION(RECOMPILE); /* CRITICAL: Ensure that only the unique constraints that exactly match get this treatment */ /* And remove any incorrect MAKE UNIQUE actions */ - UPDATE ia - SET action = NULL, - consolidation_rule = NULL, - target_index_name = NULL + UPDATE + ia + SET + ia.action = NULL, + ia.consolidation_rule = NULL, + ia.target_index_name = NULL FROM #index_analysis AS ia WHERE ia.action = N'MAKE UNIQUE' - AND NOT EXISTS ( + AND NOT EXISTS + ( /* Check if there's a unique constraint with matching keys that points to this index */ - SELECT 1 + SELECT + 1/0 FROM #index_analysis AS ia_uc WHERE ia_uc.database_id = ia.database_id - AND ia_uc.object_id = ia.object_id - AND ia_uc.key_columns = ia.key_columns - AND ia_uc.action = N'DISABLE' - AND ia_uc.target_index_name = ia.index_name - ); + AND ia_uc.object_id = ia.object_id + AND ia_uc.key_columns = ia.key_columns + AND ia_uc.action = N'DISABLE' + AND ia_uc.target_index_name = ia.index_name + ) + OPTION(RECOMPILE); /* Make sure the nonclustered index has the superseded_by field set correctly */ - UPDATE ia_nc + UPDATE + ia_nc SET ia_nc.superseded_by = CASE @@ -2843,16 +2879,17 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Special debug for uq_a and uq_i_a */ RAISERROR('Special debug for uq_a and uq_i_a after rule 7.5:', 0, 0) WITH NOWAIT; SELECT - index_name, - action, - consolidation_rule, - target_index_name, - superseded_by, - included_columns, - index_priority - FROM #index_analysis - WHERE index_name IN ('uq_a', 'uq_i_a') - ORDER BY index_name + ia.index_name, + ia.action, + ia.consolidation_rule, + ia.target_index_name, + ia.superseded_by, + ia.included_columns, + ia.index_priority + FROM #index_analysis AS ia + WHERE ia.index_name IN (N'uq_a', N'uq_i_a') + ORDER BY + ia.index_name OPTION(RECOMPILE); /* Check the merge script eligibility */ @@ -2881,7 +2918,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ON ia.database_id = ce.database_id AND ia.object_id = ce.object_id AND ia.index_id = ce.index_id - WHERE ia.index_name = 'uq_i_a' + WHERE ia.index_name = N'uq_i_a' OPTION(RECOMPILE); END; @@ -2891,7 +2928,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. UPDATE ia1 SET - ia1.consolidation_rule = 'Same Keys Different Order', + ia1.consolidation_rule = N'Same Keys Different Order', ia1.action = N'REVIEW', /* These need manual review */ ia1.target_index_name = ia2.index_name /* Reference the partner index */ FROM #index_analysis AS ia1 @@ -2927,10 +2964,10 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. id1.column_name FROM #index_details AS id1 WHERE id1.database_id = ia1.database_id - AND id1.object_id = ia1.object_id - AND id1.index_id = ia1.index_id - AND id1.is_included_column = 0 - AND id1.key_ordinal > 0 + AND id1.object_id = ia1.object_id + AND id1.index_id = ia1.index_id + AND id1.is_included_column = 0 + AND id1.key_ordinal > 0 EXCEPT @@ -3055,14 +3092,12 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. candidate.index_name FROM #index_analysis AS candidate WHERE candidate.database_id = ia.database_id - AND candidate.object_id = ia.object_id - AND candidate.key_columns = ia.key_columns - AND ISNULL(candidate.filter_definition, '') = ISNULL(ia.filter_definition, '') - AND candidate.action = N'MERGE INCLUDES' - AND candidate.consolidation_rule = 'Key Duplicate' + AND candidate.object_id = ia.object_id + AND candidate.key_columns = ia.key_columns + AND ISNULL(candidate.filter_definition, '') = ISNULL(ia.filter_definition, '') + AND candidate.action = N'MERGE INCLUDES' + AND candidate.consolidation_rule = N'Key Duplicate' ORDER BY - /* First prefer indexes with "_Extended" in the name */ - CASE WHEN candidate.index_name LIKE '%\_Extended%' ESCAPE '\' THEN 1 ELSE 0 END DESC, /* Then prefer indexes with more included columns (by length as a proxy) */ LEN(ISNULL(candidate.included_columns, '')) DESC, /* Then alphabetically for stability */ @@ -3073,16 +3108,16 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. STUFF ( ( - SELECT + SELECT DISTINCT N', ' + inner_ia.index_name FROM #index_analysis AS inner_ia WHERE inner_ia.database_id = ia.database_id - AND inner_ia.object_id = ia.object_id - AND inner_ia.key_columns = ia.key_columns - AND ISNULL(inner_ia.filter_definition, '') = ISNULL(ia.filter_definition, '') - AND inner_ia.action = N'MERGE INCLUDES' - AND inner_ia.consolidation_rule = 'Key Duplicate' + AND inner_ia.object_id = ia.object_id + AND inner_ia.key_columns = ia.key_columns + AND ISNULL(inner_ia.filter_definition, '') = ISNULL(ia.filter_definition, '') + AND inner_ia.action = N'MERGE INCLUDES' + AND inner_ia.consolidation_rule = N'Key Duplicate' ORDER BY inner_ia.index_name FOR @@ -3096,7 +3131,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ) FROM #index_analysis AS ia WHERE ia.action = N'MERGE INCLUDES' - AND ia.consolidation_rule = 'Key Duplicate' + AND ia.consolidation_rule = N'Key Duplicate' GROUP BY ia.database_id, ia.object_id, @@ -3125,15 +3160,15 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND ia.key_columns = kdd.base_key_columns AND ISNULL(ia.filter_definition, N'') = kdd.filter_definition WHERE ia.index_name <> kdd.winning_index_name - AND ia.action = N'MERGE INCLUDES' - AND ia.consolidation_rule = 'Key Duplicate' + AND ia.action = N'MERGE INCLUDES' + AND ia.consolidation_rule = N'Key Duplicate' OPTION(RECOMPILE); /* Update the winning index's superseded_by to list all other indexes */ UPDATE ia SET - ia.superseded_by = 'Supersedes ' + + ia.superseded_by = N'Supersedes ' + REPLACE ( kdd.index_list, @@ -3182,13 +3217,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND ia1.index_name <> ia2.index_name AND ia1.action = N'MERGE INCLUDES' AND ia2.action = N'MERGE INCLUDES' - AND ia1.consolidation_rule = 'Key Duplicate' - AND ia2.consolidation_rule = 'Key Duplicate' + AND ia1.consolidation_rule = N'Key Duplicate' + AND ia2.consolidation_rule = N'Key Duplicate' /* Find where subset's includes are contained within superset's includes */ AND ( - ia1.included_columns IS NULL - OR CHARINDEX(ia1.included_columns, ia2.included_columns) > 0 + ia1.included_columns IS NULL + OR CHARINDEX(ia1.included_columns, ia2.included_columns) > 0 ) /* Don't match if lengths are the same (would be exact duplicates) */ AND @@ -3225,8 +3260,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ia.superseded_by = CASE WHEN ia.superseded_by IS NULL - THEN N'Supersedes ' + isd.subset_index_name - ELSE ia.superseded_by + N', ' + isd.subset_index_name + THEN N'Supersedes ' + + isd.subset_index_name + ELSE ia.superseded_by + + N', ' + + isd.subset_index_name END FROM #index_analysis AS ia JOIN #include_subset_dedupe AS isd @@ -3244,8 +3282,6 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. FROM #index_analysis AS ia WHERE ia.action = N'MERGE INCLUDES' AND ia.superseded_by IS NOT NULL - /* Check if the index name contains "Extended" and has more included columns */ - AND (ia.index_name LIKE '%\_Extended%' ESCAPE '\' OR ia.index_name LIKE '%\_Extended' OR ia.index_name LIKE '%_Extended%') /* This should indicate it already has all the needed includes */ AND NOT EXISTS ( @@ -3435,7 +3471,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. WHERE ia.action IN (N'MERGE INCLUDES', N'MAKE UNIQUE') AND ce.can_compress = 1 AND ia.target_index_name IS NULL - ORDER BY ia.index_name + ORDER BY + ia.index_name OPTION(RECOMPILE); END; @@ -3454,35 +3491,54 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ia.is_unique, ia.index_priority, is_unique_constraint = - CASE WHEN EXISTS ( - SELECT 1 - FROM #index_details AS id - WHERE id.database_id = ia.database_id - AND id.object_id = ia.object_id - AND id.index_id = ia.index_id - AND id.is_unique_constraint = 1 - ) THEN 'YES' ELSE 'NO' END, + CASE + WHEN EXISTS + ( + SELECT + 1/0 + FROM #index_details AS id + WHERE id.database_id = ia.database_id + AND id.object_id = ia.object_id + AND id.index_id = ia.index_id + AND id.is_unique_constraint = 1 + ) + THEN 'YES' + ELSE 'NO' + END, make_unique_target = - CASE WHEN EXISTS ( - SELECT 1 - FROM #index_analysis AS ia_make - WHERE ia_make.database_id = ia.database_id - AND ia_make.object_id = ia.object_id - AND ia_make.action = 'MAKE UNIQUE' - AND ia_make.target_index_name = ia.index_name - ) THEN 'YES' ELSE 'NO' END, + CASE + WHEN EXISTS + ( + SELECT + 1/0 + FROM #index_analysis AS ia_make + WHERE ia_make.database_id = ia.database_id + AND ia_make.object_id = ia.object_id + AND ia_make.action = N'MAKE UNIQUE' + AND ia_make.target_index_name = ia.index_name + ) + THEN 'YES' + ELSE 'NO' + END, will_get_script = - CASE WHEN ia.action = 'DISABLE' AND NOT EXISTS ( - SELECT 1 - FROM #index_details AS id_uc - WHERE id_uc.database_id = ia.database_id - AND id_uc.object_id = ia.object_id - AND id_uc.index_id = ia.index_id - AND id_uc.is_unique_constraint = 1 - ) THEN 'YES' ELSE 'NO' END + CASE + WHEN ia.action = N'DISABLE' + AND NOT EXISTS + ( + SELECT + 1/0 + FROM #index_details AS id_uc + WHERE id_uc.database_id = ia.database_id + AND id_uc.object_id = ia.object_id + AND id_uc.index_id = ia.index_id + AND id_uc.is_unique_constraint = 1 + ) + THEN 'YES' + ELSE 'NO' + END FROM #index_analysis AS ia - WHERE ia.index_name LIKE 'ix_filtered_%' OR ia.index_name LIKE 'ix_desc_%' - ORDER BY ia.index_name; + ORDER BY + ia.index_name; /* Debug for all indexes marked with action = DISABLE */ RAISERROR('All indexes with action = DISABLE:', 0, 0) WITH NOWAIT; @@ -3492,8 +3548,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ia.action, ia.target_index_name FROM #index_analysis AS ia - WHERE ia.action = 'DISABLE' - ORDER BY ia.index_name; + WHERE ia.action = N'DISABLE' + ORDER BY + ia.index_name; END; INSERT INTO @@ -3522,7 +3579,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Sort duplicate/subset indexes first (20), then unused indexes last (25) */ sort_order = CASE - WHEN ia.consolidation_rule LIKE 'Unused Index%' THEN 25 + WHEN ia.consolidation_rule LIKE N'Unused Index%' THEN 25 ELSE 20 END, ia.database_name, @@ -3543,12 +3600,15 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. QUOTENAME(ia.table_name) + N' DISABLE;', CASE - WHEN ia.consolidation_rule = 'Key Subset' - THEN N'This index is superseded by a wider index: ' + ISNULL(ia.target_index_name, N'(unknown)') - WHEN ia.consolidation_rule = 'Exact Duplicate' - THEN N'This index is an exact duplicate of: ' + ISNULL(ia.target_index_name, N'(unknown)') - WHEN ia.consolidation_rule = 'Key Duplicate' - THEN N'This index has the same keys as: ' + ISNULL(ia.target_index_name, N'(unknown)') + WHEN ia.consolidation_rule = N'Key Subset' + THEN N'This index is superseded by a wider index: ' + + ISNULL(ia.target_index_name, N'(unknown)') + WHEN ia.consolidation_rule = N'Exact Duplicate' + THEN N'This index is an exact duplicate of: ' + + ISNULL(ia.target_index_name, N'(unknown)') + WHEN ia.consolidation_rule = N'Key Duplicate' + THEN N'This index has the same keys as: ' + + ISNULL(ia.target_index_name, N'(unknown)') WHEN ia.consolidation_rule LIKE 'Unused Index%' THEN ia.consolidation_rule WHEN ia.action = N'DISABLE' @@ -3782,8 +3842,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND id.is_included_column = 0 /* Get only one row per index */ AND id.key_ordinal > 0 /* Check that this index is not already in the results */ - WHERE NOT EXISTS ( - SELECT 1 FROM #index_cleanup_results AS ir + WHERE NOT EXISTS + ( + SELECT + 1/0 + FROM #index_cleanup_results AS ir WHERE ir.database_name = ia.database_name AND ir.schema_name = ia.schema_name AND ir.table_name = ia.table_name @@ -3792,9 +3855,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* And include only indexes that should be kept */ AND ( /* Include indexes marked KEEP */ - (ia.action = 'KEEP') + (ia.action = N'KEEP') /* And all indexes we haven't determined an action for (not disable, merge, etc.) */ - OR (ia.action IS NULL AND ia.index_id > 0) + OR + ( + ia.action IS NULL + AND ia.index_id > 0 + ) ) OPTION(RECOMPILE); @@ -3864,7 +3931,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Only constraints that are marked for disabling */ ia_uc.action = N'DISABLE' /* That have consolidation_rule of 'Unique Constraint Replacement' */ - AND ia_uc.consolidation_rule = 'Unique Constraint Replacement' + AND ia_uc.consolidation_rule = N'Unique Constraint Replacement' OPTION(RECOMPILE); /* Insert per-partition compression scripts */ @@ -4086,7 +4153,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ia.target_index_name, additional_info = CASE - WHEN ia.consolidation_rule = 'Same Keys Different Order' + WHEN ia.consolidation_rule = N'Same Keys Different Order' THEN N'This index has the same key columns as ' + ISNULL(ia.target_index_name, N'(unknown)') + N' but in a different order. May be redundant depending on query patterns.' ELSE N'This index needs manual review' @@ -4284,15 +4351,15 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. leaf_delete_count = SUM(os.leaf_delete_count) FROM #partition_stats AS ps LEFT JOIN #index_details AS id - ON id.database_id = ps.database_id - AND id.object_id = ps.object_id - AND id.index_id = ps.index_id - AND id.is_included_column = 0 - AND id.key_ordinal > 0 + ON id.database_id = ps.database_id + AND id.object_id = ps.object_id + AND id.index_id = ps.index_id + AND id.is_included_column = 0 + AND id.key_ordinal > 0 LEFT JOIN #operational_stats AS os - ON os.database_id = ps.database_id - AND os.object_id = ps.object_id - AND os.index_id = ps.index_id + ON os.database_id = ps.database_id + AND os.object_id = ps.object_id + AND os.index_id = ps.index_id OUTER APPLY ( /* Get actual row count per table using MAX from clustered index/heap */ @@ -4433,15 +4500,15 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. leaf_delete_count = SUM(os.leaf_delete_count) FROM #partition_stats AS ps LEFT JOIN #index_details AS id - ON id.database_id = ps.database_id - AND id.object_id = ps.object_id - AND id.index_id = ps.index_id - AND id.is_included_column = 0 - AND id.key_ordinal > 0 + ON id.database_id = ps.database_id + AND id.object_id = ps.object_id + AND id.index_id = ps.index_id + AND id.is_included_column = 0 + AND id.key_ordinal > 0 LEFT JOIN #operational_stats AS os - ON os.database_id = ps.database_id - AND os.object_id = ps.object_id - AND os.index_id = ps.index_id + ON os.database_id = ps.database_id + AND os.object_id = ps.object_id + AND os.index_id = ps.index_id GROUP BY ps.database_name, ps.database_id, @@ -4724,7 +4791,10 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ( SELECT N', ' + - sd.database_name + N' (' + sd.reason + N')' + sd.database_name + + N' (' + + sd.reason + + N')' FROM #skipped_databases AS sd GROUP BY sd.database_name, @@ -4745,17 +4815,22 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. stats = CASE WHEN @get_all_databases = 1 - THEN N'Total requested: ' + CONVERT(nvarchar(10), COUNT_BIG(*) OVER() + (SELECT COUNT_BIG(*) FROM #skipped_databases)) + - N', Processed: ' + CONVERT(nvarchar(10), SUM(CONVERT(integer, dtp.processed)) OVER()) + - N', Skipped (unprocessed): ' + CONVERT(nvarchar(10), COUNT_BIG(*) OVER() - SUM(CONVERT(integer, dtp.processed)) OVER()) + - N', Skipped (excluded): ' + CONVERT(nvarchar(10), (SELECT COUNT_BIG(*) FROM #skipped_databases)) + THEN N'Total requested: ' + + CONVERT(nvarchar(10), COUNT_BIG(*) OVER() + + (SELECT COUNT_BIG(*) FROM #skipped_databases AS sd)) + + N', Processed: ' + + CONVERT(nvarchar(10), SUM(CONVERT(integer, dtp.processed)) OVER()) + + N', Skipped (unprocessed): ' + + CONVERT(nvarchar(10), COUNT_BIG(*) OVER() - + SUM(CONVERT(integer, dtp.processed)) OVER()) + + N', Skipped (excluded): ' + + CONVERT(nvarchar(10), (SELECT COUNT_BIG(*) FROM #skipped_databases AS sd)) ELSE N'Single database mode' END FROM #databases_to_process AS dtp WHERE @database_count > 0 /* Return one row with summary data */ GROUP BY dtp.processed - /* Just to get one row */ OPTION(RECOMPILE); END; /* End of @get_all_databases = 1 section */ @@ -4769,7 +4844,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Basic identification with enhanced naming */ level = CASE - WHEN irs.summary_level = 'SUMMARY' THEN '=== OVERALL ANALYSIS ===' + WHEN irs.summary_level = 'SUMMARY' + THEN '=== OVERALL ANALYSIS ===' ELSE irs.summary_level END, @@ -4816,7 +4892,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. mergeable_indexes = FORMAT(ISNULL(irs.indexes_to_merge, 0), 'N0'), /* Percent of indexes that can be removed */ - pct_removable = + percent_removable = CASE WHEN irs.summary_level = 'SUMMARY' AND irs.index_count > 0 THEN FORMAT(100.0 * ISNULL(irs.indexes_to_disable, 0) / NULLIF(irs.index_count, 0), 'N1') + '%' @@ -4841,7 +4917,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END, /* Space reduction percentage - added this as new metric */ - space_reduction_pct = + space_reduction_percent = CASE WHEN ISNULL(irs.total_size_gb, 0) > 0 THEN FORMAT((ISNULL(irs.space_saved_gb, 0) / NULLIF(irs.total_size_gb, 0)) * 100, 'N1') + '%' @@ -4858,9 +4934,12 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. WHEN irs.summary_level <> 'SUMMARY' THEN FORMAT(ISNULL(irs.total_reads, 0), 'N0') + ' (' + - FORMAT(ISNULL(irs.user_seeks, 0), 'N0') + ' seeks, ' + - FORMAT(ISNULL(irs.user_scans, 0), 'N0') + ' scans, ' + - FORMAT(ISNULL(irs.user_lookups, 0), 'N0') + ' lookups)' + FORMAT(ISNULL(irs.user_seeks, 0), 'N0') + + ' seeks, ' + + FORMAT(ISNULL(irs.user_scans, 0), 'N0') + + ' scans, ' + + FORMAT(ISNULL(irs.user_lookups, 0), 'N0') + + ' lookups)' ELSE 'N/A' END, @@ -4877,7 +4956,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CASE WHEN irs.summary_level <> 'SUMMARY' THEN FORMAT(ISNULL(irs.user_updates / NULLIF(CONVERT(DECIMAL(10,2), - (SELECT TOP 1 server_uptime_days FROM #index_reporting_stats WHERE summary_level = 'DATABASE')), 0) * + (SELECT TOP (1) irs2.server_uptime_days FROM #index_reporting_stats AS irs2 WHERE irs2.summary_level = 'DATABASE')), 0) * (ISNULL(irs.unused_indexes, 0) / NULLIF(CONVERT(DECIMAL(10,2), irs.index_count), 0)), 0), 'N0') ELSE 'N/A' END, @@ -4896,9 +4975,12 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. avg_lock_wait_ms = CASE WHEN irs.summary_level <> 'SUMMARY' - AND (ISNULL(irs.row_lock_wait_count, 0) + ISNULL(irs.page_lock_wait_count, 0)) > 0 - THEN FORMAT(1.0 * (ISNULL(irs.row_lock_wait_in_ms, 0) + ISNULL(irs.page_lock_wait_in_ms, 0)) / - NULLIF(ISNULL(irs.row_lock_wait_count, 0) + ISNULL(irs.page_lock_wait_count, 0), 0), 'N2') + AND (ISNULL(irs.row_lock_wait_count, 0) + + ISNULL(irs.page_lock_wait_count, 0)) > 0 + THEN FORMAT(1.0 * (ISNULL(irs.row_lock_wait_in_ms, 0) + + ISNULL(irs.page_lock_wait_in_ms, 0)) / + NULLIF(ISNULL(irs.row_lock_wait_count, 0) + + ISNULL(irs.page_lock_wait_count, 0), 0), 'N2') ELSE '0.00' END, @@ -4906,9 +4988,12 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. avg_latch_wait_ms = CASE WHEN irs.summary_level <> 'SUMMARY' - AND (ISNULL(irs.page_latch_wait_count, 0) + ISNULL(irs.page_io_latch_wait_count, 0)) > 0 - THEN FORMAT(1.0 * (ISNULL(irs.page_latch_wait_in_ms, 0) + ISNULL(irs.page_io_latch_wait_in_ms, 0)) / - NULLIF(ISNULL(irs.page_latch_wait_count, 0) + ISNULL(irs.page_io_latch_wait_count, 0), 0), 'N2') + AND (ISNULL(irs.page_latch_wait_count, 0) + + ISNULL(irs.page_io_latch_wait_count, 0)) > 0 + THEN FORMAT(1.0 * (ISNULL(irs.page_latch_wait_in_ms, 0) + + ISNULL(irs.page_io_latch_wait_in_ms, 0)) / + NULLIF(ISNULL(irs.page_latch_wait_count, 0) + + ISNULL(irs.page_io_latch_wait_count, 0), 0), 'N2') ELSE '0.00' END FROM #index_reporting_stats AS irs @@ -4945,7 +5030,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SELECT /* First, show the information needed to understand the script */ - script_type = CASE WHEN ir.result_type = 'KEPT' AND ir.script_type IS NULL THEN 'KEPT' ELSE ir.script_type END, + script_type = + CASE + WHEN ir.result_type = 'KEPT' + AND ir.script_type IS NULL + THEN 'KEPT' + ELSE ir.script_type + END, ir.additional_info, /* Then show identifying information for the index */ ir.database_name, @@ -4993,12 +5084,20 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. FROM ( /* Use a subquery with ROW_NUMBER to ensure we only get one row per index */ - SELECT *, - ROW_NUMBER() OVER( - PARTITION BY database_name, schema_name, table_name, index_name - ORDER BY result_type DESC /* Prefer non-NULL result types */ - ) AS rn - FROM #index_cleanup_results + SELECT + icr.*, + rn = + ROW_NUMBER() OVER + ( + PARTITION BY + icr.database_name, + icr.schema_name, + icr.table_name, + icr.index_name + ORDER BY + icr.result_type DESC /* Prefer non-NULL result types */ + ) + FROM #index_cleanup_results AS icr ) AS ir LEFT JOIN #index_analysis AS ia ON ir.database_name = ia.database_name From 19d7211f68522850680c3c6b4ca67d065bb6b28b Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sun, 16 Mar 2025 22:49:51 -0400 Subject: [PATCH 166/246] Update sp_QuickieStore.sql --- sp_QuickieStore/sp_QuickieStore.sql | 138 ++++++++++++++++------------ 1 file changed, 81 insertions(+), 57 deletions(-) diff --git a/sp_QuickieStore/sp_QuickieStore.sql b/sp_QuickieStore/sp_QuickieStore.sql index a8419d2c..a6c9e9b2 100644 --- a/sp_QuickieStore/sp_QuickieStore.sql +++ b/sp_QuickieStore/sp_QuickieStore.sql @@ -1160,13 +1160,13 @@ CREATE TABLE #query_store_plan_feedback ( database_id integer NOT NULL, - plan_feedback_id bigint, - plan_id bigint, - feature_desc nvarchar(120), - feedback_data nvarchar(MAX), - state_desc nvarchar(120), - create_time datetimeoffset(7), - last_updated_time datetimeoffset(7) + plan_feedback_id bigint NOT NULL, + plan_id bigint NULL, + feature_desc nvarchar(120) NULL, + feedback_data nvarchar(MAX) NULL, + state_desc nvarchar(120) NULL, + create_time datetimeoffset(7) NOT NULL, + last_updated_time datetimeoffset(7) NULL ); /* @@ -1176,12 +1176,12 @@ CREATE TABLE #query_store_query_hints ( database_id integer NOT NULL, - query_hint_id bigint, - query_id bigint, - query_hint_text nvarchar(MAX), - last_query_hint_failure_reason_desc nvarchar(256), - query_hint_failure_count bigint, - source_desc nvarchar(256) + query_hint_id bigint NOT NULL, + query_id bigint NOT NULL, + query_hint_text nvarchar(MAX) NULL, + last_query_hint_failure_reason_desc nvarchar(256) NULL, + query_hint_failure_count bigint NOT NULL, + source_desc nvarchar(256) NULL ); /* @@ -1191,21 +1191,21 @@ CREATE TABLE #query_store_query_variant ( database_id integer NOT NULL, - query_variant_query_id bigint, - parent_query_id bigint, - dispatcher_plan_id bigint + query_variant_query_id bigint NOT NULL, + parent_query_id bigint NOT NULL, + dispatcher_plan_id bigint NOT NULL ); /* Replicants */ CREATE TABLE - #query_store_replicas + #query_store_replicas ( database_id integer NOT NULL, - replica_group_id bigint, - role_type smallint, - replica_name nvarchar(1288) + replica_group_id bigint NOT NULL, + role_type smallint NOT NULL, + replica_name nvarchar(1288) NULL ); /* @@ -1215,10 +1215,10 @@ CREATE TABLE #query_store_plan_forcing_locations ( database_id integer NOT NULL, - plan_forcing_location_id bigint, - query_id bigint, - plan_id bigint, - replica_group_id bigint + plan_forcing_location_id bigint NOT NULL, + query_id bigint NOT NULL, + plan_id bigint NOT NULL, + replica_group_id bigint NOT NULL ); /* @@ -1227,10 +1227,10 @@ Trouble Loves Me CREATE TABLE #troubleshoot_performance ( - id bigint IDENTITY, - current_table nvarchar(100), - start_time datetime, - end_time datetime, + id bigint IDENTITY PRIMARY KEY CLUSTERED, + current_table nvarchar(100) NOT NULL, + start_time datetime NOT NULL, + end_time datetime NOT NULL, runtime_ms AS FORMAT ( @@ -1241,7 +1241,7 @@ CREATE TABLE end_time ), 'N0' - ) + ) PERSISTED NOT NULL ); /*Gonna try gathering this based on*/ @@ -1680,7 +1680,9 @@ DECLARE @data_type sysname, @is_include bit, @requires_secondary_processing bit, - @split_sql nvarchar(MAX); + @split_sql nvarchar(MAX), + @error_msg nvarchar(2000), + @conflict_list nvarchar(max); /* In cases where we are escaping @query_text_search and @@ -2021,15 +2023,17 @@ BEGIN IF @include_databases IS NOT NULL BEGIN /* Build list of conflicting databases */ - DECLARE @conflict_list nvarchar(max) = N''; + SET @conflict_list = N''; SELECT - @conflict_list = @conflict_list + - ed.database_name + N', ' + @conflict_list = + @conflict_list + + ed.database_name + N', ' FROM #exclude_databases AS ed WHERE EXISTS ( - SELECT 1/0 + SELECT + 1/0 FROM #include_databases AS id WHERE id.database_name = ed.database_name ); @@ -2040,7 +2044,7 @@ BEGIN /* Remove trailing comma and space */ SET @conflict_list = LEFT(@conflict_list, LEN(@conflict_list) - 2); - DECLARE @error_msg nvarchar(2000) = + SET @error_msg = N'The following databases appear in both @include_databases and @exclude_databases, which creates ambiguity: ' + @conflict_list + N'. Please remove these databases from one of the lists.'; @@ -2103,6 +2107,8 @@ BEGIN BEGIN INSERT #requested_but_skipped_databases + WITH + (TABLOCK) ( database_name, reason @@ -2128,7 +2134,8 @@ BEGIN 1/0 FROM #databases AS db WHERE db.database_name = id.database_name - ); + ) + OPTION(RECOMPILE); END; END; ELSE @@ -2184,6 +2191,8 @@ BEGIN BEGIN INSERT #requested_but_skipped_databases + WITH + (TABLOCK) ( database_name, reason @@ -2221,7 +2230,8 @@ BEGIN 1/0 FROM #databases AS db WHERE db.database_name = id.database_name - ); + ) + OPTION(RECOMPILE); END; END; @@ -2671,19 +2681,19 @@ We set both _date_original variables earlier. */ SELECT @regression_baseline_start_date = - DATEADD - ( - MINUTE, - @utc_minutes_difference, - @regression_baseline_start_date_original - ), + DATEADD + ( + MINUTE, + @utc_minutes_difference, + @regression_baseline_start_date_original + ), @regression_baseline_end_date = - DATEADD - ( - MINUTE, - @utc_minutes_difference, - @regression_baseline_end_date_original - ), + DATEADD + ( + MINUTE, + @utc_minutes_difference, + @regression_baseline_end_date_original + ), @regression_comparator = ISNULL(@regression_comparator, 'absolute'), @regression_direction = @@ -3009,7 +3019,8 @@ FROM ' + @database_name_quoted + N'.sys.procedures AS p JOIN ' + @database_name_quoted + N'.sys.schemas AS s ON p.schema_id = s.schema_id WHERE s.name = @procedure_schema -AND p.name LIKE @procedure_name;' + @nc10; +AND p.name LIKE @procedure_name +OPTION(RECOMPILE);' + @nc10; IF @debug = 1 BEGIN @@ -3991,7 +4002,7 @@ BEGIN (TABLOCK) ( ' + @column_name + - N') + N') EXECUTE sys.sp_executesql @split_sql, N''@ids nvarchar(4000)'', @@ -4107,7 +4118,7 @@ BEGIN END; ELSE IF - @param_name = 'include_sql_handles' + @param_name = 'include_sql_handles' OR @param_name = 'ignore_sql_handles' BEGIN SELECT @secondary_sql = N' @@ -5547,7 +5558,11 @@ BEGIN qsq.query_hash, /* All of these but count_executions are already floats. */ regression_metric_average = - CONVERT(float, AVG(' + + CONVERT + ( + float, + AVG + (' + CASE @sort_order WHEN 'cpu' THEN N'qsrs.avg_cpu_time' WHEN 'logical reads' THEN N'qsrs.avg_logical_io_reads' @@ -5560,7 +5575,9 @@ BEGIN WHEN 'rows' THEN N'qsrs.avg_rowcount' ELSE CASE WHEN @sort_order_is_a_wait = 1 THEN N'waits.total_query_wait_time_ms' ELSE N'qsrs.avg_cpu_time' END END - + N')) + + N' + ) + ) FROM ' + @database_name_quoted + N'.sys.query_store_query AS qsq JOIN ' + @database_name_quoted + N'.sys.query_store_plan AS qsp ON qsq.query_id = qsp.query_id @@ -5647,7 +5664,11 @@ BEGIN qsq.query_hash, /* All of these but count_executions are already floats. */ current_metric_average = - CONVERT(float, AVG(' + + CONVERT + ( + float, + AVG + (' + CASE @sort_order WHEN 'cpu' THEN N'qsrs.avg_cpu_time' WHEN 'logical reads' THEN N'qsrs.avg_logical_io_reads' @@ -5660,7 +5681,9 @@ BEGIN WHEN 'rows' THEN N'qsrs.avg_rowcount' ELSE CASE WHEN @sort_order_is_a_wait = 1 THEN N'waits.total_query_wait_time_ms' ELSE N'qsrs.avg_cpu_time' END END - + N')) + + N' + ) + ) FROM ' + @database_name_quoted + N'.sys.query_store_query AS qsq JOIN ' + @database_name_quoted + N'.sys.query_store_plan AS qsp ON qsq.query_id = qsp.query_id @@ -9127,7 +9150,8 @@ BEGIN JOIN #query_store_plan_forcing_locations AS qspfl ON qsr.replica_group_id = qspfl.replica_group_id ORDER BY - qsr.replica_group_id; + qsr.replica_group_id + OPTION(RECOMPILE);; END; ELSE BEGIN From d328b42392b35416b878fb6ba71c237e82578ef4 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sun, 16 Mar 2025 23:13:39 -0400 Subject: [PATCH 167/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 116 +++++++++++++++++++--------- 1 file changed, 80 insertions(+), 36 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index b3aa1f75..fe225a09 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -24,8 +24,8 @@ ALTER PROCEDURE @min_size_gb decimal(10,2) = 0, @min_rows bigint = 0, @get_all_databases bit = 0, /* When 1, analyzes all eligible databases on the server */ - @include_databases nvarchar(MAX) = NULL, /* Comma-separated list of databases to include (used with @get_all_databases = 1) */ - @exclude_databases nvarchar(MAX) = NULL, /* Comma-separated list of databases to exclude (used with @get_all_databases = 1) */ + @include_databases nvarchar(max) = NULL, /* Comma-separated list of databases to include (used with @get_all_databases = 1) */ + @exclude_databases nvarchar(max) = NULL, /* Comma-separated list of databases to exclude (used with @get_all_databases = 1) */ @help bit = 'false', @debug bit = 'false', @version varchar(20) = NULL OUTPUT, @@ -267,12 +267,12 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. FROM sys.dm_os_sys_info AS osi ), /* Variables for multi-database processing */ - @database_cursor cursor, + @database_cursor CURSOR, @current_database_id integer, @current_database_name sysname, @database_count integer = 0, @processed_count integer = 0, - @db_list nvarchar(MAX) = N'', + @db_list nvarchar(max) = N'', @include_xml xml = N'', @exclude_xml xml = N'', @conflict_list nvarchar(max) = N'', @@ -287,14 +287,6 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ELSE 0 END; - /* - Initial checks for object validity - */ - IF @debug = 1 - BEGIN - RAISERROR('Checking paramaters...', 0, 0) WITH NOWAIT; - END; - /* Temp tables! */ @@ -637,7 +629,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. database_name = LTRIM(RTRIM(t.i.value('.', 'sysname'))) FROM @include_xml.nodes('/i') AS t(i) - WHERE LTRIM(RTRIM(t.i.value('.', 'sysname'))) <> ''; + WHERE LTRIM(RTRIM(t.i.value('.', 'sysname'))) <> '' + OPTION(RECOMPILE); /* Check for databases in both include and exclude lists */ IF @exclude_databases IS NOT NULL @@ -671,7 +664,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 1/0 FROM #database_list AS dl WHERE dl.database_name = LTRIM(RTRIM(t.i.value('.', 'sysname'))) - ); + ) + OPTION(RECOMPILE);; /* If we found any conflicts, raise an error */ IF LEN(@conflict_list) > 0 @@ -704,7 +698,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. BEGIN /* Get all eligible databases for Azure SQL */ INSERT INTO - #databases_to_process + #databases_to_process + WITH + (TABLOCK) ( database_id, database_name @@ -723,13 +719,16 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* If include list is provided, only keep databases in that list */ (@include_databases IS NULL) OR (d.name IN (SELECT database_name FROM #database_list)) - ); + ) + OPTION(RECOMPILE); END ELSE /* Regular SQL Server */ BEGIN /* Get all eligible databases with AG primary replica check */ INSERT INTO #databases_to_process + WITH + (TABLOCK) ( database_id, database_name @@ -761,7 +760,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* If include list is provided, only keep databases in that list */ (@include_databases IS NULL) OR (d.name IN (SELECT database_name FROM #database_list)) - ); + ) + OPTION(RECOMPILE);; END; /* Remove excluded databases if specified */ @@ -779,7 +779,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. FROM @exclude_xml.nodes('/i') AS t(i) WHERE dp.database_name = LTRIM(RTRIM(t.i.value('.', 'sysname'))) AND LTRIM(RTRIM(t.i.value('.', 'sysname'))) <> '' - ); + ) + OPTION(RECOMPILE);; END; IF @debug = 1 @@ -799,7 +800,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. N', ' FROM #databases_to_process AS dtp ORDER BY - dtp.database_name; + dtp.database_name + OPTION(RECOMPILE);; /* Remove trailing comma if list is not empty */ IF LEN(@db_list) > 0 @@ -815,6 +817,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. BEGIN INSERT #skipped_databases + WITH + (TABLOCK) ( database_name, reason @@ -851,7 +855,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 1/0 FROM #databases_to_process AS dp WHERE dp.database_name = dl.database_name - ); + ) + OPTION(RECOMPILE); END; /* Also track explicitly excluded databases */ @@ -859,6 +864,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. BEGIN INSERT #skipped_databases + WITH + (TABLOCK) ( database_name, reason @@ -879,7 +886,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 1/0 FROM sys.databases AS d WHERE d.name = LTRIM(RTRIM(t.i.value('.', 'sysname'))) - ); + ) + OPTION(RECOMPILE);; END; /* If no databases match criteria, exit */ @@ -917,6 +925,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. BEGIN INSERT INTO #databases_to_process + WITH + (TABLOCK) ( database_id, database_name @@ -946,7 +956,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SELECT @database_id = database_id, @database_name = database_name - FROM #databases_to_process; + FROM #databases_to_process + OPTION(RECOMPILE); END; /* @@ -955,7 +966,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. IF @get_all_databases = 1 BEGIN /* Get the count of databases for reporting */ - SELECT @database_count = COUNT_BIG(*) + SELECT + @database_count = COUNT_BIG(*) FROM #databases_to_process AS dtp OPTION(RECOMPILE); @@ -978,7 +990,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. FROM #databases_to_process AS dtp WHERE dtp.processed = 0 ORDER BY - dtp.database_name; + dtp.database_name + OPTION(RECOMPILE); OPEN @database_cursor; FETCH NEXT @@ -1022,6 +1035,12 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. RAISERROR('Processing database %s', 0, 0, @database_name) WITH NOWAIT; END; + /* Checking parameters */ + IF @debug = 1 + BEGIN + RAISERROR('Checking paramaters...', 0, 0) WITH NOWAIT; + END; + /* Continue with current database processing without recreating temp tables */ IF @schema_name IS NULL AND @table_name IS NOT NULL @@ -1220,7 +1239,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. OR SUM(ius.user_updates) >= @min_writes ) - OPTION(RECOMPILE);'; + OPTION(RECOMPILE); + '; IF @debug = 1 BEGIN @@ -1451,7 +1471,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. t.name, os.index_id, i.name - OPTION(RECOMPILE);'; + OPTION(RECOMPILE); + '; IF @debug = 1 BEGIN @@ -1633,7 +1654,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. JOIN ' + QUOTENAME(@database_name) + CONVERT ( - nvarchar(MAX), + nvarchar(max), N'.sys.columns AS c ON ic.object_id = c.object_id AND ic.column_id = c.column_id @@ -1661,7 +1682,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ) + QUOTENAME(@database_name) + CONVERT ( - nvarchar(MAX), + nvarchar(max), N'.sys.dm_db_partition_stats ps WHERE ps.object_id = t.object_id AND ps.index_id = 1 @@ -1689,7 +1710,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND so.is_ms_shipped = 0 AND so.type = N''TF'' ) - OPTION(RECOMPILE);' + OPTION(RECOMPILE); + ' ); IF @debug = 1 @@ -1898,7 +1920,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. '''' ) ) AS pc - OPTION(RECOMPILE);'; + OPTION(RECOMPILE); + '; IF @debug = 1 BEGIN @@ -2662,7 +2685,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ON ia.database_id = kss.database_id AND ia.object_id = kss.object_id AND ia.index_id = kss.index_id - WHERE ia.action = N'MERGE INCLUDES'; + WHERE ia.action = N'MERGE INCLUDES' + OPTION(RECOMPILE); IF @debug = 1 BEGIN @@ -2713,8 +2737,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END FROM #index_analysis AS ia1 WHERE ia1.consolidation_rule IS NULL /* Not already processed */ - AND ia1.action IS NULL /* Not already processed by earlier rules */ - AND EXISTS + AND ia1.action IS NULL /* Not already processed by earlier rules */ + AND EXISTS ( /* Find nonclustered indexes */ SELECT @@ -2725,7 +2749,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND id1.index_id = ia1.index_id AND id1.is_eligible_for_dedupe = 1 ) - AND EXISTS + AND EXISTS ( /* Find unique constraints with matching key columns */ SELECT @@ -3108,7 +3132,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. STUFF ( ( - SELECT DISTINCT + SELECT N', ' + inner_ia.index_name FROM #index_analysis AS inner_ia @@ -3118,6 +3142,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND ISNULL(inner_ia.filter_definition, '') = ISNULL(ia.filter_definition, '') AND inner_ia.action = N'MERGE INCLUDES' AND inner_ia.consolidation_rule = N'Key Duplicate' + GROUP BY + inner_ia.index_name ORDER BY inner_ia.index_name FOR @@ -3172,7 +3198,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. REPLACE ( kdd.index_list, - ia.index_name + N', ', N'' + ia.index_name + + N', ', + N'' ) /* Remove self from list if present */ FROM #index_analysis AS ia JOIN #key_duplicate_dedupe AS kdd @@ -3308,6 +3336,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. INSERT INTO #index_cleanup_results + WITH + (TABLOCK) ( result_type, sort_order, @@ -3555,6 +3585,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. INSERT INTO #index_cleanup_results + WITH + (TABLOCK) ( result_type, sort_order, @@ -3668,6 +3700,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. INSERT INTO #index_cleanup_results + WITH + (TABLOCK) ( result_type, sort_order, @@ -3790,6 +3824,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Insert KEPT indexes into results */ INSERT INTO #index_cleanup_results + WITH + (TABLOCK) ( result_type, sort_order, @@ -3867,6 +3903,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. INSERT INTO #index_cleanup_results + WITH + (TABLOCK) ( result_type, sort_order, @@ -3942,6 +3980,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. INSERT INTO #index_cleanup_results + WITH + (TABLOCK) ( result_type, sort_order, @@ -4259,6 +4299,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. INSERT INTO #index_reporting_stats + WITH + (TABLOCK) ( summary_level, database_name, @@ -4395,6 +4437,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. INSERT INTO #index_reporting_stats + WITH + (TABLOCK) ( summary_level, database_name, From 7ab07824ab7d44b7fadf3f4f6551c9d3b12b9015 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sun, 16 Mar 2025 23:43:15 -0400 Subject: [PATCH 168/246] make end queries dynamic make end queries dynamic --- .DS_Store | Bin 6148 -> 6148 bytes sp_QuickieStore/sp_QuickieStore.sql | 760 +++++++++++++++++----------- 2 files changed, 456 insertions(+), 304 deletions(-) diff --git a/.DS_Store b/.DS_Store index 54f54c82faac2515aa144858856cf6d6233830a6..068d98176ca291437e0540c5f458ab97646953d3 100644 GIT binary patch delta 20 bcmZoMXffFEo{inmNJl}})NJ!NHaPg5 diff --git a/sp_QuickieStore/sp_QuickieStore.sql b/sp_QuickieStore/sp_QuickieStore.sql index a6c9e9b2..9675c0ba 100644 --- a/sp_QuickieStore/sp_QuickieStore.sql +++ b/sp_QuickieStore/sp_QuickieStore.sql @@ -8651,25 +8651,30 @@ ELSE END; /* -Return special things, unformatted +Return special things: plan feedback, query hints, query variants, query text, wait stats, and query store options +This section handles all expert mode and special output formats +Format numeric values based on @format_output */ IF ( - ( - @expert_mode = 1 - OR - ( - @only_queries_with_hints = 1 - OR @only_queries_with_feedback = 1 - OR @only_queries_with_variants = 1 - ) - ) -AND @format_output = 0 + @expert_mode = 1 + OR + ( + @only_queries_with_hints = 1 + OR @only_queries_with_feedback = 1 + OR @only_queries_with_variants = 1 + ) ) BEGIN - IF @sql_2022_views = 1 + /* + SQL 2022+ features: plan feedback, query hints, and query variants + */ + IF @expert_mode = 1 BEGIN - IF @expert_mode = 1 + /* + Handle query_store_plan_feedback + */ + IF @sql_2022_views = 1 BEGIN IF EXISTS ( @@ -8722,13 +8727,15 @@ BEGIN qspf.plan_id OPTION(RECOMPILE); END; - ELSE + ELSE IF @only_queries_with_feedback = 1 BEGIN SELECT result = '#query_store_plan_feedback is empty'; END; - END; + /* + Handle query_store_query_hints + */ IF EXISTS ( SELECT @@ -8738,7 +8745,15 @@ BEGIN BEGIN SELECT @current_table = 'selecting query hints'; - + + SET @sql = N''; + + SELECT + @sql = + CONVERT + ( + nvarchar(MAX), + N' SELECT database_name = DB_NAME(qsqh.database_id), @@ -8746,12 +8761,29 @@ BEGIN qsqh.query_id, qsqh.query_hint_text, qsqh.last_query_hint_failure_reason_desc, - qsqh.query_hint_failure_count, + query_hint_failure_count = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqh.query_hint_failure_count, ''N0'')' + ELSE N'qsqh.query_hint_failure_count' + END + + N', qsqh.source_desc FROM #query_store_query_hints AS qsqh ORDER BY qsqh.query_id - OPTION(RECOMPILE); + OPTION(RECOMPILE);' + ); + + IF @debug = 1 + BEGIN + PRINT LEN(@sql); + PRINT @sql; + END; + + EXECUTE sys.sp_executesql + @sql; END; ELSE BEGIN @@ -8915,40 +8947,191 @@ BEGIN END; END; + /* + Handle resource stats + */ IF @expert_mode = 1 BEGIN IF @rc > 0 BEGIN SELECT @current_table = 'selecting resource stats'; - + + SET @sql = N''; + + SELECT + @sql = + CONVERT + ( + nvarchar(MAX), + N' SELECT source = - 'resource_stats', + ''resource_stats'', database_name = DB_NAME(qsq.database_id), qsq.query_id, qsq.object_name, - qsqt.total_grant_mb, - qsqt.last_grant_mb, - qsqt.min_grant_mb, - qsqt.max_grant_mb, - qsqt.total_used_grant_mb, - qsqt.last_used_grant_mb, - qsqt.min_used_grant_mb, - qsqt.max_used_grant_mb, - qsqt.total_ideal_grant_mb, - qsqt.last_ideal_grant_mb, - qsqt.min_ideal_grant_mb, - qsqt.max_ideal_grant_mb, - qsqt.total_reserved_threads, - qsqt.last_reserved_threads, - qsqt.min_reserved_threads, - qsqt.max_reserved_threads, - qsqt.total_used_threads, - qsqt.last_used_threads, - qsqt.min_used_threads, - qsqt.max_used_threads + total_grant_mb = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.total_grant_mb, ''N0'')' + ELSE N'qsqt.total_grant_mb' + END + + N', + last_grant_mb = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.last_grant_mb, ''N0'')' + ELSE N'qsqt.last_grant_mb' + END + + N', + min_grant_mb = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.min_grant_mb, ''N0'')' + ELSE N'qsqt.min_grant_mb' + END + + N', + max_grant_mb = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.max_grant_mb, ''N0'')' + ELSE N'qsqt.max_grant_mb' + END + + N', + total_used_grant_mb = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.total_used_grant_mb, ''N0'')' + ELSE N'qsqt.total_used_grant_mb' + END + + N', + last_used_grant_mb = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.last_used_grant_mb, ''N0'')' + ELSE N'qsqt.last_used_grant_mb' + END + + N', + min_used_grant_mb = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.min_used_grant_mb, ''N0'')' + ELSE N'qsqt.min_used_grant_mb' + END + + N', + max_used_grant_mb = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.max_used_grant_mb, ''N0'')' + ELSE N'qsqt.max_used_grant_mb' + END + + N', + total_ideal_grant_mb = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.total_ideal_grant_mb, ''N0'')' + ELSE N'qsqt.total_ideal_grant_mb' + END + + N', + last_ideal_grant_mb = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.last_ideal_grant_mb, ''N0'')' + ELSE N'qsqt.last_ideal_grant_mb' + END + + N', + min_ideal_grant_mb = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.min_ideal_grant_mb, ''N0'')' + ELSE N'qsqt.min_ideal_grant_mb' + END + + N', + max_ideal_grant_mb = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.max_ideal_grant_mb, ''N0'')' + ELSE N'qsqt.max_ideal_grant_mb' + END + + N', + total_reserved_threads = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.total_reserved_threads, ''N0'')' + ELSE N'qsqt.total_reserved_threads' + END + + N', + last_reserved_threads = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.last_reserved_threads, ''N0'')' + ELSE N'qsqt.last_reserved_threads' + END + + N', + min_reserved_threads = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.min_reserved_threads, ''N0'')' + ELSE N'qsqt.min_reserved_threads' + END + + N', + max_reserved_threads = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.max_reserved_threads, ''N0'')' + ELSE N'qsqt.max_reserved_threads' + END + + N', + total_used_threads = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.total_used_threads, ''N0'')' + ELSE N'qsqt.total_used_threads' + END + + N', + last_used_threads = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.last_used_threads, ''N0'')' + ELSE N'qsqt.last_used_threads' + END + + N', + min_used_threads = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.min_used_threads, ''N0'')' + ELSE N'qsqt.min_used_threads' + END + + N', + max_used_threads = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.max_used_threads, ''N0'')' + ELSE N'qsqt.max_used_threads' + END + + N' FROM #query_store_query AS qsq JOIN #query_store_query_text AS qsqt ON qsq.query_text_id = qsqt.query_text_id @@ -8960,7 +9143,17 @@ BEGIN ) ORDER BY qsq.query_id - OPTION(RECOMPILE); + OPTION(RECOMPILE);' + ); + + IF @debug = 1 + BEGIN + PRINT LEN(@sql); + PRINT @sql; + END; + + EXECUTE sys.sp_executesql + @sql; END; /*End resource stats query*/ ELSE @@ -8971,6 +9164,9 @@ BEGIN END; END; + /* + Handle wait stats queries + */ IF @new = 1 BEGIN IF @expert_mode = 1 @@ -8979,35 +9175,111 @@ BEGIN ( SELECT 1/0 - FROM #query_store_wait_stats AS qsws + FROM #query_store_wait_stats AS qsws ) BEGIN + /* + Wait stats by query + */ SELECT @current_table = 'selecting wait stats by query'; + SET @sql = N''; + + SELECT + @sql = + CONVERT + ( + nvarchar(MAX), + N' SELECT DISTINCT source = - 'query_store_wait_stats_by_query', + ''query_store_wait_stats_by_query'', database_name = DB_NAME(qsws.database_id), qsws.plan_id, x.object_name, qsws.wait_category_desc, - qsws.total_query_wait_time_ms, - total_query_duration_ms = - x.total_duration_ms, - qsws.avg_query_wait_time_ms, - avg_query_duration_ms = - x.avg_duration_ms, - qsws.last_query_wait_time_ms, - last_query_duration_ms = - x.last_duration_ms, - qsws.min_query_wait_time_ms, - min_query_duration_ms = - x.min_duration_ms, - qsws.max_query_wait_time_ms, - max_query_duration_ms = - x.max_duration_ms + total_query_wait_time_ms = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsws.total_query_wait_time_ms, ''N0'')' + ELSE N'qsws.total_query_wait_time_ms' + END + + N', + total_query_duration_ms = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(x.total_duration_ms, ''N0'')' + ELSE N'x.total_duration_ms' + END + + N', + avg_query_wait_time_ms = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsws.avg_query_wait_time_ms, ''N0'')' + ELSE N'qsws.avg_query_wait_time_ms' + END + + N', + avg_query_duration_ms = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(x.avg_duration_ms, ''N0'')' + ELSE N'x.avg_duration_ms' + END + + N', + last_query_wait_time_ms = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsws.last_query_wait_time_ms, ''N0'')' + ELSE N'qsws.last_query_wait_time_ms' + END + + N', + last_query_duration_ms = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(x.last_duration_ms, ''N0'')' + ELSE N'x.last_duration_ms' + END + + N', + min_query_wait_time_ms = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsws.min_query_wait_time_ms, ''N0'')' + ELSE N'qsws.min_query_wait_time_ms' + END + + N', + min_query_duration_ms = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(x.min_duration_ms, ''N0'')' + ELSE N'x.min_duration_ms' + END + + N', + max_query_wait_time_ms = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsws.max_query_wait_time_ms, ''N0'')' + ELSE N'qsws.max_query_wait_time_ms' + END + + N', + max_query_duration_ms = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(x.max_duration_ms, ''N0'')' + ELSE N'x.max_duration_ms' + END + + N' FROM #query_store_wait_stats AS qsws CROSS APPLY ( @@ -9031,37 +9303,118 @@ BEGIN ORDER BY qsws.plan_id, qsws.total_query_wait_time_ms DESC - OPTION(RECOMPILE); + OPTION(RECOMPILE);' + ); + + IF @debug = 1 + BEGIN + PRINT LEN(@sql); + PRINT @sql; + END; + + EXECUTE sys.sp_executesql + @sql; + /* + Wait stats in total + */ SELECT @current_table = 'selecting wait stats in total'; - + + SET @sql = N''; + + SELECT + @sql = + CONVERT + ( + nvarchar(MAX), + N' SELECT source = - 'query_store_wait_stats_total', + ''query_store_wait_stats_total'', database_name = DB_NAME(qsws.database_id), qsws.wait_category_desc, - total_query_wait_time_ms = - SUM(qsws.total_query_wait_time_ms), - total_query_duration_ms = - SUM(x.total_duration_ms), - avg_query_wait_time_ms = - SUM(qsws.avg_query_wait_time_ms), - avg_query_duration_ms = - SUM(x.avg_duration_ms), - last_query_wait_time_ms = - SUM(qsws.last_query_wait_time_ms), - last_query_duration_ms = - SUM(x.last_duration_ms), - min_query_wait_time_ms = - SUM(qsws.min_query_wait_time_ms), - min_query_duration_ms = - SUM(x.min_duration_ms), - max_query_wait_time_ms = - SUM(qsws.max_query_wait_time_ms), - max_query_duration_ms = - SUM(x.max_duration_ms) + total_query_wait_time_ms = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(SUM(qsws.total_query_wait_time_ms), ''N0'')' + ELSE N'SUM(qsws.total_query_wait_time_ms)' + END + + N', + total_query_duration_ms = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(SUM(x.total_duration_ms), ''N0'')' + ELSE N'SUM(x.total_duration_ms)' + END + + N', + avg_query_wait_time_ms = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(SUM(qsws.avg_query_wait_time_ms), ''N0'')' + ELSE N'SUM(qsws.avg_query_wait_time_ms)' + END + + N', + avg_query_duration_ms = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(SUM(x.avg_duration_ms), ''N0'')' + ELSE N'SUM(x.avg_duration_ms)' + END + + N', + last_query_wait_time_ms = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(SUM(qsws.last_query_wait_time_ms), ''N0'')' + ELSE N'SUM(qsws.last_query_wait_time_ms)' + END + + N', + last_query_duration_ms = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(SUM(x.last_duration_ms), ''N0'')' + ELSE N'SUM(x.last_duration_ms)' + END + + N', + min_query_wait_time_ms = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(SUM(qsws.min_query_wait_time_ms), ''N0'')' + ELSE N'SUM(qsws.min_query_wait_time_ms)' + END + + N', + min_query_duration_ms = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(SUM(x.min_duration_ms), ''N0'')' + ELSE N'SUM(x.min_duration_ms)' + END + + N', + max_query_wait_time_ms = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(SUM(qsws.max_query_wait_time_ms), ''N0'')' + ELSE N'SUM(qsws.max_query_wait_time_ms)' + END + + N', + max_query_duration_ms = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(SUM(x.max_duration_ms), ''N0'')' + ELSE N'SUM(x.max_duration_ms)' + END + + N' FROM #query_store_wait_stats AS qsws CROSS APPLY ( @@ -9086,9 +9439,19 @@ BEGIN qsws.database_id ORDER BY SUM(qsws.total_query_wait_time_ms) DESC - OPTION(RECOMPILE); + OPTION(RECOMPILE);' + ); + + IF @debug = 1 + BEGIN + PRINT LEN(@sql); + PRINT @sql; + END; + + EXECUTE sys.sp_executesql + @sql; - END; /*End unformatted wait stats*/ + END; ELSE BEGIN SELECT @@ -9235,25 +9598,9 @@ BEGIN EXECUTE sys.sp_executesql @sql; END; -END; /*End expert mode format output = 0*/ - -/* -Return special things, formatted +/* +Continue with code for special things */ -IF -( - ( - @expert_mode = 1 - OR - ( - @only_queries_with_hints = 1 - OR @only_queries_with_feedback = 1 - OR @only_queries_with_variants = 1 - ) - ) -AND @format_output = 1 -) -BEGIN IF @sql_2022_views = 1 BEGIN IF @expert_mode = 1 @@ -9590,131 +9937,11 @@ BEGIN END; END; - IF @new = 1 - BEGIN - IF EXISTS - ( - SELECT - 1/0 - FROM #query_store_wait_stats AS qsws - ) - AND @expert_mode = 1 - BEGIN - SELECT - @current_table = 'selecting wait stats by query'; - - SELECT - source = - 'query_store_wait_stats_by_query', - database_name = - DB_NAME(qsws.database_id), - qsws.plan_id, - x.object_name, - qsws.wait_category_desc, - total_query_wait_time_ms = - FORMAT(qsws.total_query_wait_time_ms, 'N0'), - total_query_duration_ms = - FORMAT(x.total_duration_ms, 'N0'), - avg_query_wait_time_ms = - FORMAT(qsws.avg_query_wait_time_ms, 'N0'), - avg_query_duration_ms = - FORMAT(x.avg_duration_ms, 'N0'), - last_query_wait_time_ms = - FORMAT(qsws.last_query_wait_time_ms, 'N0'), - last_query_duration_ms = - FORMAT(x.last_duration_ms, 'N0'), - min_query_wait_time_ms = - FORMAT(qsws.min_query_wait_time_ms, 'N0'), - min_query_duration_ms = - FORMAT(x.min_duration_ms, 'N0'), - max_query_wait_time_ms = - FORMAT(qsws.max_query_wait_time_ms, 'N0'), - max_query_duration_ms = - FORMAT(x.max_duration_ms, 'N0') - FROM #query_store_wait_stats AS qsws - CROSS APPLY - ( - SELECT - qsrs.avg_duration_ms, - qsrs.last_duration_ms, - qsrs.min_duration_ms, - qsrs.max_duration_ms, - qsrs.total_duration_ms, - qsq.object_name - FROM #query_store_runtime_stats AS qsrs - JOIN #query_store_plan AS qsp - ON qsrs.plan_id = qsp.plan_id - AND qsrs.database_id = qsp.database_id - JOIN #query_store_query AS qsq - ON qsp.query_id = qsq.query_id - AND qsp.database_id = qsq.database_id - WHERE qsws.plan_id = qsrs.plan_id - AND qsws.database_id = qsrs.database_id - ) AS x - ORDER BY - qsws.plan_id, - qsws.total_query_wait_time_ms DESC - OPTION(RECOMPILE); - - SELECT - @current_table = 'selecting wait stats in total'; - - SELECT - source = - 'query_store_wait_stats_total', - database_name = - DB_NAME(qsws.database_id), - qsws.wait_category_desc, - total_query_wait_time_ms = - FORMAT(SUM(qsws.total_query_wait_time_ms), 'N0'), - total_query_duration_ms = - FORMAT(SUM(x.total_duration_ms), 'N0'), - avg_query_wait_time_ms = - FORMAT(SUM(qsws.avg_query_wait_time_ms), 'N0'), - avg_query_duration_ms = - FORMAT(SUM(x.avg_duration_ms), 'N0'), - last_query_wait_time_ms = - FORMAT(SUM(qsws.last_query_wait_time_ms), 'N0'), - last_query_duration_ms = - FORMAT(SUM(x.last_duration_ms), 'N0'), - min_query_wait_time_ms = - FORMAT(SUM(qsws.min_query_wait_time_ms), 'N0'), - min_query_duration_ms = - FORMAT(SUM(x.min_duration_ms), 'N0'), - max_query_wait_time_ms = - FORMAT(SUM(qsws.max_query_wait_time_ms), 'N0'), - max_query_duration_ms = - FORMAT(SUM(x.max_duration_ms), 'N0') - FROM #query_store_wait_stats AS qsws - CROSS APPLY - ( - SELECT - qsrs.avg_duration_ms, - qsrs.last_duration_ms, - qsrs.min_duration_ms, - qsrs.max_duration_ms, - qsrs.total_duration_ms, - qsq.object_name - FROM #query_store_runtime_stats AS qsrs - JOIN #query_store_plan AS qsp - ON qsrs.plan_id = qsp.plan_id - AND qsrs.database_id = qsp.database_id - JOIN #query_store_query AS qsq - ON qsp.query_id = qsq.query_id - AND qsp.database_id = qsq.database_id - WHERE qsws.plan_id = qsrs.plan_id - AND qsws.database_id = qsrs.database_id - ) AS x - GROUP BY - qsws.wait_category_desc, - qsws.database_id - ORDER BY - SUM(qsws.total_query_wait_time_ms) DESC - OPTION(RECOMPILE); - - END; + /* + Wait stats sections have already been handled above with dynamic SQL + */ - END; /*End wait stats, format output = 1*/ + END; /*End wait stats queries*/ ELSE BEGIN SELECT @@ -9785,87 +10012,12 @@ BEGIN END; END; - IF @expert_mode = 1 - BEGIN - SELECT - @current_table = 'selecting query store options', - @sql = N''; - - SELECT - @sql += - CONVERT - ( - nvarchar(MAX), - N' - SELECT - source = - ''query_store_options'', - database_name = - DB_NAME(dqso.database_id), - dqso.desired_state_desc, - dqso.actual_state_desc, - dqso.readonly_reason, - current_storage_size_mb = - FORMAT(dqso.current_storage_size_mb, ''N0''), - flush_interval_seconds = - FORMAT(dqso.flush_interval_seconds, ''N0''), - interval_length_minutes = - FORMAT(dqso.interval_length_minutes, ''N0''), - max_storage_size_mb = - FORMAT(dqso.max_storage_size_mb, ''N0''), - dqso.stale_query_threshold_days, - max_plans_per_query = - FORMAT(dqso.max_plans_per_query, ''N0''), - dqso.query_capture_mode_desc,' - + - CASE - WHEN - ( - @azure = 1 - OR @product_version > 13 - ) - THEN N' - dqso.wait_stats_capture_mode_desc,' - ELSE N'' - END - + - CASE - WHEN - ( - @azure = 1 - OR @product_version > 14 - ) - THEN N' - capture_policy_execution_count = - FORMAT(dqso.capture_policy_execution_count, ''N0''), - capture_policy_total_compile_cpu_time_ms = - FORMAT(dqso.capture_policy_total_compile_cpu_time_ms, ''N0''), - capture_policy_total_execution_cpu_time_ms = - FORMAT(dqso.capture_policy_total_execution_cpu_time_ms, ''N0''), - capture_policy_stale_threshold_hours = - FORMAT(dqso.capture_policy_stale_threshold_hours, ''N0''),' - ELSE N'' - END - ); - - SELECT - @sql += N' - dqso.size_based_cleanup_mode_desc - FROM #database_query_store_options AS dqso - OPTION(RECOMPILE);'; - - - IF @debug = 1 - BEGIN - PRINT LEN(@sql); - PRINT @sql; - END; - - EXECUTE sys.sp_executesql - @sql; + /* + Query store options section has already been handled with dynamic SQL above + */ END; -END; /*End expert mode = 1, format output = 1*/ +END; /*End special output section*/ IF @query_store_trouble = 1 BEGIN From ec06cb0b513fdb2af58e20f6715b99c5fdd2019b Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 17 Mar 2025 09:07:57 -0400 Subject: [PATCH 169/246] Update sp_QuickieStore.sql --- sp_QuickieStore/sp_QuickieStore.sql | 308 +++++++++++++++++++++------- 1 file changed, 236 insertions(+), 72 deletions(-) diff --git a/sp_QuickieStore/sp_QuickieStore.sql b/sp_QuickieStore/sp_QuickieStore.sql index 9675c0ba..53d485cd 100644 --- a/sp_QuickieStore/sp_QuickieStore.sql +++ b/sp_QuickieStore/sp_QuickieStore.sql @@ -9735,14 +9735,22 @@ Continue with code for special things BEGIN SELECT @current_table = 'selecting compilation stats'; - + + /* + Use dynamic SQL to handle formatting differences based on @format_output + */ + SELECT + @sql = @isolation_level; + + SELECT + @sql += N' SELECT x.* FROM ( SELECT source = - 'compilation_stats', + ''compilation_stats'', database_name = DB_NAME(qsq.database_id), qsq.query_id, @@ -9791,46 +9799,126 @@ Continue with code for special things END, last_execution_time_utc = qsq.last_execution_time, - count_compiles = - FORMAT(qsq.count_compiles, 'N0'), - avg_compile_duration_ms = - FORMAT(qsq.avg_compile_duration_ms, 'N0'), - total_compile_duration_ms = - FORMAT(qsq.total_compile_duration_ms, 'N0'), - last_compile_duration_ms = - FORMAT(qsq.last_compile_duration_ms, 'N0'), - avg_bind_duration_ms = - FORMAT(qsq.avg_bind_duration_ms, 'N0'), - total_bind_duration_ms = - FORMAT(qsq.total_bind_duration_ms, 'N0'), - last_bind_duration_ms = - FORMAT(qsq.last_bind_duration_ms, 'N0'), - avg_bind_cpu_time_ms = - FORMAT(qsq.avg_bind_cpu_time_ms, 'N0'), - total_bind_cpu_time_ms = - FORMAT(qsq.total_bind_cpu_time_ms, 'N0'), - last_bind_cpu_time_ms = - FORMAT(qsq.last_bind_cpu_time_ms, 'N0'), - avg_optimize_duration_ms = - FORMAT(qsq.avg_optimize_duration_ms, 'N0'), - total_optimize_duration_ms = - FORMAT(qsq.total_optimize_duration_ms, 'N0'), - last_optimize_duration_ms = - FORMAT(qsq.last_optimize_duration_ms, 'N0'), - avg_optimize_cpu_time_ms = - FORMAT(qsq.avg_optimize_cpu_time_ms, 'N0'), - total_optimize_cpu_time_ms = - FORMAT(qsq.total_optimize_cpu_time_ms, 'N0'), - last_optimize_cpu_time_ms = - FORMAT(qsq.last_optimize_cpu_time_ms, 'N0'), - avg_compile_memory_mb = - FORMAT(qsq.avg_compile_memory_mb, 'N0'), - total_compile_memory_mb = - FORMAT(qsq.total_compile_memory_mb, 'N0'), - last_compile_memory_mb = - FORMAT(qsq.last_compile_memory_mb, 'N0'), - max_compile_memory_mb = - FORMAT(qsq.max_compile_memory_mb, 'N0'), + count_compiles = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsq.count_compiles, ''N0'')' + ELSE N'qsq.count_compiles' + END + N', + avg_compile_duration_ms = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsq.avg_compile_duration_ms, ''N0'')' + ELSE N'qsq.avg_compile_duration_ms' + END + N', + total_compile_duration_ms = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsq.total_compile_duration_ms, ''N0'')' + ELSE N'qsq.total_compile_duration_ms' + END + N', + last_compile_duration_ms = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsq.last_compile_duration_ms, ''N0'')' + ELSE N'qsq.last_compile_duration_ms' + END + N', + avg_bind_duration_ms = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsq.avg_bind_duration_ms, ''N0'')' + ELSE N'qsq.avg_bind_duration_ms' + END + N', + total_bind_duration_ms = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsq.total_bind_duration_ms, ''N0'')' + ELSE N'qsq.total_bind_duration_ms' + END + N', + last_bind_duration_ms = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsq.last_bind_duration_ms, ''N0'')' + ELSE N'qsq.last_bind_duration_ms' + END + N', + avg_bind_cpu_time_ms = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsq.avg_bind_cpu_time_ms, ''N0'')' + ELSE N'qsq.avg_bind_cpu_time_ms' + END + N', + total_bind_cpu_time_ms = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsq.total_bind_cpu_time_ms, ''N0'')' + ELSE N'qsq.total_bind_cpu_time_ms' + END + N', + last_bind_cpu_time_ms = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsq.last_bind_cpu_time_ms, ''N0'')' + ELSE N'qsq.last_bind_cpu_time_ms' + END + N', + avg_optimize_duration_ms = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsq.avg_optimize_duration_ms, ''N0'')' + ELSE N'qsq.avg_optimize_duration_ms' + END + N', + total_optimize_duration_ms = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsq.total_optimize_duration_ms, ''N0'')' + ELSE N'qsq.total_optimize_duration_ms' + END + N', + last_optimize_duration_ms = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsq.last_optimize_duration_ms, ''N0'')' + ELSE N'qsq.last_optimize_duration_ms' + END + N', + avg_optimize_cpu_time_ms = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsq.avg_optimize_cpu_time_ms, ''N0'')' + ELSE N'qsq.avg_optimize_cpu_time_ms' + END + N', + total_optimize_cpu_time_ms = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsq.total_optimize_cpu_time_ms, ''N0'')' + ELSE N'qsq.total_optimize_cpu_time_ms' + END + N', + last_optimize_cpu_time_ms = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsq.last_optimize_cpu_time_ms, ''N0'')' + ELSE N'qsq.last_optimize_cpu_time_ms' + END + N', + avg_compile_memory_mb = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsq.avg_compile_memory_mb, ''N0'')' + ELSE N'qsq.avg_compile_memory_mb' + END + N', + total_compile_memory_mb = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsq.total_compile_memory_mb, ''N0'')' + ELSE N'qsq.total_compile_memory_mb' + END + N', + last_compile_memory_mb = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsq.last_compile_memory_mb, ''N0'')' + ELSE N'qsq.last_compile_memory_mb' + END + N', + max_compile_memory_mb = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsq.max_compile_memory_mb, ''N0'')' + ELSE N'qsq.max_compile_memory_mb' + END + N', qsq.query_hash, qsq.batch_sql_handle, qsqt.statement_sql_handle, @@ -9858,9 +9946,20 @@ Continue with code for special things WHERE x.n = 1 ORDER BY x.query_id - OPTION(RECOMPILE); + OPTION(RECOMPILE);'; + + IF @debug = 1 + BEGIN + PRINT LEN(@sql); + PRINT @sql; + END; + + EXECUTE sys.sp_executesql + @sql, + N'@timezone sysname, @utc_offset_string nvarchar(max)', + @timezone, @utc_offset_string; - END; /*End query store query, format output = 1*/ + END; /*End compilation stats section*/ ELSE BEGIN SELECT @@ -9875,38 +9974,94 @@ Continue with code for special things BEGIN SELECT @current_table = 'selecting resource stats'; - + + /* + Use dynamic SQL to handle formatting differences based on @format_output + */ + SELECT + @sql = @isolation_level; + + SELECT + @sql += N' SELECT source = - 'resource_stats', + ''resource_stats'', database_name = DB_NAME(qsq.database_id), qsq.query_id, qsq.object_name, - total_grant_mb = - FORMAT(qsqt.total_grant_mb, 'N0'), - last_grant_mb = - FORMAT(qsqt.last_grant_mb, 'N0'), - min_grant_mb = - FORMAT(qsqt.min_grant_mb, 'N0'), - max_grant_mb = - FORMAT(qsqt.max_grant_mb, 'N0'), - total_used_grant_mb = - FORMAT(qsqt.total_used_grant_mb, 'N0'), - last_used_grant_mb = - FORMAT(qsqt.last_used_grant_mb, 'N0'), - min_used_grant_mb = - FORMAT(qsqt.min_used_grant_mb, 'N0'), - max_used_grant_mb = - FORMAT(qsqt.max_used_grant_mb, 'N0'), - total_ideal_grant_mb = - FORMAT(qsqt.total_ideal_grant_mb, 'N0'), - last_ideal_grant_mb = - FORMAT(qsqt.last_ideal_grant_mb, 'N0'), - min_ideal_grant_mb = - FORMAT(qsqt.min_ideal_grant_mb, 'N0'), - max_ideal_grant_mb = - FORMAT(qsqt.max_ideal_grant_mb, 'N0'), + total_grant_mb = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.total_grant_mb, ''N0'')' + ELSE N'qsqt.total_grant_mb' + END + N', + last_grant_mb = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.last_grant_mb, ''N0'')' + ELSE N'qsqt.last_grant_mb' + END + N', + min_grant_mb = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.min_grant_mb, ''N0'')' + ELSE N'qsqt.min_grant_mb' + END + N', + max_grant_mb = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.max_grant_mb, ''N0'')' + ELSE N'qsqt.max_grant_mb' + END + N', + total_used_grant_mb = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.total_used_grant_mb, ''N0'')' + ELSE N'qsqt.total_used_grant_mb' + END + N', + last_used_grant_mb = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.last_used_grant_mb, ''N0'')' + ELSE N'qsqt.last_used_grant_mb' + END + N', + min_used_grant_mb = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.min_used_grant_mb, ''N0'')' + ELSE N'qsqt.min_used_grant_mb' + END + N', + max_used_grant_mb = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.max_used_grant_mb, ''N0'')' + ELSE N'qsqt.max_used_grant_mb' + END + N', + total_ideal_grant_mb = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.total_ideal_grant_mb, ''N0'')' + ELSE N'qsqt.total_ideal_grant_mb' + END + N', + last_ideal_grant_mb = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.last_ideal_grant_mb, ''N0'')' + ELSE N'qsqt.last_ideal_grant_mb' + END + N', + min_ideal_grant_mb = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.min_ideal_grant_mb, ''N0'')' + ELSE N'qsqt.min_ideal_grant_mb' + END + N', + max_ideal_grant_mb = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.max_ideal_grant_mb, ''N0'')' + ELSE N'qsqt.max_ideal_grant_mb' + END + N', qsqt.total_reserved_threads, qsqt.last_reserved_threads, qsqt.min_reserved_threads, @@ -9926,9 +10081,18 @@ Continue with code for special things ) ORDER BY qsq.query_id - OPTION(RECOMPILE); + OPTION(RECOMPILE);'; + + IF @debug = 1 + BEGIN + PRINT LEN(@sql); + PRINT @sql; + END; + + EXECUTE sys.sp_executesql + @sql; - END; /*End resource stats, format output = 1*/ + END; /*End resource stats section*/ ELSE BEGIN SELECT From 85fb7bc864a955d84716a05e28de029f2b41ea09 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 17 Mar 2025 09:31:16 -0400 Subject: [PATCH 170/246] Update sp_QuickieStore.sql --- sp_QuickieStore/sp_QuickieStore.sql | 72 ++++++++++++++++++++++++++--- 1 file changed, 65 insertions(+), 7 deletions(-) diff --git a/sp_QuickieStore/sp_QuickieStore.sql b/sp_QuickieStore/sp_QuickieStore.sql index 53d485cd..50ff46db 100644 --- a/sp_QuickieStore/sp_QuickieStore.sql +++ b/sp_QuickieStore/sp_QuickieStore.sql @@ -9614,7 +9614,15 @@ Continue with code for special things BEGIN SELECT @current_table = 'selecting plan feedback'; - + + /* + Use dynamic SQL to handle formatting differences based on @format_output + */ + SELECT + @sql = @isolation_level; + + SELECT + @sql += N' SELECT database_name = DB_NAME(qspf.database_id), @@ -9654,7 +9662,18 @@ Continue with code for special things FROM #query_store_plan_feedback AS qspf ORDER BY qspf.plan_id - OPTION(RECOMPILE); + OPTION(RECOMPILE);'; + + IF @debug = 1 + BEGIN + PRINT LEN(@sql); + PRINT @sql; + END; + + EXECUTE sys.sp_executesql + @sql, + N'@timezone sysname, @utc_offset_string nvarchar(max)', + @timezone, @utc_offset_string; END; ELSE BEGIN @@ -9672,7 +9691,15 @@ Continue with code for special things BEGIN SELECT @current_table = 'selecting query hints'; - + + /* + Use dynamic SQL to handle formatting differences based on @format_output + */ + SELECT + @sql = @isolation_level; + + SELECT + @sql += N' SELECT database_name = DB_NAME(qsqh.database_id), @@ -9680,12 +9707,26 @@ Continue with code for special things qsqh.query_id, qsqh.query_hint_text, qsqh.last_query_hint_failure_reason_desc, - qsqh.query_hint_failure_count, + query_hint_failure_count = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqh.query_hint_failure_count, ''N0'')' + ELSE N'qsqh.query_hint_failure_count' + END + N', qsqh.source_desc FROM #query_store_query_hints AS qsqh ORDER BY qsqh.query_id - OPTION(RECOMPILE); + OPTION(RECOMPILE);'; + + IF @debug = 1 + BEGIN + PRINT LEN(@sql); + PRINT @sql; + END; + + EXECUTE sys.sp_executesql + @sql; END; ELSE BEGIN @@ -9704,7 +9745,15 @@ Continue with code for special things BEGIN SELECT @current_table = 'selecting query variants'; - + + /* + Use dynamic SQL to handle formatting differences based on @format_output + */ + SELECT + @sql = @isolation_level; + + SELECT + @sql += N' SELECT database_name = DB_NAME(qsqv.database_id), @@ -9714,7 +9763,16 @@ Continue with code for special things FROM #query_store_query_variant AS qsqv ORDER BY qsqv.parent_query_id - OPTION(RECOMPILE); + OPTION(RECOMPILE);'; + + IF @debug = 1 + BEGIN + PRINT LEN(@sql); + PRINT @sql; + END; + + EXECUTE sys.sp_executesql + @sql; END; ELSE BEGIN From d6fce9990ab62fead9c196139475276c917ee891 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 17 Mar 2025 09:58:42 -0400 Subject: [PATCH 171/246] Update sp_QuickieStore.sql --- sp_QuickieStore/sp_QuickieStore.sql | 54 +++-------------------------- 1 file changed, 4 insertions(+), 50 deletions(-) diff --git a/sp_QuickieStore/sp_QuickieStore.sql b/sp_QuickieStore/sp_QuickieStore.sql index 50ff46db..ca5dea05 100644 --- a/sp_QuickieStore/sp_QuickieStore.sql +++ b/sp_QuickieStore/sp_QuickieStore.sql @@ -8676,56 +8676,10 @@ BEGIN */ IF @sql_2022_views = 1 BEGIN - IF EXISTS - ( - SELECT - 1/0 - FROM #query_store_plan_feedback AS qspf - ) - BEGIN - SELECT - @current_table = 'selecting plan feedback'; - - SELECT - database_name = - DB_NAME(qspf.database_id), - qspf.plan_feedback_id, - qspf.plan_id, - qspf.feature_desc, - qspf.feedback_data, - qspf.state_desc, - create_time = - CASE - WHEN @timezone IS NULL - THEN - SWITCHOFFSET - ( - qspf.create_time, - @utc_offset_string - ) - WHEN @timezone IS NOT NULL - THEN qspf.create_time AT TIME ZONE @timezone - END, - create_time_utc = - qspf.create_time, - last_updated_time = - CASE - WHEN @timezone IS NULL - THEN - SWITCHOFFSET - ( - qspf.last_updated_time, - @utc_offset_string - ) - WHEN @timezone IS NOT NULL - THEN qspf.last_updated_time AT TIME ZONE @timezone - END, - last_updated_time_utc = - qspf.last_updated_time - FROM #query_store_plan_feedback AS qspf - ORDER BY - qspf.plan_id - OPTION(RECOMPILE); + /* + Plan feedback section has been consolidated with the dynamic SQL section below + that handles formatting differences + */ END; ELSE IF @only_queries_with_feedback = 1 BEGIN From 868c6271262a9f0a6860abf0b825bc1c6b823d14 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 17 Mar 2025 10:01:42 -0400 Subject: [PATCH 172/246] Update sp_QuickieStore.sql --- sp_QuickieStore/sp_QuickieStore.sql | 89 ++--------------------------- 1 file changed, 6 insertions(+), 83 deletions(-) diff --git a/sp_QuickieStore/sp_QuickieStore.sql b/sp_QuickieStore/sp_QuickieStore.sql index ca5dea05..7637c97e 100644 --- a/sp_QuickieStore/sp_QuickieStore.sql +++ b/sp_QuickieStore/sp_QuickieStore.sql @@ -8688,91 +8688,14 @@ BEGIN END; /* - Handle query_store_query_hints + Query hints section has been consolidated with the dynamic SQL section below + that handles formatting differences */ - IF EXISTS - ( - SELECT - 1/0 - FROM #query_store_query_hints AS qsqh - ) - BEGIN - SELECT - @current_table = 'selecting query hints'; - - SET @sql = N''; - - SELECT - @sql = - CONVERT - ( - nvarchar(MAX), - N' - SELECT - database_name = - DB_NAME(qsqh.database_id), - qsqh.query_hint_id, - qsqh.query_id, - qsqh.query_hint_text, - qsqh.last_query_hint_failure_reason_desc, - query_hint_failure_count = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqh.query_hint_failure_count, ''N0'')' - ELSE N'qsqh.query_hint_failure_count' - END - + N', - qsqh.source_desc - FROM #query_store_query_hints AS qsqh - ORDER BY - qsqh.query_id - OPTION(RECOMPILE);' - ); - - IF @debug = 1 - BEGIN - PRINT LEN(@sql); - PRINT @sql; - END; - - EXECUTE sys.sp_executesql - @sql; - END; - ELSE - BEGIN - SELECT - result = '#query_store_query_hints is empty'; - END; - IF @expert_mode = 1 - BEGIN - IF EXISTS - ( - SELECT - 1/0 - FROM #query_store_query_variant AS qsqv - ) - BEGIN - SELECT - @current_table = 'selecting query variants'; - - SELECT - database_name = - DB_NAME(qsqv.database_id), - qsqv.query_variant_query_id, - qsqv.parent_query_id, - qsqv.dispatcher_plan_id - FROM #query_store_query_variant AS qsqv - ORDER BY - qsqv.parent_query_id - OPTION(RECOMPILE); - END; - ELSE - BEGIN - SELECT - result = '#query_store_query_variant is empty'; - END; + /* + Query variants section has been consolidated with the dynamic SQL section below + that handles formatting differences + */ END; END; From 8b4565cd857246659d31f6ee0b38998960171487 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 17 Mar 2025 10:08:56 -0400 Subject: [PATCH 173/246] Update sp_QuickieStore.sql --- sp_QuickieStore/sp_QuickieStore.sql | 121 +--------------------------- 1 file changed, 4 insertions(+), 117 deletions(-) diff --git a/sp_QuickieStore/sp_QuickieStore.sql b/sp_QuickieStore/sp_QuickieStore.sql index 7637c97e..1c14e798 100644 --- a/sp_QuickieStore/sp_QuickieStore.sql +++ b/sp_QuickieStore/sp_QuickieStore.sql @@ -8699,123 +8699,10 @@ BEGIN END; END; - IF @expert_mode = 1 - BEGIN - IF EXISTS - ( - SELECT - 1/0 - FROM #query_store_query AS qsq - ) - BEGIN - SELECT - @current_table = 'selecting compilation stats'; - - SELECT - x.* - FROM - ( - SELECT - source = - 'compilation_stats', - database_name = - DB_NAME(qsq.database_id), - qsq.query_id, - qsq.object_name, - qsq.query_text_id, - qsq.query_parameterization_type_desc, - initial_compile_start_time = - CASE - WHEN @timezone IS NULL - THEN - SWITCHOFFSET - ( - qsq.initial_compile_start_time, - @utc_offset_string - ) - WHEN @timezone IS NOT NULL - THEN qsq.initial_compile_start_time AT TIME ZONE @timezone - END, - initial_compile_start_time_utc = - qsq.initial_compile_start_time, - last_compile_start_time = - CASE - WHEN @timezone IS NULL - THEN - SWITCHOFFSET - ( - qsq.last_compile_start_time, - @utc_offset_string - ) - WHEN @timezone IS NOT NULL - THEN qsq.last_compile_start_time AT TIME ZONE @timezone - END, - last_compile_start_time_utc = - qsq.last_compile_start_time, - last_execution_time = - CASE - WHEN @timezone IS NULL - THEN - SWITCHOFFSET - ( - qsq.last_execution_time, - @utc_offset_string - ) - WHEN @timezone IS NOT NULL - THEN qsq.last_execution_time AT TIME ZONE @timezone - END, - last_execution_time_utc = - qsq.last_execution_time, - qsq.count_compiles, - qsq.avg_compile_duration_ms, - qsq.total_compile_duration_ms, - qsq.last_compile_duration_ms, - qsq.avg_bind_duration_ms, - qsq.total_bind_duration_ms, - qsq.last_bind_duration_ms, - qsq.avg_bind_cpu_time_ms, - qsq.total_bind_cpu_time_ms, - qsq.last_bind_cpu_time_ms, - qsq.avg_optimize_duration_ms, - qsq.total_optimize_duration_ms, - qsq.last_optimize_duration_ms, - qsq.avg_optimize_cpu_time_ms, - qsq.total_optimize_cpu_time_ms, - qsq.last_optimize_cpu_time_ms, - qsq.avg_compile_memory_mb, - qsq.total_compile_memory_mb, - qsq.last_compile_memory_mb, - qsq.max_compile_memory_mb, - qsq.query_hash, - qsq.batch_sql_handle, - qsqt.statement_sql_handle, - qsq.last_compile_batch_sql_handle, - qsq.last_compile_batch_offset_start, - qsq.last_compile_batch_offset_end, - ROW_NUMBER() OVER - ( - PARTITION BY - qsq.query_id, - qsq.query_text_id - ORDER BY - qsq.query_id - ) AS n - FROM #query_store_query AS qsq - CROSS APPLY - ( - SELECT TOP (1) - qsqt.* - FROM #query_store_query_text AS qsqt - WHERE qsqt.query_text_id = qsq.query_text_id - AND qsqt.database_id = qsq.database_id - ) AS qsqt - ) AS x - WHERE x.n = 1 - ORDER BY - x.query_id - OPTION(RECOMPILE); - - END; /*End compilation stats query*/ + /* + Compilation stats section has been consolidated with the dynamic SQL section below + that handles formatting differences + */ ELSE BEGIN SELECT From 221af62de89e313f4db76b1df91e35453d414d57 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 17 Mar 2025 10:48:51 -0400 Subject: [PATCH 174/246] Update sp_QuickieStore.sql --- sp_QuickieStore/sp_QuickieStore.sql | 1700 +++++++++++---------------- 1 file changed, 717 insertions(+), 983 deletions(-) diff --git a/sp_QuickieStore/sp_QuickieStore.sql b/sp_QuickieStore/sp_QuickieStore.sql index 1c14e798..804d485f 100644 --- a/sp_QuickieStore/sp_QuickieStore.sql +++ b/sp_QuickieStore/sp_QuickieStore.sql @@ -8676,894 +8676,244 @@ BEGIN */ IF @sql_2022_views = 1 BEGIN - /* - Plan feedback section has been consolidated with the dynamic SQL section below - that handles formatting differences - */ + IF EXISTS + ( + SELECT + 1/0 + FROM #query_store_plan_feedback AS qspf + ) + BEGIN + SELECT + @current_table = 'selecting plan feedback'; + + /* + Use dynamic SQL to handle formatting differences based on @format_output + */ + SELECT + @sql = @isolation_level; + + SELECT + @sql += N' + SELECT + database_name = + DB_NAME(qspf.database_id), + qspf.plan_feedback_id, + qspf.plan_id, + qspf.feature_desc, + qspf.feedback_data, + qspf.state_desc, + create_time = + CASE + WHEN @timezone IS NULL + THEN + SWITCHOFFSET + ( + qspf.create_time, + @utc_offset_string + ) + WHEN @timezone IS NOT NULL + THEN qspf.create_time AT TIME ZONE @timezone + END, + create_time_utc = + qspf.create_time, + last_updated_time = + CASE + WHEN @timezone IS NULL + THEN + SWITCHOFFSET + ( + qspf.last_updated_time, + @utc_offset_string + ) + WHEN @timezone IS NOT NULL + THEN qspf.last_updated_time AT TIME ZONE @timezone + END, + last_updated_time_utc = + qspf.last_updated_time + FROM #query_store_plan_feedback AS qspf + ORDER BY + qspf.plan_id + OPTION(RECOMPILE);'; + + IF @debug = 1 + BEGIN + PRINT LEN(@sql); + PRINT @sql; + END; + + EXECUTE sys.sp_executesql + @sql, + N'@timezone sysname, @utc_offset_string nvarchar(max)', + @timezone, @utc_offset_string; END; - ELSE IF @only_queries_with_feedback = 1 + ELSE BEGIN SELECT result = '#query_store_plan_feedback is empty'; END; - - /* - Query hints section has been consolidated with the dynamic SQL section below - that handles formatting differences - */ - - /* - Query variants section has been consolidated with the dynamic SQL section below - that handles formatting differences - */ + + IF EXISTS + ( + SELECT + 1/0 + FROM #query_store_query_hints AS qsqh + ) + BEGIN + SELECT + @current_table = 'selecting query hints'; + + /* + Use dynamic SQL to handle formatting differences based on @format_output + */ + SELECT + @sql = @isolation_level; + + SELECT + @sql += N' + SELECT + database_name = + DB_NAME(qsqh.database_id), + qsqh.query_hint_id, + qsqh.query_id, + qsqh.query_hint_text, + qsqh.last_query_hint_failure_reason_desc, + query_hint_failure_count = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqh.query_hint_failure_count, ''N0'')' + ELSE N'qsqh.query_hint_failure_count' + END + N', + qsqh.source_desc + FROM #query_store_query_hints AS qsqh + ORDER BY + qsqh.query_id + OPTION(RECOMPILE);'; + + IF @debug = 1 + BEGIN + PRINT LEN(@sql); + PRINT @sql; + END; + + EXECUTE sys.sp_executesql + @sql; END; - END; - - /* - Compilation stats section has been consolidated with the dynamic SQL section below - that handles formatting differences - */ ELSE BEGIN SELECT - result = - '#query_store_query is empty'; - END; - END; + result = '#query_store_query_hints is empty'; + END; + + IF EXISTS + ( + SELECT + 1/0 + FROM #query_store_query_variant AS qsqv + ) + BEGIN + SELECT + @current_table = 'selecting query variants'; + + /* + Use dynamic SQL to handle formatting differences based on @format_output + */ + SELECT + @sql = @isolation_level; + + SELECT + @sql += N' + SELECT + database_name = + DB_NAME(qsqv.database_id), + qsqv.query_variant_query_id, + qsqv.parent_query_id, + qsqv.dispatcher_plan_id + FROM #query_store_query_variant AS qsqv + ORDER BY + qsqv.parent_query_id + OPTION(RECOMPILE);'; + + IF @debug = 1 + BEGIN + PRINT LEN(@sql); + PRINT @sql; + END; + + EXECUTE sys.sp_executesql + @sql; + END; + ELSE + BEGIN + SELECT + result = '#query_store_query_variant is empty'; + END; + + IF + ( + @sql_2022_views = 1 + AND @ags_present = 1 + ) + BEGIN + IF @expert_mode = 1 + BEGIN + IF EXISTS + ( + SELECT + 1/0 + FROM #query_store_replicas AS qsr + JOIN #query_store_plan_forcing_locations AS qspfl + ON qsr.replica_group_id = qspfl.replica_group_id + AND qsr.database_id = qspfl.database_id + ) + BEGIN + SELECT + @current_table = 'selecting #query_store_replicas and #query_store_plan_forcing_locations'; + + SELECT + database_name = + DB_NAME(qsr.database_id), + qsr.replica_group_id, + qsr.role_type, + qsr.replica_name, + qspfl.plan_forcing_location_id, + qspfl.query_id, + qspfl.plan_id, + qspfl.replica_group_id + FROM #query_store_replicas AS qsr + JOIN #query_store_plan_forcing_locations AS qspfl + ON qsr.replica_group_id = qspfl.replica_group_id + ORDER BY + qsr.replica_group_id + OPTION(RECOMPILE);; + END; + ELSE + BEGIN + SELECT + result = 'Availability Group information is empty'; + END; + END; + END; + + END; /*End 2022 views*/ - /* - Handle resource stats - */ IF @expert_mode = 1 BEGIN - IF @rc > 0 + IF EXISTS + ( + SELECT + 1/0 + FROM #query_store_query AS qsq + ) BEGIN SELECT - @current_table = 'selecting resource stats'; + @current_table = 'selecting compilation stats'; - SET @sql = N''; - + /* + Use dynamic SQL to handle formatting differences based on @format_output + */ SELECT - @sql = - CONVERT - ( - nvarchar(MAX), - N' - SELECT - source = - ''resource_stats'', - database_name = - DB_NAME(qsq.database_id), - qsq.query_id, - qsq.object_name, - total_grant_mb = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.total_grant_mb, ''N0'')' - ELSE N'qsqt.total_grant_mb' - END - + N', - last_grant_mb = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.last_grant_mb, ''N0'')' - ELSE N'qsqt.last_grant_mb' - END - + N', - min_grant_mb = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.min_grant_mb, ''N0'')' - ELSE N'qsqt.min_grant_mb' - END - + N', - max_grant_mb = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.max_grant_mb, ''N0'')' - ELSE N'qsqt.max_grant_mb' - END - + N', - total_used_grant_mb = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.total_used_grant_mb, ''N0'')' - ELSE N'qsqt.total_used_grant_mb' - END - + N', - last_used_grant_mb = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.last_used_grant_mb, ''N0'')' - ELSE N'qsqt.last_used_grant_mb' - END - + N', - min_used_grant_mb = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.min_used_grant_mb, ''N0'')' - ELSE N'qsqt.min_used_grant_mb' - END - + N', - max_used_grant_mb = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.max_used_grant_mb, ''N0'')' - ELSE N'qsqt.max_used_grant_mb' - END - + N', - total_ideal_grant_mb = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.total_ideal_grant_mb, ''N0'')' - ELSE N'qsqt.total_ideal_grant_mb' - END - + N', - last_ideal_grant_mb = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.last_ideal_grant_mb, ''N0'')' - ELSE N'qsqt.last_ideal_grant_mb' - END - + N', - min_ideal_grant_mb = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.min_ideal_grant_mb, ''N0'')' - ELSE N'qsqt.min_ideal_grant_mb' - END - + N', - max_ideal_grant_mb = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.max_ideal_grant_mb, ''N0'')' - ELSE N'qsqt.max_ideal_grant_mb' - END - + N', - total_reserved_threads = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.total_reserved_threads, ''N0'')' - ELSE N'qsqt.total_reserved_threads' - END - + N', - last_reserved_threads = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.last_reserved_threads, ''N0'')' - ELSE N'qsqt.last_reserved_threads' - END - + N', - min_reserved_threads = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.min_reserved_threads, ''N0'')' - ELSE N'qsqt.min_reserved_threads' - END - + N', - max_reserved_threads = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.max_reserved_threads, ''N0'')' - ELSE N'qsqt.max_reserved_threads' - END - + N', - total_used_threads = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.total_used_threads, ''N0'')' - ELSE N'qsqt.total_used_threads' - END - + N', - last_used_threads = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.last_used_threads, ''N0'')' - ELSE N'qsqt.last_used_threads' - END - + N', - min_used_threads = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.min_used_threads, ''N0'')' - ELSE N'qsqt.min_used_threads' - END - + N', - max_used_threads = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.max_used_threads, ''N0'')' - ELSE N'qsqt.max_used_threads' - END - + N' - FROM #query_store_query AS qsq - JOIN #query_store_query_text AS qsqt - ON qsq.query_text_id = qsqt.query_text_id - AND qsq.database_id = qsqt.database_id - WHERE - ( - qsqt.total_grant_mb IS NOT NULL - OR qsqt.total_reserved_threads IS NOT NULL - ) - ORDER BY - qsq.query_id - OPTION(RECOMPILE);' - ); - - IF @debug = 1 - BEGIN - PRINT LEN(@sql); - PRINT @sql; - END; - - EXECUTE sys.sp_executesql - @sql; - - END; /*End resource stats query*/ - ELSE - BEGIN - SELECT - result = - '#dm_exec_query_stats is empty'; - END; - END; - - /* - Handle wait stats queries - */ - IF @new = 1 - BEGIN - IF @expert_mode = 1 - BEGIN - IF EXISTS - ( - SELECT - 1/0 - FROM #query_store_wait_stats AS qsws - ) - BEGIN - /* - Wait stats by query - */ - SELECT - @current_table = 'selecting wait stats by query'; - - SET @sql = N''; - - SELECT - @sql = - CONVERT - ( - nvarchar(MAX), - N' - SELECT DISTINCT - source = - ''query_store_wait_stats_by_query'', - database_name = - DB_NAME(qsws.database_id), - qsws.plan_id, - x.object_name, - qsws.wait_category_desc, - total_query_wait_time_ms = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsws.total_query_wait_time_ms, ''N0'')' - ELSE N'qsws.total_query_wait_time_ms' - END - + N', - total_query_duration_ms = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(x.total_duration_ms, ''N0'')' - ELSE N'x.total_duration_ms' - END - + N', - avg_query_wait_time_ms = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsws.avg_query_wait_time_ms, ''N0'')' - ELSE N'qsws.avg_query_wait_time_ms' - END - + N', - avg_query_duration_ms = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(x.avg_duration_ms, ''N0'')' - ELSE N'x.avg_duration_ms' - END - + N', - last_query_wait_time_ms = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsws.last_query_wait_time_ms, ''N0'')' - ELSE N'qsws.last_query_wait_time_ms' - END - + N', - last_query_duration_ms = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(x.last_duration_ms, ''N0'')' - ELSE N'x.last_duration_ms' - END - + N', - min_query_wait_time_ms = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsws.min_query_wait_time_ms, ''N0'')' - ELSE N'qsws.min_query_wait_time_ms' - END - + N', - min_query_duration_ms = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(x.min_duration_ms, ''N0'')' - ELSE N'x.min_duration_ms' - END - + N', - max_query_wait_time_ms = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsws.max_query_wait_time_ms, ''N0'')' - ELSE N'qsws.max_query_wait_time_ms' - END - + N', - max_query_duration_ms = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(x.max_duration_ms, ''N0'')' - ELSE N'x.max_duration_ms' - END - + N' - FROM #query_store_wait_stats AS qsws - CROSS APPLY - ( - SELECT - qsrs.avg_duration_ms, - qsrs.last_duration_ms, - qsrs.min_duration_ms, - qsrs.max_duration_ms, - qsrs.total_duration_ms, - qsq.object_name - FROM #query_store_runtime_stats AS qsrs - JOIN #query_store_plan AS qsp - ON qsrs.plan_id = qsp.plan_id - AND qsrs.database_id = qsp.database_id - JOIN #query_store_query AS qsq - ON qsp.query_id = qsq.query_id - AND qsp.database_id = qsq.database_id - WHERE qsws.plan_id = qsrs.plan_id - AND qsws.database_id = qsrs.database_id - ) AS x - ORDER BY - qsws.plan_id, - qsws.total_query_wait_time_ms DESC - OPTION(RECOMPILE);' - ); - - IF @debug = 1 - BEGIN - PRINT LEN(@sql); - PRINT @sql; - END; - - EXECUTE sys.sp_executesql - @sql; - - /* - Wait stats in total - */ - SELECT - @current_table = 'selecting wait stats in total'; - - SET @sql = N''; - - SELECT - @sql = - CONVERT - ( - nvarchar(MAX), - N' - SELECT - source = - ''query_store_wait_stats_total'', - database_name = - DB_NAME(qsws.database_id), - qsws.wait_category_desc, - total_query_wait_time_ms = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(SUM(qsws.total_query_wait_time_ms), ''N0'')' - ELSE N'SUM(qsws.total_query_wait_time_ms)' - END - + N', - total_query_duration_ms = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(SUM(x.total_duration_ms), ''N0'')' - ELSE N'SUM(x.total_duration_ms)' - END - + N', - avg_query_wait_time_ms = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(SUM(qsws.avg_query_wait_time_ms), ''N0'')' - ELSE N'SUM(qsws.avg_query_wait_time_ms)' - END - + N', - avg_query_duration_ms = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(SUM(x.avg_duration_ms), ''N0'')' - ELSE N'SUM(x.avg_duration_ms)' - END - + N', - last_query_wait_time_ms = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(SUM(qsws.last_query_wait_time_ms), ''N0'')' - ELSE N'SUM(qsws.last_query_wait_time_ms)' - END - + N', - last_query_duration_ms = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(SUM(x.last_duration_ms), ''N0'')' - ELSE N'SUM(x.last_duration_ms)' - END - + N', - min_query_wait_time_ms = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(SUM(qsws.min_query_wait_time_ms), ''N0'')' - ELSE N'SUM(qsws.min_query_wait_time_ms)' - END - + N', - min_query_duration_ms = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(SUM(x.min_duration_ms), ''N0'')' - ELSE N'SUM(x.min_duration_ms)' - END - + N', - max_query_wait_time_ms = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(SUM(qsws.max_query_wait_time_ms), ''N0'')' - ELSE N'SUM(qsws.max_query_wait_time_ms)' - END - + N', - max_query_duration_ms = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(SUM(x.max_duration_ms), ''N0'')' - ELSE N'SUM(x.max_duration_ms)' - END - + N' - FROM #query_store_wait_stats AS qsws - CROSS APPLY - ( - SELECT - qsrs.avg_duration_ms, - qsrs.last_duration_ms, - qsrs.min_duration_ms, - qsrs.max_duration_ms, - qsrs.total_duration_ms, - qsq.object_name - FROM #query_store_runtime_stats AS qsrs - JOIN #query_store_plan AS qsp - ON qsrs.plan_id = qsp.plan_id - AND qsrs.database_id = qsp.database_id - JOIN #query_store_query AS qsq - ON qsp.query_id = qsq.query_id - AND qsp.database_id = qsq.database_id - WHERE qsws.plan_id = qsrs.plan_id - ) AS x - GROUP BY - qsws.wait_category_desc, - qsws.database_id - ORDER BY - SUM(qsws.total_query_wait_time_ms) DESC - OPTION(RECOMPILE);' - ); - - IF @debug = 1 - BEGIN - PRINT LEN(@sql); - PRINT @sql; - END; - - EXECUTE sys.sp_executesql - @sql; - - END; - ELSE - BEGIN - SELECT - result = - '#query_store_wait_stats is empty' + - CASE - WHEN - ( - @product_version = 13 - AND @azure = 0 - ) - THEN ' because it''s not available < 2017' - WHEN EXISTS - ( - SELECT - 1/0 - FROM #database_query_store_options AS dqso - WHERE dqso.wait_stats_capture_mode_desc <> 'ON' - ) - THEN ' because you have it disabled in your Query Store options' - ELSE ' for the queries in the results' - END; - END; - END; - END; /*End wait stats queries*/ - - IF - ( - @sql_2022_views = 1 - AND @ags_present = 1 - ) - BEGIN - IF @expert_mode = 1 - BEGIN - IF EXISTS - ( - SELECT - 1/0 - FROM #query_store_replicas AS qsr - JOIN #query_store_plan_forcing_locations AS qspfl - ON qsr.replica_group_id = qspfl.replica_group_id - AND qsr.database_id = qspfl.database_id - ) - BEGIN - SELECT - @current_table = 'selecting #query_store_replicas and #query_store_plan_forcing_locations'; - - SELECT - database_name = - DB_NAME(qsr.database_id), - qsr.replica_group_id, - qsr.role_type, - qsr.replica_name, - qspfl.plan_forcing_location_id, - qspfl.query_id, - qspfl.plan_id, - qspfl.replica_group_id - FROM #query_store_replicas AS qsr - JOIN #query_store_plan_forcing_locations AS qspfl - ON qsr.replica_group_id = qspfl.replica_group_id - ORDER BY - qsr.replica_group_id - OPTION(RECOMPILE);; - END; - ELSE - BEGIN - SELECT - result = 'Availability Group information is empty'; - END; - END; - END; - - IF @expert_mode = 1 - BEGIN - SELECT - @current_table = 'selecting query store options', - @sql = N''; - - SELECT - @sql += - CONVERT - ( - nvarchar(MAX), - N' - SELECT - source = - ''query_store_options'', - database_name = - DB_NAME(dqso.database_id), - dqso.desired_state_desc, - dqso.actual_state_desc, - dqso.readonly_reason, - dqso.current_storage_size_mb, - dqso.flush_interval_seconds, - dqso.interval_length_minutes, - dqso.max_storage_size_mb, - dqso.stale_query_threshold_days, - dqso.max_plans_per_query, - dqso.query_capture_mode_desc,' - + - CASE - WHEN - ( - @azure = 1 - OR @product_version > 13 - ) - THEN N' - dqso.wait_stats_capture_mode_desc,' - ELSE N'' - END - + - CASE - WHEN - ( - @azure = 1 - OR @product_version > 14 - ) - THEN N' - dqso.capture_policy_execution_count, - dqso.capture_policy_total_compile_cpu_time_ms, - dqso.capture_policy_total_execution_cpu_time_ms, - dqso.capture_policy_stale_threshold_hours,' - ELSE N'' - END - ); - - SELECT - @sql += - CONVERT - ( - nvarchar(MAX), - N' - dqso.size_based_cleanup_mode_desc - FROM #database_query_store_options AS dqso - OPTION(RECOMPILE);' - ); - - IF @debug = 1 - BEGIN - PRINT LEN(@sql); - PRINT @sql; - END; - - EXECUTE sys.sp_executesql - @sql; - END; -/* -Continue with code for special things -*/ - IF @sql_2022_views = 1 - BEGIN - IF @expert_mode = 1 - BEGIN - IF EXISTS - ( - SELECT - 1/0 - FROM #query_store_plan_feedback AS qspf - ) - BEGIN - SELECT - @current_table = 'selecting plan feedback'; - - /* - Use dynamic SQL to handle formatting differences based on @format_output - */ - SELECT - @sql = @isolation_level; - - SELECT - @sql += N' - SELECT - database_name = - DB_NAME(qspf.database_id), - qspf.plan_feedback_id, - qspf.plan_id, - qspf.feature_desc, - qspf.feedback_data, - qspf.state_desc, - create_time = - CASE - WHEN @timezone IS NULL - THEN - SWITCHOFFSET - ( - qspf.create_time, - @utc_offset_string - ) - WHEN @timezone IS NOT NULL - THEN qspf.create_time AT TIME ZONE @timezone - END, - create_time_utc = - qspf.create_time, - last_updated_time = - CASE - WHEN @timezone IS NULL - THEN - SWITCHOFFSET - ( - qspf.last_updated_time, - @utc_offset_string - ) - WHEN @timezone IS NOT NULL - THEN qspf.last_updated_time AT TIME ZONE @timezone - END, - last_updated_time_utc = - qspf.last_updated_time - FROM #query_store_plan_feedback AS qspf - ORDER BY - qspf.plan_id - OPTION(RECOMPILE);'; - - IF @debug = 1 - BEGIN - PRINT LEN(@sql); - PRINT @sql; - END; - - EXECUTE sys.sp_executesql - @sql, - N'@timezone sysname, @utc_offset_string nvarchar(max)', - @timezone, @utc_offset_string; - END; - ELSE - BEGIN - SELECT - result = '#query_store_plan_feedback is empty'; - END; - END; - - IF EXISTS - ( - SELECT - 1/0 - FROM #query_store_query_hints AS qsqh - ) - BEGIN - SELECT - @current_table = 'selecting query hints'; - - /* - Use dynamic SQL to handle formatting differences based on @format_output - */ - SELECT - @sql = @isolation_level; - - SELECT - @sql += N' - SELECT - database_name = - DB_NAME(qsqh.database_id), - qsqh.query_hint_id, - qsqh.query_id, - qsqh.query_hint_text, - qsqh.last_query_hint_failure_reason_desc, - query_hint_failure_count = ' + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqh.query_hint_failure_count, ''N0'')' - ELSE N'qsqh.query_hint_failure_count' - END + N', - qsqh.source_desc - FROM #query_store_query_hints AS qsqh - ORDER BY - qsqh.query_id - OPTION(RECOMPILE);'; - - IF @debug = 1 - BEGIN - PRINT LEN(@sql); - PRINT @sql; - END; - - EXECUTE sys.sp_executesql - @sql; - END; - ELSE - BEGIN - SELECT - result = '#query_store_query_hints is empty'; - END; - - IF @expert_mode = 1 - BEGIN - IF EXISTS - ( - SELECT - 1/0 - FROM #query_store_query_variant AS qsqv - ) - BEGIN - SELECT - @current_table = 'selecting query variants'; - - /* - Use dynamic SQL to handle formatting differences based on @format_output - */ - SELECT - @sql = @isolation_level; - - SELECT - @sql += N' - SELECT - database_name = - DB_NAME(qsqv.database_id), - qsqv.query_variant_query_id, - qsqv.parent_query_id, - qsqv.dispatcher_plan_id - FROM #query_store_query_variant AS qsqv - ORDER BY - qsqv.parent_query_id - OPTION(RECOMPILE);'; - - IF @debug = 1 - BEGIN - PRINT LEN(@sql); - PRINT @sql; - END; - - EXECUTE sys.sp_executesql - @sql; - END; - ELSE - BEGIN - SELECT - result = '#query_store_query_variant is empty'; - END; - END; - END; - - IF @expert_mode = 1 - BEGIN - IF EXISTS - ( - SELECT - 1/0 - FROM #query_store_query AS qsq - ) - BEGIN - SELECT - @current_table = 'selecting compilation stats'; - - /* - Use dynamic SQL to handle formatting differences based on @format_output - */ - SELECT - @sql = @isolation_level; - + @sql = @isolation_level; + SELECT @sql += N' SELECT @@ -9789,22 +9139,20 @@ Continue with code for special things '#query_store_query is empty'; END; END; - - IF @expert_mode = 1 - BEGIN + IF @rc > 0 BEGIN SELECT @current_table = 'selecting resource stats'; - - /* - Use dynamic SQL to handle formatting differences based on @format_output - */ - SELECT - @sql = @isolation_level; + + SET @sql = N''; SELECT - @sql += N' + @sql = + CONVERT + ( + nvarchar(MAX), + N' SELECT source = ''resource_stats'', @@ -9812,98 +9160,179 @@ Continue with code for special things DB_NAME(qsq.database_id), qsq.query_id, qsq.object_name, - total_grant_mb = ' + + total_grant_mb = ' + + CASE WHEN @format_output = 1 THEN N'FORMAT(qsqt.total_grant_mb, ''N0'')' ELSE N'qsqt.total_grant_mb' - END + N', - last_grant_mb = ' + + END + + N', + last_grant_mb = ' + + CASE WHEN @format_output = 1 THEN N'FORMAT(qsqt.last_grant_mb, ''N0'')' ELSE N'qsqt.last_grant_mb' - END + N', - min_grant_mb = ' + + END + + N', + min_grant_mb = ' + + CASE WHEN @format_output = 1 THEN N'FORMAT(qsqt.min_grant_mb, ''N0'')' ELSE N'qsqt.min_grant_mb' - END + N', - max_grant_mb = ' + + END + + N', + max_grant_mb = ' + + CASE WHEN @format_output = 1 THEN N'FORMAT(qsqt.max_grant_mb, ''N0'')' ELSE N'qsqt.max_grant_mb' - END + N', - total_used_grant_mb = ' + + END + + N', + total_used_grant_mb = ' + + CASE WHEN @format_output = 1 THEN N'FORMAT(qsqt.total_used_grant_mb, ''N0'')' ELSE N'qsqt.total_used_grant_mb' - END + N', - last_used_grant_mb = ' + + END + + N', + last_used_grant_mb = ' + + CASE WHEN @format_output = 1 THEN N'FORMAT(qsqt.last_used_grant_mb, ''N0'')' ELSE N'qsqt.last_used_grant_mb' - END + N', - min_used_grant_mb = ' + + END + + N', + min_used_grant_mb = ' + + CASE WHEN @format_output = 1 THEN N'FORMAT(qsqt.min_used_grant_mb, ''N0'')' ELSE N'qsqt.min_used_grant_mb' - END + N', - max_used_grant_mb = ' + + END + + N', + max_used_grant_mb = ' + + CASE WHEN @format_output = 1 THEN N'FORMAT(qsqt.max_used_grant_mb, ''N0'')' ELSE N'qsqt.max_used_grant_mb' - END + N', - total_ideal_grant_mb = ' + + END + + N', + total_ideal_grant_mb = ' + + CASE WHEN @format_output = 1 THEN N'FORMAT(qsqt.total_ideal_grant_mb, ''N0'')' ELSE N'qsqt.total_ideal_grant_mb' - END + N', - last_ideal_grant_mb = ' + + END + + N', + last_ideal_grant_mb = ' + + CASE WHEN @format_output = 1 THEN N'FORMAT(qsqt.last_ideal_grant_mb, ''N0'')' ELSE N'qsqt.last_ideal_grant_mb' - END + N', - min_ideal_grant_mb = ' + + END + + N', + min_ideal_grant_mb = ' + + CASE WHEN @format_output = 1 THEN N'FORMAT(qsqt.min_ideal_grant_mb, ''N0'')' ELSE N'qsqt.min_ideal_grant_mb' - END + N', - max_ideal_grant_mb = ' + + END + + N', + max_ideal_grant_mb = ' + + CASE WHEN @format_output = 1 THEN N'FORMAT(qsqt.max_ideal_grant_mb, ''N0'')' ELSE N'qsqt.max_ideal_grant_mb' - END + N', - qsqt.total_reserved_threads, - qsqt.last_reserved_threads, - qsqt.min_reserved_threads, - qsqt.max_reserved_threads, - qsqt.total_used_threads, - qsqt.last_used_threads, - qsqt.min_used_threads, - qsqt.max_used_threads + END + + N', + total_reserved_threads = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.total_reserved_threads, ''N0'')' + ELSE N'qsqt.total_reserved_threads' + END + + N', + last_reserved_threads = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.last_reserved_threads, ''N0'')' + ELSE N'qsqt.last_reserved_threads' + END + + N', + min_reserved_threads = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.min_reserved_threads, ''N0'')' + ELSE N'qsqt.min_reserved_threads' + END + + N', + max_reserved_threads = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.max_reserved_threads, ''N0'')' + ELSE N'qsqt.max_reserved_threads' + END + + N', + total_used_threads = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.total_used_threads, ''N0'')' + ELSE N'qsqt.total_used_threads' + END + + N', + last_used_threads = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.last_used_threads, ''N0'')' + ELSE N'qsqt.last_used_threads' + END + + N', + min_used_threads = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.min_used_threads, ''N0'')' + ELSE N'qsqt.min_used_threads' + END + + N', + max_used_threads = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.max_used_threads, ''N0'')' + ELSE N'qsqt.max_used_threads' + END + + N' FROM #query_store_query AS qsq JOIN #query_store_query_text AS qsqt ON qsq.query_text_id = qsqt.query_text_id AND qsq.database_id = qsqt.database_id WHERE ( - qsqt.total_grant_mb IS NOT NULL - OR qsqt.total_reserved_threads IS NOT NULL + qsqt.total_grant_mb IS NOT NULL + OR qsqt.total_reserved_threads IS NOT NULL ) ORDER BY qsq.query_id - OPTION(RECOMPILE);'; + OPTION(RECOMPILE);' + ); IF @debug = 1 BEGIN @@ -9914,96 +9343,401 @@ Continue with code for special things EXECUTE sys.sp_executesql @sql; - END; /*End resource stats section*/ + END; /*End resource stats query*/ ELSE BEGIN SELECT result = '#dm_exec_query_stats is empty'; END; - END; - - /* - Wait stats sections have already been handled above with dynamic SQL - */ - END; /*End wait stats queries*/ - ELSE + IF @new = 1 BEGIN - SELECT - result = - '#query_store_wait_stats is empty' + - CASE - WHEN ( - @product_version = 13 - AND @azure = 0 - ) - THEN ' because it''s not available < 2017' - WHEN EXISTS - ( - SELECT - 1/0 - FROM #database_query_store_options AS dqso - WHERE dqso.wait_stats_capture_mode_desc <> 'ON' - ) - THEN ' because you have it disabled in your Query Store options' - ELSE ' for the queries in the results' + IF @expert_mode = 1 + BEGIN + IF EXISTS + ( + SELECT + 1/0 + FROM #query_store_wait_stats AS qsws + ) + BEGIN + /* + Wait stats by query + */ + SELECT + @current_table = 'selecting wait stats by query'; + + SET @sql = N''; + + SELECT + @sql = + CONVERT + ( + nvarchar(MAX), + N' + SELECT DISTINCT + source = + ''query_store_wait_stats_by_query'', + database_name = + DB_NAME(qsws.database_id), + qsws.plan_id, + x.object_name, + qsws.wait_category_desc, + total_query_wait_time_ms = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsws.total_query_wait_time_ms, ''N0'')' + ELSE N'qsws.total_query_wait_time_ms' + END + + N', + total_query_duration_ms = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(x.total_duration_ms, ''N0'')' + ELSE N'x.total_duration_ms' + END + + N', + avg_query_wait_time_ms = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsws.avg_query_wait_time_ms, ''N0'')' + ELSE N'qsws.avg_query_wait_time_ms' + END + + N', + avg_query_duration_ms = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(x.avg_duration_ms, ''N0'')' + ELSE N'x.avg_duration_ms' + END + + N', + last_query_wait_time_ms = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsws.last_query_wait_time_ms, ''N0'')' + ELSE N'qsws.last_query_wait_time_ms' + END + + N', + last_query_duration_ms = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(x.last_duration_ms, ''N0'')' + ELSE N'x.last_duration_ms' + END + + N', + min_query_wait_time_ms = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsws.min_query_wait_time_ms, ''N0'')' + ELSE N'qsws.min_query_wait_time_ms' + END + + N', + min_query_duration_ms = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(x.min_duration_ms, ''N0'')' + ELSE N'x.min_duration_ms' + END + + N', + max_query_wait_time_ms = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsws.max_query_wait_time_ms, ''N0'')' + ELSE N'qsws.max_query_wait_time_ms' + END + + N', + max_query_duration_ms = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(x.max_duration_ms, ''N0'')' + ELSE N'x.max_duration_ms' + END + + N' + FROM #query_store_wait_stats AS qsws + CROSS APPLY + ( + SELECT + qsrs.avg_duration_ms, + qsrs.last_duration_ms, + qsrs.min_duration_ms, + qsrs.max_duration_ms, + qsrs.total_duration_ms, + qsq.object_name + FROM #query_store_runtime_stats AS qsrs + JOIN #query_store_plan AS qsp + ON qsrs.plan_id = qsp.plan_id + AND qsrs.database_id = qsp.database_id + JOIN #query_store_query AS qsq + ON qsp.query_id = qsq.query_id + AND qsp.database_id = qsq.database_id + WHERE qsws.plan_id = qsrs.plan_id + AND qsws.database_id = qsrs.database_id + ) AS x + ORDER BY + qsws.plan_id, + qsws.total_query_wait_time_ms DESC + OPTION(RECOMPILE);' + ); + + IF @debug = 1 + BEGIN + PRINT LEN(@sql); + PRINT @sql; END; - END; + + EXECUTE sys.sp_executesql + @sql; - IF - ( - @sql_2022_views = 1 - AND @ags_present = 1 - ) - BEGIN - IF @expert_mode = 1 - BEGIN - IF EXISTS - ( + /* + Wait stats in total + */ SELECT - 1/0 - FROM #query_store_replicas AS qsr - JOIN #query_store_plan_forcing_locations AS qspfl - ON qsr.replica_group_id = qspfl.replica_group_id - AND qsr.replica_group_id = qspfl.database_id - ) - BEGIN + @current_table = 'selecting wait stats in total'; + + SET @sql = N''; + SELECT - @current_table = '#query_store_replicas and #query_store_plan_forcing_locations'; - + @sql = + CONVERT + ( + nvarchar(MAX), + N' SELECT + source = + ''query_store_wait_stats_total'', database_name = - DB_NAME(qsr.database_id), - qsr.replica_group_id, - qsr.role_type, - qsr.replica_name, - qspfl.plan_forcing_location_id, - qspfl.query_id, - qspfl.plan_id, - qspfl.replica_group_id - FROM #query_store_replicas AS qsr - JOIN #query_store_plan_forcing_locations AS qspfl - ON qsr.replica_group_id = qspfl.replica_group_id - AND qsr.database_id = qspfl.database_id + DB_NAME(qsws.database_id), + qsws.wait_category_desc, + total_query_wait_time_ms = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(SUM(qsws.total_query_wait_time_ms), ''N0'')' + ELSE N'SUM(qsws.total_query_wait_time_ms)' + END + + N', + total_query_duration_ms = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(SUM(x.total_duration_ms), ''N0'')' + ELSE N'SUM(x.total_duration_ms)' + END + + N', + avg_query_wait_time_ms = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(SUM(qsws.avg_query_wait_time_ms), ''N0'')' + ELSE N'SUM(qsws.avg_query_wait_time_ms)' + END + + N', + avg_query_duration_ms = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(SUM(x.avg_duration_ms), ''N0'')' + ELSE N'SUM(x.avg_duration_ms)' + END + + N', + last_query_wait_time_ms = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(SUM(qsws.last_query_wait_time_ms), ''N0'')' + ELSE N'SUM(qsws.last_query_wait_time_ms)' + END + + N', + last_query_duration_ms = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(SUM(x.last_duration_ms), ''N0'')' + ELSE N'SUM(x.last_duration_ms)' + END + + N', + min_query_wait_time_ms = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(SUM(qsws.min_query_wait_time_ms), ''N0'')' + ELSE N'SUM(qsws.min_query_wait_time_ms)' + END + + N', + min_query_duration_ms = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(SUM(x.min_duration_ms), ''N0'')' + ELSE N'SUM(x.min_duration_ms)' + END + + N', + max_query_wait_time_ms = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(SUM(qsws.max_query_wait_time_ms), ''N0'')' + ELSE N'SUM(qsws.max_query_wait_time_ms)' + END + + N', + max_query_duration_ms = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(SUM(x.max_duration_ms), ''N0'')' + ELSE N'SUM(x.max_duration_ms)' + END + + N' + FROM #query_store_wait_stats AS qsws + CROSS APPLY + ( + SELECT + qsrs.avg_duration_ms, + qsrs.last_duration_ms, + qsrs.min_duration_ms, + qsrs.max_duration_ms, + qsrs.total_duration_ms, + qsq.object_name + FROM #query_store_runtime_stats AS qsrs + JOIN #query_store_plan AS qsp + ON qsrs.plan_id = qsp.plan_id + AND qsrs.database_id = qsp.database_id + JOIN #query_store_query AS qsq + ON qsp.query_id = qsq.query_id + AND qsp.database_id = qsq.database_id + WHERE qsws.plan_id = qsrs.plan_id + ) AS x + GROUP BY + qsws.wait_category_desc, + qsws.database_id ORDER BY - qsr.replica_group_id - OPTION(RECOMPILE); + SUM(qsws.total_query_wait_time_ms) DESC + OPTION(RECOMPILE);' + ); + + IF @debug = 1 + BEGIN + PRINT LEN(@sql); + PRINT @sql; + END; + + EXECUTE sys.sp_executesql + @sql; END; ELSE BEGIN SELECT - result = 'Availability Group information is empty'; + result = + '#query_store_wait_stats is empty' + + CASE + WHEN + ( + @product_version = 13 + AND @azure = 0 + ) + THEN ' because it''s not available < 2017' + WHEN EXISTS + ( + SELECT + 1/0 + FROM #database_query_store_options AS dqso + WHERE dqso.wait_stats_capture_mode_desc <> 'ON' + ) + THEN ' because you have it disabled in your Query Store options' + ELSE ' for the queries in the results' + END; END; END; - END; + END; /*End wait stats queries*/ - /* - Query store options section has already been handled with dynamic SQL above - */ + IF @expert_mode = 1 + BEGIN + SELECT + @current_table = 'selecting query store options', + @sql = N''; + + SELECT + @sql += + CONVERT + ( + nvarchar(MAX), + N' + SELECT + source = + ''query_store_options'', + database_name = + DB_NAME(dqso.database_id), + dqso.desired_state_desc, + dqso.actual_state_desc, + dqso.readonly_reason, + dqso.current_storage_size_mb, + dqso.flush_interval_seconds, + dqso.interval_length_minutes, + dqso.max_storage_size_mb, + dqso.stale_query_threshold_days, + dqso.max_plans_per_query, + dqso.query_capture_mode_desc,' + + + CASE + WHEN + ( + @azure = 1 + OR @product_version > 13 + ) + THEN N' + dqso.wait_stats_capture_mode_desc,' + ELSE N'' + END + + + CASE + WHEN + ( + @azure = 1 + OR @product_version > 14 + ) + THEN N' + dqso.capture_policy_execution_count, + dqso.capture_policy_total_compile_cpu_time_ms, + dqso.capture_policy_total_execution_cpu_time_ms, + dqso.capture_policy_stale_threshold_hours,' + ELSE N'' + END + ); + + SELECT + @sql += + CONVERT + ( + nvarchar(MAX), + N' + dqso.size_based_cleanup_mode_desc + FROM #database_query_store_options AS dqso + OPTION(RECOMPILE);' + ); + + IF @debug = 1 + BEGIN + PRINT LEN(@sql); + PRINT @sql; + END; + + EXECUTE sys.sp_executesql + @sql; END; -END; /*End special output section*/ + END; +END; /*End Expert Mode*/; IF @query_store_trouble = 1 BEGIN From b75b0553a20433c26b9bbb0ea7aff6529acfefda Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 17 Mar 2025 11:05:40 -0400 Subject: [PATCH 175/246] Update sp_QuickieStore.sql --- sp_QuickieStore/sp_QuickieStore.sql | 186 ++++++++++++++-------------- 1 file changed, 95 insertions(+), 91 deletions(-) diff --git a/sp_QuickieStore/sp_QuickieStore.sql b/sp_QuickieStore/sp_QuickieStore.sql index 804d485f..fe7def0c 100644 --- a/sp_QuickieStore/sp_QuickieStore.sql +++ b/sp_QuickieStore/sp_QuickieStore.sql @@ -8669,12 +8669,12 @@ BEGIN /* SQL 2022+ features: plan feedback, query hints, and query variants */ - IF @expert_mode = 1 + IF @sql_2022_views = 1 BEGIN /* Handle query_store_plan_feedback */ - IF @sql_2022_views = 1 + IF @expert_mode = 1 OR @only_queries_with_feedback = 1 BEGIN IF EXISTS ( @@ -8746,64 +8746,69 @@ BEGIN N'@timezone sysname, @utc_offset_string nvarchar(max)', @timezone, @utc_offset_string; END; - ELSE + ELSE IF @only_queries_with_feedback = 1 BEGIN SELECT result = '#query_store_plan_feedback is empty'; END; - IF EXISTS - ( - SELECT - 1/0 - FROM #query_store_query_hints AS qsqh - ) + IF @expert_mode = 1 OR @only_queries_with_hints = 1 BEGIN - SELECT - @current_table = 'selecting query hints'; + IF EXISTS + ( + SELECT + 1/0 + FROM #query_store_query_hints AS qsqh + ) + BEGIN + SELECT + @current_table = 'selecting query hints'; + + /* + Use dynamic SQL to handle formatting differences based on @format_output + */ + SELECT + @sql = @isolation_level; + + SELECT + @sql += N' + SELECT + database_name = + DB_NAME(qsqh.database_id), + qsqh.query_hint_id, + qsqh.query_id, + qsqh.query_hint_text, + qsqh.last_query_hint_failure_reason_desc, + query_hint_failure_count = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqh.query_hint_failure_count, ''N0'')' + ELSE N'qsqh.query_hint_failure_count' + END + N', + qsqh.source_desc + FROM #query_store_query_hints AS qsqh + ORDER BY + qsqh.query_id + OPTION(RECOMPILE);'; - /* - Use dynamic SQL to handle formatting differences based on @format_output - */ - SELECT - @sql = @isolation_level; + IF @debug = 1 + BEGIN + PRINT LEN(@sql); + PRINT @sql; + END; - SELECT - @sql += N' - SELECT - database_name = - DB_NAME(qsqh.database_id), - qsqh.query_hint_id, - qsqh.query_id, - qsqh.query_hint_text, - qsqh.last_query_hint_failure_reason_desc, - query_hint_failure_count = ' + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqh.query_hint_failure_count, ''N0'')' - ELSE N'qsqh.query_hint_failure_count' - END + N', - qsqh.source_desc - FROM #query_store_query_hints AS qsqh - ORDER BY - qsqh.query_id - OPTION(RECOMPILE);'; - - IF @debug = 1 + EXECUTE sys.sp_executesql + @sql; + END; + ELSE IF @only_queries_with_hints = 1 BEGIN - PRINT LEN(@sql); - PRINT @sql; + SELECT + result = '#query_store_query_hints is empty'; END; - - EXECUTE sys.sp_executesql - @sql; - END; - ELSE - BEGIN - SELECT - result = '#query_store_query_hints is empty'; END; + IF @expert_mode = 1 OR @only_queries_with_variants = 1 + BEGIN IF EXISTS ( SELECT @@ -8842,58 +8847,57 @@ BEGIN EXECUTE sys.sp_executesql @sql; END; - ELSE + ELSE IF @only_queries_with_variants = 1 BEGIN SELECT result = '#query_store_query_variant is empty'; END; + END; - IF - ( - @sql_2022_views = 1 - AND @ags_present = 1 - ) + IF + ( + @sql_2022_views = 1 + AND @ags_present = 1 + ) + BEGIN + IF @expert_mode = 1 BEGIN - IF @expert_mode = 1 + IF EXISTS + ( + SELECT + 1/0 + FROM #query_store_replicas AS qsr + JOIN #query_store_plan_forcing_locations AS qspfl + ON qsr.replica_group_id = qspfl.replica_group_id + AND qsr.database_id = qspfl.database_id + ) BEGIN - IF EXISTS - ( - SELECT - 1/0 - FROM #query_store_replicas AS qsr - JOIN #query_store_plan_forcing_locations AS qspfl - ON qsr.replica_group_id = qspfl.replica_group_id - AND qsr.database_id = qspfl.database_id - ) - BEGIN - SELECT - @current_table = 'selecting #query_store_replicas and #query_store_plan_forcing_locations'; - - SELECT - database_name = - DB_NAME(qsr.database_id), - qsr.replica_group_id, - qsr.role_type, - qsr.replica_name, - qspfl.plan_forcing_location_id, - qspfl.query_id, - qspfl.plan_id, - qspfl.replica_group_id - FROM #query_store_replicas AS qsr - JOIN #query_store_plan_forcing_locations AS qspfl - ON qsr.replica_group_id = qspfl.replica_group_id - ORDER BY - qsr.replica_group_id - OPTION(RECOMPILE);; - END; - ELSE - BEGIN - SELECT - result = 'Availability Group information is empty'; - END; + SELECT + @current_table = 'selecting #query_store_replicas and #query_store_plan_forcing_locations'; + + SELECT + database_name = + DB_NAME(qsr.database_id), + qsr.replica_group_id, + qsr.role_type, + qsr.replica_name, + qspfl.plan_forcing_location_id, + qspfl.query_id, + qspfl.plan_id, + qspfl.replica_group_id + FROM #query_store_replicas AS qsr + JOIN #query_store_plan_forcing_locations AS qspfl + ON qsr.replica_group_id = qspfl.replica_group_id + ORDER BY + qsr.replica_group_id + OPTION(RECOMPILE); + END; + ELSE + BEGIN + SELECT + result = 'Availability Group information is empty'; END; END; - END; /*End 2022 views*/ IF @expert_mode = 1 From d8643e93e4c782614868328c107cbcbade327efa Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 17 Mar 2025 11:13:57 -0400 Subject: [PATCH 176/246] Update sp_QuickieStore.sql --- sp_QuickieStore/sp_QuickieStore.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sp_QuickieStore/sp_QuickieStore.sql b/sp_QuickieStore/sp_QuickieStore.sql index fe7def0c..9ca5ad62 100644 --- a/sp_QuickieStore/sp_QuickieStore.sql +++ b/sp_QuickieStore/sp_QuickieStore.sql @@ -8898,7 +8898,8 @@ BEGIN result = 'Availability Group information is empty'; END; END; - END; /*End 2022 views*/ + END; + END; /*End 2022 views*/ IF @expert_mode = 1 BEGIN @@ -9740,8 +9741,7 @@ BEGIN @sql; END; - END; -END; /*End Expert Mode*/; +END; /*End Expert Mode*/ IF @query_store_trouble = 1 BEGIN From 2db612fb689a9342203391757faa00dd96d5bf21 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 17 Mar 2025 11:14:06 -0400 Subject: [PATCH 177/246] Update sp_QuickieStore.sql --- sp_QuickieStore/sp_QuickieStore.sql | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/sp_QuickieStore/sp_QuickieStore.sql b/sp_QuickieStore/sp_QuickieStore.sql index fe7def0c..f75d1241 100644 --- a/sp_QuickieStore/sp_QuickieStore.sql +++ b/sp_QuickieStore/sp_QuickieStore.sql @@ -8644,11 +8644,11 @@ OPTION(RECOMPILE);' @timezone; END; /*End runtime stats main query*/ ELSE - BEGIN - SELECT - result = - '#query_store_runtime_stats is empty'; - END; +BEGIN + SELECT + result = + '#query_store_runtime_stats is empty'; +END; /* Return special things: plan feedback, query hints, query variants, query text, wait stats, and query store options @@ -8674,7 +8674,8 @@ BEGIN /* Handle query_store_plan_feedback */ - IF @expert_mode = 1 OR @only_queries_with_feedback = 1 + IF @expert_mode = 1 + OR @only_queries_with_feedback = 1 BEGIN IF EXISTS ( @@ -8752,7 +8753,8 @@ BEGIN result = '#query_store_plan_feedback is empty'; END; - IF @expert_mode = 1 OR @only_queries_with_hints = 1 + IF @expert_mode = 1 + OR @only_queries_with_hints = 1 BEGIN IF EXISTS ( @@ -8807,7 +8809,8 @@ BEGIN END; END; - IF @expert_mode = 1 OR @only_queries_with_variants = 1 + IF @expert_mode = 1 + OR @only_queries_with_variants = 1 BEGIN IF EXISTS ( From 2c8b5b174117b0652d48507c355ef7e593da8f0b Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 17 Mar 2025 11:35:11 -0400 Subject: [PATCH 178/246] Update sp_QuickieStore.sql --- sp_QuickieStore/sp_QuickieStore.sql | 445 ++++++++++++++-------------- 1 file changed, 229 insertions(+), 216 deletions(-) diff --git a/sp_QuickieStore/sp_QuickieStore.sql b/sp_QuickieStore/sp_QuickieStore.sql index 9605f6ff..11785680 100644 --- a/sp_QuickieStore/sp_QuickieStore.sql +++ b/sp_QuickieStore/sp_QuickieStore.sql @@ -1241,7 +1241,7 @@ CREATE TABLE end_time ), 'N0' - ) PERSISTED NOT NULL + ) ); /*Gonna try gathering this based on*/ @@ -8752,7 +8752,8 @@ BEGIN SELECT result = '#query_store_plan_feedback is empty'; END; - + END; /*@only_queries_with_feedback*/ + IF @expert_mode = 1 OR @only_queries_with_hints = 1 BEGIN @@ -8807,7 +8808,7 @@ BEGIN SELECT result = '#query_store_query_hints is empty'; END; - END; + END; /*@only_queries_with_hints*/ IF @expert_mode = 1 OR @only_queries_with_variants = 1 @@ -8855,7 +8856,7 @@ BEGIN SELECT result = '#query_store_query_variant is empty'; END; - END; + END; /*@only_queries_with_variants*/ IF ( @@ -8901,7 +8902,7 @@ BEGIN result = 'Availability Group information is empty'; END; END; - END; + END; /*@ags_present*/ END; /*End 2022 views*/ IF @expert_mode = 1 @@ -8980,6 +8981,9 @@ BEGIN last_execution_time_utc = qsq.last_execution_time, count_compiles = ' + + CONVERT + ( + nvarchar(max), CASE WHEN @format_output = 1 THEN N'FORMAT(qsq.count_compiles, ''N0'')' @@ -9098,7 +9102,8 @@ BEGIN WHEN @format_output = 1 THEN N'FORMAT(qsq.max_compile_memory_mb, ''N0'')' ELSE N'qsq.max_compile_memory_mb' - END + N', + END + ) + N', qsq.query_hash, qsq.batch_sql_handle, qsqt.statement_sql_handle, @@ -9139,225 +9144,228 @@ BEGIN N'@timezone sysname, @utc_offset_string nvarchar(max)', @timezone, @utc_offset_string; - END; /*End compilation stats section*/ + END; /*End compilation query section*/ ELSE BEGIN SELECT result = '#query_store_query is empty'; END; - END; + END; /*compilation stats*/ - IF @rc > 0 - BEGIN - SELECT - @current_table = 'selecting resource stats'; - - SET @sql = N''; + IF @rc > 0 + BEGIN + SELECT + @current_table = 'selecting resource stats'; - SELECT - @sql = + SET @sql = N''; + + SELECT + @sql = + CONVERT + ( + nvarchar(MAX), + N' + SELECT + source = + ''resource_stats'', + database_name = + DB_NAME(qsq.database_id), + qsq.query_id, + qsq.object_name, + total_grant_mb = ' + + CONVERT ( - nvarchar(MAX), - N' - SELECT - source = - ''resource_stats'', - database_name = - DB_NAME(qsq.database_id), - qsq.query_id, - qsq.object_name, - total_grant_mb = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.total_grant_mb, ''N0'')' - ELSE N'qsqt.total_grant_mb' - END - + N', - last_grant_mb = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.last_grant_mb, ''N0'')' - ELSE N'qsqt.last_grant_mb' - END - + N', - min_grant_mb = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.min_grant_mb, ''N0'')' - ELSE N'qsqt.min_grant_mb' - END - + N', - max_grant_mb = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.max_grant_mb, ''N0'')' - ELSE N'qsqt.max_grant_mb' - END - + N', - total_used_grant_mb = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.total_used_grant_mb, ''N0'')' - ELSE N'qsqt.total_used_grant_mb' - END - + N', - last_used_grant_mb = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.last_used_grant_mb, ''N0'')' - ELSE N'qsqt.last_used_grant_mb' - END - + N', - min_used_grant_mb = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.min_used_grant_mb, ''N0'')' - ELSE N'qsqt.min_used_grant_mb' - END - + N', - max_used_grant_mb = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.max_used_grant_mb, ''N0'')' - ELSE N'qsqt.max_used_grant_mb' - END - + N', - total_ideal_grant_mb = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.total_ideal_grant_mb, ''N0'')' - ELSE N'qsqt.total_ideal_grant_mb' - END - + N', - last_ideal_grant_mb = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.last_ideal_grant_mb, ''N0'')' - ELSE N'qsqt.last_ideal_grant_mb' - END - + N', - min_ideal_grant_mb = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.min_ideal_grant_mb, ''N0'')' - ELSE N'qsqt.min_ideal_grant_mb' - END - + N', - max_ideal_grant_mb = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.max_ideal_grant_mb, ''N0'')' - ELSE N'qsqt.max_ideal_grant_mb' - END - + N', - total_reserved_threads = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.total_reserved_threads, ''N0'')' - ELSE N'qsqt.total_reserved_threads' - END - + N', - last_reserved_threads = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.last_reserved_threads, ''N0'')' - ELSE N'qsqt.last_reserved_threads' - END - + N', - min_reserved_threads = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.min_reserved_threads, ''N0'')' - ELSE N'qsqt.min_reserved_threads' - END - + N', - max_reserved_threads = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.max_reserved_threads, ''N0'')' - ELSE N'qsqt.max_reserved_threads' - END - + N', - total_used_threads = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.total_used_threads, ''N0'')' - ELSE N'qsqt.total_used_threads' - END - + N', - last_used_threads = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.last_used_threads, ''N0'')' - ELSE N'qsqt.last_used_threads' - END - + N', - min_used_threads = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.min_used_threads, ''N0'')' - ELSE N'qsqt.min_used_threads' - END - + N', - max_used_threads = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.max_used_threads, ''N0'')' - ELSE N'qsqt.max_used_threads' - END - + N' - FROM #query_store_query AS qsq - JOIN #query_store_query_text AS qsqt - ON qsq.query_text_id = qsqt.query_text_id - AND qsq.database_id = qsqt.database_id - WHERE - ( - qsqt.total_grant_mb IS NOT NULL - OR qsqt.total_reserved_threads IS NOT NULL - ) - ORDER BY - qsq.query_id - OPTION(RECOMPILE);' - ); - - IF @debug = 1 - BEGIN - PRINT LEN(@sql); - PRINT @sql; - END; - - EXECUTE sys.sp_executesql - @sql; - - END; /*End resource stats query*/ - ELSE + nvarchar(max), + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.total_grant_mb, ''N0'')' + ELSE N'qsqt.total_grant_mb' + END + + N', + last_grant_mb = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.last_grant_mb, ''N0'')' + ELSE N'qsqt.last_grant_mb' + END + + N', + min_grant_mb = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.min_grant_mb, ''N0'')' + ELSE N'qsqt.min_grant_mb' + END + + N', + max_grant_mb = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.max_grant_mb, ''N0'')' + ELSE N'qsqt.max_grant_mb' + END + + N', + total_used_grant_mb = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.total_used_grant_mb, ''N0'')' + ELSE N'qsqt.total_used_grant_mb' + END + + N', + last_used_grant_mb = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.last_used_grant_mb, ''N0'')' + ELSE N'qsqt.last_used_grant_mb' + END + + N', + min_used_grant_mb = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.min_used_grant_mb, ''N0'')' + ELSE N'qsqt.min_used_grant_mb' + END + + N', + max_used_grant_mb = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.max_used_grant_mb, ''N0'')' + ELSE N'qsqt.max_used_grant_mb' + END + + N', + total_ideal_grant_mb = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.total_ideal_grant_mb, ''N0'')' + ELSE N'qsqt.total_ideal_grant_mb' + END + + N', + last_ideal_grant_mb = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.last_ideal_grant_mb, ''N0'')' + ELSE N'qsqt.last_ideal_grant_mb' + END + + N', + min_ideal_grant_mb = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.min_ideal_grant_mb, ''N0'')' + ELSE N'qsqt.min_ideal_grant_mb' + END + + N', + max_ideal_grant_mb = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.max_ideal_grant_mb, ''N0'')' + ELSE N'qsqt.max_ideal_grant_mb' + END + + N', + total_reserved_threads = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.total_reserved_threads, ''N0'')' + ELSE N'qsqt.total_reserved_threads' + END + + N', + last_reserved_threads = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.last_reserved_threads, ''N0'')' + ELSE N'qsqt.last_reserved_threads' + END + + N', + min_reserved_threads = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.min_reserved_threads, ''N0'')' + ELSE N'qsqt.min_reserved_threads' + END + + N', + max_reserved_threads = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.max_reserved_threads, ''N0'')' + ELSE N'qsqt.max_reserved_threads' + END + + N', + total_used_threads = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.total_used_threads, ''N0'')' + ELSE N'qsqt.total_used_threads' + END + + N', + last_used_threads = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.last_used_threads, ''N0'')' + ELSE N'qsqt.last_used_threads' + END + + N', + min_used_threads = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.min_used_threads, ''N0'')' + ELSE N'qsqt.min_used_threads' + END + + N', + max_used_threads = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.max_used_threads, ''N0'')' + ELSE N'qsqt.max_used_threads' + END + ) + N' + FROM #query_store_query AS qsq + JOIN #query_store_query_text AS qsqt + ON qsq.query_text_id = qsqt.query_text_id + AND qsq.database_id = qsqt.database_id + WHERE + ( + qsqt.total_grant_mb IS NOT NULL + OR qsqt.total_reserved_threads IS NOT NULL + ) + ORDER BY + qsq.query_id + OPTION(RECOMPILE);' + ); + + IF @debug = 1 BEGIN - SELECT - result = - '#dm_exec_query_stats is empty'; + PRINT LEN(@sql); + PRINT @sql; END; + + EXECUTE sys.sp_executesql + @sql; + + END; /*End resource stats query*/ + ELSE + BEGIN + SELECT + result = + '#dm_exec_query_stats is empty'; + END; IF @new = 1 BEGIN @@ -9394,6 +9402,9 @@ BEGIN qsws.wait_category_desc, total_query_wait_time_ms = ' + + CONVERT + ( + nvarchar(max), CASE WHEN @format_output = 1 THEN N'FORMAT(qsws.total_query_wait_time_ms, ''N0'')' @@ -9471,7 +9482,7 @@ BEGIN THEN N'FORMAT(x.max_duration_ms, ''N0'')' ELSE N'x.max_duration_ms' END - + N' + ) + N' FROM #query_store_wait_stats AS qsws CROSS APPLY ( @@ -9529,6 +9540,9 @@ BEGIN qsws.wait_category_desc, total_query_wait_time_ms = ' + + CONVERT + ( + nvarchar(max), CASE WHEN @format_output = 1 THEN N'FORMAT(SUM(qsws.total_query_wait_time_ms), ''N0'')' @@ -9606,7 +9620,7 @@ BEGIN THEN N'FORMAT(SUM(x.max_duration_ms), ''N0'')' ELSE N'SUM(x.max_duration_ms)' END - + N' + ) + N' FROM #query_store_wait_stats AS qsws CROSS APPLY ( @@ -9743,7 +9757,6 @@ BEGIN EXECUTE sys.sp_executesql @sql; END; - END; /*End Expert Mode*/ IF @query_store_trouble = 1 From 6b21a8912889a37bd77f0c1dd820c077c397778f Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 17 Mar 2025 12:36:30 -0400 Subject: [PATCH 179/246] Update sp_QuickieStore.sql --- sp_QuickieStore/sp_QuickieStore.sql | 348 ++++++---------------------- 1 file changed, 65 insertions(+), 283 deletions(-) diff --git a/sp_QuickieStore/sp_QuickieStore.sql b/sp_QuickieStore/sp_QuickieStore.sql index 11785680..d6b7d9f2 100644 --- a/sp_QuickieStore/sp_QuickieStore.sql +++ b/sp_QuickieStore/sp_QuickieStore.sql @@ -8981,128 +8981,42 @@ BEGIN last_execution_time_utc = qsq.last_execution_time, count_compiles = ' + + + /* + Create a template for formatting number columns with thousands separators + */ + DECLARE + @number_format_template nvarchar(max) = CASE + WHEN @format_output = 1 + THEN N'FORMAT({column}, ''N0'')' + ELSE N'{column}' + END; + CONVERT ( nvarchar(max), - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsq.count_compiles, ''N0'')' - ELSE N'qsq.count_compiles' - END + N', - avg_compile_duration_ms = ' + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsq.avg_compile_duration_ms, ''N0'')' - ELSE N'qsq.avg_compile_duration_ms' - END + N', - total_compile_duration_ms = ' + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsq.total_compile_duration_ms, ''N0'')' - ELSE N'qsq.total_compile_duration_ms' - END + N', - last_compile_duration_ms = ' + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsq.last_compile_duration_ms, ''N0'')' - ELSE N'qsq.last_compile_duration_ms' - END + N', - avg_bind_duration_ms = ' + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsq.avg_bind_duration_ms, ''N0'')' - ELSE N'qsq.avg_bind_duration_ms' - END + N', - total_bind_duration_ms = ' + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsq.total_bind_duration_ms, ''N0'')' - ELSE N'qsq.total_bind_duration_ms' - END + N', - last_bind_duration_ms = ' + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsq.last_bind_duration_ms, ''N0'')' - ELSE N'qsq.last_bind_duration_ms' - END + N', - avg_bind_cpu_time_ms = ' + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsq.avg_bind_cpu_time_ms, ''N0'')' - ELSE N'qsq.avg_bind_cpu_time_ms' - END + N', - total_bind_cpu_time_ms = ' + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsq.total_bind_cpu_time_ms, ''N0'')' - ELSE N'qsq.total_bind_cpu_time_ms' - END + N', - last_bind_cpu_time_ms = ' + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsq.last_bind_cpu_time_ms, ''N0'')' - ELSE N'qsq.last_bind_cpu_time_ms' - END + N', - avg_optimize_duration_ms = ' + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsq.avg_optimize_duration_ms, ''N0'')' - ELSE N'qsq.avg_optimize_duration_ms' - END + N', - total_optimize_duration_ms = ' + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsq.total_optimize_duration_ms, ''N0'')' - ELSE N'qsq.total_optimize_duration_ms' - END + N', - last_optimize_duration_ms = ' + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsq.last_optimize_duration_ms, ''N0'')' - ELSE N'qsq.last_optimize_duration_ms' - END + N', - avg_optimize_cpu_time_ms = ' + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsq.avg_optimize_cpu_time_ms, ''N0'')' - ELSE N'qsq.avg_optimize_cpu_time_ms' - END + N', - total_optimize_cpu_time_ms = ' + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsq.total_optimize_cpu_time_ms, ''N0'')' - ELSE N'qsq.total_optimize_cpu_time_ms' - END + N', - last_optimize_cpu_time_ms = ' + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsq.last_optimize_cpu_time_ms, ''N0'')' - ELSE N'qsq.last_optimize_cpu_time_ms' + REPLACE(@number_format_template, N'{column}', N'qsq.count_compiles') + N', + avg_compile_duration_ms = ' + REPLACE(@number_format_template, N'{column}', N'qsq.avg_compile_duration_ms') + N', + total_compile_duration_ms = ' + REPLACE(@number_format_template, N'{column}', N'qsq.total_compile_duration_ms') + N', + last_compile_duration_ms = ' + REPLACE(@number_format_template, N'{column}', N'qsq.last_compile_duration_ms') + N', + avg_bind_duration_ms = ' + REPLACE(@number_format_template, N'{column}', N'qsq.avg_bind_duration_ms') + N', + total_bind_duration_ms = ' + REPLACE(@number_format_template, N'{column}', N'qsq.total_bind_duration_ms') + N', + last_bind_duration_ms = ' + REPLACE(@number_format_template, N'{column}', N'qsq.last_bind_duration_ms') + N', + avg_bind_cpu_time_ms = ' + REPLACE(@number_format_template, N'{column}', N'qsq.avg_bind_cpu_time_ms') + N', + total_bind_cpu_time_ms = ' + REPLACE(@number_format_template, N'{column}', N'qsq.total_bind_cpu_time_ms') + N', + last_bind_cpu_time_ms = ' + REPLACE(@number_format_template, N'{column}', N'qsq.last_bind_cpu_time_ms') + N', + avg_optimize_duration_ms = ' + REPLACE(@number_format_template, N'{column}', N'qsq.avg_optimize_duration_ms') + N', + total_optimize_duration_ms = ' + REPLACE(@number_format_template, N'{column}', N'qsq.total_optimize_duration_ms') + N', + last_optimize_duration_ms = ' + REPLACE(@number_format_template, N'{column}', N'qsq.last_optimize_duration_ms') + N', + avg_optimize_cpu_time_ms = ' + REPLACE(@number_format_template, N'{column}', N'qsq.avg_optimize_cpu_time_ms') + N', + total_optimize_cpu_time_ms = ' + REPLACE(@number_format_template, N'{column}', N'qsq.total_optimize_cpu_time_ms') + N', + last_optimize_cpu_time_ms = ' + REPLACE(@number_format_template, N'{column}', N'qsq.last_optimize_cpu_time_ms') END + N', - avg_compile_memory_mb = ' + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsq.avg_compile_memory_mb, ''N0'')' - ELSE N'qsq.avg_compile_memory_mb' - END + N', - total_compile_memory_mb = ' + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsq.total_compile_memory_mb, ''N0'')' - ELSE N'qsq.total_compile_memory_mb' + avg_compile_memory_mb = ' + REPLACE(@number_format_template, N'{column}', N'qsq.avg_compile_memory_mb') + N', + total_compile_memory_mb = ' + REPLACE(@number_format_template, N'{column}', N'qsq.total_compile_memory_mb') + N', + last_compile_memory_mb = ' + REPLACE(@number_format_template, N'{column}', N'qsq.last_compile_memory_mb') END + N', - last_compile_memory_mb = ' + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsq.last_compile_memory_mb, ''N0'')' - ELSE N'qsq.last_compile_memory_mb' - END + N', - max_compile_memory_mb = ' + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsq.max_compile_memory_mb, ''N0'')' - ELSE N'qsq.max_compile_memory_mb' - END + max_compile_memory_mb = ' + REPLACE(@number_format_template, N'{column}', N'qsq.max_compile_memory_mb') ) + N', qsq.query_hash, qsq.batch_sql_handle, @@ -9158,14 +9072,24 @@ BEGIN SELECT @current_table = 'selecting resource stats'; + /* + Create a template for formatting number columns with thousands separators + */ + DECLARE + @number_format_template nvarchar(max) = CASE + WHEN @format_output = 1 + THEN N'FORMAT({column}, ''N0'')' + ELSE N'{column}' + END; + SET @sql = N''; + /* + Build the query using a much simpler pattern - replace {column} with the actual column name + */ SELECT @sql = - CONVERT - ( - nvarchar(MAX), - N' + N' SELECT source = ''resource_stats'', @@ -9173,168 +9097,26 @@ BEGIN DB_NAME(qsq.database_id), qsq.query_id, qsq.object_name, - total_grant_mb = ' - + - CONVERT - ( - nvarchar(max), - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.total_grant_mb, ''N0'')' - ELSE N'qsqt.total_grant_mb' - END - + N', - last_grant_mb = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.last_grant_mb, ''N0'')' - ELSE N'qsqt.last_grant_mb' - END - + N', - min_grant_mb = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.min_grant_mb, ''N0'')' - ELSE N'qsqt.min_grant_mb' - END - + N', - max_grant_mb = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.max_grant_mb, ''N0'')' - ELSE N'qsqt.max_grant_mb' - END - + N', - total_used_grant_mb = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.total_used_grant_mb, ''N0'')' - ELSE N'qsqt.total_used_grant_mb' - END - + N', - last_used_grant_mb = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.last_used_grant_mb, ''N0'')' - ELSE N'qsqt.last_used_grant_mb' - END - + N', - min_used_grant_mb = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.min_used_grant_mb, ''N0'')' - ELSE N'qsqt.min_used_grant_mb' - END - + N', - max_used_grant_mb = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.max_used_grant_mb, ''N0'')' - ELSE N'qsqt.max_used_grant_mb' - END - + N', - total_ideal_grant_mb = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.total_ideal_grant_mb, ''N0'')' - ELSE N'qsqt.total_ideal_grant_mb' - END - + N', - last_ideal_grant_mb = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.last_ideal_grant_mb, ''N0'')' - ELSE N'qsqt.last_ideal_grant_mb' - END - + N', - min_ideal_grant_mb = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.min_ideal_grant_mb, ''N0'')' - ELSE N'qsqt.min_ideal_grant_mb' - END - + N', - max_ideal_grant_mb = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.max_ideal_grant_mb, ''N0'')' - ELSE N'qsqt.max_ideal_grant_mb' - END - + N', - total_reserved_threads = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.total_reserved_threads, ''N0'')' - ELSE N'qsqt.total_reserved_threads' - END - + N', - last_reserved_threads = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.last_reserved_threads, ''N0'')' - ELSE N'qsqt.last_reserved_threads' - END - + N', - min_reserved_threads = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.min_reserved_threads, ''N0'')' - ELSE N'qsqt.min_reserved_threads' - END - + N', - max_reserved_threads = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.max_reserved_threads, ''N0'')' - ELSE N'qsqt.max_reserved_threads' - END - + N', - total_used_threads = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.total_used_threads, ''N0'')' - ELSE N'qsqt.total_used_threads' - END - + N', - last_used_threads = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.last_used_threads, ''N0'')' - ELSE N'qsqt.last_used_threads' - END - + N', - min_used_threads = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.min_used_threads, ''N0'')' - ELSE N'qsqt.min_used_threads' - END - + N', - max_used_threads = ' - + - CASE - WHEN @format_output = 1 - THEN N'FORMAT(qsqt.max_used_threads, ''N0'')' - ELSE N'qsqt.max_used_threads' - END + total_grant_mb = ' + REPLACE(@number_format_template, N'{column}', N'qsqt.total_grant_mb') + N', + last_grant_mb = ' + REPLACE(@number_format_template, N'{column}', N'qsqt.last_grant_mb') + N', + min_grant_mb = ' + REPLACE(@number_format_template, N'{column}', N'qsqt.min_grant_mb') + N', + max_grant_mb = ' + REPLACE(@number_format_template, N'{column}', N'qsqt.max_grant_mb') + N', + total_used_grant_mb = ' + REPLACE(@number_format_template, N'{column}', N'qsqt.total_used_grant_mb') + N', + last_used_grant_mb = ' + REPLACE(@number_format_template, N'{column}', N'qsqt.last_used_grant_mb') + N', + min_used_grant_mb = ' + REPLACE(@number_format_template, N'{column}', N'qsqt.min_used_grant_mb') + N', + max_used_grant_mb = ' + REPLACE(@number_format_template, N'{column}', N'qsqt.max_used_grant_mb') + N', + total_ideal_grant_mb = ' + REPLACE(@number_format_template, N'{column}', N'qsqt.total_ideal_grant_mb') + N', + last_ideal_grant_mb = ' + REPLACE(@number_format_template, N'{column}', N'qsqt.last_ideal_grant_mb') + N', + min_ideal_grant_mb = ' + REPLACE(@number_format_template, N'{column}', N'qsqt.min_ideal_grant_mb') + N', + max_ideal_grant_mb = ' + REPLACE(@number_format_template, N'{column}', N'qsqt.max_ideal_grant_mb') + N', + total_reserved_threads = ' + REPLACE(@number_format_template, N'{column}', N'qsqt.total_reserved_threads') + N', + last_reserved_threads = ' + REPLACE(@number_format_template, N'{column}', N'qsqt.last_reserved_threads') + N', + min_reserved_threads = ' + REPLACE(@number_format_template, N'{column}', N'qsqt.min_reserved_threads') + N', + max_reserved_threads = ' + REPLACE(@number_format_template, N'{column}', N'qsqt.max_reserved_threads') + N', + total_used_threads = ' + REPLACE(@number_format_template, N'{column}', N'qsqt.total_used_threads') + N', + last_used_threads = ' + REPLACE(@number_format_template, N'{column}', N'qsqt.last_used_threads') + N', + min_used_threads = ' + REPLACE(@number_format_template, N'{column}', N'qsqt.min_used_threads') + N', + max_used_threads = ' + REPLACE(@number_format_template, N'{column}', N'qsqt.max_used_threads') ) + N' FROM #query_store_query AS qsq JOIN #query_store_query_text AS qsqt From 07d757be7cce4a2f25553dadec32089a731f50d7 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 17 Mar 2025 12:40:23 -0400 Subject: [PATCH 180/246] Revert "Update sp_QuickieStore.sql" This reverts commit 6b21a8912889a37bd77f0c1dd820c077c397778f. --- sp_QuickieStore/sp_QuickieStore.sql | 348 ++++++++++++++++++++++------ 1 file changed, 283 insertions(+), 65 deletions(-) diff --git a/sp_QuickieStore/sp_QuickieStore.sql b/sp_QuickieStore/sp_QuickieStore.sql index d6b7d9f2..11785680 100644 --- a/sp_QuickieStore/sp_QuickieStore.sql +++ b/sp_QuickieStore/sp_QuickieStore.sql @@ -8981,42 +8981,128 @@ BEGIN last_execution_time_utc = qsq.last_execution_time, count_compiles = ' + - - /* - Create a template for formatting number columns with thousands separators - */ - DECLARE - @number_format_template nvarchar(max) = CASE - WHEN @format_output = 1 - THEN N'FORMAT({column}, ''N0'')' - ELSE N'{column}' - END; - CONVERT ( nvarchar(max), - REPLACE(@number_format_template, N'{column}', N'qsq.count_compiles') + N', - avg_compile_duration_ms = ' + REPLACE(@number_format_template, N'{column}', N'qsq.avg_compile_duration_ms') + N', - total_compile_duration_ms = ' + REPLACE(@number_format_template, N'{column}', N'qsq.total_compile_duration_ms') + N', - last_compile_duration_ms = ' + REPLACE(@number_format_template, N'{column}', N'qsq.last_compile_duration_ms') + N', - avg_bind_duration_ms = ' + REPLACE(@number_format_template, N'{column}', N'qsq.avg_bind_duration_ms') + N', - total_bind_duration_ms = ' + REPLACE(@number_format_template, N'{column}', N'qsq.total_bind_duration_ms') + N', - last_bind_duration_ms = ' + REPLACE(@number_format_template, N'{column}', N'qsq.last_bind_duration_ms') + N', - avg_bind_cpu_time_ms = ' + REPLACE(@number_format_template, N'{column}', N'qsq.avg_bind_cpu_time_ms') + N', - total_bind_cpu_time_ms = ' + REPLACE(@number_format_template, N'{column}', N'qsq.total_bind_cpu_time_ms') + N', - last_bind_cpu_time_ms = ' + REPLACE(@number_format_template, N'{column}', N'qsq.last_bind_cpu_time_ms') + N', - avg_optimize_duration_ms = ' + REPLACE(@number_format_template, N'{column}', N'qsq.avg_optimize_duration_ms') + N', - total_optimize_duration_ms = ' + REPLACE(@number_format_template, N'{column}', N'qsq.total_optimize_duration_ms') + N', - last_optimize_duration_ms = ' + REPLACE(@number_format_template, N'{column}', N'qsq.last_optimize_duration_ms') + N', - avg_optimize_cpu_time_ms = ' + REPLACE(@number_format_template, N'{column}', N'qsq.avg_optimize_cpu_time_ms') + N', - total_optimize_cpu_time_ms = ' + REPLACE(@number_format_template, N'{column}', N'qsq.total_optimize_cpu_time_ms') + N', - last_optimize_cpu_time_ms = ' + REPLACE(@number_format_template, N'{column}', N'qsq.last_optimize_cpu_time_ms') + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsq.count_compiles, ''N0'')' + ELSE N'qsq.count_compiles' + END + N', + avg_compile_duration_ms = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsq.avg_compile_duration_ms, ''N0'')' + ELSE N'qsq.avg_compile_duration_ms' + END + N', + total_compile_duration_ms = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsq.total_compile_duration_ms, ''N0'')' + ELSE N'qsq.total_compile_duration_ms' + END + N', + last_compile_duration_ms = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsq.last_compile_duration_ms, ''N0'')' + ELSE N'qsq.last_compile_duration_ms' + END + N', + avg_bind_duration_ms = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsq.avg_bind_duration_ms, ''N0'')' + ELSE N'qsq.avg_bind_duration_ms' + END + N', + total_bind_duration_ms = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsq.total_bind_duration_ms, ''N0'')' + ELSE N'qsq.total_bind_duration_ms' + END + N', + last_bind_duration_ms = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsq.last_bind_duration_ms, ''N0'')' + ELSE N'qsq.last_bind_duration_ms' + END + N', + avg_bind_cpu_time_ms = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsq.avg_bind_cpu_time_ms, ''N0'')' + ELSE N'qsq.avg_bind_cpu_time_ms' + END + N', + total_bind_cpu_time_ms = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsq.total_bind_cpu_time_ms, ''N0'')' + ELSE N'qsq.total_bind_cpu_time_ms' + END + N', + last_bind_cpu_time_ms = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsq.last_bind_cpu_time_ms, ''N0'')' + ELSE N'qsq.last_bind_cpu_time_ms' + END + N', + avg_optimize_duration_ms = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsq.avg_optimize_duration_ms, ''N0'')' + ELSE N'qsq.avg_optimize_duration_ms' + END + N', + total_optimize_duration_ms = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsq.total_optimize_duration_ms, ''N0'')' + ELSE N'qsq.total_optimize_duration_ms' + END + N', + last_optimize_duration_ms = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsq.last_optimize_duration_ms, ''N0'')' + ELSE N'qsq.last_optimize_duration_ms' + END + N', + avg_optimize_cpu_time_ms = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsq.avg_optimize_cpu_time_ms, ''N0'')' + ELSE N'qsq.avg_optimize_cpu_time_ms' + END + N', + total_optimize_cpu_time_ms = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsq.total_optimize_cpu_time_ms, ''N0'')' + ELSE N'qsq.total_optimize_cpu_time_ms' + END + N', + last_optimize_cpu_time_ms = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsq.last_optimize_cpu_time_ms, ''N0'')' + ELSE N'qsq.last_optimize_cpu_time_ms' END + N', - avg_compile_memory_mb = ' + REPLACE(@number_format_template, N'{column}', N'qsq.avg_compile_memory_mb') + N', - total_compile_memory_mb = ' + REPLACE(@number_format_template, N'{column}', N'qsq.total_compile_memory_mb') + N', - last_compile_memory_mb = ' + REPLACE(@number_format_template, N'{column}', N'qsq.last_compile_memory_mb') + avg_compile_memory_mb = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsq.avg_compile_memory_mb, ''N0'')' + ELSE N'qsq.avg_compile_memory_mb' + END + N', + total_compile_memory_mb = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsq.total_compile_memory_mb, ''N0'')' + ELSE N'qsq.total_compile_memory_mb' END + N', - max_compile_memory_mb = ' + REPLACE(@number_format_template, N'{column}', N'qsq.max_compile_memory_mb') + last_compile_memory_mb = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsq.last_compile_memory_mb, ''N0'')' + ELSE N'qsq.last_compile_memory_mb' + END + N', + max_compile_memory_mb = ' + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsq.max_compile_memory_mb, ''N0'')' + ELSE N'qsq.max_compile_memory_mb' + END ) + N', qsq.query_hash, qsq.batch_sql_handle, @@ -9072,24 +9158,14 @@ BEGIN SELECT @current_table = 'selecting resource stats'; - /* - Create a template for formatting number columns with thousands separators - */ - DECLARE - @number_format_template nvarchar(max) = CASE - WHEN @format_output = 1 - THEN N'FORMAT({column}, ''N0'')' - ELSE N'{column}' - END; - SET @sql = N''; - /* - Build the query using a much simpler pattern - replace {column} with the actual column name - */ SELECT @sql = - N' + CONVERT + ( + nvarchar(MAX), + N' SELECT source = ''resource_stats'', @@ -9097,26 +9173,168 @@ BEGIN DB_NAME(qsq.database_id), qsq.query_id, qsq.object_name, - total_grant_mb = ' + REPLACE(@number_format_template, N'{column}', N'qsqt.total_grant_mb') + N', - last_grant_mb = ' + REPLACE(@number_format_template, N'{column}', N'qsqt.last_grant_mb') + N', - min_grant_mb = ' + REPLACE(@number_format_template, N'{column}', N'qsqt.min_grant_mb') + N', - max_grant_mb = ' + REPLACE(@number_format_template, N'{column}', N'qsqt.max_grant_mb') + N', - total_used_grant_mb = ' + REPLACE(@number_format_template, N'{column}', N'qsqt.total_used_grant_mb') + N', - last_used_grant_mb = ' + REPLACE(@number_format_template, N'{column}', N'qsqt.last_used_grant_mb') + N', - min_used_grant_mb = ' + REPLACE(@number_format_template, N'{column}', N'qsqt.min_used_grant_mb') + N', - max_used_grant_mb = ' + REPLACE(@number_format_template, N'{column}', N'qsqt.max_used_grant_mb') + N', - total_ideal_grant_mb = ' + REPLACE(@number_format_template, N'{column}', N'qsqt.total_ideal_grant_mb') + N', - last_ideal_grant_mb = ' + REPLACE(@number_format_template, N'{column}', N'qsqt.last_ideal_grant_mb') + N', - min_ideal_grant_mb = ' + REPLACE(@number_format_template, N'{column}', N'qsqt.min_ideal_grant_mb') + N', - max_ideal_grant_mb = ' + REPLACE(@number_format_template, N'{column}', N'qsqt.max_ideal_grant_mb') + N', - total_reserved_threads = ' + REPLACE(@number_format_template, N'{column}', N'qsqt.total_reserved_threads') + N', - last_reserved_threads = ' + REPLACE(@number_format_template, N'{column}', N'qsqt.last_reserved_threads') + N', - min_reserved_threads = ' + REPLACE(@number_format_template, N'{column}', N'qsqt.min_reserved_threads') + N', - max_reserved_threads = ' + REPLACE(@number_format_template, N'{column}', N'qsqt.max_reserved_threads') + N', - total_used_threads = ' + REPLACE(@number_format_template, N'{column}', N'qsqt.total_used_threads') + N', - last_used_threads = ' + REPLACE(@number_format_template, N'{column}', N'qsqt.last_used_threads') + N', - min_used_threads = ' + REPLACE(@number_format_template, N'{column}', N'qsqt.min_used_threads') + N', - max_used_threads = ' + REPLACE(@number_format_template, N'{column}', N'qsqt.max_used_threads') + total_grant_mb = ' + + + CONVERT + ( + nvarchar(max), + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.total_grant_mb, ''N0'')' + ELSE N'qsqt.total_grant_mb' + END + + N', + last_grant_mb = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.last_grant_mb, ''N0'')' + ELSE N'qsqt.last_grant_mb' + END + + N', + min_grant_mb = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.min_grant_mb, ''N0'')' + ELSE N'qsqt.min_grant_mb' + END + + N', + max_grant_mb = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.max_grant_mb, ''N0'')' + ELSE N'qsqt.max_grant_mb' + END + + N', + total_used_grant_mb = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.total_used_grant_mb, ''N0'')' + ELSE N'qsqt.total_used_grant_mb' + END + + N', + last_used_grant_mb = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.last_used_grant_mb, ''N0'')' + ELSE N'qsqt.last_used_grant_mb' + END + + N', + min_used_grant_mb = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.min_used_grant_mb, ''N0'')' + ELSE N'qsqt.min_used_grant_mb' + END + + N', + max_used_grant_mb = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.max_used_grant_mb, ''N0'')' + ELSE N'qsqt.max_used_grant_mb' + END + + N', + total_ideal_grant_mb = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.total_ideal_grant_mb, ''N0'')' + ELSE N'qsqt.total_ideal_grant_mb' + END + + N', + last_ideal_grant_mb = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.last_ideal_grant_mb, ''N0'')' + ELSE N'qsqt.last_ideal_grant_mb' + END + + N', + min_ideal_grant_mb = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.min_ideal_grant_mb, ''N0'')' + ELSE N'qsqt.min_ideal_grant_mb' + END + + N', + max_ideal_grant_mb = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.max_ideal_grant_mb, ''N0'')' + ELSE N'qsqt.max_ideal_grant_mb' + END + + N', + total_reserved_threads = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.total_reserved_threads, ''N0'')' + ELSE N'qsqt.total_reserved_threads' + END + + N', + last_reserved_threads = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.last_reserved_threads, ''N0'')' + ELSE N'qsqt.last_reserved_threads' + END + + N', + min_reserved_threads = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.min_reserved_threads, ''N0'')' + ELSE N'qsqt.min_reserved_threads' + END + + N', + max_reserved_threads = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.max_reserved_threads, ''N0'')' + ELSE N'qsqt.max_reserved_threads' + END + + N', + total_used_threads = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.total_used_threads, ''N0'')' + ELSE N'qsqt.total_used_threads' + END + + N', + last_used_threads = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.last_used_threads, ''N0'')' + ELSE N'qsqt.last_used_threads' + END + + N', + min_used_threads = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.min_used_threads, ''N0'')' + ELSE N'qsqt.min_used_threads' + END + + N', + max_used_threads = ' + + + CASE + WHEN @format_output = 1 + THEN N'FORMAT(qsqt.max_used_threads, ''N0'')' + ELSE N'qsqt.max_used_threads' + END ) + N' FROM #query_store_query AS qsq JOIN #query_store_query_text AS qsqt From 235db41f6b6a5e859ffb05411b3b4e335a50ebe8 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 17 Mar 2025 21:16:42 -0400 Subject: [PATCH 181/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index fe225a09..f1720ff1 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -81,16 +81,19 @@ BEGIN TRY IF @help = 1 BEGIN SELECT - help = N'hello, i am sp_IndexCleanup - BETA' + help = N'hello, i am sp_IndexCleanup' UNION ALL SELECT - help = N'this is a script to help clean up unused and duplicate indexes' + help = N'this is a script to help clean up unused and duplicate indexes.' UNION ALL SELECT - help = N'you are currently using a beta version, and the advice should not be followed' + help = N'it will also give you scripted out statements to add page compression to uncompressed indexes.' UNION ALL SELECT - help = N'without careful analysis and consideration. it may be harmful.'; + help = N'always validate all changes against a non-production environment!' + UNION ALL + SELECT + help = N'without careful analysis and consideration, index changes can negative impacts on performance.'; /* Parameters @@ -127,7 +130,7 @@ BEGIN TRY WHEN N'@table_name' THEN 'table name or NULL for all tables' WHEN N'@min_reads' THEN 'any positive integer or 0' WHEN N'@min_writes' THEN 'any positive integer or 0' - WHEN N'@min_size_gb' THEN 'any positive decimal number or 0' + WHEN N'@min_size_gb' THEN 'any positive decimal or 0' WHEN N'@min_rows' THEN 'any positive integer or 0' WHEN N'@get_all_databases' THEN '0 or 1' WHEN N'@include_databases' THEN 'comma-separated list of database names' From dab531ba4010464df0a9f2e77fff308a93800aeb Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 17 Mar 2025 21:21:23 -0400 Subject: [PATCH 182/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index f1720ff1..0c628aca 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -910,6 +910,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* If no database name specified, use current database if not a system database */ IF @database_name IS NULL + AND @get_all_databases = 0 AND DB_NAME() NOT IN ( N'master', @@ -1104,7 +1105,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END; /* Parameter validation for multi-database mode */ - IF @get_all_databases = 1 AND @database_name IS NOT NULL + IF @get_all_databases = 1 + AND @database_name IS NOT NULL BEGIN RAISERROR('You cannot specify both @get_all_databases = 1 and a specific @database_name. Using @get_all_databases = 1 and ignoring @database_name.', 10, 1) WITH NOWAIT; SET @database_name = NULL; From 96f182e67892ebda80c080fd46bf2a1ad221aa1d Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 17 Mar 2025 22:08:14 -0400 Subject: [PATCH 183/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 668 +++++++++++++++------------- 1 file changed, 352 insertions(+), 316 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 0c628aca..7d0a9593 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -235,11 +235,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ) IN (3, 5, 8) OR ( - CONVERT - ( - integer, - SERVERPROPERTY('EngineEdition') - ) = 2 + CONVERT + ( + integer, + SERVERPROPERTY('EngineEdition') + ) = 2 AND CONVERT ( integer, @@ -290,6 +290,71 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ELSE 0 END; + /* Parameter validation */ + IF @min_reads < 0 + OR @min_reads IS NULL + BEGIN + IF @debug = 1 + BEGIN + RAISERROR('Parameter @min_reads cannot be NULL or negative. Setting to 0.', 10, 1) WITH NOWAIT; + END; + + SET @min_reads = 0; + END; + + IF @min_writes < 0 + OR @min_writes IS NULL + BEGIN + IF @debug = 1 + BEGIN + RAISERROR('Parameter @min_writes cannot be NULL or negative. Setting to 0.', 10, 1) WITH NOWAIT; + END; + + SET @min_writes = 0; + END; + + IF @min_size_gb < 0 + OR @min_size_gb IS NULL + BEGIN + IF @debug = 1 + BEGIN + RAISERROR('Parameter @min_size_gb cannot be NULL or negative. Setting to 0.', 10, 1) WITH NOWAIT; + END; + + SET @min_size_gb = 0; + END; + + IF @min_rows < 0 + OR @min_rows IS NULL + BEGIN + IF @debug = 1 + BEGIN + RAISERROR('Parameter @min_rows cannot be NULL or negative. Setting to 0.', 10, 1) WITH NOWAIT; + END; + + SET @min_rows = 0; + END; + + IF @schema_name IS NULL + AND @table_name IS NOT NULL + BEGIN + IF @debug = 1 + BEGIN + RAISERROR('Parameter @schema_name cannot be NULL when specifying a table, defaulting to dbo', 10, 1) WITH NOWAIT; + END; + + SET @schema_name = N'dbo'; + END; + + /* Parameter validation for multi-database mode */ + IF @get_all_databases = 1 + AND @database_name IS NOT NULL + BEGIN + RAISERROR('You cannot specify both @get_all_databases = 1 and a specific @database_name. Using @get_all_databases = 1 and ignoring @database_name.', 10, 1) WITH NOWAIT; + + SET @database_name = NULL; + END; + /* Temp tables! */ @@ -589,6 +654,14 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. reason nvarchar(255) NOT NULL ); + /* Create a table to parse the include/exclude lists */ + CREATE TABLE + #database_list + ( + id integer IDENTITY PRIMARY KEY CLUSTERED, + database_name sysname NOT NULL + ); + /* Handle multi-database mode */ IF @get_all_databases = 1 BEGIN @@ -597,14 +670,6 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. RAISERROR('Multi-database mode enabled, gathering database list...', 0, 0) WITH NOWAIT; END; - /* Create a table to parse the include/exclude lists */ - CREATE TABLE - #database_list - ( - id integer IDENTITY PRIMARY KEY CLUSTERED, - database_name sysname NOT NULL - ); - /* Parse @include_databases if specified - using XML for string splitting instead of STRING_SPLIT (version compatibility) */ IF @include_databases IS NOT NULL BEGIN @@ -634,264 +699,263 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. FROM @include_xml.nodes('/i') AS t(i) WHERE LTRIM(RTRIM(t.i.value('.', 'sysname'))) <> '' OPTION(RECOMPILE); - - /* Check for databases in both include and exclude lists */ - IF @exclude_databases IS NOT NULL - BEGIN - SELECT - @exclude_xml = - CONVERT - ( - xml, - '' + - REPLACE - ( - @exclude_databases, - ',', - '' - ) + - '' - ); - - /* Build list of conflicting databases */ - SELECT - @conflict_list = - @conflict_list + - LTRIM(RTRIM(t.i.value('.', 'sysname'))) + - N', ' - FROM @exclude_xml.nodes('/i') AS t(i) - WHERE LTRIM(RTRIM(t.i.value('.', 'sysname'))) <> '' - AND EXISTS - ( - SELECT - 1/0 - FROM #database_list AS dl - WHERE dl.database_name = LTRIM(RTRIM(t.i.value('.', 'sysname'))) - ) - OPTION(RECOMPILE);; - - /* If we found any conflicts, raise an error */ - IF LEN(@conflict_list) > 0 - BEGIN - /* Remove trailing comma and space */ - SET @conflict_list = LEFT(@conflict_list, LEN(@conflict_list) - 2); - - SET @error_msg = - N'The following databases appear in both @include_databases and @exclude_databases, which creates ambiguity: ' + - @conflict_list + N'. Please remove these databases from one of the lists.'; - - RAISERROR(@error_msg, 16, 1) WITH NOWAIT; - RETURN; - END; - END; END; - /* - Check SQL Server engine edition and use appropriate query paths - */ - IF - ( - SELECT - CONVERT - ( - sysname, - SERVERPROPERTY('EngineEdition') - ) - ) IN (5, 8) /* Azure SQL DB or Managed Instance */ - BEGIN - /* Get all eligible databases for Azure SQL */ - INSERT INTO - #databases_to_process - WITH - (TABLOCK) - ( - database_id, - database_name - ) - SELECT - database_id = d.database_id, - database_name = d.name - FROM sys.databases AS d - WHERE d.name NOT IN (N'master', N'model', N'msdb', N'tempdb', N'rdsadmin') - AND d.state = 0 - AND d.is_in_standby = 0 - AND d.is_read_only = 0 - AND d.database_id > 4 /* Skip system databases */ - AND - ( - /* If include list is provided, only keep databases in that list */ - (@include_databases IS NULL) OR - (d.name IN (SELECT database_name FROM #database_list)) - ) - OPTION(RECOMPILE); - END - ELSE /* Regular SQL Server */ - BEGIN - /* Get all eligible databases with AG primary replica check */ - INSERT INTO - #databases_to_process - WITH - (TABLOCK) - ( - database_id, - database_name - ) - SELECT - database_id = d.database_id, - database_name = d.name - FROM sys.databases AS d - WHERE d.name NOT IN (N'master', N'model', N'msdb', N'tempdb', N'rdsadmin') - AND d.state = 0 - AND d.is_in_standby = 0 - AND d.is_read_only = 0 - AND d.database_id > 4 /* Skip system databases */ - /* Add AG check to ensure we only process the primary replica */ - AND NOT EXISTS - ( - SELECT - 1/0 - FROM sys.dm_hadr_availability_replica_states AS s - JOIN sys.availability_databases_cluster AS c - ON s.group_id = c.group_id - AND d.name = c.database_name - WHERE s.is_local <> 1 - AND s.role_desc <> N'PRIMARY' - AND DATABASEPROPERTYEX(c.database_name, N'Updateability') <> N'READ_WRITE' - ) - AND - ( - /* If include list is provided, only keep databases in that list */ - (@include_databases IS NULL) OR - (d.name IN (SELECT database_name FROM #database_list)) - ) - OPTION(RECOMPILE);; - END; - - /* Remove excluded databases if specified */ + /* Check for databases in both include and exclude lists */ IF @exclude_databases IS NOT NULL BEGIN - SELECT @exclude_xml = CONVERT(xml, '' + REPLACE(@exclude_databases, ',', '') + ''); - - DELETE - dp - FROM #databases_to_process AS dp - WHERE EXISTS - ( - SELECT - 1/0 - FROM @exclude_xml.nodes('/i') AS t(i) - WHERE dp.database_name = LTRIM(RTRIM(t.i.value('.', 'sysname'))) - AND LTRIM(RTRIM(t.i.value('.', 'sysname'))) <> '' - ) - OPTION(RECOMPILE);; - END; - - IF @debug = 1 - BEGIN - /* Count databases */ SELECT - database_count = COUNT_BIG(*) - FROM #databases_to_process; - - /* List databases without using STRING_AGG (version compatibility) */ - SELECT @db_list = N''; + @exclude_xml = + CONVERT + ( + xml, + '' + + REPLACE + ( + @exclude_databases, + ',', + '' + ) + + '' + ); + /* Build list of conflicting databases */ SELECT - @db_list = - @db_list + - database_name + + @conflict_list = + @conflict_list + + LTRIM(RTRIM(t.i.value('.', 'sysname'))) + N', ' - FROM #databases_to_process AS dtp - ORDER BY - dtp.database_name + FROM @exclude_xml.nodes('/i') AS t(i) + WHERE LTRIM(RTRIM(t.i.value('.', 'sysname'))) <> '' + AND EXISTS + ( + SELECT + 1/0 + FROM #database_list AS dl + WHERE dl.database_name = LTRIM(RTRIM(t.i.value('.', 'sysname'))) + ) OPTION(RECOMPILE);; - /* Remove trailing comma if list is not empty */ - IF LEN(@db_list) > 0 + /* If we found any conflicts, raise an error */ + IF LEN(@conflict_list) > 0 BEGIN - SET @db_list = LEFT(@db_list, LEN(@db_list) - 1); - END; + /* Remove trailing comma and space */ + SET @conflict_list = LEFT(@conflict_list, LEN(@conflict_list) - 2); - RAISERROR('Databases to process: %s', 0, 0, @db_list) WITH NOWAIT; + SET @error_msg = + N'The following databases appear in both @include_databases and @exclude_databases, which creates ambiguity: ' + + @conflict_list + N'. Please remove these databases from one of the lists.'; + + RAISERROR(@error_msg, 16, 1) WITH NOWAIT; + RETURN; + END; END; - /* Track databases that were requested but skipped (for better reporting) */ - IF @include_databases IS NOT NULL - BEGIN - INSERT - #skipped_databases - WITH - (TABLOCK) + /* + Check SQL Server engine edition and use appropriate query paths + */ + IF + ( + SELECT + CONVERT ( - database_name, - reason + sysname, + SERVERPROPERTY('EngineEdition') ) + ) IN (5, 8) /* Azure SQL DB or Managed Instance */ + BEGIN + /* Get all eligible databases for Azure SQL */ + INSERT INTO + #databases_to_process + WITH + (TABLOCK) + ( + database_id, + database_name + ) + SELECT + database_id = d.database_id, + database_name = d.name + FROM sys.databases AS d + WHERE d.name NOT IN (N'master', N'model', N'msdb', N'tempdb') + AND d.state = 0 + AND d.is_in_standby = 0 + AND d.is_read_only = 0 + AND d.database_id > 4 /* Skip system databases */ + AND + ( + /* If include list is provided, only keep databases in that list */ + (@include_databases IS NULL) OR + (d.name IN (SELECT database_name FROM #database_list)) + ) + OPTION(RECOMPILE); + END; + ELSE /* Regular SQL Server */ + BEGIN + /* Get all eligible databases with AG primary replica check */ + INSERT INTO + #databases_to_process + WITH + (TABLOCK) + ( + database_id, + database_name + ) + SELECT + database_id = d.database_id, + database_name = d.name + FROM sys.databases AS d + WHERE d.name NOT IN (N'master', N'model', N'msdb', N'tempdb', N'rdsadmin') + AND d.state = 0 + AND d.is_in_standby = 0 + AND d.is_read_only = 0 + AND d.database_id > 4 /* Skip system databases */ + /* Add AG check to ensure we only process the primary replica */ + AND NOT EXISTS + ( SELECT - database_name = dl.database_name, - reason = - CASE - WHEN d.name IS NULL THEN N'Database does not exist' - WHEN d.state <> 0 THEN N'Database not online' - WHEN d.is_in_standby = 1 THEN N'Database is in standby' - WHEN d.is_read_only = 1 THEN N'Database is read-only' - WHEN d.database_id <= 4 THEN N'System database' - WHEN EXISTS - ( - SELECT - 1/0 - FROM sys.dm_hadr_availability_replica_states AS s - JOIN sys.availability_databases_cluster AS c - ON s.group_id = c.group_id - AND d.name = c.database_name - WHERE s.is_local <> 1 - AND s.role_desc <> N'PRIMARY' - AND DATABASEPROPERTYEX(c.database_name, N'Updateability') <> N'READ_WRITE' - ) THEN N'AG replica issue - not primary or read-write' - ELSE N'Other issue' - END - FROM #database_list AS dl - LEFT JOIN sys.databases AS d - ON dl.database_name = d.name - WHERE NOT EXISTS - ( - SELECT - 1/0 - FROM #databases_to_process AS dp - WHERE dp.database_name = dl.database_name - ) - OPTION(RECOMPILE); - END; + 1/0 + FROM sys.dm_hadr_availability_replica_states AS s + JOIN sys.availability_databases_cluster AS c + ON s.group_id = c.group_id + AND d.name = c.database_name + WHERE s.is_local <> 1 + AND s.role_desc <> N'PRIMARY' + AND DATABASEPROPERTYEX(c.database_name, N'Updateability') <> N'READ_WRITE' + ) + AND + ( + /* If include list is provided, only keep databases in that list */ + (@include_databases IS NULL) OR + (d.name IN (SELECT database_name FROM #database_list)) + ) + OPTION(RECOMPILE);; + END; + + /* Remove excluded databases if specified */ + IF @exclude_databases IS NOT NULL + BEGIN + SELECT @exclude_xml = CONVERT(xml, '' + REPLACE(@exclude_databases, ',', '') + ''); - /* Also track explicitly excluded databases */ - IF @exclude_databases IS NOT NULL + DELETE + dp + FROM #databases_to_process AS dp + WHERE EXISTS + ( + SELECT + 1/0 + FROM @exclude_xml.nodes('/i') AS t(i) + WHERE dp.database_name = LTRIM(RTRIM(t.i.value('.', 'sysname'))) + AND LTRIM(RTRIM(t.i.value('.', 'sysname'))) <> '' + ) + OPTION(RECOMPILE);; + END; + + IF @debug = 1 + BEGIN + /* Count databases */ + SELECT + database_count = COUNT_BIG(*) + FROM #databases_to_process; + + /* List databases without using STRING_AGG (version compatibility) */ + SELECT @db_list = N''; + + SELECT + @db_list = + @db_list + + database_name + + N', ' + FROM #databases_to_process AS dtp + ORDER BY + dtp.database_name + OPTION(RECOMPILE);; + + /* Remove trailing comma if list is not empty */ + IF LEN(@db_list) > 0 BEGIN - INSERT - #skipped_databases - WITH - (TABLOCK) - ( - database_name, - reason - ) - SELECT - database_name = LTRIM(RTRIM(t.i.value('.', 'nvarchar(128)'))), - reason = N'Explicitly excluded by @exclude_databases parameter' - FROM - ( - SELECT xml_list = CONVERT(xml, N'' + - REPLACE(@exclude_databases, N',', N'') + N'') - ) AS a - CROSS APPLY a.xml_list.nodes('i') AS t(i) - WHERE LTRIM(RTRIM(t.i.value('.', 'sysname'))) <> '' - AND EXISTS - ( - SELECT - 1/0 - FROM sys.databases AS d - WHERE d.name = LTRIM(RTRIM(t.i.value('.', 'sysname'))) - ) - OPTION(RECOMPILE);; + SET @db_list = LEFT(@db_list, LEN(@db_list) - 1); END; + + RAISERROR('Databases to process: %s', 0, 0, @db_list) WITH NOWAIT; + END; + + /* Track databases that were requested but skipped (for better reporting) */ + IF @include_databases IS NOT NULL + BEGIN + INSERT + #skipped_databases + WITH + (TABLOCK) + ( + database_name, + reason + ) + SELECT + database_name = dl.database_name, + reason = + CASE + WHEN d.name IS NULL THEN N'Database does not exist' + WHEN d.state <> 0 THEN N'Database not online' + WHEN d.is_in_standby = 1 THEN N'Database is in standby' + WHEN d.is_read_only = 1 THEN N'Database is read-only' + WHEN d.database_id <= 4 THEN N'System database' + WHEN EXISTS + ( + SELECT + 1/0 + FROM sys.dm_hadr_availability_replica_states AS s + JOIN sys.availability_databases_cluster AS c + ON s.group_id = c.group_id + AND d.name = c.database_name + WHERE s.is_local <> 1 + AND s.role_desc <> N'PRIMARY' + AND DATABASEPROPERTYEX(c.database_name, N'Updateability') <> N'READ_WRITE' + ) THEN N'AG replica issue - not primary or read-write' + ELSE N'Other issue' + END + FROM #database_list AS dl + LEFT JOIN sys.databases AS d + ON dl.database_name = d.name + WHERE NOT EXISTS + ( + SELECT + 1/0 + FROM #databases_to_process AS dp + WHERE dp.database_name = dl.database_name + ) + OPTION(RECOMPILE); + END; + + /* Also track explicitly excluded databases */ + IF @exclude_databases IS NOT NULL + BEGIN + INSERT + #skipped_databases + WITH + (TABLOCK) + ( + database_name, + reason + ) + SELECT + database_name = LTRIM(RTRIM(t.i.value('.', 'nvarchar(128)'))), + reason = N'Explicitly excluded by @exclude_databases parameter' + FROM + ( + SELECT xml_list = CONVERT(xml, N'' + + REPLACE(@exclude_databases, N',', N'') + N'') + ) AS a + CROSS APPLY a.xml_list.nodes('i') AS t(i) + WHERE LTRIM(RTRIM(t.i.value('.', 'sysname'))) <> '' + AND EXISTS + ( + SELECT + 1/0 + FROM sys.databases AS d + WHERE d.name = LTRIM(RTRIM(t.i.value('.', 'sysname'))) + ) + OPTION(RECOMPILE); /* If no databases match criteria, exit */ IF NOT EXISTS (SELECT 1/0 FROM #databases_to_process AS dtp) @@ -899,7 +963,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. RAISERROR('No eligible databases found to process with the specified filters', 16, 1) WITH NOWAIT; RETURN; END; - END + END; ELSE BEGIN /* Single database mode */ @@ -924,6 +988,27 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. @database_name = DB_NAME(); END; + /*Construct the full object name*/ + IF @schema_name IS NOT NULL + AND @table_name IS NOT NULL + BEGIN + SELECT + @full_object_name = + QUOTENAME(@database_name) + + N'.' + + QUOTENAME(@schema_name) + + N'.' + + QUOTENAME(@table_name); + + SET @object_id = OBJECT_ID(@full_object_name); + END; + + IF @object_id IS NULL + BEGIN + RAISERROR('The object %s doesn''t seem to exist', 16, 1, @full_object_name) WITH NOWAIT; + RETURN; + END; + /* Add the single database to the processing list */ IF @database_name IS NOT NULL BEGIN @@ -949,7 +1034,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. RAISERROR('The specified database %s does not exist, is not in ONLINE state, or you do not have permission to access it', 16, 1, @database_name) WITH NOWAIT; RETURN; END; - END + END; ELSE BEGIN RAISERROR('No valid database specified and current database is a system database. Please specify a user database.', 16, 1) WITH NOWAIT; @@ -1039,78 +1124,29 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. RAISERROR('Processing database %s', 0, 0, @database_name) WITH NOWAIT; END; - /* Checking parameters */ - IF @debug = 1 - BEGIN - RAISERROR('Checking paramaters...', 0, 0) WITH NOWAIT; - END; - - /* Continue with current database processing without recreating temp tables */ - IF @schema_name IS NULL + IF @schema_name IS NOT NULL AND @table_name IS NOT NULL BEGIN SELECT - @schema_name = N'dbo'; + @full_object_name = + QUOTENAME(@database_name) + + N'.' + + QUOTENAME(@schema_name) + + N'.' + + QUOTENAME(@table_name); + + SET @object_id = OBJECT_ID(@full_object_name); END; - IF @schema_name IS NOT NULL - AND @table_name IS NOT NULL + IF @object_id IS NULL BEGIN - SELECT - @full_object_name = - QUOTENAME(@database_name) + - N'.' + - QUOTENAME(@schema_name) + - N'.' + - QUOTENAME(@table_name); - - SELECT - @object_id = - OBJECT_ID(@full_object_name); - - IF @object_id IS NULL - BEGIN - RAISERROR('The object %s doesn''t seem to exist', 16, 1, @full_object_name) WITH NOWAIT; - RETURN; + RAISERROR('The object %s doesn''t seem to exist', 16, 1, @full_object_name) WITH NOWAIT; + RETURN; + END; END; - END; - /* Parameter validation */ - IF @min_reads < 0 - OR @min_reads IS NULL - BEGIN - RAISERROR('Parameter @min_reads cannot be NULL or negative. Setting to 0.', 10, 1) WITH NOWAIT; - SET @min_reads = 0; - END; - - IF @min_writes < 0 - OR @min_writes IS NULL - BEGIN - RAISERROR('Parameter @min_writes cannot be NULL or negative. Setting to 0.', 10, 1) WITH NOWAIT; - SET @min_writes = 0; - END; - - IF @min_size_gb < 0 - OR @min_size_gb IS NULL - BEGIN - RAISERROR('Parameter @min_size_gb cannot be NULL or negative. Setting to 0.', 10, 1) WITH NOWAIT; - SET @min_size_gb = 0; - END; - - IF @min_rows < 0 - OR @min_rows IS NULL - BEGIN - RAISERROR('Parameter @min_rows cannot be NULL or negative. Setting to 0.', 10, 1) WITH NOWAIT; - SET @min_rows = 0; - END; + - /* Parameter validation for multi-database mode */ - IF @get_all_databases = 1 - AND @database_name IS NOT NULL - BEGIN - RAISERROR('You cannot specify both @get_all_databases = 1 and a specific @database_name. Using @get_all_databases = 1 and ignoring @database_name.', 10, 1) WITH NOWAIT; - SET @database_name = NULL; - END; /* Start insert queries */ From 13ad6a241b834e453b880dda3e8c8173944de32f Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 17 Mar 2025 22:20:23 -0400 Subject: [PATCH 184/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 97 ++++++++++++++++++++++++++--- 1 file changed, 88 insertions(+), 9 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 7d0a9593..6832f88b 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -672,6 +672,10 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Parse @include_databases if specified - using XML for string splitting instead of STRING_SPLIT (version compatibility) */ IF @include_databases IS NOT NULL + IF @debug = 1 + BEGIN + RAISERROR('processing included databases', 0, 0) WITH NOWAIT; + END; BEGIN SELECT @include_xml = @@ -703,6 +707,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Check for databases in both include and exclude lists */ IF @exclude_databases IS NOT NULL + IF @debug = 1 + BEGIN + RAISERROR('processing excluded databases', 0, 0) WITH NOWAIT; + END; + BEGIN SELECT @exclude_xml = @@ -764,6 +773,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ) ) IN (5, 8) /* Azure SQL DB or Managed Instance */ BEGIN + IF @debug = 1 + BEGIN + RAISERROR('processing databases special in azure', 0, 0) WITH NOWAIT; + END; + /* Get all eligible databases for Azure SQL */ INSERT INTO #databases_to_process @@ -792,6 +806,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END; ELSE /* Regular SQL Server */ BEGIN + IF @debug = 1 + BEGIN + RAISERROR('processing on-prem sql server databases', 0, 0) WITH NOWAIT; + END; + /* Get all eligible databases with AG primary replica check */ INSERT INTO #databases_to_process @@ -835,7 +854,25 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Remove excluded databases if specified */ IF @exclude_databases IS NOT NULL BEGIN - SELECT @exclude_xml = CONVERT(xml, '' + REPLACE(@exclude_databases, ',', '') + ''); + IF @debug = 1 + BEGIN + RAISERROR('processing excluded databases', 0, 0) WITH NOWAIT; + END; + + SELECT + @exclude_xml = + CONVERT + ( + xml, + '' + + REPLACE + ( + @exclude_databases, + ',', + '' + ) + + '' + ); DELETE dp @@ -883,6 +920,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Track databases that were requested but skipped (for better reporting) */ IF @include_databases IS NOT NULL BEGIN + IF @debug = 1 + BEGIN + RAISERROR('processing included databases for skip reasons', 0, 0) WITH NOWAIT; + END; + INSERT #skipped_databases WITH @@ -895,11 +937,16 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. database_name = dl.database_name, reason = CASE - WHEN d.name IS NULL THEN N'Database does not exist' - WHEN d.state <> 0 THEN N'Database not online' - WHEN d.is_in_standby = 1 THEN N'Database is in standby' - WHEN d.is_read_only = 1 THEN N'Database is read-only' - WHEN d.database_id <= 4 THEN N'System database' + WHEN d.name IS NULL + THEN N'Database does not exist' + WHEN d.state <> 0 + THEN N'Database not online' + WHEN d.is_in_standby = 1 + THEN N'Database is in standby' + WHEN d.is_read_only = 1 + THEN N'Database is read-only' + WHEN d.database_id <= 4 + THEN N'System database' WHEN EXISTS ( SELECT @@ -911,7 +958,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. WHERE s.is_local <> 1 AND s.role_desc <> N'PRIMARY' AND DATABASEPROPERTYEX(c.database_name, N'Updateability') <> N'READ_WRITE' - ) THEN N'AG replica issue - not primary or read-write' + ) + THEN N'AG replica issue - not primary or read-write' ELSE N'Other issue' END FROM #database_list AS dl @@ -930,6 +978,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Also track explicitly excluded databases */ IF @exclude_databases IS NOT NULL BEGIN + IF @debug = 1 + BEGIN + RAISERROR('processing explicitly excluded databases', 0, 0) WITH NOWAIT; + END; + INSERT #skipped_databases WITH @@ -943,8 +996,19 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. reason = N'Explicitly excluded by @exclude_databases parameter' FROM ( - SELECT xml_list = CONVERT(xml, N'' + - REPLACE(@exclude_databases, N',', N'') + N'') + SELECT xml_list = + CONVERT + ( + xml, + N'' + + REPLACE + ( + @exclude_databases, + N',', + N'' + ) + + N'' + ) ) AS a CROSS APPLY a.xml_list.nodes('i') AS t(i) WHERE LTRIM(RTRIM(t.i.value('.', 'sysname'))) <> '' @@ -992,6 +1056,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. IF @schema_name IS NOT NULL AND @table_name IS NOT NULL BEGIN + IF @debug = 1 + BEGIN + RAISERROR('validating object existence for %s.%s.&s.', 0, 0, @database_name, @schema_name, @table_name) WITH NOWAIT; + END; + SELECT @full_object_name = QUOTENAME(@database_name) + @@ -1012,6 +1081,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Add the single database to the processing list */ IF @database_name IS NOT NULL BEGIN + IF @debug = 1 + BEGIN + RAISERROR('inserting databases to process', 0, 0) WITH NOWAIT; + END; + INSERT INTO #databases_to_process WITH @@ -1054,6 +1128,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ IF @get_all_databases = 1 BEGIN + IF @debug = 1 + BEGIN + RAISERROR('entering main processing logic for @get_all_databases = 1', 0, 0) WITH NOWAIT; + END; + /* Get the count of databases for reporting */ SELECT @database_count = COUNT_BIG(*) From 079897428adf9c6e62dd3d153e396e2b9152945f Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 17 Mar 2025 22:20:40 -0400 Subject: [PATCH 185/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 106 ++++++++++++++++++++-------- 1 file changed, 75 insertions(+), 31 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 7d0a9593..30445d7b 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -1052,7 +1052,26 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Main processing logic - either loop through all databases or process a single database */ - IF @get_all_databases = 1 + + /* Process single database */ + IF @get_all_databases = 0 + BEGIN + /* Use the database specified in @database_name */ + SELECT + @database_id = database_id, + @database_name = database_name + FROM #databases_to_process; + + IF @debug = 1 + BEGIN + RAISERROR('Single database mode, using specified or current database: %s', 0, 0, @database_name) WITH NOWAIT; + END; + + /* Process the single database */ + GOTO ProcessDatabase; + END + /* Process multiple databases */ + ELSE IF @get_all_databases = 1 BEGIN /* Get the count of databases for reporting */ SELECT @@ -1118,43 +1137,61 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END; /* Process current database using existing logic */ - - IF @debug = 1 - BEGIN - RAISERROR('Processing database %s', 0, 0, @database_name) WITH NOWAIT; - END; - - IF @schema_name IS NOT NULL - AND @table_name IS NOT NULL - BEGIN - SELECT - @full_object_name = - QUOTENAME(@database_name) + - N'.' + - QUOTENAME(@schema_name) + - N'.' + - QUOTENAME(@table_name); - - SET @object_id = OBJECT_ID(@full_object_name); - END; - - IF @object_id IS NULL - BEGIN - RAISERROR('The object %s doesn''t seem to exist', 16, 1, @full_object_name) WITH NOWAIT; - RETURN; - END; + GOTO ProcessDatabase; + + ProcessDatabaseDone: + /* Mark database as processed */ + UPDATE #databases_to_process + SET + processed = 1, + process_date = SYSDATETIME() + WHERE database_id = @database_id; + + /* Get the next database */ + FETCH NEXT + FROM @database_cursor + INTO + @current_database_id, + @current_database_name; END; + + /* We've processed all databases, so we're done */ + GOTO AllDatabasesProcessed; + END; + + /* Process the current database (single or multiple) */ + ProcessDatabase: + + IF @debug = 1 + BEGIN + RAISERROR('Processing database %s', 0, 0, @database_name) WITH NOWAIT; + END; + IF @schema_name IS NOT NULL + AND @table_name IS NOT NULL + BEGIN + SELECT + @full_object_name = + QUOTENAME(@database_name) + + N'.' + + QUOTENAME(@schema_name) + + N'.' + + QUOTENAME(@table_name); + + SET @object_id = OBJECT_ID(@full_object_name); + END; + IF @object_id IS NULL AND @full_object_name IS NOT NULL + BEGIN + RAISERROR('The object %s doesn''t seem to exist', 16, 1, @full_object_name) WITH NOWAIT; + RETURN; + END; - /* - Start insert queries - */ - + /* Generate and execute SQL to process the current database */ IF @debug = 1 BEGIN RAISERROR('Generating #filtered_object insert', 0, 0) WITH NOWAIT; - END; + END; SELECT @sql = N' @@ -5107,6 +5144,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. irs.table_name OPTION(RECOMPILE); + /* If in multi-database mode, go back to process the next database */ + IF @get_all_databases = 1 + BEGIN + GOTO ProcessDatabaseDone; + END; + + AllDatabasesProcessed: /* Final unified results output - runs once after all databases processed */ IF @debug = 1 BEGIN From bc6f100a0bfe51a98d2f12a1b797fa3c5742c4aa Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 17 Mar 2025 22:25:22 -0400 Subject: [PATCH 186/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 41e5684b..58d2c0e8 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -1130,6 +1130,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Process single database */ IF @get_all_databases = 0 BEGIN + IF @debug = 1 + BEGIN + RAISERROR('processing for @get_all_databases = 0', 0, 0) WITH NOWAIT; + END; + /* Use the database specified in @database_name */ SELECT @database_id = database_id, @@ -1249,6 +1254,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. IF @schema_name IS NOT NULL AND @table_name IS NOT NULL BEGIN + IF @debug = 1 + BEGIN + RAISERROR('validating object existence for %s.%s.%s', 0, 0, @database_name, @schema_name, @table_name) WITH NOWAIT; + END; + SELECT @full_object_name = QUOTENAME(@database_name) + @@ -1338,6 +1348,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ) >= 13 ) /* SQL 2016+ */ BEGIN + IF @debug = 1 + BEGIN + RAISERROR('adding temporal table screening', 0, 0) WITH NOWAIT; + END; + SET @sql += N' AND NOT EXISTS ( @@ -1352,6 +1367,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. IF @object_id IS NOT NULL BEGIN + IF @debug = 1 + BEGIN + RAISERROR('adding object_id filter', 0, 0) WITH NOWAIT; + END; + SELECT @sql += N' AND t.object_id = @object_id'; END; From 09a865a8c65bfd49528d0e138d0982a1174800a2 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 17 Mar 2025 22:25:30 -0400 Subject: [PATCH 187/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 109 ++-------------------------- 1 file changed, 5 insertions(+), 104 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 41e5684b..2edbf9fb 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -1141,105 +1141,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. RAISERROR('Single database mode, using specified or current database: %s', 0, 0, @database_name) WITH NOWAIT; END; - /* Process the single database */ - GOTO ProcessDatabase; - END - /* Process multiple databases */ - ELSE IF @get_all_databases = 1 - BEGIN - IF @debug = 1 - BEGIN - RAISERROR('entering main processing logic for @get_all_databases = 1', 0, 0) WITH NOWAIT; - END; - - /* Get the count of databases for reporting */ - SELECT - @database_count = COUNT_BIG(*) - FROM #databases_to_process AS dtp - OPTION(RECOMPILE); - - IF @debug = 1 - BEGIN - RAISERROR('Beginning processing for %d databases', 0, 0, @database_count) WITH NOWAIT; - END; - - /* Set cursor variable as per coding guidelines */ - SET @database_cursor = - CURSOR - LOCAL - STATIC - READ_ONLY - FORWARD_ONLY - FOR - SELECT - dtp.database_id, - dtp.database_name - FROM #databases_to_process AS dtp - WHERE dtp.processed = 0 - ORDER BY - dtp.database_name - OPTION(RECOMPILE); - - OPEN @database_cursor; - FETCH NEXT - FROM @database_cursor - INTO - @current_database_id, - @current_database_name; - - WHILE @@FETCH_STATUS = 0 - BEGIN - SET @processed_count += 1; - - /* Update working variables for each iteration */ - SELECT - @database_id = @current_database_id, - @database_name = @current_database_name; - - IF @debug = 1 - BEGIN - RAISERROR('Processing database %d of %d: %s (ID: %d)', 0, 0, - @processed_count, @database_count, @database_name, @database_id) WITH NOWAIT; - END; - - /* Clear temp tables before processing next database */ - IF @processed_count > 1 - BEGIN - TRUNCATE TABLE #filtered_objects; - TRUNCATE TABLE #operational_stats; - TRUNCATE TABLE #partition_stats; - TRUNCATE TABLE #index_details; - TRUNCATE TABLE #index_analysis; - TRUNCATE TABLE #index_cleanup_results; - TRUNCATE TABLE #compression_eligibility; - TRUNCATE TABLE #index_reporting_stats; - END; - - /* Process current database using existing logic */ - GOTO ProcessDatabase; - - ProcessDatabaseDone: - /* Mark database as processed */ - UPDATE #databases_to_process - SET - processed = 1, - process_date = SYSDATETIME() - WHERE database_id = @database_id; - - /* Get the next database */ - FETCH NEXT - FROM @database_cursor - INTO - @current_database_id, - @current_database_name; - END; - - /* We've processed all databases, so we're done */ - GOTO AllDatabasesProcessed; END; - /* Process the current database (single or multiple) */ - ProcessDatabase: + /* Process the database - since we're in single database mode, just use the existing values */ + IF @debug = 1 + BEGIN + RAISERROR('Single database mode, using specified or current database: %s', 0, 0, @database_name) WITH NOWAIT; + END; IF @debug = 1 BEGIN @@ -5223,13 +5131,6 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. irs.table_name OPTION(RECOMPILE); - /* If in multi-database mode, go back to process the next database */ - IF @get_all_databases = 1 - BEGIN - GOTO ProcessDatabaseDone; - END; - - AllDatabasesProcessed: /* Final unified results output - runs once after all databases processed */ IF @debug = 1 BEGIN From bdfba3f9c15c481aae7c5894de55858d7d719566 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 17 Mar 2025 22:38:50 -0400 Subject: [PATCH 188/246] Create sp_IndexCleanup_Old.sql --- sp_IndexCleanup/sp_IndexCleanup_Old.sql | 4483 +++++++++++++++++++++++ 1 file changed, 4483 insertions(+) create mode 100644 sp_IndexCleanup/sp_IndexCleanup_Old.sql diff --git a/sp_IndexCleanup/sp_IndexCleanup_Old.sql b/sp_IndexCleanup/sp_IndexCleanup_Old.sql new file mode 100644 index 00000000..aac0143f --- /dev/null +++ b/sp_IndexCleanup/sp_IndexCleanup_Old.sql @@ -0,0 +1,4483 @@ +/* +EXECUTE sp_IndexCleanup + @database_name = 'StackOverflow2013', + @debug = 1; + +EXECUTE sp_IndexCleanup + @database_name = 'StackOverflow2013', + @table_name = 'Users', + @debug = 1 +*/ + +SET ANSI_WARNINGS ON; +SET ARITHABORT ON; +SET CONCAT_NULL_YIELDS_NULL ON; +SET QUOTED_IDENTIFIER ON; +SET NUMERIC_ROUNDABORT OFF; +SET IMPLICIT_TRANSACTIONS OFF; +SET STATISTICS TIME, IO OFF; +GO + +IF OBJECT_ID('dbo.sp_IndexCleanup', 'P') IS NULL +BEGIN + EXECUTE ('CREATE PROCEDURE dbo.sp_IndexCleanup AS RETURN 138;'); +END; +GO + +ALTER PROCEDURE + dbo.sp_IndexCleanup +( + @database_name sysname = NULL, + @schema_name sysname = NULL, + @table_name sysname = NULL, + @min_reads bigint = 0, + @min_writes bigint = 0, + @min_size_gb decimal(10,2) = 0, + @min_rows bigint = 0, + @help bit = 'false', + @debug bit = 'false', + @version varchar(20) = NULL OUTPUT, + @version_date datetime = NULL OUTPUT +) +WITH RECOMPILE +AS +BEGIN +SET NOCOUNT ON; + +BEGIN TRY + /* Check for SQL Server 2012 (11.0) or later for FORMAT and CONCAT functions*/ + + IF + /* Check SQL Server 2012+ for FORMAT and CONCAT functions */ + ( + CONVERT + ( + integer, + SERVERPROPERTY('EngineEdition') + ) NOT IN (5, 8) /* Not Azure SQL DB or Managed Instance */ + AND CONVERT + ( + integer, + SUBSTRING + ( + CONVERT + ( + varchar(20), + SERVERPROPERTY('ProductVersion') + ), + 1, + 2 + ) + ) < 11) /* Pre-2012 */ + BEGIN + RAISERROR('This procedure requires SQL Server 2012 (11.0) or later due to the use of FORMAT and CONCAT functions.', 11, 1); + RETURN; + END; + + SELECT + @version = '1.4', + @version_date = '20250401'; + + SELECT + for_insurance_purposes = N'Read the messages pane carefully!'; + + PRINT N' +------------------------------------------------------------------------------------------- +------------------------------------------------------------------------------------------- +------------------------------------------------------------------------------------------- +This is the BETA VERSION of sp_IndexCleanup + +It needs lots of love and testing in real environments with real indexes to fix many issues: + * Data collection + * Deduping logic + * Result correctness + * Edge cases + * May not account for specific query patterns that benefit from seemingly redundant indexes + +ALWAYS TEST THESE RECOMMENDATIONS IN A NON-PRODUCTION ENVIRONMENT FIRST" + +------------------------------------------------------------------------------------------- +------------------------------------------------------------------------------------------- +------------------------------------------------------------------------------------------- +'; + + + /* + Help section, for help. + Will become more helpful when out of beta. + */ + IF @help = 1 + BEGIN + SELECT + help = N'hello, i am sp_IndexCleanup - BETA' + UNION ALL + SELECT + help = N'this is a script to help clean up unused and duplicate indexes' + UNION ALL + SELECT + help = N'you are currently using a beta version, and the advice should not be followed' + UNION ALL + SELECT + help = N'without careful analysis and consideration. it may be harmful.'; + + /* + Parameters + */ + SELECT + parameter_name = + ap.name, + data_type = + t.name, + description = + CASE + ap.name + WHEN N'@database_name' THEN 'the name of the database you wish to analyze' + WHEN N'@schema_name' THEN 'the schema name to filter indexes by' + WHEN N'@table_name' THEN 'the table name to filter indexes by' + WHEN N'@min_reads' THEN 'minimum number of reads for an index to be considered used' + WHEN N'@min_writes' THEN 'minimum number of writes for an index to be considered used' + WHEN N'@min_size_gb' THEN 'minimum size in GB for an index to be analyzed' + WHEN N'@min_rows' THEN 'minimum number of rows for a table to be analyzed' + WHEN N'@help' THEN 'displays this help information' + WHEN N'@debug' THEN 'prints debug information during execution' + WHEN N'@version' THEN 'returns the version number of the procedure' + WHEN N'@version_date' THEN 'returns the date this version was released' + ELSE NULL + END, + valid_inputs = + CASE + ap.name + WHEN N'@database_name' THEN 'the name of a database you care about indexes in' + WHEN N'@schema_name' THEN 'schema name or NULL for all schemas' + WHEN N'@table_name' THEN 'table name or NULL for all tables' + WHEN N'@min_reads' THEN 'any positive integer or 0' + WHEN N'@min_writes' THEN 'any positive integer or 0' + WHEN N'@min_size_gb' THEN 'any positive decimal number or 0' + WHEN N'@min_rows' THEN 'any positive integer or 0' + WHEN N'@help' THEN '0 or 1' + WHEN N'@debug' THEN '0 or 1' + WHEN N'@version' THEN 'OUTPUT parameter' + WHEN N'@version_date' THEN 'OUTPUT parameter' + ELSE NULL + END, + defaults = + CASE + ap.name + WHEN N'@database_name' THEN 'NULL' + WHEN N'@schema_name' THEN 'NULL' + WHEN N'@table_name' THEN 'NULL' + WHEN N'@min_reads' THEN '0' + WHEN N'@min_writes' THEN '0' + WHEN N'@min_size_gb' THEN '0' + WHEN N'@min_rows' THEN '0' + WHEN N'@help' THEN 'false' + WHEN N'@debug' THEN 'true' + WHEN N'@version' THEN 'NULL' + WHEN N'@version_date' THEN 'NULL' + ELSE NULL + END + FROM sys.all_parameters AS ap + JOIN sys.all_objects AS o + ON ap.object_id = o.object_id + JOIN sys.types AS t + ON ap.system_type_id = t.system_type_id + AND ap.user_type_id = t.user_type_id + WHERE o.name = N'sp_IndexCleanup' + OPTION(MAXDOP 1, RECOMPILE); + + SELECT + mit_license_yo = 'i am MIT licensed, so like, do whatever' + + UNION ALL + + SELECT + mit_license_yo = 'see printed messages for full license'; + + RAISERROR(' +MIT License + +Copyright 2024 Darling Data, LLC + +https://www.erikdarling.com/ + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, +sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the +following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE +FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +', 0, 1) WITH NOWAIT; + + RETURN; + END; + + IF @debug = 1 + BEGIN + RAISERROR('Declaring variables', 0, 0) WITH NOWAIT; + END; + + DECLARE + /*general script variables*/ + @sql nvarchar(max) = N'', + @database_id integer = NULL, + @object_id integer = NULL, + @full_object_name nvarchar(768) = NULL, + @uptime_warning bit = 0, /* Will set after @uptime_days is calculated */ + /*print variables*/ + @online bit = + CASE + WHEN + CONVERT + ( + integer, + SERVERPROPERTY('EngineEdition') + ) IN (3, 5, 8) + THEN 'true' /* Enterprise, Azure SQL DB, Managed Instance */ + ELSE 'false' + END, + /* Compression variables */ + @can_compress bit = + CASE + WHEN + CONVERT(integer, SERVERPROPERTY('EngineEdition')) IN (3, 5, 8) + OR + ( + CONVERT(integer, SERVERPROPERTY('EngineEdition')) = 2 + AND CONVERT(integer, SUBSTRING(CONVERT(varchar(20), SERVERPROPERTY('ProductVersion')), 1, 2)) >= 13 + ) + THEN 1 + ELSE 0 + END, + @uptime_days nvarchar(10) = + ( + SELECT + DATEDIFF + ( + DAY, + osi.sqlserver_start_time, + SYSDATETIME() + ) + FROM sys.dm_os_sys_info AS osi + ); + + /* Set uptime warning flag after @uptime_days is calculated */ + SELECT + @uptime_warning = + CASE + WHEN CONVERT(integer, @uptime_days) < 14 + THEN 1 + ELSE 0 + END; + + /* + Initial checks for object validity + */ + IF @debug = 1 + BEGIN + RAISERROR('Checking paramaters...', 0, 0) WITH NOWAIT; + END; + + IF @database_name IS NULL + AND DB_NAME() NOT IN + ( + N'master', + N'model', + N'msdb', + N'tempdb', + N'rdsadmin' + ) + BEGIN + SELECT + @database_name = DB_NAME(); + END; + + IF @database_name IS NOT NULL + BEGIN + SELECT + @database_id = d.database_id + FROM sys.databases AS d + WHERE d.name = @database_name + OPTION(RECOMPILE); + END; + + IF @schema_name IS NULL + AND @table_name IS NOT NULL + BEGIN + SELECT + @schema_name = N'dbo'; + END; + + IF @schema_name IS NOT NULL + AND @table_name IS NOT NULL + BEGIN + SELECT + @full_object_name = + QUOTENAME(@database_name) + + N'.' + + QUOTENAME(@schema_name) + + N'.' + + QUOTENAME(@table_name); + + SELECT + @object_id = + OBJECT_ID(@full_object_name); + + IF @object_id IS NULL + BEGIN + RAISERROR('The object %s doesn''t seem to exist', 16, 1, @full_object_name) WITH NOWAIT; + RETURN; + END; + END; + + /* Parameter validation */ + IF @min_reads < 0 + OR @min_reads IS NULL + BEGIN + RAISERROR('Parameter @min_reads cannot be NULL or negative. Setting to 0.', 10, 1) WITH NOWAIT; + SET @min_reads = 0; + END; + + IF @min_writes < 0 + OR @min_writes IS NULL + BEGIN + RAISERROR('Parameter @min_writes cannot be NULL or negative. Setting to 0.', 10, 1) WITH NOWAIT; + SET @min_writes = 0; + END; + + IF @min_size_gb < 0 + OR @min_size_gb IS NULL + BEGIN + RAISERROR('Parameter @min_size_gb cannot be NULL or negative. Setting to 0.', 10, 1) WITH NOWAIT; + SET @min_size_gb = 0; + END; + + IF @min_rows < 0 + OR @min_rows IS NULL + BEGIN + RAISERROR('Parameter @min_rows cannot be NULL or negative. Setting to 0.', 10, 1) WITH NOWAIT; + SET @min_rows = 0; + END; + + /* + Temp tables! + */ + + IF @debug = 1 + BEGIN + RAISERROR('Creating temp tables', 0, 0) WITH NOWAIT; + END; + + CREATE TABLE + #filtered_objects + ( + database_id integer NOT NULL, + database_name sysname NOT NULL, + schema_id integer NOT NULL, + schema_name sysname NOT NULL, + object_id integer NOT NULL, + table_name sysname NOT NULL, + index_id integer NOT NULL, + index_name sysname NOT NULL, + can_compress bit NOT NULL + PRIMARY KEY CLUSTERED(database_id, schema_id, object_id, index_id) + ); + + CREATE TABLE + #operational_stats + ( + database_id integer NOT NULL, + database_name sysname NOT NULL, + schema_id integer NOT NULL, + schema_name sysname NOT NULL, + object_id integer NOT NULL, + table_name sysname NOT NULL, + index_id integer NOT NULL, + index_name sysname NOT NULL, + range_scan_count bigint NULL, + singleton_lookup_count bigint NULL, + forwarded_fetch_count bigint NULL, + lob_fetch_in_pages bigint NULL, + row_overflow_fetch_in_pages bigint NULL, + leaf_insert_count bigint NULL, + leaf_update_count bigint NULL, + leaf_delete_count bigint NULL, + leaf_ghost_count bigint NULL, + nonleaf_insert_count bigint NULL, + nonleaf_update_count bigint NULL, + nonleaf_delete_count bigint NULL, + leaf_allocation_count bigint NULL, + nonleaf_allocation_count bigint NULL, + row_lock_count bigint NULL, + row_lock_wait_count bigint NULL, + row_lock_wait_in_ms bigint NULL, + page_lock_count bigint NULL, + page_lock_wait_count bigint NULL, + page_lock_wait_in_ms bigint NULL, + index_lock_promotion_attempt_count bigint NULL, + index_lock_promotion_count bigint NULL, + page_latch_wait_count bigint NULL, + page_latch_wait_in_ms bigint NULL, + tree_page_latch_wait_count bigint NULL, + tree_page_latch_wait_in_ms bigint NULL, + page_io_latch_wait_count bigint NULL, + page_io_latch_wait_in_ms bigint NULL, + page_compression_attempt_count bigint NULL, + page_compression_success_count bigint NULL, + PRIMARY KEY CLUSTERED (database_id, schema_id, object_id, index_id) + ); + + CREATE TABLE + #partition_stats + ( + database_id integer NOT NULL, + database_name sysname NOT NULL, + schema_id integer NOT NULL, + schema_name sysname NOT NULL, + object_id integer NOT NULL, + table_name sysname NOT NULL, + index_id integer NOT NULL, + index_name sysname NULL, + partition_id bigint NOT NULL, + partition_number int NOT NULL, + total_rows bigint NULL, + total_space_gb decimal(38, 4) NULL, /* Using 4 decimal places for GB to maintain precision */ + reserved_lob_gb decimal(38, 4) NULL, /* Using 4 decimal places for GB to maintain precision */ + reserved_row_overflow_gb decimal(38, 4) NULL, /* Using 4 decimal places for GB to maintain precision */ + data_compression_desc nvarchar(60) NULL, + built_on sysname NULL, + partition_function_name sysname NULL, + partition_columns nvarchar(max) + PRIMARY KEY CLUSTERED(database_id, schema_id, object_id, index_id, partition_id) + ); + + CREATE TABLE + #index_details + ( + database_id integer NOT NULL, + database_name sysname NOT NULL, + schema_id integer NOT NULL, + schema_name sysname NOT NULL, + object_id integer NOT NULL, + table_name sysname NOT NULL, + index_id integer NOT NULL, + index_name sysname NULL, + column_name sysname NOT NULL, + is_primary_key bit NULL, + is_unique bit NULL, + is_unique_constraint bit NULL, + is_indexed_view integer NOT NULL, + is_foreign_key bit NULL, + is_foreign_key_reference bit NULL, + key_ordinal tinyint NOT NULL, + index_column_id integer NOT NULL, + is_descending_key bit NOT NULL, + is_included_column bit NULL, + filter_definition nvarchar(max) NULL, + is_max_length integer NOT NULL, + user_seeks bigint NOT NULL, + user_scans bigint NOT NULL, + user_lookups bigint NOT NULL, + user_updates bigint NOT NULL, + last_user_seek datetime NULL, + last_user_scan datetime NULL, + last_user_lookup datetime NULL, + last_user_update datetime NULL, + is_eligible_for_dedupe bit NOT NULL + PRIMARY KEY CLUSTERED(database_id, object_id, index_id, column_name) + ); + + CREATE TABLE + #index_analysis + ( + database_id integer NOT NULL, + database_name sysname NOT NULL, + schema_id integer NOT NULL, + schema_name sysname NOT NULL, + object_id integer NOT NULL, + table_name sysname NOT NULL, + index_id integer NULL, + index_name sysname NOT NULL, + is_unique bit NULL, + key_columns nvarchar(max) NULL, + included_columns nvarchar(max) NULL, + filter_definition nvarchar(max) NULL, + is_redundant bit NULL, + superseded_by nvarchar(256) NULL, + missing_columns nvarchar(max) NULL, + action nvarchar(30) NULL, + target_index_name sysname NULL, + consolidation_rule varchar(512) NULL, + index_priority int NULL, + original_index_definition nvarchar(max) NULL, /* Original CREATE INDEX statement */ + INDEX c CLUSTERED (database_id, schema_id, object_id, index_id) + ); + + CREATE TABLE + #compression_eligibility + ( + database_id integer NOT NULL, + database_name sysname NOT NULL, + schema_id integer NOT NULL, + schema_name sysname NOT NULL, + object_id integer NOT NULL, + table_name sysname NOT NULL, + index_id integer NOT NULL, + index_name sysname NOT NULL, + can_compress bit NOT NULL, + reason nvarchar(200) NULL, + PRIMARY KEY CLUSTERED(database_id, object_id, index_id) + ); + + CREATE TABLE + #key_duplicate_dedupe + ( + database_id integer NOT NULL, + object_id integer NOT NULL, + database_name sysname NOT NULL, + schema_name sysname NOT NULL, + table_name sysname NOT NULL, + base_key_columns nvarchar(max) NULL, + filter_definition nvarchar(max) NULL, + winning_index_name sysname NULL, + index_list nvarchar(max) NULL, + ); + + CREATE TABLE + #include_subset_dedupe + ( + database_id integer NOT NULL, + object_id integer NOT NULL, + subset_index_name sysname NULL, + superset_index_name sysname NULL, + subset_included_columns nvarchar(max) NULL, + superset_included_columns nvarchar(max) NULL + ); + + CREATE TABLE + #index_cleanup_results + ( + result_type varchar(50) NOT NULL, /* 'SUMMARY', 'MERGE', 'DISABLE', 'COMPRESS', etc. */ + sort_order integer NOT NULL, /* Keeps results in logical order */ + database_name sysname NULL, + schema_name sysname NULL, + table_name sysname NULL, + index_name sysname NULL, + script_type nvarchar(50) NULL, /* 'MERGE', 'DISABLE', 'COMPRESS', etc. */ + consolidation_rule nvarchar(256) NULL, + target_index_name sysname NULL, + script nvarchar(max) NULL, + additional_info nvarchar(max) NULL, /* For stats, constraints, etc. */ + superseded_info nvarchar(max) NULL, /* To store superseded_by information */ + original_index_definition nvarchar(max) NULL, /* Original index definition for validation */ + index_size_gb decimal(18,4) NULL, /* Size of the index in GB */ + index_rows bigint NULL, /* Number of rows in the index */ + index_reads bigint NULL, /* Total reads (seeks + scans + lookups) */ + index_writes bigint NULL /* Total writes (updates) */ + ); + + /* Create a new temp table for detailed reporting statistics */ + CREATE TABLE + #index_reporting_stats + ( + summary_level varchar(20) NOT NULL, /* 'DATABASE', 'TABLE', 'INDEX', 'SUMMARY' */ + database_name sysname NULL, + schema_name sysname NULL, + table_name sysname NULL, + index_name sysname NULL, + server_uptime_days int NULL, + uptime_warning bit NULL, + tables_analyzed int NULL, + index_count int NULL, + total_size_gb decimal(38, 4) NULL, + total_rows bigint NULL, + unused_indexes int NULL, + unused_size_gb decimal(38, 4) NULL, + indexes_to_disable int NULL, + indexes_to_merge int NULL, + avg_indexes_per_table decimal(10, 2) NULL, + space_saved_gb decimal(10, 4) NULL, + compression_min_savings_gb decimal(10, 4) NULL, + compression_max_savings_gb decimal(10, 4) NULL, + total_min_savings_gb decimal(10, 4) NULL, + total_max_savings_gb decimal(10, 4) NULL, + /* Index usage metrics */ + total_reads bigint NULL, + total_writes bigint NULL, + user_seeks bigint NULL, + user_scans bigint NULL, + user_lookups bigint NULL, + user_updates bigint NULL, + /* Operational stats */ + range_scan_count bigint NULL, + singleton_lookup_count bigint NULL, + /* Lock stats */ + row_lock_count bigint NULL, + row_lock_wait_count bigint NULL, + row_lock_wait_in_ms bigint NULL, + page_lock_count bigint NULL, + page_lock_wait_count bigint NULL, + page_lock_wait_in_ms bigint NULL, + /* Latch stats */ + page_latch_wait_count bigint NULL, + page_latch_wait_in_ms bigint NULL, + page_io_latch_wait_count bigint NULL, + page_io_latch_wait_in_ms bigint NULL, + /* Misc stats */ + forwarded_fetch_count bigint NULL, + leaf_insert_count bigint NULL, + leaf_update_count bigint NULL, + leaf_delete_count bigint NULL + ); + + /* + Start insert queries + */ + + IF @debug = 1 + BEGIN + RAISERROR('Generating #filtered_object insert', 0, 0) WITH NOWAIT; + END; + + SELECT + @sql = N' + SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;'; + + SELECT + @sql = N' + SELECT DISTINCT + @database_id, + database_name = DB_NAME(@database_id), + schema_id = t.schema_id, + schema_name = s.name, + object_id = t.object_id, + table_name = t.name, + index_id = i.index_id, + index_name = ISNULL(i.name, t.name + N''.Heap''), + can_compress = + CASE + WHEN p.index_id > 0 + AND p.data_compression = 0 + THEN 1 + ELSE 0 + END + FROM ' + QUOTENAME(@database_name) + N'.sys.tables AS t + JOIN ' + QUOTENAME(@database_name) + N'.sys.schemas AS s + ON t.schema_id = s.schema_id + JOIN ' + QUOTENAME(@database_name) + N'.sys.indexes AS i + ON t.object_id = i.object_id + JOIN ' + QUOTENAME(@database_name) + N'.sys.partitions AS p + ON i.object_id = p.object_id + AND i.index_id = p.index_id + LEFT JOIN ' + QUOTENAME(@database_name) + N'.sys.dm_db_index_usage_stats AS us + ON t.object_id = us.object_id + AND us.database_id = @database_id + WHERE t.is_ms_shipped = 0 + AND t.type <> N''TF'' + AND NOT EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@database_name) + N'.sys.views AS v + WHERE v.object_id = i.object_id + )'; + + IF /* Check SQL Server 2016+ for temporal tables support */ + ( + CONVERT + ( + integer, + SERVERPROPERTY('EngineEdition') + ) IN (5, 8) /* Azure SQL DB or Managed Instance */ + OR CONVERT + ( + integer, + SUBSTRING + ( + CONVERT + ( + varchar(20), + SERVERPROPERTY('ProductVersion') + ), + 1, + 2 + ) + ) >= 13 + ) /* SQL 2016+ */ + BEGIN + SET @sql += N' + AND NOT EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@database_name) + N'.sys.tables AS t + WHERE t.object_id = i.object_id + AND t.temporal_type > 0 + )'; + END; + + + IF @object_id IS NOT NULL + BEGIN + SELECT @sql += N' + AND t.object_id = @object_id'; + END; + + SET @sql += N' + AND EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@database_name) + N'.sys.dm_db_partition_stats AS ps + JOIN ' + QUOTENAME(@database_name) + N'.sys.allocation_units AS au + ON ps.partition_id = au.container_id + WHERE ps.object_id = t.object_id + GROUP + BY ps.object_id + HAVING + SUM(au.total_pages) * 8.0 / 1048576.0 >= @min_size_gb + ) + AND EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@database_name) + N'.sys.dm_db_partition_stats AS ps + WHERE ps.object_id = t.object_id + AND ps.index_id IN (0, 1) + GROUP + BY ps.object_id + HAVING + SUM(ps.row_count) >= @min_rows + ) + AND EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@database_name) + N'.sys.dm_db_index_usage_stats AS ius + WHERE ius.object_id = t.object_id + AND ius.database_id = @database_id + GROUP BY + ius.object_id + HAVING + SUM(ius.user_seeks + ius.user_scans + ius.user_lookups) >= @min_reads + OR + SUM(ius.user_updates) >= @min_writes + ) + OPTION(RECOMPILE);'; + + IF @debug = 1 + BEGIN + PRINT @sql; + END; + + INSERT + #filtered_objects + WITH + (TABLOCK) + ( + database_id, + database_name, + schema_id, + schema_name, + object_id, + table_name, + index_id, + index_name, + can_compress + ) + EXECUTE sys.sp_executesql + @sql, + N'@database_id int, + @min_reads bigint, + @min_writes bigint, + @min_size_gb decimal(10,2), + @min_rows bigint, + @object_id integer', + @database_id, + @min_reads, + @min_writes, + @min_size_gb, + @min_rows, + @object_id; + + IF ROWCOUNT_BIG() = 0 + BEGIN + IF @debug = 1 + BEGIN + RAISERROR('No rows inserted into #filtered_objects', 0, 0) WITH NOWAIT; + END; + END; + + IF @debug = 1 + BEGIN + SELECT + table_name = '#filtered_objects', + fo.* + FROM #filtered_objects AS fo + OPTION(RECOMPILE); + + RAISERROR('Generating #compression_eligibility insert', 0, 0) WITH NOWAIT; + END; + + /* Populate compression eligibility table */ + INSERT INTO + #compression_eligibility + WITH + (TABLOCK) + ( + database_id, + database_name, + schema_id, + schema_name, + object_id, + table_name, + index_id, + index_name, + can_compress, + reason + ) + SELECT + fo.database_id, + fo.database_name, + fo.schema_id, + fo.schema_name, + fo.object_id, + fo.table_name, + fo.index_id, + fo.index_name, + 1, /* Default to compressible */ + NULL + FROM #filtered_objects AS fo + WHERE fo.can_compress = 1 + OPTION(RECOMPILE); + + /* If SQL Server edition doesn't support compression, mark all as ineligible */ + IF @can_compress = 0 + BEGIN + UPDATE + #compression_eligibility + SET + can_compress = 0, + reason = N'SQL Server edition or version does not support compression' + WHERE can_compress = 1 + OPTION(RECOMPILE); + END; + + /* Check for sparse columns or incompatible data types */ + IF @can_compress = 1 + BEGIN + IF @debug = 1 + BEGIN + RAISERROR('Updating #compression_eligibility', 0, 0) WITH NOWAIT; + END; + + SELECT + @sql = N' + SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; + + UPDATE + ce + SET + ce.can_compress = 0, + ce.reason = ''Table contains sparse columns or incompatible data types'' + FROM #compression_eligibility AS ce + WHERE EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@database_name) + N'.sys.columns AS c + JOIN ' + QUOTENAME(@database_name) + N'.sys.types AS t + ON c.user_type_id = t.user_type_id + WHERE c.object_id = ce.object_id + AND + ( + c.is_sparse = 1 + OR t.name IN (N''text'', N''ntext'', N''image'') + ) + ) + OPTION(RECOMPILE); + '; + + IF @debug = 1 + BEGIN + PRINT @sql; + END; + + EXECUTE sys.sp_executesql + @sql; + END; + + IF @debug = 1 + BEGIN + SELECT + table_name = '#compression_eligibility', + ce.* + FROM #compression_eligibility AS ce + OPTION(RECOMPILE); + + RAISERROR('Generating #operational_stats insert', 0, 0) WITH NOWAIT; + END; + + SELECT + @sql = N' + SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;'; + + SELECT + @sql += N' + SELECT + os.database_id, + database_name = DB_NAME(os.database_id), + schema_id = s.schema_id, + schema_name = s.name, + os.object_id, + table_name = t.name, + os.index_id, + index_name = ISNULL(i.name, t.name + N''.Heap''), + range_scan_count = SUM(os.range_scan_count), + singleton_lookup_count = SUM(os.singleton_lookup_count), + forwarded_fetch_count = SUM(os.forwarded_fetch_count), + lob_fetch_in_pages = SUM(os.lob_fetch_in_pages), + row_overflow_fetch_in_pages = SUM(os.row_overflow_fetch_in_pages), + leaf_insert_count = SUM(os.leaf_insert_count), + leaf_update_count = SUM(os.leaf_update_count), + leaf_delete_count = SUM(os.leaf_delete_count), + leaf_ghost_count = SUM(os.leaf_ghost_count), + nonleaf_insert_count = SUM(os.nonleaf_insert_count), + nonleaf_update_count = SUM(os.nonleaf_update_count), + nonleaf_delete_count = SUM(os.nonleaf_delete_count), + leaf_allocation_count = SUM(os.leaf_allocation_count), + nonleaf_allocation_count = SUM(os.nonleaf_allocation_count), + row_lock_count = SUM(os.row_lock_count), + row_lock_wait_count = SUM(os.row_lock_wait_count), + row_lock_wait_in_ms = SUM(os.row_lock_wait_in_ms), + page_lock_count = SUM(os.page_lock_count), + page_lock_wait_count = SUM(os.page_lock_wait_count), + page_lock_wait_in_ms = SUM(os.page_lock_wait_in_ms), + index_lock_promotion_attempt_count = SUM(os.index_lock_promotion_attempt_count), + index_lock_promotion_count = SUM(os.index_lock_promotion_count), + page_latch_wait_count = SUM(os.page_latch_wait_count), + page_latch_wait_in_ms = SUM(os.page_latch_wait_in_ms), + tree_page_latch_wait_count = SUM(os.tree_page_latch_wait_count), + tree_page_latch_wait_in_ms = SUM(os.tree_page_latch_wait_in_ms), + page_io_latch_wait_count = SUM(os.page_io_latch_wait_count), + page_io_latch_wait_in_ms = SUM(os.page_io_latch_wait_in_ms), + page_compression_attempt_count = SUM(os.page_compression_attempt_count), + page_compression_success_count = SUM(os.page_compression_success_count) + FROM ' + QUOTENAME(@database_name) + N'.sys.dm_db_index_operational_stats + ( + @database_id, + @object_id, + NULL, + NULL + ) AS os + JOIN ' + QUOTENAME(@database_name) + N'.sys.tables AS t + ON os.object_id = t.object_id + JOIN ' + QUOTENAME(@database_name) + N'.sys.schemas AS s + ON t.schema_id = s.schema_id + JOIN ' + QUOTENAME(@database_name) + N'.sys.indexes AS i + ON os.object_id = i.object_id + AND os.index_id = i.index_id + WHERE EXISTS + ( + SELECT + 1/0 + FROM #filtered_objects AS fo + WHERE fo.database_id = os.database_id + AND fo.object_id = os.object_id + ) + GROUP BY + os.database_id, + DB_NAME(os.database_id), + s.schema_id, + s.name, + os.object_id, + t.name, + os.index_id, + i.name + OPTION(RECOMPILE);'; + + IF @debug = 1 + BEGIN + PRINT @sql; + END; + + INSERT + #operational_stats + WITH + (TABLOCK) + ( + database_id, + database_name, + schema_id, + schema_name, + object_id, + table_name, + index_id, + index_name, + range_scan_count, + singleton_lookup_count, + forwarded_fetch_count, + lob_fetch_in_pages, + row_overflow_fetch_in_pages, + leaf_insert_count, + leaf_update_count, + leaf_delete_count, + leaf_ghost_count, + nonleaf_insert_count, + nonleaf_update_count, + nonleaf_delete_count, + leaf_allocation_count, + nonleaf_allocation_count, + row_lock_count, + row_lock_wait_count, + row_lock_wait_in_ms, + page_lock_count, + page_lock_wait_count, + page_lock_wait_in_ms, + index_lock_promotion_attempt_count, + index_lock_promotion_count, + page_latch_wait_count, + page_latch_wait_in_ms, + tree_page_latch_wait_count, + tree_page_latch_wait_in_ms, + page_io_latch_wait_count, + page_io_latch_wait_in_ms, + page_compression_attempt_count, + page_compression_success_count + ) + EXECUTE sys.sp_executesql + @sql, + N'@database_id integer, + @object_id integer', + @database_id, + @object_id; + + IF ROWCOUNT_BIG() = 0 + BEGIN + IF @debug = 1 + BEGIN + RAISERROR('No rows inserted into #operational_stats', 0, 0) WITH NOWAIT; + END; + END; + + IF @debug = 1 + BEGIN + SELECT + table_name = '#operational_stats', + os.* + FROM #operational_stats AS os + OPTION(RECOMPILE); + + RAISERROR('Generating #index_details insert', 0, 0) WITH NOWAIT; + END; + + SELECT + @sql = N' + SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;'; + + SELECT + @sql += N' + SELECT + database_id = @database_id, + database_name = DB_NAME(@database_id), + t.object_id, + i.index_id, + s.schema_id, + schema_name = s.name, + table_name = t.name, + index_name = ISNULL(i.name, t.name + N''.Heap''), + column_name = c.name, + i.is_primary_key, + i.is_unique, + i.is_unique_constraint, + is_indexed_view = + CASE + WHEN EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@database_name) + N'.sys.objects AS so + WHERE i.object_id = so.object_id + AND so.is_ms_shipped = 0 + AND so.type = ''V'' + ) + THEN 1 + ELSE 0 + END, + is_foreign_key = + CASE + WHEN EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@database_name) + N'.sys.foreign_key_columns AS f + WHERE f.parent_column_id = c.column_id + AND f.parent_object_id = c.object_id + ) + THEN 1 + ELSE 0 + END, + is_foreign_key_reference = + CASE + WHEN EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@database_name) + N'.sys.foreign_key_columns AS f + WHERE f.referenced_column_id = c.column_id + AND f.referenced_object_id = c.object_id + ) + THEN 1 + ELSE 0 + END, + ic.key_ordinal, + ic.index_column_id, + ic.is_descending_key, + ic.is_included_column, + i.filter_definition, + is_max_length = + CASE + WHEN EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@database_name) + N'.sys.types AS t + WHERE c.system_type_id = t.system_type_id + AND c.user_type_id = t.user_type_id + AND t.name IN (N''varchar'', N''nvarchar'') + AND t.max_length = -1 + ) + THEN 1 + ELSE 0 + END, + user_seeks = ISNULL(us.user_seeks, 0), + user_scans = ISNULL(us.user_scans, 0), + user_lookups = ISNULL(us.user_lookups, 0), + user_updates = ISNULL(us.user_updates, 0), + us.last_user_seek, + us.last_user_scan, + us.last_user_lookup, + us.last_user_update, + is_eligible_for_dedupe = + CASE + WHEN i.type = 2 + THEN 1 + WHEN i.type = 1 + THEN 0 + END + FROM ' + QUOTENAME(@database_name) + N'.sys.tables AS t + JOIN ' + QUOTENAME(@database_name) + N'.sys.schemas AS s + ON t.schema_id = s.schema_id + JOIN ' + QUOTENAME(@database_name) + N'.sys.indexes AS i + ON t.object_id = i.object_id + JOIN ' + QUOTENAME(@database_name) + N'.sys.index_columns AS ic + ON i.object_id = ic.object_id + AND i.index_id = ic.index_id + JOIN ' + QUOTENAME(@database_name) + + CONVERT + ( + nvarchar(MAX), + N'.sys.columns AS c + ON ic.object_id = c.object_id + AND ic.column_id = c.column_id + LEFT JOIN sys.dm_db_index_usage_stats AS us + ON i.object_id = us.object_id + AND i.index_id = us.index_id + AND us.database_id = @database_id + WHERE t.is_ms_shipped = 0 + AND i.type IN (1, 2) + AND i.is_disabled = 0 + AND i.is_hypothetical = 0 + AND EXISTS + ( + SELECT + 1/0 + FROM #filtered_objects AS fo + WHERE fo.database_id = @database_id + AND fo.object_id = t.object_id + ) + AND EXISTS + ( + SELECT + 1/0 + FROM ' + ) + QUOTENAME(@database_name) + + CONVERT + ( + nvarchar(MAX), + N'.sys.dm_db_partition_stats ps + WHERE ps.object_id = t.object_id + AND ps.index_id = 1 + AND ps.row_count >= @min_rows + )' + ); + + IF @object_id IS NOT NULL + BEGIN + SELECT @sql += N' + AND t.object_id = @object_id'; + END; + + SELECT + @sql += CONVERT + ( + nvarchar(max), + N' + AND NOT EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@database_name) + N'.sys.objects AS so + WHERE i.object_id = so.object_id + AND so.is_ms_shipped = 0 + AND so.type = N''TF'' + ) + OPTION(RECOMPILE);' + ); + + IF @debug = 1 + BEGIN + PRINT SUBSTRING(@sql, 1, 4000); + PRINT SUBSTRING(@sql, 4000, 8000); + END; + + INSERT + #index_details + WITH + (TABLOCK) + ( + database_id, + database_name, + object_id, + index_id, + schema_id, + schema_name, + table_name, + index_name, + column_name, + is_primary_key, + is_unique, + is_unique_constraint, + is_indexed_view, + is_foreign_key, + is_foreign_key_reference, + key_ordinal, + index_column_id, + is_descending_key, + is_included_column, + filter_definition, + is_max_length, + user_seeks, + user_scans, + user_lookups, + user_updates, + last_user_seek, + last_user_scan, + last_user_lookup, + last_user_update, + is_eligible_for_dedupe + ) + EXECUTE sys.sp_executesql + @sql, + N'@database_id integer, + @object_id integer, + @min_rows integer', + @database_id, + @object_id, + @min_rows; + + IF ROWCOUNT_BIG() = 0 + BEGIN + IF @debug = 1 + BEGIN + RAISERROR('No rows inserted into #index_details', 0, 0) WITH NOWAIT; + END; + END; + + IF @debug = 1 + BEGIN + SELECT + table_name = '#index_details', + * + FROM #index_details AS id; + + RAISERROR('Generating #partition_stats insert', 0, 0) WITH NOWAIT; + END; + + SELECT + @sql = N' + SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;'; + + SELECT + @sql += N' + SELECT + database_id = @database_id, + database_name = DB_NAME(@database_id), + x.object_id, + x.index_id, + x.schema_id, + x.schema_name, + x.table_name, + x.index_name, + x.partition_id, + x.partition_number, + x.total_rows, + x.total_space_gb, + x.reserved_lob_gb, + x.reserved_row_overflow_gb, + x.data_compression_desc, + built_on = + ISNULL + ( + psfg.partition_scheme_name, + psfg.filegroup_name + ), + psfg.partition_function_name, + pc.partition_columns + FROM + ( + SELECT DISTINCT + ps.object_id, + ps.index_id, + s.schema_id, + schema_name = s.name, + table_name = t.name, + index_name = ISNULL(i.name, t.name + N''.Heap''), + ps.partition_id, + p.partition_number, + total_rows = ps.row_count, + total_space_gb = SUM(a.total_pages) * 8 / 1024.0 / 1024.0, /* Convert directly to GB */ + reserved_lob_gb = SUM(ps.lob_reserved_page_count) * 8. / 1024. / 1024.0, /* Convert directly to GB */ + reserved_row_overflow_gb = SUM(ps.row_overflow_reserved_page_count) * 8. / 1024. / 1024.0, /* Convert directly to GB */ + p.data_compression_desc, + i.data_space_id + FROM ' + QUOTENAME(@database_name) + N'.sys.tables AS t + JOIN ' + QUOTENAME(@database_name) + N'.sys.indexes AS i + ON t.object_id = i.object_id + JOIN ' + QUOTENAME(@database_name) + N'.sys.schemas AS s + ON t.schema_id = s.schema_id + JOIN ' + QUOTENAME(@database_name) + N'.sys.partitions AS p + ON i.object_id = p.object_id + AND i.index_id = p.index_id + JOIN ' + QUOTENAME(@database_name) + N'.sys.allocation_units AS a + ON p.partition_id = a.container_id + LEFT HASH JOIN ' + QUOTENAME(@database_name) + N'.sys.dm_db_partition_stats AS ps + ON p.partition_id = ps.partition_id + WHERE t.type <> N''TF'' + AND i.type IN (1, 2) + AND EXISTS + ( + SELECT + 1/0 + FROM #filtered_objects AS fo + WHERE fo.database_id = @database_id + AND fo.object_id = t.object_id + )'; + + IF @object_id IS NOT NULL + BEGIN + SELECT @sql += N' + AND t.object_id = @object_id'; + END; + + SELECT + @sql += N' + GROUP BY + ps.object_id, + ps.index_id, + s.schema_id, + s.name, + t.name, + i.name, + ps.partition_id, + p.partition_number, + ps.row_count, + p.data_compression_desc, + i.data_space_id + ) AS x + OUTER APPLY + ( + SELECT + filegroup_name = + fg.name, + partition_scheme_name = + ps.name, + partition_function_name = + pf.name + FROM ' + QUOTENAME(@database_name) + N'.sys.filegroups AS fg + FULL JOIN ' + QUOTENAME(@database_name) + N'.sys.partition_schemes AS ps + ON ps.data_space_id = fg.data_space_id + LEFT JOIN ' + QUOTENAME(@database_name) + N'.sys.partition_functions AS pf + ON pf.function_id = ps.function_id + WHERE x.data_space_id = fg.data_space_id + OR x.data_space_id = ps.data_space_id + ) AS psfg + OUTER APPLY + ( + SELECT + partition_columns = + STUFF + ( + ( + SELECT + N'', '' + + c.name + FROM ' + QUOTENAME(@database_name) + N'.sys.index_columns AS ic + JOIN ' + QUOTENAME(@database_name) + N'.sys.columns AS c + ON c.object_id = ic.object_id + AND c.column_id = ic.column_id + WHERE ic.object_id = x.object_id + AND ic.index_id = x.index_id + AND ic.partition_ordinal > 0 + ORDER BY + ic.partition_ordinal + FOR + XML + PATH(''''), + TYPE + ).value(''.'', ''nvarchar(max)''), + 1, + 2, + '''' + ) + ) AS pc + OPTION(RECOMPILE);'; + + IF @debug = 1 + BEGIN + PRINT SUBSTRING(@sql, 1, 4000); + PRINT SUBSTRING(@sql, 4000, 8000); + END; + + INSERT + #partition_stats WITH(TABLOCK) + ( + database_id, + database_name, + object_id, + index_id, + schema_id, + schema_name, + table_name, + index_name, + partition_id, + partition_number, + total_rows, + total_space_gb, + reserved_lob_gb, + reserved_row_overflow_gb, + data_compression_desc, + built_on, + partition_function_name, + partition_columns + ) + EXECUTE sys.sp_executesql + @sql, + N'@database_id integer, + @object_id integer', + @database_id, + @object_id; + + IF ROWCOUNT_BIG() = 0 + BEGIN + IF @debug = 1 + BEGIN + RAISERROR('No rows inserted into #partition_stats', 0, 0) WITH NOWAIT; + END; + END; + + IF @debug = 1 + BEGIN + SELECT + table_name = '#partition_stats', + * + FROM #partition_stats AS ps + OPTION(RECOMPILE); + + RAISERROR('Performing #index_analysis insert', 0, 0) WITH NOWAIT; + END; + + INSERT INTO + #index_analysis + WITH + (TABLOCK) + ( + database_id, + database_name, + schema_id, + schema_name, + table_name, + object_id, + index_id, + index_name, + is_unique, + key_columns, + included_columns, + filter_definition, + original_index_definition + ) + SELECT + @database_id, + database_name = DB_NAME(@database_id), + id1.schema_id, + id1.schema_name, + id1.table_name, + id1.object_id, + id1.index_id, + id1.index_name, + id1.is_unique, + key_columns = + STUFF + ( + ( + SELECT + N', ' + + id2.column_name + + CASE + WHEN id2.is_descending_key = 1 + THEN N' DESC' + ELSE N'' + END + FROM #index_details id2 + WHERE id2.object_id = id1.object_id + AND id2.index_id = id1.index_id + AND id2.is_included_column = 0 + GROUP BY + id2.column_name, + id2.is_descending_key, + id2.key_ordinal + ORDER BY + id2.key_ordinal + FOR + XML + PATH(''), + TYPE + ).value('text()[1]','nvarchar(max)'), + 1, + 2, + '' + ), + included_columns = + STUFF + ( + ( + SELECT + N', ' + + id2.column_name + FROM #index_details id2 + WHERE id2.object_id = id1.object_id + AND id2.index_id = id1.index_id + AND id2.is_included_column = 1 + GROUP BY + id2.column_name + ORDER BY + id2.column_name + FOR + XML + PATH(''), + TYPE + ).value('text()[1]','nvarchar(max)'), + 1, + 2, + '' + ), + id1.filter_definition, + /* Store the original index definition for validation */ + original_index_definition = + CASE + /* For unique constraints, use ALTER TABLE ADD CONSTRAINT syntax */ + WHEN id1.is_unique_constraint = 1 + THEN + N'ALTER TABLE ' + + QUOTENAME(DB_NAME(@database_id)) + + N'.' + + QUOTENAME(id1.schema_name) + + N'.' + + QUOTENAME(id1.table_name) + + N' ADD CONSTRAINT ' + + QUOTENAME(id1.index_name) + + N' UNIQUE (' + /* For regular indexes, use CREATE INDEX syntax */ + ELSE + N'CREATE ' + + CASE WHEN id1.is_unique = 1 THEN N'UNIQUE ' ELSE N'' END + + N'INDEX ' + + QUOTENAME(id1.index_name) + + N' ON ' + + QUOTENAME(DB_NAME(@database_id)) + + N'.' + + QUOTENAME(id1.schema_name) + + N'.' + + QUOTENAME(id1.table_name) + + N' (' + END + + STUFF + ( + ( + SELECT + N', ' + + id2.column_name + + CASE + WHEN id2.is_descending_key = 1 + THEN N' DESC' + ELSE N'' + END + FROM #index_details id2 + WHERE id2.object_id = id1.object_id + AND id2.index_id = id1.index_id + AND id2.is_included_column = 0 + GROUP BY + id2.column_name, + id2.is_descending_key, + id2.key_ordinal + ORDER BY + id2.key_ordinal + FOR + XML + PATH(''), + TYPE + ).value('text()[1]','nvarchar(max)'), + 1, + 2, + '' + ) + + N')' + + CASE + WHEN EXISTS + ( + SELECT + 1/0 + FROM #index_details id3 + WHERE id3.object_id = id1.object_id + AND id3.index_id = id1.index_id + AND id3.is_included_column = 1 + ) + THEN N' INCLUDE (' + + STUFF + ( + ( + SELECT + N', ' + + id4.column_name + FROM #index_details id4 + WHERE id4.object_id = id1.object_id + AND id4.index_id = id1.index_id + AND id4.is_included_column = 1 + GROUP BY + id4.column_name + ORDER BY + id4.column_name + FOR + XML + PATH(''), + TYPE + ).value('text()[1]','nvarchar(max)'), + 1, + 2, + '' + ) + + N')' + ELSE N'' + END + + CASE + WHEN id1.filter_definition IS NOT NULL + THEN N' WHERE ' + id1.filter_definition + ELSE N'' + END + FROM #index_details id1 + WHERE id1.is_eligible_for_dedupe = 1 + GROUP BY + id1.schema_name, + id1.schema_id, + id1.table_name, + id1.index_name, + id1.index_id, + id1.is_unique, + id1.object_id, + id1.index_id, + id1.filter_definition, + id1.is_unique_constraint + OPTION(RECOMPILE); + + IF ROWCOUNT_BIG() = 0 + BEGIN + IF @debug = 1 + BEGIN + RAISERROR('No rows inserted into #index_analysis', 0, 0) WITH NOWAIT; + END; + END; + + IF @debug = 1 + BEGIN + SELECT + table_name = '#index_analysis', + ia.* + FROM #index_analysis AS ia + OPTION(RECOMPILE); + + RAISERROR('Starting updates', 0, 0) WITH NOWAIT; + END; + + /* Calculate index priority scores based on actual columns that exist */ + UPDATE + #index_analysis + SET + #index_analysis.index_priority = + CASE + WHEN index_id = 1 + THEN 1000 /* Clustered indexes get highest priority */ + ELSE 0 + END + + + CASE + /* Unique indexes get high priority, but reduce priority for unique constraints */ + WHEN is_unique = 1 AND NOT EXISTS + ( + SELECT + 1/0 + FROM #index_details AS id_uc + WHERE id_uc.index_id = #index_analysis.index_id + AND id_uc.object_id = #index_analysis.object_id + AND id_uc.is_unique_constraint = 1 + ) THEN 500 + /* Unique constraints get lower priority */ + WHEN is_unique = 1 AND EXISTS + ( + SELECT + 1/0 + FROM #index_details AS id_uc + WHERE id_uc.index_id = #index_analysis.index_id + AND id_uc.object_id = #index_analysis.object_id + AND id_uc.is_unique_constraint = 1 + ) THEN 50 + ELSE 0 + END + + + CASE + WHEN EXISTS + ( + SELECT + 1/0 + FROM #index_details AS id + WHERE id.index_id = #index_analysis.index_id + AND id.object_id = #index_analysis.object_id + AND id.user_seeks > 0 + ) THEN 200 + ELSE 0 + END /* Indexes with seeks get priority */ + + + CASE + WHEN EXISTS + ( + SELECT + 1/0 + FROM #index_details AS id + WHERE id.index_id = #index_analysis.index_id + AND id.object_id = #index_analysis.object_id + AND id.user_scans > 0 + ) THEN 100 ELSE 0 + END + OPTION(RECOMPILE); /* Indexes with scans get some priority */ + + IF @debug = 1 + BEGIN + SELECT + table_name = '#index_analysis after priority score', + ia.* + FROM #index_analysis AS ia + OPTION(RECOMPILE); + END; + + /* Rule 1: Identify unused indexes */ + UPDATE + #index_analysis + SET + #index_analysis.consolidation_rule = + CASE + WHEN @uptime_warning = 1 + THEN 'Unused Index (WARNING: Server uptime < 14 days - usage data may be incomplete)' + ELSE 'Unused Index' + END, + #index_analysis.action = N'DISABLE' + WHERE EXISTS + ( + SELECT + 1/0 + FROM #index_details id + WHERE id.database_id = #index_analysis.database_id + AND id.object_id = #index_analysis.object_id + AND id.index_id = #index_analysis.index_id + AND id.user_seeks = 0 + AND id.user_scans = 0 + AND id.user_lookups = 0 + AND id.is_primary_key = 0 /* Don't disable primary keys */ + AND id.is_unique_constraint = 0 /* Don't disable unique constraints */ + AND id.is_eligible_for_dedupe = 1 /* Only eligible indexes */ + ) + AND #index_analysis.index_id <> 1 + OPTION(RECOMPILE); /* Don't disable clustered indexes */ + + IF @debug = 1 + BEGIN + SELECT + table_name = '#index_analysis after rule 1', + ia.* + FROM #index_analysis AS ia + OPTION(RECOMPILE); + END; + + /* Rule 2: Exact duplicates - matching key columns and includes */ + UPDATE + ia1 + SET + ia1.consolidation_rule = 'Exact Duplicate', + ia1.target_index_name = + CASE + WHEN ia1.index_priority > ia2.index_priority + THEN NULL /* This index is the keeper */ + WHEN ia1.index_priority = ia2.index_priority AND ia1.index_name < ia2.index_name + THEN NULL /* When tied, use alphabetical ordering for consistency */ + ELSE ia2.index_name /* Other index is the keeper */ + END, + ia1.action = + CASE + WHEN ia1.index_priority > ia2.index_priority + THEN 'KEEP' /* This index is the keeper */ + WHEN ia1.index_priority = ia2.index_priority AND ia1.index_name < ia2.index_name + THEN 'KEEP' /* When tied, use alphabetical ordering for consistency */ + ELSE 'DISABLE' /* Other index gets disabled */ + END + FROM #index_analysis AS ia1 + JOIN #index_analysis AS ia2 + ON ia1.database_id = ia2.database_id + AND ia1.object_id = ia2.object_id + AND ia1.index_name <> ia2.index_name + AND ia1.key_columns = ia2.key_columns /* Exact key match */ + AND ISNULL(ia1.included_columns, '') = ISNULL(ia2.included_columns, '') /* Exact includes match */ + AND ISNULL(ia1.filter_definition, '') = ISNULL(ia2.filter_definition, '') /* Matching filters */ + WHERE ia1.consolidation_rule IS NULL /* Not already processed */ + AND ia2.consolidation_rule IS NULL /* Not already processed */ + /* Exclude unique constraints - we'll handle those separately in Rule 7 */ + AND NOT EXISTS + ( + SELECT + 1/0 + FROM #index_details AS id1_uc + WHERE id1_uc.database_id = ia1.database_id + AND id1_uc.object_id = ia1.object_id + AND id1_uc.index_id = ia1.index_id + AND id1_uc.is_unique_constraint = 1 + ) + AND NOT EXISTS + ( + SELECT + 1/0 + FROM #index_details AS id2_uc + WHERE id2_uc.database_id = ia2.database_id + AND id2_uc.object_id = ia2.object_id + AND id2_uc.index_id = ia2.index_id + AND id2_uc.is_unique_constraint = 1 + ) + AND EXISTS + ( + SELECT + 1/0 + FROM #index_details AS id1 + WHERE id1.database_id = ia1.database_id + AND id1.object_id = ia1.object_id + AND id1.index_id = ia1.index_id + AND id1.is_eligible_for_dedupe = 1 + ) + AND EXISTS + ( + SELECT + 1/0 + FROM #index_details AS id2 + WHERE id2.database_id = ia2.database_id + AND id2.object_id = ia2.object_id + AND id2.index_id = ia2.index_id + AND id2.is_eligible_for_dedupe = 1 + ) + OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + SELECT + table_name = '#index_analysis after rule 2', + ia.* + FROM #index_analysis AS ia + OPTION(RECOMPILE); + + /* Special debug for exact duplicates */ + RAISERROR('Special debug for exact duplicates after rule 2:', 0, 0) WITH NOWAIT; + SELECT + ia1.index_name AS index1_name, + ia1.action AS index1_action, + ia1.consolidation_rule AS index1_rule, + ia1.index_priority AS index1_priority, + ia1.target_index_name AS index1_target, + ia1.filter_definition AS index1_filter, + ia2.index_name AS index2_name, + ia2.action AS index2_action, + ia2.consolidation_rule AS index2_rule, + ia2.index_priority AS index2_priority, + ia2.target_index_name AS index2_target, + ia2.filter_definition AS index2_filter + FROM #index_analysis AS ia1 + JOIN #index_analysis AS ia2 + ON ia1.database_id = ia2.database_id + AND ia1.object_id = ia2.object_id + AND ia1.index_name <> ia2.index_name + AND ia1.key_columns = ia2.key_columns /* Exact key match */ + AND ISNULL(ia1.included_columns, '') = ISNULL(ia2.included_columns, '') /* Exact includes match */ + AND ISNULL(ia1.filter_definition, '') = ISNULL(ia2.filter_definition, '') /* Matching filters */ + WHERE ia1.consolidation_rule = 'Exact Duplicate' + OR ia2.consolidation_rule = 'Exact Duplicate' + ORDER BY ia1.index_name + OPTION(RECOMPILE); + END; + + /* Rule 3: Key duplicates - matching key columns, different includes */ + UPDATE + ia1 + SET + ia1.consolidation_rule = 'Key Duplicate', + ia1.target_index_name = + CASE + /* If one is unique and the other isn't, prefer the unique one */ + WHEN ia1.is_unique = 1 + AND ia2.is_unique = 0 + THEN NULL + WHEN ia1.is_unique = 0 + AND ia2.is_unique = 1 + THEN ia2.index_name + /* Otherwise use priority */ + WHEN ia1.index_priority >= ia2.index_priority + THEN NULL + ELSE ia2.index_name + END, + ia1.action = + CASE + WHEN (ia1.is_unique = 1 AND ia2.is_unique = 0) + OR + ( + ia1.index_priority >= ia2.index_priority + AND NOT (ia1.is_unique = 0 AND ia2.is_unique = 1) + ) + AND ISNULL(ia1.included_columns, N'') <> ISNULL(ia2.included_columns, N'') + THEN 'MERGE INCLUDES' /* Keep this index but merge includes */ + ELSE 'DISABLE' /* Other index is keeper, disable this one */ + END, + /* For the winning index, set clear superseded_by text for the report */ + ia1.superseded_by = + CASE + WHEN (ia1.is_unique = 1 AND ia2.is_unique = 0) + OR + ( + ia1.index_priority >= ia2.index_priority + AND NOT (ia1.is_unique = 0 AND ia2.is_unique = 1) + ) + THEN 'Supersedes ' + + ia2.index_name + ELSE NULL + END + FROM #index_analysis AS ia1 + JOIN #index_analysis AS ia2 + ON ia1.database_id = ia2.database_id + AND ia1.object_id = ia2.object_id + AND ia1.index_name <> ia2.index_name + AND ia1.key_columns = ia2.key_columns /* Exact key match */ + AND ISNULL(ia1.included_columns, '') <> ISNULL(ia2.included_columns, '') /* Different includes */ + AND ISNULL(ia1.filter_definition, '') = ISNULL(ia2.filter_definition, '') /* Matching filters */ + WHERE ia1.consolidation_rule IS NULL /* Not already processed */ + AND ia2.consolidation_rule IS NULL /* Not already processed */ + /* Exclude pairs where either one is a unique constraint (we'll handle those separately in Rule 7) */ + AND NOT EXISTS + ( + SELECT + 1/0 + FROM #index_details AS id1_uc + WHERE id1_uc.database_id = ia1.database_id + AND id1_uc.object_id = ia1.object_id + AND id1_uc.index_id = ia1.index_id + AND id1_uc.is_unique_constraint = 1 + ) + AND NOT EXISTS + ( + SELECT + 1/0 + FROM #index_details AS id2_uc + WHERE id2_uc.database_id = ia2.database_id + AND id2_uc.object_id = ia2.object_id + AND id2_uc.index_id = ia2.index_id + AND id2_uc.is_unique_constraint = 1 + ) + AND EXISTS + ( + SELECT + 1/0 + FROM #index_details AS id1 + WHERE id1.database_id = ia1.database_id + AND id1.object_id = ia1.object_id + AND id1.index_id = ia1.index_id + AND id1.is_eligible_for_dedupe = 1 + ) + AND EXISTS + ( + SELECT + 1/0 + FROM #index_details AS id2 + WHERE id2.database_id = ia2.database_id + AND id2.object_id = ia2.object_id + AND id2.index_id = ia2.index_id + AND id2.is_eligible_for_dedupe = 1 + ) + OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + SELECT + table_name = '#index_analysis after rule 3', + ia.* + FROM #index_analysis AS ia + OPTION(RECOMPILE); + END; + + /* Rule 4: Superset/subset key columns */ + UPDATE + ia1 + SET + ia1.consolidation_rule = 'Key Subset', + ia1.target_index_name = ia2.index_name, + ia1.action = N'DISABLE' /* The narrower index gets disabled */ + FROM #index_analysis AS ia1 + JOIN #index_analysis AS ia2 + ON ia1.database_id = ia2.database_id + AND ia1.object_id = ia2.object_id + AND ia1.index_name <> ia2.index_name + AND ia2.key_columns LIKE (ia1.key_columns + '%') /* ia2 has wider key that starts with ia1's key */ + AND ISNULL(ia1.filter_definition, '') = ISNULL(ia2.filter_definition, '') /* Matching filters */ + /* Exception: If narrower index is unique and wider is not, they should not be merged */ + AND NOT (ia1.is_unique = 1 AND ia2.is_unique = 0) + WHERE ia1.consolidation_rule IS NULL /* Not already processed */ + AND ia2.consolidation_rule IS NULL /* Not already processed */ + AND EXISTS + ( + SELECT + 1/0 + FROM #index_details AS id1 + WHERE id1.database_id = ia1.database_id + AND id1.object_id = ia1.object_id + AND id1.index_id = ia1.index_id + AND id1.is_eligible_for_dedupe = 1 + ) + AND EXISTS + ( + SELECT + 1/0 + FROM #index_details AS id2 + WHERE id2.database_id = ia2.database_id + AND id2.object_id = ia2.object_id + AND id2.index_id = ia2.index_id + AND id2.is_eligible_for_dedupe = 1 + ) + OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + SELECT + table_name = '#index_analysis after rule 4', + ia.* + FROM #index_analysis AS ia + OPTION(RECOMPILE); + END; + + /* Rule 5: Mark superset indexes for merging with includes from subset */ + UPDATE + ia2 + SET + ia2.consolidation_rule = 'Key Superset', + ia2.action = N'MERGE INCLUDES', /* The wider index gets merged with includes */ + ia2.superseded_by = COALESCE(ia2.superseded_by + ', ', '') + 'Supersedes ' + ia1.index_name + FROM #index_analysis AS ia1 + JOIN #index_analysis AS ia2 + ON ia1.database_id = ia2.database_id + AND ia1.object_id = ia2.object_id + AND ia1.target_index_name = ia2.index_name /* Link from Rule 4 */ + WHERE ia1.consolidation_rule = 'Key Subset' + AND ia1.action = 'DISABLE' + AND ia2.consolidation_rule IS NULL /* Not already processed */ + OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + SELECT + table_name = '#index_analysis after rule 5', + ia.* + FROM #index_analysis AS ia + OPTION(RECOMPILE); + END; + + /* Rule 6: Merge includes from subset to superset indexes */ + WITH KeySubsetSuperset AS + ( + SELECT + superset.database_id, + superset.object_id, + superset.index_id, + superset.index_name, + superset.included_columns AS superset_includes, + subset.included_columns AS subset_includes + FROM #index_analysis AS superset + JOIN #index_analysis AS subset + ON superset.database_id = subset.database_id + AND superset.object_id = subset.object_id + AND subset.target_index_name = superset.index_name + WHERE superset.action = 'MERGE INCLUDES' + AND subset.action = 'DISABLE' + AND superset.consolidation_rule = 'Key Superset' + AND subset.consolidation_rule = 'Key Subset' + ) + UPDATE + ia + SET + ia.included_columns = + CASE + /* If both have includes, combine them without duplicates */ + WHEN kss.superset_includes IS NOT NULL + AND kss.subset_includes IS NOT NULL + THEN + /* Create combined includes using XML method that works with all SQL Server versions */ + ( + SELECT + /* Combine both sets of includes */ + combined_cols = + STUFF + ( + ( + SELECT DISTINCT + N', ' + t.c.value('.', 'sysname') + FROM + ( + /* Create XML from superset includes */ + SELECT + x = CONVERT + ( + xml, + N'' + + REPLACE(kss.superset_includes, N', ', N'') + + N'' + ) + + UNION ALL + + /* Create XML from subset includes */ + SELECT + x = CONVERT + ( + xml, + N'' + + REPLACE(kss.subset_includes, N', ', N'') + + N'' + ) + ) AS a + /* Split XML into individual columns */ + CROSS APPLY a.x.nodes('/c') AS t(c) + FOR + XML + PATH('') + ), + 1, + 2, + '' + ) + ) + /* If only subset has includes, use those */ + WHEN kss.superset_includes IS NULL AND kss.subset_includes IS NOT NULL + THEN kss.subset_includes + /* If only superset has includes or neither has includes, keep superset's includes */ + ELSE kss.superset_includes + END + FROM #index_analysis AS ia + JOIN KeySubsetSuperset AS kss + ON ia.database_id = kss.database_id + AND ia.object_id = kss.object_id + AND ia.index_id = kss.index_id + WHERE ia.action = 'MERGE INCLUDES'; + + IF @debug = 1 + BEGIN + SELECT + table_name = '#index_analysis after rule 6', + ia.* + FROM #index_analysis AS ia + OPTION(RECOMPILE); + END; + + /* Update the superseded_by column for the wider index in a separate statement */ + UPDATE + ia2 + SET + ia2.superseded_by = 'Supersedes ' + ia1.index_name + FROM #index_analysis AS ia1 + JOIN #index_analysis AS ia2 + ON ia1.database_id = ia2.database_id + AND ia1.object_id = ia2.object_id + AND ia1.index_name <> ia2.index_name + AND ia2.key_columns LIKE (ia1.key_columns + '%') /* ia2 has wider key that starts with ia1's key */ + AND ISNULL(ia1.filter_definition, '') = ISNULL(ia2.filter_definition, '') /* Matching filters */ + /* Exception: If narrower index is unique and wider is not, they should not be merged */ + AND NOT (ia1.is_unique = 1 AND ia2.is_unique = 0) + WHERE ia1.consolidation_rule = 'Key Subset' /* Use records just processed in previous UPDATE */ + AND ia1.target_index_name = ia2.index_name /* Make sure we're updating the right wider index */ + OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + SELECT + table_name = '#index_analysis after update superseded', + ia.* + FROM #index_analysis AS ia + OPTION(RECOMPILE); + END; + + /* Rule 7: Unique constraint vs. nonclustered index handling */ + UPDATE + ia1 + SET + ia1.consolidation_rule = 'Unique Constraint Replacement', + ia1.action = + CASE + WHEN ia1.is_unique = 0 + THEN 'MAKE UNIQUE' /* Convert to unique index */ + ELSE 'KEEP' /* Already unique, so just keep it */ + END + FROM #index_analysis AS ia1 + WHERE ia1.consolidation_rule IS NULL /* Not already processed */ + AND ia1.action IS NULL /* Not already processed by earlier rules */ + AND EXISTS + ( + /* Find nonclustered indexes */ + SELECT + 1/0 + FROM #index_details AS id1 + WHERE id1.database_id = ia1.database_id + AND id1.object_id = ia1.object_id + AND id1.index_id = ia1.index_id + AND id1.is_eligible_for_dedupe = 1 + ) + AND EXISTS + ( + /* Find unique constraints with matching key columns */ + SELECT + 1/0 + FROM #index_details AS id2 + WHERE id2.database_id = ia1.database_id + AND id2.object_id = ia1.object_id + AND id2.is_unique_constraint = 1 + AND NOT EXISTS + ( + /* Verify key columns match between index and unique constraint */ + SELECT + id2_inner.column_name + FROM #index_details AS id2_inner + WHERE id2_inner.database_id = id2.database_id + AND id2_inner.object_id = id2.object_id + AND id2_inner.index_id = id2.index_id + AND id2_inner.is_included_column = 0 + + EXCEPT + + SELECT + id1_inner.column_name + FROM #index_details AS id1_inner + WHERE id1_inner.database_id = ia1.database_id + AND id1_inner.object_id = ia1.object_id + AND id1_inner.index_id = ia1.index_id + AND id1_inner.is_included_column = 0 + ) + ) + OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + SELECT + table_name = '#index_analysis after rule 7', + ia.* + FROM #index_analysis AS ia + OPTION(RECOMPILE); + END; + + /* Rule 7.5: Mark unique constraints that have matching nonclustered indexes for disabling */ + /* First, mark unique constraints for disabling */ + UPDATE + ia_uc + SET + ia_uc.consolidation_rule = 'Unique Constraint Replacement', + ia_uc.action = N'DISABLE', /* Mark unique constraint for disabling */ + ia_uc.target_index_name = ia_nc.index_name /* Point to the nonclustered index that will replace it */ + FROM #index_analysis AS ia_uc /* Unique constraint */ + JOIN #index_details AS id_uc /* Join to get unique constraint details */ + ON id_uc.database_id = ia_uc.database_id + AND id_uc.object_id = ia_uc.object_id + AND id_uc.index_id = ia_uc.index_id + AND id_uc.is_unique_constraint = 1 /* This is a unique constraint */ + JOIN #index_analysis AS ia_nc /* Join to find nonclustered index */ + ON ia_nc.database_id = ia_uc.database_id + AND ia_nc.object_id = ia_uc.object_id + AND ia_nc.index_name <> ia_uc.index_name /* Different index */ + WHERE + /* Verify key columns EXACT match between index and unique constraint */ + ia_uc.key_columns = ia_nc.key_columns + OPTION(RECOMPILE); + + /* Second, mark nonclustered indexes to be made unique */ + UPDATE + ia_nc + SET + ia_nc.consolidation_rule = 'Unique Constraint Replacement', + ia_nc.action = N'MAKE UNIQUE', /* Mark nonclustered index to be made unique */ + /* CRITICAL: Set target_index_name to NULL to ensure it gets a MERGE script */ + ia_nc.target_index_name = NULL + FROM #index_analysis AS ia_nc /* Nonclustered index */ + JOIN #index_details AS id_nc /* Join to get nonclustered index details */ + ON id_nc.database_id = ia_nc.database_id + AND id_nc.object_id = ia_nc.object_id + AND id_nc.index_id = ia_nc.index_id + AND id_nc.is_unique_constraint = 0 /* This is not a unique constraint */ + WHERE + /* Two conditions for matching: + 1. Index key columns exactly match a unique constraint's key columns + 2. A unique constraint is already marked for DISABLE and has this index as target */ + (EXISTS ( + /* Find unique constraint with matching keys that should be disabled */ + SELECT 1 + FROM #index_analysis AS ia_uc + JOIN #index_details AS id_uc + ON id_uc.database_id = ia_uc.database_id + AND id_uc.object_id = ia_uc.object_id + AND id_uc.index_id = ia_uc.index_id + AND id_uc.is_unique_constraint = 1 + WHERE + ia_uc.database_id = ia_nc.database_id + AND ia_uc.object_id = ia_nc.object_id + /* Check that both indexes have EXACTLY the same key columns */ + AND ia_uc.key_columns = ia_nc.key_columns + )) + OPTION(RECOMPILE); + + /* CRITICAL: Ensure that only the unique constraints that exactly match get this treatment */ + /* And remove any incorrect MAKE UNIQUE actions */ + UPDATE ia + SET action = NULL, + consolidation_rule = NULL, + target_index_name = NULL + FROM #index_analysis AS ia + WHERE ia.action = N'MAKE UNIQUE' + AND NOT EXISTS ( + /* Check if there's a unique constraint with matching keys that points to this index */ + SELECT 1 + FROM #index_analysis AS ia_uc + WHERE ia_uc.database_id = ia.database_id + AND ia_uc.object_id = ia.object_id + AND ia_uc.key_columns = ia.key_columns + AND ia_uc.action = N'DISABLE' + AND ia_uc.target_index_name = ia.index_name + ); + + /* Make sure the nonclustered index has the superseded_by field set correctly */ + UPDATE ia_nc + SET + ia_nc.superseded_by = + CASE + WHEN ia_nc.superseded_by IS NULL THEN N'Will replace constraint ' + ia_uc.index_name + ELSE ia_nc.superseded_by + N', will replace constraint ' + ia_uc.index_name + END + FROM #index_analysis AS ia_nc + JOIN #index_analysis AS ia_uc + ON ia_uc.database_id = ia_nc.database_id + AND ia_uc.object_id = ia_nc.object_id + AND ia_uc.action = N'DISABLE' + AND ia_uc.target_index_name = ia_nc.index_name + WHERE ia_nc.action = N'MAKE UNIQUE' + OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + SELECT + table_name = '#index_analysis after rule 7.5', + ia.* + FROM #index_analysis AS ia + OPTION(RECOMPILE); + + /* Special debug for uq_a and uq_i_a */ + RAISERROR('Special debug for uq_a and uq_i_a after rule 7.5:', 0, 0) WITH NOWAIT; + SELECT + index_name, + action, + consolidation_rule, + target_index_name, + superseded_by, + included_columns, + index_priority + FROM #index_analysis + WHERE index_name IN ('uq_a', 'uq_i_a') + ORDER BY index_name + OPTION(RECOMPILE); + + /* Check the merge script eligibility */ + RAISERROR('Checking MERGE script eligibility for uq_i_a:', 0, 0) WITH NOWAIT; + SELECT + 'uq_i_a eligibility check', + ia.index_name, + ia.action, + ia.target_index_name, + ce.can_compress, + /* Show which conditions are being met for script generation */ + condition1 = CASE WHEN ia.action IN (N'MERGE INCLUDES', N'MAKE UNIQUE') THEN 'YES' ELSE 'NO' END, + condition2 = CASE WHEN ce.can_compress = 1 THEN 'YES' ELSE 'NO' END, + condition3 = CASE WHEN ia.target_index_name IS NULL THEN 'YES' ELSE 'NO' END, + /* Will this index get a MERGE script? */ + will_get_merge_script = + CASE + WHEN ia.action IN (N'MERGE INCLUDES', N'MAKE UNIQUE') + AND ce.can_compress = 1 + AND ia.target_index_name IS NULL + THEN 'YES' + ELSE 'NO' + END + FROM #index_analysis AS ia + JOIN #compression_eligibility AS ce + ON ia.database_id = ce.database_id + AND ia.object_id = ce.object_id + AND ia.index_id = ce.index_id + WHERE ia.index_name = 'uq_i_a' + OPTION(RECOMPILE); + END; + + /* Rule 8: Identify indexes with same keys but in different order after first column */ + /* This rule flags indexes that have the same set of key columns but ordered differently */ + /* These need manual review as they may be redundant depending on query patterns */ + UPDATE + ia1 + SET + ia1.consolidation_rule = 'Same Keys Different Order', + ia1.action = N'REVIEW', /* These need manual review */ + ia1.target_index_name = ia2.index_name /* Reference the partner index */ + FROM #index_analysis AS ia1 + JOIN #index_analysis AS ia2 + ON ia1.database_id = ia2.database_id + AND ia1.object_id = ia2.object_id + AND ia1.index_name < ia2.index_name /* Only process each pair once */ + AND ia1.consolidation_rule IS NULL /* Not already processed */ + AND ia2.consolidation_rule IS NULL /* Not already processed */ + WHERE + /* Leading columns match */ + EXISTS + ( + SELECT + 1/0 + FROM #index_details AS id1 + JOIN #index_details AS id2 + ON id1.database_id = id2.database_id + AND id1.object_id = id2.object_id + AND id1.column_name = id2.column_name + AND id1.key_ordinal = 1 + AND id2.key_ordinal = 1 + WHERE id1.database_id = ia1.database_id + AND id1.object_id = ia1.object_id + AND id1.index_id = ia1.index_id + AND id2.index_id = ia2.index_id + ) + /* Same set of key columns but in different order */ + AND NOT EXISTS + ( + /* Make sure the sets of key columns are exactly the same */ + SELECT + id1.column_name + FROM #index_details AS id1 + WHERE id1.database_id = ia1.database_id + AND id1.object_id = ia1.object_id + AND id1.index_id = ia1.index_id + AND id1.is_included_column = 0 + AND id1.key_ordinal > 0 + + EXCEPT + + SELECT + id2.column_name + FROM #index_details AS id2 + WHERE id2.database_id = ia2.database_id + AND id2.object_id = ia2.object_id + AND id2.index_id = ia2.index_id + AND id2.is_included_column = 0 + AND id2.key_ordinal > 0 + ) + /* But the order is different (excluding the first column) */ + AND EXISTS + ( + /* There's at least one column in a different position */ + SELECT + 1/0 + FROM #index_details AS id1 + JOIN #index_details AS id2 + ON id1.database_id = id2.database_id + AND id1.object_id = id2.object_id + AND id1.column_name = id2.column_name + AND id1.key_ordinal <> id2.key_ordinal + AND id1.key_ordinal > 1 /* After the first column */ + AND id2.key_ordinal > 1 /* After the first column */ + WHERE id1.database_id = ia1.database_id + AND id1.object_id = ia1.object_id + AND id1.index_id = ia1.index_id + AND id2.index_id = ia2.index_id + ) + OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + SELECT + table_name = '#index_analysis after rule 8', + ia.* + FROM #index_analysis AS ia + OPTION(RECOMPILE); + END; + + /* Create a reference to the detailed summary that will appear at the end */ + IF @debug = 1 + BEGIN + RAISERROR('Generating #index_cleanup_results insert', 0, 0) WITH NOWAIT; + END; + + INSERT INTO + #index_cleanup_results + ( + result_type, + sort_order, + database_name, + schema_name, + table_name, + index_name, + script_type, + additional_info, + target_index_name, + superseded_info, + original_index_definition, + index_size_gb, + index_rows, + index_reads, + index_writes + ) + SELECT + result_type = 'SUMMARY', + sort_order = 1, + database_name = '', + schema_name = '', + table_name = '', + index_name = '', + script_type = 'Index Cleanup Scripts', + additional_info = N'A detailed index analysis report appears after these scripts', + target_index_name = '', + superseded_info = '', + original_index_definition = '', + index_size_gb = 0, + index_rows = 0, + index_reads = 0, + index_writes = 0 + OPTION(RECOMPILE); + + + /* Identify key duplicates where both indexes have MERGE INCLUDES action */ + IF @debug = 1 + BEGIN + RAISERROR('Generating #key_duplicate_dedupe insert', 0, 0) WITH NOWAIT; + END; + + INSERT INTO + #key_duplicate_dedupe + WITH + (TABLOCK) + ( + database_id, + object_id, + database_name, + schema_name, + table_name, + base_key_columns, + filter_definition, + winning_index_name, + index_list + ) + SELECT + ia.database_id, + ia.object_id, + database_name = MAX(ia.database_name), + schema_name = MAX(ia.schema_name), + table_name = MAX(ia.table_name), + base_key_columns = ia.key_columns, + filter_definition = ISNULL(ia.filter_definition, N''), + /* Choose the index with most included columns as the winner (or first alphabetically if tied) */ + winning_index_name = + ( + SELECT TOP (1) + candidate.index_name + FROM #index_analysis AS candidate + WHERE candidate.database_id = ia.database_id + AND candidate.object_id = ia.object_id + AND candidate.key_columns = ia.key_columns + AND ISNULL(candidate.filter_definition, '') = ISNULL(ia.filter_definition, '') + AND candidate.action = N'MERGE INCLUDES' + AND candidate.consolidation_rule = 'Key Duplicate' + ORDER BY + /* First prefer indexes with "_Extended" in the name */ + CASE WHEN candidate.index_name LIKE '%\_Extended%' ESCAPE '\' THEN 1 ELSE 0 END DESC, + /* Then prefer indexes with more included columns (by length as a proxy) */ + LEN(ISNULL(candidate.included_columns, '')) DESC, + /* Then alphabetically for stability */ + candidate.index_name + ), + /* Build a list of other indexes in this group */ + index_list = + STUFF + ( + ( + SELECT + N', ' + + inner_ia.index_name + FROM #index_analysis AS inner_ia + WHERE inner_ia.database_id = ia.database_id + AND inner_ia.object_id = ia.object_id + AND inner_ia.key_columns = ia.key_columns + AND ISNULL(inner_ia.filter_definition, '') = ISNULL(ia.filter_definition, '') + AND inner_ia.action = N'MERGE INCLUDES' + AND inner_ia.consolidation_rule = 'Key Duplicate' + ORDER BY + inner_ia.index_name + FOR + XML + PATH(''), + TYPE + ).value('.', 'nvarchar(max)'), + 1, + 2, + '' + ) + FROM #index_analysis AS ia + WHERE ia.action = N'MERGE INCLUDES' + AND ia.consolidation_rule = 'Key Duplicate' + GROUP BY + ia.database_id, + ia.object_id, + ia.key_columns, + ia.filter_definition + HAVING + COUNT_BIG(*) > 1 + OPTION(RECOMPILE); /* Only groups with multiple MERGE INCLUDES */ + + /* Update the index_analysis table to make only one index the winner in each group */ + IF @debug = 1 + BEGIN + RAISERROR('Generating #index_analysis updates', 0, 0) WITH NOWAIT; + END; + + UPDATE + ia + SET + ia.action = N'DISABLE', + ia.target_index_name = kdd.winning_index_name, + ia.superseded_by = NULL + FROM #index_analysis AS ia + JOIN #key_duplicate_dedupe AS kdd + ON ia.database_id = kdd.database_id + AND ia.object_id = kdd.object_id + AND ia.key_columns = kdd.base_key_columns + AND ISNULL(ia.filter_definition, N'') = kdd.filter_definition + WHERE ia.index_name <> kdd.winning_index_name + AND ia.action = N'MERGE INCLUDES' + AND ia.consolidation_rule = 'Key Duplicate' + OPTION(RECOMPILE); + + /* Update the winning index's superseded_by to list all other indexes */ + UPDATE + ia + SET + ia.superseded_by = 'Supersedes ' + + REPLACE + ( + kdd.index_list, + ia.index_name + N', ', N'' + ) /* Remove self from list if present */ + FROM #index_analysis AS ia + JOIN #key_duplicate_dedupe AS kdd + ON ia.database_id = kdd.database_id + AND ia.object_id = kdd.object_id + AND ia.key_columns = kdd.base_key_columns + AND ISNULL(ia.filter_definition, '') = kdd.filter_definition + WHERE ia.index_name = kdd.winning_index_name + OPTION(RECOMPILE); + + /* Find indexes with same key columns where one has includes that are a subset of another */ + IF @debug = 1 + BEGIN + RAISERROR('Generating #include_subset_dedupe insert', 0, 0) WITH NOWAIT; + END; + + INSERT INTO + #include_subset_dedupe + WITH + (TABLOCK) + ( + database_id, + object_id, + subset_index_name, + superset_index_name, + subset_included_columns, + superset_included_columns + ) + SELECT + ia1.database_id, + ia1.object_id, + ia1.index_name AS subset_index_name, + ia2.index_name AS superset_index_name, + ia1.included_columns AS subset_included_columns, + ia2.included_columns AS superset_included_columns + FROM #index_analysis AS ia1 + JOIN #index_analysis AS ia2 + ON ia1.database_id = ia2.database_id + AND ia1.object_id = ia2.object_id + AND ia1.key_columns = ia2.key_columns + AND ISNULL(ia1.filter_definition, N'') = ISNULL(ia2.filter_definition, N'') + AND ia1.index_name <> ia2.index_name + AND ia1.action = N'MERGE INCLUDES' + AND ia2.action = N'MERGE INCLUDES' + AND ia1.consolidation_rule = 'Key Duplicate' + AND ia2.consolidation_rule = 'Key Duplicate' + /* Find where subset's includes are contained within superset's includes */ + AND + ( + ia1.included_columns IS NULL + OR CHARINDEX(ia1.included_columns, ia2.included_columns) > 0 + ) + /* Don't match if lengths are the same (would be exact duplicates) */ + AND + ( + ia1.included_columns IS NULL + OR ia2.included_columns IS NULL + OR LEN(ia1.included_columns) < LEN(ia2.included_columns) + ) + OPTION(RECOMPILE); + + /* Update the subset indexes to be disabled, since supersets already contain their columns */ + IF @debug = 1 + BEGIN + RAISERROR('Generating #index_analysis updates', 0, 0) WITH NOWAIT; + END; + + UPDATE + ia + SET + ia.action = N'DISABLE', + ia.target_index_name = isd.superset_index_name, + ia.superseded_by = NULL + FROM #index_analysis AS ia + JOIN #include_subset_dedupe AS isd + ON ia.database_id = isd.database_id + AND ia.object_id = isd.object_id + AND ia.index_name = isd.subset_index_name + OPTION(RECOMPILE); + + /* Update the superset indexes to indicate they supersede the subset indexes */ + UPDATE + ia + SET + ia.superseded_by = + CASE + WHEN ia.superseded_by IS NULL + THEN N'Supersedes ' + isd.subset_index_name + ELSE ia.superseded_by + N', ' + isd.subset_index_name + END + FROM #index_analysis AS ia + JOIN #include_subset_dedupe AS isd + ON ia.database_id = isd.database_id + AND ia.object_id = isd.object_id + AND ia.index_name = isd.superset_index_name + OPTION(RECOMPILE); + + /* Update winning indexes that don't actually need changes to have action = N'KEEP' */ + UPDATE + ia + SET + /* Change action to 'KEEP' for indexes that don't need to be modified */ + ia.action = N'KEEP' + FROM #index_analysis AS ia + WHERE ia.action = N'MERGE INCLUDES' + AND ia.superseded_by IS NOT NULL + /* Check if the index name contains "Extended" and has more included columns */ + AND (ia.index_name LIKE '%\_Extended%' ESCAPE '\' OR ia.index_name LIKE '%\_Extended' OR ia.index_name LIKE '%_Extended%') + /* This should indicate it already has all the needed includes */ + AND NOT EXISTS + ( + /* Find any indexes it supersedes that have includes not in this index */ + SELECT + 1/0 + FROM #index_analysis AS ia_subset + WHERE ia_subset.database_id = ia.database_id + AND ia_subset.object_id = ia.object_id + AND ia_subset.key_columns = ia.key_columns + AND ia_subset.action = N'DISABLE' + AND ia_subset.target_index_name = ia.index_name + /* This complex check handles cases where the superset doesn't contain all subset columns */ + AND CHARINDEX(ISNULL(ia_subset.included_columns, N''), ISNULL(ia.included_columns, N'')) = 0 + AND ISNULL(ia_subset.included_columns, N'') <> N'' + ) + OPTION(RECOMPILE); + + /* Insert merge scripts for indexes */ + IF @debug = 1 + BEGIN + RAISERROR('Generating #index_cleanup_results insert, MERGE', 0, 0) WITH NOWAIT; + END; + + INSERT INTO + #index_cleanup_results + ( + result_type, + sort_order, + database_name, + schema_name, + table_name, + index_name, + script_type, + consolidation_rule, + target_index_name, + script, + additional_info, + superseded_info, + original_index_definition, + index_size_gb, + index_rows, + index_reads, + index_writes + ) + SELECT DISTINCT + result_type = 'MERGE', + /* Put merge target indexes higher in sort order (5) so they appear before + indexes that will be disabled (20) */ + sort_order = 5, + ia.database_name, + ia.schema_name, + ia.table_name, + ia.index_name, + script_type = N'MERGE SCRIPT', + ia.consolidation_rule, + ia.target_index_name, + script = + CASE + WHEN ia.action = N'MAKE UNIQUE' + THEN N'CREATE UNIQUE ' + WHEN ia.action = N'MERGE INCLUDES' + THEN N'CREATE ' + ELSE N'CREATE ' + END + + N'INDEX ' + + QUOTENAME(ia.index_name) + + N' ON ' + + QUOTENAME(ia.database_name) + + N'.' + + QUOTENAME(ia.schema_name) + + N'.' + + QUOTENAME(ia.table_name) + + N' (' + + ia.key_columns + + N')' + + CASE + WHEN ia.included_columns IS NOT NULL + AND LEN(ia.included_columns) > 0 + AND ia.action = N'MERGE INCLUDES' + THEN N' INCLUDE (' + + ia.included_columns + + N')' + WHEN ia.included_columns IS NOT NULL + AND LEN(ia.included_columns) > 0 + THEN N' INCLUDE (' + + ia.included_columns + + N')' + ELSE N'' + END + + CASE + WHEN ia.filter_definition IS NOT NULL + THEN N' WHERE ' + + ia.filter_definition + ELSE N'' + END + + N' WITH (DROP_EXISTING = ON, FILLFACTOR = 100, SORT_IN_TEMPDB = ON, ONLINE = ' + + CASE + WHEN @online = 1 + THEN N'ON' + ELSE N'OFF' + END + + CASE + WHEN ce.can_compress = 1 + THEN ', DATA_COMPRESSION = PAGE' + ELSE N'' + END + + N')' + + CASE + WHEN ps.partition_function_name IS NOT NULL + THEN N' ON ' + + QUOTENAME(ps.partition_function_name) + + N'(' + + ISNULL(ps.partition_columns, N'') + + N')' + WHEN ps.built_on IS NOT NULL + THEN N' ON ' + + QUOTENAME(ps.built_on) + ELSE N'' + END + N';', + /* Additional info about what this script does */ + additional_info = + CASE + WHEN ia.action = N'MERGE INCLUDES' + THEN N'This index will absorb includes from duplicate indexes' + WHEN ia.action = N'MAKE UNIQUE' + THEN N'This index will replace a unique constraint' + ELSE NULL + END, + /* Add superseded_by information if available */ + ia.superseded_by, + /* Original index definition for validation */ + ia.original_index_definition, + NULL, + NULL, + NULL, + NULL + FROM #index_analysis AS ia + LEFT JOIN + ( + /* Get the partition info for each index */ + SELECT + ps.database_id, + ps.object_id, + ps.index_id, + ps.built_on, + ps.partition_function_name, + ps.partition_columns + FROM #partition_stats ps + GROUP BY + ps.database_id, + ps.object_id, + ps.index_id, + ps.built_on, + ps.partition_function_name, + ps.partition_columns + ) AS ps + ON ia.database_id = ps.database_id + AND ia.object_id = ps.object_id + AND ia.index_id = ps.index_id + JOIN #compression_eligibility AS ce + ON ia.database_id = ce.database_id + AND ia.object_id = ce.object_id + AND ia.index_id = ce.index_id + WHERE ia.action IN (N'MERGE INCLUDES', N'MAKE UNIQUE') + AND ce.can_compress = 1 + /* Only create merge scripts for the indexes that should remain after merging */ + AND ia.target_index_name IS NULL + OPTION(RECOMPILE); + + /* Debug which indexes are getting MERGE scripts */ + IF @debug = 1 + BEGIN + RAISERROR('Indexes getting MERGE scripts:', 0, 0) WITH NOWAIT; + SELECT + ia.index_name, + ia.action, + ia.consolidation_rule, + ia.target_index_name, + script_type = 'WILL GET MERGE SCRIPT', + ia.included_columns + FROM #index_analysis AS ia + JOIN #compression_eligibility AS ce + ON ia.database_id = ce.database_id + AND ia.object_id = ce.object_id + AND ia.index_id = ce.index_id + WHERE ia.action IN (N'MERGE INCLUDES', N'MAKE UNIQUE') + AND ce.can_compress = 1 + AND ia.target_index_name IS NULL + ORDER BY ia.index_name + OPTION(RECOMPILE); + END; + + /* Insert disable scripts for unneeded indexes */ + IF @debug = 1 + BEGIN + RAISERROR('Generating #index_cleanup_results insert, DISABLE', 0, 0) WITH NOWAIT; + + /* Debug for indexes that should get DISABLE scripts */ + RAISERROR('Indexes that should get DISABLE scripts:', 0, 0) WITH NOWAIT; + SELECT + ia.index_name, + ia.consolidation_rule, + ia.action, + ia.target_index_name, + ia.is_unique, + ia.index_priority, + is_unique_constraint = + CASE WHEN EXISTS ( + SELECT 1 + FROM #index_details AS id + WHERE id.database_id = ia.database_id + AND id.object_id = ia.object_id + AND id.index_id = ia.index_id + AND id.is_unique_constraint = 1 + ) THEN 'YES' ELSE 'NO' END, + make_unique_target = + CASE WHEN EXISTS ( + SELECT 1 + FROM #index_analysis AS ia_make + WHERE ia_make.database_id = ia.database_id + AND ia_make.object_id = ia.object_id + AND ia_make.action = 'MAKE UNIQUE' + AND ia_make.target_index_name = ia.index_name + ) THEN 'YES' ELSE 'NO' END, + will_get_script = + CASE WHEN ia.action = 'DISABLE' AND NOT EXISTS ( + SELECT 1 + FROM #index_details AS id_uc + WHERE id_uc.database_id = ia.database_id + AND id_uc.object_id = ia.object_id + AND id_uc.index_id = ia.index_id + AND id_uc.is_unique_constraint = 1 + ) THEN 'YES' ELSE 'NO' END + FROM #index_analysis AS ia + WHERE ia.index_name LIKE 'ix_filtered_%' OR ia.index_name LIKE 'ix_desc_%' + ORDER BY ia.index_name; + + /* Debug for all indexes marked with action = DISABLE */ + RAISERROR('All indexes with action = DISABLE:', 0, 0) WITH NOWAIT; + SELECT + ia.index_name, + ia.consolidation_rule, + ia.action, + ia.target_index_name + FROM #index_analysis AS ia + WHERE ia.action = 'DISABLE' + ORDER BY ia.index_name; + END; + + INSERT INTO + #index_cleanup_results + ( + result_type, + sort_order, + database_name, + schema_name, + table_name, + index_name, + script_type, + consolidation_rule, + script, + additional_info, + target_index_name, + superseded_info, + original_index_definition, + index_size_gb, + index_rows, + index_reads, + index_writes + ) + SELECT DISTINCT + result_type = 'DISABLE', + /* Sort duplicate/subset indexes first (20), then unused indexes last (25) */ + sort_order = + CASE + WHEN ia.consolidation_rule LIKE 'Unused Index%' THEN 25 + ELSE 20 + END, + ia.database_name, + ia.schema_name, + ia.table_name, + ia.index_name, + script_type = 'DISABLE SCRIPT', + ia.consolidation_rule, + script = + /* Use regular DISABLE syntax for indexes */ + N'ALTER INDEX ' + + QUOTENAME(ia.index_name) + + N' ON ' + + QUOTENAME(ia.database_name) + + N'.' + + QUOTENAME(ia.schema_name) + + N'.' + + QUOTENAME(ia.table_name) + + N' DISABLE;', + CASE + WHEN ia.consolidation_rule = 'Key Subset' + THEN N'This index is superseded by a wider index: ' + ISNULL(ia.target_index_name, N'(unknown)') + WHEN ia.consolidation_rule = 'Exact Duplicate' + THEN N'This index is an exact duplicate of: ' + ISNULL(ia.target_index_name, N'(unknown)') + WHEN ia.consolidation_rule = 'Key Duplicate' + THEN N'This index has the same keys as: ' + ISNULL(ia.target_index_name, N'(unknown)') + WHEN ia.consolidation_rule LIKE 'Unused Index%' + THEN ia.consolidation_rule + WHEN ia.action = N'DISABLE' + THEN N'This index is redundant and will be disabled' + ELSE N'This index is redundant' + END, + ia.target_index_name, /* Include the target index name */ + superseded_info = NULL, /* Don't need superseded_by info for disabled indexes */ + /* Original index definition for validation */ + ia.original_index_definition, + ps.total_space_gb, + ps.total_rows, + index_reads = + (id.user_seeks + id.user_scans + id.user_lookups), + id.user_updates + FROM #index_analysis AS ia + LEFT JOIN #partition_stats AS ps + ON ia.database_id = ps.database_id + AND ia.object_id = ps.object_id + AND ia.index_id = ps.index_id + LEFT JOIN #index_details AS id + ON id.database_id = ia.database_id + AND id.object_id = ia.object_id + AND id.index_id = ia.index_id + AND id.is_included_column = 0 /* Get only one row per index */ + AND id.key_ordinal > 0 + WHERE ia.action = N'DISABLE' + /* Exclude unique constraints - they are handled by DISABLE CONSTRAINT scripts */ + AND NOT EXISTS + ( + SELECT + 1/0 + FROM #index_details AS id_uc + WHERE id_uc.database_id = ia.database_id + AND id_uc.object_id = ia.object_id + AND id_uc.index_id = ia.index_id + AND id_uc.is_unique_constraint = 1 + ) + /* Also exclude any index that is also going to be made unique in rule 7.5 */ + AND NOT EXISTS + ( + SELECT + 1/0 + FROM #index_analysis AS ia_unique + WHERE ia_unique.database_id = ia.database_id + AND ia_unique.object_id = ia.object_id + AND ia_unique.index_name = ia.index_name + AND ia_unique.action = N'MAKE UNIQUE' + ) + OPTION(RECOMPILE); + + /* Insert compression scripts for remaining indexes */ + IF @debug = 1 + BEGIN + RAISERROR('Generating #index_cleanup_results insert, COMPRESS', 0, 0) WITH NOWAIT; + END; + + INSERT INTO + #index_cleanup_results + ( + result_type, + sort_order, + database_name, + schema_name, + table_name, + index_name, + script_type, + script, + additional_info, + target_index_name, + superseded_info, + original_index_definition, + index_size_gb, + index_rows, + index_reads, + index_writes + ) + SELECT DISTINCT + result_type = 'COMPRESS', + sort_order = 40, + ia.database_name, + ia.schema_name, + ia.table_name, + ia.index_name, + script_type = 'COMPRESSION SCRIPT', + script = + N'ALTER INDEX ' + + QUOTENAME(ia.index_name) + + N' ON ' + + QUOTENAME(ia.database_name) + + N'.' + + QUOTENAME(ia.schema_name) + + N'.' + + QUOTENAME(ia.table_name) + + CASE + WHEN ps.partition_function_name IS NOT NULL + THEN N' REBUILD PARTITION = ALL' + ELSE N' REBUILD' + END + + N' WITH (FILLFACTOR = 100, SORT_IN_TEMPDB = ON, ONLINE = ' + + CASE + WHEN @online = 1 + THEN N'ON' + ELSE N'OFF' + END + + CASE + WHEN ce.can_compress = 1 + THEN ', DATA_COMPRESSION = PAGE' + ELSE N'' + END + + N')', + additional_info = N'Compression type: All Partitions', + superseded_info = NULL, /* No target index for compression scripts */ + ia.superseded_by, /* Include superseded_by info for compression scripts */ + /* Original index definition for validation */ + ia.original_index_definition, + ps_full.total_space_gb, + ps_full.total_rows, + index_reads = + (id.user_seeks + id.user_scans + id.user_lookups), + id.user_updates + FROM #index_analysis AS ia + LEFT JOIN + ( + /* Get the partition info for each index */ + SELECT + ps.database_id, + ps.object_id, + ps.index_id, + ps.built_on, + ps.partition_function_name, + ps.partition_columns + FROM #partition_stats ps + GROUP BY + ps.database_id, + ps.object_id, + ps.index_id, + ps.built_on, + ps.partition_function_name, + ps.partition_columns + ) + AS ps + ON ia.database_id = ps.database_id + AND ia.object_id = ps.object_id + AND ia.index_id = ps.index_id + LEFT JOIN #partition_stats AS ps_full + ON ia.database_id = ps_full.database_id + AND ia.object_id = ps_full.object_id + AND ia.index_id = ps_full.index_id + LEFT JOIN #index_details AS id + ON id.database_id = ia.database_id + AND id.object_id = ia.object_id + AND id.index_id = ia.index_id + AND id.is_included_column = 0 /* Get only one row per index */ + AND id.key_ordinal > 0 + JOIN #compression_eligibility AS ce + ON ia.database_id = ce.database_id + AND ia.object_id = ce.object_id + AND ia.index_id = ce.index_id + WHERE + /* Indexes that are not being disabled or merged */ + (ia.action IS NULL OR ia.action = N'KEEP') + /* Only indexes eligible for compression */ + AND ce.can_compress = 1 + OPTION(RECOMPILE); + + /* Insert disable scripts for unique constraints */ + IF @debug = 1 + BEGIN + RAISERROR('Generating #index_cleanup_results insert, CONSTRAINT', 0, 0) WITH NOWAIT; + END; + + /* Add code to insert KEPT indexes into the results - THESE WERE MISSING! */ + IF @debug = 1 + BEGIN + RAISERROR('Generating #index_cleanup_results insert, KEPT', 0, 0) WITH NOWAIT; + END; + + /* Insert KEPT indexes into results */ + INSERT INTO + #index_cleanup_results + ( + result_type, + sort_order, + database_name, + schema_name, + table_name, + index_name, + script_type, + consolidation_rule, + additional_info, + script, + original_index_definition, + index_size_gb, + index_rows, + index_reads, + index_writes + ) + SELECT DISTINCT + result_type = 'KEPT', + sort_order = 95, /* Put kept indexes at the end */ + ia.database_name, + ia.schema_name, + ia.table_name, + ia.index_name, + script_type = NULL, + ia.consolidation_rule, + additional_info = + CASE + WHEN ia.consolidation_rule IS NOT NULL + THEN 'This index is being kept' + ELSE NULL + END, + script = NULL, /* No script for kept indexes */ + /* Original index definition for validation */ + ia.original_index_definition, + ps.total_space_gb, + ps.total_rows, + index_reads = + (id.user_seeks + id.user_scans + id.user_lookups), + id.user_updates + FROM #index_analysis AS ia + LEFT JOIN #partition_stats AS ps + ON ia.database_id = ps.database_id + AND ia.object_id = ps.object_id + AND ia.index_id = ps.index_id + LEFT JOIN #index_details AS id + ON id.database_id = ia.database_id + AND id.object_id = ia.object_id + AND id.index_id = ia.index_id + AND id.is_included_column = 0 /* Get only one row per index */ + AND id.key_ordinal > 0 + /* Check that this index is not already in the results */ + WHERE NOT EXISTS ( + SELECT 1 FROM #index_cleanup_results AS ir + WHERE ir.database_name = ia.database_name + AND ir.schema_name = ia.schema_name + AND ir.table_name = ia.table_name + AND ir.index_name = ia.index_name + ) + /* And include only indexes that should be kept */ + AND ( + /* Include indexes marked KEEP */ + (ia.action = 'KEEP') + /* And all indexes we haven't determined an action for (not disable, merge, etc.) */ + OR (ia.action IS NULL AND ia.index_id > 0) + ) + OPTION(RECOMPILE); + + INSERT INTO + #index_cleanup_results + ( + result_type, + sort_order, + database_name, + schema_name, + table_name, + index_name, + script_type, + additional_info, + script, + original_index_definition, + index_size_gb, + index_rows, + index_reads, + index_writes + ) + SELECT DISTINCT + result_type = 'CONSTRAINT', + sort_order = 30, + ia_uc.database_name, + ia_uc.schema_name, + ia_uc.table_name, + ia_uc.index_name, + script_type = 'DISABLE CONSTRAINT SCRIPT', + additional_info = + N'This constraint is being replaced by: ' + + ISNULL(ia_uc.target_index_name, N'(unknown)'), + script = + N'ALTER TABLE ' + + QUOTENAME(ia_uc.database_name) + + N'.' + + QUOTENAME(ia_uc.schema_name) + + N'.' + + QUOTENAME(ia_uc.table_name) + + N' NOCHECK CONSTRAINT ' + + QUOTENAME(ia_uc.index_name) + + N';', + /* Original index definition for validation */ + original_index_definition = ia_uc.original_index_definition, + ps.total_space_gb, + ps.total_rows, + index_reads = + (id2.user_seeks + id2.user_scans + id2.user_lookups), + id2.user_updates + FROM #index_analysis AS ia_uc + JOIN #index_details AS id + ON id.database_id = ia_uc.database_id + AND id.object_id = ia_uc.object_id + AND id.index_id = ia_uc.index_id + AND id.is_unique_constraint = 1 + LEFT JOIN #index_details AS id2 + ON id2.database_id = ia_uc.database_id + AND id2.object_id = ia_uc.object_id + AND id2.index_id = ia_uc.index_id + AND id2.is_included_column = 0 /* Get only one row per index */ + AND id2.key_ordinal > 0 + LEFT JOIN #partition_stats AS ps + ON ia_uc.database_id = ps.database_id + AND ia_uc.object_id = ps.object_id + AND ia_uc.index_id = ps.index_id + WHERE + /* Only constraints that are marked for disabling */ + ia_uc.action = N'DISABLE' + /* That have consolidation_rule of 'Unique Constraint Replacement' */ + AND ia_uc.consolidation_rule = 'Unique Constraint Replacement' + OPTION(RECOMPILE); + + /* Insert per-partition compression scripts */ + IF @debug = 1 + BEGIN + RAISERROR('Generating #index_cleanup_results insert, COMPRESS_PARTITION', 0, 0) WITH NOWAIT; + END; + + INSERT INTO + #index_cleanup_results + ( + result_type, + sort_order, + database_name, + schema_name, + table_name, + index_name, + script_type, + script, + additional_info, + target_index_name, + superseded_info, + original_index_definition, + index_size_gb, + index_rows, + index_reads, + index_writes + ) + SELECT DISTINCT + result_type = 'COMPRESS_PARTITION', + sort_order = 50, + ia.database_name, + ia.schema_name, + ia.table_name, + ia.index_name, + script_type = 'PARTITION COMPRESSION SCRIPT', + script = + N'ALTER INDEX ' + + QUOTENAME(ia.index_name) + + N' ON ' + + QUOTENAME(ia.database_name) + + N'.' + + QUOTENAME(ia.schema_name) + + N'.' + + QUOTENAME(ia.table_name) + + N' REBUILD PARTITION = ' + + CONVERT + ( + nvarchar(20), + ps.partition_number + ) + + N' WITH (FILLFACTOR = 100, SORT_IN_TEMPDB = ON, ONLINE = ' + + CASE + WHEN @online = 1 + THEN N'ON' + ELSE N'OFF' + END + + CASE + WHEN ce.can_compress = 1 + THEN ', DATA_COMPRESSION = PAGE' + ELSE N'' + END + + N')', + N'Compression type: Per Partition | Partition: ' + + CONVERT + ( + nvarchar(20), + ps.partition_number + ) + + N' | Rows: ' + + CONVERT + ( + nvarchar(20), + ps.total_rows + ) + + N' | Size: ' + + CONVERT + ( + nvarchar(20), + CONVERT + ( + decimal(10,4), + ps.total_space_gb + ) + ) + + N' GB', + target_index_name = NULL, + superseded_info = NULL, + ia.original_index_definition, + ps.total_space_gb, + ps.total_rows, + index_reads = + (id.user_seeks + id.user_scans + id.user_lookups), + id.user_updates + FROM #index_analysis AS ia + JOIN #partition_stats AS ps + ON ia.database_id = ps.database_id + AND ia.object_id = ps.object_id + AND ia.index_id = ps.index_id + LEFT JOIN #index_details AS id + ON id.database_id = ia.database_id + AND id.object_id = ia.object_id + AND id.index_id = ia.index_id + AND id.is_included_column = 0 /* Get only one row per index */ + AND id.key_ordinal > 0 + JOIN #compression_eligibility AS ce + ON ia.database_id = ce.database_id + AND ia.object_id = ce.object_id + AND ia.index_id = ce.index_id + WHERE + /* Only partitioned indexes */ + ps.partition_function_name IS NOT NULL + /* Indexes that are not being disabled or merged */ + AND (ia.action IS NULL OR ia.action = N'KEEP') + /* Only indexes eligible for compression */ + AND ce.can_compress = 1 + OPTION(RECOMPILE); + + /* Insert compression ineligible info */ + IF @debug = 1 + BEGIN + RAISERROR('Generating #index_cleanup_results insert, INELIGIBLE', 0, 0) WITH NOWAIT; + END; + + INSERT INTO + #index_cleanup_results + WITH + (TABLOCK) + ( + result_type, + sort_order, + database_name, + schema_name, + table_name, + index_name, + script_type, + additional_info, + original_index_definition, + index_size_gb, + index_rows, + index_reads, + index_writes + ) + SELECT DISTINCT + result_type = 'INELIGIBLE', + sort_order = 90, + ce.database_name, + ce.schema_name, + ce.table_name, + ce.index_name, + script_type = 'INELIGIBLE FOR COMPRESSION', + ce.reason, + /* Original index definition for validation */ + original_index_definition = + ( + SELECT TOP (1) + ia.original_index_definition + FROM #index_analysis AS ia + WHERE ia.database_id = ce.database_id + AND ia.object_id = ce.object_id + AND ia.index_id = ce.index_id + ), + ps.total_space_gb, + ps.total_rows, + index_reads = + (id.user_seeks + id.user_scans + id.user_lookups), + id.user_updates + FROM #compression_eligibility AS ce + LEFT JOIN #partition_stats AS ps + ON ce.database_id = ps.database_id + AND ce.object_id = ps.object_id + AND ce.index_id = ps.index_id + LEFT JOIN #index_details AS id + ON id.database_id = ce.database_id + AND id.object_id = ce.object_id + AND id.index_id = ce.index_id + AND id.is_included_column = 0 /* Get only one row per index */ + AND id.key_ordinal > 0 + WHERE ce.can_compress = 0 + OPTION(RECOMPILE); + + + /* Insert indexes identified for manual review */ + IF @debug = 1 + BEGIN + RAISERROR('Generating #index_cleanup_results insert, REVIEW', 0, 0) WITH NOWAIT; + END; + + INSERT INTO + #index_cleanup_results + WITH + (TABLOCK) + ( + result_type, + sort_order, + database_name, + schema_name, + table_name, + index_name, + script_type, + consolidation_rule, + target_index_name, + additional_info, + original_index_definition, + index_size_gb, + index_rows, + index_reads, + index_writes + ) + SELECT DISTINCT + result_type = 'REVIEW', + sort_order = 93, /* Just before KEPT indexes */ + ia.database_name, + ia.schema_name, + ia.table_name, + ia.index_name, + script_type = 'NEEDS REVIEW', + ia.consolidation_rule, + ia.target_index_name, + additional_info = + CASE + WHEN ia.consolidation_rule = 'Same Keys Different Order' + THEN N'This index has the same key columns as ' + ISNULL(ia.target_index_name, N'(unknown)') + + N' but in a different order. May be redundant depending on query patterns.' + ELSE N'This index needs manual review' + END, + /* Original index definition for validation */ + ia.original_index_definition, + ps.total_space_gb, + ps.total_rows, + index_reads = + (id.user_seeks + id.user_scans + id.user_lookups), + id.user_updates + FROM #index_analysis AS ia + LEFT JOIN #partition_stats AS ps + ON ia.database_id = ps.database_id + AND ia.object_id = ps.object_id + AND ia.index_id = ps.index_id + LEFT JOIN #index_details AS id + ON id.database_id = ia.database_id + AND id.object_id = ia.object_id + AND id.index_id = ia.index_id + AND id.is_included_column = 0 /* Get only one row per index */ + AND id.key_ordinal > 0 + WHERE ia.action = N'REVIEW' + OPTION(RECOMPILE); + + + /* Insert indexes that are being kept (superset indexes and others) */ + IF @debug = 1 + BEGIN + RAISERROR('Generating #index_cleanup_results insert, KEEP', 0, 0) WITH NOWAIT; + END; + + INSERT INTO + #index_cleanup_results + WITH + (TABLOCK) + ( + result_type, + sort_order, + database_name, + schema_name, + table_name, + index_name, + script_type, + consolidation_rule, + superseded_info, + additional_info, + original_index_definition, + index_size_gb, + index_rows, + index_reads, + index_writes + ) + SELECT DISTINCT + result_type = 'KEEP', + sort_order = 95, /* Just before END OF REPORT at 99 */ + ia.database_name, + ia.schema_name, + ia.table_name, + ia.index_name, + script_type = 'KEPT', + ia.consolidation_rule, + ia.superseded_by, + additional_info = + CASE + WHEN ia.superseded_by IS NOT NULL + THEN 'This index supersedes other indexes and already has all needed columns' + WHEN ia.action = N'KEEP' + THEN 'This index is being kept' + ELSE NULL + END, + /* Original index definition for validation */ + ia.original_index_definition, + ps.total_space_gb, + ps.total_rows, + index_reads = + (id.user_seeks + id.user_scans + id.user_lookups), + id.user_updates + FROM #index_analysis AS ia + LEFT JOIN #partition_stats AS ps + ON ia.database_id = ps.database_id + AND ia.object_id = ps.object_id + AND ia.index_id = ps.index_id + LEFT JOIN #index_details AS id + ON id.database_id = ia.database_id + AND id.object_id = ia.object_id + AND id.index_id = ia.index_id + AND id.is_included_column = 0 /* Get only one row per index */ + AND id.key_ordinal > 0 + WHERE ia.action = N'KEEP' + OR + ( + ia.action IS NULL + AND ia.consolidation_rule IS NULL + ) + OPTION(RECOMPILE); + + /* Insert database-level summaries */ + IF @debug = 1 + BEGIN + RAISERROR('Generating #index_reporting_stats insert, DATABASE', 0, 0) WITH NOWAIT; + END; + + INSERT INTO + #index_reporting_stats + ( + summary_level, + database_name, + index_count, + total_size_gb, + total_rows, + indexes_to_merge, + unused_indexes, + unused_size_gb, + total_reads, + total_writes, + user_seeks, + user_scans, + user_lookups, + user_updates, + range_scan_count, + singleton_lookup_count, + row_lock_count, + row_lock_wait_count, + row_lock_wait_in_ms, + page_lock_count, + page_lock_wait_count, + page_lock_wait_in_ms, + page_latch_wait_count, + page_latch_wait_in_ms, + page_io_latch_wait_count, + page_io_latch_wait_in_ms, + forwarded_fetch_count, + leaf_insert_count, + leaf_update_count, + leaf_delete_count + ) + SELECT + summary_level = + 'DATABASE', + ps.database_name, + index_count = + COUNT_BIG(DISTINCT CONCAT(ps.object_id, N'.', ps.index_id)), + total_size_gb = SUM(ps.total_space_gb), + /* Use a simple aggregation to avoid double-counting */ + /* Get actual row count by grabbing the real row count from clustered index/heap per table */ + total_rows = SUM(DISTINCT d.actual_rows), + indexes_to_merge = + ( + SELECT + COUNT_BIG(*) + FROM #index_analysis AS ia + WHERE ia.action IN (N'MERGE INCLUDES', N'MAKE UNIQUE') + AND ia.database_id = ps.database_id + ), + /* Use count from analysis to keep consistent with SUMMARY level */ + unused_indexes = + ( + SELECT + COUNT_BIG(*) + FROM #index_analysis AS ia + WHERE ia.action = N'DISABLE' + AND ia.database_id = ps.database_id + ), + unused_size_gb = + SUM + ( + CASE + WHEN id.user_seeks + id.user_scans + id.user_lookups = 0 + THEN ps.total_space_gb + ELSE 0 + END + ), + total_reads = SUM(id.user_seeks + id.user_scans + id.user_lookups), + total_writes = SUM(id.user_updates), + user_seeks = SUM(id.user_seeks), + user_scans = SUM(id.user_scans), + user_lookups = SUM(id.user_lookups), + user_updates = SUM(id.user_updates), + range_scan_count = SUM(os.range_scan_count), + singleton_lookup_count = SUM(os.singleton_lookup_count), + row_lock_count = SUM(os.row_lock_count), + row_lock_wait_count = SUM(os.row_lock_wait_count), + row_lock_wait_in_ms = SUM(os.row_lock_wait_in_ms), + page_lock_count = SUM(os.page_lock_count), + page_lock_wait_count = SUM(os.page_lock_wait_count), + page_lock_wait_in_ms = SUM(os.page_lock_wait_in_ms), + page_latch_wait_count = SUM(os.page_latch_wait_count), + page_latch_wait_in_ms = SUM(os.page_latch_wait_in_ms), + page_io_latch_wait_count = SUM(os.page_io_latch_wait_count), + page_io_latch_wait_in_ms = SUM(os.page_io_latch_wait_in_ms), + forwarded_fetch_count = SUM(os.forwarded_fetch_count), + leaf_insert_count = SUM(os.leaf_insert_count), + leaf_update_count = SUM(os.leaf_update_count), + leaf_delete_count = SUM(os.leaf_delete_count) + FROM #partition_stats AS ps + LEFT JOIN #index_details AS id + ON id.database_id = ps.database_id + AND id.object_id = ps.object_id + AND id.index_id = ps.index_id + AND id.is_included_column = 0 + AND id.key_ordinal > 0 + LEFT JOIN #operational_stats AS os + ON os.database_id = ps.database_id + AND os.object_id = ps.object_id + AND os.index_id = ps.index_id + OUTER APPLY + ( + /* Get actual row count per table using MAX from clustered index/heap */ + SELECT + actual_rows = + MAX + ( + CASE + WHEN ps2.index_id IN (0, 1) + THEN ps2.total_rows + ELSE 0 + END + ) + FROM #partition_stats AS ps2 + WHERE ps2.database_id = ps.database_id + AND ps2.object_id = ps.object_id + AND ps2.index_id IN (0, 1) + GROUP BY + ps2.object_id + ) AS d + GROUP BY + ps.database_name, + ps.database_id + OPTION(RECOMPILE); + + /* Insert table-level summaries */ + IF @debug = 1 + BEGIN + RAISERROR('Generating #index_reporting_stats insert, TABLE', 0, 0) WITH NOWAIT; + END; + + /* No need for a temporary table - we'll use a simpler approach */ + + INSERT INTO + #index_reporting_stats + ( + summary_level, + database_name, + schema_name, + table_name, + index_count, + total_size_gb, + total_rows, + indexes_to_merge, + unused_indexes, + unused_size_gb, + total_reads, + total_writes, + user_seeks, + user_scans, + user_lookups, + user_updates, + range_scan_count, + singleton_lookup_count, + row_lock_count, + row_lock_wait_count, + row_lock_wait_in_ms, + page_lock_count, + page_lock_wait_count, + page_lock_wait_in_ms, + page_latch_wait_count, + page_latch_wait_in_ms, + page_io_latch_wait_count, + page_io_latch_wait_in_ms, + forwarded_fetch_count, + leaf_insert_count, + leaf_update_count, + leaf_delete_count + ) + SELECT + summary_level = 'TABLE', + ps.database_name, + ps.schema_name, + ps.table_name, + index_count = COUNT_BIG(DISTINCT ps.index_id), + total_size_gb = SUM(ps.total_space_gb), + /* Use MAX to get the row count from the clustered index or heap */ + total_rows = + MAX + ( + CASE + WHEN ps.index_id IN (0, 1) + THEN ps.total_rows + ELSE 0 + END + ), + indexes_to_merge = + ( + SELECT + COUNT_BIG(*) + FROM #index_analysis AS ia + WHERE ia.action IN (N'MERGE INCLUDES', N'MAKE UNIQUE') + AND ia.database_id = ps.database_id + AND ia.schema_id = ps.schema_id + AND ia.object_id = ps.object_id + ), + /* Use count from analysis to keep consistent with SUMMARY level */ + unused_indexes = + ( + SELECT + COUNT_BIG(*) + FROM #index_analysis AS ia + WHERE ia.action = N'DISABLE' + AND ia.database_id = ps.database_id + AND ia.schema_id = ps.schema_id + AND ia.object_id = ps.object_id + ), + unused_size_gb = + SUM + ( + CASE + WHEN id.user_seeks + id.user_scans + id.user_lookups = 0 + THEN ps.total_space_gb + ELSE 0 + END + ), + total_reads = SUM(id.user_seeks + id.user_scans + id.user_lookups), + total_writes = SUM(id.user_updates), + user_seeks = SUM(id.user_seeks), + user_scans = SUM(id.user_scans), + user_lookups = SUM(id.user_lookups), + user_updates = SUM(id.user_updates), + range_scan_count = SUM(os.range_scan_count), + singleton_lookup_count = SUM(os.singleton_lookup_count), + row_lock_count = SUM(os.row_lock_count), + row_lock_wait_count = SUM(os.row_lock_wait_count), + row_lock_wait_in_ms = SUM(os.row_lock_wait_in_ms), + page_lock_count = SUM(os.page_lock_count), + page_lock_wait_count = SUM(os.page_lock_wait_count), + page_lock_wait_in_ms = SUM(os.page_lock_wait_in_ms), + page_latch_wait_count = SUM(os.page_latch_wait_count), + page_latch_wait_in_ms = SUM(os.page_latch_wait_in_ms), + page_io_latch_wait_count = SUM(os.page_io_latch_wait_count), + page_io_latch_wait_in_ms = SUM(os.page_io_latch_wait_in_ms), + forwarded_fetch_count = SUM(os.forwarded_fetch_count), + leaf_insert_count = SUM(os.leaf_insert_count), + leaf_update_count = SUM(os.leaf_update_count), + leaf_delete_count = SUM(os.leaf_delete_count) + FROM #partition_stats AS ps + LEFT JOIN #index_details AS id + ON id.database_id = ps.database_id + AND id.object_id = ps.object_id + AND id.index_id = ps.index_id + AND id.is_included_column = 0 + AND id.key_ordinal > 0 + LEFT JOIN #operational_stats AS os + ON os.database_id = ps.database_id + AND os.object_id = ps.object_id + AND os.index_id = ps.index_id + GROUP BY + ps.database_name, + ps.database_id, + ps.schema_name, + ps.schema_id, + ps.table_name, + ps.object_id + OPTION(RECOMPILE); + + /* We're not doing index-level summaries - focusing on database and table level reports */ + + /* + Return the consolidated results in a single result set + Results are ordered by: + 1. Summary information (overall stats, savings estimates) + 2. Merge scripts (includes merges and unique conversions) - sort_order 5 + 3. Disable scripts (for redundant indexes) - sort_order 20 + 4. Constraint scripts (for unique constraints to disable) + 5. Compression scripts (for tables eligible for compression) + 6. Partition-specific compression scripts + 7. Ineligible objects (tables that can't be compressed) + 8. Kept indexes - sort_order 95 + + Note: Merge target scripts are sorted higher in the results (sort_order 5) + so that new merged indexes are created before subset indexes are disabled. + + Within each category, indexes are sorted by size and impact for better prioritization. + */ + IF @debug = 1 + BEGIN + RAISERROR('Generating #index_cleanup_results, RESULTS', 0, 0) WITH NOWAIT; + END; + + SELECT + /* First, show the information needed to understand the script */ + script_type = CASE WHEN ir.result_type = 'KEPT' AND ir.script_type IS NULL THEN 'KEPT' ELSE ir.script_type END, + ir.additional_info, + /* Then show identifying information for the index */ + ir.database_name, + ir.schema_name, + ir.table_name, + ir.index_name, + /* Then show relationship information */ + ir.consolidation_rule, + ir.target_index_name, + /* Include superseded_by info for winning indexes */ + superseded_info = + CASE + WHEN ia.superseded_by IS NOT NULL + THEN ia.superseded_by + ELSE ir.superseded_info + END, + /* Add size and usage metrics */ + index_size_gb = + CASE + WHEN ir.result_type = 'SUMMARY' + THEN '0.0000' + ELSE FORMAT(ISNULL(ir.index_size_gb, 0), 'N4') + END, + index_rows = + CASE + WHEN ir.result_type = 'SUMMARY' + THEN '0' + ELSE FORMAT(ISNULL(ir.index_rows, 0), 'N0') + END, + index_reads = + CASE + WHEN ir.result_type = 'SUMMARY' + THEN '0' + ELSE FORMAT(ISNULL(ir.index_reads, 0), 'N0') + END, + index_writes = + CASE + WHEN ir.result_type = 'SUMMARY' + THEN '0' + ELSE FORMAT(ISNULL(ir.index_writes, 0), 'N0') + END, + ia.original_index_definition, + /* Finally show the actual script */ + ir.script + FROM + ( + /* Use a subquery with ROW_NUMBER to ensure we only get one row per index */ + SELECT *, + ROW_NUMBER() OVER( + PARTITION BY database_name, schema_name, table_name, index_name + ORDER BY result_type DESC /* Prefer non-NULL result types */ + ) AS rn + FROM #index_cleanup_results + ) AS ir + LEFT JOIN #index_analysis AS ia + ON ir.database_name = ia.database_name + AND ir.schema_name = ia.schema_name + AND ir.table_name = ia.table_name + AND ir.index_name = ia.index_name + WHERE ir.rn = 1 /* Take only the first row for each index */ + ORDER BY + ir.sort_order, + ir.database_name, + /* Within each sort_order group, prioritize by size and usage */ + CASE + /* For SUMMARY, keep the original order */ + WHEN ir.result_type = 'SUMMARY' + THEN 0 + /* For script categories, order by size and impact */ + ELSE ISNULL(ir.index_size_gb, 0) + END DESC, + CASE + /* For SUMMARY, keep the original order */ + WHEN ir.result_type = 'SUMMARY' + THEN 0 + /* For script categories, consider rows as secondary sort */ + ELSE ISNULL(ir.index_rows, 0) + END DESC, + /* Then by database, schema, table, index name for consistent ordering */ + ir.schema_name, + ir.table_name, + ir.index_name + OPTION(RECOMPILE); + + /* Insert overall summary information */ + IF @debug = 1 + BEGIN + RAISERROR('Generating #index_reporting_stats insert, SUMMARY', 0, 0) WITH NOWAIT; + END; + + INSERT INTO + #index_reporting_stats + WITH + (TABLOCK) + ( + summary_level, + server_uptime_days, + uptime_warning, + tables_analyzed, + index_count, + indexes_to_disable, + indexes_to_merge, + avg_indexes_per_table, + space_saved_gb, + compression_min_savings_gb, + compression_max_savings_gb, + total_min_savings_gb, + total_max_savings_gb, + total_rows + ) + SELECT + summary_level = 'SUMMARY', + server_uptime_days = @uptime_days, + uptime_warning = @uptime_warning, + tables_analyzed = + COUNT_BIG(DISTINCT CONCAT(ia.database_id, N'.', ia.schema_id, N'.', ia.object_id)), + index_count = + COUNT_BIG(*), + indexes_to_disable = + SUM + ( + CASE + WHEN ia.action = N'DISABLE' + THEN 1 + ELSE 0 + END + ), + indexes_to_merge = + SUM + ( + CASE + WHEN ia.action IN (N'MERGE INCLUDES', N'MAKE UNIQUE') + THEN 1 + ELSE 0 + END + ), + avg_indexes_per_table = + COUNT_BIG(*) * 1.0 / + NULLIF + ( + COUNT_BIG(DISTINCT CONCAT(ia.database_id, N'.', ia.schema_id, N'.', ia.object_id)), + 0 + ), + /* Space savings from cleanup */ + space_saved_gb = + SUM + ( + CASE + WHEN ia.action IN (N'DISABLE', N'MERGE INCLUDES', N'MAKE UNIQUE') + THEN ps.total_space_gb + ELSE 0 + END + ), + /* Conservative compression savings estimate (20%) */ + compression_min_savings_gb = + SUM + ( + CASE + WHEN (ia.action IS NULL OR ia.action = N'KEEP') + AND ce.can_compress = 1 + THEN ps.total_space_gb * 0.20 + ELSE 0 + END + ), + /* Optimistic compression savings estimate (60%) */ + compression_max_savings_gb = + SUM + ( + CASE + WHEN (ia.action IS NULL OR ia.action = N'KEEP') + AND ce.can_compress = 1 + THEN ps.total_space_gb * 0.60 + ELSE 0 + END + ), + /* Total conservative savings */ + total_min_savings_gb = + SUM + ( + CASE + WHEN ia.action IN (N'DISABLE', N'MERGE INCLUDES', N'MAKE UNIQUE') + THEN ps.total_space_gb + WHEN (ia.action IS NULL OR ia.action = N'KEEP') + AND ce.can_compress = 1 + THEN ps.total_space_gb * 0.20 + ELSE 0 + END + ), + /* Total optimistic savings */ + total_max_savings_gb = + SUM + ( + CASE + WHEN ia.action IN (N'DISABLE', N'MERGE INCLUDES', N'MAKE UNIQUE') + THEN ps.total_space_gb + WHEN (ia.action IS NULL OR ia.action = N'KEEP') + AND ce.can_compress = 1 + THEN ps.total_space_gb * 0.60 + ELSE 0 + END + ), + /* Get total rows from database unique tables */ + total_rows = + ( + SELECT + SUM(t.row_count) + FROM + ( + SELECT + ps_distinct.object_id, + row_count = + MAX + ( + CASE + WHEN ps_distinct.index_id IN (0, 1) + THEN ps_distinct.total_rows + ELSE 0 + END + ) + FROM #partition_stats AS ps_distinct + WHERE ps_distinct.index_id IN (0, 1) + GROUP BY + ps_distinct.object_id + ) AS t + ) + FROM #index_analysis AS ia + LEFT JOIN #partition_stats AS ps + ON ia.database_id = ps.database_id + AND ia.object_id = ps.object_id + AND ia.index_id = ps.index_id + LEFT JOIN #compression_eligibility AS ce + ON ia.database_id = ce.database_id + AND ia.object_id = ce.object_id + AND ia.index_id = ce.index_id + OPTION(RECOMPILE); + + /* Return enhanced database impact summaries */ + IF @debug = 1 + BEGIN + RAISERROR('Generating enhanced summary reports', 0, 0) WITH NOWAIT; + END; + + /* + This section now REPLACES the existing summary view rather than supplementing it + We'll modify the existing query below rather than creating new output panes + */ + + /* Return streamlined reporting statistics focused on key metrics */ + IF @debug = 1 + BEGIN + RAISERROR('Generating #index_reporting_stats, REPORT', 0, 0) WITH NOWAIT; + END; + + SELECT + /* Basic identification with enhanced naming */ + level = + CASE + WHEN irs.summary_level = 'SUMMARY' THEN '=== OVERALL ANALYSIS ===' + ELSE irs.summary_level + END, + + /* Server info (for summary) or database name */ + database_info = + CASE + WHEN irs.summary_level = 'SUMMARY' + AND irs.uptime_warning = 1 + THEN 'WARNING: Server uptime only ' + + CONVERT(varchar(10), irs.server_uptime_days) + + ' days - usage data may be incomplete!' + WHEN irs.summary_level = 'SUMMARY' + THEN 'Server uptime: ' + + CONVERT(varchar(10), irs.server_uptime_days) + + ' days' + ELSE irs.database_name + END, + + /* Schema and table names (except for summary) */ + schema_name = ISNULL(irs.schema_name, 'N/A'), + table_name = ISNULL(irs.table_name, 'N/A'), + + /* ===== Section 1: Index Counts ===== */ + /* Tables analyzed (summary only) */ + tables_analyzed = + CASE + WHEN irs.summary_level = 'SUMMARY' + THEN FORMAT(irs.tables_analyzed, 'N0') + ELSE FORMAT(0, 'N0') /* Show 0 instead of NULL */ + END, + + /* Total indexes */ + total_indexes = FORMAT(ISNULL(irs.index_count, 0), 'N0'), + + /* Removable indexes - report consistent values across levels */ + removable_indexes = + CASE + WHEN irs.summary_level = 'SUMMARY' + THEN FORMAT(ISNULL(irs.indexes_to_disable, 0), 'N0') /* Indexes that will be disabled based on analysis */ + ELSE FORMAT(ISNULL(irs.unused_indexes, 0), 'N0') /* Unused indexes at database/table level */ + END, + + /* Show mergeable indexes across all levels */ + mergeable_indexes = FORMAT(ISNULL(irs.indexes_to_merge, 0), 'N0'), + + /* Percent of indexes that can be removed */ + pct_removable = + CASE + WHEN irs.summary_level = 'SUMMARY' AND irs.index_count > 0 + THEN FORMAT(100.0 * ISNULL(irs.indexes_to_disable, 0) / NULLIF(irs.index_count, 0), 'N1') + '%' + WHEN irs.index_count > 0 + THEN FORMAT(100.0 * ISNULL(irs.unused_indexes, 0) / NULLIF(irs.index_count, 0), 'N1') + '%' + ELSE '0.0%' + END, + + /* ===== Section 2: Size and Space Savings with Before/After comparison ===== */ + /* Current size in GB */ + current_size_gb = FORMAT(ISNULL(irs.total_size_gb, 0), 'N2'), + + /* Size after cleanup - added this as new metric */ + size_after_cleanup_gb = FORMAT(ISNULL(irs.total_size_gb, 0) - ISNULL(irs.space_saved_gb, 0), 'N2'), + + /* Size that can be saved through cleanup */ + space_saved_gb = + CASE + WHEN irs.summary_level = 'SUMMARY' + THEN FORMAT(ISNULL(irs.space_saved_gb, 0), 'N2') + ELSE FORMAT(ISNULL(irs.unused_size_gb, 0), 'N2') + END, + + /* Space reduction percentage - added this as new metric */ + space_reduction_pct = + CASE + WHEN ISNULL(irs.total_size_gb, 0) > 0 + THEN FORMAT((ISNULL(irs.space_saved_gb, 0) / NULLIF(irs.total_size_gb, 0)) * 100, 'N1') + '%' + ELSE '0.0%' + END, + + /* ===== Section 3: Table and Usage Statistics ===== */ + /* Row count */ + total_rows = FORMAT(ISNULL(irs.total_rows, 0), 'N0'), + + /* Total reads - combined total and breakdown */ + reads_breakdown = + CASE + WHEN irs.summary_level <> 'SUMMARY' + THEN FORMAT(ISNULL(irs.total_reads, 0), 'N0') + + ' (' + + FORMAT(ISNULL(irs.user_seeks, 0), 'N0') + ' seeks, ' + + FORMAT(ISNULL(irs.user_scans, 0), 'N0') + ' scans, ' + + FORMAT(ISNULL(irs.user_lookups, 0), 'N0') + ' lookups)' + ELSE 'N/A' + END, + + /* Total writes */ + writes = + CASE + WHEN irs.summary_level <> 'SUMMARY' + THEN FORMAT(ISNULL(irs.total_writes, 0), 'N0') + ELSE 'N/A' + END, + + /* Daily write operations saved - added as new metric */ + daily_write_ops_saved = + CASE + WHEN irs.summary_level <> 'SUMMARY' + THEN FORMAT(ISNULL(irs.user_updates / NULLIF(CONVERT(DECIMAL(10,2), + (SELECT TOP 1 server_uptime_days FROM #index_reporting_stats WHERE summary_level = 'DATABASE')), 0) * + (ISNULL(irs.unused_indexes, 0) / NULLIF(CONVERT(DECIMAL(10,2), irs.index_count), 0)), 0), 'N0') + ELSE 'N/A' + END, + + /* ===== Section 4: Consolidated Performance Metrics ===== */ + /* Total count of lock waits (row + page) */ + lock_wait_count = + CASE + WHEN irs.summary_level <> 'SUMMARY' + THEN FORMAT(ISNULL(irs.row_lock_wait_count, 0) + + ISNULL(irs.page_lock_wait_count, 0), 'N0') + ELSE '0' + END, + + /* Average lock wait time in ms */ + avg_lock_wait_ms = + CASE + WHEN irs.summary_level <> 'SUMMARY' + AND (ISNULL(irs.row_lock_wait_count, 0) + ISNULL(irs.page_lock_wait_count, 0)) > 0 + THEN FORMAT(1.0 * (ISNULL(irs.row_lock_wait_in_ms, 0) + ISNULL(irs.page_lock_wait_in_ms, 0)) / + NULLIF(ISNULL(irs.row_lock_wait_count, 0) + ISNULL(irs.page_lock_wait_count, 0), 0), 'N2') + ELSE '0.00' + END, + + /* Combined latch wait time in ms */ + avg_latch_wait_ms = + CASE + WHEN irs.summary_level <> 'SUMMARY' + AND (ISNULL(irs.page_latch_wait_count, 0) + ISNULL(irs.page_io_latch_wait_count, 0)) > 0 + THEN FORMAT(1.0 * (ISNULL(irs.page_latch_wait_in_ms, 0) + ISNULL(irs.page_io_latch_wait_in_ms, 0)) / + NULLIF(ISNULL(irs.page_latch_wait_count, 0) + ISNULL(irs.page_io_latch_wait_count, 0), 0), 'N2') + ELSE '0.00' + END + FROM #index_reporting_stats AS irs + WHERE irs.summary_level IN ('SUMMARY', 'DATABASE', 'TABLE') /* Filter out INDEX level */ + ORDER BY + /* Order by level - summary first */ + CASE + WHEN irs.summary_level = 'SUMMARY' THEN 0 + WHEN irs.summary_level = 'DATABASE' THEN 1 + WHEN irs.summary_level = 'TABLE' THEN 2 + ELSE 3 + END, + /* Then by database name */ + irs.database_name, + /* For tables, sort by potential savings and size */ + CASE + WHEN irs.summary_level = 'TABLE' THEN irs.unused_size_gb + ELSE 0 + END DESC, + CASE + WHEN irs.summary_level = 'TABLE' THEN irs.total_size_gb + ELSE 0 + END DESC, + /* Then by schema, table */ + irs.schema_name, + irs.table_name + OPTION(RECOMPILE); + +END TRY +BEGIN CATCH + THROW; +END CATCH; +END; /*Final End*/ +GO From 88dc33c5d156459d553c40b796476ad6ef4640ab Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 17 Mar 2025 22:42:41 -0400 Subject: [PATCH 189/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 2cb55161..f396c910 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -1416,6 +1416,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* If SQL Server edition doesn't support compression, mark all as ineligible */ IF @can_compress = 0 BEGIN + IF @debug = 1 + BEGIN + RAISERROR('updating compression eligibility', 0, 0) WITH NOWAIT; + END; + UPDATE #compression_eligibility SET From 882357f1d9e62fd2aa69f9a3ad9d483431304f98 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 17 Mar 2025 22:49:14 -0400 Subject: [PATCH 190/246] trying to fix the loop dammit --- .DS_Store | Bin 6148 -> 8196 bytes sp_IndexCleanup/sp_IndexCleanup.sql | 123 ++++++++++++++++++++++++++-- 2 files changed, 116 insertions(+), 7 deletions(-) diff --git a/.DS_Store b/.DS_Store index 068d98176ca291437e0540c5f458ab97646953d3..0a0c9b2264b9c02712a1949ac68eefb9fb3b8f7b 100644 GIT binary patch delta 158 zcmZoMXmOBWU|?W$DortDU;r^WfEYvza8E20o2aMA$gweCH}hr%jz7$c**Q2SHn1>q zZ02E+V4Up7#xYrrO?+}LTeGl%j)JbanNh8dLbbW2xsHOFvFT)Hc59IQW>MAvrp+xP u4a^f8cm$b&MgxHaH;`}z*|J%X<2&wxd!wdkv9UjF1 delta 108 zcmZp1XfcprU|?W$DortDU=RQ@Ie-{Mvv5r;6q~50$jG)aU^g=(+h!gC3C7J=im@z2C4!A0d64S3evE#@H_Klei=`Yb_OPhQ6SS9HplbKVFm!is1W%8 diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index f396c910..db5d3bad 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -1145,15 +1145,123 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. BEGIN RAISERROR('Single database mode, using specified or current database: %s', 0, 0, @database_name) WITH NOWAIT; END; - - END; - - /* Process the database - since we're in single database mode, just use the existing values */ - IF @debug = 1 + END + /* Process multiple databases */ + ELSE IF @get_all_databases = 1 BEGIN - RAISERROR('Single database mode, using specified or current database: %s', 0, 0, @database_name) WITH NOWAIT; + IF @debug = 1 + BEGIN + RAISERROR('Processing all databases with @get_all_databases = 1', 0, 0) WITH NOWAIT; + END; + + /* Get the count of databases for reporting */ + SELECT + @database_count = COUNT_BIG(*) + FROM #databases_to_process AS dtp + OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + RAISERROR('Beginning processing for %d databases', 0, 0, @database_count) WITH NOWAIT; + END; + + /* Set up database cursor */ + DECLARE @database_cursor CURSOR; + SET @database_cursor = + CURSOR + LOCAL + STATIC + READ_ONLY + FORWARD_ONLY + FOR + SELECT + dtp.database_id, + dtp.database_name + FROM #databases_to_process AS dtp + WHERE dtp.processed = 0 + ORDER BY + dtp.database_name + OPTION(RECOMPILE); + + OPEN @database_cursor; + FETCH NEXT + FROM @database_cursor + INTO + @database_id, + @database_name; + + WHILE @@FETCH_STATUS = 0 + BEGIN + /* Process current database */ + IF @debug = 1 + BEGIN + RAISERROR('Processing database %s', 0, 0, @database_name) WITH NOWAIT; + END; + + /* Start main database processing logic */ + + /* Check for schema/table parameters */ + IF @schema_name IS NOT NULL + AND @table_name IS NOT NULL + BEGIN + IF @debug = 1 + BEGIN + RAISERROR('validating object existence for %s.%s.%s', 0, 0, @database_name, @schema_name, @table_name) WITH NOWAIT; + END; + + SELECT + @full_object_name = + QUOTENAME(@database_name) + + N'.' + + QUOTENAME(@schema_name) + + N'.' + + QUOTENAME(@table_name); + + SET @object_id = OBJECT_ID(@full_object_name); + + IF @object_id IS NULL AND @full_object_name IS NOT NULL + BEGIN + RAISERROR('The object %s doesn''t seem to exist', 16, 1, @full_object_name) WITH NOWAIT; + + /* Skip this database and continue to the next one */ + GOTO NextDatabase; + END; + END; + + /* Process the current database */ + IF @debug = 1 + BEGIN + RAISERROR('Generating #filtered_object insert for database %s', 0, 0, @database_name) WITH NOWAIT; + END; + + /* INSERT DATABASE PROCESSING LOGIC HERE */ + + /* Rest of the database processing will go here */ + + /* Update processed flag for this database */ +NextDatabase: + UPDATE #databases_to_process + SET + processed = 1, + process_date = SYSDATETIME() + WHERE database_id = @database_id; + + /* Get next database */ + FETCH NEXT + FROM @database_cursor + INTO + @database_id, + @database_name; + END; + + CLOSE @database_cursor; + DEALLOCATE @database_cursor; + + /* After processing all databases, return to show consolidated results */ + GOTO GenerateResults; END; + /* For single database mode - process the single database */ IF @debug = 1 BEGIN RAISERROR('Processing database %s', 0, 0, @database_name) WITH NOWAIT; @@ -1164,7 +1272,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. BEGIN IF @debug = 1 BEGIN - RAISERROR('validating object existence for %s.%s.%s', 0, 0, @database_name, @schema_name, @table_name) WITH NOWAIT; + RAISERROR('validating object existence for %s.%s.%s', 0, 0, @database_name, @schema_name, @table_name) WITH NOWAIT; END; SELECT @@ -4968,6 +5076,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. OPTION(RECOMPILE); END; /* End of @get_all_databases = 1 section */ +GenerateResults: /* Return consolidated reporting statistics for all databases processed */ IF @debug = 1 BEGIN From 003ebb21eaa99db7f8cd6f206dad99fe3d9ff4e5 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 17 Mar 2025 22:50:25 -0400 Subject: [PATCH 191/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index db5d3bad..a28896c1 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -1166,7 +1166,6 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END; /* Set up database cursor */ - DECLARE @database_cursor CURSOR; SET @database_cursor = CURSOR LOCAL From 8b31632f4d9c2e1d4f8bbbd82d0c08211cedfc34 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Mon, 17 Mar 2025 23:00:51 -0400 Subject: [PATCH 192/246] Update CLAUDE.md --- CLAUDE.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index 6b932363..4a926227 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -21,6 +21,7 @@ This document outlines the T-SQL coding style preferences for Erik Darling (Darl - **Object creation**: Generally use CREATE OR ALTER for objects instead of DROP/CREATE - **Table aliases**: Tables must always have aliases, even in simple queries - **Column references**: Always qualify columns with their table alias +- **Commas**: Trailing commas always. ## Comments @@ -49,7 +50,7 @@ This document outlines the T-SQL coding style preferences for Erik Darling (Darl - **SELECT statements**: - SELECT keyword on first line - Column list starts on next line, indented - - Leading commas for multi-line column lists + - Trailing commas for multi-line column lists - Columns aligned vertically for readability - FROM clause on new line at same indent level as SELECT - Column aliases should always use the pattern: column_name = column_expression From 313d60205f73ca77d2890bfc465b14773f391c32 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 18 Mar 2025 00:22:15 -0400 Subject: [PATCH 193/246] starting over starting over --- sp_IndexCleanup/sp_IndexCleanup.sql | 1375 +++++------------------ sp_IndexCleanup/sp_IndexCleanup_Old.sql | 687 ++++++----- 2 files changed, 689 insertions(+), 1373 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index a28896c1..79155d19 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -23,9 +23,6 @@ ALTER PROCEDURE @min_writes bigint = 0, @min_size_gb decimal(10,2) = 0, @min_rows bigint = 0, - @get_all_databases bit = 0, /* When 1, analyzes all eligible databases on the server */ - @include_databases nvarchar(max) = NULL, /* Comma-separated list of databases to include (used with @get_all_databases = 1) */ - @exclude_databases nvarchar(max) = NULL, /* Comma-separated list of databases to exclude (used with @get_all_databases = 1) */ @help bit = 'false', @debug bit = 'false', @version varchar(20) = NULL OUTPUT, @@ -113,9 +110,6 @@ BEGIN TRY WHEN N'@min_writes' THEN 'minimum number of writes for an index to be considered used' WHEN N'@min_size_gb' THEN 'minimum size in GB for an index to be analyzed' WHEN N'@min_rows' THEN 'minimum number of rows for a table to be analyzed' - WHEN N'@get_all_databases' THEN 'when set to 1, analyzes all eligible databases on the server' - WHEN N'@include_databases' THEN 'comma-separated list of databases to include (used with @get_all_databases = 1)' - WHEN N'@exclude_databases' THEN 'comma-separated list of databases to exclude (used with @get_all_databases = 1)' WHEN N'@help' THEN 'displays this help information' WHEN N'@debug' THEN 'prints debug information during execution' WHEN N'@version' THEN 'returns the version number of the procedure' @@ -132,9 +126,6 @@ BEGIN TRY WHEN N'@min_writes' THEN 'any positive integer or 0' WHEN N'@min_size_gb' THEN 'any positive decimal or 0' WHEN N'@min_rows' THEN 'any positive integer or 0' - WHEN N'@get_all_databases' THEN '0 or 1' - WHEN N'@include_databases' THEN 'comma-separated list of database names' - WHEN N'@exclude_databases' THEN 'comma-separated list of database names' WHEN N'@help' THEN '0 or 1' WHEN N'@debug' THEN '0 or 1' WHEN N'@version' THEN 'OUTPUT parameter' @@ -151,9 +142,6 @@ BEGIN TRY WHEN N'@min_writes' THEN '0' WHEN N'@min_size_gb' THEN '0' WHEN N'@min_rows' THEN '0' - WHEN N'@get_all_databases' THEN '0' - WHEN N'@include_databases' THEN 'NULL' - WHEN N'@exclude_databases' THEN 'NULL' WHEN N'@help' THEN 'false' WHEN N'@debug' THEN 'true' WHEN N'@version' THEN 'NULL' @@ -180,7 +168,7 @@ BEGIN TRY RAISERROR(' MIT License -Copyright 2024 Darling Data, LLC +Copyright 2025 Darling Data, LLC https://www.erikdarling.com/ @@ -268,18 +256,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SYSDATETIME() ) FROM sys.dm_os_sys_info AS osi - ), - /* Variables for multi-database processing */ - @database_cursor CURSOR, - @current_database_id integer, - @current_database_name sysname, - @database_count integer = 0, - @processed_count integer = 0, - @db_list nvarchar(max) = N'', - @include_xml xml = N'', - @exclude_xml xml = N'', - @conflict_list nvarchar(max) = N'', - @error_msg nvarchar(2000); + ); /* Set uptime warning flag after @uptime_days is calculated */ SELECT @@ -290,6 +267,79 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ELSE 0 END; + /* + Initial checks for object validity + */ + IF @debug = 1 + BEGIN + RAISERROR('Checking paramaters...', 0, 0) WITH NOWAIT; + END; + + IF @database_name IS NULL + AND DB_NAME() NOT IN + ( + N'master', + N'model', + N'msdb', + N'tempdb', + N'rdsadmin' + ) + BEGIN + SELECT + @database_name = DB_NAME(); + END; + + IF @database_name IS NOT NULL + BEGIN + SELECT + @database_id = d.database_id + FROM sys.databases AS d + WHERE d.database_id = DB_ID(@database_name) + AND d.state = 0 + AND d.is_in_standby = 0 + AND d.is_read_only = 0 + OPTION(RECOMPILE); + END; + + IF @schema_name IS NULL + AND @table_name IS NOT NULL + BEGIN + IF @debug = 1 + BEGIN + RAISERROR('Parameter @schema_name cannot be NULL when specifying a table, defaulting to dbo', 10, 1) WITH NOWAIT; + END; + + SELECT + @schema_name = N'dbo'; + END; + + IF @schema_name IS NOT NULL + AND @table_name IS NOT NULL + BEGIN + IF @debug = 1 + BEGIN + RAISERROR('validating object existence for %s.%s.&s.', 0, 0, @database_name, @schema_name, @table_name) WITH NOWAIT; + END; + + SELECT + @full_object_name = + QUOTENAME(@database_name) + + N'.' + + QUOTENAME(@schema_name) + + N'.' + + QUOTENAME(@table_name); + + SELECT + @object_id = + OBJECT_ID(@full_object_name); + + IF @object_id IS NULL + BEGIN + RAISERROR('The object %s doesn''t seem to exist', 16, 1, @full_object_name) WITH NOWAIT; + RETURN; + END; + END; + /* Parameter validation */ IF @min_reads < 0 OR @min_reads IS NULL @@ -335,26 +385,6 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SET @min_rows = 0; END; - IF @schema_name IS NULL - AND @table_name IS NOT NULL - BEGIN - IF @debug = 1 - BEGIN - RAISERROR('Parameter @schema_name cannot be NULL when specifying a table, defaulting to dbo', 10, 1) WITH NOWAIT; - END; - - SET @schema_name = N'dbo'; - END; - - /* Parameter validation for multi-database mode */ - IF @get_all_databases = 1 - AND @database_name IS NOT NULL - BEGIN - RAISERROR('You cannot specify both @get_all_databases = 1 and a specific @database_name. Using @get_all_databases = 1 and ignoring @database_name.', 10, 1) WITH NOWAIT; - - SET @database_name = NULL; - END; - /* Temp tables! */ @@ -495,11 +525,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. index_id integer NOT NULL, index_name sysname NOT NULL, is_unique bit NULL, - key_columns nvarchar(max) NULL, - included_columns nvarchar(max) NULL, - filter_definition nvarchar(max) NULL, + key_columns nvarchar(MAX) NULL, + included_columns nvarchar(MAX) NULL, + filter_definition nvarchar(MAX) NULL, /* Query plan for original CREATE INDEX statement */ - original_index_definition nvarchar(max) NULL, + original_index_definition nvarchar(MAX) NULL, /* Consolidation rule that matched (e.g., Key Duplicate, Key Subset, etc) For exact duplicates, use one of: Exact Duplicate, Reverse Duplicate, or Equal Except For Filter @@ -518,6 +548,22 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. index_priority decimal(10,6) NULL PRIMARY KEY CLUSTERED(database_id, object_id, index_id) ); + + CREATE TABLE + #compression_eligibility + ( + database_id integer NOT NULL, + database_name sysname NOT NULL, + schema_id integer NOT NULL, + schema_name sysname NOT NULL, + object_id integer NOT NULL, + table_name sysname NOT NULL, + index_id integer NOT NULL, + index_name sysname NOT NULL, + can_compress bit NOT NULL, + reason nvarchar(200) NULL, + PRIMARY KEY CLUSTERED(database_id, object_id, index_id) + ); CREATE TABLE #index_cleanup_results @@ -541,22 +587,6 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. script nvarchar(max) NULL /* Script to execute the action */ ); - CREATE TABLE - #compression_eligibility - ( - database_id integer NOT NULL, - database_name sysname NOT NULL, - schema_id integer NOT NULL, - schema_name sysname NOT NULL, - object_id integer NOT NULL, - table_name sysname NOT NULL, - index_id integer NOT NULL, - index_name sysname NOT NULL, - can_compress bit NOT NULL, - reason nvarchar(200) NULL, - PRIMARY KEY CLUSTERED(database_id, object_id, index_id) - ); - CREATE TABLE #key_duplicate_dedupe ( @@ -582,7 +612,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. superset_included_columns nvarchar(max) NULL ); - CREATE TABLE + /* Create a new temp table for detailed reporting statistics */ + CREATE TABLE #index_reporting_stats ( summary_level varchar(20) NOT NULL, /* 'DATABASE', 'TABLE', 'INDEX', 'SUMMARY' */ @@ -635,667 +666,14 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. leaf_delete_count bigint NULL ); - /* Create a table to store databases to process */ - CREATE TABLE - #databases_to_process - ( - database_id integer NOT NULL, - database_name sysname NOT NULL, - processed bit NOT NULL DEFAULT 0, - process_date datetime2(7) NULL, - PRIMARY KEY CLUSTERED(database_id) - ); - - /* Create a table to track databases that were requested but couldn't be processed */ - CREATE TABLE - #skipped_databases - ( - database_name sysname NOT NULL, - reason nvarchar(255) NOT NULL - ); - - /* Create a table to parse the include/exclude lists */ - CREATE TABLE - #database_list - ( - id integer IDENTITY PRIMARY KEY CLUSTERED, - database_name sysname NOT NULL - ); - - /* Handle multi-database mode */ - IF @get_all_databases = 1 - BEGIN - IF @debug = 1 - BEGIN - RAISERROR('Multi-database mode enabled, gathering database list...', 0, 0) WITH NOWAIT; - END; - - /* Parse @include_databases if specified - using XML for string splitting instead of STRING_SPLIT (version compatibility) */ - IF @include_databases IS NOT NULL - IF @debug = 1 - BEGIN - RAISERROR('processing included databases', 0, 0) WITH NOWAIT; - END; - BEGIN - SELECT - @include_xml = - CONVERT - ( - xml, - '' + - REPLACE - ( - @include_databases, - ',', - '' - ) + - '' - ); - - INSERT INTO - #database_list - ( - database_name - ) - SELECT - database_name = - LTRIM(RTRIM(t.i.value('.', 'sysname'))) - FROM @include_xml.nodes('/i') AS t(i) - WHERE LTRIM(RTRIM(t.i.value('.', 'sysname'))) <> '' - OPTION(RECOMPILE); - END; - - /* Check for databases in both include and exclude lists */ - IF @exclude_databases IS NOT NULL - IF @debug = 1 - BEGIN - RAISERROR('processing excluded databases', 0, 0) WITH NOWAIT; - END; - - BEGIN - SELECT - @exclude_xml = - CONVERT - ( - xml, - '' + - REPLACE - ( - @exclude_databases, - ',', - '' - ) + - '' - ); - - /* Build list of conflicting databases */ - SELECT - @conflict_list = - @conflict_list + - LTRIM(RTRIM(t.i.value('.', 'sysname'))) + - N', ' - FROM @exclude_xml.nodes('/i') AS t(i) - WHERE LTRIM(RTRIM(t.i.value('.', 'sysname'))) <> '' - AND EXISTS - ( - SELECT - 1/0 - FROM #database_list AS dl - WHERE dl.database_name = LTRIM(RTRIM(t.i.value('.', 'sysname'))) - ) - OPTION(RECOMPILE);; - - /* If we found any conflicts, raise an error */ - IF LEN(@conflict_list) > 0 - BEGIN - /* Remove trailing comma and space */ - SET @conflict_list = LEFT(@conflict_list, LEN(@conflict_list) - 2); - - SET @error_msg = - N'The following databases appear in both @include_databases and @exclude_databases, which creates ambiguity: ' + - @conflict_list + N'. Please remove these databases from one of the lists.'; - - RAISERROR(@error_msg, 16, 1) WITH NOWAIT; - RETURN; - END; - END; - - /* - Check SQL Server engine edition and use appropriate query paths - */ - IF - ( - SELECT - CONVERT - ( - sysname, - SERVERPROPERTY('EngineEdition') - ) - ) IN (5, 8) /* Azure SQL DB or Managed Instance */ - BEGIN - IF @debug = 1 - BEGIN - RAISERROR('processing databases special in azure', 0, 0) WITH NOWAIT; - END; - - /* Get all eligible databases for Azure SQL */ - INSERT INTO - #databases_to_process - WITH - (TABLOCK) - ( - database_id, - database_name - ) - SELECT - database_id = d.database_id, - database_name = d.name - FROM sys.databases AS d - WHERE d.name NOT IN (N'master', N'model', N'msdb', N'tempdb') - AND d.state = 0 - AND d.is_in_standby = 0 - AND d.is_read_only = 0 - AND d.database_id > 4 /* Skip system databases */ - AND - ( - /* If include list is provided, only keep databases in that list */ - (@include_databases IS NULL) OR - (d.name IN (SELECT database_name FROM #database_list)) - ) - OPTION(RECOMPILE); - END; - ELSE /* Regular SQL Server */ - BEGIN - IF @debug = 1 - BEGIN - RAISERROR('processing on-prem sql server databases', 0, 0) WITH NOWAIT; - END; - - /* Get all eligible databases with AG primary replica check */ - INSERT INTO - #databases_to_process - WITH - (TABLOCK) - ( - database_id, - database_name - ) - SELECT - database_id = d.database_id, - database_name = d.name - FROM sys.databases AS d - WHERE d.name NOT IN (N'master', N'model', N'msdb', N'tempdb', N'rdsadmin') - AND d.state = 0 - AND d.is_in_standby = 0 - AND d.is_read_only = 0 - AND d.database_id > 4 /* Skip system databases */ - /* Add AG check to ensure we only process the primary replica */ - AND NOT EXISTS - ( - SELECT - 1/0 - FROM sys.dm_hadr_availability_replica_states AS s - JOIN sys.availability_databases_cluster AS c - ON s.group_id = c.group_id - AND d.name = c.database_name - WHERE s.is_local <> 1 - AND s.role_desc <> N'PRIMARY' - AND DATABASEPROPERTYEX(c.database_name, N'Updateability') <> N'READ_WRITE' - ) - AND - ( - /* If include list is provided, only keep databases in that list */ - (@include_databases IS NULL) OR - (d.name IN (SELECT database_name FROM #database_list)) - ) - OPTION(RECOMPILE);; - END; - - /* Remove excluded databases if specified */ - IF @exclude_databases IS NOT NULL - BEGIN - IF @debug = 1 - BEGIN - RAISERROR('processing excluded databases', 0, 0) WITH NOWAIT; - END; - - SELECT - @exclude_xml = - CONVERT - ( - xml, - '' + - REPLACE - ( - @exclude_databases, - ',', - '' - ) + - '' - ); - - DELETE - dp - FROM #databases_to_process AS dp - WHERE EXISTS - ( - SELECT - 1/0 - FROM @exclude_xml.nodes('/i') AS t(i) - WHERE dp.database_name = LTRIM(RTRIM(t.i.value('.', 'sysname'))) - AND LTRIM(RTRIM(t.i.value('.', 'sysname'))) <> '' - ) - OPTION(RECOMPILE);; - END; - - IF @debug = 1 - BEGIN - /* Count databases */ - SELECT - database_count = COUNT_BIG(*) - FROM #databases_to_process; - - /* List databases without using STRING_AGG (version compatibility) */ - SELECT @db_list = N''; - - SELECT - @db_list = - @db_list + - database_name + - N', ' - FROM #databases_to_process AS dtp - ORDER BY - dtp.database_name - OPTION(RECOMPILE);; - - /* Remove trailing comma if list is not empty */ - IF LEN(@db_list) > 0 - BEGIN - SET @db_list = LEFT(@db_list, LEN(@db_list) - 1); - END; - - RAISERROR('Databases to process: %s', 0, 0, @db_list) WITH NOWAIT; - END; - - /* Track databases that were requested but skipped (for better reporting) */ - IF @include_databases IS NOT NULL - BEGIN - IF @debug = 1 - BEGIN - RAISERROR('processing included databases for skip reasons', 0, 0) WITH NOWAIT; - END; - - INSERT - #skipped_databases - WITH - (TABLOCK) - ( - database_name, - reason - ) - SELECT - database_name = dl.database_name, - reason = - CASE - WHEN d.name IS NULL - THEN N'Database does not exist' - WHEN d.state <> 0 - THEN N'Database not online' - WHEN d.is_in_standby = 1 - THEN N'Database is in standby' - WHEN d.is_read_only = 1 - THEN N'Database is read-only' - WHEN d.database_id <= 4 - THEN N'System database' - WHEN EXISTS - ( - SELECT - 1/0 - FROM sys.dm_hadr_availability_replica_states AS s - JOIN sys.availability_databases_cluster AS c - ON s.group_id = c.group_id - AND d.name = c.database_name - WHERE s.is_local <> 1 - AND s.role_desc <> N'PRIMARY' - AND DATABASEPROPERTYEX(c.database_name, N'Updateability') <> N'READ_WRITE' - ) - THEN N'AG replica issue - not primary or read-write' - ELSE N'Other issue' - END - FROM #database_list AS dl - LEFT JOIN sys.databases AS d - ON dl.database_name = d.name - WHERE NOT EXISTS - ( - SELECT - 1/0 - FROM #databases_to_process AS dp - WHERE dp.database_name = dl.database_name - ) - OPTION(RECOMPILE); - END; - - /* Also track explicitly excluded databases */ - IF @exclude_databases IS NOT NULL - BEGIN - IF @debug = 1 - BEGIN - RAISERROR('processing explicitly excluded databases', 0, 0) WITH NOWAIT; - END; - - INSERT - #skipped_databases - WITH - (TABLOCK) - ( - database_name, - reason - ) - SELECT - database_name = LTRIM(RTRIM(t.i.value('.', 'nvarchar(128)'))), - reason = N'Explicitly excluded by @exclude_databases parameter' - FROM - ( - SELECT xml_list = - CONVERT - ( - xml, - N'' + - REPLACE - ( - @exclude_databases, - N',', - N'' - ) + - N'' - ) - ) AS a - CROSS APPLY a.xml_list.nodes('i') AS t(i) - WHERE LTRIM(RTRIM(t.i.value('.', 'sysname'))) <> '' - AND EXISTS - ( - SELECT - 1/0 - FROM sys.databases AS d - WHERE d.name = LTRIM(RTRIM(t.i.value('.', 'sysname'))) - ) - OPTION(RECOMPILE); - - /* If no databases match criteria, exit */ - IF NOT EXISTS (SELECT 1/0 FROM #databases_to_process AS dtp) - BEGIN - RAISERROR('No eligible databases found to process with the specified filters', 16, 1) WITH NOWAIT; - RETURN; - END; - END; - ELSE - BEGIN - /* Single database mode */ - IF @debug = 1 - BEGIN - RAISERROR('Single database mode, using specified or current database', 0, 0) WITH NOWAIT; - END; - - /* If no database name specified, use current database if not a system database */ - IF @database_name IS NULL - AND @get_all_databases = 0 - AND DB_NAME() NOT IN - ( - N'master', - N'model', - N'msdb', - N'tempdb', - N'rdsadmin' - ) - BEGIN - SELECT - @database_name = DB_NAME(); - END; - - /*Construct the full object name*/ - IF @schema_name IS NOT NULL - AND @table_name IS NOT NULL - BEGIN - IF @debug = 1 - BEGIN - RAISERROR('validating object existence for %s.%s.&s.', 0, 0, @database_name, @schema_name, @table_name) WITH NOWAIT; - END; - - SELECT - @full_object_name = - QUOTENAME(@database_name) + - N'.' + - QUOTENAME(@schema_name) + - N'.' + - QUOTENAME(@table_name); - - SET @object_id = OBJECT_ID(@full_object_name); - END; - - IF @object_id IS NULL - BEGIN - RAISERROR('The object %s doesn''t seem to exist', 16, 1, @full_object_name) WITH NOWAIT; - RETURN; - END; - - /* Add the single database to the processing list */ - IF @database_name IS NOT NULL - BEGIN - IF @debug = 1 - BEGIN - RAISERROR('inserting databases to process', 0, 0) WITH NOWAIT; - END; - - INSERT INTO - #databases_to_process - WITH - (TABLOCK) - ( - database_id, - database_name - ) - SELECT - database_id = d.database_id, - database_name = d.name - FROM sys.databases AS d - WHERE d.name = @database_name - AND d.state_desc = N'ONLINE' - OPTION(RECOMPILE); - - /* Validate the database exists and is accessible */ - IF NOT EXISTS (SELECT 1/0 FROM #databases_to_process AS dtp) - BEGIN - RAISERROR('The specified database %s does not exist, is not in ONLINE state, or you do not have permission to access it', 16, 1, @database_name) WITH NOWAIT; - RETURN; - END; - END; - ELSE - BEGIN - RAISERROR('No valid database specified and current database is a system database. Please specify a user database.', 16, 1) WITH NOWAIT; - RETURN; - END; - - /* Set @database_id for single database mode (for backward compatibility) */ - SELECT - @database_id = database_id, - @database_name = database_name - FROM #databases_to_process - OPTION(RECOMPILE); - END; - /* - Main processing logic - either loop through all databases or process a single database + Start insert queries */ - - /* Process single database */ - IF @get_all_databases = 0 - BEGIN - IF @debug = 1 - BEGIN - RAISERROR('processing for @get_all_databases = 0', 0, 0) WITH NOWAIT; - END; - - /* Use the database specified in @database_name */ - SELECT - @database_id = database_id, - @database_name = database_name - FROM #databases_to_process; - - IF @debug = 1 - BEGIN - RAISERROR('Single database mode, using specified or current database: %s', 0, 0, @database_name) WITH NOWAIT; - END; - END - /* Process multiple databases */ - ELSE IF @get_all_databases = 1 - BEGIN - IF @debug = 1 - BEGIN - RAISERROR('Processing all databases with @get_all_databases = 1', 0, 0) WITH NOWAIT; - END; - - /* Get the count of databases for reporting */ - SELECT - @database_count = COUNT_BIG(*) - FROM #databases_to_process AS dtp - OPTION(RECOMPILE); - - IF @debug = 1 - BEGIN - RAISERROR('Beginning processing for %d databases', 0, 0, @database_count) WITH NOWAIT; - END; - - /* Set up database cursor */ - SET @database_cursor = - CURSOR - LOCAL - STATIC - READ_ONLY - FORWARD_ONLY - FOR - SELECT - dtp.database_id, - dtp.database_name - FROM #databases_to_process AS dtp - WHERE dtp.processed = 0 - ORDER BY - dtp.database_name - OPTION(RECOMPILE); - - OPEN @database_cursor; - FETCH NEXT - FROM @database_cursor - INTO - @database_id, - @database_name; - - WHILE @@FETCH_STATUS = 0 - BEGIN - /* Process current database */ - IF @debug = 1 - BEGIN - RAISERROR('Processing database %s', 0, 0, @database_name) WITH NOWAIT; - END; - - /* Start main database processing logic */ - - /* Check for schema/table parameters */ - IF @schema_name IS NOT NULL - AND @table_name IS NOT NULL - BEGIN - IF @debug = 1 - BEGIN - RAISERROR('validating object existence for %s.%s.%s', 0, 0, @database_name, @schema_name, @table_name) WITH NOWAIT; - END; - - SELECT - @full_object_name = - QUOTENAME(@database_name) + - N'.' + - QUOTENAME(@schema_name) + - N'.' + - QUOTENAME(@table_name); - - SET @object_id = OBJECT_ID(@full_object_name); - - IF @object_id IS NULL AND @full_object_name IS NOT NULL - BEGIN - RAISERROR('The object %s doesn''t seem to exist', 16, 1, @full_object_name) WITH NOWAIT; - - /* Skip this database and continue to the next one */ - GOTO NextDatabase; - END; - END; - - /* Process the current database */ - IF @debug = 1 - BEGIN - RAISERROR('Generating #filtered_object insert for database %s', 0, 0, @database_name) WITH NOWAIT; - END; - - /* INSERT DATABASE PROCESSING LOGIC HERE */ - - /* Rest of the database processing will go here */ - - /* Update processed flag for this database */ -NextDatabase: - UPDATE #databases_to_process - SET - processed = 1, - process_date = SYSDATETIME() - WHERE database_id = @database_id; - - /* Get next database */ - FETCH NEXT - FROM @database_cursor - INTO - @database_id, - @database_name; - END; - - CLOSE @database_cursor; - DEALLOCATE @database_cursor; - - /* After processing all databases, return to show consolidated results */ - GOTO GenerateResults; - END; - - /* For single database mode - process the single database */ - IF @debug = 1 - BEGIN - RAISERROR('Processing database %s', 0, 0, @database_name) WITH NOWAIT; - END; - - IF @schema_name IS NOT NULL - AND @table_name IS NOT NULL - BEGIN - IF @debug = 1 - BEGIN - RAISERROR('validating object existence for %s.%s.%s', 0, 0, @database_name, @schema_name, @table_name) WITH NOWAIT; - END; - - SELECT - @full_object_name = - QUOTENAME(@database_name) + - N'.' + - QUOTENAME(@schema_name) + - N'.' + - QUOTENAME(@table_name); - - SET @object_id = OBJECT_ID(@full_object_name); - END; - IF @object_id IS NULL AND @full_object_name IS NOT NULL - BEGIN - RAISERROR('The object %s doesn''t seem to exist', 16, 1, @full_object_name) WITH NOWAIT; - RETURN; - END; - - /* Generate and execute SQL to process the current database */ IF @debug = 1 BEGIN RAISERROR('Generating #filtered_object insert', 0, 0) WITH NOWAIT; - END; + END; SELECT @sql = N' @@ -1386,7 +764,7 @@ NextDatabase: BEGIN RAISERROR('adding object_id filter', 0, 0) WITH NOWAIT; END; - + SELECT @sql += N' AND t.object_id = @object_id'; END; @@ -1473,7 +851,8 @@ NextDatabase: BEGIN IF @debug = 1 BEGIN - RAISERROR('No rows inserted into #filtered_objects', 0, 0) WITH NOWAIT; + RAISERROR('No rows inserted into #filtered_objects, nothing to do!', 10, 0) WITH NOWAIT; + RETURN; END; END; @@ -1514,29 +893,33 @@ NextDatabase: fo.table_name, fo.index_id, fo.index_name, - 1, /* Default to compressible */ - NULL + can_compress = + CASE + @can_compress + WHEN 0 + THEN 0 + ELSE 1 + END, + reason = + CASE + @can_compress + WHEN 0 + THEN N'SQL Server edition or version does not support compression' + ELSE NULL + END FROM #filtered_objects AS fo WHERE fo.can_compress = 1 OPTION(RECOMPILE); - - /* If SQL Server edition doesn't support compression, mark all as ineligible */ - IF @can_compress = 0 + + IF @debug = 1 BEGIN - IF @debug = 1 - BEGIN - RAISERROR('updating compression eligibility', 0, 0) WITH NOWAIT; - END; - - UPDATE - #compression_eligibility - SET - #compression_eligibility.can_compress = 0, - #compression_eligibility.reason = N'SQL Server edition or version does not support compression' - WHERE #compression_eligibility.can_compress = 1 + SELECT + table_name = '#compression_eligibility before update', + ce.* + FROM #compression_eligibility AS ce OPTION(RECOMPILE); END; - + /* Check for sparse columns or incompatible data types */ IF @can_compress = 1 BEGIN @@ -1584,7 +967,7 @@ NextDatabase: IF @debug = 1 BEGIN SELECT - table_name = '#compression_eligibility', + table_name = '#compression_eligibility after update', ce.* FROM #compression_eligibility AS ce OPTION(RECOMPILE); @@ -1851,7 +1234,7 @@ NextDatabase: JOIN ' + QUOTENAME(@database_name) + CONVERT ( - nvarchar(max), + nvarchar(MAX), N'.sys.columns AS c ON ic.object_id = c.object_id AND ic.column_id = c.column_id @@ -1879,7 +1262,7 @@ NextDatabase: ) + QUOTENAME(@database_name) + CONVERT ( - nvarchar(max), + nvarchar(MAX), N'.sys.dm_db_partition_stats ps WHERE ps.object_id = t.object_id AND ps.index_id = 1 @@ -1889,6 +1272,11 @@ NextDatabase: IF @object_id IS NOT NULL BEGIN + IF @debug = 1 + BEGIN + RAISERROR('adding object+id filter', 0, 0) WITH NOWAIT; + END; + SELECT @sql += N' AND t.object_id = @object_id'; END; @@ -2052,6 +1440,11 @@ NextDatabase: IF @object_id IS NOT NULL BEGIN + IF @debug = 1 + BEGIN + RAISERROR('adding in object_id filter', 0, 0) WITH NOWAIT; + END; + SELECT @sql += N' AND t.object_id = @object_id'; END; @@ -2512,17 +1905,19 @@ NextDatabase: CASE WHEN ia1.index_priority > ia2.index_priority THEN NULL /* This index is the keeper */ - WHEN ia1.index_priority = ia2.index_priority AND ia1.index_name < ia2.index_name + WHEN ia1.index_priority = ia2.index_priority + AND ia1.index_name < ia2.index_name THEN NULL /* When tied, use alphabetical ordering for consistency */ ELSE ia2.index_name /* Other index is the keeper */ END, ia1.action = CASE WHEN ia1.index_priority > ia2.index_priority - THEN 'KEEP' /* This index is the keeper */ - WHEN ia1.index_priority = ia2.index_priority AND ia1.index_name < ia2.index_name - THEN 'KEEP' /* When tied, use alphabetical ordering for consistency */ - ELSE 'DISABLE' /* Other index gets disabled */ + THEN N'KEEP' /* This index is the keeper */ + WHEN ia1.index_priority = ia2.index_priority + AND ia1.index_name < ia2.index_name + THEN N'KEEP' /* When tied, use alphabetical ordering for consistency */ + ELSE N'DISABLE' /* Other index gets disabled */ END FROM #index_analysis AS ia1 JOIN #index_analysis AS ia2 @@ -2642,8 +2037,8 @@ NextDatabase: AND NOT (ia1.is_unique = 0 AND ia2.is_unique = 1) ) AND ISNULL(ia1.included_columns, N'') <> ISNULL(ia2.included_columns, N'') - THEN 'MERGE INCLUDES' /* Keep this index but merge includes */ - ELSE 'DISABLE' /* Other index is keeper, disable this one */ + THEN N'MERGE INCLUDES' /* Keep this index but merge includes */ + ELSE N'DISABLE' /* Other index is keeper, disable this one */ END, /* For the winning index, set clear superseded_by text for the report */ ia1.superseded_by = @@ -2654,7 +2049,7 @@ NextDatabase: ia1.index_priority >= ia2.index_priority AND NOT (ia1.is_unique = 0 AND ia2.is_unique = 1) ) - THEN 'Supersedes ' + + THEN N'Supersedes ' + ia2.index_name ELSE NULL END @@ -2775,7 +2170,15 @@ NextDatabase: SET ia2.consolidation_rule = N'Key Superset', ia2.action = N'MERGE INCLUDES', /* The wider index gets merged with includes */ - ia2.superseded_by = COALESCE(ia2.superseded_by + ', ', '') + 'Supersedes ' + ia1.index_name + ia2.superseded_by = + ISNULL + ( + ia2.superseded_by + + ', ', + '' + ) + + N'Supersedes ' + + ia1.index_name FROM #index_analysis AS ia1 JOIN #index_analysis AS ia2 ON ia1.database_id = ia2.database_id @@ -2796,7 +2199,8 @@ NextDatabase: END; /* Rule 6: Merge includes from subset to superset indexes */ - WITH KeySubsetSuperset AS + WITH + KeySubsetSuperset AS ( SELECT superset.database_id, @@ -2822,7 +2226,7 @@ NextDatabase: CASE /* If both have includes, combine them without duplicates */ WHEN kss.superset_includes IS NOT NULL - AND kss.subset_includes IS NOT NULL + AND kss.subset_includes IS NOT NULL THEN /* Create combined includes using XML method that works with all SQL Server versions */ ( @@ -2929,13 +2333,13 @@ NextDatabase: ia1.action = CASE WHEN ia1.is_unique = 0 - THEN N'MAKE UNIQUE' /* Convert to unique index */ - ELSE N'KEEP' /* Already unique, so just keep it */ + THEN 'MAKE UNIQUE' /* Convert to unique index */ + ELSE 'KEEP' /* Already unique, so just keep it */ END FROM #index_analysis AS ia1 WHERE ia1.consolidation_rule IS NULL /* Not already processed */ - AND ia1.action IS NULL /* Not already processed by earlier rules */ - AND EXISTS + AND ia1.action IS NULL /* Not already processed by earlier rules */ + AND EXISTS ( /* Find nonclustered indexes */ SELECT @@ -2946,7 +2350,7 @@ NextDatabase: AND id1.index_id = ia1.index_id AND id1.is_eligible_for_dedupe = 1 ) - AND EXISTS + AND EXISTS ( /* Find unique constraints with matching key columns */ SELECT @@ -3052,16 +2456,14 @@ NextDatabase: UPDATE ia SET - ia.action = NULL, - ia.consolidation_rule = NULL, - ia.target_index_name = NULL + action = NULL, + consolidation_rule = NULL, + target_index_name = NULL FROM #index_analysis AS ia WHERE ia.action = N'MAKE UNIQUE' - AND NOT EXISTS - ( + AND NOT EXISTS ( /* Check if there's a unique constraint with matching keys that points to this index */ - SELECT - 1/0 + SELECT 1 FROM #index_analysis AS ia_uc WHERE ia_uc.database_id = ia.database_id AND ia_uc.object_id = ia.object_id @@ -3077,8 +2479,11 @@ NextDatabase: SET ia_nc.superseded_by = CASE - WHEN ia_nc.superseded_by IS NULL THEN N'Will replace constraint ' + ia_uc.index_name - ELSE ia_nc.superseded_by + N', will replace constraint ' + ia_uc.index_name + WHEN ia_nc.superseded_by IS NULL + THEN N'Will replace constraint ' + + ia_uc.index_name + ELSE ia_nc.superseded_by + + N', will replace constraint ' + ia_uc.index_name END FROM #index_analysis AS ia_nc JOIN #index_analysis AS ia_uc @@ -3096,51 +2501,6 @@ NextDatabase: ia.* FROM #index_analysis AS ia OPTION(RECOMPILE); - - /* Special debug for uq_a and uq_i_a */ - RAISERROR('Special debug for uq_a and uq_i_a after rule 7.5:', 0, 0) WITH NOWAIT; - SELECT - ia.index_name, - ia.action, - ia.consolidation_rule, - ia.target_index_name, - ia.superseded_by, - ia.included_columns, - ia.index_priority - FROM #index_analysis AS ia - WHERE ia.index_name IN (N'uq_a', N'uq_i_a') - ORDER BY - ia.index_name - OPTION(RECOMPILE); - - /* Check the merge script eligibility */ - RAISERROR('Checking MERGE script eligibility for uq_i_a:', 0, 0) WITH NOWAIT; - SELECT - 'uq_i_a eligibility check', - ia.index_name, - ia.action, - ia.target_index_name, - ce.can_compress, - /* Show which conditions are being met for script generation */ - condition1 = CASE WHEN ia.action IN (N'MERGE INCLUDES', N'MAKE UNIQUE') THEN 'YES' ELSE 'NO' END, - condition2 = CASE WHEN ce.can_compress = 1 THEN 'YES' ELSE 'NO' END, - condition3 = CASE WHEN ia.target_index_name IS NULL THEN 'YES' ELSE 'NO' END, - /* Will this index get a MERGE script? */ - will_get_merge_script = - CASE - WHEN ia.action IN (N'MERGE INCLUDES', N'MAKE UNIQUE') - AND ce.can_compress = 1 - AND ia.target_index_name IS NULL - THEN 'YES' - ELSE 'NO' - END - FROM #index_analysis AS ia - JOIN #compression_eligibility AS ce - ON ia.database_id = ce.database_id - AND ia.object_id = ce.object_id - AND ia.index_id = ce.index_id - WHERE ia.index_name = N'uq_i_a' - OPTION(RECOMPILE); END; /* Rule 8: Identify indexes with same keys but in different order after first column */ @@ -3313,11 +2673,11 @@ NextDatabase: candidate.index_name FROM #index_analysis AS candidate WHERE candidate.database_id = ia.database_id - AND candidate.object_id = ia.object_id - AND candidate.key_columns = ia.key_columns - AND ISNULL(candidate.filter_definition, '') = ISNULL(ia.filter_definition, '') - AND candidate.action = N'MERGE INCLUDES' - AND candidate.consolidation_rule = N'Key Duplicate' + AND candidate.object_id = ia.object_id + AND candidate.key_columns = ia.key_columns + AND ISNULL(candidate.filter_definition, '') = ISNULL(ia.filter_definition, '') + AND candidate.action = N'MERGE INCLUDES' + AND candidate.consolidation_rule = N'Key Duplicate' ORDER BY /* Then prefer indexes with more included columns (by length as a proxy) */ LEN(ISNULL(candidate.included_columns, '')) DESC, @@ -3329,16 +2689,16 @@ NextDatabase: STUFF ( ( - SELECT + SELECT N', ' + inner_ia.index_name FROM #index_analysis AS inner_ia WHERE inner_ia.database_id = ia.database_id - AND inner_ia.object_id = ia.object_id - AND inner_ia.key_columns = ia.key_columns - AND ISNULL(inner_ia.filter_definition, '') = ISNULL(ia.filter_definition, '') - AND inner_ia.action = N'MERGE INCLUDES' - AND inner_ia.consolidation_rule = N'Key Duplicate' + AND inner_ia.object_id = ia.object_id + AND inner_ia.key_columns = ia.key_columns + AND ISNULL(inner_ia.filter_definition, '') = ISNULL(ia.filter_definition, '') + AND inner_ia.action = N'MERGE INCLUDES' + AND inner_ia.consolidation_rule = N'Key Duplicate' GROUP BY inner_ia.index_name ORDER BY @@ -3395,8 +2755,7 @@ NextDatabase: REPLACE ( kdd.index_list, - ia.index_name + - N', ', + ia.index_name + N', ', N'' ) /* Remove self from list if present */ FROM #index_analysis AS ia @@ -3750,22 +3109,24 @@ NextDatabase: will_get_script = CASE WHEN ia.action = N'DISABLE' - AND NOT EXISTS + AND NOT EXISTS ( - SELECT - 1/0 + SELECT 1 FROM #index_details AS id_uc WHERE id_uc.database_id = ia.database_id - AND id_uc.object_id = ia.object_id - AND id_uc.index_id = ia.index_id - AND id_uc.is_unique_constraint = 1 + AND id_uc.object_id = ia.object_id + AND id_uc.index_id = ia.index_id + AND id_uc.is_unique_constraint = 1 ) THEN 'YES' ELSE 'NO' END FROM #index_analysis AS ia + WHERE ia.index_name LIKE 'ix_filtered_%' + OR ia.index_name LIKE 'ix_desc_%' ORDER BY - ia.index_name; + ia.index_name + OPTION(RECOMPILE); /* Debug for all indexes marked with action = DISABLE */ RAISERROR('All indexes with action = DISABLE:', 0, 0) WITH NOWAIT; @@ -3777,7 +3138,8 @@ NextDatabase: FROM #index_analysis AS ia WHERE ia.action = N'DISABLE' ORDER BY - ia.index_name; + ia.index_name + OPTION(RECOMPILE); END; INSERT INTO @@ -3808,7 +3170,7 @@ NextDatabase: /* Sort duplicate/subset indexes first (20), then unused indexes last (25) */ sort_order = CASE - WHEN ia.consolidation_rule LIKE N'Unused Index%' THEN 25 + WHEN ia.consolidation_rule LIKE 'Unused Index%' THEN 25 ELSE 20 END, ia.database_name, @@ -4086,9 +3448,10 @@ NextDatabase: AND ir.index_name = ia.index_name ) /* And include only indexes that should be kept */ - AND ( + AND + ( /* Include indexes marked KEEP */ - (ia.action = N'KEEP') + ia.action = N'KEEP' /* And all indexes we haven't determined an action for (not disable, merge, etc.) */ OR ( @@ -4391,7 +3754,8 @@ NextDatabase: additional_info = CASE WHEN ia.consolidation_rule = N'Same Keys Different Order' - THEN N'This index has the same key columns as ' + ISNULL(ia.target_index_name, N'(unknown)') + + THEN N'This index has the same key columns as ' + + ISNULL(ia.target_index_name, N'(unknown)') + N' but in a different order. May be redundant depending on query patterns.' ELSE N'This index needs manual review' END, @@ -4630,8 +3994,6 @@ NextDatabase: RAISERROR('Generating #index_reporting_stats insert, TABLE', 0, 0) WITH NOWAIT; END; - /* No need for a temporary table - we'll use a simpler approach */ - INSERT INTO #index_reporting_stats WITH @@ -4778,7 +4140,97 @@ NextDatabase: Within each category, indexes are sorted by size and impact for better prioritization. */ - /* Save the final output query for later - will run after all databases are processed */ + IF @debug = 1 + BEGIN + RAISERROR('Generating #index_cleanup_results, RESULTS', 0, 0) WITH NOWAIT; + END; + + SELECT + /* First, show the information needed to understand the script */ + script_type = CASE WHEN ir.result_type = 'KEPT' AND ir.script_type IS NULL THEN 'KEPT' ELSE ir.script_type END, + ir.additional_info, + /* Then show identifying information for the index */ + ir.database_name, + ir.schema_name, + ir.table_name, + ir.index_name, + /* Then show relationship information */ + ir.consolidation_rule, + ir.target_index_name, + /* Include superseded_by info for winning indexes */ + superseded_info = + CASE + WHEN ia.superseded_by IS NOT NULL + THEN ia.superseded_by + ELSE ir.superseded_info + END, + /* Add size and usage metrics */ + index_size_gb = + CASE + WHEN ir.result_type = 'SUMMARY' + THEN '0.0000' + ELSE FORMAT(ISNULL(ir.index_size_gb, 0), 'N4') + END, + index_rows = + CASE + WHEN ir.result_type = 'SUMMARY' + THEN '0' + ELSE FORMAT(ISNULL(ir.index_rows, 0), 'N0') + END, + index_reads = + CASE + WHEN ir.result_type = 'SUMMARY' + THEN '0' + ELSE FORMAT(ISNULL(ir.index_reads, 0), 'N0') + END, + index_writes = + CASE + WHEN ir.result_type = 'SUMMARY' + THEN '0' + ELSE FORMAT(ISNULL(ir.index_writes, 0), 'N0') + END, + ia.original_index_definition, + /* Finally show the actual script */ + ir.script + FROM + ( + /* Use a subquery with ROW_NUMBER to ensure we only get one row per index */ + SELECT *, + ROW_NUMBER() OVER( + PARTITION BY database_name, schema_name, table_name, index_name + ORDER BY result_type DESC /* Prefer non-NULL result types */ + ) AS rn + FROM #index_cleanup_results + ) AS ir + LEFT JOIN #index_analysis AS ia + ON ir.database_name = ia.database_name + AND ir.schema_name = ia.schema_name + AND ir.table_name = ia.table_name + AND ir.index_name = ia.index_name + WHERE ir.rn = 1 /* Take only the first row for each index */ + ORDER BY + ir.sort_order, + ir.database_name, + /* Within each sort_order group, prioritize by size and usage */ + CASE + /* For SUMMARY, keep the original order */ + WHEN ir.result_type = 'SUMMARY' + THEN 0 + /* For script categories, order by size and impact */ + ELSE ISNULL(ir.index_size_gb, 0) + END DESC, + CASE + /* For SUMMARY, keep the original order */ + WHEN ir.result_type = 'SUMMARY' + THEN 0 + /* For script categories, consider rows as secondary sort */ + ELSE ISNULL(ir.index_rows, 0) + END DESC, + /* Then by database, schema, table, index name for consistent ordering */ + ir.schema_name, + ir.table_name, + ir.index_name + OPTION(RECOMPILE); /* Insert overall summary information */ IF @debug = 1 @@ -4943,140 +4395,7 @@ NextDatabase: We'll modify the existing query below rather than creating new output panes */ - /* Save the overall analysis report for after all databases are processed */ - - /* Update processed flag for this database */ - UPDATE - #databases_to_process - SET - #databases_to_process.processed = 1, - #databases_to_process.process_date = SYSDATETIME() - WHERE #databases_to_process.database_id = @database_id; - - /* Get next database */ - FETCH NEXT - FROM @database_cursor - INTO - @current_database_id, - @current_database_name; - END; /* End of cursor WHILE loop */ - - IF @debug = 1 - BEGIN - RAISERROR('Finished processing %d databases', 0, 0, @processed_count) WITH NOWAIT; - END; - - /* Create a summary table with database processing info */ - SELECT - database_summary = N'Database Processing Summary', - processed_databases = - N'Processed: ' + - ISNULL - ( - STUFF - ( - ( - SELECT - N', ' + - dtp.database_name - FROM #databases_to_process AS dtp - WHERE dtp.processed = 1 - GROUP BY - dtp.database_name - ORDER BY - dtp.database_name - FOR - XML - PATH(''), - TYPE - ).value('.', 'nvarchar(max)'), - 1, - 2, - N'' - ), - N'None' - ), - skipped_unprocessed = - N'Skipped (unprocessed): ' + - ISNULL - ( - STUFF - ( - ( - SELECT - N', ' + - dtp.database_name - FROM #databases_to_process AS dtp - WHERE dtp.processed = 0 - GROUP BY - dtp.database_name - ORDER BY - dtp.database_name - FOR - XML - PATH(''), - TYPE - ).value('.', 'nvarchar(max)'), - 1, - 2, - N'' - ), - N'None' - ), - skipped_with_reasons = - N'Skipped (excluded): ' + - ISNULL - ( - STUFF - ( - ( - SELECT - N', ' + - sd.database_name + - N' (' + - sd.reason + - N')' - FROM #skipped_databases AS sd - GROUP BY - sd.database_name, - sd.reason - ORDER BY - sd.database_name - FOR - XML - PATH(''), - TYPE - ).value('.', 'nvarchar(max)'), - 1, - 2, - N'' - ), - N'None' - ), - stats = - CASE - WHEN @get_all_databases = 1 - THEN N'Total requested: ' + - CONVERT(nvarchar(10), COUNT_BIG(*) OVER() + - (SELECT COUNT_BIG(*) FROM #skipped_databases AS sd)) + - N', Processed: ' + - CONVERT(nvarchar(10), SUM(CONVERT(integer, dtp.processed)) OVER()) + - N', Skipped (unprocessed): ' + - CONVERT(nvarchar(10), COUNT_BIG(*) OVER() - - SUM(CONVERT(integer, dtp.processed)) OVER()) + - N', Skipped (excluded): ' + - CONVERT(nvarchar(10), (SELECT COUNT_BIG(*) FROM #skipped_databases AS sd)) - ELSE N'Single database mode' - END - FROM #databases_to_process AS dtp - WHERE @database_count > 0 /* Return one row with summary data */ - GROUP BY - dtp.processed - OPTION(RECOMPILE); - END; /* End of @get_all_databases = 1 section */ - -GenerateResults: - /* Return consolidated reporting statistics for all databases processed */ + /* Return streamlined reporting statistics focused on key metrics */ IF @debug = 1 BEGIN RAISERROR('Generating #index_reporting_stats, REPORT', 0, 0) WITH NOWAIT; @@ -5263,113 +4582,7 @@ GenerateResults: irs.schema_name, irs.table_name OPTION(RECOMPILE); - - /* Final unified results output - runs once after all databases processed */ - IF @debug = 1 - BEGIN - RAISERROR('Generating final consolidated output for all databases', 0, 0) WITH NOWAIT; - END; - SELECT - /* First, show the information needed to understand the script */ - script_type = - CASE - WHEN ir.result_type = 'KEPT' - AND ir.script_type IS NULL - THEN 'KEPT' - ELSE ir.script_type - END, - ir.additional_info, - /* Then show identifying information for the index */ - ir.database_name, - ir.schema_name, - ir.table_name, - ir.index_name, - /* Then show relationship information */ - ir.consolidation_rule, - ir.target_index_name, - /* Include superseded_by info for winning indexes */ - superseded_info = - CASE - WHEN ia.superseded_by IS NOT NULL - THEN ia.superseded_by - ELSE ir.superseded_info - END, - /* Add size and usage metrics */ - index_size_gb = - CASE - WHEN ir.result_type = 'SUMMARY' - THEN '0.0000' - ELSE FORMAT(ISNULL(ir.index_size_gb, 0), 'N4') - END, - index_rows = - CASE - WHEN ir.result_type = 'SUMMARY' - THEN '0' - ELSE FORMAT(ISNULL(ir.index_rows, 0), 'N0') - END, - index_reads = - CASE - WHEN ir.result_type = 'SUMMARY' - THEN '0' - ELSE FORMAT(ISNULL(ir.index_reads, 0), 'N0') - END, - index_writes = - CASE - WHEN ir.result_type = 'SUMMARY' - THEN '0' - ELSE FORMAT(ISNULL(ir.index_writes, 0), 'N0') - END, - ia.original_index_definition, - /* Finally show the actual script */ - ir.script - FROM - ( - /* Use a subquery with ROW_NUMBER to ensure we only get one row per index */ - SELECT - icr.*, - rn = - ROW_NUMBER() OVER - ( - PARTITION BY - icr.database_name, - icr.schema_name, - icr.table_name, - icr.index_name - ORDER BY - icr.result_type DESC /* Prefer non-NULL result types */ - ) - FROM #index_cleanup_results AS icr - ) AS ir - LEFT JOIN #index_analysis AS ia - ON ir.database_name = ia.database_name - AND ir.schema_name = ia.schema_name - AND ir.table_name = ia.table_name - AND ir.index_name = ia.index_name - WHERE ir.rn = 1 /* Take only the first row for each index */ - ORDER BY - ir.sort_order, - ir.database_name, - /* Within each sort_order group, prioritize by size and usage */ - CASE - /* For SUMMARY, keep the original order */ - WHEN ir.result_type = 'SUMMARY' - THEN 0 - /* For script categories, order by size and impact */ - ELSE ISNULL(ir.index_size_gb, 0) - END DESC, - CASE - /* For SUMMARY, keep the original order */ - WHEN ir.result_type = 'SUMMARY' - THEN 0 - /* For script categories, consider rows as secondary sort */ - ELSE ISNULL(ir.index_rows, 0) - END DESC, - /* Then by database, schema, table, index name for consistent ordering */ - ir.schema_name, - ir.table_name, - ir.index_name - OPTION(RECOMPILE); END TRY BEGIN CATCH THROW; diff --git a/sp_IndexCleanup/sp_IndexCleanup_Old.sql b/sp_IndexCleanup/sp_IndexCleanup_Old.sql index aac0143f..acaa3d27 100644 --- a/sp_IndexCleanup/sp_IndexCleanup_Old.sql +++ b/sp_IndexCleanup/sp_IndexCleanup_Old.sql @@ -1,14 +1,3 @@ -/* -EXECUTE sp_IndexCleanup - @database_name = 'StackOverflow2013', - @debug = 1; - -EXECUTE sp_IndexCleanup - @database_name = 'StackOverflow2013', - @table_name = 'Users', - @debug = 1 -*/ - SET ANSI_WARNINGS ON; SET ARITHABORT ON; SET CONCAT_NULL_YIELDS_NULL ON; @@ -79,28 +68,8 @@ BEGIN TRY @version_date = '20250401'; SELECT - for_insurance_purposes = N'Read the messages pane carefully!'; - - PRINT N' -------------------------------------------------------------------------------------------- -------------------------------------------------------------------------------------------- -------------------------------------------------------------------------------------------- -This is the BETA VERSION of sp_IndexCleanup - -It needs lots of love and testing in real environments with real indexes to fix many issues: - * Data collection - * Deduping logic - * Result correctness - * Edge cases - * May not account for specific query patterns that benefit from seemingly redundant indexes - -ALWAYS TEST THESE RECOMMENDATIONS IN A NON-PRODUCTION ENVIRONMENT FIRST" - -------------------------------------------------------------------------------------------- -------------------------------------------------------------------------------------------- -------------------------------------------------------------------------------------------- -'; - + for_insurance_purposes = + N'ALWAYS TEST THESE RECOMMENDATIONS IN A NON-PRODUCTION ENVIRONMENT FIRST!'; /* Help section, for help. @@ -109,16 +78,19 @@ ALWAYS TEST THESE RECOMMENDATIONS IN A NON-PRODUCTION ENVIRONMENT FIRST" IF @help = 1 BEGIN SELECT - help = N'hello, i am sp_IndexCleanup - BETA' + help = N'hello, i am sp_IndexCleanup' UNION ALL SELECT - help = N'this is a script to help clean up unused and duplicate indexes' + help = N'this is a script to help clean up unused and duplicate indexes.' UNION ALL SELECT - help = N'you are currently using a beta version, and the advice should not be followed' + help = N'it will also give you scripted out statements to add page compression to uncompressed indexes.' UNION ALL SELECT - help = N'without careful analysis and consideration. it may be harmful.'; + help = N'always validate all changes against a non-production environment!' + UNION ALL + SELECT + help = N'without careful analysis and consideration, index changes can negative impacts on performance.'; /* Parameters @@ -152,7 +124,7 @@ ALWAYS TEST THESE RECOMMENDATIONS IN A NON-PRODUCTION ENVIRONMENT FIRST" WHEN N'@table_name' THEN 'table name or NULL for all tables' WHEN N'@min_reads' THEN 'any positive integer or 0' WHEN N'@min_writes' THEN 'any positive integer or 0' - WHEN N'@min_size_gb' THEN 'any positive decimal number or 0' + WHEN N'@min_size_gb' THEN 'any positive decimal or 0' WHEN N'@min_rows' THEN 'any positive integer or 0' WHEN N'@help' THEN '0 or 1' WHEN N'@debug' THEN '0 or 1' @@ -196,7 +168,7 @@ ALWAYS TEST THESE RECOMMENDATIONS IN A NON-PRODUCTION ENVIRONMENT FIRST" RAISERROR(' MIT License -Copyright 2024 Darling Data, LLC +Copyright 2025 Darling Data, LLC https://www.erikdarling.com/ @@ -244,11 +216,32 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. @can_compress bit = CASE WHEN - CONVERT(integer, SERVERPROPERTY('EngineEdition')) IN (3, 5, 8) + CONVERT + ( + integer, + SERVERPROPERTY('EngineEdition') + ) IN (3, 5, 8) OR ( - CONVERT(integer, SERVERPROPERTY('EngineEdition')) = 2 - AND CONVERT(integer, SUBSTRING(CONVERT(varchar(20), SERVERPROPERTY('ProductVersion')), 1, 2)) >= 13 + CONVERT + ( + integer, + SERVERPROPERTY('EngineEdition') + ) = 2 + AND CONVERT + ( + integer, + SUBSTRING + ( + CONVERT + ( + varchar(20), + SERVERPROPERTY('ProductVersion') + ), + 1, + 2 + ) + ) >= 13 ) THEN 1 ELSE 0 @@ -301,13 +294,21 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SELECT @database_id = d.database_id FROM sys.databases AS d - WHERE d.name = @database_name + WHERE d.name NOT IN (N'master', N'model', N'msdb', N'tempdb', 'rdsadmin') + AND d.state = 0 + AND d.is_in_standby = 0 + AND d.is_read_only = 0 OPTION(RECOMPILE); END; IF @schema_name IS NULL AND @table_name IS NOT NULL BEGIN + IF @debug = 1 + BEGIN + RAISERROR('Parameter @schema_name cannot be NULL when specifying a table, defaulting to dbo', 10, 1) WITH NOWAIT; + END; + SELECT @schema_name = N'dbo'; END; @@ -315,6 +316,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. IF @schema_name IS NOT NULL AND @table_name IS NOT NULL BEGIN + IF @debug = 1 + BEGIN + RAISERROR('validating object existence for %s.%s.&s.', 0, 0, @database_name, @schema_name, @table_name) WITH NOWAIT; + END; + SELECT @full_object_name = QUOTENAME(@database_name) + @@ -338,28 +344,44 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. IF @min_reads < 0 OR @min_reads IS NULL BEGIN - RAISERROR('Parameter @min_reads cannot be NULL or negative. Setting to 0.', 10, 1) WITH NOWAIT; + IF @debug = 1 + BEGIN + RAISERROR('Parameter @min_reads cannot be NULL or negative. Setting to 0.', 10, 1) WITH NOWAIT; + END; + SET @min_reads = 0; END; IF @min_writes < 0 OR @min_writes IS NULL BEGIN - RAISERROR('Parameter @min_writes cannot be NULL or negative. Setting to 0.', 10, 1) WITH NOWAIT; + IF @debug = 1 + BEGIN + RAISERROR('Parameter @min_writes cannot be NULL or negative. Setting to 0.', 10, 1) WITH NOWAIT; + END; + SET @min_writes = 0; END; IF @min_size_gb < 0 OR @min_size_gb IS NULL BEGIN - RAISERROR('Parameter @min_size_gb cannot be NULL or negative. Setting to 0.', 10, 1) WITH NOWAIT; + IF @debug = 1 + BEGIN + RAISERROR('Parameter @min_size_gb cannot be NULL or negative. Setting to 0.', 10, 1) WITH NOWAIT; + END; + SET @min_size_gb = 0; END; IF @min_rows < 0 OR @min_rows IS NULL BEGIN - RAISERROR('Parameter @min_rows cannot be NULL or negative. Setting to 0.', 10, 1) WITH NOWAIT; + IF @debug = 1 + BEGIN + RAISERROR('Parameter @min_rows cannot be NULL or negative. Setting to 0.', 10, 1) WITH NOWAIT; + END; + SET @min_rows = 0; END; @@ -443,7 +465,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. index_id integer NOT NULL, index_name sysname NULL, partition_id bigint NOT NULL, - partition_number int NOT NULL, + partition_number integer NOT NULL, total_rows bigint NULL, total_space_gb decimal(38, 4) NULL, /* Using 4 decimal places for GB to maintain precision */ reserved_lob_gb decimal(38, 4) NULL, /* Using 4 decimal places for GB to maintain precision */ @@ -500,21 +522,31 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. schema_name sysname NOT NULL, object_id integer NOT NULL, table_name sysname NOT NULL, - index_id integer NULL, + index_id integer NOT NULL, index_name sysname NOT NULL, is_unique bit NULL, - key_columns nvarchar(max) NULL, - included_columns nvarchar(max) NULL, - filter_definition nvarchar(max) NULL, - is_redundant bit NULL, - superseded_by nvarchar(256) NULL, - missing_columns nvarchar(max) NULL, - action nvarchar(30) NULL, + key_columns nvarchar(MAX) NULL, + included_columns nvarchar(MAX) NULL, + filter_definition nvarchar(MAX) NULL, + /* Query plan for original CREATE INDEX statement */ + original_index_definition nvarchar(MAX) NULL, + /* + Consolidation rule that matched (e.g., Key Duplicate, Key Subset, etc) + For exact duplicates, use one of: Exact Duplicate, Reverse Duplicate, or Equal Except For Filter + */ + consolidation_rule nvarchar(256) NULL, + /* + Action to take (e.g., DISABLE, MERGE INCLUDES, KEEP) + If NULL, no action to be taken + */ + action nvarchar(100) NULL, + /* Target index to merge with or use instead of this one */ target_index_name sysname NULL, - consolidation_rule varchar(512) NULL, - index_priority int NULL, - original_index_definition nvarchar(max) NULL, /* Original CREATE INDEX statement */ - INDEX c CLUSTERED (database_id, schema_id, object_id, index_id) + /* When this is a target, the index which points to it as a supersedes in consolidation */ + superseded_by nvarchar(4000) NULL, + /* Priority score from 0-1 to determine which index to keep (higher is better) */ + index_priority decimal(10,6) NULL + PRIMARY KEY CLUSTERED(database_id, object_id, index_id) ); CREATE TABLE @@ -533,6 +565,28 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. PRIMARY KEY CLUSTERED(database_id, object_id, index_id) ); + CREATE TABLE + #index_cleanup_results + ( + result_type varchar(100) NOT NULL, + sort_order integer NOT NULL, + database_name sysname NULL, + schema_name sysname NULL, + table_name sysname NULL, + index_name sysname NULL, + script_type nvarchar(60) NULL, /* Type of script (e.g., MERGE SCRIPT, DISABLE SCRIPT, etc.) */ + consolidation_rule nvarchar(256) NULL, /* Reason for action (e.g., Exact Duplicate, Key Subset) */ + target_index_name sysname NULL, /* If this index is a duplicate, indicates which index is the preferred one */ + superseded_info nvarchar(4000) NULL, /* If this is a kept index, indicates which indexes it supersedes */ + additional_info nvarchar(max) NULL, /* Additional information about the action */ + original_index_definition nvarchar(max) NULL, /* Original statement to create the index */ + index_size_gb decimal(38, 4) NULL, /* Size of the index in GB */ + index_rows bigint NULL, /* Number of rows in the index */ + index_reads bigint NULL, /* Total reads (seeks + scans + lookups) */ + index_writes bigint NULL, /* Total writes */ + script nvarchar(max) NULL /* Script to execute the action */ + ); + CREATE TABLE #key_duplicate_dedupe ( @@ -558,28 +612,6 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. superset_included_columns nvarchar(max) NULL ); - CREATE TABLE - #index_cleanup_results - ( - result_type varchar(50) NOT NULL, /* 'SUMMARY', 'MERGE', 'DISABLE', 'COMPRESS', etc. */ - sort_order integer NOT NULL, /* Keeps results in logical order */ - database_name sysname NULL, - schema_name sysname NULL, - table_name sysname NULL, - index_name sysname NULL, - script_type nvarchar(50) NULL, /* 'MERGE', 'DISABLE', 'COMPRESS', etc. */ - consolidation_rule nvarchar(256) NULL, - target_index_name sysname NULL, - script nvarchar(max) NULL, - additional_info nvarchar(max) NULL, /* For stats, constraints, etc. */ - superseded_info nvarchar(max) NULL, /* To store superseded_by information */ - original_index_definition nvarchar(max) NULL, /* Original index definition for validation */ - index_size_gb decimal(18,4) NULL, /* Size of the index in GB */ - index_rows bigint NULL, /* Number of rows in the index */ - index_reads bigint NULL, /* Total reads (seeks + scans + lookups) */ - index_writes bigint NULL /* Total writes (updates) */ - ); - /* Create a new temp table for detailed reporting statistics */ CREATE TABLE #index_reporting_stats @@ -589,16 +621,16 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. schema_name sysname NULL, table_name sysname NULL, index_name sysname NULL, - server_uptime_days int NULL, + server_uptime_days integer NULL, uptime_warning bit NULL, - tables_analyzed int NULL, - index_count int NULL, + tables_analyzed integer NULL, + index_count integer NULL, total_size_gb decimal(38, 4) NULL, total_rows bigint NULL, - unused_indexes int NULL, + unused_indexes integer NULL, unused_size_gb decimal(38, 4) NULL, - indexes_to_disable int NULL, - indexes_to_merge int NULL, + indexes_to_disable integer NULL, + indexes_to_merge integer NULL, avg_indexes_per_table decimal(10, 2) NULL, space_saved_gb decimal(10, 4) NULL, compression_min_savings_gb decimal(10, 4) NULL, @@ -686,7 +718,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. WHERE v.object_id = i.object_id )'; - IF /* Check SQL Server 2016+ for temporal tables support */ + IF /* Check for temporal tables support */ ( CONVERT ( @@ -709,6 +741,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ) >= 13 ) /* SQL 2016+ */ BEGIN + IF @debug = 1 + BEGIN + RAISERROR('adding temporal table screening', 0, 0) WITH NOWAIT; + END; + SET @sql += N' AND NOT EXISTS ( @@ -723,6 +760,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. IF @object_id IS NOT NULL BEGIN + IF @debug = 1 + BEGIN + RAISERROR('adding object_id filter', 0, 0) WITH NOWAIT; + END; + SELECT @sql += N' AND t.object_id = @object_id'; END; @@ -736,8 +778,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. JOIN ' + QUOTENAME(@database_name) + N'.sys.allocation_units AS au ON ps.partition_id = au.container_id WHERE ps.object_id = t.object_id - GROUP - BY ps.object_id + GROUP BY + ps.object_id HAVING SUM(au.total_pages) * 8.0 / 1048576.0 >= @min_size_gb ) @@ -748,8 +790,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. FROM ' + QUOTENAME(@database_name) + N'.sys.dm_db_partition_stats AS ps WHERE ps.object_id = t.object_id AND ps.index_id IN (0, 1) - GROUP - BY ps.object_id + GROUP BY + ps.object_id HAVING SUM(ps.row_count) >= @min_rows ) @@ -767,7 +809,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. OR SUM(ius.user_updates) >= @min_writes ) - OPTION(RECOMPILE);'; + OPTION(RECOMPILE); + '; IF @debug = 1 BEGIN @@ -858,12 +901,17 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* If SQL Server edition doesn't support compression, mark all as ineligible */ IF @can_compress = 0 BEGIN + IF @debug = 1 + BEGIN + RAISERROR('updating compression eligibility', 0, 0) WITH NOWAIT; + END; + UPDATE #compression_eligibility SET - can_compress = 0, - reason = N'SQL Server edition or version does not support compression' - WHERE can_compress = 1 + #compression_eligibility.can_compress = 0, + #compression_eligibility.reason = N'SQL Server edition or version does not support compression' + WHERE #compression_eligibility.can_compress = 1 OPTION(RECOMPILE); END; @@ -998,7 +1046,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. t.name, os.index_id, i.name - OPTION(RECOMPILE);'; + OPTION(RECOMPILE); + '; IF @debug = 1 BEGIN @@ -1218,6 +1267,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. IF @object_id IS NOT NULL BEGIN + IF @debug = 1 + BEGIN + RAISERROR('adding object+id filter', 0, 0) WITH NOWAIT; + END; + SELECT @sql += N' AND t.object_id = @object_id'; END; @@ -1236,7 +1290,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND so.is_ms_shipped = 0 AND so.type = N''TF'' ) - OPTION(RECOMPILE);' + OPTION(RECOMPILE); + ' ); IF @debug = 1 @@ -1380,6 +1435,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. IF @object_id IS NOT NULL BEGIN + IF @debug = 1 + BEGIN + RAISERROR('adding in object_id filter', 0, 0) WITH NOWAIT; + END; + SELECT @sql += N' AND t.object_id = @object_id'; END; @@ -1445,7 +1505,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. '''' ) ) AS pc - OPTION(RECOMPILE);'; + OPTION(RECOMPILE); + '; IF @debug = 1 BEGIN @@ -1728,14 +1789,14 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SET #index_analysis.index_priority = CASE - WHEN index_id = 1 + WHEN #index_analysis.index_id = 1 THEN 1000 /* Clustered indexes get highest priority */ ELSE 0 END + CASE /* Unique indexes get high priority, but reduce priority for unique constraints */ - WHEN is_unique = 1 AND NOT EXISTS + WHEN #index_analysis.is_unique = 1 AND NOT EXISTS ( SELECT 1/0 @@ -1745,7 +1806,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND id_uc.is_unique_constraint = 1 ) THEN 500 /* Unique constraints get lower priority */ - WHEN is_unique = 1 AND EXISTS + WHEN #index_analysis.is_unique = 1 AND EXISTS ( SELECT 1/0 @@ -1834,22 +1895,24 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. UPDATE ia1 SET - ia1.consolidation_rule = 'Exact Duplicate', + ia1.consolidation_rule = N'Exact Duplicate', ia1.target_index_name = CASE WHEN ia1.index_priority > ia2.index_priority THEN NULL /* This index is the keeper */ - WHEN ia1.index_priority = ia2.index_priority AND ia1.index_name < ia2.index_name + WHEN ia1.index_priority = ia2.index_priority + AND ia1.index_name < ia2.index_name THEN NULL /* When tied, use alphabetical ordering for consistency */ ELSE ia2.index_name /* Other index is the keeper */ END, ia1.action = CASE WHEN ia1.index_priority > ia2.index_priority - THEN 'KEEP' /* This index is the keeper */ - WHEN ia1.index_priority = ia2.index_priority AND ia1.index_name < ia2.index_name - THEN 'KEEP' /* When tied, use alphabetical ordering for consistency */ - ELSE 'DISABLE' /* Other index gets disabled */ + THEN N'KEEP' /* This index is the keeper */ + WHEN ia1.index_priority = ia2.index_priority + AND ia1.index_name < ia2.index_name + THEN N'KEEP' /* When tied, use alphabetical ordering for consistency */ + ELSE N'DISABLE' /* Other index gets disabled */ END FROM #index_analysis AS ia1 JOIN #index_analysis AS ia2 @@ -1935,8 +1998,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND ia1.key_columns = ia2.key_columns /* Exact key match */ AND ISNULL(ia1.included_columns, '') = ISNULL(ia2.included_columns, '') /* Exact includes match */ AND ISNULL(ia1.filter_definition, '') = ISNULL(ia2.filter_definition, '') /* Matching filters */ - WHERE ia1.consolidation_rule = 'Exact Duplicate' - OR ia2.consolidation_rule = 'Exact Duplicate' + WHERE ia1.consolidation_rule = N'Exact Duplicate' + OR ia2.consolidation_rule = N'Exact Duplicate' ORDER BY ia1.index_name OPTION(RECOMPILE); END; @@ -1945,7 +2008,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. UPDATE ia1 SET - ia1.consolidation_rule = 'Key Duplicate', + ia1.consolidation_rule = N'Key Duplicate', ia1.target_index_name = CASE /* If one is unique and the other isn't, prefer the unique one */ @@ -1969,8 +2032,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND NOT (ia1.is_unique = 0 AND ia2.is_unique = 1) ) AND ISNULL(ia1.included_columns, N'') <> ISNULL(ia2.included_columns, N'') - THEN 'MERGE INCLUDES' /* Keep this index but merge includes */ - ELSE 'DISABLE' /* Other index is keeper, disable this one */ + THEN N'MERGE INCLUDES' /* Keep this index but merge includes */ + ELSE N'DISABLE' /* Other index is keeper, disable this one */ END, /* For the winning index, set clear superseded_by text for the report */ ia1.superseded_by = @@ -1981,7 +2044,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ia1.index_priority >= ia2.index_priority AND NOT (ia1.is_unique = 0 AND ia2.is_unique = 1) ) - THEN 'Supersedes ' + + THEN N'Supersedes ' + ia2.index_name ELSE NULL END @@ -2051,7 +2114,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. UPDATE ia1 SET - ia1.consolidation_rule = 'Key Subset', + ia1.consolidation_rule = N'Key Subset', ia1.target_index_name = ia2.index_name, ia1.action = N'DISABLE' /* The narrower index gets disabled */ FROM #index_analysis AS ia1 @@ -2100,16 +2163,24 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. UPDATE ia2 SET - ia2.consolidation_rule = 'Key Superset', + ia2.consolidation_rule = N'Key Superset', ia2.action = N'MERGE INCLUDES', /* The wider index gets merged with includes */ - ia2.superseded_by = COALESCE(ia2.superseded_by + ', ', '') + 'Supersedes ' + ia1.index_name + ia2.superseded_by = + ISNULL + ( + ia2.superseded_by + + ', ', + '' + ) + + N'Supersedes ' + + ia1.index_name FROM #index_analysis AS ia1 JOIN #index_analysis AS ia2 ON ia1.database_id = ia2.database_id AND ia1.object_id = ia2.object_id AND ia1.target_index_name = ia2.index_name /* Link from Rule 4 */ - WHERE ia1.consolidation_rule = 'Key Subset' - AND ia1.action = 'DISABLE' + WHERE ia1.consolidation_rule = N'Key Subset' + AND ia1.action = N'DISABLE' AND ia2.consolidation_rule IS NULL /* Not already processed */ OPTION(RECOMPILE); @@ -2123,7 +2194,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END; /* Rule 6: Merge includes from subset to superset indexes */ - WITH KeySubsetSuperset AS + WITH + KeySubsetSuperset AS ( SELECT superset.database_id, @@ -2137,10 +2209,10 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ON superset.database_id = subset.database_id AND superset.object_id = subset.object_id AND subset.target_index_name = superset.index_name - WHERE superset.action = 'MERGE INCLUDES' - AND subset.action = 'DISABLE' - AND superset.consolidation_rule = 'Key Superset' - AND subset.consolidation_rule = 'Key Subset' + WHERE superset.action = N'MERGE INCLUDES' + AND subset.action = N'DISABLE' + AND superset.consolidation_rule = N'Key Superset' + AND subset.consolidation_rule = N'Key Subset' ) UPDATE ia @@ -2160,7 +2232,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ( ( SELECT DISTINCT - N', ' + t.c.value('.', 'sysname') + N', ' + + t.c.value('.', 'sysname') FROM ( /* Create XML from superset includes */ @@ -2197,7 +2270,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ) ) /* If only subset has includes, use those */ - WHEN kss.superset_includes IS NULL AND kss.subset_includes IS NOT NULL + WHEN kss.superset_includes IS NULL + AND kss.subset_includes IS NOT NULL THEN kss.subset_includes /* If only superset has includes or neither has includes, keep superset's includes */ ELSE kss.superset_includes @@ -2207,7 +2281,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ON ia.database_id = kss.database_id AND ia.object_id = kss.object_id AND ia.index_id = kss.index_id - WHERE ia.action = 'MERGE INCLUDES'; + WHERE ia.action = N'MERGE INCLUDES' + OPTION(RECOMPILE); IF @debug = 1 BEGIN @@ -2222,17 +2297,17 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. UPDATE ia2 SET - ia2.superseded_by = 'Supersedes ' + ia1.index_name + ia2.superseded_by = N'Supersedes ' + ia1.index_name FROM #index_analysis AS ia1 JOIN #index_analysis AS ia2 ON ia1.database_id = ia2.database_id AND ia1.object_id = ia2.object_id AND ia1.index_name <> ia2.index_name - AND ia2.key_columns LIKE (ia1.key_columns + '%') /* ia2 has wider key that starts with ia1's key */ + AND ia2.key_columns LIKE (ia1.key_columns + N'%') /* ia2 has wider key that starts with ia1's key */ AND ISNULL(ia1.filter_definition, '') = ISNULL(ia2.filter_definition, '') /* Matching filters */ /* Exception: If narrower index is unique and wider is not, they should not be merged */ AND NOT (ia1.is_unique = 1 AND ia2.is_unique = 0) - WHERE ia1.consolidation_rule = 'Key Subset' /* Use records just processed in previous UPDATE */ + WHERE ia1.consolidation_rule = N'Key Subset' /* Use records just processed in previous UPDATE */ AND ia1.target_index_name = ia2.index_name /* Make sure we're updating the right wider index */ OPTION(RECOMPILE); @@ -2249,7 +2324,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. UPDATE ia1 SET - ia1.consolidation_rule = 'Unique Constraint Replacement', + ia1.consolidation_rule = N'Unique Constraint Replacement', ia1.action = CASE WHEN ia1.is_unique = 0 @@ -2317,7 +2392,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. UPDATE ia_uc SET - ia_uc.consolidation_rule = 'Unique Constraint Replacement', + ia_uc.consolidation_rule = N'Unique Constraint Replacement', ia_uc.action = N'DISABLE', /* Mark unique constraint for disabling */ ia_uc.target_index_name = ia_nc.index_name /* Point to the nonclustered index that will replace it */ FROM #index_analysis AS ia_uc /* Unique constraint */ @@ -2339,7 +2414,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. UPDATE ia_nc SET - ia_nc.consolidation_rule = 'Unique Constraint Replacement', + ia_nc.consolidation_rule = N'Unique Constraint Replacement', ia_nc.action = N'MAKE UNIQUE', /* Mark nonclustered index to be made unique */ /* CRITICAL: Set target_index_name to NULL to ensure it gets a MERGE script */ ia_nc.target_index_name = NULL @@ -2353,27 +2428,30 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Two conditions for matching: 1. Index key columns exactly match a unique constraint's key columns 2. A unique constraint is already marked for DISABLE and has this index as target */ - (EXISTS ( + EXISTS + ( /* Find unique constraint with matching keys that should be disabled */ - SELECT 1 + SELECT + 1/0 FROM #index_analysis AS ia_uc JOIN #index_details AS id_uc ON id_uc.database_id = ia_uc.database_id AND id_uc.object_id = ia_uc.object_id AND id_uc.index_id = ia_uc.index_id AND id_uc.is_unique_constraint = 1 - WHERE - ia_uc.database_id = ia_nc.database_id - AND ia_uc.object_id = ia_nc.object_id - /* Check that both indexes have EXACTLY the same key columns */ - AND ia_uc.key_columns = ia_nc.key_columns - )) + WHERE ia_uc.database_id = ia_nc.database_id + AND ia_uc.object_id = ia_nc.object_id + /* Check that both indexes have EXACTLY the same key columns */ + AND ia_uc.key_columns = ia_nc.key_columns + ) OPTION(RECOMPILE); /* CRITICAL: Ensure that only the unique constraints that exactly match get this treatment */ /* And remove any incorrect MAKE UNIQUE actions */ - UPDATE ia - SET action = NULL, + UPDATE + ia + SET + action = NULL, consolidation_rule = NULL, target_index_name = NULL FROM #index_analysis AS ia @@ -2383,19 +2461,24 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SELECT 1 FROM #index_analysis AS ia_uc WHERE ia_uc.database_id = ia.database_id - AND ia_uc.object_id = ia.object_id - AND ia_uc.key_columns = ia.key_columns - AND ia_uc.action = N'DISABLE' - AND ia_uc.target_index_name = ia.index_name - ); + AND ia_uc.object_id = ia.object_id + AND ia_uc.key_columns = ia.key_columns + AND ia_uc.action = N'DISABLE' + AND ia_uc.target_index_name = ia.index_name + ) + OPTION(RECOMPILE); /* Make sure the nonclustered index has the superseded_by field set correctly */ - UPDATE ia_nc + UPDATE + ia_nc SET ia_nc.superseded_by = CASE - WHEN ia_nc.superseded_by IS NULL THEN N'Will replace constraint ' + ia_uc.index_name - ELSE ia_nc.superseded_by + N', will replace constraint ' + ia_uc.index_name + WHEN ia_nc.superseded_by IS NULL + THEN N'Will replace constraint ' + + ia_uc.index_name + ELSE ia_nc.superseded_by + + N', will replace constraint ' + ia_uc.index_name END FROM #index_analysis AS ia_nc JOIN #index_analysis AS ia_uc @@ -2413,50 +2496,6 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ia.* FROM #index_analysis AS ia OPTION(RECOMPILE); - - /* Special debug for uq_a and uq_i_a */ - RAISERROR('Special debug for uq_a and uq_i_a after rule 7.5:', 0, 0) WITH NOWAIT; - SELECT - index_name, - action, - consolidation_rule, - target_index_name, - superseded_by, - included_columns, - index_priority - FROM #index_analysis - WHERE index_name IN ('uq_a', 'uq_i_a') - ORDER BY index_name - OPTION(RECOMPILE); - - /* Check the merge script eligibility */ - RAISERROR('Checking MERGE script eligibility for uq_i_a:', 0, 0) WITH NOWAIT; - SELECT - 'uq_i_a eligibility check', - ia.index_name, - ia.action, - ia.target_index_name, - ce.can_compress, - /* Show which conditions are being met for script generation */ - condition1 = CASE WHEN ia.action IN (N'MERGE INCLUDES', N'MAKE UNIQUE') THEN 'YES' ELSE 'NO' END, - condition2 = CASE WHEN ce.can_compress = 1 THEN 'YES' ELSE 'NO' END, - condition3 = CASE WHEN ia.target_index_name IS NULL THEN 'YES' ELSE 'NO' END, - /* Will this index get a MERGE script? */ - will_get_merge_script = - CASE - WHEN ia.action IN (N'MERGE INCLUDES', N'MAKE UNIQUE') - AND ce.can_compress = 1 - AND ia.target_index_name IS NULL - THEN 'YES' - ELSE 'NO' - END - FROM #index_analysis AS ia - JOIN #compression_eligibility AS ce - ON ia.database_id = ce.database_id - AND ia.object_id = ce.object_id - AND ia.index_id = ce.index_id - WHERE ia.index_name = 'uq_i_a' - OPTION(RECOMPILE); END; /* Rule 8: Identify indexes with same keys but in different order after first column */ @@ -2465,7 +2504,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. UPDATE ia1 SET - ia1.consolidation_rule = 'Same Keys Different Order', + ia1.consolidation_rule = N'Same Keys Different Order', ia1.action = N'REVIEW', /* These need manual review */ ia1.target_index_name = ia2.index_name /* Reference the partner index */ FROM #index_analysis AS ia1 @@ -2501,10 +2540,10 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. id1.column_name FROM #index_details AS id1 WHERE id1.database_id = ia1.database_id - AND id1.object_id = ia1.object_id - AND id1.index_id = ia1.index_id - AND id1.is_included_column = 0 - AND id1.key_ordinal > 0 + AND id1.object_id = ia1.object_id + AND id1.index_id = ia1.index_id + AND id1.is_included_column = 0 + AND id1.key_ordinal > 0 EXCEPT @@ -2562,6 +2601,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. schema_name, table_name, index_name, + consolidation_rule, script_type, additional_info, target_index_name, @@ -2579,6 +2619,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. schema_name = '', table_name = '', index_name = '', + consolidation_rule = N'', script_type = 'Index Cleanup Scripts', additional_info = N'A detailed index analysis report appears after these scripts', target_index_name = '', @@ -2631,10 +2672,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND candidate.key_columns = ia.key_columns AND ISNULL(candidate.filter_definition, '') = ISNULL(ia.filter_definition, '') AND candidate.action = N'MERGE INCLUDES' - AND candidate.consolidation_rule = 'Key Duplicate' + AND candidate.consolidation_rule = N'Key Duplicate' ORDER BY - /* First prefer indexes with "_Extended" in the name */ - CASE WHEN candidate.index_name LIKE '%\_Extended%' ESCAPE '\' THEN 1 ELSE 0 END DESC, /* Then prefer indexes with more included columns (by length as a proxy) */ LEN(ISNULL(candidate.included_columns, '')) DESC, /* Then alphabetically for stability */ @@ -2654,7 +2693,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND inner_ia.key_columns = ia.key_columns AND ISNULL(inner_ia.filter_definition, '') = ISNULL(ia.filter_definition, '') AND inner_ia.action = N'MERGE INCLUDES' - AND inner_ia.consolidation_rule = 'Key Duplicate' + AND inner_ia.consolidation_rule = N'Key Duplicate' + GROUP BY + inner_ia.index_name ORDER BY inner_ia.index_name FOR @@ -2668,7 +2709,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ) FROM #index_analysis AS ia WHERE ia.action = N'MERGE INCLUDES' - AND ia.consolidation_rule = 'Key Duplicate' + AND ia.consolidation_rule = N'Key Duplicate' GROUP BY ia.database_id, ia.object_id, @@ -2697,19 +2738,20 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND ia.key_columns = kdd.base_key_columns AND ISNULL(ia.filter_definition, N'') = kdd.filter_definition WHERE ia.index_name <> kdd.winning_index_name - AND ia.action = N'MERGE INCLUDES' - AND ia.consolidation_rule = 'Key Duplicate' + AND ia.action = N'MERGE INCLUDES' + AND ia.consolidation_rule = N'Key Duplicate' OPTION(RECOMPILE); /* Update the winning index's superseded_by to list all other indexes */ UPDATE ia SET - ia.superseded_by = 'Supersedes ' + + ia.superseded_by = N'Supersedes ' + REPLACE ( kdd.index_list, - ia.index_name + N', ', N'' + ia.index_name + N', ', + N'' ) /* Remove self from list if present */ FROM #index_analysis AS ia JOIN #key_duplicate_dedupe AS kdd @@ -2754,13 +2796,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND ia1.index_name <> ia2.index_name AND ia1.action = N'MERGE INCLUDES' AND ia2.action = N'MERGE INCLUDES' - AND ia1.consolidation_rule = 'Key Duplicate' - AND ia2.consolidation_rule = 'Key Duplicate' + AND ia1.consolidation_rule = N'Key Duplicate' + AND ia2.consolidation_rule = N'Key Duplicate' /* Find where subset's includes are contained within superset's includes */ AND ( - ia1.included_columns IS NULL - OR CHARINDEX(ia1.included_columns, ia2.included_columns) > 0 + ia1.included_columns IS NULL + OR CHARINDEX(ia1.included_columns, ia2.included_columns) > 0 ) /* Don't match if lengths are the same (would be exact duplicates) */ AND @@ -2797,8 +2839,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ia.superseded_by = CASE WHEN ia.superseded_by IS NULL - THEN N'Supersedes ' + isd.subset_index_name - ELSE ia.superseded_by + N', ' + isd.subset_index_name + THEN N'Supersedes ' + + isd.subset_index_name + ELSE ia.superseded_by + + N', ' + + isd.subset_index_name END FROM #index_analysis AS ia JOIN #include_subset_dedupe AS isd @@ -2816,8 +2861,6 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. FROM #index_analysis AS ia WHERE ia.action = N'MERGE INCLUDES' AND ia.superseded_by IS NOT NULL - /* Check if the index name contains "Extended" and has more included columns */ - AND (ia.index_name LIKE '%\_Extended%' ESCAPE '\' OR ia.index_name LIKE '%\_Extended' OR ia.index_name LIKE '%_Extended%') /* This should indicate it already has all the needed includes */ AND NOT EXISTS ( @@ -2844,6 +2887,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. INSERT INTO #index_cleanup_results + WITH + (TABLOCK) ( result_type, sort_order, @@ -3007,7 +3052,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. WHERE ia.action IN (N'MERGE INCLUDES', N'MAKE UNIQUE') AND ce.can_compress = 1 AND ia.target_index_name IS NULL - ORDER BY ia.index_name + ORDER BY + ia.index_name OPTION(RECOMPILE); END; @@ -3026,35 +3072,56 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ia.is_unique, ia.index_priority, is_unique_constraint = - CASE WHEN EXISTS ( - SELECT 1 - FROM #index_details AS id - WHERE id.database_id = ia.database_id - AND id.object_id = ia.object_id - AND id.index_id = ia.index_id - AND id.is_unique_constraint = 1 - ) THEN 'YES' ELSE 'NO' END, + CASE + WHEN EXISTS + ( + SELECT + 1/0 + FROM #index_details AS id + WHERE id.database_id = ia.database_id + AND id.object_id = ia.object_id + AND id.index_id = ia.index_id + AND id.is_unique_constraint = 1 + ) + THEN 'YES' + ELSE 'NO' + END, make_unique_target = - CASE WHEN EXISTS ( - SELECT 1 - FROM #index_analysis AS ia_make - WHERE ia_make.database_id = ia.database_id - AND ia_make.object_id = ia.object_id - AND ia_make.action = 'MAKE UNIQUE' - AND ia_make.target_index_name = ia.index_name - ) THEN 'YES' ELSE 'NO' END, + CASE + WHEN EXISTS + ( + SELECT + 1/0 + FROM #index_analysis AS ia_make + WHERE ia_make.database_id = ia.database_id + AND ia_make.object_id = ia.object_id + AND ia_make.action = N'MAKE UNIQUE' + AND ia_make.target_index_name = ia.index_name + ) + THEN 'YES' + ELSE 'NO' + END, will_get_script = - CASE WHEN ia.action = 'DISABLE' AND NOT EXISTS ( - SELECT 1 - FROM #index_details AS id_uc - WHERE id_uc.database_id = ia.database_id - AND id_uc.object_id = ia.object_id - AND id_uc.index_id = ia.index_id - AND id_uc.is_unique_constraint = 1 - ) THEN 'YES' ELSE 'NO' END + CASE + WHEN ia.action = N'DISABLE' + AND NOT EXISTS + ( + SELECT 1 + FROM #index_details AS id_uc + WHERE id_uc.database_id = ia.database_id + AND id_uc.object_id = ia.object_id + AND id_uc.index_id = ia.index_id + AND id_uc.is_unique_constraint = 1 + ) + THEN 'YES' + ELSE 'NO' + END FROM #index_analysis AS ia - WHERE ia.index_name LIKE 'ix_filtered_%' OR ia.index_name LIKE 'ix_desc_%' - ORDER BY ia.index_name; + WHERE ia.index_name LIKE 'ix_filtered_%' + OR ia.index_name LIKE 'ix_desc_%' + ORDER BY + ia.index_name + OPTION(RECOMPILE); /* Debug for all indexes marked with action = DISABLE */ RAISERROR('All indexes with action = DISABLE:', 0, 0) WITH NOWAIT; @@ -3064,12 +3131,16 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ia.action, ia.target_index_name FROM #index_analysis AS ia - WHERE ia.action = 'DISABLE' - ORDER BY ia.index_name; + WHERE ia.action = N'DISABLE' + ORDER BY + ia.index_name + OPTION(RECOMPILE); END; INSERT INTO #index_cleanup_results + WITH + (TABLOCK) ( result_type, sort_order, @@ -3115,12 +3186,15 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. QUOTENAME(ia.table_name) + N' DISABLE;', CASE - WHEN ia.consolidation_rule = 'Key Subset' - THEN N'This index is superseded by a wider index: ' + ISNULL(ia.target_index_name, N'(unknown)') - WHEN ia.consolidation_rule = 'Exact Duplicate' - THEN N'This index is an exact duplicate of: ' + ISNULL(ia.target_index_name, N'(unknown)') - WHEN ia.consolidation_rule = 'Key Duplicate' - THEN N'This index has the same keys as: ' + ISNULL(ia.target_index_name, N'(unknown)') + WHEN ia.consolidation_rule = N'Key Subset' + THEN N'This index is superseded by a wider index: ' + + ISNULL(ia.target_index_name, N'(unknown)') + WHEN ia.consolidation_rule = N'Exact Duplicate' + THEN N'This index is an exact duplicate of: ' + + ISNULL(ia.target_index_name, N'(unknown)') + WHEN ia.consolidation_rule = N'Key Duplicate' + THEN N'This index has the same keys as: ' + + ISNULL(ia.target_index_name, N'(unknown)') WHEN ia.consolidation_rule LIKE 'Unused Index%' THEN ia.consolidation_rule WHEN ia.action = N'DISABLE' @@ -3180,6 +3254,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. INSERT INTO #index_cleanup_results + WITH + (TABLOCK) ( result_type, sort_order, @@ -3302,6 +3378,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Insert KEPT indexes into results */ INSERT INTO #index_cleanup_results + WITH + (TABLOCK) ( result_type, sort_order, @@ -3354,24 +3432,34 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND id.is_included_column = 0 /* Get only one row per index */ AND id.key_ordinal > 0 /* Check that this index is not already in the results */ - WHERE NOT EXISTS ( - SELECT 1 FROM #index_cleanup_results AS ir + WHERE NOT EXISTS + ( + SELECT + 1/0 + FROM #index_cleanup_results AS ir WHERE ir.database_name = ia.database_name AND ir.schema_name = ia.schema_name AND ir.table_name = ia.table_name AND ir.index_name = ia.index_name ) /* And include only indexes that should be kept */ - AND ( + AND + ( /* Include indexes marked KEEP */ - (ia.action = 'KEEP') + ia.action = N'KEEP' /* And all indexes we haven't determined an action for (not disable, merge, etc.) */ - OR (ia.action IS NULL AND ia.index_id > 0) + OR + ( + ia.action IS NULL + AND ia.index_id > 0 + ) ) OPTION(RECOMPILE); INSERT INTO #index_cleanup_results + WITH + (TABLOCK) ( result_type, sort_order, @@ -3436,7 +3524,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Only constraints that are marked for disabling */ ia_uc.action = N'DISABLE' /* That have consolidation_rule of 'Unique Constraint Replacement' */ - AND ia_uc.consolidation_rule = 'Unique Constraint Replacement' + AND ia_uc.consolidation_rule = N'Unique Constraint Replacement' OPTION(RECOMPILE); /* Insert per-partition compression scripts */ @@ -3447,6 +3535,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. INSERT INTO #index_cleanup_results + WITH + (TABLOCK) ( result_type, sort_order, @@ -3658,8 +3748,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ia.target_index_name, additional_info = CASE - WHEN ia.consolidation_rule = 'Same Keys Different Order' - THEN N'This index has the same key columns as ' + ISNULL(ia.target_index_name, N'(unknown)') + + WHEN ia.consolidation_rule = N'Same Keys Different Order' + THEN N'This index has the same key columns as ' + + ISNULL(ia.target_index_name, N'(unknown)') + N' but in a different order. May be redundant depending on query patterns.' ELSE N'This index needs manual review' END, @@ -3764,6 +3855,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. INSERT INTO #index_reporting_stats + WITH + (TABLOCK) ( summary_level, database_name, @@ -3856,15 +3949,15 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. leaf_delete_count = SUM(os.leaf_delete_count) FROM #partition_stats AS ps LEFT JOIN #index_details AS id - ON id.database_id = ps.database_id - AND id.object_id = ps.object_id - AND id.index_id = ps.index_id - AND id.is_included_column = 0 - AND id.key_ordinal > 0 + ON id.database_id = ps.database_id + AND id.object_id = ps.object_id + AND id.index_id = ps.index_id + AND id.is_included_column = 0 + AND id.key_ordinal > 0 LEFT JOIN #operational_stats AS os - ON os.database_id = ps.database_id - AND os.object_id = ps.object_id - AND os.index_id = ps.index_id + ON os.database_id = ps.database_id + AND os.object_id = ps.object_id + AND os.index_id = ps.index_id OUTER APPLY ( /* Get actual row count per table using MAX from clustered index/heap */ @@ -3896,10 +3989,10 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. RAISERROR('Generating #index_reporting_stats insert, TABLE', 0, 0) WITH NOWAIT; END; - /* No need for a temporary table - we'll use a simpler approach */ - INSERT INTO #index_reporting_stats + WITH + (TABLOCK) ( summary_level, database_name, @@ -4005,15 +4098,15 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. leaf_delete_count = SUM(os.leaf_delete_count) FROM #partition_stats AS ps LEFT JOIN #index_details AS id - ON id.database_id = ps.database_id - AND id.object_id = ps.object_id - AND id.index_id = ps.index_id - AND id.is_included_column = 0 - AND id.key_ordinal > 0 + ON id.database_id = ps.database_id + AND id.object_id = ps.object_id + AND id.index_id = ps.index_id + AND id.is_included_column = 0 + AND id.key_ordinal > 0 LEFT JOIN #operational_stats AS os - ON os.database_id = ps.database_id - AND os.object_id = ps.object_id - AND os.index_id = ps.index_id + ON os.database_id = ps.database_id + AND os.object_id = ps.object_id + AND os.index_id = ps.index_id GROUP BY ps.database_name, ps.database_id, @@ -4307,7 +4400,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Basic identification with enhanced naming */ level = CASE - WHEN irs.summary_level = 'SUMMARY' THEN '=== OVERALL ANALYSIS ===' + WHEN irs.summary_level = 'SUMMARY' + THEN '=== OVERALL ANALYSIS ===' ELSE irs.summary_level END, @@ -4354,7 +4448,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. mergeable_indexes = FORMAT(ISNULL(irs.indexes_to_merge, 0), 'N0'), /* Percent of indexes that can be removed */ - pct_removable = + percent_removable = CASE WHEN irs.summary_level = 'SUMMARY' AND irs.index_count > 0 THEN FORMAT(100.0 * ISNULL(irs.indexes_to_disable, 0) / NULLIF(irs.index_count, 0), 'N1') + '%' @@ -4379,7 +4473,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END, /* Space reduction percentage - added this as new metric */ - space_reduction_pct = + space_reduction_percent = CASE WHEN ISNULL(irs.total_size_gb, 0) > 0 THEN FORMAT((ISNULL(irs.space_saved_gb, 0) / NULLIF(irs.total_size_gb, 0)) * 100, 'N1') + '%' @@ -4396,9 +4490,12 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. WHEN irs.summary_level <> 'SUMMARY' THEN FORMAT(ISNULL(irs.total_reads, 0), 'N0') + ' (' + - FORMAT(ISNULL(irs.user_seeks, 0), 'N0') + ' seeks, ' + - FORMAT(ISNULL(irs.user_scans, 0), 'N0') + ' scans, ' + - FORMAT(ISNULL(irs.user_lookups, 0), 'N0') + ' lookups)' + FORMAT(ISNULL(irs.user_seeks, 0), 'N0') + + ' seeks, ' + + FORMAT(ISNULL(irs.user_scans, 0), 'N0') + + ' scans, ' + + FORMAT(ISNULL(irs.user_lookups, 0), 'N0') + + ' lookups)' ELSE 'N/A' END, @@ -4415,7 +4512,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CASE WHEN irs.summary_level <> 'SUMMARY' THEN FORMAT(ISNULL(irs.user_updates / NULLIF(CONVERT(DECIMAL(10,2), - (SELECT TOP 1 server_uptime_days FROM #index_reporting_stats WHERE summary_level = 'DATABASE')), 0) * + (SELECT TOP (1) irs2.server_uptime_days FROM #index_reporting_stats AS irs2 WHERE irs2.summary_level = 'DATABASE')), 0) * (ISNULL(irs.unused_indexes, 0) / NULLIF(CONVERT(DECIMAL(10,2), irs.index_count), 0)), 0), 'N0') ELSE 'N/A' END, @@ -4434,9 +4531,12 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. avg_lock_wait_ms = CASE WHEN irs.summary_level <> 'SUMMARY' - AND (ISNULL(irs.row_lock_wait_count, 0) + ISNULL(irs.page_lock_wait_count, 0)) > 0 - THEN FORMAT(1.0 * (ISNULL(irs.row_lock_wait_in_ms, 0) + ISNULL(irs.page_lock_wait_in_ms, 0)) / - NULLIF(ISNULL(irs.row_lock_wait_count, 0) + ISNULL(irs.page_lock_wait_count, 0), 0), 'N2') + AND (ISNULL(irs.row_lock_wait_count, 0) + + ISNULL(irs.page_lock_wait_count, 0)) > 0 + THEN FORMAT(1.0 * (ISNULL(irs.row_lock_wait_in_ms, 0) + + ISNULL(irs.page_lock_wait_in_ms, 0)) / + NULLIF(ISNULL(irs.row_lock_wait_count, 0) + + ISNULL(irs.page_lock_wait_count, 0), 0), 'N2') ELSE '0.00' END, @@ -4444,9 +4544,12 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. avg_latch_wait_ms = CASE WHEN irs.summary_level <> 'SUMMARY' - AND (ISNULL(irs.page_latch_wait_count, 0) + ISNULL(irs.page_io_latch_wait_count, 0)) > 0 - THEN FORMAT(1.0 * (ISNULL(irs.page_latch_wait_in_ms, 0) + ISNULL(irs.page_io_latch_wait_in_ms, 0)) / - NULLIF(ISNULL(irs.page_latch_wait_count, 0) + ISNULL(irs.page_io_latch_wait_count, 0), 0), 'N2') + AND (ISNULL(irs.page_latch_wait_count, 0) + + ISNULL(irs.page_io_latch_wait_count, 0)) > 0 + THEN FORMAT(1.0 * (ISNULL(irs.page_latch_wait_in_ms, 0) + + ISNULL(irs.page_io_latch_wait_in_ms, 0)) / + NULLIF(ISNULL(irs.page_latch_wait_count, 0) + + ISNULL(irs.page_io_latch_wait_count, 0), 0), 'N2') ELSE '0.00' END FROM #index_reporting_stats AS irs From 0148405ac543af59ef016d11b0039edf9bfe77bf Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 18 Mar 2025 00:37:51 -0400 Subject: [PATCH 194/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 79155d19..60948b37 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -2887,6 +2887,12 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Insert merge scripts for indexes */ IF @debug = 1 BEGIN + SELECT + table_name = '#index_analysis after all updates', + ia.* + FROM #index_analysis AS ia + OPTION(RECOMPILE); + RAISERROR('Generating #index_cleanup_results insert, MERGE', 0, 0) WITH NOWAIT; END; From 8203aea6c9e7b1172a582075fd22c9597bc626ba Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 18 Mar 2025 01:03:21 -0400 Subject: [PATCH 195/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 41 ++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 60948b37..e993bbd1 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -3861,6 +3861,12 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Insert database-level summaries */ IF @debug = 1 BEGIN + SELECT + table_name = '#index_cleanup_results', + icr.* + FROM #index_cleanup_results AS icr + OPTION(RECOMPILE); + RAISERROR('Generating #index_reporting_stats insert, DATABASE', 0, 0) WITH NOWAIT; END; @@ -4127,6 +4133,15 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ps.object_id OPTION(RECOMPILE); + IF @debug = 1 + BEGIN + SELECT + table_name = '#index_reporting_stats', + irs.* + FROM #index_reporting_stats AS irs + OPTION(RECOMPILE); + END; + /* We're not doing index-level summaries - focusing on database and table level reports */ /* @@ -4153,7 +4168,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SELECT /* First, show the information needed to understand the script */ - script_type = CASE WHEN ir.result_type = 'KEPT' AND ir.script_type IS NULL THEN 'KEPT' ELSE ir.script_type END, + script_type = + CASE + WHEN ir.result_type = 'KEPT' + AND ir.script_type IS NULL + THEN 'KEPT' + ELSE ir.script_type + END, ir.additional_info, /* Then show identifying information for the index */ ir.database_name, @@ -4201,12 +4222,20 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. FROM ( /* Use a subquery with ROW_NUMBER to ensure we only get one row per index */ - SELECT *, - ROW_NUMBER() OVER( - PARTITION BY database_name, schema_name, table_name, index_name - ORDER BY result_type DESC /* Prefer non-NULL result types */ + SELECT + irs.*, + ROW_NUMBER() OVER + ( + PARTITION BY + database_name, + schema_name, + table_name, + index_name, + irs.script_type + ORDER BY + result_type DESC /* Prefer non-NULL result types */ ) AS rn - FROM #index_cleanup_results + FROM #index_cleanup_results AS irs ) AS ir LEFT JOIN #index_analysis AS ia ON ir.database_name = ia.database_name From 00a38b642d24d1debf89f0494008a5716d645305 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 18 Mar 2025 01:26:47 -0400 Subject: [PATCH 196/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 190 +++++++++++++--------------- 1 file changed, 87 insertions(+), 103 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index e993bbd1..ba951523 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -3380,93 +3380,6 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. RAISERROR('Generating #index_cleanup_results insert, CONSTRAINT', 0, 0) WITH NOWAIT; END; - /* Add code to insert KEPT indexes into the results - THESE WERE MISSING! */ - IF @debug = 1 - BEGIN - RAISERROR('Generating #index_cleanup_results insert, KEPT', 0, 0) WITH NOWAIT; - END; - - /* Insert KEPT indexes into results */ - INSERT INTO - #index_cleanup_results - WITH - (TABLOCK) - ( - result_type, - sort_order, - database_name, - schema_name, - table_name, - index_name, - script_type, - consolidation_rule, - additional_info, - script, - original_index_definition, - index_size_gb, - index_rows, - index_reads, - index_writes - ) - SELECT DISTINCT - result_type = 'KEPT', - sort_order = 95, /* Put kept indexes at the end */ - ia.database_name, - ia.schema_name, - ia.table_name, - ia.index_name, - script_type = NULL, - ia.consolidation_rule, - additional_info = - CASE - WHEN ia.consolidation_rule IS NOT NULL - THEN 'This index is being kept' - ELSE NULL - END, - script = NULL, /* No script for kept indexes */ - /* Original index definition for validation */ - ia.original_index_definition, - ps.total_space_gb, - ps.total_rows, - index_reads = - (id.user_seeks + id.user_scans + id.user_lookups), - id.user_updates - FROM #index_analysis AS ia - LEFT JOIN #partition_stats AS ps - ON ia.database_id = ps.database_id - AND ia.object_id = ps.object_id - AND ia.index_id = ps.index_id - LEFT JOIN #index_details AS id - ON id.database_id = ia.database_id - AND id.object_id = ia.object_id - AND id.index_id = ia.index_id - AND id.is_included_column = 0 /* Get only one row per index */ - AND id.key_ordinal > 0 - /* Check that this index is not already in the results */ - WHERE NOT EXISTS - ( - SELECT - 1/0 - FROM #index_cleanup_results AS ir - WHERE ir.database_name = ia.database_name - AND ir.schema_name = ia.schema_name - AND ir.table_name = ia.table_name - AND ir.index_name = ia.index_name - ) - /* And include only indexes that should be kept */ - AND - ( - /* Include indexes marked KEEP */ - ia.action = N'KEEP' - /* And all indexes we haven't determined an action for (not disable, merge, etc.) */ - OR - ( - ia.action IS NULL - AND ia.index_id > 0 - ) - ) - OPTION(RECOMPILE); - INSERT INTO #index_cleanup_results WITH @@ -3573,7 +3486,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ia.schema_name, ia.table_name, ia.index_name, - script_type = 'PARTITION COMPRESSION SCRIPT', + script_type = 'COMPRESSION SCRIPT - PARTITION', script = N'ALTER INDEX ' + QUOTENAME(ia.index_name) + @@ -3787,13 +3700,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. OPTION(RECOMPILE); - /* Insert indexes that are being kept (superset indexes and others) */ + /* Insert kept indexes into results - Consolidated all kept indexes logic in one place */ IF @debug = 1 BEGIN - RAISERROR('Generating #index_cleanup_results insert, KEEP', 0, 0) WITH NOWAIT; + RAISERROR('Generating #index_cleanup_results insert, KEPT INDEXES', 0, 0) WITH NOWAIT; END; - - INSERT INTO + + INSERT INTO #index_cleanup_results WITH (TABLOCK) @@ -3812,23 +3725,29 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. index_size_gb, index_rows, index_reads, - index_writes + index_writes, + script ) SELECT DISTINCT - result_type = 'KEEP', - sort_order = 95, /* Just before END OF REPORT at 99 */ + result_type = 'KEPT', + sort_order = 95, /* Put kept indexes at the end */ ia.database_name, ia.schema_name, ia.table_name, ia.index_name, - script_type = 'KEPT', + script_type = + CASE + /* Add compression status to script_type */ + WHEN ce.can_compress = 1 THEN 'KEPT - NEEDS COMPRESSION' + ELSE 'KEPT' + END, ia.consolidation_rule, ia.superseded_by, additional_info = CASE - WHEN ia.superseded_by IS NOT NULL + WHEN ia.superseded_by IS NOT NULL THEN 'This index supersedes other indexes and already has all needed columns' - WHEN ia.action = N'KEEP' + WHEN ia.action = N'KEEP' THEN 'This index is being kept' ELSE NULL END, @@ -3838,23 +3757,88 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ps.total_rows, index_reads = (id.user_seeks + id.user_scans + id.user_lookups), - id.user_updates + id.user_updates, + /* Include compression script directly on KEPT records when needed */ + script = + CASE + WHEN ce.can_compress = 1 + THEN N'ALTER INDEX ' + + QUOTENAME(ia.index_name) + + N' ON ' + + QUOTENAME(ia.database_name) + + N'.' + + QUOTENAME(ia.schema_name) + + N'.' + + QUOTENAME(ia.table_name) + + CASE + WHEN ps_part.partition_function_name IS NOT NULL + THEN N' REBUILD PARTITION = ALL' + ELSE N' REBUILD' + END + + N' WITH (FILLFACTOR = 100, SORT_IN_TEMPDB = ON, ONLINE = ' + + CASE + WHEN @online = 1 + THEN N'ON' + ELSE N'OFF' + END + + N', DATA_COMPRESSION = PAGE)' + ELSE NULL + END FROM #index_analysis AS ia LEFT JOIN #partition_stats AS ps ON ia.database_id = ps.database_id AND ia.object_id = ps.object_id AND ia.index_id = ps.index_id + LEFT JOIN + ( + /* Get the partition info for each index */ + SELECT + ps.database_id, + ps.object_id, + ps.index_id, + ps.partition_function_name + FROM #partition_stats ps + GROUP BY + ps.database_id, + ps.object_id, + ps.index_id, + ps.partition_function_name + ) + AS ps_part + ON ia.database_id = ps_part.database_id + AND ia.object_id = ps_part.object_id + AND ia.index_id = ps_part.index_id LEFT JOIN #index_details AS id ON id.database_id = ia.database_id AND id.object_id = ia.object_id AND id.index_id = ia.index_id AND id.is_included_column = 0 /* Get only one row per index */ AND id.key_ordinal > 0 - WHERE ia.action = N'KEEP' - OR + LEFT JOIN #compression_eligibility AS ce + ON ia.database_id = ce.database_id + AND ia.object_id = ce.object_id + AND ia.index_id = ce.index_id + /* Check that this index is not already in the results */ + WHERE NOT EXISTS ( - ia.action IS NULL - AND ia.consolidation_rule IS NULL + SELECT + 1/0 + FROM #index_cleanup_results AS ir + WHERE ir.database_name = ia.database_name + AND ir.schema_name = ia.schema_name + AND ir.table_name = ia.table_name + AND ir.index_name = ia.index_name + AND ir.script_type NOT LIKE N'COMPRESSION%' + ) + /* Include only indexes that should be kept */ + AND + ( + ia.action = N'KEEP' + OR + ( + ia.action IS NULL + AND ia.index_id > 0 + ) ) OPTION(RECOMPILE); From 371530ec5e930b768834c13f9d01286df521ed04 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 18 Mar 2025 09:39:27 -0400 Subject: [PATCH 197/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 38 ++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index ba951523..65b02a64 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -1970,6 +1970,24 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND id2.index_id = ia2.index_id AND id2.is_eligible_for_dedupe = 1 ) + AND NOT EXISTS + ( + SELECT + 1/0 + FROM #index_details AS id1 + JOIN #index_details AS id2 + ON id2.database_id = id1.database_id + AND id2.object_id = id1.object_id + AND id2.column_name = id1.column_name + AND id2.key_ordinal = id1.key_ordinal + WHERE id1.database_id = ia1.database_id + AND id1.object_id = ia1.object_id + AND id1.index_id = ia1.index_id + AND id2.database_id = ia2.database_id + AND id2.object_id = ia2.object_id + AND id2.index_id = ia2.index_id + AND id1.is_descending_key <> id2.is_descending_key /* Different sort direction */ + ) OPTION(RECOMPILE); IF @debug = 1 @@ -2152,6 +2170,24 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND id2.object_id = ia2.object_id AND id2.index_id = ia2.index_id AND id2.is_eligible_for_dedupe = 1 + ) + AND NOT EXISTS + ( + SELECT + 1/0 + FROM #index_details AS id1 + JOIN #index_details AS id2 + ON id2.database_id = id1.database_id + AND id2.object_id = id1.object_id + AND id2.column_name = id1.column_name + AND id2.key_ordinal = id1.key_ordinal + WHERE id1.database_id = ia1.database_id + AND id1.object_id = ia1.object_id + AND id1.index_id = ia1.index_id + AND id2.database_id = ia2.database_id + AND id2.object_id = ia2.object_id + AND id2.index_id = ia2.index_id + AND id1.is_descending_key <> id2.is_descending_key /* Different sort direction */ ) OPTION(RECOMPILE); @@ -3601,7 +3637,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ce.schema_name, ce.table_name, ce.index_name, - script_type = 'INELIGIBLE FOR COMPRESSION', + script_type = 'COMPRESSION INELIGIBLE', ce.reason, /* Original index definition for validation */ original_index_definition = From 4e85c20dbf6e71dd6dde78f190a815374d0e13de Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 18 Mar 2025 10:33:05 -0400 Subject: [PATCH 198/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 361 ++++++++++++++-------------- 1 file changed, 182 insertions(+), 179 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 65b02a64..771ae3e8 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -3878,15 +3878,167 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ) OPTION(RECOMPILE); - /* Insert database-level summaries */ + /* Insert overall summary information */ IF @debug = 1 BEGIN - SELECT - table_name = '#index_cleanup_results', - icr.* - FROM #index_cleanup_results AS icr - OPTION(RECOMPILE); - + RAISERROR('Generating #index_reporting_stats insert, SUMMARY', 0, 0) WITH NOWAIT; + END; + + INSERT INTO + #index_reporting_stats + WITH + (TABLOCK) + ( + summary_level, + server_uptime_days, + uptime_warning, + tables_analyzed, + index_count, + indexes_to_disable, + indexes_to_merge, + avg_indexes_per_table, + space_saved_gb, + compression_min_savings_gb, + compression_max_savings_gb, + total_min_savings_gb, + total_max_savings_gb, + total_rows + ) + SELECT + summary_level = 'SUMMARY', + server_uptime_days = @uptime_days, + uptime_warning = @uptime_warning, + tables_analyzed = + COUNT_BIG(DISTINCT CONCAT(ia.database_id, N'.', ia.schema_id, N'.', ia.object_id)), + index_count = + COUNT_BIG(*), + indexes_to_disable = + SUM + ( + CASE + WHEN ia.action = N'DISABLE' + THEN 1 + ELSE 0 + END + ), + indexes_to_merge = + SUM + ( + CASE + WHEN ia.action IN (N'MERGE INCLUDES', N'MAKE UNIQUE') + THEN 1 + ELSE 0 + END + ), + avg_indexes_per_table = + COUNT_BIG(*) * 1.0 / + NULLIF + ( + COUNT_BIG(DISTINCT CONCAT(ia.database_id, N'.', ia.schema_id, N'.', ia.object_id)), + 0 + ), + /* Space savings from cleanup */ + space_saved_gb = + SUM + ( + CASE + WHEN ia.action IN (N'DISABLE', N'MERGE INCLUDES', N'MAKE UNIQUE') + THEN ps.total_space_gb + ELSE 0 + END + ), + /* Conservative compression savings estimate (20%) */ + compression_min_savings_gb = + SUM + ( + CASE + WHEN (ia.action IS NULL OR ia.action = N'KEEP') + AND ce.can_compress = 1 + THEN ps.total_space_gb * 0.20 + ELSE 0 + END + ), + /* Optimistic compression savings estimate (60%) */ + compression_max_savings_gb = + SUM + ( + CASE + WHEN (ia.action IS NULL OR ia.action = N'KEEP') + AND ce.can_compress = 1 + THEN ps.total_space_gb * 0.60 + ELSE 0 + END + ), + /* Total conservative savings */ + total_min_savings_gb = + SUM + ( + CASE + WHEN ia.action IN (N'DISABLE', N'MERGE INCLUDES', N'MAKE UNIQUE') + THEN ps.total_space_gb + WHEN (ia.action IS NULL OR ia.action = N'KEEP') + AND ce.can_compress = 1 + THEN ps.total_space_gb * 0.20 + ELSE 0 + END + ), + /* Total optimistic savings */ + total_max_savings_gb = + SUM + ( + CASE + WHEN ia.action IN (N'DISABLE', N'MERGE INCLUDES', N'MAKE UNIQUE') + THEN ps.total_space_gb + WHEN (ia.action IS NULL OR ia.action = N'KEEP') + AND ce.can_compress = 1 + THEN ps.total_space_gb * 0.60 + ELSE 0 + END + ), + /* Get total rows from database unique tables */ + total_rows = + ( + SELECT + SUM(t.row_count) + FROM + ( + SELECT + ps_distinct.object_id, + row_count = + MAX + ( + CASE + WHEN ps_distinct.index_id IN (0, 1) + THEN ps_distinct.total_rows + ELSE 0 + END + ) + FROM #partition_stats AS ps_distinct + WHERE ps_distinct.index_id IN (0, 1) + GROUP BY + ps_distinct.object_id + ) AS t + ) + FROM #index_analysis AS ia + LEFT JOIN #partition_stats AS ps + ON ia.database_id = ps.database_id + AND ia.object_id = ps.object_id + AND ia.index_id = ps.index_id + LEFT JOIN #compression_eligibility AS ce + ON ia.database_id = ce.database_id + AND ia.object_id = ce.object_id + AND ia.index_id = ce.index_id + OPTION(RECOMPILE); + + /* Return enhanced database impact summaries */ + IF @debug = 1 + BEGIN + RAISERROR('Generating enhanced summary reports', 0, 0) WITH NOWAIT; + END; + + /* Insert database-level summaries */ + IF @debug = 1 + BEGIN RAISERROR('Generating #index_reporting_stats insert, DATABASE', 0, 0) WITH NOWAIT; END; @@ -4153,15 +4305,6 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ps.object_id OPTION(RECOMPILE); - IF @debug = 1 - BEGIN - SELECT - table_name = '#index_reporting_stats', - irs.* - FROM #index_reporting_stats AS irs - OPTION(RECOMPILE); - END; - /* We're not doing index-level summaries - focusing on database and table level reports */ /* @@ -4183,6 +4326,18 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ IF @debug = 1 BEGIN + SELECT + table_name = '#index_reporting_stats', + irs.* + FROM #index_reporting_stats AS irs + OPTION(RECOMPILE); + + SELECT + table_name = '#index_cleanup_results', + icr.* + FROM #index_cleanup_results AS icr + OPTION(RECOMPILE); + RAISERROR('Generating #index_cleanup_results, RESULTS', 0, 0) WITH NOWAIT; END; @@ -4287,164 +4442,6 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ir.index_name OPTION(RECOMPILE); - /* Insert overall summary information */ - IF @debug = 1 - BEGIN - RAISERROR('Generating #index_reporting_stats insert, SUMMARY', 0, 0) WITH NOWAIT; - END; - - INSERT INTO - #index_reporting_stats - WITH - (TABLOCK) - ( - summary_level, - server_uptime_days, - uptime_warning, - tables_analyzed, - index_count, - indexes_to_disable, - indexes_to_merge, - avg_indexes_per_table, - space_saved_gb, - compression_min_savings_gb, - compression_max_savings_gb, - total_min_savings_gb, - total_max_savings_gb, - total_rows - ) - SELECT - summary_level = 'SUMMARY', - server_uptime_days = @uptime_days, - uptime_warning = @uptime_warning, - tables_analyzed = - COUNT_BIG(DISTINCT CONCAT(ia.database_id, N'.', ia.schema_id, N'.', ia.object_id)), - index_count = - COUNT_BIG(*), - indexes_to_disable = - SUM - ( - CASE - WHEN ia.action = N'DISABLE' - THEN 1 - ELSE 0 - END - ), - indexes_to_merge = - SUM - ( - CASE - WHEN ia.action IN (N'MERGE INCLUDES', N'MAKE UNIQUE') - THEN 1 - ELSE 0 - END - ), - avg_indexes_per_table = - COUNT_BIG(*) * 1.0 / - NULLIF - ( - COUNT_BIG(DISTINCT CONCAT(ia.database_id, N'.', ia.schema_id, N'.', ia.object_id)), - 0 - ), - /* Space savings from cleanup */ - space_saved_gb = - SUM - ( - CASE - WHEN ia.action IN (N'DISABLE', N'MERGE INCLUDES', N'MAKE UNIQUE') - THEN ps.total_space_gb - ELSE 0 - END - ), - /* Conservative compression savings estimate (20%) */ - compression_min_savings_gb = - SUM - ( - CASE - WHEN (ia.action IS NULL OR ia.action = N'KEEP') - AND ce.can_compress = 1 - THEN ps.total_space_gb * 0.20 - ELSE 0 - END - ), - /* Optimistic compression savings estimate (60%) */ - compression_max_savings_gb = - SUM - ( - CASE - WHEN (ia.action IS NULL OR ia.action = N'KEEP') - AND ce.can_compress = 1 - THEN ps.total_space_gb * 0.60 - ELSE 0 - END - ), - /* Total conservative savings */ - total_min_savings_gb = - SUM - ( - CASE - WHEN ia.action IN (N'DISABLE', N'MERGE INCLUDES', N'MAKE UNIQUE') - THEN ps.total_space_gb - WHEN (ia.action IS NULL OR ia.action = N'KEEP') - AND ce.can_compress = 1 - THEN ps.total_space_gb * 0.20 - ELSE 0 - END - ), - /* Total optimistic savings */ - total_max_savings_gb = - SUM - ( - CASE - WHEN ia.action IN (N'DISABLE', N'MERGE INCLUDES', N'MAKE UNIQUE') - THEN ps.total_space_gb - WHEN (ia.action IS NULL OR ia.action = N'KEEP') - AND ce.can_compress = 1 - THEN ps.total_space_gb * 0.60 - ELSE 0 - END - ), - /* Get total rows from database unique tables */ - total_rows = - ( - SELECT - SUM(t.row_count) - FROM - ( - SELECT - ps_distinct.object_id, - row_count = - MAX - ( - CASE - WHEN ps_distinct.index_id IN (0, 1) - THEN ps_distinct.total_rows - ELSE 0 - END - ) - FROM #partition_stats AS ps_distinct - WHERE ps_distinct.index_id IN (0, 1) - GROUP BY - ps_distinct.object_id - ) AS t - ) - FROM #index_analysis AS ia - LEFT JOIN #partition_stats AS ps - ON ia.database_id = ps.database_id - AND ia.object_id = ps.object_id - AND ia.index_id = ps.index_id - LEFT JOIN #compression_eligibility AS ce - ON ia.database_id = ce.database_id - AND ia.object_id = ce.object_id - AND ia.index_id = ce.index_id - OPTION(RECOMPILE); - - /* Return enhanced database impact summaries */ - IF @debug = 1 - BEGIN - RAISERROR('Generating enhanced summary reports', 0, 0) WITH NOWAIT; - END; - /* This section now REPLACES the existing summary view rather than supplementing it We'll modify the existing query below rather than creating new output panes @@ -4510,10 +4507,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Percent of indexes that can be removed */ percent_removable = CASE - WHEN irs.summary_level = 'SUMMARY' AND irs.index_count > 0 - THEN FORMAT(100.0 * ISNULL(irs.indexes_to_disable, 0) / NULLIF(irs.index_count, 0), 'N1') + '%' + WHEN irs.summary_level = 'SUMMARY' + AND irs.index_count > 0 + THEN FORMAT(100.0 * ISNULL(irs.indexes_to_disable, 0) + / NULLIF(irs.index_count, 0), 'N1') + '%' WHEN irs.index_count > 0 - THEN FORMAT(100.0 * ISNULL(irs.unused_indexes, 0) / NULLIF(irs.index_count, 0), 'N1') + '%' + THEN FORMAT(100.0 * ISNULL(irs.unused_indexes, 0) + / NULLIF(irs.index_count, 0), 'N1') + '%' ELSE '0.0%' END, @@ -4522,7 +4522,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. current_size_gb = FORMAT(ISNULL(irs.total_size_gb, 0), 'N2'), /* Size after cleanup - added this as new metric */ - size_after_cleanup_gb = FORMAT(ISNULL(irs.total_size_gb, 0) - ISNULL(irs.space_saved_gb, 0), 'N2'), + size_after_cleanup_gb = + FORMAT(ISNULL(irs.total_size_gb, 0) - + ISNULL(irs.space_saved_gb, 0), 'N2'), /* Size that can be saved through cleanup */ space_saved_gb = @@ -4536,7 +4538,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. space_reduction_percent = CASE WHEN ISNULL(irs.total_size_gb, 0) > 0 - THEN FORMAT((ISNULL(irs.space_saved_gb, 0) / NULLIF(irs.total_size_gb, 0)) * 100, 'N1') + '%' + THEN FORMAT((ISNULL(irs.space_saved_gb, 0) / + NULLIF(irs.total_size_gb, 0)) * 100, 'N1') + '%' ELSE '0.0%' END, From 77f649e428bee692d054f0c99a6451c1ff5cc97c Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 18 Mar 2025 10:59:16 -0400 Subject: [PATCH 199/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 398 +++++++++++++++++++++++----- 1 file changed, 336 insertions(+), 62 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 65b02a64..9bc6b2c1 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -23,6 +23,9 @@ ALTER PROCEDURE @min_writes bigint = 0, @min_size_gb decimal(10,2) = 0, @min_rows bigint = 0, + @get_all_databases bit = 0, /*looks for all accessible user databases and returns combined results*/ + @include_databases nvarchar(max) = NULL, /*comma-separated list of databases to include (only when @get_all_databases = 1)*/ + @exclude_databases nvarchar(max) = NULL, /*comma-separated list of databases to exclude (only when @get_all_databases = 1)*/ @help bit = 'false', @debug bit = 'false', @version varchar(20) = NULL OUTPUT, @@ -110,6 +113,9 @@ BEGIN TRY WHEN N'@min_writes' THEN 'minimum number of writes for an index to be considered used' WHEN N'@min_size_gb' THEN 'minimum size in GB for an index to be analyzed' WHEN N'@min_rows' THEN 'minimum number of rows for a table to be analyzed' + WHEN N'@get_all_databases' THEN 'set to 1 to analyze all accessible user databases' + WHEN N'@include_databases' THEN 'comma-separated list of databases to include when @get_all_databases = 1' + WHEN N'@exclude_databases' THEN 'comma-separated list of databases to exclude when @get_all_databases = 1' WHEN N'@help' THEN 'displays this help information' WHEN N'@debug' THEN 'prints debug information during execution' WHEN N'@version' THEN 'returns the version number of the procedure' @@ -126,6 +132,9 @@ BEGIN TRY WHEN N'@min_writes' THEN 'any positive integer or 0' WHEN N'@min_size_gb' THEN 'any positive decimal or 0' WHEN N'@min_rows' THEN 'any positive integer or 0' + WHEN N'@get_all_databases' THEN '0 or 1' + WHEN N'@include_databases' THEN 'comma-separated list of database names' + WHEN N'@exclude_databases' THEN 'comma-separated list of database names' WHEN N'@help' THEN '0 or 1' WHEN N'@debug' THEN '0 or 1' WHEN N'@version' THEN 'OUTPUT parameter' @@ -142,6 +151,9 @@ BEGIN TRY WHEN N'@min_writes' THEN '0' WHEN N'@min_size_gb' THEN '0' WHEN N'@min_rows' THEN '0' + WHEN N'@get_all_databases' THEN '0' + WHEN N'@include_databases' THEN 'NULL' + WHEN N'@exclude_databases' THEN 'NULL' WHEN N'@help' THEN 'false' WHEN N'@debug' THEN 'true' WHEN N'@version' THEN 'NULL' @@ -275,29 +287,237 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. RAISERROR('Checking paramaters...', 0, 0) WITH NOWAIT; END; - IF @database_name IS NULL - AND DB_NAME() NOT IN + /* Create temp tables for database filtering */ + CREATE TABLE #include_databases + ( + database_name sysname NOT NULL PRIMARY KEY + ); + + CREATE TABLE #exclude_databases + ( + database_name sysname NOT NULL PRIMARY KEY + ); + + CREATE TABLE #databases + ( + database_name sysname NOT NULL PRIMARY KEY, + database_id int NOT NULL + ); + + CREATE TABLE #requested_but_skipped_databases + ( + database_name sysname NOT NULL PRIMARY KEY, + reason nvarchar(100) NOT NULL + ); + + /* Parse @include_databases comma-separated list */ + IF @get_all_databases = 1 AND @include_databases IS NOT NULL + BEGIN + INSERT + #include_databases ( - N'master', - N'model', - N'msdb', - N'tempdb', - N'rdsadmin' + database_name ) + SELECT DISTINCT + database_name = + LTRIM(RTRIM(c.value(N'(./text())[1]', N'sysname'))) + FROM + ( + SELECT + x = CONVERT + ( + xml, + N'' + + REPLACE + ( + @include_databases, + N',', + N'' + ) + + N'' + ) + ) AS a + CROSS APPLY x.nodes(N'//i') AS t(c) + WHERE LTRIM(RTRIM(c.value(N'(./text())[1]', N'sysname'))) <> N''; + + IF @debug = 1 + BEGIN + SELECT + table_name = '#include_databases', + id.* + FROM #include_databases AS id + OPTION(RECOMPILE); + END; + END; + + /* Parse @exclude_databases comma-separated list */ + IF @get_all_databases = 1 AND @exclude_databases IS NOT NULL BEGIN - SELECT - @database_name = DB_NAME(); + INSERT + #exclude_databases + ( + database_name + ) + SELECT DISTINCT + database_name = + LTRIM(RTRIM(c.value(N'(./text())[1]', N'sysname'))) + FROM + ( + SELECT + x = CONVERT + ( + xml, + N'' + + REPLACE + ( + @exclude_databases, + N',', + N'' + ) + + N'' + ) + ) AS a + CROSS APPLY x.nodes(N'//i') AS t(c) + WHERE LTRIM(RTRIM(c.value(N'(./text())[1]', N'sysname'))) <> N''; + + IF @debug = 1 + BEGIN + SELECT + table_name = '#exclude_databases', + ed.* + FROM #exclude_databases AS ed + OPTION(RECOMPILE); + END; END; - IF @database_name IS NOT NULL + /* Check for conflicts between include and exclude lists */ + IF @get_all_databases = 1 + AND @include_databases IS NOT NULL + AND @exclude_databases IS NOT NULL BEGIN + DECLARE @conflict_list nvarchar(max) = N''; + + SELECT + @conflict_list = + @conflict_list + + ed.database_name + N', ' + FROM #exclude_databases AS ed + WHERE EXISTS + ( + SELECT + 1/0 + FROM #include_databases AS id + WHERE id.database_name = ed.database_name + ); + + /* If we found any conflicts, raise an error */ + IF LEN(@conflict_list) > 0 + BEGIN + /* Remove trailing comma and space */ + SET @conflict_list = LEFT(@conflict_list, LEN(@conflict_list) - 2); + + SET @error_msg = + N'The following databases appear in both @include_databases and @exclude_databases, which creates ambiguity: ' + + @conflict_list + N'. Please remove these databases from one of the lists.'; + + RAISERROR(@error_msg, 16, 1); + RETURN; + END; + END; + + /* Handle contradictory parameters */ + IF @get_all_databases = 1 AND @database_name IS NOT NULL + BEGIN + IF @debug = 1 + BEGIN + RAISERROR(N'@database name being ignored since @get_all_databases is set to 1', 0, 0) WITH NOWAIT; + END; + SET @database_name = NULL; + END; + + /* Build the #databases table */ + IF @get_all_databases = 0 + BEGIN + /* Default to current database if not system db */ + IF @database_name IS NULL + AND DB_NAME() NOT IN + ( + N'master', + N'model', + N'msdb', + N'tempdb', + N'rdsadmin' + ) + BEGIN + SELECT + @database_name = DB_NAME(); + END; + + /* Single database mode */ + IF @database_name IS NOT NULL + BEGIN + INSERT #databases + ( + database_name, + database_id + ) + SELECT + d.name, + d.database_id + FROM sys.databases AS d + WHERE d.database_id = DB_ID(@database_name) + AND d.state = 0 + AND d.is_in_standby = 0 + AND d.is_read_only = 0 + OPTION(RECOMPILE); + + /* Get the database_id for backwards compatibility */ + SELECT + @database_id = d.database_id + FROM #databases AS d; + END; + END + ELSE + BEGIN + /* Multi-database mode */ + INSERT #databases + ( + database_name, + database_id + ) SELECT - @database_id = d.database_id + d.name, + d.database_id FROM sys.databases AS d - WHERE d.database_id = DB_ID(@database_name) + WHERE d.database_id > 4 /* Skip system databases */ AND d.state = 0 AND d.is_in_standby = 0 AND d.is_read_only = 0 + AND ( + @include_databases IS NULL + OR EXISTS (SELECT 1/0 FROM #include_databases AS id WHERE id.database_name = d.name) + ) + AND ( + @exclude_databases IS NULL + OR NOT EXISTS (SELECT 1/0 FROM #exclude_databases AS ed WHERE ed.database_name = d.name) + ) + OPTION(RECOMPILE); + END; + + /* Check for empty database list */ + IF (SELECT COUNT(*) FROM #databases) = 0 + BEGIN + RAISERROR('No valid databases found to process.', 16, 1); + RETURN; + END; + + /* Show database list in debug mode */ + IF @debug = 1 + BEGIN + SELECT + table_name = '#databases', + d.* + FROM #databases AS d OPTION(RECOMPILE); END; @@ -667,56 +887,92 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ); /* - Start insert queries + Set up database cursor processing */ + DECLARE + @database_cursor CURSOR, + @current_database_name sysname, + @current_database_id int; + + /* Create a cursor to process each database */ + SET + @database_cursor = + CURSOR + LOCAL + SCROLL + DYNAMIC + READ_ONLY + FOR + SELECT + d.database_name, + d.database_id + FROM #databases AS d + ORDER BY d.database_name; + + OPEN @database_cursor; + + FETCH FIRST + FROM @database_cursor + INTO @current_database_name, @current_database_id; + /* + Start insert queries + */ IF @debug = 1 BEGIN RAISERROR('Generating #filtered_object insert', 0, 0) WITH NOWAIT; - END; + END; - SELECT - @sql = N' - SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;'; - - SELECT - @sql = N' - SELECT DISTINCT - @database_id, - database_name = DB_NAME(@database_id), - schema_id = t.schema_id, - schema_name = s.name, - object_id = t.object_id, - table_name = t.name, - index_id = i.index_id, - index_name = ISNULL(i.name, t.name + N''.Heap''), - can_compress = - CASE - WHEN p.index_id > 0 - AND p.data_compression = 0 - THEN 1 - ELSE 0 - END - FROM ' + QUOTENAME(@database_name) + N'.sys.tables AS t - JOIN ' + QUOTENAME(@database_name) + N'.sys.schemas AS s - ON t.schema_id = s.schema_id - JOIN ' + QUOTENAME(@database_name) + N'.sys.indexes AS i - ON t.object_id = i.object_id - JOIN ' + QUOTENAME(@database_name) + N'.sys.partitions AS p - ON i.object_id = p.object_id - AND i.index_id = p.index_id - LEFT JOIN ' + QUOTENAME(@database_name) + N'.sys.dm_db_index_usage_stats AS us - ON t.object_id = us.object_id - AND us.database_id = @database_id - WHERE t.is_ms_shipped = 0 - AND t.type <> N''TF'' - AND NOT EXISTS - ( - SELECT - 1/0 - FROM ' + QUOTENAME(@database_name) + N'.sys.views AS v - WHERE v.object_id = i.object_id - )'; + WHILE @@FETCH_STATUS = 0 + BEGIN + /* Process current database */ + IF @debug = 1 + BEGIN + RAISERROR('Processing database: %s (ID: %d)', 0, 0, @current_database_name, @current_database_id) WITH NOWAIT; + END; + + SELECT + @sql = N' + SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;'; + + SELECT + @sql = N' + SELECT DISTINCT + @database_id, + database_name = DB_NAME(@database_id), + schema_id = t.schema_id, + schema_name = s.name, + object_id = t.object_id, + table_name = t.name, + index_id = i.index_id, + index_name = ISNULL(i.name, t.name + N''.Heap''), + can_compress = + CASE + WHEN p.index_id > 0 + AND p.data_compression = 0 + THEN 1 + ELSE 0 + END + FROM ' + QUOTENAME(@current_database_name) + N'.sys.tables AS t + JOIN ' + QUOTENAME(@current_database_name) + N'.sys.schemas AS s + ON t.schema_id = s.schema_id + JOIN ' + QUOTENAME(@current_database_name) + N'.sys.indexes AS i + ON t.object_id = i.object_id + JOIN ' + QUOTENAME(@current_database_name) + N'.sys.partitions AS p + ON i.object_id = p.object_id + AND i.index_id = p.index_id + LEFT JOIN ' + QUOTENAME(@current_database_name) + N'.sys.dm_db_index_usage_stats AS us + ON t.object_id = us.object_id + AND us.database_id = @database_id + WHERE t.is_ms_shipped = 0 + AND t.type <> N''TF'' + AND NOT EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@current_database_name) + N'.sys.views AS v + WHERE v.object_id = i.object_id + )'; IF /* Check for temporal tables support */ ( @@ -751,7 +1007,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ( SELECT 1/0 - FROM ' + QUOTENAME(@database_name) + N'.sys.tables AS t + FROM ' + QUOTENAME(@current_database_name) + N'.sys.tables AS t WHERE t.object_id = i.object_id AND t.temporal_type > 0 )'; @@ -774,8 +1030,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ( SELECT 1/0 - FROM ' + QUOTENAME(@database_name) + N'.sys.dm_db_partition_stats AS ps - JOIN ' + QUOTENAME(@database_name) + N'.sys.allocation_units AS au + FROM ' + QUOTENAME(@current_database_name) + N'.sys.dm_db_partition_stats AS ps + JOIN ' + QUOTENAME(@current_database_name) + N'.sys.allocation_units AS au ON ps.partition_id = au.container_id WHERE ps.object_id = t.object_id GROUP BY @@ -787,7 +1043,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ( SELECT 1/0 - FROM ' + QUOTENAME(@database_name) + N'.sys.dm_db_partition_stats AS ps + FROM ' + QUOTENAME(@current_database_name) + N'.sys.dm_db_partition_stats AS ps WHERE ps.object_id = t.object_id AND ps.index_id IN (0, 1) GROUP BY @@ -799,7 +1055,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ( SELECT 1/0 - FROM ' + QUOTENAME(@database_name) + N'.sys.dm_db_index_usage_stats AS ius + FROM ' + QUOTENAME(@current_database_name) + N'.sys.dm_db_index_usage_stats AS ius WHERE ius.object_id = t.object_id AND ius.database_id = @database_id GROUP BY @@ -840,13 +1096,31 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. @min_size_gb decimal(10,2), @min_rows bigint, @object_id integer', - @database_id, + @current_database_id, @min_reads, @min_writes, @min_size_gb, @min_rows, @object_id; + /* Get the next database */ + FETCH NEXT + FROM @database_cursor + INTO @current_database_name, @current_database_id; + END; + + CLOSE @database_cursor; + DEALLOCATE @database_cursor; + + /* Set database_id for backwards compatibility when processing single database */ + IF @get_all_databases = 0 AND (SELECT COUNT(*) FROM #databases) = 1 + BEGIN + SELECT + @database_id = d.database_id, + @database_name = d.database_name + FROM #databases AS d; + END; + IF ROWCOUNT_BIG() = 0 BEGIN IF @debug = 1 From 13024fe863d5a46192f6ca3db0d9fc9484ac6543 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 18 Mar 2025 12:19:11 -0400 Subject: [PATCH 200/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 593 +++++++++++++++------------- 1 file changed, 311 insertions(+), 282 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 6b763080..481141c2 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -268,7 +268,12 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SYSDATETIME() ) FROM sys.dm_os_sys_info AS osi - ); + ), + @database_cursor CURSOR, + @current_database_name sysname, + @current_database_id integer, + @error_msg nvarchar(2048), + @conflict_list nvarchar(max) = N'' /* Set uptime warning flag after @uptime_days is calculated */ SELECT @@ -287,240 +292,6 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. RAISERROR('Checking paramaters...', 0, 0) WITH NOWAIT; END; - /* Create temp tables for database filtering */ - CREATE TABLE #include_databases - ( - database_name sysname NOT NULL PRIMARY KEY - ); - - CREATE TABLE #exclude_databases - ( - database_name sysname NOT NULL PRIMARY KEY - ); - - CREATE TABLE #databases - ( - database_name sysname NOT NULL PRIMARY KEY, - database_id int NOT NULL - ); - - CREATE TABLE #requested_but_skipped_databases - ( - database_name sysname NOT NULL PRIMARY KEY, - reason nvarchar(100) NOT NULL - ); - - /* Parse @include_databases comma-separated list */ - IF @get_all_databases = 1 AND @include_databases IS NOT NULL - BEGIN - INSERT - #include_databases - ( - database_name - ) - SELECT DISTINCT - database_name = - LTRIM(RTRIM(c.value(N'(./text())[1]', N'sysname'))) - FROM - ( - SELECT - x = CONVERT - ( - xml, - N'' + - REPLACE - ( - @include_databases, - N',', - N'' - ) + - N'' - ) - ) AS a - CROSS APPLY x.nodes(N'//i') AS t(c) - WHERE LTRIM(RTRIM(c.value(N'(./text())[1]', N'sysname'))) <> N''; - - IF @debug = 1 - BEGIN - SELECT - table_name = '#include_databases', - id.* - FROM #include_databases AS id - OPTION(RECOMPILE); - END; - END; - - /* Parse @exclude_databases comma-separated list */ - IF @get_all_databases = 1 AND @exclude_databases IS NOT NULL - BEGIN - INSERT - #exclude_databases - ( - database_name - ) - SELECT DISTINCT - database_name = - LTRIM(RTRIM(c.value(N'(./text())[1]', N'sysname'))) - FROM - ( - SELECT - x = CONVERT - ( - xml, - N'' + - REPLACE - ( - @exclude_databases, - N',', - N'' - ) + - N'' - ) - ) AS a - CROSS APPLY x.nodes(N'//i') AS t(c) - WHERE LTRIM(RTRIM(c.value(N'(./text())[1]', N'sysname'))) <> N''; - - IF @debug = 1 - BEGIN - SELECT - table_name = '#exclude_databases', - ed.* - FROM #exclude_databases AS ed - OPTION(RECOMPILE); - END; - END; - - /* Check for conflicts between include and exclude lists */ - IF @get_all_databases = 1 - AND @include_databases IS NOT NULL - AND @exclude_databases IS NOT NULL - BEGIN - DECLARE @conflict_list nvarchar(max) = N''; - - SELECT - @conflict_list = - @conflict_list + - ed.database_name + N', ' - FROM #exclude_databases AS ed - WHERE EXISTS - ( - SELECT - 1/0 - FROM #include_databases AS id - WHERE id.database_name = ed.database_name - ); - - /* If we found any conflicts, raise an error */ - IF LEN(@conflict_list) > 0 - BEGIN - /* Remove trailing comma and space */ - SET @conflict_list = LEFT(@conflict_list, LEN(@conflict_list) - 2); - - SET @error_msg = - N'The following databases appear in both @include_databases and @exclude_databases, which creates ambiguity: ' + - @conflict_list + N'. Please remove these databases from one of the lists.'; - - RAISERROR(@error_msg, 16, 1); - RETURN; - END; - END; - - /* Handle contradictory parameters */ - IF @get_all_databases = 1 AND @database_name IS NOT NULL - BEGIN - IF @debug = 1 - BEGIN - RAISERROR(N'@database name being ignored since @get_all_databases is set to 1', 0, 0) WITH NOWAIT; - END; - SET @database_name = NULL; - END; - - /* Build the #databases table */ - IF @get_all_databases = 0 - BEGIN - /* Default to current database if not system db */ - IF @database_name IS NULL - AND DB_NAME() NOT IN - ( - N'master', - N'model', - N'msdb', - N'tempdb', - N'rdsadmin' - ) - BEGIN - SELECT - @database_name = DB_NAME(); - END; - - /* Single database mode */ - IF @database_name IS NOT NULL - BEGIN - INSERT #databases - ( - database_name, - database_id - ) - SELECT - d.name, - d.database_id - FROM sys.databases AS d - WHERE d.database_id = DB_ID(@database_name) - AND d.state = 0 - AND d.is_in_standby = 0 - AND d.is_read_only = 0 - OPTION(RECOMPILE); - - /* Get the database_id for backwards compatibility */ - SELECT - @database_id = d.database_id - FROM #databases AS d; - END; - END - ELSE - BEGIN - /* Multi-database mode */ - INSERT #databases - ( - database_name, - database_id - ) - SELECT - d.name, - d.database_id - FROM sys.databases AS d - WHERE d.database_id > 4 /* Skip system databases */ - AND d.state = 0 - AND d.is_in_standby = 0 - AND d.is_read_only = 0 - AND ( - @include_databases IS NULL - OR EXISTS (SELECT 1/0 FROM #include_databases AS id WHERE id.database_name = d.name) - ) - AND ( - @exclude_databases IS NULL - OR NOT EXISTS (SELECT 1/0 FROM #exclude_databases AS ed WHERE ed.database_name = d.name) - ) - OPTION(RECOMPILE); - END; - - /* Check for empty database list */ - IF (SELECT COUNT(*) FROM #databases) = 0 - BEGIN - RAISERROR('No valid databases found to process.', 16, 1); - RETURN; - END; - - /* Show database list in debug mode */ - IF @debug = 1 - BEGIN - SELECT - table_name = '#databases', - d.* - FROM #databases AS d - OPTION(RECOMPILE); - END; - IF @schema_name IS NULL AND @table_name IS NOT NULL BEGIN @@ -533,34 +304,6 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. @schema_name = N'dbo'; END; - IF @schema_name IS NOT NULL - AND @table_name IS NOT NULL - BEGIN - IF @debug = 1 - BEGIN - RAISERROR('validating object existence for %s.%s.&s.', 0, 0, @database_name, @schema_name, @table_name) WITH NOWAIT; - END; - - SELECT - @full_object_name = - QUOTENAME(@database_name) + - N'.' + - QUOTENAME(@schema_name) + - N'.' + - QUOTENAME(@table_name); - - SELECT - @object_id = - OBJECT_ID(@full_object_name); - - IF @object_id IS NULL - BEGIN - RAISERROR('The object %s doesn''t seem to exist', 16, 1, @full_object_name) WITH NOWAIT; - RETURN; - END; - END; - - /* Parameter validation */ IF @min_reads < 0 OR @min_reads IS NULL BEGIN @@ -886,17 +629,261 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. leaf_delete_count bigint NULL ); + /* Create temp tables for database filtering */ + CREATE TABLE + #include_databases + ( + database_name sysname NOT NULL PRIMARY KEY + ); + + CREATE TABLE + #exclude_databases + ( + database_name sysname NOT NULL PRIMARY KEY + ); + + CREATE TABLE + #databases + ( + database_name sysname NOT NULL PRIMARY KEY, + database_id int NOT NULL + ); + + CREATE TABLE + #requested_but_skipped_databases + ( + database_name sysname NOT NULL PRIMARY KEY, + reason nvarchar(100) NOT NULL + ); + + /* Parse @include_databases comma-separated list */ + IF @get_all_databases = 1 + AND @include_databases IS NOT NULL + BEGIN + INSERT + #include_databases + WITH + (TABLOCK) + ( + database_name + ) + SELECT DISTINCT + database_name = + LTRIM(RTRIM(c.value(N'(./text())[1]', N'sysname'))) + FROM + ( + SELECT + x = CONVERT + ( + xml, + N'' + + REPLACE + ( + @include_databases, + N',', + N'' + ) + + N'' + ) + ) AS a + CROSS APPLY x.nodes(N'//i') AS t(c) + WHERE LTRIM(RTRIM(c.value(N'(./text())[1]', N'sysname'))) <> N''; + + IF @debug = 1 + BEGIN + SELECT + table_name = '#include_databases', + id.* + FROM #include_databases AS id + OPTION(RECOMPILE); + END; + END; + + /* Parse @exclude_databases comma-separated list */ + IF @get_all_databases = 1 + AND @exclude_databases IS NOT NULL + BEGIN + INSERT + #exclude_databases + WITH + (TABLOCK) + ( + database_name + ) + SELECT DISTINCT + database_name = + LTRIM(RTRIM(c.value(N'(./text())[1]', N'sysname'))) + FROM + ( + SELECT + x = CONVERT + ( + xml, + N'' + + REPLACE + ( + @exclude_databases, + N',', + N'' + ) + + N'' + ) + ) AS a + CROSS APPLY x.nodes(N'//i') AS t(c) + WHERE LTRIM(RTRIM(c.value(N'(./text())[1]', N'sysname'))) <> N''; + + IF @debug = 1 + BEGIN + SELECT + table_name = '#exclude_databases', + ed.* + FROM #exclude_databases AS ed + OPTION(RECOMPILE); + END; + END; + + /* Check for conflicts between include and exclude lists */ + IF @get_all_databases = 1 + AND @include_databases IS NOT NULL + AND @exclude_databases IS NOT NULL + BEGIN + SELECT + @conflict_list = + @conflict_list + + ed.database_name + N', ' + FROM #exclude_databases AS ed + WHERE EXISTS + ( + SELECT + 1/0 + FROM #include_databases AS id + WHERE id.database_name = ed.database_name + ); + + /* If we found any conflicts, raise an error */ + IF LEN(@conflict_list) > 0 + BEGIN + /* Remove trailing comma and space */ + SET @conflict_list = LEFT(@conflict_list, LEN(@conflict_list) - 2); + + SET @error_msg = + N'The following databases appear in both @include_databases and @exclude_databases, which creates ambiguity: ' + + @conflict_list + N'. Please remove these databases from one of the lists.'; + + RAISERROR(@error_msg, 16, 1); + RETURN; + END; + END; + + /* Handle contradictory parameters */ + IF @get_all_databases = 1 + AND @database_name IS NOT NULL + BEGIN + IF @debug = 1 + BEGIN + RAISERROR(N'@database name being ignored since @get_all_databases is set to 1', 0, 0) WITH NOWAIT; + END; + SET @database_name = NULL; + END; + + /* Build the #databases table */ + IF @get_all_databases = 0 + BEGIN + /* Default to current database if not system db */ + IF @database_name IS NULL + AND DB_NAME() NOT IN + ( + N'master', + N'model', + N'msdb', + N'tempdb', + N'rdsadmin' + ) + BEGIN + SELECT + @database_name = DB_NAME(); + END; + + /* Single database mode */ + IF @database_name IS NOT NULL + BEGIN + INSERT + #databases + WITH + (TABLOCK) + ( + database_name, + database_id + ) + SELECT + d.name, + d.database_id + FROM sys.databases AS d + WHERE d.database_id = DB_ID(@database_name) + AND d.state = 0 + AND d.is_in_standby = 0 + AND d.is_read_only = 0 + OPTION(RECOMPILE); + + /* Get the database_id for backwards compatibility */ + SELECT + @database_id = d.database_id + FROM #databases AS d; + END; + END + ELSE + BEGIN + /* Multi-database mode */ + INSERT + #databases + WITH + (TABLOCK) + ( + database_name, + database_id + ) + SELECT + d.name, + d.database_id + FROM sys.databases AS d + WHERE d.database_id > 4 /* Skip system databases */ + AND d.state = 0 + AND d.is_in_standby = 0 + AND d.is_read_only = 0 + AND ( + @include_databases IS NULL + OR EXISTS (SELECT 1/0 FROM #include_databases AS id WHERE id.database_name = d.name) + ) + AND ( + @exclude_databases IS NULL + OR NOT EXISTS (SELECT 1/0 FROM #exclude_databases AS ed WHERE ed.database_name = d.name) + ) + OPTION(RECOMPILE); + END; + + /* Check for empty database list */ + IF (SELECT COUNT_BIG(*) FROM #databases AS d) = 0 + BEGIN + RAISERROR('No valid databases found to process.', 16, 1); + RETURN; + END; + + /* Show database list in debug mode */ + IF @debug = 1 + BEGIN + SELECT + table_name = '#databases', + d.* + FROM #databases AS d + OPTION(RECOMPILE); + END; + /* Set up database cursor processing */ - DECLARE - @database_cursor CURSOR, - @current_database_name sysname, - @current_database_id int; /* Create a cursor to process each database */ - SET - @database_cursor = + SET @database_cursor = CURSOR LOCAL SCROLL @@ -913,7 +900,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. FETCH FIRST FROM @database_cursor - INTO @current_database_name, @current_database_id; + INTO + @current_database_name, + @current_database_id; /* Start insert queries @@ -925,6 +914,34 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. WHILE @@FETCH_STATUS = 0 BEGIN + /*Validate searched objects per-database*/ + IF @schema_name IS NOT NULL + AND @table_name IS NOT NULL + BEGIN + IF @debug = 1 + BEGIN + RAISERROR('validating object existence for %s.%s.&s.', 0, 0, @database_name, @schema_name, @table_name) WITH NOWAIT; + END; + + SELECT + @full_object_name = + QUOTENAME(@current_database_name) + + N'.' + + QUOTENAME(@schema_name) + + N'.' + + QUOTENAME(@table_name); + + SELECT + @object_id = + OBJECT_ID(@full_object_name); + + IF @object_id IS NULL + BEGIN + RAISERROR('The object %s doesn''t seem to exist', 16, 1, @full_object_name) WITH NOWAIT; + RETURN; + END; + END; + /* Process current database */ IF @debug = 1 BEGIN @@ -1090,7 +1107,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ) EXECUTE sys.sp_executesql @sql, - N'@database_id int, + N'@database_id integer, @min_reads bigint, @min_writes bigint, @min_size_gb decimal(10,2), @@ -1103,17 +1120,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. @min_rows, @object_id; - /* Get the next database */ - FETCH NEXT - FROM @database_cursor - INTO @current_database_name, @current_database_id; - END; - - CLOSE @database_cursor; - DEALLOCATE @database_cursor; - /* Set database_id for backwards compatibility when processing single database */ - IF @get_all_databases = 0 AND (SELECT COUNT(*) FROM #databases) = 1 + IF @get_all_databases = 0 + AND (SELECT COUNT_BIG(*) FROM #databases AS d) = 1 BEGIN SELECT @database_id = d.database_id, @@ -1126,7 +1135,19 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. IF @debug = 1 BEGIN RAISERROR('No rows inserted into #filtered_objects, nothing to do!', 10, 0) WITH NOWAIT; - RETURN; + IF @get_all_databases = 0 + BEGIN + RETURN; + END; + IF @get_all_databases = 1 + BEGIN + /* Get the next database */ + FETCH NEXT + FROM @database_cursor + INTO + @current_database_name, + @current_database_id; + END; END; END; @@ -4615,6 +4636,14 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. RAISERROR('Generating #index_cleanup_results, RESULTS', 0, 0) WITH NOWAIT; END; + /* Get the next database */ + FETCH NEXT + FROM @database_cursor + INTO + @current_database_name, + @current_database_id; + END; + SELECT /* First, show the information needed to understand the script */ script_type = From 4e8e829c2c99c26974d12c7a2b62eb082c40af77 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 18 Mar 2025 12:34:22 -0400 Subject: [PATCH 201/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 33 +++++++++++++++-------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 481141c2..83d946a6 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -920,7 +920,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. BEGIN IF @debug = 1 BEGIN - RAISERROR('validating object existence for %s.%s.&s.', 0, 0, @database_name, @schema_name, @table_name) WITH NOWAIT; + RAISERROR('validating object existence for %s.%s.%s.', 0, 0, @current_database_name, @schema_name, @table_name) WITH NOWAIT; END; SELECT @@ -1134,21 +1134,22 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. BEGIN IF @debug = 1 BEGIN - RAISERROR('No rows inserted into #filtered_objects, nothing to do!', 10, 0) WITH NOWAIT; - IF @get_all_databases = 0 - BEGIN - RETURN; - END; - IF @get_all_databases = 1 - BEGIN - /* Get the next database */ - FETCH NEXT - FROM @database_cursor - INTO - @current_database_name, - @current_database_id; - END; - END; + RAISERROR('No rows inserted into #filtered_objects from %s, continuing to next database...', 10, 0, @current_database_name) WITH NOWAIT; + END; + + IF @get_all_databases = 0 + BEGIN + RETURN; + END; + + /* Get the next database and continue the loop */ + FETCH NEXT + FROM @database_cursor + INTO + @current_database_name, + @current_database_id; + + CONTINUE; END; IF @debug = 1 From 9717dc262c74da72f7febf4acacbd5ce6f9008af Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 18 Mar 2025 12:52:09 -0400 Subject: [PATCH 202/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 122 +++++++++++++++------------- 1 file changed, 67 insertions(+), 55 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 83d946a6..6d7d5ead 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -894,7 +894,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. d.database_name, d.database_id FROM #databases AS d - ORDER BY d.database_name; + ORDER BY + d.database_id; OPEN @database_cursor; @@ -937,15 +938,27 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. IF @object_id IS NULL BEGIN - RAISERROR('The object %s doesn''t seem to exist', 16, 1, @full_object_name) WITH NOWAIT; - RETURN; + RAISERROR('The object %s doesn''t seem to exist', 10, 1, @full_object_name) WITH NOWAIT; + + IF @get_all_databases = 0 + BEGIN + RETURN; + END; + + /* Get the next database and continue the loop */ + FETCH NEXT + FROM @database_cursor + INTO + @current_database_name, + @current_database_id; + CONTINUE; END; END; /* Process current database */ IF @debug = 1 BEGIN - RAISERROR('Processing database: %s (ID: %d)', 0, 0, @current_database_name, @current_database_id) WITH NOWAIT; + RAISERROR('Processing @current_database_name: %s and @current_database_id: %d', 0, 0, @current_database_name, @current_database_id) WITH NOWAIT; END; SELECT @@ -1020,14 +1033,14 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END; SET @sql += N' - AND NOT EXISTS - ( - SELECT - 1/0 - FROM ' + QUOTENAME(@current_database_name) + N'.sys.tables AS t - WHERE t.object_id = i.object_id - AND t.temporal_type > 0 - )'; + AND NOT EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@current_database_name) + N'.sys.tables AS t + WHERE t.object_id = i.object_id + AND t.temporal_type > 0 + )'; END; @@ -1039,50 +1052,50 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END; SELECT @sql += N' - AND t.object_id = @object_id'; + AND t.object_id = @object_id'; END; SET @sql += N' - AND EXISTS - ( - SELECT - 1/0 - FROM ' + QUOTENAME(@current_database_name) + N'.sys.dm_db_partition_stats AS ps - JOIN ' + QUOTENAME(@current_database_name) + N'.sys.allocation_units AS au - ON ps.partition_id = au.container_id - WHERE ps.object_id = t.object_id - GROUP BY - ps.object_id - HAVING - SUM(au.total_pages) * 8.0 / 1048576.0 >= @min_size_gb - ) - AND EXISTS - ( - SELECT - 1/0 - FROM ' + QUOTENAME(@current_database_name) + N'.sys.dm_db_partition_stats AS ps - WHERE ps.object_id = t.object_id - AND ps.index_id IN (0, 1) - GROUP BY - ps.object_id - HAVING - SUM(ps.row_count) >= @min_rows - ) - AND EXISTS - ( - SELECT - 1/0 - FROM ' + QUOTENAME(@current_database_name) + N'.sys.dm_db_index_usage_stats AS ius - WHERE ius.object_id = t.object_id - AND ius.database_id = @database_id - GROUP BY - ius.object_id - HAVING - SUM(ius.user_seeks + ius.user_scans + ius.user_lookups) >= @min_reads - OR - SUM(ius.user_updates) >= @min_writes - ) - OPTION(RECOMPILE); + AND EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@current_database_name) + N'.sys.dm_db_partition_stats AS ps + JOIN ' + QUOTENAME(@current_database_name) + N'.sys.allocation_units AS au + ON ps.partition_id = au.container_id + WHERE ps.object_id = t.object_id + GROUP BY + ps.object_id + HAVING + SUM(au.total_pages) * 8.0 / 1048576.0 >= @min_size_gb + ) + AND EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@current_database_name) + N'.sys.dm_db_partition_stats AS ps + WHERE ps.object_id = t.object_id + AND ps.index_id IN (0, 1) + GROUP BY + ps.object_id + HAVING + SUM(ps.row_count) >= @min_rows + ) + AND EXISTS + ( + SELECT + 1/0 + FROM ' + QUOTENAME(@current_database_name) + N'.sys.dm_db_index_usage_stats AS ius + WHERE ius.object_id = t.object_id + AND ius.database_id = @database_id + GROUP BY + ius.object_id + HAVING + SUM(ius.user_seeks + ius.user_scans + ius.user_lookups) >= @min_reads + OR + SUM(ius.user_updates) >= @min_writes + ) + OPTION(RECOMPILE); '; IF @debug = 1 @@ -1147,8 +1160,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. FROM @database_cursor INTO @current_database_name, - @current_database_id; - + @current_database_id; CONTINUE; END; From 889c64b392e1d573f57a68a17cb11683a308b493 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 18 Mar 2025 13:21:51 -0400 Subject: [PATCH 203/246] working on index cleanup all databases working on index cleanup all databases --- sp_IndexCleanup/sp_IndexCleanup.sql | 75 +++++++++++++++-------------- sp_QuickieStore/sp_QuickieStore.sql | 6 ++- 2 files changed, 43 insertions(+), 38 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 6d7d5ead..a05f27ce 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -273,7 +273,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. @current_database_name sysname, @current_database_id integer, @error_msg nvarchar(2048), - @conflict_list nvarchar(max) = N'' + @conflict_list nvarchar(max) = N'', + @rc bigint; /* Set uptime warning flag after @uptime_days is calculated */ SELECT @@ -827,7 +828,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Get the database_id for backwards compatibility */ SELECT - @database_id = d.database_id + @current_database_id = d.database_id FROM #databases AS d; END; END @@ -950,7 +951,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. FROM @database_cursor INTO @current_database_name, - @current_database_id; + @current_database_id; CONTINUE; END; END; @@ -1133,6 +1134,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. @min_rows, @object_id; + SET @rc = ROWCOUNT_BIG(); + /* Set database_id for backwards compatibility when processing single database */ IF @get_all_databases = 0 AND (SELECT COUNT_BIG(*) FROM #databases AS d) = 1 @@ -1143,7 +1146,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. FROM #databases AS d; END; - IF ROWCOUNT_BIG() = 0 + IF @rc = 0 BEGIN IF @debug = 1 BEGIN @@ -1160,7 +1163,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. FROM @database_cursor INTO @current_database_name, - @current_database_id; + @current_database_id; CONTINUE; END; @@ -1328,18 +1331,18 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. page_io_latch_wait_in_ms = SUM(os.page_io_latch_wait_in_ms), page_compression_attempt_count = SUM(os.page_compression_attempt_count), page_compression_success_count = SUM(os.page_compression_success_count) - FROM ' + QUOTENAME(@database_name) + N'.sys.dm_db_index_operational_stats + FROM ' + QUOTENAME(@current_database_name) + N'.sys.dm_db_index_operational_stats ( @database_id, @object_id, NULL, NULL ) AS os - JOIN ' + QUOTENAME(@database_name) + N'.sys.tables AS t + JOIN ' + QUOTENAME(@current_database_name) + N'.sys.tables AS t ON os.object_id = t.object_id - JOIN ' + QUOTENAME(@database_name) + N'.sys.schemas AS s + JOIN ' + QUOTENAME(@current_database_name) + N'.sys.schemas AS s ON t.schema_id = s.schema_id - JOIN ' + QUOTENAME(@database_name) + N'.sys.indexes AS i + JOIN ' + QUOTENAME(@current_database_name) + N'.sys.indexes AS i ON os.object_id = i.object_id AND os.index_id = i.index_id WHERE EXISTS @@ -1415,7 +1418,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. @sql, N'@database_id integer, @object_id integer', - @database_id, + @current_database_id, @object_id; IF ROWCOUNT_BIG() = 0 @@ -1462,7 +1465,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ( SELECT 1/0 - FROM ' + QUOTENAME(@database_name) + N'.sys.objects AS so + FROM ' + QUOTENAME(@current_database_name) + N'.sys.objects AS so WHERE i.object_id = so.object_id AND so.is_ms_shipped = 0 AND so.type = ''V'' @@ -1476,7 +1479,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ( SELECT 1/0 - FROM ' + QUOTENAME(@database_name) + N'.sys.foreign_key_columns AS f + FROM ' + QUOTENAME(@current_database_name) + N'.sys.foreign_key_columns AS f WHERE f.parent_column_id = c.column_id AND f.parent_object_id = c.object_id ) @@ -1489,7 +1492,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ( SELECT 1/0 - FROM ' + QUOTENAME(@database_name) + N'.sys.foreign_key_columns AS f + FROM ' + QUOTENAME(@current_database_name) + N'.sys.foreign_key_columns AS f WHERE f.referenced_column_id = c.column_id AND f.referenced_object_id = c.object_id ) @@ -1507,7 +1510,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ( SELECT 1/0 - FROM ' + QUOTENAME(@database_name) + N'.sys.types AS t + FROM ' + QUOTENAME(@current_database_name) + N'.sys.types AS t WHERE c.system_type_id = t.system_type_id AND c.user_type_id = t.user_type_id AND t.name IN (N''varchar'', N''nvarchar'') @@ -1531,15 +1534,15 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. WHEN i.type = 1 THEN 0 END - FROM ' + QUOTENAME(@database_name) + N'.sys.tables AS t - JOIN ' + QUOTENAME(@database_name) + N'.sys.schemas AS s + FROM ' + QUOTENAME(@current_database_name) + N'.sys.tables AS t + JOIN ' + QUOTENAME(@current_database_name) + N'.sys.schemas AS s ON t.schema_id = s.schema_id - JOIN ' + QUOTENAME(@database_name) + N'.sys.indexes AS i + JOIN ' + QUOTENAME(@current_database_name) + N'.sys.indexes AS i ON t.object_id = i.object_id - JOIN ' + QUOTENAME(@database_name) + N'.sys.index_columns AS ic + JOIN ' + QUOTENAME(@current_database_name) + N'.sys.index_columns AS ic ON i.object_id = ic.object_id AND i.index_id = ic.index_id - JOIN ' + QUOTENAME(@database_name) + + JOIN ' + QUOTENAME(@current_database_name) + CONVERT ( nvarchar(MAX), @@ -1567,7 +1570,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SELECT 1/0 FROM ' - ) + QUOTENAME(@database_name) + + ) + QUOTENAME(@current_database_name) + CONVERT ( nvarchar(MAX), @@ -1598,7 +1601,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ( SELECT 1/0 - FROM ' + QUOTENAME(@database_name) + N'.sys.objects AS so + FROM ' + QUOTENAME(@current_database_name) + N'.sys.objects AS so WHERE i.object_id = so.object_id AND so.is_ms_shipped = 0 AND so.type = N''TF'' @@ -1654,7 +1657,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. N'@database_id integer, @object_id integer, @min_rows integer', - @database_id, + @current_database_id, @object_id, @min_rows; @@ -1723,17 +1726,17 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. reserved_row_overflow_gb = SUM(ps.row_overflow_reserved_page_count) * 8. / 1024. / 1024.0, /* Convert directly to GB */ p.data_compression_desc, i.data_space_id - FROM ' + QUOTENAME(@database_name) + N'.sys.tables AS t - JOIN ' + QUOTENAME(@database_name) + N'.sys.indexes AS i + FROM ' + QUOTENAME(@current_database_name) + N'.sys.tables AS t + JOIN ' + QUOTENAME(@current_database_name) + N'.sys.indexes AS i ON t.object_id = i.object_id - JOIN ' + QUOTENAME(@database_name) + N'.sys.schemas AS s + JOIN ' + QUOTENAME(@current_database_name) + N'.sys.schemas AS s ON t.schema_id = s.schema_id - JOIN ' + QUOTENAME(@database_name) + N'.sys.partitions AS p + JOIN ' + QUOTENAME(@current_database_name) + N'.sys.partitions AS p ON i.object_id = p.object_id AND i.index_id = p.index_id - JOIN ' + QUOTENAME(@database_name) + N'.sys.allocation_units AS a + JOIN ' + QUOTENAME(@current_database_name) + N'.sys.allocation_units AS a ON p.partition_id = a.container_id - LEFT HASH JOIN ' + QUOTENAME(@database_name) + N'.sys.dm_db_partition_stats AS ps + LEFT HASH JOIN ' + QUOTENAME(@current_database_name) + N'.sys.dm_db_partition_stats AS ps ON p.partition_id = ps.partition_id WHERE t.type <> N''TF'' AND i.type IN (1, 2) @@ -1781,10 +1784,10 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ps.name, partition_function_name = pf.name - FROM ' + QUOTENAME(@database_name) + N'.sys.filegroups AS fg - FULL JOIN ' + QUOTENAME(@database_name) + N'.sys.partition_schemes AS ps + FROM ' + QUOTENAME(@current_database_name) + N'.sys.filegroups AS fg + FULL JOIN ' + QUOTENAME(@current_database_name) + N'.sys.partition_schemes AS ps ON ps.data_space_id = fg.data_space_id - LEFT JOIN ' + QUOTENAME(@database_name) + N'.sys.partition_functions AS pf + LEFT JOIN ' + QUOTENAME(@current_database_name) + N'.sys.partition_functions AS pf ON pf.function_id = ps.function_id WHERE x.data_space_id = fg.data_space_id OR x.data_space_id = ps.data_space_id @@ -1799,8 +1802,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SELECT N'', '' + c.name - FROM ' + QUOTENAME(@database_name) + N'.sys.index_columns AS ic - JOIN ' + QUOTENAME(@database_name) + N'.sys.columns AS c + FROM ' + QUOTENAME(@current_database_name) + N'.sys.index_columns AS ic + JOIN ' + QUOTENAME(@current_database_name) + N'.sys.columns AS c ON c.object_id = ic.object_id AND c.column_id = ic.column_id WHERE ic.object_id = x.object_id @@ -1853,7 +1856,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. @sql, N'@database_id integer, @object_id integer', - @database_id, + @current_database_id, @object_id; IF ROWCOUNT_BIG() = 0 @@ -1967,7 +1970,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. WHEN id1.is_unique_constraint = 1 THEN N'ALTER TABLE ' + - QUOTENAME(DB_NAME(@database_id)) + + QUOTENAME(DB_NAME(@current_database_id)) + N'.' + QUOTENAME(id1.schema_name) + N'.' + @@ -1982,7 +1985,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. N'INDEX ' + QUOTENAME(id1.index_name) + N' ON ' + - QUOTENAME(DB_NAME(@database_id)) + + QUOTENAME(DB_NAME(@current_database_id)) + N'.' + QUOTENAME(id1.schema_name) + N'.' + diff --git a/sp_QuickieStore/sp_QuickieStore.sql b/sp_QuickieStore/sp_QuickieStore.sql index 11785680..fcaf1fa2 100644 --- a/sp_QuickieStore/sp_QuickieStore.sql +++ b/sp_QuickieStore/sp_QuickieStore.sql @@ -2103,7 +2103,8 @@ BEGIN OPTION(RECOMPILE); /* Track which requested databases were skipped */ - IF @include_databases IS NOT NULL AND @get_all_databases = 1 + IF @include_databases IS NOT NULL + AND @get_all_databases = 1 BEGIN INSERT #requested_but_skipped_databases @@ -2187,7 +2188,8 @@ BEGIN OPTION(RECOMPILE); /* Track which requested databases were skipped */ - IF @include_databases IS NOT NULL AND @get_all_databases = 1 + IF @include_databases IS NOT NULL + AND @get_all_databases = 1 BEGIN INSERT #requested_but_skipped_databases From 3717cdd61c6ded6ca003adfdb3ad920f563e50fd Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 18 Mar 2025 13:32:54 -0400 Subject: [PATCH 204/246] Update sp_IndexCleanup.sql gobblefuck mother shit ass --- sp_IndexCleanup/sp_IndexCleanup.sql | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index a05f27ce..537f018e 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -1136,16 +1136,6 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SET @rc = ROWCOUNT_BIG(); - /* Set database_id for backwards compatibility when processing single database */ - IF @get_all_databases = 0 - AND (SELECT COUNT_BIG(*) FROM #databases AS d) = 1 - BEGIN - SELECT - @database_id = d.database_id, - @database_name = d.database_name - FROM #databases AS d; - END; - IF @rc = 0 BEGIN IF @debug = 1 @@ -1898,8 +1888,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. original_index_definition ) SELECT - @database_id, - database_name = DB_NAME(@database_id), + @current_database_id, + database_name = DB_NAME(@current_database_id), id1.schema_id, id1.schema_name, id1.table_name, @@ -4648,8 +4638,6 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. icr.* FROM #index_cleanup_results AS icr OPTION(RECOMPILE); - - RAISERROR('Generating #index_cleanup_results, RESULTS', 0, 0) WITH NOWAIT; END; /* Get the next database */ @@ -4660,6 +4648,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. @current_database_id; END; + IF @debug = 1 + BEGIN + RAISERROR('Generating #index_cleanup_results, RESULTS', 0, 0) WITH NOWAIT; + END; + SELECT /* First, show the information needed to understand the script */ script_type = From 60cc2e30a0688236f5d165ec43c6888c564a02d2 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 18 Mar 2025 13:33:21 -0400 Subject: [PATCH 205/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index a05f27ce..5271b418 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -1253,8 +1253,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ( SELECT 1/0 - FROM ' + QUOTENAME(@database_name) + N'.sys.columns AS c - JOIN ' + QUOTENAME(@database_name) + N'.sys.types AS t + FROM ' + QUOTENAME(@current_database_name) + N'.sys.columns AS c + JOIN ' + QUOTENAME(@current_database_name) + N'.sys.types AS t ON c.user_type_id = t.user_type_id WHERE c.object_id = ce.object_id AND @@ -1745,7 +1745,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SELECT 1/0 FROM #filtered_objects AS fo - WHERE fo.database_id = @database_id + WHERE fo.database_id = @current_database_id AND fo.object_id = t.object_id )'; From f66199ade27dda9be4a478d8e27ee832c0f3f11a Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 18 Mar 2025 13:37:09 -0400 Subject: [PATCH 206/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index e0aa72cc..0ba6df75 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -4731,8 +4731,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND ir.index_name = ia.index_name WHERE ir.rn = 1 /* Take only the first row for each index */ ORDER BY - ir.sort_order, ir.database_name, + ir.sort_order, /* Within each sort_order group, prioritize by size and usage */ CASE /* For SUMMARY, keep the original order */ @@ -4930,15 +4930,15 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. FROM #index_reporting_stats AS irs WHERE irs.summary_level IN ('SUMMARY', 'DATABASE', 'TABLE') /* Filter out INDEX level */ ORDER BY - /* Order by level - summary first */ + /* Order by database name */ + irs.database_name, + /* Then order by level - summary first */ CASE WHEN irs.summary_level = 'SUMMARY' THEN 0 WHEN irs.summary_level = 'DATABASE' THEN 1 WHEN irs.summary_level = 'TABLE' THEN 2 ELSE 3 END, - /* Then by database name */ - irs.database_name, /* For tables, sort by potential savings and size */ CASE WHEN irs.summary_level = 'TABLE' THEN irs.unused_size_gb From c85e88ba689ee28622afdb1d5203822dda6d0241 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 18 Mar 2025 13:55:52 -0400 Subject: [PATCH 207/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 0ba6df75..dd222a43 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -4799,6 +4799,12 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CASE WHEN irs.summary_level = 'SUMMARY' THEN FORMAT(irs.tables_analyzed, 'N0') + WHEN irs.summary_level = 'DATABASE' + THEN FORMAT((SELECT COUNT_BIG(DISTINCT CONCAT(ia.schema_id, N'.', ia.object_id)) + FROM #index_analysis AS ia + WHERE ia.database_name = irs.database_name), 'N0') + WHEN irs.summary_level = 'TABLE' + THEN FORMAT(1, 'N0') /* Each table row represents 1 analyzed table */ ELSE FORMAT(0, 'N0') /* Show 0 instead of NULL */ END, From 8ffb8b3e7bd58dd866a40f76ff1456bf434dcab1 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 18 Mar 2025 13:57:34 -0400 Subject: [PATCH 208/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index dd222a43..40fde3bd 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -1735,7 +1735,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SELECT 1/0 FROM #filtered_objects AS fo - WHERE fo.database_id = @current_database_id + WHERE fo.database_id = @database_id AND fo.object_id = t.object_id )'; From 9d2b42cb21b617e447956abcfe699b3086582a98 Mon Sep 17 00:00:00 2001 From: Hannah Vernon Date: Tue, 18 Mar 2025 14:55:54 -0500 Subject: [PATCH 209/246] added QUOTENAME to the output of the 'original_index_definition' column. Added terminating semi-colon to the `original_index_definition` column. Modified the order of included columns to match the `column_id` from sys.columns to more closely resemble the output from SSMS script index command. --- sp_IndexCleanup/sp_IndexCleanup.sql | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 40fde3bd..9e85c918 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -453,6 +453,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. index_id integer NOT NULL, index_name sysname NULL, column_name sysname NOT NULL, + column_id int NOT NULL, is_primary_key bit NULL, is_unique bit NULL, is_unique_constraint bit NULL, @@ -1446,6 +1447,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. table_name = t.name, index_name = ISNULL(i.name, t.name + N''.Heap''), column_name = c.name, + column_id = c.column_id, i.is_primary_key, i.is_unique, i.is_unique_constraint, @@ -1620,6 +1622,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. table_name, index_name, column_name, + column_id, is_primary_key, is_unique, is_unique_constraint, @@ -1903,7 +1906,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ( SELECT N', ' + - id2.column_name + + QUOTENAME(id2.column_name) + CASE WHEN id2.is_descending_key = 1 THEN N' DESC' @@ -1934,7 +1937,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ( SELECT N', ' + - id2.column_name + QUOTENAME(id2.column_name) FROM #index_details id2 WHERE id2.object_id = id1.object_id AND id2.index_id = id1.index_id @@ -1972,6 +1975,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ELSE N'CREATE ' + CASE WHEN id1.is_unique = 1 THEN N'UNIQUE ' ELSE N'' END + + CASE WHEN id1.index_id > 0 THEN N'NONCLUSTERED ' ELSE N'' END + N'INDEX ' + QUOTENAME(id1.index_name) + N' ON ' + @@ -1987,7 +1991,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ( SELECT N', ' + - id2.column_name + + QUOTENAME(id2.column_name) + CASE WHEN id2.is_descending_key = 1 THEN N' DESC' @@ -2029,14 +2033,16 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ( SELECT N', ' + - id4.column_name + QUOTENAME(id4.column_name) FROM #index_details id4 WHERE id4.object_id = id1.object_id AND id4.index_id = id1.index_id AND id4.is_included_column = 1 GROUP BY + id4.column_id, id4.column_name ORDER BY + id4.column_id, id4.column_name FOR XML @@ -2054,7 +2060,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. WHEN id1.filter_definition IS NOT NULL THEN N' WHERE ' + id1.filter_definition ELSE N'' - END + END + + N';' FROM #index_details id1 WHERE id1.is_eligible_for_dedupe = 1 GROUP BY From 94d14ce5457b902b31570cb1eacbbfb5748fcace Mon Sep 17 00:00:00 2001 From: Hannah Vernon Date: Tue, 18 Mar 2025 15:21:54 -0500 Subject: [PATCH 210/246] added `@verbose_output` parameter to control output of NONUNIQUE and NONCLUSTERED to the original_index_definition output column. --- sp_IndexCleanup/sp_IndexCleanup.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 9e85c918..c27f4c80 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -26,6 +26,7 @@ ALTER PROCEDURE @get_all_databases bit = 0, /*looks for all accessible user databases and returns combined results*/ @include_databases nvarchar(max) = NULL, /*comma-separated list of databases to include (only when @get_all_databases = 1)*/ @exclude_databases nvarchar(max) = NULL, /*comma-separated list of databases to exclude (only when @get_all_databases = 1)*/ + @verbose_output tinyint = 0, /* 0 -> no verbose output, 1 -> add NONUNIQUE, NONCLUSTERED type output in the original_index_defintion output */ @help bit = 'false', @debug bit = 'false', @version varchar(20) = NULL OUTPUT, @@ -1974,8 +1975,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* For regular indexes, use CREATE INDEX syntax */ ELSE N'CREATE ' + - CASE WHEN id1.is_unique = 1 THEN N'UNIQUE ' ELSE N'' END + - CASE WHEN id1.index_id > 0 THEN N'NONCLUSTERED ' ELSE N'' END + + CASE WHEN id1.is_unique = 1 THEN N'UNIQUE ' WHEN id1.is_unique = 0 AND @verbose_output >= 1 THEN N'NONUNIQUE ' ELSE N'' END + + CASE WHEN id1.index_id = 0 THEN N'CLUSTERED ' WHEN id1.index_id > 0 AND @verbose_output >= 1 THEN N'NONCLUSTERED ' ELSE N'' END + N'INDEX ' + QUOTENAME(id1.index_name) + N' ON ' + From 64e8c372f5fbe1022fe025991ef08c88503309ee Mon Sep 17 00:00:00 2001 From: Hannah Vernon Date: Tue, 18 Mar 2025 16:03:18 -0500 Subject: [PATCH 211/246] Removed the syntactically invalid `NONUNIQUE` bit from the `original_index_definition` column. --- sp_IndexCleanup/sp_IndexCleanup.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index c27f4c80..ec7da4c4 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -1975,7 +1975,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* For regular indexes, use CREATE INDEX syntax */ ELSE N'CREATE ' + - CASE WHEN id1.is_unique = 1 THEN N'UNIQUE ' WHEN id1.is_unique = 0 AND @verbose_output >= 1 THEN N'NONUNIQUE ' ELSE N'' END + + CASE WHEN id1.is_unique = 1 THEN N'UNIQUE ' ELSE N'' END + CASE WHEN id1.index_id = 0 THEN N'CLUSTERED ' WHEN id1.index_id > 0 AND @verbose_output >= 1 THEN N'NONCLUSTERED ' ELSE N'' END + N'INDEX ' + QUOTENAME(id1.index_name) + From b7ab24246fd719cd2e8fe56f9fd71e0936051cf9 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 18 Mar 2025 17:18:45 -0400 Subject: [PATCH 212/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index ec7da4c4..a99aeff4 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -918,6 +918,20 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. WHILE @@FETCH_STATUS = 0 BEGIN + /*Truncate temp tables between database iterations*/ + IF @debug = 1 + BEGIN + RAISERROR('Truncating per-database temp tables for the next iteration', 0, 0) WITH NOWAIT; + END; + + TRUNCATE TABLE #filtered_objects; + TRUNCATE TABLE #operational_stats; + TRUNCATE TABLE #partition_stats; + TRUNCATE TABLE #index_details; + TRUNCATE TABLE #compression_eligibility; + TRUNCATE TABLE #key_duplicate_dedupe; + TRUNCATE TABLE #include_subset_dedupe; + /*Validate searched objects per-database*/ IF @schema_name IS NOT NULL AND @table_name IS NOT NULL From bd57a09b01e82e2a75936a097d7938c7b9b68200 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 18 Mar 2025 17:18:57 -0400 Subject: [PATCH 213/246] Update sp_QuickieStore.sql --- sp_QuickieStore/sp_QuickieStore.sql | 199 ++++++++++++++-------------- 1 file changed, 101 insertions(+), 98 deletions(-) diff --git a/sp_QuickieStore/sp_QuickieStore.sql b/sp_QuickieStore/sp_QuickieStore.sql index fcaf1fa2..90beadba 100644 --- a/sp_QuickieStore/sp_QuickieStore.sql +++ b/sp_QuickieStore/sp_QuickieStore.sql @@ -2260,6 +2260,107 @@ INTO @database_name; WHILE @@FETCH_STATUS = 0 BEGIN +/* +These tables need to get cleared out +to avoid result pollution and +primary key violations +*/ +IF @debug = 1 +BEGIN + RAISERROR('Truncating per-database temp tables for the next iteration', 0, 0) WITH NOWAIT; +END; + +TRUNCATE TABLE + #regression_baseline_runtime_stats; + +TRUNCATE TABLE + #regression_current_runtime_stats; + +TRUNCATE TABLE + #distinct_plans; + +TRUNCATE TABLE + #procedure_plans; + +TRUNCATE TABLE + #procedure_object_ids; + +TRUNCATE TABLE + #maintenance_plans; + +TRUNCATE TABLE + #query_text_search; + +TRUNCATE TABLE + #query_text_search_not; + +TRUNCATE TABLE + #dm_exec_query_stats; + +TRUNCATE TABLE + #query_types; + +TRUNCATE TABLE + #wait_filter; + +TRUNCATE TABLE + #only_queries_with_hints; + +TRUNCATE TABLE + #only_queries_with_feedback; + +TRUNCATE TABLE + #only_queries_with_variants; + +TRUNCATE TABLE + #forced_plans_failures; + +TRUNCATE TABLE + #include_plan_ids; + +TRUNCATE TABLE + #include_query_ids; + +TRUNCATE TABLE + #include_query_hashes; + +TRUNCATE TABLE + #include_plan_hashes; + +TRUNCATE TABLE + #include_sql_handles; + +TRUNCATE TABLE + #ignore_plan_ids; + +TRUNCATE TABLE + #ignore_query_ids; + +TRUNCATE TABLE + #ignore_query_hashes; + +TRUNCATE TABLE + #ignore_plan_hashes; + +TRUNCATE TABLE + #ignore_sql_handles; + +TRUNCATE TABLE + #only_queries_with_hints; + +TRUNCATE TABLE + #only_queries_with_feedback; + +TRUNCATE TABLE + #only_queries_with_variants; + +TRUNCATE TABLE + #forced_plans_failures; + +TRUNCATE TABLE + #query_hash_totals; + + /* Some variable assignment, because why not? */ @@ -8101,104 +8202,6 @@ OPTION(RECOMPILE);' + @nc10; END; /*End AG queries*/ END; /*End SQL 2022 views*/ -/* -These tables need to get cleared out -to avoid result pollution and -primary key violations -*/ -IF @get_all_databases = 1 -BEGIN - TRUNCATE TABLE - #regression_baseline_runtime_stats; - - TRUNCATE TABLE - #regression_current_runtime_stats; - - TRUNCATE TABLE - #distinct_plans; - - TRUNCATE TABLE - #procedure_plans; - - TRUNCATE TABLE - #procedure_object_ids; - - TRUNCATE TABLE - #maintenance_plans; - - TRUNCATE TABLE - #query_text_search; - - TRUNCATE TABLE - #query_text_search_not; - - TRUNCATE TABLE - #dm_exec_query_stats; - - TRUNCATE TABLE - #query_types; - - TRUNCATE TABLE - #wait_filter; - - TRUNCATE TABLE - #only_queries_with_hints; - - TRUNCATE TABLE - #only_queries_with_feedback; - - TRUNCATE TABLE - #only_queries_with_variants; - - TRUNCATE TABLE - #forced_plans_failures; - - TRUNCATE TABLE - #include_plan_ids; - - TRUNCATE TABLE - #include_query_ids; - - TRUNCATE TABLE - #include_query_hashes; - - TRUNCATE TABLE - #include_plan_hashes; - - TRUNCATE TABLE - #include_sql_handles; - - TRUNCATE TABLE - #ignore_plan_ids; - - TRUNCATE TABLE - #ignore_query_ids; - - TRUNCATE TABLE - #ignore_query_hashes; - - TRUNCATE TABLE - #ignore_plan_hashes; - - TRUNCATE TABLE - #ignore_sql_handles; - - TRUNCATE TABLE - #only_queries_with_hints; - - TRUNCATE TABLE - #only_queries_with_feedback; - - TRUNCATE TABLE - #only_queries_with_variants; - - TRUNCATE TABLE - #forced_plans_failures; - - TRUNCATE TABLE - #query_hash_totals; -END; - FETCH NEXT FROM @database_cursor INTO @database_name; From b984113a8c93118dd374985c58c01ef8d7b99158 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 18 Mar 2025 17:29:58 -0400 Subject: [PATCH 214/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index a99aeff4..de226c13 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -476,7 +476,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. last_user_lookup datetime NULL, last_user_update datetime NULL, is_eligible_for_dedupe bit NOT NULL - PRIMARY KEY CLUSTERED(database_id, object_id, index_id, column_name) + PRIMARY KEY CLUSTERED(database_id, schema_id, object_id, index_id, column_id) ); CREATE TABLE @@ -512,7 +512,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. superseded_by nvarchar(4000) NULL, /* Priority score from 0-1 to determine which index to keep (higher is better) */ index_priority decimal(10,6) NULL - PRIMARY KEY CLUSTERED(database_id, object_id, index_id) + PRIMARY KEY CLUSTERED(database_id, schema_id, object_id, index_id) ); CREATE TABLE @@ -528,7 +528,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. index_name sysname NOT NULL, can_compress bit NOT NULL, reason nvarchar(200) NULL, - PRIMARY KEY CLUSTERED(database_id, object_id, index_id) + PRIMARY KEY CLUSTERED(database_id, schema_id, object_id, index_id) ); CREATE TABLE @@ -564,7 +564,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. base_key_columns nvarchar(max) NULL, filter_definition nvarchar(max) NULL, winning_index_name sysname NULL, - index_list nvarchar(max) NULL, + index_list nvarchar(max) NULL ); CREATE TABLE @@ -924,13 +924,20 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. RAISERROR('Truncating per-database temp tables for the next iteration', 0, 0) WITH NOWAIT; END; - TRUNCATE TABLE #filtered_objects; - TRUNCATE TABLE #operational_stats; - TRUNCATE TABLE #partition_stats; - TRUNCATE TABLE #index_details; - TRUNCATE TABLE #compression_eligibility; - TRUNCATE TABLE #key_duplicate_dedupe; - TRUNCATE TABLE #include_subset_dedupe; + TRUNCATE TABLE + #filtered_objects; + TRUNCATE TABLE + #operational_stats; + TRUNCATE TABLE + #partition_stats; + TRUNCATE TABLE + #index_details; + TRUNCATE TABLE + #compression_eligibility; + TRUNCATE TABLE + #key_duplicate_dedupe; + TRUNCATE TABLE + #include_subset_dedupe; /*Validate searched objects per-database*/ IF @schema_name IS NOT NULL From d6c91715741574cf27f5ad1523bb6a7a64c1d137 Mon Sep 17 00:00:00 2001 From: Hannah Vernon Date: Tue, 18 Mar 2025 16:43:11 -0500 Subject: [PATCH 215/246] added note about index_count column to the README.md --- sp_IndexCleanup/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/sp_IndexCleanup/README.md b/sp_IndexCleanup/README.md index 3e3a3706..fd781644 100644 --- a/sp_IndexCleanup/README.md +++ b/sp_IndexCleanup/README.md @@ -84,6 +84,7 @@ EXECUTE dbo.sp_IndexCleanup - The multi-database processing feature (@get_all_databases) analyzes each database sequentially for better performance and resource management - System databases (master, model, msdb, tempdb, rdsadmin) are always excluded from processing - When using @get_all_databases, results for all databases are combined in a single result set +- The index_count column for the SUMMARY row in the output table will likely indicate a lower number than is shown at the DATABASE level. The SUMMARY level only includes indexes that have been analyzed; excluding things like clustered indexes, heaps, xml indexes, etc. The DATABASE level index_count value is the total number of indexes in the database. Copyright 2024 Darling Data, LLC Released under MIT license \ No newline at end of file From 7cac5c2b0078fb27003468c04490febfb08e080a Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 18 Mar 2025 18:02:53 -0400 Subject: [PATCH 216/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index de226c13..6ebcf018 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -4917,13 +4917,14 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ELSE 'N/A' END, - /* Daily write operations saved - added as new metric */ - daily_write_ops_saved = + /* Write operations saved - added as new metric */ + write_ops_saved = CASE WHEN irs.summary_level <> 'SUMMARY' - THEN FORMAT(ISNULL(irs.user_updates / NULLIF(CONVERT(DECIMAL(10,2), + THEN FORMAT(ISNULL(irs.user_updates / + NULLIF(CONVERT(decimal(10,2), (SELECT TOP (1) irs2.server_uptime_days FROM #index_reporting_stats AS irs2 WHERE irs2.summary_level = 'DATABASE')), 0) * - (ISNULL(irs.unused_indexes, 0) / NULLIF(CONVERT(DECIMAL(10,2), irs.index_count), 0)), 0), 'N0') + (ISNULL(irs.unused_indexes, 0) / NULLIF(CONVERT(decimal(10,2), irs.index_count), 0)), 0), 'N0') ELSE 'N/A' END, From 9899fc3af2ac092fbd90aeaec2d4513a3bb2996b Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 18 Mar 2025 18:22:11 -0400 Subject: [PATCH 217/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 33 ++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 6ebcf018..d98f9d5a 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -4918,7 +4918,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END, /* Write operations saved - added as new metric */ - write_ops_saved = + daily_write_ops_saved = CASE WHEN irs.summary_level <> 'SUMMARY' THEN FORMAT(ISNULL(irs.user_updates / @@ -4937,6 +4937,17 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ISNULL(irs.page_lock_wait_count, 0), 'N0') ELSE '0' END, + + /* Lock waits saved - new column */ + daily_lock_waits_saved = + CASE + WHEN irs.summary_level <> 'SUMMARY' + THEN FORMAT(ISNULL((irs.row_lock_wait_count + irs.page_lock_wait_count) / + NULLIF(CONVERT(decimal(10,2), + (SELECT TOP (1) irs2.server_uptime_days FROM #index_reporting_stats AS irs2 WHERE irs2.summary_level = 'DATABASE')), 0) * + (ISNULL(irs.unused_indexes, 0) / NULLIF(CONVERT(decimal(10,2), irs.index_count), 0)), 0), 'N0') + ELSE 'N/A' + END, /* Average lock wait time in ms */ avg_lock_wait_ms = @@ -4951,6 +4962,26 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ELSE '0.00' END, + /* Total count of latch waits (page + io) - new column */ + latch_wait_count = + CASE + WHEN irs.summary_level <> 'SUMMARY' + THEN FORMAT(ISNULL(irs.page_latch_wait_count, 0) + + ISNULL(irs.page_io_latch_wait_count, 0), 'N0') + ELSE '0' + END, + + /* Latch waits saved - new column */ + daily_latch_waits_saved = + CASE + WHEN irs.summary_level <> 'SUMMARY' + THEN FORMAT(ISNULL((irs.page_latch_wait_count + irs.page_io_latch_wait_count) / + NULLIF(CONVERT(decimal(10,2), + (SELECT TOP (1) irs2.server_uptime_days FROM #index_reporting_stats AS irs2 WHERE irs2.summary_level = 'DATABASE')), 0) * + (ISNULL(irs.unused_indexes, 0) / NULLIF(CONVERT(decimal(10,2), irs.index_count), 0)), 0), 'N0') + ELSE 'N/A' + END, + /* Combined latch wait time in ms */ avg_latch_wait_ms = CASE From 8304b773c9219a915db0fd2b3b9d29a45b5107e6 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 18 Mar 2025 18:34:27 -0400 Subject: [PATCH 218/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 129 +++++++++++++++++++++++++--- 1 file changed, 117 insertions(+), 12 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index d98f9d5a..f4acc5cc 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -4921,10 +4921,45 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. daily_write_ops_saved = CASE WHEN irs.summary_level <> 'SUMMARY' - THEN FORMAT(ISNULL(irs.user_updates / - NULLIF(CONVERT(decimal(10,2), - (SELECT TOP (1) irs2.server_uptime_days FROM #index_reporting_stats AS irs2 WHERE irs2.summary_level = 'DATABASE')), 0) * - (ISNULL(irs.unused_indexes, 0) / NULLIF(CONVERT(decimal(10,2), irs.index_count), 0)), 0), 'N0') + THEN FORMAT + ( + ISNULL + ( + irs.user_updates / + NULLIF + ( + CONVERT + ( + decimal(10,2), + ( + SELECT TOP (1) + irs2.server_uptime_days + FROM #index_reporting_stats AS irs2 + WHERE irs2.summary_level = 'DATABASE' + ) + ), + 0 + ) * + ( + ISNULL + ( + irs.unused_indexes, + 0 + ) / + NULLIF + ( + CONVERT + ( + decimal(10,2), + irs.index_count + ), + 0 + ) + ), + 0 + ), + 'N0' + ) ELSE 'N/A' END, @@ -4942,10 +4977,45 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. daily_lock_waits_saved = CASE WHEN irs.summary_level <> 'SUMMARY' - THEN FORMAT(ISNULL((irs.row_lock_wait_count + irs.page_lock_wait_count) / - NULLIF(CONVERT(decimal(10,2), - (SELECT TOP (1) irs2.server_uptime_days FROM #index_reporting_stats AS irs2 WHERE irs2.summary_level = 'DATABASE')), 0) * - (ISNULL(irs.unused_indexes, 0) / NULLIF(CONVERT(decimal(10,2), irs.index_count), 0)), 0), 'N0') + THEN FORMAT + ( + ISNULL + ( + (irs.row_lock_wait_count + irs.page_lock_wait_count) / + NULLIF + ( + CONVERT + ( + decimal(10,2), + ( + SELECT TOP (1) + irs2.server_uptime_days + FROM #index_reporting_stats AS irs2 + WHERE irs2.summary_level = 'DATABASE' + ) + ), + 0 + ) * + ( + ISNULL + ( + irs.unused_indexes, + 0 + ) / + NULLIF + ( + CONVERT + ( + decimal(10,2), + irs.index_count + ), + 0 + ) + ), + 0 + ), + 'N0' + ) ELSE 'N/A' END, @@ -4975,10 +5045,45 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. daily_latch_waits_saved = CASE WHEN irs.summary_level <> 'SUMMARY' - THEN FORMAT(ISNULL((irs.page_latch_wait_count + irs.page_io_latch_wait_count) / - NULLIF(CONVERT(decimal(10,2), - (SELECT TOP (1) irs2.server_uptime_days FROM #index_reporting_stats AS irs2 WHERE irs2.summary_level = 'DATABASE')), 0) * - (ISNULL(irs.unused_indexes, 0) / NULLIF(CONVERT(decimal(10,2), irs.index_count), 0)), 0), 'N0') + THEN FORMAT + ( + ISNULL + ( + (irs.page_latch_wait_count + irs.page_io_latch_wait_count) / + NULLIF + ( + CONVERT + ( + decimal(10,2), + ( + SELECT TOP (1) + irs2.server_uptime_days + FROM #index_reporting_stats AS irs2 + WHERE irs2.summary_level = 'DATABASE' + ) + ), + 0 + ) * + ( + ISNULL + ( + irs.unused_indexes, + 0 + ) / + NULLIF + ( + CONVERT + ( + decimal(10,2), + irs.index_count + ), + 0 + ) + ), + 0 + ), + 'N0' + ) ELSE 'N/A' END, From dac8fc181b1a89adb78a7c7817d538e5f7fc046a Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 18 Mar 2025 19:25:36 -0400 Subject: [PATCH 219/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 256 +++++++++++++++------------- 1 file changed, 140 insertions(+), 116 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index f4acc5cc..8ff91402 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -4845,7 +4845,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CASE WHEN irs.summary_level = 'SUMMARY' THEN FORMAT(ISNULL(irs.indexes_to_disable, 0), 'N0') /* Indexes that will be disabled based on analysis */ - ELSE FORMAT(ISNULL(irs.unused_indexes, 0), 'N0') /* Unused indexes at database/table level */ + ELSE FORMAT(ISNULL(irs.unused_indexes, 0) + ISNULL(irs.indexes_to_merge, 0), 'N0') /* All removable indexes (unused + mergeable) */ END, /* Show mergeable indexes across all levels */ @@ -4859,7 +4859,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. THEN FORMAT(100.0 * ISNULL(irs.indexes_to_disable, 0) / NULLIF(irs.index_count, 0), 'N1') + '%' WHEN irs.index_count > 0 - THEN FORMAT(100.0 * ISNULL(irs.unused_indexes, 0) + THEN FORMAT(100.0 * (ISNULL(irs.unused_indexes, 0) + ISNULL(irs.indexes_to_merge, 0)) / NULLIF(irs.index_count, 0), 'N1') + '%' ELSE '0.0%' END, @@ -4878,7 +4878,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CASE WHEN irs.summary_level = 'SUMMARY' THEN FORMAT(ISNULL(irs.space_saved_gb, 0), 'N2') - ELSE FORMAT(ISNULL(irs.unused_size_gb, 0), 'N2') + /* Include size saved from both unused and mergeable indexes */ + ELSE FORMAT(ISNULL(irs.unused_size_gb, 0) + ISNULL(irs.merged_size_gb, 0), 'N2') END, /* Space reduction percentage - added this as new metric */ @@ -4921,45 +4922,52 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. daily_write_ops_saved = CASE WHEN irs.summary_level <> 'SUMMARY' - THEN FORMAT - ( - ISNULL - ( - irs.user_updates / - NULLIF + THEN + /* For rows with any removable indexes (unused or mergeable), calculate estimated savings */ + CASE + WHEN (ISNULL(irs.unused_indexes, 0) + ISNULL(irs.indexes_to_merge, 0)) > 0 + THEN FORMAT ( - CONVERT + ISNULL ( - decimal(10,2), + irs.user_updates / + NULLIF + ( + CONVERT + ( + decimal(10,2), + ( + SELECT TOP (1) + irs2.server_uptime_days + FROM #index_reporting_stats AS irs2 + WHERE irs2.summary_level = 'DATABASE' + ) + ), + 0 + ) * ( - SELECT TOP (1) - irs2.server_uptime_days - FROM #index_reporting_stats AS irs2 - WHERE irs2.summary_level = 'DATABASE' - ) + ISNULL + ( + irs.unused_indexes + irs.indexes_to_merge, + 0 + ) / + NULLIF + ( + CONVERT + ( + decimal(10,2), + irs.index_count + ), + 0 + ) + ), + 0 ), - 0 - ) * - ( - ISNULL - ( - irs.unused_indexes, - 0 - ) / - NULLIF - ( - CONVERT - ( - decimal(10,2), - irs.index_count - ), - 0 - ) - ), - 0 - ), - 'N0' - ) + 'N0' + ) + /* Rows without removable indexes have no savings */ + ELSE '0' + END ELSE 'N/A' END, @@ -4977,45 +4985,53 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. daily_lock_waits_saved = CASE WHEN irs.summary_level <> 'SUMMARY' - THEN FORMAT - ( - ISNULL - ( - (irs.row_lock_wait_count + irs.page_lock_wait_count) / - NULLIF - ( - CONVERT - ( - decimal(10,2), - ( - SELECT TOP (1) - irs2.server_uptime_days - FROM #index_reporting_stats AS irs2 - WHERE irs2.summary_level = 'DATABASE' - ) - ), - 0 - ) * - ( - ISNULL - ( - irs.unused_indexes, - 0 - ) / - NULLIF - ( - CONVERT - ( - decimal(10,2), - irs.index_count - ), - 0 - ) - ), - 0 - ), - 'N0' - ) + THEN + /* For rows with any removable indexes (unused or mergeable), calculate estimated savings */ + CASE + WHEN (ISNULL(irs.unused_indexes, 0) + ISNULL(irs.indexes_to_merge, 0)) > 0 + THEN + FORMAT + ( + ISNULL + ( + (irs.row_lock_wait_count + irs.page_lock_wait_count) / + NULLIF + ( + CONVERT + ( + decimal(10,2), + ( + SELECT TOP (1) + irs2.server_uptime_days + FROM #index_reporting_stats AS irs2 + WHERE irs2.summary_level = 'DATABASE' + ) + ), + 0 + ) * + ( + ISNULL + ( + irs.unused_indexes + irs.indexes_to_merge, + 0 + ) / + NULLIF + ( + CONVERT + ( + decimal(10,2), + irs.index_count + ), + 0 + ) + ), + 0 + ), + 'N0' + ) + /* Rows without removable indexes have no savings */ + ELSE '0' + END ELSE 'N/A' END, @@ -5045,45 +5061,53 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. daily_latch_waits_saved = CASE WHEN irs.summary_level <> 'SUMMARY' - THEN FORMAT - ( - ISNULL - ( - (irs.page_latch_wait_count + irs.page_io_latch_wait_count) / - NULLIF - ( - CONVERT - ( - decimal(10,2), - ( - SELECT TOP (1) - irs2.server_uptime_days - FROM #index_reporting_stats AS irs2 - WHERE irs2.summary_level = 'DATABASE' - ) - ), - 0 - ) * - ( - ISNULL - ( - irs.unused_indexes, - 0 - ) / - NULLIF - ( - CONVERT - ( - decimal(10,2), - irs.index_count - ), - 0 - ) - ), - 0 - ), - 'N0' - ) + THEN + /* For rows with any removable indexes (unused or mergeable), calculate estimated savings */ + CASE + WHEN (ISNULL(irs.unused_indexes, 0) + ISNULL(irs.indexes_to_merge, 0)) > 0 + THEN + FORMAT + ( + ISNULL + ( + (irs.page_latch_wait_count + irs.page_io_latch_wait_count) / + NULLIF + ( + CONVERT + ( + decimal(10,2), + ( + SELECT TOP (1) + irs2.server_uptime_days + FROM #index_reporting_stats AS irs2 + WHERE irs2.summary_level = 'DATABASE' + ) + ), + 0 + ) * + ( + ISNULL + ( + irs.unused_indexes + irs.indexes_to_merge, + 0 + ) / + NULLIF + ( + CONVERT + ( + decimal(10,2), + irs.index_count + ), + 0 + ) + ), + 0 + ), + 'N0' + ) + /* Rows without removable indexes have no savings */ + ELSE '0' + END ELSE 'N/A' END, From 9eea8708111addd1ece2d21b39f8c0f40c9c87d8 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 18 Mar 2025 19:28:20 -0400 Subject: [PATCH 220/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 8ff91402..b2ccde05 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -4829,9 +4829,16 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. WHEN irs.summary_level = 'SUMMARY' THEN FORMAT(irs.tables_analyzed, 'N0') WHEN irs.summary_level = 'DATABASE' - THEN FORMAT((SELECT COUNT_BIG(DISTINCT CONCAT(ia.schema_id, N'.', ia.object_id)) - FROM #index_analysis AS ia - WHERE ia.database_name = irs.database_name), 'N0') + THEN FORMAT + ( + ( + SELECT + COUNT_BIG(DISTINCT CONCAT(ia.schema_id, N'.', ia.object_id)) + FROM #index_analysis AS ia + WHERE ia.database_name = irs.database_name + ), + 'N0' + ) WHEN irs.summary_level = 'TABLE' THEN FORMAT(1, 'N0') /* Each table row represents 1 analyzed table */ ELSE FORMAT(0, 'N0') /* Show 0 instead of NULL */ From 3cfd09ed66d6de8fb511c3f07ff1ab20b1cbb93f Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 18 Mar 2025 20:42:45 -0400 Subject: [PATCH 221/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 31 ++++++++++++++--------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index b2ccde05..52b90894 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -4852,7 +4852,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CASE WHEN irs.summary_level = 'SUMMARY' THEN FORMAT(ISNULL(irs.indexes_to_disable, 0), 'N0') /* Indexes that will be disabled based on analysis */ - ELSE FORMAT(ISNULL(irs.unused_indexes, 0) + ISNULL(irs.indexes_to_merge, 0), 'N0') /* All removable indexes (unused + mergeable) */ + ELSE FORMAT(ISNULL(irs.unused_indexes, 0), 'N0') /* Unused indexes at database/table level */ END, /* Show mergeable indexes across all levels */ @@ -4866,7 +4866,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. THEN FORMAT(100.0 * ISNULL(irs.indexes_to_disable, 0) / NULLIF(irs.index_count, 0), 'N1') + '%' WHEN irs.index_count > 0 - THEN FORMAT(100.0 * (ISNULL(irs.unused_indexes, 0) + ISNULL(irs.indexes_to_merge, 0)) + THEN FORMAT(100.0 * ISNULL(irs.unused_indexes, 0) / NULLIF(irs.index_count, 0), 'N1') + '%' ELSE '0.0%' END, @@ -4885,8 +4885,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CASE WHEN irs.summary_level = 'SUMMARY' THEN FORMAT(ISNULL(irs.space_saved_gb, 0), 'N2') - /* Include size saved from both unused and mergeable indexes */ - ELSE FORMAT(ISNULL(irs.unused_size_gb, 0) + ISNULL(irs.merged_size_gb, 0), 'N2') + ELSE FORMAT(ISNULL(irs.unused_size_gb, 0), 'N2') END, /* Space reduction percentage - added this as new metric */ @@ -4930,9 +4929,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CASE WHEN irs.summary_level <> 'SUMMARY' THEN - /* For rows with any removable indexes (unused or mergeable), calculate estimated savings */ + /* For rows with unused indexes, calculate estimated savings */ CASE - WHEN (ISNULL(irs.unused_indexes, 0) + ISNULL(irs.indexes_to_merge, 0)) > 0 + WHEN ISNULL(irs.unused_indexes, 0) > 0 THEN FORMAT ( ISNULL @@ -4955,7 +4954,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ( ISNULL ( - irs.unused_indexes + irs.indexes_to_merge, + irs.unused_indexes, 0 ) / NULLIF @@ -4972,7 +4971,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ), 'N0' ) - /* Rows without removable indexes have no savings */ + /* Rows without unused indexes have no savings */ ELSE '0' END ELSE 'N/A' @@ -4993,9 +4992,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CASE WHEN irs.summary_level <> 'SUMMARY' THEN - /* For rows with any removable indexes (unused or mergeable), calculate estimated savings */ + /* For rows with unused indexes, calculate estimated savings */ CASE - WHEN (ISNULL(irs.unused_indexes, 0) + ISNULL(irs.indexes_to_merge, 0)) > 0 + WHEN ISNULL(irs.unused_indexes, 0) > 0 THEN FORMAT ( @@ -5019,7 +5018,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ( ISNULL ( - irs.unused_indexes + irs.indexes_to_merge, + irs.unused_indexes, 0 ) / NULLIF @@ -5036,7 +5035,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ), 'N0' ) - /* Rows without removable indexes have no savings */ + /* Rows without unused indexes have no savings */ ELSE '0' END ELSE 'N/A' @@ -5069,9 +5068,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CASE WHEN irs.summary_level <> 'SUMMARY' THEN - /* For rows with any removable indexes (unused or mergeable), calculate estimated savings */ + /* For rows with unused indexes, calculate estimated savings */ CASE - WHEN (ISNULL(irs.unused_indexes, 0) + ISNULL(irs.indexes_to_merge, 0)) > 0 + WHEN ISNULL(irs.unused_indexes, 0) > 0 THEN FORMAT ( @@ -5095,7 +5094,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ( ISNULL ( - irs.unused_indexes + irs.indexes_to_merge, + irs.unused_indexes, 0 ) / NULLIF @@ -5112,7 +5111,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ), 'N0' ) - /* Rows without removable indexes have no savings */ + /* Rows without unused indexes have no savings */ ELSE '0' END ELSE 'N/A' From 4774b7d1fc5584694a0e9eea6e90888a0dee5cd7 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 18 Mar 2025 21:48:03 -0400 Subject: [PATCH 222/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 176 ++++++++++++++-------------- 1 file changed, 91 insertions(+), 85 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 52b90894..9c607321 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -4934,40 +4934,42 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. WHEN ISNULL(irs.unused_indexes, 0) > 0 THEN FORMAT ( - ISNULL - ( - irs.user_updates / - NULLIF + CONVERT(decimal(38,2), + ISNULL ( - CONVERT + irs.user_updates / + NULLIF ( - decimal(10,2), + CONVERT ( - SELECT TOP (1) - irs2.server_uptime_days - FROM #index_reporting_stats AS irs2 - WHERE irs2.summary_level = 'DATABASE' - ) + decimal(38,2), + ( + SELECT TOP (1) + irs2.server_uptime_days + FROM #index_reporting_stats AS irs2 + WHERE irs2.summary_level = 'DATABASE' + ) + ), + 0 + ) * + ( + ISNULL + ( + irs.unused_indexes, + 0 + ) / + NULLIF + ( + CONVERT + ( + decimal(38,2), + irs.index_count + ), + 0 + ) ), 0 - ) * - ( - ISNULL - ( - irs.unused_indexes, - 0 - ) / - NULLIF - ( - CONVERT - ( - decimal(10,2), - irs.index_count - ), - 0 - ) - ), - 0 + ) ), 'N0' ) @@ -4998,40 +5000,42 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. THEN FORMAT ( - ISNULL - ( - (irs.row_lock_wait_count + irs.page_lock_wait_count) / - NULLIF + CONVERT(decimal(38,2), + ISNULL ( - CONVERT + (irs.row_lock_wait_count + irs.page_lock_wait_count) / + NULLIF ( - decimal(10,2), + CONVERT ( - SELECT TOP (1) - irs2.server_uptime_days - FROM #index_reporting_stats AS irs2 - WHERE irs2.summary_level = 'DATABASE' - ) + decimal(38,2), + ( + SELECT TOP (1) + irs2.server_uptime_days + FROM #index_reporting_stats AS irs2 + WHERE irs2.summary_level = 'DATABASE' + ) + ), + 0 + ) * + ( + ISNULL + ( + irs.unused_indexes, + 0 + ) / + NULLIF + ( + CONVERT + ( + decimal(38,2), + irs.index_count + ), + 0 + ) ), 0 - ) * - ( - ISNULL - ( - irs.unused_indexes, - 0 - ) / - NULLIF - ( - CONVERT - ( - decimal(10,2), - irs.index_count - ), - 0 - ) - ), - 0 + ) ), 'N0' ) @@ -5074,40 +5078,42 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. THEN FORMAT ( - ISNULL - ( - (irs.page_latch_wait_count + irs.page_io_latch_wait_count) / - NULLIF - ( - CONVERT - ( - decimal(10,2), - ( - SELECT TOP (1) - irs2.server_uptime_days - FROM #index_reporting_stats AS irs2 - WHERE irs2.summary_level = 'DATABASE' - ) - ), - 0 - ) * + CONVERT(decimal(38,2), + ISNULL ( - ISNULL - ( - irs.unused_indexes, - 0 - ) / + (irs.page_latch_wait_count + irs.page_io_latch_wait_count) / NULLIF ( CONVERT ( - decimal(10,2), - irs.index_count + decimal(38,2), + ( + SELECT TOP (1) + irs2.server_uptime_days + FROM #index_reporting_stats AS irs2 + WHERE irs2.summary_level = 'DATABASE' + ) ), 0 - ) - ), - 0 + ) * + ( + ISNULL + ( + irs.unused_indexes, + 0 + ) / + NULLIF + ( + CONVERT + ( + decimal(38,2), + irs.index_count + ), + 0 + ) + ), + 0 + ) ), 'N0' ) From f4b877f71bc27a8c5f0f6484a1233d2f8f243d21 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 18 Mar 2025 22:02:27 -0400 Subject: [PATCH 223/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 9c607321..44ebccc5 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -582,7 +582,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CREATE TABLE #index_reporting_stats ( - summary_level varchar(20) NOT NULL, /* 'DATABASE', 'TABLE', 'INDEX', 'SUMMARY' */ + summary_level varchar(20) NOT NULL, /* 'SUMMARY', 'TABLE', 'INDEX', 'SUMMARY' */ database_name sysname NULL, schema_name sysname NULL, table_name sysname NULL, @@ -4410,7 +4410,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ) SELECT summary_level = - 'DATABASE', + 'SUMMARY', ps.database_name, index_count = COUNT_BIG(DISTINCT CONCAT(ps.object_id, N'.', ps.index_id)), @@ -4828,7 +4828,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CASE WHEN irs.summary_level = 'SUMMARY' THEN FORMAT(irs.tables_analyzed, 'N0') - WHEN irs.summary_level = 'DATABASE' + WHEN irs.summary_level = 'SUMMARY' THEN FORMAT ( ( @@ -4877,8 +4877,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Size after cleanup - added this as new metric */ size_after_cleanup_gb = - FORMAT(ISNULL(irs.total_size_gb, 0) - - ISNULL(irs.space_saved_gb, 0), 'N2'), + CASE + WHEN irs.summary_level = 'SUMMARY' + THEN FORMAT(ISNULL(irs.total_size_gb, 0) - ISNULL(irs.space_saved_gb, 0), 'N2') + ELSE FORMAT(ISNULL(irs.total_size_gb, 0) - ISNULL(irs.unused_size_gb, 0), 'N2') + END, /* Size that can be saved through cleanup */ space_saved_gb = @@ -4947,7 +4950,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SELECT TOP (1) irs2.server_uptime_days FROM #index_reporting_stats AS irs2 - WHERE irs2.summary_level = 'DATABASE' + WHERE irs2.summary_level = 'SUMMARY' ) ), 0 @@ -5013,7 +5016,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SELECT TOP (1) irs2.server_uptime_days FROM #index_reporting_stats AS irs2 - WHERE irs2.summary_level = 'DATABASE' + WHERE irs2.summary_level = 'SUMMARY' ) ), 0 @@ -5091,7 +5094,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SELECT TOP (1) irs2.server_uptime_days FROM #index_reporting_stats AS irs2 - WHERE irs2.summary_level = 'DATABASE' + WHERE irs2.summary_level = 'SUMMARY' ) ), 0 @@ -5136,14 +5139,14 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ELSE '0.00' END FROM #index_reporting_stats AS irs - WHERE irs.summary_level IN ('SUMMARY', 'DATABASE', 'TABLE') /* Filter out INDEX level */ + WHERE irs.summary_level IN ('SUMMARY', 'SUMMARY', 'TABLE') /* Filter out INDEX level */ ORDER BY /* Order by database name */ irs.database_name, /* Then order by level - summary first */ CASE WHEN irs.summary_level = 'SUMMARY' THEN 0 - WHEN irs.summary_level = 'DATABASE' THEN 1 + WHEN irs.summary_level = 'SUMMARY' THEN 1 WHEN irs.summary_level = 'TABLE' THEN 2 ELSE 3 END, From e00ffcb2af755205fb703c1364ed68393fc5f45f Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 18 Mar 2025 22:02:45 -0400 Subject: [PATCH 224/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 9c607321..038e0a7c 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -4947,7 +4947,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SELECT TOP (1) irs2.server_uptime_days FROM #index_reporting_stats AS irs2 - WHERE irs2.summary_level = 'DATABASE' + WHERE irs2.summary_level = 'SUMMARY' ) ), 0 @@ -5013,7 +5013,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SELECT TOP (1) irs2.server_uptime_days FROM #index_reporting_stats AS irs2 - WHERE irs2.summary_level = 'DATABASE' + WHERE irs2.summary_level = 'SUMMARY' ) ), 0 @@ -5091,7 +5091,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. SELECT TOP (1) irs2.server_uptime_days FROM #index_reporting_stats AS irs2 - WHERE irs2.summary_level = 'DATABASE' + WHERE irs2.summary_level = 'SUMMARY' ) ), 0 From b2e0c5adc93329edd7d497944fb45e3491bb9a44 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 18 Mar 2025 22:04:45 -0400 Subject: [PATCH 225/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 44ebccc5..c3b3d46b 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -4799,7 +4799,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. level = CASE WHEN irs.summary_level = 'SUMMARY' - THEN '=== OVERALL ANALYSIS ===' + THEN 'OVERALL ANALYSIS' ELSE irs.summary_level END, From 44cfa2d6443ee964f3796b76dd8bd113244f384f Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 18 Mar 2025 22:31:10 -0400 Subject: [PATCH 226/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index c3b3d46b..b29f4a74 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -582,7 +582,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CREATE TABLE #index_reporting_stats ( - summary_level varchar(20) NOT NULL, /* 'SUMMARY', 'TABLE', 'INDEX', 'SUMMARY' */ + summary_level varchar(20) NOT NULL, /* 'DATABASE', 'TABLE', 'INDEX', 'SUMMARY' */ database_name sysname NULL, schema_name sysname NULL, table_name sysname NULL, @@ -4224,6 +4224,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. uptime_warning, tables_analyzed, index_count, + total_size_gb, indexes_to_disable, indexes_to_merge, avg_indexes_per_table, @@ -4240,8 +4241,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. uptime_warning = @uptime_warning, tables_analyzed = COUNT_BIG(DISTINCT CONCAT(ia.database_id, N'.', ia.schema_id, N'.', ia.object_id)), - index_count = - COUNT_BIG(*), + index_count = COUNT_BIG(*), + total_size_gb = SUM(ps.total_space_gb), indexes_to_disable = SUM ( @@ -4409,12 +4410,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. leaf_delete_count ) SELECT - summary_level = - 'SUMMARY', + summary_level = 'DATABASE', ps.database_name, index_count = COUNT_BIG(DISTINCT CONCAT(ps.object_id, N'.', ps.index_id)), - total_size_gb = SUM(ps.total_space_gb), + total_size_gb = SUM(DISTINCT ps.total_space_gb), /* Use a simple aggregation to avoid double-counting */ /* Get actual row count by grabbing the real row count from clustered index/heap per table */ total_rows = SUM(DISTINCT d.actual_rows), @@ -4552,7 +4552,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ps.schema_name, ps.table_name, index_count = COUNT_BIG(DISTINCT ps.index_id), - total_size_gb = SUM(ps.total_space_gb), + total_size_gb = SUM(DISTINCT ps.total_space_gb), /* Use MAX to get the row count from the clustered index or heap */ total_rows = MAX @@ -4799,7 +4799,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. level = CASE WHEN irs.summary_level = 'SUMMARY' - THEN 'OVERALL ANALYSIS' + THEN 'ANALYZED OBJECT DETAILS' ELSE irs.summary_level END, @@ -4828,7 +4828,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CASE WHEN irs.summary_level = 'SUMMARY' THEN FORMAT(irs.tables_analyzed, 'N0') - WHEN irs.summary_level = 'SUMMARY' + WHEN irs.summary_level = 'DATABASE' THEN FORMAT ( ( @@ -5139,14 +5139,14 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ELSE '0.00' END FROM #index_reporting_stats AS irs - WHERE irs.summary_level IN ('SUMMARY', 'SUMMARY', 'TABLE') /* Filter out INDEX level */ + WHERE irs.summary_level IN ('SUMMARY', 'DATABASE', 'TABLE') /* Filter out INDEX level */ ORDER BY /* Order by database name */ irs.database_name, /* Then order by level - summary first */ CASE WHEN irs.summary_level = 'SUMMARY' THEN 0 - WHEN irs.summary_level = 'SUMMARY' THEN 1 + WHEN irs.summary_level = 'DATABASE' THEN 1 WHEN irs.summary_level = 'TABLE' THEN 2 ELSE 3 END, From 9c1e1f468034f0c66dd2f93c1db635c02ea43c74 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 18 Mar 2025 22:43:36 -0400 Subject: [PATCH 227/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index b29f4a74..a1ccb917 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -4895,8 +4895,14 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. space_reduction_percent = CASE WHEN ISNULL(irs.total_size_gb, 0) > 0 - THEN FORMAT((ISNULL(irs.space_saved_gb, 0) / - NULLIF(irs.total_size_gb, 0)) * 100, 'N1') + '%' + THEN + CASE + WHEN irs.summary_level = 'SUMMARY' + THEN FORMAT((ISNULL(irs.space_saved_gb, 0) / + NULLIF(irs.total_size_gb, 0)) * 100, 'N1') + '%' + ELSE FORMAT((ISNULL(irs.unused_size_gb, 0) / + NULLIF(irs.total_size_gb, 0)) * 100, 'N1') + '%' + END ELSE '0.0%' END, From 5f4cea0f92ef174d4a38cbe64ac915795dedc6b1 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 18 Mar 2025 22:52:34 -0400 Subject: [PATCH 228/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index a1ccb917..5f7c37df 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -4268,12 +4268,12 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. COUNT_BIG(DISTINCT CONCAT(ia.database_id, N'.', ia.schema_id, N'.', ia.object_id)), 0 ), - /* Space savings from cleanup */ + /* Space savings from cleanup - only count DISABLE actions */ space_saved_gb = SUM ( CASE - WHEN ia.action IN (N'DISABLE', N'MERGE INCLUDES', N'MAKE UNIQUE') + WHEN ia.action = N'DISABLE' THEN ps.total_space_gb ELSE 0 END @@ -4300,12 +4300,12 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ELSE 0 END ), - /* Total conservative savings */ + /* Total conservative savings - only count DISABLE actions for space savings */ total_min_savings_gb = SUM ( CASE - WHEN ia.action IN (N'DISABLE', N'MERGE INCLUDES', N'MAKE UNIQUE') + WHEN ia.action = N'DISABLE' THEN ps.total_space_gb WHEN (ia.action IS NULL OR ia.action = N'KEEP') AND ce.can_compress = 1 @@ -4313,12 +4313,12 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ELSE 0 END ), - /* Total optimistic savings */ + /* Total optimistic savings - only count DISABLE actions for space savings */ total_max_savings_gb = SUM ( CASE - WHEN ia.action IN (N'DISABLE', N'MERGE INCLUDES', N'MAKE UNIQUE') + WHEN ia.action = N'DISABLE' THEN ps.total_space_gb WHEN (ia.action IS NULL OR ia.action = N'KEEP') AND ce.can_compress = 1 From b4ca6818fc7cab8e099e76980e5f5ff2f110f2ce Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 18 Mar 2025 22:57:26 -0400 Subject: [PATCH 229/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 32 ++++++++++++++++++----------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 5f7c37df..ead69687 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -4436,13 +4436,16 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND ia.database_id = ps.database_id ), unused_size_gb = - SUM ( - CASE - WHEN id.user_seeks + id.user_scans + id.user_lookups = 0 - THEN ps.total_space_gb - ELSE 0 - END + SELECT + SUM(subps.total_space_gb) + FROM #partition_stats AS subps + JOIN #index_analysis AS subia + ON subps.database_id = subia.database_id + AND subps.object_id = subia.object_id + AND subps.index_id = subia.index_id + WHERE subia.action = N'DISABLE' + AND subia.database_id = ps.database_id ), total_reads = SUM(id.user_seeks + id.user_scans + id.user_lookups), total_writes = SUM(id.user_updates), @@ -4585,13 +4588,18 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND ia.object_id = ps.object_id ), unused_size_gb = - SUM ( - CASE - WHEN id.user_seeks + id.user_scans + id.user_lookups = 0 - THEN ps.total_space_gb - ELSE 0 - END + SELECT + SUM(subps.total_space_gb) + FROM #partition_stats AS subps + JOIN #index_analysis AS subia + ON subps.database_id = subia.database_id + AND subps.object_id = subia.object_id + AND subps.index_id = subia.index_id + WHERE subia.action = N'DISABLE' + AND subia.database_id = ps.database_id + AND subia.schema_id = ps.schema_id + AND subia.object_id = ps.object_id ), total_reads = SUM(id.user_seeks + id.user_scans + id.user_lookups), total_writes = SUM(id.user_updates), From 57e7987c1aeedf87c342c40f607d909d376da723 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 18 Mar 2025 23:09:54 -0400 Subject: [PATCH 230/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 201 ++++++++++++++++++++++++++++ 1 file changed, 201 insertions(+) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index ead69687..91258bd7 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -4386,6 +4386,10 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. indexes_to_merge, unused_indexes, unused_size_gb, + compression_min_savings_gb, + compression_max_savings_gb, + total_min_savings_gb, + total_max_savings_gb, total_reads, total_writes, user_seeks, @@ -4447,6 +4451,92 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. WHERE subia.action = N'DISABLE' AND subia.database_id = ps.database_id ), + /* Conservative compression savings estimate (20%) */ + compression_min_savings_gb = + ( + SELECT + SUM(subps.total_space_gb * 0.20) + FROM #partition_stats AS subps + JOIN #index_analysis AS subia + ON subps.database_id = subia.database_id + AND subps.object_id = subia.object_id + AND subps.index_id = subia.index_id + JOIN #compression_eligibility AS subce + ON subce.database_id = subia.database_id + AND subce.object_id = subia.object_id + AND subce.index_id = subia.index_id + WHERE (subia.action IS NULL OR subia.action = N'KEEP') + AND subce.can_compress = 1 + AND subia.database_id = ps.database_id + ), + /* Optimistic compression savings estimate (60%) */ + compression_max_savings_gb = + ( + SELECT + SUM(subps.total_space_gb * 0.60) + FROM #partition_stats AS subps + JOIN #index_analysis AS subia + ON subps.database_id = subia.database_id + AND subps.object_id = subia.object_id + AND subps.index_id = subia.index_id + JOIN #compression_eligibility AS subce + ON subce.database_id = subia.database_id + AND subce.object_id = subia.object_id + AND subce.index_id = subia.index_id + WHERE (subia.action IS NULL OR subia.action = N'KEEP') + AND subce.can_compress = 1 + AND subia.database_id = ps.database_id + ), + /* Total conservative savings */ + total_min_savings_gb = + ( + SELECT + SUM( + CASE + WHEN subia.action = N'DISABLE' + THEN subps.total_space_gb + WHEN (subia.action IS NULL OR subia.action = N'KEEP') + AND subce.can_compress = 1 + THEN subps.total_space_gb * 0.20 + ELSE 0 + END + ) + FROM #partition_stats AS subps + JOIN #index_analysis AS subia + ON subps.database_id = subia.database_id + AND subps.object_id = subia.object_id + AND subps.index_id = subia.index_id + LEFT JOIN #compression_eligibility AS subce + ON subce.database_id = subia.database_id + AND subce.object_id = subia.object_id + AND subce.index_id = subia.index_id + WHERE subia.database_id = ps.database_id + ), + /* Total optimistic savings */ + total_max_savings_gb = + ( + SELECT + SUM( + CASE + WHEN subia.action = N'DISABLE' + THEN subps.total_space_gb + WHEN (subia.action IS NULL OR subia.action = N'KEEP') + AND subce.can_compress = 1 + THEN subps.total_space_gb * 0.60 + ELSE 0 + END + ) + FROM #partition_stats AS subps + JOIN #index_analysis AS subia + ON subps.database_id = subia.database_id + AND subps.object_id = subia.object_id + AND subps.index_id = subia.index_id + LEFT JOIN #compression_eligibility AS subce + ON subce.database_id = subia.database_id + AND subce.object_id = subia.object_id + AND subce.index_id = subia.index_id + WHERE subia.database_id = ps.database_id + ), total_reads = SUM(id.user_seeks + id.user_scans + id.user_lookups), total_writes = SUM(id.user_updates), user_seeks = SUM(id.user_seeks), @@ -4526,6 +4616,10 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. indexes_to_merge, unused_indexes, unused_size_gb, + compression_min_savings_gb, + compression_max_savings_gb, + total_min_savings_gb, + total_max_savings_gb, total_reads, total_writes, user_seeks, @@ -4601,6 +4695,100 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AND subia.schema_id = ps.schema_id AND subia.object_id = ps.object_id ), + /* Conservative compression savings estimate (20%) */ + compression_min_savings_gb = + ( + SELECT + SUM(subps.total_space_gb * 0.20) + FROM #partition_stats AS subps + JOIN #index_analysis AS subia + ON subps.database_id = subia.database_id + AND subps.object_id = subia.object_id + AND subps.index_id = subia.index_id + JOIN #compression_eligibility AS subce + ON subce.database_id = subia.database_id + AND subce.object_id = subia.object_id + AND subce.index_id = subia.index_id + WHERE (subia.action IS NULL OR subia.action = N'KEEP') + AND subce.can_compress = 1 + AND subia.database_id = ps.database_id + AND subia.schema_id = ps.schema_id + AND subia.object_id = ps.object_id + ), + /* Optimistic compression savings estimate (60%) */ + compression_max_savings_gb = + ( + SELECT + SUM(subps.total_space_gb * 0.60) + FROM #partition_stats AS subps + JOIN #index_analysis AS subia + ON subps.database_id = subia.database_id + AND subps.object_id = subia.object_id + AND subps.index_id = subia.index_id + JOIN #compression_eligibility AS subce + ON subce.database_id = subia.database_id + AND subce.object_id = subia.object_id + AND subce.index_id = subia.index_id + WHERE (subia.action IS NULL OR subia.action = N'KEEP') + AND subce.can_compress = 1 + AND subia.database_id = ps.database_id + AND subia.schema_id = ps.schema_id + AND subia.object_id = ps.object_id + ), + /* Total conservative savings */ + total_min_savings_gb = + ( + SELECT + SUM( + CASE + WHEN subia.action = N'DISABLE' + THEN subps.total_space_gb + WHEN (subia.action IS NULL OR subia.action = N'KEEP') + AND subce.can_compress = 1 + THEN subps.total_space_gb * 0.20 + ELSE 0 + END + ) + FROM #partition_stats AS subps + JOIN #index_analysis AS subia + ON subps.database_id = subia.database_id + AND subps.object_id = subia.object_id + AND subps.index_id = subia.index_id + LEFT JOIN #compression_eligibility AS subce + ON subce.database_id = subia.database_id + AND subce.object_id = subia.object_id + AND subce.index_id = subia.index_id + WHERE subia.database_id = ps.database_id + AND subia.schema_id = ps.schema_id + AND subia.object_id = ps.object_id + ), + /* Total optimistic savings */ + total_max_savings_gb = + ( + SELECT + SUM( + CASE + WHEN subia.action = N'DISABLE' + THEN subps.total_space_gb + WHEN (subia.action IS NULL OR subia.action = N'KEEP') + AND subce.can_compress = 1 + THEN subps.total_space_gb * 0.60 + ELSE 0 + END + ) + FROM #partition_stats AS subps + JOIN #index_analysis AS subia + ON subps.database_id = subia.database_id + AND subps.object_id = subia.object_id + AND subps.index_id = subia.index_id + LEFT JOIN #compression_eligibility AS subce + ON subce.database_id = subia.database_id + AND subce.object_id = subia.object_id + AND subce.index_id = subia.index_id + WHERE subia.database_id = ps.database_id + AND subia.schema_id = ps.schema_id + AND subia.object_id = ps.object_id + ), total_reads = SUM(id.user_seeks + id.user_scans + id.user_lookups), total_writes = SUM(id.user_updates), user_seeks = SUM(id.user_seeks), @@ -4913,6 +5101,19 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END ELSE '0.0%' END, + + /* ===== Additional Space Savings from Compression ===== */ + /* Conservative compression estimate (20%) */ + compression_min_savings_gb = FORMAT(ISNULL(irs.compression_min_savings_gb, 0), 'N2'), + + /* Optimistic compression estimate (60%) */ + compression_max_savings_gb = FORMAT(ISNULL(irs.compression_max_savings_gb, 0), 'N2'), + + /* Total savings (removal + conservative compression) */ + total_min_savings_gb = FORMAT(ISNULL(irs.total_min_savings_gb, 0), 'N2'), + + /* Total savings (removal + optimistic compression) */ + total_max_savings_gb = FORMAT(ISNULL(irs.total_max_savings_gb, 0), 'N2'), /* ===== Section 3: Table and Usage Statistics ===== */ /* Row count */ From 19ed9901828e2b1a1783a3a16eff8bf05f969b22 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 18 Mar 2025 23:55:53 -0400 Subject: [PATCH 231/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 162 +++++++++++++++++++++++++++- 1 file changed, 161 insertions(+), 1 deletion(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 91258bd7..16dc6a4b 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -1997,7 +1997,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ELSE N'CREATE ' + CASE WHEN id1.is_unique = 1 THEN N'UNIQUE ' ELSE N'' END + - CASE WHEN id1.index_id = 0 THEN N'CLUSTERED ' WHEN id1.index_id > 0 AND @verbose_output >= 1 THEN N'NONCLUSTERED ' ELSE N'' END + + CASE WHEN id1.index_id = 1 THEN N'CLUSTERED ' WHEN id1.index_id > 1 AND @verbose_output >= 1 THEN N'NONCLUSTERED ' ELSE N'' END + N'INDEX ' + QUOTENAME(id1.index_name) + N' ON ' + @@ -3623,6 +3623,166 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ) OPTION(RECOMPILE); + /* Add clustered indexes to #index_analysis specifically for compression purposes */ + IF @debug = 1 + BEGIN + RAISERROR('Adding clustered indexes to #index_analysis for compression', 0, 0) WITH NOWAIT; + END; + + INSERT INTO + #index_analysis + WITH + (TABLOCK) + ( + database_id, + database_name, + schema_id, + schema_name, + table_name, + object_id, + index_id, + index_name, + is_unique, + key_columns, + included_columns, + filter_definition, + original_index_definition + ) + SELECT + fo.database_id, + fo.database_name, + fo.schema_id, + fo.schema_name, + fo.table_name, + fo.object_id, + fo.index_id, + fo.index_name, + is_unique = + CASE + WHEN ce.can_compress = 1 + THEN id.is_unique + ELSE NULL + END, + key_columns = + STUFF + ( + ( + SELECT + N', ' + + QUOTENAME(id2.column_name) + + CASE + WHEN id2.is_descending_key = 1 + THEN N' DESC' + ELSE N'' + END + FROM #index_details id2 + WHERE id2.object_id = fo.object_id + AND id2.index_id = fo.index_id + AND id2.is_included_column = 0 + GROUP BY + id2.column_name, + id2.is_descending_key, + id2.key_ordinal + ORDER BY + id2.key_ordinal + FOR + XML + PATH(''), + TYPE + ).value('text()[1]','nvarchar(max)'), + 1, + 2, + '' + ), + included_columns = NULL, /* Clustered indexes cannot have included columns */ + filter_definition = NULL, /* Clustered indexes cannot have filters */ + original_index_definition = + N'CREATE CLUSTERED INDEX ' + + QUOTENAME(fo.index_name) + + N' ON ' + + QUOTENAME(fo.database_name) + + N'.' + + QUOTENAME(fo.schema_name) + + N'.' + + QUOTENAME(fo.table_name) + + N' (' + + STUFF + ( + ( + SELECT + N', ' + + QUOTENAME(id2.column_name) + + CASE + WHEN id2.is_descending_key = 1 + THEN N' DESC' + ELSE N'' + END + FROM #index_details id2 + WHERE id2.object_id = fo.object_id + AND id2.index_id = fo.index_id + AND id2.is_included_column = 0 + GROUP BY + id2.column_name, + id2.is_descending_key, + id2.key_ordinal + ORDER BY + id2.key_ordinal + FOR + XML + PATH(''), + TYPE + ).value('text()[1]','nvarchar(max)'), + 1, + 2, + '' + ) + + N');' + FROM #filtered_objects AS fo + JOIN #index_details AS id + ON id.database_id = fo.database_id + AND id.object_id = fo.object_id + AND id.index_id = fo.index_id + AND id.key_ordinal = 1 /* Only need one row per index */ + JOIN #compression_eligibility AS ce + ON ce.database_id = fo.database_id + AND ce.object_id = fo.object_id + AND ce.index_id = fo.index_id + WHERE fo.index_id = 1 /* Clustered indexes only */ + AND ce.can_compress = 1 /* Only those eligible for compression */ + /* Only add if not already in #index_analysis */ + AND NOT EXISTS + ( + SELECT + 1/0 + FROM #index_analysis AS ia + WHERE ia.database_id = fo.database_id + AND ia.object_id = fo.object_id + AND ia.index_id = fo.index_id + ) + OPTION(RECOMPILE); + + /* If any clustered indexes were added, mark them as KEEP */ + UPDATE #index_analysis + SET action = N'KEEP' + WHERE index_id = 1 /* Clustered indexes */ + AND action IS NULL; + + /* Update index priority for clustered indexes to ensure they're not chosen for deduplication */ + UPDATE #index_analysis + SET index_priority = 1000 /* Maximum priority */ + WHERE index_id = 1 /* Clustered indexes */ + AND index_priority IS NULL; + + IF @debug = 1 + BEGIN + SELECT + table_name = '#index_analysis after adding clustered indexes', + * + FROM #index_analysis AS ia + WHERE ia.index_id = 1 + OPTION(RECOMPILE); + END; + /* Insert compression scripts for remaining indexes */ IF @debug = 1 BEGIN From 80a1e51452f184cd3484bed6e6222a0761738ed9 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 18 Mar 2025 23:56:04 -0400 Subject: [PATCH 232/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 91258bd7..5f008298 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -1545,7 +1545,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CASE WHEN i.type = 2 THEN 1 - WHEN i.type = 1 + WHEN + ( + i.type = 1 + OR i.is_primary_key = 1 + ) THEN 0 END FROM ' + QUOTENAME(@current_database_name) + N'.sys.tables AS t From d50341d44f3ff5ec565a55a171c476ded6f408e4 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Wed, 19 Mar 2025 00:12:34 -0400 Subject: [PATCH 233/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 156 ++++++++++++++++++++-------- 1 file changed, 115 insertions(+), 41 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 00d4767c..5afde043 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -2000,8 +2000,19 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* For regular indexes, use CREATE INDEX syntax */ ELSE N'CREATE ' + - CASE WHEN id1.is_unique = 1 THEN N'UNIQUE ' ELSE N'' END + - CASE WHEN id1.index_id = 1 THEN N'CLUSTERED ' WHEN id1.index_id > 1 AND @verbose_output >= 1 THEN N'NONCLUSTERED ' ELSE N'' END + + CASE + WHEN id1.is_unique = 1 + THEN N'UNIQUE ' + ELSE N'' + END + + CASE + WHEN id1.index_id = 1 + THEN N'CLUSTERED ' + WHEN id1.index_id > 1 + AND @verbose_output >= 1 + THEN N'NONCLUSTERED ' + ELSE N'' + END + N'INDEX ' + QUOTENAME(id1.index_name) + N' ON ' + @@ -3701,46 +3712,105 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. included_columns = NULL, /* Clustered indexes cannot have included columns */ filter_definition = NULL, /* Clustered indexes cannot have filters */ original_index_definition = - N'CREATE CLUSTERED INDEX ' + - QUOTENAME(fo.index_name) + - N' ON ' + - QUOTENAME(fo.database_name) + - N'.' + - QUOTENAME(fo.schema_name) + - N'.' + - QUOTENAME(fo.table_name) + - N' (' + - STUFF - ( - ( - SELECT - N', ' + - QUOTENAME(id2.column_name) + + CASE + WHEN id.is_primary_key = 1 + THEN + N'ALTER TABLE ' + + QUOTENAME(fo.database_name) + + N'.' + + QUOTENAME(fo.schema_name) + + N'.' + + QUOTENAME(fo.table_name) + + N' ADD CONSTRAINT ' + + QUOTENAME(fo.index_name) + + N' PRIMARY KEY ' + CASE - WHEN id2.is_descending_key = 1 - THEN N' DESC' - ELSE N'' + WHEN ce.index_id = 1 + THEN N'CLUSTERED' + ELSE N'NONCLUSTERED' END - FROM #index_details id2 - WHERE id2.object_id = fo.object_id - AND id2.index_id = fo.index_id - AND id2.is_included_column = 0 - GROUP BY - id2.column_name, - id2.is_descending_key, - id2.key_ordinal - ORDER BY - id2.key_ordinal - FOR - XML - PATH(''), - TYPE - ).value('text()[1]','nvarchar(max)'), - 1, - 2, - '' - ) + - N');' + + + N' (' + + STUFF + ( + ( + SELECT + N', ' + + QUOTENAME(id2.column_name) + + CASE + WHEN id2.is_descending_key = 1 + THEN N' DESC' + ELSE N'' + END + FROM #index_details id2 + WHERE id2.object_id = fo.object_id + AND id2.index_id = fo.index_id + AND id2.is_included_column = 0 + GROUP BY + id2.column_name, + id2.is_descending_key, + id2.key_ordinal + ORDER BY + id2.key_ordinal + FOR + XML + PATH(''), + TYPE + ).value('text()[1]','nvarchar(max)'), + 1, + 2, + '' + ) + + N');' + WHEN id.is_primary_key = 0 + THEN N'CREATE ' + + CASE + WHEN id.is_unique = 1 + THEN N'UNIQUE ' + ELSE N'' + END + + N'CLUSTERED INDEX' + + QUOTENAME(fo.index_name) + + N' ON ' + + QUOTENAME(fo.database_name) + + N'.' + + QUOTENAME(fo.schema_name) + + N'.' + + QUOTENAME(fo.table_name) + + N' (' + + STUFF + ( + ( + SELECT + N', ' + + QUOTENAME(id2.column_name) + + CASE + WHEN id2.is_descending_key = 1 + THEN N' DESC' + ELSE N'' + END + FROM #index_details id2 + WHERE id2.object_id = fo.object_id + AND id2.index_id = fo.index_id + AND id2.is_included_column = 0 + GROUP BY + id2.column_name, + id2.is_descending_key, + id2.key_ordinal + ORDER BY + id2.key_ordinal + FOR + XML + PATH(''), + TYPE + ).value('text()[1]','nvarchar(max)'), + 1, + 2, + '' + ) + + N');' + ELSE N'' + END FROM #filtered_objects AS fo JOIN #index_details AS id ON id.database_id = fo.database_id @@ -3751,7 +3821,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ON ce.database_id = fo.database_id AND ce.object_id = fo.object_id AND ce.index_id = fo.index_id - WHERE fo.index_id = 1 /* Clustered indexes only */ + WHERE + ( + fo.index_id = 1 /* Clustered indexes only */ + OR id.is_primary_key = 1 + ) AND ce.can_compress = 1 /* Only those eligible for compression */ /* Only add if not already in #index_analysis */ AND NOT EXISTS From c3a276ab864fbd70fc92423a650478f335c795a2 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Wed, 19 Mar 2025 00:13:05 -0400 Subject: [PATCH 234/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 170 +++++++++++++++++++++++++++- 1 file changed, 166 insertions(+), 4 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 16dc6a4b..6a320d52 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -5305,6 +5305,60 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Write operations saved - added as new metric */ daily_write_ops_saved = CASE + WHEN irs.summary_level = 'SUMMARY' + THEN + /* For SUMMARY row, calculate the sum of daily write ops saved across all databases */ + FORMAT + ( + ( + SELECT + SUM + ( + CONVERT + ( + decimal(38,2), + CASE + WHEN ISNULL(irs3.unused_indexes, 0) > 0 + THEN + ISNULL + ( + irs3.user_updates / + NULLIF + ( + CONVERT + ( + decimal(38,2), + irs.server_uptime_days /* Use SUMMARY row's uptime */ + ), + 0 + ) * + ( + ISNULL + ( + irs3.unused_indexes, + 0 + ) / + NULLIF + ( + CONVERT + ( + decimal(38,2), + irs3.index_count + ), + 0 + ) + ), + 0 + ) + ELSE 0 + END + ) + ) + FROM #index_reporting_stats AS irs3 + WHERE irs3.summary_level = 'DATABASE' + ), + 'N0' + ) WHEN irs.summary_level <> 'SUMMARY' THEN /* For rows with unused indexes, calculate estimated savings */ @@ -5354,7 +5408,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Rows without unused indexes have no savings */ ELSE '0' END - ELSE 'N/A' + ELSE '0' END, /* ===== Section 4: Consolidated Performance Metrics ===== */ @@ -5370,6 +5424,60 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Lock waits saved - new column */ daily_lock_waits_saved = CASE + WHEN irs.summary_level = 'SUMMARY' + THEN + /* For SUMMARY row, calculate the sum of daily lock waits saved across all databases */ + FORMAT + ( + ( + SELECT + SUM + ( + CONVERT + ( + decimal(38,2), + CASE + WHEN ISNULL(irs3.unused_indexes, 0) > 0 + THEN + ISNULL + ( + (irs3.row_lock_wait_count + irs3.page_lock_wait_count) / + NULLIF + ( + CONVERT + ( + decimal(38,2), + irs.server_uptime_days /* Use SUMMARY row's uptime */ + ), + 0 + ) * + ( + ISNULL + ( + irs3.unused_indexes, + 0 + ) / + NULLIF + ( + CONVERT + ( + decimal(38,2), + irs3.index_count + ), + 0 + ) + ), + 0 + ) + ELSE 0 + END + ) + ) + FROM #index_reporting_stats AS irs3 + WHERE irs3.summary_level = 'DATABASE' + ), + 'N0' + ) WHEN irs.summary_level <> 'SUMMARY' THEN /* For rows with unused indexes, calculate estimated savings */ @@ -5420,7 +5528,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Rows without unused indexes have no savings */ ELSE '0' END - ELSE 'N/A' + ELSE '0' END, /* Average lock wait time in ms */ @@ -5448,9 +5556,63 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Latch waits saved - new column */ daily_latch_waits_saved = CASE + WHEN irs.summary_level = 'SUMMARY' + THEN + /* For SUMMARY row, calculate the sum of daily latch waits saved across all databases */ + FORMAT + ( + ( + SELECT + SUM + ( + CONVERT + ( + decimal(38,2), + CASE + WHEN ISNULL(irs3.unused_indexes, 0) > 0 + THEN + ISNULL + ( + (irs3.page_latch_wait_count + irs3.page_io_latch_wait_count) / + NULLIF + ( + CONVERT + ( + decimal(38,2), + irs.server_uptime_days /* Use SUMMARY row's uptime */ + ), + 0 + ) * + ( + ISNULL + ( + irs3.unused_indexes, + 0 + ) / + NULLIF + ( + CONVERT + ( + decimal(38,2), + irs3.index_count + ), + 0 + ) + ), + 0 + ) + ELSE 0 + END + ) + ) + FROM #index_reporting_stats AS irs3 + WHERE irs3.summary_level = 'DATABASE' + ), + 'N0' + ) WHEN irs.summary_level <> 'SUMMARY' THEN - /* For rows with unused indexes, calculate estimated savings */ + /* For DATABASE and TABLE rows, calculate estimated savings */ CASE WHEN ISNULL(irs.unused_indexes, 0) > 0 THEN @@ -5498,7 +5660,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Rows without unused indexes have no savings */ ELSE '0' END - ELSE 'N/A' + ELSE '0' END, /* Combined latch wait time in ms */ From 12e5c367addb3e412a731c2b14be15dd6bac0fe8 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Wed, 19 Mar 2025 00:19:31 -0400 Subject: [PATCH 235/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index d0e5e924..6268adaa 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -5406,7 +5406,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CONVERT ( decimal(38,2), - irs.server_uptime_days /* Use SUMMARY row's uptime */ + sd.days /* Get server_uptime_days from subquery */ ), 0 ) * @@ -5433,6 +5433,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ) ) FROM #index_reporting_stats AS irs3 + CROSS JOIN (SELECT days = MAX(server_uptime_days) FROM #index_reporting_stats WHERE summary_level = 'SUMMARY') AS sd WHERE irs3.summary_level = 'DATABASE' ), 'N0' @@ -5525,7 +5526,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CONVERT ( decimal(38,2), - irs.server_uptime_days /* Use SUMMARY row's uptime */ + sd.days /* Get server_uptime_days from subquery */ ), 0 ) * @@ -5552,6 +5553,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ) ) FROM #index_reporting_stats AS irs3 + CROSS JOIN (SELECT days = MAX(server_uptime_days) FROM #index_reporting_stats WHERE summary_level = 'SUMMARY') AS sd WHERE irs3.summary_level = 'DATABASE' ), 'N0' @@ -5657,7 +5659,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CONVERT ( decimal(38,2), - irs.server_uptime_days /* Use SUMMARY row's uptime */ + sd.days /* Get server_uptime_days from subquery */ ), 0 ) * @@ -5684,6 +5686,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ) ) FROM #index_reporting_stats AS irs3 + CROSS JOIN (SELECT days = MAX(server_uptime_days) FROM #index_reporting_stats WHERE summary_level = 'SUMMARY') AS sd WHERE irs3.summary_level = 'DATABASE' ), 'N0' From c7060b5f591e4e2c7d414a5c9e661328d8dd8f2a Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Wed, 19 Mar 2025 00:35:46 -0400 Subject: [PATCH 236/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 117 ++++++++++++++++------------ 1 file changed, 69 insertions(+), 48 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 6268adaa..b14961c9 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -5384,36 +5384,43 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. daily_write_ops_saved = CASE WHEN irs.summary_level = 'SUMMARY' + THEN 'N/A' /* For SUMMARY row, use N/A to be consistent with other metrics */ + WHEN irs.summary_level = 'DATABASE' THEN - /* For SUMMARY row, calculate the sum of daily write ops saved across all databases */ + /* For DATABASE level, calculate based on sum of unused indexes from tables */ FORMAT ( ( - SELECT + SELECT SUM ( CONVERT ( decimal(38,2), CASE - WHEN ISNULL(irs3.unused_indexes, 0) > 0 + WHEN ISNULL(irt.unused_indexes, 0) > 0 THEN ISNULL ( - irs3.user_updates / + irt.user_updates / NULLIF ( CONVERT ( - decimal(38,2), - sd.days /* Get server_uptime_days from subquery */ - ), + decimal(38,2), + ( + SELECT TOP (1) + irs2.server_uptime_days + FROM #index_reporting_stats AS irs2 + WHERE irs2.summary_level = 'SUMMARY' + ) + ), 0 ) * ( ISNULL ( - irs3.unused_indexes, + irt.unused_indexes, 0 ) / NULLIF @@ -5421,26 +5428,26 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CONVERT ( decimal(38,2), - irs3.index_count - ), + irt.index_count + ), 0 ) - ), + ), 0 ) ELSE 0 END ) ) - FROM #index_reporting_stats AS irs3 - CROSS JOIN (SELECT days = MAX(server_uptime_days) FROM #index_reporting_stats WHERE summary_level = 'SUMMARY') AS sd - WHERE irs3.summary_level = 'DATABASE' + FROM #index_reporting_stats AS irt + WHERE irt.summary_level = 'TABLE' + AND irt.database_name = irs.database_name ), 'N0' ) - WHEN irs.summary_level <> 'SUMMARY' + WHEN irs.summary_level = 'TABLE' THEN - /* For rows with unused indexes, calculate estimated savings */ + /* For TABLE rows, calculate estimated savings */ CASE WHEN ISNULL(irs.unused_indexes, 0) > 0 THEN FORMAT @@ -5504,36 +5511,43 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. daily_lock_waits_saved = CASE WHEN irs.summary_level = 'SUMMARY' + THEN 'N/A' /* For SUMMARY row, use N/A to be consistent with other metrics */ + WHEN irs.summary_level = 'DATABASE' THEN - /* For SUMMARY row, calculate the sum of daily lock waits saved across all databases */ + /* For DATABASE level, calculate based on sum of unused indexes from tables */ FORMAT ( ( - SELECT + SELECT SUM ( CONVERT ( decimal(38,2), CASE - WHEN ISNULL(irs3.unused_indexes, 0) > 0 + WHEN ISNULL(irt.unused_indexes, 0) > 0 THEN ISNULL ( - (irs3.row_lock_wait_count + irs3.page_lock_wait_count) / + (irt.row_lock_wait_count + irt.page_lock_wait_count) / NULLIF ( CONVERT ( - decimal(38,2), - sd.days /* Get server_uptime_days from subquery */ - ), + decimal(38,2), + ( + SELECT TOP (1) + irs2.server_uptime_days + FROM #index_reporting_stats AS irs2 + WHERE irs2.summary_level = 'SUMMARY' + ) + ), 0 ) * ( ISNULL ( - irs3.unused_indexes, + irt.unused_indexes, 0 ) / NULLIF @@ -5541,26 +5555,26 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CONVERT ( decimal(38,2), - irs3.index_count - ), + irt.index_count + ), 0 ) - ), + ), 0 ) ELSE 0 END ) ) - FROM #index_reporting_stats AS irs3 - CROSS JOIN (SELECT days = MAX(server_uptime_days) FROM #index_reporting_stats WHERE summary_level = 'SUMMARY') AS sd - WHERE irs3.summary_level = 'DATABASE' + FROM #index_reporting_stats AS irt + WHERE irt.summary_level = 'TABLE' + AND irt.database_name = irs.database_name ), 'N0' ) - WHEN irs.summary_level <> 'SUMMARY' + WHEN irs.summary_level = 'TABLE' THEN - /* For rows with unused indexes, calculate estimated savings */ + /* For TABLE rows, calculate estimated savings */ CASE WHEN ISNULL(irs.unused_indexes, 0) > 0 THEN @@ -5637,36 +5651,43 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. daily_latch_waits_saved = CASE WHEN irs.summary_level = 'SUMMARY' + THEN 'N/A' /* For SUMMARY row, use N/A to be consistent with other metrics */ + WHEN irs.summary_level = 'DATABASE' THEN - /* For SUMMARY row, calculate the sum of daily latch waits saved across all databases */ + /* For DATABASE level, calculate based on sum of unused indexes from tables */ FORMAT ( ( - SELECT + SELECT SUM ( CONVERT ( decimal(38,2), CASE - WHEN ISNULL(irs3.unused_indexes, 0) > 0 + WHEN ISNULL(irt.unused_indexes, 0) > 0 THEN ISNULL ( - (irs3.page_latch_wait_count + irs3.page_io_latch_wait_count) / + (irt.page_latch_wait_count + irt.page_io_latch_wait_count) / NULLIF ( CONVERT ( - decimal(38,2), - sd.days /* Get server_uptime_days from subquery */ - ), + decimal(38,2), + ( + SELECT TOP (1) + irs2.server_uptime_days + FROM #index_reporting_stats AS irs2 + WHERE irs2.summary_level = 'SUMMARY' + ) + ), 0 ) * ( ISNULL ( - irs3.unused_indexes, + irt.unused_indexes, 0 ) / NULLIF @@ -5674,26 +5695,26 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. CONVERT ( decimal(38,2), - irs3.index_count - ), + irt.index_count + ), 0 ) - ), + ), 0 ) ELSE 0 END ) ) - FROM #index_reporting_stats AS irs3 - CROSS JOIN (SELECT days = MAX(server_uptime_days) FROM #index_reporting_stats WHERE summary_level = 'SUMMARY') AS sd - WHERE irs3.summary_level = 'DATABASE' + FROM #index_reporting_stats AS irt + WHERE irt.summary_level = 'TABLE' + AND irt.database_name = irs.database_name ), 'N0' ) - WHEN irs.summary_level <> 'SUMMARY' + WHEN irs.summary_level = 'TABLE' THEN - /* For DATABASE and TABLE rows, calculate estimated savings */ + /* For TABLE rows, calculate estimated savings */ CASE WHEN ISNULL(irs.unused_indexes, 0) > 0 THEN From 76f89e8be1a0e52b3ccba95ec7d4d0f6df5f84cf Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Wed, 19 Mar 2025 00:41:19 -0400 Subject: [PATCH 237/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 293 +++++++++++++--------------- 1 file changed, 131 insertions(+), 162 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index b14961c9..bd763d0e 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -5387,64 +5387,53 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. THEN 'N/A' /* For SUMMARY row, use N/A to be consistent with other metrics */ WHEN irs.summary_level = 'DATABASE' THEN - /* For DATABASE level, calculate based on sum of unused indexes from tables */ - FORMAT - ( - ( - SELECT - SUM - ( - CONVERT + /* For DATABASE level, use the same calculation as TABLE level but with DATABASE row values */ + CASE + WHEN ISNULL(irs.unused_indexes, 0) > 0 + THEN FORMAT + ( + CONVERT(decimal(38,2), + ISNULL ( - decimal(38,2), - CASE - WHEN ISNULL(irt.unused_indexes, 0) > 0 - THEN - ISNULL + irs.user_updates / + NULLIF + ( + CONVERT + ( + decimal(38,2), ( - irt.user_updates / - NULLIF - ( - CONVERT - ( - decimal(38,2), - ( - SELECT TOP (1) - irs2.server_uptime_days - FROM #index_reporting_stats AS irs2 - WHERE irs2.summary_level = 'SUMMARY' - ) - ), - 0 - ) * - ( - ISNULL - ( - irt.unused_indexes, - 0 - ) / - NULLIF - ( - CONVERT - ( - decimal(38,2), - irt.index_count - ), - 0 - ) - ), - 0 + SELECT TOP (1) + irs2.server_uptime_days + FROM #index_reporting_stats AS irs2 + WHERE irs2.summary_level = 'SUMMARY' ) - ELSE 0 - END + ), + 0 + ) * + ( + ISNULL + ( + irs.unused_indexes, + 0 + ) / + NULLIF + ( + CONVERT + ( + decimal(38,2), + irs.index_count + ), + 0 + ) + ), + 0 ) - ) - FROM #index_reporting_stats AS irt - WHERE irt.summary_level = 'TABLE' - AND irt.database_name = irs.database_name - ), - 'N0' - ) + ), + 'N0' + ) + /* Rows without unused indexes have no savings */ + ELSE '0' + END WHEN irs.summary_level = 'TABLE' THEN /* For TABLE rows, calculate estimated savings */ @@ -5514,64 +5503,54 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. THEN 'N/A' /* For SUMMARY row, use N/A to be consistent with other metrics */ WHEN irs.summary_level = 'DATABASE' THEN - /* For DATABASE level, calculate based on sum of unused indexes from tables */ - FORMAT - ( - ( - SELECT - SUM - ( - CONVERT + /* For DATABASE level, use the same calculation as TABLE level but with DATABASE row values */ + CASE + WHEN ISNULL(irs.unused_indexes, 0) > 0 + THEN + FORMAT + ( + CONVERT(decimal(38,2), + ISNULL ( - decimal(38,2), - CASE - WHEN ISNULL(irt.unused_indexes, 0) > 0 - THEN - ISNULL + (irs.row_lock_wait_count + irs.page_lock_wait_count) / + NULLIF + ( + CONVERT + ( + decimal(38,2), ( - (irt.row_lock_wait_count + irt.page_lock_wait_count) / - NULLIF - ( - CONVERT - ( - decimal(38,2), - ( - SELECT TOP (1) - irs2.server_uptime_days - FROM #index_reporting_stats AS irs2 - WHERE irs2.summary_level = 'SUMMARY' - ) - ), - 0 - ) * - ( - ISNULL - ( - irt.unused_indexes, - 0 - ) / - NULLIF - ( - CONVERT - ( - decimal(38,2), - irt.index_count - ), - 0 - ) - ), - 0 + SELECT TOP (1) + irs2.server_uptime_days + FROM #index_reporting_stats AS irs2 + WHERE irs2.summary_level = 'SUMMARY' ) - ELSE 0 - END + ), + 0 + ) * + ( + ISNULL + ( + irs.unused_indexes, + 0 + ) / + NULLIF + ( + CONVERT + ( + decimal(38,2), + irs.index_count + ), + 0 + ) + ), + 0 ) - ) - FROM #index_reporting_stats AS irt - WHERE irt.summary_level = 'TABLE' - AND irt.database_name = irs.database_name - ), - 'N0' - ) + ), + 'N0' + ) + /* Rows without unused indexes have no savings */ + ELSE '0' + END WHEN irs.summary_level = 'TABLE' THEN /* For TABLE rows, calculate estimated savings */ @@ -5654,64 +5633,54 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. THEN 'N/A' /* For SUMMARY row, use N/A to be consistent with other metrics */ WHEN irs.summary_level = 'DATABASE' THEN - /* For DATABASE level, calculate based on sum of unused indexes from tables */ - FORMAT - ( - ( - SELECT - SUM - ( - CONVERT + /* For DATABASE level, use the same calculation as TABLE level but with DATABASE row values */ + CASE + WHEN ISNULL(irs.unused_indexes, 0) > 0 + THEN + FORMAT + ( + CONVERT(decimal(38,2), + ISNULL ( - decimal(38,2), - CASE - WHEN ISNULL(irt.unused_indexes, 0) > 0 - THEN - ISNULL + (irs.page_latch_wait_count + irs.page_io_latch_wait_count) / + NULLIF + ( + CONVERT + ( + decimal(38,2), ( - (irt.page_latch_wait_count + irt.page_io_latch_wait_count) / - NULLIF - ( - CONVERT - ( - decimal(38,2), - ( - SELECT TOP (1) - irs2.server_uptime_days - FROM #index_reporting_stats AS irs2 - WHERE irs2.summary_level = 'SUMMARY' - ) - ), - 0 - ) * - ( - ISNULL - ( - irt.unused_indexes, - 0 - ) / - NULLIF - ( - CONVERT - ( - decimal(38,2), - irt.index_count - ), - 0 - ) - ), - 0 + SELECT TOP (1) + irs2.server_uptime_days + FROM #index_reporting_stats AS irs2 + WHERE irs2.summary_level = 'SUMMARY' ) - ELSE 0 - END + ), + 0 + ) * + ( + ISNULL + ( + irs.unused_indexes, + 0 + ) / + NULLIF + ( + CONVERT + ( + decimal(38,2), + irs.index_count + ), + 0 + ) + ), + 0 ) - ) - FROM #index_reporting_stats AS irt - WHERE irt.summary_level = 'TABLE' - AND irt.database_name = irs.database_name - ), - 'N0' - ) + ), + 'N0' + ) + /* Rows without unused indexes have no savings */ + ELSE '0' + END WHEN irs.summary_level = 'TABLE' THEN /* For TABLE rows, calculate estimated savings */ From 1e9536b0060c2a7542dc9004741a0b29b3a40ba4 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Wed, 19 Mar 2025 01:05:54 -0400 Subject: [PATCH 238/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 203 +++++++++------------------- 1 file changed, 63 insertions(+), 140 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index bd763d0e..7bf53312 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -5387,53 +5387,28 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. THEN 'N/A' /* For SUMMARY row, use N/A to be consistent with other metrics */ WHEN irs.summary_level = 'DATABASE' THEN - /* For DATABASE level, use the same calculation as TABLE level but with DATABASE row values */ - CASE - WHEN ISNULL(irs.unused_indexes, 0) > 0 - THEN FORMAT - ( - CONVERT(decimal(38,2), - ISNULL - ( - irs.user_updates / - NULLIF - ( - CONVERT - ( - decimal(38,2), - ( - SELECT TOP (1) - irs2.server_uptime_days - FROM #index_reporting_stats AS irs2 - WHERE irs2.summary_level = 'SUMMARY' - ) - ), - 0 - ) * - ( - ISNULL - ( - irs.unused_indexes, - 0 - ) / - NULLIF - ( - CONVERT - ( - decimal(38,2), - irs.index_count - ), - 0 - ) - ), - 0 + /* For DATABASE level, sum up the table-level values */ + FORMAT + ( + ( + SELECT + SUM + ( + TRY_CAST( + REPLACE( + REPLACE( + REPLACE(irt.daily_write_ops_saved, ',', ''), + 'N/A', '0'), + ' ', '') + AS decimal(38,2) ) - ), - 'N0' - ) - /* Rows without unused indexes have no savings */ - ELSE '0' - END + ) + FROM #index_reporting_stats AS irt + WHERE irt.summary_level = 'TABLE' + AND irt.database_name = irs.database_name + ), + 'N0' + ) WHEN irs.summary_level = 'TABLE' THEN /* For TABLE rows, calculate estimated savings */ @@ -5503,54 +5478,28 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. THEN 'N/A' /* For SUMMARY row, use N/A to be consistent with other metrics */ WHEN irs.summary_level = 'DATABASE' THEN - /* For DATABASE level, use the same calculation as TABLE level but with DATABASE row values */ - CASE - WHEN ISNULL(irs.unused_indexes, 0) > 0 - THEN - FORMAT - ( - CONVERT(decimal(38,2), - ISNULL - ( - (irs.row_lock_wait_count + irs.page_lock_wait_count) / - NULLIF - ( - CONVERT - ( - decimal(38,2), - ( - SELECT TOP (1) - irs2.server_uptime_days - FROM #index_reporting_stats AS irs2 - WHERE irs2.summary_level = 'SUMMARY' - ) - ), - 0 - ) * - ( - ISNULL - ( - irs.unused_indexes, - 0 - ) / - NULLIF - ( - CONVERT - ( - decimal(38,2), - irs.index_count - ), - 0 - ) - ), - 0 + /* For DATABASE level, sum up the table-level values */ + FORMAT + ( + ( + SELECT + SUM + ( + TRY_CAST( + REPLACE( + REPLACE( + REPLACE(irt.daily_lock_waits_saved, ',', ''), + 'N/A', '0'), + ' ', '') + AS decimal(38,2) ) - ), - 'N0' - ) - /* Rows without unused indexes have no savings */ - ELSE '0' - END + ) + FROM #index_reporting_stats AS irt + WHERE irt.summary_level = 'TABLE' + AND irt.database_name = irs.database_name + ), + 'N0' + ) WHEN irs.summary_level = 'TABLE' THEN /* For TABLE rows, calculate estimated savings */ @@ -5633,54 +5582,28 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. THEN 'N/A' /* For SUMMARY row, use N/A to be consistent with other metrics */ WHEN irs.summary_level = 'DATABASE' THEN - /* For DATABASE level, use the same calculation as TABLE level but with DATABASE row values */ - CASE - WHEN ISNULL(irs.unused_indexes, 0) > 0 - THEN - FORMAT - ( - CONVERT(decimal(38,2), - ISNULL - ( - (irs.page_latch_wait_count + irs.page_io_latch_wait_count) / - NULLIF - ( - CONVERT - ( - decimal(38,2), - ( - SELECT TOP (1) - irs2.server_uptime_days - FROM #index_reporting_stats AS irs2 - WHERE irs2.summary_level = 'SUMMARY' - ) - ), - 0 - ) * - ( - ISNULL - ( - irs.unused_indexes, - 0 - ) / - NULLIF - ( - CONVERT - ( - decimal(38,2), - irs.index_count - ), - 0 - ) - ), - 0 + /* For DATABASE level, sum up the table-level values */ + FORMAT + ( + ( + SELECT + SUM + ( + TRY_CAST( + REPLACE( + REPLACE( + REPLACE(irt.daily_latch_waits_saved, ',', ''), + 'N/A', '0'), + ' ', '') + AS decimal(38,2) ) - ), - 'N0' - ) - /* Rows without unused indexes have no savings */ - ELSE '0' - END + ) + FROM #index_reporting_stats AS irt + WHERE irt.summary_level = 'TABLE' + AND irt.database_name = irs.database_name + ), + 'N0' + ) WHEN irs.summary_level = 'TABLE' THEN /* For TABLE rows, calculate estimated savings */ From 4b62704fd5635839b7890e896dff8b5defb0562a Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Wed, 19 Mar 2025 01:17:44 -0400 Subject: [PATCH 239/246] Update sp_IndexCleanup.sql --- sp_IndexCleanup/sp_IndexCleanup.sql | 80 +++-------------------------- 1 file changed, 7 insertions(+), 73 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 7bf53312..05c6494b 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -5386,29 +5386,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. WHEN irs.summary_level = 'SUMMARY' THEN 'N/A' /* For SUMMARY row, use N/A to be consistent with other metrics */ WHEN irs.summary_level = 'DATABASE' - THEN - /* For DATABASE level, sum up the table-level values */ - FORMAT - ( - ( - SELECT - SUM - ( - TRY_CAST( - REPLACE( - REPLACE( - REPLACE(irt.daily_write_ops_saved, ',', ''), - 'N/A', '0'), - ' ', '') - AS decimal(38,2) - ) - ) - FROM #index_reporting_stats AS irt - WHERE irt.summary_level = 'TABLE' - AND irt.database_name = irs.database_name - ), - 'N0' - ) + THEN 'N/A' WHEN irs.summary_level = 'TABLE' THEN /* For TABLE rows, calculate estimated savings */ @@ -5468,7 +5446,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. WHEN irs.summary_level <> 'SUMMARY' THEN FORMAT(ISNULL(irs.row_lock_wait_count, 0) + ISNULL(irs.page_lock_wait_count, 0), 'N0') - ELSE '0' + ELSE 'N/A' END, /* Lock waits saved - new column */ @@ -5477,29 +5455,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. WHEN irs.summary_level = 'SUMMARY' THEN 'N/A' /* For SUMMARY row, use N/A to be consistent with other metrics */ WHEN irs.summary_level = 'DATABASE' - THEN - /* For DATABASE level, sum up the table-level values */ - FORMAT - ( - ( - SELECT - SUM - ( - TRY_CAST( - REPLACE( - REPLACE( - REPLACE(irt.daily_lock_waits_saved, ',', ''), - 'N/A', '0'), - ' ', '') - AS decimal(38,2) - ) - ) - FROM #index_reporting_stats AS irt - WHERE irt.summary_level = 'TABLE' - AND irt.database_name = irs.database_name - ), - 'N0' - ) + THEN 'N/A' WHEN irs.summary_level = 'TABLE' THEN /* For TABLE rows, calculate estimated savings */ @@ -5563,7 +5519,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ISNULL(irs.page_lock_wait_in_ms, 0)) / NULLIF(ISNULL(irs.row_lock_wait_count, 0) + ISNULL(irs.page_lock_wait_count, 0), 0), 'N2') - ELSE '0.00' + ELSE '0' END, /* Total count of latch waits (page + io) - new column */ @@ -5572,7 +5528,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. WHEN irs.summary_level <> 'SUMMARY' THEN FORMAT(ISNULL(irs.page_latch_wait_count, 0) + ISNULL(irs.page_io_latch_wait_count, 0), 'N0') - ELSE '0' + ELSE 'N/A' END, /* Latch waits saved - new column */ @@ -5581,29 +5537,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. WHEN irs.summary_level = 'SUMMARY' THEN 'N/A' /* For SUMMARY row, use N/A to be consistent with other metrics */ WHEN irs.summary_level = 'DATABASE' - THEN - /* For DATABASE level, sum up the table-level values */ - FORMAT - ( - ( - SELECT - SUM - ( - TRY_CAST( - REPLACE( - REPLACE( - REPLACE(irt.daily_latch_waits_saved, ',', ''), - 'N/A', '0'), - ' ', '') - AS decimal(38,2) - ) - ) - FROM #index_reporting_stats AS irt - WHERE irt.summary_level = 'TABLE' - AND irt.database_name = irs.database_name - ), - 'N0' - ) + THEN 'N/A' WHEN irs.summary_level = 'TABLE' THEN /* For TABLE rows, calculate estimated savings */ @@ -5667,7 +5601,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ISNULL(irs.page_io_latch_wait_in_ms, 0)) / NULLIF(ISNULL(irs.page_latch_wait_count, 0) + ISNULL(irs.page_io_latch_wait_count, 0), 0), 'N2') - ELSE '0.00' + ELSE 'N/A' END FROM #index_reporting_stats AS irs WHERE irs.summary_level IN ('SUMMARY', 'DATABASE', 'TABLE') /* Filter out INDEX level */ From 0d70c15d881ed5897401ff96ed5c41ee521f8277 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Wed, 19 Mar 2025 10:11:01 -0400 Subject: [PATCH 240/246] tidying making sure the output looks normal --- sp_IndexCleanup/sp_IndexCleanup.sql | 179 ++++++++++++++++++++++++---- sp_QuickieStore/sp_QuickieStore.sql | 18 ++- 2 files changed, 166 insertions(+), 31 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 05c6494b..1907c26e 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -71,10 +71,6 @@ BEGIN TRY @version = '1.4', @version_date = '20250401'; - SELECT - for_insurance_purposes = - N'ALWAYS TEST THESE RECOMMENDATIONS IN A NON-PRODUCTION ENVIRONMENT FIRST!'; - /* Help section, for help. Will become more helpful when out of beta. @@ -88,13 +84,13 @@ BEGIN TRY help = N'this is a script to help clean up unused and duplicate indexes.' UNION ALL SELECT - help = N'it will also give you scripted out statements to add page compression to uncompressed indexes.' + help = N'it will also help you add page compression to uncompressed indexes.' UNION ALL SELECT help = N'always validate all changes against a non-production environment!' UNION ALL SELECT - help = N'without careful analysis and consideration, index changes can negative impacts on performance.'; + help = N'please test carefully.'; /* Parameters @@ -209,7 +205,6 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. DECLARE /*general script variables*/ @sql nvarchar(max) = N'', - @database_id integer = NULL, @object_id integer = NULL, @full_object_name nvarchar(768) = NULL, @uptime_warning bit = 0, /* Will set after @uptime_days is calculated */ @@ -702,6 +697,55 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END; END; + IF @get_all_databases = 1 + AND @include_databases IS NOT NULL + BEGIN + INSERT + #requested_but_skipped_databases + WITH + (TABLOCK) + ( + database_name, + reason + ) + SELECT + id.database_name, + reason = + CASE + WHEN d.name IS NULL + THEN 'Database does not exist' + WHEN d.state <> 0 + THEN 'Database not online' + WHEN d.is_in_standby = 1 + THEN 'Database is in standby' + WHEN d.is_read_only = 1 + THEN 'Database is read-only' + WHEN d.database_id <= 4 + THEN 'System database' + ELSE 'Other issue' + END + FROM #include_databases AS id + LEFT JOIN sys.databases AS d + ON id.database_name = d.name + WHERE NOT EXISTS + ( + SELECT + 1/0 + FROM #databases AS db + WHERE db.database_name = id.database_name + ) + OPTION(RECOMPILE); + + IF @debug = 1 + BEGIN + SELECT + table_name = '#requested_but_skipped_databases', + rbsd.* + FROM #requested_but_skipped_databases AS rbsd + OPTION(RECOMPILE); + END; + END; + /* Parse @exclude_databases comma-separated list */ IF @get_all_databases = 1 AND @exclude_databases IS NOT NULL @@ -2993,6 +3037,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. target_index_name, superseded_info, original_index_definition, + script, index_size_gb, index_rows, index_reads, @@ -3000,17 +3045,72 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ) SELECT result_type = 'SUMMARY', - sort_order = 1, - database_name = '', - schema_name = '', - table_name = '', - index_name = '', - consolidation_rule = N'', - script_type = 'Index Cleanup Scripts', + sort_order = -1, + database_name = + N'processed databases: ' + + CASE + WHEN @get_all_databases = 0 + THEN ISNULL(@database_name, N'None') + ELSE + ISNULL + ( + STUFF + ( + ( + SELECT + N', ' + + d.database_name + FROM #databases AS d + ORDER BY + d.database_name + FOR + XML + PATH(''), + TYPE + ).value('.', 'nvarchar(max)'), + 1, + 2, + N'' + ), + N'None' + ) + END, + schema_name = + N'skipped databases: ' + + ISNULL + ( + STUFF + ( + ( + SELECT + N', ' + + rbs.database_name + + N' (' + + rbs.reason + + N')' + FROM #requested_but_skipped_databases AS rbs + ORDER BY + rbs.database_name + FOR + XML + PATH(''), + TYPE + ).value('.', 'nvarchar(MAX)'), + 1, + 2, + N'' + ), + N'None' + ), + table_name = N'brought to you by erikdarling.com', + index_name = N'for support: https://code.erikdarling.com/', + consolidation_rule = N'run date: ' + CONVERT(nvarchar(30), SYSDATETIME(), 120), + script_type = N'Index Cleanup Scripts', additional_info = N'A detailed index analysis report appears after these scripts', - target_index_name = '', - superseded_info = '', - original_index_definition = '', + target_index_name = N'ALWAYS TEST THESE RECOMMENDATIONS', + superseded_info = N'IN A NON-PRODUCTION ENVIRONMENT FIRST!', + original_index_definition = N'please enjoy responsibly!', + script = N'happy index cleaning!', index_size_gb = 0, index_rows = 0, index_reads = 0, @@ -4597,6 +4697,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ON ia.database_id = ce.database_id AND ia.object_id = ce.object_id AND ia.index_id = ce.index_id + WHERE ia.index_id > 1 OPTION(RECOMPILE); /* Return enhanced database impact summaries */ @@ -5166,7 +5267,12 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. THEN '0' ELSE FORMAT(ISNULL(ir.index_writes, 0), 'N0') END, - ia.original_index_definition, + original_index_definition = + CASE + WHEN ir.result_type = 'SUMMARY' + THEN N'please enjoy responsibly!' + ELSE ia.original_index_definition + END, /* Finally show the actual script */ ir.script FROM @@ -5253,8 +5359,22 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END, /* Schema and table names (except for summary) */ - schema_name = ISNULL(irs.schema_name, 'N/A'), - table_name = ISNULL(irs.table_name, 'N/A'), + schema_name = + CASE + WHEN irs.summary_level = 'SUMMARY' + THEN ISNULL(irs.schema_name, 'ALWAYS TEST THESE RECOMMENDATIONS') + WHEN irs.summary_level = 'DATABASE' + THEN N'N/A' + ELSE irs.schema_name + END, + table_name = + CASE + WHEN irs.summary_level = 'SUMMARY' + THEN ISNULL(irs.table_name, 'IN A NON-PRODUCTION ENVIRONMENT FIRST!') + WHEN irs.summary_level = 'DATABASE' + THEN N'N/A' + ELSE irs.table_name + END, /* ===== Section 1: Index Counts ===== */ /* Tables analyzed (summary only) */ @@ -5375,9 +5495,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Total writes */ writes = CASE + WHEN irs.summary_level = 'SUMMARY' + THEN 'N/A' WHEN irs.summary_level <> 'SUMMARY' THEN FORMAT(ISNULL(irs.total_writes, 0), 'N0') - ELSE 'N/A' + ELSE '0' END, /* Write operations saved - added as new metric */ @@ -5443,10 +5565,12 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Total count of lock waits (row + page) */ lock_wait_count = CASE + WHEN irs.summary_level = 'SUMMARY' + THEN 'N/A' WHEN irs.summary_level <> 'SUMMARY' THEN FORMAT(ISNULL(irs.row_lock_wait_count, 0) + ISNULL(irs.page_lock_wait_count, 0), 'N0') - ELSE 'N/A' + ELSE '0' END, /* Lock waits saved - new column */ @@ -5512,6 +5636,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Average lock wait time in ms */ avg_lock_wait_ms = CASE + WHEN irs.summary_level = 'SUMMARY' + THEN 'N/A' WHEN irs.summary_level <> 'SUMMARY' AND (ISNULL(irs.row_lock_wait_count, 0) + ISNULL(irs.page_lock_wait_count, 0)) > 0 @@ -5525,10 +5651,12 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Total count of latch waits (page + io) - new column */ latch_wait_count = CASE + WHEN irs.summary_level = 'SUMMARY' + THEN 'N/A' WHEN irs.summary_level <> 'SUMMARY' THEN FORMAT(ISNULL(irs.page_latch_wait_count, 0) + ISNULL(irs.page_io_latch_wait_count, 0), 'N0') - ELSE 'N/A' + ELSE '0' END, /* Latch waits saved - new column */ @@ -5594,6 +5722,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* Combined latch wait time in ms */ avg_latch_wait_ms = CASE + WHEN irs.summary_level = 'SUMMARY' + THEN 'N/A' WHEN irs.summary_level <> 'SUMMARY' AND (ISNULL(irs.page_latch_wait_count, 0) + ISNULL(irs.page_io_latch_wait_count, 0)) > 0 @@ -5601,10 +5731,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ISNULL(irs.page_io_latch_wait_in_ms, 0)) / NULLIF(ISNULL(irs.page_latch_wait_count, 0) + ISNULL(irs.page_io_latch_wait_count, 0), 0), 'N2') - ELSE 'N/A' + ELSE '0' END FROM #index_reporting_stats AS irs - WHERE irs.summary_level IN ('SUMMARY', 'DATABASE', 'TABLE') /* Filter out INDEX level */ ORDER BY /* Order by database name */ irs.database_name, diff --git a/sp_QuickieStore/sp_QuickieStore.sql b/sp_QuickieStore/sp_QuickieStore.sql index 90beadba..c1415700 100644 --- a/sp_QuickieStore/sp_QuickieStore.sql +++ b/sp_QuickieStore/sp_QuickieStore.sql @@ -2118,12 +2118,18 @@ BEGIN id.database_name, reason = CASE - WHEN d.name IS NULL THEN 'Database does not exist' - WHEN d.state <> 0 THEN 'Database not online' - WHEN d.is_query_store_on = 0 THEN 'Query Store not enabled' - WHEN d.is_in_standby = 1 THEN 'Database is in standby' - WHEN d.is_read_only = 1 THEN 'Database is read-only' - WHEN d.database_id <= 4 THEN 'System database' + WHEN d.name IS NULL + THEN 'Database does not exist' + WHEN d.state <> 0 + THEN 'Database not online' + WHEN d.is_query_store_on = 0 + THEN 'Query Store not enabled' + WHEN d.is_in_standby = 1 + THEN 'Database is in standby' + WHEN d.is_read_only = 1 + THEN 'Database is read-only' + WHEN d.database_id <= 4 + THEN 'System database' ELSE 'Other issue' END FROM #include_databases AS id From 4561708430af75989641eac863bf1ed9054b051e Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Wed, 19 Mar 2025 10:35:41 -0400 Subject: [PATCH 241/246] recompiles and such yay --- sp_IndexCleanup/sp_IndexCleanup.sql | 9 ++++++--- sp_QuickieStore/sp_QuickieStore.sql | 19 ++++++++++++------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 1907c26e..9d0ca9cd 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -685,7 +685,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ) ) AS a CROSS APPLY x.nodes(N'//i') AS t(c) - WHERE LTRIM(RTRIM(c.value(N'(./text())[1]', N'sysname'))) <> N''; + WHERE LTRIM(RTRIM(c.value(N'(./text())[1]', N'sysname'))) <> N'' + OPTION(RECOMPILE); IF @debug = 1 BEGIN @@ -777,7 +778,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ) ) AS a CROSS APPLY x.nodes(N'//i') AS t(c) - WHERE LTRIM(RTRIM(c.value(N'(./text())[1]', N'sysname'))) <> N''; + WHERE LTRIM(RTRIM(c.value(N'(./text())[1]', N'sysname'))) <> N'' + OPTION(RECOMPILE); IF @debug = 1 BEGIN @@ -805,7 +807,8 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 1/0 FROM #include_databases AS id WHERE id.database_name = ed.database_name - ); + ) + OPTION(RECOMPILE); /* If we found any conflicts, raise an error */ IF LEN(@conflict_list) > 0 diff --git a/sp_QuickieStore/sp_QuickieStore.sql b/sp_QuickieStore/sp_QuickieStore.sql index c1415700..456b1605 100644 --- a/sp_QuickieStore/sp_QuickieStore.sql +++ b/sp_QuickieStore/sp_QuickieStore.sql @@ -1682,7 +1682,7 @@ DECLARE @requires_secondary_processing bit, @split_sql nvarchar(MAX), @error_msg nvarchar(2000), - @conflict_list nvarchar(max); + @conflict_list nvarchar(max) = N''; /* In cases where we are escaping @query_text_search and @@ -1963,6 +1963,8 @@ BEGIN BEGIN INSERT #include_databases + WITH + (TABLOCK) ( database_name ) @@ -1986,7 +1988,8 @@ BEGIN ) ) AS a CROSS APPLY x.nodes(N'//i') AS t(c) - WHERE LTRIM(RTRIM(c.value(N'(./text())[1]', N'sysname'))) <> N''; + WHERE LTRIM(RTRIM(c.value(N'(./text())[1]', N'sysname'))) <> N'' + OPTION(RECOMPILE); END; /* Parse @exclude_databases if specified using XML for compatibility */ @@ -1994,6 +1997,8 @@ BEGIN BEGIN INSERT #exclude_databases + WITH + (TABLOCK) ( database_name ) @@ -2017,14 +2022,13 @@ BEGIN ) ) AS a CROSS APPLY x.nodes(N'//i') AS t(c) - WHERE LTRIM(RTRIM(c.value(N'(./text())[1]', N'sysname'))) <> N''; + WHERE LTRIM(RTRIM(c.value(N'(./text())[1]', N'sysname'))) <> N'' + OPTION(RECOMPILE); /* Check for databases in both include and exclude lists */ IF @include_databases IS NOT NULL BEGIN - /* Build list of conflicting databases */ - SET @conflict_list = N''; - + /* Build list of conflicting databases */ SELECT @conflict_list = @conflict_list + @@ -2036,7 +2040,8 @@ BEGIN 1/0 FROM #include_databases AS id WHERE id.database_name = ed.database_name - ); + ) + OPTION(RECOMPILE); /* If we found any conflicts, raise an error */ IF LEN(@conflict_list) > 0 From bf05c7ddd4bd883193b61b6babdeda4a9509528a Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Wed, 19 Mar 2025 10:47:54 -0400 Subject: [PATCH 242/246] Update sp_IndexCleanup.sql nulls look dumb in dark mode --- sp_IndexCleanup/sp_IndexCleanup.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 9d0ca9cd..1f875a1c 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -5236,14 +5236,14 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ir.table_name, ir.index_name, /* Then show relationship information */ - ir.consolidation_rule, - ir.target_index_name, + consolidation_rule = ISNULL(ir.consolidation_rule, N'N/A'), + target_index_name = ISNULL(ir.target_index_name, N'N/A'), /* Include superseded_by info for winning indexes */ superseded_info = CASE WHEN ia.superseded_by IS NOT NULL THEN ia.superseded_by - ELSE ir.superseded_info + ELSE ISNULL(ir.superseded_info, N'N/A') END, /* Add size and usage metrics */ index_size_gb = From a89338cb912cbde96d5820b0ff95f896e6756525 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Wed, 19 Mar 2025 10:59:21 -0400 Subject: [PATCH 243/246] Update sp_IndexCleanup.sql redo the compression savings columns --- sp_IndexCleanup/sp_IndexCleanup.sql | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 1f875a1c..6a05dbd3 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -5465,16 +5465,18 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. /* ===== Additional Space Savings from Compression ===== */ /* Conservative compression estimate (20%) */ - compression_min_savings_gb = FORMAT(ISNULL(irs.compression_min_savings_gb, 0), 'N2'), - - /* Optimistic compression estimate (60%) */ - compression_max_savings_gb = FORMAT(ISNULL(irs.compression_max_savings_gb, 0), 'N2'), - - /* Total savings (removal + conservative compression) */ - total_min_savings_gb = FORMAT(ISNULL(irs.total_min_savings_gb, 0), 'N2'), - - /* Total savings (removal + optimistic compression) */ - total_max_savings_gb = FORMAT(ISNULL(irs.total_max_savings_gb, 0), 'N2'), + compression_savings_potential = + N'minimum: ' + + FORMAT(ISNULL(irs.compression_min_savings_gb, 0), 'N2') + + N' GB maximum ' + + FORMAT(ISNULL(irs.compression_max_savings_gb, 0), 'N2') + + N'GB', + compression_savings_potential_total = + N'total minimum: ' + + FORMAT(ISNULL(irs.total_min_savings_gb, 0), 'N2') + + N' GB total maximum: ' + + FORMAT(ISNULL(irs.total_max_savings_gb, 0), 'N2') + + N'GB', /* ===== Section 3: Table and Usage Statistics ===== */ /* Row count */ From 24e1f32649b142b33107c0089cc1e8ed13cdd460 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Wed, 19 Mar 2025 11:14:25 -0400 Subject: [PATCH 244/246] Update sp_IndexCleanup.sql my output is always verbose --- sp_IndexCleanup/sp_IndexCleanup.sql | 2 -- 1 file changed, 2 deletions(-) diff --git a/sp_IndexCleanup/sp_IndexCleanup.sql b/sp_IndexCleanup/sp_IndexCleanup.sql index 6a05dbd3..41bc6d28 100644 --- a/sp_IndexCleanup/sp_IndexCleanup.sql +++ b/sp_IndexCleanup/sp_IndexCleanup.sql @@ -26,7 +26,6 @@ ALTER PROCEDURE @get_all_databases bit = 0, /*looks for all accessible user databases and returns combined results*/ @include_databases nvarchar(max) = NULL, /*comma-separated list of databases to include (only when @get_all_databases = 1)*/ @exclude_databases nvarchar(max) = NULL, /*comma-separated list of databases to exclude (only when @get_all_databases = 1)*/ - @verbose_output tinyint = 0, /* 0 -> no verbose output, 1 -> add NONUNIQUE, NONCLUSTERED type output in the original_index_defintion output */ @help bit = 'false', @debug bit = 'false', @version varchar(20) = NULL OUTPUT, @@ -2056,7 +2055,6 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. WHEN id1.index_id = 1 THEN N'CLUSTERED ' WHEN id1.index_id > 1 - AND @verbose_output >= 1 THEN N'NONCLUSTERED ' ELSE N'' END + From bb51cb0dee403d69662ddfcb486d70827289099b Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Wed, 19 Mar 2025 11:19:17 -0400 Subject: [PATCH 245/246] update URLs short and sweet --- Alerts/Alerts.sql | 2 +- Clear Token Perm/ClearTokenPerm Agent Job.sql | 4 ++-- Clear Token Perm/ClearTokenPerm.sql | 2 +- ...Security Cache Demo And Analysis Script.sql | 2 +- Create Long IN List/Longingly.sql | 2 +- Helper Views/WhatsUpIndexes.sql | 2 +- Helper Views/WhatsUpLocks.sql | 2 +- Helper Views/WhatsUpMemory.sql | 2 +- Helper Views/tempdb_tester.sql | 2 +- .../EffectiveAnnualInterestRate.sql | 2 +- Inline Financial Functions/FutureValue.sql | 2 +- Inline Financial Functions/InterestPayment.sql | 2 +- .../NumberOfPayments.sql | 2 +- Inline Financial Functions/NumberOfYears.sql | 2 +- Inline Financial Functions/Payment.sql | 2 +- .../PrincipalPayment.sql | 2 +- Install-All/DarlingData.sql | 18 +++++++++--------- .../MakeBigStackCCS.sql | 2 +- .../Script StackOverflowCS.sql | 2 +- String Functions/get_letters.sql | 4 ++-- String Functions/get_numbers.sql | 4 ++-- String Functions/strip_characters.sql | 4 ++-- sp_HealthParser/sp_HealthParser.sql | 2 +- .../sp_Human Events Agent Job Example.sql | 2 +- sp_HumanEvents/sp_HumanEvents.sql | 2 +- sp_HumanEvents/sp_HumanEventsBlockViewer.sql | 6 +++--- sp_LogHunter/sp_LogHunter.sql | 2 +- sp_PressureDetector/sp_PressureDetector.sql | 2 +- sp_QuickieStore/sp_QuickieStore.sql | 4 ++-- 29 files changed, 44 insertions(+), 44 deletions(-) diff --git a/Alerts/Alerts.sql b/Alerts/Alerts.sql index daec9eaa..361082ad 100644 --- a/Alerts/Alerts.sql +++ b/Alerts/Alerts.sql @@ -15,7 +15,7 @@ Copyright 2025 Darling Data, LLC https://www.erikdarling.com/ For support, head over to GitHub: -https://github.com/erikdarlingdata/DarlingData +https://code.erikdarling.com Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: diff --git a/Clear Token Perm/ClearTokenPerm Agent Job.sql b/Clear Token Perm/ClearTokenPerm Agent Job.sql index 06cedb4a..34b052df 100644 --- a/Clear Token Perm/ClearTokenPerm Agent Job.sql +++ b/Clear Token Perm/ClearTokenPerm Agent Job.sql @@ -20,7 +20,7 @@ Copyright 2025 Darling Data, LLC https://www.erikdarling.com/ For support, head over to GitHub: -https://github.com/erikdarlingdata/DarlingData +https://code.erikdarling.com Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: @@ -77,7 +77,7 @@ Copyright 2022 Darling Data, LLC https://www.erikdarling.com/ For support, head over to GitHub: -https://github.com/erikdarlingdata/DarlingData', +https://code.erikdarling.com', @category_name = N'[Uncategorized (Local)]', @owner_login_name = N'sa', @job_id = @jobId OUTPUT; diff --git a/Clear Token Perm/ClearTokenPerm.sql b/Clear Token Perm/ClearTokenPerm.sql index c70c9e31..9bf7d3db 100644 --- a/Clear Token Perm/ClearTokenPerm.sql +++ b/Clear Token Perm/ClearTokenPerm.sql @@ -20,7 +20,7 @@ Copyright 2025 Darling Data, LLC https://www.erikdarling.com/ For support, head over to GitHub: -https://github.com/erikdarlingdata/DarlingData +https://code.erikdarling.com Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: diff --git a/Clear Token Perm/Inflate Security Cache Demo And Analysis Script.sql b/Clear Token Perm/Inflate Security Cache Demo And Analysis Script.sql index 02bef443..9b9cd264 100644 --- a/Clear Token Perm/Inflate Security Cache Demo And Analysis Script.sql +++ b/Clear Token Perm/Inflate Security Cache Demo And Analysis Script.sql @@ -17,7 +17,7 @@ Copyright 2024 Darling Data, LLC https://www.erikdarling.com/ For support, head over to GitHub: -https://github.com/erikdarlingdata/DarlingData +https://code.erikdarling.com MIT License diff --git a/Create Long IN List/Longingly.sql b/Create Long IN List/Longingly.sql index 3f022f66..92c190e2 100644 --- a/Create Long IN List/Longingly.sql +++ b/Create Long IN List/Longingly.sql @@ -17,7 +17,7 @@ Copyright 2025 Darling Data, LLC https://www.erikdarling.com/ For support, head over to GitHub: -https://github.com/erikdarlingdata/DarlingData +https://code.erikdarling.com Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: diff --git a/Helper Views/WhatsUpIndexes.sql b/Helper Views/WhatsUpIndexes.sql index 605b9c71..f714455d 100644 --- a/Helper Views/WhatsUpIndexes.sql +++ b/Helper Views/WhatsUpIndexes.sql @@ -12,7 +12,7 @@ GO /* This is a quick one-off script I use in some presentations to look at index sizes. -https://github.com/erikdarlingdata/DarlingData +https://code.erikdarling.com Copyright (c) 2025 Darling Data, LLC https://www.erikdarling.com/ diff --git a/Helper Views/WhatsUpLocks.sql b/Helper Views/WhatsUpLocks.sql index 163c0501..19ab9af5 100644 --- a/Helper Views/WhatsUpLocks.sql +++ b/Helper Views/WhatsUpLocks.sql @@ -13,7 +13,7 @@ GO This is a helper function I use in some of my presentations to look at locks taken. It's definitely not a replacement for sp_WhoIsActive, it just gives me what I care about at the moment. -https://github.com/erikdarlingdata/DarlingData +https://code.erikdarling.com Copyright (c) 2025 Darling Data, LLC https://www.erikdarling.com/ diff --git a/Helper Views/WhatsUpMemory.sql b/Helper Views/WhatsUpMemory.sql index ae5c858f..8f5b4402 100644 --- a/Helper Views/WhatsUpMemory.sql +++ b/Helper Views/WhatsUpMemory.sql @@ -15,7 +15,7 @@ I probably wouldn't run this in production, especially on servers with a lot of The dm_os_buffer_descriptors DMV especially can be really slow at times -https://github.com/erikdarlingdata/DarlingData +https://code.erikdarling.com Copyright (c) 2025 Darling Data, LLC https://www.erikdarling.com/ diff --git a/Helper Views/tempdb_tester.sql b/Helper Views/tempdb_tester.sql index 8aa6c80f..0eb38d38 100644 --- a/Helper Views/tempdb_tester.sql +++ b/Helper Views/tempdb_tester.sql @@ -17,7 +17,7 @@ Copyright 2025 Darling Data, LLC https://www.erikdarling.com/ For support, head over to GitHub: -https://github.com/erikdarlingdata/DarlingData +https://code.erikdarling.com Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: diff --git a/Inline Financial Functions/EffectiveAnnualInterestRate.sql b/Inline Financial Functions/EffectiveAnnualInterestRate.sql index 05934eff..616aa5fb 100644 --- a/Inline Financial Functions/EffectiveAnnualInterestRate.sql +++ b/Inline Financial Functions/EffectiveAnnualInterestRate.sql @@ -20,7 +20,7 @@ AS RETURN /* For support, head over to GitHub: -https://github.com/erikdarlingdata/DarlingData +https://code.erikdarling.com */ SELECT Rate = diff --git a/Inline Financial Functions/FutureValue.sql b/Inline Financial Functions/FutureValue.sql index 1b37df3c..17a8fd0f 100644 --- a/Inline Financial Functions/FutureValue.sql +++ b/Inline Financial Functions/FutureValue.sql @@ -23,7 +23,7 @@ AS RETURN /* For support, head over to GitHub: -https://github.com/erikdarlingdata/DarlingData +https://code.erikdarling.com */ WITH pre AS ( diff --git a/Inline Financial Functions/InterestPayment.sql b/Inline Financial Functions/InterestPayment.sql index d7b7b78e..f3973642 100644 --- a/Inline Financial Functions/InterestPayment.sql +++ b/Inline Financial Functions/InterestPayment.sql @@ -23,7 +23,7 @@ RETURNS TABLE AS /* For support, head over to GitHub: -https://github.com/erikdarlingdata/DarlingData +https://code.erikdarling.com */ RETURN WITH pre AS diff --git a/Inline Financial Functions/NumberOfPayments.sql b/Inline Financial Functions/NumberOfPayments.sql index 4c65bbba..66a6a0fb 100644 --- a/Inline Financial Functions/NumberOfPayments.sql +++ b/Inline Financial Functions/NumberOfPayments.sql @@ -25,7 +25,7 @@ AS RETURN /* For support, head over to GitHub: -https://github.com/erikdarlingdata/DarlingData +https://code.erikdarling.com */ SELECT NumPayments = diff --git a/Inline Financial Functions/NumberOfYears.sql b/Inline Financial Functions/NumberOfYears.sql index 8b5431c4..65e69375 100644 --- a/Inline Financial Functions/NumberOfYears.sql +++ b/Inline Financial Functions/NumberOfYears.sql @@ -25,7 +25,7 @@ AS RETURN /* For support, head over to GitHub: -https://github.com/erikdarlingdata/DarlingData +https://code.erikdarling.com */ SELECT NumberOfYears = diff --git a/Inline Financial Functions/Payment.sql b/Inline Financial Functions/Payment.sql index 7b17318e..790ee7dc 100644 --- a/Inline Financial Functions/Payment.sql +++ b/Inline Financial Functions/Payment.sql @@ -23,7 +23,7 @@ AS RETURN /* For support, head over to GitHub: -https://github.com/erikdarlingdata/DarlingData +https://code.erikdarling.com */ WITH pre AS ( diff --git a/Inline Financial Functions/PrincipalPayment.sql b/Inline Financial Functions/PrincipalPayment.sql index 84aa8412..f03c00f5 100644 --- a/Inline Financial Functions/PrincipalPayment.sql +++ b/Inline Financial Functions/PrincipalPayment.sql @@ -24,7 +24,7 @@ AS RETURN /* For support, head over to GitHub: -https://github.com/erikdarlingdata/DarlingData +https://code.erikdarling.com */ SELECT PrincipalPayment = diff --git a/Install-All/DarlingData.sql b/Install-All/DarlingData.sql index 0624039d..31974e47 100644 --- a/Install-All/DarlingData.sql +++ b/Install-All/DarlingData.sql @@ -30,7 +30,7 @@ Copyright 2025 Darling Data, LLC https://www.erikdarling.com/ For support, head over to GitHub: -https://github.com/erikdarlingdata/DarlingData +https://code.erikdarling.com */ IF OBJECT_ID('dbo.sp_HealthParser') IS NULL @@ -2916,7 +2916,7 @@ EXECUTE sp_HumanEvents @debug = 1; For support, head over to GitHub: -https://github.com/erikdarlingdata/DarlingData +https://code.erikdarling.com */ @@ -7695,7 +7695,7 @@ EXECUTE sp_HumanEventsBlockViewer @debug = 1; For support, head over to GitHub: -https://github.com/erikdarlingdata/DarlingData +https://code.erikdarling.com */ IF OBJECT_ID('dbo.sp_HumanEventsBlockViewer') IS NULL @@ -9767,7 +9767,7 @@ SELECT check_id = -1, database_name = N'erikdarling.com', object_name = N'sp_HumanEventsBlockViewer version ' + CONVERT(nvarchar(30), @version) + N'.', - finding_group = N'https://github.com/erikdarlingdata/DarlingData', + finding_group = N'https://code.erikdarling.com', finding = N'blocking for period ' + CONVERT(nvarchar(30), @start_date_original, 126) + N' through ' + CONVERT(nvarchar(30), @end_date_original, 126) + N'.', 1; @@ -10598,7 +10598,7 @@ SELECT check_id = 2147483647, database_name = N'erikdarling.com', object_name = N'sp_HumanEventsBlockViewer version ' + CONVERT(nvarchar(30), @version) + N'.', - finding_group = N'https://github.com/erikdarlingdata/DarlingData', + finding_group = N'https://code.erikdarling.com', finding = N'thanks for using me!', 2147483647; @@ -12267,7 +12267,7 @@ EXECUTE sp_LogHunter @debug = 1; For support, head over to GitHub: -https://github.com/erikdarlingdata/DarlingData +https://code.erikdarling.com EXECUTE sp_LogHunter; @@ -12993,7 +12993,7 @@ EXECUTE sp_PressureDetector @debug = 1; For support, head over to GitHub: -https://github.com/erikdarlingdata/DarlingData +https://code.erikdarling.com */ @@ -16106,7 +16106,7 @@ EXECUTE sp_QuickieStore @troubleshoot_performance = 1; For support, head over to GitHub: -https://github.com/erikdarlingdata/DarlingData +https://code.erikdarling.com */ @@ -27237,7 +27237,7 @@ BEGIN all_done = 'https://www.erikdarling.com/', support = - 'https://github.com/erikdarlingdata/DarlingData', + 'https://code.erikdarling.com', help = 'EXECUTE sp_QuickieStore @help = 1;', problems = diff --git a/Stack Column Store Database/MakeBigStackCCS.sql b/Stack Column Store Database/MakeBigStackCCS.sql index 6f582e9e..b269a4fd 100644 --- a/Stack Column Store Database/MakeBigStackCCS.sql +++ b/Stack Column Store Database/MakeBigStackCCS.sql @@ -6,7 +6,7 @@ Copyright (c) 2025 Darling Data, LLC https://www.erikdarling.com/ For support, head over to GitHub: -https://github.com/erikdarlingdata/DarlingData +https://code.erikdarling.com Permission is hereby granted, free of charge, to any person obtaining a copy diff --git a/Stack Column Store Database/Script StackOverflowCS.sql b/Stack Column Store Database/Script StackOverflowCS.sql index 4acb6084..16ce00fa 100644 --- a/Stack Column Store Database/Script StackOverflowCS.sql +++ b/Stack Column Store Database/Script StackOverflowCS.sql @@ -6,7 +6,7 @@ Copyright (c) 2025 Darling Data, LLC https://www.erikdarling.com/ For support, head over to GitHub: -https://github.com/erikdarlingdata/DarlingData +https://code.erikdarling.com Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/String Functions/get_letters.sql b/String Functions/get_letters.sql index 3c922c08..aed18558 100644 --- a/String Functions/get_letters.sql +++ b/String Functions/get_letters.sql @@ -11,7 +11,7 @@ WITH SCHEMABINDING AS /* For support: -https://github.com/erikdarlingdata/DarlingData +https://code.erikdarling.com Copyright 2025 Darling Data, LLC https://erikdarling.com @@ -79,7 +79,7 @@ WITH SCHEMABINDING AS /* For support: -https://github.com/erikdarlingdata/DarlingData +https://code.erikdarling.com Copyright 2025 Darling Data, LLC https://erikdarling.com diff --git a/String Functions/get_numbers.sql b/String Functions/get_numbers.sql index fe91bfbc..0cff51d9 100644 --- a/String Functions/get_numbers.sql +++ b/String Functions/get_numbers.sql @@ -11,7 +11,7 @@ WITH SCHEMABINDING AS /* For support: -https://github.com/erikdarlingdata/DarlingData +https://code.erikdarling.com Copyright 2025 Darling Data, LLC https://erikdarling.com @@ -80,7 +80,7 @@ WITH SCHEMABINDING AS /* For support: -https://github.com/erikdarlingdata/DarlingData +https://code.erikdarling.com Copyright 2025 Darling Data, LLC https://erikdarling.com diff --git a/String Functions/strip_characters.sql b/String Functions/strip_characters.sql index a6fb4e0b..93eb8983 100644 --- a/String Functions/strip_characters.sql +++ b/String Functions/strip_characters.sql @@ -12,7 +12,7 @@ WITH SCHEMABINDING AS /* For support: -https://github.com/erikdarlingdata/DarlingData +https://code.erikdarling.com Copyright 2025 Darling Data, LLC https://erikdarling.com @@ -82,7 +82,7 @@ WITH SCHEMABINDING AS /* For support: -https://github.com/erikdarlingdata/DarlingData +https://code.erikdarling.com Copyright 2025 Darling Data, LLC https://erikdarling.com diff --git a/sp_HealthParser/sp_HealthParser.sql b/sp_HealthParser/sp_HealthParser.sql index 4b41ccba..bc89599a 100644 --- a/sp_HealthParser/sp_HealthParser.sql +++ b/sp_HealthParser/sp_HealthParser.sql @@ -29,7 +29,7 @@ Copyright 2025 Darling Data, LLC https://www.erikdarling.com/ For support, head over to GitHub: -https://github.com/erikdarlingdata/DarlingData +https://code.erikdarling.com */ IF OBJECT_ID(N'dbo.sp_HealthParser', N'P') IS NULL diff --git a/sp_HumanEvents/sp_Human Events Agent Job Example.sql b/sp_HumanEvents/sp_Human Events Agent Job Example.sql index d3ea4070..a811dc50 100644 --- a/sp_HumanEvents/sp_Human Events Agent Job Example.sql +++ b/sp_HumanEvents/sp_Human Events Agent Job Example.sql @@ -11,7 +11,7 @@ Copyright 2025 Darling Data, LLC https://www.erikdarling.com/ For support, head over to GitHub: -https://github.com/erikdarlingdata/DarlingData +https://code.erikdarling.com MIT License diff --git a/sp_HumanEvents/sp_HumanEvents.sql b/sp_HumanEvents/sp_HumanEvents.sql index 489f07c0..2eeddbeb 100644 --- a/sp_HumanEvents/sp_HumanEvents.sql +++ b/sp_HumanEvents/sp_HumanEvents.sql @@ -36,7 +36,7 @@ EXECUTE sp_HumanEvents @debug = 1; For support, head over to GitHub: -https://github.com/erikdarlingdata/DarlingData +https://code.erikdarling.com */ diff --git a/sp_HumanEvents/sp_HumanEventsBlockViewer.sql b/sp_HumanEvents/sp_HumanEventsBlockViewer.sql index 91b3908c..faca0bb1 100644 --- a/sp_HumanEvents/sp_HumanEventsBlockViewer.sql +++ b/sp_HumanEvents/sp_HumanEventsBlockViewer.sql @@ -50,7 +50,7 @@ EXECUTE sp_HumanEventsBlockViewer @debug = 1; For support, head over to GitHub: -https://github.com/erikdarlingdata/DarlingData +https://code.erikdarling.com */ IF OBJECT_ID(N'dbo.sp_HumanEventsBlockViewer', N'P') IS NULL @@ -2780,7 +2780,7 @@ BEGIN check_id = -1, database_name = N'erikdarling.com', object_name = N'sp_HumanEventsBlockViewer version ' + CONVERT(nvarchar(30), @version) + N'.', - finding_group = N'https://github.com/erikdarlingdata/DarlingData', + finding_group = N'https://code.erikdarling.com', finding = N'blocking for period ' + CONVERT(nvarchar(30), @start_date_original, 126) + N' through ' + CONVERT(nvarchar(30), @end_date_original, 126) + N'.', 1; @@ -3611,7 +3611,7 @@ BEGIN check_id = 2147483647, database_name = N'erikdarling.com', object_name = N'sp_HumanEventsBlockViewer version ' + CONVERT(nvarchar(30), @version) + N'.', - finding_group = N'https://github.com/erikdarlingdata/DarlingData', + finding_group = N'https://code.erikdarling.com', finding = N'thanks for using me!', 2147483647; diff --git a/sp_LogHunter/sp_LogHunter.sql b/sp_LogHunter/sp_LogHunter.sql index bf99cad6..4f0035ec 100644 --- a/sp_LogHunter/sp_LogHunter.sql +++ b/sp_LogHunter/sp_LogHunter.sql @@ -36,7 +36,7 @@ EXECUTE sp_LogHunter @debug = 1; For support, head over to GitHub: -https://github.com/erikdarlingdata/DarlingData +https://code.erikdarling.com EXECUTE sp_LogHunter; diff --git a/sp_PressureDetector/sp_PressureDetector.sql b/sp_PressureDetector/sp_PressureDetector.sql index 8a0709bb..caa64fe2 100644 --- a/sp_PressureDetector/sp_PressureDetector.sql +++ b/sp_PressureDetector/sp_PressureDetector.sql @@ -37,7 +37,7 @@ EXECUTE sp_PressureDetector @debug = 1; For support, head over to GitHub: -https://github.com/erikdarlingdata/DarlingData +https://code.erikdarling.com */ diff --git a/sp_QuickieStore/sp_QuickieStore.sql b/sp_QuickieStore/sp_QuickieStore.sql index 456b1605..3b1ab85f 100644 --- a/sp_QuickieStore/sp_QuickieStore.sql +++ b/sp_QuickieStore/sp_QuickieStore.sql @@ -41,7 +41,7 @@ EXECUTE sp_QuickieStore @troubleshoot_performance = 1; For support, head over to GitHub: -https://github.com/erikdarlingdata/DarlingData +https://code.erikdarling.com */ @@ -9965,7 +9965,7 @@ BEGIN N'None' ), support = - 'https://github.com/erikdarlingdata/DarlingData', + 'https://code.erikdarling.com', help = 'EXECUTE sp_QuickieStore @help = 1;', problems = From d80a79de50c212eeadb4641ac4ba10d10380bd19 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Wed, 19 Mar 2025 12:07:47 -0400 Subject: [PATCH 246/246] Update sp_HealthParser.sql fix output window stuff when logging to a table --- sp_HealthParser/sp_HealthParser.sql | 456 +++++++++++++++------------- 1 file changed, 239 insertions(+), 217 deletions(-) diff --git a/sp_HealthParser/sp_HealthParser.sql b/sp_HealthParser/sp_HealthParser.sql index bc89599a..5cfca6da 100644 --- a/sp_HealthParser/sp_HealthParser.sql +++ b/sp_HealthParser/sp_HealthParser.sql @@ -1755,26 +1755,28 @@ AND ca.utc_timestamp < @end_date'; FROM #waits_queries AS wq ) BEGIN - /* No results logic, only return if not logging */ - SELECT - finding = - CASE - WHEN @what_to_check NOT IN ('all', 'waits') - THEN 'waits skipped, @what_to_check set to ' + - @what_to_check - WHEN @what_to_check IN ('all', 'waits') - THEN 'no queries with significant waits found between ' + - RTRIM(CONVERT(date, @start_date)) + - ' and ' + - RTRIM(CONVERT(date, @end_date)) + - ' with a minimum duration of ' + - RTRIM(@wait_duration_ms) + - '.' - ELSE 'no queries with significant waits found!' - END - WHERE @log_to_table = 0; - - RAISERROR('No queries with significant waits found', 0, 0) WITH NOWAIT; + IF @log_to_table = 0 + BEGIN + /* No results logic, only return if not logging */ + SELECT + finding = + CASE + WHEN @what_to_check NOT IN ('all', 'waits') + THEN 'waits skipped, @what_to_check set to ' + + @what_to_check + WHEN @what_to_check IN ('all', 'waits') + THEN 'no queries with significant waits found between ' + + RTRIM(CONVERT(date, @start_date)) + + ' and ' + + RTRIM(CONVERT(date, @end_date)) + + ' with a minimum duration of ' + + RTRIM(@wait_duration_ms) + + '.' + ELSE 'no queries with significant waits found!' + END; + + RAISERROR('No queries with significant waits found', 0, 0) WITH NOWAIT; + END; END; ELSE BEGIN @@ -2006,24 +2008,26 @@ AND ca.utc_timestamp < @end_date'; FROM #tc AS t ) BEGIN - /* No results logic, only return if not logging */ - SELECT - finding = - CASE - WHEN @what_to_check NOT IN ('all', 'waits') - THEN 'waits skipped, @what_to_check set to ' + - @what_to_check - WHEN @what_to_check IN ('all', 'waits') - THEN 'no significant waits found between ' + - RTRIM(CONVERT(date, @start_date)) + - ' and ' + - RTRIM(CONVERT(date, @end_date)) + - '.' - ELSE 'no significant waits found!' - END - WHERE @log_to_table = 0; - - RAISERROR('No waits by count found', 0, 0) WITH NOWAIT; + IF @log_to_table = 0 + BEGIN + /* No results logic, only return if not logging */ + SELECT + finding = + CASE + WHEN @what_to_check NOT IN ('all', 'waits') + THEN 'waits skipped, @what_to_check set to ' + + @what_to_check + WHEN @what_to_check IN ('all', 'waits') + THEN 'no significant waits found between ' + + RTRIM(CONVERT(date, @start_date)) + + ' and ' + + RTRIM(CONVERT(date, @end_date)) + + '.' + ELSE 'no significant waits found!' + END + + RAISERROR('No waits by count found', 0, 0) WITH NOWAIT; + END; END; ELSE BEGIN @@ -2263,26 +2267,28 @@ AND ca.utc_timestamp < @end_date'; FROM #td AS t ) BEGIN - /* No results logic, only return if not logging */ - SELECT - finding = - CASE - WHEN @what_to_check NOT IN ('all', 'waits') - THEN 'waits skipped, @what_to_check set to ' + - @what_to_check - WHEN @what_to_check IN ('all', 'waits') - THEN 'no significant waits found between ' + - RTRIM(CONVERT(date, @start_date)) + - ' and ' + - RTRIM(CONVERT(date, @end_date)) + - ' with a minimum average duration of ' + - RTRIM(@wait_duration_ms) + - '.' - ELSE 'no significant waits found!' - END - WHERE @log_to_table = 0; - - RAISERROR('No waits by duration', 0, 0) WITH NOWAIT; + IF @log_to_table = 0 + BEGIN + /* No results logic, only return if not logging */ + SELECT + finding = + CASE + WHEN @what_to_check NOT IN ('all', 'waits') + THEN 'waits skipped, @what_to_check set to ' + + @what_to_check + WHEN @what_to_check IN ('all', 'waits') + THEN 'no significant waits found between ' + + RTRIM(CONVERT(date, @start_date)) + + ' and ' + + RTRIM(CONVERT(date, @end_date)) + + ' with a minimum average duration of ' + + RTRIM(@wait_duration_ms) + + '.' + ELSE 'no significant waits found!' + END + + RAISERROR('No waits by duration', 0, 0) WITH NOWAIT; + END; END; ELSE BEGIN @@ -2505,11 +2511,12 @@ AND ca.utc_timestamp < @end_date'; i.intervalLongIos, i.totalLongIos, longestPendingRequests_duration_ms = - ISNULL(SUM(i.longestPendingRequests_duration_ms), 0), + SUM(i.longestPendingRequests_duration_ms), longestPendingRequests_filePath = ISNULL(i.longestPendingRequests_filePath, 'N/A') INTO #i FROM #io AS i + WHERE i.longestPendingRequests_duration_ms IS NOT NULL GROUP BY i.event_time, i.state, @@ -2527,26 +2534,27 @@ AND ca.utc_timestamp < @end_date'; FROM #i AS i ) BEGIN - /* No results logic, only return if not logging */ - SELECT - finding = - CASE - WHEN @what_to_check NOT IN ('all', 'disk') - THEN 'disk skipped, @what_to_check set to ' + - @what_to_check - WHEN @what_to_check IN ('all', 'disk') - THEN 'no io issues found between ' + - RTRIM(CONVERT(date, @start_date)) + - ' and ' + - RTRIM(CONVERT(date, @end_date)) + - ' with @warnings_only set to ' + - RTRIM(@warnings_only) + - '.' - ELSE 'no io issues found!' - END - WHERE @log_to_table = 0; - - RAISERROR('No io data found', 0, 0) WITH NOWAIT; + IF @log_to_table = 0 + BEGIN + /* No results logic, only return if not logging */ + SELECT + finding = + CASE + WHEN @what_to_check NOT IN ('all', 'disk') + THEN 'disk skipped, @what_to_check set to ' + + @what_to_check + WHEN @what_to_check IN ('all', 'disk') + THEN 'no io issues found between ' + + RTRIM(CONVERT(date, @start_date)) + + ' and ' + + RTRIM(CONVERT(date, @end_date)) + + ' with @warnings_only set to ' + + RTRIM(@warnings_only) + + '.' + ELSE 'no io issues found!' + END + RAISERROR('No io data found', 0, 0) WITH NOWAIT; + END; END; ELSE BEGIN @@ -2725,26 +2733,28 @@ END; FROM #scheduler_details AS sd ) BEGIN - /* No results logic, only return if not logging */ - SELECT - finding = - CASE - WHEN @what_to_check NOT IN ('all', 'cpu') - THEN 'cpu skipped, @what_to_check set to ' + - @what_to_check - WHEN @what_to_check IN ('all', 'cpu') - THEN 'no cpu issues found between ' + - RTRIM(CONVERT(date, @start_date)) + - ' and ' + - RTRIM(CONVERT(date, @end_date)) + - ' with @warnings_only set to ' + - RTRIM(@warnings_only) + - '.' - ELSE 'no cpu issues found!' - END - WHERE @log_to_table = 0; - - RAISERROR('No scheduler data found', 0, 0) WITH NOWAIT; + IF @log_to_table = 0 + BEGIN + /* No results logic, only return if not logging */ + SELECT + finding = + CASE + WHEN @what_to_check NOT IN ('all', 'cpu') + THEN 'cpu skipped, @what_to_check set to ' + + @what_to_check + WHEN @what_to_check IN ('all', 'cpu') + THEN 'no cpu issues found between ' + + RTRIM(CONVERT(date, @start_date)) + + ' and ' + + RTRIM(CONVERT(date, @end_date)) + + ' with @warnings_only set to ' + + RTRIM(@warnings_only) + + '.' + ELSE 'no cpu issues found!' + END + + RAISERROR('No scheduler data found', 0, 0) WITH NOWAIT; + END; END; ELSE BEGIN @@ -2930,26 +2940,28 @@ END; FROM #memory AS m ) BEGIN - /* No results logic, only return if not logging */ - SELECT - finding = - CASE - WHEN @what_to_check NOT IN ('all', 'memory') - THEN 'memory skipped, @what_to_check set to ' + - @what_to_check - WHEN @what_to_check IN ('all', 'memory') - THEN 'no memory issues found between ' + - RTRIM(CONVERT(date, @start_date)) + - ' and ' + - RTRIM(CONVERT(date, @end_date)) + - ' with @warnings_only set to ' + - RTRIM(@warnings_only) + - '.' - ELSE 'no memory issues found!' - END - WHERE @log_to_table = 0; - - RAISERROR('No memory condition data found', 0, 0) WITH NOWAIT; + IF @log_to_table = 0 + BEGIN + /* No results logic, only return if not logging */ + SELECT + finding = + CASE + WHEN @what_to_check NOT IN ('all', 'memory') + THEN 'memory skipped, @what_to_check set to ' + + @what_to_check + WHEN @what_to_check IN ('all', 'memory') + THEN 'no memory issues found between ' + + RTRIM(CONVERT(date, @start_date)) + + ' and ' + + RTRIM(CONVERT(date, @end_date)) + + ' with @warnings_only set to ' + + RTRIM(@warnings_only) + + '.' + ELSE 'no memory issues found!' + END + + RAISERROR('No memory condition data found', 0, 0) WITH NOWAIT; + END; END; ELSE BEGIN @@ -3155,26 +3167,28 @@ END; FROM #memory_broker_info AS mbi ) BEGIN - /* No results logic, only return if not logging */ - SELECT - finding = - CASE - WHEN @what_to_check NOT IN ('all', 'memory') - THEN 'memory broker skipped, @what_to_check set to ' + - @what_to_check - WHEN @what_to_check IN ('all', 'memory') - THEN 'no memory pressure events found between ' + - RTRIM(CONVERT(date, @start_date)) + - ' and ' + - RTRIM(CONVERT(date, @end_date)) + - ' with @warnings_only set to ' + - RTRIM(@warnings_only) + - '.' - ELSE 'no memory pressure events found!' - END - WHERE @log_to_table = 0; - - RAISERROR('No memory broker data found', 0, 0) WITH NOWAIT; + IF @log_to_table = 0 + BEGIN + /* No results logic, only return if not logging */ + SELECT + finding = + CASE + WHEN @what_to_check NOT IN ('all', 'memory') + THEN 'memory broker skipped, @what_to_check set to ' + + @what_to_check + WHEN @what_to_check IN ('all', 'memory') + THEN 'no memory pressure events found between ' + + RTRIM(CONVERT(date, @start_date)) + + ' and ' + + RTRIM(CONVERT(date, @end_date)) + + ' with @warnings_only set to ' + + RTRIM(@warnings_only) + + '.' + ELSE 'no memory pressure events found!' + END + + RAISERROR('No memory broker data found', 0, 0) WITH NOWAIT; + END; END; ELSE BEGIN @@ -3360,24 +3374,26 @@ END; FROM #memory_node_oom_info AS mnoi ) BEGIN - /* No results logic, only return if not logging */ - SELECT - finding = - CASE - WHEN @what_to_check NOT IN ('all', 'memory') - THEN 'memory node OOM skipped, @what_to_check set to ' + - @what_to_check - WHEN @what_to_check IN ('all', 'memory') - THEN 'no memory node OOM events found between ' + - RTRIM(CONVERT(date, @start_date)) + - ' and ' + - RTRIM(CONVERT(date, @end_date)) + - '.' - ELSE 'no memory node OOM events found!' - END - WHERE @log_to_table = 0; - - RAISERROR('No memory oom data found', 0, 0) WITH NOWAIT; + IF @log_to_table = 0 + BEGIN + /* No results logic, only return if not logging */ + SELECT + finding = + CASE + WHEN @what_to_check NOT IN ('all', 'memory') + THEN 'memory node OOM skipped, @what_to_check set to ' + + @what_to_check + WHEN @what_to_check IN ('all', 'memory') + THEN 'no memory node OOM events found between ' + + RTRIM(CONVERT(date, @start_date)) + + ' and ' + + RTRIM(CONVERT(date, @end_date)) + + '.' + ELSE 'no memory node OOM events found!' + END + + RAISERROR('No memory oom data found', 0, 0) WITH NOWAIT; + END; END; ELSE BEGIN @@ -3642,26 +3658,28 @@ END; FROM #health AS h ) BEGIN - /* No results logic, only return if not logging */ - SELECT - finding = - CASE - WHEN @what_to_check NOT IN ('all', 'system') - THEN 'system health skipped, @what_to_check set to ' + - @what_to_check - WHEN @what_to_check IN ('all', 'system') - THEN 'no system health issues found between ' + - RTRIM(CONVERT(date, @start_date)) + - ' and ' + - RTRIM(CONVERT(date, @end_date)) + - ' with @warnings_only set to ' + - RTRIM(@warnings_only) + - '.' - ELSE 'no system health issues found!' - END - WHERE @log_to_table = 0; - - RAISERROR('No system health data found', 0, 0) WITH NOWAIT; + IF @log_to_table = 0 + BEGIN + /* No results logic, only return if not logging */ + SELECT + finding = + CASE + WHEN @what_to_check NOT IN ('all', 'system') + THEN 'system health skipped, @what_to_check set to ' + + @what_to_check + WHEN @what_to_check IN ('all', 'system') + THEN 'no system health issues found between ' + + RTRIM(CONVERT(date, @start_date)) + + ' and ' + + RTRIM(CONVERT(date, @end_date)) + + ' with @warnings_only set to ' + + RTRIM(@warnings_only) + + '.' + ELSE 'no system health issues found!' + END + + RAISERROR('No system health data found', 0, 0) WITH NOWAIT; + END; END; ELSE BEGIN @@ -3835,26 +3853,28 @@ END; FROM #scheduler_issues AS si ) BEGIN - /* No results logic, only return if not logging */ - SELECT - finding = - CASE - WHEN @what_to_check NOT IN ('all', 'system', 'cpu') - THEN 'scheduler monitoring skipped, @what_to_check set to ' + - @what_to_check - WHEN @what_to_check IN ('all', 'system', 'cpu') - THEN 'no scheduler issues found between ' + - RTRIM(CONVERT(date, @start_date)) + - ' and ' + - RTRIM(CONVERT(date, @end_date)) + - ' with @warnings_only set to ' + - RTRIM(@warnings_only) + - '.' - ELSE 'no scheduler issues found!' - END - WHERE @log_to_table = 0; - - RAISERROR('No scheduler issues data found', 0, 0) WITH NOWAIT; + IF @log_to_table = 0 + BEGIN + /* No results logic, only return if not logging */ + SELECT + finding = + CASE + WHEN @what_to_check NOT IN ('all', 'system', 'cpu') + THEN 'scheduler monitoring skipped, @what_to_check set to ' + + @what_to_check + WHEN @what_to_check IN ('all', 'system', 'cpu') + THEN 'no scheduler issues found between ' + + RTRIM(CONVERT(date, @start_date)) + + ' and ' + + RTRIM(CONVERT(date, @end_date)) + + ' with @warnings_only set to ' + + RTRIM(@warnings_only) + + '.' + ELSE 'no scheduler issues found!' + END + + RAISERROR('No scheduler issues data found', 0, 0) WITH NOWAIT; + END; END; ELSE BEGIN @@ -4059,26 +4079,28 @@ END; FROM #error_info AS ei ) BEGIN - /* No results logic, only return if not logging */ - SELECT - finding = - CASE - WHEN @what_to_check NOT IN ('all', 'system') - THEN 'error reporting skipped, @what_to_check set to ' + - @what_to_check - WHEN @what_to_check IN ('all', 'system') - THEN 'no severe errors found between ' + - RTRIM(CONVERT(date, @start_date)) + - ' and ' + - RTRIM(CONVERT(date, @end_date)) + - ' with @warnings_only set to ' + - RTRIM(@warnings_only) + - '.' - ELSE 'no severe errors found!' - END - WHERE @log_to_table = 0; - - RAISERROR('No error data found', 0, 0) WITH NOWAIT; + IF @log_to_table = 0 + BEGIN + /* No results logic, only return if not logging */ + SELECT + finding = + CASE + WHEN @what_to_check NOT IN ('all', 'system') + THEN 'error reporting skipped, @what_to_check set to ' + + @what_to_check + WHEN @what_to_check IN ('all', 'system') + THEN 'no severe errors found between ' + + RTRIM(CONVERT(date, @start_date)) + + ' and ' + + RTRIM(CONVERT(date, @end_date)) + + ' with @warnings_only set to ' + + RTRIM(@warnings_only) + + '.' + ELSE 'no severe errors found!' + END + + RAISERROR('No error data found', 0, 0) WITH NOWAIT; + END; END; ELSE BEGIN