Merge pull request #88 from erikdarlingdata/feature/86-collector-chain-triggers

Chain-trigger parsers, optimize XML collection, tune schedule

Author: erikdarlingdata

install/04_create_schedule_table.sql

@@ -43,39 +43,39 @@ SELECT
 FROM
 (
     VALUES
-    (N'wait_stats_collector', 1, 5, 2, 30, N'Wait statistics - high frequency for trending'),
+    (N'wait_stats_collector', 1, 1, 2, 30, N'Wait statistics - high frequency for trending'),
     (N'query_stats_collector', 1, 2, 5, 30, N'Plan cache queries - recent activity focused'),
-    (N'memory_stats_collector', 1, 5, 2, 30, N'Memory pressure monitoring'),
-    (N'memory_pressure_events_collector', 1, 5, 5, 30, N'Ring buffer system events'),
+    (N'memory_stats_collector', 1, 1, 2, 30, N'Memory pressure monitoring'),
+    (N'memory_pressure_events_collector', 1, 1, 5, 30, N'Ring buffer system events'),
     (N'system_health_collector', 1, 5, 10, 30, N'System health extended events via sp_HealthParser'),
-    (N'blocked_process_xml_collector', 1, 5, 2, 30, N'Fast blocked process XML collection'),
-    (N'deadlock_xml_collector', 1, 5, 3, 30, N'Fast deadlock XML collection'),
-    (N'process_blocked_process_xml', 1, 5, 5, 30, N'Parse blocked process XML via sp_HumanEventsBlockViewer'),
-    (N'blocking_deadlock_analyzer', 1, 5, 5, 30, N'Analyze blocking/deadlock trends and alert on significant increases'),
-    (N'process_deadlock_xml', 1, 5, 5, 30, N'Parse deadlock XML via sp_BlitzLock'),
-    (N'query_store_collector', 1, 5, 10, 30, N'Query Store data collection'),
-    (N'procedure_stats_collector', 1, 5, 10, 30, N'Procedure/trigger/function statistics'),
+    (N'blocked_process_xml_collector', 1, 1, 2, 30, N'Fast blocked process XML collection (chain-triggers parser + analyzer)'),
+    (N'deadlock_xml_collector', 1, 1, 3, 30, N'Fast deadlock XML collection (chain-triggers parser + analyzer)'),
+    (N'process_blocked_process_xml', 1, 5, 5, 30, N'Parse blocked process XML via sp_HumanEventsBlockViewer (also chain-triggered)'),
+    (N'blocking_deadlock_analyzer', 1, 5, 5, 30, N'Analyze blocking/deadlock trends (also chain-triggered)'),
+    (N'process_deadlock_xml', 1, 5, 5, 30, N'Parse deadlock XML via sp_BlitzLock (also chain-triggered)'),
+    (N'query_store_collector', 1, 2, 10, 30, N'Query Store data collection'),
+    (N'procedure_stats_collector', 1, 2, 10, 30, N'Procedure/trigger/function statistics'),
     (N'query_snapshots_collector', 1, 1, 2, 10, N'Currently executing queries with session wait stats (every minute - high frequency)'),
-    (N'file_io_stats_collector', 1, 5, 2, 30, N'File I/O statistics from dm_io_virtual_file_stats'),
-    (N'memory_grant_stats_collector', 1, 5, 2, 30, N'Memory grant semaphore pressure monitoring'),
-    (N'cpu_scheduler_stats_collector', 1, 5, 2, 30, N'CPU scheduler and workload group statistics'),
+    (N'file_io_stats_collector', 1, 1, 2, 30, N'File I/O statistics from dm_io_virtual_file_stats'),
+    (N'memory_grant_stats_collector', 1, 1, 2, 30, N'Memory grant semaphore pressure monitoring'),
+    (N'cpu_scheduler_stats_collector', 1, 1, 2, 30, N'CPU scheduler and workload group statistics'),
     (N'memory_clerks_stats_collector', 1, 5, 3, 30, N'Memory clerk allocation tracking'),
     (N'perfmon_stats_collector', 1, 5, 2, 30, N'Performance counter statistics from dm_os_performance_counters'),
-    (N'cpu_utilization_stats_collector', 1, 5, 2, 30, N'CPU utilization from ring buffer (SQL vs other processes)'),
+    (N'cpu_utilization_stats_collector', 1, 1, 2, 30, N'CPU utilization from ring buffer (SQL vs other processes)'),
     (N'trace_management_collector', 1, 1440, 5, 30, N'SQL Trace management for long-running queries'),
-    (N'trace_analysis_collector', 1, 5, 5, 30, N'Process trace files into analysis tables'),
+    (N'trace_analysis_collector', 1, 2, 5, 30, N'Process trace files into analysis tables'),
     (N'default_trace_collector', 1, 5, 3, 30, N'System events from default trace (memory, autogrow, config changes)'),
     (N'server_configuration_collector', 1, 1440, 5, 30, N'Server-level configuration settings and trace flags (daily collection)'),
     (N'database_configuration_collector', 1, 1440, 10, 30, N'Database-level configuration settings including scoped configs (daily collection)'),
     (N'configuration_issues_analyzer', 1, 1, 2, 30, N'Analyze configuration for issues: database config (Query Store, auto shrink/close), memory/CPU pressure warnings, server config (MAXDOP, priority boost)'),
-    (N'latch_stats_collector', 1, 5, 3, 30, N'Latch contention statistics - internal synchronization object waits'),
-    (N'spinlock_stats_collector', 1, 5, 3, 30, N'Spinlock contention statistics - lightweight synchronization primitive collisions'),
-    (N'tempdb_stats_collector', 1, 5, 2, 30, N'TempDB space usage - version store, user/internal objects, allocation contention'),
+    (N'latch_stats_collector', 1, 1, 3, 30, N'Latch contention statistics - internal synchronization object waits'),
+    (N'spinlock_stats_collector', 1, 1, 3, 30, N'Spinlock contention statistics - lightweight synchronization primitive collisions'),
+    (N'tempdb_stats_collector', 1, 1, 2, 30, N'TempDB space usage - version store, user/internal objects, allocation contention'),
     (N'plan_cache_stats_collector', 1, 5, 5, 30, N'Plan cache composition statistics - single-use plans and plan cache bloat detection'),
-    (N'session_stats_collector', 1, 5, 2, 30, N'Session and connection statistics - connection leaks and application patterns'),
-    (N'waiting_tasks_collector', 1, 5, 2, 30, N'Currently waiting tasks - blocking chains and wait analysis'),
+    (N'session_stats_collector', 1, 1, 2, 30, N'Session and connection statistics - connection leaks and application patterns'),
+    (N'waiting_tasks_collector', 1, 1, 2, 30, N'Currently waiting tasks - blocking chains and wait analysis'),
     (N'session_wait_stats_collector', 1, 1, 2, 30, N'Per-session wait statistics - correlates waits with specific sessions/queries (requires SQL Server 2016 SP1+)'),
-    (N'running_jobs_collector', 1, 5, 2, 7, N'Currently running SQL Agent jobs with historical duration comparison')
+    (N'running_jobs_collector', 1, 1, 2, 7, N'Currently running SQL Agent jobs with historical duration comparison')
 ) AS v (collector_name, enabled, frequency_minutes, max_duration_minutes, retention_days, description)
 WHERE NOT EXISTS
 (

install/22_collect_blocked_processes.sql

@@ -162,89 +162,103 @@ BEGIN
         IF @is_azure_sql_db = 1
         BEGIN
             SET @sql = N'
-            WITH
-                ring_buffer_xml AS
+            DECLARE
+                @ring_buffer TABLE
             (
-                SELECT
-                    target_data = TRY_CAST(xet.target_data AS xml)
-                FROM sys.dm_xe_database_session_targets AS xet
-                JOIN sys.dm_xe_database_sessions AS xes
-                  ON xes.address = xet.event_session_address
-                WHERE xes.name = @session_name
-                AND   xet.target_name = N''ring_buffer''
-            ),
-                recent_events AS
+                ring_buffer xml NOT NULL
+            );
+
+            INSERT
+                @ring_buffer
             (
-                SELECT TOP (1000)
-                    event_time = evt.value(''(@timestamp)[1]'', ''datetime2(7)''),
-                    blocked_process_xml = evt.query(''.'')
-                FROM ring_buffer_xml AS rb
-                CROSS APPLY rb.target_data.nodes(''RingBufferTarget/event[@name="blocked_process_report"]'') AS q(evt)
-                WHERE evt.value(''(@timestamp)[1]'', ''datetime2(7)'') >= @cutoff_time
-                ORDER BY
-                    evt.value(''(@timestamp)[1]'', ''datetime2(7)'') DESC
+                ring_buffer
             )
+            SELECT
+                ring_xml = TRY_CAST(xet.target_data AS xml)
+            FROM sys.dm_xe_database_session_targets AS xet
+            JOIN sys.dm_xe_database_sessions AS xes
+              ON xes.address = xet.event_session_address
+            WHERE xes.name = @session_name
+            AND   xet.target_name = N''ring_buffer''
+            OPTION(RECOMPILE);
+
             INSERT INTO
                 collect.blocked_process_xml
             (
                 event_time,
                 blocked_process_xml
             )
-            SELECT
-                re.event_time,
-                re.blocked_process_xml
-            FROM recent_events AS re
-            WHERE NOT EXISTS
+            SELECT TOP (1000)
+                event_time = evt.value(''(@timestamp)[1]'', ''datetime2(7)''),
+                blocked_process_xml = evt.query(''.'')
+            FROM
+            (
+                SELECT
+                    rb.ring_buffer
+                FROM @ring_buffer AS rb
+            ) AS rb
+            CROSS APPLY rb.ring_buffer.nodes(''RingBufferTarget/event[@name="blocked_process_report"]'') AS q(evt)
+            WHERE evt.value(''(@timestamp)[1]'', ''datetime2(7)'') >= @cutoff_time
+            AND NOT EXISTS
             (
                 SELECT
                     1/0
                 FROM collect.blocked_process_xml AS bx
-                WHERE bx.event_time = re.event_time
+                WHERE bx.event_time = evt.value(''(@timestamp)[1]'', ''datetime2(7)'')
             )
+            ORDER BY
+                evt.value(''(@timestamp)[1]'', ''datetime2(7)'') DESC
             OPTION(RECOMPILE);';
         END;
         ELSE
         BEGIN
             SET @sql = N'
-            WITH
-                ring_buffer_xml AS
+            DECLARE
+                @ring_buffer TABLE
             (
-                SELECT
-                    target_data = TRY_CAST(xet.target_data AS xml)
-                FROM sys.dm_xe_session_targets AS xet
-                JOIN sys.dm_xe_sessions AS xes
-                  ON xes.address = xet.event_session_address
-                WHERE xes.name = @session_name
-                AND   xet.target_name = N''ring_buffer''
-            ),
-                recent_events AS
+                ring_buffer xml NOT NULL
+            );
+
+            INSERT
+                @ring_buffer
             (
-                SELECT TOP (1000)
-                    event_time = evt.value(''(@timestamp)[1]'', ''datetime2(7)''),
-                    blocked_process_xml = evt.query(''.'')
-                FROM ring_buffer_xml AS rb
-                CROSS APPLY rb.target_data.nodes(''RingBufferTarget/event[@name="blocked_process_report"]'') AS q(evt)
-                WHERE evt.value(''(@timestamp)[1]'', ''datetime2(7)'') >= @cutoff_time
-                ORDER BY
-                    evt.value(''(@timestamp)[1]'', ''datetime2(7)'') DESC
+                ring_buffer
             )
+            SELECT
+                ring_xml = TRY_CAST(xet.target_data AS xml)
+            FROM sys.dm_xe_session_targets AS xet
+            JOIN sys.dm_xe_sessions AS xes
+              ON xes.address = xet.event_session_address
+            WHERE xes.name = @session_name
+            AND   xet.target_name = N''ring_buffer''
+            OPTION(RECOMPILE);
+
             INSERT INTO
                 collect.blocked_process_xml
             (
                 event_time,
                 blocked_process_xml
             )
-            SELECT
-                re.event_time,
-                re.blocked_process_xml
-            FROM recent_events AS re
-            WHERE NOT EXISTS
+            SELECT TOP (1000)
+                event_time = evt.value(''(@timestamp)[1]'', ''datetime2(7)''),
+                blocked_process_xml = evt.query(''.'')
+            FROM
+            (
+                SELECT
+                    rb.ring_buffer
+                FROM @ring_buffer AS rb
+            ) AS rb
+            CROSS APPLY rb.ring_buffer.nodes(''RingBufferTarget/event[@name="blocked_process_report"]'') AS q(evt)
+            WHERE evt.value(''(@timestamp)[1]'', ''datetime2(7)'') >= @cutoff_time
+            AND NOT EXISTS
             (
                 SELECT
                     1/0
                 FROM collect.blocked_process_xml AS bx
-                WHERE bx.event_time = re.event_time
+                WHERE bx.event_time = evt.value(''(@timestamp)[1]'', ''datetime2(7)'')
             )
+            ORDER BY
+                evt.value(''(@timestamp)[1]'', ''datetime2(7)'') DESC
             OPTION(RECOMPILE);';
         END;
 
@@ -303,6 +317,64 @@ BEGIN
 
         COMMIT TRANSACTION;
 
+        /*
+        Chain-trigger: when new blocked process XML is found, immediately
+        parse it and run the analyzer instead of waiting for their next
+        scheduled runs. This eliminates up to 10 minutes of pipeline latency.
+        Parser/analyzer errors are logged but do not fail this collector.
+        */
+        IF @rows_collected > 0
+        BEGIN
+            IF @debug = 1
+            BEGIN
+                RAISERROR(N'Chain-triggering blocked process parser and analyzer', 0, 1) WITH NOWAIT;
+            END;
+
+            BEGIN TRY
+                EXECUTE collect.process_blocked_process_xml
+                    @debug = @debug;
+            END TRY
+            BEGIN CATCH
+                INSERT INTO
+                    config.collection_log
+                (
+                    collector_name,
+                    collection_status,
+                    duration_ms,
+                    error_message
+                )
+                VALUES
+                (
+                    N'blocked_process_xml_collector',
+                    N'CHAIN_ERROR',
+                    DATEDIFF(MILLISECOND, @start_time, SYSDATETIME()),
+                    N'Chain-triggered parser failed: ' + ERROR_MESSAGE()
+                );
+            END CATCH;
+
+            BEGIN TRY
+                EXECUTE collect.blocking_deadlock_analyzer
+                    @debug = @debug;
+            END TRY
+            BEGIN CATCH
+                INSERT INTO
+                    config.collection_log
+                (
+                    collector_name,
+                    collection_status,
+                    duration_ms,
+                    error_message
+                )
+                VALUES
+                (
+                    N'blocked_process_xml_collector',
+                    N'CHAIN_ERROR',
+                    DATEDIFF(MILLISECOND, @start_time, SYSDATETIME()),
+                    N'Chain-triggered analyzer failed: ' + ERROR_MESSAGE()
+                );
+            END CATCH;
+        END;
+
     END TRY
     BEGIN CATCH
         IF @@TRANCOUNT > 0