Merge branch 'master' into subnet-metrics-endpoint

Master landed #11335, the upstream version of this branch's
consumed-cycles refactor, which makes
`SubnetMetrics::consumed_cycles_total_including_canisters` a private
field behind a getter written only by
`SubnetMetrics::refresh_consumed_cycles`. Every conflict is between
that and this branch's own variant of the same change, so master's
version wins throughout; the `subnet_metrics` endpoint and its tests
now read the total through the getter.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: mraszyk

ic-os/components/setupos/check-ntp.sh

@@ -39,7 +39,13 @@ function set_hwclock_utc() {
 
 main() {
     log_start "$(basename $0)"
-    check_ntp
+    if check_cmdline_var ic.setupos.run_checks; then
+        check_ntp
+    else
+        # NTP is not available in test environments; test VMs take their clock
+        # from the hypervisor.
+        echo "* NTP synchronization check skipped by request via kernel command line"
+    fi
     set_hwclock_utc
     log_end "$(basename $0)"
 }

rs/canister_sandbox/src/replica_controller/sandboxed_execution_controller.rs

@@ -27,7 +27,6 @@ use ic_metrics::buckets::{decimal_buckets_with_zero, exponential_buckets};
 use ic_replicated_state::canister_state::execution_state::{
     SandboxMemory, SandboxMemoryHandle, SandboxMemoryOwner, WasmBinary, WasmExecutionMode,
 };
-use ic_replicated_state::metrics::instructions_buckets;
 use ic_replicated_state::{
     EmbedderCache, ExecutionState, ExportedFunctions, Memory, PageMap, ReplicatedState,
     page_map::allocated_pages_count,
@@ -157,7 +156,6 @@ struct SandboxedExecutionMetrics {
     mprotect_count: HistogramVec,
     copy_page_count: HistogramVec,
     sigsegv_handler_duration: HistogramVec,
-    dmt_projected_message_cost: HistogramVec,
 }
 
 impl SandboxedExecutionMetrics {
@@ -411,12 +409,6 @@ impl SandboxedExecutionMetrics {
                 decimal_buckets_with_zero(-4, 1),
                 &["api_type", "memory_type"],
             ),
-            dmt_projected_message_cost: metrics_registry.histogram_vec(
-                "sandboxed_execution_dmt_projected_message_cost",
-                "Cost of a message when the DMT charges for all page accesses.",
-                instructions_buckets(), /* same as scheduler_instructions_consumed_per_message for comparison */
-                &["api_type", "memory_type"],
-            ),
         }
     }
 
@@ -502,10 +494,6 @@ impl SandboxedExecutionMetrics {
             .with_label_values(&[api_type_label, "stable"])
             .observe(instance_stats.stable_sigsegv_handler_duration.as_secs_f64());
 
-        self.dmt_projected_message_cost
-            .with_label_values(&[api_type_label, "both"])
-            .observe(instance_stats.dmt_projected_message_cost as f64);
-
         self.allocated_pages.set(allocated_pages_count() as i64);
     }
 }
@@ -1709,7 +1697,7 @@ impl SandboxedExecutionController {
                 });
                 return WasmExecutionResult::Paused(slice, paused);
             }
-            CompletionResult::Finished(mut exec_output) => {
+            CompletionResult::Finished(exec_output) => {
                 let execution_status = match exec_output.wasm.wasm_result.clone() {
                     Ok(Some(WasmResult::Reply(_))) => "Success",
                     Ok(Some(WasmResult::Reject(_))) => "Reject",
@@ -1721,19 +1709,6 @@ impl SandboxedExecutionController {
                     execution_status,
                     execution_state.wasm_execution_mode.as_str(),
                 );
-                // Temporary metric: How much will the message cost when we charge via DMT.
-                let instructions_used = message_instruction_limit
-                    .saturating_sub(&exec_output.wasm.num_instructions_left);
-                let stable_writes = exec_output.wasm.instance_stats.stable_dirty_pages;
-                let stable_reads = exec_output.wasm.instance_stats.stable_accessed_pages;
-                let heap_writes = exec_output.wasm.instance_stats.wasm_dirty_pages;
-                let heap_reads = exec_output.wasm.instance_stats.wasm_accessed_pages - heap_writes;
-                // we currently charge 1000 for stable and heap writes (plus 3000 for copy overhead, which remains separate), and 0 for everything else.
-                let additional_cost =
-                    (stable_writes + heap_writes) * 4000 + (stable_reads + heap_reads) * 5000;
-                exec_output.wasm.instance_stats.dmt_projected_message_cost =
-                    instructions_used.get() as usize + additional_cost;
-
                 self.metrics
                     .observe_instance_stats(&exec_output.wasm.instance_stats, api_type_label);
                 exec_output

rs/canonical_state/src/encoding/tests/compatibility.rs

@@ -307,8 +307,9 @@ fn canonical_encoding_stream_header_v26() {
 /// Starting with `V29`, the `Instructions` (80B) and
 /// `RequestAndResponseTransmission` (20B) use case entries are no longer added
 /// on top of the deleted canisters scalar (fixing the double counting), while
-/// the cycles consumed by non-deleted canisters (50B, passed to
-/// `encode_subnet_metrics`) are added instead. Hence the expected value becomes
+/// the cycles consumed by non-deleted canisters (50B, folded into the stored
+/// aggregate by `SubnetMetrics::refresh_consumed_cycles`) are added instead.
+/// Hence the expected value becomes
 /// 0 (deleted) + 50B (HTTP) + 100B (ECDSA) + 50B (canisters) = 200B
 /// (`1B 0000002E90EDD000`).
 ///
@@ -342,10 +343,7 @@ fn canonical_encoding_subnet_metrics() {
         metrics.threshold_signature_agreements =
             BTreeMap::from([(schnorr_key_id, 15), (ecdsa_key_id, 16)]);
 
-        // From `V29` on the reported total is this stored aggregate: the subnet-level
-        // total plus the part consumed by the canisters that still exist.
-        metrics.consumed_cycles_total_including_canisters =
-            metrics.consumed_cycles_total() + NominalCycles::new(50_000_000_000);
+        metrics.refresh_consumed_cycles(NominalCycles::new(50_000_000_000));
 
         let expected = if certification_version >= CertificationVersion::V29 {
             "A4 00 05 01 1A 00 50 00 00 02 A2 00 1B 00 00 00 2E 90 ED D0 00 01 00 03 19 10 68"

rs/canonical_state/src/encoding/types.rs

@@ -738,10 +738,11 @@ impl
         // by deleted canisters and does not account for non-deleted canisters.
         //
         // Starting with `V29`, the reported total is the stored
-        // `SubnetMetrics::consumed_cycles_total_including_canisters`, which no longer
-        // double counts deleted canisters and does account for the existing ones.
+        // `SubnetMetrics::consumed_cycles_total_including_canisters`, which no
+        // longer double counts deleted canisters and does account for the existing
+        // ones.
         let consumed_cycles_total = if certification_version >= CertificationVersion::V29 {
-            metrics.consumed_cycles_total_including_canisters
+            metrics.consumed_cycles_total_including_canisters()
         } else {
             metrics.consumed_cycles_total_v28()
         };

rs/canonical_state/src/lazy_tree_conversion.rs

@@ -1144,11 +1144,7 @@ fn subnets_as_tree<'a>(
                         subnet_id == &own_subnet_id,
                         "metrics",
                         // Starting with `V29`, the reported total also includes
-                        // the cycles consumed by all non-deleted canisters, read
-                        // from the stored
-                        // `SubnetMetrics::consumed_cycles_total_including_canisters`
-                        // (refreshed by
-                        // `ReplicatedState::refresh_consumed_cycles`).
+                        // the cycles consumed by all non-deleted canisters.
                         blob(move || encode_subnet_metrics(metrics, certification_version)),
                     )
                     .with_tree_if(

rs/canonical_state/src/traversal.rs

@@ -1199,7 +1199,9 @@ mod tests {
     #[test]
     fn test_traverse_subnet_metrics_includes_canister_consumed_cycles_at_v29() {
         use crate::encoding::encode_subnet_metrics;
-        use ic_types_cycles::{CompoundCycles, Instructions, NominalCycles};
+        use ic_types_cycles::{
+            CompoundCycles, CyclesUseCase, Instructions, NominalCycles, NominalCyclesTesting,
+        };
 
         let own_subnet_id = subnet_test_id(1);
         let mut state = ReplicatedState::new(own_subnet_id, SubnetType::Application);
@@ -1227,6 +1229,13 @@ mod tests {
             );
         });
 
+        // Non-zero subnet-level consumption, so that the reported total covers both
+        // parts: the subnet-level aggregate and the canisters' part below.
+        let subnet_metrics = &mut state.metadata.subnet_metrics;
+        subnet_metrics.observe_consumed_cycles_by_deleted_canisters(NominalCycles::new(1_000));
+        subnet_metrics
+            .observe_consumed_cycles_with_use_case(CyclesUseCase::VetKd, NominalCycles::new(4));
+
         // Add a non-deleted canister that has consumed some cycles.
         let mut canister_state = new_canister_state(
             canister_test_id(2),
@@ -1252,19 +1261,18 @@ mod tests {
             state
                 .metadata
                 .subnet_metrics
-                .consumed_cycles_total_including_canisters,
+                .consumed_cycles_total_including_canisters(),
             NominalCycles::zero()
         );
 
-        // The refresh publishes the fold into `SubnetMetrics`. This subnet has no
-        // subnet-level consumption, so the canisters' part is the whole total.
         let subnet_level = state.metadata.subnet_metrics.consumed_cycles_total();
+        assert!(subnet_level > NominalCycles::zero());
         state.refresh_consumed_cycles();
         assert_eq!(
             state
                 .metadata
                 .subnet_metrics
-                .consumed_cycles_total_including_canisters,
+                .consumed_cycles_total_including_canisters(),
             subnet_level + consumed_by_canisters
         );
 
@@ -1293,8 +1301,7 @@ mod tests {
 
             // The canister's consumed cycles are included only starting with V29.
             let mut metrics_without_canisters = state.metadata.subnet_metrics.clone();
-            metrics_without_canisters.consumed_cycles_total_including_canisters =
-                NominalCycles::zero();
+            metrics_without_canisters.refresh_consumed_cycles(NominalCycles::zero());
             let without_canisters =
                 encode_subnet_metrics(&metrics_without_canisters, certification_version);
             if certification_version >= CertificationVersion::V29 {

rs/config/src/embedders.rs

@@ -222,11 +222,12 @@ pub struct Config {
     /// overridden at runtime by the registry's `maximum_state_delta`.
     pub default_subnet_heap_delta_capacity: NumBytes,
 
-    /// Dirty page overhead. The number of instructions to charge for each dirty
-    /// page created by a write to stable memory. The default value should be
+    /// The number of instructions to charge for every OS page of heap or stable
+    /// memory that a message touches: once when the page is first accessed and
+    /// once more when it is first written to. The default value should be
     /// replaced with the correct value at runtime when the hypervisor is
     /// created.
-    pub dirty_page_overhead: NumInstructions,
+    pub page_overhead: NumInstructions,
 
     /// If this flag is enabled, then execution of a slice will produce a log
     /// entry with the number of executed instructions and the duration.
@@ -281,7 +282,7 @@ impl Config {
             max_sandbox_count: DEFAULT_MAX_SANDBOX_COUNT,
             max_sandbox_idle_time: DEFAULT_MAX_SANDBOX_IDLE_TIME,
             default_subnet_heap_delta_capacity: SUBNET_HEAP_DELTA_CAPACITY,
-            dirty_page_overhead: NumInstructions::new(0),
+            page_overhead: NumInstructions::new(0),
             trace_execution: FlagStatus::Disabled,
             max_dirty_pages_without_optimization: DEFAULT_MAX_DIRTY_PAGES_WITHOUT_OPTIMIZATION,
             dirty_page_copy_overhead: DIRTY_PAGE_COPY_OVERHEAD,

rs/config/src/subnet_config.rs

@@ -138,8 +138,8 @@ pub const DEFAULT_REFERENCE_SUBNET_SIZE: usize = 13;
 /// Reference subnet size for SEV-enabled application subnets.
 pub const SEV_REFERENCE_SUBNET_SIZE: usize = 7;
 
-/// Costs for each newly created dirty page in stable memory.
-pub const DEFAULT_DIRTY_PAGE_OVERHEAD: NumInstructions = NumInstructions::new(5_000);
+/// Cost of touching a single OS page of canister memory.
+pub const DEFAULT_PAGE_OVERHEAD: NumInstructions = NumInstructions::new(5_000);
 
 /// Accumulated priority reset interval, rounds.
 ///
@@ -276,8 +276,10 @@ pub struct SchedulerConfig {
     /// rounds until they are back under the allowed rate.
     pub install_code_rate_limit: NumInstructions,
 
-    /// Cost for each newly created dirty page in stable memory.
-    pub dirty_page_overhead: NumInstructions,
+    /// The number of instructions to charge for every OS page of heap or stable
+    /// memory that a message touches: once when the page is first accessed and
+    /// once more when it is first written to.
+    pub page_overhead: NumInstructions,
 
     /// Accumulated priority reset interval, rounds.
     pub accumulated_priority_reset_interval: ExecutionRound,
@@ -318,7 +320,7 @@ impl SchedulerConfig {
                 MAX_MESSAGE_DURATION_BEFORE_WARN_IN_SECONDS,
             heap_delta_rate_limit: NumBytes::from(75 * 1024 * 1024),
             install_code_rate_limit: MAX_INSTRUCTIONS_PER_SLICE,
-            dirty_page_overhead: DEFAULT_DIRTY_PAGE_OVERHEAD,
+            page_overhead: DEFAULT_PAGE_OVERHEAD,
             accumulated_priority_reset_interval: ACCUMULATED_PRIORITY_RESET_INTERVAL,
             upload_wasm_chunk_instructions: DEFAULT_UPLOAD_CHUNK_INSTRUCTIONS,
             canister_snapshot_baseline_instructions:
@@ -366,7 +368,7 @@ impl SchedulerConfig {
             // This limit should be high enough (1000T) to effectively disable
             // rate-limiting for the system subnets.
             install_code_rate_limit: NumInstructions::from(1_000_000_000_000_000),
-            dirty_page_overhead: DEFAULT_DIRTY_PAGE_OVERHEAD,
+            page_overhead: DEFAULT_PAGE_OVERHEAD,
             accumulated_priority_reset_interval: ACCUMULATED_PRIORITY_RESET_INTERVAL,
             upload_wasm_chunk_instructions: NumInstructions::from(0),
             canister_snapshot_baseline_instructions: NumInstructions::from(0),

rs/consensus/idkg/src/signer.rs

@@ -573,10 +573,7 @@ impl ThresholdSignerImpl {
         let reconstruction_threshold = match inputs {
             ThresholdSigInputs::Ecdsa(inputs) => inputs.reconstruction_threshold().get() as usize,
             ThresholdSigInputs::Schnorr(inputs) => inputs.reconstruction_threshold().get() as usize,
-            // VetKd's API does not expose the number of shares needed for reconstruction directly.
-            // As this code path is an optimization, we conservatively assume that we do not have
-            // enough shares if the inputs are for VetKd.
-            // The worst thing that can happen is to validate a few extra shares.
+            // Validate all VetKD shares, to allow for failures during share combination.
             ThresholdSigInputs::VetKd(_inputs) => return false,
         };
 

rs/consensus/utils/src/chain_key.rs

@@ -80,6 +80,7 @@ pub fn build_signature_inputs<'a>(
             debug_assert_eq!(context.derivation_path.len(), 1);
             const EMPTY_VEC_REF: &Vec<u8> = &vec![];
             let inputs = ThresholdSigInputs::VetKd(VetKdArgs {
+                request_id,
                 context: VetKdDerivationContextRef {
                     caller: context.request.sender.get_ref(),
                     context: context.derivation_path.first().unwrap_or(EMPTY_VEC_REF),