diff --git a/packages/ic-management-canister-types/CHANGELOG.md b/packages/ic-management-canister-types/CHANGELOG.md index ee55fa420b1b..44082367c5ca 100644 --- a/packages/ic-management-canister-types/CHANGELOG.md +++ b/packages/ic-management-canister-types/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Types for `subnet_metrics`: + - Added the types `SubnetMetricsArgs` and `SubnetMetricsResult`. + ## [0.9.0] - 2026-08-13 ### Added diff --git a/packages/ic-management-canister-types/src/lib.rs b/packages/ic-management-canister-types/src/lib.rs index 72f04f41b060..48316ff909ac 100644 --- a/packages/ic-management-canister-types/src/lib.rs +++ b/packages/ic-management-canister-types/src/lib.rs @@ -1395,6 +1395,65 @@ pub struct SubnetInfoResult { pub registry_version: u64, } +/// # Subnet Metrics Args. +/// +/// Argument type of [`subnet_metrics`](https://docs.internetcomputer.org/references/management-canister/#subnet_metrics). +#[derive( + CandidType, Serialize, Deserialize, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Clone, +)] +pub struct SubnetMetricsArgs { + /// Subnet ID. + pub subnet_id: Principal, +} + +/// # Subnet Metrics Result. +/// +/// Result type of [`subnet_metrics`](https://docs.internetcomputer.org/references/management-canister/#subnet_metrics). +/// +/// This API is EXPERIMENTAL and may evolve in a non-backward-compatible way. +/// +/// # Freshness +/// +/// Only `block_height` is current as of the block in which the call is executed. +/// The other four are read from the subnet's aggregated metrics, which the replica +/// updates at the *end* of a round, so they describe the state as of an earlier +/// block: +/// +/// - `num_canisters`, `consumed_cycles_total` and `update_transactions_total` are +/// as of the end of the previous round. +/// - `canister_state_bytes` is recomputed only every 10 rounds, because summing it +/// over every canister is expensive and it does not need to be exact. It can +/// therefore be up to ten rounds stale, and reads as `0` for the first rounds +/// after the subnet is created. +/// +/// These are the same values, with the same staleness, that `read_state` returns +/// for the `/subnet//metrics` path, so the two agree. +#[derive( + CandidType, Serialize, Deserialize, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Clone, +)] +pub struct SubnetMetricsResult { + /// Height of the block in whose execution the call is processed. + /// Monotonically non-decreasing for a given subnet; the heights of different + /// subnets are unrelated. + pub block_height: Nat, + /// Number of canisters on the subnet, as of the end of the previous round. + pub num_canisters: Nat, + /// Total size in bytes of the state taken by the canisters on the subnet, as + /// of the end of the previous round. + /// + /// Recomputed only every 10 rounds, so this can be up to ten rounds stale + /// (and reads as `0` for the first rounds after the subnet is created). See + /// the type-level "Freshness" note. + pub canister_state_bytes: Nat, + /// Total cycles removed from circulation on the subnet by all current and + /// deleted canisters, as of the end of the previous round. + pub consumed_cycles_total: Nat, + /// Total number of transactions processed on the subnet, i.e. the total + /// number of messages executed in replicated mode, as of the end of the + /// previous round. + pub update_transactions_total: Nat, +} + /// # Canister ID Range. /// /// A closed range of canister IDs, both endpoints inclusive. diff --git a/packages/ic-management-canister-types/tests/candid_equality.rs b/packages/ic-management-canister-types/tests/candid_equality.rs index ae0f1c6120f2..d97240f62e16 100644 --- a/packages/ic-management-canister-types/tests/candid_equality.rs +++ b/packages/ic-management-canister-types/tests/candid_equality.rs @@ -132,6 +132,11 @@ fn node_metrics_history(_: NodeMetricsHistoryArgs) -> NodeMetricsHistoryResult { unimplemented!() } +#[candid_method(update)] +fn subnet_metrics(_: SubnetMetricsArgs) -> SubnetMetricsResult { + unimplemented!() +} + #[candid_method(update)] fn provisional_create_canister_with_cycles( _: ProvisionalCreateCanisterWithCyclesArgs, diff --git a/packages/ic-management-canister-types/tests/ic.did b/packages/ic-management-canister-types/tests/ic.did index cc195f61006a..e6cb2687df9b 100644 --- a/packages/ic-management-canister-types/tests/ic.did +++ b/packages/ic-management-canister-types/tests/ic.did @@ -447,6 +447,38 @@ type node_metrics_history_result = vec record { node_metrics : vec node_metrics; }; +type subnet_metrics_args = record { + subnet_id : principal; +}; + +// This API is EXPERIMENTAL and may evolve in a non-backward-compatible way. +// +// Only `block_height` is current as of the block in which the call is executed. +// The other four fields are read from the subnet's aggregated metrics, which the +// replica updates at the *end* of a round, so they describe the state as of an +// earlier block; see the individual fields. +type subnet_metrics_result = record { + // Height of the block in whose execution this call is processed. + // Monotonically non-decreasing for a given subnet; the heights of different + // subnets are unrelated. + block_height : nat; + // Number of canisters on the subnet, as of the end of the previous round. + num_canisters : nat; + // Total size in bytes of the state taken by the canisters on the subnet, as + // of the end of the previous round. Recomputed only every 10 rounds, because + // summing it over every canister is expensive and it does not need to be + // exact, so it can be up to ten rounds stale and reads as 0 for the first + // rounds after the subnet is created. + canister_state_bytes : nat; + // Total cycles removed from circulation on the subnet by all current and + // deleted canisters, as of the end of the previous round. + consumed_cycles_total : nat; + // Total number of transactions processed on the subnet, i.e. the total + // number of messages executed in replicated mode, as of the end of the + // previous round. + update_transactions_total : nat; +}; + type subnet_info_args = record { subnet_id : principal; }; @@ -709,6 +741,7 @@ service ic : { // metrics interface node_metrics_history : (node_metrics_history_args) -> (node_metrics_history_result); + subnet_metrics : (subnet_metrics_args) -> (subnet_metrics_result); // subnet info subnet_info : (subnet_info_args) -> (subnet_info_result); diff --git a/rs/embedders/src/wasmtime_embedder/system_api/routing.rs b/rs/embedders/src/wasmtime_embedder/system_api/routing.rs index def610efd866..cd4a6df30bda 100644 --- a/rs/embedders/src/wasmtime_embedder/system_api/routing.rs +++ b/rs/embedders/src/wasmtime_embedder/system_api/routing.rs @@ -14,9 +14,9 @@ use ic_management_canister_types_private::{ NodeMetricsHistoryArgs, Payload, ProvisionalTopUpCanisterArgs, ReadCanisterSnapshotDataArgs, ReadCanisterSnapshotMetadataArgs, RenameCanisterArgs, ReshareChainKeyArgs, SchnorrPublicKeyArgs, SetupInitialDKGArgs, SignWithECDSAArgs, SignWithSchnorrArgs, - StoredChunksArgs, SubnetInfoArgs, TakeCanisterSnapshotArgs, UninstallCodeArgs, - UpdateSettingsArgs, UploadCanisterSnapshotDataArgs, UploadCanisterSnapshotMetadataArgs, - UploadChunkArgs, VetKdDeriveKeyArgs, VetKdPublicKeyArgs, + StoredChunksArgs, SubnetInfoArgs, SubnetMetricsArgs, TakeCanisterSnapshotArgs, + UninstallCodeArgs, UpdateSettingsArgs, UploadCanisterSnapshotDataArgs, + UploadCanisterSnapshotMetadataArgs, UploadChunkArgs, VetKdDeriveKeyArgs, VetKdPublicKeyArgs, }; use ic_replicated_state::NetworkTopology; use itertools::Itertools; @@ -201,6 +201,7 @@ pub(super) fn resolve_destination( Ok(Ic00Method::NodeMetricsHistory) => { Ok(NodeMetricsHistoryArgs::decode(payload)?.subnet_id) } + Ok(Ic00Method::SubnetMetrics) => Ok(SubnetMetricsArgs::decode(payload)?.subnet_id), Ok(Ic00Method::SubnetInfo) => Ok(SubnetInfoArgs::decode(payload)?.subnet_id), Ok(Ic00Method::FetchCanisterLogs) => { let canister_id = FetchCanisterLogsRequest::decode(payload)?.get_canister_id(); diff --git a/rs/embedders/src/wasmtime_embedder/system_api/sandbox_safe_system_state.rs b/rs/embedders/src/wasmtime_embedder/system_api/sandbox_safe_system_state.rs index 24f93b6dec91..583cc6bf6c66 100644 --- a/rs/embedders/src/wasmtime_embedder/system_api/sandbox_safe_system_state.rs +++ b/rs/embedders/src/wasmtime_embedder/system_api/sandbox_safe_system_state.rs @@ -338,6 +338,7 @@ impl SystemStateModifications { | Ok(Ic00Method::BitcoinSendTransaction) | Ok(Ic00Method::BitcoinGetCurrentFeePercentiles) | Ok(Ic00Method::NodeMetricsHistory) + | Ok(Ic00Method::SubnetMetrics) | Ok(Ic00Method::SubnetInfo) | Ok(Ic00Method::FetchCanisterLogs) | Ok(Ic00Method::UploadChunk) diff --git a/rs/execution_environment/src/canister_manager.rs b/rs/execution_environment/src/canister_manager.rs index 485da60a882b..59f1dc50e579 100644 --- a/rs/execution_environment/src/canister_manager.rs +++ b/rs/execution_environment/src/canister_manager.rs @@ -151,6 +151,10 @@ impl CanisterManager { | Ok(Ic00Method::BitcoinSendTransactionInternal) | Ok(Ic00Method::BitcoinGetCurrentFeePercentiles) | Ok(Ic00Method::NodeMetricsHistory) + // Unreachable for `SubnetMetrics`: `extract_effective_canister_id` + // rejects it earlier, at the ingress filter. Listed for exhaustiveness + // and as defence in depth. + | Ok(Ic00Method::SubnetMetrics) | Ok(Ic00Method::SubnetInfo) // `RenameCanister` can only be called from the NNS subnet. | Ok(Ic00Method::RenameCanister) => Err(UserError::new( diff --git a/rs/execution_environment/src/canister_manager/tests.rs b/rs/execution_environment/src/canister_manager/tests.rs index ed628e6c45f1..e65219fde989 100644 --- a/rs/execution_environment/src/canister_manager/tests.rs +++ b/rs/execution_environment/src/canister_manager/tests.rs @@ -44,8 +44,8 @@ use ic_management_canister_types_private::{ InstallCodeArgsV2, Method, NodeMetricsHistoryArgs, NodeMetricsHistoryResponse, OnLowWasmMemoryHookStatus, Payload, ProvisionalCreateCanisterWithCyclesArgs, RenameCanisterArgs, RenameToArgs, StoredChunksArgs, StoredChunksReply, SubnetInfoArgs, - SubnetInfoResponse, TakeCanisterSnapshotArgs, UpdateSettingsArgs, UploadChunkArgs, - UploadChunkReply, WasmMemoryPersistence, + SubnetInfoResponse, SubnetMetricsArgs, SubnetMetricsResponse, TakeCanisterSnapshotArgs, + UpdateSettingsArgs, UploadChunkArgs, UploadChunkReply, WasmMemoryPersistence, }; use ic_metrics::MetricsRegistry; use ic_registry_provisional_whitelist::ProvisionalWhitelist; @@ -6328,6 +6328,134 @@ fn subnet_info_ingress_fails() { ); } +/// Sends the given payload to `subnet_metrics` as an inter-canister call from a +/// canister on a remote subnet, executes it, and returns the decoded response or +/// the reject. +fn subnet_metrics_raw_call( + test: &mut ExecutionTest, + payload: Vec, +) -> Result { + test.inject_call_to_ic00(Method::SubnetMetrics, payload, Cycles::zero()); + test.execute_subnet_message(); + // Route the response back towards the caller (on a different subnet) so that + // it can be inspected via `xnet_messages`. + test.induct_messages(); + // The response to the one injected call is the only message crossing the + // subnet boundary. + assert_eq!(test.xnet_messages().len(), 1); + match &test.get_xnet_response(0).response_payload { + ic_types::messages::Payload::Data(bytes) => { + Ok(Decode!(bytes, SubnetMetricsResponse).unwrap()) + } + ic_types::messages::Payload::Reject(context) => { + Err((context.code(), context.message().to_string())) + } + } +} + +/// As [`subnet_metrics_raw_call`], with a well-formed payload naming `subnet_id`. +fn subnet_metrics_call( + test: &mut ExecutionTest, + subnet_id: PrincipalId, +) -> Result { + subnet_metrics_raw_call(test, SubnetMetricsArgs { subnet_id }.encode()) +} + +#[test] +fn subnet_metrics_ingress_update_fails_at_ingress_filter() { + let own_subnet_id = subnet_test_id(1); + let mut test = ExecutionTestBuilder::new() + .with_own_subnet_id(own_subnet_id) + .build(); + let payload = SubnetMetricsArgs { + subnet_id: own_subnet_id.get(), + } + .encode(); + + let result = test.should_accept_ingress_message(IC_00, Method::SubnetMetrics, payload); + assert_eq!( + result, + Err(UserError::new( + ErrorCode::CanisterRejectedMessage, + "ic00 method subnet_metrics can not be called via ingress messages" + )) + ); +} + +#[test] +fn subnet_metrics_ingress_update_fails_at_execution() { + let own_subnet_id = subnet_test_id(1); + let mut test = ExecutionTestBuilder::new() + .with_own_subnet_id(own_subnet_id) + .build(); + let payload = SubnetMetricsArgs { + subnet_id: own_subnet_id.get(), + } + .encode(); + test.subnet_message(Method::SubnetMetrics, payload) + .unwrap_err() + .assert_contains( + ErrorCode::CanisterContractViolation, + "subnet_metrics cannot be called by a user", + ); +} + +#[test] +fn subnet_metrics_ingress_query_fails() { + let own_subnet_id = subnet_test_id(1); + let mut test = ExecutionTestBuilder::new() + .with_own_subnet_id(own_subnet_id) + .build(); + let payload = SubnetMetricsArgs { + subnet_id: own_subnet_id.get(), + } + .encode(); + test.non_replicated_query(CanisterId::ic_00(), "subnet_metrics", payload) + .unwrap_err() + .assert_contains( + ErrorCode::CanisterMethodNotFound, + "Query method subnet_metrics not found.", + ); +} + +#[test] +fn subnet_metrics_foreign_subnet_id_is_rejected() { + let own_subnet_id = subnet_test_id(1); + let other_subnet_id = subnet_test_id(3); + let caller_canister = canister_test_id(1); + let mut test = ExecutionTestBuilder::new() + .with_own_subnet_id(own_subnet_id) + .with_caller(subnet_test_id(2), caller_canister) + .build(); + + let (code, message) = subnet_metrics_call(&mut test, other_subnet_id.get()).unwrap_err(); + assert_eq!(code, RejectCode::CanisterReject); + assert!( + message.contains("does not match current subnet ID"), + "unexpected reject message: {message}" + ); +} + +#[test] +fn subnet_metrics_malformed_payload_is_rejected() { + let own_subnet_id = subnet_test_id(1); + let caller_canister = canister_test_id(1); + let mut test = ExecutionTestBuilder::new() + .with_own_subnet_id(own_subnet_id) + .with_caller(subnet_test_id(2), caller_canister) + .build(); + + let (code, message) = + subnet_metrics_raw_call(&mut test, EmptyBlob.encode()).expect_err("expected a reject"); + // The Candid decode failure surfaces as `ErrorCode::InvalidManagementPayload` + // (`candid_error_to_user_error`), which maps to `RejectCode::CanisterReject`. + assert_eq!(code, RejectCode::CanisterReject); + assert!( + message.contains("Error decoding candid"), + "unexpected reject message: {message}" + ); +} + #[test] fn node_metrics_history_update_succeeds() { let own_subnet_id = subnet_test_id(1); diff --git a/rs/execution_environment/src/execution_environment.rs b/rs/execution_environment/src/execution_environment.rs index 4c51beac034b..aeb5ae2ae131 100644 --- a/rs/execution_environment/src/execution_environment.rs +++ b/rs/execution_environment/src/execution_environment.rs @@ -42,10 +42,10 @@ use ic_management_canister_types_private::{ ReadCanisterSnapshotDataArgs, ReadCanisterSnapshotMetadataArgs, RenameCanisterArgs, ReshareChainKeyArgs, SchnorrAlgorithm, SchnorrPublicKeyArgs, SchnorrPublicKeyResponse, SetupInitialDKGArgs, SignWithECDSAArgs, SignWithSchnorrArgs, SignWithSchnorrAux, - StoredChunksArgs, SubnetInfoArgs, SubnetInfoResponse, TakeCanisterSnapshotArgs, - UninstallCodeArgs, UpdateSettingsArgs, UploadCanisterSnapshotDataArgs, - UploadCanisterSnapshotMetadataArgs, UploadChunkArgs, VetKdDeriveKeyArgs, VetKdPublicKeyArgs, - VetKdPublicKeyResult, + StoredChunksArgs, SubnetInfoArgs, SubnetInfoResponse, SubnetMetricsArgs, SubnetMetricsResponse, + TakeCanisterSnapshotArgs, UninstallCodeArgs, UpdateSettingsArgs, + UploadCanisterSnapshotDataArgs, UploadCanisterSnapshotMetadataArgs, UploadChunkArgs, + VetKdDeriveKeyArgs, VetKdPublicKeyArgs, VetKdPublicKeyResult, }; use ic_metrics::MetricsRegistry; use ic_registry_provisional_whitelist::ProvisionalWhitelist; @@ -1886,6 +1886,20 @@ impl ExecutionEnvironment { } }, + Ok(Ic00Method::SubnetMetrics) => match &msg { + CanisterCall::Ingress(_) => { + self.reject_unexpected_ingress(Ic00Method::SubnetMetrics) + } + CanisterCall::Request(_) => { + let res = SubnetMetricsArgs::decode(payload) + .and_then(|args| self.subnet_metrics(&state, current_round, args)); + ExecuteSubnetMessageResult::Finished { + response: res.map(|res| (res, None)), + refund: msg.take_cycles(), + } + } + }, + Ok(Ic00Method::SubnetInfo) => match &msg { CanisterCall::Ingress(_) => self.reject_unexpected_ingress(Ic00Method::SubnetInfo), CanisterCall::Request(_) => { @@ -3386,6 +3400,60 @@ impl ExecutionEnvironment { Ok(Encode!(&res).unwrap()) } + /// Computes the response to the `subnet_metrics` management canister method. + /// + /// Charges no round instructions: every field comes from adding up a fixed + /// number of already-aggregated `SubnetMetrics` fields, so there is no work + /// here to price. See the `counts_toward_round_limit: false` grouping in + /// `ic00_permissions.rs`. + fn subnet_metrics( + &self, + state: &ReplicatedState, + current_round: ExecutionRound, + args: SubnetMetricsArgs, + ) -> Result, UserError> { + if args.subnet_id != self.own_subnet_id.get() { + return Err(UserError::new( + ErrorCode::CanisterRejectedMessage, + format!( + "Provided target subnet ID {} does not match current subnet ID {}.", + args.subnet_id, self.own_subnet_id + ), + )); + } + let metrics = &state.metadata.subnet_metrics; + // The same stored aggregate the certified state tree at + // `/subnet//metrics` reads from certification version `V29` on, so + // the two cannot drift. It is refreshed on every `commit_and_certify` + // (`rs/state_manager/src/lib.rs`), so a call executing in round N reads the + // end-of-round-(N-1) value -- the same one-round lag as `num_canisters` + // below. Reading it rather than recomputing the total is also what keeps a + // canister deleted earlier in this same round from being counted twice. + let consumed_cycles_total = metrics.consumed_cycles_total_including_canisters(); + let res = SubnetMetricsResponse { + // The height of the block in whose execution this call is processed. + // `ExecutionRound` is numerically the finalized consensus block + // height; see `rs/messaging/src/state_machine.rs`. + block_height: candid::Nat::from(current_round.get()), + // `num_canisters` and `update_transactions_total` are written at the + // *end* of a round (`message_routing.rs`, `scheduler.rs`), so a call + // executing in round N reports the end-of-round-(N-1) values. That + // one-round lag is what `read_state` reports for height N-1 too, so the + // two agree; it is nonetheless not literally "current". + num_canisters: candid::Nat::from(metrics.num_canisters), + // Read from the stored `SubnetMetrics` field rather than recomputed + // live, so that the value agrees with the certified state tree. Note + // that message routing only refreshes the stored field every 10 + // rounds by design (`rs/messaging/src/message_routing.rs`), so + // recomputing it here would make `subnet_metrics` disagree with + // `read_state` on 9 rounds out of 10. + canister_state_bytes: candid::Nat::from(metrics.canister_state_bytes.get()), + consumed_cycles_total: candid::Nat::from(consumed_cycles_total.get()), + update_transactions_total: candid::Nat::from(metrics.update_transactions_total), + }; + Ok(Encode!(&res).unwrap()) + } + // Executes an inter-canister response. // // Returns a tuple with the result, along with a flag indicating whether or diff --git a/rs/execution_environment/src/execution_environment_metrics.rs b/rs/execution_environment/src/execution_environment_metrics.rs index b5b6d3a4dd32..b0dc9e965377 100644 --- a/rs/execution_environment/src/execution_environment_metrics.rs +++ b/rs/execution_environment/src/execution_environment_metrics.rs @@ -313,6 +313,7 @@ impl ExecutionEnvironmentMetrics { | ic00::Method::BitcoinSendTransaction | ic00::Method::BitcoinGetCurrentFeePercentiles | ic00::Method::NodeMetricsHistory + | ic00::Method::SubnetMetrics | ic00::Method::SubnetInfo | ic00::Method::FetchCanisterLogs | ic00::Method::ProvisionalCreateCanisterWithCycles diff --git a/rs/execution_environment/src/ic00_permissions.rs b/rs/execution_environment/src/ic00_permissions.rs index 76d39db95c7d..44802bb5abab 100644 --- a/rs/execution_environment/src/ic00_permissions.rs +++ b/rs/execution_environment/src/ic00_permissions.rs @@ -60,6 +60,7 @@ impl Ic00MethodPermissions { | Ic00Method::BitcoinGetSuccessors | Ic00Method::NodeMetricsHistory | Ic00Method::SubnetInfo + | Ic00Method::SubnetMetrics | Ic00Method::ProvisionalCreateCanisterWithCycles | Ic00Method::ProvisionalTopUpCanister | Ic00Method::StoredChunks diff --git a/rs/execution_environment/src/scheduler.rs b/rs/execution_environment/src/scheduler.rs index c63b673ffdcf..1a67a0cd16f2 100644 --- a/rs/execution_environment/src/scheduler.rs +++ b/rs/execution_environment/src/scheduler.rs @@ -1928,6 +1928,7 @@ fn get_instruction_limits_for_subnet_message( | BitcoinGetCurrentFeePercentiles | BitcoinGetSuccessors | NodeMetricsHistory + | SubnetMetrics | SubnetInfo | FetchCanisterLogs | ProvisionalCreateCanisterWithCycles diff --git a/rs/execution_environment/tests/execution_test.rs b/rs/execution_environment/tests/execution_test.rs index 793dde956469..7d6fa4fcc302 100644 --- a/rs/execution_environment/tests/execution_test.rs +++ b/rs/execution_environment/tests/execution_test.rs @@ -9,12 +9,14 @@ use ic_config::{ }; use ic_embedders::wasmtime_embedder::system_api::MAX_CALL_TIMEOUT_SECONDS; use ic_execution_environment::units::{GIB, MIB}; +use ic_interfaces_state_manager::StateReader; use ic_management_canister_types_private::{ CanisterIdRecord, CanisterInstallModeV2, CanisterMetadataRequest, CanisterMetadataResponse, CanisterMetricsArgs, CanisterSettingsArgs, CanisterSettingsArgsBuilder, CanisterStatusResultV2, CreateCanisterArgs, DerivationPath, EcdsaKeyId, EmptyBlob, IC_00, InstallCodeArgsV2, ListCanistersResponse, LoadCanisterSnapshotArgs, MasterPublicKeyId, Method, Payload, - SignWithECDSAArgs, TakeCanisterSnapshotArgs, UpdateSettingsArgs, + SignWithECDSAArgs, SubnetMetricsArgs, SubnetMetricsResponse, TakeCanisterSnapshotArgs, + UpdateSettingsArgs, }; use ic_registry_resource_limits::ResourceLimits; use ic_registry_subnet_type::SubnetType; @@ -2881,6 +2883,263 @@ fn list_canisters_via_inter_canister_call_rejected_for_non_admin() { assert_eq!(env.subnet_message_instructions(), instructions_baseline); } +/// Number of rounds one [`read_subnet_metrics`] takes: the ingress executes in +/// the first, the `subnet_metrics` request it makes in the second, and the reply +/// reaches the caller in the third. +const ROUNDS_PER_SUBNET_METRICS_READ: u64 = 3; + +/// Calls `subnet_metrics` for the subnet under test from `caller` and returns the +/// reply. The call has to go through a canister, as `subnet_metrics` cannot be +/// called by a user. +/// +/// Asserts on the way that `block_height` is the height of the block in whose +/// execution the call was processed, i.e. the second of the +/// [`ROUNDS_PER_SUBNET_METRICS_READ`] rounds this read takes. +fn read_subnet_metrics(env: &StateMachine, caller: CanisterId) -> SubnetMetricsResponse { + let payload = SubnetMetricsArgs { + subnet_id: env.get_subnet_id().get(), + } + .encode(); + let call = wasm() + .call_simple( + CanisterId::ic_00(), + Method::SubnetMetrics, + call_args() + .other_side(payload) + .on_reject(wasm().reject_message().reject()), + ) + .build(); + + let height_before = env.state_manager.latest_state_height().get(); + let reply = get_reply(env.execute_ingress(caller, "update", call)); + let response = SubnetMetricsResponse::decode(&reply).unwrap(); + assert_eq!( + env.state_manager.latest_state_height().get(), + height_before + ROUNDS_PER_SUBNET_METRICS_READ + ); + assert_eq!(response.block_height, candid::Nat::from(height_before + 2)); + + response +} + +/// Covers the semantics of every `subnet_metrics` field on a running subnet: +/// `block_height` is the height of the block in whose execution the call is +/// processed (asserted by `read_subnet_metrics` on each read below), while the +/// four aggregate fields report `SystemMetadata::subnet_metrics`, which is +/// written at the end of a round and hence lags by (at least) one round. +#[test] +fn subnet_metrics_reports_the_subnets_metrics() { + let env = StateMachineBuilder::new() + .with_config(Some(StateMachineConfig::new( + SubnetConfig::new(SubnetType::Application), + HypervisorConfig::default(), + ))) + .with_subnet_type(SubnetType::Application) + .build(); + let caller = create_universal_canister_with_cycles( + &env, + Some(CanisterSettingsArgsBuilder::new().build()), + INITIAL_CYCLES_BALANCE, + ); + + // Message routing recomputes `canister_state_bytes` only in rounds whose + // height is a multiple of 10. Idle the subnet up to such a round, so that the + // stored value is the memory the canisters take with nothing in flight; the + // three rounds of the read below are then 1, 2 and 3 modulo 10, so none of + // them can refresh it in between. + while !env + .state_manager + .latest_state_height() + .get() + .is_multiple_of(10) + { + env.tick(); + } + let (metrics_before, memory_usage) = { + let state = env.get_latest_state(); + ( + state.metadata.subnet_metrics.clone(), + state.total_canister_memory_usage(), + ) + }; + assert_eq!(metrics_before.canister_state_bytes, memory_usage); + assert_gt!(memory_usage.get(), 0); + + let response = read_subnet_metrics(&env, caller); + + // `num_canisters`: the single canister installed above. + assert_eq!(response.num_canisters, candid::Nat::from(1_u64)); + // `canister_state_bytes`: the memory the canisters take, as of the last + // refresh -- computed here independently of the handler. + assert_eq!( + response.canister_state_bytes, + candid::Nat::from(memory_usage.get()) + ); + // `update_transactions_total`: the messages executed in replicated mode. + // Exactly one was executed since the snapshot above, namely the ingress that + // made this call. The `subnet_metrics` request itself executes one round + // later still, and the handler reports the round before that, so it is not + // counted yet. + assert_eq!( + response.update_transactions_total, + candid::Nat::from(metrics_before.update_transactions_total + 1) + ); + // `consumed_cycles_total`: the subnet-wide aggregate, including the cycles + // consumed by the canisters that still exist -- non-zero here only because + // `commit_and_certify` refreshes the canisters' part on every committed + // state. Creating and installing the caller charged cycles, and executing the + // ingress that made this call charged more, so both the reported total and + // the total the state holds now are above the snapshot. + // + // Deliberately no upper bound: while the call is in flight the caller has + // prepaid for a maximum-size response, and the prepayment counts as consumed + // until the unused part is refunded, so the reported total is in fact *above* + // the total once the call has completed. + let consumed_before = metrics_before.consumed_cycles_total_including_canisters(); + let consumed_now = env + .get_latest_state() + .metadata + .subnet_metrics + .consumed_cycles_total_including_canisters(); + assert_gt!(consumed_before.get(), 0); + assert_gt!(consumed_now.get(), consumed_before.get()); + assert_gt!( + response.consumed_cycles_total, + candid::Nat::from(consumed_now.get()) + ); + + // `num_canisters` follows the canister population in both directions. + let other = env.create_canister_with_cycles(None, INITIAL_CYCLES_BALANCE, None); + assert_eq!( + read_subnet_metrics(&env, caller).num_canisters, + candid::Nat::from(2_u64) + ); + env.stop_canister(other).unwrap(); + env.delete_canister(other).unwrap(); + assert_eq!( + read_subnet_metrics(&env, caller).num_canisters, + candid::Nat::from(1_u64) + ); +} + +/// Regression test: the `consumed_cycles_total` the endpoint reports is the +/// aggregate of the last *committed* state, read from +/// `SubnetMetrics::consumed_cycles_total_including_canisters`, and not recomputed +/// from the fields as they stand mid-round. +/// +/// `delete_canister` adds the deleted canister's consumed cycles to +/// `consumed_cycles_by_deleted_canisters` straight away, while the canisters' part +/// of the stored aggregate -- which still counts that canister -- is only refreshed +/// when the next state is committed. Recomputing the total in between therefore +/// counts the deleted canister twice. +/// +/// The test forces exactly that window: a single update issues `delete_canister` +/// and then `subnet_metrics`, so both requests sit in the caller's output queue to +/// `ic00` in that order and `drain_subnet_queues` executes them in the same round, +/// the deletion first. +#[test] +fn subnet_metrics_consumed_cycles_total_is_the_committed_aggregate() { + let env = StateMachineBuilder::new() + .with_config(Some(StateMachineConfig::new( + SubnetConfig::new(SubnetType::Application), + HypervisorConfig::default(), + ))) + .with_subnet_type(SubnetType::Application) + .build(); + let caller = create_universal_canister_with_cycles( + &env, + Some(CanisterSettingsArgsBuilder::new().build()), + INITIAL_CYCLES_BALANCE, + ); + + // A stopped canister that `caller` controls, and can therefore delete. The + // anonymous principal is a controller too, so that `stop_canister` is allowed. + let victim = env.create_canister_with_cycles( + None, + INITIAL_CYCLES_BALANCE, + Some( + CanisterSettingsArgsBuilder::new() + .with_controllers(vec![PrincipalId::new_anonymous(), caller.get()]) + .build(), + ), + ); + env.stop_canister(victim).unwrap(); + + let call = wasm() + // Whatever the deletion answers is irrelevant: the update replies with the + // `subnet_metrics` response below. + .call_simple( + CanisterId::ic_00(), + Method::DeleteCanister, + call_args() + .other_side(CanisterIdRecord::from(victim).encode()) + .on_reply(wasm().noop()) + .on_reject(wasm().noop()), + ) + .call_simple( + CanisterId::ic_00(), + Method::SubnetMetrics, + call_args() + .other_side( + SubnetMetricsArgs { + subnet_id: env.get_subnet_id().get(), + } + .encode(), + ) + .on_reject(wasm().reject_message().reject()), + ) + .build(); + + // The aggregate as of the last committed state, and the victim's consumption -- + // which a recomputation after the deletion would count a second time. + let (pre_total, victim_consumed) = { + let state = env.get_latest_state(); + ( + state + .metadata + .subnet_metrics + .consumed_cycles_total_including_canisters(), + state + .canister_state(&victim) + .expect("the victim must still exist at this point") + .system_state + .canister_metrics() + .consumed_cycles(), + ) + }; + + let reply = get_reply(env.execute_ingress(caller, "update", call)); + let response = SubnetMetricsResponse::decode(&reply).unwrap(); + + // The deletion did happen. + assert!(env.get_latest_state().canister_state(&victim).is_none()); + let post_total = env + .get_latest_state() + .metadata + .subnet_metrics + .consumed_cycles_total_including_canisters(); + + // The reported total is a committed snapshot, so it cannot predate the state + // the call started from. + assert_ge!( + response.consumed_cycles_total, + candid::Nat::from(pre_total.get()) + ); + // And this is what pins the fix. Deleting the victim burns its remaining + // balance into `consumed_cycles_by_deleted_canisters`, so `post_total` is far + // above every snapshot taken before the deletion committed -- and the handler + // reported one of those. Recomputing the total when the handler ran instead, + // i.e. after the deletion but before the refresh that drops the victim from the + // stored aggregate, would have counted the victim twice and + // returned at least `post_total + victim_consumed`; either way, at least + // `post_total`. Hence the strict `<`. + assert_lt!( + response.consumed_cycles_total, + candid::Nat::from(post_total.get()) + ); + assert_gt!(victim_consumed.get(), 0); +} + #[test] fn maximum_state_size() { let maximum_state_size = NumBytes::new(1 << 30); diff --git a/rs/tests/execution/BUILD.bazel b/rs/tests/execution/BUILD.bazel index 3667a408e77a..f8c5687a41e4 100644 --- a/rs/tests/execution/BUILD.bazel +++ b/rs/tests/execution/BUILD.bazel @@ -216,6 +216,21 @@ system_test( ], ) +system_test( + name = "subnet_metrics_test", + cpus = MIN_LOCAL_CPUS + 3 * DEFAULT_VCPUS_PER_VM, # 3 IC Node VMs * 6 vCPUs. + runtime_deps = UNIVERSAL_CANISTER_RUNTIME_DEPS, + deps = [ + # Keep sorted. + "//rs/registry/subnet_type", + "//rs/tests/driver:ic-system-test-driver", + "//rs/types/management_canister_types", + "//rs/universal_canister/lib", + "@crate_index//:anyhow", + "@crate_index//:candid", + ], +) + system_test( name = "system_api_security_test", cpus = MIN_LOCAL_CPUS + 2 * DEFAULT_VCPUS_PER_VM, # 2 IC Node VMs * 6 vCPUs. diff --git a/rs/tests/execution/Cargo.toml b/rs/tests/execution/Cargo.toml index 1eccb4e87cb7..59dbe10cade0 100644 --- a/rs/tests/execution/Cargo.toml +++ b/rs/tests/execution/Cargo.toml @@ -90,3 +90,7 @@ path = "cycles_cost_schedule_test.rs" [[bin]] name = "canister_migration_test" path = "canister_migration_test.rs" + +[[bin]] +name = "subnet_metrics_test" +path = "subnet_metrics_test.rs" diff --git a/rs/tests/execution/subnet_metrics_test.rs b/rs/tests/execution/subnet_metrics_test.rs new file mode 100644 index 000000000000..49f439edc182 --- /dev/null +++ b/rs/tests/execution/subnet_metrics_test.rs @@ -0,0 +1,146 @@ +use anyhow::Result; +use candid::Decode; +use ic_management_canister_types_private::{self as ic00, Method, Payload}; +use ic_registry_subnet_type::SubnetType; +use ic_system_test_driver::driver::group::SystemTestGroup; +use ic_system_test_driver::driver::test_env_api::{GetFirstHealthyNodeSnapshot, HasPublicApiUrl}; +use ic_system_test_driver::driver::{ + ic::{InternetComputer, Subnet}, + test_env::TestEnv, +}; +use ic_system_test_driver::systest; +use ic_system_test_driver::util::{UniversalCanister, block_on, create_and_install}; +use ic_universal_canister::{UNIVERSAL_CANISTER_WASM, call_args, wasm}; + +fn main() -> Result<()> { + SystemTestGroup::new() + .with_setup(setup) + .add_test(systest!(subnet_metrics_another_subnet_succeeds)) + .execute_from_args()?; + + Ok(()) +} + +pub fn setup(env: TestEnv) { + InternetComputer::new() + .add_subnet(Subnet::fast_single_node(SubnetType::System)) + .add_subnet(Subnet::fast_single_node(SubnetType::VerifiedApplication)) + .add_subnet(Subnet::fast_single_node(SubnetType::Application)) + .setup_and_start(&env) + .expect("failed to setup IC under test"); +} + +/// A canister on the application subnet calls `subnet_metrics` naming a +/// *different* subnet. Message routing delivers the call to that subnet, which +/// executes it and answers with **its own** metrics. +/// +/// That a reply arrives at all already shows the call reached the named subnet, +/// since the handler rejects any `subnet_id` other than the executing subnet's +/// own. What the assertions below add is that the numbers in the reply are the +/// *remote* subnet's: the test perturbs only that subnet, by installing a +/// canister there, and pins how `num_canisters` moves. Were the reply carrying +/// the caller's own subnet's metrics, a canister created on the remote subnet +/// could not have moved it. +/// +/// This test has an IC to itself and is the only test in its group, so nothing +/// else creates or deletes canisters on the remote subnet in the meantime and the +/// canister count can be pinned exactly. +/// +/// Note also: unlike `node_metrics_history_another_subnet_succeeds` in +/// `general_execution_tests/api_tests.rs`, which calls +/// `get_first_healthy_application_node_snapshot()` twice and so ends up naming +/// its *own* subnet, this test names the verified-application subnet, so the call +/// really does cross a subnet boundary. +pub fn subnet_metrics_another_subnet_succeeds(env: TestEnv) { + // Arrange. + let app_node = env.get_first_healthy_application_node_snapshot(); + let agent = app_node.build_default_agent(); + let other_node = env.get_first_healthy_verified_application_node_snapshot(); + let other_agent = other_node.build_default_agent(); + let logger = env.logger(); + let other_subnet_id = other_node.subnet_id().unwrap().get(); + assert_ne!(other_subnet_id, app_node.subnet_id().unwrap().get()); + block_on({ + async move { + let canister = UniversalCanister::new_with_retries( + &agent, + app_node.effective_canister_id(), + &logger, + ) + .await; + + let read_remote = || async { + let result = canister + .update( + wasm().call_simple( + ic00::IC_00, + Method::SubnetMetrics, + call_args().other_side( + ic00::SubnetMetricsArgs { + subnet_id: other_subnet_id, + } + .encode(), + ), + ), + ) + .await; + let bytes = result.expect("cross-subnet subnet_metrics call failed"); + Decode!(&bytes, ic00::SubnetMetricsResponse).unwrap() + }; + + // Act. + let before = read_remote().await; + // Perturb only the remote subnet, by exactly one canister. Note the + // retry-free helper: `UniversalCanister::new_with_retries` retries + // creation *and* installation together, so a failed install would + // leave a canister behind and the exact assertion below would fail + // spuriously. + let _remote_canister = create_and_install( + &other_agent, + other_node.effective_canister_id(), + &UNIVERSAL_CANISTER_WASM, + ) + .await; + let after = read_remote().await; + + // Assert: the reply reports the *target* subnet's population, so + // creating exactly one canister there moves it by exactly one. + assert_eq!( + after.num_canisters, + before.num_canisters.clone() + candid::Nat::from(1_u64), + "cross-subnet subnet_metrics did not report the target subnet's \ + canister population: num_canisters was {} before and {} after \ + creating a canister on subnet {other_subnet_id}", + before.num_canisters, + after.num_canisters, + ); + // Sanity: the counters advance on the target subnet too. Both strict + // comparisons also pin the fields non-zero, since `before` cannot be + // negative. + assert!(after.block_height > before.block_height); + assert!(after.update_transactions_total > before.update_transactions_total); + + // `canister_state_bytes` is refreshed only on rounds whose batch number + // is a multiple of 10 (`rs/messaging/src/message_routing.rs`), so it + // legitimately reads 0 for the first rounds after the subnet is created. + // Whether a given read lands before or after a refresh is a race, so + // re-read until it is populated instead of assuming. Each read is an + // update executing on the target subnet, so it advances at least one + // round there; exhausting the bound means the field never refreshed, + // which is a real failure. + let mut canister_state_bytes = after.canister_state_bytes; + for _ in 0..30 { + if canister_state_bytes > 0_u64 { + break; + } + canister_state_bytes = read_remote().await.canister_state_bytes; + } + assert!( + canister_state_bytes > 0_u64, + "canister_state_bytes never refreshed off 0 across 30 rounds on \ + subnet {other_subnet_id}; expected a multiple-of-10 batch to have \ + refreshed it by now" + ); + } + }) +} diff --git a/rs/types/management_canister_types/src/lib.rs b/rs/types/management_canister_types/src/lib.rs index 5742a153ce1c..dc3f87962a13 100644 --- a/rs/types/management_canister_types/src/lib.rs +++ b/rs/types/management_canister_types/src/lib.rs @@ -122,6 +122,7 @@ pub enum Method { // Subnet information NodeMetricsHistory, + SubnetMetrics, SubnetInfo, FetchCanisterLogs, @@ -3796,6 +3797,49 @@ pub struct SubnetInfoResponse { impl Payload<'_> for SubnetInfoResponse {} +/// `CandidType` for `SubnetMetricsArgs` +/// ```text +/// record { +/// subnet_id : principal; +/// } +/// ``` +#[derive(Clone, Debug, Default, CandidType, Deserialize)] +pub struct SubnetMetricsArgs { + pub subnet_id: PrincipalId, +} + +impl Payload<'_> for SubnetMetricsArgs {} + +/// `CandidType` for `SubnetMetricsResponse` +/// ```text +/// record { +/// block_height : nat; +/// num_canisters : nat; +/// canister_state_bytes : nat; +/// consumed_cycles_total : nat; +/// update_transactions_total : nat; +/// } +/// ``` +/// +/// Freshness: only `block_height` is current as of the block in which the call is +/// executed. The other four are read from `SystemMetadata::subnet_metrics`, which is +/// written at the *end* of a round, so they are as of the end of the previous round +/// — except `canister_state_bytes`, which message routing recomputes only every 10 +/// rounds (summing it over every canister is expensive and it need not be exact), +/// so it can be up to ten rounds stale and reads as `0` for the first rounds after +/// the subnet is created. These are the same values, with the same staleness, that +/// `read_state` serves at `/subnet//metrics`. +#[derive(Clone, Debug, Deserialize, CandidType, Serialize, PartialEq)] +pub struct SubnetMetricsResponse { + pub block_height: candid::Nat, + pub num_canisters: candid::Nat, + pub canister_state_bytes: candid::Nat, + pub consumed_cycles_total: candid::Nat, + pub update_transactions_total: candid::Nat, +} + +impl Payload<'_> for SubnetMetricsResponse {} + /// `CandidType` for `NodeMetricsHistoryArgs` /// ```text /// record { diff --git a/rs/types/management_canister_types/tests/candid_equality.rs b/rs/types/management_canister_types/tests/candid_equality.rs index 3b4c5bf99e2c..2c29faaefd90 100644 --- a/rs/types/management_canister_types/tests/candid_equality.rs +++ b/rs/types/management_canister_types/tests/candid_equality.rs @@ -16,6 +16,7 @@ type CanisterInfoResult = CanisterInfoResponse; type CanisterMetadataArgs = CanisterMetadataRequest; type CanisterMetadataResult = CanisterMetadataResponse; type SubnetInfoResult = SubnetInfoResponse; +type SubnetMetricsResult = SubnetMetricsResponse; type DeleteCanisterArgs = CanisterIdRecord; type DepositCyclesArgs = CanisterIdRecord; type RawRandResult = Vec; @@ -151,6 +152,11 @@ fn node_metrics_history(_: NodeMetricsHistoryArgs) -> NodeMetricsHistoryResult { unreachable!() } +#[candid_method(update)] +fn subnet_metrics(_: SubnetMetricsArgs) -> SubnetMetricsResult { + unreachable!() +} + #[candid_method(update)] fn provisional_create_canister_with_cycles( _: ProvisionalCreateCanisterWithCyclesArgs, diff --git a/rs/types/management_canister_types/tests/ic.did b/rs/types/management_canister_types/tests/ic.did index 2a52b291f8fe..26de23a9ab78 100644 --- a/rs/types/management_canister_types/tests/ic.did +++ b/rs/types/management_canister_types/tests/ic.did @@ -453,6 +453,38 @@ type node_metrics_history_result = vec record { node_metrics : vec node_metrics; }; +type subnet_metrics_args = record { + subnet_id : principal; +}; + +// This API is EXPERIMENTAL and may evolve in a non-backward-compatible way. +// +// Only `block_height` is current as of the block in which the call is executed. +// The other four fields are read from the subnet's aggregated metrics, which the +// replica updates at the *end* of a round, so they describe the state as of an +// earlier block; see the individual fields. +type subnet_metrics_result = record { + // Height of the block in whose execution this call is processed. + // Monotonically non-decreasing for a given subnet; the heights of different + // subnets are unrelated. + block_height : nat; + // Number of canisters on the subnet, as of the end of the previous round. + num_canisters : nat; + // Total size in bytes of the state taken by the canisters on the subnet, as + // of the end of the previous round. Recomputed only every 10 rounds, because + // summing it over every canister is expensive and it does not need to be + // exact, so it can be up to ten rounds stale and reads as 0 for the first + // rounds after the subnet is created. + canister_state_bytes : nat; + // Total cycles removed from circulation on the subnet by all current and + // deleted canisters, as of the end of the previous round. + consumed_cycles_total : nat; + // Total number of transactions processed on the subnet, i.e. the total + // number of messages executed in replicated mode, as of the end of the + // previous round. + update_transactions_total : nat; +}; + type subnet_info_args = record { subnet_id : principal; }; @@ -694,6 +726,7 @@ service ic : { // metrics interface node_metrics_history : (node_metrics_history_args) -> (node_metrics_history_result); + subnet_metrics : (subnet_metrics_args) -> (subnet_metrics_result); // subnet info subnet_info : (subnet_info_args) -> (subnet_info_result); diff --git a/rs/types/types/src/messages/ingress_messages.rs b/rs/types/types/src/messages/ingress_messages.rs index 0685491cc8f8..19ee6f7a31e1 100644 --- a/rs/types/types/src/messages/ingress_messages.rs +++ b/rs/types/types/src/messages/ingress_messages.rs @@ -702,6 +702,7 @@ pub fn extract_effective_canister_id( | Ok(Method::BitcoinGetSuccessors) | Ok(Method::BitcoinGetCurrentFeePercentiles) | Ok(Method::NodeMetricsHistory) + | Ok(Method::SubnetMetrics) | Ok(Method::SubnetInfo) | Ok(Method::FetchCanisterLogs) => { // Subnet method not allowed for ingress. diff --git a/rs/types/types/src/messages/inter_canister.rs b/rs/types/types/src/messages/inter_canister.rs index 135635864aa2..fd57f9b31162 100644 --- a/rs/types/types/src/messages/inter_canister.rs +++ b/rs/types/types/src/messages/inter_canister.rs @@ -287,6 +287,7 @@ impl Request { | Ok(Method::BitcoinGetSuccessors) | Ok(Method::BitcoinGetCurrentFeePercentiles) | Ok(Method::NodeMetricsHistory) + | Ok(Method::SubnetMetrics) | Ok(Method::SubnetInfo) => { // No effective canister id. None