Skip to content
45 changes: 26 additions & 19 deletions core/service/src/bin/kms-server.rs
Original file line number Diff line number Diff line change
Expand Up @@ -616,8 +616,8 @@ async fn main_exec() -> anyhow::Result<()> {
.unwrap_or_else(|e| panic!("Could not bind to {service_socket_addr} \n {e:?}"));

// load key
let base_kms = match get_core_signing_key(&private_vault).await {
Ok(sk) => BaseKmsStruct::new(kms_type, sk)?,
let (base_kms, recovery_mode) = match get_core_signing_key(&private_vault).await {
Ok(sk) => (BaseKmsStruct::new(kms_type, sk)?, false),
Err(e) => {
tracing::warn!("Error loading signing key: {e:?}");
tracing::warn!(
Expand All @@ -627,7 +627,7 @@ async fn main_exec() -> anyhow::Result<()> {
let verf_key = public_storage
.read_data(&SIGNING_KEY_ID, &PubDataType::VerfKey.to_string())
.await?;
BaseKmsStruct::new_no_signing_key(kms_type, verf_key)
(BaseKmsStruct::new_no_signing_key(kms_type, verf_key), true)
}
};

Expand All @@ -645,22 +645,29 @@ async fn main_exec() -> anyhow::Result<()> {
let mpc_listener = make_mpc_listener(threshold_config).await;

let tls_identity = match &threshold_config.tls {
Some(tls_config) => Some({
build_tls_config(
&threshold_config.peers,
tls_config,
security_module.clone(),
private_vault
.keychain
.as_ref()
.map(|x| x.root_key_measurements()),
&public_vault,
base_kms.sig_key()?,
#[cfg(feature = "insecure")]
core_config.mock_enclave.is_some_and(|m| m),
)
.await?
}),
Some(tls_config) => {
if recovery_mode {
tracing::warn!("TLS identity is configured but signing key is not available, TLS will not be available in recovery mode");
Comment thread
jot2re marked this conversation as resolved.
Outdated
None
} else {
Some(
build_tls_config(
&threshold_config.peers,
tls_config,
security_module.clone(),
private_vault
.keychain
.as_ref()
.map(|x| x.root_key_measurements()),
&public_vault,
base_kms.sig_key()?,
#[cfg(feature = "insecure")]
core_config.mock_enclave.is_some_and(|m| m),
)
.await?,
)
}
}
None => {
tracing::warn!(
"No TLS identity - using plaintext communication between MPC nodes"
Expand Down
22 changes: 2 additions & 20 deletions core/service/src/conf/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ use crate::util::rate_limiter::RateLimiterConfig;
use clap::ValueEnum;
use observability::{
conf::{Settings, TelemetryConfig},
telemetry::{init_telemetry, SdkMeterProvider, SdkTracerProvider},
telemetry::{init_telemetry, ConfigTracing, SdkMeterProvider, SdkTracerProvider},
};
use serde::{Deserialize, Serialize};
use std::{cmp, path::PathBuf};
Expand Down Expand Up @@ -78,10 +78,6 @@ pub struct ServiceEndpoint {
pub grpc_max_message_size: usize,
}

pub trait ConfigTracing {
fn telemetry(&self) -> Option<TelemetryConfig>;
}

impl ConfigTracing for CoreConfig {
fn telemetry(&self) -> Option<TelemetryConfig> {
self.telemetry.clone()
Expand Down Expand Up @@ -214,24 +210,10 @@ pub async fn init_conf_kms_core_telemetry<
) -> anyhow::Result<(T, SdkTracerProvider, SdkMeterProvider)> {
let full_config: T = init_conf(config_file)?;
full_config.validate()?;
let telemetry = full_config.telemetry().unwrap_or_else(|| {
TelemetryConfig::builder()
.tracing_service_name("kms_core".to_string())
.build()
});
let (tracer_provider, meter_provider) = init_telemetry(&telemetry).await?;
let (tracer_provider, meter_provider) = init_telemetry(&full_config).await?;
Ok((full_config, tracer_provider, meter_provider))
}

/// Initialize the tracing configuration with default values
pub async fn init_kms_core_telemetry() -> anyhow::Result<(SdkTracerProvider, SdkMeterProvider)> {
let telemetry = TelemetryConfig::builder()
.tracing_service_name("kms_core".to_string())
.build();
let (tracer_provider, meter_provider) = init_telemetry(&telemetry).await?;
Ok((tracer_provider, meter_provider))
}

#[cfg(test)]
mod tests {
use std::path::PathBuf;
Expand Down
9 changes: 7 additions & 2 deletions core/service/src/conf/threshold.rs
Original file line number Diff line number Diff line change
Expand Up @@ -120,8 +120,13 @@ pub enum TlsCert {
impl TlsCert {
pub fn unchecked_cert_string(&self) -> anyhow::Result<String> {
match self {
TlsCert::Path(ref cert_path) => std::fs::read_to_string(cert_path)
.map_err(|e| anyhow::anyhow!("Failed to open file {}: {}", cert_path.display(), e)),
TlsCert::Path(ref cert_path) => std::fs::read_to_string(cert_path).map_err(|e| {
anyhow::anyhow!(
"Failed to open TLS cert file {}: {}",
cert_path.display(),
e
)
}),
TlsCert::Pem(ref cert_bytes) => Ok(cert_bytes.to_string()),
}
}
Expand Down
4 changes: 3 additions & 1 deletion docs/operations/advanced/troubleshooting.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,9 @@ kubectl top pods -n <NAMESPACE>
# Health check (may not be available in Chainguard enclave containers)
# Use external health check or test endpoints directly
kubectl port-forward -n <NAMESPACE> <POD_NAME> 9646:9646 &
curl -f http://localhost:9646/health
curl -f http://localhost:9646/health # Health status
curl -f http://localhost:9646/version # Version of the KMS running
curl -f http://localhost:9646/config # The configuration used
Comment thread
jot2re marked this conversation as resolved.

# Recent logs and events - replace <POD_NAME> with actual pod name
kubectl logs -n <NAMESPACE> <POD_NAME> --tail=100 | grep -E "(ERROR|FATAL|PANIC|WARN)"
Expand Down
52 changes: 39 additions & 13 deletions observability/src/telemetry.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ use opentelemetry_sdk::propagation::{BaggagePropagator, TraceContextPropagator};
pub use opentelemetry_sdk::trace::SdkTracerProvider;
use opentelemetry_sdk::{resource::Resource, trace::Sampler};
use prometheus::{Encoder, Registry as PrometheusRegistry, TextEncoder};
use serde::Serialize;
use std::{
env,
net::SocketAddr,
Expand All @@ -38,6 +39,7 @@ use tracing_subscriber::fmt::format::FmtSpan;
use tracing_subscriber::fmt::{layer, Layer};
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::{util::SubscriberInitExt, EnvFilter};
use validator::Validate;

#[cfg(target_os = "linux")]
use prometheus::process_collector::ProcessCollector;
Expand All @@ -46,15 +48,21 @@ use prometheus::process_collector::ProcessCollector;
pub const TRACER_REQUEST_ID: &str = "x-zama-kms-request-id";
pub const TRACER_PARENT_SPAN_ID: &str = "x-zama-kms-parent-span-id";

pub trait ConfigTracing {
fn telemetry(&self) -> Option<TelemetryConfig>;
}

#[derive(Clone)]
struct MetricsState {
config: String, // Store the config as a string for the /config endpoint
registry: Arc<PrometheusRegistry>,
start_time: std::time::SystemTime,
Comment thread
jot2re marked this conversation as resolved.
}

impl MetricsState {
fn new(registry: PrometheusRegistry) -> Self {
fn new(registry: PrometheusRegistry, config: String) -> Self {
Self {
config,
registry: Arc::new(registry),
start_time: std::time::SystemTime::now(),
}
Expand Down Expand Up @@ -99,11 +107,24 @@ async fn liveness_handler() -> impl IntoResponse {
(StatusCode::OK, "alive")
}

pub fn init_metrics(settings: &TelemetryConfig) -> Result<SdkMeterProvider, anyhow::Error> {
async fn config_handler(State(state): State<MetricsState>) -> impl IntoResponse {
(StatusCode::OK, state.config)
}
Comment thread
jot2re marked this conversation as resolved.

pub fn init_metrics<T: Serialize + std::fmt::Debug + ConfigTracing + Validate>(
Comment thread
jot2re marked this conversation as resolved.
Outdated
config: &T,
) -> Result<SdkMeterProvider, anyhow::Error> {
if matches!(*ENVIRONMENT, ExecutionEnvironment::Integration) {
return Ok(SdkMeterProvider::default());
}

let telemetry_settings = config.telemetry().unwrap_or_else(|| {
tracing::warn!("No telemetry configuration found, using defaults");
TelemetryConfig::builder()
.tracing_service_name("kms_core".to_string())
.build()
});
let config_json = serde_json::to_string_pretty(&config)
.map_err(|e| anyhow::anyhow!("Failed to serialize configuration: {:?}", e))?;
let registry = PrometheusRegistry::new();

// Add process collector for system metrics
Expand All @@ -123,7 +144,7 @@ pub fn init_metrics(settings: &TelemetryConfig) -> Result<SdkMeterProvider, anyh
.with_attributes(vec![
KeyValue::new(
opentelemetry_semantic_conventions::resource::SERVICE_NAME.to_string(),
settings
telemetry_settings
.tracing_service_name()
.unwrap_or("unknown-service")
.to_string(),
Expand All @@ -144,13 +165,13 @@ pub fn init_metrics(settings: &TelemetryConfig) -> Result<SdkMeterProvider, anyh
opentelemetry::global::set_meter_provider(provider.clone());

// Start metrics server if configured
let metrics_addr = settings
let metrics_addr = telemetry_settings
.metrics_bind_address()
.unwrap_or("0.0.0.0:9464")
.parse::<SocketAddr>()
Comment thread
jot2re marked this conversation as resolved.
.context("Failed to parse metrics bind address")?;

let state = MetricsState::new(registry);
let state = MetricsState::new(registry, config_json);

// Use the global METRICS instance also as a sanity check that metrics are working
METRICS.increment_request_counter("system_startup");
Expand All @@ -165,6 +186,7 @@ pub fn init_metrics(settings: &TelemetryConfig) -> Result<SdkMeterProvider, anyh
.route("/ready", get(readiness_handler))
.route("/version", get(version_handler))
.route("/live", get(liveness_handler))
.route("/config", get(config_handler))
.with_state(state);

let listener = tokio::net::TcpListener::bind(metrics_addr)
Expand Down Expand Up @@ -406,24 +428,28 @@ pub async fn init_tracing(settings: &TelemetryConfig) -> Result<SdkTracerProvide
Ok(provider)
}

pub async fn init_telemetry(
settings: &TelemetryConfig,
pub async fn init_telemetry<T: Serialize + std::fmt::Debug + ConfigTracing + Validate>(
Comment thread
jot2re marked this conversation as resolved.
Outdated
config: &T,
) -> anyhow::Result<(SdkTracerProvider, SdkMeterProvider)> {
println!("Starting telemetry initialization...");

let telemetry_conf = config.telemetry().unwrap_or_else(|| {
TelemetryConfig::builder()
.tracing_service_name("kms_core".to_string())
.build()
});
// First initialize tracing as it's more critical
println!("Initializing tracing subsystem...");
let tracer_provider = init_tracing(settings).await?;
let tracer_provider = init_tracing(&telemetry_conf).await?;

// Now that tracing is initialized, we can use info! tracing macros
info!("Tracing initialization completed successfully");

println!("Initializing metrics subsystem...");
let meter_provider = init_metrics(settings)?;
let meter_provider = init_metrics(config)?;
info!("Metrics initialization completed successfully");

if settings.enable_sys_metrics() {
start_sys_metrics_collection(settings.refresh_interval())?;
if telemetry_conf.enable_sys_metrics() {
start_sys_metrics_collection(telemetry_conf.refresh_interval())?;
}

info!("Telemetry stack initialization completed");
Expand Down
Loading