Skip to content

Commit 0cce6d9

Browse files
xli-oaicopyberry
authored andcommitted
Export turn cost as an OTEL metric
## What changed - Emit `codex.turn.cost_microusd` as a counter with turn, conversation, interruption, speed, and reasoning-effort attributes. Convert the estimated USD string to microdollars, rounding to the nearest microdollar and skipping values that cannot be represented safely. - Start turn-cost collection when an OTLP metrics exporter is configured, even when the OTEL log exporter is disabled. ## Testing - Verify cost conversion, rounding, and metric attributes with an in-memory metrics snapshot. - Verify that a metrics-only OTLP configuration starts the turn-cost worker. GitOrigin-RevId: e60587079f097f10cc1b6d1bf889d3f7a0dc3d5f
1 parent fb9311d commit 0cce6d9

7 files changed

Lines changed: 151 additions & 2 deletions

File tree

codex-rs/app-server/src/turn_cost_worker.rs

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,10 +95,16 @@ enum BackendAvailability {
9595

9696
impl TurnCostWorker {
9797
pub(crate) fn spawn(config: Arc<Config>, auth_manager: Arc<AuthManager>) -> Option<Self> {
98-
if !matches!(
98+
let has_otel_log_exporter = matches!(
9999
config.otel.exporter,
100100
OtelExporterKind::OtlpHttp { .. } | OtelExporterKind::OtlpGrpc { .. }
101-
) || config.model_provider.is_amazon_bedrock()
101+
);
102+
let has_otel_metrics_exporter = matches!(
103+
config.otel.metrics_exporter,
104+
OtelExporterKind::OtlpHttp { .. } | OtelExporterKind::OtlpGrpc { .. }
105+
);
106+
if !(has_otel_log_exporter || has_otel_metrics_exporter)
107+
|| config.model_provider.is_amazon_bedrock()
102108
{
103109
return None;
104110
}

codex-rs/app-server/src/turn_cost_worker_tests.rs

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,39 @@ use wiremock::matchers::path;
2121

2222
const TURN_COST_PATH: &str = "/v1/analytics/codex/turn-costs";
2323

24+
#[tokio::test]
25+
async fn worker_starts_with_otlp_metrics_exporter_without_log_exporter() {
26+
let server = MockServer::start().await;
27+
Mock::given(method("POST"))
28+
.and(path(TURN_COST_PATH))
29+
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
30+
"turns": []
31+
})))
32+
.expect(1)
33+
.mount(&server)
34+
.await;
35+
let codex_home = TempDir::new().expect("temporary Codex home");
36+
let mut config = ConfigBuilder::default()
37+
.codex_home(codex_home.path().to_path_buf())
38+
.build()
39+
.await
40+
.expect("test config");
41+
config.chatgpt_base_url = server.uri();
42+
config.otel.exporter = OtelExporterKind::None;
43+
config.otel.metrics_exporter = OtelExporterKind::OtlpGrpc {
44+
endpoint: server.uri(),
45+
headers: HashMap::new(),
46+
tls: None,
47+
};
48+
let auth_manager = AuthManager::from_auth_for_testing(CodexAuth::from_api_key("sk-test"));
49+
50+
let worker = TurnCostWorker::spawn(Arc::new(config), auth_manager)
51+
.expect("OTLP metrics exporter should enable turn-cost collection");
52+
wait_for_request_count(&server, /*expected*/ 1).await;
53+
worker.shutdown();
54+
server.verify().await;
55+
}
56+
2457
#[tokio::test]
2558
async fn handle_observes_only_matching_model_provider() {
2659
let codex_home = TempDir::new().expect("temporary Codex home");

codex-rs/otel/src/events/session_telemetry.rs

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ use crate::metrics::STARTUP_PHASE_DURATION_METRIC;
2323
use crate::metrics::SessionMetricTagValues;
2424
use crate::metrics::TOOL_CALL_COUNT_METRIC;
2525
use crate::metrics::TOOL_CALL_DURATION_METRIC;
26+
use crate::metrics::TURN_COST_MICROUSD_METRIC;
2627
use crate::metrics::TURN_TTFT_DURATION_METRIC;
2728
use crate::metrics::WEBSOCKET_EVENT_COUNT_METRIC;
2829
use crate::metrics::WEBSOCKET_EVENT_DURATION_METRIC;
@@ -281,6 +282,45 @@ impl SessionTelemetry {
281282
speed: Option<&str>,
282283
reasoning_effort: Option<&str>,
283284
) {
285+
let (dollars, fractional) = estimated_usd.split_once('.').unwrap_or((estimated_usd, ""));
286+
let fractional = fractional.as_bytes();
287+
let fractional_precision = 6_usize;
288+
let estimated_microusd = dollars.parse::<u64>().ok().and_then(|dollars| {
289+
if !fractional.iter().all(u8::is_ascii_digit) {
290+
return None;
291+
}
292+
let fractional_microusd = fractional
293+
.iter()
294+
.take(fractional_precision)
295+
.fold(0_u64, |value, digit| value * 10 + u64::from(digit - b'0'))
296+
* 10_u64.pow(fractional_precision.saturating_sub(fractional.len()) as u32);
297+
let round_up = fractional
298+
.get(fractional_precision)
299+
.is_some_and(|digit| *digit >= b'5');
300+
let estimated_microusd = dollars
301+
.checked_mul(1_000_000)?
302+
.checked_add(fractional_microusd)?
303+
.checked_add(u64::from(round_up))?;
304+
i64::try_from(estimated_microusd).ok()
305+
});
306+
if let Some(estimated_microusd) = estimated_microusd {
307+
let conversation_id = self.metadata.conversation_id.to_string();
308+
let mut tags = vec![
309+
("turn.id", turn_id),
310+
("conversation.id", conversation_id.as_str()),
311+
(
312+
"turn.interrupted",
313+
if interrupted { "true" } else { "false" },
314+
),
315+
];
316+
if let Some(speed) = speed {
317+
tags.push(("speed", speed));
318+
}
319+
if let Some(reasoning_effort) = reasoning_effort {
320+
tags.push(("reasoning_effort", reasoning_effort));
321+
}
322+
self.counter(TURN_COST_MICROUSD_METRIC, estimated_microusd, &tags);
323+
}
284324
log_event!(
285325
self,
286326
event.name = "codex.turn_cost",

codex-rs/otel/src/metrics/config.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ use crate::metrics::names::RESPONSES_API_ENGINE_SERVICE_TBT_DURATION_METRIC;
77
use crate::metrics::names::RESPONSES_API_ENGINE_SERVICE_TTFT_DURATION_METRIC;
88
use crate::metrics::names::TOOL_CALL_COUNT_METRIC;
99
use crate::metrics::names::TOOL_CALL_DURATION_METRIC;
10+
use crate::metrics::names::TURN_COST_MICROUSD_METRIC;
1011
use crate::metrics::names::TURN_TOKEN_USAGE_METRIC;
1112
use crate::metrics::validation::validate_tag_key;
1213
use crate::metrics::validation::validate_tag_value;
@@ -27,6 +28,7 @@ const STATSIG_DISABLED_METRICS: &[&str] = &[
2728
RESPONSES_API_ENGINE_SERVICE_TTFT_DURATION_METRIC,
2829
TOOL_CALL_COUNT_METRIC,
2930
TOOL_CALL_DURATION_METRIC,
31+
TURN_COST_MICROUSD_METRIC,
3032
TURN_TOKEN_USAGE_METRIC,
3133
];
3234

codex-rs/otel/src/metrics/names.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ pub const TURN_NETWORK_PROXY_METRIC: &str = "codex.turn.network_proxy";
3131
pub const TURN_MEMORY_METRIC: &str = "codex.turn.memory";
3232
pub const TURN_TOOL_CALL_METRIC: &str = "codex.turn.tool.call";
3333
pub const TURN_TOKEN_USAGE_METRIC: &str = "codex.turn.token_usage";
34+
pub const TURN_COST_MICROUSD_METRIC: &str = "codex.turn.cost_microusd";
3435
pub const TURN_UNIFIED_EXEC_RUNNING_PROCESSES_METRIC: &str =
3536
"codex.turn.unified_exec.running_processes";
3637
pub const GUARDIAN_REVIEW_COUNT_METRIC: &str = "codex.guardian.review";

codex-rs/otel/src/provider.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -604,6 +604,7 @@ mod tests {
604604
use crate::metrics::RESPONSES_API_ENGINE_SERVICE_TTFT_DURATION_METRIC;
605605
use crate::metrics::TOOL_CALL_COUNT_METRIC;
606606
use crate::metrics::TOOL_CALL_DURATION_METRIC;
607+
use crate::metrics::TURN_COST_MICROUSD_METRIC;
607608
use crate::metrics::TURN_TOKEN_USAGE_METRIC;
608609
use opentelemetry_sdk::metrics::InMemoryMetricExporter;
609610
use pretty_assertions::assert_eq;
@@ -741,6 +742,7 @@ mod tests {
741742
)?;
742743
metrics.counter(TOOL_CALL_COUNT_METRIC, /*inc*/ 1, &[])?;
743744
metrics.record_duration(TOOL_CALL_DURATION_METRIC, Duration::from_millis(25), &[])?;
745+
metrics.counter(TURN_COST_MICROUSD_METRIC, /*inc*/ 1, &[])?;
744746
metrics.histogram(TURN_TOKEN_USAGE_METRIC, /*value*/ 100, &[])?;
745747
metrics.counter("codex.turns", /*inc*/ 1, &[])?;
746748
metrics.shutdown()?;

codex-rs/otel/tests/suite/snapshot.rs

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -155,3 +155,68 @@ fn manager_snapshot_metrics_collects_without_shutdown() -> Result<()> {
155155

156156
Ok(())
157157
}
158+
159+
#[test]
160+
fn manager_turn_cost_records_microusd_metric() -> Result<()> {
161+
let exporter = InMemoryMetricExporter::default();
162+
let config = MetricsConfig::in_memory("test", "codex-cli", env!("CARGO_PKG_VERSION"), exporter)
163+
.with_runtime_reader();
164+
let metrics = MetricsClient::new(config)?;
165+
let thread_id = ThreadId::new();
166+
let conversation_id = thread_id.to_string();
167+
let manager = SessionTelemetry::new(
168+
thread_id,
169+
"gpt-5.6",
170+
"gpt-5.6",
171+
/*account_id*/ None,
172+
/*account_email*/ None,
173+
Some(TelemetryAuthMode::ApiKey),
174+
"test_originator".to_string(),
175+
/*log_user_prompts*/ false,
176+
"tty".to_string(),
177+
SessionSource::Cli,
178+
)
179+
.with_metrics(metrics);
180+
181+
manager.record_turn_cost(
182+
"turn-123",
183+
"0.0001245",
184+
/*interrupted*/ false,
185+
Some("fast"),
186+
Some("high"),
187+
);
188+
189+
let snapshot = manager.snapshot_metrics()?;
190+
let metric = find_metric(&snapshot, "codex.turn.cost_microusd")
191+
.expect("turn-cost microdollar metric missing");
192+
let point = match metric.data() {
193+
AggregatedMetrics::U64(MetricData::Sum(sum)) => {
194+
sum.data_points().next().expect("turn-cost data point")
195+
}
196+
_ => panic!("unexpected turn-cost metric data type"),
197+
};
198+
assert_eq!(point.value(), 125);
199+
assert_eq!(
200+
attributes_to_map(point.attributes()),
201+
BTreeMap::from([
202+
(
203+
"app.version".to_string(),
204+
env!("CARGO_PKG_VERSION").to_string(),
205+
),
206+
(
207+
"auth_mode".to_string(),
208+
TelemetryAuthMode::ApiKey.to_string(),
209+
),
210+
("conversation.id".to_string(), conversation_id),
211+
("model".to_string(), "gpt-5.6".to_string()),
212+
("originator".to_string(), "test_originator".to_string()),
213+
("reasoning_effort".to_string(), "high".to_string()),
214+
("session_source".to_string(), "cli".to_string()),
215+
("speed".to_string(), "fast".to_string()),
216+
("turn.id".to_string(), "turn-123".to_string()),
217+
("turn.interrupted".to_string(), "false".to_string()),
218+
])
219+
);
220+
221+
Ok(())
222+
}

0 commit comments

Comments
 (0)