Skip to content

Commit 3a8760c

Browse files
author
Miagkov, Oleg
committed
feat: add provider cost eval evidence
1 parent b7bed4b commit 3a8760c

8 files changed

Lines changed: 125 additions & 9 deletions

File tree

‎apps/backend/cli/runtime_commands.py‎

Lines changed: 71 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@
5858
PROVIDER_RELIABILITY_DIRECT_API_PROVIDERS,
5959
PROVIDER_SMOKE_HISTORY_RELATIVE_PATH,
6060
)
61+
from core.providers.cost_calculator import MODEL_PRICING, estimate_session_cost
6162

6263
DEFAULT_MCP_DIAGNOSTIC_SERVERS = tuple(MCP_SERVER_CATALOG)
6364
DEFAULT_EXTERNAL_MCP_SMOKE_SERVERS = registered_external_mcp_servers()
@@ -111,6 +112,8 @@
111112
"audit_artifact",
112113
"mutating_tool_classification",
113114
)
115+
RUNTIME_COMPARATIVE_COST_ESTIMATE_INPUT_TOKENS = 10_000
116+
RUNTIME_COMPARATIVE_COST_ESTIMATE_OUTPUT_TOKENS = 2_000
114117

115118
RUNTIME_POLICY_PHASES = (
116119
{
@@ -1066,6 +1069,54 @@ def _runtime_provider_eval_metrics(provider_stats: dict[str, Any]) -> dict[str,
10661069
}
10671070

10681071

1072+
def _runtime_provider_cost_pricing_model(model: Any) -> str | None:
1073+
"""Return a known pricing model from direct provider history."""
1074+
if not isinstance(model, str):
1075+
return None
1076+
trimmed = model.strip()
1077+
if not trimmed:
1078+
return None
1079+
if trimmed in MODEL_PRICING and trimmed != "default":
1080+
return trimmed
1081+
for segment in reversed(trimmed.split("/")):
1082+
if segment in MODEL_PRICING and segment != "default":
1083+
return segment
1084+
return None
1085+
1086+
1087+
def _runtime_provider_cost_estimate(provider_stats: dict[str, Any]) -> dict[str, Any]:
1088+
"""Return cost estimate evidence from the latest observed provider model."""
1089+
pricing_model = _runtime_provider_cost_pricing_model(
1090+
provider_stats.get("last_model")
1091+
)
1092+
if not pricing_model:
1093+
return {
1094+
"cost_status": "not_recorded",
1095+
"cost_pricing_model": None,
1096+
"cost_pricing_provider": None,
1097+
"cost_estimate_usd": None,
1098+
"cost_estimate_formatted": None,
1099+
"cost_estimate_input_tokens": RUNTIME_COMPARATIVE_COST_ESTIMATE_INPUT_TOKENS,
1100+
"cost_estimate_output_tokens": RUNTIME_COMPARATIVE_COST_ESTIMATE_OUTPUT_TOKENS,
1101+
}
1102+
1103+
estimate = estimate_session_cost(
1104+
pricing_model,
1105+
RUNTIME_COMPARATIVE_COST_ESTIMATE_INPUT_TOKENS,
1106+
RUNTIME_COMPARATIVE_COST_ESTIMATE_OUTPUT_TOKENS,
1107+
)
1108+
estimated_cost = estimate["estimated_cost"]
1109+
return {
1110+
"cost_status": "local_zero_cost" if estimated_cost == 0 else "estimated",
1111+
"cost_pricing_model": pricing_model,
1112+
"cost_pricing_provider": estimate.get("provider"),
1113+
"cost_estimate_usd": estimated_cost,
1114+
"cost_estimate_formatted": estimate["formatted"],
1115+
"cost_estimate_input_tokens": estimate["input_tokens"],
1116+
"cost_estimate_output_tokens": estimate["output_tokens"],
1117+
}
1118+
1119+
10691120
def _runtime_provider_live_fault_coverage_complete(
10701121
provider_stats: dict[str, Any],
10711122
) -> bool:
@@ -1227,6 +1278,7 @@ def build_runtime_eval_history(
12271278
"last_provider_e2e_status": provider_stats.get(
12281279
"last_provider_e2e_status",
12291280
),
1281+
"last_model": provider_stats.get("last_model"),
12301282
"last_run_at": provider_stats.get("last_run_at"),
12311283
**provider_metrics,
12321284
}
@@ -1277,11 +1329,25 @@ def build_runtime_comparative_eval_matrix(
12771329
has_full_runtime = provider_row.full_autonomous == "yes"
12781330
provider_stats = provider_history.get(provider, {})
12791331
provider_metrics = _runtime_provider_eval_metrics(provider_stats)
1332+
provider_cost = _runtime_provider_cost_estimate(provider_stats)
12801333
quality_status = (
12811334
"not_recorded"
12821335
if has_full_runtime
12831336
else str(provider_stats.get("status") or "not_observed")
12841337
)
1338+
cost_fields = (
1339+
{
1340+
"cost_status": "not_recorded",
1341+
"cost_pricing_model": None,
1342+
"cost_pricing_provider": None,
1343+
"cost_estimate_usd": None,
1344+
"cost_estimate_formatted": None,
1345+
"cost_estimate_input_tokens": None,
1346+
"cost_estimate_output_tokens": None,
1347+
}
1348+
if has_full_runtime
1349+
else provider_cost
1350+
)
12851351
matrix.append(
12861352
{
12871353
"provider": provider,
@@ -1295,7 +1361,7 @@ def build_runtime_comparative_eval_matrix(
12951361
"stability_score": None
12961362
if has_full_runtime
12971363
else provider_metrics["recent_pass_rate_percent"],
1298-
"cost_status": "not_recorded",
1364+
**cost_fields,
12991365
"safety_status": "native_runtime_policy"
13001366
if has_full_runtime
13011367
else "policy_gated",
@@ -1988,6 +2054,8 @@ def format_runtime_modes_text(payload: dict[str, Any] | None = None) -> str:
19882054
_format_optional_percent(row.get("quality_score")),
19892055
_format_optional_percent(row.get("stability_score")),
19902056
row["cost_status"],
2057+
row.get("cost_estimate_formatted") or "n/a",
2058+
row.get("cost_pricing_model") or "n/a",
19912059
row["safety_status"],
19922060
_format_optional_percent(row.get("safety_score")),
19932061
", ".join(row["blockers"]) or "none",
@@ -2187,6 +2255,8 @@ def format_runtime_modes_text(payload: dict[str, Any] | None = None) -> str:
21872255
"Quality score",
21882256
"Stability score",
21892257
"Cost",
2258+
"Cost estimate",
2259+
"Pricing model",
21902260
"Safety",
21912261
"Safety score",
21922262
"Blockers",

‎apps/frontend/src/renderer/components/settings/ProviderSettingsSection.test.tsx‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,7 @@ const translate = (key: string) =>
132132
'settings:aiProvider.runtimeDiagnosticValues.dynamicMutatingToolPolicy':
133133
'Dynamic mutating tool policy',
134134
'settings:aiProvider.runtimeDiagnosticValues.enforced': 'Enforced',
135+
'settings:aiProvider.runtimeDiagnosticValues.estimated': 'Estimated',
135136
'settings:aiProvider.runtimeDiagnosticValues.failed': 'Failed',
136137
'settings:aiProvider.runtimeDiagnosticValues.inspectDiff': 'Inspect diff',
137138
'settings:aiProvider.runtimeDiagnosticValues.isolated': 'Isolated',
@@ -202,6 +203,7 @@ const translate = (key: string) =>
202203
'Live provider e2e required',
203204
'settings:aiProvider.runtimeDiagnosticValues.localModelQualityVaries':
204205
'Local model quality varies',
206+
'settings:aiProvider.runtimeDiagnosticValues.localZeroCost': 'Local zero cost',
205207
'settings:aiProvider.runtimeDiagnosticValues.limited': 'Limited',
206208
'settings:aiProvider.runtimeDiagnosticValues.missingFullAutonomousRuntime':
207209
'Missing full autonomous runtime',
@@ -495,7 +497,12 @@ describe('buildRuntimeComparativeEvalDiagnosticRows', () => {
495497
quality_status: 'passed',
496498
quality_score: 75,
497499
stability_score: 75,
498-
cost_status: 'not_recorded',
500+
cost_status: 'estimated',
501+
cost_estimate_usd: 0.045,
502+
cost_estimate_formatted: '$0.0450',
503+
cost_pricing_model: 'gpt-4o',
504+
cost_estimate_input_tokens: 10000,
505+
cost_estimate_output_tokens: 2000,
499506
safety_status: 'policy_gated',
500507
safety_score: 50,
501508
evidence_source: '.auto-Codex/provider-smoke-history.json',
@@ -507,7 +514,7 @@ describe('buildRuntimeComparativeEvalDiagnosticRows', () => {
507514
{
508515
labelKey: 'settings:aiProvider.controlPlane.runtimeComparativeEval',
509516
value: (
510-
'OpenAI: Passed / Not recorded / Policy gated '
517+
'OpenAI: Passed / Estimated $0.0450 gpt-4o / Policy gated '
511518
+ '(Quality 75%, Stability 75%, Safety 50%; '
512519
+ 'Provider e2e, Generic edit recovery, MCP bridge contract)'
513520
),

‎apps/frontend/src/renderer/components/settings/ProviderSettingsSection.tsx‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -243,6 +243,7 @@ const RUNTIME_DIAGNOSTIC_TRANSLATION_KEYS: Record<string, string> = {
243243
enable_live_fault_probes:
244244
'settings:aiProvider.runtimeDiagnosticValues.enableLiveFaultProbes',
245245
error: 'settings:aiProvider.runtimeDiagnosticValues.error',
246+
estimated: 'settings:aiProvider.runtimeDiagnosticValues.estimated',
246247
external_mcp_client: 'settings:aiProvider.runtimeDiagnosticValues.externalMcpClient',
247248
failed: 'settings:aiProvider.runtimeDiagnosticValues.failed',
248249
fallback_active: 'settings:aiProvider.runtimeDiagnosticValues.fallbackActive',
@@ -290,6 +291,7 @@ const RUNTIME_DIAGNOSTIC_TRANSLATION_KEYS: Record<string, string> = {
290291
live_provider_e2e_required: 'settings:aiProvider.runtimeDiagnosticValues.liveProviderE2eRequired',
291292
local_bridge: 'settings:aiProvider.runtimeDiagnosticValues.localBridge',
292293
local_model_quality_varies: 'settings:aiProvider.runtimeDiagnosticValues.localModelQualityVaries',
294+
local_zero_cost: 'settings:aiProvider.runtimeDiagnosticValues.localZeroCost',
293295
limited: 'settings:aiProvider.runtimeDiagnosticValues.limited',
294296
limited_autonomous_until_evidence_stable:
295297
'settings:aiProvider.runtimeDiagnosticValues.limitedAutonomousUntilEvidenceStable',
@@ -699,6 +701,17 @@ function formatRuntimePercentMetric(
699701
return `${formatRuntimeDiagnosticValue(translate, key)} ${value}%`;
700702
}
701703

704+
function formatRuntimeComparativeEvalCost(
705+
translate: RuntimeDiagnosticTranslate,
706+
row: RuntimeComparativeEvalMatrixRow
707+
): string {
708+
const status = formatRuntimeDiagnosticValue(translate, row.cost_status);
709+
const details = [row.cost_estimate_formatted, row.cost_pricing_model]
710+
.filter((value): value is string => Boolean(value))
711+
.join(' ');
712+
return [status, details].filter(Boolean).join(' ');
713+
}
714+
702715
function formatRuntimeDiagnosticBoolean(
703716
translate: RuntimeDiagnosticTranslate,
704717
value?: boolean | null
@@ -1278,7 +1291,7 @@ export function buildRuntimeComparativeEvalDiagnosticRows(
12781291
.map((row) => {
12791292
const provider = formatRuntimeDiagnosticValue(translate, row.provider);
12801293
const quality = formatRuntimeDiagnosticValue(translate, row.quality_status);
1281-
const cost = formatRuntimeDiagnosticValue(translate, row.cost_status);
1294+
const cost = formatRuntimeComparativeEvalCost(translate, row);
12821295
const safety = formatRuntimeDiagnosticValue(translate, row.safety_status);
12831296
const blockers = formatRuntimeDiagnosticList(translate, row.blockers);
12841297
const metrics = [

‎apps/frontend/src/shared/i18n/locales/en/settings.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1004,6 +1004,7 @@
10041004
"enforced": "Enforced",
10051005
"enableLiveFaultProbes": "Enable live fault probes",
10061006
"error": "Error",
1007+
"estimated": "Estimated",
10071008
"externalMcpClient": "External MCP client",
10081009
"failed": "Failed",
10091010
"fallbackActive": "Fallback active",
@@ -1044,6 +1045,7 @@
10441045
"liveFaultProbeMissing": "Live fault probe missing",
10451046
"liveFaultProbesPassed": "Live fault probes passed",
10461047
"localBridge": "Local bridge",
1048+
"localZeroCost": "Local zero cost",
10471049
"localModelQualityVaries": "Local model quality varies",
10481050
"limited": "Limited",
10491051
"limitedAutonomousUntilEvidenceStable": "Limited autonomous until evidence stable",

‎apps/frontend/src/shared/i18n/locales/fr/settings.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -952,6 +952,7 @@
952952
"enforced": "Appliqué",
953953
"enableLiveFaultProbes": "Activer les probes live de panne",
954954
"error": "Erreur",
955+
"estimated": "Estimé",
955956
"externalMcpClient": "Client MCP externe",
956957
"failed": "Échoué",
957958
"fallbackActive": "Fallback actif",
@@ -992,6 +993,7 @@
992993
"liveFaultProbeMissing": "Probe live de panne manquante",
993994
"liveFaultProbesPassed": "Probes live de panne réussies",
994995
"localBridge": "Bridge local",
996+
"localZeroCost": "Coût local nul",
995997
"localModelQualityVaries": "Qualité du modèle local variable",
996998
"limited": "Limité",
997999
"limitedAutonomousUntilEvidenceStable": "Autonomie limitée jusqu'à stabilisation des preuves",

‎apps/frontend/src/shared/types/settings.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -938,6 +938,7 @@ export interface RuntimeEvalHistoryProviderRow {
938938
last_status?: string | null;
939939
last_reliability_status?: string | null;
940940
last_provider_e2e_status?: string | null;
941+
last_model?: string | null;
941942
last_run_at?: string | null;
942943
}
943944

@@ -960,6 +961,12 @@ export interface RuntimeComparativeEvalMatrixRow {
960961
quality_score?: number | null;
961962
stability_score?: number | null;
962963
cost_status: string;
964+
cost_pricing_model?: string | null;
965+
cost_pricing_provider?: string | null;
966+
cost_estimate_usd?: number | null;
967+
cost_estimate_formatted?: string | null;
968+
cost_estimate_input_tokens?: number | null;
969+
cost_estimate_output_tokens?: number | null;
963970
safety_status: string;
964971
safety_score?: number | null;
965972
evidence_source: string;

0 commit comments

Comments
 (0)