Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions backend-python/.env.example
Original file line number Diff line number Diff line change
Expand Up @@ -241,3 +241,9 @@ OBSERVABILITY_ENABLED=false
# staging/CI: 0.25 (25%)
# production: 0.05 (5%)
# OTEL_TRACES_SAMPLE_RATIO=1.0

# Cost accounting (honoured only when OBSERVABILITY_ENABLED=true).
# Rates live in git-tracked config/model_pricing.yaml — not env-overridable.
# OBSERVABILITY_COST_PRICING_FILE=config/model_pricing.yaml
# Must match pricing_version in the pricing file at startup.
# OBSERVABILITY_COST_PRICING_VERSION=2026-08
43 changes: 43 additions & 0 deletions backend-python/alembic/versions/0008_observability_usage_cost.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
"""0008 observability usage cost

Revision ID: 0008_observability_usage_cost
Revises: 0007_workflow_tables
Create Date: 2026-08-08

Epic 07 Phase 5: additive ``cost_usd`` / ``pricing_version`` on ``usage_events``.
"""

from typing import Sequence, Union

from alembic import op
import sqlalchemy as sa

revision: str = "0008_observability_usage_cost"
down_revision: Union[str, None] = "0007_workflow_tables"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None


def upgrade() -> None:
op.add_column(
"usage_events",
sa.Column("cost_usd", sa.Numeric(precision=12, scale=6), nullable=True),
)
op.add_column(
"usage_events",
sa.Column("pricing_version", sa.Text(), nullable=True),
)
op.create_index(
"ix_usage_events_provider_model_created",
"usage_events",
["provider", "model", "created_at"],
)


def downgrade() -> None:
op.drop_index(
"ix_usage_events_provider_model_created",
table_name="usage_events",
)
op.drop_column("usage_events", "pricing_version")
op.drop_column("usage_events", "cost_usd")
4 changes: 4 additions & 0 deletions backend-python/app/ai/observability/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
See ``docs/plans/post-mvp-v2-epic-07-observability-and-evaluation.md`` Part I § Public APIs.
"""

from app.ai.observability.cost.calculator import CostCalculator
from app.ai.observability.cost.pricing import ModelPricingTable
from app.ai.observability.exceptions import (
ObservabilityConfigError,
ObservabilityDisabledError,
Expand All @@ -28,7 +30,9 @@

__all__ = [
"OBSERVABILITY_ENABLED",
"CostCalculator",
"MeterRegistry",
"ModelPricingTable",
"ObservabilityConfigError",
"ObservabilityDisabledError",
"ObservabilityError",
Expand Down
11 changes: 11 additions & 0 deletions backend-python/app/ai/observability/cost/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
"""Cost accounting public API."""

from app.ai.observability.cost.calculator import CostCalculator, CostRegistry
from app.ai.observability.cost.pricing import ModelPricingEntry, ModelPricingTable

__all__ = [
"CostCalculator",
"CostRegistry",
"ModelPricingEntry",
"ModelPricingTable",
]
86 changes: 84 additions & 2 deletions backend-python/app/ai/observability/cost/calculator.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,87 @@
"""Approximate token cost calculator — implemented in Phase 5."""
"""Approximate token cost calculator — invoked only from ``SqlUsageStore.record()``."""

from __future__ import annotations

# TODO(epic-07): Phase 5 — CostCalculator.price(provider, model, usage).
from app.ai.observability.cost.pricing import ModelPricingTable
from app.ai.observability.exceptions import ObservabilityConfigError
from app.core.config import Settings
from app.core.logging import get_logger
from app.providers.base import ProviderUsage

logger = get_logger(__name__)


class CostCalculator:
"""Convert ``ProviderUsage`` into approximate USD cost using a pricing table."""

def __init__(self, table: ModelPricingTable) -> None:
self._table = table

@property
def pricing_version(self) -> str:
return self._table.pricing_version

@property
def pricing_table(self) -> ModelPricingTable:
return self._table

def price(
self,
provider: str,
model: str,
usage: ProviderUsage | None,
) -> tuple[float | None, str | None]:
if usage is None:
return None, None
if usage.prompt_tokens is None or usage.completion_tokens is None:
return None, None

entry = self._table.lookup(provider, model)
if entry is None:
return None, None

cost = (usage.prompt_tokens / 1000.0) * entry.input_usd_per_1k + (
usage.completion_tokens / 1000.0
) * entry.output_usd_per_1k
return cost, self._table.pricing_version
Comment on lines +34 to +46

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🗄️ Data Integrity & Integration | 🟡 Minor | ⚡ Quick win

🧩 Analysis chain

🏁 Script executed:

#!/bin/bash
set -euo pipefail

ast-grep outline backend-python/app/providers/base.py --match ProviderUsage --view expanded
rg -n -C 5 'class ProviderUsage|prompt_tokens|completion_tokens' backend-python/app/providers/base.py

Repository: pateatlau/fullstack-ai-platform

Length of output: 1060


🏁 Script executed:

#!/bin/bash
set -euo pipefail

echo "Calculator and SqlUsageStore references:"
rg -n -C 4 'calculate.*cost|record\(|record_usage|cost.*usage|ProviderUsage|prompt_tokens|completion_tokens' backend-python/app/ai backend-python/app | head -n 240

echo
echo "Candidate files:"
git ls-files backend-python/app | rg '(calculator|usage|store|base|provider)'

Repository: pateatlau/fullstack-ai-platform

Length of output: 20778


🏁 Script executed:

#!/bin/bash
set -euo pipefail

echo "calculator.py"
cat -n backend-python/app/ai/observability/cost/calculator.py

echo
echo "usage_service.py"
cat -n backend-python/app/services/usage_service.py

echo
echo "db usage models"
fd -a 'usage.py|tables.py|schema.py|models.py' backend-python/app/db | while read -r f; do
  echo "--- $f"
  wc -l "$f"
  rg -n -C 4 'class .*Usage|Usage|input_tokens|completion_tokens|prompt_tokens|price|cost|record' "$f" || true
done

echo
echo "observability usages"
rg -n -C 5 'calculate|record_usage|SqlUsageStore|UsageStore|ProviderUsage' backend-python/app/services backend-python/app/ai backend-python/app/db

Repository: pateatlau/fullstack-ai-platform

Length of output: 35777


🏁 Script executed:

#!/bin/bash
set -euo pipefail

echo "ChatService usage recording slices:"
sed -n '792,850p' backend-python/app/services/chat_service.py
sed -n '1040,1105p' backend-python/app/services/chat_service.py
sed -n '170,225p' backend-python/app/ai/agent/adapters/chat_adapter.py

echo
echo "SqlUsageStore.record bounded slice:"
sed -n '28,95p' backend-python/app/db/usage.py

echo
echo "Behavioral probe of current bounds and calculation:"
python3 - <<'PY'
from dataclasses import dataclass
from typing import Any

`@dataclass`(frozen=True)
class PricingEntry:
    input_usd_per_1k: float = 0.0
    output_usd_per_1k: float = 0.0

`@dataclass`(frozen=True)
class ModelPricingTable:
    data: dict[tuple[str, str], PricingEntry]
    pricing_version = "probe"

    def lookup(self, provider: str, model: str) -> PricingEntry | None:
        return self.data.get((provider, model))

class CostCalculator:
    def __init__(self, table: ModelPricingTable) -> None:
        self._table = table

    def price(self, provider: str, model: str, usage: Any | None) -> tuple[float | None, str | None]:
        if usage is None:
            return None, None
        if usage.prompt_tokens is None or usage.completion_tokens is None:
            return None, None
        entry = self._table.lookup(provider, model)
        if entry is None:
            return None, None
        cost = (usage.prompt_tokens / 1000.0) * entry.input_usd_per_1k + (
            usage.completion_tokens / 1000.0
        ) * entry.output_usd_per_1k
        return cost, self._table.pricing_version

ProviderUsage = type("ProviderUsage", (), {})
ProviderUsage = dataclass(frozen=True)(type(
    "ProviderUsage", (), {
        "__annotations__": {"prompt_tokens": "int | None", "completion_tokens": "int | None", "total_tokens": "int | None"}
    }
))
ProviderUsage.prompt_tokens = None
ProviderUsage.completion_tokens = None
ProviderUsage.total_tokens = None

calculator = CostCalculator(ModelPricingTable({("p", "m"): PricingEntry(1.0, 2.0)}))
for tokens in [
    (10, 5),
    (0, 0),
    (-1, 5),
    (10, -2),
    (-3, -4),
]:
    usage = ProviderUsage(prompt_tokens=tokens[0], completion_tokens=tokens[1], total_tokens=None)
    print(tokens, calculator.price("p", "m", usage))
PY

Repository: pateatlau/fullstack-ai-platform

Length of output: 9296


🏁 Script executed:

#!/bin/bash
set -euo pipefail

echo "ChatService surrounding error handling:"
sed -n '830,915p' backend-python/app/services/chat_service.py
sed -n '220,345p' backend-python/app/ai/agent/adapters/chat_adapter.py

echo
echo "Simple arithmetic probe:"
python3 - <<'PY'
def current_price(prompt_tokens, completion_tokens, in_price, out_price):
    if prompt_tokens is None or completion_tokens is None:
        return None
    return (prompt_tokens / 1000.0) * in_price + (completion_tokens / 1000.0) * out_price

for prompt_tokens, completion_tokens in [
    (10, 5),
    (0, 0),
    (-1, 5),
    (10, -2),
    (-3, -4),
]:
    print(f"input={prompt_tokens}, completion={completion_tokens} -> cost={current_price(prompt_tokens, completion_tokens, 1.0, 2.0)}")
PY

Repository: pateatlau/fullstack-ai-platform

Length of output: 6014


Reject negative usage before calculating cost.

ProviderUsage accepts negative integer token counts, and SqlUsageStore.record() persists those inputs into usage_events; valid pricing rows would then produce a negative cost_usd. Reject prompt_tokens < 0 or completion_tokens < 0 in CostCalculator.price(), or return (None, None).

🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@backend-python/app/ai/observability/cost/calculator.py` around lines 34 - 46,
Update CostCalculator.price() to return (None, None) when usage.prompt_tokens or
usage.completion_tokens is negative, before looking up pricing or calculating
cost. Preserve the existing handling for missing usage, missing token values,
and unavailable pricing entries.



class CostRegistry:
"""Process-wide ``CostCalculator`` accessor (real or unset when disabled)."""

_calculator: CostCalculator | None = None
_initialized = False

@classmethod
def initialize(cls, settings: Settings) -> None:
if cls._initialized:
return

cls._initialized = True
if not settings.observability_enabled:
cls._calculator = None
return

try:
table = ModelPricingTable.load(settings)
except ObservabilityConfigError:
raise
except Exception as exc:
raise ObservabilityConfigError(
f"Failed to load model pricing table: {exc}"
) from exc

cls._calculator = CostCalculator(table)

@classmethod
def get_calculator(cls) -> CostCalculator | None:
return cls._calculator

@classmethod
def is_enabled(cls) -> bool:
return cls._calculator is not None

@classmethod
def reset_for_tests(cls) -> None:
cls._initialized = False
cls._calculator = None
137 changes: 135 additions & 2 deletions backend-python/app/ai/observability/cost/pricing.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,138 @@
"""Model pricing table loader — implemented in Phase 5."""
"""Model pricing table loader — git-tracked ``config/model_pricing.yaml``."""

from __future__ import annotations

# TODO(epic-07): Phase 5 — ModelPricingTable loads config/model_pricing.yaml.
from dataclasses import dataclass
from pathlib import Path
from typing import Any

import yaml

from app.ai.observability.exceptions import ObservabilityConfigError
from app.core.config import Settings

_BACKEND_ROOT = Path(__file__).resolve().parents[4]


@dataclass(frozen=True)
class ModelPricingEntry:
provider: str
model: str
input_usd_per_1k: float
output_usd_per_1k: float


class ModelPricingTable:
"""Version-locked per-(provider, model) token rates loaded at startup."""

def __init__(
self,
*,
pricing_version: str,
entries: dict[tuple[str, str], ModelPricingEntry],
) -> None:
self._pricing_version = pricing_version
self._entries = entries

@property
def pricing_version(self) -> str:
return self._pricing_version

@property
def model_registry(self) -> frozenset[str]:
return frozenset(entry.model for entry in self._entries.values())

def lookup(self, provider: str, model: str) -> ModelPricingEntry | None:
return self._entries.get((provider, model))

@classmethod
def load(
cls, settings: Settings, *, pricing_file: Path | None = None
) -> ModelPricingTable:
path = pricing_file or (
_BACKEND_ROOT / settings.observability_cost_pricing_file
)
if not path.is_file():
raise ObservabilityConfigError(f"Model pricing file not found: {path}")

raw = yaml.safe_load(path.read_text(encoding="utf-8"))
if not isinstance(raw, dict):
raise ObservabilityConfigError(
"Model pricing file must contain a YAML mapping at the top level."
)

file_version = raw.get("pricing_version")
if not isinstance(file_version, str) or not file_version.strip():
raise ObservabilityConfigError(
"Model pricing file requires a non-empty pricing_version."
)
if file_version != settings.observability_cost_pricing_version:
raise ObservabilityConfigError(
"Model pricing version mismatch: "
f"file has {file_version!r}, "
f"settings.observability_cost_pricing_version is "
f"{settings.observability_cost_pricing_version!r}."
)

models_raw = raw.get("models")
if not isinstance(models_raw, list) or not models_raw:
raise ObservabilityConfigError(
"Model pricing file requires a non-empty models list."
)

entries: dict[tuple[str, str], ModelPricingEntry] = {}
for index, item in enumerate(models_raw):
entry = cls._parse_entry(item, index=index)
key = (entry.provider, entry.model)
if key in entries:
raise ObservabilityConfigError(
f"Duplicate pricing entry for provider={entry.provider!r}, "
f"model={entry.model!r}."
)
entries[key] = entry

return cls(pricing_version=file_version, entries=entries)

@staticmethod
def _parse_entry(raw: Any, *, index: int) -> ModelPricingEntry:
if not isinstance(raw, dict):
raise ObservabilityConfigError(f"models[{index}] must be a mapping.")

provider = raw.get("provider")
model = raw.get("model")
input_rate = raw.get("input_usd_per_1k")
output_rate = raw.get("output_usd_per_1k")

if not isinstance(provider, str) or not provider.strip():
raise ObservabilityConfigError(
f"models[{index}].provider must be a non-empty string."
)
if not isinstance(model, str) or not model.strip():
raise ObservabilityConfigError(
f"models[{index}].model must be a non-empty string."
)

validated_input_rate = input_rate
validated_output_rate = output_rate
for field_name, value in (
("input_usd_per_1k", validated_input_rate),
("output_usd_per_1k", validated_output_rate),
):
if not isinstance(value, (int, float)) or isinstance(value, bool):
raise ObservabilityConfigError(
f"models[{index}].{field_name} must be a finite number >= 0."
)
if value < 0:
raise ObservabilityConfigError(
f"models[{index}].{field_name} must be >= 0."
)
Comment on lines +121 to +128

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🗄️ Data Integrity & Integration | 🟠 Major | ⚡ Quick win

Reject non-finite pricing rates.

.nan and .inf pass these checks because both are float values and neither is less than zero. The loader then accepts invalid rates despite the error text requiring finite values. This can persist invalid cost_usd values and emit invalid cost metrics.

Use math.isfinite(value) in this condition. Add .nan and .inf loader test cases.

Proposed fix
+import math
+
-            if not isinstance(value, (int, float)) or isinstance(value, bool):
+            if (
+                not isinstance(value, (int, float))
+                or isinstance(value, bool)
+                or not math.isfinite(value)
+            ):
                 raise ObservabilityConfigError(
                     f"models[{index}].{field_name} must be a finite number >= 0."
                 )
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
if not isinstance(value, (int, float)) or isinstance(value, bool):
raise ObservabilityConfigError(
f"models[{index}].{field_name} must be a finite number >= 0."
)
if value < 0:
raise ObservabilityConfigError(
f"models[{index}].{field_name} must be >= 0."
)
import math
if (
not isinstance(value, (int, float))
or isinstance(value, bool)
or not math.isfinite(value)
):
raise ObservabilityConfigError(
f"models[{index}].{field_name} must be a finite number >= 0."
)
if value < 0:
raise ObservabilityConfigError(
f"models[{index}].{field_name} must be >= 0."
)
🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@backend-python/app/ai/observability/cost/pricing.py` around lines 121 - 128,
Update the pricing validation in the model-loading logic around the existing
type and range checks to also reject non-finite numeric values by applying
math.isfinite(value). Preserve the current errors for invalid and negative
rates, and add loader tests covering both NaN and positive/negative infinity
inputs.


assert isinstance(validated_input_rate, (int, float))
assert isinstance(validated_output_rate, (int, float))

return ModelPricingEntry(
provider=provider.strip(),
model=model.strip(),
input_usd_per_1k=float(validated_input_rate),
output_usd_per_1k=float(validated_output_rate),
)
Loading
Loading