|
| 1 | +"""Evaluation — score a model on a task, not just its training loss. |
| 2 | +
|
| 3 | +`finetune` reports next-token *loss*; this reports task *quality*. Point a loaded |
| 4 | +model at a dataset, pick a metric, and get one number plus a per-row breakdown: |
| 5 | +
|
| 6 | + res = slm.evaluate(model, "qa.jsonl") # contains-match |
| 7 | + res = slm.evaluate(model, ds, metric="exact") # exact-match |
| 8 | + res = slm.evaluate(model, ds, judge=judge) # LLM-as-judge |
| 9 | + res = slm.evaluate(model, ds, metric=my_score_fn) # custom scorer |
| 10 | + print(res.score, res.sparkline()) |
| 11 | +
|
| 12 | +This is the front half of an "eval gate" — the same capture/judge primitives, |
| 13 | +turned toward *measuring* a model instead of training one. Pure ShadowLM: it |
| 14 | +only needs a loaded model's `.chat()`. The built-in scorers are reused from APO |
| 15 | +(`apo._contains_score`, `apo._judge_one`) so eval and prompt-optimization agree |
| 16 | +on what a good answer is. |
| 17 | +""" |
| 18 | + |
| 19 | +from __future__ import annotations |
| 20 | + |
| 21 | +from dataclasses import dataclass, field |
| 22 | + |
| 23 | +from .data import Dataset |
| 24 | + |
| 25 | + |
| 26 | +def _exact_score(output: str, expected: str) -> float: |
| 27 | + """1.0 when the output equals the expected answer (case/space-insensitive).""" |
| 28 | + from .apo import _norm # noqa: PLC0415 |
| 29 | + |
| 30 | + o, e = _norm(output), _norm(expected) |
| 31 | + return 1.0 if e and o == e else 0.0 |
| 32 | + |
| 33 | + |
| 34 | +@dataclass |
| 35 | +class EvalResult: |
| 36 | + """The outcome of an `evaluate` run: an aggregate score plus per-row detail.""" |
| 37 | + |
| 38 | + metric: str |
| 39 | + score: float # mean of `scores` |
| 40 | + scores: list[float] = field(default_factory=list) |
| 41 | + examples: list[dict] = field(default_factory=list) # [{input, output, expected, score}] |
| 42 | + n: int = 0 |
| 43 | + |
| 44 | + def sparkline(self) -> str: |
| 45 | + """A tiny unicode bar of per-row scores — handy in a REPL or log line.""" |
| 46 | + if not self.scores: |
| 47 | + return "" |
| 48 | + bars = "▁▂▃▄▅▆▇█" |
| 49 | + lo, hi = min(self.scores), max(self.scores) |
| 50 | + rng = (hi - lo) or 1.0 |
| 51 | + return "".join(bars[min(7, int((s - lo) / rng * 7))] for s in self.scores) |
| 52 | + |
| 53 | + def worst(self, k: int = 5) -> list[dict]: |
| 54 | + """The k lowest-scoring examples (for eyeballing where the model fails).""" |
| 55 | + return sorted(self.examples, key=lambda e: e["score"])[:k] |
| 56 | + |
| 57 | + def to_dict(self) -> dict: |
| 58 | + return {"metric": self.metric, "score": self.score, "n": self.n, |
| 59 | + "scores": self.scores, "examples": self.examples} |
| 60 | + |
| 61 | + def __repr__(self) -> str: |
| 62 | + return f"EvalResult(metric={self.metric!r}, score={self.score:.4f}, n={self.n})" |
| 63 | + |
| 64 | + |
| 65 | +def _row_io(row: dict, fmt: str) -> tuple[list[dict], str]: |
| 66 | + """Pull (history, expected_answer) out of a row, by dataset format. |
| 67 | +
|
| 68 | + `history` is the conversation to feed the model — a full multi-turn prefix |
| 69 | + for chat rows, or a single user turn for QA/preference rows. `expected` may |
| 70 | + be "" when the dataset carries no reference answer (e.g. judge scoring on |
| 71 | + prompts alone). |
| 72 | + """ |
| 73 | + from .data import CHAT, PREFERENCE # noqa: PLC0415 |
| 74 | + |
| 75 | + if fmt == CHAT or "messages" in row: |
| 76 | + msgs = [{"role": m.get("role", "user"), "content": m.get("content") or ""} |
| 77 | + for m in row.get("messages", [])] |
| 78 | + # Everything up to the final assistant turn is context; that turn is the |
| 79 | + # reference — so a multi-turn row is answered in its full conversation, |
| 80 | + # not scored as "answer the opening question". |
| 81 | + last_asst = next((i for i in range(len(msgs) - 1, -1, -1) |
| 82 | + if msgs[i]["role"] == "assistant"), None) |
| 83 | + if last_asst is None: |
| 84 | + return (msgs or [{"role": "user", "content": ""}]), "" |
| 85 | + history = msgs[:last_asst] or [{"role": "user", "content": ""}] |
| 86 | + return history, msgs[last_asst]["content"] |
| 87 | + if fmt == PREFERENCE or ("chosen" in row and "prompt" in row): |
| 88 | + return [{"role": "user", "content": str(row.get("prompt", ""))}], \ |
| 89 | + str(row.get("chosen", "")) |
| 90 | + # instruction / QA / raw dict — auto-detect the prompt & answer columns |
| 91 | + from .apo import _cols # noqa: PLC0415 |
| 92 | + |
| 93 | + pcol, acol = _cols(row) |
| 94 | + if not pcol: |
| 95 | + from .apo import _PROMPT_KEYS # noqa: PLC0415 |
| 96 | + |
| 97 | + raise ValueError( |
| 98 | + f"no prompt column found in row (looked for {_PROMPT_KEYS}); " |
| 99 | + "pass chat-format rows or a dataset with a prompt/question column") |
| 100 | + prompt = str(row[pcol]) |
| 101 | + # alpaca-style extra context column, when distinct from the prompt |
| 102 | + if pcol != "input" and row.get("input"): |
| 103 | + prompt = f"{prompt}\n\n{row['input']}" |
| 104 | + return [{"role": "user", "content": prompt}], \ |
| 105 | + (str(row.get(acol, "")) if acol else "") |
| 106 | + |
| 107 | + |
| 108 | +def _resolve_scorer(metric, judge): |
| 109 | + """Map the metric arg to a scorer `(output, expected, prompt) -> float`.""" |
| 110 | + if callable(metric): |
| 111 | + return metric, getattr(metric, "__name__", "custom") |
| 112 | + from .apo import _contains_score, _judge_one # noqa: PLC0415 |
| 113 | + |
| 114 | + if metric == "contains": |
| 115 | + return (lambda out, exp, q: _contains_score(out, exp)), "contains" |
| 116 | + if metric == "exact": |
| 117 | + return (lambda out, exp, q: _exact_score(out, exp)), "exact" |
| 118 | + if metric == "judge": |
| 119 | + if judge is None: |
| 120 | + raise ValueError("metric='judge' needs a judge model: evaluate(..., judge=model)") |
| 121 | + return (lambda out, exp, q: _judge_one(judge, q, out, exp)), "judge" |
| 122 | + raise ValueError( |
| 123 | + f"unknown metric {metric!r} (expected 'contains', 'exact', 'judge', or a callable)") |
| 124 | + |
| 125 | + |
| 126 | +def evaluate( |
| 127 | + model, |
| 128 | + data: Dataset | list[dict] | str, |
| 129 | + *, |
| 130 | + metric="contains", |
| 131 | + judge=None, |
| 132 | + system: str | None = None, |
| 133 | + sample: int | None = None, |
| 134 | + max_new_tokens: int = 256, |
| 135 | + temperature: float = 0.0, |
| 136 | + verbose: bool = True, |
| 137 | +) -> EvalResult: |
| 138 | + """Score `model` on `data`, returning an `EvalResult`. |
| 139 | +
|
| 140 | + model: a loaded shadowlm Model (answers each row via `.chat`). |
| 141 | + data: a Dataset, rows, or a path to a dataset file (jsonl/json/csv/parquet). |
| 142 | + metric: "contains" (default — expected answer appears in the output), "exact" |
| 143 | + (normalized equality), "judge" (LLM-as-judge, needs `judge=`), or a custom |
| 144 | + callable `(output, expected, prompt) -> float in [0, 1]`. |
| 145 | + judge: a Model that scores answers 0–1. Passing it defaults `metric` to "judge". |
| 146 | + system: optional system prompt prepended to every query. |
| 147 | + sample: evaluate only the first N rows. |
| 148 | + temperature: generation temperature — 0.0 (default) for deterministic scoring. |
| 149 | + """ |
| 150 | + if isinstance(data, str): |
| 151 | + data = Dataset.load(data) |
| 152 | + fmt = data.format if isinstance(data, Dataset) else None |
| 153 | + rows = list(data.rows if isinstance(data, Dataset) else data) |
| 154 | + if sample is not None: |
| 155 | + rows = rows[:sample] |
| 156 | + if not rows: |
| 157 | + raise ValueError("evaluate needs at least one row") |
| 158 | + if judge is not None and metric == "contains": |
| 159 | + metric = "judge" # passing a judge implies judge scoring |
| 160 | + if fmt is None: |
| 161 | + from .data import _detect_format # noqa: PLC0415 |
| 162 | + |
| 163 | + fmt = _detect_format(rows) |
| 164 | + scorer, metric_name = _resolve_scorer(metric, judge) |
| 165 | + |
| 166 | + scores: list[float] = [] |
| 167 | + examples: list[dict] = [] |
| 168 | + for r in rows: |
| 169 | + history, expected = _row_io(r, fmt) |
| 170 | + msgs = ([{"role": "system", "content": system}] if system else []) + history |
| 171 | + out = str(model.chat(msgs, temperature=temperature, max_new_tokens=max_new_tokens)) |
| 172 | + # the last user turn is the "question" passed to a judge / shown in output |
| 173 | + question = next((m["content"] for m in reversed(history) |
| 174 | + if m["role"] == "user"), "") |
| 175 | + s = max(0.0, min(1.0, float(scorer(out, expected, question)))) |
| 176 | + scores.append(s) |
| 177 | + examples.append({"input": question, "output": out, "expected": expected, "score": s}) |
| 178 | + |
| 179 | + score = sum(scores) / len(scores) |
| 180 | + if verbose: |
| 181 | + print(f"[eval] {metric_name} · {score:.3f} over {len(scores)} rows", flush=True) |
| 182 | + return EvalResult(metric=metric_name, score=score, scores=scores, |
| 183 | + examples=examples, n=len(scores)) |
0 commit comments