engraphis/docs/benchmark-evidence/offline-fixtures-v1.json at main · Coding-Dev-Tools/engraphis · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
{
  "schema": "engraphis-public-offline-fixtures/v1",
  "generated_on": "2026-08-08",
  "privacy": {
    "contains_raw_questions": false,
    "contains_answers": false,
    "contains_prompts": false,
    "contains_customer_data": false,
    "contains_per_record_fingerprints": false
  },
  "suite": {
    "digest": "4bfdfd6ccdf34ff7daa7441b8e788371c31985efe031b6520a265ef11f71ed1b",
    "digest_method": "sha256(canonical compact JSON mapping each sorted path to its file SHA-256)",
    "files": {
      "eval/chunking_eval.py": "a16544353940c0a8c40cea3b9932d3399b35ea5994b809b78f5dbe4a952c467f",
      "eval/datasets/codemem.jsonl": "341313023c22850a2e14f02742b571ad1deca824f886a1654a59541304c01f3c",
      "eval/datasets/longdoc.jsonl": "7f5ade95e1f283d0db8cf78e53ed8995d3534f847e616d2c0005fd8da37ac790",
      "eval/grounded.py": "75ba96a4427508f2718323d283f7889fc90901a6176f12c4b3518f1ede5d96dd",
      "eval/performance.py": "e17ea78095e4e592717bc5d9d8e34d55fd98c3fd28d1a8104a20c227d4d619c9"
    }
  },
  "runs": [
    {
      "id": "offline-chunking",
      "command": "python -m eval.chunking_eval --dataset eval/datasets/longdoc.jsonl --k 5",
      "config_digest": "c1c8196aa7e1568ef3844a9fb2d76b87f342c39108e32d6ad144b885a76143b8",
      "config_digest_method": "sha256(UTF-8 exact command)",
      "boundary": "Deterministic offline retrieval fixture; normalized-character token estimator; not external QA or provider billing.",
      "result": {
        "documents": 6,
        "questions": 18,
        "k": 5,
        "token_counter": "engraphis.chars4.v1",
        "whole": {
          "memories": 6,
          "recall_at_k": 1.0,
          "mean_context_tokens": 740.3,
          "mean_evidence_tokens": 162.2,
          "max_stored_tokens": 213
        },
        "chunked": {
          "memories": 24,
          "recall_at_k": 1.0,
          "mean_context_tokens": 214.3,
          "mean_evidence_tokens": 42.4,
          "max_stored_tokens": 59
        },
        "context_reduction_pct": 71.1
      }
    },
    {
      "id": "offline-performance",
      "command": "python -m eval.performance --dataset eval/datasets/codemem.jsonl --k 5 --iterations 10 --json",
      "config_digest": "bbe4aca81e58d4830e50a8fc7729a1d15b71d97a6299bccd79432b7f119677d7",
      "config_digest_method": "sha256(UTF-8 exact command)",
      "boundary": "Deterministic offline CodeMem fixture; serialized JSON-shape payload proxies, not MCP transport responses, provider billing, or latency claims.",
      "result": {
        "dataset_cases": 14,
        "memories": 44,
        "questions": 26,
        "timed_recalls": 260,
        "k": 5,
        "token_budget": 1500,
        "token_counter": "engraphis.regex.v1",
        "recall_at_k": 1.0,
        "hit_at_k": 1.0,
        "answer_token_recall": 1.0,
        "mean_context_tokens": 85.38,
        "max_context_tokens": 108,
        "full_serialized_payload_tokens": 23810,
        "compact_serialized_payload_tokens": 10202,
        "saved_serialized_payload_tokens": 13608,
        "serialized_payload_savings_ratio": 0.5715
      }
    },
    {
      "id": "offline-grounded",
      "command": "python -m eval.grounded",
      "config_digest": "590442e51e3642c10489165759919dc86ffac62c182937330c153e7f8d5fc26f",
      "config_digest_method": "sha256(UTF-8 exact command)",
      "boundary": "Deterministic offline support/abstention fixture; not a frontier-model answer-quality score.",
      "result": {
        "answerable": 5,
        "grounded": 5,
        "off_topic": 5,
        "abstained": 5,
        "decision_accuracy": 1.0
      }
    }
  ]
}