diff --git a/.claude-plugin/skill-assets.sha256 b/.claude-plugin/skill-assets.sha256
index 2f565d53..e472307d 100644
--- a/.claude-plugin/skill-assets.sha256
+++ b/.claude-plugin/skill-assets.sha256
@@ -2,5 +2,5 @@
94bfa06317a8fe6a6a7e204bb70c5abdc9e4bbc34d79dd6f8447a30140bc8b85 .claude-plugin/plugin.json
055655db84af07561d002f0c69744313d8413c39f3e873f941f0fa0b1e76dc66 skills/engraphis-memory/references/CONVENTIONS.md
62019760766ff472a76a0f81437898f39e3c1fe2631732b7b7733e50c1ad837f skills/engraphis-memory/references/SCOPING.md
-96c8e9b9cee1b3cb43c4bef9e48c57ed92af707f7f7a5d28d73b1ac247d2f0c6 skills/engraphis-memory/references/TOOLS.md
+1f62ba2b6abf3dab266d5b4d9c85f2d7fd7fe4ece4e85e2413cfc3fe460ef2c1 skills/engraphis-memory/references/TOOLS.md
0f98098df695b9a00dc78402911124ebf09a4a058f6c8bec2c6234ec61fac13a skills/engraphis-memory/SKILL.md
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
index f00bc1e7..277ac103 100644
--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@@ -26,3 +26,9 @@ updates:
schedule:
interval: "weekly"
open-pull-requests-limit: 3
+ groups:
+ # codeql-action's init and analyze steps must always bump together —
+ # a mixed-version pair fails CodeQL's analyze post-action step.
+ codeql-action:
+ patterns:
+ - "github/codeql-action/*"
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index d18fc4ac..6d058e78 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -37,7 +37,7 @@ jobs:
- name: Unit tests (full suite — extras-gated tests included)
run: |
python -c "import fastapi, httpx, mcp, multipart, pydantic, uvicorn"
- ENGRAPHIS_INDEX_ROOTS="${GITHUB_WORKSPACE}:${RUNNER_TEMP}" python -m pytest -o addopts="" tests/ -q -rs --basetemp="${RUNNER_TEMP}/engraphis-pytest"
+ ENGRAPHIS_INDEX_ROOTS="${GITHUB_WORKSPACE}:${RUNNER_TEMP}:$(python -c 'import tempfile; print(tempfile.gettempdir())')" python -m pytest -o addopts="" tests/ -q -rs --basetemp="${RUNNER_TEMP}/engraphis-pytest"
- name: Retrieval eval gate
run: python -m eval.harness --dataset eval/datasets/sample.jsonl --k 5
- name: Retrieval eval gate — CodeMem (coding-agent wedge, incl. conflict resolution)
@@ -48,6 +48,10 @@ jobs:
run: python -m eval.reinforcement
- name: Adversarial memory prompt-boundary gate
run: python -m eval.adversarial_memory_security
+ - name: Grounded-recall decision gate
+ run: python -m eval.grounded
+ - name: Code-agent arm gate
+ run: python -m eval.code_arm
typecheck:
name: core + backends typecheck (Python 3.11)
@@ -110,9 +114,11 @@ jobs:
python -m pip install --upgrade pip
pip install numpy "pytest<9"
- name: Unit tests (extras-gated tests skip; the core must pass)
- run: ENGRAPHIS_INDEX_ROOTS="${GITHUB_WORKSPACE}:${RUNNER_TEMP}" python -m pytest -o addopts="" tests/ -q -rs --basetemp="${RUNNER_TEMP}/engraphis-pytest"
+ run: ENGRAPHIS_INDEX_ROOTS="${GITHUB_WORKSPACE}:${RUNNER_TEMP}:$(python -c 'import tempfile; print(tempfile.gettempdir())')" python -m pytest -o addopts="" tests/ -q -rs --basetemp="${RUNNER_TEMP}/engraphis-pytest"
- name: Retrieval eval gate
run: python -m eval.harness --dataset eval/datasets/sample.jsonl --k 5
+ - name: Retrieval eval gate — CodeMem (coding-agent wedge, incl. conflict resolution)
+ run: python -m eval.harness --dataset eval/datasets/codemem.jsonl --k 5
- name: Ablation
run: python -m eval.ablation
- name: Reinforcement state-transition gate
@@ -178,7 +184,7 @@ jobs:
python -m pip install --upgrade pip
pip install -e ".[test]" pytest-cov
- name: Coverage run (all extras-gated tests, tracked modules)
- run: ENGRAPHIS_INDEX_ROOTS="${GITHUB_WORKSPACE}:${RUNNER_TEMP}" python -m pytest -o addopts="" tests/ -q -rs --basetemp="${RUNNER_TEMP}/engraphis-pytest" --cov=engraphis --cov-report=term-missing --cov-fail-under=60
+ run: ENGRAPHIS_INDEX_ROOTS="${GITHUB_WORKSPACE}:${RUNNER_TEMP}:$(python -c 'import tempfile; print(tempfile.gettempdir())')" python -m pytest -o addopts="" tests/ -q -rs --basetemp="${RUNNER_TEMP}/engraphis-pytest" --cov=engraphis --cov-report=term-missing --cov-fail-under=60
hygiene:
name: repo hygiene gate (no stray DBs/logs)
diff --git a/.gitignore b/.gitignore
index 4bb10535..be5c0ce6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -114,3 +114,16 @@ cookies.txt
# uv lockfile (generated tooling, not a project dependency)
uv.lock
+
+# Schema-migration flock lives next to the DB (engraphis/config.py _migration_lock);
+# regenerable runtime state like *.db itself. Held live while the server runs.
+.*.migration.lock
+
+# One-off diagnostic/report dumps that keep landing at the repo root — regenerable
+# command output, never package content (same policy as /_*.mjs above).
+/404_paths.txt
+/disk_report.txt
+/large_files.txt
+/stats_pm2.txt
+/venv_status.txt
+/r3.txt
diff --git a/AGENTS.md b/AGENTS.md
index 5ed35ab5..23b0a800 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -49,6 +49,8 @@ python -m eval.harness --dataset eval/datasets/codemem.jsonl --k 5 # coding/con
python -m eval.ablation # vector-only vs hybrid
python -m eval.reinforcement # bounded retention trajectory
python -m eval.adversarial_memory_security # prompt/graph boundary
+python -m eval.grounded # grounded-abstain decision gate
+python -m eval.code_arm # coding-agent arm gate
pyright # core + backends typecheck
# ── External benchmarks (real numbers need torch + the dataset; see eval/external.py) ──
@@ -89,7 +91,9 @@ python -m scripts.migrate_to_v2 --old engraphis_v1.db --new engraphis_v2.db
`requires-python >= 3.9` (ruff targets `py39`). CI tests the NumPy-only core on 3.9, the full
offline stack on 3.10–3.14, and Pyright on 3.11; dedicated jobs also exercise encryption and built
-artifacts. `.github/workflows/ci.yml` is authoritative when the matrix changes.
+artifacts, and further jobs run the coverage gate (`--cov-fail-under=60`), repo hygiene, the Pi
+extension, browser accessibility, and the Docker smoke. `.github/workflows/ci.yml` is
+authoritative when the matrix changes.
---
diff --git a/BENCHMARKS.md b/BENCHMARKS.md
index 644b2e1f..219f6ee2 100644
--- a/BENCHMARKS.md
+++ b/BENCHMARKS.md
@@ -11,7 +11,7 @@ For the locked operator sequence for a public canonical run, see
Every exact public aggregate retained below comes from the checked-in, public-safe
[`offline-fixtures-v1.json`](docs/benchmark-evidence/offline-fixtures-v1.json) artifact. Its
SHA-256 is
-`c3a74f1770ad3f868f55261ba11680e2dadca30167082ac2cb6669f9e3bdfad2`, also recorded in the
+`0f60b0868444f676fe14c5f94d7db2c475e22669930c4d760881d0842eaa6800`, also recorded in the
adjacent `.sha256` file. The artifact contains no raw questions, answers, prompts, customer data,
or per-record content fingerprints.
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 62da69d9..f118ca59 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,7 +7,6 @@ All notable changes to Engraphis are documented here. Format loosely follows
### Changed
-
- Direct black-hole children now receive compact, deterministic orbital lanes near the black
hole instead of inheriting the farthest authored radius. Each lane keeps phase and painted
clearance, while community-child planets remain in their local moving frame; oversized Galaxy
@@ -97,9 +96,28 @@ All notable changes to Engraphis are documented here. Format loosely follows
- Added `docs/GRAPH_PERFORMANCE.md` documenting the two graph presentation profiles,
worker layout, progressive rendering, and the 20,000-node / 200,000-relation safety
ceilings.
+- Source-import manifest paging now uses keyset (cursor) pagination instead of OFFSET,
+ so concurrent writes during a source re-import can no longer skip or duplicate rows
+ mid-scan (PR #154).
+- Local file/folder imports now accept up to 1,500 files per batch (was 500), with the total
+ batch ceiling scaled to 750 MB so the average per-file allowance is unchanged; document-wizard
+ scanner ceilings move in lockstep.
+- Folder imports report truncation explicitly: a folder with more matching files than the
+ ceiling now warns and returns `truncated`/`matched_total`/`unreadable` fields instead of
+ silently importing an alphabetically-first slice that looks complete.
### Fixed
+- Importing more than 1,000 files through the dashboard no longer fails with "Internal Server
+ Error": wizard upload routes parse multipart forms under the advertised 1,500-file ceiling
+ instead of Starlette's hidden 1,000-part parser default, oversized batches return a clear 413,
+ and large vault uploads no longer trip the dashboard's 8 MB default body limit.
+- One unreadable or pathological file (locked, deep-nested JSON, concurrent writer) now degrades
+ to a per-file error instead of rolling back the entire import batch with a 500.
+- Document/Obsidian import jobs whose worker died with the process are marked failed on the next
+ status poll (`worker_lease_expired`) instead of reporting `running` forever.
+- Cloud-placeholder files (OneDrive Files-On-Demand) on Windows are hydrated and imported rather
+ than rejected as non-regular files; symlinks and junctions remain blocked.
- Galaxy layout now packs each complete solar-system envelope before orbital seeding and keeps
those envelopes separated with rigid carrier translations during live motion. Compact server
targets can no longer stack large systems near the black hole, while local planet positions,
@@ -131,15 +149,28 @@ All notable changes to Engraphis are documented here. Format loosely follows
to register, instead of replaying the same broken asset response.
- Existing Galaxy preferences migrate only the retired `48` orbital-separation default to `60`;
deliberate custom values, including Gravity `0`, remain unchanged.
+- Source-import hardening lands via separate PR #154: deterministic missing-item detection
+ now guards an unknown baseline instead of reporting spurious misses, denial-guard
+ supersession binds digests computed from the parsed record rather than raw input,
+ import-job finalization is generation-guarded so a stale worker cannot finalize over a
+ newer attempt, and the finalized-state check completes in constant time.
### Security
+
- HTTP error responses in `vault.py` and `service.py` no longer echo user-controlled paths back
to the client, preventing filesystem structure leakage (SEC-001).
- Graph visibility SQL helpers now use parameterized queries instead of `repr(float)` string
interpolation, eliminating a fragile SQL construction pattern (SEC-002).
- The `pypdf` dependency floor is raised to `>=6.15.0` to address PYSEC-2026-3655 and
PYSEC-2026-3656 (arbitrary code execution via crafted PDF objects).
+
+### Removed
+
+- The Hermes memory-provider plugin integration (`integrations/hermes/`, its
+ `ENGRAPHIS_HERMES_*` environment surface, and its integration test) is withdrawn from
+ the repository ahead of the v1.6 tag. The provider remains available in the v1.5
+ release history for anyone who already copied it.
## [1.6] - 2026-08-15
Minor release advancing the v2 engine through schema 16 with deterministic sync state, trusted
diff --git a/README.md b/README.md
index 9724ff10..6f31b48b 100644
--- a/README.md
+++ b/README.md
@@ -1,796 +1,798 @@
-# Engraphis
-
-[](https://pypi.org/project/engraphis/)
-[](https://github.com/Coding-Dev-Tools/engraphis/blob/main/LICENSE)
-[](https://buymeacoffee.com/Jaixii)
-
-[https://engraphis.com/](https://engraphis.com/)
-
-[https://discord.com/invite/Wfr2ejBmY](https://discord.com/invite/Wfr2ejBmY)
-
-**Give your AI agents a memory. See it, search it, and maintain it, all in a beautiful WebUI on your own machine.**
-
-
-
-
- Knowledge Graph · run engraphis-dashboard to see it live
-
-
-**Grounded, not guessed.** Memory with receipts. Local by default. [Explore the proof gallery](https://github.com/Coding-Dev-Tools/engraphis/tree/main/docs/advertising) or [read the campaign guide](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/advertising/campaign.md).
-
----
-
-> **Open-core boundary:** this repository contains the free local engine, dashboard, MCP server,
-> and customer-side clients. Hosted sync, analytics, automation, and team services run on the
-> official hosted service; their server implementations are not distributed here.
-
-> **Support continued Engraphis development with Pro.** [Start a 3-day Pro trial](https://api.engraphis.com/account?plan=pro&interval=monthly&utm_source=engraphis&utm_medium=docs&utm_campaign=pro_conversion&utm_content=readme_intro&trial=pro#billing)
-> or [subscribe to Pro](https://api.engraphis.com/account?plan=pro&interval=monthly&utm_source=engraphis&utm_medium=docs&utm_campaign=pro_conversion&utm_content=readme_intro#billing).
-
----
-
-## Measured token and context savings
-
-### Runtime estimator
-
-The dashboard Overview and Audit/Receipts views also show a receipt-backed estimate from
-real context deliveries. It compares the host history or retrieved source baseline with the
-context Engraphis actually emitted, keeps token counters and release versions separate, and
-labels adaptive history reductions separately from packing savings. Receipts without estimator
-metadata remain historical/unclassified. This measures estimated prompt-context reduction; it
-does not measure provider billing. The `/context-savings` API and
-`engraphis_context_savings` MCP tool aggregate the complete history across all visible workspaces
-by default, or accept an explicit workspace plus optional `from_ts`, `to_ts`, and
-`release_version` filters.
-
-
-
-
- Less repeated history means more room for the task, tools, and useful evidence.
-
-
-
-See benchmark details and reproduce the results
-
-### Controlled before-and-after example
-
-| Retrieval mode | Mean returned memory content | Recall@5 |
-|---|---:|---:|
-| Whole documents | 740.3 tokens | 1.000 |
-| Engraphis structure-aware chunks | 214.3 tokens | 1.000 |
-
-The chunked mode returns the relevant passage instead of the whole document: **526.0 fewer tokens
-per question**. Under the same model-context budget, that leaves roughly **526 tokens** for task
-instructions or other relevant evidence. This is evidence ID `offline-chunking` in the registered
-artifact below.
-
-### Measurement details and reproducibility
-
-The table below contains every exact token/context aggregate currently published here and keeps
-its counting boundary explicit.
-
-| What is counted | Comparison | Measured reduction | Quality held constant |
-|---|---|---|---|
-| Retrieved top-5 memory content, averaged per question | Whole documents: **740.3** tokens → structure-aware chunks: **214.3** tokens | **526.0 fewer tokens per question** (**71.1% lower**, about **3.5× smaller**) | Recall@5 **1.000** in both modes across 6 documents and 18 questions |
-| Smallest returned memory that contains the reference evidence | Whole documents: **162.2** tokens → chunks: **42.4** tokens | **119.8 fewer tokens to evidence** (**73.9% lower**, about **3.8× smaller**) | The same 18 questions had a returned evidence-holding memory in both modes |
-| Full versus compact recall payload proxy across one 26-question pass within a 260-timed-recall CodeMem run | Full proxy: **23,810** `engraphis.regex.v1` tokens → compact proxy: **10,202** tokens | **13,608 proxy tokens avoided** (**57.15% lower**) | 26 payload samples; 260 timed recalls; Recall@5, hit@5, and answer-token recall all **1.000** |
-| Packed prompt-context usage in the same 26-question CodeMem sample pass | Hard budget: **1,500** tokens; observed mean: **85.38**; observed maximum: **108** | A hard cap prevents a recall from exceeding its configured context budget | This is usage accounting, not a before/after savings comparison |
-
-These values are evidence IDs `offline-chunking` and `offline-performance` in
-[`offline-fixtures-v1.json`](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/benchmark-evidence/offline-fixtures-v1.json),
-SHA-256
-`c3a74f1770ad3f868f55261ba11680e2dadca30167082ac2cb6669f9e3bdfad2`.
-[`BENCHMARKS.md`](https://github.com/Coding-Dev-Tools/engraphis/blob/main/BENCHMARKS.md#public-numeric-evidence-registry)
-records the matching suite digest, exact commands, and per-command config digests. External,
-model-dependent, consolidation, productivity, and latency results remain unpublished until the
-same evidence exists for them.
-
-The compact payload shape avoids duplicating full memory bodies when the packed context and source
-list are enough. The evaluator tokenizes JSON-shaped full and compact payload proxies built from
-recall results; it does **not** serialize the MCP envelope or measure a transport response. The
-fixture therefore does not measure model-provider charges, end-to-end task time, or customer cost
-savings.
-
-The measures are deliberately separate and **must not be added together**: chunking counts the
-content of retrieved memory records before `ContextPacker`, whereas compact recall counts a
-serialized JSON-shape payload proxy. “Tokens to evidence” is the size of the smallest
-retrieved memory record holding the reference evidence; it is not latency or end-to-end answer
-accuracy. Chunking creates more focused stored records, so this is a context-efficiency result,
-not a storage-reduction claim.
-
-Reproduce the registered quality and token/context measurements without a network connection or
-API key:
-
-```bash
-python -m eval.grounded
-python -m eval.chunking_eval --dataset eval/datasets/longdoc.jsonl --k 5
-python -m eval.performance --dataset eval/datasets/codemem.jsonl --k 5 --iterations 10 --json
-```
-
-These are small deterministic correctness and efficiency fixtures, not official LoCoMo /
-LongMemEval QA scores or a third-party leaderboard result. Compact-response counts use the exact
-`engraphis.regex.v1` counter; the chunking evaluation uses its documented deterministic
-normalized-character estimator. Chunking measures retrieved memory content, while compact recall
-measures a serialized JSON-shape payload proxy, not an MCP transport response. See the registered
-artifact and [`BENCHMARKS.md`](https://github.com/Coding-Dev-Tools/engraphis/blob/main/BENCHMARKS.md)
-for definitions, limitations, and canonical external-evaluation requirements.
-
-
-
----
-
-## Full Engraphis install: pip install "engraphis[all]"
-
-The complete `engraphis[all]` install is the default way to use Engraphis: it includes the local
-dashboard, Smart MCP server, documents, Cloud Sync client, and supported optional integrations.
-Python 3.10+ is required.
-
-```bash
-pip install "engraphis[all]"
-engraphis-dashboard
-```
-
-The dashboard opens at [http://127.0.0.1:8700](http://127.0.0.1:8700). Local memory needs no
-account or API key.
-
-### Smaller installation options
-
-Use a smaller package only when you intentionally need a limited surface. The NumPy-only core
-continues to support Python 3.9+.
-
-| Goal | Install | Start |
-|---|---|---|
-| Local dashboard and REST API | `pip install "engraphis[server]"` | `engraphis-dashboard` |
-| Coding-agent memory over Smart MCP | `pip install "engraphis[mcp]"` | `codex mcp add engraphis -- engraphis-mcp` |
-| Native SQLite vector acceleration | `pip install "engraphis[vector]"` | Server entrypoints select it automatically |
-| Offline Python library | `pip install engraphis` | `MemoryService.create("engraphis.db")` |
-
-For MCP clients other than Codex, configure a stdio server whose command is `engraphis-mcp`; see
-the [agent connection guide](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/AGENT_CONNECT.md).
-
-### Updating
-
-Use `engraphis-update` to upgrade the installation using its detected install method. Package
-metadata does not record which extras were selected, so the updater defaults to the safe
-superset `engraphis[all]` rather than silently dropping an optional surface. For a deliberate
-selection, set `ENGRAPHIS_UPDATE_EXTRAS` to a comma-separated list (for example
-`server,mcp`), or set it to `none` for the base package only.
-
-> **Upgrading to 1.4:** `engraphis-mcp` now exposes the nine-tool Smart gateway. Integrations that
-> require the former 34 direct tool names should run `engraphis-mcp-classic`. The SQLite schema
-> in the 1.4.0 release was version 9. Existing v7-to-v8 databases already contain `confidence`
-> and `pinned_at`/`unpinned_at`; v9 adds the `memory_tombstones` repository-scope column/table
-> and performs a one-time entity-canonicalization repair, then migrates automatically on first
-> open. A tombstone with a known `repo_id` is terminal only in that repository; legacy repo-less
-> tombstones remain global. See the [1.4.0 release notes](https://github.com/Coding-Dev-Tools/engraphis/blob/main/CHANGELOG.md#140---2026-08-02).
-
-> **Upgrading to 1.5:** schema 10 bounds legacy retention state and schema 11 backfills explicit
-> approval only for eligible pre-review local memories. Pending and quarantined evidence remains
-> gated. Existing 1.4.x databases migrate automatically when Engraphis 1.5 opens them; see the
-> [1.5 release notes](https://github.com/Coding-Dev-Tools/engraphis/blob/main/CHANGELOG.md#15---2026-08-04).
-
-> **Upgrading to 1.6:** existing 1.5 databases migrate automatically through schema 12, which
-> classifies content-free erasure markers before sync: existing markers become local-only
-> `never_export`, while new secure erasures become `remote_erasure` only for non-secret
-> `workspace`/`repo` records already eligible for sharing. Schema 13 adds per-memory hybrid
-> logical clocks for deterministic descriptive-state sync and durable, content-free proof that a
-> memory crossed a sync boundary. Schema 14 adds the Obsidian collection and import manifests;
-> schema 15 generalizes them to source-neutral local documents, preserves temporal source lineage
-> across re-imports, binds adapters and target scopes, and retains only bounded, content-free
-> per-job format/result metadata. The schema 16 migration persists each import job's optional session target
-> and requires source lineage and job-item attachments to remain in that exact session. See the
-> [1.6 release notes](https://github.com/Coding-Dev-Tools/engraphis/blob/main/CHANGELOG.md#16---2026-08-15).
-
----
-
-## What Engraphis gives an agent
-
-An agent should not have to reconstruct a project from scattered chat history on every task.
-Engraphis turns local project knowledge into scoped, time-aware memory; retrieves the evidence
-that supports the current question; and returns a bounded, attributable context packet.
-
-The core task is continuity: retrieve the current, supported project decision without dragging the
-whole history into the next prompt. See [measured token and context savings](#measured-token-and-context-savings)
-for the short version of how much less history an agent has to carry.
-
-| Agent need | What Engraphis changes |
-|---|---|
-| Remember a project across sessions | Stores typed memory in a `workspace → repo → session` hierarchy and provides a last-session handoff. |
-| Find support for the current task | Fuses vector, lexical, graph, and code-aware retrieval instead of relying on one search signal; `fast` can skip graph traversal for small or latency-sensitive vaults. |
-| Know what is true now and what changed | Preserves bi-temporal history and supersession chains instead of silently overwriting a fact. |
-| Avoid confident guesses | Returns cited evidence or explicitly abstains when support is too weak. |
-| Avoid dragging the whole project into every prompt | Packs context to a configured hard budget and can return a compact MCP response. |
-| Keep knowledge in the operator's control | Runs local-first and offline-capable, with scopes, audit records, and optional privacy-safe receipts. |
-
-## Dashboard and local UI
-
-The Engraphis dashboard opens `http://127.0.0.1:8700`. Local memory needs no cloud account,
-signup, or API key and stays in a SQLite file on your machine.
-
-**Ledger** is the primary local interface for recall, memories, graph exploration, provenance,
-workspaces, and manual consolidation. **Classic** preserves the former full tool suite; both use
-the same local data. Switch in **Manage → Settings → Interface** (Ledger) or **Settings →
-Appearance & Engine** (Classic).
-
-### Start it on every platform
-
-| Platform | How |
-|----------|-----|
-| **Windows** | Double-click **Engraphis Dashboard** on your Desktop or Start Menu (install: `engraphis-dashboard --install-shortcuts`) |
-| **macOS** | Double-click **Engraphis Dashboard.app** on your Desktop (install: same command) |
-| **Linux** | Desktop entry in Applications → Development (GNOME/KDE/etc.) |
-| **Docker** | `docker compose up`: see `docker-compose.yml` for the one-command deployment |
-| **Any** | `engraphis-dashboard` in a terminal |
-
-In a source checkout, `scripts/launch_dashboard.ps1` is only a Windows convenience wrapper. It
-delegates configuration, startup health, browser opening, and process lifecycle to the same
-`engraphis-dashboard` entrypoint rather than maintaining a second behavior path.
-
-### Accessibility-first inspection, built in
-
-Inspect memories, supersession diffs, recall scores, timelines, links, consolidation, and audit
-records in the dashboard. The offline graph renderer is vendored, and the interface is keyboard-
-navigable with light and dark themes. Graph exploration offers a focused **High quality** view and
-an explicit worker-backed **Show all nodes** view for complete entity projections up to 20,000
-nodes and 200,000 relationships; see the [graph performance profiles](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/GRAPH_PERFORMANCE.md).
-
----
-
-## How it works
-
-Engraphis gives agents durable, scoped, *explainable* project knowledge. The local engine combines
-Ebbinghaus decay, bi-temporal facts, and hybrid vector/lexical/graph recall; it runs offline with
-SQLite, local embeddings, and `numpy` only.
-
-- **Grounded and governed:** deterministic conflict resolution, cited answers or abstention,
- explicit correction/promotion/forgetting, and a complete history.
-- **Agent-ready:** MCP tools, hard-budget context packets, handoffs, and code-aware retrieval.
-- **Auditable:** content-free receipt chains, provenance, and temporal/entity/code relationships.
-- **Practical:** local file and code ingest, optional PDF/OCR/transcription, and SQLCipher at rest.
-
-### Optional LLM providers
-
-The memory engine, embeddings, conflict resolution, and recall stay local without an LLM. An
-explicitly configured provider adds structured extraction, cited synthesis, consolidation, and
-retention supervision. Configure it in **Settings → Connect an LLM**. The activity view records
-outcomes, never keys, prompts, or raw provider responses. See the
-[LLM provider guide](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/LLM_PROVIDERS.md) for setup and privacy choices.
-
-> Privacy boundary: text sent to an explicitly selected provider leaves the local process under
-> that provider's terms. Use `ENGRAPHIS_RETENTION_SUPERVISOR=none` (the default) and the offline
-> `chunk` extractor when ingestion must remain entirely local.
-
-Choose and configure an external LLM with the [LLM provider guide](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/LLM_PROVIDERS.md),
-including OpenAI, Anthropic, Google, OpenRouter, Ollama, Cohere Command, Command Code Provider,
-and other compatible endpoints. The guide also covers Codex subscription MCP connections.
-
----
-
-## Install
-
-```bash
-pip install "engraphis[all]" # self-hosted dashboard, MCP, code graph, documents, transcription, PostgreSQL, and Cloud Sync
-pip install "engraphis[server]" # dashboard + REST API
-pip install "engraphis[mcp]" # MCP server only
-pip install "engraphis[documents]" # PDF + image OCR bindings
-pip install "engraphis[transcription]" # faster-whisper audio/video
-pip install "engraphis[postgres]" # PostgreSQL schema introspection
-pip install "engraphis[code]" # tree-sitter code graph indexing
-pip install "engraphis[vector]" # native sqlite-vec exact-KNN acceleration
-pip install "engraphis[cloud-sync]" # Cloud Sync client crypto/runtime
-pip install "engraphis[encryption]" # SQLCipher encryption-at-rest extra
-pip install engraphis # core library: numpy only, fully offline
-```
-
-The official Docker image includes the local Tesseract executable for image OCR. Outside
-Docker, the `documents` extra installs its Python bindings; install Tesseract through your
-operating system as well if you enable image OCR.
-
-The NumPy-only core library supports Python 3.9+. Current patched releases of the WebUI
-stack, MCP SDK, image parser, and Cloud Sync client require Python 3.10+, so use Python 3.10
-or newer for the `server`, `mcp`, `documents`, `cloud-sync`, or `all` installation paths.
-
-The default `NumpyVectorIndex` performs an exact full scan. There is no universal memory-count
-cutoff because latency depends on vector size, hardware, filters, and the rest of the recall
-pipeline. Measure your machine with `python -m eval.vector_scale --backend numpy`, then run
-`python -m eval.performance` on a representative corpus. If exact scans miss your latency target,
-install `engraphis[vector]`, create the engine with `vector_backend="sqlite-vec"`, and remeasure.
-The stable sqlite-vec `vec0` backend executes exact KNN in native code; it is acceleration, not a
-claim of sublinear ANN scaling. See [BENCHMARKS.md](https://github.com/Coding-Dev-Tools/engraphis/blob/main/BENCHMARKS.md) for the reproducible commands
-and reporting limits.
-
-Dashboard, REST, and MCP entrypoints default to `ENGRAPHIS_VECTOR_BACKEND=auto`: they use
-sqlite-vec when the `vector` extra is installed and compatible, then safely fall back to NumPy.
-Programmatic `MemoryEngine.create()` and `MemoryService.create()` retain the deterministic
-`numpy` default unless a backend is requested explicitly.
-Use `python -m eval.vector_scale --backend sqlite-vec` for an input-identical direct-search
-comparison; setup/index-build time is explicitly excluded from the timed search envelope.
-
-Persistent vectors fail closed unless the embedder can publish a durable, secret-free space
-fingerprint. Sentence Transformers use the loaded Hub commit or a manifest of local artifacts;
-when a remote model's immutable identity cannot be resolved, persistent vector recall remains
-gated instead of mixing spaces. For programmatic OpenAI-compatible embeddings, construct
-`ApiEmbedder` with an operator/provider `space_version`; without it the adapter remains usable for
-ephemeral embedding only. Its `base_url` may be a provider root or a `/v1` root and is normalized
-to exactly one `/v1/embeddings` endpoint.
-
-`sqlcipher3-binary` publishes CPython manylinux x86-64 wheels. On that target,
-`engraphis[encryption]` installs the driver. The cross-platform `all` extra deliberately
-omits it so `all` remains resolvable on macOS, Windows, Linux ARM, and musl; on those
-targets, provision a compatible SQLCipher driver separately before enabling a database
-key. The programmatic core remains plaintext unless a database key is configured. For a
-fresh database, `engraphis-init` enables SQLCipher automatically when a compatible driver is
-available, creates a private key sidecar, and can be overridden with `--no-encryption`.
-
-> **Linux / macOS:** if `pip install` fails with `error: externally-managed-environment`,
-> your system Python is marked read-only (PEP 668). Install into a virtual environment
-> instead. Run `python3 -m venv venv && source venv/bin/activate && pip install "engraphis[server]"`
-> Alternatively, use Docker (`docker compose up`). `pipx install "engraphis[server]"` also works.
-
-> First run downloads `all-MiniLM-L6-v2` (~80 MB). Without it, the engine falls back
-> to deterministic feature hashing so it always runs offline. That fallback captures lexical
-> overlap, not meaning: recall and grounded MCP responses set `degraded_mode=true` and
-> `semantic_support=false`, and disable vector retrieval plus semantic-cosine evidence. Install
-> a declared embedding model for semantic retrieval.
-
-> To require a model that is already local, set `ENGRAPHIS_EMBED_MODEL=local:/absolute/model/path`
-> or `local:`. This path never downloads a model. If it is unavailable, Engraphis
-> explicitly enters lexical degraded mode instead of presenting hash-vector scores as semantic.
-
----
-
-## Quickstart: dashboard
-
-```bash
-pip install "engraphis[server]"
-engraphis-dashboard # → http://127.0.0.1:8700
-engraphis-dashboard --install-shortcuts # → Desktop + Start Menu icons
-```
-
-### Docker
-
-```bash
-docker compose up # → http://127.0.0.1:8700
-```
-
-For Docker Compose persistence and loopback-port configuration, see the
-[Docker deployment guide](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/DOCKER.md).
-`engraphis-server` and `engraphis server` are headless compatibility aliases
-for this same v2 service, so every public surface has the same scoped recall and retention model.
-
-For optional LAN exposure, token configuration, and HTTP MCP setup, see the
-[Docker deployment guide](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/DOCKER.md).
-
-Set `ENGRAPHIS_API_TOKEN` to require API authentication and `ENGRAPHIS_DB_KEY` to encrypt
-the local database at rest. Hosted-plan credentials configure customer clients; they do not
-install premium server implementations into this image. See `docker-compose.yml` for options.
-
----
-
-## Quickstart: MCP server (for coding agents)
-
-```bash
-pip install "engraphis[mcp]"
-engraphis-init # writes ~/.engraphis/config.env + prints config snippets
-claude mcp add engraphis -- engraphis-mcp
-codex mcp add engraphis -- engraphis-mcp # Codex subscription
-
-```
-For Codex subscription setup and verification, see the [agent connection guide](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/AGENT_CONNECT.md)
-and the [LLM provider guide](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/LLM_PROVIDERS.md).
-
-`engraphis-mcp` is zero-configuration Smart MCP: agents begin with nine compact tools for sessions,
-prompt-ready recall, durable memory, governed record read/update, conflict review, action discovery,
-and safe execution. For code graphs,
-governance, audit, or other advanced work, the agent calls `engraphis_discover_actions` and then
-the indicated read or action executor; no profile selection is required. The gateway validates
-the discovered capability again before it runs it, and clients remain responsible for their
-normal destructive-action approval boundary.
-
-Existing clients that pin the historical 34 named tools can use
-`engraphis-mcp-classic` (or `engraphis-mcp-http --classic`). The complete classic inventory,
-including `engraphis_check_update`, is in the [MCP tool reference](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/MCP_TOOLS.md).
-
-### Pi extension
-
-For installation, configuration, lifecycle commands, and the local trust boundary, see the
-[Pi extension guide](https://github.com/Coding-Dev-Tools/engraphis/blob/main/integrations/pi/README.md).
-
-### Hermes provider
-
-Engraphis also ships a native Hermes memory-provider plugin with local prefetch, bounded turn
-capture, scoped recall, and explicit secure erase. Install Engraphis in the Hermes Python
-environment, copy the provider, then select it with `hermes memory setup`. See the
-[Hermes integration guide](https://github.com/Coding-Dev-Tools/engraphis/blob/main/integrations/hermes/README.md). The provider never installs itself or
-downloads an embedding model.
-
-## Quickstart: repository graph
-
-```bash
-pip install "engraphis[code]"
-engraphis-graph index -w acme -r api --root .
-engraphis-graph search -w acme -r api "UserService"
-# `query`/`explain` blend code search with your stored memories: query matches symbol
-# and file NAMES (a full question sentence won't match anything), and explain's answer
-# is drawn from memories recorded against the repo; both are empty on a fresh index.
-engraphis-graph query -w acme -r api "UserService"
-engraphis-graph explain -w acme -r api "why does deploy depend on approval?"
-engraphis-graph path -w acme -r api UserService DatabasePool
-engraphis-graph impact -w acme -r api --root . --git-range origin/main...HEAD
-engraphis-graph prs -w acme -r api --base main --head HEAD
-engraphis-graph export -w acme -r api -o engraphis-graph-out
-engraphis-graph install-merge-driver --root .
-```
-
-The export contains `graph.json`, a self-contained `graph.html`, and `GRAPH_REPORT.md`.
-Indexing supports Python, JavaScript, TypeScript, Go, Rust, Java, C#, C, C++, SQL, and
-Terraform. Tree-sitter is used when available; the dependency-free regex backend remains a
-functional fallback. Definitions, methods, calls, imports, ownership, variables,
-inheritance/implementation, and docstrings/comments are indexed. Indexing is incremental by
-content hash, honors `.engraphisignore`, and does not follow file symlinks outside the repository
-root. Call edges are name-based and best-effort rather than type-resolved. The optional Git merge
-driver validates bounded graph JSON and deterministically unions nodes and edges instead of
-choosing one export side.
-
-For a read-only recall and graph API that can be shared without exposing write operations:
-
-```bash
-pip install "engraphis[server]"
-engraphis-graph-server # API at http://127.0.0.1:8720; schema at /openapi.json
-```
-
-A non-loopback bind fails closed unless `ENGRAPHIS_GRAPH_TOKEN` (or
-`ENGRAPHIS_API_TOKEN`) is set. See [the v3 architecture/design document](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/ARCHITECTURE_V3.md).
-
----
-
-## Quickstart: Python library
-
-```python
-from engraphis.service import MemoryService
-
-mem = MemoryService.create("engraphis.db")
-mem.remember("Auth migrated from JWT to PASETO.", workspace="acme", repo="api")
-hit = mem.recall("why did we change auth?", workspace="acme", repo="api")
-print(hit["context"])
-```
-
-The same `MemoryService` backs the dashboard and the MCP server.
-
-New writes support `session`, `repo`, and `workspace` visibility. `scope="user"` is reserved and
-rejected until records carry an immutable owner identity; it must not be treated as private
-per-person memory. Historical user-scope rows remain workspace-bound for compatibility.
-
-After an upgrade, `stats()` reports prompt-eligibility counts and active embedding-space
-coverage. Zero-result recall identifies a review-gated scope instead of silently looking empty,
-and `engraphis-cli review list|approve` provides a dry-run-first local bulk workflow. Embedding
-model changes trigger a guarded rebuild; vector recall stays disabled until every stored vector
-matches the new fingerprint. See [recall recovery](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/RECALL_RECOVERY.md).
-
-Agent hosts can avoid retrieval when their existing history already fits:
-
-```python
-decision = mem.adaptive_context(
- "what should the agent do next?",
- current_history,
- workspace="acme",
- repo="api",
- max_context_tokens=8_192,
- retrieval_token_budget=1_024,
-)
-prompt_context = decision["context"]
-```
-
-The decision is `history_bypass` when the history fits, `retrieval` when compact evidence is
-strong, and `history_fallback` when weak retrieval should widen back to recent raw history.
-
-For an agent prompt, prefer `engraphis_recall_context`: it returns one hard-budget packed
-`context` plus compact `sources`, deterministic `usage` accounting (`budget_tokens`, `context_tokens`,
-`source_tokens`, `saved_tokens`, `savings_ratio`, `packed_count`, `omitted_count`, and
-`token_counter`), and optional diagnostics. Accounting is exact for the named counter; inject the
-reader's tokenizer when reader-model token parity is required. `engraphis_recall` remains the compatible full-recall
-surface; use `response_mode="compact"` when the packed context is enough and full memory bodies
-would duplicate it. For advanced query-planning configuration, see the
-[architecture guide](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/ARCHITECTURE_V3.md#query-planning).
-
-For bi-temporal reads, `valid_at` selects what was true at a Unix timestamp and `known_at` selects
-what Engraphis had learned then. `as_of` remains a compatibility alias for `valid_at`; supplying
-both is allowed only when they match.
-
-For a mutable claim, pass a stable `subject_key` and optional `claim_kind`, such as
-`subject_key="api.rate_limit", claim_kind="configured_value"`. Offline conflict resolution
-deterministically adds, reinforces, relates, or supersedes records while preserving temporal
-history; it does not need an LLM. Matching claim identities let it supersede substantially
-reworded mutable facts. Without them, the dependency-free lexical embedder cannot reliably infer
-that a paraphrase is a contradiction, so keep both records or use an explicit `correct` operation.
-
----
-
-## Govern memories without losing history
-
-Engraphis separates automatic write resolution from explicit human governance:
-
-| Operation | Use it when | What happens to history |
-|---|---|---|
-| `remember` | Adding or restating one fact | Adds, reinforces, safely supersedes, or relates an uncertain neighbor |
-| `correct` | Replacing one known-wrong memory | Closes the old validity window and links the replacement |
-| `promote` | A narrow learning now applies more broadly | Writes a wider-scope successor and closes/links the source instead of editing scope in place |
-| `merge` | Combining two or more overlapping memories | Retires every source and creates one memory that supersedes all of them |
-| `retire` | Removing a memory from live recall | Bi-temporally closes it; the audit/history record remains |
-| `consolidate` | Distilling recurring episodic memories automatically | Creates linked semantic digests; source episodes remain live |
-
-Manual N→1 merge is available through `MemoryService.merge()` and `POST /api/merge`:
-
-```python
-a = mem.remember("Deploys happen Friday at 3pm.", workspace="acme")
-b = mem.remember("We deploy Fridays around 15:00.", workspace="acme")
-
-merged = mem.merge(
- [a["id"], b["id"]],
- "Deploys ship every Friday at approximately 15:00.",
- workspace="acme",
- reason="deduplicate the deployment schedule",
-)
-print(merged["compaction"])
-```
-
-`retire` is intentionally not deletion: it preserves temporal history, FTS, and vector
-evidence for historical reads. If a credential was captured, new writes are blocked before
-storage; for a legacy leak use the explicitly destructive `MemoryService.secure_erase()` or
-`POST /api/secure-erase`/`engraphis_secure_erase`. That flow removes the one memory and local
-FTS/vector-index and derived graph/link rows, runs SQLite secure-delete, WAL checkpoint, and
-VACUUM, and scans recognised local SQLite recovery backups. It cannot erase exports, filesystem
-snapshots, remote peers, unknown backups, or information a running/compromised agent already
-read; rotate the credential. See [secure-erasure limits](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/SECURE_ERASURE.md). `forget`
-remains a deprecated compatibility alias for `retire`.
-
-All sources must belong to the named workspace. The result inherits the strictest source
-sensitivity, remains untrusted if any source was untrusted, and stays pinned if any source was
-pinned. The full multi-predecessor chain remains visible through inspection, Why, and Timeline.
-
----
-
-## Free forever vs. hosted plans
-
-The core engine, local dashboard, MCP server, and manual consolidation are Apache-2.0 and free.
-**Pro and Team are services** that provide optional access to the official hosted service; its
-control-plane, billing, relay, compute, and Team identity modules live in a private repository.
-They do not limit the local core. See
-[hosted plans](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/HOSTED_PLANS.md), [licensing](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/LICENSING.md), and
-[Cloud Sync](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/SYNC.md) for service boundaries, lifecycle, and pricing.
-
-[Subscribe to Pro](https://api.engraphis.com/account?plan=pro&interval=monthly&utm_source=engraphis&utm_medium=docs&utm_campaign=pro_conversion&utm_content=readme_pricing#billing)
-to support the project and add hosted services.
-
-[Compare hosted plans](https://api.engraphis.com/account?plan=pro&interval=monthly&utm_source=engraphis&utm_medium=docs&utm_campaign=pro_conversion&utm_content=readme_intro#billing)
-when you are ready to evaluate the service boundary and billing options.
-
-| | Free (available now) | Pro: $10/mo or $100/yr | Team: $20/seat/mo or $200/seat/yr |
-|---|---|---|---|
-| Dashboard WebUI (with built-in inspector) | ✓ | ✓ | ✓ |
-| Memory engine + Smart MCP (Classic 34-tool compatibility) | ✓ | ✓ | ✓ |
-| Version-chain diffs, offline knowledge graph | ✓ | ✓ | ✓ |
-| Manual local consolidation (dry-run by default) | ✓ | ✓ | ✓ |
-| Local workspace export (portable v2 JSON: memories, source manifests, graph/code evidence, sessions, audit, and receipts) | ✓ | ✓ | ✓ |
-| Hosted Cloud Sync | | ✓ | ✓ |
-| Hosted Analytics | | ✓ | ✓ |
-| Hosted Auto Consolidation + retention policy | | ✓ | ✓ |
-| Hosted Auto Dreaming + managed proposals | | ✓ | ✓ |
-| Priority support | | ✓ | ✓ |
-| Hosted multi-user dashboard: invitations, logins, roles, seat management | | | ✓ |
-| Hosted Team audit log + CSV export | | | ✓ |
-| 72-hour pending invitations (resend/revoke) | | | ✓ |
-| Scoped, expiring per-user agent and sync tokens | | | ✓ |
-
----
-
-## MCP tools
-
-Engraphis exposes a zero-configuration Smart MCP gateway plus a 34-tool Classic compatibility
-server across memory, recall, code graphs, governance, sessions, and privacy-safe audit receipts.
-The focused [MCP tool reference](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/MCP_TOOLS.md) is the source for
-the full inventory and parameters.
-
----
-
-## Graphs and privacy-safe receipts
-
-Memory, entity, and code relationships live in one local graph. Engraphis also provides
-content-free operation receipts for inspectable audit evidence. See the
-[architecture](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/ARCHITECTURE_V3.md), [MCP tool reference](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/MCP_TOOLS.md), and
-[security policy](https://github.com/Coding-Dev-Tools/engraphis/blob/main/SECURITY.md) for the data model, tools, and guarantees.
-
----
-
-## Cloud sync
-
-Cloud Sync is an optional hosted Pro/Team service. The public package includes the customer client
-and deterministic merge implementation; hosted relay and account operations are separate. See
-[Cloud Sync](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/SYNC.md) for setup, encryption, merge behavior, and the local folder exchange.
-
----
-
-## Security and trust boundaries
-
-Engraphis is local-first and binds to loopback by default. Read the
-[security policy](https://github.com/Coding-Dev-Tools/engraphis/blob/main/SECURITY.md) before remote deployment or integrating external resources; it
-covers supported versions, data protections, threat model, and vulnerability reporting.
-
----
-
-## Encryption at rest
-
-Set `ENGRAPHIS_DB_KEY` (or `ENGRAPHIS_DB_KEY_FILE`) and install the extra:
-
-```bash
-pip install "engraphis[encryption]"
-```
-
-The entire main memory database file is transparently encrypted with AES-256 via SQLCipher;
-full-text search, the graph, and every query keep working unchanged. Customer authentication
-and managed-service state use their respective deployment protections. When a key is set for the
-main database, Engraphis **fails closed with an error** rather than silently falling back to
-plaintext. Generate a strong key:
-
-```bash
-python -c "import secrets; print(secrets.token_hex(32))"
-```
-
-When using `ENGRAPHIS_DB_KEY_FILE`, provision a regular secret file readable only by the
-service identity. Engraphis rejects links, reparse points, hard links, malformed text, and
-oversized key files rather than following an unexpected filesystem object.
-
-> An existing plaintext database cannot be opened with a key: migrate it (dump → import
-> into a fresh keyed DB). See `.env.example` for all encryption options.
-
----
-
-## Import files and folders
-
-The dependency-free universal core scans Markdown, plain text, RST, HTML, JSON/JSONL, CSV/TSV,
-configuration/XML text, source code, RTF, DOCX/ODT, XLSX/ODS, PPTX/ODP, and EPUB into the normal
-v2 memory path. Installed local resource adapters add PDF text, image OCR, and explicitly
-local-model audio/video transcription.
-Start with a zero-write
-preview, then confirm the same source collection explicitly:
-
-```bash
-engraphis import documents /path/to/collection --workspace acme --dry-run
-engraphis import documents /path/to/collection --workspace acme --repo product --yes
-```
-
-The CLI never downloads an embedding model during import. Use a model that is already cached,
-set `ENGRAPHIS_EMBED_MODEL=local:/absolute/model/path`, or explicitly set
-`ENGRAPHIS_EMBED_MODEL` to an empty value to use dependency-free deterministic hashing in
-lexical degraded mode.
-
-The dashboard’s **Import local documents** flow offers the same preview, target scope, source
-label, conflict policy, cancellation, and resumable progress. Re-imports are idempotent,
-preserve temporal history, and report source removals without hard-deleting memories. Obsidian
-remains the rich Markdown adapter for frontmatter, aliases, wikilinks, and attachment references:
-
-```bash
-engraphis import obsidian /path/to/vault --workspace acme --dry-run
-```
-
-See the [document import guide](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/DOCUMENT_IMPORT.md)
-for supported formats, source safety, resume and conflict behavior, optional adapters, and
-limitations; see the [Obsidian adapter guide](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/OBSIDIAN_IMPORT.md)
-for Markdown-specific behavior.
-
----
-
-## Consolidation and automation
-
-Manual consolidation is free, local, and dry-run by default; use the dashboard, SDK, CLI, or
-MCP. Hosted Pro and Team automation is optional managed compute that produces reviewable
-proposals rather than silently changing local data. See [hosted plans](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/HOSTED_PLANS.md),
-[licensing](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/LICENSING.md), and the [MCP tool reference](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/MCP_TOOLS.md) for scope and use.
-
----
-
-## Configuration
-
-Values come from the process environment. Engraphis also loads the owner-private
-`~/.engraphis/config.env`; `ENGRAPHIS_ENV_FILE` can select another absolute owner-private regular
-file. It never searches the working directory for `.env`, and explicit process variables win.
-
-| Env Var | Default | Description |
-|---------|---------|-------------|
-| `ENGRAPHIS_ENV_FILE` | `~/.engraphis/config.env` | Optional trusted config leaf selected before trusted values load. Its bounded dependency-free parser performs no interpolation. An explicit value must be an absolute path to an owner-private regular file; arbitrary working-directory `.env` files are ignored. |
-| `ENGRAPHIS_DB_PATH` | Source: `/engraphis.db`; installed: platform user-data directory | SQLite database file. Installed defaults are `%LOCALAPPDATA%\engraphis\engraphis.db` (Windows), `~/Library/Application Support/engraphis/engraphis.db` (macOS), and `$XDG_DATA_HOME/engraphis/engraphis.db` or `~/.local/share/engraphis/engraphis.db` (Linux). The environment variable overrides every default; a relative value is resolved from the trusted `~/.engraphis/config.env` directory so launch CWD cannot select a different workspace database. |
-| `ENGRAPHIS_HOST` | `127.0.0.1` | Server bind address |
-| `ENGRAPHIS_PORT` | `8700` | Dashboard port |
-| `ENGRAPHIS_SERVICE_MODE` | `customer` | The public package supports only `customer`; hosted vendor, relay, compute, and worker roles are not distributed here |
-| `ENGRAPHIS_API_TOKEN` | Not set | Optional bearer credential for this single-user local customer node; never reuse a hosted credential |
-| `ENGRAPHIS_CORS_ORIGINS` | loopback on `ENGRAPHIS_PORT` | Comma-separated REST CORS allow-list; defaults to `127.0.0.1` and `localhost` on the configured port |
-| `ENGRAPHIS_INDEX_ROOTS` | Working, home, and temporary directories | Optional path-separator-delimited absolute-path allow-list that replaces the default roots accepted by local code indexing |
-| `ENGRAPHIS_HTTP_INDEX_ROOT` | First `ENGRAPHIS_INDEX_ROOTS` entry, or current directory | Single root for dashboard and REST `POST /api/code/index`; submitted paths resolve beneath it. An explicit root (or fallback entry) must be absolute; an explicit HTTP root is included in the engine-approved set. MCP and CLI indexing continue to use `ENGRAPHIS_INDEX_ROOTS`. |
-| `ENGRAPHIS_DB_KEY` | Not set | Encrypt the database at rest (SQLCipher). Or use `ENGRAPHIS_DB_KEY_FILE` |
-| `ENGRAPHIS_EMBED_MODEL` | `sentence-transformers/all-MiniLM-L6-v2` | sentence-transformers model |
-| `ENGRAPHIS_EMBED_REVISION` | Not set | Optional immutable lowercase 40-hex Hugging Face commit for the embedding model. Loaded Hub commits or local artifact manifests identify persistent vector spaces; unresolved mutable identities keep vector recall fail-closed. |
-| `ENGRAPHIS_RERANK_MODEL` | Not set | Optional sentence-transformers cross-encoder reranker |
-| `ENGRAPHIS_RERANK_REVISION` | Not set | Optional immutable lowercase 40-hex Hugging Face commit for the reranker |
+# Engraphis
+
+[](https://pypi.org/project/engraphis/)
+[](https://github.com/Coding-Dev-Tools/engraphis/blob/main/LICENSE)
+[](https://buymeacoffee.com/Jaixii)
+
+[https://engraphis.com/](https://engraphis.com/)
+
+[https://discord.com/invite/Wfr2ejBmY](https://discord.com/invite/Wfr2ejBmY)
+
+**Give your AI agents a memory. See it, search it, and maintain it, all in a beautiful WebUI on your own machine.**
+
+
+
+
+ Knowledge Graph · run engraphis-dashboard to see it live
+
+
+**Grounded, not guessed.** Memory with receipts. Local by default. [Explore the proof gallery](https://github.com/Coding-Dev-Tools/engraphis/tree/main/docs/advertising) or [read the campaign guide](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/advertising/campaign.md).
+
+---
+
+> **Open-core boundary:** this repository contains the free local engine, dashboard, MCP server,
+> and customer-side clients. Hosted sync, analytics, automation, and team services run on the
+> official hosted service; their server implementations are not distributed here.
+
+> **Support continued Engraphis development with Pro.** [Start a 3-day Pro trial](https://api.engraphis.com/account?plan=pro&interval=monthly&utm_source=engraphis&utm_medium=docs&utm_campaign=pro_conversion&utm_content=readme_intro&trial=pro#billing)
+> or [subscribe to Pro](https://api.engraphis.com/account?plan=pro&interval=monthly&utm_source=engraphis&utm_medium=docs&utm_campaign=pro_conversion&utm_content=readme_intro#billing).
+
+---
+
+## Measured token and context savings
+
+### Runtime estimator
+
+The dashboard Overview and Audit/Receipts views also show a receipt-backed estimate from
+real context deliveries. It compares the host history or retrieved source baseline with the
+context Engraphis actually emitted, keeps token counters and release versions separate, and
+labels adaptive history reductions separately from packing savings. Receipts without estimator
+metadata remain historical/unclassified. This measures estimated prompt-context reduction; it
+does not measure provider billing. The `/context-savings` API and
+`engraphis_context_savings` MCP tool aggregate the complete history across all visible workspaces
+by default, or accept an explicit workspace plus optional `from_ts`, `to_ts`, and
+`release_version` filters.
+
+
+
+
+ Less repeated history means more room for the task, tools, and useful evidence.
+
+
+
+See benchmark details and reproduce the results
+
+### Controlled before-and-after example
+
+| Retrieval mode | Mean returned memory content | Recall@5 |
+|---|---:|---:|
+| Whole documents | 740.3 tokens | 1.000 |
+| Engraphis structure-aware chunks | 214.3 tokens | 1.000 |
+
+The chunked mode returns the relevant passage instead of the whole document: **526.0 fewer tokens
+per question**. Under the same model-context budget, that leaves roughly **526 tokens** for task
+instructions or other relevant evidence. This is evidence ID `offline-chunking` in the registered
+artifact below.
+
+### Measurement details and reproducibility
+
+The table below contains every exact token/context aggregate currently published here and keeps
+its counting boundary explicit.
+
+| What is counted | Comparison | Measured reduction | Quality held constant |
+|---|---|---|---|
+| Retrieved top-5 memory content, averaged per question | Whole documents: **740.3** tokens → structure-aware chunks: **214.3** tokens | **526.0 fewer tokens per question** (**71.1% lower**, about **3.5× smaller**) | Recall@5 **1.000** in both modes across 6 documents and 18 questions |
+| Smallest returned memory that contains the reference evidence | Whole documents: **162.2** tokens → chunks: **42.4** tokens | **119.8 fewer tokens to evidence** (**73.9% lower**, about **3.8× smaller**) | The same 18 questions had a returned evidence-holding memory in both modes |
+| Full versus compact recall payload proxy across one 26-question pass within a 260-timed-recall CodeMem run | Full proxy: **23,810** `engraphis.regex.v1` tokens → compact proxy: **10,202** tokens | **13,608 proxy tokens avoided** (**57.15% lower**) | 26 payload samples; 260 timed recalls; Recall@5, hit@5, and answer-token recall all **1.000** |
+| Packed prompt-context usage in the same 26-question CodeMem sample pass | Hard budget: **1,500** tokens; observed mean: **85.38**; observed maximum: **108** | A hard cap prevents a recall from exceeding its configured context budget | This is usage accounting, not a before/after savings comparison |
+
+These values are evidence IDs `offline-chunking` and `offline-performance` in
+[`offline-fixtures-v1.json`](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/benchmark-evidence/offline-fixtures-v1.json),
+SHA-256
+`0f60b0868444f676fe14c5f94d7db2c475e22669930c4d760881d0842eaa6800`.
+[`BENCHMARKS.md`](https://github.com/Coding-Dev-Tools/engraphis/blob/main/BENCHMARKS.md#public-numeric-evidence-registry)
+records the matching suite digest, exact commands, and per-command config digests. External,
+model-dependent, consolidation, productivity, and latency results remain unpublished until the
+same evidence exists for them.
+
+The compact payload shape avoids duplicating full memory bodies when the packed context and source
+list are enough. The evaluator tokenizes JSON-shaped full and compact payload proxies built from
+recall results; it does **not** serialize the MCP envelope or measure a transport response. The
+fixture therefore does not measure model-provider charges, end-to-end task time, or customer cost
+savings.
+
+The measures are deliberately separate and **must not be added together**: chunking counts the
+content of retrieved memory records before `ContextPacker`, whereas compact recall counts a
+serialized JSON-shape payload proxy. “Tokens to evidence” is the size of the smallest
+retrieved memory record holding the reference evidence; it is not latency or end-to-end answer
+accuracy. Chunking creates more focused stored records, so this is a context-efficiency result,
+not a storage-reduction claim.
+
+Reproduce the registered quality and token/context measurements without a network connection or
+API key:
+
+```bash
+python -m eval.grounded
+python -m eval.chunking_eval --dataset eval/datasets/longdoc.jsonl --k 5
+python -m eval.performance --dataset eval/datasets/codemem.jsonl --k 5 --iterations 10 --json
+```
+
+These are small deterministic correctness and efficiency fixtures, not official LoCoMo /
+LongMemEval QA scores or a third-party leaderboard result. Compact-response counts use the exact
+`engraphis.regex.v1` counter; the chunking evaluation uses its documented deterministic
+normalized-character estimator. Chunking measures retrieved memory content, while compact recall
+measures a serialized JSON-shape payload proxy, not an MCP transport response. See the registered
+artifact and [`BENCHMARKS.md`](https://github.com/Coding-Dev-Tools/engraphis/blob/main/BENCHMARKS.md)
+for definitions, limitations, and canonical external-evaluation requirements.
+
+
+
+---
+
+## Full Engraphis install: pip install "engraphis[all]"
+
+The complete `engraphis[all]` install is the default way to use Engraphis: it includes the local
+dashboard, Smart MCP server, documents, Cloud Sync client, and supported optional integrations.
+Python 3.10+ is required.
+
+```bash
+pip install "engraphis[all]"
+engraphis-dashboard
+```
+
+The dashboard opens at [http://127.0.0.1:8700](http://127.0.0.1:8700). Local memory needs no
+account or API key.
+
+### Smaller installation options
+
+Use a smaller package only when you intentionally need a limited surface. The NumPy-only core
+continues to support Python 3.9+.
+
+| Goal | Install | Start |
+|---|---|---|
+| Local dashboard and REST API | `pip install "engraphis[server]"` | `engraphis-dashboard` |
+| Coding-agent memory over Smart MCP | `pip install "engraphis[mcp]"` | `codex mcp add engraphis -- engraphis-mcp` |
+| Native SQLite vector acceleration | `pip install "engraphis[vector]"` | Server entrypoints select it automatically |
+| Offline Python library | `pip install engraphis` | `MemoryService.create("engraphis.db")` |
+
+For MCP clients other than Codex, configure a stdio server whose command is `engraphis-mcp`; see
+the [agent connection guide](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/AGENT_CONNECT.md).
+
+### Updating
+
+Use `engraphis-update` to upgrade the installation using its detected install method. Package
+metadata does not record which extras were selected, so the updater defaults to the safe
+superset `engraphis[all]` rather than silently dropping an optional surface. For a deliberate
+selection, set `ENGRAPHIS_UPDATE_EXTRAS` to a comma-separated list (for example
+`server,mcp`), or set it to `none` for the base package only.
+
+> **Upgrading to 1.4:** `engraphis-mcp` now exposes the nine-tool Smart gateway. Integrations that
+> require the former 34 direct tool names should run `engraphis-mcp-classic`. The SQLite schema
+> in the 1.4.0 release was version 9. Existing v7-to-v8 databases already contain `confidence`
+> and `pinned_at`/`unpinned_at`; v9 adds the `memory_tombstones` repository-scope column/table
+> and performs a one-time entity-canonicalization repair, then migrates automatically on first
+> open. A tombstone with a known `repo_id` is terminal only in that repository; legacy repo-less
+> tombstones remain global. See the [1.4.0 release notes](https://github.com/Coding-Dev-Tools/engraphis/blob/main/CHANGELOG.md#140---2026-08-02).
+
+> **Upgrading to 1.5:** schema 10 bounds legacy retention state and schema 11 backfills explicit
+> approval only for eligible pre-review local memories. Pending and quarantined evidence remains
+> gated. Existing 1.4.x databases migrate automatically when Engraphis 1.5 opens them; see the
+> [1.5 release notes](https://github.com/Coding-Dev-Tools/engraphis/blob/main/CHANGELOG.md#15---2026-08-04).
+
+> **Upgrading to 1.6:** existing 1.5 databases migrate automatically through schema 12, which
+> classifies content-free erasure markers before sync: existing markers become local-only
+> `never_export`, while new secure erasures become `remote_erasure` only for non-secret
+> `workspace`/`repo` records already eligible for sharing. Schema 13 adds per-memory hybrid
+> logical clocks for deterministic descriptive-state sync and durable, content-free proof that a
+> memory crossed a sync boundary. Schema 14 adds the Obsidian collection and import manifests;
+> schema 15 generalizes them to source-neutral local documents, preserves temporal source lineage
+> across re-imports, binds adapters and target scopes, and retains only bounded, content-free
+> per-job format/result metadata. The schema 16 migration persists each import job's optional session target
+> and requires source lineage and job-item attachments to remain in that exact session. See the
+> [1.6 release notes](https://github.com/Coding-Dev-Tools/engraphis/blob/main/CHANGELOG.md#16---2026-08-15).
+
+---
+
+## What Engraphis gives an agent
+
+An agent should not have to reconstruct a project from scattered chat history on every task.
+Engraphis turns local project knowledge into scoped, time-aware memory; retrieves the evidence
+that supports the current question; and returns a bounded, attributable context packet.
+
+The core task is continuity: retrieve the current, supported project decision without dragging the
+whole history into the next prompt. See [measured token and context savings](#measured-token-and-context-savings)
+for the short version of how much less history an agent has to carry.
+
+| Agent need | What Engraphis changes |
+|---|---|
+| Remember a project across sessions | Stores typed memory in a `workspace → repo → session` hierarchy and provides a last-session handoff. |
+| Find support for the current task | Fuses vector, lexical, graph, and code-aware retrieval instead of relying on one search signal; `fast` can skip graph traversal for small or latency-sensitive vaults. |
+| Know what is true now and what changed | Preserves bi-temporal history and supersession chains instead of silently overwriting a fact. |
+| Avoid confident guesses | Returns cited evidence or explicitly abstains when support is too weak. |
+| Avoid dragging the whole project into every prompt | Packs context to a configured hard budget and can return a compact MCP response. |
+| Keep knowledge in the operator's control | Runs local-first and offline-capable, with scopes, audit records, and optional privacy-safe receipts. |
+
+## Dashboard and local UI
+
+The Engraphis dashboard opens `http://127.0.0.1:8700`. Local memory needs no cloud account,
+signup, or API key and stays in a SQLite file on your machine.
+
+**Ledger** is the primary local interface for recall, memories, graph exploration, provenance,
+workspaces, and manual consolidation. **Classic** preserves the former full tool suite; both use
+the same local data. Switch in **Manage → Settings → Interface** (Ledger) or **Settings →
+Appearance & Engine** (Classic).
+
+### Start it on every platform
+
+| Platform | How |
+|----------|-----|
+| **Windows** | Double-click **Engraphis Dashboard** on your Desktop or Start Menu (install: `engraphis-dashboard --install-shortcuts`) |
+| **macOS** | Double-click **Engraphis Dashboard.app** on your Desktop (install: same command) |
+| **Linux** | Desktop entry in Applications → Development (GNOME/KDE/etc.) |
+| **Docker** | `docker compose up`: see `docker-compose.yml` for the one-command deployment |
+| **Any** | `engraphis-dashboard` in a terminal |
+
+In a source checkout, `scripts/launch_dashboard.ps1` is only a Windows convenience wrapper. It
+delegates configuration, startup health, browser opening, and process lifecycle to the same
+`engraphis-dashboard` entrypoint rather than maintaining a second behavior path.
+
+### Accessibility-first inspection, built in
+
+Inspect memories, supersession diffs, recall scores, timelines, links, consolidation, and audit
+records in the dashboard. The offline graph renderer is vendored, and the interface is keyboard-
+navigable with light and dark themes. Graph exploration offers a focused **High quality** view and
+an explicit worker-backed **Show all nodes** view for complete entity projections up to 20,000
+nodes and 200,000 relationships; see the [graph performance profiles](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/GRAPH_PERFORMANCE.md).
+
+---
+
+## How it works
+
+Engraphis gives agents durable, scoped, *explainable* project knowledge. The local engine combines
+Ebbinghaus decay, bi-temporal facts, and hybrid vector/lexical/graph recall; it runs offline with
+SQLite, local embeddings, and `numpy` only.
+
+- **Grounded and governed:** deterministic conflict resolution, cited answers or abstention,
+ explicit correction/promotion/forgetting, and a complete history.
+- **Agent-ready:** MCP tools, hard-budget context packets, handoffs, and code-aware retrieval.
+- **Auditable:** content-free receipt chains, provenance, and temporal/entity/code relationships.
+- **Practical:** local file and code ingest, optional PDF/OCR/transcription, and SQLCipher at rest.
+
+### Optional LLM providers
+
+The memory engine, embeddings, conflict resolution, and recall stay local without an LLM. An
+explicitly configured provider adds structured extraction, cited synthesis, consolidation, and
+retention supervision. Configure it in **Settings → Connect an LLM**. The activity view records
+outcomes, never keys, prompts, or raw provider responses. See the
+[LLM provider guide](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/LLM_PROVIDERS.md) for setup and privacy choices.
+
+> Privacy boundary: text sent to an explicitly selected provider leaves the local process under
+> that provider's terms. Use `ENGRAPHIS_RETENTION_SUPERVISOR=none` (the default) and the offline
+> `chunk` extractor when ingestion must remain entirely local.
+
+Choose and configure an external LLM with the [LLM provider guide](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/LLM_PROVIDERS.md),
+including OpenAI, Anthropic, Google, OpenRouter, Ollama, Cohere Command, Command Code Provider,
+and other compatible endpoints. The guide also covers Codex subscription MCP connections.
+
+---
+
+## Install
+
+```bash
+pip install "engraphis[all]" # self-hosted dashboard, MCP, code graph, documents, transcription, PostgreSQL, and Cloud Sync
+pip install "engraphis[server]" # dashboard + REST API
+pip install "engraphis[mcp]" # MCP server only
+pip install "engraphis[documents]" # PDF + image OCR bindings
+pip install "engraphis[transcription]" # faster-whisper audio/video
+pip install "engraphis[postgres]" # PostgreSQL schema introspection
+pip install "engraphis[code]" # tree-sitter code graph indexing
+pip install "engraphis[vector]" # native sqlite-vec exact-KNN acceleration
+pip install "engraphis[cloud-sync]" # Cloud Sync client crypto/runtime
+pip install "engraphis[encryption]" # SQLCipher encryption-at-rest extra
+pip install engraphis # core library: numpy only, fully offline
+```
+
+The official Docker image includes the local Tesseract executable for image OCR. Outside
+Docker, the `documents` extra installs its Python bindings; install Tesseract through your
+operating system as well if you enable image OCR.
+
+The NumPy-only core library supports Python 3.9+. Current patched releases of the WebUI
+stack, MCP SDK, image parser, and Cloud Sync client require Python 3.10+, so use Python 3.10
+or newer for the `server`, `mcp`, `documents`, `cloud-sync`, or `all` installation paths.
+
+The default `NumpyVectorIndex` performs an exact full scan. There is no universal memory-count
+cutoff because latency depends on vector size, hardware, filters, and the rest of the recall
+pipeline. Measure your machine with `python -m eval.vector_scale --backend numpy`, then run
+`python -m eval.performance` on a representative corpus. If exact scans miss your latency target,
+install `engraphis[vector]`, create the engine with `vector_backend="sqlite-vec"`, and remeasure.
+The stable sqlite-vec `vec0` backend executes exact KNN in native code; it is acceleration, not a
+claim of sublinear ANN scaling. See [BENCHMARKS.md](https://github.com/Coding-Dev-Tools/engraphis/blob/main/BENCHMARKS.md) for the reproducible commands
+and reporting limits.
+
+Dashboard, REST, and MCP entrypoints default to `ENGRAPHIS_VECTOR_BACKEND=auto`: they use
+sqlite-vec when the `vector` extra is installed and compatible, then safely fall back to NumPy.
+Programmatic `MemoryEngine.create()` and `MemoryService.create()` retain the deterministic
+`numpy` default unless a backend is requested explicitly.
+Use `python -m eval.vector_scale --backend sqlite-vec` for an input-identical direct-search
+comparison; setup/index-build time is explicitly excluded from the timed search envelope.
+
+Persistent vectors fail closed unless the embedder can publish a durable, secret-free space
+fingerprint. Sentence Transformers use the loaded Hub commit or a manifest of local artifacts;
+when a remote model's immutable identity cannot be resolved, persistent vector recall remains
+gated instead of mixing spaces. For programmatic OpenAI-compatible embeddings, construct
+`ApiEmbedder` with an operator/provider `space_version`; without it the adapter remains usable for
+ephemeral embedding only. Its `base_url` may be a provider root or a `/v1` root and is normalized
+to exactly one `/v1/embeddings` endpoint.
+
+`sqlcipher3-binary` publishes CPython manylinux x86-64 wheels. On that target,
+`engraphis[encryption]` installs the driver. The cross-platform `all` extra deliberately
+omits it so `all` remains resolvable on macOS, Windows, Linux ARM, and musl; on those
+targets, provision a compatible SQLCipher driver separately before enabling a database
+key. The programmatic core remains plaintext unless a database key is configured. For a
+fresh database, `engraphis-init` enables SQLCipher automatically when a compatible driver is
+available, creates a private key sidecar, and can be overridden with `--no-encryption`.
+
+> **Linux / macOS:** if `pip install` fails with `error: externally-managed-environment`,
+> your system Python is marked read-only (PEP 668). Install into a virtual environment
+> instead. Run `python3 -m venv venv && source venv/bin/activate && pip install "engraphis[server]"`
+> Alternatively, use Docker (`docker compose up`). `pipx install "engraphis[server]"` also works.
+
+> First run downloads `all-MiniLM-L6-v2` (~80 MB). Without it, the engine falls back
+> to deterministic feature hashing so it always runs offline. That fallback captures lexical
+> overlap, not meaning: recall and grounded MCP responses set `degraded_mode=true` and
+> `semantic_support=false`, and disable vector retrieval plus semantic-cosine evidence. Install
+> a declared embedding model for semantic retrieval.
+
+> To require a model that is already local, set `ENGRAPHIS_EMBED_MODEL=local:/absolute/model/path`
+> or `local:`. This path never downloads a model. If it is unavailable, Engraphis
+> explicitly enters lexical degraded mode instead of presenting hash-vector scores as semantic.
+
+---
+
+## Quickstart: dashboard
+
+```bash
+pip install "engraphis[server]"
+engraphis-dashboard # → http://127.0.0.1:8700
+engraphis-dashboard --install-shortcuts # → Desktop + Start Menu icons
+```
+
+### Docker
+
+```bash
+docker compose up # → http://127.0.0.1:8700
+```
+
+For Docker Compose persistence and loopback-port configuration, see the
+[Docker deployment guide](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/DOCKER.md).
+`engraphis-server` and `engraphis server` are headless compatibility aliases
+for this same v2 service, so every public surface has the same scoped recall and retention model.
+
+For optional LAN exposure, token configuration, and HTTP MCP setup, see the
+[Docker deployment guide](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/DOCKER.md).
+
+Set `ENGRAPHIS_API_TOKEN` to require API authentication and `ENGRAPHIS_DB_KEY` to encrypt
+the local database at rest. Hosted-plan credentials configure customer clients; they do not
+install premium server implementations into this image. See `docker-compose.yml` for options.
+
+---
+
+## Quickstart: MCP server (for coding agents)
+
+```bash
+pip install "engraphis[mcp]"
+engraphis-init # writes ~/.engraphis/config.env + prints config snippets
+claude mcp add engraphis -- engraphis-mcp
+codex mcp add engraphis -- engraphis-mcp # Codex subscription
+
+```
+For Codex subscription setup and verification, see the [agent connection guide](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/AGENT_CONNECT.md)
+and the [LLM provider guide](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/LLM_PROVIDERS.md).
+
+`engraphis-mcp` is zero-configuration Smart MCP: agents begin with nine compact tools for sessions,
+prompt-ready recall, durable memory, governed record read/update, conflict review, action discovery,
+and safe execution. For code graphs,
+governance, audit, or other advanced work, the agent calls `engraphis_discover_actions` and then
+the indicated read or action executor; no profile selection is required. The gateway validates
+the discovered capability again before it runs it, and clients remain responsible for their
+normal destructive-action approval boundary.
+
+Existing clients that pin the historical 34 named tools can use
+`engraphis-mcp-classic` (or `engraphis-mcp-http --classic`). The complete classic inventory,
+including `engraphis_check_update`, is in the [MCP tool reference](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/MCP_TOOLS.md).
+
+### Pi extension
+
+For installation, configuration, lifecycle commands, and the local trust boundary, see the
+[Pi extension guide](https://github.com/Coding-Dev-Tools/engraphis/blob/main/integrations/pi/README.md).
+
+## Quickstart: repository graph
+
+```bash
+pip install "engraphis[code]"
+engraphis-graph index -w acme -r api --root .
+engraphis-graph search -w acme -r api "UserService"
+# `query`/`explain` blend code search with your stored memories: query matches symbol
+# and file NAMES (a full question sentence won't match anything), and explain's answer
+# is drawn from memories recorded against the repo; both are empty on a fresh index.
+engraphis-graph query -w acme -r api "UserService"
+engraphis-graph explain -w acme -r api "why does deploy depend on approval?"
+engraphis-graph path -w acme -r api UserService DatabasePool
+engraphis-graph impact -w acme -r api --root . --git-range origin/main...HEAD
+engraphis-graph prs -w acme -r api --base main --head HEAD
+engraphis-graph export -w acme -r api -o engraphis-graph-out
+engraphis-graph install-merge-driver --root .
+```
+
+The export contains `graph.json`, a self-contained `graph.html`, and `GRAPH_REPORT.md`.
+Indexing supports Python, JavaScript, TypeScript, Go, Rust, Java, C#, C, C++, SQL, and
+Terraform. Tree-sitter is used when available; the dependency-free regex backend remains a
+functional fallback. Definitions, methods, calls, imports, ownership, variables,
+inheritance/implementation, and docstrings/comments are indexed. Indexing is incremental by
+content hash, honors `.engraphisignore`, and does not follow file symlinks outside the repository
+root. Call edges are name-based and best-effort rather than type-resolved. The optional Git merge
+driver validates bounded graph JSON and deterministically unions nodes and edges instead of
+choosing one export side.
+
+For a read-only recall and graph API that can be shared without exposing write operations:
+
+```bash
+pip install "engraphis[server]"
+engraphis-graph-server # API at http://127.0.0.1:8720; schema at /openapi.json
+```
+
+A non-loopback bind fails closed unless `ENGRAPHIS_GRAPH_TOKEN` (or
+`ENGRAPHIS_API_TOKEN`) is set. See [the v3 architecture/design document](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/ARCHITECTURE_V3.md).
+
+---
+
+## Quickstart: Python library
+
+```python
+from engraphis.service import MemoryService
+
+mem = MemoryService.create("engraphis.db")
+mem.remember("Auth migrated from JWT to PASETO.", workspace="acme", repo="api")
+hit = mem.recall("why did we change auth?", workspace="acme", repo="api")
+print(hit["context"])
+```
+
+The same `MemoryService` backs the dashboard and the MCP server. The package root also
+intentionally exposes the low-level engine facade (`MemoryEngine`, `create_memory_engine`)
+for advanced composition, while `MemoryService` remains the high-level service API.
+
+New writes support `session`, `repo`, and `workspace` visibility. `scope="user"` is reserved and
+rejected until records carry an immutable owner identity; it must not be treated as private
+per-person memory. Historical user-scope rows remain workspace-bound for compatibility.
+
+After an upgrade, `stats()` reports prompt-eligibility counts and active embedding-space
+coverage. Zero-result recall identifies a review-gated scope instead of silently looking empty,
+and `engraphis-cli review list|approve` provides a dry-run-first local bulk workflow. Embedding
+model changes trigger a guarded rebuild; vector recall stays disabled until every stored vector
+matches the new fingerprint. See [recall recovery](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/RECALL_RECOVERY.md).
+
+Agent hosts can avoid retrieval when their existing history already fits:
+
+```python
+decision = mem.adaptive_context(
+ "what should the agent do next?",
+ current_history,
+ workspace="acme",
+ repo="api",
+ max_context_tokens=8_192,
+ retrieval_token_budget=1_024,
+)
+prompt_context = decision["context"]
+```
+
+The decision is `history_bypass` when the history fits, `retrieval` when compact evidence is
+strong, and `history_fallback` when weak retrieval should widen back to recent raw history.
+
+For an agent prompt, prefer `engraphis_recall_context`: it returns one hard-budget packed
+`context` plus compact `sources`, deterministic `usage` accounting (`budget_tokens`, `context_tokens`,
+`source_tokens`, `saved_tokens`, `savings_ratio`, `packed_count`, `omitted_count`, and
+`token_counter`), and optional diagnostics. Accounting is exact for the named counter; inject the
+reader's tokenizer when reader-model token parity is required. `engraphis_recall` remains the compatible full-recall
+surface; use `response_mode="compact"` when the packed context is enough and full memory bodies
+would duplicate it. For advanced query-planning configuration, see the
+[architecture guide](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/ARCHITECTURE_V3.md#query-planning).
+
+For bi-temporal reads, `valid_at` selects what was true at a Unix timestamp and `known_at` selects
+what Engraphis had learned then. `as_of` remains a compatibility alias for `valid_at`; supplying
+both is allowed only when they match.
+
+For a mutable claim, pass a stable `subject_key` and optional `claim_kind`, such as
+`subject_key="api.rate_limit", claim_kind="configured_value"`. Offline conflict resolution
+deterministically adds, reinforces, relates, or supersedes records while preserving temporal
+history; it does not need an LLM. Matching claim identities let it supersede substantially
+reworded mutable facts. Without them, the dependency-free lexical embedder cannot reliably infer
+that a paraphrase is a contradiction, so keep both records or use an explicit `correct` operation.
+
+---
+
+## Govern memories without losing history
+
+Engraphis separates automatic write resolution from explicit human governance:
+
+| Operation | Use it when | What happens to history |
+|---|---|---|
+| `remember` | Adding or restating one fact | Adds, reinforces, safely supersedes, or relates an uncertain neighbor |
+| `correct` | Replacing one known-wrong memory | Closes the old validity window and links the replacement |
+| `promote` | A narrow learning now applies more broadly | Writes a wider-scope successor and closes/links the source instead of editing scope in place |
+| `merge` | Combining two or more overlapping memories | Retires every source and creates one memory that supersedes all of them |
+| `retire` | Removing a memory from live recall | Bi-temporally closes it; the audit/history record remains |
+| `consolidate` | Distilling recurring episodic memories automatically | Creates linked semantic digests; source episodes remain live |
+
+Manual N→1 merge is available through `MemoryService.merge()` and `POST /api/merge`:
+
+```python
+a = mem.remember("Deploys happen Friday at 3pm.", workspace="acme")
+b = mem.remember("We deploy Fridays around 15:00.", workspace="acme")
+
+merged = mem.merge(
+ [a["id"], b["id"]],
+ "Deploys ship every Friday at approximately 15:00.",
+ workspace="acme",
+ reason="deduplicate the deployment schedule",
+)
+print(merged["compaction"])
+```
+
+`retire` is intentionally not deletion: it preserves temporal history, FTS, and vector
+evidence for historical reads. If a credential was captured, new writes are blocked before
+storage; for a legacy leak use the explicitly destructive `MemoryService.secure_erase()` or
+`POST /api/secure-erase`/`engraphis_secure_erase`. That flow removes the one memory and local
+FTS/vector-index and derived graph/link rows, runs SQLite secure-delete, WAL checkpoint, and
+VACUUM, and scans recognised local SQLite recovery backups. It cannot erase exports, filesystem
+snapshots, remote peers, unknown backups, or information a running/compromised agent already
+read; rotate the credential. See [secure-erasure limits](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/SECURE_ERASURE.md). `forget`
+remains a deprecated compatibility alias for `retire`.
+
+All sources must belong to the named workspace. The result inherits the strictest source
+sensitivity, remains untrusted if any source was untrusted, and stays pinned if any source was
+pinned. The full multi-predecessor chain remains visible through inspection, Why, and Timeline.
+
+---
+
+## Free forever vs. hosted plans
+
+The core engine, local dashboard, MCP server, and manual consolidation are Apache-2.0 and free.
+**Pro and Team are services** that provide optional access to the official hosted service; its
+control-plane, billing, relay, compute, and Team identity modules live in a private repository.
+They do not limit the local core. See
+[hosted plans](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/HOSTED_PLANS.md), [licensing](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/LICENSING.md), and
+[Cloud Sync](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/SYNC.md) for service boundaries, lifecycle, and pricing.
+
+[Subscribe to Pro](https://api.engraphis.com/account?plan=pro&interval=monthly&utm_source=engraphis&utm_medium=docs&utm_campaign=pro_conversion&utm_content=readme_pricing#billing)
+to support the project and add hosted services.
+
+[Compare hosted plans](https://api.engraphis.com/account?plan=pro&interval=monthly&utm_source=engraphis&utm_medium=docs&utm_campaign=pro_conversion&utm_content=readme_intro#billing)
+when you are ready to evaluate the service boundary and billing options.
+
+| | Free (available now) | Pro: $10/mo or $100/yr | Team: $20/seat/mo or $200/seat/yr |
+|---|---|---|---|
+| Dashboard WebUI (with built-in inspector) | ✓ | ✓ | ✓ |
+| Memory engine + Smart MCP (Classic 34-tool compatibility) | ✓ | ✓ | ✓ |
+| Version-chain diffs, offline knowledge graph | ✓ | ✓ | ✓ |
+| Manual local consolidation (dry-run by default) | ✓ | ✓ | ✓ |
+| Local workspace export (portable v2 JSON: memories, source manifests, graph/code evidence, sessions, audit, and receipts) | ✓ | ✓ | ✓ |
+| Hosted Cloud Sync | | ✓ | ✓ |
+| Hosted Analytics | | ✓ | ✓ |
+| Hosted Auto Consolidation + retention policy | | ✓ | ✓ |
+| Hosted Auto Dreaming + managed proposals | | ✓ | ✓ |
+| Priority support | | ✓ | ✓ |
+| Hosted multi-user dashboard: invitations, logins, roles, seat management | | | ✓ |
+| Hosted Team audit log + CSV export | | | ✓ |
+| 72-hour pending invitations (resend/revoke) | | | ✓ |
+| Scoped, expiring per-user agent and sync tokens | | | ✓ |
+
+---
+
+## MCP tools
+
+Engraphis exposes a zero-configuration Smart MCP gateway plus a 34-tool Classic compatibility
+server across memory, recall, code graphs, governance, sessions, and privacy-safe audit receipts.
+The focused [MCP tool reference](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/MCP_TOOLS.md) is the source for
+the full inventory and parameters.
+
+---
+
+## Graphs and privacy-safe receipts
+
+Memory, entity, and code relationships live in one local graph. Engraphis also provides
+content-free operation receipts for inspectable audit evidence. See the
+[architecture](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/ARCHITECTURE_V3.md), [MCP tool reference](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/MCP_TOOLS.md), and
+[security policy](https://github.com/Coding-Dev-Tools/engraphis/blob/main/SECURITY.md) for the data model, tools, and guarantees.
+
+---
+
+## Cloud sync
+
+Cloud Sync is an optional hosted Pro/Team service. The public package includes the customer client
+and deterministic merge implementation; hosted relay and account operations are separate. See
+[Cloud Sync](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/SYNC.md) for setup, encryption, merge behavior, and the local folder exchange.
+
+The public package ships the same sync client as a console script and CLI verb:
+`engraphis-sync` (installed entry point), `engraphis sync ...`, and
+`python -m scripts.sync --status` for local-only state without network activity. See
+[Cloud Sync](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/SYNC.md) for
+flags, encryption, merge behavior, and the local folder exchange.
+
+---
+
+## Security and trust boundaries
+
+Engraphis is local-first and binds to loopback by default. Read the
+[security policy](https://github.com/Coding-Dev-Tools/engraphis/blob/main/SECURITY.md) before remote deployment or integrating external resources; it
+covers supported versions, data protections, threat model, and vulnerability reporting.
+
+---
+
+## Encryption at rest
+
+Set `ENGRAPHIS_DB_KEY` (or `ENGRAPHIS_DB_KEY_FILE`) and install the extra:
+
+```bash
+pip install "engraphis[encryption]"
+```
+
+The entire main memory database file is transparently encrypted with AES-256 via SQLCipher;
+full-text search, the graph, and every query keep working unchanged. Customer authentication
+and managed-service state use their respective deployment protections. When a key is set for the
+main database, Engraphis **fails closed with an error** rather than silently falling back to
+plaintext. Generate a strong key:
+
+```bash
+python -c "import secrets; print(secrets.token_hex(32))"
+```
+
+When using `ENGRAPHIS_DB_KEY_FILE`, provision a regular secret file readable only by the
+service identity. Engraphis rejects links, reparse points, hard links, malformed text, and
+oversized key files rather than following an unexpected filesystem object.
+
+> An existing plaintext database cannot be opened with a key: migrate it (dump → import
+> into a fresh keyed DB). See `.env.example` for all encryption options.
+
+---
+
+## Import files and folders
+
+The dependency-free universal core scans Markdown, plain text, RST, HTML, JSON/JSONL, CSV/TSV,
+configuration/XML text, source code, RTF, DOCX/ODT, XLSX/ODS, PPTX/ODP, and EPUB into the normal
+v2 memory path. Installed local resource adapters add PDF text, image OCR, and explicitly
+local-model audio/video transcription.
+Start with a zero-write
+preview, then confirm the same source collection explicitly:
+
+```bash
+engraphis import documents /path/to/collection --workspace acme --dry-run
+engraphis import documents /path/to/collection --workspace acme --repo product --yes
+```
+
+The CLI never downloads an embedding model during import. Use a model that is already cached,
+set `ENGRAPHIS_EMBED_MODEL=local:/absolute/model/path`, or explicitly set
+`ENGRAPHIS_EMBED_MODEL` to an empty value to use dependency-free deterministic hashing in
+lexical degraded mode.
+
+The dashboard’s **Import local documents** flow offers the same preview, target scope, source
+label, conflict policy, cancellation, and resumable progress. Re-imports are idempotent,
+preserve temporal history, and report source removals without hard-deleting memories. Obsidian
+remains the rich Markdown adapter for frontmatter, aliases, wikilinks, and attachment references:
+
+```bash
+engraphis import obsidian /path/to/vault --workspace acme --dry-run
+```
+
+See the [document import guide](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/DOCUMENT_IMPORT.md)
+for supported formats, source safety, resume and conflict behavior, optional adapters, and
+limitations; see the [Obsidian adapter guide](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/OBSIDIAN_IMPORT.md)
+for Markdown-specific behavior.
+
+---
+
+## Consolidation and automation
+
+Manual consolidation is free, local, and dry-run by default; use the dashboard, SDK, CLI, or
+MCP. Hosted Pro and Team automation is optional managed compute that produces reviewable
+proposals rather than silently changing local data. See [hosted plans](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/HOSTED_PLANS.md),
+[licensing](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/LICENSING.md), and the [MCP tool reference](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/MCP_TOOLS.md) for scope and use.
+
+---
+
+## Configuration
+
+Values come from the process environment. Engraphis also loads the owner-private
+`~/.engraphis/config.env`; `ENGRAPHIS_ENV_FILE` can select another absolute owner-private regular
+file. It never searches the working directory for `.env`, and explicit process variables win.
+
+| Env Var | Default | Description |
+|---------|---------|-------------|
+| `ENGRAPHIS_ENV_FILE` | `~/.engraphis/config.env` | Optional trusted config leaf selected before trusted values load. Its bounded dependency-free parser performs no interpolation. An explicit value must be an absolute path to an owner-private regular file; arbitrary working-directory `.env` files are ignored. |
+| `ENGRAPHIS_DB_PATH` | Source: `/engraphis.db`; installed: platform user-data directory | SQLite database file. Installed defaults are `%LOCALAPPDATA%\engraphis\engraphis.db` (Windows), `~/Library/Application Support/engraphis/engraphis.db` (macOS), and `$XDG_DATA_HOME/engraphis/engraphis.db` or `~/.local/share/engraphis/engraphis.db` (Linux). The environment variable overrides every default; a relative value is resolved from the trusted `~/.engraphis/config.env` directory so launch CWD cannot select a different workspace database. |
+| `ENGRAPHIS_HOST` | `127.0.0.1` | Server bind address |
+| `ENGRAPHIS_PORT` | `8700` | Dashboard port |
+| `ENGRAPHIS_SERVICE_MODE` | `customer` | The public package supports only `customer`; hosted vendor, relay, compute, and worker roles are not distributed here |
+| `ENGRAPHIS_API_TOKEN` | Not set | Optional bearer credential for this single-user local customer node; never reuse a hosted credential |
+| `ENGRAPHIS_CORS_ORIGINS` | loopback on `ENGRAPHIS_PORT` | Comma-separated REST CORS allow-list; defaults to `127.0.0.1` and `localhost` on the configured port |
+| `ENGRAPHIS_INDEX_ROOTS` | Working, home, and temporary directories | Optional path-separator-delimited absolute-path allow-list that replaces the default roots accepted by local code indexing |
+| `ENGRAPHIS_HTTP_INDEX_ROOT` | First `ENGRAPHIS_INDEX_ROOTS` entry, or current directory | Single root for dashboard and REST `POST /api/code/index`; submitted paths resolve beneath it. An explicit root (or fallback entry) must be absolute; an explicit HTTP root is included in the engine-approved set. MCP and CLI indexing continue to use `ENGRAPHIS_INDEX_ROOTS`. |
+| `ENGRAPHIS_DB_KEY` | Not set | Encrypt the database at rest (SQLCipher). Or use `ENGRAPHIS_DB_KEY_FILE` |
+| `ENGRAPHIS_EMBED_MODEL` | `sentence-transformers/all-MiniLM-L6-v2` | sentence-transformers model |
+| `ENGRAPHIS_EMBED_REVISION` | Not set | Optional immutable lowercase 40-hex Hugging Face commit for the embedding model. Loaded Hub commits or local artifact manifests identify persistent vector spaces; unresolved mutable identities keep vector recall fail-closed. |
+| `ENGRAPHIS_RERANK_MODEL` | Not set | Optional sentence-transformers cross-encoder reranker |
+| `ENGRAPHIS_RERANK_REVISION` | Not set | Optional immutable lowercase 40-hex Hugging Face commit for the reranker |
| `ENGRAPHIS_REQUIRE_IMMUTABLE_MODELS` | `false` | When enabled, require a 40-hex commit before loading remote embedding models, rerankers, or chunk tokenizers; `local:` selectors and filesystem paths remain permitted |
| `ENGRAPHIS_REQUIRE_EXACT_BACKENDS` | `false` | When enabled, dashboard and standalone MCP startup fails if a configured optional backend is unavailable instead of silently falling back |
-| `ENGRAPHIS_EXTRACTOR` | `none` | `none` = verbatim; `chunk` = offline structure-aware chunks; `llm` = free-form LLM facts; `llm_structured` = schema-validated facts + graph metadata |
-| `ENGRAPHIS_CHUNK_TOKENIZER_MODEL` | Not set | Optional Hugging Face tokenizer used to enforce chunk budgets with the downstream reader's real tokenization; requires the optional `transformers` package |
-| `ENGRAPHIS_CHUNK_TOKENIZER_REVISION` | Not set | Optional immutable tokenizer/model revision recorded in the chunk-counter identity; pin this for reproducible benchmark artifacts |
-| `ENGRAPHIS_GRAPH_EXTRACTOR` | `regex` | `regex` = offline heuristic NER; `none` = disable heuristic text extraction (validated `llm_structured` metadata still feeds the graph) |
-| `ENGRAPHIS_RETENTION_SUPERVISOR` | `none` | `none` = deterministic only; `llm` = sends a bounded excerpt to the configured provider for advisory ephemeral/normal/critical classification |
-| `ENGRAPHIS_ALLOW_AUTOMATIC_CRITICAL_RETENTION` | `false` | Opt in only when an LLM supervisor may automatically assign the long-lived `critical` class; explicit user-selected critical retention is unaffected |
-| `ENGRAPHIS_WHISPER_MODEL` | Not set | Enables local faster-whisper audio/video transcription |
-| `ENGRAPHIS_POSTGRES_DSN` | Not set | CLI-only PostgreSQL source; used for the connection and never stored |
-| `ENGRAPHIS_POSTGRES_CONNECT_TIMEOUT` | `10` | PostgreSQL introspection connection timeout in seconds (bounded to 1–120) |
-| `ENGRAPHIS_POSTGRES_STATEMENT_TIMEOUT_MS` | `30000` | Per-introspection PostgreSQL statement timeout in milliseconds (bounded to 1–300000) |
-| `ENGRAPHIS_GRAPH_TOKEN` | Not set | Bearer token for `engraphis-graph-server`; required off-loopback |
-| `ENGRAPHIS_GRAPH_HOST` / `ENGRAPHIS_GRAPH_PORT` | `127.0.0.1` / `8720` | Read-only graph/recall server bind address |
-| `ENGRAPHIS_LLM_PROVIDER` | `openai` | `openai \| anthropic \| google \| openrouter \| custom` |
-| `ENGRAPHIS_LLM_MODEL` | `gpt-4o-mini` | Model name (provider-specific) |
-| `ENGRAPHIS_LLM_API_KEY` | Not set | API key for chat/synthesis, `llm` / `llm_structured` extraction, and structured consolidation |
-| `ENGRAPHIS_LLM_BASE_URL` | Not set | Base URL for openrouter / custom OpenAI-compatible endpoints |
-| `ENGRAPHIS_LLM_AUTO_EXTRACT` | `0` | Opt in to switching the running engine to `llm_structured` after a successful live connection test; the dashboard's extraction Off button persists `0`, and its On button restores `1` |
-| `ENGRAPHIS_FORWARDED_ALLOW_IPS` | *(none)* | Proxies trusted for forwarded client/TLS headers (`*` only when the service is reachable exclusively through that proxy) |
-| `ENGRAPHIS_LOCAL_TRUSTED_PEERS` | *(none)* | Exact peers/CIDRs treated as local without forwarding headers; use only for trusted Docker/LAN peers, never public deployments |
-| `ENGRAPHIS_UPDATE_CACHE` | `86400` | Update-check cache TTL in seconds, bounded to `1..31622400`; this is never a cache-file path |
-| `ENGRAPHIS_CLOUD_CONTROL_URL` | hosted default | Official entitlement, organization, and credential control API. A saved rotating credential stays bound to the control endpoint recorded for its family; reconnect to change it. |
-| `ENGRAPHIS_CLOUD_COMPUTE_URL` | hosted default | Official Analytics and managed-automation API. A saved rotating credential stays bound to its recorded compute endpoint; reconnect to change it. |
-| `ENGRAPHIS_CLOUD_ORGANIZATION_ID` | Not set | Hosted organization bound to this customer session |
-| `ENGRAPHIS_CLOUD_REFRESH_CREDENTIAL` | Not set | Bootstrap-only rotating hosted credential; after first use the owner-only cloud session replacement takes precedence |
-| `ENGRAPHIS_CLOUD_TOKEN_SUBJECT` | `member` | Subject fixed during hosted bootstrap (`device` or `member`); set explicitly with an environment-only refresh credential |
-| `ENGRAPHIS_CLOUD_ACCESS_TOKEN` | Not set | Optional short-lived access token for ephemeral jobs |
-| `ENGRAPHIS_MANAGED_COMPUTE_CONSENT` | *(auto)* | Operator override only; default follows whether a cloud session is configured (connected = allowed, local-only = never). `0` opts a connected installation out; `1` permits local snapshot preparation but does not create a cloud credential or authorize an upload |
-
-See `.env.example` for the full variable inventory. Supply those values through the process
-environment or the trusted config file above; copying it to an arbitrary `./.env` does not make
-Engraphis load it.
-
----
-
-## Project structure
-
-```
-engraphis/
-├── engraphis/
-│ ├── core/ # v2 engine: interfaces, store, recall, scoring, schema, sync
-│ ├── backends/ # pluggable embedder / vector index / reranker / codegraph / sync transports / encryption
-│ ├── factory.py # outer v2 composition root; selects and injects concrete backends
-│ ├── service.py # validated MemoryService facade
-│ ├── mcp_server.py # Smart MCP gateway + 34-tool Classic compatibility server
-│ ├── dashboard_app.py # dashboard WebUI (FastAPI)
-│ ├── dashboard_assets/ # primary Ledger interface + graph engine
-│ ├── classic_assets/ # selectable full operator dashboard backup
-│ ├── read_only_api.py # token-protected recall/repository-graph HTTP surface
-│ ├── hosted_client.py # hosted URLs, plan labels, and endpoint validation only
-│ ├── licensing.py # compatibility facade for hosted presentation metadata
-│ ├── cloud_session.py # rotating hosted customer-session client
-│ ├── cloud_features.py # consented managed-feature protocol client
-│ ├── config.py / app.py # env settings / REST server
-│ └── static/ # compatibility dashboard asset paths
-├── eval/ # offline retrieval eval harness + datasets
-├── tests/ # offline-first pytest suite and release/security contracts
-├── scripts/ # dashboard, server, graph, CLI, connect, update, consolidation, sync
-├── docs/ # product, API, hosting, sync, and provider guides
-├── Dockerfile / docker-compose.yml
-└── pyproject.toml
-```
-
-New capability belongs in the v2 path (`engraphis/core/`, `engraphis/backends/`, and
-`MemoryService`) behind the interfaces in `core/interfaces.py`. Algorithm modules in `core/`
-remain backend-agnostic; `engraphis/factory.py` is the outer composition root used by
-`engraphis.create_memory_engine()` and the compatibility `MemoryEngine.create()` entry point, then
-injects the selected collaborators into `core/engine.py`. The flat-namespace v1 server under
-`engraphis/app.py`, `routes/`, `stores/`, and `engines/` remains a
-compatibility/reference surface; `engraphis-dashboard`, the MCP server, and the Python quickstart
-above use v2.
-
----
-
-## License
-
-Apache-2.0. See [LICENSE](https://github.com/Coding-Dev-Tools/engraphis/blob/main/LICENSE) and [NOTICE](https://github.com/Coding-Dev-Tools/engraphis/blob/main/NOTICE). "Engraphis" is a trademark of the
-Engraphis project; the license does not grant trademark rights. Code already distributed
-under Apache-2.0 keeps that grant; later releases cannot retroactively withdraw it. The
-official hosted control plane, its production credentials and records, managed operations,
-support, and future separately delivered commercial modules are outside the public source
-grant. See [`docs/LICENSING.md`](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/LICENSING.md) for the complete boundary.
+| `ENGRAPHIS_EXTRACTOR` | `none` | `none` = verbatim; `chunk` = offline structure-aware chunks; `llm` = free-form LLM facts; `llm_structured` = schema-validated facts + graph metadata |
+| `ENGRAPHIS_CHUNK_TOKENIZER_MODEL` | Not set | Optional Hugging Face tokenizer used to enforce chunk budgets with the downstream reader's real tokenization; requires the optional `transformers` package |
+| `ENGRAPHIS_CHUNK_TOKENIZER_REVISION` | Not set | Optional immutable tokenizer/model revision recorded in the chunk-counter identity; pin this for reproducible benchmark artifacts |
+| `ENGRAPHIS_GRAPH_EXTRACTOR` | `regex` | `regex` = offline heuristic NER; `none` = disable heuristic text extraction (validated `llm_structured` metadata still feeds the graph) |
+| `ENGRAPHIS_RETENTION_SUPERVISOR` | `none` | `none` = deterministic only; `llm` = sends a bounded excerpt to the configured provider for advisory ephemeral/normal/critical classification |
+| `ENGRAPHIS_ALLOW_AUTOMATIC_CRITICAL_RETENTION` | `false` | Opt in only when an LLM supervisor may automatically assign the long-lived `critical` class; explicit user-selected critical retention is unaffected |
+| `ENGRAPHIS_WHISPER_MODEL` | Not set | Enables local faster-whisper audio/video transcription |
+| `ENGRAPHIS_POSTGRES_DSN` | Not set | CLI-only PostgreSQL source; used for the connection and never stored |
+| `ENGRAPHIS_POSTGRES_CONNECT_TIMEOUT` | `10` | PostgreSQL introspection connection timeout in seconds (bounded to 1–120) |
+| `ENGRAPHIS_POSTGRES_STATEMENT_TIMEOUT_MS` | `30000` | Per-introspection PostgreSQL statement timeout in milliseconds (bounded to 1–300000) |
+| `ENGRAPHIS_GRAPH_TOKEN` | Not set | Bearer token for `engraphis-graph-server`; required off-loopback |
+| `ENGRAPHIS_GRAPH_HOST` / `ENGRAPHIS_GRAPH_PORT` | `127.0.0.1` / `8720` | Read-only graph/recall server bind address |
+| `ENGRAPHIS_LLM_PROVIDER` | `openai` | `openai \| anthropic \| google \| openrouter \| custom` |
+| `ENGRAPHIS_LLM_MODEL` | `gpt-4o-mini` | Model name (provider-specific) |
+| `ENGRAPHIS_LLM_API_KEY` | Not set | API key for chat/synthesis, `llm` / `llm_structured` extraction, and structured consolidation |
+| `ENGRAPHIS_LLM_BASE_URL` | Not set | Base URL for openrouter / custom OpenAI-compatible endpoints |
+| `ENGRAPHIS_LLM_AUTO_EXTRACT` | `0` | Opt in to switching the running engine to `llm_structured` after a successful live connection test; the dashboard's extraction Off button persists `0`, and its On button restores `1` |
+| `ENGRAPHIS_FORWARDED_ALLOW_IPS` | *(none)* | Proxies trusted for forwarded client/TLS headers (`*` only when the service is reachable exclusively through that proxy) |
+| `ENGRAPHIS_LOCAL_TRUSTED_PEERS` | *(none)* | Exact peers/CIDRs treated as local without forwarding headers; use only for trusted Docker/LAN peers, never public deployments |
+| `ENGRAPHIS_UPDATE_CACHE` | `86400` | Update-check cache TTL in seconds, bounded to `1..31622400`; this is never a cache-file path |
+| `ENGRAPHIS_UPDATE_CHECK` | Off | Opt-in release reminder surfaced in the dashboard, server startup log, and MCP. Update checks run only when this is set to an affirmative value; `0` keeps them off. |
+| `ENGRAPHIS_UPDATE_URL` | Not set | Overrides the release-check source URL; the outbound client accepts HTTPS and rejects private/reserved destinations. |
+| `ENGRAPHIS_CLOUD_CONTROL_URL` | hosted default | Official entitlement, organization, and credential control API. A saved rotating credential stays bound to the control endpoint recorded for its family; reconnect to change it. |
+| `ENGRAPHIS_CLOUD_COMPUTE_URL` | hosted default | Official Analytics and managed-automation API. A saved rotating credential stays bound to its recorded compute endpoint; reconnect to change it. |
+| `ENGRAPHIS_CLOUD_ORGANIZATION_ID` | Not set | Hosted organization bound to this customer session |
+| `ENGRAPHIS_CLOUD_REFRESH_CREDENTIAL` | Not set | Bootstrap-only rotating hosted credential; after first use the owner-only cloud session replacement takes precedence |
+| `ENGRAPHIS_CLOUD_TOKEN_SUBJECT` | `member` | Subject fixed during hosted bootstrap (`device` or `member`); set explicitly with an environment-only refresh credential |
+| `ENGRAPHIS_CLOUD_ACCESS_TOKEN` | Not set | Optional short-lived access token for ephemeral jobs |
+| `ENGRAPHIS_MANAGED_COMPUTE_CONSENT` | *(auto)* | Operator override only; default follows whether a cloud session is configured (connected = allowed, local-only = never). `0` opts a connected installation out; `1` permits local snapshot preparation but does not create a cloud credential or authorize an upload |
+
+See `.env.example` for the full variable inventory. Supply those values through the process
+environment or the trusted config file above; copying it to an arbitrary `./.env` does not make
+Engraphis load it.
+
+---
+
+## Project structure
+
+```
+engraphis/
+├── engraphis/
+│ ├── core/ # v2 engine: interfaces, store, recall, scoring, schema, sync
+│ ├── backends/ # pluggable embedder / vector index / reranker / codegraph / sync transports / encryption
+│ ├── factory.py # outer v2 composition root; selects and injects concrete backends
+│ ├── service.py # validated MemoryService facade
+│ ├── mcp_server.py # Smart MCP gateway + 34-tool Classic compatibility server
+│ ├── dashboard_app.py # dashboard WebUI (FastAPI)
+│ ├── dashboard_assets/ # primary Ledger interface + graph engine
+│ ├── classic_assets/ # selectable full operator dashboard backup
+│ ├── read_only_api.py # token-protected recall/repository-graph HTTP surface
+│ ├── hosted_client.py # hosted URLs, plan labels, and endpoint validation only
+│ ├── licensing.py # compatibility facade for hosted presentation metadata
+│ ├── cloud_session.py # rotating hosted customer-session client
+│ ├── cloud_features.py # consented managed-feature protocol client
+│ ├── config.py / app.py # env settings / REST server
+│ └── static/ # compatibility dashboard asset paths
+├── eval/ # offline retrieval eval harness + datasets
+├── tests/ # offline-first pytest suite and release/security contracts
+├── scripts/ # dashboard, server, graph, CLI, connect, update, consolidation, sync
+├── docs/ # product, API, hosting, sync, and provider guides
+├── Dockerfile / docker-compose.yml
+└── pyproject.toml
+```
+
+New capability belongs in the v2 path (`engraphis/core/`, `engraphis/backends/`, and
+`MemoryService`) behind the interfaces in `core/interfaces.py`. Algorithm modules in `core/`
+remain backend-agnostic; `engraphis/factory.py` is the outer composition root used by
+`engraphis.create_memory_engine()` and the compatibility `MemoryEngine.create()` entry point, then
+injects the selected collaborators into `core/engine.py`. The flat-namespace v1 server under
+`engraphis/app.py`, `routes/`, `stores/`, and `engines/` remains a
+compatibility/reference surface; `engraphis-dashboard`, the MCP server, and the Python quickstart
+above use v2.
+
+---
+
+## License
+
+Apache-2.0. See [LICENSE](https://github.com/Coding-Dev-Tools/engraphis/blob/main/LICENSE) and [NOTICE](https://github.com/Coding-Dev-Tools/engraphis/blob/main/NOTICE). "Engraphis" is a trademark of the
+Engraphis project; the license does not grant trademark rights. Code already distributed
+under Apache-2.0 keeps that grant; later releases cannot retroactively withdraw it. The
+official hosted control plane, its production credentials and records, managed operations,
+support, and future separately delivered commercial modules are outside the public source
+grant. See [`docs/LICENSING.md`](https://github.com/Coding-Dev-Tools/engraphis/blob/main/docs/LICENSING.md) for the complete boundary.
diff --git a/conftest.py b/conftest.py
index ea53f236..6f4fdbd0 100644
--- a/conftest.py
+++ b/conftest.py
@@ -12,6 +12,13 @@
# explicitly via monkeypatch (see tests/test_update_check.py).
os.environ.setdefault("ENGRAPHIS_UPDATE_CHECK", "0")
+# Owner machines may configure ENGRAPHIS_EXTRACTOR=llm (via ~/.engraphis/config.env),
+# which would make every ingest-path test block on live LLM extraction calls. The unit
+# suite is offline-inert by contract (AGENTS.md §1 "primary offline gate"); tests that
+# exercise extraction opt back in explicitly via monkeypatch.setenv. setdefault keeps a
+# real shell override working, matching how config.env itself defers to the environment.
+os.environ.setdefault("ENGRAPHIS_EXTRACTOR", "none")
+
# The legacy scripts/test_*.py files are HTTP smoke tests (need a running server +
# httpx), not unit tests. Keep pytest focused on the tests/ suite.
collect_ignore_glob = ["scripts/*"]
diff --git a/docs/ARCHITECTURE_V3.md b/docs/ARCHITECTURE_V3.md
index 912f84f4..8c0c0a9e 100644
--- a/docs/ARCHITECTURE_V3.md
+++ b/docs/ARCHITECTURE_V3.md
@@ -1,13 +1,14 @@
# Engraphis v3 architecture
This document is the design outline for the repo-graph, intent-native memory, resource-ingestion,
-retention-supervision, and privacy-receipt additions introduced with schema version 3.
+retention-supervision, and privacy-receipt additions introduced in the schema-3 era (the
+current schema version is 16).
```mermaid
flowchart LR
Agent["Agent / host LLM"] --> Intent["remember · link · recall_context (compact) · recall"]
CLI["engraphis-graph CLI"] --> Service["MemoryService"]
- MCP["Smart MCP (9 tools) / Classic MCP (34 tools)"] --> Service
+ MCP["Smart MCP (9 tools) / Classic MCP (35 tools)"] --> Service
HTTP["Dashboard + read-only graph HTTP"] --> Service
Import["Local resources / PostgreSQL catalog"] --> Extractors["Optional local extractors"]
Extractors --> Service
diff --git a/docs/DOCUMENT_IMPORT.md b/docs/DOCUMENT_IMPORT.md
index c3450215..6804eddc 100644
--- a/docs/DOCUMENT_IMPORT.md
+++ b/docs/DOCUMENT_IMPORT.md
@@ -99,7 +99,7 @@ collection. Reports never echo secret-like source content.
Default filename exclusions include `.env` variants, credentials, secrets, tokens, recovery
codes, SSH identity files, and `.pem`, `.key`, `.p12`, and `.pfx` material. A collection is
-bounded to 10,000 encountered files and 250 MB of read bytes; an individual adapter input is
+bounded to 10,000 encountered files and 750 MB of read bytes; an individual adapter input is
bounded to 100 MB, while canonical memory text is capped at 100,000 characters and is rejected
rather than silently split. Containers are additionally capped at 2,000 members and 20 MB of
declared decompressed content. Invalid UTF-8/UTF-16 in permitted text is replaced explicitly and
diff --git a/docs/KILO_CODE_INTEGRATION.md b/docs/KILO_CODE_INTEGRATION.md
index 56ab764d..843ebd0b 100644
--- a/docs/KILO_CODE_INTEGRATION.md
+++ b/docs/KILO_CODE_INTEGRATION.md
@@ -226,11 +226,12 @@ class, and the appropriate executor revalidates all of it before running.
preserves the former 34-tool surface below; new Kilo Code installations should keep the zero-config
Smart command shown above.
-### Classic 34-tool inventory
+### Classic 35-tool inventory
| Category | Tool | What it does |
|---|---|---|
| **Write** | `engraphis_remember` | Store a fact; deterministically resolved to add / reinforce (noop) / supersede (invalidate). |
+| Write | `engraphis_remember_many` | Store a fan-out batch of facts in one transaction; within-batch dedup/supersession, plus evidence-labeled edges between siblings sharing a `subject_key` or declared `evidence_source`. |
| Write | `engraphis_record_event` | Append one raw occurrence to an event ledger; event rows are not recalled, deduplicated, or consolidated as memories. |
| Write | `engraphis_link` | Explicitly connect two related memories (e.g. a bug ↔ its fix). |
| Write | `engraphis_ingest` | Store raw/undistilled text; extracts discrete facts first when an LLM extractor is configured. |
diff --git a/docs/MCP_TOOLS.md b/docs/MCP_TOOLS.md
index 8526a0af..2021a43b 100644
--- a/docs/MCP_TOOLS.md
+++ b/docs/MCP_TOOLS.md
@@ -27,7 +27,7 @@ discovery and the validated executors.
No user profile choice or tool switching is required. The dashboard `/mcp` endpoint and
`engraphis-mcp-http` use this Smart surface by default. `engraphis-mcp-classic` (or
-`engraphis-mcp-http --classic`) preserves the 34 direct tools below for integrations that pin
+`engraphis-mcp-http --classic`) preserves the 35 direct tools below for integrations that pin
their historical names and response shapes.
Hosts which already own chat history should use `POST /api/adaptive-context`, not an MCP action.
@@ -78,6 +78,7 @@ the [memory write trust model](WRITE_REVIEW.md) and [recall recovery guide](RECA
| Category | Tool | What it does |
|---|---|---|
| Write | `engraphis_remember` | Stores a fact and resolves it as a new memory, reinforcement, safe supersession, or related memory. |
+| Write | `engraphis_remember_many` | Stores a fan-out batch of facts in one transaction: within-batch dedup/supersession, plus evidence-labeled edges between siblings sharing a `subject_key` or declared `evidence_source`. |
| Write | `engraphis_record_event` | Appends one raw occurrence to the event ledger; event rows are not recalled, deduplicated, reinforced, or consolidated as memories. |
| Write | `engraphis_link` | Connects two related memories. |
| Write | `engraphis_ingest` | Applies the configured extractor (`chunk`, `llm`, or `llm_structured`). With `none`, it stores one verbatim memory. |
@@ -109,7 +110,7 @@ the [memory write trust model](WRITE_REVIEW.md) and [recall recovery guide](RECA
| Governance | `engraphis_promote` | Widens an explicitly approved memory's scope while preserving and linking its narrower history. |
| Session | `engraphis_start_session` / `engraphis_end_session` | Starts or closes a work session. Exact retries are safe; `force_new=true` creates another session. |
| Operations | `engraphis_stats` | Returns memory counts for health checks. |
-| Operations | `engraphis_check_update` | Refreshes the release cache and reports whether a newer version is available. |
+| Operations | `engraphis_check_update` | Refreshes the release cache and reports whether a newer version is available. Update checks are OFF unless `ENGRAPHIS_UPDATE_CHECK` is set to an affirmative value; `=0` keeps them off. |
The classic recall, grounded, and answer tools (`engraphis_recall`,
`engraphis_recall_grounded`, and the `engraphis_answer` alias) accept `planning="off"|"auto"`,
diff --git a/docs/SYNC.md b/docs/SYNC.md
index d6918bfa..5099f44b 100644
--- a/docs/SYNC.md
+++ b/docs/SYNC.md
@@ -110,6 +110,17 @@ has no hosted identity, seat, availability, support, or managed-storage guarante
Folder caps, oversize omissions, and snapshot races are observable incomplete failures rather than
successful partial backups.
+Anyone who can write to the shared folder can also choose the target `workspace_name`, so
+content arriving from a peer you do not control should be treated as untrusted: it is
+quarantined under local `trusted: false` provenance until you review and approve it.
+
+Operator note: the `operation_receipts`, `events`, and `audit` tables in the local SQLite
+database grow append-only by design - rows are hash-chained, and pruning them would break
+chain verification. Watch their size in the database file (for example with
+`sqlite3 engraphis.db "SELECT count(*) FROM operation_receipts"`) when planning capacity;
+the supported path for long-lived installations is archiving or rotating the whole database,
+not deleting rows.
+
## Merge semantics
Sync exchanges bounded workspace snapshots and merges them deterministically. Existing
diff --git a/docs/benchmark-evidence/offline-fixtures-v1.json b/docs/benchmark-evidence/offline-fixtures-v1.json
index 342f3691..f7140efd 100644
--- a/docs/benchmark-evidence/offline-fixtures-v1.json
+++ b/docs/benchmark-evidence/offline-fixtures-v1.json
@@ -9,13 +9,13 @@
"contains_per_record_fingerprints": false
},
"suite": {
- "digest": "4d7e40607319cd4bf8caee3897f1e416dbe5b81998b37a7e4839409ee2923537",
+ "digest": "4bfdfd6ccdf34ff7daa7441b8e788371c31985efe031b6520a265ef11f71ed1b",
"digest_method": "sha256(canonical compact JSON mapping each sorted path to its file SHA-256)",
"files": {
"eval/chunking_eval.py": "a16544353940c0a8c40cea3b9932d3399b35ea5994b809b78f5dbe4a952c467f",
"eval/datasets/codemem.jsonl": "341313023c22850a2e14f02742b571ad1deca824f886a1654a59541304c01f3c",
"eval/datasets/longdoc.jsonl": "7f5ade95e1f283d0db8cf78e53ed8995d3534f847e616d2c0005fd8da37ac790",
- "eval/grounded.py": "a5dd62d10c079b0098917a4640315254c65a4f1d7d71d8c3a669f290a29277e5",
+ "eval/grounded.py": "75ba96a4427508f2718323d283f7889fc90901a6176f12c4b3518f1ede5d96dd",
"eval/performance.py": "e17ea78095e4e592717bc5d9d8e34d55fd98c3fd28d1a8104a20c227d4d619c9"
}
},
diff --git a/docs/benchmark-evidence/offline-fixtures-v1.json.sha256 b/docs/benchmark-evidence/offline-fixtures-v1.json.sha256
index d679044b..ed19362f 100644
--- a/docs/benchmark-evidence/offline-fixtures-v1.json.sha256
+++ b/docs/benchmark-evidence/offline-fixtures-v1.json.sha256
@@ -1 +1 @@
-c3a74f1770ad3f868f55261ba11680e2dadca30167082ac2cb6669f9e3bdfad2 offline-fixtures-v1.json
+0f60b0868444f676fe14c5f94d7db2c475e22669930c4d760881d0842eaa6800 offline-fixtures-v1.json
diff --git a/docs/images/context-efficiency.svg b/docs/images/context-efficiency.svg
index 0e5c1156..0df330b9 100644
--- a/docs/images/context-efficiency.svg
+++ b/docs/images/context-efficiency.svg
@@ -1,6 +1,6 @@
diff --git a/engraphis/backends/codegraph.py b/engraphis/backends/codegraph.py
index c35a48f2..e51333c5 100644
--- a/engraphis/backends/codegraph.py
+++ b/engraphis/backends/codegraph.py
@@ -658,8 +658,51 @@ def index_file(self, file_path: str, content: str, lang: str) -> FileIndex:
src=name, dst=base, relation=relation,
file=file_path, line=lineno,
))
+ self._extract_call_edges(out, lines, lang, file_path)
return out
+ def _extract_call_edges(self, out: FileIndex, lines: list, lang: str,
+ file_path: str) -> None:
+ """Best-effort ``calls`` edges for the flat regex model.
+ The AST backend emits caller→callee edges from real call nodes; without
+ it the code-arm bridge cannot hop a caller to its callee, so retrieval
+ silently degrades to definitions-only on the numpy-only floor. Here any
+ reference to another symbol defined in the same file, inside a detected
+ function's body, becomes a calls edge. Bounded by construction: one pass
+ over already-split lines, callees restricted to indexed symbol names,
+ one edge per (caller, callee) pair. Best-effort by design — strings and
+ comments can fool it, and the AST backend remains authoritative where
+ a grammar exists.
+ """
+ if lang in {"sql", "terraform"}:
+ return
+ func_kinds = {"function", "method"}
+ callers = sorted(
+ ((int(str(s.span).split("-", 1)[0]), s.name)
+ for s in out.symbols if s.kind in func_kinds),
+ )
+ if not callers:
+ return
+ known = {s.name for s in out.symbols}
+ emitted: set = set()
+ call_re = re.compile(r"\b([A-Za-z_]\w*)\s*\(")
+ total = len(lines)
+ for idx, (fn_line, fn_name) in enumerate(callers):
+ body_end = callers[idx + 1][0] - 1 if idx + 1 < len(callers) else total
+ for lineno in range(fn_line + 1, min(body_end, total) + 1):
+ line = lines[lineno - 1]
+ if len(line) > self._MAX_LINE_LEN:
+ continue
+ for m in call_re.finditer(line):
+ callee = m.group(1)
+ pair = (fn_name, callee)
+ if callee in known and callee != fn_name and pair not in emitted:
+ emitted.add(pair)
+ out.edges.append(CodeEdge(
+ src=fn_name, dst=callee, relation="calls",
+ file=file_path, line=lineno,
+ ))
+
class CompositeSymbolIndexer:
"""Route each language to the best backend that supports it: AST (tree-sitter)
diff --git a/engraphis/backends/resources.py b/engraphis/backends/resources.py
index 57cb2a70..d8c4c802 100644
--- a/engraphis/backends/resources.py
+++ b/engraphis/backends/resources.py
@@ -29,7 +29,7 @@
from xml.etree import ElementTree
from engraphis.core.interfaces import ResourceDocument, ResourceExtractor
-from engraphis.core.fsutil import is_reparse_point as _is_reparse_point
+from engraphis.core.fsutil import is_link_indirection as _is_link_indirection
TEXT_EXTENSIONS = {
".txt", ".md", ".markdown", ".rst", ".log", ".json", ".jsonl", ".csv", ".tsv",
@@ -134,7 +134,7 @@ def _read_path_snapshot(source: Path) -> bytes:
if (
not stat.S_ISREG(before.st_mode)
or stat.S_ISLNK(before.st_mode)
- or _is_reparse_point(before)
+ or _is_link_indirection(before)
):
raise ResourceExtractionError("resource path is not a regular file")
flags = (
@@ -148,7 +148,7 @@ def _read_path_snapshot(source: Path) -> bytes:
opened = os.fstat(descriptor)
if (
not stat.S_ISREG(opened.st_mode)
- or _is_reparse_point(opened)
+ or _is_link_indirection(opened)
or _snapshot_identity(opened) != _snapshot_identity(before)
):
raise ResourceExtractionError("resource path changed before it was opened")
diff --git a/engraphis/backends/vector_sqlitevec.py b/engraphis/backends/vector_sqlitevec.py
index 785ba46a..5bfcc58f 100644
--- a/engraphis/backends/vector_sqlitevec.py
+++ b/engraphis/backends/vector_sqlitevec.py
@@ -14,6 +14,7 @@
from __future__ import annotations
import importlib
+import logging
import re
import sys
from numbers import Integral
@@ -24,10 +25,15 @@
from engraphis.backends.embedder_deterministic import MAX_EMBEDDING_DIM
from engraphis.backends.vector_numpy import NumpyVectorIndex
from engraphis.core.interfaces import SearchFilter, VectorIndex
-from engraphis.core.store import Store
+from engraphis.core.store import IN_CLAUSE_CHUNK, Store
+
+logger = logging.getLogger("engraphis")
_INDEX_FORMAT_VERSION = 3
-_VISIBILITY_BATCH_SIZE = 8
+# Visibility checks are plain IN-chunks over canonical ids (identical semantics at
+# any chunk size); match the store's IN_CLAUSE_CHUNK so a widening round costs
+# len(unchecked)/500 round-trips instead of len(unchecked)/8.
+_VISIBILITY_BATCH_SIZE = IN_CLAUSE_CHUNK
_COVERAGE_BATCH_SIZE = 500
_DELETE_BATCH_SIZE = 500
_COVERAGE_RTOL = 1e-6
@@ -549,7 +555,11 @@ def get_vector_index(store: Store, *, dim: int = 384, prefer: str = "auto") -> V
return NumpyVectorIndex(store, dim=dimension)
try:
return SqliteVecVectorIndex(store, dimension)
- except Exception:
+ except Exception as exc:
if prefer == "sqlite-vec":
raise
+ logger.warning(
+ "sqlite-vec vector index unavailable (%s); falling back to NumpyVectorIndex",
+ type(exc).__name__,
+ )
return NumpyVectorIndex(store, dim=dimension)
diff --git a/engraphis/classic_assets/dashboard.js b/engraphis/classic_assets/dashboard.js
index 549110af..b45e0720 100644
--- a/engraphis/classic_assets/dashboard.js
+++ b/engraphis/classic_assets/dashboard.js
@@ -488,11 +488,11 @@ async function wsCreate(){
function wsSwitch(name){setWS(name);toast('Switched to '+name,'ok');navTo('overview')}
/* import (files/folders from this PC — see MemoryService.import_folder/import_files) */
-async function importUpload(items){if(!canCreateWs()){toast('Viewers can’t import','err');return}if(!WS){toast('Create or select a folder first','err');return}if(!items||!items.length)return;const fd=new FormData();fd.append('workspace',WS);fd.append('memory_type','semantic');fd.append('derive_facts',document.getElementById('import-derive').checked?'true':'false');for(const it of items)fd.append('files',it.file,it.name);const el=document.getElementById('import-status');el.textContent='Extracting and importing '+items.length+' file(s)…';try{const r=await api('/workspaces/import-files',{method:'POST',body:fd});const wc=(r.warnings||[]).length;el.textContent=r.imported+' imported, '+r.skipped+' skipped, '+r.errors+' error(s), '+(r.derived_facts||0)+' derived fact(s)'+(wc?', '+wc+' warning(s)':'');toast(r.imported+' resource'+(r.imported===1?'':'s')+' imported into "'+WS+'"','ok');refreshFolders()}catch(e){el.textContent='';toast('Import failed: '+e.message,'err')}}
+async function importUpload(items){if(!canCreateWs()){toast('Viewers can’t import','err');return}if(!WS){toast('Create or select a folder first','err');return}if(!items||!items.length)return;const fd=new FormData();fd.append('workspace',WS);fd.append('memory_type','semantic');fd.append('derive_facts',document.getElementById('import-derive').checked?'true':'false');for(const it of items)fd.append('files',it.file,it.name);const el=document.getElementById('import-status');el.textContent='Extracting and importing '+items.length+' file(s)…';try{const r=await api('/workspaces/import-files',{method:'POST',body:fd});const wc=(r.warnings||[]).length;el.textContent=r.imported+' imported, '+r.skipped+' skipped, '+r.errors+' error(s), '+(r.derived_facts||0)+' derived fact(s)'+(wc?', '+wc+' warning(s)':'')+(r.truncated?', TRUNCATED at '+r.scanned+' of '+r.matched_total+' matching files':'')+(r.unreadable?', '+r.unreadable+' unreadable':'');toast(r.imported+' resource'+(r.imported===1?'':'s')+' imported into "'+WS+'"'+(r.truncated?' (truncated at max '+r.scanned+')':''),'ok');refreshFolders()}catch(e){el.textContent='';toast('Import failed: '+e.message,'err')}}
function importFilesPicked(fileList,el){const items=Array.from(fileList||[]).map(f=>({file:f,name:f.webkitRelativePath||f.name}));if(el)el.value='';importUpload(items)}
async function importWalkEntry(entry,path,out){if(entry.isFile){await new Promise(res=>entry.file(f=>{out.push({file:f,name:(path?path+'/':'')+f.name});res()},()=>res()))}else if(entry.isDirectory){const reader=entry.createReader();const readBatch=()=>new Promise(res=>reader.readEntries(res,()=>res([])));let batch;do{batch=await readBatch();for(const e of batch)await importWalkEntry(e,(path?path+'/':'')+entry.name,out)}while(batch.length)}}
async function importDrop(e){e.preventDefault();e.currentTarget.classList.remove('drag');const items=e.dataTransfer.items;const out=[];if(items&&items.length&&items[0].webkitGetAsEntry){for(const it of items){const entry=it.webkitGetAsEntry&&it.webkitGetAsEntry();if(entry)await importWalkEntry(entry,'',out)}}else{for(const f of e.dataTransfer.files)out.push({file:f,name:f.name})}importUpload(out)}
-async function importFromPath(){if(!canCreateWs()){toast('Viewers can’t import','err');return}if(!WS){toast('Create or select a folder first','err');return}const path=(document.getElementById('import-path').value||'').trim();const pattern=(document.getElementById('import-pattern').value||'*').trim()||'*';if(!path){toast('Enter a path','err');return}const el=document.getElementById('import-status');el.textContent='Extracting and importing…';try{const r=await api('/workspaces/import-folder',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({workspace:WS,path,file_pattern:pattern,memory_type:'semantic',derive_facts:document.getElementById('import-derive').checked})});const wc=(r.warnings||[]).length;el.textContent=r.imported+' imported, '+r.skipped+' skipped, '+r.errors+' error(s), '+(r.derived_facts||0)+' derived fact(s), scanned '+r.scanned+(wc?', '+wc+' warning(s)':'');toast(r.imported+' resource'+(r.imported===1?'':'s')+' imported into "'+WS+'"','ok');refreshFolders()}catch(e){el.textContent='';toast('Import failed: '+e.message,'err')}}
+async function importFromPath(){if(!canCreateWs()){toast('Viewers can’t import','err');return}if(!WS){toast('Create or select a folder first','err');return}const path=(document.getElementById('import-path').value||'').trim();const pattern=(document.getElementById('import-pattern').value||'*').trim()||'*';if(!path){toast('Enter a path','err');return}const el=document.getElementById('import-status');el.textContent='Extracting and importing…';try{const r=await api('/workspaces/import-folder',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({workspace:WS,path,file_pattern:pattern,memory_type:'semantic',derive_facts:document.getElementById('import-derive').checked})});const wc=(r.warnings||[]).length;el.textContent=r.imported+' imported, '+r.skipped+' skipped, '+r.errors+' error(s), '+(r.derived_facts||0)+' derived fact(s), scanned '+r.scanned+(wc?', '+wc+' warning(s)':'')+(r.truncated?', TRUNCATED at '+r.scanned+' of '+r.matched_total+' matching files':'')+(r.unreadable?', '+r.unreadable+' unreadable':'');toast(r.imported+' resource'+(r.imported===1?'':'s')+' imported into "'+WS+'"'+(r.truncated?' (truncated at max '+r.scanned+')':''),'ok');refreshFolders()}catch(e){el.textContent='';toast('Import failed: '+e.message,'err')}}
async function indexRepository(){if(!WS){toast('Select a workspace first','err');return}const repo=(document.getElementById('code-repo').value||'').trim(),root=(document.getElementById('code-root').value||'').trim(),el=document.getElementById('code-import-status');if(!repo||!root){toast('Enter a repository name and path','err');return}el.textContent='Incrementally indexing repository…';try{const r=await api('/code/index',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({workspace:WS,repo:repo,root_path:root})});el.textContent=`${r.files_indexed} changed, ${r.files_unchanged} unchanged · ${r.symbols} symbols · ${r.edges} edges · ${r.code_memory_links||0} memory links`;toast('Repository graph updated','ok')}catch(e){el.textContent='';toast(e.message,'err')}}
async function importPostgresSchema(){if(!WS){toast('Select a workspace first','err');return}const dsn=(document.getElementById('postgres-dsn').value||'').trim(),repo=(document.getElementById('postgres-repo').value||'').trim(),el=document.getElementById('code-import-status');if(!dsn){toast('Enter a PostgreSQL DSN','err');return}el.textContent='Reading PostgreSQL catalog…';try{const r=await api('/resources/postgres',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({workspace:WS,repo:repo||null,dsn:dsn})});document.getElementById('postgres-dsn').value='';el.textContent=`Imported ${r.schema.tables||0} tables, ${r.entities} entities, and ${r.relations} relations`;toast('Database schema imported','ok')}catch(e){el.textContent='';toast(e.message,'err')}}
async function wsRename(name){const nn=await textAction('Rename workspace','Choose a new name for "'+name+'".','Workspace name',name,{submit:'Rename'});if(nn===null)return;const v=nn.trim();if(!v||v===name)return;try{await api('/workspaces/rename',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({workspace:name,new_name:v})});if(WS===name)setWS(v);toast('Renamed','ok');refreshFolders()}catch(e){toast(e.message,'err')}}
@@ -569,7 +569,7 @@ async function exportWorkspace(){try{const d=await api('/export?workspace='+enco
async function loadTeam(){const el=document.getElementById('team-body'),teamCta=hostedCta('team','team_tab');try{const st=await api('/auth/state');if(teamCta.href==='#'&&st&&st.cloud_url)teamCta.href=safeUrl(st.cloud_url)}catch(e){}el.innerHTML=`
Engraphis Team Cloud HOSTED
Organizations, invitations, roles, named seats, scoped device credentials, and team audit run on the private hosted service. This local dashboard is intentionally single-user.
${esc(teamTeaserNote())} Private-service account grace is capped at 24 hours, never extends Team access, and never restricts the free local core.
`}
/* health + settings */
function connectionContext(){const host=(location.hostname||'').toLowerCase();return host==='localhost'||host==='127.0.0.1'||host==='::1'||host.endsWith('.localhost')?'Local engine':'Remote customer node'}
-async function checkHealth(){const label=connectionContext();try{await api('/health');const d=document.getElementById('health-dot'),t=document.getElementById('health-text');if(d){d.classList.add('health-ok');d.classList.remove('health-error')}if(t)t.textContent=label+' connected'}catch(e){const d=document.getElementById('health-dot'),t=document.getElementById('health-text');if(d){d.classList.add('health-error');d.classList.remove('health-ok')}if(t)t.textContent=label+' unavailable'}try{const auth=await api('/auth/state');const m=document.getElementById('deployment-mode-indicator');if(m){const isLocal=auth.deployment_mode==='local';m.textContent=isLocal?'LOCAL':'HOSTED';m.title=isLocal?'Local mode: no hosted cloud configured':'Hosted mode: connected to Engraphis Cloud';m.className='deployment-mode '+(isLocal?'mode-local':'mode-hosted');m.hidden=false}}catch(e){}}
+async function checkHealth(){const label=connectionContext();try{await api('/health');const d=document.getElementById('health-dot'),t=document.getElementById('health-text');if(d){d.classList.add('health-ok');d.classList.remove('health-error')}if(t)t.textContent=label+' connected'}catch(e){const d=document.getElementById('health-dot'),t=document.getElementById('health-text');if(d){d.classList.add('health-error');d.classList.remove('health-ok')}if(t)t.textContent=label+' unavailable'}try{const auth=await api('/auth/state');const m=document.getElementById('deployment-mode-indicator');if(m){const isLocal=auth.deployment_mode==='local';m.textContent=isLocal?'LOCAL':'HOSTED';m.title=isLocal?'Local mode: no hosted cloud configured':'Hosted mode: connected to Engraphis Cloud';m.className='deployment-mode '+(isLocal?'mode-local':'mode-hosted');m.hidden=false}}catch(e){}}
function loadSettings(){loadLicense();loadSyncStatus();loadHostedAgentAccess();loadLlmStatus();const s=document.getElementById('cfg-store');if(s)s.textContent=location.host;api('/info').then(function(d){var v=document.getElementById('cfg-version');if(v&&d&&d.version)v.textContent=d.version}).catch(function(){})}
async function loadLlmStatus(){const el=document.getElementById('llm-body');if(!el)return;try{const st=await api('/llm/status');const ok=st.configured;const badge=ok?'configured':'not configured';const keyLine=st.key_set?'API key set ✓':'No API key set';let modelSel='';let provSel='';el.innerHTML=`
Provider · Model${badge}
${provSel}${modelSel}
${keyLine} · extractor: ${esc(st.extractor)}
Add this to your .env and restart Engraphis:
LLM extraction${st.extractor_enabled?'ON':'OFF'}
While ON, ingested memory content is sent to your LLM provider for schema-validated extraction. OFF disables extraction transfers only; retention supervision is configured separately.
`}catch(e){el.innerHTML='
'+esc(e.message)+'
'}}
@@ -597,7 +597,7 @@ const syncNowBase=syncNow;
syncNow=async function(){if(!await confirmCloudTransfer('Sync shared workspaces','Cloud Sync sends eligible changes from your shared workspaces to Engraphis Cloud and receives authorized changes from your other installations; secret and session-scoped rows stay local.','Sync now',CLOUD_SYNC_PRIVACY_COPY))return;return syncNowBase()}
/* ─── knowledge graph (force-graph + d3-force: compact defaults and selectable layouts) ─── */
-let GRAPH=null, FG=null, GRAPH_ENGINE=null, GRESIZE=false, GRESIZEFRAME=0, GADJ={}, GCOMM_ADJ={}, GCOMPONENTS={}, GCOMPONENT_LAYOUT=null, GHILITE=null, GHOVERSET=null, GLABELRANK={}, GLABELBOXES=[], GDATA_CACHE=null, GACTIVE_DATA=null, GREDRAWFRAME=0, GPERF={large:false,dense:false}, GRAPH_FULL=false, GRAPH_SCOPE_BEFORE_FULL=null, GRAPH_LOAD_REQUEST=0, GRAPH_LOAD_CONTROLLER=null;
+let GRAPH=null, FG=null, GRAPH_ENGINE=null, GRESIZE=false, GRESIZEFRAME=0, GADJ={}, GCOMM_ADJ={}, GCOMPONENTS={}, GCOMPONENT_LAYOUT=null, GHILITE=null, GHOVERSET=null, GLABELRANK={}, GLABELBOXES=[], GDATA_CACHE=null, GACTIVE_DATA=null, GREDRAWFRAME=0, GPERF={large:false,dense:false}, GRAPH_FULL=false, GRAPH_SCOPE_BEFORE_FULL=null, GRAPH_LOAD_REQUEST=0, GRAPH_LOAD_CONTROLLER=null;
const GRAPH_PRESETS={
original:{label:'Original force',repel:120,link:30,gravity:14,font:13,size:3,linkw:1,labelDensity:40,curve:0,particles:0},
compact:{label:'Compact clusters',repel:42,link:20,gravity:26,font:12,size:3,linkw:.7,labelDensity:30,curve:.08,particles:0},
@@ -733,9 +733,9 @@ function graphRenderEngine(data,fit,reheat){
}
showAs(empty,false);GPERF={large:data.nodes.length>600||data.links.length>2400,dense:data.links.length>1500};
const created=!GRAPH_ENGINE;
- if(created){
- GRAPH_ENGINE=EngraphisGraph.create(element,{
- renderMode:fullGraph?'all':'overview',
+ if(created){
+ GRAPH_ENGINE=EngraphisGraph.create(element,{
+ renderMode:fullGraph?'all':'overview',
reducedMotion:prefersReducedMotion,
onNodeClick:node=>{syncGraphExplorerSelection(node.id);graphNodeClick(node.label||node.name||node.id)},
onBackgroundClick:()=>graphSetHighlight(null),
@@ -753,7 +753,7 @@ function graphRenderEngine(data,fit,reheat){
const isolated=document.getElementById('graph-show-iso'),showUnlinked=fullGraph||!!(isolated&&isolated.checked);
GRAPH_ENGINE.apply(engine=>{
engine.setSettings({...window.GSET});
- if(typeof engine.setRenderMode==='function')engine.setRenderMode(fullGraph?'all':'overview');
+ if(typeof engine.setRenderMode==='function')engine.setRenderMode(fullGraph?'all':'overview');
engine.setStyle(typeof GSTYLE!=='undefined'?GSTYLE:'cyber');
engine.setColorBy(typeof GCOLORBY!=='undefined'?GCOLORBY:'community');
engine.setThemeColors(graphThemeTypeColors());
@@ -775,7 +775,7 @@ function graphRenderEngine(data,fit,reheat){
null. Re-apply the parked state here so a renderer created against a hidden pane never
starts a rAF that nothing will stop. */
if(GRAPH_ENGINE_PARKED)GRAPH_ENGINE.pause();
- graphSetSimulationStatus(fullGraph?'All nodes · settled LOD':(window.GSET.frozen?'Layout frozen':'Adaptive layout'),false);
+ graphSetSimulationStatus(fullGraph?'All nodes · settled LOD':(window.GSET.frozen?'Layout frozen':'Adaptive layout'),false);
return true;
}catch(error){
graphEngineFallback(error);
@@ -795,11 +795,11 @@ function graphInvalidateData(){
if(GRAPH_ENGINE){try{GRAPH_ENGINE.destroy()}catch(e){}GRAPH_ENGINE=null}
GDATA_CACHE=null;GACTIVE_DATA=null;GCOMPONENT_LAYOUT=null;GHILITE=null;GHOVERSET=null
}
-async function loadLegacyGraph(){
- const request=++GRAPH_LOAD_REQUEST,targetFull=GRAPH_FULL;
- const previousController=GRAPH_LOAD_CONTROLLER,controller=new AbortController();GRAPH_LOAD_CONTROLLER=controller;
- if(previousController&&!previousController.signal.aborted)previousController.abort();
- graphInjectCss();graphInvalidateData();GRAPH=null;
+async function loadLegacyGraph(){
+ const request=++GRAPH_LOAD_REQUEST,targetFull=GRAPH_FULL;
+ const previousController=GRAPH_LOAD_CONTROLLER,controller=new AbortController();GRAPH_LOAD_CONTROLLER=controller;
+ if(previousController&&!previousController.signal.aborted)previousController.abort();
+ graphInjectCss();graphInvalidateData();GRAPH=null;
const empty=document.getElementById('graph-empty'),net=document.getElementById('graph-net'),nodesBox=document.getElementById('graph-entity-list'),edgesBox=document.getElementById('graph-relation-list');
showAs(empty,true,'flex');empty.textContent='Loading graph…';graphSetLayoutStatus('Loading data',true);
if(net)net.setAttribute('aria-busy','true');
@@ -811,32 +811,32 @@ async function loadLegacyGraph(){
GRESIZEFRAME=requestAnimationFrame(()=>{GRESIZEFRAME=0;const element=document.getElementById('graph-net');if(GRAPH_ENGINE)GRAPH_ENGINE.resize();else if(FG&&element)FG.width(element.clientWidth).height(element.clientHeight)});
});
}
- const layerInputs=Array.from(document.querySelectorAll('#graph-layer-filters input')),selectedLayers=layerInputs.filter(input=>input.checked).map(input=>input.value),layerFilter=selectedLayers.length===layerInputs.length?'':'&layers='+encodeURIComponent(selectedLayers.join(',')),includeCode=document.getElementById('graph-include-code').checked,repo=(document.getElementById('graph-repo-filter').value||'').trim(),showUnlinked=targetFull||!!document.getElementById('graph-show-iso').checked;
- try{
- const query=encodeURIComponent(WS||'')+(repo?'&repo='+encodeURIComponent(repo):'')+layerFilter;
- let nextGraph;
- if(targetFull){
- /* The complete scene and its dedicated renderer are independent requests. Starting them
- together avoids adding an asset round-trip after a potentially large scene response, and
- awaiting both guarantees that complete data can never fall into the legacy ForceGraph. */
- const [response]=await Promise.all([
- api('/graph/scene?workspace='+query+'&level=complete&presentation=all&include_memory_nodes=false',{signal:controller.signal}),
- loadGraphEngine(true)
- ]);
- const scene=response.scene||response;
- nextGraph={nodes:(scene.nodes||[]).map(node=>({...node,id:node.id,label:node.label||node.name||node.id,degree:node.degree??node.weighted_degree??0,etype:node.etype||'entity'})),edges:(scene.edges||[]).map(edge=>({...edge,from:edge.from??edge.source??edge.src,to:edge.to??edge.target??edge.dst,label:edge.label||edge.relation||'related',layer:edge.layer||'semantic'})),meta:scene.meta||{}};
- }else{
- nextGraph=await api('/graph?workspace='+query+'&include_code='+(includeCode?'true':'false')+'&limit=1000&node_limit=1000&edge_limit=2000'+(showUnlinked?'':'&connected_only=true'),{signal:controller.signal});
- }
- if(request!==GRAPH_LOAD_REQUEST||targetFull!==GRAPH_FULL)return;
- GRAPH=nextGraph;
- renderGraphSide();graphRender();
- }catch(error){
- if(request!==GRAPH_LOAD_REQUEST||error.name==='AbortError')return;
- showAs(empty,true,'flex');empty.textContent='Graph failed: '+error.message;graphSetLayoutStatus('Load failed',false);
- }finally{
- if(request!==GRAPH_LOAD_REQUEST)return;
- if(GRAPH_LOAD_CONTROLLER===controller)GRAPH_LOAD_CONTROLLER=null;
+ const layerInputs=Array.from(document.querySelectorAll('#graph-layer-filters input')),selectedLayers=layerInputs.filter(input=>input.checked).map(input=>input.value),layerFilter=selectedLayers.length===layerInputs.length?'':'&layers='+encodeURIComponent(selectedLayers.join(',')),includeCode=document.getElementById('graph-include-code').checked,repo=(document.getElementById('graph-repo-filter').value||'').trim(),showUnlinked=targetFull||!!document.getElementById('graph-show-iso').checked;
+ try{
+ const query=encodeURIComponent(WS||'')+(repo?'&repo='+encodeURIComponent(repo):'')+layerFilter;
+ let nextGraph;
+ if(targetFull){
+ /* The complete scene and its dedicated renderer are independent requests. Starting them
+ together avoids adding an asset round-trip after a potentially large scene response, and
+ awaiting both guarantees that complete data can never fall into the legacy ForceGraph. */
+ const [response]=await Promise.all([
+ api('/graph/scene?workspace='+query+'&level=complete&presentation=all&include_memory_nodes=false',{signal:controller.signal}),
+ loadGraphEngine(true)
+ ]);
+ const scene=response.scene||response;
+ nextGraph={nodes:(scene.nodes||[]).map(node=>({...node,id:node.id,label:node.label||node.name||node.id,degree:node.degree??node.weighted_degree??0,etype:node.etype||'entity'})),edges:(scene.edges||[]).map(edge=>({...edge,from:edge.from??edge.source??edge.src,to:edge.to??edge.target??edge.dst,label:edge.label||edge.relation||'related',layer:edge.layer||'semantic'})),meta:scene.meta||{}};
+ }else{
+ nextGraph=await api('/graph?workspace='+query+'&include_code='+(includeCode?'true':'false')+'&limit=1000&node_limit=1000&edge_limit=2000'+(showUnlinked?'':'&connected_only=true'),{signal:controller.signal});
+ }
+ if(request!==GRAPH_LOAD_REQUEST||targetFull!==GRAPH_FULL)return;
+ GRAPH=nextGraph;
+ renderGraphSide();graphRender();
+ }catch(error){
+ if(request!==GRAPH_LOAD_REQUEST||error.name==='AbortError')return;
+ showAs(empty,true,'flex');empty.textContent='Graph failed: '+error.message;graphSetLayoutStatus('Load failed',false);
+ }finally{
+ if(request!==GRAPH_LOAD_REQUEST)return;
+ if(GRAPH_LOAD_CONTROLLER===controller)GRAPH_LOAD_CONTROLLER=null;
if(net)net.setAttribute('aria-busy','false');
if(!GRAPH){
if(FG)FG.graphData({nodes:[],links:[]});
@@ -846,27 +846,27 @@ async function loadLegacyGraph(){
}
}
}
-function graphUpdateAllNodesControl(){
- const full=GRAPH_FULL,button=document.getElementById('graph-show-all'),isolated=document.getElementById('graph-show-iso'),includeCode=document.getElementById('graph-include-code');
- if(button){button.textContent=full?'High quality':'Show all nodes';button.setAttribute('aria-pressed',String(full));button.title=full?'Return to the high-quality graph view':'Load every node, including unconnected entities, for this graph view'}
- if(isolated){isolated.disabled=full;isolated.title=full?'All nodes are already visible.':'Show entities that have no relations (unlinked nodes). Hidden by default to keep the graph readable.'}
- if(includeCode){includeCode.disabled=full;includeCode.title=full?'Code overlay is available in High quality mode.':''}
-}
+function graphUpdateAllNodesControl(){
+ const full=GRAPH_FULL,button=document.getElementById('graph-show-all'),isolated=document.getElementById('graph-show-iso'),includeCode=document.getElementById('graph-include-code');
+ if(button){button.textContent=full?'High quality':'Show all nodes';button.setAttribute('aria-pressed',String(full));button.title=full?'Return to the high-quality graph view':'Load every node, including unconnected entities, for this graph view'}
+ if(isolated){isolated.disabled=full;isolated.title=full?'All nodes are already visible.':'Show entities that have no relations (unlinked nodes). Hidden by default to keep the graph readable.'}
+ if(includeCode){includeCode.disabled=full;includeCode.title=full?'Code overlay is available in High quality mode.':''}
+}
function graphToggleAllNodes(){
const isolated=document.getElementById('graph-show-iso');
if(!GRAPH_FULL){GRAPH_SCOPE_BEFORE_FULL={showUnlinked:!!(isolated&&isolated.checked)};GRAPH_FULL=true;if(isolated)isolated.checked=true}
else{GRAPH_FULL=false;if(isolated&&GRAPH_SCOPE_BEFORE_FULL)isolated.checked=GRAPH_SCOPE_BEFORE_FULL.showUnlinked;GRAPH_SCOPE_BEFORE_FULL=null}
graphUpdateAllNodesControl();loadLegacyGraph();
}
-function graphData(){
- const _si=document.getElementById('graph-show-iso');const hideIso=!(_si&&_si.checked);
- if(GDATA_CACHE&&GDATA_CACHE.graph===GRAPH&&GDATA_CACHE.hideIso===hideIso)return GDATA_CACHE.data;
- if(GRAPH_FULL){
- /* The flat all-node worker accepts the scene's node and from/to edge shapes directly.
- Avoid cloning and decorating up to 20k nodes and 200k relations for quality-only paint. */
- const data={nodes:GRAPH.nodes||[],links:GRAPH.edges||[]};GDATA_CACHE={graph:GRAPH,hideIso,data};return data;
- }
- let sourceNodes=GRAPH.nodes;if(hideIso)sourceNodes=sourceNodes.filter(node=>node.degree>0);
+function graphData(){
+ const _si=document.getElementById('graph-show-iso');const hideIso=!(_si&&_si.checked);
+ if(GDATA_CACHE&&GDATA_CACHE.graph===GRAPH&&GDATA_CACHE.hideIso===hideIso)return GDATA_CACHE.data;
+ if(GRAPH_FULL){
+ /* The flat all-node worker accepts the scene's node and from/to edge shapes directly.
+ Avoid cloning and decorating up to 20k nodes and 200k relations for quality-only paint. */
+ const data={nodes:GRAPH.nodes||[],links:GRAPH.edges||[]};GDATA_CACHE={graph:GRAPH,hideIso,data};return data;
+ }
+ let sourceNodes=GRAPH.nodes;if(hideIso)sourceNodes=sourceNodes.filter(node=>node.degree>0);
const names=new Set(sourceNodes.map(node=>node.id));
const nodes=sourceNodes.map(node=>({id:node.id,label:node.label||node.id,displayLabel:(node.label||node.id).length>30?(node.label||node.id).slice(0,29)+'…':(node.label||node.id),etype:node.etype,degree:node.degree||0,val:1+(node.degree||0)}));
const maxDegree=Math.max(1,...nodes.map(node=>node.degree||0));
@@ -1222,53 +1222,53 @@ function loadForceGraph(){
});
return FORCE_GRAPH_LOADING;
}
-let GRAPH_ENGINE_LOADING=null,ALL_GRAPH_ENGINE_LOADING=null;
-function loadAllGraphEngine(){
- if(typeof EngraphisAllGraph!=='undefined')return Promise.resolve();
- if(!ALL_GRAPH_ENGINE_LOADING){
- ALL_GRAPH_ENGINE_LOADING=new Promise((resolve,reject)=>{
- const script=document.createElement('script');script.src='/v2-assets/engraphis-graph-all.js?v=20260814-all-controls-2';
- script.onload=()=>{typeof EngraphisAllGraph==='undefined'?reject(new Error('All-node graph asset loaded without registering EngraphisAllGraph')):resolve()};
- script.onerror=()=>reject(new Error('All-node graph asset could not load'));
- document.head.appendChild(script);
- });
- ALL_GRAPH_ENGINE_LOADING.catch(()=>{});
- }
- return ALL_GRAPH_ENGINE_LOADING;
-}
-function loadGraphEngine(loadAll=false){
- let engineReady;
- if(typeof EngraphisGraph!=='undefined')engineReady=Promise.resolve();
- else{
- if(!GRAPH_ENGINE_LOADING){
- GRAPH_ENGINE_LOADING=new Promise((resolve,reject)=>{
- const script=document.createElement('script');
- script.src='/v2-assets/engraphis-graph.js?v=20260814-galaxy-gravity-3';
- /* A 200 that never registers the global is a corrupt/truncated asset, not a success —
- resolving there would hand graphRenderEngine() an undefined EngraphisGraph. */
- script.onload=()=>{typeof EngraphisGraph==='undefined'?reject(new Error('Graph engine asset loaded without registering EngraphisGraph')):resolve()};
- script.onerror=()=>reject(new Error('Graph engine could not load'));
- document.head.appendChild(script);
- });
- GRAPH_ENGINE_LOADING.catch(()=>{});
- }
- engineReady=GRAPH_ENGINE_LOADING;
- }
- /* Mark the memoized promise handled. graphRender() can start this fetch on a pass that
- returns before attaching its own handler, and an unhandled rejection would print the exact
- console error this lazy-loading exists to remove. Callers still receive the rejection. */
- return loadAll?engineReady.then(()=>loadAllGraphEngine()):engineReady;
-}
+let GRAPH_ENGINE_LOADING=null,ALL_GRAPH_ENGINE_LOADING=null;
+function loadAllGraphEngine(){
+ if(typeof EngraphisAllGraph!=='undefined')return Promise.resolve();
+ if(!ALL_GRAPH_ENGINE_LOADING){
+ ALL_GRAPH_ENGINE_LOADING=new Promise((resolve,reject)=>{
+ const script=document.createElement('script');script.src='/v2-assets/engraphis-graph-all.js?v=20260814-all-controls-2';
+ script.onload=()=>{typeof EngraphisAllGraph==='undefined'?reject(new Error('All-node graph asset loaded without registering EngraphisAllGraph')):resolve()};
+ script.onerror=()=>reject(new Error('All-node graph asset could not load'));
+ document.head.appendChild(script);
+ });
+ ALL_GRAPH_ENGINE_LOADING.catch(()=>{});
+ }
+ return ALL_GRAPH_ENGINE_LOADING;
+}
+function loadGraphEngine(loadAll=false){
+ let engineReady;
+ if(typeof EngraphisGraph!=='undefined')engineReady=Promise.resolve();
+ else{
+ if(!GRAPH_ENGINE_LOADING){
+ GRAPH_ENGINE_LOADING=new Promise((resolve,reject)=>{
+ const script=document.createElement('script');
+ script.src='/v2-assets/engraphis-graph.js?v=20260814-galaxy-gravity-3';
+ /* A 200 that never registers the global is a corrupt/truncated asset, not a success —
+ resolving there would hand graphRenderEngine() an undefined EngraphisGraph. */
+ script.onload=()=>{typeof EngraphisGraph==='undefined'?reject(new Error('Graph engine asset loaded without registering EngraphisGraph')):resolve()};
+ script.onerror=()=>reject(new Error('Graph engine could not load'));
+ document.head.appendChild(script);
+ });
+ GRAPH_ENGINE_LOADING.catch(()=>{});
+ }
+ engineReady=GRAPH_ENGINE_LOADING;
+ }
+ /* Mark the memoized promise handled. graphRender() can start this fetch on a pass that
+ returns before attaching its own handler, and an unhandled rejection would print the exact
+ console error this lazy-loading exists to remove. Callers still receive the rejection. */
+ return loadAll?engineReady.then(()=>loadAllGraphEngine()):engineReady;
+}
function graphRender(fit=true,reheat=true){
const empty=document.getElementById('graph-empty');
const graphFull=typeof GRAPH_FULL!=='undefined'&&GRAPH_FULL;
/* Kick the opt-in engine off alongside the vendor bundle instead of after it, so a
`?graph-engine=next` deep link costs one round trip rather than two. */
- const engineMissing=typeof EngraphisGraph==='undefined'||(graphFull&&typeof EngraphisAllGraph==='undefined');
- /* All mode owns a dedicated bounded renderer and must remain available after a quality-renderer
- runtime failure. The quality failure latch only authorizes the small legacy overview. */
- const enginePending=(graphFull||(!GRAPH_ENGINE_FAILED&&graphEngineEnabled()))&&engineMissing?loadGraphEngine(graphFull):null;
- if(!graphFull&&typeof ForceGraph==='undefined'){
+ const engineMissing=typeof EngraphisGraph==='undefined'||(graphFull&&typeof EngraphisAllGraph==='undefined');
+ /* All mode owns a dedicated bounded renderer and must remain available after a quality-renderer
+ runtime failure. The quality failure latch only authorizes the small legacy overview. */
+ const enginePending=(graphFull||(!GRAPH_ENGINE_FAILED&&graphEngineEnabled()))&&engineMissing?loadGraphEngine(graphFull):null;
+ if(!graphFull&&typeof ForceGraph==='undefined'){
showAs(empty,true,'flex');empty.textContent='Loading graph engine…';
graphSetLayoutStatus('Loading engine',true);
loadForceGraph().then(()=>graphRender(fit,reheat)).catch(error=>{
@@ -1284,14 +1284,14 @@ function graphRender(fit=true,reheat=true){
someone who explicitly asked for next. Only a real load failure degrades, and it is
announced through graphEngineFallback() rather than silent. */
showAs(empty,true,'flex');empty.textContent='Loading graph engine…';
- graphSetLayoutStatus('Loading engine',true);
- enginePending.then(()=>graphRender(fit,reheat)).catch(error=>{
- if(graphFull){
- empty.textContent=error.message+'; return to High quality or reload the dashboard assets.';
- graphSetLayoutStatus('All-node engine unavailable',false);
- return;
- }
- /* Latches GRAPH_ENGINE_FAILED, so the re-entry below takes the classic path and this
+ graphSetLayoutStatus('Loading engine',true);
+ enginePending.then(()=>graphRender(fit,reheat)).catch(error=>{
+ if(graphFull){
+ empty.textContent=error.message+'; return to High quality or reload the dashboard assets.';
+ graphSetLayoutStatus('All-node engine unavailable',false);
+ return;
+ }
+ /* Latches GRAPH_ENGINE_FAILED, so the re-entry below takes the classic path and this
cannot loop. */
graphEngineFallback(error);
graphRender(fit,reheat);
@@ -1299,14 +1299,14 @@ function graphRender(fit=true,reheat=true){
return;
}
const element=document.getElementById('graph-net'),settings=window.GSET,mode=GRAPH_PRESETS[settings.mode]||GRAPH_PRESETS.compact,data=graphData();
- if(graphFull){
- if(graphRenderEngine(data,fit,reheat))return;
- showAs(empty,true,'flex');
- empty.textContent='All-node renderer unavailable; return to High quality or reload the dashboard assets.';
- graphSetLayoutStatus('All-node engine unavailable',false);
- return;
- }
- if(graphEngineEnabled()&&graphRenderEngine(data,fit,reheat))return;
+ if(graphFull){
+ if(graphRenderEngine(data,fit,reheat))return;
+ showAs(empty,true,'flex');
+ empty.textContent='All-node renderer unavailable; return to High quality or reload the dashboard assets.';
+ graphSetLayoutStatus('All-node engine unavailable',false);
+ return;
+ }
+ if(graphEngineEnabled()&&graphRenderEngine(data,fit,reheat))return;
/* Read AFTER the opt-in attempt: a failing engine resets GACTIVE_DATA precisely so the
classic renderer below rebuilds from scratch instead of assuming the canvas is current. */
const dataChanged=GACTIVE_DATA!==data;
@@ -1510,14 +1510,14 @@ function graphSearch(){
function closeEntityMems(){document.getElementById('mm-overlay').classList.remove('show')}
async function graphNodeClick(name){const ov=document.getElementById('mm-overlay');ov.classList.add('show');document.getElementById('mm-title').textContent=name;document.getElementById('mm-meta').innerHTML='entity';document.getElementById('mm-body').innerHTML='';document.getElementById('mm-actions').innerHTML='';try{const d=await api('/memories?q='+encodeURIComponent(name)+'&workspace='+encodeURIComponent(WS||'')+'&limit=12');document.getElementById('mm-body').innerHTML=d.memories.length?('
';
const topCount=document.getElementById('graph-top-count');if(topCount)topCount.textContent=top.length===((graph.top||[]).length)?String(top.length):(top.length+' of '+(graph.top||[]).length);
@@ -1536,22 +1536,22 @@ function graphKeyboard(event){
const node=nodes[GKEYINDEX],net=document.getElementById('graph-net');graphFocus(node.id);net.setAttribute('aria-label','Selected entity '+(node.label||node.id)+', '+(node.degree||0)+' relations. Press Enter to open. Use arrow keys to move.');
}
function syncGraphExplorerSelection(id){document.querySelectorAll('#graph-entity-list [data-entity]').forEach(button=>{const active=button.dataset.entity===id;button.classList.toggle('active',active);if(active)button.setAttribute('aria-current','true');else button.removeAttribute('aria-current')})}
-function graphQueueExplorer(query){clearTimeout(GEXPLORER_TIMER);GEXPLORER_TIMER=setTimeout(()=>renderGraphExplorer(query,true),GRAPH_FULL?280:120)}
+function graphQueueExplorer(query){clearTimeout(GEXPLORER_TIMER);GEXPLORER_TIMER=setTimeout(()=>renderGraphExplorer(query,true),GRAPH_FULL?280:120)}
function graphExplorerMore(kind){
if(kind==='nodes')GEXPLORER.nodeLimit+=GRAPH_EXPLORER_PAGE.nodes;else GEXPLORER.edgeLimit+=GRAPH_EXPLORER_PAGE.edges;
renderGraphExplorer(GEXPLORER.query,false);
}
-function renderGraphExplorer(query,reset=false){
+function renderGraphExplorer(query,reset=false){
const nodesBox=document.getElementById('graph-entity-list'),edgesBox=document.getElementById('graph-relation-list');if(!nodesBox||!edgesBox)return;
if(!GRAPH){nodesBox.innerHTML='
Graph data is loading.
';edgesBox.innerHTML='
Graph data is loading.
';return}
- const normalized=(query||'').trim().toLowerCase();
- if(reset||GEXPLORER.graph!==GRAPH||GEXPLORER.query!==normalized){
- const nodes=GKEYNODES,edges=GRAPH.edges||[];
- const shownNodes=normalized?nodes.filter(node=>(GGRAPHSEARCHNAMES.get(node.id)||'').includes(normalized)||String(node.etype||'').toLowerCase().includes(normalized)):nodes;
- const shownEdges=normalized?edges.filter(edge=>(GGRAPHSEARCHNAMES.get(edge.from)||'').includes(normalized)||(GGRAPHSEARCHNAMES.get(edge.to)||'').includes(normalized)||String(edge.label||'').toLowerCase().includes(normalized)||String(edge.layer||'').toLowerCase().includes(normalized)):edges;
- GEXPLORER={graph:GRAPH,query:normalized,nodeLimit:GRAPH_EXPLORER_PAGE.nodes,edgeLimit:GRAPH_EXPLORER_PAGE.edges,nodes:shownNodes,edges:shownEdges};
- }
- const shownNodes=GEXPLORER.nodes,shownEdges=GEXPLORER.edges;
+ const normalized=(query||'').trim().toLowerCase();
+ if(reset||GEXPLORER.graph!==GRAPH||GEXPLORER.query!==normalized){
+ const nodes=GKEYNODES,edges=GRAPH.edges||[];
+ const shownNodes=normalized?nodes.filter(node=>(GGRAPHSEARCHNAMES.get(node.id)||'').includes(normalized)||String(node.etype||'').toLowerCase().includes(normalized)):nodes;
+ const shownEdges=normalized?edges.filter(edge=>(GGRAPHSEARCHNAMES.get(edge.from)||'').includes(normalized)||(GGRAPHSEARCHNAMES.get(edge.to)||'').includes(normalized)||String(edge.label||'').toLowerCase().includes(normalized)||String(edge.layer||'').toLowerCase().includes(normalized)):edges;
+ GEXPLORER={graph:GRAPH,query:normalized,nodeLimit:GRAPH_EXPLORER_PAGE.nodes,edgeLimit:GRAPH_EXPLORER_PAGE.edges,nodes:shownNodes,edges:shownEdges};
+ }
+ const shownNodes=GEXPLORER.nodes,shownEdges=GEXPLORER.edges;
const nodePage=shownNodes.slice(0,GEXPLORER.nodeLimit),edgePage=shownEdges.slice(0,GEXPLORER.edgeLimit);
document.getElementById('graph-explorer-node-count').textContent=nodePage.length+' of '+shownNodes.length;
document.getElementById('graph-explorer-edge-count').textContent=edgePage.length+' of '+shownEdges.length;
@@ -1822,4 +1822,4 @@ h143:function(event){graphExplorerMore('nodes')},
h144:function(event){graphExplorerMore('edges')},
h145:function(event){boot()},
});
-for(const type of ['click','keydown','input','change','dragover','dragleave','drop','dragstart','dragend']){document.addEventListener(type,function(event){const target=event.target instanceof Element?event.target.closest('[data-on'+type+']'):null;if(!target||!document.documentElement.contains(target))return;const handler=CSP_EVENT_HANDLERS[target.getAttribute('data-on'+type)];if(!handler)return;const result=handler.call(target,event);if(result===false){event.preventDefault();event.stopPropagation()}},false)}
+for(const type of ['click','keydown','input','change','dragover','dragleave','drop','dragstart','dragend']){document.addEventListener(type,function(event){const target=event.target instanceof Element?event.target.closest('[data-on'+type+']'):null;if(!target||!document.documentElement.contains(target))return;const handler=CSP_EVENT_HANDLERS[target.getAttribute('data-on'+type)];if(!handler)return;const result=handler.call(target,event);if(result===false){event.preventDefault();event.stopPropagation()}},false)}
diff --git a/engraphis/core/consolidate.py b/engraphis/core/consolidate.py
index 2d240be2..73a1015c 100644
--- a/engraphis/core/consolidate.py
+++ b/engraphis/core/consolidate.py
@@ -1155,6 +1155,17 @@ def consolidate(engine, *, workspace_id: str, repo_id: Optional[str] = None,
r = scoring.retention(m.stability, m.last_access, now)
if r >= archive_below:
continue
+ # A coarse host clock (~15.6 ms ticks on Windows) can tie the sweep's
+ # ``now`` to a memory's ingest instant. Closing [t, t) there would be
+ # invisible to every as_of read, and fabricating width (valid_from +
+ # 1us) would leave the row live-visible until the next tick sample —
+ # both wrong. Defer instead: leave the memory live for this sweep and
+ # let a strictly later sweep close it with ordinary half-open
+ # semantics. Production sweeps are minutes apart, so deferral is
+ # unobservable there; only same-tick test fixtures can hit it.
+ if m.valid_from is not None and now <= m.valid_from:
+ report["archive_deferred"] = report.get("archive_deferred", 0) + 1
+ continue
archived_tokens += _mem_tokens(m)
report["archived"].append({"id": m.id, "retention": round(r, 4),
"tokens_freed": _mem_tokens(m)})
@@ -1349,6 +1360,11 @@ def _inherit_safety(engine, memory_id: str, sources: list[MemoryRecord]) -> tupl
metadata["provenance"] = provenance
try:
engine.store.advance_memory_modified_hlc(memory_id, commit=False)
+ engine.store.audit(
+ "consolidation", "safety_inherit", memory_id,
+ f"sensitivity={sensitivity}; trusted={trusted}",
+ commit=False,
+ )
engine.store.conn.execute(
"UPDATE memories SET sensitivity=?, metadata=?, provenance=? WHERE id=?",
(sensitivity,
diff --git a/engraphis/core/documents.py b/engraphis/core/documents.py
index 8b947322..8a72d6d1 100644
--- a/engraphis/core/documents.py
+++ b/engraphis/core/documents.py
@@ -32,7 +32,7 @@
from engraphis.core.obsidian import parse_obsidian_note
from engraphis.core.secrets import secret_kind
-from engraphis.core.fsutil import is_reparse_point as _is_reparse_point
+from engraphis.core.fsutil import is_link_indirection as _is_link_indirection
IMPORTER_VERSION = "1"
@@ -40,7 +40,9 @@
MAX_DOCUMENT_CHARS = 100_000
MAX_DOCUMENT_WARNINGS = 100
MAX_DOCUMENT_FILES = 10_000
-MAX_DOCUMENT_TREE_BYTES = 250_000_000
+# Lockstep with service.MAX_IMPORT_TOTAL_BYTES (750 MB): the wizard scanner must never
+# silently undercut the upload transport ceiling.
+MAX_DOCUMENT_TREE_BYTES = 750_000_000
MAX_CONTAINER_MEMBERS = 2_000
MAX_CONTAINER_XML_BYTES = 20_000_000
MAX_XML_ATTRIBUTE_METADATA_CHARS = 8_000
@@ -382,7 +384,7 @@ def scan_document_tree(
selected = Path(root_path)
try:
selected_info = os.lstat(selected)
- if selected.is_symlink() or _is_reparse_point(selected_info):
+ if selected.is_symlink() or _is_link_indirection(selected_info):
raise DocumentParseError("source root cannot be a symlink")
root = selected.resolve(strict=True)
except OSError as exc:
@@ -1663,7 +1665,7 @@ def _safe_reason(exc: BaseException) -> str:
def _read_tree_file(root: Path, path: Path) -> Tuple[bytes, int]:
before = os.lstat(path)
- if not stat.S_ISREG(before.st_mode) or stat.S_ISLNK(before.st_mode) or _is_reparse_point(before):
+ if not stat.S_ISREG(before.st_mode) or stat.S_ISLNK(before.st_mode) or _is_link_indirection(before):
raise DocumentParseError("unsafe file type")
if not _is_within(root, path.resolve(strict=True)):
raise DocumentParseError("path escapes source root")
@@ -1671,7 +1673,7 @@ def _read_tree_file(root: Path, path: Path) -> Tuple[bytes, int]:
fd = os.open(path, flags)
try:
opened = os.fstat(fd)
- if not stat.S_ISREG(opened.st_mode) or _is_reparse_point(opened) or not _same_identity(before, opened):
+ if not stat.S_ISREG(opened.st_mode) or _is_link_indirection(opened) or not _same_identity(before, opened):
raise DocumentParseError("file changed during scan")
if opened.st_size > MAX_DOCUMENT_BYTES:
raise DocumentParseError("document exceeds 100000000 byte safety limit")
@@ -1688,7 +1690,7 @@ def _read_tree_file(root: Path, path: Path) -> Tuple[bytes, int]:
finished, after = os.fstat(fd), os.lstat(path)
if (not _same_identity(opened, finished) or opened.st_size != finished.st_size
or opened.st_mtime_ns != finished.st_mtime_ns or stat.S_ISLNK(after.st_mode)
- or _is_reparse_point(after)
+ or _is_link_indirection(after)
or not _same_identity(finished, after) or not _is_within(root, path.resolve(strict=True))):
raise DocumentParseError("file changed during scan")
return b"".join(chunks), int(finished.st_mtime_ns)
@@ -1725,7 +1727,7 @@ def _walk_tree(root: Path, directory: Path) -> Iterable[Tuple[Path, Optional[str
try:
relative = entry.relative_to(root)
info = entry.lstat()
- if entry.is_symlink() or _is_reparse_point(info):
+ if entry.is_symlink() or _is_link_indirection(info):
yield entry, "symlink skipped"
elif not _is_within(root, entry.resolve()):
yield entry, "path escapes source root"
diff --git a/engraphis/core/engine.py b/engraphis/core/engine.py
index 2c0e71bf..e689c44c 100644
--- a/engraphis/core/engine.py
+++ b/engraphis/core/engine.py
@@ -31,6 +31,7 @@
from engraphis.core.interfaces import (
MemoryRecord,
MemoryType,
+ FactSpec,
GraphTraversalPolicy,
QueryPlanner,
RetentionDecision,
@@ -130,6 +131,11 @@ def configure_engine_factory(factory: Callable) -> None:
# Bounded so hub memories don't accrete unbounded link lists (link quality > quantity).
EVOLVE_MAX_LINKS = 3
+# Batch writes (remember_many): maximum facts accepted per call. Matches the sync
+# APPLY_BATCH ceiling; callers with more facts must chunk. Bounds the pairwise
+# evidence-edge scan in _evolve_batch.
+MAX_FACTS_PER_BATCH = 500
+
# The deterministic detector's contradiction/obsolete reports below this severity are
# too weak to justify a durable ``conflicts_with`` relation. The detector floors its
# own reports at 0.74 (numeric) / 0.78 (polarity) / 0.82 (assertion), so this only
@@ -613,6 +619,9 @@ def _rebuild_versioned_embeddings(self) -> None:
only when their stored vectors need this lifecycle. The marker is committed
*after* every eligible record is indexed, so an interrupted rebuild safely
repeats on the next startup rather than leaving a mixed mapping marked current.
+ Paging covers only records whose canonical vector is missing or stamped with
+ another fingerprint, so a restart resumes where the previous pass stopped
+ instead of re-embedding the whole store from scratch.
"""
identity = str(getattr(self.embedder, "embedding_identity", "") or "").strip()
version = str(getattr(self.embedder, "embedding_version", "") or "").strip()
@@ -658,8 +667,9 @@ def _rebuild_versioned_embeddings(self) -> None:
after_id = ""
try:
while True:
- records = self.store.list_memories_page(
- after_id=after_id, limit=EMBEDDING_REBUILD_BATCH, include_invalid=True,
+ records = self.store.list_memories_needing_vectors_page(
+ fingerprint=fingerprint, after_id=after_id,
+ limit=EMBEDDING_REBUILD_BATCH,
)
if not records:
break
@@ -839,7 +849,8 @@ def remember_with_resolution(self, content: str, *, workspace_id: str,
subject_key: str = "", claim_kind: str = "",
_trusted_graph_keys: Optional[frozenset] = None,
_approval_override: bool = False,
- _transactional_finalizer: Optional[Callable[[str], None]] = None) -> dict:
+ _transactional_finalizer: Optional[Callable[[str], None]] = None,
+ extra_neighbors: Optional[list] = None) -> dict:
"""Store one memory with deterministic conflict resolution.
Returns ``{"id", "op", ...}`` where ``op`` is one of:
@@ -1003,6 +1014,7 @@ def remember_with_resolution(self, content: str, *, workspace_id: str,
claim_kind=claim_kind, trusted_graph_keys=_trusted_graph_keys,
poisoning=poisoning, trusted_write=trusted_write,
defer_external_index=defer_external_index,
+ extra_neighbors=extra_neighbors,
)
if (
owns_session_transaction
@@ -1024,6 +1036,7 @@ def remember_with_resolution(self, content: str, *, workspace_id: str,
poisoning=poisoning, trusted_write=trusted_write,
transactional_finalizer=_transactional_finalizer,
defer_external_index=defer_external_index,
+ extra_neighbors=extra_neighbors,
)
if owns_lifecycle_transaction:
self.store.conn.commit()
@@ -1036,6 +1049,328 @@ def remember_with_resolution(self, content: str, *, workspace_id: str,
self.store.conn.rollback()
raise
+ def remember_many(self, facts, *, workspace_id: str,
+ repo_id: Optional[str] = None, session_id: Optional[str] = None,
+ mtype: MemoryType = MemoryType.SEMANTIC,
+ scope: Optional[Scope] = None) -> list[dict]:
+ """Store a batch of facts with within-batch resolution and evidence edges.
+
+ Implements the fan-out → collect → wire lifecycle for parallel-agent output:
+ all facts are embedded in one call, each fact is resolved against existing
+ memory AND the siblings already resolved earlier in this batch (so duplicates
+ deduplicate and keyed claims supersede within the batch), every insert shares
+ one transaction (all-or-nothing), and afterwards batch siblings that share a
+ non-empty ``subject_key`` or ``provenance.source`` are wired together with
+ evidence-labeled ``related`` edges ("no shared source, no edge" — similarity
+ alone never creates a sibling edge).
+
+ Accepts ``FactSpec`` items or bare strings. Returns one result dict per fact,
+ in input order, with the same shape as ``remember_with_resolution`` results.
+ The whole batch rolls back if any fact fails.
+ """
+ specs: list[FactSpec] = []
+ for fact in facts:
+ if isinstance(fact, str):
+ specs.append(FactSpec(content=fact))
+ elif isinstance(fact, FactSpec):
+ specs.append(fact)
+ else:
+ raise TypeError(
+ "facts must contain FactSpec instances or strings, "
+ f"got {type(fact).__name__}"
+ )
+ if not specs:
+ return []
+ if len(specs) > MAX_FACTS_PER_BATCH:
+ raise ValueError(
+ f"batch exceeds MAX_FACTS_PER_BATCH ({MAX_FACTS_PER_BATCH}); chunk the input"
+ )
+
+ scope_was_omitted = scope is None
+ sc = (
+ Scope.REPO if (repo_id or session_id) else Scope.WORKSPACE
+ ) if scope is None else Scope(scope)
+ if sc == Scope.USER:
+ raise ValueError(_USER_SCOPE_WRITE_ERROR)
+ if session_id:
+ session = self.store.get_session(session_id)
+ if session is None:
+ raise ValueError(f"no session with id '{session_id}'")
+ if session["workspace_id"] != workspace_id or (
+ repo_id is not None and session.get("repo_id") != repo_id):
+ raise ValueError("session_id does not belong to that workspace/repo")
+ if sc in (Scope.SESSION, Scope.REPO) and repo_id is None:
+ repo_id = session.get("repo_id")
+ if sc == Scope.SESSION and not session_id:
+ raise ValueError("session scope requires session_id")
+ if sc == Scope.REPO and not repo_id:
+ if scope_was_omitted:
+ sc = Scope.WORKSPACE
+ else:
+ raise ValueError("repo scope requires repo_id")
+ if sc in (Scope.WORKSPACE, Scope.USER) and repo_id:
+ raise ValueError(f"{sc.value} scope requires repo_id to be omitted")
+
+ # Per-fact validation and poisoning assessment happen before embedding; the
+ # metadata/provenance normalization below mirrors _resolve_and_store's own
+ # handling, so each fact's resolution trust follows its normalized provenance.
+ prepared: list[dict] = []
+ texts: list[str] = []
+ for spec in specs:
+ content = str(spec.content or "").strip()
+ if not content:
+ raise ValueError("every fact needs non-empty content")
+ title = str(spec.title or "").strip()
+ keywords = list(spec.keywords or [])
+ reject_secrets((("title", title), ("content", content),
+ ("keywords", keywords), ("metadata", spec.metadata),
+ ("subject_key", spec.subject_key),
+ ("claim_kind", spec.claim_kind)))
+ valid_from = spec.valid_from
+ if valid_from is not None:
+ try:
+ valid_from = float(valid_from)
+ except (TypeError, ValueError) as exc:
+ raise ValueError("valid_from must be a finite timestamp") from exc
+ if not math.isfinite(valid_from):
+ raise ValueError("valid_from must be a finite timestamp")
+ write_metadata = dict(spec.metadata or {})
+ provenance = write_metadata.get("provenance")
+ if isinstance(provenance, dict):
+ provenance = dict(provenance)
+ else:
+ provenance = dict(spec.provenance) if isinstance(
+ spec.provenance, dict) else {}
+ provenance.setdefault("trusted", True)
+ provenance.setdefault("trust_origin", "local_engine")
+ provenance.setdefault("source", "local_engine")
+ if provenance.get("trusted") is True:
+ provenance.setdefault("review_state", REVIEW_APPROVED)
+ else:
+ provenance.setdefault("review_state", REVIEW_PENDING)
+ write_metadata["provenance"] = provenance
+ # Mirror the single-write path: whether this fact may resolve against,
+ # supersede, or mint graph state for existing memory is decided by its
+ # normalized provenance envelope, never by a blanket True default.
+ trusted_write = prompt_eligible(provenance, write_metadata)
+ if self.embedding_space:
+ write_metadata["embed_model"] = self.embedding_space
+ poisoning = assess_untrusted_payload(
+ content, title=title, metadata=write_metadata)
+ mt = spec.mtype if spec.mtype is not None else mtype
+ text = f"{title}\n{content}" if title else content
+ evidence_source = str(spec.evidence_source or "").strip()
+ prepared.append({
+ "content": content, "title": title, "text": text, "mtype": mt,
+ "importance": float(spec.importance or 0.0), "keywords": keywords,
+ "metadata": write_metadata, "valid_from": valid_from,
+ "subject_key": str(spec.subject_key or "").strip(),
+ "claim_kind": str(spec.claim_kind or "").strip(),
+ "poisoning": poisoning,
+ "trusted_write": trusted_write,
+ "evidence_source": evidence_source,
+ })
+ if not poisoning.quarantined:
+ texts.append(text)
+
+ persistent_store = not _is_memory_database_path(self.store.path)
+ if texts and persistent_store:
+ if not self.embedding_space:
+ raise RuntimeError(
+ "persistent writes require an embedder with a durable "
+ "embedding_identity and embedding_version"
+ )
+ if not self.store.embedding_space_ready(self.embedding_space):
+ raise RuntimeError(
+ "the configured embedding space is not active; restart through "
+ "MemoryEngine.create() to complete the guarded rebuild"
+ )
+ # One embed call for the whole batch, before taking the write lock — same
+ # posture as single writes: the expensive part stays outside serialization.
+ vectors_by_text: dict[str, np.ndarray] = {}
+ if texts:
+ embedded = self.embedder.embed(texts)
+ if len(embedded) != len(texts):
+ raise RuntimeError("embedder returned the wrong number of vectors")
+ vectors_by_text = dict(zip(texts, embedded))
+ for item in prepared:
+ item["vec"] = (
+ None if item["poisoning"].quarantined else vectors_by_text[item["text"]]
+ )
+ # Pairwise cosine between batch siblings (vectors L2-normalized first so
+ # dot == cosine). Row i holds sibling i's similarity to every earlier
+ # sibling, used to feed real evidence into within-batch resolution.
+ batch_sims: list[list[float]] = []
+ vecs = [item["vec"] for item in prepared]
+ if any(v is not None for v in vecs):
+ dim = self.embedder.dim
+ matrix = np.zeros((len(vecs), dim), dtype=np.float32)
+ for idx, v in enumerate(vecs):
+ if v is not None:
+ norm = float(np.linalg.norm(v))
+ matrix[idx] = (
+ np.asarray(v, dtype=np.float32) / norm if norm > 0 else 0.0
+ )
+ gram = matrix @ matrix.T
+ batch_sims = [
+ [float(gram[i][j]) if vecs[j] is not None else 0.0
+ for j in range(len(vecs))]
+ for i in range(len(vecs))
+ ]
+ else:
+ batch_sims = [[0.0] * len(vecs) for _ in vecs]
+
+ results: list[dict] = []
+ resolved: list[tuple[int, MemoryRecord]] = [] # (prepared idx, record)
+ inserted: list[tuple[str, dict]] = [] # (memory_id, prepared item)
+ pending_vectors: list[tuple[str, np.ndarray]] = []
+
+ with self._write_lock:
+ caller_owned_transaction = (
+ self.store.conn.transaction_owned_by_current_thread()
+ )
+ if (
+ caller_owned_transaction
+ and any(item["vec"] is not None for item in prepared)
+ and vector_index_requires_sync(self.index, self.store)
+ and not vector_index_shares_store_transaction(self.index, self.store)
+ ):
+ raise RuntimeError(
+ "caller-owned transactions cannot write through a separate vector "
+ "index; commit or roll back before remembering"
+ )
+ owns_transaction = False
+ try:
+ if session_id:
+ # The pre-transaction existence/ownership check cannot serialize
+ # with a concurrent end_session; re-read status under the same
+ # write lock the batch commits in so a close that wins first
+ # rejects this batch instead of inheriting its writes.
+ owns_transaction = self.store.begin_session_write(
+ session_id, workspace_id=workspace_id, repo_id=repo_id
+ )
+ elif not caller_owned_transaction:
+ self.store.conn.execute("BEGIN IMMEDIATE")
+ owns_transaction = True
+ with self.store.conn.defer_commits():
+ for index_i, item in enumerate(prepared):
+ # Vector-search candidates are already filtered to the
+ # resolving fact's memory type; siblings must obey the same
+ # rule or an overlapping cross-type restatement could drive
+ # the ADD/NOOP/INVALIDATE decision unweighted.
+ extra_neighbors = [
+ (batch_sims[index_i][sibling_i], rec)
+ for sibling_i, rec in resolved
+ if rec.mtype == item["mtype"]
+ ]
+ result = self._resolve_and_store(
+ item["content"], text=item["text"], vec=item["vec"],
+ workspace_id=workspace_id, repo_id=repo_id,
+ session_id=session_id, mtype=item["mtype"], scope=sc,
+ title=item["title"], importance=item["importance"],
+ confidence=None, keywords=item["keywords"],
+ metadata=item["metadata"],
+ valid_from=item["valid_from"],
+ resolve_conflicts=True, candidate_k=5,
+ subject_key=item["subject_key"],
+ claim_kind=item["claim_kind"],
+ poisoning=item["poisoning"],
+ trusted_write=item["trusted_write"],
+ defer_external_index=True,
+ extra_neighbors=extra_neighbors,
+ )
+ results.append(result)
+ mid = result.get("id")
+ if (
+ result.get("op") in {"add", "invalidate", "relate"}
+ and isinstance(mid, str) and mid
+ ):
+ rec = self.store.get_memory(mid)
+ if rec is not None:
+ resolved.append((index_i, rec))
+ inserted.append((mid, item))
+ if (
+ result.get("op") in {"add", "invalidate", "relate"}
+ and item["vec"] is not None
+ and isinstance(mid, str) and mid
+ ):
+ # Only a newly inserted record may publish its vector:
+ # a noop's mid belongs to the pre-existing memory, and
+ # republishing would overwrite it with the candidate's
+ # vector even though the stored content never changed.
+ pending_vectors.append((mid, item["vec"]))
+ linked_pairs = self._evolve_batch(inserted)
+ self._audit_batch_evolve(linked_pairs)
+ if owns_transaction:
+ self.store.conn.commit()
+ for memory_id, vec in pending_vectors:
+ self._upsert_external_vector(memory_id, vec)
+ return results
+ except BaseException:
+ if ((owns_transaction or caller_owned_transaction)
+ and self.store.conn.transaction_owned_by_current_thread()):
+ self.store.conn.rollback()
+ raise
+
+ def _evolve_batch(self, inserted: list) -> list[tuple[str, str, str]]:
+ """Wire evidence-based edges between batch siblings.
+
+ Two inserted facts get a ``related`` edge only when they share a non-empty
+ ``subject_key`` or the same *explicitly declared* ``evidence_source``
+ (``FactSpec.evidence_source`` / a caller-supplied ``provenance.source``) —
+ shared, citeable evidence, never mere embedding proximity and never the
+ engine's own default provenance. Bounded by EVOLVE_MAX_LINKS per memory;
+ best-effort: failures warn and never break the batch.
+ """
+ links: list[tuple[str, str, str]] = []
+ link_counts: dict[str, int] = {}
+ for i in range(len(inserted)):
+ mid_a, item_a = inserted[i]
+ for j in range(i + 1, len(inserted)):
+ mid_b, item_b = inserted[j]
+ reason = ""
+ key_a = item_a["subject_key"]
+ key_b = item_b["subject_key"]
+ if key_a and key_a == key_b:
+ reason = f"shared subject_key: {key_a}"
+ else:
+ src_a = item_a.get("evidence_source") or ""
+ src_b = item_b.get("evidence_source") or ""
+ if src_a and src_a == src_b:
+ reason = f"shared source: {src_a}"
+ if not reason:
+ continue
+ if link_counts.get(mid_a, 0) >= EVOLVE_MAX_LINKS or \
+ link_counts.get(mid_b, 0) >= EVOLVE_MAX_LINKS:
+ continue
+ try:
+ if self.store.has_link(mid_a, mid_b):
+ continue
+ self.store.add_link(
+ mid_a, mid_b, "related", reason=reason, commit=False,
+ )
+ except Exception as exc: # noqa: BLE001 — best-effort wiring
+ self._warn_redacted_failure("batch evolution", exc)
+ continue
+ links.append((mid_a, mid_b, reason))
+ link_counts[mid_a] = link_counts.get(mid_a, 0) + 1
+ link_counts[mid_b] = link_counts.get(mid_b, 0) + 1
+ return links
+
+ def _audit_batch_evolve(self, links: list) -> None:
+ """Best-effort audit row summarizing the batch's evidence-edge wiring."""
+ if not links:
+ return
+ try:
+ detail = "; ".join(f"{a} <-> {b} ({reason})" for a, b, reason in links[:20])
+ self.store.audit(
+ "resolver", "batch_evolve",
+ links[0][0], f"{len(links)} evidence links: {detail}"[:1000],
+ commit=False,
+ )
+ except Exception as exc: # noqa: BLE001
+ self._warn_redacted_failure("batch evolution audit", exc)
+
def _publish_result_vector(self, result: dict, vec: Optional[np.ndarray]) -> None:
"""Publish one newly committed Store vector to a separate injected index."""
if vec is None or result.get("op") not in {"add", "invalidate", "relate"}:
@@ -1083,7 +1418,8 @@ def _resolve_and_store(self, content: str, *, text: str, vec: Optional[np.ndarra
poisoning: Optional[PoisoningDecision] = None,
trusted_write: bool = True,
transactional_finalizer: Optional[Callable[[str], None]] = None,
- defer_external_index: bool = False) -> dict:
+ defer_external_index: bool = False,
+ extra_neighbors: Optional[list] = None) -> dict:
"""The resolve→insert body of ``remember_with_resolution``. The caller holds
``self._write_lock`` for the whole call (atomicity of the resolve decision).
@@ -1103,6 +1439,7 @@ def _resolve_and_store(self, content: str, *, text: str, vec: Optional[np.ndarra
session_id=session_id, scope=scope, mtype=mtype,
candidate_k=candidate_k, subject_key=subject_key,
claim_kind=claim_kind, valid_at=valid_from, content=content,
+ extra_neighbors=extra_neighbors,
)
if (resolve_conflicts and trusted_write and not poisoning.quarantined
and subject_key and valid_from is not None):
@@ -1415,7 +1752,14 @@ def _resolve_and_store(self, content: str, *, text: str, vec: Optional[np.ndarra
)
self.store.conn.commit()
except Exception as exc: # noqa: BLE001 — best-effort repair, never fail the write
- if self.store.conn.transaction_owned_by_current_thread():
+ # Inside commit deferral (batch writes) a rollback here would target the
+ # OUTER savepoint and discard earlier facts in the same batch. Deferral
+ # keeps the failed repair's partial statements inside the caller's
+ # boundary; the outer owner decides settle-or-discard for the whole batch.
+ if (
+ self.store.conn.transaction_owned_by_current_thread()
+ and not getattr(self.store.conn._pin, "defer_commits", 0)
+ ):
self.store.conn.rollback()
self._warn_redacted_failure("conflict repair", exc)
out: dict[str, object]
@@ -1698,12 +2042,20 @@ def _resolve_against_neighbors(self, text: str, vec: np.ndarray, *, workspace_id
scope: Scope, mtype: MemoryType, candidate_k: int,
subject_key: str = "", claim_kind: str = "",
valid_at: Optional[float] = None,
- content: Optional[str] = None):
+ content: Optional[str] = None,
+ extra_neighbors: Optional[list] = None):
"""Fetch same-scope neighbors via the vector index and run the deterministic
resolver (``core.resolve``). Returns ``(decision, neighbors, conflicted_with)``
so the caller can also evolve the neighborhood and persist a conflict repair.
An injected-index failure uses the canonical stored-vector mirror; if that scan
- also fails, resolution aborts rather than blindly inserting overlapping truth."""
+ also fails, resolution aborts rather than blindly inserting overlapping truth.
+
+ ``extra_neighbors`` are additional ``(similarity, MemoryRecord)`` pairs the
+ caller already holds (batch siblings resolved earlier in the same transaction,
+ which deferred vector publication cannot yet surface). They are appended to
+ the candidate list before ``resolve()`` runs; the resolver applies the same
+ key/similarity floors to them as to any other neighbor.
+ """
flt = SearchFilter(
workspace_id=workspace_id, repo_id=repo_id,
session_id=session_id if scope == Scope.SESSION else None,
@@ -1818,6 +2170,11 @@ def append_visible_neighbors(
for record in authoritative:
if record.id not in known_ids:
neighbors.append((1.0, record))
+ if extra_neighbors:
+ known_ids = {rec.id for _, rec in neighbors}
+ for sim, rec in extra_neighbors:
+ if rec.id not in known_ids:
+ neighbors.append((sim, rec))
decision = resolve(
text, neighbors, subject_key=subject_key, claim_kind=claim_kind,
candidate_content=content,
@@ -1858,7 +2215,25 @@ def _repair_conflicts(self, new_id: str, new_text: str, neighbors: list, *,
"""
try:
conflicts = detect_conflicts(new_text, (rec for _, rec in neighbors))
- except Exception:
+ except Exception as exc:
+ failure_type = type(exc).__name__
+ logger.warning(
+ "conflict detection failed (%s); treating as no conflict",
+ failure_type,
+ )
+ try:
+ self.store.audit(
+ "resolver",
+ "conflict_detect_failed",
+ new_id or workspace_id or "resolution",
+ "failure_type=%s" % failure_type,
+ commit=not self.store.conn.transaction_owned_by_current_thread(),
+ )
+ except Exception as audit_exc:
+ logger.warning(
+ "could not audit conflict-detection failure (%s)",
+ type(audit_exc).__name__,
+ )
return None
if not conflicts:
return None
diff --git a/engraphis/core/fsutil.py b/engraphis/core/fsutil.py
index 04a25c86..ee491e12 100644
--- a/engraphis/core/fsutil.py
+++ b/engraphis/core/fsutil.py
@@ -18,3 +18,30 @@ def is_reparse_point(info: object) -> bool:
"""
marker = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400)
return bool(getattr(info, "st_file_attributes", 0) & marker)
+
+
+# Cloud-files placeholders (OneDrive Files-On-Demand et al.) are reparse points, but
+# unlike symlinks/junctions they name a real tree item that hydrates transparently on
+# open — reading one never redirects outside its path. Both attribute constants are
+# only ever set together with the reparse attribute on placeholder entries.
+_PLACEHOLDER_ATTRS = (
+ getattr(stat, "FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS", 0x400000)
+ | getattr(stat, "FILE_ATTRIBUTE_RECALL_ON_OPEN", 0x40000)
+)
+
+
+def is_cloud_placeholder(info: object) -> bool:
+ """Return whether *info* is a cloud-files placeholder rather than a link.
+
+ Such entries must be allowed through link guards: rejecting them makes every
+ not-locally-cached OneDrive file unimportable on Windows even though opening
+ the file is safe and simply downloads it.
+ """
+ attributes = getattr(info, "st_file_attributes", 0)
+ return bool(attributes & _PLACEHOLDER_ATTRS)
+
+
+def is_link_indirection(info: object) -> bool:
+ """Return whether *info* is a reparse point that must stay rejected: any symlink
+ or junction — i.e. a reparse point that is not a benign cloud placeholder."""
+ return is_reparse_point(info) and not is_cloud_placeholder(info)
diff --git a/engraphis/core/interfaces.py b/engraphis/core/interfaces.py
index 4e892754..76aea875 100644
--- a/engraphis/core/interfaces.py
+++ b/engraphis/core/interfaces.py
@@ -382,6 +382,32 @@ class ExtractedFact:
metadata: dict[str, Any] = field(default_factory=dict)
+@dataclass
+class FactSpec:
+ """One fact in a batch submitted to ``MemoryEngine.remember_many``.
+
+ Mirrors ``ExtractedFact`` plus the durable claim identity fields
+ (``subject_key``/``claim_kind``) and an optional per-fact ``provenance``
+ dict. Batch siblings that share a non-empty ``subject_key`` or a
+ ``provenance.source`` are wired together with evidence-labeled edges after
+ insertion ("no shared source, no edge").
+ """
+ content: str
+ title: str = ""
+ mtype: Optional[MemoryType] = None
+ importance: float = 0.0
+ keywords: list[str] = field(default_factory=list)
+ metadata: dict[str, Any] = field(default_factory=dict)
+ subject_key: str = ""
+ claim_kind: str = ""
+ valid_from: Optional[float] = None
+ provenance: Optional[dict[str, Any]] = None
+ # Citeable sibling-evidence origin declared by the caller (e.g. "subagent-7").
+ # Only an explicitly declared source participates in batch edge wiring; the
+ # engine's default provenance never counts as shared evidence.
+ evidence_source: Optional[str] = None
+
+
@dataclass
class RetentionDecision:
"""Optional host/LLM supervision signal for a new memory.
diff --git a/engraphis/core/obsidian.py b/engraphis/core/obsidian.py
index 4d2bb8a2..1cbceb61 100644
--- a/engraphis/core/obsidian.py
+++ b/engraphis/core/obsidian.py
@@ -16,14 +16,16 @@
import unicodedata
from engraphis.core.secrets import secret_kind
-from engraphis.core.fsutil import is_reparse_point as _is_reparse_point
+from engraphis.core.fsutil import is_link_indirection as _is_link_indirection
IMPORTER_VERSION = "1"
MAX_NOTE_CHARS = 100_000
MAX_NOTE_BYTES = 2_000_000
MAX_VAULT_FILES = 10_000
-MAX_VAULT_BYTES = 250_000_000
+# Lockstep with service.MAX_IMPORT_TOTAL_BYTES (750 MB): the wizard scanner must never
+# silently undercut the upload transport ceiling.
+MAX_VAULT_BYTES = 750_000_000
MAX_SOURCE_PATH_CHARS = 4_096
ATTACHMENT_SUFFIXES = {
".aac", ".avif", ".bmp", ".csv", ".epub", ".gif", ".jpeg", ".jpg",
@@ -201,7 +203,7 @@ def scan_obsidian_vault(vault_path: Union[os.PathLike[str], str]) -> ObsidianVau
selected_root = Path(vault_path)
try:
selected_info = os.lstat(selected_root)
- if selected_root.is_symlink() or _is_reparse_point(selected_info):
+ if selected_root.is_symlink() or _is_link_indirection(selected_info):
raise ValueError("vault root cannot be a symlink")
root = selected_root.resolve(strict=True)
except OSError as exc:
@@ -284,7 +286,7 @@ def _read_vault_note(root: Path, path: Path) -> Tuple[bytes, int]:
unavailable (notably Windows) and discard bytes if the directory entry changed.
"""
before = os.lstat(path)
- if stat.S_ISLNK(before.st_mode) or _is_reparse_point(before) or not stat.S_ISREG(before.st_mode):
+ if stat.S_ISLNK(before.st_mode) or _is_link_indirection(before) or not stat.S_ISREG(before.st_mode):
raise ValueError("unsafe file type")
if not _is_within(root, path.resolve(strict=True)):
raise ValueError("path escapes vault")
@@ -295,7 +297,7 @@ def _read_vault_note(root: Path, path: Path) -> Tuple[bytes, int]:
fd = os.open(path, flags)
try:
opened = os.fstat(fd)
- if not stat.S_ISREG(opened.st_mode) or _is_reparse_point(opened) or not _same_file_identity(before, opened):
+ if not stat.S_ISREG(opened.st_mode) or _is_link_indirection(opened) or not _same_file_identity(before, opened):
raise ValueError("file changed during scan")
if opened.st_size > MAX_NOTE_BYTES:
raise ValueError("note exceeds 2000000 byte safety limit")
@@ -316,7 +318,7 @@ def _read_vault_note(root: Path, path: Path) -> Tuple[bytes, int]:
or opened.st_size != finished.st_size
or opened.st_mtime_ns != finished.st_mtime_ns
or stat.S_ISLNK(after.st_mode)
- or _is_reparse_point(after)
+ or _is_link_indirection(after)
or not _same_file_identity(finished, after)
or not _is_within(root, path.resolve(strict=True))
):
@@ -350,7 +352,7 @@ def _walk_vault(root: Path, directory: Path) -> Iterable[Tuple[Path, Optional[st
try:
relative = entry.relative_to(root)
info = entry.lstat()
- if entry.is_symlink() or _is_reparse_point(info):
+ if entry.is_symlink() or _is_link_indirection(info):
yield entry, "symlink skipped"
continue
if not _is_within(root, entry.resolve()):
diff --git a/engraphis/core/recall.py b/engraphis/core/recall.py
index e89bea2f..f0c2b99b 100644
--- a/engraphis/core/recall.py
+++ b/engraphis/core/recall.py
@@ -1168,6 +1168,27 @@ def _prompt_eligible_edges(
)
]
+ def _query_entity_seeds(self, query: str, flt: SearchFilter) -> list[str]:
+ """Return scoped entity ids whose names occur in ``query``.
+
+ Shared seeding step for both graph arms (PPR and 1-hop): the bounded
+ scoped entity map from :meth:`_seed_entity_map`, filtered to the entities
+ whose folded name is a substring of the folded query and whose word-boundary
+ pattern matches the raw query.
+ """
+ entity_map = self._seed_entity_map(query, flt)
+ patterns = {
+ eid: (name.casefold(), _entity_pattern(name))
+ for eid, name in entity_map.items()
+ if name
+ }
+ query_folded = query.casefold()
+ return [
+ eid
+ for eid, (needle, pattern) in patterns.items()
+ if needle in query_folded and pattern.search(query)
+ ]
+
def _graph_arm_ppr(
self,
query: str,
@@ -1184,18 +1205,7 @@ def _graph_arm_ppr(
memories by walk probability. Multi-hop associations surface without
expanding an explicit hop count; entity nodes are prefixed so names can
never collide with memory ids."""
- entity_map = self._seed_entity_map(query, flt)
- patterns = {
- eid: (name.casefold(), _entity_pattern(name))
- for eid, name in entity_map.items()
- if name
- }
- query_folded = query.casefold()
- seeds = [
- eid
- for eid, (needle, pattern) in patterns.items()
- if needle in query_folded and pattern.search(query)
- ]
+ seeds = self._query_entity_seeds(query, flt)
if not seeds:
return {}
@@ -1354,18 +1364,7 @@ def _graph_arm_1hop(
candidate_k: int = 50,
prompt_only: bool = False,
) -> dict[str, float]:
- entity_map = self._seed_entity_map(query, flt)
- patterns = {
- eid: (name.casefold(), _entity_pattern(name))
- for eid, name in entity_map.items()
- if name
- }
- query_folded = query.casefold()
- seed_ids = [
- eid
- for eid, (needle, pattern) in patterns.items()
- if needle in query_folded and pattern.search(query)
- ]
+ seed_ids = self._query_entity_seeds(query, flt)
if not seed_ids:
return {}
related_ids = set(seed_ids)
@@ -1580,11 +1579,6 @@ def _entity_map(self, flt: SearchFilter, *, limit: int = 2048) -> dict[str, str]
for row in self.store.conn.execute(sql, params).fetchall()
}
- def _pack(self, cands: list[Candidate]) -> str:
- """Compatibility helper for callers that exercised the old private method."""
- context, _, _ = self.context_packer.pack("", cands, self.token_budget)
- return context
-
def _sanitize_plan(
proposed: RetrievalPlan,
@@ -2017,7 +2011,11 @@ def append_visible(value: object) -> None:
if store is not None and flt is not None:
try:
source = store.get_memory(memory_id)
- except Exception:
+ except Exception as exc:
+ logger.debug(
+ "consolidation evidence source lookup failed (%s)",
+ type(exc).__name__,
+ )
return
if source is None or not memory_matches_filter(source, flt):
return
@@ -2049,11 +2047,20 @@ def append_visible(value: object) -> None:
relation = str(link.get("relation") or "")
if relation not in ("consolidates", "profiles"):
continue
- other = link.get("b") if link.get("a") == record.id else link.get("a")
+ endpoint_a = str(link.get("a") or "").strip()
+ endpoint_b = str(link.get("b") or "").strip()
+ # The digest is one endpoint of the link; the other is the
+ # summarized source memory it must expose as evidence.
+ other = endpoint_b if endpoint_a == record.id else endpoint_a
+ if not other or other == record.id:
+ continue
append_visible(other)
- except Exception:
+ except Exception as exc:
# Link lookup is best-effort evidence enrichment, never a recall failure.
- pass
+ logger.warning(
+ "consolidation evidence link lookup failed (%s)",
+ type(exc).__name__,
+ )
return evidence
diff --git a/engraphis/core/store.py b/engraphis/core/store.py
index 6ba3cf18..d95075a9 100644
--- a/engraphis/core/store.py
+++ b/engraphis/core/store.py
@@ -4788,6 +4788,32 @@ def list_memories_page(self, flt: Optional[SearchFilter] = None, *,
rows = self.conn.execute(sql, params).fetchall()
return [_row_to_record(row) for row in rows]
+ def list_memories_needing_vectors_page(self, *, fingerprint: str,
+ after_id: str = "",
+ limit: int = 500) -> list[MemoryRecord]:
+ """Return one keyset page of memories whose canonical vector is missing or stale.
+
+ The versioned-embedding rebuild must be resumable: paging only rows whose
+ ``mem_vectors`` row is absent or stamped with another fingerprint lets an
+ interrupted rebuild skip already-converted records on restart. A missing
+ vector needs embedding just as much as a stale one, so rows without any
+ ``mem_vectors`` row are returned too. Unscoped and invalid-inclusive, like
+ the rebuild's previous full-scan paging.
+ """
+ sql = (
+ "SELECT m.* FROM memories AS m "
+ "LEFT JOIN mem_vectors AS v ON v.id = m.id "
+ "WHERE (v.id IS NULL OR COALESCE(v.model, '') <> ?)"
+ )
+ params: list[Any] = [str(fingerprint)]
+ if after_id:
+ sql += " AND m.id>?"
+ params.append(after_id)
+ sql += " ORDER BY m.id LIMIT ?"
+ params.append(max(1, int(limit)))
+ rows = self.conn.execute(sql, params).fetchall()
+ return [_row_to_record(row) for row in rows]
+
def close_validity(self, memory_id: str, *, at: Optional[float] = None,
actor: str = "system", reason: str = "contradicted",
@@ -5520,6 +5546,11 @@ def secure_erase_memory(
),
}
+ def search(self, query: str, k: int = 20,
+ *, filter: Optional[SearchFilter] = None) -> list[tuple[str, float]]:
+ """LexicalIndex protocol surface (``core.interfaces``); the BM25/LIKE arm."""
+ return self.fts_search(query, k, filter=filter)
+
def fts_search(self, query: str, k: int = 20,
*, filter: Optional[SearchFilter] = None) -> list[tuple[str, float]]:
"""Lexical arm. Uses FTS5 BM25 when available, else a LIKE fallback."""
@@ -7768,6 +7799,12 @@ def memories_mentioning(self, repo_id: str, text: str, *,
sql += " AND " + " AND ".join(where)
params.extend(visibility_params)
sql += " ORDER BY m.ingested_at DESC"
+ # Bounded SQL window: prompt-eligible rows can be sparse relative to the raw
+ # LIKE match set, so cap the scan instead of streaming the whole repo. The
+ # Python-side eligibility filter and result cap below are unchanged — with a
+ # normal match set the window never truncates.
+ sql += " LIMIT ?"
+ params.append(max(int(limit) * 50, 1000))
# This derived bridge feeds impact analysis. Filter sources before counting
# them, so a newer pending import cannot consume the bounded public window.
out = []
diff --git a/engraphis/core/sync.py b/engraphis/core/sync.py
index c6b5a0e0..02c6d9d4 100644
--- a/engraphis/core/sync.py
+++ b/engraphis/core/sync.py
@@ -661,6 +661,23 @@ def dict_to_record(d: Any) -> Optional[MemoryRecord]:
content = d.get("content")
if not isinstance(mid, str) or not mid or not isinstance(content, str) or not content:
return None
+ # Scope pointers are overwritten during apply (re-homed into local scope), but
+ # ``dict_to_record`` is itself a trust boundary (dry-run, hashing): a non-string
+ # or empty pointer is malformed exactly like a bad id/content row.
+ ws_id = d.get("workspace_id")
+ repo_id = d.get("repo_id")
+ for scope_ptr in (ws_id, repo_id):
+ if scope_ptr is not None and (
+ not isinstance(scope_ptr, str) or not scope_ptr):
+ return None
+ if ws_id is not None:
+ ws_id = _clamp_str(ws_id, 128)
+ if not ws_id:
+ return None
+ if repo_id is not None:
+ repo_id = _clamp_str(repo_id, 128)
+ if not repo_id:
+ return None
# Sync is an external memory write path. Reject the row before it can reach the
# raw Store upsert, FTS, or a locally rebuilt vector; a secret-bearing peer row is
# simply counted as rejected like any other malformed bundle entry.
@@ -718,7 +735,7 @@ def dict_to_record(d: Any) -> Optional[MemoryRecord]:
return MemoryRecord(
id=_clamp_str(mid, 128), content=_clamp_str(content, MAX_CONTENT_CHARS),
mtype=_mtype(d.get("mtype")), scope=_scope(d.get("scope")),
- workspace_id=d.get("workspace_id"), repo_id=d.get("repo_id"),
+ workspace_id=ws_id, repo_id=repo_id,
session_id=_clamp_str(d.get("session_id"), MAX_SESSION_ID_CHARS)
if isinstance(d.get("session_id"), str) else None,
title=_clamp_str(d.get("title"), MAX_TITLE_CHARS),
diff --git a/engraphis/dashboard_app.py b/engraphis/dashboard_app.py
index 15a880f7..0f86f153 100644
--- a/engraphis/dashboard_app.py
+++ b/engraphis/dashboard_app.py
@@ -21,11 +21,15 @@
import threading
import time
-from fastapi import FastAPI, File, Form, HTTPException, Request, UploadFile
+from fastapi import (
+ APIRouter, FastAPI, File, Form, HTTPException, Request, UploadFile,
+)
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import FileResponse, JSONResponse, Response
+from fastapi.routing import APIRoute
from fastapi.staticfiles import StaticFiles
from pydantic import BaseModel, Field
+from starlette.exceptions import HTTPException as StarletteHTTPException
from engraphis import licensing
from engraphis.config import settings
@@ -62,6 +66,13 @@
_DASHBOARD_REQUEST_BODY_LIMITS = {
"/api/auth/session": 8 * 1024,
"/api/workspaces/import-files": _DASHBOARD_UPLOAD_REQUEST_BYTES,
+ # Wizard multipart routes carry the same upload ceilings as the classic import:
+ # without these entries they fall back to the 8 MB JSON default and large vaults
+ # are rejected by the body middleware before the bounded parser is ever reached.
+ "/api/workspaces/import-documents/preview": _DASHBOARD_UPLOAD_REQUEST_BYTES,
+ "/api/workspaces/import-documents/run": _DASHBOARD_UPLOAD_REQUEST_BYTES,
+ "/api/workspaces/import-obsidian/preview": _DASHBOARD_UPLOAD_REQUEST_BYTES,
+ "/api/workspaces/import-obsidian/run": _DASHBOARD_UPLOAD_REQUEST_BYTES,
}
@@ -140,6 +151,69 @@ async def _too_large(scope, receive, send, max_bytes):
)(scope, receive, send)
+class _BoundedUploadRoute(APIRoute):
+ """Parse import uploads with their strict multipart limits before FastAPI binds files.
+
+ FastAPI otherwise resolves ``UploadFile`` parameters with Starlette's default
+ 1,000-file ceiling before the route can inspect ``len(files)`` — an unhandled
+ MultiPartException that surfaces as a bare 500 "Internal Server Error" for any
+ folder above 1,000 files. Mirrors routes/vault.py's _bounded_upload_form.
+ """
+
+ # Non-file form fields on the widest wizard route (documents preview): workspace,
+ # repo, session_id, scope, memory_type, source_id, source_label, on_conflict,
+ # source_mode, confirmed, attachment_manifest — headroom above the current
+ # 12-field maximum so adding one field does not 400 legitimate uploads.
+ _MAX_FORM_FIELDS = 14
+
+ def get_route_handler(self):
+ route_handler = super().get_route_handler()
+
+ async def bounded_route_handler(request: Request):
+ try:
+ await request.form(
+ max_files=MAX_IMPORT_FILES,
+ max_fields=self._MAX_FORM_FIELDS,
+ )
+ except StarletteHTTPException as exc:
+ detail = str(getattr(exc, "detail", ""))
+ lowered = detail.lower()
+ if exc.status_code == 400 and lowered.startswith("too many files"):
+ raise HTTPException(
+ status_code=413,
+ detail={"error": f"too many files (max {MAX_IMPORT_FILES})"},
+ ) from exc
+ if exc.status_code == 400 and lowered.startswith("too many fields"):
+ raise HTTPException(
+ status_code=400,
+ detail={"error": "invalid upload form"},
+ ) from exc
+ raise
+ return await route_handler(request)
+
+ return bounded_route_handler
+
+
+# Document/Obsidian wizard multipart routes on this app. (The classic quick import
+# /api/workspaces/import-files lives in routes/v2_api.py and already bounds its own
+# parser with request.form(max_files=...).)
+_BOUNDED_UPLOAD_PATHS = frozenset({
+ "/api/workspaces/import-documents/preview",
+ "/api/workspaces/import-documents/run",
+ "/api/workspaces/import-obsidian/preview",
+ "/api/workspaces/import-obsidian/run",
+})
+
+
+class _BoundedUploadRouter(APIRouter):
+ """Install the bounded parser only on the multipart import routes."""
+
+ def add_api_route(self, path: str, endpoint, **kwargs):
+ if path in _BOUNDED_UPLOAD_PATHS:
+ kwargs["route_class_override"] = _BoundedUploadRoute
+ return super().add_api_route(path, endpoint, **kwargs)
+
+
async def _dashboard_consolidation_loop(service: MemoryService) -> None:
"""Run opt-in v2 consolidation from the dashboard's actual lifespan.
@@ -451,6 +525,11 @@ def _discard_unbound_service() -> None:
raise
app.include_router(v2_api.router)
+ # Wizard multipart routes register through the bounded router (included further
+ # below, after those routes are defined) so uploads are parsed with MAX_IMPORT_FILES
+ # ceilings instead of Starlette's 1,000-file default.
+ bounded_router = _BoundedUploadRouter()
+
app.state.auth_store = None
app.state.team_enabled = False
@@ -898,7 +977,7 @@ def document_formats(request: Request):
_require_document_browser_owner(request)
return {"extensions": sorted(_DOCUMENT_SUFFIXES)}
- @app.post("/api/workspaces/import-documents/preview", include_in_schema=False)
+ @bounded_router.post("/api/workspaces/import-documents/preview", include_in_schema=False)
async def document_preview(
request: Request,
workspace: str = Form(...), repo: str = Form(""), session_id: str = Form(""),
@@ -961,7 +1040,7 @@ async def document_preview(
report, owner_binding=owner_binding, digest=digest,
)
- @app.post("/api/workspaces/import-documents/run", include_in_schema=False)
+ @bounded_router.post("/api/workspaces/import-documents/run", include_in_schema=False)
async def document_run(
request: Request,
workspace: str = Form(...), repo: str = Form(""), session_id: str = Form(""),
@@ -1057,7 +1136,7 @@ def obsidian_vaults(workspace: str, request: Request):
except (ValueError, KeyError):
raise HTTPException(status_code=400, detail={"error": "invalid request"}) from None
- @app.post("/api/workspaces/import-obsidian/preview", include_in_schema=False)
+ @bounded_router.post("/api/workspaces/import-obsidian/preview", include_in_schema=False)
async def obsidian_preview_alias(
request: Request, workspace: str = Form(...), repo: str = Form(""),
session_id: str = Form(""), scope: str = Form("workspace"),
@@ -1092,7 +1171,7 @@ async def obsidian_preview_alias(
report, owner_binding=owner_binding, digest=digest,
)
- @app.post("/api/workspaces/import-obsidian/run", include_in_schema=False)
+ @bounded_router.post("/api/workspaces/import-obsidian/run", include_in_schema=False)
async def obsidian_run_alias(
request: Request, workspace: str = Form(...), repo: str = Form(""),
session_id: str = Form(""), scope: str = Form("workspace"),
@@ -1145,6 +1224,10 @@ def cancel_obsidian_job_alias(job_id: str, request: Request, workspace: str = Fo
except (ValueError, KeyError):
raise HTTPException(status_code=404, detail={"error": "import job not found"}) from None
+ # Include AFTER the wizard routes are registered: include_router snapshots routes
+ # at call time, so including earlier would mount an empty router.
+ app.include_router(bounded_router)
+
from engraphis.netutil import is_local_request
@app.middleware("http")
diff --git a/engraphis/http_security.py b/engraphis/http_security.py
index ecd06e68..b3716349 100644
--- a/engraphis/http_security.py
+++ b/engraphis/http_security.py
@@ -131,6 +131,13 @@ def install(app) -> None:
csp_override = os.environ.get("ENGRAPHIS_CSP")
csp = DEFAULT_CSP if csp_override is None else csp_override.strip()
+ if csp_override is not None and not csp:
+ # Runs once per app (the idempotency guard above): a deployment that opted
+ # out of CSP entirely must be observable at startup, not silent.
+ logger.warning(
+ "ENGRAPHIS_CSP is set to an empty string: no Content-Security-Policy "
+ "header will be sent on any response."
+ )
hsts = os.environ.get("ENGRAPHIS_HSTS")
hsts = DEFAULT_HSTS if hsts is None else hsts.strip()
diff --git a/engraphis/llm/client.py b/engraphis/llm/client.py
index 13cfb095..7b459b77 100644
--- a/engraphis/llm/client.py
+++ b/engraphis/llm/client.py
@@ -173,9 +173,13 @@ def chat(
return self._chat_openai_compat(messages, system, temperature, max_tokens, timeout)
def synthesize_thought(self, context: str, *, temperature: float = 0.3,
- max_tokens: int = 512,
+ max_tokens: int = 4096,
thought_prompt: Optional[str] = None) -> dict[str, Any]:
- """Phase 2 thought synthesis — returns parsed JSON latent state."""
+ """Phase 2 thought synthesis — returns parsed JSON latent state.
+
+ The default completion budget leaves headroom for reasoning models that
+ spend hidden reasoning tokens before emitting the JSON payload.
+ """
# Security: user-supplied thought_prompt is appended as guidance, never
# allowed to replace the system prompt entirely. This prevents prompt
# injection via the /memories/thoughts route.
@@ -243,9 +247,12 @@ def ping(self) -> dict[str, Any]:
key, 401, wrong base URL, unreachable host) without a stack trace.
"""
try:
+ # Reasoning models spend the completion budget on hidden reasoning
+ # tokens before any visible content; a tiny cap can return an empty
+ # reply and read as a false negative.
reply = self.chat(
[{"role": "user", "content": "Reply with the single word: ok"}],
- temperature=0.0, max_tokens=5,
+ temperature=0.0, max_tokens=1024,
)
return {"ok": True, "reply": (reply or "").strip()[:200],
"error": "", "provider": self.provider, "model": self.model}
diff --git a/engraphis/mcp_http_cli.py b/engraphis/mcp_http_cli.py
index 4b00b60a..a9857ebf 100644
--- a/engraphis/mcp_http_cli.py
+++ b/engraphis/mcp_http_cli.py
@@ -131,6 +131,15 @@ def main(argv=None) -> None:
server.settings.host = args.host
server.settings.port = args.port
server.settings.transport_security = _transport_security(args.host, args.port)
+ # Restart-resilient transport. FastMCP's default *stateful* mode tracks MCP
+ # session ids in memory, so every service bounce (pm2 resurrect, watchdog,
+ # manual restart) invalidates all live session ids: the client's next request
+ # gets a 404, the mcp SDK raises "Session terminated", and Hermes' gateway
+ # client parks for its full retry interval with zero registered tools.
+ # Stateless mode makes each POST self-contained per the MCP spec, so any
+ # healthy process can answer any request. Spec-compliant clients handle the
+ # absent GET SSE stream (the server answers 405 and clients skip it).
+ server.settings.stateless_http = True
_eager_exact_backend_check()
server.run(transport=args.transport)
diff --git a/engraphis/mcp_server.py b/engraphis/mcp_server.py
index e2e227eb..8ebaf21f 100644
--- a/engraphis/mcp_server.py
+++ b/engraphis/mcp_server.py
@@ -130,8 +130,10 @@ def _err(exc: Exception) -> str:
return f"Error: {exc}"
exc_type = type(exc).__name__
# Redact exception messages to prevent credential/path/memory leakage.
- # Log only a safe class marker and never attach exc_info/tracebacks.
- logger.error("MCP tool operation failed", extra={"error_class": exc_type})
+ # Log only a safe class marker and never attach exc_info/tracebacks. The
+ # class goes INTO the message: `extra=` fields are dropped by most
+ # formatters, which made every failure log identically unattributable.
+ logger.error("MCP tool operation failed (%s)", exc_type)
return "Error: operation failed. Check the Engraphis server logs for details."
@@ -485,6 +487,76 @@ def engraphis_remember(
return _err(exc)
+@mcp.tool(
+ name="engraphis_remember_many",
+ annotations={"title": "Remember a batch of facts", "readOnlyHint": False,
+ "destructiveHint": False, "idempotentHint": False, "openWorldHint": False},
+)
+def engraphis_remember_many(
+ facts: Annotated[List[dict], Field(description="The facts collected from a fan-out "
+ "(parallel sub-agents, research, a review council), "
+ "as a list of objects: each needs 'content' and "
+ "optionally 'title', 'importance' (0..1), "
+ "'keywords', 'subject_key' (stable claim subject "
+ "like 'api.rate_limit'), 'claim_kind', "
+ "'evidence_source' (per-fact origin label; facts "
+ "sharing one get evidence-labeled links), and "
+ "'valid_from' (Unix timestamp). All facts are "
+ "stored in one transaction; each is deduplicated "
+ "against the others, and facts that share a "
+ "subject_key or evidence_source are linked with "
+ "evidence-labeled edges.", min_length=1,
+ max_length=500)],
+ workspace: Annotated[str, Field(description="Top-level scope, e.g. an org or product "
+ "name ('acme'). Defaults to 'default' if omitted.",
+ min_length=1, max_length=200)] = "default",
+ repo: Annotated[Optional[str], Field(description="Repository scope within the workspace "
+ "('backend'). Omit for workspace-wide memories.",
+ max_length=200)] = None,
+ session_id: Annotated[Optional[str], Field(description="Session id from "
+ "engraphis_start_session, if this batch belongs to one.")] = None,
+ mtype: Annotated[str, Field(description="Default memory type for facts without their "
+ "own: 'semantic' (facts/conventions), 'episodic' (events/decisions), "
+ "'procedural' (how-tos), or 'working' (transient).")] = "semantic",
+ scope: Annotated[Optional[str], Field(
+ description="Visibility: session, repo, workspace, or user. Omit to infer the "
+ "compatible default: repo when repo or a repo-backed session_id is "
+ "present, otherwise workspace. Session visibility must be explicit.")] = None,
+ source: Annotated[str, Field(description="Origin of the content. Web, import, sync, and "
+ "other external origins are always untrusted even if trusted=true; "
+ "use the default agent only for facts the connected local agent "
+ "authored or independently verified.", max_length=200)] = "agent",
+ trusted: Annotated[bool, Field(description="Local-agent confidence label. External origins "
+ "cannot elevate themselves with this field.")] = True,
+) -> str:
+ """Store a batch of facts from parallel agents in one atomic, deduplicated write.
+
+ Use this instead of many ``engraphis_remember`` calls when one turn produced a
+ set of findings (fan-out sub-agents, a research sweep, a review council): the
+ whole batch lands in a single transaction, each fact is resolved against the
+ others (duplicates reinforce, keyed claims supersede), and facts sharing a
+ ``subject_key`` or an explicit per-fact ``evidence_source`` get
+ evidence-labeled graph edges so the merge is a growing graph rather than a
+ pile of prose.
+
+ Returns:
+ str: JSON ``{"workspace","repo","scope","stored":true,"total","ops",
+ "results":[{"id","op",...}]}`` with one entry per input fact, in order.
+ Returns ``"Error: "`` if validation fails or any fact cannot be
+ stored (the whole batch rolls back in that case).
+ """
+ try:
+ return _ok(service().remember_many(
+ facts, workspace=workspace, repo=repo, session_id=session_id,
+ mtype=mtype, scope=scope,
+ source=source, trusted=trusted,
+ _local_agent_operator=bool(trusted),
+ _ingress="mcp",
+ ))
+ except Exception as exc: # noqa: BLE001 - surface a safe, actionable message
+ return _err(exc)
+
+
@mcp.tool(
name="engraphis_recall",
annotations={"title": "Recall relevant memories", "readOnlyHint": False,
diff --git a/engraphis/routes/v2_api.py b/engraphis/routes/v2_api.py
index 2bba88d0..7c597a93 100644
--- a/engraphis/routes/v2_api.py
+++ b/engraphis/routes/v2_api.py
@@ -227,6 +227,12 @@ def _run(fn, *a, **k):
#: status-keyed public text (see ``_managed_error_message``), so nothing legitimate comes
#: close; a message that does is by definition not the fixed copy and is dropped.
_MANAGED_ERROR_MAX_CHARS = 300
+#: Cooldown for identical managed-cloud warnings. The dashboard UI polls these
+#: endpoints on a cadence, so a lapsed account would otherwise write one
+#: identical warning per poll. Logging only: ``_record_authoritative_denial``
+#: below still runs on every denial.
+_MANAGED_WARN_COOLDOWN_SECONDS = 300.0
+_managed_warn_last: dict = {}
def _managed_error_message(exc) -> str:
@@ -270,8 +276,16 @@ def _managed_call(fn, *args, **kwargs):
# until a later background entitlement poll happens to run.
if exc.status in {401, 402, 403}:
_record_authoritative_denial()
- logger.warning("managed cloud operation failed (%s, status=%s, transient=%s)",
- type(exc).__name__, exc.status, exc.transient)
+ warn_key = (type(exc).__name__, exc.status, bool(exc.transient))
+ now = time.monotonic()
+ last_warn = _managed_warn_last.get(warn_key)
+ if last_warn is None or now - last_warn >= _MANAGED_WARN_COOLDOWN_SECONDS:
+ _managed_warn_last[warn_key] = now
+ logger.warning("managed cloud operation failed (%s, status=%s, transient=%s)",
+ type(exc).__name__, exc.status, exc.transient)
+ else:
+ logger.debug("managed cloud operation failed (%s, status=%s, transient=%s)",
+ type(exc).__name__, exc.status, exc.transient)
detail = {"error": _managed_error_message(exc), "managed_cloud": True,
"transient": exc.transient}
if exc.code in {"consent_required", "cloud_unconfigured"}:
diff --git a/engraphis/routes/vault.py b/engraphis/routes/vault.py
index a504fba5..5839497b 100644
--- a/engraphis/routes/vault.py
+++ b/engraphis/routes/vault.py
@@ -35,7 +35,7 @@
from engraphis.stores import blob_to_vector, get_conn, now_ts
from engraphis.stores import vaults as vault_store
from engraphis.stores import vectors as mem_store
-from engraphis.core.fsutil import is_reparse_point as _is_reparse_point
+from engraphis.core.fsutil import is_link_indirection as _is_link_indirection
logger = logging.getLogger("engraphis.routes.vault")
# Multipart boundaries and per-part headers count toward the HTTP request size even
@@ -78,7 +78,7 @@ def _read_import_file(folder: Path, path: Path, limit: int) -> bytes:
enumeration phase and this read cannot escape the import root.
"""
before = os.lstat(path)
- if not stat.S_ISREG(before.st_mode) or stat.S_ISLNK(before.st_mode) or _is_reparse_point(before):
+ if not stat.S_ISREG(before.st_mode) or stat.S_ISLNK(before.st_mode) or _is_link_indirection(before):
raise OSError("unsafe file type")
if not _is_within(folder, path.resolve(strict=True)):
raise OSError("path escapes import root")
@@ -86,7 +86,7 @@ def _read_import_file(folder: Path, path: Path, limit: int) -> bytes:
fd = os.open(path, flags)
try:
opened = os.fstat(fd)
- if not stat.S_ISREG(opened.st_mode) or _is_reparse_point(opened) or not _same_identity(before, opened):
+ if not stat.S_ISREG(opened.st_mode) or _is_link_indirection(opened) or not _same_identity(before, opened):
raise OSError("file changed during import")
if opened.st_size > limit:
raise OSError("import resource exceeds its byte limit")
@@ -106,7 +106,7 @@ def _read_import_file(folder: Path, path: Path, limit: int) -> bytes:
or opened.st_size != finished.st_size
or opened.st_mtime_ns != finished.st_mtime_ns
or stat.S_ISLNK(after.st_mode)
- or _is_reparse_point(after)
+ or _is_link_indirection(after)
or not _same_identity(finished, after)
or not _is_within(folder, path.resolve(strict=True))
):
diff --git a/engraphis/service.py b/engraphis/service.py
index d98ca8da..da6b6775 100644
--- a/engraphis/service.py
+++ b/engraphis/service.py
@@ -24,6 +24,7 @@
import logging
import math
import copy
+import sqlite3
import time
import threading
import unicodedata
@@ -51,7 +52,7 @@
from engraphis.core.ids import new_id as make_id
from engraphis.core.savings import annotate_usage, normalize_release_version
from engraphis.core.interfaces import (
- Edge, GraphLayer, MemoryType, Node, Scope, SearchFilter,
+ Edge, FactSpec, GraphLayer, MemoryType, Node, Scope, SearchFilter,
embedder_capabilities, embedding_space_fingerprint,
vector_index_requires_sync,
vector_index_shares_store_transaction,
@@ -201,6 +202,30 @@ def _recall_score_semantics(capabilities: dict) -> dict:
)
return semantics
+
+def _vector_index_backend_label(index: Any) -> str:
+ """Label the active vector index backend for the recall envelope (additive)."""
+ if index is None:
+ return "numpy"
+ name = type(index).__name__
+ if "SqliteVec" in name:
+ return "sqlite-vec"
+ if name == "NumpyVectorIndex":
+ return "numpy"
+ return name
+
+
+def _reranker_mode_label(reranker: Any) -> str:
+ """Label the active reranker mode for the recall envelope (additive)."""
+ if reranker is None:
+ return "identity"
+ name = type(reranker).__name__
+ if "CrossEncoder" in name:
+ return "cross-encoder"
+ if name == "IdentityReranker":
+ return "identity"
+ return name
+
def _finite_float(value: Any, default: float = 0.0) -> float:
"""Coerce persisted numeric fields without exposing NaN/Infinity downstream."""
try:
@@ -234,11 +259,17 @@ def _with_retrieval_capabilities(payload: dict, embedder, store=None) -> dict:
MAX_AGENT_STATE_CHARS = 20_000
# import_folder/import_files (SECURITY.md §5 — reads/accepts local-content by path or
# upload; these bound resource use, not access scope, same framing as index_repo's
-# max_files/max_file_bytes).
-MAX_IMPORT_FILES = 500
+# max_files/max_file_bytes). Count raised 500→1,500 with total scaled 250 MB→750 MB so
+# the average per-file allowance (0.5 MB) is unchanged; per-file caps stay fixed.
+# Upload transports buffer accepted parts in RAM up to this total before dispatch —
+# acceptable for the local-first single-user posture; network deployments should keep
+# tighter reverse-proxy body caps (SECURITY.md §2). Keep MAX_VAULT_BYTES (core/
+# obsidian.py) and MAX_DOCUMENT_TREE_BYTES (core/documents.py) in lockstep so wizard
+# scanners never silently undercut the transport ceiling.
+MAX_IMPORT_FILES = 1_500
MAX_IMPORT_FILE_BYTES = 2_000_000
MAX_IMPORT_RESOURCE_BYTES = 100_000_000
-MAX_IMPORT_TOTAL_BYTES = 250_000_000
+MAX_IMPORT_TOTAL_BYTES = 750_000_000
# Analytical graph scenes rank the candidate graph before applying the much smaller
# browser scene budget. Keep that server-side candidate set finite as well: graph rows
# are user/sync writable, and an unbounded Louvain/PageRank request would otherwise be a
@@ -259,6 +290,7 @@ def _with_retrieval_capabilities(payload: dict, embedder, store=None) -> dict:
MAX_GRAPH_INDEX_WORKERS = 2
GRAPH_INDEX_BATCH_SIZE = 100
GRAPH_INDEX_LEASE_SECONDS = 60.0
+IMPORT_JOB_LEASE_SECONDS = 900.0
GRAPH_INDEX_JOB_HISTORY = 100
GRAPH_INDEX_SHUTDOWN_SECONDS = 10.0
DEFAULT_CODE_QUERY_CAPACITY = 10_000
@@ -904,20 +936,28 @@ def _resolve_import_root(raw_path: str) -> Path:
return folder
-def _iter_import_files(folder: Path, pattern: str, max_files: int) -> list:
+def _iter_import_files(
+ folder: Path, pattern: str, max_files: int,
+) -> tuple[list, int, int]:
"""Files under ``folder`` matching the glob ``pattern`` (default ``*.md``), skipping
VCS/dependency directories and capped at ``max_files`` — a resource bound, not a
security boundary (the boundary is ``_resolve_import_root``).
+ Returns ``(files, matched_total, unreadable)`` so callers can surface silent
+ truncation instead of importing an alphabetically-first slice that looks complete.
+ ``matched_total`` counts every regular-file match seen before the cap; ``unreadable``
+ counts candidates whose stat/resolve failed (e.g. paths beyond the Windows MAX_PATH
+ limit without LongPathsEnabled) — previously these vanished from all counts.
+
Symlink escape guard: ``rglob`` follows symlinked directories, so a symlink placed
somewhere under an allowed root (by anything that ever had write access there) could
point outside the allowed root entirely and defeat ``_resolve_import_root`` — every
candidate is re-resolved and re-contained here, the same check the root itself got."""
import fnmatch
files: list = []
+ matched_total = 0
+ unreadable = 0
for f in sorted(folder.rglob("*")):
- if len(files) >= max_files:
- break
if not f.is_file() or not fnmatch.fnmatch(f.name, pattern):
continue
try:
@@ -926,12 +966,16 @@ def _iter_import_files(folder: Path, pattern: str, max_files: int) -> list:
real = f.resolve(strict=True)
rel = real.relative_to(folder)
except (OSError, ValueError):
+ unreadable += 1
continue
parts = rel.parts
if any(p == "node_modules" or p == ".git" or p.startswith(".") for p in parts[:-1]):
continue
+ matched_total += 1
+ if len(files) >= max_files:
+ continue
files.append(real)
- return files
+ return files, matched_total, unreadable
def _title_from_content(content: str, fallback: str) -> str:
@@ -1879,6 +1923,159 @@ def remember_batch(self, memories: list[dict], *, workspace: str) -> dict:
"results": results,
}
+ def remember_many(self, facts: list[dict], *, workspace: str,
+ repo: Optional[str] = None, session_id: Optional[str] = None,
+ mtype: str = "semantic", scope: Optional[str] = None,
+ source: str = "agent", trusted: bool = False,
+ _local_agent_operator: bool = False,
+ _ingress: str = "service") -> dict:
+ """Store a fan-out batch with within-batch resolution and evidence edges.
+
+ Unlike :meth:`remember_batch` (which loops ordinary single writes and can
+ leave duplicates across items), this runs the engine's batch assembly:
+ one shared transaction, each fact also resolved against its already-resolved
+ siblings, and afterwards batch siblings sharing a non-empty ``subject_key``
+ or ``provenance.source`` are wired with evidence-labeled ``related`` edges.
+ All-or-nothing: any engine failure rolls back every fact in the batch.
+
+ Each item accepts ``content`` (required) plus optional ``title``, ``mtype``,
+ ``importance``, ``keywords``, ``metadata``, ``subject_key``, ``claim_kind``,
+ and ``valid_from``. Provenance/trust is decided once for the whole batch —
+ a sub-agent fleet shares one origin.
+ """
+ if not isinstance(facts, list):
+ raise ValidationError("facts must be a list")
+ if not facts:
+ raise ValidationError("facts list must not be empty")
+ if len(facts) > 500:
+ raise ValidationError("facts list must not exceed 500 items")
+
+ ws = self._clean_ws(workspace)
+ rp = _clean_name(repo, field="repo") if repo else None
+ default_mt = _enum(mtype, MemoryType, "mtype")
+ scope_was_omitted = scope is None
+ sc = _write_scope(scope, repo=rp, session_id=session_id)
+ local_agent_provenance = (
+ _local_agent_provenance(source, ingress=_ingress)
+ if _local_agent_operator else None
+ )
+ provenance = (
+ _local_cli_provenance()
+ if _local_agent_operator and source == "cli" else
+ local_agent_provenance
+ if local_agent_provenance is not None else
+ _canonical_write_provenance(
+ source, trusted, raw_ingest=False, ingress=_ingress
+ )
+ )
+ wid = self._get_or_create_workspace(ws)
+ rid = self.store.get_or_create_repo(wid, rp) if rp else None
+ session = self._session_for_write(session_id, wid, rid)
+ if sc in (Scope.SESSION, Scope.REPO) and rid is None and session:
+ rid = session.get("repo_id")
+ if rid:
+ row = self.store.conn.execute(
+ "SELECT name FROM repos WHERE id=?", (rid,)
+ ).fetchone()
+ rp = row["name"] if row else None
+ if sc == Scope.REPO and rid is None:
+ if scope_was_omitted:
+ sc = Scope.WORKSPACE
+ else:
+ raise ValidationError("repo scope requires a repo-backed session_id")
+
+ specs: list[FactSpec] = []
+ for fact in facts:
+ if not isinstance(fact, dict):
+ raise ValidationError("each fact must be a dict")
+ content = _clean_text(
+ fact.get("content"), field="content", max_chars=MAX_CONTENT_CHARS
+ )
+ title = _clean_text(
+ fact.get("title", ""), field="title", max_chars=MAX_TITLE_CHARS,
+ required=False,
+ )
+ _reject_secret_capture((
+ ("content", content), ("title", title),
+ ("keywords", fact.get("keywords")),
+ ("metadata", fact.get("metadata")),
+ ("subject_key", fact.get("subject_key", "")),
+ ("claim_kind", fact.get("claim_kind", "")),
+ ))
+ mt = (
+ _enum(fact["mtype"], MemoryType, "mtype")
+ if fact.get("mtype") else default_mt
+ )
+ try:
+ importance = float(fact.get("importance", 0.0))
+ except (TypeError, ValueError, OverflowError):
+ raise ValidationError("importance must be a number")
+ if not math.isfinite(importance):
+ raise ValidationError("importance must be finite")
+ importance = max(0.0, min(1.0, importance))
+ valid_from = _optional_timestamp(
+ fact.get("valid_from"), field="valid_from"
+ )
+ evidence_source = _clean_text(
+ fact.get("evidence_source", ""), field="evidence_source",
+ max_chars=MAX_NAME_CHARS, required=False,
+ )
+ specs.append(FactSpec(
+ content=content,
+ title=title,
+ mtype=mt,
+ importance=importance,
+ keywords=_clean_keywords(fact.get("keywords")),
+ metadata={
+ **_clean_metadata(fact.get("metadata")),
+ "provenance": provenance,
+ },
+ subject_key=_clean_text(
+ fact.get("subject_key", ""), field="subject_key",
+ max_chars=MAX_TITLE_CHARS, required=False,
+ ),
+ claim_kind=_clean_text(
+ fact.get("claim_kind", ""), field="claim_kind",
+ max_chars=MAX_NAME_CHARS, required=False,
+ ),
+ valid_from=valid_from,
+ evidence_source=evidence_source or None,
+ ))
+
+ try:
+ results = self.engine.remember_many(
+ specs, workspace_id=wid, repo_id=rid, session_id=session_id,
+ scope=sc,
+ )
+ except ValueError as exc:
+ if str(exc).startswith("valid_from "):
+ raise ValidationError(str(exc)) from exc
+ if session_id and str(exc) in {
+ f"no session with id '{session_id}'",
+ "session_id does not belong to that workspace/repo",
+ "session_id is not active",
+ }:
+ raise ValidationError(str(exc)) from exc
+ raise
+ ops = [r.get("op", "") for r in results]
+ out = {
+ "workspace": ws, "repo": rp, "scope": sc.value, "stored": True,
+ "total": len(results), "ops": ops,
+ "results": [
+ {"id": r.get("id"), "op": r.get("op"),
+ **({"reason": r["reason"]} if r.get("reason") else {}),
+ **({"superseded": r["superseded"]}
+ if r.get("superseded") is not None else {})}
+ for r in results
+ ],
+ }
+ self.store.record_receipt(
+ "remember_many", workspace_id=wid, repo_id=rid or "",
+ actor=provenance["source"], target_count=len(results),
+ status="batch", metadata={"scope": sc.value, "ops": ops},
+ )
+ return out
+
def ingest(self, content: str, *, workspace: str, repo: Optional[str] = None,
session_id: Optional[str] = None, mtype: str = "semantic",
scope: Optional[str] = None, metadata: Optional[dict] = None,
@@ -2103,7 +2300,12 @@ def _import_one(self, name: str, content: str, *, ws: str, mt: MemoryType,
metadata={**(extra_provenance or {}), "import_file": name},
)
return {"file": name, "id": r["id"], "op": r["op"]}
- except ValidationError as exc:
+ except (ValidationError, ValueError, sqlite3.Error, RecursionError,
+ MemoryError) as exc:
+ # One bad file must degrade to a per-file error, not void the whole batch
+ # (e.g. sqlite3.OperationalError "database is locked" from a concurrent
+ # CLI/MCP writer, embedder ValueError, or a crafted deep-nested JSON upload
+ # blowing json.loads recursion).
logger.info("uploaded resource import rejected (%s)", type(exc).__name__)
return {"file": name, "error": "resource could not be imported"}
@@ -2164,7 +2366,9 @@ def import_folder(self, *, workspace: str, path: str, file_pattern: str = "*.md"
folder = _resolve_import_root(raw_path)
wid = self._get_or_create_workspace(ws)
- files = _iter_import_files(folder, pattern, MAX_IMPORT_FILES)
+ files, matched_total, unreadable = _iter_import_files(
+ folder, pattern, MAX_IMPORT_FILES,
+ )
total_bytes = 0
for file in files:
try:
@@ -2175,7 +2379,10 @@ def import_folder(self, *, workspace: str, path: str, file_pattern: str = "*.md"
raise ValidationError(
f"import batch is too large (max {MAX_IMPORT_TOTAL_BYTES} bytes)"
)
- from engraphis.backends.resources import get_resource_extractor
+ from engraphis.backends.resources import (
+ ResourceExtractionError,
+ get_resource_extractor,
+ )
resource_extractor = get_resource_extractor()
imported, skipped, errors, derived_facts = 0, 0, 0, 0
@@ -2187,13 +2394,24 @@ def import_folder(self, *, workspace: str, path: str, file_pattern: str = "*.md"
details.append({"file": f.name, "error": "file too large"})
continue
resource = resource_extractor.extract_path(str(f))
- except (OSError, ValueError) as exc:
+ except ResourceExtractionError as exc:
if "no extractable text" in str(exc):
skipped += 1
continue
+ logger.warning("folder import failed for one file (%s): %s",
+ type(exc).__name__, exc)
+ errors += 1
+ from engraphis.core.documents import _safe_reason
+ details.append({"file": f.name, "error": _safe_reason(exc)})
+ continue
+ except (OSError, ValueError, RecursionError, MemoryError) as exc:
+ # One unreadable/oversized/pathological file must degrade to a per-file
+ # error, not void the whole batch with an unhandled 500.
logger.warning("folder import failed for one file (%s)", type(exc).__name__)
errors += 1
- details.append({"file": f.name, "error": "file could not be imported"})
+ reason = "file is locked or unreadable" if isinstance(exc, OSError) \
+ else "file content could not be processed"
+ details.append({"file": f.name, "error": reason})
continue
rel = f.relative_to(folder).as_posix()
resource_meta = {
@@ -2236,10 +2454,27 @@ def import_folder(self, *, workspace: str, path: str, file_pattern: str = "*.md"
self.store.audit(actor, "import_folder", wid,
f"{raw_path} ({imported} imported)")
self.store.conn.commit()
- return {"workspace": ws, "path": str(folder), "scanned": len(files),
+ truncated = matched_total > len(files)
+ if truncated:
+ warnings.insert(0, {
+ "file": "", "warnings": [
+ f"folder contains {matched_total} matching files; imported the "
+ f"first {len(files)} (max {MAX_IMPORT_FILES}). Narrow the path or "
+ f"file pattern to reach the rest.",
+ ],
+ })
+ if unreadable:
+ warnings.append({
+ "file": "", "warnings": [
+ f"{unreadable} file(s) could not be read (locked, or path too long)",
+ ],
+ })
+ return {"workspace": ws, "path": str(folder),
+ "scanned": len(files), "matched_total": matched_total,
+ "truncated": truncated, "unreadable": unreadable,
"imported": imported, "skipped": skipped, "errors": errors,
- "derived_facts": derived_facts, "details": details[:50],
- "warnings": warnings[:50]}
+ "derived_facts": derived_facts, "details": details[:200],
+ "warnings": warnings[:200]}
@_rollback_service_transaction
def import_files(self, *, workspace: str, files: list, memory_type: str = "semantic",
@@ -2273,7 +2508,10 @@ def import_files(self, *, workspace: str, files: list, memory_type: str = "seman
)
wid = self._get_or_create_workspace(ws)
- from engraphis.backends.resources import get_resource_extractor
+ from engraphis.backends.resources import (
+ ResourceExtractionError,
+ get_resource_extractor,
+ )
resource_extractor = get_resource_extractor()
imported, skipped, errors, derived_facts = 0, 0, 0, 0
details, warnings = [], []
@@ -2297,13 +2535,24 @@ def import_files(self, *, workspace: str, files: list, memory_type: str = "seman
continue
try:
resource = resource_extractor.extract_bytes(name, bytes(raw))
- except ValueError as exc:
+ except ResourceExtractionError as exc:
if "no extractable text" in str(exc):
skipped += 1
continue
+ logger.info("uploaded resource extraction failed (%s): %s",
+ type(exc).__name__, exc)
+ errors += 1
+ from engraphis.core.documents import _safe_reason
+ details.append({"file": name, "error": _safe_reason(exc)})
+ continue
+ except (OSError, ValueError, RecursionError, MemoryError) as exc:
+ # One unreadable/pathological upload must degrade to a per-file error,
+ # not void the whole batch with an unhandled 500.
logger.info("uploaded resource extraction failed (%s)", type(exc).__name__)
errors += 1
- details.append({"file": name, "error": "resource could not be imported"})
+ reason = "upload is locked or unreadable" if isinstance(exc, OSError) \
+ else "upload content could not be processed"
+ details.append({"file": name, "error": reason})
continue
resource_meta = {
**resource.metadata,
@@ -2345,7 +2594,7 @@ def import_files(self, *, workspace: str, files: list, memory_type: str = "seman
self.store.conn.commit()
return {"workspace": ws, "scanned": len(files), "imported": imported,
"skipped": skipped, "errors": errors, "derived_facts": derived_facts,
- "details": details[:50], "warnings": warnings[:50]}
+ "details": details[:200], "warnings": warnings[:200]}
# ── Universal local document import (v2 source manifest) ────────────────
def _document_registered_target(
@@ -2755,6 +3004,7 @@ def get_document_import_job(self, job_id: str, *, workspace: str) -> dict:
clean_id = _clean_text(job_id, field="job_id", max_chars=MAX_NAME_CHARS)
if wid is None:
raise KeyError(clean_id)
+ self._recover_stale_import_jobs()
row = self.store.conn.execute(
"SELECT * FROM jobs WHERE id=? AND workspace_id=? "
"AND kind IN ('document_import','obsidian_import')",
@@ -3222,6 +3472,7 @@ def get_obsidian_import_job(self, job_id: str, *, workspace: str) -> dict:
clean_id = _clean_text(job_id, field="job_id", max_chars=MAX_NAME_CHARS)
if wid is None:
raise KeyError(clean_id)
+ self._recover_stale_import_jobs()
row = self.store.conn.execute(
"SELECT * FROM jobs WHERE id=? AND workspace_id=? AND kind='obsidian_import'",
(clean_id, wid),
@@ -3662,6 +3913,8 @@ def recall(self, query: str, *, workspace: Optional[str] = None,
"embedding_mode": result.embedding_mode,
"degraded_reason": result.degraded_reason,
"vector_search_ready": result.vector_search_ready,
+ "vector_index_backend": _vector_index_backend_label(self.engine.index),
+ "reranker_mode": _reranker_mode_label(self.engine.reranker),
}
out = {
"query": query, "count": result.count,
@@ -7427,6 +7680,60 @@ def _recover_stale_graph_jobs(self, workspace_id: Optional[str] = None) -> int:
raise
return len(rows)
+ def _recover_stale_import_jobs(self) -> int:
+ """Fail document/obsidian import jobs whose worker died with its process.
+
+ Import workers heartbeat per document batch (obsidian_import._update_job_progress),
+ so a stale heartbeat means the process died — daemon threads never survive a
+ restart. Mirrors _recover_stale_graph_jobs' lease semantics so a crashed wizard
+ import reports 'failed: worker_lease_expired' instead of polling as 'running'
+ forever.
+
+ Import jobs use a substantially longer lease (IMPORT_JOB_LEASE_SECONDS) than
+ graph-index workers: the per-batch heartbeat only advances between documents,
+ and a single large file (big PDF, OCR image, transcription) can legitimately
+ spend longer than the graph lease inside one parse/ingest. The heartbeat is
+ the progress-aware expiry marker — it resets the lease age whenever the job's
+ processed-files count advances, so a live mid-file worker is never failed.
+ """
+ now = time.time()
+ cutoff = now - IMPORT_JOB_LEASE_SECONDS
+ where = ("kind IN ('document_import','obsidian_import') "
+ "AND state IN ('queued','running') "
+ "AND COALESCE(heartbeat_at, created_at) None:
for workspace_id in dict.fromkeys(value for value in workspace_ids if value):
self._recover_stale_graph_jobs(workspace_id)
@@ -10801,6 +11108,22 @@ def stats(self, *, workspace: Optional[str] = None) -> dict:
scopes=[Scope.WORKSPACE, Scope.REPO, Scope.USER],
)
eligibility = self.store.prompt_eligibility_counts(eligibility_filter)
+
+ def _table_count(table: str) -> Optional[int]:
+ """Best-effort row count for an internal ledger table (None if unreadable)."""
+ try:
+ return int(
+ conn.execute(f"SELECT COUNT(*) AS n FROM {table}").fetchone()["n"]
+ )
+ except Exception:
+ return None
+
+ # Additive health/observability counts; never part of the memory totals.
+ ledger_counts = {
+ "operation_receipts": _table_count("operation_receipts"),
+ "events": _table_count("events"),
+ "audit": _table_count("audit"),
+ }
embedding = self.store.embedding_space_health(
embedding_space_fingerprint(self.engine.embedder)
)
@@ -10811,6 +11134,7 @@ def stats(self, *, workspace: Optional[str] = None) -> dict:
"schema_version": self.store.schema_version,
"prompt_eligibility": eligibility,
"embedding": embedding,
+ **ledger_counts,
}
def memory_health(self, *, workspace: str) -> dict:
diff --git a/engraphis/static/dashboard.js b/engraphis/static/dashboard.js
index 549110af..b45e0720 100644
--- a/engraphis/static/dashboard.js
+++ b/engraphis/static/dashboard.js
@@ -488,11 +488,11 @@ async function wsCreate(){
function wsSwitch(name){setWS(name);toast('Switched to '+name,'ok');navTo('overview')}
/* import (files/folders from this PC — see MemoryService.import_folder/import_files) */
-async function importUpload(items){if(!canCreateWs()){toast('Viewers can’t import','err');return}if(!WS){toast('Create or select a folder first','err');return}if(!items||!items.length)return;const fd=new FormData();fd.append('workspace',WS);fd.append('memory_type','semantic');fd.append('derive_facts',document.getElementById('import-derive').checked?'true':'false');for(const it of items)fd.append('files',it.file,it.name);const el=document.getElementById('import-status');el.textContent='Extracting and importing '+items.length+' file(s)…';try{const r=await api('/workspaces/import-files',{method:'POST',body:fd});const wc=(r.warnings||[]).length;el.textContent=r.imported+' imported, '+r.skipped+' skipped, '+r.errors+' error(s), '+(r.derived_facts||0)+' derived fact(s)'+(wc?', '+wc+' warning(s)':'');toast(r.imported+' resource'+(r.imported===1?'':'s')+' imported into "'+WS+'"','ok');refreshFolders()}catch(e){el.textContent='';toast('Import failed: '+e.message,'err')}}
+async function importUpload(items){if(!canCreateWs()){toast('Viewers can’t import','err');return}if(!WS){toast('Create or select a folder first','err');return}if(!items||!items.length)return;const fd=new FormData();fd.append('workspace',WS);fd.append('memory_type','semantic');fd.append('derive_facts',document.getElementById('import-derive').checked?'true':'false');for(const it of items)fd.append('files',it.file,it.name);const el=document.getElementById('import-status');el.textContent='Extracting and importing '+items.length+' file(s)…';try{const r=await api('/workspaces/import-files',{method:'POST',body:fd});const wc=(r.warnings||[]).length;el.textContent=r.imported+' imported, '+r.skipped+' skipped, '+r.errors+' error(s), '+(r.derived_facts||0)+' derived fact(s)'+(wc?', '+wc+' warning(s)':'')+(r.truncated?', TRUNCATED at '+r.scanned+' of '+r.matched_total+' matching files':'')+(r.unreadable?', '+r.unreadable+' unreadable':'');toast(r.imported+' resource'+(r.imported===1?'':'s')+' imported into "'+WS+'"'+(r.truncated?' (truncated at max '+r.scanned+')':''),'ok');refreshFolders()}catch(e){el.textContent='';toast('Import failed: '+e.message,'err')}}
function importFilesPicked(fileList,el){const items=Array.from(fileList||[]).map(f=>({file:f,name:f.webkitRelativePath||f.name}));if(el)el.value='';importUpload(items)}
async function importWalkEntry(entry,path,out){if(entry.isFile){await new Promise(res=>entry.file(f=>{out.push({file:f,name:(path?path+'/':'')+f.name});res()},()=>res()))}else if(entry.isDirectory){const reader=entry.createReader();const readBatch=()=>new Promise(res=>reader.readEntries(res,()=>res([])));let batch;do{batch=await readBatch();for(const e of batch)await importWalkEntry(e,(path?path+'/':'')+entry.name,out)}while(batch.length)}}
async function importDrop(e){e.preventDefault();e.currentTarget.classList.remove('drag');const items=e.dataTransfer.items;const out=[];if(items&&items.length&&items[0].webkitGetAsEntry){for(const it of items){const entry=it.webkitGetAsEntry&&it.webkitGetAsEntry();if(entry)await importWalkEntry(entry,'',out)}}else{for(const f of e.dataTransfer.files)out.push({file:f,name:f.name})}importUpload(out)}
-async function importFromPath(){if(!canCreateWs()){toast('Viewers can’t import','err');return}if(!WS){toast('Create or select a folder first','err');return}const path=(document.getElementById('import-path').value||'').trim();const pattern=(document.getElementById('import-pattern').value||'*').trim()||'*';if(!path){toast('Enter a path','err');return}const el=document.getElementById('import-status');el.textContent='Extracting and importing…';try{const r=await api('/workspaces/import-folder',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({workspace:WS,path,file_pattern:pattern,memory_type:'semantic',derive_facts:document.getElementById('import-derive').checked})});const wc=(r.warnings||[]).length;el.textContent=r.imported+' imported, '+r.skipped+' skipped, '+r.errors+' error(s), '+(r.derived_facts||0)+' derived fact(s), scanned '+r.scanned+(wc?', '+wc+' warning(s)':'');toast(r.imported+' resource'+(r.imported===1?'':'s')+' imported into "'+WS+'"','ok');refreshFolders()}catch(e){el.textContent='';toast('Import failed: '+e.message,'err')}}
+async function importFromPath(){if(!canCreateWs()){toast('Viewers can’t import','err');return}if(!WS){toast('Create or select a folder first','err');return}const path=(document.getElementById('import-path').value||'').trim();const pattern=(document.getElementById('import-pattern').value||'*').trim()||'*';if(!path){toast('Enter a path','err');return}const el=document.getElementById('import-status');el.textContent='Extracting and importing…';try{const r=await api('/workspaces/import-folder',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({workspace:WS,path,file_pattern:pattern,memory_type:'semantic',derive_facts:document.getElementById('import-derive').checked})});const wc=(r.warnings||[]).length;el.textContent=r.imported+' imported, '+r.skipped+' skipped, '+r.errors+' error(s), '+(r.derived_facts||0)+' derived fact(s), scanned '+r.scanned+(wc?', '+wc+' warning(s)':'')+(r.truncated?', TRUNCATED at '+r.scanned+' of '+r.matched_total+' matching files':'')+(r.unreadable?', '+r.unreadable+' unreadable':'');toast(r.imported+' resource'+(r.imported===1?'':'s')+' imported into "'+WS+'"'+(r.truncated?' (truncated at max '+r.scanned+')':''),'ok');refreshFolders()}catch(e){el.textContent='';toast('Import failed: '+e.message,'err')}}
async function indexRepository(){if(!WS){toast('Select a workspace first','err');return}const repo=(document.getElementById('code-repo').value||'').trim(),root=(document.getElementById('code-root').value||'').trim(),el=document.getElementById('code-import-status');if(!repo||!root){toast('Enter a repository name and path','err');return}el.textContent='Incrementally indexing repository…';try{const r=await api('/code/index',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({workspace:WS,repo:repo,root_path:root})});el.textContent=`${r.files_indexed} changed, ${r.files_unchanged} unchanged · ${r.symbols} symbols · ${r.edges} edges · ${r.code_memory_links||0} memory links`;toast('Repository graph updated','ok')}catch(e){el.textContent='';toast(e.message,'err')}}
async function importPostgresSchema(){if(!WS){toast('Select a workspace first','err');return}const dsn=(document.getElementById('postgres-dsn').value||'').trim(),repo=(document.getElementById('postgres-repo').value||'').trim(),el=document.getElementById('code-import-status');if(!dsn){toast('Enter a PostgreSQL DSN','err');return}el.textContent='Reading PostgreSQL catalog…';try{const r=await api('/resources/postgres',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({workspace:WS,repo:repo||null,dsn:dsn})});document.getElementById('postgres-dsn').value='';el.textContent=`Imported ${r.schema.tables||0} tables, ${r.entities} entities, and ${r.relations} relations`;toast('Database schema imported','ok')}catch(e){el.textContent='';toast(e.message,'err')}}
async function wsRename(name){const nn=await textAction('Rename workspace','Choose a new name for "'+name+'".','Workspace name',name,{submit:'Rename'});if(nn===null)return;const v=nn.trim();if(!v||v===name)return;try{await api('/workspaces/rename',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({workspace:name,new_name:v})});if(WS===name)setWS(v);toast('Renamed','ok');refreshFolders()}catch(e){toast(e.message,'err')}}
@@ -569,7 +569,7 @@ async function exportWorkspace(){try{const d=await api('/export?workspace='+enco
async function loadTeam(){const el=document.getElementById('team-body'),teamCta=hostedCta('team','team_tab');try{const st=await api('/auth/state');if(teamCta.href==='#'&&st&&st.cloud_url)teamCta.href=safeUrl(st.cloud_url)}catch(e){}el.innerHTML=`
Engraphis Team Cloud HOSTED
Organizations, invitations, roles, named seats, scoped device credentials, and team audit run on the private hosted service. This local dashboard is intentionally single-user.
${esc(teamTeaserNote())} Private-service account grace is capped at 24 hours, never extends Team access, and never restricts the free local core.
`}
/* health + settings */
function connectionContext(){const host=(location.hostname||'').toLowerCase();return host==='localhost'||host==='127.0.0.1'||host==='::1'||host.endsWith('.localhost')?'Local engine':'Remote customer node'}
-async function checkHealth(){const label=connectionContext();try{await api('/health');const d=document.getElementById('health-dot'),t=document.getElementById('health-text');if(d){d.classList.add('health-ok');d.classList.remove('health-error')}if(t)t.textContent=label+' connected'}catch(e){const d=document.getElementById('health-dot'),t=document.getElementById('health-text');if(d){d.classList.add('health-error');d.classList.remove('health-ok')}if(t)t.textContent=label+' unavailable'}try{const auth=await api('/auth/state');const m=document.getElementById('deployment-mode-indicator');if(m){const isLocal=auth.deployment_mode==='local';m.textContent=isLocal?'LOCAL':'HOSTED';m.title=isLocal?'Local mode: no hosted cloud configured':'Hosted mode: connected to Engraphis Cloud';m.className='deployment-mode '+(isLocal?'mode-local':'mode-hosted');m.hidden=false}}catch(e){}}
+async function checkHealth(){const label=connectionContext();try{await api('/health');const d=document.getElementById('health-dot'),t=document.getElementById('health-text');if(d){d.classList.add('health-ok');d.classList.remove('health-error')}if(t)t.textContent=label+' connected'}catch(e){const d=document.getElementById('health-dot'),t=document.getElementById('health-text');if(d){d.classList.add('health-error');d.classList.remove('health-ok')}if(t)t.textContent=label+' unavailable'}try{const auth=await api('/auth/state');const m=document.getElementById('deployment-mode-indicator');if(m){const isLocal=auth.deployment_mode==='local';m.textContent=isLocal?'LOCAL':'HOSTED';m.title=isLocal?'Local mode: no hosted cloud configured':'Hosted mode: connected to Engraphis Cloud';m.className='deployment-mode '+(isLocal?'mode-local':'mode-hosted');m.hidden=false}}catch(e){}}
function loadSettings(){loadLicense();loadSyncStatus();loadHostedAgentAccess();loadLlmStatus();const s=document.getElementById('cfg-store');if(s)s.textContent=location.host;api('/info').then(function(d){var v=document.getElementById('cfg-version');if(v&&d&&d.version)v.textContent=d.version}).catch(function(){})}
async function loadLlmStatus(){const el=document.getElementById('llm-body');if(!el)return;try{const st=await api('/llm/status');const ok=st.configured;const badge=ok?'configured':'not configured';const keyLine=st.key_set?'API key set ✓':'No API key set';let modelSel='';let provSel='';el.innerHTML=`
Provider · Model${badge}
${provSel}${modelSel}
${keyLine} · extractor: ${esc(st.extractor)}
Add this to your .env and restart Engraphis:
LLM extraction${st.extractor_enabled?'ON':'OFF'}
While ON, ingested memory content is sent to your LLM provider for schema-validated extraction. OFF disables extraction transfers only; retention supervision is configured separately.
`}catch(e){el.innerHTML='
'+esc(e.message)+'
'}}
@@ -597,7 +597,7 @@ const syncNowBase=syncNow;
syncNow=async function(){if(!await confirmCloudTransfer('Sync shared workspaces','Cloud Sync sends eligible changes from your shared workspaces to Engraphis Cloud and receives authorized changes from your other installations; secret and session-scoped rows stay local.','Sync now',CLOUD_SYNC_PRIVACY_COPY))return;return syncNowBase()}
/* ─── knowledge graph (force-graph + d3-force: compact defaults and selectable layouts) ─── */
-let GRAPH=null, FG=null, GRAPH_ENGINE=null, GRESIZE=false, GRESIZEFRAME=0, GADJ={}, GCOMM_ADJ={}, GCOMPONENTS={}, GCOMPONENT_LAYOUT=null, GHILITE=null, GHOVERSET=null, GLABELRANK={}, GLABELBOXES=[], GDATA_CACHE=null, GACTIVE_DATA=null, GREDRAWFRAME=0, GPERF={large:false,dense:false}, GRAPH_FULL=false, GRAPH_SCOPE_BEFORE_FULL=null, GRAPH_LOAD_REQUEST=0, GRAPH_LOAD_CONTROLLER=null;
+let GRAPH=null, FG=null, GRAPH_ENGINE=null, GRESIZE=false, GRESIZEFRAME=0, GADJ={}, GCOMM_ADJ={}, GCOMPONENTS={}, GCOMPONENT_LAYOUT=null, GHILITE=null, GHOVERSET=null, GLABELRANK={}, GLABELBOXES=[], GDATA_CACHE=null, GACTIVE_DATA=null, GREDRAWFRAME=0, GPERF={large:false,dense:false}, GRAPH_FULL=false, GRAPH_SCOPE_BEFORE_FULL=null, GRAPH_LOAD_REQUEST=0, GRAPH_LOAD_CONTROLLER=null;
const GRAPH_PRESETS={
original:{label:'Original force',repel:120,link:30,gravity:14,font:13,size:3,linkw:1,labelDensity:40,curve:0,particles:0},
compact:{label:'Compact clusters',repel:42,link:20,gravity:26,font:12,size:3,linkw:.7,labelDensity:30,curve:.08,particles:0},
@@ -733,9 +733,9 @@ function graphRenderEngine(data,fit,reheat){
}
showAs(empty,false);GPERF={large:data.nodes.length>600||data.links.length>2400,dense:data.links.length>1500};
const created=!GRAPH_ENGINE;
- if(created){
- GRAPH_ENGINE=EngraphisGraph.create(element,{
- renderMode:fullGraph?'all':'overview',
+ if(created){
+ GRAPH_ENGINE=EngraphisGraph.create(element,{
+ renderMode:fullGraph?'all':'overview',
reducedMotion:prefersReducedMotion,
onNodeClick:node=>{syncGraphExplorerSelection(node.id);graphNodeClick(node.label||node.name||node.id)},
onBackgroundClick:()=>graphSetHighlight(null),
@@ -753,7 +753,7 @@ function graphRenderEngine(data,fit,reheat){
const isolated=document.getElementById('graph-show-iso'),showUnlinked=fullGraph||!!(isolated&&isolated.checked);
GRAPH_ENGINE.apply(engine=>{
engine.setSettings({...window.GSET});
- if(typeof engine.setRenderMode==='function')engine.setRenderMode(fullGraph?'all':'overview');
+ if(typeof engine.setRenderMode==='function')engine.setRenderMode(fullGraph?'all':'overview');
engine.setStyle(typeof GSTYLE!=='undefined'?GSTYLE:'cyber');
engine.setColorBy(typeof GCOLORBY!=='undefined'?GCOLORBY:'community');
engine.setThemeColors(graphThemeTypeColors());
@@ -775,7 +775,7 @@ function graphRenderEngine(data,fit,reheat){
null. Re-apply the parked state here so a renderer created against a hidden pane never
starts a rAF that nothing will stop. */
if(GRAPH_ENGINE_PARKED)GRAPH_ENGINE.pause();
- graphSetSimulationStatus(fullGraph?'All nodes · settled LOD':(window.GSET.frozen?'Layout frozen':'Adaptive layout'),false);
+ graphSetSimulationStatus(fullGraph?'All nodes · settled LOD':(window.GSET.frozen?'Layout frozen':'Adaptive layout'),false);
return true;
}catch(error){
graphEngineFallback(error);
@@ -795,11 +795,11 @@ function graphInvalidateData(){
if(GRAPH_ENGINE){try{GRAPH_ENGINE.destroy()}catch(e){}GRAPH_ENGINE=null}
GDATA_CACHE=null;GACTIVE_DATA=null;GCOMPONENT_LAYOUT=null;GHILITE=null;GHOVERSET=null
}
-async function loadLegacyGraph(){
- const request=++GRAPH_LOAD_REQUEST,targetFull=GRAPH_FULL;
- const previousController=GRAPH_LOAD_CONTROLLER,controller=new AbortController();GRAPH_LOAD_CONTROLLER=controller;
- if(previousController&&!previousController.signal.aborted)previousController.abort();
- graphInjectCss();graphInvalidateData();GRAPH=null;
+async function loadLegacyGraph(){
+ const request=++GRAPH_LOAD_REQUEST,targetFull=GRAPH_FULL;
+ const previousController=GRAPH_LOAD_CONTROLLER,controller=new AbortController();GRAPH_LOAD_CONTROLLER=controller;
+ if(previousController&&!previousController.signal.aborted)previousController.abort();
+ graphInjectCss();graphInvalidateData();GRAPH=null;
const empty=document.getElementById('graph-empty'),net=document.getElementById('graph-net'),nodesBox=document.getElementById('graph-entity-list'),edgesBox=document.getElementById('graph-relation-list');
showAs(empty,true,'flex');empty.textContent='Loading graph…';graphSetLayoutStatus('Loading data',true);
if(net)net.setAttribute('aria-busy','true');
@@ -811,32 +811,32 @@ async function loadLegacyGraph(){
GRESIZEFRAME=requestAnimationFrame(()=>{GRESIZEFRAME=0;const element=document.getElementById('graph-net');if(GRAPH_ENGINE)GRAPH_ENGINE.resize();else if(FG&&element)FG.width(element.clientWidth).height(element.clientHeight)});
});
}
- const layerInputs=Array.from(document.querySelectorAll('#graph-layer-filters input')),selectedLayers=layerInputs.filter(input=>input.checked).map(input=>input.value),layerFilter=selectedLayers.length===layerInputs.length?'':'&layers='+encodeURIComponent(selectedLayers.join(',')),includeCode=document.getElementById('graph-include-code').checked,repo=(document.getElementById('graph-repo-filter').value||'').trim(),showUnlinked=targetFull||!!document.getElementById('graph-show-iso').checked;
- try{
- const query=encodeURIComponent(WS||'')+(repo?'&repo='+encodeURIComponent(repo):'')+layerFilter;
- let nextGraph;
- if(targetFull){
- /* The complete scene and its dedicated renderer are independent requests. Starting them
- together avoids adding an asset round-trip after a potentially large scene response, and
- awaiting both guarantees that complete data can never fall into the legacy ForceGraph. */
- const [response]=await Promise.all([
- api('/graph/scene?workspace='+query+'&level=complete&presentation=all&include_memory_nodes=false',{signal:controller.signal}),
- loadGraphEngine(true)
- ]);
- const scene=response.scene||response;
- nextGraph={nodes:(scene.nodes||[]).map(node=>({...node,id:node.id,label:node.label||node.name||node.id,degree:node.degree??node.weighted_degree??0,etype:node.etype||'entity'})),edges:(scene.edges||[]).map(edge=>({...edge,from:edge.from??edge.source??edge.src,to:edge.to??edge.target??edge.dst,label:edge.label||edge.relation||'related',layer:edge.layer||'semantic'})),meta:scene.meta||{}};
- }else{
- nextGraph=await api('/graph?workspace='+query+'&include_code='+(includeCode?'true':'false')+'&limit=1000&node_limit=1000&edge_limit=2000'+(showUnlinked?'':'&connected_only=true'),{signal:controller.signal});
- }
- if(request!==GRAPH_LOAD_REQUEST||targetFull!==GRAPH_FULL)return;
- GRAPH=nextGraph;
- renderGraphSide();graphRender();
- }catch(error){
- if(request!==GRAPH_LOAD_REQUEST||error.name==='AbortError')return;
- showAs(empty,true,'flex');empty.textContent='Graph failed: '+error.message;graphSetLayoutStatus('Load failed',false);
- }finally{
- if(request!==GRAPH_LOAD_REQUEST)return;
- if(GRAPH_LOAD_CONTROLLER===controller)GRAPH_LOAD_CONTROLLER=null;
+ const layerInputs=Array.from(document.querySelectorAll('#graph-layer-filters input')),selectedLayers=layerInputs.filter(input=>input.checked).map(input=>input.value),layerFilter=selectedLayers.length===layerInputs.length?'':'&layers='+encodeURIComponent(selectedLayers.join(',')),includeCode=document.getElementById('graph-include-code').checked,repo=(document.getElementById('graph-repo-filter').value||'').trim(),showUnlinked=targetFull||!!document.getElementById('graph-show-iso').checked;
+ try{
+ const query=encodeURIComponent(WS||'')+(repo?'&repo='+encodeURIComponent(repo):'')+layerFilter;
+ let nextGraph;
+ if(targetFull){
+ /* The complete scene and its dedicated renderer are independent requests. Starting them
+ together avoids adding an asset round-trip after a potentially large scene response, and
+ awaiting both guarantees that complete data can never fall into the legacy ForceGraph. */
+ const [response]=await Promise.all([
+ api('/graph/scene?workspace='+query+'&level=complete&presentation=all&include_memory_nodes=false',{signal:controller.signal}),
+ loadGraphEngine(true)
+ ]);
+ const scene=response.scene||response;
+ nextGraph={nodes:(scene.nodes||[]).map(node=>({...node,id:node.id,label:node.label||node.name||node.id,degree:node.degree??node.weighted_degree??0,etype:node.etype||'entity'})),edges:(scene.edges||[]).map(edge=>({...edge,from:edge.from??edge.source??edge.src,to:edge.to??edge.target??edge.dst,label:edge.label||edge.relation||'related',layer:edge.layer||'semantic'})),meta:scene.meta||{}};
+ }else{
+ nextGraph=await api('/graph?workspace='+query+'&include_code='+(includeCode?'true':'false')+'&limit=1000&node_limit=1000&edge_limit=2000'+(showUnlinked?'':'&connected_only=true'),{signal:controller.signal});
+ }
+ if(request!==GRAPH_LOAD_REQUEST||targetFull!==GRAPH_FULL)return;
+ GRAPH=nextGraph;
+ renderGraphSide();graphRender();
+ }catch(error){
+ if(request!==GRAPH_LOAD_REQUEST||error.name==='AbortError')return;
+ showAs(empty,true,'flex');empty.textContent='Graph failed: '+error.message;graphSetLayoutStatus('Load failed',false);
+ }finally{
+ if(request!==GRAPH_LOAD_REQUEST)return;
+ if(GRAPH_LOAD_CONTROLLER===controller)GRAPH_LOAD_CONTROLLER=null;
if(net)net.setAttribute('aria-busy','false');
if(!GRAPH){
if(FG)FG.graphData({nodes:[],links:[]});
@@ -846,27 +846,27 @@ async function loadLegacyGraph(){
}
}
}
-function graphUpdateAllNodesControl(){
- const full=GRAPH_FULL,button=document.getElementById('graph-show-all'),isolated=document.getElementById('graph-show-iso'),includeCode=document.getElementById('graph-include-code');
- if(button){button.textContent=full?'High quality':'Show all nodes';button.setAttribute('aria-pressed',String(full));button.title=full?'Return to the high-quality graph view':'Load every node, including unconnected entities, for this graph view'}
- if(isolated){isolated.disabled=full;isolated.title=full?'All nodes are already visible.':'Show entities that have no relations (unlinked nodes). Hidden by default to keep the graph readable.'}
- if(includeCode){includeCode.disabled=full;includeCode.title=full?'Code overlay is available in High quality mode.':''}
-}
+function graphUpdateAllNodesControl(){
+ const full=GRAPH_FULL,button=document.getElementById('graph-show-all'),isolated=document.getElementById('graph-show-iso'),includeCode=document.getElementById('graph-include-code');
+ if(button){button.textContent=full?'High quality':'Show all nodes';button.setAttribute('aria-pressed',String(full));button.title=full?'Return to the high-quality graph view':'Load every node, including unconnected entities, for this graph view'}
+ if(isolated){isolated.disabled=full;isolated.title=full?'All nodes are already visible.':'Show entities that have no relations (unlinked nodes). Hidden by default to keep the graph readable.'}
+ if(includeCode){includeCode.disabled=full;includeCode.title=full?'Code overlay is available in High quality mode.':''}
+}
function graphToggleAllNodes(){
const isolated=document.getElementById('graph-show-iso');
if(!GRAPH_FULL){GRAPH_SCOPE_BEFORE_FULL={showUnlinked:!!(isolated&&isolated.checked)};GRAPH_FULL=true;if(isolated)isolated.checked=true}
else{GRAPH_FULL=false;if(isolated&&GRAPH_SCOPE_BEFORE_FULL)isolated.checked=GRAPH_SCOPE_BEFORE_FULL.showUnlinked;GRAPH_SCOPE_BEFORE_FULL=null}
graphUpdateAllNodesControl();loadLegacyGraph();
}
-function graphData(){
- const _si=document.getElementById('graph-show-iso');const hideIso=!(_si&&_si.checked);
- if(GDATA_CACHE&&GDATA_CACHE.graph===GRAPH&&GDATA_CACHE.hideIso===hideIso)return GDATA_CACHE.data;
- if(GRAPH_FULL){
- /* The flat all-node worker accepts the scene's node and from/to edge shapes directly.
- Avoid cloning and decorating up to 20k nodes and 200k relations for quality-only paint. */
- const data={nodes:GRAPH.nodes||[],links:GRAPH.edges||[]};GDATA_CACHE={graph:GRAPH,hideIso,data};return data;
- }
- let sourceNodes=GRAPH.nodes;if(hideIso)sourceNodes=sourceNodes.filter(node=>node.degree>0);
+function graphData(){
+ const _si=document.getElementById('graph-show-iso');const hideIso=!(_si&&_si.checked);
+ if(GDATA_CACHE&&GDATA_CACHE.graph===GRAPH&&GDATA_CACHE.hideIso===hideIso)return GDATA_CACHE.data;
+ if(GRAPH_FULL){
+ /* The flat all-node worker accepts the scene's node and from/to edge shapes directly.
+ Avoid cloning and decorating up to 20k nodes and 200k relations for quality-only paint. */
+ const data={nodes:GRAPH.nodes||[],links:GRAPH.edges||[]};GDATA_CACHE={graph:GRAPH,hideIso,data};return data;
+ }
+ let sourceNodes=GRAPH.nodes;if(hideIso)sourceNodes=sourceNodes.filter(node=>node.degree>0);
const names=new Set(sourceNodes.map(node=>node.id));
const nodes=sourceNodes.map(node=>({id:node.id,label:node.label||node.id,displayLabel:(node.label||node.id).length>30?(node.label||node.id).slice(0,29)+'…':(node.label||node.id),etype:node.etype,degree:node.degree||0,val:1+(node.degree||0)}));
const maxDegree=Math.max(1,...nodes.map(node=>node.degree||0));
@@ -1222,53 +1222,53 @@ function loadForceGraph(){
});
return FORCE_GRAPH_LOADING;
}
-let GRAPH_ENGINE_LOADING=null,ALL_GRAPH_ENGINE_LOADING=null;
-function loadAllGraphEngine(){
- if(typeof EngraphisAllGraph!=='undefined')return Promise.resolve();
- if(!ALL_GRAPH_ENGINE_LOADING){
- ALL_GRAPH_ENGINE_LOADING=new Promise((resolve,reject)=>{
- const script=document.createElement('script');script.src='/v2-assets/engraphis-graph-all.js?v=20260814-all-controls-2';
- script.onload=()=>{typeof EngraphisAllGraph==='undefined'?reject(new Error('All-node graph asset loaded without registering EngraphisAllGraph')):resolve()};
- script.onerror=()=>reject(new Error('All-node graph asset could not load'));
- document.head.appendChild(script);
- });
- ALL_GRAPH_ENGINE_LOADING.catch(()=>{});
- }
- return ALL_GRAPH_ENGINE_LOADING;
-}
-function loadGraphEngine(loadAll=false){
- let engineReady;
- if(typeof EngraphisGraph!=='undefined')engineReady=Promise.resolve();
- else{
- if(!GRAPH_ENGINE_LOADING){
- GRAPH_ENGINE_LOADING=new Promise((resolve,reject)=>{
- const script=document.createElement('script');
- script.src='/v2-assets/engraphis-graph.js?v=20260814-galaxy-gravity-3';
- /* A 200 that never registers the global is a corrupt/truncated asset, not a success —
- resolving there would hand graphRenderEngine() an undefined EngraphisGraph. */
- script.onload=()=>{typeof EngraphisGraph==='undefined'?reject(new Error('Graph engine asset loaded without registering EngraphisGraph')):resolve()};
- script.onerror=()=>reject(new Error('Graph engine could not load'));
- document.head.appendChild(script);
- });
- GRAPH_ENGINE_LOADING.catch(()=>{});
- }
- engineReady=GRAPH_ENGINE_LOADING;
- }
- /* Mark the memoized promise handled. graphRender() can start this fetch on a pass that
- returns before attaching its own handler, and an unhandled rejection would print the exact
- console error this lazy-loading exists to remove. Callers still receive the rejection. */
- return loadAll?engineReady.then(()=>loadAllGraphEngine()):engineReady;
-}
+let GRAPH_ENGINE_LOADING=null,ALL_GRAPH_ENGINE_LOADING=null;
+function loadAllGraphEngine(){
+ if(typeof EngraphisAllGraph!=='undefined')return Promise.resolve();
+ if(!ALL_GRAPH_ENGINE_LOADING){
+ ALL_GRAPH_ENGINE_LOADING=new Promise((resolve,reject)=>{
+ const script=document.createElement('script');script.src='/v2-assets/engraphis-graph-all.js?v=20260814-all-controls-2';
+ script.onload=()=>{typeof EngraphisAllGraph==='undefined'?reject(new Error('All-node graph asset loaded without registering EngraphisAllGraph')):resolve()};
+ script.onerror=()=>reject(new Error('All-node graph asset could not load'));
+ document.head.appendChild(script);
+ });
+ ALL_GRAPH_ENGINE_LOADING.catch(()=>{});
+ }
+ return ALL_GRAPH_ENGINE_LOADING;
+}
+function loadGraphEngine(loadAll=false){
+ let engineReady;
+ if(typeof EngraphisGraph!=='undefined')engineReady=Promise.resolve();
+ else{
+ if(!GRAPH_ENGINE_LOADING){
+ GRAPH_ENGINE_LOADING=new Promise((resolve,reject)=>{
+ const script=document.createElement('script');
+ script.src='/v2-assets/engraphis-graph.js?v=20260814-galaxy-gravity-3';
+ /* A 200 that never registers the global is a corrupt/truncated asset, not a success —
+ resolving there would hand graphRenderEngine() an undefined EngraphisGraph. */
+ script.onload=()=>{typeof EngraphisGraph==='undefined'?reject(new Error('Graph engine asset loaded without registering EngraphisGraph')):resolve()};
+ script.onerror=()=>reject(new Error('Graph engine could not load'));
+ document.head.appendChild(script);
+ });
+ GRAPH_ENGINE_LOADING.catch(()=>{});
+ }
+ engineReady=GRAPH_ENGINE_LOADING;
+ }
+ /* Mark the memoized promise handled. graphRender() can start this fetch on a pass that
+ returns before attaching its own handler, and an unhandled rejection would print the exact
+ console error this lazy-loading exists to remove. Callers still receive the rejection. */
+ return loadAll?engineReady.then(()=>loadAllGraphEngine()):engineReady;
+}
function graphRender(fit=true,reheat=true){
const empty=document.getElementById('graph-empty');
const graphFull=typeof GRAPH_FULL!=='undefined'&&GRAPH_FULL;
/* Kick the opt-in engine off alongside the vendor bundle instead of after it, so a
`?graph-engine=next` deep link costs one round trip rather than two. */
- const engineMissing=typeof EngraphisGraph==='undefined'||(graphFull&&typeof EngraphisAllGraph==='undefined');
- /* All mode owns a dedicated bounded renderer and must remain available after a quality-renderer
- runtime failure. The quality failure latch only authorizes the small legacy overview. */
- const enginePending=(graphFull||(!GRAPH_ENGINE_FAILED&&graphEngineEnabled()))&&engineMissing?loadGraphEngine(graphFull):null;
- if(!graphFull&&typeof ForceGraph==='undefined'){
+ const engineMissing=typeof EngraphisGraph==='undefined'||(graphFull&&typeof EngraphisAllGraph==='undefined');
+ /* All mode owns a dedicated bounded renderer and must remain available after a quality-renderer
+ runtime failure. The quality failure latch only authorizes the small legacy overview. */
+ const enginePending=(graphFull||(!GRAPH_ENGINE_FAILED&&graphEngineEnabled()))&&engineMissing?loadGraphEngine(graphFull):null;
+ if(!graphFull&&typeof ForceGraph==='undefined'){
showAs(empty,true,'flex');empty.textContent='Loading graph engine…';
graphSetLayoutStatus('Loading engine',true);
loadForceGraph().then(()=>graphRender(fit,reheat)).catch(error=>{
@@ -1284,14 +1284,14 @@ function graphRender(fit=true,reheat=true){
someone who explicitly asked for next. Only a real load failure degrades, and it is
announced through graphEngineFallback() rather than silent. */
showAs(empty,true,'flex');empty.textContent='Loading graph engine…';
- graphSetLayoutStatus('Loading engine',true);
- enginePending.then(()=>graphRender(fit,reheat)).catch(error=>{
- if(graphFull){
- empty.textContent=error.message+'; return to High quality or reload the dashboard assets.';
- graphSetLayoutStatus('All-node engine unavailable',false);
- return;
- }
- /* Latches GRAPH_ENGINE_FAILED, so the re-entry below takes the classic path and this
+ graphSetLayoutStatus('Loading engine',true);
+ enginePending.then(()=>graphRender(fit,reheat)).catch(error=>{
+ if(graphFull){
+ empty.textContent=error.message+'; return to High quality or reload the dashboard assets.';
+ graphSetLayoutStatus('All-node engine unavailable',false);
+ return;
+ }
+ /* Latches GRAPH_ENGINE_FAILED, so the re-entry below takes the classic path and this
cannot loop. */
graphEngineFallback(error);
graphRender(fit,reheat);
@@ -1299,14 +1299,14 @@ function graphRender(fit=true,reheat=true){
return;
}
const element=document.getElementById('graph-net'),settings=window.GSET,mode=GRAPH_PRESETS[settings.mode]||GRAPH_PRESETS.compact,data=graphData();
- if(graphFull){
- if(graphRenderEngine(data,fit,reheat))return;
- showAs(empty,true,'flex');
- empty.textContent='All-node renderer unavailable; return to High quality or reload the dashboard assets.';
- graphSetLayoutStatus('All-node engine unavailable',false);
- return;
- }
- if(graphEngineEnabled()&&graphRenderEngine(data,fit,reheat))return;
+ if(graphFull){
+ if(graphRenderEngine(data,fit,reheat))return;
+ showAs(empty,true,'flex');
+ empty.textContent='All-node renderer unavailable; return to High quality or reload the dashboard assets.';
+ graphSetLayoutStatus('All-node engine unavailable',false);
+ return;
+ }
+ if(graphEngineEnabled()&&graphRenderEngine(data,fit,reheat))return;
/* Read AFTER the opt-in attempt: a failing engine resets GACTIVE_DATA precisely so the
classic renderer below rebuilds from scratch instead of assuming the canvas is current. */
const dataChanged=GACTIVE_DATA!==data;
@@ -1510,14 +1510,14 @@ function graphSearch(){
function closeEntityMems(){document.getElementById('mm-overlay').classList.remove('show')}
async function graphNodeClick(name){const ov=document.getElementById('mm-overlay');ov.classList.add('show');document.getElementById('mm-title').textContent=name;document.getElementById('mm-meta').innerHTML='entity';document.getElementById('mm-body').innerHTML='';document.getElementById('mm-actions').innerHTML='';try{const d=await api('/memories?q='+encodeURIComponent(name)+'&workspace='+encodeURIComponent(WS||'')+'&limit=12');document.getElementById('mm-body').innerHTML=d.memories.length?('
- 
-