feat: bundle retrieval models into an offline wheel for air-gapped installs

Chetic · claude · Chetic · commit 3ec97e4fe30d · 2026-06-04T19:12:04.000+02:00
Ship the embedding (BAAI/bge-small-en-v1.5) and rerank
(ms-marco-MiniLM-L-12-v2) models inside the package so a pip-installed
wheel runs with no network access. cfgload transparently falls back to
the bundled models (offline mode) when the configured cache lacks them,
while still respecting a user-provided cache that already has them.

The model-bundled wheel (~260 MB) exceeds public PyPI's 100 MB/file
limit, so it is built in a dedicated release job and attached to the
GitHub Release for download into a self-hosted index (Artifactory); the
lean wheel continues to publish to PyPI unchanged. The offline wheel is
built against chunksilo-2.3.3-linux-x86_64.lock (pip --no-deps) so the
dependency closure is identical to v2.3.3 with no upgrades.

- cfgload: _resolve_bundled_models() picks the packaged cache + offline
- pyproject: package-data includes _bundled_models, excludes HF blobs/
- manual-release: build_offline_wheel job downloads/dereferences models
- add v2.3.3 fully-pinned transitive lock; gitignore the bundle dir

Co-Authored-By: Claude Opus 4.8 (1M context) &lt;noreply@anthropic.com&gt;
diff --git a/.github/workflows/manual-release.yml b/.github/workflows/manual-release.yml
@@ -251,6 +251,59 @@ jobs:
             packages/*.tar.gz
 
 
+  # Offline wheel: the same chunksilo wheel with the embedding + rerank models
+  # bundled in (~260 MB). Too large for public PyPI's 100 MB/file limit, so it is
+  # NEVER published there — it is attached to the GitHub Release for download into
+  # an air-gapped, self-hosted index (Artifactory). Installs with zero network.
+  build_offline_wheel:
+    needs: [prepare, release]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Set package version
+        run: |
+          VERSION="${{ needs.prepare.outputs.version }}"
+          sed -i "s/^version = .*/version = \"$VERSION\"/" pyproject.toml
+          sed -i "s/^__version__ = .*/__version__ = \"$VERSION\"/" src/chunksilo/__init__.py
+
+      - name: Bundle retrieval models into the wheel
+        env:
+          # Prefer real files in snapshots/ over symlinks into blobs/ so the models
+          # survive being packed into a wheel (a zip cannot carry symlinks).
+          HF_HUB_DISABLE_SYMLINKS: "1"
+        run: |
+          # Install the exact dependency set the release used; no upgrades.
+          pip install --no-deps -r chunksilo-2.3.3-linux-x86_64.lock
+          BUNDLE=src/chunksilo/_bundled_models
+          PYTHONPATH=src python -m chunksilo.index --download-models \
+            --model-cache-dir "$BUNDLE"
+          # Belt-and-suspenders: materialize any remaining symlinks into real files
+          # (independent of the hf_hub version honoring HF_HUB_DISABLE_SYMLINKS),
+          # then drop the now-redundant blob store. exclude-package-data also keeps
+          # blobs/ out of the wheel in case anything is left behind.
+          find "$BUNDLE" -type l | while read -r link; do
+            target=$(readlink -f "$link"); rm "$link"; cp "$target" "$link"
+          done
+          find "$BUNDLE" -type d -name blobs -prune -exec rm -rf {} +
+
+      - name: Build offline wheel
+        run: |
+          pip install --no-deps build
+          python -m build --wheel
+          ls -lh dist/
+
+      - name: Attach offline wheel to the GitHub Release
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set -eo pipefail
+          gh release upload "v${{ needs.prepare.outputs.version }}" dist/*.whl --clobber
+
   publish_pypi:
     needs: [prepare, release]
     runs-on: ubuntu-latest
diff --git a/.gitignore b/.gitignore
@@ -29,6 +29,8 @@ wheels/
 data/
 storage/
 models/
+# Retrieval models bundled into the wheel at build time (never committed)
+src/chunksilo/_bundled_models/
 
 # Test artifacts (generated by test suite)
 test_data/
diff --git a/chunksilo-2.3.3-linux-x86_64.lock b/chunksilo-2.3.3-linux-x86_64.lock
@@ -0,0 +1,157 @@
+# Fully-pinned transitive dependency lock for chunksilo 2.3.3
+#
+# Derived from the exact wheels/sdists bundled in the official release:
+#   asset:    chunksilo-2.3.3-manylinux_2_34_x86_64.tar.gz  (dependencies/)
+#   digest:   sha256:6d00e7b2a216932e6032ab7d4dae8e901a34cd819161135256520d2d7aa0fd2d
+#   platform: manylinux_2_34 x86_64 / CPython 3.11
+#   includes: base deps + 'confluence' and 'jira' extras (as the release bundles them)
+#
+# Reproduce the release's exact dependency set:
+#   pip install --no-deps -r chunksilo-2.3.3-linux-x86_64.lock
+#
+# Notes:
+#  - No --hash lines: versions come from bundled wheel filenames, not a hashed index.
+#  - Linux/x86_64/py3.11 closure: some pins are platform-specific and won't
+#    resolve on macOS/Windows or other Python versions.
+#  - chunksilo==2.3.3 itself is the app, not listed as a dependency.
+#
+aiohappyeyeballs==2.6.1
+aiohttp==3.13.2
+aiosignal==1.4.0
+aiosqlite==0.21.0
+annotated-types==0.7.0
+anyio==4.12.0
+atlassian-python-api==4.0.7
+attrs==25.4.0
+banks==2.2.0
+beautifulsoup4==4.14.3
+bm25s==0.2.14
+certifi==2025.11.12
+cffi==2.0.0
+charset-normalizer==3.4.4
+click==8.3.1
+colorama==0.4.6
+coloredlogs==15.0.1
+cryptography==46.0.3
+cssselect2==0.8.0
+dataclasses-json==0.6.7
+defusedxml==0.7.1
+Deprecated==1.2.18
+dirtyjson==1.0.8
+distro==1.9.0
+docx2txt==0.8
+fastembed==0.7.4
+filelock==3.20.0
+filetype==1.2.0
+FlashRank==0.2.10
+flatbuffers==25.9.23
+frozenlist==1.8.0
+fsspec==2025.12.0
+greenlet==3.3.0
+griffe==1.15.0
+h11==0.16.0
+hf-xet==1.2.0
+httpcore==1.0.9
+httpx-sse==0.4.3
+httpx==0.28.1
+huggingface-hub==1.1.7
+humanfriendly==10.0
+idna==3.11
+jinja2==3.1.6
+jira==3.10.5
+jiter==0.12.0
+jmespath==1.0.1
+joblib==1.5.2
+jsonschema-specifications==2025.9.1
+jsonschema==4.25.1
+llama-cloud-services==0.6.54
+llama-cloud==0.1.35
+llama-index-cli==0.5.3
+llama-index-core==0.14.10
+llama-index-embeddings-fastembed==0.5.0
+llama-index-embeddings-openai==0.5.1
+llama-index-indices-managed-llama-cloud==0.9.4
+llama-index-instrumentation==0.4.2
+llama-index-llms-openai==0.6.10
+llama-index-readers-confluence==0.6.0
+llama-index-readers-file==0.5.5
+llama-index-readers-llama-parse==0.5.1
+llama-index-retrievers-bm25==0.6.5
+llama-index-workflows==2.11.5
+llama-index==0.14.10
+llama-parse==0.6.54
+loguru==0.7.3
+lxml==6.0.2
+markdownify==1.2.2
+markupsafe==3.0.3
+marshmallow==3.26.1
+mcp==1.23.1
+mmh3==5.2.0
+mpmath==1.3.0
+multidict==6.7.0
+mypy-extensions==1.1.0
+nest-asyncio==1.6.0
+networkx==3.6
+nltk==3.9.2
+numpy==2.3.5
+oauthlib==3.3.1
+onnxruntime==1.23.2
+openai==2.9.0
+packaging==25.0
+pandas==2.2.3
+pdf2image==1.17.0
+pillow==10.3.0
+platformdirs==4.5.0
+propcache==0.4.1
+protobuf==6.33.1
+py-rust-stemmers==0.1.5
+pycparser==2.23
+pydantic-core==2.41.5
+pydantic-settings==2.12.0
+pydantic==2.12.5
+PyJWT==2.10.1
+pypdf==6.4.0
+PyStemmer==2.2.0.3
+pytesseract==0.3.13
+python-dateutil==2.9.0.post0
+python-docx==1.2.0
+python-dotenv==1.2.1
+python-multipart==0.0.20
+pytz==2025.2
+pyyaml==6.0.3
+referencing==0.37.0
+regex==2025.11.3
+reportlab==4.4.9
+requests-oauthlib==2.0.0
+requests-toolbelt==1.0.0
+requests==2.32.5
+retrying==1.4.2
+rpds-py==0.30.0
+scipy==1.17.1
+setuptools==82.0.0
+shellingham==1.5.4
+six==1.17.0
+sniffio==1.3.1
+soupsieve==2.8
+sqlalchemy==2.0.44
+sse-starlette==3.0.3
+starlette==0.50.0
+striprtf==0.0.26
+svglib==1.5.1
+sympy==1.14.0
+tenacity==9.1.2
+tiktoken==0.12.0
+tinycss2==1.5.1
+tokenizers==0.22.1
+tqdm==4.67.1
+typer-slim==0.20.0
+typing-extensions==4.15.0
+typing-inspect==0.9.0
+typing-inspection==0.4.2
+tzdata==2025.2
+urllib3==2.5.0
+uvicorn==0.38.0
+webencodings==0.5.1
+wrapt==1.17.3
+xlrd==2.0.2
+yarl==1.22.0
diff --git a/pyproject.toml b/pyproject.toml
@@ -43,6 +43,16 @@ dependencies = {file = ["requirements.txt"]}
 [tool.setuptools.packages.find]
 where = ["src"]
 
+# Bundle the retrieval models into the wheel so air-gapped installs work with no
+# network access. Populated at build time (see the publish job / prepare-common.sh).
+# The HuggingFace `blobs/` copies are excluded: with HF_HUB_DISABLE_SYMLINKS=1 the
+# `snapshots/` tree holds real files, so blobs would only duplicate the bytes.
+[tool.setuptools.package-data]
+chunksilo = ["_bundled_models/**/*"]
+
+[tool.setuptools.exclude-package-data]
+chunksilo = ["_bundled_models/**/blobs/**"]
+
 [tool.ruff]
 target-version = "py311"
 line-length = 120
diff --git a/src/chunksilo/cfgload.py b/src/chunksilo/cfgload.py
@@ -5,6 +5,7 @@
 
 Loads configuration from config.yaml, searching in standard locations.
 """
+import copy
 import logging
 import os
 from pathlib import Path
@@ -14,6 +15,10 @@
 
 logger = logging.getLogger(__name__)
 
+# Retrieval models bundled inside the installed package (populated at wheel-build
+# time). Present in air-gapped wheel installs; absent in plain source checkouts.
+_BUNDLED_MODELS_DIR = Path(__file__).resolve().parent / "_bundled_models"
+
 
 def _find_config() -> Path:
     """Find config.yaml using a priority-based search.
@@ -137,6 +142,34 @@ def _deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any
     return result
 
 
+def _resolve_bundled_models(config: dict[str, Any]) -> dict[str, Any]:
+    """Transparently fall back to models bundled in the package.
+
+    If the wheel ships the retrieval models and the configured cache directory
+    does not already contain them, point the cache at the bundled copy and enable
+    offline mode. This lets a pip-installed wheel run in an air-gapped environment
+    with zero extra configuration. A user-configured cache that already holds the
+    models always wins, so explicit setups are never overridden.
+    """
+    embed_name = config["retrieval"]["embed_model_name"]
+    hf_dirname = f"models--{embed_name.replace('/', '--')}"
+
+    if not (_BUNDLED_MODELS_DIR / hf_dirname).exists():
+        return config  # No bundled models for this embedding model.
+
+    configured = Path(config["storage"]["model_cache_dir"]).expanduser()
+    if (configured / hf_dirname).exists():
+        return config  # Configured cache already has the models; respect it.
+
+    result = copy.deepcopy(config)
+    result["storage"]["model_cache_dir"] = str(_BUNDLED_MODELS_DIR)
+    result["retrieval"]["offline"] = True
+    logger.info(
+        "Using bundled retrieval models at %s (offline mode)", _BUNDLED_MODELS_DIR
+    )
+    return result
+
+
 def load_config(config_path: Path | None = None) -> dict[str, Any]:
     """Load configuration from YAML file with defaults.
 
@@ -157,14 +190,15 @@ def load_config(config_path: Path | None = None) -> dict[str, Any]:
 
     if not path.exists():
         logger.info("Config file not found at %s; using built-in defaults", path)
-        return _DEFAULTS.copy()
+        return _resolve_bundled_models(_DEFAULTS.copy())
 
     logger.info("Using config: %s", path)
 
     with open(path, encoding="utf-8") as f:
         user_config = yaml.safe_load(f) or {}
 
     result = _deep_merge(_DEFAULTS, user_config)
+    result = _resolve_bundled_models(result)
 
     # Cache result only for default path
     if config_path is None:
