Add fastwarc-grpc, a streaming gRPC server for FastWARC #247
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build Packages | |
| on: | |
| push: | |
| tags: | |
| - '*' | |
| branches: | |
| - develop | |
| - ci/* | |
| pull_request: | |
| types: | |
| - opened | |
| - synchronize | |
| branches: | |
| - develop | |
| env: | |
| PYTHON_VERSION: "3.14" | |
| FORCE_COLOR: "1" | |
| MACOSX_DEPLOYMENT_TARGET: "12.0" | |
| jobs: | |
| build-rust: | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-latest ] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| - name: Cache Cargo | |
| if: "!startsWith(github.ref, 'refs/tags/v')" | |
| uses: actions/cache@v5 | |
| with: | |
| path: | | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} | |
| # Build only FastWARC at this time. Resiliparse needs additional vcpkg dependencies. | |
| # TODO: Fix this! | |
| - name: Build | |
| run: cargo build --verbose --color always --package fastwarc | |
| - name: Run tests | |
| run: cargo test --verbose --color always --package fastwarc | |
| # fastwarc-grpc needs protoc for stub generation, so it builds in its own job. | |
| build-fastwarc-grpc: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| - name: Cache Cargo | |
| if: "!startsWith(github.ref, 'refs/tags/v')" | |
| uses: actions/cache@v5 | |
| with: | |
| path: | | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| key: ${{ runner.os }}-cargo-grpc-${{ hashFiles('**/Cargo.lock') }} | |
| - name: Install protoc | |
| run: sudo apt-get update && sudo apt-get install -y protobuf-compiler | |
| - name: Build | |
| run: cargo build --verbose --color always --package fastwarc-grpc | |
| - name: Run tests | |
| run: cargo test --verbose --color always --package fastwarc-grpc | |
| build-wheels: | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-latest ] | |
| env: | |
| CIBW_SKIP: "*cp38* *cp39* *-musllinux*" | |
| CIBW_TEST_SKIP: "*-macosx_x86_64" # x86_64 tests are flaky on arm64 | |
| CIBW_ARCHS_MACOS: "x86_64 arm64" | |
| CIBW_MANYLINUX_X86_64_IMAGE: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_x86_64:latest | |
| CIBW_MANYLINUX_AARCH64_IMAGE: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_aarch64:latest | |
| CIBW_REPAIR_WHEEL_COMMAND_MACOS: >- | |
| DYLD_LIBRARY_PATH=$(pwd)/vcpkg_installed/$(echo {delocate_archs} | sed s/x86_64/x64/)-osx/lib | |
| delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel} | |
| CIBW_BEFORE_BUILD_WINDOWS: "python -m pip install delvewheel" | |
| CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: >- | |
| delvewheel repair --add-path vcpkg_installed\x64-windows\bin -w {dest_dir} {wheel} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| - name: Cache Vcpkg | |
| uses: actions/cache@v5 | |
| id: cache-vcpkg | |
| if: "!startsWith(github.ref, 'refs/tags/v') && (runner.os == 'macOS' || runner.os == 'Windows')" | |
| with: | |
| path: | | |
| ~/.cache/vcpkg/archives | |
| ~/AppData/Local/vcpkg/archives | |
| key: ${{ runner.os }}-vcpkg-${{ hashFiles('**/vcpkg.json', '**/vcpkg-lock.json') }} | |
| restore-keys: | | |
| ${{ runner.os }}-vcpkg- | |
| - name: Install Vcpkg Dependencies | |
| if: runner.os == 'macOS' || runner.os == 'Windows' | |
| shell: bash | |
| run: | | |
| set -xe | |
| # Windows | |
| if [ -d /c/vcpkg ]; then | |
| vcpkg install --triplet=x64-windows | |
| # MacOS | |
| elif [ "$(uname -s)" == "Darwin" ]; then | |
| git clone https://github.com/Microsoft/vcpkg | |
| ./vcpkg/bootstrap-vcpkg.sh | |
| # Use to two install roots and merge manually, otherwise the second install clears out the previous | |
| ./vcpkg/vcpkg install --triplet=x64-osx | |
| ./vcpkg/vcpkg install --triplet=arm64-osx --x-install-root vcpkg_installed2 | |
| cp -a vcpkg_installed2/* vcpkg_installed/ | |
| rm -r vcpkg_installed2 | |
| else | |
| echo "Unsupported platform." >&2 | |
| exit 1 | |
| fi | |
| - name: Install Rust Cross-compilation Targets | |
| if: runner.os == 'macOS' | |
| shell: bash | |
| run: | | |
| rustup target add aarch64-apple-darwin | |
| rustup target add x86_64-apple-darwin | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Build FastWARC | |
| uses: pypa/cibuildwheel@v3.4 | |
| with: | |
| package-dir: fastwarc-py | |
| output-dir: wheelhouse | |
| env: | |
| CIBW_TEST_COMMAND: python -m pytest --capture=sys --verbose {project}/fastwarc-py/fastwarc-pytest | |
| - name: Build Resiliparse | |
| uses: pypa/cibuildwheel@v3.4 | |
| with: | |
| package-dir: resiliparse-py | |
| output-dir: wheelhouse | |
| env: | |
| CIBW_BEFORE_TEST: >- | |
| python -c "import glob, platform, re, sysconfig; open('fastwarc.txt', 'w').write(glob.glob('wheelhouse/fastwarc-*-cp*-cp' + re.match(r'^.*(?:cpython|cp)-?(3\d+t?)', sysconfig.get_config_var('EXT_SUFFIX'))[1] + '-*_' + platform.machine().lower() + '.whl')[0])" && | |
| python -m pip install -r fastwarc.txt | |
| CIBW_TEST_COMMAND: python -m pytest --capture=sys --verbose {project}/tests/resiliparse | |
| - name: Upload Wheels | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: wheels-${{ matrix.os }} | |
| path: ./wheelhouse/*.whl | |
| check-mypy: | |
| runs-on: ubuntu-latest | |
| container: | |
| image: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_x86_64:latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| - name: Install mypy | |
| run: python${PYTHON_VERSION} -m pip install mypy | |
| - name: Build FastWARC | |
| run: python${PYTHON_VERSION} -m pip install -e "fastwarc-py[all,test]" | |
| - name: Check stubs | |
| run: | | |
| set -ex | |
| cd fastwarc-py | |
| python${PYTHON_VERSION} -m mypy.stubtest fastwarc --mypy-config-file pyproject.toml --allowlist=stubtest-allowlist.txt | |
| build-sdist: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install Build Module | |
| run: python -m pip install build | |
| - name: Build FastWARC Source Dist | |
| run: python -m build --sdist --outdir dist fastwarc-py | |
| - name: Build Resiliparse Source Dist | |
| run: python -m build --sdist --outdir dist resiliparse-py | |
| - name: Upload Source Dists | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: sdist | |
| path: ./dist/*.tar.gz | |
| build-asan: | |
| runs-on: ubuntu-latest | |
| container: | |
| image: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_x86_64:latest | |
| env: | |
| DEBUG: "1" | |
| SETUPTOOLS_RUST_CARGO_PROFILE: "dev" | |
| ASAN: "1" | |
| ASAN_OPTIONS: leak_check_at_exit=0 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Build Extensions | |
| run: | | |
| set -ex | |
| python${PYTHON_VERSION} -m pip install -e "fastwarc-py[all,test]" | |
| python${PYTHON_VERSION} -m pip install -e "resiliparse-py[all,test]" | |
| - name: Run Tests | |
| run: | | |
| export LD_PRELOAD="$(ldconfig -p | grep libasan | head -n1 | awk '{print $4}')" | |
| python${PYTHON_VERSION} -m pytest --capture=sys --verbose tests fastwarc-py/fastwarc-pytest | |
| build-coverage: | |
| runs-on: ubuntu-latest | |
| container: | |
| image: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_x86_64:latest | |
| env: | |
| TRACE: "1" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Build Extensions | |
| shell: bash | |
| run: | | |
| set -ex | |
| # Set up llvm-cov env | |
| source <(cargo llvm-cov show-env --sh) | |
| # Build wheels | |
| python${PYTHON_VERSION} -m pip install cython setuptools setuptools-rust tomli-w codecov-cli | |
| python${PYTHON_VERSION} -m pip install --no-build-isolation -e "fastwarc-py[all,test]" | |
| python${PYTHON_VERSION} -m pip install --no-build-isolation -e "resiliparse-py[all,test]" | |
| - name: Run Tests | |
| shell: bash | |
| run: | | |
| set -ex | |
| # Set up llvm-cov env | |
| source <(cargo llvm-cov show-env --sh) | |
| # Run FastWARC tests | |
| # TODO: Run Resiliparse as well | |
| cargo test --package fastwarc | |
| # Run Python tests | |
| python${PYTHON_VERSION} -m pytest --cov \ | |
| --cov-report xml:pytest-coverage.xml \ | |
| --junitxml=pytest-report.junit.xml \ | |
| tests \ | |
| fastwarc-py/fastwarc-pytest | |
| # Generate Rust coverage report | |
| cargo llvm-cov report --lcov --output-path cargo-coverage.info | |
| # We cannot use the Codecov GitHub action due to incompatible Glibc versions | |
| - name: Upload to Codecov | |
| shell: bash | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| TOKENLESS: ${{ github.event.pull_request.head.label }} | |
| run: | | |
| set -ex | |
| PR="" | |
| if [ -n '${{ github.event.pull_request.head.sha }}' ]; then | |
| PR="--pr ${{ github.event.number }}" | |
| fi | |
| # Upload test result report | |
| python${PYTHON_VERSION} -m codecov_cli.main do-upload \ | |
| --file pytest-report.junit.xml \ | |
| --disable-search \ | |
| --report-type test_results \ | |
| --sha ${{ github.event.pull_request.head.sha || github.sha }} \ | |
| --branch ${{ github.event.pull_request.head.label || github.ref_name }} \ | |
| $PR \ | |
| --slug ${{ github.repository }} \ | |
| --git-service github \ | |
| --fail-on-error | |
| # Upload coverage | |
| python${PYTHON_VERSION} -m codecov_cli.main upload-process \ | |
| --file pytest-coverage.xml \ | |
| --file cargo-coverage.info \ | |
| --disable-search \ | |
| --report-type coverage \ | |
| --sha ${{ github.event.pull_request.head.sha || github.sha }} \ | |
| --branch ${{ github.event.pull_request.head.label || github.ref_name }} \ | |
| $PR \ | |
| --slug ${{ github.repository }} \ | |
| --git-service github \ | |
| --fail-on-error | |
| - name: Upload Artifacts | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: coverage | |
| path: | | |
| pytest-report.junit.xml | |
| pytest-coverage.xml | |
| cargo-coverage.info | |
| build-rust-docs: | |
| runs-on: ubuntu-latest | |
| needs: build-rust | |
| container: | |
| image: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_x86_64:latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| - name: Build Rust Documentation | |
| shell: bash | |
| run: | | |
| set -ex | |
| RUSTDOCFLAGS="-D warnings" cargo doc --package fastwarc --no-deps | |
| build-python-docs: | |
| runs-on: ubuntu-latest | |
| needs: [ build-wheels, build-asan, build-sdist, check-mypy, build-rust ] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| - name: Download Artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: wheels-ubuntu-latest | |
| path: wheelhouse | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Build Python Documentation | |
| shell: bash | |
| run: | | |
| set -ex | |
| python${PYTHON_VERSION} -m pip install -r <(grep -vE "fastwarc|resiliparse" docs/requirements.txt) | |
| PYTHON_ABI="$(python${PYTHON_VERSION} -c 'from packaging.tags import sys_tags; print(next(iter(sys_tags())).interpreter)')" | |
| find wheelhouse -name "fastwarc-*-cp*-${PYTHON_ABI}-manylinux*_x86_64.whl" | xargs -I% python${PYTHON_VERSION} -m pip install "%[all]" | |
| python${PYTHON_VERSION} -c 'import fastwarc' 2> /dev/null || { echo 'No FastWARC wheel matched architecture' >&2; exit 1; } | |
| find wheelhouse -name "resiliparse-*-cp*-${PYTHON_ABI}-manylinux*_x86_64.whl" | xargs -I% python${PYTHON_VERSION} -m pip install "%[all]" | |
| python${PYTHON_VERSION} -c 'import resiliparse' 2> /dev/null || { echo 'No Resiliparse wheel matched architecture' >&2; exit 1; } | |
| cd docs | |
| make html | |
| - name: Wait | |
| if: github.ref == 'refs/heads/develop' | |
| run: sleep 120 | |
| - name: Trigger Readthedocs Build (Latest) | |
| if: github.ref == 'refs/heads/develop' | |
| uses: dfm/rtds-action@v1 | |
| with: | |
| webhook_url: ${{ secrets.RTDS_WEBHOOK_URL }} | |
| webhook_token: ${{ secrets.RTDS_WEBHOOK_TOKEN }} | |
| commit_ref: ${{ github.ref }} | |
| publish-crates: | |
| runs-on: ubuntu-latest | |
| if: startsWith(github.ref, 'refs/tags/v') | |
| container: | |
| image: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_x86_64:latest | |
| needs: [ build-rust-docs, build-python-docs, build-coverage ] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| - name: Publish to Crates.io | |
| env: | |
| CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} | |
| run: > | |
| cargo publish | |
| --verbose | |
| --locked | |
| --no-verify | |
| --workspace | |
| publish-wheels: | |
| runs-on: ubuntu-latest | |
| if: startsWith(github.ref, 'refs/tags/v') | |
| container: | |
| image: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_x86_64:latest | |
| needs: [ build-rust-docs, build-python-docs, build-coverage ] | |
| steps: | |
| - name: Download Wheels | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: wheels-* | |
| merge-multiple: true | |
| - name: Download Source Dist | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: sdist | |
| - name: Publish to PyPi | |
| env: | |
| TWINE_USERNAME: __token__ | |
| TWINE_PASSWORD1: ${{ secrets.PYPI_FASTWARC_API_TOKEN }} | |
| TWINE_PASSWORD2: ${{ secrets.PYPI_RESILIPARSE_API_TOKEN }} | |
| run: | | |
| set -ex | |
| python${PYTHON_VERSION} -m pip install --upgrade twine packaging | |
| TWINE_PASSWORD="$TWINE_PASSWORD1" python${PYTHON_VERSION} -m twine upload fastwarc-*.whl fastwarc-*.tar.gz | |
| TWINE_PASSWORD="$TWINE_PASSWORD2" python${PYTHON_VERSION} -m twine upload resiliparse-*.whl resiliparse-*.tar.gz | |
| - name: Wait | |
| run: sleep 120 | |
| - name: Trigger Readthedocs Build (Stable) | |
| uses: dfm/rtds-action@v1 | |
| with: | |
| webhook_url: ${{ secrets.RTDS_WEBHOOK_URL }} | |
| webhook_token: ${{ secrets.RTDS_WEBHOOK_TOKEN }} | |
| commit_ref: ${{ github.ref }} |