fix(master): stop MergeWal starving Compact in the scheduler (#239) #470
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Rust Tests | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| paths: | |
| - crates/** | |
| - .github/workflows/rust-test.yml | |
| - Cargo.toml | |
| - Cargo.lock | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| CARGO_TERM_COLOR: always | |
| RUSTFLAGS: "-C debuginfo=1" | |
| RUST_BACKTRACE: "1" | |
| CARGO_INCREMENTAL: "0" | |
| jobs: | |
| test: | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 30 | |
| strategy: | |
| matrix: | |
| toolchain: | |
| - stable | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup rust toolchain | |
| run: | | |
| rustup toolchain install ${{ matrix.toolchain }} | |
| rustup default ${{ matrix.toolchain }} | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| shared-key: "lance-context-deps" | |
| workspaces: "." | |
| - name: Install dependencies | |
| run: | | |
| sudo apt update | |
| sudo apt install -y clang libclang-dev protobuf-compiler | |
| - name: Build tests | |
| run: cargo test --workspace --all-targets --no-run | |
| - name: Run unit and integration tests | |
| # `--all-targets` matters: the previous `--lib` compiled the | |
| # crates/lance-context-core/tests/*.rs integration tests and then threw | |
| # them away without running them, so every WAL-merge and concurrency | |
| # regression test was dead weight in CI. `--workspace` covers the crates | |
| # that were silently untested (api, server, client, metrics, the facade). | |
| run: cargo test --workspace --all-targets | |
| - name: Start etcd for HA scheduler test | |
| run: | | |
| ETCD_VERSION=3.7.0 | |
| curl -fsSL -o /tmp/etcd.tar.gz \ | |
| "https://github.com/etcd-io/etcd/releases/download/v${ETCD_VERSION}/etcd-v${ETCD_VERSION}-linux-amd64.tar.gz" | |
| mkdir -p /tmp/etcd | |
| tar -xzf /tmp/etcd.tar.gz -C /tmp/etcd --strip-components=1 | |
| nohup /tmp/etcd/etcd \ | |
| --data-dir /tmp/etcd-data \ | |
| --listen-client-urls http://127.0.0.1:2379 \ | |
| --advertise-client-urls http://127.0.0.1:2379 \ | |
| >/tmp/etcd.log 2>&1 & | |
| for _ in $(seq 1 30); do | |
| curl -fsS http://127.0.0.1:2379/health && exit 0 | |
| sleep 1 | |
| done | |
| cat /tmp/etcd.log | |
| exit 1 | |
| # CI already pays to download and start etcd, so restricting this step to a | |
| # single named test left the other 18 ignored tests — the scheduler, | |
| # routes, and state etcd paths — completely untested. Run the whole suite. | |
| - name: Run etcd-backed HA test suite | |
| env: | |
| ETCD_TEST_ENDPOINTS: http://127.0.0.1:2379 | |
| run: cargo test -p lance-context-master --lib -- --ignored | |
| coverage: | |
| runs-on: ubuntu-24.04 | |
| # Coverage builds are much slower than the plain test job: instrumentation | |
| # defeats most caching wins and the core crate's suite alone is ~6.5 min. | |
| timeout-minutes: 60 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup rust toolchain | |
| run: | | |
| rustup toolchain install stable | |
| rustup default stable | |
| rustup component add llvm-tools-preview | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| shared-key: "lance-context-coverage" | |
| workspaces: "." | |
| - name: Install dependencies | |
| run: | | |
| sudo apt update | |
| sudo apt install -y clang libclang-dev protobuf-compiler | |
| - uses: taiki-e/install-action@cargo-llvm-cov | |
| - name: Start etcd for HA scheduler test | |
| run: | | |
| ETCD_VERSION=3.7.0 | |
| curl -fsSL -o /tmp/etcd.tar.gz \ | |
| "https://github.com/etcd-io/etcd/releases/download/v${ETCD_VERSION}/etcd-v${ETCD_VERSION}-linux-amd64.tar.gz" | |
| mkdir -p /tmp/etcd | |
| tar -xzf /tmp/etcd.tar.gz -C /tmp/etcd --strip-components=1 | |
| nohup /tmp/etcd/etcd \ | |
| --data-dir /tmp/etcd-data \ | |
| --listen-client-urls http://127.0.0.1:2379 \ | |
| --advertise-client-urls http://127.0.0.1:2379 \ | |
| >/tmp/etcd.log 2>&1 & | |
| for _ in $(seq 1 30); do | |
| curl -fsS http://127.0.0.1:2379/health && exit 0 | |
| sleep 1 | |
| done | |
| cat /tmp/etcd.log | |
| exit 1 | |
| - name: Clean stale coverage profile data | |
| run: cargo llvm-cov clean --workspace | |
| - name: Collect coverage (unit and integration tests) | |
| run: cargo llvm-cov --no-report --workspace --all-features | |
| - name: Collect coverage (etcd-backed ignored tests) | |
| env: | |
| ETCD_TEST_ENDPOINTS: http://127.0.0.1:2379 | |
| run: | | |
| cargo llvm-cov --no-report --all-features \ | |
| -p lance-context-master --lib -- --ignored | |
| - name: Merge coverage reports | |
| run: cargo llvm-cov report --lcov --output-path lcov.info | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| files: lcov.info | |
| fail_ci_if_error: false | |
| token: ${{ secrets.CODECOV_TOKEN }} |