feat(shuffle): add swap-or-not shuffling module and binding #2108
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| branches: | |
| - main | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }}-ci | |
| cancel-in-progress: true | |
| env: | |
| ZIG_VERSION: "0.16.0" | |
| NODE_VERSION: "24" | |
| jobs: | |
| build-test: | |
| name: build & test (${{ matrix.os }}) | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - ubuntu-latest | |
| - ubuntu-24.04-arm | |
| - macos-latest | |
| # - macos-13 # skip because hashtree does not support macos x64 yet | |
| # - windows-latest # TODO: https://github.com/ChainSafe/state-transition-z/issues/10 | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 | |
| - name: Install Zig | |
| uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 | |
| with: | |
| version: ${{ env.ZIG_VERSION }} | |
| cache-key: ${{ matrix.os }}-${{ env.ZIG_VERSION }} | |
| # Zig fmt is failing on windows and is redundant to run all platforms. So we run only on linux. | |
| - name: Check formatting | |
| if: matrix.os == 'ubuntu-latest' | |
| run: zig fmt --check . | |
| - name: Run hashing tests | |
| run: | | |
| zig build test:hashing | |
| - name: Run persistent_merkle_tree tests | |
| run: | | |
| zig build test:persistent_merkle_tree | |
| - name: Run ssz tests | |
| run: | | |
| zig build test:ssz | |
| - name: Run hex tests | |
| run: | | |
| zig build test:hex | |
| - name: Run cpu_count tests | |
| run: | | |
| zig build test:cpu_count | |
| - name: Run config tests | |
| run: | | |
| zig build test:config | |
| - name: Run constants tests | |
| run: | | |
| zig build test:constants | |
| - name: Run consensus-type tests | |
| run: | | |
| zig build test:consensus_types | |
| - name: Run fork_types tests | |
| run: | | |
| zig build test:fork_types | |
| - name: Run era tests | |
| run: | | |
| zig build test:era | |
| - name: Run preset tests | |
| run: | | |
| zig build test:preset | |
| - name: Run bls tests | |
| run: | | |
| zig build test:bls | |
| - name: Run swap-or-not-shuffle tests | |
| run: | | |
| zig build test:swap_or_not_shuffle | |
| - name: Run state-transition tests | |
| run: | | |
| zig build test:state_transition | |
| - name: Run fork-choice tests | |
| run: | | |
| zig build test:fork_choice | |
| - name: Run clock tests | |
| run: | | |
| zig build test:clock | |
| - name: Run beacon-node tests | |
| run: | | |
| zig build test:beacon_node | |
| - name: Build benchmarks | |
| run: | | |
| zig build build-exe:bench_hashing build-exe:bench_merkle_gindex build-exe:bench_merkle_node build-exe:bench_ssz_attestation build-exe:bench_ssz_block build-exe:bench_ssz_state build-exe:bench_process_block build-exe:bench_process_epoch build-exe:bench_fork_choice_update_head build-exe:bench_fork_choice_compute_deltas build-exe:bench_fork_choice_on_attestation | |
| build-test-slow: | |
| name: slow tests | |
| # It takes time to download and run these tests, so we only run them on ubuntu-latest | |
| runs-on: ubuntu-latest | |
| needs: build-test | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 | |
| - name: Install Zig | |
| uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 | |
| with: | |
| version: ${{ env.ZIG_VERSION }} | |
| cache-key: ubuntu-latest-${{ env.ZIG_VERSION }} | |
| - name: Restore spec tests cache | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 | |
| with: | |
| path: fixtures/era | |
| key: era-files-${{ hashFiles('build.zig') }} # downloaded era files defined in build.zig | |
| - name: Download era files | |
| run: | | |
| zig build run:download_era_files | |
| - name: Run slow int tests | |
| run: | | |
| zig build test:int | |
| spec-test-prep: | |
| name: spec tests - prepare | |
| runs-on: ubuntu-latest | |
| outputs: | |
| spec-test-version: ${{ steps.spec-test-version.outputs.version }} | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 | |
| - name: Install Zig | |
| uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 | |
| with: | |
| version: ${{ env.ZIG_VERSION }} | |
| cache-key: ubuntu-latest-${{ env.ZIG_VERSION }} | |
| - name: Read spec test version | |
| id: spec-test-version | |
| run: | | |
| version=$(awk ' | |
| /\.spec_test_version = \./ {in_block=1} | |
| in_block && /\.default = / { | |
| match($0, /"[^"]+"/) | |
| print substr($0, RSTART + 1, RLENGTH - 2) | |
| exit | |
| } | |
| ' build.zig.zon) | |
| test -n "$version" | |
| echo "version=$version" >> "$GITHUB_OUTPUT" | |
| - name: Restore spec tests cache | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 | |
| with: | |
| path: test/spec/spec_tests | |
| key: spec-test-data-${{ steps.spec-test-version.outputs.version }} | |
| - name: Download spec tests | |
| run: | | |
| zig build run:download_spec_tests | |
| - name: Write spec tests | |
| run: | | |
| zig build run:write_spec_tests | |
| - name: Write ssz_generic spec tests | |
| run: | | |
| zig build run:write_ssz_generic_spec_tests | |
| - name: Write ssz_static spec tests | |
| run: | | |
| zig build run:write_ssz_static_spec_tests | |
| - name: Write BLS spec tests | |
| run: | | |
| zig build run:write_bls_spec_tests | |
| - name: Upload generated test files | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: spec-test-generated | |
| path: | | |
| test/spec/root*.zig | |
| test/spec/test_case/ | |
| test/spec/ssz/generic_tests.zig | |
| test/spec/ssz/static_tests.zig | |
| test/spec/bls/spec_tests.zig | |
| retention-days: 1 | |
| spec-test-ssz-generic: | |
| name: spec tests - ssz generic | |
| runs-on: ubuntu-latest | |
| needs: [build-test, spec-test-prep] | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 | |
| - name: Install Zig | |
| uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 | |
| with: | |
| version: ${{ env.ZIG_VERSION }} | |
| cache-key: ubuntu-latest-${{ env.ZIG_VERSION }} | |
| - name: Restore spec tests cache | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 | |
| with: | |
| path: test/spec/spec_tests | |
| key: spec-test-data-${{ needs['spec-test-prep'].outputs['spec-test-version'] }} | |
| - name: Download generated test files | |
| uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 | |
| with: | |
| name: spec-test-generated | |
| path: test/spec | |
| - name: Run ssz_generic spec tests | |
| run: | | |
| zig build test:ssz_generic_spec_tests | |
| spec-test-ssz-static-minimal: | |
| name: spec tests - ssz static minimal | |
| runs-on: ubuntu-latest | |
| needs: [build-test, spec-test-prep] | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 | |
| - name: Install Zig | |
| uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 | |
| with: | |
| version: ${{ env.ZIG_VERSION }} | |
| cache-key: ubuntu-latest-${{ env.ZIG_VERSION }} | |
| - name: Restore spec tests cache | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 | |
| with: | |
| path: test/spec/spec_tests | |
| key: spec-test-data-${{ needs['spec-test-prep'].outputs['spec-test-version'] }} | |
| - name: Download generated test files | |
| uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 | |
| with: | |
| name: spec-test-generated | |
| path: test/spec | |
| - name: Run minimal ssz_static spec tests | |
| run: | | |
| zig build test:ssz_static_spec_tests -Dpreset=minimal -Doptimize=ReleaseSafe | |
| spec-test-ssz-static-mainnet: | |
| name: spec tests - ssz static mainnet | |
| runs-on: ubuntu-latest | |
| needs: [build-test, spec-test-prep] | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 | |
| - name: Install Zig | |
| uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 | |
| with: | |
| version: ${{ env.ZIG_VERSION }} | |
| cache-key: ubuntu-latest-${{ env.ZIG_VERSION }} | |
| - name: Restore spec tests cache | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 | |
| with: | |
| path: test/spec/spec_tests | |
| key: spec-test-data-${{ needs['spec-test-prep'].outputs['spec-test-version'] }} | |
| - name: Download generated test files | |
| uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 | |
| with: | |
| name: spec-test-generated | |
| path: test/spec | |
| - name: Run mainnet ssz_static spec tests | |
| run: | | |
| zig build test:ssz_static_spec_tests -Dpreset=mainnet | |
| spec-test-bls: | |
| name: spec tests - bls | |
| runs-on: ubuntu-latest | |
| needs: [build-test, spec-test-prep] | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 | |
| - name: Install Zig | |
| uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 | |
| with: | |
| version: ${{ env.ZIG_VERSION }} | |
| cache-key: ubuntu-latest-${{ env.ZIG_VERSION }} | |
| - name: Restore spec tests cache | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 | |
| with: | |
| path: test/spec/spec_tests | |
| key: spec-test-data-${{ needs['spec-test-prep'].outputs['spec-test-version'] }} | |
| - name: Download generated test files | |
| uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 | |
| with: | |
| name: spec-test-generated | |
| path: test/spec | |
| - name: Run BLS spec tests | |
| run: | | |
| zig build test:bls_spec_tests | |
| spec-test-minimal: | |
| name: spec tests - minimal (${{ matrix.runner }}) | |
| runs-on: ubuntu-latest | |
| needs: [build-test, spec-test-prep] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| runner: | |
| - operations | |
| - epoch_processing | |
| - sanity | |
| - rewards | |
| - transition | |
| - random | |
| - fork | |
| - finality | |
| - merkle_proof | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 | |
| - name: Install Zig | |
| uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 | |
| with: | |
| version: ${{ env.ZIG_VERSION }} | |
| cache-key: ubuntu-latest-${{ env.ZIG_VERSION }} | |
| - name: Restore spec tests cache | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 | |
| with: | |
| path: test/spec/spec_tests | |
| key: spec-test-data-${{ needs['spec-test-prep'].outputs['spec-test-version'] }} | |
| - name: Download generated test files | |
| uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 | |
| with: | |
| name: spec-test-generated | |
| path: test/spec | |
| - name: Run spec tests - minimal (${{ matrix.runner }}) | |
| run: | | |
| zig build test:spec_tests -Dpreset=minimal -Dspec_tests.filters=${{ matrix.runner }} | |
| spec-test-mainnet: | |
| name: spec tests - mainnet (${{ matrix.runner }}) | |
| runs-on: ubuntu-latest | |
| needs: [build-test, spec-test-prep] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| runner: | |
| - operations | |
| - epoch_processing | |
| - sanity | |
| - rewards | |
| - transition | |
| - random | |
| - fork | |
| - finality | |
| - merkle_proof | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 | |
| - name: Install Zig | |
| uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 | |
| with: | |
| version: ${{ env.ZIG_VERSION }} | |
| cache-key: ubuntu-latest-${{ env.ZIG_VERSION }} | |
| - name: Restore spec tests cache | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 | |
| with: | |
| path: test/spec/spec_tests | |
| key: spec-test-data-${{ needs['spec-test-prep'].outputs['spec-test-version'] }} | |
| - name: Download generated test files | |
| uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 | |
| with: | |
| name: spec-test-generated | |
| path: test/spec | |
| - name: Run spec tests - mainnet (${{ matrix.runner }}) | |
| run: | | |
| zig build test:spec_tests -Dpreset=mainnet -Doptimize=ReleaseSafe -Dspec_tests.filters=${{ matrix.runner }} | |
| build-fuzz: | |
| name: build fuzz harnesses | |
| runs-on: ubuntu-latest | |
| needs: build-test | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 | |
| - name: Install Zig | |
| uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 | |
| with: | |
| version: ${{ env.ZIG_VERSION }} | |
| cache-key: ubuntu-latest-${{ env.ZIG_VERSION }} | |
| - name: Install AFL++ and LLVM | |
| run: | | |
| sudo apt-get update | |
| sudo DEBIAN_FRONTEND=noninteractive apt-get install -y afl++ llvm | |
| - name: Verify AFL++ and LLVM are available | |
| run: | | |
| afl-cc --version | |
| if command -v llvm-config >/dev/null 2>&1; then | |
| llvm-config --version | |
| else | |
| llvm-config-18 --version | |
| fi | |
| - name: Build fuzz harnesses | |
| # Build in ReleaseFast so Zig 0.16's Debug `sanitize_c = .full` | |
| # doesn't emit `__ubsan_handle_*` calls into blst's C objects that | |
| # `afl-cc` can't resolve without `libubsan`. Fuzzers want throughput | |
| # anyway; devs can still use `zig build` in Debug locally. | |
| run: | | |
| cd test/fuzz && zig build -Doptimize=ReleaseFast | |
| bindings-test: | |
| name: bindings - test | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 | |
| - name: Setup pnpm | |
| uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4.3.0 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| cache: pnpm | |
| - name: Install Zig | |
| uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 | |
| with: | |
| version: ${{ env.ZIG_VERSION }} | |
| cache-key: ubuntu-latest-${{ env.ZIG_VERSION }} | |
| - name: Install dependencies | |
| run: | | |
| pnpm install | |
| - name: Run bindings formatter | |
| run: | | |
| pnpm biome ci | |
| - name: Build bindings | |
| run: | | |
| zig build build-lib:bindings -Doptimize=ReleaseSafe | |
| - name: Restore era files cache | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 | |
| with: | |
| path: fixtures/era | |
| key: era-files-${{ hashFiles('build.zig') }} # downloaded era files defined in build.zig | |
| - name: Download era files | |
| run: | | |
| zig build run:download_era_files | |
| - name: Test bindings | |
| run: | | |
| pnpm test | |
| bindings-artifacts: | |
| name: bindings - build artifacts | |
| # Runs the exact multi-target build the Release workflow publishes | |
| # (see publish-bindings.yml), so cross-compile breakage is caught | |
| # pre-merge instead of after a release tag already exists. | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 | |
| - name: Setup pnpm | |
| uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4.3.0 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| cache: pnpm | |
| - name: Install Zig | |
| uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 | |
| with: | |
| version: ${{ env.ZIG_VERSION }} | |
| cache-key: ubuntu-latest-${{ env.ZIG_VERSION }} | |
| - name: Install dependencies | |
| run: | | |
| pnpm install --frozen-lockfile | |
| - name: Build release artifacts | |
| run: | | |
| pnpm zapi build-artifacts --optimize ReleaseSafe | |
| # metrics_stf depends on httpz, which is temporarily disabled for Zig 0.16 migration | |
| # build-examples: | |
| # name: build-examples | |
| # runs-on: ubuntu-latest | |
| # steps: | |
| # - name: Checkout Repository | |
| # uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 | |
| # - name: Install Zig | |
| # uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 | |
| # with: | |
| # version: ${{ env.ZIG_VERSION }} | |
| # cache-key: ubuntu-latest-${{ env.ZIG_VERSION }} | |
| # - name: Build metrics example | |
| # run: | | |
| # zig build build-exe:metrics_stf |