chore: release 9.11.4 (security patch — GHSA-q2hj-ggqm-4g62, GHSA-946… #7279
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Standalone | |
| on: | |
| push: | |
| branches: | |
| - master | |
| - 'releases/**' | |
| pull_request: | |
| branches: | |
| - '*' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.sha }} | |
| cancel-in-progress: true | |
| env: | |
| # apt on the hosted images can block forever on an interactive | |
| # needrestart/debconf prompt; -y does not suppress those. | |
| DEBIAN_FRONTEND: noninteractive | |
| NEEDRESTART_MODE: a | |
| permissions: | |
| contents: read | |
| jobs: | |
| build_vendor_linux: | |
| name: core-cli.${{ matrix.os }}.amd64.${{ matrix.build_type }}.${{ matrix.compiler.cxx }}.standalone | |
| strategy: | |
| matrix: | |
| os: ["ubuntu-22.04"] | |
| build_type: ["Debug", "Release"] | |
| compiler: | |
| - { cc: "gcc", cxx: "g++"} | |
| - { cc: "clang", cxx: "clang++"} | |
| runs-on: ${{matrix.os}} | |
| timeout-minutes: 90 | |
| steps: | |
| - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 | |
| with: | |
| persist-credentials: false | |
| submodules: false | |
| - name: Init submodules with retry | |
| shell: bash | |
| run: | | |
| git config --global --add safe.directory "$(pwd)" | |
| for attempt in 1 2 3 4 5; do | |
| if git submodule update --init --recursive; then | |
| echo "Submodule init succeeded on attempt $attempt" | |
| exit 0 | |
| fi | |
| echo "Attempt $attempt failed, retrying in 15s..." | |
| sleep 15 | |
| done | |
| echo "Submodule init failed after 5 attempts" | |
| exit 1 | |
| - name: Install requirements | |
| run: bash "$GITHUB_WORKSPACE/.github/scripts/apt-install.sh" ninja-build libboost-test-dev | |
| # zizmor's cache-poisoning heuristic treats the `releases/**` push filter | |
| # above as a publishing trigger. The only artifact this workflow uploads is | |
| # a vw binary with retention-days: 1, consumed by a downstream test job; | |
| # the workflows that produce VW's release deliverables (python_wheels, | |
| # dotnet_nugets, native_nugets, java-publish) use no caching whatsoever. | |
| - name: Cache FetchContent downloads | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 # zizmor: ignore[cache-poisoning] | |
| with: | |
| path: build/_deps | |
| key: fetchcontent-${{ matrix.os }}-${{ hashFiles('cmake/VowpalWabbitUtils.cmake') }} | |
| - name: Configure | |
| env: | |
| CC: ${{matrix.compiler.cc}} | |
| CXX: ${{matrix.compiler.cxx}} | |
| run: > | |
| cmake -S . -B build -G Ninja | |
| -DCMAKE_BUILD_TYPE=${{matrix.build_type}} | |
| -DVW_FEAT_FLATBUFFERS=Off | |
| -DVW_FEAT_CSV=On | |
| -DVW_FEAT_CB_GRAPH_FEEDBACK=On | |
| -DRAPIDJSON_SYS_DEP=Off | |
| -DFMT_SYS_DEP=Off | |
| -DSPDLOG_SYS_DEP=Off | |
| -DVW_ZLIB_SYS_DEP=Off | |
| -DVW_BOOST_MATH_SYS_DEP=Off | |
| -DVW_INSTALL=Off | |
| -DWARNINGS=On | |
| -DWARNING_AS_ERROR=On | |
| -DVW_CXX_STANDARD=17 | |
| -DVW_SIMD_INV_SQRT=OFF | |
| -DCMAKE_POLICY_VERSION_MINIMUM=3.5 | |
| - name: Build | |
| run: cmake --build build | |
| - name: Upload vw binary | |
| if: matrix.build_type == 'Release' && matrix.compiler.cc == 'gcc' | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: vw-linux-release | |
| path: build/vowpalwabbit/cli/vw | |
| if-no-files-found: error | |
| retention-days: 1 | |
| - name: Unit tests | |
| working-directory: build | |
| run: ctest --output-on-failure --no-tests=error --label-regex VWTestList --parallel 2 | |
| - name: Test | |
| run: python3 test/run_tests.py -f --clean_dirty -E 0.001 | |
| big-tests: | |
| name: big-tests.[${{ matrix.segment }}] | |
| needs: build_vendor_linux | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 90 | |
| strategy: | |
| matrix: | |
| segment: [1, 2] | |
| steps: | |
| - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 | |
| with: | |
| persist-credentials: false | |
| submodules: false | |
| - name: Download vw binary | |
| uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 | |
| with: | |
| name: vw-linux-release | |
| - name: Prepare vw binary | |
| run: chmod +x vw | |
| - name: Download and prepare MNIST dataset | |
| if: matrix.segment == 1 | |
| working-directory: big_tests | |
| run: | | |
| mkdir -p dataSets/mnist.dir | |
| cd dataSets/mnist.dir | |
| for f in train-labels-idx1-ubyte.gz train-images-idx3-ubyte.gz \ | |
| t10k-labels-idx1-ubyte.gz t10k-images-idx3-ubyte.gz; do | |
| wget -nv "https://storage.googleapis.com/cvdf-datasets/mnist/$f" | |
| done | |
| cd ../../testCode | |
| g++ -O3 -Wall mnist.extractfeatures.cpp -o mnist.extractfeatures | |
| cd ../dataSets/mnist.dir | |
| export testCodeDir=$(pwd)/../../testCode | |
| $testCodeDir/mnist.munge.sh train-labels-idx1-ubyte.gz train-images-idx3-ubyte.gz \ | |
| | $testCodeDir/shuffle.pl > train.prep | |
| $testCodeDir/mnist.munge.sh t10k-labels-idx1-ubyte.gz t10k-images-idx3-ubyte.gz \ | |
| > test.prep | |
| - name: Download and prepare Covertype dataset | |
| if: matrix.segment == 1 | |
| working-directory: big_tests | |
| run: | | |
| mkdir -p dataSets/covtype.dir | |
| wget -nv -P dataSets/covtype.dir \ | |
| https://archive.ics.uci.edu/ml/machine-learning-databases/covtype/covtype.data.gz | |
| export testCodeDir=$(pwd)/testCode | |
| cd dataSets/covtype.dir | |
| $testCodeDir/covtype.munge.sh covtype.data.gz > prep | |
| - name: Download and prepare URLRep dataset | |
| if: matrix.segment == 1 | |
| working-directory: big_tests | |
| run: | | |
| mkdir -p dataSets/URLRep.dir | |
| wget -nv -P dataSets/URLRep.dir \ | |
| https://archive.ics.uci.edu/ml/machine-learning-databases/url/url_svmlight.tar.gz | |
| export testCodeDir=$(pwd)/testCode | |
| cd dataSets/URLRep.dir | |
| $testCodeDir/URLRep.munge.sh url_svmlight.tar.gz > prep | |
| - name: Download and prepare MovieLens dataset | |
| if: matrix.segment == 2 | |
| working-directory: big_tests | |
| run: | | |
| mkdir -p dataSets/movielens.dir | |
| # files.grouplens.org is serving an expired TLS certificate, which fails wget's | |
| # verification and blocks this job. The dataset itself is unchanged (Last-Modified | |
| # 2019-12-03), so integrity is enforced with a pinned SHA-256 instead of relying on | |
| # the certificate. Do not drop the checksum: it is the only thing authenticating this | |
| # download while verification is off. | |
| wget -nv --no-check-certificate -P dataSets/movielens.dir \ | |
| https://files.grouplens.org/datasets/movielens/ml-1m.zip | |
| echo "a6898adb50b9ca05aa231689da44c217cb524e7ebd39d264c56e2832f2c54e20 dataSets/movielens.dir/ml-1m.zip" \ | |
| | sha256sum --check --strict | |
| cd dataSets/movielens.dir | |
| unzip -ou ml-1m.zip | |
| ../../testCode/movielensRatings2vw.pl pre.train.vw test.prep ml-1m/ratings.dat | |
| perl -ne 'BEGIN { srand 8675309; }; 1; print join "\t", rand (), $_;' \ | |
| pre.train.vw | sort -k1 | cut -f2- > train.prep | |
| - name: Download and prepare OCR dataset | |
| if: matrix.segment == 2 | |
| working-directory: big_tests | |
| run: | | |
| mkdir -p dataSets/OCR.dir | |
| wget -nv -P dataSets/OCR.dir \ | |
| http://ai.stanford.edu/~btaskar/ocr/letter.data.gz \ | |
| http://ai.stanford.edu/~btaskar/ocr/letter.names | |
| python3 testCode/ocr2vw.py \ | |
| dataSets/OCR.dir/letter.data.gz dataSets/OCR.dir/letter.names \ | |
| dataSets/OCR.dir/train.prep dataSets/OCR.dir/test.prep | |
| - name: Run tests | |
| working-directory: big_tests | |
| run: | | |
| VW=${{ github.workspace }}/vw | |
| DIFF="perl testCode/floatingTolerance.pl" | |
| TOL=0.001 | |
| run_test() { | |
| local name="$1" | |
| shift | |
| echo "=== Running test $name ===" | |
| mkdir -p runs/$name.dir | |
| $VW "$@" > runs/$name.dir/raw_out 2> runs/$name.dir/raw_err | |
| grep "average loss" runs/$name.dir/raw_err > runs/$name.dir/err | |
| echo "Result: $(cat runs/$name.dir/err)" | |
| $DIFF $TOL $TOL runs/$name.dir/err expected/$name/err | |
| echo "=== Test $name PASSED ===" | |
| } | |
| if [ "${{ matrix.segment }}" = "1" ]; then | |
| # Test 1a: MNIST train | |
| run_test 1a --oaa 10 -d dataSets/mnist.dir/train.prep \ | |
| -f runs/1a.dir/mnist.model -b 24 --adaptive --invariant --holdout_off \ | |
| -l 0.1 --nn 40 --passes 24 -k --compressed --cache_file runs/1a.dir/mnist.cache | |
| # Test 1b: MNIST predict | |
| run_test 1b -t -d dataSets/mnist.dir/test.prep -i runs/1a.dir/mnist.model | |
| # Test 2: Covertype | |
| run_test 2 --oaa 7 -d dataSets/covtype.dir/prep | |
| # Test 3: URL Reputation | |
| run_test 3 -d dataSets/URLRep.dir/prep | |
| else | |
| # Test 5a: MovieLens train | |
| run_test 5a --loss_function quantile -l 0.45 -b 24 --passes 100 -k \ | |
| --cache_file runs/5a.dir/movielens.cache -d dataSets/movielens.dir/train.prep \ | |
| --holdout_off --lrq um14 --lrqdropout --adaptive --invariant \ | |
| -f runs/5a.dir/movielens.model | |
| # Test 5b: MovieLens predict | |
| run_test 5b --loss_function quantile -t \ | |
| -i runs/5a.dir/movielens.model -d dataSets/movielens.dir/test.prep | |
| # Test 6a: OCR train | |
| run_test 6a -d dataSets/OCR.dir/train.prep \ | |
| -f runs/6a.dir/OCR.model --cache_file runs/6a.dir/OCR.cache -k \ | |
| --oaa 26 --adaptive --invariant --holdout_off --loss_function logistic --passes 14 | |
| # Test 6b: OCR predict | |
| run_test 6b -i runs/6a.dir/OCR.model -d dataSets/OCR.dir/test.prep --testonly | |
| fi | |
| build_vendor_windows: | |
| name: core-cli.${{ matrix.os }}.amd64.${{ matrix.build_type }}.msvc.standalone | |
| runs-on: ${{matrix.os}} | |
| timeout-minutes: 90 | |
| strategy: | |
| matrix: | |
| os: ["windows-latest"] | |
| build_type: ["Debug", "Release"] | |
| env: | |
| CMAKE_BUILD_DIR: ${{ github.workspace }}/vw/build | |
| SOURCE_DIR: ${{ github.workspace }}/vw | |
| steps: | |
| - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 | |
| with: | |
| persist-credentials: false | |
| path: 'vw' | |
| submodules: false | |
| - name: Init submodules with retry | |
| shell: bash | |
| working-directory: vw | |
| run: | | |
| git config --global --add safe.directory "$(pwd)" | |
| for attempt in 1 2 3 4 5; do | |
| if git submodule update --init --recursive; then | |
| echo "Submodule init succeeded on attempt $attempt" | |
| exit 0 | |
| fi | |
| echo "Attempt $attempt failed, retrying in 15s..." | |
| sleep 15 | |
| done | |
| echo "Submodule init failed after 5 attempts" | |
| exit 1 | |
| - uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 | |
| # zizmor's cache-poisoning heuristic treats the `releases/**` push filter | |
| # above as a publishing trigger. The only artifact this workflow uploads is | |
| # a vw binary with retention-days: 1, consumed by a downstream test job; | |
| # the workflows that produce VW's release deliverables (python_wheels, | |
| # dotnet_nugets, native_nugets, java-publish) use no caching whatsoever. | |
| - name: Cache FetchContent downloads | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 # zizmor: ignore[cache-poisoning] | |
| with: | |
| path: ${{ env.CMAKE_BUILD_DIR }}/_deps | |
| key: fetchcontent-${{ matrix.os }}-${{ hashFiles('vw/cmake/VowpalWabbitUtils.cmake') }} | |
| - name: Configure | |
| run: > | |
| cmake -S "$env:SOURCE_DIR" -B "$env:CMAKE_BUILD_DIR" -A "x64" | |
| -DUSE_LATEST_STD=On | |
| -DVW_FEAT_FLATBUFFERS=Off | |
| -DVW_FEAT_CSV=On | |
| -DVW_FEAT_CB_GRAPH_FEEDBACK=On | |
| -DRAPIDJSON_SYS_DEP=Off | |
| -DFMT_SYS_DEP=Off | |
| -DSPDLOG_SYS_DEP=Off | |
| -DVW_ZLIB_SYS_DEP=Off | |
| -DVW_BOOST_MATH_SYS_DEP=Off | |
| -DVW_INSTALL=Off | |
| -DVW_SIMD_INV_SQRT=OFF | |
| "-DCMAKE_POLICY_VERSION_MINIMUM=3.5" | |
| - name: Build | |
| run: cmake --build "$env:CMAKE_BUILD_DIR" --config ${{ matrix.build_type }} | |
| - name: Test run_tests.py | |
| run: python3 "$env:SOURCE_DIR/test/run_tests.py" -f --clean_dirty -E 0.01 --skip_spanning_tree_tests --vw_bin_path "$env:CMAKE_BUILD_DIR/vowpalwabbit/cli/${{ matrix.build_type }}/vw.exe" | |
| - name: Test unit tests | |
| working-directory: ${{ github.workspace }}/vw/build | |
| run: ctest --output-on-failure --no-tests=error --label-regex VWTestList --build-config ${{ matrix.build_type }} --parallel 2 | |
| build_vendor_macos: | |
| runs-on: ${{matrix.os}} | |
| timeout-minutes: 90 | |
| name: core-cli.${{ matrix.os }}.amd64.${{ matrix.build_type }}.AppleClang.standalone | |
| strategy: | |
| matrix: | |
| os: [macos-14] | |
| build_type: [Debug, Release] | |
| steps: | |
| - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 | |
| with: | |
| persist-credentials: false | |
| submodules: false | |
| - name: Init submodules with retry | |
| shell: bash | |
| run: | | |
| git config --global --add safe.directory "$(pwd)" | |
| for attempt in 1 2 3 4 5; do | |
| if git submodule update --init --recursive; then | |
| echo "Submodule init succeeded on attempt $attempt" | |
| exit 0 | |
| fi | |
| echo "Attempt $attempt failed, retrying in 15s..." | |
| sleep 15 | |
| done | |
| echo "Submodule init failed after 5 attempts" | |
| exit 1 | |
| - name: Install dependencies | |
| run: brew install cmake ninja | |
| # zizmor's cache-poisoning heuristic treats the `releases/**` push filter | |
| # above as a publishing trigger. The only artifact this workflow uploads is | |
| # a vw binary with retention-days: 1, consumed by a downstream test job; | |
| # the workflows that produce VW's release deliverables (python_wheels, | |
| # dotnet_nugets, native_nugets, java-publish) use no caching whatsoever. | |
| - name: Cache FetchContent downloads | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 # zizmor: ignore[cache-poisoning] | |
| with: | |
| path: build/_deps | |
| key: fetchcontent-${{ matrix.os }}-${{ hashFiles('cmake/VowpalWabbitUtils.cmake') }} | |
| - name: Configure | |
| run: > | |
| cmake -S . -B build -G Ninja | |
| -DCMAKE_BUILD_TYPE=${{matrix.build_type}} | |
| -DVW_FEAT_FLATBUFFERS=Off | |
| -DVW_FEAT_CSV=On | |
| -DVW_FEAT_CB_GRAPH_FEEDBACK=On | |
| -DRAPIDJSON_SYS_DEP=Off | |
| -DFMT_SYS_DEP=Off | |
| -DSPDLOG_SYS_DEP=Off | |
| -DVW_ZLIB_SYS_DEP=Off | |
| -DVW_BOOST_MATH_SYS_DEP=Off | |
| -DVW_INSTALL=Off | |
| -DVW_SIMD_INV_SQRT=OFF | |
| -DCMAKE_POLICY_VERSION_MINIMUM=3.5 | |
| - name: Build | |
| run: cmake --build build | |
| - name: Unit tests | |
| working-directory: build | |
| run: ctest --output-on-failure --no-tests=error --label-regex VWTestList --parallel 2 | |
| - name: Test | |
| run: | | |
| python3 test/run_tests.py -f --clean_dirty -E 0.001 |