Merge pull request #202 from KhiopsML/dev

Release 10.2.2.0

Author: popescu-v

.github/workflows/api-docs.yml

@@ -68,7 +68,7 @@ jobs:
           path: doc/_build/html/
   # Deploy only when the user explicitly (and manually) orders it
   deploy:
-    if: ${{ github.event_name == 'workflow_dispatch' || inputs.deploy-gh-pages == true }}
+    if: github.event_name == 'workflow_dispatch' && inputs.deploy-gh-pages == true
     runs-on: ubuntu-latest
     needs: build
     environment:

.github/workflows/conda.yml

@@ -1,17 +1,19 @@
 ---
-name: Conda Packages
+name: Conda Package
 env:
-  DEFAULT_KHIOPS_REVISION: main
-  DEFAULT_SAMPLES_REVISION: main
+  DEFAULT_SAMPLES_VERSION: 10.2.0
+  # Note: The default Khiops version must never be an alpha release as they are
+  #       ephemeral. To test alpha versions run the workflow manually.
+  DEFAULT_KHIOPS_CORE_VERSION: 10.2.2
 on:
   workflow_dispatch:
     inputs:
-      khiops-revision:
-        default: main
-        description: khiops repo revision
-      samples-revision:
-        default: main
-        description: khiops-samples repo revision
+      khiops-core-version:
+        default: 10.2.2
+        description: khiops-core version for testing
+      khiops-samples-version:
+        default: 10.2.0
+        description: khiops-samples version
       release-channel:
         type: choice
         default: khiops-dev
@@ -29,18 +31,7 @@ concurrency:
   cancel-in-progress: true
 jobs:
   build:
-    strategy:
-      fail-fast: false
-      matrix:
-        #  Use the oldest supported Mac OS and Ubuntu versions for GLIBC compatibility
-        include:
-          - os: ubuntu-20.04
-            os-family: linux
-          - os: windows-latest
-            os-family: windows
-          - os: macos-11
-            os-family: macos
-    runs-on: ${{ matrix.os }}
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout Sources
         uses: actions/checkout@v4
@@ -52,34 +43,19 @@ jobs:
         uses: conda-incubator/setup-miniconda@v3
         with:
           miniconda-version: latest
-          python-version: '3.11'
+          python-version: '3.12'
       - name: Install Dependency Requirements for Building Conda Packages
-        run: conda install conda-build=3.27.0 conda-verify
-      # We need MacOS SDK 10.10 to build on Big Sur
-      - name: Install Mac OS SDK 10.10
-        if: runner.os == 'macOS'
-        run: |
-          wget https://github.com/phracker/MacOSX-SDKs/releases/download/11.3/MacOSX10.10.sdk.tar.xz
-          sudo tar -zxvf MacOSX10.10.sdk.tar.xz -C /opt
-      - name: Set KHIOPS_REVISION build input parameter
+        run: conda install -y conda-build
+      - name: Build the Conda Package
+        # Note: The "khiops-dev" conda channel is needed to retrieve the "khiops-core" package.
+        #       The "test" part of the conda recipe needs this package.
         run: |
-          KHIOPS_REVISION="${{ inputs.khiops-revision || env.DEFAULT_KHIOPS_REVISION }}"
-          echo "KHIOPS_REVISION=$KHIOPS_REVISION" >> "$GITHUB_ENV"
-      - name: Build Khiops Conda Package (Windows)
-        if: runner.os == 'Windows'
-        run: |
-          mkdir khiops-conda
-          conda build --output-folder khiops-conda ./packaging/conda
-      # In Linux/macOS we need the conda-forge channel to install their pinned versions
-      - name: Build Khiops Conda Package (Linux/macOS)
-        if: runner.os != 'Windows'
-        run: |
-          mkdir khiops-conda
-          conda build --channel conda-forge --output-folder khiops-conda ./packaging/conda
-      - name: Upload Khiops Conda Package
+          conda build --channel conda-forge --channel khiops-dev \
+            --output-folder ./khiops-conda ./packaging/conda
+      - name: Upload Conda Package Artifact
         uses: actions/upload-artifact@v4
         with:
-          name: khiops-conda-${{ matrix.os-family }}
+          name: khiops-conda
           path: ./khiops-conda
           retention-days: 7
   # Test Conda package on brand new environments
@@ -88,26 +64,20 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        env:
-          - {os: ubuntu-20.04, os-family: linux}
-          - {os: ubuntu-22.04, os-family: linux}
-          - {os: windows-2019, os-family: windows}
-          - {os: windows-2022, os-family: windows}
-          - {os: macos-11, os-family: macos}
-          - {os: macos-12, os-family: macos}
-          - {os: macos-13, os-family: macos}
         python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
+        env:
+          - {os: ubuntu-20.04, json-image: '{"image": null}'}
+          - {os: ubuntu-22.04, json-image: '{"image": null}'}
+          - {os: ubuntu-22.04, json-image: '{"image": "rockylinux:8"}'}
+          - {os: ubuntu-22.04, json-image: '{"image": "rockylinux:9"}'}
+          - {os: windows-2019, json-image: '{"image": null}'}
+          - {os: windows-2022, json-image: '{"image": null}'}
+          - {os: macos-12, json-image: '{"image": null}'}
+          - {os: macos-13, json-image: '{"image": null}'}
+          - {os: macos-14, json-image: '{"image": null}'}
     runs-on: ${{ matrix.env.os }}
-    env:
-      KHIOPS_SAMPLES_DIR: ./khiops-samples-repo
+    container: ${{ fromJSON(matrix.env.json-image) }}
     steps:
-      - name: Checkout Khiops samples
-        uses: actions/checkout@v4
-        with:
-          repository: khiopsml/khiops-samples
-          ref: ${{ inputs.samples-revision || env.DEFAULT_SAMPLES_REVISION }}
-          token: ${{ secrets.GITHUB_TOKEN }}
-          path: ${{ env.KHIOPS_SAMPLES_DIR }}
       - name: Install Miniconda
         uses: conda-incubator/setup-miniconda@v3
         with:
@@ -116,18 +86,33 @@ jobs:
       - name: Download Conda Package Artifact
         uses: actions/download-artifact@v4
         with:
-          name: khiops-conda-${{ matrix.env.os-family }}
-          path: khiops-conda
-      - name: Install the Khiops Conda pagkage (Windows)
+          name: khiops-conda
+          path: ./khiops-conda
+      - name: Put the khiops-core Version in the Environment
+        run: |
+          KHIOPS_CORE_VERSION="${{ inputs.khiops-core-version || env.DEFAULT_KHIOPS_CORE_VERSION }}"
+          echo "KHIOPS_CORE_VERSION=$KHIOPS_CORE_VERSION" >> "$GITHUB_ENV"
+      - name: Install the Khiops Conda package (Windows)
         if: runner.os == 'Windows'
-        run: conda install -c ./khiops-conda/ khiops
+        run: |
+          conda install --channel khiops-dev khiops-core=$KHIOPS_CORE_VERSION
+          conda install --override-channels --channel conda-forge --channel ./khiops-conda/ khiops
       # In Linux/macOS we need the conda-forge channel to install their pinned versions
       - name: Install the Khiops Conda package (Linux/macOS)
         if: runner.os != 'Windows'
-        run: conda install -c conda-forge -c ./khiops-conda/ khiops
+        run: |
+          conda install --channel conda-forge --channel khiops-dev khiops-core=$KHIOPS_CORE_VERSION
+          conda install --channel ./khiops-conda/ khiops
       - name: Test Khiops Installation Status
         run: kh-status
+      - name: Download Sample Datasets
+        run: |
+          kh-download-datasets \
+            --version ${{ inputs.khiops-samples-version || env.DEFAULT_SAMPLES_VERSION }}
       - name: Test Conda Package Installation on Samples
+        env:
+          # Force > 2 CPU cores to launch mpiexec
+          KHIOPS_PROC_NUMBER: 4
         run: |
           kh-samples core -i train_predictor -e
           kh-samples core -i train_predictor_error_handling -e
@@ -140,66 +125,37 @@ jobs:
   release:
     if: github.ref_type == 'tag'
     needs: test
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     permissions:
       contents: write
     steps:
       - name: Download package artifacts
         uses: actions/download-artifact@v4
         with:
           # See the upload-artifact step in the build job for the explanation of this pattern
+          name: khiops-conda
           path: ./khiops-conda
-          pattern: khiops-conda-*
-          merge-multiple: true
       - name: Install Miniconda
         uses: conda-incubator/setup-miniconda@v3
         with:
           miniconda-version: latest
-          python-version: '3.11'
-      - name: Install requirement packages
-        run: conda install -y anaconda-client conda-build=3.27.0
+          python-version: '3.12'
+      - name: Install Requirement Packages
+        run: conda install -y anaconda-client conda-index
       - name: Reindex the package directory
-        run: conda-index ./khiops-conda
-      - name: Upload the packages to anaconda.org
-        run: |
+        run: python -m conda_index ./khiops-conda
+      - name: Upload the Package to anaconda.org
+        run: |-
           # Set the anaconda.org channel
           ANACONDA_CHANNEL="${{ inputs.release-channel || 'khiops-dev' }}"
 
           # For the release channel: upload without forcing
           if [[ "$ANACONDA_CHANNEL" == "khiops" ]]
           then
             anaconda --token "${{ secrets.KHIOPS_ANACONDA_CHANNEL_TOKEN }}" upload \
-              --user "$ANACONDA_CHANNEL" ./khiops-conda/*/*.tar.bz2
+              --user "$ANACONDA_CHANNEL" ./khiops-conda/noarch/*.tar.bz2
           # For the dev channel: upload with forcing
           else
             anaconda --token "${{ secrets.KHIOPS_DEV_ANACONDA_CHANNEL_TOKEN }}" upload \
-              --user "$ANACONDA_CHANNEL" --force ./khiops-conda/*/*.tar.bz2
+              --user "$ANACONDA_CHANNEL" --force ./khiops-conda/noarch/*.tar.bz2
           fi
-      - name: Extract package version
-        run: |
-          PKG_VERSION=$(\
-              conda search --override-channels --channel ./khiops-conda/ khiops \
-                | awk '!/#|channels/ {print $2}' \
-                | sort -u \
-          )
-          echo "PKG_VERSION=$PKG_VERSION" >> "$GITHUB_ENV"
-      - name: Create the release zip archive
-        uses: thedoctor0/zip-release@0.7.6
-        with:
-          type: zip
-          path: ./khiops-conda/
-          filename: khiops-${{ env.PKG_VERSION }}-conda.zip
-      - name: Upload conda package artifacts for all platforms
-        uses: actions/upload-artifact@v4
-        with:
-          name: khiops-conda-all
-          path: ./khiops-${{ env.PKG_VERSION }}-conda.zip
-      - name: Release the zip archive
-        uses: ncipollo/release-action@v1
-        with:
-          allowUpdates: true
-          artifacts: ./khiops-${{ env.PKG_VERSION }}-conda.zip
-          draft: false
-          makeLatest: false
-          prerelease: true
-          updateOnlyUnreleased: true

.github/workflows/dev-docker.yml

@@ -1,8 +1,9 @@
 ---
 name: Dev Docker
 env:
-  DEFAULT_KHIOPS_REVISION: main
+  DEFAULT_KHIOPS_REVISION: 10.2.2
   DEFAULT_SERVER_REVISION: main
+  DEFAULT_PYTHON_VERSIONS: 3.8 3.9 3.10 3.11 3.12
 on:
   pull_request:
     paths: [packaging/docker/khiopspydev/Dockerfile.*, .github/workflows/dev-docker.yml]
@@ -20,6 +21,10 @@ on:
         type: boolean
         default: true
         description: Push to GH Registry
+      python-versions:
+        type: string
+        default: 3.8 3.9 3.10 3.11 3.12
+        description: Python versions supported by khiops-python
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
@@ -60,6 +65,7 @@ jobs:
             "KHIOPS_REVISION=${{ env.KHIOPS_REVISION }}"
             "KHIOPSDEV_OS=${{ matrix.khiopsdev-os }}"
             "SERVER_REVISION=${{ env.SERVER_REVISION }}"
+            "PYTHON_VERSIONS=${{ inputs.python-versions || env.DEFAULT_PYTHON_VERSIONS }}"
           tags: ghcr.io/khiopsml/khiops-python/khiopspydev-${{ matrix.khiopsdev-os }}:latest
           # Push only on manual request
           push: ${{ inputs.push || false }}

.github/workflows/pip.yml

@@ -71,11 +71,21 @@ jobs:
       - name: Run tests
         env:
           KHIOPS_SAMPLES_DIR: ${{ github.workspace }}/khiops-samples
+          # This is needed so that OpenMPI's mpiexec can be run as root
+          OMPI_ALLOW_RUN_AS_ROOT: 1
+          OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1
+          # Force > 2 CPU cores to launch mpiexec
+          KHIOPS_PROC_NUMBER: 4
+          # Oversubscribe for MPI 4.x
+          rmaps_base_oversubscribe: true
+          # Oversubscribe for MPI > 4.x
+          OMPI_MCA_rmaps_base_oversubscribe: true
+          PRTE_MCA_rmaps_default_mapping_policy: :oversubscribe
         run: |-
           # Make sure MPI support is not loaded through env modules
           # Note: As the Docker container's shell is non-interactive, environment
           # modules are currently not initializing the shell anyway
-          if [[ -n "$MODULESHOME" ]]; then module unload mpi; fi
+          if [ -n "$MODULESHOME" ]; then module unload mpi; fi
 
           # Print khiops installation status
           kh-status