Skip to content

Commit 6daf9c9

Browse files
folmos-at-orangefolmos-at-orange
authored
Merge pull request #149 from KhiopsML/dev
Release 10.2.0.0
2 parents 32755fb + 3f58477 commit 6daf9c9

188 files changed

Lines changed: 17617 additions & 14941 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.gitattributes‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,4 @@
22
.git_commit_template eol=lf
33

44
# Versioneer
5-
pykhiops/_version.py export-subst
5+
khiops/_version.py export-subst

‎.github/ISSUE_TEMPLATE/generic-issue-template.md‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
---
2+
name: Generic issue template
3+
about: Should be used for all issues of this repository.
4+
title: ''
5+
labels: ''
6+
assignees: ''
7+
8+
---
9+
10+
#### Description
11+
_What, Why_
12+
13+
#### Questions/Ideas
14+
- _A potential solution__
15+
- _A potential solution detail_
16+
- _A potential problem_
17+
- _Something I still don't understand_

‎.github/workflows/build-conda-package.yml‎

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
---
2+
name: Conda
3+
env:
4+
DEFAULT_KHIOPS_REVISION: dev
5+
DEFAULT_SAMPLES_REVISION: main
6+
on:
7+
workflow_dispatch:
8+
inputs:
9+
khiops-revision:
10+
default: dev
11+
description: khiops repo revision
12+
samples-revision:
13+
default: main
14+
description: khiops-samples repo revision
15+
push:
16+
tags: [v*]
17+
pull_request:
18+
paths: [.github/workflows/build-conda-package.yml]
19+
defaults:
20+
run:
21+
shell: bash -el {0}
22+
concurrency:
23+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
24+
cancel-in-progress: true
25+
jobs:
26+
build-packages:
27+
name: Build Packages
28+
strategy:
29+
fail-fast: false
30+
matrix:
31+
# Use the oldest supported Mac OS and Ubuntu versions for GLIBC compatibility
32+
os: [ubuntu-20.04, windows-latest, macos-11]
33+
runs-on: ${{ matrix.os }}
34+
outputs:
35+
pkg-version: ${{ steps.get-pkg-version.outputs.pkg-version }}
36+
steps:
37+
- name: Checkout Sources
38+
uses: actions/checkout@v3
39+
with:
40+
# Checkout the full repository to have the tags so versioneer works properly
41+
# See issue https://github.com/actions/checkout/issues/701
42+
fetch-depth: 0
43+
# We move KHIOPS_REVISION to the environment so that we can use
44+
# them in both push and workflow_dispatch events
45+
- name: Move KHIOPS_REVISION to the environment (push/PR)
46+
if: github.event_name == 'push' || github.event_name == 'pull_request'
47+
run: echo "KHIOPS_REVISION=${DEFAULT_KHIOPS_REVISION}" >> "$GITHUB_ENV"
48+
- name: Move KHIOPS_REVISION to the environment (workflow_dispatch)
49+
if: github.event_name == 'workflow_dispatch'
50+
run: echo "KHIOPS_REVISION=${{ inputs.khiops-revision }}" >> "$GITHUB_ENV"
51+
- name: Install Miniconda
52+
uses: conda-incubator/setup-miniconda@v3
53+
with:
54+
activate-environment: khiops-python-env
55+
- name: Install Dependency Requirements for Building Conda Packages
56+
run: conda install conda-build=3.27.0 conda-verify
57+
# We need MacOS SDK 10.10 to build on Big Sur
58+
- name: Install Mac OS SDK 10.10
59+
if: runner.os == 'macOS'
60+
run: |
61+
wget https://github.com/phracker/MacOSX-SDKs/releases/download/11.3/MacOSX10.10.sdk.tar.xz
62+
sudo tar -zxvf MacOSX10.10.sdk.tar.xz -C /opt
63+
- name: Build Khiops Conda Package (Windows)
64+
if: runner.os == 'Windows'
65+
run: |
66+
mkdir khiops-conda
67+
conda build --output-folder khiops-conda ./packaging/conda
68+
# In Linux/macOS we need the conda-forge channel to install their pinned versions
69+
- name: Build Khiops Conda Package (Linux/macOS)
70+
if: runner.os != 'Windows'
71+
run: |
72+
mkdir khiops-conda
73+
conda build --channel conda-forge --output-folder khiops-conda ./packaging/conda
74+
- name: Extract Khiops Package Version
75+
id: get-pkg-version
76+
run: |
77+
PKG_VERSION=$(\
78+
conda search --override-channels --channel ./khiops-conda/ khiops \
79+
| awk '!/#|channels/ {print $2}' \
80+
| sort -u \
81+
)
82+
echo "pkg-version=$PKG_VERSION" >> "$GITHUB_OUTPUT"
83+
- name: Upload Khiops Conda Package
84+
uses: actions/upload-artifact@v3
85+
with:
86+
name: khiops-${{ steps.get-pkg-version.outputs.pkg-version }}-conda
87+
path: ./khiops-conda
88+
retention-days: 7
89+
# Test Conda package on brand new environments
90+
test-packages:
91+
needs: build-packages
92+
strategy:
93+
fail-fast: false
94+
matrix:
95+
os:
96+
- ubuntu-20.04
97+
- ubuntu-22.04
98+
- windows-2019
99+
- windows-2022
100+
- macos-11
101+
- macos-12
102+
- macos-13
103+
python-version: ['3.8', '3.9', '3.10', '3.11']
104+
runs-on: ${{ matrix.os }}
105+
env:
106+
KHIOPS_SAMPLES_DIR: ./khiops-samples-repo
107+
steps:
108+
# We move SAMPLES_REVISION to the environment so that we can use
109+
# them in both push and workflow_dispatch events
110+
- name: Move SAMPLES_REVISION to the environment (push event)
111+
if: github.event_name == 'push'
112+
run: echo "SAMPLES_REVISION=${DEFAULT_SAMPLES_REVISION}" >> "$GITHUB_ENV"
113+
- name: Move SAMPLES_REVISION to the environment (workflow_dispatch event)
114+
if: github.event_name == 'workflow_dispatch'
115+
run: echo "SAMPLES_REVISION=${{ inputs.samples-revision }}" >> "$GITHUB_ENV"
116+
- name: Checkout Khiops samples
117+
uses: actions/checkout@v3
118+
with:
119+
repository: khiopsml/khiops-samples
120+
ref: ${{ env.SAMPLES_REVISION }}
121+
token: ${{ secrets.GITHUB_TOKEN }}
122+
path: ${{ env.KHIOPS_SAMPLES_DIR }}
123+
- name: Install Miniconda
124+
uses: conda-incubator/setup-miniconda@v3
125+
with:
126+
miniconda-version: latest # needed for Mac OS 13
127+
python-version: ${{ matrix.python-version }}
128+
activate-environment: khiops-python-env
129+
- name: Download Conda Package Artifact
130+
uses: actions/download-artifact@v3
131+
with:
132+
name: khiops-${{ needs.build-packages.outputs.pkg-version }}-conda
133+
path: khiops-conda
134+
- name: Install the Khiops Conda pagkage (Windows)
135+
if: runner.os == 'Windows'
136+
run: conda install -c ./khiops-conda/ khiops
137+
# In Linux/macOS we need the conda-forge channel to install their pinned versions
138+
- name: Install the Khiops Conda package (Linux/macOS)
139+
if: runner.os != 'Windows'
140+
run: conda install -c conda-forge -c ./khiops-conda/ khiops
141+
- name: Test Khiops Installation Status
142+
run: kh-status
143+
- name: Test Conda Package Installation on Samples
144+
run: |
145+
kh-samples core -i train_predictor -e
146+
kh-samples core -i train_predictor_error_handling -e
147+
kh-samples core -i train_coclustering -e
148+
kh-samples sklearn -i khiops_classifier -e
149+
kh-samples sklearn -i khiops_coclustering -e
150+
# Build and push Conda package release archive
151+
release-packages:
152+
# We need `build-packages` because we use its output
153+
needs: [build-packages, test-packages]
154+
runs-on: ubuntu-latest
155+
permissions:
156+
contents: write
157+
packages: read
158+
steps:
159+
- name: Release the zip archive
160+
if: github.event_name == 'push'
161+
uses: ncipollo/release-action@v1
162+
with:
163+
artifacts: khiops-${{ needs.build-packages.outputs.pkg-version }}-conda

‎.github/workflows/build-dev-container-on-push.yml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
---
2+
name: Build development container [PR]
3+
on:
4+
pull_request:
5+
paths: [packaging/docker/khiopspydev/Dockerfile]
6+
concurrency:
7+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
8+
cancel-in-progress: true
9+
jobs:
10+
build-dev-container:
11+
permissions:
12+
packages: write
13+
uses: ./.github/workflows/build-dev-container.yml

‎.github/workflows/build-dev-container.yml‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
---
2+
name: Build development container
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
khiops-revision:
7+
default: dev
8+
description: Git tag, branch or commit of the Khiops repository
9+
khiopsdev-os:
10+
default: ubuntu22.04
11+
description: Github runner OS
12+
server-revision:
13+
default: main
14+
description: Tag of the server Docker image
15+
workflow_call:
16+
inputs:
17+
khiops-revision:
18+
type: string
19+
default: dev
20+
description: Git tag, branch or commit of the Khiops repository
21+
khiopsdev-os:
22+
type: string
23+
default: ubuntu22.04
24+
description: Github runner OS
25+
server-revision:
26+
type: string
27+
default: main
28+
description: Tag of the server Docker image
29+
concurrency:
30+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
31+
cancel-in-progress: true
32+
jobs:
33+
build-dev-container:
34+
runs-on: ubuntu-latest
35+
permissions:
36+
packages: write # to write in the Github package registry
37+
steps:
38+
- name: Checkout khiops-python sources
39+
uses: actions/checkout@v3
40+
- name: Set up Docker Buildx
41+
id: buildx
42+
uses: docker/setup-buildx-action@v3
43+
- name: Login to Github Packages
44+
uses: docker/login-action@v3
45+
with:
46+
registry: ghcr.io
47+
username: ${{ github.actor }}
48+
password: ${{ secrets.GITHUB_TOKEN }}
49+
- name: Get Linux codename from OS version
50+
run: |
51+
OS_CODENAME=$(echo ${{ inputs.khiopsdev-os }} | tr -d '0-9.')
52+
echo "OS_CODENAME=$OS_CODENAME" >> "$GITHUB_ENV"
53+
- name: Build image and push to GitHub Container Registry
54+
uses: docker/build-push-action@v5
55+
with:
56+
context: ./packaging/docker/khiopspydev/
57+
file: ./packaging/docker/khiopspydev/Dockerfile.${{ env.OS_CODENAME }}
58+
build-args: |
59+
"KHIOPS_REVISION=${{ inputs.khiops-revision }}"
60+
"KHIOPSDEV_OS=${{ inputs.khiopsdev-os }}"
61+
"SERVER_REVISION=${{ inputs.server-revision }}"
62+
tags: ghcr.io/khiopsml/khiops-python/khiopspydev-${{ inputs.khiopsdev-os }}:latest
63+
push: true
64+
- name: Display the image digest
65+
run: echo ${{ steps.docker_build.outputs.digest }}
66+
test-dev-container:
67+
needs: build-dev-container
68+
runs-on: ubuntu-latest
69+
container:
70+
image: ghcr.io/khiopsml/khiops-python/khiopspydev-${{ inputs.khiopsdev-os }}:latest
71+
steps:
72+
- name: Check Khiops executables
73+
run: |-
74+
khiops -v
75+
khiops_coclustering -v

‎.github/workflows/build-pip-package.yml‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
---
2+
name: Build Pip Package
3+
env:
4+
DEFAULT_SAMPLES_REVISION: main
5+
on:
6+
workflow_dispatch:
7+
inputs:
8+
samples-revision:
9+
default: main
10+
description: Git tag, branch or commit for the khiops-samples repository
11+
pull_request:
12+
concurrency:
13+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
14+
cancel-in-progress: true
15+
jobs:
16+
build-and-test-pip-package:
17+
strategy:
18+
fail-fast: false
19+
matrix:
20+
container: [ubuntu22.04, rocky9]
21+
runs-on: ubuntu-latest
22+
container:
23+
image: ghcr.io/khiopsml/khiops-python/khiopspydev-${{ matrix.container }}:latest
24+
permissions:
25+
checks: write
26+
contents: read
27+
id-token: write
28+
packages: read
29+
steps:
30+
- name: Checkout sources
31+
uses: actions/checkout@v3
32+
with:
33+
# Get Git tags so that versioneer can function correctly
34+
# See issue https://github.com/actions/checkout/issues/701
35+
fetch-depth: 0
36+
# We move SAMPLES_REVISION to the environment so that we can use
37+
# them in both push and workflow_dispatch events
38+
- name: Move SAMPLES_REVISION to the environment (push event)
39+
if: github.event_name == 'push'
40+
run: echo "SAMPLES_REVISION=${DEFAULT_SAMPLES_REVISION}" >> "$GITHUB_ENV"
41+
- name: Move SAMPLES_REVISION to the environment (workflow_dispatch event)
42+
if: github.event_name == 'workflow_dispatch'
43+
run: echo "SAMPLES_REVISION=${{ inputs.samples-revision }}" >> "$GITHUB_ENV"
44+
- name: Checkout Khiops samples
45+
uses: actions/checkout@v3
46+
with:
47+
repository: khiopsml/khiops-samples
48+
ref: ${{ env.SAMPLES_REVISION }}
49+
token: ${{ secrets.GITHUB_TOKEN }}
50+
path: khiops-samples
51+
- name: Build and Install Pip Package
52+
run: |
53+
# This is needed so that the Git tag is parsed and the khiops-python
54+
# version is retrieved
55+
git config --global --add safe.directory $(realpath .)
56+
python3 setup.py sdist --dist-dir=./test_dist
57+
pip install $(ls ./test_dist/*.tar.gz)
58+
- name: Test Pip package
59+
env:
60+
KHIOPS_SAMPLES_DIR: ${{ github.workspace }}/khiops-samples
61+
run: |-
62+
# Make sure MPI support is not loaded through env modules
63+
# Note: As Docker container's shell is non-interactive, environment
64+
# modules are currently not initializing the shell anyway
65+
if [[ -n "$MODULESHOME" ]]; then module unload mpi; fi
66+
cd ./test_dist
67+
kh-samples core -i train_predictor -e
68+
kh-samples sklearn -i khiops_classifier -e
69+
# Test that the line containing "MPI command" also contains
70+
# "mpiexec", which means that `mpiexec` has been found
71+
kh-status | grep "MPI command" | grep -wq mpiexec

0 commit comments

Comments
 (0)