Skip to content

Commit 45f657f

Browse files
mikediasmikedias
committed
Merge branch 'master' into mdias/master/buckets-per-partition
2 parents 2360910 + 5a64cd4 commit 45f657f

855 files changed

Lines changed: 84865 additions & 14414 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/paimon-python-checks.yml‎

Lines changed: 4 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,7 @@ jobs:
7474
curl \
7575
pkg-config \
7676
libssl-dev \
77+
libclang-dev \
7778
&& apt-get clean \
7879
&& rm -rf /var/lib/apt/lists/*
7980
@@ -132,11 +133,12 @@ jobs:
132133
else
133134
python -m pip install --upgrade pip
134135
pip install torch --index-url https://download.pytorch.org/whl/cpu
135-
python -m pip install pyroaring readerwriterlock==1.0.9 fsspec==2024.3.1 cachetools==5.3.3 ossfs==2023.12.0 ray==2.48.0 fastavro==1.11.1 pyarrow==16.0.0 zstandard==0.24.0 polars==1.32.0 duckdb==1.3.2 numpy==1.24.3 pandas==2.0.3 pylance==0.39.0 cramjam flake8==4.0.1 pytest~=7.0 py4j==0.10.9.9 requests parameterized==0.9.0 'daft>=0.7.6'
136+
python -m pip install pyroaring readerwriterlock==1.0.9 fsspec==2024.3.1 cachetools==5.3.3 ossfs==2023.12.0 ray==2.54.0 fastavro==1.11.1 pyarrow==16.0.0 zstandard==0.24.0 polars==1.32.0 duckdb==1.3.2 numpy==1.24.3 pandas==2.0.3 pylance==0.39.0 cramjam flake8==4.0.1 pytest~=7.0 py4j==0.10.9.9 requests parameterized==0.9.0 'daft>=0.7.6' pypaimon-rust==0.2.0 'datafusion>=52'
136137
python -m pip install 'lumina-data>=${{ env.LUMINA_DATA_VERSION }}' -i https://pypi.org/simple/
137138
if python -c "import sys; sys.exit(0 if sys.version_info >= (3, 11) else 1)"; then
138139
python -m pip install vortex-data==0.70.0
139140
fi
141+
python -m pip install 'paimon-mosaic>=0.1.0'
140142
fi
141143
df -h
142144
@@ -150,15 +152,6 @@ jobs:
150152
maturin build --release
151153
pip install target/wheels/tantivy-*.whl
152154
153-
- name: Build and install pypaimon-rust from source
154-
if: matrix.python-version != '3.6.15'
155-
shell: bash
156-
run: |
157-
git clone https://github.com/apache/paimon-rust.git /tmp/paimon-rust
158-
cd /tmp/paimon-rust/bindings/python
159-
maturin build --release -o dist
160-
pip install dist/pypaimon_rust-*.whl
161-
pip install 'datafusion>=52'
162155
163156
- name: Run lint-python.sh
164157
shell: bash
@@ -192,7 +185,7 @@ jobs:
192185
run: |
193186
python -m pip install --upgrade pip
194187
pip install torch --index-url https://download.pytorch.org/whl/cpu
195-
python -m pip install pyroaring readerwriterlock==1.0.9 fsspec==2024.3.1 cachetools==5.3.3 ossfs==2023.12.0 ray==2.48.0 fastavro==1.11.1 pyarrow==16.0.0 zstandard==0.24.0 polars==1.32.0 duckdb==1.3.2 numpy==1.24.3 pandas==2.0.3 pylance==0.39.0 flake8==4.0.1 pytest~=7.0 py4j==0.10.9.9 requests parameterized==0.9.0
188+
python -m pip install pyroaring readerwriterlock==1.0.9 fsspec==2024.3.1 cachetools==5.3.3 ossfs==2023.12.0 ray==2.54.0 fastavro==1.11.1 pyarrow==16.0.0 zstandard==0.24.0 polars==1.32.0 duckdb==1.3.2 numpy==1.24.3 pandas==2.0.3 pylance==0.39.0 flake8==4.0.1 pytest~=7.0 py4j==0.10.9.9 requests parameterized==0.9.0
196189
python -m pip install 'lumina-data>=${{ env.LUMINA_DATA_VERSION }}' -i https://pypi.org/simple/
197190
- name: Run lint-python.sh
198191
shell: bash

.github/workflows/utitcase-rust-native.yml renamed to .github/workflows/utcase-tantivy.yml

Lines changed: 1 addition & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -16,16 +16,14 @@
1616
# limitations under the License.
1717
################################################################################
1818

19-
name: UTCase Rust Native
19+
name: UTCase Tantivy
2020

2121
on:
2222
push:
2323
paths:
24-
- 'paimon-vortex/**'
2524
- 'paimon-tantivy/**'
2625
pull_request:
2726
paths:
28-
- 'paimon-vortex/**'
2927
- 'paimon-tantivy/**'
3028

3129
env:
@@ -37,44 +35,6 @@ concurrency:
3735
cancel-in-progress: true
3836

3937
jobs:
40-
vortex_test:
41-
runs-on: ubuntu-latest
42-
43-
steps:
44-
- name: Checkout code
45-
uses: actions/checkout@v6
46-
47-
- name: Set up JDK ${{ env.JDK_VERSION }}
48-
uses: actions/setup-java@v5
49-
with:
50-
java-version: ${{ env.JDK_VERSION }}
51-
distribution: 'temurin'
52-
53-
- name: Install Rust toolchain
54-
run: |
55-
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable --profile minimal
56-
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
57-
58-
- name: Clone and build Vortex native library
59-
run: |
60-
git clone --depth 1 -b 0.69.0 https://github.com/spiraldb/vortex.git ${RUNNER_TEMP}/vortex
61-
cd ${RUNNER_TEMP}/vortex
62-
cargo build --package vortex-jni --release
63-
64-
- name: Copy native library to resources
65-
run: |
66-
RESOURCE_DIR=paimon-vortex/paimon-vortex-jni/src/main/resources/native/linux-amd64
67-
mkdir -p ${RESOURCE_DIR}
68-
cp ${RUNNER_TEMP}/vortex/target/release/libvortex_jni.so ${RESOURCE_DIR}/
69-
70-
- name: Build and test Vortex modules
71-
timeout-minutes: 30
72-
run: |
73-
mvn -T 2C -B -ntp clean install -DskipTests
74-
mvn -B -ntp verify -pl paimon-vortex/paimon-vortex-jni,paimon-vortex/paimon-vortex-format -Dcheckstyle.skip=true -Dspotless.check.skip=true
75-
env:
76-
MAVEN_OPTS: -Xmx4096m
77-
7838
tantivy_test:
7939
runs-on: ubuntu-latest
8040

‎.github/workflows/utcase-vector-index.yml‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
################################################################################
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
################################################################################
18+
19+
name: UTCase Vector Index
20+
21+
on:
22+
push:
23+
paths:
24+
- 'paimon-vector/**'
25+
pull_request:
26+
paths:
27+
- 'paimon-vector/**'
28+
29+
env:
30+
JDK_VERSION: 8
31+
MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true
32+
33+
concurrency:
34+
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.number || github.run_id }}
35+
cancel-in-progress: true
36+
37+
jobs:
38+
vector_index_test:
39+
runs-on: ubuntu-latest
40+
41+
steps:
42+
- name: Checkout code
43+
uses: actions/checkout@v6
44+
45+
- name: Set up JDK ${{ env.JDK_VERSION }}
46+
uses: actions/setup-java@v5
47+
with:
48+
java-version: ${{ env.JDK_VERSION }}
49+
distribution: 'temurin'
50+
51+
- name: Install Rust toolchain
52+
run: |
53+
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable --profile minimal
54+
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
55+
56+
- name: Clone and build paimon-vector-index native library
57+
run: |
58+
git clone --depth 1 https://github.com/apache/paimon-vector-index.git /tmp/paimon-vector-index
59+
cd /tmp/paimon-vector-index
60+
cargo build --release -p paimon-vindex-jni
61+
62+
- name: Copy native library to resources
63+
run: |
64+
RESOURCE_DIR=paimon-vector/paimon-vector-jni/src/main/resources/native/linux-amd64
65+
mkdir -p ${RESOURCE_DIR}
66+
cp /tmp/paimon-vector-index/target/release/libpaimon_vindex_jni.so ${RESOURCE_DIR}/
67+
68+
- name: Build and test vector index modules
69+
timeout-minutes: 30
70+
run: |
71+
mvn -T 2C -B -ntp clean install -DskipTests
72+
mvn -B -ntp verify -pl paimon-vector/paimon-vector-jni,paimon-vector/paimon-vector-index -Dcheckstyle.skip=true -Dspotless.check.skip=true
73+
env:
74+
MAVEN_OPTS: -Xmx4096m

‎.idea/vcs.xml‎

Lines changed: 0 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)