ARROW-10385: [C++][Gandiva] Add support for LLVM 11

LLVM link failures in MinGW build are caused by msys2/MINGW-packages#7170 .
They aren't fixed by this pull request. The problem will be fixed in upstream.

Closes apache#8521 from kou/cpp-llvm-11

Authored-by: Sutou Kouhei <kou@clear-code.com>
Signed-off-by: Sutou Kouhei <kou@clear-code.com>

Author: kou

.env

@@ -24,9 +24,9 @@ ARCH=amd64
 CUDA=9.1
 DEBIAN=10
 UBUNTU=18.04
-FEDORA=32
+FEDORA=33
 PYTHON=3.6
-LLVM=10
+LLVM=11
 CLANG_TOOLS=8
 RUST=nightly-2020-04-22
 GO=1.12

.github/workflows/cpp.yml

@@ -336,7 +336,6 @@ jobs:
       CMAKE_ARGS: >-
         -DARROW_PACKAGE_PREFIX=/mingw${{ matrix.mingw-n-bits }}
         -DBoost_NO_BOOST_CMAKE=ON
-      CMAKE_GENERATOR: MSYS Makefiles
       CMAKE_UNITY_BUILD: ON
     steps:
       - name: Disable Crash Dialogs

.github/workflows/cpp_cron.yml

@@ -46,7 +46,7 @@ jobs:
       matrix:
         name:
           - amd64-debian-10-cpp
-          - amd64-fedora-32-cpp
+          - amd64-fedora-33-cpp
           - amd64-ubuntu-16.04-cpp
           - amd64-ubuntu-18.04-cpp
           - amd64-ubuntu-18.04-cpp-cmake32
@@ -55,10 +55,10 @@ jobs:
             image: debian-cpp
             title: AMD64 Debian 10 C++
             debian: 10
-          - name: amd64-fedora-32-cpp
+          - name: amd64-fedora-33-cpp
             image: fedora-cpp
-            title: AMD64 Fedora 32 C++
-            fedora: 32
+            title: AMD64 Fedora 33 C++
+            fedora: 33
           - name: amd64-ubuntu-16.04-cpp
             image: ubuntu-cpp
             title: AMD64 Ubuntu 16.04 C++
@@ -75,7 +75,7 @@ jobs:
       # the defaults here should correspond to the values in .env
       ARCH: 'amd64'
       DEBIAN: ${{ matrix.debian || 10 }}
-      FEDORA: ${{ matrix.fedora || 32 }}
+      FEDORA: ${{ matrix.fedora || 33 }}
       UBUNTU: ${{ matrix.ubuntu || 18.04 }}
     steps:
       - name: Checkout Arrow

.github/workflows/python_cron.yml

@@ -45,7 +45,7 @@ jobs:
       matrix:
         name:
           - debian-10-python-3
-          - fedora-32-python-3
+          - fedora-33-python-3
           - ubuntu-18.04-python-3
           - conda-python-3.7-dask-latest
           - conda-python-3.7-turbodbc-latest
@@ -59,11 +59,11 @@ jobs:
             image: debian-python
             title: AMD64 Debian 10 Python 3
             debian: 10
-          - name: fedora-32-python-3
-            cache: fedora-32-python-3
+          - name: fedora-33-python-3
+            cache: fedora-33-python-3
             image: fedora-python
-            title: AMD64 Fedora 32 Python 3
-            fedora: 32
+            title: AMD64 Fedora 33 Python 3
+            fedora: 33
           - name: ubuntu-18.04-python-3
             cache: ubuntu-18.04-python-3
             image: ubuntu-python
@@ -102,7 +102,7 @@ jobs:
     env:
       # the defaults here should correspond to the values in .env
       DEBIAN: ${{ matrix.debian || 10 }}
-      FEDORA: ${{ matrix.fedora || 32 }}
+      FEDORA: ${{ matrix.fedora || 33 }}
       UBUNTU: ${{ matrix.ubuntu || 18.04 }}
       PYTHON: ${{ matrix.python || 3.7 }}
       HDFS: ${{ matrix.hdfs || '2.9.2' }}

.github/workflows/ruby.yml

@@ -187,7 +187,6 @@ jobs:
       CMAKE_ARGS: >-
         -DARROW_PACKAGE_PREFIX=/mingw${{ matrix.mingw-n-bits }}
         -DBoost_NO_BOOST_CMAKE=ON
-      CMAKE_GENERATOR: MSYS Makefiles
       CMAKE_UNITY_BUILD: ON
     steps:
       - name: Disable Crash Dialogs

.travis.yml

@@ -40,6 +40,34 @@ jobs:
         ARCH: arm64v8
         ARROW_CI_MODULES: "CPP"
         DOCKER_IMAGE_ID: ubuntu-cpp
+        # ARROW_USE_GLOG=OFF is needed to avoid build error caused by
+        # glog and CMAKE_UNITY_BUILD=ON.
+        #
+        # Disable ARROW_S3 because it often causes "No output has
+        # been received in the last 10m0s, this potentially indicates
+        # a stalled build or something wrong with the build itself."
+        # on Travis CI.
+        DOCKER_RUN_ARGS: >-
+          "
+          -e ARROW_BUILD_STATIC=OFF
+          -e ARROW_ORC=OFF
+          -e ARROW_S3=OFF
+          -e ARROW_USE_GLOG=OFF
+          -e CMAKE_UNITY_BUILD=ON
+          "
+        # We need to use smaller build when cache doesn't exist
+        # because Travis CI has "No output has been received in the
+        # last 10m0s" limitation. If we build many modules, we reach
+        # the limitation.
+        DOCKER_RUN_ARGS_NO_CACHE: >-
+          "
+          -e ARROW_BUILD_TESTS=OFF
+          -e ARROW_GANDIVA=OFF
+          -e ARROW_PARQUET=OFF
+          "
+        # The LLVM's APT repository provides only arm64 binaries.
+        # We should use LLVM provided by Ubuntu.
+        LLVM: "10"
         UBUNTU: "20.04"
     - name: "C++ on s390x"
       os: linux
@@ -50,9 +78,25 @@ jobs:
         ARROW_FLIGHT: "ON"
         ARROW_PARQUET: "OFF"
         DOCKER_IMAGE_ID: ubuntu-cpp
-        PARQUET_BUILD_EXAMPLES: "OFF"
-        PARQUET_BUILD_EXECUTABLES: "OFF"
-        Protobuf_SOURCE: "BUNDLED"
+        # Can't use CMAKE_UNITIFY_BUILD=ON because of compiler crash.
+        # Can't enable ARROW_S3 because compiler is killed while compiling
+        # aws-sdk-cpp.
+        DOCKER_RUN_ARGS: >-
+          "
+          -e ARROW_BUILD_STATIC=OFF
+          -e ARROW_FLIGHT=ON
+          -e ARROW_ORC=OFF
+          -e ARROW_PARQUET=OFF
+          -e ARROW_S3=OFF
+          -e PARQUET_BUILD_EXAMPLES=OFF
+          -e PARQUET_BUILD_EXECUTABLES=OFF
+          -e Protobuf_SOURCE=BUNDLED
+          -e cares_SOURCE=BUNDLED
+          -e gRPC_SOURCE=BUNDLED
+          "
+        # The LLVM's APT repository provides only arm64 binaries.
+        # We should use LLVM provided by Ubuntu.
+        LLVM: "10"
         UBUNTU: "20.04"
         cares_SOURCE: "BUNDLED"
         gRPC_SOURCE: "BUNDLED"

ci/conda_env_gandiva.yml

@@ -15,6 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-clangdev=10
-llvmdev=10
+clangdev=11
+llvmdev=11
 re2

ci/docker/fedora-32-cpp.dockerfile ci/docker/fedora-33-cpp.dockerfileci/docker/fedora-32-cpp.dockerfile renamed to ci/docker/fedora-33-cpp.dockerfile

@@ -16,7 +16,8 @@
 # under the License.
 
 ARG arch
-FROM ${arch}/fedora:32
+FROM ${arch}/fedora:33
+ARG arch
 
 # install dependencies
 RUN dnf update -y && \
@@ -30,24 +31,26 @@ RUN dnf update -y && \
         clang-devel \
         cmake \
         flatbuffers-devel \
-        java-1.8.0-openjdk-devel \
-        java-1.8.0-openjdk-headless \
         gcc \
         gcc-c++ \
-        glog-devel \
         gflags-devel \
+        git \
+        glog-devel \
         gmock-devel \
         google-benchmark-devel \
-        protobuf-devel \
+        grpc-devel \
+        grpc-plugins \
         gtest-devel \
-        git \
+        java-latest-openjdk-devel \
+        java-latest-openjdk-headless \
         libzstd-devel \
         llvm-devel \
         llvm-static \
         lz4-devel \
         make \
         ninja-build \
         openssl-devel \
+        protobuf-devel \
         python \
         rapidjson-devel \
         re2-devel \
@@ -56,7 +59,10 @@ RUN dnf update -y && \
         which \
         zlib-devel
 
-# * gRPC 1.26 in Fedora 32 may have a problem. arrow-flight-test is stuck.
+COPY ci/scripts/install_minio.sh \
+     /arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
+
 ENV ARROW_BUILD_TESTS=ON \
     ARROW_DEPENDENCY_SOURCE=SYSTEM \
     ARROW_DATASET=ON \
@@ -75,7 +81,6 @@ ENV ARROW_BUILD_TESTS=ON \
     ARROW_WITH_ZSTD=ON \
     CC=gcc \
     CXX=g++ \
-    gRPC_SOURCE=BUNDLED \
     ORC_SOURCE=BUNDLED \
     PARQUET_BUILD_EXECUTABLES=ON \
     PARQUET_BUILD_EXAMPLES=ON \

ci/docker/linux-apt-c-glib.dockerfile

@@ -25,6 +25,7 @@ RUN apt-get update -y -q && \
         gtk-doc-tools \
         libgirepository1.0-dev \
         libglib2.0-doc \
+        lsb-release \
         luarocks \
         pkg-config \
         ruby-dev && \

ci/docker/ubuntu-20.04-cpp.dockerfile

@@ -29,7 +29,22 @@ RUN echo "debconf debconf/frontend select Noninteractive" | \
 # while debugging package list with docker build.
 ARG clang_tools
 ARG llvm
-RUN apt-get update -y -q && \
+RUN if [ "${llvm}" -gt "10" ]; then \
+      apt-get update -y -q && \
+      apt-get install -y -q --no-install-recommends \
+          apt-transport-https \
+          ca-certificates \
+          gnupg \
+          wget && \
+      wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
+      echo "deb https://apt.llvm.org/focal/ llvm-toolchain-focal-${llvm} main" > \
+         /etc/apt/sources.list.d/llvm.list && \
+      if [ "${clang_tools}" != "${llvm}" -a "${clang_tools}" -gt 10 ]; then \
+        echo "deb https://apt.llvm.org/focal/ llvm-toolchain-focal-${clang_tools} main" > \
+           /etc/apt/sources.list.d/clang-tools.list; \
+      fi \
+    fi && \
+    apt-get update -y -q && \
     apt-get install -y -q --no-install-recommends \
         clang-${clang_tools} \
         clang-${llvm} \