ARROW-9004: [C++][Gandiva] Support building with LLVM 10

LLVM 7 and 8 are still supported.

Clang Tools still use 8 because Clang Tools 10 reports an error for
RaipdJSON in sanitizer build. We should work on this as a separated
task.

e.g.:

https://github.com/kou/arrow/runs/725786805?check_suite_focus=true

    /usr/include/rapidjson/internal/stack.h:117:13: runtime error: applying non-zero offset 16 to null pointer
         #0 0x7f4d336c0e2c in void arrow::rapidjson::internal::Stack<arrow::rapidjson::CrtAllocator>::Reserve<arrow::rapidjson::GenericValue<arrow::rapidjson::UTF8<char>, arrow::rapidjson::MemoryPoolAllocator<arrow::rapidjson::CrtAllocator> > >(unsigned long) /usr/include/rapidjson/internal/stack.h:117:13

Ubuntu 16.04 still use LLVM 8 because LLVM 10 requires C++ 14 but g++
5 on Ubuntu 16.04 is too old to use
cpp/src/arrow/vendored/datetime/date.h.

CentOS 8 packages still use LLVM 8 because there are no packages for
LLVM 10 on CentOS 8.

manylinux1 still use LLVM 8 because g++ 4.8.2 is old to build LLVM 9
or later.

Conda package for Windows still use LLVM 8 because llvmdev 9 or later
use Visual Studio 2017. We need to use Visual Studio 2017 not 2015 to
use llvmdev 9 or later.

Closes #7323 from kou/cpp-llvm-10

Authored-by: Sutou Kouhei <kou@clear-code.com>
Signed-off-by: Sutou Kouhei <kou@clear-code.com>

Author: kou

.env

@@ -24,9 +24,9 @@ ARCH=amd64
 CUDA=9.1
 DEBIAN=10
 UBUNTU=18.04
-FEDORA=30
+FEDORA=32
 PYTHON=3.6
-LLVM=8
+LLVM=10
 CLANG_TOOLS=8
 RUST=nightly-2020-04-22
 GO=1.12

.github/workflows/cpp_cron.yml

@@ -47,7 +47,7 @@ jobs:
       matrix:
         name:
           - debian-10-cpp
-          - fedora-30-cpp
+          - fedora-32-cpp
           - ubuntu-16.04-cpp
           - ubuntu-18.04-cpp
           - ubuntu-18.04-cpp-cmake32
@@ -56,10 +56,10 @@ jobs:
             image: debian-cpp
             title: AMD64 Debian 10 C++
             debian: 10
-          - name: fedora-30-cpp
+          - name: fedora-32-cpp
             image: fedora-cpp
-            title: AMD64 Fedora 30 C++
-            fedora: 30
+            title: AMD64 Fedora 32 C++
+            fedora: 32
           - name: ubuntu-16.04-cpp
             image: ubuntu-cpp
             title: AMD64 Ubuntu 16.04 C++
@@ -75,7 +75,7 @@ jobs:
     env:
       # the defaults here should correspond to the values in .env
       DEBIAN: ${{ matrix.debian || 10 }}
-      FEDORA: ${{ matrix.fedora || 30 }}
+      FEDORA: ${{ matrix.fedora || 32 }}
       UBUNTU: ${{ matrix.ubuntu || 18.04 }}
     steps:
       - name: Checkout Arrow

.github/workflows/python_cron.yml

@@ -45,7 +45,7 @@ jobs:
       matrix:
         name:
           - debian-10-python-3
-          - fedora-30-python-3
+          - fedora-32-python-3
           - ubuntu-18.04-python-3
           - conda-python-3.7-dask-latest
           - conda-python-3.7-turbodbc-latest
@@ -59,11 +59,11 @@ jobs:
             image: debian-python
             title: AMD64 Debian 10 Python 3
             debian: 10
-          - name: fedora-30-python-3
-            cache: fedora-30-python-3
+          - name: fedora-32-python-3
+            cache: fedora-32-python-3
             image: fedora-python
-            title: AMD64 Fedora 30 Python 3
-            fedora: 30
+            title: AMD64 Fedora 32 Python 3
+            fedora: 32
           - name: ubuntu-18.04-python-3
             cache: ubuntu-18.04-python-3
             image: ubuntu-python
@@ -102,7 +102,7 @@ jobs:
     env:
       # the defaults here should correspond to the values in .env
       DEBIAN: ${{ matrix.debian || 10 }}
-      FEDORA: ${{ matrix.fedora || 30 }}
+      FEDORA: ${{ matrix.fedora || 32 }}
       UBUNTU: ${{ matrix.ubuntu || 18.04 }}
       PYTHON: ${{ matrix.python || 3.7 }}
       HDFS: ${{ matrix.hdfs || '2.9.2' }}

LICENSE.txt

@@ -50,7 +50,7 @@ set CONDA_PACKAGES=
 
 if "%ARROW_BUILD_GANDIVA%" == "ON" (
   @rem Install llvmdev in the toolchain if building gandiva.dll
-  set CONDA_PACKAGES=%CONDA_PACKAGES% --file=ci\conda_env_gandiva.yml
+  set CONDA_PACKAGES=%CONDA_PACKAGES% --file=ci\conda_env_gandiva_win.yml
 )
 if "%JOB%" == "Toolchain" (
   @rem Install pre-built "toolchain" packages for faster builds

ci/appveyor-cpp-setup.bat

@@ -15,6 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-clangdev=8
-llvmdev=8
+clangdev=10
+llvmdev=10
 re2

ci/conda_env_gandiva.yml

@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# llvmdev=9 or later require Visual Studio 2017
+clangdev=8
+llvmdev=8
+re2

ci/conda_env_gandiva_win.yml

@@ -26,6 +26,15 @@ RUN \
 
 ARG llvm
 RUN apt-get update -y -q && \
+     apt-get install -y -q --no-install-recommends \
+        apt-transport-https \
+        ca-certificates \
+        gnupg \
+        wget && \
+    wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
+    echo "deb https://apt.llvm.org/buster/ llvm-toolchain-buster-${llvm} main" > \
+        /etc/apt/sources.list.d/llvm.list && \
+    apt-get update -y -q && \
     apt-get install -y -q --no-install-recommends \
         autoconf \
         ca-certificates \
@@ -58,8 +67,7 @@ RUN apt-get update -y -q && \
         protobuf-compiler \
         rapidjson-dev \
         tzdata \
-        zlib1g-dev \
-        wget && \
+        zlib1g-dev && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 

ci/docker/debian-10-cpp.dockerfile

@@ -16,7 +16,7 @@
 # under the License.
 
 ARG arch
-FROM ${arch}/fedora:30
+FROM ${arch}/fedora:32
 
 # install dependencies
 RUN dnf update -y && \
@@ -25,19 +25,21 @@ RUN dnf update -y && \
         boost-devel \
         brotli-devel \
         bzip2-devel \
+        c-ares-devel \
         ccache \
         clang-devel \
         cmake \
         flatbuffers-devel \
-        java-openjdk-devel \
-        java-openjdk-headless \
+        java-1.8.0-openjdk-devel \
+        java-1.8.0-openjdk-headless \
         gcc \
         gcc-c++ \
         glog-devel \
         gflags-devel \
-        gtest-devel \
         gmock-devel \
         google-benchmark-devel \
+        protobuf-devel \
+        gtest-devel \
         git \
         libzstd-devel \
         llvm-devel \
@@ -50,17 +52,17 @@ RUN dnf update -y && \
         rapidjson-devel \
         re2-devel \
         snappy-devel \
+        thrift-devel \
+        which \
         zlib-devel
 
-# * c-ares cmake config is not installed on Fedora but gRPC needs it
-#   when built via ExternalProject: https://bugzilla.redhat.com/show_bug.cgi?id=1687844
-# * protobuf libraries in Fedora 30 are too old for gRPC
+# * gRPC 1.26 in Fedora 32 may have a problem. arrow-flight-test is stuck.
 ENV ARROW_BUILD_TESTS=ON \
     ARROW_DEPENDENCY_SOURCE=SYSTEM \
     ARROW_DATASET=ON \
     ARROW_FLIGHT=ON \
     ARROW_GANDIVA_JAVA=ON \
-    ARROW_GANDIVA=OFF \
+    ARROW_GANDIVA=ON \
     ARROW_HOME=/usr/local \
     ARROW_ORC=ON \
     ARROW_PARQUET=ON \
@@ -71,13 +73,10 @@ ENV ARROW_BUILD_TESTS=ON \
     ARROW_WITH_SNAPPY=ON \
     ARROW_WITH_ZLIB=ON \
     ARROW_WITH_ZSTD=ON \
-    cares_SOURCE=BUNDLED \
     CC=gcc \
     CXX=g++ \
     gRPC_SOURCE=BUNDLED \
     ORC_SOURCE=BUNDLED \
     PARQUET_BUILD_EXECUTABLES=ON \
     PARQUET_BUILD_EXAMPLES=ON \
-    PATH=/usr/lib/ccache/:$PATH \
-    Protobuf_SOURCE=BUNDLED \
-    Thrift_SOURCE=BUNDLED
+    PATH=/usr/lib/ccache/:$PATH

ci/docker/fedora-30-cpp.dockerfile ci/docker/fedora-32-cpp.dockerfileci/docker/fedora-30-cpp.dockerfile renamed to ci/docker/fedora-32-cpp.dockerfile

@@ -22,7 +22,10 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 ENV DEBIAN_FRONTEND noninteractive
 
-ARG llvm
+# LLVM 10 or later requires C++ 14 but g++-5's C++ 14 support is limited.
+# cpp/src/arrow/vendored/datetime/date.h doesn't work.
+# ARG llvm
+ENV llvm=8
 RUN apt-get update -y -q && \
     apt-get install -y -q --no-install-recommends \
         apt-transport-https \