ARROW-6437: [R] Add AWS SDK to system dependencies for macOS and Windows

This adds S3 support to the binary macOS and Windows (Rtools40 only, i.e. R >= 4.0) packages.

Closes #7928 from nealrichardson/aws-sdk-r

Authored-by: Neal Richardson <neal.p.richardson@gmail.com>
Signed-off-by: Neal Richardson <neal.p.richardson@gmail.com>

Author: nealrichardson

ci/scripts/PKGBUILD

@@ -24,7 +24,8 @@ pkgdesc="Apache Arrow is a cross-language development platform for in-memory dat
 arch=("any")
 url="https://arrow.apache.org/"
 license=("Apache-2.0")
-depends=("${MINGW_PACKAGE_PREFIX}-thrift"
+depends=("${MINGW_PACKAGE_PREFIX}-aws-sdk-cpp"
+         "${MINGW_PACKAGE_PREFIX}-thrift"
          "${MINGW_PACKAGE_PREFIX}-snappy"
          "${MINGW_PACKAGE_PREFIX}-zlib"
          "${MINGW_PACKAGE_PREFIX}-lz4"
@@ -74,6 +75,9 @@ build() {
     export PATH="/C/Rtools${MINGW_PREFIX/mingw/mingw_}/bin:$PATH"
     export CPPFLAGS="${CPPFLAGS} -I${MINGW_PREFIX}/include"
     export LIBS="-L${MINGW_PREFIX}/libs"
+    export ARROW_S3=OFF
+  else
+    export ARROW_S3=ON
   fi
 
   MSYS2_ARG_CONV_EXCL="-DCMAKE_INSTALL_PREFIX=" \
@@ -94,6 +98,7 @@ build() {
     -DARROW_MIMALLOC=ON \
     -DARROW_PACKAGE_PREFIX="${MINGW_PREFIX}" \
     -DARROW_PARQUET=ON \
+    -DARROW_S3="${ARROW_S3}" \
     -DARROW_SNAPPY_USE_SHARED=OFF \
     -DARROW_USE_GLOG=OFF \
     -DARROW_WITH_LZ4=ON \

ci/scripts/r_windows_build.sh

@@ -34,6 +34,12 @@ if [ "$RTOOLS_VERSION" = "35" ]; then
   # lib-4.9.3 is for libraries compiled with gcc 4.9 (Rtools 3.5)
   RWINLIB_LIB_DIR="lib-4.9.3"
 else
+  # Uncomment L38-41 if you're testing a new rtools dependency that hasn't yet sync'd to CRAN
+  # curl https://raw.githubusercontent.com/r-windows/rtools-packages/master/pacman.conf > /etc/pacman.conf
+  # curl -OSsl "http://repo.msys2.org/msys/x86_64/msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz"
+  # pacman -U --noconfirm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz && rm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz
+  # pacman --noconfirm -Scc
+
   pacman --noconfirm -Syy
   RWINLIB_LIB_DIR="lib"
 fi
@@ -84,8 +90,8 @@ cp $MSYS_LIB_DIR/mingw64/lib/lib{thrift,snappy}.a $DST_DIR/${RWINLIB_LIB_DIR}/x6
 cp $MSYS_LIB_DIR/mingw32/lib/lib{thrift,snappy}.a $DST_DIR/${RWINLIB_LIB_DIR}/i386
 
 # These are from https://dl.bintray.com/rtools/mingw{32,64}/
-cp $MSYS_LIB_DIR/mingw64/lib/lib{zstd,lz4,crypto}.a $DST_DIR/lib/x64
-cp $MSYS_LIB_DIR/mingw32/lib/lib{zstd,lz4,crypto}.a $DST_DIR/lib/i386
+cp $MSYS_LIB_DIR/mingw64/lib/lib{zstd,lz4,crypto,aws*}.a $DST_DIR/lib/x64
+cp $MSYS_LIB_DIR/mingw32/lib/lib{zstd,lz4,crypto,aws*}.a $DST_DIR/lib/i386
 
 # Create build artifact
 zip -r ${DST_DIR}.zip $DST_DIR

dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb

@@ -32,6 +32,7 @@ class ApacheArrow < Formula
   # NOTE: if you add something here, be sure to add to PKG_LIBS in r/tools/autobrew
   depends_on "boost" => :build
   depends_on "cmake" => :build
+  depends_on "aws-sdk-cpp"
   depends_on "lz4"
   depends_on "snappy"
   depends_on "thrift"
@@ -50,14 +51,14 @@ def install
       -DARROW_JEMALLOC=ON
       -DARROW_USE_GLOG=OFF
       -DARROW_PYTHON=OFF
-      -DARROW_S3=OFF
+      -DARROW_S3=ON
       -DARROW_WITH_LZ4=ON
       -DARROW_WITH_ZLIB=ON
       -DARROW_WITH_SNAPPY=ON
       -DARROW_WITH_UTF8PROC=OFF
       -DARROW_BUILD_UTILITIES=ON
       -DARROW_VERBOSE_THIRDPARTY_BUILD=ON
-      -DCMAKE_UNITY_BUILD=ON
+      -DCMAKE_UNITY_BUILD=OFF
       -DPARQUET_BUILD_EXECUTABLES=ON
       -DLZ4_HOME=#{Formula["lz4"].prefix}
       -DTHRIFT_HOME=#{Formula["thrift"].prefix}

dev/tasks/homebrew-formulae/travis.osx.r.yml

@@ -49,6 +49,5 @@ script:
 - Rscript -e 'install.packages("rcmdcheck")'
 # Note that this is not --as-cran. CRAN doesn't do macOS checks --as-cran
 - travis_wait Rscript -e "rcmdcheck::rcmdcheck(build_args = '--no-build-vignettes', args = c('--no-manual', '--ignore-vignettes', '--run-donttest'), error_on = 'warning', check_dir = 'check')"
-after_failure:
-# If there's a build failure, it's probably in this log
-- cat arrow.Rcheck/00install.out
+# If there's a build failure, it's probably in this log. Let's print it regardless though
+- cat check/arrow.Rcheck/00install.out

r/configure

@@ -98,6 +98,8 @@ else
       if [ "$FORCE_AUTOBREW" != "true" ] && [ "`command -v brew`" ] && [ "`brew ls --versions ${PKG_BREW_NAME}`" != "" ]; then
         echo "*** Using Homebrew ${PKG_BREW_NAME}"
         BREWDIR=`brew --prefix`
+        PKG_LIBS="-L$BREWDIR/opt/$PKG_BREW_NAME/lib $PKG_LIBS -larrow_bundled_dependencies"
+        PKG_CFLAGS="-I$BREWDIR/opt/$PKG_BREW_NAME/include"
       else
         echo "*** Downloading ${PKG_BREW_NAME}"
         if [ -f "autobrew" ]; then
@@ -112,9 +114,8 @@ else
         if [ $? -ne 0 ]; then
           echo "Failed to retrieve binary for ${PKG_BREW_NAME}"
         fi
+        # autobrew sets `PKG_LIBS` and `PKG_CFLAGS`
       fi
-      PKG_CFLAGS="-I$BREWDIR/opt/$PKG_BREW_NAME/include"
-      PKG_LIBS="-L$BREWDIR/opt/$PKG_BREW_NAME/lib $PKG_LIBS -larrow_bundled_dependencies"
     elif [ "$UNAME" = "Linux" ]; then
       # Set some default values/backwards compatibility
       if [ "${LIBARROW_DOWNLOAD}" = "" ] && [ "${NOT_CRAN}" != "" ]; then

r/configure.win

@@ -44,11 +44,22 @@ else
 fi
 OPENSSL_LIBS="-lcrypto -lcrypt32"
 MIMALLOC_LIBS="-lbcrypt -lpsapi"
+AWS_LIBS="-laws-cpp-sdk-config -laws-cpp-sdk-transfer -laws-cpp-sdk-identity-management -laws-cpp-sdk-cognito-identity -laws-cpp-sdk-sts -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-c-event-stream -laws-checksums -laws-c-common -lUserenv -lversion -lws2_32 -lBcrypt -lWininet -lwinhttp"
 
 # NOTE: If you make changes to the libraries below, you should also change
 # ci/scripts/r_windows_build.sh and ci/scripts/PKGBUILD
 PKG_CFLAGS="-I${RWINLIB}/include -DARROW_STATIC -DPARQUET_STATIC -DARROW_DS_STATIC -DARROW_R_WITH_ARROW"
-PKG_LIBS="-L${RWINLIB}/lib"'$(subst gcc,,$(COMPILED_BY))$(R_ARCH) '"-L${RWINLIB}/lib"'$(R_ARCH) '"-lparquet -larrow_dataset -larrow -larrow_bundled_dependencies -lthrift -lsnappy -lz -lzstd -llz4 ${MIMALLOC_LIBS} ${OPENSSL_LIBS} -lws2_32"
+PKG_LIBS="-L${RWINLIB}/lib"'$(subst gcc,,$(COMPILED_BY))$(R_ARCH) '"-L${RWINLIB}/lib"'$(R_ARCH) '"-lparquet -larrow_dataset -larrow -larrow_bundled_dependencies -lthrift -lsnappy -lz -lzstd -llz4 ${MIMALLOC_LIBS} ${OPENSSL_LIBS}"
+
+# S3 support only for Rtools40 (i.e. R >= 4.0)
+"${R_HOME}/bin${R_ARCH_BIN}/Rscript.exe" -e 'R.version$major >= 4' | grep TRUE >/dev/null 2>&1
+if [ $? -eq 0 ]; then
+  PKG_CFLAGS="${PKG_CFLAGS} -DARROW_R_WITH_S3"
+  PKG_LIBS="${PKG_LIBS} ${AWS_LIBS}"
+else
+  # It seems that order matters
+  PKG_LIBS="${PKG_LIBS} -lws2_32"
+fi
 
 # Set any user-defined CXXFLAGS
 if [ "$ARROW_R_CXXFLAGS" ]; then

r/tools/autobrew

@@ -40,14 +40,15 @@ if [ -f "$LOCAL_FORMULA" ]; then
   $BREW deps -n "$LOCAL_FORMULA" 2>/dev/null
   BREW_DEPS=$($BREW deps -n "$LOCAL_FORMULA" 2>/dev/null)
   $BREW install --force-bottle $BREW_DEPS 2>&1 | perl -pe 's/Warning/Note/gi'
-  $BREW install --build-from-source --HEAD "$LOCAL_FORMULA" 2>&1 | perl -pe 's/Warning/Note/gi'
+  $BREW install -v --build-from-source --HEAD "$LOCAL_FORMULA" 2>&1 | perl -pe 's/Warning/Note/gi'
 else
   $BREW install --force-bottle $BREW_DEPS $PKG_BREW_NAME 2>&1 | perl -pe 's/Warning/Note/gi'
 fi
 
 # Hardcode this for my custom autobrew build
 rm -f $BREWDIR/lib/*.dylib
-PKG_LIBS="-L$BREWDIR/lib -lparquet -larrow_dataset -larrow -lthrift -llz4 -lsnappy"
+AWS_LIBS="-laws-cpp-sdk-config -laws-cpp-sdk-transfer -laws-cpp-sdk-identity-management -laws-cpp-sdk-cognito-identity -laws-cpp-sdk-sts -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-c-event-stream -laws-checksums -laws-c-common -lpthread -lcurl"
+PKG_LIBS="-L$BREWDIR/lib -lparquet -larrow_dataset -larrow -larrow_bundled_dependencies -lthrift -llz4 -lsnappy $AWS_LIBS"
 
 # Prevent CRAN builder from linking against old libs in /usr/local/lib
 for FILE in $BREWDIR/Cellar/*/*/lib/*.a; do
@@ -58,5 +59,7 @@ for FILE in $BREWDIR/Cellar/*/*/lib/*.a; do
   PKG_LIBS=`echo $PKG_LIBS | sed "s/-l$LIBNAME/-lbrew$LIBNAME/g"`
 done
 
+PKG_CFLAGS="-I$BREWDIR/opt/$PKG_BREW_NAME/include -DARROW_R_WITH_S3"
+
 unset HOMEBREW_NO_ANALYTICS
 unset HOMEBREW_NO_AUTO_UPDATE