ARROW-14638: [C++][R] Unknown C compiler / ccache on Arch Linux

Closes apache#11666 from jonkeane/ARROW-14638-ccache Lead-authored-by: Jonathan Keane <[email protected]> Co-authored-by: Neal Richardson <[email protected]> Signed-off-by: Jonathan Keane <[email protected]>
NetEase · Apr 21, 2022 · f7bccc5 · f7bccc5
1 parent a40102a
commit f7bccc5
Show file tree

Hide file tree

Showing 7 changed files with 50 additions and 4 deletions.
diff --git a/.env b/.env
@@ -73,11 +73,13 @@ SPARK=master
 TURBODBC=latest
 
 # These correspond to images on Docker Hub that contain R, e.g. rhub/ubuntu-gcc-release:latest
-ARROW_R_DEV=TRUE
 R_IMAGE=ubuntu-gcc-release
 R_ORG=rhub
-R_PRUNE_DEPS=FALSE
 R_TAG=latest
+
+# Env vars for R builds
+ARROW_R_DEV=TRUE
+R_PRUNE_DEPS=FALSE
 TZ=UTC
 
 # -1 does not attempt to install a devtoolset version, any positive integer will install devtoolset-n

diff --git a/ci/docker/linux-r.dockerfile b/ci/docker/linux-r.dockerfile
@@ -33,6 +33,9 @@ ENV DEVTOOLSET_VERSION=${devtoolset_version}
 ARG r_prune_deps=FALSE
 ENV R_PRUNE_DEPS=${r_prune_deps}
 
+ARG r_custom_ccache=false
+ENV R_CUSTOM_CCACHE=${r_custom_ccache}
+
 ARG tz="UTC"
 ENV TZ=${tz}
 

diff --git a/ci/scripts/r_docker_configure.sh b/ci/scripts/r_docker_configure.sh
@@ -42,6 +42,30 @@ else
   apt-get update
 fi
 
+# Enable ccache if requested based on http://dirk.eddelbuettel.com/blog/2017/11/27/
+: ${R_CUSTOM_CCACHE:=FALSE}
+R_CUSTOM_CCACHE=`echo $R_CUSTOM_CCACHE | tr '[:upper:]' '[:lower:]'`
+if [ ${R_CUSTOM_CCACHE} = "true" ]; then
+  # install ccache
+  $PACKAGE_MANAGER install -y epel-release || true
+  $PACKAGE_MANAGER install -y ccache
+
+  mkdir -p ~/.R
+  echo "VER=
+CCACHE=ccache
+CC=\$(CCACHE) gcc\$(VER)
+CXX=\$(CCACHE) g++\$(VER)
+CXX11=\$(CCACHE) g++\$(VER)" >> ~/.R/Makevars
+
+  mkdir -p ~/.ccache/
+  echo "max_size = 5.0G
+# important for R CMD INSTALL *.tar.gz as tarballs are expanded freshly -> fresh ctime
+sloppiness = include_file_ctime
+# also important as the (temp.) directory name will differ
+hash_dir = false" >> ~/.ccache/ccache.conf
+fi
+
+
 # Special hacking to try to reproduce quirks on fedora-clang-devel on CRAN
 # which uses a bespoke clang compiled to use libc++
 # https://www.stats.ox.ac.uk/pub/bdr/Rconfig/r-devel-linux-x86_64-fedora-clang

diff --git a/dev/tasks/r/azure.linux.yml b/dev/tasks/r/azure.linux.yml
@@ -43,6 +43,7 @@ jobs:
           export R_IMAGE={{ r_image }}
           export R_TAG={{ r_tag }}
           export DEVTOOLSET_VERSION={{ devtoolset_version|default("-1") }}
+          export R_CUSTOM_CCACHE={{ r_custom_ccache|default("false") }}
           docker-compose pull --ignore-pull-failures r
           docker-compose build r
         displayName: Docker build

diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
@@ -1279,6 +1279,15 @@ tasks:
     template: r/github.linux.offline.build.yml
 
 
+  test-r-rhub-debian-gcc-release-custom-ccache:
+    ci: azure
+    template: r/azure.linux.yml
+    params:
+      r_org: rhub
+      r_image: debian-gcc-release
+      r_tag: latest
+      r_custom_ccache: true
+
 {% for r_org, r_image, r_tag in [("rhub", "ubuntu-gcc-release", "latest"),
                                  ("rocker", "r-base", "latest"),
                                  ("rstudio", "r-base", "4.1-focal"),

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1209,6 +1209,7 @@ services:
         devtoolset_version: ${DEVTOOLSET_VERSION}
         tz: ${TZ}
         r_prune_deps: ${R_PRUNE_DEPS}
+        r_custom_ccache: ${R_CUSTOM_CCACHE}
     shm_size: *shm-size
     environment:
       LIBARROW_DOWNLOAD: "false"

diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R
@@ -301,8 +301,14 @@ build_libarrow <- function(src_dir, dst_dir) {
     # EXTRA_CMAKE_FLAGS will often be "", but it's convenient later to have it defined
     EXTRA_CMAKE_FLAGS = Sys.getenv("EXTRA_CMAKE_FLAGS"),
     # Make sure we build with the same compiler settings that R is using
-    CC = R_CMD_config("CC"),
-    CXX = paste(R_CMD_config("CXX11"), R_CMD_config("CXX11STD")),
+    # Exception: if you've added ccache to CC and CXX following
+    # http://dirk.eddelbuettel.com/blog/2017/11/27/, some libarrow
+    # third party dependencies will error on compilation. But don't
+    # worry, `ARROW_USE_CCACHE=ON` by default, so if ccache
+    # is found, it will be used by the libarrow build, and this does
+    # not affect how R compiles the arrow bindings.
+    CC = sub("^.*ccache", "", R_CMD_config("CC")),
+    CXX = paste(sub("^.*ccache", "", R_CMD_config("CXX11")), R_CMD_config("CXX11STD")),
     # CXXFLAGS = R_CMD_config("CXX11FLAGS"), # We don't want the same debug symbols
     LDFLAGS = R_CMD_config("LDFLAGS")
   )