ARROW-17423: [CI][C++] Fix building CUDA docker images (apache#13896)

* Update the CUDA runtime version as CUDA 9.1 images are not available anymore * Fix passing child command arguments to "docker run" Checked locally under a Ubuntu 20.04 host with: ``` UBUNTU=18.04 archery --debug docker run ubuntu-cuda-cpp UBUNTU=20.04 archery --debug docker run ubuntu-cuda-cpp ``` Authored-by: Antoine Pitrou <[email protected]> Signed-off-by: Antoine Pitrou <[email protected]>
NetEase · Aug 17, 2022 · f6127fc · f6127fc
1 parent 050876c
commit f6127fc
Show file tree

Hide file tree

Showing 12 changed files with 41 additions and 27 deletions.
diff --git a/.env b/.env
@@ -54,7 +54,7 @@ UBUNTU=20.04
 
 # Default versions for various dependencies
 CLANG_TOOLS=12
-CUDA=9.1
+CUDA=11.0.3
 DASK=latest
 DOTNET=6.0
 GCC_VERSION=""

diff --git a/dev/archery/archery/docker/core.py b/dev/archery/archery/docker/core.py
@@ -16,14 +16,15 @@
 # under the License.
 
 import os
-import re
+import shlex
 import subprocess
 from io import StringIO
 
 from dotenv import dotenv_values
 from ruamel.yaml import YAML
 
 from ..utils.command import Command, default_bin
+from ..utils.source import arrow_path
 from ..compat import _ensure_path
 
 
@@ -42,12 +43,6 @@ def flatten(node, parents=None):
         raise TypeError(node)
 
 
-def _sanitize_command(cmd):
-    if isinstance(cmd, list):
-        cmd = " ".join(cmd)
-    return re.sub(r"\s+", " ", cmd)
-
-
 _arch_short_mapping = {
     'arm64v8': 'arm64',
 }
@@ -294,7 +289,7 @@ def _build(service, use_cache):
 
                 args.extend([
                     '--output', 'type=docker',
-                    '-f', service['build']['dockerfile'],
+                    '-f', arrow_path(service['build']['dockerfile']),
                     '-t', service['image'],
                     service['build'].get('context', '.')
                 ])
@@ -306,7 +301,7 @@ def _build(service, use_cache):
                 for img in cache_from:
                     args.append('--cache-from="{}"'.format(img))
                 args.extend([
-                    '-f', service['build']['dockerfile'],
+                    '-f', arrow_path(service['build']['dockerfile']),
                     '-t', service['image'],
                     service['build'].get('context', '.')
                 ])
@@ -381,10 +376,9 @@ def run(self, service_name, command=None, *, env=None, volumes=None,
             if command is not None:
                 args.append(command)
             else:
-                # replace whitespaces from the preformatted compose command
-                cmd = _sanitize_command(service.get('command', ''))
+                cmd = service.get('command', '')
                 if cmd:
-                    args.append(cmd)
+                    args.extend(shlex.split(cmd))
 
             # execute as a plain docker cli command
             self._execute_docker('run', '--rm', *args)

diff --git a/dev/archery/archery/docker/tests/test_docker.py b/dev/archery/archery/docker/tests/test_docker.py
@@ -509,7 +509,7 @@ def test_image_with_gpu(arrow_compose_path):
             "-e", "OTHER_ENV=2",
             "-v", "/host:/container:rw",
             "org/ubuntu-cuda",
-            '/bin/bash -c "echo 1 > /tmp/dummy && cat /tmp/dummy"'
+            "/bin/bash", "-c", "echo 1 > /tmp/dummy && cat /tmp/dummy",
         ]
     ]
     with assert_docker_calls(compose, expected_calls):

diff --git a/dev/archery/archery/integration/runner.py b/dev/archery/archery/integration/runner.py
@@ -35,8 +35,8 @@
 from .tester_java import JavaTester
 from .tester_js import JSTester
 from .tester_csharp import CSharpTester
-from .util import (ARROW_ROOT_DEFAULT, guid, SKIP_ARROW, SKIP_FLIGHT,
-                   printer)
+from .util import guid, SKIP_ARROW, SKIP_FLIGHT, printer
+from ..utils.source import ARROW_ROOT_DEFAULT
 from . import datagen
 
 

diff --git a/dev/archery/archery/integration/tester_cpp.py b/dev/archery/archery/integration/tester_cpp.py
@@ -20,7 +20,8 @@
 import subprocess
 
 from .tester import Tester
-from .util import run_cmd, ARROW_ROOT_DEFAULT, log
+from .util import run_cmd, log
+from ..utils.source import ARROW_ROOT_DEFAULT
 
 
 _EXE_PATH = os.environ.get(

diff --git a/dev/archery/archery/integration/tester_csharp.py b/dev/archery/archery/integration/tester_csharp.py
@@ -18,7 +18,8 @@
 import os
 
 from .tester import Tester
-from .util import run_cmd, ARROW_ROOT_DEFAULT, log
+from .util import run_cmd, log
+from ..utils.source import ARROW_ROOT_DEFAULT
 
 
 _EXE_PATH = os.path.join(

diff --git a/dev/archery/archery/integration/tester_java.py b/dev/archery/archery/integration/tester_java.py
@@ -20,7 +20,8 @@
 import subprocess
 
 from .tester import Tester
-from .util import run_cmd, ARROW_ROOT_DEFAULT, log
+from .util import run_cmd, log
+from ..utils.source import ARROW_ROOT_DEFAULT
 
 
 def load_version_from_pom():

diff --git a/dev/archery/archery/integration/tester_js.py b/dev/archery/archery/integration/tester_js.py
@@ -18,7 +18,9 @@
 import os
 
 from .tester import Tester
-from .util import run_cmd, ARROW_ROOT_DEFAULT, log
+from .util import run_cmd, log
+from ..utils.source import ARROW_ROOT_DEFAULT
+
 
 _EXE_PATH = os.path.join(ARROW_ROOT_DEFAULT, 'js/bin')
 _VALIDATE = os.path.join(_EXE_PATH, 'integration.js')

diff --git a/dev/archery/archery/integration/tester_rust.py b/dev/archery/archery/integration/tester_rust.py
@@ -20,7 +20,8 @@
 import subprocess
 
 from .tester import Tester
-from .util import run_cmd, ARROW_ROOT_DEFAULT, log
+from .util import run_cmd, log
+from ..utils.source import ARROW_ROOT_DEFAULT
 
 
 _EXE_PATH = os.path.join(ARROW_ROOT_DEFAULT, "rust/target/debug")

diff --git a/dev/archery/archery/integration/util.py b/dev/archery/archery/integration/util.py
@@ -17,7 +17,6 @@
 
 import contextlib
 import io
-import os
 import random
 import socket
 import subprocess
@@ -36,11 +35,6 @@ def guid():
 SKIP_ARROW = 'arrow'
 SKIP_FLIGHT = 'flight'
 
-ARROW_ROOT_DEFAULT = os.environ.get(
-    'ARROW_ROOT',
-    os.path.abspath(__file__).rsplit("/", 5)[0]
-)
-
 
 class _Printer:
     """

diff --git a/dev/archery/archery/utils/source.py b/dev/archery/archery/utils/source.py
@@ -22,6 +22,19 @@
 from .git import git
 
 
+ARROW_ROOT_DEFAULT = os.environ.get(
+    'ARROW_ROOT',
+    Path(__file__).resolve().parents[4]
+)
+
+
+def arrow_path(path):
+    """
+    Return full path to a file given its path inside the Arrow repo.
+    """
+    return os.path.join(ARROW_ROOT_DEFAULT, path)
+
+
 class InvalidArrowSource(Exception):
     pass
 

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -489,7 +489,14 @@ services:
     ulimits: *ulimits
     environment:
       <<: *ccache
+      ARROW_BUILD_STATIC: "OFF"
       ARROW_CUDA: "ON"
+      ARROW_GANDIVA: "OFF"
+      ARROW_GCS: "OFF"
+      ARROW_ORC: "OFF"
+      ARROW_S3: "OFF"
+      ARROW_SUBSTRAIT: "OFF"
+      ARROW_WITH_OPENTELEMETRY: "OFF"
     volumes: *ubuntu-volumes
     command: *cpp-command