Merge pull request #1331 from rmartin16/docker-desktop

Adds support for Docker Desktop and rootless Docker

changes/1083.feature.rst

@@ -0,0 +1 @@
+On Linux, Docker Desktop and rootless Docker are now supported.

setup.cfg

@@ -74,7 +74,7 @@ install_requires =
     # cause API breakages). If the package uses calver, we don't pin the upper
     # version, as the upper version provides no basis for determining API
     # stability.
-    cookiecutter >= 2.1, < 3.0
+    cookiecutter >= 2.2, < 3.0
     dmgbuild >= 1.6, < 2.0; sys_platform == "darwin"
     GitPython >= 3.1, < 4.0
     platformdirs >= 2.6, < 4.0

src/briefcase/integrations/docker.py

@@ -111,9 +111,32 @@ class Docker(Tool):
         "Linux": "https://docs.docker.com/engine/install/#server",
     }
 
+    def __init__(self, tools: ToolCache, image_tag: str | None = None):
+        """A wrapper for the user-installed Docker.
+
+        :param tools: ToolCache of available tools
+        :param image_tag: An optional image used to access attributes of the Docker
+            environment, such as how user permissions are managed in bind mounts. A
+            lightweight image will be used if one is not specified but this image is not
+            at all bound to the instance.
+        """
+        super().__init__(tools=tools)
+        self.is_user_mapped = self._is_user_mapping_enabled(image_tag)
+
     @classmethod
-    def verify_install(cls, tools: ToolCache, **kwargs) -> Docker:
-        """Verify Docker is installed and operational."""
+    def verify_install(
+        cls,
+        tools: ToolCache,
+        image_tag: str | None = None,
+        **kwargs,
+    ) -> Docker:
+        """Verify Docker is installed and operational.
+
+        :param tools: ToolCache of available tools
+        :param image_tag: An optional image used during verification to access
+            attributes of the local Docker environment. This image is not bound to the
+            instance and only used during instantiation.
+        """
         # short circuit since already verified and available
         if hasattr(tools, "docker"):
             return tools.docker
@@ -122,7 +145,7 @@ def verify_install(cls, tools: ToolCache, **kwargs) -> Docker:
         cls._user_access(tools=tools)
         cls._buildx_installed(tools=tools)
 
-        tools.docker = Docker(tools=tools)
+        tools.docker = Docker(tools=tools, image_tag=image_tag)
         return tools.docker
 
     @classmethod
@@ -189,16 +212,156 @@ def _buildx_installed(cls, tools: ToolCache):
         except subprocess.CalledProcessError:
             raise BriefcaseCommandError(cls.BUILDX_PLUGIN_MISSING)
 
+    def _write_test_path(self) -> Path:
+        """Host system filepath to perform write test from a container."""
+        return Path.cwd() / "build" / "container_write_test"
+
+    def _is_user_mapping_enabled(self, image_tag: str | None = None) -> bool:
+        """Determine whether Docker is mapping users between the container and the host.
+
+        Docker can be installed in different ways on Linux that significantly impact how
+        containers interact with the host system. Of particular note is ownership of
+        files and directories in bind mounts (i.e. mounts using --volume).
+
+        Traditionally, Docker would pass through the UID/GID of the user used inside a
+        container as the owner of files created within the bind mount. And since the
+        default user inside containers is root, the files would be owned by root on the
+        host file system; this prevents later interaction with those files by the host.
+        To work around this, the Dockerfile can use a step-down user with a UID and GID
+        that matches the host user running Docker.
+
+        Other installation methods of Docker, though, are not compatible with using such
+        a step-down user. This includes Docker Desktop and rootless Docker (although,
+        even a traditional installation of Docker Engine can be configured similarly).
+        In these modes, Docker maps the host user to the root user inside the container;
+        this mapping is transparent and would require changes to the host environment to
+        disable, if it can be disabled at all. This allows files created in bind mounts
+        inside the container to be owned on the host file system by the user running
+        Docker. Additionally, though, because the host user is mapped to root inside the
+        container, any files that were created by the host user in the bind mount
+        outside the container are owned by root inside the container; therefore, a step-
+        down user could not interact with such bind mount files inside the container.
+
+        To accommodate these different modes, this checks which user owns a file that is
+        created inside a bind mount in the container. If the owning user of that file on
+        the host file system is root, then a step-down user is necessary inside
+        containers. If the owning user is the host user, root should be used.
+
+        On macOS, Docker Desktop is the only option to use Docker and user mapping
+        happens differently such that any user in the container is mapped to the host
+        user. Instead of leveraging user namespaces as on Linux, this user mapping
+        manifests as a consequence of bind mounts being implemented as NFS shares
+        between macOS and the Linux VM that Docker Desktop runs containers in. So,
+        using a step-down user on macOS is effectively inconsequential.
+
+        On Windows WSL 2, Docker Desktop operates similarly to how it does on Linux.
+        However, user namespace mapping is not possible because the Docker Desktop VM
+        and the WSL distro are already running in different user namespaces...and
+        therefore, Docker cannot even see the users in the distro to map them in to the
+        container. So, a step-down user is always used.
+
+        ref: https://docs.docker.com/engine/security/userns-remap/
+
+        :param image_tag: The image:tag to use to create the container for the test; if
+            one is not specified, then `alpine:latest` will be used.
+        :returns: True if users are being mapped; False otherwise
+        """
+        host_write_test_path = self._write_test_path()
+        container_write_test_path = PurePosixPath(
+            "/host_write_test", host_write_test_path.name
+        )
+
+        docker_run_cmd = [
+            "docker",
+            "run",
+            "--rm",
+            "--volume",
+            f"{host_write_test_path.parent}:{container_write_test_path.parent}:z",
+            "alpine" if image_tag is None else image_tag,
+        ]
+
+        host_write_test_path.parent.mkdir(exist_ok=True)
+
+        try:
+            host_write_test_path.unlink(missing_ok=True)
+        except OSError as e:
+            raise BriefcaseCommandError(
+                f"""\
+The file path used to determine how Docker is mapping users between the host
+and Docker containers already exists and cannot be automatically deleted.
+
+    {host_write_test_path}
+
+Delete this file and run Briefcase again.
+"""
+            ) from e
+
+        try:
+            self.tools.subprocess.run(
+                docker_run_cmd + ["touch", container_write_test_path],
+                check=True,
+                stream_output=False,
+            )
+        except subprocess.CalledProcessError as e:
+            raise BriefcaseCommandError(
+                "Unable to determine if Docker is mapping users"
+            ) from e
+
+        # if the file is not owned by `root`, then Docker is mapping usernames
+        is_user_mapped = 0 != self.tools.os.stat(host_write_test_path).st_uid
+
+        try:
+            self.tools.subprocess.run(
+                docker_run_cmd + ["rm", "-f", container_write_test_path],
+                check=True,
+                stream_output=False,
+            )
+        except subprocess.CalledProcessError as e:
+            raise BriefcaseCommandError(
+                "Unable to clean up from determining if Docker is mapping users"
+            ) from e
+
+        return is_user_mapped
+
+    def cache_image(self, image_tag: str):
+        """Ensures an image is available and cached locally.
+
+        While many Docker commands for an image will pull that image in-line with the
+        command if it isn't already cached, this pollutes the console output with
+        details about pulling the image. This can be particularly troublesome when the
+        output from a command run inside a container using the image is desired.
+
+        Note: This will not update an already cached image if a newer version is
+        available in the registry.
+
+        :param image_tag: Image name/tag to pull if not locally cached
+        """
+        image_id = self.tools.subprocess.check_output(
+            ["docker", "images", "-q", image_tag]
+        ).strip()
+
+        if not image_id:
+            try:
+                self.tools.subprocess.run(["docker", "pull", image_tag], check=True)
+            except subprocess.CalledProcessError as e:
+                raise BriefcaseCommandError(
+                    f"Unable to obtain the Docker image for {image_tag}. "
+                    "Is the image name correct?"
+                ) from e
+
     def check_output(self, args: list[SubprocessArgT], image_tag: str) -> str:
         """Run a process inside a Docker container, capturing output.
 
-        This is a bare Docker invocation; it's really only useful for running simple
-        commands on an image, ensuring that the container is destroyed afterward. In
-        most cases, you'll want to use an app context, rather than this.
+        This ensures the image is locally cached and then runs a bare Docker invocation;
+        it's really only useful for running simple commands on an image, ensuring that
+        the container is destroyed afterward. In most cases, you'll want to use an app
+        context, rather than this.
 
         :param args: The list of arguments to pass to the Docker instance.
         :param image_tag: The Docker image to run
         """
+        self.cache_image(image_tag)
+
         # Any exceptions from running the process are *not* caught.
         # This ensures that "docker.check_output()" behaves as closely to
         # "subprocess.check_output()" as possible.
@@ -212,28 +375,6 @@ def check_output(self, args: list[SubprocessArgT], image_tag: str) -> str:
             + args,
         )
 
-    def prepare(self, image_tag: str):
-        """Ensure that the given image exists, and is cached locally.
-
-        This is achieved by trying to run a no-op command (echo) on the image; if it
-        succeeds, the image exists locally.
-
-        A pull is forced, so you can be certain that the image is up-to-date.
-
-        :param image_tag: The Docker image to prepare
-        """
-        try:
-            self.tools.subprocess.run(
-                ["docker", "run", "--rm", image_tag, "printf", ""],
-                check=True,
-                stream_output=False,
-            )
-        except subprocess.CalledProcessError as e:
-            raise BriefcaseCommandError(
-                f"Unable to obtain the Docker base image {image_tag}. "
-                "Is the image name correct?"
-            ) from e
-
 
 class DockerAppContext(Tool):
     name = "docker_app_context"
@@ -354,7 +495,7 @@ def _dockerize_path(self, arg: str) -> str:  # pragma: no-cover-if-is-windows
         filesystem.
 
         Converts:
-        * any reference to sys.executable into the python executable in the docker container
+        * any reference to `sys.executable` into the python executable in the docker container
         * any path in <build path> into the equivalent stemming from /app
         * any path in <data path> into the equivalent in ~/.cache/briefcase
 

src/briefcase/platforms/linux/appimage.py

@@ -150,6 +150,7 @@ class LinuxAppImageCreateCommand(
 
     def output_format_template_context(self, app: AppConfig):
         context = super().output_format_template_context(app)
+
         # Add the manylinux tag to the template context.
         try:
             tag = getattr(app, "manylinux_image_tag", "latest")
@@ -167,6 +168,12 @@ def output_format_template_context(self, app: AppConfig):
         except AttributeError:
             pass
 
+        # Use the non-root user if Docker is not mapping usernames
+        try:
+            context["use_non_root_user"] = not self.tools.docker.is_user_mapped
+        except AttributeError:
+            pass  # ignore if not using Docker
+
         return context
 
     def _cleanup_app_support_package(self, support_path):

src/briefcase/platforms/linux/system.py

@@ -163,6 +163,29 @@ def finalize_app_config(self, app: AppConfig):
 
         if not self.use_docker:
             app.target_image = f"{app.target_vendor}:{app.target_codename}"
+        else:
+            # If we're building for Arch, and Docker does user mapping, we can't build,
+            # because Arch won't let makepkg run as root. Docker on macOS *does* map the
+            # user, but introducing a step-down user doesn't alter behavior, so we can
+            # allow it.
+            if (
+                app.target_vendor_base == ARCH
+                and self.tools.docker.is_user_mapped
+                and self.tools.host_os != "Darwin"
+            ):
+                raise BriefcaseCommandError(
+                    """\
+Briefcase cannot use this Docker installation to target Arch Linux since the
+tools to build packages for Arch cannot be run as root.
+
+The Docker available to Briefcase requires the use of the root user in
+containers to maintain accurate file permissions of the build artefacts.
+
+This most likely means you're using Docker Desktop or rootless Docker.
+
+Install Docker Engine and try again or run Briefcase on an Arch host system.
+"""
+                )
 
         # Merge target-specific configuration items into the app config This
         # means:
@@ -271,16 +294,15 @@ def platform_freedesktop_info(self, app):
             # Preserve the target image on the command line as the app's target
             app.target_image = self.target_image
 
-            # Ensure that the Docker base image is available.
-            self.logger.info(f"Checking Docker target image {app.target_image}...")
-            self.tools.docker.prepare(app.target_image)
-
             # Extract release information from the image.
-            output = self.tools.docker.check_output(
-                ["cat", "/etc/os-release"],
-                image_tag=app.target_image,
-            )
-            freedesktop_info = parse_freedesktop_os_release(output)
+            with self.input.wait_bar(
+                f"Checking Docker target image {app.target_image}..."
+            ):
+                output = self.tools.docker.check_output(
+                    ["cat", "/etc/os-release"],
+                    image_tag=app.target_image,
+                )
+                freedesktop_info = parse_freedesktop_os_release(output)
         else:
             freedesktop_info = super().platform_freedesktop_info(app)
 
@@ -294,7 +316,7 @@ def verify_tools(self):
         """If we're using Docker, verify that it is available."""
         super().verify_tools()
         if self.use_docker:
-            Docker.verify(tools=self.tools)
+            Docker.verify(tools=self.tools, image_tag=self.target_image)
 
     def add_options(self, parser):
         super().add_options(parser)
@@ -559,6 +581,20 @@ def output_format_template_context(self, app: AppConfig):
         # Add the vendor base
         context["vendor_base"] = app.target_vendor_base
 
+        # Use the non-root user if Docker is not mapping usernames. Also use a non-root
+        # user if we're on macOS; user mapping doesn't alter Docker operation, but some
+        # packaging tools (e.g., Arch's makepkg) don't like running as root. If we're
+        # not using Docker, this will fall back to the template default, which should be
+        # enabling the root user. This might cause problems later, but it's part of a
+        # much bigger "does the project need to be updated in light of configuration
+        # changes" problem.
+        try:
+            context["use_non_root_user"] = (
+                self.tools.host_os == "Darwin" or not self.tools.docker.is_user_mapped
+            )
+        except AttributeError:
+            pass  # ignore if not using Docker
+
         return context