Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement architecture and bitness checks #1360

Merged
merged 16 commits into from
Aug 30, 2023
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changes/1360.feature.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Support for less common environments, such as Linux on ARM, is improved and error messages for unsupported platforms are more accurate.
2 changes: 2 additions & 0 deletions src/briefcase/commands/create.py
Original file line number Diff line number Diff line change
Expand Up @@ -344,6 +344,7 @@ def _download_support_package(self, app: AppConfig):
python_version_tag=self.python_version_tag,
platform=self.platform,
host_arch=self.tools.host_arch,
is_32bit=self.tools.is_32bit_python,
)

support_package_url = self.support_package_url(support_revision)
Expand Down Expand Up @@ -384,6 +385,7 @@ def _download_support_package(self, app: AppConfig):
python_version_tag=self.python_version_tag,
platform=self.platform,
host_arch=self.tools.host_arch,
is_32bit=self.tools.is_32bit_python,
) from e

def _write_requirements_file(
Expand Down
20 changes: 16 additions & 4 deletions src/briefcase/exceptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,18 @@ def __init__(self, tool):
super().__init__(msg=f"Unable to locate {tool!r}. Has it been installed?")


class IncompatibleToolError(BriefcaseCommandError):
def __init__(self, tool: str, env_var: str):
self.tool = tool
super().__init__(
msg=f"""\
Briefcase cannot install {tool} on this machine.

Install {tool} manually and specify the installation directory in the {env_var} environment variable.
"""
)


class NonManagedToolError(BriefcaseCommandError):
def __init__(self, tool):
self.tool = tool
Expand Down Expand Up @@ -153,16 +165,16 @@ def __init__(self, app_bundle_path):


class MissingSupportPackage(BriefcaseCommandError):
def __init__(self, python_version_tag, platform, host_arch):
def __init__(self, python_version_tag, platform, host_arch, is_32bit):
self.python_version_tag = python_version_tag
self.platform = platform
self.platform = f"{'32 bit ' if is_32bit else ''}{platform}"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not a huge thing - but any reason to merge is_32bit and platform, rather than keeping them separate until needed for output?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think I was getting frustrated with linters telling me a line was too long. I should figure out a different solution though.

self.host_arch = host_arch
super().__init__(
f"""\
Unable to download {self.platform} support package for Python {self.python_version_tag} on {self.host_arch}.

This is likely because either Python {self.python_version_tag} and/or {self.host_arch}
is not yet supported on {self.platform}. You will need to:
This is likely because either Python {self.python_version_tag} and/or {self.host_arch} is not yet
supported on {self.platform}. You will need to:
* Use an older version of Python; or
* Compile your own custom support package.
"""
Expand Down
41 changes: 31 additions & 10 deletions src/briefcase/integrations/android_sdk.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
from briefcase.console import InputDisabled, select_option
from briefcase.exceptions import (
BriefcaseCommandError,
IncompatibleToolError,
InvalidDeviceError,
MissingToolError,
)
Expand Down Expand Up @@ -57,15 +58,35 @@ def __init__(self, tools: ToolCache, root_path: Path):

@property
def cmdline_tools_url(self) -> str:
"""The Android SDK Command-Line Tools URL appropriate to the current operating
system."""
platform_name = self.tools.host_os.lower()
if self.tools.host_os.lower() == "darwin":
platform_name = "mac"
elif self.tools.host_os.lower() == "windows": # pragma: no branch
platform_name = "win"
"""The Android SDK Command-Line Tools URL appropriate for the current machine.

return f"https://dl.google.com/android/repository/commandlinetools-{platform_name}-{self.cmdline_tools_version}_latest.zip" # noqa: E501
The SDK largely only supports typical development environments; if a machine is
using an unsupported architecture, `sdkmanager` will error while installing the
emulator as a dependency of the build-tools. However, for some of the platforms
that are unsupported by sdkmanager, users can set up their own SDK install.
"""
try:
platform_name = {
"Darwin": {
"arm64": "mac",
"x86_64": "mac",
},
"Linux": {
"x86_64": "linux",
},
"Windows": {
"AMD64": "win",
},
}[self.tools.host_os][self.tools.host_arch]
except KeyError as e:
raise IncompatibleToolError(
tool=self.full_name, env_var="ANDROID_HOME"
) from e

return (
f"https://dl.google.com/android/repository/commandlinetools-"
f"{platform_name}-{self.cmdline_tools_version}_latest.zip"
)

@property
def cmdline_tools_path(self) -> Path:
Expand All @@ -74,11 +95,11 @@ def cmdline_tools_path(self) -> Path:
@property
def cmdline_tools_version(self) -> str:
# This is the version of the Android SDK Command-line tools that
# are current as of May 2022. These tools can generally self-update,
# are current as of July 2023. These tools can generally self-update,
# so using a fixed download URL isn't a problem.
# However, if/when this version number is changed, ensure that the
# checks done during verification include any required upgrade steps.
return "8092744"
return "9477386"
freakboy3742 marked this conversation as resolved.
Show resolved Hide resolved

@property
def cmdline_tools_version_path(self) -> Path:
Expand Down
2 changes: 2 additions & 0 deletions src/briefcase/integrations/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,8 @@ def __init__(

self.host_arch = self.platform.machine()
self.host_os = self.platform.system()
# Python is 32bit if its pointers can only address with 32 bits or fewer
self.is_32bit_python = self.sys.maxsize <= 2**32

self.app_tools: DefaultDict[AppConfig, ToolCache] = defaultdict(
lambda: ToolCache(
Expand Down
55 changes: 33 additions & 22 deletions src/briefcase/integrations/java.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,11 @@
import subprocess
from pathlib import Path

from briefcase.exceptions import BriefcaseCommandError, MissingToolError
from briefcase.exceptions import (
BriefcaseCommandError,
IncompatibleToolError,
MissingToolError,
)
from briefcase.integrations.base import ManagedTool, ToolCache


Expand All @@ -16,7 +20,7 @@ class JDK(ManagedTool):
# As of 12 May 2023, 17.0.7+7 is the current OpenJDK
# https://adoptium.net/temurin/releases/
JDK_MAJOR_VER = "17"
JDK_RELEASE = "17.0.7"
JDK_RELEASE = "17.0.8"
JDK_BUILD = "7"
JDK_INSTALL_DIR_NAME = f"java{JDK_MAJOR_VER}"

Expand All @@ -26,23 +30,30 @@ def __init__(self, tools: ToolCache, java_home: Path):

@property
def OpenJDK_download_url(self):
arch = {
"x86_64": "x64", # Linux\macOS x86-64
"aarch64": "aarch64", # Linux arm64
"armv6l": "arm", # Linux arm
"arm64": "aarch64", # macOS arm64
"AMD64": "x64", # Windows x86-64
}.get(self.tools.host_arch)

platform = {
"Darwin": "mac",
"Windows": "windows",
"Linux": "linux",
}.get(self.tools.host_os)

extension = {
"Windows": "zip",
}.get(self.tools.host_os, "tar.gz")
"""The OpenJDK download URL appropriate for the current machine."""
system_arch = self.tools.host_arch
# use a 32bit JDK if using 32bit Python on 64bit hardware
if self.tools.is_32bit_python and self.tools.host_arch == "aarch64":
system_arch = "armv7l"
freakboy3742 marked this conversation as resolved.
Show resolved Hide resolved

try:
platform, arch, extension = {
"Darwin": {
"arm64": ("mac", "aarch64", "tar.gz"),
"x86_64": ("mac", "x64", "tar.gz"),
},
"Linux": {
"armv7l": ("linux", "arm", "tar.gz"),
"armv8l": ("linux", "arm", "tar.gz"),
"aarch64": ("linux", "aarch64", "tar.gz"),
"x86_64": ("linux", "x64", "tar.gz"),
},
"Windows": {
"AMD64": ("windows", "x64", "zip"),
},
}[self.tools.host_os][system_arch]
except KeyError as e:
raise IncompatibleToolError(tool=self.full_name, env_var="JAVA_HOME") from e

return (
f"https://github.com/adoptium/temurin{self.JDK_MAJOR_VER}-binaries/"
Expand All @@ -64,15 +75,15 @@ def version_from_path(cls, tools: ToolCache, java_path: str | Path) -> str:

:param tools: ToolCache of available tools
:param java_path: File path to a candidate JDK install
:return: JDK release version; e.g. "17.0.7"
:return: JDK release version; e.g. "17.0.8"
"""
output = tools.subprocess.check_output(
[
os.fsdecode(Path(java_path) / "bin" / "javac"),
"-version",
],
)
# javac's output should look like "javac 17.0.7\n"
# javac's output should look like "javac 17.0.8\n"
return output.strip("\n").split(" ")[1]

@classmethod
Expand Down Expand Up @@ -292,7 +303,7 @@ def install(self):

jdk_zip_path.unlink() # Zip file no longer needed once unpacked.

# The tarball will unpack into <briefcase data dir>/tools/jdk-17.0.7+7
# The tarball will unpack into <briefcase data dir>/tools/jdk-17.0.8+7
# (or whatever name matches the current release).
# We turn this into <briefcase data dir>/tools/java so we have a consistent name.
java_unpack_path = (
Expand Down
20 changes: 18 additions & 2 deletions src/briefcase/integrations/linuxdeploy.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
BriefcaseCommandError,
CorruptToolError,
MissingToolError,
UnsupportedHostError,
)
from briefcase.integrations.base import ManagedTool, Tool, ToolCache

Expand Down Expand Up @@ -48,6 +49,21 @@ def download_url(self) -> str:
def file_path(self) -> Path:
"""The folder on the local filesystem that contains the file_name."""

@classmethod
def arch(cls, host_os: str, host_arch: str) -> str:
# always use the x86-64 arch on macOS since the
# manylinux image will always be x86-64 for macOS
arch = "x86_64" if host_os == "Darwin" else host_arch
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

...is that true? The manylinux2014_x86_64 image will be; but I can pull quay.io/pypa/manylinux2014_aarch64 and it works fine on M1 macOS.

Copy link
Member Author

@rmartin16 rmartin16 Jul 31, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's true in this implementation since this same code is used to determine the x86-64 manylinux image should always be used on macOS.

Nonetheless, this review has included a lot of questions about building AppImages on macOS. My intention was to support building x86_64 AppImages by using the x86_64 image of manylinux on macOS irrespective of hardware. Therefore, while the aarch64 image of manylinux works on Apple Silicon, you can't then build an x86_64 AppImage with that image....and without aarch64 support from linuxdeploy, you won't be able to build any AppImage.

Does that help clarify our difference of position? I'm not sure exactly where we aren't aligned.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I think that clarifies things. I think I was reading some of this PR as an attempt at "make all the architectures work everywhere"; keeping it to "native architecture only" definitely make sense as a scoping exercise, and avoids a bunch of gnarly compatibility issues.

That said - as a longer term issue, I'd be interested in whether running x86 docker on M1 Mac images is viable for Linux System and Windows - mostly because it would make my personal testing life a lot simpler.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd be interested in whether running x86 docker on M1 Mac images is viable for Linux System and Windows - mostly because it would make my personal testing life a lot simpler.

That's is definitely in scope currently for #1392 😃

try:
return {
"x86_64": "x86_64",
"i686": "i386",
}[arch]
freakboy3742 marked this conversation as resolved.
Show resolved Hide resolved
except KeyError as e:
raise UnsupportedHostError(
f"Linux AppImages cannot be built on {host_arch}."
) from e

def exists(self) -> bool:
return (self.file_path / self.file_name).is_file()

Expand Down Expand Up @@ -208,7 +224,7 @@ class LinuxDeployQtPlugin(LinuxDeployPluginBase, ManagedTool):

@property
def file_name(self) -> str:
return f"linuxdeploy-plugin-qt-{self.tools.host_arch}.AppImage"
return f"linuxdeploy-plugin-qt-{self.arch(self.tools.host_os, self.tools.host_arch)}.AppImage"

@property
def download_url(self) -> str:
Expand Down Expand Up @@ -319,7 +335,7 @@ def file_path(self) -> Path:

@property
def file_name(self) -> str:
return f"linuxdeploy-{self.tools.host_arch}.AppImage"
return f"linuxdeploy-{self.arch(self.tools.host_os, self.tools.host_arch)}.AppImage"

@property
def download_url(self) -> str:
Expand Down
9 changes: 8 additions & 1 deletion src/briefcase/platforms/linux/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,10 +66,17 @@ def support_package_url(self, support_revision):
System packages don't use a support package; this is defined by
the template, so this method won't be invoked
"""
python_download_arch = self.tools.host_arch
# use a 32bit Python if using 32bit Python on 64bit hardware
if self.tools.is_32bit_python and self.tools.host_arch == "aarch64":
python_download_arch = "armv7"
rmartin16 marked this conversation as resolved.
Show resolved Hide resolved
elif self.tools.is_32bit_python and self.tools.host_arch == "x86_64":
python_download_arch = "i686"

version, datestamp = support_revision.split("+")
return (
"https://github.com/indygreg/python-build-standalone/releases/download/"
f"{datestamp}/cpython-{support_revision}-{self.tools.host_arch}-unknown-linux-gnu-install_only.tar.gz"
f"{datestamp}/cpython-{support_revision}-{python_download_arch}-unknown-linux-gnu-install_only.tar.gz"
)

def vendor_details(self, freedesktop_info):
Expand Down
19 changes: 12 additions & 7 deletions src/briefcase/platforms/linux/appimage.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,14 +44,20 @@ def project_path(self, app):

def binary_name(self, app):
safe_name = app.formal_name.replace(" ", "_")
return f"{safe_name}-{app.version}-{self.tools.host_arch}.AppImage"
arch = LinuxDeploy.arch(self.tools.host_os, self.tools.host_arch)
return f"{safe_name}-{app.version}-{arch}.AppImage"

def binary_path(self, app):
return self.bundle_path(app) / self.binary_name(app)

def distribution_path(self, app):
return self.dist_path / self.binary_name(app)

def verify_tools(self):
"""Verify the AppImage LinuxDeploy tool and its plugins exist."""
super().verify_tools()
LinuxDeploy.verify(tools=self.tools)
Comment on lines +56 to +59
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Verify LinuxDeploy before building an entire Docker image and then informing the user that LinuxDeploy can't run on ARM.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Completely agreed that we should verify LinuxDeploy before building the Docker image.

However, this isn't working for me on my M1. When I run briefcase package linux appimage, it downloads the x86_64 linuxdeploy, then builds the Docker image, then downloads the GTK plugin, then starts building the app image (with a warning that the image being used is linux/amd64), and then fails due to a bad file descriptor. Log file attached.

briefcase.2023_07_31-11_57_12.package.log

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this is something I've discovered in #1392; if you don't specify an explicit platform to use and the image only supports a platform that's different than the host platform, you get this warning.

Nonetheless, (and ignoring the completely corrupted PATH setting), I think this is AppImage/AppImageKit#965 which suggests that building x86-64 AppImages on Apple Silicon may not currently be possible.

I haven't fully processed that issue....so, I look more at this later.


def add_options(self, parser):
super().add_options(parser)
parser.add_argument(
Expand Down Expand Up @@ -154,7 +160,11 @@ def output_format_template_context(self, app: AppConfig):
# Add the manylinux tag to the template context.
try:
tag = getattr(app, "manylinux_image_tag", "latest")
context["manylinux_image"] = f"{app.manylinux}_{self.tools.host_arch}:{tag}"
manylinux_arch = {
"x86_64": "x86_64",
"i386": "i686",
freakboy3742 marked this conversation as resolved.
Show resolved Hide resolved
}[LinuxDeploy.arch(self.tools.host_os, self.tools.host_arch)]
context["manylinux_image"] = f"{app.manylinux}_{manylinux_arch}:{tag}"
if app.manylinux in {"manylinux1", "manylinux2010", "manylinux2014"}:
context["vendor_base"] = "centos"
elif app.manylinux == "manylinux_2_24":
Expand Down Expand Up @@ -213,11 +223,6 @@ class LinuxAppImageOpenCommand(LinuxAppImageMostlyPassiveMixin, DockerOpenComman
class LinuxAppImageBuildCommand(LinuxAppImageMixin, BuildCommand):
description = "Build a Linux AppImage."

def verify_tools(self):
"""Verify the AppImage linuxdeploy tool and plugins exist."""
super().verify_tools()
LinuxDeploy.verify(tools=self.tools)

def build_app(self, app: AppConfig, **kwargs): # pragma: no-cover-if-is-windows
"""Build an application.

Expand Down
Loading
Loading