python: update to 3.13

Author: T-Dynamos

pythonforandroid/bootstraps/common/build/jni/application/src/start.c

@@ -31,6 +31,7 @@
 #define ENTRYPOINT_MAXLEN 128
 #define LOG(n, x) __android_log_write(ANDROID_LOG_INFO, (n), (x))
 #define LOGP(x) LOG("python", (x))
+#define P4A_MIN_VER 11
 
 static PyObject *androidembed_log(PyObject *self, PyObject *args) {
   char *logstr = NULL;
@@ -154,11 +155,6 @@ int main(int argc, char *argv[]) {
   Py_NoSiteFlag=1;
 #endif
 
-#if PY_MAJOR_VERSION < 3
-  Py_SetProgramName("android_python");
-#else
-  Py_SetProgramName(L"android_python");
-#endif
 
 #if PY_MAJOR_VERSION >= 3
   /* our logging module for android
@@ -174,6 +170,21 @@ int main(int argc, char *argv[]) {
   char python_bundle_dir[256];
   snprintf(python_bundle_dir, 256,
            "%s/_python_bundle", getenv("ANDROID_UNPACK"));
+
+  #if PY_MAJOR_VERSION >= 3
+
+    #if PY_MINOR_VERSION >= P4A_MIN_VER
+      PyConfig config;
+      PyConfig_InitPythonConfig(&config);
+      config.program_name = L"android_python";
+    #else
+      Py_SetProgramName(L"android_python");
+    #endif
+
+  #else
+    Py_SetProgramName("android_python");
+  #endif
+
   if (dir_exists(python_bundle_dir)) {
     LOGP("_python_bundle dir exists");
     snprintf(paths, 256,
@@ -185,7 +196,14 @@ int main(int argc, char *argv[]) {
 
     #if PY_MAJOR_VERSION >= 3
         wchar_t *wchar_paths = Py_DecodeLocale(paths, NULL);
+
+        // Py_SetPath is deprecated in python 3.13
+        #if PY_MINOR_VERSION >= P4A_MIN_VER
+        PyWideStringList_Append(&config.module_search_paths, wchar_paths);
+        #else
         Py_SetPath(wchar_paths);
+        #endif
+
     #endif
 
         LOGP("set wchar paths...");
@@ -194,13 +212,21 @@ int main(int argc, char *argv[]) {
            " recipes should have this folder, should we expect a crash soon?");
   }
 
-  Py_Initialize();
+  #if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= P4A_MIN_VER
+      Py_InitializeFromConfig(&config);
+  #else
+      Py_Initialize();
+  #endif
+
   LOGP("Initialized python");
 
-  /* ensure threads will work.
-   */
-  LOGP("AND: Init threads");
-  PyEval_InitThreads();
+  /* < 3.9 requires explicit GIL initialization
+  *  3.9+ PyEval_InitThreads() is deprecated and unnecessary
+  */
+  #if PY_VERSION_HEX < 0x03090000
+    LOGP("Initializing threads (required for Python < 3.9)");
+    PyEval_InitThreads();
+  #endif
 
 #if PY_MAJOR_VERSION < 3
   initandroidembed();
@@ -325,21 +351,6 @@ int main(int argc, char *argv[]) {
 
   LOGP("Python for android ended.");
 
-  /* Shut down: since regular shutdown causes issues sometimes
-     (seems to be an incomplete shutdown breaking next launch)
-     we'll use sys.exit(ret) to shutdown, since that one works.
-
-     Reference discussion:
-
-     https://github.com/kivy/kivy/pull/6107#issue-246120816
-   */
-  char terminatecmd[256];
-  snprintf(
-    terminatecmd, sizeof(terminatecmd),
-    "import sys; sys.exit(%d)\n", ret
-  );
-  PyRun_SimpleString(terminatecmd);
-
   /* This should never actually be reached, but we'll leave the clean-up
    * here just to be safe.
    */

pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonUtil.java

@@ -39,27 +39,22 @@ protected static void addLibraryIfExists(ArrayList<String> libsList, String patt
     }
 
     protected static ArrayList<String> getLibraries(File libsDir) {
-        ArrayList<String> libsList = new ArrayList<String>();
-        addLibraryIfExists(libsList, "sqlite3", libsDir);
-        addLibraryIfExists(libsList, "ffi", libsDir);
-        addLibraryIfExists(libsList, "png16", libsDir);
-        addLibraryIfExists(libsList, "ssl.*", libsDir);
-        addLibraryIfExists(libsList, "crypto.*", libsDir);
-        addLibraryIfExists(libsList, "SDL2", libsDir);
-        addLibraryIfExists(libsList, "SDL2_image", libsDir);
-        addLibraryIfExists(libsList, "SDL2_mixer", libsDir);
-        addLibraryIfExists(libsList, "SDL2_ttf", libsDir);
-        addLibraryIfExists(libsList, "SDL3", libsDir);
-        addLibraryIfExists(libsList, "SDL3_image", libsDir);
-        addLibraryIfExists(libsList, "SDL3_mixer", libsDir);
-        addLibraryIfExists(libsList, "SDL3_ttf", libsDir);
-        libsList.add("python3.5m");
-        libsList.add("python3.6m");
-        libsList.add("python3.7m");
-        libsList.add("python3.8");
-        libsList.add("python3.9");
-        libsList.add("python3.10");
-        libsList.add("python3.11");
+        ArrayList<String> libsList = new ArrayList<>();
+
+        String[] libNames = {
+            "sqlite3", "ffi", "png16", "ssl.*", "crypto.*",
+            "SDL2", "SDL2_image", "SDL2_mixer", "SDL2_ttf",
+            "SDL3", "SDL3_image", "SDL3_mixer", "SDL3_ttf"
+        };
+
+        for (String name : libNames) {
+            addLibraryIfExists(libsList, name, libsDir);
+        }
+
+        for (int v = 5; v <= 13; v++) {
+            libsList.add("python3." + v + (v <= 7 ? "m" : ""));
+        }
+
         libsList.add("main");
         return libsList;
     }
@@ -79,7 +74,7 @@ public static void loadLibraries(File filesDir, File libsDir) {
                 // load, and it has failed, give a more
                 // general error
                 Log.v(TAG, "Library loading error: " + e.getMessage());
-                if (lib.startsWith("python3.11") && !foundPython) {
+                if (lib.startsWith("python3.13") && !foundPython) {
                     throw new RuntimeException("Could not load any libpythonXXX.so");
                 } else if (lib.startsWith("python")) {
                     continue;

pythonforandroid/recipe.py

@@ -12,6 +12,7 @@
 from urllib.request import urlretrieve
 from os import listdir, unlink, environ, curdir, walk
 from sys import stdout
+from multiprocessing import cpu_count
 import time
 try:
     from urlparse import urlparse
@@ -517,7 +518,7 @@ def unpack(self, arch):
                     for entry in listdir(extraction_filename):
                         # Previously we filtered out the .git folder, but during the build process for some recipes
                         # (e.g. when version is parsed by `setuptools_scm`) that may be needed.
-                        shprint(sh.cp, '-Rv',
+                        shprint(sh.cp, '-R',
                                 join(extraction_filename, entry),
                                 directory_name)
                 else:
@@ -830,6 +831,8 @@ def build_arch(self, arch, *extra_args):
             shprint(
                 sh.Command(join(self.ctx.ndk_dir, "ndk-build")),
                 'V=1',
+                "-j",
+                str(cpu_count()),
                 'NDK_DEBUG=' + ("1" if self.ctx.build_as_debuggable else "0"),
                 'APP_PLATFORM=android-' + str(self.ctx.ndk_api),
                 'APP_ABI=' + arch.arch,
@@ -878,6 +881,8 @@ class PythonRecipe(Recipe):
     hostpython_prerequisites = []
     '''List of hostpython packages required to build a recipe'''
 
+    _host_recipe = None
+
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         if 'python3' not in self.depends:
@@ -890,6 +895,10 @@ def __init__(self, *args, **kwargs):
             depends = list(set(depends))
             self.depends = depends
 
+    def prebuild_arch(self, arch):
+        self._host_recipe = Recipe.get_recipe("hostpython3", self.ctx)
+        return super().prebuild_arch(arch)
+
     def clean_build(self, arch=None):
         super().clean_build(arch=arch)
         name = self.folder_name
@@ -907,8 +916,7 @@ def clean_build(self, arch=None):
     def real_hostpython_location(self):
         host_name = 'host{}'.format(self.ctx.python_recipe.name)
         if host_name == 'hostpython3':
-            python_recipe = Recipe.get_recipe(host_name, self.ctx)
-            return python_recipe.python_exe
+            return self._host_recipe.python_exe
         else:
             python_recipe = self.ctx.python_recipe
             return 'python{}'.format(python_recipe.version)
@@ -927,14 +935,44 @@ def folder_name(self):
             name = self.name
         return name
 
+    def patch_shebang(self, _file, original_bin):
+        _file_des = open(_file, "r")
+
+        try:
+            data = _file_des.readlines()
+        except UnicodeDecodeError:
+            return
+
+        if "#!" in (line := data[0]):
+            if line.split("#!")[-1].strip() == original_bin:
+                return
+
+            info(f"Fixing shebang for '{_file}'")
+            data.pop(0)
+            data.insert(0, "#!" + original_bin + "\n")
+            _file_des.close()
+            _file_des = open(_file, "w")
+            _file_des.write("".join(data))
+            _file_des.close()
+
+    def patch_shebangs(self, path, original_bin):
+        # set correct shebang
+        for file in listdir(path):
+            _file = join(path, file)
+            self.patch_shebang(_file, original_bin)
+
     def get_recipe_env(self, arch=None, with_flags_in_cc=True):
+        if self._host_recipe is None:
+            self._host_recipe = Recipe.get_recipe("hostpython3", self.ctx)
+
         env = super().get_recipe_env(arch, with_flags_in_cc)
-        env['PYTHONNOUSERSITE'] = '1'
         # Set the LANG, this isn't usually important but is a better default
         # as it occasionally matters how Python e.g. reads files
         env['LANG'] = "en_GB.UTF-8"
         # Binaries made by packages installed by pip
-        env["PATH"] = join(self.hostpython_site_dir, "bin") + ":" + env["PATH"]
+        env["PATH"] = self._host_recipe.site_bin + ":" + env["PATH"]
+        host_env = self.get_hostrecipe_env()
+        env['PYTHONPATH'] = host_env["PYTHONPATH"]
 
         if not self.call_hostpython_via_targetpython:
             env['CFLAGS'] += ' -I{}'.format(
@@ -945,18 +983,6 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True):
                 self.ctx.python_recipe.link_version,
             )
 
-            hppath = []
-            hppath.append(join(dirname(self.hostpython_location), 'Lib'))
-            hppath.append(join(hppath[0], 'site-packages'))
-            builddir = join(dirname(self.hostpython_location), 'build')
-            if exists(builddir):
-                hppath += [join(builddir, d) for d in listdir(builddir)
-                           if isdir(join(builddir, d))]
-            if len(hppath) > 0:
-                if 'PYTHONPATH' in env:
-                    env['PYTHONPATH'] = ':'.join(hppath + [env['PYTHONPATH']])
-                else:
-                    env['PYTHONPATH'] = ':'.join(hppath)
         return env
 
     def should_build(self, arch):
@@ -993,25 +1019,36 @@ def install_python_package(self, arch, name=None, env=None, is_dir=True):
                     '--install-lib=.',
                     _env=hpenv, *self.setup_extra_args)
 
-            # If asked, also install in the hostpython build dir
-            if self.install_in_hostpython:
-                self.install_hostpython_package(arch)
+            if isfile("setup.py"):
+                shprint(hostpython, 'setup.py', 'install', '-O2',
+                        '--root={}'.format(self.ctx.get_python_install_dir(arch.arch)),
+                        '--install-lib=.',
+                        _env=hpenv, *self.setup_extra_args)
+
+                # If asked, also install in the hostpython build dir
+                if self.install_in_hostpython:
+                    self.install_hostpython_package(arch)
+            else:
+                warning("`PythonRecipe.install_python_package` called without `setup.py` file!")
 
-    def get_hostrecipe_env(self, arch):
+    def get_hostrecipe_env(self):
         env = environ.copy()
-        env['PYTHONPATH'] = self.hostpython_site_dir
+        _python_path = self._host_recipe.get_path_to_python()
+        env['PYTHONPATH'] = self._host_recipe.site_dir + ":" + join(
+            _python_path, "Modules") + ":" + glob.glob(join(_python_path, "build", "lib*"))[0]
         return env
 
     @property
     def hostpython_site_dir(self):
         return join(dirname(self.real_hostpython_location), 'Lib', 'site-packages')
 
     def install_hostpython_package(self, arch):
-        env = self.get_hostrecipe_env(arch)
+        env = self.get_hostrecipe_env()
         real_hostpython = sh.Command(self.real_hostpython_location)
         shprint(real_hostpython, 'setup.py', 'install', '-O2',
                 '--root={}'.format(dirname(self.real_hostpython_location)),
                 '--install-lib=Lib/site-packages',
+                '--root={}'.format(self._host_recipe.site_root),
                 _env=env, *self.setup_extra_args)
 
     @property
@@ -1029,15 +1066,17 @@ def install_hostpython_prerequisites(self, packages=None, force_upgrade=True):
         pip_options = [
             "install",
             *packages,
-            "--target", self.hostpython_site_dir, "--python-version",
+            "--target", self._host_recipe.site_dir, "--python-version",
             self.ctx.python_recipe.version,
             # Don't use sources, instead wheels
             "--only-binary=:all:",
         ]
         if force_upgrade:
             pip_options.append("--upgrade")
         # Use system's pip
-        shprint(sh.pip, *pip_options)
+        pip_env = self.get_hostrecipe_env()
+        pip_env["HOME"] = "/tmp"
+        shprint(sh.Command(self.real_hostpython_location), "-m", "pip", *pip_options, _env=pip_env)
 
     def restore_hostpython_prerequisites(self, packages):
         _packages = []
@@ -1076,7 +1115,7 @@ def build_compiled_components(self, arch):
                     env['STRIP'], '{}', ';', _env=env)
 
     def install_hostpython_package(self, arch):
-        env = self.get_hostrecipe_env(arch)
+        env = self.get_hostrecipe_env()
         self.rebuild_compiled_components(arch, env)
         super().install_hostpython_package(arch)
 
@@ -1231,7 +1270,7 @@ def get_recipe_env(self, arch, **kwargs):
         return env
 
     def get_wheel_platform_tag(self, arch):
-        return "android_" + {
+        return f"android_{self.ctx.ndk_api}_" + {
             "armeabi-v7a": "arm",
             "arm64-v8a": "aarch64",
             "x86_64": "x86_64",
@@ -1268,6 +1307,9 @@ def build_arch(self, arch):
         self.install_hostpython_prerequisites(
             packages=["build[virtualenv]", "pip"] + self.hostpython_prerequisites
         )
+        python_bin_dir = join(self._host_recipe.site_dir, "bin")
+        self.patch_shebangs(python_bin_dir, self.real_hostpython_location)
+
         build_dir = self.get_build_dir(arch.arch)
         env = self.get_recipe_env(arch, with_flags_in_cc=True)
         # make build dir separately