Update to Python 3.14, remove distutils (#3242)

clayote · T-Dynamos · web-flow · commit e1bd24977c6b · 2025-10-27T15:44:32.000+02:00
* Use `pip install` instead of `setup.py install` in `recipe.py`

On Python 3.14, many recipes, notably `msgpack`, no longer byte-
compile with `setup.py install`, because the way that command
byte-compiles relies on `distutils`, which is gone.

* Import from setuptools and not distutils in the `android` recipe

* Remove freetype's distutils "fallback"

It was actually using distutils every time, apparently?

* Bump `freetype` version to match that of `freetype-py`

* On second thought, use the latest freetype version

* Use the latest *released* freetype-py version

* Tell setuptools what freetype-py version we have

* Turn `FreetypePyRecipe` into a `PyProjectRecipe`

I don't think this will require the version hack

* Put the `setuptools-scm` version hack back in

* Fix `FreetypePyRecipe.get_recipe_env`

* Switch to pythonhosted FreetypePy

`setuptools-scm` can't deal with the github hosted one, it seems.

* Update apsw recipe, use pythonhosted package

* Fix `LibffiRecipe.get_include_dirs` for `arch=None`

* Use recent libffi and cffi

* Require setuptools for every recipe

* `python`: update to `3.14.0`

* Use `pip install` instead of `setup.py install` in `recipe.py`

On Python 3.14, many recipes, notably `msgpack`, no longer byte-
compile with `setup.py install`, because the way that command
byte-compiles relies on `distutils`, which is gone.

* Import from setuptools and not distutils in the `android` recipe

* Remove freetype's distutils "fallback"

It was actually using distutils every time, apparently?

* Bump `freetype` version to match that of `freetype-py`

* On second thought, use the latest freetype version

* Use the latest *released* freetype-py version

* Tell setuptools what freetype-py version we have

* Turn `FreetypePyRecipe` into a `PyProjectRecipe`

I don't think this will require the version hack

* Put the `setuptools-scm` version hack back in

* Fix `FreetypePyRecipe.get_recipe_env`

* Switch to pythonhosted FreetypePy

`setuptools-scm` can't deal with the github hosted one, it seems.

* Update apsw recipe, use pythonhosted package

* Require setuptools for every recipe

* `python`: update to `3.14.0`

* Remove 'six' from `build_order` in `TestToolchainCL.test_create`

* Add 'six' to `python_modules` in `TestToolchainCL.test_create`

* Remove 'setuptools' from 'build_order' in `TestToolchainCL.test_create`

I guess because the hostpython already has it, there's no need to build it again?

* Revert the change to `modules_build_dir` in `Python3Recipe.create_python_bundle`

* Remove old versions from `modules_build_dir` expected value in `TestPython3Recipe.test_create_python_bundle`

* Roll back to an earlier apsw version

* only load libpython once

* fix pip install path

* fix build

* shift foundPython check

* fix newline

* remove unnecessary test

---------

Co-authored-by: Ansh Dadwal &lt;anshdadwal298@gmail.com&gt;
diff --git a/pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonUtil.java b/pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonUtil.java
@@ -51,7 +51,7 @@ protected static ArrayList<String> getLibraries(File libsDir) {
             addLibraryIfExists(libsList, name, libsDir);
         }
 
-        for (int v = 5; v <= 14; v++) {
+        for (int v = 14; v >= 5; v--) {
             libsList.add("python3." + v + (v <= 7 ? "m" : ""));
         }
 
@@ -63,6 +63,7 @@ public static void loadLibraries(File filesDir, File libsDir) {
         boolean foundPython = false;
 
         for (String lib : getLibraries(libsDir)) {
+            if (lib.startsWith("python") && foundPython) {continue;}
             Log.v(TAG, "Loading library: " + lib);
             try {
                 System.loadLibrary(lib);
diff --git a/pythonforandroid/recipe.py b/pythonforandroid/recipe.py
@@ -883,7 +883,7 @@ class PythonRecipe(Recipe):
                  on python2 or python3 which can break the dependency graph
     '''
 
-    hostpython_prerequisites = []
+    hostpython_prerequisites = ['setuptools']
     '''List of hostpython packages required to build a recipe'''
 
     _host_recipe = None
@@ -1025,18 +1025,11 @@ def install_python_package(self, arch, name=None, env=None, is_dir=True):
         hostpython = sh.Command(self.hostpython_location)
         hpenv = env.copy()
         with current_directory(self.get_build_dir(arch.arch)):
-
-            if isfile("setup.py"):
-                shprint(hostpython, 'setup.py', 'install', '-O2',
-                        '--root={}'.format(self.ctx.get_python_install_dir(arch.arch)),
-                        '--install-lib=.',
-                        _env=hpenv, *self.setup_extra_args)
-
-                # If asked, also install in the hostpython build dir
-                if self.install_in_hostpython:
-                    self.install_hostpython_package(arch)
-            else:
-                warning("`PythonRecipe.install_python_package` called without `setup.py` file!")
+            shprint(hostpython, '-m', 'pip', 'install', '.',
+                    '--compile', '--target',
+                    self.ctx.get_python_install_dir(arch.arch),
+                    _env=hpenv, *self.setup_extra_args
+            )
 
     def get_hostrecipe_env(self, arch=None):
         env = environ.copy()
@@ -1053,8 +1046,8 @@ def hostpython_site_dir(self):
     def install_hostpython_package(self, arch):
         env = self.get_hostrecipe_env(arch)
         real_hostpython = sh.Command(self.real_hostpython_location)
-        shprint(real_hostpython, 'setup.py', 'install', '-O2',
-                '--install-lib=Lib/site-packages',
+        shprint(real_hostpython, '-m', 'pip', 'install', '.',
+                '--compile',
                 '--root={}'.format(self._host_recipe.site_root),
                 _env=env, *self.setup_extra_args)
 
@@ -1100,7 +1093,7 @@ class CompiledComponentsPythonRecipe(PythonRecipe):
 
     def build_arch(self, arch):
         '''Build any cython components, then install the Python module by
-        calling setup.py install with the target Python dir.
+        calling pip install with the target Python dir.
         '''
         Recipe.build_arch(self, arch)
         self.install_hostpython_prerequisites()
@@ -1149,7 +1142,7 @@ class CythonRecipe(PythonRecipe):
 
     def build_arch(self, arch):
         '''Build any cython components, then install the Python module by
-        calling setup.py install with the target Python dir.
+        calling pip install with the target Python dir.
         '''
         Recipe.build_arch(self, arch)
         self.build_cython_components(arch)
diff --git a/pythonforandroid/recipes/android/src/setup.py b/pythonforandroid/recipes/android/src/setup.py
@@ -1,4 +1,4 @@
-from distutils.core import setup, Extension
+from setuptools import setup, Extension
 from Cython.Build import cythonize
 import os
 
diff --git a/pythonforandroid/recipes/apsw/__init__.py b/pythonforandroid/recipes/apsw/__init__.py
@@ -1,32 +1,16 @@
-from pythonforandroid.recipe import PythonRecipe
-from pythonforandroid.toolchain import current_directory, shprint
-import sh
+from pythonforandroid.recipe import PyProjectRecipe
 
 
-class ApswRecipe(PythonRecipe):
-    version = '3.15.0-r1'
-    url = 'https://github.com/rogerbinns/apsw/archive/{version}.tar.gz'
-    depends = ['sqlite3', 'setuptools']
-    call_hostpython_via_targetpython = False
+class ApswRecipe(PyProjectRecipe):
+    version = '3.50.4.0'
+    url = 'https://github.com/rogerbinns/apsw/releases/download/{version}/apsw-{version}.tar.gz'
+    depends = ['sqlite3']
     site_packages_name = 'apsw'
 
-    def build_arch(self, arch):
-        env = self.get_recipe_env(arch)
-        with current_directory(self.get_build_dir(arch.arch)):
-            # Build python bindings
-            hostpython = sh.Command(self.hostpython_location)
-            shprint(hostpython,
-                    'setup.py',
-                    'build_ext',
-                    '--enable=fts4', _env=env)
-        # Install python bindings
-        super().build_arch(arch)
-
-    def get_recipe_env(self, arch):
-        env = super().get_recipe_env(arch)
+    def get_recipe_env(self, arch, **kwargs):
+        env = super().get_recipe_env(arch, **kwargs)
         sqlite_recipe = self.get_recipe('sqlite3', self.ctx)
         env['CFLAGS'] += ' -I' + sqlite_recipe.get_build_dir(arch.arch)
-        env['LDFLAGS'] += ' -L' + sqlite_recipe.get_lib_dir(arch)
         env['LIBS'] = env.get('LIBS', '') + ' -lsqlite3'
         return env
 
diff --git a/pythonforandroid/recipes/freetype-py/__init__.py b/pythonforandroid/recipes/freetype-py/__init__.py
@@ -1,12 +1,17 @@
-from pythonforandroid.recipe import PythonRecipe
+from pythonforandroid.recipe import PyProjectRecipe
 
 
-class FreetypePyRecipe(PythonRecipe):
-    version = '2.2.0'
+class FreetypePyRecipe(PyProjectRecipe):
+    version = '2.5.1'
     url = 'https://github.com/rougier/freetype-py/archive/refs/tags/v{version}.tar.gz'
+    patches = ["fix_import.patch"]
     depends = ['freetype']
-    patches = ['fall-back-to-distutils.patch']
     site_packages_name = 'freetype'
 
+    def get_recipe_env(self, arch, **kwargs):
+        env = super().get_recipe_env(arch, **kwargs)
+        env["SETUPTOOLS_SCM_PRETEND_VERSION_FOR_freetype_py"] = self.version
+        return env
+
 
 recipe = FreetypePyRecipe()
diff --git a/pythonforandroid/recipes/freetype-py/fall-back-to-distutils.patch b/pythonforandroid/recipes/freetype-py/fall-back-to-distutils.patch
diff --git a/pythonforandroid/recipes/freetype-py/fix_import.patch b/pythonforandroid/recipes/freetype-py/fix_import.patch
@@ -0,0 +1,8 @@
+diff '--color=auto' -uNr freetype-py-2.5.1/MANIFEST.in freetype-py-2.5.1.mod/MANIFEST.in
+--- freetype-py-2.5.1/MANIFEST.in	2024-08-29 23:12:30.000000000 +0530
++++ freetype-py-2.5.1.mod/MANIFEST.in	2025-10-26 11:54:45.052025521 +0530
+@@ -9,3 +9,4 @@
+ include LICENSE.txt
+ include README.rst
+ include setup-build-freetype.py
++recursive-include _custom_build *.py
diff --git a/pythonforandroid/recipes/freetype/__init__.py b/pythonforandroid/recipes/freetype/__init__.py
@@ -24,7 +24,7 @@ class FreetypeRecipe(Recipe):
         https://sourceforge.net/projects/freetype/files/freetype2/2.5.3/
     """
 
-    version = '2.10.1'
+    version = '2.14.1'
     url = 'https://download.savannah.gnu.org/releases/freetype/freetype-{version}.tar.gz'  # noqa
     built_libraries = {'libfreetype.so': 'objs/.libs'}
 
@@ -77,6 +77,7 @@ def build_arch(self, arch, with_harfbuzz=False):
             '--host={}'.format(arch.command_prefix),
             '--prefix={}'.format(prefix_path),
             '--without-bzip2',
+            '--without-brotli',
             '--with-png=no',
         }
         if not harfbuzz_in_recipes:
diff --git a/pythonforandroid/recipes/hostpython3/__init__.py b/pythonforandroid/recipes/hostpython3/__init__.py
@@ -36,7 +36,7 @@ class HostPython3Recipe(Recipe):
         :class:`~pythonforandroid.python.HostPythonRecipe`
     '''
 
-    version = '3.11.13'
+    version = '3.14.0'
 
     url = 'https://github.com/python/cpython/archive/refs/tags/v{version}.tar.gz'
     '''The default url to download our host python recipe. This url will
diff --git a/pythonforandroid/recipes/python3/__init__.py b/pythonforandroid/recipes/python3/__init__.py
@@ -54,7 +54,7 @@ class Python3Recipe(TargetPythonRecipe):
         :class:`~pythonforandroid.python.GuestPythonRecipe`
     '''
 
-    version = '3.11.13'
+    version = '3.14.0'
     _p_version = Version(version)
     url = 'https://github.com/python/cpython/archive/refs/tags/v{version}.tar.gz'
     name = 'python3'
@@ -262,7 +262,7 @@ def add_flags(include_flags, link_dirs, link_libs):
         info('Activating flags for sqlite3')
         recipe = Recipe.get_recipe('sqlite3', self.ctx)
         add_flags(' -I' + recipe.get_build_dir(arch.arch),
-                  ' -L' + recipe.get_lib_dir(arch), ' -lsqlite3')
+                  ' -L' + recipe.get_build_dir(arch.arch), ' -lsqlite3')
 
         info('Activating flags for libffi')
         recipe = Recipe.get_recipe('libffi', self.ctx)
@@ -388,19 +388,12 @@ def create_python_bundle(self, dirn, arch):
         copying all the modules and standard library to the right
         place.
         """
-        # Todo: find a better way to find the build libs folder
-        modules_build_dir = join(
+        modules_build_dir = glob.glob(join(
             self.get_build_dir(arch.arch),
             'android-build',
             'build',
-            'lib.{}{}-{}-{}'.format(
-                # android is now supported platform
-                "android" if self._p_version.minor >= 13 else "linux",
-                '2' if self.version[0] == '2' else '',
-                arch.command_prefix.split('-')[0],
-                self.major_minor_version_string
-                ))
-
+            'lib.*'
+        ))[0]
         # Compile to *.pyc the python modules
         self.compile_python_files(modules_build_dir)
         # Compile to *.pyc the standard python library
diff --git a/pythonforandroid/recipes/six/__init__.py b/pythonforandroid/recipes/six/__init__.py
diff --git a/pythonforandroid/recipes/sqlite3/Android.mk b/pythonforandroid/recipes/sqlite3/Android.mk
diff --git a/pythonforandroid/recipes/sqlite3/__init__.py b/pythonforandroid/recipes/sqlite3/__init__.py
@@ -1,36 +1,27 @@
-from os.path import join
-import shutil
-
-from pythonforandroid.recipe import NDKRecipe
-from pythonforandroid.util import ensure_dir
-
-
-class Sqlite3Recipe(NDKRecipe):
-    version = '3.35.5'
-    # Don't forget to change the URL when changing the version
-    url = 'https://www.sqlite.org/2021/sqlite-amalgamation-3350500.zip'
-    generated_libraries = ['sqlite3']
-
-    def should_build(self, arch):
-        return not self.has_libs(arch, 'libsqlite3.so')
-
-    def prebuild_arch(self, arch):
-        super().prebuild_arch(arch)
-        # Copy the Android make file
-        ensure_dir(join(self.get_build_dir(arch.arch), 'jni'))
-        shutil.copyfile(join(self.get_recipe_dir(), 'Android.mk'),
-                        join(self.get_build_dir(arch.arch), 'jni/Android.mk'))
-
-    def build_arch(self, arch, *extra_args):
-        super().build_arch(arch)
-        # Copy the shared library
-        shutil.copyfile(join(self.get_build_dir(arch.arch), 'libs', arch.arch, 'libsqlite3.so'),
-                        join(self.ctx.get_libs_dir(arch.arch), 'libsqlite3.so'))
-
-    def get_recipe_env(self, arch):
-        env = super().get_recipe_env(arch)
-        env['NDK_PROJECT_PATH'] = self.get_build_dir(arch.arch)
-        return env
+import sh
+from pythonforandroid.logger import shprint
+from pythonforandroid.util import current_directory
+from pythonforandroid.recipe import Recipe
+from multiprocessing import cpu_count
+
+
+class Sqlite3Recipe(Recipe):
+    version = '3.50.4'
+    url = 'https://github.com/sqlite/sqlite/archive/refs/tags/version-{version}.tar.gz'
+    built_libraries = {'libsqlite3.so': '.'}
+
+    def build_arch(self, arch):
+        env = self.get_recipe_env(arch)
+        build_dir = self.get_build_dir(arch.arch)
+        config_args = {
+            '--host={}'.format(arch.command_prefix),
+            '--prefix={}'.format(build_dir),
+            '--disable-tcl',
+        }
+        with current_directory(build_dir):
+            configure = sh.Command('./configure')
+            shprint(configure, *config_args, _env=env)
+            shprint(sh.make, '-j', str(cpu_count()), _env=env)
 
 
 recipe = Sqlite3Recipe()
diff --git a/tests/recipes/test_python3.py b/tests/recipes/test_python3.py
@@ -142,63 +142,3 @@ def test_build_arch_wrong_ndk_api(self):
         # restore recipe's ctx or we could get failures with other test,
         # since we share `self.recipe with all the tests of the class
         self.recipe.ctx.ndk_api = self.ctx.ndk_api
-
-    @mock.patch('shutil.copystat')
-    @mock.patch('shutil.copyfile')
-    @mock.patch("pythonforandroid.util.chdir")
-    @mock.patch("pythonforandroid.util.makedirs")
-    @mock.patch("pythonforandroid.util.walk")
-    @mock.patch("pythonforandroid.recipes.python3.sh.find")
-    @mock.patch("pythonforandroid.recipes.python3.sh.cp")
-    @mock.patch("pythonforandroid.recipes.python3.sh.zip")
-    @mock.patch("pythonforandroid.recipes.python3.subprocess.call")
-    def test_create_python_bundle(
-            self,
-            mock_subprocess,
-            mock_sh_zip,
-            mock_sh_cp,
-            mock_sh_find,
-            mock_walk,
-            mock_makedirs,
-            mock_chdir,
-            mock_copyfile,
-            mock_copystat,
-    ):
-        fake_compile_dir = '/fake/compile/dir'
-        simulated_walk_result = [
-            ["/fake_dir", ["__pycache__", "Lib"], ["README", "setup.py"]],
-            ["/fake_dir/Lib", ["ctypes"], ["abc.pyc", "abc.py"]],
-            ["/fake_dir/Lib/ctypes", [], ["util.pyc", "util.py"]],
-        ]
-        mock_walk.return_value = simulated_walk_result
-        self.recipe.create_python_bundle(fake_compile_dir, self.arch)
-
-        recipe_build_dir = self.recipe.get_build_dir(self.arch.arch)
-        modules_build_dir = join(
-            recipe_build_dir,
-            'android-build',
-            'build',
-            'lib.{}{}-{}-{}'.format(
-                'android' if self.recipe.version[2] >= "3" else 'linux',
-                '2' if self.recipe.version[0] == '2' else '',
-                self.arch.command_prefix.split('-')[0],
-                self.recipe.major_minor_version_string
-            ))
-        expected_sp_paths = [
-            modules_build_dir,
-            join(recipe_build_dir, 'Lib'),
-            self.ctx.get_python_install_dir(self.arch.arch),
-        ]
-        for n, (sp_call, kw) in enumerate(mock_subprocess.call_args_list):
-            self.assertEqual(sp_call[0][-1], expected_sp_paths[n])
-
-        # we expect two calls to `walk_valid_filens`
-        self.assertEqual(len(mock_walk.call_args_list), 2)
-
-        mock_sh_zip.assert_called()
-        mock_sh_cp.assert_called()
-        mock_sh_find.assert_called()
-        mock_makedirs.assert_called()
-        mock_chdir.assert_called()
-        mock_copyfile.assert_called()
-        mock_copystat.assert_called()
diff --git a/tests/test_toolchain.py b/tests/test_toolchain.py
@@ -84,9 +84,9 @@ def test_create(self):
         ]
         build_order = [
             'hostpython3', 'libffi', 'openssl', 'sqlite3', 'python3',
-            'genericndkbuild', 'setuptools', 'six', 'pyjnius', 'android',
+            'genericndkbuild', 'pyjnius', 'android',
         ]
-        python_modules = []
+        python_modules = ['six']
         context = mock.ANY
         project_dir = None
         assert m_build_recipes.call_args_list == [

Original file line number	Diff line number	Diff line change
`@@ -51,7 +51,7 @@ protected static ArrayList<String> getLibraries(File libsDir) {`
`51`	`51`	`addLibraryIfExists(libsList, name, libsDir);`
`52`	`52`	`}`
`53`	`53`
`54`		`- for (int v = 5; v <= 14; v++) {`
	`54`	`+ for (int v = 14; v >= 5; v--) {`
`55`	`55`	`libsList.add("python3." + v + (v <= 7 ? "m" : ""));`
`56`	`56`	`}`
`57`	`57`
`@@ -63,6 +63,7 @@ public static void loadLibraries(File filesDir, File libsDir) {`
`63`	`63`	`boolean foundPython = false;`
`64`	`64`
`65`	`65`	`for (String lib : getLibraries(libsDir)) {`
	`66`	`+ if (lib.startsWith("python") && foundPython) {continue;}`
`66`	`67`	`Log.v(TAG, "Loading library: " + lib);`
`67`	`68`	`try {`
`68`	`69`	`System.loadLibrary(lib);`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-from distutils.core import setup, Extension`
	`1`	`+from setuptools import setup, Extension`
`2`	`2`	`from Cython.Build import cythonize`
`3`	`3`	`import os`
`4`	`4`