Use python3's venv instead of virtualenv (kivy#2152)

* 👽 Migrate to `toml` and apply `isort` to `pythonpackage.py`

Because `pytoml` is deprecated

See also: https://pypi.org/project/pytoml/

* 🐛 Avoid copying `.tox` folder when extracting metadata from package

Because when running local tests for `test_pythonpackages` and we have
some virtual envs created with tox the function also copies the hidden
tox folder causing error on our tests

* 🔥 Use python3's venv instead of `virtualenv`

* 💚 Add new dependency to docker `libssl-dev`

So this way we will get hostpython's _ssl module build,
and we will be able to use hostpython's pip without security errors.

* 📝 Update outdated inline comment

* 🍎 Don't run p4a inside virtualenv/venv

because it seems that hostpython's build scripts gets confused on about locating the `pyconfig.h` file

**Note:** This only happens to macOS platform

Author: opacam

.travis.yml

@@ -23,9 +23,7 @@ jobs:
         #   See also: https://github.com/travis-ci/travis-ci/issues/8589
         - type -t deactivate && deactivate || true
         - export PATH=/opt/python/3.7/bin:$PATH
-        # Install tox & virtualenv
-        #   Note: venv/virtualenv are both used by tests/test_pythonpackage.py
-        - pip3.7 install -U virtualenv
+        # Install tox
         - pip3.7 install tox>=2.0
         # Install coveralls & dependencies
         #   Note: pyOpenSSL needed to send the coveralls reports
@@ -54,7 +52,7 @@ jobs:
         # installs java 1.8, android's SDK/NDK and p4a
         - make -f ci/makefiles/osx.mk
         - export JAVA_HOME=/Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
-      script: make testapps/armeabi-v7a PYTHON_WITH_VERSION=python3
+      script: make testapps-no-venv/armeabi-v7a
     - <<: *testapps
       name: Rebuild updated recipes
       script: travis_wait 30 make docker/run/make/rebuild_updated_recipes

Dockerfile

@@ -66,6 +66,7 @@ RUN dpkg --add-architecture i386 \
     libncurses5:i386 \
     libpangox-1.0-0:i386 \
     libpangoxft-1.0-0:i386 \
+    libssl-dev \
     libstdc++6:i386 \
     libtool \
     openjdk-8-jdk \
@@ -77,7 +78,6 @@ RUN dpkg --add-architecture i386 \
     python3-venv \
     sudo \
     unzip \
-    virtualenv \
     wget \
     zip \
     zlib1g-dev \

Makefile

@@ -19,7 +19,7 @@ ANDROID_NDK_HOME ?= $(HOME)/.android/android-ndk
 all: virtualenv
 
 $(VIRTUAL_ENV):
-	virtualenv --python=$(PYTHON_WITH_VERSION) $(VIRTUAL_ENV)
+	python3 -m venv $(VIRTUAL_ENV)
 	$(PIP) install Cython==0.28.6
 	$(PIP) install -e .
 
@@ -47,6 +47,14 @@ testapps/%: virtualenv
     python setup.py apk --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \
     --arch=$($@_APP_ARCH)
 
+testapps-no-venv/%:
+	pip3 install Cython==0.28.6
+	pip3 install -e .
+	$(eval $@_APP_ARCH := $(shell basename $*))
+	cd testapps/on_device_unit_tests/ && \
+    python3 setup.py apk --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \
+    --arch=$($@_APP_ARCH)
+
 clean:
 	find . -type d -name "__pycache__" -exec rm -r {} +
 	find . -type d -name "*.egg-info" -exec rm -r {} +

pythonforandroid/build.py

@@ -13,8 +13,8 @@
 import subprocess
 
 from pythonforandroid.util import (
-    current_directory, ensure_dir, get_virtualenv_executable,
-    BuildInterruptingException
+    current_directory, ensure_dir,
+    BuildInterruptingException,
 )
 from pythonforandroid.logger import (info, warning, info_notify, info_main, shprint)
 from pythonforandroid.archs import ArchARM, ArchARMv7_a, ArchAarch_64, Archx86, Archx86_64
@@ -357,13 +357,6 @@ def prepare_build_environment(self,
 
         check_ndk_api(ndk_api, self.android_api)
 
-        virtualenv = get_virtualenv_executable()
-        if virtualenv is None:
-            raise IOError('Couldn\'t find a virtualenv executable, '
-                          'you must install this to use p4a.')
-        self.virtualenv = virtualenv
-        info('Found virtualenv at {}'.format(virtualenv))
-
         # path to some tools
         self.ccache = sh.which("ccache")
         if not self.ccache:
@@ -765,15 +758,10 @@ def run_pymodules_install(ctx, modules, project_dir=None,
         info('Will process project install, if it fails then the '
              'project may not be compatible for Android install.')
 
-    venv = sh.Command(ctx.virtualenv)
+    # Use our hostpython to create the virtualenv
+    host_python = sh.Command(ctx.hostpython)
     with current_directory(join(ctx.build_dir)):
-        shprint(venv,
-                '--python=python{}'.format(
-                    ctx.python_recipe.major_minor_version_string.
-                    partition(".")[0]
-                    ),
-                'venv'
-               )
+        shprint(host_python, '-m', 'venv', 'venv')
 
         # Prepare base environment and upgrade pip:
         base_env = copy.copy(os.environ)

pythonforandroid/pythonpackage.py

@@ -33,26 +33,23 @@
 """
 
 
-from io import open  # needed for python 2
 import functools
 import os
-from pep517.envbuild import BuildEnvironment
-from pep517.wrappers import Pep517HookCaller
-import pytoml
 import shutil
 import subprocess
 import sys
 import tarfile
 import tempfile
 import textwrap
 import time
-try:
-    from urllib.parse import urlparse
-    from urllib.parse import unquote as urlunquote
-except ImportError:  # Python 2...
-    from urlparse import urlparse
-    from urlparse import unquote as urlunquote
 import zipfile
+from io import open  # needed for python 2
+from urllib.parse import unquote as urlunquote
+from urllib.parse import urlparse
+
+import toml
+from pep517.envbuild import BuildEnvironment
+from pep517.wrappers import Pep517HookCaller
 
 
 def transform_dep_for_pip(dependency):
@@ -111,7 +108,8 @@ def extract_metainfo_files_from_package(
         if is_filesystem_path(package):
             shutil.copytree(
                 parse_as_folder_reference(package),
-                os.path.join(temp_folder, "package")
+                os.path.join(temp_folder, "package"),
+                ignore=shutil.ignore_patterns(".tox")
             )
             package = os.path.join(temp_folder, "package")
 
@@ -486,7 +484,7 @@ def _extract_metainfo_files_from_package_unsafe(
 
             # Get build backend and requirements from pyproject.toml:
             with open(os.path.join(path, 'pyproject.toml')) as f:
-                build_sys = pytoml.load(f)['build-system']
+                build_sys = toml.load(f)['build-system']
                 backend = build_sys["build-backend"]
                 build_requires.extend(build_sys["requires"])
 
@@ -630,7 +628,7 @@ def _extract_info_from_package(dependency,
                               ) and include_build_requirements:
                 # Read build system from pyproject.toml file: (PEP518)
                 with open(os.path.join(output_folder, 'pyproject.toml')) as f:
-                    build_sys = pytoml.load(f)['build-system']
+                    build_sys = toml.load(f)['build-system']
                     if "requires" in build_sys:
                         requirements += build_sys["requires"]
             elif include_build_requirements:

pythonforandroid/util.py

@@ -1,7 +1,6 @@
 import contextlib
 from os.path import exists, join
 from os import getcwd, chdir, makedirs, walk, uname
-import sh
 import shutil
 from fnmatch import fnmatch
 from tempfile import mkdtemp
@@ -55,17 +54,6 @@ def ensure_dir(filename):
         makedirs(filename)
 
 
-def get_virtualenv_executable():
-    virtualenv = None
-    if virtualenv is None:
-        virtualenv = sh.which('virtualenv2')
-    if virtualenv is None:
-        virtualenv = sh.which('virtualenv-2.7')
-    if virtualenv is None:
-        virtualenv = sh.which('virtualenv')
-    return virtualenv
-
-
 def walk_valid_filens(base_dir, invalid_dir_names, invalid_file_patterns):
     """Recursively walks all the files and directories in ``dirn``,
     ignoring directories that match any pattern in ``invalid_dirns``

setup.py

@@ -23,9 +23,9 @@
 install_reqs = [
     'appdirs', 'colorama>=0.3.3', 'jinja2', 'six',
     'enum34; python_version<"3.4"', 'sh>=1.10; sys_platform!="nt"',
-    'pep517<0.7.0"', 'pytoml', 'virtualenv<20'
+    'pep517<0.7.0"', 'toml',
 ]
-# (pep517, pytoml and virtualenv are used by pythonpackage.py)
+# (pep517 and toml are used by pythonpackage.py)
 
 # By specifying every file manually, package_data will be able to
 # include them in binary distributions. Note that we have to add

tests/test_pythonpackage_basic.py

@@ -6,7 +6,6 @@
 """
 
 import os
-import pytest
 import shutil
 import sys
 import subprocess
@@ -279,43 +278,6 @@ def test_systemwide_python(self):
             else:
                 raise
 
-    def test_virtualenv(self):
-        """ Verifies that _get_system_python_executable() works correctly
-            if called with a python binary as found inside a virtualenv.
-        """
-
-        # Get system-wide python bin seen from here first:
-        pybin = _get_system_python_executable()
-        # (this call was not a test, we really just need the path here)
-
-        test_dir = tempfile.mkdtemp()
-        try:
-            # Check that in a virtualenv, the system-wide python is returned:
-            subprocess.check_output([
-                pybin, "-m", "virtualenv",
-                "--python=" + str(sys.executable),
-                "--",
-                os.path.join(test_dir, "virtualenv")
-            ])
-            subprocess.check_output([
-                os.path.join(test_dir, "virtualenv", "bin", "pip"),
-                "install", "-U", "pip"
-            ])
-            subprocess.check_output([
-                os.path.join(test_dir, "virtualenv", "bin", "pip"),
-                "install", "-U", "pep517<0.7.0"
-            ])
-            sys_python_path = self.run__get_system_python_executable(
-                os.path.join(test_dir, "virtualenv", "bin", "python")
-            )
-            assert os.path.normpath(sys_python_path).startswith(
-                os.path.normpath(pybin)
-            )
-        finally:
-            shutil.rmtree(test_dir)
-
-    @pytest.mark.skipif(int(sys.version.partition(".")[0]) < 3,
-                        reason="venv is python 3 only")
     def test_venv(self):
         """ Verifies that _get_system_python_executable() works correctly
             in a 'venv' (Python 3 only feature).
@@ -340,6 +302,10 @@ def test_venv(self):
                 os.path.join(test_dir, "venv", "bin", "pip"),
                 "install", "-U", "pep517<0.7.0"
             ])
+            subprocess.check_output([
+                os.path.join(test_dir, "venv", "bin", "pip"),
+                "install", "-U", "toml"
+            ])
             sys_python_path = self.run__get_system_python_executable(
                 os.path.join(test_dir, "venv", "bin", "python")
             )

tests/test_util.py

@@ -70,46 +70,6 @@ def test_current_directory_exception(self):
         ):
             pass
 
-    @mock.patch("pythonforandroid.util.sh.which")
-    def test_get_virtualenv_executable(self, mock_sh_which):
-        """
-        Test method :meth:`~pythonforandroid.util.get_virtualenv_executable`.
-        In here we test:
-
-            - that all calls to `sh.which` are performed, so we expect the
-              first two `sh.which` calls should be None and the last one should
-              return the expected virtualenv (the python3 one)
-            - that we don't have virtualenv installed, so all calls to
-              `sh.which` should return None
-        """
-        expected_venv = os.path.join(
-            os.path.expanduser("~"), ".local/bin/virtualenv"
-        )
-        mock_sh_which.side_effect = [None, None, expected_venv]
-        self.assertEqual(util.get_virtualenv_executable(), expected_venv)
-        mock_sh_which.assert_has_calls(
-            [
-                mock.call("virtualenv2"),
-                mock.call("virtualenv-2.7"),
-                mock.call("virtualenv"),
-            ]
-        )
-        self.assertEqual(mock_sh_which.call_count, 3)
-        mock_sh_which.reset_mock()
-
-        # Now test that we don't have virtualenv installed, so all calls to
-        # `sh.which` should return None
-        mock_sh_which.side_effect = [None, None, None]
-        self.assertIsNone(util.get_virtualenv_executable())
-        self.assertEqual(mock_sh_which.call_count, 3)
-        mock_sh_which.assert_has_calls(
-            [
-                mock.call("virtualenv2"),
-                mock.call("virtualenv-2.7"),
-                mock.call("virtualenv"),
-            ]
-        )
-
     @mock.patch("pythonforandroid.util.walk")
     def test_walk_valid_filens(self, mock_walk):
         """

tox.ini

@@ -5,7 +5,6 @@ basepython = python3
 [testenv]
 deps =
     pytest
-    virtualenv
     py3: coveralls
     backports.tempfile
 # posargs will be replaced by the tox args, so you can override pytest