Merge pull request #2941 from abn/fix/sonnet

Fix inconsistencies with release scripts

.cirrus.yml

@@ -28,25 +28,44 @@ release_task:
   name: "Release / FreeBSD"
   only_if: $CIRRUS_TAG != ''
   env:
-    GITHUB_TOKEN: ENCRYPTED[e637a1887c028ea4e44867715a8a4b7c5e27568559a15c45a3e33739b0d97fc050394c728a44e9bfe7c246179810cad7]
+    GITHUB_TOKEN: ENCRYPTED[2b573a2d28a03523ac6fb5b3c2f513a41c0a98db81e40e50e1d103b171f85c57e58ae38d957499dbf7fd7635cfcfd7be]
+    PYTHON: python3.8
+    PYTHON27: python2.7
+    PYTHON35: python3.5
+    PYTHON36: python3.6
+    PYTHON37: python3.7
+    PYTHON38: python3.8
   freebsd_instance:
     matrix:
       - image_family: freebsd-12-1-snap
       - image_family: freebsd-11-3-snap
-  python_script: pkg install -y curl python3 python27 python35 python36 python37 python38
+  python_script: pkg install -y curl bash python3 python27 python35 python36 python37 python38
   pip_script:
     - python2.7 -m ensurepip
     - python3.5 -m ensurepip
     - python3.6 -m ensurepip
     - python3.7 -m ensurepip
     - python3.8 -m ensurepip
-  build_script: ./make-nix-release.sh
+  build_script: bash ./make-nix-release.sh
   upload_script: |
+    #!/usr/bin/env bash
+
+    if [[ "$CIRRUS_RELEASE" == "" ]]; then
+      echo "Not a release. No need to deploy!"
+      exit 0
+    fi
+
+    if [[ "$GITHUB_TOKEN" == "" ]]; then
+      echo "Please provide GitHub access token via GITHUB_TOKEN environment variable!"
+      exit 1
+    fi
+
     for fpath in releases/*
     do
       echo "Uploading $fpath..."
       name=$(basename "$fpath")
-      url_to_upload="https://uploads.github.com/repos/$CIRRUS_REPO_FULL_NAME/releases/$CIRRUS_TAG/assets?name=$name"
+      url_to_upload="https://uploads.github.com/repos/$CIRRUS_REPO_FULL_NAME/releases/$CIRRUS_RELEASE/assets?name=$name"
+      echo "Uploading to $url_to_upload"
       curl -X POST \
         --data-binary @$fpath \
         --header "Authorization: token $GITHUB_TOKEN" \

.github/workflows/main.yml

@@ -36,15 +36,16 @@ jobs:
         shell: bash
         run: echo ::set-output name=version::$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))")
 
-      - name: Install poetry
+      - name: Bootstrap poetry
         shell: bash
         run: |
-          python get-poetry.py -y --preview
-          echo "::set-env name=PATH::$HOME/.poetry/bin:$PATH"
+          python -m ensurepip
+          python -m pip install --upgrade pip
+          python -m pip install .
 
       - name: Configure poetry
         shell: bash
-        run: poetry config virtualenvs.in-project true
+        run: python -m poetry config virtualenvs.in-project true
 
       - name: Set up cache
         uses: actions/cache@v2
@@ -56,16 +57,12 @@ jobs:
       - name: Ensure cache is healthy
         if: steps.cache.outputs.cache-hit == 'true'
         shell: bash
-        run: poetry run pip --version >/dev/null 2>&1 || rm -rf .venv
-
-      - name: Upgrade pip
-        shell: bash
-        run: poetry run python -m pip install pip -U
+        run: python -m poetry run pip --version >/dev/null 2>&1 || rm -rf .venv
 
       - name: Install dependencies
         shell: bash
-        run: poetry install
+        run: python -m poetry install
 
       - name: Run pytest
         shell: bash
-        run: poetry run python -m pytest -v tests
+        run: python -m poetry run python -m pytest -v tests

make-nix-release.sh

@@ -19,6 +19,6 @@ else
   POETRY="$PYTHON -m poetry"
 fi
 
-$POETRY config virtualenvs.create false
+$POETRY config virtualenvs.in-project true
 $POETRY install --no-dev
 $POETRY run python sonnet make release ${RUNTIMES[@]}

poetry/installation/executor.py

@@ -12,6 +12,8 @@
 from poetry.core.packages.file_dependency import FileDependency
 from poetry.core.packages.utils.link import Link
 from poetry.io.null_io import NullIO
+from poetry.utils._compat import PY2
+from poetry.utils._compat import WINDOWS
 from poetry.utils._compat import OrderedDict
 from poetry.utils._compat import Path
 from poetry.utils._compat import cpu_count
@@ -39,7 +41,7 @@ def __init__(self, env, pool, config, io, parallel=True):
         self._chef = Chef(config, self._env)
         self._chooser = Chooser(pool, self._env)
 
-        if parallel:
+        if parallel and not (PY2 and WINDOWS):
             # This should be directly handled by ThreadPoolExecutor
             # however, on some systems the number of CPUs cannot be determined
             # (it raises a NotImplementedError), so, in this case, we assume

sonnet

@@ -55,10 +55,7 @@ class MakeReleaseCommand(Command):
         from poetry.repositories.pool import Pool
         from poetry.repositories.repository import Repository
         from poetry.utils._compat import Path
-        from poetry.utils._compat import decode
-        from poetry.utils._compat import encode
-        from poetry.utils._compat import subprocess
-        from poetry.utils.env import GET_BASE_PREFIX
+        from poetry.utils.env import EnvManager
         from poetry.utils.env import VirtualEnv
         from poetry.utils.helpers import temporary_directory
 
@@ -100,40 +97,34 @@ class MakeReleaseCommand(Command):
                         version
                     )
                 )
-                prefix = decode(
-                    subprocess.run(
-                        [python],
-                        stdout=subprocess.PIPE,
-                        stderr=subprocess.STDOUT,
-                        input=encode(GET_BASE_PREFIX),
-                        check=True,
-                        shell=WINDOWS,
-                    ).stdout
-                )
-                env = VirtualEnv(Path(prefix.strip()), base=Path(prefix.strip()))
-                solver = Solver(package, pool, Repository(), Repository(), self.io)
-                with solver.use_environment(env):
-                    ops = solver.solve()
-                    for op in ops:
-                        if not env.is_valid_for_marker(op.package.marker):
-                            op.skip("Not needed for the current environment")
-
-                self.vendorize_for_python(
-                    python,
-                    [op.package for op in ops if not op.skipped],
-                    poetry_dir,
-                    version,
-                )
-                vendor_dir = Path(
-                    os.path.join(poetry_dir, "_vendor", "py{}".format(python))
-                )
-                created_files += [
-                    p.relative_to(Path(tmp_dir))
-                    for p in vendor_dir.glob("**/*")
-                    if p.is_file()
-                    and p.suffix != ".pyc"
-                    and str(p.relative_to(Path(tmp_dir))) not in vcs_excluded
-                ]
+
+                with temporary_directory() as tmp_venv_dir:
+                    venv_dir = Path(tmp_venv_dir) / ".venv"
+                    EnvManager.build_venv(venv_dir.as_posix(), executable=python)
+
+                    env = VirtualEnv(venv_dir, venv_dir)
+                    solver = Solver(package, pool, Repository(), Repository(), self.io)
+                    with solver.use_environment(env):
+                        ops = solver.solve()
+                        for op in ops:
+                            if not env.is_valid_for_marker(op.package.marker):
+                                op.skip("Not needed for the current environment")
+
+                    vendor_dir = Path(
+                        self.vendorize_for_python(
+                            env,
+                            [op.package for op in ops if not op.skipped],
+                            poetry_dir,
+                            version,
+                        )
+                    )
+                    created_files += [
+                        p.relative_to(Path(tmp_dir))
+                        for p in vendor_dir.glob("**/*")
+                        if p.is_file()
+                        and p.suffix != ".pyc"
+                        and str(p.relative_to(Path(tmp_dir))) not in vcs_excluded
+                    ]
 
                 self.line("")
 
@@ -227,7 +218,7 @@ class MakeReleaseCommand(Command):
             except subprocess.CalledProcessError:
                 raise RuntimeError("Python {} is not available".format(version))
 
-    def vendorize_for_python(self, python, packages, dest, python_version):
+    def vendorize_for_python(self, env, packages, dest, python_version):
         vendor_dir = os.path.join(dest, "_vendor", "py{}".format(python_version))
 
         bar = self.progress_bar(max=len(packages))
@@ -239,26 +230,21 @@ class MakeReleaseCommand(Command):
         )
         bar.start()
         for package in packages:
-            subprocess.check_output(
-                [
-                    python,
-                    "-m",
-                    "pip",
-                    "install",
-                    "{}=={}".format(package.name, package.version),
-                    "--no-deps",
-                    "--target",
-                    vendor_dir,
-                ],
-                stderr=subprocess.STDOUT,
-                shell=WINDOWS,
+            env.run_pip(
+                "install",
+                "{}=={}".format(package.name, package.version),
+                "--no-deps",
+                "--target",
+                vendor_dir,
             )
             bar.advance()
 
         bar.finish()
 
         self.line("")
 
+        return vendor_dir
+
 
 class MakeCommand(Command):
     """