From 37ddcc6c0cc385572c6da367b6d641e46911befd Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Fri, 18 Sep 2020 19:59:35 +0200 Subject: [PATCH 1/7] cirrus: ensure env vars are set for build task --- .cirrus.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.cirrus.yml b/.cirrus.yml index 330e48bdeba..7c896a158f5 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -29,6 +29,12 @@ release_task: only_if: $CIRRUS_TAG != '' env: GITHUB_TOKEN: ENCRYPTED[e637a1887c028ea4e44867715a8a4b7c5e27568559a15c45a3e33739b0d97fc050394c728a44e9bfe7c246179810cad7] + PYTHON: python3.8 + PYTHON27: python2.7 + PYTHON35: python3.5 + PYTHON36: python3.6 + PYTHON37: python3.7 + PYTHON38: python3.8 freebsd_instance: matrix: - image_family: freebsd-12-1-snap From 4d47e54f41ae5cb01d3bd36a7a192b6567a91ee3 Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Sat, 19 Sep 2020 16:00:20 +0200 Subject: [PATCH 2/7] cirrus: use bash to execute script --- .cirrus.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 7c896a158f5..467216af82f 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -39,14 +39,14 @@ release_task: matrix: - image_family: freebsd-12-1-snap - image_family: freebsd-11-3-snap - python_script: pkg install -y curl python3 python27 python35 python36 python37 python38 + python_script: pkg install -y curl bash python3 python27 python35 python36 python37 python38 pip_script: - python2.7 -m ensurepip - python3.5 -m ensurepip - python3.6 -m ensurepip - python3.7 -m ensurepip - python3.8 -m ensurepip - build_script: ./make-nix-release.sh + build_script: bash ./make-nix-release.sh upload_script: | for fpath in releases/* do From 25ecf42f35c057cefb3b9ccbfede2bfcda247d13 Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Sat, 19 Sep 2020 16:00:26 +0200 Subject: [PATCH 3/7] sonnet: fix incorrect environment handling --- make-nix-release.sh | 2 +- sonnet | 90 +++++++++++++++++++-------------------------- 2 files changed, 39 insertions(+), 53 deletions(-) diff --git a/make-nix-release.sh b/make-nix-release.sh index 15072cf621e..b3828db2c38 100755 --- a/make-nix-release.sh +++ b/make-nix-release.sh @@ -19,6 +19,6 @@ else POETRY="$PYTHON -m poetry" fi -$POETRY config virtualenvs.create false +$POETRY config virtualenvs.in-project true $POETRY install --no-dev $POETRY run python sonnet make release ${RUNTIMES[@]} diff --git a/sonnet b/sonnet index cbe4842b2ed..ecfda99037b 100755 --- a/sonnet +++ b/sonnet @@ -55,10 +55,7 @@ class MakeReleaseCommand(Command): from poetry.repositories.pool import Pool from poetry.repositories.repository import Repository from poetry.utils._compat import Path - from poetry.utils._compat import decode - from poetry.utils._compat import encode - from poetry.utils._compat import subprocess - from poetry.utils.env import GET_BASE_PREFIX + from poetry.utils.env import EnvManager from poetry.utils.env import VirtualEnv from poetry.utils.helpers import temporary_directory @@ -100,40 +97,34 @@ class MakeReleaseCommand(Command): version ) ) - prefix = decode( - subprocess.run( - [python], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - input=encode(GET_BASE_PREFIX), - check=True, - shell=WINDOWS, - ).stdout - ) - env = VirtualEnv(Path(prefix.strip()), base=Path(prefix.strip())) - solver = Solver(package, pool, Repository(), Repository(), self.io) - with solver.use_environment(env): - ops = solver.solve() - for op in ops: - if not env.is_valid_for_marker(op.package.marker): - op.skip("Not needed for the current environment") - - self.vendorize_for_python( - python, - [op.package for op in ops if not op.skipped], - poetry_dir, - version, - ) - vendor_dir = Path( - os.path.join(poetry_dir, "_vendor", "py{}".format(python)) - ) - created_files += [ - p.relative_to(Path(tmp_dir)) - for p in vendor_dir.glob("**/*") - if p.is_file() - and p.suffix != ".pyc" - and str(p.relative_to(Path(tmp_dir))) not in vcs_excluded - ] + + with temporary_directory() as tmp_venv_dir: + venv_dir = Path(tmp_venv_dir) / ".venv" + EnvManager.build_venv(venv_dir.as_posix(), executable=python) + + env = VirtualEnv(venv_dir, venv_dir) + solver = Solver(package, pool, Repository(), Repository(), self.io) + with solver.use_environment(env): + ops = solver.solve() + for op in ops: + if not env.is_valid_for_marker(op.package.marker): + op.skip("Not needed for the current environment") + + vendor_dir = Path( + self.vendorize_for_python( + env, + [op.package for op in ops if not op.skipped], + poetry_dir, + version, + ) + ) + created_files += [ + p.relative_to(Path(tmp_dir)) + for p in vendor_dir.glob("**/*") + if p.is_file() + and p.suffix != ".pyc" + and str(p.relative_to(Path(tmp_dir))) not in vcs_excluded + ] self.line("") @@ -227,7 +218,7 @@ class MakeReleaseCommand(Command): except subprocess.CalledProcessError: raise RuntimeError("Python {} is not available".format(version)) - def vendorize_for_python(self, python, packages, dest, python_version): + def vendorize_for_python(self, env, packages, dest, python_version): vendor_dir = os.path.join(dest, "_vendor", "py{}".format(python_version)) bar = self.progress_bar(max=len(packages)) @@ -239,19 +230,12 @@ class MakeReleaseCommand(Command): ) bar.start() for package in packages: - subprocess.check_output( - [ - python, - "-m", - "pip", - "install", - "{}=={}".format(package.name, package.version), - "--no-deps", - "--target", - vendor_dir, - ], - stderr=subprocess.STDOUT, - shell=WINDOWS, + env.run_pip( + "install", + "{}=={}".format(package.name, package.version), + "--no-deps", + "--target", + vendor_dir, ) bar.advance() @@ -259,6 +243,8 @@ class MakeReleaseCommand(Command): self.line("") + return vendor_dir + class MakeCommand(Command): """ From 427fc4aa64db10ca0af8259a88fbc49565a58507 Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Sat, 19 Sep 2020 16:33:18 +0200 Subject: [PATCH 4/7] cirrus: update token and use CIRRUS_RELEASE --- .cirrus.yml | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 467216af82f..126ded1f6bd 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -28,7 +28,7 @@ release_task: name: "Release / FreeBSD" only_if: $CIRRUS_TAG != '' env: - GITHUB_TOKEN: ENCRYPTED[e637a1887c028ea4e44867715a8a4b7c5e27568559a15c45a3e33739b0d97fc050394c728a44e9bfe7c246179810cad7] + GITHUB_TOKEN: ENCRYPTED[2b573a2d28a03523ac6fb5b3c2f513a41c0a98db81e40e50e1d103b171f85c57e58ae38d957499dbf7fd7635cfcfd7be] PYTHON: python3.8 PYTHON27: python2.7 PYTHON35: python3.5 @@ -48,11 +48,24 @@ release_task: - python3.8 -m ensurepip build_script: bash ./make-nix-release.sh upload_script: | + #!/usr/bin/env bash + + if [[ "$CIRRUS_RELEASE" == "" ]]; then + echo "Not a release. No need to deploy!" + exit 0 + fi + + if [[ "$GITHUB_TOKEN" == "" ]]; then + echo "Please provide GitHub access token via GITHUB_TOKEN environment variable!" + exit 1 + fi + for fpath in releases/* do echo "Uploading $fpath..." name=$(basename "$fpath") - url_to_upload="https://uploads.github.com/repos/$CIRRUS_REPO_FULL_NAME/releases/$CIRRUS_TAG/assets?name=$name" + url_to_upload="https://uploads.github.com/repos/$CIRRUS_REPO_FULL_NAME/releases/$CIRRUS_RELEASE/assets?name=$name" + echo "Uploading to $url_to_upload" curl -X POST \ --data-binary @$fpath \ --header "Authorization: token $GITHUB_TOKEN" \ From 3ae62a372a80c4eb07f7c7d014bfb6961155b706 Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Mon, 21 Sep 2020 12:20:46 +0200 Subject: [PATCH 5/7] executor: fix compatibility issue w/ py2 on windows Python 2 running on Windows, causes thread leaks which leads console io write errors when using pastel. This works around the issue by limiting to a single worker in these environments. --- poetry/installation/executor.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/poetry/installation/executor.py b/poetry/installation/executor.py index 1e63394985f..f6c60497fd9 100644 --- a/poetry/installation/executor.py +++ b/poetry/installation/executor.py @@ -12,6 +12,8 @@ from poetry.core.packages.file_dependency import FileDependency from poetry.core.packages.utils.link import Link from poetry.io.null_io import NullIO +from poetry.utils._compat import PY2 +from poetry.utils._compat import WINDOWS from poetry.utils._compat import OrderedDict from poetry.utils._compat import Path from poetry.utils._compat import cpu_count @@ -39,7 +41,7 @@ def __init__(self, env, pool, config, io, parallel=True): self._chef = Chef(config, self._env) self._chooser = Chooser(pool, self._env) - if parallel: + if parallel and not (PY2 and WINDOWS): # This should be directly handled by ThreadPoolExecutor # however, on some systems the number of CPUs cannot be determined # (it raises a NotImplementedError), so, in this case, we assume From 2986076b85e47eaa24fe7590c682612b723bf488 Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Mon, 21 Sep 2020 12:28:49 +0200 Subject: [PATCH 6/7] ci: bootstrap poetry instead of using installer --- .github/workflows/main.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 02616088d2f..4c55a7acc60 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -36,15 +36,16 @@ jobs: shell: bash run: echo ::set-output name=version::$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))") - - name: Install poetry + - name: Bootstrap poetry shell: bash run: | - python get-poetry.py -y --preview - echo "::set-env name=PATH::$HOME/.poetry/bin:$PATH" + python -m ensurepip + python -m pip install --upgrade pip + python -m pip install . - name: Configure poetry shell: bash - run: poetry config virtualenvs.in-project true + run: python -m poetry config virtualenvs.in-project true - name: Set up cache uses: actions/cache@v2 @@ -56,16 +57,16 @@ jobs: - name: Ensure cache is healthy if: steps.cache.outputs.cache-hit == 'true' shell: bash - run: poetry run pip --version >/dev/null 2>&1 || rm -rf .venv + run: python -m poetry run pip --version >/dev/null 2>&1 || rm -rf .venv - name: Upgrade pip shell: bash - run: poetry run python -m pip install pip -U + run: python -m poetry run python -m pip install pip -U - name: Install dependencies shell: bash - run: poetry install + run: python -m poetry install - name: Run pytest shell: bash - run: poetry run python -m pytest -v tests + run: python -m poetry run python -m pytest -v tests From 95523da4d7388820e9e36215d4d55f09510e82b5 Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Mon, 21 Sep 2020 12:30:38 +0200 Subject: [PATCH 7/7] ci: remove upgrade pip step --- .github/workflows/main.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4c55a7acc60..1c90c595fbf 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -59,10 +59,6 @@ jobs: shell: bash run: python -m poetry run pip --version >/dev/null 2>&1 || rm -rf .venv - - name: Upgrade pip - shell: bash - run: python -m poetry run python -m pip install pip -U - - name: Install dependencies shell: bash run: python -m poetry install