diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1718cc6..f4525f9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -50,4 +50,5 @@ jobs: - name: Test run: | export ISOLATE_PYENV_EXECUTABLE=pyenv/bin/pyenv + export AGENT_REQUIREMENTS_TXT=tools/agent_requirements.txt python -m pytest diff --git a/poetry.lock b/poetry.lock index f1e1593..797af63 100644 --- a/poetry.lock +++ b/poetry.lock @@ -7,7 +7,7 @@ optional = false python-versions = "*" [package.extras] -test = ["flake8 (==3.7.8)", "hypothesis (==3.55.3)"] +test = ["hypothesis (==3.55.3)", "flake8 (==3.7.8)"] [[package]] name = "distlib" @@ -26,8 +26,8 @@ optional = false python-versions = ">=3.7" [package.extras] -docs = ["furo (>=2022.6.21)", "sphinx (>=5.1.1)", "sphinx-autodoc-typehints (>=1.19.1)"] -testing = ["covdefaults (>=2.2)", "coverage (>=6.4.2)", "pytest (>=7.1.2)", "pytest-cov (>=3)", "pytest-timeout (>=2.1)"] +testing = ["pytest-timeout (>=2.1)", "pytest-cov (>=3)", "pytest (>=7.1.2)", "coverage (>=6.4.2)", "covdefaults (>=2.2)"] +docs = ["sphinx-autodoc-typehints (>=1.19.1)", "sphinx (>=5.1.1)", "furo (>=2022.6.21)"] [[package]] name = "grpcio" @@ -69,8 +69,8 @@ optional = false python-versions = ">=3.7" [package.extras] -docs = ["furo (>=2021.7.5b38)", "proselint (>=0.10.2)", "sphinx-autodoc-typehints (>=1.12)", "sphinx (>=4)"] -test = ["appdirs (==1.4.4)", "pytest-cov (>=2.7)", "pytest-mock (>=3.6)", "pytest (>=6)"] +test = ["pytest (>=6)", "pytest-mock (>=3.6)", "pytest-cov (>=2.7)", "appdirs (==1.4.4)"] +docs = ["sphinx (>=4)", "sphinx-autodoc-typehints (>=1.12)", "proselint (>=0.10.2)", "furo (>=2021.7.5b38)"] [[package]] name = "protobuf" @@ -91,6 +91,14 @@ python-versions = ">=3.6" [package.extras] plugins = ["importlib-metadata"] +[[package]] +name = "pyyaml" +version = "6.0" +description = "YAML parser and emitter for Python" +category = "main" +optional = false +python-versions = ">=3.6" + [[package]] name = "rich" version = "12.6.0" @@ -168,7 +176,7 @@ server = [] [metadata] lock-version = "1.1" python-versions = ">=3.7,<4.0" -content-hash = "0a5c65e4262d6400277faca3109be24bc7c3e83c88ec22bf9b3ad327f1a42825" +content-hash = "ba7c204557b1423dced5872301ac90a9cd4b3e427671a0e803ee071eee84e63b" [metadata.files] commonmark = [ @@ -258,6 +266,41 @@ pygments = [ {file = "Pygments-2.13.0-py3-none-any.whl", hash = "sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42"}, {file = "Pygments-2.13.0.tar.gz", hash = "sha256:56a8508ae95f98e2b9bdf93a6be5ae3f7d8af858b43e02c5a2ff083726be40c1"}, ] +pyyaml = [ + {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, + {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, + {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, + {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, + {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"}, + {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"}, + {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"}, + {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"}, + {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"}, + {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"}, + {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"}, + {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"}, + {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"}, + {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"}, + {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"}, + {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"}, + {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, + {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, +] rich = [ {file = "rich-12.6.0-py3-none-any.whl", hash = "sha256:a4eb26484f2c82589bd9a17c73d32a010b1e29d89f1604cd9bf3a2097b81bb5e"}, {file = "rich-12.6.0.tar.gz", hash = "sha256:ba3a3775974105c221d31141f2c116f4fd65c5ceb0698657a11e9f295ec93fd0"}, diff --git a/pyproject.toml b/pyproject.toml index ceab4ce..5495577 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ rich = ">=12.0" grpcio = ">=1.49" protobuf = "*" tblib = "^1.7.0" +PyYAML = "^6.0" [tool.poetry.extras] grpc = [] diff --git a/src/isolate/backends/conda.py b/src/isolate/backends/conda.py index 816a560..6876e55 100644 --- a/src/isolate/backends/conda.py +++ b/src/isolate/backends/conda.py @@ -4,6 +4,8 @@ import os import shutil import subprocess +import tempfile +import yaml from dataclasses import dataclass, field from pathlib import Path from typing import Any, ClassVar, Dict, List, Optional @@ -33,6 +35,7 @@ class CondaEnvironment(BaseEnvironment[Path]): packages: List[str] = field(default_factory=list) python_version: Optional[str] = None + env_dict: Optional[Dict[str, Any]] = None @classmethod def from_config( @@ -40,17 +43,30 @@ def from_config( config: Dict[str, Any], settings: IsolateSettings = DEFAULT_SETTINGS, ) -> BaseEnvironment: + if config.get('env_dict') and config.get('env_yml_str'): + raise EnvironmentCreationError("Either env_dict or env_yml_str can be provided, not both!") + if config.get('env_yml_str'): + config['env_dict'] = yaml.safe_load(config['env_yml_str']) + del config['env_yml_str'] environment = cls(**config) environment.apply_settings(settings) return environment @property def key(self) -> str: + if self.env_dict: + return sha256_digest_of(str(self._compute_dependencies())) return sha256_digest_of(*self._compute_dependencies()) - def _compute_dependencies(self) -> List[str]: - user_dependencies = self.packages.copy() + def _compute_dependencies(self) -> List[Any]: + if self.env_dict: + user_dependencies = self.env_dict.get('dependencies', []).copy() + else: + user_dependencies = self.packages.copy() for raw_requirement in user_dependencies: + # It could be 'pip': [...] + if type(raw_requirement) is dict: + continue # Get rid of all whitespace characters (python = 3.8 becomes python=3.8) raw_requirement = raw_requirement.replace(" ", "") if not raw_requirement.startswith("python"): @@ -87,28 +103,46 @@ def create(self) -> Path: if env_path.exists(): return env_path - # Since our agent needs Python to be installed (at very least) - # we need to make sure that the base environment is created with - # the same Python version as the one that is used to run the - # isolate agent. - dependencies = self._compute_dependencies() - - self.log(f"Creating the environment at '{env_path}'") - self.log(f"Installing packages: {', '.join(dependencies)}") - - try: - self._run_conda( - "create", - "--yes", - "--prefix", - env_path, - *dependencies, - ) - except subprocess.SubprocessError as exc: - raise EnvironmentCreationError("Failure during 'conda create'") from exc - - self.log(f"New environment cached at '{env_path}'") - return env_path + if self.env_dict: + self.env_dict['dependencies'] = self._compute_dependencies() + with tempfile.NamedTemporaryFile(mode='w', suffix='.yml') as tf: + yaml.dump(self.env_dict, tf) + tf.flush() + try: + self._run_conda( + "env", + "create", + "-f", + tf.name, + "--prefix", + env_path + ) + except subprocess.SubprocessError as exc: + raise EnvironmentCreationError("Failure during 'conda create'") from exc + + else: + # Since our agent needs Python to be installed (at very least) + # we need to make sure that the base environment is created with + # the same Python version as the one that is used to run the + # isolate agent. + dependencies = self._compute_dependencies() + + self.log(f"Creating the environment at '{env_path}'") + self.log(f"Installing packages: {', '.join(dependencies)}") + + try: + self._run_conda( + "create", + "--yes", + "--prefix", + env_path, + *dependencies, + ) + except subprocess.SubprocessError as exc: + raise EnvironmentCreationError("Failure during 'conda create'") from exc + + self.log(f"New environment cached at '{env_path}'") + return env_path def destroy(self, connection_key: Path) -> None: with self.settings.cache_lock_for(connection_key): diff --git a/tests/test_backends.py b/tests/test_backends.py index 479465b..c04646e 100644 --- a/tests/test_backends.py +++ b/tests/test_backends.py @@ -380,6 +380,11 @@ class TestConda(GenericEnvironmentTests): "new-python": { "python_version": "3.10", }, + "yml-with-isolate": { + "env_yml_str": 'name: test\n' + \ + 'channels:\n - defaults\n' + \ + 'dependencies:\n - pip:\n - isolate==0.7.1\n - pyjokes==0.5.0\n' + } } creation_entry_point = ("subprocess.check_call", subprocess.SubprocessError) @@ -431,6 +436,10 @@ def test_fail_when_user_overwrites_python( ): environment.create() + def test_environment_with_yml(self, tmp_path): + environment = self.get_project_environment(tmp_path, "yml-with-isolate") + connection_key = environment.create() + assert self.get_example_version(environment, connection_key) == "0.5.0" def test_local_python_environment(): """Since 'local' environment does not support installation of extra dependencies diff --git a/tools/Dockerfile b/tools/Dockerfile index a114617..518b451 100644 --- a/tools/Dockerfile +++ b/tools/Dockerfile @@ -9,6 +9,27 @@ FROM python:3.9 RUN apt-get update && apt-get install -y git + +RUN mkdir -p /opt +RUN git clone https://github.com/pyenv/pyenv --branch v2.3.6 --depth=1 /opt/pyenv +# TODO: Investigate whether we can leverage the compiled pyenv extension. +ENV ISOLATE_PYENV_EXECUTABLE=/opt/pyenv/bin/pyenv + +#### CONDA #### +# Will copy from existing Docker image +COPY --from=continuumio/miniconda3:4.12.0 /opt/conda /opt/conda + +ENV PATH=/opt/conda/bin:$PATH +ENV ISOLATE_CONDA_HOME=/opt/conda/bin + +# Usage examples +RUN set -ex && \ + conda config --set always_yes yes --set changeps1 no && \ + conda info -a && \ + conda config --add channels conda-forge && \ + conda install --quiet --freeze-installed -c main conda-pack + +#### END CONDA #### RUN pip install --upgrade pip virtualenv wheel poetry-core # Since system-level debian does not comply with @@ -22,9 +43,10 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH" COPY tools/requirements.txt /tmp/requirements.txt RUN pip install -r /tmp/requirements.txt -COPY . . -RUN pip install . +COPY . /isolate +RUN pip install /isolate[server] ENV ISOLATE_INHERIT_FROM_LOCAL=1 +ENV AGENT_REQUIREMENTS_TXT=/isolate/tools/agent_requirements.txt CMD ["python", "-m", "isolate.server.server"] diff --git a/tools/agent_requirements.txt b/tools/agent_requirements.txt new file mode 100644 index 0000000..e0c97d2 --- /dev/null +++ b/tools/agent_requirements.txt @@ -0,0 +1,3 @@ +/isolate[server] +dill==0.3.5.1 +google-cloud-storage==2.6.0