Merge pull request #2099 from jaimergp/pixi

conda_smithy/configure_feedstock.py

@@ -16,6 +16,7 @@
 from itertools import chain, product
 from os import fspath
 from pathlib import Path, PurePath
+from typing import Dict
 
 import requests
 import yaml
@@ -663,7 +664,26 @@ def _yaml_represent_ordereddict(yaml_representer, data):
     )
 
 
-def _santize_remote_ci_setup(remote_ci_setup):
+def _has_local_ci_setup(forge_dir, forge_config):
+    # If the recipe has its own conda_forge_ci_setup package, then
+    # install that
+    return os.path.exists(
+        os.path.join(
+            forge_dir,
+            forge_config["recipe_dir"],
+            "conda_forge_ci_setup",
+            "__init__.py",
+        )
+    ) and os.path.exists(
+        os.path.join(
+            forge_dir,
+            forge_config["recipe_dir"],
+            "setup.py",
+        )
+    )
+
+
+def _sanitize_remote_ci_setup(remote_ci_setup):
     remote_ci_setup_ = conda_build.utils.ensure_list(remote_ci_setup)
     remote_ci_setup = []
     for package in remote_ci_setup_:
@@ -675,6 +695,31 @@ def _santize_remote_ci_setup(remote_ci_setup):
     return remote_ci_setup
 
 
+def _sanitize_build_tool_deps_as_dict(
+    forge_dir, forge_config
+) -> Dict[str, str]:
+    """
+    Aggregates different sources of build tool dependencies in
+    mapping of package names to OR-merged version constraints.
+    """
+    deps = [
+        *forge_config["conda_build_tool_deps"].split(),
+        *forge_config["remote_ci_setup"],
+    ]
+    merged = {
+        spec.name: str(spec.version or "*")
+        for spec in MatchSpec.merge([dep.strip("\"'") for dep in deps])
+    }
+    if forge_config.get("local_ci_setup") or _has_local_ci_setup(
+        forge_dir, forge_config
+    ):
+        # We need to conda uninstall conda-forge-ci-setup
+        # and then pip install on top
+        merged.setdefault("conda", "*")
+        merged.setdefault("pip", "*")
+    return merged
+
+
 def finalize_config(config, platform, arch, forge_config):
     """For configs without essential parameters like docker_image
     add fallback value.
@@ -1188,25 +1233,9 @@ def _render_ci_provider(
             fast_finish_text=fast_finish_text,
         )
 
-        # If the recipe has its own conda_forge_ci_setup package, then
-        # install that
-        if os.path.exists(
-            os.path.join(
-                forge_dir,
-                forge_config["recipe_dir"],
-                "conda_forge_ci_setup",
-                "__init__.py",
-            )
-        ) and os.path.exists(
-            os.path.join(
-                forge_dir,
-                forge_config["recipe_dir"],
-                "setup.py",
-            )
-        ):
-            forge_config["local_ci_setup"] = True
-        else:
-            forge_config["local_ci_setup"] = False
+        forge_config["local_ci_setup"] = _has_local_ci_setup(
+            forge_dir, forge_config
+        )
 
         # hook for extending with whatever platform specific junk we need.
         #     Function passed in as argument
@@ -2194,6 +2223,34 @@ def render_github_actions_services(jinja_env, forge_config, forge_dir):
             fh.write(new_file_contents)
 
 
+def render_pixi(jinja_env, forge_config, forge_dir):
+    target_fname = os.path.join(forge_dir, "pixi.toml")
+    remove_file_or_dir(target_fname)
+    if forge_config["conda_install_tool"] != "pixi":
+        return
+    template = jinja_env.get_template("pixi.toml.tmpl")
+    ci_support_path = os.path.join(forge_dir, ".ci_support")
+    variants = []
+    if os.path.exists(ci_support_path):
+        for filename in os.listdir(ci_support_path):
+            if filename.endswith(".yaml"):
+                variant_name, _ = os.path.splitext(filename)
+                variants.append(variant_name)
+    platforms = {
+        platform.replace("_", "-")
+        for platform, service in forge_config["provider"].items()
+        if service
+    }
+    new_file_contents = template.render(
+        smithy_version=__version__,
+        platforms=sorted(platforms),
+        variants=variants,
+        **forge_config,
+    )
+    with write_file(target_fname) as fh:
+        fh.write(new_file_contents)
+
+
 def copy_feedstock_content(forge_config, forge_dir):
     feedstock_content = os.path.join(conda_forge_content, "feedstock_content")
     skip_files = _get_skip_files(forge_config)
@@ -2410,7 +2467,7 @@ def _load_forge_config(forge_dir, exclusive_config_file, forge_yml=None):
     if config["provider"]["linux_s390x"] in {"default", "native"}:
         config["provider"]["linux_s390x"] = ["travis"]
 
-    config["remote_ci_setup"] = _santize_remote_ci_setup(
+    config["remote_ci_setup"] = _sanitize_remote_ci_setup(
         config["remote_ci_setup"]
     )
     if config["conda_install_tool"] == "conda":
@@ -2421,6 +2478,11 @@ def _load_forge_config(forge_dir, exclusive_config_file, forge_yml=None):
     else:
         config["remote_ci_setup_update"] = config["remote_ci_setup"]
 
+    # Post-process requirements so they are tidier for the TOML files
+    config["build_tool_deps_dict"] = _sanitize_build_tool_deps_as_dict(
+        forge_dir, config
+    )
+
     if not config["github_actions"]["triggers"]:
         self_hosted = config["github_actions"]["self_hosted"]
         config["github_actions"]["triggers"] = (
@@ -2778,7 +2840,6 @@ def main(
         )
 
     config = _load_forge_config(forge_dir, exclusive_config_file, forge_yml)
-
     config["feedstock_name"] = os.path.basename(forge_dir)
 
     env = make_jinja_env(forge_dir)
@@ -2792,50 +2853,51 @@ def main(
     clear_variants(forge_dir)
     clear_scripts(forge_dir)
     set_migration_fns(forge_dir, config)
-
     logger.debug("migration fns set")
 
     # the order of these calls appears to matter
     render_info = []
     render_info.append(
         render_circle(env, config, forge_dir, return_metadata=True)
     )
-
     logger.debug("circle rendered")
+
     render_info.append(
         render_travis(env, config, forge_dir, return_metadata=True)
     )
-
     logger.debug("travis rendered")
+
     render_info.append(
         render_appveyor(env, config, forge_dir, return_metadata=True)
     )
-
     logger.debug("appveyor rendered")
+
     render_info.append(
         render_azure(env, config, forge_dir, return_metadata=True)
     )
-
     logger.debug("azure rendered")
+
     render_info.append(
         render_drone(env, config, forge_dir, return_metadata=True)
     )
-
     logger.debug("drone rendered")
+
     render_info.append(
         render_woodpecker(env, config, forge_dir, return_metadata=True)
     )
-
     logger.debug("woodpecker rendered")
+
     render_info.append(
         render_github_actions(env, config, forge_dir, return_metadata=True)
     )
-
     logger.debug("github_actions rendered")
-    render_github_actions_services(env, config, forge_dir)
 
+    render_github_actions_services(env, config, forge_dir)
     logger.debug("github_actions services rendered")
 
+    render_pixi(env, config, forge_dir)
+    logger.debug("pixi config rendered")
+
     # put azure first just in case
     azure_ind = ([ri["provider_name"] for ri in render_info]).index("azure")
     tmp = render_info[0]

conda_smithy/data/conda-forge.json

@@ -1768,7 +1768,8 @@
           "enum": [
             "conda",
             "mamba",
-            "micromamba"
+            "micromamba",
+            "pixi"
           ],
           "type": "string"
         },
@@ -1777,7 +1778,7 @@
         }
       ],
       "default": "micromamba",
-      "description": "Use this option to choose which tool is used to provision the tooling in your\nfeedstock. Defaults to micromamba.\n\nIf conda or mamba are chosen, the latest Miniforge will be used to\nprovision the base environment. If micromamba is chosen, Miniforge\nis not involved; the environment is created directly by micromamba.",
+      "description": "Use this option to choose which tool is used to provision the tooling in your\nfeedstock. Defaults to micromamba.\n\nIf conda or mamba are chosen, the latest Miniforge will be used to\nprovision the base environment. If micromamba or pixi are chosen,\nMiniforge is not involved; the environment is created directly by\nmicromamba or pixi.",
       "title": "Conda Install Tool"
     },
     "conda_forge_output_validation": {

conda_smithy/schema.py

@@ -626,20 +626,21 @@ class ConfigModel(BaseModel):
         ),
     )
 
-    conda_install_tool: Optional[Literal["conda", "mamba", "micromamba"]] = (
-        Field(
-            default="micromamba",
-            description=cleandoc(
-                """
+    conda_install_tool: Optional[
+        Literal["conda", "mamba", "micromamba", "pixi"]
+    ] = Field(
+        default="micromamba",
+        description=cleandoc(
+            """
                 Use this option to choose which tool is used to provision the tooling in your
                 feedstock. Defaults to micromamba.
 
                 If conda or mamba are chosen, the latest Miniforge will be used to
-                provision the base environment. If micromamba is chosen, Miniforge
-                is not involved; the environment is created directly by micromamba.
+                provision the base environment. If micromamba or pixi are chosen,
+                Miniforge is not involved; the environment is created directly by
+                micromamba or pixi.
                 """
-            ),
-        )
+        ),
     )
 
     conda_forge_output_validation: Optional[bool] = Field(

conda_smithy/templates/build_steps.sh.tmpl

@@ -40,6 +40,22 @@ echo > /opt/conda/conda-meta/history
 micromamba install --root-prefix ~/.conda --prefix /opt/conda \
     --yes --override-channels --channel conda-forge --strict-channel-priority \
     pip {{ conda_install_tool_deps }} {{ conda_build_tool_deps }} {{ " ".join(remote_ci_setup) }}
+{%- elif conda_install_tool == "pixi" %}
+curl -fsSL https://pixi.sh/install.sh | bash
+export PATH="~/.pixi/bin:$PATH"
+pushd "${FEEDSTOCK_ROOT}"
+arch=$(uname -m)
+if [[ "$arch" == "x86_64" ]]; then
+  arch="64"
+fi
+sed -i.bak "s/platforms = .*/platforms = [\"linux-${arch}\"]/" pixi.toml
+echo "Creating environment"
+PIXI_CACHE_DIR=/opt/conda pixi install
+pixi list
+echo "Activating environment"
+eval "$(pixi shell-hook)"
+mv pixi.toml.bak pixi.toml
+popd
 {%- endif %}
 
 {%- if conda_build_tool == "mambabuild" %}
@@ -66,7 +82,7 @@ export CONDA_LIBMAMBA_SOLVER_NO_CHANNELS_FROM_INSTALLED=1
 {%- endif %}
 {% if local_ci_setup %}
 conda uninstall --quiet --yes --force {{ " ".join(remote_ci_setup)}}
-pip install --no-deps ${RECIPE_ROOT}/.
+pip install --no-deps "${RECIPE_ROOT}/."
 {%- endif %}
 # set up the condarc
 setup_conda_rc "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}"

conda_smithy/templates/pixi.toml.tmpl

@@ -0,0 +1,46 @@
+# This file was generated automatically from conda-smithy. To update this configuration,
+# update the conda-forge.yml and/or the recipe/meta.yaml.
+# -*- mode: toml -*-
+
+[project]
+name = "{{ feedstock_name }}"
+version = "{{ smithy_version }}"
+description = "Pixi configuration for conda-forge/{{ feedstock_name }}"
+authors = ["@conda-forge/{{ feedstock_name[:-10] }}"]
+channels = ["conda-forge"]
+platforms = {{ platforms }}
+
+[dependencies]
+{%- for spec_name, spec_constraints in build_tool_deps_dict.items() %}
+{{ spec_name }} = "{{ spec_constraints }}"
+{%- endfor %}
+
+[tasks]
+inspect-all = "inspect_artifacts --all-packages"
+{%- if conda_build_tool != "rattler-build" %}
+build = "{{ conda_build_tool }} build {{ recipe_dir }}"
+debug = "{{ conda_build_tool }} debug {{ recipe_dir }}"
+{%- else %}
+build = "{{ conda_build_tool }} build --recipe {{ recipe_dir }}"
+{%- endif %}
+{%- for variant in variants %}
+{%- if conda_build_tool != "rattler-build" %}
+"build-{{ variant }}" = "{{ conda_build_tool }} build {{ recipe_dir }} -m .ci_support/{{ variant }}.yaml --suppress-variables --clobber-file .ci_support/clobber_{{ variant }}.yaml"
+"debug-{{ variant }}" = "{{ conda_build_tool }} debug {{ recipe_dir }} -m .ci_support/{{ variant }}.yaml"
+{%- else %}
+"build-{{ variant }}" = "{{ conda_build_tool }} build --recipe {{ recipe_dir }} -m .ci_support/{{ variant }}.yaml"
+{%- endif %}
+"inspect-{{ variant }}" = "inspect_artifacts --recipe-dir {{ recipe_dir }} -m .ci_support/{{ variant }}.yaml"
+{%- endfor %}
+
+[feature.smithy.dependencies]
+conda-smithy = "*"
+
+[feature.smithy.tasks]
+build-locally = "python ./build-locally.py"
+smithy = "conda-smithy"
+rerender = "conda-smithy rerender"
+lint = "conda-smithy lint {{ recipe_dir}}"
+
+[environments]
+smithy = ["smithy"]

conda_smithy/templates/run_osx_build.sh.tmpl

@@ -32,6 +32,22 @@ mv "${MAMBA_ROOT_PREFIX}/pkgs" "${MINIFORGE_HOME}"
 echo "Cleaning up micromamba"
 rm -rf "${MAMBA_ROOT_PREFIX}" "${micromamba_exe}" || true
 ( endgroup "Provisioning base env with micromamba" ) 2> /dev/null
+{%- elif conda_install_tool == "pixi" %}
+( startgroup "Provisioning base env with pixi" ) 2> /dev/null
+curl -fsSL https://pixi.sh/install.sh | bash
+export PATH="~/.pixi/bin:$PATH"
+arch=$(uname -m)
+if [[ "$arch" == "x86_64" ]]; then
+  arch="64"
+fi
+sed -i.bak "s/platforms = .*/platforms = [\"osx-${arch}\"]/" pixi.toml
+echo "Creating environment"
+pixi install
+pixi list
+echo "Activating environment"
+eval "$(pixi shell-hook)"
+mv pixi.toml.bak pixi.toml
+( endgroup "Provisioning base env with pixi" ) 2> /dev/null
 {%- else %}
 ( startgroup "Installing a fresh version of Miniforge" ) 2> /dev/null
 
@@ -57,8 +73,11 @@ bash "${MINIFORGE_FILE}" -b -p "${MINIFORGE_HOME}"
 {%- set BUILD_CMD="conda-build" %}
 {%- endif %}
 
+{%- if conda_install_tool != "pixi" %}
+echo "Activating environment"
 source "${MINIFORGE_HOME}/etc/profile.d/conda.sh"
 conda activate base
+{%- endif %}
 {%- if conda_solver %}
 export CONDA_SOLVER="{{ conda_solver }}"
 {%- endif %}