fix: if a spec is source, don't merge but force the source (#405)

Author: ruben-arts

backends/pixi-build-ros/pixi.lock

@@ -20,7 +20,10 @@ scripts = ["../../scripts/activate.sh"]
 [dependencies]
 pydantic = ">=2.8.2,<3"
 py_rattler = ">=0.15.0,<0.16"
-pixi-build-ros = { path = "." }
+py-pixi-build-backend = "*"
+
+[pypi-dependencies]
+pixi-build-ros = { path = ".", editable = true }
 
 [environments]
 default = { features = ["test", "lint"], solve-group = "default" }
@@ -74,5 +77,5 @@ pixi-build-api-version = ">=2,<3"
 # should be added to `py-pixi-build-backend`
 typing-extensions = "*"
 # this depends has to match the test dependency, so switch comments if needed
-# py-pixi-build-backend = "*"
+#py-pixi-build-backend = "*"
 py-pixi-build-backend = { path = "../../py-pixi-build-backend" }

backends/pixi-build-ros/pixi.toml

@@ -7,7 +7,8 @@ dependencies = [
   "pyyaml",
   "pydantic",
   "py_rattler",
-  "py-pixi-build-backend",
+  #  TODO: publish to pypi
+  #  "py-pixi-build-backend",
 ]
 name = "pixi-build-ros"
 version = "0.3.0"

backends/pixi-build-ros/pyproject.toml

@@ -358,6 +358,8 @@ liblz4-dev:
   robostack: [lz4]
 libmicrohttpd:
   robostack: [libmicrohttpd]
+libnanoflann-dev:
+  robostack: [nanoflann]
 libncurses-dev:
   robostack:
     linux: [ncurses]
@@ -904,10 +906,14 @@ python3-tk:
   robostack: [tk]
 python3-tornado:
   robostack: [tornado]
+python3-transforms3d:
+  robostack: [transforms3d]
 python3-twisted:
   robostack: [twisted]
 python3-typeguard:
   robostack: [typeguard]
+python3-types-pyyaml:
+  robostack: [types-pyyaml]
 python3-unidiff:
   robostack: [unidiff]
 python3-usb:

backends/pixi-build-ros/robostack.yaml

@@ -1,5 +1,5 @@
 """
-Python generator implementation using Python bindings.
+ROS generator implementation using Python bindings.
 """
 
 from pathlib import Path
@@ -11,9 +11,9 @@
     GeneratedRecipe,
 )
 from .metadata_provider import ROSPackageXmlMetadataProvider
-from pixi_build_backend.types.intermediate_recipe import Script, ConditionalRequirements
+from pixi_build_backend.types.intermediate_recipe import Script
 
-from pixi_build_backend.types.item import ItemPackageDependency, VecItemPackageDependency
+from pixi_build_backend.types.item import ItemPackageDependency
 from pixi_build_backend.types.platform import Platform
 from pixi_build_backend.types.project_model import ProjectModelV1
 from pixi_build_backend.types.python_params import PythonParams
@@ -25,6 +25,7 @@
     convert_package_xml_to_catkin_package,
     get_package_xml_content,
     load_package_map_data,
+    merge_requirements,
 )
 from .config import ROSBackendConfig, PackageMappingSource
 
@@ -153,69 +154,3 @@ def default_variants(self, host_platform: Platform) -> dict[str, Any]:
         if host_platform.is_windows:
             variants["cxx_compiler"] = ["vs2019"]
         return variants
-
-
-def merge_requirements(
-    model_requirements: ConditionalRequirements,
-    package_requirements: ConditionalRequirements,
-) -> ConditionalRequirements:
-    """Merge two sets of requirements."""
-    merged = ConditionalRequirements()
-
-    merged.host = merge_unique_items(model_requirements.host, package_requirements.host)
-    merged.build = merge_unique_items(model_requirements.build, package_requirements.build)
-    merged.run = merge_unique_items(model_requirements.run, package_requirements.run)
-
-    # If the dependency is of type Source in one of the requirements, we need to set them to Source for all variants
-    return merged
-
-
-def merge_unique_items(
-    model: list[ItemPackageDependency] | VecItemPackageDependency,
-    package: list[ItemPackageDependency] | VecItemPackageDependency,
-) -> list[ItemPackageDependency]:
-    """Merge unique items from source into target."""
-
-    def _find_matching(list_to_find: list[ItemPackageDependency], name: str) -> ItemPackageDependency | None:
-        for dep in list_to_find:
-            if dep.concrete.package_name == name:
-                return dep
-        else:
-            return None
-
-    def _merge_specs(spec1: str, spec2: str, package_name: str) -> str:
-        # remove the package name
-        version_spec1 = spec1.removeprefix(package_name).strip()
-        version_spec2 = spec2.removeprefix(package_name).strip()
-
-        if " " in version_spec1 or " " in version_spec2:
-            raise ValueError(f"{version_spec1}, or {version_spec2} contains spaces, cannot merge specifiers.")
-
-        # early out with *, empty or ==
-        if version_spec1 in ["*", ""] or "==" in version_spec2 or version_spec1 == version_spec2:
-            return spec2
-        if version_spec2 in ["*", ""] or "==" in version_spec1:
-            return spec1
-        return package_name + " " + ",".join([version_spec1, version_spec2])
-
-    result: list[ItemPackageDependency] = []
-    templates_in_model = [str(i.template) for i in model]
-    for item in list(model) + list(package):
-        # It's concrete (i.e. no template)
-        if item.concrete is not None:
-            # It does not exist yet in model
-            item_in_result = _find_matching(result, item.concrete.package_name)
-            if not item_in_result:
-                result.append(item)
-            else:
-                new_dep = ItemPackageDependency(
-                    name=_merge_specs(
-                        item_in_result.concrete.binary_spec, item.concrete.binary_spec, item.concrete.package_name
-                    )
-                )
-                result.remove(item_in_result)
-                result.append(new_dep)
-
-        elif str(item.template) not in templates_in_model:
-            result.append(item)
-    return result

backends/pixi-build-ros/src/pixi_build_ros/ros_generator.py

@@ -7,7 +7,7 @@
 from catkin_pkg.package import Package as CatkinPackage, parse_package_string, Dependency
 
 from pixi_build_backend.types.intermediate_recipe import ConditionalRequirements
-from pixi_build_backend.types.item import ItemPackageDependency
+from pixi_build_backend.types.item import ItemPackageDependency, VecItemPackageDependency
 from pixi_build_backend.types.platform import Platform
 from pixi_build_ros.distro import Distro
 from rattler import Version
@@ -262,3 +262,85 @@ def package_xml_to_conda_requirements(
     cond.run = run_requirements
 
     return cond
+
+
+def find_matching(list_to_find: list[ItemPackageDependency], name: str) -> ItemPackageDependency | None:
+    for dep in list_to_find:
+        if dep.concrete.package_name == name:
+            return dep
+    else:
+        return None
+
+
+def normalize_spec(spec: str | None, package_name: str) -> str:
+    """Normalize a spec by removing package name and handling None."""
+    if not spec:
+        return ""
+    return spec.removeprefix(package_name).strip()
+
+
+def merge_specs(spec1: str | None, spec2: str | None, package_name: str) -> str:
+    # remove the package name
+    version_spec1 = normalize_spec(spec1, package_name)
+    version_spec2 = normalize_spec(spec2, package_name)
+
+    if " " in version_spec1 or " " in version_spec2:
+        raise ValueError(f"{version_spec1}, or {version_spec2} contains spaces, cannot merge specifiers.")
+
+    # early out with *, empty or ==
+    if version_spec1 in ["*", ""] or "==" in version_spec2 or version_spec1 == version_spec2:
+        return spec2 or ""
+    if version_spec2 in ["*", ""] or "==" in version_spec1:
+        return spec1 or ""
+    return package_name + " " + ",".join([version_spec1, version_spec2])
+
+
+def merge_unique_items(
+    model: list[ItemPackageDependency] | VecItemPackageDependency,
+    package: list[ItemPackageDependency] | VecItemPackageDependency,
+) -> list[ItemPackageDependency]:
+    """Merge unique items from source into target."""
+
+    result: list[ItemPackageDependency] = []
+    templates_in_model = [str(i.template) for i in model]
+    for item in list(model) + list(package):
+        # It's concrete (i.e. no template)
+        if item.concrete is not None:
+            # It does not exist yet in model
+            item_in_result = find_matching(result, item.concrete.package_name)
+            if not item_in_result:
+                result.append(item)
+            elif item_in_result.concrete.is_source:
+                # If the existing dependency is source, don't merge - keep the source one
+                continue
+            elif item.concrete.is_source:
+                # If a new item is source, replace the existing one
+                result = [dep for dep in result if dep.concrete.package_name != item.concrete.package_name]
+                result.append(item)
+            else:
+                new_dep = ItemPackageDependency(
+                    name=merge_specs(
+                        item_in_result.concrete.binary_spec, item.concrete.binary_spec, item.concrete.package_name
+                    )
+                )
+                result.remove(item_in_result)
+                result.append(new_dep)
+
+        elif str(item.template) not in templates_in_model:
+            result.append(item)
+    return result
+
+
+def merge_requirements(
+    model_requirements: ConditionalRequirements,
+    package_requirements: ConditionalRequirements,
+) -> ConditionalRequirements:
+    """Merge two sets of requirements."""
+    merged = ConditionalRequirements()
+
+    merged.host = merge_unique_items(model_requirements.host, package_requirements.host)
+    merged.build = merge_unique_items(model_requirements.build, package_requirements.build)
+    merged.run = merge_unique_items(model_requirements.run, package_requirements.run)
+
+    # If the dependency is of type Source in one of the requirements, we need to set them to Source for all variants
+    return merged

backends/pixi-build-ros/src/pixi_build_ros/utils.py

@@ -1,6 +1,6 @@
 import pytest
 from pixi_build_backend.types.item import ItemPackageDependency
-from pixi_build_ros.ros_generator import merge_unique_items
+from pixi_build_ros.utils import merge_unique_items
 
 
 def test_with_star_items():
@@ -40,3 +40,17 @@ def test_specs_with_spaces():
     with pytest.raises(ValueError) as exc:
         merge_unique_items(list1, list2)
     assert "contains spaces" in str(exc)
+
+
+def test_specs_with_none():
+    list1 = [ItemPackageDependency("ros-noetic")]
+    list2 = [ItemPackageDependency("ros-noetic <=2.0,<3.0")]
+    result = merge_unique_items(list1, list2)
+    assert result[0].concrete.binary_spec == list2[0].concrete.binary_spec
+
+
+def test_specs_with_source():
+    list1 = [ItemPackageDependency('ros-noetic[url="https://blabla"]')]
+    list2 = [ItemPackageDependency("ros-noetic <=2.0,<3.0")]
+    result = merge_unique_items(list1, list2)
+    assert str(result[0].concrete.source_spec.spec) == str(list1[0].concrete.source_spec.spec)