feat(workflow): naming metadata for command parameters (#2071)

mohammad-alisafaee · web-flow · commit b1e7a9b29138 · 2021-05-17T12:29:01.000+02:00
* feat(workflow): naming metadata for command parameters
diff --git a/renku/core/models/workflow/dependency_graph.py b/renku/core/models/workflow/dependency_graph.py
@@ -44,7 +44,7 @@ def __init__(self, plans=None):
         self._connect_all_nodes()
 
     @property
-    def plans(self):
+    def plans(self) -> List[Plan]:
         """A list of all plans in the graph."""
         return list(self._plans)
 
diff --git a/renku/core/models/workflow/parameters.py b/renku/core/models/workflow/parameters.py
@@ -27,6 +27,9 @@
 
 from renku.core.models.calamus import JsonLDSchema, Nested, fields, rdfs, renku, schema
 from renku.core.models.entities import CollectionSchema, EntitySchema
+from renku.core.utils.urls import get_slug
+
+RANDOM_ID_LENGTH = 4
 
 
 @attr.s(eq=False, order=False)
@@ -80,25 +83,44 @@ def as_jsonld(self):
 
 
 @attr.s(eq=False, order=False)
-class CommandParameter(object):
+class CommandParameter:
     """Represents a parameter for an execution template."""
 
     _id = attr.ib(default=None, kw_only=True)
     _label = attr.ib(default=None, kw_only=True)
 
+    default_value = attr.ib(default=None, kw_only=True,)
+
+    description = attr.ib(default=None, kw_only=True,)
+
+    name: str = attr.ib(default=None, kw_only=True)
+
     position = attr.ib(default=None, type=int, kw_only=True,)
 
     prefix = attr.ib(default=None, type=str, kw_only=True,)
 
-    default_value = attr.ib(default=None, kw_only=True,)
-
     @property
     def sanitized_id(self):
         """Return ``_id`` sanitized for use in non-jsonld contexts."""
         if "/steps/" in self._id:
             return "/".join(self._id.split("/")[-4:])
         return "/".join(self._id.split("/")[-2:])
 
+    def default_label(self):
+        """Set default label."""
+        raise NotImplementedError
+
+    def default_name(self):
+        """Create a default name."""
+        raise NotImplementedError
+
+    def __attrs_post_init__(self):
+        """Post-init hook."""
+        if not self._label:
+            self._label = self.default_label()
+        if not self.name:
+            self.name = self.default_name()
+
 
 @attr.s(eq=False, order=False)
 class CommandArgument(CommandParameter):
@@ -119,6 +141,10 @@ def default_label(self):
         """Set default label."""
         return 'Command Argument "{}"'.format(self.default_value)
 
+    def default_name(self):
+        """Create a default name."""
+        return _generate_name(base="param", prefix=self.prefix, position=self.position)
+
     def to_argv(self):
         """String representation (sames as cmd argument)."""
         if self.prefix:
@@ -130,8 +156,7 @@ def to_argv(self):
 
     def __attrs_post_init__(self):
         """Post-init hook."""
-        if not self._label:
-            self._label = self.default_label()
+        super().__attrs_post_init__()
 
         if not self.default_value:
             self.default_value = self.value
@@ -172,6 +197,10 @@ def default_label(self):
         """Set default label."""
         return 'Command Input "{}"'.format(self.default_value)
 
+    def default_name(self):
+        """Create a default name."""
+        return _generate_name(base="input", prefix=self.prefix, position=self.position)
+
     def to_argv(self):
         """String representation (sames as cmd argument)."""
         if self.prefix:
@@ -190,8 +219,7 @@ def to_stream_repr(self):
 
     def __attrs_post_init__(self):
         """Post-init hook."""
-        if not self._label:
-            self._label = self.default_label()
+        super().__attrs_post_init__()
 
         if not self.default_value:
             self.default_value = self.consumes.path
@@ -224,6 +252,10 @@ def generate_id(plan_id, position=None, id_=None):
             id_ = str(position) if position else uuid.uuid4().hex
         return f"{plan_id}/inputs/{id_}"
 
+    def default_name(self):
+        """Create a default name."""
+        return _generate_name(base="input", prefix=self.prefix, position=self.position)
+
     def default_label(self):
         """Set default label."""
         return 'Command Input Template "{}"'.format(self.default_value)
@@ -239,11 +271,6 @@ def to_stream_repr(self):
 
         return " < {}".format(self.default_value)
 
-    def __attrs_post_init__(self):
-        """Post-init hook."""
-        if not self._label:
-            self._label = self.default_label()
-
     @classmethod
     def from_jsonld(cls, data):
         """Create an instance from JSON-LD data."""
@@ -282,6 +309,10 @@ def default_label(self):
         """Set default label."""
         return 'Command Output "{}"'.format(self.default_value)
 
+    def default_name(self):
+        """Create a default name."""
+        return _generate_name(base="output", prefix=self.prefix, position=self.position)
+
     def to_argv(self):
         """String representation (sames as cmd argument)."""
         if self.prefix:
@@ -303,12 +334,11 @@ def to_stream_repr(self):
 
     def __attrs_post_init__(self):
         """Post-init hook."""
+        super().__attrs_post_init__()
+
         if not self.default_value:
             self.default_value = self.produces.path
 
-        if not self._label:
-            self._label = self.default_label()
-
     @classmethod
     def from_jsonld(cls, data):
         """Create an instance from JSON-LD data."""
@@ -389,6 +419,10 @@ def default_label(self):
         """Set default label."""
         return 'Command Output Template "{}"'.format(self.default_value)
 
+    def default_name(self):
+        """Create a default name."""
+        return _generate_name(base="output", prefix=self.prefix, position=self.position)
+
     def to_argv(self):
         """String representation (sames as cmd argument)."""
         raise RuntimeError("Cannot use CommandOutputTemplate in a command.")
@@ -403,11 +437,6 @@ def to_stream_repr(self):
 
         return " 2> {}".format(self.default_value)
 
-    def __attrs_post_init__(self):
-        """Post-init hook."""
-        if not self._label:
-            self._label = self.default_label()
-
     @classmethod
     def from_jsonld(cls, data):
         """Create an instance from JSON-LD data."""
@@ -450,9 +479,11 @@ class Meta:
 
     _id = fields.Id(init_name="id")
     _label = fields.String(rdfs.label, init_name="label")
+    default_value = fields.Raw(schema.defaultValue, missing=None)
+    description = fields.String(schema.description, missing=None)
+    name = fields.String(schema.name, missing=None)
     position = fields.Integer(renku.position, missing=None)
     prefix = fields.String(renku.prefix, missing=None)
-    default_value = fields.Raw(schema.defaultValue, missing=None)
 
 
 class CommandArgumentSchema(CommandParameterSchema):
@@ -525,7 +556,7 @@ class Meta:
 
 
 class RunParameterSchema(JsonLDSchema):
-    """CommandParameter schema."""
+    """RunParameter schema."""
 
     class Meta:
         """Meta class."""
@@ -539,3 +570,9 @@ class Meta:
     name = fields.String(schema.name)
     value = fields.String(renku.value)
     type = fields.String(renku.type)
+
+
+def _generate_name(base, prefix, position):
+    name = get_slug(prefix.strip(" -=")) if prefix else base
+    position = position or uuid.uuid4().hex[:RANDOM_ID_LENGTH]
+    return f"{name}-{position}"
diff --git a/renku/core/models/workflow/plan.py b/renku/core/models/workflow/plan.py
@@ -120,7 +120,7 @@ def _generate_name(run):
         if not run:
             return uuid.uuid4().hex[:MAX_GENERATED_NAME_LENGTH]
 
-        name = "-".join(run.to_argv())
+        name = "-".join(str(a) for a in run.to_argv())
         name = secure_filename(name)
         rand_length = 5
         return f"{name[:MAX_GENERATED_NAME_LENGTH - rand_length -1]}-{uuid.uuid4().hex[:rand_length]}"
@@ -179,7 +179,9 @@ def convert_input(input_: CommandInputTemplate) -> CommandInput:
             return CommandInput(
                 id=input_._id.replace(self.id_, run_id),
                 consumes=entity,
+                description=input_.description,
                 mapped_to=input_.mapped_to,
+                name=input_.name,
                 position=input_.position,
                 prefix=input_.prefix,
             )
@@ -192,11 +194,13 @@ def convert_output(output: CommandOutputTemplate) -> CommandOutput:
 
             return CommandOutput(
                 id=output._id.replace(self.id_, run_id),
-                produces=entity,
+                create_folder=output.create_folder,
+                description=output.description,
                 mapped_to=output.mapped_to,
+                name=output.name,
                 position=output.position,
                 prefix=output.prefix,
-                create_folder=output.create_folder,
+                produces=entity,
             )
 
         uuid_ = self._extract_uuid()
@@ -234,7 +238,9 @@ def _convert_command_input(input_: CommandInput, plan_id) -> CommandInputTemplat
     return CommandInputTemplate(
         id=CommandInputTemplate.generate_id(plan_id=plan_id, id_=Path(input_._id).name),
         default_value=consumes,
+        description=input_.description,
         mapped_to=input_.mapped_to,
+        name=input_.name,
         position=input_.position,
         prefix=input_.prefix,
     )
@@ -251,7 +257,9 @@ def _convert_command_output(output: CommandOutput, plan_id) -> CommandOutputTemp
     return CommandOutputTemplate(
         id=CommandOutputTemplate.generate_id(plan_id=plan_id, id_=Path(output._id).name),
         default_value=produces,
+        description=output.description,
         mapped_to=output.mapped_to,
+        name=output.name,
         position=output.position,
         prefix=output.prefix,
         create_folder=output.create_folder,
diff --git a/renku/data/new_graph_shacl_shape.json b/renku/data/new_graph_shacl_shape.json
@@ -862,12 +862,27 @@
          "closed": true,
          "targetClass": "prov:Plan",
          "property": [
+            {
+               "nodeKind": "sh:Literal",
+               "path": "schema:description",
+               "datatype": {
+                  "@id": "xsd:string"
+               },
+               "maxCount": 1
+            },
             {
                "sh:class": {
                   "@id": "renku:CommandArgument"
                },
                "path": "renku:hasArguments"
             },
+            {
+               "nodeKind": "sh:Literal",
+               "path": "schema:keywords",
+               "datatype": {
+                  "@id": "xsd:string"
+               }
+            },
             {
                "nodeKind": "sh:Literal",
                "path": "renku:command",
@@ -916,13 +931,30 @@
          "closed": true,
          "targetClass": "renku:CommandArgument",
          "property": [
+            {
+               "nodeKind": "sh:Literal",
+               "path": "schema:description",
+               "datatype": {
+                  "@id": "xsd:string"
+               },
+               "maxCount": 1
+            },
             {
                "nodeKind": "sh:Literal",
                "path": "rdfs:label",
                "datatype": {
                   "@id": "xsd:string"
                }
             },
+            {
+               "nodeKind": "sh:Literal",
+               "path": "schema:name",
+               "datatype": {
+                  "@id": "xsd:string"
+               },
+               "minCount": 1,
+               "maxCount": 1
+            },
             {
                "nodeKind": "sh:Literal",
                "path": "renku:position",
@@ -1013,13 +1045,29 @@
          "closed": true,
          "targetClass": "renku:CommandInputTemplate",
          "property": [
+            {
+               "nodeKind": "sh:Literal",
+               "path": "schema:description",
+               "datatype": {
+                  "@id": "xsd:string"
+               },
+               "maxCount": 1
+            },
             {
                "nodeKind": "sh:Literal",
                "path": "rdfs:label",
                "datatype": {
                   "@id": "xsd:string"
                }
             },
+            {
+               "nodeKind": "sh:Literal",
+               "path": "schema:name",
+               "datatype": {
+                  "@id": "xsd:string"
+               },
+               "maxCount": 1
+            },
             {
                "nodeKind": "sh:Literal",
                "path": "renku:position",
@@ -1064,13 +1112,29 @@
          "closed": true,
          "targetClass": "renku:CommandOutputTemplate",
          "property": [
+            {
+               "nodeKind": "sh:Literal",
+               "path": "schema:description",
+               "datatype": {
+                  "@id": "xsd:string"
+               },
+               "maxCount": 1
+            },
             {
                "nodeKind": "sh:Literal",
                "path": "rdfs:label",
                "datatype": {
                   "@id": "xsd:string"
                }
             },
+            {
+               "nodeKind": "sh:Literal",
+               "path": "schema:name",
+               "datatype": {
+                  "@id": "xsd:string"
+               },
+               "maxCount": 1
+            },
             {
                "nodeKind": "sh:Literal",
                "path": "renku:position",
diff --git a/renku/data/shacl_shape.json b/renku/data/shacl_shape.json
diff --git a/tests/cli/test_run.py b/tests/cli/test_run.py
