fix(dataset): deserialization error (#1675)

Author: mohammad-alisafaee

conftest.py

@@ -413,6 +413,25 @@ def dataset(client):
     return dataset
 
 
+@pytest.fixture
+def client_with_datasets(client, directory_tree):
+    """A client with datasets."""
+    from renku.core.models.provenance.agents import Person
+
+    person_1 = Person.from_string("P1 <[email protected]> [IANA]")
+    person_2 = Person.from_string("P2 <[email protected]>")
+
+    client.create_dataset(name="dataset-1", keywords=["dataset", "1"], creators=[person_1])
+
+    with client.with_dataset("dataset-2", create=True) as dataset:
+        dataset.keywords = ["dataset", "2"]
+        dataset.creators = [person_1, person_2]
+
+        client.add_data_to_dataset(dataset=dataset, urls=[str(p) for p in directory_tree.glob("*")])
+
+    yield client
+
+
 @pytest.fixture(params=[".", "some/sub/directory"])
 def subdirectory(project, request):
     """Runs tests in root directory and a subdirectory."""

renku/core/management/migrations/models/v3.py

@@ -22,7 +22,18 @@
 from marshmallow import EXCLUDE, post_load, pre_load
 
 from renku.core.models import jsonld
-from renku.core.models.calamus import JsonLDSchema, Uri, fields, prov, rdfs, renku, schema, wfprov
+from renku.core.models.calamus import (
+    DateTimeList,
+    JsonLDSchema,
+    StringList,
+    Uri,
+    fields,
+    prov,
+    rdfs,
+    renku,
+    schema,
+    wfprov,
+)
 from renku.core.models.datasets import generate_dataset_tag_id, generate_url_id
 from renku.core.models.git import get_user_info
 from renku.core.models.projects import generate_project_id
@@ -36,6 +47,8 @@ def __init__(self, **kwargs):
         """Initialize an instance."""
         self.client = None
 
+        kwargs.setdefault("_id", None)
+
         for k, v in kwargs.items():
             setattr(self, k, v)
 
@@ -185,11 +198,11 @@ class Meta:
         unknown = EXCLUDE
 
     _id = fields.Id()
-    name = fields.String(schema.name)
+    name = StringList(schema.name)
     email = fields.String(schema.email, missing=None)
-    label = fields.String(rdfs.label)
-    affiliation = fields.String(schema.affiliation, missing=None)
-    alternate_name = fields.String(schema.alternateName, missing=None)
+    label = StringList(rdfs.label)
+    affiliation = StringList(schema.affiliation, missing=None)
+    alternate_name = StringList(schema.alternateName, missing=None)
 
     @post_load
     def make_instance(self, data, **kwargs):
@@ -213,8 +226,8 @@ class Meta:
     _id = fields.Id(missing=None)
     agent_version = fields.String(schema.agent, missing="pre-0.11.0")
     name = fields.String(schema.name, missing=None)
-    created = fields.DateTime(schema.dateCreated, missing=None)
-    version = fields.String(schema.schemaVersion, missing=1)
+    created = DateTimeList(schema.dateCreated, missing=None)
+    version = StringList(schema.schemaVersion, missing="1")
     creator = fields.Nested(schema.creator, PersonSchemaV3, missing=None)
 
 

renku/core/management/migrations/models/v8.py

@@ -18,16 +18,33 @@
 """Migration models V8."""
 
 import os
+from pathlib import Path
 
 from marshmallow import EXCLUDE, pre_dump
 
 from renku.core.models import jsonld
 from renku.core.models.calamus import Uri, fields, schema
+from renku.core.models.entities import generate_file_id
 
 from .v3 import CreatorMixinSchemaV3, DatasetTagSchemaV3, EntitySchemaV3, LanguageSchemaV3, PersonSchemaV3, UrlSchemaV3
 from .v7 import Base, DatasetFileSchemaV7
 
 
+class DatasetFile(Base):
+    """DatasetFile migration model."""
+
+    def __init__(self, **kwargs):
+        """Initialize an instance."""
+        super().__init__(**kwargs)
+
+        if hasattr(self, "path") and (not self._id or self._id.startswith("_:")):
+            hexsha = "UNCOMMITTED"
+            if self.client and Path(self.path).exists():
+                hexsha = self.client.find_previous_commit().hexsha
+
+            self._id = generate_file_id(client=self.client, hexsha=hexsha, path=self.path)
+
+
 class Dataset(Base):
     """Dataset migration model."""
 
@@ -43,11 +60,25 @@ def to_yaml(self, path=None):
         """Write content to a YAML file."""
         from renku.core.management import LocalClient
 
+        for file_ in self.files:
+            file_._project = self._project
+
         data = DatasetSchemaV8(flattened=True).dump(self)
         path = path or self._metadata_path or os.path.join(self.path, LocalClient.METADATA)
         jsonld.write_yaml(path=path, data=data)
 
 
+class DatasetFileSchemaV8(DatasetFileSchemaV7):
+    """DatasetFile schema."""
+
+    class Meta:
+        """Meta class."""
+
+        rdf_type = schema.DigitalDocument
+        model = DatasetFile
+        unknown = EXCLUDE
+
+
 class DatasetSchemaV8(CreatorMixinSchemaV3, EntitySchemaV3):
     """Dataset schema."""
 
@@ -62,7 +93,7 @@ class Meta:
     date_created = fields.DateTime(schema.dateCreated, missing=None)
     date_published = fields.DateTime(schema.datePublished, missing=None)
     description = fields.String(schema.description, missing=None)
-    files = fields.Nested(schema.hasPart, DatasetFileSchemaV7, many=True)
+    files = fields.Nested(schema.hasPart, DatasetFileSchemaV8, many=True)
     identifier = fields.String(schema.identifier)
     in_language = fields.Nested(schema.inLanguage, LanguageSchemaV3, missing=None)
     keywords = fields.List(schema.keywords, fields.String())

renku/core/models/calamus.py

@@ -116,34 +116,22 @@ def _deserialize(self, value, attr, data, **kwargs):
             raise ValueError("Invalid type for field {}: {}".format(self.name, type(value)))
 
 
-class StringList(fields._JsonLDField, marshmallow.fields.String, marshmallow.fields.List):
+class StringList(fields.String):
     """A String field that might be a list when deserializing."""
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args, return_max_value=True, **kwargs):
         """Create an instance."""
         super().__init__(*args, **kwargs)
-
-    def _serialize(self, value, attr, obj, **kwargs):
-        if isinstance(value, list):
-            value = value[0] if value else None
-        if isinstance(value, str):
-            value = super(fields._JsonLDField, self)._serialize(value, attr, obj, **kwargs)
-            if self.parent.opts.add_value_types:
-                value = {"@value": value, "@type": "http://www.w3.org/2001/XMLSchema#string"}
-
-        return value
+        self.return_max_value = return_max_value
 
     def _deserialize(self, value, attr, data, **kwargs):
         value = normalize_value(value)
-        if not value:
-            return None
-        elif isinstance(value, str):
-            return value
-        elif isinstance(value, list):
-            value = super(marshmallow.fields.String, self)._deserialize(value, attr, data, **kwargs)
-            return value[0] if len(value) > 0 else None
-        else:
-            raise ValueError("Invalid type for field {}: {}".format(self.name, type(value)))
+
+        if isinstance(value, (list, tuple, set)):
+            value = sorted(value, reverse=self.return_max_value)
+            value = value[0] if len(value) > 0 else None
+
+        return super()._deserialize(value, attr, data, **kwargs)
 
 
 class DateTimeList(fields.DateTime):

renku/core/models/projects.py

@@ -26,7 +26,7 @@
 
 from renku.core.management.migrate import SUPPORTED_PROJECT_VERSION
 from renku.core.models import jsonld
-from renku.core.models.calamus import JsonLDSchema, Nested, fields, prov, renku, schema
+from renku.core.models.calamus import DateTimeList, JsonLDSchema, Nested, StringList, fields, prov, renku, schema
 from renku.core.models.datastructures import Collection
 from renku.core.models.provenance.agents import Person, PersonSchema
 from renku.core.utils.datetime8601 import parse_date
@@ -184,8 +184,8 @@ class Meta:
         unknown = EXCLUDE
 
     name = fields.String(schema.name, missing=None)
-    created = fields.DateTime(schema.dateCreated, missing=None, format="iso", extra_formats=("%Y-%m-%d",))
-    version = fields.String(schema.schemaVersion, missing=1)
+    created = DateTimeList(schema.dateCreated, missing=None, format="iso", extra_formats=("%Y-%m-%d",))
+    version = StringList(schema.schemaVersion, missing="1")
     agent_version = fields.String(schema.agent, missing="pre-0.11.0")
     template_source = fields.String(renku.templateSource, missing=None)
     template_ref = fields.String(renku.templateReference, missing=None)

renku/core/models/provenance/agents.py

@@ -141,11 +141,11 @@ class Meta:
         model = Person
         unknown = EXCLUDE
 
-    name = StringList(schema.name, fields.String(), missing=None)
+    name = StringList(schema.name, missing=None)
     email = fields.String(schema.email, missing=None)
-    label = StringList(rdfs.label, fields.String(), missing=None)
-    affiliation = StringList(schema.affiliation, fields.String(), missing=None)
-    alternate_name = StringList(schema.alternateName, fields.String(), missing=None)
+    label = StringList(rdfs.label, missing=None)
+    affiliation = StringList(schema.affiliation, missing=None)
+    alternate_name = StringList(schema.alternateName, missing=None)
     _id = fields.Id(init_name="id")
 
 

tests/core/commands/test_serialization.py

@@ -87,3 +87,17 @@ def test_calamus(client, dataset_metadata_before_calamus):
     file_ = dataset.find_file("data/dataverse/local/result.csv")
     assert file_.external is False
     assert "file://../../../../tmp/result.csv" == file_.url
+
+
+def test_dataset_with_multiple_project_version(client_with_datasets):
+    """Test deserialization of a dataset where contains different project versions."""
+    max_version = "42000"
+
+    # Change project version for a single file
+    with client_with_datasets.with_dataset("dataset-2") as dataset:
+        file_ = dataset.find_file(dataset.data_dir / "file1")
+        file_._project.version = max_version
+
+    dataset = client_with_datasets.load_dataset("dataset-2")
+
+    assert {max_version} == {f._project.version for f in dataset.files}