feat(dataset): update local files (#2049)

* feat(dataset): update local files

Author: mohammad-alisafaee

renku/cli/dataset.py

@@ -217,6 +217,9 @@
 with those datasets. Modifying those datasets locally will prevent them from
 being updated.
 
+The update command also checks for file changes in the project and updates
+datasets' metadata accordingly.
+
 You can limit the scope of updated files by specifying dataset names, using
 ``--include`` and ``--exclude`` to filter based on file names, or using
 ``--creators`` to filter based on creators. For example, the following command

renku/core/commands/dataset.py

@@ -652,13 +652,16 @@ def _update_datasets(client, names, creators, include, exclude, ref, delete, ext
     possible_updates = []
     unique_remotes = set()
     external_files = []
+    local_files = []
 
     for file_ in records:
         if file_.based_on:
             possible_updates.append(file_)
             unique_remotes.add(file_.based_on.source)
         elif file_.external:
             external_files.append(file_)
+        else:
+            local_files.append(file_)
 
     if ref and len(unique_remotes) > 1:
         raise ParameterError(
@@ -672,17 +675,24 @@ def _update_datasets(client, names, creators, include, exclude, ref, delete, ext
         else:
             communication.echo("To update external files run update command with '--external' flag.")
 
-    if not possible_updates:
-        return
+    updated_files = []
+    deleted_files = []
+
+    if possible_updates:
+        updated_files, deleted_files = client.update_dataset_git_files(files=possible_updates, ref=ref, delete=delete)
 
-    updated_files, deleted_files = client.update_dataset_git_files(files=possible_updates, ref=ref, delete=delete)
+    if local_files:
+        updated, deleted = client.update_dataset_local_files(records=local_files, delete=delete)
+        updated_files.extend(updated)
+        deleted_files.extend(deleted)
 
     if deleted_files and not delete:
-        communication.echo(
-            "Some files are deleted from remote. To also delete them locally "
-            "run update command with '--delete' flag."
-        )
-    communication.echo("Updated {} files".format(len(updated_files)))
+        communication.echo("Some files are deleted. To also delete them from datasets' metadata use '--delete' flag.")
+
+    message = f"Updated {len(updated_files)} files"
+    if delete:
+        message += f" and deleted {len(deleted_files)} files"
+    communication.echo(message)
 
 
 def update_datasets():

renku/core/management/datasets.py

@@ -1049,6 +1049,58 @@ def move_files(self, files, to_dataset, commit):
         if to_dataset:
             to_dataset.to_yaml()
 
+    def update_dataset_local_files(self, records, delete=False):
+        """Update files metadata from the git history."""
+        updated_files = []
+        deleted_files = []
+        progress_text = "Checking for local updates"
+
+        try:
+            communication.start_progress(progress_text, len(records))
+            for file_ in records:
+                communication.update_progress(progress_text, 1)
+
+                if file_.based_on or file_.external:
+                    continue
+
+                if not Path(file_.path).exists():
+                    deleted_files.append(file_)
+                    continue
+
+                try:
+                    commit = self.find_previous_commit(file_.path)
+                except KeyError:
+                    deleted_files.append(file_)
+                else:
+                    if self._get_commit_sha_from_label(file_) != commit.hexsha:
+                        updated_files.append(file_)
+        finally:
+            communication.finalize_progress(progress_text)
+
+        if updated_files or (deleted_files and delete):
+            self._update_datasets_metadata(updated_files, deleted_files, delete)
+
+        return updated_files, deleted_files
+
+    def _update_datasets_metadata(self, updated_files, deleted_files, delete):
+        modified_datasets = {}
+
+        for file_ in updated_files:
+            new_file = DatasetFile.from_revision(
+                self, path=file_.path, based_on=file_.based_on, url=file_.url, source=file_.source
+            )
+            file_.dataset.update_files([new_file])
+            modified_datasets[file_.dataset.name] = file_.dataset
+
+        if delete:
+            for file_ in deleted_files:
+                file_.dataset.unlink_file(file_.path)
+                modified_datasets[file_.dataset.name] = file_.dataset
+
+        for dataset in modified_datasets.values():
+            dataset.to_yaml()
+            self.update_datasets_provenance(dataset)
+
     def update_dataset_git_files(self, files, ref, delete=False):
         """Update files and dataset metadata according to their remotes.
 
@@ -1133,25 +1185,7 @@ def update_dataset_git_files(self, files, ref, delete=False):
             skip_hooks=skip_hooks,
         )
 
-        # Update datasets' metadata
-
-        modified_datasets = {}
-
-        for file_ in updated_files:
-            new_file = DatasetFile.from_revision(
-                self, path=file_.path, based_on=file_.based_on, url=file_.url, source=file_.source
-            )
-            file_.dataset.update_files([new_file])
-            modified_datasets[file_.dataset.name] = file_.dataset
-
-        if delete:
-            for file_ in deleted_files:
-                file_.dataset.unlink_file(file_.path)
-                modified_datasets[file_.dataset.name] = file_.dataset
-
-        for dataset in modified_datasets.values():
-            dataset.to_yaml()
-            self.update_datasets_provenance(dataset)
+        self._update_datasets_metadata(updated_files, deleted_files, delete)
 
         return updated_files, deleted_files
 

renku/core/management/migrations/models/v8.py

@@ -40,7 +40,7 @@ def __init__(self, **kwargs):
         if hasattr(self, "path") and (not self._id or self._id.startswith("_:")):
             hexsha = "UNCOMMITTED"
             if self.client and Path(self.path).exists():
-                hexsha = self.client.find_previous_commit().hexsha
+                hexsha = self.client.find_previous_commit(self.path).hexsha
 
             self._id = generate_file_id(client=self.client, hexsha=hexsha, path=self.path)
 

tests/cli/test_datasets.py

@@ -1987,3 +1987,65 @@ def test_authorized_import(mock_kg, client, runner):
     assert 1 == result.exit_code
     assert "Unauthorized access to knowledge graph" not in result.output
     assert "Resource not found in knowledge graph" in result.output
+
+
+def test_update_local_file(runner, client, directory_tree):
+    """Check updating local files."""
+    assert 0 == runner.invoke(cli, ["dataset", "add", "-c", "my-data", str(directory_tree)]).exit_code
+
+    file1 = Path(DATA_DIR) / "my-data" / directory_tree.name / "file1"
+    file1.write_text("some updates")
+    client.repo.git.add("--all")
+    client.repo.index.commit("file1")
+    commit_sha_after_file1 = client.repo.head.object.hexsha
+
+    file2 = Path(DATA_DIR) / "my-data" / directory_tree.name / "dir1" / "file2"
+    file2.write_text("some updates")
+    client.repo.git.add("--all")
+    client.repo.index.commit("file2")
+    commit_sha_after_file2 = client.repo.head.object.hexsha
+
+    old_dataset = client.load_dataset("my-data")
+
+    result = runner.invoke(cli, ["dataset", "update", "my-data"])
+
+    assert 0 == result.exit_code
+    dataset = client.load_dataset("my-data")
+    assert commit_sha_after_file1 in dataset.find_file(file1)._label
+    assert commit_sha_after_file2 in dataset.find_file(file2)._label
+    assert_dataset_is_mutated(old=old_dataset, new=dataset)
+
+
+def test_update_local_deleted_file(runner, client, directory_tree):
+    """Check updating local deleted files."""
+    assert 0 == runner.invoke(cli, ["dataset", "add", "-c", "my-data", str(directory_tree)]).exit_code
+
+    file1 = Path(DATA_DIR) / "my-data" / directory_tree.name / "file1"
+    file1.unlink()
+    client.repo.git.add("--all")
+    client.repo.index.commit("deleted file1")
+    commit_sha_after_file1_delete = client.repo.head.object.hexsha
+
+    result = runner.invoke(cli, ["dataset", "update", "my-data"])
+
+    assert 0 == result.exit_code
+    assert "Some files are deleted." in result.output
+    assert "Updated 0 files" in result.output
+    assert commit_sha_after_file1_delete == client.repo.head.object.hexsha
+    old_dataset = client.load_dataset("my-data")
+    assert old_dataset.find_file(file1)
+
+    # NOTE: Update with `--delete`
+    result = runner.invoke(cli, ["dataset", "update", "--delete", "my-data"])
+
+    assert 0 == result.exit_code
+    assert "Updated 0 files and deleted 1 files" in result.output
+    assert commit_sha_after_file1_delete != client.repo.head.object.hexsha
+    dataset = client.load_dataset("my-data")
+    assert dataset.find_file(file1) is None
+    assert_dataset_is_mutated(old=old_dataset, new=dataset)
+
+    result = runner.invoke(cli, ["dataset", "update", "--delete", "my-data"])
+
+    assert 0 == result.exit_code
+    assert "Updated 0 files and deleted 0 files" in result.output

tests/cli/test_integration_datasets.py

@@ -1102,7 +1102,7 @@ def test_dataset_update_remove_file(client, runner):
 
     result = runner.invoke(cli, ["dataset", "update", "--ref", "v0.5.0"], catch_exceptions=False)
     assert 0 == result.exit_code, result.output + str(result.stderr_bytes)
-    assert "Some files are deleted from remote." in result.output
+    assert "Some files are deleted." in result.output
     assert file_path.exists()
 
     result = runner.invoke(cli, ["dataset", "update", "--ref", "v0.5.0", "--delete"], catch_exceptions=False)