feat(dataset): wildcard support when adding data from git (#1128)

* feat(dataset): wildcards when adding from git

Author: mohammad-alisafaee

Pipfile.lock

@@ -118,6 +118,15 @@
       my-dataset/
         datafile
 
+You can use shell-like wildcards (e.g. *, **, ?) when specifying paths to be
+added. Put wildcard patterns in quotes to prevent your shell from expanding
+them.
+
+.. code-block:: console
+
+    $ renku dataset add my-dataset --source 'path/**/datafile' \
+        git+ssh://host.io/namespace/project.git
+
 You can use ``--destination`` or ``-d`` flag to change the name of the target
 file or directory. The semantics here are similar to the POSIX copy command:
 if the destination does not exist or if it is a file then the source will be

renku/cli/dataset.py

@@ -26,6 +26,7 @@
 import time
 import uuid
 import warnings
+from collections import OrderedDict
 from configparser import NoSectionError
 from contextlib import contextmanager
 from pathlib import Path
@@ -36,6 +37,7 @@
 import patoolib
 import requests
 from git import GitCommandError, GitError, Repo
+from wcmatch import glob
 
 from renku.core import errors
 from renku.core.management.clone import clone
@@ -520,33 +522,37 @@ def _add_from_git(self, dataset, url, sources, destination, ref):
 
         # Get all files from repo that match sources
         repo, repo_path = self.prepare_git_repo(url, ref)
-        copied_sources = set()
         files = set()
+        used_sources = set()
         for file in repo.head.commit.tree.traverse():
             path = file.path
             result = self._get_src_and_dst(
-                path, repo_path, sources, destination
+                path, repo_path, sources, destination, used_sources
             )
 
             if result:
                 files.add(result)
-                source = result[3]
-                copied_sources.add(source)
 
-        uncopied_sources = sources - copied_sources
-        if uncopied_sources:
-            uncopied_sources = {str(s) for s in uncopied_sources}
+        unused_sources = set(sources.keys()) - used_sources
+        if unused_sources:
+            unused_sources = {str(s) for s in unused_sources}
             raise errors.ParameterError(
-                'No such file or directory', param_hint=uncopied_sources
+                'No such file or directory', param_hint=unused_sources
             )
 
+        if destination.exists() and not destination.is_dir():
+            if len(files) > 1:
+                raise errors.ParameterError(
+                    'Cannot copy multiple files or directories to a file'
+                )
+
         # Create metadata and move files to dataset
         results = []
         remote_client = LocalClient(repo_path)
 
         # Pull files from LFS
         paths = set()
-        for path, src, _, __ in files:
+        for path, src, _ in files:
             if src.is_dir():
                 continue
             if src.is_symlink():
@@ -561,7 +567,7 @@ def _add_from_git(self, dataset, url, sources, destination, ref):
         paths = {f[0] for f in files}
         metadata = self._fetch_files_metadata(remote_client, paths)
 
-        for path, src, dst, _ in files:
+        for path, src, dst in files:
             if not src.is_dir():
                 # Use original metadata if it exists
                 based_on = metadata.get(path)
@@ -605,7 +611,11 @@ def _check_overwrite(self, files, force):
 
     def _resolve_paths(self, root_path, paths):
         """Check if paths are within a root path and resolve them."""
-        return {self._resolve_path(root_path, p) for p in paths}
+        result = OrderedDict()  # Used as an ordered-set
+        for path in paths:
+            r = self._resolve_path(root_path, path)
+            result[r] = None
+        return result
 
     def _resolve_path(self, root_path, path):
         """Check if a path is within a root path and resolve it."""
@@ -618,18 +628,27 @@ def _resolve_path(self, root_path, path):
                 'File {} is not within path {}'.format(path, root_path)
             )
 
-    def _get_src_and_dst(self, path, repo_path, sources, dst_root):
+    def _get_src_and_dst(
+        self, path, repo_path, sources, dst_root, used_sources
+    ):
+        is_wildcard = False
+
         if not sources:
             source = Path('.')
         else:
             source = None
-            for s in sources:
+            for s in sources.keys():
                 try:
                     Path(path).relative_to(s)
                 except ValueError:
-                    pass
+                    if glob.globmatch(path, str(s), flags=glob.GLOBSTAR):
+                        is_wildcard = True
+                        source = path
+                        used_sources.add(s)
+                        break
                 else:
                     source = s
+                    used_sources.add(source)
                     break
 
             if not source:
@@ -639,24 +658,26 @@ def _get_src_and_dst(self, path, repo_path, sources, dst_root):
         source_name = Path(source).name
         relative_path = Path(path).relative_to(source)
 
-        if not dst_root.exists():
-            if len(sources) == 1:
+        if src.is_dir() and is_wildcard:
+            sources[source] = None
+            used_sources.add(source)
+
+        if not dst_root.exists():  # Destination will be a file or directory
+            if len(sources) == 1 and not is_wildcard:
                 dst = dst_root / relative_path
             else:  # Treat destination as a directory
                 dst = dst_root / source_name / relative_path
         elif dst_root.is_dir():
             dst = dst_root / source_name / relative_path
         else:  # Destination is an existing file
-            if len(sources) == 1 and not src.is_dir():
-                dst = dst_root
-            elif not sources:
-                raise errors.ParameterError('Cannot copy repo to file')
-            else:
+            if src.is_dir():
                 raise errors.ParameterError(
                     'Cannot copy multiple files or directories to a file'
                 )
+            # Later we need to check if we are copying multiple files
+            dst = dst_root
 
-        return (path, src, dst, source)
+        return (path, src, dst)
 
     def _fetch_lfs_files(self, repo_path, paths):
         """Fetch and checkout paths that are tracked by Git LFS."""

renku/core/management/datasets.py

@@ -182,6 +182,7 @@ def run(self):
     'tabulate>=0.7.7',
     'tqdm>=4.31.1',
     'walrus>=0.8.0',
+    'wcmatch>=6.0.0',
     'werkzeug>=0.15.5',
 ]
 

setup.py

@@ -784,6 +784,44 @@ def test_add_data_from_git(runner, client, params, path):
     assert Path(path).exists()
 
 
+@pytest.mark.integration
+@pytest.mark.parametrize(
+    'params,files', [
+        (['-s', 'docker*'], {'docker'}),
+        (['-s', 'docker/*'
+          ], {'py3.7', 'cuda10.0-tf1.14', 'cuda9.2', 'r3.6.1', 'bioc3_10'}),
+        (['-s', 'docker/**'], {
+            'powerline.config', 'Dockerfile', 'entrypoint.sh',
+            'requirements.txt', 'fix-permissions.sh',
+            'LICENSE-fix-permissions', 'powerline.bashrc', 'git-config.bashrc'
+        }),
+        (['-s', 'docker/*/*sh'], {'entrypoint.sh', 'fix-permissions.sh'}),
+    ]
+)
+@flaky(max_runs=10, min_passes=1)
+def test_add_data_from_git_with_wildcards(runner, client, params, files):
+    """Test add data using wildcards to datasets from a git repository."""
+    REMOTE = 'https://github.com/SwissDataScienceCenter/renku-jupyter.git'
+
+    result = runner.invoke(
+        cli,
+        ['dataset', 'add', 'remote', '--create', '--ref', '0.5.2', REMOTE] +
+        params,
+        catch_exceptions=False,
+    )
+    assert 0 == result.exit_code
+    assert files == set(os.listdir('data/remote'))
+
+    result = runner.invoke(
+        cli,
+        ['dataset', 'add', 'remote', '--ref', '0.5.2', '-d', 'new', REMOTE] +
+        params,
+        catch_exceptions=False,
+    )
+    assert 0 == result.exit_code
+    assert files == set(os.listdir('data/remote/new'))
+
+
 @pytest.mark.integration
 @flaky(max_runs=10, min_passes=1)
 def test_add_from_git_copies_metadata(runner, client):
@@ -814,11 +852,13 @@ def test_add_from_git_copies_metadata(runner, client):
         (['-s', 'file'], 2, 'Cannot add multiple URLs'),
         (['-d', 'file'], 2, 'Cannot add multiple URLs'),
         (['-s', 'non-existing'], 1, 'No such file or directory'),
+        (['-s', 'docker/*Dockerfile'], 1, 'No such file or directory'),
         (['-s', 'docker', '-d', 'LICENSE'
           ], 1, 'Cannot copy multiple files or directories to a file'),
         (['-s', 'LICENSE', '-s', 'Makefile', '-d', 'LICENSE'
           ], 1, 'Cannot copy multiple files or directories to a file'),
-        (['-d', 'LICENSE'], 1, 'Cannot copy repo to file'),
+        (['-d', 'LICENSE'
+          ], 1, 'Cannot copy multiple files or directories to a file'),
     ]
 )
 @flaky(max_runs=10, min_passes=1)