diff --git a/docs/reference/process.mdx b/docs/reference/process.mdx index bf9ed3a95a..edcfffb6d6 100644 --- a/docs/reference/process.mdx +++ b/docs/reference/process.mdx @@ -129,7 +129,7 @@ When `true`, the file name is interpreted as a glob pattern (default: `true`). ###### `hidden: Boolean` -When `true`, hidden files are included in the matching output files (default: `false`). +When `true`, hidden files and directories are included in the matching output files (default: `false`). By default the leading `.` of a hidden name must be matched explicitly by the corresponding component of the pattern, and hidden directories are not traversed. ###### `includeInputs: Boolean` @@ -259,7 +259,7 @@ When `true`, the specified name is interpreted as a glob pattern (default: `true ###### `hidden` -When `true`, hidden files are included in the matching output files (default: `false`). +When `true`, hidden files and directories are included in the matching output files (default: `false`). By default the leading `.` of a hidden name must be matched explicitly by the corresponding component of the pattern, and hidden directories are not traversed. ###### `includeInputs` diff --git a/docs/reference/stdlib-namespaces/channel.mdx b/docs/reference/stdlib-namespaces/channel.mdx index 0f580d80ee..92d20a223a 100644 --- a/docs/reference/stdlib-namespaces/channel.mdx +++ b/docs/reference/stdlib-namespaces/channel.mdx @@ -57,7 +57,7 @@ When `true`, follows symbolic links when traversing a directory tree, otherwise ###### `hidden` -When `true`, matches hidden files when using a glob pattern (default: `false`). +When `true`, matches hidden files and directories when using a glob pattern, without requiring the pattern to match the leading `.` explicitly (default: `false`). ###### `maxDepth` @@ -144,7 +144,7 @@ channel.fromPath('data/**/*.fa') channel.fromPath('data/file_{1,2}.fq') ``` -By default, glob patterns do not match hidden files (i.e. files with names that start with `.`). Use a glob pattern that explicitly starts with `.` or set `hidden: true` to match hidden files: +By default, glob patterns do not match hidden files and directories (i.e. names that start with `.`). As in Bash, the leading `.` must be matched explicitly by the corresponding component of the pattern, and hidden directories are not traversed unless a component of the pattern matches them. Alternatively, set `hidden: true` to match every hidden file and directory: ```nextflow // match hidden files in `data` @@ -153,8 +153,13 @@ channel.fromPath('data/*', hidden: true) // match hidden files in `data` with `fa` extension channel.fromPath('data/.*.fa') + +// match `fa` files in any hidden `.cache` directory under `data` +channel.fromPath('data/**/.cache/*.fa') ``` +Since the `.` is matched for each component separately, `data/*/*.fa` does not match `data/.cache/file.fa`, because the pattern says nothing about a hidden `.cache` directory. + By default, glob patterns only match regular files, not directories. Use the `type` option to control whether to match files, directories, or both: ```nextflow @@ -181,7 +186,7 @@ When `true`, interprets the characters `*`, `?`, `[]`, and `{}` as glob wildcard ###### `hidden` -When `true`, matches hidden files when using a glob pattern (default: `false`). +When `true`, matches hidden files and directories when using a glob pattern, without requiring the pattern to match the leading `.` explicitly (default: `false`). ###### `maxDepth` diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskFileCollector.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskFileCollector.groovy index 2407465038..452bd530bb 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskFileCollector.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskFileCollector.groovy @@ -117,7 +117,11 @@ class TaskFileCollector { protected Map visitOptions(String pattern) { return [ relative: false, - hidden: opts.hidden ?: pattern.startsWith('.'), + // note: only the explicit `hidden` option is forwarded. The dot rule for the pattern + // itself is applied by `FileHelper.visitFiles` per path component, therefore deriving + // it here from `pattern.startsWith('.')` would turn a single dotted component into a + // tree-wide authorisation e.g. `.config/**/*.txt` would also collect `.config/a/.secret/x.txt` + hidden: opts.hidden as boolean, followLinks: opts.followLinks, maxDepth: opts.maxDepth, type: opts.type ? opts.type : ( pattern.contains('**') ? 'file' : 'any' ) diff --git a/modules/nextflow/src/test/groovy/nextflow/processor/TaskFileCollectorTest.groovy b/modules/nextflow/src/test/groovy/nextflow/processor/TaskFileCollectorTest.groovy index 5639fb5647..bd96c037e7 100644 --- a/modules/nextflow/src/test/groovy/nextflow/processor/TaskFileCollectorTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/processor/TaskFileCollectorTest.groovy @@ -140,6 +140,65 @@ class TaskFileCollectorTest extends Specification { } + def 'should not collect the files left in the work dir by a hidden directory'() { + + given: + // mimic a task work dir where Fusion has left its own metadata files under + // `/.fusion/v1//md.json` -- see issue #7480 + def folder = Files.createTempDirectory('test') + folder.resolve('sample/outs').mkdirs() + folder.resolve('sample/outs/real.txt').text = 'real output' + and: + folder.resolve('.fusion/v1/s3/bucket/work/ab/cdef/sample/outs').mkdirs() + folder.resolve('.fusion/v1/s3/bucket/work/ab/cdef/sample/outs/md.json').text = '{"internal":"metadata"}' + and: + def result + + when: 'the pattern does not name the hidden directory' + result = fetchResultFiles('**/outs/**', folder) + then: + result == ['sample/outs/real.txt'] + + when: 'hidden files are explicitly requested' + result = fetchResultFiles('**/outs/**', folder, hidden: true) + then: + result == ['.fusion/v1/s3/bucket/work/ab/cdef/sample/outs/md.json', 'sample/outs/real.txt'] + + cleanup: + folder?.deleteDir() + + } + + def 'should apply the dot rule per component for a pattern starting with a dot'() { + + given: + // a dotted component authorises a hidden name where it appears and nowhere else, + // therefore `.config` is visited while `.secret` and `.dotleaf.txt` are not. + // bash 5.3 and zsh 5.9 return `.config/a/plain/y.txt` alone for this tree + def folder = Files.createTempDirectory('test') + folder.resolve('.config/a/.secret').mkdirs() + folder.resolve('.config/a/plain').mkdirs() + folder.resolve('.config/a/.secret/x.txt').text = 'x' + folder.resolve('.config/a/plain/y.txt').text = 'y' + folder.resolve('.config/a/.dotleaf.txt').text = 'z' + and: + def result + + when: + result = fetchResultFiles('.config/**/*.txt', folder) + then: + result == ['.config/a/plain/y.txt'] + + when: 'hidden files are explicitly requested' + result = fetchResultFiles('.config/**/*.txt', folder, hidden: true) + then: + result == ['.config/a/.dotleaf.txt', '.config/a/.secret/x.txt', '.config/a/plain/y.txt'] + + cleanup: + folder?.deleteDir() + + } + def defaultCollector(Map opts) { return new TaskFileCollector([], opts, Mock(TaskRun)) } @@ -154,7 +213,9 @@ class TaskFileCollectorTest extends Specification { then: collector.visitOptions('file.txt') == [type:'any', followLinks: true, maxDepth: null, hidden: false, relative: false] collector.visitOptions('path/**') == [type:'file', followLinks: true, maxDepth: null, hidden: false, relative: false] - collector.visitOptions('.hidden_file') == [type:'any', followLinks: true, maxDepth: null, hidden: true, relative: false] + // note: a dotted pattern no longer forces `hidden` here. The dot rule is applied by + // `FileHelper.visitFiles` per path component, see issue #7480 + collector.visitOptions('.hidden_file') == [type:'any', followLinks: true, maxDepth: null, hidden: false, relative: false] when: collector = defaultCollector([type: 'dir']) diff --git a/modules/nf-commons/src/main/nextflow/file/FileHelper.groovy b/modules/nf-commons/src/main/nextflow/file/FileHelper.groovy index 2ded21331e..477f0245ee 100644 --- a/modules/nf-commons/src/main/nextflow/file/FileHelper.groovy +++ b/modules/nf-commons/src/main/nextflow/file/FileHelper.groovy @@ -854,12 +854,37 @@ class FileHelper { final type = options.type ?: 'any' final walkOptions = options.followLinks == false ? EnumSet.noneOf(FileVisitOption.class) : EnumSet.of(FileVisitOption.FOLLOW_LINKS) final int maxDepth = getMaxDepth(options.maxDepth, filePattern) - final includeHidden = options.hidden as Boolean ?: filePattern.startsWith('.') + final syntax = options.syntax ?: 'glob' + // NOTE: the dot rule below can only be applied to a glob pattern. When the `regex` syntax + // is used the pattern is a regular expression, hence a leading dot is the any-char + // meta-character and the slash is not a component separator, therefore splitting it into + // path components would be meaningless. For any non-glob syntax the legacy behaviour is + // preserved i.e. only a pattern *starting* with a dot enables hidden files, and hidden + // directories are not pruned from the tree walk. + final isGlob = syntax == 'glob' + final includeHidden = options.hidden as Boolean ?: (isGlob ? false : filePattern.startsWith('.')) + final dotMatchers = isGlob && !includeHidden + ? hiddenComponents(filePattern).collect { getPathMatcherFor("glob:$it", folder.fileSystem) } + : Collections.emptyList() final includeDir = type in ['dir','any'] final includeFile = type in ['file','any'] - final syntax = options.syntax ?: 'glob' final relative = options.relative == true + // Implements the POSIX rule (Issue 8, section 2.14.3) also honoured by bash and zsh: a + // leading dot in a file name must be matched explicitly by a dot in the corresponding + // component of the pattern. Therefore a dot-prefixed component does *not* enable hidden + // paths for the whole pattern, it only authorises a hidden name where it appears, e.g. + // `.config/**/*.txt` visits `.config` but not `.config/a/.secret`. + // + // NOTE: the authorisation is checked against the file name alone rather than against the + // position of the component within the pattern. Since `**` matches an arbitrary number of + // components, a pattern component cannot be reliably aligned with a path component, so the + // dot-prefixed components of the pattern are collected up-front and a hidden name is + // allowed when it matches any of them as a single-component glob. + final Closure allowed = { Path it -> + includeHidden || !isHidden(it) || matchesAnyName(dotMatchers, it) + } + final matcher = getPathMatcherFor("$syntax:${filePattern}", folder.fileSystem) final singleParam = action.getMaximumNumberOfParameters() == 1 @@ -871,7 +896,13 @@ class FileHelper { final path = relativize0(folder, fullPath) log.trace "visitFiles > dir=$path; depth=$depth; includeDir=$includeDir; matches=${matcher.matches(path)}; isDir=${attrs.isDirectory()}" - if (depth>0 && includeDir && matcher.matches(path) && attrs.isDirectory() && (includeHidden || !isHidden(fullPath))) { + // do not descend into a hidden directory unless the pattern authorises it + // note: the `depth>0` check is needed because the start folder itself can be + // hidden or nested under a hidden directory e.g. a work dir /scratch/.tmp/ab/cdef + if( depth>0 && isGlob && !allowed(fullPath) ) + return FileVisitResult.SKIP_SUBTREE + + if (depth>0 && includeDir && matcher.matches(path) && attrs.isDirectory() && allowed(fullPath)) { def result = relative ? path : fullPath singleParam ? action.call(result) : action.call(result,attrs) } @@ -886,7 +917,7 @@ class FileHelper { : fullPath log.trace "visitFiles > file=$path; includeFile=$includeFile; matches=${matcher.matches(path)}; isRegularFile=${attrs.isRegularFile()}" - if (includeFile && matcher.matches(path) && (attrs.isRegularFile() || (options.followLinks == false && attrs.isSymbolicLink())) && (includeHidden || !isHidden(fullPath))) { + if (includeFile && matcher.matches(path) && (attrs.isRegularFile() || (options.followLinks == false && attrs.isSymbolicLink())) && allowed(fullPath)) { def result = relative ? path : fullPath singleParam ? action.call(result) : action.call(result,attrs) } @@ -926,6 +957,66 @@ class FileHelper { len>0 ? Paths.get(str.substring(0,str.length()-1)) : Paths.get('') } + /** + * Collect the dot-prefixed components of the given glob pattern, e.g. {@code .config} for + * the pattern {@code .config/**}. Each of them authorises a hidden file name in the visit, + * as prescribed by the POSIX dot rule. + * + * Note: the {@code .} and {@code ..} components are *not* considered hidden names. + * + * @param pattern A glob pattern e.g. {@code foo/.bar/*} + * @return The list of dot-prefixed components, each one being itself a glob pattern + */ + @PackageScope + static List hiddenComponents(String pattern) { + final result = new ArrayList() + if( !pattern ) + return result + for( String it : pattern.tokenize('/') ) { + // strip the escape char, since a dot can be escaped to match it literally + final name = it.startsWith('\\') ? it.substring(1) : it + if( name=='.' || name=='..' ) + continue + if( name.startsWith('.') ) + result.add(it) + } + return result + } + + /** + * Check if the *name* of the given path matches any of the specified single-component matchers + * + * Note: the matchers can be the fallback implementation returned by + * {@link #getDefaultPathMatcher(java.lang.String)}, which is used when the file system does + * not support {@link java.nio.file.FileSystem#getPathMatcher(java.lang.String)} e.g. S3. + * That implementation matches against the path {@code toString()}, therefore the name is + * matched as a *relative* single-component path, so that no file system specific prefix + * (e.g. the S3 bucket) can be prepended to it. + * + * @param matchers A list of matchers built from a single pattern component + * @param path The path whose file name should be checked + * @return {@code true} when the path name matches at least one of the given matchers + */ + @PackageScope + static boolean matchesAnyName(List matchers, Path path) { + if( !matchers ) + return false + // note: fileName can be null for the root path + final fileName = path.getFileName() + if( fileName==null ) + return false + // some file systems report a directory name with a trailing slash, in that case the name + // has to be rebuilt because the matcher would not match the ending separator + final str = fileName.toString() + final name = str.endsWith('/') + ? path.getFileSystem().getPath(str.substring(0, str.length()-1)) + : fileName + for( PathMatcher it : matchers ) + if( it.matches(name) ) + return true + return false + } + private static boolean isHidden(Path path) { // note: fileName can be null for root path def fileName = path.getFileName() diff --git a/modules/nf-commons/src/test/nextflow/file/FileHelperTest.groovy b/modules/nf-commons/src/test/nextflow/file/FileHelperTest.groovy index c95513f1b8..1abb48df89 100644 --- a/modules/nf-commons/src/test/nextflow/file/FileHelperTest.groovy +++ b/modules/nf-commons/src/test/nextflow/file/FileHelperTest.groovy @@ -19,13 +19,18 @@ package nextflow.file import static java.nio.file.LinkOption.* import java.nio.file.FileAlreadyExistsException +import java.nio.file.FileStore import java.nio.file.FileSystem +import java.nio.file.FileSystems import java.nio.file.Files import java.nio.file.LinkOption import java.nio.file.NoSuchFileException import java.nio.file.Path +import java.nio.file.PathMatcher import java.nio.file.Paths import java.nio.file.StandardCopyOption +import java.nio.file.WatchService +import java.nio.file.attribute.UserPrincipalLookupService import java.nio.file.spi.FileSystemProvider import com.google.common.jimfs.Configuration @@ -599,6 +604,251 @@ class FileHelperTest extends Specification { folder.deleteDir() } + def 'should not visit files inside a hidden directory' () { + given: + def folder = Files.createTempDirectory('test') + and: + folder.resolve('sub/outs').mkdirs() + folder.resolve('sub/outs/keep.txt').text = 'keep me' + and: + // a file whose own name is *not* dot-prefixed, but that lives inside a hidden directory + // e.g. the metadata files created by Fusion under `/.fusion/v1/...` + folder.resolve('.fusion/v1/sub/outs').mkdirs() + folder.resolve('.fusion/v1/sub/outs/md.json').text = 'fusion internal' + + when: 'hidden files are excluded by default' + def result = [] + FileHelper.visitFiles(folder, '**/outs/**', type: 'file', relative: true) { result << it.toString() } + then: + result.sort() == ['sub/outs/keep.txt'] + + when: 'hidden files are explicitly requested' + result = [] + FileHelper.visitFiles(folder, '**/outs/**', type: 'file', relative: true, hidden: true) { result << it.toString() } + then: + result.sort() == ['.fusion/v1/sub/outs/md.json', 'sub/outs/keep.txt'] + + cleanup: + folder?.deleteDir() + } + + def 'should visit files inside a hidden directory named by the pattern' () { + given: + def folder = Files.createTempDirectory('test') + and: + folder.resolve('.fusion/v1/.nested').mkdirs() + folder.resolve('.fusion/v1/md.json').text = 'fusion internal' + // these are hidden in a position the pattern does not authorise + folder.resolve('.fusion/v1/.nested/deep.json').text = 'nested hidden dir' + folder.resolve('.fusion/.dotfile').text = 'nested hidden file' + and: + folder.resolve('sub/.hidden').mkdirs() + folder.resolve('sub/.hidden/data.txt').text = 'data' + + when: 'the hidden component is the first one' + def result = [] + FileHelper.visitFiles(folder, '.fusion/**', type: 'file', relative: true) { result << it.toString() } + then: + // the leading `.fusion` authorises that name only, exactly as `shopt -u dotglob` in bash + result.sort() == ['.fusion/v1/md.json'] + + when: 'the hidden component is *not* the first one' + result = [] + FileHelper.visitFiles(folder, 'sub/.hidden/*', type: 'file', relative: true) { result << it.toString() } + then: + result.sort() == ['sub/.hidden/data.txt'] + + cleanup: + folder?.deleteDir() + } + + def 'should not visit a hidden file in a visible directory' () { + given: + def folder = Files.createTempDirectory('test') + and: + folder.resolve('sub').mkdirs() + folder.resolve('sub/file.txt').text = 'visible' + folder.resolve('sub/.secret.txt').text = 'hidden' + + when: + def result = [] + FileHelper.visitFiles(folder, '**', type: 'file', relative: true) { result << it.toString() } + then: + result.sort() == ['sub/file.txt'] + + when: + result = [] + FileHelper.visitFiles(folder, '**', type: 'file', relative: true, hidden: true) { result << it.toString() } + then: + result.sort() == ['sub/.secret.txt', 'sub/file.txt'] + + cleanup: + folder?.deleteDir() + } + + def 'should visit files when the start folder is itself hidden' () { + given: + // mimic a task work directory nested under a dot-directory e.g. /scratch/.tmp/ab/cdef + def root = Files.createTempDirectory('test') + def folder = root.resolve('.tmp/ab/cdef') + folder.mkdirs() + and: + folder.resolve('outs').mkdirs() + folder.resolve('outs/keep.txt').text = 'keep me' + folder.resolve('.fusion/v1/outs').mkdirs() + folder.resolve('.fusion/v1/outs/md.json').text = 'fusion internal' + + when: + def result = [] + FileHelper.visitFiles(folder, 'outs/**', type: 'file', relative: true) { result << it.toString() } + then: + result.sort() == ['outs/keep.txt'] + + cleanup: + root?.deleteDir() + } + + def 'should keep the legacy behaviour when using regex syntax' () { + given: + def folder = Files.createTempDirectory('test') + and: + folder.resolve('rx/.hid').mkdirs() + folder.resolve('rx/.hid/keep.json').text = 'keep me' + + when: 'a regex pattern is used, hidden directories are not pruned' + def result = [] + FileHelper.visitFiles(folder, /rx\/\.hid\/keep\.json/, syntax: 'regex', type: 'file', relative: true) { result << it.toString() } + then: + result == ['rx/.hid/keep.json'] + + cleanup: + folder?.deleteDir() + } + + @Unroll + def 'should collect hidden components for #PATTERN' () { + expect: + FileHelper.hiddenComponents(PATTERN) == EXPECTED + + where: + PATTERN | EXPECTED + null | [] + '' | [] + '*' | [] + '**' | [] + 'foo' | [] + 'foo/**' | [] + '.' | [] + '..' | [] + '../foo' | [] + './foo' | [] + 'foo/./bar' | [] + 'foo/../bar' | [] + '.git' | ['.git'] + '.git/config' | ['.git'] + './.git/config' | ['.git'] + 'a/.b/c' | ['.b'] + '*/.git/config' | ['.git'] + 'outs/.fusion/**' | ['.fusion'] + '/abs/.hidden/x' | ['.hidden'] + '**/.*/outs/*' | ['.*'] + '.a/x/.b/y' | ['.a','.b'] + // a dot can be escaped to be matched literally + '\\.git/config' | ['\\.git'] + 'a/\\.b/c' | ['\\.b'] + '\\.' | [] + } + + def 'should apply the dot rule to each pattern component separately' () { + given: + // expected values verified against bash 5.3 and zsh 5.9: + // $ shopt -s globstar; echo .config/**/*.txt + // .config/a/plain/y.txt + def folder = Files.createTempDirectory('test') + and: + folder.resolve('.config/a/.secret').mkdirs() + folder.resolve('.config/a/plain').mkdirs() + folder.resolve('.config/a/.secret/x.txt').text = 'nested hidden dir' + folder.resolve('.config/a/plain/y.txt').text = 'visible' + folder.resolve('.config/a/.dotleaf.txt').text = 'nested hidden file' + + when: 'the leading dot component only authorises a hidden name in that position' + def result = [] + FileHelper.visitFiles(folder, '.config/**/*.txt', type: 'file', relative: true) { result << it.toString() } + then: + result.sort() == ['.config/a/plain/y.txt'] + + when: 'hidden files are explicitly requested' + result = [] + FileHelper.visitFiles(folder, '.config/**/*.txt', type: 'file', relative: true, hidden: true) { result << it.toString() } + then: + result.sort() == ['.config/a/.dotleaf.txt', '.config/a/.secret/x.txt', '.config/a/plain/y.txt'] + + cleanup: + folder?.deleteDir() + } + + @Unroll + def 'should apply the dot rule as bash does for #PATTERN' () { + given: + // expected values verified against bash 5.3, zsh 5.9 and CPython glob + def folder = Files.createTempDirectory('test') + folder.resolve('.foo').mkdirs() + folder.resolve('pub').mkdirs() + folder.resolve('.foo/bar.txt').text = 'hidden dir' + folder.resolve('.secret.txt').text = 'hidden file' + folder.resolve('pub/bar.txt').text = 'visible dir' + folder.resolve('visible.txt').text = 'visible file' + + when: + def result = [] + FileHelper.visitFiles(folder, PATTERN, type: 'file', relative: true) { result << it.toString() } + then: + result.sort() == EXPECTED + + cleanup: + folder?.deleteDir() + + where: + PATTERN | EXPECTED + '*.txt' | ['visible.txt'] + '*/bar.txt' | ['pub/bar.txt'] + '.*/bar.txt' | ['.foo/bar.txt'] + } + + @Unroll + def 'should authorise a hidden directory named by the pattern for #PATTERN' () { + given: + // expected values verified against bash 5.3 and zsh 5.9 + // note: the hidden dirs are nested below `sub` because the Java glob `**` does not + // match zero directories, unlike the bash and zsh one + def folder = Files.createTempDirectory('test') + folder.resolve('sub/.hiddenmid/outs').mkdirs() + folder.resolve('sub/.other/outs').mkdirs() + folder.resolve('sub/vis/outs').mkdirs() + folder.resolve('sub/.hiddenmid/outs/b.txt').text = 'b' + folder.resolve('sub/.other/outs/d.txt').text = 'd' + folder.resolve('sub/vis/outs/c.txt').text = 'c' + + when: + def result = [] + FileHelper.visitFiles(folder, PATTERN, type: 'file', relative: true) { result << it.toString() } + then: + result.sort() == EXPECTED + + cleanup: + folder?.deleteDir() + + where: + PATTERN | EXPECTED + // a literal dotted component authorises only that name + '**/.hiddenmid/outs/*' | ['sub/.hiddenmid/outs/b.txt'] + // a dotted glob component authorises any hidden name + '**/.*/outs/*' | ['sub/.hiddenmid/outs/b.txt', 'sub/.other/outs/d.txt'] + // no dotted component, all hidden dirs are pruned -- this is the issue #7480 case + '**/outs/**' | ['sub/vis/outs/c.txt'] + } + def 'get max depth'() { expect: FileHelper.getMaxDepth(1,null) == 1 @@ -1154,4 +1404,61 @@ class FileHelperTest extends Specification { 's3:///example.com////another///path' | 's3:///example.com/another/path' 'ftp://example.com//file//path' | 'ftp://example.com/file/path' } + + /** + * A file system that does not support `getPathMatcher`, like the S3 one, so that the + * {@link FileHelper#getDefaultPathMatcher} fallback is used instead + */ + static class NoMatcherFileSystem extends FileSystem { + private FileSystem target = FileSystems.getDefault() + + @Override + PathMatcher getPathMatcher(String syntaxAndPattern) { throw new UnsupportedOperationException() } + + @Override FileSystemProvider provider() { target.provider() } + @Override void close() throws IOException { } + @Override boolean isOpen() { target.isOpen() } + @Override boolean isReadOnly() { target.isReadOnly() } + @Override String getSeparator() { target.getSeparator() } + @Override Iterable getRootDirectories() { target.getRootDirectories() } + @Override Iterable getFileStores() { target.getFileStores() } + @Override Set supportedFileAttributeViews() { target.supportedFileAttributeViews() } + @Override Path getPath(String first, String... more) { target.getPath(first, more) } + @Override UserPrincipalLookupService getUserPrincipalLookupService() { throw new UnsupportedOperationException() } + @Override WatchService newWatchService() throws IOException { throw new UnsupportedOperationException() } + } + + @Unroll + def 'should match the hidden component #COMPONENT with the fallback matcher'() { + given: + // a file system that does not implement `getPathMatcher` e.g. the S3 one, hence the + // matcher is the `getDefaultPathMatcher` fallback which matches the path `toString()` + def fs = new NoMatcherFileSystem() + def matchers = [ FileHelper.getPathMatcherFor("glob:$COMPONENT", fs) ] + + expect: + FileHelper.matchesAnyName(matchers, Paths.get(PATH)) == EXPECTED + + where: + COMPONENT | PATH | EXPECTED + '.fusion' | '/work/ab/cdef/.fusion' | true + '.fusion' | '.fusion' | true + '.fusion' | '/work/ab/.fusion-other' | false + '.fusion' | '/work/ab/cdef' | false + '.*' | '/work/ab/cdef/.fusion' | true + '.*' | '/work/ab/cdef/.git' | true + '\\.bar' | '/work/ab/.bar' | true + '.b*' | '/work/ab/.bar' | true + '.b*' | '/work/ab/.zzz' | false + } + + def 'should not match a name when there is no hidden component'() { + expect: + !FileHelper.matchesAnyName([], Paths.get('/work/.fusion')) + and: + !FileHelper.matchesAnyName(null, Paths.get('/work/.fusion')) + and: + // the root path has no file name + !FileHelper.matchesAnyName([FileHelper.getDefaultPathMatcher('glob:.foo')], Paths.get('/')) + } } diff --git a/plugins/nf-amazon/src/test/nextflow/file/FileHelperS3Test.groovy b/plugins/nf-amazon/src/test/nextflow/file/FileHelperS3Test.groovy index e8843cf905..d748d3070e 100644 --- a/plugins/nf-amazon/src/test/nextflow/file/FileHelperS3Test.groovy +++ b/plugins/nf-amazon/src/test/nextflow/file/FileHelperS3Test.groovy @@ -82,4 +82,57 @@ class FileHelperS3Test extends Specification { 's3://foo//this/that' | new URI('s3:///foo/this/that') 's3://foo//this///that' | new URI('s3:///foo/this/that') } + + def 'should not implement a path matcher' () { + given: + Global.session = Mock(Session) { getConfig() >> [:] } + def path = FileHelper.asPath('s3://my-bucket/work/ab/cdef/.fusion') + + when: + path.getFileSystem().getPathMatcher('glob:*') + then: + // this is the reason why the `getDefaultPathMatcher` fallback is needed + thrown(UnsupportedOperationException) + } + + def 'should return a bucket-less file name' () { + given: + Global.session = Mock(Session) { getConfig() >> [:] } + def path = FileHelper.asPath('s3://my-bucket/work/ab/cdef/.fusion') + + expect: + // the name must not be turned into a bucket, otherwise it could not be matched + path.getFileName().toString() == '.fusion' + and: + !path.getFileName().isAbsolute() + and: + path.getFileSystem().getPath('.fusion').toString() == '.fusion' + } + + @Unroll + def 'should match the hidden component #COMPONENT of a s3 path' () { + given: + Global.session = Mock(Session) { getConfig() >> [:] } + def path = FileHelper.asPath(PATH) + and: + // the S3 file system does not implement `getPathMatcher`, hence this is the + // `getDefaultPathMatcher` fallback, matching against the path `toString()` + def matchers = [ FileHelper.getPathMatcherFor("glob:$COMPONENT", path.getFileSystem()) ] + + expect: + FileHelper.matchesAnyName(matchers, path) == EXPECTED + + where: + COMPONENT | PATH | EXPECTED + '.fusion' | 's3://my-bucket/work/ab/cdef/.fusion' | true + '.fusion' | 's3://my-bucket/.fusion' | true + '.fusion' | 's3://my-bucket/work/ab/cdef/.fusion-other' | false + '.fusion' | 's3://my-bucket/work/ab/cdef' | false + '.fusion' | 's3://.fusion/work/ab/cdef' | false + '.*' | 's3://my-bucket/work/ab/cdef/.fusion' | true + '.*' | 's3://my-bucket/work/ab/.git' | true + '\\.bar' | 's3://my-bucket/work/ab/.bar' | true + '.b*' | 's3://my-bucket/work/ab/.bar' | true + '.b*' | 's3://my-bucket/work/ab/.zzz' | false + } }