Apply the POSIX dot rule to each glob pattern component

docs/reference/process.mdx

@@ -129,7 +129,7 @@ When `true`, the file name is interpreted as a glob pattern (default: `true`).
 
 ###### `hidden: Boolean`
 
-When `true`, hidden files are included in the matching output files (default: `false`).
+When `true`, hidden files and directories are included in the matching output files (default: `false`). By default the leading `.` of a hidden name must be matched explicitly by the corresponding component of the pattern, and hidden directories are not traversed.
 
 ###### `includeInputs: Boolean`
 
@@ -259,7 +259,7 @@ When `true`, the specified name is interpreted as a glob pattern (default: `true
 
 ###### `hidden`
 
-When `true`, hidden files are included in the matching output files (default: `false`).
+When `true`, hidden files and directories are included in the matching output files (default: `false`). By default the leading `.` of a hidden name must be matched explicitly by the corresponding component of the pattern, and hidden directories are not traversed.
 
 ###### `includeInputs`
 

docs/reference/stdlib-namespaces/channel.mdx

@@ -57,7 +57,7 @@ When `true`, follows symbolic links when traversing a directory tree, otherwise
 
 ###### `hidden`
 
-When `true`, matches hidden files when using a glob pattern (default: `false`).
+When `true`, matches hidden files and directories when using a glob pattern, without requiring the pattern to match the leading `.` explicitly (default: `false`).
 
 ###### `maxDepth`
 
@@ -144,7 +144,7 @@ channel.fromPath('data/**/*.fa')
 channel.fromPath('data/file_{1,2}.fq')
 ```
 
-By default, glob patterns do not match hidden files (i.e. files with names that start with `.`). Use a glob pattern that explicitly starts with `.` or set `hidden: true` to match hidden files:
+By default, glob patterns do not match hidden files and directories (i.e. names that start with `.`). As in Bash, the leading `.` must be matched explicitly by the corresponding component of the pattern, and hidden directories are not traversed unless a component of the pattern matches them. Alternatively, set `hidden: true` to match every hidden file and directory:
 
 ```nextflow
 // match hidden files in `data`
@@ -153,8 +153,13 @@ channel.fromPath('data/*', hidden: true)
 
 // match hidden files in `data` with `fa` extension
 channel.fromPath('data/.*.fa')
+
+// match `fa` files in any hidden `.cache` directory under `data`
+channel.fromPath('data/**/.cache/*.fa')
 ```
 
+Since the `.` is matched for each component separately, `data/*/*.fa` does not match `data/.cache/file.fa`, because the pattern says nothing about a hidden `.cache` directory.
+
 By default, glob patterns only match regular files, not directories. Use the `type` option to control whether to match files, directories, or both:
 
 ```nextflow
@@ -181,7 +186,7 @@ When `true`, interprets the characters `*`, `?`, `[]`, and `{}` as glob wildcard
 
 ###### `hidden`
 
-When `true`, matches hidden files when using a glob pattern (default: `false`).
+When `true`, matches hidden files and directories when using a glob pattern, without requiring the pattern to match the leading `.` explicitly (default: `false`).
 
 ###### `maxDepth`
 

modules/nextflow/src/main/groovy/nextflow/processor/TaskFileCollector.groovy

@@ -117,7 +117,11 @@ class TaskFileCollector {
     protected Map<String,?> visitOptions(String pattern) {
         return [
             relative: false,
-            hidden: opts.hidden ?: pattern.startsWith('.'),
+            // note: only the explicit `hidden` option is forwarded. The dot rule for the pattern
+            // itself is applied by `FileHelper.visitFiles` per path component, therefore deriving
+            // it here from `pattern.startsWith('.')` would turn a single dotted component into a
+            // tree-wide authorisation e.g. `.config/**/*.txt` would also collect `.config/a/.secret/x.txt`
+            hidden: opts.hidden as boolean,
             followLinks: opts.followLinks,
             maxDepth: opts.maxDepth,
             type: opts.type ? opts.type : ( pattern.contains('**') ? 'file' : 'any' )

modules/nextflow/src/test/groovy/nextflow/processor/TaskFileCollectorTest.groovy

@@ -140,6 +140,65 @@ class TaskFileCollectorTest extends Specification {
 
     }
 
+    def 'should not collect the files left in the work dir by a hidden directory'() {
+
+        given:
+        // mimic a task work dir where Fusion has left its own metadata files under
+        // `<workDir>/.fusion/v1/<absolute path>/md.json` -- see issue #7480
+        def folder = Files.createTempDirectory('test')
+        folder.resolve('sample/outs').mkdirs()
+        folder.resolve('sample/outs/real.txt').text = 'real output'
+        and:
+        folder.resolve('.fusion/v1/s3/bucket/work/ab/cdef/sample/outs').mkdirs()
+        folder.resolve('.fusion/v1/s3/bucket/work/ab/cdef/sample/outs/md.json').text = '{"internal":"metadata"}'
+        and:
+        def result
+
+        when: 'the pattern does not name the hidden directory'
+        result = fetchResultFiles('**/outs/**', folder)
+        then:
+        result == ['sample/outs/real.txt']
+
+        when: 'hidden files are explicitly requested'
+        result = fetchResultFiles('**/outs/**', folder, hidden: true)
+        then:
+        result == ['.fusion/v1/s3/bucket/work/ab/cdef/sample/outs/md.json', 'sample/outs/real.txt']
+
+        cleanup:
+        folder?.deleteDir()
+
+    }
+
+    def 'should apply the dot rule per component for a pattern starting with a dot'() {
+
+        given:
+        // a dotted component authorises a hidden name where it appears and nowhere else,
+        // therefore `.config` is visited while `.secret` and `.dotleaf.txt` are not.
+        // bash 5.3 and zsh 5.9 return `.config/a/plain/y.txt` alone for this tree
+        def folder = Files.createTempDirectory('test')
+        folder.resolve('.config/a/.secret').mkdirs()
+        folder.resolve('.config/a/plain').mkdirs()
+        folder.resolve('.config/a/.secret/x.txt').text = 'x'
+        folder.resolve('.config/a/plain/y.txt').text = 'y'
+        folder.resolve('.config/a/.dotleaf.txt').text = 'z'
+        and:
+        def result
+
+        when:
+        result = fetchResultFiles('.config/**/*.txt', folder)
+        then:
+        result == ['.config/a/plain/y.txt']
+
+        when: 'hidden files are explicitly requested'
+        result = fetchResultFiles('.config/**/*.txt', folder, hidden: true)
+        then:
+        result == ['.config/a/.dotleaf.txt', '.config/a/.secret/x.txt', '.config/a/plain/y.txt']
+
+        cleanup:
+        folder?.deleteDir()
+
+    }
+
     def defaultCollector(Map opts) {
         return new TaskFileCollector([], opts, Mock(TaskRun))
     }
@@ -154,7 +213,9 @@ class TaskFileCollectorTest extends Specification {
         then:
         collector.visitOptions('file.txt') == [type:'any', followLinks: true, maxDepth: null, hidden: false, relative: false]
         collector.visitOptions('path/**') == [type:'file', followLinks: true, maxDepth: null, hidden: false, relative: false]
-        collector.visitOptions('.hidden_file') == [type:'any', followLinks: true, maxDepth: null, hidden: true, relative: false]
+        // note: a dotted pattern no longer forces `hidden` here. The dot rule is applied by
+        // `FileHelper.visitFiles` per path component, see issue #7480
+        collector.visitOptions('.hidden_file') == [type:'any', followLinks: true, maxDepth: null, hidden: false, relative: false]
 
         when:
         collector = defaultCollector([type: 'dir'])

modules/nf-commons/src/main/nextflow/file/FileHelper.groovy

@@ -854,12 +854,37 @@ class FileHelper {
         final type = options.type ?: 'any'
         final walkOptions = options.followLinks == false ? EnumSet.noneOf(FileVisitOption.class) : EnumSet.of(FileVisitOption.FOLLOW_LINKS)
         final int maxDepth = getMaxDepth(options.maxDepth, filePattern)
-        final includeHidden = options.hidden as Boolean ?: filePattern.startsWith('.')
+        final syntax = options.syntax ?: 'glob'
+        // NOTE: the dot rule below can only be applied to a glob pattern. When the `regex` syntax
+        // is used the pattern is a regular expression, hence a leading dot is the any-char
+        // meta-character and the slash is not a component separator, therefore splitting it into
+        // path components would be meaningless. For any non-glob syntax the legacy behaviour is
+        // preserved i.e. only a pattern *starting* with a dot enables hidden files, and hidden
+        // directories are not pruned from the tree walk.
+        final isGlob = syntax == 'glob'
+        final includeHidden = options.hidden as Boolean ?: (isGlob ? false : filePattern.startsWith('.'))
+        final dotMatchers = isGlob && !includeHidden
+                ? hiddenComponents(filePattern).collect { getPathMatcherFor("glob:$it", folder.fileSystem) }
+                : Collections.<PathMatcher>emptyList()
         final includeDir = type in ['dir','any']
         final includeFile = type in ['file','any']
-        final syntax = options.syntax ?: 'glob'
         final relative = options.relative == true
 
+        // Implements the POSIX rule (Issue 8, section 2.14.3) also honoured by bash and zsh: a
+        // leading dot in a file name must be matched explicitly by a dot in the corresponding
+        // component of the pattern. Therefore a dot-prefixed component does *not* enable hidden
+        // paths for the whole pattern, it only authorises a hidden name where it appears, e.g.
+        // `.config/**/*.txt` visits `.config` but not `.config/a/.secret`.
+        //
+        // NOTE: the authorisation is checked against the file name alone rather than against the
+        // position of the component within the pattern. Since `**` matches an arbitrary number of
+        // components, a pattern component cannot be reliably aligned with a path component, so the
+        // dot-prefixed components of the pattern are collected up-front and a hidden name is
+        // allowed when it matches any of them as a single-component glob.
+        final Closure<Boolean> allowed = { Path it ->
+            includeHidden || !isHidden(it) || matchesAnyName(dotMatchers, it)
+        }
+
         final matcher = getPathMatcherFor("$syntax:${filePattern}", folder.fileSystem)
         final singleParam = action.getMaximumNumberOfParameters() == 1
 
@@ -871,7 +896,13 @@ class FileHelper {
                 final path = relativize0(folder, fullPath)
                 log.trace "visitFiles > dir=$path; depth=$depth; includeDir=$includeDir; matches=${matcher.matches(path)}; isDir=${attrs.isDirectory()}"
 
-                if (depth>0 && includeDir && matcher.matches(path) && attrs.isDirectory() && (includeHidden || !isHidden(fullPath))) {
+                // do not descend into a hidden directory unless the pattern authorises it
+                // note: the `depth>0` check is needed because the start folder itself can be
+                // hidden or nested under a hidden directory e.g. a work dir /scratch/.tmp/ab/cdef
+                if( depth>0 && isGlob && !allowed(fullPath) )
+                    return FileVisitResult.SKIP_SUBTREE
+
+                if (depth>0 && includeDir && matcher.matches(path) && attrs.isDirectory() && allowed(fullPath)) {
                     def result = relative ? path : fullPath
                     singleParam ? action.call(result) : action.call(result,attrs)
                 }
@@ -886,7 +917,7 @@ class FileHelper {
                     : fullPath
                 log.trace "visitFiles > file=$path; includeFile=$includeFile; matches=${matcher.matches(path)}; isRegularFile=${attrs.isRegularFile()}"
 
-                if (includeFile && matcher.matches(path) && (attrs.isRegularFile() || (options.followLinks == false && attrs.isSymbolicLink())) && (includeHidden || !isHidden(fullPath))) {
+                if (includeFile && matcher.matches(path) && (attrs.isRegularFile() || (options.followLinks == false && attrs.isSymbolicLink())) && allowed(fullPath)) {
                     def result = relative ? path : fullPath
                     singleParam ? action.call(result) : action.call(result,attrs)
                 }
@@ -926,6 +957,66 @@ class FileHelper {
         len>0 ? Paths.get(str.substring(0,str.length()-1)) : Paths.get('')
     }
 
+    /**
+     * Collect the dot-prefixed components of the given glob pattern, e.g. {@code .config} for
+     * the pattern {@code .config/**}. Each of them authorises a hidden file name in the visit,
+     * as prescribed by the POSIX dot rule.
+     *
+     * Note: the {@code .} and {@code ..} components are *not* considered hidden names.
+     *
+     * @param pattern A glob pattern e.g. {@code foo/.bar/*}
+     * @return The list of dot-prefixed components, each one being itself a glob pattern
+     */
+    @PackageScope
+    static List<String> hiddenComponents(String pattern) {
+        final result = new ArrayList<String>()
+        if( !pattern )
+            return result
+        for( String it : pattern.tokenize('/') ) {
+            // strip the escape char, since a dot can be escaped to match it literally
+            final name = it.startsWith('\\') ? it.substring(1) : it
+            if( name=='.' || name=='..' )
+                continue
+            if( name.startsWith('.') )
+                result.add(it)
+        }
+        return result
+    }
+
+    /**
+     * Check if the *name* of the given path matches any of the specified single-component matchers
+     *
+     * Note: the matchers can be the fallback implementation returned by
+     * {@link #getDefaultPathMatcher(java.lang.String)}, which is used when the file system does
+     * not support {@link java.nio.file.FileSystem#getPathMatcher(java.lang.String)} e.g. S3.
+     * That implementation matches against the path {@code toString()}, therefore the name is
+     * matched as a *relative* single-component path, so that no file system specific prefix
+     * (e.g. the S3 bucket) can be prepended to it.
+     *
+     * @param matchers A list of matchers built from a single pattern component
+     * @param path The path whose file name should be checked
+     * @return {@code true} when the path name matches at least one of the given matchers
+     */
+    @PackageScope
+    static boolean matchesAnyName(List<PathMatcher> matchers, Path path) {
+        if( !matchers )
+            return false
+        // note: fileName can be null for the root path
+        final fileName = path.getFileName()
+        if( fileName==null )
+            return false
+        // some file systems report a directory name with a trailing slash, in that case the name
+        // has to be rebuilt because the matcher would not match the ending separator
+        final str = fileName.toString()
+        final name = str.endsWith('/')
+                ? path.getFileSystem().getPath(str.substring(0, str.length()-1))
+                : fileName
+        for( PathMatcher it : matchers )
+            if( it.matches(name) )
+                return true
+        return false
+    }
+
     private static boolean isHidden(Path path) {
         // note: fileName can be null for root path
         def fileName = path.getFileName()