Merge pull request #144 from marytts/custom-basenames-lst

Custom basenames.lst

.github/workflows/main.yml

@@ -18,6 +18,11 @@ jobs:
           - 8
           - 11
           - 13
+        exclude:
+          - os: macos-latest
+            java_version: 8
+          - os: macos-latest
+            java_version: 13
     runs-on: ${{ matrix.os }}
 
     steps:

CHANGELOG.md

@@ -6,19 +6,23 @@ Gradle MaryTTS voicebuilding plugin
 
 ### Added
 
-- Plugin usage in readme
+- Plugin usage documented in readme
 - Testing on OpenJDK 11 and 13
 
 ### Changed
 
 - Migrate testing from Travis CI to GitHub Actions
-- Refactor to depends to gradle-marytts-component-plugin v0.2.2
-- Refactor to replace the dependency on the defunct marytts-builder by a dependecy on marytts-runtime
+- Refactor to depend on gradle-marytts-component-plugin v0.2.2
+- Refactor to replace the dependency on the defunct marytts-builder with a dependecy on marytts-runtime
 - Build with Gradle v6.3
 - Upgrade test dependencies
 - Drop testing on Oracle JDK
 - [all changes since v5.4]
 
+### Fixed
+
+- Utterance order in `basenames.lst` is now deterministic, either sorted (the default) or specified via a provided `srcFile` (optional)
+
 [v5.4] - 2018-12-12
 -------------------
 

src/main/groovy/de/dfki/mary/voicebuilding/tasks/GenerateBasenamesList.groovy

@@ -8,6 +8,10 @@ import org.gradle.api.tasks.*
 
 class GenerateBasenamesList extends DefaultTask {
 
+    @Optional
+    @InputFile
+    final RegularFileProperty srcFile = project.objects.fileProperty()
+
     @InputDirectory
     final DirectoryProperty wavDir = project.objects.directoryProperty()
 
@@ -39,20 +43,26 @@ class GenerateBasenamesList extends DefaultTask {
     @TaskAction
     void generate() {
         destFile.get().asFile.withWriter('UTF-8') { writer ->
-            project.fileTree(wavDir).matching {
-                include this.includes.getOrElse('*').collect { it + '.wav' }
-                exclude this.excludes.getOrElse([]).collect { it + '.wav' }
-            }.toSorted().each { wavFile ->
-                def basename = wavFile.name - '.wav'
+            def basenames
+            if (srcFile.getOrNull()) {
+                basenames = srcFile.get().asFile.readLines().findAll { !it.trim().startsWith('#') }
+            } else {
+                basenames = project.fileTree(wavDir).matching {
+                    include this.includes.getOrElse('*').collect { it + '.wav' }
+                    exclude this.excludes.getOrElse([]).collect { it + '.wav' }
+                }.collect { it.name - '.wav' }.toSorted()
+            }
+            basenames.each { basename ->
+                def wavFile = wavDir.file("${basename}.wav").get().asFile
+                if (!wavFile.canRead())
+                    project.logger.warn "WARNING: Could not read from $wavFile"
                 def textFile = textDir.file("${basename}.txt").get().asFile
-                if (!textFile.canRead()) {
+                if (!textFile.canRead())
                     project.logger.warn "WARNING: Could not read from $textFile"
-                }
                 def labFile = labDir.file("${basename}.lab").get().asFile
-                if (!labFile.canRead()) {
+                if (!labFile.canRead())
                     project.logger.warn "WARNING: Could not read from $labFile"
-                }
-                if (textFile.canRead() && labFile.canRead()) {
+                if (wavFile.canRead() && textFile.canRead() && labFile.canRead()) {
                     writer.println basename
                 }
             }

src/test/groovy/de/dfki/mary/voicebuilding/tasks/GenerateBasenamesListFunctionalTest.groovy

@@ -0,0 +1,161 @@
+package de.dfki.mary.voicebuilding.tasks
+
+import org.gradle.testkit.runner.BuildResult
+import org.gradle.testkit.runner.GradleRunner
+import org.testng.annotations.Test
+
+class GenerateBasenamesListFunctionalTest {
+
+    @Test
+    void 'Given data directories, When basenames list is generated, Then basenames are in sort order'() {
+        def projectDir = File.createTempDir()
+
+        def basenames = generateBasenames(5)
+        createDataDirectories(projectDir, basenames)
+
+        generateBuildScript(projectDir)
+        runGradle(projectDir)
+
+        def expected = basenames
+        def actual = new File(projectDir, 'basenames.lst').readLines()
+        assert expected == actual
+    }
+
+    @Test
+    void 'Given data directories with some missing files, When basenames list is generated, Then basenames exclude them'() {
+        def projectDir = File.createTempDir()
+
+        def basenames = generateBasenames(6)
+        createDataDirectories(projectDir, basenames)
+
+        new File(projectDir.path + File.separator + 'wav', 'test_0002.wav').delete()
+        new File(projectDir.path + File.separator + 'txt', 'test_0004.txt').delete()
+        new File(projectDir.path + File.separator + 'lab', 'test_0006.lab').delete()
+
+        generateBuildScript(projectDir)
+        runGradle(projectDir)
+
+        def expected = basenames - ['test_0002', 'test_0004', 'test_0006']
+        def actual = new File(projectDir, 'basenames.lst').readLines()
+        assert expected == actual
+    }
+
+    @Test
+    void 'Given data directories, When custom list is provided, Then basenames are in custom order'() {
+        def projectDir = File.createTempDir()
+
+        def basenames = generateBasenames(5)
+        createDataDirectories(projectDir, basenames)
+
+        Collections.shuffle(basenames)
+        new File(projectDir, 'custom.lst').withWriter { customList ->
+            basenames.each { basename ->
+                customList.writeLine(basename)
+            }
+        }
+
+        def buildScript = generateBuildScript(projectDir)
+        buildScript << "basenames.srcFile = file('custom.lst')"
+        runGradle(projectDir)
+
+        def expected = basenames
+        def actual = new File(projectDir, 'basenames.lst').readLines()
+        assert expected == actual
+    }
+
+    @Test
+    void 'Given data directories, When custom list with comments is provided, Then basenames are in custom order'() {
+        def projectDir = File.createTempDir()
+
+        def basenames = generateBasenames(5)
+        createDataDirectories(projectDir, basenames)
+
+        Collections.shuffle(basenames)
+        new File(projectDir, 'custom.lst').withWriter { customList ->
+            customList.writeLine('# This is the custom order:')
+            basenames.eachWithIndex { basename, b ->
+                customList.writeLine(basename)
+                if (b == 3)
+                    customList.writeLine('  # Another comment!')
+            }
+        }
+
+        def buildScript = generateBuildScript(projectDir)
+        buildScript << "basenames.srcFile = file('custom.lst')"
+        runGradle(projectDir)
+
+        def expected = basenames
+        def actual = new File(projectDir, 'basenames.lst').readLines()
+        assert expected == actual
+    }
+
+    @Test
+    void 'Given data directories with some missing files, When custom list is provided, Then basenames exclude them and are in custom order'() {
+        def projectDir = File.createTempDir()
+
+        def basenames = generateBasenames(6)
+        createDataDirectories(projectDir, basenames)
+
+        Collections.shuffle(basenames)
+        new File(projectDir, 'custom.lst').withWriter { customList ->
+            basenames.each { basename ->
+                customList.writeLine(basename)
+            }
+        }
+
+        new File(projectDir.path + File.separator + 'wav', 'test_0002.wav').delete()
+        new File(projectDir.path + File.separator + 'txt', 'test_0004.txt').delete()
+        new File(projectDir.path + File.separator + 'lab', 'test_0006.lab').delete()
+
+        def buildScript = generateBuildScript(projectDir)
+        buildScript << "basenames.srcFile = file('custom.lst')"
+        runGradle(projectDir)
+
+        def expected = basenames - ['test_0002', 'test_0004', 'test_0006']
+        def actual = new File(projectDir, 'basenames.lst').readLines()
+        assert expected == actual
+    }
+
+    private static List<String> generateBasenames(int n) {
+        (1..n).collect { i ->
+            String.format('test_%04d', i)
+        }
+    }
+
+    private static List<String> createDataDirectories(projectDir, basenames) {
+        ['wav', 'txt', 'lab'].each { dirName ->
+            def dir = new File(projectDir, dirName)
+            dir.mkdir()
+            basenames.each { basename ->
+                new File(dir, "${basename}.${dirName}").createNewFile()
+            }
+        }
+    }
+
+    private static File generateBuildScript(File projectDir) {
+        def buildScript = new File(projectDir, 'build.gradle')
+        buildScript.text = '''
+            plugins {
+                id 'de.dfki.mary.voicebuilding-data'
+            }
+            
+            basenames {
+                wavDir = file('wav')
+                textDir = file('txt')
+                labDir = file('lab')
+                destFile = file('basenames.lst')
+            }
+            '''.stripMargin()
+        buildScript
+    }
+
+    private static BuildResult runGradle(File projectDir) {
+        def defaultArgs = ['--warning-mode', 'all', '--stacktrace', '--info']
+        GradleRunner.create()
+                .withProjectDir(projectDir)
+                .withPluginClasspath()
+                .forwardOutput()
+                .withArguments(defaultArgs + 'basenames')
+                .build()
+    }
+}

src/test/resources/de/dfki/mary/voicebuilding/dataPluginFunctionalTestBuildScript.gradle

@@ -9,15 +9,6 @@ marytts {
 }
 
 repositories {
-    ivy {
-        url 'https://dl.bintray.com/marytts/marytts'
-        patternLayout {
-            artifact '[organisation]/[module]/[artifact].[ext]'
-        }
-        metadataSources {
-            artifact()
-        }
-    }
     ivy {
         url 'http://festvox.org/examples'
         allowInsecureProtocol = true

src/test/resources/de/dfki/mary/voicebuilding/festvoxPluginFunctionalTestBuildScript.gradle

@@ -3,15 +3,6 @@ plugins {
 }
 
 repositories {
-    ivy {
-        url 'https://dl.bintray.com/marytts/marytts'
-        patternLayout {
-            artifact '[organisation]/[module]/[artifact].[ext]'
-        }
-        metadataSources {
-            artifact()
-        }
-    }
     ivy {
         url 'http://festvox.org/examples'
         allowInsecureProtocol = true

src/test/resources/de/dfki/mary/voicebuilding/legacyPluginFunctionalTestBuildScript.gradle

@@ -29,15 +29,6 @@ def voiceLocale = new Locale.Builder()
         .build()
 
 repositories {
-    ivy {
-        url 'https://dl.bintray.com/marytts/marytts'
-        patternLayout {
-            artifact '[organisation]/[module]/[artifact].[ext]'
-        }
-        metadataSources {
-            artifact()
-        }
-    }
     ivy {
         url 'http://festvox.org/examples'
         allowInsecureProtocol = true