feat: much progress toward running all regress cases

everyvoice/tests/regression/README.md

@@ -0,0 +1,38 @@
+# EveryVoice regression test suite
+
+## Preparing the regression training data:
+
+ - Download LJ 1.1 from https://keithito.com/LJ-Speech-Dataset/
+ - Download Sinhala TTS from https://openslr.org/30/
+ - Download High quality TTS data for four South African languages (af, st, tn,
+   xh) from https://openslr.org/32
+ - See [`prep-datasets.sh`](prep-datasets.sh) to see where these datasets are expected to be found.
+ - Run this to create the regression testing directory structure:
+
+       mkdir regress-1  # or any suffix you want
+       cd regress-1
+       ../prep-datasets.sh
+
+## Running the regression tests
+
+On a Slurm cluster:
+
+    for dir in regress-*; do
+        pushd $dir
+        sbatch ../../regression-test.sh
+        popd
+    done
+
+Or just use `../../regression-test.sh` directly in the loop if you're not on a cluster.
+
+## Test data provenance
+
+ - `test-si.txt`: copied from https://en.wikipedia.org/wiki/Sinhala_script CC BY-SA-4.0
+    - the first line means Sinhala script, found at the top of the page
+    - the rest is the first verse from the Pali Dhammapada lower on the same page
+ - `test2-si.txt`: one word from that first line
+
+ - `test*-xh.txt`: individual words copied from https://en.wikipedia.org/wiki/Xhosa_language
+   CC BY-SA-4.0
+
+ - `test*-lj.txt`: written by Eric Joanis

everyvoice/tests/regression/combine-coverage.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+find . -name .coverage\* | coverage combine --keep
+coverage report --include='*/everyvoice/*' | sed 's/.*EveryVoice\/everyvoice/everyvoice/' > coverage.txt
+coverage html --include='*/everyvoice/*'
+coverage xml --include='*/everyvoice/*'
+sed -i 's/"[^"]*EveryVoice.everyvoice/"everyvoice/g' coverage.xml

everyvoice/tests/regression/go.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+# Automated application of the instructions in README.md
+
+set -o errexit
+
+TOP_LEVEL_DIR=$(mktemp --directory regress-XXX)
+cd "$TOP_LEVEL_DIR"
+
+../prep-datasets.sh
+for DIR in regress-*; do
+    pushd "$DIR"
+    sbatch ../../regression-test.sh
+    popd
+done
+
+coverage run -p -m everyvoice test
+
+JOB_COUNT=$(find . -maxdepth 1 -name regress-\* | wc -l)
+while true; do
+    DONE_COUNT=$(find . -maxdepth 2 -name DONE | wc -l)
+    if (( DONE_COUNT + 2 >= JOB_COUNT )); then
+        break
+    fi
+    echo "$DONE_COUNT/$JOB_COUNT regression job(s) done. Still waiting."
+    date
+    sleep $(( 60 * 5 ))
+done
+
+echo "$DONE_COUNT regression jobs done. Calculating coverage now, but some jobs may still be running."
+../combine-coverage.sh
+cat coverage.txt
+
+while true; do
+    DONE_COUNT=$(find . -maxdepth 2 -name DONE | wc -l)
+    if (( DONE_COUNT >= JOB_COUNT )); then
+        break
+    fi
+    echo "$DONE_COUNT/$JOB_COUNT regression job(s) done. Still waiting."
+    date
+    sleep $(( 60 * 5 ))
+done
+
+echo "All $DONE_COUNT regression jobs done. Calculating final coverage."
+rm .coverage
+../combine-coverage.sh
+cat coverage.txt

everyvoice/tests/regression/prep-datasets.sh

@@ -4,9 +4,6 @@
 
 set -o errexit
 
-LJ_SPEECH_DATASET=$HOME/sgile/data/LJSpeech-1.1
-LJSLICES="150 600 1600 full"
-
 # Usage: cat my_file | get_slice lines_to_keep > out
 # Use a number of lines or full to get all lines
 get_slice() {
@@ -18,21 +15,34 @@ get_slice() {
     fi
 }
 
+EVERYVOICE_REGRESS_ROOT=$(python -c 'import everyvoice; print(everyvoice.__path__[0])')/tests/regression
+
+LJ_SPEECH_DATASET=$HOME/sgile/data/LJSpeech-1.1
+LJSLICES="150 600 1600 full"
 for slice in $LJSLICES; do
     dir=regress-lj-$slice
     mkdir "$dir"
     ln -s "$LJ_SPEECH_DATASET/wavs" "$dir"/
     get_slice "$slice" < "$LJ_SPEECH_DATASET/metadata.csv" > "$dir"/metadata.csv
+    cp "$EVERYVOICE_REGRESS_ROOT"/wizard-resume-lj "$dir"/wizard-resume
+    cp "$EVERYVOICE_REGRESS_ROOT"/test-lj.txt "$dir"/test.txt
+    cp "$EVERYVOICE_REGRESS_ROOT"/test2-lj.txt "$dir"/test2.txt
 done
 
 SinhalaTTS=$HOME/sgile/data/SinhalaTTS
 dir=regress-si
 mkdir $dir
 ln -s "$SinhalaTTS/wavs" $dir/
 cp "$SinhalaTTS/si_lk.lines.txt" $dir/
+cp "$EVERYVOICE_REGRESS_ROOT"/wizard-resume-si "$dir"/wizard-resume
+cp "$EVERYVOICE_REGRESS_ROOT"/test-si.txt "$dir"/test.txt
+cp "$EVERYVOICE_REGRESS_ROOT"/test2-si.txt "$dir"/test2.txt
 
 isiXhosa=$HOME/sgile/data/OpenSLR32-four-South-Afican-languages/xh_za/za/xho
 dir=regress-xh
 mkdir $dir
 ln -s "$isiXhosa/wavs" $dir/
 cp "$isiXhosa/line_index.tsv" $dir/
+cp "$EVERYVOICE_REGRESS_ROOT"/wizard-resume-xh "$dir"/wizard-resume
+cp "$EVERYVOICE_REGRESS_ROOT"/test-xh.txt "$dir"/test.txt
+cp "$EVERYVOICE_REGRESS_ROOT"/test2-xh.txt "$dir"/test2.txt

everyvoice/tests/regression/regression-test.sh

@@ -12,76 +12,68 @@
 #SBATCH --output=./%x.o%j
 #SBATCH --error=./%x.e%j
 
-set -o errexit
+# User env config -- set ACTIVATE_SCRIPT to point to something that will activate the
+# right Python environment, or leave it empty if you don't need it.
+ACTIVATE_SCRIPT=$HOME/start_ev.sh
 
 # Run a command, logging it first
 r() {
     cmd="$*"
     printf "\n\n======================================================================\n"
     printf 'Running "%s"\n' "$cmd"
-    printf "\n\n======================================================================\n"
+    date
+    printf "======================================================================\n"
     eval "$cmd" 2>&1
     rc=$?
     if [[ $rc != 0 ]]; then
-        echo "Command \"$cmd\" exited with non-zero return code $rc"
+        printf "\n\nXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\n"
+        echo "Command \"$cmd\" exited with non-zero return code $rc."
+        date
+        printf "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\n"
     fi
     return $rc
 }
 
-# User env config -- adjust this as necessary before running:
-ACTIVATE_SCRIPT=$HOME/start_ev.sh
-LJ_SPEECH_DATASET=$HOME/tts/corpora/Speech/LJ.Speech.Dataset/LJSpeech-1.1
+echo "Start at $(date)"
+date > START
 
 # Regression config
 [[ -e "$ACTIVATE_SCRIPT" ]] && source "$ACTIVATE_SCRIPT"
 export TQDM_MININTERVAL=5
-LINES_TO_USE=2000
-EPOCHS=2
-EVERYVOICE_ROOT=$(python -c 'import everyvoice; print(everyvoice.__path__[0])')
-if [[ $SLURM_JOBID ]]; then
-    WORKDIR_SUFFIX="$SLURM_JOBID"
-else
-    WORKDIR_SUFFIX="$(date +'%Y%m%d')"
-fi
-WORKDIR=regress-"$WORKDIR_SUFFIX"
-mkdir "$WORKDIR"
-cd "$WORKDIR"
-
-# 1: create a small dataset with *no header line*
-grep -v '^basename|' "$LJ_SPEECH_DATASET/metadata.csv" | head -"$LINES_TO_USE" > metadata.csv
-ln -s "$LJ_SPEECH_DATASET/wavs" .
-
-# 2: run the new-project wizard
-r "coverage run -p -m everyvoice new-project --resume-from '$EVERYVOICE_ROOT/tests/regress-lj2k-resume'"
-
-# 3: preprocess
-
-cd regress
+EPOCHS=10
+
+# Run the new-project wizard
+r "coverage run -p -m everyvoice new-project --resume-from wizard-resume"
+
+# Preprocess
+
+cd regress || { echo "Cannot cd into regress directory, aborting."; date > DONE ; exit 1; }
 r "coverage run -p -m everyvoice preprocess config/everyvoice-text-to-spec.yaml"
 
-# 4: train the fs2 model
+# Train the fs2 model
 r "coverage run -p -m everyvoice train text-to-spec config/everyvoice-text-to-spec.yaml --config-args training.max_epochs=$EPOCHS"
 FS2=logs_and_checkpoints/FeaturePredictionExperiment/base/checkpoints/last.ckpt
 ls $FS2
 
-# 5: train the vocoder
+# Train the vocoder
 r "coverage run -p -m everyvoice train spec-to-wav config/everyvoice-spec-to-wav.yaml --config-args training.max_epochs=$EPOCHS"
 VOCODER=logs_and_checkpoints/VocoderExperiment/base/checkpoints/last.ckpt
 ls $VOCODER
 
-# 6: synthesize some text
+# Synthesize some text
 r "coverage run -p -m everyvoice synthesize from-text \
     --output-type wav --output-type spec --output-type textgrid --output-type readalong-xml --output-type readalong-html \
-    --filelist '$EVERYVOICE_ROOT/tests/regress-text.txt' \
+    --filelist ../test.txt \
     --vocoder-path '$VOCODER' \
     '$FS2'"
 # TODO: check the synthesized files, somehow
 
 # Exercise two-step synthesis
-r "coverage run -p -m everyvoice synthesize from-text --output-type spec --text 'Test via spec.' '$FS2'"
+ONE_WORD=$(cat ../test2.txt)
+r "coverage run -p -m everyvoice synthesize from-text --output-type spec --text '$ONE_WORD' '$FS2'"
 r "coverage run -p -m everyvoice synthesize from-spec \
-    --input synthesis_output/synthesized_spec/Test-via-spec---speaker_0--eng--spec-pred-22050-mel-librosa.pt \
-    --vocoder-path '$VOCODER'"
+    --input synthesis_output/synthesized_spec/'$ONE_WORD'-*.pt \
+    --model '$VOCODER'"
 
 # Exercise DeepForceAligner
 # Meh, this appears to be broken...
@@ -96,12 +88,5 @@ r "coverage run -p -m everyvoice synthesize from-spec \
 # 8: use playwright to synthesize something using the demo
 # TODO...
 
-
-# Run the regular everyvoice test suite to complete coverage
-cd ..
-r "coverage run -p -m everyvoice test"
-
-# Collect coverage data
-coverage combine . regress
-coverage html --include='*/everyvoice/*'
-coverage report --include='*/everyvoice/*'
+echo "Done at $(date)"
+date > ../DONE

everyvoice/tests/regression/test-si.txt

@@ -0,0 +1,4 @@
+සිංහල අක්ෂර මාලාව
+මනොපුබ්‌බඞ්‌ගමා ධම්‌මා, මනොසෙට්‌ඨා මනොමයා;
+මනසා චෙ පදුට්‌ඨෙන, භාසති වා කරොති වා;
+තතො නං දුක්‌ඛමන්‌වෙති, චක්‌කංව වහතො පදං.

everyvoice/tests/regression/test-xh.txt

@@ -0,0 +1,3 @@
+ukukrwentshwa
+uqeqesho
+iimpumlo

everyvoice/tests/regression/test2-lj.txt

@@ -0,0 +1 @@
+spec

everyvoice/tests/regression/test2-si.txt

@@ -0,0 +1 @@
+අක-ෂර

everyvoice/tests/regression/test2-xh.txt

@@ -0,0 +1 @@
+isiXhosa