Skip to content

Commit

Permalink
[Template merge] more test scripts to core
Browse files Browse the repository at this point in the history
  • Loading branch information
@flammie
flammie committed Jul 5, 2024
1 parent b54ddac commit 60b70ae
Show file tree
Hide file tree
Showing 3 changed files with 25 additions and 121 deletions.
8 changes: 3 additions & 5 deletions m4/giella-macros.m4
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ AC_MSG_RESULT([$GIELLA_CORE])
###############################################################
### This is the version of the Giella Core that we require. ###
### UPDATE AS NEEDED.
_giella_core_min_version=1.0.1
_giella_core_min_version=1.0.2
# GIELLA_CORE/GTCORE env. variable, required by the infrastructure to find scripts:
AC_ARG_VAR([GIELLA_CORE], [directory for the Giella infra core scripts and other required resources])
Expand Down Expand Up @@ -1048,10 +1048,8 @@ AS_IF([test "x$enable_yamltests" = "xno"],
[*darwin*], [AC_MSG_WARN([YAML testing could not be automatically enabled.
To enable it, on MacOSX please do:
sudo port install python38 py38-pip
sudo python3 -m pip install PyYAML
replace 38 in python38 with current python])],
sudo brew install python pyyaml
sudo python3 -m pip install PyYAML])],
[*linux*], [AC_MSG_WARN([YAML testing could not be automatically enabled.
To enable it on linux please use your package manager to install python an pyyaml,
or install pip and use it to install pyyaml.])],
Expand Down
98 changes: 9 additions & 89 deletions src/fst/test/run-lexc-testcases.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,102 +19,22 @@ fi

###### Variables: #######
transducer=gt-norm
Fail=0
Tests_found=no
Skipped=no
testtype=full
concat_lexc_file="lexicon.lexc"

relpath=$GIELLA_CORE/scripts/
testrunner=run-morph-tester.sh
testrunner=run-lexc-testcases.sh

if ! test -x "$relpath/$testrunner" ; then
echo "$0: No test runner found in $relpath/$testrunner!"
exit 77
fi

# Get list of source files:
source_files="$(find ${srcdir}/../ -name '*.lexc' \
-not -name "$concat_lexc_file")"

# One empty line in the beginning:
echo ""

# Loop over all lexc source files:
for file in ${source_files}; do
fileshort=$(echo "$(basename \
$(dirname $file))/$(basename $file)")

# For each lexc file, extract all fst's specified in it:
fsts=$(grep '^\!\!€[^ :]' $file | cut -d':' -f1 \
| sed 's/\(.*\)€\(.*\)/\2/g' | sort -u)

# Check whether there are test cases in the file:
tests=$(grep '^\!\!€ ' $file)

# If no tests or fst's are found, skip the file:
if test "$fsts" == "" -a "$tests" == ""; then
if test "$Tests_found" == "no" ; then
Skipped=yes
fi
echo "SKIPPED: no tests in $fileshort"

# If there are tests, but no specified fst, skip (future: run default fst)
elif test "$fsts" == "" -a ! "$tests" == ""; then
# (( i += 1 ))
# echo "$file has tests, but no fst specified - defaulting to $transducer"
# source ./run-yaml-testcases.sh $transducer $file
echo "* WARNING: $fileshort has tests, but no fst specified - SKIPPED"

# For each specified fst in the lexc file, run those tests:
else
# echo "TESTING: found tests in $fileshort" # debug

Tests_found=yes
Skipped=no

for fst in $fsts; do
(( i += 1 ))
echo "Running $fst tests:"
# Empty line before each new fst:
echo
leadtext="LEXC test $i: "

# Check for possible one-sided tests (default is two-sided/full):
if [[ "$fst" == *.gen ]]; then
testtype="gen"
fst=$(basename "$fst" .gen)
elif [[ "$fst" == *.ana ]] ; then
testtype="ana"
fst=$(basename "$fst" .ana)
fi

# Run the actual tests for the given fst:
for tk in $TESTKITS ; do
"$relpath"/run-morph-tester.sh \
"$fst" "$file" "$relpath" "$testtype" all "$srcdir" "$tk" "$leadtext"
done
# echo "The $fst testing is done using $testtype testing." # debug

# Reset testtype to default:
testtype=full
done
fi
for tk in $TESTKITS ; do
GIELLA_CORE=$GIELLA_CORE "$relpath/$testrunner" "$srcdir" "$tk"
rv=$?
if test $rv = 77 ; then
exit 77
elif test $rv -ge 1 ; then
exit 1
fi
done


if test "$Skipped" == "yes" ; then
exit 77
fi

# At least one of the Xerox or HFST tests failed:
if test "$Fail" -ge 1; then
exit 1
fi

# Skip if no transducer were found - failing is problematic for lexc tests:
if test "$Tests_found" = 0 ; then
echo "No lexcy tests found"
exit 77
fi

40 changes: 13 additions & 27 deletions tools/spellcheckers/test/fstbased/desktop/hfst/accept-all-lemmas.sh.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,37 +15,23 @@

###### Variables: #######
GIELLA_LANG=@GTLANG2@
ospell=@HFST_OSPELL@
### in ###
source_files=${srcdir}/../../../../../../src/fst/morphology/stems/*.lexc
speller_dir=./../../../../../../tools/spellcheckers

### out ###
lemmas=./filtered-lemmas.txt
rejected_lemmas=rejected_lemmas.txt
checked_lemmas=checked_lemmas.txt
source_files=@abs_top_srcdir@/src/fst/morphology/stems/*.lexc
speller_dir=@abs_builddir@/../../../../
exclusions="(CmpN/Only|ShCmp|\+Cmp/SplitR| Rreal | R | Rnoun |\+V\+|NOT-TO-LEMMATEST)"

##### Extract lemmas - add additional egrep pattern as parameters: #####
##### --include "(pattern1|pattern2|...)" #####
##### --exclude "(pattern1|pattern2|...)" #####
@GTCORE@/scripts/extract-lemmas.sh \
--exclude "(CmpN/Only|ShCmp|\+Cmp\/SplitR| Rreal | R | Rnoun |\+V\+|NOT-TO-LEMMATEST)" \
$source_files > $lemmas

if ! test -f $speller_dir/${GIELLA_LANG}.zhfst ; then
echo missing $speller_dir/${GIELLA_LANG}.zhfst
fi

####### Start testing: #######

$ospell $speller_dir/${GIELLA_LANG}.zhfst < $lemmas > $checked_lemmas

grep 'is NOT in the lexicon' $checked_lemmas > $rejected_lemmas

if [ -s $rejected_lemmas ] ; then
head $rejected_lemmas
echo see $rejected_lemmas for more
@GTCORE@/scripts/accept-all-lemmas.sh "$speller_dir/$GIELLA_LANG.zhfst" \
"$exclusions" \
"$source_files"

rv=$?
if test $rv = 77 ; then
echo skipped
exit 77
elif test $rv -ge 1 ; then
echo failed
exit 1
fi

rm -f $lemmas $rejected_lemmas

0 comments on commit 60b70ae

Please sign in to comment.