Utils: Update spaCy's sentence recognizers, word tokenizers, part-of-…

…speech taggers, lemmatizers, and dependency parsers
BLKSerene · Aug 8, 2023 · 6eba084 · 6eba084
1 parent 047d5a4
commit 6eba084
Show file tree

Hide file tree

Showing 109 changed files with 3,075 additions and 133,160 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -53,21 +53,17 @@ jobs:
       # Run tests and collect coverage
       - name: Run Tests and collect coverage
         run: |
-          pytest --cov=./ --cov-report=xml --cov-append tests/wl_tests_nlp/test_dependency_parsing.py
-          pytest --cov=./ --cov-report=xml --cov-append tests/wl_tests_nlp/test_lemmatization.py
-          pytest --cov=./ --cov-report=xml --cov-append tests/wl_tests_nlp/test_matching.py
-          pytest --cov=./ --cov-report=xml --cov-append tests/wl_tests_nlp/test_nlp_utils.py
-          pytest --cov=./ --cov-report=xml --cov-append tests/wl_tests_nlp/test_pos_tagging.py
+          pytest --cov=./ --cov-report=xml --cov-append tests/wl_tests_nlp/wl_tests_spacy/test_spacy_eng.py
+
           pytest --cov=./ --cov-report=xml --cov-append tests/wl_tests_nlp/test_sentence_tokenization.py
-          pytest --cov=./ --cov-report=xml --cov-append tests/wl_tests_nlp/test_stop_word_lists.py
-          pytest --cov=./ --cov-report=xml --cov-append tests/wl_tests_nlp/test_syl_tokenization.py
-          pytest --cov=./ --cov-report=xml --cov-append tests/wl_tests_nlp/test_word_detokenization.py
           pytest --cov=./ --cov-report=xml --cov-append tests/wl_tests_nlp/test_word_tokenization.py
+          pytest --cov=./ --cov-report=xml --cov-append tests/wl_tests_nlp/test_syl_tokenization.py
+          pytest --cov=./ --cov-report=xml --cov-append tests/wl_tests_nlp/test_pos_tagging.py
+          pytest --cov=./ --cov-report=xml --cov-append tests/wl_tests_nlp/test_lemmatization.py
 
-          pytest --cov=./ --cov-report=xml --cov-append tests/wl_tests_file_area
-          pytest --cov=./ --cov-report=xml --cov-append tests/wl_tests_work_area
+          pytest --cov=./ --cov-report=xml --cov-append tests/wl_tests_nlp/ --ignore=tests/wl_tests_nlp/wl_tests_spacy --ignore=tests/wl_tests_nlp/test_sentence_tokenization.py --ignore=tests/wl_tests_nlp/test_word_tokenization.py --ignore=tests/wl_tests_nlp/test_syl_tokenization.py --ignore=tests/wl_tests_nlp/test_pos_tagging.py --ignore=tests/wl_tests_nlp/test_lemmatization.py
 
-          pytest --cov=./ --cov-report=xml --cov-append --ignore=tests/wl_tests_nlp --ignore=tests/wl_tests_file_area --ignore=tests/wl_tests_work_area
+          pytest --cov=./ --cov-report=xml --cov-append --ignore=tests/wl_tests_nlp
 
       # Upload coverage to Codecov
       - name: "Upload coverage to Codecov"
@@ -103,22 +99,18 @@ jobs:
       # Run tests
       - name: Run Tests
         run: |
-          pytest tests/wl_tests_nlp/test_dependency_parsing.py
-          pytest tests/wl_tests_nlp/test_lemmatization.py
-          pytest tests/wl_tests_nlp/test_matching.py
-          pytest tests/wl_tests_nlp/test_nlp_utils.py
-          pytest tests/wl_tests_nlp/test_pos_tagging.py
+          pytest tests/wl_tests_nlp/wl_tests_spacy/test_spacy_eng.py
+
           pytest tests/wl_tests_nlp/test_sentence_tokenization.py
-          pytest tests/wl_tests_nlp/test_stop_word_lists.py
-          pytest tests/wl_tests_nlp/test_syl_tokenization.py
-          pytest tests/wl_tests_nlp/test_word_detokenization.py
           pytest tests/wl_tests_nlp/test_word_tokenization.py
+          pytest tests/wl_tests_nlp/test_syl_tokenization.py
+          pytest tests/wl_tests_nlp/test_pos_tagging.py
+          pytest tests/wl_tests_nlp/test_lemmatization.py
 
-          pytest tests/wl_tests_file_area
-          # Ignore tests of Profiler due to unknown errors
-          pytest tests/wl_tests_work_area --ignore=tests/wl_tests_work_area/test_profiler.py
+          pytest tests/wl_tests_nlp/ --ignore=tests/wl_tests_nlp/wl_tests_spacy --ignore=tests/wl_tests_nlp/test_sentence_tokenization.py --ignore=tests/wl_tests_nlp/test_word_tokenization.py --ignore=tests/wl_tests_nlp/test_syl_tokenization.py --ignore=tests/wl_tests_nlp/test_pos_tagging.py --ignore=tests/wl_tests_nlp/test_lemmatization.py
 
-          pytest --ignore=tests/wl_tests_nlp --ignore=tests/wl_tests_file_area --ignore=tests/wl_tests_work_area
+          # Ignore tests of Profiler due to unknown errors
+          pytest --ignore=tests/wl_tests_nlp --ignore=tests/wl_tests_work_area/test_profiler.py
 
   # Linux
   build-linux:
@@ -153,18 +145,14 @@ jobs:
           # Fix PyQt
           export QT_QPA_PLATFORM=offscreen
 
-          pytest tests/wl_tests_nlp/test_dependency_parsing.py
-          pytest tests/wl_tests_nlp/test_lemmatization.py
-          pytest tests/wl_tests_nlp/test_matching.py
-          pytest tests/wl_tests_nlp/test_nlp_utils.py
-          pytest tests/wl_tests_nlp/test_pos_tagging.py
+          pytest tests/wl_tests_nlp/wl_tests_spacy/test_spacy_eng.py
+
           pytest tests/wl_tests_nlp/test_sentence_tokenization.py
-          pytest tests/wl_tests_nlp/test_stop_word_lists.py
-          pytest tests/wl_tests_nlp/test_syl_tokenization.py
-          pytest tests/wl_tests_nlp/test_word_detokenization.py
           pytest tests/wl_tests_nlp/test_word_tokenization.py
+          pytest tests/wl_tests_nlp/test_syl_tokenization.py
+          pytest tests/wl_tests_nlp/test_pos_tagging.py
+          pytest tests/wl_tests_nlp/test_lemmatization.py
 
-          pytest tests/wl_tests_file_area
-          pytest tests/wl_tests_work_area
+          pytest tests/wl_tests_nlp/ --ignore=tests/wl_tests_nlp/wl_tests_spacy --ignore=tests/wl_tests_nlp/test_sentence_tokenization.py --ignore=tests/wl_tests_nlp/test_word_tokenization.py --ignore=tests/wl_tests_nlp/test_syl_tokenization.py --ignore=tests/wl_tests_nlp/test_pos_tagging.py --ignore=tests/wl_tests_nlp/test_lemmatization.py
 
-          pytest --ignore=tests/wl_tests_nlp --ignore=tests/wl_tests_file_area --ignore=tests/wl_tests_work_area
+          pytest --ignore=tests/wl_tests_nlp
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -29,6 +29,7 @@
 - Work Area: Add Profiler - Readability - Bormuth's Cloze Mean / Bormuth's Grade Placement / Coleman's Readability Formula / Danielson-Bryan's Readability Formula / Degrees of Reading Power / Easy Listening Formula / Fucks's Stilcharakteristik / Strain Index / Tränkle & Bailer's Readability Formula / Tuldava's Text Difficulty / Wheeler & Smith's Readability Formula
 
 ### ✨ Improvements
+- Utils: Update spaCy's sentence recognizers, word tokenizers, part-of-speech taggers, lemmatizers, and dependency parsers
 - Utils: Update Wordless's sentence and sentence segment splitters
 - Work Area: Update Profiler - Readability - Automated Readability Index / Dale-Chall Readability Formula / Flesch Reading Ease / Flesch Reading Ease (Farr-Jenkins-Paterson) / Gunning Fog Index / Spache Grade Level
 

diff --git a/appveyor.yml b/appveyor.yml
@@ -60,21 +60,17 @@ for:
 
     # Run tests
     test_script:
-      - pytest tests/wl_tests_nlp/test_dependency_parsing.py
-      - pytest tests/wl_tests_nlp/test_lemmatization.py
-      - pytest tests/wl_tests_nlp/test_matching.py
-      - pytest tests/wl_tests_nlp/test_nlp_utils.py
-      - pytest tests/wl_tests_nlp/test_pos_tagging.py
+      - pytest tests/wl_tests_nlp/wl_tests_spacy/test_spacy_eng.py
+
       - pytest tests/wl_tests_nlp/test_sentence_tokenization.py
-      - pytest tests/wl_tests_nlp/test_stop_word_lists.py
-      - pytest tests/wl_tests_nlp/test_syl_tokenization.py
-      - pytest tests/wl_tests_nlp/test_word_detokenization.py
       - pytest tests/wl_tests_nlp/test_word_tokenization.py
+      - pytest tests/wl_tests_nlp/test_syl_tokenization.py
+      - pytest tests/wl_tests_nlp/test_pos_tagging.py
+      - pytest tests/wl_tests_nlp/test_lemmatization.py
 
-      - pytest tests/wl_tests_file_area
-      - pytest tests/wl_tests_work_area
+      - pytest tests/wl_tests_nlp/ --ignore=tests/wl_tests_nlp/wl_tests_spacy --ignore=tests/wl_tests_nlp/test_sentence_tokenization.py --ignore=tests/wl_tests_nlp/test_word_tokenization.py --ignore=tests/wl_tests_nlp/test_syl_tokenization.py --ignore=tests/wl_tests_nlp/test_pos_tagging.py --ignore=tests/wl_tests_nlp/test_lemmatization.py
 
-      - pytest --ignore=tests/wl_tests_nlp --ignore=tests/wl_tests_file_area --ignore=tests/wl_tests_work_area
+      - pytest --ignore=tests/wl_tests_nlp
 
   # macOS
   -
@@ -94,22 +90,18 @@ for:
 
     # Run tests
     test_script:
-      - pytest tests/wl_tests_nlp/test_dependency_parsing.py
-      - pytest tests/wl_tests_nlp/test_lemmatization.py
-      - pytest tests/wl_tests_nlp/test_matching.py
-      - pytest tests/wl_tests_nlp/test_nlp_utils.py
-      - pytest tests/wl_tests_nlp/test_pos_tagging.py
+      - pytest tests/wl_tests_nlp/wl_tests_spacy/test_spacy_eng.py
+
       - pytest tests/wl_tests_nlp/test_sentence_tokenization.py
-      - pytest tests/wl_tests_nlp/test_stop_word_lists.py
-      - pytest tests/wl_tests_nlp/test_syl_tokenization.py
-      - pytest tests/wl_tests_nlp/test_word_detokenization.py
       - pytest tests/wl_tests_nlp/test_word_tokenization.py
+      - pytest tests/wl_tests_nlp/test_syl_tokenization.py
+      - pytest tests/wl_tests_nlp/test_pos_tagging.py
+      - pytest tests/wl_tests_nlp/test_lemmatization.py
 
-      - pytest tests/wl_tests_file_area
-      # Ignore tests of Profiler due to unknown errors
-      - pytest tests/wl_tests_work_area --ignore=tests/wl_tests_work_area/test_profiler.py
+      - pytest tests/wl_tests_nlp/ --ignore=tests/wl_tests_nlp/wl_tests_spacy --ignore=tests/wl_tests_nlp/test_sentence_tokenization.py --ignore=tests/wl_tests_nlp/test_word_tokenization.py --ignore=tests/wl_tests_nlp/test_syl_tokenization.py --ignore=tests/wl_tests_nlp/test_pos_tagging.py --ignore=tests/wl_tests_nlp/test_lemmatization.py
 
-      - pytest --ignore=tests/wl_tests_nlp --ignore=tests/wl_tests_file_area --ignore=tests/wl_tests_work_area
+      # Ignore tests of Profiler due to unknown errors
+      - pytest --ignore=tests/wl_tests_nlp --ignore=tests/wl_tests_work_area/test_profiler.py
 
   # Ubuntu
   -
@@ -132,18 +124,14 @@ for:
       # Fix PyQt
       - export QT_QPA_PLATFORM=offscreen
 
-      - pytest tests/wl_tests_nlp/test_dependency_parsing.py
-      - pytest tests/wl_tests_nlp/test_lemmatization.py
-      - pytest tests/wl_tests_nlp/test_matching.py
-      - pytest tests/wl_tests_nlp/test_nlp_utils.py
-      - pytest tests/wl_tests_nlp/test_pos_tagging.py
+      - pytest tests/wl_tests_nlp/wl_tests_spacy/test_spacy_eng.py
+
       - pytest tests/wl_tests_nlp/test_sentence_tokenization.py
-      - pytest tests/wl_tests_nlp/test_stop_word_lists.py
-      - pytest tests/wl_tests_nlp/test_syl_tokenization.py
-      - pytest tests/wl_tests_nlp/test_word_detokenization.py
       - pytest tests/wl_tests_nlp/test_word_tokenization.py
+      - pytest tests/wl_tests_nlp/test_syl_tokenization.py
+      - pytest tests/wl_tests_nlp/test_pos_tagging.py
+      - pytest tests/wl_tests_nlp/test_lemmatization.py
 
-      - pytest tests/wl_tests_file_area
-      - pytest tests/wl_tests_work_area
+      - pytest tests/wl_tests_nlp/ --ignore=tests/wl_tests_nlp/wl_tests_spacy --ignore=tests/wl_tests_nlp/test_sentence_tokenization.py --ignore=tests/wl_tests_nlp/test_word_tokenization.py --ignore=tests/wl_tests_nlp/test_syl_tokenization.py --ignore=tests/wl_tests_nlp/test_pos_tagging.py --ignore=tests/wl_tests_nlp/test_lemmatization.py
 
-      - pytest --ignore=tests/wl_tests_nlp --ignore=tests/wl_tests_file_area --ignore=tests/wl_tests_work_area
+      - pytest --ignore=tests/wl_tests_nlp
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -54,21 +54,17 @@ jobs:
 
       # Run tests
       - script: |
-          pytest tests/wl_tests_nlp/test_dependency_parsing.py
-          pytest tests/wl_tests_nlp/test_lemmatization.py
-          pytest tests/wl_tests_nlp/test_matching.py
-          pytest tests/wl_tests_nlp/test_nlp_utils.py
-          pytest tests/wl_tests_nlp/test_pos_tagging.py
+          pytest tests/wl_tests_nlp/wl_tests_spacy/test_spacy_eng.py
+
           pytest tests/wl_tests_nlp/test_sentence_tokenization.py
-          pytest tests/wl_tests_nlp/test_stop_word_lists.py
-          pytest tests/wl_tests_nlp/test_syl_tokenization.py
-          pytest tests/wl_tests_nlp/test_word_detokenization.py
           pytest tests/wl_tests_nlp/test_word_tokenization.py
+          pytest tests/wl_tests_nlp/test_syl_tokenization.py
+          pytest tests/wl_tests_nlp/test_pos_tagging.py
+          pytest tests/wl_tests_nlp/test_lemmatization.py
 
-          pytest tests/wl_tests_file_area
-          pytest tests/wl_tests_work_area
+          pytest tests/wl_tests_nlp/ --ignore=tests/wl_tests_nlp/wl_tests_spacy --ignore=tests/wl_tests_nlp/test_sentence_tokenization.py --ignore=tests/wl_tests_nlp/test_word_tokenization.py --ignore=tests/wl_tests_nlp/test_syl_tokenization.py --ignore=tests/wl_tests_nlp/test_pos_tagging.py --ignore=tests/wl_tests_nlp/test_lemmatization.py
 
-          pytest --ignore=tests/wl_tests_nlp --ignore=tests/wl_tests_file_area --ignore=tests/wl_tests_work_area
+          pytest --ignore=tests/wl_tests_nlp
         displayName: 'Run tests'
 
   # macOS
@@ -104,22 +100,18 @@ jobs:
 
       # Run tests
       - script: |
-          pytest tests/wl_tests_nlp/test_dependency_parsing.py
-          pytest tests/wl_tests_nlp/test_lemmatization.py
-          pytest tests/wl_tests_nlp/test_matching.py
-          pytest tests/wl_tests_nlp/test_nlp_utils.py
-          pytest tests/wl_tests_nlp/test_pos_tagging.py
+          pytest tests/wl_tests_nlp/wl_tests_spacy/test_spacy_eng.py
+
           pytest tests/wl_tests_nlp/test_sentence_tokenization.py
-          pytest tests/wl_tests_nlp/test_stop_word_lists.py
-          pytest tests/wl_tests_nlp/test_syl_tokenization.py
-          pytest tests/wl_tests_nlp/test_word_detokenization.py
           pytest tests/wl_tests_nlp/test_word_tokenization.py
+          pytest tests/wl_tests_nlp/test_syl_tokenization.py
+          pytest tests/wl_tests_nlp/test_pos_tagging.py
+          pytest tests/wl_tests_nlp/test_lemmatization.py
 
-          pytest tests/wl_tests_file_area
-          # Ignore tests of Profiler due to unknown errors
-          pytest tests/wl_tests_work_area --ignore=tests/wl_tests_work_area/test_profiler.py
+          pytest tests/wl_tests_nlp/ --ignore=tests/wl_tests_nlp/wl_tests_spacy --ignore=tests/wl_tests_nlp/test_sentence_tokenization.py --ignore=tests/wl_tests_nlp/test_word_tokenization.py --ignore=tests/wl_tests_nlp/test_syl_tokenization.py --ignore=tests/wl_tests_nlp/test_pos_tagging.py --ignore=tests/wl_tests_nlp/test_lemmatization.py
 
-          pytest --ignore=tests/wl_tests_nlp --ignore=tests/wl_tests_file_area --ignore=tests/wl_tests_work_area
+          # Ignore tests of Profiler due to unknown errors
+          pytest --ignore=tests/wl_tests_nlp --ignore=tests/wl_tests_work_area/test_profiler.py
         displayName: 'Run tests'
 
   # Linux
@@ -158,19 +150,15 @@ jobs:
           # Fix PyQt
           export QT_QPA_PLATFORM=offscreen
 
-          pytest tests/wl_tests_nlp/test_dependency_parsing.py
-          pytest tests/wl_tests_nlp/test_lemmatization.py
-          pytest tests/wl_tests_nlp/test_matching.py
-          pytest tests/wl_tests_nlp/test_nlp_utils.py
-          pytest tests/wl_tests_nlp/test_pos_tagging.py
+          pytest tests/wl_tests_nlp/wl_tests_spacy/test_spacy_eng.py
+
           pytest tests/wl_tests_nlp/test_sentence_tokenization.py
-          pytest tests/wl_tests_nlp/test_stop_word_lists.py
-          pytest tests/wl_tests_nlp/test_syl_tokenization.py
-          pytest tests/wl_tests_nlp/test_word_detokenization.py
           pytest tests/wl_tests_nlp/test_word_tokenization.py
+          pytest tests/wl_tests_nlp/test_syl_tokenization.py
+          pytest tests/wl_tests_nlp/test_pos_tagging.py
+          pytest tests/wl_tests_nlp/test_lemmatization.py
 
-          pytest tests/wl_tests_file_area
-          pytest tests/wl_tests_work_area
+          pytest tests/wl_tests_nlp/ --ignore=tests/wl_tests_nlp/wl_tests_spacy --ignore=tests/wl_tests_nlp/test_sentence_tokenization.py --ignore=tests/wl_tests_nlp/test_word_tokenization.py --ignore=tests/wl_tests_nlp/test_syl_tokenization.py --ignore=tests/wl_tests_nlp/test_pos_tagging.py --ignore=tests/wl_tests_nlp/test_lemmatization.py
 
-          pytest --ignore=tests/wl_tests_nlp --ignore=tests/wl_tests_file_area --ignore=tests/wl_tests_work_area
+          pytest --ignore=tests/wl_tests_nlp
         displayName: 'Run tests'