diff --git a/app/public/cantusdata/management/commands/index_manuscript_mei.py b/app/public/cantusdata/management/commands/index_manuscript_mei.py index 88e7d95e..77c9f683 100644 --- a/app/public/cantusdata/management/commands/index_manuscript_mei.py +++ b/app/public/cantusdata/management/commands/index_manuscript_mei.py @@ -1,21 +1,30 @@ from typing import Any, Dict from os import path, listdir +import re from django.core.management.base import BaseCommand, CommandParser from django.conf import settings +from solr.core import SolrConnection # type: ignore + from cantusdata.helpers.mei_processing.mei_tokenizer import MEITokenizer from cantusdata.models.folio import Folio -from solr.core import SolrConnection # type: ignore MEI4_DIR = path.join("/code", "production-mei-files") +FOLIO_NUMBER_REGEX = re.compile(r"[a-zA-Z]?\d+[a-z]?") class Command(BaseCommand): help = ( "This command indexes the contents of MEI files in Solr, using" "the MEITokenizer class to extract n-grams from the MEI files." - "Files must be named in the format [some string]_[folio number].mei." + "Files must be named in the format [some string]_[folio number].mei," + "where [folio number] is an optional single letter followed by " + "some number of digits followed by an optional" + "lowercase single letter. The command currently has a workaround for folios " + "that have MEI files but are NOT in CantusDB. See #891 for details " + "about how to handle this case -- the command will alert the user " + "when it encounters this case." ) def add_arguments(self, parser: CommandParser) -> None: @@ -74,7 +83,7 @@ def handle(self, *args: Any, **options: Any) -> None: raise ValueError(f"No folios found for manuscript {manuscript_id}.") manuscript_mei_path = path.join(options["mei_dir"], str(manuscript_id)) if not path.exists(manuscript_mei_path): - raise FileNotFoundError(f"--mei-dir path does not exist.") + raise FileNotFoundError("--mei-dir path does not exist.") manuscript_mei_files = [ f for f in listdir(manuscript_mei_path) if f.endswith(".mei") ] @@ -82,10 +91,19 @@ def handle(self, *args: Any, **options: Any) -> None: raise FileNotFoundError(f"No MEI files found in {manuscript_mei_path}.") for mei_file in manuscript_mei_files: folio_number: str = mei_file.split("_")[-1].split(".")[0] - if not folio_number in folio_map: + if not FOLIO_NUMBER_REGEX.match(folio_number): raise ValueError( - f"Folio number {folio_number} in MEI file {mei_file} does not exist in the database." + f"MEI file {mei_file} does not match the expected format." + ) + if not folio_number in folio_map: + self.stdout.write( + self.style.WARNING( + f"Folio number {folio_number} in MEI file " + f"{mei_file} did not exist in the database. Creating record. " + "See #891 for details on how to handle this case." + ) ) + Folio.objects.create(manuscript_id=manuscript_id, number=folio_number) tokenizer = MEITokenizer( path.join(manuscript_mei_path, mei_file), min_ngram=options["min_ngram"], @@ -98,6 +116,7 @@ def handle(self, *args: Any, **options: Any) -> None: doc["image_uri"] = folio_map.get(folio_number, "") solr_conn.add_many(ngram_docs) solr_conn.commit() + return None def flush_manuscript_ngrams_from_index( self, solr_conn: SolrConnection, manuscript_id: int diff --git a/app/public/cantusdata/test/core/helpers/mei_processing/test_mei_files/123723/cdn-hsmu-m2149l4_999r.mei b/app/public/cantusdata/test/core/helpers/mei_processing/test_mei_files/123723/cdn-hsmu-m2149l4_999r.mei new file mode 100644 index 00000000..1a1d7b5a --- /dev/null +++ b/app/public/cantusdata/test/core/helpers/mei_processing/test_mei_files/123723/cdn-hsmu-m2149l4_999r.mei @@ -0,0 +1,1150 @@ + + + + + + + + MEI Encoding Output (1.0.0) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + Ec + + + + + + ce + + + + + + + no + + + + + + men + + + + + + do + + + + + + mi + + + + + + + ni + + + + + + ve + + + + + + + + + + + nit + + + + + + + + + + + de + + + + + + lon + + + + + + + gin + + + + + + + + + + + + + quo + + + + + + + et + + + + + + cla + + + + + + ri + + + + + + tas + + + + + + e + + + + + + + + + + jus + + + + + + + re + + + + + + + plet + + + + + + or + + + + + + + bem + + + + + + ter + + + + + + + ra + + + + + + rum + + + + + + E + + + + + + u + + + + + + + + + o + + + + + + u + + + + + + + a + + + + + + e + + + + + Quem + + + + + + ter + + + + + + + ra + + + + + + + pon + + + + + + thus + + + + + + + + + et + + + + + + + eth + + + + + + + + + + ra + + + col + + + + + + lunt + + + + + + a + + + + + + do + + + + + + + rant + + + + + + prae + + + + + + di + + + + + + + cant + + + + + + tri + + + + + + + + + nam + + + + + + re + + + + + + + gen + + + + + + + tem + + + + + + ma + + + + + + + chi + + + + + + + + nam + + + + + + claus + + + + + + trum + + + + + + ma + + + + + + + + + + + ri + + + e + + + + + + + + + ba + + + + + + + + iu + + + + + + + lat + + + + + + Cui + + + + + + + lu + + + + + + + na + + + + + + + sol + + + + + + et + + + + + + sy + + + + + + + + + + + de + + + + + + ra + + + de + + + + + + ser + + + + + + vi + + + + + + + + + unt + + + + + + + per + + + + + + tem + + + + + + po + + + + + + + ra + + + + + + per + + + + + + fu + + + + + + sa + + + + + + + ce + + + + + + + li + + + + + + gra + + + + + + + + + + + + ti + + + a + + + + + + ges + + + + + + tant + + + + + + + + + pu + + + + + + + + + + + el + + + le + + + + + + + + + + + vis + + + + + + + ce + + + ra + + + + + + Be + + + + + + + + + + a + + + ta + + + + + + + ma + + + + + + ter + + + + + + mu + + + + + + + + + + + ne + + + + + + re + + + + + + cu + + + + + + ius + + + + + + su + + + + + + per + + + + + + + nus + + + + + + ar + + + + + + ti + + + + + + + fex + + + + + + mun + + + + + + dum + + + + + + pu + + + + + + + gil + + + + + + + + +
+ + + + + \ No newline at end of file diff --git a/app/public/cantusdata/test/core/management/commands/test_index_manuscript_mei.py b/app/public/cantusdata/test/core/management/commands/test_index_manuscript_mei.py index 9c5b9fbd..f2f12e5f 100644 --- a/app/public/cantusdata/test/core/management/commands/test_index_manuscript_mei.py +++ b/app/public/cantusdata/test/core/management/commands/test_index_manuscript_mei.py @@ -27,6 +27,10 @@ def setUpTestData(cls) -> None: Folio.objects.create(number="001v", manuscript=manuscript) def test_index_manuscript_mei(self) -> None: + # Assert that prior to the command run, the folio "999r" does not + # exist in the database + with self.assertRaises(Folio.DoesNotExist): + Folio.objects.get(manuscript_id=123723, number="999r") call_command( "index_manuscript_mei", "123723", @@ -37,6 +41,10 @@ def test_index_manuscript_mei(self) -> None: "--mei-dir", TEST_MEI_FILES_PATH, ) + # Assert that the folio "999r" now exists in the database + # (will raise exception if it does not) + with self.subTest("Test creation of non-existent folio"): + Folio.objects.get(manuscript_id=123723, number="999r") results = self.solr_conn.query("*:*", fq="type:omr_ngram") with self.subTest("Test total number of indexed documents"): total_exp_ngrams_001r = calculate_expected_total_ngrams( @@ -45,8 +53,12 @@ def test_index_manuscript_mei(self) -> None: total_exp_ngrams_001v = calculate_expected_total_ngrams( f"{TEST_MEI_FILES_PATH}/123723/cdn-hsmu-m2149l4_001v.mei", 1, 5 ) + total_exp_ngrams_999r = calculate_expected_total_ngrams( + f"{TEST_MEI_FILES_PATH}/123723/cdn-hsmu-m2149l4_999r.mei", 1, 5 + ) self.assertEqual( - results.numFound, total_exp_ngrams_001r + total_exp_ngrams_001v + results.numFound, + total_exp_ngrams_001r + total_exp_ngrams_001v + total_exp_ngrams_999r, ) def test_flush_option(self) -> None: