From 4f1046d238690ec8e4ec9c621caed55a7c884d62 Mon Sep 17 00:00:00 2001 From: lucasmarchd01 Date: Thu, 13 Jun 2024 14:56:44 +0000 Subject: [PATCH 01/14] feat(cantusindex): add json-text api function --- django/cantusdb_project/cantusindex.py | 32 ++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/django/cantusdb_project/cantusindex.py b/django/cantusdb_project/cantusindex.py index b00ef8ee3..e0aa1dc91 100644 --- a/django/cantusdb_project/cantusindex.py +++ b/django/cantusdb_project/cantusindex.py @@ -7,6 +7,7 @@ from typing import Optional, Union, Callable from main_app.models import Genre import json +from requests.exceptions import SSLError, Timeout CANTUS_INDEX_DOMAIN: str = "https://cantusindex.uwaterloo.ca" DEFAULT_TIMEOUT: float = 2 # seconds @@ -167,6 +168,37 @@ def get_merged_cantus_ids() -> Optional[list]: return merge_events +def get_ci_text_search(search_term: str) -> Optional[list[Optional[dict]]]: + """Fetch data from Cantus Index for a given search term. + To do a text search on CI, use 'https://cantusindex.org/json-text/ + """ + + # We have to use the old CI domain since the API is still not available on + # cantusindex.uwaterloo.ca. Once it's available, we can use get_json_from_ci_api + # json: Union[dict, list, None] = get_json_from_ci_api(uri) + uri: str = f"https://cantusindex.org/json-text/{search_term}" + try: + response: requests.Response = requests.get( + uri, + timeout=DEFAULT_TIMEOUT, + ) + except (SSLError, Timeout, requests.HTTPError) as exc: + return None + if not response.status_code == 200: + return None + response.encoding = "utf-8-sig" + raw_text: str = response.text + text_without_bom: str = raw_text.encode().decode("utf-8-sig") + if not text_without_bom: + return None + text_search_results: list = json.loads(text_without_bom) + # if cantus index returns an empty table + if not text_search_results or not isinstance(text_search_results, list): + return None + + return text_search_results + + def get_json_from_ci_api( path: str, timeout: float = DEFAULT_TIMEOUT ) -> Union[dict, list, None]: From 89928746d5200b2461d2fd74d7c88a063c32c260 Mon Sep 17 00:00:00 2001 From: lucasmarchd01 Date: Thu, 13 Jun 2024 14:57:06 +0000 Subject: [PATCH 02/14] feat(urls): add cantus index input tool --- django/cantusdb_project/main_app/urls.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/django/cantusdb_project/main_app/urls.py b/django/cantusdb_project/main_app/urls.py index 739ea1954..05c4e47d6 100644 --- a/django/cantusdb_project/main_app/urls.py +++ b/django/cantusdb_project/main_app/urls.py @@ -18,6 +18,7 @@ ChantEditSyllabificationView, ChantSearchView, ChantSearchMSView, + CISearchView, MelodySearchView, SourceEditChantsView, ) @@ -369,6 +370,11 @@ ChantSearchMSView.as_view(), name="chant-search-ms", ), + path( + "ci-search/", + CISearchView.as_view(), + name="ci-search", + ), path( "search/", views.redirect_search, From bc9b1ec47fbbca81fe16ba34c64178414919d737 Mon Sep 17 00:00:00 2001 From: lucasmarchd01 Date: Thu, 13 Jun 2024 14:57:26 +0000 Subject: [PATCH 03/14] feat(chant create): add input tool card on sidebar --- .../main_app/templates/chant_create.html | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/django/cantusdb_project/main_app/templates/chant_create.html b/django/cantusdb_project/main_app/templates/chant_create.html index 39bb752cc..51681c8ad 100644 --- a/django/cantusdb_project/main_app/templates/chant_create.html +++ b/django/cantusdb_project/main_app/templates/chant_create.html @@ -292,7 +292,7 @@
{{ source.siglum } {% if previous_chant %} -
+
Suggestions based on previous chant:
@@ -319,12 +319,34 @@
{{ source.siglum } {{ suggestion.genre_name }} - {{ suggestion.incipit }} ({{ suggestion.occurrences }}x)
{% endfor %} {% else %} - Sorry! No suggestions found.
+ Sorry! No suggestions found. Please use the search form below.
{% endif %}
{% endif %} +
+
+
Input Tool
+
+ +
+ Enter any text from chant: + + + +
+ +
From e9c62e9cd40cf450e0eb72962f3669a84bffc3bc Mon Sep 17 00:00:00 2001 From: lucasmarchd01 Date: Thu, 13 Jun 2024 14:57:50 +0000 Subject: [PATCH 04/14] feat(ci search): add input tool search window page --- .../main_app/templates/ci_search.html | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 django/cantusdb_project/main_app/templates/ci_search.html diff --git a/django/cantusdb_project/main_app/templates/ci_search.html b/django/cantusdb_project/main_app/templates/ci_search.html new file mode 100644 index 000000000..664f70656 --- /dev/null +++ b/django/cantusdb_project/main_app/templates/ci_search.html @@ -0,0 +1,58 @@ + + + + + + + + + + +

Select the chant by clicking "OK" at the left. If your chant is not included here, please add it into + Cantus Index or contact the administrator +

+ + {# if we don't include 'style="display: table;"', the table is very narrow when viewed in certain browsers (e.g. Firefox) #} + + + + + + + + + + + {% for cantus_id,genre,full_text in results %} + + + + + + + {% endfor %} + +
SelectCantus IDGenreFulltext
{{ cantus_id }}{{ genre }}{{ full_text }}
+ + + + \ No newline at end of file From 4b6e35767ace98b111940424b1bb4bb5ae4d0130 Mon Sep 17 00:00:00 2001 From: lucasmarchd01 Date: Thu, 13 Jun 2024 14:58:31 +0000 Subject: [PATCH 05/14] feat(cisearchview): add ci search view --- .../cantusdb_project/main_app/views/chant.py | 42 ++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/django/cantusdb_project/main_app/views/chant.py b/django/cantusdb_project/main_app/views/chant.py index 7c743ee2e..7ce0b64f7 100644 --- a/django/cantusdb_project/main_app/views/chant.py +++ b/django/cantusdb_project/main_app/views/chant.py @@ -47,7 +47,11 @@ user_can_view_chant, ) -from cantusindex import get_suggested_chants, get_suggested_fulltext +from cantusindex import ( + get_suggested_chants, + get_suggested_fulltext, + get_ci_text_search, +) CHANT_SEARCH_TEMPLATE_VALUES: tuple[str, ...] = ( # for views that use chant_search.html, this allows them to @@ -869,6 +873,42 @@ def get_success_url(self): return reverse("source-edit-chants", args=[self.object.source.id]) +class CISearchView(TemplateView): + """search in CI and write results in get_context_data + Shown on the chant create page as the "Input Tool" + """ + + template_name = "ci_search.html" + + def get_context_data(self, **kwargs): + MAX_RESULTS = 500 + context = super().get_context_data(**kwargs) + context["genres"] = list( + Genre.objects.all().order_by("name").values("id", "name") + ) + search_term: str = kwargs["search_term"] + search_term: str = search_term.replace(" ", "+") # for multiple keywords + + text_search_results: Optional[list[dict]] = get_ci_text_search(search_term) + + cantus_id = [] + genre = [] + full_text = [] + + if text_search_results: + for result in text_search_results: + cantus_id.append(result["cid"]) + genre.append(result["genre"]) + full_text.append(result["fulltext"]) + + # for looping through three lists in template, we have to zip it here + if len(cantus_id) == 0: + context["results"] = [["No results", "No results", "No results"]] + else: + context["results"] = zip(cantus_id, genre, full_text) + return context + + class SourceEditChantsView(LoginRequiredMixin, UserPassesTestMixin, UpdateView): template_name = "chant_edit.html" model = Chant From f4a0a0ceaa1972b2ed5815bd3dfa0a123f6286b8 Mon Sep 17 00:00:00 2001 From: lucasmarchd01 Date: Thu, 13 Jun 2024 15:00:38 +0000 Subject: [PATCH 06/14] fix(cantusindex): fix typing and bug with json parsing --- django/cantusdb_project/cantusindex.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/django/cantusdb_project/cantusindex.py b/django/cantusdb_project/cantusindex.py index e0aa1dc91..7bd116d6f 100644 --- a/django/cantusdb_project/cantusindex.py +++ b/django/cantusdb_project/cantusindex.py @@ -133,7 +133,7 @@ def get_suggested_fulltext(cantus_id: str) -> Optional[str]: return suggested_fulltext -def get_merged_cantus_ids() -> Optional[list]: +def get_merged_cantus_ids() -> Optional[list[Optional[dict]]]: """Retrieve merged Cantus IDs from the Cantus Index API (/json-merged-chants) This function sends a request to the Cantus Index API endpoint for merged chants @@ -161,6 +161,8 @@ def get_merged_cantus_ids() -> Optional[list]: response.encoding = "utf-8-sig" raw_text: str = response.text text_without_bom: str = raw_text.encode().decode("utf-8-sig") + if not text_without_bom: + return None merge_events: list = json.loads(text_without_bom) if not isinstance(merge_events, list): From d673f508888cefdc1bbe36f36182d4360111bce4 Mon Sep 17 00:00:00 2001 From: lucasmarchd01 Date: Thu, 13 Jun 2024 15:08:39 +0000 Subject: [PATCH 07/14] fix(cantusindex): fix error handling --- django/cantusdb_project/cantusindex.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django/cantusdb_project/cantusindex.py b/django/cantusdb_project/cantusindex.py index 7bd116d6f..cfb26fc47 100644 --- a/django/cantusdb_project/cantusindex.py +++ b/django/cantusdb_project/cantusindex.py @@ -7,7 +7,7 @@ from typing import Optional, Union, Callable from main_app.models import Genre import json -from requests.exceptions import SSLError, Timeout +from requests.exceptions import SSLError, Timeout, HTTPError CANTUS_INDEX_DOMAIN: str = "https://cantusindex.uwaterloo.ca" DEFAULT_TIMEOUT: float = 2 # seconds @@ -184,7 +184,7 @@ def get_ci_text_search(search_term: str) -> Optional[list[Optional[dict]]]: uri, timeout=DEFAULT_TIMEOUT, ) - except (SSLError, Timeout, requests.HTTPError) as exc: + except (SSLError, Timeout, HTTPError): return None if not response.status_code == 200: return None From d5de8322cf7ff75a525d2e8c6727257d4b233f18 Mon Sep 17 00:00:00 2001 From: lucasmarchd01 Date: Mon, 17 Jun 2024 13:08:27 +0000 Subject: [PATCH 08/14] feat(cantusindex): add genre dal selector --- .../main_app/templates/ci_search.html | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/django/cantusdb_project/main_app/templates/ci_search.html b/django/cantusdb_project/main_app/templates/ci_search.html index 664f70656..a8e151c8d 100644 --- a/django/cantusdb_project/main_app/templates/ci_search.html +++ b/django/cantusdb_project/main_app/templates/ci_search.html @@ -38,7 +38,7 @@ + @@ -46,19 +49,17 @@ // get the option value corresponding to the genre name var genres = {{ genres|safe }}; // genres contains "id" and "name" of all Genre objects in the database var genreObj = genres.find(item => item.name === genre); - var genreId = genreId = genreObj ? genreObj.id : null; + var genreID = genreObj ? genreObj.id : null; opener.document.getElementById('id_cantus_id').value = cantus_id; - if (genreId) { - var genreSelect = opener.document.$('#id_genre'); - - if (genreSelect.find("option[value='" + genreId + "']").length) { - genreSelect.val(genreId).trigger('change'); + if (genreID) { + if (opener.$('#id_genre').find("option[value='" + genreID + "']").length) { + opener.$('#id_genre').val(genreID).trigger('change'); } else { // Create a new DOM Option and pre-select it by default - var newOption = new Option(genre, genreId, true, true); + var newOption = new Option(genre, genreID, true, true); // Append it to the select and trigger change - genreSelect.append(newOption).trigger('change'); + opener.$('#id_genre').append(newOption).trigger('change'); } } opener.document.getElementById('id_manuscript_full_text_std_spelling').value = full_text; From 971a744352a07b301e4a2a61ef0aab1944c327ad Mon Sep 17 00:00:00 2001 From: Lucas Date: Mon, 17 Jun 2024 14:21:47 +0000 Subject: [PATCH 10/14] refactor(cantusindex): rewrite docstrings and remove errant code --- django/cantusdb_project/cantusindex.py | 4 ++-- django/cantusdb_project/main_app/views/chant.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/django/cantusdb_project/cantusindex.py b/django/cantusdb_project/cantusindex.py index cfb26fc47..3bd7c6284 100644 --- a/django/cantusdb_project/cantusindex.py +++ b/django/cantusdb_project/cantusindex.py @@ -172,10 +172,10 @@ def get_merged_cantus_ids() -> Optional[list[Optional[dict]]]: def get_ci_text_search(search_term: str) -> Optional[list[Optional[dict]]]: """Fetch data from Cantus Index for a given search term. - To do a text search on CI, use 'https://cantusindex.org/json-text/ + To do a text search on CI, we use 'https://cantusindex.org/json-text/ """ - # We have to use the old CI domain since the API is still not available on + # We have to use the old CI domain since this API is still not available on # cantusindex.uwaterloo.ca. Once it's available, we can use get_json_from_ci_api # json: Union[dict, list, None] = get_json_from_ci_api(uri) uri: str = f"https://cantusindex.org/json-text/{search_term}" diff --git a/django/cantusdb_project/main_app/views/chant.py b/django/cantusdb_project/main_app/views/chant.py index 7ce0b64f7..b1825ecb4 100644 --- a/django/cantusdb_project/main_app/views/chant.py +++ b/django/cantusdb_project/main_app/views/chant.py @@ -874,14 +874,13 @@ def get_success_url(self): class CISearchView(TemplateView): - """search in CI and write results in get_context_data + """Search in CI and write results in get_context_data Shown on the chant create page as the "Input Tool" """ template_name = "ci_search.html" def get_context_data(self, **kwargs): - MAX_RESULTS = 500 context = super().get_context_data(**kwargs) context["genres"] = list( Genre.objects.all().order_by("name").values("id", "name") From f7755c291f0f56ef45410ded5b39af454e83900c Mon Sep 17 00:00:00 2001 From: Lucas Date: Mon, 17 Jun 2024 14:22:57 +0000 Subject: [PATCH 11/14] refactor(cantusindex): remove comment --- django/cantusdb_project/main_app/views/chant.py | 1 - 1 file changed, 1 deletion(-) diff --git a/django/cantusdb_project/main_app/views/chant.py b/django/cantusdb_project/main_app/views/chant.py index b1825ecb4..29e41db5d 100644 --- a/django/cantusdb_project/main_app/views/chant.py +++ b/django/cantusdb_project/main_app/views/chant.py @@ -900,7 +900,6 @@ def get_context_data(self, **kwargs): genre.append(result["genre"]) full_text.append(result["fulltext"]) - # for looping through three lists in template, we have to zip it here if len(cantus_id) == 0: context["results"] = [["No results", "No results", "No results"]] else: From 676268d7f458702ee85806a3fb4db43531e3cee3 Mon Sep 17 00:00:00 2001 From: Lucas Date: Mon, 17 Jun 2024 16:25:40 +0000 Subject: [PATCH 12/14] refactor(cantusindex): add comment for dal option selector --- django/cantusdb_project/main_app/templates/ci_search.html | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/django/cantusdb_project/main_app/templates/ci_search.html b/django/cantusdb_project/main_app/templates/ci_search.html index 8ad58281b..8ea28d1ba 100644 --- a/django/cantusdb_project/main_app/templates/ci_search.html +++ b/django/cantusdb_project/main_app/templates/ci_search.html @@ -52,13 +52,15 @@ var genreID = genreObj ? genreObj.id : null; opener.document.getElementById('id_cantus_id').value = cantus_id; + // Since we're using a django-autocomplete-light widget for the Genre selector, + // we need to follow a special process in selecting a value from the widget: + // Set the value, creating a new option if necessary if (genreID) { if (opener.$('#id_genre').find("option[value='" + genreID + "']").length) { opener.$('#id_genre').val(genreID).trigger('change'); } else { // Create a new DOM Option and pre-select it by default var newOption = new Option(genre, genreID, true, true); - // Append it to the select and trigger change opener.$('#id_genre').append(newOption).trigger('change'); } } From 71823a69337bccda8d9064c1a1f031dbb8d4a309 Mon Sep 17 00:00:00 2001 From: Lucas Date: Tue, 18 Jun 2024 13:29:03 +0000 Subject: [PATCH 13/14] fix(cantusindex): fix typing in CISearchView --- django/cantusdb_project/main_app/views/chant.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/django/cantusdb_project/main_app/views/chant.py b/django/cantusdb_project/main_app/views/chant.py index 29e41db5d..b45e7850b 100644 --- a/django/cantusdb_project/main_app/views/chant.py +++ b/django/cantusdb_project/main_app/views/chant.py @@ -888,7 +888,9 @@ def get_context_data(self, **kwargs): search_term: str = kwargs["search_term"] search_term: str = search_term.replace(" ", "+") # for multiple keywords - text_search_results: Optional[list[dict]] = get_ci_text_search(search_term) + text_search_results: Optional[list[Optional[dict]]] = get_ci_text_search( + search_term + ) cantus_id = [] genre = [] @@ -896,9 +898,10 @@ def get_context_data(self, **kwargs): if text_search_results: for result in text_search_results: - cantus_id.append(result["cid"]) - genre.append(result["genre"]) - full_text.append(result["fulltext"]) + if result: + cantus_id.append(result.get("cid", None)) + genre.append(result.get("genre", None)) + full_text.append(result.get("fulltext", None)) if len(cantus_id) == 0: context["results"] = [["No results", "No results", "No results"]] From 1b803aaf679562718b6d6f56350d97cccd75d61e Mon Sep 17 00:00:00 2001 From: Lucas Date: Tue, 18 Jun 2024 15:24:34 +0000 Subject: [PATCH 14/14] refactor(cantusindex): modify message on ci search --- django/cantusdb_project/main_app/templates/ci_search.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/django/cantusdb_project/main_app/templates/ci_search.html b/django/cantusdb_project/main_app/templates/ci_search.html index 8ea28d1ba..1dc1d6c47 100644 --- a/django/cantusdb_project/main_app/templates/ci_search.html +++ b/django/cantusdb_project/main_app/templates/ci_search.html @@ -12,7 +12,8 @@ -

Select the chant by clicking "OK" at the left. If your chant is not included here, please add it into +

Select the chant by clicking "OK" at the left. Please note that the search results are limited to the first 50 chants. + If your chant is not included here, please add it into Cantus Index or contact the administrator