Merge pull request #1538 from lucasmarchd01/issue-1391

Re-implement cantus index input tool

django/cantusdb_project/cantusindex.py

@@ -7,6 +7,7 @@
 from typing import Optional, Union, Callable
 from main_app.models import Genre
 import json
+from requests.exceptions import SSLError, Timeout, HTTPError
 
 CANTUS_INDEX_DOMAIN: str = "https://cantusindex.uwaterloo.ca"
 DEFAULT_TIMEOUT: float = 2  # seconds
@@ -132,7 +133,7 @@ def get_suggested_fulltext(cantus_id: str) -> Optional[str]:
     return suggested_fulltext
 
 
-def get_merged_cantus_ids() -> Optional[list]:
+def get_merged_cantus_ids() -> Optional[list[Optional[dict]]]:
     """Retrieve merged Cantus IDs from the Cantus Index API (/json-merged-chants)
 
     This function sends a request to the Cantus Index API endpoint for merged chants
@@ -160,13 +161,46 @@ def get_merged_cantus_ids() -> Optional[list]:
     response.encoding = "utf-8-sig"
     raw_text: str = response.text
     text_without_bom: str = raw_text.encode().decode("utf-8-sig")
+    if not text_without_bom:
+        return None
     merge_events: list = json.loads(text_without_bom)
 
     if not isinstance(merge_events, list):
         return None
     return merge_events
 
 
+def get_ci_text_search(search_term: str) -> Optional[list[Optional[dict]]]:
+    """Fetch data from Cantus Index for a given search term.
+    To do a text search on CI, we use 'https://cantusindex.org/json-text/<text to search>
+    """
+
+    # We have to use the old CI domain since this API is still not available on
+    # cantusindex.uwaterloo.ca. Once it's available, we can use get_json_from_ci_api
+    # json: Union[dict, list, None] = get_json_from_ci_api(uri)
+    uri: str = f"https://cantusindex.org/json-text/{search_term}"
+    try:
+        response: requests.Response = requests.get(
+            uri,
+            timeout=DEFAULT_TIMEOUT,
+        )
+    except (SSLError, Timeout, HTTPError):
+        return None
+    if not response.status_code == 200:
+        return None
+    response.encoding = "utf-8-sig"
+    raw_text: str = response.text
+    text_without_bom: str = raw_text.encode().decode("utf-8-sig")
+    if not text_without_bom:
+        return None
+    text_search_results: list = json.loads(text_without_bom)
+    # if cantus index returns an empty table
+    if not text_search_results or not isinstance(text_search_results, list):
+        return None
+
+    return text_search_results
+
+
 def get_json_from_ci_api(
     path: str, timeout: float = DEFAULT_TIMEOUT
 ) -> Union[dict, list, None]:

django/cantusdb_project/main_app/admin/institution_identifier.py

@@ -6,5 +6,5 @@
 
 @admin.register(InstitutionIdentifier)
 class InstitutionIdentifierAdmin(BaseModelAdmin):
-    list_display = ('identifier', 'identifier_type')
+    list_display = ("identifier", "identifier_type")
     raw_id_fields = ("institution",)

django/cantusdb_project/main_app/permissions.py

@@ -162,4 +162,8 @@ def user_can_manage_source_editors(user: User) -> bool:
     Checks if the user has permission to change the editors assigned to a Source.
     Used in SourceDetailView.
     """
-    return user.is_superuser or user.is_staff or user.groups.filter(name="project manager").exists()
+    return (
+        user.is_superuser
+        or user.is_staff
+        or user.groups.filter(name="project manager").exists()
+    )

django/cantusdb_project/main_app/templates/chant_create.html

@@ -295,38 +295,63 @@ <h5><a id="source" href="{% url 'source-detail' source.id %}">{{ source.siglum }
         </div>
     </div>
 
-    {% if previous_chant %}
-        <div class="card w-100">
-            <div class="card-body">
-                <small>
-                    <b>Suggestions based on previous chant:</b><br>
-                    <div>
-                        {{ previous_chant.folio }} {{ previous_chant.c_sequence}} <a href="{% url 'chant-detail' previous_chant.id %}" target="_blank">{{ previous_chant.incipit }}</a><br>
-                        {% if previous_chant.cantus_id %}
-                            (Cantus ID: <b><a href="https://cantusindex.org/id/{{ previous_chant.cantus_id }}" target="_blank">{{ previous_chant.cantus_id }}</a></b>)
+            {% if previous_chant %}
+                <div class="card w-100 mb-3">
+                    <div class="card-body">
+                        <small>
+                            <b>Suggestions based on previous chant:</b><br>
+                            <div>
+                                {{ previous_chant.folio }} {{ previous_chant.c_sequence}} <a href="{% url 'chant-detail' previous_chant.id %}" target="_blank">{{ previous_chant.incipit }}</a><br>
+                                {% if previous_chant.cantus_id %}
+                                    (Cantus ID: <b><a href="https://cantusindex.org/id/{{ previous_chant.cantus_id }}" target="_blank">{{ previous_chant.cantus_id }}</a></b>)
+                                {% endif %}
+                            </div>
+                        {% if suggested_chants %}
+                            {% for suggestion in suggested_chants %}
+                                <input
+                                    type="button"
+                                    style="width: 80px"
+                                    value="{{ suggestion.cantus_id }}"
+                                    title="{{ suggestion.cantus_id }}"
+                                    onclick='autoFillSuggestedChant(
+                                        "{{ suggestion.genre_name }}",
+                                        {{ suggestion.genre_id | default_if_none:"null" }},
+                                        "{{ suggestion.cantus_id }}",
+                                        "{{ suggestion.fulltext }}"
+                                    )'
+                                >
+                                <strong>{{ suggestion.genre_name }}</strong> - <span title="{{ suggestion.fulltext }}">{{ suggestion.incipit }}</span> (<strong>{{ suggestion.occurrences }}x</strong>)<br>
+                            {% endfor %}
+                        {% else %}
+                            Sorry! No suggestions found. Please use the search form below.<br>
                         {% endif %}
+                        </small>
                     </div>
-                {% if suggested_chants %}
-                    {% for suggestion in suggested_chants %}
-                        <input
-                            type="button"
-                            style="width: 80px"
-                            value="{{ suggestion.cantus_id }}"
-                            title="{{ suggestion.cantus_id }}"
-                            onclick='autoFillSuggestedChant(
-                                "{{ suggestion.genre_name }}",
-                                {{ suggestion.genre_id | default_if_none:"null" }},
-                                "{{ suggestion.cantus_id }}",
-                                "{{ suggestion.fulltext }}"
-                            )'
-                        >
-                        <strong>{{ suggestion.genre_name }}</strong> - <span title="{{ suggestion.fulltext }}">{{ suggestion.incipit }}</span> (<strong>{{ suggestion.occurrences }}x</strong>)<br>
-                    {% endfor %}
-                {% else %}
-                    Sorry! No suggestions found.<br>
-                {% endif %}
-                </small>
+                </div>
+            {% endif %}
+            <div class="card w-100">
+                <div class="card-header">
+                    <h5>Input Tool</h5>
+                </div>
+
+                <div class=" card-body" style="font-size: 15px">
+                    Enter any text from chant:
+                    <input type="text" id="incipitsearch" />
+                    <button class="btn btn-danger btn-sm" onclick="openCiSearchWin()">Search ID</button>
+                    <script>
+                        function openCiSearchWin() {
+                            search_term = document.getElementById("incipitsearch").value;
+                            if (search_term.length < 3) {
+                                search_term = window.prompt("Enter at least 3 characters of incipit", "");
+                            }
+                            address = "/ci-search/" + search_term;
+                            var ci = window.open(address, "ci", "height=400px, width=1200px, top=0, left=50px, status=no, toolbar=no, location=no, scrollbars=yes, resizable=yes");
+                        }
+                    </script>
+                </div>
+
             </div>
         </div>
-    {% endif %}
+    </div>
+</div>
 {% endblock %}

django/cantusdb_project/main_app/templates/ci_search.html

@@ -0,0 +1,73 @@
+<!DOCTYPE html>
+{% load static %}
+<script type="text/javascript" src="{% static 'admin/js/vendor/jquery/jquery.js' %}"></script>
+
+<html>
+
+<head>
+    <!-- Bootstrap CSS -->
+    <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.5.0/css/bootstrap.min.css"
+        integrity="sha384-9aIt2nRpC12Uk9gS9baDl411NQApFmC26EwAOH8WgZl5MYYxFfc+NcPb1dKGj7Sk" crossorigin="anonymous">
+</head>
+
+<body>
+
+    <p>Select the chant by clicking "OK" at the left. Please note that the search results are limited to the first 50 chants.
+        If your chant is not included here, please add it into
+        <a href="https://cantusindex.org/">Cantus Index</a> or contact the <a
+            href="mailto:[email protected]">administrator</a>
+    </p>
+    <table class="table table-responsive table-sm small table-bordered table-striped table-hover" style="display: table;">
+        {# if we don't include 'style="display: table;"', the table is very narrow when viewed in certain browsers (e.g. Firefox) #}
+        <thead class="thead-dark">
+            <tr>
+                <th width="50">Select</th>
+                <th width="80">Cantus ID</th>
+                <th width="80">Genre</th>
+                <th width="100">Fulltext</th>
+            </tr>
+        </thead>
+
+        <tbody>
+            {% for cantus_id,genre,full_text in results %}
+            <tr id="{{ cantus_id }}">
+                <td><input type="button" value="OK" onclick="autoFill()" /></td>
+                <td>{{ cantus_id }}</td>
+                <td>{{ genre }}</td>
+                <td>{{ full_text }}</td>
+            </tr>
+            {% endfor %}
+        </tbody>
+    </table>
+
+    <script>
+        function autoFill() {
+            var rowId = event.target.parentNode.parentNode.id; //this gives id of whose button was clicked 
+            var cantus_id = document.getElementById(rowId).cells[1].innerText;
+            var genre = document.getElementById(rowId).cells[2].innerText;
+            var full_text = document.getElementById(rowId).cells[3].innerText;
+
+            // get the option value corresponding to the genre name
+            var genres = {{ genres|safe }}; // genres contains "id" and "name" of all Genre objects in the database
+            var genreObj = genres.find(item => item.name === genre);
+            var genreID = genreObj ? genreObj.id : null;
+
+            opener.document.getElementById('id_cantus_id').value = cantus_id;
+            // Since we're using a django-autocomplete-light widget for the Genre selector,
+            // we need to follow a special process in selecting a value from the widget:
+            // Set the value, creating a new option if necessary            
+            if (genreID) {              
+                if (opener.$('#id_genre').find("option[value='" + genreID + "']").length) {
+                    opener.$('#id_genre').val(genreID).trigger('change');
+                } else { 
+                    // Create a new DOM Option and pre-select it by default
+                    var newOption = new Option(genre, genreID, true, true);
+                    opener.$('#id_genre').append(newOption).trigger('change');
+                }
+            }
+            opener.document.getElementById('id_manuscript_full_text_std_spelling').value = full_text;
+            close()
+        }
+    </script>
+</body>
+</html>

django/cantusdb_project/main_app/tests/test_views.py

@@ -4963,7 +4963,9 @@ def test_dd_column(self):
         response = self.client.get(reverse("source-inventory", args=[source.id]))
         html: str = str(response.content)
         self.assertIn(diff_id, html)
-        expected_html_substring: str = f'<a href="https://differentiaedatabase.ca/differentia/{diff_id}" target="_blank">'
+        expected_html_substring: str = (
+            f'<a href="https://differentiaedatabase.ca/differentia/{diff_id}" target="_blank">'
+        )
         self.assertIn(expected_html_substring, html)
 
     def test_redirect_with_source_parameter(self):

django/cantusdb_project/main_app/urls.py

@@ -18,6 +18,7 @@
     ChantEditSyllabificationView,
     ChantSearchView,
     ChantSearchMSView,
+    CISearchView,
     MelodySearchView,
     SourceEditChantsView,
 )
@@ -368,6 +369,11 @@
         ChantSearchMSView.as_view(),
         name="chant-search-ms",
     ),
+    path(
+        "ci-search/<str:search_term>",
+        CISearchView.as_view(),
+        name="ci-search",
+    ),
     path(
         "search/",
         views.redirect_search,

django/cantusdb_project/main_app/views/chant.py

@@ -47,7 +47,11 @@
     user_can_view_chant,
 )
 
-from cantusindex import get_suggested_chants, get_suggested_fulltext
+from cantusindex import (
+    get_suggested_chants,
+    get_suggested_fulltext,
+    get_ci_text_search,
+)
 
 CHANT_SEARCH_TEMPLATE_VALUES: tuple[str, ...] = (
     # for views that use chant_search.html, this allows them to
@@ -869,6 +873,43 @@ def get_success_url(self):
         return reverse("source-edit-chants", args=[self.object.source.id])
 
 
+class CISearchView(TemplateView):
+    """Search in CI and write results in get_context_data
+    Shown on the chant create page as the "Input Tool"
+    """
+
+    template_name = "ci_search.html"
+
+    def get_context_data(self, **kwargs):
+        context = super().get_context_data(**kwargs)
+        context["genres"] = list(
+            Genre.objects.all().order_by("name").values("id", "name")
+        )
+        search_term: str = kwargs["search_term"]
+        search_term: str = search_term.replace(" ", "+")  # for multiple keywords
+
+        text_search_results: Optional[list[Optional[dict]]] = get_ci_text_search(
+            search_term
+        )
+
+        cantus_id = []
+        genre = []
+        full_text = []
+
+        if text_search_results:
+            for result in text_search_results:
+                if result:
+                    cantus_id.append(result.get("cid", None))
+                    genre.append(result.get("genre", None))
+                    full_text.append(result.get("fulltext", None))
+
+        if len(cantus_id) == 0:
+            context["results"] = [["No results", "No results", "No results"]]
+        else:
+            context["results"] = zip(cantus_id, genre, full_text)
+        return context
+
+
 class SourceEditChantsView(LoginRequiredMixin, UserPassesTestMixin, UpdateView):
     template_name = "chant_edit.html"
     model = Chant