Merge pull request #1500 from DDMAL/staging

Merge staging into production

.github/workflows/django_tests.yml

@@ -0,0 +1,29 @@
+name: django-tests
+on:
+    pull_request:
+        types: [opened, synchronize]
+jobs:
+    run-tests:
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v3
+            - run: mkdir -p config/envs
+            - uses: SpicyPizza/[email protected]
+              with:
+                envkey_POSTGRES_DB: test_cantusdb
+                envkey_POSTGRES_USER: test_user
+                envkey_POSTGRES_HOST: postgres
+                envkey_POSTGRES_PORT: 5432
+                envkey_PROJECT_ENVIRONMENT: PRODUCTION
+                envkey_CANTUSDB_STATIC_ROOT: /path/to/static
+                envkey_CANTUSDB_MEDIA_ROOT: /path/to/media
+                envkey_CANTUSDB_HOST: somehost
+                envkey_CANTUSDB_SECRET_KEY: "hereisakey1234"
+                envkey_POSTGRES_PASSWORD: woahagreatpasswordabc
+                envkey_AWS_EMAIL_HOST_USER: test_user
+                envkey_AWS_EMAIL_HOST_PASSWORD: test_password
+                directory: config/envs
+                file_name: dev_env
+            - run: docker compose -f docker-compose-development.yml build
+            - run: docker compose -f docker-compose-development.yml up -d
+            - run: docker compose -f docker-compose-development.yml exec -T django python manage.py test main_app.tests

config/nginx/conf.d/cantusdb.conf.development

@@ -10,6 +10,7 @@ server {
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
         proxy_set_header Host $host;
         proxy_redirect off;
+        proxy_intercept_errors on;
     }
 
     location /static {
@@ -23,26 +24,11 @@ server {
         alias /resources/api_cache/concordances.json;
         expires modified +24h;
     }
-
-    location = /style.css {
-        root /;
-    }
-    location = /background.jpg {
-        root /;
-    }
-    location = /CantusLogoSmall.gif {
-        root /;
-    }
-    location = /favicon.ico {
-        root /;
-    }
 
-    error_page 502 /502.html;
-    location = /502.html {
-        root /;
-    }
-    error_page 504 /504.html;
-    location = /504.html {
+    error_page 500 /error_pages/500.html;
+    error_page 502 /error_pages/502.html;
+    error_page 504 /error_pages/504.html;
+    location /error_pages {
         root /;
     }
-}
+}

cron/management/manage.sh

@@ -7,4 +7,4 @@
 DOCKER_COMPOSE_FILE=$1                                              # This is the path to the docker-compose file.
 COMMAND=$2                                                          # This is the command to execute.
 
-/usr/local/bin/docker-compose -f $DOCKER_COMPOSE_FILE exec -T django python manage.py $COMMAND
+/usr/bin/docker compose -f $DOCKER_COMPOSE_FILE exec -T django python manage.py $COMMAND

cron/postgres/db_backup.sh

@@ -20,9 +20,9 @@ BACKUP_FILENAME=$(date "+%Y-%m-%dT%H:%M:%S").sql.gz                 # This is th
 
 # Create the backup and copy it to the daily backup directory
 mkdir -p $BACKUP_DIR/daily $BACKUP_DIR/weekly $BACKUP_DIR/monthly $BACKUP_DIR/yearly
-/usr/bin/docker exec cantusdb_postgres_1 /usr/local/bin/postgres_backup.sh $BACKUP_FILENAME
-/usr/bin/docker cp cantusdb_postgres_1:/var/lib/postgresql/backups/$BACKUP_FILENAME $BACKUP_DIR/daily
-/usr/bin/docker exec cantusdb_postgres_1 rm /var/lib/postgresql/backups/$BACKUP_FILENAME
+/usr/bin/docker exec cantusdb-postgres-1 /usr/local/bin/postgres_backup.sh $BACKUP_FILENAME
+/usr/bin/docker cp cantusdb-postgres-1:/var/lib/postgresql/backups/$BACKUP_FILENAME $BACKUP_DIR/daily
+/usr/bin/docker exec cantusdb-postgres-1 rm /var/lib/postgresql/backups/$BACKUP_FILENAME
 
 # Manage retention of daily backups
 FILES_TO_REMOVE=$(ls -td $BACKUP_DIR/daily/* | tail -n +8)

django/cantusdb_project/articles/templates/article_detail.html

@@ -1,40 +1,39 @@
-{% extends "base.html" %}
+{% extends "base_page_with_side_cards.html" %}
 {% load helper_tags %} {# for recent_articles #}
-{% block content %}
-<div class="container">
-    <title>{{ article.title }} | Cantus Manuscript Database</title>
+{% block title %}
+<title>{{ article.title }} | Cantus Database</title>
+{% endblock %}
+
+{% block uppersidebar %}
+    <div class="search-bar mb-3">
+        {% include "global_search_bar.html" %}
+    </div>
+{% endblock %}
+
+{% block maincontent %}
+    <h3>
+        {{ article.title }}
+    </h3>
     <div class="row">
-        <div class="p-3 col-lg-8 bg-white rounded main-content">
-            <h3>
-                {{ article.title }}
-            </h3>
-            <div class="row">
-                <div class="col">
-                    <div class="container">
-                        <div class="container text-wrap">
-                            <small>Submitted by <a href="{% url 'user-detail' article.author.id %}">{{ article.author }}</a> on {{ article.date_created|date:"D, m/d/Y - H:i" }}</small>
-                            <div style="padding-top: 1em;">
-                                {{ article.body.html|safe }}
-                            </div>
-                        </div>
+        <div class="col">
+            <div class="container">
+                <div class="container text-wrap">
+                    <small>Submitted by <a href="{% url 'user-detail' article.author.id %}">{{ article.author }}</a> on {{ article.date_created|date:"D, m/d/Y - H:i" }}</small>
+                    <div style="padding-top: 1em;">
+                        {{ article.body.html|safe }}
                     </div>
                 </div>
             </div>
         </div>
-        <div class="col p-0 sidebar">
-            <div class="search-bar mb-3">
-                {% include "global_search_bar.html" %}
-            </div>
-            <div class="card mt-3 w-100">
-                <div class="card-header">
-                    What's New
-                </div>
-                <div class="card-body">
-                    {% recent_articles %}
-                </div>
-            </div>
+    </div>
+{% endblock %}
+{% block lowersidebar %}
+    <div class="card">
+        <div class="card-header">
+            What's New
+        </div>
+        <div class="card-body">
+            {% recent_articles %}
         </div>
     </div>
-
-</div>
 {% endblock %}

django/cantusdb_project/articles/templates/article_list.html

@@ -1,43 +1,39 @@
-{% extends "base.html" %}
-{% load helper_tags %} {# for recent_articles #}
-{% block content %}
-<div class="container">
-    <title>What's New | Cantus Manuscript Database</title>
+{% extends "base_page_with_side_cards.html" %}
+{% load helper_tags %} {# for recent_articles #}\
+{% block title %}
+<title>What's New | Cantus Database</title>
+{% endblock %}
+{% block uppersidebar %}
+<div class="search-bar mb-3">
+    {% include "global_search_bar.html" %}
+</div>
+{% endblock %}
+{% block maincontent %}
+    <h3>What's New</h3>
+    {% for article in articles %}
     <div class="row">
-        <div class="p-3 col-lg-8 bg-white rounded main-content">
-            <h3>What's New</h3>
-            {% for article in articles %}
-            <div class="row">
-                <div class="col">
-                    <small>{{ article.date_created|date:"D, m/d/Y - H:i" }}</small>
-                    <h4>
-                        <a href="{% url 'article-detail' article.id %}">{{ article.title }}</a>
-                    </h4>
-                    <div class="container">
-                        <small>
-                        {{ article.body.html|safe|truncatechars_html:3000 }}
-                        </small>
-                    </div>
-                </div>
+        <div class="col">
+            <small>{{ article.date_created|date:"D, m/d/Y - H:i" }}</small>
+            <h4>
+                <a href="{% url 'article-detail' article.id %}">{{ article.title }}</a>
+            </h4>
+            <div class="container">
+                <small>
+                {{ article.body.html|safe|truncatechars_html:3000 }}
+                </small>
             </div>
-            {% endfor %}
-            {% include "pagination.html" %}
-            <br>
         </div>
-        <div class="col p-0 sidebar">
-            <div class="search-bar mb-3">
-                {% include "global_search_bar.html" %}
-            </div>
-
-            <div class="card mt-3 w-100">
-                <div class="card-header">
-                    What's New
-                </div>
-                <div class="card-body">
-                    {% recent_articles %}
-                </div>
-            </div>
+    </div>
+    {% endfor %}
+    {% include "pagination.html" %}
+{% endblock %}
+{% block lowersidebar %}
+    <div class="card">
+        <div class="card-header">
+            What's New
+        </div>
+        <div class="card-body">
+            {% recent_articles %}
         </div>
     </div>
-</div>
 {% endblock %}

django/cantusdb_project/cantusdb/settings.py

@@ -197,6 +197,9 @@
         "debug_toolbar.panels.redirects.RedirectsPanel",
         "debug_toolbar.panels.profiling.ProfilingPanel",
     },
+    "SHOW_TOOLBAR_CALLBACK": lambda request: (
+        False if request.headers.get("x-requested-with") == "XMLHttpRequest" else True
+    ),
 }
 
 INTERNAL_IPS = [

django/cantusdb_project/cantusindex.py

@@ -6,8 +6,11 @@
 import requests
 from typing import Optional, Union, Callable
 from main_app.models import Genre
+import json
+from requests.exceptions import SSLError, Timeout, HTTPError
 
 CANTUS_INDEX_DOMAIN: str = "https://cantusindex.uwaterloo.ca"
+OLD_CANTUS_INDEX_DOMAIN: str = "https://cantusindex.org"
 DEFAULT_TIMEOUT: float = 2  # seconds
 NUMBER_OF_SUGGESTED_CHANTS: int = 3  # this number can't be too large,
 # since for each suggested chant, we make a request to Cantus Index.
@@ -90,9 +93,12 @@ def get_suggested_chant(
         # mostly, in case of a timeout within get_json_from_ci_api
         return None
 
-    fulltext: str = json["info"]["field_full_text"]
-    incipit: str = " ".join(fulltext.split(" ")[:5])
-    genre_name: str = json["info"]["field_genre"]
+    try:
+        fulltext: str = json["info"]["field_full_text"]
+        incipit: str = " ".join(fulltext.split(" ")[:5])
+        genre_name: str = json["info"]["field_genre"]
+    except TypeError:
+        return None
     genre_id: Optional[int] = None
     try:
         genre_id = Genre.objects.get(name=genre_name).id
@@ -112,7 +118,7 @@ def get_suggested_chant(
     }
 
 
-def get_suggested_fulltext(cantus_id: str) -> str:
+def get_suggested_fulltext(cantus_id: str) -> Optional[str]:
     endpoint_path: str = f"/json-cid/{cantus_id}"
     json: Union[dict, list, None] = get_json_from_ci_api(endpoint_path)
 
@@ -128,6 +134,75 @@ def get_suggested_fulltext(cantus_id: str) -> str:
     return suggested_fulltext
 
 
+def get_merged_cantus_ids() -> Optional[list[Optional[dict]]]:
+    """Retrieve merged Cantus IDs from the Cantus Index API (/json-merged-chants)
+
+    This function sends a request to the Cantus Index API endpoint for merged chants
+    and retrieves the response. The response is expected to be a list of dictionaries,
+    each containing information about a merged Cantus ID, including the old Cantus ID,
+    the new Cantus ID, and the date of the merge.
+
+    Returns:
+        Optional[list]: A list of dictionaries representing merged chant information,
+    or None if there was an error retrieving the data or the response format is invalid.
+
+    """
+    endpoint_path: str = "/json-merged-chants"
+
+    # We have to use the old CI domain since the API is still not available on
+    # cantusindex.uwaterloo.ca. Once it's available, we can use get_json_from_ci_api
+    # json: Union[dict, list, None] = get_json_from_ci_api(endpoint_path)
+    uri: str = f"{OLD_CANTUS_INDEX_DOMAIN}{endpoint_path}"
+    try:
+        response: requests.Response = requests.get(uri, timeout=DEFAULT_TIMEOUT)
+    except (SSLError, Timeout, HTTPError):
+        return None
+    if not response.status_code == 200:
+        return None
+    response.encoding = "utf-8-sig"
+    raw_text: str = response.text
+    text_without_bom: str = raw_text.encode().decode("utf-8-sig")
+    if not text_without_bom:
+        return None
+    merge_events: list = json.loads(text_without_bom)
+
+    if not isinstance(merge_events, list):
+        return None
+    return merge_events
+
+
+def get_ci_text_search(search_term: str) -> Optional[list[Optional[dict]]]:
+    """Fetch data from Cantus Index for a given search term.
+    To do a text search on CI, we use 'https://cantusindex.org/json-text/<text to search>
+    """
+
+    # We have to use the old CI domain since this API is still not available on
+    # cantusindex.uwaterloo.ca. Once it's available, we can use get_json_from_ci_api
+    # json: Union[dict, list, None] = get_json_from_ci_api(uri)
+    endpoint_path: str = f"/json-text/{search_term}"
+    uri: str = f"{OLD_CANTUS_INDEX_DOMAIN}{endpoint_path}"
+    try:
+        response: requests.Response = requests.get(
+            uri,
+            timeout=DEFAULT_TIMEOUT,
+        )
+    except (SSLError, Timeout, HTTPError):
+        return None
+    if not response.status_code == 200:
+        return None
+    response.encoding = "utf-8-sig"
+    raw_text: str = response.text
+    text_without_bom: str = raw_text.encode().decode("utf-8-sig")
+    if not text_without_bom:
+        return None
+    text_search_results: list = json.loads(text_without_bom)
+    # if cantus index returns an empty table
+    if not text_search_results or not isinstance(text_search_results, list):
+        return None
+
+    return text_search_results
+
+
 def get_json_from_ci_api(
     path: str, timeout: float = DEFAULT_TIMEOUT
 ) -> Union[dict, list, None]: