Merge pull request #1622 from DDMAL/develop

.github/workflows/broken-link-checker.yaml

@@ -31,7 +31,7 @@ jobs:
         id: lychee
         uses: lycheeverse/[email protected]
         with:
-          args: --exclude http:\/\/cantus\.sk.* ${{ matrix.links }}
+          args: --exclude http:\/\/cantus\.sk.* --exclude https:\/\/us06web\.zoom\.us* ${{ matrix.links }}
           format: json
           output: /tmp/link-checker-output.txt
       - name: Curating Link Checker Output

config/nginx/conf.d/cantusdb.conf.development

@@ -10,7 +10,7 @@ server {
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
         proxy_set_header Host $host;
         proxy_redirect off;
-        proxy_intercept_errors on;
+        proxy_intercept_errors off;
     }
 
     location /static {

cron/postgres/db_backup.sh

@@ -19,7 +19,7 @@ BACKUP_FILENAME=$(date "+%Y-%m-%dT%H:%M:%S").sql.gz                 # This is th
 
 
 # Create the backup and copy it to the daily backup directory
-mkdir -p $BACKUP_DIR/daily $BACKUP_DIR/weekly $BACKUP_DIR/monthly $BACKUP_DIR/yearly
+mkdir -p $BACKUP_DIR/daily $BACKUP_DIR/weekly $BACKUP_DIR/monthly $BACKUP_DIR/yearly $BACKUP_DIR/rism
 /usr/bin/docker exec cantusdb-postgres-1 /usr/local/bin/postgres_backup.sh $BACKUP_FILENAME
 /usr/bin/docker cp cantusdb-postgres-1:/var/lib/postgresql/backups/$BACKUP_FILENAME $BACKUP_DIR/daily
 /usr/bin/docker exec cantusdb-postgres-1 rm /var/lib/postgresql/backups/$BACKUP_FILENAME
@@ -34,10 +34,14 @@ MONTH_OF_YEAR=$(date "+%m")
 
 # Retain weekly backups on Mondays
 # Manage retention of weekly backups
+# Copy the partial export for RISM purposes (created weekly on Mondays)
 if [ $DAY_OF_WEEK -eq 1 ]; then
     cp $BACKUP_DIR/daily/$BACKUP_FILENAME $BACKUP_DIR/weekly
     FILES_TO_REMOVE=$(ls -td $BACKUP_DIR/weekly/* | tail -n +9)
     [[ ! -z "$FILES_TO_REMOVE" ]] && rm $FILES_TO_REMOVE
+    # Copy the partial DB backup for RISM purposes to $BACKUP_DIR/rism
+    /usr/bin/docker cp cantusdb-postgres-1:/var/lib/postgresql/backups/cantusdb_rism_export.sql.gz $BACKUP_DIR/rism
+    /usr/bin/docker exec cantusdb-postgres-1 rm /var/lib/postgresql/backups/cantusdb_rism_export.sql.gz
 fi
 
 # Retain a monthly backup on the first day of the month

django/cantusdb_project/main_app/admin/institution.py

@@ -32,9 +32,18 @@ class InstitutionAdmin(BaseModelAdmin):
         "get_city_region",
         "country",
         "is_private_collector",
+        "is_private_collection",
     )
-    search_fields = ("name", "siglum", "city", "region", "alternate_names")
-    list_filter = ("is_private_collector", "city")
+    search_fields = (
+        "name",
+        "siglum",
+        "city",
+        "region",
+        "alternate_names",
+        "migrated_identifier",
+    )
+    readonly_fields = ("migrated_identifier",)
+    list_filter = ("is_private_collector", "is_private_collection", "city")
     inlines = (InstitutionIdentifierInline, InstitutionSourceInline)
     fieldsets = [
         (
@@ -48,6 +57,8 @@ class InstitutionAdmin(BaseModelAdmin):
                     "alternate_names",
                     "former_sigla",
                     "private_notes",
+                    "is_private_collection",
+                    "migrated_identifier",
                 )
             },
         ),

django/cantusdb_project/main_app/admin/source.py

@@ -25,6 +25,7 @@ class SourceAdmin(BaseModelAdmin):
         "shelfmark",
         "holding_institution__siglum",
         "holding_institution__name",
+        "holding_institution__migrated_identifier",
         "id",
         "provenance_notes",
     )

django/cantusdb_project/main_app/forms.py

@@ -4,6 +4,7 @@
     Chant,
     Service,
     Genre,
+    Institution,
     Notation,
     Feast,
     Source,
@@ -215,10 +216,6 @@ class Meta:
         widgets = {
             # "title": TextInputWidget(),
             # "siglum": TextInputWidget(),
-            "holding_institution": autocomplete.ModelSelect2(
-                url="holding-autocomplete"
-            ),
-            "shelfmark": TextInputWidget(),
             "provenance": autocomplete.ModelSelect2(url="provenance-autocomplete"),
             "provenance_notes": TextInputWidget(),
             "date": TextInputWidget(),
@@ -254,6 +251,17 @@ class Meta:
             "segment_m2m": CheckboxNameModelMultipleChoiceField,
         }
 
+    holding_institution = forms.ModelChoiceField(
+        queryset=Institution.objects.all(),
+        required=True,
+        widget=autocomplete.ModelSelect2(url="holding-autocomplete"),
+    )
+
+    shelfmark = forms.CharField(
+        required=True,
+        widget=TextInputWidget,
+    )
+
     TRUE_FALSE_CHOICES_SOURCE = (
         (True, "Full source"),
         (False, "Fragment or Fragmented"),
@@ -407,10 +415,6 @@ class Meta:
             "other_editors",
         ]
         widgets = {
-            "holding_institution": autocomplete.ModelSelect2(
-                url="holding-autocomplete"
-            ),
-            "shelfmark": TextInputWidget(),
             "segment_m2m": CheckboxSelectMultiple(),
             "provenance": autocomplete.ModelSelect2(url="provenance-autocomplete"),
             "provenance_notes": TextInputWidget(),
@@ -447,6 +451,17 @@ class Meta:
             "segment_m2m": CheckboxNameModelMultipleChoiceField,
         }
 
+    shelfmark = forms.CharField(
+        required=True,
+        widget=TextInputWidget,
+    )
+
+    holding_institution = forms.ModelChoiceField(
+        queryset=Institution.objects.all(),
+        required=True,
+        widget=autocomplete.ModelSelect2(url="holding-autocomplete"),
+    )
+
     CHOICES_FULL_SOURCE = (
         (None, "None"),
         (True, "Full source"),
@@ -718,6 +733,16 @@ class Meta:
         help_text="RISM-style siglum + Shelf-mark (e.g. GB-Ob 202).",
     )
 
+    shelfmark = forms.CharField(
+        required=True,
+        widget=TextInputWidget,
+    )
+
+    holding_institution = forms.ModelChoiceField(
+        queryset=Institution.objects.all().order_by("name"),
+        required=True,
+    )
+
     provenance = forms.ModelChoiceField(
         queryset=Provenance.objects.all().order_by("name"),
         required=False,

django/cantusdb_project/main_app/management/commands/migrate_records.py

@@ -24,6 +24,15 @@
     "US-OakCApc",
     "US-CApc",
     "ZA-Newpc",
+    "CDN-pc",
+    "CDN-pc(AB)",
+    "CDN-pc(BC)",
+    "US-pc(Texas),",
+    "US-pc(MA)",
+    "US-pc(OH)",
+    "US-pc(CA)",
+    "US-pc(VA)",
+    "US-pc(Washington)",
     "CDN-MtlQCpc",
     "CDN-MasQCpc",
     "CDN-HalNSpc",
@@ -187,9 +196,14 @@ def handle(self, *args, **options):
 
                 if siglum in private_collections:
                     iobj["is_private_collector"] = True
+                    iobj["is_private_collection"] = True
+                    iobj["migrated_identifier"] = siglum.strip()
                 else:
                     iobj["siglum"] = siglum
 
+                    if "private" in institution_name.lower():
+                        iobj["is_private_collection"] = True
+
                 institution = Institution.objects.create(**iobj)
 
                 rism_id = None

django/cantusdb_project/main_app/migrations/0029_institution_migrated_identifier.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.2.14 on 2024-08-23 08:04
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("main_app", "0028_alter_institution_options_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="institution",
+            name="migrated_identifier",
+            field=models.CharField(
+                blank=True,
+                help_text="Former Cantus identifier. Should not be used for new records.",
+                max_length=64,
+                null=True,
+            ),
+        ),
+    ]

django/cantusdb_project/main_app/migrations/0030_institution_is_private_collection.py

@@ -0,0 +1,21 @@
+# Generated by Django 4.2.14 on 2024-08-23 08:21
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("main_app", "0029_institution_migrated_identifier"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="institution",
+            name="is_private_collection",
+            field=models.BooleanField(
+                default=False,
+                help_text="Mark this instititution as being a private collection. This is used to identify private \ncollectors regardless of whether they have a RISM siglum or not.\n",
+            ),
+        ),
+    ]

django/cantusdb_project/main_app/migrations/0031_alter_source_holding_institution_and_more.py

@@ -0,0 +1,32 @@
+# Generated by Django 4.2.14 on 2024-08-30 13:58
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("main_app", "0030_institution_is_private_collection"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="source",
+            name="holding_institution",
+            field=models.ForeignKey(
+                default=19,
+                on_delete=django.db.models.deletion.PROTECT,
+                to="main_app.institution",
+            ),
+            preserve_default=False,
+        ),
+        migrations.AlterField(
+            model_name="source",
+            name="shelfmark",
+            field=models.CharField(
+                default="XX-NN",
+                max_length=255,
+            ),
+        ),
+    ]

django/cantusdb_project/main_app/mixins.py

@@ -0,0 +1,59 @@
+from typing import Any
+from django.http.response import JsonResponse, HttpResponse
+from django.http.request import HttpRequest
+from django.core.exceptions import ImproperlyConfigured
+from django.template.response import TemplateResponse
+
+
+class JSONResponseMixin:
+    """
+    Mixin to negotiate content type. Designed for use with
+    DetailView and ListView classes only.
+
+    If the request contains an `Accept` header with the value
+    `application/json`, the response will be a JSON object.
+    Otherwise, the response will render the HTML template as
+    usual.
+
+    The parent view must define an attribute "json_fields" that
+    lists the fields to be included in the JSON response.
+    """
+
+    def render_to_response(
+        self, context: dict[Any, Any], **response_kwargs: dict[Any, Any]
+    ) -> HttpResponse:
+        """
+        Returns a JSON response if the request accepts JSON.
+        Otherwise, returns the default response.
+        """
+        try:
+            request: HttpRequest = self.request  # type: ignore[attr-defined]
+        except AttributeError as exc:
+            raise ImproperlyConfigured(
+                "A JSONResponseMixin must be used with a DetailView or ListView."
+            ) from exc
+        try:
+            json_fields = self.json_fields  # type: ignore[attr-defined]
+        except AttributeError as exc:
+            raise ImproperlyConfigured(
+                "A JSONResponseMixin must define a json_fields attribute."
+            ) from exc
+        if "application/json" in request.META.get("HTTP_ACCEPT", ""):
+            obj = context.get("object")
+            if obj:
+                obj_json = {}
+                for field in json_fields:
+                    obj_json[field] = getattr(obj, field)
+                return JsonResponse({obj.get_verbose_name(): obj_json})
+            q_s = context["object_list"].values(*json_fields)
+            q_s_name = str(q_s.model.get_verbose_name_plural())
+            return JsonResponse({q_s_name: list(q_s)})
+        try:
+            template_response: TemplateResponse = super().render_to_response(  # type: ignore[misc]
+                context, **response_kwargs
+            )
+        except AttributeError as exc:
+            raise ImproperlyConfigured(
+                "A JSONResponseMixin must be used with a DetailView or ListView."
+            ) from exc
+        return template_response

django/cantusdb_project/main_app/models/institution.py

@@ -7,6 +7,9 @@
 region_help_text = """Province / State / Canton / County. Used to disambiguate cities, e.g., "London (Ontario)"."""
 city_help_text = """City / Town / Village / Settlement"""
 private_collector_help = """Mark this institution as private collector."""
+private_collection_help = """Mark this instititution as being a private collection. This is used to identify private 
+collectors regardless of whether they have a RISM siglum or not.
+"""
 
 
 class Institution(BaseModel):
@@ -37,6 +40,10 @@ class Meta:
         default=False,
         help_text=private_collector_help,
     )
+    is_private_collection = models.BooleanField(
+        default=False,
+        help_text=private_collection_help,
+    )
     city = models.CharField(
         max_length=64, blank=True, null=True, help_text=city_help_text
     )
@@ -55,6 +62,12 @@ class Meta:
         null=True,
         help_text="Notes about this institution that are not publicly visible.",
     )
+    migrated_identifier = models.CharField(
+        max_length=64,
+        blank=True,
+        null=True,
+        help_text="Former Cantus identifier. Should not be used for new records.",
+    )
 
     def __str__(self) -> str:
         sigl: str = f" ({self.siglum})" if self.siglum else ""

django/cantusdb_project/main_app/models/project.py

@@ -13,4 +13,4 @@ class Project(BaseModel):
     name = models.CharField(max_length=63)
 
     def __str__(self):
-        return f"{self.name} ({self.id})"
+        return f"{self.name}"

django/cantusdb_project/main_app/models/source.py

@@ -42,13 +42,13 @@ class Source(BaseModel):
     holding_institution = models.ForeignKey(
         "Institution",
         on_delete=models.PROTECT,
-        null=True,
-        blank=True,
+        null=False,
+        blank=False,
     )
     shelfmark = models.CharField(
         max_length=255,
-        blank=True,
-        null=True,
+        blank=False,
+        null=False,
     )
     provenance = models.ForeignKey(
         "Provenance",