Merge pull request #762 from security-force-monitor/hcg/sources-patch

Deduplicate source lists on entity detail pages, update column headers

organization/views.py

@@ -1,5 +1,4 @@
 from datetime import date
-from itertools import chain
 import json
 
 from django.conf import settings
@@ -33,6 +32,7 @@
 from sfm_pc.templatetags.countries import country_name
 from sfm_pc.base_views import BaseUpdateView, BaseCreateView, BaseDetailView, \
     BaseDeleteView, BaseDeleteRelationshipView
+from source.models import Source
 
 
 class EditButtonsMixin:
@@ -48,9 +48,9 @@ class OrganizationDetail(BaseDetailView):
     slug_field = 'uuid'
 
     def get_sources(self, context):
-        sources = list(context['organization'].sources)
+        sources = set()
 
-        original_list = sources.copy()
+        sources.update(list(context['organization'].sources.values_list('uuid', flat=True)))
 
         related_entities = (
             'person_members',
@@ -64,11 +64,11 @@ def get_sources(self, context):
         )
 
         for relation in related_entities:
-            sources += list(
-                chain.from_iterable(entity.sources for entity in context[relation])
-            )
+            for entity in context[relation]:
+                sources.update(list(entity.sources.values_list('uuid', flat=True)))
 
-        return sorted(set(sources), key=lambda x: x.get_published_date() or date.min, reverse=True)
+        return Source.objects.filter(uuid__in=sources).order_by('source_url', '-accesspoint__accessed_on')\
+                                                      .distinct('source_url')
 
     def get_context_data(self, **kwargs):
         context = super().get_context_data(**kwargs)

person/views.py

@@ -16,6 +16,7 @@
 from membershipperson.models import MembershipPersonMember, MembershipPerson
 from sfm_pc.base_views import BaseUpdateView, BaseCreateView, BaseDetailView, \
     BaseDeleteView, BaseDeleteRelationshipView
+from source.models import Source
 
 
 class PersonDetail(BaseDetailView):
@@ -24,21 +25,22 @@ class PersonDetail(BaseDetailView):
     slug_field = 'uuid'
 
     def get_sources(self, context):
-        sources = list(context['person'].sources)
+        sources = set()
 
-        sources += list(
-            chain.from_iterable(m.sources for m in context['memberships'])
-        )
+        sources.update(list(context['person'].sources.values_list('uuid', flat=True)))
 
-        sources += list(
-            chain.from_iterable(sub['commander'].sources for sub in context['subordinates'])
-        )
+        for membership in context['memberships']:
+            sources.update(
+                list(membership.sources.values_list('uuid', flat=True))
+            )
 
-        sources += list(
-            chain.from_iterable(sub['commander'].sources for sub in context['superiors'])
-        )
+        for entity in chain(context['subordinates'], context['superiors']):
+            sources.update(
+                list(entity['commander'].sources.values_list('uuid', flat=True))
+            )
 
-        return sorted(set(sources), key=lambda x: x.get_published_date() or date.min, reverse=True)
+        return Source.objects.filter(uuid__in=sources).order_by('source_url', '-accesspoint__accessed_on')\
+                                                      .distinct('source_url')
 
     def get_context_data(self, **kwargs):
         context = super().get_context_data(**kwargs)

sfm_pc/utils.py

@@ -880,5 +880,3 @@ def get_source_context(field_name, access_point, uncommitted=True):
         context['accessed_on'] = access_point.accessed_on.strftime('%Y-%m-%dT%H:%M:%S')
 
     return context
-
-

source/mixins.py

@@ -1,18 +1,24 @@
 import itertools
 
 from complex_fields.models import ComplexFieldContainer, ComplexFieldListContainer
+from django.db.models import QuerySet
+
+from source.models import Source
 
 
 class SourcesMixin:
     @property
     def sources(self):
-        sources = []
+        access_points = set()
 
         for field in itertools.chain(self.complex_fields, self.complex_lists):
             if isinstance(field, ComplexFieldListContainer):
                 field = field.get_complex_field(None)
                 assert field
 
-            sources += [accesspoint.source for accesspoint in field.get_sources()]
+            field_sources = field.get_sources()
+
+            if field_sources and isinstance(field_sources, QuerySet):  # Empty list if unsourced
+                access_points.update(list(field_sources))
 
-        return set(sources)
+        return Source.objects.filter(accesspoint__in=access_points).distinct('source_url')

source/models.py

@@ -46,7 +46,7 @@ class Source(models.Model, GetSpreadsheetFieldNameMixin, VersionsMixin):
     title = source_fields.TextField(verbose_name=_("title"), spreadsheet_field_name='source:title')
     type = source_fields.CharField(max_length=1000, null=True, blank=True, spreadsheet_field_name='source:type')
     author = source_fields.CharField(max_length=1000, null=True, blank=True, spreadsheet_field_name='source:author')
-    publication = source_fields.TextField(null=True, verbose_name=_("publication"), spreadsheet_field_name='source:publication_name')
+    publication = source_fields.TextField(null=True, verbose_name=_("publisher"), spreadsheet_field_name='source:publication_name')
     publication_country = source_fields.CharField(
         max_length=1000,
         verbose_name=_("publication country"),

templates/partials/source_overview_table.html

@@ -15,7 +15,7 @@ <h3 id="sources">
                     <th>{{ models.Source|verbose_field_name:"publication" }}</th>
                     <th>{{ models.Source|verbose_field_name:"title" }}</th>
                     <th>{{ models.AccessPoint|verbose_field_name:"accessed_on" }}</th>
-                    <th>Permalink</th>
+                    <th>Archive Link</th>
                 </tr>
             </thead>
             <tbody>
@@ -44,7 +44,7 @@ <h3 id="sources">
                             {{ source.title }}
                         </a>
                     </td>
-                    {% with access_date=source.accesspoint_set.get.accessed_on %}
+                    {% with access_date=source.accesspoint_set.last.accessed_on %}
                         <td data-order="{{ access_date.year|default:0 }}-{{ access_date.month|default:0 }}-{{access_date.day|default:0 }}">
                             {% if access_date.month and access_date.day %}
                                 {{ access_date|date:"d F Y" }}
@@ -57,9 +57,9 @@ <h3 id="sources">
                     {% endwith %}
                     <td>
                         <a
-                            href="{{ source.accesspoint_set.get.archive_url }}"
+                            href="{{ source.accesspoint_set.last.archive_url }}"
                             target="_blank"
-                            onclick="_paq.push(['trackEvent', 'Source Table Interaction', 'Archive Link Click', '{{ source.accesspoint_set.get.archive_url }}']);"
+                            onclick="_paq.push(['trackEvent', 'Source Table Interaction', 'Archive Link Click', '{{ source.accesspoint_set.last.archive_url }}']);"
                         >
                             <i class="fas fa-link"></i>
                         </a>

violation/views.py

@@ -23,11 +23,8 @@ class ViolationDetail(BaseDetailView):
     slug_field = 'uuid'
 
     def get_sources(self, context):
-        return sorted(
-            context['violation'].sources,
-            key=lambda x: x.get_published_date() or date.min,
-            reverse=True
-        )
+        return context['violation'].sources.order_by('source_url', '-accesspoint__accessed_on')\
+                                           .distinct('source_url')
 
     def get_page_title(self):
         title = _('Incident')