Cria funcionalidade para notificar ausência de arquivos de log

collection/models.py

@@ -233,3 +233,7 @@ def create_or_update(
     @property
     def name(self):
         return CollectionName.objects.filter(collection=self).iterator()
+
+    @property
+    def acron2_list(self):
+        return [col.acron2 for col in Collection.objects.iterator()]

collection/wagtail_hooks.py

@@ -18,6 +18,7 @@ class CollectionSnippetViewSet(SnippetViewSet):
     list_display = (
         "main_name",
         "acron3",
+        "acron2",
         "code",
         "status",
         "collection_type",

config/settings/local.py

@@ -29,11 +29,13 @@
 EMAIL_HOST = env("EMAIL_HOST", default="mailhog")
 # https://docs.djangoproject.com/en/dev/ref/settings/#email-port
 EMAIL_PORT = 1025
+# https://docs.djangoproject.com/en/dev/ref/settings/#email-host-user
+EMAIL_HOST_USER = env.str("DJANGO_EMAIL_HOST_USER", default="[email protected]")
 
 # ADMIN
 # ------------------------------------------------------------------------------
 # https://docs.wagtail.org/en/stable/reference/settings.html#wagtailadmin-base-url
-WAGTAILADMIN_BASE_URL = 'https://usage.scielo.org'
+WAGTAILADMIN_BASE_URL = 'http://0.0.0.0:8009'
 
 # WhiteNoise
 # ------------------------------------------------------------------------------

core/templates/wagtailadmin/home.html

@@ -12,7 +12,7 @@
 
     {% else %}
 
-        SciELO Core
+        SciELO Usage
 
     {% endif %}
 

log_manager/choices.py

@@ -28,12 +28,14 @@
 COLLECTION_CONFIG_TYPE_DIRECTORY_PROCESSED_DATA = 'PRO'
 COLLECTION_CONFIG_TYPE_DIRECTORY_METRICS = 'MTS'
 COLLECTION_CONFIG_TYPE_FILES_PER_DAY = 'DAY'
+COLLECTION_CONFIG_TYPE_EMAIL = 'EMA'
 
 COLLECTION_CONFIG_TYPE = [
     (COLLECTION_CONFIG_TYPE_DIRECTORY_LOGS, _('Logs')),
     (COLLECTION_CONFIG_TYPE_DIRECTORY_PROCESSED_DATA, _('Processed Data')),
     (COLLECTION_CONFIG_TYPE_DIRECTORY_METRICS, _('Metrics')),
     (COLLECTION_CONFIG_TYPE_FILES_PER_DAY, _('Files per Day')),
+    (COLLECTION_CONFIG_TYPE_EMAIL, _('E-mail')),
 ]
 
 
@@ -52,6 +54,12 @@
 ]
 
 
-TEMPORAL_REFERENCE_YESTERDAY = 'yesterday'
-TEMPORAL_REFERENCE_LAST_WEEK = 'last week'
-TEMPORAL_REFERENCE_LAST_MONTH = 'last month'
+COLLECTION_LOG_FILE_DATE_COUNT_OK = 'OK'
+COLLECTION_LOG_FILE_DATE_COUNT_MISSING_FILES = 'MIS'
+COLLECTION_LOG_FILE_DATE_COUNT_EXTRA_FILES = 'EXT'
+
+COLLECTION_LOG_FILE_DATE_COUNT = [
+    (COLLECTION_LOG_FILE_DATE_COUNT_OK, _("OK")),
+    (COLLECTION_LOG_FILE_DATE_COUNT_MISSING_FILES, _("Missing Files")),
+    (COLLECTION_LOG_FILE_DATE_COUNT_EXTRA_FILES, _("Extra files")),
+]

log_manager/exceptions.py

@@ -12,3 +12,6 @@ class UndefinedApplicationConfigError(Exception):
 
 class UndefinedCollectionConfigError(Exception):
     ...
+
+class MultipleCollectionConfigError(Exception):
+    ...

log_manager/models.py

@@ -1,6 +1,7 @@
 from datetime import datetime
 
 from django.db import models
+from django.db.models import Q
 from django.db.utils import IntegrityError
 from django.utils.translation import gettext_lazy as _
 from wagtail.admin.panels import FieldPanel
@@ -12,7 +13,11 @@
 from tracker.models import UnexpectedEvent
 
 from . import choices
-from .exceptions import LogFileAlreadyExistsError
+from .exceptions import (
+    LogFileAlreadyExistsError, 
+    MultipleCollectionConfigError,
+    UndefinedCollectionConfigError,
+)
 
 
 class ApplicationConfig(CommonControlField):
@@ -144,7 +149,7 @@ class CollectionConfig(CommonControlField):
 
     class Meta:
         ordering = ['collection', 'value']
-        unique_together = ('config_type', 'value',)
+        unique_together = ('collection', 'config_type', 'start_date', 'end_date', 'value',)
         verbose_name = _("Collection Configuration")
         verbose_name_plural = _("Collection Configurations")
 
@@ -155,6 +160,24 @@ def filter_by_collection_and_config_type(cls, collection_acron2, config_type, is
             config_type=config_type, 
             is_enabled=is_enabled
         )
+
+    @classmethod
+    def get_number_of_expected_files_by_day(cls, collection_acron2, date, is_enabled=True):
+        files_by_day = cls.objects.filter(
+            Q(collection__acron2=collection_acron2) &
+            Q(start_date__lte=date) &
+            (Q(end_date__gte=date) | Q(end_date__isnull=True)) &
+            Q(config_type=choices.COLLECTION_CONFIG_TYPE_FILES_PER_DAY) &
+            Q(is_enabled=is_enabled)
+        )
+
+        if files_by_day.count() > 1:
+            raise MultipleCollectionConfigError(_("ERROR. Please, keep only one configuration enabled for the FILES_BY_DAY attribute."))
+
+        if files_by_day.count() == 0:
+            raise UndefinedCollectionConfigError(_("ERROR. Please, add an Application Configuration for the FILES_BY_DAY attribute."))
+
+        return int(files_by_day.get().value)
 
     def __str__(self):
         return f'{self.value}'
@@ -201,11 +224,132 @@ def create(cls, user, log_file, date):
         obj.save()
 
         return obj
+
+    @classmethod
+    def filter_by_collection_and_date(cls, collection_acron2, date):
+        return cls.objects.filter(
+            ~Q(log_file__status__in=[
+                choices.LOG_FILE_STATUS_CREATED, 
+                choices.LOG_FILE_STATUS_INVALIDATED
+            ]),
+            log_file__collection__acron2=collection_acron2,
+            date=date,
+        )
+
+    @classmethod
+    def get_number_of_found_files_for_date(cls, collection_acron2, date):
+        return cls.objects.filter(
+            ~Q(log_file__status__in=[
+                choices.LOG_FILE_STATUS_CREATED, 
+                choices.LOG_FILE_STATUS_INVALIDATED
+            ]),
+            log_file__collection__acron2=collection_acron2,
+            date=date,
+        ).count()
 
     def __str__(self):
         return f'{self.log_file.path}-{self.date}'
 
 
+class CollectionLogFileDateCount(CommonControlField):
+    collection = models.ForeignKey(
+        Collection, 
+        verbose_name=_('Collection'), 
+        on_delete=models.DO_NOTHING, 
+        null=False, 
+        blank=False,
+    )
+
+    date = models.DateField(
+        _('Date'),
+        null=False,
+        blank=False,
+    )
+
+    year = models.IntegerField(
+        _('Year'),
+        max_length=4,
+        null=False,
+        blank=False,
+    )
+
+    month = models.IntegerField(
+        _('Month'),
+        max_length=2,
+        null=False,
+        blank=False,
+    )
+
+    found_log_files = models.IntegerField(
+        verbose_name=_('Number of Found Valid Log Files'), 
+        max_length=8,
+        default=0,
+    )
+
+    expected_log_files = models.IntegerField(
+        verbose_name=_('Number of Expected Valid Log Files'),
+        max_length=8,
+        blank=True,
+        null=True,
+    )
+
+    status = models.CharField(
+        verbose_name=_('Status'),
+        choices=choices.COLLECTION_LOG_FILE_DATE_COUNT,
+        max_length=3,
+    )
+
+    @classmethod
+    def create_or_update(cls, user, collection, date, expected_log_files, found_log_files):
+        obj, created = cls.objects.get_or_create(
+            collection=collection, 
+            date=date,
+            month=date.month,
+            year=date.year,
+        )
+
+        if not created:
+            obj.updated_by = user
+            obj.updated = datetime.utcnow()
+        else:
+            obj.creator = user
+            obj.created = datetime.utcnow()
+
+        obj.expected_log_files = expected_log_files            
+        obj.found_log_files = found_log_files
+
+        if found_log_files < expected_log_files:
+            obj.status = choices.COLLECTION_LOG_FILE_DATE_COUNT_MISSING_FILES
+        elif found_log_files > expected_log_files:
+            obj.status = choices.COLLECTION_LOG_FILE_DATE_COUNT_EXTRA_FILES
+        else:
+            obj.status = choices.COLLECTION_LOG_FILE_DATE_COUNT_OK 
+
+        try:
+            obj.save()
+            return obj        
+        except IntegrityError:
+            ...
+
+    class Meta:
+        ordering = ['-date']
+        verbose_name = _("Collection Log File Date Count")
+        unique_together = (
+            'collection',
+            'date',
+        )
+
+    panels = [
+        AutocompletePanel('collection'),
+        FieldPanel('date'),
+        FieldPanel('year'),
+        FieldPanel('month'),
+        FieldPanel('found_log_files'),
+        FieldPanel('expected_log_files'),
+        FieldPanel('status'),
+    ]
+
+
 class LogFile(CommonControlField):
     hash = models.CharField(_("Hash MD5"), max_length=32, null=True, blank=True, unique=True)