From 3f2d57bd510a3e7de1230bf58baa02d17460397f Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Tue, 25 Jun 2024 22:37:30 -0300 Subject: [PATCH 01/19] Altera ordem de menu para acomodar nova app metrics --- config/menu.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/config/menu.py b/config/menu.py index e2266cb..6dcda53 100644 --- a/config/menu.py +++ b/config/menu.py @@ -1,8 +1,9 @@ WAGTAIL_MENU_APPS_ORDER = { "collection": 100, - "log_manager": 200, - "tasks": 300, - "unexpected-error": 400, + "metrics": 200, + "log_manager": 300, + "tasks": 400, + "unexpected-error": 500, } def get_menu_order(app_name): From e926938bde9f0362ed7560da7f16cd90d8c7b917 Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Tue, 25 Jun 2024 22:38:44 -0300 Subject: [PATCH 02/19] =?UTF-8?q?Importa=20app=20metrics=20em=20settings,?= =?UTF-8?q?=20inclui=20configura=C3=A7=C3=B5es=20b=C3=A1sicas=20e=20ajusta?= =?UTF-8?q?=20outras?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/settings/base.py | 29 ++++++++++++++++++++++------- local.yml | 8 ++++---- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/config/settings/base.py b/config/settings/base.py index 32a807e..7e375b7 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -118,6 +118,7 @@ "collection", "core", "log_manager", + "metrics", "tracker", ] @@ -403,7 +404,21 @@ HAYSTACK_CONNECTIONS = { "default": { "ENGINE": "haystack.backends.solr_backend.SolrEngine", - "URL": "%s%s" % (env("SOLR_URL", default="http://solr:8983/solr/"), "usage"), + "URL": "%s%s" % (env("SOLR_URL", default="http://solr:8983/solr/"), "metrics"), + "ADMIN_URL": "http://solr:8983/solr/admin/cores", + "SILENTLY_FAIL": False, + "SOLR_TIMEOUT": 10, + }, + "metrics": { + "ENGINE": "haystack.backends.solr_backend.SolrEngine", + "URL": "%s%s" % (env("SOLR_URL", default="http://solr:8983/solr/"), "metrics"), + "ADMIN_URL": "http://solr:8983/solr/admin/cores", + "SILENTLY_FAIL": False, + "SOLR_TIMEOUT": 10, + }, + "log_manager": { + "ENGINE": "haystack.backends.solr_backend.SolrEngine", + "URL": "%s%s" % (env("SOLR_URL", default="http://solr:8983/solr/"), "log_manager"), "ADMIN_URL": "http://solr:8983/solr/admin/cores", "SILENTLY_FAIL": False, "SOLR_TIMEOUT": 10, @@ -421,10 +436,10 @@ SEARCH_PAGINATION_ITEMS_PER_PAGE = 10 -SEARCH_FACET_ITEMS_PER_MORE = 5 +# SEARCH_FACET_ITEMS_PER_MORE = 5 -SEARCH_FACET_LIST = [ - "server_time", - "action_name", - "browser_name", -] +# SEARCH_FACET_LIST = [ +# "server_time", +# "action_name", +# "browser_name", +# ] diff --git a/local.yml b/local.yml index ab0785a..6f8e814 100644 --- a/local.yml +++ b/local.yml @@ -19,7 +19,7 @@ services: - solr volumes: - .:/app:z - - ../scms_data/usage/log_manager:/data + - ../scms_data/usage/data:/data env_file: - ./.envs/.local/.django - ./.envs/.local/.postgres @@ -60,7 +60,7 @@ services: env_file: - ./.envs/.local/.minio volumes: - - ../scms_data/usage/minio:/data + - ../scms_data/minio:/data command: server --console-address ":9001" /data postgres: @@ -70,8 +70,8 @@ services: image: scielo_usage_local_postgres container_name: scielo_usage_local_postgres volumes: - - ../scms_data/usage/postgresql:/var/lib/postgresql/data:Z - - ../scms_data/usage/postgresql_backup:/backups:z + - ../scms_data/postgresql:/var/lib/postgresql/data:Z + - ../scms_data/postgresql_backup:/backups:z ports: - "5439:5432" env_file: From 3b456c6316262400de34d5fd37717d415b754c20 Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Tue, 25 Jun 2024 22:39:52 -0300 Subject: [PATCH 03/19] =?UTF-8?q?Altera=20local=20de=20=C3=ADndice=20relac?= =?UTF-8?q?ionado=20=C3=A0=20app=20log=5Fmanager?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- index/data/{usage => log_manager/conf}/schema.xml | 2 +- index/data/{usage => log_manager/conf}/solrconfig.xml | 0 index/data/{usage => log_manager}/core.properties | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename index/data/{usage => log_manager/conf}/schema.xml (99%) rename index/data/{usage => log_manager/conf}/solrconfig.xml (100%) rename index/data/{usage => log_manager}/core.properties (87%) diff --git a/index/data/usage/schema.xml b/index/data/log_manager/conf/schema.xml similarity index 99% rename from index/data/usage/schema.xml rename to index/data/log_manager/conf/schema.xml index 0fab467..f04bca8 100755 --- a/index/data/usage/schema.xml +++ b/index/data/log_manager/conf/schema.xml @@ -1,5 +1,5 @@ - + diff --git a/index/data/usage/solrconfig.xml b/index/data/log_manager/conf/solrconfig.xml similarity index 100% rename from index/data/usage/solrconfig.xml rename to index/data/log_manager/conf/solrconfig.xml diff --git a/index/data/usage/core.properties b/index/data/log_manager/core.properties similarity index 87% rename from index/data/usage/core.properties rename to index/data/log_manager/core.properties index f537e65..75456b3 100755 --- a/index/data/usage/core.properties +++ b/index/data/log_manager/core.properties @@ -2,5 +2,5 @@ #Fri Mar 29 13:50:09 UTC 2024 schema=schema.xml dataDir=data -name=usage +name=log_manager config=solrconfig.xml From 8712a5898c9ef35cdc0aac8bb11869b3b04bbd92 Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Tue, 25 Jun 2024 22:40:07 -0300 Subject: [PATCH 04/19] =?UTF-8?q?Cria=20=C3=ADndice=20para=20app=20metrics?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- index/data/metrics/conf/schema.xml | 108 +++++++++++++++++++++++++ index/data/metrics/conf/solrconfig.xml | 98 ++++++++++++++++++++++ index/data/metrics/core.properties | 6 ++ 3 files changed, 212 insertions(+) create mode 100755 index/data/metrics/conf/schema.xml create mode 100755 index/data/metrics/conf/solrconfig.xml create mode 100755 index/data/metrics/core.properties diff --git a/index/data/metrics/conf/schema.xml b/index/data/metrics/conf/schema.xml new file mode 100755 index 0000000..dd82bff --- /dev/null +++ b/index/data/metrics/conf/schema.xml @@ -0,0 +1,108 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + + + + \ No newline at end of file diff --git a/index/data/metrics/conf/solrconfig.xml b/index/data/metrics/conf/solrconfig.xml new file mode 100755 index 0000000..e77ebc9 --- /dev/null +++ b/index/data/metrics/conf/solrconfig.xml @@ -0,0 +1,98 @@ + + + + + + + LUCENE_42 + + + + ${solr.core0.data.dir:} + + + + + + + ${solr.core0.data.dir:} + + + + + + + true + + + + + + + + + + + + + + + + solrpingquery + + + all + + + + + + solr + + + + diff --git a/index/data/metrics/core.properties b/index/data/metrics/core.properties new file mode 100755 index 0000000..4b23656 --- /dev/null +++ b/index/data/metrics/core.properties @@ -0,0 +1,6 @@ +#Written by CorePropertiesLocator +#Fri Mar 29 13:50:09 UTC 2024 +schema=schema.xml +dataDir=data +name=metrics +config=solrconfig.xml From 064ce951dfb8fd6d6b70ab19c1bb5b140b5a5ade Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Tue, 25 Jun 2024 22:40:43 -0300 Subject: [PATCH 05/19] =?UTF-8?q?Remove=20atributo=20desnecess=C3=A1rio=20?= =?UTF-8?q?em=20modelos=20log=5Fmanager?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- log_manager/models.py | 4 ---- log_manager/search_indexes.py | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/log_manager/models.py b/log_manager/models.py index 0e965ca..6c7b4b2 100644 --- a/log_manager/models.py +++ b/log_manager/models.py @@ -268,27 +268,23 @@ class CollectionLogFileDateCount(CommonControlField): year = models.IntegerField( _('Year'), - max_length=4, null=False, blank=False, ) month = models.IntegerField( _('Month'), - max_length=2, null=False, blank=False, ) found_log_files = models.IntegerField( verbose_name=_('Number of Found Valid Log Files'), - max_length=8, default=0, ) expected_log_files = models.IntegerField( verbose_name=_('Number of Expected Valid Log Files'), - max_length=8, blank=True, null=True, ) diff --git a/log_manager/search_indexes.py b/log_manager/search_indexes.py index 99c7275..26c44e5 100644 --- a/log_manager/search_indexes.py +++ b/log_manager/search_indexes.py @@ -50,4 +50,4 @@ def get_model(self): return LogProcessedRow def index_queryset(self, using=None): - return self.get_model().objects.filter(log_file__collection__acron2='wi').all() + return self.get_model().objects.all() From b2bc593c13ec440c71dabf47bdc0de4f952e5882 Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Tue, 25 Jun 2024 22:41:16 -0300 Subject: [PATCH 06/19] Corrige template para povoar campo txt em schema Solr de log_manager --- .../search/indexes/log_manager/logprocessedrow_text.txt | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/log_manager/templates/search/indexes/log_manager/logprocessedrow_text.txt b/log_manager/templates/search/indexes/log_manager/logprocessedrow_text.txt index 07070f0..c2c0146 100644 --- a/log_manager/templates/search/indexes/log_manager/logprocessedrow_text.txt +++ b/log_manager/templates/search/indexes/log_manager/logprocessedrow_text.txt @@ -1,2 +1,7 @@ -{{ object.title }} -{{ object.body }} \ No newline at end of file +{{server_time}} +{{browser_name}} +{{browser_version}} +{{ip}} +{{latitude}} +{{longitude}} +{{action_name}} \ No newline at end of file From cd67f5b573c1a07734b0cd20b18585eee84612a2 Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Tue, 25 Jun 2024 22:41:56 -0300 Subject: [PATCH 07/19] =?UTF-8?q?Adiciona=20estrutura=20b=C3=A1sica=20de?= =?UTF-8?q?=20app=20metrics?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metrics/__init__.py | 0 metrics/admin.py | 3 +++ metrics/apps.py | 6 ++++++ metrics/migrations/__init__.py | 0 4 files changed, 9 insertions(+) create mode 100755 metrics/__init__.py create mode 100755 metrics/admin.py create mode 100755 metrics/apps.py create mode 100755 metrics/migrations/__init__.py diff --git a/metrics/__init__.py b/metrics/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/metrics/admin.py b/metrics/admin.py new file mode 100755 index 0000000..8c38f3f --- /dev/null +++ b/metrics/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/metrics/apps.py b/metrics/apps.py new file mode 100755 index 0000000..e609fae --- /dev/null +++ b/metrics/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class MetricsConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "metrics" diff --git a/metrics/migrations/__init__.py b/metrics/migrations/__init__.py new file mode 100755 index 0000000..e69de29 From 24701cae20318363464105459e85b3f849f8b515 Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Tue, 25 Jun 2024 22:50:34 -0300 Subject: [PATCH 08/19] Adiciona modelo para acomodar Top100Articles acessados --- metrics/tests/test_models.py | 87 ++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 metrics/tests/test_models.py diff --git a/metrics/tests/test_models.py b/metrics/tests/test_models.py new file mode 100644 index 0000000..457bb17 --- /dev/null +++ b/metrics/tests/test_models.py @@ -0,0 +1,87 @@ +from django.db.utils import IntegrityError +from django.test import TestCase + +from core.models import User +from metrics.models import Top100Articles + + +class Top100ArticlesModelTestCase(TestCase): + + def setUp(self): + self.user = User.objects.create(username='testuser') + + def test_create_valid_article(self): + article = Top100Articles.create( + user=self.user, + key_issn='2448-6132', + year_month_day='2024-06-01', + print_issn='2007-428X', + online_issn='2448-6132', + collection='mex', + pid='S2448-61322023000100101', + yop=2023, + total_item_requests=10, + total_item_investigations=5, + unique_item_requests=8, + unique_item_investigations=4, + ) + self.assertIsNotNone(article) + + self.assertEqual(article.key_issn, '2448-6132') + self.assertEqual(article.year_month_day, '2024-06-01') + self.assertEqual(article.print_issn, '2007-428X') + self.assertEqual(article.online_issn, '2448-6132') + self.assertEqual(article.collection, 'mex') + self.assertEqual(article.pid, 'S2448-61322023000100101') + self.assertEqual(article.yop, 2023) + self.assertEqual(article.total_item_requests, 10) + self.assertEqual(article.total_item_investigations, 5) + self.assertEqual(article.unique_item_requests, 8) + self.assertEqual(article.unique_item_investigations, 4) + + def test_unique_together_constraint(self): + Top100Articles.create( + user=self.user, + key_issn='2448-6132', + year_month_day='2024-06-01', + print_issn='2007-428X', + online_issn='2448-6132', + collection='mex', + pid='S2448-61322023000100101', + yop=2024, + total_item_requests=10, + total_item_investigations=5, + unique_item_requests=8, + unique_item_investigations=4, + ) + with self.assertRaises(IntegrityError): + Top100Articles.create( + user=self.user, + key_issn='2448-6132', + year_month_day='2024-06-01', + print_issn='2007-428X', + online_issn='2448-6132', + collection='mex', + pid='S2448-61322023000100101', + yop=2023, + total_item_requests=10, + total_item_investigations=5, + unique_item_requests=8, + unique_item_investigations=4, + ) + + def test_str_method(self): + article = Top100Articles( + key_issn='2448-6132', + year_month_day='2024-06-01', + print_issn='2007-428X', + online_issn='2448-6132', + collection='mex', + pid='S2448-61322023000100101', + yop=2023, + total_item_requests=10, + total_item_investigations=5, + unique_item_requests=8, + unique_item_investigations=4, + ) + self.assertEqual(str(article), '2448-6132, S2448-61322023000100101, 10') From 8e175c6b50bdd4851a9b0bed75454ab1264e778d Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Tue, 25 Jun 2024 22:51:31 -0300 Subject: [PATCH 09/19] =?UTF-8?q?Adiciona=20utilit=C3=A1rio=20para=20carre?= =?UTF-8?q?gar=20dados=20de=20arquivos=20extra=C3=ADdos=20do=20Matomo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metrics/utils.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 metrics/utils.py diff --git a/metrics/utils.py b/metrics/utils.py new file mode 100644 index 0000000..e0afa5c --- /dev/null +++ b/metrics/utils.py @@ -0,0 +1,7 @@ +import csv + + +def load_csv(path, delimiter='\t'): + with open(path) as fin: + for row in csv.DictReader(fin, fieldnames=fin.readline().strip().split(delimiter), delimiter=delimiter): + yield row From 0cb32ef586c79b3a63ad7f4b632034b4ab17c758 Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Tue, 25 Jun 2024 22:51:42 -0300 Subject: [PATCH 10/19] Adiciona arquivo views.py --- metrics/views.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100755 metrics/views.py diff --git a/metrics/views.py b/metrics/views.py new file mode 100755 index 0000000..91ea44a --- /dev/null +++ b/metrics/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. From 6d2b8e6d04bc51b26bab2cbec784e7f1523dab6f Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Tue, 25 Jun 2024 22:51:59 -0300 Subject: [PATCH 11/19] Adiciona wagtailhooks --- metrics/wagtail_hooks.py | 50 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 metrics/wagtail_hooks.py diff --git a/metrics/wagtail_hooks.py b/metrics/wagtail_hooks.py new file mode 100644 index 0000000..b5a18c7 --- /dev/null +++ b/metrics/wagtail_hooks.py @@ -0,0 +1,50 @@ +from django.utils.translation import gettext_lazy as _ +from wagtail.snippets.views.snippets import SnippetViewSet, SnippetViewSetGroup +from wagtail.snippets.models import register_snippet + +from config.menu import get_menu_order + +from metrics.models import ( + Top100Articles, +) + + +class Top100Articles(SnippetViewSet): + model = Top100Articles + menu_label = _("Top 100 Articles") + icon = "folder" + menu_order = 100 + + list_display = ( + 'collection', + 'key_issn', + 'pid', + 'yop', + 'year_month_day', + 'total_item_requests', + 'total_item_investigations', + 'unique_item_requests', + 'unique_item_investigations', + ) + list_filter = ( + "collection", + "key_issn", + 'yop', + 'year_month_day', + ) + search_fields = ( + "pid", + ) + + +class MetricsViewSetGroup(SnippetViewSetGroup): + menu_name = 'metrics' + menu_label = _("Usage Metrics") + menu_icon = "folder-open-inverse" + menu_order = get_menu_order("metrics") + items = ( + Top100Articles, + ) + + +register_snippet(MetricsViewSetGroup) From e772c79522b1cbe7e3448695fea7fca4ddc24dd6 Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Tue, 25 Jun 2024 22:52:58 -0300 Subject: [PATCH 12/19] =?UTF-8?q?Cria=20arquivo=20search=5Findexes.py=20pa?= =?UTF-8?q?ra=20aplica=C3=A7=C3=A3o=20metrics?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metrics/search_indexes.py | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 metrics/search_indexes.py diff --git a/metrics/search_indexes.py b/metrics/search_indexes.py new file mode 100644 index 0000000..8bb487e --- /dev/null +++ b/metrics/search_indexes.py @@ -0,0 +1,4 @@ +# coding: utf-8 +from haystack import indexes + +from .models import Top100Articles From fe793c35fb8280385a9e4cf1f83dfec8ccae42cd Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Tue, 25 Jun 2024 22:53:43 -0300 Subject: [PATCH 13/19] Cria template para povoar campo txt de Top100ArticlesIndex --- .../search/indexes/metrics/top100articles_text.txt | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 metrics/templates/search/indexes/metrics/top100articles_text.txt diff --git a/metrics/templates/search/indexes/metrics/top100articles_text.txt b/metrics/templates/search/indexes/metrics/top100articles_text.txt new file mode 100644 index 0000000..ccf5e94 --- /dev/null +++ b/metrics/templates/search/indexes/metrics/top100articles_text.txt @@ -0,0 +1,10 @@ +{{ object.collection }} +{{ object.key_issn }} +{{ object.pid }} +{{ object.yop }} +{{ object.language }} +{{ object.country }} +{{ object.total_item_requests }} +{{ object.total_item_investigations }} +{{ object.unique_item_requests }} +{{ object.unique_item_investigations }} \ No newline at end of file From b7f4179f4373d298369a99912e13b20c367cdaa5 Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Tue, 25 Jun 2024 22:54:25 -0300 Subject: [PATCH 14/19] =?UTF-8?q?Adiciona=20=C3=ADndice=20Top100ArticlesIn?= =?UTF-8?q?dex?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metrics/search_indexes.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/metrics/search_indexes.py b/metrics/search_indexes.py index 8bb487e..135c508 100644 --- a/metrics/search_indexes.py +++ b/metrics/search_indexes.py @@ -2,3 +2,29 @@ from haystack import indexes from .models import Top100Articles + + +class Top100ArticlesIndex(indexes.SearchIndex, indexes.Indexable): + text = indexes.CharField(document=True, use_template=True, null=True) + + collection = indexes.CharField(model_attr='collection', null=True) + key_issn = indexes.CharField(model_attr='key_issn', null=False) + pid = indexes.CharField(model_attr='pid', null=False) + yop = indexes.CharField(model_attr='yop', null=False) + total_item_requests = indexes.IntegerField(model_attr='total_item_requests', null=False) + total_item_investigations = indexes.IntegerField(model_attr='total_item_investigations', null=False) + unique_item_requests = indexes.IntegerField(model_attr='unique_item_requests', null=False) + unique_item_investigations = indexes.IntegerField(model_attr='unique_item_investigations', null=False) + + year_month_day = indexes.DateField(model_attr='year_month_day', null=False) + + metric_scope = indexes.CharField(null=False) + + def prepare_metric_scope(self, obj): + return 'top100articles' + + def get_model(self): + return Top100Articles + + def index_queryset(self, using=None): + return self.get_model().objects.all() From a061d7636b9264f0d5adf929735be7bf8e295c25 Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Tue, 25 Jun 2024 22:55:12 -0300 Subject: [PATCH 15/19] Adiciona task para fazer carga de modelo Top100Articles a partir de csv --- metrics/tasks.py | 93 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 metrics/tasks.py diff --git a/metrics/tasks.py b/metrics/tasks.py new file mode 100644 index 0000000..4b65b24 --- /dev/null +++ b/metrics/tasks.py @@ -0,0 +1,93 @@ +import logging +import os + +from django.contrib.auth import get_user_model +from django.utils.translation import gettext as _ + +from core.utils.utils import _get_user +from config import celery_app + +from . import ( + models, + utils, +) + + +User = get_user_model() + + +@celery_app.task(bind=True, name=_('Load Top100 Article Metrics from directory')) +def task_load_top100_from_dir(self, dir=None, user_id=None, username=None): + """ + Task to load most accessed articles by journal. + + Parameters: + dir (path): Directory path containing the top100 metrics files + user_id + username + + Returns: + None. + """ + user = _get_user(self.request, username=username, user_id=user_id) + + for root, sub_dirs, files in os.walk(dir): + for name in files: + if not name.lower().endswith('.csv'): + continue + + file_path = os.path.join(root, name) + for row in utils.load_csv(file_path): + try: + models.Top100ArticlesByJournal.create( + user=user, + key_issn=row['pid_issn'], + online_issn=row['online_issn'], + print_issn=row['print_issn'], + year_month_day=row['year_month_day'], + collection=row['collection'], + pid=row['pid'], + yop=row['yop'], + total_item_requests=row['total_item_requests'], + total_item_investigations=row['total_item_investigations'], + unique_item_requests=row['unique_item_requests'], + unique_item_investigations=row['unique_item_investigations'], + ) + except KeyError as e: + logging.info(_(f'File {file_path} does not contain all of the necessary data. Message: {e}')) + + +@celery_app.task(bind=True, name=_('Load Top100 Article Metrics from file')) +def task_load_top100_from_file(self, file_path, user_id=None, username=None): + """ + Task to load most accessed articles by journal. + + Args: + path (str): File path of the top100 metrics data. + user_id + username + + Returns: + None. + """ + user = _get_user(self.request, username=username, user_id=user_id) + + logging.info(f'Processing file {file_path}') + for row in utils.load_csv(file_path): + try: + models.Top100ArticlesByJournal.create( + user=user, + key_issn=row['pid_issn'], + online_issn=row['online_issn'], + print_issn=row['print_issn'], + year_month_day=row['year_month_day'], + collection=row['collection'], + pid=row['pid'], + yop=row['yop'], + total_item_requests=row['total_item_requests'], + total_item_investigations=row['total_item_investigations'], + unique_item_requests=row['unique_item_requests'], + unique_item_investigations=row['unique_item_investigations'], + ) + except KeyError as e: + logging.info(_(f'File {file_path} does not contain all of the necessary data. Message: {e}')) From 2ff5fc16414f1b230779bfb6bdb31975cb78666f Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Tue, 25 Jun 2024 22:57:40 -0300 Subject: [PATCH 16/19] Atualiza gitignore com pasta de dados de logs solr --- .gitignore | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index f341aa9..5bfb226 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # Solr -index/usage/data -index/data/usage/data/ +index/data/log_manager/data +index/data/metrics/data +index/logs # Byte-compiled / optimized / DLL files __pycache__/ From ac8006da8ccedc75f5bfa9739de58fd8094beb54 Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Tue, 25 Jun 2024 23:16:21 -0300 Subject: [PATCH 17/19] Corrige import --- metrics/wagtail_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metrics/wagtail_hooks.py b/metrics/wagtail_hooks.py index b5a18c7..06a3838 100644 --- a/metrics/wagtail_hooks.py +++ b/metrics/wagtail_hooks.py @@ -4,7 +4,7 @@ from config.menu import get_menu_order -from metrics.models import ( +from .models import ( Top100Articles, ) From 7d0c401056df1ae8b11b9284615b621844b1208e Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Tue, 25 Jun 2024 23:29:58 -0300 Subject: [PATCH 18/19] Recupera arquivo perdido models.py com Top100Articles --- metrics/models.py | 93 ++++++++++++++++++++++++++++++++++++ metrics/tests/test_models.py | 2 +- 2 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 metrics/models.py diff --git a/metrics/models.py b/metrics/models.py new file mode 100644 index 0000000..c17176b --- /dev/null +++ b/metrics/models.py @@ -0,0 +1,93 @@ +from datetime import datetime + +from django.db import models +from django.db.utils import IntegrityError +from django.utils.translation import gettext_lazy as _ +from wagtail.admin.panels import FieldPanel + +from core.models import CommonControlField + + +class Top100Articles(CommonControlField): + key_issn = models.CharField('Key ISSN', max_length=9, null=False, blank=False) + year_month_day = models.DateField('Date of access', null=False, blank=False) + + print_issn = models.CharField('Print ISSN', max_length=9, null=True, blank=True) + online_issn = models.CharField('Online ISSN', max_length=9, null=True, blank=True) + + collection = models.CharField('Collection Acronym 3', max_length=3, null=False, blank=False) + pid = models.CharField('Publication ID', null=False, blank=False) + yop = models.PositiveSmallIntegerField('Year of Publication', null=False, blank=False) + + total_item_requests = models.IntegerField('Total Item Requests', null=False, blank=False) + total_item_investigations = models.IntegerField('Total Item Investigations', null=False, blank=False) + unique_item_requests = models.IntegerField('Unique Item Requests', null=False, blank=False) + unique_item_investigations = models.IntegerField('Unique Item Investigations', null=False, blank=False) + + panels = [ + FieldPanel('key_issn'), + FieldPanel('year_month_day'), + FieldPanel('print_issn'), + FieldPanel('online_issn'), + FieldPanel('collection'), + FieldPanel('pid'), + FieldPanel('yop'), + FieldPanel('total_item_requests'), + FieldPanel('total_item_investigations'), + FieldPanel('unique_item_requests'), + FieldPanel('unique_item_investigations'), + ] + + class Meta: + unique_together = ( + 'collection', + 'key_issn', + 'pid', + 'year_month_day', + ) + verbose_name = _('Top 100 Articles') + indexes = [ + models.Index(fields=['key_issn']), + models.Index(fields=['year_month_day']), + ] + + @classmethod + def create( + cls, + user, + key_issn, + year_month_day, + print_issn, + online_issn, + collection, + pid, + yop, + total_item_requests, + total_item_investigations, + unique_item_requests, + unique_item_investigations, + ): + obj = cls() + obj.creator = user + obj.created = datetime.utcnow() + + obj.key_issn = key_issn + obj.year_month_day = year_month_day + obj.print_issn = print_issn + obj.online_issn = online_issn + obj.collection = collection + obj.pid = pid + obj.yop = yop + obj.total_item_requests = total_item_requests + obj.total_item_investigations = total_item_investigations + obj.unique_item_requests = unique_item_requests + obj.unique_item_investigations = unique_item_investigations + + try: + obj.save() + return obj + except IntegrityError: + raise + + def __str__(self): + return f'{self.key_issn}, {self.pid}, {self.total_item_requests}' diff --git a/metrics/tests/test_models.py b/metrics/tests/test_models.py index 457bb17..9055beb 100644 --- a/metrics/tests/test_models.py +++ b/metrics/tests/test_models.py @@ -2,7 +2,7 @@ from django.test import TestCase from core.models import User -from metrics.models import Top100Articles +from .models import Top100Articles class Top100ArticlesModelTestCase(TestCase): From dfc76ae976413be163da373518f08cb0e2463698 Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Tue, 25 Jun 2024 23:34:20 -0300 Subject: [PATCH 19/19] Corrige imports quebrados --- metrics/tests/test_models.py | 2 +- metrics/wagtail_hooks.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/metrics/tests/test_models.py b/metrics/tests/test_models.py index 9055beb..457bb17 100644 --- a/metrics/tests/test_models.py +++ b/metrics/tests/test_models.py @@ -2,7 +2,7 @@ from django.test import TestCase from core.models import User -from .models import Top100Articles +from metrics.models import Top100Articles class Top100ArticlesModelTestCase(TestCase): diff --git a/metrics/wagtail_hooks.py b/metrics/wagtail_hooks.py index 06a3838..6fc7d50 100644 --- a/metrics/wagtail_hooks.py +++ b/metrics/wagtail_hooks.py @@ -4,9 +4,7 @@ from config.menu import get_menu_order -from .models import ( - Top100Articles, -) +from metrics.models import Top100Articles class Top100Articles(SnippetViewSet):