Merge branch 'main' of github.com:scieloorg/core

article/sources/xmlsps.py

@@ -13,10 +13,10 @@
 from packtools.sps.models.funding_group import FundingGroup
 from packtools.sps.models.journal_meta import Title as Journal
 from packtools.sps.models.kwd_group import KwdGroup
+from packtools.sps.pid_provider.xml_sps_lib import XMLWithPre
 
 from article import models
 from issue.models import TocSection
-from xmlsps.xml_sps_lib import XMLWithPre
 
 
 class XMLSPSArticleSaveError(Exception):

compose/production/postgres/Dockerfile

@@ -1,4 +1,4 @@
-FROM postgres:14.8
+FROM postgres:15.3
 
 COPY ./compose/production/postgres/maintenance /usr/local/bin/maintenance
 RUN chmod +x /usr/local/bin/maintenance/*

pid_provider/api/v1/views.py

@@ -84,9 +84,10 @@ def create(self, request, format="zip"):
             results = list(results)
             resp_status = None
             for item in results:
+                logging.info(item)
                 try:
                     xml_with_pre = item.pop("xml_with_pre")
-                    if item["xml_changed"]:
+                    if item.get("xml_changed"):
                         item["xml"] = xml_with_pre.tostring()
                 except KeyError:
                     resp_status = status.HTTP_400_BAD_REQUEST

pid_provider/controller.py

@@ -7,11 +7,11 @@
 import requests
 from django.contrib.auth import get_user_model
 from django.utils.translation import gettext as _
+from packtools.sps.pid_provider.xml_sps_lib import XMLWithPre
 from requests.auth import HTTPBasicAuth
 
 from pid_provider import exceptions
 from pid_provider.models import PidProviderConfig, PidProviderXML
-from xmlsps.xml_sps_lib import XMLWithPre
 
 User = get_user_model()
 
@@ -27,7 +27,15 @@ class PidProvider:
     def __init__(self):
         pass
 
-    def provide_pid_for_xml_zip(self, zip_xml_file_path, user):
+    def provide_pid_for_xml_zip(
+        self,
+        zip_xml_file_path,
+        user,
+        filename=None,
+        origin_date=None,
+        force_update=None,
+        is_published=None,
+    ):
         """
         Fornece / Valida PID para o XML em um arquivo compactado
 
@@ -43,6 +51,9 @@ def provide_pid_for_xml_zip(self, zip_xml_file_path, user):
                         xml_with_pre,
                         xml_with_pre.filename,
                         user,
+                        origin_date=origin_date,
+                        force_update=force_update,
+                        is_published=is_published,
                     )
                     registered["filename"] = xml_with_pre.filename
                     logging.info(registered)
@@ -60,7 +71,15 @@ def provide_pid_for_xml_zip(self, zip_xml_file_path, user):
                 "error_type": str(type(e)),
             }
 
-    def provide_pid_for_xml_uri(self, xml_uri, name, user):
+    def provide_pid_for_xml_uri(
+        self,
+        xml_uri,
+        name,
+        user,
+        origin_date=None,
+        force_update=None,
+        is_published=None,
+    ):
         """
         Fornece / Valida PID de um XML disponível por um URI
 
@@ -77,16 +96,34 @@ def provide_pid_for_xml_uri(self, xml_uri, name, user):
                 "error_type": str(type(e)),
             }
         else:
-            return self.provide_pid_for_xml_with_pre(xml_with_pre, name, user)
-
-    def provide_pid_for_xml_with_pre(self, xml_with_pre, name, user):
+            return self.provide_pid_for_xml_with_pre(
+                xml_with_pre,
+                name,
+                user,
+                origin_date=origin_date,
+                force_update=force_update,
+                is_published=is_published,
+            )
+
+    def provide_pid_for_xml_with_pre(
+        self,
+        xml_with_pre,
+        name,
+        user,
+        origin_date=None,
+        force_update=None,
+        is_published=None,
+    ):
         """
         Fornece / Valida PID para o XML no formato de objeto de XMLWithPre
         """
         registered = PidProviderXML.register(
             xml_with_pre,
             name,
             user,
+            origin_date=origin_date,
+            force_update=force_update,
+            is_published=is_published,
         )
         logging.info("")
         registered["xml_with_pre"] = xml_with_pre

pid_provider/migrations/0002_pidproviderxml_origin_date.py

@@ -0,0 +1,19 @@
+# Generated by Django 4.1.10 on 2023-08-02 02:11
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("pid_provider", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="pidproviderxml",
+            name="origin_date",
+            field=models.CharField(
+                blank=True, max_length=10, null=True, verbose_name="Origin date"
+            ),
+        ),
+    ]

pid_provider/models.py

@@ -11,10 +11,11 @@
 from django.db import models
 from django.utils.translation import gettext as _
 from wagtail.admin.panels import FieldPanel
+from packtools.sps.pid_provider import v3_gen, xml_sps_adapter
 
 from core.forms import CoreAdminModelForm
 from core.models import CommonControlField
-from pid_provider import exceptions, v3_gen, xml_sps_adapter
+from pid_provider import exceptions
 from xmlsps.models import XMLSPS, XMLIssue, XMLJournal, XMLVersion
 
 LOGGER = logging.getLogger(__name__)
@@ -288,6 +289,9 @@ class PidProviderXML(CommonControlField):
     z_partial_body = models.CharField(
         _("partial_body"), max_length=64, null=True, blank=True
     )
+    origin_date = models.CharField(
+        _("Origin date"), max_length=10, null=True, blank=True
+    )
 
     base_form_class = CoreAdminModelForm
 
@@ -370,6 +374,8 @@ def register(
         xml_with_pre,
         filename,
         user,
+        origin_date=None,
+        force_update=None,
         is_published=False,
     ):
         """
@@ -412,12 +418,12 @@ def register(
             # consulta se documento já está registrado
             registered = cls._query_document(xml_adapter)
 
-            if registered and registered.is_equal_to(xml_adapter):
-                # retorna item registrado
-                return registered.data
-
             # analisa se aceita ou rejeita registro
-            cls.evaluate_registration(xml_adapter, registered)
+            updated_data = cls.skip_registration(
+                xml_adapter, registered, force_update, origin_date
+            )
+            if updated_data:
+                return updated_data
 
             # verfica os PIDs encontrados no XML / atualiza-os se necessário
             changed_pids = cls._complete_pids(xml_adapter, registered)
@@ -432,7 +438,9 @@ def register(
                 )
 
             # cria ou atualiza registro
-            registered = cls._save(registered, xml_adapter, user, changed_pids)
+            registered = cls._save(
+                registered, xml_adapter, user, changed_pids, origin_date
+            )
 
             # cria ou atualiza XMLSPS
             registered._create_or_update_xmlsps(user, is_published)
@@ -475,6 +483,7 @@ def _save(
         xml_adapter,
         user,
         changed_pids,
+        origin_date=None,
     ):
         if registered:
             registered.updated_by = user
@@ -484,6 +493,7 @@ def _save(
             registered.creator = user
             registered.created = utcnow()
 
+        registered.origin_date = origin_date
         registered._add_data(xml_adapter, user)
         registered._add_journal(xml_adapter)
         registered._add_issue(xml_adapter, registered.journal)
@@ -497,22 +507,44 @@ def _save(
         return registered
 
     @classmethod
-    def evaluate_registration(cls, xml_adapter, registered):
+    def skip_registration(cls, xml_adapter, registered, force_update, origin_date):
         """
         XML é versão AOP, mas
         documento está registrado com versão VoR (fascículo),
         então, recusar o registro,
         pois está tentando registrar uma versão desatualizada
         """
-        logging.info("PidProviderXML.evaluate_registration")
+        logging.info("PidProviderXML.skip_registration")
+
+        if force_update:
+            return
+
+        if not registered:
+            return
+
+        # verifica se é necessário atualizar
+        if registered.is_equal_to(xml_adapter):
+            # XML fornecido é igual ao registrado, não precisa continuar
+            logging.info(f"Skip update: equal")
+            return registered.data
+
         if xml_adapter.is_aop and registered and not registered.is_aop:
+            logging.info(f"Skip update: forbidden")
             raise exceptions.ForbiddenPidProviderXMLRegistrationError(
                 _(
                     "The XML content is an ahead of print version "
                     "but the document {} is already published in an issue"
                 ).format(registered)
             )
-        return True
+
+        if (
+            origin_date
+            and registered.origin_date
+            and registered.origin_date > origin_date
+        ):
+            # retorna item registrado que está mais atualizado
+            logging.info(f"Skip update: is already up-to-date")
+            return registered.data
 
     def is_equal_to(self, xml_adapter):
         return bool(
@@ -579,18 +611,18 @@ def _query_document(cls, xml_adapter):
         items = xml_adapter.query_list
         for params in items:
             cls.validate_query_params(params)
-            xml_adapter.adapt_query_params(params)
+            adapted_params = xml_adapter.adapt_query_params(params)
 
             try:
-                return cls.objects.get(**params)
+                return cls.objects.get(**adapted_params)
             except cls.DoesNotExist:
                 continue
             except cls.MultipleObjectsReturned as e:
                 # seria inesperado já que os dados informados devem encontrar
                 # ocorrência única ou None
-                logging.info(f"params={params} | e={e}")
+                logging.info(f"params={adapted_params} | e={e}")
                 raise exceptions.QueryDocumentMultipleObjectsReturnedError(
-                    _("Found more than one document matching to {}").format(params)
+                    _("Found more than one document matching to {}").format(adapted_params)
                 )
 
     def _add_data(self, xml_adapter, user):

pid_provider/scripts/provide_pid_for_am_xmls.py

@@ -1,40 +1,16 @@
 from pid_provider import tasks
 
 
-def run(username, collection_acron=None, pids=None):
-    if not collection_acron and not pids:
-        return tasks.harvest_pids.apply_async(
-            kwargs={
-                "username": username,
-                "stop": 100,
-            }
-        )
-
-    if not pids:
-        return tasks.harvest_pids.apply_async(
-            kwargs={
-                "username": username,
-                "collection_acron": collection_acron,
-                "stop": 100,
-            }
-        )
-
-    if "," in pids:
-        items = [
-            {"collection_acron": collection_acron, "pid_v2": pid_v2}
-            for pid_v2 in pids.split(",")
-        ]
-        return tasks.provide_pid_for_am_xmls.apply_async(
-            kwargs={
-                "username": username,
-                "items": items,
-            }
-        )
-
-    return tasks.provide_pid_for_am_xml.apply_async(
+def run(
+    username, collection_acron, from_date=None, limit=None, stop=None, force_update=None
+):
+    return tasks.harvest_pids.apply_async(
         kwargs={
             "username": username,
             "collection_acron": collection_acron,
-            "pid_v2": pids,
+            "from_date": from_date or "1900-01-01",
+            "limit": int(limit or 1000),
+            "force_update": force_update or False,
+            "stop": int(stop or 1000),
         }
     )

pid_provider/sources/am.py

@@ -1,7 +1,7 @@
 import logging
 
 from pid_provider.controller import PidProvider
-from pid_provider.models import PidRequest
+from pid_provider.models import PidRequest, PidProviderXML
 
 
 class AMHarvesting:
@@ -48,7 +48,7 @@ def __init__(self, collection_acron=None, from_date=None, limit=None, stop=None)
             atribuir valor para forçar a interrompção das requisições antes de coletar tudo
         """
         self._collection_acron = collection_acron
-        self._limit = limit or 10
+        self._limit = limit or 1000
         self._offset = 0
         self._total = None
         self._stop = stop
@@ -107,11 +107,25 @@ def uris(self):
                 break
 
 
-def request_pid_v3(user, uri, collection_acron, pid_v2, processing_date):
-    pp = PidProvider()
+def request_pid_v3(user, uri, collection_acron, pid_v2, processing_date, force_update):
+    if not force_update:
+        # skip update
+        try:
+            return PidProviderXML.objects.get(v2=pid_v2).data
+        except PidProviderXML.DoesNotExist:
+            pass
 
     try:
-        response = pp.provide_pid_for_xml_uri(uri, pid_v2 + ".xml", user)
+        logging.info(f"Request pid for {uri}")
+        pp = PidProvider()
+        response = pp.provide_pid_for_xml_uri(
+            uri,
+            pid_v2 + ".xml",
+            user,
+            force_update=force_update,
+            origin_date=processing_date,
+            is_published=True,
+        )
     except Exception as e:
         return PidRequest.register_failure(
             e=e,