Merge pull request #1175 from scieloorg/beta

Incorporação de códigos estáveis

docs/dev/rpc_spec/getScanArticlesBatch.rst

@@ -31,3 +31,6 @@ Exemplo::
 
             batch_id = result.next_batch_id
 
+
+.. note:: A lista de resultados de um lote não é ordenada por data ou qualquer 
+          campo. 

docs/dev/rpc_spec/scanArticles.rst

@@ -42,6 +42,8 @@ Os índices que podem ser utilizados na consulta são:
 +----------------------+------------------------------------------------------+
 | source               | O documento XML codificado em utf-8.                 |
 +----------------------+------------------------------------------------------+
+| timestamp            | Data e hora conforme ISO-8601 (combined).            |
++----------------------+------------------------------------------------------+
 
 
 .. note:: A quantidade de registros por lote não é definida pelo cliente, e 
@@ -51,8 +53,39 @@ Os índices que podem ser utilizados na consulta são:
 Exemplo::
 
     batch_id = client.scanArticles('{"query": {"match": {"year": "2013"}}}')
-    next_batch_id, articles = client.getScanArticlesBatch(batch_id)
+    results = client.getScanArticlesBatch(batch_id)
 
-    for article in articles:
+    for article in results.articles:
         ...
 
+
+Casos de uso
+------------
+
+1. Coleta incremental de lotes, com base na data e hora de modificação dos 
+   registros::
+
+    {
+        "query": {
+            "filtered": {
+                "query": {
+                    "match_all": {}
+                },
+                "filter": {
+                    "range": {
+                        "timestamp": {
+                            "gte": "2015-08-04T02:04:37.993423",
+                            "lte": "now"
+                        }
+                    }
+                }
+            }
+        }     
+    }
+
+
+É importante notar que os resultados da consulta '''não''' são ordenados, mesmo 
+quando utilizada a instrução ``sort``. O cliente deverá identificar o maior 
+valor de ``timestamp`` entre os registros recuperados para a recuperação do 
+próximo incremento.
+

requirements.txt

@@ -1,4 +1,4 @@
-Django==1.4.10
+Django<1.5
 South
 psycopg2==2.4.1
 wsgiref==0.1.2

scielomanager/accounts/templates/accounts/my_account.html

@@ -142,7 +142,7 @@ <h4>{% trans "API Token" %}:</h4>
  <script>
     $(document).ready(function() {
       $('input', '#change_password_form').removeClass('span3').addClass('span12');
-      $('select', '#profile_form').addClass('span12').chosen();
+      $('select', '#profile_form').addClass('span12').chosen(defaultChosenOptions);
     });
   </script>
 {% endblock %}

scielomanager/editorialmanager/templates/board/board_member_delete.html

@@ -32,10 +32,9 @@ <h2>{% trans "Delete Board Member" %}?:</h2>
         $('#board-member-form').find('form').submit();
       });
       $('input').addClass('span12');
-      $(".chzn-select").chosen({
-        no_results_text: "{% trans 'No results found for' %}:",
-        width: "100%",
-      });
+      var chosenOptions = defaultChosenOptions;
+      chosenOptions['width'] = "100%";
+      $(".chzn-select").chosen(chosenOptions);
     });
   </script>
 {% endblock %}

scielomanager/editorialmanager/templates/board/board_member_edit.html

@@ -33,10 +33,9 @@ <h2>{% trans "Edit Board Member" %}:</h2>
         $('#board-member-form').find('form').submit();
       });
       $('input').addClass('span12');
-      $(".chzn-select").chosen({
-        no_results_text: "{% trans 'No results found for' %}:",
-        width: "100%",
-      });
+      var chosenOptions = defaultChosenOptions;
+      chosenOptions['width'] = "100%";
+      $(".chzn-select").chosen(chosenOptions);
       {% include "includes/add_member_form_loaded_callbacks.js" %}
     });
   </script>

scielomanager/editorialmanager/templates/journal/edit_journal.html

@@ -142,7 +142,7 @@ <h2>{% trans "Editor" %}:</h2>
       prefix: '{{ missionformset.prefix }}'
     });
 
-    $(".chzn-select").chosen({no_results_text: "{% trans 'No results found for' %}:"});
+    $(".chzn-select").chosen(defaultChosenOptions);
 
   </script>
 {% endblock %}

scielomanager/journalmanager/tasks.py

@@ -62,6 +62,7 @@ def _gen_es_struct_from_article(article):
         'is_aop': article.is_aop,
         'source': article_as_octets,
         'aid': article.aid,
+        'timestamp': datetime.datetime.now(),
     }
 
     es_struct.update(partial_struct)

scielomanager/journalmanager/templates/journalmanager/add_user.html

@@ -205,7 +205,7 @@ <h2>{% trans "User Collections" %}:</h2>
 {{ block.super }}
  <script>
     $(document).ready(function() {
-      $('.chosen-select').chosen()
+      $('.chosen-select').chosen(defaultChosenOptions)
       $( "#id_user-collection" ).combobox({ autoFocus: true });
       $(".help-text").popover('hide');
       $( "#toggle" ).click(function() {

scielomanager/journalmanager/templates/journalmanager/editor.html

@@ -69,12 +69,12 @@
 
      });
 
-    $(".chzn-select").chosen({no_results_text: "Nenhum resultado encontrado para: "})
+    $(".chzn-select").chosen(defaultChosenOptions)
 
     $('#modal').on('shown', function () {
 
       $("select[name='editor']").attr('class', 'chzn-select span12');
-      $(".chzn-select").chosen({no_results_text: "Nenhum resultado encontrado para: "});
+      $(".chzn-select").chosen(defaultChosenOptions);
 
     });
 

scielomanager/journalmanager/templates/journalmanager/user_list.html

@@ -82,6 +82,6 @@
 
 {% block extrafooter %}
   <script>
-    $(".chzn-select").chosen({no_results_text: "No users found for:"})
+    $(".chzn-select").chosen(defaultChosenOptions)
   </script>
 {% endblock %}

scielomanager/scielomanager/connectors/__init__.py

@@ -4,10 +4,15 @@
 Um conector é apenas um link entre dois ou mais elementos conectáveis (e.g.,
 portas, interfaces etc).
 """
+import os
 
 # se apoia na definição da variável __all__ nos módulos.
 from .storage import *
 
 
-__all__ = (storage.__all__,)
+_CWD = os.path.dirname(os.path.abspath(__file__))
+ES_ARTICLE_MAPPING_PATH = os.path.join(_CWD, 'article_mapping.json')
+
+
+__all__ = (storage.__all__, ES_ARTICLE_MAPPING_PATH)
 

scielomanager/scielomanager/connectors/article_mapping.json

@@ -0,0 +1,59 @@
+{
+   "article": {
+      "_timestamp": {
+          "enabled": true,
+          "path": "timestamp"
+      },
+      "properties": {
+         "abbrev_journal_title": {
+            "type": "string"
+         },
+         "aid": {
+            "type": "string",
+            "index": "not_analyzed"
+         },
+         "article_type": {
+            "type": "string",
+            "index": "not_analyzed"
+         },
+         "doi": {
+            "type": "string",
+            "index": "not_analyzed"
+         },
+         "epub": {
+            "type": "string",
+            "index": "not_analyzed"
+         },
+         "head_subject": {
+            "type": "string"
+         },
+         "is_aop": {
+            "type": "boolean"
+         },
+         "issue": {
+            "type": "string",
+            "index": "not_analyzed"
+         },
+         "pid": {
+            "type": "string",
+            "index": "not_analyzed"
+         },
+         "ppub": {
+            "type": "string",
+            "index": "not_analyzed"
+         },
+         "version": {
+            "type": "string",
+            "index": "not_analyzed"
+         },
+         "volume": {
+            "type": "string",
+            "index": "not_analyzed"
+         },
+         "year": {
+            "type": "date",
+            "format": "yyyy"
+         }
+      }
+   }
+}

scielomanager/scielomanager/management/commands/es.py

@@ -0,0 +1,90 @@
+# coding: utf-8
+"""Tarefas de manutenção da integração com o Elasticsearch.
+"""
+from django.core.management.base import BaseCommand, CommandError
+import elasticsearch
+
+from scielomanager import connectors
+
+
+_TYPE_CLIENT_MAP = {
+        'article': connectors.ArticleElasticsearch,
+}
+
+
+_HELP = u"""\
+Executa tarefas de manutenção da integração com o Elasticsearch.
+
+Os comandos são:
+    putmapping <type>     Adiciona ou atualiza o mapping de um tipo.
+    createindex <type>    Cria o índice e define o mapping do tipo.
+
+Os valores de <type> são:
+    %s
+""" % (_TYPE_CLIENT_MAP.keys(),)
+
+
+def _to_bytestring(text):
+    return text.encode('utf-8')
+
+
+def _get_article_mapping():
+    with open(connectors.ES_ARTICLE_MAPPING_PATH, 'r') as mapping_file:
+        mapping_data = mapping_file.read()
+
+    return mapping_data
+
+
+def _put_mapping(type):
+    """
+    """
+    client_cls = _TYPE_CLIENT_MAP.get(type)
+    if client_cls is None:
+        raise CommandError(
+                _to_bytestring(u'"%s" não é um tipo válido' % type))
+
+    client = client_cls()
+    mapping_data = _get_article_mapping()
+
+    client.es_client.indices.put_mapping(index=client.index,
+            doc_type=client.doctype, body=mapping_data, ignore_conflicts=False,
+            timeout=2)
+
+
+def _create_index(type):
+    client_cls = _TYPE_CLIENT_MAP.get(type)
+    if client_cls is None:
+        raise CommandError(
+                _to_bytestring(u'"%s" não é um tipo válido' % type))
+
+    client = client_cls()
+
+    client.es_client.indices.create(index=client.index, timeout=2)
+    _put_mapping(type)
+
+
+class Command(BaseCommand):
+    args = '<command> [<args>]'
+    help = _HELP
+    requires_model_validation = False
+
+    def handle(self, *args, **options):
+        if not args:
+            raise CommandError(
+                    _to_bytestring("'./manage.py help es' para ajuda."))
+
+        command = args[0]
+        arguments = args[1:]
+
+        try:
+            if command == 'putmapping':
+                _put_mapping(*arguments)
+            elif command == 'createindex':
+                _create_index(*arguments)
+            else:
+                raise CommandError(
+                        _to_bytestring(u'Comando inválido'))
+
+        except elasticsearch.exceptions.TransportError as exc:
+            raise CommandError(str(exc))
+

scielomanager/scielomanager/settings.py

@@ -177,6 +177,7 @@
     'export',
     'health',
     'thrift',
+    'scielomanager',  # apenas para management commands
 )
 
 TEMPLATE_CONTEXT_PROCESSORS = (

scielomanager/scielomanager/static/js/modal.js

@@ -35,10 +35,9 @@ $(function() {
   });
   $('.modal').on('loaded', function () {
     $(this).find('input').addClass('span12');
-    $(this).find(".chzn-select").chosen({
-      no_results_text: "{% trans 'No results found for' %}:",
-      width: "100%",
-    });
+    var chosenOptions = defaultChosenOptions;
+    chosenOptions['width'] = "100%";
+    $(this).find(".chzn-select").chosen(chosenOptions);
   })
 
   /* actions: */

scielomanager/scielomanager/templates/base_lv0.html

@@ -22,6 +22,9 @@
     <script type="text/javascript" src="{{ ASSET_URL }}"></script>
   {% endassets %}
   <script type="text/javascript">
+    var defaultChosenOptions = {
+      'no_results_text': "{% trans 'No results found for' %}:",
+    }
     $(document).ready(function(){
       $('#list_model').change(function(){
         $('#search-form').attr('action', $('#list_model :selected').val());

scielomanager/thrift/scielomanager.thrift

@@ -31,7 +31,7 @@ namespace py scielomanager
  * IMPORTANTE! Alterar o valor de VERSION após qualquer alteração na interface.
  * Regras em: http://semver.org/lang/pt-BR/
  */
-const string VERSION = "1.0.0"
+const string VERSION = "1.1.0"
 
 
 #
@@ -72,6 +72,7 @@ struct Article {
     12: required string version;
     13: optional bool is_aop;
     14: required string source;
+    15: required string timestamp;
 }
 
 /*