installation: fix python packages dependencies

* Fixes strange behaviours during the resolution dependencies with
pipenv. This will make the new python packages easy.
* Adds the documentation in the python packages file (Pipefile).
* Adds support of a new version of the python import order check tool (isort>=4.3.10).
* Adds new bootstrap and server command for pipenv.
* Replaces scripts commands by pipenv run:
  * bootstrap
  * console
  * server
  * setup
  * update
* Adds virtualenv detection in scripts/* for python>3.6.4.

Co-Authored-by: Johnny Mariéthoz <[email protected]>
Co-Authored-by: Olivier DOSSMANN <[email protected]>

.editorconfig

@@ -39,6 +39,8 @@ known_first_party = rero_ils
 multi_line_output = 2
 default_section = THIRDPARTY
 skip = .eggs
+# Needed for isort since version 4.3.10
+reverse_relative = true
 
 # RST files (used by sphinx)
 [*.rst]

Dockerfile

@@ -36,4 +36,4 @@ ARG UI_TGZ=""
 
 ENV INVENIO_COLLECT_STORAGE='flask_collect.storage.file'
 
-RUN ./scripts/bootstrap --deploy ${UI_TGZ} && rm -fr ui/node_modules
+RUN pipenv run bootstrap --deploy ${UI_TGZ} && rm -fr ui/node_modules

INSTALL.rst

@@ -60,7 +60,7 @@ all Python dependencies and build all static assets):
 .. code-block:: console
 
     $ cd rero-ils
-    $ ./scripts/bootstrap
+    $ pipenv run bootstrap
 
 Start all dependent services using docker-compose (this will start PostgreSQL,
 Elasticsearch 6, RabbitMQ and Redis):
@@ -97,21 +97,21 @@ Start the webserver and the celery worker:
 
 .. code-block:: console
 
-    $ ./scripts/server
+    $ pipenv run server
 
 Start a Python shell:
 
 .. code-block:: console
 
-    $ ./scripts/console
+    $ pipenv run console
 
 Upgrading
 ---------
 In order to upgrade an existing instance simply run:
 
 .. code-block:: console
 
-    $ ./scripts/update
+    $ pipenv run update
 
 Testing
 -------

Pipfile

@@ -1,62 +1,92 @@
+# RERO ILS
+# Copyright (C) 2020 RERO
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
 [[source]]
 url = "https://pypi.org/simple"
 verify_ssl = true
 name = "pypi"
 
 [packages]
-# TODO: try to remove it when werkzeug will be fixed.
-werkzeug = "==0.16.0"
-# TODO: check why we need this.
-flask-login = "<0.5"
+## Python packages dependencies (order matters)
+#------------------------------------------------------------------------------
+## Note: pipenv evaluates the package constraints from the top to the bottom
+## and respects only the last constraint to install the package.
+## Thus the most strength constraint should be at the end.
+## Imagine the following package list for the foo package with the
+## most recent version equal to 5.0:
+## A foo Pipfile example:
+## ----------------------
+## foo = ">1.0" # first will install 5.0
+## foo = "==1.0" # then will downgrade to 1.0
+## foo = ">3.0" # finally will upgrade to 5.0
+## when the python code will be executed a conflict exception will be raised.
+#------------------------------------------------------------------------------
 # Default from Invenio
 Babel = ">=2.4.0"
 Flask-BabelEx = ">=0.9.3"
-uwsgi = ">=2.0"
-uwsgitop = ">=0.11"
-uwsgi-tools = ">=1.1.1"
-lxml = ">=3.5.0,<4.2.6"
-# RERO ILS specific python packages
-PyYAML = ">=3.13"
-dateparser = ">=0.7.0"
-isbnlib = ">=3.9.1"
-requests = ">=2.20.0"
-pycountry = ">=19.7.15"
-# flask-oauthlib = "*"
-# requests-oauthlib = "==1.1"
-# TODO: to be removed when the thumbnail will be refactored
-angular-gettext-babel= ">=0.1"
-# TODO: causes datetime serialize error
-celery = "<4.3.0"
+## Third party invenio modules used by RERO ILS
 invenio-oaiharvester = {editable = true,ref = "v1.0.0a4",git = "https://github.com/inveniosoftware/invenio-oaiharvester.git"}
 invenio-circulation = {editable = true,git = "https://github.com/inveniosoftware/invenio-circulation.git",ref = "v1.0.0a16"}
-invenio = {version = "~=3.1.0",extras = ["base", "postgresql", "auth", "elasticsearch6" ]}
-dojson = ">=1.3.2"
-invenio-accounts = "<=1.1.2"
-# get rid of invenio-search-ui
+## Invenio base modules used by RERO ILS
+# same as invenio metadata extras without invenio-search-ui
 invenio-indexer = ">=1.0.1,<1.1.0"
 invenio-jsonschemas = ">=1.0.0,<1.1.0"
 invenio-oaiserver = ">=1.0.3,<1.1.0"
 invenio-pidstore = ">=1.0.0,<1.1.0"
 invenio-records-rest = ">=1.4.0,<1.5.0"
 invenio-records-ui = ">=1.0.1,<1.1.0"
-# separate tables
-invenio-records = ">=1.3.0"
-# TODO: remove this contraint once it is solved in invenio
-raven = {version = ">=6.0",extras = ["flask"]}
-# self install
-rero-ils = {editable = true,path = "."}
+## Default from Invenio
+invenio = {version = "~=3.1.0",extras = ["base", "postgresql", "auth", "elasticsearch6"]}
+uwsgi = ">=2.0"
+uwsgitop = ">=0.11"
+uwsgi-tools = ">=1.1.1"
+## Additionnal constraints on invenio base modules
+# TODO: remove when https://github.com/inveniosoftware/invenio-accounts/issues/306 will be solved
+invenio-accounts = "<=1.1.2"
 # TODO: remove when DynamicPermission will be solved
 invenio-access = "<1.2.0"
+# separate tables
+invenio-records = ">=1.3.0"
+## RERO ILS specific python modules
+PyYAML = ">=3.13"
+dateparser = ">=0.7.0"
+isbnlib = ">=3.9.1"
+requests = ">=2.20.0"
 polib = "*"
+pycountry = ">=19.7.15"
 xmltodict = "*"
+# TODO: to be removed when the thumbnail will be refactored
+angular-gettext-babel= ">=0.1"
+## Additionnal constraints on python modules
+# solves datetime serialize error
+celery = "<4.3.0"
+## Self install
+rero-ils = {editable = true,path = "."}
+## Deployment
+raven = "*"
+lxml = ">=3.5.0,<4.2.6"
 
 [dev-packages]
+## Python packages development dependencies (order matters)
+#----------------------------------------------------------
+## Default from Invenio
 Flask-Debugtoolbar = ">=0.10.1"
 Sphinx = ">=1.5.1"
-autoflake = ">=1.3.1"
 check-manifest = ">=0.35"
 coverage = ">=4.4.1"
-isort = ">=4.3.5,<4.3.10"
+isort = ">=4.3"
 mock = ">=2.0.0"
 marshmallow = ">=2.15.1,<3.0.0"
 pydocstyle = ">=2.0.0"
@@ -69,6 +99,8 @@ pytest-random-order = ">=0.5.4"
 pytest-runner = ">=3.0.0,<5"
 # FIXME: Remove this once this issue is resolved https://github.com/pypa/pipenv/issues/3865.
 docutils = "==0.15"
+## RERO ILS specific python packages
+autoflake = ">=1.3.1"
 transifex-client = ">=0.12.5"
 
 [requires]
@@ -78,4 +110,8 @@ python_version = "3.6"
 test = "python setup.py test"
 build_sphinx = "python setup.py build_sphinx"
 dev = "pytest --no-cov -vs tests"
+bootstrap = "./scripts/bootstrap"
+console = "./scripts/console"
 setup = "./scripts/setup"
+server = "./scripts/server"
+update = "./scripts/update"

scripts/bootstrap

@@ -92,6 +92,11 @@ do
   shift
 done
 
+# PIPENV is a mandatory condition to launch this program!
+if [[ -z  "${VIRTUAL_ENV}" ]]; then
+  error_msg+exit "Error - Launch this script via pipenv command:\n\tpipenv run ${PROGRAM}"
+fi
+
 if ! $deploy || $ci ; then
     flags=("--dev")
 fi
@@ -113,40 +118,40 @@ ${cmd} ${flags[@]}
 # to avoid IPython dependency problem on Mac OS X
 if [ "$(uname -s)" == "Darwin" ]; then
   info_msg "Install 'appnope' for Mac OS X"
-  pipenv run pip install appnope
+  pip install appnope
 fi
 
 # install assets utils
 virtualenv_path=`pipenv --venv`
 info_msg "Install npm assets utils in: ${virtualenv_path}"
-pipenv run npm i npm@latest -g --prefix "${virtualenv_path}" && pipenv run npm install --prefix "${virtualenv_path}" --silent -g [email protected] [email protected] [email protected] [email protected] @angular/[email protected] yarn
+npm i npm@latest -g --prefix "${virtualenv_path}" && npm install --prefix "${virtualenv_path}" --silent -g [email protected] [email protected] [email protected] [email protected] @angular/[email protected] yarn
 
 # collect static files and compile html/css assets
 # ------------------------------------------------
 # install the npm dependencies
 info_msg "Install npm dependencies"
-pipenv run invenio npm
+invenio npm
 info_msg "Search static folder location"
 static_folder=$(pipenv run invenio shell --no-term-title -c "print('static_folder:%s' % app.static_folder)"|grep static_folder| cut -d: -f2-)
 info_msg "Install static folder npm dependencies in: ${static_folder}"
-pipenv run npm install --prefix "${static_folder}"
+npm install --prefix "${static_folder}"
 if [ -f ${tgz_file} ]
 then
   info_msg "Install RERO-ILS-UI from tgz: ${tgz_file}"
-  pipenv run npm install "${tgz_file}" --prefix "${static_folder}"
+  npm install "${tgz_file}" --prefix "${static_folder}"
 fi
 
 # build the web assets
 info_msg "Build web assets: collect"
-pipenv run invenio collect -v
+invenio collect -v
 info_msg "Build web assets: check (build command)"
-pipenv run invenio assets build
+invenio assets build
 
 # compile json files (resolve $ref)
 info_msg "Compile JSON files to resolve \$ref"
 echo
-pipenv run invenio utils compile_json ./rero_ils/modules/documents/jsonschemas/documents/document-minimal-v0.0.1_src.json -o ./rero_ils/modules/documents/jsonschemas/documents/document-minimal-v0.0.1.json
-pipenv run invenio utils compile_json ./rero_ils/modules/documents/jsonschemas/documents/document-v0.0.1_src.json -o ./rero_ils/modules/documents/jsonschemas/documents/document-v0.0.1.json
+invenio utils compile_json ./rero_ils/modules/documents/jsonschemas/documents/document-minimal-v0.0.1_src.json -o ./rero_ils/modules/documents/jsonschemas/documents/document-minimal-v0.0.1.json
+invenio utils compile_json ./rero_ils/modules/documents/jsonschemas/documents/document-v0.0.1_src.json -o ./rero_ils/modules/documents/jsonschemas/documents/document-v0.0.1.json
 
 success_msg "${PROGRAM} finished!"
 exit 0