Release 1.0.0

deeppavlov · Nov 8, 2022 · 1a59449 · 1a59449
2 parents 27ad09e + ef1253c
commit 1a59449
Show file tree

Hide file tree

Showing 569 changed files with 8,450 additions and 64,133 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -10,26 +10,15 @@ node('cuda-module') {
             stage('Setup') {
                 env.TFHUB_CACHE_DIR="tfhub_cache"
                 sh """
-                    virtualenv --python=python3.7 '.venv-$BUILD_NUMBER'
-                    . '.venv-$BUILD_NUMBER/bin/activate'
-                    pip install .[tests,docs]
-                    pip install -r deeppavlov/requirements/tf-gpu.txt
-                    rm -rf `find . -mindepth 1 -maxdepth 1 ! -name tests ! -name Jenkinsfile ! -name docs ! -name '.venv-$BUILD_NUMBER'`
+                    EPOCH=\$(date +%s) docker-compose -f utils/Docker/docker-compose.yml -p $BUILD_TAG build
                 """
             }
             stage('Tests') {
                 sh """
-                    . /etc/profile
-                    module add cuda/10.0
-                    . .venv-$BUILD_NUMBER/bin/activate
-
-                    cd docs
-                    make clean
-                    make html
-                    cd ..
-
-                    flake8 `python -c 'import deeppavlov; print(deeppavlov.__path__[0])'` --count --select=E9,F63,F7,F82 --show-source --statistics
-                    pytest -v --disable-warnings
+                    docker-compose -f utils/Docker/docker-compose.yml -p $BUILD_TAG up py36 py37
+                    docker-compose -f utils/Docker/docker-compose.yml -p $BUILD_TAG ps | grep Exit | grep -v 'Exit 0' && exit 1
+                    docker-compose -f utils/Docker/docker-compose.yml -p $BUILD_TAG up py38 py39
+                    docker-compose -f utils/Docker/docker-compose.yml -p $BUILD_TAG ps | grep Exit | grep -v 'Exit 0' && exit 1 || exit 0
                 """
                 currentBuild.result = 'SUCCESS'
             }
@@ -39,6 +28,10 @@ node('cuda-module') {
             throw e
         }
         finally {
+            sh """
+                docker-compose -f utils/Docker/docker-compose.yml -p $BUILD_TAG rm -f
+                docker network rm \$(echo $BUILD_TAG | awk '{print tolower(\$0)}')_default
+            """
             emailext to: "\${DEFAULT_RECIPIENTS}",
                 subject: "${env.JOB_NAME} - Build # ${currentBuild.number} - ${currentBuild.result}!",
                 body: '${BRANCH_NAME} - ${BUILD_URL}',

diff --git a/README.md b/README.md
@@ -1,10 +1,9 @@
 [![License Apache 2.0](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](LICENSE)
-![Python 3.6, 3.7](https://img.shields.io/badge/python-3.6%20%7C%203.7-green.svg)
+![Python 3.6, 3.7, 3.8, 3.9](https://img.shields.io/badge/python-3.6%20%7C%203.7%20%7C%203.8%20%7C%203.9-green.svg)
 [![Downloads](https://pepy.tech/badge/deeppavlov)](https://pepy.tech/project/deeppavlov)
 <img align="right" height="27%" width="27%" src="docs/_static/deeppavlov_logo.png"/>
 
-DeepPavlov is an open-source conversational AI library built on [TensorFlow](https://www.tensorflow.org/), [Keras](https://keras.io/)
-and [PyTorch](https://pytorch.org/).
+DeepPavlov is an open-source conversational AI library built on [PyTorch](https://pytorch.org/).
 
 DeepPavlov is designed for
 * development of production ready chat-bots and complex conversational systems,
@@ -27,25 +26,19 @@ Please leave us [your feedback](https://forms.gle/i64fowQmiVhMMC7f9) on how we c
 
 **Models**
 
-[Named Entity Recognition](http://docs.deeppavlov.ai/en/master/features/models/ner.html) | [Slot filling](http://docs.deeppavlov.ai/en/master/features/models/slot_filling.html)
+[Named Entity Recognition](http://docs.deeppavlov.ai/en/master/features/models/NER.html) | [Intent/Sentence Classification](http://docs.deeppavlov.ai/en/master/features/models/classifiers.html) |
 
-[Intent/Sentence Classification](http://docs.deeppavlov.ai/en/master/features/models/classifiers.html) |  [Question Answering over Text (SQuAD)](http://docs.deeppavlov.ai/en/master/features/models/squad.html) 
-
-[Knowledge Base Question Answering](http://docs.deeppavlov.ai/en/master/features/models/kbqa.html)
+[Question Answering over Text (SQuAD)](http://docs.deeppavlov.ai/en/master/features/models/SQuAD.html) | [Knowledge Base Question Answering](http://docs.deeppavlov.ai/en/master/features/models/kbqa.html)
 
 [Sentence Similarity/Ranking](http://docs.deeppavlov.ai/en/master/features/models/neural_ranking.html) | [TF-IDF Ranking](http://docs.deeppavlov.ai/en/master/features/models/tfidf_ranking.html) 
 
-[Morphological tagging](http://docs.deeppavlov.ai/en/master/features/models/morphotagger.html) | [Syntactic parsing](http://docs.deeppavlov.ai/en/master/features/models/syntaxparser.html)
-
-[Automatic Spelling Correction](http://docs.deeppavlov.ai/en/master/features/models/spelling_correction.html) | [ELMo training and fine-tuning](http://docs.deeppavlov.ai/en/master/apiref/models/elmo.html)
-
-[Speech recognition and synthesis (ASR and TTS)](http://docs.deeppavlov.ai/en/master/features/models/nemo.html) based on [NVIDIA NeMo](https://nvidia.github.io/NeMo/index.html)
+[Automatic Spelling Correction](http://docs.deeppavlov.ai/en/master/features/models/spelling_correction.html) | [Entity Linking](http://docs.deeppavlov.ai/en/master/features/models/entity_linking.html)
 
-[Entity Linking](http://docs.deeppavlov.ai/en/master/features/models/entity_linking.html) | [Multitask BERT](http://docs.deeppavlov.ai/en/master/features/models/multitask_bert.html)
+[Russian SuperGLUE](http://docs.deeppavlov.ai/en/master/features/models/superglue.html)
 
 **Skills**
 
-[Goal(Task)-oriented Bot](http://docs.deeppavlov.ai/en/master/features/skills/go_bot.html) | [Open Domain Questions Answering](http://docs.deeppavlov.ai/en/master/features/skills/odqa.html)
+[Open Domain Questions Answering](http://docs.deeppavlov.ai/en/master/features/skills/odqa.html)
 
 [Frequently Asked Questions Answering](http://docs.deeppavlov.ai/en/master/features/skills/faq.html)
 
@@ -63,29 +56,21 @@ Please leave us [your feedback](https://forms.gle/i64fowQmiVhMMC7f9) on how we c
 
 **Integrations**
 
-[REST API](http://docs.deeppavlov.ai/en/master/integrations/rest_api.html) | [Socket API](http://docs.deeppavlov.ai/en/master/integrations/socket_api.html) | [Yandex Alice](http://docs.deeppavlov.ai/en/master/integrations/yandex_alice.html)
+[REST API](http://docs.deeppavlov.ai/en/master/integrations/rest_api.html) | [Socket API](http://docs.deeppavlov.ai/en/master/integrations/socket_api.html)
 
-[Telegram](http://docs.deeppavlov.ai/en/master/integrations/telegram.html) | [Microsoft Bot Framework](http://docs.deeppavlov.ai/en/master/integrations/ms_bot.html)
-
-[Amazon Alexa](http://docs.deeppavlov.ai/en/master/integrations/amazon_alexa.html) | [Amazon AWS](http://docs.deeppavlov.ai/en/master/integrations/aws_ec2.html)
+[Amazon AWS](http://docs.deeppavlov.ai/en/master/integrations/aws_ec2.html)
 
 ## Installation
 
-0. We support `Linux` and `Windows` platforms, `Python 3.6` and `Python 3.7`
+0. We support `Linux` platform, `Python 3.6`, `3.7`, `3.8` and `3.9`
     * **`Python 3.5` is not supported!**
-    * **installation for `Windows` requires `Git`(for example, [git](https://git-scm.com/download/win)) and  `Visual Studio 2015/2017` with `C++` build tools installed!**
 
 1. Create and activate a virtual environment:
     * `Linux`
     ```
     python -m venv env
     source ./env/bin/activate
     ```
-    * `Windows`
-    ```
-    python -m venv env
-    .\env\Scripts\activate.bat
-    ```
 2. Install the package inside the environment:
     ```
     pip install deeppavlov
@@ -112,29 +97,8 @@ evaluate and infer it:
 
 #### GPU requirements
 
-To run supported DeepPavlov models on GPU you should have [CUDA](https://developer.nvidia.com/cuda-toolkit) 10.0
-installed on your host machine and TensorFlow with GPU support (`tensorflow-gpu`)
-installed in your python environment. Current supported TensorFlow version is 1.15.2.
-Run
-
-```
-pip install tensorflow-gpu==1.15.2
-```
-
-before installing model's package requirements to install supported `tensorflow-gpu` version.
-
-
-Before making choice of an interface, install model's package requirements
-(CLI):
-
-```bash
-python -m deeppavlov install <config_path>
-```
-
-* where `<config_path>` is path to the chosen model's config file (e.g.
-  `deeppavlov/configs/ner/slotfill_dstc2.json`) or just name without
-  *.json* extension (e.g. `slotfill_dstc2`)
-
+To run supported DeepPavlov models on GPU you should have [CUDA](https://developer.nvidia.com/cuda-toolkit) compatible
+with used GPU and [library PyTorch version](deeppavlov/requirements/pytorch.txt).
 
 ### Command line interface (CLI)
 
@@ -172,10 +136,6 @@ python -m deeppavlov <action> <config_path> [-d]
     * `interact` to interact via CLI,
     * `riseapi` to run a REST API server (see
     [doc](http://docs.deeppavlov.ai/en/master/integrations/rest_api.html)),
-    * `telegram` to run as a Telegram bot (see
-    [doc](http://docs.deeppavlov.ai/en/master/integrations/telegram.html)),
-    * `msbot` to run a Miscrosoft Bot Framework server (see
-    [doc](http://docs.deeppavlov.ai/en/master/integrations/ms_bot.html)),
     * `predict` to get prediction for samples from *stdin* or from
       *<file_path>* if `-f <file_path>` is specified.
 * `<config_path>` specifies path (or name) of model's config file
@@ -228,78 +188,6 @@ from deeppavlov import evaluate_model
 model = evaluate_model(<config_path>, download=True)
 ```
 
-There are also available integrations with various messengers, see
-[Telegram Bot doc page](http://docs.deeppavlov.ai/en/master/integrations/telegram.html)
-and others in the Integrations section for more info.
-
-
-## Breaking Changes
-
-**Breaking changes in version 0.15.0**
-- [bert_as_summarizer](https://github.com/deepmipt/DeepPavlov/pull/1391), [seq2seq_go_bot](https://github.com/deepmipt/DeepPavlov/pull/1434) and all deeppavlov.deprecated components were removed
-- hyperparameter optimization by neural evolution was [removed](https://github.com/deepmipt/DeepPavlov/pull/1436)
-
-**Breaking changes in version 0.7.0**
-- in dialog logger config file [dialog_logger_config.json](deeppavlov/utils/settings/dialog_logger_config.json) `agent_name` parameter was renamed to `logger_name`,
-  the default value was changed
-- Agent, Skill, eCommerce Bot and Pattern Matching classes were moved to [deeppavlov.deprecated](deeppavlov/deprecated)
-- [AIML Skill](http://docs.deeppavlov.ai/en/0.7.0/features/skills/aiml_skill.html),
-  [RASA Skill](http://docs.deeppavlov.ai/en/0.7.0/features/skills/rasa_skill.html),
-  [Yandex Alice](http://docs.deeppavlov.ai/en/0.7.0/integrations/yandex_alice.html),
-  [Amazon Alexa](http://docs.deeppavlov.ai/en/0.7.0/integrations/amazon_alexa.html),
-  [Microsoft Bot Framework](http://docs.deeppavlov.ai/en/0.7.0/integrations/ms_bot.html) and
-  [Telegram integration](http://docs.deeppavlov.ai/en/0.7.0/integrations/telegram.html) interfaces were changed
-- `/start` and `/help` Telegram messages were moved from `models_info.json` to [server_config.json](deeppavlov/utils/settings/server_config.json)
-- [risesocket](http://docs.deeppavlov.ai/en/0.7.0/integrations/socket_api.html) request and response format was changed
-- [riseapi](http://docs.deeppavlov.ai/en/0.7.0/integrations/rest_api.html#advanced-configuration) and
-  [risesocket](http://docs.deeppavlov.ai/en/0.7.0/integrations/socket_api.html#advanced-configuration) model-specific
-  properties parametrization was changed
-
-**Breaking changes in version 0.6.0**
-- [REST API](http://docs.deeppavlov.ai/en/0.6.0/integrations/rest_api.html):
-  - all models default endpoints were renamed to `/model`
-  - by default model arguments names are taken from `chainer.in`
-    [configuration parameter](http://docs.deeppavlov.ai/en/0.6.0/intro/configuration.html) instead of pre-set names
-    from a [settings file](http://docs.deeppavlov.ai/en/0.6.0/integrations/settings.html)
-  - swagger api endpoint moved from `/apidocs` to `/docs`
-- when using `"max_proba": true` in
-  a [`proba2labels` component](http://docs.deeppavlov.ai/en/0.6.0/apiref/models/classifiers.html) for classification,
-  it will return single label for every batch element instead of a list. One can set `"top_n": 1`
-  to get batches of single item lists as before
-
-**Breaking changes in version 0.5.0**
-- dependencies have to be reinstalled for most pipeline configurations
-- models depending on `tensorflow` require `CUDA 10.0` to run on GPU instead of `CUDA 9.0`
-- scikit-learn models have to be redownloaded or retrained
-
-**Breaking changes in version 0.4.0!**
-- default target variable name for [neural evolution](https://docs.deeppavlov.ai/en/0.4.0/intro/hypersearch.html#parameters-evolution-for-deeppavlov-models)
-was changed from `MODELS_PATH` to `MODEL_PATH`.
-
-**Breaking changes in version 0.3.0!**
-- component option `fit_on_batch` in configuration files was removed and replaced with adaptive usage of the `fit_on` parameter.
-
-**Breaking changes in version 0.2.0!**
-- `utils` module was moved from repository root in to `deeppavlov` module
-- `ms_bot_framework_utils`,`server_utils`, `telegram utils` modules was renamed to `ms_bot_framework`, `server` and `telegram` correspondingly
-- rename metric functions `exact_match` to `squad_v2_em` and  `squad_f1` to `squad_v2_f1`
-- replace dashes in configs name with underscores
-
-**Breaking changes in version 0.1.0!**
-- As of `version 0.1.0` all models, embeddings and other downloaded data for provided configurations are
- by default downloaded to the `.deeppavlov` directory in current user's home directory.
- This can be changed on per-model basis by modifying
- a `ROOT_PATH` [variable](http://docs.deeppavlov.ai/en/master/intro/configuration.html#variables)
- or related fields one by one in model's configuration file.
-
-- In configuration files, for all features/models, dataset readers and iterators `"name"` and `"class"` fields are combined
-into the `"class_name"` field.
-
-- `deeppavlov.core.commands.infer.build_model_from_config()` was renamed to `build_model` and can be imported from the
- `deeppavlov` module directly.
-
-- The way arguments are passed to metrics functions during training and evaluation was changed and
- [documented](http://docs.deeppavlov.ai/en/0.4.0/intro/config_description.html#metrics).
 
 ## License
 

diff --git a/deeppavlov/_meta.py b/deeppavlov/_meta.py
@@ -1,4 +1,4 @@
-__version__ = '0.17.6'
+__version__ = '1.0.0'
 __author__ = 'Neural Networks and Deep Learning lab, MIPT'
 __description__ = 'An open source library for building end-to-end dialog systems and training chatbots.'
 __keywords__ = ['NLP', 'NER', 'SQUAD', 'Intents', 'Chatbot']

diff --git a/deeppavlov/configs/classifiers/boolqa_rubert.json b/deeppavlov/configs/classifiers/boolqa_rubert.json
@@ -13,24 +13,21 @@
     "in_y": ["y"],
     "pipe": [
       {
-        "class_name": "bert_preprocessor",
-        "vocab_file": "{DOWNLOADS_PATH}/bert_models/rubert_cased_L-12_H-768_A-12_v1/vocab.txt",
+        "class_name": "torch_transformers_preprocessor",
+        "vocab_file": "{TRANSFORMER}",
         "do_lower_case": false,
         "max_seq_length": 128,
         "in": ["text_a", "text_b"],
         "out": ["bert_features"]
       },
       {
-        "class_name": "bert_classifier",
+        "class_name": "torch_transformers_classifier",
         "n_classes": 2,
-        "one_hot_labels": false,
-        "bert_config_file": "{DOWNLOADS_PATH}/bert_models/rubert_cased_L-12_H-768_A-12_v1/bert_config.json",
-        "pretrained_bert": "{DOWNLOADS_PATH}/bert_models/rubert_cased_L-12_H-768_A-12_v1/bert_model.ckpt",
+        "pretrained_bert": "{TRANSFORMER}",
         "save_path": "{MODELS_PATH}/boolqa_rubert/model_rubert",
         "load_path": "{MODELS_PATH}/boolqa_rubert/model_rubert",
-        "keep_prob": 0.5,
-        "optimizer": "tf.train:AdamOptimizer",
-        "learning_rate": 2e-05,
+        "optimizer": "AdamW",
+        "optimizer_parameters": {"lr": 2e-05},
         "learning_rate_drop_patience": 3,
         "learning_rate_drop_div": 2.0,
         "in": ["bert_features"],
@@ -50,19 +47,14 @@
     "log_every_n_epochs": 1,
     "evaluation_targets": ["valid", "train"],
     "show_examples": false,
-    "tensorboard_log_dir": "{MODELS_PATH}/boolqa_rubert/logs"
+    "class_name": "torch_trainer"
   },
   "metadata": {
     "variables": {
       "ROOT_PATH": "~/.deeppavlov",
       "DOWNLOADS_PATH": "{ROOT_PATH}/downloads",
-      "MODELS_PATH": "{ROOT_PATH}/models"
-    },
-    "download": [
-      {
-        "url": "http://files.deeppavlov.ai/deeppavlov_data/bert/rubert_cased_L-12_H-768_A-12_v1.tar.gz",
-        "subdir": "{DOWNLOADS_PATH}/bert_models"
-      }
-    ]
+      "MODELS_PATH": "{ROOT_PATH}/models",
+      "TRANSFORMER": "DeepPavlov/rubert-base-cased"
+    }
   }
 }
diff --git a/deeppavlov/configs/classifiers/entity_ranking_bert_eng_no_mention.json b/deeppavlov/configs/classifiers/entity_ranking_bert_eng_no_mention.json