Ray_with_weight_decay.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Ray with weight_decay.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyOXYci+jhjBBtxnwazdT6lP",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/olgasem10/Unsafe/blob/main/Ray_with_weight_decay.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NU9VZeyYjRLR"
      },
      "source": [
        "!pip install ray[tune]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "40eBULRBxCJ-"
      },
      "source": [
        "!pip install transformers"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "8W4GCnPhwYRU"
      },
      "source": [
        "from transformers import BertTokenizer, BertForSequenceClassification, Trainer, TrainingArguments\n",
        "import torch\n",
        "from torch.utils.data import Dataset\n",
        "from sklearn.metrics import accuracy_score, precision_recall_fscore_support, roc_auc_score\n",
        "import numpy as np\n",
        "import pandas as pd"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "K0jUe33twYRb"
      },
      "source": [
        "### Данные (с добавлением спец токена)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AD13wCHKw0-Z",
        "outputId": "b43688d3-aff2-43be-95fe-12b1f20d4bdb"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "gDkrwx_fwYRc"
      },
      "source": [
        "train_df = pd.read_csv('/content/drive/MyDrive/internship/data/train_randst0.csv')\n",
        "test_df = pd.read_csv('/content/drive/MyDrive/internship/data/val_randst0.csv')\n",
        "#train_df = train_df[[\"text\", \"unsafe\"]]\n",
        "#test_df = test_df[[\"text\", \"unsafe\"]]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "NQfSJcEfwYRc"
      },
      "source": [
        "train_df = train_df.loc[(train_df['unsafe'] >= 0.8) | (train_df['unsafe'] <= 0.2)]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "ageVXh5cwYRd"
      },
      "source": [
        "translator = {\n",
        "    'crime_real' : 'криминал',\n",
        "    'crime_web' : 'веб-криминал',\n",
        "    'drugs' : 'наркотики',\n",
        "    'gambling' : 'азартные игры',\n",
        "    'pornography' : 'порнография',\n",
        "    'prostitution' : 'проституция',\n",
        "    'slavery' : 'рабство',\n",
        "    'suicide' : 'суицид',\n",
        "    'terrorism' : 'терроризм',\n",
        "    'weapons' : 'оружие', \n",
        "    'body_shaming' : 'уродства',\n",
        "    'halth_shaming' : 'инвалидность',\n",
        "    'politics' : 'политика',\n",
        "    'racism' : 'расизм',\n",
        "    'religion' : 'религия',\n",
        "    'sex_minorities' : 'гомосексуализм',\n",
        "    'sexism' : 'сексизм',\n",
        "    'social' : 'социальное',\n",
        "    'none' : ''\n",
        "}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "IMSsdZ23wYRd"
      },
      "source": [
        "col_names = ['crime_real',\n",
        " 'crime_web',\n",
        " 'drugs',\n",
        " 'gambling',\n",
        " 'pornography',\n",
        " 'prostitution',\n",
        " 'slavery',\n",
        " 'suicide',\n",
        " 'terrorism',\n",
        " 'weapons',\n",
        " 'body_shaming',\n",
        " 'halth_shaming',\n",
        " 'politics',\n",
        " 'racism',\n",
        " 'religion',\n",
        " 'sex_minorities',\n",
        " 'sexism',\n",
        " 'social']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "Qqal9_MswYRd"
      },
      "source": [
        "def binary(val):\n",
        "    return round(val)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "Ecz9np7AwYRe"
      },
      "source": [
        "for name in col_names:\n",
        "    train_df[name] = train_df[name].apply(binary)\n",
        "    test_df[name] = test_df[name].apply(binary)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "TClIDbMgwYRe"
      },
      "source": [
        "train_df['unsafe'] = train_df['unsafe'].apply(binary)\n",
        "test_df['unsafe'] = test_df['unsafe'].apply(binary)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "8rSn7SWAwYRe"
      },
      "source": [
        "def add_special_tokens(dataframe):\n",
        "    new_dataframe = dataframe.copy()\n",
        "    for column in col_names:\n",
        "        indices = dataframe.index[dataframe[column] == 1]\n",
        "        new_dataframe.loc[indices, ['text']] = '<' + translator[column] + '> ' + dataframe.loc[indices, ['text']].astype(str)\n",
        "    return new_dataframe"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "ZGgZBs4VwYRf"
      },
      "source": [
        "train_df = add_special_tokens(train_df)\n",
        "test_df = add_special_tokens(test_df)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "HFilo2apwYRf"
      },
      "source": [
        "train_df = train_df.sample(frac=1)\n",
        "test_df = test_df.sample(frac=1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "ckpk_F6bwYRg"
      },
      "source": [
        "x_train = train_df['text'].tolist()\n",
        "y_train = train_df['unsafe'].tolist()\n",
        "x_test = test_df['text'].tolist()\n",
        "y_test = test_df['unsafe'].tolist()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "Kh1AUG-bwYRh"
      },
      "source": [
        "class UnsafeData(Dataset):\n",
        "\n",
        "    def __init__(self, texts, targets, tokenizer, max_len):\n",
        "        \n",
        "        super().__init__()\n",
        "        \n",
        "        self.texts = texts\n",
        "        self.targets = targets        \n",
        "        self.max_len = max_len\n",
        "        self.tokenizer = tokenizer\n",
        "\n",
        "    def __len__(self):\n",
        "        \n",
        "        return len(self.texts)\n",
        "\n",
        "    \n",
        "    def __getitem__(self, index):\n",
        "        x = self.texts[index]\n",
        "\n",
        "        enc_dict = self.tokenizer(x, truncation=True, max_length=self.max_len, padding='max_length')\n",
        "      \n",
        "        item = {key: torch.tensor(val).long() for key, val in enc_dict.items()}\n",
        "        item['labels'] = torch.tensor(self.targets[index]).long()\n",
        "\n",
        "        return item "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZCKuigpQwYRi"
      },
      "source": [
        "### Обучение"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "xSsL1BnjwYRi"
      },
      "source": [
        "def compute_metrics(pred):\n",
        "    labels = pred.label_ids\n",
        "    preds = pred.predictions.argmax(-1)\n",
        "    precision, recall, f1, _ = precision_recall_fscore_support(labels, preds, average='weighted')\n",
        "    acc = accuracy_score(labels, preds)\n",
        "    return {\n",
        "        'accuracy': acc,\n",
        "        'f1': f1,\n",
        "        'precision': precision,\n",
        "        'recall': recall\n",
        "    }"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "tjl4ELuIwYRj"
      },
      "source": [
        "model_name = 'DeepPavlov/rubert-base-cased-conversational'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "OHc35n4owYRj"
      },
      "source": [
        "tokenizer = BertTokenizer.from_pretrained(model_name)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "L8eiVMBBwYRh"
      },
      "source": [
        "train_dataset = UnsafeData(x_train, y_train, tokenizer, max_len = 60)\n",
        "test_dataset = UnsafeData(x_test, y_test, tokenizer, max_len = 60)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VWf8M4B9wYRi",
        "outputId": "ab72e680-0a6c-42de-9931-467ee6cc717f"
      },
      "source": [
        "len(train_dataset), len(test_dataset)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(15000, 2000)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "c8Jo4T66wYRi",
        "outputId": "5209ff79-a3fe-4fd9-8896-f82105e4e879"
      },
      "source": [
        "train_dataset[10000]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
              "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),\n",
              " 'input_ids': tensor([  101,   160, 35408,   164, 11830,  6554,   132,   997,  1070, 63603,\n",
              "           128,   322, 17144,  3869,   885,   883,  8801, 95818,  2933,   371,\n",
              "          1199,   879,  1088,  2949, 10896, 35140,   344, 35401,   128,  3847,\n",
              "          1928,   838,  2785, 10209, 32648,  3941,  1757,   102,     0,     0,\n",
              "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0]),\n",
              " 'labels': tensor(1),\n",
              " 'token_type_ids': tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "Gy8nTG0LwYRj"
      },
      "source": [
        "def model_init():\n",
        "    return BertForSequenceClassification.from_pretrained(model_name, return_dict=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "r55TgRD4wYRk",
        "outputId": "5ed76ca1-8a08-4499-f1c5-1a0995335ed0"
      },
      "source": [
        "training_args = TrainingArguments(\n",
        "    output_dir='/bert1',\n",
        "    logging_steps = 600,\n",
        "    evaluation_strategy = 'steps',\n",
        "    logging_dir='/bert1/logs',\n",
        "    do_eval=True,\n",
        "    save_steps = 10000,\n",
        "    save_total_limit = 2,\n",
        "    disable_tqdm=True\n",
        ")\n",
        "\n",
        "trainer = Trainer(\n",
        "    args=training_args,\n",
        "    tokenizer=tokenizer,\n",
        "    train_dataset=train_dataset,\n",
        "    eval_dataset=test_dataset,\n",
        "    model_init=model_init,\n",
        "    compute_metrics=compute_metrics,\n",
        ")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at DeepPavlov/rubert-base-cased-conversational and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "o6DUgdQewYRk"
      },
      "source": [
        "### По какой метрике оптимизировать\n",
        "\n",
        "def my_func(metrics):\n",
        "    return metrics['eval_f1']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B0E9DlfXkg0M"
      },
      "source": [
        "from ray import tune"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y2w9YlTQiTYg"
      },
      "source": [
        "##scheduler  другие: https://docs.ray.io/en/latest/tune/api_docs/schedulers.html\n",
        "\n",
        "from ray.tune.schedulers import PopulationBasedTraining\n",
        "\n",
        "scheduler = PopulationBasedTraining(\n",
        "        time_attr=\"training_iteration\",\n",
        "        metric=\"eval_f1\",\n",
        "        mode=\"max\",\n",
        "        perturbation_interval=2,\n",
        "        hyperparam_mutations={\n",
        "            \"weight_decay\": tune.uniform(0.0, 0.1),\n",
        "            \"learning_rate\": tune.loguniform(1e-6, 1e-4),\n",
        "            \"per_device_train_batch_size\": tune.choice([16, 32, 64]),\n",
        "            \"num_train_epochs\": tune.choice([1,2])\n",
        "\n",
        "        })"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TSnA-GOL7Wlf"
      },
      "source": [
        "### алгоритм  другие: https://docs.ray.io/en/master/tune/api_docs/suggestion.html\n",
        "\n",
        "from ray.tune.suggest.hyperopt import HyperOptSearch\n",
        "\n",
        "alg = HyperOptSearch(metric = 'eval_f1', mode = 'max')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SzdFUMPH1gF8"
      },
      "source": [
        "### Stopper  другие:\n",
        "\n",
        "from ray.tune.stopper import TrialPlateauStopper\n",
        "\n",
        "stopper = TrialPlateauStopper(metric = 'eval_f1', mode = 'max', grace_period = 3)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Dq5kdBpm5JJ0"
      },
      "source": [
        "### Функция для пространства парметров\n",
        "\n",
        "def my_space(arg = None):\n",
        "    return {\n",
        "        \"learning_rate\": tune.loguniform(1e-6, 1e-3),\n",
        "        \"num_train_epochs\": tune.choice([1,2]),\n",
        "        #\"seed\": tune.uniform(1, 40),\n",
        "        \"per_device_train_batch_size\": tune.choice([16, 32, 64]),\n",
        "        \"weight_decay\": tune.uniform(0.0, 0.1),\n",
        "    }"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 870
        },
        "id": "B1MjHDnUkMXL",
        "outputId": "3070d7d8-4319-4e0c-f078-4873dc09f15b"
      },
      "source": [
        "best_run = trainer.hyperparameter_search(\n",
        "    hp_space = my_space,\n",
        "    n_trials=2,\n",
        "    compute_objective = my_func,\n",
        "    direction=\"maximize\",\n",
        "    backend = 'ray',\n",
        "    search_alg=alg,\n",
        "    scheduler=scheduler,\n",
        "    stop = stopper,\n",
        "    keep_checkpoints_num=1\n",
        "    \n",
        ")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-03-18 10:46:31,615\tWARNING util.py:152 -- The `start_trial` operation took 1.103 s, which may be a performance bottleneck.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "== Status ==\n",
            "Memory usage on this node: 6.3/12.7 GiB\n",
            "PopulationBasedTraining: 0 checkpoints, 0 perturbs\n",
            "Resources requested: 1/2 CPUs, 0/0 GPUs, 0.0/7.52 GiB heap, 0.0/2.59 GiB objects\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_10-46-27\n",
            "Number of trials: 1/2 (1 RUNNING)\n",
            "+-----------------+----------+-------+-----------------+--------------------+-------------------------------+----------------+\n",
            "| Trial name      | status   | loc   |   learning_rate |   num_train_epochs |   per_device_train_batch_size |   weight_decay |\n",
            "|-----------------+----------+-------+-----------------+--------------------+-------------------------------+----------------|\n",
            "| _inner_32774bb4 | RUNNING  |       |     8.94172e-05 |                  2 |                            16 |      0.0749374 |\n",
            "+-----------------+----------+-------+-----------------+--------------------+-------------------------------+----------------+\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1542)\u001b[0m Some weights of BertForSequenceClassification were not initialized from the model checkpoint at DeepPavlov/rubert-base-cased-conversational and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
            "\u001b[2m\u001b[36m(pid=1542)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "\u001b[2m\u001b[36m(pid=1541)\u001b[0m Some weights of BertForSequenceClassification were not initialized from the model checkpoint at DeepPavlov/rubert-base-cased-conversational and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "\u001b[2m\u001b[36m(pid=1541)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "\u001b[2m\u001b[36m(pid=1542)\u001b[0m Using deprecated `--per_gpu_train_batch_size` argument which will be removed in a future version. Using `--per_device_train_batch_size` is preferred.\n",
            "\u001b[2m\u001b[36m(pid=1542)\u001b[0m Using deprecated `--per_gpu_train_batch_size` argument which will be removed in a future version. Using `--per_device_train_batch_size` is preferred.\n",
            "\u001b[2m\u001b[36m(pid=1541)\u001b[0m Using deprecated `--per_gpu_train_batch_size` argument which will be removed in a future version. Using `--per_device_train_batch_size` is preferred.\n",
            "\u001b[2m\u001b[36m(pid=1541)\u001b[0m Using deprecated `--per_gpu_train_batch_size` argument which will be removed in a future version. Using `--per_device_train_batch_size` is preferred.\n",
            "2021-03-18 10:46:43,547\tWARNING worker.py:1107 -- The actor or task with ID ffffffffffffffff4ee449587774c1f0770a083801000000 cannot be scheduled right now. It requires {CPU: 1.000000} for placement, but this node only has remaining {0.000000/2.000000 CPU, 7.519531 GiB/7.519531 GiB memory, 2.587891 GiB/2.587891 GiB object_store_memory, 1.000000/1.000000 node:172.28.0.2}\n",
            ". In total there are 0 pending tasks and 2 pending actors on this node. This is likely due to all cluster resources being claimed by actors. To resolve the issue, consider creating fewer actors or increase the resources available to this Ray cluster. You can ignore this message if this Ray cluster is expected to auto-scale.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-33-1efeff5c8add>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      7\u001b[0m     \u001b[0msearch_alg\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mHyperOptSearch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmetric\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'eval_f1'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'max'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0mscheduler\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mscheduler\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m     \u001b[0mkeep_checkpoints_num\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m )\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/transformers/trainer.py\u001b[0m in \u001b[0;36mhyperparameter_search\u001b[0;34m(self, hp_space, compute_objective, n_trials, direction, backend, hp_name, **kwargs)\u001b[0m\n\u001b[1;32m   1380\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1381\u001b[0m         \u001b[0mrun_hp_search\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrun_hp_search_optuna\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mbackend\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mHPSearchBackend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOPTUNA\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mrun_hp_search_ray\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1382\u001b[0;31m         \u001b[0mbest_run\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrun_hp_search\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_trials\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdirection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1383\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1384\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhp_search_backend\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/transformers/integrations.py\u001b[0m in \u001b[0;36mrun_hp_search_ray\u001b[0;34m(trainer, n_trials, direction, **kwargs)\u001b[0m\n\u001b[1;32m    235\u001b[0m         \u001b[0mconfig\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhp_space\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    236\u001b[0m         \u001b[0mnum_samples\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mn_trials\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 237\u001b[0;31m         \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    238\u001b[0m     )\n\u001b[1;32m    239\u001b[0m     \u001b[0mbest_trial\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0manalysis\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_best_trial\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmetric\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"objective\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdirection\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/ray/tune/tune.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(run_or_experiment, name, metric, mode, stop, time_budget_s, config, resources_per_trial, num_samples, local_dir, search_alg, scheduler, keep_checkpoints_num, checkpoint_score_attr, checkpoint_freq, checkpoint_at_end, verbose, progress_reporter, log_to_file, trial_name_creator, trial_dirname_creator, sync_config, export_formats, max_failures, fail_fast, restore, server_port, resume, queue_trials, reuse_actors, trial_executor, raise_on_failed_trial, callbacks, loggers, ray_auto_init, run_errored_only, global_checkpoint_period, with_server, upload_dir, sync_to_cloud, sync_to_driver, sync_on_checkpoint)\u001b[0m\n\u001b[1;32m    419\u001b[0m     \u001b[0mtune_start\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    420\u001b[0m     \u001b[0;32mwhile\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mrunner\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_finished\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 421\u001b[0;31m         \u001b[0mrunner\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    422\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mhas_verbosity\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mVerbosity\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mV1_EXPERIMENT\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    423\u001b[0m             \u001b[0m_report_progress\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrunner\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprogress_reporter\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/ray/tune/trial_runner.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    400\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrial_executor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0min_staging_grace_period\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    401\u001b[0m                     \u001b[0mtimeout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0.1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 402\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_process_events\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# blocking\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    403\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    404\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrial_executor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_no_available_trials\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/ray/tune/trial_runner.py\u001b[0m in \u001b[0;36m_process_events\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    521\u001b[0m             \u001b[0;31m#  fetch_result functionality so that we don't timeout on fetch.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    522\u001b[0m             trial = self.trial_executor.get_next_available_trial(\n\u001b[0;32m--> 523\u001b[0;31m                 timeout=timeout)  # blocking\n\u001b[0m\u001b[1;32m    524\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mtrial\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    525\u001b[0m                 \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/ray/tune/ray_trial_executor.py\u001b[0m in \u001b[0;36mget_next_available_trial\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    579\u001b[0m         \u001b[0;31m# See https://github.com/ray-project/ray/issues/4211 for details.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    580\u001b[0m         \u001b[0mstart\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 581\u001b[0;31m         \u001b[0mready\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mray\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mshuffled_results\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    582\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mready\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    583\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/ray/_private/client_mode_hook.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     45\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mclient_mode_enabled\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0m_client_hook_enabled\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     46\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mray\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 47\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     48\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     49\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/ray/worker.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(object_refs, num_returns, timeout, fetch_local)\u001b[0m\n\u001b[1;32m   1596\u001b[0m             \u001b[0mtimeout_milliseconds\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1597\u001b[0m             \u001b[0mworker\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcurrent_task_id\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1598\u001b[0;31m             \u001b[0mfetch_local\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1599\u001b[0m         )\n\u001b[1;32m   1600\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mready_ids\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mremaining_ids\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32mpython/ray/_raylet.pyx\u001b[0m in \u001b[0;36mray._raylet.CoreWorker.wait\u001b[0;34m()\u001b[0m\n",
            "\u001b[0;32mpython/ray/_raylet.pyx\u001b[0m in \u001b[0;36mray._raylet.check_status\u001b[0;34m()\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    }
  ]
}