FLAN-T5-zeroshot.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "870e8279",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "\n",
    "import torch\n",
    "import tqdm\n",
    "\n",
    "from transformers import T5Tokenizer, T5ForConditionalGeneration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "677130fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "split = \"train.json\"\n",
    "data = json.load(open(f\"Complete_dataset/{split}\"))\n",
    "files = os.listdir(\"Complete_dataset/CT json/\")\n",
    "files.remove(\".DS_Store\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d5df4e00",
   "metadata": {},
   "outputs": [],
   "source": [
    "files_data = {file[:-5]:json.load(open(f\"Complete_dataset/CT json/{file}\")) for file in files}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "07636837",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_expanded = []\n",
    "for _id, value in data.items():\n",
    "    temp = {}\n",
    "    temp[\"id\"] = _id\n",
    "    p_nctid = value[\"Primary_id\"]\n",
    "    s_nctid = value.get(\"Secondary_id\")\n",
    "    section_id = value[\"Section_id\"]\n",
    "    statement = value[\"Statement\"]\n",
    "    primary_evidence = files_data[p_nctid][section_id]\n",
    "    temp[\"statement\"] = statement\n",
    "    temp[\"primary_evidence\"] = primary_evidence\n",
    "    temp[\"label\"] = value[\"Label\"]\n",
    "    \n",
    "    if s_nctid is not None:\n",
    "        secondary_evidence = files_data[s_nctid][section_id]\n",
    "        temp[\"secondary_evidence\"] = secondary_evidence\n",
    "    \n",
    "    data_expanded.append(temp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e0a6c139",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = T5Tokenizer.from_pretrained(\"google/flan-t5-xl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "dce56a86",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9724d8e0af044ee99736adebd7be2e91",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "model = T5ForConditionalGeneration.from_pretrained(\"google/flan-t5-xl\", cache_dir=\"/mnt/data/huggingface_cache\", device_map=\"auto\",)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ab1349c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_input_text(premise, hypothesis):\n",
    "    options_prefix = \"OPTIONS:\\n- \"\n",
    "    separator = \"\\n- \"\n",
    "    options_ = options_prefix + f\"{separator}\".join([\"Entailment\",\"Contradiction\"])\n",
    "    return f\"{premise} \\n Question: Does this imply that {hypothesis}? {options_}\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "a307dd6d",
   "metadata": {},
   "outputs": [],
   "source": [
    "samples = []\n",
    "for sample in data_expanded:\n",
    "    primary_evidence = \"\".join(sample['primary_evidence'])\n",
    "    sentence = f\"Primary trial evidence are {primary_evidence}\"\n",
    "    secondary_evidence = sample.get(\"secondary_evidence\")\n",
    "    if secondary_evidence:\n",
    "        secondary_evidence = \"\".join(sample['secondary_evidence'])\n",
    "        sentence = f\"{sentence} Secondary trial evidence are {secondary_evidence}\"\n",
    "    input_text = get_input_text(sentence, sample['statement'])\n",
    "    temp = {\"text\":input_text, \"label\":sample['label']}\n",
    "    samples.append(temp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f5d62610",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/1700 [00:00<?, ?it/s]/home/kamal_raj/transformers/src/transformers/generation/utils.py:1234: UserWarning: Neither `max_length` nor `max_new_tokens` has been set, `max_length` will default to 20 (`generation_config.max_length`). Controlling `max_length` via the config is deprecated and `max_length` will be removed from the config in v5 of Transformers -- we recommend using `max_new_tokens` to control the maximum length of the generation.\n",
      "  warnings.warn(\n",
      "  0%|          | 1/1700 [00:01<50:26,  1.78s/it]Token indices sequence length is longer than the specified maximum sequence length for this model (1258 > 512). Running this sequence through the model will result in indexing errors\n",
      "100%|██████████| 1700/1700 [11:05<00:00,  2.55it/s]\n"
     ]
    }
   ],
   "source": [
    "labels = []\n",
    "pred = []\n",
    "with torch.inference_mode():\n",
    "    for sample in tqdm.tqdm(samples):\n",
    "        labels.append(sample[\"label\"])\n",
    "        input_ids = tokenizer(sample[\"text\"], return_tensors=\"pt\",).input_ids.to(\"cuda\")\n",
    "        outputs = model.generate(input_ids)\n",
    "        pred.append(tokenizer.decode(outputs[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "708cd824",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> No</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> No</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> No</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> No</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> No</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> No</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> No</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> No</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " '<pad> Contradiction</s>',\n",
       " '<pad> Entailment</s>',\n",
       " ...]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "32f3b8fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = [p[5:][:-4].strip() for p in pred]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "5d349880",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Contradiction', 'Entailment', 'No', 'Yes'}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "6fbbb1d3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Contradiction',\n",
       " 'Contradiction',\n",
       " 'Entailment',\n",
       " 'Contradiction',\n",
       " 'Contradiction',\n",
       " 'Contradiction',\n",
       " 'Contradiction',\n",
       " 'Contradiction',\n",
       " 'Entailment',\n",
       " 'Contradiction']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "6e025405",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import f1_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "184b5e30",
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Target is multiclass but average='binary'. Please choose another average setting, one of [None, 'micro', 'macro', 'weighted'].",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[15], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m f1_score(pred, labels,pos_label\u001b[39m=\u001b[39;49m\u001b[39m\"\u001b[39;49m\u001b[39mEntailment\u001b[39;49m\u001b[39m\"\u001b[39;49m)\n",
      "File \u001b[0;32m~/.virtualenvs/hf_env/lib/python3.9/site-packages/sklearn/metrics/_classification.py:1146\u001b[0m, in \u001b[0;36mf1_score\u001b[0;34m(y_true, y_pred, labels, pos_label, average, sample_weight, zero_division)\u001b[0m\n\u001b[1;32m   1011\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mf1_score\u001b[39m(\n\u001b[1;32m   1012\u001b[0m     y_true,\n\u001b[1;32m   1013\u001b[0m     y_pred,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1019\u001b[0m     zero_division\u001b[39m=\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mwarn\u001b[39m\u001b[39m\"\u001b[39m,\n\u001b[1;32m   1020\u001b[0m ):\n\u001b[1;32m   1021\u001b[0m \u001b[39m    \u001b[39m\u001b[39m\"\"\"Compute the F1 score, also known as balanced F-score or F-measure.\u001b[39;00m\n\u001b[1;32m   1022\u001b[0m \n\u001b[1;32m   1023\u001b[0m \u001b[39m    The F1 score can be interpreted as a harmonic mean of the precision and\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1144\u001b[0m \u001b[39m    array([0.66666667, 1.        , 0.66666667])\u001b[39;00m\n\u001b[1;32m   1145\u001b[0m \u001b[39m    \"\"\"\u001b[39;00m\n\u001b[0;32m-> 1146\u001b[0m     \u001b[39mreturn\u001b[39;00m fbeta_score(\n\u001b[1;32m   1147\u001b[0m         y_true,\n\u001b[1;32m   1148\u001b[0m         y_pred,\n\u001b[1;32m   1149\u001b[0m         beta\u001b[39m=\u001b[39;49m\u001b[39m1\u001b[39;49m,\n\u001b[1;32m   1150\u001b[0m         labels\u001b[39m=\u001b[39;49mlabels,\n\u001b[1;32m   1151\u001b[0m         pos_label\u001b[39m=\u001b[39;49mpos_label,\n\u001b[1;32m   1152\u001b[0m         average\u001b[39m=\u001b[39;49maverage,\n\u001b[1;32m   1153\u001b[0m         sample_weight\u001b[39m=\u001b[39;49msample_weight,\n\u001b[1;32m   1154\u001b[0m         zero_division\u001b[39m=\u001b[39;49mzero_division,\n\u001b[1;32m   1155\u001b[0m     )\n",
      "File \u001b[0;32m~/.virtualenvs/hf_env/lib/python3.9/site-packages/sklearn/metrics/_classification.py:1287\u001b[0m, in \u001b[0;36mfbeta_score\u001b[0;34m(y_true, y_pred, beta, labels, pos_label, average, sample_weight, zero_division)\u001b[0m\n\u001b[1;32m   1158\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mfbeta_score\u001b[39m(\n\u001b[1;32m   1159\u001b[0m     y_true,\n\u001b[1;32m   1160\u001b[0m     y_pred,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1167\u001b[0m     zero_division\u001b[39m=\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mwarn\u001b[39m\u001b[39m\"\u001b[39m,\n\u001b[1;32m   1168\u001b[0m ):\n\u001b[1;32m   1169\u001b[0m \u001b[39m    \u001b[39m\u001b[39m\"\"\"Compute the F-beta score.\u001b[39;00m\n\u001b[1;32m   1170\u001b[0m \n\u001b[1;32m   1171\u001b[0m \u001b[39m    The F-beta score is the weighted harmonic mean of precision and recall,\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1284\u001b[0m \u001b[39m    array([0.71..., 0.        , 0.        ])\u001b[39;00m\n\u001b[1;32m   1285\u001b[0m \u001b[39m    \"\"\"\u001b[39;00m\n\u001b[0;32m-> 1287\u001b[0m     _, _, f, _ \u001b[39m=\u001b[39m precision_recall_fscore_support(\n\u001b[1;32m   1288\u001b[0m         y_true,\n\u001b[1;32m   1289\u001b[0m         y_pred,\n\u001b[1;32m   1290\u001b[0m         beta\u001b[39m=\u001b[39;49mbeta,\n\u001b[1;32m   1291\u001b[0m         labels\u001b[39m=\u001b[39;49mlabels,\n\u001b[1;32m   1292\u001b[0m         pos_label\u001b[39m=\u001b[39;49mpos_label,\n\u001b[1;32m   1293\u001b[0m         average\u001b[39m=\u001b[39;49maverage,\n\u001b[1;32m   1294\u001b[0m         warn_for\u001b[39m=\u001b[39;49m(\u001b[39m\"\u001b[39;49m\u001b[39mf-score\u001b[39;49m\u001b[39m\"\u001b[39;49m,),\n\u001b[1;32m   1295\u001b[0m         sample_weight\u001b[39m=\u001b[39;49msample_weight,\n\u001b[1;32m   1296\u001b[0m         zero_division\u001b[39m=\u001b[39;49mzero_division,\n\u001b[1;32m   1297\u001b[0m     )\n\u001b[1;32m   1298\u001b[0m     \u001b[39mreturn\u001b[39;00m f\n",
      "File \u001b[0;32m~/.virtualenvs/hf_env/lib/python3.9/site-packages/sklearn/metrics/_classification.py:1573\u001b[0m, in \u001b[0;36mprecision_recall_fscore_support\u001b[0;34m(y_true, y_pred, beta, labels, pos_label, average, warn_for, sample_weight, zero_division)\u001b[0m\n\u001b[1;32m   1571\u001b[0m \u001b[39mif\u001b[39;00m beta \u001b[39m<\u001b[39m \u001b[39m0\u001b[39m:\n\u001b[1;32m   1572\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\u001b[39m\"\u001b[39m\u001b[39mbeta should be >=0 in the F-beta score\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[0;32m-> 1573\u001b[0m labels \u001b[39m=\u001b[39m _check_set_wise_labels(y_true, y_pred, average, labels, pos_label)\n\u001b[1;32m   1575\u001b[0m \u001b[39m# Calculate tp_sum, pred_sum, true_sum ###\u001b[39;00m\n\u001b[1;32m   1576\u001b[0m samplewise \u001b[39m=\u001b[39m average \u001b[39m==\u001b[39m \u001b[39m\"\u001b[39m\u001b[39msamples\u001b[39m\u001b[39m\"\u001b[39m\n",
      "File \u001b[0;32m~/.virtualenvs/hf_env/lib/python3.9/site-packages/sklearn/metrics/_classification.py:1391\u001b[0m, in \u001b[0;36m_check_set_wise_labels\u001b[0;34m(y_true, y_pred, average, labels, pos_label)\u001b[0m\n\u001b[1;32m   1389\u001b[0m         \u001b[39mif\u001b[39;00m y_type \u001b[39m==\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mmulticlass\u001b[39m\u001b[39m\"\u001b[39m:\n\u001b[1;32m   1390\u001b[0m             average_options\u001b[39m.\u001b[39mremove(\u001b[39m\"\u001b[39m\u001b[39msamples\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[0;32m-> 1391\u001b[0m         \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\n\u001b[1;32m   1392\u001b[0m             \u001b[39m\"\u001b[39m\u001b[39mTarget is \u001b[39m\u001b[39m%s\u001b[39;00m\u001b[39m but average=\u001b[39m\u001b[39m'\u001b[39m\u001b[39mbinary\u001b[39m\u001b[39m'\u001b[39m\u001b[39m. Please \u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m   1393\u001b[0m             \u001b[39m\"\u001b[39m\u001b[39mchoose another average setting, one of \u001b[39m\u001b[39m%r\u001b[39;00m\u001b[39m.\u001b[39m\u001b[39m\"\u001b[39m \u001b[39m%\u001b[39m (y_type, average_options)\n\u001b[1;32m   1394\u001b[0m         )\n\u001b[1;32m   1395\u001b[0m \u001b[39melif\u001b[39;00m pos_label \u001b[39mnot\u001b[39;00m \u001b[39min\u001b[39;00m (\u001b[39mNone\u001b[39;00m, \u001b[39m1\u001b[39m):\n\u001b[1;32m   1396\u001b[0m     warnings\u001b[39m.\u001b[39mwarn(\n\u001b[1;32m   1397\u001b[0m         \u001b[39m\"\u001b[39m\u001b[39mNote that pos_label (set to \u001b[39m\u001b[39m%r\u001b[39;00m\u001b[39m) is ignored when \u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m   1398\u001b[0m         \u001b[39m\"\u001b[39m\u001b[39maverage != \u001b[39m\u001b[39m'\u001b[39m\u001b[39mbinary\u001b[39m\u001b[39m'\u001b[39m\u001b[39m (got \u001b[39m\u001b[39m%r\u001b[39;00m\u001b[39m). You may use \u001b[39m\u001b[39m\"\u001b[39m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1401\u001b[0m         \u001b[39mUserWarning\u001b[39;00m,\n\u001b[1;32m   1402\u001b[0m     )\n",
      "\u001b[0;31mValueError\u001b[0m: Target is multiclass but average='binary'. Please choose another average setting, one of [None, 'micro', 'macro', 'weighted']."
     ]
    }
   ],
   "source": [
    "f1_score(pred, labels,pos_label=\"Entailment\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "93cb321e",
   "metadata": {},
   "outputs": [],
   "source": [
    "prediction_dict = {}\n",
    "for _id,pred_x in zip(data, pred):\n",
    "    prediction_dict[str(_id)] = {\"Prediction\":pred_x}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a15a49fa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "updating: results.json (deflated 73%)\n"
     ]
    }
   ],
   "source": [
    "json.dump(prediction_dict, open(\"results.json\",'w'),indent=4)\n",
    "!zip results_xl_zero_dev.zip results.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "9beac770",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import f1_score\n",
    "uuid_list = list(prediction_dict.keys())\n",
    "results_pred = []\n",
    "gold_labels = []\n",
    "for i in range(len(uuid_list)):\n",
    "    if prediction_dict[uuid_list[i]][\"Prediction\"] in [\"Entailment\", \"Yes\"]:\n",
    "        results_pred.append(1)\n",
    "    else:\n",
    "        results_pred.append(0)\n",
    "    if data[uuid_list[i]][\"Label\"] in [\"Entailment\", \"No\"]:\n",
    "        gold_labels.append(1)\n",
    "    else:\n",
    "        gold_labels.append(0)\n",
    "f1_score(gold_labels,results_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "1930d449",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6754896770778189"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e7c89969",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  },
  "vscode": {
   "interpreter": {
    "hash": "6cf230bfe48388054af6957fef277c9681be8115aa3a1a86509ea0819612252f"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}